[dpdk-dev] [RFC] drivers: introduce mlx5 crypto PMD

[dpdk-dev] [RFC] drivers: introduce mlx5 crypto PMD

[Matan Azrad]

From: Shiri Kuzin <shirik@nvidia.com>

Add a new PMD for Mellanox devices- crypto PMD.

The crypto PMD will be supported starting Mellanox ConnectX6 and
BlueField2.

The crypto PMD will add the support of encryption and decryption using
the AES-XTS symmetric algorithm.

The crypto PMD requires rdma-core and uses mlx5 DevX.

This patch adds the PCI probing, basic functions, build files and
log utility.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 MAINTAINERS                             |   4 +
 drivers/common/mlx5/mlx5_common.h       |   1 +
 drivers/common/mlx5/mlx5_common_pci.c   |  14 ++
 drivers/common/mlx5/mlx5_common_pci.h   |  21 +--
 drivers/crypto/meson.build              |   1 +
 drivers/crypto/mlx5/meson.build         |  26 +++
 drivers/crypto/mlx5/mlx5_crypto.c       | 288 ++++++++++++++++++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto_utils.h |  19 +++
 drivers/crypto/mlx5/version.map         |   3 +
 9 files changed, 367 insertions(+), 10 deletions(-)
 create mode 100644 drivers/crypto/mlx5/meson.build
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto.c
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_utils.h
 create mode 100644 drivers/crypto/mlx5/version.map

diff --git a/MAINTAINERS b/MAINTAINERS
index 1962284..a449180 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -1064,6 +1064,10 @@ F: drivers/crypto/octeontx2/
 F: doc/guides/cryptodevs/octeontx2.rst
 F: doc/guides/cryptodevs/features/octeontx2.ini
 
+Mellanox mlx5
+M: Matan Azrad <matan@nvidia.com>
+F: drivers/crypto/mlx5/
+
 Null Crypto
 M: Declan Doherty <declan.doherty@intel.com>
 F: drivers/crypto/null/
diff --git a/drivers/common/mlx5/mlx5_common.h b/drivers/common/mlx5/mlx5_common.h
index 99efa22..5b4d8d1 100644
--- a/drivers/common/mlx5/mlx5_common.h
+++ b/drivers/common/mlx5/mlx5_common.h
@@ -219,6 +219,7 @@ enum mlx5_class {
 	MLX5_CLASS_VDPA = RTE_BIT64(1),
 	MLX5_CLASS_REGEX = RTE_BIT64(2),
 	MLX5_CLASS_COMPRESS = RTE_BIT64(3),
+	MLX5_CLASS_CRYPTO = RTE_BIT64(4),
 };
 
 #define MLX5_DBR_SIZE RTE_CACHE_LINE_SIZE
diff --git a/drivers/common/mlx5/mlx5_common_pci.c b/drivers/common/mlx5/mlx5_common_pci.c
index 4ecc4d6..167d55b 100644
--- a/drivers/common/mlx5/mlx5_common_pci.c
+++ b/drivers/common/mlx5/mlx5_common_pci.c
@@ -30,6 +30,7 @@ static TAILQ_HEAD(mlx5_pci_devices_head, mlx5_pci_device) devices_list =
 	{ .name = "net", .driver_class = MLX5_CLASS_NET },
 	{ .name = "regex", .driver_class = MLX5_CLASS_REGEX },
 	{ .name = "compress", .driver_class = MLX5_CLASS_COMPRESS },
+	{ .name = "crypto", .driver_class = MLX5_CLASS_CRYPTO },
 };
 
 static const unsigned int mlx5_class_combinations[] = {
@@ -37,13 +38,26 @@ static TAILQ_HEAD(mlx5_pci_devices_head, mlx5_pci_device) devices_list =
 	MLX5_CLASS_VDPA,
 	MLX5_CLASS_REGEX,
 	MLX5_CLASS_COMPRESS,
+	MLX5_CLASS_CRYPTO,
 	MLX5_CLASS_NET | MLX5_CLASS_REGEX,
 	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX,
 	MLX5_CLASS_NET | MLX5_CLASS_COMPRESS,
 	MLX5_CLASS_VDPA | MLX5_CLASS_COMPRESS,
 	MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
+	MLX5_CLASS_NET | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_VDPA | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO,
 	MLX5_CLASS_NET | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
 	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
+	MLX5_CLASS_NET | MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_NET | MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_VDPA | MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_NET | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS |
+	MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS |
+	MLX5_CLASS_CRYPTO,
 	/* New class combination should be added here. */
 };
 
diff --git a/drivers/common/mlx5/mlx5_common_pci.h b/drivers/common/mlx5/mlx5_common_pci.h
index de89bb9..cb8d2f5 100644
--- a/drivers/common/mlx5/mlx5_common_pci.h
+++ b/drivers/common/mlx5/mlx5_common_pci.h
@@ -9,17 +9,18 @@
  * @file
  *
  * RTE Mellanox PCI Driver Interface
- * Mellanox ConnectX PCI device supports multiple class: net,vdpa,regex and
- * compress devices. This layer enables creating such multiple class of devices
- * on a single PCI device by allowing to bind multiple class specific device
- * driver to attach to mlx5_pci driver.
+ * Mellanox ConnectX PCI device supports multiple class: net,vdpa,regex,compress
+ * and crypto devices. This layer enables creating such multiple class of
+ * devices on a single PCI device by allowing to bind multiple class specific
+ * device driver to attach to mlx5_pci driver.
  *
- * -----------    ------------    -------------    ----------------
- * |   mlx5  |    |   mlx5   |    |   mlx5    |    |     mlx5     |
- * | net pmd |    | vdpa pmd |    | regex pmd |    | compress pmd |
- * -----------    ------------    -------------    ----------------
- *      \              \                    /              /
- *       \              \                  /              /
+ * --------    --------    ---------    ------------    ----------
+ * | mlx5 |    | mlx5 |    | mlx5  |    |   mlx5   |    |  mlx5  |
+ * | net  |    | vdpa |    | regex |    | compress |    | crypto |
+ * | pmd  |    | pmd  |    |  pmd  |    |   pmd    |    |  pmd   |
+ * --------    --------    ---------    ------------    ----------
+ *      \              \         |          /              /
+ *       \              \        |         /              /
  *        \              \_--------------_/              /
  *         \_______________|   mlx5     |_______________/
  *                         | pci common |
diff --git a/drivers/crypto/meson.build b/drivers/crypto/meson.build
index c927c5f..90b2149 100644
--- a/drivers/crypto/meson.build
+++ b/drivers/crypto/meson.build
@@ -14,6 +14,7 @@ drivers = ['aesni_gcm',
 	   'dpaa_sec',
 	   'dpaa2_sec',
 	   'kasumi',
+	   'mlx5',
 	   'mvsam',
 	   'nitrox',
 	   'null',
diff --git a/drivers/crypto/mlx5/meson.build b/drivers/crypto/mlx5/meson.build
new file mode 100644
index 0000000..5bf0912
--- /dev/null
+++ b/drivers/crypto/mlx5/meson.build
@@ -0,0 +1,26 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2021 Mellanox Technologies, Ltd
+
+if not is_linux
+	build = false
+	reason = 'only supported on Linux'
+	subdir_done()
+endif
+
+fmt_name = 'mlx5_crypto'
+deps += ['common_mlx5', 'eal', 'cryptodev']
+sources = files(
+	'mlx5_crypto.c',
+)
+cflags_options = [
+	'-std=c11',
+	'-Wno-strict-prototypes',
+	'-D_BSD_SOURCE',
+	'-D_DEFAULT_SOURCE',
+	'-D_XOPEN_SOURCE=600'
+]
+foreach option:cflags_options
+	if cc.has_argument(option)
+		cflags += option
+	endif
+endforeach
\ No newline at end of file
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
new file mode 100644
index 0000000..6e40176
--- /dev/null
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -0,0 +1,288 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#include <rte_malloc.h>
+#include <rte_log.h>
+#include <rte_errno.h>
+#include <rte_pci.h>
+#include <rte_crypto.h>
+#include <rte_cryptodev.h>
+#include <rte_cryptodev_pmd.h>
+
+#include <mlx5_glue.h>
+#include <mlx5_common.h>
+#include <mlx5_common_pci.h>
+#include <mlx5_devx_cmds.h>
+#include <mlx5_common_os.h>
+
+#include "mlx5_crypto_utils.h"
+
+#define MLX5_CRYPTO_DRIVER_NAME mlx5_crypto
+#define MLX5_CRYPTO_LOG_NAME    pmd.crypto.mlx5
+
+struct mlx5_crypto_priv {
+	TAILQ_ENTRY(mlx5_crypto_priv) next;
+	struct ibv_context *ctx; /* Device context. */
+	struct rte_pci_device *pci_dev;
+	struct rte_cryptodev *crypto_dev;
+	void *uar; /* User Access Region. */
+	uint32_t pdn; /* Protection Domain number. */
+	struct ibv_pd *pd;
+};
+
+TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
+				TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
+static pthread_mutex_t priv_list_lock = PTHREAD_MUTEX_INITIALIZER;
+
+int mlx5_crypto_logtype;
+
+uint8_t mlx5_crypto_driver_id;
+
+static const char mlx5_crypto_drv_name[] = RTE_STR(MLX5_CRYPTO_DRIVER_NAME);
+
+static const struct rte_driver mlx5_drv = {
+	.name = mlx5_crypto_drv_name,
+	.alias = mlx5_crypto_drv_name
+};
+
+static struct cryptodev_driver mlx5_cryptodev_driver;
+
+static struct rte_cryptodev_ops mlx5_crypto_ops = {
+	.dev_configure			= NULL,
+	.dev_start			= NULL,
+	.dev_stop			= NULL,
+	.dev_close			= NULL,
+	.dev_infos_get			= NULL,
+	.stats_get			= NULL,
+	.stats_reset			= NULL,
+	.queue_pair_setup		= NULL,
+	.queue_pair_release		= NULL,
+	.sym_session_get_size		= NULL,
+	.sym_session_configure		= NULL,
+	.sym_session_clear		= NULL,
+	.sym_get_raw_dp_ctx_size	= NULL,
+	.sym_configure_raw_dp_ctx	= NULL,
+};
+
+static void
+mlx5_crypto_hw_global_release(struct mlx5_crypto_priv *priv)
+{
+	if (priv->pd != NULL) {
+		claim_zero(mlx5_glue->dealloc_pd(priv->pd));
+		priv->pd = NULL;
+	}
+	if (priv->uar != NULL) {
+		mlx5_glue->devx_free_uar(priv->uar);
+		priv->uar = NULL;
+	}
+}
+
+static int
+mlx5_crypto_pd_create(struct mlx5_crypto_priv *priv)
+{
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+	struct mlx5dv_obj obj;
+	struct mlx5dv_pd pd_info;
+	int ret;
+
+	priv->pd = mlx5_glue->alloc_pd(priv->ctx);
+	if (priv->pd == NULL) {
+		DRV_LOG(ERR, "Failed to allocate PD.");
+		return errno ? -errno : -ENOMEM;
+	}
+	obj.pd.in = priv->pd;
+	obj.pd.out = &pd_info;
+	ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_PD);
+	if (ret != 0) {
+		DRV_LOG(ERR, "Fail to get PD object info.");
+		mlx5_glue->dealloc_pd(priv->pd);
+		priv->pd = NULL;
+		return -errno;
+	}
+	priv->pdn = pd_info.pdn;
+	return 0;
+#else
+	(void)priv;
+	DRV_LOG(ERR, "Cannot get pdn - no DV support.");
+	return -ENOTSUP;
+#endif /* HAVE_IBV_FLOW_DV_SUPPORT */
+}
+
+static int
+mlx5_crypto_hw_global_prepare(struct mlx5_crypto_priv *priv)
+{
+	if (mlx5_crypto_pd_create(priv) != 0)
+		return -1;
+	priv->uar = mlx5_devx_alloc_uar(priv->ctx, -1);
+	if (priv->uar == NULL || mlx5_os_get_devx_uar_reg_addr(priv->uar) ==
+	    NULL) {
+		rte_errno = errno;
+		claim_zero(mlx5_glue->dealloc_pd(priv->pd));
+		DRV_LOG(ERR, "Failed to allocate UAR.");
+		return -1;
+	}
+	return 0;
+}
+
+/**
+ * DPDK callback to register a PCI device.
+ *
+ * This function spawns crypto device out of a given PCI device.
+ *
+ * @param[in] pci_drv
+ *   PCI driver structure (mlx5_crypto_driver).
+ * @param[in] pci_dev
+ *   PCI device information.
+ *
+ * @return
+ *   0 on success, 1 to skip this driver, a negative errno value otherwise
+ *   and rte_errno is set.
+ */
+static int
+mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
+			struct rte_pci_device *pci_dev)
+{
+	struct ibv_device *ibv;
+	struct rte_cryptodev *crypto_dev;
+	struct ibv_context *ctx;
+	struct mlx5_crypto_priv *priv;
+	struct mlx5_hca_attr attr = { 0 };
+	struct rte_cryptodev_pmd_init_params init_params = {
+		.name = "",
+		.private_data_size = sizeof(struct mlx5_crypto_priv),
+		.socket_id = pci_dev->device.numa_node,
+		.max_nb_queue_pairs =
+				RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
+	};
+	RTE_SET_USED(pci_drv);
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+		DRV_LOG(ERR, "Non-primary process type is not supported.");
+		rte_errno = ENOTSUP;
+		return -rte_errno;
+	}
+	ibv = mlx5_os_get_ib_device_match(&pci_dev->addr);
+	if (ibv == NULL) {
+		DRV_LOG(ERR, "No matching IB device for PCI slot "
+			PCI_PRI_FMT ".", pci_dev->addr.domain,
+			pci_dev->addr.bus, pci_dev->addr.devid,
+			pci_dev->addr.function);
+		return -rte_errno;
+	}
+	DRV_LOG(INFO, "PCI information matches for device \"%s\".", ibv->name);
+	ctx = mlx5_glue->dv_open_device(ibv);
+	if (ctx == NULL) {
+		DRV_LOG(ERR, "Failed to open IB device \"%s\".", ibv->name);
+		rte_errno = ENODEV;
+		return -rte_errno;
+	}
+	if (mlx5_devx_cmd_query_hca_attr(ctx, &attr) != 0 ||
+	    attr.crypto == 0 || attr.aes_xts == 0) {
+		DRV_LOG(ERR, "Not enough capabilities to support crypto "
+			"operations, maybe old FW/OFED version?");
+		claim_zero(mlx5_glue->close_device(ctx));
+		rte_errno = ENOTSUP;
+		return -ENOTSUP;
+	}
+	crypto_dev = rte_cryptodev_pmd_create(ibv->name, &pci_dev->device,
+					&init_params);
+	if (crypto_dev == NULL) {
+		DRV_LOG(ERR, "Failed to create device \"%s\".", ibv->name);
+		claim_zero(mlx5_glue->close_device(ctx));
+		return -ENODEV;
+	}
+	DRV_LOG(INFO,
+		"Crypto device %s was created successfully.", ibv->name);
+	crypto_dev->dev_ops = &mlx5_crypto_ops;
+	crypto_dev->dequeue_burst = NULL;
+	crypto_dev->enqueue_burst = NULL;
+	crypto_dev->feature_flags = RTE_CRYPTODEV_FF_HW_ACCELERATED;
+	crypto_dev->driver_id = mlx5_crypto_driver_id;
+	priv = crypto_dev->data->dev_private;
+	priv->ctx = ctx;
+	priv->pci_dev = pci_dev;
+	priv->crypto_dev = crypto_dev;
+	if (mlx5_crypto_hw_global_prepare(priv) != 0) {
+		rte_cryptodev_pmd_destroy(priv->crypto_dev);
+		claim_zero(mlx5_glue->close_device(priv->ctx));
+		return -1;
+	}
+	pthread_mutex_lock(&priv_list_lock);
+	TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next);
+	pthread_mutex_unlock(&priv_list_lock);
+	return 0;
+}
+
+static int
+mlx5_crypto_pci_remove(struct rte_pci_device *pdev)
+{
+	struct mlx5_crypto_priv *priv = NULL;
+
+	pthread_mutex_lock(&priv_list_lock);
+	TAILQ_FOREACH(priv, &mlx5_crypto_priv_list, next)
+		if (rte_pci_addr_cmp(&priv->pci_dev->addr, &pdev->addr) != 0)
+			break;
+	if (priv)
+		TAILQ_REMOVE(&mlx5_crypto_priv_list, priv, next);
+	pthread_mutex_unlock(&priv_list_lock);
+	if (priv) {
+		mlx5_crypto_hw_global_release(priv);
+		rte_cryptodev_pmd_destroy(priv->crypto_dev);
+		claim_zero(mlx5_glue->close_device(priv->ctx));
+	}
+	return 0;
+}
+
+static const struct rte_pci_id mlx5_crypto_pci_id_map[] = {
+		{
+			RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
+					PCI_DEVICE_ID_MELLANOX_CONNECTX6)
+		},
+		{
+			RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
+					PCI_DEVICE_ID_MELLANOX_CONNECTX6DX)
+		},
+		{
+			RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
+					PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF)
+		},
+		{
+			RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
+					PCI_DEVICE_ID_MELLANOX_CONNECTX7)
+		},
+		{
+			RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
+					PCI_DEVICE_ID_MELLANOX_CONNECTX7BF)
+		},
+		{
+			.vendor_id = 0
+		}
+	};
+
+static struct mlx5_pci_driver mlx5_crypto_driver = {
+	.driver_class = MLX5_CLASS_CRYPTO,
+	.pci_driver = {
+		.driver = {
+			.name = RTE_STR(MLX5_CRYPTO_DRIVER_NAME),
+		},
+		.id_table = mlx5_crypto_pci_id_map,
+		.probe = mlx5_crypto_pci_probe,
+		.remove = mlx5_crypto_pci_remove,
+		.drv_flags = 0,
+	},
+};
+
+RTE_INIT(rte_mlx5_crypto_init)
+{
+	mlx5_common_init();
+	if (mlx5_glue != NULL)
+		mlx5_pci_driver_register(&mlx5_crypto_driver);
+}
+
+RTE_PMD_REGISTER_CRYPTO_DRIVER(mlx5_cryptodev_driver, mlx5_drv,
+			       mlx5_crypto_driver_id);
+
+RTE_LOG_REGISTER(mlx5_crypto_logtype, MLX5_CRYPTO_LOG_NAME, NOTICE)
+RTE_PMD_EXPORT_NAME(MLX5_CRYPTO_DRIVER_NAME, __COUNTER__);
+RTE_PMD_REGISTER_PCI_TABLE(MLX5_CRYPTO_DRIVER_NAME, mlx5_crypto_pci_id_map);
+RTE_PMD_REGISTER_KMOD_DEP(MLX5_CRYPTO_DRIVER_NAME, "* ib_uverbs & mlx5_core & mlx5_ib");
diff --git a/drivers/crypto/mlx5/mlx5_crypto_utils.h b/drivers/crypto/mlx5/mlx5_crypto_utils.h
new file mode 100644
index 0000000..cef4b07
--- /dev/null
+++ b/drivers/crypto/mlx5/mlx5_crypto_utils.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#ifndef RTE_PMD_MLX5_CRYPTO_UTILS_H_
+#define RTE_PMD_MLX5_CRYPTO_UTILS_H_
+
+#include <mlx5_common.h>
+
+extern int mlx5_crypto_logtype;
+
+#define MLX5_CRYPTO_LOG_PREFIX "mlx5_crypto"
+/* Generic printf()-like logging macro with automatic line feed. */
+#define DRV_LOG(level, ...) \
+	PMD_DRV_LOG_(level, mlx5_crypto_logtype, MLX5_CRYPTO_LOG_PREFIX, \
+		__VA_ARGS__ PMD_DRV_LOG_STRIP PMD_DRV_LOG_OPAREN, \
+		PMD_DRV_LOG_CPAREN)
+
+#endif /* RTE_PMD_MLX5_CRYPTO_UTILS_H_ */
diff --git a/drivers/crypto/mlx5/version.map b/drivers/crypto/mlx5/version.map
new file mode 100644
index 0000000..4a76d1d
--- /dev/null
+++ b/drivers/crypto/mlx5/version.map
@@ -0,0 +1,3 @@
+DPDK_21 {
+	local: *;
+};
-- 
1.8.3.1

[dpdk-dev] [PATCH 00/24] drivers: introduce mlx5 crypto PMD

[Shiri Kuzin]

Add a new PMD for Nvidia devices- crypto PMD.

The crypto PMD will be supported on Nvidia ConnectX6-Dx and
Bluefield 2.

The crypto PMD will add the support of encryption and decryption using
the AES-XTS symmetric algorithm.

The crypto PMD requires rdma-core and uses mlx5 DevX.

This series includes control path PMD implementation.
We are currently almost done working on the data path implementation
and doing our best effort to send the data path for rc1, but we might
be done for rc2.
If there is a suggestion or need for the series earlier, we will make
the effort to meet the requirement.

Dekel Peled (13):
  common/mlx5: remove redundant spaces in header file
  common/mlx5: update GENEVE TLV OPT obj name
  common/mlx5: optimize read of general obj type caps
  common/mlx5: add HCA cap for AES-XTS crypto
  common/mlx5: support general object DEK create op
  common/mlx5: add crypto en field to MKEY context
  common/mlx5: support umr en field in MKEY context
  common/mlx5: support general obj IMPORT KEK create
  common/mlx5: support general obj CRYPTO LOGIN create
  common/mlx5: add crypto BSF struct and defines
  common/mlx5: support general obj CREDENTIAL create
  common/mlx5: add crypto register structs and defs
  common/mlx5: support register write access

Shiri Kuzin (11):
  common/mlx5: share hash list tool
  common/mlx5: share get ib device match function
  drivers: introduce mlx5 crypto PMD
  crypto/mlx5: add DEK object management
  crypto/mlx5: support session operations
  crypto/mlx5: add basic operations
  crypto/mlx5: support queue pairs operations
  crypto/mlx5: add dev stop and start operations
  crypto/mlx5: add memory region management
  crypto/mlx5: create login object using DevX
  crypto/mlx5: adjust to the multiple data unit API

 MAINTAINERS                                   |   4 +
 drivers/common/mlx5/linux/mlx5_common_os.c    |  30 +-
 drivers/common/mlx5/linux/mlx5_common_os.h    |   5 +
 drivers/common/mlx5/linux/mlx5_nl.c           |   2 +-
 drivers/common/mlx5/meson.build               |   1 +
 drivers/common/mlx5/mlx5_common.c             |   2 +-
 drivers/common/mlx5/mlx5_common.h             |   2 +
 drivers/common/mlx5/mlx5_common_devx.c        |   2 +-
 drivers/common/mlx5/mlx5_common_log.h         |  21 +
 drivers/common/mlx5/mlx5_common_mp.c          |   2 +-
 drivers/common/mlx5/mlx5_common_mr.c          |   2 +-
 drivers/common/mlx5/mlx5_common_pci.c         |  17 +-
 drivers/common/mlx5/mlx5_common_pci.h         |  21 +-
 drivers/common/mlx5/mlx5_common_utils.c       | 221 ++++++
 drivers/common/mlx5/mlx5_common_utils.h       | 202 ++++-
 drivers/common/mlx5/mlx5_devx_cmds.c          | 314 +++++++-
 drivers/common/mlx5/mlx5_devx_cmds.h          |  66 ++
 drivers/common/mlx5/mlx5_malloc.c             |   2 +-
 drivers/common/mlx5/mlx5_prm.h                | 545 +++++++++-----
 .../common/mlx5/rte_common_mlx5_exports.def   |   7 +
 drivers/common/mlx5/version.map               |  24 +-
 drivers/common/mlx5/windows/mlx5_common_os.c  |   2 +-
 drivers/common/mlx5/windows/mlx5_glue.c       |   2 +-
 drivers/compress/mlx5/mlx5_compress.c         |  30 +-
 drivers/crypto/meson.build                    |   1 +
 drivers/crypto/mlx5/meson.build               |  27 +
 drivers/crypto/mlx5/mlx5_crypto.c             | 687 ++++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto.h             |  67 ++
 drivers/crypto/mlx5/mlx5_crypto_dek.c         | 135 ++++
 drivers/crypto/mlx5/mlx5_crypto_utils.h       |  19 +
 drivers/crypto/mlx5/version.map               |   3 +
 drivers/net/mlx5/mlx5_utils.c                 | 209 ------
 drivers/net/mlx5/mlx5_utils.h                 | 194 +----
 drivers/regex/mlx5/mlx5_regex.c               |  30 +-
 drivers/vdpa/mlx5/mlx5_vdpa.c                 |  32 +-
 35 files changed, 2216 insertions(+), 714 deletions(-)
 create mode 100644 drivers/common/mlx5/mlx5_common_log.h
 create mode 100644 drivers/common/mlx5/mlx5_common_utils.c
 create mode 100644 drivers/crypto/mlx5/meson.build
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto.c
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto.h
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_dek.c
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_utils.h
 create mode 100644 drivers/crypto/mlx5/version.map

-- 
2.21.0

[dpdk-dev] [PATCH 01/24] common/mlx5: remove redundant spaces in header file

[Shiri Kuzin]

From: Dekel Peled <dekelp@nvidia.com>

File drivers/common/mlx5/mlx5_prm.h includes structs representing
data items as defined in PRM document.
Some of these structs were copied as-is from kernel file mlx5_ifc.h.
As result the structs are not all aligned with the same spacing.

This patch removes redundant spaces and new lines from several structs,
to align all structs in mlx5_prm.h to the same format.

Signed-off-by: Dekel Peled <dekelp@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_prm.h | 334 +++++++++++++++------------------
 1 file changed, 155 insertions(+), 179 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index 0ef0574f92..f0a4f70d7d 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -953,163 +953,139 @@ enum {
 
 /* Flow counters. */
 struct mlx5_ifc_alloc_flow_counter_out_bits {
-	u8         status[0x8];
-	u8         reserved_at_8[0x18];
-	u8         syndrome[0x20];
-	u8         flow_counter_id[0x20];
-	u8         reserved_at_60[0x20];
+	u8 status[0x8];
+	u8 reserved_at_8[0x18];
+	u8 syndrome[0x20];
+	u8 flow_counter_id[0x20];
+	u8 reserved_at_60[0x20];
 };
 
 struct mlx5_ifc_alloc_flow_counter_in_bits {
-	u8         opcode[0x10];
-	u8         reserved_at_10[0x10];
-	u8         reserved_at_20[0x10];
-	u8         op_mod[0x10];
-	u8         flow_counter_id[0x20];
-	u8         reserved_at_40[0x18];
-	u8         flow_counter_bulk[0x8];
+	u8 opcode[0x10];
+	u8 reserved_at_10[0x10];
+	u8 reserved_at_20[0x10];
+	u8 op_mod[0x10];
+	u8 flow_counter_id[0x20];
+	u8 reserved_at_40[0x18];
+	u8 flow_counter_bulk[0x8];
 };
 
 struct mlx5_ifc_dealloc_flow_counter_out_bits {
-	u8         status[0x8];
-	u8         reserved_at_8[0x18];
-	u8         syndrome[0x20];
-	u8         reserved_at_40[0x40];
+	u8 status[0x8];
+	u8 reserved_at_8[0x18];
+	u8 syndrome[0x20];
+	u8 reserved_at_40[0x40];
 };
 
 struct mlx5_ifc_dealloc_flow_counter_in_bits {
-	u8         opcode[0x10];
-	u8         reserved_at_10[0x10];
-	u8         reserved_at_20[0x10];
-	u8         op_mod[0x10];
-	u8         flow_counter_id[0x20];
-	u8         reserved_at_60[0x20];
+	u8 opcode[0x10];
+	u8 reserved_at_10[0x10];
+	u8 reserved_at_20[0x10];
+	u8 op_mod[0x10];
+	u8 flow_counter_id[0x20];
+	u8 reserved_at_60[0x20];
 };
 
 struct mlx5_ifc_traffic_counter_bits {
-	u8         packets[0x40];
-	u8         octets[0x40];
+	u8 packets[0x40];
+	u8 octets[0x40];
 };
 
 struct mlx5_ifc_query_flow_counter_out_bits {
-	u8         status[0x8];
-	u8         reserved_at_8[0x18];
-	u8         syndrome[0x20];
-	u8         reserved_at_40[0x40];
+	u8 status[0x8];
+	u8 reserved_at_8[0x18];
+	u8 syndrome[0x20];
+	u8 reserved_at_40[0x40];
 	struct mlx5_ifc_traffic_counter_bits flow_statistics[];
 };
 
 struct mlx5_ifc_query_flow_counter_in_bits {
-	u8         opcode[0x10];
-	u8         reserved_at_10[0x10];
-	u8         reserved_at_20[0x10];
-	u8         op_mod[0x10];
-	u8         reserved_at_40[0x20];
-	u8         mkey[0x20];
-	u8         address[0x40];
-	u8         clear[0x1];
-	u8         dump_to_memory[0x1];
-	u8         num_of_counters[0x1e];
-	u8         flow_counter_id[0x20];
+	u8 opcode[0x10];
+	u8 reserved_at_10[0x10];
+	u8 reserved_at_20[0x10];
+	u8 op_mod[0x10];
+	u8 reserved_at_40[0x20];
+	u8 mkey[0x20];
+	u8 address[0x40];
+	u8 clear[0x1];
+	u8 dump_to_memory[0x1];
+	u8 num_of_counters[0x1e];
+	u8 flow_counter_id[0x20];
 };
 
 #define MLX5_MAX_KLM_BYTE_COUNT 0x80000000u
 #define MLX5_MIN_KLM_FIXED_BUFFER_SIZE 0x1000u
 
-
 struct mlx5_ifc_klm_bits {
-	u8         byte_count[0x20];
-	u8         mkey[0x20];
-	u8         address[0x40];
+	u8 byte_count[0x20];
+	u8 mkey[0x20];
+	u8 address[0x40];
 };
 
 struct mlx5_ifc_mkc_bits {
-	u8         reserved_at_0[0x1];
-	u8         free[0x1];
-	u8         reserved_at_2[0x1];
-	u8         access_mode_4_2[0x3];
-	u8         reserved_at_6[0x7];
-	u8         relaxed_ordering_write[0x1];
-	u8         reserved_at_e[0x1];
-	u8         small_fence_on_rdma_read_response[0x1];
-	u8         umr_en[0x1];
-	u8         a[0x1];
-	u8         rw[0x1];
-	u8         rr[0x1];
-	u8         lw[0x1];
-	u8         lr[0x1];
-	u8         access_mode_1_0[0x2];
-	u8         reserved_at_18[0x8];
-
-	u8         qpn[0x18];
-	u8         mkey_7_0[0x8];
-
-	u8         reserved_at_40[0x20];
-
-	u8         length64[0x1];
-	u8         bsf_en[0x1];
-	u8         sync_umr[0x1];
-	u8         reserved_at_63[0x2];
-	u8         expected_sigerr_count[0x1];
-	u8         reserved_at_66[0x1];
-	u8         en_rinval[0x1];
-	u8         pd[0x18];
-
-	u8         start_addr[0x40];
-
-	u8         len[0x40];
-
-	u8         bsf_octword_size[0x20];
-
-	u8         reserved_at_120[0x80];
-
-	u8         translations_octword_size[0x20];
-
-	u8         reserved_at_1c0[0x19];
-	u8		   relaxed_ordering_read[0x1];
-	u8		   reserved_at_1da[0x1];
-	u8         log_page_size[0x5];
-
-	u8         reserved_at_1e0[0x20];
+	u8 reserved_at_0[0x1];
+	u8 free[0x1];
+	u8 reserved_at_2[0x1];
+	u8 access_mode_4_2[0x3];
+	u8 reserved_at_6[0x7];
+	u8 relaxed_ordering_write[0x1];
+	u8 reserved_at_e[0x1];
+	u8 small_fence_on_rdma_read_response[0x1];
+	u8 umr_en[0x1];
+	u8 a[0x1];
+	u8 rw[0x1];
+	u8 rr[0x1];
+	u8 lw[0x1];
+	u8 lr[0x1];
+	u8 access_mode_1_0[0x2];
+	u8 reserved_at_18[0x8];
+	u8 qpn[0x18];
+	u8 mkey_7_0[0x8];
+	u8 reserved_at_40[0x20];
+	u8 length64[0x1];
+	u8 bsf_en[0x1];
+	u8 sync_umr[0x1];
+	u8 reserved_at_63[0x2];
+	u8 expected_sigerr_count[0x1];
+	u8 reserved_at_66[0x1];
+	u8 en_rinval[0x1];
+	u8 pd[0x18];
+	u8 start_addr[0x40];
+	u8 len[0x40];
+	u8 bsf_octword_size[0x20];
+	u8 reserved_at_120[0x80];
+	u8 translations_octword_size[0x20];
+	u8 reserved_at_1c0[0x19];
+	u8 relaxed_ordering_read[0x1];
+	u8 reserved_at_1da[0x1];
+	u8 log_page_size[0x5];
+	u8 reserved_at_1e0[0x20];
 };
 
 struct mlx5_ifc_create_mkey_out_bits {
-	u8         status[0x8];
-	u8         reserved_at_8[0x18];
-
-	u8         syndrome[0x20];
-
-	u8         reserved_at_40[0x8];
-	u8         mkey_index[0x18];
-
-	u8         reserved_at_60[0x20];
+	u8 status[0x8];
+	u8 reserved_at_8[0x18];
+	u8 syndrome[0x20];
+	u8 reserved_at_40[0x8];
+	u8 mkey_index[0x18];
+	u8 reserved_at_60[0x20];
 };
 
 struct mlx5_ifc_create_mkey_in_bits {
-	u8         opcode[0x10];
-	u8         reserved_at_10[0x10];
-
-	u8         reserved_at_20[0x10];
-	u8         op_mod[0x10];
-
-	u8         reserved_at_40[0x20];
-
-	u8         pg_access[0x1];
-	u8         reserved_at_61[0x1f];
-
+	u8 opcode[0x10];
+	u8 reserved_at_10[0x10];
+	u8 reserved_at_20[0x10];
+	u8 op_mod[0x10];
+	u8 reserved_at_40[0x20];
+	u8 pg_access[0x1];
+	u8 reserved_at_61[0x1f];
 	struct mlx5_ifc_mkc_bits memory_key_mkey_entry;
-
-	u8         reserved_at_280[0x80];
-
-	u8         translations_octword_actual_size[0x20];
-
-	u8         mkey_umem_id[0x20];
-
-	u8         mkey_umem_offset[0x40];
-
-	u8         reserved_at_380[0x500];
-
-	u8         klm_pas_mtt[][0x20];
+	u8 reserved_at_280[0x80];
+	u8 translations_octword_actual_size[0x20];
+	u8 mkey_umem_id[0x20];
+	u8 mkey_umem_offset[0x40];
+	u8 reserved_at_380[0x500];
+	u8 klm_pas_mtt[][0x20];
 };
 
 enum {
@@ -2262,27 +2238,27 @@ enum {
 };
 
 struct mlx5_ifc_flow_meter_parameters_bits {
-	u8         valid[0x1];			// 00h
-	u8         bucket_overflow[0x1];
-	u8         start_color[0x2];
-	u8         both_buckets_on_green[0x1];
-	u8         meter_mode[0x2];
-	u8         reserved_at_1[0x19];
-	u8         reserved_at_2[0x20]; //04h
-	u8         reserved_at_3[0x3];
-	u8         cbs_exponent[0x5];		// 08h
-	u8         cbs_mantissa[0x8];
-	u8         reserved_at_4[0x3];
-	u8         cir_exponent[0x5];
-	u8         cir_mantissa[0x8];
-	u8         reserved_at_5[0x20];		// 0Ch
-	u8         reserved_at_6[0x3];
-	u8         ebs_exponent[0x5];		// 10h
-	u8         ebs_mantissa[0x8];
-	u8         reserved_at_7[0x3];
-	u8         eir_exponent[0x5];
-	u8         eir_mantissa[0x8];
-	u8         reserved_at_8[0x60];		// 14h-1Ch
+	u8 valid[0x1];
+	u8 bucket_overflow[0x1];
+	u8 start_color[0x2];
+	u8 both_buckets_on_green[0x1];
+	u8 meter_mode[0x2];
+	u8 reserved_at_1[0x19];
+	u8 reserved_at_2[0x20];
+	u8 reserved_at_3[0x3];
+	u8 cbs_exponent[0x5];
+	u8 cbs_mantissa[0x8];
+	u8 reserved_at_4[0x3];
+	u8 cir_exponent[0x5];
+	u8 cir_mantissa[0x8];
+	u8 reserved_at_5[0x20];
+	u8 reserved_at_6[0x3];
+	u8 ebs_exponent[0x5];
+	u8 ebs_mantissa[0x8];
+	u8 reserved_at_7[0x3];
+	u8 eir_exponent[0x5];
+	u8 eir_mantissa[0x8];
+	u8 reserved_at_8[0x60];
 };
 
 enum {
@@ -2340,46 +2316,46 @@ struct mlx5_ifc_cqc_bits {
 };
 
 struct mlx5_ifc_health_buffer_bits {
-	u8         reserved_0[0x100];
-	u8         assert_existptr[0x20];
-	u8         assert_callra[0x20];
-	u8         reserved_1[0x40];
-	u8         fw_version[0x20];
-	u8         hw_id[0x20];
-	u8         reserved_2[0x20];
-	u8         irisc_index[0x8];
-	u8         synd[0x8];
-	u8         ext_synd[0x10];
+	u8 reserved_0[0x100];
+	u8 assert_existptr[0x20];
+	u8 assert_callra[0x20];
+	u8 reserved_1[0x40];
+	u8 fw_version[0x20];
+	u8 hw_id[0x20];
+	u8 reserved_2[0x20];
+	u8 irisc_index[0x8];
+	u8 synd[0x8];
+	u8 ext_synd[0x10];
 };
 
 struct mlx5_ifc_initial_seg_bits {
-	u8         fw_rev_minor[0x10];
-	u8         fw_rev_major[0x10];
-	u8         cmd_interface_rev[0x10];
-	u8         fw_rev_subminor[0x10];
-	u8         reserved_0[0x40];
-	u8         cmdq_phy_addr_63_32[0x20];
-	u8         cmdq_phy_addr_31_12[0x14];
-	u8         reserved_1[0x2];
-	u8         nic_interface[0x2];
-	u8         log_cmdq_size[0x4];
-	u8         log_cmdq_stride[0x4];
-	u8         command_doorbell_vector[0x20];
-	u8         reserved_2[0xf00];
-	u8         initializing[0x1];
-	u8         nic_interface_supported[0x7];
-	u8         reserved_4[0x18];
+	u8 fw_rev_minor[0x10];
+	u8 fw_rev_major[0x10];
+	u8 cmd_interface_rev[0x10];
+	u8 fw_rev_subminor[0x10];
+	u8 reserved_0[0x40];
+	u8 cmdq_phy_addr_63_32[0x20];
+	u8 cmdq_phy_addr_31_12[0x14];
+	u8 reserved_1[0x2];
+	u8 nic_interface[0x2];
+	u8 log_cmdq_size[0x4];
+	u8 log_cmdq_stride[0x4];
+	u8 command_doorbell_vector[0x20];
+	u8 reserved_2[0xf00];
+	u8 initializing[0x1];
+	u8 nic_interface_supported[0x7];
+	u8 reserved_4[0x18];
 	struct mlx5_ifc_health_buffer_bits health_buffer;
-	u8         no_dram_nic_offset[0x20];
-	u8         reserved_5[0x6de0];
-	u8         internal_timer_h[0x20];
-	u8         internal_timer_l[0x20];
-	u8         reserved_6[0x20];
-	u8         reserved_7[0x1f];
-	u8         clear_int[0x1];
-	u8         health_syndrome[0x8];
-	u8         health_counter[0x18];
-	u8         reserved_8[0x17fc0];
+	u8 no_dram_nic_offset[0x20];
+	u8 reserved_5[0x6de0];
+	u8 internal_timer_h[0x20];
+	u8 internal_timer_l[0x20];
+	u8 reserved_6[0x20];
+	u8 reserved_7[0x1f];
+	u8 clear_int[0x1];
+	u8 health_syndrome[0x8];
+	u8 health_counter[0x18];
+	u8 reserved_8[0x17fc0];
 };
 
 struct mlx5_ifc_create_cq_out_bits {
-- 
2.21.0

[dpdk-dev] [PATCH 02/24] common/mlx5: update GENEVE TLV OPT obj name

[Shiri Kuzin]

From: Dekel Peled <dekelp@nvidia.com>

Rename MLX5_OBJ_TYPE_GENEVE_TLV_OPT as
MLX5_GENERAL_OBJ_TYPE_GENEVE_TLV_OPT, to align with other general
objects names.

Signed-off-by: Dekel Peled <dekelp@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_devx_cmds.c | 2 +-
 drivers/common/mlx5/mlx5_prm.h       | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c
index c90e020643..dfd2cb6af4 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.c
+++ b/drivers/common/mlx5/mlx5_devx_cmds.c
@@ -2176,7 +2176,7 @@ mlx5_devx_cmd_create_geneve_tlv_option(void *ctx,
 	MLX5_SET(general_obj_in_cmd_hdr, hdr, opcode,
 			MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
 	MLX5_SET(general_obj_in_cmd_hdr, hdr, obj_type,
-			MLX5_OBJ_TYPE_GENEVE_TLV_OPT);
+		 MLX5_GENERAL_OBJ_TYPE_GENEVE_TLV_OPT);
 	MLX5_SET(geneve_tlv_option, opt, option_class,
 			rte_be_to_cpu_16(class));
 	MLX5_SET(geneve_tlv_option, opt, option_type, type);
diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index f0a4f70d7d..e3ec4201c3 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -1106,7 +1106,7 @@ enum {
 #define MLX5_GENERAL_OBJ_TYPES_CAP_FLOW_HIT_ASO \
 			(1ULL << MLX5_GENERAL_OBJ_TYPE_FLOW_HIT_ASO)
 #define MLX5_GENERAL_OBJ_TYPES_CAP_GENEVE_TLV_OPT \
-			(1ULL << MLX5_OBJ_TYPE_GENEVE_TLV_OPT)
+			(1ULL << MLX5_GENERAL_OBJ_TYPE_GENEVE_TLV_OPT)
 
 enum {
 	MLX5_HCA_CAP_OPMOD_GET_MAX   = 0,
@@ -2383,7 +2383,7 @@ struct mlx5_ifc_create_cq_in_bits {
 };
 
 enum {
-	MLX5_OBJ_TYPE_GENEVE_TLV_OPT = 0x000b,
+	MLX5_GENERAL_OBJ_TYPE_GENEVE_TLV_OPT = 0x000b,
 	MLX5_GENERAL_OBJ_TYPE_VIRTQ = 0x000d,
 	MLX5_GENERAL_OBJ_TYPE_VIRTIO_Q_COUNTERS = 0x001c,
 	MLX5_GENERAL_OBJ_TYPE_FLEX_PARSE_GRAPH = 0x0022,
-- 
2.21.0

[dpdk-dev] [PATCH 03/24] common/mlx5: optimize read of general obj type caps

[Shiri Kuzin]

From: Dekel Peled <dekelp@nvidia.com>

General object types support is indicated in bitmap general_obj_types,
which is part of HCA capabilities list.
Currently this bitmap is read multiple times, and each time a different
bit is extracted.

This patch optimizes the code, reading the bitmap once into a local
variable, and then extracting the required bits.

Signed-off-by: Dekel Peled <dekelp@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_devx_cmds.c | 30 +++++++++++++++-------------
 1 file changed, 16 insertions(+), 14 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c
index dfd2cb6af4..8e4a5d71b6 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.c
+++ b/drivers/common/mlx5/mlx5_devx_cmds.c
@@ -646,6 +646,7 @@ mlx5_devx_cmd_query_hca_attr(void *ctx,
 	uint32_t out[MLX5_ST_SZ_DW(query_hca_cap_out)] = {0};
 	void *hcattr;
 	int status, syndrome, rc, i;
+	uint64_t general_obj_types_supported = 0;
 
 	MLX5_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP);
 	MLX5_SET(query_hca_cap_in, in, op_mod,
@@ -695,15 +696,6 @@ mlx5_devx_cmd_query_hca_attr(void *ctx,
 	attr->max_geneve_tlv_option_data_len = MLX5_GET(cmd_hca_cap, hcattr,
 			max_geneve_tlv_option_data_len);
 	attr->qos.sup = MLX5_GET(cmd_hca_cap, hcattr, qos);
-	attr->vdpa.valid = !!(MLX5_GET64(cmd_hca_cap, hcattr,
-					 general_obj_types) &
-			      MLX5_GENERAL_OBJ_TYPES_CAP_VIRTQ_NET_Q);
-	attr->vdpa.queue_counters_valid = !!(MLX5_GET64(cmd_hca_cap, hcattr,
-							general_obj_types) &
-				  MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_Q_COUNTERS);
-	attr->parse_graph_flex_node = !!(MLX5_GET64(cmd_hca_cap, hcattr,
-					 general_obj_types) &
-			      MLX5_GENERAL_OBJ_TYPES_CAP_PARSE_GRAPH_FLEX_NODE);
 	attr->wqe_index_ignore = MLX5_GET(cmd_hca_cap, hcattr,
 					  wqe_index_ignore_cap);
 	attr->cross_channel = MLX5_GET(cmd_hca_cap, hcattr, cd);
@@ -721,12 +713,22 @@ mlx5_devx_cmd_query_hca_attr(void *ctx,
 	attr->regex = MLX5_GET(cmd_hca_cap, hcattr, regexp);
 	attr->regexp_num_of_engines = MLX5_GET(cmd_hca_cap, hcattr,
 					       regexp_num_of_engines);
-	attr->flow_hit_aso = !!(MLX5_GET64(cmd_hca_cap, hcattr,
-					   general_obj_types) &
+	/* Read the general_obj_types bitmap and extract the relevant bits. */
+	general_obj_types_supported = MLX5_GET64(cmd_hca_cap, hcattr,
+						 general_obj_types);
+	attr->vdpa.valid = !!(general_obj_types_supported &
+			      MLX5_GENERAL_OBJ_TYPES_CAP_VIRTQ_NET_Q);
+	attr->vdpa.queue_counters_valid =
+			!!(general_obj_types_supported &
+			   MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_Q_COUNTERS);
+	attr->parse_graph_flex_node =
+			!!(general_obj_types_supported &
+			   MLX5_GENERAL_OBJ_TYPES_CAP_PARSE_GRAPH_FLEX_NODE);
+	attr->flow_hit_aso = !!(general_obj_types_supported &
 				MLX5_GENERAL_OBJ_TYPES_CAP_FLOW_HIT_ASO);
-	attr->geneve_tlv_opt = !!(MLX5_GET64(cmd_hca_cap, hcattr,
-					   general_obj_types) &
-				MLX5_GENERAL_OBJ_TYPES_CAP_GENEVE_TLV_OPT);
+	attr->geneve_tlv_opt = !!(general_obj_types_supported &
+				  MLX5_GENERAL_OBJ_TYPES_CAP_GENEVE_TLV_OPT);
+	/* Add reading of other GENERAL_OBJ_TYPES_CAP bits above this line. */
 	attr->log_max_cq = MLX5_GET(cmd_hca_cap, hcattr, log_max_cq);
 	attr->log_max_qp = MLX5_GET(cmd_hca_cap, hcattr, log_max_qp);
 	attr->log_max_cq_sz = MLX5_GET(cmd_hca_cap, hcattr, log_max_cq_sz);
-- 
2.21.0

[dpdk-dev] [PATCH 04/24] common/mlx5: add HCA cap for AES-XTS crypto

[Shiri Kuzin]

From: Dekel Peled <dekelp@nvidia.com>

Update the PRM structure and HCA capabilities reading, to include
relevant capabilities for AES-XTS crypto.

Signed-off-by: Dekel Peled <dekelp@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_devx_cmds.c | 3 +++
 drivers/common/mlx5/mlx5_devx_cmds.h | 2 ++
 drivers/common/mlx5/mlx5_prm.h       | 5 ++++-
 3 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c
index 8e4a5d71b6..f96f706ccd 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.c
+++ b/drivers/common/mlx5/mlx5_devx_cmds.c
@@ -754,6 +754,9 @@ mlx5_devx_cmd_query_hca_attr(void *ctx,
 						mini_cqe_resp_flow_tag);
 	attr->mini_cqe_resp_l3_l4_tag = MLX5_GET(cmd_hca_cap, hcattr,
 						 mini_cqe_resp_l3_l4_tag);
+	attr->crypto = MLX5_GET(cmd_hca_cap, hcattr, crypto);
+	if (attr->crypto)
+		attr->aes_xts = MLX5_GET(cmd_hca_cap, hcattr, aes_xts);
 	if (attr->qos.sup) {
 		MLX5_SET(query_hca_cap_in, in, op_mod,
 			 MLX5_GET_HCA_CAP_OP_MOD_QOS_CAP |
diff --git a/drivers/common/mlx5/mlx5_devx_cmds.h b/drivers/common/mlx5/mlx5_devx_cmds.h
index 2826c0b2c6..0c7a9ed76e 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.h
+++ b/drivers/common/mlx5/mlx5_devx_cmds.h
@@ -127,6 +127,8 @@ struct mlx5_hca_attr {
 	uint32_t qp_ts_format:2;
 	uint32_t regex:1;
 	uint32_t reg_c_preserve:1;
+	uint32_t crypto:1; /* Crypto engine is supported. */
+	uint32_t aes_xts:1; /* AES-XTS crypto is supported. */
 	uint32_t regexp_num_of_engines;
 	uint32_t log_max_ft_sampler_num:8;
 	uint32_t geneve_tlv_opt;
diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index e3ec4201c3..c7f973953e 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -1425,7 +1425,10 @@ struct mlx5_ifc_cmd_hca_cap_bits {
 	u8 sq_ts_format[0x2];
 	u8 rq_ts_format[0x2];
 	u8 reserved_at_444[0x1C];
-	u8 reserved_at_460[0x10];
+	u8 reserved_at_460[0x8];
+	u8 aes_xts[0x1];
+	u8 crypto[0x1];
+	u8 reserved_at_46a[0x6];
 	u8 max_num_eqs[0x10];
 	u8 reserved_at_480[0x3];
 	u8 log_max_l2_table[0x5];
-- 
2.21.0

[dpdk-dev] [PATCH 05/24] common/mlx5: support general object DEK create op

[Shiri Kuzin]

From: Dekel Peled <dekelp@nvidia.com>

Data Encryption Keys (DEKs) are the keys used for data
encryption/decryption operations.

Add reading of DEK support capability.
Add function to create general object type DEK, using DevX API.

Signed-off-by: Dekel Peled <dekelp@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_devx_cmds.c | 53 ++++++++++++++++++++++++++++
 drivers/common/mlx5/mlx5_devx_cmds.h | 17 +++++++++
 drivers/common/mlx5/mlx5_prm.h       | 39 ++++++++++++++++++++
 drivers/common/mlx5/version.map      |  1 +
 4 files changed, 110 insertions(+)

diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c
index f96f706ccd..bad474aeae 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.c
+++ b/drivers/common/mlx5/mlx5_devx_cmds.c
@@ -728,6 +728,8 @@ mlx5_devx_cmd_query_hca_attr(void *ctx,
 				MLX5_GENERAL_OBJ_TYPES_CAP_FLOW_HIT_ASO);
 	attr->geneve_tlv_opt = !!(general_obj_types_supported &
 				  MLX5_GENERAL_OBJ_TYPES_CAP_GENEVE_TLV_OPT);
+	attr->dek = !!(general_obj_types_supported &
+		       MLX5_GENERAL_OBJ_TYPES_CAP_DEK);
 	/* Add reading of other GENERAL_OBJ_TYPES_CAP bits above this line. */
 	attr->log_max_cq = MLX5_GET(cmd_hca_cap, hcattr, log_max_cq);
 	attr->log_max_qp = MLX5_GET(cmd_hca_cap, hcattr, log_max_qp);
@@ -2299,3 +2301,54 @@ mlx5_devx_cmd_queue_counter_query(struct mlx5_devx_obj *dcs, int clear,
 	*out_of_buffers = MLX5_GET(query_q_counter_out, out, out_of_buffer);
 	return 0;
 }
+
+/**
+ * Create general object of type DEK using DevX API.
+ *
+ * @param[in] ctx
+ *   Context returned from mlx5 open_device() glue function.
+ * @param [in] attr
+ *   Pointer to DEK attributes structure.
+ *
+ * @return
+ *   The DevX object created, NULL otherwise and rte_errno is set.
+ */
+struct mlx5_devx_obj *
+mlx5_devx_cmd_create_dek_obj(void *ctx, struct mlx5_devx_dek_attr *attr)
+{
+	uint32_t in[MLX5_ST_SZ_DW(create_dek_in)] = {0};
+	uint32_t out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};
+	struct mlx5_devx_obj *dek_obj = NULL;
+	void *ptr = NULL, *key_addr = NULL;
+
+	dek_obj = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*dek_obj),
+			      0, SOCKET_ID_ANY);
+	if (dek_obj == NULL) {
+		DRV_LOG(ERR, "Failed to allocate DEK object data");
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+	ptr = MLX5_ADDR_OF(create_dek_in, in, hdr);
+	MLX5_SET(general_obj_in_cmd_hdr, ptr, opcode,
+		 MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
+	MLX5_SET(general_obj_in_cmd_hdr, ptr, obj_type,
+		 MLX5_GENERAL_OBJ_TYPE_DEK);
+	ptr = MLX5_ADDR_OF(create_dek_in, in, dek);
+	MLX5_SET(dek, ptr, key_size, attr->key_size);
+	MLX5_SET(dek, ptr, has_keytag, attr->has_keytag);
+	MLX5_SET(dek, ptr, key_purpose, attr->key_purpose);
+	MLX5_SET(dek, ptr, pd, attr->pd);
+	MLX5_SET64(dek, ptr, opaque, attr->opaque);
+	key_addr = MLX5_ADDR_OF(dek, ptr, key);
+	memcpy(key_addr, (void *)(attr->key), MLX5_CRYPTO_KEY_MAX_SIZE);
+	dek_obj->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in),
+						  out, sizeof(out));
+	if (dek_obj->obj == NULL) {
+		rte_errno = errno;
+		DRV_LOG(ERR, "Failed to create DEK obj using DevX.");
+		mlx5_free(dek_obj);
+		return NULL;
+	}
+	dek_obj->id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
+	return dek_obj;
+}
diff --git a/drivers/common/mlx5/mlx5_devx_cmds.h b/drivers/common/mlx5/mlx5_devx_cmds.h
index 0c7a9ed76e..e17bad6bce 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.h
+++ b/drivers/common/mlx5/mlx5_devx_cmds.h
@@ -129,6 +129,7 @@ struct mlx5_hca_attr {
 	uint32_t reg_c_preserve:1;
 	uint32_t crypto:1; /* Crypto engine is supported. */
 	uint32_t aes_xts:1; /* AES-XTS crypto is supported. */
+	uint32_t dek:1; /* General obj type DEK is supported. */
 	uint32_t regexp_num_of_engines;
 	uint32_t log_max_ft_sampler_num:8;
 	uint32_t geneve_tlv_opt;
@@ -423,6 +424,18 @@ struct mlx5_devx_graph_node_attr {
 	struct mlx5_devx_graph_arc_attr out[MLX5_GRAPH_NODE_ARC_NUM];
 };
 
+/* Encryption key size is up to 1024 bit, 128 bytes. */
+#define MLX5_CRYPTO_KEY_MAX_SIZE	128
+
+struct mlx5_devx_dek_attr {
+	uint32_t key_size:4;
+	uint32_t has_keytag:1;
+	uint32_t key_purpose:4;
+	uint32_t pd:24;
+	uint64_t opaque;
+	uint8_t key[MLX5_CRYPTO_KEY_MAX_SIZE];
+};
+
 /* mlx5_devx_cmds.c */
 
 __rte_internal
@@ -557,4 +570,8 @@ struct mlx5_devx_obj *mlx5_devx_cmd_queue_counter_alloc(void *ctx);
 __rte_internal
 int mlx5_devx_cmd_queue_counter_query(struct mlx5_devx_obj *dcs, int clear,
 				      uint32_t *out_of_buffers);
+__rte_internal
+struct mlx5_devx_obj *
+mlx5_devx_cmd_create_dek_obj(void *ctx, struct mlx5_devx_dek_attr *attr);
+
 #endif /* RTE_PMD_MLX5_DEVX_CMDS_H_ */
diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index c7f973953e..fbe191cc94 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -1107,6 +1107,8 @@ enum {
 			(1ULL << MLX5_GENERAL_OBJ_TYPE_FLOW_HIT_ASO)
 #define MLX5_GENERAL_OBJ_TYPES_CAP_GENEVE_TLV_OPT \
 			(1ULL << MLX5_GENERAL_OBJ_TYPE_GENEVE_TLV_OPT)
+#define MLX5_GENERAL_OBJ_TYPES_CAP_DEK \
+			(1ULL << MLX5_GENERAL_OBJ_TYPE_DEK)
 
 enum {
 	MLX5_HCA_CAP_OPMOD_GET_MAX   = 0,
@@ -2387,6 +2389,7 @@ struct mlx5_ifc_create_cq_in_bits {
 
 enum {
 	MLX5_GENERAL_OBJ_TYPE_GENEVE_TLV_OPT = 0x000b,
+	MLX5_GENERAL_OBJ_TYPE_DEK = 0x000c,
 	MLX5_GENERAL_OBJ_TYPE_VIRTQ = 0x000d,
 	MLX5_GENERAL_OBJ_TYPE_VIRTIO_Q_COUNTERS = 0x001c,
 	MLX5_GENERAL_OBJ_TYPE_FLEX_PARSE_GRAPH = 0x0022,
@@ -2447,6 +2450,42 @@ struct mlx5_ifc_create_geneve_tlv_option_in_bits {
 	struct mlx5_ifc_geneve_tlv_option_bits geneve_tlv_opt;
 };
 
+enum {
+	MLX5_CRYPTO_KEY_SIZE_128b = 0x0,
+	MLX5_CRYPTO_KEY_SIZE_256b = 0x1,
+};
+
+enum {
+	MLX5_CRYPTO_KEY_PURPOSE_TLS	= 0x1,
+	MLX5_CRYPTO_KEY_PURPOSE_IPSEC	= 0x2,
+	MLX5_CRYPTO_KEY_PURPOSE_AES_XTS	= 0x3,
+	MLX5_CRYPTO_KEY_PURPOSE_MACSEC	= 0x4,
+	MLX5_CRYPTO_KEY_PURPOSE_GCM	= 0x5,
+	MLX5_CRYPTO_KEY_PURPOSE_PSP	= 0x6,
+};
+
+struct mlx5_ifc_dek_bits {
+	u8 modify_field_select[0x40];
+	u8 state[0x8];
+	u8 reserved_at_48[0xc];
+	u8 key_size[0x4];
+	u8 has_keytag[0x1];
+	u8 reserved_at_59[0x3];
+	u8 key_purpose[0x4];
+	u8 reserved_at_60[0x8];
+	u8 pd[0x18];
+	u8 reserved_at_80[0x100];
+	u8 opaque[0x40];
+	u8 reserved_at_1c0[0x40];
+	u8 key[0x400];
+	u8 reserved_at_600[0x200];
+};
+
+struct mlx5_ifc_create_dek_in_bits {
+	struct mlx5_ifc_general_obj_in_cmd_hdr_bits hdr;
+	struct mlx5_ifc_dek_bits dek;
+};
+
 enum {
 	MLX5_VIRTQ_STATE_INIT = 0,
 	MLX5_VIRTQ_STATE_RDY = 1,
diff --git a/drivers/common/mlx5/version.map b/drivers/common/mlx5/version.map
index 91f3fa5779..50017f9005 100644
--- a/drivers/common/mlx5/version.map
+++ b/drivers/common/mlx5/version.map
@@ -24,6 +24,7 @@ INTERNAL {
 	mlx5_devx_cmd_create_virtq;
 	mlx5_devx_cmd_create_flow_hit_aso_obj;
 	mlx5_devx_cmd_create_geneve_tlv_option;
+        mlx5_devx_cmd_create_dek_obj;
 	mlx5_devx_cmd_destroy;
 	mlx5_devx_cmd_flow_counter_alloc;
 	mlx5_devx_cmd_flow_counter_query;
-- 
2.21.0

[dpdk-dev] [PATCH 06/24] common/mlx5: add crypto en field to MKEY context

[Shiri Kuzin]

From: Dekel Peled <dekelp@nvidia.com>

MKEY that will be used for crypto purposes must be created with
crypto_en attribute.
This patch adds support for crypto_en attribute in MKEY context.

Signed-off-by: Dekel Peled <dekelp@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_devx_cmds.c |  1 +
 drivers/common/mlx5/mlx5_devx_cmds.h |  1 +
 drivers/common/mlx5/mlx5_prm.h       | 10 +++++++++-
 3 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c
index bad474aeae..262930675a 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.c
+++ b/drivers/common/mlx5/mlx5_devx_cmds.c
@@ -272,6 +272,7 @@ mlx5_devx_cmd_mkey_create(void *ctx,
 	MLX5_SET(mkc, mkc, relaxed_ordering_read, attr->relaxed_ordering_read);
 	MLX5_SET64(mkc, mkc, start_addr, attr->addr);
 	MLX5_SET64(mkc, mkc, len, attr->size);
+	MLX5_SET(mkc, mkc, crypto_en, attr->crypto_en);
 	mkey->obj = mlx5_glue->devx_obj_create(ctx, in, in_size_dw * 4, out,
 					       sizeof(out));
 	if (!mkey->obj) {
diff --git a/drivers/common/mlx5/mlx5_devx_cmds.h b/drivers/common/mlx5/mlx5_devx_cmds.h
index e17bad6bce..94bd8f292f 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.h
+++ b/drivers/common/mlx5/mlx5_devx_cmds.h
@@ -31,6 +31,7 @@ struct mlx5_devx_mkey_attr {
 	uint32_t pg_access:1;
 	uint32_t relaxed_ordering_write:1;
 	uint32_t relaxed_ordering_read:1;
+	uint32_t crypto_en:2;
 	struct mlx5_klm *klm_array;
 	int klm_num;
 };
diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index fbe191cc94..4a06e656fd 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -1059,7 +1059,15 @@ struct mlx5_ifc_mkc_bits {
 	u8 relaxed_ordering_read[0x1];
 	u8 reserved_at_1da[0x1];
 	u8 log_page_size[0x5];
-	u8 reserved_at_1e0[0x20];
+	u8 reserved_at_1e0[0x3];
+	u8 crypto_en[0x2];
+	u8 reserved_at_1e5[0x1b];
+};
+
+/* Range of values for MKEY context crypto_en field. */
+enum {
+	MLX5_MKEY_CRYPTO_DISABLED = 0x0,
+	MLX5_MKEY_CRYPTO_ENABLED = 0x1,
 };
 
 struct mlx5_ifc_create_mkey_out_bits {
-- 
2.21.0

[dpdk-dev] [PATCH 07/24] common/mlx5: support umr en field in MKEY context

[Shiri Kuzin]

From: Dekel Peled <dekelp@nvidia.com>

MKEY that will be used in UMR operations must be created with
umr_en attribute.
This patch adds support for setting umr_en attribute in MKEY context.

Signed-off-by: Dekel Peled <dekelp@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_devx_cmds.c | 1 +
 drivers/common/mlx5/mlx5_devx_cmds.h | 1 +
 2 files changed, 2 insertions(+)

diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c
index 262930675a..7208ae1f11 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.c
+++ b/drivers/common/mlx5/mlx5_devx_cmds.c
@@ -273,6 +273,7 @@ mlx5_devx_cmd_mkey_create(void *ctx,
 	MLX5_SET64(mkc, mkc, start_addr, attr->addr);
 	MLX5_SET64(mkc, mkc, len, attr->size);
 	MLX5_SET(mkc, mkc, crypto_en, attr->crypto_en);
+	MLX5_SET(mkc, mkc, umr_en, attr->umr_en);
 	mkey->obj = mlx5_glue->devx_obj_create(ctx, in, in_size_dw * 4, out,
 					       sizeof(out));
 	if (!mkey->obj) {
diff --git a/drivers/common/mlx5/mlx5_devx_cmds.h b/drivers/common/mlx5/mlx5_devx_cmds.h
index 94bd8f292f..1571542dbc 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.h
+++ b/drivers/common/mlx5/mlx5_devx_cmds.h
@@ -32,6 +32,7 @@ struct mlx5_devx_mkey_attr {
 	uint32_t relaxed_ordering_write:1;
 	uint32_t relaxed_ordering_read:1;
 	uint32_t crypto_en:2;
+	uint32_t umr_en:1;
 	struct mlx5_klm *klm_array;
 	int klm_num;
 };
-- 
2.21.0

[dpdk-dev] [PATCH 08/24] common/mlx5: support general obj IMPORT KEK create

[Shiri Kuzin]

From: Dekel Peled <dekelp@nvidia.com>

IMPORT_KEK object is used to wrap (encrypt) critical security
parameters, such as other keys and credentials, when those need
to be passed between the device and the software.

This patch add support of IMPORT_KEK object create operation.
Add reading of IMPORT_KEK support capability.
Add function to create general object type IMPORT_KEK, using DevX API.

Signed-off-by: Dekel Peled <dekelp@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_devx_cmds.c | 50 ++++++++++++++++++++++++++++
 drivers/common/mlx5/mlx5_devx_cmds.h | 13 ++++++++
 drivers/common/mlx5/mlx5_prm.h       | 18 ++++++++++
 drivers/common/mlx5/version.map      |  1 +
 4 files changed, 82 insertions(+)

diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c
index 7208ae1f11..5855b5ead5 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.c
+++ b/drivers/common/mlx5/mlx5_devx_cmds.c
@@ -732,6 +732,8 @@ mlx5_devx_cmd_query_hca_attr(void *ctx,
 				  MLX5_GENERAL_OBJ_TYPES_CAP_GENEVE_TLV_OPT);
 	attr->dek = !!(general_obj_types_supported &
 		       MLX5_GENERAL_OBJ_TYPES_CAP_DEK);
+	attr->import_kek = !!(general_obj_types_supported &
+			      MLX5_GENERAL_OBJ_TYPES_CAP_IMPORT_KEK);
 	/* Add reading of other GENERAL_OBJ_TYPES_CAP bits above this line. */
 	attr->log_max_cq = MLX5_GET(cmd_hca_cap, hcattr, log_max_cq);
 	attr->log_max_qp = MLX5_GET(cmd_hca_cap, hcattr, log_max_qp);
@@ -2354,3 +2356,51 @@ mlx5_devx_cmd_create_dek_obj(void *ctx, struct mlx5_devx_dek_attr *attr)
 	dek_obj->id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
 	return dek_obj;
 }
+
+/**
+ * Create general object of type IMPORT_KEK using DevX API.
+ *
+ * @param[in] ctx
+ *   Context returned from mlx5 open_device() glue function.
+ * @param [in] attr
+ *   Pointer to IMPORT_KEK attributes structure.
+ *
+ * @return
+ *   The DevX object created, NULL otherwise and rte_errno is set.
+ */
+struct mlx5_devx_obj *
+mlx5_devx_cmd_create_import_kek_obj(void *ctx,
+				    struct mlx5_devx_import_kek_attr *attr)
+{
+	uint32_t in[MLX5_ST_SZ_DW(create_import_kek_in)] = {0};
+	uint32_t out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};
+	struct mlx5_devx_obj *import_kek_obj = NULL;
+	void *ptr = NULL, *key_addr = NULL;
+
+	import_kek_obj = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*import_kek_obj),
+				     0, SOCKET_ID_ANY);
+	if (import_kek_obj == NULL) {
+		DRV_LOG(ERR, "Failed to allocate IMPORT_KEK object data");
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+	ptr = MLX5_ADDR_OF(create_import_kek_in, in, hdr);
+	MLX5_SET(general_obj_in_cmd_hdr, ptr, opcode,
+		 MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
+	MLX5_SET(general_obj_in_cmd_hdr, ptr, obj_type,
+		 MLX5_GENERAL_OBJ_TYPE_IMPORT_KEK);
+	ptr = MLX5_ADDR_OF(create_import_kek_in, in, import_kek);
+	MLX5_SET(import_kek, ptr, key_size, attr->key_size);
+	key_addr = MLX5_ADDR_OF(import_kek, ptr, key);
+	memcpy(key_addr, (void *)(attr->key), MLX5_CRYPTO_KEY_MAX_SIZE);
+	import_kek_obj->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in),
+							 out, sizeof(out));
+	if (import_kek_obj->obj == NULL) {
+		rte_errno = errno;
+		DRV_LOG(ERR, "Failed to create IMPORT_KEK object using DevX.");
+		mlx5_free(import_kek_obj);
+		return NULL;
+	}
+	import_kek_obj->id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
+	return import_kek_obj;
+}
diff --git a/drivers/common/mlx5/mlx5_devx_cmds.h b/drivers/common/mlx5/mlx5_devx_cmds.h
index 1571542dbc..eebe05911f 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.h
+++ b/drivers/common/mlx5/mlx5_devx_cmds.h
@@ -132,6 +132,7 @@ struct mlx5_hca_attr {
 	uint32_t crypto:1; /* Crypto engine is supported. */
 	uint32_t aes_xts:1; /* AES-XTS crypto is supported. */
 	uint32_t dek:1; /* General obj type DEK is supported. */
+	uint32_t import_kek:1; /* General obj type IMPORT_KEK supported. */
 	uint32_t regexp_num_of_engines;
 	uint32_t log_max_ft_sampler_num:8;
 	uint32_t geneve_tlv_opt;
@@ -438,6 +439,13 @@ struct mlx5_devx_dek_attr {
 	uint8_t key[MLX5_CRYPTO_KEY_MAX_SIZE];
 };
 
+struct mlx5_devx_import_kek_attr {
+	uint64_t modify_field_select;
+	uint32_t state:8;
+	uint32_t key_size:4;
+	uint8_t key[MLX5_CRYPTO_KEY_MAX_SIZE];
+};
+
 /* mlx5_devx_cmds.c */
 
 __rte_internal
@@ -576,4 +584,9 @@ __rte_internal
 struct mlx5_devx_obj *
 mlx5_devx_cmd_create_dek_obj(void *ctx, struct mlx5_devx_dek_attr *attr);
 
+__rte_internal
+struct mlx5_devx_obj *
+mlx5_devx_cmd_create_import_kek_obj(void *ctx,
+				    struct mlx5_devx_import_kek_attr *attr);
+
 #endif /* RTE_PMD_MLX5_DEVX_CMDS_H_ */
diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index 4a06e656fd..1569dae9a4 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -1117,6 +1117,8 @@ enum {
 			(1ULL << MLX5_GENERAL_OBJ_TYPE_GENEVE_TLV_OPT)
 #define MLX5_GENERAL_OBJ_TYPES_CAP_DEK \
 			(1ULL << MLX5_GENERAL_OBJ_TYPE_DEK)
+#define MLX5_GENERAL_OBJ_TYPES_CAP_IMPORT_KEK \
+			(1ULL << MLX5_GENERAL_OBJ_TYPE_IMPORT_KEK)
 
 enum {
 	MLX5_HCA_CAP_OPMOD_GET_MAX   = 0,
@@ -2400,6 +2402,7 @@ enum {
 	MLX5_GENERAL_OBJ_TYPE_DEK = 0x000c,
 	MLX5_GENERAL_OBJ_TYPE_VIRTQ = 0x000d,
 	MLX5_GENERAL_OBJ_TYPE_VIRTIO_Q_COUNTERS = 0x001c,
+	MLX5_GENERAL_OBJ_TYPE_IMPORT_KEK = 0x001d,
 	MLX5_GENERAL_OBJ_TYPE_FLEX_PARSE_GRAPH = 0x0022,
 	MLX5_GENERAL_OBJ_TYPE_FLOW_HIT_ASO = 0x0025,
 };
@@ -2494,6 +2497,21 @@ struct mlx5_ifc_create_dek_in_bits {
 	struct mlx5_ifc_dek_bits dek;
 };
 
+struct mlx5_ifc_import_kek_bits {
+	u8 modify_field_select[0x40];
+	u8 state[0x8];
+	u8 reserved_at_48[0xc];
+	u8 key_size[0x4];
+	u8 reserved_at_58[0x1a8];
+	u8 key[0x400];
+	u8 reserved_at_600[0x200];
+};
+
+struct mlx5_ifc_create_import_kek_in_bits {
+	struct mlx5_ifc_general_obj_in_cmd_hdr_bits hdr;
+	struct mlx5_ifc_import_kek_bits import_kek;
+};
+
 enum {
 	MLX5_VIRTQ_STATE_INIT = 0,
 	MLX5_VIRTQ_STATE_RDY = 1,
diff --git a/drivers/common/mlx5/version.map b/drivers/common/mlx5/version.map
index 50017f9005..1407e52d02 100644
--- a/drivers/common/mlx5/version.map
+++ b/drivers/common/mlx5/version.map
@@ -25,6 +25,7 @@ INTERNAL {
 	mlx5_devx_cmd_create_flow_hit_aso_obj;
 	mlx5_devx_cmd_create_geneve_tlv_option;
         mlx5_devx_cmd_create_dek_obj;
+        mlx5_devx_cmd_create_import_kek_obj;
 	mlx5_devx_cmd_destroy;
 	mlx5_devx_cmd_flow_counter_alloc;
 	mlx5_devx_cmd_flow_counter_query;
-- 
2.21.0

[dpdk-dev] [PATCH 09/24] common/mlx5: support general obj CRYPTO LOGIN create

[Shiri Kuzin]

From: Dekel Peled <dekelp@nvidia.com>

CRYPTO_LOGIN Object is used to login to the device as crypto user
or crypto officer.
Required in order to perform any crypto related control operations.

This patch adds support of CRYPTO_LOGIN object create operation.
Add reading of CRYPTO_LOGIN support capability.
Add function to create general object type CRYPTO_LOGIN, using DevX API.

Signed-off-by: Dekel Peled <dekelp@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_devx_cmds.c | 54 ++++++++++++++++++++++++++++
 drivers/common/mlx5/mlx5_devx_cmds.h | 15 ++++++++
 drivers/common/mlx5/mlx5_prm.h       | 19 ++++++++++
 drivers/common/mlx5/version.map      |  1 +
 4 files changed, 89 insertions(+)

diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c
index 5855b5ead5..c3b3b50b51 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.c
+++ b/drivers/common/mlx5/mlx5_devx_cmds.c
@@ -734,6 +734,8 @@ mlx5_devx_cmd_query_hca_attr(void *ctx,
 		       MLX5_GENERAL_OBJ_TYPES_CAP_DEK);
 	attr->import_kek = !!(general_obj_types_supported &
 			      MLX5_GENERAL_OBJ_TYPES_CAP_IMPORT_KEK);
+	attr->crypto_login = !!(general_obj_types_supported &
+				MLX5_GENERAL_OBJ_TYPES_CAP_CRYPTO_LOGIN);
 	/* Add reading of other GENERAL_OBJ_TYPES_CAP bits above this line. */
 	attr->log_max_cq = MLX5_GET(cmd_hca_cap, hcattr, log_max_cq);
 	attr->log_max_qp = MLX5_GET(cmd_hca_cap, hcattr, log_max_qp);
@@ -2404,3 +2406,55 @@ mlx5_devx_cmd_create_import_kek_obj(void *ctx,
 	import_kek_obj->id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
 	return import_kek_obj;
 }
+
+/**
+ * Create general object of type CRYPTO_LOGIN using DevX API.
+ *
+ * @param[in] ctx
+ *   Context returned from mlx5 open_device() glue function.
+ * @param [in] attr
+ *   Pointer to CRYPTO_LOGIN attributes structure.
+ *
+ * @return
+ *   The DevX object created, NULL otherwise and rte_errno is set.
+ */
+struct mlx5_devx_obj *
+mlx5_devx_cmd_create_crypto_login_obj(void *ctx,
+				      struct mlx5_devx_crypto_login_attr *attr)
+{
+	uint32_t in[MLX5_ST_SZ_DW(create_crypto_login_in)] = {0};
+	uint32_t out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};
+	struct mlx5_devx_obj *crypto_login_obj = NULL;
+	void *ptr = NULL, *credential_addr = NULL;
+
+	crypto_login_obj = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*crypto_login_obj),
+				       0, SOCKET_ID_ANY);
+	if (crypto_login_obj == NULL) {
+		DRV_LOG(ERR, "Failed to allocate CRYPTO_LOGIN object data");
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+	ptr = MLX5_ADDR_OF(create_crypto_login_in, in, hdr);
+	MLX5_SET(general_obj_in_cmd_hdr, ptr, opcode,
+		 MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
+	MLX5_SET(general_obj_in_cmd_hdr, ptr, obj_type,
+		 MLX5_GENERAL_OBJ_TYPE_CRYPTO_LOGIN);
+	ptr = MLX5_ADDR_OF(create_crypto_login_in, in, crypto_login);
+	MLX5_SET(crypto_login, ptr, credential_pointer,
+		 attr->credential_pointer);
+	MLX5_SET(crypto_login, ptr, session_import_kek_ptr,
+		 attr->session_import_kek_ptr);
+	credential_addr = MLX5_ADDR_OF(crypto_login, ptr, credential);
+	memcpy(credential_addr, (void *)(attr->credential),
+	       MLX5_CRYPTO_LOGIN_CREDENTIAL_SIZE);
+	crypto_login_obj->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in),
+							   out, sizeof(out));
+	if (crypto_login_obj->obj == NULL) {
+		rte_errno = errno;
+		DRV_LOG(ERR, "Failed to create CRYPTO_LOGIN obj using DevX.");
+		mlx5_free(crypto_login_obj);
+		return NULL;
+	}
+	crypto_login_obj->id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
+	return crypto_login_obj;
+}
diff --git a/drivers/common/mlx5/mlx5_devx_cmds.h b/drivers/common/mlx5/mlx5_devx_cmds.h
index eebe05911f..b723aee126 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.h
+++ b/drivers/common/mlx5/mlx5_devx_cmds.h
@@ -133,6 +133,7 @@ struct mlx5_hca_attr {
 	uint32_t aes_xts:1; /* AES-XTS crypto is supported. */
 	uint32_t dek:1; /* General obj type DEK is supported. */
 	uint32_t import_kek:1; /* General obj type IMPORT_KEK supported. */
+	uint32_t crypto_login:1; /* General obj type CRYPTO_LOGIN supported. */
 	uint32_t regexp_num_of_engines;
 	uint32_t log_max_ft_sampler_num:8;
 	uint32_t geneve_tlv_opt;
@@ -446,6 +447,15 @@ struct mlx5_devx_import_kek_attr {
 	uint8_t key[MLX5_CRYPTO_KEY_MAX_SIZE];
 };
 
+#define MLX5_CRYPTO_LOGIN_CREDENTIAL_SIZE	48
+
+struct mlx5_devx_crypto_login_attr {
+	uint64_t modify_field_select;
+	uint32_t credential_pointer:24;
+	uint32_t session_import_kek_ptr:24;
+	uint8_t credential[MLX5_CRYPTO_LOGIN_CREDENTIAL_SIZE];
+};
+
 /* mlx5_devx_cmds.c */
 
 __rte_internal
@@ -589,4 +599,9 @@ struct mlx5_devx_obj *
 mlx5_devx_cmd_create_import_kek_obj(void *ctx,
 				    struct mlx5_devx_import_kek_attr *attr);
 
+__rte_internal
+struct mlx5_devx_obj *
+mlx5_devx_cmd_create_crypto_login_obj(void *ctx,
+				      struct mlx5_devx_crypto_login_attr *attr);
+
 #endif /* RTE_PMD_MLX5_DEVX_CMDS_H_ */
diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index 1569dae9a4..ca2e70881d 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -1119,6 +1119,8 @@ enum {
 			(1ULL << MLX5_GENERAL_OBJ_TYPE_DEK)
 #define MLX5_GENERAL_OBJ_TYPES_CAP_IMPORT_KEK \
 			(1ULL << MLX5_GENERAL_OBJ_TYPE_IMPORT_KEK)
+#define MLX5_GENERAL_OBJ_TYPES_CAP_CRYPTO_LOGIN \
+			(1ULL << MLX5_GENERAL_OBJ_TYPE_CRYPTO_LOGIN)
 
 enum {
 	MLX5_HCA_CAP_OPMOD_GET_MAX   = 0,
@@ -2403,6 +2405,7 @@ enum {
 	MLX5_GENERAL_OBJ_TYPE_VIRTQ = 0x000d,
 	MLX5_GENERAL_OBJ_TYPE_VIRTIO_Q_COUNTERS = 0x001c,
 	MLX5_GENERAL_OBJ_TYPE_IMPORT_KEK = 0x001d,
+	MLX5_GENERAL_OBJ_TYPE_CRYPTO_LOGIN = 0x001f,
 	MLX5_GENERAL_OBJ_TYPE_FLEX_PARSE_GRAPH = 0x0022,
 	MLX5_GENERAL_OBJ_TYPE_FLOW_HIT_ASO = 0x0025,
 };
@@ -2512,6 +2515,22 @@ struct mlx5_ifc_create_import_kek_in_bits {
 	struct mlx5_ifc_import_kek_bits import_kek;
 };
 
+struct mlx5_ifc_crypto_login_bits {
+	u8 modify_field_select[0x40];
+	u8 reserved_at_40[0x48];
+	u8 credential_pointer[0x18];
+	u8 reserved_at_a0[0x8];
+	u8 session_import_kek_ptr[0x18];
+	u8 reserved_at_c0[0x140];
+	u8 credential[0x180];
+	u8 reserved_at_380[0x480];
+};
+
+struct mlx5_ifc_create_crypto_login_in_bits {
+	struct mlx5_ifc_general_obj_in_cmd_hdr_bits hdr;
+	struct mlx5_ifc_crypto_login_bits crypto_login;
+};
+
 enum {
 	MLX5_VIRTQ_STATE_INIT = 0,
 	MLX5_VIRTQ_STATE_RDY = 1,
diff --git a/drivers/common/mlx5/version.map b/drivers/common/mlx5/version.map
index 1407e52d02..120be7a99d 100644
--- a/drivers/common/mlx5/version.map
+++ b/drivers/common/mlx5/version.map
@@ -26,6 +26,7 @@ INTERNAL {
 	mlx5_devx_cmd_create_geneve_tlv_option;
         mlx5_devx_cmd_create_dek_obj;
         mlx5_devx_cmd_create_import_kek_obj;
+        mlx5_devx_cmd_create_crypto_login_obj;
 	mlx5_devx_cmd_destroy;
 	mlx5_devx_cmd_flow_counter_alloc;
 	mlx5_devx_cmd_flow_counter_query;
-- 
2.21.0

[dpdk-dev] [PATCH 10/24] common/mlx5: add crypto BSF struct and defines

[Shiri Kuzin]

From: Dekel Peled <dekelp@nvidia.com>

This patch adds the struct defining crypto BSF segment of UMR WQE,
and the related value definitions and offsets.

Signed-off-by: Dekel Peled <dekelp@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_prm.h | 66 ++++++++++++++++++++++++++++++++++
 1 file changed, 66 insertions(+)

diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index ca2e70881d..736badb6df 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -1096,6 +1096,72 @@ struct mlx5_ifc_create_mkey_in_bits {
 	u8 klm_pas_mtt[][0x20];
 };
 
+enum {
+	MLX5_BSF_SIZE_16B = 0x0,
+	MLX5_BSF_SIZE_32B = 0x1,
+	MLX5_BSF_SIZE_64B = 0x2,
+	MLX5_BSF_SIZE_128B = 0x3,
+};
+
+enum {
+	MLX5_BSF_P_TYPE_SIGNATURE = 0x0,
+	MLX5_BSF_P_TYPE_CRYPTO = 0x1,
+};
+
+enum {
+	MLX5_ENCRYPTION_ORDER_ENCRYPTED_WIRE_SIGNATURE = 0x0,
+	MLX5_ENCRYPTION_ORDER_ENCRYPTED_MEMORY_SIGNATURE = 0x1,
+	MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_WIRE = 0x2,
+	MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_MEMORY = 0x3,
+};
+
+enum {
+	MLX5_ENCRYPTION_STANDARD_AES_XTS = 0x0,
+};
+
+enum {
+	MLX5_BLOCK_SIZE_512B	= 0x1,
+	MLX5_BLOCK_SIZE_520B	= 0x2,
+	MLX5_BLOCK_SIZE_4096B	= 0x3,
+	MLX5_BLOCK_SIZE_4160B	= 0x4,
+	MLX5_BLOCK_SIZE_1MB	= 0x5,
+	MLX5_BLOCK_SIZE_4048B	= 0x6,
+};
+
+#define MLX5_BSF_SIZE_OFFSET		30
+#define MLX5_BSF_P_TYPE_OFFSET		24
+#define MLX5_ENCRYPTION_ORDER_OFFSET	16
+#define MLX5_BLOCK_SIZE_OFFSET		24
+
+struct mlx5_wqe_umr_bsf_seg {
+	/*
+	 * bs_bpt_eo_es contains:
+	 * bs	bsf_size		2 bits at MLX5_BSF_SIZE_OFFSET
+	 * bpt	bsf_p_type		2 bits at MLX5_BSF_P_TYPE_OFFSET
+	 * eo	encryption_order	4 bits at MLX5_ENCRYPTION_ORDER_OFFSET
+	 * es	encryption_standard	4 bits at offset 0
+	 */
+	uint32_t bs_bpt_eo_es;
+	uint32_t raw_data_size;
+	/*
+	 * bsp_res contains:
+	 * bsp	crypto_block_size_pointer	8 bits at MLX5_BLOCK_SIZE_OFFSET
+	 * res	reserved 24 bits
+	 */
+	uint32_t bsp_res;
+	uint32_t reserved0;
+	uint8_t xts_initial_tweak[16];
+	/*
+	 * res_dp contains:
+	 * res	reserved 8 bits
+	 * dp	dek_pointer		24 bits at offset 0
+	 */
+	uint32_t res_dp;
+	uint32_t reserved1;
+	uint64_t keytag;
+	uint32_t reserved2[4];
+} __rte_packed;
+
 enum {
 	MLX5_GET_HCA_CAP_OP_MOD_GENERAL_DEVICE = 0x0 << 1,
 	MLX5_GET_HCA_CAP_OP_MOD_ETHERNET_OFFLOAD_CAPS = 0x1 << 1,
-- 
2.21.0

[dpdk-dev] [PATCH 11/24] common/mlx5: share hash list tool

[Shiri Kuzin]

In order to use the hash list defined in net in other drivers, the
hash list is moved to common utilities.

In addition, the log definition was moved from the common utilities to
a dedicated new log file in common in order to prevent a conflict.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/linux/mlx5_common_os.c    |   2 +-
 drivers/common/mlx5/linux/mlx5_nl.c           |   2 +-
 drivers/common/mlx5/meson.build               |   1 +
 drivers/common/mlx5/mlx5_common.c             |   2 +-
 drivers/common/mlx5/mlx5_common_devx.c        |   2 +-
 drivers/common/mlx5/mlx5_common_log.h         |  21 ++
 drivers/common/mlx5/mlx5_common_mp.c          |   2 +-
 drivers/common/mlx5/mlx5_common_mr.c          |   2 +-
 drivers/common/mlx5/mlx5_common_pci.c         |   3 +-
 drivers/common/mlx5/mlx5_common_utils.c       | 221 ++++++++++++++++++
 drivers/common/mlx5/mlx5_common_utils.h       | 202 +++++++++++++++-
 drivers/common/mlx5/mlx5_devx_cmds.c          |   2 +-
 drivers/common/mlx5/mlx5_malloc.c             |   2 +-
 .../common/mlx5/rte_common_mlx5_exports.def   |   7 +
 drivers/common/mlx5/version.map               |   6 +
 drivers/common/mlx5/windows/mlx5_common_os.c  |   2 +-
 drivers/common/mlx5/windows/mlx5_glue.c       |   2 +-
 drivers/net/mlx5/mlx5_utils.c                 | 209 -----------------
 drivers/net/mlx5/mlx5_utils.h                 | 194 +--------------
 19 files changed, 464 insertions(+), 420 deletions(-)
 create mode 100644 drivers/common/mlx5/mlx5_common_log.h
 create mode 100644 drivers/common/mlx5/mlx5_common_utils.c

diff --git a/drivers/common/mlx5/linux/mlx5_common_os.c b/drivers/common/mlx5/linux/mlx5_common_os.c
index 5cf9576921..fba8245b8b 100644
--- a/drivers/common/mlx5/linux/mlx5_common_os.c
+++ b/drivers/common/mlx5/linux/mlx5_common_os.c
@@ -15,7 +15,7 @@
 #include <rte_string_fns.h>
 
 #include "mlx5_common.h"
-#include "mlx5_common_utils.h"
+#include "mlx5_common_log.h"
 #include "mlx5_glue.h"
 
 #ifdef MLX5_GLUE
diff --git a/drivers/common/mlx5/linux/mlx5_nl.c b/drivers/common/mlx5/linux/mlx5_nl.c
index 752c57b33d..f0d04f9473 100644
--- a/drivers/common/mlx5/linux/mlx5_nl.c
+++ b/drivers/common/mlx5/linux/mlx5_nl.c
@@ -20,7 +20,7 @@
 #include <rte_errno.h>
 
 #include "mlx5_nl.h"
-#include "mlx5_common_utils.h"
+#include "../mlx5_common_log.h"
 #include "mlx5_malloc.h"
 #ifdef HAVE_DEVLINK
 #include <linux/devlink.h>
diff --git a/drivers/common/mlx5/meson.build b/drivers/common/mlx5/meson.build
index b2efea4432..1459b9bd89 100644
--- a/drivers/common/mlx5/meson.build
+++ b/drivers/common/mlx5/meson.build
@@ -16,6 +16,7 @@ sources += files(
 	'mlx5_malloc.c',
 	'mlx5_common_pci.c',
 	'mlx5_common_devx.c',
+	'mlx5_common_utils.c',
 )
 
 cflags_options = [
diff --git a/drivers/common/mlx5/mlx5_common.c b/drivers/common/mlx5/mlx5_common.c
index c26a2cfa30..acb785f9ce 100644
--- a/drivers/common/mlx5/mlx5_common.c
+++ b/drivers/common/mlx5/mlx5_common.c
@@ -11,7 +11,7 @@
 
 #include "mlx5_common.h"
 #include "mlx5_common_os.h"
-#include "mlx5_common_utils.h"
+#include "mlx5_common_log.h"
 #include "mlx5_common_pci.h"
 
 int mlx5_common_logtype;
diff --git a/drivers/common/mlx5/mlx5_common_devx.c b/drivers/common/mlx5/mlx5_common_devx.c
index d19be122bd..22c8d356c4 100644
--- a/drivers/common/mlx5/mlx5_common_devx.c
+++ b/drivers/common/mlx5/mlx5_common_devx.c
@@ -12,7 +12,7 @@
 
 #include "mlx5_prm.h"
 #include "mlx5_devx_cmds.h"
-#include "mlx5_common_utils.h"
+#include "mlx5_common_log.h"
 #include "mlx5_malloc.h"
 #include "mlx5_common.h"
 #include "mlx5_common_devx.h"
diff --git a/drivers/common/mlx5/mlx5_common_log.h b/drivers/common/mlx5/mlx5_common_log.h
new file mode 100644
index 0000000000..26b13fedaf
--- /dev/null
+++ b/drivers/common/mlx5/mlx5_common_log.h
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2019 Mellanox Technologies, Ltd
+ */
+
+#ifndef RTE_PMD_MLX5_COMMON_LOG_H_
+#define RTE_PMD_MLX5_COMMON_LOG_H_
+
+#include "mlx5_common.h"
+
+
+extern int mlx5_common_logtype;
+
+#define MLX5_COMMON_LOG_PREFIX "mlx5_common"
+/* Generic printf()-like logging macro with automatic line feed. */
+#define DRV_LOG(level, ...) \
+	PMD_DRV_LOG_(level, mlx5_common_logtype, MLX5_COMMON_LOG_PREFIX, \
+		__VA_ARGS__ PMD_DRV_LOG_STRIP PMD_DRV_LOG_OPAREN, \
+		PMD_DRV_LOG_CPAREN)
+
+#endif /* RTE_PMD_MLX5_COMMON_LOG_H_ */
+
diff --git a/drivers/common/mlx5/mlx5_common_mp.c b/drivers/common/mlx5/mlx5_common_mp.c
index 40e3956e45..673a7c31de 100644
--- a/drivers/common/mlx5/mlx5_common_mp.c
+++ b/drivers/common/mlx5/mlx5_common_mp.c
@@ -10,7 +10,7 @@
 #include <rte_errno.h>
 
 #include "mlx5_common_mp.h"
-#include "mlx5_common_utils.h"
+#include "mlx5_common_log.h"
 #include "mlx5_malloc.h"
 
 /**
diff --git a/drivers/common/mlx5/mlx5_common_mr.c b/drivers/common/mlx5/mlx5_common_mr.c
index e1ed0caf3a..afb5b3d0a7 100644
--- a/drivers/common/mlx5/mlx5_common_mr.c
+++ b/drivers/common/mlx5/mlx5_common_mr.c
@@ -11,7 +11,7 @@
 #include "mlx5_glue.h"
 #include "mlx5_common_mp.h"
 #include "mlx5_common_mr.h"
-#include "mlx5_common_utils.h"
+#include "mlx5_common_log.h"
 #include "mlx5_malloc.h"
 
 struct mr_find_contig_memsegs_data {
diff --git a/drivers/common/mlx5/mlx5_common_pci.c b/drivers/common/mlx5/mlx5_common_pci.c
index 21b1acdf87..177ccf62d9 100644
--- a/drivers/common/mlx5/mlx5_common_pci.c
+++ b/drivers/common/mlx5/mlx5_common_pci.c
@@ -5,7 +5,8 @@
 #include <stdlib.h>
 #include <rte_malloc.h>
 #include <rte_class.h>
-#include "mlx5_common_utils.h"
+
+#include "mlx5_common_log.h"
 #include "mlx5_common_pci.h"
 
 struct mlx5_pci_device {
diff --git a/drivers/common/mlx5/mlx5_common_utils.c b/drivers/common/mlx5/mlx5_common_utils.c
new file mode 100644
index 0000000000..ad2011e858
--- /dev/null
+++ b/drivers/common/mlx5/mlx5_common_utils.c
@@ -0,0 +1,221 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2019 Mellanox Technologies, Ltd
+ */
+
+#include <rte_malloc.h>
+#include <rte_hash_crc.h>
+#include <rte_errno.h>
+
+#include <mlx5_malloc.h>
+
+#include "mlx5_common_utils.h"
+#include "mlx5_common_log.h"
+
+/********************* Hash List **********************/
+
+static struct mlx5_hlist_entry *
+mlx5_hlist_default_create_cb(struct mlx5_hlist *h, uint64_t key __rte_unused,
+			     void *ctx __rte_unused)
+{
+	return mlx5_malloc(MLX5_MEM_ZERO, h->entry_sz, 0, SOCKET_ID_ANY);
+}
+
+static void
+mlx5_hlist_default_remove_cb(struct mlx5_hlist *h __rte_unused,
+			     struct mlx5_hlist_entry *entry)
+{
+	mlx5_free(entry);
+}
+
+struct mlx5_hlist *
+mlx5_hlist_create(const char *name, uint32_t size, uint32_t entry_size,
+		  uint32_t flags, mlx5_hlist_create_cb cb_create,
+		  mlx5_hlist_match_cb cb_match, mlx5_hlist_remove_cb cb_remove)
+{
+	struct mlx5_hlist *h;
+	uint32_t act_size;
+	uint32_t alloc_size;
+	uint32_t i;
+
+	if (!size || !cb_match || (!cb_create ^ !cb_remove))
+		return NULL;
+	/* Align to the next power of 2, 32bits integer is enough now. */
+	if (!rte_is_power_of_2(size)) {
+		act_size = rte_align32pow2(size);
+		DRV_LOG(DEBUG, "Size 0x%" PRIX32 " is not power of 2, "
+			"will be aligned to 0x%" PRIX32 ".", size, act_size);
+	} else {
+		act_size = size;
+	}
+	alloc_size = sizeof(struct mlx5_hlist) +
+		     sizeof(struct mlx5_hlist_bucket) * act_size;
+	/* Using zmalloc, then no need to initialize the heads. */
+	h = mlx5_malloc(MLX5_MEM_ZERO, alloc_size, RTE_CACHE_LINE_SIZE,
+			SOCKET_ID_ANY);
+	if (!h) {
+		DRV_LOG(ERR, "No memory for hash list %s creation",
+			name ? name : "None");
+		return NULL;
+	}
+	if (name)
+		snprintf(h->name, MLX5_HLIST_NAMESIZE, "%s", name);
+	h->table_sz = act_size;
+	h->mask = act_size - 1;
+	h->entry_sz = entry_size;
+	h->direct_key = !!(flags & MLX5_HLIST_DIRECT_KEY);
+	h->write_most = !!(flags & MLX5_HLIST_WRITE_MOST);
+	h->cb_create = cb_create ? cb_create : mlx5_hlist_default_create_cb;
+	h->cb_match = cb_match;
+	h->cb_remove = cb_remove ? cb_remove : mlx5_hlist_default_remove_cb;
+	for (i = 0; i < act_size; i++)
+		rte_rwlock_init(&h->buckets[i].lock);
+	DRV_LOG(DEBUG, "Hash list with %s size 0x%" PRIX32 " is created.",
+		h->name, act_size);
+	return h;
+}
+
+static struct mlx5_hlist_entry *
+__hlist_lookup(struct mlx5_hlist *h, uint64_t key, uint32_t idx,
+	       void *ctx, bool reuse)
+{
+	struct mlx5_hlist_head *first;
+	struct mlx5_hlist_entry *node;
+
+	MLX5_ASSERT(h);
+	first = &h->buckets[idx].head;
+	LIST_FOREACH(node, first, next) {
+		if (!h->cb_match(h, node, key, ctx)) {
+			if (reuse) {
+				__atomic_add_fetch(&node->ref_cnt, 1,
+						   __ATOMIC_RELAXED);
+				DRV_LOG(DEBUG, "Hash list %s entry %p "
+					"reuse: %u.",
+					h->name, (void *)node, node->ref_cnt);
+			}
+			break;
+		}
+	}
+	return node;
+}
+
+static struct mlx5_hlist_entry *
+hlist_lookup(struct mlx5_hlist *h, uint64_t key, uint32_t idx,
+	     void *ctx, bool reuse)
+{
+	struct mlx5_hlist_entry *node;
+
+	MLX5_ASSERT(h);
+	rte_rwlock_read_lock(&h->buckets[idx].lock);
+	node = __hlist_lookup(h, key, idx, ctx, reuse);
+	rte_rwlock_read_unlock(&h->buckets[idx].lock);
+	return node;
+}
+
+struct mlx5_hlist_entry *
+mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
+{
+	uint32_t idx;
+
+	if (h->direct_key)
+		idx = (uint32_t)(key & h->mask);
+	else
+		idx = rte_hash_crc_8byte(key, 0) & h->mask;
+	return hlist_lookup(h, key, idx, ctx, false);
+}
+
+struct mlx5_hlist_entry*
+mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
+{
+	uint32_t idx;
+	struct mlx5_hlist_head *first;
+	struct mlx5_hlist_bucket *b;
+	struct mlx5_hlist_entry *entry;
+	uint32_t prev_gen_cnt = 0;
+
+	if (h->direct_key)
+		idx = (uint32_t)(key & h->mask);
+	else
+		idx = rte_hash_crc_8byte(key, 0) & h->mask;
+	MLX5_ASSERT(h);
+	b = &h->buckets[idx];
+	/* Use write lock directly for write-most list. */
+	if (!h->write_most) {
+		prev_gen_cnt = __atomic_load_n(&b->gen_cnt, __ATOMIC_ACQUIRE);
+		entry = hlist_lookup(h, key, idx, ctx, true);
+		if (entry)
+			return entry;
+	}
+	rte_rwlock_write_lock(&b->lock);
+	/* Check if the list changed by other threads. */
+	if (h->write_most ||
+	    prev_gen_cnt != __atomic_load_n(&b->gen_cnt, __ATOMIC_ACQUIRE)) {
+		entry = __hlist_lookup(h, key, idx, ctx, true);
+		if (entry)
+			goto done;
+	}
+	first = &b->head;
+	entry = h->cb_create(h, key, ctx);
+	if (!entry) {
+		rte_errno = ENOMEM;
+		DRV_LOG(DEBUG, "Can't allocate hash list %s entry.", h->name);
+		goto done;
+	}
+	entry->idx = idx;
+	entry->ref_cnt = 1;
+	LIST_INSERT_HEAD(first, entry, next);
+	__atomic_add_fetch(&b->gen_cnt, 1, __ATOMIC_ACQ_REL);
+	DRV_LOG(DEBUG, "Hash list %s entry %p new: %u.",
+		h->name, (void *)entry, entry->ref_cnt);
+done:
+	rte_rwlock_write_unlock(&b->lock);
+	return entry;
+}
+
+int
+mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry)
+{
+	uint32_t idx = entry->idx;
+
+	rte_rwlock_write_lock(&h->buckets[idx].lock);
+	MLX5_ASSERT(entry && entry->ref_cnt && entry->next.le_prev);
+	DRV_LOG(DEBUG, "Hash list %s entry %p deref: %u.",
+		h->name, (void *)entry, entry->ref_cnt);
+	if (--entry->ref_cnt) {
+		rte_rwlock_write_unlock(&h->buckets[idx].lock);
+		return 1;
+	}
+	LIST_REMOVE(entry, next);
+	/* Set to NULL to get rid of removing action for more than once. */
+	entry->next.le_prev = NULL;
+	h->cb_remove(h, entry);
+	rte_rwlock_write_unlock(&h->buckets[idx].lock);
+	DRV_LOG(DEBUG, "Hash list %s entry %p removed.",
+		h->name, (void *)entry);
+	return 0;
+}
+
+void
+mlx5_hlist_destroy(struct mlx5_hlist *h)
+{
+	uint32_t idx;
+	struct mlx5_hlist_entry *entry;
+
+	MLX5_ASSERT(h);
+	for (idx = 0; idx < h->table_sz; ++idx) {
+		/* No LIST_FOREACH_SAFE, using while instead. */
+		while (!LIST_EMPTY(&h->buckets[idx].head)) {
+			entry = LIST_FIRST(&h->buckets[idx].head);
+			LIST_REMOVE(entry, next);
+			/*
+			 * The owner of whole element which contains data entry
+			 * is the user, so it's the user's duty to do the clean
+			 * up and the free work because someone may not put the
+			 * hlist entry at the beginning(suggested to locate at
+			 * the beginning). Or else the default free function
+			 * will be used.
+			 */
+			h->cb_remove(h, entry);
+		}
+	}
+	mlx5_free(h);
+}
diff --git a/drivers/common/mlx5/mlx5_common_utils.h b/drivers/common/mlx5/mlx5_common_utils.h
index 6cba39c8cc..ed378ce9bd 100644
--- a/drivers/common/mlx5/mlx5_common_utils.h
+++ b/drivers/common/mlx5/mlx5_common_utils.h
@@ -7,14 +7,202 @@
 
 #include "mlx5_common.h"
 
+#define MLX5_HLIST_DIRECT_KEY 0x0001 /* Use the key directly as hash index. */
+#define MLX5_HLIST_WRITE_MOST 0x0002 /* List mostly used for append new. */
 
-extern int mlx5_common_logtype;
+/** Maximum size of string for naming the hlist table. */
+#define MLX5_HLIST_NAMESIZE			32
 
-#define MLX5_COMMON_LOG_PREFIX "mlx5_common"
-/* Generic printf()-like logging macro with automatic line feed. */
-#define DRV_LOG(level, ...) \
-	PMD_DRV_LOG_(level, mlx5_common_logtype, MLX5_COMMON_LOG_PREFIX, \
-		__VA_ARGS__ PMD_DRV_LOG_STRIP PMD_DRV_LOG_OPAREN, \
-		PMD_DRV_LOG_CPAREN)
+struct mlx5_hlist;
+
+/**
+ * Structure of the entry in the hash list, user should define its own struct
+ * that contains this in order to store the data. The 'key' is 64-bits right
+ * now and its user's responsibility to guarantee there is no collision.
+ */
+struct mlx5_hlist_entry {
+	LIST_ENTRY(mlx5_hlist_entry) next; /* entry pointers in the list. */
+	uint32_t idx; /* Bucket index the entry belongs to. */
+	uint32_t ref_cnt; /* Reference count. */
+};
+
+/** Structure for hash head. */
+LIST_HEAD(mlx5_hlist_head, mlx5_hlist_entry);
+
+/**
+ * Type of callback function for entry removal.
+ *
+ * @param list
+ *   The hash list.
+ * @param entry
+ *   The entry in the list.
+ */
+typedef void (*mlx5_hlist_remove_cb)(struct mlx5_hlist *list,
+				     struct mlx5_hlist_entry *entry);
+
+/**
+ * Type of function for user defined matching.
+ *
+ * @param list
+ *   The hash list.
+ * @param entry
+ *   The entry in the list.
+ * @param key
+ *   The new entry key.
+ * @param ctx
+ *   The pointer to new entry context.
+ *
+ * @return
+ *   0 if matching, non-zero number otherwise.
+ */
+typedef int (*mlx5_hlist_match_cb)(struct mlx5_hlist *list,
+				   struct mlx5_hlist_entry *entry,
+				   uint64_t key, void *ctx);
+
+/**
+ * Type of function for user defined hash list entry creation.
+ *
+ * @param list
+ *   The hash list.
+ * @param key
+ *   The key of the new entry.
+ * @param ctx
+ *   The pointer to new entry context.
+ *
+ * @return
+ *   Pointer to allocated entry on success, NULL otherwise.
+ */
+typedef struct mlx5_hlist_entry *(*mlx5_hlist_create_cb)
+				  (struct mlx5_hlist *list,
+				   uint64_t key, void *ctx);
+
+/* Hash list bucket head. */
+struct mlx5_hlist_bucket {
+	struct mlx5_hlist_head head; /* List head. */
+	rte_rwlock_t lock; /* Bucket lock. */
+	uint32_t gen_cnt; /* List modification will update generation count. */
+} __rte_cache_aligned;
+
+/**
+ * Hash list table structure
+ *
+ * Entry in hash list could be reused if entry already exists, reference
+ * count will increase and the existing entry returns.
+ *
+ * When destroy an entry from list, decrease reference count and only
+ * destroy when no further reference.
+ */
+struct mlx5_hlist {
+	char name[MLX5_HLIST_NAMESIZE]; /**< Name of the hash list. */
+	/**< number of heads, need to be power of 2. */
+	uint32_t table_sz;
+	uint32_t entry_sz; /**< Size of entry, used to allocate entry. */
+	/**< mask to get the index of the list heads. */
+	uint32_t mask;
+	bool direct_key; /* Use the new entry key directly as hash index. */
+	bool write_most; /* List mostly used for append new or destroy. */
+	void *ctx;
+	mlx5_hlist_create_cb cb_create; /**< entry create callback. */
+	mlx5_hlist_match_cb cb_match; /**< entry match callback. */
+	mlx5_hlist_remove_cb cb_remove; /**< entry remove callback. */
+	struct mlx5_hlist_bucket buckets[] __rte_cache_aligned;
+	/**< list bucket arrays. */
+};
+
+/**
+ * Create a hash list table, the user can specify the list heads array size
+ * of the table, now the size should be a power of 2 in order to get better
+ * distribution for the entries. Each entry is a part of the whole data element
+ * and the caller should be responsible for the data element's allocation and
+ * cleanup / free. Key of each entry will be calculated with CRC in order to
+ * generate a little fairer distribution.
+ *
+ * @param name
+ *   Name of the hash list(optional).
+ * @param size
+ *   Heads array size of the hash list.
+ * @param entry_size
+ *   Entry size to allocate if cb_create not specified.
+ * @param flags
+ *   The hash list attribute flags.
+ * @param cb_create
+ *   Callback function for entry create.
+ * @param cb_match
+ *   Callback function for entry match.
+ * @param cb_destroy
+ *   Callback function for entry destroy.
+ * @return
+ *   Pointer of the hash list table created, NULL on failure.
+ */
+__rte_internal
+struct mlx5_hlist *mlx5_hlist_create(const char *name, uint32_t size,
+				     uint32_t entry_size, uint32_t flags,
+				     mlx5_hlist_create_cb cb_create,
+				     mlx5_hlist_match_cb cb_match,
+				     mlx5_hlist_remove_cb cb_destroy);
+
+/**
+ * Search an entry matching the key.
+ *
+ * Result returned might be destroyed by other thread, must use
+ * this function only in main thread.
+ *
+ * @param h
+ *   Pointer to the hast list table.
+ * @param key
+ *   Key for the searching entry.
+ * @param ctx
+ *   Common context parameter used by entry callback function.
+ *
+ * @return
+ *   Pointer of the hlist entry if found, NULL otherwise.
+ */
+__rte_internal
+struct mlx5_hlist_entry *mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key,
+					   void *ctx);
+
+/**
+ * Insert an entry to the hash list table, the entry is only part of whole data
+ * element and a 64B key is used for matching. User should construct the key or
+ * give a calculated hash signature and guarantee there is no collision.
+ *
+ * @param h
+ *   Pointer to the hast list table.
+ * @param entry
+ *   Entry to be inserted into the hash list table.
+ * @param ctx
+ *   Common context parameter used by callback function.
+ *
+ * @return
+ *   registered entry on success, NULL otherwise
+ */
+__rte_internal
+struct mlx5_hlist_entry *mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key,
+					     void *ctx);
+
+/**
+ * Remove an entry from the hash list table. User should guarantee the validity
+ * of the entry.
+ *
+ * @param h
+ *   Pointer to the hast list table. (not used)
+ * @param entry
+ *   Entry to be removed from the hash list table.
+ * @return
+ *   0 on entry removed, 1 on entry still referenced.
+ */
+__rte_internal
+int mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry);
+
+/**
+ * Destroy the hash list table, all the entries already inserted into the lists
+ * will be handled by the callback function provided by the user (including
+ * free if needed) before the table is freed.
+ *
+ * @param h
+ *   Pointer to the hast list table.
+ */
+__rte_internal
+void mlx5_hlist_destroy(struct mlx5_hlist *h);
 
 #endif /* RTE_PMD_MLX5_COMMON_UTILS_H_ */
diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c
index c3b3b50b51..1b1227eb16 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.c
+++ b/drivers/common/mlx5/mlx5_devx_cmds.c
@@ -9,7 +9,7 @@
 
 #include "mlx5_prm.h"
 #include "mlx5_devx_cmds.h"
-#include "mlx5_common_utils.h"
+#include "mlx5_common_log.h"
 #include "mlx5_malloc.h"
 
 
diff --git a/drivers/common/mlx5/mlx5_malloc.c b/drivers/common/mlx5/mlx5_malloc.c
index 9d30cedbaa..b19501e1bc 100644
--- a/drivers/common/mlx5/mlx5_malloc.c
+++ b/drivers/common/mlx5/mlx5_malloc.c
@@ -8,7 +8,7 @@
 #include <stdbool.h>
 #include <string.h>
 
-#include "mlx5_common_utils.h"
+#include "mlx5_common_log.h"
 #include "mlx5_common_os.h"
 #include "mlx5_malloc.h"
 
diff --git a/drivers/common/mlx5/rte_common_mlx5_exports.def b/drivers/common/mlx5/rte_common_mlx5_exports.def
index fd62b806ca..f44ae61d52 100644
--- a/drivers/common/mlx5/rte_common_mlx5_exports.def
+++ b/drivers/common/mlx5/rte_common_mlx5_exports.def
@@ -73,3 +73,10 @@ EXPORTS
 	mlx5_os_reg_mr
 	mlx5_os_umem_reg
 	mlx5_os_umem_dereg
+
+	mlx5_hlist_create
+	mlx5_hlist_lookup
+	mlx5_hlist_register
+	mlx5_hlist_unregister
+	mlx5_hlist_destroy
+
diff --git a/drivers/common/mlx5/version.map b/drivers/common/mlx5/version.map
index 120be7a99d..5c2cc4dca9 100644
--- a/drivers/common/mlx5/version.map
+++ b/drivers/common/mlx5/version.map
@@ -108,4 +108,10 @@ INTERNAL {
 	mlx5_free;
 
 	mlx5_pci_driver_register;
+
+	mlx5_hlist_create;
+	mlx5_hlist_lookup;
+	mlx5_hlist_register;
+	mlx5_hlist_unregister;
+	mlx5_hlist_destroy;
 };
diff --git a/drivers/common/mlx5/windows/mlx5_common_os.c b/drivers/common/mlx5/windows/mlx5_common_os.c
index f2d781a965..2e6e172a96 100644
--- a/drivers/common/mlx5/windows/mlx5_common_os.c
+++ b/drivers/common/mlx5/windows/mlx5_common_os.c
@@ -11,7 +11,7 @@
 #include <rte_errno.h>
 
 #include "mlx5_devx_cmds.h"
-#include "mlx5_common_utils.h"
+#include "../mlx5_common_log.h"
 #include "mlx5_common.h"
 #include "mlx5_common_os.h"
 #include "mlx5_malloc.h"
diff --git a/drivers/common/mlx5/windows/mlx5_glue.c b/drivers/common/mlx5/windows/mlx5_glue.c
index aef6d3b5f4..535487a8d4 100644
--- a/drivers/common/mlx5/windows/mlx5_glue.c
+++ b/drivers/common/mlx5/windows/mlx5_glue.c
@@ -12,7 +12,7 @@
 #include <rte_malloc.h>
 
 #include "mlx5_glue.h"
-#include "mlx5_common_utils.h"
+#include "../mlx5_common_log.h"
 #include "mlx5_win_ext.h"
 
 /*
diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index a39b5edddc..18fe23e4fb 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -3,220 +3,11 @@
  */
 
 #include <rte_malloc.h>
-#include <rte_hash_crc.h>
 
 #include <mlx5_malloc.h>
 
 #include "mlx5_utils.h"
 
-/********************* Hash List **********************/
-
-static struct mlx5_hlist_entry *
-mlx5_hlist_default_create_cb(struct mlx5_hlist *h, uint64_t key __rte_unused,
-			     void *ctx __rte_unused)
-{
-	return mlx5_malloc(MLX5_MEM_ZERO, h->entry_sz, 0, SOCKET_ID_ANY);
-}
-
-static void
-mlx5_hlist_default_remove_cb(struct mlx5_hlist *h __rte_unused,
-			     struct mlx5_hlist_entry *entry)
-{
-	mlx5_free(entry);
-}
-
-struct mlx5_hlist *
-mlx5_hlist_create(const char *name, uint32_t size, uint32_t entry_size,
-		  uint32_t flags, mlx5_hlist_create_cb cb_create,
-		  mlx5_hlist_match_cb cb_match, mlx5_hlist_remove_cb cb_remove)
-{
-	struct mlx5_hlist *h;
-	uint32_t act_size;
-	uint32_t alloc_size;
-	uint32_t i;
-
-	if (!size || !cb_match || (!cb_create ^ !cb_remove))
-		return NULL;
-	/* Align to the next power of 2, 32bits integer is enough now. */
-	if (!rte_is_power_of_2(size)) {
-		act_size = rte_align32pow2(size);
-		DRV_LOG(DEBUG, "Size 0x%" PRIX32 " is not power of 2, "
-			"will be aligned to 0x%" PRIX32 ".", size, act_size);
-	} else {
-		act_size = size;
-	}
-	alloc_size = sizeof(struct mlx5_hlist) +
-		     sizeof(struct mlx5_hlist_bucket) * act_size;
-	/* Using zmalloc, then no need to initialize the heads. */
-	h = mlx5_malloc(MLX5_MEM_ZERO, alloc_size, RTE_CACHE_LINE_SIZE,
-			SOCKET_ID_ANY);
-	if (!h) {
-		DRV_LOG(ERR, "No memory for hash list %s creation",
-			name ? name : "None");
-		return NULL;
-	}
-	if (name)
-		snprintf(h->name, MLX5_HLIST_NAMESIZE, "%s", name);
-	h->table_sz = act_size;
-	h->mask = act_size - 1;
-	h->entry_sz = entry_size;
-	h->direct_key = !!(flags & MLX5_HLIST_DIRECT_KEY);
-	h->write_most = !!(flags & MLX5_HLIST_WRITE_MOST);
-	h->cb_create = cb_create ? cb_create : mlx5_hlist_default_create_cb;
-	h->cb_match = cb_match;
-	h->cb_remove = cb_remove ? cb_remove : mlx5_hlist_default_remove_cb;
-	for (i = 0; i < act_size; i++)
-		rte_rwlock_init(&h->buckets[i].lock);
-	DRV_LOG(DEBUG, "Hash list with %s size 0x%" PRIX32 " is created.",
-		h->name, act_size);
-	return h;
-}
-
-static struct mlx5_hlist_entry *
-__hlist_lookup(struct mlx5_hlist *h, uint64_t key, uint32_t idx,
-	       void *ctx, bool reuse)
-{
-	struct mlx5_hlist_head *first;
-	struct mlx5_hlist_entry *node;
-
-	MLX5_ASSERT(h);
-	first = &h->buckets[idx].head;
-	LIST_FOREACH(node, first, next) {
-		if (!h->cb_match(h, node, key, ctx)) {
-			if (reuse) {
-				__atomic_add_fetch(&node->ref_cnt, 1,
-						   __ATOMIC_RELAXED);
-				DRV_LOG(DEBUG, "Hash list %s entry %p "
-					"reuse: %u.",
-					h->name, (void *)node, node->ref_cnt);
-			}
-			break;
-		}
-	}
-	return node;
-}
-
-static struct mlx5_hlist_entry *
-hlist_lookup(struct mlx5_hlist *h, uint64_t key, uint32_t idx,
-	     void *ctx, bool reuse)
-{
-	struct mlx5_hlist_entry *node;
-
-	MLX5_ASSERT(h);
-	rte_rwlock_read_lock(&h->buckets[idx].lock);
-	node = __hlist_lookup(h, key, idx, ctx, reuse);
-	rte_rwlock_read_unlock(&h->buckets[idx].lock);
-	return node;
-}
-
-struct mlx5_hlist_entry *
-mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
-{
-	uint32_t idx;
-
-	if (h->direct_key)
-		idx = (uint32_t)(key & h->mask);
-	else
-		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	return hlist_lookup(h, key, idx, ctx, false);
-}
-
-struct mlx5_hlist_entry*
-mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
-{
-	uint32_t idx;
-	struct mlx5_hlist_head *first;
-	struct mlx5_hlist_bucket *b;
-	struct mlx5_hlist_entry *entry;
-	uint32_t prev_gen_cnt = 0;
-
-	if (h->direct_key)
-		idx = (uint32_t)(key & h->mask);
-	else
-		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	MLX5_ASSERT(h);
-	b = &h->buckets[idx];
-	/* Use write lock directly for write-most list. */
-	if (!h->write_most) {
-		prev_gen_cnt = __atomic_load_n(&b->gen_cnt, __ATOMIC_ACQUIRE);
-		entry = hlist_lookup(h, key, idx, ctx, true);
-		if (entry)
-			return entry;
-	}
-	rte_rwlock_write_lock(&b->lock);
-	/* Check if the list changed by other threads. */
-	if (h->write_most ||
-	    prev_gen_cnt != __atomic_load_n(&b->gen_cnt, __ATOMIC_ACQUIRE)) {
-		entry = __hlist_lookup(h, key, idx, ctx, true);
-		if (entry)
-			goto done;
-	}
-	first = &b->head;
-	entry = h->cb_create(h, key, ctx);
-	if (!entry) {
-		rte_errno = ENOMEM;
-		DRV_LOG(DEBUG, "Can't allocate hash list %s entry.", h->name);
-		goto done;
-	}
-	entry->idx = idx;
-	entry->ref_cnt = 1;
-	LIST_INSERT_HEAD(first, entry, next);
-	__atomic_add_fetch(&b->gen_cnt, 1, __ATOMIC_ACQ_REL);
-	DRV_LOG(DEBUG, "Hash list %s entry %p new: %u.",
-		h->name, (void *)entry, entry->ref_cnt);
-done:
-	rte_rwlock_write_unlock(&b->lock);
-	return entry;
-}
-
-int
-mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry)
-{
-	uint32_t idx = entry->idx;
-
-	rte_rwlock_write_lock(&h->buckets[idx].lock);
-	MLX5_ASSERT(entry && entry->ref_cnt && entry->next.le_prev);
-	DRV_LOG(DEBUG, "Hash list %s entry %p deref: %u.",
-		h->name, (void *)entry, entry->ref_cnt);
-	if (--entry->ref_cnt) {
-		rte_rwlock_write_unlock(&h->buckets[idx].lock);
-		return 1;
-	}
-	LIST_REMOVE(entry, next);
-	/* Set to NULL to get rid of removing action for more than once. */
-	entry->next.le_prev = NULL;
-	h->cb_remove(h, entry);
-	rte_rwlock_write_unlock(&h->buckets[idx].lock);
-	DRV_LOG(DEBUG, "Hash list %s entry %p removed.",
-		h->name, (void *)entry);
-	return 0;
-}
-
-void
-mlx5_hlist_destroy(struct mlx5_hlist *h)
-{
-	uint32_t idx;
-	struct mlx5_hlist_entry *entry;
-
-	MLX5_ASSERT(h);
-	for (idx = 0; idx < h->table_sz; ++idx) {
-		/* No LIST_FOREACH_SAFE, using while instead. */
-		while (!LIST_EMPTY(&h->buckets[idx].head)) {
-			entry = LIST_FIRST(&h->buckets[idx].head);
-			LIST_REMOVE(entry, next);
-			/*
-			 * The owner of whole element which contains data entry
-			 * is the user, so it's the user's duty to do the clean
-			 * up and the free work because someone may not put the
-			 * hlist entry at the beginning(suggested to locate at
-			 * the beginning). Or else the default free function
-			 * will be used.
-			 */
-			h->cb_remove(h, entry);
-		}
-	}
-	mlx5_free(h);
-}
 
 /********************* Cache list ************************/
 
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index 5088c95e86..008a017402 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -18,6 +18,7 @@
 #include <rte_bitmap.h>
 
 #include <mlx5_common.h>
+#include <mlx5_common_utils.h>
 
 #include "mlx5_defs.h"
 
@@ -261,199 +262,6 @@ log2above(unsigned int v)
 	return l + r;
 }
 
-#define MLX5_HLIST_DIRECT_KEY 0x0001 /* Use the key directly as hash index. */
-#define MLX5_HLIST_WRITE_MOST 0x0002 /* List mostly used for append new. */
-
-/** Maximum size of string for naming the hlist table. */
-#define MLX5_HLIST_NAMESIZE			32
-
-struct mlx5_hlist;
-
-/**
- * Structure of the entry in the hash list, user should define its own struct
- * that contains this in order to store the data. The 'key' is 64-bits right
- * now and its user's responsibility to guarantee there is no collision.
- */
-struct mlx5_hlist_entry {
-	LIST_ENTRY(mlx5_hlist_entry) next; /* entry pointers in the list. */
-	uint32_t idx; /* Bucket index the entry belongs to. */
-	uint32_t ref_cnt; /* Reference count. */
-};
-
-/** Structure for hash head. */
-LIST_HEAD(mlx5_hlist_head, mlx5_hlist_entry);
-
-/**
- * Type of callback function for entry removal.
- *
- * @param list
- *   The hash list.
- * @param entry
- *   The entry in the list.
- */
-typedef void (*mlx5_hlist_remove_cb)(struct mlx5_hlist *list,
-				     struct mlx5_hlist_entry *entry);
-
-/**
- * Type of function for user defined matching.
- *
- * @param list
- *   The hash list.
- * @param entry
- *   The entry in the list.
- * @param key
- *   The new entry key.
- * @param ctx
- *   The pointer to new entry context.
- *
- * @return
- *   0 if matching, non-zero number otherwise.
- */
-typedef int (*mlx5_hlist_match_cb)(struct mlx5_hlist *list,
-				   struct mlx5_hlist_entry *entry,
-				   uint64_t key, void *ctx);
-
-/**
- * Type of function for user defined hash list entry creation.
- *
- * @param list
- *   The hash list.
- * @param key
- *   The key of the new entry.
- * @param ctx
- *   The pointer to new entry context.
- *
- * @return
- *   Pointer to allocated entry on success, NULL otherwise.
- */
-typedef struct mlx5_hlist_entry *(*mlx5_hlist_create_cb)
-				  (struct mlx5_hlist *list,
-				   uint64_t key, void *ctx);
-
-/* Hash list bucket head. */
-struct mlx5_hlist_bucket {
-	struct mlx5_hlist_head head; /* List head. */
-	rte_rwlock_t lock; /* Bucket lock. */
-	uint32_t gen_cnt; /* List modification will update generation count. */
-} __rte_cache_aligned;
-
-/**
- * Hash list table structure
- *
- * Entry in hash list could be reused if entry already exists, reference
- * count will increase and the existing entry returns.
- *
- * When destroy an entry from list, decrease reference count and only
- * destroy when no further reference.
- */
-struct mlx5_hlist {
-	char name[MLX5_HLIST_NAMESIZE]; /**< Name of the hash list. */
-	/**< number of heads, need to be power of 2. */
-	uint32_t table_sz;
-	uint32_t entry_sz; /**< Size of entry, used to allocate entry. */
-	/**< mask to get the index of the list heads. */
-	uint32_t mask;
-	bool direct_key; /* Use the new entry key directly as hash index. */
-	bool write_most; /* List mostly used for append new or destroy. */
-	void *ctx;
-	mlx5_hlist_create_cb cb_create; /**< entry create callback. */
-	mlx5_hlist_match_cb cb_match; /**< entry match callback. */
-	mlx5_hlist_remove_cb cb_remove; /**< entry remove callback. */
-	struct mlx5_hlist_bucket buckets[] __rte_cache_aligned;
-	/**< list bucket arrays. */
-};
-
-/**
- * Create a hash list table, the user can specify the list heads array size
- * of the table, now the size should be a power of 2 in order to get better
- * distribution for the entries. Each entry is a part of the whole data element
- * and the caller should be responsible for the data element's allocation and
- * cleanup / free. Key of each entry will be calculated with CRC in order to
- * generate a little fairer distribution.
- *
- * @param name
- *   Name of the hash list(optional).
- * @param size
- *   Heads array size of the hash list.
- * @param entry_size
- *   Entry size to allocate if cb_create not specified.
- * @param flags
- *   The hash list attribute flags.
- * @param cb_create
- *   Callback function for entry create.
- * @param cb_match
- *   Callback function for entry match.
- * @param cb_destroy
- *   Callback function for entry destroy.
- * @return
- *   Pointer of the hash list table created, NULL on failure.
- */
-struct mlx5_hlist *mlx5_hlist_create(const char *name, uint32_t size,
-				     uint32_t entry_size, uint32_t flags,
-				     mlx5_hlist_create_cb cb_create,
-				     mlx5_hlist_match_cb cb_match,
-				     mlx5_hlist_remove_cb cb_destroy);
-
-/**
- * Search an entry matching the key.
- *
- * Result returned might be destroyed by other thread, must use
- * this function only in main thread.
- *
- * @param h
- *   Pointer to the hast list table.
- * @param key
- *   Key for the searching entry.
- * @param ctx
- *   Common context parameter used by entry callback function.
- *
- * @return
- *   Pointer of the hlist entry if found, NULL otherwise.
- */
-struct mlx5_hlist_entry *mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key,
-					   void *ctx);
-
-/**
- * Insert an entry to the hash list table, the entry is only part of whole data
- * element and a 64B key is used for matching. User should construct the key or
- * give a calculated hash signature and guarantee there is no collision.
- *
- * @param h
- *   Pointer to the hast list table.
- * @param entry
- *   Entry to be inserted into the hash list table.
- * @param ctx
- *   Common context parameter used by callback function.
- *
- * @return
- *   registered entry on success, NULL otherwise
- */
-struct mlx5_hlist_entry *mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key,
-					     void *ctx);
-
-/**
- * Remove an entry from the hash list table. User should guarantee the validity
- * of the entry.
- *
- * @param h
- *   Pointer to the hast list table. (not used)
- * @param entry
- *   Entry to be removed from the hash list table.
- * @return
- *   0 on entry removed, 1 on entry still referenced.
- */
-int mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry);
-
-/**
- * Destroy the hash list table, all the entries already inserted into the lists
- * will be handled by the callback function provided by the user (including
- * free if needed) before the table is freed.
- *
- * @param h
- *   Pointer to the hast list table.
- */
-void mlx5_hlist_destroy(struct mlx5_hlist *h);
-
 /************************ cache list *****************************/
 
 /** Maximum size of string for naming. */
-- 
2.21.0

[dpdk-dev] [PATCH 12/24] common/mlx5: share get ib device match function

[Shiri Kuzin]

The get_ib_device_match function iterates over the list of ib devices
returned by the get_device_list glue function and returns the ib device
matching the provided address.

Since this function is in use by several drivers, in this patch we
share the function in common part.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/linux/mlx5_common_os.c | 28 +++++++++++++++++++
 drivers/common/mlx5/linux/mlx5_common_os.h |  5 ++++
 drivers/common/mlx5/mlx5_common.h          |  1 +
 drivers/common/mlx5/version.map            |  2 ++
 drivers/compress/mlx5/mlx5_compress.c      | 30 +-------------------
 drivers/regex/mlx5/mlx5_regex.c            | 30 ++------------------
 drivers/vdpa/mlx5/mlx5_vdpa.c              | 32 ++--------------------
 7 files changed, 41 insertions(+), 87 deletions(-)

diff --git a/drivers/common/mlx5/linux/mlx5_common_os.c b/drivers/common/mlx5/linux/mlx5_common_os.c
index fba8245b8b..037147fe31 100644
--- a/drivers/common/mlx5/linux/mlx5_common_os.c
+++ b/drivers/common/mlx5/linux/mlx5_common_os.c
@@ -16,6 +16,7 @@
 
 #include "mlx5_common.h"
 #include "mlx5_common_log.h"
+#include "mlx5_common_os.h"
 #include "mlx5_glue.h"
 
 #ifdef MLX5_GLUE
@@ -423,3 +424,30 @@ mlx5_glue_constructor(void)
 	mlx5_glue = NULL;
 }
 
+struct ibv_device *
+mlx5_os_get_ib_device_match(struct rte_pci_addr *addr)
+{
+	int n;
+	struct ibv_device **ibv_list = mlx5_glue->get_device_list(&n);
+	struct ibv_device *ibv_match = NULL;
+
+	if (ibv_list == NULL) {
+		rte_errno = ENOSYS;
+		return NULL;
+	}
+	while (n-- > 0) {
+		struct rte_pci_addr paddr;
+
+		DRV_LOG(DEBUG, "Checking device \"%s\"..", ibv_list[n]->name);
+		if (mlx5_dev_to_pci_addr(ibv_list[n]->ibdev_path, &paddr) != 0)
+			continue;
+		if (rte_pci_addr_cmp(addr, &paddr) != 0)
+			continue;
+		ibv_match = ibv_list[n];
+		break;
+	}
+	if (ibv_match == NULL)
+		rte_errno = ENOENT;
+	mlx5_glue->free_device_list(ibv_list);
+	return ibv_match;
+}
diff --git a/drivers/common/mlx5/linux/mlx5_common_os.h b/drivers/common/mlx5/linux/mlx5_common_os.h
index 63f070d9c4..0eb304e0e3 100644
--- a/drivers/common/mlx5/linux/mlx5_common_os.h
+++ b/drivers/common/mlx5/linux/mlx5_common_os.h
@@ -284,4 +284,9 @@ mlx5_os_free(void *addr)
 {
 	free(addr);
 }
+
+__rte_internal
+struct ibv_device *
+mlx5_os_get_ib_device_match(struct rte_pci_addr *addr);
+
 #endif /* RTE_PMD_MLX5_COMMON_OS_H_ */
diff --git a/drivers/common/mlx5/mlx5_common.h b/drivers/common/mlx5/mlx5_common.h
index 327374fdcf..f1ff16eb3e 100644
--- a/drivers/common/mlx5/mlx5_common.h
+++ b/drivers/common/mlx5/mlx5_common.h
@@ -17,6 +17,7 @@
 
 #include "mlx5_prm.h"
 #include "mlx5_devx_cmds.h"
+#include "mlx5_common_os.h"
 
 /* Reported driver name. */
 #define MLX5_PCI_DRIVER_NAME "mlx5_pci"
diff --git a/drivers/common/mlx5/version.map b/drivers/common/mlx5/version.map
index 5c2cc4dca9..045c0fc6fe 100644
--- a/drivers/common/mlx5/version.map
+++ b/drivers/common/mlx5/version.map
@@ -99,6 +99,8 @@ INTERNAL {
 	mlx5_nl_vlan_vmwa_create;
 	mlx5_nl_vlan_vmwa_delete;
 
+	mlx5_os_get_ib_device_match;
+
 	mlx5_translate_port_name;
 
 	mlx5_malloc_mem_select;
diff --git a/drivers/compress/mlx5/mlx5_compress.c b/drivers/compress/mlx5/mlx5_compress.c
index ec3c237512..ff988d26ef 100644
--- a/drivers/compress/mlx5/mlx5_compress.c
+++ b/drivers/compress/mlx5/mlx5_compress.c
@@ -647,34 +647,6 @@ mlx5_compress_dequeue_burst(void *queue_pair, struct rte_comp_op **ops,
 	return i;
 }
 
-static struct ibv_device *
-mlx5_compress_get_ib_device_match(struct rte_pci_addr *addr)
-{
-	int n;
-	struct ibv_device **ibv_list = mlx5_glue->get_device_list(&n);
-	struct ibv_device *ibv_match = NULL;
-
-	if (ibv_list == NULL) {
-		rte_errno = ENOSYS;
-		return NULL;
-	}
-	while (n-- > 0) {
-		struct rte_pci_addr paddr;
-
-		DRV_LOG(DEBUG, "Checking device \"%s\"..", ibv_list[n]->name);
-		if (mlx5_dev_to_pci_addr(ibv_list[n]->ibdev_path, &paddr) != 0)
-			continue;
-		if (rte_pci_addr_cmp(addr, &paddr) != 0)
-			continue;
-		ibv_match = ibv_list[n];
-		break;
-	}
-	if (ibv_match == NULL)
-		rte_errno = ENOENT;
-	mlx5_glue->free_device_list(ibv_list);
-	return ibv_match;
-}
-
 static void
 mlx5_compress_hw_global_release(struct mlx5_compress_priv *priv)
 {
@@ -774,7 +746,7 @@ mlx5_compress_pci_probe(struct rte_pci_driver *pci_drv,
 		rte_errno = ENOTSUP;
 		return -rte_errno;
 	}
-	ibv = mlx5_compress_get_ib_device_match(&pci_dev->addr);
+	ibv = mlx5_os_get_ib_device_match(&pci_dev->addr);
 	if (ibv == NULL) {
 		DRV_LOG(ERR, "No matching IB device for PCI slot "
 			PCI_PRI_FMT ".", pci_dev->addr.domain,
diff --git a/drivers/regex/mlx5/mlx5_regex.c b/drivers/regex/mlx5/mlx5_regex.c
index ac5b205fa9..88221186e8 100644
--- a/drivers/regex/mlx5/mlx5_regex.c
+++ b/drivers/regex/mlx5/mlx5_regex.c
@@ -11,6 +11,7 @@
 #include <rte_regexdev_driver.h>
 
 #include <mlx5_common_pci.h>
+#include <mlx5_common.h>
 #include <mlx5_glue.h>
 #include <mlx5_devx_cmds.h>
 #include <mlx5_prm.h>
@@ -52,33 +53,6 @@ mlx5_regex_close(struct rte_regexdev *dev __rte_unused)
 	return 0;
 }
 
-static struct ibv_device *
-mlx5_regex_get_ib_device_match(struct rte_pci_addr *addr)
-{
-	int n;
-	struct ibv_device **ibv_list = mlx5_glue->get_device_list(&n);
-	struct ibv_device *ibv_match = NULL;
-
-	if (!ibv_list) {
-		rte_errno = ENOSYS;
-		return NULL;
-	}
-	while (n-- > 0) {
-		struct rte_pci_addr pci_addr;
-
-		DRV_LOG(DEBUG, "Checking device \"%s\"..", ibv_list[n]->name);
-		if (mlx5_dev_to_pci_addr(ibv_list[n]->ibdev_path, &pci_addr))
-			continue;
-		if (rte_pci_addr_cmp(addr, &pci_addr))
-			continue;
-		ibv_match = ibv_list[n];
-		break;
-	}
-	if (!ibv_match)
-		rte_errno = ENOENT;
-	mlx5_glue->free_device_list(ibv_list);
-	return ibv_match;
-}
 static int
 mlx5_regex_engines_status(struct ibv_context *ctx, int num_engines)
 {
@@ -121,7 +95,7 @@ mlx5_regex_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 	int ret;
 	uint32_t val;
 
-	ibv = mlx5_regex_get_ib_device_match(&pci_dev->addr);
+	ibv = mlx5_os_get_ib_device_match(&pci_dev->addr);
 	if (!ibv) {
 		DRV_LOG(ERR, "No matching IB device for PCI slot "
 			PCI_PRI_FMT ".", pci_dev->addr.domain,
diff --git a/drivers/vdpa/mlx5/mlx5_vdpa.c b/drivers/vdpa/mlx5/mlx5_vdpa.c
index 619e31d61c..9e5857205d 100644
--- a/drivers/vdpa/mlx5/mlx5_vdpa.c
+++ b/drivers/vdpa/mlx5/mlx5_vdpa.c
@@ -472,34 +472,6 @@ static struct rte_vdpa_dev_ops mlx5_vdpa_ops = {
 	.reset_stats = mlx5_vdpa_reset_stats,
 };
 
-static struct ibv_device *
-mlx5_vdpa_get_ib_device_match(struct rte_pci_addr *addr)
-{
-	int n;
-	struct ibv_device **ibv_list = mlx5_glue->get_device_list(&n);
-	struct ibv_device *ibv_match = NULL;
-
-	if (!ibv_list) {
-		rte_errno = ENOSYS;
-		return NULL;
-	}
-	while (n-- > 0) {
-		struct rte_pci_addr pci_addr;
-
-		DRV_LOG(DEBUG, "Checking device \"%s\"..", ibv_list[n]->name);
-		if (mlx5_dev_to_pci_addr(ibv_list[n]->ibdev_path, &pci_addr))
-			continue;
-		if (rte_pci_addr_cmp(addr, &pci_addr))
-			continue;
-		ibv_match = ibv_list[n];
-		break;
-	}
-	if (!ibv_match)
-		rte_errno = ENOENT;
-	mlx5_glue->free_device_list(ibv_list);
-	return ibv_match;
-}
-
 /* Try to disable ROCE by Netlink\Devlink. */
 static int
 mlx5_vdpa_nl_roce_disable(const char *addr)
@@ -595,7 +567,7 @@ mlx5_vdpa_roce_disable(struct rte_pci_addr *addr, struct ibv_device **ibv)
 		struct ibv_device *ibv_new;
 
 		for (r = MLX5_VDPA_MAX_RETRIES; r; r--) {
-			ibv_new = mlx5_vdpa_get_ib_device_match(addr);
+			ibv_new = mlx5_os_get_ib_device_match(addr);
 			if (ibv_new) {
 				*ibv = ibv_new;
 				return 0;
@@ -698,7 +670,7 @@ mlx5_vdpa_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 	struct mlx5_hca_attr attr;
 	int ret;
 
-	ibv = mlx5_vdpa_get_ib_device_match(&pci_dev->addr);
+	ibv = mlx5_os_get_ib_device_match(&pci_dev->addr);
 	if (!ibv) {
 		DRV_LOG(ERR, "No matching IB device for PCI slot "
 			PCI_PRI_FMT ".", pci_dev->addr.domain,
-- 
2.21.0

[dpdk-dev] [PATCH 13/24] common/mlx5: support general obj CREDENTIAL create

[Shiri Kuzin]

From: Dekel Peled <dekelp@nvidia.com>

CREDENTIAL object is used for any crypto operation in wrapped mode.

This patch add support of CREDENTIAL object create operation.
Add reading of CREDENTIAL support capability.
Add function to create general object type CREDENTIAL, using DevX API.

Signed-off-by: Dekel Peled <dekelp@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_devx_cmds.c | 53 +++++++++++++++++++++++++++-
 drivers/common/mlx5/mlx5_devx_cmds.h | 17 +++++++--
 drivers/common/mlx5/mlx5_prm.h       | 23 ++++++++++++
 drivers/common/mlx5/version.map      |  1 +
 4 files changed, 91 insertions(+), 3 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c
index 1b1227eb16..c43e320f87 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.c
+++ b/drivers/common/mlx5/mlx5_devx_cmds.c
@@ -734,6 +734,8 @@ mlx5_devx_cmd_query_hca_attr(void *ctx,
 		       MLX5_GENERAL_OBJ_TYPES_CAP_DEK);
 	attr->import_kek = !!(general_obj_types_supported &
 			      MLX5_GENERAL_OBJ_TYPES_CAP_IMPORT_KEK);
+	attr->credential = !!(general_obj_types_supported &
+			      MLX5_GENERAL_OBJ_TYPES_CAP_CREDENTIAL);
 	attr->crypto_login = !!(general_obj_types_supported &
 				MLX5_GENERAL_OBJ_TYPES_CAP_CRYPTO_LOGIN);
 	/* Add reading of other GENERAL_OBJ_TYPES_CAP bits above this line. */
@@ -2407,6 +2409,55 @@ mlx5_devx_cmd_create_import_kek_obj(void *ctx,
 	return import_kek_obj;
 }
 
+/**
+ * Create general object of type CREDENTIAL using DevX API.
+ *
+ * @param[in] ctx
+ *   Context returned from mlx5 open_device() glue function.
+ * @param [in] attr
+ *   Pointer to CREDENTIAL attributes structure.
+ *
+ * @return
+ *   The DevX object created, NULL otherwise and rte_errno is set.
+ */
+struct mlx5_devx_obj *
+mlx5_devx_cmd_create_credential_obj(void *ctx,
+				    struct mlx5_devx_credential_attr *attr)
+{
+	uint32_t in[MLX5_ST_SZ_DW(create_credential_in)] = {0};
+	uint32_t out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};
+	struct mlx5_devx_obj *credential_obj = NULL;
+	void *ptr = NULL, *credential_addr = NULL;
+
+	credential_obj = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*credential_obj),
+				     0, SOCKET_ID_ANY);
+	if (credential_obj == NULL) {
+		DRV_LOG(ERR, "Failed to allocate CREDENTIAL object data");
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+	ptr = MLX5_ADDR_OF(create_credential_in, in, hdr);
+	MLX5_SET(general_obj_in_cmd_hdr, ptr, opcode,
+		 MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
+	MLX5_SET(general_obj_in_cmd_hdr, ptr, obj_type,
+		 MLX5_GENERAL_OBJ_TYPE_CREDENTIAL);
+	ptr = MLX5_ADDR_OF(create_credential_in, in, credential);
+	MLX5_SET(credential, ptr, credential_role, attr->credential_role);
+	credential_addr = MLX5_ADDR_OF(credential, ptr, credential);
+	memcpy(credential_addr, (void *)(attr->credential),
+	       MLX5_CRYPTO_CREDENTIAL_SIZE);
+	credential_obj->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in),
+							 out, sizeof(out));
+	if (credential_obj->obj == NULL) {
+		rte_errno = errno;
+		DRV_LOG(ERR, "Failed to create CREDENTIAL object using DevX.");
+		mlx5_free(credential_obj);
+		return NULL;
+	}
+	credential_obj->id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
+	return credential_obj;
+}
+
 /**
  * Create general object of type CRYPTO_LOGIN using DevX API.
  *
@@ -2446,7 +2497,7 @@ mlx5_devx_cmd_create_crypto_login_obj(void *ctx,
 		 attr->session_import_kek_ptr);
 	credential_addr = MLX5_ADDR_OF(crypto_login, ptr, credential);
 	memcpy(credential_addr, (void *)(attr->credential),
-	       MLX5_CRYPTO_LOGIN_CREDENTIAL_SIZE);
+	       MLX5_CRYPTO_CREDENTIAL_SIZE);
 	crypto_login_obj->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in),
 							   out, sizeof(out));
 	if (crypto_login_obj->obj == NULL) {
diff --git a/drivers/common/mlx5/mlx5_devx_cmds.h b/drivers/common/mlx5/mlx5_devx_cmds.h
index b723aee126..1c952495bf 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.h
+++ b/drivers/common/mlx5/mlx5_devx_cmds.h
@@ -133,6 +133,7 @@ struct mlx5_hca_attr {
 	uint32_t aes_xts:1; /* AES-XTS crypto is supported. */
 	uint32_t dek:1; /* General obj type DEK is supported. */
 	uint32_t import_kek:1; /* General obj type IMPORT_KEK supported. */
+	uint32_t credential:1; /* General obj type CREDENTIAL supported. */
 	uint32_t crypto_login:1; /* General obj type CRYPTO_LOGIN supported. */
 	uint32_t regexp_num_of_engines;
 	uint32_t log_max_ft_sampler_num:8;
@@ -447,13 +448,20 @@ struct mlx5_devx_import_kek_attr {
 	uint8_t key[MLX5_CRYPTO_KEY_MAX_SIZE];
 };
 
-#define MLX5_CRYPTO_LOGIN_CREDENTIAL_SIZE	48
+#define MLX5_CRYPTO_CREDENTIAL_SIZE	48
+
+struct mlx5_devx_credential_attr {
+	uint64_t modify_field_select;
+	uint32_t state:8;
+	uint32_t credential_role:8;
+	uint8_t credential[MLX5_CRYPTO_CREDENTIAL_SIZE];
+};
 
 struct mlx5_devx_crypto_login_attr {
 	uint64_t modify_field_select;
 	uint32_t credential_pointer:24;
 	uint32_t session_import_kek_ptr:24;
-	uint8_t credential[MLX5_CRYPTO_LOGIN_CREDENTIAL_SIZE];
+	uint8_t credential[MLX5_CRYPTO_CREDENTIAL_SIZE];
 };
 
 /* mlx5_devx_cmds.c */
@@ -599,6 +607,11 @@ struct mlx5_devx_obj *
 mlx5_devx_cmd_create_import_kek_obj(void *ctx,
 				    struct mlx5_devx_import_kek_attr *attr);
 
+__rte_internal
+struct mlx5_devx_obj *
+mlx5_devx_cmd_create_credential_obj(void *ctx,
+				    struct mlx5_devx_credential_attr *attr);
+
 __rte_internal
 struct mlx5_devx_obj *
 mlx5_devx_cmd_create_crypto_login_obj(void *ctx,
diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index 736badb6df..69d6de8cbd 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -1185,6 +1185,8 @@ enum {
 			(1ULL << MLX5_GENERAL_OBJ_TYPE_DEK)
 #define MLX5_GENERAL_OBJ_TYPES_CAP_IMPORT_KEK \
 			(1ULL << MLX5_GENERAL_OBJ_TYPE_IMPORT_KEK)
+#define MLX5_GENERAL_OBJ_TYPES_CAP_CREDENTIAL \
+			(1ULL << MLX5_GENERAL_OBJ_TYPE_CREDENTIAL)
 #define MLX5_GENERAL_OBJ_TYPES_CAP_CRYPTO_LOGIN \
 			(1ULL << MLX5_GENERAL_OBJ_TYPE_CRYPTO_LOGIN)
 
@@ -2471,6 +2473,7 @@ enum {
 	MLX5_GENERAL_OBJ_TYPE_VIRTQ = 0x000d,
 	MLX5_GENERAL_OBJ_TYPE_VIRTIO_Q_COUNTERS = 0x001c,
 	MLX5_GENERAL_OBJ_TYPE_IMPORT_KEK = 0x001d,
+	MLX5_GENERAL_OBJ_TYPE_CREDENTIAL = 0x001e,
 	MLX5_GENERAL_OBJ_TYPE_CRYPTO_LOGIN = 0x001f,
 	MLX5_GENERAL_OBJ_TYPE_FLEX_PARSE_GRAPH = 0x0022,
 	MLX5_GENERAL_OBJ_TYPE_FLOW_HIT_ASO = 0x0025,
@@ -2581,6 +2584,26 @@ struct mlx5_ifc_create_import_kek_in_bits {
 	struct mlx5_ifc_import_kek_bits import_kek;
 };
 
+enum {
+	MLX5_CREDENTIAL_ROLE_OFFICER = 0x0,
+	MLX5_CREDENTIAL_ROLE_USER = 0x1,
+};
+
+struct mlx5_ifc_credential_bits {
+	u8 modify_field_select[0x40];
+	u8 state[0x8];
+	u8 reserved_at_48[0x10];
+	u8 credential_role[0x8];
+	u8 reserved_at_60[0x1a0];
+	u8 credential[0x180];
+	u8 reserved_at_380[0x480];
+};
+
+struct mlx5_ifc_create_credential_in_bits {
+	struct mlx5_ifc_general_obj_in_cmd_hdr_bits hdr;
+	struct mlx5_ifc_credential_bits credential;
+};
+
 struct mlx5_ifc_crypto_login_bits {
 	u8 modify_field_select[0x40];
 	u8 reserved_at_40[0x48];
diff --git a/drivers/common/mlx5/version.map b/drivers/common/mlx5/version.map
index 045c0fc6fe..e1f0b324a0 100644
--- a/drivers/common/mlx5/version.map
+++ b/drivers/common/mlx5/version.map
@@ -26,6 +26,7 @@ INTERNAL {
 	mlx5_devx_cmd_create_geneve_tlv_option;
         mlx5_devx_cmd_create_dek_obj;
         mlx5_devx_cmd_create_import_kek_obj;
+        mlx5_devx_cmd_create_credential_obj;
         mlx5_devx_cmd_create_crypto_login_obj;
 	mlx5_devx_cmd_destroy;
 	mlx5_devx_cmd_flow_counter_alloc;
-- 
2.21.0

[dpdk-dev] [PATCH 14/24] common/mlx5: add crypto register structs and defs

[Shiri Kuzin]

From: Dekel Peled <dekelp@nvidia.com>

Encryption key management requires use of several related registers.
This patch adds the relevant structs and values, according to PRM
definitions.

Signed-off-by: Dekel Peled <dekelp@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_prm.h | 41 ++++++++++++++++++++++++++++++++++
 1 file changed, 41 insertions(+)

diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index 69d6de8cbd..4518304889 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -3232,6 +3232,10 @@ enum {
 
 enum {
 	MLX5_REGISTER_ID_MTUTC  = 0x9055,
+	MLX5_CRYPTO_OPERATIONAL_REGISTER_ID = 0xC002,
+	MLX5_CRYPTO_COMMISSIONING_REGISTER_ID = 0xC003,
+	MLX5_IMPORT_KEK_HANDLE_REGISTER_ID = 0xC004,
+	MLX5_CREDENTIAL_HANDLE_REGISTER_ID = 0xC005,
 };
 
 struct mlx5_ifc_register_mtutc_bits {
@@ -3249,6 +3253,43 @@ struct mlx5_ifc_register_mtutc_bits {
 #define MLX5_MTUTC_TIMESTAMP_MODE_INTERNAL_TIMER 0
 #define MLX5_MTUTC_TIMESTAMP_MODE_REAL_TIME 1
 
+struct mlx5_ifc_crypto_operational_register_bits {
+	u8 wrapped_crypto_operational[0x1];
+	u8 reserved_at_1[0x1b];
+	u8 kek_size[0x4];
+	u8 reserved_at_20[0x20];
+	u8 credential[0x140];
+	u8 kek[0x100];
+	u8 reserved_at_280[0x180];
+};
+
+struct mlx5_ifc_crypto_commissioning_register_bits {
+	u8 token[0x1]; /* TODO: add size after PRM update */
+};
+
+struct mlx5_ifc_import_kek_handle_register_bits {
+	struct mlx5_ifc_crypto_login_bits crypto_login_object;
+	struct mlx5_ifc_import_kek_bits import_kek_object;
+	u8 reserved_at_200[0x4];
+	u8 write_operation[0x4];
+	u8 import_kek_id[0x18];
+	u8 reserved_at_220[0xe0];
+};
+
+struct mlx5_ifc_credential_handle_register_bits {
+	struct mlx5_ifc_crypto_login_bits crypto_login_object;
+	struct mlx5_ifc_credential_bits credential_object;
+	u8 reserved_at_200[0x4];
+	u8 write_operation[0x4];
+	u8 credential_id[0x18];
+	u8 reserved_at_220[0xe0];
+};
+
+enum {
+	MLX5_REGISTER_ADD_OPERATION = 0x1,
+	MLX5_REGISTER_DELETE_OPERATION = 0x2,
+};
+
 struct mlx5_ifc_parse_graph_arc_bits {
 	u8 start_inner_tunnel[0x1];
 	u8 reserved_at_1[0x7];
-- 
2.21.0

[dpdk-dev] [PATCH 15/24] common/mlx5: support register write access

[Shiri Kuzin]

From: Dekel Peled <dekelp@nvidia.com>

This patch adds support of write operation to NIC registers.

Signed-off-by: Dekel Peled <dekelp@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_devx_cmds.c | 67 +++++++++++++++++++++++++++-
 drivers/common/mlx5/mlx5_devx_cmds.h |  4 ++
 drivers/common/mlx5/version.map      | 20 +++++----
 3 files changed, 81 insertions(+), 10 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c
index c43e320f87..49042d99ee 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.c
+++ b/drivers/common/mlx5/mlx5_devx_cmds.c
@@ -12,7 +12,6 @@
 #include "mlx5_common_log.h"
 #include "mlx5_malloc.h"
 
-
 /**
  * Perform read access to the registers. Reads data from register
  * and writes ones to the specified buffer.
@@ -61,7 +60,7 @@ mlx5_devx_cmd_register_read(void *ctx, uint16_t reg_id, uint32_t arg,
 	if (status) {
 		int syndrome = MLX5_GET(access_register_out, out, syndrome);
 
-		DRV_LOG(DEBUG, "Failed to access NIC register 0x%X, "
+		DRV_LOG(DEBUG, "Failed to read access NIC register 0x%X, "
 			       "status %x, syndrome = %x",
 			       reg_id, status, syndrome);
 		return -1;
@@ -74,6 +73,70 @@ mlx5_devx_cmd_register_read(void *ctx, uint16_t reg_id, uint32_t arg,
 	return rc;
 }
 
+/**
+ * Perform write access to the registers.
+ *
+ * @param[in] ctx
+ *   Context returned from mlx5 open_device() glue function.
+ * @param[in] reg_id
+ *   Register identifier according to the PRM.
+ * @param[in] arg
+ *   Register access auxiliary parameter according to the PRM.
+ * @param[out] data
+ *   Pointer to the buffer containing data to write.
+ * @param[in] dw_cnt
+ *   Buffer size in double words (32bit units).
+ *
+ * @return
+ *   0 on success, a negative value otherwise.
+ */
+int
+mlx5_devx_cmd_register_write(void *ctx, uint16_t reg_id, uint32_t arg,
+			     uint32_t *data, uint32_t dw_cnt)
+{
+	uint32_t in[MLX5_ST_SZ_DW(access_register_in) +
+		    MLX5_ACCESS_REGISTER_DATA_DWORD_MAX] = {0};
+	uint32_t out[MLX5_ST_SZ_DW(access_register_out)] = {0};
+	int status, rc;
+	void *ptr;
+
+	MLX5_ASSERT(data && dw_cnt);
+	MLX5_ASSERT(dw_cnt <= MLX5_ACCESS_REGISTER_DATA_DWORD_MAX);
+	if (dw_cnt > MLX5_ACCESS_REGISTER_DATA_DWORD_MAX) {
+		DRV_LOG(ERR, "Data to write exceeds max size");
+		return -1;
+	}
+	MLX5_SET(access_register_in, in, opcode,
+		 MLX5_CMD_OP_ACCESS_REGISTER_USER);
+	MLX5_SET(access_register_in, in, op_mod,
+		 MLX5_ACCESS_REGISTER_IN_OP_MOD_WRITE);
+	MLX5_SET(access_register_in, in, register_id, reg_id);
+	MLX5_SET(access_register_in, in, argument, arg);
+	ptr = MLX5_ADDR_OF(access_register_in, in, register_data);
+	memcpy(ptr, data, dw_cnt * sizeof(uint32_t));
+	rc = mlx5_glue->devx_general_cmd(ctx, in, sizeof(in), out, sizeof(out));
+
+	rc = mlx5_glue->devx_general_cmd(ctx, in,
+					 MLX5_ST_SZ_BYTES(access_register_in) +
+					 dw_cnt * sizeof(uint32_t),
+					 out, sizeof(out));
+	if (rc)
+		goto error;
+	status = MLX5_GET(access_register_out, out, status);
+	if (status) {
+		int syndrome = MLX5_GET(access_register_out, out, syndrome);
+
+		DRV_LOG(DEBUG, "Failed to write access NIC register 0x%X, "
+			       "status %x, syndrome = %x",
+			       reg_id, status, syndrome);
+		return -1;
+	}
+	return 0;
+error:
+	rc = (rc > 0) ? -rc : rc;
+	return rc;
+}
+
 /**
  * Allocate flow counters via devx interface.
  *
diff --git a/drivers/common/mlx5/mlx5_devx_cmds.h b/drivers/common/mlx5/mlx5_devx_cmds.h
index 1c952495bf..6e961f77db 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.h
+++ b/drivers/common/mlx5/mlx5_devx_cmds.h
@@ -552,6 +552,10 @@ __rte_internal
 int mlx5_devx_cmd_register_read(void *ctx, uint16_t reg_id,
 				uint32_t arg, uint32_t *data, uint32_t dw_cnt);
 
+__rte_internal
+int mlx5_devx_cmd_register_write(void *ctx, uint16_t reg_id,
+				 uint32_t arg, uint32_t *data, uint32_t dw_cnt);
+
 __rte_internal
 struct mlx5_devx_obj *
 mlx5_devx_cmd_create_geneve_tlv_option(void *ctx,
diff --git a/drivers/common/mlx5/version.map b/drivers/common/mlx5/version.map
index e1f0b324a0..2e213bd3b9 100644
--- a/drivers/common/mlx5/version.map
+++ b/drivers/common/mlx5/version.map
@@ -11,8 +11,16 @@ INTERNAL {
 	mlx5_dev_to_pci_addr;
 
 	mlx5_devx_cmd_alloc_pd;
+	mlx5_devx_alloc_uar;
+
 	mlx5_devx_cmd_create_cq;
+	mlx5_devx_cmd_create_credential_obj;
+	mlx5_devx_cmd_create_crypto_login_obj;
+	mlx5_devx_cmd_create_dek_obj;
 	mlx5_devx_cmd_create_flex_parser;
+	mlx5_devx_cmd_create_flow_hit_aso_obj;
+	mlx5_devx_cmd_create_geneve_tlv_option;
+	mlx5_devx_cmd_create_import_kek_obj;
 	mlx5_devx_cmd_create_qp;
 	mlx5_devx_cmd_create_rq;
 	mlx5_devx_cmd_create_rqt;
@@ -22,12 +30,6 @@ INTERNAL {
 	mlx5_devx_cmd_create_tis;
 	mlx5_devx_cmd_create_virtio_q_counters;
 	mlx5_devx_cmd_create_virtq;
-	mlx5_devx_cmd_create_flow_hit_aso_obj;
-	mlx5_devx_cmd_create_geneve_tlv_option;
-        mlx5_devx_cmd_create_dek_obj;
-        mlx5_devx_cmd_create_import_kek_obj;
-        mlx5_devx_cmd_create_credential_obj;
-        mlx5_devx_cmd_create_crypto_login_obj;
 	mlx5_devx_cmd_destroy;
 	mlx5_devx_cmd_flow_counter_alloc;
 	mlx5_devx_cmd_flow_counter_query;
@@ -47,12 +49,14 @@ INTERNAL {
 	mlx5_devx_cmd_queue_counter_alloc;
 	mlx5_devx_cmd_queue_counter_query;
 	mlx5_devx_cmd_register_read;
+	mlx5_devx_cmd_register_write;
 	mlx5_devx_cmd_wq_query;
-	mlx5_devx_get_out_command_status;
-	mlx5_devx_alloc_uar;
 
 	mlx5_devx_cq_create;
 	mlx5_devx_cq_destroy;
+
+	mlx5_devx_get_out_command_status;
+
 	mlx5_devx_rq_create;
 	mlx5_devx_rq_destroy;
 	mlx5_devx_sq_create;
-- 
2.21.0

[dpdk-dev] [PATCH 16/24] drivers: introduce mlx5 crypto PMD

[Shiri Kuzin]

Add a new PMD for Nvidia devices- crypto PMD.

The crypto PMD will be supported starting Nvidia ConnectX6 and
BlueField2.

The crypto PMD will add the support of encryption and decryption using
the AES-XTS symmetric algorithm.

The crypto PMD requires rdma-core and uses mlx5 DevX.

This patch adds the PCI probing, basic functions, build files and
log utility.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 MAINTAINERS                             |   4 +
 drivers/common/mlx5/mlx5_common.h       |   1 +
 drivers/common/mlx5/mlx5_common_pci.c   |  14 ++
 drivers/common/mlx5/mlx5_common_pci.h   |  21 +-
 drivers/crypto/meson.build              |   1 +
 drivers/crypto/mlx5/meson.build         |  26 +++
 drivers/crypto/mlx5/mlx5_crypto.c       | 288 ++++++++++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto_utils.h |  19 ++
 drivers/crypto/mlx5/version.map         |   3 +
 9 files changed, 367 insertions(+), 10 deletions(-)
 create mode 100644 drivers/crypto/mlx5/meson.build
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto.c
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_utils.h
 create mode 100644 drivers/crypto/mlx5/version.map

diff --git a/MAINTAINERS b/MAINTAINERS
index 0ec5588540..448fd32186 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -1064,6 +1064,10 @@ F: drivers/crypto/octeontx2/
 F: doc/guides/cryptodevs/octeontx2.rst
 F: doc/guides/cryptodevs/features/octeontx2.ini
 
+Mellanox mlx5
+M: Matan Azrad <matan@nvidia.com>
+F: drivers/crypto/mlx5/
+
 Null Crypto
 M: Declan Doherty <declan.doherty@intel.com>
 F: drivers/crypto/null/
diff --git a/drivers/common/mlx5/mlx5_common.h b/drivers/common/mlx5/mlx5_common.h
index f1ff16eb3e..7858cdb7dc 100644
--- a/drivers/common/mlx5/mlx5_common.h
+++ b/drivers/common/mlx5/mlx5_common.h
@@ -215,6 +215,7 @@ enum mlx5_class {
 	MLX5_CLASS_VDPA = RTE_BIT64(1),
 	MLX5_CLASS_REGEX = RTE_BIT64(2),
 	MLX5_CLASS_COMPRESS = RTE_BIT64(3),
+	MLX5_CLASS_CRYPTO = RTE_BIT64(4),
 };
 
 #define MLX5_DBR_SIZE RTE_CACHE_LINE_SIZE
diff --git a/drivers/common/mlx5/mlx5_common_pci.c b/drivers/common/mlx5/mlx5_common_pci.c
index 177ccf62d9..c27d8dd492 100644
--- a/drivers/common/mlx5/mlx5_common_pci.c
+++ b/drivers/common/mlx5/mlx5_common_pci.c
@@ -31,6 +31,7 @@ static const struct {
 	{ .name = "eth", .driver_class = MLX5_CLASS_NET },
 	{ .name = "regex", .driver_class = MLX5_CLASS_REGEX },
 	{ .name = "compress", .driver_class = MLX5_CLASS_COMPRESS },
+	{ .name = "crypto", .driver_class = MLX5_CLASS_CRYPTO },
 };
 
 static const unsigned int mlx5_class_combinations[] = {
@@ -38,13 +39,26 @@ static const unsigned int mlx5_class_combinations[] = {
 	MLX5_CLASS_VDPA,
 	MLX5_CLASS_REGEX,
 	MLX5_CLASS_COMPRESS,
+	MLX5_CLASS_CRYPTO,
 	MLX5_CLASS_NET | MLX5_CLASS_REGEX,
 	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX,
 	MLX5_CLASS_NET | MLX5_CLASS_COMPRESS,
 	MLX5_CLASS_VDPA | MLX5_CLASS_COMPRESS,
 	MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
+	MLX5_CLASS_NET | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_VDPA | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO,
 	MLX5_CLASS_NET | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
 	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
+	MLX5_CLASS_NET | MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_NET | MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_VDPA | MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_NET | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS |
+	MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS |
+	MLX5_CLASS_CRYPTO,
 	/* New class combination should be added here. */
 };
 
diff --git a/drivers/common/mlx5/mlx5_common_pci.h b/drivers/common/mlx5/mlx5_common_pci.h
index de89bb98bc..cb8d2f5f87 100644
--- a/drivers/common/mlx5/mlx5_common_pci.h
+++ b/drivers/common/mlx5/mlx5_common_pci.h
@@ -9,17 +9,18 @@
  * @file
  *
  * RTE Mellanox PCI Driver Interface
- * Mellanox ConnectX PCI device supports multiple class: net,vdpa,regex and
- * compress devices. This layer enables creating such multiple class of devices
- * on a single PCI device by allowing to bind multiple class specific device
- * driver to attach to mlx5_pci driver.
+ * Mellanox ConnectX PCI device supports multiple class: net,vdpa,regex,compress
+ * and crypto devices. This layer enables creating such multiple class of
+ * devices on a single PCI device by allowing to bind multiple class specific
+ * device driver to attach to mlx5_pci driver.
  *
- * -----------    ------------    -------------    ----------------
- * |   mlx5  |    |   mlx5   |    |   mlx5    |    |     mlx5     |
- * | net pmd |    | vdpa pmd |    | regex pmd |    | compress pmd |
- * -----------    ------------    -------------    ----------------
- *      \              \                    /              /
- *       \              \                  /              /
+ * --------    --------    ---------    ------------    ----------
+ * | mlx5 |    | mlx5 |    | mlx5  |    |   mlx5   |    |  mlx5  |
+ * | net  |    | vdpa |    | regex |    | compress |    | crypto |
+ * | pmd  |    | pmd  |    |  pmd  |    |   pmd    |    |  pmd   |
+ * --------    --------    ---------    ------------    ----------
+ *      \              \         |          /              /
+ *       \              \        |         /              /
  *        \              \_--------------_/              /
  *         \_______________|   mlx5     |_______________/
  *                         | pci common |
diff --git a/drivers/crypto/meson.build b/drivers/crypto/meson.build
index c927c5f8d3..90b21494e1 100644
--- a/drivers/crypto/meson.build
+++ b/drivers/crypto/meson.build
@@ -14,6 +14,7 @@ drivers = ['aesni_gcm',
 	   'dpaa_sec',
 	   'dpaa2_sec',
 	   'kasumi',
+	   'mlx5',
 	   'mvsam',
 	   'nitrox',
 	   'null',
diff --git a/drivers/crypto/mlx5/meson.build b/drivers/crypto/mlx5/meson.build
new file mode 100644
index 0000000000..5bf0912766
--- /dev/null
+++ b/drivers/crypto/mlx5/meson.build
@@ -0,0 +1,26 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2021 Mellanox Technologies, Ltd
+
+if not is_linux
+	build = false
+	reason = 'only supported on Linux'
+	subdir_done()
+endif
+
+fmt_name = 'mlx5_crypto'
+deps += ['common_mlx5', 'eal', 'cryptodev']
+sources = files(
+	'mlx5_crypto.c',
+)
+cflags_options = [
+	'-std=c11',
+	'-Wno-strict-prototypes',
+	'-D_BSD_SOURCE',
+	'-D_DEFAULT_SOURCE',
+	'-D_XOPEN_SOURCE=600'
+]
+foreach option:cflags_options
+	if cc.has_argument(option)
+		cflags += option
+	endif
+endforeach
\ No newline at end of file
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
new file mode 100644
index 0000000000..6e40176087
--- /dev/null
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -0,0 +1,288 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#include <rte_malloc.h>
+#include <rte_log.h>
+#include <rte_errno.h>
+#include <rte_pci.h>
+#include <rte_crypto.h>
+#include <rte_cryptodev.h>
+#include <rte_cryptodev_pmd.h>
+
+#include <mlx5_glue.h>
+#include <mlx5_common.h>
+#include <mlx5_common_pci.h>
+#include <mlx5_devx_cmds.h>
+#include <mlx5_common_os.h>
+
+#include "mlx5_crypto_utils.h"
+
+#define MLX5_CRYPTO_DRIVER_NAME mlx5_crypto
+#define MLX5_CRYPTO_LOG_NAME    pmd.crypto.mlx5
+
+struct mlx5_crypto_priv {
+	TAILQ_ENTRY(mlx5_crypto_priv) next;
+	struct ibv_context *ctx; /* Device context. */
+	struct rte_pci_device *pci_dev;
+	struct rte_cryptodev *crypto_dev;
+	void *uar; /* User Access Region. */
+	uint32_t pdn; /* Protection Domain number. */
+	struct ibv_pd *pd;
+};
+
+TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
+				TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
+static pthread_mutex_t priv_list_lock = PTHREAD_MUTEX_INITIALIZER;
+
+int mlx5_crypto_logtype;
+
+uint8_t mlx5_crypto_driver_id;
+
+static const char mlx5_crypto_drv_name[] = RTE_STR(MLX5_CRYPTO_DRIVER_NAME);
+
+static const struct rte_driver mlx5_drv = {
+	.name = mlx5_crypto_drv_name,
+	.alias = mlx5_crypto_drv_name
+};
+
+static struct cryptodev_driver mlx5_cryptodev_driver;
+
+static struct rte_cryptodev_ops mlx5_crypto_ops = {
+	.dev_configure			= NULL,
+	.dev_start			= NULL,
+	.dev_stop			= NULL,
+	.dev_close			= NULL,
+	.dev_infos_get			= NULL,
+	.stats_get			= NULL,
+	.stats_reset			= NULL,
+	.queue_pair_setup		= NULL,
+	.queue_pair_release		= NULL,
+	.sym_session_get_size		= NULL,
+	.sym_session_configure		= NULL,
+	.sym_session_clear		= NULL,
+	.sym_get_raw_dp_ctx_size	= NULL,
+	.sym_configure_raw_dp_ctx	= NULL,
+};
+
+static void
+mlx5_crypto_hw_global_release(struct mlx5_crypto_priv *priv)
+{
+	if (priv->pd != NULL) {
+		claim_zero(mlx5_glue->dealloc_pd(priv->pd));
+		priv->pd = NULL;
+	}
+	if (priv->uar != NULL) {
+		mlx5_glue->devx_free_uar(priv->uar);
+		priv->uar = NULL;
+	}
+}
+
+static int
+mlx5_crypto_pd_create(struct mlx5_crypto_priv *priv)
+{
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+	struct mlx5dv_obj obj;
+	struct mlx5dv_pd pd_info;
+	int ret;
+
+	priv->pd = mlx5_glue->alloc_pd(priv->ctx);
+	if (priv->pd == NULL) {
+		DRV_LOG(ERR, "Failed to allocate PD.");
+		return errno ? -errno : -ENOMEM;
+	}
+	obj.pd.in = priv->pd;
+	obj.pd.out = &pd_info;
+	ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_PD);
+	if (ret != 0) {
+		DRV_LOG(ERR, "Fail to get PD object info.");
+		mlx5_glue->dealloc_pd(priv->pd);
+		priv->pd = NULL;
+		return -errno;
+	}
+	priv->pdn = pd_info.pdn;
+	return 0;
+#else
+	(void)priv;
+	DRV_LOG(ERR, "Cannot get pdn - no DV support.");
+	return -ENOTSUP;
+#endif /* HAVE_IBV_FLOW_DV_SUPPORT */
+}
+
+static int
+mlx5_crypto_hw_global_prepare(struct mlx5_crypto_priv *priv)
+{
+	if (mlx5_crypto_pd_create(priv) != 0)
+		return -1;
+	priv->uar = mlx5_devx_alloc_uar(priv->ctx, -1);
+	if (priv->uar == NULL || mlx5_os_get_devx_uar_reg_addr(priv->uar) ==
+	    NULL) {
+		rte_errno = errno;
+		claim_zero(mlx5_glue->dealloc_pd(priv->pd));
+		DRV_LOG(ERR, "Failed to allocate UAR.");
+		return -1;
+	}
+	return 0;
+}
+
+/**
+ * DPDK callback to register a PCI device.
+ *
+ * This function spawns crypto device out of a given PCI device.
+ *
+ * @param[in] pci_drv
+ *   PCI driver structure (mlx5_crypto_driver).
+ * @param[in] pci_dev
+ *   PCI device information.
+ *
+ * @return
+ *   0 on success, 1 to skip this driver, a negative errno value otherwise
+ *   and rte_errno is set.
+ */
+static int
+mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
+			struct rte_pci_device *pci_dev)
+{
+	struct ibv_device *ibv;
+	struct rte_cryptodev *crypto_dev;
+	struct ibv_context *ctx;
+	struct mlx5_crypto_priv *priv;
+	struct mlx5_hca_attr attr = { 0 };
+	struct rte_cryptodev_pmd_init_params init_params = {
+		.name = "",
+		.private_data_size = sizeof(struct mlx5_crypto_priv),
+		.socket_id = pci_dev->device.numa_node,
+		.max_nb_queue_pairs =
+				RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
+	};
+	RTE_SET_USED(pci_drv);
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+		DRV_LOG(ERR, "Non-primary process type is not supported.");
+		rte_errno = ENOTSUP;
+		return -rte_errno;
+	}
+	ibv = mlx5_os_get_ib_device_match(&pci_dev->addr);
+	if (ibv == NULL) {
+		DRV_LOG(ERR, "No matching IB device for PCI slot "
+			PCI_PRI_FMT ".", pci_dev->addr.domain,
+			pci_dev->addr.bus, pci_dev->addr.devid,
+			pci_dev->addr.function);
+		return -rte_errno;
+	}
+	DRV_LOG(INFO, "PCI information matches for device \"%s\".", ibv->name);
+	ctx = mlx5_glue->dv_open_device(ibv);
+	if (ctx == NULL) {
+		DRV_LOG(ERR, "Failed to open IB device \"%s\".", ibv->name);
+		rte_errno = ENODEV;
+		return -rte_errno;
+	}
+	if (mlx5_devx_cmd_query_hca_attr(ctx, &attr) != 0 ||
+	    attr.crypto == 0 || attr.aes_xts == 0) {
+		DRV_LOG(ERR, "Not enough capabilities to support crypto "
+			"operations, maybe old FW/OFED version?");
+		claim_zero(mlx5_glue->close_device(ctx));
+		rte_errno = ENOTSUP;
+		return -ENOTSUP;
+	}
+	crypto_dev = rte_cryptodev_pmd_create(ibv->name, &pci_dev->device,
+					&init_params);
+	if (crypto_dev == NULL) {
+		DRV_LOG(ERR, "Failed to create device \"%s\".", ibv->name);
+		claim_zero(mlx5_glue->close_device(ctx));
+		return -ENODEV;
+	}
+	DRV_LOG(INFO,
+		"Crypto device %s was created successfully.", ibv->name);
+	crypto_dev->dev_ops = &mlx5_crypto_ops;
+	crypto_dev->dequeue_burst = NULL;
+	crypto_dev->enqueue_burst = NULL;
+	crypto_dev->feature_flags = RTE_CRYPTODEV_FF_HW_ACCELERATED;
+	crypto_dev->driver_id = mlx5_crypto_driver_id;
+	priv = crypto_dev->data->dev_private;
+	priv->ctx = ctx;
+	priv->pci_dev = pci_dev;
+	priv->crypto_dev = crypto_dev;
+	if (mlx5_crypto_hw_global_prepare(priv) != 0) {
+		rte_cryptodev_pmd_destroy(priv->crypto_dev);
+		claim_zero(mlx5_glue->close_device(priv->ctx));
+		return -1;
+	}
+	pthread_mutex_lock(&priv_list_lock);
+	TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next);
+	pthread_mutex_unlock(&priv_list_lock);
+	return 0;
+}
+
+static int
+mlx5_crypto_pci_remove(struct rte_pci_device *pdev)
+{
+	struct mlx5_crypto_priv *priv = NULL;
+
+	pthread_mutex_lock(&priv_list_lock);
+	TAILQ_FOREACH(priv, &mlx5_crypto_priv_list, next)
+		if (rte_pci_addr_cmp(&priv->pci_dev->addr, &pdev->addr) != 0)
+			break;
+	if (priv)
+		TAILQ_REMOVE(&mlx5_crypto_priv_list, priv, next);
+	pthread_mutex_unlock(&priv_list_lock);
+	if (priv) {
+		mlx5_crypto_hw_global_release(priv);
+		rte_cryptodev_pmd_destroy(priv->crypto_dev);
+		claim_zero(mlx5_glue->close_device(priv->ctx));
+	}
+	return 0;
+}
+
+static const struct rte_pci_id mlx5_crypto_pci_id_map[] = {
+		{
+			RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
+					PCI_DEVICE_ID_MELLANOX_CONNECTX6)
+		},
+		{
+			RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
+					PCI_DEVICE_ID_MELLANOX_CONNECTX6DX)
+		},
+		{
+			RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
+					PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF)
+		},
+		{
+			RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
+					PCI_DEVICE_ID_MELLANOX_CONNECTX7)
+		},
+		{
+			RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
+					PCI_DEVICE_ID_MELLANOX_CONNECTX7BF)
+		},
+		{
+			.vendor_id = 0
+		}
+	};
+
+static struct mlx5_pci_driver mlx5_crypto_driver = {
+	.driver_class = MLX5_CLASS_CRYPTO,
+	.pci_driver = {
+		.driver = {
+			.name = RTE_STR(MLX5_CRYPTO_DRIVER_NAME),
+		},
+		.id_table = mlx5_crypto_pci_id_map,
+		.probe = mlx5_crypto_pci_probe,
+		.remove = mlx5_crypto_pci_remove,
+		.drv_flags = 0,
+	},
+};
+
+RTE_INIT(rte_mlx5_crypto_init)
+{
+	mlx5_common_init();
+	if (mlx5_glue != NULL)
+		mlx5_pci_driver_register(&mlx5_crypto_driver);
+}
+
+RTE_PMD_REGISTER_CRYPTO_DRIVER(mlx5_cryptodev_driver, mlx5_drv,
+			       mlx5_crypto_driver_id);
+
+RTE_LOG_REGISTER(mlx5_crypto_logtype, MLX5_CRYPTO_LOG_NAME, NOTICE)
+RTE_PMD_EXPORT_NAME(MLX5_CRYPTO_DRIVER_NAME, __COUNTER__);
+RTE_PMD_REGISTER_PCI_TABLE(MLX5_CRYPTO_DRIVER_NAME, mlx5_crypto_pci_id_map);
+RTE_PMD_REGISTER_KMOD_DEP(MLX5_CRYPTO_DRIVER_NAME, "* ib_uverbs & mlx5_core & mlx5_ib");
diff --git a/drivers/crypto/mlx5/mlx5_crypto_utils.h b/drivers/crypto/mlx5/mlx5_crypto_utils.h
new file mode 100644
index 0000000000..cef4b07a36
--- /dev/null
+++ b/drivers/crypto/mlx5/mlx5_crypto_utils.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#ifndef RTE_PMD_MLX5_CRYPTO_UTILS_H_
+#define RTE_PMD_MLX5_CRYPTO_UTILS_H_
+
+#include <mlx5_common.h>
+
+extern int mlx5_crypto_logtype;
+
+#define MLX5_CRYPTO_LOG_PREFIX "mlx5_crypto"
+/* Generic printf()-like logging macro with automatic line feed. */
+#define DRV_LOG(level, ...) \
+	PMD_DRV_LOG_(level, mlx5_crypto_logtype, MLX5_CRYPTO_LOG_PREFIX, \
+		__VA_ARGS__ PMD_DRV_LOG_STRIP PMD_DRV_LOG_OPAREN, \
+		PMD_DRV_LOG_CPAREN)
+
+#endif /* RTE_PMD_MLX5_CRYPTO_UTILS_H_ */
diff --git a/drivers/crypto/mlx5/version.map b/drivers/crypto/mlx5/version.map
new file mode 100644
index 0000000000..4a76d1d52d
--- /dev/null
+++ b/drivers/crypto/mlx5/version.map
@@ -0,0 +1,3 @@
+DPDK_21 {
+	local: *;
+};
-- 
2.21.0

[dpdk-dev] [PATCH 17/24] crypto/mlx5: add DEK object management

[Shiri Kuzin]

A DEK(Data encryption Key) is an mlx5 HW object which represents the
cipher algorithm key.
The DEKs are used during data encryption/decryption operations.

In symmetric algorithms like AES-STS, we use the same DEK for both
encryption and decryption.

Use the mlx5 hash-list tool to manage the DEK objects in the PMD.

Provide the compare, create and destroy functions to manage DEKs in
hash-list and introduce an internal API to setup and unset the DEK
management and to prepare and destroy specific DEK object.

The DEK hash-list will be created in dev_configure routine and
destroyed in dev_close routine.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/meson.build       |   1 +
 drivers/crypto/mlx5/mlx5_crypto.c     |  44 +++++----
 drivers/crypto/mlx5/mlx5_crypto.h     |  51 ++++++++++
 drivers/crypto/mlx5/mlx5_crypto_dek.c | 135 ++++++++++++++++++++++++++
 4 files changed, 214 insertions(+), 17 deletions(-)
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto.h
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_dek.c

diff --git a/drivers/crypto/mlx5/meson.build b/drivers/crypto/mlx5/meson.build
index 5bf0912766..0666c35094 100644
--- a/drivers/crypto/mlx5/meson.build
+++ b/drivers/crypto/mlx5/meson.build
@@ -11,6 +11,7 @@ fmt_name = 'mlx5_crypto'
 deps += ['common_mlx5', 'eal', 'cryptodev']
 sources = files(
 	'mlx5_crypto.c',
+	'mlx5_crypto_dek.c',
 )
 cflags_options = [
 	'-std=c11',
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 6e40176087..17aaaaa53d 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -3,12 +3,9 @@
  */
 
 #include <rte_malloc.h>
-#include <rte_log.h>
 #include <rte_errno.h>
+#include <rte_log.h>
 #include <rte_pci.h>
-#include <rte_crypto.h>
-#include <rte_cryptodev.h>
-#include <rte_cryptodev_pmd.h>
 
 #include <mlx5_glue.h>
 #include <mlx5_common.h>
@@ -17,19 +14,10 @@
 #include <mlx5_common_os.h>
 
 #include "mlx5_crypto_utils.h"
+#include "mlx5_crypto.h"
 
 #define MLX5_CRYPTO_DRIVER_NAME mlx5_crypto
-#define MLX5_CRYPTO_LOG_NAME    pmd.crypto.mlx5
-
-struct mlx5_crypto_priv {
-	TAILQ_ENTRY(mlx5_crypto_priv) next;
-	struct ibv_context *ctx; /* Device context. */
-	struct rte_pci_device *pci_dev;
-	struct rte_cryptodev *crypto_dev;
-	void *uar; /* User Access Region. */
-	uint32_t pdn; /* Protection Domain number. */
-	struct ibv_pd *pd;
-};
+#define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
 
 TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
 				TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
@@ -48,11 +36,33 @@ static const struct rte_driver mlx5_drv = {
 
 static struct cryptodev_driver mlx5_cryptodev_driver;
 
+static int
+mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
+		struct rte_cryptodev_config *config __rte_unused)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+
+	if (mlx5_crypto_dek_setup(priv) != 0) {
+		DRV_LOG(ERR, "Dek hash list creation has failed.");
+		return -ENOMEM;
+	}
+	return 0;
+}
+
+static int
+mlx5_crypto_dev_close(struct rte_cryptodev *dev)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+
+	mlx5_crypto_dek_unset(priv);
+	return 0;
+}
+
 static struct rte_cryptodev_ops mlx5_crypto_ops = {
-	.dev_configure			= NULL,
+	.dev_configure			= mlx5_crypto_dev_configure,
 	.dev_start			= NULL,
 	.dev_stop			= NULL,
-	.dev_close			= NULL,
+	.dev_close			= mlx5_crypto_dev_close,
 	.dev_infos_get			= NULL,
 	.stats_get			= NULL,
 	.stats_reset			= NULL,
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
new file mode 100644
index 0000000000..4ec67a7e0f
--- /dev/null
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -0,0 +1,51 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#ifndef MLX5_CRYPTO_H_
+#define MLX5_CRYPTO_H_
+
+#include <stdbool.h>
+
+#include <rte_cryptodev.h>
+#include <rte_cryptodev_pmd.h>
+
+#include <mlx5_common_utils.h>
+
+#define MLX5_CRYPTO_DEK_HTABLE_SZ (1 << 11)
+#define MLX5_CRYPTO_KEY_LENGTH 80
+
+struct mlx5_crypto_priv {
+	TAILQ_ENTRY(mlx5_crypto_priv) next;
+	struct ibv_context *ctx; /* Device context. */
+	struct rte_pci_device *pci_dev;
+	struct rte_cryptodev *crypto_dev;
+	void *uar; /* User Access Region. */
+	uint32_t pdn; /* Protection Domain number. */
+	struct ibv_pd *pd;
+	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
+};
+
+struct mlx5_crypto_dek {
+	struct mlx5_hlist_entry entry; /* Pointer to DEK hash list entry. */
+	struct mlx5_devx_obj *obj; /* Pointer to DEK DevX object. */
+	uint8_t data[MLX5_CRYPTO_KEY_LENGTH]; /* DEK key data. */
+	bool size_is_48; /* Whether the key\data size is 48 bytes or not. */
+};
+
+int
+mlx5_crypto_dek_destroy(struct mlx5_crypto_priv *priv,
+			struct mlx5_crypto_dek *dek);
+
+struct mlx5_crypto_dek *
+mlx5_crypto_dek_prepare(struct mlx5_crypto_priv *priv,
+			struct rte_crypto_cipher_xform *cipher);
+
+int
+mlx5_crypto_dek_setup(struct mlx5_crypto_priv *priv);
+
+void
+mlx5_crypto_dek_unset(struct mlx5_crypto_priv *priv);
+
+#endif /* MLX5_CRYPTO_H_ */
+
diff --git a/drivers/crypto/mlx5/mlx5_crypto_dek.c b/drivers/crypto/mlx5/mlx5_crypto_dek.c
new file mode 100644
index 0000000000..70410fed57
--- /dev/null
+++ b/drivers/crypto/mlx5/mlx5_crypto_dek.c
@@ -0,0 +1,135 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2018 Mellanox Technologies, Ltd
+ */
+
+#include <rte_ip.h>
+#include <rte_common.h>
+#include <rte_errno.h>
+#include <rte_log.h>
+
+#include <mlx5_prm.h>
+#include <mlx5_devx_cmds.h>
+
+#include "mlx5_crypto_utils.h"
+#include "mlx5_crypto.h"
+
+struct mlx5_crypto_dek_ctx {
+	struct rte_crypto_cipher_xform *cipher;
+	struct mlx5_crypto_priv *priv;
+};
+
+int
+mlx5_crypto_dek_destroy(struct mlx5_crypto_priv *priv,
+			struct mlx5_crypto_dek *dek)
+{
+	return mlx5_hlist_unregister(priv->dek_hlist, &dek->entry);
+}
+
+struct mlx5_crypto_dek *
+mlx5_crypto_dek_prepare(struct mlx5_crypto_priv *priv,
+			struct rte_crypto_cipher_xform *cipher)
+{
+	struct mlx5_hlist *dek_hlist = priv->dek_hlist;
+	struct mlx5_crypto_dek_ctx dek_ctx = {
+		.cipher = cipher,
+		.priv = priv,
+	};
+	struct rte_crypto_cipher_xform *cipher_ctx = cipher;
+	uint64_t key64 = __rte_raw_cksum(cipher_ctx->key.data,
+					 cipher_ctx->key.length, 0);
+	struct mlx5_hlist_entry *entry = mlx5_hlist_register(dek_hlist,
+							     key64, &dek_ctx);
+
+	return entry == NULL ? NULL :
+			     container_of(entry, struct mlx5_crypto_dek, entry);
+}
+
+static int
+mlx5_crypto_dek_match_cb(struct mlx5_hlist *list __rte_unused,
+			 struct mlx5_hlist_entry *entry,
+			 uint64_t key __rte_unused, void *cb_ctx)
+{
+	struct mlx5_crypto_dek_ctx *ctx = cb_ctx;
+	struct rte_crypto_cipher_xform *cipher_ctx = ctx->cipher;
+	struct mlx5_crypto_dek *dek =
+			container_of(entry, typeof(*dek), entry);
+	uint32_t key_len = dek->size_is_48 ? 48 : 80;
+
+	if (key_len != cipher_ctx->key.length)
+		return -1;
+	return memcmp(cipher_ctx->key.data, dek->data, key_len);
+}
+
+static struct mlx5_hlist_entry *
+mlx5_crypto_dek_create_cb(struct mlx5_hlist *list __rte_unused,
+			  uint64_t key __rte_unused, void *cb_ctx)
+{
+	struct mlx5_crypto_dek_ctx *ctx = cb_ctx;
+	struct rte_crypto_cipher_xform *cipher_ctx = ctx->cipher;
+	struct mlx5_crypto_dek *dek = rte_zmalloc(__func__, sizeof(*dek),
+						  RTE_CACHE_LINE_SIZE);
+	struct mlx5_devx_dek_attr dek_attr = {
+		.pd = ctx->priv->pdn,
+		.key_purpose = MLX5_CRYPTO_KEY_PURPOSE_AES_XTS,
+	};
+
+	if (dek == NULL) {
+		DRV_LOG(ERR, "Failed to allocate dek memory.");
+		return NULL;
+	}
+	switch (cipher_ctx->key.length) {
+	case 48:
+		dek->size_is_48 = true;
+		dek_attr.key_size = MLX5_CRYPTO_KEY_SIZE_128b;
+		break;
+	case 80:
+		dek->size_is_48 = false;
+		dek_attr.key_size = MLX5_CRYPTO_KEY_SIZE_256b;
+		break;
+	default:
+		DRV_LOG(ERR, "Key size not supported.");
+		return NULL;
+	}
+	rte_memcpy(&dek_attr.key, cipher_ctx->key.data, cipher_ctx->key.length);
+	dek->obj = mlx5_devx_cmd_create_dek_obj(ctx->priv->ctx, &dek_attr);
+	if (dek->obj == NULL) {
+		rte_free(dek);
+		return NULL;
+	}
+	rte_memcpy(&dek->data, cipher_ctx->key.data, cipher_ctx->key.length);
+	return &dek->entry;
+}
+
+static void
+mlx5_crypto_dek_remove_cb(struct mlx5_hlist *list __rte_unused,
+			  struct mlx5_hlist_entry *entry)
+{
+	struct mlx5_crypto_dek *dek =
+		container_of(entry, typeof(*dek), entry);
+
+	claim_zero(mlx5_devx_cmd_destroy(dek->obj));
+	rte_free(dek);
+}
+
+
+int
+mlx5_crypto_dek_setup(struct mlx5_crypto_priv *priv)
+{
+	priv->dek_hlist = mlx5_hlist_create("dek_hlist",
+				 MLX5_CRYPTO_DEK_HTABLE_SZ,
+				 0, MLX5_HLIST_WRITE_MOST |
+				 MLX5_HLIST_DIRECT_KEY,
+				 mlx5_crypto_dek_create_cb,
+				 mlx5_crypto_dek_match_cb,
+				 mlx5_crypto_dek_remove_cb);
+	if (priv->dek_hlist == NULL)
+		return -1;
+	return 0;
+}
+
+void
+mlx5_crypto_dek_unset(struct mlx5_crypto_priv *priv)
+{
+	mlx5_hlist_destroy(priv->dek_hlist);
+	priv->dek_hlist = NULL;
+}
-- 
2.21.0

[dpdk-dev] [PATCH 18/24] crypto/mlx5: support session operations

[Shiri Kuzin]

Sessions are used in symmetric transformations in order to prepare
objects and data for packet processing stage.

A mlx5 session includes iv_offset, pointer to mlx5_crypto_dek struct,
bsf_size, bsf_p_type, encryption_order and encryption standard.

Implement the next session operations:
        mlx5_crypto_sym_session_get_size- returns the size of the mlx5
	session struct.
	mlx5_crypto_sym_session_configure- prepares the DEK hash-list
	and saves all the session data.
	mlx5_crypto_sym_session_clear - destroys the DEK hash-list.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 92 ++++++++++++++++++++++++++++++-
 1 file changed, 89 insertions(+), 3 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 17aaaaa53d..b0242afec4 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -3,6 +3,7 @@
  */
 
 #include <rte_malloc.h>
+#include <rte_mempool.h>
 #include <rte_errno.h>
 #include <rte_log.h>
 #include <rte_pci.h>
@@ -36,6 +37,23 @@ static const struct rte_driver mlx5_drv = {
 
 static struct cryptodev_driver mlx5_cryptodev_driver;
 
+struct mlx5_crypto_session {
+	uint32_t bs_bpt_eo_es;
+	/*
+	 * bsf_size, bsf_p_type, encryption_order and encryption standard,
+	 * saved in big endian format.
+	 */
+	uint32_t iv_offset:16;
+	/* Starting point for Initialisation Vector. */
+	struct mlx5_crypto_dek *dek; /* Pointer to dek struct. */
+} __rte_packed;
+
+static unsigned int
+mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
+{
+	return sizeof(struct mlx5_crypto_session);
+}
+
 static int
 mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
 		struct rte_cryptodev_config *config __rte_unused)
@@ -58,6 +76,74 @@ mlx5_crypto_dev_close(struct rte_cryptodev *dev)
 	return 0;
 }
 
+static int
+mlx5_crypto_sym_session_configure(struct rte_cryptodev *dev,
+				  struct rte_crypto_sym_xform *xform,
+				  struct rte_cryptodev_sym_session *session,
+				  struct rte_mempool *mp)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+	struct mlx5_crypto_session *sess_private_data;
+	struct rte_crypto_cipher_xform *cipher;
+	uint8_t encryption_order;
+	int ret;
+
+	if (unlikely(xform->next != NULL)) {
+		DRV_LOG(ERR, "Xform next is not supported.");
+		return -ENOTSUP;
+	}
+	if (unlikely((xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) ||
+		     (xform->cipher.algo != RTE_CRYPTO_CIPHER_AES_XTS))) {
+		DRV_LOG(ERR, "Only AES-XTS algorithm is supported.");
+		return -ENOTSUP;
+	}
+	ret = rte_mempool_get(mp, (void *)&sess_private_data);
+	if (ret != 0) {
+		DRV_LOG(ERR,
+			"Failed to get session %p private data from mempool.",
+			sess_private_data);
+		return -ENOMEM;
+	}
+	cipher = &xform->cipher;
+	sess_private_data->dek = mlx5_crypto_dek_prepare(priv, cipher);
+	if (sess_private_data->dek == NULL) {
+		rte_mempool_put(mp, sess_private_data);
+		DRV_LOG(ERR, "Failed to prepare dek.");
+		return -ENOMEM;
+	}
+	if (cipher->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
+		encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_MEMORY;
+	else
+		encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_WIRE;
+	sess_private_data->bs_bpt_eo_es = rte_cpu_to_be_32
+			(MLX5_BSF_SIZE_64B << MLX5_BSF_SIZE_OFFSET |
+			 MLX5_BSF_P_TYPE_CRYPTO << MLX5_BSF_P_TYPE_OFFSET |
+			 encryption_order << MLX5_ENCRYPTION_ORDER_OFFSET |
+			 MLX5_ENCRYPTION_STANDARD_AES_XTS);
+	sess_private_data->iv_offset = cipher->iv.offset;
+	set_sym_session_private_data(session, dev->driver_id,
+				     sess_private_data);
+	DRV_LOG(DEBUG, "Session %p was configured.", sess_private_data);
+	return 0;
+}
+
+static void
+mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev,
+			      struct rte_cryptodev_sym_session *sess)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+	struct mlx5_crypto_session *sess_private_data =
+			get_sym_session_private_data(sess, dev->driver_id);
+
+	if (unlikely(sess_private_data == NULL)) {
+		DRV_LOG(ERR, "Failed to get session %p private data.",
+				sess_private_data);
+		return;
+	}
+	mlx5_crypto_dek_destroy(priv, sess_private_data->dek);
+	DRV_LOG(DEBUG, "Session %p was cleared.", sess_private_data);
+}
+
 static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_configure			= mlx5_crypto_dev_configure,
 	.dev_start			= NULL,
@@ -68,9 +154,9 @@ static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.stats_reset			= NULL,
 	.queue_pair_setup		= NULL,
 	.queue_pair_release		= NULL,
-	.sym_session_get_size		= NULL,
-	.sym_session_configure		= NULL,
-	.sym_session_clear		= NULL,
+	.sym_session_get_size		= mlx5_crypto_sym_session_get_size,
+	.sym_session_configure		= mlx5_crypto_sym_session_configure,
+	.sym_session_clear		= mlx5_crypto_sym_session_clear,
 	.sym_get_raw_dp_ctx_size	= NULL,
 	.sym_configure_raw_dp_ctx	= NULL,
 };
-- 
2.21.0

[dpdk-dev] [PATCH 19/24] crypto/mlx5: add basic operations

[Shiri Kuzin]

The basic dev control operations are configure, close and get info.

Extended the existing support of configure and close:
	-mlx5_crypto_dev_configure- function used to configure device.
	-mlx5_crypto_dev_close-  function used to close a configured
	 device.

Added support of get info function:
	-mlx5_crypto_dev_infos_get- function used to get specific
	 information of a device.

Added config struct to user private data with the fields socket id,
number of queue pairs and feature flags to be disabled.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 46 +++++++++++++++++++++++++++----
 drivers/crypto/mlx5/mlx5_crypto.h |  1 +
 2 files changed, 42 insertions(+), 5 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index b0242afec4..3807a01357 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -19,6 +19,7 @@
 
 #define MLX5_CRYPTO_DRIVER_NAME mlx5_crypto
 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
+#define MLX5_CRYPTO_MAX_QPS 1024
 
 TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
 				TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
@@ -28,6 +29,9 @@ int mlx5_crypto_logtype;
 
 uint8_t mlx5_crypto_driver_id;
 
+const struct rte_cryptodev_capabilities
+		mlx5_crypto_caps[RTE_CRYPTO_OP_TYPE_UNDEFINED];
+
 static const char mlx5_crypto_drv_name[] = RTE_STR(MLX5_CRYPTO_DRIVER_NAME);
 
 static const struct rte_driver mlx5_drv = {
@@ -48,22 +52,47 @@ struct mlx5_crypto_session {
 	struct mlx5_crypto_dek *dek; /* Pointer to dek struct. */
 } __rte_packed;
 
-static unsigned int
-mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
+static void
+mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev,
+			  struct rte_cryptodev_info *dev_info)
 {
-	return sizeof(struct mlx5_crypto_session);
+	RTE_SET_USED(dev);
+	if (dev_info != NULL) {
+		dev_info->driver_id = mlx5_crypto_driver_id;
+		dev_info->feature_flags = 0;
+		dev_info->capabilities = mlx5_crypto_caps;
+		dev_info->max_nb_queue_pairs = MLX5_CRYPTO_MAX_QPS;
+		dev_info->min_mbuf_headroom_req = 0;
+		dev_info->min_mbuf_tailroom_req = 0;
+		dev_info->sym.max_nb_sessions = 0;
+		/*
+		 * If 0, the device does not have any limitation in number of
+		 * sessions that can be used.
+		 */
+	}
 }
 
 static int
 mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
-		struct rte_cryptodev_config *config __rte_unused)
+			  struct rte_cryptodev_config *config)
 {
 	struct mlx5_crypto_priv *priv = dev->data->dev_private;
 
+	if (config == NULL) {
+		DRV_LOG(ERR, "Invalid crypto dev configure parameters.");
+		return -EINVAL;
+	}
+	if ((config->ff_disable & RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO) != 0) {
+		DRV_LOG(ERR,
+			"Disabled symmetric crypto feature is not supported.");
+		return -ENOTSUP;
+	}
 	if (mlx5_crypto_dek_setup(priv) != 0) {
 		DRV_LOG(ERR, "Dek hash list creation has failed.");
 		return -ENOMEM;
 	}
+	priv->dev_config = *config;
+	DRV_LOG(DEBUG, "Device %u was configured.", dev->driver_id);
 	return 0;
 }
 
@@ -73,9 +102,16 @@ mlx5_crypto_dev_close(struct rte_cryptodev *dev)
 	struct mlx5_crypto_priv *priv = dev->data->dev_private;
 
 	mlx5_crypto_dek_unset(priv);
+	DRV_LOG(DEBUG, "Device %u was closed.", dev->driver_id);
 	return 0;
 }
 
+static unsigned int
+mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
+{
+	return sizeof(struct mlx5_crypto_session);
+}
+
 static int
 mlx5_crypto_sym_session_configure(struct rte_cryptodev *dev,
 				  struct rte_crypto_sym_xform *xform,
@@ -149,7 +185,7 @@ static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_start			= NULL,
 	.dev_stop			= NULL,
 	.dev_close			= mlx5_crypto_dev_close,
-	.dev_infos_get			= NULL,
+	.dev_infos_get			= mlx5_crypto_dev_infos_get,
 	.stats_get			= NULL,
 	.stats_reset			= NULL,
 	.queue_pair_setup		= NULL,
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 4ec67a7e0f..5e270d3d5a 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -24,6 +24,7 @@ struct mlx5_crypto_priv {
 	uint32_t pdn; /* Protection Domain number. */
 	struct ibv_pd *pd;
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
+	struct rte_cryptodev_config dev_config;
 };
 
 struct mlx5_crypto_dek {
-- 
2.21.0

[dpdk-dev] [PATCH 20/24] crypto/mlx5: support queue pairs operations

[Shiri Kuzin]

The HW queue pairs are a pair of send queue and receive queue of
independent work queues packed together in one object for the purpose
of transferring data between nodes of a network.

Completion Queue is a FIFO queue of completed work requests.

In crypto driver we use one QP in loopback in order to encrypt and
decrypt data locally without sending it to the wire.
In the configured QP we only use the SQ to preform the encryption and
decryption operations.

Added implementation for the QP setup function which creates the CQ,
creates the QP and changes its state to RTS (ready to send).

Added implementation for the release QP function to release all the QP
resources.

Added the ops structure that contains any operation which is supported
by the cryptodev.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 124 +++++++++++++++++++++++++++++-
 drivers/crypto/mlx5/mlx5_crypto.h |  11 +++
 2 files changed, 133 insertions(+), 2 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 3807a01357..07bc6e3a1c 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -7,6 +7,7 @@
 #include <rte_errno.h>
 #include <rte_log.h>
 #include <rte_pci.h>
+#include <rte_memory.h>
 
 #include <mlx5_glue.h>
 #include <mlx5_common.h>
@@ -180,6 +181,125 @@ mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev,
 	DRV_LOG(DEBUG, "Session %p was cleared.", sess_private_data);
 }
 
+static int
+mlx5_crypto_queue_pair_release(struct rte_cryptodev *dev, uint16_t qp_id)
+{
+	struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
+
+	if (qp->qp_obj != NULL)
+		claim_zero(mlx5_devx_cmd_destroy(qp->qp_obj));
+	if (qp->umem_obj != NULL)
+		claim_zero(mlx5_glue->devx_umem_dereg(qp->umem_obj));
+	if (qp->umem_buf != NULL)
+		rte_free(qp->umem_buf);
+	mlx5_devx_cq_destroy(&qp->cq_obj);
+	rte_free(qp);
+	dev->data->queue_pairs[qp_id] = NULL;
+	return 0;
+}
+
+static int
+mlx5_crypto_qp2rts(struct mlx5_crypto_qp *qp)
+{
+	/*
+	 * In Order to configure self loopback, when calling these functions the
+	 * remote QP id that is used is the id of the same QP.
+	 */
+	if (mlx5_devx_cmd_modify_qp_state(qp->qp_obj, MLX5_CMD_OP_RST2INIT_QP,
+					  qp->qp_obj->id)) {
+		DRV_LOG(ERR, "Failed to modify QP to INIT state(%u).",
+			rte_errno);
+		return -1;
+	}
+	if (mlx5_devx_cmd_modify_qp_state(qp->qp_obj, MLX5_CMD_OP_INIT2RTR_QP,
+					  qp->qp_obj->id)) {
+		DRV_LOG(ERR, "Failed to modify QP to RTR state(%u).",
+			rte_errno);
+		return -1;
+	}
+	if (mlx5_devx_cmd_modify_qp_state(qp->qp_obj, MLX5_CMD_OP_RTR2RTS_QP,
+					  qp->qp_obj->id)) {
+		DRV_LOG(ERR, "Failed to modify QP to RTS state(%u).",
+			rte_errno);
+		return -1;
+	}
+	return 0;
+}
+
+static int
+mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
+			     const struct rte_cryptodev_qp_conf *qp_conf,
+			     int socket_id)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+	struct mlx5_devx_qp_attr attr = {0};
+	struct mlx5_crypto_qp *qp;
+	uint16_t log_nb_desc = rte_log2_u32(qp_conf->nb_descriptors);
+	uint32_t umem_size = RTE_BIT32(log_nb_desc) *
+			      MLX5_CRYPTO_WQE_SET_SIZE +
+			      sizeof(*qp->db_rec) * 2;
+	uint32_t alloc_size = sizeof(*qp);
+	struct mlx5_devx_cq_attr cq_attr = {
+		.uar_page_id = mlx5_os_get_devx_uar_page_id(priv->uar),
+	};
+
+	alloc_size = RTE_ALIGN(alloc_size, RTE_CACHE_LINE_SIZE);
+	alloc_size += sizeof(struct rte_crypto_op *) * RTE_BIT32(log_nb_desc);
+	qp = rte_zmalloc_socket(__func__, alloc_size, RTE_CACHE_LINE_SIZE,
+				socket_id);
+	if (qp == NULL) {
+		DRV_LOG(ERR, "Failed to allocate QP memory.");
+		rte_errno = ENOMEM;
+		return -rte_errno;
+	}
+	if (mlx5_devx_cq_create(priv->ctx, &qp->cq_obj, log_nb_desc,
+				&cq_attr, socket_id) != 0) {
+		DRV_LOG(ERR, "Failed to create CQ.");
+		goto error;
+	}
+	qp->umem_buf = rte_zmalloc_socket(__func__, umem_size, 4096, socket_id);
+	if (qp->umem_buf == NULL) {
+		DRV_LOG(ERR, "Failed to allocate QP umem.");
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	qp->umem_obj = mlx5_glue->devx_umem_reg(priv->ctx,
+					       (void *)(uintptr_t)qp->umem_buf,
+					       umem_size,
+					       IBV_ACCESS_LOCAL_WRITE);
+	if (qp->umem_obj == NULL) {
+		DRV_LOG(ERR, "Failed to register QP umem.");
+		goto error;
+	}
+	attr.pd = priv->pdn;
+	attr.uar_index = mlx5_os_get_devx_uar_page_id(priv->uar);
+	attr.cqn = qp->cq_obj.cq->id;
+	attr.log_page_size = rte_log2_u32(sysconf(_SC_PAGESIZE));
+	attr.rq_size =  0;
+	attr.sq_size = RTE_BIT32(log_nb_desc);
+	attr.dbr_umem_valid = 1;
+	attr.wq_umem_id = qp->umem_obj->umem_id;
+	attr.wq_umem_offset = 0;
+	attr.dbr_umem_id = qp->umem_obj->umem_id;
+	attr.dbr_address = RTE_BIT64(log_nb_desc) *
+			   MLX5_CRYPTO_WQE_SET_SIZE;
+	qp->qp_obj = mlx5_devx_cmd_create_qp(priv->ctx, &attr);
+	if (qp->qp_obj == NULL) {
+		DRV_LOG(ERR, "Failed to create QP(%u).", rte_errno);
+		goto error;
+	}
+	qp->db_rec = RTE_PTR_ADD(qp->umem_buf, (uintptr_t)attr.dbr_address);
+	if (mlx5_crypto_qp2rts(qp))
+		goto error;
+	qp->ops = (struct rte_crypto_op **)RTE_ALIGN((uintptr_t)(qp + 1),
+							   RTE_CACHE_LINE_SIZE);
+	dev->data->queue_pairs[qp_id] = qp;
+	return 0;
+error:
+	mlx5_crypto_queue_pair_release(dev, qp_id);
+	return -1;
+}
+
 static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_configure			= mlx5_crypto_dev_configure,
 	.dev_start			= NULL,
@@ -188,8 +308,8 @@ static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_infos_get			= mlx5_crypto_dev_infos_get,
 	.stats_get			= NULL,
 	.stats_reset			= NULL,
-	.queue_pair_setup		= NULL,
-	.queue_pair_release		= NULL,
+	.queue_pair_setup		= mlx5_crypto_queue_pair_setup,
+	.queue_pair_release		= mlx5_crypto_queue_pair_release,
 	.sym_session_get_size		= mlx5_crypto_sym_session_get_size,
 	.sym_session_configure		= mlx5_crypto_sym_session_configure,
 	.sym_session_clear		= mlx5_crypto_sym_session_clear,
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 5e270d3d5a..f5313b89f2 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -11,9 +11,11 @@
 #include <rte_cryptodev_pmd.h>
 
 #include <mlx5_common_utils.h>
+#include <mlx5_common_devx.h>
 
 #define MLX5_CRYPTO_DEK_HTABLE_SZ (1 << 11)
 #define MLX5_CRYPTO_KEY_LENGTH 80
+#define MLX5_CRYPTO_WQE_SET_SIZE 1024
 
 struct mlx5_crypto_priv {
 	TAILQ_ENTRY(mlx5_crypto_priv) next;
@@ -27,6 +29,15 @@ struct mlx5_crypto_priv {
 	struct rte_cryptodev_config dev_config;
 };
 
+struct mlx5_crypto_qp {
+	struct mlx5_devx_cq cq_obj;
+	struct mlx5_devx_obj *qp_obj;
+	struct mlx5dv_devx_umem *umem_obj;
+	void *umem_buf;
+	volatile uint32_t *db_rec;
+	struct rte_crypto_op **ops;
+};
+
 struct mlx5_crypto_dek {
 	struct mlx5_hlist_entry entry; /* Pointer to DEK hash list entry. */
 	struct mlx5_devx_obj *obj; /* Pointer to DEK DevX object. */
-- 
2.21.0

[dpdk-dev] [PATCH 21/24] crypto/mlx5: add dev stop and start operations

[Shiri Kuzin]

Add the dev_start function that is used to start a configured device.
Add the dev_stop function that is used to stop a configured device.

Both functions set the dev parameter as used and return 0.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 07bc6e3a1c..cb5716ba2a 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -97,6 +97,19 @@ mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
 	return 0;
 }
 
+static void
+mlx5_crypto_dev_stop(struct rte_cryptodev *dev)
+{
+	RTE_SET_USED(dev);
+}
+
+static int
+mlx5_crypto_dev_start(struct rte_cryptodev *dev)
+{
+	RTE_SET_USED(dev);
+	return 0;
+}
+
 static int
 mlx5_crypto_dev_close(struct rte_cryptodev *dev)
 {
@@ -302,8 +315,8 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 
 static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_configure			= mlx5_crypto_dev_configure,
-	.dev_start			= NULL,
-	.dev_stop			= NULL,
+	.dev_start			= mlx5_crypto_dev_start,
+	.dev_stop			= mlx5_crypto_dev_stop,
 	.dev_close			= mlx5_crypto_dev_close,
 	.dev_infos_get			= mlx5_crypto_dev_infos_get,
 	.stats_get			= NULL,
-- 
2.21.0

[dpdk-dev] [PATCH 22/24] crypto/mlx5: add memory region management

[Shiri Kuzin]

Mellanox user space drivers don't deal with physical addresses as part
of a memory protection mechanism.
The device translates the given virtual address to a physical address
using the given memory key as an address space identifier.
That's why any mbuf virtual address is moved directly to the HW
descriptor(WQE).

The mapping between the virtual address to the physical address is saved
in MR configured by the kernel to the HW.

Each MR has a key that should also be moved to the WQE by the SW.

When the SW sees an unmapped address, it extends the address range and
creates a MR using a system call.

Add memory region cache management:
	- 2 level cache per queue-pair - no locks.
	- 1 shared cache between all the queues using a lock.

Using this way, the MR key search per data-path address is optimized.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 20 ++++++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto.h |  3 +++
 2 files changed, 23 insertions(+)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index cb5716ba2a..f71de5a724 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -205,6 +205,7 @@ mlx5_crypto_queue_pair_release(struct rte_cryptodev *dev, uint16_t qp_id)
 		claim_zero(mlx5_glue->devx_umem_dereg(qp->umem_obj));
 	if (qp->umem_buf != NULL)
 		rte_free(qp->umem_buf);
+	mlx5_mr_btree_free(&qp->mr_ctrl.cache_bh);
 	mlx5_devx_cq_destroy(&qp->cq_obj);
 	rte_free(qp);
 	dev->data->queue_pairs[qp_id] = NULL;
@@ -284,6 +285,13 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 		DRV_LOG(ERR, "Failed to register QP umem.");
 		goto error;
 	}
+	if (mlx5_mr_btree_init(&qp->mr_ctrl.cache_bh, MLX5_MR_BTREE_CACHE_N,
+			       priv->dev_config.socket_id) != 0) {
+		DRV_LOG(ERR, "Cannot allocate MR Btree for qp %u.",
+			(uint32_t)qp_id);
+		rte_errno = ENOMEM;
+		goto error;
+	}
 	attr.pd = priv->pdn;
 	attr.uar_index = mlx5_os_get_devx_uar_page_id(priv->uar);
 	attr.cqn = qp->cq_obj.cq->id;
@@ -472,6 +480,17 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 		claim_zero(mlx5_glue->close_device(priv->ctx));
 		return -1;
 	}
+	if (mlx5_mr_btree_init(&priv->mr_scache.cache,
+			     MLX5_MR_BTREE_CACHE_N * 2, rte_socket_id()) != 0) {
+		DRV_LOG(ERR, "Failed to allocate shared cache MR memory.");
+		mlx5_crypto_hw_global_release(priv);
+		rte_cryptodev_pmd_destroy(priv->crypto_dev);
+		claim_zero(mlx5_glue->close_device(priv->ctx));
+		rte_errno = ENOMEM;
+		return -rte_errno;
+	}
+	priv->mr_scache.reg_mr_cb = mlx5_common_verbs_reg_mr;
+	priv->mr_scache.dereg_mr_cb = mlx5_common_verbs_dereg_mr;
 	pthread_mutex_lock(&priv_list_lock);
 	TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next);
 	pthread_mutex_unlock(&priv_list_lock);
@@ -491,6 +510,7 @@ mlx5_crypto_pci_remove(struct rte_pci_device *pdev)
 		TAILQ_REMOVE(&mlx5_crypto_priv_list, priv, next);
 	pthread_mutex_unlock(&priv_list_lock);
 	if (priv) {
+		mlx5_mr_release_cache(&priv->mr_scache);
 		mlx5_crypto_hw_global_release(priv);
 		rte_cryptodev_pmd_destroy(priv->crypto_dev);
 		claim_zero(mlx5_glue->close_device(priv->ctx));
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index f5313b89f2..397267d249 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -12,6 +12,7 @@
 
 #include <mlx5_common_utils.h>
 #include <mlx5_common_devx.h>
+#include <mlx5_common_mr.h>
 
 #define MLX5_CRYPTO_DEK_HTABLE_SZ (1 << 11)
 #define MLX5_CRYPTO_KEY_LENGTH 80
@@ -27,6 +28,7 @@ struct mlx5_crypto_priv {
 	struct ibv_pd *pd;
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
 	struct rte_cryptodev_config dev_config;
+	struct mlx5_mr_share_cache mr_scache; /* Global shared MR cache. */
 };
 
 struct mlx5_crypto_qp {
@@ -36,6 +38,7 @@ struct mlx5_crypto_qp {
 	void *umem_buf;
 	volatile uint32_t *db_rec;
 	struct rte_crypto_op **ops;
+	struct mlx5_mr_ctrl mr_ctrl;
 };
 
 struct mlx5_crypto_dek {
-- 
2.21.0

[dpdk-dev] [PATCH 23/24] crypto/mlx5: create login object using DevX

[Shiri Kuzin]

To work with crypto engines that are marked with wrapped_import_method,
a login session is required.
A crypto login object needs to be created using DevX.

The crypto login object contains:
	- The credential pointer.
	- The import_KEK pointer to be used for all secured information
	  communicated in crypto commands (key fields), including the
	  provided credential in this command.
	- The credential secret, wrapped by the import_KEK indicated in
	  this command. Size includes 8 bytes IV for wrapping.

Added devargs for the required login values:
	- wcs_file - path to the file containing the credential.
	- import_kek_id - the import KEK pointer.
	- credential_id - the credential pointer.

Create the login DevX object in pci_probe function and destroy it in
pci_remove.
Destroying the crypto login object means logout.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 89 +++++++++++++++++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto.h |  1 +
 2 files changed, 90 insertions(+)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index f71de5a724..25a435a999 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -398,6 +398,87 @@ mlx5_crypto_hw_global_prepare(struct mlx5_crypto_priv *priv)
 	return 0;
 }
 
+
+static int
+mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque)
+{
+	struct mlx5_devx_crypto_login_attr *attr = opaque;
+	unsigned long tmp;
+	FILE *file;
+	int ret;
+
+	if (strcmp(key, "class") == 0)
+		return 0;
+	if (strcmp(key, "wcs_file") == 0) {
+		file = fopen(val, "rb");
+		if (file == NULL) {
+			rte_errno = ENOTSUP;
+			return -rte_errno;
+		}
+		ret = fscanf(file, "%" RTE_STR(MLX5_CRYPTO_CREDENTIAL_SIZE) "s",
+			     &attr->credential[0]);
+		if (ret <= 0) {
+			fclose(file);
+			DRV_LOG(ERR, "Failed to read credential from file.");
+			rte_errno = EINVAL;
+			return -rte_errno;
+		}
+		fclose(file);
+		return 0;
+	}
+	errno = 0;
+	tmp = strtoul(val, NULL, 0);
+	if (errno) {
+		DRV_LOG(WARNING, "%s: \"%s\" is an invalid integer.", key, val);
+		return -errno;
+	}
+	if (strcmp(key, "import_kek_id") == 0)
+		attr->session_import_kek_ptr = (uint32_t)tmp;
+	else if (strcmp(key, "credential_id") == 0)
+		attr->credential_pointer = (uint32_t)tmp;
+	else
+		DRV_LOG(WARNING, "Invalid key %s.", key);
+	return 0;
+}
+
+static struct mlx5_devx_obj *
+mlx5_crypto_config_login(struct rte_devargs *devargs,
+			 struct ibv_context *ctx)
+{
+	/*
+	 * Set credential pointer and session import KEK pointer to a default
+	 * value of 0.
+	 */
+	struct mlx5_devx_crypto_login_attr attr = {
+			.credential_pointer = 0,
+			.session_import_kek_ptr = 0,
+	};
+	struct rte_kvargs *kvlist;
+
+	if (devargs == NULL) {
+		DRV_LOG(ERR,
+	"No login devargs in order to enable crypto operations in the device.");
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	kvlist = rte_kvargs_parse(devargs->args, NULL);
+	if (kvlist == NULL) {
+		DRV_LOG(ERR, "Failed to parse devargs.");
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	rte_kvargs_process(kvlist, NULL, mlx5_crypto_args_check_handler, &attr);
+	rte_kvargs_free(kvlist);
+	if (attr.credential == NULL) {
+		DRV_LOG(ERR,
+	"No login credential devarg in order to enable crypto operations "
+	"in the device.");
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	return mlx5_devx_cmd_create_crypto_login_obj(ctx, &attr);
+}
+
 /**
  * DPDK callback to register a PCI device.
  *
@@ -419,6 +500,7 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	struct ibv_device *ibv;
 	struct rte_cryptodev *crypto_dev;
 	struct ibv_context *ctx;
+	struct mlx5_devx_obj *login;
 	struct mlx5_crypto_priv *priv;
 	struct mlx5_hca_attr attr = { 0 };
 	struct rte_cryptodev_pmd_init_params init_params = {
@@ -457,6 +539,11 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 		rte_errno = ENOTSUP;
 		return -ENOTSUP;
 	}
+	login = mlx5_crypto_config_login(pci_dev->device.devargs, ctx);
+	if (login == NULL) {
+		DRV_LOG(ERR, "Failed to configure login.");
+		return -rte_errno;
+	}
 	crypto_dev = rte_cryptodev_pmd_create(ibv->name, &pci_dev->device,
 					&init_params);
 	if (crypto_dev == NULL) {
@@ -473,6 +560,7 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	crypto_dev->driver_id = mlx5_crypto_driver_id;
 	priv = crypto_dev->data->dev_private;
 	priv->ctx = ctx;
+	priv->login_obj = login;
 	priv->pci_dev = pci_dev;
 	priv->crypto_dev = crypto_dev;
 	if (mlx5_crypto_hw_global_prepare(priv) != 0) {
@@ -513,6 +601,7 @@ mlx5_crypto_pci_remove(struct rte_pci_device *pdev)
 		mlx5_mr_release_cache(&priv->mr_scache);
 		mlx5_crypto_hw_global_release(priv);
 		rte_cryptodev_pmd_destroy(priv->crypto_dev);
+		claim_zero(mlx5_devx_cmd_destroy(priv->login_obj));
 		claim_zero(mlx5_glue->close_device(priv->ctx));
 	}
 	return 0;
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 397267d249..0056d9e3e8 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -29,6 +29,7 @@ struct mlx5_crypto_priv {
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
 	struct rte_cryptodev_config dev_config;
 	struct mlx5_mr_share_cache mr_scache; /* Global shared MR cache. */
+	struct mlx5_devx_obj *login_obj;
 };
 
 struct mlx5_crypto_qp {
-- 
2.21.0

[dpdk-dev] [PATCH 24/24] crypto/mlx5: adjust to the multiple data unit API

[Shiri Kuzin]

In AES-XTS the data to be encrypted\decrypted does not have to be
in multiples of 16B size, the unit of data is called data-unit.

As a result of patch [1] a new field is added to the cipher capability,
called dataunit_set, where the devices can report the range of
supported data-unit sizes.

The new field enables saving the data-unit size in the session
structure to the block size pointer variable in order to support
several data-unit sizes.

[1] https://www.mail-archive.com/dev@dpdk.org/msg203590.html

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 25a435a999..8810872aff 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -48,6 +48,11 @@ struct mlx5_crypto_session {
 	 * bsf_size, bsf_p_type, encryption_order and encryption standard,
 	 * saved in big endian format.
 	 */
+	uint32_t bsp_res;
+	/*
+	 * crypto_block_size_pointer and reserved 24 bits saved in big endian
+	 * format.
+	 */
 	uint32_t iv_offset:16;
 	/* Starting point for Initialisation Vector. */
 	struct mlx5_crypto_dek *dek; /* Pointer to dek struct. */
@@ -170,6 +175,26 @@ mlx5_crypto_sym_session_configure(struct rte_cryptodev *dev,
 			 MLX5_BSF_P_TYPE_CRYPTO << MLX5_BSF_P_TYPE_OFFSET |
 			 encryption_order << MLX5_ENCRYPTION_ORDER_OFFSET |
 			 MLX5_ENCRYPTION_STANDARD_AES_XTS);
+	switch (xform->cipher.dataunit_len) {
+	case RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_512_BYTES:
+		sess_private_data->bsp_res = rte_cpu_to_be_32
+					     ((uint32_t)MLX5_BLOCK_SIZE_512B <<
+					     MLX5_BLOCK_SIZE_OFFSET);
+		break;
+	case RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_4096_BYTES:
+		sess_private_data->bsp_res = rte_cpu_to_be_32
+					     ((uint32_t)MLX5_BLOCK_SIZE_4096B <<
+					     MLX5_BLOCK_SIZE_OFFSET);
+		break;
+	case RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_1M_BYTES:
+		sess_private_data->bsp_res = rte_cpu_to_be_32
+					     ((uint32_t)MLX5_BLOCK_SIZE_1MB <<
+					     MLX5_BLOCK_SIZE_OFFSET);
+		break;
+	default:
+		DRV_LOG(ERR, "Cipher data unit length is not supported.");
+		return -ENOTSUP;
+	}
 	sess_private_data->iv_offset = cipher->iv.offset;
 	set_sym_session_private_data(session, dev->driver_id,
 				     sess_private_data);
-- 
2.21.0

Re: [dpdk-dev] [EXT] [PATCH 00/24] drivers: introduce mlx5 crypto PMD

[Akhil Goyal]

Hi Matan/Shiri
> Dekel Peled (13):
>   common/mlx5: remove redundant spaces in header file
>   common/mlx5: update GENEVE TLV OPT obj name
>   common/mlx5: optimize read of general obj type caps
>   common/mlx5: add HCA cap for AES-XTS crypto
>   common/mlx5: support general object DEK create op
>   common/mlx5: add crypto en field to MKEY context
>   common/mlx5: support umr en field in MKEY context
>   common/mlx5: support general obj IMPORT KEK create
>   common/mlx5: support general obj CRYPTO LOGIN create
>   common/mlx5: add crypto BSF struct and defines
>   common/mlx5: support general obj CREDENTIAL create
>   common/mlx5: add crypto register structs and defs
>   common/mlx5: support register write access
> 
> Shiri Kuzin (11):
>   common/mlx5: share hash list tool
>   common/mlx5: share get ib device match function
>   drivers: introduce mlx5 crypto PMD
>   crypto/mlx5: add DEK object management
>   crypto/mlx5: support session operations
>   crypto/mlx5: add basic operations
>   crypto/mlx5: support queue pairs operations
>   crypto/mlx5: add dev stop and start operations
>   crypto/mlx5: add memory region management
>   crypto/mlx5: create login object using DevX
>   crypto/mlx5: adjust to the multiple data unit API
> 
I would recommend this series be split into 2.
1. non-crypto code for common/mlx5 for next-net-mlx tree.
Eg: common/mlx5: remove redundant spaces in header file
       common/mlx5: update GENEVE TLV OPT obj name
       common/mlx5: optimize read of general obj type caps
       common/mlx5: support register write access
2. crypto PMD and crypto related changes in common/mlx5 for crypto tree.

Regards,
Akhil

[dpdk-dev] [PATCH v2 00/16] mlx5 common part for crypto driver

[Matan Azrad]

The crypto PMD will be supported on Nvidia ConnectX6
The crypto PMD will add the support of encryption and decryption using the AES-XTS symmetric algorithm.
The crypto PMD requires rdma-core and uses mlx5 DevX.
This is the mlx5 common part that added support for DevX commands needed for crypto driver.
Also includes PRM definitions.

v2:
Add data-path part.

Dekel Peled (12):
  common/mlx5: remove redundant spaces in header file
  common/mlx5: update GENEVE TLV OPT obj name
  common/mlx5: optimize read of general obj type caps
  common/mlx5: add HCA cap for AES-XTS crypto
  common/mlx5: support general object DEK create op
  common/mlx5: adjust DevX MKEY fields for crypto
  common/mlx5: support general obj IMPORT KEK create
  common/mlx5: support general obj CRYPTO LOGIN create
  common/mlx5: add crypto BSF struct and defines
  common/mlx5: support general obj CREDENTIAL create
  common/mlx5: add crypto register structs and defs
  common/mlx5: support register write access

Shiri Kuzin (2):
  common/mlx5: share hash list tool
  common/mlx5: share get ib device match function

Suanming Mou (2):
  common/mlx5: fix cypto bsf attr
  common/mlx5: add UMR and RDMA write WQE defines

 drivers/common/mlx5/linux/mlx5_common_os.c   |  30 +-
 drivers/common/mlx5/linux/mlx5_common_os.h   |   5 +
 drivers/common/mlx5/linux/mlx5_nl.c          |   2 +-
 drivers/common/mlx5/meson.build              |  15 +-
 drivers/common/mlx5/mlx5_common.c            |   2 +-
 drivers/common/mlx5/mlx5_common.h            |   2 +
 drivers/common/mlx5/mlx5_common_devx.c       |   2 +-
 drivers/common/mlx5/mlx5_common_log.h        |  21 +
 drivers/common/mlx5/mlx5_common_mp.c         |   2 +-
 drivers/common/mlx5/mlx5_common_mr.c         |   2 +-
 drivers/common/mlx5/mlx5_common_pci.c        |   4 +-
 drivers/common/mlx5/mlx5_common_utils.c      | 221 +++++++
 drivers/common/mlx5/mlx5_common_utils.h      | 202 +++++-
 drivers/common/mlx5/mlx5_devx_cmds.c         | 312 +++++++++-
 drivers/common/mlx5/mlx5_devx_cmds.h         |  66 ++
 drivers/common/mlx5/mlx5_malloc.c            |   2 +-
 drivers/common/mlx5/mlx5_prm.h               | 610 +++++++++++++------
 drivers/common/mlx5/version.map              |  24 +-
 drivers/common/mlx5/windows/mlx5_common_os.c |   2 +-
 drivers/common/mlx5/windows/mlx5_glue.c      |   2 +-
 drivers/compress/mlx5/mlx5_compress.c        |  30 +-
 drivers/net/mlx5/mlx5_utils.c                | 209 -------
 drivers/net/mlx5/mlx5_utils.h                | 194 +-----
 drivers/regex/mlx5/mlx5_regex.c              |  30 +-
 drivers/vdpa/mlx5/mlx5_vdpa.c                |  32 +-
 25 files changed, 1317 insertions(+), 706 deletions(-)
 create mode 100644 drivers/common/mlx5/mlx5_common_log.h
 create mode 100644 drivers/common/mlx5/mlx5_common_utils.c

-- 
2.25.1

[dpdk-dev] [PATCH v2 01/16] common/mlx5: remove redundant spaces in header file

[Matan Azrad]

From: Dekel Peled <dekelp@nvidia.com>

File drivers/common/mlx5/mlx5_prm.h includes structs representing
data items as defined in PRM document.
Some of these structs were copied as-is from kernel file mlx5_ifc.h.
As result the structs are not all aligned with the same spacing.

This patch removes redundant spaces and new lines from several structs,
to align all structs in mlx5_prm.h to the same format.

Signed-off-by: Dekel Peled <dekelp@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_prm.h | 334 +++++++++++++++------------------
 1 file changed, 155 insertions(+), 179 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index efa5ae67bf..da1510ac1e 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -953,163 +953,139 @@ enum {
 
 /* Flow counters. */
 struct mlx5_ifc_alloc_flow_counter_out_bits {
-	u8         status[0x8];
-	u8         reserved_at_8[0x18];
-	u8         syndrome[0x20];
-	u8         flow_counter_id[0x20];
-	u8         reserved_at_60[0x20];
+	u8 status[0x8];
+	u8 reserved_at_8[0x18];
+	u8 syndrome[0x20];
+	u8 flow_counter_id[0x20];
+	u8 reserved_at_60[0x20];
 };
 
 struct mlx5_ifc_alloc_flow_counter_in_bits {
-	u8         opcode[0x10];
-	u8         reserved_at_10[0x10];
-	u8         reserved_at_20[0x10];
-	u8         op_mod[0x10];
-	u8         flow_counter_id[0x20];
-	u8         reserved_at_40[0x18];
-	u8         flow_counter_bulk[0x8];
+	u8 opcode[0x10];
+	u8 reserved_at_10[0x10];
+	u8 reserved_at_20[0x10];
+	u8 op_mod[0x10];
+	u8 flow_counter_id[0x20];
+	u8 reserved_at_40[0x18];
+	u8 flow_counter_bulk[0x8];
 };
 
 struct mlx5_ifc_dealloc_flow_counter_out_bits {
-	u8         status[0x8];
-	u8         reserved_at_8[0x18];
-	u8         syndrome[0x20];
-	u8         reserved_at_40[0x40];
+	u8 status[0x8];
+	u8 reserved_at_8[0x18];
+	u8 syndrome[0x20];
+	u8 reserved_at_40[0x40];
 };
 
 struct mlx5_ifc_dealloc_flow_counter_in_bits {
-	u8         opcode[0x10];
-	u8         reserved_at_10[0x10];
-	u8         reserved_at_20[0x10];
-	u8         op_mod[0x10];
-	u8         flow_counter_id[0x20];
-	u8         reserved_at_60[0x20];
+	u8 opcode[0x10];
+	u8 reserved_at_10[0x10];
+	u8 reserved_at_20[0x10];
+	u8 op_mod[0x10];
+	u8 flow_counter_id[0x20];
+	u8 reserved_at_60[0x20];
 };
 
 struct mlx5_ifc_traffic_counter_bits {
-	u8         packets[0x40];
-	u8         octets[0x40];
+	u8 packets[0x40];
+	u8 octets[0x40];
 };
 
 struct mlx5_ifc_query_flow_counter_out_bits {
-	u8         status[0x8];
-	u8         reserved_at_8[0x18];
-	u8         syndrome[0x20];
-	u8         reserved_at_40[0x40];
+	u8 status[0x8];
+	u8 reserved_at_8[0x18];
+	u8 syndrome[0x20];
+	u8 reserved_at_40[0x40];
 	struct mlx5_ifc_traffic_counter_bits flow_statistics[];
 };
 
 struct mlx5_ifc_query_flow_counter_in_bits {
-	u8         opcode[0x10];
-	u8         reserved_at_10[0x10];
-	u8         reserved_at_20[0x10];
-	u8         op_mod[0x10];
-	u8         reserved_at_40[0x20];
-	u8         mkey[0x20];
-	u8         address[0x40];
-	u8         clear[0x1];
-	u8         dump_to_memory[0x1];
-	u8         num_of_counters[0x1e];
-	u8         flow_counter_id[0x20];
+	u8 opcode[0x10];
+	u8 reserved_at_10[0x10];
+	u8 reserved_at_20[0x10];
+	u8 op_mod[0x10];
+	u8 reserved_at_40[0x20];
+	u8 mkey[0x20];
+	u8 address[0x40];
+	u8 clear[0x1];
+	u8 dump_to_memory[0x1];
+	u8 num_of_counters[0x1e];
+	u8 flow_counter_id[0x20];
 };
 
 #define MLX5_MAX_KLM_BYTE_COUNT 0x80000000u
 #define MLX5_MIN_KLM_FIXED_BUFFER_SIZE 0x1000u
 
-
 struct mlx5_ifc_klm_bits {
-	u8         byte_count[0x20];
-	u8         mkey[0x20];
-	u8         address[0x40];
+	u8 byte_count[0x20];
+	u8 mkey[0x20];
+	u8 address[0x40];
 };
 
 struct mlx5_ifc_mkc_bits {
-	u8         reserved_at_0[0x1];
-	u8         free[0x1];
-	u8         reserved_at_2[0x1];
-	u8         access_mode_4_2[0x3];
-	u8         reserved_at_6[0x7];
-	u8         relaxed_ordering_write[0x1];
-	u8         reserved_at_e[0x1];
-	u8         small_fence_on_rdma_read_response[0x1];
-	u8         umr_en[0x1];
-	u8         a[0x1];
-	u8         rw[0x1];
-	u8         rr[0x1];
-	u8         lw[0x1];
-	u8         lr[0x1];
-	u8         access_mode_1_0[0x2];
-	u8         reserved_at_18[0x8];
-
-	u8         qpn[0x18];
-	u8         mkey_7_0[0x8];
-
-	u8         reserved_at_40[0x20];
-
-	u8         length64[0x1];
-	u8         bsf_en[0x1];
-	u8         sync_umr[0x1];
-	u8         reserved_at_63[0x2];
-	u8         expected_sigerr_count[0x1];
-	u8         reserved_at_66[0x1];
-	u8         en_rinval[0x1];
-	u8         pd[0x18];
-
-	u8         start_addr[0x40];
-
-	u8         len[0x40];
-
-	u8         bsf_octword_size[0x20];
-
-	u8         reserved_at_120[0x80];
-
-	u8         translations_octword_size[0x20];
-
-	u8         reserved_at_1c0[0x19];
-	u8		   relaxed_ordering_read[0x1];
-	u8		   reserved_at_1da[0x1];
-	u8         log_page_size[0x5];
-
-	u8         reserved_at_1e0[0x20];
+	u8 reserved_at_0[0x1];
+	u8 free[0x1];
+	u8 reserved_at_2[0x1];
+	u8 access_mode_4_2[0x3];
+	u8 reserved_at_6[0x7];
+	u8 relaxed_ordering_write[0x1];
+	u8 reserved_at_e[0x1];
+	u8 small_fence_on_rdma_read_response[0x1];
+	u8 umr_en[0x1];
+	u8 a[0x1];
+	u8 rw[0x1];
+	u8 rr[0x1];
+	u8 lw[0x1];
+	u8 lr[0x1];
+	u8 access_mode_1_0[0x2];
+	u8 reserved_at_18[0x8];
+	u8 qpn[0x18];
+	u8 mkey_7_0[0x8];
+	u8 reserved_at_40[0x20];
+	u8 length64[0x1];
+	u8 bsf_en[0x1];
+	u8 sync_umr[0x1];
+	u8 reserved_at_63[0x2];
+	u8 expected_sigerr_count[0x1];
+	u8 reserved_at_66[0x1];
+	u8 en_rinval[0x1];
+	u8 pd[0x18];
+	u8 start_addr[0x40];
+	u8 len[0x40];
+	u8 bsf_octword_size[0x20];
+	u8 reserved_at_120[0x80];
+	u8 translations_octword_size[0x20];
+	u8 reserved_at_1c0[0x19];
+	u8 relaxed_ordering_read[0x1];
+	u8 reserved_at_1da[0x1];
+	u8 log_page_size[0x5];
+	u8 reserved_at_1e0[0x20];
 };
 
 struct mlx5_ifc_create_mkey_out_bits {
-	u8         status[0x8];
-	u8         reserved_at_8[0x18];
-
-	u8         syndrome[0x20];
-
-	u8         reserved_at_40[0x8];
-	u8         mkey_index[0x18];
-
-	u8         reserved_at_60[0x20];
+	u8 status[0x8];
+	u8 reserved_at_8[0x18];
+	u8 syndrome[0x20];
+	u8 reserved_at_40[0x8];
+	u8 mkey_index[0x18];
+	u8 reserved_at_60[0x20];
 };
 
 struct mlx5_ifc_create_mkey_in_bits {
-	u8         opcode[0x10];
-	u8         reserved_at_10[0x10];
-
-	u8         reserved_at_20[0x10];
-	u8         op_mod[0x10];
-
-	u8         reserved_at_40[0x20];
-
-	u8         pg_access[0x1];
-	u8         reserved_at_61[0x1f];
-
+	u8 opcode[0x10];
+	u8 reserved_at_10[0x10];
+	u8 reserved_at_20[0x10];
+	u8 op_mod[0x10];
+	u8 reserved_at_40[0x20];
+	u8 pg_access[0x1];
+	u8 reserved_at_61[0x1f];
 	struct mlx5_ifc_mkc_bits memory_key_mkey_entry;
-
-	u8         reserved_at_280[0x80];
-
-	u8         translations_octword_actual_size[0x20];
-
-	u8         mkey_umem_id[0x20];
-
-	u8         mkey_umem_offset[0x40];
-
-	u8         reserved_at_380[0x500];
-
-	u8         klm_pas_mtt[][0x20];
+	u8 reserved_at_280[0x80];
+	u8 translations_octword_actual_size[0x20];
+	u8 mkey_umem_id[0x20];
+	u8 mkey_umem_offset[0x40];
+	u8 reserved_at_380[0x500];
+	u8 klm_pas_mtt[][0x20];
 };
 
 enum {
@@ -2272,27 +2248,27 @@ enum {
 };
 
 struct mlx5_ifc_flow_meter_parameters_bits {
-	u8         valid[0x1];			// 00h
-	u8         bucket_overflow[0x1];
-	u8         start_color[0x2];
-	u8         both_buckets_on_green[0x1];
-	u8         meter_mode[0x2];
-	u8         reserved_at_1[0x19];
-	u8         reserved_at_2[0x20]; //04h
-	u8         reserved_at_3[0x3];
-	u8         cbs_exponent[0x5];		// 08h
-	u8         cbs_mantissa[0x8];
-	u8         reserved_at_4[0x3];
-	u8         cir_exponent[0x5];
-	u8         cir_mantissa[0x8];
-	u8         reserved_at_5[0x20];		// 0Ch
-	u8         reserved_at_6[0x3];
-	u8         ebs_exponent[0x5];		// 10h
-	u8         ebs_mantissa[0x8];
-	u8         reserved_at_7[0x3];
-	u8         eir_exponent[0x5];
-	u8         eir_mantissa[0x8];
-	u8         reserved_at_8[0x60];		// 14h-1Ch
+	u8 valid[0x1];
+	u8 bucket_overflow[0x1];
+	u8 start_color[0x2];
+	u8 both_buckets_on_green[0x1];
+	u8 meter_mode[0x2];
+	u8 reserved_at_1[0x19];
+	u8 reserved_at_2[0x20];
+	u8 reserved_at_3[0x3];
+	u8 cbs_exponent[0x5];
+	u8 cbs_mantissa[0x8];
+	u8 reserved_at_4[0x3];
+	u8 cir_exponent[0x5];
+	u8 cir_mantissa[0x8];
+	u8 reserved_at_5[0x20];
+	u8 reserved_at_6[0x3];
+	u8 ebs_exponent[0x5];
+	u8 ebs_mantissa[0x8];
+	u8 reserved_at_7[0x3];
+	u8 eir_exponent[0x5];
+	u8 eir_mantissa[0x8];
+	u8 reserved_at_8[0x60];
 };
 #define MLX5_IFC_FLOW_METER_PARAM_MASK UINT64_C(0x80FFFFFF)
 #define MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL 0x14BF00C8
@@ -2359,46 +2335,46 @@ struct mlx5_ifc_cqc_bits {
 };
 
 struct mlx5_ifc_health_buffer_bits {
-	u8         reserved_0[0x100];
-	u8         assert_existptr[0x20];
-	u8         assert_callra[0x20];
-	u8         reserved_1[0x40];
-	u8         fw_version[0x20];
-	u8         hw_id[0x20];
-	u8         reserved_2[0x20];
-	u8         irisc_index[0x8];
-	u8         synd[0x8];
-	u8         ext_synd[0x10];
+	u8 reserved_0[0x100];
+	u8 assert_existptr[0x20];
+	u8 assert_callra[0x20];
+	u8 reserved_1[0x40];
+	u8 fw_version[0x20];
+	u8 hw_id[0x20];
+	u8 reserved_2[0x20];
+	u8 irisc_index[0x8];
+	u8 synd[0x8];
+	u8 ext_synd[0x10];
 };
 
 struct mlx5_ifc_initial_seg_bits {
-	u8         fw_rev_minor[0x10];
-	u8         fw_rev_major[0x10];
-	u8         cmd_interface_rev[0x10];
-	u8         fw_rev_subminor[0x10];
-	u8         reserved_0[0x40];
-	u8         cmdq_phy_addr_63_32[0x20];
-	u8         cmdq_phy_addr_31_12[0x14];
-	u8         reserved_1[0x2];
-	u8         nic_interface[0x2];
-	u8         log_cmdq_size[0x4];
-	u8         log_cmdq_stride[0x4];
-	u8         command_doorbell_vector[0x20];
-	u8         reserved_2[0xf00];
-	u8         initializing[0x1];
-	u8         nic_interface_supported[0x7];
-	u8         reserved_4[0x18];
+	u8 fw_rev_minor[0x10];
+	u8 fw_rev_major[0x10];
+	u8 cmd_interface_rev[0x10];
+	u8 fw_rev_subminor[0x10];
+	u8 reserved_0[0x40];
+	u8 cmdq_phy_addr_63_32[0x20];
+	u8 cmdq_phy_addr_31_12[0x14];
+	u8 reserved_1[0x2];
+	u8 nic_interface[0x2];
+	u8 log_cmdq_size[0x4];
+	u8 log_cmdq_stride[0x4];
+	u8 command_doorbell_vector[0x20];
+	u8 reserved_2[0xf00];
+	u8 initializing[0x1];
+	u8 nic_interface_supported[0x7];
+	u8 reserved_4[0x18];
 	struct mlx5_ifc_health_buffer_bits health_buffer;
-	u8         no_dram_nic_offset[0x20];
-	u8         reserved_5[0x6de0];
-	u8         internal_timer_h[0x20];
-	u8         internal_timer_l[0x20];
-	u8         reserved_6[0x20];
-	u8         reserved_7[0x1f];
-	u8         clear_int[0x1];
-	u8         health_syndrome[0x8];
-	u8         health_counter[0x18];
-	u8         reserved_8[0x17fc0];
+	u8 no_dram_nic_offset[0x20];
+	u8 reserved_5[0x6de0];
+	u8 internal_timer_h[0x20];
+	u8 internal_timer_l[0x20];
+	u8 reserved_6[0x20];
+	u8 reserved_7[0x1f];
+	u8 clear_int[0x1];
+	u8 health_syndrome[0x8];
+	u8 health_counter[0x18];
+	u8 reserved_8[0x17fc0];
 };
 
 struct mlx5_ifc_create_cq_out_bits {
-- 
2.25.1

[dpdk-dev] [PATCH v2 02/16] common/mlx5: update GENEVE TLV OPT obj name

[Matan Azrad]

From: Dekel Peled <dekelp@nvidia.com>

Rename MLX5_OBJ_TYPE_GENEVE_TLV_OPT as
MLX5_GENERAL_OBJ_TYPE_GENEVE_TLV_OPT, to align with other general
objects names.

Signed-off-by: Dekel Peled <dekelp@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_devx_cmds.c | 2 +-
 drivers/common/mlx5/mlx5_prm.h       | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c
index 79fff6457c..831175efc5 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.c
+++ b/drivers/common/mlx5/mlx5_devx_cmds.c
@@ -2265,7 +2265,7 @@ mlx5_devx_cmd_create_geneve_tlv_option(void *ctx,
 	MLX5_SET(general_obj_in_cmd_hdr, hdr, opcode,
 			MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
 	MLX5_SET(general_obj_in_cmd_hdr, hdr, obj_type,
-			MLX5_OBJ_TYPE_GENEVE_TLV_OPT);
+		 MLX5_GENERAL_OBJ_TYPE_GENEVE_TLV_OPT);
 	MLX5_SET(geneve_tlv_option, opt, option_class,
 			rte_be_to_cpu_16(class));
 	MLX5_SET(geneve_tlv_option, opt, option_type, type);
diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index da1510ac1e..2e5e42f6e9 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -1108,7 +1108,7 @@ enum {
 #define MLX5_GENERAL_OBJ_TYPES_CAP_FLOW_METER_ASO \
 			(1ULL << MLX5_GENERAL_OBJ_TYPE_FLOW_METER_ASO)
 #define MLX5_GENERAL_OBJ_TYPES_CAP_GENEVE_TLV_OPT \
-			(1ULL << MLX5_OBJ_TYPE_GENEVE_TLV_OPT)
+			(1ULL << MLX5_GENERAL_OBJ_TYPE_GENEVE_TLV_OPT)
 
 enum {
 	MLX5_HCA_CAP_OPMOD_GET_MAX   = 0,
@@ -2402,7 +2402,7 @@ struct mlx5_ifc_create_cq_in_bits {
 };
 
 enum {
-	MLX5_OBJ_TYPE_GENEVE_TLV_OPT = 0x000b,
+	MLX5_GENERAL_OBJ_TYPE_GENEVE_TLV_OPT = 0x000b,
 	MLX5_GENERAL_OBJ_TYPE_VIRTQ = 0x000d,
 	MLX5_GENERAL_OBJ_TYPE_VIRTIO_Q_COUNTERS = 0x001c,
 	MLX5_GENERAL_OBJ_TYPE_FLEX_PARSE_GRAPH = 0x0022,
-- 
2.25.1

[dpdk-dev] [PATCH v2 03/16] common/mlx5: optimize read of general obj type caps

[Matan Azrad]

From: Dekel Peled <dekelp@nvidia.com>

General object types support is indicated in bitmap general_obj_types,
which is part of HCA capabilities list.
Currently this bitmap is read multiple times, and each time a different
bit is extracted.

This patch optimizes the code, reading the bitmap once into a local
variable, and then extracting the required bits.

Signed-off-by: Dekel Peled <dekelp@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_devx_cmds.c | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c
index 831175efc5..a0bf0d3009 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.c
+++ b/drivers/common/mlx5/mlx5_devx_cmds.c
@@ -647,6 +647,7 @@ mlx5_devx_cmd_query_hca_attr(void *ctx,
 	uint32_t out[MLX5_ST_SZ_DW(query_hca_cap_out)] = {0};
 	void *hcattr;
 	int status, syndrome, rc, i;
+	uint64_t general_obj_types_supported = 0;
 
 	MLX5_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP);
 	MLX5_SET(query_hca_cap_in, in, op_mod,
@@ -725,12 +726,22 @@ mlx5_devx_cmd_query_hca_attr(void *ctx,
 	attr->regex = MLX5_GET(cmd_hca_cap, hcattr, regexp);
 	attr->regexp_num_of_engines = MLX5_GET(cmd_hca_cap, hcattr,
 					       regexp_num_of_engines);
-	attr->flow_hit_aso = !!(MLX5_GET64(cmd_hca_cap, hcattr,
-					   general_obj_types) &
+	/* Read the general_obj_types bitmap and extract the relevant bits. */
+	general_obj_types_supported = MLX5_GET64(cmd_hca_cap, hcattr,
+						 general_obj_types);
+	attr->vdpa.valid = !!(general_obj_types_supported &
+			      MLX5_GENERAL_OBJ_TYPES_CAP_VIRTQ_NET_Q);
+	attr->vdpa.queue_counters_valid =
+			!!(general_obj_types_supported &
+			   MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_Q_COUNTERS);
+	attr->parse_graph_flex_node =
+			!!(general_obj_types_supported &
+			   MLX5_GENERAL_OBJ_TYPES_CAP_PARSE_GRAPH_FLEX_NODE);
+	attr->flow_hit_aso = !!(general_obj_types_supported &
 				MLX5_GENERAL_OBJ_TYPES_CAP_FLOW_HIT_ASO);
-	attr->geneve_tlv_opt = !!(MLX5_GET64(cmd_hca_cap, hcattr,
-					   general_obj_types) &
-				MLX5_GENERAL_OBJ_TYPES_CAP_GENEVE_TLV_OPT);
+	attr->geneve_tlv_opt = !!(general_obj_types_supported &
+				  MLX5_GENERAL_OBJ_TYPES_CAP_GENEVE_TLV_OPT);
+	/* Add reading of other GENERAL_OBJ_TYPES_CAP bits above this line. */
 	attr->log_max_cq = MLX5_GET(cmd_hca_cap, hcattr, log_max_cq);
 	attr->log_max_qp = MLX5_GET(cmd_hca_cap, hcattr, log_max_qp);
 	attr->log_max_cq_sz = MLX5_GET(cmd_hca_cap, hcattr, log_max_cq_sz);
-- 
2.25.1

[dpdk-dev] [PATCH v2 04/16] common/mlx5: add HCA cap for AES-XTS crypto

[Matan Azrad]

From: Dekel Peled <dekelp@nvidia.com>

Update the PRM structure and HCA capabilities reading, to include
relevant capabilities for AES-XTS crypto.

Signed-off-by: Dekel Peled <dekelp@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_devx_cmds.c | 3 +++
 drivers/common/mlx5/mlx5_devx_cmds.h | 2 ++
 drivers/common/mlx5/mlx5_prm.h       | 5 ++++-
 3 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c
index a0bf0d3009..7ca767944e 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.c
+++ b/drivers/common/mlx5/mlx5_devx_cmds.c
@@ -771,6 +771,9 @@ mlx5_devx_cmd_query_hca_attr(void *ctx,
 		MLX5_GET(cmd_hca_cap, hcattr, umr_indirect_mkey_disabled);
 	attr->umr_modify_entity_size_disabled =
 		MLX5_GET(cmd_hca_cap, hcattr, umr_modify_entity_size_disabled);
+	attr->crypto = MLX5_GET(cmd_hca_cap, hcattr, crypto);
+	if (attr->crypto)
+		attr->aes_xts = MLX5_GET(cmd_hca_cap, hcattr, aes_xts);
 	if (attr->qos.sup) {
 		MLX5_SET(query_hca_cap_in, in, op_mod,
 			 MLX5_GET_HCA_CAP_OP_MOD_QOS_CAP |
diff --git a/drivers/common/mlx5/mlx5_devx_cmds.h b/drivers/common/mlx5/mlx5_devx_cmds.h
index 870bdb6b30..28ade5bbc4 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.h
+++ b/drivers/common/mlx5/mlx5_devx_cmds.h
@@ -137,6 +137,8 @@ struct mlx5_hca_attr {
 	uint32_t qp_ts_format:2;
 	uint32_t regex:1;
 	uint32_t reg_c_preserve:1;
+	uint32_t crypto:1; /* Crypto engine is supported. */
+	uint32_t aes_xts:1; /* AES-XTS crypto is supported. */
 	uint32_t regexp_num_of_engines;
 	uint32_t log_max_ft_sampler_num:8;
 	uint32_t geneve_tlv_opt;
diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index 2e5e42f6e9..a8fbfbb0f5 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -1427,7 +1427,10 @@ struct mlx5_ifc_cmd_hca_cap_bits {
 	u8 sq_ts_format[0x2];
 	u8 rq_ts_format[0x2];
 	u8 reserved_at_444[0x1C];
-	u8 reserved_at_460[0x10];
+	u8 reserved_at_460[0x8];
+	u8 aes_xts[0x1];
+	u8 crypto[0x1];
+	u8 reserved_at_46a[0x6];
 	u8 max_num_eqs[0x10];
 	u8 reserved_at_480[0x3];
 	u8 log_max_l2_table[0x5];
-- 
2.25.1

[dpdk-dev] [PATCH v2 05/16] common/mlx5: support general object DEK create op

[Matan Azrad]

From: Dekel Peled <dekelp@nvidia.com>

Data Encryption Keys (DEKs) are the keys used for data
encryption/decryption operations.

Add reading of DEK support capability.
Add function to create general object type DEK, using DevX API.

Signed-off-by: Dekel Peled <dekelp@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_devx_cmds.c | 53 ++++++++++++++++++++++++++++
 drivers/common/mlx5/mlx5_devx_cmds.h | 17 +++++++++
 drivers/common/mlx5/mlx5_prm.h       | 39 ++++++++++++++++++++
 drivers/common/mlx5/version.map      |  1 +
 4 files changed, 110 insertions(+)

diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c
index 7ca767944e..742c82cca4 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.c
+++ b/drivers/common/mlx5/mlx5_devx_cmds.c
@@ -741,6 +741,8 @@ mlx5_devx_cmd_query_hca_attr(void *ctx,
 				MLX5_GENERAL_OBJ_TYPES_CAP_FLOW_HIT_ASO);
 	attr->geneve_tlv_opt = !!(general_obj_types_supported &
 				  MLX5_GENERAL_OBJ_TYPES_CAP_GENEVE_TLV_OPT);
+	attr->dek = !!(general_obj_types_supported &
+		       MLX5_GENERAL_OBJ_TYPES_CAP_DEK);
 	/* Add reading of other GENERAL_OBJ_TYPES_CAP bits above this line. */
 	attr->log_max_cq = MLX5_GET(cmd_hca_cap, hcattr, log_max_cq);
 	attr->log_max_qp = MLX5_GET(cmd_hca_cap, hcattr, log_max_qp);
@@ -2397,3 +2399,54 @@ mlx5_devx_cmd_queue_counter_query(struct mlx5_devx_obj *dcs, int clear,
 	*out_of_buffers = MLX5_GET(query_q_counter_out, out, out_of_buffer);
 	return 0;
 }
+
+/**
+ * Create general object of type DEK using DevX API.
+ *
+ * @param[in] ctx
+ *   Context returned from mlx5 open_device() glue function.
+ * @param [in] attr
+ *   Pointer to DEK attributes structure.
+ *
+ * @return
+ *   The DevX object created, NULL otherwise and rte_errno is set.
+ */
+struct mlx5_devx_obj *
+mlx5_devx_cmd_create_dek_obj(void *ctx, struct mlx5_devx_dek_attr *attr)
+{
+	uint32_t in[MLX5_ST_SZ_DW(create_dek_in)] = {0};
+	uint32_t out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};
+	struct mlx5_devx_obj *dek_obj = NULL;
+	void *ptr = NULL, *key_addr = NULL;
+
+	dek_obj = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*dek_obj),
+			      0, SOCKET_ID_ANY);
+	if (dek_obj == NULL) {
+		DRV_LOG(ERR, "Failed to allocate DEK object data");
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+	ptr = MLX5_ADDR_OF(create_dek_in, in, hdr);
+	MLX5_SET(general_obj_in_cmd_hdr, ptr, opcode,
+		 MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
+	MLX5_SET(general_obj_in_cmd_hdr, ptr, obj_type,
+		 MLX5_GENERAL_OBJ_TYPE_DEK);
+	ptr = MLX5_ADDR_OF(create_dek_in, in, dek);
+	MLX5_SET(dek, ptr, key_size, attr->key_size);
+	MLX5_SET(dek, ptr, has_keytag, attr->has_keytag);
+	MLX5_SET(dek, ptr, key_purpose, attr->key_purpose);
+	MLX5_SET(dek, ptr, pd, attr->pd);
+	MLX5_SET64(dek, ptr, opaque, attr->opaque);
+	key_addr = MLX5_ADDR_OF(dek, ptr, key);
+	memcpy(key_addr, (void *)(attr->key), MLX5_CRYPTO_KEY_MAX_SIZE);
+	dek_obj->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in),
+						  out, sizeof(out));
+	if (dek_obj->obj == NULL) {
+		rte_errno = errno;
+		DRV_LOG(ERR, "Failed to create DEK obj using DevX.");
+		mlx5_free(dek_obj);
+		return NULL;
+	}
+	dek_obj->id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
+	return dek_obj;
+}
diff --git a/drivers/common/mlx5/mlx5_devx_cmds.h b/drivers/common/mlx5/mlx5_devx_cmds.h
index 28ade5bbc4..b9ff7ab87d 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.h
+++ b/drivers/common/mlx5/mlx5_devx_cmds.h
@@ -139,6 +139,7 @@ struct mlx5_hca_attr {
 	uint32_t reg_c_preserve:1;
 	uint32_t crypto:1; /* Crypto engine is supported. */
 	uint32_t aes_xts:1; /* AES-XTS crypto is supported. */
+	uint32_t dek:1; /* General obj type DEK is supported. */
 	uint32_t regexp_num_of_engines;
 	uint32_t log_max_ft_sampler_num:8;
 	uint32_t geneve_tlv_opt;
@@ -435,6 +436,18 @@ struct mlx5_devx_graph_node_attr {
 	struct mlx5_devx_graph_arc_attr out[MLX5_GRAPH_NODE_ARC_NUM];
 };
 
+/* Encryption key size is up to 1024 bit, 128 bytes. */
+#define MLX5_CRYPTO_KEY_MAX_SIZE	128
+
+struct mlx5_devx_dek_attr {
+	uint32_t key_size:4;
+	uint32_t has_keytag:1;
+	uint32_t key_purpose:4;
+	uint32_t pd:24;
+	uint64_t opaque;
+	uint8_t key[MLX5_CRYPTO_KEY_MAX_SIZE];
+};
+
 /* mlx5_devx_cmds.c */
 
 __rte_internal
@@ -587,4 +600,8 @@ int mlx5_devx_cmd_queue_counter_query(struct mlx5_devx_obj *dcs, int clear,
 __rte_internal
 struct mlx5_devx_obj *mlx5_devx_cmd_create_flow_meter_aso_obj(void *ctx,
 					uint32_t pd, uint32_t log_obj_size);
+__rte_internal
+struct mlx5_devx_obj *
+mlx5_devx_cmd_create_dek_obj(void *ctx, struct mlx5_devx_dek_attr *attr);
+
 #endif /* RTE_PMD_MLX5_DEVX_CMDS_H_ */
diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index a8fbfbb0f5..bc9f58ad03 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -1109,6 +1109,8 @@ enum {
 			(1ULL << MLX5_GENERAL_OBJ_TYPE_FLOW_METER_ASO)
 #define MLX5_GENERAL_OBJ_TYPES_CAP_GENEVE_TLV_OPT \
 			(1ULL << MLX5_GENERAL_OBJ_TYPE_GENEVE_TLV_OPT)
+#define MLX5_GENERAL_OBJ_TYPES_CAP_DEK \
+			(1ULL << MLX5_GENERAL_OBJ_TYPE_DEK)
 
 enum {
 	MLX5_HCA_CAP_OPMOD_GET_MAX   = 0,
@@ -2406,6 +2408,7 @@ struct mlx5_ifc_create_cq_in_bits {
 
 enum {
 	MLX5_GENERAL_OBJ_TYPE_GENEVE_TLV_OPT = 0x000b,
+	MLX5_GENERAL_OBJ_TYPE_DEK = 0x000c,
 	MLX5_GENERAL_OBJ_TYPE_VIRTQ = 0x000d,
 	MLX5_GENERAL_OBJ_TYPE_VIRTIO_Q_COUNTERS = 0x001c,
 	MLX5_GENERAL_OBJ_TYPE_FLEX_PARSE_GRAPH = 0x0022,
@@ -2469,6 +2472,42 @@ struct mlx5_ifc_create_geneve_tlv_option_in_bits {
 	struct mlx5_ifc_geneve_tlv_option_bits geneve_tlv_opt;
 };
 
+enum {
+	MLX5_CRYPTO_KEY_SIZE_128b = 0x0,
+	MLX5_CRYPTO_KEY_SIZE_256b = 0x1,
+};
+
+enum {
+	MLX5_CRYPTO_KEY_PURPOSE_TLS	= 0x1,
+	MLX5_CRYPTO_KEY_PURPOSE_IPSEC	= 0x2,
+	MLX5_CRYPTO_KEY_PURPOSE_AES_XTS	= 0x3,
+	MLX5_CRYPTO_KEY_PURPOSE_MACSEC	= 0x4,
+	MLX5_CRYPTO_KEY_PURPOSE_GCM	= 0x5,
+	MLX5_CRYPTO_KEY_PURPOSE_PSP	= 0x6,
+};
+
+struct mlx5_ifc_dek_bits {
+	u8 modify_field_select[0x40];
+	u8 state[0x8];
+	u8 reserved_at_48[0xc];
+	u8 key_size[0x4];
+	u8 has_keytag[0x1];
+	u8 reserved_at_59[0x3];
+	u8 key_purpose[0x4];
+	u8 reserved_at_60[0x8];
+	u8 pd[0x18];
+	u8 reserved_at_80[0x100];
+	u8 opaque[0x40];
+	u8 reserved_at_1c0[0x40];
+	u8 key[0x400];
+	u8 reserved_at_600[0x200];
+};
+
+struct mlx5_ifc_create_dek_in_bits {
+	struct mlx5_ifc_general_obj_in_cmd_hdr_bits hdr;
+	struct mlx5_ifc_dek_bits dek;
+};
+
 enum {
 	MLX5_VIRTQ_STATE_INIT = 0,
 	MLX5_VIRTQ_STATE_RDY = 1,
diff --git a/drivers/common/mlx5/version.map b/drivers/common/mlx5/version.map
index 18dc96276d..2976edce0b 100644
--- a/drivers/common/mlx5/version.map
+++ b/drivers/common/mlx5/version.map
@@ -27,6 +27,7 @@ INTERNAL {
 	mlx5_devx_cmd_create_flow_hit_aso_obj;
 	mlx5_devx_cmd_create_flow_meter_aso_obj;
 	mlx5_devx_cmd_create_geneve_tlv_option;
+        mlx5_devx_cmd_create_dek_obj;
 	mlx5_devx_cmd_destroy;
 	mlx5_devx_cmd_flow_counter_alloc;
 	mlx5_devx_cmd_flow_counter_query;
-- 
2.25.1

[dpdk-dev] [PATCH v2 06/16] common/mlx5: adjust DevX MKEY fields for crypto

[Matan Azrad]

From: Dekel Peled <dekelp@nvidia.com>

MKEY that will be used for crypto purposes must be created with
crypto_en and remote access attributes.
This patch adds support for them in the DevX MKEY context.

Signed-off-by: Dekel Peled <dekelp@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_devx_cmds.c |  5 +++++
 drivers/common/mlx5/mlx5_devx_cmds.h |  2 ++
 drivers/common/mlx5/mlx5_prm.h       | 10 +++++++++-
 3 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c
index 742c82cca4..68a10b149a 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.c
+++ b/drivers/common/mlx5/mlx5_devx_cmds.c
@@ -263,6 +263,10 @@ mlx5_devx_cmd_mkey_create(void *ctx,
 	MLX5_SET(create_mkey_in, in, pg_access, attr->pg_access);
 	MLX5_SET(mkc, mkc, lw, 0x1);
 	MLX5_SET(mkc, mkc, lr, 0x1);
+	if (attr->set_remote_rw) {
+		MLX5_SET(mkc, mkc, rw, 0x1);
+		MLX5_SET(mkc, mkc, rr, 0x1);
+	}
 	MLX5_SET(mkc, mkc, qpn, 0xffffff);
 	MLX5_SET(mkc, mkc, pd, attr->pd);
 	MLX5_SET(mkc, mkc, mkey_7_0, attr->umem_id & 0xFF);
@@ -273,6 +277,7 @@ mlx5_devx_cmd_mkey_create(void *ctx,
 	MLX5_SET(mkc, mkc, relaxed_ordering_read, attr->relaxed_ordering_read);
 	MLX5_SET64(mkc, mkc, start_addr, attr->addr);
 	MLX5_SET64(mkc, mkc, len, attr->size);
+	MLX5_SET(mkc, mkc, crypto_en, attr->crypto_en);
 	mkey->obj = mlx5_glue->devx_obj_create(ctx, in, in_size_dw * 4, out,
 					       sizeof(out));
 	if (!mkey->obj) {
diff --git a/drivers/common/mlx5/mlx5_devx_cmds.h b/drivers/common/mlx5/mlx5_devx_cmds.h
index b9ff7ab87d..600577f18a 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.h
+++ b/drivers/common/mlx5/mlx5_devx_cmds.h
@@ -32,6 +32,8 @@ struct mlx5_devx_mkey_attr {
 	uint32_t relaxed_ordering_write:1;
 	uint32_t relaxed_ordering_read:1;
 	uint32_t umr_en:1;
+	uint32_t crypto_en:2;
+	uint32_t set_remote_rw:1;
 	struct mlx5_klm *klm_array;
 	int klm_num;
 };
diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index bc9f58ad03..25f6f8ff00 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -1059,7 +1059,15 @@ struct mlx5_ifc_mkc_bits {
 	u8 relaxed_ordering_read[0x1];
 	u8 reserved_at_1da[0x1];
 	u8 log_page_size[0x5];
-	u8 reserved_at_1e0[0x20];
+	u8 reserved_at_1e0[0x3];
+	u8 crypto_en[0x2];
+	u8 reserved_at_1e5[0x1b];
+};
+
+/* Range of values for MKEY context crypto_en field. */
+enum {
+	MLX5_MKEY_CRYPTO_DISABLED = 0x0,
+	MLX5_MKEY_CRYPTO_ENABLED = 0x1,
 };
 
 struct mlx5_ifc_create_mkey_out_bits {
-- 
2.25.1

[dpdk-dev] [PATCH v2 07/16] common/mlx5: fix cypto bsf attr

[Matan Azrad]

From: Suanming Mou <suanmingm@nvidia.com>

This commit should be squash to crypto en commit.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
---
 drivers/common/mlx5/mlx5_devx_cmds.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c
index 68a10b149a..7e3c8b55fa 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.c
+++ b/drivers/common/mlx5/mlx5_devx_cmds.c
@@ -278,6 +278,10 @@ mlx5_devx_cmd_mkey_create(void *ctx,
 	MLX5_SET64(mkc, mkc, start_addr, attr->addr);
 	MLX5_SET64(mkc, mkc, len, attr->size);
 	MLX5_SET(mkc, mkc, crypto_en, attr->crypto_en);
+	if (attr->crypto_en) {
+		MLX5_SET(mkc, mkc, bsf_en, attr->crypto_en);
+		MLX5_SET(mkc, mkc, bsf_octword_size, 4);
+	}
 	mkey->obj = mlx5_glue->devx_obj_create(ctx, in, in_size_dw * 4, out,
 					       sizeof(out));
 	if (!mkey->obj) {
-- 
2.25.1

[dpdk-dev] [PATCH v2 08/16] common/mlx5: support general obj IMPORT KEK create

[Matan Azrad]

From: Dekel Peled <dekelp@nvidia.com>

IMPORT_KEK object is used to wrap (encrypt) critical security
parameters, such as other keys and credentials, when those need
to be passed between the device and the software.

This patch add support of IMPORT_KEK object create operation.
Add reading of IMPORT_KEK support capability.
Add function to create general object type IMPORT_KEK, using DevX API.

Signed-off-by: Dekel Peled <dekelp@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_devx_cmds.c | 50 ++++++++++++++++++++++++++++
 drivers/common/mlx5/mlx5_devx_cmds.h | 13 ++++++++
 drivers/common/mlx5/mlx5_prm.h       | 18 ++++++++++
 drivers/common/mlx5/version.map      |  1 +
 4 files changed, 82 insertions(+)

diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c
index 7e3c8b55fa..afef7a5f63 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.c
+++ b/drivers/common/mlx5/mlx5_devx_cmds.c
@@ -752,6 +752,8 @@ mlx5_devx_cmd_query_hca_attr(void *ctx,
 				  MLX5_GENERAL_OBJ_TYPES_CAP_GENEVE_TLV_OPT);
 	attr->dek = !!(general_obj_types_supported &
 		       MLX5_GENERAL_OBJ_TYPES_CAP_DEK);
+	attr->import_kek = !!(general_obj_types_supported &
+			      MLX5_GENERAL_OBJ_TYPES_CAP_IMPORT_KEK);
 	/* Add reading of other GENERAL_OBJ_TYPES_CAP bits above this line. */
 	attr->log_max_cq = MLX5_GET(cmd_hca_cap, hcattr, log_max_cq);
 	attr->log_max_qp = MLX5_GET(cmd_hca_cap, hcattr, log_max_qp);
@@ -2459,3 +2461,51 @@ mlx5_devx_cmd_create_dek_obj(void *ctx, struct mlx5_devx_dek_attr *attr)
 	dek_obj->id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
 	return dek_obj;
 }
+
+/**
+ * Create general object of type IMPORT_KEK using DevX API.
+ *
+ * @param[in] ctx
+ *   Context returned from mlx5 open_device() glue function.
+ * @param [in] attr
+ *   Pointer to IMPORT_KEK attributes structure.
+ *
+ * @return
+ *   The DevX object created, NULL otherwise and rte_errno is set.
+ */
+struct mlx5_devx_obj *
+mlx5_devx_cmd_create_import_kek_obj(void *ctx,
+				    struct mlx5_devx_import_kek_attr *attr)
+{
+	uint32_t in[MLX5_ST_SZ_DW(create_import_kek_in)] = {0};
+	uint32_t out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};
+	struct mlx5_devx_obj *import_kek_obj = NULL;
+	void *ptr = NULL, *key_addr = NULL;
+
+	import_kek_obj = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*import_kek_obj),
+				     0, SOCKET_ID_ANY);
+	if (import_kek_obj == NULL) {
+		DRV_LOG(ERR, "Failed to allocate IMPORT_KEK object data");
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+	ptr = MLX5_ADDR_OF(create_import_kek_in, in, hdr);
+	MLX5_SET(general_obj_in_cmd_hdr, ptr, opcode,
+		 MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
+	MLX5_SET(general_obj_in_cmd_hdr, ptr, obj_type,
+		 MLX5_GENERAL_OBJ_TYPE_IMPORT_KEK);
+	ptr = MLX5_ADDR_OF(create_import_kek_in, in, import_kek);
+	MLX5_SET(import_kek, ptr, key_size, attr->key_size);
+	key_addr = MLX5_ADDR_OF(import_kek, ptr, key);
+	memcpy(key_addr, (void *)(attr->key), MLX5_CRYPTO_KEY_MAX_SIZE);
+	import_kek_obj->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in),
+							 out, sizeof(out));
+	if (import_kek_obj->obj == NULL) {
+		rte_errno = errno;
+		DRV_LOG(ERR, "Failed to create IMPORT_KEK object using DevX.");
+		mlx5_free(import_kek_obj);
+		return NULL;
+	}
+	import_kek_obj->id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
+	return import_kek_obj;
+}
diff --git a/drivers/common/mlx5/mlx5_devx_cmds.h b/drivers/common/mlx5/mlx5_devx_cmds.h
index 600577f18a..6423610dae 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.h
+++ b/drivers/common/mlx5/mlx5_devx_cmds.h
@@ -142,6 +142,7 @@ struct mlx5_hca_attr {
 	uint32_t crypto:1; /* Crypto engine is supported. */
 	uint32_t aes_xts:1; /* AES-XTS crypto is supported. */
 	uint32_t dek:1; /* General obj type DEK is supported. */
+	uint32_t import_kek:1; /* General obj type IMPORT_KEK supported. */
 	uint32_t regexp_num_of_engines;
 	uint32_t log_max_ft_sampler_num:8;
 	uint32_t geneve_tlv_opt;
@@ -450,6 +451,13 @@ struct mlx5_devx_dek_attr {
 	uint8_t key[MLX5_CRYPTO_KEY_MAX_SIZE];
 };
 
+struct mlx5_devx_import_kek_attr {
+	uint64_t modify_field_select;
+	uint32_t state:8;
+	uint32_t key_size:4;
+	uint8_t key[MLX5_CRYPTO_KEY_MAX_SIZE];
+};
+
 /* mlx5_devx_cmds.c */
 
 __rte_internal
@@ -606,4 +614,9 @@ __rte_internal
 struct mlx5_devx_obj *
 mlx5_devx_cmd_create_dek_obj(void *ctx, struct mlx5_devx_dek_attr *attr);
 
+__rte_internal
+struct mlx5_devx_obj *
+mlx5_devx_cmd_create_import_kek_obj(void *ctx,
+				    struct mlx5_devx_import_kek_attr *attr);
+
 #endif /* RTE_PMD_MLX5_DEVX_CMDS_H_ */
diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index 25f6f8ff00..bc339566a6 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -1119,6 +1119,8 @@ enum {
 			(1ULL << MLX5_GENERAL_OBJ_TYPE_GENEVE_TLV_OPT)
 #define MLX5_GENERAL_OBJ_TYPES_CAP_DEK \
 			(1ULL << MLX5_GENERAL_OBJ_TYPE_DEK)
+#define MLX5_GENERAL_OBJ_TYPES_CAP_IMPORT_KEK \
+			(1ULL << MLX5_GENERAL_OBJ_TYPE_IMPORT_KEK)
 
 enum {
 	MLX5_HCA_CAP_OPMOD_GET_MAX   = 0,
@@ -2419,6 +2421,7 @@ enum {
 	MLX5_GENERAL_OBJ_TYPE_DEK = 0x000c,
 	MLX5_GENERAL_OBJ_TYPE_VIRTQ = 0x000d,
 	MLX5_GENERAL_OBJ_TYPE_VIRTIO_Q_COUNTERS = 0x001c,
+	MLX5_GENERAL_OBJ_TYPE_IMPORT_KEK = 0x001d,
 	MLX5_GENERAL_OBJ_TYPE_FLEX_PARSE_GRAPH = 0x0022,
 	MLX5_GENERAL_OBJ_TYPE_FLOW_METER_ASO = 0x0024,
 	MLX5_GENERAL_OBJ_TYPE_FLOW_HIT_ASO = 0x0025,
@@ -2516,6 +2519,21 @@ struct mlx5_ifc_create_dek_in_bits {
 	struct mlx5_ifc_dek_bits dek;
 };
 
+struct mlx5_ifc_import_kek_bits {
+	u8 modify_field_select[0x40];
+	u8 state[0x8];
+	u8 reserved_at_48[0xc];
+	u8 key_size[0x4];
+	u8 reserved_at_58[0x1a8];
+	u8 key[0x400];
+	u8 reserved_at_600[0x200];
+};
+
+struct mlx5_ifc_create_import_kek_in_bits {
+	struct mlx5_ifc_general_obj_in_cmd_hdr_bits hdr;
+	struct mlx5_ifc_import_kek_bits import_kek;
+};
+
 enum {
 	MLX5_VIRTQ_STATE_INIT = 0,
 	MLX5_VIRTQ_STATE_RDY = 1,
diff --git a/drivers/common/mlx5/version.map b/drivers/common/mlx5/version.map
index 2976edce0b..56a1b77b0a 100644
--- a/drivers/common/mlx5/version.map
+++ b/drivers/common/mlx5/version.map
@@ -28,6 +28,7 @@ INTERNAL {
 	mlx5_devx_cmd_create_flow_meter_aso_obj;
 	mlx5_devx_cmd_create_geneve_tlv_option;
         mlx5_devx_cmd_create_dek_obj;
+        mlx5_devx_cmd_create_import_kek_obj;
 	mlx5_devx_cmd_destroy;
 	mlx5_devx_cmd_flow_counter_alloc;
 	mlx5_devx_cmd_flow_counter_query;
-- 
2.25.1

[dpdk-dev] [PATCH v2 09/16] common/mlx5: support general obj CRYPTO LOGIN create

[Matan Azrad]

From: Dekel Peled <dekelp@nvidia.com>

CRYPTO_LOGIN Object is used to login to the device as crypto user
or crypto officer.
Required in order to perform any crypto related control operations.

This patch adds support of CRYPTO_LOGIN object create operation.
Add reading of CRYPTO_LOGIN support capability.
Add function to create general object type CRYPTO_LOGIN, using DevX API.

Signed-off-by: Dekel Peled <dekelp@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_devx_cmds.c | 54 ++++++++++++++++++++++++++++
 drivers/common/mlx5/mlx5_devx_cmds.h | 15 ++++++++
 drivers/common/mlx5/mlx5_prm.h       | 19 ++++++++++
 drivers/common/mlx5/version.map      |  1 +
 4 files changed, 89 insertions(+)

diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c
index afef7a5f63..5e082ebb78 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.c
+++ b/drivers/common/mlx5/mlx5_devx_cmds.c
@@ -754,6 +754,8 @@ mlx5_devx_cmd_query_hca_attr(void *ctx,
 		       MLX5_GENERAL_OBJ_TYPES_CAP_DEK);
 	attr->import_kek = !!(general_obj_types_supported &
 			      MLX5_GENERAL_OBJ_TYPES_CAP_IMPORT_KEK);
+	attr->crypto_login = !!(general_obj_types_supported &
+				MLX5_GENERAL_OBJ_TYPES_CAP_CRYPTO_LOGIN);
 	/* Add reading of other GENERAL_OBJ_TYPES_CAP bits above this line. */
 	attr->log_max_cq = MLX5_GET(cmd_hca_cap, hcattr, log_max_cq);
 	attr->log_max_qp = MLX5_GET(cmd_hca_cap, hcattr, log_max_qp);
@@ -2509,3 +2511,55 @@ mlx5_devx_cmd_create_import_kek_obj(void *ctx,
 	import_kek_obj->id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
 	return import_kek_obj;
 }
+
+/**
+ * Create general object of type CRYPTO_LOGIN using DevX API.
+ *
+ * @param[in] ctx
+ *   Context returned from mlx5 open_device() glue function.
+ * @param [in] attr
+ *   Pointer to CRYPTO_LOGIN attributes structure.
+ *
+ * @return
+ *   The DevX object created, NULL otherwise and rte_errno is set.
+ */
+struct mlx5_devx_obj *
+mlx5_devx_cmd_create_crypto_login_obj(void *ctx,
+				      struct mlx5_devx_crypto_login_attr *attr)
+{
+	uint32_t in[MLX5_ST_SZ_DW(create_crypto_login_in)] = {0};
+	uint32_t out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};
+	struct mlx5_devx_obj *crypto_login_obj = NULL;
+	void *ptr = NULL, *credential_addr = NULL;
+
+	crypto_login_obj = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*crypto_login_obj),
+				       0, SOCKET_ID_ANY);
+	if (crypto_login_obj == NULL) {
+		DRV_LOG(ERR, "Failed to allocate CRYPTO_LOGIN object data");
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+	ptr = MLX5_ADDR_OF(create_crypto_login_in, in, hdr);
+	MLX5_SET(general_obj_in_cmd_hdr, ptr, opcode,
+		 MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
+	MLX5_SET(general_obj_in_cmd_hdr, ptr, obj_type,
+		 MLX5_GENERAL_OBJ_TYPE_CRYPTO_LOGIN);
+	ptr = MLX5_ADDR_OF(create_crypto_login_in, in, crypto_login);
+	MLX5_SET(crypto_login, ptr, credential_pointer,
+		 attr->credential_pointer);
+	MLX5_SET(crypto_login, ptr, session_import_kek_ptr,
+		 attr->session_import_kek_ptr);
+	credential_addr = MLX5_ADDR_OF(crypto_login, ptr, credential);
+	memcpy(credential_addr, (void *)(attr->credential),
+	       MLX5_CRYPTO_LOGIN_CREDENTIAL_SIZE);
+	crypto_login_obj->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in),
+							   out, sizeof(out));
+	if (crypto_login_obj->obj == NULL) {
+		rte_errno = errno;
+		DRV_LOG(ERR, "Failed to create CRYPTO_LOGIN obj using DevX.");
+		mlx5_free(crypto_login_obj);
+		return NULL;
+	}
+	crypto_login_obj->id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
+	return crypto_login_obj;
+}
diff --git a/drivers/common/mlx5/mlx5_devx_cmds.h b/drivers/common/mlx5/mlx5_devx_cmds.h
index 6423610dae..709e28bfba 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.h
+++ b/drivers/common/mlx5/mlx5_devx_cmds.h
@@ -143,6 +143,7 @@ struct mlx5_hca_attr {
 	uint32_t aes_xts:1; /* AES-XTS crypto is supported. */
 	uint32_t dek:1; /* General obj type DEK is supported. */
 	uint32_t import_kek:1; /* General obj type IMPORT_KEK supported. */
+	uint32_t crypto_login:1; /* General obj type CRYPTO_LOGIN supported. */
 	uint32_t regexp_num_of_engines;
 	uint32_t log_max_ft_sampler_num:8;
 	uint32_t geneve_tlv_opt;
@@ -458,6 +459,15 @@ struct mlx5_devx_import_kek_attr {
 	uint8_t key[MLX5_CRYPTO_KEY_MAX_SIZE];
 };
 
+#define MLX5_CRYPTO_LOGIN_CREDENTIAL_SIZE	48
+
+struct mlx5_devx_crypto_login_attr {
+	uint64_t modify_field_select;
+	uint32_t credential_pointer:24;
+	uint32_t session_import_kek_ptr:24;
+	uint8_t credential[MLX5_CRYPTO_LOGIN_CREDENTIAL_SIZE];
+};
+
 /* mlx5_devx_cmds.c */
 
 __rte_internal
@@ -619,4 +629,9 @@ struct mlx5_devx_obj *
 mlx5_devx_cmd_create_import_kek_obj(void *ctx,
 				    struct mlx5_devx_import_kek_attr *attr);
 
+__rte_internal
+struct mlx5_devx_obj *
+mlx5_devx_cmd_create_crypto_login_obj(void *ctx,
+				      struct mlx5_devx_crypto_login_attr *attr);
+
 #endif /* RTE_PMD_MLX5_DEVX_CMDS_H_ */
diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index bc339566a6..a2437faec0 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -1121,6 +1121,8 @@ enum {
 			(1ULL << MLX5_GENERAL_OBJ_TYPE_DEK)
 #define MLX5_GENERAL_OBJ_TYPES_CAP_IMPORT_KEK \
 			(1ULL << MLX5_GENERAL_OBJ_TYPE_IMPORT_KEK)
+#define MLX5_GENERAL_OBJ_TYPES_CAP_CRYPTO_LOGIN \
+			(1ULL << MLX5_GENERAL_OBJ_TYPE_CRYPTO_LOGIN)
 
 enum {
 	MLX5_HCA_CAP_OPMOD_GET_MAX   = 0,
@@ -2422,6 +2424,7 @@ enum {
 	MLX5_GENERAL_OBJ_TYPE_VIRTQ = 0x000d,
 	MLX5_GENERAL_OBJ_TYPE_VIRTIO_Q_COUNTERS = 0x001c,
 	MLX5_GENERAL_OBJ_TYPE_IMPORT_KEK = 0x001d,
+	MLX5_GENERAL_OBJ_TYPE_CRYPTO_LOGIN = 0x001f,
 	MLX5_GENERAL_OBJ_TYPE_FLEX_PARSE_GRAPH = 0x0022,
 	MLX5_GENERAL_OBJ_TYPE_FLOW_METER_ASO = 0x0024,
 	MLX5_GENERAL_OBJ_TYPE_FLOW_HIT_ASO = 0x0025,
@@ -2534,6 +2537,22 @@ struct mlx5_ifc_create_import_kek_in_bits {
 	struct mlx5_ifc_import_kek_bits import_kek;
 };
 
+struct mlx5_ifc_crypto_login_bits {
+	u8 modify_field_select[0x40];
+	u8 reserved_at_40[0x48];
+	u8 credential_pointer[0x18];
+	u8 reserved_at_a0[0x8];
+	u8 session_import_kek_ptr[0x18];
+	u8 reserved_at_c0[0x140];
+	u8 credential[0x180];
+	u8 reserved_at_380[0x480];
+};
+
+struct mlx5_ifc_create_crypto_login_in_bits {
+	struct mlx5_ifc_general_obj_in_cmd_hdr_bits hdr;
+	struct mlx5_ifc_crypto_login_bits crypto_login;
+};
+
 enum {
 	MLX5_VIRTQ_STATE_INIT = 0,
 	MLX5_VIRTQ_STATE_RDY = 1,
diff --git a/drivers/common/mlx5/version.map b/drivers/common/mlx5/version.map
index 56a1b77b0a..bbef436fde 100644
--- a/drivers/common/mlx5/version.map
+++ b/drivers/common/mlx5/version.map
@@ -29,6 +29,7 @@ INTERNAL {
 	mlx5_devx_cmd_create_geneve_tlv_option;
         mlx5_devx_cmd_create_dek_obj;
         mlx5_devx_cmd_create_import_kek_obj;
+        mlx5_devx_cmd_create_crypto_login_obj;
 	mlx5_devx_cmd_destroy;
 	mlx5_devx_cmd_flow_counter_alloc;
 	mlx5_devx_cmd_flow_counter_query;
-- 
2.25.1

[dpdk-dev] [PATCH v2 10/16] common/mlx5: add crypto BSF struct and defines

[Matan Azrad]

From: Dekel Peled <dekelp@nvidia.com>

This patch adds the struct defining crypto BSF segment of UMR WQE,
and the related value definitions and offsets.

Signed-off-by: Dekel Peled <dekelp@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_prm.h | 66 ++++++++++++++++++++++++++++++++++
 1 file changed, 66 insertions(+)

diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index a2437faec0..a9dcbfa63c 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -1096,6 +1096,72 @@ struct mlx5_ifc_create_mkey_in_bits {
 	u8 klm_pas_mtt[][0x20];
 };
 
+enum {
+	MLX5_BSF_SIZE_16B = 0x0,
+	MLX5_BSF_SIZE_32B = 0x1,
+	MLX5_BSF_SIZE_64B = 0x2,
+	MLX5_BSF_SIZE_128B = 0x3,
+};
+
+enum {
+	MLX5_BSF_P_TYPE_SIGNATURE = 0x0,
+	MLX5_BSF_P_TYPE_CRYPTO = 0x1,
+};
+
+enum {
+	MLX5_ENCRYPTION_ORDER_ENCRYPTED_WIRE_SIGNATURE = 0x0,
+	MLX5_ENCRYPTION_ORDER_ENCRYPTED_MEMORY_SIGNATURE = 0x1,
+	MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_WIRE = 0x2,
+	MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_MEMORY = 0x3,
+};
+
+enum {
+	MLX5_ENCRYPTION_STANDARD_AES_XTS = 0x0,
+};
+
+enum {
+	MLX5_BLOCK_SIZE_512B	= 0x1,
+	MLX5_BLOCK_SIZE_520B	= 0x2,
+	MLX5_BLOCK_SIZE_4096B	= 0x3,
+	MLX5_BLOCK_SIZE_4160B	= 0x4,
+	MLX5_BLOCK_SIZE_1MB	= 0x5,
+	MLX5_BLOCK_SIZE_4048B	= 0x6,
+};
+
+#define MLX5_BSF_SIZE_OFFSET		30
+#define MLX5_BSF_P_TYPE_OFFSET		24
+#define MLX5_ENCRYPTION_ORDER_OFFSET	16
+#define MLX5_BLOCK_SIZE_OFFSET		24
+
+struct mlx5_wqe_umr_bsf_seg {
+	/*
+	 * bs_bpt_eo_es contains:
+	 * bs	bsf_size		2 bits at MLX5_BSF_SIZE_OFFSET
+	 * bpt	bsf_p_type		2 bits at MLX5_BSF_P_TYPE_OFFSET
+	 * eo	encryption_order	4 bits at MLX5_ENCRYPTION_ORDER_OFFSET
+	 * es	encryption_standard	4 bits at offset 0
+	 */
+	uint32_t bs_bpt_eo_es;
+	uint32_t raw_data_size;
+	/*
+	 * bsp_res contains:
+	 * bsp	crypto_block_size_pointer	8 bits at MLX5_BLOCK_SIZE_OFFSET
+	 * res	reserved 24 bits
+	 */
+	uint32_t bsp_res;
+	uint32_t reserved0;
+	uint8_t xts_initial_tweak[16];
+	/*
+	 * res_dp contains:
+	 * res	reserved 8 bits
+	 * dp	dek_pointer		24 bits at offset 0
+	 */
+	uint32_t res_dp;
+	uint32_t reserved1;
+	uint64_t keytag;
+	uint32_t reserved2[4];
+} __rte_packed;
+
 enum {
 	MLX5_GET_HCA_CAP_OP_MOD_GENERAL_DEVICE = 0x0 << 1,
 	MLX5_GET_HCA_CAP_OP_MOD_ETHERNET_OFFLOAD_CAPS = 0x1 << 1,
-- 
2.25.1

[dpdk-dev] [PATCH v2 11/16] common/mlx5: share hash list tool

[Matan Azrad]

From: Shiri Kuzin <shirik@nvidia.com>

In order to use the hash list defined in net in other drivers, the
hash list is moved to common utilities.

In addition, the log definition was moved from the common utilities to
a dedicated new log file in common in order to prevent a conflict.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/linux/mlx5_common_os.c   |   2 +-
 drivers/common/mlx5/linux/mlx5_nl.c          |   2 +-
 drivers/common/mlx5/meson.build              |  15 +-
 drivers/common/mlx5/mlx5_common.c            |   2 +-
 drivers/common/mlx5/mlx5_common_devx.c       |   2 +-
 drivers/common/mlx5/mlx5_common_log.h        |  21 ++
 drivers/common/mlx5/mlx5_common_mp.c         |   2 +-
 drivers/common/mlx5/mlx5_common_mr.c         |   2 +-
 drivers/common/mlx5/mlx5_common_pci.c        |   4 +-
 drivers/common/mlx5/mlx5_common_utils.c      | 221 +++++++++++++++++++
 drivers/common/mlx5/mlx5_common_utils.h      | 202 ++++++++++++++++-
 drivers/common/mlx5/mlx5_devx_cmds.c         |   2 +-
 drivers/common/mlx5/mlx5_malloc.c            |   2 +-
 drivers/common/mlx5/version.map              |   6 +
 drivers/common/mlx5/windows/mlx5_common_os.c |   2 +-
 drivers/common/mlx5/windows/mlx5_glue.c      |   2 +-
 drivers/net/mlx5/mlx5_utils.c                | 209 ------------------
 drivers/net/mlx5/mlx5_utils.h                | 194 +---------------
 18 files changed, 465 insertions(+), 427 deletions(-)
 create mode 100644 drivers/common/mlx5/mlx5_common_log.h
 create mode 100644 drivers/common/mlx5/mlx5_common_utils.c

diff --git a/drivers/common/mlx5/linux/mlx5_common_os.c b/drivers/common/mlx5/linux/mlx5_common_os.c
index 5cf9576921..fba8245b8b 100644
--- a/drivers/common/mlx5/linux/mlx5_common_os.c
+++ b/drivers/common/mlx5/linux/mlx5_common_os.c
@@ -15,7 +15,7 @@
 #include <rte_string_fns.h>
 
 #include "mlx5_common.h"
-#include "mlx5_common_utils.h"
+#include "mlx5_common_log.h"
 #include "mlx5_glue.h"
 
 #ifdef MLX5_GLUE
diff --git a/drivers/common/mlx5/linux/mlx5_nl.c b/drivers/common/mlx5/linux/mlx5_nl.c
index 752c57b33d..f0d04f9473 100644
--- a/drivers/common/mlx5/linux/mlx5_nl.c
+++ b/drivers/common/mlx5/linux/mlx5_nl.c
@@ -20,7 +20,7 @@
 #include <rte_errno.h>
 
 #include "mlx5_nl.h"
-#include "mlx5_common_utils.h"
+#include "../mlx5_common_log.h"
 #include "mlx5_malloc.h"
 #ifdef HAVE_DEVLINK
 #include <linux/devlink.h>
diff --git a/drivers/common/mlx5/meson.build b/drivers/common/mlx5/meson.build
index b5585d2cc3..cd371fe6e1 100644
--- a/drivers/common/mlx5/meson.build
+++ b/drivers/common/mlx5/meson.build
@@ -9,13 +9,14 @@ endif
 
 deps += ['hash', 'pci', 'bus_pci', 'net', 'eal', 'kvargs']
 sources += files(
-        'mlx5_devx_cmds.c',
-        'mlx5_common.c',
-        'mlx5_common_mp.c',
-        'mlx5_common_mr.c',
-        'mlx5_malloc.c',
-        'mlx5_common_pci.c',
-        'mlx5_common_devx.c',
+	'mlx5_devx_cmds.c',
+	'mlx5_common.c',
+	'mlx5_common_mp.c',
+	'mlx5_common_mr.c',
+	'mlx5_malloc.c',
+	'mlx5_common_pci.c',
+	'mlx5_common_devx.c',
+	'mlx5_common_utils.c',
 )
 
 cflags_options = [
diff --git a/drivers/common/mlx5/mlx5_common.c b/drivers/common/mlx5/mlx5_common.c
index f92f05bda5..d397459a3d 100644
--- a/drivers/common/mlx5/mlx5_common.c
+++ b/drivers/common/mlx5/mlx5_common.c
@@ -11,7 +11,7 @@
 
 #include "mlx5_common.h"
 #include "mlx5_common_os.h"
-#include "mlx5_common_utils.h"
+#include "mlx5_common_log.h"
 #include "mlx5_common_pci.h"
 
 uint8_t haswell_broadwell_cpu;
diff --git a/drivers/common/mlx5/mlx5_common_devx.c b/drivers/common/mlx5/mlx5_common_devx.c
index d19be122bd..22c8d356c4 100644
--- a/drivers/common/mlx5/mlx5_common_devx.c
+++ b/drivers/common/mlx5/mlx5_common_devx.c
@@ -12,7 +12,7 @@
 
 #include "mlx5_prm.h"
 #include "mlx5_devx_cmds.h"
-#include "mlx5_common_utils.h"
+#include "mlx5_common_log.h"
 #include "mlx5_malloc.h"
 #include "mlx5_common.h"
 #include "mlx5_common_devx.h"
diff --git a/drivers/common/mlx5/mlx5_common_log.h b/drivers/common/mlx5/mlx5_common_log.h
new file mode 100644
index 0000000000..26b13fedaf
--- /dev/null
+++ b/drivers/common/mlx5/mlx5_common_log.h
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2019 Mellanox Technologies, Ltd
+ */
+
+#ifndef RTE_PMD_MLX5_COMMON_LOG_H_
+#define RTE_PMD_MLX5_COMMON_LOG_H_
+
+#include "mlx5_common.h"
+
+
+extern int mlx5_common_logtype;
+
+#define MLX5_COMMON_LOG_PREFIX "mlx5_common"
+/* Generic printf()-like logging macro with automatic line feed. */
+#define DRV_LOG(level, ...) \
+	PMD_DRV_LOG_(level, mlx5_common_logtype, MLX5_COMMON_LOG_PREFIX, \
+		__VA_ARGS__ PMD_DRV_LOG_STRIP PMD_DRV_LOG_OPAREN, \
+		PMD_DRV_LOG_CPAREN)
+
+#endif /* RTE_PMD_MLX5_COMMON_LOG_H_ */
+
diff --git a/drivers/common/mlx5/mlx5_common_mp.c b/drivers/common/mlx5/mlx5_common_mp.c
index 40e3956e45..673a7c31de 100644
--- a/drivers/common/mlx5/mlx5_common_mp.c
+++ b/drivers/common/mlx5/mlx5_common_mp.c
@@ -10,7 +10,7 @@
 #include <rte_errno.h>
 
 #include "mlx5_common_mp.h"
-#include "mlx5_common_utils.h"
+#include "mlx5_common_log.h"
 #include "mlx5_malloc.h"
 
 /**
diff --git a/drivers/common/mlx5/mlx5_common_mr.c b/drivers/common/mlx5/mlx5_common_mr.c
index e1ed0caf3a..afb5b3d0a7 100644
--- a/drivers/common/mlx5/mlx5_common_mr.c
+++ b/drivers/common/mlx5/mlx5_common_mr.c
@@ -11,7 +11,7 @@
 #include "mlx5_glue.h"
 #include "mlx5_common_mp.h"
 #include "mlx5_common_mr.h"
-#include "mlx5_common_utils.h"
+#include "mlx5_common_log.h"
 #include "mlx5_malloc.h"
 
 struct mr_find_contig_memsegs_data {
diff --git a/drivers/common/mlx5/mlx5_common_pci.c b/drivers/common/mlx5/mlx5_common_pci.c
index 5a73ffa60a..3f16cd21cf 100644
--- a/drivers/common/mlx5/mlx5_common_pci.c
+++ b/drivers/common/mlx5/mlx5_common_pci.c
@@ -4,7 +4,9 @@
 
 #include <stdlib.h>
 #include <rte_malloc.h>
-#include "mlx5_common_utils.h"
+#include <rte_class.h>
+
+#include "mlx5_common_log.h"
 #include "mlx5_common_pci.h"
 
 struct mlx5_pci_device {
diff --git a/drivers/common/mlx5/mlx5_common_utils.c b/drivers/common/mlx5/mlx5_common_utils.c
new file mode 100644
index 0000000000..ad2011e858
--- /dev/null
+++ b/drivers/common/mlx5/mlx5_common_utils.c
@@ -0,0 +1,221 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2019 Mellanox Technologies, Ltd
+ */
+
+#include <rte_malloc.h>
+#include <rte_hash_crc.h>
+#include <rte_errno.h>
+
+#include <mlx5_malloc.h>
+
+#include "mlx5_common_utils.h"
+#include "mlx5_common_log.h"
+
+/********************* Hash List **********************/
+
+static struct mlx5_hlist_entry *
+mlx5_hlist_default_create_cb(struct mlx5_hlist *h, uint64_t key __rte_unused,
+			     void *ctx __rte_unused)
+{
+	return mlx5_malloc(MLX5_MEM_ZERO, h->entry_sz, 0, SOCKET_ID_ANY);
+}
+
+static void
+mlx5_hlist_default_remove_cb(struct mlx5_hlist *h __rte_unused,
+			     struct mlx5_hlist_entry *entry)
+{
+	mlx5_free(entry);
+}
+
+struct mlx5_hlist *
+mlx5_hlist_create(const char *name, uint32_t size, uint32_t entry_size,
+		  uint32_t flags, mlx5_hlist_create_cb cb_create,
+		  mlx5_hlist_match_cb cb_match, mlx5_hlist_remove_cb cb_remove)
+{
+	struct mlx5_hlist *h;
+	uint32_t act_size;
+	uint32_t alloc_size;
+	uint32_t i;
+
+	if (!size || !cb_match || (!cb_create ^ !cb_remove))
+		return NULL;
+	/* Align to the next power of 2, 32bits integer is enough now. */
+	if (!rte_is_power_of_2(size)) {
+		act_size = rte_align32pow2(size);
+		DRV_LOG(DEBUG, "Size 0x%" PRIX32 " is not power of 2, "
+			"will be aligned to 0x%" PRIX32 ".", size, act_size);
+	} else {
+		act_size = size;
+	}
+	alloc_size = sizeof(struct mlx5_hlist) +
+		     sizeof(struct mlx5_hlist_bucket) * act_size;
+	/* Using zmalloc, then no need to initialize the heads. */
+	h = mlx5_malloc(MLX5_MEM_ZERO, alloc_size, RTE_CACHE_LINE_SIZE,
+			SOCKET_ID_ANY);
+	if (!h) {
+		DRV_LOG(ERR, "No memory for hash list %s creation",
+			name ? name : "None");
+		return NULL;
+	}
+	if (name)
+		snprintf(h->name, MLX5_HLIST_NAMESIZE, "%s", name);
+	h->table_sz = act_size;
+	h->mask = act_size - 1;
+	h->entry_sz = entry_size;
+	h->direct_key = !!(flags & MLX5_HLIST_DIRECT_KEY);
+	h->write_most = !!(flags & MLX5_HLIST_WRITE_MOST);
+	h->cb_create = cb_create ? cb_create : mlx5_hlist_default_create_cb;
+	h->cb_match = cb_match;
+	h->cb_remove = cb_remove ? cb_remove : mlx5_hlist_default_remove_cb;
+	for (i = 0; i < act_size; i++)
+		rte_rwlock_init(&h->buckets[i].lock);
+	DRV_LOG(DEBUG, "Hash list with %s size 0x%" PRIX32 " is created.",
+		h->name, act_size);
+	return h;
+}
+
+static struct mlx5_hlist_entry *
+__hlist_lookup(struct mlx5_hlist *h, uint64_t key, uint32_t idx,
+	       void *ctx, bool reuse)
+{
+	struct mlx5_hlist_head *first;
+	struct mlx5_hlist_entry *node;
+
+	MLX5_ASSERT(h);
+	first = &h->buckets[idx].head;
+	LIST_FOREACH(node, first, next) {
+		if (!h->cb_match(h, node, key, ctx)) {
+			if (reuse) {
+				__atomic_add_fetch(&node->ref_cnt, 1,
+						   __ATOMIC_RELAXED);
+				DRV_LOG(DEBUG, "Hash list %s entry %p "
+					"reuse: %u.",
+					h->name, (void *)node, node->ref_cnt);
+			}
+			break;
+		}
+	}
+	return node;
+}
+
+static struct mlx5_hlist_entry *
+hlist_lookup(struct mlx5_hlist *h, uint64_t key, uint32_t idx,
+	     void *ctx, bool reuse)
+{
+	struct mlx5_hlist_entry *node;
+
+	MLX5_ASSERT(h);
+	rte_rwlock_read_lock(&h->buckets[idx].lock);
+	node = __hlist_lookup(h, key, idx, ctx, reuse);
+	rte_rwlock_read_unlock(&h->buckets[idx].lock);
+	return node;
+}
+
+struct mlx5_hlist_entry *
+mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
+{
+	uint32_t idx;
+
+	if (h->direct_key)
+		idx = (uint32_t)(key & h->mask);
+	else
+		idx = rte_hash_crc_8byte(key, 0) & h->mask;
+	return hlist_lookup(h, key, idx, ctx, false);
+}
+
+struct mlx5_hlist_entry*
+mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
+{
+	uint32_t idx;
+	struct mlx5_hlist_head *first;
+	struct mlx5_hlist_bucket *b;
+	struct mlx5_hlist_entry *entry;
+	uint32_t prev_gen_cnt = 0;
+
+	if (h->direct_key)
+		idx = (uint32_t)(key & h->mask);
+	else
+		idx = rte_hash_crc_8byte(key, 0) & h->mask;
+	MLX5_ASSERT(h);
+	b = &h->buckets[idx];
+	/* Use write lock directly for write-most list. */
+	if (!h->write_most) {
+		prev_gen_cnt = __atomic_load_n(&b->gen_cnt, __ATOMIC_ACQUIRE);
+		entry = hlist_lookup(h, key, idx, ctx, true);
+		if (entry)
+			return entry;
+	}
+	rte_rwlock_write_lock(&b->lock);
+	/* Check if the list changed by other threads. */
+	if (h->write_most ||
+	    prev_gen_cnt != __atomic_load_n(&b->gen_cnt, __ATOMIC_ACQUIRE)) {
+		entry = __hlist_lookup(h, key, idx, ctx, true);
+		if (entry)
+			goto done;
+	}
+	first = &b->head;
+	entry = h->cb_create(h, key, ctx);
+	if (!entry) {
+		rte_errno = ENOMEM;
+		DRV_LOG(DEBUG, "Can't allocate hash list %s entry.", h->name);
+		goto done;
+	}
+	entry->idx = idx;
+	entry->ref_cnt = 1;
+	LIST_INSERT_HEAD(first, entry, next);
+	__atomic_add_fetch(&b->gen_cnt, 1, __ATOMIC_ACQ_REL);
+	DRV_LOG(DEBUG, "Hash list %s entry %p new: %u.",
+		h->name, (void *)entry, entry->ref_cnt);
+done:
+	rte_rwlock_write_unlock(&b->lock);
+	return entry;
+}
+
+int
+mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry)
+{
+	uint32_t idx = entry->idx;
+
+	rte_rwlock_write_lock(&h->buckets[idx].lock);
+	MLX5_ASSERT(entry && entry->ref_cnt && entry->next.le_prev);
+	DRV_LOG(DEBUG, "Hash list %s entry %p deref: %u.",
+		h->name, (void *)entry, entry->ref_cnt);
+	if (--entry->ref_cnt) {
+		rte_rwlock_write_unlock(&h->buckets[idx].lock);
+		return 1;
+	}
+	LIST_REMOVE(entry, next);
+	/* Set to NULL to get rid of removing action for more than once. */
+	entry->next.le_prev = NULL;
+	h->cb_remove(h, entry);
+	rte_rwlock_write_unlock(&h->buckets[idx].lock);
+	DRV_LOG(DEBUG, "Hash list %s entry %p removed.",
+		h->name, (void *)entry);
+	return 0;
+}
+
+void
+mlx5_hlist_destroy(struct mlx5_hlist *h)
+{
+	uint32_t idx;
+	struct mlx5_hlist_entry *entry;
+
+	MLX5_ASSERT(h);
+	for (idx = 0; idx < h->table_sz; ++idx) {
+		/* No LIST_FOREACH_SAFE, using while instead. */
+		while (!LIST_EMPTY(&h->buckets[idx].head)) {
+			entry = LIST_FIRST(&h->buckets[idx].head);
+			LIST_REMOVE(entry, next);
+			/*
+			 * The owner of whole element which contains data entry
+			 * is the user, so it's the user's duty to do the clean
+			 * up and the free work because someone may not put the
+			 * hlist entry at the beginning(suggested to locate at
+			 * the beginning). Or else the default free function
+			 * will be used.
+			 */
+			h->cb_remove(h, entry);
+		}
+	}
+	mlx5_free(h);
+}
diff --git a/drivers/common/mlx5/mlx5_common_utils.h b/drivers/common/mlx5/mlx5_common_utils.h
index 6cba39c8cc..ed378ce9bd 100644
--- a/drivers/common/mlx5/mlx5_common_utils.h
+++ b/drivers/common/mlx5/mlx5_common_utils.h
@@ -7,14 +7,202 @@
 
 #include "mlx5_common.h"
 
+#define MLX5_HLIST_DIRECT_KEY 0x0001 /* Use the key directly as hash index. */
+#define MLX5_HLIST_WRITE_MOST 0x0002 /* List mostly used for append new. */
 
-extern int mlx5_common_logtype;
+/** Maximum size of string for naming the hlist table. */
+#define MLX5_HLIST_NAMESIZE			32
 
-#define MLX5_COMMON_LOG_PREFIX "mlx5_common"
-/* Generic printf()-like logging macro with automatic line feed. */
-#define DRV_LOG(level, ...) \
-	PMD_DRV_LOG_(level, mlx5_common_logtype, MLX5_COMMON_LOG_PREFIX, \
-		__VA_ARGS__ PMD_DRV_LOG_STRIP PMD_DRV_LOG_OPAREN, \
-		PMD_DRV_LOG_CPAREN)
+struct mlx5_hlist;
+
+/**
+ * Structure of the entry in the hash list, user should define its own struct
+ * that contains this in order to store the data. The 'key' is 64-bits right
+ * now and its user's responsibility to guarantee there is no collision.
+ */
+struct mlx5_hlist_entry {
+	LIST_ENTRY(mlx5_hlist_entry) next; /* entry pointers in the list. */
+	uint32_t idx; /* Bucket index the entry belongs to. */
+	uint32_t ref_cnt; /* Reference count. */
+};
+
+/** Structure for hash head. */
+LIST_HEAD(mlx5_hlist_head, mlx5_hlist_entry);
+
+/**
+ * Type of callback function for entry removal.
+ *
+ * @param list
+ *   The hash list.
+ * @param entry
+ *   The entry in the list.
+ */
+typedef void (*mlx5_hlist_remove_cb)(struct mlx5_hlist *list,
+				     struct mlx5_hlist_entry *entry);
+
+/**
+ * Type of function for user defined matching.
+ *
+ * @param list
+ *   The hash list.
+ * @param entry
+ *   The entry in the list.
+ * @param key
+ *   The new entry key.
+ * @param ctx
+ *   The pointer to new entry context.
+ *
+ * @return
+ *   0 if matching, non-zero number otherwise.
+ */
+typedef int (*mlx5_hlist_match_cb)(struct mlx5_hlist *list,
+				   struct mlx5_hlist_entry *entry,
+				   uint64_t key, void *ctx);
+
+/**
+ * Type of function for user defined hash list entry creation.
+ *
+ * @param list
+ *   The hash list.
+ * @param key
+ *   The key of the new entry.
+ * @param ctx
+ *   The pointer to new entry context.
+ *
+ * @return
+ *   Pointer to allocated entry on success, NULL otherwise.
+ */
+typedef struct mlx5_hlist_entry *(*mlx5_hlist_create_cb)
+				  (struct mlx5_hlist *list,
+				   uint64_t key, void *ctx);
+
+/* Hash list bucket head. */
+struct mlx5_hlist_bucket {
+	struct mlx5_hlist_head head; /* List head. */
+	rte_rwlock_t lock; /* Bucket lock. */
+	uint32_t gen_cnt; /* List modification will update generation count. */
+} __rte_cache_aligned;
+
+/**
+ * Hash list table structure
+ *
+ * Entry in hash list could be reused if entry already exists, reference
+ * count will increase and the existing entry returns.
+ *
+ * When destroy an entry from list, decrease reference count and only
+ * destroy when no further reference.
+ */
+struct mlx5_hlist {
+	char name[MLX5_HLIST_NAMESIZE]; /**< Name of the hash list. */
+	/**< number of heads, need to be power of 2. */
+	uint32_t table_sz;
+	uint32_t entry_sz; /**< Size of entry, used to allocate entry. */
+	/**< mask to get the index of the list heads. */
+	uint32_t mask;
+	bool direct_key; /* Use the new entry key directly as hash index. */
+	bool write_most; /* List mostly used for append new or destroy. */
+	void *ctx;
+	mlx5_hlist_create_cb cb_create; /**< entry create callback. */
+	mlx5_hlist_match_cb cb_match; /**< entry match callback. */
+	mlx5_hlist_remove_cb cb_remove; /**< entry remove callback. */
+	struct mlx5_hlist_bucket buckets[] __rte_cache_aligned;
+	/**< list bucket arrays. */
+};
+
+/**
+ * Create a hash list table, the user can specify the list heads array size
+ * of the table, now the size should be a power of 2 in order to get better
+ * distribution for the entries. Each entry is a part of the whole data element
+ * and the caller should be responsible for the data element's allocation and
+ * cleanup / free. Key of each entry will be calculated with CRC in order to
+ * generate a little fairer distribution.
+ *
+ * @param name
+ *   Name of the hash list(optional).
+ * @param size
+ *   Heads array size of the hash list.
+ * @param entry_size
+ *   Entry size to allocate if cb_create not specified.
+ * @param flags
+ *   The hash list attribute flags.
+ * @param cb_create
+ *   Callback function for entry create.
+ * @param cb_match
+ *   Callback function for entry match.
+ * @param cb_destroy
+ *   Callback function for entry destroy.
+ * @return
+ *   Pointer of the hash list table created, NULL on failure.
+ */
+__rte_internal
+struct mlx5_hlist *mlx5_hlist_create(const char *name, uint32_t size,
+				     uint32_t entry_size, uint32_t flags,
+				     mlx5_hlist_create_cb cb_create,
+				     mlx5_hlist_match_cb cb_match,
+				     mlx5_hlist_remove_cb cb_destroy);
+
+/**
+ * Search an entry matching the key.
+ *
+ * Result returned might be destroyed by other thread, must use
+ * this function only in main thread.
+ *
+ * @param h
+ *   Pointer to the hast list table.
+ * @param key
+ *   Key for the searching entry.
+ * @param ctx
+ *   Common context parameter used by entry callback function.
+ *
+ * @return
+ *   Pointer of the hlist entry if found, NULL otherwise.
+ */
+__rte_internal
+struct mlx5_hlist_entry *mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key,
+					   void *ctx);
+
+/**
+ * Insert an entry to the hash list table, the entry is only part of whole data
+ * element and a 64B key is used for matching. User should construct the key or
+ * give a calculated hash signature and guarantee there is no collision.
+ *
+ * @param h
+ *   Pointer to the hast list table.
+ * @param entry
+ *   Entry to be inserted into the hash list table.
+ * @param ctx
+ *   Common context parameter used by callback function.
+ *
+ * @return
+ *   registered entry on success, NULL otherwise
+ */
+__rte_internal
+struct mlx5_hlist_entry *mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key,
+					     void *ctx);
+
+/**
+ * Remove an entry from the hash list table. User should guarantee the validity
+ * of the entry.
+ *
+ * @param h
+ *   Pointer to the hast list table. (not used)
+ * @param entry
+ *   Entry to be removed from the hash list table.
+ * @return
+ *   0 on entry removed, 1 on entry still referenced.
+ */
+__rte_internal
+int mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry);
+
+/**
+ * Destroy the hash list table, all the entries already inserted into the lists
+ * will be handled by the callback function provided by the user (including
+ * free if needed) before the table is freed.
+ *
+ * @param h
+ *   Pointer to the hast list table.
+ */
+__rte_internal
+void mlx5_hlist_destroy(struct mlx5_hlist *h);
 
 #endif /* RTE_PMD_MLX5_COMMON_UTILS_H_ */
diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c
index 5e082ebb78..c0061741e8 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.c
+++ b/drivers/common/mlx5/mlx5_devx_cmds.c
@@ -9,7 +9,7 @@
 
 #include "mlx5_prm.h"
 #include "mlx5_devx_cmds.h"
-#include "mlx5_common_utils.h"
+#include "mlx5_common_log.h"
 #include "mlx5_malloc.h"
 
 
diff --git a/drivers/common/mlx5/mlx5_malloc.c b/drivers/common/mlx5/mlx5_malloc.c
index 9d30cedbaa..b19501e1bc 100644
--- a/drivers/common/mlx5/mlx5_malloc.c
+++ b/drivers/common/mlx5/mlx5_malloc.c
@@ -8,7 +8,7 @@
 #include <stdbool.h>
 #include <string.h>
 
-#include "mlx5_common_utils.h"
+#include "mlx5_common_log.h"
 #include "mlx5_common_os.h"
 #include "mlx5_malloc.h"
 
diff --git a/drivers/common/mlx5/version.map b/drivers/common/mlx5/version.map
index bbef436fde..d16e484ffa 100644
--- a/drivers/common/mlx5/version.map
+++ b/drivers/common/mlx5/version.map
@@ -121,4 +121,10 @@ INTERNAL {
 	mlx5_free;
 
 	mlx5_pci_driver_register;
+
+	mlx5_hlist_create;
+	mlx5_hlist_lookup;
+	mlx5_hlist_register;
+	mlx5_hlist_unregister;
+	mlx5_hlist_destroy;
 };
diff --git a/drivers/common/mlx5/windows/mlx5_common_os.c b/drivers/common/mlx5/windows/mlx5_common_os.c
index f2d781a965..2e6e172a96 100644
--- a/drivers/common/mlx5/windows/mlx5_common_os.c
+++ b/drivers/common/mlx5/windows/mlx5_common_os.c
@@ -11,7 +11,7 @@
 #include <rte_errno.h>
 
 #include "mlx5_devx_cmds.h"
-#include "mlx5_common_utils.h"
+#include "../mlx5_common_log.h"
 #include "mlx5_common.h"
 #include "mlx5_common_os.h"
 #include "mlx5_malloc.h"
diff --git a/drivers/common/mlx5/windows/mlx5_glue.c b/drivers/common/mlx5/windows/mlx5_glue.c
index aef6d3b5f4..535487a8d4 100644
--- a/drivers/common/mlx5/windows/mlx5_glue.c
+++ b/drivers/common/mlx5/windows/mlx5_glue.c
@@ -12,7 +12,7 @@
 #include <rte_malloc.h>
 
 #include "mlx5_glue.h"
-#include "mlx5_common_utils.h"
+#include "../mlx5_common_log.h"
 #include "mlx5_win_ext.h"
 
 /*
diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index a39b5edddc..18fe23e4fb 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -3,220 +3,11 @@
  */
 
 #include <rte_malloc.h>
-#include <rte_hash_crc.h>
 
 #include <mlx5_malloc.h>
 
 #include "mlx5_utils.h"
 
-/********************* Hash List **********************/
-
-static struct mlx5_hlist_entry *
-mlx5_hlist_default_create_cb(struct mlx5_hlist *h, uint64_t key __rte_unused,
-			     void *ctx __rte_unused)
-{
-	return mlx5_malloc(MLX5_MEM_ZERO, h->entry_sz, 0, SOCKET_ID_ANY);
-}
-
-static void
-mlx5_hlist_default_remove_cb(struct mlx5_hlist *h __rte_unused,
-			     struct mlx5_hlist_entry *entry)
-{
-	mlx5_free(entry);
-}
-
-struct mlx5_hlist *
-mlx5_hlist_create(const char *name, uint32_t size, uint32_t entry_size,
-		  uint32_t flags, mlx5_hlist_create_cb cb_create,
-		  mlx5_hlist_match_cb cb_match, mlx5_hlist_remove_cb cb_remove)
-{
-	struct mlx5_hlist *h;
-	uint32_t act_size;
-	uint32_t alloc_size;
-	uint32_t i;
-
-	if (!size || !cb_match || (!cb_create ^ !cb_remove))
-		return NULL;
-	/* Align to the next power of 2, 32bits integer is enough now. */
-	if (!rte_is_power_of_2(size)) {
-		act_size = rte_align32pow2(size);
-		DRV_LOG(DEBUG, "Size 0x%" PRIX32 " is not power of 2, "
-			"will be aligned to 0x%" PRIX32 ".", size, act_size);
-	} else {
-		act_size = size;
-	}
-	alloc_size = sizeof(struct mlx5_hlist) +
-		     sizeof(struct mlx5_hlist_bucket) * act_size;
-	/* Using zmalloc, then no need to initialize the heads. */
-	h = mlx5_malloc(MLX5_MEM_ZERO, alloc_size, RTE_CACHE_LINE_SIZE,
-			SOCKET_ID_ANY);
-	if (!h) {
-		DRV_LOG(ERR, "No memory for hash list %s creation",
-			name ? name : "None");
-		return NULL;
-	}
-	if (name)
-		snprintf(h->name, MLX5_HLIST_NAMESIZE, "%s", name);
-	h->table_sz = act_size;
-	h->mask = act_size - 1;
-	h->entry_sz = entry_size;
-	h->direct_key = !!(flags & MLX5_HLIST_DIRECT_KEY);
-	h->write_most = !!(flags & MLX5_HLIST_WRITE_MOST);
-	h->cb_create = cb_create ? cb_create : mlx5_hlist_default_create_cb;
-	h->cb_match = cb_match;
-	h->cb_remove = cb_remove ? cb_remove : mlx5_hlist_default_remove_cb;
-	for (i = 0; i < act_size; i++)
-		rte_rwlock_init(&h->buckets[i].lock);
-	DRV_LOG(DEBUG, "Hash list with %s size 0x%" PRIX32 " is created.",
-		h->name, act_size);
-	return h;
-}
-
-static struct mlx5_hlist_entry *
-__hlist_lookup(struct mlx5_hlist *h, uint64_t key, uint32_t idx,
-	       void *ctx, bool reuse)
-{
-	struct mlx5_hlist_head *first;
-	struct mlx5_hlist_entry *node;
-
-	MLX5_ASSERT(h);
-	first = &h->buckets[idx].head;
-	LIST_FOREACH(node, first, next) {
-		if (!h->cb_match(h, node, key, ctx)) {
-			if (reuse) {
-				__atomic_add_fetch(&node->ref_cnt, 1,
-						   __ATOMIC_RELAXED);
-				DRV_LOG(DEBUG, "Hash list %s entry %p "
-					"reuse: %u.",
-					h->name, (void *)node, node->ref_cnt);
-			}
-			break;
-		}
-	}
-	return node;
-}
-
-static struct mlx5_hlist_entry *
-hlist_lookup(struct mlx5_hlist *h, uint64_t key, uint32_t idx,
-	     void *ctx, bool reuse)
-{
-	struct mlx5_hlist_entry *node;
-
-	MLX5_ASSERT(h);
-	rte_rwlock_read_lock(&h->buckets[idx].lock);
-	node = __hlist_lookup(h, key, idx, ctx, reuse);
-	rte_rwlock_read_unlock(&h->buckets[idx].lock);
-	return node;
-}
-
-struct mlx5_hlist_entry *
-mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
-{
-	uint32_t idx;
-
-	if (h->direct_key)
-		idx = (uint32_t)(key & h->mask);
-	else
-		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	return hlist_lookup(h, key, idx, ctx, false);
-}
-
-struct mlx5_hlist_entry*
-mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
-{
-	uint32_t idx;
-	struct mlx5_hlist_head *first;
-	struct mlx5_hlist_bucket *b;
-	struct mlx5_hlist_entry *entry;
-	uint32_t prev_gen_cnt = 0;
-
-	if (h->direct_key)
-		idx = (uint32_t)(key & h->mask);
-	else
-		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	MLX5_ASSERT(h);
-	b = &h->buckets[idx];
-	/* Use write lock directly for write-most list. */
-	if (!h->write_most) {
-		prev_gen_cnt = __atomic_load_n(&b->gen_cnt, __ATOMIC_ACQUIRE);
-		entry = hlist_lookup(h, key, idx, ctx, true);
-		if (entry)
-			return entry;
-	}
-	rte_rwlock_write_lock(&b->lock);
-	/* Check if the list changed by other threads. */
-	if (h->write_most ||
-	    prev_gen_cnt != __atomic_load_n(&b->gen_cnt, __ATOMIC_ACQUIRE)) {
-		entry = __hlist_lookup(h, key, idx, ctx, true);
-		if (entry)
-			goto done;
-	}
-	first = &b->head;
-	entry = h->cb_create(h, key, ctx);
-	if (!entry) {
-		rte_errno = ENOMEM;
-		DRV_LOG(DEBUG, "Can't allocate hash list %s entry.", h->name);
-		goto done;
-	}
-	entry->idx = idx;
-	entry->ref_cnt = 1;
-	LIST_INSERT_HEAD(first, entry, next);
-	__atomic_add_fetch(&b->gen_cnt, 1, __ATOMIC_ACQ_REL);
-	DRV_LOG(DEBUG, "Hash list %s entry %p new: %u.",
-		h->name, (void *)entry, entry->ref_cnt);
-done:
-	rte_rwlock_write_unlock(&b->lock);
-	return entry;
-}
-
-int
-mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry)
-{
-	uint32_t idx = entry->idx;
-
-	rte_rwlock_write_lock(&h->buckets[idx].lock);
-	MLX5_ASSERT(entry && entry->ref_cnt && entry->next.le_prev);
-	DRV_LOG(DEBUG, "Hash list %s entry %p deref: %u.",
-		h->name, (void *)entry, entry->ref_cnt);
-	if (--entry->ref_cnt) {
-		rte_rwlock_write_unlock(&h->buckets[idx].lock);
-		return 1;
-	}
-	LIST_REMOVE(entry, next);
-	/* Set to NULL to get rid of removing action for more than once. */
-	entry->next.le_prev = NULL;
-	h->cb_remove(h, entry);
-	rte_rwlock_write_unlock(&h->buckets[idx].lock);
-	DRV_LOG(DEBUG, "Hash list %s entry %p removed.",
-		h->name, (void *)entry);
-	return 0;
-}
-
-void
-mlx5_hlist_destroy(struct mlx5_hlist *h)
-{
-	uint32_t idx;
-	struct mlx5_hlist_entry *entry;
-
-	MLX5_ASSERT(h);
-	for (idx = 0; idx < h->table_sz; ++idx) {
-		/* No LIST_FOREACH_SAFE, using while instead. */
-		while (!LIST_EMPTY(&h->buckets[idx].head)) {
-			entry = LIST_FIRST(&h->buckets[idx].head);
-			LIST_REMOVE(entry, next);
-			/*
-			 * The owner of whole element which contains data entry
-			 * is the user, so it's the user's duty to do the clean
-			 * up and the free work because someone may not put the
-			 * hlist entry at the beginning(suggested to locate at
-			 * the beginning). Or else the default free function
-			 * will be used.
-			 */
-			h->cb_remove(h, entry);
-		}
-	}
-	mlx5_free(h);
-}
 
 /********************* Cache list ************************/
 
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index 289941cebc..b54517c6df 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -18,6 +18,7 @@
 #include <rte_bitmap.h>
 
 #include <mlx5_common.h>
+#include <mlx5_common_utils.h>
 
 #include "mlx5_defs.h"
 
@@ -261,199 +262,6 @@ log2above(unsigned int v)
 	return l + r;
 }
 
-#define MLX5_HLIST_DIRECT_KEY 0x0001 /* Use the key directly as hash index. */
-#define MLX5_HLIST_WRITE_MOST 0x0002 /* List mostly used for append new. */
-
-/** Maximum size of string for naming the hlist table. */
-#define MLX5_HLIST_NAMESIZE			32
-
-struct mlx5_hlist;
-
-/**
- * Structure of the entry in the hash list, user should define its own struct
- * that contains this in order to store the data. The 'key' is 64-bits right
- * now and its user's responsibility to guarantee there is no collision.
- */
-struct mlx5_hlist_entry {
-	LIST_ENTRY(mlx5_hlist_entry) next; /* entry pointers in the list. */
-	uint32_t idx; /* Bucket index the entry belongs to. */
-	uint32_t ref_cnt; /* Reference count. */
-};
-
-/** Structure for hash head. */
-LIST_HEAD(mlx5_hlist_head, mlx5_hlist_entry);
-
-/**
- * Type of callback function for entry removal.
- *
- * @param list
- *   The hash list.
- * @param entry
- *   The entry in the list.
- */
-typedef void (*mlx5_hlist_remove_cb)(struct mlx5_hlist *list,
-				     struct mlx5_hlist_entry *entry);
-
-/**
- * Type of function for user defined matching.
- *
- * @param list
- *   The hash list.
- * @param entry
- *   The entry in the list.
- * @param key
- *   The new entry key.
- * @param ctx
- *   The pointer to new entry context.
- *
- * @return
- *   0 if matching, non-zero number otherwise.
- */
-typedef int (*mlx5_hlist_match_cb)(struct mlx5_hlist *list,
-				   struct mlx5_hlist_entry *entry,
-				   uint64_t key, void *ctx);
-
-/**
- * Type of function for user defined hash list entry creation.
- *
- * @param list
- *   The hash list.
- * @param key
- *   The key of the new entry.
- * @param ctx
- *   The pointer to new entry context.
- *
- * @return
- *   Pointer to allocated entry on success, NULL otherwise.
- */
-typedef struct mlx5_hlist_entry *(*mlx5_hlist_create_cb)
-				  (struct mlx5_hlist *list,
-				   uint64_t key, void *ctx);
-
-/* Hash list bucket head. */
-struct mlx5_hlist_bucket {
-	struct mlx5_hlist_head head; /* List head. */
-	rte_rwlock_t lock; /* Bucket lock. */
-	uint32_t gen_cnt; /* List modification will update generation count. */
-} __rte_cache_aligned;
-
-/**
- * Hash list table structure
- *
- * Entry in hash list could be reused if entry already exists, reference
- * count will increase and the existing entry returns.
- *
- * When destroy an entry from list, decrease reference count and only
- * destroy when no further reference.
- */
-struct mlx5_hlist {
-	char name[MLX5_HLIST_NAMESIZE]; /**< Name of the hash list. */
-	/**< number of heads, need to be power of 2. */
-	uint32_t table_sz;
-	uint32_t entry_sz; /**< Size of entry, used to allocate entry. */
-	/**< mask to get the index of the list heads. */
-	uint32_t mask;
-	bool direct_key; /* Use the new entry key directly as hash index. */
-	bool write_most; /* List mostly used for append new or destroy. */
-	void *ctx;
-	mlx5_hlist_create_cb cb_create; /**< entry create callback. */
-	mlx5_hlist_match_cb cb_match; /**< entry match callback. */
-	mlx5_hlist_remove_cb cb_remove; /**< entry remove callback. */
-	struct mlx5_hlist_bucket buckets[] __rte_cache_aligned;
-	/**< list bucket arrays. */
-};
-
-/**
- * Create a hash list table, the user can specify the list heads array size
- * of the table, now the size should be a power of 2 in order to get better
- * distribution for the entries. Each entry is a part of the whole data element
- * and the caller should be responsible for the data element's allocation and
- * cleanup / free. Key of each entry will be calculated with CRC in order to
- * generate a little fairer distribution.
- *
- * @param name
- *   Name of the hash list(optional).
- * @param size
- *   Heads array size of the hash list.
- * @param entry_size
- *   Entry size to allocate if cb_create not specified.
- * @param flags
- *   The hash list attribute flags.
- * @param cb_create
- *   Callback function for entry create.
- * @param cb_match
- *   Callback function for entry match.
- * @param cb_destroy
- *   Callback function for entry destroy.
- * @return
- *   Pointer of the hash list table created, NULL on failure.
- */
-struct mlx5_hlist *mlx5_hlist_create(const char *name, uint32_t size,
-				     uint32_t entry_size, uint32_t flags,
-				     mlx5_hlist_create_cb cb_create,
-				     mlx5_hlist_match_cb cb_match,
-				     mlx5_hlist_remove_cb cb_destroy);
-
-/**
- * Search an entry matching the key.
- *
- * Result returned might be destroyed by other thread, must use
- * this function only in main thread.
- *
- * @param h
- *   Pointer to the hast list table.
- * @param key
- *   Key for the searching entry.
- * @param ctx
- *   Common context parameter used by entry callback function.
- *
- * @return
- *   Pointer of the hlist entry if found, NULL otherwise.
- */
-struct mlx5_hlist_entry *mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key,
-					   void *ctx);
-
-/**
- * Insert an entry to the hash list table, the entry is only part of whole data
- * element and a 64B key is used for matching. User should construct the key or
- * give a calculated hash signature and guarantee there is no collision.
- *
- * @param h
- *   Pointer to the hast list table.
- * @param entry
- *   Entry to be inserted into the hash list table.
- * @param ctx
- *   Common context parameter used by callback function.
- *
- * @return
- *   registered entry on success, NULL otherwise
- */
-struct mlx5_hlist_entry *mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key,
-					     void *ctx);
-
-/**
- * Remove an entry from the hash list table. User should guarantee the validity
- * of the entry.
- *
- * @param h
- *   Pointer to the hast list table. (not used)
- * @param entry
- *   Entry to be removed from the hash list table.
- * @return
- *   0 on entry removed, 1 on entry still referenced.
- */
-int mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry);
-
-/**
- * Destroy the hash list table, all the entries already inserted into the lists
- * will be handled by the callback function provided by the user (including
- * free if needed) before the table is freed.
- *
- * @param h
- *   Pointer to the hast list table.
- */
-void mlx5_hlist_destroy(struct mlx5_hlist *h);
-
 /************************ cache list *****************************/
 
 /** Maximum size of string for naming. */
-- 
2.25.1

[dpdk-dev] [PATCH v2 12/16] common/mlx5: share get ib device match function

[Matan Azrad]

From: Shiri Kuzin <shirik@nvidia.com>

The get_ib_device_match function iterates over the list of ib devices
returned by the get_device_list glue function and returns the ib device
matching the provided address.

Since this function is in use by several drivers, in this patch we
share the function in common part.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/linux/mlx5_common_os.c | 28 +++++++++++++++++++
 drivers/common/mlx5/linux/mlx5_common_os.h |  5 ++++
 drivers/common/mlx5/mlx5_common.h          |  2 ++
 drivers/common/mlx5/version.map            |  1 +
 drivers/compress/mlx5/mlx5_compress.c      | 30 +-------------------
 drivers/regex/mlx5/mlx5_regex.c            | 30 ++------------------
 drivers/vdpa/mlx5/mlx5_vdpa.c              | 32 ++--------------------
 7 files changed, 41 insertions(+), 87 deletions(-)

diff --git a/drivers/common/mlx5/linux/mlx5_common_os.c b/drivers/common/mlx5/linux/mlx5_common_os.c
index fba8245b8b..037147fe31 100644
--- a/drivers/common/mlx5/linux/mlx5_common_os.c
+++ b/drivers/common/mlx5/linux/mlx5_common_os.c
@@ -16,6 +16,7 @@
 
 #include "mlx5_common.h"
 #include "mlx5_common_log.h"
+#include "mlx5_common_os.h"
 #include "mlx5_glue.h"
 
 #ifdef MLX5_GLUE
@@ -423,3 +424,30 @@ mlx5_glue_constructor(void)
 	mlx5_glue = NULL;
 }
 
+struct ibv_device *
+mlx5_os_get_ib_device_match(struct rte_pci_addr *addr)
+{
+	int n;
+	struct ibv_device **ibv_list = mlx5_glue->get_device_list(&n);
+	struct ibv_device *ibv_match = NULL;
+
+	if (ibv_list == NULL) {
+		rte_errno = ENOSYS;
+		return NULL;
+	}
+	while (n-- > 0) {
+		struct rte_pci_addr paddr;
+
+		DRV_LOG(DEBUG, "Checking device \"%s\"..", ibv_list[n]->name);
+		if (mlx5_dev_to_pci_addr(ibv_list[n]->ibdev_path, &paddr) != 0)
+			continue;
+		if (rte_pci_addr_cmp(addr, &paddr) != 0)
+			continue;
+		ibv_match = ibv_list[n];
+		break;
+	}
+	if (ibv_match == NULL)
+		rte_errno = ENOENT;
+	mlx5_glue->free_device_list(ibv_list);
+	return ibv_match;
+}
diff --git a/drivers/common/mlx5/linux/mlx5_common_os.h b/drivers/common/mlx5/linux/mlx5_common_os.h
index d1c7e3dce0..9fff9cdb83 100644
--- a/drivers/common/mlx5/linux/mlx5_common_os.h
+++ b/drivers/common/mlx5/linux/mlx5_common_os.h
@@ -288,4 +288,9 @@ mlx5_os_free(void *addr)
 {
 	free(addr);
 }
+
+__rte_internal
+struct ibv_device *
+mlx5_os_get_ib_device_match(struct rte_pci_addr *addr);
+
 #endif /* RTE_PMD_MLX5_COMMON_OS_H_ */
diff --git a/drivers/common/mlx5/mlx5_common.h b/drivers/common/mlx5/mlx5_common.h
index f3c6beb23b..89aca32305 100644
--- a/drivers/common/mlx5/mlx5_common.h
+++ b/drivers/common/mlx5/mlx5_common.h
@@ -18,6 +18,7 @@
 
 #include "mlx5_prm.h"
 #include "mlx5_devx_cmds.h"
+#include "mlx5_common_os.h"
 
 /* Reported driver name. */
 #define MLX5_PCI_DRIVER_NAME "mlx5_pci"
@@ -215,6 +216,7 @@ enum mlx5_class {
 	MLX5_CLASS_VDPA = RTE_BIT64(1),
 	MLX5_CLASS_REGEX = RTE_BIT64(2),
 	MLX5_CLASS_COMPRESS = RTE_BIT64(3),
+	MLX5_CLASS_CRYPTO = RTE_BIT64(4),
 };
 
 #define MLX5_DBR_SIZE RTE_CACHE_LINE_SIZE
diff --git a/drivers/common/mlx5/version.map b/drivers/common/mlx5/version.map
index d16e484ffa..00df37e81a 100644
--- a/drivers/common/mlx5/version.map
+++ b/drivers/common/mlx5/version.map
@@ -111,6 +111,7 @@ INTERNAL {
 	mlx5_os_reg_mr;
 	mlx5_os_umem_reg;
 	mlx5_os_umem_dereg;
+	mlx5_os_get_ib_device_match; # WINDOWS_NO_EXPORT
 
 	mlx5_translate_port_name; # WINDOWS_NO_EXPORT
 
diff --git a/drivers/compress/mlx5/mlx5_compress.c b/drivers/compress/mlx5/mlx5_compress.c
index ec3c237512..ff988d26ef 100644
--- a/drivers/compress/mlx5/mlx5_compress.c
+++ b/drivers/compress/mlx5/mlx5_compress.c
@@ -647,34 +647,6 @@ mlx5_compress_dequeue_burst(void *queue_pair, struct rte_comp_op **ops,
 	return i;
 }
 
-static struct ibv_device *
-mlx5_compress_get_ib_device_match(struct rte_pci_addr *addr)
-{
-	int n;
-	struct ibv_device **ibv_list = mlx5_glue->get_device_list(&n);
-	struct ibv_device *ibv_match = NULL;
-
-	if (ibv_list == NULL) {
-		rte_errno = ENOSYS;
-		return NULL;
-	}
-	while (n-- > 0) {
-		struct rte_pci_addr paddr;
-
-		DRV_LOG(DEBUG, "Checking device \"%s\"..", ibv_list[n]->name);
-		if (mlx5_dev_to_pci_addr(ibv_list[n]->ibdev_path, &paddr) != 0)
-			continue;
-		if (rte_pci_addr_cmp(addr, &paddr) != 0)
-			continue;
-		ibv_match = ibv_list[n];
-		break;
-	}
-	if (ibv_match == NULL)
-		rte_errno = ENOENT;
-	mlx5_glue->free_device_list(ibv_list);
-	return ibv_match;
-}
-
 static void
 mlx5_compress_hw_global_release(struct mlx5_compress_priv *priv)
 {
@@ -774,7 +746,7 @@ mlx5_compress_pci_probe(struct rte_pci_driver *pci_drv,
 		rte_errno = ENOTSUP;
 		return -rte_errno;
 	}
-	ibv = mlx5_compress_get_ib_device_match(&pci_dev->addr);
+	ibv = mlx5_os_get_ib_device_match(&pci_dev->addr);
 	if (ibv == NULL) {
 		DRV_LOG(ERR, "No matching IB device for PCI slot "
 			PCI_PRI_FMT ".", pci_dev->addr.domain,
diff --git a/drivers/regex/mlx5/mlx5_regex.c b/drivers/regex/mlx5/mlx5_regex.c
index 82c485e50c..5b81666d21 100644
--- a/drivers/regex/mlx5/mlx5_regex.c
+++ b/drivers/regex/mlx5/mlx5_regex.c
@@ -11,6 +11,7 @@
 #include <rte_regexdev_driver.h>
 
 #include <mlx5_common_pci.h>
+#include <mlx5_common.h>
 #include <mlx5_glue.h>
 #include <mlx5_devx_cmds.h>
 #include <mlx5_prm.h>
@@ -52,33 +53,6 @@ mlx5_regex_close(struct rte_regexdev *dev __rte_unused)
 	return 0;
 }
 
-static struct ibv_device *
-mlx5_regex_get_ib_device_match(struct rte_pci_addr *addr)
-{
-	int n;
-	struct ibv_device **ibv_list = mlx5_glue->get_device_list(&n);
-	struct ibv_device *ibv_match = NULL;
-
-	if (!ibv_list) {
-		rte_errno = ENOSYS;
-		return NULL;
-	}
-	while (n-- > 0) {
-		struct rte_pci_addr pci_addr;
-
-		DRV_LOG(DEBUG, "Checking device \"%s\"..", ibv_list[n]->name);
-		if (mlx5_dev_to_pci_addr(ibv_list[n]->ibdev_path, &pci_addr))
-			continue;
-		if (rte_pci_addr_cmp(addr, &pci_addr))
-			continue;
-		ibv_match = ibv_list[n];
-		break;
-	}
-	if (!ibv_match)
-		rte_errno = ENOENT;
-	mlx5_glue->free_device_list(ibv_list);
-	return ibv_match;
-}
 static int
 mlx5_regex_engines_status(struct ibv_context *ctx, int num_engines)
 {
@@ -121,7 +95,7 @@ mlx5_regex_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 	int ret;
 	uint32_t val;
 
-	ibv = mlx5_regex_get_ib_device_match(&pci_dev->addr);
+	ibv = mlx5_os_get_ib_device_match(&pci_dev->addr);
 	if (!ibv) {
 		DRV_LOG(ERR, "No matching IB device for PCI slot "
 			PCI_PRI_FMT ".", pci_dev->addr.domain,
diff --git a/drivers/vdpa/mlx5/mlx5_vdpa.c b/drivers/vdpa/mlx5/mlx5_vdpa.c
index 898e50f807..9a946ac8a7 100644
--- a/drivers/vdpa/mlx5/mlx5_vdpa.c
+++ b/drivers/vdpa/mlx5/mlx5_vdpa.c
@@ -472,34 +472,6 @@ static struct rte_vdpa_dev_ops mlx5_vdpa_ops = {
 	.reset_stats = mlx5_vdpa_reset_stats,
 };
 
-static struct ibv_device *
-mlx5_vdpa_get_ib_device_match(struct rte_pci_addr *addr)
-{
-	int n;
-	struct ibv_device **ibv_list = mlx5_glue->get_device_list(&n);
-	struct ibv_device *ibv_match = NULL;
-
-	if (!ibv_list) {
-		rte_errno = ENOSYS;
-		return NULL;
-	}
-	while (n-- > 0) {
-		struct rte_pci_addr pci_addr;
-
-		DRV_LOG(DEBUG, "Checking device \"%s\"..", ibv_list[n]->name);
-		if (mlx5_dev_to_pci_addr(ibv_list[n]->ibdev_path, &pci_addr))
-			continue;
-		if (rte_pci_addr_cmp(addr, &pci_addr))
-			continue;
-		ibv_match = ibv_list[n];
-		break;
-	}
-	if (!ibv_match)
-		rte_errno = ENOENT;
-	mlx5_glue->free_device_list(ibv_list);
-	return ibv_match;
-}
-
 /* Try to disable ROCE by Netlink\Devlink. */
 static int
 mlx5_vdpa_nl_roce_disable(const char *addr)
@@ -595,7 +567,7 @@ mlx5_vdpa_roce_disable(struct rte_pci_addr *addr, struct ibv_device **ibv)
 		struct ibv_device *ibv_new;
 
 		for (r = MLX5_VDPA_MAX_RETRIES; r; r--) {
-			ibv_new = mlx5_vdpa_get_ib_device_match(addr);
+			ibv_new = mlx5_os_get_ib_device_match(addr);
 			if (ibv_new) {
 				*ibv = ibv_new;
 				return 0;
@@ -698,7 +670,7 @@ mlx5_vdpa_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 	struct mlx5_hca_attr attr;
 	int ret;
 
-	ibv = mlx5_vdpa_get_ib_device_match(&pci_dev->addr);
+	ibv = mlx5_os_get_ib_device_match(&pci_dev->addr);
 	if (!ibv) {
 		DRV_LOG(ERR, "No matching IB device for PCI slot "
 			PCI_PRI_FMT ".", pci_dev->addr.domain,
-- 
2.25.1

[dpdk-dev] [PATCH v2 13/16] common/mlx5: support general obj CREDENTIAL create

[Matan Azrad]

From: Dekel Peled <dekelp@nvidia.com>

CREDENTIAL object is used for any crypto operation in wrapped mode.

This patch add support of CREDENTIAL object create operation.
Add reading of CREDENTIAL support capability.
Add function to create general object type CREDENTIAL, using DevX API.

Signed-off-by: Dekel Peled <dekelp@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_devx_cmds.c | 53 +++++++++++++++++++++++++++-
 drivers/common/mlx5/mlx5_devx_cmds.h | 17 +++++++--
 drivers/common/mlx5/mlx5_prm.h       | 23 ++++++++++++
 drivers/common/mlx5/version.map      |  1 +
 4 files changed, 91 insertions(+), 3 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c
index c0061741e8..c0a0853c3a 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.c
+++ b/drivers/common/mlx5/mlx5_devx_cmds.c
@@ -754,6 +754,8 @@ mlx5_devx_cmd_query_hca_attr(void *ctx,
 		       MLX5_GENERAL_OBJ_TYPES_CAP_DEK);
 	attr->import_kek = !!(general_obj_types_supported &
 			      MLX5_GENERAL_OBJ_TYPES_CAP_IMPORT_KEK);
+	attr->credential = !!(general_obj_types_supported &
+			      MLX5_GENERAL_OBJ_TYPES_CAP_CREDENTIAL);
 	attr->crypto_login = !!(general_obj_types_supported &
 				MLX5_GENERAL_OBJ_TYPES_CAP_CRYPTO_LOGIN);
 	/* Add reading of other GENERAL_OBJ_TYPES_CAP bits above this line. */
@@ -2512,6 +2514,55 @@ mlx5_devx_cmd_create_import_kek_obj(void *ctx,
 	return import_kek_obj;
 }
 
+/**
+ * Create general object of type CREDENTIAL using DevX API.
+ *
+ * @param[in] ctx
+ *   Context returned from mlx5 open_device() glue function.
+ * @param [in] attr
+ *   Pointer to CREDENTIAL attributes structure.
+ *
+ * @return
+ *   The DevX object created, NULL otherwise and rte_errno is set.
+ */
+struct mlx5_devx_obj *
+mlx5_devx_cmd_create_credential_obj(void *ctx,
+				    struct mlx5_devx_credential_attr *attr)
+{
+	uint32_t in[MLX5_ST_SZ_DW(create_credential_in)] = {0};
+	uint32_t out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};
+	struct mlx5_devx_obj *credential_obj = NULL;
+	void *ptr = NULL, *credential_addr = NULL;
+
+	credential_obj = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*credential_obj),
+				     0, SOCKET_ID_ANY);
+	if (credential_obj == NULL) {
+		DRV_LOG(ERR, "Failed to allocate CREDENTIAL object data");
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+	ptr = MLX5_ADDR_OF(create_credential_in, in, hdr);
+	MLX5_SET(general_obj_in_cmd_hdr, ptr, opcode,
+		 MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
+	MLX5_SET(general_obj_in_cmd_hdr, ptr, obj_type,
+		 MLX5_GENERAL_OBJ_TYPE_CREDENTIAL);
+	ptr = MLX5_ADDR_OF(create_credential_in, in, credential);
+	MLX5_SET(credential, ptr, credential_role, attr->credential_role);
+	credential_addr = MLX5_ADDR_OF(credential, ptr, credential);
+	memcpy(credential_addr, (void *)(attr->credential),
+	       MLX5_CRYPTO_CREDENTIAL_SIZE);
+	credential_obj->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in),
+							 out, sizeof(out));
+	if (credential_obj->obj == NULL) {
+		rte_errno = errno;
+		DRV_LOG(ERR, "Failed to create CREDENTIAL object using DevX.");
+		mlx5_free(credential_obj);
+		return NULL;
+	}
+	credential_obj->id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
+	return credential_obj;
+}
+
 /**
  * Create general object of type CRYPTO_LOGIN using DevX API.
  *
@@ -2551,7 +2602,7 @@ mlx5_devx_cmd_create_crypto_login_obj(void *ctx,
 		 attr->session_import_kek_ptr);
 	credential_addr = MLX5_ADDR_OF(crypto_login, ptr, credential);
 	memcpy(credential_addr, (void *)(attr->credential),
-	       MLX5_CRYPTO_LOGIN_CREDENTIAL_SIZE);
+	       MLX5_CRYPTO_CREDENTIAL_SIZE);
 	crypto_login_obj->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in),
 							   out, sizeof(out));
 	if (crypto_login_obj->obj == NULL) {
diff --git a/drivers/common/mlx5/mlx5_devx_cmds.h b/drivers/common/mlx5/mlx5_devx_cmds.h
index 709e28bfba..811e7a1462 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.h
+++ b/drivers/common/mlx5/mlx5_devx_cmds.h
@@ -143,6 +143,7 @@ struct mlx5_hca_attr {
 	uint32_t aes_xts:1; /* AES-XTS crypto is supported. */
 	uint32_t dek:1; /* General obj type DEK is supported. */
 	uint32_t import_kek:1; /* General obj type IMPORT_KEK supported. */
+	uint32_t credential:1; /* General obj type CREDENTIAL supported. */
 	uint32_t crypto_login:1; /* General obj type CRYPTO_LOGIN supported. */
 	uint32_t regexp_num_of_engines;
 	uint32_t log_max_ft_sampler_num:8;
@@ -459,13 +460,20 @@ struct mlx5_devx_import_kek_attr {
 	uint8_t key[MLX5_CRYPTO_KEY_MAX_SIZE];
 };
 
-#define MLX5_CRYPTO_LOGIN_CREDENTIAL_SIZE	48
+#define MLX5_CRYPTO_CREDENTIAL_SIZE	48
+
+struct mlx5_devx_credential_attr {
+	uint64_t modify_field_select;
+	uint32_t state:8;
+	uint32_t credential_role:8;
+	uint8_t credential[MLX5_CRYPTO_CREDENTIAL_SIZE];
+};
 
 struct mlx5_devx_crypto_login_attr {
 	uint64_t modify_field_select;
 	uint32_t credential_pointer:24;
 	uint32_t session_import_kek_ptr:24;
-	uint8_t credential[MLX5_CRYPTO_LOGIN_CREDENTIAL_SIZE];
+	uint8_t credential[MLX5_CRYPTO_CREDENTIAL_SIZE];
 };
 
 /* mlx5_devx_cmds.c */
@@ -629,6 +637,11 @@ struct mlx5_devx_obj *
 mlx5_devx_cmd_create_import_kek_obj(void *ctx,
 				    struct mlx5_devx_import_kek_attr *attr);
 
+__rte_internal
+struct mlx5_devx_obj *
+mlx5_devx_cmd_create_credential_obj(void *ctx,
+				    struct mlx5_devx_credential_attr *attr);
+
 __rte_internal
 struct mlx5_devx_obj *
 mlx5_devx_cmd_create_crypto_login_obj(void *ctx,
diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index a9dcbfa63c..432c8fdb63 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -1187,6 +1187,8 @@ enum {
 			(1ULL << MLX5_GENERAL_OBJ_TYPE_DEK)
 #define MLX5_GENERAL_OBJ_TYPES_CAP_IMPORT_KEK \
 			(1ULL << MLX5_GENERAL_OBJ_TYPE_IMPORT_KEK)
+#define MLX5_GENERAL_OBJ_TYPES_CAP_CREDENTIAL \
+			(1ULL << MLX5_GENERAL_OBJ_TYPE_CREDENTIAL)
 #define MLX5_GENERAL_OBJ_TYPES_CAP_CRYPTO_LOGIN \
 			(1ULL << MLX5_GENERAL_OBJ_TYPE_CRYPTO_LOGIN)
 
@@ -2490,6 +2492,7 @@ enum {
 	MLX5_GENERAL_OBJ_TYPE_VIRTQ = 0x000d,
 	MLX5_GENERAL_OBJ_TYPE_VIRTIO_Q_COUNTERS = 0x001c,
 	MLX5_GENERAL_OBJ_TYPE_IMPORT_KEK = 0x001d,
+	MLX5_GENERAL_OBJ_TYPE_CREDENTIAL = 0x001e,
 	MLX5_GENERAL_OBJ_TYPE_CRYPTO_LOGIN = 0x001f,
 	MLX5_GENERAL_OBJ_TYPE_FLEX_PARSE_GRAPH = 0x0022,
 	MLX5_GENERAL_OBJ_TYPE_FLOW_METER_ASO = 0x0024,
@@ -2603,6 +2606,26 @@ struct mlx5_ifc_create_import_kek_in_bits {
 	struct mlx5_ifc_import_kek_bits import_kek;
 };
 
+enum {
+	MLX5_CREDENTIAL_ROLE_OFFICER = 0x0,
+	MLX5_CREDENTIAL_ROLE_USER = 0x1,
+};
+
+struct mlx5_ifc_credential_bits {
+	u8 modify_field_select[0x40];
+	u8 state[0x8];
+	u8 reserved_at_48[0x10];
+	u8 credential_role[0x8];
+	u8 reserved_at_60[0x1a0];
+	u8 credential[0x180];
+	u8 reserved_at_380[0x480];
+};
+
+struct mlx5_ifc_create_credential_in_bits {
+	struct mlx5_ifc_general_obj_in_cmd_hdr_bits hdr;
+	struct mlx5_ifc_credential_bits credential;
+};
+
 struct mlx5_ifc_crypto_login_bits {
 	u8 modify_field_select[0x40];
 	u8 reserved_at_40[0x48];
diff --git a/drivers/common/mlx5/version.map b/drivers/common/mlx5/version.map
index 00df37e81a..1885cb8f6a 100644
--- a/drivers/common/mlx5/version.map
+++ b/drivers/common/mlx5/version.map
@@ -29,6 +29,7 @@ INTERNAL {
 	mlx5_devx_cmd_create_geneve_tlv_option;
         mlx5_devx_cmd_create_dek_obj;
         mlx5_devx_cmd_create_import_kek_obj;
+        mlx5_devx_cmd_create_credential_obj;
         mlx5_devx_cmd_create_crypto_login_obj;
 	mlx5_devx_cmd_destroy;
 	mlx5_devx_cmd_flow_counter_alloc;
-- 
2.25.1

[dpdk-dev] [PATCH v2 14/16] common/mlx5: add crypto register structs and defs

[Matan Azrad]

From: Dekel Peled <dekelp@nvidia.com>

Encryption key management requires use of several related registers.
This patch adds the relevant structs and values, according to PRM
definitions.

Signed-off-by: Dekel Peled <dekelp@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_prm.h | 41 ++++++++++++++++++++++++++++++++++
 1 file changed, 41 insertions(+)

diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index 432c8fdb63..c2cd2d9f70 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -3307,6 +3307,10 @@ enum {
 
 enum {
 	MLX5_REGISTER_ID_MTUTC  = 0x9055,
+	MLX5_CRYPTO_OPERATIONAL_REGISTER_ID = 0xC002,
+	MLX5_CRYPTO_COMMISSIONING_REGISTER_ID = 0xC003,
+	MLX5_IMPORT_KEK_HANDLE_REGISTER_ID = 0xC004,
+	MLX5_CREDENTIAL_HANDLE_REGISTER_ID = 0xC005,
 };
 
 struct mlx5_ifc_register_mtutc_bits {
@@ -3324,6 +3328,43 @@ struct mlx5_ifc_register_mtutc_bits {
 #define MLX5_MTUTC_TIMESTAMP_MODE_INTERNAL_TIMER 0
 #define MLX5_MTUTC_TIMESTAMP_MODE_REAL_TIME 1
 
+struct mlx5_ifc_crypto_operational_register_bits {
+	u8 wrapped_crypto_operational[0x1];
+	u8 reserved_at_1[0x1b];
+	u8 kek_size[0x4];
+	u8 reserved_at_20[0x20];
+	u8 credential[0x140];
+	u8 kek[0x100];
+	u8 reserved_at_280[0x180];
+};
+
+struct mlx5_ifc_crypto_commissioning_register_bits {
+	u8 token[0x1]; /* TODO: add size after PRM update */
+};
+
+struct mlx5_ifc_import_kek_handle_register_bits {
+	struct mlx5_ifc_crypto_login_bits crypto_login_object;
+	struct mlx5_ifc_import_kek_bits import_kek_object;
+	u8 reserved_at_200[0x4];
+	u8 write_operation[0x4];
+	u8 import_kek_id[0x18];
+	u8 reserved_at_220[0xe0];
+};
+
+struct mlx5_ifc_credential_handle_register_bits {
+	struct mlx5_ifc_crypto_login_bits crypto_login_object;
+	struct mlx5_ifc_credential_bits credential_object;
+	u8 reserved_at_200[0x4];
+	u8 write_operation[0x4];
+	u8 credential_id[0x18];
+	u8 reserved_at_220[0xe0];
+};
+
+enum {
+	MLX5_REGISTER_ADD_OPERATION = 0x1,
+	MLX5_REGISTER_DELETE_OPERATION = 0x2,
+};
+
 struct mlx5_ifc_parse_graph_arc_bits {
 	u8 start_inner_tunnel[0x1];
 	u8 reserved_at_1[0x7];
-- 
2.25.1

[dpdk-dev] [PATCH v2 15/16] common/mlx5: support register write access

[Matan Azrad]

From: Dekel Peled <dekelp@nvidia.com>

This patch adds support of write operation to NIC registers.

Signed-off-by: Dekel Peled <dekelp@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_devx_cmds.c | 67 +++++++++++++++++++++++++++-
 drivers/common/mlx5/mlx5_devx_cmds.h |  4 ++
 drivers/common/mlx5/version.map      | 21 ++++++---
 3 files changed, 83 insertions(+), 9 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c
index c0a0853c3a..0b421933ce 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.c
+++ b/drivers/common/mlx5/mlx5_devx_cmds.c
@@ -12,7 +12,6 @@
 #include "mlx5_common_log.h"
 #include "mlx5_malloc.h"
 
-
 /**
  * Perform read access to the registers. Reads data from register
  * and writes ones to the specified buffer.
@@ -61,7 +60,7 @@ mlx5_devx_cmd_register_read(void *ctx, uint16_t reg_id, uint32_t arg,
 	if (status) {
 		int syndrome = MLX5_GET(access_register_out, out, syndrome);
 
-		DRV_LOG(DEBUG, "Failed to access NIC register 0x%X, "
+		DRV_LOG(DEBUG, "Failed to read access NIC register 0x%X, "
 			       "status %x, syndrome = %x",
 			       reg_id, status, syndrome);
 		return -1;
@@ -74,6 +73,70 @@ mlx5_devx_cmd_register_read(void *ctx, uint16_t reg_id, uint32_t arg,
 	return rc;
 }
 
+/**
+ * Perform write access to the registers.
+ *
+ * @param[in] ctx
+ *   Context returned from mlx5 open_device() glue function.
+ * @param[in] reg_id
+ *   Register identifier according to the PRM.
+ * @param[in] arg
+ *   Register access auxiliary parameter according to the PRM.
+ * @param[out] data
+ *   Pointer to the buffer containing data to write.
+ * @param[in] dw_cnt
+ *   Buffer size in double words (32bit units).
+ *
+ * @return
+ *   0 on success, a negative value otherwise.
+ */
+int
+mlx5_devx_cmd_register_write(void *ctx, uint16_t reg_id, uint32_t arg,
+			     uint32_t *data, uint32_t dw_cnt)
+{
+	uint32_t in[MLX5_ST_SZ_DW(access_register_in) +
+		    MLX5_ACCESS_REGISTER_DATA_DWORD_MAX] = {0};
+	uint32_t out[MLX5_ST_SZ_DW(access_register_out)] = {0};
+	int status, rc;
+	void *ptr;
+
+	MLX5_ASSERT(data && dw_cnt);
+	MLX5_ASSERT(dw_cnt <= MLX5_ACCESS_REGISTER_DATA_DWORD_MAX);
+	if (dw_cnt > MLX5_ACCESS_REGISTER_DATA_DWORD_MAX) {
+		DRV_LOG(ERR, "Data to write exceeds max size");
+		return -1;
+	}
+	MLX5_SET(access_register_in, in, opcode,
+		 MLX5_CMD_OP_ACCESS_REGISTER_USER);
+	MLX5_SET(access_register_in, in, op_mod,
+		 MLX5_ACCESS_REGISTER_IN_OP_MOD_WRITE);
+	MLX5_SET(access_register_in, in, register_id, reg_id);
+	MLX5_SET(access_register_in, in, argument, arg);
+	ptr = MLX5_ADDR_OF(access_register_in, in, register_data);
+	memcpy(ptr, data, dw_cnt * sizeof(uint32_t));
+	rc = mlx5_glue->devx_general_cmd(ctx, in, sizeof(in), out, sizeof(out));
+
+	rc = mlx5_glue->devx_general_cmd(ctx, in,
+					 MLX5_ST_SZ_BYTES(access_register_in) +
+					 dw_cnt * sizeof(uint32_t),
+					 out, sizeof(out));
+	if (rc)
+		goto error;
+	status = MLX5_GET(access_register_out, out, status);
+	if (status) {
+		int syndrome = MLX5_GET(access_register_out, out, syndrome);
+
+		DRV_LOG(DEBUG, "Failed to write access NIC register 0x%X, "
+			       "status %x, syndrome = %x",
+			       reg_id, status, syndrome);
+		return -1;
+	}
+	return 0;
+error:
+	rc = (rc > 0) ? -rc : rc;
+	return rc;
+}
+
 /**
  * Allocate flow counters via devx interface.
  *
diff --git a/drivers/common/mlx5/mlx5_devx_cmds.h b/drivers/common/mlx5/mlx5_devx_cmds.h
index 811e7a1462..ce570ad28a 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.h
+++ b/drivers/common/mlx5/mlx5_devx_cmds.h
@@ -566,6 +566,10 @@ __rte_internal
 int mlx5_devx_cmd_register_read(void *ctx, uint16_t reg_id,
 				uint32_t arg, uint32_t *data, uint32_t dw_cnt);
 
+__rte_internal
+int mlx5_devx_cmd_register_write(void *ctx, uint16_t reg_id,
+				 uint32_t arg, uint32_t *data, uint32_t dw_cnt);
+
 __rte_internal
 struct mlx5_devx_obj *
 mlx5_devx_cmd_create_geneve_tlv_option(void *ctx,
diff --git a/drivers/common/mlx5/version.map b/drivers/common/mlx5/version.map
index 1885cb8f6a..1dc2d063ff 100644
--- a/drivers/common/mlx5/version.map
+++ b/drivers/common/mlx5/version.map
@@ -13,8 +13,17 @@ INTERNAL {
 	mlx5_dev_to_pci_addr; # WINDOWS_NO_EXPORT
 
 	mlx5_devx_cmd_alloc_pd;
+	mlx5_devx_alloc_uar;
+
 	mlx5_devx_cmd_create_cq;
+	mlx5_devx_cmd_create_credential_obj;
+	mlx5_devx_cmd_create_crypto_login_obj;
+	mlx5_devx_cmd_create_dek_obj;
 	mlx5_devx_cmd_create_flex_parser;
+	mlx5_devx_cmd_create_flow_hit_aso_obj;
+	mlx5_devx_cmd_create_flow_meter_aso_obj;
+	mlx5_devx_cmd_create_geneve_tlv_option;
+	mlx5_devx_cmd_create_import_kek_obj;
 	mlx5_devx_cmd_create_qp;
 	mlx5_devx_cmd_create_rq;
 	mlx5_devx_cmd_create_rqt;
@@ -24,13 +33,6 @@ INTERNAL {
 	mlx5_devx_cmd_create_tis;
 	mlx5_devx_cmd_create_virtio_q_counters; # WINDOWS_NO_EXPORT
 	mlx5_devx_cmd_create_virtq;
-	mlx5_devx_cmd_create_flow_hit_aso_obj;
-	mlx5_devx_cmd_create_flow_meter_aso_obj;
-	mlx5_devx_cmd_create_geneve_tlv_option;
-        mlx5_devx_cmd_create_dek_obj;
-        mlx5_devx_cmd_create_import_kek_obj;
-        mlx5_devx_cmd_create_credential_obj;
-        mlx5_devx_cmd_create_crypto_login_obj;
 	mlx5_devx_cmd_destroy;
 	mlx5_devx_cmd_flow_counter_alloc;
 	mlx5_devx_cmd_flow_counter_query;
@@ -51,12 +53,17 @@ INTERNAL {
 	mlx5_devx_cmd_queue_counter_alloc; # WINDOWS_NO_EXPORT
 	mlx5_devx_cmd_queue_counter_query; # WINDOWS_NO_EXPORT
 	mlx5_devx_cmd_register_read;
+	mlx5_devx_cmd_register_write;
 	mlx5_devx_cmd_wq_query; # WINDOWS_NO_EXPORT
 	mlx5_devx_get_out_command_status;
 	mlx5_devx_alloc_uar; # WINDOWS_NO_EXPORT
 
+
 	mlx5_devx_cq_create;
 	mlx5_devx_cq_destroy;
+
+	mlx5_devx_get_out_command_status;
+
 	mlx5_devx_rq_create;
 	mlx5_devx_rq_destroy;
 	mlx5_devx_sq_create;
-- 
2.25.1

[dpdk-dev] [PATCH v2 16/16] common/mlx5: add UMR and RDMA write WQE defines

[Matan Azrad]

From: Suanming Mou <suanmingm@nvidia.com>

This patch adds the struct defining UMR and RDMA write WQEs.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_prm.h | 187 +++++++++++++++++++++------------
 1 file changed, 121 insertions(+), 66 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index c2cd2d9f70..1ffee5fd56 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -412,6 +412,127 @@ struct mlx5_cqe_ts {
 	uint8_t op_own;
 };
 
+struct mlx5_wqe_rseg {
+	uint64_t raddr;
+	uint32_t rkey;
+	uint32_t reserved;
+} __rte_packed;
+
+#define MLX5_UMRC_IF_OFFSET 31u
+#define MLX5_UMRC_KO_OFFSET 16u
+#define MLX5_UMRC_TO_BS_OFFSET 0u
+
+struct mlx5_wqe_umr_cseg {
+	uint32_t if_cf_toe_cq_res;
+	uint32_t ko_to_bs;
+	uint64_t mkey_mask;
+	uint32_t rsvd1[8];
+} __rte_packed;
+
+struct mlx5_wqe_mkey_cseg {
+	uint32_t fr_res_af_sf;
+	uint32_t qpn_mkey;
+	uint32_t reserved2;
+	uint32_t flags_pd;
+	uint64_t start_addr;
+	uint64_t len;
+	uint32_t bsf_octword_size;
+	uint32_t reserved3[4];
+	uint32_t translations_octword_size;
+	uint32_t res4_lps;
+	uint32_t reserved;
+} __rte_packed;
+
+enum {
+	MLX5_BSF_SIZE_16B = 0x0,
+	MLX5_BSF_SIZE_32B = 0x1,
+	MLX5_BSF_SIZE_64B = 0x2,
+	MLX5_BSF_SIZE_128B = 0x3,
+};
+
+enum {
+	MLX5_BSF_P_TYPE_SIGNATURE = 0x0,
+	MLX5_BSF_P_TYPE_CRYPTO = 0x1,
+};
+
+enum {
+	MLX5_ENCRYPTION_ORDER_ENCRYPTED_WIRE_SIGNATURE = 0x0,
+	MLX5_ENCRYPTION_ORDER_ENCRYPTED_MEMORY_SIGNATURE = 0x1,
+	MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_WIRE = 0x2,
+	MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_MEMORY = 0x3,
+};
+
+enum {
+	MLX5_ENCRYPTION_STANDARD_AES_XTS = 0x0,
+};
+
+enum {
+	MLX5_BLOCK_SIZE_512B	= 0x1,
+	MLX5_BLOCK_SIZE_520B	= 0x2,
+	MLX5_BLOCK_SIZE_4096B	= 0x3,
+	MLX5_BLOCK_SIZE_4160B	= 0x4,
+	MLX5_BLOCK_SIZE_1MB	= 0x5,
+	MLX5_BLOCK_SIZE_4048B	= 0x6,
+};
+
+#define MLX5_BSF_SIZE_OFFSET		30
+#define MLX5_BSF_P_TYPE_OFFSET		24
+#define MLX5_ENCRYPTION_ORDER_OFFSET	16
+#define MLX5_BLOCK_SIZE_OFFSET		24
+
+struct mlx5_wqe_umr_bsf_seg {
+	/*
+	 * bs_bpt_eo_es contains:
+	 * bs	bsf_size		2 bits at MLX5_BSF_SIZE_OFFSET
+	 * bpt	bsf_p_type		2 bits at MLX5_BSF_P_TYPE_OFFSET
+	 * eo	encryption_order	4 bits at MLX5_ENCRYPTION_ORDER_OFFSET
+	 * es	encryption_standard	4 bits at offset 0
+	 */
+	uint32_t bs_bpt_eo_es;
+	uint32_t raw_data_size;
+	/*
+	 * bsp_res contains:
+	 * bsp	crypto_block_size_pointer	8 bits at MLX5_BLOCK_SIZE_OFFSET
+	 * res	reserved 24 bits
+	 */
+	uint32_t bsp_res;
+	uint32_t reserved0;
+	uint8_t xts_initial_tweak[16];
+	/*
+	 * res_dp contains:
+	 * res	reserved 8 bits
+	 * dp	dek_pointer		24 bits at offset 0
+	 */
+	uint32_t res_dp;
+	uint32_t reserved1;
+	uint64_t keytag;
+	uint32_t reserved2[4];
+} __rte_packed;
+
+#ifdef PEDANTIC
+#pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+
+struct mlx5_umr_wqe {
+	struct mlx5_wqe_cseg ctr;
+	struct mlx5_wqe_umr_cseg ucseg;
+	struct mlx5_wqe_mkey_cseg mkc;
+	union {
+		struct mlx5_wqe_dseg kseg[0];
+		struct mlx5_wqe_umr_bsf_seg bsf[0];
+	};
+} __rte_packed;
+
+struct mlx5_rdma_write_wqe {
+	struct mlx5_wqe_cseg ctr;
+	struct mlx5_wqe_rseg rseg;
+	struct mlx5_wqe_dseg dseg[0];
+} __rte_packed;
+
+#ifdef PEDANTIC
+#pragma GCC diagnostic error "-Wpedantic"
+#endif
+
 /* GGA */
 /* MMO metadata segment */
 
@@ -1096,72 +1217,6 @@ struct mlx5_ifc_create_mkey_in_bits {
 	u8 klm_pas_mtt[][0x20];
 };
 
-enum {
-	MLX5_BSF_SIZE_16B = 0x0,
-	MLX5_BSF_SIZE_32B = 0x1,
-	MLX5_BSF_SIZE_64B = 0x2,
-	MLX5_BSF_SIZE_128B = 0x3,
-};
-
-enum {
-	MLX5_BSF_P_TYPE_SIGNATURE = 0x0,
-	MLX5_BSF_P_TYPE_CRYPTO = 0x1,
-};
-
-enum {
-	MLX5_ENCRYPTION_ORDER_ENCRYPTED_WIRE_SIGNATURE = 0x0,
-	MLX5_ENCRYPTION_ORDER_ENCRYPTED_MEMORY_SIGNATURE = 0x1,
-	MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_WIRE = 0x2,
-	MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_MEMORY = 0x3,
-};
-
-enum {
-	MLX5_ENCRYPTION_STANDARD_AES_XTS = 0x0,
-};
-
-enum {
-	MLX5_BLOCK_SIZE_512B	= 0x1,
-	MLX5_BLOCK_SIZE_520B	= 0x2,
-	MLX5_BLOCK_SIZE_4096B	= 0x3,
-	MLX5_BLOCK_SIZE_4160B	= 0x4,
-	MLX5_BLOCK_SIZE_1MB	= 0x5,
-	MLX5_BLOCK_SIZE_4048B	= 0x6,
-};
-
-#define MLX5_BSF_SIZE_OFFSET		30
-#define MLX5_BSF_P_TYPE_OFFSET		24
-#define MLX5_ENCRYPTION_ORDER_OFFSET	16
-#define MLX5_BLOCK_SIZE_OFFSET		24
-
-struct mlx5_wqe_umr_bsf_seg {
-	/*
-	 * bs_bpt_eo_es contains:
-	 * bs	bsf_size		2 bits at MLX5_BSF_SIZE_OFFSET
-	 * bpt	bsf_p_type		2 bits at MLX5_BSF_P_TYPE_OFFSET
-	 * eo	encryption_order	4 bits at MLX5_ENCRYPTION_ORDER_OFFSET
-	 * es	encryption_standard	4 bits at offset 0
-	 */
-	uint32_t bs_bpt_eo_es;
-	uint32_t raw_data_size;
-	/*
-	 * bsp_res contains:
-	 * bsp	crypto_block_size_pointer	8 bits at MLX5_BLOCK_SIZE_OFFSET
-	 * res	reserved 24 bits
-	 */
-	uint32_t bsp_res;
-	uint32_t reserved0;
-	uint8_t xts_initial_tweak[16];
-	/*
-	 * res_dp contains:
-	 * res	reserved 8 bits
-	 * dp	dek_pointer		24 bits at offset 0
-	 */
-	uint32_t res_dp;
-	uint32_t reserved1;
-	uint64_t keytag;
-	uint32_t reserved2[4];
-} __rte_packed;
-
 enum {
 	MLX5_GET_HCA_CAP_OP_MOD_GENERAL_DEVICE = 0x0 << 1,
 	MLX5_GET_HCA_CAP_OP_MOD_ETHERNET_OFFLOAD_CAPS = 0x1 << 1,
-- 
2.25.1

[dpdk-dev] [PATCH v2 00/15] drivers: introduce mlx5 crypto PMD

[Matan Azrad]

Add a new PMD for Nvidia devices- crypto PMD.
The crypto PMD will be supported on Nvidia ConnectX6
The crypto PMD will add the support of encryption and decryption using the AES-XTS symmetric algorithm.
The crypto PMD requires rdma-core and uses mlx5 DevX.

v2:
Add data-path part.

Shiri Kuzin (10):
  drivers: introduce mlx5 crypto PMD
  crypto/mlx5: add DEK object management
  crypto/mlx5: support session operations
  crypto/mlx5: add basic operations
  crypto/mlx5: support queue pairs operations
  crypto/mlx5: add dev stop and start operations
  crypto/mlx5: add memory region management
  crypto/mlx5: create login object using DevX
  crypto/mlx5: adjust to the multiple data unit API
  crypto/mlx5: set feature flags and capabilities

Suanming Mou (5):
  crypto/mlx5: add keytag device argument
  crypto/mlx5: add maximum segments device argument
  crypto/mlx5: add WQE set initialization
  crypto/mlx5: add enqueue and dequeue operations
  crypto/mlx5: add statistic get and reset operations

 MAINTAINERS                             |    4 +
 doc/guides/cryptodevs/features/mlx5.ini |   37 +
 doc/guides/cryptodevs/index.rst         |    1 +
 doc/guides/cryptodevs/mlx5.rst          |  152 ++++
 doc/guides/rel_notes/release_21_05.rst  |    5 +
 drivers/common/mlx5/mlx5_common_pci.c   |   14 +
 drivers/common/mlx5/mlx5_common_pci.h   |   21 +-
 drivers/crypto/meson.build              |    1 +
 drivers/crypto/mlx5/meson.build         |   27 +
 drivers/crypto/mlx5/mlx5_crypto.c       | 1092 +++++++++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto.h       |   92 ++
 drivers/crypto/mlx5/mlx5_crypto_dek.c   |  136 +++
 drivers/crypto/mlx5/mlx5_crypto_utils.h |   19 +
 drivers/crypto/mlx5/version.map         |    3 +
 14 files changed, 1594 insertions(+), 10 deletions(-)
 create mode 100644 doc/guides/cryptodevs/features/mlx5.ini
 create mode 100644 doc/guides/cryptodevs/mlx5.rst
 create mode 100644 drivers/crypto/mlx5/meson.build
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto.c
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto.h
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_dek.c
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_utils.h
 create mode 100644 drivers/crypto/mlx5/version.map

-- 
2.25.1

[dpdk-dev] [PATCH v2 01/15] drivers: introduce mlx5 crypto PMD

[Matan Azrad]

From: Shiri Kuzin <shirik@nvidia.com>

Add a new PMD for Nvidia devices- crypto PMD.

The crypto PMD will be supported starting Nvidia ConnectX6 and
BlueField2.

The crypto PMD will add the support of encryption and decryption using
the AES-XTS symmetric algorithm.

The crypto PMD requires rdma-core and uses mlx5 DevX.

This patch adds the PCI probing, basic functions, build files and
log utility.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 MAINTAINERS                             |   4 +
 drivers/common/mlx5/mlx5_common_pci.c   |  14 ++
 drivers/common/mlx5/mlx5_common_pci.h   |  21 +-
 drivers/crypto/meson.build              |   1 +
 drivers/crypto/mlx5/meson.build         |  26 +++
 drivers/crypto/mlx5/mlx5_crypto.c       | 272 ++++++++++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto_utils.h |  19 ++
 drivers/crypto/mlx5/version.map         |   3 +
 8 files changed, 350 insertions(+), 10 deletions(-)
 create mode 100644 drivers/crypto/mlx5/meson.build
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto.c
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_utils.h
 create mode 100644 drivers/crypto/mlx5/version.map

diff --git a/MAINTAINERS b/MAINTAINERS
index 44f3d322ed..707e5a43b4 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -1081,6 +1081,10 @@ F: drivers/crypto/octeontx2/
 F: doc/guides/cryptodevs/octeontx2.rst
 F: doc/guides/cryptodevs/features/octeontx2.ini
 
+Mellanox mlx5
+M: Matan Azrad <matan@nvidia.com>
+F: drivers/crypto/mlx5/
+
 Null Crypto
 M: Declan Doherty <declan.doherty@intel.com>
 F: drivers/crypto/null/
diff --git a/drivers/common/mlx5/mlx5_common_pci.c b/drivers/common/mlx5/mlx5_common_pci.c
index 3f16cd21cf..8a47afee20 100644
--- a/drivers/common/mlx5/mlx5_common_pci.c
+++ b/drivers/common/mlx5/mlx5_common_pci.c
@@ -31,6 +31,7 @@ static const struct {
 	{ .name = "net", .driver_class = MLX5_CLASS_NET },
 	{ .name = "regex", .driver_class = MLX5_CLASS_REGEX },
 	{ .name = "compress", .driver_class = MLX5_CLASS_COMPRESS },
+	{ .name = "crypto", .driver_class = MLX5_CLASS_CRYPTO },
 };
 
 static const unsigned int mlx5_class_combinations[] = {
@@ -38,13 +39,26 @@ static const unsigned int mlx5_class_combinations[] = {
 	MLX5_CLASS_VDPA,
 	MLX5_CLASS_REGEX,
 	MLX5_CLASS_COMPRESS,
+	MLX5_CLASS_CRYPTO,
 	MLX5_CLASS_NET | MLX5_CLASS_REGEX,
 	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX,
 	MLX5_CLASS_NET | MLX5_CLASS_COMPRESS,
 	MLX5_CLASS_VDPA | MLX5_CLASS_COMPRESS,
 	MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
+	MLX5_CLASS_NET | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_VDPA | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO,
 	MLX5_CLASS_NET | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
 	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
+	MLX5_CLASS_NET | MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_NET | MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_VDPA | MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_NET | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS |
+	MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS |
+	MLX5_CLASS_CRYPTO,
 	/* New class combination should be added here. */
 };
 
diff --git a/drivers/common/mlx5/mlx5_common_pci.h b/drivers/common/mlx5/mlx5_common_pci.h
index de89bb98bc..cb8d2f5f87 100644
--- a/drivers/common/mlx5/mlx5_common_pci.h
+++ b/drivers/common/mlx5/mlx5_common_pci.h
@@ -9,17 +9,18 @@
  * @file
  *
  * RTE Mellanox PCI Driver Interface
- * Mellanox ConnectX PCI device supports multiple class: net,vdpa,regex and
- * compress devices. This layer enables creating such multiple class of devices
- * on a single PCI device by allowing to bind multiple class specific device
- * driver to attach to mlx5_pci driver.
+ * Mellanox ConnectX PCI device supports multiple class: net,vdpa,regex,compress
+ * and crypto devices. This layer enables creating such multiple class of
+ * devices on a single PCI device by allowing to bind multiple class specific
+ * device driver to attach to mlx5_pci driver.
  *
- * -----------    ------------    -------------    ----------------
- * |   mlx5  |    |   mlx5   |    |   mlx5    |    |     mlx5     |
- * | net pmd |    | vdpa pmd |    | regex pmd |    | compress pmd |
- * -----------    ------------    -------------    ----------------
- *      \              \                    /              /
- *       \              \                  /              /
+ * --------    --------    ---------    ------------    ----------
+ * | mlx5 |    | mlx5 |    | mlx5  |    |   mlx5   |    |  mlx5  |
+ * | net  |    | vdpa |    | regex |    | compress |    | crypto |
+ * | pmd  |    | pmd  |    |  pmd  |    |   pmd    |    |  pmd   |
+ * --------    --------    ---------    ------------    ----------
+ *      \              \         |          /              /
+ *       \              \        |         /              /
  *        \              \_--------------_/              /
  *         \_______________|   mlx5     |_______________/
  *                         | pci common |
diff --git a/drivers/crypto/meson.build b/drivers/crypto/meson.build
index b9fdf9392f..7d9bd64cd1 100644
--- a/drivers/crypto/meson.build
+++ b/drivers/crypto/meson.build
@@ -16,6 +16,7 @@ drivers = [
         'dpaa2_sec',
         'kasumi',
         'mvsam',
+        'mlx5',
         'nitrox',
         'null',
         'octeontx',
diff --git a/drivers/crypto/mlx5/meson.build b/drivers/crypto/mlx5/meson.build
new file mode 100644
index 0000000000..5bf0912766
--- /dev/null
+++ b/drivers/crypto/mlx5/meson.build
@@ -0,0 +1,26 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2021 Mellanox Technologies, Ltd
+
+if not is_linux
+	build = false
+	reason = 'only supported on Linux'
+	subdir_done()
+endif
+
+fmt_name = 'mlx5_crypto'
+deps += ['common_mlx5', 'eal', 'cryptodev']
+sources = files(
+	'mlx5_crypto.c',
+)
+cflags_options = [
+	'-std=c11',
+	'-Wno-strict-prototypes',
+	'-D_BSD_SOURCE',
+	'-D_DEFAULT_SOURCE',
+	'-D_XOPEN_SOURCE=600'
+]
+foreach option:cflags_options
+	if cc.has_argument(option)
+		cflags += option
+	endif
+endforeach
\ No newline at end of file
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
new file mode 100644
index 0000000000..8a3a307f7b
--- /dev/null
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -0,0 +1,272 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#include <rte_malloc.h>
+#include <rte_log.h>
+#include <rte_errno.h>
+#include <rte_pci.h>
+#include <rte_crypto.h>
+#include <rte_cryptodev.h>
+#include <rte_cryptodev_pmd.h>
+
+#include <mlx5_glue.h>
+#include <mlx5_common.h>
+#include <mlx5_common_pci.h>
+#include <mlx5_devx_cmds.h>
+#include <mlx5_common_os.h>
+
+#include "mlx5_crypto_utils.h"
+
+#define MLX5_CRYPTO_DRIVER_NAME mlx5_crypto
+#define MLX5_CRYPTO_LOG_NAME    pmd.crypto.mlx5
+
+struct mlx5_crypto_priv {
+	TAILQ_ENTRY(mlx5_crypto_priv) next;
+	struct ibv_context *ctx; /* Device context. */
+	struct rte_pci_device *pci_dev;
+	struct rte_cryptodev *crypto_dev;
+	void *uar; /* User Access Region. */
+	uint32_t pdn; /* Protection Domain number. */
+	struct ibv_pd *pd;
+};
+
+TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
+				TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
+static pthread_mutex_t priv_list_lock = PTHREAD_MUTEX_INITIALIZER;
+
+int mlx5_crypto_logtype;
+
+uint8_t mlx5_crypto_driver_id;
+
+static const char mlx5_crypto_drv_name[] = RTE_STR(MLX5_CRYPTO_DRIVER_NAME);
+
+static const struct rte_driver mlx5_drv = {
+	.name = mlx5_crypto_drv_name,
+	.alias = mlx5_crypto_drv_name
+};
+
+static struct cryptodev_driver mlx5_cryptodev_driver;
+
+static struct rte_cryptodev_ops mlx5_crypto_ops = {
+	.dev_configure			= NULL,
+	.dev_start			= NULL,
+	.dev_stop			= NULL,
+	.dev_close			= NULL,
+	.dev_infos_get			= NULL,
+	.stats_get			= NULL,
+	.stats_reset			= NULL,
+	.queue_pair_setup		= NULL,
+	.queue_pair_release		= NULL,
+	.sym_session_get_size		= NULL,
+	.sym_session_configure		= NULL,
+	.sym_session_clear		= NULL,
+	.sym_get_raw_dp_ctx_size	= NULL,
+	.sym_configure_raw_dp_ctx	= NULL,
+};
+
+static void
+mlx5_crypto_hw_global_release(struct mlx5_crypto_priv *priv)
+{
+	if (priv->pd != NULL) {
+		claim_zero(mlx5_glue->dealloc_pd(priv->pd));
+		priv->pd = NULL;
+	}
+	if (priv->uar != NULL) {
+		mlx5_glue->devx_free_uar(priv->uar);
+		priv->uar = NULL;
+	}
+}
+
+static int
+mlx5_crypto_pd_create(struct mlx5_crypto_priv *priv)
+{
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+	struct mlx5dv_obj obj;
+	struct mlx5dv_pd pd_info;
+	int ret;
+
+	priv->pd = mlx5_glue->alloc_pd(priv->ctx);
+	if (priv->pd == NULL) {
+		DRV_LOG(ERR, "Failed to allocate PD.");
+		return errno ? -errno : -ENOMEM;
+	}
+	obj.pd.in = priv->pd;
+	obj.pd.out = &pd_info;
+	ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_PD);
+	if (ret != 0) {
+		DRV_LOG(ERR, "Fail to get PD object info.");
+		mlx5_glue->dealloc_pd(priv->pd);
+		priv->pd = NULL;
+		return -errno;
+	}
+	priv->pdn = pd_info.pdn;
+	return 0;
+#else
+	(void)priv;
+	DRV_LOG(ERR, "Cannot get pdn - no DV support.");
+	return -ENOTSUP;
+#endif /* HAVE_IBV_FLOW_DV_SUPPORT */
+}
+
+static int
+mlx5_crypto_hw_global_prepare(struct mlx5_crypto_priv *priv)
+{
+	if (mlx5_crypto_pd_create(priv) != 0)
+		return -1;
+	priv->uar = mlx5_devx_alloc_uar(priv->ctx, -1);
+	if (priv->uar == NULL || mlx5_os_get_devx_uar_reg_addr(priv->uar) ==
+	    NULL) {
+		rte_errno = errno;
+		claim_zero(mlx5_glue->dealloc_pd(priv->pd));
+		DRV_LOG(ERR, "Failed to allocate UAR.");
+		return -1;
+	}
+	return 0;
+}
+
+/**
+ * DPDK callback to register a PCI device.
+ *
+ * This function spawns crypto device out of a given PCI device.
+ *
+ * @param[in] pci_drv
+ *   PCI driver structure (mlx5_crypto_driver).
+ * @param[in] pci_dev
+ *   PCI device information.
+ *
+ * @return
+ *   0 on success, 1 to skip this driver, a negative errno value otherwise
+ *   and rte_errno is set.
+ */
+static int
+mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
+			struct rte_pci_device *pci_dev)
+{
+	struct ibv_device *ibv;
+	struct rte_cryptodev *crypto_dev;
+	struct ibv_context *ctx;
+	struct mlx5_crypto_priv *priv;
+	struct mlx5_hca_attr attr = { 0 };
+	struct rte_cryptodev_pmd_init_params init_params = {
+		.name = "",
+		.private_data_size = sizeof(struct mlx5_crypto_priv),
+		.socket_id = pci_dev->device.numa_node,
+		.max_nb_queue_pairs =
+				RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
+	};
+	RTE_SET_USED(pci_drv);
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+		DRV_LOG(ERR, "Non-primary process type is not supported.");
+		rte_errno = ENOTSUP;
+		return -rte_errno;
+	}
+	ibv = mlx5_os_get_ib_device_match(&pci_dev->addr);
+	if (ibv == NULL) {
+		DRV_LOG(ERR, "No matching IB device for PCI slot "
+			PCI_PRI_FMT ".", pci_dev->addr.domain,
+			pci_dev->addr.bus, pci_dev->addr.devid,
+			pci_dev->addr.function);
+		return -rte_errno;
+	}
+	DRV_LOG(INFO, "PCI information matches for device \"%s\".", ibv->name);
+	ctx = mlx5_glue->dv_open_device(ibv);
+	if (ctx == NULL) {
+		DRV_LOG(ERR, "Failed to open IB device \"%s\".", ibv->name);
+		rte_errno = ENODEV;
+		return -rte_errno;
+	}
+	if (mlx5_devx_cmd_query_hca_attr(ctx, &attr) != 0 ||
+	    attr.crypto == 0 || attr.aes_xts == 0) {
+		DRV_LOG(ERR, "Not enough capabilities to support crypto "
+			"operations, maybe old FW/OFED version?");
+		claim_zero(mlx5_glue->close_device(ctx));
+		rte_errno = ENOTSUP;
+		return -ENOTSUP;
+	}
+	crypto_dev = rte_cryptodev_pmd_create(ibv->name, &pci_dev->device,
+					&init_params);
+	if (crypto_dev == NULL) {
+		DRV_LOG(ERR, "Failed to create device \"%s\".", ibv->name);
+		claim_zero(mlx5_glue->close_device(ctx));
+		return -ENODEV;
+	}
+	DRV_LOG(INFO,
+		"Crypto device %s was created successfully.", ibv->name);
+	crypto_dev->dev_ops = &mlx5_crypto_ops;
+	crypto_dev->dequeue_burst = NULL;
+	crypto_dev->enqueue_burst = NULL;
+	crypto_dev->feature_flags = RTE_CRYPTODEV_FF_HW_ACCELERATED;
+	crypto_dev->driver_id = mlx5_crypto_driver_id;
+	priv = crypto_dev->data->dev_private;
+	priv->ctx = ctx;
+	priv->pci_dev = pci_dev;
+	priv->crypto_dev = crypto_dev;
+	if (mlx5_crypto_hw_global_prepare(priv) != 0) {
+		rte_cryptodev_pmd_destroy(priv->crypto_dev);
+		claim_zero(mlx5_glue->close_device(priv->ctx));
+		return -1;
+	}
+	pthread_mutex_lock(&priv_list_lock);
+	TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next);
+	pthread_mutex_unlock(&priv_list_lock);
+	return 0;
+}
+
+static int
+mlx5_crypto_pci_remove(struct rte_pci_device *pdev)
+{
+	struct mlx5_crypto_priv *priv = NULL;
+
+	pthread_mutex_lock(&priv_list_lock);
+	TAILQ_FOREACH(priv, &mlx5_crypto_priv_list, next)
+		if (rte_pci_addr_cmp(&priv->pci_dev->addr, &pdev->addr) != 0)
+			break;
+	if (priv)
+		TAILQ_REMOVE(&mlx5_crypto_priv_list, priv, next);
+	pthread_mutex_unlock(&priv_list_lock);
+	if (priv) {
+		mlx5_crypto_hw_global_release(priv);
+		rte_cryptodev_pmd_destroy(priv->crypto_dev);
+		claim_zero(mlx5_glue->close_device(priv->ctx));
+	}
+	return 0;
+}
+
+static const struct rte_pci_id mlx5_crypto_pci_id_map[] = {
+		{
+			RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
+					PCI_DEVICE_ID_MELLANOX_CONNECTX6)
+		},
+		{
+			.vendor_id = 0
+		}
+	};
+
+static struct mlx5_pci_driver mlx5_crypto_driver = {
+	.driver_class = MLX5_CLASS_CRYPTO,
+	.pci_driver = {
+		.driver = {
+			.name = RTE_STR(MLX5_CRYPTO_DRIVER_NAME),
+		},
+		.id_table = mlx5_crypto_pci_id_map,
+		.probe = mlx5_crypto_pci_probe,
+		.remove = mlx5_crypto_pci_remove,
+		.drv_flags = 0,
+	},
+};
+
+RTE_INIT(rte_mlx5_crypto_init)
+{
+	mlx5_common_init();
+	if (mlx5_glue != NULL)
+		mlx5_pci_driver_register(&mlx5_crypto_driver);
+}
+
+RTE_PMD_REGISTER_CRYPTO_DRIVER(mlx5_cryptodev_driver, mlx5_drv,
+			       mlx5_crypto_driver_id);
+
+RTE_LOG_REGISTER(mlx5_crypto_logtype, MLX5_CRYPTO_LOG_NAME, NOTICE)
+RTE_PMD_EXPORT_NAME(MLX5_CRYPTO_DRIVER_NAME, __COUNTER__);
+RTE_PMD_REGISTER_PCI_TABLE(MLX5_CRYPTO_DRIVER_NAME, mlx5_crypto_pci_id_map);
+RTE_PMD_REGISTER_KMOD_DEP(MLX5_CRYPTO_DRIVER_NAME, "* ib_uverbs & mlx5_core & mlx5_ib");
diff --git a/drivers/crypto/mlx5/mlx5_crypto_utils.h b/drivers/crypto/mlx5/mlx5_crypto_utils.h
new file mode 100644
index 0000000000..cef4b07a36
--- /dev/null
+++ b/drivers/crypto/mlx5/mlx5_crypto_utils.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#ifndef RTE_PMD_MLX5_CRYPTO_UTILS_H_
+#define RTE_PMD_MLX5_CRYPTO_UTILS_H_
+
+#include <mlx5_common.h>
+
+extern int mlx5_crypto_logtype;
+
+#define MLX5_CRYPTO_LOG_PREFIX "mlx5_crypto"
+/* Generic printf()-like logging macro with automatic line feed. */
+#define DRV_LOG(level, ...) \
+	PMD_DRV_LOG_(level, mlx5_crypto_logtype, MLX5_CRYPTO_LOG_PREFIX, \
+		__VA_ARGS__ PMD_DRV_LOG_STRIP PMD_DRV_LOG_OPAREN, \
+		PMD_DRV_LOG_CPAREN)
+
+#endif /* RTE_PMD_MLX5_CRYPTO_UTILS_H_ */
diff --git a/drivers/crypto/mlx5/version.map b/drivers/crypto/mlx5/version.map
new file mode 100644
index 0000000000..4a76d1d52d
--- /dev/null
+++ b/drivers/crypto/mlx5/version.map
@@ -0,0 +1,3 @@
+DPDK_21 {
+	local: *;
+};
-- 
2.25.1

[dpdk-dev] [PATCH v2 02/15] crypto/mlx5: add DEK object management

[Matan Azrad]

From: Shiri Kuzin <shirik@nvidia.com>

A DEK(Data encryption Key) is an mlx5 HW object which represents the
cipher algorithm key.
The DEKs are used during data encryption/decryption operations.

In symmetric algorithms like AES-STS, we use the same DEK for both
encryption and decryption.

Use the mlx5 hash-list tool to manage the DEK objects in the PMD.

Provide the compare, create and destroy functions to manage DEKs in
hash-list and introduce an internal API to setup and unset the DEK
management and to prepare and destroy specific DEK object.

The DEK hash-list will be created in dev_configure routine and
destroyed in dev_close routine.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/meson.build       |   1 +
 drivers/crypto/mlx5/mlx5_crypto.c     |  44 +++++----
 drivers/crypto/mlx5/mlx5_crypto.h     |  51 ++++++++++
 drivers/crypto/mlx5/mlx5_crypto_dek.c | 136 ++++++++++++++++++++++++++
 4 files changed, 215 insertions(+), 17 deletions(-)
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto.h
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_dek.c

diff --git a/drivers/crypto/mlx5/meson.build b/drivers/crypto/mlx5/meson.build
index 5bf0912766..0666c35094 100644
--- a/drivers/crypto/mlx5/meson.build
+++ b/drivers/crypto/mlx5/meson.build
@@ -11,6 +11,7 @@ fmt_name = 'mlx5_crypto'
 deps += ['common_mlx5', 'eal', 'cryptodev']
 sources = files(
 	'mlx5_crypto.c',
+	'mlx5_crypto_dek.c',
 )
 cflags_options = [
 	'-std=c11',
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 8a3a307f7b..e7c70c521f 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -3,12 +3,9 @@
  */
 
 #include <rte_malloc.h>
-#include <rte_log.h>
 #include <rte_errno.h>
+#include <rte_log.h>
 #include <rte_pci.h>
-#include <rte_crypto.h>
-#include <rte_cryptodev.h>
-#include <rte_cryptodev_pmd.h>
 
 #include <mlx5_glue.h>
 #include <mlx5_common.h>
@@ -17,19 +14,10 @@
 #include <mlx5_common_os.h>
 
 #include "mlx5_crypto_utils.h"
+#include "mlx5_crypto.h"
 
 #define MLX5_CRYPTO_DRIVER_NAME mlx5_crypto
-#define MLX5_CRYPTO_LOG_NAME    pmd.crypto.mlx5
-
-struct mlx5_crypto_priv {
-	TAILQ_ENTRY(mlx5_crypto_priv) next;
-	struct ibv_context *ctx; /* Device context. */
-	struct rte_pci_device *pci_dev;
-	struct rte_cryptodev *crypto_dev;
-	void *uar; /* User Access Region. */
-	uint32_t pdn; /* Protection Domain number. */
-	struct ibv_pd *pd;
-};
+#define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
 
 TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
 				TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
@@ -48,11 +36,33 @@ static const struct rte_driver mlx5_drv = {
 
 static struct cryptodev_driver mlx5_cryptodev_driver;
 
+static int
+mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
+		struct rte_cryptodev_config *config __rte_unused)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+
+	if (mlx5_crypto_dek_setup(priv) != 0) {
+		DRV_LOG(ERR, "Dek hash list creation has failed.");
+		return -ENOMEM;
+	}
+	return 0;
+}
+
+static int
+mlx5_crypto_dev_close(struct rte_cryptodev *dev)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+
+	mlx5_crypto_dek_unset(priv);
+	return 0;
+}
+
 static struct rte_cryptodev_ops mlx5_crypto_ops = {
-	.dev_configure			= NULL,
+	.dev_configure			= mlx5_crypto_dev_configure,
 	.dev_start			= NULL,
 	.dev_stop			= NULL,
-	.dev_close			= NULL,
+	.dev_close			= mlx5_crypto_dev_close,
 	.dev_infos_get			= NULL,
 	.stats_get			= NULL,
 	.stats_reset			= NULL,
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
new file mode 100644
index 0000000000..4ec67a7e0f
--- /dev/null
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -0,0 +1,51 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#ifndef MLX5_CRYPTO_H_
+#define MLX5_CRYPTO_H_
+
+#include <stdbool.h>
+
+#include <rte_cryptodev.h>
+#include <rte_cryptodev_pmd.h>
+
+#include <mlx5_common_utils.h>
+
+#define MLX5_CRYPTO_DEK_HTABLE_SZ (1 << 11)
+#define MLX5_CRYPTO_KEY_LENGTH 80
+
+struct mlx5_crypto_priv {
+	TAILQ_ENTRY(mlx5_crypto_priv) next;
+	struct ibv_context *ctx; /* Device context. */
+	struct rte_pci_device *pci_dev;
+	struct rte_cryptodev *crypto_dev;
+	void *uar; /* User Access Region. */
+	uint32_t pdn; /* Protection Domain number. */
+	struct ibv_pd *pd;
+	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
+};
+
+struct mlx5_crypto_dek {
+	struct mlx5_hlist_entry entry; /* Pointer to DEK hash list entry. */
+	struct mlx5_devx_obj *obj; /* Pointer to DEK DevX object. */
+	uint8_t data[MLX5_CRYPTO_KEY_LENGTH]; /* DEK key data. */
+	bool size_is_48; /* Whether the key\data size is 48 bytes or not. */
+};
+
+int
+mlx5_crypto_dek_destroy(struct mlx5_crypto_priv *priv,
+			struct mlx5_crypto_dek *dek);
+
+struct mlx5_crypto_dek *
+mlx5_crypto_dek_prepare(struct mlx5_crypto_priv *priv,
+			struct rte_crypto_cipher_xform *cipher);
+
+int
+mlx5_crypto_dek_setup(struct mlx5_crypto_priv *priv);
+
+void
+mlx5_crypto_dek_unset(struct mlx5_crypto_priv *priv);
+
+#endif /* MLX5_CRYPTO_H_ */
+
diff --git a/drivers/crypto/mlx5/mlx5_crypto_dek.c b/drivers/crypto/mlx5/mlx5_crypto_dek.c
new file mode 100644
index 0000000000..c76e208845
--- /dev/null
+++ b/drivers/crypto/mlx5/mlx5_crypto_dek.c
@@ -0,0 +1,136 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2018 Mellanox Technologies, Ltd
+ */
+
+#include <rte_ip.h>
+#include <rte_common.h>
+#include <rte_errno.h>
+#include <rte_log.h>
+
+#include <mlx5_prm.h>
+#include <mlx5_devx_cmds.h>
+
+#include "mlx5_crypto_utils.h"
+#include "mlx5_crypto.h"
+
+struct mlx5_crypto_dek_ctx {
+	struct rte_crypto_cipher_xform *cipher;
+	struct mlx5_crypto_priv *priv;
+};
+
+int
+mlx5_crypto_dek_destroy(struct mlx5_crypto_priv *priv,
+			struct mlx5_crypto_dek *dek)
+{
+	return mlx5_hlist_unregister(priv->dek_hlist, &dek->entry);
+}
+
+struct mlx5_crypto_dek *
+mlx5_crypto_dek_prepare(struct mlx5_crypto_priv *priv,
+			struct rte_crypto_cipher_xform *cipher)
+{
+	struct mlx5_hlist *dek_hlist = priv->dek_hlist;
+	struct mlx5_crypto_dek_ctx dek_ctx = {
+		.cipher = cipher,
+		.priv = priv,
+	};
+	struct rte_crypto_cipher_xform *cipher_ctx = cipher;
+	uint64_t key64 = __rte_raw_cksum(cipher_ctx->key.data,
+					 cipher_ctx->key.length, 0);
+	struct mlx5_hlist_entry *entry = mlx5_hlist_register(dek_hlist,
+							     key64, &dek_ctx);
+
+	return entry == NULL ? NULL :
+			     container_of(entry, struct mlx5_crypto_dek, entry);
+}
+
+static int
+mlx5_crypto_dek_match_cb(struct mlx5_hlist *list __rte_unused,
+			 struct mlx5_hlist_entry *entry,
+			 uint64_t key __rte_unused, void *cb_ctx)
+{
+	struct mlx5_crypto_dek_ctx *ctx = cb_ctx;
+	struct rte_crypto_cipher_xform *cipher_ctx = ctx->cipher;
+	struct mlx5_crypto_dek *dek =
+			container_of(entry, typeof(*dek), entry);
+	uint32_t key_len = dek->size_is_48 ? 48 : 80;
+
+	if (key_len != cipher_ctx->key.length)
+		return -1;
+	return memcmp(cipher_ctx->key.data, dek->data, key_len);
+}
+
+static struct mlx5_hlist_entry *
+mlx5_crypto_dek_create_cb(struct mlx5_hlist *list __rte_unused,
+			  uint64_t key __rte_unused, void *cb_ctx)
+{
+	struct mlx5_crypto_dek_ctx *ctx = cb_ctx;
+	struct rte_crypto_cipher_xform *cipher_ctx = ctx->cipher;
+	struct mlx5_crypto_dek *dek = rte_zmalloc(__func__, sizeof(*dek),
+						  RTE_CACHE_LINE_SIZE);
+	struct mlx5_devx_dek_attr dek_attr = {
+		.pd = ctx->priv->pdn,
+		.key_purpose = MLX5_CRYPTO_KEY_PURPOSE_AES_XTS,
+		.has_keytag = 1,
+	};
+
+	if (dek == NULL) {
+		DRV_LOG(ERR, "Failed to allocate dek memory.");
+		return NULL;
+	}
+	switch (cipher_ctx->key.length) {
+	case 48:
+		dek->size_is_48 = true;
+		dek_attr.key_size = MLX5_CRYPTO_KEY_SIZE_128b;
+		break;
+	case 80:
+		dek->size_is_48 = false;
+		dek_attr.key_size = MLX5_CRYPTO_KEY_SIZE_256b;
+		break;
+	default:
+		DRV_LOG(ERR, "Key size not supported.");
+		return NULL;
+	}
+	rte_memcpy(&dek_attr.key, cipher_ctx->key.data, cipher_ctx->key.length);
+	dek->obj = mlx5_devx_cmd_create_dek_obj(ctx->priv->ctx, &dek_attr);
+	if (dek->obj == NULL) {
+		rte_free(dek);
+		return NULL;
+	}
+	rte_memcpy(&dek->data, cipher_ctx->key.data, cipher_ctx->key.length);
+	return &dek->entry;
+}
+
+static void
+mlx5_crypto_dek_remove_cb(struct mlx5_hlist *list __rte_unused,
+			  struct mlx5_hlist_entry *entry)
+{
+	struct mlx5_crypto_dek *dek =
+		container_of(entry, typeof(*dek), entry);
+
+	claim_zero(mlx5_devx_cmd_destroy(dek->obj));
+	rte_free(dek);
+}
+
+
+int
+mlx5_crypto_dek_setup(struct mlx5_crypto_priv *priv)
+{
+	priv->dek_hlist = mlx5_hlist_create("dek_hlist",
+				 MLX5_CRYPTO_DEK_HTABLE_SZ,
+				 0, MLX5_HLIST_WRITE_MOST |
+				 MLX5_HLIST_DIRECT_KEY,
+				 mlx5_crypto_dek_create_cb,
+				 mlx5_crypto_dek_match_cb,
+				 mlx5_crypto_dek_remove_cb);
+	if (priv->dek_hlist == NULL)
+		return -1;
+	return 0;
+}
+
+void
+mlx5_crypto_dek_unset(struct mlx5_crypto_priv *priv)
+{
+	mlx5_hlist_destroy(priv->dek_hlist);
+	priv->dek_hlist = NULL;
+}
-- 
2.25.1

[dpdk-dev] [PATCH v2 03/15] crypto/mlx5: support session operations

[Matan Azrad]

From: Shiri Kuzin <shirik@nvidia.com>

Sessions are used in symmetric transformations in order to prepare
objects and data for packet processing stage.

A mlx5 session includes iv_offset, pointer to mlx5_crypto_dek struct,
bsf_size, bsf_p_type, encryption_order and encryption standard.

Implement the next session operations:
        mlx5_crypto_sym_session_get_size- returns the size of the mlx5
	session struct.
	mlx5_crypto_sym_session_configure- prepares the DEK hash-list
	and saves all the session data.
	mlx5_crypto_sym_session_clear - destroys the DEK hash-list.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 96 ++++++++++++++++++++++++++++++-
 1 file changed, 93 insertions(+), 3 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index e7c70c521f..c494ed00d1 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -3,6 +3,7 @@
  */
 
 #include <rte_malloc.h>
+#include <rte_mempool.h>
 #include <rte_errno.h>
 #include <rte_log.h>
 #include <rte_pci.h>
@@ -36,6 +37,24 @@ static const struct rte_driver mlx5_drv = {
 
 static struct cryptodev_driver mlx5_cryptodev_driver;
 
+struct mlx5_crypto_session {
+	uint32_t bs_bpt_eo_es;
+	/*
+	 * bsf_size, bsf_p_type, encryption_order and encryption standard,
+	 * saved in big endian format.
+	 */
+	uint32_t iv_offset:16;
+	/* Starting point for Initialisation Vector. */
+	struct mlx5_crypto_dek *dek; /* Pointer to dek struct. */
+	uint32_t dek_id; /* DEK ID */
+} __rte_packed;
+
+static unsigned int
+mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
+{
+	return sizeof(struct mlx5_crypto_session);
+}
+
 static int
 mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
 		struct rte_cryptodev_config *config __rte_unused)
@@ -58,6 +77,77 @@ mlx5_crypto_dev_close(struct rte_cryptodev *dev)
 	return 0;
 }
 
+static int
+mlx5_crypto_sym_session_configure(struct rte_cryptodev *dev,
+				  struct rte_crypto_sym_xform *xform,
+				  struct rte_cryptodev_sym_session *session,
+				  struct rte_mempool *mp)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+	struct mlx5_crypto_session *sess_private_data;
+	struct rte_crypto_cipher_xform *cipher;
+	uint8_t encryption_order;
+	int ret;
+
+	if (unlikely(xform->next != NULL)) {
+		DRV_LOG(ERR, "Xform next is not supported.");
+		return -ENOTSUP;
+	}
+	if (unlikely((xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) ||
+		     (xform->cipher.algo != RTE_CRYPTO_CIPHER_AES_XTS))) {
+		DRV_LOG(ERR, "Only AES-XTS algorithm is supported.");
+		return -ENOTSUP;
+	}
+	ret = rte_mempool_get(mp, (void *)&sess_private_data);
+	if (ret != 0) {
+		DRV_LOG(ERR,
+			"Failed to get session %p private data from mempool.",
+			sess_private_data);
+		return -ENOMEM;
+	}
+	cipher = &xform->cipher;
+	sess_private_data->dek = mlx5_crypto_dek_prepare(priv, cipher);
+	if (sess_private_data->dek == NULL) {
+		rte_mempool_put(mp, sess_private_data);
+		DRV_LOG(ERR, "Failed to prepare dek.");
+		return -ENOMEM;
+	}
+	if (cipher->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
+		encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_MEMORY;
+	else
+		encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_WIRE;
+	sess_private_data->bs_bpt_eo_es = rte_cpu_to_be_32
+			(MLX5_BSF_SIZE_64B << MLX5_BSF_SIZE_OFFSET |
+			 MLX5_BSF_P_TYPE_CRYPTO << MLX5_BSF_P_TYPE_OFFSET |
+			 encryption_order << MLX5_ENCRYPTION_ORDER_OFFSET |
+			 MLX5_ENCRYPTION_STANDARD_AES_XTS);
+	sess_private_data->iv_offset = cipher->iv.offset;
+	sess_private_data->dek_id =
+			rte_cpu_to_be_32(sess_private_data->dek->obj->id &
+					 0xffffff);
+	set_sym_session_private_data(session, dev->driver_id,
+				     sess_private_data);
+	DRV_LOG(DEBUG, "Session %p was configured.", sess_private_data);
+	return 0;
+}
+
+static void
+mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev,
+			      struct rte_cryptodev_sym_session *sess)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+	struct mlx5_crypto_session *sess_private_data =
+			get_sym_session_private_data(sess, dev->driver_id);
+
+	if (unlikely(sess_private_data == NULL)) {
+		DRV_LOG(ERR, "Failed to get session %p private data.",
+				sess_private_data);
+		return;
+	}
+	mlx5_crypto_dek_destroy(priv, sess_private_data->dek);
+	DRV_LOG(DEBUG, "Session %p was cleared.", sess_private_data);
+}
+
 static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_configure			= mlx5_crypto_dev_configure,
 	.dev_start			= NULL,
@@ -68,9 +158,9 @@ static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.stats_reset			= NULL,
 	.queue_pair_setup		= NULL,
 	.queue_pair_release		= NULL,
-	.sym_session_get_size		= NULL,
-	.sym_session_configure		= NULL,
-	.sym_session_clear		= NULL,
+	.sym_session_get_size		= mlx5_crypto_sym_session_get_size,
+	.sym_session_configure		= mlx5_crypto_sym_session_configure,
+	.sym_session_clear		= mlx5_crypto_sym_session_clear,
 	.sym_get_raw_dp_ctx_size	= NULL,
 	.sym_configure_raw_dp_ctx	= NULL,
 };
-- 
2.25.1

[dpdk-dev] [PATCH v2 04/15] crypto/mlx5: add basic operations

[Matan Azrad]

From: Shiri Kuzin <shirik@nvidia.com>

The basic dev control operations are configure, close and get info.

Extended the existing support of configure and close:
	-mlx5_crypto_dev_configure- function used to configure device.
	-mlx5_crypto_dev_close-  function used to close a configured
	 device.

Added support of get info function:
	-mlx5_crypto_dev_infos_get- function used to get specific
	 information of a device.

Added config struct to user private data with the fields socket id,
number of queue pairs and feature flags to be disabled.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 46 +++++++++++++++++++++++++++----
 drivers/crypto/mlx5/mlx5_crypto.h |  1 +
 2 files changed, 42 insertions(+), 5 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index c494ed00d1..cc11cabbea 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -19,6 +19,7 @@
 
 #define MLX5_CRYPTO_DRIVER_NAME mlx5_crypto
 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
+#define MLX5_CRYPTO_MAX_QPS 1024
 
 TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
 				TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
@@ -28,6 +29,9 @@ int mlx5_crypto_logtype;
 
 uint8_t mlx5_crypto_driver_id;
 
+const struct rte_cryptodev_capabilities
+		mlx5_crypto_caps[RTE_CRYPTO_OP_TYPE_UNDEFINED];
+
 static const char mlx5_crypto_drv_name[] = RTE_STR(MLX5_CRYPTO_DRIVER_NAME);
 
 static const struct rte_driver mlx5_drv = {
@@ -49,22 +53,47 @@ struct mlx5_crypto_session {
 	uint32_t dek_id; /* DEK ID */
 } __rte_packed;
 
-static unsigned int
-mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
+static void
+mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev,
+			  struct rte_cryptodev_info *dev_info)
 {
-	return sizeof(struct mlx5_crypto_session);
+	RTE_SET_USED(dev);
+	if (dev_info != NULL) {
+		dev_info->driver_id = mlx5_crypto_driver_id;
+		dev_info->feature_flags = 0;
+		dev_info->capabilities = mlx5_crypto_caps;
+		dev_info->max_nb_queue_pairs = MLX5_CRYPTO_MAX_QPS;
+		dev_info->min_mbuf_headroom_req = 0;
+		dev_info->min_mbuf_tailroom_req = 0;
+		dev_info->sym.max_nb_sessions = 0;
+		/*
+		 * If 0, the device does not have any limitation in number of
+		 * sessions that can be used.
+		 */
+	}
 }
 
 static int
 mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
-		struct rte_cryptodev_config *config __rte_unused)
+			  struct rte_cryptodev_config *config)
 {
 	struct mlx5_crypto_priv *priv = dev->data->dev_private;
 
+	if (config == NULL) {
+		DRV_LOG(ERR, "Invalid crypto dev configure parameters.");
+		return -EINVAL;
+	}
+	if ((config->ff_disable & RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO) != 0) {
+		DRV_LOG(ERR,
+			"Disabled symmetric crypto feature is not supported.");
+		return -ENOTSUP;
+	}
 	if (mlx5_crypto_dek_setup(priv) != 0) {
 		DRV_LOG(ERR, "Dek hash list creation has failed.");
 		return -ENOMEM;
 	}
+	priv->dev_config = *config;
+	DRV_LOG(DEBUG, "Device %u was configured.", dev->driver_id);
 	return 0;
 }
 
@@ -74,9 +103,16 @@ mlx5_crypto_dev_close(struct rte_cryptodev *dev)
 	struct mlx5_crypto_priv *priv = dev->data->dev_private;
 
 	mlx5_crypto_dek_unset(priv);
+	DRV_LOG(DEBUG, "Device %u was closed.", dev->driver_id);
 	return 0;
 }
 
+static unsigned int
+mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
+{
+	return sizeof(struct mlx5_crypto_session);
+}
+
 static int
 mlx5_crypto_sym_session_configure(struct rte_cryptodev *dev,
 				  struct rte_crypto_sym_xform *xform,
@@ -153,7 +189,7 @@ static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_start			= NULL,
 	.dev_stop			= NULL,
 	.dev_close			= mlx5_crypto_dev_close,
-	.dev_infos_get			= NULL,
+	.dev_infos_get			= mlx5_crypto_dev_infos_get,
 	.stats_get			= NULL,
 	.stats_reset			= NULL,
 	.queue_pair_setup		= NULL,
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 4ec67a7e0f..5e270d3d5a 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -24,6 +24,7 @@ struct mlx5_crypto_priv {
 	uint32_t pdn; /* Protection Domain number. */
 	struct ibv_pd *pd;
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
+	struct rte_cryptodev_config dev_config;
 };
 
 struct mlx5_crypto_dek {
-- 
2.25.1

[dpdk-dev] [PATCH v2 05/15] crypto/mlx5: support queue pairs operations

[Matan Azrad]

From: Shiri Kuzin <shirik@nvidia.com>

The HW queue pairs are a pair of send queue and receive queue of
independent work queues packed together in one object for the purpose
of transferring data between nodes of a network.

Completion Queue is a FIFO queue of completed work requests.

In crypto driver we use one QP in loopback in order to encrypt and
decrypt data locally without sending it to the wire.
In the configured QP we only use the SQ to perform the encryption and
decryption operations.

Added implementation for the QP setup function which creates the CQ,
creates the QP and changes its state to RTS (ready to send).

Added implementation for the release QP function to release all the QP
resources.

Added the ops structure that contains any operation which is supported
by the cryptodev.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 124 +++++++++++++++++++++++++++++-
 drivers/crypto/mlx5/mlx5_crypto.h |  11 +++
 2 files changed, 133 insertions(+), 2 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index cc11cabbea..5a14e05d55 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -7,6 +7,7 @@
 #include <rte_errno.h>
 #include <rte_log.h>
 #include <rte_pci.h>
+#include <rte_memory.h>
 
 #include <mlx5_glue.h>
 #include <mlx5_common.h>
@@ -184,6 +185,125 @@ mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev,
 	DRV_LOG(DEBUG, "Session %p was cleared.", sess_private_data);
 }
 
+static int
+mlx5_crypto_queue_pair_release(struct rte_cryptodev *dev, uint16_t qp_id)
+{
+	struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
+
+	if (qp->qp_obj != NULL)
+		claim_zero(mlx5_devx_cmd_destroy(qp->qp_obj));
+	if (qp->umem_obj != NULL)
+		claim_zero(mlx5_glue->devx_umem_dereg(qp->umem_obj));
+	if (qp->umem_buf != NULL)
+		rte_free(qp->umem_buf);
+	mlx5_devx_cq_destroy(&qp->cq_obj);
+	rte_free(qp);
+	dev->data->queue_pairs[qp_id] = NULL;
+	return 0;
+}
+
+static int
+mlx5_crypto_qp2rts(struct mlx5_crypto_qp *qp)
+{
+	/*
+	 * In Order to configure self loopback, when calling these functions the
+	 * remote QP id that is used is the id of the same QP.
+	 */
+	if (mlx5_devx_cmd_modify_qp_state(qp->qp_obj, MLX5_CMD_OP_RST2INIT_QP,
+					  qp->qp_obj->id)) {
+		DRV_LOG(ERR, "Failed to modify QP to INIT state(%u).",
+			rte_errno);
+		return -1;
+	}
+	if (mlx5_devx_cmd_modify_qp_state(qp->qp_obj, MLX5_CMD_OP_INIT2RTR_QP,
+					  qp->qp_obj->id)) {
+		DRV_LOG(ERR, "Failed to modify QP to RTR state(%u).",
+			rte_errno);
+		return -1;
+	}
+	if (mlx5_devx_cmd_modify_qp_state(qp->qp_obj, MLX5_CMD_OP_RTR2RTS_QP,
+					  qp->qp_obj->id)) {
+		DRV_LOG(ERR, "Failed to modify QP to RTS state(%u).",
+			rte_errno);
+		return -1;
+	}
+	return 0;
+}
+
+static int
+mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
+			     const struct rte_cryptodev_qp_conf *qp_conf,
+			     int socket_id)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+	struct mlx5_devx_qp_attr attr = {0};
+	struct mlx5_crypto_qp *qp;
+	uint16_t log_nb_desc = rte_log2_u32(qp_conf->nb_descriptors);
+	uint32_t umem_size = RTE_BIT32(log_nb_desc) *
+			      MLX5_CRYPTO_WQE_SET_SIZE +
+			      sizeof(*qp->db_rec) * 2;
+	uint32_t alloc_size = sizeof(*qp);
+	struct mlx5_devx_cq_attr cq_attr = {
+		.uar_page_id = mlx5_os_get_devx_uar_page_id(priv->uar),
+	};
+
+	alloc_size = RTE_ALIGN(alloc_size, RTE_CACHE_LINE_SIZE);
+	alloc_size += sizeof(struct rte_crypto_op *) * RTE_BIT32(log_nb_desc);
+	qp = rte_zmalloc_socket(__func__, alloc_size, RTE_CACHE_LINE_SIZE,
+				socket_id);
+	if (qp == NULL) {
+		DRV_LOG(ERR, "Failed to allocate QP memory.");
+		rte_errno = ENOMEM;
+		return -rte_errno;
+	}
+	if (mlx5_devx_cq_create(priv->ctx, &qp->cq_obj, log_nb_desc,
+				&cq_attr, socket_id) != 0) {
+		DRV_LOG(ERR, "Failed to create CQ.");
+		goto error;
+	}
+	qp->umem_buf = rte_zmalloc_socket(__func__, umem_size, 4096, socket_id);
+	if (qp->umem_buf == NULL) {
+		DRV_LOG(ERR, "Failed to allocate QP umem.");
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	qp->umem_obj = mlx5_glue->devx_umem_reg(priv->ctx,
+					       (void *)(uintptr_t)qp->umem_buf,
+					       umem_size,
+					       IBV_ACCESS_LOCAL_WRITE);
+	if (qp->umem_obj == NULL) {
+		DRV_LOG(ERR, "Failed to register QP umem.");
+		goto error;
+	}
+	attr.pd = priv->pdn;
+	attr.uar_index = mlx5_os_get_devx_uar_page_id(priv->uar);
+	attr.cqn = qp->cq_obj.cq->id;
+	attr.log_page_size = rte_log2_u32(sysconf(_SC_PAGESIZE));
+	attr.rq_size =  0;
+	attr.sq_size = RTE_BIT32(log_nb_desc);
+	attr.dbr_umem_valid = 1;
+	attr.wq_umem_id = qp->umem_obj->umem_id;
+	attr.wq_umem_offset = 0;
+	attr.dbr_umem_id = qp->umem_obj->umem_id;
+	attr.dbr_address = RTE_BIT64(log_nb_desc) *
+			   MLX5_CRYPTO_WQE_SET_SIZE;
+	qp->qp_obj = mlx5_devx_cmd_create_qp(priv->ctx, &attr);
+	if (qp->qp_obj == NULL) {
+		DRV_LOG(ERR, "Failed to create QP(%u).", rte_errno);
+		goto error;
+	}
+	qp->db_rec = RTE_PTR_ADD(qp->umem_buf, (uintptr_t)attr.dbr_address);
+	if (mlx5_crypto_qp2rts(qp))
+		goto error;
+	qp->ops = (struct rte_crypto_op **)RTE_ALIGN((uintptr_t)(qp + 1),
+							   RTE_CACHE_LINE_SIZE);
+	dev->data->queue_pairs[qp_id] = qp;
+	return 0;
+error:
+	mlx5_crypto_queue_pair_release(dev, qp_id);
+	return -1;
+}
+
 static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_configure			= mlx5_crypto_dev_configure,
 	.dev_start			= NULL,
@@ -192,8 +312,8 @@ static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_infos_get			= mlx5_crypto_dev_infos_get,
 	.stats_get			= NULL,
 	.stats_reset			= NULL,
-	.queue_pair_setup		= NULL,
-	.queue_pair_release		= NULL,
+	.queue_pair_setup		= mlx5_crypto_queue_pair_setup,
+	.queue_pair_release		= mlx5_crypto_queue_pair_release,
 	.sym_session_get_size		= mlx5_crypto_sym_session_get_size,
 	.sym_session_configure		= mlx5_crypto_sym_session_configure,
 	.sym_session_clear		= mlx5_crypto_sym_session_clear,
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 5e270d3d5a..f5313b89f2 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -11,9 +11,11 @@
 #include <rte_cryptodev_pmd.h>
 
 #include <mlx5_common_utils.h>
+#include <mlx5_common_devx.h>
 
 #define MLX5_CRYPTO_DEK_HTABLE_SZ (1 << 11)
 #define MLX5_CRYPTO_KEY_LENGTH 80
+#define MLX5_CRYPTO_WQE_SET_SIZE 1024
 
 struct mlx5_crypto_priv {
 	TAILQ_ENTRY(mlx5_crypto_priv) next;
@@ -27,6 +29,15 @@ struct mlx5_crypto_priv {
 	struct rte_cryptodev_config dev_config;
 };
 
+struct mlx5_crypto_qp {
+	struct mlx5_devx_cq cq_obj;
+	struct mlx5_devx_obj *qp_obj;
+	struct mlx5dv_devx_umem *umem_obj;
+	void *umem_buf;
+	volatile uint32_t *db_rec;
+	struct rte_crypto_op **ops;
+};
+
 struct mlx5_crypto_dek {
 	struct mlx5_hlist_entry entry; /* Pointer to DEK hash list entry. */
 	struct mlx5_devx_obj *obj; /* Pointer to DEK DevX object. */
-- 
2.25.1

[dpdk-dev] [PATCH v2 06/15] crypto/mlx5: add dev stop and start operations

[Matan Azrad]

From: Shiri Kuzin <shirik@nvidia.com>

Add the dev_start function that is used to start a configured device.
Add the dev_stop function that is used to stop a configured device.

Both functions set the dev parameter as used and return 0.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 5a14e05d55..495d5448b2 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -98,6 +98,19 @@ mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
 	return 0;
 }
 
+static void
+mlx5_crypto_dev_stop(struct rte_cryptodev *dev)
+{
+	RTE_SET_USED(dev);
+}
+
+static int
+mlx5_crypto_dev_start(struct rte_cryptodev *dev)
+{
+	RTE_SET_USED(dev);
+	return 0;
+}
+
 static int
 mlx5_crypto_dev_close(struct rte_cryptodev *dev)
 {
@@ -306,8 +319,8 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 
 static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_configure			= mlx5_crypto_dev_configure,
-	.dev_start			= NULL,
-	.dev_stop			= NULL,
+	.dev_start			= mlx5_crypto_dev_start,
+	.dev_stop			= mlx5_crypto_dev_stop,
 	.dev_close			= mlx5_crypto_dev_close,
 	.dev_infos_get			= mlx5_crypto_dev_infos_get,
 	.stats_get			= NULL,
-- 
2.25.1

[dpdk-dev] [PATCH v2 07/15] crypto/mlx5: add memory region management

[Matan Azrad]

From: Shiri Kuzin <shirik@nvidia.com>

Mellanox user space drivers don't deal with physical addresses as part
of a memory protection mechanism.
The device translates the given virtual address to a physical address
using the given memory key as an address space identifier.
That's why any mbuf virtual address is moved directly to the HW
descriptor(WQE).

The mapping between the virtual address to the physical address is saved
in MR configured by the kernel to the HW.

Each MR has a key that should also be moved to the WQE by the SW.

When the SW sees an unmapped address, it extends the address range and
creates a MR using a system call.

Add memory region cache management:
	- 2 level cache per queue-pair - no locks.
	- 1 shared cache between all the queues using a lock.

Using this way, the MR key search per data-path address is optimized.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 20 ++++++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto.h |  3 +++
 2 files changed, 23 insertions(+)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 495d5448b2..79e3d3ee45 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -209,6 +209,7 @@ mlx5_crypto_queue_pair_release(struct rte_cryptodev *dev, uint16_t qp_id)
 		claim_zero(mlx5_glue->devx_umem_dereg(qp->umem_obj));
 	if (qp->umem_buf != NULL)
 		rte_free(qp->umem_buf);
+	mlx5_mr_btree_free(&qp->mr_ctrl.cache_bh);
 	mlx5_devx_cq_destroy(&qp->cq_obj);
 	rte_free(qp);
 	dev->data->queue_pairs[qp_id] = NULL;
@@ -288,6 +289,13 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 		DRV_LOG(ERR, "Failed to register QP umem.");
 		goto error;
 	}
+	if (mlx5_mr_btree_init(&qp->mr_ctrl.cache_bh, MLX5_MR_BTREE_CACHE_N,
+			       priv->dev_config.socket_id) != 0) {
+		DRV_LOG(ERR, "Cannot allocate MR Btree for qp %u.",
+			(uint32_t)qp_id);
+		rte_errno = ENOMEM;
+		goto error;
+	}
 	attr.pd = priv->pdn;
 	attr.uar_index = mlx5_os_get_devx_uar_page_id(priv->uar);
 	attr.cqn = qp->cq_obj.cq->id;
@@ -476,6 +484,17 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 		claim_zero(mlx5_glue->close_device(priv->ctx));
 		return -1;
 	}
+	if (mlx5_mr_btree_init(&priv->mr_scache.cache,
+			     MLX5_MR_BTREE_CACHE_N * 2, rte_socket_id()) != 0) {
+		DRV_LOG(ERR, "Failed to allocate shared cache MR memory.");
+		mlx5_crypto_hw_global_release(priv);
+		rte_cryptodev_pmd_destroy(priv->crypto_dev);
+		claim_zero(mlx5_glue->close_device(priv->ctx));
+		rte_errno = ENOMEM;
+		return -rte_errno;
+	}
+	priv->mr_scache.reg_mr_cb = mlx5_common_verbs_reg_mr;
+	priv->mr_scache.dereg_mr_cb = mlx5_common_verbs_dereg_mr;
 	pthread_mutex_lock(&priv_list_lock);
 	TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next);
 	pthread_mutex_unlock(&priv_list_lock);
@@ -495,6 +514,7 @@ mlx5_crypto_pci_remove(struct rte_pci_device *pdev)
 		TAILQ_REMOVE(&mlx5_crypto_priv_list, priv, next);
 	pthread_mutex_unlock(&priv_list_lock);
 	if (priv) {
+		mlx5_mr_release_cache(&priv->mr_scache);
 		mlx5_crypto_hw_global_release(priv);
 		rte_cryptodev_pmd_destroy(priv->crypto_dev);
 		claim_zero(mlx5_glue->close_device(priv->ctx));
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index f5313b89f2..397267d249 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -12,6 +12,7 @@
 
 #include <mlx5_common_utils.h>
 #include <mlx5_common_devx.h>
+#include <mlx5_common_mr.h>
 
 #define MLX5_CRYPTO_DEK_HTABLE_SZ (1 << 11)
 #define MLX5_CRYPTO_KEY_LENGTH 80
@@ -27,6 +28,7 @@ struct mlx5_crypto_priv {
 	struct ibv_pd *pd;
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
 	struct rte_cryptodev_config dev_config;
+	struct mlx5_mr_share_cache mr_scache; /* Global shared MR cache. */
 };
 
 struct mlx5_crypto_qp {
@@ -36,6 +38,7 @@ struct mlx5_crypto_qp {
 	void *umem_buf;
 	volatile uint32_t *db_rec;
 	struct rte_crypto_op **ops;
+	struct mlx5_mr_ctrl mr_ctrl;
 };
 
 struct mlx5_crypto_dek {
-- 
2.25.1

[dpdk-dev] [PATCH v2 08/15] crypto/mlx5: create login object using DevX

[Matan Azrad]

From: Shiri Kuzin <shirik@nvidia.com>

To work with crypto engines that are marked with wrapped_import_method,
a login session is required.
A crypto login object needs to be created using DevX.

The crypto login object contains:
	- The credential pointer.
	- The import_KEK pointer to be used for all secured information
	  communicated in crypto commands (key fields), including the
	  provided credential in this command.
	- The credential secret, wrapped by the import_KEK indicated in
	  this command. Size includes 8 bytes IV for wrapping.

Added devargs for the required login values:
	- wcs_file - path to the file containing the credential.
	- import_kek_id - the import KEK pointer.
	- credential_id - the credential pointer.

Create the login DevX object in pci_probe function and destroy it in
pci_remove.
Destroying the crypto login object means logout.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 103 ++++++++++++++++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto.h |   7 ++
 2 files changed, 110 insertions(+)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 79e3d3ee45..44038f0e05 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -402,6 +402,101 @@ mlx5_crypto_hw_global_prepare(struct mlx5_crypto_priv *priv)
 	return 0;
 }
 
+
+static int
+mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque)
+{
+	struct mlx5_crypto_devarg_params *devarg_prms = opaque;
+	struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr;
+	unsigned long tmp;
+	FILE *file;
+	int ret;
+	int i;
+
+	if (strcmp(key, "class") == 0)
+		return 0;
+	if (strcmp(key, "wcs_file") == 0) {
+		file = fopen(val, "rb");
+		if (file == NULL) {
+			rte_errno = ENOTSUP;
+			return -rte_errno;
+		}
+		for (i = 0 ; i < MLX5_CRYPTO_CREDENTIAL_SIZE ; i++) {
+			ret = fscanf(file, "%02hhX", &attr->credential[i]);
+			if (ret <= 0) {
+				fclose(file);
+				DRV_LOG(ERR,
+					"Failed to read credential from file.");
+				rte_errno = EINVAL;
+				return -rte_errno;
+			}
+		}
+		fclose(file);
+		devarg_prms->login_devarg = true;
+		return 0;
+	}
+	errno = 0;
+	tmp = strtoul(val, NULL, 0);
+	if (errno) {
+		DRV_LOG(WARNING, "%s: \"%s\" is an invalid integer.", key, val);
+		return -errno;
+	}
+	if (strcmp(key, "import_kek_id") == 0)
+		attr->session_import_kek_ptr = (uint32_t)tmp;
+	else if (strcmp(key, "credential_id") == 0)
+		attr->credential_pointer = (uint32_t)tmp;
+	else
+		DRV_LOG(WARNING, "Invalid key %s.", key);
+	return 0;
+}
+
+static struct mlx5_devx_obj *
+mlx5_crypto_config_login(struct rte_devargs *devargs,
+			 struct ibv_context *ctx)
+{
+	/*
+	 * Set credential pointer and session import KEK pointer to a default
+	 * value of 0.
+	 */
+	struct mlx5_crypto_devarg_params login = {
+			.login_devarg = false,
+			.login_attr = {
+					.credential_pointer = 0,
+					.session_import_kek_ptr = 0,
+			}
+	};
+	struct rte_kvargs *kvlist;
+
+	if (devargs == NULL) {
+		DRV_LOG(ERR,
+	"No login devargs in order to enable crypto operations in the device.");
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	kvlist = rte_kvargs_parse(devargs->args, NULL);
+	if (kvlist == NULL) {
+		DRV_LOG(ERR, "Failed to parse devargs.");
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	if (rte_kvargs_process(kvlist, NULL, mlx5_crypto_args_check_handler,
+			   &login) != 0) {
+		DRV_LOG(ERR, "Devargs handler function Failed.");
+		rte_kvargs_free(kvlist);
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	rte_kvargs_free(kvlist);
+	if (login.login_devarg == false) {
+		DRV_LOG(ERR,
+	"No login credential devarg in order to enable crypto operations "
+	"in the device.");
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	return mlx5_devx_cmd_create_crypto_login_obj(ctx, &login.login_attr);
+}
+
 /**
  * DPDK callback to register a PCI device.
  *
@@ -423,6 +518,7 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	struct ibv_device *ibv;
 	struct rte_cryptodev *crypto_dev;
 	struct ibv_context *ctx;
+	struct mlx5_devx_obj *login;
 	struct mlx5_crypto_priv *priv;
 	struct mlx5_hca_attr attr = { 0 };
 	struct rte_cryptodev_pmd_init_params init_params = {
@@ -461,6 +557,11 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 		rte_errno = ENOTSUP;
 		return -ENOTSUP;
 	}
+	login = mlx5_crypto_config_login(pci_dev->device.devargs, ctx);
+	if (login == NULL) {
+		DRV_LOG(ERR, "Failed to configure login.");
+		return -rte_errno;
+	}
 	crypto_dev = rte_cryptodev_pmd_create(ibv->name, &pci_dev->device,
 					&init_params);
 	if (crypto_dev == NULL) {
@@ -477,6 +578,7 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	crypto_dev->driver_id = mlx5_crypto_driver_id;
 	priv = crypto_dev->data->dev_private;
 	priv->ctx = ctx;
+	priv->login_obj = login;
 	priv->pci_dev = pci_dev;
 	priv->crypto_dev = crypto_dev;
 	if (mlx5_crypto_hw_global_prepare(priv) != 0) {
@@ -517,6 +619,7 @@ mlx5_crypto_pci_remove(struct rte_pci_device *pdev)
 		mlx5_mr_release_cache(&priv->mr_scache);
 		mlx5_crypto_hw_global_release(priv);
 		rte_cryptodev_pmd_destroy(priv->crypto_dev);
+		claim_zero(mlx5_devx_cmd_destroy(priv->login_obj));
 		claim_zero(mlx5_glue->close_device(priv->ctx));
 	}
 	return 0;
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 397267d249..0aef804b92 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -29,6 +29,7 @@ struct mlx5_crypto_priv {
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
 	struct rte_cryptodev_config dev_config;
 	struct mlx5_mr_share_cache mr_scache; /* Global shared MR cache. */
+	struct mlx5_devx_obj *login_obj;
 };
 
 struct mlx5_crypto_qp {
@@ -48,6 +49,12 @@ struct mlx5_crypto_dek {
 	bool size_is_48; /* Whether the key\data size is 48 bytes or not. */
 };
 
+
+struct mlx5_crypto_devarg_params {
+	bool login_devarg;
+	struct mlx5_devx_crypto_login_attr login_attr;
+};
+
 int
 mlx5_crypto_dek_destroy(struct mlx5_crypto_priv *priv,
 			struct mlx5_crypto_dek *dek);
-- 
2.25.1

[dpdk-dev] [PATCH v2 09/15] crypto/mlx5: adjust to the multiple data unit API

[Matan Azrad]

From: Shiri Kuzin <shirik@nvidia.com>

In AES-XTS the data to be encrypted\decrypted does not have to be
in multiples of 16B size, the unit of data is called data-unit.

As a result of patch [1] a new field is added to the cipher capability,
called dataunit_set, where the devices can report the range of
supported data-unit sizes.

The new field enables saving the data-unit size in the session
structure to the block size pointer variable in order to support
several data-unit sizes.

[1] https://www.mail-archive.com/dev@dpdk.org/msg205337.html

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 44038f0e05..1dcebce04c 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -48,6 +48,11 @@ struct mlx5_crypto_session {
 	 * bsf_size, bsf_p_type, encryption_order and encryption standard,
 	 * saved in big endian format.
 	 */
+	uint32_t bsp_res;
+	/*
+	 * crypto_block_size_pointer and reserved 24 bits saved in big endian
+	 * format.
+	 */
 	uint32_t iv_offset:16;
 	/* Starting point for Initialisation Vector. */
 	struct mlx5_crypto_dek *dek; /* Pointer to dek struct. */
@@ -171,6 +176,24 @@ mlx5_crypto_sym_session_configure(struct rte_cryptodev *dev,
 			 MLX5_BSF_P_TYPE_CRYPTO << MLX5_BSF_P_TYPE_OFFSET |
 			 encryption_order << MLX5_ENCRYPTION_ORDER_OFFSET |
 			 MLX5_ENCRYPTION_STANDARD_AES_XTS);
+	switch (xform->cipher.dataunit_len) {
+	case 0:
+		sess_private_data->bsp_res = 0;
+		break;
+	case 512:
+		sess_private_data->bsp_res = rte_cpu_to_be_32
+					     ((uint32_t)MLX5_BLOCK_SIZE_512B <<
+					     MLX5_BLOCK_SIZE_OFFSET);
+		break;
+	case 4096:
+		sess_private_data->bsp_res = rte_cpu_to_be_32
+					     ((uint32_t)MLX5_BLOCK_SIZE_4096B <<
+					     MLX5_BLOCK_SIZE_OFFSET);
+		break;
+	default:
+		DRV_LOG(ERR, "Cipher data unit length is not supported.");
+		return -ENOTSUP;
+	}
 	sess_private_data->iv_offset = cipher->iv.offset;
 	sess_private_data->dek_id =
 			rte_cpu_to_be_32(sess_private_data->dek->obj->id &
-- 
2.25.1

[dpdk-dev] [PATCH v2 10/15] crypto/mlx5: add keytag device argument

[Matan Azrad]

From: Suanming Mou <suanmingm@nvidia.com>

A keytag is a piece of data encrypted together with a DEK.

When a DEK is referenced by an MKEY.bsf through its index, the keytag is
also supplied in the BSF as plaintext. The HW will decrypt the DEK (and
the attached keytag) and will fail the operation if the keytags don't
match.

This commit adds the configuration of the keytag with devargs.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 50 +++++++++++++++++--------------
 drivers/crypto/mlx5/mlx5_crypto.h |  3 +-
 2 files changed, 30 insertions(+), 23 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 1dcebce04c..9f270e4382 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -468,56 +468,52 @@ mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque)
 		attr->session_import_kek_ptr = (uint32_t)tmp;
 	else if (strcmp(key, "credential_id") == 0)
 		attr->credential_pointer = (uint32_t)tmp;
+	else if (strcmp(key, "keytag") == 0)
+		devarg_prms->keytag = tmp;
 	else
 		DRV_LOG(WARNING, "Invalid key %s.", key);
 	return 0;
 }
 
-static struct mlx5_devx_obj *
-mlx5_crypto_config_login(struct rte_devargs *devargs,
-			 struct ibv_context *ctx)
+static int
+mlx5_crypto_parse_devargs(struct rte_devargs *devargs,
+			  struct mlx5_crypto_devarg_params *devarg_prms)
 {
-	/*
-	 * Set credential pointer and session import KEK pointer to a default
-	 * value of 0.
-	 */
-	struct mlx5_crypto_devarg_params login = {
-			.login_devarg = false,
-			.login_attr = {
-					.credential_pointer = 0,
-					.session_import_kek_ptr = 0,
-			}
-	};
+	struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr;
 	struct rte_kvargs *kvlist;
 
+	/* Default values. */
+	attr->credential_pointer = 0;
+	attr->session_import_kek_ptr = 0;
+	devarg_prms->keytag = 0;
 	if (devargs == NULL) {
 		DRV_LOG(ERR,
 	"No login devargs in order to enable crypto operations in the device.");
 		rte_errno = EINVAL;
-		return NULL;
+		return -1;
 	}
 	kvlist = rte_kvargs_parse(devargs->args, NULL);
 	if (kvlist == NULL) {
 		DRV_LOG(ERR, "Failed to parse devargs.");
 		rte_errno = EINVAL;
-		return NULL;
+		return -1;
 	}
 	if (rte_kvargs_process(kvlist, NULL, mlx5_crypto_args_check_handler,
-			   &login) != 0) {
+			   devarg_prms) != 0) {
 		DRV_LOG(ERR, "Devargs handler function Failed.");
 		rte_kvargs_free(kvlist);
 		rte_errno = EINVAL;
-		return NULL;
+		return -1;
 	}
 	rte_kvargs_free(kvlist);
-	if (login.login_devarg == false) {
+	if (devarg_prms->login_devarg == false) {
 		DRV_LOG(ERR,
 	"No login credential devarg in order to enable crypto operations "
 	"in the device.");
 		rte_errno = EINVAL;
-		return NULL;
+		return -1;
 	}
-	return mlx5_devx_cmd_create_crypto_login_obj(ctx, &login.login_attr);
+	return 0;
 }
 
 /**
@@ -543,6 +539,7 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	struct ibv_context *ctx;
 	struct mlx5_devx_obj *login;
 	struct mlx5_crypto_priv *priv;
+	struct mlx5_crypto_devarg_params devarg_prms = { 0 };
 	struct mlx5_hca_attr attr = { 0 };
 	struct rte_cryptodev_pmd_init_params init_params = {
 		.name = "",
@@ -551,6 +548,8 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 		.max_nb_queue_pairs =
 				RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
 	};
+	int ret;
+
 	RTE_SET_USED(pci_drv);
 	if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
 		DRV_LOG(ERR, "Non-primary process type is not supported.");
@@ -580,7 +579,13 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 		rte_errno = ENOTSUP;
 		return -ENOTSUP;
 	}
-	login = mlx5_crypto_config_login(pci_dev->device.devargs, ctx);
+	ret = mlx5_crypto_parse_devargs(pci_dev->device.devargs, &devarg_prms);
+	if (ret) {
+		DRV_LOG(ERR, "Failed to parse devargs.");
+		return -rte_errno;
+	}
+	login = mlx5_devx_cmd_create_crypto_login_obj(ctx,
+						      &devarg_prms.login_attr);
 	if (login == NULL) {
 		DRV_LOG(ERR, "Failed to configure login.");
 		return -rte_errno;
@@ -620,6 +625,7 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	}
 	priv->mr_scache.reg_mr_cb = mlx5_common_verbs_reg_mr;
 	priv->mr_scache.dereg_mr_cb = mlx5_common_verbs_dereg_mr;
+	priv->keytag = rte_cpu_to_be_64(devarg_prms.keytag);
 	pthread_mutex_lock(&priv_list_lock);
 	TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next);
 	pthread_mutex_unlock(&priv_list_lock);
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 0aef804b92..34c65f9a24 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -30,6 +30,7 @@ struct mlx5_crypto_priv {
 	struct rte_cryptodev_config dev_config;
 	struct mlx5_mr_share_cache mr_scache; /* Global shared MR cache. */
 	struct mlx5_devx_obj *login_obj;
+	uint64_t keytag;
 };
 
 struct mlx5_crypto_qp {
@@ -49,10 +50,10 @@ struct mlx5_crypto_dek {
 	bool size_is_48; /* Whether the key\data size is 48 bytes or not. */
 };
 
-
 struct mlx5_crypto_devarg_params {
 	bool login_devarg;
 	struct mlx5_devx_crypto_login_attr login_attr;
+	uint64_t keytag;
 };
 
 int
-- 
2.25.1

[dpdk-dev] [PATCH v2 11/15] crypto/mlx5: add maximum segments device argument

[Matan Azrad]

From: Suanming Mou <suanmingm@nvidia.com>

The mlx5 HW crypto operations are done by attaching crypto property
to a memory region. Once done, every access to the memory via the
crypto-enabled memory region will result with in-line encryption or
decryption of the data.

As a result, the design choice is to provide two types of WQEs. One
is UMR WQE which sets the crypto property and the other is rdma write
WQE which sends DMA command to copy data from local MR to remote MR.

The size of the WQEs will be defined by a new devarg called
max_segs_num.

This devarg also defines the maximum segments in mbuf chain that will be
supported for crypto operations.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 35 +++++++++++++++++++++++++++----
 drivers/crypto/mlx5/mlx5_crypto.h |  7 +++++++
 2 files changed, 38 insertions(+), 4 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 9f270e4382..5e8e7b63d0 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -21,6 +21,7 @@
 #define MLX5_CRYPTO_DRIVER_NAME mlx5_crypto
 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
 #define MLX5_CRYPTO_MAX_QPS 1024
+#define MLX5_CRYPTO_MAX_SEGS 56
 
 TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
 				TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
@@ -464,14 +465,24 @@ mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque)
 		DRV_LOG(WARNING, "%s: \"%s\" is an invalid integer.", key, val);
 		return -errno;
 	}
-	if (strcmp(key, "import_kek_id") == 0)
+	if (strcmp(key, "max_segs_num") == 0) {
+		if (!tmp || tmp > MLX5_CRYPTO_MAX_SEGS) {
+			DRV_LOG(WARNING, "Invalid max_segs_num: %d, should"
+				" be less than %d.",
+				(uint32_t)tmp, MLX5_CRYPTO_MAX_SEGS);
+			rte_errno = EINVAL;
+			return -rte_errno;
+		}
+		devarg_prms->max_segs_num = (uint32_t)tmp;
+	} else if (strcmp(key, "import_kek_id") == 0) {
 		attr->session_import_kek_ptr = (uint32_t)tmp;
-	else if (strcmp(key, "credential_id") == 0)
+	} else if (strcmp(key, "credential_id") == 0) {
 		attr->credential_pointer = (uint32_t)tmp;
-	else if (strcmp(key, "keytag") == 0)
+	} else if (strcmp(key, "keytag") == 0) {
 		devarg_prms->keytag = tmp;
-	else
+	} else {
 		DRV_LOG(WARNING, "Invalid key %s.", key);
+	}
 	return 0;
 }
 
@@ -486,6 +497,7 @@ mlx5_crypto_parse_devargs(struct rte_devargs *devargs,
 	attr->credential_pointer = 0;
 	attr->session_import_kek_ptr = 0;
 	devarg_prms->keytag = 0;
+	devarg_prms->max_segs_num = 8;
 	if (devargs == NULL) {
 		DRV_LOG(ERR,
 	"No login devargs in order to enable crypto operations in the device.");
@@ -626,6 +638,21 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	priv->mr_scache.reg_mr_cb = mlx5_common_verbs_reg_mr;
 	priv->mr_scache.dereg_mr_cb = mlx5_common_verbs_dereg_mr;
 	priv->keytag = rte_cpu_to_be_64(devarg_prms.keytag);
+	priv->max_segs_num = devarg_prms.max_segs_num;
+	priv->umr_wqe_size = sizeof(struct mlx5_wqe_umr_bsf_seg) +
+			     sizeof(struct mlx5_umr_wqe) +
+			     RTE_ALIGN(priv->max_segs_num, 4) *
+			     sizeof(struct mlx5_wqe_dseg);
+	priv->rdmw_wqe_size = sizeof(struct mlx5_rdma_write_wqe) +
+			      sizeof(struct mlx5_wqe_dseg) *
+			      (priv->max_segs_num <= 2 ? 2 : 2 +
+			       RTE_ALIGN(priv->max_segs_num - 2, 4));
+	priv->wqe_set_size = priv->umr_wqe_size + priv->rdmw_wqe_size;
+	priv->wqe_stride = (priv->umr_wqe_size + priv->rdmw_wqe_size) /
+							       MLX5_SEND_WQE_BB;
+	priv->max_rdmaw_klm_n = (priv->rdmw_wqe_size -
+				 sizeof(struct mlx5_rdma_write_wqe)) /
+				 sizeof(struct mlx5_wqe_dseg);
 	pthread_mutex_lock(&priv_list_lock);
 	TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next);
 	pthread_mutex_unlock(&priv_list_lock);
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 34c65f9a24..81452bd700 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -25,12 +25,18 @@ struct mlx5_crypto_priv {
 	struct rte_cryptodev *crypto_dev;
 	void *uar; /* User Access Region. */
 	uint32_t pdn; /* Protection Domain number. */
+	uint32_t max_segs_num; /* Maximum supported data segs. */
 	struct ibv_pd *pd;
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
 	struct rte_cryptodev_config dev_config;
 	struct mlx5_mr_share_cache mr_scache; /* Global shared MR cache. */
 	struct mlx5_devx_obj *login_obj;
 	uint64_t keytag;
+	uint16_t wqe_set_size;
+	uint16_t umr_wqe_size;
+	uint16_t rdmw_wqe_size;
+	uint16_t wqe_stride;
+	uint16_t max_rdmaw_klm_n;
 };
 
 struct mlx5_crypto_qp {
@@ -54,6 +60,7 @@ struct mlx5_crypto_devarg_params {
 	bool login_devarg;
 	struct mlx5_devx_crypto_login_attr login_attr;
 	uint64_t keytag;
+	uint32_t max_segs_num;
 };
 
 int
-- 
2.25.1

[dpdk-dev] [PATCH v2 12/15] crypto/mlx5: add WQE set initialization

[Matan Azrad]

From: Suanming Mou <suanmingm@nvidia.com>

Currently, HW handles the WQEs much faster than the software,
Using the constant WQE set layout can initialize most of the WQE
segments in advanced, and software only needs to configure very
limited segments in datapath. This accelerates the software WQE
organize in datapath.

This commit initializes the fixed WQE set segments.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 83 +++++++++++++++++++++++++++++--
 drivers/crypto/mlx5/mlx5_crypto.h | 10 +++-
 2 files changed, 87 insertions(+), 6 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 5e8e7b63d0..e8f68eb115 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -268,6 +268,69 @@ mlx5_crypto_qp2rts(struct mlx5_crypto_qp *qp)
 	return 0;
 }
 
+static void
+mlx5_crypto_qp_init(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp)
+{
+	uint32_t i;
+
+	for (i = 0 ; i < qp->entries_n; i++) {
+		struct mlx5_wqe_cseg *cseg = RTE_PTR_ADD(qp->umem_buf, i *
+							 priv->wqe_set_size);
+		struct mlx5_wqe_umr_cseg *ucseg = (struct mlx5_wqe_umr_cseg *)
+								     (cseg + 1);
+		struct mlx5_wqe_umr_bsf_seg *bsf =
+			(struct mlx5_wqe_umr_bsf_seg *)(RTE_PTR_ADD(cseg,
+						       priv->umr_wqe_size)) - 1;
+		struct mlx5_wqe_rseg *rseg;
+
+		/* Init UMR WQE. */
+		cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj->id << 8) |
+					 (priv->umr_wqe_size / MLX5_WSEG_SIZE));
+		cseg->flags = RTE_BE32(MLX5_COMP_ONLY_FIRST_ERR <<
+				       MLX5_COMP_MODE_OFFSET);
+		cseg->misc = rte_cpu_to_be_32(qp->mkey[i]->id);
+		ucseg->if_cf_toe_cq_res = RTE_BE32(1u << MLX5_UMRC_IF_OFFSET);
+		ucseg->mkey_mask = RTE_BE64(1u << 0); /* Mkey length bit. */
+		ucseg->ko_to_bs = rte_cpu_to_be_32
+			((RTE_ALIGN(priv->max_segs_num, 4u) <<
+			 MLX5_UMRC_KO_OFFSET) | (4 << MLX5_UMRC_TO_BS_OFFSET));
+		bsf->keytag = priv->keytag;
+		/* Init RDMA WRITE WQE. */
+		cseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size);
+		cseg->flags = RTE_BE32((MLX5_COMP_ALWAYS <<
+				      MLX5_COMP_MODE_OFFSET) |
+				      MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE);
+		rseg = (struct mlx5_wqe_rseg *)(cseg + 1);
+		rseg->rkey = rte_cpu_to_be_32(qp->mkey[i]->id);
+	}
+}
+
+static int
+mlx5_crypto_indirect_mkeys_prepare(struct mlx5_crypto_priv *priv,
+				  struct mlx5_crypto_qp *qp)
+{
+	struct mlx5_umr_wqe *umr;
+	uint32_t i;
+	struct mlx5_devx_mkey_attr attr = {
+		.pd = priv->pdn,
+		.umr_en = 1,
+		.crypto_en = 1,
+		.set_remote_rw = 1,
+		.klm_num = RTE_ALIGN(priv->max_segs_num, 4),
+	};
+
+	for (umr = (struct mlx5_umr_wqe *)qp->umem_buf, i = 0;
+	   i < qp->entries_n; i++, umr = RTE_PTR_ADD(umr, priv->wqe_set_size)) {
+		attr.klm_array = (struct mlx5_klm *)&umr->kseg[0];
+		qp->mkey[i] = mlx5_devx_cmd_mkey_create(priv->ctx, &attr);
+		if (!qp->mkey[i]) {
+			DRV_LOG(ERR, "Failed to allocate indirect mkey.");
+			return -1;
+		}
+	}
+	return 0;
+}
+
 static int
 mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 			     const struct rte_cryptodev_qp_conf *qp_conf,
@@ -278,7 +341,7 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	struct mlx5_crypto_qp *qp;
 	uint16_t log_nb_desc = rte_log2_u32(qp_conf->nb_descriptors);
 	uint32_t umem_size = RTE_BIT32(log_nb_desc) *
-			      MLX5_CRYPTO_WQE_SET_SIZE +
+			      priv->wqe_set_size +
 			      sizeof(*qp->db_rec) * 2;
 	uint32_t alloc_size = sizeof(*qp);
 	struct mlx5_devx_cq_attr cq_attr = {
@@ -286,7 +349,9 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	};
 
 	alloc_size = RTE_ALIGN(alloc_size, RTE_CACHE_LINE_SIZE);
-	alloc_size += sizeof(struct rte_crypto_op *) * RTE_BIT32(log_nb_desc);
+	alloc_size += (sizeof(struct rte_crypto_op *) +
+		       sizeof(struct mlx5_devx_obj *)) *
+		       RTE_BIT32(log_nb_desc);
 	qp = rte_zmalloc_socket(__func__, alloc_size, RTE_CACHE_LINE_SIZE,
 				socket_id);
 	if (qp == NULL) {
@@ -330,8 +395,7 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	attr.wq_umem_id = qp->umem_obj->umem_id;
 	attr.wq_umem_offset = 0;
 	attr.dbr_umem_id = qp->umem_obj->umem_id;
-	attr.dbr_address = RTE_BIT64(log_nb_desc) *
-			   MLX5_CRYPTO_WQE_SET_SIZE;
+	attr.dbr_address = RTE_BIT64(log_nb_desc) * priv->wqe_set_size;
 	qp->qp_obj = mlx5_devx_cmd_create_qp(priv->ctx, &attr);
 	if (qp->qp_obj == NULL) {
 		DRV_LOG(ERR, "Failed to create QP(%u).", rte_errno);
@@ -340,8 +404,17 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	qp->db_rec = RTE_PTR_ADD(qp->umem_buf, (uintptr_t)attr.dbr_address);
 	if (mlx5_crypto_qp2rts(qp))
 		goto error;
-	qp->ops = (struct rte_crypto_op **)RTE_ALIGN((uintptr_t)(qp + 1),
+	qp->mkey = (struct mlx5_devx_obj **)RTE_ALIGN((uintptr_t)(qp + 1),
 							   RTE_CACHE_LINE_SIZE);
+	qp->ops = (struct rte_crypto_op **)(qp->mkey + RTE_BIT32(log_nb_desc));
+	qp->entries_n = 1 << log_nb_desc;
+	if (mlx5_crypto_indirect_mkeys_prepare(priv, qp)) {
+		DRV_LOG(ERR, "Cannot allocate indirect memory regions.");
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	mlx5_crypto_qp_init(priv, qp);
+	qp->priv = priv;
 	dev->data->queue_pairs[qp_id] = qp;
 	return 0;
 error:
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 81452bd700..52fcf5217f 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -16,7 +16,6 @@
 
 #define MLX5_CRYPTO_DEK_HTABLE_SZ (1 << 11)
 #define MLX5_CRYPTO_KEY_LENGTH 80
-#define MLX5_CRYPTO_WQE_SET_SIZE 1024
 
 struct mlx5_crypto_priv {
 	TAILQ_ENTRY(mlx5_crypto_priv) next;
@@ -24,6 +23,7 @@ struct mlx5_crypto_priv {
 	struct rte_pci_device *pci_dev;
 	struct rte_cryptodev *crypto_dev;
 	void *uar; /* User Access Region. */
+	volatile uint64_t *uar_addr;
 	uint32_t pdn; /* Protection Domain number. */
 	uint32_t max_segs_num; /* Maximum supported data segs. */
 	struct ibv_pd *pd;
@@ -40,13 +40,21 @@ struct mlx5_crypto_priv {
 };
 
 struct mlx5_crypto_qp {
+	struct mlx5_crypto_priv *priv;
 	struct mlx5_devx_cq cq_obj;
 	struct mlx5_devx_obj *qp_obj;
+	struct rte_cryptodev_stats stats;
 	struct mlx5dv_devx_umem *umem_obj;
 	void *umem_buf;
 	volatile uint32_t *db_rec;
 	struct rte_crypto_op **ops;
+	struct mlx5_devx_obj **mkey; /* WQE's indirect mekys. */
 	struct mlx5_mr_ctrl mr_ctrl;
+	uint8_t *wqe;
+	uint16_t entries_n;
+	uint16_t pi;
+	uint16_t ci;
+	uint16_t db_pi;
 };
 
 struct mlx5_crypto_dek {
-- 
2.25.1

[dpdk-dev] [PATCH v2 13/15] crypto/mlx5: add enqueue and dequeue operations

[Matan Azrad]

From: Suanming Mou <suanmingm@nvidia.com>

The crypto operations are done with the WQE set which contains
one UMR WQE and one rdma write WQE. Most segments of the WQE
set are initialized properly during queue setup, only limited
segments are initialized according to the crypto detail in the
datapath process.

This commit adds the enquue and dequeue operations and updates
the WQE set segments accordingly.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 243 +++++++++++++++++++++++++++++-
 drivers/crypto/mlx5/mlx5_crypto.h |   3 +
 2 files changed, 241 insertions(+), 5 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index e8f68eb115..08a8c1e925 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -268,6 +268,239 @@ mlx5_crypto_qp2rts(struct mlx5_crypto_qp *qp)
 	return 0;
 }
 
+static __rte_noinline uint32_t
+mlx5_crypto_get_block_size(struct rte_crypto_op *op)
+{
+	uint32_t bl = op->sym->cipher.data.length;
+
+	switch (bl) {
+	case (1 << 20):
+		return RTE_BE32(MLX5_BLOCK_SIZE_1MB << MLX5_BLOCK_SIZE_OFFSET);
+	case (1 << 12):
+		return RTE_BE32(MLX5_BLOCK_SIZE_4096B <<
+				MLX5_BLOCK_SIZE_OFFSET);
+	case (1 << 9):
+		return RTE_BE32(MLX5_BLOCK_SIZE_512B << MLX5_BLOCK_SIZE_OFFSET);
+	default:
+		DRV_LOG(ERR, "Unknown block size: %u.", bl);
+		return UINT32_MAX;
+	}
+}
+
+static __rte_always_inline uint32_t
+mlx5_crypto_klm_set(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp,
+		      struct rte_mbuf *mbuf, struct mlx5_wqe_dseg *klm,
+		      uint32_t offset, uint32_t *remain)
+{
+	uint32_t data_len = (rte_pktmbuf_data_len(mbuf) - offset);
+	uintptr_t addr = rte_pktmbuf_mtod_offset(mbuf, uintptr_t, offset);
+
+	if (data_len > *remain)
+		data_len = *remain;
+	*remain -= data_len;
+	klm->bcount = rte_cpu_to_be_32(data_len);
+	klm->pbuf = rte_cpu_to_be_64(addr);
+	klm->lkey = mlx5_mr_addr2mr_bh(priv->pd, 0,
+		&priv->mr_scache, &qp->mr_ctrl, addr,
+		!!(mbuf->ol_flags & EXT_ATTACHED_MBUF));
+	return klm->lkey;
+
+}
+
+static __rte_always_inline uint32_t
+mlx5_crypto_klms_set(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp,
+		     struct rte_crypto_op *op, struct rte_mbuf *mbuf,
+		     struct mlx5_wqe_dseg *klm)
+{
+	uint32_t remain_len = op->sym->cipher.data.length;
+	uint32_t nb_segs = mbuf->nb_segs;
+	uint32_t klm_n = 1;
+
+	/* First mbuf needs to take the cipher offset. */
+	if (unlikely(mlx5_crypto_klm_set(priv, qp, mbuf, klm,
+		     op->sym->cipher.data.offset, &remain_len) == UINT32_MAX)) {
+		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return 0;
+	}
+	while (remain_len) {
+		nb_segs--;
+		mbuf = mbuf->next;
+		if (unlikely(mbuf == NULL || nb_segs == 0)) {
+			op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+			return 0;
+		}
+		if (unlikely(mlx5_crypto_klm_set(priv, qp, mbuf, klm, 0,
+						 &remain_len) == UINT32_MAX)) {
+			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			return 0;
+		}
+		klm_n++;
+	}
+	return klm_n;
+}
+
+static __rte_always_inline int
+mlx5_crypto_wqe_set(struct mlx5_crypto_priv *priv,
+			 struct mlx5_crypto_qp *qp,
+			 struct rte_crypto_op *op,
+			 struct mlx5_umr_wqe *umr)
+{
+	struct mlx5_crypto_session *sess = get_sym_session_private_data
+				(op->sym->session, mlx5_crypto_driver_id);
+	struct mlx5_wqe_cseg *cseg = &umr->ctr;
+	struct mlx5_wqe_mkey_cseg *mkc = &umr->mkc;
+	struct mlx5_wqe_dseg *klms = &umr->kseg[0];
+	struct mlx5_wqe_umr_bsf_seg *bsf = ((struct mlx5_wqe_umr_bsf_seg *)
+				      RTE_PTR_ADD(umr, priv->umr_wqe_size)) - 1;
+	uint16_t nop_ds;
+	/* Set UMR WQE. */
+	uint32_t klm_n = mlx5_crypto_klms_set(priv, qp, op,
+			op->sym->m_dst ? op->sym->m_dst : op->sym->m_src, klms);
+
+	if (unlikely(klm_n == 0))
+		return 0;
+	bsf->bs_bpt_eo_es = sess->bs_bpt_eo_es;
+	if (unlikely(!sess->bsp_res)) {
+		bsf->bsp_res = mlx5_crypto_get_block_size(op);
+		if (unlikely(bsf->bsp_res == UINT32_MAX)) {
+			op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+			return 0;
+		}
+	} else {
+		bsf->bsp_res = sess->bsp_res;
+	}
+	bsf->raw_data_size = rte_cpu_to_be_32(op->sym->cipher.data.length);
+	memcpy(bsf->xts_initial_tweak,
+	       rte_crypto_op_ctod_offset(op, uint8_t *, sess->iv_offset), 16);
+	bsf->res_dp = sess->dek_id;
+	cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) | MLX5_OPCODE_UMR);
+	mkc->len = rte_cpu_to_be_64(op->sym->cipher.data.length);
+	/* Set RDMA_WRITE WQE. */
+	cseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size);
+	klms = RTE_PTR_ADD(cseg, sizeof(struct mlx5_rdma_write_wqe));
+	cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) |
+					MLX5_OPCODE_RDMA_WRITE);
+	if (op->sym->m_dst != op->sym->m_src) {
+		klm_n = mlx5_crypto_klms_set(priv, qp, op, op->sym->m_src,
+					     klms);
+		if (unlikely(klm_n == 0))
+			return 0;
+	} else {
+		memcpy(klms, &umr->kseg[0], sizeof(*klms) * klm_n);
+	}
+	cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj->id << 8) | (2 + klm_n));
+	qp->db_pi += priv->wqe_stride;
+	/* Set NOP WQE if needed. */
+	klm_n = RTE_ALIGN(klm_n + 2, 4) - 2;
+	nop_ds = priv->max_rdmaw_klm_n - klm_n;
+	if (nop_ds) {
+		cseg = (struct mlx5_wqe_cseg *)(klms + klm_n);
+		cseg->opcode = rte_cpu_to_be_32(((qp->db_pi - (nop_ds >> 2)) <<
+						 8) | MLX5_OPCODE_NOP);
+		cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj->id << 8) | nop_ds);
+	}
+	qp->wqe = (uint8_t *)cseg;
+	return 1;
+}
+
+static __rte_always_inline void
+mlx5_crypto_uar_write(uint64_t val, struct mlx5_crypto_priv *priv)
+{
+#ifdef RTE_ARCH_64
+	*priv->uar_addr = val;
+#else /* !RTE_ARCH_64 */
+	rte_spinlock_lock(&priv->uar32_sl);
+	*(volatile uint32_t *)priv->uar_addr = val;
+	rte_io_wmb();
+	*((volatile uint32_t *)priv->uar_addr + 1) = val >> 32;
+	rte_spinlock_unlock(&priv->uar32_sl);
+#endif
+}
+
+static uint16_t
+mlx5_crypto_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
+			  uint16_t nb_ops)
+{
+	struct mlx5_crypto_qp *qp = queue_pair;
+	struct mlx5_crypto_priv *priv = qp->priv;
+	struct mlx5_umr_wqe *umr;
+	struct rte_crypto_op *op;
+	uint16_t mask = qp->entries_n - 1;
+	uint16_t remain = qp->entries_n - (qp->pi - qp->ci);
+
+	if (remain < nb_ops)
+		nb_ops = remain;
+	else
+		remain = nb_ops;
+	if (unlikely(remain == 0))
+		return 0;
+	do {
+		op = *ops++;
+		umr = RTE_PTR_ADD(qp->umem_buf, priv->wqe_set_size * qp->pi);
+		if (unlikely(mlx5_crypto_wqe_set(priv, qp, op, umr) == 0))
+			break;
+		qp->ops[qp->pi] = op;
+		qp->pi = (qp->pi + 1) & mask;
+	} while (--remain);
+	rte_io_wmb();
+	qp->db_rec[MLX5_SND_DBR] = rte_cpu_to_be_32(qp->db_pi);
+	rte_wmb();
+	mlx5_crypto_uar_write(*(volatile uint64_t *)qp->wqe, qp->priv);
+	rte_wmb();
+	return nb_ops;
+}
+
+static __rte_noinline void
+mlx5_crypto_cqe_err_handle(struct mlx5_crypto_qp *qp, struct rte_crypto_op *op)
+{
+	const uint32_t idx = qp->ci & (qp->entries_n - 1);
+	volatile struct mlx5_err_cqe *cqe = (volatile struct mlx5_err_cqe *)
+							&qp->cq_obj.cqes[idx];
+
+	op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+	DRV_LOG(ERR, "CQE ERR:%x.\n", rte_be_to_cpu_32(cqe->syndrome));
+}
+
+static uint16_t
+mlx5_crypto_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
+			  uint16_t nb_ops)
+{
+	struct mlx5_crypto_qp *qp = queue_pair;
+	volatile struct mlx5_cqe *restrict cqe;
+	struct rte_crypto_op *restrict op;
+	const unsigned int cq_size = qp->entries_n;
+	const unsigned int mask = cq_size - 1;
+	uint32_t idx;
+	uint32_t next_idx = qp->ci & mask;
+	const uint16_t max = RTE_MIN((uint16_t)(qp->pi - qp->ci), nb_ops);
+	uint16_t i = 0;
+	int ret;
+
+	if (unlikely(max == 0))
+		return 0;
+	do {
+		idx = next_idx;
+		next_idx = (qp->ci + 1) & mask;
+		op = qp->ops[idx];
+		cqe = &qp->cq_obj.cqes[idx];
+		ret = check_cqe(cqe, cq_size, qp->ci);
+		rte_io_rmb();
+		if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) {
+			if (unlikely(ret != MLX5_CQE_STATUS_HW_OWN))
+				mlx5_crypto_cqe_err_handle(qp, op);
+			break;
+		}
+		op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+		ops[i++] = op;
+		qp->ci++;
+	} while (i < max);
+	if (likely(i != 0)) {
+		rte_io_wmb();
+		qp->cq_obj.db_rec[0] = rte_cpu_to_be_32(qp->ci);
+	}
+	return i;
+}
+
 static void
 mlx5_crypto_qp_init(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp)
 {
@@ -489,8 +722,9 @@ mlx5_crypto_hw_global_prepare(struct mlx5_crypto_priv *priv)
 	if (mlx5_crypto_pd_create(priv) != 0)
 		return -1;
 	priv->uar = mlx5_devx_alloc_uar(priv->ctx, -1);
-	if (priv->uar == NULL || mlx5_os_get_devx_uar_reg_addr(priv->uar) ==
-	    NULL) {
+	if (priv->uar)
+		priv->uar_addr = mlx5_os_get_devx_uar_reg_addr(priv->uar);
+	if (priv->uar == NULL || priv->uar_addr == NULL) {
 		rte_errno = errno;
 		claim_zero(mlx5_glue->dealloc_pd(priv->pd));
 		DRV_LOG(ERR, "Failed to allocate UAR.");
@@ -685,9 +919,8 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	DRV_LOG(INFO,
 		"Crypto device %s was created successfully.", ibv->name);
 	crypto_dev->dev_ops = &mlx5_crypto_ops;
-	crypto_dev->dequeue_burst = NULL;
-	crypto_dev->enqueue_burst = NULL;
-	crypto_dev->feature_flags = RTE_CRYPTODEV_FF_HW_ACCELERATED;
+	crypto_dev->dequeue_burst = mlx5_crypto_dequeue_burst;
+	crypto_dev->enqueue_burst = mlx5_crypto_enqueue_burst;
 	crypto_dev->driver_id = mlx5_crypto_driver_id;
 	priv = crypto_dev->data->dev_private;
 	priv->ctx = ctx;
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 52fcf5217f..ac4ad1834f 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -37,6 +37,9 @@ struct mlx5_crypto_priv {
 	uint16_t rdmw_wqe_size;
 	uint16_t wqe_stride;
 	uint16_t max_rdmaw_klm_n;
+#ifndef RTE_ARCH_64
+	rte_spinlock_t uar32_sl;
+#endif /* RTE_ARCH_64 */
 };
 
 struct mlx5_crypto_qp {
-- 
2.25.1

[dpdk-dev] [PATCH v2 14/15] crypto/mlx5: add statistic get and reset operations

[Matan Azrad]

From: Suanming Mou <suanmingm@nvidia.com>

This commit adds mlx5 crypto statistic get and reset operations.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 39 ++++++++++++++++++++++++++++---
 1 file changed, 36 insertions(+), 3 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 08a8c1e925..af8985939e 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -437,11 +437,14 @@ mlx5_crypto_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
 	do {
 		op = *ops++;
 		umr = RTE_PTR_ADD(qp->umem_buf, priv->wqe_set_size * qp->pi);
-		if (unlikely(mlx5_crypto_wqe_set(priv, qp, op, umr) == 0))
+		if (unlikely(mlx5_crypto_wqe_set(priv, qp, op, umr) == 0)) {
+			qp->stats.enqueue_err_count++;
 			break;
+		}
 		qp->ops[qp->pi] = op;
 		qp->pi = (qp->pi + 1) & mask;
 	} while (--remain);
+	qp->stats.enqueued_count += nb_ops;
 	rte_io_wmb();
 	qp->db_rec[MLX5_SND_DBR] = rte_cpu_to_be_32(qp->db_pi);
 	rte_wmb();
@@ -458,6 +461,7 @@ mlx5_crypto_cqe_err_handle(struct mlx5_crypto_qp *qp, struct rte_crypto_op *op)
 							&qp->cq_obj.cqes[idx];
 
 	op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+	qp->stats.dequeue_err_count++;
 	DRV_LOG(ERR, "CQE ERR:%x.\n", rte_be_to_cpu_32(cqe->syndrome));
 }
 
@@ -497,6 +501,7 @@ mlx5_crypto_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
 	if (likely(i != 0)) {
 		rte_io_wmb();
 		qp->cq_obj.db_rec[0] = rte_cpu_to_be_32(qp->ci);
+		qp->stats.dequeued_count += i;
 	}
 	return i;
 }
@@ -655,14 +660,42 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	return -1;
 }
 
+static void
+mlx5_crypto_stats_get(struct rte_cryptodev *dev,
+		      struct rte_cryptodev_stats *stats)
+{
+	int qp_id;
+
+	for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
+		struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
+
+		stats->enqueued_count += qp->stats.enqueued_count;
+		stats->dequeued_count += qp->stats.dequeued_count;
+		stats->enqueue_err_count += qp->stats.enqueue_err_count;
+		stats->dequeue_err_count += qp->stats.dequeue_err_count;
+	}
+}
+
+static void
+mlx5_crypto_stats_reset(struct rte_cryptodev *dev)
+{
+	int qp_id;
+
+	for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
+		struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
+
+		memset(&qp->stats, 0, sizeof(qp->stats));
+	}
+}
+
 static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_configure			= mlx5_crypto_dev_configure,
 	.dev_start			= mlx5_crypto_dev_start,
 	.dev_stop			= mlx5_crypto_dev_stop,
 	.dev_close			= mlx5_crypto_dev_close,
 	.dev_infos_get			= mlx5_crypto_dev_infos_get,
-	.stats_get			= NULL,
-	.stats_reset			= NULL,
+	.stats_get			= mlx5_crypto_stats_get,
+	.stats_reset			= mlx5_crypto_stats_reset,
 	.queue_pair_setup		= mlx5_crypto_queue_pair_setup,
 	.queue_pair_release		= mlx5_crypto_queue_pair_release,
 	.sym_session_get_size		= mlx5_crypto_sym_session_get_size,
-- 
2.25.1

[dpdk-dev] [PATCH v2 15/15] crypto/mlx5: set feature flags and capabilities

[Matan Azrad]

From: Shiri Kuzin <shirik@nvidia.com>

Add the supported capabilities to the crypto driver.

Add supported feature flags.

Add crypto driver documentation.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
 doc/guides/cryptodevs/features/mlx5.ini |  37 ++++++
 doc/guides/cryptodevs/index.rst         |   1 +
 doc/guides/cryptodevs/mlx5.rst          | 152 ++++++++++++++++++++++++
 doc/guides/rel_notes/release_21_05.rst  |   5 +
 drivers/crypto/mlx5/mlx5_crypto.c       |  39 +++++-
 5 files changed, 231 insertions(+), 3 deletions(-)
 create mode 100644 doc/guides/cryptodevs/features/mlx5.ini
 create mode 100644 doc/guides/cryptodevs/mlx5.rst

diff --git a/doc/guides/cryptodevs/features/mlx5.ini b/doc/guides/cryptodevs/features/mlx5.ini
new file mode 100644
index 0000000000..a89526add0
--- /dev/null
+++ b/doc/guides/cryptodevs/features/mlx5.ini
@@ -0,0 +1,37 @@
+;
+; Features of a mlx5 crypto driver.
+;
+; Refer to default.ini for the full list of available PMD features.
+;
+[Features]
+Symmetric crypto       = Y
+HW Accelerated         = Y
+In Place SGL           = Y
+OOP SGL In SGL Out     = Y
+OOP SGL In LB  Out     = Y
+OOP LB  In SGL Out     = Y
+OOP LB  In LB  Out     = Y
+Cipher multiple data units = Y
+Cipher wrapped key     = Y
+
+;
+; Supported crypto algorithms of a mlx5 crypto driver.
+;
+[Cipher]
+AES XTS (128)  = Y
+AES XTS (256)  = Y
+
+;
+; Supported authentication algorithms of a mlx5 crypto driver.
+;
+[Auth]
+
+;
+; Supported AEAD algorithms of a mlx5 crypto driver.
+;
+[AEAD]
+
+;
+; Supported Asymmetric algorithms of a mlx5 crypto driver.
+;
+[Asymmetric]
diff --git a/doc/guides/cryptodevs/index.rst b/doc/guides/cryptodevs/index.rst
index 279f56a002..747409c441 100644
--- a/doc/guides/cryptodevs/index.rst
+++ b/doc/guides/cryptodevs/index.rst
@@ -22,6 +22,7 @@ Crypto Device Drivers
     octeontx
     octeontx2
     openssl
+    mlx5
     mvsam
     nitrox
     null
diff --git a/doc/guides/cryptodevs/mlx5.rst b/doc/guides/cryptodevs/mlx5.rst
new file mode 100644
index 0000000000..4ccec78be8
--- /dev/null
+++ b/doc/guides/cryptodevs/mlx5.rst
@@ -0,0 +1,152 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+   Copyright 2021 Mellanox Technologies, Ltd
+
+.. include:: <isonum.txt>
+
+MLX5 Crypto Driver
+==================
+
+The MLX5 crypto driver library
+(**librte_crypto_mlx5**) provides support for **Mellanox ConnectX-6**
+family adapters.
+
+Overview
+--------
+
+The device can provide disk encryption services, allowing data encryption
+and decryption towards a disk. Having all encryption/decryption
+operations done in a single device can reduce cost and overheads of the related
+FIPS certification, as ConnectX-6 is FIPS 140-2 level-2 ready.
+The encryption cipher is AES-XTS of 256/512 key size.
+
+MKEY is a memory region object in the hardware, that holds address translation information and
+attributes per memory area. Its ID must be tied to addresses provided to the hardware.
+The encryption operations are performed with MKEY read\write transactions, when
+the MKEY is configured to perform crypto operations.
+
+The encryption does not require text to be aligned to the AES block size (128b).
+
+In order to move the device to crypto operational mode, credential and KEK
+(Key Encrypting Key) should be set as the first step.
+The credential will be used by the software in order to perform crypto login, and the KEK is
+the AES Key Wrap Algorithm (rfc3394) key that will be used for sensitive data
+wrapping.
+The credential and the AES-XTS keys should be provided to the hardware, as ciphertext
+encrypted by the KEK.
+
+A keytag (64 bits) should be appended to the AES-XTS keys (before wrapping),
+and will be validated when the hardware attempts to access it.
+
+For security reasons and to increase robustness, this driver only deals with virtual
+memory addresses. The way resources allocations are handled by the kernel,
+combined with hardware specifications that allow handling virtual memory
+addresses directly, ensure that DPDK applications cannot access random
+physical memory (or memory that does not belong to the current process).
+
+The PMD uses libibverbs and libmlx5 to access the device firmware or to
+access the hardware components directly.
+There are different levels of objects and bypassing abilities.
+To get the best performances:
+
+- Verbs is a complete high-level generic API.
+- Direct Verbs is a device-specific API.
+- DevX allows to access firmware objects.
+
+Enabling librte_crypto_mlx5 causes DPDK applications to be linked against
+libibverbs.
+
+Mellanox mlx5 PCI device can be probed by a number of different PCI devices, such as
+net / vDPA / RegEx. To select the crypto PMD, ``class=crypto``
+should be specified as a device parameter. The crypto device can be probed and
+used with other Mellanox classes by adding more options in the class.
+For example: ``class=net:crypto`` will probe both the net PMD and the crypto
+PMD.
+
+When crypto engines are defined to work in wrapped import method, they come out
+of the factory in Commissioning mode, and thus, cannot be used for crypto operations
+yet. A dedicated tool is used for changing the mode from Commissioning to
+Operational, while setting the first import_KEK and credential in plaintext.
+The mlxreg dedicated tool should be used as follows:
+
+- Set CRYPTO_OPERATIONAL register to set the device in crypto operational mode.
+
+  The input to this tool is:
+    The first credential in plaintext, 40B.
+    The first import_KEK in plaintext: kek size 0 for 16B or 1 for 32B, kek data.
+
+  Example:
+  mlxreg -d /dev/mst/mt4123_pciconf0 --reg_name CRYPTO_OPERATIONAL --get
+
+  The "wrapped_crypto_operational" value will be "0x00000000".
+  The command to set the register should be executed only once, and all the
+  values mentioned above should be specified in the same command.
+
+  Example:
+  mlxreg -d /dev/mst/mt4123_pciconf0 --reg_name CRYPTO_OPERATIONAL
+  --set "credential[0]=0x10000000, credential[1]=0x10000000, kek[0]=0x00000000"
+
+  All values not specified will remain 0.
+  "wrapped_crypto_going_to_commissioning" and  "wrapped_crypto_operational"
+  should not be specified.
+
+  All the device ports should set it in order to move to operational mode.
+
+- Query CRYPTO_OPERATIONAL register to make sure the device is in Operational
+  mode.
+
+  Example:
+  mlxreg -d /dev/mst/mt4123_pciconf0 --reg_name CRYPTO_OPERATIONAL --get
+  The "wrapped_crypto_operational" value will be "0x00000001" if the mode was
+  successfully changed to operational mode.
+
+
+Driver options
+--------------
+
+- ``class`` parameter [string]
+
+  Select the class of the driver that should probe the device.
+  `crypto` for the mlx5 crypto driver.
+
+- ``wcs_file`` parameter [string] - mandatory
+
+  File path including only the wrapped credential in string format of hexadecimal
+  numbers, represent 48 bytes (8 bytes IV added by the AES key wrap algorithm).
+
+- ``import_kek_id`` parameter [int]
+
+  The identifier of the KEK, default value is 0 represents the operational
+  register import_kek..
+
+- ``credential_id`` parameter [int]
+
+  The identifier of the credential, default value is 0 represents the operational
+  register credential.
+
+- ``max_segs_num`` parameter [int]
+
+  Maximum number of mbuf chain segments(src or dest), default value is 8.
+
+- ``keytag`` parameter [int]
+
+  The plaintext of the keytag appanded to the AES-XTS keys, default value is 0.
+
+
+Limitations
+-----------
+
+- AES-XTS keys provided in Xform must include keytag and should be wrappend.
+- The supported data-unit lengths are: 512B, 1KB, 1MB. In case the `dataunit_len`
+  is not provided in the cipher Xform, the OP length is limitted to the above values.
+
+
+Supported NICs
+--------------
+
+* Mellanox\ |reg| ConnectX\ |reg|-6 200G MCX654106A-HCAT (2x200G)
+
+Prerequisites
+-------------
+
+- Mellanox OFED version: **5.3**
+  see :doc:`../../nics/mlx5` guide for more Mellanox OFED details.
diff --git a/doc/guides/rel_notes/release_21_05.rst b/doc/guides/rel_notes/release_21_05.rst
index 133a0dbbf2..dc508fc63c 100644
--- a/doc/guides/rel_notes/release_21_05.rst
+++ b/doc/guides/rel_notes/release_21_05.rst
@@ -273,6 +273,11 @@ New Features
   * Added support for crypto adapter forward mode in octeontx2 event and crypto
     device driver.
 
+* **Added support for Nvidia crypto device driver.**
+
+  * Added mlx5 crypto driver to support AES-XTS cipher operations.
+    the first device to support it is ConnectX-6.
+
 
 Removed Items
 -------------
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index af8985939e..c2e75905fd 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -22,6 +22,14 @@
 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
 #define MLX5_CRYPTO_MAX_QPS 1024
 #define MLX5_CRYPTO_MAX_SEGS 56
+#define MLX5_CRYPTO_FEATURE_FLAGS \
+	(RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | \
+	RTE_CRYPTODEV_FF_IN_PLACE_SGL | RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT | \
+	RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | \
+	RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT | \
+	RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | \
+	RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY | \
+	RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS)
 
 TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
 				TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
@@ -31,8 +39,32 @@ int mlx5_crypto_logtype;
 
 uint8_t mlx5_crypto_driver_id;
 
-const struct rte_cryptodev_capabilities
-		mlx5_crypto_caps[RTE_CRYPTO_OP_TYPE_UNDEFINED];
+const struct rte_cryptodev_capabilities mlx5_crypto_caps[] = {
+	{		/* AES XTS */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+			{.cipher = {
+				.algo = RTE_CRYPTO_CIPHER_AES_XTS,
+				.block_size = 16,
+				.key_size = {
+					.min = 32,
+					.max = 64,
+					.increment = 32
+				},
+				.iv_size = {
+					.min = 16,
+					.max = 16,
+					.increment = 0
+				},
+				.dataunit_set =
+				RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_512_BYTES |
+				RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_4096_BYTES,
+			}, }
+		}, }
+	},
+};
+
 
 static const char mlx5_crypto_drv_name[] = RTE_STR(MLX5_CRYPTO_DRIVER_NAME);
 
@@ -67,7 +99,7 @@ mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev,
 	RTE_SET_USED(dev);
 	if (dev_info != NULL) {
 		dev_info->driver_id = mlx5_crypto_driver_id;
-		dev_info->feature_flags = 0;
+		dev_info->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
 		dev_info->capabilities = mlx5_crypto_caps;
 		dev_info->max_nb_queue_pairs = MLX5_CRYPTO_MAX_QPS;
 		dev_info->min_mbuf_headroom_req = 0;
@@ -954,6 +986,7 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	crypto_dev->dev_ops = &mlx5_crypto_ops;
 	crypto_dev->dequeue_burst = mlx5_crypto_dequeue_burst;
 	crypto_dev->enqueue_burst = mlx5_crypto_enqueue_burst;
+	crypto_dev->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
 	crypto_dev->driver_id = mlx5_crypto_driver_id;
 	priv = crypto_dev->data->dev_private;
 	priv->ctx = ctx;
-- 
2.25.1

Re: [dpdk-dev] [PATCH v2 07/16] common/mlx5: fix cypto bsf attr

[Matan Azrad]

Hi Raslan

See below , need to squash this to
" common/mlx5: adjust DevX MKEY fields for crypto"

The one below it...missed that, sorry.

From: Matan Azrad <matan@nvidia.com>
> From: Suanming Mou <suanmingm@nvidia.com>
> 
> This commit should be squash to crypto en commit.
> 
> Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
> ---
>  drivers/common/mlx5/mlx5_devx_cmds.c | 4 ++++
>  1 file changed, 4 insertions(+)
> 
> diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c
> b/drivers/common/mlx5/mlx5_devx_cmds.c
> index 68a10b149a..7e3c8b55fa 100644
> --- a/drivers/common/mlx5/mlx5_devx_cmds.c
> +++ b/drivers/common/mlx5/mlx5_devx_cmds.c
> @@ -278,6 +278,10 @@ mlx5_devx_cmd_mkey_create(void *ctx,
>  	MLX5_SET64(mkc, mkc, start_addr, attr->addr);
>  	MLX5_SET64(mkc, mkc, len, attr->size);
>  	MLX5_SET(mkc, mkc, crypto_en, attr->crypto_en);
> +	if (attr->crypto_en) {
> +		MLX5_SET(mkc, mkc, bsf_en, attr->crypto_en);
> +		MLX5_SET(mkc, mkc, bsf_octword_size, 4);
> +	}
>  	mkey->obj = mlx5_glue->devx_obj_create(ctx, in, in_size_dw * 4,
> out,
>  					       sizeof(out));
>  	if (!mkey->obj) {
> --
> 2.25.1

Re: [dpdk-dev] [PATCH v2 00/15] drivers: introduce mlx5 crypto PMD

[Matan Azrad]

From: Matan Azrad <matan@nvidia.com>
 
> Add a new PMD for Nvidia devices- crypto PMD.
> The crypto PMD will be supported on Nvidia ConnectX6 The crypto PMD will
> add the support of encryption and decryption using the AES-XTS symmetric
> algorithm.
> The crypto PMD requires rdma-core and uses mlx5 DevX.
> 
> v2:
> Add data-path part.
> 
> Shiri Kuzin (10):
>   drivers: introduce mlx5 crypto PMD
>   crypto/mlx5: add DEK object management
>   crypto/mlx5: support session operations
>   crypto/mlx5: add basic operations
>   crypto/mlx5: support queue pairs operations
>   crypto/mlx5: add dev stop and start operations
>   crypto/mlx5: add memory region management
>   crypto/mlx5: create login object using DevX
>   crypto/mlx5: adjust to the multiple data unit API
>   crypto/mlx5: set feature flags and capabilities
> 
> Suanming Mou (5):
>   crypto/mlx5: add keytag device argument
>   crypto/mlx5: add maximum segments device argument
>   crypto/mlx5: add WQE set initialization
>   crypto/mlx5: add enqueue and dequeue operations
>   crypto/mlx5: add statistic get and reset operations


Series-acked-by: Matan Azrad <matan@nvidia.com>
 
>  MAINTAINERS                             |    4 +
>  doc/guides/cryptodevs/features/mlx5.ini |   37 +
>  doc/guides/cryptodevs/index.rst         |    1 +
>  doc/guides/cryptodevs/mlx5.rst          |  152 ++++
>  doc/guides/rel_notes/release_21_05.rst  |    5 +
>  drivers/common/mlx5/mlx5_common_pci.c   |   14 +
>  drivers/common/mlx5/mlx5_common_pci.h   |   21 +-
>  drivers/crypto/meson.build              |    1 +
>  drivers/crypto/mlx5/meson.build         |   27 +
>  drivers/crypto/mlx5/mlx5_crypto.c       | 1092 +++++++++++++++++++++++
>  drivers/crypto/mlx5/mlx5_crypto.h       |   92 ++
>  drivers/crypto/mlx5/mlx5_crypto_dek.c   |  136 +++
>  drivers/crypto/mlx5/mlx5_crypto_utils.h |   19 +
>  drivers/crypto/mlx5/version.map         |    3 +
>  14 files changed, 1594 insertions(+), 10 deletions(-)  create mode 100644
> doc/guides/cryptodevs/features/mlx5.ini
>  create mode 100644 doc/guides/cryptodevs/mlx5.rst  create mode 100644
> drivers/crypto/mlx5/meson.build  create mode 100644
> drivers/crypto/mlx5/mlx5_crypto.c  create mode 100644
> drivers/crypto/mlx5/mlx5_crypto.h  create mode 100644
> drivers/crypto/mlx5/mlx5_crypto_dek.c
>  create mode 100644 drivers/crypto/mlx5/mlx5_crypto_utils.h
>  create mode 100644 drivers/crypto/mlx5/version.map
> 
> --
> 2.25.1

Re: [dpdk-dev] [PATCH v2 00/16] mlx5 common part for crypto driver

[Matan Azrad]

From: Matan Azrad <matan@nvidia.com>
> The crypto PMD will be supported on Nvidia ConnectX6 The crypto PMD will
> add the support of encryption and decryption using the AES-XTS symmetric
> algorithm.
> The crypto PMD requires rdma-core and uses mlx5 DevX.
> This is the mlx5 common part that added support for DevX commands
> needed for crypto driver.
> Also includes PRM definitions.
> 
> v2:
> Add data-path part.
> 
> Dekel Peled (12):
>   common/mlx5: remove redundant spaces in header file
>   common/mlx5: update GENEVE TLV OPT obj name
>   common/mlx5: optimize read of general obj type caps
>   common/mlx5: add HCA cap for AES-XTS crypto
>   common/mlx5: support general object DEK create op
>   common/mlx5: adjust DevX MKEY fields for crypto
>   common/mlx5: support general obj IMPORT KEK create
>   common/mlx5: support general obj CRYPTO LOGIN create
>   common/mlx5: add crypto BSF struct and defines
>   common/mlx5: support general obj CREDENTIAL create
>   common/mlx5: add crypto register structs and defs
>   common/mlx5: support register write access
> 
> Shiri Kuzin (2):
>   common/mlx5: share hash list tool
>   common/mlx5: share get ib device match function
> 
> Suanming Mou (2):
>   common/mlx5: fix cypto bsf attr
>   common/mlx5: add UMR and RDMA write WQE defines

Series-acked-by: Matan Azrad <matan@nvidia.com>

>  drivers/common/mlx5/linux/mlx5_common_os.c   |  30 +-
>  drivers/common/mlx5/linux/mlx5_common_os.h   |   5 +
>  drivers/common/mlx5/linux/mlx5_nl.c          |   2 +-
>  drivers/common/mlx5/meson.build              |  15 +-
>  drivers/common/mlx5/mlx5_common.c            |   2 +-
>  drivers/common/mlx5/mlx5_common.h            |   2 +
>  drivers/common/mlx5/mlx5_common_devx.c       |   2 +-
>  drivers/common/mlx5/mlx5_common_log.h        |  21 +
>  drivers/common/mlx5/mlx5_common_mp.c         |   2 +-
>  drivers/common/mlx5/mlx5_common_mr.c         |   2 +-
>  drivers/common/mlx5/mlx5_common_pci.c        |   4 +-
>  drivers/common/mlx5/mlx5_common_utils.c      | 221 +++++++
>  drivers/common/mlx5/mlx5_common_utils.h      | 202 +++++-
>  drivers/common/mlx5/mlx5_devx_cmds.c         | 312 +++++++++-
>  drivers/common/mlx5/mlx5_devx_cmds.h         |  66 ++
>  drivers/common/mlx5/mlx5_malloc.c            |   2 +-
>  drivers/common/mlx5/mlx5_prm.h               | 610 +++++++++++++------
>  drivers/common/mlx5/version.map              |  24 +-
>  drivers/common/mlx5/windows/mlx5_common_os.c |   2 +-
>  drivers/common/mlx5/windows/mlx5_glue.c      |   2 +-
>  drivers/compress/mlx5/mlx5_compress.c        |  30 +-
>  drivers/net/mlx5/mlx5_utils.c                | 209 -------
>  drivers/net/mlx5/mlx5_utils.h                | 194 +-----
>  drivers/regex/mlx5/mlx5_regex.c              |  30 +-
>  drivers/vdpa/mlx5/mlx5_vdpa.c                |  32 +-
>  25 files changed, 1317 insertions(+), 706 deletions(-)  create mode 100644
> drivers/common/mlx5/mlx5_common_log.h
>  create mode 100644 drivers/common/mlx5/mlx5_common_utils.c
> 
> --
> 2.25.1

Re: [dpdk-dev] [PATCH v2 12/16] common/mlx5: share get ib device match function

[Thomas Monjalon]

29/04/2021 17:43, Matan Azrad:
> From: Shiri Kuzin <shirik@nvidia.com>
> 
> The get_ib_device_match function iterates over the list of ib devices
> returned by the get_device_list glue function and returns the ib device
> matching the provided address.
> 
> Since this function is in use by several drivers, in this patch we
> share the function in common part.
> 
> Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
> Acked-by: Matan Azrad <matan@nvidia.com>
> ---
> --- a/drivers/common/mlx5/linux/mlx5_common_os.c
> +++ b/drivers/common/mlx5/linux/mlx5_common_os.c
> +struct ibv_device *
> +mlx5_os_get_ib_device_match(struct rte_pci_addr *addr)

It would be better as mlx5_get_pci_ibv_device()
in drivers/common/mlx5/mlx5_common_pci.c
Just need to forward declare struct ibv_device
in drivers/common/mlx5/mlx5_common.h
This is what is done internally for sub-functions.

> --- a/drivers/common/mlx5/mlx5_common.h
> +++ b/drivers/common/mlx5/mlx5_common.h
> @@ -18,6 +18,7 @@
>  
>  #include "mlx5_prm.h"
>  #include "mlx5_devx_cmds.h"
> +#include "mlx5_common_os.h"
>  
>  /* Reported driver name. */
>  #define MLX5_PCI_DRIVER_NAME "mlx5_pci"
> @@ -215,6 +216,7 @@ enum mlx5_class {
>  	MLX5_CLASS_VDPA = RTE_BIT64(1),
>  	MLX5_CLASS_REGEX = RTE_BIT64(2),
>  	MLX5_CLASS_COMPRESS = RTE_BIT64(3),
> +	MLX5_CLASS_CRYPTO = RTE_BIT64(4),
>  };

This change has nothing to do in this patch I think.

> --- a/drivers/common/mlx5/version.map
> +++ b/drivers/common/mlx5/version.map
> @@ -111,6 +111,7 @@ INTERNAL {
>  	mlx5_os_reg_mr;
>  	mlx5_os_umem_reg;
>  	mlx5_os_umem_dereg;
> +	mlx5_os_get_ib_device_match; # WINDOWS_NO_EXPORT

Please keep alphabetical order.

Re: [dpdk-dev] [PATCH v2 15/16] common/mlx5: support register write access

[Thomas Monjalon]

29/04/2021 17:43, Matan Azrad:
> From: Dekel Peled <dekelp@nvidia.com>
> 
> This patch adds support of write operation to NIC registers.
> 
> Signed-off-by: Dekel Peled <dekelp@nvidia.com>
> Acked-by: Matan Azrad <matan@nvidia.com>
> ---
> --- a/drivers/common/mlx5/version.map
> +++ b/drivers/common/mlx5/version.map
> @@ -13,8 +13,17 @@ INTERNAL {
>  	mlx5_dev_to_pci_addr; # WINDOWS_NO_EXPORT
>  
>  	mlx5_devx_cmd_alloc_pd;
> +	mlx5_devx_alloc_uar;

Please keep alphabetical order.

> +
>  	mlx5_devx_cmd_create_cq;
> +	mlx5_devx_cmd_create_credential_obj;
> +	mlx5_devx_cmd_create_crypto_login_obj;
> +	mlx5_devx_cmd_create_dek_obj;
>  	mlx5_devx_cmd_create_flex_parser;
> +	mlx5_devx_cmd_create_flow_hit_aso_obj;
> +	mlx5_devx_cmd_create_flow_meter_aso_obj;
> +	mlx5_devx_cmd_create_geneve_tlv_option;
> +	mlx5_devx_cmd_create_import_kek_obj;
>  	mlx5_devx_cmd_create_qp;
>  	mlx5_devx_cmd_create_rq;
>  	mlx5_devx_cmd_create_rqt;
> @@ -24,13 +33,6 @@ INTERNAL {
>  	mlx5_devx_cmd_create_tis;
>  	mlx5_devx_cmd_create_virtio_q_counters; # WINDOWS_NO_EXPORT
>  	mlx5_devx_cmd_create_virtq;
> -	mlx5_devx_cmd_create_flow_hit_aso_obj;
> -	mlx5_devx_cmd_create_flow_meter_aso_obj;
> -	mlx5_devx_cmd_create_geneve_tlv_option;
> -        mlx5_devx_cmd_create_dek_obj;
> -        mlx5_devx_cmd_create_import_kek_obj;
> -        mlx5_devx_cmd_create_credential_obj;
> -        mlx5_devx_cmd_create_crypto_login_obj;

Are they changes of previous patches?
It seems some cleaning is required in previous patches.

>  	mlx5_devx_cmd_destroy;
>  	mlx5_devx_cmd_flow_counter_alloc;
>  	mlx5_devx_cmd_flow_counter_query;
> @@ -51,12 +53,17 @@ INTERNAL {
>  	mlx5_devx_cmd_queue_counter_alloc; # WINDOWS_NO_EXPORT
>  	mlx5_devx_cmd_queue_counter_query; # WINDOWS_NO_EXPORT
>  	mlx5_devx_cmd_register_read;
> +	mlx5_devx_cmd_register_write;
>  	mlx5_devx_cmd_wq_query; # WINDOWS_NO_EXPORT
>  	mlx5_devx_get_out_command_status;
>  	mlx5_devx_alloc_uar; # WINDOWS_NO_EXPORT
>  
> +
>  	mlx5_devx_cq_create;

There is an extra blank line here.

Re: [dpdk-dev] [PATCH v2 12/16] common/mlx5: share get ib device match function

[Matan Azrad]

Hi Thomas

From: Thomas Monjalon
> 29/04/2021 17:43, Matan Azrad:
> > From: Shiri Kuzin <shirik@nvidia.com>
> >
> > The get_ib_device_match function iterates over the list of ib devices
> > returned by the get_device_list glue function and returns the ib
> > device matching the provided address.
> >
> > Since this function is in use by several drivers, in this patch we
> > share the function in common part.
> >
> > Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
> > Acked-by: Matan Azrad <matan@nvidia.com>
> > ---
> > --- a/drivers/common/mlx5/linux/mlx5_common_os.c
> > +++ b/drivers/common/mlx5/linux/mlx5_common_os.c
> > +struct ibv_device *
> > +mlx5_os_get_ib_device_match(struct rte_pci_addr *addr)
> 
> It would be better as mlx5_get_pci_ibv_device() in
> drivers/common/mlx5/mlx5_common_pci.c
> Just need to forward declare struct ibv_device in
> drivers/common/mlx5/mlx5_common.h This is what is done internally for
> sub-functions.

struct ibv_device Is linux only, defined in Verbs library.
So, no need it in OS common place.


> > --- a/drivers/common/mlx5/mlx5_common.h
> > +++ b/drivers/common/mlx5/mlx5_common.h
> > @@ -18,6 +18,7 @@
> >
> >  #include "mlx5_prm.h"
> >  #include "mlx5_devx_cmds.h"
> > +#include "mlx5_common_os.h"
> >
> >  /* Reported driver name. */
> >  #define MLX5_PCI_DRIVER_NAME "mlx5_pci"
> > @@ -215,6 +216,7 @@ enum mlx5_class {
> >       MLX5_CLASS_VDPA = RTE_BIT64(1),
> >       MLX5_CLASS_REGEX = RTE_BIT64(2),
> >       MLX5_CLASS_COMPRESS = RTE_BIT64(3),
> > +     MLX5_CLASS_CRYPTO = RTE_BIT64(4),
> >  };
> 
> This change has nothing to do in this patch I think.
> 

Yes, will handle in v3.

> > --- a/drivers/common/mlx5/version.map
> > +++ b/drivers/common/mlx5/version.map
> > @@ -111,6 +111,7 @@ INTERNAL {
> >       mlx5_os_reg_mr;
> >       mlx5_os_umem_reg;
> >       mlx5_os_umem_dereg;
> > +     mlx5_os_get_ib_device_match; # WINDOWS_NO_EXPORT
> 
> Please keep alphabetical order.

Sure.

Re: [dpdk-dev] [PATCH v2 15/16] common/mlx5: support register write access

[Matan Azrad]

From: Thomas Monjalon
> 29/04/2021 17:43, Matan Azrad:
> > From: Dekel Peled <dekelp@nvidia.com>
> >
> > This patch adds support of write operation to NIC registers.
> >
> > Signed-off-by: Dekel Peled <dekelp@nvidia.com>
> > Acked-by: Matan Azrad <matan@nvidia.com>
> > ---
> > --- a/drivers/common/mlx5/version.map
> > +++ b/drivers/common/mlx5/version.map
> > @@ -13,8 +13,17 @@ INTERNAL {
> >       mlx5_dev_to_pci_addr; # WINDOWS_NO_EXPORT
> >
> >       mlx5_devx_cmd_alloc_pd;
> > +     mlx5_devx_alloc_uar;
> 
> Please keep alphabetical order.
> 
> > +
> >       mlx5_devx_cmd_create_cq;
> > +     mlx5_devx_cmd_create_credential_obj;
> > +     mlx5_devx_cmd_create_crypto_login_obj;
> > +     mlx5_devx_cmd_create_dek_obj;
> >       mlx5_devx_cmd_create_flex_parser;
> > +     mlx5_devx_cmd_create_flow_hit_aso_obj;
> > +     mlx5_devx_cmd_create_flow_meter_aso_obj;
> > +     mlx5_devx_cmd_create_geneve_tlv_option;
> > +     mlx5_devx_cmd_create_import_kek_obj;
> >       mlx5_devx_cmd_create_qp;
> >       mlx5_devx_cmd_create_rq;
> >       mlx5_devx_cmd_create_rqt;
> > @@ -24,13 +33,6 @@ INTERNAL {
> >       mlx5_devx_cmd_create_tis;
> >       mlx5_devx_cmd_create_virtio_q_counters; # WINDOWS_NO_EXPORT
> >       mlx5_devx_cmd_create_virtq;
> > -     mlx5_devx_cmd_create_flow_hit_aso_obj;
> > -     mlx5_devx_cmd_create_flow_meter_aso_obj;
> > -     mlx5_devx_cmd_create_geneve_tlv_option;
> > -        mlx5_devx_cmd_create_dek_obj;
> > -        mlx5_devx_cmd_create_import_kek_obj;
> > -        mlx5_devx_cmd_create_credential_obj;
> > -        mlx5_devx_cmd_create_crypto_login_obj;
> 
> Are they changes of previous patches?
> It seems some cleaning is required in previous patches.

Yes, this one tried to do the cleaning, I will handle it in the first patch (of this series) accessing this file. 

> >       mlx5_devx_cmd_destroy;
> >       mlx5_devx_cmd_flow_counter_alloc;
> >       mlx5_devx_cmd_flow_counter_query; @@ -51,12 +53,17 @@
> INTERNAL {
> >       mlx5_devx_cmd_queue_counter_alloc; # WINDOWS_NO_EXPORT
> >       mlx5_devx_cmd_queue_counter_query; # WINDOWS_NO_EXPORT
> >       mlx5_devx_cmd_register_read;
> > +     mlx5_devx_cmd_register_write;
> >       mlx5_devx_cmd_wq_query; # WINDOWS_NO_EXPORT
> >       mlx5_devx_get_out_command_status;
> >       mlx5_devx_alloc_uar; # WINDOWS_NO_EXPORT
> >
> > +
> >       mlx5_devx_cq_create;
> 
> There is an extra blank line here.
> 
> 

[dpdk-dev] [PATCH v3 00/15] mlx5 common part for crypto driver

[Matan Azrad]

The crypto PMD will be supported on Nvidia ConnectX6.
The crypto PMD will add the support of encryption and decryption using the AES-XTS symmetric algorithm.
The crypto PMD requires rdma-core and uses mlx5 DevX.
This is the mlx5 common part that added support for DevX commands needed for crypto driver.
Also includes PRM definitions.

v2:
Add data-path part.

v3:
Squash fix commit to the one before it.
Change get IB device function name.
Rearrange common version.map file.
Moved class definition to the driver seires.

Dekel Peled (12):
  common/mlx5: remove redundant spaces in header file
  common/mlx5: update GENEVE TLV OPT obj name
  common/mlx5: optimize read of general obj type caps
  common/mlx5: add HCA cap for AES-XTS crypto
  common/mlx5: support general object DEK create op
  common/mlx5: adjust DevX MKEY fields for crypto
  common/mlx5: support general obj IMPORT KEK create
  common/mlx5: support general obj CRYPTO LOGIN create
  common/mlx5: add crypto BSF struct and defines
  common/mlx5: support general obj CREDENTIAL create
  common/mlx5: add crypto register structs and defs
  common/mlx5: support register write access

Shiri Kuzin (2):
  common/mlx5: share hash list tool
  common/mlx5: share get ib device match function

Suanming Mou (1):
  common/mlx5: add UMR and RDMA write WQE defines

 drivers/common/mlx5/linux/mlx5_common_os.c   |  30 +-
 drivers/common/mlx5/linux/mlx5_common_os.h   |   5 +
 drivers/common/mlx5/linux/mlx5_nl.c          |   2 +-
 drivers/common/mlx5/meson.build              |  15 +-
 drivers/common/mlx5/mlx5_common.c            |   2 +-
 drivers/common/mlx5/mlx5_common.h            |   1 +
 drivers/common/mlx5/mlx5_common_devx.c       |   2 +-
 drivers/common/mlx5/mlx5_common_log.h        |  21 +
 drivers/common/mlx5/mlx5_common_mp.c         |   2 +-
 drivers/common/mlx5/mlx5_common_mr.c         |   2 +-
 drivers/common/mlx5/mlx5_common_pci.c        |   4 +-
 drivers/common/mlx5/mlx5_common_utils.c      | 221 +++++++
 drivers/common/mlx5/mlx5_common_utils.h      | 202 +++++-
 drivers/common/mlx5/mlx5_devx_cmds.c         | 312 +++++++++-
 drivers/common/mlx5/mlx5_devx_cmds.h         |  66 ++
 drivers/common/mlx5/mlx5_malloc.c            |   2 +-
 drivers/common/mlx5/mlx5_prm.h               | 610 +++++++++++++------
 drivers/common/mlx5/version.map              |  66 +-
 drivers/common/mlx5/windows/mlx5_common_os.c |   2 +-
 drivers/common/mlx5/windows/mlx5_glue.c      |   2 +-
 drivers/compress/mlx5/mlx5_compress.c        |  30 +-
 drivers/net/mlx5/mlx5_utils.c                | 209 -------
 drivers/net/mlx5/mlx5_utils.h                | 194 +-----
 drivers/regex/mlx5/mlx5_regex.c              |  30 +-
 drivers/vdpa/mlx5/mlx5_vdpa.c                |  32 +-
 25 files changed, 1337 insertions(+), 727 deletions(-)
 create mode 100644 drivers/common/mlx5/mlx5_common_log.h
 create mode 100644 drivers/common/mlx5/mlx5_common_utils.c

-- 
2.25.1

[dpdk-dev] [PATCH v3 01/15] common/mlx5: remove redundant spaces in header file

[Matan Azrad]

From: Dekel Peled <dekelp@nvidia.com>

File drivers/common/mlx5/mlx5_prm.h includes structs representing
data items as defined in PRM document.
Some of these structs were copied as-is from kernel file mlx5_ifc.h.
As result the structs are not all aligned with the same spacing.

This patch removes redundant spaces and new lines from several structs,
to align all structs in mlx5_prm.h to the same format.

Signed-off-by: Dekel Peled <dekelp@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_prm.h | 334 +++++++++++++++------------------
 1 file changed, 155 insertions(+), 179 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index efa5ae67bf..da1510ac1e 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -953,163 +953,139 @@ enum {
 
 /* Flow counters. */
 struct mlx5_ifc_alloc_flow_counter_out_bits {
-	u8         status[0x8];
-	u8         reserved_at_8[0x18];
-	u8         syndrome[0x20];
-	u8         flow_counter_id[0x20];
-	u8         reserved_at_60[0x20];
+	u8 status[0x8];
+	u8 reserved_at_8[0x18];
+	u8 syndrome[0x20];
+	u8 flow_counter_id[0x20];
+	u8 reserved_at_60[0x20];
 };
 
 struct mlx5_ifc_alloc_flow_counter_in_bits {
-	u8         opcode[0x10];
-	u8         reserved_at_10[0x10];
-	u8         reserved_at_20[0x10];
-	u8         op_mod[0x10];
-	u8         flow_counter_id[0x20];
-	u8         reserved_at_40[0x18];
-	u8         flow_counter_bulk[0x8];
+	u8 opcode[0x10];
+	u8 reserved_at_10[0x10];
+	u8 reserved_at_20[0x10];
+	u8 op_mod[0x10];
+	u8 flow_counter_id[0x20];
+	u8 reserved_at_40[0x18];
+	u8 flow_counter_bulk[0x8];
 };
 
 struct mlx5_ifc_dealloc_flow_counter_out_bits {
-	u8         status[0x8];
-	u8         reserved_at_8[0x18];
-	u8         syndrome[0x20];
-	u8         reserved_at_40[0x40];
+	u8 status[0x8];
+	u8 reserved_at_8[0x18];
+	u8 syndrome[0x20];
+	u8 reserved_at_40[0x40];
 };
 
 struct mlx5_ifc_dealloc_flow_counter_in_bits {
-	u8         opcode[0x10];
-	u8         reserved_at_10[0x10];
-	u8         reserved_at_20[0x10];
-	u8         op_mod[0x10];
-	u8         flow_counter_id[0x20];
-	u8         reserved_at_60[0x20];
+	u8 opcode[0x10];
+	u8 reserved_at_10[0x10];
+	u8 reserved_at_20[0x10];
+	u8 op_mod[0x10];
+	u8 flow_counter_id[0x20];
+	u8 reserved_at_60[0x20];
 };
 
 struct mlx5_ifc_traffic_counter_bits {
-	u8         packets[0x40];
-	u8         octets[0x40];
+	u8 packets[0x40];
+	u8 octets[0x40];
 };
 
 struct mlx5_ifc_query_flow_counter_out_bits {
-	u8         status[0x8];
-	u8         reserved_at_8[0x18];
-	u8         syndrome[0x20];
-	u8         reserved_at_40[0x40];
+	u8 status[0x8];
+	u8 reserved_at_8[0x18];
+	u8 syndrome[0x20];
+	u8 reserved_at_40[0x40];
 	struct mlx5_ifc_traffic_counter_bits flow_statistics[];
 };
 
 struct mlx5_ifc_query_flow_counter_in_bits {
-	u8         opcode[0x10];
-	u8         reserved_at_10[0x10];
-	u8         reserved_at_20[0x10];
-	u8         op_mod[0x10];
-	u8         reserved_at_40[0x20];
-	u8         mkey[0x20];
-	u8         address[0x40];
-	u8         clear[0x1];
-	u8         dump_to_memory[0x1];
-	u8         num_of_counters[0x1e];
-	u8         flow_counter_id[0x20];
+	u8 opcode[0x10];
+	u8 reserved_at_10[0x10];
+	u8 reserved_at_20[0x10];
+	u8 op_mod[0x10];
+	u8 reserved_at_40[0x20];
+	u8 mkey[0x20];
+	u8 address[0x40];
+	u8 clear[0x1];
+	u8 dump_to_memory[0x1];
+	u8 num_of_counters[0x1e];
+	u8 flow_counter_id[0x20];
 };
 
 #define MLX5_MAX_KLM_BYTE_COUNT 0x80000000u
 #define MLX5_MIN_KLM_FIXED_BUFFER_SIZE 0x1000u
 
-
 struct mlx5_ifc_klm_bits {
-	u8         byte_count[0x20];
-	u8         mkey[0x20];
-	u8         address[0x40];
+	u8 byte_count[0x20];
+	u8 mkey[0x20];
+	u8 address[0x40];
 };
 
 struct mlx5_ifc_mkc_bits {
-	u8         reserved_at_0[0x1];
-	u8         free[0x1];
-	u8         reserved_at_2[0x1];
-	u8         access_mode_4_2[0x3];
-	u8         reserved_at_6[0x7];
-	u8         relaxed_ordering_write[0x1];
-	u8         reserved_at_e[0x1];
-	u8         small_fence_on_rdma_read_response[0x1];
-	u8         umr_en[0x1];
-	u8         a[0x1];
-	u8         rw[0x1];
-	u8         rr[0x1];
-	u8         lw[0x1];
-	u8         lr[0x1];
-	u8         access_mode_1_0[0x2];
-	u8         reserved_at_18[0x8];
-
-	u8         qpn[0x18];
-	u8         mkey_7_0[0x8];
-
-	u8         reserved_at_40[0x20];
-
-	u8         length64[0x1];
-	u8         bsf_en[0x1];
-	u8         sync_umr[0x1];
-	u8         reserved_at_63[0x2];
-	u8         expected_sigerr_count[0x1];
-	u8         reserved_at_66[0x1];
-	u8         en_rinval[0x1];
-	u8         pd[0x18];
-
-	u8         start_addr[0x40];
-
-	u8         len[0x40];
-
-	u8         bsf_octword_size[0x20];
-
-	u8         reserved_at_120[0x80];
-
-	u8         translations_octword_size[0x20];
-
-	u8         reserved_at_1c0[0x19];
-	u8		   relaxed_ordering_read[0x1];
-	u8		   reserved_at_1da[0x1];
-	u8         log_page_size[0x5];
-
-	u8         reserved_at_1e0[0x20];
+	u8 reserved_at_0[0x1];
+	u8 free[0x1];
+	u8 reserved_at_2[0x1];
+	u8 access_mode_4_2[0x3];
+	u8 reserved_at_6[0x7];
+	u8 relaxed_ordering_write[0x1];
+	u8 reserved_at_e[0x1];
+	u8 small_fence_on_rdma_read_response[0x1];
+	u8 umr_en[0x1];
+	u8 a[0x1];
+	u8 rw[0x1];
+	u8 rr[0x1];
+	u8 lw[0x1];
+	u8 lr[0x1];
+	u8 access_mode_1_0[0x2];
+	u8 reserved_at_18[0x8];
+	u8 qpn[0x18];
+	u8 mkey_7_0[0x8];
+	u8 reserved_at_40[0x20];
+	u8 length64[0x1];
+	u8 bsf_en[0x1];
+	u8 sync_umr[0x1];
+	u8 reserved_at_63[0x2];
+	u8 expected_sigerr_count[0x1];
+	u8 reserved_at_66[0x1];
+	u8 en_rinval[0x1];
+	u8 pd[0x18];
+	u8 start_addr[0x40];
+	u8 len[0x40];
+	u8 bsf_octword_size[0x20];
+	u8 reserved_at_120[0x80];
+	u8 translations_octword_size[0x20];
+	u8 reserved_at_1c0[0x19];
+	u8 relaxed_ordering_read[0x1];
+	u8 reserved_at_1da[0x1];
+	u8 log_page_size[0x5];
+	u8 reserved_at_1e0[0x20];
 };
 
 struct mlx5_ifc_create_mkey_out_bits {
-	u8         status[0x8];
-	u8         reserved_at_8[0x18];
-
-	u8         syndrome[0x20];
-
-	u8         reserved_at_40[0x8];
-	u8         mkey_index[0x18];
-
-	u8         reserved_at_60[0x20];
+	u8 status[0x8];
+	u8 reserved_at_8[0x18];
+	u8 syndrome[0x20];
+	u8 reserved_at_40[0x8];
+	u8 mkey_index[0x18];
+	u8 reserved_at_60[0x20];
 };
 
 struct mlx5_ifc_create_mkey_in_bits {
-	u8         opcode[0x10];
-	u8         reserved_at_10[0x10];
-
-	u8         reserved_at_20[0x10];
-	u8         op_mod[0x10];
-
-	u8         reserved_at_40[0x20];
-
-	u8         pg_access[0x1];
-	u8         reserved_at_61[0x1f];
-
+	u8 opcode[0x10];
+	u8 reserved_at_10[0x10];
+	u8 reserved_at_20[0x10];
+	u8 op_mod[0x10];
+	u8 reserved_at_40[0x20];
+	u8 pg_access[0x1];
+	u8 reserved_at_61[0x1f];
 	struct mlx5_ifc_mkc_bits memory_key_mkey_entry;
-
-	u8         reserved_at_280[0x80];
-
-	u8         translations_octword_actual_size[0x20];
-
-	u8         mkey_umem_id[0x20];
-
-	u8         mkey_umem_offset[0x40];
-
-	u8         reserved_at_380[0x500];
-
-	u8         klm_pas_mtt[][0x20];
+	u8 reserved_at_280[0x80];
+	u8 translations_octword_actual_size[0x20];
+	u8 mkey_umem_id[0x20];
+	u8 mkey_umem_offset[0x40];
+	u8 reserved_at_380[0x500];
+	u8 klm_pas_mtt[][0x20];
 };
 
 enum {
@@ -2272,27 +2248,27 @@ enum {
 };
 
 struct mlx5_ifc_flow_meter_parameters_bits {
-	u8         valid[0x1];			// 00h
-	u8         bucket_overflow[0x1];
-	u8         start_color[0x2];
-	u8         both_buckets_on_green[0x1];
-	u8         meter_mode[0x2];
-	u8         reserved_at_1[0x19];
-	u8         reserved_at_2[0x20]; //04h
-	u8         reserved_at_3[0x3];
-	u8         cbs_exponent[0x5];		// 08h
-	u8         cbs_mantissa[0x8];
-	u8         reserved_at_4[0x3];
-	u8         cir_exponent[0x5];
-	u8         cir_mantissa[0x8];
-	u8         reserved_at_5[0x20];		// 0Ch
-	u8         reserved_at_6[0x3];
-	u8         ebs_exponent[0x5];		// 10h
-	u8         ebs_mantissa[0x8];
-	u8         reserved_at_7[0x3];
-	u8         eir_exponent[0x5];
-	u8         eir_mantissa[0x8];
-	u8         reserved_at_8[0x60];		// 14h-1Ch
+	u8 valid[0x1];
+	u8 bucket_overflow[0x1];
+	u8 start_color[0x2];
+	u8 both_buckets_on_green[0x1];
+	u8 meter_mode[0x2];
+	u8 reserved_at_1[0x19];
+	u8 reserved_at_2[0x20];
+	u8 reserved_at_3[0x3];
+	u8 cbs_exponent[0x5];
+	u8 cbs_mantissa[0x8];
+	u8 reserved_at_4[0x3];
+	u8 cir_exponent[0x5];
+	u8 cir_mantissa[0x8];
+	u8 reserved_at_5[0x20];
+	u8 reserved_at_6[0x3];
+	u8 ebs_exponent[0x5];
+	u8 ebs_mantissa[0x8];
+	u8 reserved_at_7[0x3];
+	u8 eir_exponent[0x5];
+	u8 eir_mantissa[0x8];
+	u8 reserved_at_8[0x60];
 };
 #define MLX5_IFC_FLOW_METER_PARAM_MASK UINT64_C(0x80FFFFFF)
 #define MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL 0x14BF00C8
@@ -2359,46 +2335,46 @@ struct mlx5_ifc_cqc_bits {
 };
 
 struct mlx5_ifc_health_buffer_bits {
-	u8         reserved_0[0x100];
-	u8         assert_existptr[0x20];
-	u8         assert_callra[0x20];
-	u8         reserved_1[0x40];
-	u8         fw_version[0x20];
-	u8         hw_id[0x20];
-	u8         reserved_2[0x20];
-	u8         irisc_index[0x8];
-	u8         synd[0x8];
-	u8         ext_synd[0x10];
+	u8 reserved_0[0x100];
+	u8 assert_existptr[0x20];
+	u8 assert_callra[0x20];
+	u8 reserved_1[0x40];
+	u8 fw_version[0x20];
+	u8 hw_id[0x20];
+	u8 reserved_2[0x20];
+	u8 irisc_index[0x8];
+	u8 synd[0x8];
+	u8 ext_synd[0x10];
 };
 
 struct mlx5_ifc_initial_seg_bits {
-	u8         fw_rev_minor[0x10];
-	u8         fw_rev_major[0x10];
-	u8         cmd_interface_rev[0x10];
-	u8         fw_rev_subminor[0x10];
-	u8         reserved_0[0x40];
-	u8         cmdq_phy_addr_63_32[0x20];
-	u8         cmdq_phy_addr_31_12[0x14];
-	u8         reserved_1[0x2];
-	u8         nic_interface[0x2];
-	u8         log_cmdq_size[0x4];
-	u8         log_cmdq_stride[0x4];
-	u8         command_doorbell_vector[0x20];
-	u8         reserved_2[0xf00];
-	u8         initializing[0x1];
-	u8         nic_interface_supported[0x7];
-	u8         reserved_4[0x18];
+	u8 fw_rev_minor[0x10];
+	u8 fw_rev_major[0x10];
+	u8 cmd_interface_rev[0x10];
+	u8 fw_rev_subminor[0x10];
+	u8 reserved_0[0x40];
+	u8 cmdq_phy_addr_63_32[0x20];
+	u8 cmdq_phy_addr_31_12[0x14];
+	u8 reserved_1[0x2];
+	u8 nic_interface[0x2];
+	u8 log_cmdq_size[0x4];
+	u8 log_cmdq_stride[0x4];
+	u8 command_doorbell_vector[0x20];
+	u8 reserved_2[0xf00];
+	u8 initializing[0x1];
+	u8 nic_interface_supported[0x7];
+	u8 reserved_4[0x18];
 	struct mlx5_ifc_health_buffer_bits health_buffer;
-	u8         no_dram_nic_offset[0x20];
-	u8         reserved_5[0x6de0];
-	u8         internal_timer_h[0x20];
-	u8         internal_timer_l[0x20];
-	u8         reserved_6[0x20];
-	u8         reserved_7[0x1f];
-	u8         clear_int[0x1];
-	u8         health_syndrome[0x8];
-	u8         health_counter[0x18];
-	u8         reserved_8[0x17fc0];
+	u8 no_dram_nic_offset[0x20];
+	u8 reserved_5[0x6de0];
+	u8 internal_timer_h[0x20];
+	u8 internal_timer_l[0x20];
+	u8 reserved_6[0x20];
+	u8 reserved_7[0x1f];
+	u8 clear_int[0x1];
+	u8 health_syndrome[0x8];
+	u8 health_counter[0x18];
+	u8 reserved_8[0x17fc0];
 };
 
 struct mlx5_ifc_create_cq_out_bits {
-- 
2.25.1

[dpdk-dev] [PATCH v3 02/15] common/mlx5: update GENEVE TLV OPT obj name

[Matan Azrad]

From: Dekel Peled <dekelp@nvidia.com>

Rename MLX5_OBJ_TYPE_GENEVE_TLV_OPT as
MLX5_GENERAL_OBJ_TYPE_GENEVE_TLV_OPT, to align with other general
objects names.

Signed-off-by: Dekel Peled <dekelp@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_devx_cmds.c | 2 +-
 drivers/common/mlx5/mlx5_prm.h       | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c
index 79fff6457c..831175efc5 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.c
+++ b/drivers/common/mlx5/mlx5_devx_cmds.c
@@ -2265,7 +2265,7 @@ mlx5_devx_cmd_create_geneve_tlv_option(void *ctx,
 	MLX5_SET(general_obj_in_cmd_hdr, hdr, opcode,
 			MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
 	MLX5_SET(general_obj_in_cmd_hdr, hdr, obj_type,
-			MLX5_OBJ_TYPE_GENEVE_TLV_OPT);
+		 MLX5_GENERAL_OBJ_TYPE_GENEVE_TLV_OPT);
 	MLX5_SET(geneve_tlv_option, opt, option_class,
 			rte_be_to_cpu_16(class));
 	MLX5_SET(geneve_tlv_option, opt, option_type, type);
diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index da1510ac1e..2e5e42f6e9 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -1108,7 +1108,7 @@ enum {
 #define MLX5_GENERAL_OBJ_TYPES_CAP_FLOW_METER_ASO \
 			(1ULL << MLX5_GENERAL_OBJ_TYPE_FLOW_METER_ASO)
 #define MLX5_GENERAL_OBJ_TYPES_CAP_GENEVE_TLV_OPT \
-			(1ULL << MLX5_OBJ_TYPE_GENEVE_TLV_OPT)
+			(1ULL << MLX5_GENERAL_OBJ_TYPE_GENEVE_TLV_OPT)
 
 enum {
 	MLX5_HCA_CAP_OPMOD_GET_MAX   = 0,
@@ -2402,7 +2402,7 @@ struct mlx5_ifc_create_cq_in_bits {
 };
 
 enum {
-	MLX5_OBJ_TYPE_GENEVE_TLV_OPT = 0x000b,
+	MLX5_GENERAL_OBJ_TYPE_GENEVE_TLV_OPT = 0x000b,
 	MLX5_GENERAL_OBJ_TYPE_VIRTQ = 0x000d,
 	MLX5_GENERAL_OBJ_TYPE_VIRTIO_Q_COUNTERS = 0x001c,
 	MLX5_GENERAL_OBJ_TYPE_FLEX_PARSE_GRAPH = 0x0022,
-- 
2.25.1

[dpdk-dev] [PATCH v3 03/15] common/mlx5: optimize read of general obj type caps

[Matan Azrad]

From: Dekel Peled <dekelp@nvidia.com>

General object types support is indicated in bitmap general_obj_types,
which is part of HCA capabilities list.
Currently this bitmap is read multiple times, and each time a different
bit is extracted.

This patch optimizes the code, reading the bitmap once into a local
variable, and then extracting the required bits.

Signed-off-by: Dekel Peled <dekelp@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_devx_cmds.c | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c
index 831175efc5..a0bf0d3009 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.c
+++ b/drivers/common/mlx5/mlx5_devx_cmds.c
@@ -647,6 +647,7 @@ mlx5_devx_cmd_query_hca_attr(void *ctx,
 	uint32_t out[MLX5_ST_SZ_DW(query_hca_cap_out)] = {0};
 	void *hcattr;
 	int status, syndrome, rc, i;
+	uint64_t general_obj_types_supported = 0;
 
 	MLX5_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP);
 	MLX5_SET(query_hca_cap_in, in, op_mod,
@@ -725,12 +726,22 @@ mlx5_devx_cmd_query_hca_attr(void *ctx,
 	attr->regex = MLX5_GET(cmd_hca_cap, hcattr, regexp);
 	attr->regexp_num_of_engines = MLX5_GET(cmd_hca_cap, hcattr,
 					       regexp_num_of_engines);
-	attr->flow_hit_aso = !!(MLX5_GET64(cmd_hca_cap, hcattr,
-					   general_obj_types) &
+	/* Read the general_obj_types bitmap and extract the relevant bits. */
+	general_obj_types_supported = MLX5_GET64(cmd_hca_cap, hcattr,
+						 general_obj_types);
+	attr->vdpa.valid = !!(general_obj_types_supported &
+			      MLX5_GENERAL_OBJ_TYPES_CAP_VIRTQ_NET_Q);
+	attr->vdpa.queue_counters_valid =
+			!!(general_obj_types_supported &
+			   MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_Q_COUNTERS);
+	attr->parse_graph_flex_node =
+			!!(general_obj_types_supported &
+			   MLX5_GENERAL_OBJ_TYPES_CAP_PARSE_GRAPH_FLEX_NODE);
+	attr->flow_hit_aso = !!(general_obj_types_supported &
 				MLX5_GENERAL_OBJ_TYPES_CAP_FLOW_HIT_ASO);
-	attr->geneve_tlv_opt = !!(MLX5_GET64(cmd_hca_cap, hcattr,
-					   general_obj_types) &
-				MLX5_GENERAL_OBJ_TYPES_CAP_GENEVE_TLV_OPT);
+	attr->geneve_tlv_opt = !!(general_obj_types_supported &
+				  MLX5_GENERAL_OBJ_TYPES_CAP_GENEVE_TLV_OPT);
+	/* Add reading of other GENERAL_OBJ_TYPES_CAP bits above this line. */
 	attr->log_max_cq = MLX5_GET(cmd_hca_cap, hcattr, log_max_cq);
 	attr->log_max_qp = MLX5_GET(cmd_hca_cap, hcattr, log_max_qp);
 	attr->log_max_cq_sz = MLX5_GET(cmd_hca_cap, hcattr, log_max_cq_sz);
-- 
2.25.1

[dpdk-dev] [PATCH v3 04/15] common/mlx5: add HCA cap for AES-XTS crypto

[Matan Azrad]

From: Dekel Peled <dekelp@nvidia.com>

Update the PRM structure and HCA capabilities reading, to include
relevant capabilities for AES-XTS crypto.

Signed-off-by: Dekel Peled <dekelp@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_devx_cmds.c | 3 +++
 drivers/common/mlx5/mlx5_devx_cmds.h | 2 ++
 drivers/common/mlx5/mlx5_prm.h       | 5 ++++-
 3 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c
index a0bf0d3009..7ca767944e 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.c
+++ b/drivers/common/mlx5/mlx5_devx_cmds.c
@@ -771,6 +771,9 @@ mlx5_devx_cmd_query_hca_attr(void *ctx,
 		MLX5_GET(cmd_hca_cap, hcattr, umr_indirect_mkey_disabled);
 	attr->umr_modify_entity_size_disabled =
 		MLX5_GET(cmd_hca_cap, hcattr, umr_modify_entity_size_disabled);
+	attr->crypto = MLX5_GET(cmd_hca_cap, hcattr, crypto);
+	if (attr->crypto)
+		attr->aes_xts = MLX5_GET(cmd_hca_cap, hcattr, aes_xts);
 	if (attr->qos.sup) {
 		MLX5_SET(query_hca_cap_in, in, op_mod,
 			 MLX5_GET_HCA_CAP_OP_MOD_QOS_CAP |
diff --git a/drivers/common/mlx5/mlx5_devx_cmds.h b/drivers/common/mlx5/mlx5_devx_cmds.h
index 870bdb6b30..28ade5bbc4 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.h
+++ b/drivers/common/mlx5/mlx5_devx_cmds.h
@@ -137,6 +137,8 @@ struct mlx5_hca_attr {
 	uint32_t qp_ts_format:2;
 	uint32_t regex:1;
 	uint32_t reg_c_preserve:1;
+	uint32_t crypto:1; /* Crypto engine is supported. */
+	uint32_t aes_xts:1; /* AES-XTS crypto is supported. */
 	uint32_t regexp_num_of_engines;
 	uint32_t log_max_ft_sampler_num:8;
 	uint32_t geneve_tlv_opt;
diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index 2e5e42f6e9..a8fbfbb0f5 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -1427,7 +1427,10 @@ struct mlx5_ifc_cmd_hca_cap_bits {
 	u8 sq_ts_format[0x2];
 	u8 rq_ts_format[0x2];
 	u8 reserved_at_444[0x1C];
-	u8 reserved_at_460[0x10];
+	u8 reserved_at_460[0x8];
+	u8 aes_xts[0x1];
+	u8 crypto[0x1];
+	u8 reserved_at_46a[0x6];
 	u8 max_num_eqs[0x10];
 	u8 reserved_at_480[0x3];
 	u8 log_max_l2_table[0x5];
-- 
2.25.1

[dpdk-dev] [PATCH v3 05/15] common/mlx5: support general object DEK create op

[Matan Azrad]

From: Dekel Peled <dekelp@nvidia.com>

Data Encryption Keys (DEKs) are the keys used for data
encryption/decryption operations.

Add reading of DEK support capability.
Add function to create general object type DEK, using DevX API.

Arrange common version.map file in alphabetical order.

Signed-off-by: Dekel Peled <dekelp@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_devx_cmds.c | 53 +++++++++++++++++++++++++++
 drivers/common/mlx5/mlx5_devx_cmds.h | 17 +++++++++
 drivers/common/mlx5/mlx5_prm.h       | 39 ++++++++++++++++++++
 drivers/common/mlx5/version.map      | 55 ++++++++++++++++------------
 4 files changed, 140 insertions(+), 24 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c
index 7ca767944e..742c82cca4 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.c
+++ b/drivers/common/mlx5/mlx5_devx_cmds.c
@@ -741,6 +741,8 @@ mlx5_devx_cmd_query_hca_attr(void *ctx,
 				MLX5_GENERAL_OBJ_TYPES_CAP_FLOW_HIT_ASO);
 	attr->geneve_tlv_opt = !!(general_obj_types_supported &
 				  MLX5_GENERAL_OBJ_TYPES_CAP_GENEVE_TLV_OPT);
+	attr->dek = !!(general_obj_types_supported &
+		       MLX5_GENERAL_OBJ_TYPES_CAP_DEK);
 	/* Add reading of other GENERAL_OBJ_TYPES_CAP bits above this line. */
 	attr->log_max_cq = MLX5_GET(cmd_hca_cap, hcattr, log_max_cq);
 	attr->log_max_qp = MLX5_GET(cmd_hca_cap, hcattr, log_max_qp);
@@ -2397,3 +2399,54 @@ mlx5_devx_cmd_queue_counter_query(struct mlx5_devx_obj *dcs, int clear,
 	*out_of_buffers = MLX5_GET(query_q_counter_out, out, out_of_buffer);
 	return 0;
 }
+
+/**
+ * Create general object of type DEK using DevX API.
+ *
+ * @param[in] ctx
+ *   Context returned from mlx5 open_device() glue function.
+ * @param [in] attr
+ *   Pointer to DEK attributes structure.
+ *
+ * @return
+ *   The DevX object created, NULL otherwise and rte_errno is set.
+ */
+struct mlx5_devx_obj *
+mlx5_devx_cmd_create_dek_obj(void *ctx, struct mlx5_devx_dek_attr *attr)
+{
+	uint32_t in[MLX5_ST_SZ_DW(create_dek_in)] = {0};
+	uint32_t out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};
+	struct mlx5_devx_obj *dek_obj = NULL;
+	void *ptr = NULL, *key_addr = NULL;
+
+	dek_obj = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*dek_obj),
+			      0, SOCKET_ID_ANY);
+	if (dek_obj == NULL) {
+		DRV_LOG(ERR, "Failed to allocate DEK object data");
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+	ptr = MLX5_ADDR_OF(create_dek_in, in, hdr);
+	MLX5_SET(general_obj_in_cmd_hdr, ptr, opcode,
+		 MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
+	MLX5_SET(general_obj_in_cmd_hdr, ptr, obj_type,
+		 MLX5_GENERAL_OBJ_TYPE_DEK);
+	ptr = MLX5_ADDR_OF(create_dek_in, in, dek);
+	MLX5_SET(dek, ptr, key_size, attr->key_size);
+	MLX5_SET(dek, ptr, has_keytag, attr->has_keytag);
+	MLX5_SET(dek, ptr, key_purpose, attr->key_purpose);
+	MLX5_SET(dek, ptr, pd, attr->pd);
+	MLX5_SET64(dek, ptr, opaque, attr->opaque);
+	key_addr = MLX5_ADDR_OF(dek, ptr, key);
+	memcpy(key_addr, (void *)(attr->key), MLX5_CRYPTO_KEY_MAX_SIZE);
+	dek_obj->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in),
+						  out, sizeof(out));
+	if (dek_obj->obj == NULL) {
+		rte_errno = errno;
+		DRV_LOG(ERR, "Failed to create DEK obj using DevX.");
+		mlx5_free(dek_obj);
+		return NULL;
+	}
+	dek_obj->id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
+	return dek_obj;
+}
diff --git a/drivers/common/mlx5/mlx5_devx_cmds.h b/drivers/common/mlx5/mlx5_devx_cmds.h
index 28ade5bbc4..b9ff7ab87d 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.h
+++ b/drivers/common/mlx5/mlx5_devx_cmds.h
@@ -139,6 +139,7 @@ struct mlx5_hca_attr {
 	uint32_t reg_c_preserve:1;
 	uint32_t crypto:1; /* Crypto engine is supported. */
 	uint32_t aes_xts:1; /* AES-XTS crypto is supported. */
+	uint32_t dek:1; /* General obj type DEK is supported. */
 	uint32_t regexp_num_of_engines;
 	uint32_t log_max_ft_sampler_num:8;
 	uint32_t geneve_tlv_opt;
@@ -435,6 +436,18 @@ struct mlx5_devx_graph_node_attr {
 	struct mlx5_devx_graph_arc_attr out[MLX5_GRAPH_NODE_ARC_NUM];
 };
 
+/* Encryption key size is up to 1024 bit, 128 bytes. */
+#define MLX5_CRYPTO_KEY_MAX_SIZE	128
+
+struct mlx5_devx_dek_attr {
+	uint32_t key_size:4;
+	uint32_t has_keytag:1;
+	uint32_t key_purpose:4;
+	uint32_t pd:24;
+	uint64_t opaque;
+	uint8_t key[MLX5_CRYPTO_KEY_MAX_SIZE];
+};
+
 /* mlx5_devx_cmds.c */
 
 __rte_internal
@@ -587,4 +600,8 @@ int mlx5_devx_cmd_queue_counter_query(struct mlx5_devx_obj *dcs, int clear,
 __rte_internal
 struct mlx5_devx_obj *mlx5_devx_cmd_create_flow_meter_aso_obj(void *ctx,
 					uint32_t pd, uint32_t log_obj_size);
+__rte_internal
+struct mlx5_devx_obj *
+mlx5_devx_cmd_create_dek_obj(void *ctx, struct mlx5_devx_dek_attr *attr);
+
 #endif /* RTE_PMD_MLX5_DEVX_CMDS_H_ */
diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index a8fbfbb0f5..bc9f58ad03 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -1109,6 +1109,8 @@ enum {
 			(1ULL << MLX5_GENERAL_OBJ_TYPE_FLOW_METER_ASO)
 #define MLX5_GENERAL_OBJ_TYPES_CAP_GENEVE_TLV_OPT \
 			(1ULL << MLX5_GENERAL_OBJ_TYPE_GENEVE_TLV_OPT)
+#define MLX5_GENERAL_OBJ_TYPES_CAP_DEK \
+			(1ULL << MLX5_GENERAL_OBJ_TYPE_DEK)
 
 enum {
 	MLX5_HCA_CAP_OPMOD_GET_MAX   = 0,
@@ -2406,6 +2408,7 @@ struct mlx5_ifc_create_cq_in_bits {
 
 enum {
 	MLX5_GENERAL_OBJ_TYPE_GENEVE_TLV_OPT = 0x000b,
+	MLX5_GENERAL_OBJ_TYPE_DEK = 0x000c,
 	MLX5_GENERAL_OBJ_TYPE_VIRTQ = 0x000d,
 	MLX5_GENERAL_OBJ_TYPE_VIRTIO_Q_COUNTERS = 0x001c,
 	MLX5_GENERAL_OBJ_TYPE_FLEX_PARSE_GRAPH = 0x0022,
@@ -2469,6 +2472,42 @@ struct mlx5_ifc_create_geneve_tlv_option_in_bits {
 	struct mlx5_ifc_geneve_tlv_option_bits geneve_tlv_opt;
 };
 
+enum {
+	MLX5_CRYPTO_KEY_SIZE_128b = 0x0,
+	MLX5_CRYPTO_KEY_SIZE_256b = 0x1,
+};
+
+enum {
+	MLX5_CRYPTO_KEY_PURPOSE_TLS	= 0x1,
+	MLX5_CRYPTO_KEY_PURPOSE_IPSEC	= 0x2,
+	MLX5_CRYPTO_KEY_PURPOSE_AES_XTS	= 0x3,
+	MLX5_CRYPTO_KEY_PURPOSE_MACSEC	= 0x4,
+	MLX5_CRYPTO_KEY_PURPOSE_GCM	= 0x5,
+	MLX5_CRYPTO_KEY_PURPOSE_PSP	= 0x6,
+};
+
+struct mlx5_ifc_dek_bits {
+	u8 modify_field_select[0x40];
+	u8 state[0x8];
+	u8 reserved_at_48[0xc];
+	u8 key_size[0x4];
+	u8 has_keytag[0x1];
+	u8 reserved_at_59[0x3];
+	u8 key_purpose[0x4];
+	u8 reserved_at_60[0x8];
+	u8 pd[0x18];
+	u8 reserved_at_80[0x100];
+	u8 opaque[0x40];
+	u8 reserved_at_1c0[0x40];
+	u8 key[0x400];
+	u8 reserved_at_600[0x200];
+};
+
+struct mlx5_ifc_create_dek_in_bits {
+	struct mlx5_ifc_general_obj_in_cmd_hdr_bits hdr;
+	struct mlx5_ifc_dek_bits dek;
+};
+
 enum {
 	MLX5_VIRTQ_STATE_INIT = 0,
 	MLX5_VIRTQ_STATE_RDY = 1,
diff --git a/drivers/common/mlx5/version.map b/drivers/common/mlx5/version.map
index 18dc96276d..42bb985fb1 100644
--- a/drivers/common/mlx5/version.map
+++ b/drivers/common/mlx5/version.map
@@ -12,21 +12,24 @@ INTERNAL {
 
 	mlx5_dev_to_pci_addr; # WINDOWS_NO_EXPORT
 
+	mlx5_devx_alloc_uar; # WINDOWS_NO_EXPORT
+
 	mlx5_devx_cmd_alloc_pd;
 	mlx5_devx_cmd_create_cq;
+	mlx5_devx_cmd_create_dek_obj;
 	mlx5_devx_cmd_create_flex_parser;
+	mlx5_devx_cmd_create_flow_hit_aso_obj;
+	mlx5_devx_cmd_create_flow_meter_aso_obj;
+	mlx5_devx_cmd_create_geneve_tlv_option;
 	mlx5_devx_cmd_create_qp;
 	mlx5_devx_cmd_create_rq;
 	mlx5_devx_cmd_create_rqt;
 	mlx5_devx_cmd_create_sq;
-	mlx5_devx_cmd_create_tir;
 	mlx5_devx_cmd_create_td;
+	mlx5_devx_cmd_create_tir;
 	mlx5_devx_cmd_create_tis;
 	mlx5_devx_cmd_create_virtio_q_counters; # WINDOWS_NO_EXPORT
 	mlx5_devx_cmd_create_virtq;
-	mlx5_devx_cmd_create_flow_hit_aso_obj;
-	mlx5_devx_cmd_create_flow_meter_aso_obj;
-	mlx5_devx_cmd_create_geneve_tlv_option;
 	mlx5_devx_cmd_destroy;
 	mlx5_devx_cmd_flow_counter_alloc;
 	mlx5_devx_cmd_flow_counter_query;
@@ -48,41 +51,49 @@ INTERNAL {
 	mlx5_devx_cmd_queue_counter_query; # WINDOWS_NO_EXPORT
 	mlx5_devx_cmd_register_read;
 	mlx5_devx_cmd_wq_query; # WINDOWS_NO_EXPORT
-	mlx5_devx_get_out_command_status;
-	mlx5_devx_alloc_uar; # WINDOWS_NO_EXPORT
 
 	mlx5_devx_cq_create;
 	mlx5_devx_cq_destroy;
+
+	mlx5_devx_get_out_command_status;
+
 	mlx5_devx_rq_create;
 	mlx5_devx_rq_destroy;
 	mlx5_devx_sq_create;
 	mlx5_devx_sq_destroy;
 
+	mlx5_free;
+
 	mlx5_get_ifname_sysfs; # WINDOWS_NO_EXPORT
 
 	mlx5_glue;
 
+	mlx5_malloc;
+	mlx5_malloc_mem_select;
+
+	mlx5_memory_stat_dump; # WINDOWS_NO_EXPORT
+
 	mlx5_mp_init_primary; # WINDOWS_NO_EXPORT
-	mlx5_mp_uninit_primary; # WINDOWS_NO_EXPORT
 	mlx5_mp_init_secondary; # WINDOWS_NO_EXPORT
-	mlx5_mp_uninit_secondary; # WINDOWS_NO_EXPORT
 	mlx5_mp_req_mr_create; # WINDOWS_NO_EXPORT
 	mlx5_mp_req_queue_state_modify;
 	mlx5_mp_req_verbs_cmd_fd; # WINDOWS_NO_EXPORT
+	mlx5_mp_uninit_primary; # WINDOWS_NO_EXPORT
+	mlx5_mp_uninit_secondary; # WINDOWS_NO_EXPORT
 
-	mlx5_mr_btree_init;
-	mlx5_mr_btree_free;
-	mlx5_mr_btree_dump;
 	mlx5_mr_addr2mr_bh;
-	mlx5_mr_release_cache;
+	mlx5_mr_btree_dump;
+	mlx5_mr_btree_free;
+	mlx5_mr_btree_init;
+	mlx5_mr_create_primary;
 	mlx5_mr_dump_cache;
-	mlx5_mr_rebuild_cache;
+	mlx5_mr_flush_local_cache;
+	mlx5_mr_free;
 	mlx5_mr_insert_cache;
 	mlx5_mr_lookup_cache;
 	mlx5_mr_lookup_list;
-	mlx5_mr_create_primary;
-	mlx5_mr_flush_local_cache;
-	mlx5_mr_free;
+	mlx5_mr_rebuild_cache;
+	mlx5_mr_release_cache;
 
 	mlx5_nl_allmulti; # WINDOWS_NO_EXPORT
 	mlx5_nl_devlink_family_id_get; # WINDOWS_NO_EXPORT
@@ -102,20 +113,16 @@ INTERNAL {
 	mlx5_nl_vlan_vmwa_create; # WINDOWS_NO_EXPORT
 	mlx5_nl_vlan_vmwa_delete; # WINDOWS_NO_EXPORT
 
+	mlx5_pci_driver_register;
+
 	mlx5_os_alloc_pd;
 	mlx5_os_dealloc_pd;
 	mlx5_os_dereg_mr;
 	mlx5_os_reg_mr;
-	mlx5_os_umem_reg;
 	mlx5_os_umem_dereg;
+	mlx5_os_umem_reg;
 
-	mlx5_translate_port_name; # WINDOWS_NO_EXPORT
-
-	mlx5_malloc_mem_select;
-	mlx5_memory_stat_dump; # WINDOWS_NO_EXPORT
-	mlx5_malloc;
 	mlx5_realloc;
-	mlx5_free;
 
-	mlx5_pci_driver_register;
+	mlx5_translate_port_name; # WINDOWS_NO_EXPORT
 };
-- 
2.25.1

[dpdk-dev] [PATCH v3 06/15] common/mlx5: adjust DevX MKEY fields for crypto

[Matan Azrad]

From: Dekel Peled <dekelp@nvidia.com>

MKEY that will be used for crypto purposes must be created with
crypto_en and remote access attributes.
This patch adds support for them in the DevX MKEY context.

Signed-off-by: Dekel Peled <dekelp@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_devx_cmds.c |  9 +++++++++
 drivers/common/mlx5/mlx5_devx_cmds.h |  2 ++
 drivers/common/mlx5/mlx5_prm.h       | 10 +++++++++-
 3 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c
index 742c82cca4..7e3c8b55fa 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.c
+++ b/drivers/common/mlx5/mlx5_devx_cmds.c
@@ -263,6 +263,10 @@ mlx5_devx_cmd_mkey_create(void *ctx,
 	MLX5_SET(create_mkey_in, in, pg_access, attr->pg_access);
 	MLX5_SET(mkc, mkc, lw, 0x1);
 	MLX5_SET(mkc, mkc, lr, 0x1);
+	if (attr->set_remote_rw) {
+		MLX5_SET(mkc, mkc, rw, 0x1);
+		MLX5_SET(mkc, mkc, rr, 0x1);
+	}
 	MLX5_SET(mkc, mkc, qpn, 0xffffff);
 	MLX5_SET(mkc, mkc, pd, attr->pd);
 	MLX5_SET(mkc, mkc, mkey_7_0, attr->umem_id & 0xFF);
@@ -273,6 +277,11 @@ mlx5_devx_cmd_mkey_create(void *ctx,
 	MLX5_SET(mkc, mkc, relaxed_ordering_read, attr->relaxed_ordering_read);
 	MLX5_SET64(mkc, mkc, start_addr, attr->addr);
 	MLX5_SET64(mkc, mkc, len, attr->size);
+	MLX5_SET(mkc, mkc, crypto_en, attr->crypto_en);
+	if (attr->crypto_en) {
+		MLX5_SET(mkc, mkc, bsf_en, attr->crypto_en);
+		MLX5_SET(mkc, mkc, bsf_octword_size, 4);
+	}
 	mkey->obj = mlx5_glue->devx_obj_create(ctx, in, in_size_dw * 4, out,
 					       sizeof(out));
 	if (!mkey->obj) {
diff --git a/drivers/common/mlx5/mlx5_devx_cmds.h b/drivers/common/mlx5/mlx5_devx_cmds.h
index b9ff7ab87d..600577f18a 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.h
+++ b/drivers/common/mlx5/mlx5_devx_cmds.h
@@ -32,6 +32,8 @@ struct mlx5_devx_mkey_attr {
 	uint32_t relaxed_ordering_write:1;
 	uint32_t relaxed_ordering_read:1;
 	uint32_t umr_en:1;
+	uint32_t crypto_en:2;
+	uint32_t set_remote_rw:1;
 	struct mlx5_klm *klm_array;
 	int klm_num;
 };
diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index bc9f58ad03..25f6f8ff00 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -1059,7 +1059,15 @@ struct mlx5_ifc_mkc_bits {
 	u8 relaxed_ordering_read[0x1];
 	u8 reserved_at_1da[0x1];
 	u8 log_page_size[0x5];
-	u8 reserved_at_1e0[0x20];
+	u8 reserved_at_1e0[0x3];
+	u8 crypto_en[0x2];
+	u8 reserved_at_1e5[0x1b];
+};
+
+/* Range of values for MKEY context crypto_en field. */
+enum {
+	MLX5_MKEY_CRYPTO_DISABLED = 0x0,
+	MLX5_MKEY_CRYPTO_ENABLED = 0x1,
 };
 
 struct mlx5_ifc_create_mkey_out_bits {
-- 
2.25.1

[dpdk-dev] [PATCH v3 07/15] common/mlx5: support general obj IMPORT KEK create

[Matan Azrad]

From: Dekel Peled <dekelp@nvidia.com>

IMPORT_KEK object is used to wrap (encrypt) critical security
parameters, such as other keys and credentials, when those need
to be passed between the device and the software.

This patch add support of IMPORT_KEK object create operation.
Add reading of IMPORT_KEK support capability.
Add function to create general object type IMPORT_KEK, using DevX API.

Signed-off-by: Dekel Peled <dekelp@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_devx_cmds.c | 50 ++++++++++++++++++++++++++++
 drivers/common/mlx5/mlx5_devx_cmds.h | 13 ++++++++
 drivers/common/mlx5/mlx5_prm.h       | 18 ++++++++++
 drivers/common/mlx5/version.map      |  1 +
 4 files changed, 82 insertions(+)

diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c
index 7e3c8b55fa..afef7a5f63 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.c
+++ b/drivers/common/mlx5/mlx5_devx_cmds.c
@@ -752,6 +752,8 @@ mlx5_devx_cmd_query_hca_attr(void *ctx,
 				  MLX5_GENERAL_OBJ_TYPES_CAP_GENEVE_TLV_OPT);
 	attr->dek = !!(general_obj_types_supported &
 		       MLX5_GENERAL_OBJ_TYPES_CAP_DEK);
+	attr->import_kek = !!(general_obj_types_supported &
+			      MLX5_GENERAL_OBJ_TYPES_CAP_IMPORT_KEK);
 	/* Add reading of other GENERAL_OBJ_TYPES_CAP bits above this line. */
 	attr->log_max_cq = MLX5_GET(cmd_hca_cap, hcattr, log_max_cq);
 	attr->log_max_qp = MLX5_GET(cmd_hca_cap, hcattr, log_max_qp);
@@ -2459,3 +2461,51 @@ mlx5_devx_cmd_create_dek_obj(void *ctx, struct mlx5_devx_dek_attr *attr)
 	dek_obj->id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
 	return dek_obj;
 }
+
+/**
+ * Create general object of type IMPORT_KEK using DevX API.
+ *
+ * @param[in] ctx
+ *   Context returned from mlx5 open_device() glue function.
+ * @param [in] attr
+ *   Pointer to IMPORT_KEK attributes structure.
+ *
+ * @return
+ *   The DevX object created, NULL otherwise and rte_errno is set.
+ */
+struct mlx5_devx_obj *
+mlx5_devx_cmd_create_import_kek_obj(void *ctx,
+				    struct mlx5_devx_import_kek_attr *attr)
+{
+	uint32_t in[MLX5_ST_SZ_DW(create_import_kek_in)] = {0};
+	uint32_t out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};
+	struct mlx5_devx_obj *import_kek_obj = NULL;
+	void *ptr = NULL, *key_addr = NULL;
+
+	import_kek_obj = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*import_kek_obj),
+				     0, SOCKET_ID_ANY);
+	if (import_kek_obj == NULL) {
+		DRV_LOG(ERR, "Failed to allocate IMPORT_KEK object data");
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+	ptr = MLX5_ADDR_OF(create_import_kek_in, in, hdr);
+	MLX5_SET(general_obj_in_cmd_hdr, ptr, opcode,
+		 MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
+	MLX5_SET(general_obj_in_cmd_hdr, ptr, obj_type,
+		 MLX5_GENERAL_OBJ_TYPE_IMPORT_KEK);
+	ptr = MLX5_ADDR_OF(create_import_kek_in, in, import_kek);
+	MLX5_SET(import_kek, ptr, key_size, attr->key_size);
+	key_addr = MLX5_ADDR_OF(import_kek, ptr, key);
+	memcpy(key_addr, (void *)(attr->key), MLX5_CRYPTO_KEY_MAX_SIZE);
+	import_kek_obj->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in),
+							 out, sizeof(out));
+	if (import_kek_obj->obj == NULL) {
+		rte_errno = errno;
+		DRV_LOG(ERR, "Failed to create IMPORT_KEK object using DevX.");
+		mlx5_free(import_kek_obj);
+		return NULL;
+	}
+	import_kek_obj->id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
+	return import_kek_obj;
+}
diff --git a/drivers/common/mlx5/mlx5_devx_cmds.h b/drivers/common/mlx5/mlx5_devx_cmds.h
index 600577f18a..6423610dae 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.h
+++ b/drivers/common/mlx5/mlx5_devx_cmds.h
@@ -142,6 +142,7 @@ struct mlx5_hca_attr {
 	uint32_t crypto:1; /* Crypto engine is supported. */
 	uint32_t aes_xts:1; /* AES-XTS crypto is supported. */
 	uint32_t dek:1; /* General obj type DEK is supported. */
+	uint32_t import_kek:1; /* General obj type IMPORT_KEK supported. */
 	uint32_t regexp_num_of_engines;
 	uint32_t log_max_ft_sampler_num:8;
 	uint32_t geneve_tlv_opt;
@@ -450,6 +451,13 @@ struct mlx5_devx_dek_attr {
 	uint8_t key[MLX5_CRYPTO_KEY_MAX_SIZE];
 };
 
+struct mlx5_devx_import_kek_attr {
+	uint64_t modify_field_select;
+	uint32_t state:8;
+	uint32_t key_size:4;
+	uint8_t key[MLX5_CRYPTO_KEY_MAX_SIZE];
+};
+
 /* mlx5_devx_cmds.c */
 
 __rte_internal
@@ -606,4 +614,9 @@ __rte_internal
 struct mlx5_devx_obj *
 mlx5_devx_cmd_create_dek_obj(void *ctx, struct mlx5_devx_dek_attr *attr);
 
+__rte_internal
+struct mlx5_devx_obj *
+mlx5_devx_cmd_create_import_kek_obj(void *ctx,
+				    struct mlx5_devx_import_kek_attr *attr);
+
 #endif /* RTE_PMD_MLX5_DEVX_CMDS_H_ */
diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index 25f6f8ff00..bc339566a6 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -1119,6 +1119,8 @@ enum {
 			(1ULL << MLX5_GENERAL_OBJ_TYPE_GENEVE_TLV_OPT)
 #define MLX5_GENERAL_OBJ_TYPES_CAP_DEK \
 			(1ULL << MLX5_GENERAL_OBJ_TYPE_DEK)
+#define MLX5_GENERAL_OBJ_TYPES_CAP_IMPORT_KEK \
+			(1ULL << MLX5_GENERAL_OBJ_TYPE_IMPORT_KEK)
 
 enum {
 	MLX5_HCA_CAP_OPMOD_GET_MAX   = 0,
@@ -2419,6 +2421,7 @@ enum {
 	MLX5_GENERAL_OBJ_TYPE_DEK = 0x000c,
 	MLX5_GENERAL_OBJ_TYPE_VIRTQ = 0x000d,
 	MLX5_GENERAL_OBJ_TYPE_VIRTIO_Q_COUNTERS = 0x001c,
+	MLX5_GENERAL_OBJ_TYPE_IMPORT_KEK = 0x001d,
 	MLX5_GENERAL_OBJ_TYPE_FLEX_PARSE_GRAPH = 0x0022,
 	MLX5_GENERAL_OBJ_TYPE_FLOW_METER_ASO = 0x0024,
 	MLX5_GENERAL_OBJ_TYPE_FLOW_HIT_ASO = 0x0025,
@@ -2516,6 +2519,21 @@ struct mlx5_ifc_create_dek_in_bits {
 	struct mlx5_ifc_dek_bits dek;
 };
 
+struct mlx5_ifc_import_kek_bits {
+	u8 modify_field_select[0x40];
+	u8 state[0x8];
+	u8 reserved_at_48[0xc];
+	u8 key_size[0x4];
+	u8 reserved_at_58[0x1a8];
+	u8 key[0x400];
+	u8 reserved_at_600[0x200];
+};
+
+struct mlx5_ifc_create_import_kek_in_bits {
+	struct mlx5_ifc_general_obj_in_cmd_hdr_bits hdr;
+	struct mlx5_ifc_import_kek_bits import_kek;
+};
+
 enum {
 	MLX5_VIRTQ_STATE_INIT = 0,
 	MLX5_VIRTQ_STATE_RDY = 1,
diff --git a/drivers/common/mlx5/version.map b/drivers/common/mlx5/version.map
index 42bb985fb1..60bff5f799 100644
--- a/drivers/common/mlx5/version.map
+++ b/drivers/common/mlx5/version.map
@@ -21,6 +21,7 @@ INTERNAL {
 	mlx5_devx_cmd_create_flow_hit_aso_obj;
 	mlx5_devx_cmd_create_flow_meter_aso_obj;
 	mlx5_devx_cmd_create_geneve_tlv_option;
+	mlx5_devx_cmd_create_import_kek_obj;
 	mlx5_devx_cmd_create_qp;
 	mlx5_devx_cmd_create_rq;
 	mlx5_devx_cmd_create_rqt;
-- 
2.25.1

[dpdk-dev] [PATCH v3 08/15] common/mlx5: support general obj CRYPTO LOGIN create

[Matan Azrad]

From: Dekel Peled <dekelp@nvidia.com>

CRYPTO_LOGIN Object is used to login to the device as crypto user
or crypto officer.
Required in order to perform any crypto related control operations.

This patch adds support of CRYPTO_LOGIN object create operation.
Add reading of CRYPTO_LOGIN support capability.
Add function to create general object type CRYPTO_LOGIN, using DevX API.

Signed-off-by: Dekel Peled <dekelp@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_devx_cmds.c | 54 ++++++++++++++++++++++++++++
 drivers/common/mlx5/mlx5_devx_cmds.h | 15 ++++++++
 drivers/common/mlx5/mlx5_prm.h       | 19 ++++++++++
 drivers/common/mlx5/version.map      |  1 +
 4 files changed, 89 insertions(+)

diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c
index afef7a5f63..5e082ebb78 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.c
+++ b/drivers/common/mlx5/mlx5_devx_cmds.c
@@ -754,6 +754,8 @@ mlx5_devx_cmd_query_hca_attr(void *ctx,
 		       MLX5_GENERAL_OBJ_TYPES_CAP_DEK);
 	attr->import_kek = !!(general_obj_types_supported &
 			      MLX5_GENERAL_OBJ_TYPES_CAP_IMPORT_KEK);
+	attr->crypto_login = !!(general_obj_types_supported &
+				MLX5_GENERAL_OBJ_TYPES_CAP_CRYPTO_LOGIN);
 	/* Add reading of other GENERAL_OBJ_TYPES_CAP bits above this line. */
 	attr->log_max_cq = MLX5_GET(cmd_hca_cap, hcattr, log_max_cq);
 	attr->log_max_qp = MLX5_GET(cmd_hca_cap, hcattr, log_max_qp);
@@ -2509,3 +2511,55 @@ mlx5_devx_cmd_create_import_kek_obj(void *ctx,
 	import_kek_obj->id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
 	return import_kek_obj;
 }
+
+/**
+ * Create general object of type CRYPTO_LOGIN using DevX API.
+ *
+ * @param[in] ctx
+ *   Context returned from mlx5 open_device() glue function.
+ * @param [in] attr
+ *   Pointer to CRYPTO_LOGIN attributes structure.
+ *
+ * @return
+ *   The DevX object created, NULL otherwise and rte_errno is set.
+ */
+struct mlx5_devx_obj *
+mlx5_devx_cmd_create_crypto_login_obj(void *ctx,
+				      struct mlx5_devx_crypto_login_attr *attr)
+{
+	uint32_t in[MLX5_ST_SZ_DW(create_crypto_login_in)] = {0};
+	uint32_t out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};
+	struct mlx5_devx_obj *crypto_login_obj = NULL;
+	void *ptr = NULL, *credential_addr = NULL;
+
+	crypto_login_obj = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*crypto_login_obj),
+				       0, SOCKET_ID_ANY);
+	if (crypto_login_obj == NULL) {
+		DRV_LOG(ERR, "Failed to allocate CRYPTO_LOGIN object data");
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+	ptr = MLX5_ADDR_OF(create_crypto_login_in, in, hdr);
+	MLX5_SET(general_obj_in_cmd_hdr, ptr, opcode,
+		 MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
+	MLX5_SET(general_obj_in_cmd_hdr, ptr, obj_type,
+		 MLX5_GENERAL_OBJ_TYPE_CRYPTO_LOGIN);
+	ptr = MLX5_ADDR_OF(create_crypto_login_in, in, crypto_login);
+	MLX5_SET(crypto_login, ptr, credential_pointer,
+		 attr->credential_pointer);
+	MLX5_SET(crypto_login, ptr, session_import_kek_ptr,
+		 attr->session_import_kek_ptr);
+	credential_addr = MLX5_ADDR_OF(crypto_login, ptr, credential);
+	memcpy(credential_addr, (void *)(attr->credential),
+	       MLX5_CRYPTO_LOGIN_CREDENTIAL_SIZE);
+	crypto_login_obj->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in),
+							   out, sizeof(out));
+	if (crypto_login_obj->obj == NULL) {
+		rte_errno = errno;
+		DRV_LOG(ERR, "Failed to create CRYPTO_LOGIN obj using DevX.");
+		mlx5_free(crypto_login_obj);
+		return NULL;
+	}
+	crypto_login_obj->id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
+	return crypto_login_obj;
+}
diff --git a/drivers/common/mlx5/mlx5_devx_cmds.h b/drivers/common/mlx5/mlx5_devx_cmds.h
index 6423610dae..709e28bfba 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.h
+++ b/drivers/common/mlx5/mlx5_devx_cmds.h
@@ -143,6 +143,7 @@ struct mlx5_hca_attr {
 	uint32_t aes_xts:1; /* AES-XTS crypto is supported. */
 	uint32_t dek:1; /* General obj type DEK is supported. */
 	uint32_t import_kek:1; /* General obj type IMPORT_KEK supported. */
+	uint32_t crypto_login:1; /* General obj type CRYPTO_LOGIN supported. */
 	uint32_t regexp_num_of_engines;
 	uint32_t log_max_ft_sampler_num:8;
 	uint32_t geneve_tlv_opt;
@@ -458,6 +459,15 @@ struct mlx5_devx_import_kek_attr {
 	uint8_t key[MLX5_CRYPTO_KEY_MAX_SIZE];
 };
 
+#define MLX5_CRYPTO_LOGIN_CREDENTIAL_SIZE	48
+
+struct mlx5_devx_crypto_login_attr {
+	uint64_t modify_field_select;
+	uint32_t credential_pointer:24;
+	uint32_t session_import_kek_ptr:24;
+	uint8_t credential[MLX5_CRYPTO_LOGIN_CREDENTIAL_SIZE];
+};
+
 /* mlx5_devx_cmds.c */
 
 __rte_internal
@@ -619,4 +629,9 @@ struct mlx5_devx_obj *
 mlx5_devx_cmd_create_import_kek_obj(void *ctx,
 				    struct mlx5_devx_import_kek_attr *attr);
 
+__rte_internal
+struct mlx5_devx_obj *
+mlx5_devx_cmd_create_crypto_login_obj(void *ctx,
+				      struct mlx5_devx_crypto_login_attr *attr);
+
 #endif /* RTE_PMD_MLX5_DEVX_CMDS_H_ */
diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index bc339566a6..a2437faec0 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -1121,6 +1121,8 @@ enum {
 			(1ULL << MLX5_GENERAL_OBJ_TYPE_DEK)
 #define MLX5_GENERAL_OBJ_TYPES_CAP_IMPORT_KEK \
 			(1ULL << MLX5_GENERAL_OBJ_TYPE_IMPORT_KEK)
+#define MLX5_GENERAL_OBJ_TYPES_CAP_CRYPTO_LOGIN \
+			(1ULL << MLX5_GENERAL_OBJ_TYPE_CRYPTO_LOGIN)
 
 enum {
 	MLX5_HCA_CAP_OPMOD_GET_MAX   = 0,
@@ -2422,6 +2424,7 @@ enum {
 	MLX5_GENERAL_OBJ_TYPE_VIRTQ = 0x000d,
 	MLX5_GENERAL_OBJ_TYPE_VIRTIO_Q_COUNTERS = 0x001c,
 	MLX5_GENERAL_OBJ_TYPE_IMPORT_KEK = 0x001d,
+	MLX5_GENERAL_OBJ_TYPE_CRYPTO_LOGIN = 0x001f,
 	MLX5_GENERAL_OBJ_TYPE_FLEX_PARSE_GRAPH = 0x0022,
 	MLX5_GENERAL_OBJ_TYPE_FLOW_METER_ASO = 0x0024,
 	MLX5_GENERAL_OBJ_TYPE_FLOW_HIT_ASO = 0x0025,
@@ -2534,6 +2537,22 @@ struct mlx5_ifc_create_import_kek_in_bits {
 	struct mlx5_ifc_import_kek_bits import_kek;
 };
 
+struct mlx5_ifc_crypto_login_bits {
+	u8 modify_field_select[0x40];
+	u8 reserved_at_40[0x48];
+	u8 credential_pointer[0x18];
+	u8 reserved_at_a0[0x8];
+	u8 session_import_kek_ptr[0x18];
+	u8 reserved_at_c0[0x140];
+	u8 credential[0x180];
+	u8 reserved_at_380[0x480];
+};
+
+struct mlx5_ifc_create_crypto_login_in_bits {
+	struct mlx5_ifc_general_obj_in_cmd_hdr_bits hdr;
+	struct mlx5_ifc_crypto_login_bits crypto_login;
+};
+
 enum {
 	MLX5_VIRTQ_STATE_INIT = 0,
 	MLX5_VIRTQ_STATE_RDY = 1,
diff --git a/drivers/common/mlx5/version.map b/drivers/common/mlx5/version.map
index 60bff5f799..89f0ee04cb 100644
--- a/drivers/common/mlx5/version.map
+++ b/drivers/common/mlx5/version.map
@@ -16,6 +16,7 @@ INTERNAL {
 
 	mlx5_devx_cmd_alloc_pd;
 	mlx5_devx_cmd_create_cq;
+	mlx5_devx_cmd_create_crypto_login_obj;
 	mlx5_devx_cmd_create_dek_obj;
 	mlx5_devx_cmd_create_flex_parser;
 	mlx5_devx_cmd_create_flow_hit_aso_obj;
-- 
2.25.1

[dpdk-dev] [PATCH v3 09/15] common/mlx5: add crypto BSF struct and defines

[Matan Azrad]

From: Dekel Peled <dekelp@nvidia.com>

This patch adds the struct defining crypto BSF segment of UMR WQE,
and the related value definitions and offsets.

Signed-off-by: Dekel Peled <dekelp@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_prm.h | 66 ++++++++++++++++++++++++++++++++++
 1 file changed, 66 insertions(+)

diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index a2437faec0..a9dcbfa63c 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -1096,6 +1096,72 @@ struct mlx5_ifc_create_mkey_in_bits {
 	u8 klm_pas_mtt[][0x20];
 };
 
+enum {
+	MLX5_BSF_SIZE_16B = 0x0,
+	MLX5_BSF_SIZE_32B = 0x1,
+	MLX5_BSF_SIZE_64B = 0x2,
+	MLX5_BSF_SIZE_128B = 0x3,
+};
+
+enum {
+	MLX5_BSF_P_TYPE_SIGNATURE = 0x0,
+	MLX5_BSF_P_TYPE_CRYPTO = 0x1,
+};
+
+enum {
+	MLX5_ENCRYPTION_ORDER_ENCRYPTED_WIRE_SIGNATURE = 0x0,
+	MLX5_ENCRYPTION_ORDER_ENCRYPTED_MEMORY_SIGNATURE = 0x1,
+	MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_WIRE = 0x2,
+	MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_MEMORY = 0x3,
+};
+
+enum {
+	MLX5_ENCRYPTION_STANDARD_AES_XTS = 0x0,
+};
+
+enum {
+	MLX5_BLOCK_SIZE_512B	= 0x1,
+	MLX5_BLOCK_SIZE_520B	= 0x2,
+	MLX5_BLOCK_SIZE_4096B	= 0x3,
+	MLX5_BLOCK_SIZE_4160B	= 0x4,
+	MLX5_BLOCK_SIZE_1MB	= 0x5,
+	MLX5_BLOCK_SIZE_4048B	= 0x6,
+};
+
+#define MLX5_BSF_SIZE_OFFSET		30
+#define MLX5_BSF_P_TYPE_OFFSET		24
+#define MLX5_ENCRYPTION_ORDER_OFFSET	16
+#define MLX5_BLOCK_SIZE_OFFSET		24
+
+struct mlx5_wqe_umr_bsf_seg {
+	/*
+	 * bs_bpt_eo_es contains:
+	 * bs	bsf_size		2 bits at MLX5_BSF_SIZE_OFFSET
+	 * bpt	bsf_p_type		2 bits at MLX5_BSF_P_TYPE_OFFSET
+	 * eo	encryption_order	4 bits at MLX5_ENCRYPTION_ORDER_OFFSET
+	 * es	encryption_standard	4 bits at offset 0
+	 */
+	uint32_t bs_bpt_eo_es;
+	uint32_t raw_data_size;
+	/*
+	 * bsp_res contains:
+	 * bsp	crypto_block_size_pointer	8 bits at MLX5_BLOCK_SIZE_OFFSET
+	 * res	reserved 24 bits
+	 */
+	uint32_t bsp_res;
+	uint32_t reserved0;
+	uint8_t xts_initial_tweak[16];
+	/*
+	 * res_dp contains:
+	 * res	reserved 8 bits
+	 * dp	dek_pointer		24 bits at offset 0
+	 */
+	uint32_t res_dp;
+	uint32_t reserved1;
+	uint64_t keytag;
+	uint32_t reserved2[4];
+} __rte_packed;
+
 enum {
 	MLX5_GET_HCA_CAP_OP_MOD_GENERAL_DEVICE = 0x0 << 1,
 	MLX5_GET_HCA_CAP_OP_MOD_ETHERNET_OFFLOAD_CAPS = 0x1 << 1,
-- 
2.25.1

[dpdk-dev] [PATCH v3 10/15] common/mlx5: share hash list tool

[Matan Azrad]

From: Shiri Kuzin <shirik@nvidia.com>

In order to use the hash list defined in net in other drivers, the
hash list is moved to common utilities.

In addition, the log definition was moved from the common utilities to
a dedicated new log file in common in order to prevent a conflict.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/linux/mlx5_common_os.c   |   2 +-
 drivers/common/mlx5/linux/mlx5_nl.c          |   2 +-
 drivers/common/mlx5/meson.build              |  15 +-
 drivers/common/mlx5/mlx5_common.c            |   2 +-
 drivers/common/mlx5/mlx5_common_devx.c       |   2 +-
 drivers/common/mlx5/mlx5_common_log.h        |  21 ++
 drivers/common/mlx5/mlx5_common_mp.c         |   2 +-
 drivers/common/mlx5/mlx5_common_mr.c         |   2 +-
 drivers/common/mlx5/mlx5_common_pci.c        |   4 +-
 drivers/common/mlx5/mlx5_common_utils.c      | 221 +++++++++++++++++++
 drivers/common/mlx5/mlx5_common_utils.h      | 202 ++++++++++++++++-
 drivers/common/mlx5/mlx5_devx_cmds.c         |   2 +-
 drivers/common/mlx5/mlx5_malloc.c            |   2 +-
 drivers/common/mlx5/version.map              |   6 +
 drivers/common/mlx5/windows/mlx5_common_os.c |   2 +-
 drivers/common/mlx5/windows/mlx5_glue.c      |   2 +-
 drivers/net/mlx5/mlx5_utils.c                | 209 ------------------
 drivers/net/mlx5/mlx5_utils.h                | 194 +---------------
 18 files changed, 465 insertions(+), 427 deletions(-)
 create mode 100644 drivers/common/mlx5/mlx5_common_log.h
 create mode 100644 drivers/common/mlx5/mlx5_common_utils.c

diff --git a/drivers/common/mlx5/linux/mlx5_common_os.c b/drivers/common/mlx5/linux/mlx5_common_os.c
index 5cf9576921..fba8245b8b 100644
--- a/drivers/common/mlx5/linux/mlx5_common_os.c
+++ b/drivers/common/mlx5/linux/mlx5_common_os.c
@@ -15,7 +15,7 @@
 #include <rte_string_fns.h>
 
 #include "mlx5_common.h"
-#include "mlx5_common_utils.h"
+#include "mlx5_common_log.h"
 #include "mlx5_glue.h"
 
 #ifdef MLX5_GLUE
diff --git a/drivers/common/mlx5/linux/mlx5_nl.c b/drivers/common/mlx5/linux/mlx5_nl.c
index 752c57b33d..f0d04f9473 100644
--- a/drivers/common/mlx5/linux/mlx5_nl.c
+++ b/drivers/common/mlx5/linux/mlx5_nl.c
@@ -20,7 +20,7 @@
 #include <rte_errno.h>
 
 #include "mlx5_nl.h"
-#include "mlx5_common_utils.h"
+#include "../mlx5_common_log.h"
 #include "mlx5_malloc.h"
 #ifdef HAVE_DEVLINK
 #include <linux/devlink.h>
diff --git a/drivers/common/mlx5/meson.build b/drivers/common/mlx5/meson.build
index e78b4f47bc..c7ce4c7045 100644
--- a/drivers/common/mlx5/meson.build
+++ b/drivers/common/mlx5/meson.build
@@ -9,13 +9,14 @@ endif
 
 deps += ['hash', 'pci', 'bus_pci', 'net', 'eal', 'kvargs']
 sources += files(
-        'mlx5_devx_cmds.c',
-        'mlx5_common.c',
-        'mlx5_common_mp.c',
-        'mlx5_common_mr.c',
-        'mlx5_malloc.c',
-        'mlx5_common_pci.c',
-        'mlx5_common_devx.c',
+	'mlx5_devx_cmds.c',
+	'mlx5_common.c',
+	'mlx5_common_mp.c',
+	'mlx5_common_mr.c',
+	'mlx5_malloc.c',
+	'mlx5_common_pci.c',
+	'mlx5_common_devx.c',
+	'mlx5_common_utils.c',
 )
 
 cflags_options = [
diff --git a/drivers/common/mlx5/mlx5_common.c b/drivers/common/mlx5/mlx5_common.c
index f92f05bda5..d397459a3d 100644
--- a/drivers/common/mlx5/mlx5_common.c
+++ b/drivers/common/mlx5/mlx5_common.c
@@ -11,7 +11,7 @@
 
 #include "mlx5_common.h"
 #include "mlx5_common_os.h"
-#include "mlx5_common_utils.h"
+#include "mlx5_common_log.h"
 #include "mlx5_common_pci.h"
 
 uint8_t haswell_broadwell_cpu;
diff --git a/drivers/common/mlx5/mlx5_common_devx.c b/drivers/common/mlx5/mlx5_common_devx.c
index d19be122bd..22c8d356c4 100644
--- a/drivers/common/mlx5/mlx5_common_devx.c
+++ b/drivers/common/mlx5/mlx5_common_devx.c
@@ -12,7 +12,7 @@
 
 #include "mlx5_prm.h"
 #include "mlx5_devx_cmds.h"
-#include "mlx5_common_utils.h"
+#include "mlx5_common_log.h"
 #include "mlx5_malloc.h"
 #include "mlx5_common.h"
 #include "mlx5_common_devx.h"
diff --git a/drivers/common/mlx5/mlx5_common_log.h b/drivers/common/mlx5/mlx5_common_log.h
new file mode 100644
index 0000000000..26b13fedaf
--- /dev/null
+++ b/drivers/common/mlx5/mlx5_common_log.h
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2019 Mellanox Technologies, Ltd
+ */
+
+#ifndef RTE_PMD_MLX5_COMMON_LOG_H_
+#define RTE_PMD_MLX5_COMMON_LOG_H_
+
+#include "mlx5_common.h"
+
+
+extern int mlx5_common_logtype;
+
+#define MLX5_COMMON_LOG_PREFIX "mlx5_common"
+/* Generic printf()-like logging macro with automatic line feed. */
+#define DRV_LOG(level, ...) \
+	PMD_DRV_LOG_(level, mlx5_common_logtype, MLX5_COMMON_LOG_PREFIX, \
+		__VA_ARGS__ PMD_DRV_LOG_STRIP PMD_DRV_LOG_OPAREN, \
+		PMD_DRV_LOG_CPAREN)
+
+#endif /* RTE_PMD_MLX5_COMMON_LOG_H_ */
+
diff --git a/drivers/common/mlx5/mlx5_common_mp.c b/drivers/common/mlx5/mlx5_common_mp.c
index 40e3956e45..673a7c31de 100644
--- a/drivers/common/mlx5/mlx5_common_mp.c
+++ b/drivers/common/mlx5/mlx5_common_mp.c
@@ -10,7 +10,7 @@
 #include <rte_errno.h>
 
 #include "mlx5_common_mp.h"
-#include "mlx5_common_utils.h"
+#include "mlx5_common_log.h"
 #include "mlx5_malloc.h"
 
 /**
diff --git a/drivers/common/mlx5/mlx5_common_mr.c b/drivers/common/mlx5/mlx5_common_mr.c
index e1ed0caf3a..afb5b3d0a7 100644
--- a/drivers/common/mlx5/mlx5_common_mr.c
+++ b/drivers/common/mlx5/mlx5_common_mr.c
@@ -11,7 +11,7 @@
 #include "mlx5_glue.h"
 #include "mlx5_common_mp.h"
 #include "mlx5_common_mr.h"
-#include "mlx5_common_utils.h"
+#include "mlx5_common_log.h"
 #include "mlx5_malloc.h"
 
 struct mr_find_contig_memsegs_data {
diff --git a/drivers/common/mlx5/mlx5_common_pci.c b/drivers/common/mlx5/mlx5_common_pci.c
index 5a73ffa60a..3f16cd21cf 100644
--- a/drivers/common/mlx5/mlx5_common_pci.c
+++ b/drivers/common/mlx5/mlx5_common_pci.c
@@ -4,7 +4,9 @@
 
 #include <stdlib.h>
 #include <rte_malloc.h>
-#include "mlx5_common_utils.h"
+#include <rte_class.h>
+
+#include "mlx5_common_log.h"
 #include "mlx5_common_pci.h"
 
 struct mlx5_pci_device {
diff --git a/drivers/common/mlx5/mlx5_common_utils.c b/drivers/common/mlx5/mlx5_common_utils.c
new file mode 100644
index 0000000000..ad2011e858
--- /dev/null
+++ b/drivers/common/mlx5/mlx5_common_utils.c
@@ -0,0 +1,221 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2019 Mellanox Technologies, Ltd
+ */
+
+#include <rte_malloc.h>
+#include <rte_hash_crc.h>
+#include <rte_errno.h>
+
+#include <mlx5_malloc.h>
+
+#include "mlx5_common_utils.h"
+#include "mlx5_common_log.h"
+
+/********************* Hash List **********************/
+
+static struct mlx5_hlist_entry *
+mlx5_hlist_default_create_cb(struct mlx5_hlist *h, uint64_t key __rte_unused,
+			     void *ctx __rte_unused)
+{
+	return mlx5_malloc(MLX5_MEM_ZERO, h->entry_sz, 0, SOCKET_ID_ANY);
+}
+
+static void
+mlx5_hlist_default_remove_cb(struct mlx5_hlist *h __rte_unused,
+			     struct mlx5_hlist_entry *entry)
+{
+	mlx5_free(entry);
+}
+
+struct mlx5_hlist *
+mlx5_hlist_create(const char *name, uint32_t size, uint32_t entry_size,
+		  uint32_t flags, mlx5_hlist_create_cb cb_create,
+		  mlx5_hlist_match_cb cb_match, mlx5_hlist_remove_cb cb_remove)
+{
+	struct mlx5_hlist *h;
+	uint32_t act_size;
+	uint32_t alloc_size;
+	uint32_t i;
+
+	if (!size || !cb_match || (!cb_create ^ !cb_remove))
+		return NULL;
+	/* Align to the next power of 2, 32bits integer is enough now. */
+	if (!rte_is_power_of_2(size)) {
+		act_size = rte_align32pow2(size);
+		DRV_LOG(DEBUG, "Size 0x%" PRIX32 " is not power of 2, "
+			"will be aligned to 0x%" PRIX32 ".", size, act_size);
+	} else {
+		act_size = size;
+	}
+	alloc_size = sizeof(struct mlx5_hlist) +
+		     sizeof(struct mlx5_hlist_bucket) * act_size;
+	/* Using zmalloc, then no need to initialize the heads. */
+	h = mlx5_malloc(MLX5_MEM_ZERO, alloc_size, RTE_CACHE_LINE_SIZE,
+			SOCKET_ID_ANY);
+	if (!h) {
+		DRV_LOG(ERR, "No memory for hash list %s creation",
+			name ? name : "None");
+		return NULL;
+	}
+	if (name)
+		snprintf(h->name, MLX5_HLIST_NAMESIZE, "%s", name);
+	h->table_sz = act_size;
+	h->mask = act_size - 1;
+	h->entry_sz = entry_size;
+	h->direct_key = !!(flags & MLX5_HLIST_DIRECT_KEY);
+	h->write_most = !!(flags & MLX5_HLIST_WRITE_MOST);
+	h->cb_create = cb_create ? cb_create : mlx5_hlist_default_create_cb;
+	h->cb_match = cb_match;
+	h->cb_remove = cb_remove ? cb_remove : mlx5_hlist_default_remove_cb;
+	for (i = 0; i < act_size; i++)
+		rte_rwlock_init(&h->buckets[i].lock);
+	DRV_LOG(DEBUG, "Hash list with %s size 0x%" PRIX32 " is created.",
+		h->name, act_size);
+	return h;
+}
+
+static struct mlx5_hlist_entry *
+__hlist_lookup(struct mlx5_hlist *h, uint64_t key, uint32_t idx,
+	       void *ctx, bool reuse)
+{
+	struct mlx5_hlist_head *first;
+	struct mlx5_hlist_entry *node;
+
+	MLX5_ASSERT(h);
+	first = &h->buckets[idx].head;
+	LIST_FOREACH(node, first, next) {
+		if (!h->cb_match(h, node, key, ctx)) {
+			if (reuse) {
+				__atomic_add_fetch(&node->ref_cnt, 1,
+						   __ATOMIC_RELAXED);
+				DRV_LOG(DEBUG, "Hash list %s entry %p "
+					"reuse: %u.",
+					h->name, (void *)node, node->ref_cnt);
+			}
+			break;
+		}
+	}
+	return node;
+}
+
+static struct mlx5_hlist_entry *
+hlist_lookup(struct mlx5_hlist *h, uint64_t key, uint32_t idx,
+	     void *ctx, bool reuse)
+{
+	struct mlx5_hlist_entry *node;
+
+	MLX5_ASSERT(h);
+	rte_rwlock_read_lock(&h->buckets[idx].lock);
+	node = __hlist_lookup(h, key, idx, ctx, reuse);
+	rte_rwlock_read_unlock(&h->buckets[idx].lock);
+	return node;
+}
+
+struct mlx5_hlist_entry *
+mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
+{
+	uint32_t idx;
+
+	if (h->direct_key)
+		idx = (uint32_t)(key & h->mask);
+	else
+		idx = rte_hash_crc_8byte(key, 0) & h->mask;
+	return hlist_lookup(h, key, idx, ctx, false);
+}
+
+struct mlx5_hlist_entry*
+mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
+{
+	uint32_t idx;
+	struct mlx5_hlist_head *first;
+	struct mlx5_hlist_bucket *b;
+	struct mlx5_hlist_entry *entry;
+	uint32_t prev_gen_cnt = 0;
+
+	if (h->direct_key)
+		idx = (uint32_t)(key & h->mask);
+	else
+		idx = rte_hash_crc_8byte(key, 0) & h->mask;
+	MLX5_ASSERT(h);
+	b = &h->buckets[idx];
+	/* Use write lock directly for write-most list. */
+	if (!h->write_most) {
+		prev_gen_cnt = __atomic_load_n(&b->gen_cnt, __ATOMIC_ACQUIRE);
+		entry = hlist_lookup(h, key, idx, ctx, true);
+		if (entry)
+			return entry;
+	}
+	rte_rwlock_write_lock(&b->lock);
+	/* Check if the list changed by other threads. */
+	if (h->write_most ||
+	    prev_gen_cnt != __atomic_load_n(&b->gen_cnt, __ATOMIC_ACQUIRE)) {
+		entry = __hlist_lookup(h, key, idx, ctx, true);
+		if (entry)
+			goto done;
+	}
+	first = &b->head;
+	entry = h->cb_create(h, key, ctx);
+	if (!entry) {
+		rte_errno = ENOMEM;
+		DRV_LOG(DEBUG, "Can't allocate hash list %s entry.", h->name);
+		goto done;
+	}
+	entry->idx = idx;
+	entry->ref_cnt = 1;
+	LIST_INSERT_HEAD(first, entry, next);
+	__atomic_add_fetch(&b->gen_cnt, 1, __ATOMIC_ACQ_REL);
+	DRV_LOG(DEBUG, "Hash list %s entry %p new: %u.",
+		h->name, (void *)entry, entry->ref_cnt);
+done:
+	rte_rwlock_write_unlock(&b->lock);
+	return entry;
+}
+
+int
+mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry)
+{
+	uint32_t idx = entry->idx;
+
+	rte_rwlock_write_lock(&h->buckets[idx].lock);
+	MLX5_ASSERT(entry && entry->ref_cnt && entry->next.le_prev);
+	DRV_LOG(DEBUG, "Hash list %s entry %p deref: %u.",
+		h->name, (void *)entry, entry->ref_cnt);
+	if (--entry->ref_cnt) {
+		rte_rwlock_write_unlock(&h->buckets[idx].lock);
+		return 1;
+	}
+	LIST_REMOVE(entry, next);
+	/* Set to NULL to get rid of removing action for more than once. */
+	entry->next.le_prev = NULL;
+	h->cb_remove(h, entry);
+	rte_rwlock_write_unlock(&h->buckets[idx].lock);
+	DRV_LOG(DEBUG, "Hash list %s entry %p removed.",
+		h->name, (void *)entry);
+	return 0;
+}
+
+void
+mlx5_hlist_destroy(struct mlx5_hlist *h)
+{
+	uint32_t idx;
+	struct mlx5_hlist_entry *entry;
+
+	MLX5_ASSERT(h);
+	for (idx = 0; idx < h->table_sz; ++idx) {
+		/* No LIST_FOREACH_SAFE, using while instead. */
+		while (!LIST_EMPTY(&h->buckets[idx].head)) {
+			entry = LIST_FIRST(&h->buckets[idx].head);
+			LIST_REMOVE(entry, next);
+			/*
+			 * The owner of whole element which contains data entry
+			 * is the user, so it's the user's duty to do the clean
+			 * up and the free work because someone may not put the
+			 * hlist entry at the beginning(suggested to locate at
+			 * the beginning). Or else the default free function
+			 * will be used.
+			 */
+			h->cb_remove(h, entry);
+		}
+	}
+	mlx5_free(h);
+}
diff --git a/drivers/common/mlx5/mlx5_common_utils.h b/drivers/common/mlx5/mlx5_common_utils.h
index 6cba39c8cc..ed378ce9bd 100644
--- a/drivers/common/mlx5/mlx5_common_utils.h
+++ b/drivers/common/mlx5/mlx5_common_utils.h
@@ -7,14 +7,202 @@
 
 #include "mlx5_common.h"
 
+#define MLX5_HLIST_DIRECT_KEY 0x0001 /* Use the key directly as hash index. */
+#define MLX5_HLIST_WRITE_MOST 0x0002 /* List mostly used for append new. */
 
-extern int mlx5_common_logtype;
+/** Maximum size of string for naming the hlist table. */
+#define MLX5_HLIST_NAMESIZE			32
 
-#define MLX5_COMMON_LOG_PREFIX "mlx5_common"
-/* Generic printf()-like logging macro with automatic line feed. */
-#define DRV_LOG(level, ...) \
-	PMD_DRV_LOG_(level, mlx5_common_logtype, MLX5_COMMON_LOG_PREFIX, \
-		__VA_ARGS__ PMD_DRV_LOG_STRIP PMD_DRV_LOG_OPAREN, \
-		PMD_DRV_LOG_CPAREN)
+struct mlx5_hlist;
+
+/**
+ * Structure of the entry in the hash list, user should define its own struct
+ * that contains this in order to store the data. The 'key' is 64-bits right
+ * now and its user's responsibility to guarantee there is no collision.
+ */
+struct mlx5_hlist_entry {
+	LIST_ENTRY(mlx5_hlist_entry) next; /* entry pointers in the list. */
+	uint32_t idx; /* Bucket index the entry belongs to. */
+	uint32_t ref_cnt; /* Reference count. */
+};
+
+/** Structure for hash head. */
+LIST_HEAD(mlx5_hlist_head, mlx5_hlist_entry);
+
+/**
+ * Type of callback function for entry removal.
+ *
+ * @param list
+ *   The hash list.
+ * @param entry
+ *   The entry in the list.
+ */
+typedef void (*mlx5_hlist_remove_cb)(struct mlx5_hlist *list,
+				     struct mlx5_hlist_entry *entry);
+
+/**
+ * Type of function for user defined matching.
+ *
+ * @param list
+ *   The hash list.
+ * @param entry
+ *   The entry in the list.
+ * @param key
+ *   The new entry key.
+ * @param ctx
+ *   The pointer to new entry context.
+ *
+ * @return
+ *   0 if matching, non-zero number otherwise.
+ */
+typedef int (*mlx5_hlist_match_cb)(struct mlx5_hlist *list,
+				   struct mlx5_hlist_entry *entry,
+				   uint64_t key, void *ctx);
+
+/**
+ * Type of function for user defined hash list entry creation.
+ *
+ * @param list
+ *   The hash list.
+ * @param key
+ *   The key of the new entry.
+ * @param ctx
+ *   The pointer to new entry context.
+ *
+ * @return
+ *   Pointer to allocated entry on success, NULL otherwise.
+ */
+typedef struct mlx5_hlist_entry *(*mlx5_hlist_create_cb)
+				  (struct mlx5_hlist *list,
+				   uint64_t key, void *ctx);
+
+/* Hash list bucket head. */
+struct mlx5_hlist_bucket {
+	struct mlx5_hlist_head head; /* List head. */
+	rte_rwlock_t lock; /* Bucket lock. */
+	uint32_t gen_cnt; /* List modification will update generation count. */
+} __rte_cache_aligned;
+
+/**
+ * Hash list table structure
+ *
+ * Entry in hash list could be reused if entry already exists, reference
+ * count will increase and the existing entry returns.
+ *
+ * When destroy an entry from list, decrease reference count and only
+ * destroy when no further reference.
+ */
+struct mlx5_hlist {
+	char name[MLX5_HLIST_NAMESIZE]; /**< Name of the hash list. */
+	/**< number of heads, need to be power of 2. */
+	uint32_t table_sz;
+	uint32_t entry_sz; /**< Size of entry, used to allocate entry. */
+	/**< mask to get the index of the list heads. */
+	uint32_t mask;
+	bool direct_key; /* Use the new entry key directly as hash index. */
+	bool write_most; /* List mostly used for append new or destroy. */
+	void *ctx;
+	mlx5_hlist_create_cb cb_create; /**< entry create callback. */
+	mlx5_hlist_match_cb cb_match; /**< entry match callback. */
+	mlx5_hlist_remove_cb cb_remove; /**< entry remove callback. */
+	struct mlx5_hlist_bucket buckets[] __rte_cache_aligned;
+	/**< list bucket arrays. */
+};
+
+/**
+ * Create a hash list table, the user can specify the list heads array size
+ * of the table, now the size should be a power of 2 in order to get better
+ * distribution for the entries. Each entry is a part of the whole data element
+ * and the caller should be responsible for the data element's allocation and
+ * cleanup / free. Key of each entry will be calculated with CRC in order to
+ * generate a little fairer distribution.
+ *
+ * @param name
+ *   Name of the hash list(optional).
+ * @param size
+ *   Heads array size of the hash list.
+ * @param entry_size
+ *   Entry size to allocate if cb_create not specified.
+ * @param flags
+ *   The hash list attribute flags.
+ * @param cb_create
+ *   Callback function for entry create.
+ * @param cb_match
+ *   Callback function for entry match.
+ * @param cb_destroy
+ *   Callback function for entry destroy.
+ * @return
+ *   Pointer of the hash list table created, NULL on failure.
+ */
+__rte_internal
+struct mlx5_hlist *mlx5_hlist_create(const char *name, uint32_t size,
+				     uint32_t entry_size, uint32_t flags,
+				     mlx5_hlist_create_cb cb_create,
+				     mlx5_hlist_match_cb cb_match,
+				     mlx5_hlist_remove_cb cb_destroy);
+
+/**
+ * Search an entry matching the key.
+ *
+ * Result returned might be destroyed by other thread, must use
+ * this function only in main thread.
+ *
+ * @param h
+ *   Pointer to the hast list table.
+ * @param key
+ *   Key for the searching entry.
+ * @param ctx
+ *   Common context parameter used by entry callback function.
+ *
+ * @return
+ *   Pointer of the hlist entry if found, NULL otherwise.
+ */
+__rte_internal
+struct mlx5_hlist_entry *mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key,
+					   void *ctx);
+
+/**
+ * Insert an entry to the hash list table, the entry is only part of whole data
+ * element and a 64B key is used for matching. User should construct the key or
+ * give a calculated hash signature and guarantee there is no collision.
+ *
+ * @param h
+ *   Pointer to the hast list table.
+ * @param entry
+ *   Entry to be inserted into the hash list table.
+ * @param ctx
+ *   Common context parameter used by callback function.
+ *
+ * @return
+ *   registered entry on success, NULL otherwise
+ */
+__rte_internal
+struct mlx5_hlist_entry *mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key,
+					     void *ctx);
+
+/**
+ * Remove an entry from the hash list table. User should guarantee the validity
+ * of the entry.
+ *
+ * @param h
+ *   Pointer to the hast list table. (not used)
+ * @param entry
+ *   Entry to be removed from the hash list table.
+ * @return
+ *   0 on entry removed, 1 on entry still referenced.
+ */
+__rte_internal
+int mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry);
+
+/**
+ * Destroy the hash list table, all the entries already inserted into the lists
+ * will be handled by the callback function provided by the user (including
+ * free if needed) before the table is freed.
+ *
+ * @param h
+ *   Pointer to the hast list table.
+ */
+__rte_internal
+void mlx5_hlist_destroy(struct mlx5_hlist *h);
 
 #endif /* RTE_PMD_MLX5_COMMON_UTILS_H_ */
diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c
index 5e082ebb78..c0061741e8 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.c
+++ b/drivers/common/mlx5/mlx5_devx_cmds.c
@@ -9,7 +9,7 @@
 
 #include "mlx5_prm.h"
 #include "mlx5_devx_cmds.h"
-#include "mlx5_common_utils.h"
+#include "mlx5_common_log.h"
 #include "mlx5_malloc.h"
 
 
diff --git a/drivers/common/mlx5/mlx5_malloc.c b/drivers/common/mlx5/mlx5_malloc.c
index 9d30cedbaa..b19501e1bc 100644
--- a/drivers/common/mlx5/mlx5_malloc.c
+++ b/drivers/common/mlx5/mlx5_malloc.c
@@ -8,7 +8,7 @@
 #include <stdbool.h>
 #include <string.h>
 
-#include "mlx5_common_utils.h"
+#include "mlx5_common_log.h"
 #include "mlx5_common_os.h"
 #include "mlx5_malloc.h"
 
diff --git a/drivers/common/mlx5/version.map b/drivers/common/mlx5/version.map
index 89f0ee04cb..40b0d713ac 100644
--- a/drivers/common/mlx5/version.map
+++ b/drivers/common/mlx5/version.map
@@ -70,6 +70,12 @@ INTERNAL {
 
 	mlx5_glue;
 
+	mlx5_hlist_create;
+	mlx5_hlist_lookup;
+	mlx5_hlist_register;
+	mlx5_hlist_unregister;
+	mlx5_hlist_destroy;
+
 	mlx5_malloc;
 	mlx5_malloc_mem_select;
 
diff --git a/drivers/common/mlx5/windows/mlx5_common_os.c b/drivers/common/mlx5/windows/mlx5_common_os.c
index cebf42d53c..78975d9d4d 100644
--- a/drivers/common/mlx5/windows/mlx5_common_os.c
+++ b/drivers/common/mlx5/windows/mlx5_common_os.c
@@ -11,7 +11,7 @@
 #include <rte_errno.h>
 
 #include "mlx5_devx_cmds.h"
-#include "mlx5_common_utils.h"
+#include "../mlx5_common_log.h"
 #include "mlx5_common.h"
 #include "mlx5_common_os.h"
 #include "mlx5_malloc.h"
diff --git a/drivers/common/mlx5/windows/mlx5_glue.c b/drivers/common/mlx5/windows/mlx5_glue.c
index aef6d3b5f4..535487a8d4 100644
--- a/drivers/common/mlx5/windows/mlx5_glue.c
+++ b/drivers/common/mlx5/windows/mlx5_glue.c
@@ -12,7 +12,7 @@
 #include <rte_malloc.h>
 
 #include "mlx5_glue.h"
-#include "mlx5_common_utils.h"
+#include "../mlx5_common_log.h"
 #include "mlx5_win_ext.h"
 
 /*
diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index a39b5edddc..18fe23e4fb 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -3,220 +3,11 @@
  */
 
 #include <rte_malloc.h>
-#include <rte_hash_crc.h>
 
 #include <mlx5_malloc.h>
 
 #include "mlx5_utils.h"
 
-/********************* Hash List **********************/
-
-static struct mlx5_hlist_entry *
-mlx5_hlist_default_create_cb(struct mlx5_hlist *h, uint64_t key __rte_unused,
-			     void *ctx __rte_unused)
-{
-	return mlx5_malloc(MLX5_MEM_ZERO, h->entry_sz, 0, SOCKET_ID_ANY);
-}
-
-static void
-mlx5_hlist_default_remove_cb(struct mlx5_hlist *h __rte_unused,
-			     struct mlx5_hlist_entry *entry)
-{
-	mlx5_free(entry);
-}
-
-struct mlx5_hlist *
-mlx5_hlist_create(const char *name, uint32_t size, uint32_t entry_size,
-		  uint32_t flags, mlx5_hlist_create_cb cb_create,
-		  mlx5_hlist_match_cb cb_match, mlx5_hlist_remove_cb cb_remove)
-{
-	struct mlx5_hlist *h;
-	uint32_t act_size;
-	uint32_t alloc_size;
-	uint32_t i;
-
-	if (!size || !cb_match || (!cb_create ^ !cb_remove))
-		return NULL;
-	/* Align to the next power of 2, 32bits integer is enough now. */
-	if (!rte_is_power_of_2(size)) {
-		act_size = rte_align32pow2(size);
-		DRV_LOG(DEBUG, "Size 0x%" PRIX32 " is not power of 2, "
-			"will be aligned to 0x%" PRIX32 ".", size, act_size);
-	} else {
-		act_size = size;
-	}
-	alloc_size = sizeof(struct mlx5_hlist) +
-		     sizeof(struct mlx5_hlist_bucket) * act_size;
-	/* Using zmalloc, then no need to initialize the heads. */
-	h = mlx5_malloc(MLX5_MEM_ZERO, alloc_size, RTE_CACHE_LINE_SIZE,
-			SOCKET_ID_ANY);
-	if (!h) {
-		DRV_LOG(ERR, "No memory for hash list %s creation",
-			name ? name : "None");
-		return NULL;
-	}
-	if (name)
-		snprintf(h->name, MLX5_HLIST_NAMESIZE, "%s", name);
-	h->table_sz = act_size;
-	h->mask = act_size - 1;
-	h->entry_sz = entry_size;
-	h->direct_key = !!(flags & MLX5_HLIST_DIRECT_KEY);
-	h->write_most = !!(flags & MLX5_HLIST_WRITE_MOST);
-	h->cb_create = cb_create ? cb_create : mlx5_hlist_default_create_cb;
-	h->cb_match = cb_match;
-	h->cb_remove = cb_remove ? cb_remove : mlx5_hlist_default_remove_cb;
-	for (i = 0; i < act_size; i++)
-		rte_rwlock_init(&h->buckets[i].lock);
-	DRV_LOG(DEBUG, "Hash list with %s size 0x%" PRIX32 " is created.",
-		h->name, act_size);
-	return h;
-}
-
-static struct mlx5_hlist_entry *
-__hlist_lookup(struct mlx5_hlist *h, uint64_t key, uint32_t idx,
-	       void *ctx, bool reuse)
-{
-	struct mlx5_hlist_head *first;
-	struct mlx5_hlist_entry *node;
-
-	MLX5_ASSERT(h);
-	first = &h->buckets[idx].head;
-	LIST_FOREACH(node, first, next) {
-		if (!h->cb_match(h, node, key, ctx)) {
-			if (reuse) {
-				__atomic_add_fetch(&node->ref_cnt, 1,
-						   __ATOMIC_RELAXED);
-				DRV_LOG(DEBUG, "Hash list %s entry %p "
-					"reuse: %u.",
-					h->name, (void *)node, node->ref_cnt);
-			}
-			break;
-		}
-	}
-	return node;
-}
-
-static struct mlx5_hlist_entry *
-hlist_lookup(struct mlx5_hlist *h, uint64_t key, uint32_t idx,
-	     void *ctx, bool reuse)
-{
-	struct mlx5_hlist_entry *node;
-
-	MLX5_ASSERT(h);
-	rte_rwlock_read_lock(&h->buckets[idx].lock);
-	node = __hlist_lookup(h, key, idx, ctx, reuse);
-	rte_rwlock_read_unlock(&h->buckets[idx].lock);
-	return node;
-}
-
-struct mlx5_hlist_entry *
-mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
-{
-	uint32_t idx;
-
-	if (h->direct_key)
-		idx = (uint32_t)(key & h->mask);
-	else
-		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	return hlist_lookup(h, key, idx, ctx, false);
-}
-
-struct mlx5_hlist_entry*
-mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
-{
-	uint32_t idx;
-	struct mlx5_hlist_head *first;
-	struct mlx5_hlist_bucket *b;
-	struct mlx5_hlist_entry *entry;
-	uint32_t prev_gen_cnt = 0;
-
-	if (h->direct_key)
-		idx = (uint32_t)(key & h->mask);
-	else
-		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	MLX5_ASSERT(h);
-	b = &h->buckets[idx];
-	/* Use write lock directly for write-most list. */
-	if (!h->write_most) {
-		prev_gen_cnt = __atomic_load_n(&b->gen_cnt, __ATOMIC_ACQUIRE);
-		entry = hlist_lookup(h, key, idx, ctx, true);
-		if (entry)
-			return entry;
-	}
-	rte_rwlock_write_lock(&b->lock);
-	/* Check if the list changed by other threads. */
-	if (h->write_most ||
-	    prev_gen_cnt != __atomic_load_n(&b->gen_cnt, __ATOMIC_ACQUIRE)) {
-		entry = __hlist_lookup(h, key, idx, ctx, true);
-		if (entry)
-			goto done;
-	}
-	first = &b->head;
-	entry = h->cb_create(h, key, ctx);
-	if (!entry) {
-		rte_errno = ENOMEM;
-		DRV_LOG(DEBUG, "Can't allocate hash list %s entry.", h->name);
-		goto done;
-	}
-	entry->idx = idx;
-	entry->ref_cnt = 1;
-	LIST_INSERT_HEAD(first, entry, next);
-	__atomic_add_fetch(&b->gen_cnt, 1, __ATOMIC_ACQ_REL);
-	DRV_LOG(DEBUG, "Hash list %s entry %p new: %u.",
-		h->name, (void *)entry, entry->ref_cnt);
-done:
-	rte_rwlock_write_unlock(&b->lock);
-	return entry;
-}
-
-int
-mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry)
-{
-	uint32_t idx = entry->idx;
-
-	rte_rwlock_write_lock(&h->buckets[idx].lock);
-	MLX5_ASSERT(entry && entry->ref_cnt && entry->next.le_prev);
-	DRV_LOG(DEBUG, "Hash list %s entry %p deref: %u.",
-		h->name, (void *)entry, entry->ref_cnt);
-	if (--entry->ref_cnt) {
-		rte_rwlock_write_unlock(&h->buckets[idx].lock);
-		return 1;
-	}
-	LIST_REMOVE(entry, next);
-	/* Set to NULL to get rid of removing action for more than once. */
-	entry->next.le_prev = NULL;
-	h->cb_remove(h, entry);
-	rte_rwlock_write_unlock(&h->buckets[idx].lock);
-	DRV_LOG(DEBUG, "Hash list %s entry %p removed.",
-		h->name, (void *)entry);
-	return 0;
-}
-
-void
-mlx5_hlist_destroy(struct mlx5_hlist *h)
-{
-	uint32_t idx;
-	struct mlx5_hlist_entry *entry;
-
-	MLX5_ASSERT(h);
-	for (idx = 0; idx < h->table_sz; ++idx) {
-		/* No LIST_FOREACH_SAFE, using while instead. */
-		while (!LIST_EMPTY(&h->buckets[idx].head)) {
-			entry = LIST_FIRST(&h->buckets[idx].head);
-			LIST_REMOVE(entry, next);
-			/*
-			 * The owner of whole element which contains data entry
-			 * is the user, so it's the user's duty to do the clean
-			 * up and the free work because someone may not put the
-			 * hlist entry at the beginning(suggested to locate at
-			 * the beginning). Or else the default free function
-			 * will be used.
-			 */
-			h->cb_remove(h, entry);
-		}
-	}
-	mlx5_free(h);
-}
 
 /********************* Cache list ************************/
 
diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h
index 289941cebc..b54517c6df 100644
--- a/drivers/net/mlx5/mlx5_utils.h
+++ b/drivers/net/mlx5/mlx5_utils.h
@@ -18,6 +18,7 @@
 #include <rte_bitmap.h>
 
 #include <mlx5_common.h>
+#include <mlx5_common_utils.h>
 
 #include "mlx5_defs.h"
 
@@ -261,199 +262,6 @@ log2above(unsigned int v)
 	return l + r;
 }
 
-#define MLX5_HLIST_DIRECT_KEY 0x0001 /* Use the key directly as hash index. */
-#define MLX5_HLIST_WRITE_MOST 0x0002 /* List mostly used for append new. */
-
-/** Maximum size of string for naming the hlist table. */
-#define MLX5_HLIST_NAMESIZE			32
-
-struct mlx5_hlist;
-
-/**
- * Structure of the entry in the hash list, user should define its own struct
- * that contains this in order to store the data. The 'key' is 64-bits right
- * now and its user's responsibility to guarantee there is no collision.
- */
-struct mlx5_hlist_entry {
-	LIST_ENTRY(mlx5_hlist_entry) next; /* entry pointers in the list. */
-	uint32_t idx; /* Bucket index the entry belongs to. */
-	uint32_t ref_cnt; /* Reference count. */
-};
-
-/** Structure for hash head. */
-LIST_HEAD(mlx5_hlist_head, mlx5_hlist_entry);
-
-/**
- * Type of callback function for entry removal.
- *
- * @param list
- *   The hash list.
- * @param entry
- *   The entry in the list.
- */
-typedef void (*mlx5_hlist_remove_cb)(struct mlx5_hlist *list,
-				     struct mlx5_hlist_entry *entry);
-
-/**
- * Type of function for user defined matching.
- *
- * @param list
- *   The hash list.
- * @param entry
- *   The entry in the list.
- * @param key
- *   The new entry key.
- * @param ctx
- *   The pointer to new entry context.
- *
- * @return
- *   0 if matching, non-zero number otherwise.
- */
-typedef int (*mlx5_hlist_match_cb)(struct mlx5_hlist *list,
-				   struct mlx5_hlist_entry *entry,
-				   uint64_t key, void *ctx);
-
-/**
- * Type of function for user defined hash list entry creation.
- *
- * @param list
- *   The hash list.
- * @param key
- *   The key of the new entry.
- * @param ctx
- *   The pointer to new entry context.
- *
- * @return
- *   Pointer to allocated entry on success, NULL otherwise.
- */
-typedef struct mlx5_hlist_entry *(*mlx5_hlist_create_cb)
-				  (struct mlx5_hlist *list,
-				   uint64_t key, void *ctx);
-
-/* Hash list bucket head. */
-struct mlx5_hlist_bucket {
-	struct mlx5_hlist_head head; /* List head. */
-	rte_rwlock_t lock; /* Bucket lock. */
-	uint32_t gen_cnt; /* List modification will update generation count. */
-} __rte_cache_aligned;
-
-/**
- * Hash list table structure
- *
- * Entry in hash list could be reused if entry already exists, reference
- * count will increase and the existing entry returns.
- *
- * When destroy an entry from list, decrease reference count and only
- * destroy when no further reference.
- */
-struct mlx5_hlist {
-	char name[MLX5_HLIST_NAMESIZE]; /**< Name of the hash list. */
-	/**< number of heads, need to be power of 2. */
-	uint32_t table_sz;
-	uint32_t entry_sz; /**< Size of entry, used to allocate entry. */
-	/**< mask to get the index of the list heads. */
-	uint32_t mask;
-	bool direct_key; /* Use the new entry key directly as hash index. */
-	bool write_most; /* List mostly used for append new or destroy. */
-	void *ctx;
-	mlx5_hlist_create_cb cb_create; /**< entry create callback. */
-	mlx5_hlist_match_cb cb_match; /**< entry match callback. */
-	mlx5_hlist_remove_cb cb_remove; /**< entry remove callback. */
-	struct mlx5_hlist_bucket buckets[] __rte_cache_aligned;
-	/**< list bucket arrays. */
-};
-
-/**
- * Create a hash list table, the user can specify the list heads array size
- * of the table, now the size should be a power of 2 in order to get better
- * distribution for the entries. Each entry is a part of the whole data element
- * and the caller should be responsible for the data element's allocation and
- * cleanup / free. Key of each entry will be calculated with CRC in order to
- * generate a little fairer distribution.
- *
- * @param name
- *   Name of the hash list(optional).
- * @param size
- *   Heads array size of the hash list.
- * @param entry_size
- *   Entry size to allocate if cb_create not specified.
- * @param flags
- *   The hash list attribute flags.
- * @param cb_create
- *   Callback function for entry create.
- * @param cb_match
- *   Callback function for entry match.
- * @param cb_destroy
- *   Callback function for entry destroy.
- * @return
- *   Pointer of the hash list table created, NULL on failure.
- */
-struct mlx5_hlist *mlx5_hlist_create(const char *name, uint32_t size,
-				     uint32_t entry_size, uint32_t flags,
-				     mlx5_hlist_create_cb cb_create,
-				     mlx5_hlist_match_cb cb_match,
-				     mlx5_hlist_remove_cb cb_destroy);
-
-/**
- * Search an entry matching the key.
- *
- * Result returned might be destroyed by other thread, must use
- * this function only in main thread.
- *
- * @param h
- *   Pointer to the hast list table.
- * @param key
- *   Key for the searching entry.
- * @param ctx
- *   Common context parameter used by entry callback function.
- *
- * @return
- *   Pointer of the hlist entry if found, NULL otherwise.
- */
-struct mlx5_hlist_entry *mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key,
-					   void *ctx);
-
-/**
- * Insert an entry to the hash list table, the entry is only part of whole data
- * element and a 64B key is used for matching. User should construct the key or
- * give a calculated hash signature and guarantee there is no collision.
- *
- * @param h
- *   Pointer to the hast list table.
- * @param entry
- *   Entry to be inserted into the hash list table.
- * @param ctx
- *   Common context parameter used by callback function.
- *
- * @return
- *   registered entry on success, NULL otherwise
- */
-struct mlx5_hlist_entry *mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key,
-					     void *ctx);
-
-/**
- * Remove an entry from the hash list table. User should guarantee the validity
- * of the entry.
- *
- * @param h
- *   Pointer to the hast list table. (not used)
- * @param entry
- *   Entry to be removed from the hash list table.
- * @return
- *   0 on entry removed, 1 on entry still referenced.
- */
-int mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry);
-
-/**
- * Destroy the hash list table, all the entries already inserted into the lists
- * will be handled by the callback function provided by the user (including
- * free if needed) before the table is freed.
- *
- * @param h
- *   Pointer to the hast list table.
- */
-void mlx5_hlist_destroy(struct mlx5_hlist *h);
-
 /************************ cache list *****************************/
 
 /** Maximum size of string for naming. */
-- 
2.25.1

[dpdk-dev] [PATCH v3 11/15] common/mlx5: share get ib device match function

[Matan Azrad]

From: Shiri Kuzin <shirik@nvidia.com>

The get_ib_device_match function iterates over the list of ib devices
returned by the get_device_list glue function and returns the ib device
matching the provided address.

Since this function is in use by several drivers, in this patch we
share the function in common part.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/linux/mlx5_common_os.c | 28 +++++++++++++++++++
 drivers/common/mlx5/linux/mlx5_common_os.h |  5 ++++
 drivers/common/mlx5/mlx5_common.h          |  1 +
 drivers/common/mlx5/version.map            |  1 +
 drivers/compress/mlx5/mlx5_compress.c      | 30 +-------------------
 drivers/regex/mlx5/mlx5_regex.c            | 30 ++------------------
 drivers/vdpa/mlx5/mlx5_vdpa.c              | 32 ++--------------------
 7 files changed, 40 insertions(+), 87 deletions(-)

diff --git a/drivers/common/mlx5/linux/mlx5_common_os.c b/drivers/common/mlx5/linux/mlx5_common_os.c
index fba8245b8b..ea0b71e425 100644
--- a/drivers/common/mlx5/linux/mlx5_common_os.c
+++ b/drivers/common/mlx5/linux/mlx5_common_os.c
@@ -16,6 +16,7 @@
 
 #include "mlx5_common.h"
 #include "mlx5_common_log.h"
+#include "mlx5_common_os.h"
 #include "mlx5_glue.h"
 
 #ifdef MLX5_GLUE
@@ -423,3 +424,30 @@ mlx5_glue_constructor(void)
 	mlx5_glue = NULL;
 }
 
+struct ibv_device *
+mlx5_os_get_ibv_device(struct rte_pci_addr *addr)
+{
+	int n;
+	struct ibv_device **ibv_list = mlx5_glue->get_device_list(&n);
+	struct ibv_device *ibv_match = NULL;
+
+	if (ibv_list == NULL) {
+		rte_errno = ENOSYS;
+		return NULL;
+	}
+	while (n-- > 0) {
+		struct rte_pci_addr paddr;
+
+		DRV_LOG(DEBUG, "Checking device \"%s\"..", ibv_list[n]->name);
+		if (mlx5_dev_to_pci_addr(ibv_list[n]->ibdev_path, &paddr) != 0)
+			continue;
+		if (rte_pci_addr_cmp(addr, &paddr) != 0)
+			continue;
+		ibv_match = ibv_list[n];
+		break;
+	}
+	if (ibv_match == NULL)
+		rte_errno = ENOENT;
+	mlx5_glue->free_device_list(ibv_list);
+	return ibv_match;
+}
diff --git a/drivers/common/mlx5/linux/mlx5_common_os.h b/drivers/common/mlx5/linux/mlx5_common_os.h
index d1c7e3dce0..72d6bf828b 100644
--- a/drivers/common/mlx5/linux/mlx5_common_os.h
+++ b/drivers/common/mlx5/linux/mlx5_common_os.h
@@ -288,4 +288,9 @@ mlx5_os_free(void *addr)
 {
 	free(addr);
 }
+
+__rte_internal
+struct ibv_device *
+mlx5_os_get_ibv_device(struct rte_pci_addr *addr);
+
 #endif /* RTE_PMD_MLX5_COMMON_OS_H_ */
diff --git a/drivers/common/mlx5/mlx5_common.h b/drivers/common/mlx5/mlx5_common.h
index f3c6beb23b..1fbefe0fa6 100644
--- a/drivers/common/mlx5/mlx5_common.h
+++ b/drivers/common/mlx5/mlx5_common.h
@@ -18,6 +18,7 @@
 
 #include "mlx5_prm.h"
 #include "mlx5_devx_cmds.h"
+#include "mlx5_common_os.h"
 
 /* Reported driver name. */
 #define MLX5_PCI_DRIVER_NAME "mlx5_pci"
diff --git a/drivers/common/mlx5/version.map b/drivers/common/mlx5/version.map
index 40b0d713ac..5706c6bbef 100644
--- a/drivers/common/mlx5/version.map
+++ b/drivers/common/mlx5/version.map
@@ -126,6 +126,7 @@ INTERNAL {
 	mlx5_os_alloc_pd;
 	mlx5_os_dealloc_pd;
 	mlx5_os_dereg_mr;
+	mlx5_os_get_ibv_device; # WINDOWS_NO_EXPORT
 	mlx5_os_reg_mr;
 	mlx5_os_umem_dereg;
 	mlx5_os_umem_reg;
diff --git a/drivers/compress/mlx5/mlx5_compress.c b/drivers/compress/mlx5/mlx5_compress.c
index ec3c237512..0581dee505 100644
--- a/drivers/compress/mlx5/mlx5_compress.c
+++ b/drivers/compress/mlx5/mlx5_compress.c
@@ -647,34 +647,6 @@ mlx5_compress_dequeue_burst(void *queue_pair, struct rte_comp_op **ops,
 	return i;
 }
 
-static struct ibv_device *
-mlx5_compress_get_ib_device_match(struct rte_pci_addr *addr)
-{
-	int n;
-	struct ibv_device **ibv_list = mlx5_glue->get_device_list(&n);
-	struct ibv_device *ibv_match = NULL;
-
-	if (ibv_list == NULL) {
-		rte_errno = ENOSYS;
-		return NULL;
-	}
-	while (n-- > 0) {
-		struct rte_pci_addr paddr;
-
-		DRV_LOG(DEBUG, "Checking device \"%s\"..", ibv_list[n]->name);
-		if (mlx5_dev_to_pci_addr(ibv_list[n]->ibdev_path, &paddr) != 0)
-			continue;
-		if (rte_pci_addr_cmp(addr, &paddr) != 0)
-			continue;
-		ibv_match = ibv_list[n];
-		break;
-	}
-	if (ibv_match == NULL)
-		rte_errno = ENOENT;
-	mlx5_glue->free_device_list(ibv_list);
-	return ibv_match;
-}
-
 static void
 mlx5_compress_hw_global_release(struct mlx5_compress_priv *priv)
 {
@@ -774,7 +746,7 @@ mlx5_compress_pci_probe(struct rte_pci_driver *pci_drv,
 		rte_errno = ENOTSUP;
 		return -rte_errno;
 	}
-	ibv = mlx5_compress_get_ib_device_match(&pci_dev->addr);
+	ibv = mlx5_os_get_ibv_device(&pci_dev->addr);
 	if (ibv == NULL) {
 		DRV_LOG(ERR, "No matching IB device for PCI slot "
 			PCI_PRI_FMT ".", pci_dev->addr.domain,
diff --git a/drivers/regex/mlx5/mlx5_regex.c b/drivers/regex/mlx5/mlx5_regex.c
index 82c485e50c..dd96436d42 100644
--- a/drivers/regex/mlx5/mlx5_regex.c
+++ b/drivers/regex/mlx5/mlx5_regex.c
@@ -11,6 +11,7 @@
 #include <rte_regexdev_driver.h>
 
 #include <mlx5_common_pci.h>
+#include <mlx5_common.h>
 #include <mlx5_glue.h>
 #include <mlx5_devx_cmds.h>
 #include <mlx5_prm.h>
@@ -52,33 +53,6 @@ mlx5_regex_close(struct rte_regexdev *dev __rte_unused)
 	return 0;
 }
 
-static struct ibv_device *
-mlx5_regex_get_ib_device_match(struct rte_pci_addr *addr)
-{
-	int n;
-	struct ibv_device **ibv_list = mlx5_glue->get_device_list(&n);
-	struct ibv_device *ibv_match = NULL;
-
-	if (!ibv_list) {
-		rte_errno = ENOSYS;
-		return NULL;
-	}
-	while (n-- > 0) {
-		struct rte_pci_addr pci_addr;
-
-		DRV_LOG(DEBUG, "Checking device \"%s\"..", ibv_list[n]->name);
-		if (mlx5_dev_to_pci_addr(ibv_list[n]->ibdev_path, &pci_addr))
-			continue;
-		if (rte_pci_addr_cmp(addr, &pci_addr))
-			continue;
-		ibv_match = ibv_list[n];
-		break;
-	}
-	if (!ibv_match)
-		rte_errno = ENOENT;
-	mlx5_glue->free_device_list(ibv_list);
-	return ibv_match;
-}
 static int
 mlx5_regex_engines_status(struct ibv_context *ctx, int num_engines)
 {
@@ -121,7 +95,7 @@ mlx5_regex_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 	int ret;
 	uint32_t val;
 
-	ibv = mlx5_regex_get_ib_device_match(&pci_dev->addr);
+	ibv = mlx5_os_get_ibv_device(&pci_dev->addr);
 	if (!ibv) {
 		DRV_LOG(ERR, "No matching IB device for PCI slot "
 			PCI_PRI_FMT ".", pci_dev->addr.domain,
diff --git a/drivers/vdpa/mlx5/mlx5_vdpa.c b/drivers/vdpa/mlx5/mlx5_vdpa.c
index 898e50f807..17753ae63f 100644
--- a/drivers/vdpa/mlx5/mlx5_vdpa.c
+++ b/drivers/vdpa/mlx5/mlx5_vdpa.c
@@ -472,34 +472,6 @@ static struct rte_vdpa_dev_ops mlx5_vdpa_ops = {
 	.reset_stats = mlx5_vdpa_reset_stats,
 };
 
-static struct ibv_device *
-mlx5_vdpa_get_ib_device_match(struct rte_pci_addr *addr)
-{
-	int n;
-	struct ibv_device **ibv_list = mlx5_glue->get_device_list(&n);
-	struct ibv_device *ibv_match = NULL;
-
-	if (!ibv_list) {
-		rte_errno = ENOSYS;
-		return NULL;
-	}
-	while (n-- > 0) {
-		struct rte_pci_addr pci_addr;
-
-		DRV_LOG(DEBUG, "Checking device \"%s\"..", ibv_list[n]->name);
-		if (mlx5_dev_to_pci_addr(ibv_list[n]->ibdev_path, &pci_addr))
-			continue;
-		if (rte_pci_addr_cmp(addr, &pci_addr))
-			continue;
-		ibv_match = ibv_list[n];
-		break;
-	}
-	if (!ibv_match)
-		rte_errno = ENOENT;
-	mlx5_glue->free_device_list(ibv_list);
-	return ibv_match;
-}
-
 /* Try to disable ROCE by Netlink\Devlink. */
 static int
 mlx5_vdpa_nl_roce_disable(const char *addr)
@@ -595,7 +567,7 @@ mlx5_vdpa_roce_disable(struct rte_pci_addr *addr, struct ibv_device **ibv)
 		struct ibv_device *ibv_new;
 
 		for (r = MLX5_VDPA_MAX_RETRIES; r; r--) {
-			ibv_new = mlx5_vdpa_get_ib_device_match(addr);
+			ibv_new = mlx5_os_get_ibv_device(addr);
 			if (ibv_new) {
 				*ibv = ibv_new;
 				return 0;
@@ -698,7 +670,7 @@ mlx5_vdpa_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 	struct mlx5_hca_attr attr;
 	int ret;
 
-	ibv = mlx5_vdpa_get_ib_device_match(&pci_dev->addr);
+	ibv = mlx5_os_get_ibv_device(&pci_dev->addr);
 	if (!ibv) {
 		DRV_LOG(ERR, "No matching IB device for PCI slot "
 			PCI_PRI_FMT ".", pci_dev->addr.domain,
-- 
2.25.1

[dpdk-dev] [PATCH v3 12/15] common/mlx5: support general obj CREDENTIAL create

[Matan Azrad]

From: Dekel Peled <dekelp@nvidia.com>

CREDENTIAL object is used for any crypto operation in wrapped mode.

This patch add support of CREDENTIAL object create operation.
Add reading of CREDENTIAL support capability.
Add function to create general object type CREDENTIAL, using DevX API.

Signed-off-by: Dekel Peled <dekelp@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_devx_cmds.c | 53 +++++++++++++++++++++++++++-
 drivers/common/mlx5/mlx5_devx_cmds.h | 17 +++++++--
 drivers/common/mlx5/mlx5_prm.h       | 23 ++++++++++++
 drivers/common/mlx5/version.map      |  1 +
 4 files changed, 91 insertions(+), 3 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c
index c0061741e8..c0a0853c3a 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.c
+++ b/drivers/common/mlx5/mlx5_devx_cmds.c
@@ -754,6 +754,8 @@ mlx5_devx_cmd_query_hca_attr(void *ctx,
 		       MLX5_GENERAL_OBJ_TYPES_CAP_DEK);
 	attr->import_kek = !!(general_obj_types_supported &
 			      MLX5_GENERAL_OBJ_TYPES_CAP_IMPORT_KEK);
+	attr->credential = !!(general_obj_types_supported &
+			      MLX5_GENERAL_OBJ_TYPES_CAP_CREDENTIAL);
 	attr->crypto_login = !!(general_obj_types_supported &
 				MLX5_GENERAL_OBJ_TYPES_CAP_CRYPTO_LOGIN);
 	/* Add reading of other GENERAL_OBJ_TYPES_CAP bits above this line. */
@@ -2512,6 +2514,55 @@ mlx5_devx_cmd_create_import_kek_obj(void *ctx,
 	return import_kek_obj;
 }
 
+/**
+ * Create general object of type CREDENTIAL using DevX API.
+ *
+ * @param[in] ctx
+ *   Context returned from mlx5 open_device() glue function.
+ * @param [in] attr
+ *   Pointer to CREDENTIAL attributes structure.
+ *
+ * @return
+ *   The DevX object created, NULL otherwise and rte_errno is set.
+ */
+struct mlx5_devx_obj *
+mlx5_devx_cmd_create_credential_obj(void *ctx,
+				    struct mlx5_devx_credential_attr *attr)
+{
+	uint32_t in[MLX5_ST_SZ_DW(create_credential_in)] = {0};
+	uint32_t out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};
+	struct mlx5_devx_obj *credential_obj = NULL;
+	void *ptr = NULL, *credential_addr = NULL;
+
+	credential_obj = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*credential_obj),
+				     0, SOCKET_ID_ANY);
+	if (credential_obj == NULL) {
+		DRV_LOG(ERR, "Failed to allocate CREDENTIAL object data");
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+	ptr = MLX5_ADDR_OF(create_credential_in, in, hdr);
+	MLX5_SET(general_obj_in_cmd_hdr, ptr, opcode,
+		 MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
+	MLX5_SET(general_obj_in_cmd_hdr, ptr, obj_type,
+		 MLX5_GENERAL_OBJ_TYPE_CREDENTIAL);
+	ptr = MLX5_ADDR_OF(create_credential_in, in, credential);
+	MLX5_SET(credential, ptr, credential_role, attr->credential_role);
+	credential_addr = MLX5_ADDR_OF(credential, ptr, credential);
+	memcpy(credential_addr, (void *)(attr->credential),
+	       MLX5_CRYPTO_CREDENTIAL_SIZE);
+	credential_obj->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in),
+							 out, sizeof(out));
+	if (credential_obj->obj == NULL) {
+		rte_errno = errno;
+		DRV_LOG(ERR, "Failed to create CREDENTIAL object using DevX.");
+		mlx5_free(credential_obj);
+		return NULL;
+	}
+	credential_obj->id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
+	return credential_obj;
+}
+
 /**
  * Create general object of type CRYPTO_LOGIN using DevX API.
  *
@@ -2551,7 +2602,7 @@ mlx5_devx_cmd_create_crypto_login_obj(void *ctx,
 		 attr->session_import_kek_ptr);
 	credential_addr = MLX5_ADDR_OF(crypto_login, ptr, credential);
 	memcpy(credential_addr, (void *)(attr->credential),
-	       MLX5_CRYPTO_LOGIN_CREDENTIAL_SIZE);
+	       MLX5_CRYPTO_CREDENTIAL_SIZE);
 	crypto_login_obj->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in),
 							   out, sizeof(out));
 	if (crypto_login_obj->obj == NULL) {
diff --git a/drivers/common/mlx5/mlx5_devx_cmds.h b/drivers/common/mlx5/mlx5_devx_cmds.h
index 709e28bfba..811e7a1462 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.h
+++ b/drivers/common/mlx5/mlx5_devx_cmds.h
@@ -143,6 +143,7 @@ struct mlx5_hca_attr {
 	uint32_t aes_xts:1; /* AES-XTS crypto is supported. */
 	uint32_t dek:1; /* General obj type DEK is supported. */
 	uint32_t import_kek:1; /* General obj type IMPORT_KEK supported. */
+	uint32_t credential:1; /* General obj type CREDENTIAL supported. */
 	uint32_t crypto_login:1; /* General obj type CRYPTO_LOGIN supported. */
 	uint32_t regexp_num_of_engines;
 	uint32_t log_max_ft_sampler_num:8;
@@ -459,13 +460,20 @@ struct mlx5_devx_import_kek_attr {
 	uint8_t key[MLX5_CRYPTO_KEY_MAX_SIZE];
 };
 
-#define MLX5_CRYPTO_LOGIN_CREDENTIAL_SIZE	48
+#define MLX5_CRYPTO_CREDENTIAL_SIZE	48
+
+struct mlx5_devx_credential_attr {
+	uint64_t modify_field_select;
+	uint32_t state:8;
+	uint32_t credential_role:8;
+	uint8_t credential[MLX5_CRYPTO_CREDENTIAL_SIZE];
+};
 
 struct mlx5_devx_crypto_login_attr {
 	uint64_t modify_field_select;
 	uint32_t credential_pointer:24;
 	uint32_t session_import_kek_ptr:24;
-	uint8_t credential[MLX5_CRYPTO_LOGIN_CREDENTIAL_SIZE];
+	uint8_t credential[MLX5_CRYPTO_CREDENTIAL_SIZE];
 };
 
 /* mlx5_devx_cmds.c */
@@ -629,6 +637,11 @@ struct mlx5_devx_obj *
 mlx5_devx_cmd_create_import_kek_obj(void *ctx,
 				    struct mlx5_devx_import_kek_attr *attr);
 
+__rte_internal
+struct mlx5_devx_obj *
+mlx5_devx_cmd_create_credential_obj(void *ctx,
+				    struct mlx5_devx_credential_attr *attr);
+
 __rte_internal
 struct mlx5_devx_obj *
 mlx5_devx_cmd_create_crypto_login_obj(void *ctx,
diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index a9dcbfa63c..432c8fdb63 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -1187,6 +1187,8 @@ enum {
 			(1ULL << MLX5_GENERAL_OBJ_TYPE_DEK)
 #define MLX5_GENERAL_OBJ_TYPES_CAP_IMPORT_KEK \
 			(1ULL << MLX5_GENERAL_OBJ_TYPE_IMPORT_KEK)
+#define MLX5_GENERAL_OBJ_TYPES_CAP_CREDENTIAL \
+			(1ULL << MLX5_GENERAL_OBJ_TYPE_CREDENTIAL)
 #define MLX5_GENERAL_OBJ_TYPES_CAP_CRYPTO_LOGIN \
 			(1ULL << MLX5_GENERAL_OBJ_TYPE_CRYPTO_LOGIN)
 
@@ -2490,6 +2492,7 @@ enum {
 	MLX5_GENERAL_OBJ_TYPE_VIRTQ = 0x000d,
 	MLX5_GENERAL_OBJ_TYPE_VIRTIO_Q_COUNTERS = 0x001c,
 	MLX5_GENERAL_OBJ_TYPE_IMPORT_KEK = 0x001d,
+	MLX5_GENERAL_OBJ_TYPE_CREDENTIAL = 0x001e,
 	MLX5_GENERAL_OBJ_TYPE_CRYPTO_LOGIN = 0x001f,
 	MLX5_GENERAL_OBJ_TYPE_FLEX_PARSE_GRAPH = 0x0022,
 	MLX5_GENERAL_OBJ_TYPE_FLOW_METER_ASO = 0x0024,
@@ -2603,6 +2606,26 @@ struct mlx5_ifc_create_import_kek_in_bits {
 	struct mlx5_ifc_import_kek_bits import_kek;
 };
 
+enum {
+	MLX5_CREDENTIAL_ROLE_OFFICER = 0x0,
+	MLX5_CREDENTIAL_ROLE_USER = 0x1,
+};
+
+struct mlx5_ifc_credential_bits {
+	u8 modify_field_select[0x40];
+	u8 state[0x8];
+	u8 reserved_at_48[0x10];
+	u8 credential_role[0x8];
+	u8 reserved_at_60[0x1a0];
+	u8 credential[0x180];
+	u8 reserved_at_380[0x480];
+};
+
+struct mlx5_ifc_create_credential_in_bits {
+	struct mlx5_ifc_general_obj_in_cmd_hdr_bits hdr;
+	struct mlx5_ifc_credential_bits credential;
+};
+
 struct mlx5_ifc_crypto_login_bits {
 	u8 modify_field_select[0x40];
 	u8 reserved_at_40[0x48];
diff --git a/drivers/common/mlx5/version.map b/drivers/common/mlx5/version.map
index 5706c6bbef..04b2179b2c 100644
--- a/drivers/common/mlx5/version.map
+++ b/drivers/common/mlx5/version.map
@@ -16,6 +16,7 @@ INTERNAL {
 
 	mlx5_devx_cmd_alloc_pd;
 	mlx5_devx_cmd_create_cq;
+	mlx5_devx_cmd_create_credential_obj;
 	mlx5_devx_cmd_create_crypto_login_obj;
 	mlx5_devx_cmd_create_dek_obj;
 	mlx5_devx_cmd_create_flex_parser;
-- 
2.25.1

[dpdk-dev] [PATCH v3 13/15] common/mlx5: add crypto register structs and defs

[Matan Azrad]

From: Dekel Peled <dekelp@nvidia.com>

Encryption key management requires use of several related registers.
This patch adds the relevant structs and values, according to PRM
definitions.

Signed-off-by: Dekel Peled <dekelp@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_prm.h | 41 ++++++++++++++++++++++++++++++++++
 1 file changed, 41 insertions(+)

diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index 432c8fdb63..c2cd2d9f70 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -3307,6 +3307,10 @@ enum {
 
 enum {
 	MLX5_REGISTER_ID_MTUTC  = 0x9055,
+	MLX5_CRYPTO_OPERATIONAL_REGISTER_ID = 0xC002,
+	MLX5_CRYPTO_COMMISSIONING_REGISTER_ID = 0xC003,
+	MLX5_IMPORT_KEK_HANDLE_REGISTER_ID = 0xC004,
+	MLX5_CREDENTIAL_HANDLE_REGISTER_ID = 0xC005,
 };
 
 struct mlx5_ifc_register_mtutc_bits {
@@ -3324,6 +3328,43 @@ struct mlx5_ifc_register_mtutc_bits {
 #define MLX5_MTUTC_TIMESTAMP_MODE_INTERNAL_TIMER 0
 #define MLX5_MTUTC_TIMESTAMP_MODE_REAL_TIME 1
 
+struct mlx5_ifc_crypto_operational_register_bits {
+	u8 wrapped_crypto_operational[0x1];
+	u8 reserved_at_1[0x1b];
+	u8 kek_size[0x4];
+	u8 reserved_at_20[0x20];
+	u8 credential[0x140];
+	u8 kek[0x100];
+	u8 reserved_at_280[0x180];
+};
+
+struct mlx5_ifc_crypto_commissioning_register_bits {
+	u8 token[0x1]; /* TODO: add size after PRM update */
+};
+
+struct mlx5_ifc_import_kek_handle_register_bits {
+	struct mlx5_ifc_crypto_login_bits crypto_login_object;
+	struct mlx5_ifc_import_kek_bits import_kek_object;
+	u8 reserved_at_200[0x4];
+	u8 write_operation[0x4];
+	u8 import_kek_id[0x18];
+	u8 reserved_at_220[0xe0];
+};
+
+struct mlx5_ifc_credential_handle_register_bits {
+	struct mlx5_ifc_crypto_login_bits crypto_login_object;
+	struct mlx5_ifc_credential_bits credential_object;
+	u8 reserved_at_200[0x4];
+	u8 write_operation[0x4];
+	u8 credential_id[0x18];
+	u8 reserved_at_220[0xe0];
+};
+
+enum {
+	MLX5_REGISTER_ADD_OPERATION = 0x1,
+	MLX5_REGISTER_DELETE_OPERATION = 0x2,
+};
+
 struct mlx5_ifc_parse_graph_arc_bits {
 	u8 start_inner_tunnel[0x1];
 	u8 reserved_at_1[0x7];
-- 
2.25.1

[dpdk-dev] [PATCH v3 14/15] common/mlx5: support register write access

[Matan Azrad]

From: Dekel Peled <dekelp@nvidia.com>

This patch adds support of write operation to NIC registers.

Signed-off-by: Dekel Peled <dekelp@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_devx_cmds.c | 67 +++++++++++++++++++++++++++-
 drivers/common/mlx5/mlx5_devx_cmds.h |  4 ++
 drivers/common/mlx5/version.map      |  1 +
 3 files changed, 70 insertions(+), 2 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c
index c0a0853c3a..0b421933ce 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.c
+++ b/drivers/common/mlx5/mlx5_devx_cmds.c
@@ -12,7 +12,6 @@
 #include "mlx5_common_log.h"
 #include "mlx5_malloc.h"
 
-
 /**
  * Perform read access to the registers. Reads data from register
  * and writes ones to the specified buffer.
@@ -61,7 +60,7 @@ mlx5_devx_cmd_register_read(void *ctx, uint16_t reg_id, uint32_t arg,
 	if (status) {
 		int syndrome = MLX5_GET(access_register_out, out, syndrome);
 
-		DRV_LOG(DEBUG, "Failed to access NIC register 0x%X, "
+		DRV_LOG(DEBUG, "Failed to read access NIC register 0x%X, "
 			       "status %x, syndrome = %x",
 			       reg_id, status, syndrome);
 		return -1;
@@ -74,6 +73,70 @@ mlx5_devx_cmd_register_read(void *ctx, uint16_t reg_id, uint32_t arg,
 	return rc;
 }
 
+/**
+ * Perform write access to the registers.
+ *
+ * @param[in] ctx
+ *   Context returned from mlx5 open_device() glue function.
+ * @param[in] reg_id
+ *   Register identifier according to the PRM.
+ * @param[in] arg
+ *   Register access auxiliary parameter according to the PRM.
+ * @param[out] data
+ *   Pointer to the buffer containing data to write.
+ * @param[in] dw_cnt
+ *   Buffer size in double words (32bit units).
+ *
+ * @return
+ *   0 on success, a negative value otherwise.
+ */
+int
+mlx5_devx_cmd_register_write(void *ctx, uint16_t reg_id, uint32_t arg,
+			     uint32_t *data, uint32_t dw_cnt)
+{
+	uint32_t in[MLX5_ST_SZ_DW(access_register_in) +
+		    MLX5_ACCESS_REGISTER_DATA_DWORD_MAX] = {0};
+	uint32_t out[MLX5_ST_SZ_DW(access_register_out)] = {0};
+	int status, rc;
+	void *ptr;
+
+	MLX5_ASSERT(data && dw_cnt);
+	MLX5_ASSERT(dw_cnt <= MLX5_ACCESS_REGISTER_DATA_DWORD_MAX);
+	if (dw_cnt > MLX5_ACCESS_REGISTER_DATA_DWORD_MAX) {
+		DRV_LOG(ERR, "Data to write exceeds max size");
+		return -1;
+	}
+	MLX5_SET(access_register_in, in, opcode,
+		 MLX5_CMD_OP_ACCESS_REGISTER_USER);
+	MLX5_SET(access_register_in, in, op_mod,
+		 MLX5_ACCESS_REGISTER_IN_OP_MOD_WRITE);
+	MLX5_SET(access_register_in, in, register_id, reg_id);
+	MLX5_SET(access_register_in, in, argument, arg);
+	ptr = MLX5_ADDR_OF(access_register_in, in, register_data);
+	memcpy(ptr, data, dw_cnt * sizeof(uint32_t));
+	rc = mlx5_glue->devx_general_cmd(ctx, in, sizeof(in), out, sizeof(out));
+
+	rc = mlx5_glue->devx_general_cmd(ctx, in,
+					 MLX5_ST_SZ_BYTES(access_register_in) +
+					 dw_cnt * sizeof(uint32_t),
+					 out, sizeof(out));
+	if (rc)
+		goto error;
+	status = MLX5_GET(access_register_out, out, status);
+	if (status) {
+		int syndrome = MLX5_GET(access_register_out, out, syndrome);
+
+		DRV_LOG(DEBUG, "Failed to write access NIC register 0x%X, "
+			       "status %x, syndrome = %x",
+			       reg_id, status, syndrome);
+		return -1;
+	}
+	return 0;
+error:
+	rc = (rc > 0) ? -rc : rc;
+	return rc;
+}
+
 /**
  * Allocate flow counters via devx interface.
  *
diff --git a/drivers/common/mlx5/mlx5_devx_cmds.h b/drivers/common/mlx5/mlx5_devx_cmds.h
index 811e7a1462..ce570ad28a 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.h
+++ b/drivers/common/mlx5/mlx5_devx_cmds.h
@@ -566,6 +566,10 @@ __rte_internal
 int mlx5_devx_cmd_register_read(void *ctx, uint16_t reg_id,
 				uint32_t arg, uint32_t *data, uint32_t dw_cnt);
 
+__rte_internal
+int mlx5_devx_cmd_register_write(void *ctx, uint16_t reg_id,
+				 uint32_t arg, uint32_t *data, uint32_t dw_cnt);
+
 __rte_internal
 struct mlx5_devx_obj *
 mlx5_devx_cmd_create_geneve_tlv_option(void *ctx,
diff --git a/drivers/common/mlx5/version.map b/drivers/common/mlx5/version.map
index 04b2179b2c..c630696213 100644
--- a/drivers/common/mlx5/version.map
+++ b/drivers/common/mlx5/version.map
@@ -53,6 +53,7 @@ INTERNAL {
 	mlx5_devx_cmd_queue_counter_alloc; # WINDOWS_NO_EXPORT
 	mlx5_devx_cmd_queue_counter_query; # WINDOWS_NO_EXPORT
 	mlx5_devx_cmd_register_read;
+	mlx5_devx_cmd_register_write;
 	mlx5_devx_cmd_wq_query; # WINDOWS_NO_EXPORT
 
 	mlx5_devx_cq_create;
-- 
2.25.1

[dpdk-dev] [PATCH v3 15/15] common/mlx5: add UMR and RDMA write WQE defines

[Matan Azrad]

From: Suanming Mou <suanmingm@nvidia.com>

This patch adds the struct defining UMR and RDMA write WQEs.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/mlx5_prm.h | 187 +++++++++++++++++++++------------
 1 file changed, 121 insertions(+), 66 deletions(-)

diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index c2cd2d9f70..1ffee5fd56 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -412,6 +412,127 @@ struct mlx5_cqe_ts {
 	uint8_t op_own;
 };
 
+struct mlx5_wqe_rseg {
+	uint64_t raddr;
+	uint32_t rkey;
+	uint32_t reserved;
+} __rte_packed;
+
+#define MLX5_UMRC_IF_OFFSET 31u
+#define MLX5_UMRC_KO_OFFSET 16u
+#define MLX5_UMRC_TO_BS_OFFSET 0u
+
+struct mlx5_wqe_umr_cseg {
+	uint32_t if_cf_toe_cq_res;
+	uint32_t ko_to_bs;
+	uint64_t mkey_mask;
+	uint32_t rsvd1[8];
+} __rte_packed;
+
+struct mlx5_wqe_mkey_cseg {
+	uint32_t fr_res_af_sf;
+	uint32_t qpn_mkey;
+	uint32_t reserved2;
+	uint32_t flags_pd;
+	uint64_t start_addr;
+	uint64_t len;
+	uint32_t bsf_octword_size;
+	uint32_t reserved3[4];
+	uint32_t translations_octword_size;
+	uint32_t res4_lps;
+	uint32_t reserved;
+} __rte_packed;
+
+enum {
+	MLX5_BSF_SIZE_16B = 0x0,
+	MLX5_BSF_SIZE_32B = 0x1,
+	MLX5_BSF_SIZE_64B = 0x2,
+	MLX5_BSF_SIZE_128B = 0x3,
+};
+
+enum {
+	MLX5_BSF_P_TYPE_SIGNATURE = 0x0,
+	MLX5_BSF_P_TYPE_CRYPTO = 0x1,
+};
+
+enum {
+	MLX5_ENCRYPTION_ORDER_ENCRYPTED_WIRE_SIGNATURE = 0x0,
+	MLX5_ENCRYPTION_ORDER_ENCRYPTED_MEMORY_SIGNATURE = 0x1,
+	MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_WIRE = 0x2,
+	MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_MEMORY = 0x3,
+};
+
+enum {
+	MLX5_ENCRYPTION_STANDARD_AES_XTS = 0x0,
+};
+
+enum {
+	MLX5_BLOCK_SIZE_512B	= 0x1,
+	MLX5_BLOCK_SIZE_520B	= 0x2,
+	MLX5_BLOCK_SIZE_4096B	= 0x3,
+	MLX5_BLOCK_SIZE_4160B	= 0x4,
+	MLX5_BLOCK_SIZE_1MB	= 0x5,
+	MLX5_BLOCK_SIZE_4048B	= 0x6,
+};
+
+#define MLX5_BSF_SIZE_OFFSET		30
+#define MLX5_BSF_P_TYPE_OFFSET		24
+#define MLX5_ENCRYPTION_ORDER_OFFSET	16
+#define MLX5_BLOCK_SIZE_OFFSET		24
+
+struct mlx5_wqe_umr_bsf_seg {
+	/*
+	 * bs_bpt_eo_es contains:
+	 * bs	bsf_size		2 bits at MLX5_BSF_SIZE_OFFSET
+	 * bpt	bsf_p_type		2 bits at MLX5_BSF_P_TYPE_OFFSET
+	 * eo	encryption_order	4 bits at MLX5_ENCRYPTION_ORDER_OFFSET
+	 * es	encryption_standard	4 bits at offset 0
+	 */
+	uint32_t bs_bpt_eo_es;
+	uint32_t raw_data_size;
+	/*
+	 * bsp_res contains:
+	 * bsp	crypto_block_size_pointer	8 bits at MLX5_BLOCK_SIZE_OFFSET
+	 * res	reserved 24 bits
+	 */
+	uint32_t bsp_res;
+	uint32_t reserved0;
+	uint8_t xts_initial_tweak[16];
+	/*
+	 * res_dp contains:
+	 * res	reserved 8 bits
+	 * dp	dek_pointer		24 bits at offset 0
+	 */
+	uint32_t res_dp;
+	uint32_t reserved1;
+	uint64_t keytag;
+	uint32_t reserved2[4];
+} __rte_packed;
+
+#ifdef PEDANTIC
+#pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+
+struct mlx5_umr_wqe {
+	struct mlx5_wqe_cseg ctr;
+	struct mlx5_wqe_umr_cseg ucseg;
+	struct mlx5_wqe_mkey_cseg mkc;
+	union {
+		struct mlx5_wqe_dseg kseg[0];
+		struct mlx5_wqe_umr_bsf_seg bsf[0];
+	};
+} __rte_packed;
+
+struct mlx5_rdma_write_wqe {
+	struct mlx5_wqe_cseg ctr;
+	struct mlx5_wqe_rseg rseg;
+	struct mlx5_wqe_dseg dseg[0];
+} __rte_packed;
+
+#ifdef PEDANTIC
+#pragma GCC diagnostic error "-Wpedantic"
+#endif
+
 /* GGA */
 /* MMO metadata segment */
 
@@ -1096,72 +1217,6 @@ struct mlx5_ifc_create_mkey_in_bits {
 	u8 klm_pas_mtt[][0x20];
 };
 
-enum {
-	MLX5_BSF_SIZE_16B = 0x0,
-	MLX5_BSF_SIZE_32B = 0x1,
-	MLX5_BSF_SIZE_64B = 0x2,
-	MLX5_BSF_SIZE_128B = 0x3,
-};
-
-enum {
-	MLX5_BSF_P_TYPE_SIGNATURE = 0x0,
-	MLX5_BSF_P_TYPE_CRYPTO = 0x1,
-};
-
-enum {
-	MLX5_ENCRYPTION_ORDER_ENCRYPTED_WIRE_SIGNATURE = 0x0,
-	MLX5_ENCRYPTION_ORDER_ENCRYPTED_MEMORY_SIGNATURE = 0x1,
-	MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_WIRE = 0x2,
-	MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_MEMORY = 0x3,
-};
-
-enum {
-	MLX5_ENCRYPTION_STANDARD_AES_XTS = 0x0,
-};
-
-enum {
-	MLX5_BLOCK_SIZE_512B	= 0x1,
-	MLX5_BLOCK_SIZE_520B	= 0x2,
-	MLX5_BLOCK_SIZE_4096B	= 0x3,
-	MLX5_BLOCK_SIZE_4160B	= 0x4,
-	MLX5_BLOCK_SIZE_1MB	= 0x5,
-	MLX5_BLOCK_SIZE_4048B	= 0x6,
-};
-
-#define MLX5_BSF_SIZE_OFFSET		30
-#define MLX5_BSF_P_TYPE_OFFSET		24
-#define MLX5_ENCRYPTION_ORDER_OFFSET	16
-#define MLX5_BLOCK_SIZE_OFFSET		24
-
-struct mlx5_wqe_umr_bsf_seg {
-	/*
-	 * bs_bpt_eo_es contains:
-	 * bs	bsf_size		2 bits at MLX5_BSF_SIZE_OFFSET
-	 * bpt	bsf_p_type		2 bits at MLX5_BSF_P_TYPE_OFFSET
-	 * eo	encryption_order	4 bits at MLX5_ENCRYPTION_ORDER_OFFSET
-	 * es	encryption_standard	4 bits at offset 0
-	 */
-	uint32_t bs_bpt_eo_es;
-	uint32_t raw_data_size;
-	/*
-	 * bsp_res contains:
-	 * bsp	crypto_block_size_pointer	8 bits at MLX5_BLOCK_SIZE_OFFSET
-	 * res	reserved 24 bits
-	 */
-	uint32_t bsp_res;
-	uint32_t reserved0;
-	uint8_t xts_initial_tweak[16];
-	/*
-	 * res_dp contains:
-	 * res	reserved 8 bits
-	 * dp	dek_pointer		24 bits at offset 0
-	 */
-	uint32_t res_dp;
-	uint32_t reserved1;
-	uint64_t keytag;
-	uint32_t reserved2[4];
-} __rte_packed;
-
 enum {
 	MLX5_GET_HCA_CAP_OP_MOD_GENERAL_DEVICE = 0x0 << 1,
 	MLX5_GET_HCA_CAP_OP_MOD_ETHERNET_OFFLOAD_CAPS = 0x1 << 1,
-- 
2.25.1

[dpdk-dev] [PATCH v3 00/15] drivers: introduce mlx5 crypto PMD

[Matan Azrad]

Add a new PMD for Nvidia devices- crypto PMD.
The crypto PMD will be supported on Nvidia ConnectX6 The crypto PMD will add the support of encryption and decryption using the AES-XTS symmetric algorithm.
The crypto PMD requires rdma-core and uses mlx5 DevX.

v2:
Add data-path part.

v3:
Rebase.

Shiri Kuzin (10):
  drivers: introduce mlx5 crypto PMD
  crypto/mlx5: add DEK object management
  crypto/mlx5: support session operations
  crypto/mlx5: add basic operations
  crypto/mlx5: support queue pairs operations
  crypto/mlx5: add dev stop and start operations
  crypto/mlx5: add memory region management
  crypto/mlx5: create login object using DevX
  crypto/mlx5: adjust to the multiple data unit API
  crypto/mlx5: set feature flags and capabilities

Suanming Mou (5):
  crypto/mlx5: add keytag device argument
  crypto/mlx5: add maximum segments device argument
  crypto/mlx5: add WQE set initialization
  crypto/mlx5: add enqueue and dequeue operations
  crypto/mlx5: add statistic get and reset operations

 MAINTAINERS                             |    4 +
 doc/guides/cryptodevs/features/mlx5.ini |   37 +
 doc/guides/cryptodevs/index.rst         |    1 +
 doc/guides/cryptodevs/mlx5.rst          |  152 ++++
 doc/guides/rel_notes/release_21_05.rst  |    5 +
 drivers/common/mlx5/mlx5_common.h       |    1 +
 drivers/common/mlx5/mlx5_common_pci.c   |   14 +
 drivers/common/mlx5/mlx5_common_pci.h   |   21 +-
 drivers/crypto/meson.build              |    1 +
 drivers/crypto/mlx5/meson.build         |   27 +
 drivers/crypto/mlx5/mlx5_crypto.c       | 1092 +++++++++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto.h       |   92 ++
 drivers/crypto/mlx5/mlx5_crypto_dek.c   |  136 +++
 drivers/crypto/mlx5/mlx5_crypto_utils.h |   19 +
 drivers/crypto/mlx5/version.map         |    3 +
 15 files changed, 1595 insertions(+), 10 deletions(-)
 create mode 100644 doc/guides/cryptodevs/features/mlx5.ini
 create mode 100644 doc/guides/cryptodevs/mlx5.rst
 create mode 100644 drivers/crypto/mlx5/meson.build
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto.c
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto.h
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_dek.c
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_utils.h
 create mode 100644 drivers/crypto/mlx5/version.map

-- 
2.25.1

[dpdk-dev] [PATCH v3 01/15] drivers: introduce mlx5 crypto PMD

[Matan Azrad]

From: Shiri Kuzin <shirik@nvidia.com>

Add a new PMD for Nvidia devices- crypto PMD.

The crypto PMD will be supported starting Nvidia ConnectX6 and
BlueField2.

The crypto PMD will add the support of encryption and decryption using
the AES-XTS symmetric algorithm.

The crypto PMD requires rdma-core and uses mlx5 DevX.

This patch adds the PCI probing, basic functions, build files and
log utility.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 MAINTAINERS                             |   4 +
 drivers/common/mlx5/mlx5_common.h       |   1 +
 drivers/common/mlx5/mlx5_common_pci.c   |  14 ++
 drivers/common/mlx5/mlx5_common_pci.h   |  21 +-
 drivers/crypto/meson.build              |   1 +
 drivers/crypto/mlx5/meson.build         |  26 +++
 drivers/crypto/mlx5/mlx5_crypto.c       | 272 ++++++++++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto_utils.h |  19 ++
 drivers/crypto/mlx5/version.map         |   3 +
 9 files changed, 351 insertions(+), 10 deletions(-)
 create mode 100644 drivers/crypto/mlx5/meson.build
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto.c
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_utils.h
 create mode 100644 drivers/crypto/mlx5/version.map

diff --git a/MAINTAINERS b/MAINTAINERS
index b40d8ae266..165474c91f 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -1078,6 +1078,10 @@ F: drivers/crypto/octeontx2/
 F: doc/guides/cryptodevs/octeontx2.rst
 F: doc/guides/cryptodevs/features/octeontx2.ini
 
+Mellanox mlx5
+M: Matan Azrad <matan@nvidia.com>
+F: drivers/crypto/mlx5/
+
 Null Crypto
 M: Declan Doherty <declan.doherty@intel.com>
 F: drivers/crypto/null/
diff --git a/drivers/common/mlx5/mlx5_common.h b/drivers/common/mlx5/mlx5_common.h
index 1fbefe0fa6..89aca32305 100644
--- a/drivers/common/mlx5/mlx5_common.h
+++ b/drivers/common/mlx5/mlx5_common.h
@@ -216,6 +216,7 @@ enum mlx5_class {
 	MLX5_CLASS_VDPA = RTE_BIT64(1),
 	MLX5_CLASS_REGEX = RTE_BIT64(2),
 	MLX5_CLASS_COMPRESS = RTE_BIT64(3),
+	MLX5_CLASS_CRYPTO = RTE_BIT64(4),
 };
 
 #define MLX5_DBR_SIZE RTE_CACHE_LINE_SIZE
diff --git a/drivers/common/mlx5/mlx5_common_pci.c b/drivers/common/mlx5/mlx5_common_pci.c
index 3f16cd21cf..8a47afee20 100644
--- a/drivers/common/mlx5/mlx5_common_pci.c
+++ b/drivers/common/mlx5/mlx5_common_pci.c
@@ -31,6 +31,7 @@ static const struct {
 	{ .name = "net", .driver_class = MLX5_CLASS_NET },
 	{ .name = "regex", .driver_class = MLX5_CLASS_REGEX },
 	{ .name = "compress", .driver_class = MLX5_CLASS_COMPRESS },
+	{ .name = "crypto", .driver_class = MLX5_CLASS_CRYPTO },
 };
 
 static const unsigned int mlx5_class_combinations[] = {
@@ -38,13 +39,26 @@ static const unsigned int mlx5_class_combinations[] = {
 	MLX5_CLASS_VDPA,
 	MLX5_CLASS_REGEX,
 	MLX5_CLASS_COMPRESS,
+	MLX5_CLASS_CRYPTO,
 	MLX5_CLASS_NET | MLX5_CLASS_REGEX,
 	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX,
 	MLX5_CLASS_NET | MLX5_CLASS_COMPRESS,
 	MLX5_CLASS_VDPA | MLX5_CLASS_COMPRESS,
 	MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
+	MLX5_CLASS_NET | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_VDPA | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO,
 	MLX5_CLASS_NET | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
 	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
+	MLX5_CLASS_NET | MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_NET | MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_VDPA | MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_NET | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS |
+	MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS |
+	MLX5_CLASS_CRYPTO,
 	/* New class combination should be added here. */
 };
 
diff --git a/drivers/common/mlx5/mlx5_common_pci.h b/drivers/common/mlx5/mlx5_common_pci.h
index de89bb98bc..cb8d2f5f87 100644
--- a/drivers/common/mlx5/mlx5_common_pci.h
+++ b/drivers/common/mlx5/mlx5_common_pci.h
@@ -9,17 +9,18 @@
  * @file
  *
  * RTE Mellanox PCI Driver Interface
- * Mellanox ConnectX PCI device supports multiple class: net,vdpa,regex and
- * compress devices. This layer enables creating such multiple class of devices
- * on a single PCI device by allowing to bind multiple class specific device
- * driver to attach to mlx5_pci driver.
+ * Mellanox ConnectX PCI device supports multiple class: net,vdpa,regex,compress
+ * and crypto devices. This layer enables creating such multiple class of
+ * devices on a single PCI device by allowing to bind multiple class specific
+ * device driver to attach to mlx5_pci driver.
  *
- * -----------    ------------    -------------    ----------------
- * |   mlx5  |    |   mlx5   |    |   mlx5    |    |     mlx5     |
- * | net pmd |    | vdpa pmd |    | regex pmd |    | compress pmd |
- * -----------    ------------    -------------    ----------------
- *      \              \                    /              /
- *       \              \                  /              /
+ * --------    --------    ---------    ------------    ----------
+ * | mlx5 |    | mlx5 |    | mlx5  |    |   mlx5   |    |  mlx5  |
+ * | net  |    | vdpa |    | regex |    | compress |    | crypto |
+ * | pmd  |    | pmd  |    |  pmd  |    |   pmd    |    |  pmd   |
+ * --------    --------    ---------    ------------    ----------
+ *      \              \         |          /              /
+ *       \              \        |         /              /
  *        \              \_--------------_/              /
  *         \_______________|   mlx5     |_______________/
  *                         | pci common |
diff --git a/drivers/crypto/meson.build b/drivers/crypto/meson.build
index b9fdf9392f..6951607def 100644
--- a/drivers/crypto/meson.build
+++ b/drivers/crypto/meson.build
@@ -15,6 +15,7 @@ drivers = [
         'dpaa_sec',
         'dpaa2_sec',
         'kasumi',
+        'mlx5',
         'mvsam',
         'nitrox',
         'null',
diff --git a/drivers/crypto/mlx5/meson.build b/drivers/crypto/mlx5/meson.build
new file mode 100644
index 0000000000..fd00283665
--- /dev/null
+++ b/drivers/crypto/mlx5/meson.build
@@ -0,0 +1,26 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2021 Mellanox Technologies, Ltd
+
+if not is_linux
+	build = false
+	reason = 'only supported on Linux'
+	subdir_done()
+endif
+
+fmt_name = 'mlx5_crypto'
+deps += ['common_mlx5', 'eal', 'cryptodev']
+sources = files(
+	'mlx5_crypto.c',
+)
+cflags_options = [
+	'-std=c11',
+	'-Wno-strict-prototypes',
+	'-D_BSD_SOURCE',
+	'-D_DEFAULT_SOURCE',
+	'-D_XOPEN_SOURCE=600',
+]
+foreach option:cflags_options
+	if cc.has_argument(option)
+		cflags += option
+	endif
+endforeach
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
new file mode 100644
index 0000000000..ffbce5d68a
--- /dev/null
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -0,0 +1,272 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#include <rte_malloc.h>
+#include <rte_log.h>
+#include <rte_errno.h>
+#include <rte_pci.h>
+#include <rte_crypto.h>
+#include <rte_cryptodev.h>
+#include <rte_cryptodev_pmd.h>
+
+#include <mlx5_glue.h>
+#include <mlx5_common.h>
+#include <mlx5_common_pci.h>
+#include <mlx5_devx_cmds.h>
+#include <mlx5_common_os.h>
+
+#include "mlx5_crypto_utils.h"
+
+#define MLX5_CRYPTO_DRIVER_NAME mlx5_crypto
+#define MLX5_CRYPTO_LOG_NAME    pmd.crypto.mlx5
+
+struct mlx5_crypto_priv {
+	TAILQ_ENTRY(mlx5_crypto_priv) next;
+	struct ibv_context *ctx; /* Device context. */
+	struct rte_pci_device *pci_dev;
+	struct rte_cryptodev *crypto_dev;
+	void *uar; /* User Access Region. */
+	uint32_t pdn; /* Protection Domain number. */
+	struct ibv_pd *pd;
+};
+
+TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
+				TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
+static pthread_mutex_t priv_list_lock = PTHREAD_MUTEX_INITIALIZER;
+
+int mlx5_crypto_logtype;
+
+uint8_t mlx5_crypto_driver_id;
+
+static const char mlx5_crypto_drv_name[] = RTE_STR(MLX5_CRYPTO_DRIVER_NAME);
+
+static const struct rte_driver mlx5_drv = {
+	.name = mlx5_crypto_drv_name,
+	.alias = mlx5_crypto_drv_name
+};
+
+static struct cryptodev_driver mlx5_cryptodev_driver;
+
+static struct rte_cryptodev_ops mlx5_crypto_ops = {
+	.dev_configure			= NULL,
+	.dev_start			= NULL,
+	.dev_stop			= NULL,
+	.dev_close			= NULL,
+	.dev_infos_get			= NULL,
+	.stats_get			= NULL,
+	.stats_reset			= NULL,
+	.queue_pair_setup		= NULL,
+	.queue_pair_release		= NULL,
+	.sym_session_get_size		= NULL,
+	.sym_session_configure		= NULL,
+	.sym_session_clear		= NULL,
+	.sym_get_raw_dp_ctx_size	= NULL,
+	.sym_configure_raw_dp_ctx	= NULL,
+};
+
+static void
+mlx5_crypto_hw_global_release(struct mlx5_crypto_priv *priv)
+{
+	if (priv->pd != NULL) {
+		claim_zero(mlx5_glue->dealloc_pd(priv->pd));
+		priv->pd = NULL;
+	}
+	if (priv->uar != NULL) {
+		mlx5_glue->devx_free_uar(priv->uar);
+		priv->uar = NULL;
+	}
+}
+
+static int
+mlx5_crypto_pd_create(struct mlx5_crypto_priv *priv)
+{
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+	struct mlx5dv_obj obj;
+	struct mlx5dv_pd pd_info;
+	int ret;
+
+	priv->pd = mlx5_glue->alloc_pd(priv->ctx);
+	if (priv->pd == NULL) {
+		DRV_LOG(ERR, "Failed to allocate PD.");
+		return errno ? -errno : -ENOMEM;
+	}
+	obj.pd.in = priv->pd;
+	obj.pd.out = &pd_info;
+	ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_PD);
+	if (ret != 0) {
+		DRV_LOG(ERR, "Fail to get PD object info.");
+		mlx5_glue->dealloc_pd(priv->pd);
+		priv->pd = NULL;
+		return -errno;
+	}
+	priv->pdn = pd_info.pdn;
+	return 0;
+#else
+	(void)priv;
+	DRV_LOG(ERR, "Cannot get pdn - no DV support.");
+	return -ENOTSUP;
+#endif /* HAVE_IBV_FLOW_DV_SUPPORT */
+}
+
+static int
+mlx5_crypto_hw_global_prepare(struct mlx5_crypto_priv *priv)
+{
+	if (mlx5_crypto_pd_create(priv) != 0)
+		return -1;
+	priv->uar = mlx5_devx_alloc_uar(priv->ctx, -1);
+	if (priv->uar == NULL || mlx5_os_get_devx_uar_reg_addr(priv->uar) ==
+	    NULL) {
+		rte_errno = errno;
+		claim_zero(mlx5_glue->dealloc_pd(priv->pd));
+		DRV_LOG(ERR, "Failed to allocate UAR.");
+		return -1;
+	}
+	return 0;
+}
+
+/**
+ * DPDK callback to register a PCI device.
+ *
+ * This function spawns crypto device out of a given PCI device.
+ *
+ * @param[in] pci_drv
+ *   PCI driver structure (mlx5_crypto_driver).
+ * @param[in] pci_dev
+ *   PCI device information.
+ *
+ * @return
+ *   0 on success, 1 to skip this driver, a negative errno value otherwise
+ *   and rte_errno is set.
+ */
+static int
+mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
+			struct rte_pci_device *pci_dev)
+{
+	struct ibv_device *ibv;
+	struct rte_cryptodev *crypto_dev;
+	struct ibv_context *ctx;
+	struct mlx5_crypto_priv *priv;
+	struct mlx5_hca_attr attr = { 0 };
+	struct rte_cryptodev_pmd_init_params init_params = {
+		.name = "",
+		.private_data_size = sizeof(struct mlx5_crypto_priv),
+		.socket_id = pci_dev->device.numa_node,
+		.max_nb_queue_pairs =
+				RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
+	};
+	RTE_SET_USED(pci_drv);
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+		DRV_LOG(ERR, "Non-primary process type is not supported.");
+		rte_errno = ENOTSUP;
+		return -rte_errno;
+	}
+	ibv = mlx5_os_get_ibv_device(&pci_dev->addr);
+	if (ibv == NULL) {
+		DRV_LOG(ERR, "No matching IB device for PCI slot "
+			PCI_PRI_FMT ".", pci_dev->addr.domain,
+			pci_dev->addr.bus, pci_dev->addr.devid,
+			pci_dev->addr.function);
+		return -rte_errno;
+	}
+	DRV_LOG(INFO, "PCI information matches for device \"%s\".", ibv->name);
+	ctx = mlx5_glue->dv_open_device(ibv);
+	if (ctx == NULL) {
+		DRV_LOG(ERR, "Failed to open IB device \"%s\".", ibv->name);
+		rte_errno = ENODEV;
+		return -rte_errno;
+	}
+	if (mlx5_devx_cmd_query_hca_attr(ctx, &attr) != 0 ||
+	    attr.crypto == 0 || attr.aes_xts == 0) {
+		DRV_LOG(ERR, "Not enough capabilities to support crypto "
+			"operations, maybe old FW/OFED version?");
+		claim_zero(mlx5_glue->close_device(ctx));
+		rte_errno = ENOTSUP;
+		return -ENOTSUP;
+	}
+	crypto_dev = rte_cryptodev_pmd_create(ibv->name, &pci_dev->device,
+					&init_params);
+	if (crypto_dev == NULL) {
+		DRV_LOG(ERR, "Failed to create device \"%s\".", ibv->name);
+		claim_zero(mlx5_glue->close_device(ctx));
+		return -ENODEV;
+	}
+	DRV_LOG(INFO,
+		"Crypto device %s was created successfully.", ibv->name);
+	crypto_dev->dev_ops = &mlx5_crypto_ops;
+	crypto_dev->dequeue_burst = NULL;
+	crypto_dev->enqueue_burst = NULL;
+	crypto_dev->feature_flags = RTE_CRYPTODEV_FF_HW_ACCELERATED;
+	crypto_dev->driver_id = mlx5_crypto_driver_id;
+	priv = crypto_dev->data->dev_private;
+	priv->ctx = ctx;
+	priv->pci_dev = pci_dev;
+	priv->crypto_dev = crypto_dev;
+	if (mlx5_crypto_hw_global_prepare(priv) != 0) {
+		rte_cryptodev_pmd_destroy(priv->crypto_dev);
+		claim_zero(mlx5_glue->close_device(priv->ctx));
+		return -1;
+	}
+	pthread_mutex_lock(&priv_list_lock);
+	TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next);
+	pthread_mutex_unlock(&priv_list_lock);
+	return 0;
+}
+
+static int
+mlx5_crypto_pci_remove(struct rte_pci_device *pdev)
+{
+	struct mlx5_crypto_priv *priv = NULL;
+
+	pthread_mutex_lock(&priv_list_lock);
+	TAILQ_FOREACH(priv, &mlx5_crypto_priv_list, next)
+		if (rte_pci_addr_cmp(&priv->pci_dev->addr, &pdev->addr) != 0)
+			break;
+	if (priv)
+		TAILQ_REMOVE(&mlx5_crypto_priv_list, priv, next);
+	pthread_mutex_unlock(&priv_list_lock);
+	if (priv) {
+		mlx5_crypto_hw_global_release(priv);
+		rte_cryptodev_pmd_destroy(priv->crypto_dev);
+		claim_zero(mlx5_glue->close_device(priv->ctx));
+	}
+	return 0;
+}
+
+static const struct rte_pci_id mlx5_crypto_pci_id_map[] = {
+		{
+			RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
+					PCI_DEVICE_ID_MELLANOX_CONNECTX6)
+		},
+		{
+			.vendor_id = 0
+		}
+	};
+
+static struct mlx5_pci_driver mlx5_crypto_driver = {
+	.driver_class = MLX5_CLASS_CRYPTO,
+	.pci_driver = {
+		.driver = {
+			.name = RTE_STR(MLX5_CRYPTO_DRIVER_NAME),
+		},
+		.id_table = mlx5_crypto_pci_id_map,
+		.probe = mlx5_crypto_pci_probe,
+		.remove = mlx5_crypto_pci_remove,
+		.drv_flags = 0,
+	},
+};
+
+RTE_INIT(rte_mlx5_crypto_init)
+{
+	mlx5_common_init();
+	if (mlx5_glue != NULL)
+		mlx5_pci_driver_register(&mlx5_crypto_driver);
+}
+
+RTE_PMD_REGISTER_CRYPTO_DRIVER(mlx5_cryptodev_driver, mlx5_drv,
+			       mlx5_crypto_driver_id);
+
+RTE_LOG_REGISTER(mlx5_crypto_logtype, MLX5_CRYPTO_LOG_NAME, NOTICE)
+RTE_PMD_EXPORT_NAME(MLX5_CRYPTO_DRIVER_NAME, __COUNTER__);
+RTE_PMD_REGISTER_PCI_TABLE(MLX5_CRYPTO_DRIVER_NAME, mlx5_crypto_pci_id_map);
+RTE_PMD_REGISTER_KMOD_DEP(MLX5_CRYPTO_DRIVER_NAME, "* ib_uverbs & mlx5_core & mlx5_ib");
diff --git a/drivers/crypto/mlx5/mlx5_crypto_utils.h b/drivers/crypto/mlx5/mlx5_crypto_utils.h
new file mode 100644
index 0000000000..cef4b07a36
--- /dev/null
+++ b/drivers/crypto/mlx5/mlx5_crypto_utils.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#ifndef RTE_PMD_MLX5_CRYPTO_UTILS_H_
+#define RTE_PMD_MLX5_CRYPTO_UTILS_H_
+
+#include <mlx5_common.h>
+
+extern int mlx5_crypto_logtype;
+
+#define MLX5_CRYPTO_LOG_PREFIX "mlx5_crypto"
+/* Generic printf()-like logging macro with automatic line feed. */
+#define DRV_LOG(level, ...) \
+	PMD_DRV_LOG_(level, mlx5_crypto_logtype, MLX5_CRYPTO_LOG_PREFIX, \
+		__VA_ARGS__ PMD_DRV_LOG_STRIP PMD_DRV_LOG_OPAREN, \
+		PMD_DRV_LOG_CPAREN)
+
+#endif /* RTE_PMD_MLX5_CRYPTO_UTILS_H_ */
diff --git a/drivers/crypto/mlx5/version.map b/drivers/crypto/mlx5/version.map
new file mode 100644
index 0000000000..4a76d1d52d
--- /dev/null
+++ b/drivers/crypto/mlx5/version.map
@@ -0,0 +1,3 @@
+DPDK_21 {
+	local: *;
+};
-- 
2.25.1

[dpdk-dev] [PATCH v3 02/15] crypto/mlx5: add DEK object management

[Matan Azrad]

From: Shiri Kuzin <shirik@nvidia.com>

A DEK(Data encryption Key) is an mlx5 HW object which represents the
cipher algorithm key.
The DEKs are used during data encryption/decryption operations.

In symmetric algorithms like AES-STS, we use the same DEK for both
encryption and decryption.

Use the mlx5 hash-list tool to manage the DEK objects in the PMD.

Provide the compare, create and destroy functions to manage DEKs in
hash-list and introduce an internal API to setup and unset the DEK
management and to prepare and destroy specific DEK object.

The DEK hash-list will be created in dev_configure routine and
destroyed in dev_close routine.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/meson.build       |   1 +
 drivers/crypto/mlx5/mlx5_crypto.c     |  44 +++++----
 drivers/crypto/mlx5/mlx5_crypto.h     |  51 ++++++++++
 drivers/crypto/mlx5/mlx5_crypto_dek.c | 136 ++++++++++++++++++++++++++
 4 files changed, 215 insertions(+), 17 deletions(-)
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto.h
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_dek.c

diff --git a/drivers/crypto/mlx5/meson.build b/drivers/crypto/mlx5/meson.build
index fd00283665..9cb3bd214f 100644
--- a/drivers/crypto/mlx5/meson.build
+++ b/drivers/crypto/mlx5/meson.build
@@ -11,6 +11,7 @@ fmt_name = 'mlx5_crypto'
 deps += ['common_mlx5', 'eal', 'cryptodev']
 sources = files(
 	'mlx5_crypto.c',
+	'mlx5_crypto_dek.c',
 )
 cflags_options = [
 	'-std=c11',
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index ffbce5d68a..2bdfb1a10f 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -3,12 +3,9 @@
  */
 
 #include <rte_malloc.h>
-#include <rte_log.h>
 #include <rte_errno.h>
+#include <rte_log.h>
 #include <rte_pci.h>
-#include <rte_crypto.h>
-#include <rte_cryptodev.h>
-#include <rte_cryptodev_pmd.h>
 
 #include <mlx5_glue.h>
 #include <mlx5_common.h>
@@ -17,19 +14,10 @@
 #include <mlx5_common_os.h>
 
 #include "mlx5_crypto_utils.h"
+#include "mlx5_crypto.h"
 
 #define MLX5_CRYPTO_DRIVER_NAME mlx5_crypto
-#define MLX5_CRYPTO_LOG_NAME    pmd.crypto.mlx5
-
-struct mlx5_crypto_priv {
-	TAILQ_ENTRY(mlx5_crypto_priv) next;
-	struct ibv_context *ctx; /* Device context. */
-	struct rte_pci_device *pci_dev;
-	struct rte_cryptodev *crypto_dev;
-	void *uar; /* User Access Region. */
-	uint32_t pdn; /* Protection Domain number. */
-	struct ibv_pd *pd;
-};
+#define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
 
 TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
 				TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
@@ -48,11 +36,33 @@ static const struct rte_driver mlx5_drv = {
 
 static struct cryptodev_driver mlx5_cryptodev_driver;
 
+static int
+mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
+		struct rte_cryptodev_config *config __rte_unused)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+
+	if (mlx5_crypto_dek_setup(priv) != 0) {
+		DRV_LOG(ERR, "Dek hash list creation has failed.");
+		return -ENOMEM;
+	}
+	return 0;
+}
+
+static int
+mlx5_crypto_dev_close(struct rte_cryptodev *dev)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+
+	mlx5_crypto_dek_unset(priv);
+	return 0;
+}
+
 static struct rte_cryptodev_ops mlx5_crypto_ops = {
-	.dev_configure			= NULL,
+	.dev_configure			= mlx5_crypto_dev_configure,
 	.dev_start			= NULL,
 	.dev_stop			= NULL,
-	.dev_close			= NULL,
+	.dev_close			= mlx5_crypto_dev_close,
 	.dev_infos_get			= NULL,
 	.stats_get			= NULL,
 	.stats_reset			= NULL,
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
new file mode 100644
index 0000000000..4ec67a7e0f
--- /dev/null
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -0,0 +1,51 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#ifndef MLX5_CRYPTO_H_
+#define MLX5_CRYPTO_H_
+
+#include <stdbool.h>
+
+#include <rte_cryptodev.h>
+#include <rte_cryptodev_pmd.h>
+
+#include <mlx5_common_utils.h>
+
+#define MLX5_CRYPTO_DEK_HTABLE_SZ (1 << 11)
+#define MLX5_CRYPTO_KEY_LENGTH 80
+
+struct mlx5_crypto_priv {
+	TAILQ_ENTRY(mlx5_crypto_priv) next;
+	struct ibv_context *ctx; /* Device context. */
+	struct rte_pci_device *pci_dev;
+	struct rte_cryptodev *crypto_dev;
+	void *uar; /* User Access Region. */
+	uint32_t pdn; /* Protection Domain number. */
+	struct ibv_pd *pd;
+	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
+};
+
+struct mlx5_crypto_dek {
+	struct mlx5_hlist_entry entry; /* Pointer to DEK hash list entry. */
+	struct mlx5_devx_obj *obj; /* Pointer to DEK DevX object. */
+	uint8_t data[MLX5_CRYPTO_KEY_LENGTH]; /* DEK key data. */
+	bool size_is_48; /* Whether the key\data size is 48 bytes or not. */
+};
+
+int
+mlx5_crypto_dek_destroy(struct mlx5_crypto_priv *priv,
+			struct mlx5_crypto_dek *dek);
+
+struct mlx5_crypto_dek *
+mlx5_crypto_dek_prepare(struct mlx5_crypto_priv *priv,
+			struct rte_crypto_cipher_xform *cipher);
+
+int
+mlx5_crypto_dek_setup(struct mlx5_crypto_priv *priv);
+
+void
+mlx5_crypto_dek_unset(struct mlx5_crypto_priv *priv);
+
+#endif /* MLX5_CRYPTO_H_ */
+
diff --git a/drivers/crypto/mlx5/mlx5_crypto_dek.c b/drivers/crypto/mlx5/mlx5_crypto_dek.c
new file mode 100644
index 0000000000..c76e208845
--- /dev/null
+++ b/drivers/crypto/mlx5/mlx5_crypto_dek.c
@@ -0,0 +1,136 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2018 Mellanox Technologies, Ltd
+ */
+
+#include <rte_ip.h>
+#include <rte_common.h>
+#include <rte_errno.h>
+#include <rte_log.h>
+
+#include <mlx5_prm.h>
+#include <mlx5_devx_cmds.h>
+
+#include "mlx5_crypto_utils.h"
+#include "mlx5_crypto.h"
+
+struct mlx5_crypto_dek_ctx {
+	struct rte_crypto_cipher_xform *cipher;
+	struct mlx5_crypto_priv *priv;
+};
+
+int
+mlx5_crypto_dek_destroy(struct mlx5_crypto_priv *priv,
+			struct mlx5_crypto_dek *dek)
+{
+	return mlx5_hlist_unregister(priv->dek_hlist, &dek->entry);
+}
+
+struct mlx5_crypto_dek *
+mlx5_crypto_dek_prepare(struct mlx5_crypto_priv *priv,
+			struct rte_crypto_cipher_xform *cipher)
+{
+	struct mlx5_hlist *dek_hlist = priv->dek_hlist;
+	struct mlx5_crypto_dek_ctx dek_ctx = {
+		.cipher = cipher,
+		.priv = priv,
+	};
+	struct rte_crypto_cipher_xform *cipher_ctx = cipher;
+	uint64_t key64 = __rte_raw_cksum(cipher_ctx->key.data,
+					 cipher_ctx->key.length, 0);
+	struct mlx5_hlist_entry *entry = mlx5_hlist_register(dek_hlist,
+							     key64, &dek_ctx);
+
+	return entry == NULL ? NULL :
+			     container_of(entry, struct mlx5_crypto_dek, entry);
+}
+
+static int
+mlx5_crypto_dek_match_cb(struct mlx5_hlist *list __rte_unused,
+			 struct mlx5_hlist_entry *entry,
+			 uint64_t key __rte_unused, void *cb_ctx)
+{
+	struct mlx5_crypto_dek_ctx *ctx = cb_ctx;
+	struct rte_crypto_cipher_xform *cipher_ctx = ctx->cipher;
+	struct mlx5_crypto_dek *dek =
+			container_of(entry, typeof(*dek), entry);
+	uint32_t key_len = dek->size_is_48 ? 48 : 80;
+
+	if (key_len != cipher_ctx->key.length)
+		return -1;
+	return memcmp(cipher_ctx->key.data, dek->data, key_len);
+}
+
+static struct mlx5_hlist_entry *
+mlx5_crypto_dek_create_cb(struct mlx5_hlist *list __rte_unused,
+			  uint64_t key __rte_unused, void *cb_ctx)
+{
+	struct mlx5_crypto_dek_ctx *ctx = cb_ctx;
+	struct rte_crypto_cipher_xform *cipher_ctx = ctx->cipher;
+	struct mlx5_crypto_dek *dek = rte_zmalloc(__func__, sizeof(*dek),
+						  RTE_CACHE_LINE_SIZE);
+	struct mlx5_devx_dek_attr dek_attr = {
+		.pd = ctx->priv->pdn,
+		.key_purpose = MLX5_CRYPTO_KEY_PURPOSE_AES_XTS,
+		.has_keytag = 1,
+	};
+
+	if (dek == NULL) {
+		DRV_LOG(ERR, "Failed to allocate dek memory.");
+		return NULL;
+	}
+	switch (cipher_ctx->key.length) {
+	case 48:
+		dek->size_is_48 = true;
+		dek_attr.key_size = MLX5_CRYPTO_KEY_SIZE_128b;
+		break;
+	case 80:
+		dek->size_is_48 = false;
+		dek_attr.key_size = MLX5_CRYPTO_KEY_SIZE_256b;
+		break;
+	default:
+		DRV_LOG(ERR, "Key size not supported.");
+		return NULL;
+	}
+	rte_memcpy(&dek_attr.key, cipher_ctx->key.data, cipher_ctx->key.length);
+	dek->obj = mlx5_devx_cmd_create_dek_obj(ctx->priv->ctx, &dek_attr);
+	if (dek->obj == NULL) {
+		rte_free(dek);
+		return NULL;
+	}
+	rte_memcpy(&dek->data, cipher_ctx->key.data, cipher_ctx->key.length);
+	return &dek->entry;
+}
+
+static void
+mlx5_crypto_dek_remove_cb(struct mlx5_hlist *list __rte_unused,
+			  struct mlx5_hlist_entry *entry)
+{
+	struct mlx5_crypto_dek *dek =
+		container_of(entry, typeof(*dek), entry);
+
+	claim_zero(mlx5_devx_cmd_destroy(dek->obj));
+	rte_free(dek);
+}
+
+
+int
+mlx5_crypto_dek_setup(struct mlx5_crypto_priv *priv)
+{
+	priv->dek_hlist = mlx5_hlist_create("dek_hlist",
+				 MLX5_CRYPTO_DEK_HTABLE_SZ,
+				 0, MLX5_HLIST_WRITE_MOST |
+				 MLX5_HLIST_DIRECT_KEY,
+				 mlx5_crypto_dek_create_cb,
+				 mlx5_crypto_dek_match_cb,
+				 mlx5_crypto_dek_remove_cb);
+	if (priv->dek_hlist == NULL)
+		return -1;
+	return 0;
+}
+
+void
+mlx5_crypto_dek_unset(struct mlx5_crypto_priv *priv)
+{
+	mlx5_hlist_destroy(priv->dek_hlist);
+	priv->dek_hlist = NULL;
+}
-- 
2.25.1

[dpdk-dev] [PATCH v3 03/15] crypto/mlx5: support session operations

[Matan Azrad]

From: Shiri Kuzin <shirik@nvidia.com>

Sessions are used in symmetric transformations in order to prepare
objects and data for packet processing stage.

A mlx5 session includes iv_offset, pointer to mlx5_crypto_dek struct,
bsf_size, bsf_p_type, encryption_order and encryption standard.

Implement the next session operations:
        mlx5_crypto_sym_session_get_size- returns the size of the mlx5
	session struct.
	mlx5_crypto_sym_session_configure- prepares the DEK hash-list
	and saves all the session data.
	mlx5_crypto_sym_session_clear - destroys the DEK hash-list.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 96 ++++++++++++++++++++++++++++++-
 1 file changed, 93 insertions(+), 3 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 2bdfb1a10f..32f5077066 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -3,6 +3,7 @@
  */
 
 #include <rte_malloc.h>
+#include <rte_mempool.h>
 #include <rte_errno.h>
 #include <rte_log.h>
 #include <rte_pci.h>
@@ -36,6 +37,24 @@ static const struct rte_driver mlx5_drv = {
 
 static struct cryptodev_driver mlx5_cryptodev_driver;
 
+struct mlx5_crypto_session {
+	uint32_t bs_bpt_eo_es;
+	/*
+	 * bsf_size, bsf_p_type, encryption_order and encryption standard,
+	 * saved in big endian format.
+	 */
+	uint32_t iv_offset:16;
+	/* Starting point for Initialisation Vector. */
+	struct mlx5_crypto_dek *dek; /* Pointer to dek struct. */
+	uint32_t dek_id; /* DEK ID */
+} __rte_packed;
+
+static unsigned int
+mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
+{
+	return sizeof(struct mlx5_crypto_session);
+}
+
 static int
 mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
 		struct rte_cryptodev_config *config __rte_unused)
@@ -58,6 +77,77 @@ mlx5_crypto_dev_close(struct rte_cryptodev *dev)
 	return 0;
 }
 
+static int
+mlx5_crypto_sym_session_configure(struct rte_cryptodev *dev,
+				  struct rte_crypto_sym_xform *xform,
+				  struct rte_cryptodev_sym_session *session,
+				  struct rte_mempool *mp)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+	struct mlx5_crypto_session *sess_private_data;
+	struct rte_crypto_cipher_xform *cipher;
+	uint8_t encryption_order;
+	int ret;
+
+	if (unlikely(xform->next != NULL)) {
+		DRV_LOG(ERR, "Xform next is not supported.");
+		return -ENOTSUP;
+	}
+	if (unlikely((xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) ||
+		     (xform->cipher.algo != RTE_CRYPTO_CIPHER_AES_XTS))) {
+		DRV_LOG(ERR, "Only AES-XTS algorithm is supported.");
+		return -ENOTSUP;
+	}
+	ret = rte_mempool_get(mp, (void *)&sess_private_data);
+	if (ret != 0) {
+		DRV_LOG(ERR,
+			"Failed to get session %p private data from mempool.",
+			sess_private_data);
+		return -ENOMEM;
+	}
+	cipher = &xform->cipher;
+	sess_private_data->dek = mlx5_crypto_dek_prepare(priv, cipher);
+	if (sess_private_data->dek == NULL) {
+		rte_mempool_put(mp, sess_private_data);
+		DRV_LOG(ERR, "Failed to prepare dek.");
+		return -ENOMEM;
+	}
+	if (cipher->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
+		encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_MEMORY;
+	else
+		encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_WIRE;
+	sess_private_data->bs_bpt_eo_es = rte_cpu_to_be_32
+			(MLX5_BSF_SIZE_64B << MLX5_BSF_SIZE_OFFSET |
+			 MLX5_BSF_P_TYPE_CRYPTO << MLX5_BSF_P_TYPE_OFFSET |
+			 encryption_order << MLX5_ENCRYPTION_ORDER_OFFSET |
+			 MLX5_ENCRYPTION_STANDARD_AES_XTS);
+	sess_private_data->iv_offset = cipher->iv.offset;
+	sess_private_data->dek_id =
+			rte_cpu_to_be_32(sess_private_data->dek->obj->id &
+					 0xffffff);
+	set_sym_session_private_data(session, dev->driver_id,
+				     sess_private_data);
+	DRV_LOG(DEBUG, "Session %p was configured.", sess_private_data);
+	return 0;
+}
+
+static void
+mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev,
+			      struct rte_cryptodev_sym_session *sess)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+	struct mlx5_crypto_session *sess_private_data =
+			get_sym_session_private_data(sess, dev->driver_id);
+
+	if (unlikely(sess_private_data == NULL)) {
+		DRV_LOG(ERR, "Failed to get session %p private data.",
+				sess_private_data);
+		return;
+	}
+	mlx5_crypto_dek_destroy(priv, sess_private_data->dek);
+	DRV_LOG(DEBUG, "Session %p was cleared.", sess_private_data);
+}
+
 static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_configure			= mlx5_crypto_dev_configure,
 	.dev_start			= NULL,
@@ -68,9 +158,9 @@ static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.stats_reset			= NULL,
 	.queue_pair_setup		= NULL,
 	.queue_pair_release		= NULL,
-	.sym_session_get_size		= NULL,
-	.sym_session_configure		= NULL,
-	.sym_session_clear		= NULL,
+	.sym_session_get_size		= mlx5_crypto_sym_session_get_size,
+	.sym_session_configure		= mlx5_crypto_sym_session_configure,
+	.sym_session_clear		= mlx5_crypto_sym_session_clear,
 	.sym_get_raw_dp_ctx_size	= NULL,
 	.sym_configure_raw_dp_ctx	= NULL,
 };
-- 
2.25.1

[dpdk-dev] [PATCH v3 04/15] crypto/mlx5: add basic operations

[Matan Azrad]

From: Shiri Kuzin <shirik@nvidia.com>

The basic dev control operations are configure, close and get info.

Extended the existing support of configure and close:
	-mlx5_crypto_dev_configure- function used to configure device.
	-mlx5_crypto_dev_close-  function used to close a configured
	 device.

Added support of get info function:
	-mlx5_crypto_dev_infos_get- function used to get specific
	 information of a device.

Added config struct to user private data with the fields socket id,
number of queue pairs and feature flags to be disabled.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 46 +++++++++++++++++++++++++++----
 drivers/crypto/mlx5/mlx5_crypto.h |  1 +
 2 files changed, 42 insertions(+), 5 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 32f5077066..cec21dbea7 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -19,6 +19,7 @@
 
 #define MLX5_CRYPTO_DRIVER_NAME mlx5_crypto
 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
+#define MLX5_CRYPTO_MAX_QPS 1024
 
 TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
 				TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
@@ -28,6 +29,9 @@ int mlx5_crypto_logtype;
 
 uint8_t mlx5_crypto_driver_id;
 
+const struct rte_cryptodev_capabilities
+		mlx5_crypto_caps[RTE_CRYPTO_OP_TYPE_UNDEFINED];
+
 static const char mlx5_crypto_drv_name[] = RTE_STR(MLX5_CRYPTO_DRIVER_NAME);
 
 static const struct rte_driver mlx5_drv = {
@@ -49,22 +53,47 @@ struct mlx5_crypto_session {
 	uint32_t dek_id; /* DEK ID */
 } __rte_packed;
 
-static unsigned int
-mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
+static void
+mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev,
+			  struct rte_cryptodev_info *dev_info)
 {
-	return sizeof(struct mlx5_crypto_session);
+	RTE_SET_USED(dev);
+	if (dev_info != NULL) {
+		dev_info->driver_id = mlx5_crypto_driver_id;
+		dev_info->feature_flags = 0;
+		dev_info->capabilities = mlx5_crypto_caps;
+		dev_info->max_nb_queue_pairs = MLX5_CRYPTO_MAX_QPS;
+		dev_info->min_mbuf_headroom_req = 0;
+		dev_info->min_mbuf_tailroom_req = 0;
+		dev_info->sym.max_nb_sessions = 0;
+		/*
+		 * If 0, the device does not have any limitation in number of
+		 * sessions that can be used.
+		 */
+	}
 }
 
 static int
 mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
-		struct rte_cryptodev_config *config __rte_unused)
+			  struct rte_cryptodev_config *config)
 {
 	struct mlx5_crypto_priv *priv = dev->data->dev_private;
 
+	if (config == NULL) {
+		DRV_LOG(ERR, "Invalid crypto dev configure parameters.");
+		return -EINVAL;
+	}
+	if ((config->ff_disable & RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO) != 0) {
+		DRV_LOG(ERR,
+			"Disabled symmetric crypto feature is not supported.");
+		return -ENOTSUP;
+	}
 	if (mlx5_crypto_dek_setup(priv) != 0) {
 		DRV_LOG(ERR, "Dek hash list creation has failed.");
 		return -ENOMEM;
 	}
+	priv->dev_config = *config;
+	DRV_LOG(DEBUG, "Device %u was configured.", dev->driver_id);
 	return 0;
 }
 
@@ -74,9 +103,16 @@ mlx5_crypto_dev_close(struct rte_cryptodev *dev)
 	struct mlx5_crypto_priv *priv = dev->data->dev_private;
 
 	mlx5_crypto_dek_unset(priv);
+	DRV_LOG(DEBUG, "Device %u was closed.", dev->driver_id);
 	return 0;
 }
 
+static unsigned int
+mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
+{
+	return sizeof(struct mlx5_crypto_session);
+}
+
 static int
 mlx5_crypto_sym_session_configure(struct rte_cryptodev *dev,
 				  struct rte_crypto_sym_xform *xform,
@@ -153,7 +189,7 @@ static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_start			= NULL,
 	.dev_stop			= NULL,
 	.dev_close			= mlx5_crypto_dev_close,
-	.dev_infos_get			= NULL,
+	.dev_infos_get			= mlx5_crypto_dev_infos_get,
 	.stats_get			= NULL,
 	.stats_reset			= NULL,
 	.queue_pair_setup		= NULL,
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 4ec67a7e0f..5e270d3d5a 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -24,6 +24,7 @@ struct mlx5_crypto_priv {
 	uint32_t pdn; /* Protection Domain number. */
 	struct ibv_pd *pd;
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
+	struct rte_cryptodev_config dev_config;
 };
 
 struct mlx5_crypto_dek {
-- 
2.25.1

[dpdk-dev] [PATCH v3 05/15] crypto/mlx5: support queue pairs operations

[Matan Azrad]

From: Shiri Kuzin <shirik@nvidia.com>

The HW queue pairs are a pair of send queue and receive queue of
independent work queues packed together in one object for the purpose
of transferring data between nodes of a network.

Completion Queue is a FIFO queue of completed work requests.

In crypto driver we use one QP in loopback in order to encrypt and
decrypt data locally without sending it to the wire.
In the configured QP we only use the SQ to perform the encryption and
decryption operations.

Added implementation for the QP setup function which creates the CQ,
creates the QP and changes its state to RTS (ready to send).

Added implementation for the release QP function to release all the QP
resources.

Added the ops structure that contains any operation which is supported
by the cryptodev.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 124 +++++++++++++++++++++++++++++-
 drivers/crypto/mlx5/mlx5_crypto.h |  11 +++
 2 files changed, 133 insertions(+), 2 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index cec21dbea7..8c3417ee96 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -7,6 +7,7 @@
 #include <rte_errno.h>
 #include <rte_log.h>
 #include <rte_pci.h>
+#include <rte_memory.h>
 
 #include <mlx5_glue.h>
 #include <mlx5_common.h>
@@ -184,6 +185,125 @@ mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev,
 	DRV_LOG(DEBUG, "Session %p was cleared.", sess_private_data);
 }
 
+static int
+mlx5_crypto_queue_pair_release(struct rte_cryptodev *dev, uint16_t qp_id)
+{
+	struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
+
+	if (qp->qp_obj != NULL)
+		claim_zero(mlx5_devx_cmd_destroy(qp->qp_obj));
+	if (qp->umem_obj != NULL)
+		claim_zero(mlx5_glue->devx_umem_dereg(qp->umem_obj));
+	if (qp->umem_buf != NULL)
+		rte_free(qp->umem_buf);
+	mlx5_devx_cq_destroy(&qp->cq_obj);
+	rte_free(qp);
+	dev->data->queue_pairs[qp_id] = NULL;
+	return 0;
+}
+
+static int
+mlx5_crypto_qp2rts(struct mlx5_crypto_qp *qp)
+{
+	/*
+	 * In Order to configure self loopback, when calling these functions the
+	 * remote QP id that is used is the id of the same QP.
+	 */
+	if (mlx5_devx_cmd_modify_qp_state(qp->qp_obj, MLX5_CMD_OP_RST2INIT_QP,
+					  qp->qp_obj->id)) {
+		DRV_LOG(ERR, "Failed to modify QP to INIT state(%u).",
+			rte_errno);
+		return -1;
+	}
+	if (mlx5_devx_cmd_modify_qp_state(qp->qp_obj, MLX5_CMD_OP_INIT2RTR_QP,
+					  qp->qp_obj->id)) {
+		DRV_LOG(ERR, "Failed to modify QP to RTR state(%u).",
+			rte_errno);
+		return -1;
+	}
+	if (mlx5_devx_cmd_modify_qp_state(qp->qp_obj, MLX5_CMD_OP_RTR2RTS_QP,
+					  qp->qp_obj->id)) {
+		DRV_LOG(ERR, "Failed to modify QP to RTS state(%u).",
+			rte_errno);
+		return -1;
+	}
+	return 0;
+}
+
+static int
+mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
+			     const struct rte_cryptodev_qp_conf *qp_conf,
+			     int socket_id)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+	struct mlx5_devx_qp_attr attr = {0};
+	struct mlx5_crypto_qp *qp;
+	uint16_t log_nb_desc = rte_log2_u32(qp_conf->nb_descriptors);
+	uint32_t umem_size = RTE_BIT32(log_nb_desc) *
+			      MLX5_CRYPTO_WQE_SET_SIZE +
+			      sizeof(*qp->db_rec) * 2;
+	uint32_t alloc_size = sizeof(*qp);
+	struct mlx5_devx_cq_attr cq_attr = {
+		.uar_page_id = mlx5_os_get_devx_uar_page_id(priv->uar),
+	};
+
+	alloc_size = RTE_ALIGN(alloc_size, RTE_CACHE_LINE_SIZE);
+	alloc_size += sizeof(struct rte_crypto_op *) * RTE_BIT32(log_nb_desc);
+	qp = rte_zmalloc_socket(__func__, alloc_size, RTE_CACHE_LINE_SIZE,
+				socket_id);
+	if (qp == NULL) {
+		DRV_LOG(ERR, "Failed to allocate QP memory.");
+		rte_errno = ENOMEM;
+		return -rte_errno;
+	}
+	if (mlx5_devx_cq_create(priv->ctx, &qp->cq_obj, log_nb_desc,
+				&cq_attr, socket_id) != 0) {
+		DRV_LOG(ERR, "Failed to create CQ.");
+		goto error;
+	}
+	qp->umem_buf = rte_zmalloc_socket(__func__, umem_size, 4096, socket_id);
+	if (qp->umem_buf == NULL) {
+		DRV_LOG(ERR, "Failed to allocate QP umem.");
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	qp->umem_obj = mlx5_glue->devx_umem_reg(priv->ctx,
+					       (void *)(uintptr_t)qp->umem_buf,
+					       umem_size,
+					       IBV_ACCESS_LOCAL_WRITE);
+	if (qp->umem_obj == NULL) {
+		DRV_LOG(ERR, "Failed to register QP umem.");
+		goto error;
+	}
+	attr.pd = priv->pdn;
+	attr.uar_index = mlx5_os_get_devx_uar_page_id(priv->uar);
+	attr.cqn = qp->cq_obj.cq->id;
+	attr.log_page_size = rte_log2_u32(sysconf(_SC_PAGESIZE));
+	attr.rq_size =  0;
+	attr.sq_size = RTE_BIT32(log_nb_desc);
+	attr.dbr_umem_valid = 1;
+	attr.wq_umem_id = qp->umem_obj->umem_id;
+	attr.wq_umem_offset = 0;
+	attr.dbr_umem_id = qp->umem_obj->umem_id;
+	attr.dbr_address = RTE_BIT64(log_nb_desc) *
+			   MLX5_CRYPTO_WQE_SET_SIZE;
+	qp->qp_obj = mlx5_devx_cmd_create_qp(priv->ctx, &attr);
+	if (qp->qp_obj == NULL) {
+		DRV_LOG(ERR, "Failed to create QP(%u).", rte_errno);
+		goto error;
+	}
+	qp->db_rec = RTE_PTR_ADD(qp->umem_buf, (uintptr_t)attr.dbr_address);
+	if (mlx5_crypto_qp2rts(qp))
+		goto error;
+	qp->ops = (struct rte_crypto_op **)RTE_ALIGN((uintptr_t)(qp + 1),
+							   RTE_CACHE_LINE_SIZE);
+	dev->data->queue_pairs[qp_id] = qp;
+	return 0;
+error:
+	mlx5_crypto_queue_pair_release(dev, qp_id);
+	return -1;
+}
+
 static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_configure			= mlx5_crypto_dev_configure,
 	.dev_start			= NULL,
@@ -192,8 +312,8 @@ static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_infos_get			= mlx5_crypto_dev_infos_get,
 	.stats_get			= NULL,
 	.stats_reset			= NULL,
-	.queue_pair_setup		= NULL,
-	.queue_pair_release		= NULL,
+	.queue_pair_setup		= mlx5_crypto_queue_pair_setup,
+	.queue_pair_release		= mlx5_crypto_queue_pair_release,
 	.sym_session_get_size		= mlx5_crypto_sym_session_get_size,
 	.sym_session_configure		= mlx5_crypto_sym_session_configure,
 	.sym_session_clear		= mlx5_crypto_sym_session_clear,
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 5e270d3d5a..f5313b89f2 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -11,9 +11,11 @@
 #include <rte_cryptodev_pmd.h>
 
 #include <mlx5_common_utils.h>
+#include <mlx5_common_devx.h>
 
 #define MLX5_CRYPTO_DEK_HTABLE_SZ (1 << 11)
 #define MLX5_CRYPTO_KEY_LENGTH 80
+#define MLX5_CRYPTO_WQE_SET_SIZE 1024
 
 struct mlx5_crypto_priv {
 	TAILQ_ENTRY(mlx5_crypto_priv) next;
@@ -27,6 +29,15 @@ struct mlx5_crypto_priv {
 	struct rte_cryptodev_config dev_config;
 };
 
+struct mlx5_crypto_qp {
+	struct mlx5_devx_cq cq_obj;
+	struct mlx5_devx_obj *qp_obj;
+	struct mlx5dv_devx_umem *umem_obj;
+	void *umem_buf;
+	volatile uint32_t *db_rec;
+	struct rte_crypto_op **ops;
+};
+
 struct mlx5_crypto_dek {
 	struct mlx5_hlist_entry entry; /* Pointer to DEK hash list entry. */
 	struct mlx5_devx_obj *obj; /* Pointer to DEK DevX object. */
-- 
2.25.1

[dpdk-dev] [PATCH v3 06/15] crypto/mlx5: add dev stop and start operations

[Matan Azrad]

From: Shiri Kuzin <shirik@nvidia.com>

Add the dev_start function that is used to start a configured device.
Add the dev_stop function that is used to stop a configured device.

Both functions set the dev parameter as used and return 0.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 8c3417ee96..538fe5ce56 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -98,6 +98,19 @@ mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
 	return 0;
 }
 
+static void
+mlx5_crypto_dev_stop(struct rte_cryptodev *dev)
+{
+	RTE_SET_USED(dev);
+}
+
+static int
+mlx5_crypto_dev_start(struct rte_cryptodev *dev)
+{
+	RTE_SET_USED(dev);
+	return 0;
+}
+
 static int
 mlx5_crypto_dev_close(struct rte_cryptodev *dev)
 {
@@ -306,8 +319,8 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 
 static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_configure			= mlx5_crypto_dev_configure,
-	.dev_start			= NULL,
-	.dev_stop			= NULL,
+	.dev_start			= mlx5_crypto_dev_start,
+	.dev_stop			= mlx5_crypto_dev_stop,
 	.dev_close			= mlx5_crypto_dev_close,
 	.dev_infos_get			= mlx5_crypto_dev_infos_get,
 	.stats_get			= NULL,
-- 
2.25.1

[dpdk-dev] [PATCH v3 07/15] crypto/mlx5: add memory region management

[Matan Azrad]

From: Shiri Kuzin <shirik@nvidia.com>

Mellanox user space drivers don't deal with physical addresses as part
of a memory protection mechanism.
The device translates the given virtual address to a physical address
using the given memory key as an address space identifier.
That's why any mbuf virtual address is moved directly to the HW
descriptor(WQE).

The mapping between the virtual address to the physical address is saved
in MR configured by the kernel to the HW.

Each MR has a key that should also be moved to the WQE by the SW.

When the SW sees an unmapped address, it extends the address range and
creates a MR using a system call.

Add memory region cache management:
	- 2 level cache per queue-pair - no locks.
	- 1 shared cache between all the queues using a lock.

Using this way, the MR key search per data-path address is optimized.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 20 ++++++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto.h |  3 +++
 2 files changed, 23 insertions(+)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 538fe5ce56..b95aea0068 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -209,6 +209,7 @@ mlx5_crypto_queue_pair_release(struct rte_cryptodev *dev, uint16_t qp_id)
 		claim_zero(mlx5_glue->devx_umem_dereg(qp->umem_obj));
 	if (qp->umem_buf != NULL)
 		rte_free(qp->umem_buf);
+	mlx5_mr_btree_free(&qp->mr_ctrl.cache_bh);
 	mlx5_devx_cq_destroy(&qp->cq_obj);
 	rte_free(qp);
 	dev->data->queue_pairs[qp_id] = NULL;
@@ -288,6 +289,13 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 		DRV_LOG(ERR, "Failed to register QP umem.");
 		goto error;
 	}
+	if (mlx5_mr_btree_init(&qp->mr_ctrl.cache_bh, MLX5_MR_BTREE_CACHE_N,
+			       priv->dev_config.socket_id) != 0) {
+		DRV_LOG(ERR, "Cannot allocate MR Btree for qp %u.",
+			(uint32_t)qp_id);
+		rte_errno = ENOMEM;
+		goto error;
+	}
 	attr.pd = priv->pdn;
 	attr.uar_index = mlx5_os_get_devx_uar_page_id(priv->uar);
 	attr.cqn = qp->cq_obj.cq->id;
@@ -476,6 +484,17 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 		claim_zero(mlx5_glue->close_device(priv->ctx));
 		return -1;
 	}
+	if (mlx5_mr_btree_init(&priv->mr_scache.cache,
+			     MLX5_MR_BTREE_CACHE_N * 2, rte_socket_id()) != 0) {
+		DRV_LOG(ERR, "Failed to allocate shared cache MR memory.");
+		mlx5_crypto_hw_global_release(priv);
+		rte_cryptodev_pmd_destroy(priv->crypto_dev);
+		claim_zero(mlx5_glue->close_device(priv->ctx));
+		rte_errno = ENOMEM;
+		return -rte_errno;
+	}
+	priv->mr_scache.reg_mr_cb = mlx5_common_verbs_reg_mr;
+	priv->mr_scache.dereg_mr_cb = mlx5_common_verbs_dereg_mr;
 	pthread_mutex_lock(&priv_list_lock);
 	TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next);
 	pthread_mutex_unlock(&priv_list_lock);
@@ -495,6 +514,7 @@ mlx5_crypto_pci_remove(struct rte_pci_device *pdev)
 		TAILQ_REMOVE(&mlx5_crypto_priv_list, priv, next);
 	pthread_mutex_unlock(&priv_list_lock);
 	if (priv) {
+		mlx5_mr_release_cache(&priv->mr_scache);
 		mlx5_crypto_hw_global_release(priv);
 		rte_cryptodev_pmd_destroy(priv->crypto_dev);
 		claim_zero(mlx5_glue->close_device(priv->ctx));
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index f5313b89f2..397267d249 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -12,6 +12,7 @@
 
 #include <mlx5_common_utils.h>
 #include <mlx5_common_devx.h>
+#include <mlx5_common_mr.h>
 
 #define MLX5_CRYPTO_DEK_HTABLE_SZ (1 << 11)
 #define MLX5_CRYPTO_KEY_LENGTH 80
@@ -27,6 +28,7 @@ struct mlx5_crypto_priv {
 	struct ibv_pd *pd;
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
 	struct rte_cryptodev_config dev_config;
+	struct mlx5_mr_share_cache mr_scache; /* Global shared MR cache. */
 };
 
 struct mlx5_crypto_qp {
@@ -36,6 +38,7 @@ struct mlx5_crypto_qp {
 	void *umem_buf;
 	volatile uint32_t *db_rec;
 	struct rte_crypto_op **ops;
+	struct mlx5_mr_ctrl mr_ctrl;
 };
 
 struct mlx5_crypto_dek {
-- 
2.25.1

[dpdk-dev] [PATCH v3 08/15] crypto/mlx5: create login object using DevX

[Matan Azrad]

From: Shiri Kuzin <shirik@nvidia.com>

To work with crypto engines that are marked with wrapped_import_method,
a login session is required.
A crypto login object needs to be created using DevX.

The crypto login object contains:
	- The credential pointer.
	- The import_KEK pointer to be used for all secured information
	  communicated in crypto commands (key fields), including the
	  provided credential in this command.
	- The credential secret, wrapped by the import_KEK indicated in
	  this command. Size includes 8 bytes IV for wrapping.

Added devargs for the required login values:
	- wcs_file - path to the file containing the credential.
	- import_kek_id - the import KEK pointer.
	- credential_id - the credential pointer.

Create the login DevX object in pci_probe function and destroy it in
pci_remove.
Destroying the crypto login object means logout.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 103 ++++++++++++++++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto.h |   7 ++
 2 files changed, 110 insertions(+)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index b95aea0068..18b1a6be88 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -402,6 +402,101 @@ mlx5_crypto_hw_global_prepare(struct mlx5_crypto_priv *priv)
 	return 0;
 }
 
+
+static int
+mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque)
+{
+	struct mlx5_crypto_devarg_params *devarg_prms = opaque;
+	struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr;
+	unsigned long tmp;
+	FILE *file;
+	int ret;
+	int i;
+
+	if (strcmp(key, "class") == 0)
+		return 0;
+	if (strcmp(key, "wcs_file") == 0) {
+		file = fopen(val, "rb");
+		if (file == NULL) {
+			rte_errno = ENOTSUP;
+			return -rte_errno;
+		}
+		for (i = 0 ; i < MLX5_CRYPTO_CREDENTIAL_SIZE ; i++) {
+			ret = fscanf(file, "%02hhX", &attr->credential[i]);
+			if (ret <= 0) {
+				fclose(file);
+				DRV_LOG(ERR,
+					"Failed to read credential from file.");
+				rte_errno = EINVAL;
+				return -rte_errno;
+			}
+		}
+		fclose(file);
+		devarg_prms->login_devarg = true;
+		return 0;
+	}
+	errno = 0;
+	tmp = strtoul(val, NULL, 0);
+	if (errno) {
+		DRV_LOG(WARNING, "%s: \"%s\" is an invalid integer.", key, val);
+		return -errno;
+	}
+	if (strcmp(key, "import_kek_id") == 0)
+		attr->session_import_kek_ptr = (uint32_t)tmp;
+	else if (strcmp(key, "credential_id") == 0)
+		attr->credential_pointer = (uint32_t)tmp;
+	else
+		DRV_LOG(WARNING, "Invalid key %s.", key);
+	return 0;
+}
+
+static struct mlx5_devx_obj *
+mlx5_crypto_config_login(struct rte_devargs *devargs,
+			 struct ibv_context *ctx)
+{
+	/*
+	 * Set credential pointer and session import KEK pointer to a default
+	 * value of 0.
+	 */
+	struct mlx5_crypto_devarg_params login = {
+			.login_devarg = false,
+			.login_attr = {
+					.credential_pointer = 0,
+					.session_import_kek_ptr = 0,
+			}
+	};
+	struct rte_kvargs *kvlist;
+
+	if (devargs == NULL) {
+		DRV_LOG(ERR,
+	"No login devargs in order to enable crypto operations in the device.");
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	kvlist = rte_kvargs_parse(devargs->args, NULL);
+	if (kvlist == NULL) {
+		DRV_LOG(ERR, "Failed to parse devargs.");
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	if (rte_kvargs_process(kvlist, NULL, mlx5_crypto_args_check_handler,
+			   &login) != 0) {
+		DRV_LOG(ERR, "Devargs handler function Failed.");
+		rte_kvargs_free(kvlist);
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	rte_kvargs_free(kvlist);
+	if (login.login_devarg == false) {
+		DRV_LOG(ERR,
+	"No login credential devarg in order to enable crypto operations "
+	"in the device.");
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	return mlx5_devx_cmd_create_crypto_login_obj(ctx, &login.login_attr);
+}
+
 /**
  * DPDK callback to register a PCI device.
  *
@@ -423,6 +518,7 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	struct ibv_device *ibv;
 	struct rte_cryptodev *crypto_dev;
 	struct ibv_context *ctx;
+	struct mlx5_devx_obj *login;
 	struct mlx5_crypto_priv *priv;
 	struct mlx5_hca_attr attr = { 0 };
 	struct rte_cryptodev_pmd_init_params init_params = {
@@ -461,6 +557,11 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 		rte_errno = ENOTSUP;
 		return -ENOTSUP;
 	}
+	login = mlx5_crypto_config_login(pci_dev->device.devargs, ctx);
+	if (login == NULL) {
+		DRV_LOG(ERR, "Failed to configure login.");
+		return -rte_errno;
+	}
 	crypto_dev = rte_cryptodev_pmd_create(ibv->name, &pci_dev->device,
 					&init_params);
 	if (crypto_dev == NULL) {
@@ -477,6 +578,7 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	crypto_dev->driver_id = mlx5_crypto_driver_id;
 	priv = crypto_dev->data->dev_private;
 	priv->ctx = ctx;
+	priv->login_obj = login;
 	priv->pci_dev = pci_dev;
 	priv->crypto_dev = crypto_dev;
 	if (mlx5_crypto_hw_global_prepare(priv) != 0) {
@@ -517,6 +619,7 @@ mlx5_crypto_pci_remove(struct rte_pci_device *pdev)
 		mlx5_mr_release_cache(&priv->mr_scache);
 		mlx5_crypto_hw_global_release(priv);
 		rte_cryptodev_pmd_destroy(priv->crypto_dev);
+		claim_zero(mlx5_devx_cmd_destroy(priv->login_obj));
 		claim_zero(mlx5_glue->close_device(priv->ctx));
 	}
 	return 0;
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 397267d249..0aef804b92 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -29,6 +29,7 @@ struct mlx5_crypto_priv {
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
 	struct rte_cryptodev_config dev_config;
 	struct mlx5_mr_share_cache mr_scache; /* Global shared MR cache. */
+	struct mlx5_devx_obj *login_obj;
 };
 
 struct mlx5_crypto_qp {
@@ -48,6 +49,12 @@ struct mlx5_crypto_dek {
 	bool size_is_48; /* Whether the key\data size is 48 bytes or not. */
 };
 
+
+struct mlx5_crypto_devarg_params {
+	bool login_devarg;
+	struct mlx5_devx_crypto_login_attr login_attr;
+};
+
 int
 mlx5_crypto_dek_destroy(struct mlx5_crypto_priv *priv,
 			struct mlx5_crypto_dek *dek);
-- 
2.25.1

[dpdk-dev] [PATCH v3 09/15] crypto/mlx5: adjust to the multiple data unit API

[Matan Azrad]

From: Shiri Kuzin <shirik@nvidia.com>

In AES-XTS the data to be encrypted\decrypted does not have to be
in multiples of 16B size, the unit of data is called data-unit.

As a result of patch [1] a new field is added to the cipher capability,
called dataunit_set, where the devices can report the range of
supported data-unit sizes.

The new field enables saving the data-unit size in the session
structure to the block size pointer variable in order to support
several data-unit sizes.

[1] https://www.mail-archive.com/dev@dpdk.org/msg205337.html

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 18b1a6be88..8cc29ced21 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -48,6 +48,11 @@ struct mlx5_crypto_session {
 	 * bsf_size, bsf_p_type, encryption_order and encryption standard,
 	 * saved in big endian format.
 	 */
+	uint32_t bsp_res;
+	/*
+	 * crypto_block_size_pointer and reserved 24 bits saved in big endian
+	 * format.
+	 */
 	uint32_t iv_offset:16;
 	/* Starting point for Initialisation Vector. */
 	struct mlx5_crypto_dek *dek; /* Pointer to dek struct. */
@@ -171,6 +176,24 @@ mlx5_crypto_sym_session_configure(struct rte_cryptodev *dev,
 			 MLX5_BSF_P_TYPE_CRYPTO << MLX5_BSF_P_TYPE_OFFSET |
 			 encryption_order << MLX5_ENCRYPTION_ORDER_OFFSET |
 			 MLX5_ENCRYPTION_STANDARD_AES_XTS);
+	switch (xform->cipher.dataunit_len) {
+	case 0:
+		sess_private_data->bsp_res = 0;
+		break;
+	case 512:
+		sess_private_data->bsp_res = rte_cpu_to_be_32
+					     ((uint32_t)MLX5_BLOCK_SIZE_512B <<
+					     MLX5_BLOCK_SIZE_OFFSET);
+		break;
+	case 4096:
+		sess_private_data->bsp_res = rte_cpu_to_be_32
+					     ((uint32_t)MLX5_BLOCK_SIZE_4096B <<
+					     MLX5_BLOCK_SIZE_OFFSET);
+		break;
+	default:
+		DRV_LOG(ERR, "Cipher data unit length is not supported.");
+		return -ENOTSUP;
+	}
 	sess_private_data->iv_offset = cipher->iv.offset;
 	sess_private_data->dek_id =
 			rte_cpu_to_be_32(sess_private_data->dek->obj->id &
-- 
2.25.1

[dpdk-dev] [PATCH v3 10/15] crypto/mlx5: add keytag device argument

[Matan Azrad]

From: Suanming Mou <suanmingm@nvidia.com>

A keytag is a piece of data encrypted together with a DEK.

When a DEK is referenced by an MKEY.bsf through its index, the keytag is
also supplied in the BSF as plaintext. The HW will decrypt the DEK (and
the attached keytag) and will fail the operation if the keytags don't
match.

This commit adds the configuration of the keytag with devargs.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 50 +++++++++++++++++--------------
 drivers/crypto/mlx5/mlx5_crypto.h |  3 +-
 2 files changed, 30 insertions(+), 23 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 8cc29ced21..73cca8136b 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -468,56 +468,52 @@ mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque)
 		attr->session_import_kek_ptr = (uint32_t)tmp;
 	else if (strcmp(key, "credential_id") == 0)
 		attr->credential_pointer = (uint32_t)tmp;
+	else if (strcmp(key, "keytag") == 0)
+		devarg_prms->keytag = tmp;
 	else
 		DRV_LOG(WARNING, "Invalid key %s.", key);
 	return 0;
 }
 
-static struct mlx5_devx_obj *
-mlx5_crypto_config_login(struct rte_devargs *devargs,
-			 struct ibv_context *ctx)
+static int
+mlx5_crypto_parse_devargs(struct rte_devargs *devargs,
+			  struct mlx5_crypto_devarg_params *devarg_prms)
 {
-	/*
-	 * Set credential pointer and session import KEK pointer to a default
-	 * value of 0.
-	 */
-	struct mlx5_crypto_devarg_params login = {
-			.login_devarg = false,
-			.login_attr = {
-					.credential_pointer = 0,
-					.session_import_kek_ptr = 0,
-			}
-	};
+	struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr;
 	struct rte_kvargs *kvlist;
 
+	/* Default values. */
+	attr->credential_pointer = 0;
+	attr->session_import_kek_ptr = 0;
+	devarg_prms->keytag = 0;
 	if (devargs == NULL) {
 		DRV_LOG(ERR,
 	"No login devargs in order to enable crypto operations in the device.");
 		rte_errno = EINVAL;
-		return NULL;
+		return -1;
 	}
 	kvlist = rte_kvargs_parse(devargs->args, NULL);
 	if (kvlist == NULL) {
 		DRV_LOG(ERR, "Failed to parse devargs.");
 		rte_errno = EINVAL;
-		return NULL;
+		return -1;
 	}
 	if (rte_kvargs_process(kvlist, NULL, mlx5_crypto_args_check_handler,
-			   &login) != 0) {
+			   devarg_prms) != 0) {
 		DRV_LOG(ERR, "Devargs handler function Failed.");
 		rte_kvargs_free(kvlist);
 		rte_errno = EINVAL;
-		return NULL;
+		return -1;
 	}
 	rte_kvargs_free(kvlist);
-	if (login.login_devarg == false) {
+	if (devarg_prms->login_devarg == false) {
 		DRV_LOG(ERR,
 	"No login credential devarg in order to enable crypto operations "
 	"in the device.");
 		rte_errno = EINVAL;
-		return NULL;
+		return -1;
 	}
-	return mlx5_devx_cmd_create_crypto_login_obj(ctx, &login.login_attr);
+	return 0;
 }
 
 /**
@@ -543,6 +539,7 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	struct ibv_context *ctx;
 	struct mlx5_devx_obj *login;
 	struct mlx5_crypto_priv *priv;
+	struct mlx5_crypto_devarg_params devarg_prms = { 0 };
 	struct mlx5_hca_attr attr = { 0 };
 	struct rte_cryptodev_pmd_init_params init_params = {
 		.name = "",
@@ -551,6 +548,8 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 		.max_nb_queue_pairs =
 				RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
 	};
+	int ret;
+
 	RTE_SET_USED(pci_drv);
 	if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
 		DRV_LOG(ERR, "Non-primary process type is not supported.");
@@ -580,7 +579,13 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 		rte_errno = ENOTSUP;
 		return -ENOTSUP;
 	}
-	login = mlx5_crypto_config_login(pci_dev->device.devargs, ctx);
+	ret = mlx5_crypto_parse_devargs(pci_dev->device.devargs, &devarg_prms);
+	if (ret) {
+		DRV_LOG(ERR, "Failed to parse devargs.");
+		return -rte_errno;
+	}
+	login = mlx5_devx_cmd_create_crypto_login_obj(ctx,
+						      &devarg_prms.login_attr);
 	if (login == NULL) {
 		DRV_LOG(ERR, "Failed to configure login.");
 		return -rte_errno;
@@ -620,6 +625,7 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	}
 	priv->mr_scache.reg_mr_cb = mlx5_common_verbs_reg_mr;
 	priv->mr_scache.dereg_mr_cb = mlx5_common_verbs_dereg_mr;
+	priv->keytag = rte_cpu_to_be_64(devarg_prms.keytag);
 	pthread_mutex_lock(&priv_list_lock);
 	TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next);
 	pthread_mutex_unlock(&priv_list_lock);
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 0aef804b92..34c65f9a24 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -30,6 +30,7 @@ struct mlx5_crypto_priv {
 	struct rte_cryptodev_config dev_config;
 	struct mlx5_mr_share_cache mr_scache; /* Global shared MR cache. */
 	struct mlx5_devx_obj *login_obj;
+	uint64_t keytag;
 };
 
 struct mlx5_crypto_qp {
@@ -49,10 +50,10 @@ struct mlx5_crypto_dek {
 	bool size_is_48; /* Whether the key\data size is 48 bytes or not. */
 };
 
-
 struct mlx5_crypto_devarg_params {
 	bool login_devarg;
 	struct mlx5_devx_crypto_login_attr login_attr;
+	uint64_t keytag;
 };
 
 int
-- 
2.25.1

[dpdk-dev] [PATCH v3 11/15] crypto/mlx5: add maximum segments device argument

[Matan Azrad]

From: Suanming Mou <suanmingm@nvidia.com>

The mlx5 HW crypto operations are done by attaching crypto property
to a memory region. Once done, every access to the memory via the
crypto-enabled memory region will result with in-line encryption or
decryption of the data.

As a result, the design choice is to provide two types of WQEs. One
is UMR WQE which sets the crypto property and the other is rdma write
WQE which sends DMA command to copy data from local MR to remote MR.

The size of the WQEs will be defined by a new devarg called
max_segs_num.

This devarg also defines the maximum segments in mbuf chain that will be
supported for crypto operations.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 35 +++++++++++++++++++++++++++----
 drivers/crypto/mlx5/mlx5_crypto.h |  7 +++++++
 2 files changed, 38 insertions(+), 4 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 73cca8136b..6de44398bd 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -21,6 +21,7 @@
 #define MLX5_CRYPTO_DRIVER_NAME mlx5_crypto
 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
 #define MLX5_CRYPTO_MAX_QPS 1024
+#define MLX5_CRYPTO_MAX_SEGS 56
 
 TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
 				TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
@@ -464,14 +465,24 @@ mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque)
 		DRV_LOG(WARNING, "%s: \"%s\" is an invalid integer.", key, val);
 		return -errno;
 	}
-	if (strcmp(key, "import_kek_id") == 0)
+	if (strcmp(key, "max_segs_num") == 0) {
+		if (!tmp || tmp > MLX5_CRYPTO_MAX_SEGS) {
+			DRV_LOG(WARNING, "Invalid max_segs_num: %d, should"
+				" be less than %d.",
+				(uint32_t)tmp, MLX5_CRYPTO_MAX_SEGS);
+			rte_errno = EINVAL;
+			return -rte_errno;
+		}
+		devarg_prms->max_segs_num = (uint32_t)tmp;
+	} else if (strcmp(key, "import_kek_id") == 0) {
 		attr->session_import_kek_ptr = (uint32_t)tmp;
-	else if (strcmp(key, "credential_id") == 0)
+	} else if (strcmp(key, "credential_id") == 0) {
 		attr->credential_pointer = (uint32_t)tmp;
-	else if (strcmp(key, "keytag") == 0)
+	} else if (strcmp(key, "keytag") == 0) {
 		devarg_prms->keytag = tmp;
-	else
+	} else {
 		DRV_LOG(WARNING, "Invalid key %s.", key);
+	}
 	return 0;
 }
 
@@ -486,6 +497,7 @@ mlx5_crypto_parse_devargs(struct rte_devargs *devargs,
 	attr->credential_pointer = 0;
 	attr->session_import_kek_ptr = 0;
 	devarg_prms->keytag = 0;
+	devarg_prms->max_segs_num = 8;
 	if (devargs == NULL) {
 		DRV_LOG(ERR,
 	"No login devargs in order to enable crypto operations in the device.");
@@ -626,6 +638,21 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	priv->mr_scache.reg_mr_cb = mlx5_common_verbs_reg_mr;
 	priv->mr_scache.dereg_mr_cb = mlx5_common_verbs_dereg_mr;
 	priv->keytag = rte_cpu_to_be_64(devarg_prms.keytag);
+	priv->max_segs_num = devarg_prms.max_segs_num;
+	priv->umr_wqe_size = sizeof(struct mlx5_wqe_umr_bsf_seg) +
+			     sizeof(struct mlx5_umr_wqe) +
+			     RTE_ALIGN(priv->max_segs_num, 4) *
+			     sizeof(struct mlx5_wqe_dseg);
+	priv->rdmw_wqe_size = sizeof(struct mlx5_rdma_write_wqe) +
+			      sizeof(struct mlx5_wqe_dseg) *
+			      (priv->max_segs_num <= 2 ? 2 : 2 +
+			       RTE_ALIGN(priv->max_segs_num - 2, 4));
+	priv->wqe_set_size = priv->umr_wqe_size + priv->rdmw_wqe_size;
+	priv->wqe_stride = (priv->umr_wqe_size + priv->rdmw_wqe_size) /
+							       MLX5_SEND_WQE_BB;
+	priv->max_rdmaw_klm_n = (priv->rdmw_wqe_size -
+				 sizeof(struct mlx5_rdma_write_wqe)) /
+				 sizeof(struct mlx5_wqe_dseg);
 	pthread_mutex_lock(&priv_list_lock);
 	TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next);
 	pthread_mutex_unlock(&priv_list_lock);
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 34c65f9a24..81452bd700 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -25,12 +25,18 @@ struct mlx5_crypto_priv {
 	struct rte_cryptodev *crypto_dev;
 	void *uar; /* User Access Region. */
 	uint32_t pdn; /* Protection Domain number. */
+	uint32_t max_segs_num; /* Maximum supported data segs. */
 	struct ibv_pd *pd;
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
 	struct rte_cryptodev_config dev_config;
 	struct mlx5_mr_share_cache mr_scache; /* Global shared MR cache. */
 	struct mlx5_devx_obj *login_obj;
 	uint64_t keytag;
+	uint16_t wqe_set_size;
+	uint16_t umr_wqe_size;
+	uint16_t rdmw_wqe_size;
+	uint16_t wqe_stride;
+	uint16_t max_rdmaw_klm_n;
 };
 
 struct mlx5_crypto_qp {
@@ -54,6 +60,7 @@ struct mlx5_crypto_devarg_params {
 	bool login_devarg;
 	struct mlx5_devx_crypto_login_attr login_attr;
 	uint64_t keytag;
+	uint32_t max_segs_num;
 };
 
 int
-- 
2.25.1

[dpdk-dev] [PATCH v3 12/15] crypto/mlx5: add WQE set initialization

[Matan Azrad]

From: Suanming Mou <suanmingm@nvidia.com>

Currently, HW handles the WQEs much faster than the software,
Using the constant WQE set layout can initialize most of the WQE
segments in advanced, and software only needs to configure very
limited segments in datapath. This accelerates the software WQE
organize in datapath.

This commit initializes the fixed WQE set segments.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 83 +++++++++++++++++++++++++++++--
 drivers/crypto/mlx5/mlx5_crypto.h | 10 +++-
 2 files changed, 87 insertions(+), 6 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 6de44398bd..7bffe08bfe 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -268,6 +268,69 @@ mlx5_crypto_qp2rts(struct mlx5_crypto_qp *qp)
 	return 0;
 }
 
+static void
+mlx5_crypto_qp_init(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp)
+{
+	uint32_t i;
+
+	for (i = 0 ; i < qp->entries_n; i++) {
+		struct mlx5_wqe_cseg *cseg = RTE_PTR_ADD(qp->umem_buf, i *
+							 priv->wqe_set_size);
+		struct mlx5_wqe_umr_cseg *ucseg = (struct mlx5_wqe_umr_cseg *)
+								     (cseg + 1);
+		struct mlx5_wqe_umr_bsf_seg *bsf =
+			(struct mlx5_wqe_umr_bsf_seg *)(RTE_PTR_ADD(cseg,
+						       priv->umr_wqe_size)) - 1;
+		struct mlx5_wqe_rseg *rseg;
+
+		/* Init UMR WQE. */
+		cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj->id << 8) |
+					 (priv->umr_wqe_size / MLX5_WSEG_SIZE));
+		cseg->flags = RTE_BE32(MLX5_COMP_ONLY_FIRST_ERR <<
+				       MLX5_COMP_MODE_OFFSET);
+		cseg->misc = rte_cpu_to_be_32(qp->mkey[i]->id);
+		ucseg->if_cf_toe_cq_res = RTE_BE32(1u << MLX5_UMRC_IF_OFFSET);
+		ucseg->mkey_mask = RTE_BE64(1u << 0); /* Mkey length bit. */
+		ucseg->ko_to_bs = rte_cpu_to_be_32
+			((RTE_ALIGN(priv->max_segs_num, 4u) <<
+			 MLX5_UMRC_KO_OFFSET) | (4 << MLX5_UMRC_TO_BS_OFFSET));
+		bsf->keytag = priv->keytag;
+		/* Init RDMA WRITE WQE. */
+		cseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size);
+		cseg->flags = RTE_BE32((MLX5_COMP_ALWAYS <<
+				      MLX5_COMP_MODE_OFFSET) |
+				      MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE);
+		rseg = (struct mlx5_wqe_rseg *)(cseg + 1);
+		rseg->rkey = rte_cpu_to_be_32(qp->mkey[i]->id);
+	}
+}
+
+static int
+mlx5_crypto_indirect_mkeys_prepare(struct mlx5_crypto_priv *priv,
+				  struct mlx5_crypto_qp *qp)
+{
+	struct mlx5_umr_wqe *umr;
+	uint32_t i;
+	struct mlx5_devx_mkey_attr attr = {
+		.pd = priv->pdn,
+		.umr_en = 1,
+		.crypto_en = 1,
+		.set_remote_rw = 1,
+		.klm_num = RTE_ALIGN(priv->max_segs_num, 4),
+	};
+
+	for (umr = (struct mlx5_umr_wqe *)qp->umem_buf, i = 0;
+	   i < qp->entries_n; i++, umr = RTE_PTR_ADD(umr, priv->wqe_set_size)) {
+		attr.klm_array = (struct mlx5_klm *)&umr->kseg[0];
+		qp->mkey[i] = mlx5_devx_cmd_mkey_create(priv->ctx, &attr);
+		if (!qp->mkey[i]) {
+			DRV_LOG(ERR, "Failed to allocate indirect mkey.");
+			return -1;
+		}
+	}
+	return 0;
+}
+
 static int
 mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 			     const struct rte_cryptodev_qp_conf *qp_conf,
@@ -278,7 +341,7 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	struct mlx5_crypto_qp *qp;
 	uint16_t log_nb_desc = rte_log2_u32(qp_conf->nb_descriptors);
 	uint32_t umem_size = RTE_BIT32(log_nb_desc) *
-			      MLX5_CRYPTO_WQE_SET_SIZE +
+			      priv->wqe_set_size +
 			      sizeof(*qp->db_rec) * 2;
 	uint32_t alloc_size = sizeof(*qp);
 	struct mlx5_devx_cq_attr cq_attr = {
@@ -286,7 +349,9 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	};
 
 	alloc_size = RTE_ALIGN(alloc_size, RTE_CACHE_LINE_SIZE);
-	alloc_size += sizeof(struct rte_crypto_op *) * RTE_BIT32(log_nb_desc);
+	alloc_size += (sizeof(struct rte_crypto_op *) +
+		       sizeof(struct mlx5_devx_obj *)) *
+		       RTE_BIT32(log_nb_desc);
 	qp = rte_zmalloc_socket(__func__, alloc_size, RTE_CACHE_LINE_SIZE,
 				socket_id);
 	if (qp == NULL) {
@@ -330,8 +395,7 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	attr.wq_umem_id = qp->umem_obj->umem_id;
 	attr.wq_umem_offset = 0;
 	attr.dbr_umem_id = qp->umem_obj->umem_id;
-	attr.dbr_address = RTE_BIT64(log_nb_desc) *
-			   MLX5_CRYPTO_WQE_SET_SIZE;
+	attr.dbr_address = RTE_BIT64(log_nb_desc) * priv->wqe_set_size;
 	qp->qp_obj = mlx5_devx_cmd_create_qp(priv->ctx, &attr);
 	if (qp->qp_obj == NULL) {
 		DRV_LOG(ERR, "Failed to create QP(%u).", rte_errno);
@@ -340,8 +404,17 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	qp->db_rec = RTE_PTR_ADD(qp->umem_buf, (uintptr_t)attr.dbr_address);
 	if (mlx5_crypto_qp2rts(qp))
 		goto error;
-	qp->ops = (struct rte_crypto_op **)RTE_ALIGN((uintptr_t)(qp + 1),
+	qp->mkey = (struct mlx5_devx_obj **)RTE_ALIGN((uintptr_t)(qp + 1),
 							   RTE_CACHE_LINE_SIZE);
+	qp->ops = (struct rte_crypto_op **)(qp->mkey + RTE_BIT32(log_nb_desc));
+	qp->entries_n = 1 << log_nb_desc;
+	if (mlx5_crypto_indirect_mkeys_prepare(priv, qp)) {
+		DRV_LOG(ERR, "Cannot allocate indirect memory regions.");
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	mlx5_crypto_qp_init(priv, qp);
+	qp->priv = priv;
 	dev->data->queue_pairs[qp_id] = qp;
 	return 0;
 error:
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 81452bd700..52fcf5217f 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -16,7 +16,6 @@
 
 #define MLX5_CRYPTO_DEK_HTABLE_SZ (1 << 11)
 #define MLX5_CRYPTO_KEY_LENGTH 80
-#define MLX5_CRYPTO_WQE_SET_SIZE 1024
 
 struct mlx5_crypto_priv {
 	TAILQ_ENTRY(mlx5_crypto_priv) next;
@@ -24,6 +23,7 @@ struct mlx5_crypto_priv {
 	struct rte_pci_device *pci_dev;
 	struct rte_cryptodev *crypto_dev;
 	void *uar; /* User Access Region. */
+	volatile uint64_t *uar_addr;
 	uint32_t pdn; /* Protection Domain number. */
 	uint32_t max_segs_num; /* Maximum supported data segs. */
 	struct ibv_pd *pd;
@@ -40,13 +40,21 @@ struct mlx5_crypto_priv {
 };
 
 struct mlx5_crypto_qp {
+	struct mlx5_crypto_priv *priv;
 	struct mlx5_devx_cq cq_obj;
 	struct mlx5_devx_obj *qp_obj;
+	struct rte_cryptodev_stats stats;
 	struct mlx5dv_devx_umem *umem_obj;
 	void *umem_buf;
 	volatile uint32_t *db_rec;
 	struct rte_crypto_op **ops;
+	struct mlx5_devx_obj **mkey; /* WQE's indirect mekys. */
 	struct mlx5_mr_ctrl mr_ctrl;
+	uint8_t *wqe;
+	uint16_t entries_n;
+	uint16_t pi;
+	uint16_t ci;
+	uint16_t db_pi;
 };
 
 struct mlx5_crypto_dek {
-- 
2.25.1

[dpdk-dev] [PATCH v3 13/15] crypto/mlx5: add enqueue and dequeue operations

[Matan Azrad]

From: Suanming Mou <suanmingm@nvidia.com>

The crypto operations are done with the WQE set which contains
one UMR WQE and one rdma write WQE. Most segments of the WQE
set are initialized properly during queue setup, only limited
segments are initialized according to the crypto detail in the
datapath process.

This commit adds the enquue and dequeue operations and updates
the WQE set segments accordingly.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 243 +++++++++++++++++++++++++++++-
 drivers/crypto/mlx5/mlx5_crypto.h |   3 +
 2 files changed, 241 insertions(+), 5 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 7bffe08bfe..fd0afb9fb2 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -268,6 +268,239 @@ mlx5_crypto_qp2rts(struct mlx5_crypto_qp *qp)
 	return 0;
 }
 
+static __rte_noinline uint32_t
+mlx5_crypto_get_block_size(struct rte_crypto_op *op)
+{
+	uint32_t bl = op->sym->cipher.data.length;
+
+	switch (bl) {
+	case (1 << 20):
+		return RTE_BE32(MLX5_BLOCK_SIZE_1MB << MLX5_BLOCK_SIZE_OFFSET);
+	case (1 << 12):
+		return RTE_BE32(MLX5_BLOCK_SIZE_4096B <<
+				MLX5_BLOCK_SIZE_OFFSET);
+	case (1 << 9):
+		return RTE_BE32(MLX5_BLOCK_SIZE_512B << MLX5_BLOCK_SIZE_OFFSET);
+	default:
+		DRV_LOG(ERR, "Unknown block size: %u.", bl);
+		return UINT32_MAX;
+	}
+}
+
+static __rte_always_inline uint32_t
+mlx5_crypto_klm_set(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp,
+		      struct rte_mbuf *mbuf, struct mlx5_wqe_dseg *klm,
+		      uint32_t offset, uint32_t *remain)
+{
+	uint32_t data_len = (rte_pktmbuf_data_len(mbuf) - offset);
+	uintptr_t addr = rte_pktmbuf_mtod_offset(mbuf, uintptr_t, offset);
+
+	if (data_len > *remain)
+		data_len = *remain;
+	*remain -= data_len;
+	klm->bcount = rte_cpu_to_be_32(data_len);
+	klm->pbuf = rte_cpu_to_be_64(addr);
+	klm->lkey = mlx5_mr_addr2mr_bh(priv->pd, 0,
+		&priv->mr_scache, &qp->mr_ctrl, addr,
+		!!(mbuf->ol_flags & EXT_ATTACHED_MBUF));
+	return klm->lkey;
+
+}
+
+static __rte_always_inline uint32_t
+mlx5_crypto_klms_set(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp,
+		     struct rte_crypto_op *op, struct rte_mbuf *mbuf,
+		     struct mlx5_wqe_dseg *klm)
+{
+	uint32_t remain_len = op->sym->cipher.data.length;
+	uint32_t nb_segs = mbuf->nb_segs;
+	uint32_t klm_n = 1;
+
+	/* First mbuf needs to take the cipher offset. */
+	if (unlikely(mlx5_crypto_klm_set(priv, qp, mbuf, klm,
+		     op->sym->cipher.data.offset, &remain_len) == UINT32_MAX)) {
+		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return 0;
+	}
+	while (remain_len) {
+		nb_segs--;
+		mbuf = mbuf->next;
+		if (unlikely(mbuf == NULL || nb_segs == 0)) {
+			op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+			return 0;
+		}
+		if (unlikely(mlx5_crypto_klm_set(priv, qp, mbuf, klm, 0,
+						 &remain_len) == UINT32_MAX)) {
+			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			return 0;
+		}
+		klm_n++;
+	}
+	return klm_n;
+}
+
+static __rte_always_inline int
+mlx5_crypto_wqe_set(struct mlx5_crypto_priv *priv,
+			 struct mlx5_crypto_qp *qp,
+			 struct rte_crypto_op *op,
+			 struct mlx5_umr_wqe *umr)
+{
+	struct mlx5_crypto_session *sess = get_sym_session_private_data
+				(op->sym->session, mlx5_crypto_driver_id);
+	struct mlx5_wqe_cseg *cseg = &umr->ctr;
+	struct mlx5_wqe_mkey_cseg *mkc = &umr->mkc;
+	struct mlx5_wqe_dseg *klms = &umr->kseg[0];
+	struct mlx5_wqe_umr_bsf_seg *bsf = ((struct mlx5_wqe_umr_bsf_seg *)
+				      RTE_PTR_ADD(umr, priv->umr_wqe_size)) - 1;
+	uint16_t nop_ds;
+	/* Set UMR WQE. */
+	uint32_t klm_n = mlx5_crypto_klms_set(priv, qp, op,
+			op->sym->m_dst ? op->sym->m_dst : op->sym->m_src, klms);
+
+	if (unlikely(klm_n == 0))
+		return 0;
+	bsf->bs_bpt_eo_es = sess->bs_bpt_eo_es;
+	if (unlikely(!sess->bsp_res)) {
+		bsf->bsp_res = mlx5_crypto_get_block_size(op);
+		if (unlikely(bsf->bsp_res == UINT32_MAX)) {
+			op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+			return 0;
+		}
+	} else {
+		bsf->bsp_res = sess->bsp_res;
+	}
+	bsf->raw_data_size = rte_cpu_to_be_32(op->sym->cipher.data.length);
+	memcpy(bsf->xts_initial_tweak,
+	       rte_crypto_op_ctod_offset(op, uint8_t *, sess->iv_offset), 16);
+	bsf->res_dp = sess->dek_id;
+	cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) | MLX5_OPCODE_UMR);
+	mkc->len = rte_cpu_to_be_64(op->sym->cipher.data.length);
+	/* Set RDMA_WRITE WQE. */
+	cseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size);
+	klms = RTE_PTR_ADD(cseg, sizeof(struct mlx5_rdma_write_wqe));
+	cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) |
+					MLX5_OPCODE_RDMA_WRITE);
+	if (op->sym->m_dst != op->sym->m_src) {
+		klm_n = mlx5_crypto_klms_set(priv, qp, op, op->sym->m_src,
+					     klms);
+		if (unlikely(klm_n == 0))
+			return 0;
+	} else {
+		memcpy(klms, &umr->kseg[0], sizeof(*klms) * klm_n);
+	}
+	cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj->id << 8) | (2 + klm_n));
+	qp->db_pi += priv->wqe_stride;
+	/* Set NOP WQE if needed. */
+	klm_n = RTE_ALIGN(klm_n + 2, 4) - 2;
+	nop_ds = priv->max_rdmaw_klm_n - klm_n;
+	if (nop_ds) {
+		cseg = (struct mlx5_wqe_cseg *)(klms + klm_n);
+		cseg->opcode = rte_cpu_to_be_32(((qp->db_pi - (nop_ds >> 2)) <<
+						 8) | MLX5_OPCODE_NOP);
+		cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj->id << 8) | nop_ds);
+	}
+	qp->wqe = (uint8_t *)cseg;
+	return 1;
+}
+
+static __rte_always_inline void
+mlx5_crypto_uar_write(uint64_t val, struct mlx5_crypto_priv *priv)
+{
+#ifdef RTE_ARCH_64
+	*priv->uar_addr = val;
+#else /* !RTE_ARCH_64 */
+	rte_spinlock_lock(&priv->uar32_sl);
+	*(volatile uint32_t *)priv->uar_addr = val;
+	rte_io_wmb();
+	*((volatile uint32_t *)priv->uar_addr + 1) = val >> 32;
+	rte_spinlock_unlock(&priv->uar32_sl);
+#endif
+}
+
+static uint16_t
+mlx5_crypto_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
+			  uint16_t nb_ops)
+{
+	struct mlx5_crypto_qp *qp = queue_pair;
+	struct mlx5_crypto_priv *priv = qp->priv;
+	struct mlx5_umr_wqe *umr;
+	struct rte_crypto_op *op;
+	uint16_t mask = qp->entries_n - 1;
+	uint16_t remain = qp->entries_n - (qp->pi - qp->ci);
+
+	if (remain < nb_ops)
+		nb_ops = remain;
+	else
+		remain = nb_ops;
+	if (unlikely(remain == 0))
+		return 0;
+	do {
+		op = *ops++;
+		umr = RTE_PTR_ADD(qp->umem_buf, priv->wqe_set_size * qp->pi);
+		if (unlikely(mlx5_crypto_wqe_set(priv, qp, op, umr) == 0))
+			break;
+		qp->ops[qp->pi] = op;
+		qp->pi = (qp->pi + 1) & mask;
+	} while (--remain);
+	rte_io_wmb();
+	qp->db_rec[MLX5_SND_DBR] = rte_cpu_to_be_32(qp->db_pi);
+	rte_wmb();
+	mlx5_crypto_uar_write(*(volatile uint64_t *)qp->wqe, qp->priv);
+	rte_wmb();
+	return nb_ops;
+}
+
+static __rte_noinline void
+mlx5_crypto_cqe_err_handle(struct mlx5_crypto_qp *qp, struct rte_crypto_op *op)
+{
+	const uint32_t idx = qp->ci & (qp->entries_n - 1);
+	volatile struct mlx5_err_cqe *cqe = (volatile struct mlx5_err_cqe *)
+							&qp->cq_obj.cqes[idx];
+
+	op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+	DRV_LOG(ERR, "CQE ERR:%x.\n", rte_be_to_cpu_32(cqe->syndrome));
+}
+
+static uint16_t
+mlx5_crypto_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
+			  uint16_t nb_ops)
+{
+	struct mlx5_crypto_qp *qp = queue_pair;
+	volatile struct mlx5_cqe *restrict cqe;
+	struct rte_crypto_op *restrict op;
+	const unsigned int cq_size = qp->entries_n;
+	const unsigned int mask = cq_size - 1;
+	uint32_t idx;
+	uint32_t next_idx = qp->ci & mask;
+	const uint16_t max = RTE_MIN((uint16_t)(qp->pi - qp->ci), nb_ops);
+	uint16_t i = 0;
+	int ret;
+
+	if (unlikely(max == 0))
+		return 0;
+	do {
+		idx = next_idx;
+		next_idx = (qp->ci + 1) & mask;
+		op = qp->ops[idx];
+		cqe = &qp->cq_obj.cqes[idx];
+		ret = check_cqe(cqe, cq_size, qp->ci);
+		rte_io_rmb();
+		if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) {
+			if (unlikely(ret != MLX5_CQE_STATUS_HW_OWN))
+				mlx5_crypto_cqe_err_handle(qp, op);
+			break;
+		}
+		op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+		ops[i++] = op;
+		qp->ci++;
+	} while (i < max);
+	if (likely(i != 0)) {
+		rte_io_wmb();
+		qp->cq_obj.db_rec[0] = rte_cpu_to_be_32(qp->ci);
+	}
+	return i;
+}
+
 static void
 mlx5_crypto_qp_init(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp)
 {
@@ -489,8 +722,9 @@ mlx5_crypto_hw_global_prepare(struct mlx5_crypto_priv *priv)
 	if (mlx5_crypto_pd_create(priv) != 0)
 		return -1;
 	priv->uar = mlx5_devx_alloc_uar(priv->ctx, -1);
-	if (priv->uar == NULL || mlx5_os_get_devx_uar_reg_addr(priv->uar) ==
-	    NULL) {
+	if (priv->uar)
+		priv->uar_addr = mlx5_os_get_devx_uar_reg_addr(priv->uar);
+	if (priv->uar == NULL || priv->uar_addr == NULL) {
 		rte_errno = errno;
 		claim_zero(mlx5_glue->dealloc_pd(priv->pd));
 		DRV_LOG(ERR, "Failed to allocate UAR.");
@@ -685,9 +919,8 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	DRV_LOG(INFO,
 		"Crypto device %s was created successfully.", ibv->name);
 	crypto_dev->dev_ops = &mlx5_crypto_ops;
-	crypto_dev->dequeue_burst = NULL;
-	crypto_dev->enqueue_burst = NULL;
-	crypto_dev->feature_flags = RTE_CRYPTODEV_FF_HW_ACCELERATED;
+	crypto_dev->dequeue_burst = mlx5_crypto_dequeue_burst;
+	crypto_dev->enqueue_burst = mlx5_crypto_enqueue_burst;
 	crypto_dev->driver_id = mlx5_crypto_driver_id;
 	priv = crypto_dev->data->dev_private;
 	priv->ctx = ctx;
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 52fcf5217f..ac4ad1834f 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -37,6 +37,9 @@ struct mlx5_crypto_priv {
 	uint16_t rdmw_wqe_size;
 	uint16_t wqe_stride;
 	uint16_t max_rdmaw_klm_n;
+#ifndef RTE_ARCH_64
+	rte_spinlock_t uar32_sl;
+#endif /* RTE_ARCH_64 */
 };
 
 struct mlx5_crypto_qp {
-- 
2.25.1

[dpdk-dev] [PATCH v3 14/15] crypto/mlx5: add statistic get and reset operations

[Matan Azrad]

From: Suanming Mou <suanmingm@nvidia.com>

This commit adds mlx5 crypto statistic get and reset operations.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 39 ++++++++++++++++++++++++++++---
 1 file changed, 36 insertions(+), 3 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index fd0afb9fb2..896ca60866 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -437,11 +437,14 @@ mlx5_crypto_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
 	do {
 		op = *ops++;
 		umr = RTE_PTR_ADD(qp->umem_buf, priv->wqe_set_size * qp->pi);
-		if (unlikely(mlx5_crypto_wqe_set(priv, qp, op, umr) == 0))
+		if (unlikely(mlx5_crypto_wqe_set(priv, qp, op, umr) == 0)) {
+			qp->stats.enqueue_err_count++;
 			break;
+		}
 		qp->ops[qp->pi] = op;
 		qp->pi = (qp->pi + 1) & mask;
 	} while (--remain);
+	qp->stats.enqueued_count += nb_ops;
 	rte_io_wmb();
 	qp->db_rec[MLX5_SND_DBR] = rte_cpu_to_be_32(qp->db_pi);
 	rte_wmb();
@@ -458,6 +461,7 @@ mlx5_crypto_cqe_err_handle(struct mlx5_crypto_qp *qp, struct rte_crypto_op *op)
 							&qp->cq_obj.cqes[idx];
 
 	op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+	qp->stats.dequeue_err_count++;
 	DRV_LOG(ERR, "CQE ERR:%x.\n", rte_be_to_cpu_32(cqe->syndrome));
 }
 
@@ -497,6 +501,7 @@ mlx5_crypto_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
 	if (likely(i != 0)) {
 		rte_io_wmb();
 		qp->cq_obj.db_rec[0] = rte_cpu_to_be_32(qp->ci);
+		qp->stats.dequeued_count += i;
 	}
 	return i;
 }
@@ -655,14 +660,42 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	return -1;
 }
 
+static void
+mlx5_crypto_stats_get(struct rte_cryptodev *dev,
+		      struct rte_cryptodev_stats *stats)
+{
+	int qp_id;
+
+	for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
+		struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
+
+		stats->enqueued_count += qp->stats.enqueued_count;
+		stats->dequeued_count += qp->stats.dequeued_count;
+		stats->enqueue_err_count += qp->stats.enqueue_err_count;
+		stats->dequeue_err_count += qp->stats.dequeue_err_count;
+	}
+}
+
+static void
+mlx5_crypto_stats_reset(struct rte_cryptodev *dev)
+{
+	int qp_id;
+
+	for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
+		struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
+
+		memset(&qp->stats, 0, sizeof(qp->stats));
+	}
+}
+
 static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_configure			= mlx5_crypto_dev_configure,
 	.dev_start			= mlx5_crypto_dev_start,
 	.dev_stop			= mlx5_crypto_dev_stop,
 	.dev_close			= mlx5_crypto_dev_close,
 	.dev_infos_get			= mlx5_crypto_dev_infos_get,
-	.stats_get			= NULL,
-	.stats_reset			= NULL,
+	.stats_get			= mlx5_crypto_stats_get,
+	.stats_reset			= mlx5_crypto_stats_reset,
 	.queue_pair_setup		= mlx5_crypto_queue_pair_setup,
 	.queue_pair_release		= mlx5_crypto_queue_pair_release,
 	.sym_session_get_size		= mlx5_crypto_sym_session_get_size,
-- 
2.25.1

[dpdk-dev] [PATCH v3 15/15] crypto/mlx5: set feature flags and capabilities

[Matan Azrad]

From: Shiri Kuzin <shirik@nvidia.com>

Add the supported capabilities to the crypto driver.

Add supported feature flags.

Add crypto driver documentation.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
 doc/guides/cryptodevs/features/mlx5.ini |  37 ++++++
 doc/guides/cryptodevs/index.rst         |   1 +
 doc/guides/cryptodevs/mlx5.rst          | 152 ++++++++++++++++++++++++
 doc/guides/rel_notes/release_21_05.rst  |   5 +
 drivers/crypto/mlx5/mlx5_crypto.c       |  39 +++++-
 5 files changed, 231 insertions(+), 3 deletions(-)
 create mode 100644 doc/guides/cryptodevs/features/mlx5.ini
 create mode 100644 doc/guides/cryptodevs/mlx5.rst

diff --git a/doc/guides/cryptodevs/features/mlx5.ini b/doc/guides/cryptodevs/features/mlx5.ini
new file mode 100644
index 0000000000..a89526add0
--- /dev/null
+++ b/doc/guides/cryptodevs/features/mlx5.ini
@@ -0,0 +1,37 @@
+;
+; Features of a mlx5 crypto driver.
+;
+; Refer to default.ini for the full list of available PMD features.
+;
+[Features]
+Symmetric crypto       = Y
+HW Accelerated         = Y
+In Place SGL           = Y
+OOP SGL In SGL Out     = Y
+OOP SGL In LB  Out     = Y
+OOP LB  In SGL Out     = Y
+OOP LB  In LB  Out     = Y
+Cipher multiple data units = Y
+Cipher wrapped key     = Y
+
+;
+; Supported crypto algorithms of a mlx5 crypto driver.
+;
+[Cipher]
+AES XTS (128)  = Y
+AES XTS (256)  = Y
+
+;
+; Supported authentication algorithms of a mlx5 crypto driver.
+;
+[Auth]
+
+;
+; Supported AEAD algorithms of a mlx5 crypto driver.
+;
+[AEAD]
+
+;
+; Supported Asymmetric algorithms of a mlx5 crypto driver.
+;
+[Asymmetric]
diff --git a/doc/guides/cryptodevs/index.rst b/doc/guides/cryptodevs/index.rst
index 279f56a002..747409c441 100644
--- a/doc/guides/cryptodevs/index.rst
+++ b/doc/guides/cryptodevs/index.rst
@@ -22,6 +22,7 @@ Crypto Device Drivers
     octeontx
     octeontx2
     openssl
+    mlx5
     mvsam
     nitrox
     null
diff --git a/doc/guides/cryptodevs/mlx5.rst b/doc/guides/cryptodevs/mlx5.rst
new file mode 100644
index 0000000000..4ccec78be8
--- /dev/null
+++ b/doc/guides/cryptodevs/mlx5.rst
@@ -0,0 +1,152 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+   Copyright 2021 Mellanox Technologies, Ltd
+
+.. include:: <isonum.txt>
+
+MLX5 Crypto Driver
+==================
+
+The MLX5 crypto driver library
+(**librte_crypto_mlx5**) provides support for **Mellanox ConnectX-6**
+family adapters.
+
+Overview
+--------
+
+The device can provide disk encryption services, allowing data encryption
+and decryption towards a disk. Having all encryption/decryption
+operations done in a single device can reduce cost and overheads of the related
+FIPS certification, as ConnectX-6 is FIPS 140-2 level-2 ready.
+The encryption cipher is AES-XTS of 256/512 key size.
+
+MKEY is a memory region object in the hardware, that holds address translation information and
+attributes per memory area. Its ID must be tied to addresses provided to the hardware.
+The encryption operations are performed with MKEY read\write transactions, when
+the MKEY is configured to perform crypto operations.
+
+The encryption does not require text to be aligned to the AES block size (128b).
+
+In order to move the device to crypto operational mode, credential and KEK
+(Key Encrypting Key) should be set as the first step.
+The credential will be used by the software in order to perform crypto login, and the KEK is
+the AES Key Wrap Algorithm (rfc3394) key that will be used for sensitive data
+wrapping.
+The credential and the AES-XTS keys should be provided to the hardware, as ciphertext
+encrypted by the KEK.
+
+A keytag (64 bits) should be appended to the AES-XTS keys (before wrapping),
+and will be validated when the hardware attempts to access it.
+
+For security reasons and to increase robustness, this driver only deals with virtual
+memory addresses. The way resources allocations are handled by the kernel,
+combined with hardware specifications that allow handling virtual memory
+addresses directly, ensure that DPDK applications cannot access random
+physical memory (or memory that does not belong to the current process).
+
+The PMD uses libibverbs and libmlx5 to access the device firmware or to
+access the hardware components directly.
+There are different levels of objects and bypassing abilities.
+To get the best performances:
+
+- Verbs is a complete high-level generic API.
+- Direct Verbs is a device-specific API.
+- DevX allows to access firmware objects.
+
+Enabling librte_crypto_mlx5 causes DPDK applications to be linked against
+libibverbs.
+
+Mellanox mlx5 PCI device can be probed by a number of different PCI devices, such as
+net / vDPA / RegEx. To select the crypto PMD, ``class=crypto``
+should be specified as a device parameter. The crypto device can be probed and
+used with other Mellanox classes by adding more options in the class.
+For example: ``class=net:crypto`` will probe both the net PMD and the crypto
+PMD.
+
+When crypto engines are defined to work in wrapped import method, they come out
+of the factory in Commissioning mode, and thus, cannot be used for crypto operations
+yet. A dedicated tool is used for changing the mode from Commissioning to
+Operational, while setting the first import_KEK and credential in plaintext.
+The mlxreg dedicated tool should be used as follows:
+
+- Set CRYPTO_OPERATIONAL register to set the device in crypto operational mode.
+
+  The input to this tool is:
+    The first credential in plaintext, 40B.
+    The first import_KEK in plaintext: kek size 0 for 16B or 1 for 32B, kek data.
+
+  Example:
+  mlxreg -d /dev/mst/mt4123_pciconf0 --reg_name CRYPTO_OPERATIONAL --get
+
+  The "wrapped_crypto_operational" value will be "0x00000000".
+  The command to set the register should be executed only once, and all the
+  values mentioned above should be specified in the same command.
+
+  Example:
+  mlxreg -d /dev/mst/mt4123_pciconf0 --reg_name CRYPTO_OPERATIONAL
+  --set "credential[0]=0x10000000, credential[1]=0x10000000, kek[0]=0x00000000"
+
+  All values not specified will remain 0.
+  "wrapped_crypto_going_to_commissioning" and  "wrapped_crypto_operational"
+  should not be specified.
+
+  All the device ports should set it in order to move to operational mode.
+
+- Query CRYPTO_OPERATIONAL register to make sure the device is in Operational
+  mode.
+
+  Example:
+  mlxreg -d /dev/mst/mt4123_pciconf0 --reg_name CRYPTO_OPERATIONAL --get
+  The "wrapped_crypto_operational" value will be "0x00000001" if the mode was
+  successfully changed to operational mode.
+
+
+Driver options
+--------------
+
+- ``class`` parameter [string]
+
+  Select the class of the driver that should probe the device.
+  `crypto` for the mlx5 crypto driver.
+
+- ``wcs_file`` parameter [string] - mandatory
+
+  File path including only the wrapped credential in string format of hexadecimal
+  numbers, represent 48 bytes (8 bytes IV added by the AES key wrap algorithm).
+
+- ``import_kek_id`` parameter [int]
+
+  The identifier of the KEK, default value is 0 represents the operational
+  register import_kek..
+
+- ``credential_id`` parameter [int]
+
+  The identifier of the credential, default value is 0 represents the operational
+  register credential.
+
+- ``max_segs_num`` parameter [int]
+
+  Maximum number of mbuf chain segments(src or dest), default value is 8.
+
+- ``keytag`` parameter [int]
+
+  The plaintext of the keytag appanded to the AES-XTS keys, default value is 0.
+
+
+Limitations
+-----------
+
+- AES-XTS keys provided in Xform must include keytag and should be wrappend.
+- The supported data-unit lengths are: 512B, 1KB, 1MB. In case the `dataunit_len`
+  is not provided in the cipher Xform, the OP length is limitted to the above values.
+
+
+Supported NICs
+--------------
+
+* Mellanox\ |reg| ConnectX\ |reg|-6 200G MCX654106A-HCAT (2x200G)
+
+Prerequisites
+-------------
+
+- Mellanox OFED version: **5.3**
+  see :doc:`../../nics/mlx5` guide for more Mellanox OFED details.
diff --git a/doc/guides/rel_notes/release_21_05.rst b/doc/guides/rel_notes/release_21_05.rst
index 0970a2331a..9030fd8b98 100644
--- a/doc/guides/rel_notes/release_21_05.rst
+++ b/doc/guides/rel_notes/release_21_05.rst
@@ -275,6 +275,11 @@ New Features
   * Added support for crypto adapter forward mode in octeontx2 event and crypto
     device driver.
 
+* **Added support for Nvidia crypto device driver.**
+
+  * Added mlx5 crypto driver to support AES-XTS cipher operations.
+    the first device to support it is ConnectX-6.
+
 
 Removed Items
 -------------
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 896ca60866..f318ff4682 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -22,6 +22,14 @@
 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
 #define MLX5_CRYPTO_MAX_QPS 1024
 #define MLX5_CRYPTO_MAX_SEGS 56
+#define MLX5_CRYPTO_FEATURE_FLAGS \
+	(RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | \
+	RTE_CRYPTODEV_FF_IN_PLACE_SGL | RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT | \
+	RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | \
+	RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT | \
+	RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | \
+	RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY | \
+	RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS)
 
 TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
 				TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
@@ -31,8 +39,32 @@ int mlx5_crypto_logtype;
 
 uint8_t mlx5_crypto_driver_id;
 
-const struct rte_cryptodev_capabilities
-		mlx5_crypto_caps[RTE_CRYPTO_OP_TYPE_UNDEFINED];
+const struct rte_cryptodev_capabilities mlx5_crypto_caps[] = {
+	{		/* AES XTS */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+			{.cipher = {
+				.algo = RTE_CRYPTO_CIPHER_AES_XTS,
+				.block_size = 16,
+				.key_size = {
+					.min = 32,
+					.max = 64,
+					.increment = 32
+				},
+				.iv_size = {
+					.min = 16,
+					.max = 16,
+					.increment = 0
+				},
+				.dataunit_set =
+				RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_512_BYTES |
+				RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_4096_BYTES,
+			}, }
+		}, }
+	},
+};
+
 
 static const char mlx5_crypto_drv_name[] = RTE_STR(MLX5_CRYPTO_DRIVER_NAME);
 
@@ -67,7 +99,7 @@ mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev,
 	RTE_SET_USED(dev);
 	if (dev_info != NULL) {
 		dev_info->driver_id = mlx5_crypto_driver_id;
-		dev_info->feature_flags = 0;
+		dev_info->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
 		dev_info->capabilities = mlx5_crypto_caps;
 		dev_info->max_nb_queue_pairs = MLX5_CRYPTO_MAX_QPS;
 		dev_info->min_mbuf_headroom_req = 0;
@@ -954,6 +986,7 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	crypto_dev->dev_ops = &mlx5_crypto_ops;
 	crypto_dev->dequeue_burst = mlx5_crypto_dequeue_burst;
 	crypto_dev->enqueue_burst = mlx5_crypto_enqueue_burst;
+	crypto_dev->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
 	crypto_dev->driver_id = mlx5_crypto_driver_id;
 	priv = crypto_dev->data->dev_private;
 	priv->ctx = ctx;
-- 
2.25.1

Re: [dpdk-dev] [PATCH v3 00/15] mlx5 common part for crypto driver

[Thomas Monjalon]

04/05/2021 19:54, Matan Azrad:
> Dekel Peled (12):
>   common/mlx5: remove redundant spaces in header file
>   common/mlx5: update GENEVE TLV OPT obj name
>   common/mlx5: optimize read of general obj type caps
>   common/mlx5: add HCA cap for AES-XTS crypto
>   common/mlx5: support general object DEK create op
>   common/mlx5: adjust DevX MKEY fields for crypto
>   common/mlx5: support general obj IMPORT KEK create
>   common/mlx5: support general obj CRYPTO LOGIN create
>   common/mlx5: add crypto BSF struct and defines
>   common/mlx5: support general obj CREDENTIAL create
>   common/mlx5: add crypto register structs and defs
>   common/mlx5: support register write access
> 
> Shiri Kuzin (2):
>   common/mlx5: share hash list tool
>   common/mlx5: share get ib device match function
> 
> Suanming Mou (1):
>   common/mlx5: add UMR and RDMA write WQE defines

Applied, thanks.

Re: [dpdk-dev] [EXT] [PATCH v3 01/15] drivers: introduce mlx5 crypto PMD

[Akhil Goyal]

> diff --git a/drivers/crypto/mlx5/meson.build
> b/drivers/crypto/mlx5/meson.build
> new file mode 100644
> index 0000000000..fd00283665
> --- /dev/null
> +++ b/drivers/crypto/mlx5/meson.build
> @@ -0,0 +1,26 @@
> +# SPDX-License-Identifier: BSD-3-Clause
> +# Copyright 2021 Mellanox Technologies, Ltd
> +
> +if not is_linux
> +	build = false
> +	reason = 'only supported on Linux'
> +	subdir_done()
> +endif
> +
> +fmt_name = 'mlx5_crypto'
> +deps += ['common_mlx5', 'eal', 'cryptodev']

I don't see a reason for eal dependency. Do you really need it?

> +sources = files(
> +	'mlx5_crypto.c',
> +)
> +cflags_options = [
> +	'-std=c11',
> +	'-Wno-strict-prototypes',
> +	'-D_BSD_SOURCE',
> +	'-D_DEFAULT_SOURCE',
> +	'-D_XOPEN_SOURCE=600',
> +]

Any specific reason to use '-Wno-strict-prototypes'
Compilation works without this as well.

> +foreach option:cflags_options
> +	if cc.has_argument(option)
> +		cflags += option
> +	endif
> +endforeach

Re: [dpdk-dev] [EXT] [PATCH v3 15/15] crypto/mlx5: set feature flags and capabilities

[Akhil Goyal]

> +
> +Overview
> +--------
> +
> +The device can provide disk encryption services, allowing data encryption
> +and decryption towards a disk. Having all encryption/decryption
> +operations done in a single device can reduce cost and overheads of the
> related
> +FIPS certification, as ConnectX-6 is FIPS 140-2 level-2 ready.
> +The encryption cipher is AES-XTS of 256/512 key size.

256/512 bit key size

> +
> +MKEY is a memory region object in the hardware, that holds address
> translation information and
> +attributes per memory area. Its ID must be tied to addresses provided to
> the hardware.
> +The encryption operations are performed with MKEY read\write

read/write

> transactions, when
> +the MKEY is configured to perform crypto operations.
> +
> +The encryption does not require text to be aligned to the AES block size
> (128b).
> +
> +In order to move the device to crypto operational mode, credential and KEK
> +(Key Encrypting Key) should be set as the first step.
> +The credential will be used by the software in order to perform crypto login,
> and the KEK is
> +the AES Key Wrap Algorithm (rfc3394) key that will be used for sensitive
> data
> +wrapping.
> +The credential and the AES-XTS keys should be provided to the hardware, as
> ciphertext
> +encrypted by the KEK.
> +
> +A keytag (64 bits) should be appended to the AES-XTS keys (before
> wrapping),
> +and will be validated when the hardware attempts to access it.
> +
> +For security reasons and to increase robustness, this driver only deals with
> virtual
> +memory addresses. The way resources allocations are handled by the
> kernel,
> +combined with hardware specifications that allow handling virtual memory
> +addresses directly, ensure that DPDK applications cannot access random
> +physical memory (or memory that does not belong to the current process).
> +
> +The PMD uses libibverbs and libmlx5 to access the device firmware or to
> +access the hardware components directly.
> +There are different levels of objects and bypassing abilities.
> +To get the best performances:
> +
> +- Verbs is a complete high-level generic API.
> +- Direct Verbs is a device-specific API.
> +- DevX allows to access firmware objects.
> +
> +Enabling librte_crypto_mlx5 causes DPDK applications to be linked against
> +libibverbs.
> +
> +Mellanox mlx5 PCI device can be probed by a number of different PCI
> devices, such as
> +net / vDPA / RegEx. To select the crypto PMD, ``class=crypto``
> +should be specified as a device parameter. The crypto device can be probed
> and
> +used with other Mellanox classes by adding more options in the class.
> +For example: ``class=net:crypto`` will probe both the net PMD and the
> crypto
> +PMD.
> +
> +When crypto engines are defined to work in wrapped import method, they
> come out
> +of the factory in Commissioning mode, and thus, cannot be used for crypto
> operations
> +yet. A dedicated tool is used for changing the mode from Commissioning to
> +Operational, while setting the first import_KEK and credential in plaintext.
> +The mlxreg dedicated tool should be used as follows:
> +
> +- Set CRYPTO_OPERATIONAL register to set the device in crypto operational
> mode.
> +
> +  The input to this tool is:
> +    The first credential in plaintext, 40B.
> +    The first import_KEK in plaintext: kek size 0 for 16B or 1 for 32B, kek data.
> +
> +  Example:
> +  mlxreg -d /dev/mst/mt4123_pciconf0 --reg_name CRYPTO_OPERATIONAL -
> -get
> +
> +  The "wrapped_crypto_operational" value will be "0x00000000".
> +  The command to set the register should be executed only once, and all the
> +  values mentioned above should be specified in the same command.
> +
> +  Example:
> +  mlxreg -d /dev/mst/mt4123_pciconf0 --reg_name CRYPTO_OPERATIONAL
> +  --set "credential[0]=0x10000000, credential[1]=0x10000000,
> kek[0]=0x00000000"
> +
> +  All values not specified will remain 0.
> +  "wrapped_crypto_going_to_commissioning" and
> "wrapped_crypto_operational"
> +  should not be specified.
> +
> +  All the device ports should set it in order to move to operational mode.
> +
> +- Query CRYPTO_OPERATIONAL register to make sure the device is in
> Operational
> +  mode.
> +
> +  Example:
> +  mlxreg -d /dev/mst/mt4123_pciconf0 --reg_name CRYPTO_OPERATIONAL -
> -get
> +  The "wrapped_crypto_operational" value will be "0x00000001" if the
> mode was
> +  successfully changed to operational mode.
> +
> +
> +Driver options
> +--------------
> +
> +- ``class`` parameter [string]
> +
> +  Select the class of the driver that should probe the device.
> +  `crypto` for the mlx5 crypto driver.
> +
> +- ``wcs_file`` parameter [string] - mandatory
> +
> +  File path including only the wrapped credential in string format of
> hexadecimal
> +  numbers, represent 48 bytes (8 bytes IV added by the AES key wrap
> algorithm).
> +
> +- ``import_kek_id`` parameter [int]
> +
> +  The identifier of the KEK, default value is 0 represents the operational
> +  register import_kek..
> +
> +- ``credential_id`` parameter [int]
> +
> +  The identifier of the credential, default value is 0 represents the
> operational
> +  register credential.
> +
> +- ``max_segs_num`` parameter [int]
> +
> +  Maximum number of mbuf chain segments(src or dest), default value is 8.
> +
> +- ``keytag`` parameter [int]
> +
> +  The plaintext of the keytag appanded to the AES-XTS keys, default value is
> 0.
> +
> +
> +Limitations
> +-----------
> +
> +- AES-XTS keys provided in Xform must include keytag and should be
> wrappend.
> +- The supported data-unit lengths are: 512B, 1KB, 1MB. In case the
> `dataunit_len`
> +  is not provided in the cipher Xform, the OP length is limitted to the above

Xform should be xform

Spell check  - limited

> values.
> +
> +
> +Supported NICs
> +--------------
> +
> +* Mellanox\ |reg| ConnectX\ |reg|-6 200G MCX654106A-HCAT (2x200G)
> +
> +Prerequisites
> +-------------
> +
> +- Mellanox OFED version: **5.3**
> +  see :doc:`../../nics/mlx5` guide for more Mellanox OFED details.

Since the driver is by default compiled off due to the dependency on external
Libraries, I would recommend to add few lines here as well for compilation.
Like to compile rdma-core and set PKG_CONFIG_LIBDIR.

And I do not see any updates to the test application for testing this driver.
Is this driver really tested or is it work in progress? If it is work in progress,
We should defer this PMD to next release.

> diff --git a/doc/guides/rel_notes/release_21_05.rst
> b/doc/guides/rel_notes/release_21_05.rst
> index 0970a2331a..9030fd8b98 100644
> --- a/doc/guides/rel_notes/release_21_05.rst
> +++ b/doc/guides/rel_notes/release_21_05.rst
> @@ -275,6 +275,11 @@ New Features
>    * Added support for crypto adapter forward mode in octeontx2 event and
> crypto
>      device driver.
> 
> +* **Added support for Nvidia crypto device driver.**
> +
> +  * Added mlx5 crypto driver to support AES-XTS cipher operations.
> +    the first device to support it is ConnectX-6.

'The' 

> +
> 
>  Removed Items
>  -------------
> diff --git a/drivers/crypto/mlx5/mlx5_crypto.c
> b/drivers/crypto/mlx5/mlx5_crypto.c
> index 896ca60866..f318ff4682 100644
> --- a/drivers/crypto/mlx5/mlx5_crypto.c
> +++ b/drivers/crypto/mlx5/mlx5_crypto.c
> @@ -22,6 +22,14 @@
>  #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
>  #define MLX5_CRYPTO_MAX_QPS 1024
>  #define MLX5_CRYPTO_MAX_SEGS 56
> +#define MLX5_CRYPTO_FEATURE_FLAGS \
> +	(RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
> RTE_CRYPTODEV_FF_HW_ACCELERATED | \
> +	RTE_CRYPTODEV_FF_IN_PLACE_SGL |
> RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT | \
> +	RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | \
> +	RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT | \
> +	RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | \
> +	RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY | \
> +	RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS)
> 
>  TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
> 
> 	TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
> @@ -31,8 +39,32 @@ int mlx5_crypto_logtype;
> 
>  uint8_t mlx5_crypto_driver_id;
> 
> -const struct rte_cryptodev_capabilities
> -		mlx5_crypto_caps[RTE_CRYPTO_OP_TYPE_UNDEFINED];
> +const struct rte_cryptodev_capabilities mlx5_crypto_caps[] = {
> +	{		/* AES XTS */
> +		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
> +		{.sym = {
> +			.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
> +			{.cipher = {
> +				.algo = RTE_CRYPTO_CIPHER_AES_XTS,
> +				.block_size = 16,
> +				.key_size = {
> +					.min = 32,
> +					.max = 64,
> +					.increment = 32
> +				},
> +				.iv_size = {
> +					.min = 16,
> +					.max = 16,
> +					.increment = 0
> +				},
> +				.dataunit_set =
> +
> 	RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_512_BYTES |
> +
> 	RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_4096_BYTES,
> +			}, }
> +		}, }
> +	},
> +};
> +
> 
>  static const char mlx5_crypto_drv_name[] =
> RTE_STR(MLX5_CRYPTO_DRIVER_NAME);
> 
> @@ -67,7 +99,7 @@ mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev,
>  	RTE_SET_USED(dev);
>  	if (dev_info != NULL) {
>  		dev_info->driver_id = mlx5_crypto_driver_id;
> -		dev_info->feature_flags = 0;
> +		dev_info->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
>  		dev_info->capabilities = mlx5_crypto_caps;
>  		dev_info->max_nb_queue_pairs = MLX5_CRYPTO_MAX_QPS;
>  		dev_info->min_mbuf_headroom_req = 0;
> @@ -954,6 +986,7 @@ mlx5_crypto_pci_probe(struct rte_pci_driver
> *pci_drv,
>  	crypto_dev->dev_ops = &mlx5_crypto_ops;
>  	crypto_dev->dequeue_burst = mlx5_crypto_dequeue_burst;
>  	crypto_dev->enqueue_burst = mlx5_crypto_enqueue_burst;
> +	crypto_dev->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
>  	crypto_dev->driver_id = mlx5_crypto_driver_id;
>  	priv = crypto_dev->data->dev_private;
>  	priv->ctx = ctx;
> --
> 2.25.1

Re: [dpdk-dev] [EXT] [PATCH v3 13/15] crypto/mlx5: add enqueue and dequeue operations

[Akhil Goyal]

> @@ -685,9 +919,8 @@ mlx5_crypto_pci_probe(struct rte_pci_driver
> *pci_drv,
>  	DRV_LOG(INFO,
>  		"Crypto device %s was created successfully.", ibv->name);
>  	crypto_dev->dev_ops = &mlx5_crypto_ops;
> -	crypto_dev->dequeue_burst = NULL;
> -	crypto_dev->enqueue_burst = NULL;
> -	crypto_dev->feature_flags =
> RTE_CRYPTODEV_FF_HW_ACCELERATED;
It should be set once in the patchset.
I see this getting set in 1/15 and removed here and again set in 15/15.
And whenever you are setting it, it should be added in the .ini file as well.

> +	crypto_dev->dequeue_burst = mlx5_crypto_dequeue_burst;
> +	crypto_dev->enqueue_burst = mlx5_crypto_enqueue_burst;
>  	crypto_dev->driver_id = mlx5_crypto_driver_id;
>  	priv = crypto_dev->data->dev_private;
>  	priv->ctx = ctx;
> diff --git a/drivers/crypto/mlx5/mlx5_crypto.h
> b/drivers/crypto/mlx5/mlx5_crypto.h
> index 52fcf5217f..ac4ad1834f 100644
> --- a/drivers/crypto/mlx5/mlx5_crypto.h
> +++ b/drivers/crypto/mlx5/mlx5_crypto.h
> @@ -37,6 +37,9 @@ struct mlx5_crypto_priv {
>  	uint16_t rdmw_wqe_size;
>  	uint16_t wqe_stride;
>  	uint16_t max_rdmaw_klm_n;
> +#ifndef RTE_ARCH_64
> +	rte_spinlock_t uar32_sl;
> +#endif /* RTE_ARCH_64 */
>  };
> 
>  struct mlx5_crypto_qp {
> --
> 2.25.1

Re: [dpdk-dev] [EXT] [PATCH v3 09/15] crypto/mlx5: adjust to the multiple data unit API

[Akhil Goyal]

> From: Shiri Kuzin <shirik@nvidia.com>
> 
> In AES-XTS the data to be encrypted\decrypted does not have to be
> in multiples of 16B size, the unit of data is called data-unit.
> 
> As a result of patch [1] a new field is added to the cipher capability,
> called dataunit_set, where the devices can report the range of
> supported data-unit sizes.
> 
> The new field enables saving the data-unit size in the session
> structure to the block size pointer variable in order to support
> several data-unit sizes.
> 
> [1] https://www.mail-archive.com/dev@dpdk.org/msg205337.html 
> 
Since [1] patch is already merged, no need to refer it in the patch description.
Title can be rephrased as
Crypto/mlx5: support multiple data units

Also set feature flag in the code and the documentation in this patch.
I see that you are setting all of them in a single patch in the end.
This is not correct. It should be added where the feature is supported.
Please fix this for all the feature flags.


> Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
> ---
>  drivers/crypto/mlx5/mlx5_crypto.c | 23 +++++++++++++++++++++++
>  1 file changed, 23 insertions(+)
> 
> diff --git a/drivers/crypto/mlx5/mlx5_crypto.c
> b/drivers/crypto/mlx5/mlx5_crypto.c
> index 18b1a6be88..8cc29ced21 100644
> --- a/drivers/crypto/mlx5/mlx5_crypto.c
> +++ b/drivers/crypto/mlx5/mlx5_crypto.c
> @@ -48,6 +48,11 @@ struct mlx5_crypto_session {
>  	 * bsf_size, bsf_p_type, encryption_order and encryption standard,
>  	 * saved in big endian format.
>  	 */
> +	uint32_t bsp_res;
> +	/*
> +	 * crypto_block_size_pointer and reserved 24 bits saved in big endian
> +	 * format.
> +	 */
>  	uint32_t iv_offset:16;
>  	/* Starting point for Initialisation Vector. */
>  	struct mlx5_crypto_dek *dek; /* Pointer to dek struct. */
> @@ -171,6 +176,24 @@ mlx5_crypto_sym_session_configure(struct
> rte_cryptodev *dev,
>  			 MLX5_BSF_P_TYPE_CRYPTO <<
> MLX5_BSF_P_TYPE_OFFSET |
>  			 encryption_order <<
> MLX5_ENCRYPTION_ORDER_OFFSET |
>  			 MLX5_ENCRYPTION_STANDARD_AES_XTS);
> +	switch (xform->cipher.dataunit_len) {
> +	case 0:
> +		sess_private_data->bsp_res = 0;
> +		break;
> +	case 512:
> +		sess_private_data->bsp_res = rte_cpu_to_be_32
> +
> ((uint32_t)MLX5_BLOCK_SIZE_512B <<
> +					     MLX5_BLOCK_SIZE_OFFSET);
> +		break;
> +	case 4096:
> +		sess_private_data->bsp_res = rte_cpu_to_be_32
> +
> ((uint32_t)MLX5_BLOCK_SIZE_4096B <<
> +					     MLX5_BLOCK_SIZE_OFFSET);
> +		break;
> +	default:
> +		DRV_LOG(ERR, "Cipher data unit length is not supported.");
> +		return -ENOTSUP;
> +	}
>  	sess_private_data->iv_offset = cipher->iv.offset;
>  	sess_private_data->dek_id =
>  			rte_cpu_to_be_32(sess_private_data->dek->obj->id
> &
> --
> 2.25.1

Re: [dpdk-dev] [EXT] [PATCH v3 10/15] crypto/mlx5: add keytag device argument

[Akhil Goyal]

> From: Suanming Mou <suanmingm@nvidia.com>
> 
> A keytag is a piece of data encrypted together with a DEK.
> 
> When a DEK is referenced by an MKEY.bsf through its index, the keytag is
> also supplied in the BSF as plaintext. The HW will decrypt the DEK (and
> the attached keytag) and will fail the operation if the keytags don't
> match.
> 
> This commit adds the configuration of the keytag with devargs.
> 
> Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
> Signed-off-by: Matan Azrad <matan@nvidia.com>
> ---
Documentation for devargs should be part of this patch.
Please split the last patch accordingly.

Title can be shortened to
Crypto/mlx5: add keytag devarg

Fix other patches of devargs accordingly.

Re: [dpdk-dev] [EXT] [PATCH v3 03/15] crypto/mlx5: support session operations

[Akhil Goyal]

> +
> +static void
> +mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev,
> +			      struct rte_cryptodev_sym_session *sess)
> +{
> +	struct mlx5_crypto_priv *priv = dev->data->dev_private;
> +	struct mlx5_crypto_session *sess_private_data =
> +			get_sym_session_private_data(sess, dev->driver_id);
> +
> +	if (unlikely(sess_private_data == NULL)) {
> +		DRV_LOG(ERR, "Failed to get session %p private data.",
> +				sess_private_data);
> +		return;
> +	}
> +	mlx5_crypto_dek_destroy(priv, sess_private_data->dek);
> +	DRV_LOG(DEBUG, "Session %p was cleared.", sess_private_data);
> +}

Memory leakage, mempool is not freed.
IMO, this driver is not properly tested with the unit test app.
Please add a note in the documentation that it is tested with autotest.

Re: [dpdk-dev] [EXT] [PATCH v3 01/15] drivers: introduce mlx5 crypto PMD

[Matan Azrad]

Hi Akhil

Thanks for the review, I know it was too tied for you to review, so, I really appreciate..

From: Akhil Goyal
> > diff --git a/drivers/crypto/mlx5/meson.build
> > b/drivers/crypto/mlx5/meson.build new file mode 100644 index
> > 0000000000..fd00283665
> > --- /dev/null
> > +++ b/drivers/crypto/mlx5/meson.build
> > @@ -0,0 +1,26 @@
> > +# SPDX-License-Identifier: BSD-3-Clause # Copyright 2021 Mellanox
> > +Technologies, Ltd
> > +
> > +if not is_linux
> > +     build = false
> > +     reason = 'only supported on Linux'
> > +     subdir_done()
> > +endif
> > +
> > +fmt_name = 'mlx5_crypto'
> > +deps += ['common_mlx5', 'eal', 'cryptodev']
> 
> I don't see a reason for eal dependency. Do you really need it?

See RTE_LOG_REGISTER.

> > +sources = files(
> > +     'mlx5_crypto.c',
> > +)
> > +cflags_options = [
> > +     '-std=c11',
> > +     '-Wno-strict-prototypes',
> > +     '-D_BSD_SOURCE',
> > +     '-D_DEFAULT_SOURCE',
> > +     '-D_XOPEN_SOURCE=600',
> > +]
> 
> Any specific reason to use '-Wno-strict-prototypes'
> Compilation works without this as well.

We use it in Mellanox driver usually.
This is internal mlx5 code convention.


 
> > +foreach option:cflags_options
> > +     if cc.has_argument(option)
> > +             cflags += option
> > +     endif
> > +endforeach

Re: [dpdk-dev] [EXT] [PATCH v3 03/15] crypto/mlx5: support session operations

[Matan Azrad]

From: Akhil Goyal
> > +static void
> > +mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev,
> > +                           struct rte_cryptodev_sym_session *sess) {
> > +     struct mlx5_crypto_priv *priv = dev->data->dev_private;
> > +     struct mlx5_crypto_session *sess_private_data =
> > +                     get_sym_session_private_data(sess,
> > +dev->driver_id);
> > +
> > +     if (unlikely(sess_private_data == NULL)) {
> > +             DRV_LOG(ERR, "Failed to get session %p private data.",
> > +                             sess_private_data);
> > +             return;
> > +     }
> > +     mlx5_crypto_dek_destroy(priv, sess_private_data->dek);
> > +     DRV_LOG(DEBUG, "Session %p was cleared.", sess_private_data); }
> 
> Memory leakage, mempool is not freed.

Yes, good catch, this part was missed.

> IMO, this driver is not properly tested with the unit test app.

The only app we tested until now is l2fwd_crypto and it works fine!
We can add it to doc.

> Please add a note in the documentation that it is tested with autotest.


The next app we want to test with, is test-crypto-perf.
 

Re: [dpdk-dev] [EXT] [PATCH v3 09/15] crypto/mlx5: adjust to the multiple data unit API

[Matan Azrad]

From: Akhil Goyal
> > From: Shiri Kuzin <shirik@nvidia.com>
> >
> > In AES-XTS the data to be encrypted\decrypted does not have to be in
> > multiples of 16B size, the unit of data is called data-unit.
> >
> > As a result of patch [1] a new field is added to the cipher
> > capability, called dataunit_set, where the devices can report the
> > range of supported data-unit sizes.
> >
> > The new field enables saving the data-unit size in the session
> > structure to the block size pointer variable in order to support
> > several data-unit sizes.
> >
> > [1] https://www.mail-archive.com/dev@dpdk.org/msg205337.html
> >
> Since [1] patch is already merged, no need to refer it in the patch description.
> Title can be rephrased as
> Crypto/mlx5: support multiple data units

Ok.

 
> Also set feature flag in the code and the documentation in this patch.
> I see that you are setting all of them in a single patch in the end.
> This is not correct. It should be added where the feature is supported.
> Please fix this for all the feature flags.

No, in this stage no feature is really supported, the actual time it will be supported is after the datapath patches and capabilities set.

> 
> > Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
> > ---
> >  drivers/crypto/mlx5/mlx5_crypto.c | 23 +++++++++++++++++++++++
> >  1 file changed, 23 insertions(+)
> >
> > diff --git a/drivers/crypto/mlx5/mlx5_crypto.c
> > b/drivers/crypto/mlx5/mlx5_crypto.c
> > index 18b1a6be88..8cc29ced21 100644
> > --- a/drivers/crypto/mlx5/mlx5_crypto.c
> > +++ b/drivers/crypto/mlx5/mlx5_crypto.c
> > @@ -48,6 +48,11 @@ struct mlx5_crypto_session {
> >        * bsf_size, bsf_p_type, encryption_order and encryption standard,
> >        * saved in big endian format.
> >        */
> > +     uint32_t bsp_res;
> > +     /*
> > +      * crypto_block_size_pointer and reserved 24 bits saved in big endian
> > +      * format.
> > +      */
> >       uint32_t iv_offset:16;
> >       /* Starting point for Initialisation Vector. */
> >       struct mlx5_crypto_dek *dek; /* Pointer to dek struct. */ @@
> > -171,6 +176,24 @@ mlx5_crypto_sym_session_configure(struct
> > rte_cryptodev *dev,
> >                        MLX5_BSF_P_TYPE_CRYPTO <<
> > MLX5_BSF_P_TYPE_OFFSET |
> >                        encryption_order <<
> > MLX5_ENCRYPTION_ORDER_OFFSET |
> >                        MLX5_ENCRYPTION_STANDARD_AES_XTS);
> > +     switch (xform->cipher.dataunit_len) {
> > +     case 0:
> > +             sess_private_data->bsp_res = 0;
> > +             break;
> > +     case 512:
> > +             sess_private_data->bsp_res = rte_cpu_to_be_32
> > +
> > ((uint32_t)MLX5_BLOCK_SIZE_512B <<
> > +                                          MLX5_BLOCK_SIZE_OFFSET);
> > +             break;
> > +     case 4096:
> > +             sess_private_data->bsp_res = rte_cpu_to_be_32
> > +
> > ((uint32_t)MLX5_BLOCK_SIZE_4096B <<
> > +                                          MLX5_BLOCK_SIZE_OFFSET);
> > +             break;
> > +     default:
> > +             DRV_LOG(ERR, "Cipher data unit length is not supported.");
> > +             return -ENOTSUP;
> > +     }
> >       sess_private_data->iv_offset = cipher->iv.offset;
> >       sess_private_data->dek_id =
> >                       rte_cpu_to_be_32(sess_private_data->dek->obj->id
> > &
> > --
> > 2.25.1

Re: [dpdk-dev] [EXT] [PATCH v3 10/15] crypto/mlx5: add keytag device argument

[Matan Azrad]

From: Akhil Goyal
> > From: Suanming Mou <suanmingm@nvidia.com>
> >
> > A keytag is a piece of data encrypted together with a DEK.
> >
> > When a DEK is referenced by an MKEY.bsf through its index, the keytag
> > is also supplied in the BSF as plaintext. The HW will decrypt the DEK
> > (and the attached keytag) and will fail the operation if the keytags
> > don't match.
> >
> > This commit adds the configuration of the keytag with devargs.
> >
> > Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
> > Signed-off-by: Matan Azrad <matan@nvidia.com>
> > ---
> Documentation for devargs should be part of this patch.
> Please split the last patch accordingly.
> 
> Title can be shortened to
> Crypto/mlx5: add keytag devarg
> 
> Fix other patches of devargs accordingly.

As I said before, no devargs is really active before adding datapath patches.
The option to add all the supported features \ documentations in the patch which actually adds the support is correct.

The last patch adds the capabilities and docs when all of them are really supported.

Ok for title.

Re: [dpdk-dev] [EXT] [PATCH v3 13/15] crypto/mlx5: add enqueue and dequeue operations

[Matan Azrad]

From: Akhil Goyal
> > @@ -685,9 +919,8 @@ mlx5_crypto_pci_probe(struct rte_pci_driver
> > *pci_drv,
> >       DRV_LOG(INFO,
> >               "Crypto device %s was created successfully.", ibv->name);
> >       crypto_dev->dev_ops = &mlx5_crypto_ops;
> > -     crypto_dev->dequeue_burst = NULL;
> > -     crypto_dev->enqueue_burst = NULL;
> > -     crypto_dev->feature_flags =
> > RTE_CRYPTODEV_FF_HW_ACCELERATED;
> It should be set once in the patchset.
> I see this getting set in 1/15 and removed here and again set in 15/15.
> And whenever you are setting it, it should be added in the .ini file as well.

Moved it to last patch too

> > +     crypto_dev->dequeue_burst = mlx5_crypto_dequeue_burst;
> > +     crypto_dev->enqueue_burst = mlx5_crypto_enqueue_burst;
> >       crypto_dev->driver_id = mlx5_crypto_driver_id;
> >       priv = crypto_dev->data->dev_private;
> >       priv->ctx = ctx;
> > diff --git a/drivers/crypto/mlx5/mlx5_crypto.h
> > b/drivers/crypto/mlx5/mlx5_crypto.h
> > index 52fcf5217f..ac4ad1834f 100644
> > --- a/drivers/crypto/mlx5/mlx5_crypto.h
> > +++ b/drivers/crypto/mlx5/mlx5_crypto.h
> > @@ -37,6 +37,9 @@ struct mlx5_crypto_priv {
> >       uint16_t rdmw_wqe_size;
> >       uint16_t wqe_stride;
> >       uint16_t max_rdmaw_klm_n;
> > +#ifndef RTE_ARCH_64
> > +     rte_spinlock_t uar32_sl;
> > +#endif /* RTE_ARCH_64 */
> >  };
> >
> >  struct mlx5_crypto_qp {
> > --
> > 2.25.1

Re: [dpdk-dev] [EXT] [PATCH v3 15/15] crypto/mlx5: set feature flags and capabilities

[Matan Azrad]

From: Akhil Goyal
> > +
> > +Overview
> > +--------
> > +
> > +The device can provide disk encryption services, allowing data
> > +encryption and decryption towards a disk. Having all
> > +encryption/decryption operations done in a single device can reduce
> > +cost and overheads of the
> > related
> > +FIPS certification, as ConnectX-6 is FIPS 140-2 level-2 ready.
> > +The encryption cipher is AES-XTS of 256/512 key size.
> 
> 256/512 bit key size
> 
> > +
> > +MKEY is a memory region object in the hardware, that holds address
> > translation information and
> > +attributes per memory area. Its ID must be tied to addresses provided
> > +to
> > the hardware.
> > +The encryption operations are performed with MKEY read\write
> 
> read/write
> 
> > transactions, when
> > +the MKEY is configured to perform crypto operations.
> > +
> > +The encryption does not require text to be aligned to the AES block
> > +size
> > (128b).
> > +
> > +In order to move the device to crypto operational mode, credential
> > +and KEK (Key Encrypting Key) should be set as the first step.
> > +The credential will be used by the software in order to perform
> > +crypto login,
> > and the KEK is
> > +the AES Key Wrap Algorithm (rfc3394) key that will be used for
> > +sensitive
> > data
> > +wrapping.
> > +The credential and the AES-XTS keys should be provided to the
> > +hardware, as
> > ciphertext
> > +encrypted by the KEK.
> > +
> > +A keytag (64 bits) should be appended to the AES-XTS keys (before
> > wrapping),
> > +and will be validated when the hardware attempts to access it.
> > +
> > +For security reasons and to increase robustness, this driver only
> > +deals with
> > virtual
> > +memory addresses. The way resources allocations are handled by the
> > kernel,
> > +combined with hardware specifications that allow handling virtual
> > +memory addresses directly, ensure that DPDK applications cannot
> > +access random physical memory (or memory that does not belong to the
> current process).
> > +
> > +The PMD uses libibverbs and libmlx5 to access the device firmware or
> > +to access the hardware components directly.
> > +There are different levels of objects and bypassing abilities.
> > +To get the best performances:
> > +
> > +- Verbs is a complete high-level generic API.
> > +- Direct Verbs is a device-specific API.
> > +- DevX allows to access firmware objects.
> > +
> > +Enabling librte_crypto_mlx5 causes DPDK applications to be linked
> > +against libibverbs.
> > +
> > +Mellanox mlx5 PCI device can be probed by a number of different PCI
> > devices, such as
> > +net / vDPA / RegEx. To select the crypto PMD, ``class=crypto`` should
> > +be specified as a device parameter. The crypto device can be probed
> > and
> > +used with other Mellanox classes by adding more options in the class.
> > +For example: ``class=net:crypto`` will probe both the net PMD and the
> > crypto
> > +PMD.
> > +
> > +When crypto engines are defined to work in wrapped import method,
> > +they
> > come out
> > +of the factory in Commissioning mode, and thus, cannot be used for
> > +crypto
> > operations
> > +yet. A dedicated tool is used for changing the mode from
> > +Commissioning to Operational, while setting the first import_KEK and
> credential in plaintext.
> > +The mlxreg dedicated tool should be used as follows:
> > +
> > +- Set CRYPTO_OPERATIONAL register to set the device in crypto
> > +operational
> > mode.
> > +
> > +  The input to this tool is:
> > +    The first credential in plaintext, 40B.
> > +    The first import_KEK in plaintext: kek size 0 for 16B or 1 for 32B, kek
> data.
> > +
> > +  Example:
> > +  mlxreg -d /dev/mst/mt4123_pciconf0 --reg_name
> CRYPTO_OPERATIONAL -
> > -get
> > +
> > +  The "wrapped_crypto_operational" value will be "0x00000000".
> > +  The command to set the register should be executed only once, and
> > + all the  values mentioned above should be specified in the same
> command.
> > +
> > +  Example:
> > +  mlxreg -d /dev/mst/mt4123_pciconf0 --reg_name
> CRYPTO_OPERATIONAL
> > + --set "credential[0]=0x10000000, credential[1]=0x10000000,
> > kek[0]=0x00000000"
> > +
> > +  All values not specified will remain 0.
> > +  "wrapped_crypto_going_to_commissioning" and
> > "wrapped_crypto_operational"
> > +  should not be specified.
> > +
> > +  All the device ports should set it in order to move to operational mode.
> > +
> > +- Query CRYPTO_OPERATIONAL register to make sure the device is in
> > Operational
> > +  mode.
> > +
> > +  Example:
> > +  mlxreg -d /dev/mst/mt4123_pciconf0 --reg_name
> CRYPTO_OPERATIONAL -
> > -get
> > +  The "wrapped_crypto_operational" value will be "0x00000001" if the
> > mode was
> > +  successfully changed to operational mode.
> > +
> > +
> > +Driver options
> > +--------------
> > +
> > +- ``class`` parameter [string]
> > +
> > +  Select the class of the driver that should probe the device.
> > +  `crypto` for the mlx5 crypto driver.
> > +
> > +- ``wcs_file`` parameter [string] - mandatory
> > +
> > +  File path including only the wrapped credential in string format of
> > hexadecimal
> > +  numbers, represent 48 bytes (8 bytes IV added by the AES key wrap
> > algorithm).
> > +
> > +- ``import_kek_id`` parameter [int]
> > +
> > +  The identifier of the KEK, default value is 0 represents the
> > + operational  register import_kek..
> > +
> > +- ``credential_id`` parameter [int]
> > +
> > +  The identifier of the credential, default value is 0 represents the
> > operational
> > +  register credential.
> > +
> > +- ``max_segs_num`` parameter [int]
> > +
> > +  Maximum number of mbuf chain segments(src or dest), default value is
> 8.
> > +
> > +- ``keytag`` parameter [int]
> > +
> > +  The plaintext of the keytag appanded to the AES-XTS keys, default
> > + value is
> > 0.
> > +
> > +
> > +Limitations
> > +-----------
> > +
> > +- AES-XTS keys provided in Xform must include keytag and should be
> > wrappend.
> > +- The supported data-unit lengths are: 512B, 1KB, 1MB. In case the
> > `dataunit_len`
> > +  is not provided in the cipher Xform, the OP length is limitted to
> > + the above
> 
> Xform should be xform
> 
> Spell check  - limited


Ok, for all the above comments. Will adjust in v4.
 
> > values.
> > +
> > +
> > +Supported NICs
> > +--------------
> > +
> > +* Mellanox\ |reg| ConnectX\ |reg|-6 200G MCX654106A-HCAT (2x200G)
> > +
> > +Prerequisites
> > +-------------
> > +
> > +- Mellanox OFED version: **5.3**
> > +  see :doc:`../../nics/mlx5` guide for more Mellanox OFED details.
> 
> Since the driver is by default compiled off due to the dependency on
> external Libraries, I would recommend to add few lines here as well for
> compilation.
> Like to compile rdma-core and set PKG_CONFIG_LIBDIR.

Why? all Mellanox drivers has the same external dependencies.
I added here link for the doc explains it well.

> And I do not see any updates to the test application for testing this driver.

You can see update to l2fwd_crypto, we tested with this example for the first stage.
Everything looks ok there.

> Is this driver really tested or is it work in progress? If it is work in progress, We
> should defer this PMD to next release.

We can continue test this driver for a long time.
The basics were tested in the above crypto example what proofs that driver works well.
If we will find issues in other tests, we will send fixes.

I'll send v4 with your comments adjustment soon.

> 
> > diff --git a/doc/guides/rel_notes/release_21_05.rst
> > b/doc/guides/rel_notes/release_21_05.rst
> > index 0970a2331a..9030fd8b98 100644
> > --- a/doc/guides/rel_notes/release_21_05.rst
> > +++ b/doc/guides/rel_notes/release_21_05.rst
> > @@ -275,6 +275,11 @@ New Features
> >    * Added support for crypto adapter forward mode in octeontx2 event
> > and crypto
> >      device driver.
> >
> > +* **Added support for Nvidia crypto device driver.**
> > +
> > +  * Added mlx5 crypto driver to support AES-XTS cipher operations.
> > +    the first device to support it is ConnectX-6.
> 
> 'The'
> 
> > +
> >
> >  Removed Items
> >  -------------
> > diff --git a/drivers/crypto/mlx5/mlx5_crypto.c
> > b/drivers/crypto/mlx5/mlx5_crypto.c
> > index 896ca60866..f318ff4682 100644
> > --- a/drivers/crypto/mlx5/mlx5_crypto.c
> > +++ b/drivers/crypto/mlx5/mlx5_crypto.c
> > @@ -22,6 +22,14 @@
> >  #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5  #define
> > MLX5_CRYPTO_MAX_QPS 1024  #define MLX5_CRYPTO_MAX_SEGS 56
> > +#define MLX5_CRYPTO_FEATURE_FLAGS \
> > +     (RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
> > RTE_CRYPTODEV_FF_HW_ACCELERATED | \
> > +     RTE_CRYPTODEV_FF_IN_PLACE_SGL |
> > RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT | \
> > +     RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | \
> > +     RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT | \
> > +     RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | \
> > +     RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY | \
> > +     RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS)
> >
> >  TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list
> > =
> >
> >       TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
> > @@ -31,8 +39,32 @@ int mlx5_crypto_logtype;
> >
> >  uint8_t mlx5_crypto_driver_id;
> >
> > -const struct rte_cryptodev_capabilities
> > -             mlx5_crypto_caps[RTE_CRYPTO_OP_TYPE_UNDEFINED];
> > +const struct rte_cryptodev_capabilities mlx5_crypto_caps[] = {
> > +     {               /* AES XTS */
> > +             .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
> > +             {.sym = {
> > +                     .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
> > +                     {.cipher = {
> > +                             .algo = RTE_CRYPTO_CIPHER_AES_XTS,
> > +                             .block_size = 16,
> > +                             .key_size = {
> > +                                     .min = 32,
> > +                                     .max = 64,
> > +                                     .increment = 32
> > +                             },
> > +                             .iv_size = {
> > +                                     .min = 16,
> > +                                     .max = 16,
> > +                                     .increment = 0
> > +                             },
> > +                             .dataunit_set =
> > +
> >       RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_512_BYTES |
> > +
> >       RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_4096_BYTES,
> > +                     }, }
> > +             }, }
> > +     },
> > +};
> > +
> >
> >  static const char mlx5_crypto_drv_name[] =
> > RTE_STR(MLX5_CRYPTO_DRIVER_NAME);
> >
> > @@ -67,7 +99,7 @@ mlx5_crypto_dev_infos_get(struct rte_cryptodev
> *dev,
> >       RTE_SET_USED(dev);
> >       if (dev_info != NULL) {
> >               dev_info->driver_id = mlx5_crypto_driver_id;
> > -             dev_info->feature_flags = 0;
> > +             dev_info->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
> >               dev_info->capabilities = mlx5_crypto_caps;
> >               dev_info->max_nb_queue_pairs = MLX5_CRYPTO_MAX_QPS;
> >               dev_info->min_mbuf_headroom_req = 0; @@ -954,6 +986,7 @@
> > mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
> >       crypto_dev->dev_ops = &mlx5_crypto_ops;
> >       crypto_dev->dequeue_burst = mlx5_crypto_dequeue_burst;
> >       crypto_dev->enqueue_burst = mlx5_crypto_enqueue_burst;
> > +     crypto_dev->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
> >       crypto_dev->driver_id = mlx5_crypto_driver_id;
> >       priv = crypto_dev->data->dev_private;
> >       priv->ctx = ctx;
> > --
> > 2.25.1

Re: [dpdk-dev] [EXT] [PATCH v3 10/15] crypto/mlx5: add keytag device argument

[Thomas Monjalon]

09/05/2021 10:31, Matan Azrad:
> 
> From: Akhil Goyal
> > > From: Suanming Mou <suanmingm@nvidia.com>
> > >
> > > A keytag is a piece of data encrypted together with a DEK.
> > >
> > > When a DEK is referenced by an MKEY.bsf through its index, the keytag
> > > is also supplied in the BSF as plaintext. The HW will decrypt the DEK
> > > (and the attached keytag) and will fail the operation if the keytags
> > > don't match.
> > >
> > > This commit adds the configuration of the keytag with devargs.
> > >
> > > Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
> > > Signed-off-by: Matan Azrad <matan@nvidia.com>
> > > ---
> > Documentation for devargs should be part of this patch.
> > Please split the last patch accordingly.
> > 
> > Title can be shortened to
> > Crypto/mlx5: add keytag devarg
> > 
> > Fix other patches of devargs accordingly.
> 
> As I said before, no devargs is really active before adding datapath patches.
> The option to add all the supported features \ documentations in the patch which actually adds the support is correct.
> 
> The last patch adds the capabilities and docs when all of them are really supported.

I think it is better for git history (blame, etc) to have doc
added at the same time of the code, even if the datapath is not active.
I agree with Akhil.

Re: [dpdk-dev] [EXT] [PATCH v3 15/15] crypto/mlx5: set feature flags and capabilities

[Akhil Goyal]

> > > +Supported NICs
> > > +--------------
> > > +
> > > +* Mellanox\ |reg| ConnectX\ |reg|-6 200G MCX654106A-HCAT (2x200G)
> > > +
> > > +Prerequisites
> > > +-------------
> > > +
> > > +- Mellanox OFED version: **5.3**
> > > +  see :doc:`../../nics/mlx5` guide for more Mellanox OFED details.
> >
> > Since the driver is by default compiled off due to the dependency on
> > external Libraries, I would recommend to add few lines here as well for
> > compilation.
> > Like to compile rdma-core and set PKG_CONFIG_LIBDIR.
> 
> Why? all Mellanox drivers has the same external dependencies.
> I added here link for the doc explains it well.

This is a crypto PMD, not a NIC PMD. Somebody working on crypto PMDs,
do not really care about the NIC PMDs.
Hence it would be convenient to have compilation information here as well.
You can refer to other document for details, but basic info should be added
here as well.

> 
> > And I do not see any updates to the test application for testing this driver.
> 
> You can see update to l2fwd_crypto, we tested with this example for the first
> stage.
> Everything looks ok there.

L2fwd-crypto is an app which only test data path with no packet validation.
It does not tell if your encryption is correctly done as per standards or not.
Did you test interoperability with l2fwd-crypto?
All basic configuration tests are also not done, like cleanup etc of the PMD.
I haven't seen a driver getting merge without the unit test application run.
Test app helps you comply with the way dpdk drivers are meant to be written.

> 
> > Is this driver really tested or is it work in progress? If it is work in progress,
> We
> > should defer this PMD to next release.
> 
> We can continue test this driver for a long time.
> The basics were tested in the above crypto example what proofs that driver
> works well.
> If we will find issues in other tests, we will send fixes.
> 
> I'll send v4 with your comments adjustment soon.
> 

Re: [dpdk-dev] [EXT] [PATCH v3 10/15] crypto/mlx5: add keytag device argument

[Akhil Goyal]

> From: Akhil Goyal
> > > From: Suanming Mou <suanmingm@nvidia.com>
> > >
> > > A keytag is a piece of data encrypted together with a DEK.
> > >
> > > When a DEK is referenced by an MKEY.bsf through its index, the keytag
> > > is also supplied in the BSF as plaintext. The HW will decrypt the DEK
> > > (and the attached keytag) and will fail the operation if the keytags
> > > don't match.
> > >
> > > This commit adds the configuration of the keytag with devargs.
> > >
> > > Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
> > > Signed-off-by: Matan Azrad <matan@nvidia.com>
> > > ---
> > Documentation for devargs should be part of this patch.
> > Please split the last patch accordingly.
> >
> > Title can be shortened to
> > Crypto/mlx5: add keytag devarg
> >
> > Fix other patches of devargs accordingly.
> 
> As I said before, no devargs is really active before adding datapath patches.
> The option to add all the supported features \ documentations in the patch
> which actually adds the support is correct.
> 
> The last patch adds the capabilities and docs when all of them are really
> supported.
> 
In that case split the patches in such a manner that data path is added
Before devargs patch with some dummy values and add the devargs with
Appropriate documentation update in a later patch.

Re: [dpdk-dev] [EXT] [PATCH v3 09/15] crypto/mlx5: adjust to the multiple data unit API

[Akhil Goyal]

> 
> > Also set feature flag in the code and the documentation in this patch.
> > I see that you are setting all of them in a single patch in the end.
> > This is not correct. It should be added where the feature is supported.
> > Please fix this for all the feature flags.
> 
> No, in this stage no feature is really supported, the actual time it will be
> supported is after the datapath patches and capabilities set.
> 
Move the patch after adding data path, but documentation should be part
Of this patch.

Re: [dpdk-dev] [EXT] [PATCH v3 03/15] crypto/mlx5: support session operations

[Akhil Goyal]

> > > +static void
> > > +mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev,
> > > +                           struct rte_cryptodev_sym_session *sess) {
> > > +     struct mlx5_crypto_priv *priv = dev->data->dev_private;
> > > +     struct mlx5_crypto_session *sess_private_data =
> > > +                     get_sym_session_private_data(sess,
> > > +dev->driver_id);
> > > +
> > > +     if (unlikely(sess_private_data == NULL)) {
> > > +             DRV_LOG(ERR, "Failed to get session %p private data.",
> > > +                             sess_private_data);
> > > +             return;
> > > +     }
> > > +     mlx5_crypto_dek_destroy(priv, sess_private_data->dek);
> > > +     DRV_LOG(DEBUG, "Session %p was cleared.", sess_private_data); }
> >
> > Memory leakage, mempool is not freed.
> 
> Yes, good catch, this part was missed.
> 
> > IMO, this driver is not properly tested with the unit test app.
> 
> The only app we tested until now is l2fwd_crypto and it works fine!
> We can add it to doc.
> 
> > Please add a note in the documentation that it is tested with autotest.
> 
> 
> The next app we want to test with, is test-crypto-perf.
> 
I would recommend to run the test app first.
It will catch most of your basic bugs like the one above.

Re: [dpdk-dev] [EXT] [PATCH v3 01/15] drivers: introduce mlx5 crypto PMD

[Akhil Goyal]

> > > +
> > > +fmt_name = 'mlx5_crypto'
> > > +deps += ['common_mlx5', 'eal', 'cryptodev']
> >
> > I don't see a reason for eal dependency. Do you really need it?
> 
> See RTE_LOG_REGISTER.

Compilation works fine even after removing this dependency.
I believe it is internally managed in cryptodev.
> 
> > > +sources = files(
> > > +     'mlx5_crypto.c',
> > > +)
> > > +cflags_options = [
> > > +     '-std=c11',
> > > +     '-Wno-strict-prototypes',
> > > +     '-D_BSD_SOURCE',
> > > +     '-D_DEFAULT_SOURCE',
> > > +     '-D_XOPEN_SOURCE=600',
> > > +]
> >
> > Any specific reason to use '-Wno-strict-prototypes'
> > Compilation works without this as well.
> 
> We use it in Mellanox driver usually.
> This is internal mlx5 code convention.
> 
Ok.

Re: [dpdk-dev] [EXT] [PATCH v3 15/15] crypto/mlx5: set feature flags and capabilities

[Matan Azrad]

From: Akhil Goyal
> > > > +Supported NICs
> > > > +--------------
> > > > +
> > > > +* Mellanox\ |reg| ConnectX\ |reg|-6 200G MCX654106A-HCAT
> (2x200G)
> > > > +
> > > > +Prerequisites
> > > > +-------------
> > > > +
> > > > +- Mellanox OFED version: **5.3**
> > > > +  see :doc:`../../nics/mlx5` guide for more Mellanox OFED details.
> > >
> > > Since the driver is by default compiled off due to the dependency on
> > > external Libraries, I would recommend to add few lines here as well
> > > for compilation.
> > > Like to compile rdma-core and set PKG_CONFIG_LIBDIR.
> >
> > Why? all Mellanox drivers has the same external dependencies.
> > I added here link for the doc explains it well.
> 
> This is a crypto PMD, not a NIC PMD. Somebody working on crypto PMDs, do
> not really care about the NIC PMDs.
> Hence it would be convenient to have compilation information here as well.
> You can refer to other document for details, but basic info should be added
> here as well.

The link explains how to install OFED, this is only what the user need to take from the link.
The basic is to install OFED.
I don't see a reason to duplicate doc section which are exactly the same. 

> >
> > > And I do not see any updates to the test application for testing this driver.
> >
> > You can see update to l2fwd_crypto, we tested with this example for
> > the first stage.
> > Everything looks ok there.
> 
> L2fwd-crypto is an app which only test data path with no packet validation.
> It does not tell if your encryption is correctly done as per standards or not.
> Did you test interoperability with l2fwd-crypto?
> All basic configuration tests are also not done, like cleanup etc of the PMD.
> I haven't seen a driver getting merge without the unit test application run.
> Test app helps you comply with the way dpdk drivers are meant to be
> written.

We adjusted the l2fwd-crypto to the dataunit feature and wrapped keys.
We validated data integrity from the packet returns back from the crypto net port.
As I said, encryption\decryption with AES-XTS is working well.

Now, is too late to update the test application to the above features, the driver code is here for a long time, no one ask about the test adjustment until now.

We can add the adjustment to increase validity for the next release to all the remaining crypto apps (test\test-crypto-perf).

For now, we have one validation with l2fwd-crypto And any user can run it and see how to use mlx5 driver.


> > > Is this driver really tested or is it work in progress? If it is
> > > work in progress,
> > We
> > > should defer this PMD to next release.
> >
> > We can continue test this driver for a long time.
> > The basics were tested in the above crypto example what proofs that
> > driver works well.
> > If we will find issues in other tests, we will send fixes.
> >
> > I'll send v4 with your comments adjustment soon.
> >

Re: [dpdk-dev] [EXT] [PATCH v3 10/15] crypto/mlx5: add keytag device argument

[Matan Azrad]

From: Akhil Goyal
> > From: Akhil Goyal
> > > > From: Suanming Mou <suanmingm@nvidia.com>
> > > >
> > > > A keytag is a piece of data encrypted together with a DEK.
> > > >
> > > > When a DEK is referenced by an MKEY.bsf through its index, the
> > > > keytag is also supplied in the BSF as plaintext. The HW will
> > > > decrypt the DEK (and the attached keytag) and will fail the
> > > > operation if the keytags don't match.
> > > >
> > > > This commit adds the configuration of the keytag with devargs.
> > > >
> > > > Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
> > > > Signed-off-by: Matan Azrad <matan@nvidia.com>
> > > > ---
> > > Documentation for devargs should be part of this patch.
> > > Please split the last patch accordingly.
> > >
> > > Title can be shortened to
> > > Crypto/mlx5: add keytag devarg
> > >
> > > Fix other patches of devargs accordingly.
> >
> > As I said before, no devargs is really active before adding datapath patches.
> > The option to add all the supported features \ documentations in the
> > patch which actually adds the support is correct.
> >
> > The last patch adds the capabilities and docs when all of them are
> > really supported.
> >
> In that case split the patches in such a manner that data path is added Before
> devargs patch with some dummy values and add the devargs with
> Appropriate documentation update in a later patch.


No, data-path must know all the devargs in advance, it cannot work well without them.

Re: [dpdk-dev] [EXT] [PATCH v3 09/15] crypto/mlx5: adjust to the multiple data unit API

[Matan Azrad]

> -----Original Message-----
> From: Akhil Goyal <gakhil@marvell.com>
> Sent: Sunday, May 9, 2021 12:19 PM
> To: Matan Azrad <matan@nvidia.com>; dev@dpdk.org
> Cc: Suanming Mou <suanmingm@nvidia.com>; NBU-Contact-Thomas
> Monjalon <thomas@monjalon.net>; Shiri Kuzin <shirik@nvidia.com>
> Subject: RE: [EXT] [PATCH v3 09/15] crypto/mlx5: adjust to the multiple data
> unit API
> 
> External email: Use caution opening links or attachments
> 
> 
> >
> > > Also set feature flag in the code and the documentation in this patch.
> > > I see that you are setting all of them in a single patch in the end.
> > > This is not correct. It should be added where the feature is supported.
> > > Please fix this for all the feature flags.
> >
> > No, in this stage no feature is really supported, the actual time it
> > will be supported is after the datapath patches and capabilities set.
> >
> Move the patch after adding data path, but documentation should be part Of
> this patch.

I will squash this patch to the session commit.

Re: [dpdk-dev] [EXT] [PATCH v3 03/15] crypto/mlx5: support session operations

[Matan Azrad]

> -----Original Message-----
> From: Akhil Goyal <gakhil@marvell.com>
> Sent: Sunday, May 9, 2021 12:21 PM
> To: Matan Azrad <matan@nvidia.com>; dev@dpdk.org
> Cc: Suanming Mou <suanmingm@nvidia.com>; NBU-Contact-Thomas
> Monjalon <thomas@monjalon.net>; Shiri Kuzin <shirik@nvidia.com>
> Subject: RE: [EXT] [PATCH v3 03/15] crypto/mlx5: support session operations
> 
> External email: Use caution opening links or attachments
> 
> 
> > > > +static void
> > > > +mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev,
> > > > +                           struct rte_cryptodev_sym_session *sess) {
> > > > +     struct mlx5_crypto_priv *priv = dev->data->dev_private;
> > > > +     struct mlx5_crypto_session *sess_private_data =
> > > > +                     get_sym_session_private_data(sess,
> > > > +dev->driver_id);
> > > > +
> > > > +     if (unlikely(sess_private_data == NULL)) {
> > > > +             DRV_LOG(ERR, "Failed to get session %p private data.",
> > > > +                             sess_private_data);
> > > > +             return;
> > > > +     }
> > > > +     mlx5_crypto_dek_destroy(priv, sess_private_data->dek);
> > > > +     DRV_LOG(DEBUG, "Session %p was cleared.", sess_private_data); }
> > >
> > > Memory leakage, mempool is not freed.
> >
> > Yes, good catch, this part was missed.
> >
> > > IMO, this driver is not properly tested with the unit test app.
> >
> > The only app we tested until now is l2fwd_crypto and it works fine!
> > We can add it to doc.
> >
> > > Please add a note in the documentation that it is tested with autotest.
> >
> >
> > The next app we want to test with, is test-crypto-perf.
> >
> I would recommend to run the test app first.
> It will catch most of your basic bugs like the one above.

It is too late for this, will add the test adjustment later.

Re: [dpdk-dev] [EXT] [PATCH v3 01/15] drivers: introduce mlx5 crypto PMD

[Matan Azrad]

> -----Original Message-----
> From: Akhil Goyal <gakhil@marvell.com>
> Sent: Sunday, May 9, 2021 12:33 PM
> To: Matan Azrad <matan@nvidia.com>; dev@dpdk.org
> Cc: Suanming Mou <suanmingm@nvidia.com>; NBU-Contact-Thomas
> Monjalon <thomas@monjalon.net>; Shiri Kuzin <shirik@nvidia.com>
> Subject: RE: [EXT] [PATCH v3 01/15] drivers: introduce mlx5 crypto PMD
> 
> External email: Use caution opening links or attachments
> 
> 
> > > > +
> > > > +fmt_name = 'mlx5_crypto'
> > > > +deps += ['common_mlx5', 'eal', 'cryptodev']
> > >
> > > I don't see a reason for eal dependency. Do you really need it?
> >
> > See RTE_LOG_REGISTER.
> 
> Compilation works fine even after removing this dependency.
> I believe it is internally managed in cryptodev.

Yes, but it is better to add the libs we use directly.

> > > > +sources = files(
> > > > +     'mlx5_crypto.c',
> > > > +)
> > > > +cflags_options = [
> > > > +     '-std=c11',
> > > > +     '-Wno-strict-prototypes',
> > > > +     '-D_BSD_SOURCE',
> > > > +     '-D_DEFAULT_SOURCE',
> > > > +     '-D_XOPEN_SOURCE=600',
> > > > +]
> > >
> > > Any specific reason to use '-Wno-strict-prototypes'
> > > Compilation works without this as well.
> >
> > We use it in Mellanox driver usually.
> > This is internal mlx5 code convention.
> >
> Ok.

Re: [dpdk-dev] [EXT] [PATCH v3 10/15] crypto/mlx5: add keytag device argument

[Matan Azrad]

From: Thomas Monjalon
> 09/05/2021 10:31, Matan Azrad:
> >
> > From: Akhil Goyal
> > > > From: Suanming Mou <suanmingm@nvidia.com>
> > > >
> > > > A keytag is a piece of data encrypted together with a DEK.
> > > >
> > > > When a DEK is referenced by an MKEY.bsf through its index, the
> > > > keytag is also supplied in the BSF as plaintext. The HW will
> > > > decrypt the DEK (and the attached keytag) and will fail the
> > > > operation if the keytags don't match.
> > > >
> > > > This commit adds the configuration of the keytag with devargs.
> > > >
> > > > Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
> > > > Signed-off-by: Matan Azrad <matan@nvidia.com>
> > > > ---
> > > Documentation for devargs should be part of this patch.
> > > Please split the last patch accordingly.
> > >
> > > Title can be shortened to
> > > Crypto/mlx5: add keytag devarg
> > >
> > > Fix other patches of devargs accordingly.
> >
> > As I said before, no devargs is really active before adding datapath patches.
> > The option to add all the supported features \ documentations in the patch
> which actually adds the support is correct.
> >
> > The last patch adds the capabilities and docs when all of them are really
> supported.
> 
> I think it is better for git history (blame, etc) to have doc added at the same
> time of the code, even if the datapath is not active.
> I agree with Akhil.

User can read the commit log which explain things enough for blame case. 

I don't agree, here the devargs has no any usage, just saved in the pmd mem.
The devargs is actually active in the last datapath patch, we can squash the doc to datapath patch if you realy insist...

[dpdk-dev] [PATCH v4 00/14] drivers: introduce mlx5 crypto PMD

[Matan Azrad]

Add a new PMD for Nvidia devices- crypto PMD.
The crypto PMD will be supported on Nvidia ConnectX6 The crypto PMD will add the support of encryption and decryption using the AES-XTS symmetric algorithm.
The crypto PMD requires rdma-core and uses mlx5 DevX.

v2:
Add data-path part.

v3:
Rebase.

v4 - Rebase + Address the following Akhil comments:
- Set HW feature flag in the capability patch.
- Fix mp object release in session clear.
- Some spelling and word missing in doc.
- Squash data-unit adjustment to the session operations commit.
- Wording: device argument -> devarg.


Shiri Kuzin (9):
  drivers: introduce mlx5 crypto PMD
  crypto/mlx5: add DEK object management
  crypto/mlx5: add session operations
  crypto/mlx5: add basic operations
  crypto/mlx5: add queue pairs operations
  crypto/mlx5: add dev stop and start operations
  crypto/mlx5: add memory region management
  crypto/mlx5: create login object using DevX
  crypto/mlx5: set feature flags and capabilities

Suanming Mou (5):
  crypto/mlx5: add keytag devarg
  crypto/mlx5: add maximum segments devarg
  crypto/mlx5: add WQE set initialization
  crypto/mlx5: add enqueue and dequeue operations
  crypto/mlx5: add statistic get and reset operations

 MAINTAINERS                             |    4 +
 doc/guides/cryptodevs/features/mlx5.ini |   37 +
 doc/guides/cryptodevs/index.rst         |    1 +
 doc/guides/cryptodevs/mlx5.rst          |  152 ++++
 doc/guides/rel_notes/release_21_05.rst  |    5 +
 drivers/common/mlx5/mlx5_common.h       |    1 +
 drivers/common/mlx5/mlx5_common_pci.c   |   14 +
 drivers/common/mlx5/mlx5_common_pci.h   |   21 +-
 drivers/crypto/meson.build              |    1 +
 drivers/crypto/mlx5/meson.build         |   27 +
 drivers/crypto/mlx5/mlx5_crypto.c       | 1093 +++++++++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto.h       |   92 ++
 drivers/crypto/mlx5/mlx5_crypto_dek.c   |  136 +++
 drivers/crypto/mlx5/mlx5_crypto_utils.h |   19 +
 drivers/crypto/mlx5/version.map         |    3 +
 15 files changed, 1596 insertions(+), 10 deletions(-)
 create mode 100644 doc/guides/cryptodevs/features/mlx5.ini
 create mode 100644 doc/guides/cryptodevs/mlx5.rst
 create mode 100644 drivers/crypto/mlx5/meson.build
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto.c
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto.h
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_dek.c
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_utils.h
 create mode 100644 drivers/crypto/mlx5/version.map

-- 
2.25.1

[dpdk-dev] [PATCH v4 01/14] drivers: introduce mlx5 crypto PMD

[Matan Azrad]

From: Shiri Kuzin <shirik@nvidia.com>

Add a new PMD for Mellanox devices- crypto PMD.

The crypto PMD will be supported starting Nvidia ConnectX6 and
BlueField2.

The crypto PMD will add the support of encryption and decryption using
the AES-XTS symmetric algorithm.

The crypto PMD requires rdma-core and uses mlx5 DevX.

This patch adds the PCI probing, basic functions, build files and
log utility.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 MAINTAINERS                             |   4 +
 drivers/common/mlx5/mlx5_common.h       |   1 +
 drivers/common/mlx5/mlx5_common_pci.c   |  14 ++
 drivers/common/mlx5/mlx5_common_pci.h   |  21 +-
 drivers/crypto/meson.build              |   1 +
 drivers/crypto/mlx5/meson.build         |  26 +++
 drivers/crypto/mlx5/mlx5_crypto.c       | 272 ++++++++++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto_utils.h |  19 ++
 drivers/crypto/mlx5/version.map         |   3 +
 9 files changed, 351 insertions(+), 10 deletions(-)
 create mode 100644 drivers/crypto/mlx5/meson.build
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto.c
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_utils.h
 create mode 100644 drivers/crypto/mlx5/version.map

diff --git a/MAINTAINERS b/MAINTAINERS
index 68856b3aae..fa5c88988b 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -1078,6 +1078,10 @@ F: drivers/crypto/octeontx2/
 F: doc/guides/cryptodevs/octeontx2.rst
 F: doc/guides/cryptodevs/features/octeontx2.ini
 
+Mellanox mlx5
+M: Matan Azrad <matan@nvidia.com>
+F: drivers/crypto/mlx5/
+
 Null Crypto
 M: Declan Doherty <declan.doherty@intel.com>
 F: drivers/crypto/null/
diff --git a/drivers/common/mlx5/mlx5_common.h b/drivers/common/mlx5/mlx5_common.h
index 1fbefe0fa6..89aca32305 100644
--- a/drivers/common/mlx5/mlx5_common.h
+++ b/drivers/common/mlx5/mlx5_common.h
@@ -216,6 +216,7 @@ enum mlx5_class {
 	MLX5_CLASS_VDPA = RTE_BIT64(1),
 	MLX5_CLASS_REGEX = RTE_BIT64(2),
 	MLX5_CLASS_COMPRESS = RTE_BIT64(3),
+	MLX5_CLASS_CRYPTO = RTE_BIT64(4),
 };
 
 #define MLX5_DBR_SIZE RTE_CACHE_LINE_SIZE
diff --git a/drivers/common/mlx5/mlx5_common_pci.c b/drivers/common/mlx5/mlx5_common_pci.c
index 3f16cd21cf..8a47afee20 100644
--- a/drivers/common/mlx5/mlx5_common_pci.c
+++ b/drivers/common/mlx5/mlx5_common_pci.c
@@ -31,6 +31,7 @@ static const struct {
 	{ .name = "net", .driver_class = MLX5_CLASS_NET },
 	{ .name = "regex", .driver_class = MLX5_CLASS_REGEX },
 	{ .name = "compress", .driver_class = MLX5_CLASS_COMPRESS },
+	{ .name = "crypto", .driver_class = MLX5_CLASS_CRYPTO },
 };
 
 static const unsigned int mlx5_class_combinations[] = {
@@ -38,13 +39,26 @@ static const unsigned int mlx5_class_combinations[] = {
 	MLX5_CLASS_VDPA,
 	MLX5_CLASS_REGEX,
 	MLX5_CLASS_COMPRESS,
+	MLX5_CLASS_CRYPTO,
 	MLX5_CLASS_NET | MLX5_CLASS_REGEX,
 	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX,
 	MLX5_CLASS_NET | MLX5_CLASS_COMPRESS,
 	MLX5_CLASS_VDPA | MLX5_CLASS_COMPRESS,
 	MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
+	MLX5_CLASS_NET | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_VDPA | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO,
 	MLX5_CLASS_NET | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
 	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
+	MLX5_CLASS_NET | MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_NET | MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_VDPA | MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_NET | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS |
+	MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS |
+	MLX5_CLASS_CRYPTO,
 	/* New class combination should be added here. */
 };
 
diff --git a/drivers/common/mlx5/mlx5_common_pci.h b/drivers/common/mlx5/mlx5_common_pci.h
index de89bb98bc..cb8d2f5f87 100644
--- a/drivers/common/mlx5/mlx5_common_pci.h
+++ b/drivers/common/mlx5/mlx5_common_pci.h
@@ -9,17 +9,18 @@
  * @file
  *
  * RTE Mellanox PCI Driver Interface
- * Mellanox ConnectX PCI device supports multiple class: net,vdpa,regex and
- * compress devices. This layer enables creating such multiple class of devices
- * on a single PCI device by allowing to bind multiple class specific device
- * driver to attach to mlx5_pci driver.
+ * Mellanox ConnectX PCI device supports multiple class: net,vdpa,regex,compress
+ * and crypto devices. This layer enables creating such multiple class of
+ * devices on a single PCI device by allowing to bind multiple class specific
+ * device driver to attach to mlx5_pci driver.
  *
- * -----------    ------------    -------------    ----------------
- * |   mlx5  |    |   mlx5   |    |   mlx5    |    |     mlx5     |
- * | net pmd |    | vdpa pmd |    | regex pmd |    | compress pmd |
- * -----------    ------------    -------------    ----------------
- *      \              \                    /              /
- *       \              \                  /              /
+ * --------    --------    ---------    ------------    ----------
+ * | mlx5 |    | mlx5 |    | mlx5  |    |   mlx5   |    |  mlx5  |
+ * | net  |    | vdpa |    | regex |    | compress |    | crypto |
+ * | pmd  |    | pmd  |    |  pmd  |    |   pmd    |    |  pmd   |
+ * --------    --------    ---------    ------------    ----------
+ *      \              \         |          /              /
+ *       \              \        |         /              /
  *        \              \_--------------_/              /
  *         \_______________|   mlx5     |_______________/
  *                         | pci common |
diff --git a/drivers/crypto/meson.build b/drivers/crypto/meson.build
index b9fdf9392f..6951607def 100644
--- a/drivers/crypto/meson.build
+++ b/drivers/crypto/meson.build
@@ -15,6 +15,7 @@ drivers = [
         'dpaa_sec',
         'dpaa2_sec',
         'kasumi',
+        'mlx5',
         'mvsam',
         'nitrox',
         'null',
diff --git a/drivers/crypto/mlx5/meson.build b/drivers/crypto/mlx5/meson.build
new file mode 100644
index 0000000000..fd00283665
--- /dev/null
+++ b/drivers/crypto/mlx5/meson.build
@@ -0,0 +1,26 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2021 Mellanox Technologies, Ltd
+
+if not is_linux
+	build = false
+	reason = 'only supported on Linux'
+	subdir_done()
+endif
+
+fmt_name = 'mlx5_crypto'
+deps += ['common_mlx5', 'eal', 'cryptodev']
+sources = files(
+	'mlx5_crypto.c',
+)
+cflags_options = [
+	'-std=c11',
+	'-Wno-strict-prototypes',
+	'-D_BSD_SOURCE',
+	'-D_DEFAULT_SOURCE',
+	'-D_XOPEN_SOURCE=600',
+]
+foreach option:cflags_options
+	if cc.has_argument(option)
+		cflags += option
+	endif
+endforeach
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
new file mode 100644
index 0000000000..ea259112f4
--- /dev/null
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -0,0 +1,272 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#include <rte_malloc.h>
+#include <rte_log.h>
+#include <rte_errno.h>
+#include <rte_pci.h>
+#include <rte_crypto.h>
+#include <rte_cryptodev.h>
+#include <rte_cryptodev_pmd.h>
+
+#include <mlx5_glue.h>
+#include <mlx5_common.h>
+#include <mlx5_common_pci.h>
+#include <mlx5_devx_cmds.h>
+#include <mlx5_common_os.h>
+
+#include "mlx5_crypto_utils.h"
+
+#define MLX5_CRYPTO_DRIVER_NAME mlx5_crypto
+#define MLX5_CRYPTO_LOG_NAME    pmd.crypto.mlx5
+
+struct mlx5_crypto_priv {
+	TAILQ_ENTRY(mlx5_crypto_priv) next;
+	struct ibv_context *ctx; /* Device context. */
+	struct rte_pci_device *pci_dev;
+	struct rte_cryptodev *crypto_dev;
+	void *uar; /* User Access Region. */
+	uint32_t pdn; /* Protection Domain number. */
+	struct ibv_pd *pd;
+};
+
+TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
+				TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
+static pthread_mutex_t priv_list_lock = PTHREAD_MUTEX_INITIALIZER;
+
+int mlx5_crypto_logtype;
+
+uint8_t mlx5_crypto_driver_id;
+
+static const char mlx5_crypto_drv_name[] = RTE_STR(MLX5_CRYPTO_DRIVER_NAME);
+
+static const struct rte_driver mlx5_drv = {
+	.name = mlx5_crypto_drv_name,
+	.alias = mlx5_crypto_drv_name
+};
+
+static struct cryptodev_driver mlx5_cryptodev_driver;
+
+static struct rte_cryptodev_ops mlx5_crypto_ops = {
+	.dev_configure			= NULL,
+	.dev_start			= NULL,
+	.dev_stop			= NULL,
+	.dev_close			= NULL,
+	.dev_infos_get			= NULL,
+	.stats_get			= NULL,
+	.stats_reset			= NULL,
+	.queue_pair_setup		= NULL,
+	.queue_pair_release		= NULL,
+	.sym_session_get_size		= NULL,
+	.sym_session_configure		= NULL,
+	.sym_session_clear		= NULL,
+	.sym_get_raw_dp_ctx_size	= NULL,
+	.sym_configure_raw_dp_ctx	= NULL,
+};
+
+static void
+mlx5_crypto_hw_global_release(struct mlx5_crypto_priv *priv)
+{
+	if (priv->pd != NULL) {
+		claim_zero(mlx5_glue->dealloc_pd(priv->pd));
+		priv->pd = NULL;
+	}
+	if (priv->uar != NULL) {
+		mlx5_glue->devx_free_uar(priv->uar);
+		priv->uar = NULL;
+	}
+}
+
+static int
+mlx5_crypto_pd_create(struct mlx5_crypto_priv *priv)
+{
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+	struct mlx5dv_obj obj;
+	struct mlx5dv_pd pd_info;
+	int ret;
+
+	priv->pd = mlx5_glue->alloc_pd(priv->ctx);
+	if (priv->pd == NULL) {
+		DRV_LOG(ERR, "Failed to allocate PD.");
+		return errno ? -errno : -ENOMEM;
+	}
+	obj.pd.in = priv->pd;
+	obj.pd.out = &pd_info;
+	ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_PD);
+	if (ret != 0) {
+		DRV_LOG(ERR, "Fail to get PD object info.");
+		mlx5_glue->dealloc_pd(priv->pd);
+		priv->pd = NULL;
+		return -errno;
+	}
+	priv->pdn = pd_info.pdn;
+	return 0;
+#else
+	(void)priv;
+	DRV_LOG(ERR, "Cannot get pdn - no DV support.");
+	return -ENOTSUP;
+#endif /* HAVE_IBV_FLOW_DV_SUPPORT */
+}
+
+static int
+mlx5_crypto_hw_global_prepare(struct mlx5_crypto_priv *priv)
+{
+	if (mlx5_crypto_pd_create(priv) != 0)
+		return -1;
+	priv->uar = mlx5_devx_alloc_uar(priv->ctx, -1);
+	if (priv->uar == NULL || mlx5_os_get_devx_uar_reg_addr(priv->uar) ==
+	    NULL) {
+		rte_errno = errno;
+		claim_zero(mlx5_glue->dealloc_pd(priv->pd));
+		DRV_LOG(ERR, "Failed to allocate UAR.");
+		return -1;
+	}
+	return 0;
+}
+
+/**
+ * DPDK callback to register a PCI device.
+ *
+ * This function spawns crypto device out of a given PCI device.
+ *
+ * @param[in] pci_drv
+ *   PCI driver structure (mlx5_crypto_driver).
+ * @param[in] pci_dev
+ *   PCI device information.
+ *
+ * @return
+ *   0 on success, 1 to skip this driver, a negative errno value otherwise
+ *   and rte_errno is set.
+ */
+static int
+mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
+			struct rte_pci_device *pci_dev)
+{
+	struct ibv_device *ibv;
+	struct rte_cryptodev *crypto_dev;
+	struct ibv_context *ctx;
+	struct mlx5_crypto_priv *priv;
+	struct mlx5_hca_attr attr = { 0 };
+	struct rte_cryptodev_pmd_init_params init_params = {
+		.name = "",
+		.private_data_size = sizeof(struct mlx5_crypto_priv),
+		.socket_id = pci_dev->device.numa_node,
+		.max_nb_queue_pairs =
+				RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
+	};
+	RTE_SET_USED(pci_drv);
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+		DRV_LOG(ERR, "Non-primary process type is not supported.");
+		rte_errno = ENOTSUP;
+		return -rte_errno;
+	}
+	ibv = mlx5_os_get_ibv_device(&pci_dev->addr);
+	if (ibv == NULL) {
+		DRV_LOG(ERR, "No matching IB device for PCI slot "
+			PCI_PRI_FMT ".", pci_dev->addr.domain,
+			pci_dev->addr.bus, pci_dev->addr.devid,
+			pci_dev->addr.function);
+		return -rte_errno;
+	}
+	DRV_LOG(INFO, "PCI information matches for device \"%s\".", ibv->name);
+	ctx = mlx5_glue->dv_open_device(ibv);
+	if (ctx == NULL) {
+		DRV_LOG(ERR, "Failed to open IB device \"%s\".", ibv->name);
+		rte_errno = ENODEV;
+		return -rte_errno;
+	}
+	if (mlx5_devx_cmd_query_hca_attr(ctx, &attr) != 0 ||
+	    attr.crypto == 0 || attr.aes_xts == 0) {
+		DRV_LOG(ERR, "Not enough capabilities to support crypto "
+			"operations, maybe old FW/OFED version?");
+		claim_zero(mlx5_glue->close_device(ctx));
+		rte_errno = ENOTSUP;
+		return -ENOTSUP;
+	}
+	crypto_dev = rte_cryptodev_pmd_create(ibv->name, &pci_dev->device,
+					&init_params);
+	if (crypto_dev == NULL) {
+		DRV_LOG(ERR, "Failed to create device \"%s\".", ibv->name);
+		claim_zero(mlx5_glue->close_device(ctx));
+		return -ENODEV;
+	}
+	DRV_LOG(INFO,
+		"Crypto device %s was created successfully.", ibv->name);
+	crypto_dev->dev_ops = &mlx5_crypto_ops;
+	crypto_dev->dequeue_burst = NULL;
+	crypto_dev->enqueue_burst = NULL;
+	crypto_dev->feature_flags = 0;
+	crypto_dev->driver_id = mlx5_crypto_driver_id;
+	priv = crypto_dev->data->dev_private;
+	priv->ctx = ctx;
+	priv->pci_dev = pci_dev;
+	priv->crypto_dev = crypto_dev;
+	if (mlx5_crypto_hw_global_prepare(priv) != 0) {
+		rte_cryptodev_pmd_destroy(priv->crypto_dev);
+		claim_zero(mlx5_glue->close_device(priv->ctx));
+		return -1;
+	}
+	pthread_mutex_lock(&priv_list_lock);
+	TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next);
+	pthread_mutex_unlock(&priv_list_lock);
+	return 0;
+}
+
+static int
+mlx5_crypto_pci_remove(struct rte_pci_device *pdev)
+{
+	struct mlx5_crypto_priv *priv = NULL;
+
+	pthread_mutex_lock(&priv_list_lock);
+	TAILQ_FOREACH(priv, &mlx5_crypto_priv_list, next)
+		if (rte_pci_addr_cmp(&priv->pci_dev->addr, &pdev->addr) != 0)
+			break;
+	if (priv)
+		TAILQ_REMOVE(&mlx5_crypto_priv_list, priv, next);
+	pthread_mutex_unlock(&priv_list_lock);
+	if (priv) {
+		mlx5_crypto_hw_global_release(priv);
+		rte_cryptodev_pmd_destroy(priv->crypto_dev);
+		claim_zero(mlx5_glue->close_device(priv->ctx));
+	}
+	return 0;
+}
+
+static const struct rte_pci_id mlx5_crypto_pci_id_map[] = {
+		{
+			RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
+					PCI_DEVICE_ID_MELLANOX_CONNECTX6)
+		},
+		{
+			.vendor_id = 0
+		}
+	};
+
+static struct mlx5_pci_driver mlx5_crypto_driver = {
+	.driver_class = MLX5_CLASS_CRYPTO,
+	.pci_driver = {
+		.driver = {
+			.name = RTE_STR(MLX5_CRYPTO_DRIVER_NAME),
+		},
+		.id_table = mlx5_crypto_pci_id_map,
+		.probe = mlx5_crypto_pci_probe,
+		.remove = mlx5_crypto_pci_remove,
+		.drv_flags = 0,
+	},
+};
+
+RTE_INIT(rte_mlx5_crypto_init)
+{
+	mlx5_common_init();
+	if (mlx5_glue != NULL)
+		mlx5_pci_driver_register(&mlx5_crypto_driver);
+}
+
+RTE_PMD_REGISTER_CRYPTO_DRIVER(mlx5_cryptodev_driver, mlx5_drv,
+			       mlx5_crypto_driver_id);
+
+RTE_LOG_REGISTER(mlx5_crypto_logtype, MLX5_CRYPTO_LOG_NAME, NOTICE)
+RTE_PMD_EXPORT_NAME(MLX5_CRYPTO_DRIVER_NAME, __COUNTER__);
+RTE_PMD_REGISTER_PCI_TABLE(MLX5_CRYPTO_DRIVER_NAME, mlx5_crypto_pci_id_map);
+RTE_PMD_REGISTER_KMOD_DEP(MLX5_CRYPTO_DRIVER_NAME, "* ib_uverbs & mlx5_core & mlx5_ib");
diff --git a/drivers/crypto/mlx5/mlx5_crypto_utils.h b/drivers/crypto/mlx5/mlx5_crypto_utils.h
new file mode 100644
index 0000000000..cef4b07a36
--- /dev/null
+++ b/drivers/crypto/mlx5/mlx5_crypto_utils.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#ifndef RTE_PMD_MLX5_CRYPTO_UTILS_H_
+#define RTE_PMD_MLX5_CRYPTO_UTILS_H_
+
+#include <mlx5_common.h>
+
+extern int mlx5_crypto_logtype;
+
+#define MLX5_CRYPTO_LOG_PREFIX "mlx5_crypto"
+/* Generic printf()-like logging macro with automatic line feed. */
+#define DRV_LOG(level, ...) \
+	PMD_DRV_LOG_(level, mlx5_crypto_logtype, MLX5_CRYPTO_LOG_PREFIX, \
+		__VA_ARGS__ PMD_DRV_LOG_STRIP PMD_DRV_LOG_OPAREN, \
+		PMD_DRV_LOG_CPAREN)
+
+#endif /* RTE_PMD_MLX5_CRYPTO_UTILS_H_ */
diff --git a/drivers/crypto/mlx5/version.map b/drivers/crypto/mlx5/version.map
new file mode 100644
index 0000000000..4a76d1d52d
--- /dev/null
+++ b/drivers/crypto/mlx5/version.map
@@ -0,0 +1,3 @@
+DPDK_21 {
+	local: *;
+};
-- 
2.25.1

[dpdk-dev] [PATCH v4 02/14] crypto/mlx5: add DEK object management

[Matan Azrad]

From: Shiri Kuzin <shirik@nvidia.com>

A DEK(Data encryption Key) is an mlx5 HW object which represents the
cipher algorithm key.
The DEKs are used during data encryption/decryption operations.

In symmetric algorithms like AES-STS, we use the same DEK for both
encryption and decryption.

Use the mlx5 hash-list tool to manage the DEK objects in the PMD.

Provide the compare, create and destroy functions to manage DEKs in
hash-list and introduce an internal API to setup and unset the DEK
management and to prepare and destroy specific DEK object.

The DEK hash-list will be created in dev_configure routine and
destroyed in dev_close routine.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/meson.build       |   1 +
 drivers/crypto/mlx5/mlx5_crypto.c     |  44 +++++----
 drivers/crypto/mlx5/mlx5_crypto.h     |  51 ++++++++++
 drivers/crypto/mlx5/mlx5_crypto_dek.c | 136 ++++++++++++++++++++++++++
 4 files changed, 215 insertions(+), 17 deletions(-)
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto.h
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_dek.c

diff --git a/drivers/crypto/mlx5/meson.build b/drivers/crypto/mlx5/meson.build
index fd00283665..9cb3bd214f 100644
--- a/drivers/crypto/mlx5/meson.build
+++ b/drivers/crypto/mlx5/meson.build
@@ -11,6 +11,7 @@ fmt_name = 'mlx5_crypto'
 deps += ['common_mlx5', 'eal', 'cryptodev']
 sources = files(
 	'mlx5_crypto.c',
+	'mlx5_crypto_dek.c',
 )
 cflags_options = [
 	'-std=c11',
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index ea259112f4..2825f4192e 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -3,12 +3,9 @@
  */
 
 #include <rte_malloc.h>
-#include <rte_log.h>
 #include <rte_errno.h>
+#include <rte_log.h>
 #include <rte_pci.h>
-#include <rte_crypto.h>
-#include <rte_cryptodev.h>
-#include <rte_cryptodev_pmd.h>
 
 #include <mlx5_glue.h>
 #include <mlx5_common.h>
@@ -17,19 +14,10 @@
 #include <mlx5_common_os.h>
 
 #include "mlx5_crypto_utils.h"
+#include "mlx5_crypto.h"
 
 #define MLX5_CRYPTO_DRIVER_NAME mlx5_crypto
-#define MLX5_CRYPTO_LOG_NAME    pmd.crypto.mlx5
-
-struct mlx5_crypto_priv {
-	TAILQ_ENTRY(mlx5_crypto_priv) next;
-	struct ibv_context *ctx; /* Device context. */
-	struct rte_pci_device *pci_dev;
-	struct rte_cryptodev *crypto_dev;
-	void *uar; /* User Access Region. */
-	uint32_t pdn; /* Protection Domain number. */
-	struct ibv_pd *pd;
-};
+#define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
 
 TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
 				TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
@@ -48,11 +36,33 @@ static const struct rte_driver mlx5_drv = {
 
 static struct cryptodev_driver mlx5_cryptodev_driver;
 
+static int
+mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
+		struct rte_cryptodev_config *config __rte_unused)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+
+	if (mlx5_crypto_dek_setup(priv) != 0) {
+		DRV_LOG(ERR, "Dek hash list creation has failed.");
+		return -ENOMEM;
+	}
+	return 0;
+}
+
+static int
+mlx5_crypto_dev_close(struct rte_cryptodev *dev)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+
+	mlx5_crypto_dek_unset(priv);
+	return 0;
+}
+
 static struct rte_cryptodev_ops mlx5_crypto_ops = {
-	.dev_configure			= NULL,
+	.dev_configure			= mlx5_crypto_dev_configure,
 	.dev_start			= NULL,
 	.dev_stop			= NULL,
-	.dev_close			= NULL,
+	.dev_close			= mlx5_crypto_dev_close,
 	.dev_infos_get			= NULL,
 	.stats_get			= NULL,
 	.stats_reset			= NULL,
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
new file mode 100644
index 0000000000..4ec67a7e0f
--- /dev/null
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -0,0 +1,51 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#ifndef MLX5_CRYPTO_H_
+#define MLX5_CRYPTO_H_
+
+#include <stdbool.h>
+
+#include <rte_cryptodev.h>
+#include <rte_cryptodev_pmd.h>
+
+#include <mlx5_common_utils.h>
+
+#define MLX5_CRYPTO_DEK_HTABLE_SZ (1 << 11)
+#define MLX5_CRYPTO_KEY_LENGTH 80
+
+struct mlx5_crypto_priv {
+	TAILQ_ENTRY(mlx5_crypto_priv) next;
+	struct ibv_context *ctx; /* Device context. */
+	struct rte_pci_device *pci_dev;
+	struct rte_cryptodev *crypto_dev;
+	void *uar; /* User Access Region. */
+	uint32_t pdn; /* Protection Domain number. */
+	struct ibv_pd *pd;
+	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
+};
+
+struct mlx5_crypto_dek {
+	struct mlx5_hlist_entry entry; /* Pointer to DEK hash list entry. */
+	struct mlx5_devx_obj *obj; /* Pointer to DEK DevX object. */
+	uint8_t data[MLX5_CRYPTO_KEY_LENGTH]; /* DEK key data. */
+	bool size_is_48; /* Whether the key\data size is 48 bytes or not. */
+};
+
+int
+mlx5_crypto_dek_destroy(struct mlx5_crypto_priv *priv,
+			struct mlx5_crypto_dek *dek);
+
+struct mlx5_crypto_dek *
+mlx5_crypto_dek_prepare(struct mlx5_crypto_priv *priv,
+			struct rte_crypto_cipher_xform *cipher);
+
+int
+mlx5_crypto_dek_setup(struct mlx5_crypto_priv *priv);
+
+void
+mlx5_crypto_dek_unset(struct mlx5_crypto_priv *priv);
+
+#endif /* MLX5_CRYPTO_H_ */
+
diff --git a/drivers/crypto/mlx5/mlx5_crypto_dek.c b/drivers/crypto/mlx5/mlx5_crypto_dek.c
new file mode 100644
index 0000000000..c76e208845
--- /dev/null
+++ b/drivers/crypto/mlx5/mlx5_crypto_dek.c
@@ -0,0 +1,136 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2018 Mellanox Technologies, Ltd
+ */
+
+#include <rte_ip.h>
+#include <rte_common.h>
+#include <rte_errno.h>
+#include <rte_log.h>
+
+#include <mlx5_prm.h>
+#include <mlx5_devx_cmds.h>
+
+#include "mlx5_crypto_utils.h"
+#include "mlx5_crypto.h"
+
+struct mlx5_crypto_dek_ctx {
+	struct rte_crypto_cipher_xform *cipher;
+	struct mlx5_crypto_priv *priv;
+};
+
+int
+mlx5_crypto_dek_destroy(struct mlx5_crypto_priv *priv,
+			struct mlx5_crypto_dek *dek)
+{
+	return mlx5_hlist_unregister(priv->dek_hlist, &dek->entry);
+}
+
+struct mlx5_crypto_dek *
+mlx5_crypto_dek_prepare(struct mlx5_crypto_priv *priv,
+			struct rte_crypto_cipher_xform *cipher)
+{
+	struct mlx5_hlist *dek_hlist = priv->dek_hlist;
+	struct mlx5_crypto_dek_ctx dek_ctx = {
+		.cipher = cipher,
+		.priv = priv,
+	};
+	struct rte_crypto_cipher_xform *cipher_ctx = cipher;
+	uint64_t key64 = __rte_raw_cksum(cipher_ctx->key.data,
+					 cipher_ctx->key.length, 0);
+	struct mlx5_hlist_entry *entry = mlx5_hlist_register(dek_hlist,
+							     key64, &dek_ctx);
+
+	return entry == NULL ? NULL :
+			     container_of(entry, struct mlx5_crypto_dek, entry);
+}
+
+static int
+mlx5_crypto_dek_match_cb(struct mlx5_hlist *list __rte_unused,
+			 struct mlx5_hlist_entry *entry,
+			 uint64_t key __rte_unused, void *cb_ctx)
+{
+	struct mlx5_crypto_dek_ctx *ctx = cb_ctx;
+	struct rte_crypto_cipher_xform *cipher_ctx = ctx->cipher;
+	struct mlx5_crypto_dek *dek =
+			container_of(entry, typeof(*dek), entry);
+	uint32_t key_len = dek->size_is_48 ? 48 : 80;
+
+	if (key_len != cipher_ctx->key.length)
+		return -1;
+	return memcmp(cipher_ctx->key.data, dek->data, key_len);
+}
+
+static struct mlx5_hlist_entry *
+mlx5_crypto_dek_create_cb(struct mlx5_hlist *list __rte_unused,
+			  uint64_t key __rte_unused, void *cb_ctx)
+{
+	struct mlx5_crypto_dek_ctx *ctx = cb_ctx;
+	struct rte_crypto_cipher_xform *cipher_ctx = ctx->cipher;
+	struct mlx5_crypto_dek *dek = rte_zmalloc(__func__, sizeof(*dek),
+						  RTE_CACHE_LINE_SIZE);
+	struct mlx5_devx_dek_attr dek_attr = {
+		.pd = ctx->priv->pdn,
+		.key_purpose = MLX5_CRYPTO_KEY_PURPOSE_AES_XTS,
+		.has_keytag = 1,
+	};
+
+	if (dek == NULL) {
+		DRV_LOG(ERR, "Failed to allocate dek memory.");
+		return NULL;
+	}
+	switch (cipher_ctx->key.length) {
+	case 48:
+		dek->size_is_48 = true;
+		dek_attr.key_size = MLX5_CRYPTO_KEY_SIZE_128b;
+		break;
+	case 80:
+		dek->size_is_48 = false;
+		dek_attr.key_size = MLX5_CRYPTO_KEY_SIZE_256b;
+		break;
+	default:
+		DRV_LOG(ERR, "Key size not supported.");
+		return NULL;
+	}
+	rte_memcpy(&dek_attr.key, cipher_ctx->key.data, cipher_ctx->key.length);
+	dek->obj = mlx5_devx_cmd_create_dek_obj(ctx->priv->ctx, &dek_attr);
+	if (dek->obj == NULL) {
+		rte_free(dek);
+		return NULL;
+	}
+	rte_memcpy(&dek->data, cipher_ctx->key.data, cipher_ctx->key.length);
+	return &dek->entry;
+}
+
+static void
+mlx5_crypto_dek_remove_cb(struct mlx5_hlist *list __rte_unused,
+			  struct mlx5_hlist_entry *entry)
+{
+	struct mlx5_crypto_dek *dek =
+		container_of(entry, typeof(*dek), entry);
+
+	claim_zero(mlx5_devx_cmd_destroy(dek->obj));
+	rte_free(dek);
+}
+
+
+int
+mlx5_crypto_dek_setup(struct mlx5_crypto_priv *priv)
+{
+	priv->dek_hlist = mlx5_hlist_create("dek_hlist",
+				 MLX5_CRYPTO_DEK_HTABLE_SZ,
+				 0, MLX5_HLIST_WRITE_MOST |
+				 MLX5_HLIST_DIRECT_KEY,
+				 mlx5_crypto_dek_create_cb,
+				 mlx5_crypto_dek_match_cb,
+				 mlx5_crypto_dek_remove_cb);
+	if (priv->dek_hlist == NULL)
+		return -1;
+	return 0;
+}
+
+void
+mlx5_crypto_dek_unset(struct mlx5_crypto_priv *priv)
+{
+	mlx5_hlist_destroy(priv->dek_hlist);
+	priv->dek_hlist = NULL;
+}
-- 
2.25.1

[dpdk-dev] [PATCH v4 03/14] crypto/mlx5: add session operations

[Matan Azrad]

From: Shiri Kuzin <shirik@nvidia.com>

Sessions are used in symmetric transformations in order to prepare
objects and data for packet processing stage.

A mlx5 session includes iv_offset, pointer to mlx5_crypto_dek struct,
bsf_size, bsf_p_type, block size index, encryption_order and encryption
standard.

Implement the next session operations:
        mlx5_crypto_sym_session_get_size- returns the size of the mlx5
	session struct.
	mlx5_crypto_sym_session_configure- prepares the DEK hash-list
	and saves all the session data.
	mlx5_crypto_sym_session_clear - destroys the DEK hash-list.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 120 +++++++++++++++++++++++++++++-
 1 file changed, 117 insertions(+), 3 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 2825f4192e..31d27245be 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -3,6 +3,7 @@
  */
 
 #include <rte_malloc.h>
+#include <rte_mempool.h>
 #include <rte_errno.h>
 #include <rte_log.h>
 #include <rte_pci.h>
@@ -36,6 +37,29 @@ static const struct rte_driver mlx5_drv = {
 
 static struct cryptodev_driver mlx5_cryptodev_driver;
 
+struct mlx5_crypto_session {
+	uint32_t bs_bpt_eo_es;
+	/*
+	 * bsf_size, bsf_p_type, encryption_order and encryption standard,
+	 * saved in big endian format.
+	 */
+	uint32_t bsp_res;
+	/*
+	 * crypto_block_size_pointer and reserved 24 bits saved in big endian
+	 * format.
+	 */
+	uint32_t iv_offset:16;
+	/* Starting point for Initialisation Vector. */
+	struct mlx5_crypto_dek *dek; /* Pointer to dek struct. */
+	uint32_t dek_id; /* DEK ID */
+} __rte_packed;
+
+static unsigned int
+mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
+{
+	return sizeof(struct mlx5_crypto_session);
+}
+
 static int
 mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
 		struct rte_cryptodev_config *config __rte_unused)
@@ -58,6 +82,96 @@ mlx5_crypto_dev_close(struct rte_cryptodev *dev)
 	return 0;
 }
 
+static int
+mlx5_crypto_sym_session_configure(struct rte_cryptodev *dev,
+				  struct rte_crypto_sym_xform *xform,
+				  struct rte_cryptodev_sym_session *session,
+				  struct rte_mempool *mp)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+	struct mlx5_crypto_session *sess_private_data;
+	struct rte_crypto_cipher_xform *cipher;
+	uint8_t encryption_order;
+	int ret;
+
+	if (unlikely(xform->next != NULL)) {
+		DRV_LOG(ERR, "Xform next is not supported.");
+		return -ENOTSUP;
+	}
+	if (unlikely((xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) ||
+		     (xform->cipher.algo != RTE_CRYPTO_CIPHER_AES_XTS))) {
+		DRV_LOG(ERR, "Only AES-XTS algorithm is supported.");
+		return -ENOTSUP;
+	}
+	ret = rte_mempool_get(mp, (void *)&sess_private_data);
+	if (ret != 0) {
+		DRV_LOG(ERR,
+			"Failed to get session %p private data from mempool.",
+			sess_private_data);
+		return -ENOMEM;
+	}
+	cipher = &xform->cipher;
+	sess_private_data->dek = mlx5_crypto_dek_prepare(priv, cipher);
+	if (sess_private_data->dek == NULL) {
+		rte_mempool_put(mp, sess_private_data);
+		DRV_LOG(ERR, "Failed to prepare dek.");
+		return -ENOMEM;
+	}
+	if (cipher->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
+		encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_MEMORY;
+	else
+		encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_WIRE;
+	sess_private_data->bs_bpt_eo_es = rte_cpu_to_be_32
+			(MLX5_BSF_SIZE_64B << MLX5_BSF_SIZE_OFFSET |
+			 MLX5_BSF_P_TYPE_CRYPTO << MLX5_BSF_P_TYPE_OFFSET |
+			 encryption_order << MLX5_ENCRYPTION_ORDER_OFFSET |
+			 MLX5_ENCRYPTION_STANDARD_AES_XTS);
+	switch (xform->cipher.dataunit_len) {
+	case 0:
+		sess_private_data->bsp_res = 0;
+		break;
+	case 512:
+		sess_private_data->bsp_res = rte_cpu_to_be_32
+					     ((uint32_t)MLX5_BLOCK_SIZE_512B <<
+					     MLX5_BLOCK_SIZE_OFFSET);
+		break;
+	case 4096:
+		sess_private_data->bsp_res = rte_cpu_to_be_32
+					     ((uint32_t)MLX5_BLOCK_SIZE_4096B <<
+					     MLX5_BLOCK_SIZE_OFFSET);
+		break;
+	default:
+		DRV_LOG(ERR, "Cipher data unit length is not supported.");
+		return -ENOTSUP;
+	}
+	sess_private_data->iv_offset = cipher->iv.offset;
+	sess_private_data->dek_id =
+			rte_cpu_to_be_32(sess_private_data->dek->obj->id &
+					 0xffffff);
+	set_sym_session_private_data(session, dev->driver_id,
+				     sess_private_data);
+	DRV_LOG(DEBUG, "Session %p was configured.", sess_private_data);
+	return 0;
+}
+
+static void
+mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev,
+			      struct rte_cryptodev_sym_session *sess)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+	struct mlx5_crypto_session *spriv = get_sym_session_private_data(sess,
+								dev->driver_id);
+
+	if (unlikely(spriv == NULL)) {
+		DRV_LOG(ERR, "Failed to get session %p private data.", spriv);
+		return;
+	}
+	mlx5_crypto_dek_destroy(priv, spriv->dek);
+	set_sym_session_private_data(sess, dev->driver_id, NULL);
+	rte_mempool_put(rte_mempool_from_obj(spriv), spriv);
+	DRV_LOG(DEBUG, "Session %p was cleared.", spriv);
+}
+
 static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_configure			= mlx5_crypto_dev_configure,
 	.dev_start			= NULL,
@@ -68,9 +182,9 @@ static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.stats_reset			= NULL,
 	.queue_pair_setup		= NULL,
 	.queue_pair_release		= NULL,
-	.sym_session_get_size		= NULL,
-	.sym_session_configure		= NULL,
-	.sym_session_clear		= NULL,
+	.sym_session_get_size		= mlx5_crypto_sym_session_get_size,
+	.sym_session_configure		= mlx5_crypto_sym_session_configure,
+	.sym_session_clear		= mlx5_crypto_sym_session_clear,
 	.sym_get_raw_dp_ctx_size	= NULL,
 	.sym_configure_raw_dp_ctx	= NULL,
 };
-- 
2.25.1

[dpdk-dev] [PATCH v4 04/14] crypto/mlx5: add basic operations

[Matan Azrad]

From: Shiri Kuzin <shirik@nvidia.com>

The basic dev control operations are configure, close and get info.

Extended the existing support of configure and close:
	-mlx5_crypto_dev_configure- function used to configure device.
	-mlx5_crypto_dev_close-  function used to close a configured
	 device.

Added support of get info function:
	-mlx5_crypto_dev_infos_get- function used to get specific
	 information of a device.

Added config struct to user private data with the fields socket id,
number of queue pairs and feature flags to be disabled.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 46 +++++++++++++++++++++++++++----
 drivers/crypto/mlx5/mlx5_crypto.h |  1 +
 2 files changed, 42 insertions(+), 5 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 31d27245be..c1ae012d15 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -19,6 +19,7 @@
 
 #define MLX5_CRYPTO_DRIVER_NAME mlx5_crypto
 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
+#define MLX5_CRYPTO_MAX_QPS 1024
 
 TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
 				TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
@@ -28,6 +29,9 @@ int mlx5_crypto_logtype;
 
 uint8_t mlx5_crypto_driver_id;
 
+const struct rte_cryptodev_capabilities
+		mlx5_crypto_caps[RTE_CRYPTO_OP_TYPE_UNDEFINED];
+
 static const char mlx5_crypto_drv_name[] = RTE_STR(MLX5_CRYPTO_DRIVER_NAME);
 
 static const struct rte_driver mlx5_drv = {
@@ -54,22 +58,47 @@ struct mlx5_crypto_session {
 	uint32_t dek_id; /* DEK ID */
 } __rte_packed;
 
-static unsigned int
-mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
+static void
+mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev,
+			  struct rte_cryptodev_info *dev_info)
 {
-	return sizeof(struct mlx5_crypto_session);
+	RTE_SET_USED(dev);
+	if (dev_info != NULL) {
+		dev_info->driver_id = mlx5_crypto_driver_id;
+		dev_info->feature_flags = 0;
+		dev_info->capabilities = mlx5_crypto_caps;
+		dev_info->max_nb_queue_pairs = MLX5_CRYPTO_MAX_QPS;
+		dev_info->min_mbuf_headroom_req = 0;
+		dev_info->min_mbuf_tailroom_req = 0;
+		dev_info->sym.max_nb_sessions = 0;
+		/*
+		 * If 0, the device does not have any limitation in number of
+		 * sessions that can be used.
+		 */
+	}
 }
 
 static int
 mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
-		struct rte_cryptodev_config *config __rte_unused)
+			  struct rte_cryptodev_config *config)
 {
 	struct mlx5_crypto_priv *priv = dev->data->dev_private;
 
+	if (config == NULL) {
+		DRV_LOG(ERR, "Invalid crypto dev configure parameters.");
+		return -EINVAL;
+	}
+	if ((config->ff_disable & RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO) != 0) {
+		DRV_LOG(ERR,
+			"Disabled symmetric crypto feature is not supported.");
+		return -ENOTSUP;
+	}
 	if (mlx5_crypto_dek_setup(priv) != 0) {
 		DRV_LOG(ERR, "Dek hash list creation has failed.");
 		return -ENOMEM;
 	}
+	priv->dev_config = *config;
+	DRV_LOG(DEBUG, "Device %u was configured.", dev->driver_id);
 	return 0;
 }
 
@@ -79,9 +108,16 @@ mlx5_crypto_dev_close(struct rte_cryptodev *dev)
 	struct mlx5_crypto_priv *priv = dev->data->dev_private;
 
 	mlx5_crypto_dek_unset(priv);
+	DRV_LOG(DEBUG, "Device %u was closed.", dev->driver_id);
 	return 0;
 }
 
+static unsigned int
+mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
+{
+	return sizeof(struct mlx5_crypto_session);
+}
+
 static int
 mlx5_crypto_sym_session_configure(struct rte_cryptodev *dev,
 				  struct rte_crypto_sym_xform *xform,
@@ -177,7 +213,7 @@ static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_start			= NULL,
 	.dev_stop			= NULL,
 	.dev_close			= mlx5_crypto_dev_close,
-	.dev_infos_get			= NULL,
+	.dev_infos_get			= mlx5_crypto_dev_infos_get,
 	.stats_get			= NULL,
 	.stats_reset			= NULL,
 	.queue_pair_setup		= NULL,
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 4ec67a7e0f..5e270d3d5a 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -24,6 +24,7 @@ struct mlx5_crypto_priv {
 	uint32_t pdn; /* Protection Domain number. */
 	struct ibv_pd *pd;
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
+	struct rte_cryptodev_config dev_config;
 };
 
 struct mlx5_crypto_dek {
-- 
2.25.1

[dpdk-dev] [PATCH v4 05/14] crypto/mlx5: add queue pairs operations

[Matan Azrad]

From: Shiri Kuzin <shirik@nvidia.com>

The HW queue pairs are a pair of send queue and receive queue of
independent work queues packed together in one object for the purpose
of transferring data between nodes of a network.

Completion Queue is a FIFO queue of completed work requests.

In crypto driver we use one QP in loopback in order to encrypt and
decrypt data locally without sending it to the wire.
In the configured QP we only use the SQ to perform the encryption and
decryption operations.

Added implementation for the QP setup function which creates the CQ,
creates the QP and changes its state to RTS (ready to send).

Added implementation for the release QP function to release all the QP
resources.

Added the ops structure that contains any operation which is supported
by the cryptodev.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 124 +++++++++++++++++++++++++++++-
 drivers/crypto/mlx5/mlx5_crypto.h |  11 +++
 2 files changed, 133 insertions(+), 2 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index c1ae012d15..bc14ba25b6 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -7,6 +7,7 @@
 #include <rte_errno.h>
 #include <rte_log.h>
 #include <rte_pci.h>
+#include <rte_memory.h>
 
 #include <mlx5_glue.h>
 #include <mlx5_common.h>
@@ -208,6 +209,125 @@ mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev,
 	DRV_LOG(DEBUG, "Session %p was cleared.", spriv);
 }
 
+static int
+mlx5_crypto_queue_pair_release(struct rte_cryptodev *dev, uint16_t qp_id)
+{
+	struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
+
+	if (qp->qp_obj != NULL)
+		claim_zero(mlx5_devx_cmd_destroy(qp->qp_obj));
+	if (qp->umem_obj != NULL)
+		claim_zero(mlx5_glue->devx_umem_dereg(qp->umem_obj));
+	if (qp->umem_buf != NULL)
+		rte_free(qp->umem_buf);
+	mlx5_devx_cq_destroy(&qp->cq_obj);
+	rte_free(qp);
+	dev->data->queue_pairs[qp_id] = NULL;
+	return 0;
+}
+
+static int
+mlx5_crypto_qp2rts(struct mlx5_crypto_qp *qp)
+{
+	/*
+	 * In Order to configure self loopback, when calling these functions the
+	 * remote QP id that is used is the id of the same QP.
+	 */
+	if (mlx5_devx_cmd_modify_qp_state(qp->qp_obj, MLX5_CMD_OP_RST2INIT_QP,
+					  qp->qp_obj->id)) {
+		DRV_LOG(ERR, "Failed to modify QP to INIT state(%u).",
+			rte_errno);
+		return -1;
+	}
+	if (mlx5_devx_cmd_modify_qp_state(qp->qp_obj, MLX5_CMD_OP_INIT2RTR_QP,
+					  qp->qp_obj->id)) {
+		DRV_LOG(ERR, "Failed to modify QP to RTR state(%u).",
+			rte_errno);
+		return -1;
+	}
+	if (mlx5_devx_cmd_modify_qp_state(qp->qp_obj, MLX5_CMD_OP_RTR2RTS_QP,
+					  qp->qp_obj->id)) {
+		DRV_LOG(ERR, "Failed to modify QP to RTS state(%u).",
+			rte_errno);
+		return -1;
+	}
+	return 0;
+}
+
+static int
+mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
+			     const struct rte_cryptodev_qp_conf *qp_conf,
+			     int socket_id)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+	struct mlx5_devx_qp_attr attr = {0};
+	struct mlx5_crypto_qp *qp;
+	uint16_t log_nb_desc = rte_log2_u32(qp_conf->nb_descriptors);
+	uint32_t umem_size = RTE_BIT32(log_nb_desc) *
+			      MLX5_CRYPTO_WQE_SET_SIZE +
+			      sizeof(*qp->db_rec) * 2;
+	uint32_t alloc_size = sizeof(*qp);
+	struct mlx5_devx_cq_attr cq_attr = {
+		.uar_page_id = mlx5_os_get_devx_uar_page_id(priv->uar),
+	};
+
+	alloc_size = RTE_ALIGN(alloc_size, RTE_CACHE_LINE_SIZE);
+	alloc_size += sizeof(struct rte_crypto_op *) * RTE_BIT32(log_nb_desc);
+	qp = rte_zmalloc_socket(__func__, alloc_size, RTE_CACHE_LINE_SIZE,
+				socket_id);
+	if (qp == NULL) {
+		DRV_LOG(ERR, "Failed to allocate QP memory.");
+		rte_errno = ENOMEM;
+		return -rte_errno;
+	}
+	if (mlx5_devx_cq_create(priv->ctx, &qp->cq_obj, log_nb_desc,
+				&cq_attr, socket_id) != 0) {
+		DRV_LOG(ERR, "Failed to create CQ.");
+		goto error;
+	}
+	qp->umem_buf = rte_zmalloc_socket(__func__, umem_size, 4096, socket_id);
+	if (qp->umem_buf == NULL) {
+		DRV_LOG(ERR, "Failed to allocate QP umem.");
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	qp->umem_obj = mlx5_glue->devx_umem_reg(priv->ctx,
+					       (void *)(uintptr_t)qp->umem_buf,
+					       umem_size,
+					       IBV_ACCESS_LOCAL_WRITE);
+	if (qp->umem_obj == NULL) {
+		DRV_LOG(ERR, "Failed to register QP umem.");
+		goto error;
+	}
+	attr.pd = priv->pdn;
+	attr.uar_index = mlx5_os_get_devx_uar_page_id(priv->uar);
+	attr.cqn = qp->cq_obj.cq->id;
+	attr.log_page_size = rte_log2_u32(sysconf(_SC_PAGESIZE));
+	attr.rq_size =  0;
+	attr.sq_size = RTE_BIT32(log_nb_desc);
+	attr.dbr_umem_valid = 1;
+	attr.wq_umem_id = qp->umem_obj->umem_id;
+	attr.wq_umem_offset = 0;
+	attr.dbr_umem_id = qp->umem_obj->umem_id;
+	attr.dbr_address = RTE_BIT64(log_nb_desc) *
+			   MLX5_CRYPTO_WQE_SET_SIZE;
+	qp->qp_obj = mlx5_devx_cmd_create_qp(priv->ctx, &attr);
+	if (qp->qp_obj == NULL) {
+		DRV_LOG(ERR, "Failed to create QP(%u).", rte_errno);
+		goto error;
+	}
+	qp->db_rec = RTE_PTR_ADD(qp->umem_buf, (uintptr_t)attr.dbr_address);
+	if (mlx5_crypto_qp2rts(qp))
+		goto error;
+	qp->ops = (struct rte_crypto_op **)RTE_ALIGN((uintptr_t)(qp + 1),
+							   RTE_CACHE_LINE_SIZE);
+	dev->data->queue_pairs[qp_id] = qp;
+	return 0;
+error:
+	mlx5_crypto_queue_pair_release(dev, qp_id);
+	return -1;
+}
+
 static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_configure			= mlx5_crypto_dev_configure,
 	.dev_start			= NULL,
@@ -216,8 +336,8 @@ static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_infos_get			= mlx5_crypto_dev_infos_get,
 	.stats_get			= NULL,
 	.stats_reset			= NULL,
-	.queue_pair_setup		= NULL,
-	.queue_pair_release		= NULL,
+	.queue_pair_setup		= mlx5_crypto_queue_pair_setup,
+	.queue_pair_release		= mlx5_crypto_queue_pair_release,
 	.sym_session_get_size		= mlx5_crypto_sym_session_get_size,
 	.sym_session_configure		= mlx5_crypto_sym_session_configure,
 	.sym_session_clear		= mlx5_crypto_sym_session_clear,
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 5e270d3d5a..f5313b89f2 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -11,9 +11,11 @@
 #include <rte_cryptodev_pmd.h>
 
 #include <mlx5_common_utils.h>
+#include <mlx5_common_devx.h>
 
 #define MLX5_CRYPTO_DEK_HTABLE_SZ (1 << 11)
 #define MLX5_CRYPTO_KEY_LENGTH 80
+#define MLX5_CRYPTO_WQE_SET_SIZE 1024
 
 struct mlx5_crypto_priv {
 	TAILQ_ENTRY(mlx5_crypto_priv) next;
@@ -27,6 +29,15 @@ struct mlx5_crypto_priv {
 	struct rte_cryptodev_config dev_config;
 };
 
+struct mlx5_crypto_qp {
+	struct mlx5_devx_cq cq_obj;
+	struct mlx5_devx_obj *qp_obj;
+	struct mlx5dv_devx_umem *umem_obj;
+	void *umem_buf;
+	volatile uint32_t *db_rec;
+	struct rte_crypto_op **ops;
+};
+
 struct mlx5_crypto_dek {
 	struct mlx5_hlist_entry entry; /* Pointer to DEK hash list entry. */
 	struct mlx5_devx_obj *obj; /* Pointer to DEK DevX object. */
-- 
2.25.1

[dpdk-dev] [PATCH v4 06/14] crypto/mlx5: add dev stop and start operations

[Matan Azrad]

From: Shiri Kuzin <shirik@nvidia.com>

Add the dev_start function that is used to start a configured device.
Add the dev_stop function that is used to stop a configured device.

Both functions set the dev parameter as used and return 0.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index bc14ba25b6..e3e2401a25 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -103,6 +103,19 @@ mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
 	return 0;
 }
 
+static void
+mlx5_crypto_dev_stop(struct rte_cryptodev *dev)
+{
+	RTE_SET_USED(dev);
+}
+
+static int
+mlx5_crypto_dev_start(struct rte_cryptodev *dev)
+{
+	RTE_SET_USED(dev);
+	return 0;
+}
+
 static int
 mlx5_crypto_dev_close(struct rte_cryptodev *dev)
 {
@@ -330,8 +343,8 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 
 static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_configure			= mlx5_crypto_dev_configure,
-	.dev_start			= NULL,
-	.dev_stop			= NULL,
+	.dev_start			= mlx5_crypto_dev_start,
+	.dev_stop			= mlx5_crypto_dev_stop,
 	.dev_close			= mlx5_crypto_dev_close,
 	.dev_infos_get			= mlx5_crypto_dev_infos_get,
 	.stats_get			= NULL,
-- 
2.25.1

[dpdk-dev] [PATCH v4 07/14] crypto/mlx5: add memory region management

[Matan Azrad]

From: Shiri Kuzin <shirik@nvidia.com>

Mellanox user space drivers don't deal with physical addresses as part
of a memory protection mechanism.
The device translates the given virtual address to a physical address
using the given memory key as an address space identifier.
That's why any mbuf virtual address is moved directly to the HW
descriptor(WQE).

The mapping between the virtual address to the physical address is saved
in MR configured by the kernel to the HW.

Each MR has a key that should also be moved to the WQE by the SW.

When the SW sees an unmapped address, it extends the address range and
creates a MR using a system call.

Add memory region cache management:
	- 2 level cache per queue-pair - no locks.
	- 1 shared cache between all the queues using a lock.

Using this way, the MR key search per data-path address is optimized.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 20 ++++++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto.h |  3 +++
 2 files changed, 23 insertions(+)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index e3e2401a25..d063c36e9b 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -233,6 +233,7 @@ mlx5_crypto_queue_pair_release(struct rte_cryptodev *dev, uint16_t qp_id)
 		claim_zero(mlx5_glue->devx_umem_dereg(qp->umem_obj));
 	if (qp->umem_buf != NULL)
 		rte_free(qp->umem_buf);
+	mlx5_mr_btree_free(&qp->mr_ctrl.cache_bh);
 	mlx5_devx_cq_destroy(&qp->cq_obj);
 	rte_free(qp);
 	dev->data->queue_pairs[qp_id] = NULL;
@@ -312,6 +313,13 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 		DRV_LOG(ERR, "Failed to register QP umem.");
 		goto error;
 	}
+	if (mlx5_mr_btree_init(&qp->mr_ctrl.cache_bh, MLX5_MR_BTREE_CACHE_N,
+			       priv->dev_config.socket_id) != 0) {
+		DRV_LOG(ERR, "Cannot allocate MR Btree for qp %u.",
+			(uint32_t)qp_id);
+		rte_errno = ENOMEM;
+		goto error;
+	}
 	attr.pd = priv->pdn;
 	attr.uar_index = mlx5_os_get_devx_uar_page_id(priv->uar);
 	attr.cqn = qp->cq_obj.cq->id;
@@ -500,6 +508,17 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 		claim_zero(mlx5_glue->close_device(priv->ctx));
 		return -1;
 	}
+	if (mlx5_mr_btree_init(&priv->mr_scache.cache,
+			     MLX5_MR_BTREE_CACHE_N * 2, rte_socket_id()) != 0) {
+		DRV_LOG(ERR, "Failed to allocate shared cache MR memory.");
+		mlx5_crypto_hw_global_release(priv);
+		rte_cryptodev_pmd_destroy(priv->crypto_dev);
+		claim_zero(mlx5_glue->close_device(priv->ctx));
+		rte_errno = ENOMEM;
+		return -rte_errno;
+	}
+	priv->mr_scache.reg_mr_cb = mlx5_common_verbs_reg_mr;
+	priv->mr_scache.dereg_mr_cb = mlx5_common_verbs_dereg_mr;
 	pthread_mutex_lock(&priv_list_lock);
 	TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next);
 	pthread_mutex_unlock(&priv_list_lock);
@@ -519,6 +538,7 @@ mlx5_crypto_pci_remove(struct rte_pci_device *pdev)
 		TAILQ_REMOVE(&mlx5_crypto_priv_list, priv, next);
 	pthread_mutex_unlock(&priv_list_lock);
 	if (priv) {
+		mlx5_mr_release_cache(&priv->mr_scache);
 		mlx5_crypto_hw_global_release(priv);
 		rte_cryptodev_pmd_destroy(priv->crypto_dev);
 		claim_zero(mlx5_glue->close_device(priv->ctx));
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index f5313b89f2..397267d249 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -12,6 +12,7 @@
 
 #include <mlx5_common_utils.h>
 #include <mlx5_common_devx.h>
+#include <mlx5_common_mr.h>
 
 #define MLX5_CRYPTO_DEK_HTABLE_SZ (1 << 11)
 #define MLX5_CRYPTO_KEY_LENGTH 80
@@ -27,6 +28,7 @@ struct mlx5_crypto_priv {
 	struct ibv_pd *pd;
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
 	struct rte_cryptodev_config dev_config;
+	struct mlx5_mr_share_cache mr_scache; /* Global shared MR cache. */
 };
 
 struct mlx5_crypto_qp {
@@ -36,6 +38,7 @@ struct mlx5_crypto_qp {
 	void *umem_buf;
 	volatile uint32_t *db_rec;
 	struct rte_crypto_op **ops;
+	struct mlx5_mr_ctrl mr_ctrl;
 };
 
 struct mlx5_crypto_dek {
-- 
2.25.1

[dpdk-dev] [PATCH v4 08/14] crypto/mlx5: create login object using DevX

[Matan Azrad]

From: Shiri Kuzin <shirik@nvidia.com>

To work with crypto engines that are marked with wrapped_import_method,
a login session is required.
A crypto login object needs to be created using DevX.

The crypto login object contains:
	- The credential pointer.
	- The import_KEK pointer to be used for all secured information
	  communicated in crypto commands (key fields), including the
	  provided credential in this command.
	- The credential secret, wrapped by the import_KEK indicated in
	  this command. Size includes 8 bytes IV for wrapping.

Added devargs for the required login values:
	- wcs_file - path to the file containing the credential.
	- import_kek_id - the import KEK pointer.
	- credential_id - the credential pointer.

Create the login DevX object in pci_probe function and destroy it in
pci_remove.
Destroying the crypto login object means logout.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 103 ++++++++++++++++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto.h |   7 ++
 2 files changed, 110 insertions(+)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index d063c36e9b..acf6c514d3 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -426,6 +426,101 @@ mlx5_crypto_hw_global_prepare(struct mlx5_crypto_priv *priv)
 	return 0;
 }
 
+
+static int
+mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque)
+{
+	struct mlx5_crypto_devarg_params *devarg_prms = opaque;
+	struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr;
+	unsigned long tmp;
+	FILE *file;
+	int ret;
+	int i;
+
+	if (strcmp(key, "class") == 0)
+		return 0;
+	if (strcmp(key, "wcs_file") == 0) {
+		file = fopen(val, "rb");
+		if (file == NULL) {
+			rte_errno = ENOTSUP;
+			return -rte_errno;
+		}
+		for (i = 0 ; i < MLX5_CRYPTO_CREDENTIAL_SIZE ; i++) {
+			ret = fscanf(file, "%02hhX", &attr->credential[i]);
+			if (ret <= 0) {
+				fclose(file);
+				DRV_LOG(ERR,
+					"Failed to read credential from file.");
+				rte_errno = EINVAL;
+				return -rte_errno;
+			}
+		}
+		fclose(file);
+		devarg_prms->login_devarg = true;
+		return 0;
+	}
+	errno = 0;
+	tmp = strtoul(val, NULL, 0);
+	if (errno) {
+		DRV_LOG(WARNING, "%s: \"%s\" is an invalid integer.", key, val);
+		return -errno;
+	}
+	if (strcmp(key, "import_kek_id") == 0)
+		attr->session_import_kek_ptr = (uint32_t)tmp;
+	else if (strcmp(key, "credential_id") == 0)
+		attr->credential_pointer = (uint32_t)tmp;
+	else
+		DRV_LOG(WARNING, "Invalid key %s.", key);
+	return 0;
+}
+
+static struct mlx5_devx_obj *
+mlx5_crypto_config_login(struct rte_devargs *devargs,
+			 struct ibv_context *ctx)
+{
+	/*
+	 * Set credential pointer and session import KEK pointer to a default
+	 * value of 0.
+	 */
+	struct mlx5_crypto_devarg_params login = {
+			.login_devarg = false,
+			.login_attr = {
+					.credential_pointer = 0,
+					.session_import_kek_ptr = 0,
+			}
+	};
+	struct rte_kvargs *kvlist;
+
+	if (devargs == NULL) {
+		DRV_LOG(ERR,
+	"No login devargs in order to enable crypto operations in the device.");
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	kvlist = rte_kvargs_parse(devargs->args, NULL);
+	if (kvlist == NULL) {
+		DRV_LOG(ERR, "Failed to parse devargs.");
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	if (rte_kvargs_process(kvlist, NULL, mlx5_crypto_args_check_handler,
+			   &login) != 0) {
+		DRV_LOG(ERR, "Devargs handler function Failed.");
+		rte_kvargs_free(kvlist);
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	rte_kvargs_free(kvlist);
+	if (login.login_devarg == false) {
+		DRV_LOG(ERR,
+	"No login credential devarg in order to enable crypto operations "
+	"in the device.");
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	return mlx5_devx_cmd_create_crypto_login_obj(ctx, &login.login_attr);
+}
+
 /**
  * DPDK callback to register a PCI device.
  *
@@ -447,6 +542,7 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	struct ibv_device *ibv;
 	struct rte_cryptodev *crypto_dev;
 	struct ibv_context *ctx;
+	struct mlx5_devx_obj *login;
 	struct mlx5_crypto_priv *priv;
 	struct mlx5_hca_attr attr = { 0 };
 	struct rte_cryptodev_pmd_init_params init_params = {
@@ -485,6 +581,11 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 		rte_errno = ENOTSUP;
 		return -ENOTSUP;
 	}
+	login = mlx5_crypto_config_login(pci_dev->device.devargs, ctx);
+	if (login == NULL) {
+		DRV_LOG(ERR, "Failed to configure login.");
+		return -rte_errno;
+	}
 	crypto_dev = rte_cryptodev_pmd_create(ibv->name, &pci_dev->device,
 					&init_params);
 	if (crypto_dev == NULL) {
@@ -501,6 +602,7 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	crypto_dev->driver_id = mlx5_crypto_driver_id;
 	priv = crypto_dev->data->dev_private;
 	priv->ctx = ctx;
+	priv->login_obj = login;
 	priv->pci_dev = pci_dev;
 	priv->crypto_dev = crypto_dev;
 	if (mlx5_crypto_hw_global_prepare(priv) != 0) {
@@ -541,6 +643,7 @@ mlx5_crypto_pci_remove(struct rte_pci_device *pdev)
 		mlx5_mr_release_cache(&priv->mr_scache);
 		mlx5_crypto_hw_global_release(priv);
 		rte_cryptodev_pmd_destroy(priv->crypto_dev);
+		claim_zero(mlx5_devx_cmd_destroy(priv->login_obj));
 		claim_zero(mlx5_glue->close_device(priv->ctx));
 	}
 	return 0;
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 397267d249..0aef804b92 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -29,6 +29,7 @@ struct mlx5_crypto_priv {
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
 	struct rte_cryptodev_config dev_config;
 	struct mlx5_mr_share_cache mr_scache; /* Global shared MR cache. */
+	struct mlx5_devx_obj *login_obj;
 };
 
 struct mlx5_crypto_qp {
@@ -48,6 +49,12 @@ struct mlx5_crypto_dek {
 	bool size_is_48; /* Whether the key\data size is 48 bytes or not. */
 };
 
+
+struct mlx5_crypto_devarg_params {
+	bool login_devarg;
+	struct mlx5_devx_crypto_login_attr login_attr;
+};
+
 int
 mlx5_crypto_dek_destroy(struct mlx5_crypto_priv *priv,
 			struct mlx5_crypto_dek *dek);
-- 
2.25.1

[dpdk-dev] [PATCH v4 09/14] crypto/mlx5: add keytag devarg

[Matan Azrad]

From: Suanming Mou <suanmingm@nvidia.com>

A keytag is a piece of data encrypted together with a DEK.

When a DEK is referenced by an MKEY.bsf through its index, the keytag is
also supplied in the BSF as plaintext. The HW will decrypt the DEK (and
the attached keytag) and will fail the operation if the keytags don't
match.

This commit adds the configuration of the keytag with devargs.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 50 +++++++++++++++++--------------
 drivers/crypto/mlx5/mlx5_crypto.h |  3 +-
 2 files changed, 30 insertions(+), 23 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index acf6c514d3..d4379d2b05 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -469,56 +469,52 @@ mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque)
 		attr->session_import_kek_ptr = (uint32_t)tmp;
 	else if (strcmp(key, "credential_id") == 0)
 		attr->credential_pointer = (uint32_t)tmp;
+	else if (strcmp(key, "keytag") == 0)
+		devarg_prms->keytag = tmp;
 	else
 		DRV_LOG(WARNING, "Invalid key %s.", key);
 	return 0;
 }
 
-static struct mlx5_devx_obj *
-mlx5_crypto_config_login(struct rte_devargs *devargs,
-			 struct ibv_context *ctx)
+static int
+mlx5_crypto_parse_devargs(struct rte_devargs *devargs,
+			  struct mlx5_crypto_devarg_params *devarg_prms)
 {
-	/*
-	 * Set credential pointer and session import KEK pointer to a default
-	 * value of 0.
-	 */
-	struct mlx5_crypto_devarg_params login = {
-			.login_devarg = false,
-			.login_attr = {
-					.credential_pointer = 0,
-					.session_import_kek_ptr = 0,
-			}
-	};
+	struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr;
 	struct rte_kvargs *kvlist;
 
+	/* Default values. */
+	attr->credential_pointer = 0;
+	attr->session_import_kek_ptr = 0;
+	devarg_prms->keytag = 0;
 	if (devargs == NULL) {
 		DRV_LOG(ERR,
 	"No login devargs in order to enable crypto operations in the device.");
 		rte_errno = EINVAL;
-		return NULL;
+		return -1;
 	}
 	kvlist = rte_kvargs_parse(devargs->args, NULL);
 	if (kvlist == NULL) {
 		DRV_LOG(ERR, "Failed to parse devargs.");
 		rte_errno = EINVAL;
-		return NULL;
+		return -1;
 	}
 	if (rte_kvargs_process(kvlist, NULL, mlx5_crypto_args_check_handler,
-			   &login) != 0) {
+			   devarg_prms) != 0) {
 		DRV_LOG(ERR, "Devargs handler function Failed.");
 		rte_kvargs_free(kvlist);
 		rte_errno = EINVAL;
-		return NULL;
+		return -1;
 	}
 	rte_kvargs_free(kvlist);
-	if (login.login_devarg == false) {
+	if (devarg_prms->login_devarg == false) {
 		DRV_LOG(ERR,
 	"No login credential devarg in order to enable crypto operations "
 	"in the device.");
 		rte_errno = EINVAL;
-		return NULL;
+		return -1;
 	}
-	return mlx5_devx_cmd_create_crypto_login_obj(ctx, &login.login_attr);
+	return 0;
 }
 
 /**
@@ -544,6 +540,7 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	struct ibv_context *ctx;
 	struct mlx5_devx_obj *login;
 	struct mlx5_crypto_priv *priv;
+	struct mlx5_crypto_devarg_params devarg_prms = { 0 };
 	struct mlx5_hca_attr attr = { 0 };
 	struct rte_cryptodev_pmd_init_params init_params = {
 		.name = "",
@@ -552,6 +549,8 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 		.max_nb_queue_pairs =
 				RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
 	};
+	int ret;
+
 	RTE_SET_USED(pci_drv);
 	if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
 		DRV_LOG(ERR, "Non-primary process type is not supported.");
@@ -581,7 +580,13 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 		rte_errno = ENOTSUP;
 		return -ENOTSUP;
 	}
-	login = mlx5_crypto_config_login(pci_dev->device.devargs, ctx);
+	ret = mlx5_crypto_parse_devargs(pci_dev->device.devargs, &devarg_prms);
+	if (ret) {
+		DRV_LOG(ERR, "Failed to parse devargs.");
+		return -rte_errno;
+	}
+	login = mlx5_devx_cmd_create_crypto_login_obj(ctx,
+						      &devarg_prms.login_attr);
 	if (login == NULL) {
 		DRV_LOG(ERR, "Failed to configure login.");
 		return -rte_errno;
@@ -621,6 +626,7 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	}
 	priv->mr_scache.reg_mr_cb = mlx5_common_verbs_reg_mr;
 	priv->mr_scache.dereg_mr_cb = mlx5_common_verbs_dereg_mr;
+	priv->keytag = rte_cpu_to_be_64(devarg_prms.keytag);
 	pthread_mutex_lock(&priv_list_lock);
 	TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next);
 	pthread_mutex_unlock(&priv_list_lock);
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 0aef804b92..34c65f9a24 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -30,6 +30,7 @@ struct mlx5_crypto_priv {
 	struct rte_cryptodev_config dev_config;
 	struct mlx5_mr_share_cache mr_scache; /* Global shared MR cache. */
 	struct mlx5_devx_obj *login_obj;
+	uint64_t keytag;
 };
 
 struct mlx5_crypto_qp {
@@ -49,10 +50,10 @@ struct mlx5_crypto_dek {
 	bool size_is_48; /* Whether the key\data size is 48 bytes or not. */
 };
 
-
 struct mlx5_crypto_devarg_params {
 	bool login_devarg;
 	struct mlx5_devx_crypto_login_attr login_attr;
+	uint64_t keytag;
 };
 
 int
-- 
2.25.1

[dpdk-dev] [PATCH v4 10/14] crypto/mlx5: add maximum segments devarg

[Matan Azrad]

From: Suanming Mou <suanmingm@nvidia.com>

The mlx5 HW crypto operations are done by attaching crypto property
to a memory region. Once done, every access to the memory via the
crypto-enabled memory region will result with in-line encryption or
decryption of the data.

As a result, the design choice is to provide two types of WQEs. One
is UMR WQE which sets the crypto property and the other is rdma write
WQE which sends DMA command to copy data from local MR to remote MR.

The size of the WQEs will be defined by a new devarg called
max_segs_num.

This devarg also defines the maximum segments in mbuf chain that will be
supported for crypto operations.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 35 +++++++++++++++++++++++++++----
 drivers/crypto/mlx5/mlx5_crypto.h |  7 +++++++
 2 files changed, 38 insertions(+), 4 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index d4379d2b05..1a7317aef5 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -21,6 +21,7 @@
 #define MLX5_CRYPTO_DRIVER_NAME mlx5_crypto
 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
 #define MLX5_CRYPTO_MAX_QPS 1024
+#define MLX5_CRYPTO_MAX_SEGS 56
 
 TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
 				TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
@@ -465,14 +466,24 @@ mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque)
 		DRV_LOG(WARNING, "%s: \"%s\" is an invalid integer.", key, val);
 		return -errno;
 	}
-	if (strcmp(key, "import_kek_id") == 0)
+	if (strcmp(key, "max_segs_num") == 0) {
+		if (!tmp || tmp > MLX5_CRYPTO_MAX_SEGS) {
+			DRV_LOG(WARNING, "Invalid max_segs_num: %d, should"
+				" be less than %d.",
+				(uint32_t)tmp, MLX5_CRYPTO_MAX_SEGS);
+			rte_errno = EINVAL;
+			return -rte_errno;
+		}
+		devarg_prms->max_segs_num = (uint32_t)tmp;
+	} else if (strcmp(key, "import_kek_id") == 0) {
 		attr->session_import_kek_ptr = (uint32_t)tmp;
-	else if (strcmp(key, "credential_id") == 0)
+	} else if (strcmp(key, "credential_id") == 0) {
 		attr->credential_pointer = (uint32_t)tmp;
-	else if (strcmp(key, "keytag") == 0)
+	} else if (strcmp(key, "keytag") == 0) {
 		devarg_prms->keytag = tmp;
-	else
+	} else {
 		DRV_LOG(WARNING, "Invalid key %s.", key);
+	}
 	return 0;
 }
 
@@ -487,6 +498,7 @@ mlx5_crypto_parse_devargs(struct rte_devargs *devargs,
 	attr->credential_pointer = 0;
 	attr->session_import_kek_ptr = 0;
 	devarg_prms->keytag = 0;
+	devarg_prms->max_segs_num = 8;
 	if (devargs == NULL) {
 		DRV_LOG(ERR,
 	"No login devargs in order to enable crypto operations in the device.");
@@ -627,6 +639,21 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	priv->mr_scache.reg_mr_cb = mlx5_common_verbs_reg_mr;
 	priv->mr_scache.dereg_mr_cb = mlx5_common_verbs_dereg_mr;
 	priv->keytag = rte_cpu_to_be_64(devarg_prms.keytag);
+	priv->max_segs_num = devarg_prms.max_segs_num;
+	priv->umr_wqe_size = sizeof(struct mlx5_wqe_umr_bsf_seg) +
+			     sizeof(struct mlx5_umr_wqe) +
+			     RTE_ALIGN(priv->max_segs_num, 4) *
+			     sizeof(struct mlx5_wqe_dseg);
+	priv->rdmw_wqe_size = sizeof(struct mlx5_rdma_write_wqe) +
+			      sizeof(struct mlx5_wqe_dseg) *
+			      (priv->max_segs_num <= 2 ? 2 : 2 +
+			       RTE_ALIGN(priv->max_segs_num - 2, 4));
+	priv->wqe_set_size = priv->umr_wqe_size + priv->rdmw_wqe_size;
+	priv->wqe_stride = (priv->umr_wqe_size + priv->rdmw_wqe_size) /
+							       MLX5_SEND_WQE_BB;
+	priv->max_rdmaw_klm_n = (priv->rdmw_wqe_size -
+				 sizeof(struct mlx5_rdma_write_wqe)) /
+				 sizeof(struct mlx5_wqe_dseg);
 	pthread_mutex_lock(&priv_list_lock);
 	TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next);
 	pthread_mutex_unlock(&priv_list_lock);
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 34c65f9a24..81452bd700 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -25,12 +25,18 @@ struct mlx5_crypto_priv {
 	struct rte_cryptodev *crypto_dev;
 	void *uar; /* User Access Region. */
 	uint32_t pdn; /* Protection Domain number. */
+	uint32_t max_segs_num; /* Maximum supported data segs. */
 	struct ibv_pd *pd;
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
 	struct rte_cryptodev_config dev_config;
 	struct mlx5_mr_share_cache mr_scache; /* Global shared MR cache. */
 	struct mlx5_devx_obj *login_obj;
 	uint64_t keytag;
+	uint16_t wqe_set_size;
+	uint16_t umr_wqe_size;
+	uint16_t rdmw_wqe_size;
+	uint16_t wqe_stride;
+	uint16_t max_rdmaw_klm_n;
 };
 
 struct mlx5_crypto_qp {
@@ -54,6 +60,7 @@ struct mlx5_crypto_devarg_params {
 	bool login_devarg;
 	struct mlx5_devx_crypto_login_attr login_attr;
 	uint64_t keytag;
+	uint32_t max_segs_num;
 };
 
 int
-- 
2.25.1

[dpdk-dev] [PATCH v4 11/14] crypto/mlx5: add WQE set initialization

[Matan Azrad]

From: Suanming Mou <suanmingm@nvidia.com>

Currently, HW handles the WQEs much faster than the software,
Using the constant WQE set layout can initialize most of the WQE
segments in advanced, and software only needs to configure very
limited segments in datapath. This accelerates the software WQE
organize in datapath.

This commit initializes the fixed WQE set segments.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 83 +++++++++++++++++++++++++++++--
 drivers/crypto/mlx5/mlx5_crypto.h | 10 +++-
 2 files changed, 87 insertions(+), 6 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 1a7317aef5..913276a4f3 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -269,6 +269,69 @@ mlx5_crypto_qp2rts(struct mlx5_crypto_qp *qp)
 	return 0;
 }
 
+static void
+mlx5_crypto_qp_init(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp)
+{
+	uint32_t i;
+
+	for (i = 0 ; i < qp->entries_n; i++) {
+		struct mlx5_wqe_cseg *cseg = RTE_PTR_ADD(qp->umem_buf, i *
+							 priv->wqe_set_size);
+		struct mlx5_wqe_umr_cseg *ucseg = (struct mlx5_wqe_umr_cseg *)
+								     (cseg + 1);
+		struct mlx5_wqe_umr_bsf_seg *bsf =
+			(struct mlx5_wqe_umr_bsf_seg *)(RTE_PTR_ADD(cseg,
+						       priv->umr_wqe_size)) - 1;
+		struct mlx5_wqe_rseg *rseg;
+
+		/* Init UMR WQE. */
+		cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj->id << 8) |
+					 (priv->umr_wqe_size / MLX5_WSEG_SIZE));
+		cseg->flags = RTE_BE32(MLX5_COMP_ONLY_FIRST_ERR <<
+				       MLX5_COMP_MODE_OFFSET);
+		cseg->misc = rte_cpu_to_be_32(qp->mkey[i]->id);
+		ucseg->if_cf_toe_cq_res = RTE_BE32(1u << MLX5_UMRC_IF_OFFSET);
+		ucseg->mkey_mask = RTE_BE64(1u << 0); /* Mkey length bit. */
+		ucseg->ko_to_bs = rte_cpu_to_be_32
+			((RTE_ALIGN(priv->max_segs_num, 4u) <<
+			 MLX5_UMRC_KO_OFFSET) | (4 << MLX5_UMRC_TO_BS_OFFSET));
+		bsf->keytag = priv->keytag;
+		/* Init RDMA WRITE WQE. */
+		cseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size);
+		cseg->flags = RTE_BE32((MLX5_COMP_ALWAYS <<
+				      MLX5_COMP_MODE_OFFSET) |
+				      MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE);
+		rseg = (struct mlx5_wqe_rseg *)(cseg + 1);
+		rseg->rkey = rte_cpu_to_be_32(qp->mkey[i]->id);
+	}
+}
+
+static int
+mlx5_crypto_indirect_mkeys_prepare(struct mlx5_crypto_priv *priv,
+				  struct mlx5_crypto_qp *qp)
+{
+	struct mlx5_umr_wqe *umr;
+	uint32_t i;
+	struct mlx5_devx_mkey_attr attr = {
+		.pd = priv->pdn,
+		.umr_en = 1,
+		.crypto_en = 1,
+		.set_remote_rw = 1,
+		.klm_num = RTE_ALIGN(priv->max_segs_num, 4),
+	};
+
+	for (umr = (struct mlx5_umr_wqe *)qp->umem_buf, i = 0;
+	   i < qp->entries_n; i++, umr = RTE_PTR_ADD(umr, priv->wqe_set_size)) {
+		attr.klm_array = (struct mlx5_klm *)&umr->kseg[0];
+		qp->mkey[i] = mlx5_devx_cmd_mkey_create(priv->ctx, &attr);
+		if (!qp->mkey[i]) {
+			DRV_LOG(ERR, "Failed to allocate indirect mkey.");
+			return -1;
+		}
+	}
+	return 0;
+}
+
 static int
 mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 			     const struct rte_cryptodev_qp_conf *qp_conf,
@@ -279,7 +342,7 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	struct mlx5_crypto_qp *qp;
 	uint16_t log_nb_desc = rte_log2_u32(qp_conf->nb_descriptors);
 	uint32_t umem_size = RTE_BIT32(log_nb_desc) *
-			      MLX5_CRYPTO_WQE_SET_SIZE +
+			      priv->wqe_set_size +
 			      sizeof(*qp->db_rec) * 2;
 	uint32_t alloc_size = sizeof(*qp);
 	struct mlx5_devx_cq_attr cq_attr = {
@@ -287,7 +350,9 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	};
 
 	alloc_size = RTE_ALIGN(alloc_size, RTE_CACHE_LINE_SIZE);
-	alloc_size += sizeof(struct rte_crypto_op *) * RTE_BIT32(log_nb_desc);
+	alloc_size += (sizeof(struct rte_crypto_op *) +
+		       sizeof(struct mlx5_devx_obj *)) *
+		       RTE_BIT32(log_nb_desc);
 	qp = rte_zmalloc_socket(__func__, alloc_size, RTE_CACHE_LINE_SIZE,
 				socket_id);
 	if (qp == NULL) {
@@ -331,8 +396,7 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	attr.wq_umem_id = qp->umem_obj->umem_id;
 	attr.wq_umem_offset = 0;
 	attr.dbr_umem_id = qp->umem_obj->umem_id;
-	attr.dbr_address = RTE_BIT64(log_nb_desc) *
-			   MLX5_CRYPTO_WQE_SET_SIZE;
+	attr.dbr_address = RTE_BIT64(log_nb_desc) * priv->wqe_set_size;
 	qp->qp_obj = mlx5_devx_cmd_create_qp(priv->ctx, &attr);
 	if (qp->qp_obj == NULL) {
 		DRV_LOG(ERR, "Failed to create QP(%u).", rte_errno);
@@ -341,8 +405,17 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	qp->db_rec = RTE_PTR_ADD(qp->umem_buf, (uintptr_t)attr.dbr_address);
 	if (mlx5_crypto_qp2rts(qp))
 		goto error;
-	qp->ops = (struct rte_crypto_op **)RTE_ALIGN((uintptr_t)(qp + 1),
+	qp->mkey = (struct mlx5_devx_obj **)RTE_ALIGN((uintptr_t)(qp + 1),
 							   RTE_CACHE_LINE_SIZE);
+	qp->ops = (struct rte_crypto_op **)(qp->mkey + RTE_BIT32(log_nb_desc));
+	qp->entries_n = 1 << log_nb_desc;
+	if (mlx5_crypto_indirect_mkeys_prepare(priv, qp)) {
+		DRV_LOG(ERR, "Cannot allocate indirect memory regions.");
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	mlx5_crypto_qp_init(priv, qp);
+	qp->priv = priv;
 	dev->data->queue_pairs[qp_id] = qp;
 	return 0;
 error:
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 81452bd700..52fcf5217f 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -16,7 +16,6 @@
 
 #define MLX5_CRYPTO_DEK_HTABLE_SZ (1 << 11)
 #define MLX5_CRYPTO_KEY_LENGTH 80
-#define MLX5_CRYPTO_WQE_SET_SIZE 1024
 
 struct mlx5_crypto_priv {
 	TAILQ_ENTRY(mlx5_crypto_priv) next;
@@ -24,6 +23,7 @@ struct mlx5_crypto_priv {
 	struct rte_pci_device *pci_dev;
 	struct rte_cryptodev *crypto_dev;
 	void *uar; /* User Access Region. */
+	volatile uint64_t *uar_addr;
 	uint32_t pdn; /* Protection Domain number. */
 	uint32_t max_segs_num; /* Maximum supported data segs. */
 	struct ibv_pd *pd;
@@ -40,13 +40,21 @@ struct mlx5_crypto_priv {
 };
 
 struct mlx5_crypto_qp {
+	struct mlx5_crypto_priv *priv;
 	struct mlx5_devx_cq cq_obj;
 	struct mlx5_devx_obj *qp_obj;
+	struct rte_cryptodev_stats stats;
 	struct mlx5dv_devx_umem *umem_obj;
 	void *umem_buf;
 	volatile uint32_t *db_rec;
 	struct rte_crypto_op **ops;
+	struct mlx5_devx_obj **mkey; /* WQE's indirect mekys. */
 	struct mlx5_mr_ctrl mr_ctrl;
+	uint8_t *wqe;
+	uint16_t entries_n;
+	uint16_t pi;
+	uint16_t ci;
+	uint16_t db_pi;
 };
 
 struct mlx5_crypto_dek {
-- 
2.25.1

[dpdk-dev] [PATCH v4 12/14] crypto/mlx5: add enqueue and dequeue operations

[Matan Azrad]

From: Suanming Mou <suanmingm@nvidia.com>

The crypto operations are done with the WQE set which contains
one UMR WQE and one rdma write WQE. Most segments of the WQE
set are initialized properly during queue setup, only limited
segments are initialized according to the crypto detail in the
datapath process.

This commit adds the enquue and dequeue operations and updates
the WQE set segments accordingly.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 242 +++++++++++++++++++++++++++++-
 drivers/crypto/mlx5/mlx5_crypto.h |   3 +
 2 files changed, 241 insertions(+), 4 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 913276a4f3..29e59de702 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -269,6 +269,239 @@ mlx5_crypto_qp2rts(struct mlx5_crypto_qp *qp)
 	return 0;
 }
 
+static __rte_noinline uint32_t
+mlx5_crypto_get_block_size(struct rte_crypto_op *op)
+{
+	uint32_t bl = op->sym->cipher.data.length;
+
+	switch (bl) {
+	case (1 << 20):
+		return RTE_BE32(MLX5_BLOCK_SIZE_1MB << MLX5_BLOCK_SIZE_OFFSET);
+	case (1 << 12):
+		return RTE_BE32(MLX5_BLOCK_SIZE_4096B <<
+				MLX5_BLOCK_SIZE_OFFSET);
+	case (1 << 9):
+		return RTE_BE32(MLX5_BLOCK_SIZE_512B << MLX5_BLOCK_SIZE_OFFSET);
+	default:
+		DRV_LOG(ERR, "Unknown block size: %u.", bl);
+		return UINT32_MAX;
+	}
+}
+
+static __rte_always_inline uint32_t
+mlx5_crypto_klm_set(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp,
+		      struct rte_mbuf *mbuf, struct mlx5_wqe_dseg *klm,
+		      uint32_t offset, uint32_t *remain)
+{
+	uint32_t data_len = (rte_pktmbuf_data_len(mbuf) - offset);
+	uintptr_t addr = rte_pktmbuf_mtod_offset(mbuf, uintptr_t, offset);
+
+	if (data_len > *remain)
+		data_len = *remain;
+	*remain -= data_len;
+	klm->bcount = rte_cpu_to_be_32(data_len);
+	klm->pbuf = rte_cpu_to_be_64(addr);
+	klm->lkey = mlx5_mr_addr2mr_bh(priv->pd, 0,
+		&priv->mr_scache, &qp->mr_ctrl, addr,
+		!!(mbuf->ol_flags & EXT_ATTACHED_MBUF));
+	return klm->lkey;
+
+}
+
+static __rte_always_inline uint32_t
+mlx5_crypto_klms_set(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp,
+		     struct rte_crypto_op *op, struct rte_mbuf *mbuf,
+		     struct mlx5_wqe_dseg *klm)
+{
+	uint32_t remain_len = op->sym->cipher.data.length;
+	uint32_t nb_segs = mbuf->nb_segs;
+	uint32_t klm_n = 1;
+
+	/* First mbuf needs to take the cipher offset. */
+	if (unlikely(mlx5_crypto_klm_set(priv, qp, mbuf, klm,
+		     op->sym->cipher.data.offset, &remain_len) == UINT32_MAX)) {
+		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return 0;
+	}
+	while (remain_len) {
+		nb_segs--;
+		mbuf = mbuf->next;
+		if (unlikely(mbuf == NULL || nb_segs == 0)) {
+			op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+			return 0;
+		}
+		if (unlikely(mlx5_crypto_klm_set(priv, qp, mbuf, klm, 0,
+						 &remain_len) == UINT32_MAX)) {
+			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			return 0;
+		}
+		klm_n++;
+	}
+	return klm_n;
+}
+
+static __rte_always_inline int
+mlx5_crypto_wqe_set(struct mlx5_crypto_priv *priv,
+			 struct mlx5_crypto_qp *qp,
+			 struct rte_crypto_op *op,
+			 struct mlx5_umr_wqe *umr)
+{
+	struct mlx5_crypto_session *sess = get_sym_session_private_data
+				(op->sym->session, mlx5_crypto_driver_id);
+	struct mlx5_wqe_cseg *cseg = &umr->ctr;
+	struct mlx5_wqe_mkey_cseg *mkc = &umr->mkc;
+	struct mlx5_wqe_dseg *klms = &umr->kseg[0];
+	struct mlx5_wqe_umr_bsf_seg *bsf = ((struct mlx5_wqe_umr_bsf_seg *)
+				      RTE_PTR_ADD(umr, priv->umr_wqe_size)) - 1;
+	uint16_t nop_ds;
+	/* Set UMR WQE. */
+	uint32_t klm_n = mlx5_crypto_klms_set(priv, qp, op,
+			op->sym->m_dst ? op->sym->m_dst : op->sym->m_src, klms);
+
+	if (unlikely(klm_n == 0))
+		return 0;
+	bsf->bs_bpt_eo_es = sess->bs_bpt_eo_es;
+	if (unlikely(!sess->bsp_res)) {
+		bsf->bsp_res = mlx5_crypto_get_block_size(op);
+		if (unlikely(bsf->bsp_res == UINT32_MAX)) {
+			op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+			return 0;
+		}
+	} else {
+		bsf->bsp_res = sess->bsp_res;
+	}
+	bsf->raw_data_size = rte_cpu_to_be_32(op->sym->cipher.data.length);
+	memcpy(bsf->xts_initial_tweak,
+	       rte_crypto_op_ctod_offset(op, uint8_t *, sess->iv_offset), 16);
+	bsf->res_dp = sess->dek_id;
+	cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) | MLX5_OPCODE_UMR);
+	mkc->len = rte_cpu_to_be_64(op->sym->cipher.data.length);
+	/* Set RDMA_WRITE WQE. */
+	cseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size);
+	klms = RTE_PTR_ADD(cseg, sizeof(struct mlx5_rdma_write_wqe));
+	cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) |
+					MLX5_OPCODE_RDMA_WRITE);
+	if (op->sym->m_dst != op->sym->m_src) {
+		klm_n = mlx5_crypto_klms_set(priv, qp, op, op->sym->m_src,
+					     klms);
+		if (unlikely(klm_n == 0))
+			return 0;
+	} else {
+		memcpy(klms, &umr->kseg[0], sizeof(*klms) * klm_n);
+	}
+	cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj->id << 8) | (2 + klm_n));
+	qp->db_pi += priv->wqe_stride;
+	/* Set NOP WQE if needed. */
+	klm_n = RTE_ALIGN(klm_n + 2, 4) - 2;
+	nop_ds = priv->max_rdmaw_klm_n - klm_n;
+	if (nop_ds) {
+		cseg = (struct mlx5_wqe_cseg *)(klms + klm_n);
+		cseg->opcode = rte_cpu_to_be_32(((qp->db_pi - (nop_ds >> 2)) <<
+						 8) | MLX5_OPCODE_NOP);
+		cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj->id << 8) | nop_ds);
+	}
+	qp->wqe = (uint8_t *)cseg;
+	return 1;
+}
+
+static __rte_always_inline void
+mlx5_crypto_uar_write(uint64_t val, struct mlx5_crypto_priv *priv)
+{
+#ifdef RTE_ARCH_64
+	*priv->uar_addr = val;
+#else /* !RTE_ARCH_64 */
+	rte_spinlock_lock(&priv->uar32_sl);
+	*(volatile uint32_t *)priv->uar_addr = val;
+	rte_io_wmb();
+	*((volatile uint32_t *)priv->uar_addr + 1) = val >> 32;
+	rte_spinlock_unlock(&priv->uar32_sl);
+#endif
+}
+
+static uint16_t
+mlx5_crypto_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
+			  uint16_t nb_ops)
+{
+	struct mlx5_crypto_qp *qp = queue_pair;
+	struct mlx5_crypto_priv *priv = qp->priv;
+	struct mlx5_umr_wqe *umr;
+	struct rte_crypto_op *op;
+	uint16_t mask = qp->entries_n - 1;
+	uint16_t remain = qp->entries_n - (qp->pi - qp->ci);
+
+	if (remain < nb_ops)
+		nb_ops = remain;
+	else
+		remain = nb_ops;
+	if (unlikely(remain == 0))
+		return 0;
+	do {
+		op = *ops++;
+		umr = RTE_PTR_ADD(qp->umem_buf, priv->wqe_set_size * qp->pi);
+		if (unlikely(mlx5_crypto_wqe_set(priv, qp, op, umr) == 0))
+			break;
+		qp->ops[qp->pi] = op;
+		qp->pi = (qp->pi + 1) & mask;
+	} while (--remain);
+	rte_io_wmb();
+	qp->db_rec[MLX5_SND_DBR] = rte_cpu_to_be_32(qp->db_pi);
+	rte_wmb();
+	mlx5_crypto_uar_write(*(volatile uint64_t *)qp->wqe, qp->priv);
+	rte_wmb();
+	return nb_ops;
+}
+
+static __rte_noinline void
+mlx5_crypto_cqe_err_handle(struct mlx5_crypto_qp *qp, struct rte_crypto_op *op)
+{
+	const uint32_t idx = qp->ci & (qp->entries_n - 1);
+	volatile struct mlx5_err_cqe *cqe = (volatile struct mlx5_err_cqe *)
+							&qp->cq_obj.cqes[idx];
+
+	op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+	DRV_LOG(ERR, "CQE ERR:%x.\n", rte_be_to_cpu_32(cqe->syndrome));
+}
+
+static uint16_t
+mlx5_crypto_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
+			  uint16_t nb_ops)
+{
+	struct mlx5_crypto_qp *qp = queue_pair;
+	volatile struct mlx5_cqe *restrict cqe;
+	struct rte_crypto_op *restrict op;
+	const unsigned int cq_size = qp->entries_n;
+	const unsigned int mask = cq_size - 1;
+	uint32_t idx;
+	uint32_t next_idx = qp->ci & mask;
+	const uint16_t max = RTE_MIN((uint16_t)(qp->pi - qp->ci), nb_ops);
+	uint16_t i = 0;
+	int ret;
+
+	if (unlikely(max == 0))
+		return 0;
+	do {
+		idx = next_idx;
+		next_idx = (qp->ci + 1) & mask;
+		op = qp->ops[idx];
+		cqe = &qp->cq_obj.cqes[idx];
+		ret = check_cqe(cqe, cq_size, qp->ci);
+		rte_io_rmb();
+		if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) {
+			if (unlikely(ret != MLX5_CQE_STATUS_HW_OWN))
+				mlx5_crypto_cqe_err_handle(qp, op);
+			break;
+		}
+		op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+		ops[i++] = op;
+		qp->ci++;
+	} while (i < max);
+	if (likely(i != 0)) {
+		rte_io_wmb();
+		qp->cq_obj.db_rec[0] = rte_cpu_to_be_32(qp->ci);
+	}
+	return i;
+}
+
 static void
 mlx5_crypto_qp_init(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp)
 {
@@ -490,8 +723,9 @@ mlx5_crypto_hw_global_prepare(struct mlx5_crypto_priv *priv)
 	if (mlx5_crypto_pd_create(priv) != 0)
 		return -1;
 	priv->uar = mlx5_devx_alloc_uar(priv->ctx, -1);
-	if (priv->uar == NULL || mlx5_os_get_devx_uar_reg_addr(priv->uar) ==
-	    NULL) {
+	if (priv->uar)
+		priv->uar_addr = mlx5_os_get_devx_uar_reg_addr(priv->uar);
+	if (priv->uar == NULL || priv->uar_addr == NULL) {
 		rte_errno = errno;
 		claim_zero(mlx5_glue->dealloc_pd(priv->pd));
 		DRV_LOG(ERR, "Failed to allocate UAR.");
@@ -686,8 +920,8 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	DRV_LOG(INFO,
 		"Crypto device %s was created successfully.", ibv->name);
 	crypto_dev->dev_ops = &mlx5_crypto_ops;
-	crypto_dev->dequeue_burst = NULL;
-	crypto_dev->enqueue_burst = NULL;
+	crypto_dev->dequeue_burst = mlx5_crypto_dequeue_burst;
+	crypto_dev->enqueue_burst = mlx5_crypto_enqueue_burst;
 	crypto_dev->feature_flags = 0;
 	crypto_dev->driver_id = mlx5_crypto_driver_id;
 	priv = crypto_dev->data->dev_private;
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 52fcf5217f..ac4ad1834f 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -37,6 +37,9 @@ struct mlx5_crypto_priv {
 	uint16_t rdmw_wqe_size;
 	uint16_t wqe_stride;
 	uint16_t max_rdmaw_klm_n;
+#ifndef RTE_ARCH_64
+	rte_spinlock_t uar32_sl;
+#endif /* RTE_ARCH_64 */
 };
 
 struct mlx5_crypto_qp {
-- 
2.25.1

[dpdk-dev] [PATCH v4 13/14] crypto/mlx5: add statistic get and reset operations

[Matan Azrad]

From: Suanming Mou <suanmingm@nvidia.com>

This commit adds mlx5 crypto statistic get and reset operations.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 39 ++++++++++++++++++++++++++++---
 1 file changed, 36 insertions(+), 3 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 29e59de702..60ffa6951e 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -438,11 +438,14 @@ mlx5_crypto_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
 	do {
 		op = *ops++;
 		umr = RTE_PTR_ADD(qp->umem_buf, priv->wqe_set_size * qp->pi);
-		if (unlikely(mlx5_crypto_wqe_set(priv, qp, op, umr) == 0))
+		if (unlikely(mlx5_crypto_wqe_set(priv, qp, op, umr) == 0)) {
+			qp->stats.enqueue_err_count++;
 			break;
+		}
 		qp->ops[qp->pi] = op;
 		qp->pi = (qp->pi + 1) & mask;
 	} while (--remain);
+	qp->stats.enqueued_count += nb_ops;
 	rte_io_wmb();
 	qp->db_rec[MLX5_SND_DBR] = rte_cpu_to_be_32(qp->db_pi);
 	rte_wmb();
@@ -459,6 +462,7 @@ mlx5_crypto_cqe_err_handle(struct mlx5_crypto_qp *qp, struct rte_crypto_op *op)
 							&qp->cq_obj.cqes[idx];
 
 	op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+	qp->stats.dequeue_err_count++;
 	DRV_LOG(ERR, "CQE ERR:%x.\n", rte_be_to_cpu_32(cqe->syndrome));
 }
 
@@ -498,6 +502,7 @@ mlx5_crypto_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
 	if (likely(i != 0)) {
 		rte_io_wmb();
 		qp->cq_obj.db_rec[0] = rte_cpu_to_be_32(qp->ci);
+		qp->stats.dequeued_count += i;
 	}
 	return i;
 }
@@ -656,14 +661,42 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	return -1;
 }
 
+static void
+mlx5_crypto_stats_get(struct rte_cryptodev *dev,
+		      struct rte_cryptodev_stats *stats)
+{
+	int qp_id;
+
+	for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
+		struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
+
+		stats->enqueued_count += qp->stats.enqueued_count;
+		stats->dequeued_count += qp->stats.dequeued_count;
+		stats->enqueue_err_count += qp->stats.enqueue_err_count;
+		stats->dequeue_err_count += qp->stats.dequeue_err_count;
+	}
+}
+
+static void
+mlx5_crypto_stats_reset(struct rte_cryptodev *dev)
+{
+	int qp_id;
+
+	for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
+		struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
+
+		memset(&qp->stats, 0, sizeof(qp->stats));
+	}
+}
+
 static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_configure			= mlx5_crypto_dev_configure,
 	.dev_start			= mlx5_crypto_dev_start,
 	.dev_stop			= mlx5_crypto_dev_stop,
 	.dev_close			= mlx5_crypto_dev_close,
 	.dev_infos_get			= mlx5_crypto_dev_infos_get,
-	.stats_get			= NULL,
-	.stats_reset			= NULL,
+	.stats_get			= mlx5_crypto_stats_get,
+	.stats_reset			= mlx5_crypto_stats_reset,
 	.queue_pair_setup		= mlx5_crypto_queue_pair_setup,
 	.queue_pair_release		= mlx5_crypto_queue_pair_release,
 	.sym_session_get_size		= mlx5_crypto_sym_session_get_size,
-- 
2.25.1

[dpdk-dev] [PATCH v4 14/14] crypto/mlx5: set feature flags and capabilities

[Matan Azrad]

From: Shiri Kuzin <shirik@nvidia.com>

Add the supported capabilities to the crypto driver.

Add supported feature flags.

Add crypto driver documentation.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
 doc/guides/cryptodevs/features/mlx5.ini |  37 ++++++
 doc/guides/cryptodevs/index.rst         |   1 +
 doc/guides/cryptodevs/mlx5.rst          | 152 ++++++++++++++++++++++++
 doc/guides/rel_notes/release_21_05.rst  |   5 +
 drivers/crypto/mlx5/mlx5_crypto.c       |  40 ++++++-
 5 files changed, 231 insertions(+), 4 deletions(-)
 create mode 100644 doc/guides/cryptodevs/features/mlx5.ini
 create mode 100644 doc/guides/cryptodevs/mlx5.rst

diff --git a/doc/guides/cryptodevs/features/mlx5.ini b/doc/guides/cryptodevs/features/mlx5.ini
new file mode 100644
index 0000000000..a89526add0
--- /dev/null
+++ b/doc/guides/cryptodevs/features/mlx5.ini
@@ -0,0 +1,37 @@
+;
+; Features of a mlx5 crypto driver.
+;
+; Refer to default.ini for the full list of available PMD features.
+;
+[Features]
+Symmetric crypto       = Y
+HW Accelerated         = Y
+In Place SGL           = Y
+OOP SGL In SGL Out     = Y
+OOP SGL In LB  Out     = Y
+OOP LB  In SGL Out     = Y
+OOP LB  In LB  Out     = Y
+Cipher multiple data units = Y
+Cipher wrapped key     = Y
+
+;
+; Supported crypto algorithms of a mlx5 crypto driver.
+;
+[Cipher]
+AES XTS (128)  = Y
+AES XTS (256)  = Y
+
+;
+; Supported authentication algorithms of a mlx5 crypto driver.
+;
+[Auth]
+
+;
+; Supported AEAD algorithms of a mlx5 crypto driver.
+;
+[AEAD]
+
+;
+; Supported Asymmetric algorithms of a mlx5 crypto driver.
+;
+[Asymmetric]
diff --git a/doc/guides/cryptodevs/index.rst b/doc/guides/cryptodevs/index.rst
index 279f56a002..747409c441 100644
--- a/doc/guides/cryptodevs/index.rst
+++ b/doc/guides/cryptodevs/index.rst
@@ -22,6 +22,7 @@ Crypto Device Drivers
     octeontx
     octeontx2
     openssl
+    mlx5
     mvsam
     nitrox
     null
diff --git a/doc/guides/cryptodevs/mlx5.rst b/doc/guides/cryptodevs/mlx5.rst
new file mode 100644
index 0000000000..735c0e1fa0
--- /dev/null
+++ b/doc/guides/cryptodevs/mlx5.rst
@@ -0,0 +1,152 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+   Copyright 2021 Mellanox Technologies, Ltd
+
+.. include:: <isonum.txt>
+
+MLX5 Crypto Driver
+==================
+
+The MLX5 crypto driver library
+(**librte_crypto_mlx5**) provides support for **Mellanox ConnectX-6**
+family adapters.
+
+Overview
+--------
+
+The device can provide disk encryption services, allowing data encryption
+and decryption towards a disk. Having all encryption/decryption
+operations done in a single device can reduce cost and overheads of the related
+FIPS certification, as ConnectX-6 is FIPS 140-2 level-2 ready.
+The encryption cipher is AES-XTS of 256/512 bit key size.
+
+MKEY is a memory region object in the hardware, that holds address translation information and
+attributes per memory area. Its ID must be tied to addresses provided to the hardware.
+The encryption operations are performed with MKEY read/write transactions, when
+the MKEY is configured to perform crypto operations.
+
+The encryption does not require text to be aligned to the AES block size (128b).
+
+In order to move the device to crypto operational mode, credential and KEK
+(Key Encrypting Key) should be set as the first step.
+The credential will be used by the software in order to perform crypto login, and the KEK is
+the AES Key Wrap Algorithm (rfc3394) key that will be used for sensitive data
+wrapping.
+The credential and the AES-XTS keys should be provided to the hardware, as ciphertext
+encrypted by the KEK.
+
+A keytag (64 bits) should be appended to the AES-XTS keys (before wrapping),
+and will be validated when the hardware attempts to access it.
+
+For security reasons and to increase robustness, this driver only deals with virtual
+memory addresses. The way resources allocations are handled by the kernel,
+combined with hardware specifications that allow handling virtual memory
+addresses directly, ensure that DPDK applications cannot access random
+physical memory (or memory that does not belong to the current process).
+
+The PMD uses libibverbs and libmlx5 to access the device firmware or to
+access the hardware components directly.
+There are different levels of objects and bypassing abilities.
+To get the best performances:
+
+- Verbs is a complete high-level generic API.
+- Direct Verbs is a device-specific API.
+- DevX allows to access firmware objects.
+
+Enabling librte_crypto_mlx5 causes DPDK applications to be linked against
+libibverbs.
+
+Mellanox mlx5 PCI device can be probed by a number of different PCI devices, such as
+net / vDPA / RegEx. To select the crypto PMD, ``class=crypto``
+should be specified as a device parameter. The crypto device can be probed and
+used with other Mellanox classes by adding more options in the class.
+For example: ``class=net:crypto`` will probe both the net PMD and the crypto
+PMD.
+
+When crypto engines are defined to work in wrapped import method, they come out
+of the factory in Commissioning mode, and thus, cannot be used for crypto operations
+yet. A dedicated tool is used for changing the mode from Commissioning to
+Operational, while setting the first import_KEK and credential in plaintext.
+The mlxreg dedicated tool should be used as follows:
+
+- Set CRYPTO_OPERATIONAL register to set the device in crypto operational mode.
+
+  The input to this tool is:
+    The first credential in plaintext, 40B.
+    The first import_KEK in plaintext: kek size 0 for 16B or 1 for 32B, kek data.
+
+  Example:
+  mlxreg -d /dev/mst/mt4123_pciconf0 --reg_name CRYPTO_OPERATIONAL --get
+
+  The "wrapped_crypto_operational" value will be "0x00000000".
+  The command to set the register should be executed only once, and all the
+  values mentioned above should be specified in the same command.
+
+  Example:
+  mlxreg -d /dev/mst/mt4123_pciconf0 --reg_name CRYPTO_OPERATIONAL
+  --set "credential[0]=0x10000000, credential[1]=0x10000000, kek[0]=0x00000000"
+
+  All values not specified will remain 0.
+  "wrapped_crypto_going_to_commissioning" and  "wrapped_crypto_operational"
+  should not be specified.
+
+  All the device ports should set it in order to move to operational mode.
+
+- Query CRYPTO_OPERATIONAL register to make sure the device is in Operational
+  mode.
+
+  Example:
+  mlxreg -d /dev/mst/mt4123_pciconf0 --reg_name CRYPTO_OPERATIONAL --get
+  The "wrapped_crypto_operational" value will be "0x00000001" if the mode was
+  successfully changed to operational mode.
+
+
+Driver options
+--------------
+
+- ``class`` parameter [string]
+
+  Select the class of the driver that should probe the device.
+  `crypto` for the mlx5 crypto driver.
+
+- ``wcs_file`` parameter [string] - mandatory
+
+  File path including only the wrapped credential in string format of hexadecimal
+  numbers, represent 48 bytes (8 bytes IV added by the AES key wrap algorithm).
+
+- ``import_kek_id`` parameter [int]
+
+  The identifier of the KEK, default value is 0 represents the operational
+  register import_kek..
+
+- ``credential_id`` parameter [int]
+
+  The identifier of the credential, default value is 0 represents the operational
+  register credential.
+
+- ``max_segs_num`` parameter [int]
+
+  Maximum number of mbuf chain segments(src or dest), default value is 8.
+
+- ``keytag`` parameter [int]
+
+  The plaintext of the keytag appanded to the AES-XTS keys, default value is 0.
+
+
+Limitations
+-----------
+
+- AES-XTS keys provided in xform must include keytag and should be wrappend.
+- The supported data-unit lengths are: 512B, 1KB, 1MB. In case the `dataunit_len`
+  is not provided in the cipher xform, the OP length is limited to the above values.
+
+
+Supported NICs
+--------------
+
+* Mellanox\ |reg| ConnectX\ |reg|-6 200G MCX654106A-HCAT (2x200G)
+
+Prerequisites
+-------------
+
+- Mellanox OFED version: **5.3**
+  see :doc:`../../nics/mlx5` guide for more Mellanox OFED details.
diff --git a/doc/guides/rel_notes/release_21_05.rst b/doc/guides/rel_notes/release_21_05.rst
index 30dec1c1d1..eaaa9eecbf 100644
--- a/doc/guides/rel_notes/release_21_05.rst
+++ b/doc/guides/rel_notes/release_21_05.rst
@@ -287,6 +287,11 @@ New Features
   * Added support for crypto adapter forward mode in octeontx2 event and crypto
     device driver.
 
+* **Added support for Nvidia crypto device driver.**
+
+  * Added mlx5 crypto driver to support AES-XTS cipher operations.
+    the first device to support it is ConnectX-6.
+
 
 Removed Items
 -------------
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 60ffa6951e..4c5cbf5ffe 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -22,6 +22,14 @@
 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
 #define MLX5_CRYPTO_MAX_QPS 1024
 #define MLX5_CRYPTO_MAX_SEGS 56
+#define MLX5_CRYPTO_FEATURE_FLAGS \
+	(RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | \
+	RTE_CRYPTODEV_FF_IN_PLACE_SGL | RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT | \
+	RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | \
+	RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT | \
+	RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | \
+	RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY | \
+	RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS)
 
 TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
 				TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
@@ -31,8 +39,32 @@ int mlx5_crypto_logtype;
 
 uint8_t mlx5_crypto_driver_id;
 
-const struct rte_cryptodev_capabilities
-		mlx5_crypto_caps[RTE_CRYPTO_OP_TYPE_UNDEFINED];
+const struct rte_cryptodev_capabilities mlx5_crypto_caps[] = {
+	{		/* AES XTS */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+			{.cipher = {
+				.algo = RTE_CRYPTO_CIPHER_AES_XTS,
+				.block_size = 16,
+				.key_size = {
+					.min = 32,
+					.max = 64,
+					.increment = 32
+				},
+				.iv_size = {
+					.min = 16,
+					.max = 16,
+					.increment = 0
+				},
+				.dataunit_set =
+				RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_512_BYTES |
+				RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_4096_BYTES,
+			}, }
+		}, }
+	},
+};
+
 
 static const char mlx5_crypto_drv_name[] = RTE_STR(MLX5_CRYPTO_DRIVER_NAME);
 
@@ -67,7 +99,7 @@ mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev,
 	RTE_SET_USED(dev);
 	if (dev_info != NULL) {
 		dev_info->driver_id = mlx5_crypto_driver_id;
-		dev_info->feature_flags = 0;
+		dev_info->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
 		dev_info->capabilities = mlx5_crypto_caps;
 		dev_info->max_nb_queue_pairs = MLX5_CRYPTO_MAX_QPS;
 		dev_info->min_mbuf_headroom_req = 0;
@@ -955,7 +987,7 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	crypto_dev->dev_ops = &mlx5_crypto_ops;
 	crypto_dev->dequeue_burst = mlx5_crypto_dequeue_burst;
 	crypto_dev->enqueue_burst = mlx5_crypto_enqueue_burst;
-	crypto_dev->feature_flags = 0;
+	crypto_dev->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
 	crypto_dev->driver_id = mlx5_crypto_driver_id;
 	priv = crypto_dev->data->dev_private;
 	priv->ctx = ctx;
-- 
2.25.1

Re: [dpdk-dev] [EXT] [PATCH v3 01/15] drivers: introduce mlx5 crypto PMD

[Bruce Richardson]

On Sun, May 09, 2021 at 09:32:45AM +0000, Akhil Goyal wrote:
> > > > +
> > > > +fmt_name = 'mlx5_crypto'
> > > > +deps += ['common_mlx5', 'eal', 'cryptodev']
> > >
> > > I don't see a reason for eal dependency. Do you really need it?
> > 
> > See RTE_LOG_REGISTER.
> 
> Compilation works fine even after removing this dependency.
> I believe it is internally managed in cryptodev.

Yes, meson dependencies are recursive, so taking in e.g. cryptodev also
pulls in all dependencies of cryptodev, including EAL. Having the deps
explicitly called out makes things a little clearer for the reader, but
having a minimum set of dependencies called out can make meson runs a
little faster as it doesn't have as much checking and pruning of
dependencies to do. [this was more a problem with older meson releases,
less no now.] This is why the deps list in DPDK are generally fairly short.

Either way, whatever builds is fine. :-)

/Bruce

Re: [dpdk-dev] [EXT] [PATCH v3 01/15] drivers: introduce mlx5 crypto PMD

[Akhil Goyal]

> >
> > > > > +
> > > > > +fmt_name = 'mlx5_crypto'
> > > > > +deps += ['common_mlx5', 'eal', 'cryptodev']
> > > >
> > > > I don't see a reason for eal dependency. Do you really need it?
> > >
> > > See RTE_LOG_REGISTER.
> >
> > Compilation works fine even after removing this dependency.
> > I believe it is internally managed in cryptodev.
> 
> Yes, but it is better to add the libs we use directly.
> 
Please remove eal so that it is consistent with other PMDs

Re: [dpdk-dev] [EXT] [PATCH v3 03/15] crypto/mlx5: support session operations

[Akhil Goyal]

> >
> > > > > +static void
> > > > > +mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev,
> > > > > +                           struct rte_cryptodev_sym_session *sess) {
> > > > > +     struct mlx5_crypto_priv *priv = dev->data->dev_private;
> > > > > +     struct mlx5_crypto_session *sess_private_data =
> > > > > +                     get_sym_session_private_data(sess,
> > > > > +dev->driver_id);
> > > > > +
> > > > > +     if (unlikely(sess_private_data == NULL)) {
> > > > > +             DRV_LOG(ERR, "Failed to get session %p private data.",
> > > > > +                             sess_private_data);
> > > > > +             return;
> > > > > +     }
> > > > > +     mlx5_crypto_dek_destroy(priv, sess_private_data->dek);
> > > > > +     DRV_LOG(DEBUG, "Session %p was cleared.", sess_private_data);
> }
> > > >
> > > > Memory leakage, mempool is not freed.
> > >
> > > Yes, good catch, this part was missed.
> > >
> > > > IMO, this driver is not properly tested with the unit test app.
> > >
> > > The only app we tested until now is l2fwd_crypto and it works fine!
> > > We can add it to doc.
> > >
> > > > Please add a note in the documentation that it is tested with autotest.
> > >
> > >
> > > The next app we want to test with, is test-crypto-perf.
> > >
> > I would recommend to run the test app first.
> > It will catch most of your basic bugs like the one above.
> 
> It is too late for this, will add the test adjustment later.

Can we postpone the PMD to next release. I believe test application makes
The PMD look robust as per the DPDK crypto PMD API usage.
I haven't seen a PMD getting merged without test app.
And I apologize I did not mentioned it earlier, but it is kind of obvious thing
to run test app before sending it to upstream.
L2fwd-crypto is not doing data validation hence you cannot be sure that it is
working fine as per other standard stacks like Linux.

Regards,
Akhil

Re: [dpdk-dev] [EXT] [PATCH v3 09/15] crypto/mlx5: adjust to the multiple data unit API

[Akhil Goyal]

> > >
> > > > Also set feature flag in the code and the documentation in this patch.
> > > > I see that you are setting all of them in a single patch in the end.
> > > > This is not correct. It should be added where the feature is supported.
> > > > Please fix this for all the feature flags.
> > >
> > > No, in this stage no feature is really supported, the actual time it
> > > will be supported is after the datapath patches and capabilities set.
> > >
> > Move the patch after adding data path, but documentation should be part
> Of
> > this patch.
> 
> I will squash this patch to the session commit.

I am not asking you to squash the patch to the session commit.
The point is documentation patch should be part of code patch which
Introduced that feature. This methodology has been discussed in the past
and is being followed now. Please discuss this in techboard in case of deviation.

Regards,
Akhil

Re: [dpdk-dev] [EXT] [PATCH v3 10/15] crypto/mlx5: add keytag device argument

[Akhil Goyal]

> 
> From: Akhil Goyal
> > > From: Akhil Goyal
> > > > > From: Suanming Mou <suanmingm@nvidia.com>
> > > > >
> > > > > A keytag is a piece of data encrypted together with a DEK.
> > > > >
> > > > > When a DEK is referenced by an MKEY.bsf through its index, the
> > > > > keytag is also supplied in the BSF as plaintext. The HW will
> > > > > decrypt the DEK (and the attached keytag) and will fail the
> > > > > operation if the keytags don't match.
> > > > >
> > > > > This commit adds the configuration of the keytag with devargs.
> > > > >
> > > > > Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
> > > > > Signed-off-by: Matan Azrad <matan@nvidia.com>
> > > > > ---
> > > > Documentation for devargs should be part of this patch.
> > > > Please split the last patch accordingly.
> > > >
> > > > Title can be shortened to
> > > > Crypto/mlx5: add keytag devarg
> > > >
> > > > Fix other patches of devargs accordingly.
> > >
> > > As I said before, no devargs is really active before adding datapath
> patches.
> > > The option to add all the supported features \ documentations in the
> > > patch which actually adds the support is correct.
> > >
> > > The last patch adds the capabilities and docs when all of them are
> > > really supported.
> > >
> > In that case split the patches in such a manner that data path is added
> Before
> > devargs patch with some dummy values and add the devargs with
> > Appropriate documentation update in a later patch.
> 
> 
> No, data-path must know all the devargs in advance, it cannot work well
> without them.

The suggestion is to have a sequence like this
Crypto/mlx5: add datapath -- (with some dummy values of devargs.)
Crypto/mlx5: add keytag devarg  -- (replace the dummy values from previous patch
				       with correct one. Add documentation )
Crypto/mx5: add xyz devarg  -- (same as keytag)

Regards,
Akhil

Re: [dpdk-dev] [EXT] [PATCH v3 15/15] crypto/mlx5: set feature flags and capabilities

[Akhil Goyal]

Hi Matan,
> > > > > +Prerequisites
> > > > > +-------------
> > > > > +
> > > > > +- Mellanox OFED version: **5.3**
> > > > > +  see :doc:`../../nics/mlx5` guide for more Mellanox OFED details.
> > > >
> > > > Since the driver is by default compiled off due to the dependency on
> > > > external Libraries, I would recommend to add few lines here as well
> > > > for compilation.
> > > > Like to compile rdma-core and set PKG_CONFIG_LIBDIR.
> > >
> > > Why? all Mellanox drivers has the same external dependencies.
> > > I added here link for the doc explains it well.
> >
> > This is a crypto PMD, not a NIC PMD. Somebody working on crypto PMDs,
> do
> > not really care about the NIC PMDs.
> > Hence it would be convenient to have compilation information here as
> well.
> > You can refer to other document for details, but basic info should be added
> > here as well.
> 
> The link explains how to install OFED, this is only what the user need to take
> from the link.
> The basic is to install OFED.
> I don't see a reason to duplicate doc section which are exactly the same.

As I compiled the PMD, it was not convenient to read the whole document.
And it is not needed to compile linux and everything.
I just needed rdma-core and set it in PKG_CONFIG_LIBDIR.

The reason I am insisting here is, when somebody do small changes in
Crypto library, he may need to do subsequent changes in all PMDs.
For which compilation steps should be easily accessible in the PMD doc
So that the patch can be compiled properly.

Hence I just recommend to have 3-4 lines to enable the compilation
In the PMD doc.

> > >
> > > > And I do not see any updates to the test application for testing this
> driver.
> > >
> > > You can see update to l2fwd_crypto, we tested with this example for
> > > the first stage.
> > > Everything looks ok there.
> >
> > L2fwd-crypto is an app which only test data path with no packet validation.
> > It does not tell if your encryption is correctly done as per standards or not.
> > Did you test interoperability with l2fwd-crypto?
> > All basic configuration tests are also not done, like cleanup etc of the PMD.
> > I haven't seen a driver getting merge without the unit test application run.
> > Test app helps you comply with the way dpdk drivers are meant to be
> > written.
> 
> We adjusted the l2fwd-crypto to the dataunit feature and wrapped keys.
> We validated data integrity from the packet returns back from the crypto net
> port.
> As I said, encryption\decryption with AES-XTS is working well.

Do you test interoperability here? Encryption by MLX5 and decryption
By another PMD/stack and vice-versa.
Test app is supposed to have test vectors which will work on any platform.
Hence data validation is done properly.

> 
> Now, is too late to update the test application to the above features, the
> driver code is here for a long time, no one ask about the test adjustment until
> now.

Can we defer to next release? I apologize for not asking it earlier. But this is
kind of obvious for somebody working in DPDK.
Please check that none of the PMD is merged without test app in the past 3-4yrs.

> 
> We can add the adjustment to increase validity for the next release to all the
> remaining crypto apps (test\test-crypto-perf).
> 
> For now, we have one validation with l2fwd-crypto And any user can run it
> and see how to use mlx5 driver.

The user cannot be sure of the basic things of a crypto PMD are in order or not.
As l2fwd-crypto does not test every basic thing.
For eg. Session deletion, PMD stop, PMD close, PMD restart,
setting up multiple sessions(l2fwd support single session).

Running datapath of a single use case is not sufficient for a PMD.
This is a POC and it need to comply with the environment.

I hope the doubts are clear now and we are OK to defer to next release.

Regards,
Akhil

Re: [dpdk-dev] [EXT] [PATCH v3 01/15] drivers: introduce mlx5 crypto PMD

[Matan Azrad]

From: Akhil Goyal
> > > > > > +fmt_name = 'mlx5_crypto'
> > > > > > +deps += ['common_mlx5', 'eal', 'cryptodev']
> > > > >
> > > > > I don't see a reason for eal dependency. Do you really need it?
> > > >
> > > > See RTE_LOG_REGISTER.
> > >
> > > Compilation works fine even after removing this dependency.
> > > I believe it is internally managed in cryptodev.
> >
> > Yes, but it is better to add the libs we use directly.
> >
> Please remove eal so that it is consistent with other PMDs

I don't understand why you don't give a claim to explain why not to add libs the PMD accesses directly.

One day when other dependency will remove the direct dependency, it may be problematic.

Because I see you strongly insist on it and this is tiny I will remove. 

Re: [dpdk-dev] [EXT] [PATCH v3 01/15] drivers: introduce mlx5 crypto PMD

[Akhil Goyal]

> From: Akhil Goyal
> > > > > > > +fmt_name = 'mlx5_crypto'
> > > > > > > +deps += ['common_mlx5', 'eal', 'cryptodev']
> > > > > >
> > > > > > I don't see a reason for eal dependency. Do you really need it?
> > > > >
> > > > > See RTE_LOG_REGISTER.
> > > >
> > > > Compilation works fine even after removing this dependency.
> > > > I believe it is internally managed in cryptodev.
> > >
> > > Yes, but it is better to add the libs we use directly.
> > >
> > Please remove eal so that it is consistent with other PMDs
> 
> I don't understand why you don't give a claim to explain why not to add libs
> the PMD accesses directly.

If that is the case, why not add all the other libs as well - mbuf, mempool, pci etc?
Why only eal?

> 
> One day when other dependency will remove the direct dependency, it may
> be problematic.
> 
> Because I see you strongly insist on it and this is tiny I will remove.

Re: [dpdk-dev] [EXT] [PATCH v3 03/15] crypto/mlx5: support session operations

[Matan Azrad]

From: Akhil Goyal
> > > > > > +static void
> > > > > > +mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev,
> > > > > > +                           struct rte_cryptodev_sym_session *sess) {
> > > > > > +     struct mlx5_crypto_priv *priv = dev->data->dev_private;
> > > > > > +     struct mlx5_crypto_session *sess_private_data =
> > > > > > +                     get_sym_session_private_data(sess,
> > > > > > +dev->driver_id);
> > > > > > +
> > > > > > +     if (unlikely(sess_private_data == NULL)) {
> > > > > > +             DRV_LOG(ERR, "Failed to get session %p private data.",
> > > > > > +                             sess_private_data);
> > > > > > +             return;
> > > > > > +     }
> > > > > > +     mlx5_crypto_dek_destroy(priv, sess_private_data->dek);
> > > > > > +     DRV_LOG(DEBUG, "Session %p was cleared.",
> > > > > > + sess_private_data);
> > }
> > > > >
> > > > > Memory leakage, mempool is not freed.
> > > >
> > > > Yes, good catch, this part was missed.
> > > >
> > > > > IMO, this driver is not properly tested with the unit test app.
> > > >
> > > > The only app we tested until now is l2fwd_crypto and it works fine!
> > > > We can add it to doc.
> > > >
> > > > > Please add a note in the documentation that it is tested with autotest.
> > > >
> > > >
> > > > The next app we want to test with, is test-crypto-perf.
> > > >
> > > I would recommend to run the test app first.
> > > It will catch most of your basic bugs like the one above.
> >
> > It is too late for this, will add the test adjustment later.
> 
> Can we postpone the PMD to next release.

We can, but this is not our plan.
We met all the DPDK rules to push it on time.

> I believe test application makes
> The PMD look robust as per the DPDK crypto PMD API usage.

Every test will add robustness to the PMD.

> I haven't seen a PMD getting merged without test app.

compress/mlx5, vdpa/mlx5, regex/mlx5, net/mlx5, vdev_netvsc....

> And I apologize I did not mentioned it earlier, but it is kind of obvious thing to
> run test app before sending it to upstream.

In fact, no, I added more than one PMD, no one require specific test.

> L2fwd-crypto is not doing data validation hence you cannot be sure that it is
> working fine as per other standard stacks like Linux.

It is not do data validation, but we check that the packet payload return back has the expected encrypted\decrypted data using open-ssl.
Also for us it was mandatory requirement to check it.

For us, the current validation is enough, we don't support a lot of things in the crypto PMD currently, only one algorithm in the most basic modes.

For sure we will continue to add tests and to increase robustness.
Also adding more features is in plan for future. 

If you postpone it, it yours, we don't agree with it.

> Regards,
> Akhil

Re: [dpdk-dev] [EXT] [PATCH v3 09/15] crypto/mlx5: adjust to the multiple data unit API

[Matan Azrad]

 From: Akhil Goyal
> > > > > Also set feature flag in the code and the documentation in this patch.
> > > > > I see that you are setting all of them in a single patch in the end.
> > > > > This is not correct. It should be added where the feature is supported.
> > > > > Please fix this for all the feature flags.
> > > >
> > > > No, in this stage no feature is really supported, the actual time
> > > > it will be supported is after the datapath patches and capabilities set.
> > > >
> > > Move the patch after adding data path, but documentation should be
> > > part
> > Of
> > > this patch.
> >
> > I will squash this patch to the session commit.
> 
> I am not asking you to squash the patch to the session commit.
> The point is documentation patch should be part of code patch which
> Introduced that feature. This methodology has been discussed in the past
> and is being followed now. Please discuss this in techboard in case of
> deviation.

Not sure this is mandatory when you add a new PMD.
For sure you right in general.

Anyway, the session commit is the better place to this code.
 
> Regards,
> Akhil

Re: [dpdk-dev] [EXT] [PATCH v3 10/15] crypto/mlx5: add keytag device argument

[Matan Azrad]

From: Akhil Goyal 
> > From: Akhil Goyal
> > > > From: Akhil Goyal
> > > > > > From: Suanming Mou <suanmingm@nvidia.com>
> > > > > >
> > > > > > A keytag is a piece of data encrypted together with a DEK.
> > > > > >
> > > > > > When a DEK is referenced by an MKEY.bsf through its index, the
> > > > > > keytag is also supplied in the BSF as plaintext. The HW will
> > > > > > decrypt the DEK (and the attached keytag) and will fail the
> > > > > > operation if the keytags don't match.
> > > > > >
> > > > > > This commit adds the configuration of the keytag with devargs.
> > > > > >
> > > > > > Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
> > > > > > Signed-off-by: Matan Azrad <matan@nvidia.com>
> > > > > > ---
> > > > > Documentation for devargs should be part of this patch.
> > > > > Please split the last patch accordingly.
> > > > >
> > > > > Title can be shortened to
> > > > > Crypto/mlx5: add keytag devarg
> > > > >
> > > > > Fix other patches of devargs accordingly.
> > > >
> > > > As I said before, no devargs is really active before adding
> > > > datapath
> > patches.
> > > > The option to add all the supported features \ documentations in
> > > > the patch which actually adds the support is correct.
> > > >
> > > > The last patch adds the capabilities and docs when all of them are
> > > > really supported.
> > > >
> > > In that case split the patches in such a manner that data path is
> > > added
> > Before
> > > devargs patch with some dummy values and add the devargs with
> > > Appropriate documentation update in a later patch.
> >
> >
> > No, data-path must know all the devargs in advance, it cannot work
> > well without them.
> 
> The suggestion is to have a sequence like this
> Crypto/mlx5: add datapath -- (with some dummy values of devargs.)
> Crypto/mlx5: add keytag devarg  -- (replace the dummy values from previous
> patch
>                                        with correct one. Add documentation )
> Crypto/mx5: add xyz devarg  -- (same as keytag)

Yes, it can be another option.
But I don't see it critical.
If you insist we can do it.
 

> Regards,
> Akhil

Re: [dpdk-dev] [EXT] [PATCH v3 15/15] crypto/mlx5: set feature flags and capabilities

[Matan Azrad]

From: Akhil Goyal
> Hi Matan,
> > > > > > +Prerequisites
> > > > > > +-------------
> > > > > > +
> > > > > > +- Mellanox OFED version: **5.3**
> > > > > > +  see :doc:`../../nics/mlx5` guide for more Mellanox OFED details.
> > > > >
> > > > > Since the driver is by default compiled off due to the
> > > > > dependency on external Libraries, I would recommend to add few
> > > > > lines here as well for compilation.
> > > > > Like to compile rdma-core and set PKG_CONFIG_LIBDIR.
> > > >
> > > > Why? all Mellanox drivers has the same external dependencies.
> > > > I added here link for the doc explains it well.
> > >
> > > This is a crypto PMD, not a NIC PMD. Somebody working on crypto
> > > PMDs,
> > do
> > > not really care about the NIC PMDs.
> > > Hence it would be convenient to have compilation information here as
> > well.
> > > You can refer to other document for details, but basic info should
> > > be added here as well.
> >
> > The link explains how to install OFED, this is only what the user need
> > to take from the link.
> > The basic is to install OFED.
> > I don't see a reason to duplicate doc section which are exactly the same.
> 
> As I compiled the PMD, it was not convenient to read the whole document.
> And it is not needed to compile linux and everything.
> I just needed rdma-core and set it in PKG_CONFIG_LIBDIR.

But compilation is not enough to run, you still cannot test if you break thigs by compilation.
You need to install also the kernel modules.
That's what we explain in all our drivers.

 
> The reason I am insisting here is, when somebody do small changes in Crypto
> library, he may need to do subsequent changes in all PMDs.
> For which compilation steps should be easily accessible in the PMD doc So
> that the patch can be compiled properly.

Not enough.

> Hence I just recommend to have 3-4 lines to enable the compilation In the
> PMD doc.

We are not doing it in others mlx5 drivers.
If you insist, we will do.

> > > > > And I do not see any updates to the test application for testing
> > > > > this
> > driver.
> > > >
> > > > You can see update to l2fwd_crypto, we tested with this example
> > > > for the first stage.
> > > > Everything looks ok there.
> > >
> > > L2fwd-crypto is an app which only test data path with no packet
> validation.
> > > It does not tell if your encryption is correctly done as per standards or
> not.
> > > Did you test interoperability with l2fwd-crypto?
> > > All basic configuration tests are also not done, like cleanup etc of the
> PMD.
> > > I haven't seen a driver getting merge without the unit test application
> run.
> > > Test app helps you comply with the way dpdk drivers are meant to be
> > > written.
> >
> > We adjusted the l2fwd-crypto to the dataunit feature and wrapped keys.
> > We validated data integrity from the packet returns back from the
> > crypto net port.
> > As I said, encryption\decryption with AES-XTS is working well.
> 
> Do you test interoperability here? Encryption by MLX5 and decryption By
> another PMD/stack and vice-versa.

Compared to open-ssl results.

> Test app is supposed to have test vectors which will work on any platform.
> Hence data validation is done properly.

I think open-ssl is good standard to check too.

> >
> > Now, is too late to update the test application to the above features,
> > the driver code is here for a long time, no one ask about the test
> > adjustment until now.

I acked as maintainer of this PMD.

> Can we defer to next release? I apologize for not asking it earlier. But this is
> kind of obvious for somebody working in DPDK.
> Please check that none of the PMD is merged without test app in the past 3-
> 4yrs.

This is not true.

> > We can add the adjustment to increase validity for the next release to
> > all the remaining crypto apps (test\test-crypto-perf).
> >
> > For now, we have one validation with l2fwd-crypto And any user can run
> > it and see how to use mlx5 driver.
> 
> The user cannot be sure of the basic things of a crypto PMD are in order or
> not.
> As l2fwd-crypto does not test every basic thing.
> For eg. Session deletion, PMD stop, PMD close, PMD restart, setting up
> multiple sessions(l2fwd support single session).
> 
> Running datapath of a single use case is not sufficient for a PMD.
> This is a POC and it need to comply with the environment.

There are only one\ two cases we need in this version and we tested them.

> I hope the doubts are clear now and we are OK to defer to next release.

As I said, we don't agree, it is yours. 


> Regards,
> Akhil
> 

Re: [dpdk-dev] [EXT] [PATCH v3 03/15] crypto/mlx5: support session operations

[Akhil Goyal]

> From: Akhil Goyal
> > > > > > > +static void
> > > > > > > +mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev,
> > > > > > > +                           struct rte_cryptodev_sym_session *sess) {
> > > > > > > +     struct mlx5_crypto_priv *priv = dev->data->dev_private;
> > > > > > > +     struct mlx5_crypto_session *sess_private_data =
> > > > > > > +                     get_sym_session_private_data(sess,
> > > > > > > +dev->driver_id);
> > > > > > > +
> > > > > > > +     if (unlikely(sess_private_data == NULL)) {
> > > > > > > +             DRV_LOG(ERR, "Failed to get session %p private data.",
> > > > > > > +                             sess_private_data);
> > > > > > > +             return;
> > > > > > > +     }
> > > > > > > +     mlx5_crypto_dek_destroy(priv, sess_private_data->dek);
> > > > > > > +     DRV_LOG(DEBUG, "Session %p was cleared.",
> > > > > > > + sess_private_data);
> > > }
> > > > > >
> > > > > > Memory leakage, mempool is not freed.
> > > > >
> > > > > Yes, good catch, this part was missed.
> > > > >
> > > > > > IMO, this driver is not properly tested with the unit test app.
> > > > >
> > > > > The only app we tested until now is l2fwd_crypto and it works fine!
> > > > > We can add it to doc.
> > > > >
> > > > > > Please add a note in the documentation that it is tested with
> autotest.
> > > > >
> > > > >
> > > > > The next app we want to test with, is test-crypto-perf.
> > > > >
> > > > I would recommend to run the test app first.
> > > > It will catch most of your basic bugs like the one above.
> > >
> > > It is too late for this, will add the test adjustment later.
> >
> > Can we postpone the PMD to next release.
> 
> We can, but this is not our plan.
> We met all the DPDK rules to push it on time.

On time!! Really? Incomplete v1 was submitted before RC1,
The complete PMD was submitted only during RC2.

> 
> > I believe test application makes
> > The PMD look robust as per the DPDK crypto PMD API usage.
> 
> Every test will add robustness to the PMD.
> 
> > I haven't seen a PMD getting merged without test app.
> 
> compress/mlx5, vdpa/mlx5, regex/mlx5, net/mlx5, vdev_netvsc....
> 
All these are 'not' crypto PMDs.
For crypto, UT is a must. I would like to add a statement in the guidelines if it is not there.

> > And I apologize I did not mentioned it earlier, but it is kind of obvious thing
> to
> > run test app before sending it to upstream.
> 
> In fact, no, I added more than one PMD, no one require specific test.
> 
> > L2fwd-crypto is not doing data validation hence you cannot be sure that it
> is
> > working fine as per other standard stacks like Linux.
> 
> It is not do data validation, but we check that the packet payload return back
> has the expected encrypted\decrypted data using open-ssl.
> Also for us it was mandatory requirement to check it.
> 
> For us, the current validation is enough, we don't support a lot of things in
> the crypto PMD currently, only one algorithm in the most basic modes.
> 
> For sure we will continue to add tests and to increase robustness.
> Also adding more features is in plan for future.
> 
> If you postpone it, it yours, we don't agree with it.
> 
Please fix the review comments by today, if you want it  to be merged in RC3.
IMO, the driver is not ready to be merged. 
I want to postpone it as it is not feasible to fix all the comments by RC3 timeline.
I believe all my comments are valid and need to be addressed.
Can you get somebody else(outside Nvidia) to ack your patch and counter my comments?
I will merge it without review based on that Ack.

Regards,
Akhil

Re: [dpdk-dev] [EXT] [PATCH v3 09/15] crypto/mlx5: adjust to the multiple data unit API

[Akhil Goyal]

>  From: Akhil Goyal
> > > > > > Also set feature flag in the code and the documentation in this
> patch.
> > > > > > I see that you are setting all of them in a single patch in the end.
> > > > > > This is not correct. It should be added where the feature is
> supported.
> > > > > > Please fix this for all the feature flags.
> > > > >
> > > > > No, in this stage no feature is really supported, the actual time
> > > > > it will be supported is after the datapath patches and capabilities set.
> > > > >
> > > > Move the patch after adding data path, but documentation should be
> > > > part
> > > Of
> > > > this patch.
> > >
> > > I will squash this patch to the session commit.
> >
> > I am not asking you to squash the patch to the session commit.
> > The point is documentation patch should be part of code patch which
> > Introduced that feature. This methodology has been discussed in the past
> > and is being followed now. Please discuss this in techboard in case of
> > deviation.
> 
> Not sure this is mandatory when you add a new PMD.
In case of a new PMD also we try to follow this as much as we can.
IMO, this can be done here.

> For sure you right in general.
> 
> Anyway, the session commit is the better place to this code.
> 
> > Regards,
> > Akhil

Re: [dpdk-dev] [EXT] [PATCH v3 15/15] crypto/mlx5: set feature flags and capabilities

[Akhil Goyal]

> From: Akhil Goyal
> > Hi Matan,
> > > > > > > +Prerequisites
> > > > > > > +-------------
> > > > > > > +
> > > > > > > +- Mellanox OFED version: **5.3**
> > > > > > > +  see :doc:`../../nics/mlx5` guide for more Mellanox OFED details.
> > > > > >
> > > > > > Since the driver is by default compiled off due to the
> > > > > > dependency on external Libraries, I would recommend to add few
> > > > > > lines here as well for compilation.
> > > > > > Like to compile rdma-core and set PKG_CONFIG_LIBDIR.
> > > > >
> > > > > Why? all Mellanox drivers has the same external dependencies.
> > > > > I added here link for the doc explains it well.
> > > >
> > > > This is a crypto PMD, not a NIC PMD. Somebody working on crypto
> > > > PMDs,
> > > do
> > > > not really care about the NIC PMDs.
> > > > Hence it would be convenient to have compilation information here as
> > > well.
> > > > You can refer to other document for details, but basic info should
> > > > be added here as well.
> > >
> > > The link explains how to install OFED, this is only what the user need
> > > to take from the link.
> > > The basic is to install OFED.
> > > I don't see a reason to duplicate doc section which are exactly the same.
> >
> > As I compiled the PMD, it was not convenient to read the whole document.
> > And it is not needed to compile linux and everything.
> > I just needed rdma-core and set it in PKG_CONFIG_LIBDIR.
> 
> But compilation is not enough to run, you still cannot test if you break thigs
> by compilation.
> You need to install also the kernel modules.
> That's what we explain in all our drivers.
> 

For a person doing minor changes in the Lib, he cannot test each hardware.
He can only do compilation and that is what he is expected to do.

> 
> > The reason I am insisting here is, when somebody do small changes in
> Crypto
> > library, he may need to do subsequent changes in all PMDs.
> > For which compilation steps should be easily accessible in the PMD doc So
> > that the patch can be compiled properly.
> 
> Not enough.
> 
> > Hence I just recommend to have 3-4 lines to enable the compilation In the
> > PMD doc.
> 
> We are not doing it in others mlx5 drivers.
> If you insist, we will do.
> 
> > > > > > And I do not see any updates to the test application for testing
> > > > > > this
> > > driver.
> > > > >
> > > > > You can see update to l2fwd_crypto, we tested with this example
> > > > > for the first stage.
> > > > > Everything looks ok there.
> > > >
> > > > L2fwd-crypto is an app which only test data path with no packet
> > validation.
> > > > It does not tell if your encryption is correctly done as per standards or
> > not.
> > > > Did you test interoperability with l2fwd-crypto?
> > > > All basic configuration tests are also not done, like cleanup etc of the
> > PMD.
> > > > I haven't seen a driver getting merge without the unit test application
> > run.
> > > > Test app helps you comply with the way dpdk drivers are meant to be
> > > > written.
> > >
> > > We adjusted the l2fwd-crypto to the dataunit feature and wrapped keys.
> > > We validated data integrity from the packet returns back from the
> > > crypto net port.
> > > As I said, encryption\decryption with AES-XTS is working well.
> >
> > Do you test interoperability here? Encryption by MLX5 and decryption By
> > another PMD/stack and vice-versa.
> 
> Compared to open-ssl results.
> 
> > Test app is supposed to have test vectors which will work on any platform.
> > Hence data validation is done properly.
> 
> I think open-ssl is good standard to check too.
Yes it is good standard.
OK but what about the other basic cases that I listed below.

Did you mention any limitation that all these cases are not supported as of now.

> 
> > >
> > > Now, is too late to update the test application to the above features,
> > > the driver code is here for a long time, no one ask about the test
> > > adjustment until now.
> 
> I acked as maintainer of this PMD.
Is it really enough for the new PMD?

> 
> > Can we defer to next release? I apologize for not asking it earlier. But this is
> > kind of obvious for somebody working in DPDK.
> > Please check that none of the PMD is merged without test app in the past 3-
> > 4yrs.
> 
> This is not true.
> 
> > > We can add the adjustment to increase validity for the next release to
> > > all the remaining crypto apps (test\test-crypto-perf).
> > >
> > > For now, we have one validation with l2fwd-crypto And any user can run
> > > it and see how to use mlx5 driver.
> >
> > The user cannot be sure of the basic things of a crypto PMD are in order or
> > not.
> > As l2fwd-crypto does not test every basic thing.
> > For eg. Session deletion, PMD stop, PMD close, PMD restart, setting up
> > multiple sessions(l2fwd support single session).
> >
> > Running datapath of a single use case is not sufficient for a PMD.
> > This is a POC and it need to comply with the environment.
> 
> There are only one\ two cases we need in this version and we tested them.
> 
> > I hope the doubts are clear now and we are OK to defer to next release.
> 
> As I said, we don't agree, it is yours.
> 
As I said, fix the issues, I can still merge today but not after that.

Regards,
Akhil

Re: [dpdk-dev] [EXT] [PATCH v3 03/15] crypto/mlx5: support session operations

[Thomas Monjalon]

12/05/2021 07:51, Matan Azrad:
> From: Akhil Goyal
> > > > > > > +static void
> > > > > > > +mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev,
> > > > > > > +                           struct rte_cryptodev_sym_session *sess) {
> > > > > > > +     struct mlx5_crypto_priv *priv = dev->data->dev_private;
> > > > > > > +     struct mlx5_crypto_session *sess_private_data =
> > > > > > > +                     get_sym_session_private_data(sess,
> > > > > > > +dev->driver_id);
> > > > > > > +
> > > > > > > +     if (unlikely(sess_private_data == NULL)) {
> > > > > > > +             DRV_LOG(ERR, "Failed to get session %p private data.",
> > > > > > > +                             sess_private_data);
> > > > > > > +             return;
> > > > > > > +     }
> > > > > > > +     mlx5_crypto_dek_destroy(priv, sess_private_data->dek);
> > > > > > > +     DRV_LOG(DEBUG, "Session %p was cleared.",
> > > > > > > + sess_private_data);
> > > }
> > > > > >
> > > > > > Memory leakage, mempool is not freed.
> > > > >
> > > > > Yes, good catch, this part was missed.
> > > > >
> > > > > > IMO, this driver is not properly tested with the unit test app.
> > > > >
> > > > > The only app we tested until now is l2fwd_crypto and it works fine!
> > > > > We can add it to doc.
> > > > >
> > > > > > Please add a note in the documentation that it is tested with autotest.
> > > > >
> > > > >
> > > > > The next app we want to test with, is test-crypto-perf.
> > > > >
> > > > I would recommend to run the test app first.
> > > > It will catch most of your basic bugs like the one above.
> > >
> > > It is too late for this, will add the test adjustment later.
> > 
> > Can we postpone the PMD to next release.
> 
> We can, but this is not our plan.
> We met all the DPDK rules to push it on time.

There is no guarantee a patch will be pushed. This is not due.

> > I believe test application makes
> > The PMD look robust as per the DPDK crypto PMD API usage.
> 
> Every test will add robustness to the PMD.
> 
> > I haven't seen a PMD getting merged without test app.
> 
> compress/mlx5, vdpa/mlx5, regex/mlx5, net/mlx5, vdev_netvsc....

There are functional test app for regex, and testpmd for net driver.
Running these tests are basic requirements.

> > And I apologize I did not mentioned it earlier, but it is kind of obvious thing to
> > run test app before sending it to upstream.
> 
> In fact, no, I added more than one PMD, no one require specific test.

Each driver class may have its own expectations.

> > L2fwd-crypto is not doing data validation hence you cannot be sure that it is
> > working fine as per other standard stacks like Linux.
> 
> It is not do data validation, but we check that the packet payload return back has the expected encrypted\decrypted data using open-ssl.
> Also for us it was mandatory requirement to check it.
> 
> For us, the current validation is enough, we don't support a lot of things in the crypto PMD currently, only one algorithm in the most basic modes.
> 
> For sure we will continue to add tests and to increase robustness.
> Also adding more features is in plan for future. 
> 
> If you postpone it, it yours, we don't agree with it.

Not clear who is "we".
I am OK with postponing this PMD.

Thank you Akhil for your time.

Re: [dpdk-dev] [EXT] [PATCH v3 03/15] crypto/mlx5: support session operations

[Matan Azrad]

From: Akhil Goyal
> > From: Akhil Goyal
> > > > > > > > +static void
> > > > > > > > +mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev,
> > > > > > > > +                           struct rte_cryptodev_sym_session *sess) {
> > > > > > > > +     struct mlx5_crypto_priv *priv = dev->data->dev_private;
> > > > > > > > +     struct mlx5_crypto_session *sess_private_data =
> > > > > > > > +                     get_sym_session_private_data(sess,
> > > > > > > > +dev->driver_id);
> > > > > > > > +
> > > > > > > > +     if (unlikely(sess_private_data == NULL)) {
> > > > > > > > +             DRV_LOG(ERR, "Failed to get session %p private data.",
> > > > > > > > +                             sess_private_data);
> > > > > > > > +             return;
> > > > > > > > +     }
> > > > > > > > +     mlx5_crypto_dek_destroy(priv, sess_private_data->dek);
> > > > > > > > +     DRV_LOG(DEBUG, "Session %p was cleared.",
> > > > > > > > + sess_private_data);
> > > > }
> > > > > > >
> > > > > > > Memory leakage, mempool is not freed.
> > > > > >
> > > > > > Yes, good catch, this part was missed.
> > > > > >
> > > > > > > IMO, this driver is not properly tested with the unit test app.
> > > > > >
> > > > > > The only app we tested until now is l2fwd_crypto and it works fine!
> > > > > > We can add it to doc.
> > > > > >
> > > > > > > Please add a note in the documentation that it is tested
> > > > > > > with
> > autotest.
> > > > > >
> > > > > >
> > > > > > The next app we want to test with, is test-crypto-perf.
> > > > > >
> > > > > I would recommend to run the test app first.
> > > > > It will catch most of your basic bugs like the one above.
> > > >
> > > > It is too late for this, will add the test adjustment later.
> > >
> > > Can we postpone the PMD to next release.
> >
> > We can, but this is not our plan.
> > We met all the DPDK rules to push it on time.
> 
> On time!! Really? Incomplete v1 was submitted before RC1, The complete
> PMD was submitted only during RC2.

Yes, on time, you got draft in RC1 -- most of your critical comments are relevant for that version. 
 

> > > I believe test application makes
> > > The PMD look robust as per the DPDK crypto PMD API usage.
> >
> > Every test will add robustness to the PMD.
> >
> > > I haven't seen a PMD getting merged without test app.
> >
> > compress/mlx5, vdpa/mlx5, regex/mlx5, net/mlx5, vdev_netvsc....
> >
> All these are 'not' crypto PMDs.
> For crypto, UT is a must. I would like to add a statement in the guidelines if it
> is not there.
> 
> > > And I apologize I did not mentioned it earlier, but it is kind of
> > > obvious thing
> > to
> > > run test app before sending it to upstream.
> >
> > In fact, no, I added more than one PMD, no one require specific test.
> >
> > > L2fwd-crypto is not doing data validation hence you cannot be sure
> > > that it
> > is
> > > working fine as per other standard stacks like Linux.
> >
> > It is not do data validation, but we check that the packet payload
> > return back has the expected encrypted\decrypted data using open-ssl.
> > Also for us it was mandatory requirement to check it.
> >
> > For us, the current validation is enough, we don't support a lot of
> > things in the crypto PMD currently, only one algorithm in the most basic
> modes.
> >
> > For sure we will continue to add tests and to increase robustness.
> > Also adding more features is in plan for future.
> >
> > If you postpone it, it yours, we don't agree with it.
> >
> Please fix the review comments by today, if you want it  to be merged in RC3.
> IMO, the driver is not ready to be merged.
> I want to postpone it as it is not feasible to fix all the comments by RC3
> timeline.

Yes, it is too late.
Let's continue in next release.

Thanks for your time.

> I believe all my comments are valid and need to be addressed.
> Can you get somebody else(outside Nvidia) to ack your patch and counter
> my comments?
> I will merge it without review based on that Ack.
> Regards,
> Akhil

[dpdk-dev] [PATCH v5 00/15] drivers: introduce mlx5 crypto PMD

[Shiri Kuzin]

Add a new PMD for Nvidia devices- crypto PMD.
The crypto PMD will be supported on Nvidia ConnectX6.
The crypto PMD will add the support of encryption and decryption using
the AES-XTS symmetric algorithm.
The crypto PMD requires rdma-core and uses mlx5 DevX.

v2:
Add data-path part.

v3:
Rebase.

v4:
- Rebase + Address the following Akhil comments:
- Set HW feature flag in the capability patch.
- Fix mp object release in session clear.
- Some spelling and word missing in doc.
- Squash data-unit adjustment to the session operations commit.
- Wording: device argument -> devarg.

v5: 
- Add mlx5 crypto tests into test library.
- Update documentation according to Akhil comments.
- Fix memory region management.
- Fix multi segment case in data-path code.
- Split documentation to the correct commits according to Akhil comments.
- Rebase to new version.
- Change license to Nvidia license.




Shiri Kuzin (10):
  drivers: introduce mlx5 crypto PMD
  crypto/mlx5: add DEK object management
  crypto/mlx5: add session operations
  crypto/mlx5: add basic operations
  crypto/mlx5: add queue pairs operations
  crypto/mlx5: add dev stop and start operations
  crypto/mlx5: add memory region management
  crypto/mlx5: create login object using DevX
  test/crypto: add mlx5 crypto driver
  test/crypto: add mlx5 multi segment tests

Suanming Mou (5):
  crypto/mlx5: add keytag devarg
  crypto/mlx5: add maximum segments devarg
  crypto/mlx5: add WQE set initialization
  crypto/mlx5: add enqueue and dequeue operations
  crypto/mlx5: add statistic get and reset operations

 MAINTAINERS                                 |    4 +
 app/test/meson.build                        |    1 +
 app/test/test_cryptodev.c                   |  611 ++++-
 app/test/test_cryptodev.h                   |    3 +-
 app/test/test_cryptodev_mlx5_test_vectors.h | 2499 +++++++++++++++++++
 doc/guides/cryptodevs/features/mlx5.ini     |   37 +
 doc/guides/cryptodevs/index.rst             |    1 +
 doc/guides/cryptodevs/mlx5.rst              |  150 ++
 doc/guides/rel_notes/release_21_08.rst      |    5 +
 drivers/common/mlx5/mlx5_common.h           |    1 +
 drivers/common/mlx5/mlx5_common_pci.c       |   14 +
 drivers/common/mlx5/mlx5_common_pci.h       |   21 +-
 drivers/crypto/meson.build                  |    1 +
 drivers/crypto/mlx5/meson.build             |   27 +
 drivers/crypto/mlx5/mlx5_crypto.c           | 1177 +++++++++
 drivers/crypto/mlx5/mlx5_crypto.h           |   91 +
 drivers/crypto/mlx5/mlx5_crypto_dek.c       |  136 +
 drivers/crypto/mlx5/mlx5_crypto_utils.h     |   19 +
 drivers/crypto/mlx5/version.map             |    3 +
 19 files changed, 4774 insertions(+), 27 deletions(-)
 create mode 100644 app/test/test_cryptodev_mlx5_test_vectors.h
 create mode 100644 doc/guides/cryptodevs/features/mlx5.ini
 create mode 100644 doc/guides/cryptodevs/mlx5.rst
 create mode 100644 drivers/crypto/mlx5/meson.build
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto.c
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto.h
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_dek.c
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_utils.h
 create mode 100644 drivers/crypto/mlx5/version.map

-- 
2.27.0

[dpdk-dev] [PATCH v5 01/15] drivers: introduce mlx5 crypto PMD

[Shiri Kuzin]

Add a new PMD for Mellanox devices- crypto PMD.

The crypto PMD will be supported starting Nvidia ConnectX6 and
BlueField2.

The crypto PMD will add the support of encryption and decryption using
the AES-XTS symmetric algorithm.

The crypto PMD requires rdma-core and uses mlx5 DevX.

This patch adds the PCI probing, basic functions, build files and
log utility.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 MAINTAINERS                             |   4 +
 doc/guides/cryptodevs/features/mlx5.ini |  27 +++
 doc/guides/cryptodevs/index.rst         |   1 +
 doc/guides/cryptodevs/mlx5.rst          |  63 ++++++
 doc/guides/rel_notes/release_21_08.rst  |   5 +
 drivers/common/mlx5/mlx5_common.h       |   1 +
 drivers/common/mlx5/mlx5_common_pci.c   |  14 ++
 drivers/common/mlx5/mlx5_common_pci.h   |  21 +-
 drivers/crypto/meson.build              |   1 +
 drivers/crypto/mlx5/meson.build         |  26 +++
 drivers/crypto/mlx5/mlx5_crypto.c       | 275 ++++++++++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto_utils.h |  19 ++
 drivers/crypto/mlx5/version.map         |   3 +
 13 files changed, 450 insertions(+), 10 deletions(-)
 create mode 100644 doc/guides/cryptodevs/features/mlx5.ini
 create mode 100644 doc/guides/cryptodevs/mlx5.rst
 create mode 100644 drivers/crypto/mlx5/meson.build
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto.c
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_utils.h
 create mode 100644 drivers/crypto/mlx5/version.map

diff --git a/MAINTAINERS b/MAINTAINERS
index 5877a16971..c0e2aa1f9f 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -1080,6 +1080,10 @@ F: drivers/crypto/octeontx2/
 F: doc/guides/cryptodevs/octeontx2.rst
 F: doc/guides/cryptodevs/features/octeontx2.ini
 
+Mellanox mlx5
+M: Matan Azrad <matan@nvidia.com>
+F: drivers/crypto/mlx5/
+
 Null Crypto
 M: Declan Doherty <declan.doherty@intel.com>
 F: drivers/crypto/null/
diff --git a/doc/guides/cryptodevs/features/mlx5.ini b/doc/guides/cryptodevs/features/mlx5.ini
new file mode 100644
index 0000000000..ceadd967b6
--- /dev/null
+++ b/doc/guides/cryptodevs/features/mlx5.ini
@@ -0,0 +1,27 @@
+;
+; Features of a mlx5 crypto driver.
+;
+; Refer to default.ini for the full list of available PMD features.
+;
+[Features]
+HW Accelerated         = Y
+
+;
+; Supported crypto algorithms of a mlx5 crypto driver.
+;
+[Cipher]
+
+;
+; Supported authentication algorithms of a mlx5 crypto driver.
+;
+[Auth]
+
+;
+; Supported AEAD algorithms of a mlx5 crypto driver.
+;
+[AEAD]
+
+;
+; Supported Asymmetric algorithms of a mlx5 crypto driver.
+;
+[Asymmetric]
diff --git a/doc/guides/cryptodevs/index.rst b/doc/guides/cryptodevs/index.rst
index 279f56a002..747409c441 100644
--- a/doc/guides/cryptodevs/index.rst
+++ b/doc/guides/cryptodevs/index.rst
@@ -22,6 +22,7 @@ Crypto Device Drivers
     octeontx
     octeontx2
     openssl
+    mlx5
     mvsam
     nitrox
     null
diff --git a/doc/guides/cryptodevs/mlx5.rst b/doc/guides/cryptodevs/mlx5.rst
new file mode 100644
index 0000000000..05a0a449e2
--- /dev/null
+++ b/doc/guides/cryptodevs/mlx5.rst
@@ -0,0 +1,63 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+   Copyright (c) 2021 NVIDIA Corporation & Affiliates
+
+.. include:: <isonum.txt>
+
+MLX5 Crypto Driver
+==================
+
+The MLX5 crypto driver library
+(**librte_crypto_mlx5**) provides support for **Mellanox ConnectX-6**
+family adapters.
+
+Overview
+--------
+
+The device can provide disk encryption services, allowing data encryption
+and decryption towards a disk. Having all encryption/decryption
+operations done in a single device can reduce cost and overheads of the related
+FIPS certification, as ConnectX-6 is FIPS 140-2 level-2 ready.
+The encryption cipher is AES-XTS of 256/512 bit key size.
+
+MKEY is a memory region object in the hardware, that holds address translation information and
+attributes per memory area. Its ID must be tied to addresses provided to the hardware.
+The encryption operations are performed with MKEY read/write transactions, when
+the MKEY is configured to perform crypto operations.
+
+The encryption does not require text to be aligned to the AES block size (128b).
+
+The PMD uses libibverbs and libmlx5 to access the device firmware or to
+access the hardware components directly.
+There are different levels of objects and bypassing abilities.
+To get the best performances:
+
+- Verbs is a complete high-level generic API.
+- Direct Verbs is a device-specific API.
+- DevX allows to access firmware objects.
+
+Enabling librte_crypto_mlx5 causes DPDK applications to be linked against
+libibverbs.
+
+
+Driver options
+--------------
+
+- ``class`` parameter [string]
+
+  Select the class of the driver that should probe the device.
+  `crypto` for the mlx5 crypto driver.
+
+
+Supported NICs
+--------------
+
+* Mellanox\ |reg| ConnectX\ |reg|-6 200G MCX654106A-HCAT (2x200G)
+
+Prerequisites
+-------------
+
+- Mellanox OFED version: **5.3**
+  see :doc:`../../nics/mlx5` guide for more Mellanox OFED details.
+
+- Compilation can be done also with rdma-core v15+.
+  see :doc:`../../nics/mlx5` guide for more rdma-core details.
diff --git a/doc/guides/rel_notes/release_21_08.rst b/doc/guides/rel_notes/release_21_08.rst
index a6ecfdf3ce..e12341fe55 100644
--- a/doc/guides/rel_notes/release_21_08.rst
+++ b/doc/guides/rel_notes/release_21_08.rst
@@ -55,6 +55,11 @@ New Features
      Also, make sure to start the actual text at the margin.
      =======================================================
 
+* **Added support for Nvidia crypto device driver.**
+
+  * Added mlx5 crypto driver to support AES-XTS cipher operations.
+    The first device to support it is ConnectX-6.
+
 
 Removed Items
 -------------
diff --git a/drivers/common/mlx5/mlx5_common.h b/drivers/common/mlx5/mlx5_common.h
index 1fbefe0fa6..89aca32305 100644
--- a/drivers/common/mlx5/mlx5_common.h
+++ b/drivers/common/mlx5/mlx5_common.h
@@ -216,6 +216,7 @@ enum mlx5_class {
 	MLX5_CLASS_VDPA = RTE_BIT64(1),
 	MLX5_CLASS_REGEX = RTE_BIT64(2),
 	MLX5_CLASS_COMPRESS = RTE_BIT64(3),
+	MLX5_CLASS_CRYPTO = RTE_BIT64(4),
 };
 
 #define MLX5_DBR_SIZE RTE_CACHE_LINE_SIZE
diff --git a/drivers/common/mlx5/mlx5_common_pci.c b/drivers/common/mlx5/mlx5_common_pci.c
index 3f16cd21cf..8a47afee20 100644
--- a/drivers/common/mlx5/mlx5_common_pci.c
+++ b/drivers/common/mlx5/mlx5_common_pci.c
@@ -31,6 +31,7 @@ static const struct {
 	{ .name = "net", .driver_class = MLX5_CLASS_NET },
 	{ .name = "regex", .driver_class = MLX5_CLASS_REGEX },
 	{ .name = "compress", .driver_class = MLX5_CLASS_COMPRESS },
+	{ .name = "crypto", .driver_class = MLX5_CLASS_CRYPTO },
 };
 
 static const unsigned int mlx5_class_combinations[] = {
@@ -38,13 +39,26 @@ static const unsigned int mlx5_class_combinations[] = {
 	MLX5_CLASS_VDPA,
 	MLX5_CLASS_REGEX,
 	MLX5_CLASS_COMPRESS,
+	MLX5_CLASS_CRYPTO,
 	MLX5_CLASS_NET | MLX5_CLASS_REGEX,
 	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX,
 	MLX5_CLASS_NET | MLX5_CLASS_COMPRESS,
 	MLX5_CLASS_VDPA | MLX5_CLASS_COMPRESS,
 	MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
+	MLX5_CLASS_NET | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_VDPA | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO,
 	MLX5_CLASS_NET | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
 	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
+	MLX5_CLASS_NET | MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_NET | MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_VDPA | MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_NET | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS |
+	MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS |
+	MLX5_CLASS_CRYPTO,
 	/* New class combination should be added here. */
 };
 
diff --git a/drivers/common/mlx5/mlx5_common_pci.h b/drivers/common/mlx5/mlx5_common_pci.h
index de89bb98bc..cb8d2f5f87 100644
--- a/drivers/common/mlx5/mlx5_common_pci.h
+++ b/drivers/common/mlx5/mlx5_common_pci.h
@@ -9,17 +9,18 @@
  * @file
  *
  * RTE Mellanox PCI Driver Interface
- * Mellanox ConnectX PCI device supports multiple class: net,vdpa,regex and
- * compress devices. This layer enables creating such multiple class of devices
- * on a single PCI device by allowing to bind multiple class specific device
- * driver to attach to mlx5_pci driver.
+ * Mellanox ConnectX PCI device supports multiple class: net,vdpa,regex,compress
+ * and crypto devices. This layer enables creating such multiple class of
+ * devices on a single PCI device by allowing to bind multiple class specific
+ * device driver to attach to mlx5_pci driver.
  *
- * -----------    ------------    -------------    ----------------
- * |   mlx5  |    |   mlx5   |    |   mlx5    |    |     mlx5     |
- * | net pmd |    | vdpa pmd |    | regex pmd |    | compress pmd |
- * -----------    ------------    -------------    ----------------
- *      \              \                    /              /
- *       \              \                  /              /
+ * --------    --------    ---------    ------------    ----------
+ * | mlx5 |    | mlx5 |    | mlx5  |    |   mlx5   |    |  mlx5  |
+ * | net  |    | vdpa |    | regex |    | compress |    | crypto |
+ * | pmd  |    | pmd  |    |  pmd  |    |   pmd    |    |  pmd   |
+ * --------    --------    ---------    ------------    ----------
+ *      \              \         |          /              /
+ *       \              \        |         /              /
  *        \              \_--------------_/              /
  *         \_______________|   mlx5     |_______________/
  *                         | pci common |
diff --git a/drivers/crypto/meson.build b/drivers/crypto/meson.build
index b9fdf9392f..6951607def 100644
--- a/drivers/crypto/meson.build
+++ b/drivers/crypto/meson.build
@@ -15,6 +15,7 @@ drivers = [
         'dpaa_sec',
         'dpaa2_sec',
         'kasumi',
+        'mlx5',
         'mvsam',
         'nitrox',
         'null',
diff --git a/drivers/crypto/mlx5/meson.build b/drivers/crypto/mlx5/meson.build
new file mode 100644
index 0000000000..6fd70bc477
--- /dev/null
+++ b/drivers/crypto/mlx5/meson.build
@@ -0,0 +1,26 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2021 NVIDIA Corporation & Affiliates
+
+if not is_linux
+	build = false
+	reason = 'only supported on Linux'
+	subdir_done()
+endif
+
+fmt_name = 'mlx5_crypto'
+deps += ['common_mlx5', 'eal', 'cryptodev']
+sources = files(
+	'mlx5_crypto.c',
+)
+cflags_options = [
+	'-std=c11',
+	'-Wno-strict-prototypes',
+	'-D_BSD_SOURCE',
+	'-D_DEFAULT_SOURCE',
+	'-D_XOPEN_SOURCE=600',
+]
+foreach option:cflags_options
+	if cc.has_argument(option)
+		cflags += option
+	endif
+endforeach
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
new file mode 100644
index 0000000000..f89d508a7e
--- /dev/null
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -0,0 +1,275 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2021 NVIDIA Corporation & Affiliates
+ */
+
+#include <rte_malloc.h>
+#include <rte_log.h>
+#include <rte_errno.h>
+#include <rte_pci.h>
+#include <rte_crypto.h>
+#include <rte_cryptodev.h>
+#include <rte_cryptodev_pmd.h>
+
+#include <mlx5_glue.h>
+#include <mlx5_common.h>
+#include <mlx5_common_pci.h>
+#include <mlx5_devx_cmds.h>
+#include <mlx5_common_os.h>
+
+#include "mlx5_crypto_utils.h"
+
+#define MLX5_CRYPTO_DRIVER_NAME mlx5_crypto
+#define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
+
+#define MLX5_CRYPTO_FEATURE_FLAGS \
+	RTE_CRYPTODEV_FF_HW_ACCELERATED
+
+struct mlx5_crypto_priv {
+	TAILQ_ENTRY(mlx5_crypto_priv) next;
+	struct ibv_context *ctx; /* Device context. */
+	struct rte_pci_device *pci_dev;
+	struct rte_cryptodev *crypto_dev;
+	void *uar; /* User Access Region. */
+	uint32_t pdn; /* Protection Domain number. */
+	struct ibv_pd *pd;
+};
+
+TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
+				TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
+static pthread_mutex_t priv_list_lock = PTHREAD_MUTEX_INITIALIZER;
+
+int mlx5_crypto_logtype;
+
+uint8_t mlx5_crypto_driver_id;
+
+static const char mlx5_crypto_drv_name[] = RTE_STR(MLX5_CRYPTO_DRIVER_NAME);
+
+static const struct rte_driver mlx5_drv = {
+	.name = mlx5_crypto_drv_name,
+	.alias = mlx5_crypto_drv_name
+};
+
+static struct cryptodev_driver mlx5_cryptodev_driver;
+
+static struct rte_cryptodev_ops mlx5_crypto_ops = {
+	.dev_configure			= NULL,
+	.dev_start			= NULL,
+	.dev_stop			= NULL,
+	.dev_close			= NULL,
+	.dev_infos_get			= NULL,
+	.stats_get			= NULL,
+	.stats_reset			= NULL,
+	.queue_pair_setup		= NULL,
+	.queue_pair_release		= NULL,
+	.sym_session_get_size		= NULL,
+	.sym_session_configure		= NULL,
+	.sym_session_clear		= NULL,
+	.sym_get_raw_dp_ctx_size	= NULL,
+	.sym_configure_raw_dp_ctx	= NULL,
+};
+
+static void
+mlx5_crypto_hw_global_release(struct mlx5_crypto_priv *priv)
+{
+	if (priv->pd != NULL) {
+		claim_zero(mlx5_glue->dealloc_pd(priv->pd));
+		priv->pd = NULL;
+	}
+	if (priv->uar != NULL) {
+		mlx5_glue->devx_free_uar(priv->uar);
+		priv->uar = NULL;
+	}
+}
+
+static int
+mlx5_crypto_pd_create(struct mlx5_crypto_priv *priv)
+{
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+	struct mlx5dv_obj obj;
+	struct mlx5dv_pd pd_info;
+	int ret;
+
+	priv->pd = mlx5_glue->alloc_pd(priv->ctx);
+	if (priv->pd == NULL) {
+		DRV_LOG(ERR, "Failed to allocate PD.");
+		return errno ? -errno : -ENOMEM;
+	}
+	obj.pd.in = priv->pd;
+	obj.pd.out = &pd_info;
+	ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_PD);
+	if (ret != 0) {
+		DRV_LOG(ERR, "Fail to get PD object info.");
+		mlx5_glue->dealloc_pd(priv->pd);
+		priv->pd = NULL;
+		return -errno;
+	}
+	priv->pdn = pd_info.pdn;
+	return 0;
+#else
+	(void)priv;
+	DRV_LOG(ERR, "Cannot get pdn - no DV support.");
+	return -ENOTSUP;
+#endif /* HAVE_IBV_FLOW_DV_SUPPORT */
+}
+
+static int
+mlx5_crypto_hw_global_prepare(struct mlx5_crypto_priv *priv)
+{
+	if (mlx5_crypto_pd_create(priv) != 0)
+		return -1;
+	priv->uar = mlx5_devx_alloc_uar(priv->ctx, -1);
+	if (priv->uar == NULL || mlx5_os_get_devx_uar_reg_addr(priv->uar) ==
+	    NULL) {
+		rte_errno = errno;
+		claim_zero(mlx5_glue->dealloc_pd(priv->pd));
+		DRV_LOG(ERR, "Failed to allocate UAR.");
+		return -1;
+	}
+	return 0;
+}
+
+/**
+ * DPDK callback to register a PCI device.
+ *
+ * This function spawns crypto device out of a given PCI device.
+ *
+ * @param[in] pci_drv
+ *   PCI driver structure (mlx5_crypto_driver).
+ * @param[in] pci_dev
+ *   PCI device information.
+ *
+ * @return
+ *   0 on success, 1 to skip this driver, a negative errno value otherwise
+ *   and rte_errno is set.
+ */
+static int
+mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
+			struct rte_pci_device *pci_dev)
+{
+	struct ibv_device *ibv;
+	struct rte_cryptodev *crypto_dev;
+	struct ibv_context *ctx;
+	struct mlx5_crypto_priv *priv;
+	struct mlx5_hca_attr attr = { 0 };
+	struct rte_cryptodev_pmd_init_params init_params = {
+		.name = "",
+		.private_data_size = sizeof(struct mlx5_crypto_priv),
+		.socket_id = pci_dev->device.numa_node,
+		.max_nb_queue_pairs =
+				RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
+	};
+	RTE_SET_USED(pci_drv);
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+		DRV_LOG(ERR, "Non-primary process type is not supported.");
+		rte_errno = ENOTSUP;
+		return -rte_errno;
+	}
+	ibv = mlx5_os_get_ibv_device(&pci_dev->addr);
+	if (ibv == NULL) {
+		DRV_LOG(ERR, "No matching IB device for PCI slot "
+			PCI_PRI_FMT ".", pci_dev->addr.domain,
+			pci_dev->addr.bus, pci_dev->addr.devid,
+			pci_dev->addr.function);
+		return -rte_errno;
+	}
+	DRV_LOG(INFO, "PCI information matches for device \"%s\".", ibv->name);
+	ctx = mlx5_glue->dv_open_device(ibv);
+	if (ctx == NULL) {
+		DRV_LOG(ERR, "Failed to open IB device \"%s\".", ibv->name);
+		rte_errno = ENODEV;
+		return -rte_errno;
+	}
+	if (mlx5_devx_cmd_query_hca_attr(ctx, &attr) != 0 ||
+	    attr.crypto == 0 || attr.aes_xts == 0) {
+		DRV_LOG(ERR, "Not enough capabilities to support crypto "
+			"operations, maybe old FW/OFED version?");
+		claim_zero(mlx5_glue->close_device(ctx));
+		rte_errno = ENOTSUP;
+		return -ENOTSUP;
+	}
+	crypto_dev = rte_cryptodev_pmd_create(ibv->name, &pci_dev->device,
+					&init_params);
+	if (crypto_dev == NULL) {
+		DRV_LOG(ERR, "Failed to create device \"%s\".", ibv->name);
+		claim_zero(mlx5_glue->close_device(ctx));
+		return -ENODEV;
+	}
+	DRV_LOG(INFO,
+		"Crypto device %s was created successfully.", ibv->name);
+	crypto_dev->dev_ops = &mlx5_crypto_ops;
+	crypto_dev->dequeue_burst = NULL;
+	crypto_dev->enqueue_burst = NULL;
+	crypto_dev->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
+	crypto_dev->driver_id = mlx5_crypto_driver_id;
+	priv = crypto_dev->data->dev_private;
+	priv->ctx = ctx;
+	priv->pci_dev = pci_dev;
+	priv->crypto_dev = crypto_dev;
+	if (mlx5_crypto_hw_global_prepare(priv) != 0) {
+		rte_cryptodev_pmd_destroy(priv->crypto_dev);
+		claim_zero(mlx5_glue->close_device(priv->ctx));
+		return -1;
+	}
+	pthread_mutex_lock(&priv_list_lock);
+	TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next);
+	pthread_mutex_unlock(&priv_list_lock);
+	return 0;
+}
+
+static int
+mlx5_crypto_pci_remove(struct rte_pci_device *pdev)
+{
+	struct mlx5_crypto_priv *priv = NULL;
+
+	pthread_mutex_lock(&priv_list_lock);
+	TAILQ_FOREACH(priv, &mlx5_crypto_priv_list, next)
+		if (rte_pci_addr_cmp(&priv->pci_dev->addr, &pdev->addr) != 0)
+			break;
+	if (priv)
+		TAILQ_REMOVE(&mlx5_crypto_priv_list, priv, next);
+	pthread_mutex_unlock(&priv_list_lock);
+	if (priv) {
+		mlx5_crypto_hw_global_release(priv);
+		rte_cryptodev_pmd_destroy(priv->crypto_dev);
+		claim_zero(mlx5_glue->close_device(priv->ctx));
+	}
+	return 0;
+}
+
+static const struct rte_pci_id mlx5_crypto_pci_id_map[] = {
+		{
+			RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
+					PCI_DEVICE_ID_MELLANOX_CONNECTX6)
+		},
+		{
+			.vendor_id = 0
+		}
+	};
+
+static struct mlx5_pci_driver mlx5_crypto_driver = {
+	.driver_class = MLX5_CLASS_CRYPTO,
+	.pci_driver = {
+		.driver = {
+			.name = RTE_STR(MLX5_CRYPTO_DRIVER_NAME),
+		},
+		.id_table = mlx5_crypto_pci_id_map,
+		.probe = mlx5_crypto_pci_probe,
+		.remove = mlx5_crypto_pci_remove,
+		.drv_flags = 0,
+	},
+};
+
+RTE_INIT(rte_mlx5_crypto_init)
+{
+	mlx5_common_init();
+	if (mlx5_glue != NULL)
+		mlx5_pci_driver_register(&mlx5_crypto_driver);
+}
+
+RTE_PMD_REGISTER_CRYPTO_DRIVER(mlx5_cryptodev_driver, mlx5_drv,
+			       mlx5_crypto_driver_id);
+
+RTE_LOG_REGISTER(mlx5_crypto_logtype, MLX5_CRYPTO_LOG_NAME, NOTICE)
+RTE_PMD_EXPORT_NAME(MLX5_CRYPTO_DRIVER_NAME, __COUNTER__);
+RTE_PMD_REGISTER_PCI_TABLE(MLX5_CRYPTO_DRIVER_NAME, mlx5_crypto_pci_id_map);
+RTE_PMD_REGISTER_KMOD_DEP(MLX5_CRYPTO_DRIVER_NAME, "* ib_uverbs & mlx5_core & mlx5_ib");
diff --git a/drivers/crypto/mlx5/mlx5_crypto_utils.h b/drivers/crypto/mlx5/mlx5_crypto_utils.h
new file mode 100644
index 0000000000..b6c60ca782
--- /dev/null
+++ b/drivers/crypto/mlx5/mlx5_crypto_utils.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2021 NVIDIA Corporation & Affiliates
+ */
+
+#ifndef RTE_PMD_MLX5_CRYPTO_UTILS_H_
+#define RTE_PMD_MLX5_CRYPTO_UTILS_H_
+
+#include <mlx5_common.h>
+
+extern int mlx5_crypto_logtype;
+
+#define MLX5_CRYPTO_LOG_PREFIX "mlx5_crypto"
+/* Generic printf()-like logging macro with automatic line feed. */
+#define DRV_LOG(level, ...) \
+	PMD_DRV_LOG_(level, mlx5_crypto_logtype, MLX5_CRYPTO_LOG_PREFIX, \
+		__VA_ARGS__ PMD_DRV_LOG_STRIP PMD_DRV_LOG_OPAREN, \
+		PMD_DRV_LOG_CPAREN)
+
+#endif /* RTE_PMD_MLX5_CRYPTO_UTILS_H_ */
diff --git a/drivers/crypto/mlx5/version.map b/drivers/crypto/mlx5/version.map
new file mode 100644
index 0000000000..4a76d1d52d
--- /dev/null
+++ b/drivers/crypto/mlx5/version.map
@@ -0,0 +1,3 @@
+DPDK_21 {
+	local: *;
+};
-- 
2.27.0

[dpdk-dev] [PATCH v5 02/15] crypto/mlx5: add DEK object management

[Shiri Kuzin]

A DEK(Data encryption Key) is an mlx5 HW object which represents the
cipher algorithm key.
The DEKs are used during data encryption/decryption operations.

In symmetric algorithms like AES-STS, we use the same DEK for both
encryption and decryption.

Use the mlx5 hash-list tool to manage the DEK objects in the PMD.

Provide the compare, create and destroy functions to manage DEKs in
hash-list and introduce an internal API to setup and unset the DEK
management and to prepare and destroy specific DEK object.

The DEK hash-list will be created in dev_configure routine and
destroyed in dev_close routine.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/meson.build       |   1 +
 drivers/crypto/mlx5/mlx5_crypto.c     |  42 +++++---
 drivers/crypto/mlx5/mlx5_crypto.h     |  51 ++++++++++
 drivers/crypto/mlx5/mlx5_crypto_dek.c | 136 ++++++++++++++++++++++++++
 4 files changed, 214 insertions(+), 16 deletions(-)
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto.h
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_dek.c

diff --git a/drivers/crypto/mlx5/meson.build b/drivers/crypto/mlx5/meson.build
index 6fd70bc477..d55cdbfe6f 100644
--- a/drivers/crypto/mlx5/meson.build
+++ b/drivers/crypto/mlx5/meson.build
@@ -11,6 +11,7 @@ fmt_name = 'mlx5_crypto'
 deps += ['common_mlx5', 'eal', 'cryptodev']
 sources = files(
 	'mlx5_crypto.c',
+	'mlx5_crypto_dek.c',
 )
 cflags_options = [
 	'-std=c11',
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index f89d508a7e..dfc3da3d3f 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -3,12 +3,9 @@
  */
 
 #include <rte_malloc.h>
-#include <rte_log.h>
 #include <rte_errno.h>
+#include <rte_log.h>
 #include <rte_pci.h>
-#include <rte_crypto.h>
-#include <rte_cryptodev.h>
-#include <rte_cryptodev_pmd.h>
 
 #include <mlx5_glue.h>
 #include <mlx5_common.h>
@@ -17,6 +14,7 @@
 #include <mlx5_common_os.h>
 
 #include "mlx5_crypto_utils.h"
+#include "mlx5_crypto.h"
 
 #define MLX5_CRYPTO_DRIVER_NAME mlx5_crypto
 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
@@ -24,16 +22,6 @@
 #define MLX5_CRYPTO_FEATURE_FLAGS \
 	RTE_CRYPTODEV_FF_HW_ACCELERATED
 
-struct mlx5_crypto_priv {
-	TAILQ_ENTRY(mlx5_crypto_priv) next;
-	struct ibv_context *ctx; /* Device context. */
-	struct rte_pci_device *pci_dev;
-	struct rte_cryptodev *crypto_dev;
-	void *uar; /* User Access Region. */
-	uint32_t pdn; /* Protection Domain number. */
-	struct ibv_pd *pd;
-};
-
 TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
 				TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
 static pthread_mutex_t priv_list_lock = PTHREAD_MUTEX_INITIALIZER;
@@ -51,11 +39,33 @@ static const struct rte_driver mlx5_drv = {
 
 static struct cryptodev_driver mlx5_cryptodev_driver;
 
+static int
+mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
+		struct rte_cryptodev_config *config __rte_unused)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+
+	if (mlx5_crypto_dek_setup(priv) != 0) {
+		DRV_LOG(ERR, "Dek hash list creation has failed.");
+		return -ENOMEM;
+	}
+	return 0;
+}
+
+static int
+mlx5_crypto_dev_close(struct rte_cryptodev *dev)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+
+	mlx5_crypto_dek_unset(priv);
+	return 0;
+}
+
 static struct rte_cryptodev_ops mlx5_crypto_ops = {
-	.dev_configure			= NULL,
+	.dev_configure			= mlx5_crypto_dev_configure,
 	.dev_start			= NULL,
 	.dev_stop			= NULL,
-	.dev_close			= NULL,
+	.dev_close			= mlx5_crypto_dev_close,
 	.dev_infos_get			= NULL,
 	.stats_get			= NULL,
 	.stats_reset			= NULL,
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
new file mode 100644
index 0000000000..5a54cb0dca
--- /dev/null
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -0,0 +1,51 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2021 NVIDIA Corporation & Affiliates
+ */
+
+#ifndef MLX5_CRYPTO_H_
+#define MLX5_CRYPTO_H_
+
+#include <stdbool.h>
+
+#include <rte_cryptodev.h>
+#include <rte_cryptodev_pmd.h>
+
+#include <mlx5_common_utils.h>
+
+#define MLX5_CRYPTO_DEK_HTABLE_SZ (1 << 11)
+#define MLX5_CRYPTO_KEY_LENGTH 80
+
+struct mlx5_crypto_priv {
+	TAILQ_ENTRY(mlx5_crypto_priv) next;
+	struct ibv_context *ctx; /* Device context. */
+	struct rte_pci_device *pci_dev;
+	struct rte_cryptodev *crypto_dev;
+	void *uar; /* User Access Region. */
+	uint32_t pdn; /* Protection Domain number. */
+	struct ibv_pd *pd;
+	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
+};
+
+struct mlx5_crypto_dek {
+	struct mlx5_hlist_entry entry; /* Pointer to DEK hash list entry. */
+	struct mlx5_devx_obj *obj; /* Pointer to DEK DevX object. */
+	uint8_t data[MLX5_CRYPTO_KEY_LENGTH]; /* DEK key data. */
+	bool size_is_48; /* Whether the key\data size is 48 bytes or not. */
+} __rte_cache_aligned;
+
+int
+mlx5_crypto_dek_destroy(struct mlx5_crypto_priv *priv,
+			struct mlx5_crypto_dek *dek);
+
+struct mlx5_crypto_dek *
+mlx5_crypto_dek_prepare(struct mlx5_crypto_priv *priv,
+			struct rte_crypto_cipher_xform *cipher);
+
+int
+mlx5_crypto_dek_setup(struct mlx5_crypto_priv *priv);
+
+void
+mlx5_crypto_dek_unset(struct mlx5_crypto_priv *priv);
+
+#endif /* MLX5_CRYPTO_H_ */
+
diff --git a/drivers/crypto/mlx5/mlx5_crypto_dek.c b/drivers/crypto/mlx5/mlx5_crypto_dek.c
new file mode 100644
index 0000000000..e09c20c844
--- /dev/null
+++ b/drivers/crypto/mlx5/mlx5_crypto_dek.c
@@ -0,0 +1,136 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2021 NVIDIA Corporation & Affiliates
+ */
+
+#include <rte_ip.h>
+#include <rte_common.h>
+#include <rte_errno.h>
+#include <rte_log.h>
+
+#include <mlx5_prm.h>
+#include <mlx5_devx_cmds.h>
+
+#include "mlx5_crypto_utils.h"
+#include "mlx5_crypto.h"
+
+struct mlx5_crypto_dek_ctx {
+	struct rte_crypto_cipher_xform *cipher;
+	struct mlx5_crypto_priv *priv;
+};
+
+int
+mlx5_crypto_dek_destroy(struct mlx5_crypto_priv *priv,
+			struct mlx5_crypto_dek *dek)
+{
+	return mlx5_hlist_unregister(priv->dek_hlist, &dek->entry);
+}
+
+struct mlx5_crypto_dek *
+mlx5_crypto_dek_prepare(struct mlx5_crypto_priv *priv,
+			struct rte_crypto_cipher_xform *cipher)
+{
+	struct mlx5_hlist *dek_hlist = priv->dek_hlist;
+	struct mlx5_crypto_dek_ctx dek_ctx = {
+		.cipher = cipher,
+		.priv = priv,
+	};
+	struct rte_crypto_cipher_xform *cipher_ctx = cipher;
+	uint64_t key64 = __rte_raw_cksum(cipher_ctx->key.data,
+					 cipher_ctx->key.length, 0);
+	struct mlx5_hlist_entry *entry = mlx5_hlist_register(dek_hlist,
+							     key64, &dek_ctx);
+
+	return entry == NULL ? NULL :
+			     container_of(entry, struct mlx5_crypto_dek, entry);
+}
+
+static int
+mlx5_crypto_dek_match_cb(struct mlx5_hlist *list __rte_unused,
+			 struct mlx5_hlist_entry *entry,
+			 uint64_t key __rte_unused, void *cb_ctx)
+{
+	struct mlx5_crypto_dek_ctx *ctx = cb_ctx;
+	struct rte_crypto_cipher_xform *cipher_ctx = ctx->cipher;
+	struct mlx5_crypto_dek *dek =
+			container_of(entry, typeof(*dek), entry);
+	uint32_t key_len = dek->size_is_48 ? 48 : 80;
+
+	if (key_len != cipher_ctx->key.length)
+		return -1;
+	return memcmp(cipher_ctx->key.data, dek->data, key_len);
+}
+
+static struct mlx5_hlist_entry *
+mlx5_crypto_dek_create_cb(struct mlx5_hlist *list __rte_unused,
+			  uint64_t key __rte_unused, void *cb_ctx)
+{
+	struct mlx5_crypto_dek_ctx *ctx = cb_ctx;
+	struct rte_crypto_cipher_xform *cipher_ctx = ctx->cipher;
+	struct mlx5_crypto_dek *dek = rte_zmalloc(__func__, sizeof(*dek),
+						  RTE_CACHE_LINE_SIZE);
+	struct mlx5_devx_dek_attr dek_attr = {
+		.pd = ctx->priv->pdn,
+		.key_purpose = MLX5_CRYPTO_KEY_PURPOSE_AES_XTS,
+		.has_keytag = 1,
+	};
+
+	if (dek == NULL) {
+		DRV_LOG(ERR, "Failed to allocate dek memory.");
+		return NULL;
+	}
+	switch (cipher_ctx->key.length) {
+	case 48:
+		dek->size_is_48 = true;
+		dek_attr.key_size = MLX5_CRYPTO_KEY_SIZE_128b;
+		break;
+	case 80:
+		dek->size_is_48 = false;
+		dek_attr.key_size = MLX5_CRYPTO_KEY_SIZE_256b;
+		break;
+	default:
+		DRV_LOG(ERR, "Key size not supported.");
+		return NULL;
+	}
+	memcpy(&dek_attr.key, cipher_ctx->key.data, cipher_ctx->key.length);
+	dek->obj = mlx5_devx_cmd_create_dek_obj(ctx->priv->ctx, &dek_attr);
+	if (dek->obj == NULL) {
+		rte_free(dek);
+		return NULL;
+	}
+	memcpy(&dek->data, cipher_ctx->key.data, cipher_ctx->key.length);
+	return &dek->entry;
+}
+
+static void
+mlx5_crypto_dek_remove_cb(struct mlx5_hlist *list __rte_unused,
+			  struct mlx5_hlist_entry *entry)
+{
+	struct mlx5_crypto_dek *dek =
+		container_of(entry, typeof(*dek), entry);
+
+	claim_zero(mlx5_devx_cmd_destroy(dek->obj));
+	rte_free(dek);
+}
+
+
+int
+mlx5_crypto_dek_setup(struct mlx5_crypto_priv *priv)
+{
+	priv->dek_hlist = mlx5_hlist_create("dek_hlist",
+				 MLX5_CRYPTO_DEK_HTABLE_SZ,
+				 0, MLX5_HLIST_WRITE_MOST |
+				 MLX5_HLIST_DIRECT_KEY,
+				 mlx5_crypto_dek_create_cb,
+				 mlx5_crypto_dek_match_cb,
+				 mlx5_crypto_dek_remove_cb);
+	if (priv->dek_hlist == NULL)
+		return -1;
+	return 0;
+}
+
+void
+mlx5_crypto_dek_unset(struct mlx5_crypto_priv *priv)
+{
+	mlx5_hlist_destroy(priv->dek_hlist);
+	priv->dek_hlist = NULL;
+}
-- 
2.27.0

[dpdk-dev] [PATCH v5 03/15] crypto/mlx5: add session operations

[Shiri Kuzin]

Sessions are used in symmetric transformations in order to prepare
objects and data for packet processing stage.

A mlx5 session includes iv_offset, pointer to mlx5_crypto_dek struct,
bsf_size, bsf_p_type, block size index, encryption_order and encryption
standard.

Implement the next session operations:
        mlx5_crypto_sym_session_get_size- returns the size of the mlx5
	session struct.
	mlx5_crypto_sym_session_configure- prepares the DEK hash-list
	and saves all the session data.
	mlx5_crypto_sym_session_clear - destroys the DEK hash-list.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 doc/guides/cryptodevs/features/mlx5.ini |   5 +
 doc/guides/cryptodevs/mlx5.rst          |  10 ++
 drivers/crypto/mlx5/mlx5_crypto.c       | 172 +++++++++++++++++++++++-
 3 files changed, 182 insertions(+), 5 deletions(-)

diff --git a/doc/guides/cryptodevs/features/mlx5.ini b/doc/guides/cryptodevs/features/mlx5.ini
index ceadd967b6..bd757b5211 100644
--- a/doc/guides/cryptodevs/features/mlx5.ini
+++ b/doc/guides/cryptodevs/features/mlx5.ini
@@ -4,12 +4,17 @@
 ; Refer to default.ini for the full list of available PMD features.
 ;
 [Features]
+Symmetric crypto       = Y
 HW Accelerated         = Y
+Cipher multiple data units = Y
+Cipher wrapped key     = Y
 
 ;
 ; Supported crypto algorithms of a mlx5 crypto driver.
 ;
 [Cipher]
+AES XTS (128)  = Y
+AES XTS (256)  = Y
 
 ;
 ; Supported authentication algorithms of a mlx5 crypto driver.
diff --git a/doc/guides/cryptodevs/mlx5.rst b/doc/guides/cryptodevs/mlx5.rst
index 05a0a449e2..dd1d1a615d 100644
--- a/doc/guides/cryptodevs/mlx5.rst
+++ b/doc/guides/cryptodevs/mlx5.rst
@@ -53,6 +53,16 @@ Supported NICs
 
 * Mellanox\ |reg| ConnectX\ |reg|-6 200G MCX654106A-HCAT (2x200G)
 
+
+Limitations
+-----------
+
+- AES-XTS keys provided in xform must include keytag and should be wrappend.
+- The supported data-unit lengths are 512B and 1KB. In case the `dataunit_len`
+  is not provided in the cipher xform, the OP length is limited to the above
+  values and 1MB.
+
+
 Prerequisites
 -------------
 
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index dfc3da3d3f..dd9685dcad 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -3,6 +3,7 @@
  */
 
 #include <rte_malloc.h>
+#include <rte_mempool.h>
 #include <rte_errno.h>
 #include <rte_log.h>
 #include <rte_pci.h>
@@ -20,7 +21,9 @@
 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
 
 #define MLX5_CRYPTO_FEATURE_FLAGS \
-	RTE_CRYPTODEV_FF_HW_ACCELERATED
+	(RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | \
+	 RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY | \
+	 RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS)
 
 TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
 				TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
@@ -30,6 +33,32 @@ int mlx5_crypto_logtype;
 
 uint8_t mlx5_crypto_driver_id;
 
+const struct rte_cryptodev_capabilities mlx5_crypto_caps[] = {
+	{		/* AES XTS */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+			{.cipher = {
+				.algo = RTE_CRYPTO_CIPHER_AES_XTS,
+				.block_size = 16,
+				.key_size = {
+					.min = 32,
+					.max = 64,
+					.increment = 32
+				},
+				.iv_size = {
+					.min = 16,
+					.max = 16,
+					.increment = 0
+				},
+				.dataunit_set =
+				RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_512_BYTES |
+				RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_4096_BYTES,
+			}, }
+		}, }
+	},
+};
+
 static const char mlx5_crypto_drv_name[] = RTE_STR(MLX5_CRYPTO_DRIVER_NAME);
 
 static const struct rte_driver mlx5_drv = {
@@ -39,6 +68,49 @@ static const struct rte_driver mlx5_drv = {
 
 static struct cryptodev_driver mlx5_cryptodev_driver;
 
+struct mlx5_crypto_session {
+	uint32_t bs_bpt_eo_es;
+	/*
+	 * bsf_size, bsf_p_type, encryption_order and encryption standard,
+	 * saved in big endian format.
+	 */
+	uint32_t bsp_res;
+	/*
+	 * crypto_block_size_pointer and reserved 24 bits saved in big endian
+	 * format.
+	 */
+	uint32_t iv_offset:16;
+	/* Starting point for Initialisation Vector. */
+	struct mlx5_crypto_dek *dek; /* Pointer to dek struct. */
+	uint32_t dek_id; /* DEK ID */
+} __rte_packed;
+
+static void
+mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev,
+			  struct rte_cryptodev_info *dev_info)
+{
+	RTE_SET_USED(dev);
+	if (dev_info != NULL) {
+		dev_info->driver_id = mlx5_crypto_driver_id;
+		dev_info->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
+		dev_info->capabilities = mlx5_crypto_caps;
+		dev_info->max_nb_queue_pairs = 0;
+		dev_info->min_mbuf_headroom_req = 0;
+		dev_info->min_mbuf_tailroom_req = 0;
+		dev_info->sym.max_nb_sessions = 0;
+		/*
+		 * If 0, the device does not have any limitation in number of
+		 * sessions that can be used.
+		 */
+	}
+}
+
+static unsigned int
+mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
+{
+	return sizeof(struct mlx5_crypto_session);
+}
+
 static int
 mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
 		struct rte_cryptodev_config *config __rte_unused)
@@ -61,19 +133,109 @@ mlx5_crypto_dev_close(struct rte_cryptodev *dev)
 	return 0;
 }
 
+static int
+mlx5_crypto_sym_session_configure(struct rte_cryptodev *dev,
+				  struct rte_crypto_sym_xform *xform,
+				  struct rte_cryptodev_sym_session *session,
+				  struct rte_mempool *mp)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+	struct mlx5_crypto_session *sess_private_data;
+	struct rte_crypto_cipher_xform *cipher;
+	uint8_t encryption_order;
+	int ret;
+
+	if (unlikely(xform->next != NULL)) {
+		DRV_LOG(ERR, "Xform next is not supported.");
+		return -ENOTSUP;
+	}
+	if (unlikely((xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) ||
+		     (xform->cipher.algo != RTE_CRYPTO_CIPHER_AES_XTS))) {
+		DRV_LOG(ERR, "Only AES-XTS algorithm is supported.");
+		return -ENOTSUP;
+	}
+	ret = rte_mempool_get(mp, (void *)&sess_private_data);
+	if (ret != 0) {
+		DRV_LOG(ERR,
+			"Failed to get session %p private data from mempool.",
+			sess_private_data);
+		return -ENOMEM;
+	}
+	cipher = &xform->cipher;
+	sess_private_data->dek = mlx5_crypto_dek_prepare(priv, cipher);
+	if (sess_private_data->dek == NULL) {
+		rte_mempool_put(mp, sess_private_data);
+		DRV_LOG(ERR, "Failed to prepare dek.");
+		return -ENOMEM;
+	}
+	if (cipher->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
+		encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_MEMORY;
+	else
+		encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_WIRE;
+	sess_private_data->bs_bpt_eo_es = rte_cpu_to_be_32
+			(MLX5_BSF_SIZE_64B << MLX5_BSF_SIZE_OFFSET |
+			 MLX5_BSF_P_TYPE_CRYPTO << MLX5_BSF_P_TYPE_OFFSET |
+			 encryption_order << MLX5_ENCRYPTION_ORDER_OFFSET |
+			 MLX5_ENCRYPTION_STANDARD_AES_XTS);
+	switch (xform->cipher.dataunit_len) {
+	case 0:
+		sess_private_data->bsp_res = 0;
+		break;
+	case 512:
+		sess_private_data->bsp_res = rte_cpu_to_be_32
+					     ((uint32_t)MLX5_BLOCK_SIZE_512B <<
+					     MLX5_BLOCK_SIZE_OFFSET);
+		break;
+	case 4096:
+		sess_private_data->bsp_res = rte_cpu_to_be_32
+					     ((uint32_t)MLX5_BLOCK_SIZE_4096B <<
+					     MLX5_BLOCK_SIZE_OFFSET);
+		break;
+	default:
+		DRV_LOG(ERR, "Cipher data unit length is not supported.");
+		return -ENOTSUP;
+	}
+	sess_private_data->iv_offset = cipher->iv.offset;
+	sess_private_data->dek_id =
+			rte_cpu_to_be_32(sess_private_data->dek->obj->id &
+					 0xffffff);
+	set_sym_session_private_data(session, dev->driver_id,
+				     sess_private_data);
+	DRV_LOG(DEBUG, "Session %p was configured.", sess_private_data);
+	return 0;
+}
+
+static void
+mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev,
+			      struct rte_cryptodev_sym_session *sess)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+	struct mlx5_crypto_session *spriv = get_sym_session_private_data(sess,
+								dev->driver_id);
+
+	if (unlikely(spriv == NULL)) {
+		DRV_LOG(ERR, "Failed to get session %p private data.", spriv);
+		return;
+	}
+	mlx5_crypto_dek_destroy(priv, spriv->dek);
+	set_sym_session_private_data(sess, dev->driver_id, NULL);
+	rte_mempool_put(rte_mempool_from_obj(spriv), spriv);
+	DRV_LOG(DEBUG, "Session %p was cleared.", spriv);
+}
+
 static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_configure			= mlx5_crypto_dev_configure,
 	.dev_start			= NULL,
 	.dev_stop			= NULL,
 	.dev_close			= mlx5_crypto_dev_close,
-	.dev_infos_get			= NULL,
+	.dev_infos_get			= mlx5_crypto_dev_infos_get,
 	.stats_get			= NULL,
 	.stats_reset			= NULL,
 	.queue_pair_setup		= NULL,
 	.queue_pair_release		= NULL,
-	.sym_session_get_size		= NULL,
-	.sym_session_configure		= NULL,
-	.sym_session_clear		= NULL,
+	.sym_session_get_size		= mlx5_crypto_sym_session_get_size,
+	.sym_session_configure		= mlx5_crypto_sym_session_configure,
+	.sym_session_clear		= mlx5_crypto_sym_session_clear,
 	.sym_get_raw_dp_ctx_size	= NULL,
 	.sym_configure_raw_dp_ctx	= NULL,
 };
-- 
2.27.0

[dpdk-dev] [PATCH v5 04/15] crypto/mlx5: add basic operations

[Shiri Kuzin]

The basic dev control operations are configure, close and get info.

Extended the existing support of configure and close:
	-mlx5_crypto_dev_configure- function used to configure device.
	-mlx5_crypto_dev_close-  function used to close a configured
	 device.

Added config struct to user private data with the fields socket id,
number of queue pairs and feature flags to be disabled.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 26 +++++++++++++++++++-------
 drivers/crypto/mlx5/mlx5_crypto.h |  1 +
 2 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index dd9685dcad..0ce46ffd2f 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -105,22 +105,27 @@ mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev,
 	}
 }
 
-static unsigned int
-mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
-{
-	return sizeof(struct mlx5_crypto_session);
-}
-
 static int
 mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
-		struct rte_cryptodev_config *config __rte_unused)
+			  struct rte_cryptodev_config *config)
 {
 	struct mlx5_crypto_priv *priv = dev->data->dev_private;
 
+	if (config == NULL) {
+		DRV_LOG(ERR, "Invalid crypto dev configure parameters.");
+		return -EINVAL;
+	}
+	if ((config->ff_disable & RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO) != 0) {
+		DRV_LOG(ERR,
+			"Disabled symmetric crypto feature is not supported.");
+		return -ENOTSUP;
+	}
 	if (mlx5_crypto_dek_setup(priv) != 0) {
 		DRV_LOG(ERR, "Dek hash list creation has failed.");
 		return -ENOMEM;
 	}
+	priv->dev_config = *config;
+	DRV_LOG(DEBUG, "Device %u was configured.", dev->driver_id);
 	return 0;
 }
 
@@ -130,9 +135,16 @@ mlx5_crypto_dev_close(struct rte_cryptodev *dev)
 	struct mlx5_crypto_priv *priv = dev->data->dev_private;
 
 	mlx5_crypto_dek_unset(priv);
+	DRV_LOG(DEBUG, "Device %u was closed.", dev->driver_id);
 	return 0;
 }
 
+static unsigned int
+mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
+{
+	return sizeof(struct mlx5_crypto_session);
+}
+
 static int
 mlx5_crypto_sym_session_configure(struct rte_cryptodev *dev,
 				  struct rte_crypto_sym_xform *xform,
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 5a54cb0dca..4c07356028 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -24,6 +24,7 @@ struct mlx5_crypto_priv {
 	uint32_t pdn; /* Protection Domain number. */
 	struct ibv_pd *pd;
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
+	struct rte_cryptodev_config dev_config;
 };
 
 struct mlx5_crypto_dek {
-- 
2.27.0

[dpdk-dev] [PATCH v5 05/15] crypto/mlx5: add queue pairs operations

[Shiri Kuzin]

The HW queue pairs are a pair of send queue and receive queue of
independent work queues packed together in one object for the purpose
of transferring data between nodes of a network.

Completion Queue is a FIFO queue of completed work requests.

In crypto driver we use one QP in loopback in order to encrypt and
decrypt data locally without sending it to the wire.
In the configured QP we only use the SQ to perform the encryption and
decryption operations.

Added implementation for the QP setup function which creates the CQ,
creates the QP and changes its state to RTS (ready to send).

Added implementation for the release QP function to release all the QP
resources.

Added the ops structure that contains any operation which is supported
by the cryptodev.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 129 +++++++++++++++++++++++++++++-
 drivers/crypto/mlx5/mlx5_crypto.h |  11 +++
 2 files changed, 137 insertions(+), 3 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 0ce46ffd2f..f1d690c750 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -7,6 +7,7 @@
 #include <rte_errno.h>
 #include <rte_log.h>
 #include <rte_pci.h>
+#include <rte_memory.h>
 
 #include <mlx5_glue.h>
 #include <mlx5_common.h>
@@ -19,6 +20,7 @@
 
 #define MLX5_CRYPTO_DRIVER_NAME mlx5_crypto
 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
+#define MLX5_CRYPTO_MAX_QPS 1024
 
 #define MLX5_CRYPTO_FEATURE_FLAGS \
 	(RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | \
@@ -94,7 +96,7 @@ mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev,
 		dev_info->driver_id = mlx5_crypto_driver_id;
 		dev_info->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
 		dev_info->capabilities = mlx5_crypto_caps;
-		dev_info->max_nb_queue_pairs = 0;
+		dev_info->max_nb_queue_pairs = MLX5_CRYPTO_MAX_QPS;
 		dev_info->min_mbuf_headroom_req = 0;
 		dev_info->min_mbuf_tailroom_req = 0;
 		dev_info->sym.max_nb_sessions = 0;
@@ -235,6 +237,127 @@ mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev,
 	DRV_LOG(DEBUG, "Session %p was cleared.", spriv);
 }
 
+static int
+mlx5_crypto_queue_pair_release(struct rte_cryptodev *dev, uint16_t qp_id)
+{
+	struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
+
+	if (qp->qp_obj != NULL)
+		claim_zero(mlx5_devx_cmd_destroy(qp->qp_obj));
+	if (qp->umem_obj != NULL)
+		claim_zero(mlx5_glue->devx_umem_dereg(qp->umem_obj));
+	if (qp->umem_buf != NULL)
+		rte_free(qp->umem_buf);
+	mlx5_devx_cq_destroy(&qp->cq_obj);
+	rte_free(qp);
+	dev->data->queue_pairs[qp_id] = NULL;
+	return 0;
+}
+
+static int
+mlx5_crypto_qp2rts(struct mlx5_crypto_qp *qp)
+{
+	/*
+	 * In Order to configure self loopback, when calling these functions the
+	 * remote QP id that is used is the id of the same QP.
+	 */
+	if (mlx5_devx_cmd_modify_qp_state(qp->qp_obj, MLX5_CMD_OP_RST2INIT_QP,
+					  qp->qp_obj->id)) {
+		DRV_LOG(ERR, "Failed to modify QP to INIT state(%u).",
+			rte_errno);
+		return -1;
+	}
+	if (mlx5_devx_cmd_modify_qp_state(qp->qp_obj, MLX5_CMD_OP_INIT2RTR_QP,
+					  qp->qp_obj->id)) {
+		DRV_LOG(ERR, "Failed to modify QP to RTR state(%u).",
+			rte_errno);
+		return -1;
+	}
+	if (mlx5_devx_cmd_modify_qp_state(qp->qp_obj, MLX5_CMD_OP_RTR2RTS_QP,
+					  qp->qp_obj->id)) {
+		DRV_LOG(ERR, "Failed to modify QP to RTS state(%u).",
+			rte_errno);
+		return -1;
+	}
+	return 0;
+}
+
+static int
+mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
+			     const struct rte_cryptodev_qp_conf *qp_conf,
+			     int socket_id)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+	struct mlx5_devx_qp_attr attr = {0};
+	struct mlx5_crypto_qp *qp;
+	uint16_t log_nb_desc = rte_log2_u32(qp_conf->nb_descriptors);
+	uint32_t umem_size = RTE_BIT32(log_nb_desc) *
+			      MLX5_CRYPTO_WQE_SET_SIZE +
+			      sizeof(*qp->db_rec) * 2;
+	uint32_t alloc_size = sizeof(*qp);
+	struct mlx5_devx_cq_attr cq_attr = {
+		.uar_page_id = mlx5_os_get_devx_uar_page_id(priv->uar),
+	};
+
+	if (dev->data->queue_pairs[qp_id] != NULL)
+		mlx5_crypto_queue_pair_release(dev, qp_id);
+	alloc_size = RTE_ALIGN(alloc_size, RTE_CACHE_LINE_SIZE);
+	alloc_size += sizeof(struct rte_crypto_op *) * RTE_BIT32(log_nb_desc);
+	qp = rte_zmalloc_socket(__func__, alloc_size, RTE_CACHE_LINE_SIZE,
+				socket_id);
+	if (qp == NULL) {
+		DRV_LOG(ERR, "Failed to allocate QP memory.");
+		rte_errno = ENOMEM;
+		return -rte_errno;
+	}
+	if (mlx5_devx_cq_create(priv->ctx, &qp->cq_obj, log_nb_desc,
+				&cq_attr, socket_id) != 0) {
+		DRV_LOG(ERR, "Failed to create CQ.");
+		goto error;
+	}
+	qp->umem_buf = rte_zmalloc_socket(__func__, umem_size, 4096, socket_id);
+	if (qp->umem_buf == NULL) {
+		DRV_LOG(ERR, "Failed to allocate QP umem.");
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	qp->umem_obj = mlx5_glue->devx_umem_reg(priv->ctx,
+					       (void *)(uintptr_t)qp->umem_buf,
+					       umem_size,
+					       IBV_ACCESS_LOCAL_WRITE);
+	if (qp->umem_obj == NULL) {
+		DRV_LOG(ERR, "Failed to register QP umem.");
+		goto error;
+	}
+	attr.pd = priv->pdn;
+	attr.uar_index = mlx5_os_get_devx_uar_page_id(priv->uar);
+	attr.cqn = qp->cq_obj.cq->id;
+	attr.log_page_size = rte_log2_u32(sysconf(_SC_PAGESIZE));
+	attr.rq_size =  0;
+	attr.sq_size = RTE_BIT32(log_nb_desc);
+	attr.dbr_umem_valid = 1;
+	attr.wq_umem_id = qp->umem_obj->umem_id;
+	attr.wq_umem_offset = 0;
+	attr.dbr_umem_id = qp->umem_obj->umem_id;
+	attr.dbr_address = RTE_BIT64(log_nb_desc) *
+			   MLX5_CRYPTO_WQE_SET_SIZE;
+	qp->qp_obj = mlx5_devx_cmd_create_qp(priv->ctx, &attr);
+	if (qp->qp_obj == NULL) {
+		DRV_LOG(ERR, "Failed to create QP(%u).", rte_errno);
+		goto error;
+	}
+	qp->db_rec = RTE_PTR_ADD(qp->umem_buf, (uintptr_t)attr.dbr_address);
+	if (mlx5_crypto_qp2rts(qp))
+		goto error;
+	qp->ops = (struct rte_crypto_op **)RTE_ALIGN((uintptr_t)(qp + 1),
+							   RTE_CACHE_LINE_SIZE);
+	dev->data->queue_pairs[qp_id] = qp;
+	return 0;
+error:
+	mlx5_crypto_queue_pair_release(dev, qp_id);
+	return -1;
+}
+
 static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_configure			= mlx5_crypto_dev_configure,
 	.dev_start			= NULL,
@@ -243,8 +366,8 @@ static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_infos_get			= mlx5_crypto_dev_infos_get,
 	.stats_get			= NULL,
 	.stats_reset			= NULL,
-	.queue_pair_setup		= NULL,
-	.queue_pair_release		= NULL,
+	.queue_pair_setup		= mlx5_crypto_queue_pair_setup,
+	.queue_pair_release		= mlx5_crypto_queue_pair_release,
 	.sym_session_get_size		= mlx5_crypto_sym_session_get_size,
 	.sym_session_configure		= mlx5_crypto_sym_session_configure,
 	.sym_session_clear		= mlx5_crypto_sym_session_clear,
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 4c07356028..a3bc65b0ed 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -11,9 +11,11 @@
 #include <rte_cryptodev_pmd.h>
 
 #include <mlx5_common_utils.h>
+#include <mlx5_common_devx.h>
 
 #define MLX5_CRYPTO_DEK_HTABLE_SZ (1 << 11)
 #define MLX5_CRYPTO_KEY_LENGTH 80
+#define MLX5_CRYPTO_WQE_SET_SIZE 1024
 
 struct mlx5_crypto_priv {
 	TAILQ_ENTRY(mlx5_crypto_priv) next;
@@ -27,6 +29,15 @@ struct mlx5_crypto_priv {
 	struct rte_cryptodev_config dev_config;
 };
 
+struct mlx5_crypto_qp {
+	struct mlx5_devx_cq cq_obj;
+	struct mlx5_devx_obj *qp_obj;
+	struct mlx5dv_devx_umem *umem_obj;
+	void *umem_buf;
+	volatile uint32_t *db_rec;
+	struct rte_crypto_op **ops;
+};
+
 struct mlx5_crypto_dek {
 	struct mlx5_hlist_entry entry; /* Pointer to DEK hash list entry. */
 	struct mlx5_devx_obj *obj; /* Pointer to DEK DevX object. */
-- 
2.27.0

[dpdk-dev] [PATCH v5 06/15] crypto/mlx5: add dev stop and start operations

[Shiri Kuzin]

Add the dev_start function that is used to start a configured device.
Add the dev_stop function that is used to stop a configured device.

Both functions set the dev parameter as used and return 0.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index f1d690c750..1e24cf8c25 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -131,6 +131,19 @@ mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
 	return 0;
 }
 
+static void
+mlx5_crypto_dev_stop(struct rte_cryptodev *dev)
+{
+	RTE_SET_USED(dev);
+}
+
+static int
+mlx5_crypto_dev_start(struct rte_cryptodev *dev)
+{
+	RTE_SET_USED(dev);
+	return 0;
+}
+
 static int
 mlx5_crypto_dev_close(struct rte_cryptodev *dev)
 {
@@ -360,8 +373,8 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 
 static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_configure			= mlx5_crypto_dev_configure,
-	.dev_start			= NULL,
-	.dev_stop			= NULL,
+	.dev_start			= mlx5_crypto_dev_start,
+	.dev_stop			= mlx5_crypto_dev_stop,
 	.dev_close			= mlx5_crypto_dev_close,
 	.dev_infos_get			= mlx5_crypto_dev_infos_get,
 	.stats_get			= NULL,
-- 
2.27.0

[dpdk-dev] [PATCH v5 07/15] crypto/mlx5: add memory region management

[Shiri Kuzin]

Mellanox user space drivers don't deal with physical addresses as part
of a memory protection mechanism.
The device translates the given virtual address to a physical address
using the given memory key as an address space identifier.
That's why any mbuf virtual address is moved directly to the HW
descriptor(WQE).

The mapping between the virtual address to the physical address is saved
in MR configured by the kernel to the HW.

Each MR has a key that should also be moved to the WQE by the SW.

When the SW sees an unmapped address, it extends the address range and
creates a MR using a system call.

Add memory region cache management:
	- 2 level cache per queue-pair - no locks.
	- 1 shared cache between all the queues using a lock.

Using this way, the MR key search per data-path address is optimized.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Signed-off-by: Michael Baum <michaelba@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 doc/guides/cryptodevs/mlx5.rst    |  6 +++
 drivers/crypto/mlx5/mlx5_crypto.c | 63 +++++++++++++++++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto.h |  3 ++
 3 files changed, 72 insertions(+)

diff --git a/doc/guides/cryptodevs/mlx5.rst b/doc/guides/cryptodevs/mlx5.rst
index dd1d1a615d..ece881220e 100644
--- a/doc/guides/cryptodevs/mlx5.rst
+++ b/doc/guides/cryptodevs/mlx5.rst
@@ -26,6 +26,12 @@ the MKEY is configured to perform crypto operations.
 
 The encryption does not require text to be aligned to the AES block size (128b).
 
+For security reasons and to increase robustness, this driver only deals with virtual
+memory addresses. The way resources allocations are handled by the kernel,
+combined with hardware specifications that allow handling virtual memory
+addresses directly, ensure that DPDK applications cannot access random
+physical memory (or memory that does not belong to the current process).
+
 The PMD uses libibverbs and libmlx5 to access the device firmware or to
 access the hardware components directly.
 There are different levels of objects and bypassing abilities.
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 1e24cf8c25..367a4759b3 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -261,6 +261,7 @@ mlx5_crypto_queue_pair_release(struct rte_cryptodev *dev, uint16_t qp_id)
 		claim_zero(mlx5_glue->devx_umem_dereg(qp->umem_obj));
 	if (qp->umem_buf != NULL)
 		rte_free(qp->umem_buf);
+	mlx5_mr_btree_free(&qp->mr_ctrl.cache_bh);
 	mlx5_devx_cq_destroy(&qp->cq_obj);
 	rte_free(qp);
 	dev->data->queue_pairs[qp_id] = NULL;
@@ -342,6 +343,14 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 		DRV_LOG(ERR, "Failed to register QP umem.");
 		goto error;
 	}
+	if (mlx5_mr_btree_init(&qp->mr_ctrl.cache_bh, MLX5_MR_BTREE_CACHE_N,
+			       priv->dev_config.socket_id) != 0) {
+		DRV_LOG(ERR, "Cannot allocate MR Btree for qp %u.",
+			(uint32_t)qp_id);
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	qp->mr_ctrl.dev_gen_ptr = &priv->mr_scache.dev_gen;
 	attr.pd = priv->pdn;
 	attr.uar_index = mlx5_os_get_devx_uar_page_id(priv->uar);
 	attr.cqn = qp->cq_obj.cq->id;
@@ -448,6 +457,40 @@ mlx5_crypto_hw_global_prepare(struct mlx5_crypto_priv *priv)
 	return 0;
 }
 
+/**
+ * Callback for memory event.
+ *
+ * @param event_type
+ *   Memory event type.
+ * @param addr
+ *   Address of memory.
+ * @param len
+ *   Size of memory.
+ */
+static void
+mlx5_crypto_mr_mem_event_cb(enum rte_mem_event event_type, const void *addr,
+			    size_t len, void *arg __rte_unused)
+{
+	struct mlx5_crypto_priv *priv;
+
+	/* Must be called from the primary process. */
+	MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);
+	switch (event_type) {
+	case RTE_MEM_EVENT_FREE:
+		pthread_mutex_lock(&priv_list_lock);
+		/* Iterate all the existing mlx5 devices. */
+		TAILQ_FOREACH(priv, &mlx5_crypto_priv_list, next)
+			mlx5_free_mr_by_addr(&priv->mr_scache,
+					     priv->ctx->device->name,
+					     addr, len);
+		pthread_mutex_unlock(&priv_list_lock);
+		break;
+	case RTE_MEM_EVENT_ALLOC:
+	default:
+		break;
+	}
+}
+
 /**
  * DPDK callback to register a PCI device.
  *
@@ -530,6 +573,22 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 		claim_zero(mlx5_glue->close_device(priv->ctx));
 		return -1;
 	}
+	if (mlx5_mr_btree_init(&priv->mr_scache.cache,
+			     MLX5_MR_BTREE_CACHE_N * 2, rte_socket_id()) != 0) {
+		DRV_LOG(ERR, "Failed to allocate shared cache MR memory.");
+		mlx5_crypto_hw_global_release(priv);
+		rte_cryptodev_pmd_destroy(priv->crypto_dev);
+		claim_zero(mlx5_glue->close_device(priv->ctx));
+		rte_errno = ENOMEM;
+		return -rte_errno;
+	}
+	priv->mr_scache.reg_mr_cb = mlx5_common_verbs_reg_mr;
+	priv->mr_scache.dereg_mr_cb = mlx5_common_verbs_dereg_mr;
+	/* Register callback function for global shared MR cache management. */
+	if (TAILQ_EMPTY(&mlx5_crypto_priv_list))
+		rte_mem_event_callback_register("MLX5_MEM_EVENT_CB",
+						mlx5_crypto_mr_mem_event_cb,
+						NULL);
 	pthread_mutex_lock(&priv_list_lock);
 	TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next);
 	pthread_mutex_unlock(&priv_list_lock);
@@ -549,6 +608,10 @@ mlx5_crypto_pci_remove(struct rte_pci_device *pdev)
 		TAILQ_REMOVE(&mlx5_crypto_priv_list, priv, next);
 	pthread_mutex_unlock(&priv_list_lock);
 	if (priv) {
+		if (TAILQ_EMPTY(&mlx5_crypto_priv_list))
+			rte_mem_event_callback_unregister("MLX5_MEM_EVENT_CB",
+							  NULL);
+		mlx5_mr_release_cache(&priv->mr_scache);
 		mlx5_crypto_hw_global_release(priv);
 		rte_cryptodev_pmd_destroy(priv->crypto_dev);
 		claim_zero(mlx5_glue->close_device(priv->ctx));
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index a3bc65b0ed..d1ea8024d4 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -12,6 +12,7 @@
 
 #include <mlx5_common_utils.h>
 #include <mlx5_common_devx.h>
+#include <mlx5_common_mr.h>
 
 #define MLX5_CRYPTO_DEK_HTABLE_SZ (1 << 11)
 #define MLX5_CRYPTO_KEY_LENGTH 80
@@ -27,6 +28,7 @@ struct mlx5_crypto_priv {
 	struct ibv_pd *pd;
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
 	struct rte_cryptodev_config dev_config;
+	struct mlx5_mr_share_cache mr_scache; /* Global shared MR cache. */
 };
 
 struct mlx5_crypto_qp {
@@ -36,6 +38,7 @@ struct mlx5_crypto_qp {
 	void *umem_buf;
 	volatile uint32_t *db_rec;
 	struct rte_crypto_op **ops;
+	struct mlx5_mr_ctrl mr_ctrl;
 };
 
 struct mlx5_crypto_dek {
-- 
2.27.0

[dpdk-dev] [PATCH v5 08/15] crypto/mlx5: create login object using DevX

[Shiri Kuzin]

To work with crypto engines that are marked with wrapped_import_method,
a login session is required.
A crypto login object needs to be created using DevX.

The crypto login object contains:
	- The credential pointer.
	- The import_KEK pointer to be used for all secured information
	  communicated in crypto commands (key fields), including the
	  provided credential in this command.
	- The credential secret, wrapped by the import_KEK indicated in
	  this command. Size includes 8 bytes IV for wrapping.

Added devargs for the required login values:
	- wcs_file - path to the file containing the credential.
	- import_kek_id - the import KEK pointer.
	- credential_id - the credential pointer.

Create the login DevX object in pci_probe function and destroy it in
pci_remove.
Destroying the crypto login object means logout.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 doc/guides/cryptodevs/mlx5.rst    |  60 +++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto.c | 103 ++++++++++++++++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto.h |   7 ++
 3 files changed, 170 insertions(+)

diff --git a/doc/guides/cryptodevs/mlx5.rst b/doc/guides/cryptodevs/mlx5.rst
index ece881220e..6cae5affbd 100644
--- a/doc/guides/cryptodevs/mlx5.rst
+++ b/doc/guides/cryptodevs/mlx5.rst
@@ -44,6 +44,51 @@ To get the best performances:
 Enabling librte_crypto_mlx5 causes DPDK applications to be linked against
 libibverbs.
 
+In order to move the device to crypto operational mode, credential and KEK
+(Key Encrypting Key) should be set as the first step.
+The credential will be used by the software in order to perform crypto login, and the KEK is
+the AES Key Wrap Algorithm (rfc3394) key that will be used for sensitive data
+wrapping.
+The credential and the AES-XTS keys should be provided to the hardware, as ciphertext
+encrypted by the KEK.
+
+When crypto engines are defined to work in wrapped import method, they come out
+of the factory in Commissioning mode, and thus, cannot be used for crypto operations
+yet. A dedicated tool is used for changing the mode from Commissioning to
+Operational, while setting the first import_KEK and credential in plaintext.
+The mlxreg dedicated tool should be used as follows:
+
+- Set CRYPTO_OPERATIONAL register to set the device in crypto operational mode.
+
+  The input to this tool is:
+    The first credential in plaintext, 40B.
+    The first import_KEK in plaintext: kek size 0 for 16B or 1 for 32B, kek data.
+
+  Example:
+  mlxreg -d /dev/mst/mt4123_pciconf0 --reg_name CRYPTO_OPERATIONAL --get
+
+  The "wrapped_crypto_operational" value will be "0x00000000".
+  The command to set the register should be executed only once, and all the
+  values mentioned above should be specified in the same command.
+
+  Example:
+  mlxreg -d /dev/mst/mt4123_pciconf0 --reg_name CRYPTO_OPERATIONAL
+  --set "credential[0]=0x10000000, credential[1]=0x10000000, kek[0]=0x00000000"
+
+  All values not specified will remain 0.
+  "wrapped_crypto_going_to_commissioning" and  "wrapped_crypto_operational"
+  should not be specified.
+
+  All the device ports should set it in order to move to operational mode.
+
+- Query CRYPTO_OPERATIONAL register to make sure the device is in Operational
+  mode.
+
+  Example:
+  mlxreg -d /dev/mst/mt4123_pciconf0 --reg_name CRYPTO_OPERATIONAL --get
+  The "wrapped_crypto_operational" value will be "0x00000001" if the mode was
+  successfully changed to operational mode.
+
 
 Driver options
 --------------
@@ -53,6 +98,21 @@ Driver options
   Select the class of the driver that should probe the device.
   `crypto` for the mlx5 crypto driver.
 
+- ``wcs_file`` parameter [string] - mandatory
+
+  File path including only the wrapped credential in string format of hexadecimal
+  numbers, represent 48 bytes (8 bytes IV added by the AES key wrap algorithm).
+
+- ``import_kek_id`` parameter [int]
+
+  The identifier of the KEK, default value is 0 represents the operational
+  register import_kek..
+
+- ``credential_id`` parameter [int]
+
+  The identifier of the credential, default value is 0 represents the operational
+  register credential.
+
 
 Supported NICs
 --------------
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 367a4759b3..95bbb83fc3 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -457,6 +457,101 @@ mlx5_crypto_hw_global_prepare(struct mlx5_crypto_priv *priv)
 	return 0;
 }
 
+
+static int
+mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque)
+{
+	struct mlx5_crypto_devarg_params *devarg_prms = opaque;
+	struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr;
+	unsigned long tmp;
+	FILE *file;
+	int ret;
+	int i;
+
+	if (strcmp(key, "class") == 0)
+		return 0;
+	if (strcmp(key, "wcs_file") == 0) {
+		file = fopen(val, "rb");
+		if (file == NULL) {
+			rte_errno = ENOTSUP;
+			return -rte_errno;
+		}
+		for (i = 0 ; i < MLX5_CRYPTO_CREDENTIAL_SIZE ; i++) {
+			ret = fscanf(file, "%02hhX", &attr->credential[i]);
+			if (ret <= 0) {
+				fclose(file);
+				DRV_LOG(ERR,
+					"Failed to read credential from file.");
+				rte_errno = EINVAL;
+				return -rte_errno;
+			}
+		}
+		fclose(file);
+		devarg_prms->login_devarg = true;
+		return 0;
+	}
+	errno = 0;
+	tmp = strtoul(val, NULL, 0);
+	if (errno) {
+		DRV_LOG(WARNING, "%s: \"%s\" is an invalid integer.", key, val);
+		return -errno;
+	}
+	if (strcmp(key, "import_kek_id") == 0)
+		attr->session_import_kek_ptr = (uint32_t)tmp;
+	else if (strcmp(key, "credential_id") == 0)
+		attr->credential_pointer = (uint32_t)tmp;
+	else
+		DRV_LOG(WARNING, "Invalid key %s.", key);
+	return 0;
+}
+
+static struct mlx5_devx_obj *
+mlx5_crypto_config_login(struct rte_devargs *devargs,
+			 struct ibv_context *ctx)
+{
+	/*
+	 * Set credential pointer and session import KEK pointer to a default
+	 * value of 0.
+	 */
+	struct mlx5_crypto_devarg_params login = {
+			.login_devarg = false,
+			.login_attr = {
+					.credential_pointer = 0,
+					.session_import_kek_ptr = 0,
+			}
+	};
+	struct rte_kvargs *kvlist;
+
+	if (devargs == NULL) {
+		DRV_LOG(ERR,
+	"No login devargs in order to enable crypto operations in the device.");
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	kvlist = rte_kvargs_parse(devargs->args, NULL);
+	if (kvlist == NULL) {
+		DRV_LOG(ERR, "Failed to parse devargs.");
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	if (rte_kvargs_process(kvlist, NULL, mlx5_crypto_args_check_handler,
+			   &login) != 0) {
+		DRV_LOG(ERR, "Devargs handler function Failed.");
+		rte_kvargs_free(kvlist);
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	rte_kvargs_free(kvlist);
+	if (login.login_devarg == false) {
+		DRV_LOG(ERR,
+	"No login credential devarg in order to enable crypto operations "
+	"in the device.");
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	return mlx5_devx_cmd_create_crypto_login_obj(ctx, &login.login_attr);
+}
+
 /**
  * Callback for memory event.
  *
@@ -512,6 +607,7 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	struct ibv_device *ibv;
 	struct rte_cryptodev *crypto_dev;
 	struct ibv_context *ctx;
+	struct mlx5_devx_obj *login;
 	struct mlx5_crypto_priv *priv;
 	struct mlx5_hca_attr attr = { 0 };
 	struct rte_cryptodev_pmd_init_params init_params = {
@@ -550,6 +646,11 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 		rte_errno = ENOTSUP;
 		return -ENOTSUP;
 	}
+	login = mlx5_crypto_config_login(pci_dev->device.devargs, ctx);
+	if (login == NULL) {
+		DRV_LOG(ERR, "Failed to configure login.");
+		return -rte_errno;
+	}
 	crypto_dev = rte_cryptodev_pmd_create(ibv->name, &pci_dev->device,
 					&init_params);
 	if (crypto_dev == NULL) {
@@ -566,6 +667,7 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	crypto_dev->driver_id = mlx5_crypto_driver_id;
 	priv = crypto_dev->data->dev_private;
 	priv->ctx = ctx;
+	priv->login_obj = login;
 	priv->pci_dev = pci_dev;
 	priv->crypto_dev = crypto_dev;
 	if (mlx5_crypto_hw_global_prepare(priv) != 0) {
@@ -614,6 +716,7 @@ mlx5_crypto_pci_remove(struct rte_pci_device *pdev)
 		mlx5_mr_release_cache(&priv->mr_scache);
 		mlx5_crypto_hw_global_release(priv);
 		rte_cryptodev_pmd_destroy(priv->crypto_dev);
+		claim_zero(mlx5_devx_cmd_destroy(priv->login_obj));
 		claim_zero(mlx5_glue->close_device(priv->ctx));
 	}
 	return 0;
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index d1ea8024d4..a45a210e58 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -29,6 +29,7 @@ struct mlx5_crypto_priv {
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
 	struct rte_cryptodev_config dev_config;
 	struct mlx5_mr_share_cache mr_scache; /* Global shared MR cache. */
+	struct mlx5_devx_obj *login_obj;
 };
 
 struct mlx5_crypto_qp {
@@ -48,6 +49,12 @@ struct mlx5_crypto_dek {
 	bool size_is_48; /* Whether the key\data size is 48 bytes or not. */
 } __rte_cache_aligned;
 
+
+struct mlx5_crypto_devarg_params {
+	bool login_devarg;
+	struct mlx5_devx_crypto_login_attr login_attr;
+};
+
 int
 mlx5_crypto_dek_destroy(struct mlx5_crypto_priv *priv,
 			struct mlx5_crypto_dek *dek);
-- 
2.27.0

[dpdk-dev] [PATCH v5 09/15] crypto/mlx5: add keytag devarg

[Shiri Kuzin]

From: Suanming Mou <suanmingm@nvidia.com>

A keytag is a piece of data encrypted together with a DEK.

When a DEK is referenced by an MKEY.bsf through its index, the keytag is
also supplied in the BSF as plaintext. The HW will decrypt the DEK (and
the attached keytag) and will fail the operation if the keytags don't
match.

This commit adds the configuration of the keytag with devargs.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
 doc/guides/cryptodevs/mlx5.rst    |  7 +++++
 drivers/crypto/mlx5/mlx5_crypto.c | 50 +++++++++++++++++--------------
 drivers/crypto/mlx5/mlx5_crypto.h |  3 +-
 3 files changed, 37 insertions(+), 23 deletions(-)

diff --git a/doc/guides/cryptodevs/mlx5.rst b/doc/guides/cryptodevs/mlx5.rst
index 6cae5affbd..c3632484a5 100644
--- a/doc/guides/cryptodevs/mlx5.rst
+++ b/doc/guides/cryptodevs/mlx5.rst
@@ -52,6 +52,9 @@ wrapping.
 The credential and the AES-XTS keys should be provided to the hardware, as ciphertext
 encrypted by the KEK.
 
+A keytag (64 bits) should be appended to the AES-XTS keys (before wrapping),
+and will be validated when the hardware attempts to access it.
+
 When crypto engines are defined to work in wrapped import method, they come out
 of the factory in Commissioning mode, and thus, cannot be used for crypto operations
 yet. A dedicated tool is used for changing the mode from Commissioning to
@@ -113,6 +116,10 @@ Driver options
   The identifier of the credential, default value is 0 represents the operational
   register credential.
 
+- ``keytag`` parameter [int]
+
+  The plaintext of the keytag appanded to the AES-XTS keys, default value is 0.
+
 
 Supported NICs
 --------------
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 95bbb83fc3..6fd6bd1511 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -500,56 +500,52 @@ mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque)
 		attr->session_import_kek_ptr = (uint32_t)tmp;
 	else if (strcmp(key, "credential_id") == 0)
 		attr->credential_pointer = (uint32_t)tmp;
+	else if (strcmp(key, "keytag") == 0)
+		devarg_prms->keytag = tmp;
 	else
 		DRV_LOG(WARNING, "Invalid key %s.", key);
 	return 0;
 }
 
-static struct mlx5_devx_obj *
-mlx5_crypto_config_login(struct rte_devargs *devargs,
-			 struct ibv_context *ctx)
+static int
+mlx5_crypto_parse_devargs(struct rte_devargs *devargs,
+			  struct mlx5_crypto_devarg_params *devarg_prms)
 {
-	/*
-	 * Set credential pointer and session import KEK pointer to a default
-	 * value of 0.
-	 */
-	struct mlx5_crypto_devarg_params login = {
-			.login_devarg = false,
-			.login_attr = {
-					.credential_pointer = 0,
-					.session_import_kek_ptr = 0,
-			}
-	};
+	struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr;
 	struct rte_kvargs *kvlist;
 
+	/* Default values. */
+	attr->credential_pointer = 0;
+	attr->session_import_kek_ptr = 0;
+	devarg_prms->keytag = 0;
 	if (devargs == NULL) {
 		DRV_LOG(ERR,
 	"No login devargs in order to enable crypto operations in the device.");
 		rte_errno = EINVAL;
-		return NULL;
+		return -1;
 	}
 	kvlist = rte_kvargs_parse(devargs->args, NULL);
 	if (kvlist == NULL) {
 		DRV_LOG(ERR, "Failed to parse devargs.");
 		rte_errno = EINVAL;
-		return NULL;
+		return -1;
 	}
 	if (rte_kvargs_process(kvlist, NULL, mlx5_crypto_args_check_handler,
-			   &login) != 0) {
+			   devarg_prms) != 0) {
 		DRV_LOG(ERR, "Devargs handler function Failed.");
 		rte_kvargs_free(kvlist);
 		rte_errno = EINVAL;
-		return NULL;
+		return -1;
 	}
 	rte_kvargs_free(kvlist);
-	if (login.login_devarg == false) {
+	if (devarg_prms->login_devarg == false) {
 		DRV_LOG(ERR,
 	"No login credential devarg in order to enable crypto operations "
 	"in the device.");
 		rte_errno = EINVAL;
-		return NULL;
+		return -1;
 	}
-	return mlx5_devx_cmd_create_crypto_login_obj(ctx, &login.login_attr);
+	return 0;
 }
 
 /**
@@ -609,6 +605,7 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	struct ibv_context *ctx;
 	struct mlx5_devx_obj *login;
 	struct mlx5_crypto_priv *priv;
+	struct mlx5_crypto_devarg_params devarg_prms = { 0 };
 	struct mlx5_hca_attr attr = { 0 };
 	struct rte_cryptodev_pmd_init_params init_params = {
 		.name = "",
@@ -617,6 +614,8 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 		.max_nb_queue_pairs =
 				RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
 	};
+	int ret;
+
 	RTE_SET_USED(pci_drv);
 	if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
 		DRV_LOG(ERR, "Non-primary process type is not supported.");
@@ -646,7 +645,13 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 		rte_errno = ENOTSUP;
 		return -ENOTSUP;
 	}
-	login = mlx5_crypto_config_login(pci_dev->device.devargs, ctx);
+	ret = mlx5_crypto_parse_devargs(pci_dev->device.devargs, &devarg_prms);
+	if (ret) {
+		DRV_LOG(ERR, "Failed to parse devargs.");
+		return -rte_errno;
+	}
+	login = mlx5_devx_cmd_create_crypto_login_obj(ctx,
+						      &devarg_prms.login_attr);
 	if (login == NULL) {
 		DRV_LOG(ERR, "Failed to configure login.");
 		return -rte_errno;
@@ -686,6 +691,7 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	}
 	priv->mr_scache.reg_mr_cb = mlx5_common_verbs_reg_mr;
 	priv->mr_scache.dereg_mr_cb = mlx5_common_verbs_dereg_mr;
+	priv->keytag = rte_cpu_to_be_64(devarg_prms.keytag);
 	/* Register callback function for global shared MR cache management. */
 	if (TAILQ_EMPTY(&mlx5_crypto_priv_list))
 		rte_mem_event_callback_register("MLX5_MEM_EVENT_CB",
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index a45a210e58..ad70052967 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -30,6 +30,7 @@ struct mlx5_crypto_priv {
 	struct rte_cryptodev_config dev_config;
 	struct mlx5_mr_share_cache mr_scache; /* Global shared MR cache. */
 	struct mlx5_devx_obj *login_obj;
+	uint64_t keytag;
 };
 
 struct mlx5_crypto_qp {
@@ -49,10 +50,10 @@ struct mlx5_crypto_dek {
 	bool size_is_48; /* Whether the key\data size is 48 bytes or not. */
 } __rte_cache_aligned;
 
-
 struct mlx5_crypto_devarg_params {
 	bool login_devarg;
 	struct mlx5_devx_crypto_login_attr login_attr;
+	uint64_t keytag;
 };
 
 int
-- 
2.27.0

[dpdk-dev] [PATCH v5 10/15] crypto/mlx5: add maximum segments devarg

[Shiri Kuzin]

From: Suanming Mou <suanmingm@nvidia.com>

The mlx5 HW crypto operations are done by attaching crypto property
to a memory region. Once done, every access to the memory via the
crypto-enabled memory region will result with in-line encryption or
decryption of the data.

As a result, the design choice is to provide two types of WQEs. One
is UMR WQE which sets the crypto property and the other is rdma write
WQE which sends DMA command to copy data from local MR to remote MR.

The size of the WQEs will be defined by a new devarg called
max_segs_num.

This devarg also defines the maximum segments in mbuf chain that will be
supported for crypto operations.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
 doc/guides/cryptodevs/mlx5.rst    |  4 ++++
 drivers/crypto/mlx5/mlx5_crypto.c | 33 +++++++++++++++++++++++++++----
 drivers/crypto/mlx5/mlx5_crypto.h |  6 ++++++
 3 files changed, 39 insertions(+), 4 deletions(-)

diff --git a/doc/guides/cryptodevs/mlx5.rst b/doc/guides/cryptodevs/mlx5.rst
index c3632484a5..dd4705b744 100644
--- a/doc/guides/cryptodevs/mlx5.rst
+++ b/doc/guides/cryptodevs/mlx5.rst
@@ -120,6 +120,10 @@ Driver options
 
   The plaintext of the keytag appanded to the AES-XTS keys, default value is 0.
 
+- ``max_segs_num`` parameter [int]
+
+  Maximum number of mbuf chain segments(src or dest), default value is 8.
+
 
 Supported NICs
 --------------
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 6fd6bd1511..a6949775a9 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -21,6 +21,7 @@
 #define MLX5_CRYPTO_DRIVER_NAME mlx5_crypto
 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
 #define MLX5_CRYPTO_MAX_QPS 1024
+#define MLX5_CRYPTO_MAX_SEGS 56
 
 #define MLX5_CRYPTO_FEATURE_FLAGS \
 	(RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | \
@@ -496,14 +497,24 @@ mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque)
 		DRV_LOG(WARNING, "%s: \"%s\" is an invalid integer.", key, val);
 		return -errno;
 	}
-	if (strcmp(key, "import_kek_id") == 0)
+	if (strcmp(key, "max_segs_num") == 0) {
+		if (!tmp || tmp > MLX5_CRYPTO_MAX_SEGS) {
+			DRV_LOG(WARNING, "Invalid max_segs_num: %d, should"
+				" be less than %d.",
+				(uint32_t)tmp, MLX5_CRYPTO_MAX_SEGS);
+			rte_errno = EINVAL;
+			return -rte_errno;
+		}
+		devarg_prms->max_segs_num = (uint32_t)tmp;
+	} else if (strcmp(key, "import_kek_id") == 0) {
 		attr->session_import_kek_ptr = (uint32_t)tmp;
-	else if (strcmp(key, "credential_id") == 0)
+	} else if (strcmp(key, "credential_id") == 0) {
 		attr->credential_pointer = (uint32_t)tmp;
-	else if (strcmp(key, "keytag") == 0)
+	} else if (strcmp(key, "keytag") == 0) {
 		devarg_prms->keytag = tmp;
-	else
+	} else {
 		DRV_LOG(WARNING, "Invalid key %s.", key);
+	}
 	return 0;
 }
 
@@ -518,6 +529,7 @@ mlx5_crypto_parse_devargs(struct rte_devargs *devargs,
 	attr->credential_pointer = 0;
 	attr->session_import_kek_ptr = 0;
 	devarg_prms->keytag = 0;
+	devarg_prms->max_segs_num = 8;
 	if (devargs == NULL) {
 		DRV_LOG(ERR,
 	"No login devargs in order to enable crypto operations in the device.");
@@ -614,6 +626,7 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 		.max_nb_queue_pairs =
 				RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
 	};
+	uint16_t rdmw_wqe_size;
 	int ret;
 
 	RTE_SET_USED(pci_drv);
@@ -692,6 +705,18 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	priv->mr_scache.reg_mr_cb = mlx5_common_verbs_reg_mr;
 	priv->mr_scache.dereg_mr_cb = mlx5_common_verbs_dereg_mr;
 	priv->keytag = rte_cpu_to_be_64(devarg_prms.keytag);
+	priv->max_segs_num = devarg_prms.max_segs_num;
+	priv->umr_wqe_size = sizeof(struct mlx5_wqe_umr_bsf_seg) +
+			     sizeof(struct mlx5_umr_wqe) +
+			     RTE_ALIGN(priv->max_segs_num, 4) *
+			     sizeof(struct mlx5_wqe_dseg);
+	rdmw_wqe_size = sizeof(struct mlx5_rdma_write_wqe) +
+			      sizeof(struct mlx5_wqe_dseg) *
+			      (priv->max_segs_num <= 2 ? 2 : 2 +
+			       RTE_ALIGN(priv->max_segs_num - 2, 4));
+	priv->wqe_set_size = priv->umr_wqe_size + rdmw_wqe_size;
+	priv->umr_wqe_stride = priv->umr_wqe_size / MLX5_SEND_WQE_BB;
+	priv->max_rdmar_ds = rdmw_wqe_size / sizeof(struct mlx5_wqe_dseg);
 	/* Register callback function for global shared MR cache management. */
 	if (TAILQ_EMPTY(&mlx5_crypto_priv_list))
 		rte_mem_event_callback_register("MLX5_MEM_EVENT_CB",
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index ad70052967..48ca1cb9a2 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -25,12 +25,17 @@ struct mlx5_crypto_priv {
 	struct rte_cryptodev *crypto_dev;
 	void *uar; /* User Access Region. */
 	uint32_t pdn; /* Protection Domain number. */
+	uint32_t max_segs_num; /* Maximum supported data segs. */
 	struct ibv_pd *pd;
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
 	struct rte_cryptodev_config dev_config;
 	struct mlx5_mr_share_cache mr_scache; /* Global shared MR cache. */
 	struct mlx5_devx_obj *login_obj;
 	uint64_t keytag;
+	uint16_t wqe_set_size;
+	uint16_t umr_wqe_size;
+	uint16_t umr_wqe_stride;
+	uint16_t max_rdmar_ds;
 };
 
 struct mlx5_crypto_qp {
@@ -54,6 +59,7 @@ struct mlx5_crypto_devarg_params {
 	bool login_devarg;
 	struct mlx5_devx_crypto_login_attr login_attr;
 	uint64_t keytag;
+	uint32_t max_segs_num;
 };
 
 int
-- 
2.27.0

[dpdk-dev] [PATCH v5 11/15] crypto/mlx5: add WQE set initialization

[Shiri Kuzin]

From: Suanming Mou <suanmingm@nvidia.com>

Currently, HW handles the WQEs much faster than the software,
Using the constant WQE set layout can initialize most of the WQE
segments in advanced, and software only needs to configure very
limited segments in datapath. This accelerates the software WQE
organize in datapath.

This commit initializes the fixed WQE set segments.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 83 +++++++++++++++++++++++++++++--
 drivers/crypto/mlx5/mlx5_crypto.h | 10 +++-
 2 files changed, 87 insertions(+), 6 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index a6949775a9..de45e38597 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -297,6 +297,69 @@ mlx5_crypto_qp2rts(struct mlx5_crypto_qp *qp)
 	return 0;
 }
 
+static void
+mlx5_crypto_qp_init(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp)
+{
+	uint32_t i;
+
+	for (i = 0 ; i < qp->entries_n; i++) {
+		struct mlx5_wqe_cseg *cseg = RTE_PTR_ADD(qp->umem_buf, i *
+							 priv->wqe_set_size);
+		struct mlx5_wqe_umr_cseg *ucseg = (struct mlx5_wqe_umr_cseg *)
+								     (cseg + 1);
+		struct mlx5_wqe_umr_bsf_seg *bsf =
+			(struct mlx5_wqe_umr_bsf_seg *)(RTE_PTR_ADD(cseg,
+						       priv->umr_wqe_size)) - 1;
+		struct mlx5_wqe_rseg *rseg;
+
+		/* Init UMR WQE. */
+		cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj->id << 8) |
+					 (priv->umr_wqe_size / MLX5_WSEG_SIZE));
+		cseg->flags = RTE_BE32(MLX5_COMP_ONLY_FIRST_ERR <<
+				       MLX5_COMP_MODE_OFFSET);
+		cseg->misc = rte_cpu_to_be_32(qp->mkey[i]->id);
+		ucseg->if_cf_toe_cq_res = RTE_BE32(1u << MLX5_UMRC_IF_OFFSET);
+		ucseg->mkey_mask = RTE_BE64(1u << 0); /* Mkey length bit. */
+		ucseg->ko_to_bs = rte_cpu_to_be_32
+			((RTE_ALIGN(priv->max_segs_num, 4u) <<
+			 MLX5_UMRC_KO_OFFSET) | (4 << MLX5_UMRC_TO_BS_OFFSET));
+		bsf->keytag = priv->keytag;
+		/* Init RDMA WRITE WQE. */
+		cseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size);
+		cseg->flags = RTE_BE32((MLX5_COMP_ALWAYS <<
+				      MLX5_COMP_MODE_OFFSET) |
+				      MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE);
+		rseg = (struct mlx5_wqe_rseg *)(cseg + 1);
+		rseg->rkey = rte_cpu_to_be_32(qp->mkey[i]->id);
+	}
+}
+
+static int
+mlx5_crypto_indirect_mkeys_prepare(struct mlx5_crypto_priv *priv,
+				  struct mlx5_crypto_qp *qp)
+{
+	struct mlx5_umr_wqe *umr;
+	uint32_t i;
+	struct mlx5_devx_mkey_attr attr = {
+		.pd = priv->pdn,
+		.umr_en = 1,
+		.crypto_en = 1,
+		.set_remote_rw = 1,
+		.klm_num = RTE_ALIGN(priv->max_segs_num, 4),
+	};
+
+	for (umr = (struct mlx5_umr_wqe *)qp->umem_buf, i = 0;
+	   i < qp->entries_n; i++, umr = RTE_PTR_ADD(umr, priv->wqe_set_size)) {
+		attr.klm_array = (struct mlx5_klm *)&umr->kseg[0];
+		qp->mkey[i] = mlx5_devx_cmd_mkey_create(priv->ctx, &attr);
+		if (!qp->mkey[i]) {
+			DRV_LOG(ERR, "Failed to allocate indirect mkey.");
+			return -1;
+		}
+	}
+	return 0;
+}
+
 static int
 mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 			     const struct rte_cryptodev_qp_conf *qp_conf,
@@ -307,7 +370,7 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	struct mlx5_crypto_qp *qp;
 	uint16_t log_nb_desc = rte_log2_u32(qp_conf->nb_descriptors);
 	uint32_t umem_size = RTE_BIT32(log_nb_desc) *
-			      MLX5_CRYPTO_WQE_SET_SIZE +
+			      priv->wqe_set_size +
 			      sizeof(*qp->db_rec) * 2;
 	uint32_t alloc_size = sizeof(*qp);
 	struct mlx5_devx_cq_attr cq_attr = {
@@ -317,7 +380,9 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	if (dev->data->queue_pairs[qp_id] != NULL)
 		mlx5_crypto_queue_pair_release(dev, qp_id);
 	alloc_size = RTE_ALIGN(alloc_size, RTE_CACHE_LINE_SIZE);
-	alloc_size += sizeof(struct rte_crypto_op *) * RTE_BIT32(log_nb_desc);
+	alloc_size += (sizeof(struct rte_crypto_op *) +
+		       sizeof(struct mlx5_devx_obj *)) *
+		       RTE_BIT32(log_nb_desc);
 	qp = rte_zmalloc_socket(__func__, alloc_size, RTE_CACHE_LINE_SIZE,
 				socket_id);
 	if (qp == NULL) {
@@ -362,8 +427,7 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	attr.wq_umem_id = qp->umem_obj->umem_id;
 	attr.wq_umem_offset = 0;
 	attr.dbr_umem_id = qp->umem_obj->umem_id;
-	attr.dbr_address = RTE_BIT64(log_nb_desc) *
-			   MLX5_CRYPTO_WQE_SET_SIZE;
+	attr.dbr_address = RTE_BIT64(log_nb_desc) * priv->wqe_set_size;
 	qp->qp_obj = mlx5_devx_cmd_create_qp(priv->ctx, &attr);
 	if (qp->qp_obj == NULL) {
 		DRV_LOG(ERR, "Failed to create QP(%u).", rte_errno);
@@ -372,8 +436,17 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	qp->db_rec = RTE_PTR_ADD(qp->umem_buf, (uintptr_t)attr.dbr_address);
 	if (mlx5_crypto_qp2rts(qp))
 		goto error;
-	qp->ops = (struct rte_crypto_op **)RTE_ALIGN((uintptr_t)(qp + 1),
+	qp->mkey = (struct mlx5_devx_obj **)RTE_ALIGN((uintptr_t)(qp + 1),
 							   RTE_CACHE_LINE_SIZE);
+	qp->ops = (struct rte_crypto_op **)(qp->mkey + RTE_BIT32(log_nb_desc));
+	qp->entries_n = 1 << log_nb_desc;
+	if (mlx5_crypto_indirect_mkeys_prepare(priv, qp)) {
+		DRV_LOG(ERR, "Cannot allocate indirect memory regions.");
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	mlx5_crypto_qp_init(priv, qp);
+	qp->priv = priv;
 	dev->data->queue_pairs[qp_id] = qp;
 	return 0;
 error:
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 48ca1cb9a2..12e5890bf7 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -16,7 +16,6 @@
 
 #define MLX5_CRYPTO_DEK_HTABLE_SZ (1 << 11)
 #define MLX5_CRYPTO_KEY_LENGTH 80
-#define MLX5_CRYPTO_WQE_SET_SIZE 1024
 
 struct mlx5_crypto_priv {
 	TAILQ_ENTRY(mlx5_crypto_priv) next;
@@ -24,6 +23,7 @@ struct mlx5_crypto_priv {
 	struct rte_pci_device *pci_dev;
 	struct rte_cryptodev *crypto_dev;
 	void *uar; /* User Access Region. */
+	volatile uint64_t *uar_addr;
 	uint32_t pdn; /* Protection Domain number. */
 	uint32_t max_segs_num; /* Maximum supported data segs. */
 	struct ibv_pd *pd;
@@ -39,13 +39,21 @@ struct mlx5_crypto_priv {
 };
 
 struct mlx5_crypto_qp {
+	struct mlx5_crypto_priv *priv;
 	struct mlx5_devx_cq cq_obj;
 	struct mlx5_devx_obj *qp_obj;
+	struct rte_cryptodev_stats stats;
 	struct mlx5dv_devx_umem *umem_obj;
 	void *umem_buf;
 	volatile uint32_t *db_rec;
 	struct rte_crypto_op **ops;
+	struct mlx5_devx_obj **mkey; /* WQE's indirect mekys. */
 	struct mlx5_mr_ctrl mr_ctrl;
+	uint8_t *wqe;
+	uint16_t entries_n;
+	uint16_t pi;
+	uint16_t ci;
+	uint16_t db_pi;
 };
 
 struct mlx5_crypto_dek {
-- 
2.27.0

[dpdk-dev] [PATCH v5 12/15] crypto/mlx5: add enqueue and dequeue operations

[Shiri Kuzin]

From: Suanming Mou <suanmingm@nvidia.com>

The crypto operations are done with the WQE set which contains
one UMR WQE and one rdma write WQE. Most segments of the WQE
set are initialized properly during queue setup, only limited
segments are initialized according to the crypto detail in the
datapath process.

This commit adds the enquue and dequeue operations and updates
the WQE set segments accordingly.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
Signed-off-by: Michael Baum <michaelba@nvidia.com>
---
 doc/guides/cryptodevs/features/mlx5.ini |   5 +
 drivers/crypto/mlx5/mlx5_crypto.c       | 286 +++++++++++++++++++++++-
 drivers/crypto/mlx5/mlx5_crypto.h       |   3 +
 3 files changed, 290 insertions(+), 4 deletions(-)

diff --git a/doc/guides/cryptodevs/features/mlx5.ini b/doc/guides/cryptodevs/features/mlx5.ini
index bd757b5211..a89526add0 100644
--- a/doc/guides/cryptodevs/features/mlx5.ini
+++ b/doc/guides/cryptodevs/features/mlx5.ini
@@ -6,6 +6,11 @@
 [Features]
 Symmetric crypto       = Y
 HW Accelerated         = Y
+In Place SGL           = Y
+OOP SGL In SGL Out     = Y
+OOP SGL In LB  Out     = Y
+OOP LB  In SGL Out     = Y
+OOP LB  In LB  Out     = Y
 Cipher multiple data units = Y
 Cipher wrapped key     = Y
 
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index de45e38597..47dbb9c385 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -25,6 +25,10 @@
 
 #define MLX5_CRYPTO_FEATURE_FLAGS \
 	(RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | \
+	 RTE_CRYPTODEV_FF_IN_PLACE_SGL | RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT | \
+	 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | \
+	 RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT | \
+	 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | \
 	 RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY | \
 	 RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS)
 
@@ -297,6 +301,279 @@ mlx5_crypto_qp2rts(struct mlx5_crypto_qp *qp)
 	return 0;
 }
 
+static __rte_noinline uint32_t
+mlx5_crypto_get_block_size(struct rte_crypto_op *op)
+{
+	uint32_t bl = op->sym->cipher.data.length;
+
+	switch (bl) {
+	case (1 << 20):
+		return RTE_BE32(MLX5_BLOCK_SIZE_1MB << MLX5_BLOCK_SIZE_OFFSET);
+	case (1 << 12):
+		return RTE_BE32(MLX5_BLOCK_SIZE_4096B <<
+				MLX5_BLOCK_SIZE_OFFSET);
+	case (1 << 9):
+		return RTE_BE32(MLX5_BLOCK_SIZE_512B << MLX5_BLOCK_SIZE_OFFSET);
+	default:
+		DRV_LOG(ERR, "Unknown block size: %u.", bl);
+		return UINT32_MAX;
+	}
+}
+
+/**
+ * Query LKey from a packet buffer for QP. If not found, add the mempool.
+ *
+ * @param priv
+ *   Pointer to the priv object.
+ * @param addr
+ *   Search key.
+ * @param mr_ctrl
+ *   Pointer to per-queue MR control structure.
+ * @param ol_flags
+ *   Mbuf offload features.
+ *
+ * @return
+ *   Searched LKey on success, UINT32_MAX on no match.
+ */
+static __rte_always_inline uint32_t
+mlx5_crypto_addr2mr(struct mlx5_crypto_priv *priv, uintptr_t addr,
+		    struct mlx5_mr_ctrl *mr_ctrl, uint64_t ol_flags)
+{
+	uint32_t lkey;
+
+	/* Check generation bit to see if there's any change on existing MRs. */
+	if (unlikely(*mr_ctrl->dev_gen_ptr != mr_ctrl->cur_gen))
+		mlx5_mr_flush_local_cache(mr_ctrl);
+	/* Linear search on MR cache array. */
+	lkey = mlx5_mr_lookup_lkey(mr_ctrl->cache, &mr_ctrl->mru,
+				   MLX5_MR_CACHE_N, addr);
+	if (likely(lkey != UINT32_MAX))
+		return lkey;
+	/* Take slower bottom-half on miss. */
+	return mlx5_mr_addr2mr_bh(priv->pd, 0, &priv->mr_scache, mr_ctrl, addr,
+				  !!(ol_flags & EXT_ATTACHED_MBUF));
+}
+
+static __rte_always_inline uint32_t
+mlx5_crypto_klm_set(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp,
+		      struct rte_mbuf *mbuf, struct mlx5_wqe_dseg *klm,
+		      uint32_t offset, uint32_t *remain)
+{
+	uint32_t data_len = (rte_pktmbuf_data_len(mbuf) - offset);
+	uintptr_t addr = rte_pktmbuf_mtod_offset(mbuf, uintptr_t, offset);
+
+	if (data_len > *remain)
+		data_len = *remain;
+	*remain -= data_len;
+	klm->bcount = rte_cpu_to_be_32(data_len);
+	klm->pbuf = rte_cpu_to_be_64(addr);
+	klm->lkey = mlx5_crypto_addr2mr(priv, addr, &qp->mr_ctrl,
+					mbuf->ol_flags);
+	return klm->lkey;
+
+}
+
+static __rte_always_inline uint32_t
+mlx5_crypto_klms_set(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp,
+		     struct rte_crypto_op *op, struct rte_mbuf *mbuf,
+		     struct mlx5_wqe_dseg *klm)
+{
+	uint32_t remain_len = op->sym->cipher.data.length;
+	uint32_t nb_segs = mbuf->nb_segs;
+	uint32_t klm_n = 1u;
+
+	/* First mbuf needs to take the cipher offset. */
+	if (unlikely(mlx5_crypto_klm_set(priv, qp, mbuf, klm,
+		     op->sym->cipher.data.offset, &remain_len) == UINT32_MAX)) {
+		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return 0;
+	}
+	while (remain_len) {
+		nb_segs--;
+		mbuf = mbuf->next;
+		if (unlikely(mbuf == NULL || nb_segs == 0)) {
+			op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+			return 0;
+		}
+		if (unlikely(mlx5_crypto_klm_set(priv, qp, mbuf, ++klm, 0,
+						 &remain_len) == UINT32_MAX)) {
+			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			return 0;
+		}
+		klm_n++;
+	}
+	return klm_n;
+}
+
+static __rte_always_inline int
+mlx5_crypto_wqe_set(struct mlx5_crypto_priv *priv,
+			 struct mlx5_crypto_qp *qp,
+			 struct rte_crypto_op *op,
+			 struct mlx5_umr_wqe *umr)
+{
+	struct mlx5_crypto_session *sess = get_sym_session_private_data
+				(op->sym->session, mlx5_crypto_driver_id);
+	struct mlx5_wqe_cseg *cseg = &umr->ctr;
+	struct mlx5_wqe_mkey_cseg *mkc = &umr->mkc;
+	struct mlx5_wqe_dseg *klms = &umr->kseg[0];
+	struct mlx5_wqe_umr_bsf_seg *bsf = ((struct mlx5_wqe_umr_bsf_seg *)
+				      RTE_PTR_ADD(umr, priv->umr_wqe_size)) - 1;
+	uint32_t ds;
+	bool ipl = op->sym->m_dst == NULL || op->sym->m_dst == op->sym->m_src;
+	/* Set UMR WQE. */
+	uint32_t klm_n = mlx5_crypto_klms_set(priv, qp, op,
+				   ipl ? op->sym->m_src : op->sym->m_dst, klms);
+
+	if (unlikely(klm_n == 0))
+		return 0;
+	bsf->bs_bpt_eo_es = sess->bs_bpt_eo_es;
+	if (unlikely(!sess->bsp_res)) {
+		bsf->bsp_res = mlx5_crypto_get_block_size(op);
+		if (unlikely(bsf->bsp_res == UINT32_MAX)) {
+			op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+			return 0;
+		}
+	} else {
+		bsf->bsp_res = sess->bsp_res;
+	}
+	bsf->raw_data_size = rte_cpu_to_be_32(op->sym->cipher.data.length);
+	memcpy(bsf->xts_initial_tweak,
+	       rte_crypto_op_ctod_offset(op, uint8_t *, sess->iv_offset), 16);
+	bsf->res_dp = sess->dek_id;
+	mkc->len = rte_cpu_to_be_64(op->sym->cipher.data.length);
+	cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) | MLX5_OPCODE_UMR);
+	qp->db_pi += priv->umr_wqe_stride;
+	/* Set RDMA_WRITE WQE. */
+	cseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size);
+	klms = RTE_PTR_ADD(cseg, sizeof(struct mlx5_rdma_write_wqe));
+	if (!ipl) {
+		klm_n = mlx5_crypto_klms_set(priv, qp, op, op->sym->m_src,
+					     klms);
+		if (unlikely(klm_n == 0))
+			return 0;
+	} else {
+		memcpy(klms, &umr->kseg[0], sizeof(*klms) * klm_n);
+	}
+	ds = 2 + klm_n;
+	cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj->id << 8) | ds);
+	cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) |
+							MLX5_OPCODE_RDMA_WRITE);
+	ds = RTE_ALIGN(ds, 4);
+	qp->db_pi += ds >> 2;
+	/* Set NOP WQE if needed. */
+	if (priv->max_rdmar_ds > ds) {
+		cseg += ds;
+		ds = priv->max_rdmar_ds - ds;
+		cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj->id << 8) | ds);
+		cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) |
+							       MLX5_OPCODE_NOP);
+		qp->db_pi += ds >> 2; /* Here, DS is 4 aligned for sure. */
+	}
+	qp->wqe = (uint8_t *)cseg;
+	return 1;
+}
+
+static __rte_always_inline void
+mlx5_crypto_uar_write(uint64_t val, struct mlx5_crypto_priv *priv)
+{
+#ifdef RTE_ARCH_64
+	*priv->uar_addr = val;
+#else /* !RTE_ARCH_64 */
+	rte_spinlock_lock(&priv->uar32_sl);
+	*(volatile uint32_t *)priv->uar_addr = val;
+	rte_io_wmb();
+	*((volatile uint32_t *)priv->uar_addr + 1) = val >> 32;
+	rte_spinlock_unlock(&priv->uar32_sl);
+#endif
+}
+
+static uint16_t
+mlx5_crypto_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
+			  uint16_t nb_ops)
+{
+	struct mlx5_crypto_qp *qp = queue_pair;
+	struct mlx5_crypto_priv *priv = qp->priv;
+	struct mlx5_umr_wqe *umr;
+	struct rte_crypto_op *op;
+	uint16_t mask = qp->entries_n - 1;
+	uint16_t remain = qp->entries_n - (qp->pi - qp->ci);
+
+	if (remain < nb_ops)
+		nb_ops = remain;
+	else
+		remain = nb_ops;
+	if (unlikely(remain == 0))
+		return 0;
+	do {
+		op = *ops++;
+		umr = RTE_PTR_ADD(qp->umem_buf, priv->wqe_set_size * qp->pi);
+		if (unlikely(mlx5_crypto_wqe_set(priv, qp, op, umr) == 0)) {
+			if (remain != nb_ops)
+				break;
+			return 0;
+		}
+		qp->ops[qp->pi] = op;
+		qp->pi = (qp->pi + 1) & mask;
+	} while (--remain);
+	rte_io_wmb();
+	qp->db_rec[MLX5_SND_DBR] = rte_cpu_to_be_32(qp->db_pi);
+	rte_wmb();
+	mlx5_crypto_uar_write(*(volatile uint64_t *)qp->wqe, qp->priv);
+	rte_wmb();
+	return nb_ops;
+}
+
+static __rte_noinline void
+mlx5_crypto_cqe_err_handle(struct mlx5_crypto_qp *qp, struct rte_crypto_op *op)
+{
+	const uint32_t idx = qp->ci & (qp->entries_n - 1);
+	volatile struct mlx5_err_cqe *cqe = (volatile struct mlx5_err_cqe *)
+							&qp->cq_obj.cqes[idx];
+
+	op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+	DRV_LOG(ERR, "CQE ERR:%x.\n", rte_be_to_cpu_32(cqe->syndrome));
+}
+
+static uint16_t
+mlx5_crypto_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
+			  uint16_t nb_ops)
+{
+	struct mlx5_crypto_qp *qp = queue_pair;
+	volatile struct mlx5_cqe *restrict cqe;
+	struct rte_crypto_op *restrict op;
+	const unsigned int cq_size = qp->entries_n;
+	const unsigned int mask = cq_size - 1;
+	uint32_t idx;
+	uint32_t next_idx = qp->ci & mask;
+	const uint16_t max = RTE_MIN((uint16_t)(qp->pi - qp->ci), nb_ops);
+	uint16_t i = 0;
+	int ret;
+
+	if (unlikely(max == 0))
+		return 0;
+	do {
+		idx = next_idx;
+		next_idx = (qp->ci + 1) & mask;
+		op = qp->ops[idx];
+		cqe = &qp->cq_obj.cqes[idx];
+		ret = check_cqe(cqe, cq_size, qp->ci);
+		rte_io_rmb();
+		if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) {
+			if (unlikely(ret != MLX5_CQE_STATUS_HW_OWN))
+				mlx5_crypto_cqe_err_handle(qp, op);
+			break;
+		}
+		op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+		ops[i++] = op;
+		qp->ci++;
+	} while (i < max);
+	if (likely(i != 0)) {
+		rte_io_wmb();
+		qp->cq_obj.db_rec[0] = rte_cpu_to_be_32(qp->ci);
+	}
+	return i;
+}
+
 static void
 mlx5_crypto_qp_init(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp)
 {
@@ -521,8 +798,9 @@ mlx5_crypto_hw_global_prepare(struct mlx5_crypto_priv *priv)
 	if (mlx5_crypto_pd_create(priv) != 0)
 		return -1;
 	priv->uar = mlx5_devx_alloc_uar(priv->ctx, -1);
-	if (priv->uar == NULL || mlx5_os_get_devx_uar_reg_addr(priv->uar) ==
-	    NULL) {
+	if (priv->uar)
+		priv->uar_addr = mlx5_os_get_devx_uar_reg_addr(priv->uar);
+	if (priv->uar == NULL || priv->uar_addr == NULL) {
 		rte_errno = errno;
 		claim_zero(mlx5_glue->dealloc_pd(priv->pd));
 		DRV_LOG(ERR, "Failed to allocate UAR.");
@@ -752,8 +1030,8 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	DRV_LOG(INFO,
 		"Crypto device %s was created successfully.", ibv->name);
 	crypto_dev->dev_ops = &mlx5_crypto_ops;
-	crypto_dev->dequeue_burst = NULL;
-	crypto_dev->enqueue_burst = NULL;
+	crypto_dev->dequeue_burst = mlx5_crypto_dequeue_burst;
+	crypto_dev->enqueue_burst = mlx5_crypto_enqueue_burst;
 	crypto_dev->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
 	crypto_dev->driver_id = mlx5_crypto_driver_id;
 	priv = crypto_dev->data->dev_private;
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 12e5890bf7..8e90735182 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -36,6 +36,9 @@ struct mlx5_crypto_priv {
 	uint16_t umr_wqe_size;
 	uint16_t umr_wqe_stride;
 	uint16_t max_rdmar_ds;
+#ifndef RTE_ARCH_64
+	rte_spinlock_t uar32_sl;
+#endif /* RTE_ARCH_64 */
 };
 
 struct mlx5_crypto_qp {
-- 
2.27.0

[dpdk-dev] [PATCH v5 13/15] crypto/mlx5: add statistic get and reset operations

[Shiri Kuzin]

From: Suanming Mou <suanmingm@nvidia.com>

This commit adds mlx5 crypto statistic get and reset operations.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 40 ++++++++++++++++++++++++++++---
 1 file changed, 37 insertions(+), 3 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 47dbb9c385..44273dd2cd 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -508,13 +508,17 @@ mlx5_crypto_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
 		op = *ops++;
 		umr = RTE_PTR_ADD(qp->umem_buf, priv->wqe_set_size * qp->pi);
 		if (unlikely(mlx5_crypto_wqe_set(priv, qp, op, umr) == 0)) {
-			if (remain != nb_ops)
+			qp->stats.enqueue_err_count++;
+			if (remain != nb_ops) {
+				qp->stats.enqueued_count -= remain;
 				break;
+			}
 			return 0;
 		}
 		qp->ops[qp->pi] = op;
 		qp->pi = (qp->pi + 1) & mask;
 	} while (--remain);
+	qp->stats.enqueued_count += nb_ops;
 	rte_io_wmb();
 	qp->db_rec[MLX5_SND_DBR] = rte_cpu_to_be_32(qp->db_pi);
 	rte_wmb();
@@ -531,6 +535,7 @@ mlx5_crypto_cqe_err_handle(struct mlx5_crypto_qp *qp, struct rte_crypto_op *op)
 							&qp->cq_obj.cqes[idx];
 
 	op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+	qp->stats.dequeue_err_count++;
 	DRV_LOG(ERR, "CQE ERR:%x.\n", rte_be_to_cpu_32(cqe->syndrome));
 }
 
@@ -570,6 +575,7 @@ mlx5_crypto_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
 	if (likely(i != 0)) {
 		rte_io_wmb();
 		qp->cq_obj.db_rec[0] = rte_cpu_to_be_32(qp->ci);
+		qp->stats.dequeued_count += i;
 	}
 	return i;
 }
@@ -731,14 +737,42 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	return -1;
 }
 
+static void
+mlx5_crypto_stats_get(struct rte_cryptodev *dev,
+		      struct rte_cryptodev_stats *stats)
+{
+	int qp_id;
+
+	for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
+		struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
+
+		stats->enqueued_count += qp->stats.enqueued_count;
+		stats->dequeued_count += qp->stats.dequeued_count;
+		stats->enqueue_err_count += qp->stats.enqueue_err_count;
+		stats->dequeue_err_count += qp->stats.dequeue_err_count;
+	}
+}
+
+static void
+mlx5_crypto_stats_reset(struct rte_cryptodev *dev)
+{
+	int qp_id;
+
+	for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
+		struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
+
+		memset(&qp->stats, 0, sizeof(qp->stats));
+	}
+}
+
 static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_configure			= mlx5_crypto_dev_configure,
 	.dev_start			= mlx5_crypto_dev_start,
 	.dev_stop			= mlx5_crypto_dev_stop,
 	.dev_close			= mlx5_crypto_dev_close,
 	.dev_infos_get			= mlx5_crypto_dev_infos_get,
-	.stats_get			= NULL,
-	.stats_reset			= NULL,
+	.stats_get			= mlx5_crypto_stats_get,
+	.stats_reset			= mlx5_crypto_stats_reset,
 	.queue_pair_setup		= mlx5_crypto_queue_pair_setup,
 	.queue_pair_release		= mlx5_crypto_queue_pair_release,
 	.sym_session_get_size		= mlx5_crypto_sym_session_get_size,
-- 
2.27.0

[dpdk-dev] [PATCH v5 14/15] test/crypto: add mlx5 crypto driver

[Shiri Kuzin]

In order to test the new mlx5 crypto PMD, the driver is added to the
crypto test application.

Added mlx5 vectors that include 4 testing vectors with length 512 and
4096.

Added mlx5 encryption function and decryption function that will both
use the mlx5 vectors that will also set the dataunit_len and use a
wrapped key.

The added tests will test both data integrity and correct stat values.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 app/test/meson.build                        |    1 +
 app/test/test_cryptodev.c                   |  334 ++-
 app/test/test_cryptodev.h                   |    3 +-
 app/test/test_cryptodev_mlx5_test_vectors.h | 2502 +++++++++++++++++++
 4 files changed, 2823 insertions(+), 17 deletions(-)
 create mode 100644 app/test/test_cryptodev_mlx5_test_vectors.h

diff --git a/app/test/meson.build b/app/test/meson.build
index 0a5f425578..c36655888e 100644
--- a/app/test/meson.build
+++ b/app/test/meson.build
@@ -313,6 +313,7 @@ driver_test_names = [
         'cryptodev_aesni_gcm_autotest',
         'cryptodev_dpaa_sec_autotest',
         'cryptodev_dpaa2_sec_autotest',
+        'cryptodev_mlx5_autotest',
         'cryptodev_null_autotest',
         'cryptodev_octeontx2_autotest',
         'cryptodev_openssl_autotest',
diff --git a/app/test/test_cryptodev.c b/app/test/test_cryptodev.c
index 39db52b17a..8dbe324b81 100644
--- a/app/test/test_cryptodev.c
+++ b/app/test/test_cryptodev.c
@@ -35,6 +35,7 @@
 #include "test_cryptodev_hash_test_vectors.h"
 #include "test_cryptodev_kasumi_test_vectors.h"
 #include "test_cryptodev_kasumi_hash_test_vectors.h"
+#include "test_cryptodev_mlx5_test_vectors.h"
 #include "test_cryptodev_snow3g_test_vectors.h"
 #include "test_cryptodev_snow3g_hash_test_vectors.h"
 #include "test_cryptodev_zuc_test_vectors.h"
@@ -1267,6 +1268,39 @@ negative_hmac_sha1_testsuite_setup(void)
 	return 0;
 }
 
+static int
+mlx5_testsuite_setup(void)
+{
+	struct crypto_testsuite_params *ts_params = &testsuite_params;
+	uint8_t dev_id = ts_params->valid_devs[0];
+	struct rte_cryptodev_info dev_info;
+	const enum rte_crypto_cipher_algorithm ciphers[] = {
+		RTE_CRYPTO_CIPHER_AES_XTS
+	};
+
+	rte_cryptodev_info_get(dev_id, &dev_info);
+
+	if (!(dev_info.feature_flags & RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO) &&
+			!(dev_info.feature_flags &
+				RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY) &&
+			!(dev_info.feature_flags &
+				RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS)) {
+		RTE_LOG(INFO, USER1,
+		"Feature flag requirements for MLX5 testsuite not met\n");
+
+		return TEST_SKIPPED;
+	}
+
+	if (check_cipher_capabilities_supported(ciphers,
+						RTE_DIM(ciphers)) != 0) {
+		RTE_LOG(INFO, USER1,
+		"Capability requirements for MLX5 testsuite not met\n");
+		return TEST_SKIPPED;
+	}
+
+	return 0;
+}
+
 static int
 dev_configure_and_start(uint64_t ff_disable)
 {
@@ -2366,7 +2400,7 @@ create_wireless_algo_cipher_session(uint8_t dev_id,
 			enum rte_crypto_cipher_operation op,
 			enum rte_crypto_cipher_algorithm algo,
 			const uint8_t *key, const uint8_t key_len,
-			uint8_t iv_len)
+			uint8_t iv_len, uint64_t dataunit_len)
 {
 	uint8_t cipher_key[key_len];
 	int status;
@@ -2378,13 +2412,13 @@ create_wireless_algo_cipher_session(uint8_t dev_id,
 	/* Setup Cipher Parameters */
 	ut_params->cipher_xform.type = RTE_CRYPTO_SYM_XFORM_CIPHER;
 	ut_params->cipher_xform.next = NULL;
-
 	ut_params->cipher_xform.cipher.algo = algo;
 	ut_params->cipher_xform.cipher.op = op;
 	ut_params->cipher_xform.cipher.key.data = cipher_key;
 	ut_params->cipher_xform.cipher.key.length = key_len;
 	ut_params->cipher_xform.cipher.iv.offset = IV_OFFSET;
 	ut_params->cipher_xform.cipher.iv.length = iv_len;
+	ut_params->cipher_xform.cipher.dataunit_len = dataunit_len;
 
 	debug_hexdump(stdout, "key:", key, key_len);
 
@@ -3528,7 +3562,7 @@ test_kasumi_encryption(const struct kasumi_test_data *tdata)
 					RTE_CRYPTO_CIPHER_OP_ENCRYPT,
 					RTE_CRYPTO_CIPHER_KASUMI_F8,
 					tdata->key.data, tdata->key.len,
-					tdata->cipher_iv.len);
+					tdata->cipher_iv.len, 0);
 	if (retval < 0)
 		return retval;
 
@@ -3631,7 +3665,7 @@ test_kasumi_encryption_sgl(const struct kasumi_test_data *tdata)
 					RTE_CRYPTO_CIPHER_OP_ENCRYPT,
 					RTE_CRYPTO_CIPHER_KASUMI_F8,
 					tdata->key.data, tdata->key.len,
-					tdata->cipher_iv.len);
+					tdata->cipher_iv.len, 0);
 	if (retval < 0)
 		return retval;
 
@@ -3718,7 +3752,7 @@ test_kasumi_encryption_oop(const struct kasumi_test_data *tdata)
 					RTE_CRYPTO_CIPHER_OP_ENCRYPT,
 					RTE_CRYPTO_CIPHER_KASUMI_F8,
 					tdata->key.data, tdata->key.len,
-					tdata->cipher_iv.len);
+					tdata->cipher_iv.len, 0);
 	if (retval < 0)
 		return retval;
 
@@ -3815,7 +3849,7 @@ test_kasumi_encryption_oop_sgl(const struct kasumi_test_data *tdata)
 					RTE_CRYPTO_CIPHER_OP_ENCRYPT,
 					RTE_CRYPTO_CIPHER_KASUMI_F8,
 					tdata->key.data, tdata->key.len,
-					tdata->cipher_iv.len);
+					tdata->cipher_iv.len, 0);
 	if (retval < 0)
 		return retval;
 
@@ -3896,7 +3930,7 @@ test_kasumi_decryption_oop(const struct kasumi_test_data *tdata)
 					RTE_CRYPTO_CIPHER_OP_DECRYPT,
 					RTE_CRYPTO_CIPHER_KASUMI_F8,
 					tdata->key.data, tdata->key.len,
-					tdata->cipher_iv.len);
+					tdata->cipher_iv.len, 0);
 	if (retval < 0)
 		return retval;
 
@@ -3986,7 +4020,7 @@ test_kasumi_decryption(const struct kasumi_test_data *tdata)
 					RTE_CRYPTO_CIPHER_OP_DECRYPT,
 					RTE_CRYPTO_CIPHER_KASUMI_F8,
 					tdata->key.data, tdata->key.len,
-					tdata->cipher_iv.len);
+					tdata->cipher_iv.len, 0);
 	if (retval < 0)
 		return retval;
 
@@ -4078,7 +4112,7 @@ test_snow3g_encryption(const struct snow3g_test_data *tdata)
 					RTE_CRYPTO_CIPHER_OP_ENCRYPT,
 					RTE_CRYPTO_CIPHER_SNOW3G_UEA2,
 					tdata->key.data, tdata->key.len,
-					tdata->cipher_iv.len);
+					tdata->cipher_iv.len, 0);
 	if (retval < 0)
 		return retval;
 
@@ -4162,7 +4196,7 @@ test_snow3g_encryption_oop(const struct snow3g_test_data *tdata)
 					RTE_CRYPTO_CIPHER_OP_ENCRYPT,
 					RTE_CRYPTO_CIPHER_SNOW3G_UEA2,
 					tdata->key.data, tdata->key.len,
-					tdata->cipher_iv.len);
+					tdata->cipher_iv.len, 0);
 	if (retval < 0)
 		return retval;
 
@@ -4262,7 +4296,7 @@ test_snow3g_encryption_oop_sgl(const struct snow3g_test_data *tdata)
 					RTE_CRYPTO_CIPHER_OP_ENCRYPT,
 					RTE_CRYPTO_CIPHER_SNOW3G_UEA2,
 					tdata->key.data, tdata->key.len,
-					tdata->cipher_iv.len);
+					tdata->cipher_iv.len, 0);
 	if (retval < 0)
 		return retval;
 
@@ -4376,7 +4410,7 @@ test_snow3g_encryption_offset_oop(const struct snow3g_test_data *tdata)
 					RTE_CRYPTO_CIPHER_OP_ENCRYPT,
 					RTE_CRYPTO_CIPHER_SNOW3G_UEA2,
 					tdata->key.data, tdata->key.len,
-					tdata->cipher_iv.len);
+					tdata->cipher_iv.len, 0);
 	if (retval < 0)
 		return retval;
 
@@ -4488,7 +4522,7 @@ static int test_snow3g_decryption(const struct snow3g_test_data *tdata)
 					RTE_CRYPTO_CIPHER_OP_DECRYPT,
 					RTE_CRYPTO_CIPHER_SNOW3G_UEA2,
 					tdata->key.data, tdata->key.len,
-					tdata->cipher_iv.len);
+					tdata->cipher_iv.len, 0);
 	if (retval < 0)
 		return retval;
 
@@ -4569,7 +4603,7 @@ static int test_snow3g_decryption_oop(const struct snow3g_test_data *tdata)
 					RTE_CRYPTO_CIPHER_OP_DECRYPT,
 					RTE_CRYPTO_CIPHER_SNOW3G_UEA2,
 					tdata->key.data, tdata->key.len,
-					tdata->cipher_iv.len);
+					tdata->cipher_iv.len, 0);
 	if (retval < 0)
 		return retval;
 
@@ -5811,7 +5845,7 @@ test_zuc_encryption(const struct wireless_test_data *tdata)
 					RTE_CRYPTO_CIPHER_OP_ENCRYPT,
 					RTE_CRYPTO_CIPHER_ZUC_EEA3,
 					tdata->key.data, tdata->key.len,
-					tdata->cipher_iv.len);
+					tdata->cipher_iv.len, 0);
 	if (retval < 0)
 		return retval;
 
@@ -5924,7 +5958,7 @@ test_zuc_encryption_sgl(const struct wireless_test_data *tdata)
 			RTE_CRYPTO_CIPHER_OP_ENCRYPT,
 			RTE_CRYPTO_CIPHER_ZUC_EEA3,
 			tdata->key.data, tdata->key.len,
-			tdata->cipher_iv.len);
+			tdata->cipher_iv.len, 0);
 	if (retval < 0)
 		return retval;
 
@@ -6460,6 +6494,193 @@ test_zuc_auth_cipher_sgl(const struct wireless_test_data *tdata,
 	return 0;
 }
 
+static int
+test_mlx5_encryption(const struct mlx5_test_data *tdata)
+{
+	struct crypto_testsuite_params *ts_params = &testsuite_params;
+	struct crypto_unittest_params *ut_params = &unittest_params;
+	struct rte_cryptodev_sym_capability_idx cap_idx;
+	struct rte_cryptodev_info dev_info;
+	struct rte_cryptodev_stats stats;
+	uint8_t *plaintext, *ciphertext;
+	uint64_t feat_flags;
+	unsigned int plaintext_pad_len;
+	unsigned int plaintext_len;
+	int retval;
+
+	rte_cryptodev_info_get(ts_params->valid_devs[0], &dev_info);
+	feat_flags = dev_info.feature_flags;
+	if ((global_api_test_type == CRYPTODEV_RAW_API_TEST) &&
+			(!(feat_flags & RTE_CRYPTODEV_FF_SYM_RAW_DP))) {
+		printf("Device doesn't support RAW data-path APIs.\n");
+		return -ENOTSUP;
+	}
+	if (gbl_action_type == RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO)
+		return -ENOTSUP;
+	/* Verify the capabilities */
+	cap_idx.type = RTE_CRYPTO_SYM_XFORM_CIPHER;
+	cap_idx.algo.cipher = RTE_CRYPTO_CIPHER_AES_XTS;
+	if (rte_cryptodev_sym_capability_get(ts_params->valid_devs[0],
+			&cap_idx) == NULL)
+		return -ENOTSUP;
+	/* Create mlx5 session */
+	retval = create_wireless_algo_cipher_session(ts_params->valid_devs[0],
+						RTE_CRYPTO_CIPHER_OP_ENCRYPT,
+						RTE_CRYPTO_CIPHER_AES_XTS,
+						tdata->key.data, tdata->key.len,
+						tdata->cipher_iv.len, 0);
+	if (retval < 0)
+		return retval;
+	ut_params->ibuf = rte_pktmbuf_alloc(ts_params->mbuf_pool);
+	if (unlikely(ut_params->ibuf == NULL))
+		return -ENOMEM;
+	/* Clear mbuf payload */
+	memset(rte_pktmbuf_mtod(ut_params->ibuf, uint8_t *), 0,
+	       rte_pktmbuf_tailroom(ut_params->ibuf));
+	plaintext_len = ceil_byte_length(tdata->plaintext.len);
+	/* Append data which is padded to a multiple */
+	/* of the algorithms block size */
+	plaintext_pad_len = RTE_ALIGN_CEIL(plaintext_len, 8);
+	plaintext = (uint8_t *)rte_pktmbuf_append(ut_params->ibuf,
+				plaintext_pad_len);
+	if (unlikely(plaintext == NULL))
+		return -ENOMEM;
+	memcpy(plaintext, tdata->plaintext.data, plaintext_len);
+	debug_hexdump(stdout, "plaintext:", plaintext, plaintext_len);
+	retval = create_wireless_algo_cipher_operation(tdata->cipher_iv.data,
+				tdata->cipher_iv.len, (tdata->cipher.len_bits),
+				(tdata->cipher.offset_bits));
+	if (retval < 0)
+		return retval;
+	if (global_api_test_type == CRYPTODEV_RAW_API_TEST)
+		process_sym_raw_dp_op(ts_params->valid_devs[0], 0,
+				ut_params->op, 1, 0, 1, tdata->cipher_iv.len);
+	else
+		ut_params->op = process_crypto_request(ts_params->valid_devs[0],
+						ut_params->op);
+	TEST_ASSERT_NOT_NULL(ut_params->op, "failed to retrieve obuf");
+	ut_params->obuf = ut_params->op->sym->m_dst;
+	if (ut_params->obuf)
+		ciphertext = rte_pktmbuf_mtod(ut_params->obuf, uint8_t *);
+	else
+		ciphertext = plaintext;
+	debug_hexdump(stdout, "ciphertext:", ciphertext, plaintext_len);
+	/* Validate obuf */
+	TEST_ASSERT_BUFFERS_ARE_EQUAL_BIT(
+		ciphertext,
+		tdata->ciphertext.data,
+		tdata->validCipherLenInBits.len,
+		"MLX5 Ciphertext data not as expected");
+	/* Validate stats */
+	TEST_ASSERT_SUCCESS(rte_cryptodev_stats_get(ts_params->valid_devs[0],
+			    &stats), "rte_cryptodev_stats_get failed");
+	TEST_ASSERT((stats.enqueued_count == 1),
+		   "rte_cryptodev_stats_get returned unexpected enqueued stat");
+	TEST_ASSERT((stats.dequeued_count == 1),
+		   "rte_cryptodev_stats_get returned unexpected dequeued stat");
+	TEST_ASSERT((stats.enqueue_err_count == 0),
+		   "rte_cryptodev_stats_get returned error enqueued stat");
+	TEST_ASSERT((stats.dequeue_err_count == 0),
+		   "rte_cryptodev_stats_get returned error dequeued stat");
+	return 0;
+}
+
+static int
+test_mlx5_decryption(const struct mlx5_test_data *tdata)
+{
+	struct crypto_testsuite_params *ts_params = &testsuite_params;
+	struct crypto_unittest_params *ut_params = &unittest_params;
+	struct rte_cryptodev_sym_capability_idx cap_idx;
+	struct rte_cryptodev_info dev_info;
+	struct rte_cryptodev_stats stats;
+	const uint8_t *reference_plaintext;
+	uint8_t *ciphertext, *plaintext;
+	uint64_t feat_flags;
+	unsigned int ciphertext_pad_len;
+	unsigned int ciphertext_len;
+	int retval;
+
+
+	rte_cryptodev_info_get(ts_params->valid_devs[0], &dev_info);
+	feat_flags = dev_info.feature_flags;
+	if ((global_api_test_type == CRYPTODEV_RAW_API_TEST) &&
+			(!(feat_flags & RTE_CRYPTODEV_FF_SYM_RAW_DP))) {
+		printf("Device doesn't support RAW data-path APIs.\n");
+		return -ENOTSUP;
+	}
+	if (gbl_action_type == RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO)
+		return -ENOTSUP;
+	/* Verify the capabilities */
+	cap_idx.type = RTE_CRYPTO_SYM_XFORM_CIPHER;
+	cap_idx.algo.cipher = RTE_CRYPTO_CIPHER_AES_XTS;
+	if (rte_cryptodev_sym_capability_get(ts_params->valid_devs[0],
+			&cap_idx) == NULL)
+		return -ENOTSUP;
+	/* Create mlx5 session */
+	retval = create_wireless_algo_cipher_session(ts_params->valid_devs[0],
+						RTE_CRYPTO_CIPHER_OP_DECRYPT,
+						RTE_CRYPTO_CIPHER_AES_XTS,
+						tdata->key.data, tdata->key.len,
+						tdata->cipher_iv.len, 0);
+	if (retval < 0)
+		return retval;
+	ut_params->ibuf = rte_pktmbuf_alloc(ts_params->mbuf_pool);
+	if (unlikely(ut_params->ibuf == NULL))
+		return -ENOMEM;
+	/* Clear mbuf payload */
+	memset(rte_pktmbuf_mtod(ut_params->ibuf, uint8_t *), 0,
+	       rte_pktmbuf_tailroom(ut_params->ibuf));
+	ciphertext_len = ceil_byte_length(tdata->ciphertext.len);
+	/* Append data which is padded to a multiple */
+	/* of the algorithms block size */
+	ciphertext_pad_len = RTE_ALIGN_CEIL(ciphertext_len, 8);
+	ciphertext = (uint8_t *)rte_pktmbuf_append(ut_params->ibuf,
+				ciphertext_pad_len);
+	if (unlikely(ciphertext == NULL))
+		return -ENOMEM;
+	memcpy(ciphertext, tdata->ciphertext.data, ciphertext_len);
+	debug_hexdump(stdout, "ciphertext:", ciphertext, ciphertext_len);
+	/* Create mlx5 operation */
+	retval = create_wireless_algo_cipher_operation(tdata->cipher_iv.data,
+				tdata->cipher_iv.len, (tdata->cipher.len_bits),
+				(tdata->cipher.offset_bits));
+	if (retval < 0)
+		return retval;
+	if (global_api_test_type == CRYPTODEV_RAW_API_TEST)
+		process_sym_raw_dp_op(ts_params->valid_devs[0], 0,
+				ut_params->op, 1, 0, 1, 0);
+	else
+		ut_params->op = process_crypto_request(ts_params->valid_devs[0],
+						ut_params->op);
+	TEST_ASSERT_NOT_NULL(ut_params->op, "failed to retrieve obuf");
+	ut_params->obuf = ut_params->op->sym->m_dst;
+	if (ut_params->obuf)
+		plaintext = rte_pktmbuf_mtod(ut_params->obuf, uint8_t *);
+	else
+		plaintext = ciphertext + (tdata->cipher.offset_bits);
+	debug_hexdump(stdout, "plaintext:", plaintext, ciphertext_len);
+	reference_plaintext = tdata->plaintext.data +
+				(tdata->cipher.offset_bits);
+	/* Validate obuf */
+	TEST_ASSERT_BUFFERS_ARE_EQUAL_BIT(
+		plaintext,
+		reference_plaintext,
+		tdata->validCipherLenInBits.len,
+		"MLX5 Plaintext data not as expected");
+	/* Validate stats */
+	TEST_ASSERT_SUCCESS(rte_cryptodev_stats_get(ts_params->valid_devs[0],
+			    &stats), "rte_cryptodev_stats_get failed");
+	TEST_ASSERT((stats.enqueued_count == 1),
+		   "rte_cryptodev_stats_get returned unexpected enqueued stat");
+	TEST_ASSERT((stats.dequeued_count == 1),
+		   "rte_cryptodev_stats_get returned unexpected dequeued stat");
+	TEST_ASSERT((stats.enqueue_err_count == 0),
+		   "rte_cryptodev_stats_get returned error enqueued stat");
+	TEST_ASSERT((stats.dequeue_err_count == 0),
+		   "rte_cryptodev_stats_get returned error dequeued stat");
+	return 0;
+}
+
 static int
 test_kasumi_encryption_test_case_1(void)
 {
@@ -7076,6 +7297,54 @@ test_zuc_auth_cipher_verify_test_case_1_oop_sgl(void)
 		&zuc_auth_cipher_test_case_1, OUT_OF_PLACE, 1);
 }
 
+static int
+test_mlx5_encryption_test_case_1(void)
+{
+	return test_mlx5_encryption(&mlx5_test_case_cipher_aes_xts_1);
+}
+
+static int
+test_mlx5_encryption_test_case_2(void)
+{
+	return test_mlx5_encryption(&mlx5_test_case_cipher_aes_xts_2);
+}
+
+static int
+test_mlx5_encryption_test_case_3(void)
+{
+	return test_mlx5_encryption(&mlx5_test_case_cipher_aes_xts_3);
+}
+
+static int
+test_mlx5_encryption_test_case_4(void)
+{
+	return test_mlx5_encryption(&mlx5_test_case_cipher_aes_xts_4);
+}
+
+static int
+test_mlx5_decryption_test_case_1(void)
+{
+	return test_mlx5_decryption(&mlx5_test_case_cipher_aes_xts_1);
+}
+
+static int
+test_mlx5_decryption_test_case_2(void)
+{
+	return test_mlx5_decryption(&mlx5_test_case_cipher_aes_xts_2);
+}
+
+static int
+test_mlx5_decryption_test_case_3(void)
+{
+	return test_mlx5_decryption(&mlx5_test_case_cipher_aes_xts_3);
+}
+
+static int
+test_mlx5_decryption_test_case_4(void)
+{
+	return test_mlx5_decryption(&mlx5_test_case_cipher_aes_xts_4);
+}
+
 static int
 test_mixed_check_if_unsupported(const struct mixed_cipher_auth_test_data *tdata)
 {
@@ -14458,6 +14727,31 @@ static struct unit_test_suite cryptodev_mixed_cipher_hash_testsuite  = {
 	}
 };
 
+static struct unit_test_suite cryptodev_mlx5_testsuite  = {
+	.suite_name = "MLX5 Test Suite",
+	.setup = mlx5_testsuite_setup,
+	.unit_test_cases = {
+		TEST_CASE_ST(ut_setup, ut_teardown,
+			test_mlx5_decryption_test_case_1),
+		TEST_CASE_ST(ut_setup, ut_teardown,
+			test_mlx5_decryption_test_case_2),
+		TEST_CASE_ST(ut_setup, ut_teardown,
+			test_mlx5_decryption_test_case_3),
+		TEST_CASE_ST(ut_setup, ut_teardown,
+			test_mlx5_decryption_test_case_4),
+		TEST_CASE_ST(ut_setup, ut_teardown,
+			test_mlx5_encryption_test_case_1),
+		TEST_CASE_ST(ut_setup, ut_teardown,
+			test_mlx5_encryption_test_case_2),
+		TEST_CASE_ST(ut_setup, ut_teardown,
+			test_mlx5_encryption_test_case_3),
+		TEST_CASE_ST(ut_setup, ut_teardown,
+			test_mlx5_encryption_test_case_4),
+		TEST_CASES_END()
+	}
+};
+
+
 static int
 run_cryptodev_testsuite(const char *pmd_name)
 {
@@ -14483,6 +14777,7 @@ run_cryptodev_testsuite(const char *pmd_name)
 		&cryptodev_hmac_md5_auth_testsuite,
 		&cryptodev_kasumi_testsuite,
 		&cryptodev_esn_testsuite,
+		&cryptodev_mlx5_testsuite,
 		&cryptodev_negative_aes_gcm_testsuite,
 		&cryptodev_negative_aes_gmac_testsuite,
 		&cryptodev_mixed_cipher_hash_testsuite,
@@ -14590,6 +14885,12 @@ test_cryptodev_sw_kasumi(void /*argv __rte_unused, int argc __rte_unused*/)
 	return run_cryptodev_testsuite(RTE_STR(CRYPTODEV_NAME_KASUMI_PMD));
 }
 
+static int
+test_cryptodev_sw_mlx5(void /*argv __rte_unused, int argc __rte_unused*/)
+{
+	return run_cryptodev_testsuite(RTE_STR(CRYPTODEV_NAME_MLX5_PMD));
+}
+
 static int
 test_cryptodev_sw_zuc(void /*argv __rte_unused, int argc __rte_unused*/)
 {
@@ -14791,6 +15092,7 @@ REGISTER_TEST_COMMAND(cryptodev_cpu_aesni_gcm_autotest,
 REGISTER_TEST_COMMAND(cryptodev_null_autotest, test_cryptodev_null);
 REGISTER_TEST_COMMAND(cryptodev_sw_snow3g_autotest, test_cryptodev_sw_snow3g);
 REGISTER_TEST_COMMAND(cryptodev_sw_kasumi_autotest, test_cryptodev_sw_kasumi);
+REGISTER_TEST_COMMAND(cryptodev_sw_mlx5_autotest, test_cryptodev_sw_mlx5);
 REGISTER_TEST_COMMAND(cryptodev_sw_zuc_autotest, test_cryptodev_sw_zuc);
 REGISTER_TEST_COMMAND(cryptodev_sw_armv8_autotest, test_cryptodev_armv8);
 REGISTER_TEST_COMMAND(cryptodev_sw_mvsam_autotest, test_cryptodev_mrvl);
diff --git a/app/test/test_cryptodev.h b/app/test/test_cryptodev.h
index f81f8e372f..683fe3ddc8 100644
--- a/app/test/test_cryptodev.h
+++ b/app/test/test_cryptodev.h
@@ -19,7 +19,7 @@
 #define DEFAULT_NUM_XFORMS              (2)
 #define NUM_MBUFS                       (8191)
 #define MBUF_CACHE_SIZE                 (256)
-#define MBUF_DATAPAYLOAD_SIZE		(2048 + DIGEST_BYTE_LENGTH_SHA512)
+#define MBUF_DATAPAYLOAD_SIZE		(4096 + DIGEST_BYTE_LENGTH_SHA512)
 #define MBUF_SIZE			(sizeof(struct rte_mbuf) + \
 		RTE_PKTMBUF_HEADROOM + MBUF_DATAPAYLOAD_SIZE)
 
@@ -71,6 +71,7 @@
 #define CRYPTODEV_NAME_CAAM_JR_PMD	crypto_caam_jr
 #define CRYPTODEV_NAME_NITROX_PMD	crypto_nitrox_sym
 #define CRYPTODEV_NAME_BCMFS_PMD	crypto_bcmfs
+#define CRYPTODEV_NAME_MLX5_PMD		mlx5_crypto
 
 enum cryptodev_api_test_type {
 	CRYPTODEV_API_TEST = 0,
diff --git a/app/test/test_cryptodev_mlx5_test_vectors.h b/app/test/test_cryptodev_mlx5_test_vectors.h
new file mode 100644
index 0000000000..2a05aa4626
--- /dev/null
+++ b/app/test/test_cryptodev_mlx5_test_vectors.h
@@ -0,0 +1,2502 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2021 NVIDIA Corporation & Affiliates
+ */
+#ifndef TEST_CRYPTODEV_MLX5_TEST_VECTORS_H_
+#define TEST_CRYPTODEV_MLX5_TEST_VECTORS_H_
+
+struct mlx5_test_data {
+	struct {
+		uint8_t data[64];
+		unsigned int len;
+	} key;
+
+	struct {
+		uint8_t data[64] __rte_aligned(16);
+		unsigned int len;
+	} cipher_iv;
+
+	struct {
+		uint8_t data[32768];
+		unsigned int len; /* length must be in Bits */
+	} plaintext;
+
+	struct {
+		uint8_t data[32768];
+		unsigned int len; /* length must be in Bits */
+	} ciphertext;
+
+	struct {
+		unsigned int len;
+	} validCipherLenInBits;
+
+	struct {
+		unsigned int len_bits; /* length must be in Bits */
+		unsigned int offset_bits;
+	} cipher;
+	uint16_t dataunit_len;
+};
+static struct mlx5_test_data mlx5_test_case_cipher_aes_xts_1 = {
+	.key = {
+		.data = {
+			0x94, 0xD8, 0xAE, 0xD7, 0x74, 0xBF, 0x51, 0xF1,
+			0x2D, 0x19, 0x0B, 0x9A, 0xB3, 0xDC, 0x1E, 0xA9,
+			0x5C, 0xC5, 0x47, 0xC1, 0xBB, 0x94, 0xDB, 0xFA,
+			0x2B, 0x1E, 0x54, 0xA5, 0x76, 0xAD, 0x07, 0x4B,
+			0x46, 0xE0, 0xAA, 0xF9, 0xF7, 0x5C, 0x64, 0x2D,
+			0x8F, 0x04, 0x17, 0x72, 0xE5, 0x8D, 0xFE, 0x9C
+		},
+		.len = 48
+	},
+	.cipher_iv = {
+		.data = {
+			0x4F, 0xAE, 0xF7, 0x11, 0x7C, 0xDA, 0x59, 0xC6,
+			0x6E, 0x4B, 0x92, 0x01, 0x3E, 0x76, 0x8A, 0xD5
+		},
+		.len = 16
+	},
+	.plaintext = {
+		.data = {
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79
+		},
+		.len = 512 << 3
+	},
+	.ciphertext = {
+		.data = {
+			0xD3, 0x94, 0x3E, 0xC9, 0xD3, 0x43, 0x13, 0xD3,
+			0x07, 0x7E, 0x51, 0x11, 0x97, 0xA5, 0xB1, 0xB2,
+			0xB0, 0x55, 0xF8, 0xD6, 0xA2, 0x10, 0x78, 0x76,
+			0x37, 0x95, 0x4F, 0x25, 0x99, 0xCC, 0x12, 0xD6,
+			0xBB, 0x2A, 0x24, 0xF4, 0x2D, 0x82, 0xFF, 0x56,
+			0xEC, 0x55, 0x3B, 0xDD, 0xF9, 0xE1, 0xC8, 0x48,
+			0x68, 0x16, 0xD2, 0x2B, 0x95, 0x7C, 0x1F, 0xCB,
+			0x32, 0xD6, 0x8F, 0x9A, 0x2E, 0xF7, 0x9B, 0xBE,
+			0x72, 0xFC, 0x55, 0x33, 0x27, 0x21, 0x2F, 0x69,
+			0xCA, 0x76, 0xA5, 0x9B, 0x21, 0x2E, 0x40, 0x57,
+			0x2C, 0x1C, 0x98, 0x41, 0x9A, 0x5E, 0x55, 0x38,
+			0xDE, 0xC2, 0x09, 0x57, 0x32, 0xA3, 0x34, 0x40,
+			0xC1, 0x8D, 0xCF, 0x70, 0x15, 0xF2, 0x6F, 0x4F,
+			0x6A, 0x04, 0xA6, 0x6D, 0xFF, 0x53, 0x25, 0x6A,
+			0x0E, 0xD0, 0x87, 0x63, 0xA1, 0x6C, 0xB1, 0x99,
+			0x4A, 0x42, 0xF5, 0xF6, 0xEA, 0xA4, 0xEB, 0x6D,
+			0x70, 0x9B, 0x0F, 0x85, 0xE2, 0x43, 0x6C, 0x27,
+			0x54, 0x57, 0x52, 0x1C, 0xCF, 0x72, 0x11, 0x83,
+			0xC3, 0xF5, 0xC6, 0xB6, 0x07, 0xEC, 0x1A, 0xF5,
+			0xAC, 0xA1, 0xF2, 0x3A, 0x01, 0x53, 0x0C, 0xA4,
+			0x40, 0x19, 0xAF, 0x7B, 0x2D, 0xE7, 0x56, 0x8B,
+			0x71, 0x5F, 0x8D, 0x96, 0xF1, 0x5D, 0x33, 0xAC,
+			0xE7, 0xAD, 0x96, 0xDB, 0xBB, 0xF7, 0xF5, 0x3F,
+			0x0E, 0x02, 0x2E, 0x80, 0xE9, 0xA2, 0x30, 0xD2,
+			0x82, 0x65, 0xEC, 0x1A, 0xB2, 0xE8, 0x98, 0x23,
+			0x42, 0x79, 0x43, 0x85, 0x3B, 0xF9, 0xFB, 0xBB,
+			0xC5, 0x29, 0xAD, 0x95, 0xAE, 0x22, 0x5B, 0x26,
+			0xDF, 0x76, 0x65, 0x37, 0x17, 0xBC, 0x58, 0xBB,
+			0x1B, 0x0A, 0x71, 0xDC, 0x68, 0xFF, 0x90, 0x36,
+			0x37, 0xB7, 0x49, 0x5E, 0x49, 0x4E, 0xE4, 0x1E,
+			0x24, 0x39, 0x48, 0xC7, 0x68, 0x19, 0xED, 0x94,
+			0xB5, 0xF6, 0x85, 0xFA, 0xE6, 0xB8, 0x2B, 0x9E,
+			0x07, 0x9E, 0xFB, 0x1D, 0x61, 0x06, 0x47, 0x2A,
+			0x3E, 0x1E, 0xD8, 0x52, 0xFB, 0xB7, 0xE3, 0xB4,
+			0x0D, 0xA8, 0x15, 0x1E, 0x98, 0x02, 0xBD, 0x89,
+			0x89, 0xE3, 0x38, 0x2C, 0xAB, 0x50, 0x25, 0x30,
+			0xB4, 0x5E, 0xA5, 0xCD, 0xA8, 0x9B, 0xA4, 0x2A,
+			0xED, 0x19, 0x3B, 0xC3, 0x05, 0x07, 0x57, 0xB5,
+			0x52, 0x11, 0x74, 0x95, 0x51, 0x5A, 0xD8, 0xED,
+			0xDF, 0x37, 0x91, 0x83, 0x27, 0xD5, 0x08, 0x82,
+			0xB3, 0x42, 0x08, 0xC7, 0x81, 0x35, 0x5F, 0x58,
+			0x28, 0x69, 0x0D, 0x97, 0x1D, 0x28, 0xE6, 0xB0,
+			0x58, 0x93, 0xCE, 0x2A, 0xB4, 0x7D, 0x4B, 0x83,
+			0x20, 0x1E, 0x08, 0xEF, 0x68, 0x51, 0xEB, 0xB4,
+			0xFA, 0x78, 0xB5, 0xE5, 0x2D, 0x93, 0x07, 0x99,
+			0xBB, 0xDD, 0x9A, 0x4E, 0xD6, 0xE7, 0x31, 0x9F,
+			0x4D, 0xB4, 0x05, 0x45, 0x89, 0x59, 0x42, 0xFF,
+			0x0C, 0xD4, 0xC2, 0xF3, 0xA0, 0xF7, 0xD3, 0xCE,
+			0x26, 0x2F, 0x9B, 0xCB, 0x98, 0x24, 0xDE, 0x50,
+			0xF6, 0x81, 0x24, 0xA3, 0xB4, 0x99, 0x94, 0xDB,
+			0xF5, 0xF4, 0x74, 0xAC, 0xED, 0xF1, 0xD6, 0x2F,
+			0x89, 0x48, 0x41, 0x24, 0xBB, 0xA7, 0x64, 0x48,
+			0xAE, 0x36, 0xE3, 0xF3, 0xA8, 0x37, 0x4F, 0xE6,
+			0x24, 0x92, 0xC4, 0x27, 0x86, 0x68, 0xC4, 0xA2,
+			0x2F, 0xB2, 0xA8, 0xFF, 0x27, 0x5F, 0x59, 0xB9,
+			0x86, 0x89, 0x5A, 0x09, 0xB8, 0xDA, 0xD6, 0xBA,
+			0x26, 0x86, 0x51, 0xC2, 0xDF, 0xCF, 0x07, 0x49,
+			0x1C, 0xB7, 0x1A, 0x80, 0xD4, 0x97, 0x13, 0xAA,
+			0xE8, 0x38, 0xA7, 0x2C, 0xB7, 0xE2, 0xE0, 0x33,
+			0xF9, 0xC3, 0x21, 0xF6, 0x2A, 0xAA, 0x1A, 0xF7,
+			0x39, 0x1F, 0x2F, 0x7E, 0x38, 0xC8, 0x82, 0x3F,
+			0xCC, 0xD8, 0xE2, 0xD9, 0x02, 0xE6, 0x8E, 0x31,
+			0xA7, 0x37, 0xE8, 0x59, 0x3F, 0xE1, 0x60, 0x1C,
+			0x74, 0x52, 0xE4, 0x49, 0xF7, 0x21, 0x60, 0x11
+		},
+		.len = 512 << 3
+	},
+	.validCipherLenInBits = {
+		.len = 512 << 3
+	},
+	.cipher = {
+		.len_bits = 512,
+		.offset_bits = 0
+	},
+	.dataunit_len = 512
+};
+
+static struct mlx5_test_data mlx5_test_case_cipher_aes_xts_2 = {
+	.key = {
+		.data = {
+			0x94, 0xD8, 0xAE, 0xD7, 0x74, 0xBF, 0x51, 0xF1,
+			0x2D, 0x19, 0x0B, 0x9A, 0xB3, 0xDC, 0x1E, 0xA9,
+			0x5C, 0xC5, 0x47, 0xC1, 0xBB, 0x94, 0xDB, 0xFA,
+			0x2B, 0x1E, 0x54, 0xA5, 0x76, 0xAD, 0x07, 0x4B,
+			0x46, 0xE0, 0xAA, 0xF9, 0xF7, 0x5C, 0x64, 0x2D,
+			0x8F, 0x04, 0x17, 0x72, 0xE5, 0x8D, 0xFE, 0x9C
+		},
+		.len = 48
+	},
+	.cipher_iv = {
+		.data = {
+			0x4F, 0xAE, 0xF7, 0x11, 0x7C, 0xDA, 0x59, 0xC6,
+			0x6E, 0x4B, 0x92, 0x01, 0x3E, 0x76, 0x8A, 0xD5
+		},
+		.len = 16
+	},
+	.plaintext = {
+		.data = {
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79
+		},
+		.len = 512 << 3
+	},
+	.ciphertext = {
+		.data = {
+			0xD3, 0x94, 0x3E, 0xC9, 0xD3, 0x43, 0x13, 0xD3,
+			0x07, 0x7E, 0x51, 0x11, 0x97, 0xA5, 0xB1, 0xB2,
+			0xB0, 0x55, 0xF8, 0xD6, 0xA2, 0x10, 0x78, 0x76,
+			0x37, 0x95, 0x4F, 0x25, 0x99, 0xCC, 0x12, 0xD6,
+			0xBB, 0x2A, 0x24, 0xF4, 0x2D, 0x82, 0xFF, 0x56,
+			0xEC, 0x55, 0x3B, 0xDD, 0xF9, 0xE1, 0xC8, 0x48,
+			0x68, 0x16, 0xD2, 0x2B, 0x95, 0x7C, 0x1F, 0xCB,
+			0x32, 0xD6, 0x8F, 0x9A, 0x2E, 0xF7, 0x9B, 0xBE,
+			0x72, 0xFC, 0x55, 0x33, 0x27, 0x21, 0x2F, 0x69,
+			0xCA, 0x76, 0xA5, 0x9B, 0x21, 0x2E, 0x40, 0x57,
+			0x2C, 0x1C, 0x98, 0x41, 0x9A, 0x5E, 0x55, 0x38,
+			0xDE, 0xC2, 0x09, 0x57, 0x32, 0xA3, 0x34, 0x40,
+			0xC1, 0x8D, 0xCF, 0x70, 0x15, 0xF2, 0x6F, 0x4F,
+			0x6A, 0x04, 0xA6, 0x6D, 0xFF, 0x53, 0x25, 0x6A,
+			0x0E, 0xD0, 0x87, 0x63, 0xA1, 0x6C, 0xB1, 0x99,
+			0x4A, 0x42, 0xF5, 0xF6, 0xEA, 0xA4, 0xEB, 0x6D,
+			0x70, 0x9B, 0x0F, 0x85, 0xE2, 0x43, 0x6C, 0x27,
+			0x54, 0x57, 0x52, 0x1C, 0xCF, 0x72, 0x11, 0x83,
+			0xC3, 0xF5, 0xC6, 0xB6, 0x07, 0xEC, 0x1A, 0xF5,
+			0xAC, 0xA1, 0xF2, 0x3A, 0x01, 0x53, 0x0C, 0xA4,
+			0x40, 0x19, 0xAF, 0x7B, 0x2D, 0xE7, 0x56, 0x8B,
+			0x71, 0x5F, 0x8D, 0x96, 0xF1, 0x5D, 0x33, 0xAC,
+			0xE7, 0xAD, 0x96, 0xDB, 0xBB, 0xF7, 0xF5, 0x3F,
+			0x0E, 0x02, 0x2E, 0x80, 0xE9, 0xA2, 0x30, 0xD2,
+			0x82, 0x65, 0xEC, 0x1A, 0xB2, 0xE8, 0x98, 0x23,
+			0x42, 0x79, 0x43, 0x85, 0x3B, 0xF9, 0xFB, 0xBB,
+			0xC5, 0x29, 0xAD, 0x95, 0xAE, 0x22, 0x5B, 0x26,
+			0xDF, 0x76, 0x65, 0x37, 0x17, 0xBC, 0x58, 0xBB,
+			0x1B, 0x0A, 0x71, 0xDC, 0x68, 0xFF, 0x90, 0x36,
+			0x37, 0xB7, 0x49, 0x5E, 0x49, 0x4E, 0xE4, 0x1E,
+			0x24, 0x39, 0x48, 0xC7, 0x68, 0x19, 0xED, 0x94,
+			0xB5, 0xF6, 0x85, 0xFA, 0xE6, 0xB8, 0x2B, 0x9E,
+			0x07, 0x9E, 0xFB, 0x1D, 0x61, 0x06, 0x47, 0x2A,
+			0x3E, 0x1E, 0xD8, 0x52, 0xFB, 0xB7, 0xE3, 0xB4,
+			0x0D, 0xA8, 0x15, 0x1E, 0x98, 0x02, 0xBD, 0x89,
+			0x89, 0xE3, 0x38, 0x2C, 0xAB, 0x50, 0x25, 0x30,
+			0xB4, 0x5E, 0xA5, 0xCD, 0xA8, 0x9B, 0xA4, 0x2A,
+			0xED, 0x19, 0x3B, 0xC3, 0x05, 0x07, 0x57, 0xB5,
+			0x52, 0x11, 0x74, 0x95, 0x51, 0x5A, 0xD8, 0xED,
+			0xDF, 0x37, 0x91, 0x83, 0x27, 0xD5, 0x08, 0x82,
+			0xB3, 0x42, 0x08, 0xC7, 0x81, 0x35, 0x5F, 0x58,
+			0x28, 0x69, 0x0D, 0x97, 0x1D, 0x28, 0xE6, 0xB0,
+			0x58, 0x93, 0xCE, 0x2A, 0xB4, 0x7D, 0x4B, 0x83,
+			0x20, 0x1E, 0x08, 0xEF, 0x68, 0x51, 0xEB, 0xB4,
+			0xFA, 0x78, 0xB5, 0xE5, 0x2D, 0x93, 0x07, 0x99,
+			0xBB, 0xDD, 0x9A, 0x4E, 0xD6, 0xE7, 0x31, 0x9F,
+			0x4D, 0xB4, 0x05, 0x45, 0x89, 0x59, 0x42, 0xFF,
+			0x0C, 0xD4, 0xC2, 0xF3, 0xA0, 0xF7, 0xD3, 0xCE,
+			0x26, 0x2F, 0x9B, 0xCB, 0x98, 0x24, 0xDE, 0x50,
+			0xF6, 0x81, 0x24, 0xA3, 0xB4, 0x99, 0x94, 0xDB,
+			0xF5, 0xF4, 0x74, 0xAC, 0xED, 0xF1, 0xD6, 0x2F,
+			0x89, 0x48, 0x41, 0x24, 0xBB, 0xA7, 0x64, 0x48,
+			0xAE, 0x36, 0xE3, 0xF3, 0xA8, 0x37, 0x4F, 0xE6,
+			0x24, 0x92, 0xC4, 0x27, 0x86, 0x68, 0xC4, 0xA2,
+			0x2F, 0xB2, 0xA8, 0xFF, 0x27, 0x5F, 0x59, 0xB9,
+			0x86, 0x89, 0x5A, 0x09, 0xB8, 0xDA, 0xD6, 0xBA,
+			0x26, 0x86, 0x51, 0xC2, 0xDF, 0xCF, 0x07, 0x49,
+			0x1C, 0xB7, 0x1A, 0x80, 0xD4, 0x97, 0x13, 0xAA,
+			0xE8, 0x38, 0xA7, 0x2C, 0xB7, 0xE2, 0xE0, 0x33,
+			0xF9, 0xC3, 0x21, 0xF6, 0x2A, 0xAA, 0x1A, 0xF7,
+			0x39, 0x1F, 0x2F, 0x7E, 0x38, 0xC8, 0x82, 0x3F,
+			0xCC, 0xD8, 0xE2, 0xD9, 0x02, 0xE6, 0x8E, 0x31,
+			0xA7, 0x37, 0xE8, 0x59, 0x3F, 0xE1, 0x60, 0x1C,
+			0x74, 0x52, 0xE4, 0x49, 0xF7, 0x21, 0x60, 0x11
+		},
+		.len = 512 << 3
+	},
+	.validCipherLenInBits = {
+		.len = 512 << 3
+	},
+	.cipher = {
+		.len_bits = 512,
+		.offset_bits = 0
+	},
+	.dataunit_len = 0
+};
+
+static struct mlx5_test_data mlx5_test_case_cipher_aes_xts_3 = {
+	.key = {
+		.data = {
+			0x94, 0xD8, 0xAE, 0xD7, 0x74, 0xBF, 0x51, 0xF1,
+			0x2D, 0x19, 0x0B, 0x9A, 0xB3, 0xDC, 0x1E, 0xA9,
+			0x5C, 0xC5, 0x47, 0xC1, 0xBB, 0x94, 0xDB, 0xFA,
+			0x2B, 0x1E, 0x54, 0xA5, 0x76, 0xAD, 0x07, 0x4B,
+			0x46, 0xE0, 0xAA, 0xF9, 0xF7, 0x5C, 0x64, 0x2D,
+			0x8F, 0x04, 0x17, 0x72, 0xE5, 0x8D, 0xFE, 0x9C
+		},
+		.len = 48
+	},
+	.cipher_iv = {
+		.data = {
+			0x4F, 0xAE, 0xF7, 0x11, 0x7C, 0xDA, 0x59, 0xC6,
+			0x6E, 0x4B, 0x92, 0x01, 0x3E, 0x76, 0x8A, 0xD5
+		},
+		.len = 16
+	},
+	.plaintext = {
+		.data = {
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79
+		},
+		.len = 4096 << 3
+	},
+	.ciphertext = {
+		.data = {
+			0xD3, 0x94, 0x3E, 0xC9, 0xD3, 0x43, 0x13, 0xD3,
+			0x07, 0x7E, 0x51, 0x11, 0x97, 0xA5, 0xB1, 0xB2,
+			0xB0, 0x55, 0xF8, 0xD6, 0xA2, 0x10, 0x78, 0x76,
+			0x37, 0x95, 0x4F, 0x25, 0x99, 0xCC, 0x12, 0xD6,
+			0xBB, 0x2A, 0x24, 0xF4, 0x2D, 0x82, 0xFF, 0x56,
+			0xEC, 0x55, 0x3B, 0xDD, 0xF9, 0xE1, 0xC8, 0x48,
+			0x68, 0x16, 0xD2, 0x2B, 0x95, 0x7C, 0x1F, 0xCB,
+			0x32, 0xD6, 0x8F, 0x9A, 0x2E, 0xF7, 0x9B, 0xBE,
+			0x72, 0xFC, 0x55, 0x33, 0x27, 0x21, 0x2F, 0x69,
+			0xCA, 0x76, 0xA5, 0x9B, 0x21, 0x2E, 0x40, 0x57,
+			0x2C, 0x1C, 0x98, 0x41, 0x9A, 0x5E, 0x55, 0x38,
+			0xDE, 0xC2, 0x09, 0x57, 0x32, 0xA3, 0x34, 0x40,
+			0xC1, 0x8D, 0xCF, 0x70, 0x15, 0xF2, 0x6F, 0x4F,
+			0x6A, 0x04, 0xA6, 0x6D, 0xFF, 0x53, 0x25, 0x6A,
+			0x0E, 0xD0, 0x87, 0x63, 0xA1, 0x6C, 0xB1, 0x99,
+			0x4A, 0x42, 0xF5, 0xF6, 0xEA, 0xA4, 0xEB, 0x6D,
+			0x70, 0x9B, 0x0F, 0x85, 0xE2, 0x43, 0x6C, 0x27,
+			0x54, 0x57, 0x52, 0x1C, 0xCF, 0x72, 0x11, 0x83,
+			0xC3, 0xF5, 0xC6, 0xB6, 0x07, 0xEC, 0x1A, 0xF5,
+			0xAC, 0xA1, 0xF2, 0x3A, 0x01, 0x53, 0x0C, 0xA4,
+			0x40, 0x19, 0xAF, 0x7B, 0x2D, 0xE7, 0x56, 0x8B,
+			0x71, 0x5F, 0x8D, 0x96, 0xF1, 0x5D, 0x33, 0xAC,
+			0xE7, 0xAD, 0x96, 0xDB, 0xBB, 0xF7, 0xF5, 0x3F,
+			0x0E, 0x02, 0x2E, 0x80, 0xE9, 0xA2, 0x30, 0xD2,
+			0x82, 0x65, 0xEC, 0x1A, 0xB2, 0xE8, 0x98, 0x23,
+			0x42, 0x79, 0x43, 0x85, 0x3B, 0xF9, 0xFB, 0xBB,
+			0xC5, 0x29, 0xAD, 0x95, 0xAE, 0x22, 0x5B, 0x26,
+			0xDF, 0x76, 0x65, 0x37, 0x17, 0xBC, 0x58, 0xBB,
+			0x1B, 0x0A, 0x71, 0xDC, 0x68, 0xFF, 0x90, 0x36,
+			0x37, 0xB7, 0x49, 0x5E, 0x49, 0x4E, 0xE4, 0x1E,
+			0x24, 0x39, 0x48, 0xC7, 0x68, 0x19, 0xED, 0x94,
+			0xB5, 0xF6, 0x85, 0xFA, 0xE6, 0xB8, 0x2B, 0x9E,
+			0x07, 0x9E, 0xFB, 0x1D, 0x61, 0x06, 0x47, 0x2A,
+			0x3E, 0x1E, 0xD8, 0x52, 0xFB, 0xB7, 0xE3, 0xB4,
+			0x0D, 0xA8, 0x15, 0x1E, 0x98, 0x02, 0xBD, 0x89,
+			0x89, 0xE3, 0x38, 0x2C, 0xAB, 0x50, 0x25, 0x30,
+			0xB4, 0x5E, 0xA5, 0xCD, 0xA8, 0x9B, 0xA4, 0x2A,
+			0xED, 0x19, 0x3B, 0xC3, 0x05, 0x07, 0x57, 0xB5,
+			0x52, 0x11, 0x74, 0x95, 0x51, 0x5A, 0xD8, 0xED,
+			0xDF, 0x37, 0x91, 0x83, 0x27, 0xD5, 0x08, 0x82,
+			0xB3, 0x42, 0x08, 0xC7, 0x81, 0x35, 0x5F, 0x58,
+			0x28, 0x69, 0x0D, 0x97, 0x1D, 0x28, 0xE6, 0xB0,
+			0x58, 0x93, 0xCE, 0x2A, 0xB4, 0x7D, 0x4B, 0x83,
+			0x20, 0x1E, 0x08, 0xEF, 0x68, 0x51, 0xEB, 0xB4,
+			0xFA, 0x78, 0xB5, 0xE5, 0x2D, 0x93, 0x07, 0x99,
+			0xBB, 0xDD, 0x9A, 0x4E, 0xD6, 0xE7, 0x31, 0x9F,
+			0x4D, 0xB4, 0x05, 0x45, 0x89, 0x59, 0x42, 0xFF,
+			0x0C, 0xD4, 0xC2, 0xF3, 0xA0, 0xF7, 0xD3, 0xCE,
+			0x26, 0x2F, 0x9B, 0xCB, 0x98, 0x24, 0xDE, 0x50,
+			0xF6, 0x81, 0x24, 0xA3, 0xB4, 0x99, 0x94, 0xDB,
+			0xF5, 0xF4, 0x74, 0xAC, 0xED, 0xF1, 0xD6, 0x2F,
+			0x89, 0x48, 0x41, 0x24, 0xBB, 0xA7, 0x64, 0x48,
+			0xAE, 0x36, 0xE3, 0xF3, 0xA8, 0x37, 0x4F, 0xE6,
+			0x24, 0x92, 0xC4, 0x27, 0x86, 0x68, 0xC4, 0xA2,
+			0x2F, 0xB2, 0xA8, 0xFF, 0x27, 0x5F, 0x59, 0xB9,
+			0x86, 0x89, 0x5A, 0x09, 0xB8, 0xDA, 0xD6, 0xBA,
+			0x26, 0x86, 0x51, 0xC2, 0xDF, 0xCF, 0x07, 0x49,
+			0x1C, 0xB7, 0x1A, 0x80, 0xD4, 0x97, 0x13, 0xAA,
+			0xE8, 0x38, 0xA7, 0x2C, 0xB7, 0xE2, 0xE0, 0x33,
+			0xF9, 0xC3, 0x21, 0xF6, 0x2A, 0xAA, 0x1A, 0xF7,
+			0x39, 0x1F, 0x2F, 0x7E, 0x38, 0xC8, 0x82, 0x3F,
+			0xCC, 0xD8, 0xE2, 0xD9, 0x02, 0xE6, 0x8E, 0x31,
+			0xA7, 0x37, 0xE8, 0x59, 0x3F, 0xE1, 0x60, 0x1C,
+			0x74, 0x52, 0xE4, 0x49, 0xF7, 0x21, 0x60, 0x11,
+			0x8F, 0xD0, 0xA1, 0x4B, 0xE3, 0xF1, 0x18, 0x3D,
+			0x90, 0x89, 0x54, 0x27, 0xA0, 0xF9, 0x32, 0x09,
+			0x3D, 0x9D, 0x9A, 0x09, 0x53, 0xC6, 0x7E, 0x95,
+			0x85, 0x53, 0x98, 0x4C, 0x23, 0xEA, 0x54, 0xBD,
+			0x6F, 0x50, 0xBC, 0x4C, 0xCF, 0x37, 0xC5, 0x7B,
+			0x4B, 0xCE, 0x84, 0xAF, 0xE2, 0xE2, 0x55, 0x49,
+			0xBC, 0xBF, 0x92, 0xCA, 0x1E, 0x5E, 0x10, 0xDF,
+			0x60, 0x87, 0x09, 0xA1, 0x4C, 0x1D, 0x7E, 0x1E,
+			0x59, 0xE9, 0xCF, 0xDA, 0x45, 0x3F, 0xE2, 0x0F,
+			0x53, 0x8D, 0x8B, 0x79, 0xBD, 0xD8, 0xB0, 0xE3,
+			0x5B, 0x7C, 0x55, 0x4A, 0x84, 0xF0, 0x1E, 0xF9,
+			0xE8, 0xF3, 0x09, 0x4D, 0x0B, 0xD7, 0x77, 0xCC,
+			0x3F, 0x70, 0x22, 0x7D, 0x17, 0x27, 0x48, 0x57,
+			0xE2, 0x36, 0xA0, 0x84, 0x3B, 0xDE, 0x05, 0x34,
+			0xEF, 0x55, 0x12, 0xF4, 0x9A, 0x99, 0x0D, 0x28,
+			0x86, 0x28, 0x99, 0x6B, 0x22, 0xEE, 0x63, 0xF0,
+			0x68, 0x9C, 0xE1, 0x70, 0xF6, 0x26, 0xD8, 0x3B,
+			0xF9, 0x57, 0x18, 0x3D, 0xAD, 0x66, 0xF0, 0xCF,
+			0x7B, 0x0C, 0x28, 0x4D, 0xB8, 0xEB, 0x7B, 0x04,
+			0x1E, 0x7D, 0x40, 0x5F, 0x5A, 0x1E, 0x7E, 0x08,
+			0x7F, 0x4C, 0x1E, 0x18, 0xE5, 0x3E, 0x6C, 0x90,
+			0x3C, 0x89, 0x13, 0x2A, 0xC4, 0x2A, 0x94, 0xB5,
+			0x3E, 0x18, 0x1C, 0x4C, 0xBA, 0xEA, 0x86, 0xD2,
+			0x05, 0xA9, 0x59, 0x9C, 0x80, 0xC2, 0x45, 0xAD,
+			0x30, 0x99, 0x18, 0x6A, 0x2F, 0x73, 0x8C, 0xF0,
+			0xFE, 0xA4, 0xBD, 0x44, 0x3E, 0xEB, 0x98, 0x75,
+			0x48, 0x08, 0x57, 0x45, 0xD8, 0x41, 0xDE, 0x61,
+			0x6D, 0x06, 0x93, 0xC4, 0x99, 0x1B, 0x23, 0xB5,
+			0x12, 0x22, 0x5C, 0xC7, 0x9E, 0x18, 0xEA, 0x64,
+			0xDB, 0xCE, 0x1A, 0xAC, 0x5D, 0x9B, 0x80, 0xE2,
+			0xBF, 0x3E, 0xC2, 0xA4, 0x78, 0x4F, 0xF1, 0xE3,
+			0x7D, 0x2A, 0x20, 0x94, 0x13, 0xCD, 0xF3, 0x1C,
+			0x33, 0x9C, 0xC1, 0x59, 0x85, 0x52, 0xCB, 0xDB,
+			0x03, 0xDF, 0x11, 0xE6, 0xAB, 0x95, 0x82, 0x65,
+			0x7A, 0x88, 0x73, 0xEE, 0xBA, 0x21, 0x1C, 0x2F,
+			0xCD, 0xD7, 0xC6, 0xE5, 0x13, 0xDE, 0x7A, 0x9E,
+			0xEE, 0x83, 0x8D, 0xC6, 0x47, 0x63, 0xE0, 0xC7,
+			0xC4, 0xBE, 0x19, 0x25, 0xEE, 0xCC, 0x0A, 0x13,
+			0x18, 0x9D, 0x34, 0x5B, 0x55, 0x6C, 0xC1, 0x6E,
+			0xBF, 0x5A, 0xC5, 0x61, 0x75, 0x77, 0x49, 0x8C,
+			0x67, 0x61, 0xE8, 0x72, 0x87, 0xE8, 0xCA, 0xBE,
+			0x6E, 0xC3, 0xD7, 0x81, 0x8C, 0x78, 0x79, 0xC8,
+			0x72, 0xDA, 0x1A, 0x40, 0x7D, 0x60, 0xE2, 0x5A,
+			0x47, 0x38, 0xA1, 0x21, 0x22, 0x6B, 0x54, 0x74,
+			0xDD, 0xF3, 0xBC, 0x96, 0x28, 0x7E, 0xC2, 0x8B,
+			0x13, 0xEE, 0x53, 0xBC, 0x34, 0x67, 0x07, 0x87,
+			0xD5, 0x6B, 0x93, 0x22, 0x21, 0xB9, 0xED, 0x17,
+			0xE4, 0xA1, 0x96, 0xB2, 0xC6, 0xFF, 0x79, 0xA0,
+			0xA7, 0xF9, 0xDD, 0x92, 0x78, 0xF7, 0xE3, 0x16,
+			0x79, 0xEF, 0xEF, 0x31, 0x4D, 0x1D, 0x75, 0xF9,
+			0xCF, 0x5A, 0x1B, 0x68, 0x16, 0x7F, 0xAF, 0x5F,
+			0x30, 0xB7, 0xEF, 0xF8, 0x94, 0x63, 0x73, 0x3D,
+			0xB3, 0x63, 0xE4, 0xE6, 0xD8, 0xAD, 0xF4, 0x80,
+			0x5E, 0x82, 0xA3, 0xFB, 0x3D, 0x0F, 0xCF, 0x59,
+			0xB8, 0x76, 0xAF, 0x27, 0x83, 0xE3, 0x2D, 0x6F,
+			0xE0, 0xF3, 0x11, 0xD5, 0xAE, 0x82, 0x14, 0x1D,
+			0x78, 0x95, 0xBF, 0x31, 0x22, 0x1B, 0x80, 0x12,
+			0x02, 0xD7, 0x4D, 0x1B, 0x92, 0xE3, 0x15, 0xBA,
+			0x67, 0xD6, 0x8F, 0xD4, 0xDA, 0xBF, 0xD4, 0x62,
+			0xAD, 0x76, 0xDA, 0x04, 0xA2, 0xEA, 0x98, 0xD3,
+			0xC3, 0x6E, 0x5F, 0x26, 0x3C, 0x5E, 0xD9, 0xEA,
+			0x09, 0xF0, 0x02, 0xFC, 0xD2, 0x11, 0xF8, 0xA8,
+			0x7E, 0xFF, 0x06, 0x28, 0x5B, 0xE5, 0x6F, 0x9A,
+			0x00, 0xE7, 0x7F, 0xB9, 0xFB, 0x59, 0xBB, 0xDD,
+			0x85, 0xF3, 0x40, 0xCE, 0xA3, 0x5E, 0x2E, 0x2E,
+			0x34, 0x91, 0x58, 0x41, 0x00, 0xB5, 0xE1, 0x88,
+			0x24, 0x51, 0xC7, 0xB6, 0xF3, 0x21, 0x52, 0x6E,
+			0xE7, 0xFC, 0x42, 0xE7, 0x9C, 0xCC, 0x1E, 0x51,
+			0x45, 0x39, 0xBE, 0x09, 0xFE, 0x1A, 0xC4, 0xF0,
+			0x79, 0xF4, 0x05, 0xC7, 0xA7, 0xF8, 0x0F, 0xB6,
+			0x5A, 0x7B, 0xD7, 0xE1, 0x6F, 0xF0, 0x9D, 0x67,
+			0xA3, 0xE3, 0x3E, 0x2E, 0xB9, 0x8C, 0x83, 0x9E,
+			0xFD, 0x2E, 0xA2, 0x92, 0x99, 0x3C, 0xC0, 0x99,
+			0x01, 0xAB, 0x0D, 0xFA, 0x55, 0x96, 0x04, 0x60,
+			0x1A, 0xAD, 0x4C, 0xBB, 0x3D, 0xBB, 0x7D, 0x8B,
+			0x9F, 0x28, 0x85, 0x7D, 0xB9, 0xE4, 0x05, 0x79,
+			0x7B, 0x63, 0xDD, 0x7F, 0x4D, 0xE7, 0x50, 0xD9,
+			0x41, 0xFF, 0x53, 0xB1, 0xCE, 0x42, 0x7B, 0xD6,
+			0x05, 0x1B, 0x4E, 0xAF, 0xC4, 0x8C, 0x17, 0xC8,
+			0x52, 0xBD, 0x03, 0x3B, 0x92, 0x57, 0x4E, 0xA8,
+			0x15, 0xC3, 0x26, 0x1C, 0x55, 0xC1, 0xFF, 0xAE,
+			0xA9, 0x26, 0x2D, 0xA7, 0x8E, 0x3A, 0x7F, 0xA3,
+			0x48, 0xA5, 0xBC, 0x14, 0x84, 0xF2, 0x90, 0xCE,
+			0x35, 0x0F, 0x64, 0x6B, 0xD8, 0x1C, 0x12, 0xFE,
+			0x5A, 0x4F, 0x0E, 0xCE, 0x81, 0x4E, 0x79, 0x6B,
+			0xCF, 0x56, 0xA7, 0xDB, 0x24, 0xBC, 0xB0, 0x84,
+			0x4C, 0xB0, 0xDA, 0xBE, 0xE6, 0x8F, 0xD7, 0x8E,
+			0x0E, 0xA0, 0xD3, 0x55, 0xC2, 0x4A, 0x34, 0x1C,
+			0xF9, 0xC7, 0x3D, 0x29, 0x70, 0x8B, 0xF0, 0x99,
+			0x61, 0xF5, 0x11, 0xFB, 0x82, 0xE2, 0x67, 0x35,
+			0x60, 0x78, 0x47, 0x81, 0x2A, 0x74, 0x5E, 0x4D,
+			0x48, 0xD3, 0x7C, 0x32, 0xCA, 0x1B, 0xD2, 0xA2,
+			0x5C, 0x3A, 0x2F, 0xCE, 0xB4, 0x6C, 0x3A, 0x6A,
+			0x8F, 0x67, 0x46, 0x12, 0xE7, 0xAE, 0x6A, 0x3B,
+			0x99, 0x04, 0x5E, 0x96, 0xD0, 0xB9, 0x84, 0xF6,
+			0xA7, 0x64, 0x11, 0xE8, 0x0C, 0x51, 0xFD, 0x3F,
+			0x18, 0xFA, 0xE8, 0x52, 0xD9, 0x4B, 0x99, 0x7A,
+			0x25, 0x2B, 0x1B, 0x21, 0xAD, 0x8C, 0xFE, 0x0D,
+			0x34, 0x51, 0x91, 0x75, 0x55, 0x6F, 0xEB, 0x9F,
+			0x42, 0xDC, 0x73, 0x7D, 0x31, 0x0A, 0x74, 0x13,
+			0x80, 0xB8, 0xC3, 0xED, 0x73, 0x9D, 0x79, 0x42,
+			0xC0, 0x33, 0xAB, 0xC1, 0xCB, 0xB9, 0xD0, 0xBE,
+			0xA0, 0x78, 0xB8, 0x3B, 0xEB, 0x3D, 0x1A, 0x3F,
+			0xFB, 0x9B, 0xAA, 0x8F, 0x89, 0xF9, 0xD1, 0x22,
+			0x82, 0xE6, 0x66, 0xEE, 0x2A, 0xFD, 0x9F, 0xF8,
+			0x92, 0x7E, 0x10, 0xF5, 0xD5, 0x23, 0x0B, 0xB1,
+			0xD6, 0xF1, 0x7A, 0x3D, 0x73, 0xE9, 0xCE, 0x7F,
+			0xE6, 0x0B, 0x17, 0xBC, 0x23, 0xAE, 0x72, 0xB6,
+			0xFA, 0x19, 0x46, 0xBB, 0xFE, 0xA4, 0xC1, 0x64,
+			0xA8, 0x5E, 0xE8, 0xBB, 0x63, 0x58, 0x19, 0x50,
+			0xAA, 0x36, 0xC2, 0x4B, 0x38, 0x24, 0xD1, 0x2E,
+			0xAE, 0xAD, 0x6E, 0x34, 0x64, 0xA8, 0xC8, 0xF2,
+			0x4E, 0x74, 0x5C, 0x98, 0xE8, 0xDF, 0x99, 0x8C,
+			0x41, 0x79, 0x60, 0x2D, 0xD5, 0xF4, 0xE3, 0xE9,
+			0x1D, 0xF6, 0x5A, 0xA9, 0x69, 0x8E, 0xA1, 0x4F,
+			0xD3, 0x1B, 0x09, 0xA8, 0x7A, 0xD1, 0xE1, 0xCF,
+			0xAC, 0xBA, 0xD0, 0xD1, 0x34, 0x34, 0x8F, 0xC1,
+			0xD1, 0xA8, 0xAF, 0x6E, 0x92, 0xE0, 0xB0, 0xF6,
+			0xF9, 0x08, 0xA0, 0xCB, 0x58, 0x34, 0xF6, 0x68,
+			0xA2, 0xBF, 0x05, 0x39, 0x63, 0xBA, 0x4F, 0xEF,
+			0xE3, 0x95, 0x69, 0xD5, 0x89, 0x7C, 0x64, 0x07,
+			0x13, 0x42, 0x14, 0xF1, 0xA6, 0x9B, 0x87, 0xE5,
+			0xF4, 0x49, 0xAE, 0x67, 0x65, 0xCC, 0xF2, 0x26,
+			0xF8, 0x31, 0xBD, 0x33, 0x6A, 0x87, 0x77, 0x4E,
+			0xB1, 0xEE, 0xA4, 0xA2, 0xC8, 0xA0, 0x4A, 0xC1,
+			0xDF, 0x55, 0xE0, 0xDE, 0x53, 0x15, 0x3B, 0xEC,
+			0x55, 0x32, 0xCA, 0x06, 0xE4, 0x78, 0x59, 0x63,
+			0x10, 0x68, 0xA9, 0x46, 0x1B, 0xEF, 0x73, 0x6D,
+			0x1A, 0x02, 0x64, 0x12, 0x76, 0x9B, 0xDB, 0x7C,
+			0x03, 0x35, 0x19, 0xE1, 0x58, 0x7A, 0x87, 0x0C,
+			0x76, 0xDC, 0xFC, 0xC0, 0x28, 0xE4, 0xA2, 0x07,
+			0x9C, 0x28, 0x05, 0x21, 0x13, 0x58, 0xEF, 0x05,
+			0xBB, 0xAB, 0x94, 0xA2, 0x93, 0xBC, 0x31, 0x61,
+			0x26, 0x39, 0x38, 0x0C, 0xC4, 0x67, 0xDA, 0xA5,
+			0xE4, 0x1E, 0x1B, 0xB6, 0xE5, 0x73, 0xD6, 0x6C,
+			0xEE, 0xBC, 0x9D, 0xB9, 0xE7, 0xD9, 0x45, 0x2F,
+			0xF2, 0xB6, 0x92, 0x54, 0x41, 0x05, 0xB7, 0xB7,
+			0xFC, 0x37, 0x63, 0x6A, 0xB4, 0xBE, 0xB8, 0x3E,
+			0xD8, 0x53, 0x3B, 0xF8, 0x7D, 0x9A, 0x05, 0xDF,
+			0x20, 0x02, 0x27, 0x64, 0x38, 0xFA, 0x7D, 0xAF,
+			0x7F, 0xFA, 0xD1, 0xB7, 0x32, 0xC5, 0x74, 0x3E,
+			0x04, 0xA2, 0x67, 0x79, 0x02, 0x2E, 0x6F, 0xA1,
+			0x27, 0x87, 0x07, 0xB5, 0x9F, 0x0A, 0x7D, 0x5E,
+			0x14, 0xA0, 0x31, 0x46, 0x3F, 0xA9, 0xDE, 0x98,
+			0xB9, 0x89, 0xA0, 0x4A, 0x7A, 0xBD, 0x15, 0xAE,
+			0x2D, 0x0B, 0x38, 0x9A, 0xD8, 0x0E, 0xD2, 0xBA,
+			0x6D, 0xA1, 0x04, 0x1E, 0x4E, 0x39, 0x87, 0x4B,
+			0xC8, 0x3C, 0x74, 0x35, 0x4D, 0xC8, 0x1B, 0x42,
+			0x06, 0x5B, 0x73, 0xB7, 0x33, 0x86, 0x4A, 0x10,
+			0x2A, 0x10, 0x16, 0x28, 0x6F, 0x2A, 0xE3, 0x86,
+			0xDE, 0xA3, 0x44, 0x23, 0xE2, 0x90, 0xC4, 0x20,
+			0x90, 0xE0, 0xB8, 0xE6, 0xA7, 0xB6, 0xD6, 0x92,
+			0xF4, 0xF8, 0x8A, 0xBC, 0xAC, 0x31, 0x47, 0x8F,
+			0xAA, 0xE0, 0xD9, 0xF7, 0xE3, 0xCB, 0x11, 0xA4,
+			0x6B, 0x05, 0xB3, 0xB8, 0x72, 0x69, 0xE6, 0xDD,
+			0x75, 0x0F, 0x20, 0x1D, 0x3F, 0xC6, 0x96, 0xA0,
+			0x18, 0xB6, 0x24, 0xA1, 0xA6, 0xFD, 0x0C, 0x80,
+			0x1E, 0xD2, 0x28, 0xA2, 0x1A, 0x27, 0xF4, 0x23,
+			0x59, 0x1A, 0xCC, 0x0F, 0xD4, 0x99, 0xD0, 0xB4,
+			0x1E, 0x91, 0xC7, 0xD8, 0x8F, 0x8C, 0x5B, 0xEB,
+			0xB5, 0x9F, 0xFF, 0x4F, 0xD0, 0xD5, 0xB7, 0x60,
+			0xCC, 0x0A, 0x10, 0x38, 0xBF, 0xA8, 0x2E, 0xCC,
+			0xEB, 0x26, 0xB0, 0x78, 0xB3, 0xE0, 0x40, 0xAF,
+			0xCD, 0x12, 0xC5, 0x3A, 0x24, 0xD8, 0xEE, 0x3A,
+			0x64, 0x83, 0x2E, 0xD9, 0x25, 0x21, 0x66, 0xA5,
+			0x28, 0xD1, 0xE1, 0x84, 0x25, 0x1B, 0x20, 0xB8,
+			0xF5, 0x76, 0xB6, 0x3E, 0x4B, 0xC6, 0xEC, 0xC1,
+			0xC7, 0xAC, 0xC4, 0xAD, 0xCE, 0xF0, 0xB4, 0x0F,
+			0x35, 0x1E, 0xCE, 0x4E, 0xE3, 0x57, 0x30, 0xFC,
+			0xF4, 0x9B, 0x86, 0xB0, 0xDD, 0x3F, 0x2F, 0xB6,
+			0x10, 0x20, 0xE4, 0x24, 0x17, 0x1C, 0x24, 0xC6,
+			0x89, 0xE4, 0x14, 0xAD, 0x2E, 0x41, 0x08, 0x33,
+			0x88, 0xB1, 0x6F, 0x11, 0x85, 0xAF, 0x58, 0x17,
+			0xE3, 0x91, 0xB4, 0x72, 0xA2, 0x7F, 0xA3, 0x98,
+			0xAF, 0xB7, 0x6B, 0x58, 0x76, 0xA3, 0x11, 0x1C,
+			0x8A, 0x1A, 0xE6, 0x58, 0x54, 0xB0, 0xB9, 0x6E,
+			0x46, 0xCB, 0x16, 0xC0, 0x63, 0x0C, 0xEE, 0xA2,
+			0xAE, 0xF6, 0x71, 0xEF, 0xD1, 0xB9, 0x3D, 0xB7,
+			0x76, 0xCE, 0x5B, 0x84, 0x66, 0x7C, 0x7D, 0xF1,
+			0x96, 0x60, 0x34, 0xF6, 0xD1, 0x64, 0x27, 0xD9,
+			0xF3, 0x78, 0x8B, 0xF4, 0xC3, 0x1D, 0x37, 0xC0,
+			0xF4, 0x4A, 0xD0, 0xA5, 0x9A, 0xEB, 0xDD, 0x79,
+			0x54, 0x5D, 0xEB, 0x04, 0xC1, 0xA4, 0xBC, 0xED,
+			0xE3, 0x74, 0xC3, 0xB9, 0x9A, 0x6A, 0xAA, 0x06,
+			0xD1, 0xF0, 0x0F, 0xC5, 0xEF, 0x7E, 0x0B, 0xC8,
+			0xF4, 0x94, 0x4E, 0x69, 0x0E, 0x36, 0x00, 0x13,
+			0x45, 0xCE, 0x68, 0x13, 0xFE, 0x7F, 0x29, 0xA2,
+			0x1D, 0x79, 0xDF, 0xF2, 0x27, 0xFB, 0xAE, 0x52,
+			0x05, 0x78, 0xD7, 0xB9, 0xF7, 0x38, 0x68, 0xD5,
+			0xBA, 0xD7, 0xCF, 0x09, 0xC6, 0xD2, 0x5B, 0xC6,
+			0x98, 0xE4, 0xEC, 0xD5, 0xE9, 0xC2, 0xA5, 0x1A,
+			0x52, 0xC8, 0xA7, 0xBA, 0x3D, 0x74, 0x75, 0x00,
+			0xAA, 0xDD, 0x6A, 0x3F, 0xB6, 0x2F, 0x08, 0xB7,
+			0x1C, 0x6B, 0x52, 0x0C, 0xC9, 0xE4, 0x4D, 0xF4,
+			0xC5, 0x26, 0x1F, 0x35, 0x41, 0x25, 0x68, 0x17,
+			0xA8, 0x81, 0x75, 0xF4, 0x66, 0x41, 0xB5, 0xE4,
+			0x1D, 0x92, 0xEE, 0xDA, 0x0F, 0x56, 0x76, 0xC6,
+			0xAA, 0x0F, 0xA8, 0x63, 0x8D, 0xF0, 0x69, 0x63,
+			0x93, 0x45, 0xBC, 0x76, 0x40, 0xBE, 0xA9, 0x96,
+			0x36, 0xAF, 0x2F, 0x6B, 0x3E, 0xAB, 0xF3, 0xC0,
+			0xD7, 0xD5, 0xB1, 0x23, 0x23, 0xA2, 0xA0, 0xC4,
+			0xC5, 0x70, 0xEF, 0x66, 0x79, 0x15, 0xF0, 0xD0,
+			0x40, 0x0A, 0x33, 0x0C, 0xF3, 0x32, 0x6D, 0x8D,
+			0xB4, 0x44, 0x46, 0x78, 0x3F, 0x8D, 0x75, 0x40,
+			0xA5, 0x60, 0xBC, 0x9B, 0x76, 0xDF, 0x25, 0xF4,
+			0xE9, 0xED, 0xAC, 0x74, 0x2F, 0x9A, 0x00, 0xC4,
+			0x2B, 0x52, 0x26, 0x79, 0x09, 0x19, 0x57, 0x89,
+			0x60, 0x14, 0xBE, 0x65, 0xBD, 0x7B, 0x4D, 0x7D,
+			0x9B, 0x8B, 0x9E, 0x72, 0x6C, 0x0C, 0x57, 0xC7,
+			0x00, 0x08, 0x38, 0x7C, 0x37, 0x45, 0x9D, 0x55,
+			0xA2, 0x62, 0x5E, 0x34, 0x19, 0x99, 0x31, 0x16,
+			0xF1, 0x14, 0x44, 0x2D, 0xE3, 0x7E, 0x22, 0xE1,
+			0xA2, 0xB8, 0x9A, 0x9F, 0xE0, 0x37, 0x29, 0xBB,
+			0xCD, 0x46, 0xEE, 0x0A, 0x62, 0x2B, 0x98, 0x34,
+			0xBA, 0x9E, 0x54, 0x1B, 0xB1, 0x5C, 0x4F, 0xE9,
+			0xAA, 0xE4, 0x95, 0x8C, 0xA4, 0xEF, 0xC2, 0xB1,
+			0x7F, 0xF9, 0x80, 0xDA, 0x55, 0x95, 0x92, 0xC0,
+			0x86, 0xF4, 0x2D, 0x99, 0x3E, 0x17, 0xDC, 0x55,
+			0xA6, 0x33, 0x85, 0x90, 0x31, 0xC8, 0xFF, 0x58,
+			0x83, 0xC5, 0xBA, 0x60, 0x20, 0x5F, 0x87, 0x29,
+			0x20, 0x5A, 0x7D, 0x44, 0x2B, 0xA0, 0xE2, 0x99,
+			0xC8, 0x70, 0xBE, 0x89, 0xC5, 0xBC, 0xF6, 0x0D,
+			0x04, 0xC0, 0x96, 0xD1, 0x5C, 0xD1, 0x90, 0x43,
+			0xD3, 0x7B, 0x73, 0x52, 0x30, 0xB6, 0xA9, 0x7C,
+			0x0A, 0xA3, 0x24, 0x0E, 0x80, 0xFE, 0xBE, 0x31,
+			0xFD, 0xB5, 0x96, 0x04, 0x2B, 0xCF, 0x0B, 0x28,
+			0x1F, 0x7A, 0xCF, 0xC4, 0x82, 0x78, 0x52, 0x30,
+			0xB1, 0x34, 0x12, 0x50, 0x03, 0x09, 0x1C, 0x8B,
+			0x80, 0x60, 0xE3, 0xA1, 0xE5, 0x61, 0xF7, 0xD7,
+			0xB6, 0x74, 0xBE, 0xD6, 0x58, 0x03, 0xD3, 0xE5,
+			0xF7, 0xAC, 0x07, 0x60, 0xB7, 0x8A, 0xEC, 0xFA,
+			0xC6, 0x0F, 0xF0, 0x20, 0x04, 0x6B, 0x8F, 0x61,
+			0x09, 0x92, 0x03, 0xFB, 0x85, 0x99, 0x94, 0x9D,
+			0x2E, 0x6A, 0xC2, 0x9F, 0x20, 0x46, 0x2A, 0x96,
+			0xED, 0x42, 0x7D, 0x64, 0xA9, 0xE4, 0x1B, 0xDE,
+			0x11, 0x20, 0x12, 0x93, 0xE6, 0x2B, 0xE5, 0x93,
+			0x48, 0x37, 0x8C, 0x5A, 0x54, 0x0D, 0xEB, 0xF0,
+			0x9F, 0x9D, 0xE4, 0xA5, 0xC4, 0x93, 0x6F, 0x6A,
+			0xE3, 0x99, 0x69, 0xD9, 0xFE, 0x0C, 0x4E, 0xEC,
+			0x8B, 0x30, 0x1F, 0x7A, 0xB8, 0xC8, 0x5B, 0x61,
+			0x8E, 0xC2, 0x10, 0x90, 0x57, 0xB4, 0x72, 0x58,
+			0x7F, 0x41, 0x29, 0x7E, 0xF9, 0xBE, 0x40, 0xC3,
+			0x6F, 0xA9, 0xE3, 0x00, 0xE9, 0xC8, 0xFD, 0x4B,
+			0xFD, 0x3F, 0xE3, 0x3F, 0x25, 0x22, 0xFD, 0xB7,
+			0x2D, 0x57, 0xEF, 0x91, 0x08, 0xF0, 0x20, 0x56,
+			0x30, 0xFA, 0x83, 0x69, 0xFD, 0x56, 0x5A, 0x9B,
+			0xCE, 0xF8, 0x28, 0x02, 0xB4, 0x91, 0x35, 0x75,
+			0x9E, 0x63, 0x99, 0x48, 0xCF, 0x35, 0xF5, 0x58,
+			0x0C, 0x48, 0x8F, 0x0A, 0x2D, 0x9A, 0xE6, 0x40,
+			0xF6, 0x21, 0xB5, 0x69, 0xC1, 0x09, 0x31, 0x00,
+			0xA3, 0xC1, 0x4C, 0x99, 0x70, 0x4F, 0x5A, 0x63,
+			0x17, 0x90, 0xB8, 0xF8, 0x3A, 0x0E, 0xFD, 0x67,
+			0xEA, 0x0E, 0xBA, 0x7B, 0x1E, 0xEF, 0x37, 0x84,
+			0xD5, 0x51, 0x37, 0x01, 0xD6, 0x93, 0x15, 0xDF,
+			0x56, 0x89, 0x0E, 0x54, 0xF5, 0x1D, 0xF7, 0xE5,
+			0xB7, 0xC1, 0xF1, 0xC2, 0xD9, 0x14, 0x6F, 0x40,
+			0x55, 0x67, 0x50, 0x7C, 0x58, 0x35, 0x8B, 0x39,
+			0xCB, 0xB5, 0x87, 0xF5, 0x55, 0x5E, 0x26, 0x8C,
+			0x5B, 0x73, 0x0E, 0xBB, 0x25, 0x51, 0x0E, 0xAD,
+			0x57, 0x72, 0x7B, 0x68, 0x83, 0x11, 0x1E, 0x3A,
+			0x3D, 0xA4, 0x7C, 0x18, 0xB7, 0x70, 0x18, 0xBC,
+			0x72, 0x03, 0x4A, 0xA1, 0xD2, 0xF9, 0xA9, 0x8A,
+			0x25, 0x45, 0x19, 0xEE, 0x93, 0x06, 0xB5, 0x09,
+			0x71, 0xC9, 0x2D, 0xFD, 0x2B, 0xF3, 0xC7, 0x64,
+			0x5F, 0xCE, 0x71, 0x1D, 0x81, 0x96, 0x67, 0xBF,
+			0x01, 0x39, 0x8C, 0xE7, 0xA2, 0xD0, 0x98, 0x57,
+			0x5A, 0xFD, 0x21, 0xC7, 0x46, 0xAA, 0xB4, 0xE4,
+			0x0E, 0xBE, 0xC6, 0x68, 0x3E, 0x38, 0xF5, 0xA2,
+			0xED, 0x73, 0xCC, 0x53, 0x7E, 0x7E, 0x03, 0x32,
+			0xDC, 0xB6, 0xC1, 0x03, 0x9E, 0xB3, 0x2A, 0xAD,
+			0xC0, 0xC3, 0x6E, 0x47, 0xFB, 0x1E, 0xB7, 0x0D,
+			0x86, 0x95, 0x09, 0xA6, 0x9D, 0x6F, 0x92, 0xFC,
+			0xFF, 0x2C, 0x7D, 0x09, 0x16, 0x68, 0x50, 0x3E,
+			0x4F, 0x23, 0x4C, 0x93, 0x95, 0x2A, 0xE1, 0x9B,
+			0x16, 0xF0, 0x0F, 0xFF, 0x79, 0xA8, 0x06, 0xF9,
+			0x70, 0x61, 0x72, 0x2C, 0xE8, 0x91, 0x01, 0x6D,
+			0x45, 0xE5, 0x82, 0x5D, 0x26, 0x21, 0xAD, 0x3D,
+			0x77, 0x73, 0x23, 0x04, 0x84, 0x27, 0xA3, 0x5D,
+			0x6D, 0xA8, 0x99, 0xC1, 0xCE, 0x4F, 0xA9, 0xF7,
+			0xAB, 0x5C, 0xDE, 0x01, 0xE6, 0x1E, 0xEF, 0xE6,
+			0xFD, 0xE0, 0x68, 0x85, 0x3E, 0xEE, 0xBF, 0xF1,
+			0x0D, 0x79, 0xF4, 0xA2, 0xB4, 0x14, 0xBC, 0x0C,
+			0x49, 0x77, 0x03, 0x71, 0x08, 0x3E, 0x40, 0xA6,
+			0xD7, 0x03, 0xFA, 0xE2, 0xFB, 0xC7, 0x59, 0x30,
+			0x6E, 0x07, 0x06, 0x1C, 0x7C, 0x47, 0xE5, 0x4C,
+			0x57, 0x0A, 0x91, 0x4A, 0x43, 0xE4, 0x8A, 0xCD,
+			0x6E, 0x92, 0x01, 0xE2, 0x52, 0xC1, 0x92, 0x34,
+			0x8E, 0x64, 0x0F, 0x39, 0x63, 0x53, 0xAB, 0xE5,
+			0x44, 0xD5, 0xAA, 0xAA, 0xF6, 0x03, 0x89, 0xB9,
+			0xDD, 0xB2, 0x2D, 0x56, 0x1A, 0xE0, 0x72, 0x5A,
+			0x52, 0x19, 0x46, 0xEA, 0xB3, 0xCE, 0xB3, 0x59,
+			0x46, 0x7A, 0xA7, 0x48, 0x37, 0x0C, 0x09, 0xBA,
+			0x92, 0x70, 0x17, 0x7F, 0xF5, 0xD3, 0x60, 0x44,
+			0xC4, 0xC6, 0xC6, 0x7D, 0xD2, 0x21, 0xAC, 0x3F,
+			0x62, 0x6C, 0xE9, 0xBA, 0x4C, 0xF3, 0x82, 0x7E,
+			0x6D, 0x3A, 0x92, 0xDC, 0x94, 0xE4, 0x5F, 0xA6,
+			0x8B, 0x66, 0xA0, 0xDD, 0xE2, 0x97, 0x83, 0xED,
+			0xF5, 0x9D, 0xDF, 0x74, 0x77, 0x23, 0x7D, 0xDA,
+			0xC4, 0xFB, 0x92, 0x1A, 0xD9, 0x37, 0x36, 0xD2,
+			0x88, 0xC9, 0xEA, 0x0F, 0x98, 0xBD, 0xC5, 0xF8,
+			0xAA, 0x19, 0x75, 0x12, 0x6A, 0x41, 0xB5, 0xB3,
+			0xB5, 0xA4, 0x96, 0xDC, 0x2B, 0x49, 0x86, 0x66,
+			0x35, 0xD8, 0x4A, 0x62, 0xB4, 0xCB, 0x1E, 0x27,
+			0xC1, 0xAD, 0x34, 0x0E, 0x26, 0x16, 0xF2, 0xC2,
+			0x22, 0x52, 0x84, 0xD8, 0xD1, 0x32, 0xB8, 0x9C,
+			0xFE, 0x64, 0x42, 0x9F, 0xE4, 0x69, 0xF0, 0xAE,
+			0x3B, 0xD9, 0x2C, 0xA0, 0x14, 0xEB, 0x69, 0x74,
+			0x7C, 0xE2, 0xA6, 0x60, 0xE1, 0x52, 0x1C, 0xCC,
+			0xBF, 0xE6, 0xA1, 0x83, 0x20, 0x5D, 0x9E, 0xA3,
+			0xFB, 0x84, 0x8B, 0x33, 0xE6, 0xC9, 0x32, 0x83,
+			0xC0, 0x3F, 0x98, 0x1D, 0x6E, 0xC0, 0x50, 0x71,
+			0x29, 0x60, 0x5F, 0x36, 0xB4, 0x68, 0x1D, 0xB9,
+			0x76, 0x73, 0xC3, 0x80, 0xC5, 0xBC, 0x59, 0x7B,
+			0x59, 0xB4, 0xE0, 0x6A, 0x80, 0xCD, 0x4D, 0x8C,
+			0x9E, 0xE0, 0x0B, 0x45, 0x7D, 0x54, 0xD4, 0xC4,
+			0x97, 0x6C, 0x54, 0xEF, 0x14, 0x64, 0xBD, 0x3B,
+			0xD7, 0xEE, 0xF4, 0xD1, 0x41, 0x76, 0x3A, 0x24,
+			0x7A, 0xC2, 0xCA, 0x68, 0x28, 0x53, 0x46, 0xF7,
+			0x1B, 0xDA, 0x4B, 0x7A, 0x56, 0x75, 0x86, 0xFB,
+			0x31, 0x2C, 0x27, 0xF9, 0x4D, 0x35, 0xA4, 0x82,
+			0xE7, 0x2F, 0x41, 0xB4, 0xCA, 0xCE, 0x75, 0x94,
+			0x08, 0x54, 0xE2, 0x9E, 0x99, 0xC9, 0x85, 0xDE,
+			0x6F, 0x80, 0x95, 0x59, 0x3E, 0x54, 0x9F, 0x31,
+			0xF8, 0xDE, 0xD0, 0xD7, 0xA6, 0xD4, 0xD3, 0xBB,
+			0xD9, 0xC7, 0x55, 0xDD, 0xAE, 0xAD, 0x9E, 0x57,
+			0x4A, 0x33, 0x5D, 0x7A, 0xA6, 0xA3, 0xCA, 0xF9,
+			0x4C, 0x5B, 0x51, 0xCC, 0x22, 0xBB, 0x76, 0x44,
+			0x17, 0xDE, 0x22, 0xA1, 0xDF, 0x80, 0x13, 0x7D,
+			0xE5, 0x34, 0x7E, 0x75, 0x73, 0x10, 0x40, 0xFB,
+			0x9A, 0x21, 0xCD, 0xD3, 0xD3, 0x84, 0xB6, 0x0C,
+			0x31, 0x1E, 0xB5, 0x42, 0xF4, 0x34, 0x11, 0x7F,
+			0x4A, 0x23, 0xA8, 0xA5, 0x8F, 0x20, 0xCD, 0xA9,
+			0xF2, 0xE4, 0xEE, 0xFA, 0x57, 0xD1, 0x22, 0x1C,
+			0xA5, 0xDC, 0x0B, 0x25, 0xFE, 0xC2, 0xA7, 0x7E,
+			0x09, 0x2E, 0xDA, 0x40, 0x9F, 0x6C, 0xC8, 0x71,
+			0x58, 0x91, 0x04, 0x25, 0xD0, 0x06, 0xEA, 0x1B,
+			0xCD, 0x9D, 0x50, 0xD8, 0x40, 0x24, 0xAC, 0xC3,
+			0xB4, 0x07, 0x6E, 0x76, 0xF4, 0x4C, 0xD8, 0x80,
+			0xD0, 0x20, 0xF5, 0x15, 0x6A, 0x0A, 0x12, 0xF8,
+			0x6B, 0x67, 0x77, 0x34, 0xAE, 0x60, 0x68, 0x13,
+			0x5B, 0x8E, 0xFF, 0x5E, 0x7A, 0x77, 0x67, 0x0D,
+			0xE6, 0x96, 0x43, 0x9F, 0x8F, 0x77, 0x5F, 0x97,
+			0x23, 0x91, 0x33, 0x72, 0xC7, 0x8A, 0xC7, 0x80,
+			0xCF, 0xE7, 0x71, 0x06, 0x25, 0xB7, 0x4B, 0x89,
+			0x6A, 0x46, 0x67, 0x19, 0x49, 0x44, 0x03, 0x52,
+			0x32, 0xB1, 0x8F, 0xE7, 0x9E, 0xDA, 0x03, 0x41,
+			0xA3, 0xAC, 0xE5, 0xF3, 0x96, 0xE6, 0xAC, 0xFF,
+			0xEC, 0x35, 0x4D, 0x83, 0xA9, 0xCE, 0x7C, 0x52,
+			0xF2, 0x36, 0x97, 0xF0, 0x28, 0x36, 0x54, 0x59,
+			0x96, 0xEA, 0xEE, 0xB2, 0xC1, 0xAB, 0xA4, 0x96,
+			0x62, 0xD4, 0x3C, 0xF0, 0x1F, 0x2D, 0x65, 0x0E,
+			0x46, 0x7E, 0x12, 0x31, 0x8F, 0xA7, 0x8D, 0x7A,
+			0x4A, 0x41, 0x15, 0x57, 0x90, 0xF6, 0xF1, 0xE8,
+			0xE8, 0xE3, 0x57, 0x7B, 0x55, 0x85, 0x95, 0x97,
+			0xB3, 0x29, 0x3D, 0x02, 0x73, 0x1E, 0x87, 0x1F,
+			0x01, 0x89, 0x06, 0x88, 0x9E, 0x8A, 0x2E, 0xE0,
+			0x99, 0xFC, 0xF0, 0x48, 0x60, 0x37, 0x65, 0x25,
+			0xDB, 0x89, 0xC9, 0x7A, 0x51, 0x7E, 0x35, 0x92,
+			0x00, 0xC9, 0x61, 0x3F, 0x88, 0xE3, 0x20, 0x01,
+			0x46, 0x5A, 0x2C, 0x37, 0x02, 0xC9, 0x15, 0x0F,
+			0xB2, 0xEB, 0xC3, 0x55, 0x18, 0xF0, 0x15, 0x1A,
+			0x08, 0x8E, 0xB8, 0x9D, 0x18, 0xE4, 0x9F, 0x34,
+			0x10, 0x67, 0x68, 0x57, 0x60, 0x61, 0xEC, 0xD5,
+			0xD9, 0xA8, 0x3A, 0xAB, 0xD6, 0xD2, 0x7A, 0x47,
+			0x3D, 0xA4, 0x08, 0x7C, 0x3E, 0x4D, 0x76, 0x08,
+			0x19, 0x22, 0xB2, 0x89, 0x57, 0x84, 0xC2, 0x98,
+			0x72, 0xB8, 0x8B, 0xE0, 0x85, 0xA1, 0x3A, 0xC2,
+			0xA0, 0x06, 0x43, 0x03, 0xCF, 0x4F, 0x27, 0x80,
+			0x48, 0x9A, 0xBC, 0xB3, 0x3C, 0xC4, 0x5E, 0xAC,
+			0x8B, 0x85, 0x6F, 0x21, 0xD6, 0xFE, 0x12, 0x90,
+			0x53, 0x2F, 0x65, 0x32, 0x8D, 0x03, 0xFE, 0xFE,
+			0x61, 0x04, 0x47, 0x24, 0x6A, 0xB5, 0x01, 0x98,
+			0xB9, 0x27, 0x10, 0xE1, 0x32, 0x3D, 0x2A, 0xA0,
+			0xC5, 0x70, 0xDE, 0x1E, 0x10, 0xD7, 0x01, 0x50,
+			0x4F, 0x87, 0xCA, 0xD9, 0xBF, 0x12, 0xEA, 0x38,
+			0x4C, 0x43, 0xD5, 0x5A, 0xEF, 0x3E, 0x21, 0x8E,
+			0x59, 0x77, 0x23, 0xED, 0x51, 0x09, 0x99, 0x73,
+			0xD2, 0x56, 0x04, 0xDD, 0x8F, 0x5F, 0xDF, 0x79,
+			0xFF, 0x16, 0x8C, 0xB0, 0xBA, 0x8A, 0x1A, 0x56,
+			0xAF, 0xCA, 0x19, 0xF2, 0x64, 0x1A, 0xF5, 0x1E,
+			0xA7, 0xA7, 0x84, 0x3D, 0xAD, 0xC1, 0x0E, 0x22,
+			0xA1, 0x45, 0xFC, 0xB4, 0x13, 0x91, 0x34, 0xB7,
+			0x48, 0xEF, 0x9E, 0xD9, 0x0B, 0xE3, 0x82, 0x75,
+			0x80, 0xC5, 0xD9, 0xA0, 0x77, 0xA3, 0xF9, 0xCC,
+			0x67, 0xDD, 0xCB, 0x28, 0xC6, 0xE8, 0x2C, 0xB8,
+			0xAC, 0x63, 0xBD, 0x3B, 0x28, 0x4A, 0xE9, 0x2D,
+			0x29, 0x84, 0xD7, 0x4F, 0x61, 0x06, 0xE3, 0x37,
+			0xC1, 0x58, 0x20, 0x5D, 0x0A, 0xE7, 0x45, 0x29,
+			0x7D, 0xED, 0x0F, 0xCE, 0x00, 0x95, 0x2A, 0x62,
+			0x38, 0xA8, 0x1A, 0x3A, 0x96, 0x0E, 0x56, 0xD9,
+			0x18, 0xC2, 0x25, 0xA5, 0xAA, 0x27, 0x0A, 0x6E,
+			0xDD, 0x1C, 0x35, 0x6C, 0xC1, 0x26, 0x90, 0xF6,
+			0x43, 0x1B, 0x34, 0xDA, 0xE1, 0x5D, 0x09, 0x7C,
+			0xBE, 0x0F, 0x40, 0xD3, 0x24, 0x82, 0x0B, 0xFF,
+			0xE6, 0xB7, 0x10, 0xD6, 0x36, 0xD0, 0xE6, 0xC0,
+			0xBE, 0x65, 0x4C, 0x83, 0xF1, 0xDA, 0xDE, 0xCE,
+			0xE8, 0x5A, 0x80, 0x88, 0xFE, 0x9B, 0x79, 0x54,
+			0xA3, 0xA4, 0x5A, 0x76, 0xD0, 0xE2, 0xCE, 0x92,
+			0x53, 0x7D, 0x9C, 0xDA, 0xA1, 0xED, 0x9F, 0x56,
+			0x05, 0x0A, 0xA4, 0x81, 0xC7, 0x82, 0x5B, 0xB8,
+			0xC7, 0xA8, 0x95, 0x21, 0x99, 0x0B, 0x0F, 0xD2,
+			0x66, 0x68, 0xC3, 0x07, 0x53, 0x93, 0x8C, 0x68,
+			0x5A, 0xF5, 0x6F, 0x5E, 0x07, 0x68, 0x70, 0xF2,
+			0x6A, 0x78, 0xA8, 0xDB, 0x24, 0x6F, 0xD9, 0x74,
+			0x38, 0x29, 0xBF, 0x50, 0xCC, 0xC8, 0x34, 0x82,
+			0x69, 0x1A, 0xF4, 0x1A, 0x95, 0xE1, 0x31, 0x39,
+			0x00, 0xBE, 0xF6, 0x25, 0x19, 0x5A, 0xF6, 0xA7,
+			0xB7, 0xC7, 0xAC, 0xFA, 0x52, 0x6D, 0xC7, 0xEA,
+			0xFF, 0xC4, 0xB0, 0x84, 0x41, 0x2D, 0x6B, 0x22,
+			0x57, 0x1F, 0x8A, 0x97, 0x71, 0xEF, 0x54, 0xA7,
+			0xA9, 0xB5, 0xA0, 0x7A, 0xFF, 0x52, 0xBC, 0x78,
+			0x56, 0x8C, 0x9C, 0x2A, 0xFB, 0x31, 0xA6, 0xC8,
+			0x87, 0xC8, 0x82, 0x72, 0x00, 0x68, 0x27, 0xDE,
+			0x54, 0x67, 0x4C, 0x36, 0x4E, 0xE2, 0x77, 0x7F,
+			0xAD, 0x15, 0x1B, 0xC9, 0x07, 0xC4, 0x84, 0x50,
+			0x84, 0x45, 0xB5, 0x1D, 0xCE, 0x3F, 0x7C, 0xDF,
+			0x73, 0x88, 0x3E, 0x6D, 0xAC, 0x18, 0x67, 0x2C,
+			0x1D, 0x31, 0xD5, 0x41, 0x6E, 0xFC, 0x3D, 0xFE,
+			0x5C, 0x6D, 0x3B, 0xCB, 0x67, 0x05, 0x44, 0x8B,
+			0x02, 0xB7, 0xF5, 0x05, 0xFD, 0x1D, 0x7E, 0x13,
+			0x90, 0xCE, 0xED, 0xD2, 0xAB, 0x08, 0xFF, 0xC3,
+			0x91, 0x2C, 0x79, 0x06, 0xDB, 0x54, 0xAB, 0xFF,
+			0xF6, 0x9D, 0xBB, 0xDC, 0x3C, 0xCD, 0x03, 0xE7,
+			0xD8, 0x1A, 0x4E, 0x7F, 0xCB, 0x1B, 0xA3, 0xCA,
+			0xDC, 0x14, 0xC5, 0xFE, 0x46, 0x38, 0x07, 0x82,
+			0x5E, 0x9A, 0x77, 0x9C, 0xB4, 0x44, 0x26, 0xBA,
+			0xC2, 0x27, 0xD0, 0xF4, 0x75, 0x67, 0x4A, 0x15,
+			0x2A, 0x55, 0x2A, 0x61, 0x87, 0x55, 0xA0, 0xFB,
+			0xE0, 0x93, 0xA5, 0xD7, 0xDF, 0x0D, 0x97, 0xD3,
+			0x93, 0x39, 0x0B, 0x5A, 0xC6, 0x86, 0x17, 0x7D,
+			0x6A, 0xA4, 0x07, 0x9C, 0xB9, 0x0F, 0x42, 0x15,
+			0x46, 0x71, 0x45, 0x70, 0x75, 0x4B, 0xD5, 0x80,
+			0x64, 0x62, 0x92, 0x50, 0xBC, 0x10, 0xA1, 0x68,
+			0x60, 0x5A, 0x6F, 0x31, 0x26, 0xCA, 0xB1, 0x48,
+			0xFB, 0xC6, 0xF4, 0x3A, 0xCA, 0x52, 0x20, 0x2F,
+			0x7D, 0xEC, 0x19, 0xF8, 0xAA, 0x27, 0xC2, 0x23,
+			0xD0, 0x8F, 0x60, 0xFC, 0x7F, 0xA0, 0xCB, 0xDC,
+			0xA2, 0xC6, 0xC4, 0xC5, 0x13, 0x00, 0xF3, 0x43,
+			0x48, 0x6D, 0xFD, 0x7D, 0xB9, 0xA8, 0x14, 0xB0,
+			0x0C, 0x72, 0x82, 0x2F, 0x99, 0x64, 0x41, 0x2B,
+			0xB3, 0x34, 0x73, 0x73, 0xF4, 0x26, 0x1D, 0x06,
+			0xDF, 0x6E, 0xF4, 0x20, 0x1E, 0x31, 0xE3, 0x8A,
+			0x01, 0x6C, 0xDB, 0x3C, 0xE3, 0xC6, 0xC4, 0xC5,
+			0xB8, 0x20, 0x51, 0xF1, 0xD6, 0xB0, 0x30, 0xB7,
+			0x2D, 0xDA, 0x95, 0x01, 0x0D, 0xED, 0xEE, 0x6F,
+			0x69, 0xFD, 0xCF, 0x9D, 0xDD, 0x05, 0xD6, 0xC0,
+			0xFE, 0x11, 0x67, 0xAF, 0x53, 0x94, 0x60, 0xFC,
+			0x56, 0xBA, 0x0C, 0x5F, 0xA7, 0x7E, 0xDA, 0x65
+		},
+
+		.len = 4096 << 3
+	},
+	.validCipherLenInBits = {
+		.len = 4096 << 3
+	},
+	.cipher = {
+		.len_bits = 4096,
+		.offset_bits = 0
+	},
+	.dataunit_len = 4096
+};
+
+static struct mlx5_test_data mlx5_test_case_cipher_aes_xts_4 = {
+	.key = {
+		.data = {
+			0x94, 0xD8, 0xAE, 0xD7, 0x74, 0xBF, 0x51, 0xF1,
+			0x2D, 0x19, 0x0B, 0x9A, 0xB3, 0xDC, 0x1E, 0xA9,
+			0x5C, 0xC5, 0x47, 0xC1, 0xBB, 0x94, 0xDB, 0xFA,
+			0x2B, 0x1E, 0x54, 0xA5, 0x76, 0xAD, 0x07, 0x4B,
+			0x46, 0xE0, 0xAA, 0xF9, 0xF7, 0x5C, 0x64, 0x2D,
+			0x8F, 0x04, 0x17, 0x72, 0xE5, 0x8D, 0xFE, 0x9C
+		},
+		.len = 48
+	},
+	.cipher_iv = {
+		.data = {
+			0x4F, 0xAE, 0xF7, 0x11, 0x7C, 0xDA, 0x59, 0xC6,
+			0x6E, 0x4B, 0x92, 0x01, 0x3E, 0x76, 0x8A, 0xD5
+		},
+		.len = 16
+	},
+	.plaintext = {
+		.data = {
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+			0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79
+		},
+		.len = 4096 << 3
+	},
+	.ciphertext = {
+		.data = {
+			0xD3, 0x94, 0x3E, 0xC9, 0xD3, 0x43, 0x13, 0xD3,
+			0x07, 0x7E, 0x51, 0x11, 0x97, 0xA5, 0xB1, 0xB2,
+			0xB0, 0x55, 0xF8, 0xD6, 0xA2, 0x10, 0x78, 0x76,
+			0x37, 0x95, 0x4F, 0x25, 0x99, 0xCC, 0x12, 0xD6,
+			0xBB, 0x2A, 0x24, 0xF4, 0x2D, 0x82, 0xFF, 0x56,
+			0xEC, 0x55, 0x3B, 0xDD, 0xF9, 0xE1, 0xC8, 0x48,
+			0x68, 0x16, 0xD2, 0x2B, 0x95, 0x7C, 0x1F, 0xCB,
+			0x32, 0xD6, 0x8F, 0x9A, 0x2E, 0xF7, 0x9B, 0xBE,
+			0x72, 0xFC, 0x55, 0x33, 0x27, 0x21, 0x2F, 0x69,
+			0xCA, 0x76, 0xA5, 0x9B, 0x21, 0x2E, 0x40, 0x57,
+			0x2C, 0x1C, 0x98, 0x41, 0x9A, 0x5E, 0x55, 0x38,
+			0xDE, 0xC2, 0x09, 0x57, 0x32, 0xA3, 0x34, 0x40,
+			0xC1, 0x8D, 0xCF, 0x70, 0x15, 0xF2, 0x6F, 0x4F,
+			0x6A, 0x04, 0xA6, 0x6D, 0xFF, 0x53, 0x25, 0x6A,
+			0x0E, 0xD0, 0x87, 0x63, 0xA1, 0x6C, 0xB1, 0x99,
+			0x4A, 0x42, 0xF5, 0xF6, 0xEA, 0xA4, 0xEB, 0x6D,
+			0x70, 0x9B, 0x0F, 0x85, 0xE2, 0x43, 0x6C, 0x27,
+			0x54, 0x57, 0x52, 0x1C, 0xCF, 0x72, 0x11, 0x83,
+			0xC3, 0xF5, 0xC6, 0xB6, 0x07, 0xEC, 0x1A, 0xF5,
+			0xAC, 0xA1, 0xF2, 0x3A, 0x01, 0x53, 0x0C, 0xA4,
+			0x40, 0x19, 0xAF, 0x7B, 0x2D, 0xE7, 0x56, 0x8B,
+			0x71, 0x5F, 0x8D, 0x96, 0xF1, 0x5D, 0x33, 0xAC,
+			0xE7, 0xAD, 0x96, 0xDB, 0xBB, 0xF7, 0xF5, 0x3F,
+			0x0E, 0x02, 0x2E, 0x80, 0xE9, 0xA2, 0x30, 0xD2,
+			0x82, 0x65, 0xEC, 0x1A, 0xB2, 0xE8, 0x98, 0x23,
+			0x42, 0x79, 0x43, 0x85, 0x3B, 0xF9, 0xFB, 0xBB,
+			0xC5, 0x29, 0xAD, 0x95, 0xAE, 0x22, 0x5B, 0x26,
+			0xDF, 0x76, 0x65, 0x37, 0x17, 0xBC, 0x58, 0xBB,
+			0x1B, 0x0A, 0x71, 0xDC, 0x68, 0xFF, 0x90, 0x36,
+			0x37, 0xB7, 0x49, 0x5E, 0x49, 0x4E, 0xE4, 0x1E,
+			0x24, 0x39, 0x48, 0xC7, 0x68, 0x19, 0xED, 0x94,
+			0xB5, 0xF6, 0x85, 0xFA, 0xE6, 0xB8, 0x2B, 0x9E,
+			0x07, 0x9E, 0xFB, 0x1D, 0x61, 0x06, 0x47, 0x2A,
+			0x3E, 0x1E, 0xD8, 0x52, 0xFB, 0xB7, 0xE3, 0xB4,
+			0x0D, 0xA8, 0x15, 0x1E, 0x98, 0x02, 0xBD, 0x89,
+			0x89, 0xE3, 0x38, 0x2C, 0xAB, 0x50, 0x25, 0x30,
+			0xB4, 0x5E, 0xA5, 0xCD, 0xA8, 0x9B, 0xA4, 0x2A,
+			0xED, 0x19, 0x3B, 0xC3, 0x05, 0x07, 0x57, 0xB5,
+			0x52, 0x11, 0x74, 0x95, 0x51, 0x5A, 0xD8, 0xED,
+			0xDF, 0x37, 0x91, 0x83, 0x27, 0xD5, 0x08, 0x82,
+			0xB3, 0x42, 0x08, 0xC7, 0x81, 0x35, 0x5F, 0x58,
+			0x28, 0x69, 0x0D, 0x97, 0x1D, 0x28, 0xE6, 0xB0,
+			0x58, 0x93, 0xCE, 0x2A, 0xB4, 0x7D, 0x4B, 0x83,
+			0x20, 0x1E, 0x08, 0xEF, 0x68, 0x51, 0xEB, 0xB4,
+			0xFA, 0x78, 0xB5, 0xE5, 0x2D, 0x93, 0x07, 0x99,
+			0xBB, 0xDD, 0x9A, 0x4E, 0xD6, 0xE7, 0x31, 0x9F,
+			0x4D, 0xB4, 0x05, 0x45, 0x89, 0x59, 0x42, 0xFF,
+			0x0C, 0xD4, 0xC2, 0xF3, 0xA0, 0xF7, 0xD3, 0xCE,
+			0x26, 0x2F, 0x9B, 0xCB, 0x98, 0x24, 0xDE, 0x50,
+			0xF6, 0x81, 0x24, 0xA3, 0xB4, 0x99, 0x94, 0xDB,
+			0xF5, 0xF4, 0x74, 0xAC, 0xED, 0xF1, 0xD6, 0x2F,
+			0x89, 0x48, 0x41, 0x24, 0xBB, 0xA7, 0x64, 0x48,
+			0xAE, 0x36, 0xE3, 0xF3, 0xA8, 0x37, 0x4F, 0xE6,
+			0x24, 0x92, 0xC4, 0x27, 0x86, 0x68, 0xC4, 0xA2,
+			0x2F, 0xB2, 0xA8, 0xFF, 0x27, 0x5F, 0x59, 0xB9,
+			0x86, 0x89, 0x5A, 0x09, 0xB8, 0xDA, 0xD6, 0xBA,
+			0x26, 0x86, 0x51, 0xC2, 0xDF, 0xCF, 0x07, 0x49,
+			0x1C, 0xB7, 0x1A, 0x80, 0xD4, 0x97, 0x13, 0xAA,
+			0xE8, 0x38, 0xA7, 0x2C, 0xB7, 0xE2, 0xE0, 0x33,
+			0xF9, 0xC3, 0x21, 0xF6, 0x2A, 0xAA, 0x1A, 0xF7,
+			0x39, 0x1F, 0x2F, 0x7E, 0x38, 0xC8, 0x82, 0x3F,
+			0xCC, 0xD8, 0xE2, 0xD9, 0x02, 0xE6, 0x8E, 0x31,
+			0xA7, 0x37, 0xE8, 0x59, 0x3F, 0xE1, 0x60, 0x1C,
+			0x74, 0x52, 0xE4, 0x49, 0xF7, 0x21, 0x60, 0x11,
+			0x8F, 0xD0, 0xA1, 0x4B, 0xE3, 0xF1, 0x18, 0x3D,
+			0x90, 0x89, 0x54, 0x27, 0xA0, 0xF9, 0x32, 0x09,
+			0x3D, 0x9D, 0x9A, 0x09, 0x53, 0xC6, 0x7E, 0x95,
+			0x85, 0x53, 0x98, 0x4C, 0x23, 0xEA, 0x54, 0xBD,
+			0x6F, 0x50, 0xBC, 0x4C, 0xCF, 0x37, 0xC5, 0x7B,
+			0x4B, 0xCE, 0x84, 0xAF, 0xE2, 0xE2, 0x55, 0x49,
+			0xBC, 0xBF, 0x92, 0xCA, 0x1E, 0x5E, 0x10, 0xDF,
+			0x60, 0x87, 0x09, 0xA1, 0x4C, 0x1D, 0x7E, 0x1E,
+			0x59, 0xE9, 0xCF, 0xDA, 0x45, 0x3F, 0xE2, 0x0F,
+			0x53, 0x8D, 0x8B, 0x79, 0xBD, 0xD8, 0xB0, 0xE3,
+			0x5B, 0x7C, 0x55, 0x4A, 0x84, 0xF0, 0x1E, 0xF9,
+			0xE8, 0xF3, 0x09, 0x4D, 0x0B, 0xD7, 0x77, 0xCC,
+			0x3F, 0x70, 0x22, 0x7D, 0x17, 0x27, 0x48, 0x57,
+			0xE2, 0x36, 0xA0, 0x84, 0x3B, 0xDE, 0x05, 0x34,
+			0xEF, 0x55, 0x12, 0xF4, 0x9A, 0x99, 0x0D, 0x28,
+			0x86, 0x28, 0x99, 0x6B, 0x22, 0xEE, 0x63, 0xF0,
+			0x68, 0x9C, 0xE1, 0x70, 0xF6, 0x26, 0xD8, 0x3B,
+			0xF9, 0x57, 0x18, 0x3D, 0xAD, 0x66, 0xF0, 0xCF,
+			0x7B, 0x0C, 0x28, 0x4D, 0xB8, 0xEB, 0x7B, 0x04,
+			0x1E, 0x7D, 0x40, 0x5F, 0x5A, 0x1E, 0x7E, 0x08,
+			0x7F, 0x4C, 0x1E, 0x18, 0xE5, 0x3E, 0x6C, 0x90,
+			0x3C, 0x89, 0x13, 0x2A, 0xC4, 0x2A, 0x94, 0xB5,
+			0x3E, 0x18, 0x1C, 0x4C, 0xBA, 0xEA, 0x86, 0xD2,
+			0x05, 0xA9, 0x59, 0x9C, 0x80, 0xC2, 0x45, 0xAD,
+			0x30, 0x99, 0x18, 0x6A, 0x2F, 0x73, 0x8C, 0xF0,
+			0xFE, 0xA4, 0xBD, 0x44, 0x3E, 0xEB, 0x98, 0x75,
+			0x48, 0x08, 0x57, 0x45, 0xD8, 0x41, 0xDE, 0x61,
+			0x6D, 0x06, 0x93, 0xC4, 0x99, 0x1B, 0x23, 0xB5,
+			0x12, 0x22, 0x5C, 0xC7, 0x9E, 0x18, 0xEA, 0x64,
+			0xDB, 0xCE, 0x1A, 0xAC, 0x5D, 0x9B, 0x80, 0xE2,
+			0xBF, 0x3E, 0xC2, 0xA4, 0x78, 0x4F, 0xF1, 0xE3,
+			0x7D, 0x2A, 0x20, 0x94, 0x13, 0xCD, 0xF3, 0x1C,
+			0x33, 0x9C, 0xC1, 0x59, 0x85, 0x52, 0xCB, 0xDB,
+			0x03, 0xDF, 0x11, 0xE6, 0xAB, 0x95, 0x82, 0x65,
+			0x7A, 0x88, 0x73, 0xEE, 0xBA, 0x21, 0x1C, 0x2F,
+			0xCD, 0xD7, 0xC6, 0xE5, 0x13, 0xDE, 0x7A, 0x9E,
+			0xEE, 0x83, 0x8D, 0xC6, 0x47, 0x63, 0xE0, 0xC7,
+			0xC4, 0xBE, 0x19, 0x25, 0xEE, 0xCC, 0x0A, 0x13,
+			0x18, 0x9D, 0x34, 0x5B, 0x55, 0x6C, 0xC1, 0x6E,
+			0xBF, 0x5A, 0xC5, 0x61, 0x75, 0x77, 0x49, 0x8C,
+			0x67, 0x61, 0xE8, 0x72, 0x87, 0xE8, 0xCA, 0xBE,
+			0x6E, 0xC3, 0xD7, 0x81, 0x8C, 0x78, 0x79, 0xC8,
+			0x72, 0xDA, 0x1A, 0x40, 0x7D, 0x60, 0xE2, 0x5A,
+			0x47, 0x38, 0xA1, 0x21, 0x22, 0x6B, 0x54, 0x74,
+			0xDD, 0xF3, 0xBC, 0x96, 0x28, 0x7E, 0xC2, 0x8B,
+			0x13, 0xEE, 0x53, 0xBC, 0x34, 0x67, 0x07, 0x87,
+			0xD5, 0x6B, 0x93, 0x22, 0x21, 0xB9, 0xED, 0x17,
+			0xE4, 0xA1, 0x96, 0xB2, 0xC6, 0xFF, 0x79, 0xA0,
+			0xA7, 0xF9, 0xDD, 0x92, 0x78, 0xF7, 0xE3, 0x16,
+			0x79, 0xEF, 0xEF, 0x31, 0x4D, 0x1D, 0x75, 0xF9,
+			0xCF, 0x5A, 0x1B, 0x68, 0x16, 0x7F, 0xAF, 0x5F,
+			0x30, 0xB7, 0xEF, 0xF8, 0x94, 0x63, 0x73, 0x3D,
+			0xB3, 0x63, 0xE4, 0xE6, 0xD8, 0xAD, 0xF4, 0x80,
+			0x5E, 0x82, 0xA3, 0xFB, 0x3D, 0x0F, 0xCF, 0x59,
+			0xB8, 0x76, 0xAF, 0x27, 0x83, 0xE3, 0x2D, 0x6F,
+			0xE0, 0xF3, 0x11, 0xD5, 0xAE, 0x82, 0x14, 0x1D,
+			0x78, 0x95, 0xBF, 0x31, 0x22, 0x1B, 0x80, 0x12,
+			0x02, 0xD7, 0x4D, 0x1B, 0x92, 0xE3, 0x15, 0xBA,
+			0x67, 0xD6, 0x8F, 0xD4, 0xDA, 0xBF, 0xD4, 0x62,
+			0xAD, 0x76, 0xDA, 0x04, 0xA2, 0xEA, 0x98, 0xD3,
+			0xC3, 0x6E, 0x5F, 0x26, 0x3C, 0x5E, 0xD9, 0xEA,
+			0x09, 0xF0, 0x02, 0xFC, 0xD2, 0x11, 0xF8, 0xA8,
+			0x7E, 0xFF, 0x06, 0x28, 0x5B, 0xE5, 0x6F, 0x9A,
+			0x00, 0xE7, 0x7F, 0xB9, 0xFB, 0x59, 0xBB, 0xDD,
+			0x85, 0xF3, 0x40, 0xCE, 0xA3, 0x5E, 0x2E, 0x2E,
+			0x34, 0x91, 0x58, 0x41, 0x00, 0xB5, 0xE1, 0x88,
+			0x24, 0x51, 0xC7, 0xB6, 0xF3, 0x21, 0x52, 0x6E,
+			0xE7, 0xFC, 0x42, 0xE7, 0x9C, 0xCC, 0x1E, 0x51,
+			0x45, 0x39, 0xBE, 0x09, 0xFE, 0x1A, 0xC4, 0xF0,
+			0x79, 0xF4, 0x05, 0xC7, 0xA7, 0xF8, 0x0F, 0xB6,
+			0x5A, 0x7B, 0xD7, 0xE1, 0x6F, 0xF0, 0x9D, 0x67,
+			0xA3, 0xE3, 0x3E, 0x2E, 0xB9, 0x8C, 0x83, 0x9E,
+			0xFD, 0x2E, 0xA2, 0x92, 0x99, 0x3C, 0xC0, 0x99,
+			0x01, 0xAB, 0x0D, 0xFA, 0x55, 0x96, 0x04, 0x60,
+			0x1A, 0xAD, 0x4C, 0xBB, 0x3D, 0xBB, 0x7D, 0x8B,
+			0x9F, 0x28, 0x85, 0x7D, 0xB9, 0xE4, 0x05, 0x79,
+			0x7B, 0x63, 0xDD, 0x7F, 0x4D, 0xE7, 0x50, 0xD9,
+			0x41, 0xFF, 0x53, 0xB1, 0xCE, 0x42, 0x7B, 0xD6,
+			0x05, 0x1B, 0x4E, 0xAF, 0xC4, 0x8C, 0x17, 0xC8,
+			0x52, 0xBD, 0x03, 0x3B, 0x92, 0x57, 0x4E, 0xA8,
+			0x15, 0xC3, 0x26, 0x1C, 0x55, 0xC1, 0xFF, 0xAE,
+			0xA9, 0x26, 0x2D, 0xA7, 0x8E, 0x3A, 0x7F, 0xA3,
+			0x48, 0xA5, 0xBC, 0x14, 0x84, 0xF2, 0x90, 0xCE,
+			0x35, 0x0F, 0x64, 0x6B, 0xD8, 0x1C, 0x12, 0xFE,
+			0x5A, 0x4F, 0x0E, 0xCE, 0x81, 0x4E, 0x79, 0x6B,
+			0xCF, 0x56, 0xA7, 0xDB, 0x24, 0xBC, 0xB0, 0x84,
+			0x4C, 0xB0, 0xDA, 0xBE, 0xE6, 0x8F, 0xD7, 0x8E,
+			0x0E, 0xA0, 0xD3, 0x55, 0xC2, 0x4A, 0x34, 0x1C,
+			0xF9, 0xC7, 0x3D, 0x29, 0x70, 0x8B, 0xF0, 0x99,
+			0x61, 0xF5, 0x11, 0xFB, 0x82, 0xE2, 0x67, 0x35,
+			0x60, 0x78, 0x47, 0x81, 0x2A, 0x74, 0x5E, 0x4D,
+			0x48, 0xD3, 0x7C, 0x32, 0xCA, 0x1B, 0xD2, 0xA2,
+			0x5C, 0x3A, 0x2F, 0xCE, 0xB4, 0x6C, 0x3A, 0x6A,
+			0x8F, 0x67, 0x46, 0x12, 0xE7, 0xAE, 0x6A, 0x3B,
+			0x99, 0x04, 0x5E, 0x96, 0xD0, 0xB9, 0x84, 0xF6,
+			0xA7, 0x64, 0x11, 0xE8, 0x0C, 0x51, 0xFD, 0x3F,
+			0x18, 0xFA, 0xE8, 0x52, 0xD9, 0x4B, 0x99, 0x7A,
+			0x25, 0x2B, 0x1B, 0x21, 0xAD, 0x8C, 0xFE, 0x0D,
+			0x34, 0x51, 0x91, 0x75, 0x55, 0x6F, 0xEB, 0x9F,
+			0x42, 0xDC, 0x73, 0x7D, 0x31, 0x0A, 0x74, 0x13,
+			0x80, 0xB8, 0xC3, 0xED, 0x73, 0x9D, 0x79, 0x42,
+			0xC0, 0x33, 0xAB, 0xC1, 0xCB, 0xB9, 0xD0, 0xBE,
+			0xA0, 0x78, 0xB8, 0x3B, 0xEB, 0x3D, 0x1A, 0x3F,
+			0xFB, 0x9B, 0xAA, 0x8F, 0x89, 0xF9, 0xD1, 0x22,
+			0x82, 0xE6, 0x66, 0xEE, 0x2A, 0xFD, 0x9F, 0xF8,
+			0x92, 0x7E, 0x10, 0xF5, 0xD5, 0x23, 0x0B, 0xB1,
+			0xD6, 0xF1, 0x7A, 0x3D, 0x73, 0xE9, 0xCE, 0x7F,
+			0xE6, 0x0B, 0x17, 0xBC, 0x23, 0xAE, 0x72, 0xB6,
+			0xFA, 0x19, 0x46, 0xBB, 0xFE, 0xA4, 0xC1, 0x64,
+			0xA8, 0x5E, 0xE8, 0xBB, 0x63, 0x58, 0x19, 0x50,
+			0xAA, 0x36, 0xC2, 0x4B, 0x38, 0x24, 0xD1, 0x2E,
+			0xAE, 0xAD, 0x6E, 0x34, 0x64, 0xA8, 0xC8, 0xF2,
+			0x4E, 0x74, 0x5C, 0x98, 0xE8, 0xDF, 0x99, 0x8C,
+			0x41, 0x79, 0x60, 0x2D, 0xD5, 0xF4, 0xE3, 0xE9,
+			0x1D, 0xF6, 0x5A, 0xA9, 0x69, 0x8E, 0xA1, 0x4F,
+			0xD3, 0x1B, 0x09, 0xA8, 0x7A, 0xD1, 0xE1, 0xCF,
+			0xAC, 0xBA, 0xD0, 0xD1, 0x34, 0x34, 0x8F, 0xC1,
+			0xD1, 0xA8, 0xAF, 0x6E, 0x92, 0xE0, 0xB0, 0xF6,
+			0xF9, 0x08, 0xA0, 0xCB, 0x58, 0x34, 0xF6, 0x68,
+			0xA2, 0xBF, 0x05, 0x39, 0x63, 0xBA, 0x4F, 0xEF,
+			0xE3, 0x95, 0x69, 0xD5, 0x89, 0x7C, 0x64, 0x07,
+			0x13, 0x42, 0x14, 0xF1, 0xA6, 0x9B, 0x87, 0xE5,
+			0xF4, 0x49, 0xAE, 0x67, 0x65, 0xCC, 0xF2, 0x26,
+			0xF8, 0x31, 0xBD, 0x33, 0x6A, 0x87, 0x77, 0x4E,
+			0xB1, 0xEE, 0xA4, 0xA2, 0xC8, 0xA0, 0x4A, 0xC1,
+			0xDF, 0x55, 0xE0, 0xDE, 0x53, 0x15, 0x3B, 0xEC,
+			0x55, 0x32, 0xCA, 0x06, 0xE4, 0x78, 0x59, 0x63,
+			0x10, 0x68, 0xA9, 0x46, 0x1B, 0xEF, 0x73, 0x6D,
+			0x1A, 0x02, 0x64, 0x12, 0x76, 0x9B, 0xDB, 0x7C,
+			0x03, 0x35, 0x19, 0xE1, 0x58, 0x7A, 0x87, 0x0C,
+			0x76, 0xDC, 0xFC, 0xC0, 0x28, 0xE4, 0xA2, 0x07,
+			0x9C, 0x28, 0x05, 0x21, 0x13, 0x58, 0xEF, 0x05,
+			0xBB, 0xAB, 0x94, 0xA2, 0x93, 0xBC, 0x31, 0x61,
+			0x26, 0x39, 0x38, 0x0C, 0xC4, 0x67, 0xDA, 0xA5,
+			0xE4, 0x1E, 0x1B, 0xB6, 0xE5, 0x73, 0xD6, 0x6C,
+			0xEE, 0xBC, 0x9D, 0xB9, 0xE7, 0xD9, 0x45, 0x2F,
+			0xF2, 0xB6, 0x92, 0x54, 0x41, 0x05, 0xB7, 0xB7,
+			0xFC, 0x37, 0x63, 0x6A, 0xB4, 0xBE, 0xB8, 0x3E,
+			0xD8, 0x53, 0x3B, 0xF8, 0x7D, 0x9A, 0x05, 0xDF,
+			0x20, 0x02, 0x27, 0x64, 0x38, 0xFA, 0x7D, 0xAF,
+			0x7F, 0xFA, 0xD1, 0xB7, 0x32, 0xC5, 0x74, 0x3E,
+			0x04, 0xA2, 0x67, 0x79, 0x02, 0x2E, 0x6F, 0xA1,
+			0x27, 0x87, 0x07, 0xB5, 0x9F, 0x0A, 0x7D, 0x5E,
+			0x14, 0xA0, 0x31, 0x46, 0x3F, 0xA9, 0xDE, 0x98,
+			0xB9, 0x89, 0xA0, 0x4A, 0x7A, 0xBD, 0x15, 0xAE,
+			0x2D, 0x0B, 0x38, 0x9A, 0xD8, 0x0E, 0xD2, 0xBA,
+			0x6D, 0xA1, 0x04, 0x1E, 0x4E, 0x39, 0x87, 0x4B,
+			0xC8, 0x3C, 0x74, 0x35, 0x4D, 0xC8, 0x1B, 0x42,
+			0x06, 0x5B, 0x73, 0xB7, 0x33, 0x86, 0x4A, 0x10,
+			0x2A, 0x10, 0x16, 0x28, 0x6F, 0x2A, 0xE3, 0x86,
+			0xDE, 0xA3, 0x44, 0x23, 0xE2, 0x90, 0xC4, 0x20,
+			0x90, 0xE0, 0xB8, 0xE6, 0xA7, 0xB6, 0xD6, 0x92,
+			0xF4, 0xF8, 0x8A, 0xBC, 0xAC, 0x31, 0x47, 0x8F,
+			0xAA, 0xE0, 0xD9, 0xF7, 0xE3, 0xCB, 0x11, 0xA4,
+			0x6B, 0x05, 0xB3, 0xB8, 0x72, 0x69, 0xE6, 0xDD,
+			0x75, 0x0F, 0x20, 0x1D, 0x3F, 0xC6, 0x96, 0xA0,
+			0x18, 0xB6, 0x24, 0xA1, 0xA6, 0xFD, 0x0C, 0x80,
+			0x1E, 0xD2, 0x28, 0xA2, 0x1A, 0x27, 0xF4, 0x23,
+			0x59, 0x1A, 0xCC, 0x0F, 0xD4, 0x99, 0xD0, 0xB4,
+			0x1E, 0x91, 0xC7, 0xD8, 0x8F, 0x8C, 0x5B, 0xEB,
+			0xB5, 0x9F, 0xFF, 0x4F, 0xD0, 0xD5, 0xB7, 0x60,
+			0xCC, 0x0A, 0x10, 0x38, 0xBF, 0xA8, 0x2E, 0xCC,
+			0xEB, 0x26, 0xB0, 0x78, 0xB3, 0xE0, 0x40, 0xAF,
+			0xCD, 0x12, 0xC5, 0x3A, 0x24, 0xD8, 0xEE, 0x3A,
+			0x64, 0x83, 0x2E, 0xD9, 0x25, 0x21, 0x66, 0xA5,
+			0x28, 0xD1, 0xE1, 0x84, 0x25, 0x1B, 0x20, 0xB8,
+			0xF5, 0x76, 0xB6, 0x3E, 0x4B, 0xC6, 0xEC, 0xC1,
+			0xC7, 0xAC, 0xC4, 0xAD, 0xCE, 0xF0, 0xB4, 0x0F,
+			0x35, 0x1E, 0xCE, 0x4E, 0xE3, 0x57, 0x30, 0xFC,
+			0xF4, 0x9B, 0x86, 0xB0, 0xDD, 0x3F, 0x2F, 0xB6,
+			0x10, 0x20, 0xE4, 0x24, 0x17, 0x1C, 0x24, 0xC6,
+			0x89, 0xE4, 0x14, 0xAD, 0x2E, 0x41, 0x08, 0x33,
+			0x88, 0xB1, 0x6F, 0x11, 0x85, 0xAF, 0x58, 0x17,
+			0xE3, 0x91, 0xB4, 0x72, 0xA2, 0x7F, 0xA3, 0x98,
+			0xAF, 0xB7, 0x6B, 0x58, 0x76, 0xA3, 0x11, 0x1C,
+			0x8A, 0x1A, 0xE6, 0x58, 0x54, 0xB0, 0xB9, 0x6E,
+			0x46, 0xCB, 0x16, 0xC0, 0x63, 0x0C, 0xEE, 0xA2,
+			0xAE, 0xF6, 0x71, 0xEF, 0xD1, 0xB9, 0x3D, 0xB7,
+			0x76, 0xCE, 0x5B, 0x84, 0x66, 0x7C, 0x7D, 0xF1,
+			0x96, 0x60, 0x34, 0xF6, 0xD1, 0x64, 0x27, 0xD9,
+			0xF3, 0x78, 0x8B, 0xF4, 0xC3, 0x1D, 0x37, 0xC0,
+			0xF4, 0x4A, 0xD0, 0xA5, 0x9A, 0xEB, 0xDD, 0x79,
+			0x54, 0x5D, 0xEB, 0x04, 0xC1, 0xA4, 0xBC, 0xED,
+			0xE3, 0x74, 0xC3, 0xB9, 0x9A, 0x6A, 0xAA, 0x06,
+			0xD1, 0xF0, 0x0F, 0xC5, 0xEF, 0x7E, 0x0B, 0xC8,
+			0xF4, 0x94, 0x4E, 0x69, 0x0E, 0x36, 0x00, 0x13,
+			0x45, 0xCE, 0x68, 0x13, 0xFE, 0x7F, 0x29, 0xA2,
+			0x1D, 0x79, 0xDF, 0xF2, 0x27, 0xFB, 0xAE, 0x52,
+			0x05, 0x78, 0xD7, 0xB9, 0xF7, 0x38, 0x68, 0xD5,
+			0xBA, 0xD7, 0xCF, 0x09, 0xC6, 0xD2, 0x5B, 0xC6,
+			0x98, 0xE4, 0xEC, 0xD5, 0xE9, 0xC2, 0xA5, 0x1A,
+			0x52, 0xC8, 0xA7, 0xBA, 0x3D, 0x74, 0x75, 0x00,
+			0xAA, 0xDD, 0x6A, 0x3F, 0xB6, 0x2F, 0x08, 0xB7,
+			0x1C, 0x6B, 0x52, 0x0C, 0xC9, 0xE4, 0x4D, 0xF4,
+			0xC5, 0x26, 0x1F, 0x35, 0x41, 0x25, 0x68, 0x17,
+			0xA8, 0x81, 0x75, 0xF4, 0x66, 0x41, 0xB5, 0xE4,
+			0x1D, 0x92, 0xEE, 0xDA, 0x0F, 0x56, 0x76, 0xC6,
+			0xAA, 0x0F, 0xA8, 0x63, 0x8D, 0xF0, 0x69, 0x63,
+			0x93, 0x45, 0xBC, 0x76, 0x40, 0xBE, 0xA9, 0x96,
+			0x36, 0xAF, 0x2F, 0x6B, 0x3E, 0xAB, 0xF3, 0xC0,
+			0xD7, 0xD5, 0xB1, 0x23, 0x23, 0xA2, 0xA0, 0xC4,
+			0xC5, 0x70, 0xEF, 0x66, 0x79, 0x15, 0xF0, 0xD0,
+			0x40, 0x0A, 0x33, 0x0C, 0xF3, 0x32, 0x6D, 0x8D,
+			0xB4, 0x44, 0x46, 0x78, 0x3F, 0x8D, 0x75, 0x40,
+			0xA5, 0x60, 0xBC, 0x9B, 0x76, 0xDF, 0x25, 0xF4,
+			0xE9, 0xED, 0xAC, 0x74, 0x2F, 0x9A, 0x00, 0xC4,
+			0x2B, 0x52, 0x26, 0x79, 0x09, 0x19, 0x57, 0x89,
+			0x60, 0x14, 0xBE, 0x65, 0xBD, 0x7B, 0x4D, 0x7D,
+			0x9B, 0x8B, 0x9E, 0x72, 0x6C, 0x0C, 0x57, 0xC7,
+			0x00, 0x08, 0x38, 0x7C, 0x37, 0x45, 0x9D, 0x55,
+			0xA2, 0x62, 0x5E, 0x34, 0x19, 0x99, 0x31, 0x16,
+			0xF1, 0x14, 0x44, 0x2D, 0xE3, 0x7E, 0x22, 0xE1,
+			0xA2, 0xB8, 0x9A, 0x9F, 0xE0, 0x37, 0x29, 0xBB,
+			0xCD, 0x46, 0xEE, 0x0A, 0x62, 0x2B, 0x98, 0x34,
+			0xBA, 0x9E, 0x54, 0x1B, 0xB1, 0x5C, 0x4F, 0xE9,
+			0xAA, 0xE4, 0x95, 0x8C, 0xA4, 0xEF, 0xC2, 0xB1,
+			0x7F, 0xF9, 0x80, 0xDA, 0x55, 0x95, 0x92, 0xC0,
+			0x86, 0xF4, 0x2D, 0x99, 0x3E, 0x17, 0xDC, 0x55,
+			0xA6, 0x33, 0x85, 0x90, 0x31, 0xC8, 0xFF, 0x58,
+			0x83, 0xC5, 0xBA, 0x60, 0x20, 0x5F, 0x87, 0x29,
+			0x20, 0x5A, 0x7D, 0x44, 0x2B, 0xA0, 0xE2, 0x99,
+			0xC8, 0x70, 0xBE, 0x89, 0xC5, 0xBC, 0xF6, 0x0D,
+			0x04, 0xC0, 0x96, 0xD1, 0x5C, 0xD1, 0x90, 0x43,
+			0xD3, 0x7B, 0x73, 0x52, 0x30, 0xB6, 0xA9, 0x7C,
+			0x0A, 0xA3, 0x24, 0x0E, 0x80, 0xFE, 0xBE, 0x31,
+			0xFD, 0xB5, 0x96, 0x04, 0x2B, 0xCF, 0x0B, 0x28,
+			0x1F, 0x7A, 0xCF, 0xC4, 0x82, 0x78, 0x52, 0x30,
+			0xB1, 0x34, 0x12, 0x50, 0x03, 0x09, 0x1C, 0x8B,
+			0x80, 0x60, 0xE3, 0xA1, 0xE5, 0x61, 0xF7, 0xD7,
+			0xB6, 0x74, 0xBE, 0xD6, 0x58, 0x03, 0xD3, 0xE5,
+			0xF7, 0xAC, 0x07, 0x60, 0xB7, 0x8A, 0xEC, 0xFA,
+			0xC6, 0x0F, 0xF0, 0x20, 0x04, 0x6B, 0x8F, 0x61,
+			0x09, 0x92, 0x03, 0xFB, 0x85, 0x99, 0x94, 0x9D,
+			0x2E, 0x6A, 0xC2, 0x9F, 0x20, 0x46, 0x2A, 0x96,
+			0xED, 0x42, 0x7D, 0x64, 0xA9, 0xE4, 0x1B, 0xDE,
+			0x11, 0x20, 0x12, 0x93, 0xE6, 0x2B, 0xE5, 0x93,
+			0x48, 0x37, 0x8C, 0x5A, 0x54, 0x0D, 0xEB, 0xF0,
+			0x9F, 0x9D, 0xE4, 0xA5, 0xC4, 0x93, 0x6F, 0x6A,
+			0xE3, 0x99, 0x69, 0xD9, 0xFE, 0x0C, 0x4E, 0xEC,
+			0x8B, 0x30, 0x1F, 0x7A, 0xB8, 0xC8, 0x5B, 0x61,
+			0x8E, 0xC2, 0x10, 0x90, 0x57, 0xB4, 0x72, 0x58,
+			0x7F, 0x41, 0x29, 0x7E, 0xF9, 0xBE, 0x40, 0xC3,
+			0x6F, 0xA9, 0xE3, 0x00, 0xE9, 0xC8, 0xFD, 0x4B,
+			0xFD, 0x3F, 0xE3, 0x3F, 0x25, 0x22, 0xFD, 0xB7,
+			0x2D, 0x57, 0xEF, 0x91, 0x08, 0xF0, 0x20, 0x56,
+			0x30, 0xFA, 0x83, 0x69, 0xFD, 0x56, 0x5A, 0x9B,
+			0xCE, 0xF8, 0x28, 0x02, 0xB4, 0x91, 0x35, 0x75,
+			0x9E, 0x63, 0x99, 0x48, 0xCF, 0x35, 0xF5, 0x58,
+			0x0C, 0x48, 0x8F, 0x0A, 0x2D, 0x9A, 0xE6, 0x40,
+			0xF6, 0x21, 0xB5, 0x69, 0xC1, 0x09, 0x31, 0x00,
+			0xA3, 0xC1, 0x4C, 0x99, 0x70, 0x4F, 0x5A, 0x63,
+			0x17, 0x90, 0xB8, 0xF8, 0x3A, 0x0E, 0xFD, 0x67,
+			0xEA, 0x0E, 0xBA, 0x7B, 0x1E, 0xEF, 0x37, 0x84,
+			0xD5, 0x51, 0x37, 0x01, 0xD6, 0x93, 0x15, 0xDF,
+			0x56, 0x89, 0x0E, 0x54, 0xF5, 0x1D, 0xF7, 0xE5,
+			0xB7, 0xC1, 0xF1, 0xC2, 0xD9, 0x14, 0x6F, 0x40,
+			0x55, 0x67, 0x50, 0x7C, 0x58, 0x35, 0x8B, 0x39,
+			0xCB, 0xB5, 0x87, 0xF5, 0x55, 0x5E, 0x26, 0x8C,
+			0x5B, 0x73, 0x0E, 0xBB, 0x25, 0x51, 0x0E, 0xAD,
+			0x57, 0x72, 0x7B, 0x68, 0x83, 0x11, 0x1E, 0x3A,
+			0x3D, 0xA4, 0x7C, 0x18, 0xB7, 0x70, 0x18, 0xBC,
+			0x72, 0x03, 0x4A, 0xA1, 0xD2, 0xF9, 0xA9, 0x8A,
+			0x25, 0x45, 0x19, 0xEE, 0x93, 0x06, 0xB5, 0x09,
+			0x71, 0xC9, 0x2D, 0xFD, 0x2B, 0xF3, 0xC7, 0x64,
+			0x5F, 0xCE, 0x71, 0x1D, 0x81, 0x96, 0x67, 0xBF,
+			0x01, 0x39, 0x8C, 0xE7, 0xA2, 0xD0, 0x98, 0x57,
+			0x5A, 0xFD, 0x21, 0xC7, 0x46, 0xAA, 0xB4, 0xE4,
+			0x0E, 0xBE, 0xC6, 0x68, 0x3E, 0x38, 0xF5, 0xA2,
+			0xED, 0x73, 0xCC, 0x53, 0x7E, 0x7E, 0x03, 0x32,
+			0xDC, 0xB6, 0xC1, 0x03, 0x9E, 0xB3, 0x2A, 0xAD,
+			0xC0, 0xC3, 0x6E, 0x47, 0xFB, 0x1E, 0xB7, 0x0D,
+			0x86, 0x95, 0x09, 0xA6, 0x9D, 0x6F, 0x92, 0xFC,
+			0xFF, 0x2C, 0x7D, 0x09, 0x16, 0x68, 0x50, 0x3E,
+			0x4F, 0x23, 0x4C, 0x93, 0x95, 0x2A, 0xE1, 0x9B,
+			0x16, 0xF0, 0x0F, 0xFF, 0x79, 0xA8, 0x06, 0xF9,
+			0x70, 0x61, 0x72, 0x2C, 0xE8, 0x91, 0x01, 0x6D,
+			0x45, 0xE5, 0x82, 0x5D, 0x26, 0x21, 0xAD, 0x3D,
+			0x77, 0x73, 0x23, 0x04, 0x84, 0x27, 0xA3, 0x5D,
+			0x6D, 0xA8, 0x99, 0xC1, 0xCE, 0x4F, 0xA9, 0xF7,
+			0xAB, 0x5C, 0xDE, 0x01, 0xE6, 0x1E, 0xEF, 0xE6,
+			0xFD, 0xE0, 0x68, 0x85, 0x3E, 0xEE, 0xBF, 0xF1,
+			0x0D, 0x79, 0xF4, 0xA2, 0xB4, 0x14, 0xBC, 0x0C,
+			0x49, 0x77, 0x03, 0x71, 0x08, 0x3E, 0x40, 0xA6,
+			0xD7, 0x03, 0xFA, 0xE2, 0xFB, 0xC7, 0x59, 0x30,
+			0x6E, 0x07, 0x06, 0x1C, 0x7C, 0x47, 0xE5, 0x4C,
+			0x57, 0x0A, 0x91, 0x4A, 0x43, 0xE4, 0x8A, 0xCD,
+			0x6E, 0x92, 0x01, 0xE2, 0x52, 0xC1, 0x92, 0x34,
+			0x8E, 0x64, 0x0F, 0x39, 0x63, 0x53, 0xAB, 0xE5,
+			0x44, 0xD5, 0xAA, 0xAA, 0xF6, 0x03, 0x89, 0xB9,
+			0xDD, 0xB2, 0x2D, 0x56, 0x1A, 0xE0, 0x72, 0x5A,
+			0x52, 0x19, 0x46, 0xEA, 0xB3, 0xCE, 0xB3, 0x59,
+			0x46, 0x7A, 0xA7, 0x48, 0x37, 0x0C, 0x09, 0xBA,
+			0x92, 0x70, 0x17, 0x7F, 0xF5, 0xD3, 0x60, 0x44,
+			0xC4, 0xC6, 0xC6, 0x7D, 0xD2, 0x21, 0xAC, 0x3F,
+			0x62, 0x6C, 0xE9, 0xBA, 0x4C, 0xF3, 0x82, 0x7E,
+			0x6D, 0x3A, 0x92, 0xDC, 0x94, 0xE4, 0x5F, 0xA6,
+			0x8B, 0x66, 0xA0, 0xDD, 0xE2, 0x97, 0x83, 0xED,
+			0xF5, 0x9D, 0xDF, 0x74, 0x77, 0x23, 0x7D, 0xDA,
+			0xC4, 0xFB, 0x92, 0x1A, 0xD9, 0x37, 0x36, 0xD2,
+			0x88, 0xC9, 0xEA, 0x0F, 0x98, 0xBD, 0xC5, 0xF8,
+			0xAA, 0x19, 0x75, 0x12, 0x6A, 0x41, 0xB5, 0xB3,
+			0xB5, 0xA4, 0x96, 0xDC, 0x2B, 0x49, 0x86, 0x66,
+			0x35, 0xD8, 0x4A, 0x62, 0xB4, 0xCB, 0x1E, 0x27,
+			0xC1, 0xAD, 0x34, 0x0E, 0x26, 0x16, 0xF2, 0xC2,
+			0x22, 0x52, 0x84, 0xD8, 0xD1, 0x32, 0xB8, 0x9C,
+			0xFE, 0x64, 0x42, 0x9F, 0xE4, 0x69, 0xF0, 0xAE,
+			0x3B, 0xD9, 0x2C, 0xA0, 0x14, 0xEB, 0x69, 0x74,
+			0x7C, 0xE2, 0xA6, 0x60, 0xE1, 0x52, 0x1C, 0xCC,
+			0xBF, 0xE6, 0xA1, 0x83, 0x20, 0x5D, 0x9E, 0xA3,
+			0xFB, 0x84, 0x8B, 0x33, 0xE6, 0xC9, 0x32, 0x83,
+			0xC0, 0x3F, 0x98, 0x1D, 0x6E, 0xC0, 0x50, 0x71,
+			0x29, 0x60, 0x5F, 0x36, 0xB4, 0x68, 0x1D, 0xB9,
+			0x76, 0x73, 0xC3, 0x80, 0xC5, 0xBC, 0x59, 0x7B,
+			0x59, 0xB4, 0xE0, 0x6A, 0x80, 0xCD, 0x4D, 0x8C,
+			0x9E, 0xE0, 0x0B, 0x45, 0x7D, 0x54, 0xD4, 0xC4,
+			0x97, 0x6C, 0x54, 0xEF, 0x14, 0x64, 0xBD, 0x3B,
+			0xD7, 0xEE, 0xF4, 0xD1, 0x41, 0x76, 0x3A, 0x24,
+			0x7A, 0xC2, 0xCA, 0x68, 0x28, 0x53, 0x46, 0xF7,
+			0x1B, 0xDA, 0x4B, 0x7A, 0x56, 0x75, 0x86, 0xFB,
+			0x31, 0x2C, 0x27, 0xF9, 0x4D, 0x35, 0xA4, 0x82,
+			0xE7, 0x2F, 0x41, 0xB4, 0xCA, 0xCE, 0x75, 0x94,
+			0x08, 0x54, 0xE2, 0x9E, 0x99, 0xC9, 0x85, 0xDE,
+			0x6F, 0x80, 0x95, 0x59, 0x3E, 0x54, 0x9F, 0x31,
+			0xF8, 0xDE, 0xD0, 0xD7, 0xA6, 0xD4, 0xD3, 0xBB,
+			0xD9, 0xC7, 0x55, 0xDD, 0xAE, 0xAD, 0x9E, 0x57,
+			0x4A, 0x33, 0x5D, 0x7A, 0xA6, 0xA3, 0xCA, 0xF9,
+			0x4C, 0x5B, 0x51, 0xCC, 0x22, 0xBB, 0x76, 0x44,
+			0x17, 0xDE, 0x22, 0xA1, 0xDF, 0x80, 0x13, 0x7D,
+			0xE5, 0x34, 0x7E, 0x75, 0x73, 0x10, 0x40, 0xFB,
+			0x9A, 0x21, 0xCD, 0xD3, 0xD3, 0x84, 0xB6, 0x0C,
+			0x31, 0x1E, 0xB5, 0x42, 0xF4, 0x34, 0x11, 0x7F,
+			0x4A, 0x23, 0xA8, 0xA5, 0x8F, 0x20, 0xCD, 0xA9,
+			0xF2, 0xE4, 0xEE, 0xFA, 0x57, 0xD1, 0x22, 0x1C,
+			0xA5, 0xDC, 0x0B, 0x25, 0xFE, 0xC2, 0xA7, 0x7E,
+			0x09, 0x2E, 0xDA, 0x40, 0x9F, 0x6C, 0xC8, 0x71,
+			0x58, 0x91, 0x04, 0x25, 0xD0, 0x06, 0xEA, 0x1B,
+			0xCD, 0x9D, 0x50, 0xD8, 0x40, 0x24, 0xAC, 0xC3,
+			0xB4, 0x07, 0x6E, 0x76, 0xF4, 0x4C, 0xD8, 0x80,
+			0xD0, 0x20, 0xF5, 0x15, 0x6A, 0x0A, 0x12, 0xF8,
+			0x6B, 0x67, 0x77, 0x34, 0xAE, 0x60, 0x68, 0x13,
+			0x5B, 0x8E, 0xFF, 0x5E, 0x7A, 0x77, 0x67, 0x0D,
+			0xE6, 0x96, 0x43, 0x9F, 0x8F, 0x77, 0x5F, 0x97,
+			0x23, 0x91, 0x33, 0x72, 0xC7, 0x8A, 0xC7, 0x80,
+			0xCF, 0xE7, 0x71, 0x06, 0x25, 0xB7, 0x4B, 0x89,
+			0x6A, 0x46, 0x67, 0x19, 0x49, 0x44, 0x03, 0x52,
+			0x32, 0xB1, 0x8F, 0xE7, 0x9E, 0xDA, 0x03, 0x41,
+			0xA3, 0xAC, 0xE5, 0xF3, 0x96, 0xE6, 0xAC, 0xFF,
+			0xEC, 0x35, 0x4D, 0x83, 0xA9, 0xCE, 0x7C, 0x52,
+			0xF2, 0x36, 0x97, 0xF0, 0x28, 0x36, 0x54, 0x59,
+			0x96, 0xEA, 0xEE, 0xB2, 0xC1, 0xAB, 0xA4, 0x96,
+			0x62, 0xD4, 0x3C, 0xF0, 0x1F, 0x2D, 0x65, 0x0E,
+			0x46, 0x7E, 0x12, 0x31, 0x8F, 0xA7, 0x8D, 0x7A,
+			0x4A, 0x41, 0x15, 0x57, 0x90, 0xF6, 0xF1, 0xE8,
+			0xE8, 0xE3, 0x57, 0x7B, 0x55, 0x85, 0x95, 0x97,
+			0xB3, 0x29, 0x3D, 0x02, 0x73, 0x1E, 0x87, 0x1F,
+			0x01, 0x89, 0x06, 0x88, 0x9E, 0x8A, 0x2E, 0xE0,
+			0x99, 0xFC, 0xF0, 0x48, 0x60, 0x37, 0x65, 0x25,
+			0xDB, 0x89, 0xC9, 0x7A, 0x51, 0x7E, 0x35, 0x92,
+			0x00, 0xC9, 0x61, 0x3F, 0x88, 0xE3, 0x20, 0x01,
+			0x46, 0x5A, 0x2C, 0x37, 0x02, 0xC9, 0x15, 0x0F,
+			0xB2, 0xEB, 0xC3, 0x55, 0x18, 0xF0, 0x15, 0x1A,
+			0x08, 0x8E, 0xB8, 0x9D, 0x18, 0xE4, 0x9F, 0x34,
+			0x10, 0x67, 0x68, 0x57, 0x60, 0x61, 0xEC, 0xD5,
+			0xD9, 0xA8, 0x3A, 0xAB, 0xD6, 0xD2, 0x7A, 0x47,
+			0x3D, 0xA4, 0x08, 0x7C, 0x3E, 0x4D, 0x76, 0x08,
+			0x19, 0x22, 0xB2, 0x89, 0x57, 0x84, 0xC2, 0x98,
+			0x72, 0xB8, 0x8B, 0xE0, 0x85, 0xA1, 0x3A, 0xC2,
+			0xA0, 0x06, 0x43, 0x03, 0xCF, 0x4F, 0x27, 0x80,
+			0x48, 0x9A, 0xBC, 0xB3, 0x3C, 0xC4, 0x5E, 0xAC,
+			0x8B, 0x85, 0x6F, 0x21, 0xD6, 0xFE, 0x12, 0x90,
+			0x53, 0x2F, 0x65, 0x32, 0x8D, 0x03, 0xFE, 0xFE,
+			0x61, 0x04, 0x47, 0x24, 0x6A, 0xB5, 0x01, 0x98,
+			0xB9, 0x27, 0x10, 0xE1, 0x32, 0x3D, 0x2A, 0xA0,
+			0xC5, 0x70, 0xDE, 0x1E, 0x10, 0xD7, 0x01, 0x50,
+			0x4F, 0x87, 0xCA, 0xD9, 0xBF, 0x12, 0xEA, 0x38,
+			0x4C, 0x43, 0xD5, 0x5A, 0xEF, 0x3E, 0x21, 0x8E,
+			0x59, 0x77, 0x23, 0xED, 0x51, 0x09, 0x99, 0x73,
+			0xD2, 0x56, 0x04, 0xDD, 0x8F, 0x5F, 0xDF, 0x79,
+			0xFF, 0x16, 0x8C, 0xB0, 0xBA, 0x8A, 0x1A, 0x56,
+			0xAF, 0xCA, 0x19, 0xF2, 0x64, 0x1A, 0xF5, 0x1E,
+			0xA7, 0xA7, 0x84, 0x3D, 0xAD, 0xC1, 0x0E, 0x22,
+			0xA1, 0x45, 0xFC, 0xB4, 0x13, 0x91, 0x34, 0xB7,
+			0x48, 0xEF, 0x9E, 0xD9, 0x0B, 0xE3, 0x82, 0x75,
+			0x80, 0xC5, 0xD9, 0xA0, 0x77, 0xA3, 0xF9, 0xCC,
+			0x67, 0xDD, 0xCB, 0x28, 0xC6, 0xE8, 0x2C, 0xB8,
+			0xAC, 0x63, 0xBD, 0x3B, 0x28, 0x4A, 0xE9, 0x2D,
+			0x29, 0x84, 0xD7, 0x4F, 0x61, 0x06, 0xE3, 0x37,
+			0xC1, 0x58, 0x20, 0x5D, 0x0A, 0xE7, 0x45, 0x29,
+			0x7D, 0xED, 0x0F, 0xCE, 0x00, 0x95, 0x2A, 0x62,
+			0x38, 0xA8, 0x1A, 0x3A, 0x96, 0x0E, 0x56, 0xD9,
+			0x18, 0xC2, 0x25, 0xA5, 0xAA, 0x27, 0x0A, 0x6E,
+			0xDD, 0x1C, 0x35, 0x6C, 0xC1, 0x26, 0x90, 0xF6,
+			0x43, 0x1B, 0x34, 0xDA, 0xE1, 0x5D, 0x09, 0x7C,
+			0xBE, 0x0F, 0x40, 0xD3, 0x24, 0x82, 0x0B, 0xFF,
+			0xE6, 0xB7, 0x10, 0xD6, 0x36, 0xD0, 0xE6, 0xC0,
+			0xBE, 0x65, 0x4C, 0x83, 0xF1, 0xDA, 0xDE, 0xCE,
+			0xE8, 0x5A, 0x80, 0x88, 0xFE, 0x9B, 0x79, 0x54,
+			0xA3, 0xA4, 0x5A, 0x76, 0xD0, 0xE2, 0xCE, 0x92,
+			0x53, 0x7D, 0x9C, 0xDA, 0xA1, 0xED, 0x9F, 0x56,
+			0x05, 0x0A, 0xA4, 0x81, 0xC7, 0x82, 0x5B, 0xB8,
+			0xC7, 0xA8, 0x95, 0x21, 0x99, 0x0B, 0x0F, 0xD2,
+			0x66, 0x68, 0xC3, 0x07, 0x53, 0x93, 0x8C, 0x68,
+			0x5A, 0xF5, 0x6F, 0x5E, 0x07, 0x68, 0x70, 0xF2,
+			0x6A, 0x78, 0xA8, 0xDB, 0x24, 0x6F, 0xD9, 0x74,
+			0x38, 0x29, 0xBF, 0x50, 0xCC, 0xC8, 0x34, 0x82,
+			0x69, 0x1A, 0xF4, 0x1A, 0x95, 0xE1, 0x31, 0x39,
+			0x00, 0xBE, 0xF6, 0x25, 0x19, 0x5A, 0xF6, 0xA7,
+			0xB7, 0xC7, 0xAC, 0xFA, 0x52, 0x6D, 0xC7, 0xEA,
+			0xFF, 0xC4, 0xB0, 0x84, 0x41, 0x2D, 0x6B, 0x22,
+			0x57, 0x1F, 0x8A, 0x97, 0x71, 0xEF, 0x54, 0xA7,
+			0xA9, 0xB5, 0xA0, 0x7A, 0xFF, 0x52, 0xBC, 0x78,
+			0x56, 0x8C, 0x9C, 0x2A, 0xFB, 0x31, 0xA6, 0xC8,
+			0x87, 0xC8, 0x82, 0x72, 0x00, 0x68, 0x27, 0xDE,
+			0x54, 0x67, 0x4C, 0x36, 0x4E, 0xE2, 0x77, 0x7F,
+			0xAD, 0x15, 0x1B, 0xC9, 0x07, 0xC4, 0x84, 0x50,
+			0x84, 0x45, 0xB5, 0x1D, 0xCE, 0x3F, 0x7C, 0xDF,
+			0x73, 0x88, 0x3E, 0x6D, 0xAC, 0x18, 0x67, 0x2C,
+			0x1D, 0x31, 0xD5, 0x41, 0x6E, 0xFC, 0x3D, 0xFE,
+			0x5C, 0x6D, 0x3B, 0xCB, 0x67, 0x05, 0x44, 0x8B,
+			0x02, 0xB7, 0xF5, 0x05, 0xFD, 0x1D, 0x7E, 0x13,
+			0x90, 0xCE, 0xED, 0xD2, 0xAB, 0x08, 0xFF, 0xC3,
+			0x91, 0x2C, 0x79, 0x06, 0xDB, 0x54, 0xAB, 0xFF,
+			0xF6, 0x9D, 0xBB, 0xDC, 0x3C, 0xCD, 0x03, 0xE7,
+			0xD8, 0x1A, 0x4E, 0x7F, 0xCB, 0x1B, 0xA3, 0xCA,
+			0xDC, 0x14, 0xC5, 0xFE, 0x46, 0x38, 0x07, 0x82,
+			0x5E, 0x9A, 0x77, 0x9C, 0xB4, 0x44, 0x26, 0xBA,
+			0xC2, 0x27, 0xD0, 0xF4, 0x75, 0x67, 0x4A, 0x15,
+			0x2A, 0x55, 0x2A, 0x61, 0x87, 0x55, 0xA0, 0xFB,
+			0xE0, 0x93, 0xA5, 0xD7, 0xDF, 0x0D, 0x97, 0xD3,
+			0x93, 0x39, 0x0B, 0x5A, 0xC6, 0x86, 0x17, 0x7D,
+			0x6A, 0xA4, 0x07, 0x9C, 0xB9, 0x0F, 0x42, 0x15,
+			0x46, 0x71, 0x45, 0x70, 0x75, 0x4B, 0xD5, 0x80,
+			0x64, 0x62, 0x92, 0x50, 0xBC, 0x10, 0xA1, 0x68,
+			0x60, 0x5A, 0x6F, 0x31, 0x26, 0xCA, 0xB1, 0x48,
+			0xFB, 0xC6, 0xF4, 0x3A, 0xCA, 0x52, 0x20, 0x2F,
+			0x7D, 0xEC, 0x19, 0xF8, 0xAA, 0x27, 0xC2, 0x23,
+			0xD0, 0x8F, 0x60, 0xFC, 0x7F, 0xA0, 0xCB, 0xDC,
+			0xA2, 0xC6, 0xC4, 0xC5, 0x13, 0x00, 0xF3, 0x43,
+			0x48, 0x6D, 0xFD, 0x7D, 0xB9, 0xA8, 0x14, 0xB0,
+			0x0C, 0x72, 0x82, 0x2F, 0x99, 0x64, 0x41, 0x2B,
+			0xB3, 0x34, 0x73, 0x73, 0xF4, 0x26, 0x1D, 0x06,
+			0xDF, 0x6E, 0xF4, 0x20, 0x1E, 0x31, 0xE3, 0x8A,
+			0x01, 0x6C, 0xDB, 0x3C, 0xE3, 0xC6, 0xC4, 0xC5,
+			0xB8, 0x20, 0x51, 0xF1, 0xD6, 0xB0, 0x30, 0xB7,
+			0x2D, 0xDA, 0x95, 0x01, 0x0D, 0xED, 0xEE, 0x6F,
+			0x69, 0xFD, 0xCF, 0x9D, 0xDD, 0x05, 0xD6, 0xC0,
+			0xFE, 0x11, 0x67, 0xAF, 0x53, 0x94, 0x60, 0xFC,
+			0x56, 0xBA, 0x0C, 0x5F, 0xA7, 0x7E, 0xDA, 0x65
+		},
+		.len = 4096 << 3
+	},
+	.validCipherLenInBits = {
+		.len = 4096 << 3
+	},
+	.cipher = {
+		.len_bits = 4096,
+		.offset_bits = 0
+	},
+	.dataunit_len = 0
+};
+
+
+
+
+#endif /*TEST_CRYPTODEV_MLX5_TEST_VECTORS_H_*/
-- 
2.27.0

[dpdk-dev] [PATCH v5 15/15] test/crypto: add mlx5 multi segment tests

[Shiri Kuzin]

The crypto mlx5 driver supports multi segment encryption and decryption
operations.

Added mlx5 multi segment encryption function and multi segment
decryption function that will both use the mlx5 vectors.

The added tests will test both data integrity and correct stat values.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 app/test/test_cryptodev.c                   | 277 ++++++++++++++++++++
 app/test/test_cryptodev_mlx5_test_vectors.h |   3 -
 2 files changed, 277 insertions(+), 3 deletions(-)

diff --git a/app/test/test_cryptodev.c b/app/test/test_cryptodev.c
index 8dbe324b81..4d27a9444c 100644
--- a/app/test/test_cryptodev.c
+++ b/app/test/test_cryptodev.c
@@ -6681,6 +6681,219 @@ test_mlx5_decryption(const struct mlx5_test_data *tdata)
 	return 0;
 }
 
+static int
+test_mlx5_encryption_sgl(const struct mlx5_test_data *tdata)
+{
+	struct crypto_testsuite_params *ts_params = &testsuite_params;
+	struct crypto_unittest_params *ut_params = &unittest_params;
+	struct rte_cryptodev_sym_capability_idx cap_idx;
+	struct rte_cryptodev_info dev_info;
+	struct rte_cryptodev_stats stats;
+	uint8_t buffer[10000];
+	const uint8_t *ciphertext;
+	const uint8_t *reference_ciphertext;
+	uint64_t feat_flags;
+	unsigned int plaintext_pad_len;
+	unsigned int plaintext_len;
+	int retval;
+
+	/* Verify the capabilities */
+	cap_idx.type = RTE_CRYPTO_SYM_XFORM_CIPHER;
+	cap_idx.algo.cipher = RTE_CRYPTO_CIPHER_AES_XTS;
+	if (rte_cryptodev_sym_capability_get(ts_params->valid_devs[0],
+			&cap_idx) == NULL)
+		return TEST_SKIPPED;
+	rte_cryptodev_info_get(ts_params->valid_devs[0], &dev_info);
+	feat_flags = dev_info.feature_flags;
+	if (!(feat_flags & RTE_CRYPTODEV_FF_IN_PLACE_SGL)) {
+		printf("Device doesn't support in-place scatter-gather. "
+				"Test Skipped.\n");
+		return TEST_SKIPPED;
+	}
+	if ((global_api_test_type == CRYPTODEV_RAW_API_TEST) &&
+			(!(feat_flags & RTE_CRYPTODEV_FF_SYM_RAW_DP))) {
+		printf("Device doesn't support RAW data-path APIs.\n");
+		return TEST_SKIPPED;
+	}
+	if (gbl_action_type == RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO)
+		return TEST_SKIPPED;
+	/* Create mlx5 session */
+	retval = create_wireless_algo_cipher_session(ts_params->valid_devs[0],
+					RTE_CRYPTO_CIPHER_OP_ENCRYPT,
+					RTE_CRYPTO_CIPHER_AES_XTS,
+					tdata->key.data, tdata->key.len,
+					tdata->cipher_iv.len, 0);
+	if (retval < 0)
+		return retval;
+	plaintext_len = ceil_byte_length(tdata->plaintext.len);
+	/* Append data which is padded to a multiple */
+	/* of the algorithms block size */
+	plaintext_pad_len = RTE_ALIGN_CEIL(plaintext_len, 8);
+	ut_params->ibuf = create_segmented_mbuf(ts_params->mbuf_pool,
+			plaintext_pad_len, 8, 0);
+	if (unlikely(ut_params->ibuf == NULL))
+		return -ENOMEM;
+	pktmbuf_write(ut_params->ibuf, 0, plaintext_len, tdata->plaintext.data);
+	debug_hexdump(stdout, "plaintext:", tdata->plaintext.data,
+		      plaintext_len);
+	/* Create mlx5 operation */
+	retval = create_wireless_algo_cipher_operation(tdata->cipher_iv.data,
+				tdata->cipher_iv.len, (tdata->cipher.len_bits),
+				(tdata->cipher.offset_bits));
+	if (retval < 0)
+		return retval;
+	if (global_api_test_type == CRYPTODEV_RAW_API_TEST)
+		process_sym_raw_dp_op(ts_params->valid_devs[0], 0,
+				ut_params->op, 1, 0, 1, tdata->cipher_iv.len);
+	else
+		ut_params->op = process_crypto_request(ts_params->valid_devs[0],
+						ut_params->op);
+	TEST_ASSERT_NOT_NULL(ut_params->op, "failed to retrieve obuf");
+	ut_params->obuf = ut_params->op->sym->m_dst;
+	if (ut_params->obuf)
+		ciphertext = rte_pktmbuf_read(ut_params->obuf, 0,
+				plaintext_len, buffer);
+	else
+		ciphertext = rte_pktmbuf_read(ut_params->ibuf,
+				tdata->cipher.offset_bits,
+				plaintext_len, buffer);
+	if (unlikely(ciphertext == NULL))
+		return -ENOMEM;
+	/* Validate obuf */
+	debug_hexdump(stdout, "ciphertext:", ciphertext, plaintext_len);
+	reference_ciphertext = tdata->ciphertext.data +
+				tdata->cipher.offset_bits;
+	/* Validate obuf */
+	TEST_ASSERT_BUFFERS_ARE_EQUAL_BIT(
+		ciphertext,
+		reference_ciphertext,
+		tdata->validCipherLenInBits.len,
+		"MLX5 Ciphertext data not as expected");
+	/* Validate stats */
+	TEST_ASSERT_SUCCESS(rte_cryptodev_stats_get(ts_params->valid_devs[0],
+			&stats),
+		"rte_cryptodev_stats_get failed");
+	TEST_ASSERT((stats.enqueued_count == 1),
+		"rte_cryptodev_stats_get returned unexpected enqueued stat");
+	TEST_ASSERT((stats.dequeued_count == 1),
+		   "rte_cryptodev_stats_get returned unexpected dequeued stat");
+	TEST_ASSERT((stats.enqueue_err_count == 0),
+		   "rte_cryptodev_stats_get returned error enqueued stat");
+	TEST_ASSERT((stats.dequeue_err_count == 0),
+		   "rte_cryptodev_stats_get returned error dequeued stat");
+	return 0;
+}
+
+static int
+test_mlx5_decryption_sgl(const struct mlx5_test_data *tdata)
+{
+	struct crypto_testsuite_params *ts_params = &testsuite_params;
+	struct crypto_unittest_params *ut_params = &unittest_params;
+	struct rte_cryptodev_sym_capability_idx cap_idx;
+	struct rte_cryptodev_stats stats;
+	struct rte_cryptodev_info dev_info;
+	uint8_t *ciphertext;
+	const uint8_t *plaintext;
+	const uint8_t *reference_plaintext;
+	uint8_t buffer[10000];
+	uint64_t feat_flags;
+	unsigned int ciphertext_pad_len;
+	unsigned int ciphertext_len;
+	int retval;
+
+	rte_cryptodev_info_get(ts_params->valid_devs[0], &dev_info);
+	feat_flags = dev_info.feature_flags;
+	if ((global_api_test_type == CRYPTODEV_RAW_API_TEST) &&
+			(!(feat_flags & RTE_CRYPTODEV_FF_SYM_RAW_DP))) {
+		printf("Device doesn't support RAW data-path APIs.\n");
+		return -ENOTSUP;
+	}
+	if (gbl_action_type == RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO)
+		return -ENOTSUP;
+	/* Verify the capabilities */
+	cap_idx.type = RTE_CRYPTO_SYM_XFORM_CIPHER;
+	cap_idx.algo.cipher = RTE_CRYPTO_CIPHER_AES_XTS;
+	if (rte_cryptodev_sym_capability_get(ts_params->valid_devs[0],
+			&cap_idx) == NULL)
+		return -ENOTSUP;
+	/* Create mlx5 session */
+	retval = create_wireless_algo_cipher_session(ts_params->valid_devs[0],
+					RTE_CRYPTO_CIPHER_OP_DECRYPT,
+					RTE_CRYPTO_CIPHER_AES_XTS,
+					tdata->key.data, tdata->key.len,
+					tdata->cipher_iv.len, 0);
+	if (retval < 0)
+		return retval;
+	ut_params->ibuf = rte_pktmbuf_alloc(ts_params->mbuf_pool);
+	if (unlikely(ut_params->ibuf == NULL))
+		return -ENOMEM;
+	/* Clear mbuf payload */
+	memset(rte_pktmbuf_mtod(ut_params->ibuf, uint8_t *), 0,
+	       rte_pktmbuf_tailroom(ut_params->ibuf));
+	ciphertext_len = ceil_byte_length(tdata->ciphertext.len);
+	/* Append data which is padded to a multiple */
+	/* of the algorithms block size */
+	ciphertext_pad_len = RTE_ALIGN_CEIL(ciphertext_len, 8);
+	ciphertext = (uint8_t *)rte_pktmbuf_append(ut_params->ibuf,
+				ciphertext_pad_len);
+	if (unlikely(ciphertext == NULL))
+		return -ENOMEM;
+	ut_params->ibuf = create_segmented_mbuf(ts_params->mbuf_pool,
+			ciphertext_pad_len, 8, 0);
+	if (unlikely(ut_params->ibuf == NULL))
+		return -ENOMEM;
+	pktmbuf_write(ut_params->ibuf, 0, ciphertext_len,
+			tdata->ciphertext.data);
+	memcpy(ciphertext, tdata->ciphertext.data, ciphertext_len);
+	if (unlikely(ciphertext == NULL))
+		return -ENOMEM;
+	debug_hexdump(stdout, "ciphertext:", ciphertext, ciphertext_len);
+	/* Create mlx5 operation */
+	retval = create_wireless_algo_cipher_operation(tdata->cipher_iv.data,
+				tdata->cipher_iv.len, (tdata->cipher.len_bits),
+				(tdata->cipher.offset_bits));
+	if (retval < 0)
+		return retval;
+	if (global_api_test_type == CRYPTODEV_RAW_API_TEST)
+		process_sym_raw_dp_op(ts_params->valid_devs[0], 0,
+				ut_params->op, 1, 0, 1, 0);
+	else
+		ut_params->op = process_crypto_request(ts_params->valid_devs[0],
+						ut_params->op);
+	TEST_ASSERT_NOT_NULL(ut_params->op, "failed to retrieve obuf");
+	ut_params->obuf = ut_params->op->sym->m_dst;
+	if (ut_params->obuf)
+		plaintext = rte_pktmbuf_read(ut_params->obuf, 0,
+				ciphertext_len, buffer);
+	else
+		plaintext = rte_pktmbuf_read(ut_params->ibuf,
+				tdata->cipher.offset_bits,
+				ciphertext_len, buffer);
+	debug_hexdump(stdout, "plaintext:", plaintext, ciphertext_len);
+	reference_plaintext = tdata->plaintext.data +
+				(tdata->cipher.offset_bits);
+	/* Validate obuf */
+	TEST_ASSERT_BUFFERS_ARE_EQUAL_BIT(
+		plaintext,
+		reference_plaintext,
+		tdata->validCipherLenInBits.len,
+		"MLX5 Plaintext data not as expected");
+	/* Validate stats */
+	TEST_ASSERT_SUCCESS(rte_cryptodev_stats_get(ts_params->valid_devs[0],
+			&stats),
+		"rte_cryptodev_stats_get failed");
+	TEST_ASSERT((stats.enqueued_count == 1),
+		"rte_cryptodev_stats_get returned unexpected enqueued stat");
+	TEST_ASSERT((stats.dequeued_count == 1),
+		   "rte_cryptodev_stats_get returned unexpected dequeued stat");
+	TEST_ASSERT((stats.enqueue_err_count == 0),
+		   "rte_cryptodev_stats_get returned error enqueued stat");
+	TEST_ASSERT((stats.dequeue_err_count == 0),
+		   "rte_cryptodev_stats_get returned error dequeued stat");
+	return 0;
+}
+
+
 static int
 test_kasumi_encryption_test_case_1(void)
 {
@@ -7345,6 +7558,54 @@ test_mlx5_decryption_test_case_4(void)
 	return test_mlx5_decryption(&mlx5_test_case_cipher_aes_xts_4);
 }
 
+static int
+test_mlx5_encryption_test_case_1_sgl(void)
+{
+	return test_mlx5_encryption_sgl(&mlx5_test_case_cipher_aes_xts_1);
+}
+
+static int
+test_mlx5_encryption_test_case_2_sgl(void)
+{
+	return test_mlx5_encryption_sgl(&mlx5_test_case_cipher_aes_xts_2);
+}
+
+static int
+test_mlx5_encryption_test_case_3_sgl(void)
+{
+	return test_mlx5_encryption_sgl(&mlx5_test_case_cipher_aes_xts_3);
+}
+
+static int
+test_mlx5_encryption_test_case_4_sgl(void)
+{
+	return test_mlx5_encryption_sgl(&mlx5_test_case_cipher_aes_xts_4);
+}
+
+static int
+test_mlx5_decryption_test_case_1_sgl(void)
+{
+	return test_mlx5_decryption_sgl(&mlx5_test_case_cipher_aes_xts_1);
+}
+
+static int
+test_mlx5_decryption_test_case_2_sgl(void)
+{
+	return test_mlx5_decryption_sgl(&mlx5_test_case_cipher_aes_xts_2);
+}
+
+static int
+test_mlx5_decryption_test_case_3_sgl(void)
+{
+	return test_mlx5_decryption_sgl(&mlx5_test_case_cipher_aes_xts_3);
+}
+
+static int
+test_mlx5_decryption_test_case_4_sgl(void)
+{
+	return test_mlx5_decryption_sgl(&mlx5_test_case_cipher_aes_xts_4);
+}
+
 static int
 test_mixed_check_if_unsupported(const struct mixed_cipher_auth_test_data *tdata)
 {
@@ -14747,6 +15008,22 @@ static struct unit_test_suite cryptodev_mlx5_testsuite  = {
 			test_mlx5_encryption_test_case_3),
 		TEST_CASE_ST(ut_setup, ut_teardown,
 			test_mlx5_encryption_test_case_4),
+		TEST_CASE_ST(ut_setup, ut_teardown,
+			test_mlx5_decryption_test_case_1_sgl),
+		TEST_CASE_ST(ut_setup, ut_teardown,
+			test_mlx5_decryption_test_case_2_sgl),
+		TEST_CASE_ST(ut_setup, ut_teardown,
+			test_mlx5_decryption_test_case_3_sgl),
+		TEST_CASE_ST(ut_setup, ut_teardown,
+			test_mlx5_decryption_test_case_4_sgl),
+		TEST_CASE_ST(ut_setup, ut_teardown,
+			test_mlx5_encryption_test_case_1_sgl),
+		TEST_CASE_ST(ut_setup, ut_teardown,
+			test_mlx5_encryption_test_case_2_sgl),
+		TEST_CASE_ST(ut_setup, ut_teardown,
+			test_mlx5_encryption_test_case_3_sgl),
+		TEST_CASE_ST(ut_setup, ut_teardown,
+			test_mlx5_encryption_test_case_4_sgl),
 		TEST_CASES_END()
 	}
 };
diff --git a/app/test/test_cryptodev_mlx5_test_vectors.h b/app/test/test_cryptodev_mlx5_test_vectors.h
index 2a05aa4626..db7e5fc744 100644
--- a/app/test/test_cryptodev_mlx5_test_vectors.h
+++ b/app/test/test_cryptodev_mlx5_test_vectors.h
@@ -2496,7 +2496,4 @@ static struct mlx5_test_data mlx5_test_case_cipher_aes_xts_4 = {
 	.dataunit_len = 0
 };
 
-
-
-
 #endif /*TEST_CRYPTODEV_MLX5_TEST_VECTORS_H_*/
-- 
2.27.0

Re: [dpdk-dev] [PATCH v5 01/15] drivers: introduce mlx5 crypto PMD

[David Marchand]

On Thu, Jul 1, 2021 at 3:27 PM Shiri Kuzin <shirik@nvidia.com> wrote:

[snip]

> +RTE_INIT(rte_mlx5_crypto_init)
> +{
> +       mlx5_common_init();
> +       if (mlx5_glue != NULL)
> +               mlx5_pci_driver_register(&mlx5_crypto_driver);
> +}
> +
> +RTE_PMD_REGISTER_CRYPTO_DRIVER(mlx5_cryptodev_driver, mlx5_drv,
> +                              mlx5_crypto_driver_id);
> +
> +RTE_LOG_REGISTER(mlx5_crypto_logtype, MLX5_CRYPTO_LOG_NAME, NOTICE)

Please use helpers added recently:
https://git.dpdk.org/dpdk/commit/?id=eeded2044af5bbe88220120b14933536cbb3edb6

RTE_LOG_REGISTER_DEFAULT(mlx5_crypto_logtype, NOTICE)


> +RTE_PMD_EXPORT_NAME(MLX5_CRYPTO_DRIVER_NAME, __COUNTER__);
> +RTE_PMD_REGISTER_PCI_TABLE(MLX5_CRYPTO_DRIVER_NAME, mlx5_crypto_pci_id_map);
> +RTE_PMD_REGISTER_KMOD_DEP(MLX5_CRYPTO_DRIVER_NAME, "* ib_uverbs & mlx5_core & mlx5_ib");


-- 
David Marchand

Re: [dpdk-dev] [EXT] [PATCH v5 15/15] test/crypto: add mlx5 multi segment tests

[Akhil Goyal]

> The crypto mlx5 driver supports multi segment encryption and decryption
> operations.
> 
> Added mlx5 multi segment encryption function and multi segment
> decryption function that will both use the mlx5 vectors.
> 
> The added tests will test both data integrity and correct stat values.
> 
> Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
> Acked-by: Matan Azrad <matan@nvidia.com>
> ---
>  app/test/test_cryptodev.c                   | 277 ++++++++++++++++++++
>  app/test/test_cryptodev_mlx5_test_vectors.h |   3 -
>  2 files changed, 277 insertions(+), 3 deletions(-)
> 
> diff --git a/app/test/test_cryptodev.c b/app/test/test_cryptodev.c
> index 8dbe324b81..4d27a9444c 100644
> --- a/app/test/test_cryptodev.c
> +++ b/app/test/test_cryptodev.c
> @@ -6681,6 +6681,219 @@ test_mlx5_decryption(const struct
> mlx5_test_data *tdata)
>  	return 0;
>  }
> 
> +static int
> +test_mlx5_encryption_sgl(const struct mlx5_test_data *tdata)

The test cases cannot be specific to one particular device.
Hence the name test_mlx5_xxxx cannot be accepted.

Moreover is it not possible to add aes-xts in test_blockcipher_one_case()
and add only the test vectors of aes-xts?
And probably, you do not need a new file for test vectors, XTS is variant of AES,
Hence the vectors can be part of "test_cryptodev_aes_test_vectors.h".

I don't think there is need for a separate function for XTS right now.
The current function test_blockcipher_one_case() covers all capability checks
And feature flag checks so that the test is skipped for the devices which do not
Support a specific case.

> +{
> +	struct crypto_testsuite_params *ts_params = &testsuite_params;
> +	struct crypto_unittest_params *ut_params = &unittest_params;
> +	struct rte_cryptodev_sym_capability_idx cap_idx;
> +	struct rte_cryptodev_info dev_info;
> +	struct rte_cryptodev_stats stats;
> +	uint8_t buffer[10000];
> +	const uint8_t *ciphertext;
> +	const uint8_t *reference_ciphertext;
> +	uint64_t feat_flags;
> +	unsigned int plaintext_pad_len;
> +	unsigned int plaintext_len;
> +	int retval;
> +
> +	/* Verify the capabilities */
> +	cap_idx.type = RTE_CRYPTO_SYM_XFORM_CIPHER;
> +	cap_idx.algo.cipher = RTE_CRYPTO_CIPHER_AES_XTS;
> +	if (rte_cryptodev_sym_capability_get(ts_params->valid_devs[0],
> +			&cap_idx) == NULL)
> +		return TEST_SKIPPED;
> +	rte_cryptodev_info_get(ts_params->valid_devs[0], &dev_info);
> +	feat_flags = dev_info.feature_flags;
> +	if (!(feat_flags & RTE_CRYPTODEV_FF_IN_PLACE_SGL)) {
> +		printf("Device doesn't support in-place scatter-gather. "
> +				"Test Skipped.\n");
> +		return TEST_SKIPPED;
> +	}
> +	if ((global_api_test_type == CRYPTODEV_RAW_API_TEST) &&
> +			(!(feat_flags & RTE_CRYPTODEV_FF_SYM_RAW_DP)))
> {
> +		printf("Device doesn't support RAW data-path APIs.\n");
> +		return TEST_SKIPPED;
> +	}
> +	if (gbl_action_type == RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO)
> +		return TEST_SKIPPED;
> +	/* Create mlx5 session */
> +	retval = create_wireless_algo_cipher_session(ts_params-
> >valid_devs[0],
> +					RTE_CRYPTO_CIPHER_OP_ENCRYPT,
> +					RTE_CRYPTO_CIPHER_AES_XTS,
> +					tdata->key.data, tdata->key.len,
> +					tdata->cipher_iv.len, 0);
> +	if (retval < 0)
> +		return retval;
> +	plaintext_len = ceil_byte_length(tdata->plaintext.len);
> +	/* Append data which is padded to a multiple */
> +	/* of the algorithms block size */
> +	plaintext_pad_len = RTE_ALIGN_CEIL(plaintext_len, 8);
> +	ut_params->ibuf = create_segmented_mbuf(ts_params->mbuf_pool,
> +			plaintext_pad_len, 8, 0);
> +	if (unlikely(ut_params->ibuf == NULL))
> +		return -ENOMEM;
> +	pktmbuf_write(ut_params->ibuf, 0, plaintext_len, tdata-
> >plaintext.data);
> +	debug_hexdump(stdout, "plaintext:", tdata->plaintext.data,
> +		      plaintext_len);
> +	/* Create mlx5 operation */
> +	retval = create_wireless_algo_cipher_operation(tdata-
> >cipher_iv.data,
> +				tdata->cipher_iv.len, (tdata->cipher.len_bits),
> +				(tdata->cipher.offset_bits));
> +	if (retval < 0)
> +		return retval;
> +	if (global_api_test_type == CRYPTODEV_RAW_API_TEST)
> +		process_sym_raw_dp_op(ts_params->valid_devs[0], 0,
> +				ut_params->op, 1, 0, 1, tdata->cipher_iv.len);
> +	else
> +		ut_params->op = process_crypto_request(ts_params-
> >valid_devs[0],
> +						ut_params->op);
> +	TEST_ASSERT_NOT_NULL(ut_params->op, "failed to retrieve obuf");
> +	ut_params->obuf = ut_params->op->sym->m_dst;
> +	if (ut_params->obuf)
> +		ciphertext = rte_pktmbuf_read(ut_params->obuf, 0,
> +				plaintext_len, buffer);
> +	else
> +		ciphertext = rte_pktmbuf_read(ut_params->ibuf,
> +				tdata->cipher.offset_bits,
> +				plaintext_len, buffer);
> +	if (unlikely(ciphertext == NULL))
> +		return -ENOMEM;
> +	/* Validate obuf */
> +	debug_hexdump(stdout, "ciphertext:", ciphertext, plaintext_len);
> +	reference_ciphertext = tdata->ciphertext.data +
> +				tdata->cipher.offset_bits;
> +	/* Validate obuf */
> +	TEST_ASSERT_BUFFERS_ARE_EQUAL_BIT(
> +		ciphertext,
> +		reference_ciphertext,
> +		tdata->validCipherLenInBits.len,
> +		"MLX5 Ciphertext data not as expected");
> +	/* Validate stats */
> +	TEST_ASSERT_SUCCESS(rte_cryptodev_stats_get(ts_params-
> >valid_devs[0],
> +			&stats),
> +		"rte_cryptodev_stats_get failed");
> +	TEST_ASSERT((stats.enqueued_count == 1),
> +		"rte_cryptodev_stats_get returned unexpected enqueued
> stat");
> +	TEST_ASSERT((stats.dequeued_count == 1),
> +		   "rte_cryptodev_stats_get returned unexpected dequeued
> stat");
> +	TEST_ASSERT((stats.enqueue_err_count == 0),
> +		   "rte_cryptodev_stats_get returned error enqueued stat");
> +	TEST_ASSERT((stats.dequeue_err_count == 0),
> +		   "rte_cryptodev_stats_get returned error dequeued stat");
> +	return 0;
> +}
> +
> +static int
> +test_mlx5_decryption_sgl(const struct mlx5_test_data *tdata)
> +{
> +	struct crypto_testsuite_params *ts_params = &testsuite_params;
> +	struct crypto_unittest_params *ut_params = &unittest_params;
> +	struct rte_cryptodev_sym_capability_idx cap_idx;
> +	struct rte_cryptodev_stats stats;
> +	struct rte_cryptodev_info dev_info;
> +	uint8_t *ciphertext;
> +	const uint8_t *plaintext;
> +	const uint8_t *reference_plaintext;
> +	uint8_t buffer[10000];
> +	uint64_t feat_flags;
> +	unsigned int ciphertext_pad_len;
> +	unsigned int ciphertext_len;
> +	int retval;
> +
> +	rte_cryptodev_info_get(ts_params->valid_devs[0], &dev_info);
> +	feat_flags = dev_info.feature_flags;
> +	if ((global_api_test_type == CRYPTODEV_RAW_API_TEST) &&
> +			(!(feat_flags & RTE_CRYPTODEV_FF_SYM_RAW_DP)))
> {
> +		printf("Device doesn't support RAW data-path APIs.\n");
> +		return -ENOTSUP;
> +	}
> +	if (gbl_action_type == RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO)
> +		return -ENOTSUP;
> +	/* Verify the capabilities */
> +	cap_idx.type = RTE_CRYPTO_SYM_XFORM_CIPHER;
> +	cap_idx.algo.cipher = RTE_CRYPTO_CIPHER_AES_XTS;
> +	if (rte_cryptodev_sym_capability_get(ts_params->valid_devs[0],
> +			&cap_idx) == NULL)
> +		return -ENOTSUP;
> +	/* Create mlx5 session */
> +	retval = create_wireless_algo_cipher_session(ts_params-
> >valid_devs[0],
> +					RTE_CRYPTO_CIPHER_OP_DECRYPT,
> +					RTE_CRYPTO_CIPHER_AES_XTS,
> +					tdata->key.data, tdata->key.len,
> +					tdata->cipher_iv.len, 0);
> +	if (retval < 0)
> +		return retval;
> +	ut_params->ibuf = rte_pktmbuf_alloc(ts_params->mbuf_pool);
> +	if (unlikely(ut_params->ibuf == NULL))
> +		return -ENOMEM;
> +	/* Clear mbuf payload */
> +	memset(rte_pktmbuf_mtod(ut_params->ibuf, uint8_t *), 0,
> +	       rte_pktmbuf_tailroom(ut_params->ibuf));
> +	ciphertext_len = ceil_byte_length(tdata->ciphertext.len);
> +	/* Append data which is padded to a multiple */
> +	/* of the algorithms block size */
> +	ciphertext_pad_len = RTE_ALIGN_CEIL(ciphertext_len, 8);
> +	ciphertext = (uint8_t *)rte_pktmbuf_append(ut_params->ibuf,
> +				ciphertext_pad_len);
> +	if (unlikely(ciphertext == NULL))
> +		return -ENOMEM;
> +	ut_params->ibuf = create_segmented_mbuf(ts_params->mbuf_pool,
> +			ciphertext_pad_len, 8, 0);
> +	if (unlikely(ut_params->ibuf == NULL))
> +		return -ENOMEM;
> +	pktmbuf_write(ut_params->ibuf, 0, ciphertext_len,
> +			tdata->ciphertext.data);
> +	memcpy(ciphertext, tdata->ciphertext.data, ciphertext_len);
> +	if (unlikely(ciphertext == NULL))
> +		return -ENOMEM;
> +	debug_hexdump(stdout, "ciphertext:", ciphertext, ciphertext_len);
> +	/* Create mlx5 operation */
> +	retval = create_wireless_algo_cipher_operation(tdata-
> >cipher_iv.data,
> +				tdata->cipher_iv.len, (tdata->cipher.len_bits),
> +				(tdata->cipher.offset_bits));
> +	if (retval < 0)
> +		return retval;
> +	if (global_api_test_type == CRYPTODEV_RAW_API_TEST)
> +		process_sym_raw_dp_op(ts_params->valid_devs[0], 0,
> +				ut_params->op, 1, 0, 1, 0);
> +	else
> +		ut_params->op = process_crypto_request(ts_params-
> >valid_devs[0],
> +						ut_params->op);
> +	TEST_ASSERT_NOT_NULL(ut_params->op, "failed to retrieve obuf");
> +	ut_params->obuf = ut_params->op->sym->m_dst;
> +	if (ut_params->obuf)
> +		plaintext = rte_pktmbuf_read(ut_params->obuf, 0,
> +				ciphertext_len, buffer);
> +	else
> +		plaintext = rte_pktmbuf_read(ut_params->ibuf,
> +				tdata->cipher.offset_bits,
> +				ciphertext_len, buffer);
> +	debug_hexdump(stdout, "plaintext:", plaintext, ciphertext_len);
> +	reference_plaintext = tdata->plaintext.data +
> +				(tdata->cipher.offset_bits);
> +	/* Validate obuf */
> +	TEST_ASSERT_BUFFERS_ARE_EQUAL_BIT(
> +		plaintext,
> +		reference_plaintext,
> +		tdata->validCipherLenInBits.len,
> +		"MLX5 Plaintext data not as expected");
> +	/* Validate stats */
> +	TEST_ASSERT_SUCCESS(rte_cryptodev_stats_get(ts_params-
> >valid_devs[0],
> +			&stats),
> +		"rte_cryptodev_stats_get failed");
> +	TEST_ASSERT((stats.enqueued_count == 1),
> +		"rte_cryptodev_stats_get returned unexpected enqueued
> stat");
> +	TEST_ASSERT((stats.dequeued_count == 1),
> +		   "rte_cryptodev_stats_get returned unexpected dequeued
> stat");
> +	TEST_ASSERT((stats.enqueue_err_count == 0),
> +		   "rte_cryptodev_stats_get returned error enqueued stat");
> +	TEST_ASSERT((stats.dequeue_err_count == 0),
> +		   "rte_cryptodev_stats_get returned error dequeued stat");
> +	return 0;
> +}
> +
> +
>  static int
>  test_kasumi_encryption_test_case_1(void)
>  {
> @@ -7345,6 +7558,54 @@ test_mlx5_decryption_test_case_4(void)
>  	return test_mlx5_decryption(&mlx5_test_case_cipher_aes_xts_4);
>  }
> 
> +static int
> +test_mlx5_encryption_test_case_1_sgl(void)
> +{
> +	return
> test_mlx5_encryption_sgl(&mlx5_test_case_cipher_aes_xts_1);
> +}
> +
> +static int
> +test_mlx5_encryption_test_case_2_sgl(void)
> +{
> +	return
> test_mlx5_encryption_sgl(&mlx5_test_case_cipher_aes_xts_2);
> +}
> +
> +static int
> +test_mlx5_encryption_test_case_3_sgl(void)
> +{
> +	return
> test_mlx5_encryption_sgl(&mlx5_test_case_cipher_aes_xts_3);
> +}
> +
> +static int
> +test_mlx5_encryption_test_case_4_sgl(void)
> +{
> +	return
> test_mlx5_encryption_sgl(&mlx5_test_case_cipher_aes_xts_4);
> +}
> +
> +static int
> +test_mlx5_decryption_test_case_1_sgl(void)
> +{
> +	return
> test_mlx5_decryption_sgl(&mlx5_test_case_cipher_aes_xts_1);
> +}
> +
> +static int
> +test_mlx5_decryption_test_case_2_sgl(void)
> +{
> +	return
> test_mlx5_decryption_sgl(&mlx5_test_case_cipher_aes_xts_2);
> +}
> +
> +static int
> +test_mlx5_decryption_test_case_3_sgl(void)
> +{
> +	return
> test_mlx5_decryption_sgl(&mlx5_test_case_cipher_aes_xts_3);
> +}
> +
> +static int
> +test_mlx5_decryption_test_case_4_sgl(void)
> +{
> +	return
> test_mlx5_decryption_sgl(&mlx5_test_case_cipher_aes_xts_4);
> +}
> +
>  static int
>  test_mixed_check_if_unsupported(const struct
> mixed_cipher_auth_test_data *tdata)
>  {
> @@ -14747,6 +15008,22 @@ static struct unit_test_suite
> cryptodev_mlx5_testsuite  = {
>  			test_mlx5_encryption_test_case_3),
>  		TEST_CASE_ST(ut_setup, ut_teardown,
>  			test_mlx5_encryption_test_case_4),
> +		TEST_CASE_ST(ut_setup, ut_teardown,
> +			test_mlx5_decryption_test_case_1_sgl),
> +		TEST_CASE_ST(ut_setup, ut_teardown,
> +			test_mlx5_decryption_test_case_2_sgl),
> +		TEST_CASE_ST(ut_setup, ut_teardown,
> +			test_mlx5_decryption_test_case_3_sgl),
> +		TEST_CASE_ST(ut_setup, ut_teardown,
> +			test_mlx5_decryption_test_case_4_sgl),
> +		TEST_CASE_ST(ut_setup, ut_teardown,
> +			test_mlx5_encryption_test_case_1_sgl),
> +		TEST_CASE_ST(ut_setup, ut_teardown,
> +			test_mlx5_encryption_test_case_2_sgl),
> +		TEST_CASE_ST(ut_setup, ut_teardown,
> +			test_mlx5_encryption_test_case_3_sgl),
> +		TEST_CASE_ST(ut_setup, ut_teardown,
> +			test_mlx5_encryption_test_case_4_sgl),
>  		TEST_CASES_END()
>  	}
>  };
> diff --git a/app/test/test_cryptodev_mlx5_test_vectors.h
> b/app/test/test_cryptodev_mlx5_test_vectors.h
> index 2a05aa4626..db7e5fc744 100644
> --- a/app/test/test_cryptodev_mlx5_test_vectors.h
> +++ b/app/test/test_cryptodev_mlx5_test_vectors.h
> @@ -2496,7 +2496,4 @@ static struct mlx5_test_data
> mlx5_test_case_cipher_aes_xts_4 = {
>  	.dataunit_len = 0
>  };
> 
> -
> -
> -
>  #endif /*TEST_CRYPTODEV_MLX5_TEST_VECTORS_H_*/
> --
> 2.27.0

Re: [dpdk-dev] [EXT] [PATCH v5 14/15] test/crypto: add mlx5 crypto driver

[Akhil Goyal]

> In order to test the new mlx5 crypto PMD, the driver is added to the
> crypto test application.
> 
> Added mlx5 vectors that include 4 testing vectors with length 512 and
> 4096.
> 
> Added mlx5 encryption function and decryption function that will both
> use the mlx5 vectors that will also set the dataunit_len and use a
> wrapped key.
> 
> The added tests will test both data integrity and correct stat values.
> 
> Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
> Acked-by: Matan Azrad <matan@nvidia.com>
> ---
>  app/test/meson.build                        |    1 +
>  app/test/test_cryptodev.c                   |  334 ++-
>  app/test/test_cryptodev.h                   |    3 +-
>  app/test/test_cryptodev_mlx5_test_vectors.h | 2502 +++++++++++++++++++
>  4 files changed, 2823 insertions(+), 17 deletions(-)
>  create mode 100644 app/test/test_cryptodev_mlx5_test_vectors.h
> 
> diff --git a/app/test/meson.build b/app/test/meson.build
> index 0a5f425578..c36655888e 100644
> --- a/app/test/meson.build
> +++ b/app/test/meson.build
> @@ -313,6 +313,7 @@ driver_test_names = [
>          'cryptodev_aesni_gcm_autotest',
>          'cryptodev_dpaa_sec_autotest',
>          'cryptodev_dpaa2_sec_autotest',
> +        'cryptodev_mlx5_autotest',
>          'cryptodev_null_autotest',
>          'cryptodev_octeontx2_autotest',
>          'cryptodev_openssl_autotest',
> diff --git a/app/test/test_cryptodev.c b/app/test/test_cryptodev.c
> index 39db52b17a..8dbe324b81 100644
> --- a/app/test/test_cryptodev.c
> +++ b/app/test/test_cryptodev.c
> @@ -35,6 +35,7 @@
>  #include "test_cryptodev_hash_test_vectors.h"
>  #include "test_cryptodev_kasumi_test_vectors.h"
>  #include "test_cryptodev_kasumi_hash_test_vectors.h"
> +#include "test_cryptodev_mlx5_test_vectors.h"
>  #include "test_cryptodev_snow3g_test_vectors.h"
>  #include "test_cryptodev_snow3g_hash_test_vectors.h"
>  #include "test_cryptodev_zuc_test_vectors.h"
> @@ -1267,6 +1268,39 @@ negative_hmac_sha1_testsuite_setup(void)
>  	return 0;
>  }
> 
> +static int
> +mlx5_testsuite_setup(void)

Please see how other drivers are enabled in this test app.
We no longer allow PMD specific testsuite.
Please see the cn9k/cn10k PMD which recently got merged.
static int
test_cryptodev_cn9k(void)
{
        return run_cryptodev_testsuite(RTE_STR(CRYPTODEV_NAME_CN9K_PMD));
}

The cases which are not supported by mlx5 crypto PMD will get automatically skipped
If all capabilities and feature flags are set properly.

The cases should be generic enough to be used by other PMDs as well.
@Thomas Monjalon: This patchset is adding PMD specific test cases again
which we removed in past 3-4 releases.

Regards,
Akhil

Re: [dpdk-dev] [EXT] [PATCH v5 14/15] test/crypto: add mlx5 crypto driver

[Thomas Monjalon]

06/07/2021 10:47, Akhil Goyal:
> > +static int
> > +mlx5_testsuite_setup(void)
> 
> Please see how other drivers are enabled in this test app.
> We no longer allow PMD specific testsuite.
> Please see the cn9k/cn10k PMD which recently got merged.
> static int
> test_cryptodev_cn9k(void)
> {
>         return run_cryptodev_testsuite(RTE_STR(CRYPTODEV_NAME_CN9K_PMD));
> }
> 
> The cases which are not supported by mlx5 crypto PMD will get automatically skipped
> If all capabilities and feature flags are set properly.
> 
> The cases should be generic enough to be used by other PMDs as well.
> @Thomas Monjalon: This patchset is adding PMD specific test cases again
> which we removed in past 3-4 releases.

Good catch Akhil. Indeed tests should be generic.

Re: [dpdk-dev] [EXT] [PATCH v5 15/15] test/crypto: add mlx5 multi segment tests

[Shiri Kuzin]

> -----Original Message-----
> From: Akhil Goyal <gakhil@marvell.com>
> Sent: Tuesday, July 6, 2021 10:48 AM
> To: Shiri Kuzin <shirik@nvidia.com>; dev@dpdk.org
> Cc: Matan Azrad <matan@nvidia.com>; Suanming Mou
> <suanmingm@nvidia.com>
> Subject: RE: [EXT] [PATCH v5 15/15] test/crypto: add mlx5 multi segment
> tests
> 
> > The crypto mlx5 driver supports multi segment encryption and
> > decryption operations.
> >
> > Added mlx5 multi segment encryption function and multi segment
> > decryption function that will both use the mlx5 vectors.
> >
> > The added tests will test both data integrity and correct stat values.
> >
> > Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
> > Acked-by: Matan Azrad <matan@nvidia.com>
> > ---
> >  app/test/test_cryptodev.c                   | 277 ++++++++++++++++++++
> >  app/test/test_cryptodev_mlx5_test_vectors.h |   3 -
> >  2 files changed, 277 insertions(+), 3 deletions(-)
> >
> > diff --git a/app/test/test_cryptodev.c b/app/test/test_cryptodev.c
> > index 8dbe324b81..4d27a9444c 100644
> > --- a/app/test/test_cryptodev.c
> > +++ b/app/test/test_cryptodev.c
> > @@ -6681,6 +6681,219 @@ test_mlx5_decryption(const struct
> > mlx5_test_data *tdata)
> >  	return 0;
> >  }
> >
> > +static int
> > +test_mlx5_encryption_sgl(const struct mlx5_test_data *tdata)
> 
> The test cases cannot be specific to one particular device.
> Hence the name test_mlx5_xxxx cannot be accepted.
> 
> Moreover is it not possible to add aes-xts in test_blockcipher_one_case()
> and add only the test vectors of aes-xts?
> And probably, you do not need a new file for test vectors, XTS is variant of
> AES, Hence the vectors can be part of "test_cryptodev_aes_test_vectors.h".
> 
> I don't think there is need for a separate function for XTS right now.
> The current function test_blockcipher_one_case() covers all capability checks
> And feature flag checks so that the test is skipped for the devices which do
> not Support a specific case.


Hi Akhil, 
Thank you for taking the time to review this.
I would like to change the test according to your comments as follows:
Use the existing AES-XTS testing function.
Since we require using data-unit and wrapped key I will add two new fields to the blockcipher_test_data struct:
XTS data unit- to be used by drivers that support data unit.
Wrapped key - to state whether the key is wrapped or not.
I will add vectors which will include the needed data unit and wrapped key to the "test_cryptodev_aes_test_vectors.h", which will be generic and could be used by other PMDs as well.
Will that solution align with your comments? I believe this also addresses the comments you mentioned it the other email.
Please let me know what you think.

Regards,
Shiri.

Re: [dpdk-dev] [EXT] [PATCH v5 15/15] test/crypto: add mlx5 multi segment tests

[Akhil Goyal]

> > > The crypto mlx5 driver supports multi segment encryption and
> > > decryption operations.
> > >
> > > Added mlx5 multi segment encryption function and multi segment
> > > decryption function that will both use the mlx5 vectors.
> > >
> > > The added tests will test both data integrity and correct stat values.
> > >
> > > Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
> > > Acked-by: Matan Azrad <matan@nvidia.com>
> > > ---
> > >  app/test/test_cryptodev.c                   | 277 ++++++++++++++++++++
> > >  app/test/test_cryptodev_mlx5_test_vectors.h |   3 -
> > >  2 files changed, 277 insertions(+), 3 deletions(-)
> > >
> > > diff --git a/app/test/test_cryptodev.c b/app/test/test_cryptodev.c
> > > index 8dbe324b81..4d27a9444c 100644
> > > --- a/app/test/test_cryptodev.c
> > > +++ b/app/test/test_cryptodev.c
> > > @@ -6681,6 +6681,219 @@ test_mlx5_decryption(const struct
> > > mlx5_test_data *tdata)
> > >  	return 0;
> > >  }
> > >
> > > +static int
> > > +test_mlx5_encryption_sgl(const struct mlx5_test_data *tdata)
> >
> > The test cases cannot be specific to one particular device.
> > Hence the name test_mlx5_xxxx cannot be accepted.
> >
> > Moreover is it not possible to add aes-xts in test_blockcipher_one_case()
> > and add only the test vectors of aes-xts?
> > And probably, you do not need a new file for test vectors, XTS is variant of
> > AES, Hence the vectors can be part of "test_cryptodev_aes_test_vectors.h".
> >
> > I don't think there is need for a separate function for XTS right now.
> > The current function test_blockcipher_one_case() covers all capability
> checks
> > And feature flag checks so that the test is skipped for the devices which do
> > not Support a specific case.
> 
> 
> Hi Akhil,
> Thank you for taking the time to review this.
> I would like to change the test according to your comments as follows:
> Use the existing AES-XTS testing function.
> Since we require using data-unit and wrapped key I will add two new fields to
> the blockcipher_test_data struct:
> XTS data unit- to be used by drivers that support data unit.
> Wrapped key - to state whether the key is wrapped or not.
> I will add vectors which will include the needed data unit and wrapped key to
> the "test_cryptodev_aes_test_vectors.h", which will be generic and could be
> used by other PMDs as well.
> Will that solution align with your comments? I believe this also addresses the
> comments you mentioned it the other email.
> Please let me know what you think.
> 
Yes this looks good.

You should have 2 separate patches
1. for adding mlx5 crypto PMD in test app. This should be minimal similar to cnxk PMDs.
2. adding aes-xts algo support in test app.

I believe SGL is already supported in block cipher, you wont need that one separately.

Re: [dpdk-dev] [EXT] [PATCH v5 15/15] test/crypto: add mlx5 multi segment tests

[Shiri Kuzin]

> -----Original Message-----
> From: Akhil Goyal <gakhil@marvell.com>
> Sent: Tuesday, July 6, 2021 12:38 PM
> To: Shiri Kuzin <shirik@nvidia.com>; dev@dpdk.org
> Cc: Matan Azrad <matan@nvidia.com>; Suanming Mou
> <suanmingm@nvidia.com>
> Subject: RE: [EXT] [PATCH v5 15/15] test/crypto: add mlx5 multi segment
> tests
> 
> > > > The crypto mlx5 driver supports multi segment encryption and
> > > > decryption operations.
> > > >
> > > > Added mlx5 multi segment encryption function and multi segment
> > > > decryption function that will both use the mlx5 vectors.
> > > >
> > > > The added tests will test both data integrity and correct stat values.
> > > >
> > > > Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
> > > > Acked-by: Matan Azrad <matan@nvidia.com>
> > > > ---
> > > >  app/test/test_cryptodev.c                   | 277 ++++++++++++++++++++
> > > >  app/test/test_cryptodev_mlx5_test_vectors.h |   3 -
> > > >  2 files changed, 277 insertions(+), 3 deletions(-)
> > > >
> > > > diff --git a/app/test/test_cryptodev.c b/app/test/test_cryptodev.c
> > > > index 8dbe324b81..4d27a9444c 100644
> > > > --- a/app/test/test_cryptodev.c
> > > > +++ b/app/test/test_cryptodev.c
> > > > @@ -6681,6 +6681,219 @@ test_mlx5_decryption(const struct
> > > > mlx5_test_data *tdata)
> > > >  	return 0;
> > > >  }
> > > >
> > > > +static int
> > > > +test_mlx5_encryption_sgl(const struct mlx5_test_data *tdata)
> > >
> > > The test cases cannot be specific to one particular device.
> > > Hence the name test_mlx5_xxxx cannot be accepted.
> > >
> > > Moreover is it not possible to add aes-xts in
> > > test_blockcipher_one_case() and add only the test vectors of aes-xts?
> > > And probably, you do not need a new file for test vectors, XTS is
> > > variant of AES, Hence the vectors can be part of
> "test_cryptodev_aes_test_vectors.h".
> > >
> > > I don't think there is need for a separate function for XTS right now.
> > > The current function test_blockcipher_one_case() covers all
> > > capability
> > checks
> > > And feature flag checks so that the test is skipped for the devices
> > > which do not Support a specific case.
> >
> >
> > Hi Akhil,
> > Thank you for taking the time to review this.
> > I would like to change the test according to your comments as follows:
> > Use the existing AES-XTS testing function.
> > Since we require using data-unit and wrapped key I will add two new
> > fields to the blockcipher_test_data struct:
> > XTS data unit- to be used by drivers that support data unit.
> > Wrapped key - to state whether the key is wrapped or not.
> > I will add vectors which will include the needed data unit and wrapped
> > key to the "test_cryptodev_aes_test_vectors.h", which will be generic
> > and could be used by other PMDs as well.
> > Will that solution align with your comments? I believe this also
> > addresses the comments you mentioned it the other email.
> > Please let me know what you think.
> >
> Yes this looks good.
> 
> You should have 2 separate patches
> 1. for adding mlx5 crypto PMD in test app. This should be minimal similar to
> cnxk PMDs.
> 2. adding aes-xts algo support in test app.
> 
> I believe SGL is already supported in block cipher, you wont need that one
> separately.

Thank you for the quick reply.
 Working on these changes, I will send an updated version once it is ready.

Regards,
Shiri.

[dpdk-dev] [PATCH v6 00/15] drivers: introduce mlx5 crypto PMD

[Shiri Kuzin]

Add a new PMD for Nvidia devices- crypto PMD.
The crypto PMD will be supported on Nvidia ConnectX6.
The crypto PMD will add the support of encryption and decryption using
the AES-XTS symmetric algorithm.
The crypto PMD requires rdma-core and uses mlx5 DevX.

v2:
Add data-path part.

v3:
Rebase.

v4:
- Rebase + Address the following Akhil comments:
- Set HW feature flag in the capability patch.
- Fix mp object release in session clear.
- Some spelling and word missing in doc.
- Squash data-unit adjustment to the session operations commit.
- Wording: device argument -> devarg.

v5: 
- Add mlx5 crypto tests into test library.
- Update documentation according to Akhil comments.
- Fix memory region management.
- Fix multi segment case in data-path code.
- Split documentation to the correct commits according to Akhil comments.
- Rebase to new version.
- Change license to Nvidia license.

v6:
- Rebase to new version.
- Address David's comment and update log accordingly. 
- Update testing app commits as suggested by Akhil.



Shiri Kuzin (10):
  drivers: introduce mlx5 crypto PMD
  crypto/mlx5: add DEK object management
  crypto/mlx5: add session operations
  crypto/mlx5: add basic operations
  crypto/mlx5: add queue pairs operations
  crypto/mlx5: add dev stop and start operations
  crypto/mlx5: add memory region management
  crypto/mlx5: create login object using DevX
  test/crypto: add mlx5 crypto driver
  test/crypto: add data-unit and wrapped vectors

Suanming Mou (5):
  crypto/mlx5: add keytag devarg
  crypto/mlx5: add maximum segments devarg
  crypto/mlx5: add WQE set initialization
  crypto/mlx5: add enqueue and dequeue operations
  crypto/mlx5: add statistic get and reset operations

 MAINTAINERS                                |    4 +
 app/test/test_cryptodev.c                  |    7 +
 app/test/test_cryptodev.h                  |    3 +-
 app/test/test_cryptodev_aes_test_vectors.h | 1340 ++++++++++++++++++++
 app/test/test_cryptodev_blockcipher.c      |   10 +-
 app/test/test_cryptodev_blockcipher.h      |    2 +
 doc/guides/cryptodevs/features/mlx5.ini    |   37 +
 doc/guides/cryptodevs/index.rst            |    1 +
 doc/guides/cryptodevs/mlx5.rst             |  153 +++
 doc/guides/rel_notes/release_21_08.rst     |    5 +
 drivers/common/mlx5/mlx5_common.h          |    1 +
 drivers/common/mlx5/mlx5_common_pci.c      |   14 +
 drivers/common/mlx5/mlx5_common_pci.h      |   21 +-
 drivers/crypto/meson.build                 |    1 +
 drivers/crypto/mlx5/meson.build            |   27 +
 drivers/crypto/mlx5/mlx5_crypto.c          | 1177 +++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto.h          |   91 ++
 drivers/crypto/mlx5/mlx5_crypto_dek.c      |  136 ++
 drivers/crypto/mlx5/mlx5_crypto_utils.h    |   19 +
 drivers/crypto/mlx5/version.map            |    3 +
 20 files changed, 3040 insertions(+), 12 deletions(-)
 create mode 100644 doc/guides/cryptodevs/features/mlx5.ini
 create mode 100644 doc/guides/cryptodevs/mlx5.rst
 create mode 100644 drivers/crypto/mlx5/meson.build
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto.c
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto.h
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_dek.c
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_utils.h
 create mode 100644 drivers/crypto/mlx5/version.map

-- 
2.27.0

[dpdk-dev] [PATCH v6 01/15] drivers: introduce mlx5 crypto PMD

[Shiri Kuzin]

Add a new PMD for Mellanox devices- crypto PMD.

The crypto PMD will be supported starting Nvidia ConnectX6 and
BlueField2.

The crypto PMD will add the support of encryption and decryption using
the AES-XTS symmetric algorithm.

The crypto PMD requires rdma-core and uses mlx5 DevX.

This patch adds the PCI probing, basic functions, build files and
log utility.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 MAINTAINERS                             |   4 +
 doc/guides/cryptodevs/features/mlx5.ini |  27 +++
 doc/guides/cryptodevs/index.rst         |   1 +
 doc/guides/cryptodevs/mlx5.rst          |  63 ++++++
 doc/guides/rel_notes/release_21_08.rst  |   5 +
 drivers/common/mlx5/mlx5_common.h       |   1 +
 drivers/common/mlx5/mlx5_common_pci.c   |  14 ++
 drivers/common/mlx5/mlx5_common_pci.h   |  21 +-
 drivers/crypto/meson.build              |   1 +
 drivers/crypto/mlx5/meson.build         |  26 +++
 drivers/crypto/mlx5/mlx5_crypto.c       | 275 ++++++++++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto_utils.h |  19 ++
 drivers/crypto/mlx5/version.map         |   3 +
 13 files changed, 450 insertions(+), 10 deletions(-)
 create mode 100644 doc/guides/cryptodevs/features/mlx5.ini
 create mode 100644 doc/guides/cryptodevs/mlx5.rst
 create mode 100644 drivers/crypto/mlx5/meson.build
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto.c
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_utils.h
 create mode 100644 drivers/crypto/mlx5/version.map

diff --git a/MAINTAINERS b/MAINTAINERS
index f64fce84ab..0cf9e91330 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -1088,6 +1088,10 @@ F: drivers/crypto/octeontx2/
 F: doc/guides/cryptodevs/octeontx2.rst
 F: doc/guides/cryptodevs/features/octeontx2.ini
 
+Mellanox mlx5
+M: Matan Azrad <matan@nvidia.com>
+F: drivers/crypto/mlx5/
+
 Null Crypto
 M: Declan Doherty <declan.doherty@intel.com>
 F: drivers/crypto/null/
diff --git a/doc/guides/cryptodevs/features/mlx5.ini b/doc/guides/cryptodevs/features/mlx5.ini
new file mode 100644
index 0000000000..ceadd967b6
--- /dev/null
+++ b/doc/guides/cryptodevs/features/mlx5.ini
@@ -0,0 +1,27 @@
+;
+; Features of a mlx5 crypto driver.
+;
+; Refer to default.ini for the full list of available PMD features.
+;
+[Features]
+HW Accelerated         = Y
+
+;
+; Supported crypto algorithms of a mlx5 crypto driver.
+;
+[Cipher]
+
+;
+; Supported authentication algorithms of a mlx5 crypto driver.
+;
+[Auth]
+
+;
+; Supported AEAD algorithms of a mlx5 crypto driver.
+;
+[AEAD]
+
+;
+; Supported Asymmetric algorithms of a mlx5 crypto driver.
+;
+[Asymmetric]
diff --git a/doc/guides/cryptodevs/index.rst b/doc/guides/cryptodevs/index.rst
index 279f56a002..747409c441 100644
--- a/doc/guides/cryptodevs/index.rst
+++ b/doc/guides/cryptodevs/index.rst
@@ -22,6 +22,7 @@ Crypto Device Drivers
     octeontx
     octeontx2
     openssl
+    mlx5
     mvsam
     nitrox
     null
diff --git a/doc/guides/cryptodevs/mlx5.rst b/doc/guides/cryptodevs/mlx5.rst
new file mode 100644
index 0000000000..05a0a449e2
--- /dev/null
+++ b/doc/guides/cryptodevs/mlx5.rst
@@ -0,0 +1,63 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+   Copyright (c) 2021 NVIDIA Corporation & Affiliates
+
+.. include:: <isonum.txt>
+
+MLX5 Crypto Driver
+==================
+
+The MLX5 crypto driver library
+(**librte_crypto_mlx5**) provides support for **Mellanox ConnectX-6**
+family adapters.
+
+Overview
+--------
+
+The device can provide disk encryption services, allowing data encryption
+and decryption towards a disk. Having all encryption/decryption
+operations done in a single device can reduce cost and overheads of the related
+FIPS certification, as ConnectX-6 is FIPS 140-2 level-2 ready.
+The encryption cipher is AES-XTS of 256/512 bit key size.
+
+MKEY is a memory region object in the hardware, that holds address translation information and
+attributes per memory area. Its ID must be tied to addresses provided to the hardware.
+The encryption operations are performed with MKEY read/write transactions, when
+the MKEY is configured to perform crypto operations.
+
+The encryption does not require text to be aligned to the AES block size (128b).
+
+The PMD uses libibverbs and libmlx5 to access the device firmware or to
+access the hardware components directly.
+There are different levels of objects and bypassing abilities.
+To get the best performances:
+
+- Verbs is a complete high-level generic API.
+- Direct Verbs is a device-specific API.
+- DevX allows to access firmware objects.
+
+Enabling librte_crypto_mlx5 causes DPDK applications to be linked against
+libibverbs.
+
+
+Driver options
+--------------
+
+- ``class`` parameter [string]
+
+  Select the class of the driver that should probe the device.
+  `crypto` for the mlx5 crypto driver.
+
+
+Supported NICs
+--------------
+
+* Mellanox\ |reg| ConnectX\ |reg|-6 200G MCX654106A-HCAT (2x200G)
+
+Prerequisites
+-------------
+
+- Mellanox OFED version: **5.3**
+  see :doc:`../../nics/mlx5` guide for more Mellanox OFED details.
+
+- Compilation can be done also with rdma-core v15+.
+  see :doc:`../../nics/mlx5` guide for more rdma-core details.
diff --git a/doc/guides/rel_notes/release_21_08.rst b/doc/guides/rel_notes/release_21_08.rst
index aca34105c2..d6f98811b5 100644
--- a/doc/guides/rel_notes/release_21_08.rst
+++ b/doc/guides/rel_notes/release_21_08.rst
@@ -98,6 +98,11 @@ New Features
   * Added support for Meter hierarchy.
   * Added devargs options ``allow_duplicate_pattern``.
 
+* **Added support for Nvidia crypto device driver.**
+
+  * Added mlx5 crypto driver to support AES-XTS cipher operations.
+    The first device to support it is ConnectX-6.
+
 
 Removed Items
 -------------
diff --git a/drivers/common/mlx5/mlx5_common.h b/drivers/common/mlx5/mlx5_common.h
index 306f2f1ab7..790b45f8ea 100644
--- a/drivers/common/mlx5/mlx5_common.h
+++ b/drivers/common/mlx5/mlx5_common.h
@@ -214,6 +214,7 @@ enum mlx5_class {
 	MLX5_CLASS_VDPA = RTE_BIT64(1),
 	MLX5_CLASS_REGEX = RTE_BIT64(2),
 	MLX5_CLASS_COMPRESS = RTE_BIT64(3),
+	MLX5_CLASS_CRYPTO = RTE_BIT64(4),
 };
 
 #define MLX5_DBR_SIZE RTE_CACHE_LINE_SIZE
diff --git a/drivers/common/mlx5/mlx5_common_pci.c b/drivers/common/mlx5/mlx5_common_pci.c
index 34747c4e07..5547e62d6b 100644
--- a/drivers/common/mlx5/mlx5_common_pci.c
+++ b/drivers/common/mlx5/mlx5_common_pci.c
@@ -31,6 +31,7 @@ static const struct {
 	{ .name = "net", .driver_class = MLX5_CLASS_NET },
 	{ .name = "regex", .driver_class = MLX5_CLASS_REGEX },
 	{ .name = "compress", .driver_class = MLX5_CLASS_COMPRESS },
+	{ .name = "crypto", .driver_class = MLX5_CLASS_CRYPTO },
 };
 
 static const unsigned int mlx5_class_combinations[] = {
@@ -38,13 +39,26 @@ static const unsigned int mlx5_class_combinations[] = {
 	MLX5_CLASS_VDPA,
 	MLX5_CLASS_REGEX,
 	MLX5_CLASS_COMPRESS,
+	MLX5_CLASS_CRYPTO,
 	MLX5_CLASS_NET | MLX5_CLASS_REGEX,
 	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX,
 	MLX5_CLASS_NET | MLX5_CLASS_COMPRESS,
 	MLX5_CLASS_VDPA | MLX5_CLASS_COMPRESS,
 	MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
+	MLX5_CLASS_NET | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_VDPA | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO,
 	MLX5_CLASS_NET | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
 	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
+	MLX5_CLASS_NET | MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_NET | MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_VDPA | MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_NET | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS |
+	MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS |
+	MLX5_CLASS_CRYPTO,
 	/* New class combination should be added here. */
 };
 
diff --git a/drivers/common/mlx5/mlx5_common_pci.h b/drivers/common/mlx5/mlx5_common_pci.h
index de89bb98bc..cb8d2f5f87 100644
--- a/drivers/common/mlx5/mlx5_common_pci.h
+++ b/drivers/common/mlx5/mlx5_common_pci.h
@@ -9,17 +9,18 @@
  * @file
  *
  * RTE Mellanox PCI Driver Interface
- * Mellanox ConnectX PCI device supports multiple class: net,vdpa,regex and
- * compress devices. This layer enables creating such multiple class of devices
- * on a single PCI device by allowing to bind multiple class specific device
- * driver to attach to mlx5_pci driver.
+ * Mellanox ConnectX PCI device supports multiple class: net,vdpa,regex,compress
+ * and crypto devices. This layer enables creating such multiple class of
+ * devices on a single PCI device by allowing to bind multiple class specific
+ * device driver to attach to mlx5_pci driver.
  *
- * -----------    ------------    -------------    ----------------
- * |   mlx5  |    |   mlx5   |    |   mlx5    |    |     mlx5     |
- * | net pmd |    | vdpa pmd |    | regex pmd |    | compress pmd |
- * -----------    ------------    -------------    ----------------
- *      \              \                    /              /
- *       \              \                  /              /
+ * --------    --------    ---------    ------------    ----------
+ * | mlx5 |    | mlx5 |    | mlx5  |    |   mlx5   |    |  mlx5  |
+ * | net  |    | vdpa |    | regex |    | compress |    | crypto |
+ * | pmd  |    | pmd  |    |  pmd  |    |   pmd    |    |  pmd   |
+ * --------    --------    ---------    ------------    ----------
+ *      \              \         |          /              /
+ *       \              \        |         /              /
  *        \              \_--------------_/              /
  *         \_______________|   mlx5     |_______________/
  *                         | pci common |
diff --git a/drivers/crypto/meson.build b/drivers/crypto/meson.build
index b9fdf9392f..6951607def 100644
--- a/drivers/crypto/meson.build
+++ b/drivers/crypto/meson.build
@@ -15,6 +15,7 @@ drivers = [
         'dpaa_sec',
         'dpaa2_sec',
         'kasumi',
+        'mlx5',
         'mvsam',
         'nitrox',
         'null',
diff --git a/drivers/crypto/mlx5/meson.build b/drivers/crypto/mlx5/meson.build
new file mode 100644
index 0000000000..6fd70bc477
--- /dev/null
+++ b/drivers/crypto/mlx5/meson.build
@@ -0,0 +1,26 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2021 NVIDIA Corporation & Affiliates
+
+if not is_linux
+	build = false
+	reason = 'only supported on Linux'
+	subdir_done()
+endif
+
+fmt_name = 'mlx5_crypto'
+deps += ['common_mlx5', 'eal', 'cryptodev']
+sources = files(
+	'mlx5_crypto.c',
+)
+cflags_options = [
+	'-std=c11',
+	'-Wno-strict-prototypes',
+	'-D_BSD_SOURCE',
+	'-D_DEFAULT_SOURCE',
+	'-D_XOPEN_SOURCE=600',
+]
+foreach option:cflags_options
+	if cc.has_argument(option)
+		cflags += option
+	endif
+endforeach
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
new file mode 100644
index 0000000000..fbe3c21aae
--- /dev/null
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -0,0 +1,275 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2021 NVIDIA Corporation & Affiliates
+ */
+
+#include <rte_malloc.h>
+#include <rte_log.h>
+#include <rte_errno.h>
+#include <rte_pci.h>
+#include <rte_crypto.h>
+#include <rte_cryptodev.h>
+#include <rte_cryptodev_pmd.h>
+
+#include <mlx5_glue.h>
+#include <mlx5_common.h>
+#include <mlx5_common_pci.h>
+#include <mlx5_devx_cmds.h>
+#include <mlx5_common_os.h>
+
+#include "mlx5_crypto_utils.h"
+
+#define MLX5_CRYPTO_DRIVER_NAME mlx5_crypto
+#define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
+
+#define MLX5_CRYPTO_FEATURE_FLAGS \
+	RTE_CRYPTODEV_FF_HW_ACCELERATED
+
+struct mlx5_crypto_priv {
+	TAILQ_ENTRY(mlx5_crypto_priv) next;
+	struct ibv_context *ctx; /* Device context. */
+	struct rte_pci_device *pci_dev;
+	struct rte_cryptodev *crypto_dev;
+	void *uar; /* User Access Region. */
+	uint32_t pdn; /* Protection Domain number. */
+	struct ibv_pd *pd;
+};
+
+TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
+				TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
+static pthread_mutex_t priv_list_lock = PTHREAD_MUTEX_INITIALIZER;
+
+int mlx5_crypto_logtype;
+
+uint8_t mlx5_crypto_driver_id;
+
+static const char mlx5_crypto_drv_name[] = RTE_STR(MLX5_CRYPTO_DRIVER_NAME);
+
+static const struct rte_driver mlx5_drv = {
+	.name = mlx5_crypto_drv_name,
+	.alias = mlx5_crypto_drv_name
+};
+
+static struct cryptodev_driver mlx5_cryptodev_driver;
+
+static struct rte_cryptodev_ops mlx5_crypto_ops = {
+	.dev_configure			= NULL,
+	.dev_start			= NULL,
+	.dev_stop			= NULL,
+	.dev_close			= NULL,
+	.dev_infos_get			= NULL,
+	.stats_get			= NULL,
+	.stats_reset			= NULL,
+	.queue_pair_setup		= NULL,
+	.queue_pair_release		= NULL,
+	.sym_session_get_size		= NULL,
+	.sym_session_configure		= NULL,
+	.sym_session_clear		= NULL,
+	.sym_get_raw_dp_ctx_size	= NULL,
+	.sym_configure_raw_dp_ctx	= NULL,
+};
+
+static void
+mlx5_crypto_hw_global_release(struct mlx5_crypto_priv *priv)
+{
+	if (priv->pd != NULL) {
+		claim_zero(mlx5_glue->dealloc_pd(priv->pd));
+		priv->pd = NULL;
+	}
+	if (priv->uar != NULL) {
+		mlx5_glue->devx_free_uar(priv->uar);
+		priv->uar = NULL;
+	}
+}
+
+static int
+mlx5_crypto_pd_create(struct mlx5_crypto_priv *priv)
+{
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+	struct mlx5dv_obj obj;
+	struct mlx5dv_pd pd_info;
+	int ret;
+
+	priv->pd = mlx5_glue->alloc_pd(priv->ctx);
+	if (priv->pd == NULL) {
+		DRV_LOG(ERR, "Failed to allocate PD.");
+		return errno ? -errno : -ENOMEM;
+	}
+	obj.pd.in = priv->pd;
+	obj.pd.out = &pd_info;
+	ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_PD);
+	if (ret != 0) {
+		DRV_LOG(ERR, "Fail to get PD object info.");
+		mlx5_glue->dealloc_pd(priv->pd);
+		priv->pd = NULL;
+		return -errno;
+	}
+	priv->pdn = pd_info.pdn;
+	return 0;
+#else
+	(void)priv;
+	DRV_LOG(ERR, "Cannot get pdn - no DV support.");
+	return -ENOTSUP;
+#endif /* HAVE_IBV_FLOW_DV_SUPPORT */
+}
+
+static int
+mlx5_crypto_hw_global_prepare(struct mlx5_crypto_priv *priv)
+{
+	if (mlx5_crypto_pd_create(priv) != 0)
+		return -1;
+	priv->uar = mlx5_devx_alloc_uar(priv->ctx, -1);
+	if (priv->uar == NULL || mlx5_os_get_devx_uar_reg_addr(priv->uar) ==
+	    NULL) {
+		rte_errno = errno;
+		claim_zero(mlx5_glue->dealloc_pd(priv->pd));
+		DRV_LOG(ERR, "Failed to allocate UAR.");
+		return -1;
+	}
+	return 0;
+}
+
+/**
+ * DPDK callback to register a PCI device.
+ *
+ * This function spawns crypto device out of a given PCI device.
+ *
+ * @param[in] pci_drv
+ *   PCI driver structure (mlx5_crypto_driver).
+ * @param[in] pci_dev
+ *   PCI device information.
+ *
+ * @return
+ *   0 on success, 1 to skip this driver, a negative errno value otherwise
+ *   and rte_errno is set.
+ */
+static int
+mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
+			struct rte_pci_device *pci_dev)
+{
+	struct ibv_device *ibv;
+	struct rte_cryptodev *crypto_dev;
+	struct ibv_context *ctx;
+	struct mlx5_crypto_priv *priv;
+	struct mlx5_hca_attr attr = { 0 };
+	struct rte_cryptodev_pmd_init_params init_params = {
+		.name = "",
+		.private_data_size = sizeof(struct mlx5_crypto_priv),
+		.socket_id = pci_dev->device.numa_node,
+		.max_nb_queue_pairs =
+				RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
+	};
+	RTE_SET_USED(pci_drv);
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+		DRV_LOG(ERR, "Non-primary process type is not supported.");
+		rte_errno = ENOTSUP;
+		return -rte_errno;
+	}
+	ibv = mlx5_os_get_ibv_device(&pci_dev->addr);
+	if (ibv == NULL) {
+		DRV_LOG(ERR, "No matching IB device for PCI slot "
+			PCI_PRI_FMT ".", pci_dev->addr.domain,
+			pci_dev->addr.bus, pci_dev->addr.devid,
+			pci_dev->addr.function);
+		return -rte_errno;
+	}
+	DRV_LOG(INFO, "PCI information matches for device \"%s\".", ibv->name);
+	ctx = mlx5_glue->dv_open_device(ibv);
+	if (ctx == NULL) {
+		DRV_LOG(ERR, "Failed to open IB device \"%s\".", ibv->name);
+		rte_errno = ENODEV;
+		return -rte_errno;
+	}
+	if (mlx5_devx_cmd_query_hca_attr(ctx, &attr) != 0 ||
+	    attr.crypto == 0 || attr.aes_xts == 0) {
+		DRV_LOG(ERR, "Not enough capabilities to support crypto "
+			"operations, maybe old FW/OFED version?");
+		claim_zero(mlx5_glue->close_device(ctx));
+		rte_errno = ENOTSUP;
+		return -ENOTSUP;
+	}
+	crypto_dev = rte_cryptodev_pmd_create(ibv->name, &pci_dev->device,
+					&init_params);
+	if (crypto_dev == NULL) {
+		DRV_LOG(ERR, "Failed to create device \"%s\".", ibv->name);
+		claim_zero(mlx5_glue->close_device(ctx));
+		return -ENODEV;
+	}
+	DRV_LOG(INFO,
+		"Crypto device %s was created successfully.", ibv->name);
+	crypto_dev->dev_ops = &mlx5_crypto_ops;
+	crypto_dev->dequeue_burst = NULL;
+	crypto_dev->enqueue_burst = NULL;
+	crypto_dev->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
+	crypto_dev->driver_id = mlx5_crypto_driver_id;
+	priv = crypto_dev->data->dev_private;
+	priv->ctx = ctx;
+	priv->pci_dev = pci_dev;
+	priv->crypto_dev = crypto_dev;
+	if (mlx5_crypto_hw_global_prepare(priv) != 0) {
+		rte_cryptodev_pmd_destroy(priv->crypto_dev);
+		claim_zero(mlx5_glue->close_device(priv->ctx));
+		return -1;
+	}
+	pthread_mutex_lock(&priv_list_lock);
+	TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next);
+	pthread_mutex_unlock(&priv_list_lock);
+	return 0;
+}
+
+static int
+mlx5_crypto_pci_remove(struct rte_pci_device *pdev)
+{
+	struct mlx5_crypto_priv *priv = NULL;
+
+	pthread_mutex_lock(&priv_list_lock);
+	TAILQ_FOREACH(priv, &mlx5_crypto_priv_list, next)
+		if (rte_pci_addr_cmp(&priv->pci_dev->addr, &pdev->addr) != 0)
+			break;
+	if (priv)
+		TAILQ_REMOVE(&mlx5_crypto_priv_list, priv, next);
+	pthread_mutex_unlock(&priv_list_lock);
+	if (priv) {
+		mlx5_crypto_hw_global_release(priv);
+		rte_cryptodev_pmd_destroy(priv->crypto_dev);
+		claim_zero(mlx5_glue->close_device(priv->ctx));
+	}
+	return 0;
+}
+
+static const struct rte_pci_id mlx5_crypto_pci_id_map[] = {
+		{
+			RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
+					PCI_DEVICE_ID_MELLANOX_CONNECTX6)
+		},
+		{
+			.vendor_id = 0
+		}
+	};
+
+static struct mlx5_pci_driver mlx5_crypto_driver = {
+	.driver_class = MLX5_CLASS_CRYPTO,
+	.pci_driver = {
+		.driver = {
+			.name = RTE_STR(MLX5_CRYPTO_DRIVER_NAME),
+		},
+		.id_table = mlx5_crypto_pci_id_map,
+		.probe = mlx5_crypto_pci_probe,
+		.remove = mlx5_crypto_pci_remove,
+		.drv_flags = 0,
+	},
+};
+
+RTE_INIT(rte_mlx5_crypto_init)
+{
+	mlx5_common_init();
+	if (mlx5_glue != NULL)
+		mlx5_pci_driver_register(&mlx5_crypto_driver);
+}
+
+RTE_PMD_REGISTER_CRYPTO_DRIVER(mlx5_cryptodev_driver, mlx5_drv,
+			       mlx5_crypto_driver_id);
+
+RTE_LOG_REGISTER_DEFAULT(mlx5_crypto_logtype, NOTICE)
+RTE_PMD_EXPORT_NAME(MLX5_CRYPTO_DRIVER_NAME, __COUNTER__);
+RTE_PMD_REGISTER_PCI_TABLE(MLX5_CRYPTO_DRIVER_NAME, mlx5_crypto_pci_id_map);
+RTE_PMD_REGISTER_KMOD_DEP(MLX5_CRYPTO_DRIVER_NAME, "* ib_uverbs & mlx5_core & mlx5_ib");
diff --git a/drivers/crypto/mlx5/mlx5_crypto_utils.h b/drivers/crypto/mlx5/mlx5_crypto_utils.h
new file mode 100644
index 0000000000..b6c60ca782
--- /dev/null
+++ b/drivers/crypto/mlx5/mlx5_crypto_utils.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2021 NVIDIA Corporation & Affiliates
+ */
+
+#ifndef RTE_PMD_MLX5_CRYPTO_UTILS_H_
+#define RTE_PMD_MLX5_CRYPTO_UTILS_H_
+
+#include <mlx5_common.h>
+
+extern int mlx5_crypto_logtype;
+
+#define MLX5_CRYPTO_LOG_PREFIX "mlx5_crypto"
+/* Generic printf()-like logging macro with automatic line feed. */
+#define DRV_LOG(level, ...) \
+	PMD_DRV_LOG_(level, mlx5_crypto_logtype, MLX5_CRYPTO_LOG_PREFIX, \
+		__VA_ARGS__ PMD_DRV_LOG_STRIP PMD_DRV_LOG_OPAREN, \
+		PMD_DRV_LOG_CPAREN)
+
+#endif /* RTE_PMD_MLX5_CRYPTO_UTILS_H_ */
diff --git a/drivers/crypto/mlx5/version.map b/drivers/crypto/mlx5/version.map
new file mode 100644
index 0000000000..4a76d1d52d
--- /dev/null
+++ b/drivers/crypto/mlx5/version.map
@@ -0,0 +1,3 @@
+DPDK_21 {
+	local: *;
+};
-- 
2.27.0

[dpdk-dev] [PATCH v6 02/15] crypto/mlx5: add DEK object management

[Shiri Kuzin]

A DEK(Data encryption Key) is an mlx5 HW object which represents the
cipher algorithm key.
The DEKs are used during data encryption/decryption operations.

In symmetric algorithms like AES-STS, we use the same DEK for both
encryption and decryption.

Use the mlx5 hash-list tool to manage the DEK objects in the PMD.

Provide the compare, create and destroy functions to manage DEKs in
hash-list and introduce an internal API to setup and unset the DEK
management and to prepare and destroy specific DEK object.

The DEK hash-list will be created in dev_configure routine and
destroyed in dev_close routine.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/meson.build       |   1 +
 drivers/crypto/mlx5/mlx5_crypto.c     |  42 +++++---
 drivers/crypto/mlx5/mlx5_crypto.h     |  51 ++++++++++
 drivers/crypto/mlx5/mlx5_crypto_dek.c | 136 ++++++++++++++++++++++++++
 4 files changed, 214 insertions(+), 16 deletions(-)
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto.h
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_dek.c

diff --git a/drivers/crypto/mlx5/meson.build b/drivers/crypto/mlx5/meson.build
index 6fd70bc477..d55cdbfe6f 100644
--- a/drivers/crypto/mlx5/meson.build
+++ b/drivers/crypto/mlx5/meson.build
@@ -11,6 +11,7 @@ fmt_name = 'mlx5_crypto'
 deps += ['common_mlx5', 'eal', 'cryptodev']
 sources = files(
 	'mlx5_crypto.c',
+	'mlx5_crypto_dek.c',
 )
 cflags_options = [
 	'-std=c11',
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index fbe3c21aae..d2d82c7b15 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -3,12 +3,9 @@
  */
 
 #include <rte_malloc.h>
-#include <rte_log.h>
 #include <rte_errno.h>
+#include <rte_log.h>
 #include <rte_pci.h>
-#include <rte_crypto.h>
-#include <rte_cryptodev.h>
-#include <rte_cryptodev_pmd.h>
 
 #include <mlx5_glue.h>
 #include <mlx5_common.h>
@@ -17,6 +14,7 @@
 #include <mlx5_common_os.h>
 
 #include "mlx5_crypto_utils.h"
+#include "mlx5_crypto.h"
 
 #define MLX5_CRYPTO_DRIVER_NAME mlx5_crypto
 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
@@ -24,16 +22,6 @@
 #define MLX5_CRYPTO_FEATURE_FLAGS \
 	RTE_CRYPTODEV_FF_HW_ACCELERATED
 
-struct mlx5_crypto_priv {
-	TAILQ_ENTRY(mlx5_crypto_priv) next;
-	struct ibv_context *ctx; /* Device context. */
-	struct rte_pci_device *pci_dev;
-	struct rte_cryptodev *crypto_dev;
-	void *uar; /* User Access Region. */
-	uint32_t pdn; /* Protection Domain number. */
-	struct ibv_pd *pd;
-};
-
 TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
 				TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
 static pthread_mutex_t priv_list_lock = PTHREAD_MUTEX_INITIALIZER;
@@ -51,11 +39,33 @@ static const struct rte_driver mlx5_drv = {
 
 static struct cryptodev_driver mlx5_cryptodev_driver;
 
+static int
+mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
+		struct rte_cryptodev_config *config __rte_unused)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+
+	if (mlx5_crypto_dek_setup(priv) != 0) {
+		DRV_LOG(ERR, "Dek hash list creation has failed.");
+		return -ENOMEM;
+	}
+	return 0;
+}
+
+static int
+mlx5_crypto_dev_close(struct rte_cryptodev *dev)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+
+	mlx5_crypto_dek_unset(priv);
+	return 0;
+}
+
 static struct rte_cryptodev_ops mlx5_crypto_ops = {
-	.dev_configure			= NULL,
+	.dev_configure			= mlx5_crypto_dev_configure,
 	.dev_start			= NULL,
 	.dev_stop			= NULL,
-	.dev_close			= NULL,
+	.dev_close			= mlx5_crypto_dev_close,
 	.dev_infos_get			= NULL,
 	.stats_get			= NULL,
 	.stats_reset			= NULL,
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
new file mode 100644
index 0000000000..5a54cb0dca
--- /dev/null
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -0,0 +1,51 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2021 NVIDIA Corporation & Affiliates
+ */
+
+#ifndef MLX5_CRYPTO_H_
+#define MLX5_CRYPTO_H_
+
+#include <stdbool.h>
+
+#include <rte_cryptodev.h>
+#include <rte_cryptodev_pmd.h>
+
+#include <mlx5_common_utils.h>
+
+#define MLX5_CRYPTO_DEK_HTABLE_SZ (1 << 11)
+#define MLX5_CRYPTO_KEY_LENGTH 80
+
+struct mlx5_crypto_priv {
+	TAILQ_ENTRY(mlx5_crypto_priv) next;
+	struct ibv_context *ctx; /* Device context. */
+	struct rte_pci_device *pci_dev;
+	struct rte_cryptodev *crypto_dev;
+	void *uar; /* User Access Region. */
+	uint32_t pdn; /* Protection Domain number. */
+	struct ibv_pd *pd;
+	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
+};
+
+struct mlx5_crypto_dek {
+	struct mlx5_hlist_entry entry; /* Pointer to DEK hash list entry. */
+	struct mlx5_devx_obj *obj; /* Pointer to DEK DevX object. */
+	uint8_t data[MLX5_CRYPTO_KEY_LENGTH]; /* DEK key data. */
+	bool size_is_48; /* Whether the key\data size is 48 bytes or not. */
+} __rte_cache_aligned;
+
+int
+mlx5_crypto_dek_destroy(struct mlx5_crypto_priv *priv,
+			struct mlx5_crypto_dek *dek);
+
+struct mlx5_crypto_dek *
+mlx5_crypto_dek_prepare(struct mlx5_crypto_priv *priv,
+			struct rte_crypto_cipher_xform *cipher);
+
+int
+mlx5_crypto_dek_setup(struct mlx5_crypto_priv *priv);
+
+void
+mlx5_crypto_dek_unset(struct mlx5_crypto_priv *priv);
+
+#endif /* MLX5_CRYPTO_H_ */
+
diff --git a/drivers/crypto/mlx5/mlx5_crypto_dek.c b/drivers/crypto/mlx5/mlx5_crypto_dek.c
new file mode 100644
index 0000000000..e09c20c844
--- /dev/null
+++ b/drivers/crypto/mlx5/mlx5_crypto_dek.c
@@ -0,0 +1,136 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2021 NVIDIA Corporation & Affiliates
+ */
+
+#include <rte_ip.h>
+#include <rte_common.h>
+#include <rte_errno.h>
+#include <rte_log.h>
+
+#include <mlx5_prm.h>
+#include <mlx5_devx_cmds.h>
+
+#include "mlx5_crypto_utils.h"
+#include "mlx5_crypto.h"
+
+struct mlx5_crypto_dek_ctx {
+	struct rte_crypto_cipher_xform *cipher;
+	struct mlx5_crypto_priv *priv;
+};
+
+int
+mlx5_crypto_dek_destroy(struct mlx5_crypto_priv *priv,
+			struct mlx5_crypto_dek *dek)
+{
+	return mlx5_hlist_unregister(priv->dek_hlist, &dek->entry);
+}
+
+struct mlx5_crypto_dek *
+mlx5_crypto_dek_prepare(struct mlx5_crypto_priv *priv,
+			struct rte_crypto_cipher_xform *cipher)
+{
+	struct mlx5_hlist *dek_hlist = priv->dek_hlist;
+	struct mlx5_crypto_dek_ctx dek_ctx = {
+		.cipher = cipher,
+		.priv = priv,
+	};
+	struct rte_crypto_cipher_xform *cipher_ctx = cipher;
+	uint64_t key64 = __rte_raw_cksum(cipher_ctx->key.data,
+					 cipher_ctx->key.length, 0);
+	struct mlx5_hlist_entry *entry = mlx5_hlist_register(dek_hlist,
+							     key64, &dek_ctx);
+
+	return entry == NULL ? NULL :
+			     container_of(entry, struct mlx5_crypto_dek, entry);
+}
+
+static int
+mlx5_crypto_dek_match_cb(struct mlx5_hlist *list __rte_unused,
+			 struct mlx5_hlist_entry *entry,
+			 uint64_t key __rte_unused, void *cb_ctx)
+{
+	struct mlx5_crypto_dek_ctx *ctx = cb_ctx;
+	struct rte_crypto_cipher_xform *cipher_ctx = ctx->cipher;
+	struct mlx5_crypto_dek *dek =
+			container_of(entry, typeof(*dek), entry);
+	uint32_t key_len = dek->size_is_48 ? 48 : 80;
+
+	if (key_len != cipher_ctx->key.length)
+		return -1;
+	return memcmp(cipher_ctx->key.data, dek->data, key_len);
+}
+
+static struct mlx5_hlist_entry *
+mlx5_crypto_dek_create_cb(struct mlx5_hlist *list __rte_unused,
+			  uint64_t key __rte_unused, void *cb_ctx)
+{
+	struct mlx5_crypto_dek_ctx *ctx = cb_ctx;
+	struct rte_crypto_cipher_xform *cipher_ctx = ctx->cipher;
+	struct mlx5_crypto_dek *dek = rte_zmalloc(__func__, sizeof(*dek),
+						  RTE_CACHE_LINE_SIZE);
+	struct mlx5_devx_dek_attr dek_attr = {
+		.pd = ctx->priv->pdn,
+		.key_purpose = MLX5_CRYPTO_KEY_PURPOSE_AES_XTS,
+		.has_keytag = 1,
+	};
+
+	if (dek == NULL) {
+		DRV_LOG(ERR, "Failed to allocate dek memory.");
+		return NULL;
+	}
+	switch (cipher_ctx->key.length) {
+	case 48:
+		dek->size_is_48 = true;
+		dek_attr.key_size = MLX5_CRYPTO_KEY_SIZE_128b;
+		break;
+	case 80:
+		dek->size_is_48 = false;
+		dek_attr.key_size = MLX5_CRYPTO_KEY_SIZE_256b;
+		break;
+	default:
+		DRV_LOG(ERR, "Key size not supported.");
+		return NULL;
+	}
+	memcpy(&dek_attr.key, cipher_ctx->key.data, cipher_ctx->key.length);
+	dek->obj = mlx5_devx_cmd_create_dek_obj(ctx->priv->ctx, &dek_attr);
+	if (dek->obj == NULL) {
+		rte_free(dek);
+		return NULL;
+	}
+	memcpy(&dek->data, cipher_ctx->key.data, cipher_ctx->key.length);
+	return &dek->entry;
+}
+
+static void
+mlx5_crypto_dek_remove_cb(struct mlx5_hlist *list __rte_unused,
+			  struct mlx5_hlist_entry *entry)
+{
+	struct mlx5_crypto_dek *dek =
+		container_of(entry, typeof(*dek), entry);
+
+	claim_zero(mlx5_devx_cmd_destroy(dek->obj));
+	rte_free(dek);
+}
+
+
+int
+mlx5_crypto_dek_setup(struct mlx5_crypto_priv *priv)
+{
+	priv->dek_hlist = mlx5_hlist_create("dek_hlist",
+				 MLX5_CRYPTO_DEK_HTABLE_SZ,
+				 0, MLX5_HLIST_WRITE_MOST |
+				 MLX5_HLIST_DIRECT_KEY,
+				 mlx5_crypto_dek_create_cb,
+				 mlx5_crypto_dek_match_cb,
+				 mlx5_crypto_dek_remove_cb);
+	if (priv->dek_hlist == NULL)
+		return -1;
+	return 0;
+}
+
+void
+mlx5_crypto_dek_unset(struct mlx5_crypto_priv *priv)
+{
+	mlx5_hlist_destroy(priv->dek_hlist);
+	priv->dek_hlist = NULL;
+}
-- 
2.27.0

[dpdk-dev] [PATCH v6 03/15] crypto/mlx5: add session operations

[Shiri Kuzin]

Sessions are used in symmetric transformations in order to prepare
objects and data for packet processing stage.

A mlx5 session includes iv_offset, pointer to mlx5_crypto_dek struct,
bsf_size, bsf_p_type, block size index, encryption_order and encryption
standard.

Implement the next session operations:
        mlx5_crypto_sym_session_get_size- returns the size of the mlx5
	session struct.
	mlx5_crypto_sym_session_configure- prepares the DEK hash-list
	and saves all the session data.
	mlx5_crypto_sym_session_clear - destroys the DEK hash-list.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 doc/guides/cryptodevs/features/mlx5.ini |   5 +
 doc/guides/cryptodevs/mlx5.rst          |  10 ++
 drivers/crypto/mlx5/mlx5_crypto.c       | 172 +++++++++++++++++++++++-
 3 files changed, 182 insertions(+), 5 deletions(-)

diff --git a/doc/guides/cryptodevs/features/mlx5.ini b/doc/guides/cryptodevs/features/mlx5.ini
index ceadd967b6..bd757b5211 100644
--- a/doc/guides/cryptodevs/features/mlx5.ini
+++ b/doc/guides/cryptodevs/features/mlx5.ini
@@ -4,12 +4,17 @@
 ; Refer to default.ini for the full list of available PMD features.
 ;
 [Features]
+Symmetric crypto       = Y
 HW Accelerated         = Y
+Cipher multiple data units = Y
+Cipher wrapped key     = Y
 
 ;
 ; Supported crypto algorithms of a mlx5 crypto driver.
 ;
 [Cipher]
+AES XTS (128)  = Y
+AES XTS (256)  = Y
 
 ;
 ; Supported authentication algorithms of a mlx5 crypto driver.
diff --git a/doc/guides/cryptodevs/mlx5.rst b/doc/guides/cryptodevs/mlx5.rst
index 05a0a449e2..dd1d1a615d 100644
--- a/doc/guides/cryptodevs/mlx5.rst
+++ b/doc/guides/cryptodevs/mlx5.rst
@@ -53,6 +53,16 @@ Supported NICs
 
 * Mellanox\ |reg| ConnectX\ |reg|-6 200G MCX654106A-HCAT (2x200G)
 
+
+Limitations
+-----------
+
+- AES-XTS keys provided in xform must include keytag and should be wrappend.
+- The supported data-unit lengths are 512B and 1KB. In case the `dataunit_len`
+  is not provided in the cipher xform, the OP length is limited to the above
+  values and 1MB.
+
+
 Prerequisites
 -------------
 
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index d2d82c7b15..3f0c97d081 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -3,6 +3,7 @@
  */
 
 #include <rte_malloc.h>
+#include <rte_mempool.h>
 #include <rte_errno.h>
 #include <rte_log.h>
 #include <rte_pci.h>
@@ -20,7 +21,9 @@
 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
 
 #define MLX5_CRYPTO_FEATURE_FLAGS \
-	RTE_CRYPTODEV_FF_HW_ACCELERATED
+	(RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | \
+	 RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY | \
+	 RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS)
 
 TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
 				TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
@@ -30,6 +33,32 @@ int mlx5_crypto_logtype;
 
 uint8_t mlx5_crypto_driver_id;
 
+const struct rte_cryptodev_capabilities mlx5_crypto_caps[] = {
+	{		/* AES XTS */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+			{.cipher = {
+				.algo = RTE_CRYPTO_CIPHER_AES_XTS,
+				.block_size = 16,
+				.key_size = {
+					.min = 32,
+					.max = 64,
+					.increment = 32
+				},
+				.iv_size = {
+					.min = 16,
+					.max = 16,
+					.increment = 0
+				},
+				.dataunit_set =
+				RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_512_BYTES |
+				RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_4096_BYTES,
+			}, }
+		}, }
+	},
+};
+
 static const char mlx5_crypto_drv_name[] = RTE_STR(MLX5_CRYPTO_DRIVER_NAME);
 
 static const struct rte_driver mlx5_drv = {
@@ -39,6 +68,49 @@ static const struct rte_driver mlx5_drv = {
 
 static struct cryptodev_driver mlx5_cryptodev_driver;
 
+struct mlx5_crypto_session {
+	uint32_t bs_bpt_eo_es;
+	/*
+	 * bsf_size, bsf_p_type, encryption_order and encryption standard,
+	 * saved in big endian format.
+	 */
+	uint32_t bsp_res;
+	/*
+	 * crypto_block_size_pointer and reserved 24 bits saved in big endian
+	 * format.
+	 */
+	uint32_t iv_offset:16;
+	/* Starting point for Initialisation Vector. */
+	struct mlx5_crypto_dek *dek; /* Pointer to dek struct. */
+	uint32_t dek_id; /* DEK ID */
+} __rte_packed;
+
+static void
+mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev,
+			  struct rte_cryptodev_info *dev_info)
+{
+	RTE_SET_USED(dev);
+	if (dev_info != NULL) {
+		dev_info->driver_id = mlx5_crypto_driver_id;
+		dev_info->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
+		dev_info->capabilities = mlx5_crypto_caps;
+		dev_info->max_nb_queue_pairs = 0;
+		dev_info->min_mbuf_headroom_req = 0;
+		dev_info->min_mbuf_tailroom_req = 0;
+		dev_info->sym.max_nb_sessions = 0;
+		/*
+		 * If 0, the device does not have any limitation in number of
+		 * sessions that can be used.
+		 */
+	}
+}
+
+static unsigned int
+mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
+{
+	return sizeof(struct mlx5_crypto_session);
+}
+
 static int
 mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
 		struct rte_cryptodev_config *config __rte_unused)
@@ -61,19 +133,109 @@ mlx5_crypto_dev_close(struct rte_cryptodev *dev)
 	return 0;
 }
 
+static int
+mlx5_crypto_sym_session_configure(struct rte_cryptodev *dev,
+				  struct rte_crypto_sym_xform *xform,
+				  struct rte_cryptodev_sym_session *session,
+				  struct rte_mempool *mp)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+	struct mlx5_crypto_session *sess_private_data;
+	struct rte_crypto_cipher_xform *cipher;
+	uint8_t encryption_order;
+	int ret;
+
+	if (unlikely(xform->next != NULL)) {
+		DRV_LOG(ERR, "Xform next is not supported.");
+		return -ENOTSUP;
+	}
+	if (unlikely((xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) ||
+		     (xform->cipher.algo != RTE_CRYPTO_CIPHER_AES_XTS))) {
+		DRV_LOG(ERR, "Only AES-XTS algorithm is supported.");
+		return -ENOTSUP;
+	}
+	ret = rte_mempool_get(mp, (void *)&sess_private_data);
+	if (ret != 0) {
+		DRV_LOG(ERR,
+			"Failed to get session %p private data from mempool.",
+			sess_private_data);
+		return -ENOMEM;
+	}
+	cipher = &xform->cipher;
+	sess_private_data->dek = mlx5_crypto_dek_prepare(priv, cipher);
+	if (sess_private_data->dek == NULL) {
+		rte_mempool_put(mp, sess_private_data);
+		DRV_LOG(ERR, "Failed to prepare dek.");
+		return -ENOMEM;
+	}
+	if (cipher->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
+		encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_MEMORY;
+	else
+		encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_WIRE;
+	sess_private_data->bs_bpt_eo_es = rte_cpu_to_be_32
+			(MLX5_BSF_SIZE_64B << MLX5_BSF_SIZE_OFFSET |
+			 MLX5_BSF_P_TYPE_CRYPTO << MLX5_BSF_P_TYPE_OFFSET |
+			 encryption_order << MLX5_ENCRYPTION_ORDER_OFFSET |
+			 MLX5_ENCRYPTION_STANDARD_AES_XTS);
+	switch (xform->cipher.dataunit_len) {
+	case 0:
+		sess_private_data->bsp_res = 0;
+		break;
+	case 512:
+		sess_private_data->bsp_res = rte_cpu_to_be_32
+					     ((uint32_t)MLX5_BLOCK_SIZE_512B <<
+					     MLX5_BLOCK_SIZE_OFFSET);
+		break;
+	case 4096:
+		sess_private_data->bsp_res = rte_cpu_to_be_32
+					     ((uint32_t)MLX5_BLOCK_SIZE_4096B <<
+					     MLX5_BLOCK_SIZE_OFFSET);
+		break;
+	default:
+		DRV_LOG(ERR, "Cipher data unit length is not supported.");
+		return -ENOTSUP;
+	}
+	sess_private_data->iv_offset = cipher->iv.offset;
+	sess_private_data->dek_id =
+			rte_cpu_to_be_32(sess_private_data->dek->obj->id &
+					 0xffffff);
+	set_sym_session_private_data(session, dev->driver_id,
+				     sess_private_data);
+	DRV_LOG(DEBUG, "Session %p was configured.", sess_private_data);
+	return 0;
+}
+
+static void
+mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev,
+			      struct rte_cryptodev_sym_session *sess)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+	struct mlx5_crypto_session *spriv = get_sym_session_private_data(sess,
+								dev->driver_id);
+
+	if (unlikely(spriv == NULL)) {
+		DRV_LOG(ERR, "Failed to get session %p private data.", spriv);
+		return;
+	}
+	mlx5_crypto_dek_destroy(priv, spriv->dek);
+	set_sym_session_private_data(sess, dev->driver_id, NULL);
+	rte_mempool_put(rte_mempool_from_obj(spriv), spriv);
+	DRV_LOG(DEBUG, "Session %p was cleared.", spriv);
+}
+
 static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_configure			= mlx5_crypto_dev_configure,
 	.dev_start			= NULL,
 	.dev_stop			= NULL,
 	.dev_close			= mlx5_crypto_dev_close,
-	.dev_infos_get			= NULL,
+	.dev_infos_get			= mlx5_crypto_dev_infos_get,
 	.stats_get			= NULL,
 	.stats_reset			= NULL,
 	.queue_pair_setup		= NULL,
 	.queue_pair_release		= NULL,
-	.sym_session_get_size		= NULL,
-	.sym_session_configure		= NULL,
-	.sym_session_clear		= NULL,
+	.sym_session_get_size		= mlx5_crypto_sym_session_get_size,
+	.sym_session_configure		= mlx5_crypto_sym_session_configure,
+	.sym_session_clear		= mlx5_crypto_sym_session_clear,
 	.sym_get_raw_dp_ctx_size	= NULL,
 	.sym_configure_raw_dp_ctx	= NULL,
 };
-- 
2.27.0

[dpdk-dev] [PATCH v6 04/15] crypto/mlx5: add basic operations

[Shiri Kuzin]

The basic dev control operations are configure, close and get info.

Extended the existing support of configure and close:
	-mlx5_crypto_dev_configure- function used to configure device.
	-mlx5_crypto_dev_close-  function used to close a configured
	 device.

Added config struct to user private data with the fields socket id,
number of queue pairs and feature flags to be disabled.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 26 +++++++++++++++++++-------
 drivers/crypto/mlx5/mlx5_crypto.h |  1 +
 2 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 3f0c97d081..a7e44deb9e 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -105,22 +105,27 @@ mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev,
 	}
 }
 
-static unsigned int
-mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
-{
-	return sizeof(struct mlx5_crypto_session);
-}
-
 static int
 mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
-		struct rte_cryptodev_config *config __rte_unused)
+			  struct rte_cryptodev_config *config)
 {
 	struct mlx5_crypto_priv *priv = dev->data->dev_private;
 
+	if (config == NULL) {
+		DRV_LOG(ERR, "Invalid crypto dev configure parameters.");
+		return -EINVAL;
+	}
+	if ((config->ff_disable & RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO) != 0) {
+		DRV_LOG(ERR,
+			"Disabled symmetric crypto feature is not supported.");
+		return -ENOTSUP;
+	}
 	if (mlx5_crypto_dek_setup(priv) != 0) {
 		DRV_LOG(ERR, "Dek hash list creation has failed.");
 		return -ENOMEM;
 	}
+	priv->dev_config = *config;
+	DRV_LOG(DEBUG, "Device %u was configured.", dev->driver_id);
 	return 0;
 }
 
@@ -130,9 +135,16 @@ mlx5_crypto_dev_close(struct rte_cryptodev *dev)
 	struct mlx5_crypto_priv *priv = dev->data->dev_private;
 
 	mlx5_crypto_dek_unset(priv);
+	DRV_LOG(DEBUG, "Device %u was closed.", dev->driver_id);
 	return 0;
 }
 
+static unsigned int
+mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
+{
+	return sizeof(struct mlx5_crypto_session);
+}
+
 static int
 mlx5_crypto_sym_session_configure(struct rte_cryptodev *dev,
 				  struct rte_crypto_sym_xform *xform,
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 5a54cb0dca..4c07356028 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -24,6 +24,7 @@ struct mlx5_crypto_priv {
 	uint32_t pdn; /* Protection Domain number. */
 	struct ibv_pd *pd;
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
+	struct rte_cryptodev_config dev_config;
 };
 
 struct mlx5_crypto_dek {
-- 
2.27.0

[dpdk-dev] [PATCH v6 05/15] crypto/mlx5: add queue pairs operations

[Shiri Kuzin]

The HW queue pairs are a pair of send queue and receive queue of
independent work queues packed together in one object for the purpose
of transferring data between nodes of a network.

Completion Queue is a FIFO queue of completed work requests.

In crypto driver we use one QP in loopback in order to encrypt and
decrypt data locally without sending it to the wire.
In the configured QP we only use the SQ to perform the encryption and
decryption operations.

Added implementation for the QP setup function which creates the CQ,
creates the QP and changes its state to RTS (ready to send).

Added implementation for the release QP function to release all the QP
resources.

Added the ops structure that contains any operation which is supported
by the cryptodev.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 129 +++++++++++++++++++++++++++++-
 drivers/crypto/mlx5/mlx5_crypto.h |  11 +++
 2 files changed, 137 insertions(+), 3 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index a7e44deb9e..ebaa65c7a9 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -7,6 +7,7 @@
 #include <rte_errno.h>
 #include <rte_log.h>
 #include <rte_pci.h>
+#include <rte_memory.h>
 
 #include <mlx5_glue.h>
 #include <mlx5_common.h>
@@ -19,6 +20,7 @@
 
 #define MLX5_CRYPTO_DRIVER_NAME mlx5_crypto
 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
+#define MLX5_CRYPTO_MAX_QPS 1024
 
 #define MLX5_CRYPTO_FEATURE_FLAGS \
 	(RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | \
@@ -94,7 +96,7 @@ mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev,
 		dev_info->driver_id = mlx5_crypto_driver_id;
 		dev_info->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
 		dev_info->capabilities = mlx5_crypto_caps;
-		dev_info->max_nb_queue_pairs = 0;
+		dev_info->max_nb_queue_pairs = MLX5_CRYPTO_MAX_QPS;
 		dev_info->min_mbuf_headroom_req = 0;
 		dev_info->min_mbuf_tailroom_req = 0;
 		dev_info->sym.max_nb_sessions = 0;
@@ -235,6 +237,127 @@ mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev,
 	DRV_LOG(DEBUG, "Session %p was cleared.", spriv);
 }
 
+static int
+mlx5_crypto_queue_pair_release(struct rte_cryptodev *dev, uint16_t qp_id)
+{
+	struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
+
+	if (qp->qp_obj != NULL)
+		claim_zero(mlx5_devx_cmd_destroy(qp->qp_obj));
+	if (qp->umem_obj != NULL)
+		claim_zero(mlx5_glue->devx_umem_dereg(qp->umem_obj));
+	if (qp->umem_buf != NULL)
+		rte_free(qp->umem_buf);
+	mlx5_devx_cq_destroy(&qp->cq_obj);
+	rte_free(qp);
+	dev->data->queue_pairs[qp_id] = NULL;
+	return 0;
+}
+
+static int
+mlx5_crypto_qp2rts(struct mlx5_crypto_qp *qp)
+{
+	/*
+	 * In Order to configure self loopback, when calling these functions the
+	 * remote QP id that is used is the id of the same QP.
+	 */
+	if (mlx5_devx_cmd_modify_qp_state(qp->qp_obj, MLX5_CMD_OP_RST2INIT_QP,
+					  qp->qp_obj->id)) {
+		DRV_LOG(ERR, "Failed to modify QP to INIT state(%u).",
+			rte_errno);
+		return -1;
+	}
+	if (mlx5_devx_cmd_modify_qp_state(qp->qp_obj, MLX5_CMD_OP_INIT2RTR_QP,
+					  qp->qp_obj->id)) {
+		DRV_LOG(ERR, "Failed to modify QP to RTR state(%u).",
+			rte_errno);
+		return -1;
+	}
+	if (mlx5_devx_cmd_modify_qp_state(qp->qp_obj, MLX5_CMD_OP_RTR2RTS_QP,
+					  qp->qp_obj->id)) {
+		DRV_LOG(ERR, "Failed to modify QP to RTS state(%u).",
+			rte_errno);
+		return -1;
+	}
+	return 0;
+}
+
+static int
+mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
+			     const struct rte_cryptodev_qp_conf *qp_conf,
+			     int socket_id)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+	struct mlx5_devx_qp_attr attr = {0};
+	struct mlx5_crypto_qp *qp;
+	uint16_t log_nb_desc = rte_log2_u32(qp_conf->nb_descriptors);
+	uint32_t umem_size = RTE_BIT32(log_nb_desc) *
+			      MLX5_CRYPTO_WQE_SET_SIZE +
+			      sizeof(*qp->db_rec) * 2;
+	uint32_t alloc_size = sizeof(*qp);
+	struct mlx5_devx_cq_attr cq_attr = {
+		.uar_page_id = mlx5_os_get_devx_uar_page_id(priv->uar),
+	};
+
+	if (dev->data->queue_pairs[qp_id] != NULL)
+		mlx5_crypto_queue_pair_release(dev, qp_id);
+	alloc_size = RTE_ALIGN(alloc_size, RTE_CACHE_LINE_SIZE);
+	alloc_size += sizeof(struct rte_crypto_op *) * RTE_BIT32(log_nb_desc);
+	qp = rte_zmalloc_socket(__func__, alloc_size, RTE_CACHE_LINE_SIZE,
+				socket_id);
+	if (qp == NULL) {
+		DRV_LOG(ERR, "Failed to allocate QP memory.");
+		rte_errno = ENOMEM;
+		return -rte_errno;
+	}
+	if (mlx5_devx_cq_create(priv->ctx, &qp->cq_obj, log_nb_desc,
+				&cq_attr, socket_id) != 0) {
+		DRV_LOG(ERR, "Failed to create CQ.");
+		goto error;
+	}
+	qp->umem_buf = rte_zmalloc_socket(__func__, umem_size, 4096, socket_id);
+	if (qp->umem_buf == NULL) {
+		DRV_LOG(ERR, "Failed to allocate QP umem.");
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	qp->umem_obj = mlx5_glue->devx_umem_reg(priv->ctx,
+					       (void *)(uintptr_t)qp->umem_buf,
+					       umem_size,
+					       IBV_ACCESS_LOCAL_WRITE);
+	if (qp->umem_obj == NULL) {
+		DRV_LOG(ERR, "Failed to register QP umem.");
+		goto error;
+	}
+	attr.pd = priv->pdn;
+	attr.uar_index = mlx5_os_get_devx_uar_page_id(priv->uar);
+	attr.cqn = qp->cq_obj.cq->id;
+	attr.log_page_size = rte_log2_u32(sysconf(_SC_PAGESIZE));
+	attr.rq_size =  0;
+	attr.sq_size = RTE_BIT32(log_nb_desc);
+	attr.dbr_umem_valid = 1;
+	attr.wq_umem_id = qp->umem_obj->umem_id;
+	attr.wq_umem_offset = 0;
+	attr.dbr_umem_id = qp->umem_obj->umem_id;
+	attr.dbr_address = RTE_BIT64(log_nb_desc) *
+			   MLX5_CRYPTO_WQE_SET_SIZE;
+	qp->qp_obj = mlx5_devx_cmd_create_qp(priv->ctx, &attr);
+	if (qp->qp_obj == NULL) {
+		DRV_LOG(ERR, "Failed to create QP(%u).", rte_errno);
+		goto error;
+	}
+	qp->db_rec = RTE_PTR_ADD(qp->umem_buf, (uintptr_t)attr.dbr_address);
+	if (mlx5_crypto_qp2rts(qp))
+		goto error;
+	qp->ops = (struct rte_crypto_op **)RTE_ALIGN((uintptr_t)(qp + 1),
+							   RTE_CACHE_LINE_SIZE);
+	dev->data->queue_pairs[qp_id] = qp;
+	return 0;
+error:
+	mlx5_crypto_queue_pair_release(dev, qp_id);
+	return -1;
+}
+
 static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_configure			= mlx5_crypto_dev_configure,
 	.dev_start			= NULL,
@@ -243,8 +366,8 @@ static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_infos_get			= mlx5_crypto_dev_infos_get,
 	.stats_get			= NULL,
 	.stats_reset			= NULL,
-	.queue_pair_setup		= NULL,
-	.queue_pair_release		= NULL,
+	.queue_pair_setup		= mlx5_crypto_queue_pair_setup,
+	.queue_pair_release		= mlx5_crypto_queue_pair_release,
 	.sym_session_get_size		= mlx5_crypto_sym_session_get_size,
 	.sym_session_configure		= mlx5_crypto_sym_session_configure,
 	.sym_session_clear		= mlx5_crypto_sym_session_clear,
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 4c07356028..a3bc65b0ed 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -11,9 +11,11 @@
 #include <rte_cryptodev_pmd.h>
 
 #include <mlx5_common_utils.h>
+#include <mlx5_common_devx.h>
 
 #define MLX5_CRYPTO_DEK_HTABLE_SZ (1 << 11)
 #define MLX5_CRYPTO_KEY_LENGTH 80
+#define MLX5_CRYPTO_WQE_SET_SIZE 1024
 
 struct mlx5_crypto_priv {
 	TAILQ_ENTRY(mlx5_crypto_priv) next;
@@ -27,6 +29,15 @@ struct mlx5_crypto_priv {
 	struct rte_cryptodev_config dev_config;
 };
 
+struct mlx5_crypto_qp {
+	struct mlx5_devx_cq cq_obj;
+	struct mlx5_devx_obj *qp_obj;
+	struct mlx5dv_devx_umem *umem_obj;
+	void *umem_buf;
+	volatile uint32_t *db_rec;
+	struct rte_crypto_op **ops;
+};
+
 struct mlx5_crypto_dek {
 	struct mlx5_hlist_entry entry; /* Pointer to DEK hash list entry. */
 	struct mlx5_devx_obj *obj; /* Pointer to DEK DevX object. */
-- 
2.27.0

[dpdk-dev] [PATCH v6 06/15] crypto/mlx5: add dev stop and start operations

[Shiri Kuzin]

Add the dev_start function that is used to start a configured device.
Add the dev_stop function that is used to stop a configured device.

Both functions set the dev parameter as used and return 0.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index ebaa65c7a9..37e66cf57b 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -131,6 +131,19 @@ mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
 	return 0;
 }
 
+static void
+mlx5_crypto_dev_stop(struct rte_cryptodev *dev)
+{
+	RTE_SET_USED(dev);
+}
+
+static int
+mlx5_crypto_dev_start(struct rte_cryptodev *dev)
+{
+	RTE_SET_USED(dev);
+	return 0;
+}
+
 static int
 mlx5_crypto_dev_close(struct rte_cryptodev *dev)
 {
@@ -360,8 +373,8 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 
 static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_configure			= mlx5_crypto_dev_configure,
-	.dev_start			= NULL,
-	.dev_stop			= NULL,
+	.dev_start			= mlx5_crypto_dev_start,
+	.dev_stop			= mlx5_crypto_dev_stop,
 	.dev_close			= mlx5_crypto_dev_close,
 	.dev_infos_get			= mlx5_crypto_dev_infos_get,
 	.stats_get			= NULL,
-- 
2.27.0

[dpdk-dev] [PATCH v6 07/15] crypto/mlx5: add memory region management

[Shiri Kuzin]

Mellanox user space drivers don't deal with physical addresses as part
of a memory protection mechanism.
The device translates the given virtual address to a physical address
using the given memory key as an address space identifier.
That's why any mbuf virtual address is moved directly to the HW
descriptor(WQE).

The mapping between the virtual address to the physical address is saved
in MR configured by the kernel to the HW.

Each MR has a key that should also be moved to the WQE by the SW.

When the SW sees an unmapped address, it extends the address range and
creates a MR using a system call.

Add memory region cache management:
	- 2 level cache per queue-pair - no locks.
	- 1 shared cache between all the queues using a lock.

Using this way, the MR key search per data-path address is optimized.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Signed-off-by: Michael Baum <michaelba@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 doc/guides/cryptodevs/mlx5.rst    |  6 +++
 drivers/crypto/mlx5/mlx5_crypto.c | 63 +++++++++++++++++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto.h |  3 ++
 3 files changed, 72 insertions(+)

diff --git a/doc/guides/cryptodevs/mlx5.rst b/doc/guides/cryptodevs/mlx5.rst
index dd1d1a615d..ece881220e 100644
--- a/doc/guides/cryptodevs/mlx5.rst
+++ b/doc/guides/cryptodevs/mlx5.rst
@@ -26,6 +26,12 @@ the MKEY is configured to perform crypto operations.
 
 The encryption does not require text to be aligned to the AES block size (128b).
 
+For security reasons and to increase robustness, this driver only deals with virtual
+memory addresses. The way resources allocations are handled by the kernel,
+combined with hardware specifications that allow handling virtual memory
+addresses directly, ensure that DPDK applications cannot access random
+physical memory (or memory that does not belong to the current process).
+
 The PMD uses libibverbs and libmlx5 to access the device firmware or to
 access the hardware components directly.
 There are different levels of objects and bypassing abilities.
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 37e66cf57b..38008dcb28 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -261,6 +261,7 @@ mlx5_crypto_queue_pair_release(struct rte_cryptodev *dev, uint16_t qp_id)
 		claim_zero(mlx5_glue->devx_umem_dereg(qp->umem_obj));
 	if (qp->umem_buf != NULL)
 		rte_free(qp->umem_buf);
+	mlx5_mr_btree_free(&qp->mr_ctrl.cache_bh);
 	mlx5_devx_cq_destroy(&qp->cq_obj);
 	rte_free(qp);
 	dev->data->queue_pairs[qp_id] = NULL;
@@ -342,6 +343,14 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 		DRV_LOG(ERR, "Failed to register QP umem.");
 		goto error;
 	}
+	if (mlx5_mr_btree_init(&qp->mr_ctrl.cache_bh, MLX5_MR_BTREE_CACHE_N,
+			       priv->dev_config.socket_id) != 0) {
+		DRV_LOG(ERR, "Cannot allocate MR Btree for qp %u.",
+			(uint32_t)qp_id);
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	qp->mr_ctrl.dev_gen_ptr = &priv->mr_scache.dev_gen;
 	attr.pd = priv->pdn;
 	attr.uar_index = mlx5_os_get_devx_uar_page_id(priv->uar);
 	attr.cqn = qp->cq_obj.cq->id;
@@ -448,6 +457,40 @@ mlx5_crypto_hw_global_prepare(struct mlx5_crypto_priv *priv)
 	return 0;
 }
 
+/**
+ * Callback for memory event.
+ *
+ * @param event_type
+ *   Memory event type.
+ * @param addr
+ *   Address of memory.
+ * @param len
+ *   Size of memory.
+ */
+static void
+mlx5_crypto_mr_mem_event_cb(enum rte_mem_event event_type, const void *addr,
+			    size_t len, void *arg __rte_unused)
+{
+	struct mlx5_crypto_priv *priv;
+
+	/* Must be called from the primary process. */
+	MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);
+	switch (event_type) {
+	case RTE_MEM_EVENT_FREE:
+		pthread_mutex_lock(&priv_list_lock);
+		/* Iterate all the existing mlx5 devices. */
+		TAILQ_FOREACH(priv, &mlx5_crypto_priv_list, next)
+			mlx5_free_mr_by_addr(&priv->mr_scache,
+					     priv->ctx->device->name,
+					     addr, len);
+		pthread_mutex_unlock(&priv_list_lock);
+		break;
+	case RTE_MEM_EVENT_ALLOC:
+	default:
+		break;
+	}
+}
+
 /**
  * DPDK callback to register a PCI device.
  *
@@ -530,6 +573,22 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 		claim_zero(mlx5_glue->close_device(priv->ctx));
 		return -1;
 	}
+	if (mlx5_mr_btree_init(&priv->mr_scache.cache,
+			     MLX5_MR_BTREE_CACHE_N * 2, rte_socket_id()) != 0) {
+		DRV_LOG(ERR, "Failed to allocate shared cache MR memory.");
+		mlx5_crypto_hw_global_release(priv);
+		rte_cryptodev_pmd_destroy(priv->crypto_dev);
+		claim_zero(mlx5_glue->close_device(priv->ctx));
+		rte_errno = ENOMEM;
+		return -rte_errno;
+	}
+	priv->mr_scache.reg_mr_cb = mlx5_common_verbs_reg_mr;
+	priv->mr_scache.dereg_mr_cb = mlx5_common_verbs_dereg_mr;
+	/* Register callback function for global shared MR cache management. */
+	if (TAILQ_EMPTY(&mlx5_crypto_priv_list))
+		rte_mem_event_callback_register("MLX5_MEM_EVENT_CB",
+						mlx5_crypto_mr_mem_event_cb,
+						NULL);
 	pthread_mutex_lock(&priv_list_lock);
 	TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next);
 	pthread_mutex_unlock(&priv_list_lock);
@@ -549,6 +608,10 @@ mlx5_crypto_pci_remove(struct rte_pci_device *pdev)
 		TAILQ_REMOVE(&mlx5_crypto_priv_list, priv, next);
 	pthread_mutex_unlock(&priv_list_lock);
 	if (priv) {
+		if (TAILQ_EMPTY(&mlx5_crypto_priv_list))
+			rte_mem_event_callback_unregister("MLX5_MEM_EVENT_CB",
+							  NULL);
+		mlx5_mr_release_cache(&priv->mr_scache);
 		mlx5_crypto_hw_global_release(priv);
 		rte_cryptodev_pmd_destroy(priv->crypto_dev);
 		claim_zero(mlx5_glue->close_device(priv->ctx));
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index a3bc65b0ed..d1ea8024d4 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -12,6 +12,7 @@
 
 #include <mlx5_common_utils.h>
 #include <mlx5_common_devx.h>
+#include <mlx5_common_mr.h>
 
 #define MLX5_CRYPTO_DEK_HTABLE_SZ (1 << 11)
 #define MLX5_CRYPTO_KEY_LENGTH 80
@@ -27,6 +28,7 @@ struct mlx5_crypto_priv {
 	struct ibv_pd *pd;
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
 	struct rte_cryptodev_config dev_config;
+	struct mlx5_mr_share_cache mr_scache; /* Global shared MR cache. */
 };
 
 struct mlx5_crypto_qp {
@@ -36,6 +38,7 @@ struct mlx5_crypto_qp {
 	void *umem_buf;
 	volatile uint32_t *db_rec;
 	struct rte_crypto_op **ops;
+	struct mlx5_mr_ctrl mr_ctrl;
 };
 
 struct mlx5_crypto_dek {
-- 
2.27.0

[dpdk-dev] [PATCH v6 08/15] crypto/mlx5: create login object using DevX

[Shiri Kuzin]

To work with crypto engines that are marked with wrapped_import_method,
a login session is required.
A crypto login object needs to be created using DevX.

The crypto login object contains:
	- The credential pointer.
	- The import_KEK pointer to be used for all secured information
	  communicated in crypto commands (key fields), including the
	  provided credential in this command.
	- The credential secret, wrapped by the import_KEK indicated in
	  this command. Size includes 8 bytes IV for wrapping.

Added devargs for the required login values:
	- wcs_file - path to the file containing the credential.
	- import_kek_id - the import KEK pointer.
	- credential_id - the credential pointer.

Create the login DevX object in pci_probe function and destroy it in
pci_remove.
Destroying the crypto login object means logout.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 doc/guides/cryptodevs/mlx5.rst    |  60 +++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto.c | 103 ++++++++++++++++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto.h |   7 ++
 3 files changed, 170 insertions(+)

diff --git a/doc/guides/cryptodevs/mlx5.rst b/doc/guides/cryptodevs/mlx5.rst
index ece881220e..6cae5affbd 100644
--- a/doc/guides/cryptodevs/mlx5.rst
+++ b/doc/guides/cryptodevs/mlx5.rst
@@ -44,6 +44,51 @@ To get the best performances:
 Enabling librte_crypto_mlx5 causes DPDK applications to be linked against
 libibverbs.
 
+In order to move the device to crypto operational mode, credential and KEK
+(Key Encrypting Key) should be set as the first step.
+The credential will be used by the software in order to perform crypto login, and the KEK is
+the AES Key Wrap Algorithm (rfc3394) key that will be used for sensitive data
+wrapping.
+The credential and the AES-XTS keys should be provided to the hardware, as ciphertext
+encrypted by the KEK.
+
+When crypto engines are defined to work in wrapped import method, they come out
+of the factory in Commissioning mode, and thus, cannot be used for crypto operations
+yet. A dedicated tool is used for changing the mode from Commissioning to
+Operational, while setting the first import_KEK and credential in plaintext.
+The mlxreg dedicated tool should be used as follows:
+
+- Set CRYPTO_OPERATIONAL register to set the device in crypto operational mode.
+
+  The input to this tool is:
+    The first credential in plaintext, 40B.
+    The first import_KEK in plaintext: kek size 0 for 16B or 1 for 32B, kek data.
+
+  Example:
+  mlxreg -d /dev/mst/mt4123_pciconf0 --reg_name CRYPTO_OPERATIONAL --get
+
+  The "wrapped_crypto_operational" value will be "0x00000000".
+  The command to set the register should be executed only once, and all the
+  values mentioned above should be specified in the same command.
+
+  Example:
+  mlxreg -d /dev/mst/mt4123_pciconf0 --reg_name CRYPTO_OPERATIONAL
+  --set "credential[0]=0x10000000, credential[1]=0x10000000, kek[0]=0x00000000"
+
+  All values not specified will remain 0.
+  "wrapped_crypto_going_to_commissioning" and  "wrapped_crypto_operational"
+  should not be specified.
+
+  All the device ports should set it in order to move to operational mode.
+
+- Query CRYPTO_OPERATIONAL register to make sure the device is in Operational
+  mode.
+
+  Example:
+  mlxreg -d /dev/mst/mt4123_pciconf0 --reg_name CRYPTO_OPERATIONAL --get
+  The "wrapped_crypto_operational" value will be "0x00000001" if the mode was
+  successfully changed to operational mode.
+
 
 Driver options
 --------------
@@ -53,6 +98,21 @@ Driver options
   Select the class of the driver that should probe the device.
   `crypto` for the mlx5 crypto driver.
 
+- ``wcs_file`` parameter [string] - mandatory
+
+  File path including only the wrapped credential in string format of hexadecimal
+  numbers, represent 48 bytes (8 bytes IV added by the AES key wrap algorithm).
+
+- ``import_kek_id`` parameter [int]
+
+  The identifier of the KEK, default value is 0 represents the operational
+  register import_kek..
+
+- ``credential_id`` parameter [int]
+
+  The identifier of the credential, default value is 0 represents the operational
+  register credential.
+
 
 Supported NICs
 --------------
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 38008dcb28..cbaa2e52ff 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -457,6 +457,101 @@ mlx5_crypto_hw_global_prepare(struct mlx5_crypto_priv *priv)
 	return 0;
 }
 
+
+static int
+mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque)
+{
+	struct mlx5_crypto_devarg_params *devarg_prms = opaque;
+	struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr;
+	unsigned long tmp;
+	FILE *file;
+	int ret;
+	int i;
+
+	if (strcmp(key, "class") == 0)
+		return 0;
+	if (strcmp(key, "wcs_file") == 0) {
+		file = fopen(val, "rb");
+		if (file == NULL) {
+			rte_errno = ENOTSUP;
+			return -rte_errno;
+		}
+		for (i = 0 ; i < MLX5_CRYPTO_CREDENTIAL_SIZE ; i++) {
+			ret = fscanf(file, "%02hhX", &attr->credential[i]);
+			if (ret <= 0) {
+				fclose(file);
+				DRV_LOG(ERR,
+					"Failed to read credential from file.");
+				rte_errno = EINVAL;
+				return -rte_errno;
+			}
+		}
+		fclose(file);
+		devarg_prms->login_devarg = true;
+		return 0;
+	}
+	errno = 0;
+	tmp = strtoul(val, NULL, 0);
+	if (errno) {
+		DRV_LOG(WARNING, "%s: \"%s\" is an invalid integer.", key, val);
+		return -errno;
+	}
+	if (strcmp(key, "import_kek_id") == 0)
+		attr->session_import_kek_ptr = (uint32_t)tmp;
+	else if (strcmp(key, "credential_id") == 0)
+		attr->credential_pointer = (uint32_t)tmp;
+	else
+		DRV_LOG(WARNING, "Invalid key %s.", key);
+	return 0;
+}
+
+static struct mlx5_devx_obj *
+mlx5_crypto_config_login(struct rte_devargs *devargs,
+			 struct ibv_context *ctx)
+{
+	/*
+	 * Set credential pointer and session import KEK pointer to a default
+	 * value of 0.
+	 */
+	struct mlx5_crypto_devarg_params login = {
+			.login_devarg = false,
+			.login_attr = {
+					.credential_pointer = 0,
+					.session_import_kek_ptr = 0,
+			}
+	};
+	struct rte_kvargs *kvlist;
+
+	if (devargs == NULL) {
+		DRV_LOG(ERR,
+	"No login devargs in order to enable crypto operations in the device.");
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	kvlist = rte_kvargs_parse(devargs->args, NULL);
+	if (kvlist == NULL) {
+		DRV_LOG(ERR, "Failed to parse devargs.");
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	if (rte_kvargs_process(kvlist, NULL, mlx5_crypto_args_check_handler,
+			   &login) != 0) {
+		DRV_LOG(ERR, "Devargs handler function Failed.");
+		rte_kvargs_free(kvlist);
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	rte_kvargs_free(kvlist);
+	if (login.login_devarg == false) {
+		DRV_LOG(ERR,
+	"No login credential devarg in order to enable crypto operations "
+	"in the device.");
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	return mlx5_devx_cmd_create_crypto_login_obj(ctx, &login.login_attr);
+}
+
 /**
  * Callback for memory event.
  *
@@ -512,6 +607,7 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	struct ibv_device *ibv;
 	struct rte_cryptodev *crypto_dev;
 	struct ibv_context *ctx;
+	struct mlx5_devx_obj *login;
 	struct mlx5_crypto_priv *priv;
 	struct mlx5_hca_attr attr = { 0 };
 	struct rte_cryptodev_pmd_init_params init_params = {
@@ -550,6 +646,11 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 		rte_errno = ENOTSUP;
 		return -ENOTSUP;
 	}
+	login = mlx5_crypto_config_login(pci_dev->device.devargs, ctx);
+	if (login == NULL) {
+		DRV_LOG(ERR, "Failed to configure login.");
+		return -rte_errno;
+	}
 	crypto_dev = rte_cryptodev_pmd_create(ibv->name, &pci_dev->device,
 					&init_params);
 	if (crypto_dev == NULL) {
@@ -566,6 +667,7 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	crypto_dev->driver_id = mlx5_crypto_driver_id;
 	priv = crypto_dev->data->dev_private;
 	priv->ctx = ctx;
+	priv->login_obj = login;
 	priv->pci_dev = pci_dev;
 	priv->crypto_dev = crypto_dev;
 	if (mlx5_crypto_hw_global_prepare(priv) != 0) {
@@ -614,6 +716,7 @@ mlx5_crypto_pci_remove(struct rte_pci_device *pdev)
 		mlx5_mr_release_cache(&priv->mr_scache);
 		mlx5_crypto_hw_global_release(priv);
 		rte_cryptodev_pmd_destroy(priv->crypto_dev);
+		claim_zero(mlx5_devx_cmd_destroy(priv->login_obj));
 		claim_zero(mlx5_glue->close_device(priv->ctx));
 	}
 	return 0;
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index d1ea8024d4..a45a210e58 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -29,6 +29,7 @@ struct mlx5_crypto_priv {
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
 	struct rte_cryptodev_config dev_config;
 	struct mlx5_mr_share_cache mr_scache; /* Global shared MR cache. */
+	struct mlx5_devx_obj *login_obj;
 };
 
 struct mlx5_crypto_qp {
@@ -48,6 +49,12 @@ struct mlx5_crypto_dek {
 	bool size_is_48; /* Whether the key\data size is 48 bytes or not. */
 } __rte_cache_aligned;
 
+
+struct mlx5_crypto_devarg_params {
+	bool login_devarg;
+	struct mlx5_devx_crypto_login_attr login_attr;
+};
+
 int
 mlx5_crypto_dek_destroy(struct mlx5_crypto_priv *priv,
 			struct mlx5_crypto_dek *dek);
-- 
2.27.0

[dpdk-dev] [PATCH v6 09/15] crypto/mlx5: add keytag devarg

[Shiri Kuzin]

From: Suanming Mou <suanmingm@nvidia.com>

A keytag is a piece of data encrypted together with a DEK.

When a DEK is referenced by an MKEY.bsf through its index, the keytag is
also supplied in the BSF as plaintext. The HW will decrypt the DEK (and
the attached keytag) and will fail the operation if the keytags don't
match.

This commit adds the configuration of the keytag with devargs.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
 doc/guides/cryptodevs/mlx5.rst    |  7 +++++
 drivers/crypto/mlx5/mlx5_crypto.c | 50 +++++++++++++++++--------------
 drivers/crypto/mlx5/mlx5_crypto.h |  3 +-
 3 files changed, 37 insertions(+), 23 deletions(-)

diff --git a/doc/guides/cryptodevs/mlx5.rst b/doc/guides/cryptodevs/mlx5.rst
index 6cae5affbd..c3632484a5 100644
--- a/doc/guides/cryptodevs/mlx5.rst
+++ b/doc/guides/cryptodevs/mlx5.rst
@@ -52,6 +52,9 @@ wrapping.
 The credential and the AES-XTS keys should be provided to the hardware, as ciphertext
 encrypted by the KEK.
 
+A keytag (64 bits) should be appended to the AES-XTS keys (before wrapping),
+and will be validated when the hardware attempts to access it.
+
 When crypto engines are defined to work in wrapped import method, they come out
 of the factory in Commissioning mode, and thus, cannot be used for crypto operations
 yet. A dedicated tool is used for changing the mode from Commissioning to
@@ -113,6 +116,10 @@ Driver options
   The identifier of the credential, default value is 0 represents the operational
   register credential.
 
+- ``keytag`` parameter [int]
+
+  The plaintext of the keytag appanded to the AES-XTS keys, default value is 0.
+
 
 Supported NICs
 --------------
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index cbaa2e52ff..0f786ac4ca 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -500,56 +500,52 @@ mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque)
 		attr->session_import_kek_ptr = (uint32_t)tmp;
 	else if (strcmp(key, "credential_id") == 0)
 		attr->credential_pointer = (uint32_t)tmp;
+	else if (strcmp(key, "keytag") == 0)
+		devarg_prms->keytag = tmp;
 	else
 		DRV_LOG(WARNING, "Invalid key %s.", key);
 	return 0;
 }
 
-static struct mlx5_devx_obj *
-mlx5_crypto_config_login(struct rte_devargs *devargs,
-			 struct ibv_context *ctx)
+static int
+mlx5_crypto_parse_devargs(struct rte_devargs *devargs,
+			  struct mlx5_crypto_devarg_params *devarg_prms)
 {
-	/*
-	 * Set credential pointer and session import KEK pointer to a default
-	 * value of 0.
-	 */
-	struct mlx5_crypto_devarg_params login = {
-			.login_devarg = false,
-			.login_attr = {
-					.credential_pointer = 0,
-					.session_import_kek_ptr = 0,
-			}
-	};
+	struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr;
 	struct rte_kvargs *kvlist;
 
+	/* Default values. */
+	attr->credential_pointer = 0;
+	attr->session_import_kek_ptr = 0;
+	devarg_prms->keytag = 0;
 	if (devargs == NULL) {
 		DRV_LOG(ERR,
 	"No login devargs in order to enable crypto operations in the device.");
 		rte_errno = EINVAL;
-		return NULL;
+		return -1;
 	}
 	kvlist = rte_kvargs_parse(devargs->args, NULL);
 	if (kvlist == NULL) {
 		DRV_LOG(ERR, "Failed to parse devargs.");
 		rte_errno = EINVAL;
-		return NULL;
+		return -1;
 	}
 	if (rte_kvargs_process(kvlist, NULL, mlx5_crypto_args_check_handler,
-			   &login) != 0) {
+			   devarg_prms) != 0) {
 		DRV_LOG(ERR, "Devargs handler function Failed.");
 		rte_kvargs_free(kvlist);
 		rte_errno = EINVAL;
-		return NULL;
+		return -1;
 	}
 	rte_kvargs_free(kvlist);
-	if (login.login_devarg == false) {
+	if (devarg_prms->login_devarg == false) {
 		DRV_LOG(ERR,
 	"No login credential devarg in order to enable crypto operations "
 	"in the device.");
 		rte_errno = EINVAL;
-		return NULL;
+		return -1;
 	}
-	return mlx5_devx_cmd_create_crypto_login_obj(ctx, &login.login_attr);
+	return 0;
 }
 
 /**
@@ -609,6 +605,7 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	struct ibv_context *ctx;
 	struct mlx5_devx_obj *login;
 	struct mlx5_crypto_priv *priv;
+	struct mlx5_crypto_devarg_params devarg_prms = { 0 };
 	struct mlx5_hca_attr attr = { 0 };
 	struct rte_cryptodev_pmd_init_params init_params = {
 		.name = "",
@@ -617,6 +614,8 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 		.max_nb_queue_pairs =
 				RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
 	};
+	int ret;
+
 	RTE_SET_USED(pci_drv);
 	if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
 		DRV_LOG(ERR, "Non-primary process type is not supported.");
@@ -646,7 +645,13 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 		rte_errno = ENOTSUP;
 		return -ENOTSUP;
 	}
-	login = mlx5_crypto_config_login(pci_dev->device.devargs, ctx);
+	ret = mlx5_crypto_parse_devargs(pci_dev->device.devargs, &devarg_prms);
+	if (ret) {
+		DRV_LOG(ERR, "Failed to parse devargs.");
+		return -rte_errno;
+	}
+	login = mlx5_devx_cmd_create_crypto_login_obj(ctx,
+						      &devarg_prms.login_attr);
 	if (login == NULL) {
 		DRV_LOG(ERR, "Failed to configure login.");
 		return -rte_errno;
@@ -686,6 +691,7 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	}
 	priv->mr_scache.reg_mr_cb = mlx5_common_verbs_reg_mr;
 	priv->mr_scache.dereg_mr_cb = mlx5_common_verbs_dereg_mr;
+	priv->keytag = rte_cpu_to_be_64(devarg_prms.keytag);
 	/* Register callback function for global shared MR cache management. */
 	if (TAILQ_EMPTY(&mlx5_crypto_priv_list))
 		rte_mem_event_callback_register("MLX5_MEM_EVENT_CB",
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index a45a210e58..ad70052967 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -30,6 +30,7 @@ struct mlx5_crypto_priv {
 	struct rte_cryptodev_config dev_config;
 	struct mlx5_mr_share_cache mr_scache; /* Global shared MR cache. */
 	struct mlx5_devx_obj *login_obj;
+	uint64_t keytag;
 };
 
 struct mlx5_crypto_qp {
@@ -49,10 +50,10 @@ struct mlx5_crypto_dek {
 	bool size_is_48; /* Whether the key\data size is 48 bytes or not. */
 } __rte_cache_aligned;
 
-
 struct mlx5_crypto_devarg_params {
 	bool login_devarg;
 	struct mlx5_devx_crypto_login_attr login_attr;
+	uint64_t keytag;
 };
 
 int
-- 
2.27.0

[dpdk-dev] [PATCH v6 10/15] crypto/mlx5: add maximum segments devarg

[Shiri Kuzin]

From: Suanming Mou <suanmingm@nvidia.com>

The mlx5 HW crypto operations are done by attaching crypto property
to a memory region. Once done, every access to the memory via the
crypto-enabled memory region will result with in-line encryption or
decryption of the data.

As a result, the design choice is to provide two types of WQEs. One
is UMR WQE which sets the crypto property and the other is rdma write
WQE which sends DMA command to copy data from local MR to remote MR.

The size of the WQEs will be defined by a new devarg called
max_segs_num.

This devarg also defines the maximum segments in mbuf chain that will be
supported for crypto operations.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
 doc/guides/cryptodevs/mlx5.rst    |  4 ++++
 drivers/crypto/mlx5/mlx5_crypto.c | 33 +++++++++++++++++++++++++++----
 drivers/crypto/mlx5/mlx5_crypto.h |  6 ++++++
 3 files changed, 39 insertions(+), 4 deletions(-)

diff --git a/doc/guides/cryptodevs/mlx5.rst b/doc/guides/cryptodevs/mlx5.rst
index c3632484a5..dd4705b744 100644
--- a/doc/guides/cryptodevs/mlx5.rst
+++ b/doc/guides/cryptodevs/mlx5.rst
@@ -120,6 +120,10 @@ Driver options
 
   The plaintext of the keytag appanded to the AES-XTS keys, default value is 0.
 
+- ``max_segs_num`` parameter [int]
+
+  Maximum number of mbuf chain segments(src or dest), default value is 8.
+
 
 Supported NICs
 --------------
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 0f786ac4ca..fc251f3f4c 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -21,6 +21,7 @@
 #define MLX5_CRYPTO_DRIVER_NAME mlx5_crypto
 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
 #define MLX5_CRYPTO_MAX_QPS 1024
+#define MLX5_CRYPTO_MAX_SEGS 56
 
 #define MLX5_CRYPTO_FEATURE_FLAGS \
 	(RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | \
@@ -496,14 +497,24 @@ mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque)
 		DRV_LOG(WARNING, "%s: \"%s\" is an invalid integer.", key, val);
 		return -errno;
 	}
-	if (strcmp(key, "import_kek_id") == 0)
+	if (strcmp(key, "max_segs_num") == 0) {
+		if (!tmp || tmp > MLX5_CRYPTO_MAX_SEGS) {
+			DRV_LOG(WARNING, "Invalid max_segs_num: %d, should"
+				" be less than %d.",
+				(uint32_t)tmp, MLX5_CRYPTO_MAX_SEGS);
+			rte_errno = EINVAL;
+			return -rte_errno;
+		}
+		devarg_prms->max_segs_num = (uint32_t)tmp;
+	} else if (strcmp(key, "import_kek_id") == 0) {
 		attr->session_import_kek_ptr = (uint32_t)tmp;
-	else if (strcmp(key, "credential_id") == 0)
+	} else if (strcmp(key, "credential_id") == 0) {
 		attr->credential_pointer = (uint32_t)tmp;
-	else if (strcmp(key, "keytag") == 0)
+	} else if (strcmp(key, "keytag") == 0) {
 		devarg_prms->keytag = tmp;
-	else
+	} else {
 		DRV_LOG(WARNING, "Invalid key %s.", key);
+	}
 	return 0;
 }
 
@@ -518,6 +529,7 @@ mlx5_crypto_parse_devargs(struct rte_devargs *devargs,
 	attr->credential_pointer = 0;
 	attr->session_import_kek_ptr = 0;
 	devarg_prms->keytag = 0;
+	devarg_prms->max_segs_num = 8;
 	if (devargs == NULL) {
 		DRV_LOG(ERR,
 	"No login devargs in order to enable crypto operations in the device.");
@@ -614,6 +626,7 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 		.max_nb_queue_pairs =
 				RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
 	};
+	uint16_t rdmw_wqe_size;
 	int ret;
 
 	RTE_SET_USED(pci_drv);
@@ -692,6 +705,18 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	priv->mr_scache.reg_mr_cb = mlx5_common_verbs_reg_mr;
 	priv->mr_scache.dereg_mr_cb = mlx5_common_verbs_dereg_mr;
 	priv->keytag = rte_cpu_to_be_64(devarg_prms.keytag);
+	priv->max_segs_num = devarg_prms.max_segs_num;
+	priv->umr_wqe_size = sizeof(struct mlx5_wqe_umr_bsf_seg) +
+			     sizeof(struct mlx5_umr_wqe) +
+			     RTE_ALIGN(priv->max_segs_num, 4) *
+			     sizeof(struct mlx5_wqe_dseg);
+	rdmw_wqe_size = sizeof(struct mlx5_rdma_write_wqe) +
+			      sizeof(struct mlx5_wqe_dseg) *
+			      (priv->max_segs_num <= 2 ? 2 : 2 +
+			       RTE_ALIGN(priv->max_segs_num - 2, 4));
+	priv->wqe_set_size = priv->umr_wqe_size + rdmw_wqe_size;
+	priv->umr_wqe_stride = priv->umr_wqe_size / MLX5_SEND_WQE_BB;
+	priv->max_rdmar_ds = rdmw_wqe_size / sizeof(struct mlx5_wqe_dseg);
 	/* Register callback function for global shared MR cache management. */
 	if (TAILQ_EMPTY(&mlx5_crypto_priv_list))
 		rte_mem_event_callback_register("MLX5_MEM_EVENT_CB",
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index ad70052967..48ca1cb9a2 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -25,12 +25,17 @@ struct mlx5_crypto_priv {
 	struct rte_cryptodev *crypto_dev;
 	void *uar; /* User Access Region. */
 	uint32_t pdn; /* Protection Domain number. */
+	uint32_t max_segs_num; /* Maximum supported data segs. */
 	struct ibv_pd *pd;
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
 	struct rte_cryptodev_config dev_config;
 	struct mlx5_mr_share_cache mr_scache; /* Global shared MR cache. */
 	struct mlx5_devx_obj *login_obj;
 	uint64_t keytag;
+	uint16_t wqe_set_size;
+	uint16_t umr_wqe_size;
+	uint16_t umr_wqe_stride;
+	uint16_t max_rdmar_ds;
 };
 
 struct mlx5_crypto_qp {
@@ -54,6 +59,7 @@ struct mlx5_crypto_devarg_params {
 	bool login_devarg;
 	struct mlx5_devx_crypto_login_attr login_attr;
 	uint64_t keytag;
+	uint32_t max_segs_num;
 };
 
 int
-- 
2.27.0

[dpdk-dev] [PATCH v6 11/15] crypto/mlx5: add WQE set initialization

[Shiri Kuzin]

From: Suanming Mou <suanmingm@nvidia.com>

Currently, HW handles the WQEs much faster than the software,
Using the constant WQE set layout can initialize most of the WQE
segments in advanced, and software only needs to configure very
limited segments in datapath. This accelerates the software WQE
organize in datapath.

This commit initializes the fixed WQE set segments.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 83 +++++++++++++++++++++++++++++--
 drivers/crypto/mlx5/mlx5_crypto.h | 10 +++-
 2 files changed, 87 insertions(+), 6 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index fc251f3f4c..e5f8d96ff7 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -297,6 +297,69 @@ mlx5_crypto_qp2rts(struct mlx5_crypto_qp *qp)
 	return 0;
 }
 
+static void
+mlx5_crypto_qp_init(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp)
+{
+	uint32_t i;
+
+	for (i = 0 ; i < qp->entries_n; i++) {
+		struct mlx5_wqe_cseg *cseg = RTE_PTR_ADD(qp->umem_buf, i *
+							 priv->wqe_set_size);
+		struct mlx5_wqe_umr_cseg *ucseg = (struct mlx5_wqe_umr_cseg *)
+								     (cseg + 1);
+		struct mlx5_wqe_umr_bsf_seg *bsf =
+			(struct mlx5_wqe_umr_bsf_seg *)(RTE_PTR_ADD(cseg,
+						       priv->umr_wqe_size)) - 1;
+		struct mlx5_wqe_rseg *rseg;
+
+		/* Init UMR WQE. */
+		cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj->id << 8) |
+					 (priv->umr_wqe_size / MLX5_WSEG_SIZE));
+		cseg->flags = RTE_BE32(MLX5_COMP_ONLY_FIRST_ERR <<
+				       MLX5_COMP_MODE_OFFSET);
+		cseg->misc = rte_cpu_to_be_32(qp->mkey[i]->id);
+		ucseg->if_cf_toe_cq_res = RTE_BE32(1u << MLX5_UMRC_IF_OFFSET);
+		ucseg->mkey_mask = RTE_BE64(1u << 0); /* Mkey length bit. */
+		ucseg->ko_to_bs = rte_cpu_to_be_32
+			((RTE_ALIGN(priv->max_segs_num, 4u) <<
+			 MLX5_UMRC_KO_OFFSET) | (4 << MLX5_UMRC_TO_BS_OFFSET));
+		bsf->keytag = priv->keytag;
+		/* Init RDMA WRITE WQE. */
+		cseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size);
+		cseg->flags = RTE_BE32((MLX5_COMP_ALWAYS <<
+				      MLX5_COMP_MODE_OFFSET) |
+				      MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE);
+		rseg = (struct mlx5_wqe_rseg *)(cseg + 1);
+		rseg->rkey = rte_cpu_to_be_32(qp->mkey[i]->id);
+	}
+}
+
+static int
+mlx5_crypto_indirect_mkeys_prepare(struct mlx5_crypto_priv *priv,
+				  struct mlx5_crypto_qp *qp)
+{
+	struct mlx5_umr_wqe *umr;
+	uint32_t i;
+	struct mlx5_devx_mkey_attr attr = {
+		.pd = priv->pdn,
+		.umr_en = 1,
+		.crypto_en = 1,
+		.set_remote_rw = 1,
+		.klm_num = RTE_ALIGN(priv->max_segs_num, 4),
+	};
+
+	for (umr = (struct mlx5_umr_wqe *)qp->umem_buf, i = 0;
+	   i < qp->entries_n; i++, umr = RTE_PTR_ADD(umr, priv->wqe_set_size)) {
+		attr.klm_array = (struct mlx5_klm *)&umr->kseg[0];
+		qp->mkey[i] = mlx5_devx_cmd_mkey_create(priv->ctx, &attr);
+		if (!qp->mkey[i]) {
+			DRV_LOG(ERR, "Failed to allocate indirect mkey.");
+			return -1;
+		}
+	}
+	return 0;
+}
+
 static int
 mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 			     const struct rte_cryptodev_qp_conf *qp_conf,
@@ -307,7 +370,7 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	struct mlx5_crypto_qp *qp;
 	uint16_t log_nb_desc = rte_log2_u32(qp_conf->nb_descriptors);
 	uint32_t umem_size = RTE_BIT32(log_nb_desc) *
-			      MLX5_CRYPTO_WQE_SET_SIZE +
+			      priv->wqe_set_size +
 			      sizeof(*qp->db_rec) * 2;
 	uint32_t alloc_size = sizeof(*qp);
 	struct mlx5_devx_cq_attr cq_attr = {
@@ -317,7 +380,9 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	if (dev->data->queue_pairs[qp_id] != NULL)
 		mlx5_crypto_queue_pair_release(dev, qp_id);
 	alloc_size = RTE_ALIGN(alloc_size, RTE_CACHE_LINE_SIZE);
-	alloc_size += sizeof(struct rte_crypto_op *) * RTE_BIT32(log_nb_desc);
+	alloc_size += (sizeof(struct rte_crypto_op *) +
+		       sizeof(struct mlx5_devx_obj *)) *
+		       RTE_BIT32(log_nb_desc);
 	qp = rte_zmalloc_socket(__func__, alloc_size, RTE_CACHE_LINE_SIZE,
 				socket_id);
 	if (qp == NULL) {
@@ -362,8 +427,7 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	attr.wq_umem_id = qp->umem_obj->umem_id;
 	attr.wq_umem_offset = 0;
 	attr.dbr_umem_id = qp->umem_obj->umem_id;
-	attr.dbr_address = RTE_BIT64(log_nb_desc) *
-			   MLX5_CRYPTO_WQE_SET_SIZE;
+	attr.dbr_address = RTE_BIT64(log_nb_desc) * priv->wqe_set_size;
 	qp->qp_obj = mlx5_devx_cmd_create_qp(priv->ctx, &attr);
 	if (qp->qp_obj == NULL) {
 		DRV_LOG(ERR, "Failed to create QP(%u).", rte_errno);
@@ -372,8 +436,17 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	qp->db_rec = RTE_PTR_ADD(qp->umem_buf, (uintptr_t)attr.dbr_address);
 	if (mlx5_crypto_qp2rts(qp))
 		goto error;
-	qp->ops = (struct rte_crypto_op **)RTE_ALIGN((uintptr_t)(qp + 1),
+	qp->mkey = (struct mlx5_devx_obj **)RTE_ALIGN((uintptr_t)(qp + 1),
 							   RTE_CACHE_LINE_SIZE);
+	qp->ops = (struct rte_crypto_op **)(qp->mkey + RTE_BIT32(log_nb_desc));
+	qp->entries_n = 1 << log_nb_desc;
+	if (mlx5_crypto_indirect_mkeys_prepare(priv, qp)) {
+		DRV_LOG(ERR, "Cannot allocate indirect memory regions.");
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	mlx5_crypto_qp_init(priv, qp);
+	qp->priv = priv;
 	dev->data->queue_pairs[qp_id] = qp;
 	return 0;
 error:
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 48ca1cb9a2..12e5890bf7 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -16,7 +16,6 @@
 
 #define MLX5_CRYPTO_DEK_HTABLE_SZ (1 << 11)
 #define MLX5_CRYPTO_KEY_LENGTH 80
-#define MLX5_CRYPTO_WQE_SET_SIZE 1024
 
 struct mlx5_crypto_priv {
 	TAILQ_ENTRY(mlx5_crypto_priv) next;
@@ -24,6 +23,7 @@ struct mlx5_crypto_priv {
 	struct rte_pci_device *pci_dev;
 	struct rte_cryptodev *crypto_dev;
 	void *uar; /* User Access Region. */
+	volatile uint64_t *uar_addr;
 	uint32_t pdn; /* Protection Domain number. */
 	uint32_t max_segs_num; /* Maximum supported data segs. */
 	struct ibv_pd *pd;
@@ -39,13 +39,21 @@ struct mlx5_crypto_priv {
 };
 
 struct mlx5_crypto_qp {
+	struct mlx5_crypto_priv *priv;
 	struct mlx5_devx_cq cq_obj;
 	struct mlx5_devx_obj *qp_obj;
+	struct rte_cryptodev_stats stats;
 	struct mlx5dv_devx_umem *umem_obj;
 	void *umem_buf;
 	volatile uint32_t *db_rec;
 	struct rte_crypto_op **ops;
+	struct mlx5_devx_obj **mkey; /* WQE's indirect mekys. */
 	struct mlx5_mr_ctrl mr_ctrl;
+	uint8_t *wqe;
+	uint16_t entries_n;
+	uint16_t pi;
+	uint16_t ci;
+	uint16_t db_pi;
 };
 
 struct mlx5_crypto_dek {
-- 
2.27.0

[dpdk-dev] [PATCH v6 12/15] crypto/mlx5: add enqueue and dequeue operations

[Shiri Kuzin]

From: Suanming Mou <suanmingm@nvidia.com>

The crypto operations are done with the WQE set which contains
one UMR WQE and one rdma write WQE. Most segments of the WQE
set are initialized properly during queue setup, only limited
segments are initialized according to the crypto detail in the
datapath process.

This commit adds the enquue and dequeue operations and updates
the WQE set segments accordingly.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
Signed-off-by: Michael Baum <michaelba@nvidia.com>
---
 doc/guides/cryptodevs/features/mlx5.ini |   5 +
 drivers/crypto/mlx5/mlx5_crypto.c       | 286 +++++++++++++++++++++++-
 drivers/crypto/mlx5/mlx5_crypto.h       |   3 +
 3 files changed, 290 insertions(+), 4 deletions(-)

diff --git a/doc/guides/cryptodevs/features/mlx5.ini b/doc/guides/cryptodevs/features/mlx5.ini
index bd757b5211..a89526add0 100644
--- a/doc/guides/cryptodevs/features/mlx5.ini
+++ b/doc/guides/cryptodevs/features/mlx5.ini
@@ -6,6 +6,11 @@
 [Features]
 Symmetric crypto       = Y
 HW Accelerated         = Y
+In Place SGL           = Y
+OOP SGL In SGL Out     = Y
+OOP SGL In LB  Out     = Y
+OOP LB  In SGL Out     = Y
+OOP LB  In LB  Out     = Y
 Cipher multiple data units = Y
 Cipher wrapped key     = Y
 
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index e5f8d96ff7..b467739a3c 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -25,6 +25,10 @@
 
 #define MLX5_CRYPTO_FEATURE_FLAGS \
 	(RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | \
+	 RTE_CRYPTODEV_FF_IN_PLACE_SGL | RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT | \
+	 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | \
+	 RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT | \
+	 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | \
 	 RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY | \
 	 RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS)
 
@@ -297,6 +301,279 @@ mlx5_crypto_qp2rts(struct mlx5_crypto_qp *qp)
 	return 0;
 }
 
+static __rte_noinline uint32_t
+mlx5_crypto_get_block_size(struct rte_crypto_op *op)
+{
+	uint32_t bl = op->sym->cipher.data.length;
+
+	switch (bl) {
+	case (1 << 20):
+		return RTE_BE32(MLX5_BLOCK_SIZE_1MB << MLX5_BLOCK_SIZE_OFFSET);
+	case (1 << 12):
+		return RTE_BE32(MLX5_BLOCK_SIZE_4096B <<
+				MLX5_BLOCK_SIZE_OFFSET);
+	case (1 << 9):
+		return RTE_BE32(MLX5_BLOCK_SIZE_512B << MLX5_BLOCK_SIZE_OFFSET);
+	default:
+		DRV_LOG(ERR, "Unknown block size: %u.", bl);
+		return UINT32_MAX;
+	}
+}
+
+/**
+ * Query LKey from a packet buffer for QP. If not found, add the mempool.
+ *
+ * @param priv
+ *   Pointer to the priv object.
+ * @param addr
+ *   Search key.
+ * @param mr_ctrl
+ *   Pointer to per-queue MR control structure.
+ * @param ol_flags
+ *   Mbuf offload features.
+ *
+ * @return
+ *   Searched LKey on success, UINT32_MAX on no match.
+ */
+static __rte_always_inline uint32_t
+mlx5_crypto_addr2mr(struct mlx5_crypto_priv *priv, uintptr_t addr,
+		    struct mlx5_mr_ctrl *mr_ctrl, uint64_t ol_flags)
+{
+	uint32_t lkey;
+
+	/* Check generation bit to see if there's any change on existing MRs. */
+	if (unlikely(*mr_ctrl->dev_gen_ptr != mr_ctrl->cur_gen))
+		mlx5_mr_flush_local_cache(mr_ctrl);
+	/* Linear search on MR cache array. */
+	lkey = mlx5_mr_lookup_lkey(mr_ctrl->cache, &mr_ctrl->mru,
+				   MLX5_MR_CACHE_N, addr);
+	if (likely(lkey != UINT32_MAX))
+		return lkey;
+	/* Take slower bottom-half on miss. */
+	return mlx5_mr_addr2mr_bh(priv->pd, 0, &priv->mr_scache, mr_ctrl, addr,
+				  !!(ol_flags & EXT_ATTACHED_MBUF));
+}
+
+static __rte_always_inline uint32_t
+mlx5_crypto_klm_set(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp,
+		      struct rte_mbuf *mbuf, struct mlx5_wqe_dseg *klm,
+		      uint32_t offset, uint32_t *remain)
+{
+	uint32_t data_len = (rte_pktmbuf_data_len(mbuf) - offset);
+	uintptr_t addr = rte_pktmbuf_mtod_offset(mbuf, uintptr_t, offset);
+
+	if (data_len > *remain)
+		data_len = *remain;
+	*remain -= data_len;
+	klm->bcount = rte_cpu_to_be_32(data_len);
+	klm->pbuf = rte_cpu_to_be_64(addr);
+	klm->lkey = mlx5_crypto_addr2mr(priv, addr, &qp->mr_ctrl,
+					mbuf->ol_flags);
+	return klm->lkey;
+
+}
+
+static __rte_always_inline uint32_t
+mlx5_crypto_klms_set(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp,
+		     struct rte_crypto_op *op, struct rte_mbuf *mbuf,
+		     struct mlx5_wqe_dseg *klm)
+{
+	uint32_t remain_len = op->sym->cipher.data.length;
+	uint32_t nb_segs = mbuf->nb_segs;
+	uint32_t klm_n = 1u;
+
+	/* First mbuf needs to take the cipher offset. */
+	if (unlikely(mlx5_crypto_klm_set(priv, qp, mbuf, klm,
+		     op->sym->cipher.data.offset, &remain_len) == UINT32_MAX)) {
+		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return 0;
+	}
+	while (remain_len) {
+		nb_segs--;
+		mbuf = mbuf->next;
+		if (unlikely(mbuf == NULL || nb_segs == 0)) {
+			op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+			return 0;
+		}
+		if (unlikely(mlx5_crypto_klm_set(priv, qp, mbuf, ++klm, 0,
+						 &remain_len) == UINT32_MAX)) {
+			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			return 0;
+		}
+		klm_n++;
+	}
+	return klm_n;
+}
+
+static __rte_always_inline int
+mlx5_crypto_wqe_set(struct mlx5_crypto_priv *priv,
+			 struct mlx5_crypto_qp *qp,
+			 struct rte_crypto_op *op,
+			 struct mlx5_umr_wqe *umr)
+{
+	struct mlx5_crypto_session *sess = get_sym_session_private_data
+				(op->sym->session, mlx5_crypto_driver_id);
+	struct mlx5_wqe_cseg *cseg = &umr->ctr;
+	struct mlx5_wqe_mkey_cseg *mkc = &umr->mkc;
+	struct mlx5_wqe_dseg *klms = &umr->kseg[0];
+	struct mlx5_wqe_umr_bsf_seg *bsf = ((struct mlx5_wqe_umr_bsf_seg *)
+				      RTE_PTR_ADD(umr, priv->umr_wqe_size)) - 1;
+	uint32_t ds;
+	bool ipl = op->sym->m_dst == NULL || op->sym->m_dst == op->sym->m_src;
+	/* Set UMR WQE. */
+	uint32_t klm_n = mlx5_crypto_klms_set(priv, qp, op,
+				   ipl ? op->sym->m_src : op->sym->m_dst, klms);
+
+	if (unlikely(klm_n == 0))
+		return 0;
+	bsf->bs_bpt_eo_es = sess->bs_bpt_eo_es;
+	if (unlikely(!sess->bsp_res)) {
+		bsf->bsp_res = mlx5_crypto_get_block_size(op);
+		if (unlikely(bsf->bsp_res == UINT32_MAX)) {
+			op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+			return 0;
+		}
+	} else {
+		bsf->bsp_res = sess->bsp_res;
+	}
+	bsf->raw_data_size = rte_cpu_to_be_32(op->sym->cipher.data.length);
+	memcpy(bsf->xts_initial_tweak,
+	       rte_crypto_op_ctod_offset(op, uint8_t *, sess->iv_offset), 16);
+	bsf->res_dp = sess->dek_id;
+	mkc->len = rte_cpu_to_be_64(op->sym->cipher.data.length);
+	cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) | MLX5_OPCODE_UMR);
+	qp->db_pi += priv->umr_wqe_stride;
+	/* Set RDMA_WRITE WQE. */
+	cseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size);
+	klms = RTE_PTR_ADD(cseg, sizeof(struct mlx5_rdma_write_wqe));
+	if (!ipl) {
+		klm_n = mlx5_crypto_klms_set(priv, qp, op, op->sym->m_src,
+					     klms);
+		if (unlikely(klm_n == 0))
+			return 0;
+	} else {
+		memcpy(klms, &umr->kseg[0], sizeof(*klms) * klm_n);
+	}
+	ds = 2 + klm_n;
+	cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj->id << 8) | ds);
+	cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) |
+							MLX5_OPCODE_RDMA_WRITE);
+	ds = RTE_ALIGN(ds, 4);
+	qp->db_pi += ds >> 2;
+	/* Set NOP WQE if needed. */
+	if (priv->max_rdmar_ds > ds) {
+		cseg += ds;
+		ds = priv->max_rdmar_ds - ds;
+		cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj->id << 8) | ds);
+		cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) |
+							       MLX5_OPCODE_NOP);
+		qp->db_pi += ds >> 2; /* Here, DS is 4 aligned for sure. */
+	}
+	qp->wqe = (uint8_t *)cseg;
+	return 1;
+}
+
+static __rte_always_inline void
+mlx5_crypto_uar_write(uint64_t val, struct mlx5_crypto_priv *priv)
+{
+#ifdef RTE_ARCH_64
+	*priv->uar_addr = val;
+#else /* !RTE_ARCH_64 */
+	rte_spinlock_lock(&priv->uar32_sl);
+	*(volatile uint32_t *)priv->uar_addr = val;
+	rte_io_wmb();
+	*((volatile uint32_t *)priv->uar_addr + 1) = val >> 32;
+	rte_spinlock_unlock(&priv->uar32_sl);
+#endif
+}
+
+static uint16_t
+mlx5_crypto_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
+			  uint16_t nb_ops)
+{
+	struct mlx5_crypto_qp *qp = queue_pair;
+	struct mlx5_crypto_priv *priv = qp->priv;
+	struct mlx5_umr_wqe *umr;
+	struct rte_crypto_op *op;
+	uint16_t mask = qp->entries_n - 1;
+	uint16_t remain = qp->entries_n - (qp->pi - qp->ci);
+
+	if (remain < nb_ops)
+		nb_ops = remain;
+	else
+		remain = nb_ops;
+	if (unlikely(remain == 0))
+		return 0;
+	do {
+		op = *ops++;
+		umr = RTE_PTR_ADD(qp->umem_buf, priv->wqe_set_size * qp->pi);
+		if (unlikely(mlx5_crypto_wqe_set(priv, qp, op, umr) == 0)) {
+			if (remain != nb_ops)
+				break;
+			return 0;
+		}
+		qp->ops[qp->pi] = op;
+		qp->pi = (qp->pi + 1) & mask;
+	} while (--remain);
+	rte_io_wmb();
+	qp->db_rec[MLX5_SND_DBR] = rte_cpu_to_be_32(qp->db_pi);
+	rte_wmb();
+	mlx5_crypto_uar_write(*(volatile uint64_t *)qp->wqe, qp->priv);
+	rte_wmb();
+	return nb_ops;
+}
+
+static __rte_noinline void
+mlx5_crypto_cqe_err_handle(struct mlx5_crypto_qp *qp, struct rte_crypto_op *op)
+{
+	const uint32_t idx = qp->ci & (qp->entries_n - 1);
+	volatile struct mlx5_err_cqe *cqe = (volatile struct mlx5_err_cqe *)
+							&qp->cq_obj.cqes[idx];
+
+	op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+	DRV_LOG(ERR, "CQE ERR:%x.\n", rte_be_to_cpu_32(cqe->syndrome));
+}
+
+static uint16_t
+mlx5_crypto_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
+			  uint16_t nb_ops)
+{
+	struct mlx5_crypto_qp *qp = queue_pair;
+	volatile struct mlx5_cqe *restrict cqe;
+	struct rte_crypto_op *restrict op;
+	const unsigned int cq_size = qp->entries_n;
+	const unsigned int mask = cq_size - 1;
+	uint32_t idx;
+	uint32_t next_idx = qp->ci & mask;
+	const uint16_t max = RTE_MIN((uint16_t)(qp->pi - qp->ci), nb_ops);
+	uint16_t i = 0;
+	int ret;
+
+	if (unlikely(max == 0))
+		return 0;
+	do {
+		idx = next_idx;
+		next_idx = (qp->ci + 1) & mask;
+		op = qp->ops[idx];
+		cqe = &qp->cq_obj.cqes[idx];
+		ret = check_cqe(cqe, cq_size, qp->ci);
+		rte_io_rmb();
+		if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) {
+			if (unlikely(ret != MLX5_CQE_STATUS_HW_OWN))
+				mlx5_crypto_cqe_err_handle(qp, op);
+			break;
+		}
+		op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+		ops[i++] = op;
+		qp->ci++;
+	} while (i < max);
+	if (likely(i != 0)) {
+		rte_io_wmb();
+		qp->cq_obj.db_rec[0] = rte_cpu_to_be_32(qp->ci);
+	}
+	return i;
+}
+
 static void
 mlx5_crypto_qp_init(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp)
 {
@@ -521,8 +798,9 @@ mlx5_crypto_hw_global_prepare(struct mlx5_crypto_priv *priv)
 	if (mlx5_crypto_pd_create(priv) != 0)
 		return -1;
 	priv->uar = mlx5_devx_alloc_uar(priv->ctx, -1);
-	if (priv->uar == NULL || mlx5_os_get_devx_uar_reg_addr(priv->uar) ==
-	    NULL) {
+	if (priv->uar)
+		priv->uar_addr = mlx5_os_get_devx_uar_reg_addr(priv->uar);
+	if (priv->uar == NULL || priv->uar_addr == NULL) {
 		rte_errno = errno;
 		claim_zero(mlx5_glue->dealloc_pd(priv->pd));
 		DRV_LOG(ERR, "Failed to allocate UAR.");
@@ -752,8 +1030,8 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	DRV_LOG(INFO,
 		"Crypto device %s was created successfully.", ibv->name);
 	crypto_dev->dev_ops = &mlx5_crypto_ops;
-	crypto_dev->dequeue_burst = NULL;
-	crypto_dev->enqueue_burst = NULL;
+	crypto_dev->dequeue_burst = mlx5_crypto_dequeue_burst;
+	crypto_dev->enqueue_burst = mlx5_crypto_enqueue_burst;
 	crypto_dev->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
 	crypto_dev->driver_id = mlx5_crypto_driver_id;
 	priv = crypto_dev->data->dev_private;
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 12e5890bf7..8e90735182 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -36,6 +36,9 @@ struct mlx5_crypto_priv {
 	uint16_t umr_wqe_size;
 	uint16_t umr_wqe_stride;
 	uint16_t max_rdmar_ds;
+#ifndef RTE_ARCH_64
+	rte_spinlock_t uar32_sl;
+#endif /* RTE_ARCH_64 */
 };
 
 struct mlx5_crypto_qp {
-- 
2.27.0

[dpdk-dev] [PATCH v6 13/15] crypto/mlx5: add statistic get and reset operations

[Shiri Kuzin]

From: Suanming Mou <suanmingm@nvidia.com>

This commit adds mlx5 crypto statistic get and reset operations.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 40 ++++++++++++++++++++++++++++---
 1 file changed, 37 insertions(+), 3 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index b467739a3c..0ecca32cce 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -508,13 +508,17 @@ mlx5_crypto_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
 		op = *ops++;
 		umr = RTE_PTR_ADD(qp->umem_buf, priv->wqe_set_size * qp->pi);
 		if (unlikely(mlx5_crypto_wqe_set(priv, qp, op, umr) == 0)) {
-			if (remain != nb_ops)
+			qp->stats.enqueue_err_count++;
+			if (remain != nb_ops) {
+				qp->stats.enqueued_count -= remain;
 				break;
+			}
 			return 0;
 		}
 		qp->ops[qp->pi] = op;
 		qp->pi = (qp->pi + 1) & mask;
 	} while (--remain);
+	qp->stats.enqueued_count += nb_ops;
 	rte_io_wmb();
 	qp->db_rec[MLX5_SND_DBR] = rte_cpu_to_be_32(qp->db_pi);
 	rte_wmb();
@@ -531,6 +535,7 @@ mlx5_crypto_cqe_err_handle(struct mlx5_crypto_qp *qp, struct rte_crypto_op *op)
 							&qp->cq_obj.cqes[idx];
 
 	op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+	qp->stats.dequeue_err_count++;
 	DRV_LOG(ERR, "CQE ERR:%x.\n", rte_be_to_cpu_32(cqe->syndrome));
 }
 
@@ -570,6 +575,7 @@ mlx5_crypto_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
 	if (likely(i != 0)) {
 		rte_io_wmb();
 		qp->cq_obj.db_rec[0] = rte_cpu_to_be_32(qp->ci);
+		qp->stats.dequeued_count += i;
 	}
 	return i;
 }
@@ -731,14 +737,42 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	return -1;
 }
 
+static void
+mlx5_crypto_stats_get(struct rte_cryptodev *dev,
+		      struct rte_cryptodev_stats *stats)
+{
+	int qp_id;
+
+	for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
+		struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
+
+		stats->enqueued_count += qp->stats.enqueued_count;
+		stats->dequeued_count += qp->stats.dequeued_count;
+		stats->enqueue_err_count += qp->stats.enqueue_err_count;
+		stats->dequeue_err_count += qp->stats.dequeue_err_count;
+	}
+}
+
+static void
+mlx5_crypto_stats_reset(struct rte_cryptodev *dev)
+{
+	int qp_id;
+
+	for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
+		struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
+
+		memset(&qp->stats, 0, sizeof(qp->stats));
+	}
+}
+
 static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_configure			= mlx5_crypto_dev_configure,
 	.dev_start			= mlx5_crypto_dev_start,
 	.dev_stop			= mlx5_crypto_dev_stop,
 	.dev_close			= mlx5_crypto_dev_close,
 	.dev_infos_get			= mlx5_crypto_dev_infos_get,
-	.stats_get			= NULL,
-	.stats_reset			= NULL,
+	.stats_get			= mlx5_crypto_stats_get,
+	.stats_reset			= mlx5_crypto_stats_reset,
 	.queue_pair_setup		= mlx5_crypto_queue_pair_setup,
 	.queue_pair_release		= mlx5_crypto_queue_pair_release,
 	.sym_session_get_size		= mlx5_crypto_sym_session_get_size,
-- 
2.27.0

[dpdk-dev] [PATCH v6 14/15] test/crypto: add mlx5 crypto driver

[Shiri Kuzin]

In order to test the new mlx5 crypto PMD, the driver is added to the
crypto test application.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
---
 app/test/test_cryptodev.c      | 7 +++++++
 app/test/test_cryptodev.h      | 1 +
 doc/guides/cryptodevs/mlx5.rst | 3 +++
 3 files changed, 11 insertions(+)

diff --git a/app/test/test_cryptodev.c b/app/test/test_cryptodev.c
index 39db52b17a..76a5ae789b 100644
--- a/app/test/test_cryptodev.c
+++ b/app/test/test_cryptodev.c
@@ -14572,6 +14572,12 @@ test_cryptodev_cpu_aesni_gcm(void)
 	return rc;
 }
 
+static int
+test_cryptodev_mlx5(void /*argv __rte_unused, int argc __rte_unused*/)
+{
+	return run_cryptodev_testsuite(RTE_STR(CRYPTODEV_NAME_MLX5_PMD));
+}
+
 static int
 test_cryptodev_null(void)
 {
@@ -14788,6 +14794,7 @@ REGISTER_TEST_COMMAND(cryptodev_openssl_autotest, test_cryptodev_openssl);
 REGISTER_TEST_COMMAND(cryptodev_aesni_gcm_autotest, test_cryptodev_aesni_gcm);
 REGISTER_TEST_COMMAND(cryptodev_cpu_aesni_gcm_autotest,
 	test_cryptodev_cpu_aesni_gcm);
+REGISTER_TEST_COMMAND(cryptodev_mlx5_autotest, test_cryptodev_mlx5);
 REGISTER_TEST_COMMAND(cryptodev_null_autotest, test_cryptodev_null);
 REGISTER_TEST_COMMAND(cryptodev_sw_snow3g_autotest, test_cryptodev_sw_snow3g);
 REGISTER_TEST_COMMAND(cryptodev_sw_kasumi_autotest, test_cryptodev_sw_kasumi);
diff --git a/app/test/test_cryptodev.h b/app/test/test_cryptodev.h
index f81f8e372f..e280da910c 100644
--- a/app/test/test_cryptodev.h
+++ b/app/test/test_cryptodev.h
@@ -71,6 +71,7 @@
 #define CRYPTODEV_NAME_CAAM_JR_PMD	crypto_caam_jr
 #define CRYPTODEV_NAME_NITROX_PMD	crypto_nitrox_sym
 #define CRYPTODEV_NAME_BCMFS_PMD	crypto_bcmfs
+#define CRYPTODEV_NAME_MLX5_PMD		mlx5_crypto
 
 enum cryptodev_api_test_type {
 	CRYPTODEV_API_TEST = 0,
diff --git a/doc/guides/cryptodevs/mlx5.rst b/doc/guides/cryptodevs/mlx5.rst
index dd4705b744..2023dd1b3b 100644
--- a/doc/guides/cryptodevs/mlx5.rst
+++ b/doc/guides/cryptodevs/mlx5.rst
@@ -92,6 +92,9 @@ The mlxreg dedicated tool should be used as follows:
   The "wrapped_crypto_operational" value will be "0x00000001" if the mode was
   successfully changed to operational mode.
 
+  The mlx5 crypto PMD can be verfied by running the test application:
+  ./build/app/test/dpdk-test -c 1 -n 1 -w <dev>,class=crypto,wcs_file=<file_path>
+  RTE>>cryptodev_mlx5_autotest
 
 Driver options
 --------------
-- 
2.27.0

[dpdk-dev] [PATCH v6 15/15] test/crypto: add data-unit and wrapped vectors

[Shiri Kuzin]

The AES-XTS algorithm supports using a wrapped key.
In AES-XTS the data-unit defines the data block size to be
encrypted\decrypted.

Add AES-XTS vectors with a wrapped key.
Add a variable stating whether the key is wrapped or not.
Add the AES-XTS data-unit.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
---
 app/test/test_cryptodev.h                  |    2 +-
 app/test/test_cryptodev_aes_test_vectors.h | 1340 ++++++++++++++++++++
 app/test/test_cryptodev_blockcipher.c      |   10 +-
 app/test/test_cryptodev_blockcipher.h      |    2 +
 4 files changed, 1352 insertions(+), 2 deletions(-)

diff --git a/app/test/test_cryptodev.h b/app/test/test_cryptodev.h
index e280da910c..683fe3ddc8 100644
--- a/app/test/test_cryptodev.h
+++ b/app/test/test_cryptodev.h
@@ -19,7 +19,7 @@
 #define DEFAULT_NUM_XFORMS              (2)
 #define NUM_MBUFS                       (8191)
 #define MBUF_CACHE_SIZE                 (256)
-#define MBUF_DATAPAYLOAD_SIZE		(2048 + DIGEST_BYTE_LENGTH_SHA512)
+#define MBUF_DATAPAYLOAD_SIZE		(4096 + DIGEST_BYTE_LENGTH_SHA512)
 #define MBUF_SIZE			(sizeof(struct rte_mbuf) + \
 		RTE_PKTMBUF_HEADROOM + MBUF_DATAPAYLOAD_SIZE)
 
diff --git a/app/test/test_cryptodev_aes_test_vectors.h b/app/test/test_cryptodev_aes_test_vectors.h
index ec53874d9f..f10dd4a59d 100644
--- a/app/test/test_cryptodev_aes_test_vectors.h
+++ b/app/test/test_cryptodev_aes_test_vectors.h
@@ -2609,6 +2609,1298 @@ blockcipher_test_data aes_test_data_xts_key_64_pt_48 = {
 	}
 };
 
+static const uint8_t plaintext_aes256xts_512bytes[] = {
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79
+};
+static const uint8_t ciphertext_aes256xts_512bytes[] = {
+	0xD3, 0x94, 0x3E, 0xC9, 0xD3, 0x43, 0x13, 0xD3,
+	0x07, 0x7E, 0x51, 0x11, 0x97, 0xA5, 0xB1, 0xB2,
+	0xB0, 0x55, 0xF8, 0xD6, 0xA2, 0x10, 0x78, 0x76,
+	0x37, 0x95, 0x4F, 0x25, 0x99, 0xCC, 0x12, 0xD6,
+	0xBB, 0x2A, 0x24, 0xF4, 0x2D, 0x82, 0xFF, 0x56,
+	0xEC, 0x55, 0x3B, 0xDD, 0xF9, 0xE1, 0xC8, 0x48,
+	0x68, 0x16, 0xD2, 0x2B, 0x95, 0x7C, 0x1F, 0xCB,
+	0x32, 0xD6, 0x8F, 0x9A, 0x2E, 0xF7, 0x9B, 0xBE,
+	0x72, 0xFC, 0x55, 0x33, 0x27, 0x21, 0x2F, 0x69,
+	0xCA, 0x76, 0xA5, 0x9B, 0x21, 0x2E, 0x40, 0x57,
+	0x2C, 0x1C, 0x98, 0x41, 0x9A, 0x5E, 0x55, 0x38,
+	0xDE, 0xC2, 0x09, 0x57, 0x32, 0xA3, 0x34, 0x40,
+	0xC1, 0x8D, 0xCF, 0x70, 0x15, 0xF2, 0x6F, 0x4F,
+	0x6A, 0x04, 0xA6, 0x6D, 0xFF, 0x53, 0x25, 0x6A,
+	0x0E, 0xD0, 0x87, 0x63, 0xA1, 0x6C, 0xB1, 0x99,
+	0x4A, 0x42, 0xF5, 0xF6, 0xEA, 0xA4, 0xEB, 0x6D,
+	0x70, 0x9B, 0x0F, 0x85, 0xE2, 0x43, 0x6C, 0x27,
+	0x54, 0x57, 0x52, 0x1C, 0xCF, 0x72, 0x11, 0x83,
+	0xC3, 0xF5, 0xC6, 0xB6, 0x07, 0xEC, 0x1A, 0xF5,
+	0xAC, 0xA1, 0xF2, 0x3A, 0x01, 0x53, 0x0C, 0xA4,
+	0x40, 0x19, 0xAF, 0x7B, 0x2D, 0xE7, 0x56, 0x8B,
+	0x71, 0x5F, 0x8D, 0x96, 0xF1, 0x5D, 0x33, 0xAC,
+	0xE7, 0xAD, 0x96, 0xDB, 0xBB, 0xF7, 0xF5, 0x3F,
+	0x0E, 0x02, 0x2E, 0x80, 0xE9, 0xA2, 0x30, 0xD2,
+	0x82, 0x65, 0xEC, 0x1A, 0xB2, 0xE8, 0x98, 0x23,
+	0x42, 0x79, 0x43, 0x85, 0x3B, 0xF9, 0xFB, 0xBB,
+	0xC5, 0x29, 0xAD, 0x95, 0xAE, 0x22, 0x5B, 0x26,
+	0xDF, 0x76, 0x65, 0x37, 0x17, 0xBC, 0x58, 0xBB,
+	0x1B, 0x0A, 0x71, 0xDC, 0x68, 0xFF, 0x90, 0x36,
+	0x37, 0xB7, 0x49, 0x5E, 0x49, 0x4E, 0xE4, 0x1E,
+	0x24, 0x39, 0x48, 0xC7, 0x68, 0x19, 0xED, 0x94,
+	0xB5, 0xF6, 0x85, 0xFA, 0xE6, 0xB8, 0x2B, 0x9E,
+	0x07, 0x9E, 0xFB, 0x1D, 0x61, 0x06, 0x47, 0x2A,
+	0x3E, 0x1E, 0xD8, 0x52, 0xFB, 0xB7, 0xE3, 0xB4,
+	0x0D, 0xA8, 0x15, 0x1E, 0x98, 0x02, 0xBD, 0x89,
+	0x89, 0xE3, 0x38, 0x2C, 0xAB, 0x50, 0x25, 0x30,
+	0xB4, 0x5E, 0xA5, 0xCD, 0xA8, 0x9B, 0xA4, 0x2A,
+	0xED, 0x19, 0x3B, 0xC3, 0x05, 0x07, 0x57, 0xB5,
+	0x52, 0x11, 0x74, 0x95, 0x51, 0x5A, 0xD8, 0xED,
+	0xDF, 0x37, 0x91, 0x83, 0x27, 0xD5, 0x08, 0x82,
+	0xB3, 0x42, 0x08, 0xC7, 0x81, 0x35, 0x5F, 0x58,
+	0x28, 0x69, 0x0D, 0x97, 0x1D, 0x28, 0xE6, 0xB0,
+	0x58, 0x93, 0xCE, 0x2A, 0xB4, 0x7D, 0x4B, 0x83,
+	0x20, 0x1E, 0x08, 0xEF, 0x68, 0x51, 0xEB, 0xB4,
+	0xFA, 0x78, 0xB5, 0xE5, 0x2D, 0x93, 0x07, 0x99,
+	0xBB, 0xDD, 0x9A, 0x4E, 0xD6, 0xE7, 0x31, 0x9F,
+	0x4D, 0xB4, 0x05, 0x45, 0x89, 0x59, 0x42, 0xFF,
+	0x0C, 0xD4, 0xC2, 0xF3, 0xA0, 0xF7, 0xD3, 0xCE,
+	0x26, 0x2F, 0x9B, 0xCB, 0x98, 0x24, 0xDE, 0x50,
+	0xF6, 0x81, 0x24, 0xA3, 0xB4, 0x99, 0x94, 0xDB,
+	0xF5, 0xF4, 0x74, 0xAC, 0xED, 0xF1, 0xD6, 0x2F,
+	0x89, 0x48, 0x41, 0x24, 0xBB, 0xA7, 0x64, 0x48,
+	0xAE, 0x36, 0xE3, 0xF3, 0xA8, 0x37, 0x4F, 0xE6,
+	0x24, 0x92, 0xC4, 0x27, 0x86, 0x68, 0xC4, 0xA2,
+	0x2F, 0xB2, 0xA8, 0xFF, 0x27, 0x5F, 0x59, 0xB9,
+	0x86, 0x89, 0x5A, 0x09, 0xB8, 0xDA, 0xD6, 0xBA,
+	0x26, 0x86, 0x51, 0xC2, 0xDF, 0xCF, 0x07, 0x49,
+	0x1C, 0xB7, 0x1A, 0x80, 0xD4, 0x97, 0x13, 0xAA,
+	0xE8, 0x38, 0xA7, 0x2C, 0xB7, 0xE2, 0xE0, 0x33,
+	0xF9, 0xC3, 0x21, 0xF6, 0x2A, 0xAA, 0x1A, 0xF7,
+	0x39, 0x1F, 0x2F, 0x7E, 0x38, 0xC8, 0x82, 0x3F,
+	0xCC, 0xD8, 0xE2, 0xD9, 0x02, 0xE6, 0x8E, 0x31,
+	0xA7, 0x37, 0xE8, 0x59, 0x3F, 0xE1, 0x60, 0x1C,
+	0x74, 0x52, 0xE4, 0x49, 0xF7, 0x21, 0x60, 0x11
+};
+static const struct
+blockcipher_test_data aes_test_data_xts_wrapped_key_48_pt_512_du_512 = {
+	.crypto_algo = RTE_CRYPTO_CIPHER_AES_XTS,
+	.cipher_key = {
+		.data = {
+			0x94, 0xD8, 0xAE, 0xD7, 0x74, 0xBF, 0x51, 0xF1,
+			0x2D, 0x19, 0x0B, 0x9A, 0xB3, 0xDC, 0x1E, 0xA9,
+			0x5C, 0xC5, 0x47, 0xC1, 0xBB, 0x94, 0xDB, 0xFA,
+			0x2B, 0x1E, 0x54, 0xA5, 0x76, 0xAD, 0x07, 0x4B,
+			0x46, 0xE0, 0xAA, 0xF9, 0xF7, 0x5C, 0x64, 0x2D,
+			0x8F, 0x04, 0x17, 0x72, 0xE5, 0x8D, 0xFE, 0x9C
+		},
+		.len = 48
+	},
+	.iv = {
+		.data = {
+			0x4F, 0xAE, 0xF7, 0x11, 0x7C, 0xDA, 0x59, 0xC6,
+			0x6E, 0x4B, 0x92, 0x01, 0x3E, 0x76, 0x8A, 0xD5
+		},
+		.len = 16
+	},
+	.plaintext = {
+		.data = plaintext_aes256xts_512bytes,
+		.len = 512
+	},
+	.ciphertext = {
+		.data = ciphertext_aes256xts_512bytes,
+		.len = 512
+	},
+	.xts_dataunit_len = 512,
+	.wrapped_key = true
+};
+static const uint8_t plaintext_aes256xts_4096bytes[] = {
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79
+};
+static const uint8_t ciphertext_aes256xts_4096bytes[] = {
+	0xD3, 0x94, 0x3E, 0xC9, 0xD3, 0x43, 0x13, 0xD3,
+	0x07, 0x7E, 0x51, 0x11, 0x97, 0xA5, 0xB1, 0xB2,
+	0xB0, 0x55, 0xF8, 0xD6, 0xA2, 0x10, 0x78, 0x76,
+	0x37, 0x95, 0x4F, 0x25, 0x99, 0xCC, 0x12, 0xD6,
+	0xBB, 0x2A, 0x24, 0xF4, 0x2D, 0x82, 0xFF, 0x56,
+	0xEC, 0x55, 0x3B, 0xDD, 0xF9, 0xE1, 0xC8, 0x48,
+	0x68, 0x16, 0xD2, 0x2B, 0x95, 0x7C, 0x1F, 0xCB,
+	0x32, 0xD6, 0x8F, 0x9A, 0x2E, 0xF7, 0x9B, 0xBE,
+	0x72, 0xFC, 0x55, 0x33, 0x27, 0x21, 0x2F, 0x69,
+	0xCA, 0x76, 0xA5, 0x9B, 0x21, 0x2E, 0x40, 0x57,
+	0x2C, 0x1C, 0x98, 0x41, 0x9A, 0x5E, 0x55, 0x38,
+	0xDE, 0xC2, 0x09, 0x57, 0x32, 0xA3, 0x34, 0x40,
+	0xC1, 0x8D, 0xCF, 0x70, 0x15, 0xF2, 0x6F, 0x4F,
+	0x6A, 0x04, 0xA6, 0x6D, 0xFF, 0x53, 0x25, 0x6A,
+	0x0E, 0xD0, 0x87, 0x63, 0xA1, 0x6C, 0xB1, 0x99,
+	0x4A, 0x42, 0xF5, 0xF6, 0xEA, 0xA4, 0xEB, 0x6D,
+	0x70, 0x9B, 0x0F, 0x85, 0xE2, 0x43, 0x6C, 0x27,
+	0x54, 0x57, 0x52, 0x1C, 0xCF, 0x72, 0x11, 0x83,
+	0xC3, 0xF5, 0xC6, 0xB6, 0x07, 0xEC, 0x1A, 0xF5,
+	0xAC, 0xA1, 0xF2, 0x3A, 0x01, 0x53, 0x0C, 0xA4,
+	0x40, 0x19, 0xAF, 0x7B, 0x2D, 0xE7, 0x56, 0x8B,
+	0x71, 0x5F, 0x8D, 0x96, 0xF1, 0x5D, 0x33, 0xAC,
+	0xE7, 0xAD, 0x96, 0xDB, 0xBB, 0xF7, 0xF5, 0x3F,
+	0x0E, 0x02, 0x2E, 0x80, 0xE9, 0xA2, 0x30, 0xD2,
+	0x82, 0x65, 0xEC, 0x1A, 0xB2, 0xE8, 0x98, 0x23,
+	0x42, 0x79, 0x43, 0x85, 0x3B, 0xF9, 0xFB, 0xBB,
+	0xC5, 0x29, 0xAD, 0x95, 0xAE, 0x22, 0x5B, 0x26,
+	0xDF, 0x76, 0x65, 0x37, 0x17, 0xBC, 0x58, 0xBB,
+	0x1B, 0x0A, 0x71, 0xDC, 0x68, 0xFF, 0x90, 0x36,
+	0x37, 0xB7, 0x49, 0x5E, 0x49, 0x4E, 0xE4, 0x1E,
+	0x24, 0x39, 0x48, 0xC7, 0x68, 0x19, 0xED, 0x94,
+	0xB5, 0xF6, 0x85, 0xFA, 0xE6, 0xB8, 0x2B, 0x9E,
+	0x07, 0x9E, 0xFB, 0x1D, 0x61, 0x06, 0x47, 0x2A,
+	0x3E, 0x1E, 0xD8, 0x52, 0xFB, 0xB7, 0xE3, 0xB4,
+	0x0D, 0xA8, 0x15, 0x1E, 0x98, 0x02, 0xBD, 0x89,
+	0x89, 0xE3, 0x38, 0x2C, 0xAB, 0x50, 0x25, 0x30,
+	0xB4, 0x5E, 0xA5, 0xCD, 0xA8, 0x9B, 0xA4, 0x2A,
+	0xED, 0x19, 0x3B, 0xC3, 0x05, 0x07, 0x57, 0xB5,
+	0x52, 0x11, 0x74, 0x95, 0x51, 0x5A, 0xD8, 0xED,
+	0xDF, 0x37, 0x91, 0x83, 0x27, 0xD5, 0x08, 0x82,
+	0xB3, 0x42, 0x08, 0xC7, 0x81, 0x35, 0x5F, 0x58,
+	0x28, 0x69, 0x0D, 0x97, 0x1D, 0x28, 0xE6, 0xB0,
+	0x58, 0x93, 0xCE, 0x2A, 0xB4, 0x7D, 0x4B, 0x83,
+	0x20, 0x1E, 0x08, 0xEF, 0x68, 0x51, 0xEB, 0xB4,
+	0xFA, 0x78, 0xB5, 0xE5, 0x2D, 0x93, 0x07, 0x99,
+	0xBB, 0xDD, 0x9A, 0x4E, 0xD6, 0xE7, 0x31, 0x9F,
+	0x4D, 0xB4, 0x05, 0x45, 0x89, 0x59, 0x42, 0xFF,
+	0x0C, 0xD4, 0xC2, 0xF3, 0xA0, 0xF7, 0xD3, 0xCE,
+	0x26, 0x2F, 0x9B, 0xCB, 0x98, 0x24, 0xDE, 0x50,
+	0xF6, 0x81, 0x24, 0xA3, 0xB4, 0x99, 0x94, 0xDB,
+	0xF5, 0xF4, 0x74, 0xAC, 0xED, 0xF1, 0xD6, 0x2F,
+	0x89, 0x48, 0x41, 0x24, 0xBB, 0xA7, 0x64, 0x48,
+	0xAE, 0x36, 0xE3, 0xF3, 0xA8, 0x37, 0x4F, 0xE6,
+	0x24, 0x92, 0xC4, 0x27, 0x86, 0x68, 0xC4, 0xA2,
+	0x2F, 0xB2, 0xA8, 0xFF, 0x27, 0x5F, 0x59, 0xB9,
+	0x86, 0x89, 0x5A, 0x09, 0xB8, 0xDA, 0xD6, 0xBA,
+	0x26, 0x86, 0x51, 0xC2, 0xDF, 0xCF, 0x07, 0x49,
+	0x1C, 0xB7, 0x1A, 0x80, 0xD4, 0x97, 0x13, 0xAA,
+	0xE8, 0x38, 0xA7, 0x2C, 0xB7, 0xE2, 0xE0, 0x33,
+	0xF9, 0xC3, 0x21, 0xF6, 0x2A, 0xAA, 0x1A, 0xF7,
+	0x39, 0x1F, 0x2F, 0x7E, 0x38, 0xC8, 0x82, 0x3F,
+	0xCC, 0xD8, 0xE2, 0xD9, 0x02, 0xE6, 0x8E, 0x31,
+	0xA7, 0x37, 0xE8, 0x59, 0x3F, 0xE1, 0x60, 0x1C,
+	0x74, 0x52, 0xE4, 0x49, 0xF7, 0x21, 0x60, 0x11,
+	0x8F, 0xD0, 0xA1, 0x4B, 0xE3, 0xF1, 0x18, 0x3D,
+	0x90, 0x89, 0x54, 0x27, 0xA0, 0xF9, 0x32, 0x09,
+	0x3D, 0x9D, 0x9A, 0x09, 0x53, 0xC6, 0x7E, 0x95,
+	0x85, 0x53, 0x98, 0x4C, 0x23, 0xEA, 0x54, 0xBD,
+	0x6F, 0x50, 0xBC, 0x4C, 0xCF, 0x37, 0xC5, 0x7B,
+	0x4B, 0xCE, 0x84, 0xAF, 0xE2, 0xE2, 0x55, 0x49,
+	0xBC, 0xBF, 0x92, 0xCA, 0x1E, 0x5E, 0x10, 0xDF,
+	0x60, 0x87, 0x09, 0xA1, 0x4C, 0x1D, 0x7E, 0x1E,
+	0x59, 0xE9, 0xCF, 0xDA, 0x45, 0x3F, 0xE2, 0x0F,
+	0x53, 0x8D, 0x8B, 0x79, 0xBD, 0xD8, 0xB0, 0xE3,
+	0x5B, 0x7C, 0x55, 0x4A, 0x84, 0xF0, 0x1E, 0xF9,
+	0xE8, 0xF3, 0x09, 0x4D, 0x0B, 0xD7, 0x77, 0xCC,
+	0x3F, 0x70, 0x22, 0x7D, 0x17, 0x27, 0x48, 0x57,
+	0xE2, 0x36, 0xA0, 0x84, 0x3B, 0xDE, 0x05, 0x34,
+	0xEF, 0x55, 0x12, 0xF4, 0x9A, 0x99, 0x0D, 0x28,
+	0x86, 0x28, 0x99, 0x6B, 0x22, 0xEE, 0x63, 0xF0,
+	0x68, 0x9C, 0xE1, 0x70, 0xF6, 0x26, 0xD8, 0x3B,
+	0xF9, 0x57, 0x18, 0x3D, 0xAD, 0x66, 0xF0, 0xCF,
+	0x7B, 0x0C, 0x28, 0x4D, 0xB8, 0xEB, 0x7B, 0x04,
+	0x1E, 0x7D, 0x40, 0x5F, 0x5A, 0x1E, 0x7E, 0x08,
+	0x7F, 0x4C, 0x1E, 0x18, 0xE5, 0x3E, 0x6C, 0x90,
+	0x3C, 0x89, 0x13, 0x2A, 0xC4, 0x2A, 0x94, 0xB5,
+	0x3E, 0x18, 0x1C, 0x4C, 0xBA, 0xEA, 0x86, 0xD2,
+	0x05, 0xA9, 0x59, 0x9C, 0x80, 0xC2, 0x45, 0xAD,
+	0x30, 0x99, 0x18, 0x6A, 0x2F, 0x73, 0x8C, 0xF0,
+	0xFE, 0xA4, 0xBD, 0x44, 0x3E, 0xEB, 0x98, 0x75,
+	0x48, 0x08, 0x57, 0x45, 0xD8, 0x41, 0xDE, 0x61,
+	0x6D, 0x06, 0x93, 0xC4, 0x99, 0x1B, 0x23, 0xB5,
+	0x12, 0x22, 0x5C, 0xC7, 0x9E, 0x18, 0xEA, 0x64,
+	0xDB, 0xCE, 0x1A, 0xAC, 0x5D, 0x9B, 0x80, 0xE2,
+	0xBF, 0x3E, 0xC2, 0xA4, 0x78, 0x4F, 0xF1, 0xE3,
+	0x7D, 0x2A, 0x20, 0x94, 0x13, 0xCD, 0xF3, 0x1C,
+	0x33, 0x9C, 0xC1, 0x59, 0x85, 0x52, 0xCB, 0xDB,
+	0x03, 0xDF, 0x11, 0xE6, 0xAB, 0x95, 0x82, 0x65,
+	0x7A, 0x88, 0x73, 0xEE, 0xBA, 0x21, 0x1C, 0x2F,
+	0xCD, 0xD7, 0xC6, 0xE5, 0x13, 0xDE, 0x7A, 0x9E,
+	0xEE, 0x83, 0x8D, 0xC6, 0x47, 0x63, 0xE0, 0xC7,
+	0xC4, 0xBE, 0x19, 0x25, 0xEE, 0xCC, 0x0A, 0x13,
+	0x18, 0x9D, 0x34, 0x5B, 0x55, 0x6C, 0xC1, 0x6E,
+	0xBF, 0x5A, 0xC5, 0x61, 0x75, 0x77, 0x49, 0x8C,
+	0x67, 0x61, 0xE8, 0x72, 0x87, 0xE8, 0xCA, 0xBE,
+	0x6E, 0xC3, 0xD7, 0x81, 0x8C, 0x78, 0x79, 0xC8,
+	0x72, 0xDA, 0x1A, 0x40, 0x7D, 0x60, 0xE2, 0x5A,
+	0x47, 0x38, 0xA1, 0x21, 0x22, 0x6B, 0x54, 0x74,
+	0xDD, 0xF3, 0xBC, 0x96, 0x28, 0x7E, 0xC2, 0x8B,
+	0x13, 0xEE, 0x53, 0xBC, 0x34, 0x67, 0x07, 0x87,
+	0xD5, 0x6B, 0x93, 0x22, 0x21, 0xB9, 0xED, 0x17,
+	0xE4, 0xA1, 0x96, 0xB2, 0xC6, 0xFF, 0x79, 0xA0,
+	0xA7, 0xF9, 0xDD, 0x92, 0x78, 0xF7, 0xE3, 0x16,
+	0x79, 0xEF, 0xEF, 0x31, 0x4D, 0x1D, 0x75, 0xF9,
+	0xCF, 0x5A, 0x1B, 0x68, 0x16, 0x7F, 0xAF, 0x5F,
+	0x30, 0xB7, 0xEF, 0xF8, 0x94, 0x63, 0x73, 0x3D,
+	0xB3, 0x63, 0xE4, 0xE6, 0xD8, 0xAD, 0xF4, 0x80,
+	0x5E, 0x82, 0xA3, 0xFB, 0x3D, 0x0F, 0xCF, 0x59,
+	0xB8, 0x76, 0xAF, 0x27, 0x83, 0xE3, 0x2D, 0x6F,
+	0xE0, 0xF3, 0x11, 0xD5, 0xAE, 0x82, 0x14, 0x1D,
+	0x78, 0x95, 0xBF, 0x31, 0x22, 0x1B, 0x80, 0x12,
+	0x02, 0xD7, 0x4D, 0x1B, 0x92, 0xE3, 0x15, 0xBA,
+	0x67, 0xD6, 0x8F, 0xD4, 0xDA, 0xBF, 0xD4, 0x62,
+	0xAD, 0x76, 0xDA, 0x04, 0xA2, 0xEA, 0x98, 0xD3,
+	0xC3, 0x6E, 0x5F, 0x26, 0x3C, 0x5E, 0xD9, 0xEA,
+	0x09, 0xF0, 0x02, 0xFC, 0xD2, 0x11, 0xF8, 0xA8,
+	0x7E, 0xFF, 0x06, 0x28, 0x5B, 0xE5, 0x6F, 0x9A,
+	0x00, 0xE7, 0x7F, 0xB9, 0xFB, 0x59, 0xBB, 0xDD,
+	0x85, 0xF3, 0x40, 0xCE, 0xA3, 0x5E, 0x2E, 0x2E,
+	0x34, 0x91, 0x58, 0x41, 0x00, 0xB5, 0xE1, 0x88,
+	0x24, 0x51, 0xC7, 0xB6, 0xF3, 0x21, 0x52, 0x6E,
+	0xE7, 0xFC, 0x42, 0xE7, 0x9C, 0xCC, 0x1E, 0x51,
+	0x45, 0x39, 0xBE, 0x09, 0xFE, 0x1A, 0xC4, 0xF0,
+	0x79, 0xF4, 0x05, 0xC7, 0xA7, 0xF8, 0x0F, 0xB6,
+	0x5A, 0x7B, 0xD7, 0xE1, 0x6F, 0xF0, 0x9D, 0x67,
+	0xA3, 0xE3, 0x3E, 0x2E, 0xB9, 0x8C, 0x83, 0x9E,
+	0xFD, 0x2E, 0xA2, 0x92, 0x99, 0x3C, 0xC0, 0x99,
+	0x01, 0xAB, 0x0D, 0xFA, 0x55, 0x96, 0x04, 0x60,
+	0x1A, 0xAD, 0x4C, 0xBB, 0x3D, 0xBB, 0x7D, 0x8B,
+	0x9F, 0x28, 0x85, 0x7D, 0xB9, 0xE4, 0x05, 0x79,
+	0x7B, 0x63, 0xDD, 0x7F, 0x4D, 0xE7, 0x50, 0xD9,
+	0x41, 0xFF, 0x53, 0xB1, 0xCE, 0x42, 0x7B, 0xD6,
+	0x05, 0x1B, 0x4E, 0xAF, 0xC4, 0x8C, 0x17, 0xC8,
+	0x52, 0xBD, 0x03, 0x3B, 0x92, 0x57, 0x4E, 0xA8,
+	0x15, 0xC3, 0x26, 0x1C, 0x55, 0xC1, 0xFF, 0xAE,
+	0xA9, 0x26, 0x2D, 0xA7, 0x8E, 0x3A, 0x7F, 0xA3,
+	0x48, 0xA5, 0xBC, 0x14, 0x84, 0xF2, 0x90, 0xCE,
+	0x35, 0x0F, 0x64, 0x6B, 0xD8, 0x1C, 0x12, 0xFE,
+	0x5A, 0x4F, 0x0E, 0xCE, 0x81, 0x4E, 0x79, 0x6B,
+	0xCF, 0x56, 0xA7, 0xDB, 0x24, 0xBC, 0xB0, 0x84,
+	0x4C, 0xB0, 0xDA, 0xBE, 0xE6, 0x8F, 0xD7, 0x8E,
+	0x0E, 0xA0, 0xD3, 0x55, 0xC2, 0x4A, 0x34, 0x1C,
+	0xF9, 0xC7, 0x3D, 0x29, 0x70, 0x8B, 0xF0, 0x99,
+	0x61, 0xF5, 0x11, 0xFB, 0x82, 0xE2, 0x67, 0x35,
+	0x60, 0x78, 0x47, 0x81, 0x2A, 0x74, 0x5E, 0x4D,
+	0x48, 0xD3, 0x7C, 0x32, 0xCA, 0x1B, 0xD2, 0xA2,
+	0x5C, 0x3A, 0x2F, 0xCE, 0xB4, 0x6C, 0x3A, 0x6A,
+	0x8F, 0x67, 0x46, 0x12, 0xE7, 0xAE, 0x6A, 0x3B,
+	0x99, 0x04, 0x5E, 0x96, 0xD0, 0xB9, 0x84, 0xF6,
+	0xA7, 0x64, 0x11, 0xE8, 0x0C, 0x51, 0xFD, 0x3F,
+	0x18, 0xFA, 0xE8, 0x52, 0xD9, 0x4B, 0x99, 0x7A,
+	0x25, 0x2B, 0x1B, 0x21, 0xAD, 0x8C, 0xFE, 0x0D,
+	0x34, 0x51, 0x91, 0x75, 0x55, 0x6F, 0xEB, 0x9F,
+	0x42, 0xDC, 0x73, 0x7D, 0x31, 0x0A, 0x74, 0x13,
+	0x80, 0xB8, 0xC3, 0xED, 0x73, 0x9D, 0x79, 0x42,
+	0xC0, 0x33, 0xAB, 0xC1, 0xCB, 0xB9, 0xD0, 0xBE,
+	0xA0, 0x78, 0xB8, 0x3B, 0xEB, 0x3D, 0x1A, 0x3F,
+	0xFB, 0x9B, 0xAA, 0x8F, 0x89, 0xF9, 0xD1, 0x22,
+	0x82, 0xE6, 0x66, 0xEE, 0x2A, 0xFD, 0x9F, 0xF8,
+	0x92, 0x7E, 0x10, 0xF5, 0xD5, 0x23, 0x0B, 0xB1,
+	0xD6, 0xF1, 0x7A, 0x3D, 0x73, 0xE9, 0xCE, 0x7F,
+	0xE6, 0x0B, 0x17, 0xBC, 0x23, 0xAE, 0x72, 0xB6,
+	0xFA, 0x19, 0x46, 0xBB, 0xFE, 0xA4, 0xC1, 0x64,
+	0xA8, 0x5E, 0xE8, 0xBB, 0x63, 0x58, 0x19, 0x50,
+	0xAA, 0x36, 0xC2, 0x4B, 0x38, 0x24, 0xD1, 0x2E,
+	0xAE, 0xAD, 0x6E, 0x34, 0x64, 0xA8, 0xC8, 0xF2,
+	0x4E, 0x74, 0x5C, 0x98, 0xE8, 0xDF, 0x99, 0x8C,
+	0x41, 0x79, 0x60, 0x2D, 0xD5, 0xF4, 0xE3, 0xE9,
+	0x1D, 0xF6, 0x5A, 0xA9, 0x69, 0x8E, 0xA1, 0x4F,
+	0xD3, 0x1B, 0x09, 0xA8, 0x7A, 0xD1, 0xE1, 0xCF,
+	0xAC, 0xBA, 0xD0, 0xD1, 0x34, 0x34, 0x8F, 0xC1,
+	0xD1, 0xA8, 0xAF, 0x6E, 0x92, 0xE0, 0xB0, 0xF6,
+	0xF9, 0x08, 0xA0, 0xCB, 0x58, 0x34, 0xF6, 0x68,
+	0xA2, 0xBF, 0x05, 0x39, 0x63, 0xBA, 0x4F, 0xEF,
+	0xE3, 0x95, 0x69, 0xD5, 0x89, 0x7C, 0x64, 0x07,
+	0x13, 0x42, 0x14, 0xF1, 0xA6, 0x9B, 0x87, 0xE5,
+	0xF4, 0x49, 0xAE, 0x67, 0x65, 0xCC, 0xF2, 0x26,
+	0xF8, 0x31, 0xBD, 0x33, 0x6A, 0x87, 0x77, 0x4E,
+	0xB1, 0xEE, 0xA4, 0xA2, 0xC8, 0xA0, 0x4A, 0xC1,
+	0xDF, 0x55, 0xE0, 0xDE, 0x53, 0x15, 0x3B, 0xEC,
+	0x55, 0x32, 0xCA, 0x06, 0xE4, 0x78, 0x59, 0x63,
+	0x10, 0x68, 0xA9, 0x46, 0x1B, 0xEF, 0x73, 0x6D,
+	0x1A, 0x02, 0x64, 0x12, 0x76, 0x9B, 0xDB, 0x7C,
+	0x03, 0x35, 0x19, 0xE1, 0x58, 0x7A, 0x87, 0x0C,
+	0x76, 0xDC, 0xFC, 0xC0, 0x28, 0xE4, 0xA2, 0x07,
+	0x9C, 0x28, 0x05, 0x21, 0x13, 0x58, 0xEF, 0x05,
+	0xBB, 0xAB, 0x94, 0xA2, 0x93, 0xBC, 0x31, 0x61,
+	0x26, 0x39, 0x38, 0x0C, 0xC4, 0x67, 0xDA, 0xA5,
+	0xE4, 0x1E, 0x1B, 0xB6, 0xE5, 0x73, 0xD6, 0x6C,
+	0xEE, 0xBC, 0x9D, 0xB9, 0xE7, 0xD9, 0x45, 0x2F,
+	0xF2, 0xB6, 0x92, 0x54, 0x41, 0x05, 0xB7, 0xB7,
+	0xFC, 0x37, 0x63, 0x6A, 0xB4, 0xBE, 0xB8, 0x3E,
+	0xD8, 0x53, 0x3B, 0xF8, 0x7D, 0x9A, 0x05, 0xDF,
+	0x20, 0x02, 0x27, 0x64, 0x38, 0xFA, 0x7D, 0xAF,
+	0x7F, 0xFA, 0xD1, 0xB7, 0x32, 0xC5, 0x74, 0x3E,
+	0x04, 0xA2, 0x67, 0x79, 0x02, 0x2E, 0x6F, 0xA1,
+	0x27, 0x87, 0x07, 0xB5, 0x9F, 0x0A, 0x7D, 0x5E,
+	0x14, 0xA0, 0x31, 0x46, 0x3F, 0xA9, 0xDE, 0x98,
+	0xB9, 0x89, 0xA0, 0x4A, 0x7A, 0xBD, 0x15, 0xAE,
+	0x2D, 0x0B, 0x38, 0x9A, 0xD8, 0x0E, 0xD2, 0xBA,
+	0x6D, 0xA1, 0x04, 0x1E, 0x4E, 0x39, 0x87, 0x4B,
+	0xC8, 0x3C, 0x74, 0x35, 0x4D, 0xC8, 0x1B, 0x42,
+	0x06, 0x5B, 0x73, 0xB7, 0x33, 0x86, 0x4A, 0x10,
+	0x2A, 0x10, 0x16, 0x28, 0x6F, 0x2A, 0xE3, 0x86,
+	0xDE, 0xA3, 0x44, 0x23, 0xE2, 0x90, 0xC4, 0x20,
+	0x90, 0xE0, 0xB8, 0xE6, 0xA7, 0xB6, 0xD6, 0x92,
+	0xF4, 0xF8, 0x8A, 0xBC, 0xAC, 0x31, 0x47, 0x8F,
+	0xAA, 0xE0, 0xD9, 0xF7, 0xE3, 0xCB, 0x11, 0xA4,
+	0x6B, 0x05, 0xB3, 0xB8, 0x72, 0x69, 0xE6, 0xDD,
+	0x75, 0x0F, 0x20, 0x1D, 0x3F, 0xC6, 0x96, 0xA0,
+	0x18, 0xB6, 0x24, 0xA1, 0xA6, 0xFD, 0x0C, 0x80,
+	0x1E, 0xD2, 0x28, 0xA2, 0x1A, 0x27, 0xF4, 0x23,
+	0x59, 0x1A, 0xCC, 0x0F, 0xD4, 0x99, 0xD0, 0xB4,
+	0x1E, 0x91, 0xC7, 0xD8, 0x8F, 0x8C, 0x5B, 0xEB,
+	0xB5, 0x9F, 0xFF, 0x4F, 0xD0, 0xD5, 0xB7, 0x60,
+	0xCC, 0x0A, 0x10, 0x38, 0xBF, 0xA8, 0x2E, 0xCC,
+	0xEB, 0x26, 0xB0, 0x78, 0xB3, 0xE0, 0x40, 0xAF,
+	0xCD, 0x12, 0xC5, 0x3A, 0x24, 0xD8, 0xEE, 0x3A,
+	0x64, 0x83, 0x2E, 0xD9, 0x25, 0x21, 0x66, 0xA5,
+	0x28, 0xD1, 0xE1, 0x84, 0x25, 0x1B, 0x20, 0xB8,
+	0xF5, 0x76, 0xB6, 0x3E, 0x4B, 0xC6, 0xEC, 0xC1,
+	0xC7, 0xAC, 0xC4, 0xAD, 0xCE, 0xF0, 0xB4, 0x0F,
+	0x35, 0x1E, 0xCE, 0x4E, 0xE3, 0x57, 0x30, 0xFC,
+	0xF4, 0x9B, 0x86, 0xB0, 0xDD, 0x3F, 0x2F, 0xB6,
+	0x10, 0x20, 0xE4, 0x24, 0x17, 0x1C, 0x24, 0xC6,
+	0x89, 0xE4, 0x14, 0xAD, 0x2E, 0x41, 0x08, 0x33,
+	0x88, 0xB1, 0x6F, 0x11, 0x85, 0xAF, 0x58, 0x17,
+	0xE3, 0x91, 0xB4, 0x72, 0xA2, 0x7F, 0xA3, 0x98,
+	0xAF, 0xB7, 0x6B, 0x58, 0x76, 0xA3, 0x11, 0x1C,
+	0x8A, 0x1A, 0xE6, 0x58, 0x54, 0xB0, 0xB9, 0x6E,
+	0x46, 0xCB, 0x16, 0xC0, 0x63, 0x0C, 0xEE, 0xA2,
+	0xAE, 0xF6, 0x71, 0xEF, 0xD1, 0xB9, 0x3D, 0xB7,
+	0x76, 0xCE, 0x5B, 0x84, 0x66, 0x7C, 0x7D, 0xF1,
+	0x96, 0x60, 0x34, 0xF6, 0xD1, 0x64, 0x27, 0xD9,
+	0xF3, 0x78, 0x8B, 0xF4, 0xC3, 0x1D, 0x37, 0xC0,
+	0xF4, 0x4A, 0xD0, 0xA5, 0x9A, 0xEB, 0xDD, 0x79,
+	0x54, 0x5D, 0xEB, 0x04, 0xC1, 0xA4, 0xBC, 0xED,
+	0xE3, 0x74, 0xC3, 0xB9, 0x9A, 0x6A, 0xAA, 0x06,
+	0xD1, 0xF0, 0x0F, 0xC5, 0xEF, 0x7E, 0x0B, 0xC8,
+	0xF4, 0x94, 0x4E, 0x69, 0x0E, 0x36, 0x00, 0x13,
+	0x45, 0xCE, 0x68, 0x13, 0xFE, 0x7F, 0x29, 0xA2,
+	0x1D, 0x79, 0xDF, 0xF2, 0x27, 0xFB, 0xAE, 0x52,
+	0x05, 0x78, 0xD7, 0xB9, 0xF7, 0x38, 0x68, 0xD5,
+	0xBA, 0xD7, 0xCF, 0x09, 0xC6, 0xD2, 0x5B, 0xC6,
+	0x98, 0xE4, 0xEC, 0xD5, 0xE9, 0xC2, 0xA5, 0x1A,
+	0x52, 0xC8, 0xA7, 0xBA, 0x3D, 0x74, 0x75, 0x00,
+	0xAA, 0xDD, 0x6A, 0x3F, 0xB6, 0x2F, 0x08, 0xB7,
+	0x1C, 0x6B, 0x52, 0x0C, 0xC9, 0xE4, 0x4D, 0xF4,
+	0xC5, 0x26, 0x1F, 0x35, 0x41, 0x25, 0x68, 0x17,
+	0xA8, 0x81, 0x75, 0xF4, 0x66, 0x41, 0xB5, 0xE4,
+	0x1D, 0x92, 0xEE, 0xDA, 0x0F, 0x56, 0x76, 0xC6,
+	0xAA, 0x0F, 0xA8, 0x63, 0x8D, 0xF0, 0x69, 0x63,
+	0x93, 0x45, 0xBC, 0x76, 0x40, 0xBE, 0xA9, 0x96,
+	0x36, 0xAF, 0x2F, 0x6B, 0x3E, 0xAB, 0xF3, 0xC0,
+	0xD7, 0xD5, 0xB1, 0x23, 0x23, 0xA2, 0xA0, 0xC4,
+	0xC5, 0x70, 0xEF, 0x66, 0x79, 0x15, 0xF0, 0xD0,
+	0x40, 0x0A, 0x33, 0x0C, 0xF3, 0x32, 0x6D, 0x8D,
+	0xB4, 0x44, 0x46, 0x78, 0x3F, 0x8D, 0x75, 0x40,
+	0xA5, 0x60, 0xBC, 0x9B, 0x76, 0xDF, 0x25, 0xF4,
+	0xE9, 0xED, 0xAC, 0x74, 0x2F, 0x9A, 0x00, 0xC4,
+	0x2B, 0x52, 0x26, 0x79, 0x09, 0x19, 0x57, 0x89,
+	0x60, 0x14, 0xBE, 0x65, 0xBD, 0x7B, 0x4D, 0x7D,
+	0x9B, 0x8B, 0x9E, 0x72, 0x6C, 0x0C, 0x57, 0xC7,
+	0x00, 0x08, 0x38, 0x7C, 0x37, 0x45, 0x9D, 0x55,
+	0xA2, 0x62, 0x5E, 0x34, 0x19, 0x99, 0x31, 0x16,
+	0xF1, 0x14, 0x44, 0x2D, 0xE3, 0x7E, 0x22, 0xE1,
+	0xA2, 0xB8, 0x9A, 0x9F, 0xE0, 0x37, 0x29, 0xBB,
+	0xCD, 0x46, 0xEE, 0x0A, 0x62, 0x2B, 0x98, 0x34,
+	0xBA, 0x9E, 0x54, 0x1B, 0xB1, 0x5C, 0x4F, 0xE9,
+	0xAA, 0xE4, 0x95, 0x8C, 0xA4, 0xEF, 0xC2, 0xB1,
+	0x7F, 0xF9, 0x80, 0xDA, 0x55, 0x95, 0x92, 0xC0,
+	0x86, 0xF4, 0x2D, 0x99, 0x3E, 0x17, 0xDC, 0x55,
+	0xA6, 0x33, 0x85, 0x90, 0x31, 0xC8, 0xFF, 0x58,
+	0x83, 0xC5, 0xBA, 0x60, 0x20, 0x5F, 0x87, 0x29,
+	0x20, 0x5A, 0x7D, 0x44, 0x2B, 0xA0, 0xE2, 0x99,
+	0xC8, 0x70, 0xBE, 0x89, 0xC5, 0xBC, 0xF6, 0x0D,
+	0x04, 0xC0, 0x96, 0xD1, 0x5C, 0xD1, 0x90, 0x43,
+	0xD3, 0x7B, 0x73, 0x52, 0x30, 0xB6, 0xA9, 0x7C,
+	0x0A, 0xA3, 0x24, 0x0E, 0x80, 0xFE, 0xBE, 0x31,
+	0xFD, 0xB5, 0x96, 0x04, 0x2B, 0xCF, 0x0B, 0x28,
+	0x1F, 0x7A, 0xCF, 0xC4, 0x82, 0x78, 0x52, 0x30,
+	0xB1, 0x34, 0x12, 0x50, 0x03, 0x09, 0x1C, 0x8B,
+	0x80, 0x60, 0xE3, 0xA1, 0xE5, 0x61, 0xF7, 0xD7,
+	0xB6, 0x74, 0xBE, 0xD6, 0x58, 0x03, 0xD3, 0xE5,
+	0xF7, 0xAC, 0x07, 0x60, 0xB7, 0x8A, 0xEC, 0xFA,
+	0xC6, 0x0F, 0xF0, 0x20, 0x04, 0x6B, 0x8F, 0x61,
+	0x09, 0x92, 0x03, 0xFB, 0x85, 0x99, 0x94, 0x9D,
+	0x2E, 0x6A, 0xC2, 0x9F, 0x20, 0x46, 0x2A, 0x96,
+	0xED, 0x42, 0x7D, 0x64, 0xA9, 0xE4, 0x1B, 0xDE,
+	0x11, 0x20, 0x12, 0x93, 0xE6, 0x2B, 0xE5, 0x93,
+	0x48, 0x37, 0x8C, 0x5A, 0x54, 0x0D, 0xEB, 0xF0,
+	0x9F, 0x9D, 0xE4, 0xA5, 0xC4, 0x93, 0x6F, 0x6A,
+	0xE3, 0x99, 0x69, 0xD9, 0xFE, 0x0C, 0x4E, 0xEC,
+	0x8B, 0x30, 0x1F, 0x7A, 0xB8, 0xC8, 0x5B, 0x61,
+	0x8E, 0xC2, 0x10, 0x90, 0x57, 0xB4, 0x72, 0x58,
+	0x7F, 0x41, 0x29, 0x7E, 0xF9, 0xBE, 0x40, 0xC3,
+	0x6F, 0xA9, 0xE3, 0x00, 0xE9, 0xC8, 0xFD, 0x4B,
+	0xFD, 0x3F, 0xE3, 0x3F, 0x25, 0x22, 0xFD, 0xB7,
+	0x2D, 0x57, 0xEF, 0x91, 0x08, 0xF0, 0x20, 0x56,
+	0x30, 0xFA, 0x83, 0x69, 0xFD, 0x56, 0x5A, 0x9B,
+	0xCE, 0xF8, 0x28, 0x02, 0xB4, 0x91, 0x35, 0x75,
+	0x9E, 0x63, 0x99, 0x48, 0xCF, 0x35, 0xF5, 0x58,
+	0x0C, 0x48, 0x8F, 0x0A, 0x2D, 0x9A, 0xE6, 0x40,
+	0xF6, 0x21, 0xB5, 0x69, 0xC1, 0x09, 0x31, 0x00,
+	0xA3, 0xC1, 0x4C, 0x99, 0x70, 0x4F, 0x5A, 0x63,
+	0x17, 0x90, 0xB8, 0xF8, 0x3A, 0x0E, 0xFD, 0x67,
+	0xEA, 0x0E, 0xBA, 0x7B, 0x1E, 0xEF, 0x37, 0x84,
+	0xD5, 0x51, 0x37, 0x01, 0xD6, 0x93, 0x15, 0xDF,
+	0x56, 0x89, 0x0E, 0x54, 0xF5, 0x1D, 0xF7, 0xE5,
+	0xB7, 0xC1, 0xF1, 0xC2, 0xD9, 0x14, 0x6F, 0x40,
+	0x55, 0x67, 0x50, 0x7C, 0x58, 0x35, 0x8B, 0x39,
+	0xCB, 0xB5, 0x87, 0xF5, 0x55, 0x5E, 0x26, 0x8C,
+	0x5B, 0x73, 0x0E, 0xBB, 0x25, 0x51, 0x0E, 0xAD,
+	0x57, 0x72, 0x7B, 0x68, 0x83, 0x11, 0x1E, 0x3A,
+	0x3D, 0xA4, 0x7C, 0x18, 0xB7, 0x70, 0x18, 0xBC,
+	0x72, 0x03, 0x4A, 0xA1, 0xD2, 0xF9, 0xA9, 0x8A,
+	0x25, 0x45, 0x19, 0xEE, 0x93, 0x06, 0xB5, 0x09,
+	0x71, 0xC9, 0x2D, 0xFD, 0x2B, 0xF3, 0xC7, 0x64,
+	0x5F, 0xCE, 0x71, 0x1D, 0x81, 0x96, 0x67, 0xBF,
+	0x01, 0x39, 0x8C, 0xE7, 0xA2, 0xD0, 0x98, 0x57,
+	0x5A, 0xFD, 0x21, 0xC7, 0x46, 0xAA, 0xB4, 0xE4,
+	0x0E, 0xBE, 0xC6, 0x68, 0x3E, 0x38, 0xF5, 0xA2,
+	0xED, 0x73, 0xCC, 0x53, 0x7E, 0x7E, 0x03, 0x32,
+	0xDC, 0xB6, 0xC1, 0x03, 0x9E, 0xB3, 0x2A, 0xAD,
+	0xC0, 0xC3, 0x6E, 0x47, 0xFB, 0x1E, 0xB7, 0x0D,
+	0x86, 0x95, 0x09, 0xA6, 0x9D, 0x6F, 0x92, 0xFC,
+	0xFF, 0x2C, 0x7D, 0x09, 0x16, 0x68, 0x50, 0x3E,
+	0x4F, 0x23, 0x4C, 0x93, 0x95, 0x2A, 0xE1, 0x9B,
+	0x16, 0xF0, 0x0F, 0xFF, 0x79, 0xA8, 0x06, 0xF9,
+	0x70, 0x61, 0x72, 0x2C, 0xE8, 0x91, 0x01, 0x6D,
+	0x45, 0xE5, 0x82, 0x5D, 0x26, 0x21, 0xAD, 0x3D,
+	0x77, 0x73, 0x23, 0x04, 0x84, 0x27, 0xA3, 0x5D,
+	0x6D, 0xA8, 0x99, 0xC1, 0xCE, 0x4F, 0xA9, 0xF7,
+	0xAB, 0x5C, 0xDE, 0x01, 0xE6, 0x1E, 0xEF, 0xE6,
+	0xFD, 0xE0, 0x68, 0x85, 0x3E, 0xEE, 0xBF, 0xF1,
+	0x0D, 0x79, 0xF4, 0xA2, 0xB4, 0x14, 0xBC, 0x0C,
+	0x49, 0x77, 0x03, 0x71, 0x08, 0x3E, 0x40, 0xA6,
+	0xD7, 0x03, 0xFA, 0xE2, 0xFB, 0xC7, 0x59, 0x30,
+	0x6E, 0x07, 0x06, 0x1C, 0x7C, 0x47, 0xE5, 0x4C,
+	0x57, 0x0A, 0x91, 0x4A, 0x43, 0xE4, 0x8A, 0xCD,
+	0x6E, 0x92, 0x01, 0xE2, 0x52, 0xC1, 0x92, 0x34,
+	0x8E, 0x64, 0x0F, 0x39, 0x63, 0x53, 0xAB, 0xE5,
+	0x44, 0xD5, 0xAA, 0xAA, 0xF6, 0x03, 0x89, 0xB9,
+	0xDD, 0xB2, 0x2D, 0x56, 0x1A, 0xE0, 0x72, 0x5A,
+	0x52, 0x19, 0x46, 0xEA, 0xB3, 0xCE, 0xB3, 0x59,
+	0x46, 0x7A, 0xA7, 0x48, 0x37, 0x0C, 0x09, 0xBA,
+	0x92, 0x70, 0x17, 0x7F, 0xF5, 0xD3, 0x60, 0x44,
+	0xC4, 0xC6, 0xC6, 0x7D, 0xD2, 0x21, 0xAC, 0x3F,
+	0x62, 0x6C, 0xE9, 0xBA, 0x4C, 0xF3, 0x82, 0x7E,
+	0x6D, 0x3A, 0x92, 0xDC, 0x94, 0xE4, 0x5F, 0xA6,
+	0x8B, 0x66, 0xA0, 0xDD, 0xE2, 0x97, 0x83, 0xED,
+	0xF5, 0x9D, 0xDF, 0x74, 0x77, 0x23, 0x7D, 0xDA,
+	0xC4, 0xFB, 0x92, 0x1A, 0xD9, 0x37, 0x36, 0xD2,
+	0x88, 0xC9, 0xEA, 0x0F, 0x98, 0xBD, 0xC5, 0xF8,
+	0xAA, 0x19, 0x75, 0x12, 0x6A, 0x41, 0xB5, 0xB3,
+	0xB5, 0xA4, 0x96, 0xDC, 0x2B, 0x49, 0x86, 0x66,
+	0x35, 0xD8, 0x4A, 0x62, 0xB4, 0xCB, 0x1E, 0x27,
+	0xC1, 0xAD, 0x34, 0x0E, 0x26, 0x16, 0xF2, 0xC2,
+	0x22, 0x52, 0x84, 0xD8, 0xD1, 0x32, 0xB8, 0x9C,
+	0xFE, 0x64, 0x42, 0x9F, 0xE4, 0x69, 0xF0, 0xAE,
+	0x3B, 0xD9, 0x2C, 0xA0, 0x14, 0xEB, 0x69, 0x74,
+	0x7C, 0xE2, 0xA6, 0x60, 0xE1, 0x52, 0x1C, 0xCC,
+	0xBF, 0xE6, 0xA1, 0x83, 0x20, 0x5D, 0x9E, 0xA3,
+	0xFB, 0x84, 0x8B, 0x33, 0xE6, 0xC9, 0x32, 0x83,
+	0xC0, 0x3F, 0x98, 0x1D, 0x6E, 0xC0, 0x50, 0x71,
+	0x29, 0x60, 0x5F, 0x36, 0xB4, 0x68, 0x1D, 0xB9,
+	0x76, 0x73, 0xC3, 0x80, 0xC5, 0xBC, 0x59, 0x7B,
+	0x59, 0xB4, 0xE0, 0x6A, 0x80, 0xCD, 0x4D, 0x8C,
+	0x9E, 0xE0, 0x0B, 0x45, 0x7D, 0x54, 0xD4, 0xC4,
+	0x97, 0x6C, 0x54, 0xEF, 0x14, 0x64, 0xBD, 0x3B,
+	0xD7, 0xEE, 0xF4, 0xD1, 0x41, 0x76, 0x3A, 0x24,
+	0x7A, 0xC2, 0xCA, 0x68, 0x28, 0x53, 0x46, 0xF7,
+	0x1B, 0xDA, 0x4B, 0x7A, 0x56, 0x75, 0x86, 0xFB,
+	0x31, 0x2C, 0x27, 0xF9, 0x4D, 0x35, 0xA4, 0x82,
+	0xE7, 0x2F, 0x41, 0xB4, 0xCA, 0xCE, 0x75, 0x94,
+	0x08, 0x54, 0xE2, 0x9E, 0x99, 0xC9, 0x85, 0xDE,
+	0x6F, 0x80, 0x95, 0x59, 0x3E, 0x54, 0x9F, 0x31,
+	0xF8, 0xDE, 0xD0, 0xD7, 0xA6, 0xD4, 0xD3, 0xBB,
+	0xD9, 0xC7, 0x55, 0xDD, 0xAE, 0xAD, 0x9E, 0x57,
+	0x4A, 0x33, 0x5D, 0x7A, 0xA6, 0xA3, 0xCA, 0xF9,
+	0x4C, 0x5B, 0x51, 0xCC, 0x22, 0xBB, 0x76, 0x44,
+	0x17, 0xDE, 0x22, 0xA1, 0xDF, 0x80, 0x13, 0x7D,
+	0xE5, 0x34, 0x7E, 0x75, 0x73, 0x10, 0x40, 0xFB,
+	0x9A, 0x21, 0xCD, 0xD3, 0xD3, 0x84, 0xB6, 0x0C,
+	0x31, 0x1E, 0xB5, 0x42, 0xF4, 0x34, 0x11, 0x7F,
+	0x4A, 0x23, 0xA8, 0xA5, 0x8F, 0x20, 0xCD, 0xA9,
+	0xF2, 0xE4, 0xEE, 0xFA, 0x57, 0xD1, 0x22, 0x1C,
+	0xA5, 0xDC, 0x0B, 0x25, 0xFE, 0xC2, 0xA7, 0x7E,
+	0x09, 0x2E, 0xDA, 0x40, 0x9F, 0x6C, 0xC8, 0x71,
+	0x58, 0x91, 0x04, 0x25, 0xD0, 0x06, 0xEA, 0x1B,
+	0xCD, 0x9D, 0x50, 0xD8, 0x40, 0x24, 0xAC, 0xC3,
+	0xB4, 0x07, 0x6E, 0x76, 0xF4, 0x4C, 0xD8, 0x80,
+	0xD0, 0x20, 0xF5, 0x15, 0x6A, 0x0A, 0x12, 0xF8,
+	0x6B, 0x67, 0x77, 0x34, 0xAE, 0x60, 0x68, 0x13,
+	0x5B, 0x8E, 0xFF, 0x5E, 0x7A, 0x77, 0x67, 0x0D,
+	0xE6, 0x96, 0x43, 0x9F, 0x8F, 0x77, 0x5F, 0x97,
+	0x23, 0x91, 0x33, 0x72, 0xC7, 0x8A, 0xC7, 0x80,
+	0xCF, 0xE7, 0x71, 0x06, 0x25, 0xB7, 0x4B, 0x89,
+	0x6A, 0x46, 0x67, 0x19, 0x49, 0x44, 0x03, 0x52,
+	0x32, 0xB1, 0x8F, 0xE7, 0x9E, 0xDA, 0x03, 0x41,
+	0xA3, 0xAC, 0xE5, 0xF3, 0x96, 0xE6, 0xAC, 0xFF,
+	0xEC, 0x35, 0x4D, 0x83, 0xA9, 0xCE, 0x7C, 0x52,
+	0xF2, 0x36, 0x97, 0xF0, 0x28, 0x36, 0x54, 0x59,
+	0x96, 0xEA, 0xEE, 0xB2, 0xC1, 0xAB, 0xA4, 0x96,
+	0x62, 0xD4, 0x3C, 0xF0, 0x1F, 0x2D, 0x65, 0x0E,
+	0x46, 0x7E, 0x12, 0x31, 0x8F, 0xA7, 0x8D, 0x7A,
+	0x4A, 0x41, 0x15, 0x57, 0x90, 0xF6, 0xF1, 0xE8,
+	0xE8, 0xE3, 0x57, 0x7B, 0x55, 0x85, 0x95, 0x97,
+	0xB3, 0x29, 0x3D, 0x02, 0x73, 0x1E, 0x87, 0x1F,
+	0x01, 0x89, 0x06, 0x88, 0x9E, 0x8A, 0x2E, 0xE0,
+	0x99, 0xFC, 0xF0, 0x48, 0x60, 0x37, 0x65, 0x25,
+	0xDB, 0x89, 0xC9, 0x7A, 0x51, 0x7E, 0x35, 0x92,
+	0x00, 0xC9, 0x61, 0x3F, 0x88, 0xE3, 0x20, 0x01,
+	0x46, 0x5A, 0x2C, 0x37, 0x02, 0xC9, 0x15, 0x0F,
+	0xB2, 0xEB, 0xC3, 0x55, 0x18, 0xF0, 0x15, 0x1A,
+	0x08, 0x8E, 0xB8, 0x9D, 0x18, 0xE4, 0x9F, 0x34,
+	0x10, 0x67, 0x68, 0x57, 0x60, 0x61, 0xEC, 0xD5,
+	0xD9, 0xA8, 0x3A, 0xAB, 0xD6, 0xD2, 0x7A, 0x47,
+	0x3D, 0xA4, 0x08, 0x7C, 0x3E, 0x4D, 0x76, 0x08,
+	0x19, 0x22, 0xB2, 0x89, 0x57, 0x84, 0xC2, 0x98,
+	0x72, 0xB8, 0x8B, 0xE0, 0x85, 0xA1, 0x3A, 0xC2,
+	0xA0, 0x06, 0x43, 0x03, 0xCF, 0x4F, 0x27, 0x80,
+	0x48, 0x9A, 0xBC, 0xB3, 0x3C, 0xC4, 0x5E, 0xAC,
+	0x8B, 0x85, 0x6F, 0x21, 0xD6, 0xFE, 0x12, 0x90,
+	0x53, 0x2F, 0x65, 0x32, 0x8D, 0x03, 0xFE, 0xFE,
+	0x61, 0x04, 0x47, 0x24, 0x6A, 0xB5, 0x01, 0x98,
+	0xB9, 0x27, 0x10, 0xE1, 0x32, 0x3D, 0x2A, 0xA0,
+	0xC5, 0x70, 0xDE, 0x1E, 0x10, 0xD7, 0x01, 0x50,
+	0x4F, 0x87, 0xCA, 0xD9, 0xBF, 0x12, 0xEA, 0x38,
+	0x4C, 0x43, 0xD5, 0x5A, 0xEF, 0x3E, 0x21, 0x8E,
+	0x59, 0x77, 0x23, 0xED, 0x51, 0x09, 0x99, 0x73,
+	0xD2, 0x56, 0x04, 0xDD, 0x8F, 0x5F, 0xDF, 0x79,
+	0xFF, 0x16, 0x8C, 0xB0, 0xBA, 0x8A, 0x1A, 0x56,
+	0xAF, 0xCA, 0x19, 0xF2, 0x64, 0x1A, 0xF5, 0x1E,
+	0xA7, 0xA7, 0x84, 0x3D, 0xAD, 0xC1, 0x0E, 0x22,
+	0xA1, 0x45, 0xFC, 0xB4, 0x13, 0x91, 0x34, 0xB7,
+	0x48, 0xEF, 0x9E, 0xD9, 0x0B, 0xE3, 0x82, 0x75,
+	0x80, 0xC5, 0xD9, 0xA0, 0x77, 0xA3, 0xF9, 0xCC,
+	0x67, 0xDD, 0xCB, 0x28, 0xC6, 0xE8, 0x2C, 0xB8,
+	0xAC, 0x63, 0xBD, 0x3B, 0x28, 0x4A, 0xE9, 0x2D,
+	0x29, 0x84, 0xD7, 0x4F, 0x61, 0x06, 0xE3, 0x37,
+	0xC1, 0x58, 0x20, 0x5D, 0x0A, 0xE7, 0x45, 0x29,
+	0x7D, 0xED, 0x0F, 0xCE, 0x00, 0x95, 0x2A, 0x62,
+	0x38, 0xA8, 0x1A, 0x3A, 0x96, 0x0E, 0x56, 0xD9,
+	0x18, 0xC2, 0x25, 0xA5, 0xAA, 0x27, 0x0A, 0x6E,
+	0xDD, 0x1C, 0x35, 0x6C, 0xC1, 0x26, 0x90, 0xF6,
+	0x43, 0x1B, 0x34, 0xDA, 0xE1, 0x5D, 0x09, 0x7C,
+	0xBE, 0x0F, 0x40, 0xD3, 0x24, 0x82, 0x0B, 0xFF,
+	0xE6, 0xB7, 0x10, 0xD6, 0x36, 0xD0, 0xE6, 0xC0,
+	0xBE, 0x65, 0x4C, 0x83, 0xF1, 0xDA, 0xDE, 0xCE,
+	0xE8, 0x5A, 0x80, 0x88, 0xFE, 0x9B, 0x79, 0x54,
+	0xA3, 0xA4, 0x5A, 0x76, 0xD0, 0xE2, 0xCE, 0x92,
+	0x53, 0x7D, 0x9C, 0xDA, 0xA1, 0xED, 0x9F, 0x56,
+	0x05, 0x0A, 0xA4, 0x81, 0xC7, 0x82, 0x5B, 0xB8,
+	0xC7, 0xA8, 0x95, 0x21, 0x99, 0x0B, 0x0F, 0xD2,
+	0x66, 0x68, 0xC3, 0x07, 0x53, 0x93, 0x8C, 0x68,
+	0x5A, 0xF5, 0x6F, 0x5E, 0x07, 0x68, 0x70, 0xF2,
+	0x6A, 0x78, 0xA8, 0xDB, 0x24, 0x6F, 0xD9, 0x74,
+	0x38, 0x29, 0xBF, 0x50, 0xCC, 0xC8, 0x34, 0x82,
+	0x69, 0x1A, 0xF4, 0x1A, 0x95, 0xE1, 0x31, 0x39,
+	0x00, 0xBE, 0xF6, 0x25, 0x19, 0x5A, 0xF6, 0xA7,
+	0xB7, 0xC7, 0xAC, 0xFA, 0x52, 0x6D, 0xC7, 0xEA,
+	0xFF, 0xC4, 0xB0, 0x84, 0x41, 0x2D, 0x6B, 0x22,
+	0x57, 0x1F, 0x8A, 0x97, 0x71, 0xEF, 0x54, 0xA7,
+	0xA9, 0xB5, 0xA0, 0x7A, 0xFF, 0x52, 0xBC, 0x78,
+	0x56, 0x8C, 0x9C, 0x2A, 0xFB, 0x31, 0xA6, 0xC8,
+	0x87, 0xC8, 0x82, 0x72, 0x00, 0x68, 0x27, 0xDE,
+	0x54, 0x67, 0x4C, 0x36, 0x4E, 0xE2, 0x77, 0x7F,
+	0xAD, 0x15, 0x1B, 0xC9, 0x07, 0xC4, 0x84, 0x50,
+	0x84, 0x45, 0xB5, 0x1D, 0xCE, 0x3F, 0x7C, 0xDF,
+	0x73, 0x88, 0x3E, 0x6D, 0xAC, 0x18, 0x67, 0x2C,
+	0x1D, 0x31, 0xD5, 0x41, 0x6E, 0xFC, 0x3D, 0xFE,
+	0x5C, 0x6D, 0x3B, 0xCB, 0x67, 0x05, 0x44, 0x8B,
+	0x02, 0xB7, 0xF5, 0x05, 0xFD, 0x1D, 0x7E, 0x13,
+	0x90, 0xCE, 0xED, 0xD2, 0xAB, 0x08, 0xFF, 0xC3,
+	0x91, 0x2C, 0x79, 0x06, 0xDB, 0x54, 0xAB, 0xFF,
+	0xF6, 0x9D, 0xBB, 0xDC, 0x3C, 0xCD, 0x03, 0xE7,
+	0xD8, 0x1A, 0x4E, 0x7F, 0xCB, 0x1B, 0xA3, 0xCA,
+	0xDC, 0x14, 0xC5, 0xFE, 0x46, 0x38, 0x07, 0x82,
+	0x5E, 0x9A, 0x77, 0x9C, 0xB4, 0x44, 0x26, 0xBA,
+	0xC2, 0x27, 0xD0, 0xF4, 0x75, 0x67, 0x4A, 0x15,
+	0x2A, 0x55, 0x2A, 0x61, 0x87, 0x55, 0xA0, 0xFB,
+	0xE0, 0x93, 0xA5, 0xD7, 0xDF, 0x0D, 0x97, 0xD3,
+	0x93, 0x39, 0x0B, 0x5A, 0xC6, 0x86, 0x17, 0x7D,
+	0x6A, 0xA4, 0x07, 0x9C, 0xB9, 0x0F, 0x42, 0x15,
+	0x46, 0x71, 0x45, 0x70, 0x75, 0x4B, 0xD5, 0x80,
+	0x64, 0x62, 0x92, 0x50, 0xBC, 0x10, 0xA1, 0x68,
+	0x60, 0x5A, 0x6F, 0x31, 0x26, 0xCA, 0xB1, 0x48,
+	0xFB, 0xC6, 0xF4, 0x3A, 0xCA, 0x52, 0x20, 0x2F,
+	0x7D, 0xEC, 0x19, 0xF8, 0xAA, 0x27, 0xC2, 0x23,
+	0xD0, 0x8F, 0x60, 0xFC, 0x7F, 0xA0, 0xCB, 0xDC,
+	0xA2, 0xC6, 0xC4, 0xC5, 0x13, 0x00, 0xF3, 0x43,
+	0x48, 0x6D, 0xFD, 0x7D, 0xB9, 0xA8, 0x14, 0xB0,
+	0x0C, 0x72, 0x82, 0x2F, 0x99, 0x64, 0x41, 0x2B,
+	0xB3, 0x34, 0x73, 0x73, 0xF4, 0x26, 0x1D, 0x06,
+	0xDF, 0x6E, 0xF4, 0x20, 0x1E, 0x31, 0xE3, 0x8A,
+	0x01, 0x6C, 0xDB, 0x3C, 0xE3, 0xC6, 0xC4, 0xC5,
+	0xB8, 0x20, 0x51, 0xF1, 0xD6, 0xB0, 0x30, 0xB7,
+	0x2D, 0xDA, 0x95, 0x01, 0x0D, 0xED, 0xEE, 0x6F,
+	0x69, 0xFD, 0xCF, 0x9D, 0xDD, 0x05, 0xD6, 0xC0,
+	0xFE, 0x11, 0x67, 0xAF, 0x53, 0x94, 0x60, 0xFC,
+	0x56, 0xBA, 0x0C, 0x5F, 0xA7, 0x7E, 0xDA, 0x65
+};
+static const struct
+blockcipher_test_data aes_test_data_xts_wrapped_key_48_pt_4096_du_4096 = {
+	.crypto_algo = RTE_CRYPTO_CIPHER_AES_XTS,
+	.cipher_key = {
+		.data = {
+			0x94, 0xD8, 0xAE, 0xD7, 0x74, 0xBF, 0x51, 0xF1,
+			0x2D, 0x19, 0x0B, 0x9A, 0xB3, 0xDC, 0x1E, 0xA9,
+			0x5C, 0xC5, 0x47, 0xC1, 0xBB, 0x94, 0xDB, 0xFA,
+			0x2B, 0x1E, 0x54, 0xA5, 0x76, 0xAD, 0x07, 0x4B,
+			0x46, 0xE0, 0xAA, 0xF9, 0xF7, 0x5C, 0x64, 0x2D,
+			0x8F, 0x04, 0x17, 0x72, 0xE5, 0x8D, 0xFE, 0x9C
+		},
+		.len = 48
+	},
+	.iv = {
+		.data = {
+			0x4F, 0xAE, 0xF7, 0x11, 0x7C, 0xDA, 0x59, 0xC6,
+			0x6E, 0x4B, 0x92, 0x01, 0x3E, 0x76, 0x8A, 0xD5
+		},
+		.len = 16
+	},
+	.plaintext = {
+		.data = plaintext_aes256xts_4096bytes,
+		.len = 4096
+	},
+	.ciphertext = {
+		.data = ciphertext_aes256xts_4096bytes,
+		.len = 4096
+	},
+	.xts_dataunit_len = 4096,
+	.wrapped_key = true
+};
+
+static const struct
+blockcipher_test_data aes_test_data_xts_wrapped_key_48_pt_4096_du_0 = {
+	.crypto_algo = RTE_CRYPTO_CIPHER_AES_XTS,
+	.cipher_key = {
+		.data = {
+			0x94, 0xD8, 0xAE, 0xD7, 0x74, 0xBF, 0x51, 0xF1,
+			0x2D, 0x19, 0x0B, 0x9A, 0xB3, 0xDC, 0x1E, 0xA9,
+			0x5C, 0xC5, 0x47, 0xC1, 0xBB, 0x94, 0xDB, 0xFA,
+			0x2B, 0x1E, 0x54, 0xA5, 0x76, 0xAD, 0x07, 0x4B,
+			0x46, 0xE0, 0xAA, 0xF9, 0xF7, 0x5C, 0x64, 0x2D,
+			0x8F, 0x04, 0x17, 0x72, 0xE5, 0x8D, 0xFE, 0x9C
+		},
+		.len = 48
+	},
+	.iv = {
+		.data = {
+			0x4F, 0xAE, 0xF7, 0x11, 0x7C, 0xDA, 0x59, 0xC6,
+			0x6E, 0x4B, 0x92, 0x01, 0x3E, 0x76, 0x8A, 0xD5
+		},
+		.len = 16
+	},
+	.plaintext = {
+		.data = plaintext_aes256xts_4096bytes,
+		.len = 4096
+	},
+	.ciphertext = {
+		.data = ciphertext_aes256xts_4096bytes,
+		.len = 4096
+	},
+	.xts_dataunit_len = 0,
+	.wrapped_key = true
+};
+
+static const struct
+blockcipher_test_data aes_test_data_xts_wrapped_key_48_pt_512_du_0 = {
+	.crypto_algo = RTE_CRYPTO_CIPHER_AES_XTS,
+	.cipher_key = {
+		.data = {
+			0x94, 0xD8, 0xAE, 0xD7, 0x74, 0xBF, 0x51, 0xF1,
+			0x2D, 0x19, 0x0B, 0x9A, 0xB3, 0xDC, 0x1E, 0xA9,
+			0x5C, 0xC5, 0x47, 0xC1, 0xBB, 0x94, 0xDB, 0xFA,
+			0x2B, 0x1E, 0x54, 0xA5, 0x76, 0xAD, 0x07, 0x4B,
+			0x46, 0xE0, 0xAA, 0xF9, 0xF7, 0x5C, 0x64, 0x2D,
+			0x8F, 0x04, 0x17, 0x72, 0xE5, 0x8D, 0xFE, 0x9C
+		},
+		.len = 48
+	},
+	.iv = {
+		.data = {
+			0x4F, 0xAE, 0xF7, 0x11, 0x7C, 0xDA, 0x59, 0xC6,
+			0x6E, 0x4B, 0x92, 0x01, 0x3E, 0x76, 0x8A, 0xD5
+		},
+		.len = 16
+	},
+	.plaintext = {
+		.data = plaintext_aes256xts_512bytes,
+		.len = 512
+	},
+	.ciphertext = {
+		.data = ciphertext_aes256xts_512bytes,
+		.len = 512
+	},
+	.xts_dataunit_len = 0,
+	.wrapped_key = true
+};
+
+
 /* AES-DOCSIS-BPI test vectors */
 
 /* Multiple of AES block size */
@@ -3347,6 +4639,54 @@ static const struct blockcipher_test_case aes_cipheronly_test_cases[] = {
 		.test_data = &aes_test_data_xts_key_64_pt_48,
 		.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
 	},
+	{
+		.test_descr = "AES-256-XTS Encryption (512-byte plaintext"
+			      " Dataunit 512)",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_512_du_512,
+		.op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT,
+	},
+	{
+		.test_descr = "AES-256-XTS Decryption (512-byte plaintext"
+			      " Dataunit 512)",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_512_du_512,
+		.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
+	},
+	{
+		.test_descr = "AES-256-XTS Encryption (512-byte plaintext"
+			      " Dataunit 0)",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_512_du_0,
+		.op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT,
+	},
+	{
+		.test_descr = "AES-256-XTS Decryption (512-byte plaintext"
+			      " Dataunit 0)",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_512_du_0,
+		.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
+	},
+	{
+		.test_descr = "AES-256-XTS Encryption (4096-byte plaintext"
+			      " Dataunit 4096)",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_4096_du_4096,
+		.op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT,
+	},
+	{
+		.test_descr = "AES-256-XTS Decryption (4096-byte plaintext"
+			      " Dataunit 4096)",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_4096_du_4096,
+		.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
+	},
+	{
+		.test_descr = "AES-256-XTS Encryption (4096-byte plaintext"
+			      " Dataunit 0)",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_4096_du_0,
+		.op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT,
+	},
+	{
+		.test_descr = "AES-256-XTS Decryption (4096-byte plaintext"
+			      " Dataunit 0)",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_4096_du_0,
+		.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
+	},
 	{
 		.test_descr = "cipher-only - NULL algo - x8 - encryption",
 		.test_data = &null_test_data_chain_x8_multiple,
diff --git a/app/test/test_cryptodev_blockcipher.c b/app/test/test_cryptodev_blockcipher.c
index d342c7b859..53fd4718af 100644
--- a/app/test/test_cryptodev_blockcipher.c
+++ b/app/test/test_cryptodev_blockcipher.c
@@ -36,7 +36,8 @@ verify_algo_support(const struct blockcipher_test_case *t,
 		if (capability == NULL)
 			return -1;
 
-		if (cap_idx.algo.cipher != RTE_CRYPTO_CIPHER_NULL)
+		if (cap_idx.algo.cipher != RTE_CRYPTO_CIPHER_NULL &&
+				!(t->test_data->wrapped_key))
 			ret = rte_cryptodev_sym_capability_check_cipher(capability,
 							tdata->cipher_key.len,
 							tdata->iv.len);
@@ -145,6 +146,12 @@ test_blockcipher_one_case(const struct blockcipher_test_case *t,
 
 		nb_segs = 3;
 	}
+	if (!!(feat_flags & RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY) ^
+		tdata->wrapped_key) {
+		snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN,
+			"SKIPPED");
+		return TEST_SKIPPED;
+	}
 
 	if (global_api_test_type == CRYPTODEV_RAW_API_TEST &&
 		!(feat_flags & RTE_CRYPTODEV_FF_SYM_RAW_DP)) {
@@ -450,6 +457,7 @@ test_blockcipher_one_case(const struct blockcipher_test_case *t,
 		cipher_xform->cipher.key.data = cipher_key;
 		cipher_xform->cipher.key.length = tdata->cipher_key.len;
 		cipher_xform->cipher.iv.offset = IV_OFFSET;
+		cipher_xform->cipher.dataunit_len = tdata->xts_dataunit_len;
 
 		if (tdata->crypto_algo == RTE_CRYPTO_CIPHER_NULL)
 			cipher_xform->cipher.iv.length = 0;
diff --git a/app/test/test_cryptodev_blockcipher.h b/app/test/test_cryptodev_blockcipher.h
index a06241b06d..dcaa08ae22 100644
--- a/app/test/test_cryptodev_blockcipher.h
+++ b/app/test/test_cryptodev_blockcipher.h
@@ -97,6 +97,8 @@ struct blockcipher_test_data {
 
 	unsigned int cipher_offset;
 	unsigned int auth_offset;
+	uint16_t xts_dataunit_len;
+	bool wrapped_key;
 };
 
 struct unit_test_suite *
-- 
2.27.0

[dpdk-dev] [PATCH v7 00/16] drivers: introduce mlx5 crypto PMD

[Shiri Kuzin]

Add a new PMD for Nvidia devices- crypto PMD.
The crypto PMD will be supported on Nvidia ConnectX6.
The crypto PMD will add the support of encryption and decryption using
the AES-XTS symmetric algorithm.
The crypto PMD requires rdma-core and uses mlx5 DevX.

v2:
- Add data-path part.

v3:
- Rebase.

v4:
- Rebase + Address the following Akhil comments:
- Set HW feature flag in the capability patch.
- Fix mp object release in session clear.
- Some spelling and word missing in doc.
- Squash data-unit adjustment to the session operations commit.
- Wording: device argument -> devarg.

v5: 
- Add mlx5 crypto tests into test library.
- Update documentation according to Akhil comments.
- Fix memory region management.
- Fix multi segment case in data-path code.
- Split documentation to the correct commits according to Akhil comments.
- Rebase to new version.
- Change license to Nvidia license.

v6:
- Rebase to new version.
- Address David's comment and update log accordingly. 
- Update testing app commits as suggested by Akhil.

v7:
- Rebase to new version.
- Add scatter gather and OOP test cases.



Shiri Kuzin (11):
  drivers: introduce mlx5 crypto PMD
  crypto/mlx5: add DEK object management
  crypto/mlx5: add session operations
  crypto/mlx5: add basic operations
  crypto/mlx5: add queue pairs operations
  crypto/mlx5: add dev stop and start operations
  crypto/mlx5: add memory region management
  crypto/mlx5: create login object using DevX
  test/crypto: add mlx5 crypto driver
  test/crypto: add data-unit and wrapped vectors
  test/crypto: add AES-XTS multi segment OOP tests

Suanming Mou (5):
  crypto/mlx5: add keytag devarg
  crypto/mlx5: add maximum segments devarg
  crypto/mlx5: add WQE set initialization
  crypto/mlx5: add enqueue and dequeue operations
  crypto/mlx5: add statistic get and reset operations

 MAINTAINERS                                |    4 +
 app/test/test_cryptodev.c                  |    7 +
 app/test/test_cryptodev.h                  |    4 +-
 app/test/test_cryptodev_aes_test_vectors.h | 1403 +++++++++++++++++++-
 app/test/test_cryptodev_blockcipher.c      |   10 +-
 app/test/test_cryptodev_blockcipher.h      |    2 +
 doc/guides/cryptodevs/features/mlx5.ini    |   37 +
 doc/guides/cryptodevs/index.rst            |    1 +
 doc/guides/cryptodevs/mlx5.rst             |  153 +++
 doc/guides/rel_notes/release_21_08.rst     |    5 +
 drivers/common/mlx5/mlx5_common.h          |    1 +
 drivers/common/mlx5/mlx5_common_pci.c      |   14 +
 drivers/common/mlx5/mlx5_common_pci.h      |   21 +-
 drivers/crypto/meson.build                 |    1 +
 drivers/crypto/mlx5/meson.build            |   27 +
 drivers/crypto/mlx5/mlx5_crypto.c          | 1177 ++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto.h          |   91 ++
 drivers/crypto/mlx5/mlx5_crypto_dek.c      |  161 +++
 drivers/crypto/mlx5/mlx5_crypto_utils.h    |   19 +
 drivers/crypto/mlx5/version.map            |    3 +
 20 files changed, 3127 insertions(+), 14 deletions(-)
 create mode 100644 doc/guides/cryptodevs/features/mlx5.ini
 create mode 100644 doc/guides/cryptodevs/mlx5.rst
 create mode 100644 drivers/crypto/mlx5/meson.build
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto.c
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto.h
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_dek.c
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_utils.h
 create mode 100644 drivers/crypto/mlx5/version.map

-- 
2.27.0

[dpdk-dev] [PATCH v7 01/16] drivers: introduce mlx5 crypto PMD

[Shiri Kuzin]

Add a new PMD for Mellanox devices- crypto PMD.

The crypto PMD will be supported starting Nvidia ConnectX6 and
BlueField2.

The crypto PMD will add the support of encryption and decryption using
the AES-XTS symmetric algorithm.

The crypto PMD requires rdma-core and uses mlx5 DevX.

This patch adds the PCI probing, basic functions, build files and
log utility.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 MAINTAINERS                             |   4 +
 doc/guides/cryptodevs/features/mlx5.ini |  27 +++
 doc/guides/cryptodevs/index.rst         |   1 +
 doc/guides/cryptodevs/mlx5.rst          |  63 ++++++
 doc/guides/rel_notes/release_21_08.rst  |   5 +
 drivers/common/mlx5/mlx5_common.h       |   1 +
 drivers/common/mlx5/mlx5_common_pci.c   |  14 ++
 drivers/common/mlx5/mlx5_common_pci.h   |  21 +-
 drivers/crypto/meson.build              |   1 +
 drivers/crypto/mlx5/meson.build         |  26 +++
 drivers/crypto/mlx5/mlx5_crypto.c       | 275 ++++++++++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto_utils.h |  19 ++
 drivers/crypto/mlx5/version.map         |   3 +
 13 files changed, 450 insertions(+), 10 deletions(-)
 create mode 100644 doc/guides/cryptodevs/features/mlx5.ini
 create mode 100644 doc/guides/cryptodevs/mlx5.rst
 create mode 100644 drivers/crypto/mlx5/meson.build
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto.c
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_utils.h
 create mode 100644 drivers/crypto/mlx5/version.map

diff --git a/MAINTAINERS b/MAINTAINERS
index af2a91d7c4..ba2cb4ef42 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -1102,6 +1102,10 @@ F: drivers/crypto/octeontx2/
 F: doc/guides/cryptodevs/octeontx2.rst
 F: doc/guides/cryptodevs/features/octeontx2.ini
 
+Mellanox mlx5
+M: Matan Azrad <matan@nvidia.com>
+F: drivers/crypto/mlx5/
+
 Null Crypto
 M: Declan Doherty <declan.doherty@intel.com>
 F: drivers/crypto/null/
diff --git a/doc/guides/cryptodevs/features/mlx5.ini b/doc/guides/cryptodevs/features/mlx5.ini
new file mode 100644
index 0000000000..ceadd967b6
--- /dev/null
+++ b/doc/guides/cryptodevs/features/mlx5.ini
@@ -0,0 +1,27 @@
+;
+; Features of a mlx5 crypto driver.
+;
+; Refer to default.ini for the full list of available PMD features.
+;
+[Features]
+HW Accelerated         = Y
+
+;
+; Supported crypto algorithms of a mlx5 crypto driver.
+;
+[Cipher]
+
+;
+; Supported authentication algorithms of a mlx5 crypto driver.
+;
+[Auth]
+
+;
+; Supported AEAD algorithms of a mlx5 crypto driver.
+;
+[AEAD]
+
+;
+; Supported Asymmetric algorithms of a mlx5 crypto driver.
+;
+[Asymmetric]
diff --git a/doc/guides/cryptodevs/index.rst b/doc/guides/cryptodevs/index.rst
index 067e7d784e..0f981c77b5 100644
--- a/doc/guides/cryptodevs/index.rst
+++ b/doc/guides/cryptodevs/index.rst
@@ -23,6 +23,7 @@ Crypto Device Drivers
     octeontx
     octeontx2
     openssl
+    mlx5
     mvsam
     nitrox
     null
diff --git a/doc/guides/cryptodevs/mlx5.rst b/doc/guides/cryptodevs/mlx5.rst
new file mode 100644
index 0000000000..05a0a449e2
--- /dev/null
+++ b/doc/guides/cryptodevs/mlx5.rst
@@ -0,0 +1,63 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+   Copyright (c) 2021 NVIDIA Corporation & Affiliates
+
+.. include:: <isonum.txt>
+
+MLX5 Crypto Driver
+==================
+
+The MLX5 crypto driver library
+(**librte_crypto_mlx5**) provides support for **Mellanox ConnectX-6**
+family adapters.
+
+Overview
+--------
+
+The device can provide disk encryption services, allowing data encryption
+and decryption towards a disk. Having all encryption/decryption
+operations done in a single device can reduce cost and overheads of the related
+FIPS certification, as ConnectX-6 is FIPS 140-2 level-2 ready.
+The encryption cipher is AES-XTS of 256/512 bit key size.
+
+MKEY is a memory region object in the hardware, that holds address translation information and
+attributes per memory area. Its ID must be tied to addresses provided to the hardware.
+The encryption operations are performed with MKEY read/write transactions, when
+the MKEY is configured to perform crypto operations.
+
+The encryption does not require text to be aligned to the AES block size (128b).
+
+The PMD uses libibverbs and libmlx5 to access the device firmware or to
+access the hardware components directly.
+There are different levels of objects and bypassing abilities.
+To get the best performances:
+
+- Verbs is a complete high-level generic API.
+- Direct Verbs is a device-specific API.
+- DevX allows to access firmware objects.
+
+Enabling librte_crypto_mlx5 causes DPDK applications to be linked against
+libibverbs.
+
+
+Driver options
+--------------
+
+- ``class`` parameter [string]
+
+  Select the class of the driver that should probe the device.
+  `crypto` for the mlx5 crypto driver.
+
+
+Supported NICs
+--------------
+
+* Mellanox\ |reg| ConnectX\ |reg|-6 200G MCX654106A-HCAT (2x200G)
+
+Prerequisites
+-------------
+
+- Mellanox OFED version: **5.3**
+  see :doc:`../../nics/mlx5` guide for more Mellanox OFED details.
+
+- Compilation can be done also with rdma-core v15+.
+  see :doc:`../../nics/mlx5` guide for more rdma-core details.
diff --git a/doc/guides/rel_notes/release_21_08.rst b/doc/guides/rel_notes/release_21_08.rst
index 7d289e07e3..2bf4ce7a73 100644
--- a/doc/guides/rel_notes/release_21_08.rst
+++ b/doc/guides/rel_notes/release_21_08.rst
@@ -125,6 +125,11 @@ New Features
   The experimental PMD power management API now supports managing
   multiple Ethernet Rx queues per lcore.
 
+* **Added support for Nvidia crypto device driver.**
+
+  * Added mlx5 crypto driver to support AES-XTS cipher operations.
+    The first device to support it is ConnectX-6.
+
 
 Removed Items
 -------------
diff --git a/drivers/common/mlx5/mlx5_common.h b/drivers/common/mlx5/mlx5_common.h
index 7fb7d40b38..962179a5a5 100644
--- a/drivers/common/mlx5/mlx5_common.h
+++ b/drivers/common/mlx5/mlx5_common.h
@@ -216,6 +216,7 @@ enum mlx5_class {
 	MLX5_CLASS_VDPA = RTE_BIT64(1),
 	MLX5_CLASS_REGEX = RTE_BIT64(2),
 	MLX5_CLASS_COMPRESS = RTE_BIT64(3),
+	MLX5_CLASS_CRYPTO = RTE_BIT64(4),
 };
 
 #define MLX5_DBR_SIZE RTE_CACHE_LINE_SIZE
diff --git a/drivers/common/mlx5/mlx5_common_pci.c b/drivers/common/mlx5/mlx5_common_pci.c
index 34747c4e07..5547e62d6b 100644
--- a/drivers/common/mlx5/mlx5_common_pci.c
+++ b/drivers/common/mlx5/mlx5_common_pci.c
@@ -31,6 +31,7 @@ static const struct {
 	{ .name = "net", .driver_class = MLX5_CLASS_NET },
 	{ .name = "regex", .driver_class = MLX5_CLASS_REGEX },
 	{ .name = "compress", .driver_class = MLX5_CLASS_COMPRESS },
+	{ .name = "crypto", .driver_class = MLX5_CLASS_CRYPTO },
 };
 
 static const unsigned int mlx5_class_combinations[] = {
@@ -38,13 +39,26 @@ static const unsigned int mlx5_class_combinations[] = {
 	MLX5_CLASS_VDPA,
 	MLX5_CLASS_REGEX,
 	MLX5_CLASS_COMPRESS,
+	MLX5_CLASS_CRYPTO,
 	MLX5_CLASS_NET | MLX5_CLASS_REGEX,
 	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX,
 	MLX5_CLASS_NET | MLX5_CLASS_COMPRESS,
 	MLX5_CLASS_VDPA | MLX5_CLASS_COMPRESS,
 	MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
+	MLX5_CLASS_NET | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_VDPA | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO,
 	MLX5_CLASS_NET | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
 	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
+	MLX5_CLASS_NET | MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_NET | MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_VDPA | MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_NET | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS |
+	MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS |
+	MLX5_CLASS_CRYPTO,
 	/* New class combination should be added here. */
 };
 
diff --git a/drivers/common/mlx5/mlx5_common_pci.h b/drivers/common/mlx5/mlx5_common_pci.h
index de89bb98bc..cb8d2f5f87 100644
--- a/drivers/common/mlx5/mlx5_common_pci.h
+++ b/drivers/common/mlx5/mlx5_common_pci.h
@@ -9,17 +9,18 @@
  * @file
  *
  * RTE Mellanox PCI Driver Interface
- * Mellanox ConnectX PCI device supports multiple class: net,vdpa,regex and
- * compress devices. This layer enables creating such multiple class of devices
- * on a single PCI device by allowing to bind multiple class specific device
- * driver to attach to mlx5_pci driver.
+ * Mellanox ConnectX PCI device supports multiple class: net,vdpa,regex,compress
+ * and crypto devices. This layer enables creating such multiple class of
+ * devices on a single PCI device by allowing to bind multiple class specific
+ * device driver to attach to mlx5_pci driver.
  *
- * -----------    ------------    -------------    ----------------
- * |   mlx5  |    |   mlx5   |    |   mlx5    |    |     mlx5     |
- * | net pmd |    | vdpa pmd |    | regex pmd |    | compress pmd |
- * -----------    ------------    -------------    ----------------
- *      \              \                    /              /
- *       \              \                  /              /
+ * --------    --------    ---------    ------------    ----------
+ * | mlx5 |    | mlx5 |    | mlx5  |    |   mlx5   |    |  mlx5  |
+ * | net  |    | vdpa |    | regex |    | compress |    | crypto |
+ * | pmd  |    | pmd  |    |  pmd  |    |   pmd    |    |  pmd   |
+ * --------    --------    ---------    ------------    ----------
+ *      \              \         |          /              /
+ *       \              \        |         /              /
  *        \              \_--------------_/              /
  *         \_______________|   mlx5     |_______________/
  *                         | pci common |
diff --git a/drivers/crypto/meson.build b/drivers/crypto/meson.build
index cb865aa0d5..ea239f4c56 100644
--- a/drivers/crypto/meson.build
+++ b/drivers/crypto/meson.build
@@ -16,6 +16,7 @@ drivers = [
         'dpaa_sec',
         'dpaa2_sec',
         'kasumi',
+        'mlx5',
         'mvsam',
         'nitrox',
         'null',
diff --git a/drivers/crypto/mlx5/meson.build b/drivers/crypto/mlx5/meson.build
new file mode 100644
index 0000000000..6fd70bc477
--- /dev/null
+++ b/drivers/crypto/mlx5/meson.build
@@ -0,0 +1,26 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2021 NVIDIA Corporation & Affiliates
+
+if not is_linux
+	build = false
+	reason = 'only supported on Linux'
+	subdir_done()
+endif
+
+fmt_name = 'mlx5_crypto'
+deps += ['common_mlx5', 'eal', 'cryptodev']
+sources = files(
+	'mlx5_crypto.c',
+)
+cflags_options = [
+	'-std=c11',
+	'-Wno-strict-prototypes',
+	'-D_BSD_SOURCE',
+	'-D_DEFAULT_SOURCE',
+	'-D_XOPEN_SOURCE=600',
+]
+foreach option:cflags_options
+	if cc.has_argument(option)
+		cflags += option
+	endif
+endforeach
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
new file mode 100644
index 0000000000..fbe3c21aae
--- /dev/null
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -0,0 +1,275 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2021 NVIDIA Corporation & Affiliates
+ */
+
+#include <rte_malloc.h>
+#include <rte_log.h>
+#include <rte_errno.h>
+#include <rte_pci.h>
+#include <rte_crypto.h>
+#include <rte_cryptodev.h>
+#include <rte_cryptodev_pmd.h>
+
+#include <mlx5_glue.h>
+#include <mlx5_common.h>
+#include <mlx5_common_pci.h>
+#include <mlx5_devx_cmds.h>
+#include <mlx5_common_os.h>
+
+#include "mlx5_crypto_utils.h"
+
+#define MLX5_CRYPTO_DRIVER_NAME mlx5_crypto
+#define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
+
+#define MLX5_CRYPTO_FEATURE_FLAGS \
+	RTE_CRYPTODEV_FF_HW_ACCELERATED
+
+struct mlx5_crypto_priv {
+	TAILQ_ENTRY(mlx5_crypto_priv) next;
+	struct ibv_context *ctx; /* Device context. */
+	struct rte_pci_device *pci_dev;
+	struct rte_cryptodev *crypto_dev;
+	void *uar; /* User Access Region. */
+	uint32_t pdn; /* Protection Domain number. */
+	struct ibv_pd *pd;
+};
+
+TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
+				TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
+static pthread_mutex_t priv_list_lock = PTHREAD_MUTEX_INITIALIZER;
+
+int mlx5_crypto_logtype;
+
+uint8_t mlx5_crypto_driver_id;
+
+static const char mlx5_crypto_drv_name[] = RTE_STR(MLX5_CRYPTO_DRIVER_NAME);
+
+static const struct rte_driver mlx5_drv = {
+	.name = mlx5_crypto_drv_name,
+	.alias = mlx5_crypto_drv_name
+};
+
+static struct cryptodev_driver mlx5_cryptodev_driver;
+
+static struct rte_cryptodev_ops mlx5_crypto_ops = {
+	.dev_configure			= NULL,
+	.dev_start			= NULL,
+	.dev_stop			= NULL,
+	.dev_close			= NULL,
+	.dev_infos_get			= NULL,
+	.stats_get			= NULL,
+	.stats_reset			= NULL,
+	.queue_pair_setup		= NULL,
+	.queue_pair_release		= NULL,
+	.sym_session_get_size		= NULL,
+	.sym_session_configure		= NULL,
+	.sym_session_clear		= NULL,
+	.sym_get_raw_dp_ctx_size	= NULL,
+	.sym_configure_raw_dp_ctx	= NULL,
+};
+
+static void
+mlx5_crypto_hw_global_release(struct mlx5_crypto_priv *priv)
+{
+	if (priv->pd != NULL) {
+		claim_zero(mlx5_glue->dealloc_pd(priv->pd));
+		priv->pd = NULL;
+	}
+	if (priv->uar != NULL) {
+		mlx5_glue->devx_free_uar(priv->uar);
+		priv->uar = NULL;
+	}
+}
+
+static int
+mlx5_crypto_pd_create(struct mlx5_crypto_priv *priv)
+{
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+	struct mlx5dv_obj obj;
+	struct mlx5dv_pd pd_info;
+	int ret;
+
+	priv->pd = mlx5_glue->alloc_pd(priv->ctx);
+	if (priv->pd == NULL) {
+		DRV_LOG(ERR, "Failed to allocate PD.");
+		return errno ? -errno : -ENOMEM;
+	}
+	obj.pd.in = priv->pd;
+	obj.pd.out = &pd_info;
+	ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_PD);
+	if (ret != 0) {
+		DRV_LOG(ERR, "Fail to get PD object info.");
+		mlx5_glue->dealloc_pd(priv->pd);
+		priv->pd = NULL;
+		return -errno;
+	}
+	priv->pdn = pd_info.pdn;
+	return 0;
+#else
+	(void)priv;
+	DRV_LOG(ERR, "Cannot get pdn - no DV support.");
+	return -ENOTSUP;
+#endif /* HAVE_IBV_FLOW_DV_SUPPORT */
+}
+
+static int
+mlx5_crypto_hw_global_prepare(struct mlx5_crypto_priv *priv)
+{
+	if (mlx5_crypto_pd_create(priv) != 0)
+		return -1;
+	priv->uar = mlx5_devx_alloc_uar(priv->ctx, -1);
+	if (priv->uar == NULL || mlx5_os_get_devx_uar_reg_addr(priv->uar) ==
+	    NULL) {
+		rte_errno = errno;
+		claim_zero(mlx5_glue->dealloc_pd(priv->pd));
+		DRV_LOG(ERR, "Failed to allocate UAR.");
+		return -1;
+	}
+	return 0;
+}
+
+/**
+ * DPDK callback to register a PCI device.
+ *
+ * This function spawns crypto device out of a given PCI device.
+ *
+ * @param[in] pci_drv
+ *   PCI driver structure (mlx5_crypto_driver).
+ * @param[in] pci_dev
+ *   PCI device information.
+ *
+ * @return
+ *   0 on success, 1 to skip this driver, a negative errno value otherwise
+ *   and rte_errno is set.
+ */
+static int
+mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
+			struct rte_pci_device *pci_dev)
+{
+	struct ibv_device *ibv;
+	struct rte_cryptodev *crypto_dev;
+	struct ibv_context *ctx;
+	struct mlx5_crypto_priv *priv;
+	struct mlx5_hca_attr attr = { 0 };
+	struct rte_cryptodev_pmd_init_params init_params = {
+		.name = "",
+		.private_data_size = sizeof(struct mlx5_crypto_priv),
+		.socket_id = pci_dev->device.numa_node,
+		.max_nb_queue_pairs =
+				RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
+	};
+	RTE_SET_USED(pci_drv);
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+		DRV_LOG(ERR, "Non-primary process type is not supported.");
+		rte_errno = ENOTSUP;
+		return -rte_errno;
+	}
+	ibv = mlx5_os_get_ibv_device(&pci_dev->addr);
+	if (ibv == NULL) {
+		DRV_LOG(ERR, "No matching IB device for PCI slot "
+			PCI_PRI_FMT ".", pci_dev->addr.domain,
+			pci_dev->addr.bus, pci_dev->addr.devid,
+			pci_dev->addr.function);
+		return -rte_errno;
+	}
+	DRV_LOG(INFO, "PCI information matches for device \"%s\".", ibv->name);
+	ctx = mlx5_glue->dv_open_device(ibv);
+	if (ctx == NULL) {
+		DRV_LOG(ERR, "Failed to open IB device \"%s\".", ibv->name);
+		rte_errno = ENODEV;
+		return -rte_errno;
+	}
+	if (mlx5_devx_cmd_query_hca_attr(ctx, &attr) != 0 ||
+	    attr.crypto == 0 || attr.aes_xts == 0) {
+		DRV_LOG(ERR, "Not enough capabilities to support crypto "
+			"operations, maybe old FW/OFED version?");
+		claim_zero(mlx5_glue->close_device(ctx));
+		rte_errno = ENOTSUP;
+		return -ENOTSUP;
+	}
+	crypto_dev = rte_cryptodev_pmd_create(ibv->name, &pci_dev->device,
+					&init_params);
+	if (crypto_dev == NULL) {
+		DRV_LOG(ERR, "Failed to create device \"%s\".", ibv->name);
+		claim_zero(mlx5_glue->close_device(ctx));
+		return -ENODEV;
+	}
+	DRV_LOG(INFO,
+		"Crypto device %s was created successfully.", ibv->name);
+	crypto_dev->dev_ops = &mlx5_crypto_ops;
+	crypto_dev->dequeue_burst = NULL;
+	crypto_dev->enqueue_burst = NULL;
+	crypto_dev->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
+	crypto_dev->driver_id = mlx5_crypto_driver_id;
+	priv = crypto_dev->data->dev_private;
+	priv->ctx = ctx;
+	priv->pci_dev = pci_dev;
+	priv->crypto_dev = crypto_dev;
+	if (mlx5_crypto_hw_global_prepare(priv) != 0) {
+		rte_cryptodev_pmd_destroy(priv->crypto_dev);
+		claim_zero(mlx5_glue->close_device(priv->ctx));
+		return -1;
+	}
+	pthread_mutex_lock(&priv_list_lock);
+	TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next);
+	pthread_mutex_unlock(&priv_list_lock);
+	return 0;
+}
+
+static int
+mlx5_crypto_pci_remove(struct rte_pci_device *pdev)
+{
+	struct mlx5_crypto_priv *priv = NULL;
+
+	pthread_mutex_lock(&priv_list_lock);
+	TAILQ_FOREACH(priv, &mlx5_crypto_priv_list, next)
+		if (rte_pci_addr_cmp(&priv->pci_dev->addr, &pdev->addr) != 0)
+			break;
+	if (priv)
+		TAILQ_REMOVE(&mlx5_crypto_priv_list, priv, next);
+	pthread_mutex_unlock(&priv_list_lock);
+	if (priv) {
+		mlx5_crypto_hw_global_release(priv);
+		rte_cryptodev_pmd_destroy(priv->crypto_dev);
+		claim_zero(mlx5_glue->close_device(priv->ctx));
+	}
+	return 0;
+}
+
+static const struct rte_pci_id mlx5_crypto_pci_id_map[] = {
+		{
+			RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
+					PCI_DEVICE_ID_MELLANOX_CONNECTX6)
+		},
+		{
+			.vendor_id = 0
+		}
+	};
+
+static struct mlx5_pci_driver mlx5_crypto_driver = {
+	.driver_class = MLX5_CLASS_CRYPTO,
+	.pci_driver = {
+		.driver = {
+			.name = RTE_STR(MLX5_CRYPTO_DRIVER_NAME),
+		},
+		.id_table = mlx5_crypto_pci_id_map,
+		.probe = mlx5_crypto_pci_probe,
+		.remove = mlx5_crypto_pci_remove,
+		.drv_flags = 0,
+	},
+};
+
+RTE_INIT(rte_mlx5_crypto_init)
+{
+	mlx5_common_init();
+	if (mlx5_glue != NULL)
+		mlx5_pci_driver_register(&mlx5_crypto_driver);
+}
+
+RTE_PMD_REGISTER_CRYPTO_DRIVER(mlx5_cryptodev_driver, mlx5_drv,
+			       mlx5_crypto_driver_id);
+
+RTE_LOG_REGISTER_DEFAULT(mlx5_crypto_logtype, NOTICE)
+RTE_PMD_EXPORT_NAME(MLX5_CRYPTO_DRIVER_NAME, __COUNTER__);
+RTE_PMD_REGISTER_PCI_TABLE(MLX5_CRYPTO_DRIVER_NAME, mlx5_crypto_pci_id_map);
+RTE_PMD_REGISTER_KMOD_DEP(MLX5_CRYPTO_DRIVER_NAME, "* ib_uverbs & mlx5_core & mlx5_ib");
diff --git a/drivers/crypto/mlx5/mlx5_crypto_utils.h b/drivers/crypto/mlx5/mlx5_crypto_utils.h
new file mode 100644
index 0000000000..b6c60ca782
--- /dev/null
+++ b/drivers/crypto/mlx5/mlx5_crypto_utils.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2021 NVIDIA Corporation & Affiliates
+ */
+
+#ifndef RTE_PMD_MLX5_CRYPTO_UTILS_H_
+#define RTE_PMD_MLX5_CRYPTO_UTILS_H_
+
+#include <mlx5_common.h>
+
+extern int mlx5_crypto_logtype;
+
+#define MLX5_CRYPTO_LOG_PREFIX "mlx5_crypto"
+/* Generic printf()-like logging macro with automatic line feed. */
+#define DRV_LOG(level, ...) \
+	PMD_DRV_LOG_(level, mlx5_crypto_logtype, MLX5_CRYPTO_LOG_PREFIX, \
+		__VA_ARGS__ PMD_DRV_LOG_STRIP PMD_DRV_LOG_OPAREN, \
+		PMD_DRV_LOG_CPAREN)
+
+#endif /* RTE_PMD_MLX5_CRYPTO_UTILS_H_ */
diff --git a/drivers/crypto/mlx5/version.map b/drivers/crypto/mlx5/version.map
new file mode 100644
index 0000000000..4a76d1d52d
--- /dev/null
+++ b/drivers/crypto/mlx5/version.map
@@ -0,0 +1,3 @@
+DPDK_21 {
+	local: *;
+};
-- 
2.27.0

[dpdk-dev] [PATCH v7 02/16] crypto/mlx5: add DEK object management

[Shiri Kuzin]

A DEK(Data encryption Key) is an mlx5 HW object which represents the
cipher algorithm key.
The DEKs are used during data encryption/decryption operations.

In symmetric algorithms like AES-STS, we use the same DEK for both
encryption and decryption.

Use the mlx5 hash-list tool to manage the DEK objects in the PMD.

Provide the compare, create and destroy functions to manage DEKs in
hash-list and introduce an internal API to setup and unset the DEK
management and to prepare and destroy specific DEK object.

The DEK hash-list will be created in dev_configure routine and
destroyed in dev_close routine.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/meson.build       |   1 +
 drivers/crypto/mlx5/mlx5_crypto.c     |  42 ++++---
 drivers/crypto/mlx5/mlx5_crypto.h     |  51 ++++++++
 drivers/crypto/mlx5/mlx5_crypto_dek.c | 161 ++++++++++++++++++++++++++
 4 files changed, 239 insertions(+), 16 deletions(-)
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto.h
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_dek.c

diff --git a/drivers/crypto/mlx5/meson.build b/drivers/crypto/mlx5/meson.build
index 6fd70bc477..d55cdbfe6f 100644
--- a/drivers/crypto/mlx5/meson.build
+++ b/drivers/crypto/mlx5/meson.build
@@ -11,6 +11,7 @@ fmt_name = 'mlx5_crypto'
 deps += ['common_mlx5', 'eal', 'cryptodev']
 sources = files(
 	'mlx5_crypto.c',
+	'mlx5_crypto_dek.c',
 )
 cflags_options = [
 	'-std=c11',
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index fbe3c21aae..d2d82c7b15 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -3,12 +3,9 @@
  */
 
 #include <rte_malloc.h>
-#include <rte_log.h>
 #include <rte_errno.h>
+#include <rte_log.h>
 #include <rte_pci.h>
-#include <rte_crypto.h>
-#include <rte_cryptodev.h>
-#include <rte_cryptodev_pmd.h>
 
 #include <mlx5_glue.h>
 #include <mlx5_common.h>
@@ -17,6 +14,7 @@
 #include <mlx5_common_os.h>
 
 #include "mlx5_crypto_utils.h"
+#include "mlx5_crypto.h"
 
 #define MLX5_CRYPTO_DRIVER_NAME mlx5_crypto
 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
@@ -24,16 +22,6 @@
 #define MLX5_CRYPTO_FEATURE_FLAGS \
 	RTE_CRYPTODEV_FF_HW_ACCELERATED
 
-struct mlx5_crypto_priv {
-	TAILQ_ENTRY(mlx5_crypto_priv) next;
-	struct ibv_context *ctx; /* Device context. */
-	struct rte_pci_device *pci_dev;
-	struct rte_cryptodev *crypto_dev;
-	void *uar; /* User Access Region. */
-	uint32_t pdn; /* Protection Domain number. */
-	struct ibv_pd *pd;
-};
-
 TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
 				TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
 static pthread_mutex_t priv_list_lock = PTHREAD_MUTEX_INITIALIZER;
@@ -51,11 +39,33 @@ static const struct rte_driver mlx5_drv = {
 
 static struct cryptodev_driver mlx5_cryptodev_driver;
 
+static int
+mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
+		struct rte_cryptodev_config *config __rte_unused)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+
+	if (mlx5_crypto_dek_setup(priv) != 0) {
+		DRV_LOG(ERR, "Dek hash list creation has failed.");
+		return -ENOMEM;
+	}
+	return 0;
+}
+
+static int
+mlx5_crypto_dev_close(struct rte_cryptodev *dev)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+
+	mlx5_crypto_dek_unset(priv);
+	return 0;
+}
+
 static struct rte_cryptodev_ops mlx5_crypto_ops = {
-	.dev_configure			= NULL,
+	.dev_configure			= mlx5_crypto_dev_configure,
 	.dev_start			= NULL,
 	.dev_stop			= NULL,
-	.dev_close			= NULL,
+	.dev_close			= mlx5_crypto_dev_close,
 	.dev_infos_get			= NULL,
 	.stats_get			= NULL,
 	.stats_reset			= NULL,
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
new file mode 100644
index 0000000000..167e9e57ad
--- /dev/null
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -0,0 +1,51 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2021 NVIDIA Corporation & Affiliates
+ */
+
+#ifndef MLX5_CRYPTO_H_
+#define MLX5_CRYPTO_H_
+
+#include <stdbool.h>
+
+#include <rte_cryptodev.h>
+#include <rte_cryptodev_pmd.h>
+
+#include <mlx5_common_utils.h>
+
+#define MLX5_CRYPTO_DEK_HTABLE_SZ (1 << 11)
+#define MLX5_CRYPTO_KEY_LENGTH 80
+
+struct mlx5_crypto_priv {
+	TAILQ_ENTRY(mlx5_crypto_priv) next;
+	struct ibv_context *ctx; /* Device context. */
+	struct rte_pci_device *pci_dev;
+	struct rte_cryptodev *crypto_dev;
+	void *uar; /* User Access Region. */
+	uint32_t pdn; /* Protection Domain number. */
+	struct ibv_pd *pd;
+	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
+};
+
+struct mlx5_crypto_dek {
+	struct mlx5_list_entry entry; /* Pointer to DEK hash list entry. */
+	struct mlx5_devx_obj *obj; /* Pointer to DEK DevX object. */
+	uint8_t data[MLX5_CRYPTO_KEY_LENGTH]; /* DEK key data. */
+	bool size_is_48; /* Whether the key\data size is 48 bytes or not. */
+} __rte_cache_aligned;
+
+int
+mlx5_crypto_dek_destroy(struct mlx5_crypto_priv *priv,
+			struct mlx5_crypto_dek *dek);
+
+struct mlx5_crypto_dek *
+mlx5_crypto_dek_prepare(struct mlx5_crypto_priv *priv,
+			struct rte_crypto_cipher_xform *cipher);
+
+int
+mlx5_crypto_dek_setup(struct mlx5_crypto_priv *priv);
+
+void
+mlx5_crypto_dek_unset(struct mlx5_crypto_priv *priv);
+
+#endif /* MLX5_CRYPTO_H_ */
+
diff --git a/drivers/crypto/mlx5/mlx5_crypto_dek.c b/drivers/crypto/mlx5/mlx5_crypto_dek.c
new file mode 100644
index 0000000000..43d1bcc9f8
--- /dev/null
+++ b/drivers/crypto/mlx5/mlx5_crypto_dek.c
@@ -0,0 +1,161 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2021 NVIDIA Corporation & Affiliates
+ */
+
+#include <rte_ip.h>
+#include <rte_common.h>
+#include <rte_errno.h>
+#include <rte_log.h>
+
+#include <mlx5_prm.h>
+#include <mlx5_devx_cmds.h>
+
+#include "mlx5_crypto_utils.h"
+#include "mlx5_crypto.h"
+
+struct mlx5_crypto_dek_ctx {
+	struct rte_crypto_cipher_xform *cipher;
+	struct mlx5_crypto_priv *priv;
+};
+
+int
+mlx5_crypto_dek_destroy(struct mlx5_crypto_priv *priv,
+			struct mlx5_crypto_dek *dek)
+{
+	return mlx5_hlist_unregister(priv->dek_hlist, &dek->entry);
+}
+
+struct mlx5_crypto_dek *
+mlx5_crypto_dek_prepare(struct mlx5_crypto_priv *priv,
+			struct rte_crypto_cipher_xform *cipher)
+{
+	struct mlx5_hlist *dek_hlist = priv->dek_hlist;
+	struct mlx5_crypto_dek_ctx dek_ctx = {
+		.cipher = cipher,
+		.priv = priv,
+	};
+	struct rte_crypto_cipher_xform *cipher_ctx = cipher;
+	uint64_t key64 = __rte_raw_cksum(cipher_ctx->key.data,
+					 cipher_ctx->key.length, 0);
+	struct mlx5_list_entry *entry = mlx5_hlist_register(dek_hlist,
+							     key64, &dek_ctx);
+
+	return entry == NULL ? NULL :
+			     container_of(entry, struct mlx5_crypto_dek, entry);
+}
+
+static struct mlx5_list_entry *
+mlx5_crypto_dek_clone_cb(void *tool_ctx __rte_unused,
+			 struct mlx5_list_entry *oentry,
+			 void *cb_ctx __rte_unused)
+{
+	struct mlx5_crypto_dek *entry = rte_zmalloc(__func__, sizeof(*entry),
+						    RTE_CACHE_LINE_SIZE);
+
+	if (!entry) {
+		DRV_LOG(ERR, "Cannot allocate dek resource memory.");
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+	memcpy(entry, oentry, sizeof(*entry));
+	return &entry->entry;
+}
+
+static void
+mlx5_crypto_dek_clone_free_cb(void *tool_ctx __rte_unused,
+			      struct mlx5_list_entry *entry)
+{
+	struct mlx5_crypto_dek *dek = container_of(entry,
+						struct mlx5_crypto_dek, entry);
+
+	rte_free(dek);
+}
+
+static int
+mlx5_crypto_dek_match_cb(void *tool_ctx __rte_unused,
+			 struct mlx5_list_entry *entry, void *cb_ctx)
+{
+	struct mlx5_crypto_dek_ctx *ctx = cb_ctx;
+	struct rte_crypto_cipher_xform *cipher_ctx = ctx->cipher;
+	struct mlx5_crypto_dek *dek =
+			container_of(entry, typeof(*dek), entry);
+	uint32_t key_len = dek->size_is_48 ? 48 : 80;
+
+	if (key_len != cipher_ctx->key.length)
+		return -1;
+	return memcmp(cipher_ctx->key.data, dek->data, key_len);
+}
+
+static struct mlx5_list_entry *
+mlx5_crypto_dek_create_cb(void *tool_ctx __rte_unused, void *cb_ctx)
+{
+	struct mlx5_crypto_dek_ctx *ctx = cb_ctx;
+	struct rte_crypto_cipher_xform *cipher_ctx = ctx->cipher;
+	struct mlx5_crypto_dek *dek = rte_zmalloc(__func__, sizeof(*dek),
+						  RTE_CACHE_LINE_SIZE);
+	struct mlx5_devx_dek_attr dek_attr = {
+		.pd = ctx->priv->pdn,
+		.key_purpose = MLX5_CRYPTO_KEY_PURPOSE_AES_XTS,
+		.has_keytag = 1,
+	};
+
+	if (dek == NULL) {
+		DRV_LOG(ERR, "Failed to allocate dek memory.");
+		return NULL;
+	}
+	switch (cipher_ctx->key.length) {
+	case 48:
+		dek->size_is_48 = true;
+		dek_attr.key_size = MLX5_CRYPTO_KEY_SIZE_128b;
+		break;
+	case 80:
+		dek->size_is_48 = false;
+		dek_attr.key_size = MLX5_CRYPTO_KEY_SIZE_256b;
+		break;
+	default:
+		DRV_LOG(ERR, "Key size not supported.");
+		return NULL;
+	}
+	memcpy(&dek_attr.key, cipher_ctx->key.data, cipher_ctx->key.length);
+	dek->obj = mlx5_devx_cmd_create_dek_obj(ctx->priv->ctx, &dek_attr);
+	if (dek->obj == NULL) {
+		rte_free(dek);
+		return NULL;
+	}
+	memcpy(&dek->data, cipher_ctx->key.data, cipher_ctx->key.length);
+	return &dek->entry;
+}
+
+static void
+mlx5_crypto_dek_remove_cb(void *tool_ctx __rte_unused,
+			  struct mlx5_list_entry *entry)
+{
+	struct mlx5_crypto_dek *dek =
+		container_of(entry, typeof(*dek), entry);
+
+	claim_zero(mlx5_devx_cmd_destroy(dek->obj));
+	rte_free(dek);
+}
+
+
+int
+mlx5_crypto_dek_setup(struct mlx5_crypto_priv *priv)
+{
+	priv->dek_hlist = mlx5_hlist_create("dek_hlist",
+				 MLX5_CRYPTO_DEK_HTABLE_SZ,
+				 0, 1, NULL, mlx5_crypto_dek_create_cb,
+				 mlx5_crypto_dek_match_cb,
+				 mlx5_crypto_dek_remove_cb,
+				 mlx5_crypto_dek_clone_cb,
+				 mlx5_crypto_dek_clone_free_cb);
+	if (priv->dek_hlist == NULL)
+		return -1;
+	return 0;
+}
+
+void
+mlx5_crypto_dek_unset(struct mlx5_crypto_priv *priv)
+{
+	mlx5_hlist_destroy(priv->dek_hlist);
+	priv->dek_hlist = NULL;
+}
-- 
2.27.0

[dpdk-dev] [PATCH v7 03/16] crypto/mlx5: add session operations

[Shiri Kuzin]

Sessions are used in symmetric transformations in order to prepare
objects and data for packet processing stage.

A mlx5 session includes iv_offset, pointer to mlx5_crypto_dek struct,
bsf_size, bsf_p_type, block size index, encryption_order and encryption
standard.

Implement the next session operations:
        mlx5_crypto_sym_session_get_size- returns the size of the mlx5
	session struct.
	mlx5_crypto_sym_session_configure- prepares the DEK hash-list
	and saves all the session data.
	mlx5_crypto_sym_session_clear - destroys the DEK hash-list.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 doc/guides/cryptodevs/features/mlx5.ini |   5 +
 doc/guides/cryptodevs/mlx5.rst          |  10 ++
 drivers/crypto/mlx5/mlx5_crypto.c       | 172 +++++++++++++++++++++++-
 3 files changed, 182 insertions(+), 5 deletions(-)

diff --git a/doc/guides/cryptodevs/features/mlx5.ini b/doc/guides/cryptodevs/features/mlx5.ini
index ceadd967b6..bd757b5211 100644
--- a/doc/guides/cryptodevs/features/mlx5.ini
+++ b/doc/guides/cryptodevs/features/mlx5.ini
@@ -4,12 +4,17 @@
 ; Refer to default.ini for the full list of available PMD features.
 ;
 [Features]
+Symmetric crypto       = Y
 HW Accelerated         = Y
+Cipher multiple data units = Y
+Cipher wrapped key     = Y
 
 ;
 ; Supported crypto algorithms of a mlx5 crypto driver.
 ;
 [Cipher]
+AES XTS (128)  = Y
+AES XTS (256)  = Y
 
 ;
 ; Supported authentication algorithms of a mlx5 crypto driver.
diff --git a/doc/guides/cryptodevs/mlx5.rst b/doc/guides/cryptodevs/mlx5.rst
index 05a0a449e2..dd1d1a615d 100644
--- a/doc/guides/cryptodevs/mlx5.rst
+++ b/doc/guides/cryptodevs/mlx5.rst
@@ -53,6 +53,16 @@ Supported NICs
 
 * Mellanox\ |reg| ConnectX\ |reg|-6 200G MCX654106A-HCAT (2x200G)
 
+
+Limitations
+-----------
+
+- AES-XTS keys provided in xform must include keytag and should be wrappend.
+- The supported data-unit lengths are 512B and 1KB. In case the `dataunit_len`
+  is not provided in the cipher xform, the OP length is limited to the above
+  values and 1MB.
+
+
 Prerequisites
 -------------
 
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index d2d82c7b15..3f0c97d081 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -3,6 +3,7 @@
  */
 
 #include <rte_malloc.h>
+#include <rte_mempool.h>
 #include <rte_errno.h>
 #include <rte_log.h>
 #include <rte_pci.h>
@@ -20,7 +21,9 @@
 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
 
 #define MLX5_CRYPTO_FEATURE_FLAGS \
-	RTE_CRYPTODEV_FF_HW_ACCELERATED
+	(RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | \
+	 RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY | \
+	 RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS)
 
 TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
 				TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
@@ -30,6 +33,32 @@ int mlx5_crypto_logtype;
 
 uint8_t mlx5_crypto_driver_id;
 
+const struct rte_cryptodev_capabilities mlx5_crypto_caps[] = {
+	{		/* AES XTS */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+			{.cipher = {
+				.algo = RTE_CRYPTO_CIPHER_AES_XTS,
+				.block_size = 16,
+				.key_size = {
+					.min = 32,
+					.max = 64,
+					.increment = 32
+				},
+				.iv_size = {
+					.min = 16,
+					.max = 16,
+					.increment = 0
+				},
+				.dataunit_set =
+				RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_512_BYTES |
+				RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_4096_BYTES,
+			}, }
+		}, }
+	},
+};
+
 static const char mlx5_crypto_drv_name[] = RTE_STR(MLX5_CRYPTO_DRIVER_NAME);
 
 static const struct rte_driver mlx5_drv = {
@@ -39,6 +68,49 @@ static const struct rte_driver mlx5_drv = {
 
 static struct cryptodev_driver mlx5_cryptodev_driver;
 
+struct mlx5_crypto_session {
+	uint32_t bs_bpt_eo_es;
+	/*
+	 * bsf_size, bsf_p_type, encryption_order and encryption standard,
+	 * saved in big endian format.
+	 */
+	uint32_t bsp_res;
+	/*
+	 * crypto_block_size_pointer and reserved 24 bits saved in big endian
+	 * format.
+	 */
+	uint32_t iv_offset:16;
+	/* Starting point for Initialisation Vector. */
+	struct mlx5_crypto_dek *dek; /* Pointer to dek struct. */
+	uint32_t dek_id; /* DEK ID */
+} __rte_packed;
+
+static void
+mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev,
+			  struct rte_cryptodev_info *dev_info)
+{
+	RTE_SET_USED(dev);
+	if (dev_info != NULL) {
+		dev_info->driver_id = mlx5_crypto_driver_id;
+		dev_info->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
+		dev_info->capabilities = mlx5_crypto_caps;
+		dev_info->max_nb_queue_pairs = 0;
+		dev_info->min_mbuf_headroom_req = 0;
+		dev_info->min_mbuf_tailroom_req = 0;
+		dev_info->sym.max_nb_sessions = 0;
+		/*
+		 * If 0, the device does not have any limitation in number of
+		 * sessions that can be used.
+		 */
+	}
+}
+
+static unsigned int
+mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
+{
+	return sizeof(struct mlx5_crypto_session);
+}
+
 static int
 mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
 		struct rte_cryptodev_config *config __rte_unused)
@@ -61,19 +133,109 @@ mlx5_crypto_dev_close(struct rte_cryptodev *dev)
 	return 0;
 }
 
+static int
+mlx5_crypto_sym_session_configure(struct rte_cryptodev *dev,
+				  struct rte_crypto_sym_xform *xform,
+				  struct rte_cryptodev_sym_session *session,
+				  struct rte_mempool *mp)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+	struct mlx5_crypto_session *sess_private_data;
+	struct rte_crypto_cipher_xform *cipher;
+	uint8_t encryption_order;
+	int ret;
+
+	if (unlikely(xform->next != NULL)) {
+		DRV_LOG(ERR, "Xform next is not supported.");
+		return -ENOTSUP;
+	}
+	if (unlikely((xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) ||
+		     (xform->cipher.algo != RTE_CRYPTO_CIPHER_AES_XTS))) {
+		DRV_LOG(ERR, "Only AES-XTS algorithm is supported.");
+		return -ENOTSUP;
+	}
+	ret = rte_mempool_get(mp, (void *)&sess_private_data);
+	if (ret != 0) {
+		DRV_LOG(ERR,
+			"Failed to get session %p private data from mempool.",
+			sess_private_data);
+		return -ENOMEM;
+	}
+	cipher = &xform->cipher;
+	sess_private_data->dek = mlx5_crypto_dek_prepare(priv, cipher);
+	if (sess_private_data->dek == NULL) {
+		rte_mempool_put(mp, sess_private_data);
+		DRV_LOG(ERR, "Failed to prepare dek.");
+		return -ENOMEM;
+	}
+	if (cipher->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
+		encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_MEMORY;
+	else
+		encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_WIRE;
+	sess_private_data->bs_bpt_eo_es = rte_cpu_to_be_32
+			(MLX5_BSF_SIZE_64B << MLX5_BSF_SIZE_OFFSET |
+			 MLX5_BSF_P_TYPE_CRYPTO << MLX5_BSF_P_TYPE_OFFSET |
+			 encryption_order << MLX5_ENCRYPTION_ORDER_OFFSET |
+			 MLX5_ENCRYPTION_STANDARD_AES_XTS);
+	switch (xform->cipher.dataunit_len) {
+	case 0:
+		sess_private_data->bsp_res = 0;
+		break;
+	case 512:
+		sess_private_data->bsp_res = rte_cpu_to_be_32
+					     ((uint32_t)MLX5_BLOCK_SIZE_512B <<
+					     MLX5_BLOCK_SIZE_OFFSET);
+		break;
+	case 4096:
+		sess_private_data->bsp_res = rte_cpu_to_be_32
+					     ((uint32_t)MLX5_BLOCK_SIZE_4096B <<
+					     MLX5_BLOCK_SIZE_OFFSET);
+		break;
+	default:
+		DRV_LOG(ERR, "Cipher data unit length is not supported.");
+		return -ENOTSUP;
+	}
+	sess_private_data->iv_offset = cipher->iv.offset;
+	sess_private_data->dek_id =
+			rte_cpu_to_be_32(sess_private_data->dek->obj->id &
+					 0xffffff);
+	set_sym_session_private_data(session, dev->driver_id,
+				     sess_private_data);
+	DRV_LOG(DEBUG, "Session %p was configured.", sess_private_data);
+	return 0;
+}
+
+static void
+mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev,
+			      struct rte_cryptodev_sym_session *sess)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+	struct mlx5_crypto_session *spriv = get_sym_session_private_data(sess,
+								dev->driver_id);
+
+	if (unlikely(spriv == NULL)) {
+		DRV_LOG(ERR, "Failed to get session %p private data.", spriv);
+		return;
+	}
+	mlx5_crypto_dek_destroy(priv, spriv->dek);
+	set_sym_session_private_data(sess, dev->driver_id, NULL);
+	rte_mempool_put(rte_mempool_from_obj(spriv), spriv);
+	DRV_LOG(DEBUG, "Session %p was cleared.", spriv);
+}
+
 static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_configure			= mlx5_crypto_dev_configure,
 	.dev_start			= NULL,
 	.dev_stop			= NULL,
 	.dev_close			= mlx5_crypto_dev_close,
-	.dev_infos_get			= NULL,
+	.dev_infos_get			= mlx5_crypto_dev_infos_get,
 	.stats_get			= NULL,
 	.stats_reset			= NULL,
 	.queue_pair_setup		= NULL,
 	.queue_pair_release		= NULL,
-	.sym_session_get_size		= NULL,
-	.sym_session_configure		= NULL,
-	.sym_session_clear		= NULL,
+	.sym_session_get_size		= mlx5_crypto_sym_session_get_size,
+	.sym_session_configure		= mlx5_crypto_sym_session_configure,
+	.sym_session_clear		= mlx5_crypto_sym_session_clear,
 	.sym_get_raw_dp_ctx_size	= NULL,
 	.sym_configure_raw_dp_ctx	= NULL,
 };
-- 
2.27.0

[dpdk-dev] [PATCH v7 04/16] crypto/mlx5: add basic operations

[Shiri Kuzin]

The basic dev control operations are configure, close and get info.

Extended the existing support of configure and close:
	-mlx5_crypto_dev_configure- function used to configure device.
	-mlx5_crypto_dev_close-  function used to close a configured
	 device.

Added config struct to user private data with the fields socket id,
number of queue pairs and feature flags to be disabled.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 26 +++++++++++++++++++-------
 drivers/crypto/mlx5/mlx5_crypto.h |  1 +
 2 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 3f0c97d081..a7e44deb9e 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -105,22 +105,27 @@ mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev,
 	}
 }
 
-static unsigned int
-mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
-{
-	return sizeof(struct mlx5_crypto_session);
-}
-
 static int
 mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
-		struct rte_cryptodev_config *config __rte_unused)
+			  struct rte_cryptodev_config *config)
 {
 	struct mlx5_crypto_priv *priv = dev->data->dev_private;
 
+	if (config == NULL) {
+		DRV_LOG(ERR, "Invalid crypto dev configure parameters.");
+		return -EINVAL;
+	}
+	if ((config->ff_disable & RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO) != 0) {
+		DRV_LOG(ERR,
+			"Disabled symmetric crypto feature is not supported.");
+		return -ENOTSUP;
+	}
 	if (mlx5_crypto_dek_setup(priv) != 0) {
 		DRV_LOG(ERR, "Dek hash list creation has failed.");
 		return -ENOMEM;
 	}
+	priv->dev_config = *config;
+	DRV_LOG(DEBUG, "Device %u was configured.", dev->driver_id);
 	return 0;
 }
 
@@ -130,9 +135,16 @@ mlx5_crypto_dev_close(struct rte_cryptodev *dev)
 	struct mlx5_crypto_priv *priv = dev->data->dev_private;
 
 	mlx5_crypto_dek_unset(priv);
+	DRV_LOG(DEBUG, "Device %u was closed.", dev->driver_id);
 	return 0;
 }
 
+static unsigned int
+mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
+{
+	return sizeof(struct mlx5_crypto_session);
+}
+
 static int
 mlx5_crypto_sym_session_configure(struct rte_cryptodev *dev,
 				  struct rte_crypto_sym_xform *xform,
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 167e9e57ad..a0df775407 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -24,6 +24,7 @@ struct mlx5_crypto_priv {
 	uint32_t pdn; /* Protection Domain number. */
 	struct ibv_pd *pd;
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
+	struct rte_cryptodev_config dev_config;
 };
 
 struct mlx5_crypto_dek {
-- 
2.27.0

[dpdk-dev] [PATCH v7 05/16] crypto/mlx5: add queue pairs operations

[Shiri Kuzin]

The HW queue pairs are a pair of send queue and receive queue of
independent work queues packed together in one object for the purpose
of transferring data between nodes of a network.

Completion Queue is a FIFO queue of completed work requests.

In crypto driver we use one QP in loopback in order to encrypt and
decrypt data locally without sending it to the wire.
In the configured QP we only use the SQ to perform the encryption and
decryption operations.

Added implementation for the QP setup function which creates the CQ,
creates the QP and changes its state to RTS (ready to send).

Added implementation for the release QP function to release all the QP
resources.

Added the ops structure that contains any operation which is supported
by the cryptodev.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 129 +++++++++++++++++++++++++++++-
 drivers/crypto/mlx5/mlx5_crypto.h |  11 +++
 2 files changed, 137 insertions(+), 3 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index a7e44deb9e..ebaa65c7a9 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -7,6 +7,7 @@
 #include <rte_errno.h>
 #include <rte_log.h>
 #include <rte_pci.h>
+#include <rte_memory.h>
 
 #include <mlx5_glue.h>
 #include <mlx5_common.h>
@@ -19,6 +20,7 @@
 
 #define MLX5_CRYPTO_DRIVER_NAME mlx5_crypto
 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
+#define MLX5_CRYPTO_MAX_QPS 1024
 
 #define MLX5_CRYPTO_FEATURE_FLAGS \
 	(RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | \
@@ -94,7 +96,7 @@ mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev,
 		dev_info->driver_id = mlx5_crypto_driver_id;
 		dev_info->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
 		dev_info->capabilities = mlx5_crypto_caps;
-		dev_info->max_nb_queue_pairs = 0;
+		dev_info->max_nb_queue_pairs = MLX5_CRYPTO_MAX_QPS;
 		dev_info->min_mbuf_headroom_req = 0;
 		dev_info->min_mbuf_tailroom_req = 0;
 		dev_info->sym.max_nb_sessions = 0;
@@ -235,6 +237,127 @@ mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev,
 	DRV_LOG(DEBUG, "Session %p was cleared.", spriv);
 }
 
+static int
+mlx5_crypto_queue_pair_release(struct rte_cryptodev *dev, uint16_t qp_id)
+{
+	struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
+
+	if (qp->qp_obj != NULL)
+		claim_zero(mlx5_devx_cmd_destroy(qp->qp_obj));
+	if (qp->umem_obj != NULL)
+		claim_zero(mlx5_glue->devx_umem_dereg(qp->umem_obj));
+	if (qp->umem_buf != NULL)
+		rte_free(qp->umem_buf);
+	mlx5_devx_cq_destroy(&qp->cq_obj);
+	rte_free(qp);
+	dev->data->queue_pairs[qp_id] = NULL;
+	return 0;
+}
+
+static int
+mlx5_crypto_qp2rts(struct mlx5_crypto_qp *qp)
+{
+	/*
+	 * In Order to configure self loopback, when calling these functions the
+	 * remote QP id that is used is the id of the same QP.
+	 */
+	if (mlx5_devx_cmd_modify_qp_state(qp->qp_obj, MLX5_CMD_OP_RST2INIT_QP,
+					  qp->qp_obj->id)) {
+		DRV_LOG(ERR, "Failed to modify QP to INIT state(%u).",
+			rte_errno);
+		return -1;
+	}
+	if (mlx5_devx_cmd_modify_qp_state(qp->qp_obj, MLX5_CMD_OP_INIT2RTR_QP,
+					  qp->qp_obj->id)) {
+		DRV_LOG(ERR, "Failed to modify QP to RTR state(%u).",
+			rte_errno);
+		return -1;
+	}
+	if (mlx5_devx_cmd_modify_qp_state(qp->qp_obj, MLX5_CMD_OP_RTR2RTS_QP,
+					  qp->qp_obj->id)) {
+		DRV_LOG(ERR, "Failed to modify QP to RTS state(%u).",
+			rte_errno);
+		return -1;
+	}
+	return 0;
+}
+
+static int
+mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
+			     const struct rte_cryptodev_qp_conf *qp_conf,
+			     int socket_id)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+	struct mlx5_devx_qp_attr attr = {0};
+	struct mlx5_crypto_qp *qp;
+	uint16_t log_nb_desc = rte_log2_u32(qp_conf->nb_descriptors);
+	uint32_t umem_size = RTE_BIT32(log_nb_desc) *
+			      MLX5_CRYPTO_WQE_SET_SIZE +
+			      sizeof(*qp->db_rec) * 2;
+	uint32_t alloc_size = sizeof(*qp);
+	struct mlx5_devx_cq_attr cq_attr = {
+		.uar_page_id = mlx5_os_get_devx_uar_page_id(priv->uar),
+	};
+
+	if (dev->data->queue_pairs[qp_id] != NULL)
+		mlx5_crypto_queue_pair_release(dev, qp_id);
+	alloc_size = RTE_ALIGN(alloc_size, RTE_CACHE_LINE_SIZE);
+	alloc_size += sizeof(struct rte_crypto_op *) * RTE_BIT32(log_nb_desc);
+	qp = rte_zmalloc_socket(__func__, alloc_size, RTE_CACHE_LINE_SIZE,
+				socket_id);
+	if (qp == NULL) {
+		DRV_LOG(ERR, "Failed to allocate QP memory.");
+		rte_errno = ENOMEM;
+		return -rte_errno;
+	}
+	if (mlx5_devx_cq_create(priv->ctx, &qp->cq_obj, log_nb_desc,
+				&cq_attr, socket_id) != 0) {
+		DRV_LOG(ERR, "Failed to create CQ.");
+		goto error;
+	}
+	qp->umem_buf = rte_zmalloc_socket(__func__, umem_size, 4096, socket_id);
+	if (qp->umem_buf == NULL) {
+		DRV_LOG(ERR, "Failed to allocate QP umem.");
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	qp->umem_obj = mlx5_glue->devx_umem_reg(priv->ctx,
+					       (void *)(uintptr_t)qp->umem_buf,
+					       umem_size,
+					       IBV_ACCESS_LOCAL_WRITE);
+	if (qp->umem_obj == NULL) {
+		DRV_LOG(ERR, "Failed to register QP umem.");
+		goto error;
+	}
+	attr.pd = priv->pdn;
+	attr.uar_index = mlx5_os_get_devx_uar_page_id(priv->uar);
+	attr.cqn = qp->cq_obj.cq->id;
+	attr.log_page_size = rte_log2_u32(sysconf(_SC_PAGESIZE));
+	attr.rq_size =  0;
+	attr.sq_size = RTE_BIT32(log_nb_desc);
+	attr.dbr_umem_valid = 1;
+	attr.wq_umem_id = qp->umem_obj->umem_id;
+	attr.wq_umem_offset = 0;
+	attr.dbr_umem_id = qp->umem_obj->umem_id;
+	attr.dbr_address = RTE_BIT64(log_nb_desc) *
+			   MLX5_CRYPTO_WQE_SET_SIZE;
+	qp->qp_obj = mlx5_devx_cmd_create_qp(priv->ctx, &attr);
+	if (qp->qp_obj == NULL) {
+		DRV_LOG(ERR, "Failed to create QP(%u).", rte_errno);
+		goto error;
+	}
+	qp->db_rec = RTE_PTR_ADD(qp->umem_buf, (uintptr_t)attr.dbr_address);
+	if (mlx5_crypto_qp2rts(qp))
+		goto error;
+	qp->ops = (struct rte_crypto_op **)RTE_ALIGN((uintptr_t)(qp + 1),
+							   RTE_CACHE_LINE_SIZE);
+	dev->data->queue_pairs[qp_id] = qp;
+	return 0;
+error:
+	mlx5_crypto_queue_pair_release(dev, qp_id);
+	return -1;
+}
+
 static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_configure			= mlx5_crypto_dev_configure,
 	.dev_start			= NULL,
@@ -243,8 +366,8 @@ static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_infos_get			= mlx5_crypto_dev_infos_get,
 	.stats_get			= NULL,
 	.stats_reset			= NULL,
-	.queue_pair_setup		= NULL,
-	.queue_pair_release		= NULL,
+	.queue_pair_setup		= mlx5_crypto_queue_pair_setup,
+	.queue_pair_release		= mlx5_crypto_queue_pair_release,
 	.sym_session_get_size		= mlx5_crypto_sym_session_get_size,
 	.sym_session_configure		= mlx5_crypto_sym_session_configure,
 	.sym_session_clear		= mlx5_crypto_sym_session_clear,
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index a0df775407..1c3c4855ec 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -11,9 +11,11 @@
 #include <rte_cryptodev_pmd.h>
 
 #include <mlx5_common_utils.h>
+#include <mlx5_common_devx.h>
 
 #define MLX5_CRYPTO_DEK_HTABLE_SZ (1 << 11)
 #define MLX5_CRYPTO_KEY_LENGTH 80
+#define MLX5_CRYPTO_WQE_SET_SIZE 1024
 
 struct mlx5_crypto_priv {
 	TAILQ_ENTRY(mlx5_crypto_priv) next;
@@ -27,6 +29,15 @@ struct mlx5_crypto_priv {
 	struct rte_cryptodev_config dev_config;
 };
 
+struct mlx5_crypto_qp {
+	struct mlx5_devx_cq cq_obj;
+	struct mlx5_devx_obj *qp_obj;
+	struct mlx5dv_devx_umem *umem_obj;
+	void *umem_buf;
+	volatile uint32_t *db_rec;
+	struct rte_crypto_op **ops;
+};
+
 struct mlx5_crypto_dek {
 	struct mlx5_list_entry entry; /* Pointer to DEK hash list entry. */
 	struct mlx5_devx_obj *obj; /* Pointer to DEK DevX object. */
-- 
2.27.0

[dpdk-dev] [PATCH v7 06/16] crypto/mlx5: add dev stop and start operations

[Shiri Kuzin]

Add the dev_start function that is used to start a configured device.
Add the dev_stop function that is used to stop a configured device.

Both functions set the dev parameter as used and return 0.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index ebaa65c7a9..37e66cf57b 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -131,6 +131,19 @@ mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
 	return 0;
 }
 
+static void
+mlx5_crypto_dev_stop(struct rte_cryptodev *dev)
+{
+	RTE_SET_USED(dev);
+}
+
+static int
+mlx5_crypto_dev_start(struct rte_cryptodev *dev)
+{
+	RTE_SET_USED(dev);
+	return 0;
+}
+
 static int
 mlx5_crypto_dev_close(struct rte_cryptodev *dev)
 {
@@ -360,8 +373,8 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 
 static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_configure			= mlx5_crypto_dev_configure,
-	.dev_start			= NULL,
-	.dev_stop			= NULL,
+	.dev_start			= mlx5_crypto_dev_start,
+	.dev_stop			= mlx5_crypto_dev_stop,
 	.dev_close			= mlx5_crypto_dev_close,
 	.dev_infos_get			= mlx5_crypto_dev_infos_get,
 	.stats_get			= NULL,
-- 
2.27.0

[dpdk-dev] [PATCH v7 07/16] crypto/mlx5: add memory region management

[Shiri Kuzin]

Mellanox user space drivers don't deal with physical addresses as part
of a memory protection mechanism.
The device translates the given virtual address to a physical address
using the given memory key as an address space identifier.
That's why any mbuf virtual address is moved directly to the HW
descriptor(WQE).

The mapping between the virtual address to the physical address is saved
in MR configured by the kernel to the HW.

Each MR has a key that should also be moved to the WQE by the SW.

When the SW sees an unmapped address, it extends the address range and
creates a MR using a system call.

Add memory region cache management:
	- 2 level cache per queue-pair - no locks.
	- 1 shared cache between all the queues using a lock.

Using this way, the MR key search per data-path address is optimized.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Signed-off-by: Michael Baum <michaelba@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 doc/guides/cryptodevs/mlx5.rst    |  6 +++
 drivers/crypto/mlx5/mlx5_crypto.c | 63 +++++++++++++++++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto.h |  3 ++
 3 files changed, 72 insertions(+)

diff --git a/doc/guides/cryptodevs/mlx5.rst b/doc/guides/cryptodevs/mlx5.rst
index dd1d1a615d..ece881220e 100644
--- a/doc/guides/cryptodevs/mlx5.rst
+++ b/doc/guides/cryptodevs/mlx5.rst
@@ -26,6 +26,12 @@ the MKEY is configured to perform crypto operations.
 
 The encryption does not require text to be aligned to the AES block size (128b).
 
+For security reasons and to increase robustness, this driver only deals with virtual
+memory addresses. The way resources allocations are handled by the kernel,
+combined with hardware specifications that allow handling virtual memory
+addresses directly, ensure that DPDK applications cannot access random
+physical memory (or memory that does not belong to the current process).
+
 The PMD uses libibverbs and libmlx5 to access the device firmware or to
 access the hardware components directly.
 There are different levels of objects and bypassing abilities.
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 37e66cf57b..38008dcb28 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -261,6 +261,7 @@ mlx5_crypto_queue_pair_release(struct rte_cryptodev *dev, uint16_t qp_id)
 		claim_zero(mlx5_glue->devx_umem_dereg(qp->umem_obj));
 	if (qp->umem_buf != NULL)
 		rte_free(qp->umem_buf);
+	mlx5_mr_btree_free(&qp->mr_ctrl.cache_bh);
 	mlx5_devx_cq_destroy(&qp->cq_obj);
 	rte_free(qp);
 	dev->data->queue_pairs[qp_id] = NULL;
@@ -342,6 +343,14 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 		DRV_LOG(ERR, "Failed to register QP umem.");
 		goto error;
 	}
+	if (mlx5_mr_btree_init(&qp->mr_ctrl.cache_bh, MLX5_MR_BTREE_CACHE_N,
+			       priv->dev_config.socket_id) != 0) {
+		DRV_LOG(ERR, "Cannot allocate MR Btree for qp %u.",
+			(uint32_t)qp_id);
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	qp->mr_ctrl.dev_gen_ptr = &priv->mr_scache.dev_gen;
 	attr.pd = priv->pdn;
 	attr.uar_index = mlx5_os_get_devx_uar_page_id(priv->uar);
 	attr.cqn = qp->cq_obj.cq->id;
@@ -448,6 +457,40 @@ mlx5_crypto_hw_global_prepare(struct mlx5_crypto_priv *priv)
 	return 0;
 }
 
+/**
+ * Callback for memory event.
+ *
+ * @param event_type
+ *   Memory event type.
+ * @param addr
+ *   Address of memory.
+ * @param len
+ *   Size of memory.
+ */
+static void
+mlx5_crypto_mr_mem_event_cb(enum rte_mem_event event_type, const void *addr,
+			    size_t len, void *arg __rte_unused)
+{
+	struct mlx5_crypto_priv *priv;
+
+	/* Must be called from the primary process. */
+	MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);
+	switch (event_type) {
+	case RTE_MEM_EVENT_FREE:
+		pthread_mutex_lock(&priv_list_lock);
+		/* Iterate all the existing mlx5 devices. */
+		TAILQ_FOREACH(priv, &mlx5_crypto_priv_list, next)
+			mlx5_free_mr_by_addr(&priv->mr_scache,
+					     priv->ctx->device->name,
+					     addr, len);
+		pthread_mutex_unlock(&priv_list_lock);
+		break;
+	case RTE_MEM_EVENT_ALLOC:
+	default:
+		break;
+	}
+}
+
 /**
  * DPDK callback to register a PCI device.
  *
@@ -530,6 +573,22 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 		claim_zero(mlx5_glue->close_device(priv->ctx));
 		return -1;
 	}
+	if (mlx5_mr_btree_init(&priv->mr_scache.cache,
+			     MLX5_MR_BTREE_CACHE_N * 2, rte_socket_id()) != 0) {
+		DRV_LOG(ERR, "Failed to allocate shared cache MR memory.");
+		mlx5_crypto_hw_global_release(priv);
+		rte_cryptodev_pmd_destroy(priv->crypto_dev);
+		claim_zero(mlx5_glue->close_device(priv->ctx));
+		rte_errno = ENOMEM;
+		return -rte_errno;
+	}
+	priv->mr_scache.reg_mr_cb = mlx5_common_verbs_reg_mr;
+	priv->mr_scache.dereg_mr_cb = mlx5_common_verbs_dereg_mr;
+	/* Register callback function for global shared MR cache management. */
+	if (TAILQ_EMPTY(&mlx5_crypto_priv_list))
+		rte_mem_event_callback_register("MLX5_MEM_EVENT_CB",
+						mlx5_crypto_mr_mem_event_cb,
+						NULL);
 	pthread_mutex_lock(&priv_list_lock);
 	TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next);
 	pthread_mutex_unlock(&priv_list_lock);
@@ -549,6 +608,10 @@ mlx5_crypto_pci_remove(struct rte_pci_device *pdev)
 		TAILQ_REMOVE(&mlx5_crypto_priv_list, priv, next);
 	pthread_mutex_unlock(&priv_list_lock);
 	if (priv) {
+		if (TAILQ_EMPTY(&mlx5_crypto_priv_list))
+			rte_mem_event_callback_unregister("MLX5_MEM_EVENT_CB",
+							  NULL);
+		mlx5_mr_release_cache(&priv->mr_scache);
 		mlx5_crypto_hw_global_release(priv);
 		rte_cryptodev_pmd_destroy(priv->crypto_dev);
 		claim_zero(mlx5_glue->close_device(priv->ctx));
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 1c3c4855ec..a9fbc0aa82 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -12,6 +12,7 @@
 
 #include <mlx5_common_utils.h>
 #include <mlx5_common_devx.h>
+#include <mlx5_common_mr.h>
 
 #define MLX5_CRYPTO_DEK_HTABLE_SZ (1 << 11)
 #define MLX5_CRYPTO_KEY_LENGTH 80
@@ -27,6 +28,7 @@ struct mlx5_crypto_priv {
 	struct ibv_pd *pd;
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
 	struct rte_cryptodev_config dev_config;
+	struct mlx5_mr_share_cache mr_scache; /* Global shared MR cache. */
 };
 
 struct mlx5_crypto_qp {
@@ -36,6 +38,7 @@ struct mlx5_crypto_qp {
 	void *umem_buf;
 	volatile uint32_t *db_rec;
 	struct rte_crypto_op **ops;
+	struct mlx5_mr_ctrl mr_ctrl;
 };
 
 struct mlx5_crypto_dek {
-- 
2.27.0

[dpdk-dev] [PATCH v7 08/16] crypto/mlx5: create login object using DevX

[Shiri Kuzin]

To work with crypto engines that are marked with wrapped_import_method,
a login session is required.
A crypto login object needs to be created using DevX.

The crypto login object contains:
	- The credential pointer.
	- The import_KEK pointer to be used for all secured information
	  communicated in crypto commands (key fields), including the
	  provided credential in this command.
	- The credential secret, wrapped by the import_KEK indicated in
	  this command. Size includes 8 bytes IV for wrapping.

Added devargs for the required login values:
	- wcs_file - path to the file containing the credential.
	- import_kek_id - the import KEK pointer.
	- credential_id - the credential pointer.

Create the login DevX object in pci_probe function and destroy it in
pci_remove.
Destroying the crypto login object means logout.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 doc/guides/cryptodevs/mlx5.rst    |  60 +++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto.c | 103 ++++++++++++++++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto.h |   7 ++
 3 files changed, 170 insertions(+)

diff --git a/doc/guides/cryptodevs/mlx5.rst b/doc/guides/cryptodevs/mlx5.rst
index ece881220e..6cae5affbd 100644
--- a/doc/guides/cryptodevs/mlx5.rst
+++ b/doc/guides/cryptodevs/mlx5.rst
@@ -44,6 +44,51 @@ To get the best performances:
 Enabling librte_crypto_mlx5 causes DPDK applications to be linked against
 libibverbs.
 
+In order to move the device to crypto operational mode, credential and KEK
+(Key Encrypting Key) should be set as the first step.
+The credential will be used by the software in order to perform crypto login, and the KEK is
+the AES Key Wrap Algorithm (rfc3394) key that will be used for sensitive data
+wrapping.
+The credential and the AES-XTS keys should be provided to the hardware, as ciphertext
+encrypted by the KEK.
+
+When crypto engines are defined to work in wrapped import method, they come out
+of the factory in Commissioning mode, and thus, cannot be used for crypto operations
+yet. A dedicated tool is used for changing the mode from Commissioning to
+Operational, while setting the first import_KEK and credential in plaintext.
+The mlxreg dedicated tool should be used as follows:
+
+- Set CRYPTO_OPERATIONAL register to set the device in crypto operational mode.
+
+  The input to this tool is:
+    The first credential in plaintext, 40B.
+    The first import_KEK in plaintext: kek size 0 for 16B or 1 for 32B, kek data.
+
+  Example:
+  mlxreg -d /dev/mst/mt4123_pciconf0 --reg_name CRYPTO_OPERATIONAL --get
+
+  The "wrapped_crypto_operational" value will be "0x00000000".
+  The command to set the register should be executed only once, and all the
+  values mentioned above should be specified in the same command.
+
+  Example:
+  mlxreg -d /dev/mst/mt4123_pciconf0 --reg_name CRYPTO_OPERATIONAL
+  --set "credential[0]=0x10000000, credential[1]=0x10000000, kek[0]=0x00000000"
+
+  All values not specified will remain 0.
+  "wrapped_crypto_going_to_commissioning" and  "wrapped_crypto_operational"
+  should not be specified.
+
+  All the device ports should set it in order to move to operational mode.
+
+- Query CRYPTO_OPERATIONAL register to make sure the device is in Operational
+  mode.
+
+  Example:
+  mlxreg -d /dev/mst/mt4123_pciconf0 --reg_name CRYPTO_OPERATIONAL --get
+  The "wrapped_crypto_operational" value will be "0x00000001" if the mode was
+  successfully changed to operational mode.
+
 
 Driver options
 --------------
@@ -53,6 +98,21 @@ Driver options
   Select the class of the driver that should probe the device.
   `crypto` for the mlx5 crypto driver.
 
+- ``wcs_file`` parameter [string] - mandatory
+
+  File path including only the wrapped credential in string format of hexadecimal
+  numbers, represent 48 bytes (8 bytes IV added by the AES key wrap algorithm).
+
+- ``import_kek_id`` parameter [int]
+
+  The identifier of the KEK, default value is 0 represents the operational
+  register import_kek..
+
+- ``credential_id`` parameter [int]
+
+  The identifier of the credential, default value is 0 represents the operational
+  register credential.
+
 
 Supported NICs
 --------------
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 38008dcb28..cbaa2e52ff 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -457,6 +457,101 @@ mlx5_crypto_hw_global_prepare(struct mlx5_crypto_priv *priv)
 	return 0;
 }
 
+
+static int
+mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque)
+{
+	struct mlx5_crypto_devarg_params *devarg_prms = opaque;
+	struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr;
+	unsigned long tmp;
+	FILE *file;
+	int ret;
+	int i;
+
+	if (strcmp(key, "class") == 0)
+		return 0;
+	if (strcmp(key, "wcs_file") == 0) {
+		file = fopen(val, "rb");
+		if (file == NULL) {
+			rte_errno = ENOTSUP;
+			return -rte_errno;
+		}
+		for (i = 0 ; i < MLX5_CRYPTO_CREDENTIAL_SIZE ; i++) {
+			ret = fscanf(file, "%02hhX", &attr->credential[i]);
+			if (ret <= 0) {
+				fclose(file);
+				DRV_LOG(ERR,
+					"Failed to read credential from file.");
+				rte_errno = EINVAL;
+				return -rte_errno;
+			}
+		}
+		fclose(file);
+		devarg_prms->login_devarg = true;
+		return 0;
+	}
+	errno = 0;
+	tmp = strtoul(val, NULL, 0);
+	if (errno) {
+		DRV_LOG(WARNING, "%s: \"%s\" is an invalid integer.", key, val);
+		return -errno;
+	}
+	if (strcmp(key, "import_kek_id") == 0)
+		attr->session_import_kek_ptr = (uint32_t)tmp;
+	else if (strcmp(key, "credential_id") == 0)
+		attr->credential_pointer = (uint32_t)tmp;
+	else
+		DRV_LOG(WARNING, "Invalid key %s.", key);
+	return 0;
+}
+
+static struct mlx5_devx_obj *
+mlx5_crypto_config_login(struct rte_devargs *devargs,
+			 struct ibv_context *ctx)
+{
+	/*
+	 * Set credential pointer and session import KEK pointer to a default
+	 * value of 0.
+	 */
+	struct mlx5_crypto_devarg_params login = {
+			.login_devarg = false,
+			.login_attr = {
+					.credential_pointer = 0,
+					.session_import_kek_ptr = 0,
+			}
+	};
+	struct rte_kvargs *kvlist;
+
+	if (devargs == NULL) {
+		DRV_LOG(ERR,
+	"No login devargs in order to enable crypto operations in the device.");
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	kvlist = rte_kvargs_parse(devargs->args, NULL);
+	if (kvlist == NULL) {
+		DRV_LOG(ERR, "Failed to parse devargs.");
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	if (rte_kvargs_process(kvlist, NULL, mlx5_crypto_args_check_handler,
+			   &login) != 0) {
+		DRV_LOG(ERR, "Devargs handler function Failed.");
+		rte_kvargs_free(kvlist);
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	rte_kvargs_free(kvlist);
+	if (login.login_devarg == false) {
+		DRV_LOG(ERR,
+	"No login credential devarg in order to enable crypto operations "
+	"in the device.");
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	return mlx5_devx_cmd_create_crypto_login_obj(ctx, &login.login_attr);
+}
+
 /**
  * Callback for memory event.
  *
@@ -512,6 +607,7 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	struct ibv_device *ibv;
 	struct rte_cryptodev *crypto_dev;
 	struct ibv_context *ctx;
+	struct mlx5_devx_obj *login;
 	struct mlx5_crypto_priv *priv;
 	struct mlx5_hca_attr attr = { 0 };
 	struct rte_cryptodev_pmd_init_params init_params = {
@@ -550,6 +646,11 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 		rte_errno = ENOTSUP;
 		return -ENOTSUP;
 	}
+	login = mlx5_crypto_config_login(pci_dev->device.devargs, ctx);
+	if (login == NULL) {
+		DRV_LOG(ERR, "Failed to configure login.");
+		return -rte_errno;
+	}
 	crypto_dev = rte_cryptodev_pmd_create(ibv->name, &pci_dev->device,
 					&init_params);
 	if (crypto_dev == NULL) {
@@ -566,6 +667,7 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	crypto_dev->driver_id = mlx5_crypto_driver_id;
 	priv = crypto_dev->data->dev_private;
 	priv->ctx = ctx;
+	priv->login_obj = login;
 	priv->pci_dev = pci_dev;
 	priv->crypto_dev = crypto_dev;
 	if (mlx5_crypto_hw_global_prepare(priv) != 0) {
@@ -614,6 +716,7 @@ mlx5_crypto_pci_remove(struct rte_pci_device *pdev)
 		mlx5_mr_release_cache(&priv->mr_scache);
 		mlx5_crypto_hw_global_release(priv);
 		rte_cryptodev_pmd_destroy(priv->crypto_dev);
+		claim_zero(mlx5_devx_cmd_destroy(priv->login_obj));
 		claim_zero(mlx5_glue->close_device(priv->ctx));
 	}
 	return 0;
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index a9fbc0aa82..b98c621c23 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -29,6 +29,7 @@ struct mlx5_crypto_priv {
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
 	struct rte_cryptodev_config dev_config;
 	struct mlx5_mr_share_cache mr_scache; /* Global shared MR cache. */
+	struct mlx5_devx_obj *login_obj;
 };
 
 struct mlx5_crypto_qp {
@@ -48,6 +49,12 @@ struct mlx5_crypto_dek {
 	bool size_is_48; /* Whether the key\data size is 48 bytes or not. */
 } __rte_cache_aligned;
 
+
+struct mlx5_crypto_devarg_params {
+	bool login_devarg;
+	struct mlx5_devx_crypto_login_attr login_attr;
+};
+
 int
 mlx5_crypto_dek_destroy(struct mlx5_crypto_priv *priv,
 			struct mlx5_crypto_dek *dek);
-- 
2.27.0

[dpdk-dev] [PATCH v7 09/16] crypto/mlx5: add keytag devarg

[Shiri Kuzin]

From: Suanming Mou <suanmingm@nvidia.com>

A keytag is a piece of data encrypted together with a DEK.

When a DEK is referenced by an MKEY.bsf through its index, the keytag is
also supplied in the BSF as plaintext. The HW will decrypt the DEK (and
the attached keytag) and will fail the operation if the keytags don't
match.

This commit adds the configuration of the keytag with devargs.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
 doc/guides/cryptodevs/mlx5.rst    |  7 +++++
 drivers/crypto/mlx5/mlx5_crypto.c | 50 +++++++++++++++++--------------
 drivers/crypto/mlx5/mlx5_crypto.h |  3 +-
 3 files changed, 37 insertions(+), 23 deletions(-)

diff --git a/doc/guides/cryptodevs/mlx5.rst b/doc/guides/cryptodevs/mlx5.rst
index 6cae5affbd..c3632484a5 100644
--- a/doc/guides/cryptodevs/mlx5.rst
+++ b/doc/guides/cryptodevs/mlx5.rst
@@ -52,6 +52,9 @@ wrapping.
 The credential and the AES-XTS keys should be provided to the hardware, as ciphertext
 encrypted by the KEK.
 
+A keytag (64 bits) should be appended to the AES-XTS keys (before wrapping),
+and will be validated when the hardware attempts to access it.
+
 When crypto engines are defined to work in wrapped import method, they come out
 of the factory in Commissioning mode, and thus, cannot be used for crypto operations
 yet. A dedicated tool is used for changing the mode from Commissioning to
@@ -113,6 +116,10 @@ Driver options
   The identifier of the credential, default value is 0 represents the operational
   register credential.
 
+- ``keytag`` parameter [int]
+
+  The plaintext of the keytag appanded to the AES-XTS keys, default value is 0.
+
 
 Supported NICs
 --------------
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index cbaa2e52ff..0f786ac4ca 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -500,56 +500,52 @@ mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque)
 		attr->session_import_kek_ptr = (uint32_t)tmp;
 	else if (strcmp(key, "credential_id") == 0)
 		attr->credential_pointer = (uint32_t)tmp;
+	else if (strcmp(key, "keytag") == 0)
+		devarg_prms->keytag = tmp;
 	else
 		DRV_LOG(WARNING, "Invalid key %s.", key);
 	return 0;
 }
 
-static struct mlx5_devx_obj *
-mlx5_crypto_config_login(struct rte_devargs *devargs,
-			 struct ibv_context *ctx)
+static int
+mlx5_crypto_parse_devargs(struct rte_devargs *devargs,
+			  struct mlx5_crypto_devarg_params *devarg_prms)
 {
-	/*
-	 * Set credential pointer and session import KEK pointer to a default
-	 * value of 0.
-	 */
-	struct mlx5_crypto_devarg_params login = {
-			.login_devarg = false,
-			.login_attr = {
-					.credential_pointer = 0,
-					.session_import_kek_ptr = 0,
-			}
-	};
+	struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr;
 	struct rte_kvargs *kvlist;
 
+	/* Default values. */
+	attr->credential_pointer = 0;
+	attr->session_import_kek_ptr = 0;
+	devarg_prms->keytag = 0;
 	if (devargs == NULL) {
 		DRV_LOG(ERR,
 	"No login devargs in order to enable crypto operations in the device.");
 		rte_errno = EINVAL;
-		return NULL;
+		return -1;
 	}
 	kvlist = rte_kvargs_parse(devargs->args, NULL);
 	if (kvlist == NULL) {
 		DRV_LOG(ERR, "Failed to parse devargs.");
 		rte_errno = EINVAL;
-		return NULL;
+		return -1;
 	}
 	if (rte_kvargs_process(kvlist, NULL, mlx5_crypto_args_check_handler,
-			   &login) != 0) {
+			   devarg_prms) != 0) {
 		DRV_LOG(ERR, "Devargs handler function Failed.");
 		rte_kvargs_free(kvlist);
 		rte_errno = EINVAL;
-		return NULL;
+		return -1;
 	}
 	rte_kvargs_free(kvlist);
-	if (login.login_devarg == false) {
+	if (devarg_prms->login_devarg == false) {
 		DRV_LOG(ERR,
 	"No login credential devarg in order to enable crypto operations "
 	"in the device.");
 		rte_errno = EINVAL;
-		return NULL;
+		return -1;
 	}
-	return mlx5_devx_cmd_create_crypto_login_obj(ctx, &login.login_attr);
+	return 0;
 }
 
 /**
@@ -609,6 +605,7 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	struct ibv_context *ctx;
 	struct mlx5_devx_obj *login;
 	struct mlx5_crypto_priv *priv;
+	struct mlx5_crypto_devarg_params devarg_prms = { 0 };
 	struct mlx5_hca_attr attr = { 0 };
 	struct rte_cryptodev_pmd_init_params init_params = {
 		.name = "",
@@ -617,6 +614,8 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 		.max_nb_queue_pairs =
 				RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
 	};
+	int ret;
+
 	RTE_SET_USED(pci_drv);
 	if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
 		DRV_LOG(ERR, "Non-primary process type is not supported.");
@@ -646,7 +645,13 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 		rte_errno = ENOTSUP;
 		return -ENOTSUP;
 	}
-	login = mlx5_crypto_config_login(pci_dev->device.devargs, ctx);
+	ret = mlx5_crypto_parse_devargs(pci_dev->device.devargs, &devarg_prms);
+	if (ret) {
+		DRV_LOG(ERR, "Failed to parse devargs.");
+		return -rte_errno;
+	}
+	login = mlx5_devx_cmd_create_crypto_login_obj(ctx,
+						      &devarg_prms.login_attr);
 	if (login == NULL) {
 		DRV_LOG(ERR, "Failed to configure login.");
 		return -rte_errno;
@@ -686,6 +691,7 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	}
 	priv->mr_scache.reg_mr_cb = mlx5_common_verbs_reg_mr;
 	priv->mr_scache.dereg_mr_cb = mlx5_common_verbs_dereg_mr;
+	priv->keytag = rte_cpu_to_be_64(devarg_prms.keytag);
 	/* Register callback function for global shared MR cache management. */
 	if (TAILQ_EMPTY(&mlx5_crypto_priv_list))
 		rte_mem_event_callback_register("MLX5_MEM_EVENT_CB",
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index b98c621c23..d374d3c4e9 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -30,6 +30,7 @@ struct mlx5_crypto_priv {
 	struct rte_cryptodev_config dev_config;
 	struct mlx5_mr_share_cache mr_scache; /* Global shared MR cache. */
 	struct mlx5_devx_obj *login_obj;
+	uint64_t keytag;
 };
 
 struct mlx5_crypto_qp {
@@ -49,10 +50,10 @@ struct mlx5_crypto_dek {
 	bool size_is_48; /* Whether the key\data size is 48 bytes or not. */
 } __rte_cache_aligned;
 
-
 struct mlx5_crypto_devarg_params {
 	bool login_devarg;
 	struct mlx5_devx_crypto_login_attr login_attr;
+	uint64_t keytag;
 };
 
 int
-- 
2.27.0

[dpdk-dev] [PATCH v7 10/16] crypto/mlx5: add maximum segments devarg

[Shiri Kuzin]

From: Suanming Mou <suanmingm@nvidia.com>

The mlx5 HW crypto operations are done by attaching crypto property
to a memory region. Once done, every access to the memory via the
crypto-enabled memory region will result with in-line encryption or
decryption of the data.

As a result, the design choice is to provide two types of WQEs. One
is UMR WQE which sets the crypto property and the other is rdma write
WQE which sends DMA command to copy data from local MR to remote MR.

The size of the WQEs will be defined by a new devarg called
max_segs_num.

This devarg also defines the maximum segments in mbuf chain that will be
supported for crypto operations.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
 doc/guides/cryptodevs/mlx5.rst    |  4 ++++
 drivers/crypto/mlx5/mlx5_crypto.c | 33 +++++++++++++++++++++++++++----
 drivers/crypto/mlx5/mlx5_crypto.h |  6 ++++++
 3 files changed, 39 insertions(+), 4 deletions(-)

diff --git a/doc/guides/cryptodevs/mlx5.rst b/doc/guides/cryptodevs/mlx5.rst
index c3632484a5..dd4705b744 100644
--- a/doc/guides/cryptodevs/mlx5.rst
+++ b/doc/guides/cryptodevs/mlx5.rst
@@ -120,6 +120,10 @@ Driver options
 
   The plaintext of the keytag appanded to the AES-XTS keys, default value is 0.
 
+- ``max_segs_num`` parameter [int]
+
+  Maximum number of mbuf chain segments(src or dest), default value is 8.
+
 
 Supported NICs
 --------------
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 0f786ac4ca..fc251f3f4c 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -21,6 +21,7 @@
 #define MLX5_CRYPTO_DRIVER_NAME mlx5_crypto
 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
 #define MLX5_CRYPTO_MAX_QPS 1024
+#define MLX5_CRYPTO_MAX_SEGS 56
 
 #define MLX5_CRYPTO_FEATURE_FLAGS \
 	(RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | \
@@ -496,14 +497,24 @@ mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque)
 		DRV_LOG(WARNING, "%s: \"%s\" is an invalid integer.", key, val);
 		return -errno;
 	}
-	if (strcmp(key, "import_kek_id") == 0)
+	if (strcmp(key, "max_segs_num") == 0) {
+		if (!tmp || tmp > MLX5_CRYPTO_MAX_SEGS) {
+			DRV_LOG(WARNING, "Invalid max_segs_num: %d, should"
+				" be less than %d.",
+				(uint32_t)tmp, MLX5_CRYPTO_MAX_SEGS);
+			rte_errno = EINVAL;
+			return -rte_errno;
+		}
+		devarg_prms->max_segs_num = (uint32_t)tmp;
+	} else if (strcmp(key, "import_kek_id") == 0) {
 		attr->session_import_kek_ptr = (uint32_t)tmp;
-	else if (strcmp(key, "credential_id") == 0)
+	} else if (strcmp(key, "credential_id") == 0) {
 		attr->credential_pointer = (uint32_t)tmp;
-	else if (strcmp(key, "keytag") == 0)
+	} else if (strcmp(key, "keytag") == 0) {
 		devarg_prms->keytag = tmp;
-	else
+	} else {
 		DRV_LOG(WARNING, "Invalid key %s.", key);
+	}
 	return 0;
 }
 
@@ -518,6 +529,7 @@ mlx5_crypto_parse_devargs(struct rte_devargs *devargs,
 	attr->credential_pointer = 0;
 	attr->session_import_kek_ptr = 0;
 	devarg_prms->keytag = 0;
+	devarg_prms->max_segs_num = 8;
 	if (devargs == NULL) {
 		DRV_LOG(ERR,
 	"No login devargs in order to enable crypto operations in the device.");
@@ -614,6 +626,7 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 		.max_nb_queue_pairs =
 				RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
 	};
+	uint16_t rdmw_wqe_size;
 	int ret;
 
 	RTE_SET_USED(pci_drv);
@@ -692,6 +705,18 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	priv->mr_scache.reg_mr_cb = mlx5_common_verbs_reg_mr;
 	priv->mr_scache.dereg_mr_cb = mlx5_common_verbs_dereg_mr;
 	priv->keytag = rte_cpu_to_be_64(devarg_prms.keytag);
+	priv->max_segs_num = devarg_prms.max_segs_num;
+	priv->umr_wqe_size = sizeof(struct mlx5_wqe_umr_bsf_seg) +
+			     sizeof(struct mlx5_umr_wqe) +
+			     RTE_ALIGN(priv->max_segs_num, 4) *
+			     sizeof(struct mlx5_wqe_dseg);
+	rdmw_wqe_size = sizeof(struct mlx5_rdma_write_wqe) +
+			      sizeof(struct mlx5_wqe_dseg) *
+			      (priv->max_segs_num <= 2 ? 2 : 2 +
+			       RTE_ALIGN(priv->max_segs_num - 2, 4));
+	priv->wqe_set_size = priv->umr_wqe_size + rdmw_wqe_size;
+	priv->umr_wqe_stride = priv->umr_wqe_size / MLX5_SEND_WQE_BB;
+	priv->max_rdmar_ds = rdmw_wqe_size / sizeof(struct mlx5_wqe_dseg);
 	/* Register callback function for global shared MR cache management. */
 	if (TAILQ_EMPTY(&mlx5_crypto_priv_list))
 		rte_mem_event_callback_register("MLX5_MEM_EVENT_CB",
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index d374d3c4e9..7a386fe76a 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -25,12 +25,17 @@ struct mlx5_crypto_priv {
 	struct rte_cryptodev *crypto_dev;
 	void *uar; /* User Access Region. */
 	uint32_t pdn; /* Protection Domain number. */
+	uint32_t max_segs_num; /* Maximum supported data segs. */
 	struct ibv_pd *pd;
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
 	struct rte_cryptodev_config dev_config;
 	struct mlx5_mr_share_cache mr_scache; /* Global shared MR cache. */
 	struct mlx5_devx_obj *login_obj;
 	uint64_t keytag;
+	uint16_t wqe_set_size;
+	uint16_t umr_wqe_size;
+	uint16_t umr_wqe_stride;
+	uint16_t max_rdmar_ds;
 };
 
 struct mlx5_crypto_qp {
@@ -54,6 +59,7 @@ struct mlx5_crypto_devarg_params {
 	bool login_devarg;
 	struct mlx5_devx_crypto_login_attr login_attr;
 	uint64_t keytag;
+	uint32_t max_segs_num;
 };
 
 int
-- 
2.27.0

[dpdk-dev] [PATCH v7 11/16] crypto/mlx5: add WQE set initialization

[Shiri Kuzin]

From: Suanming Mou <suanmingm@nvidia.com>

Currently, HW handles the WQEs much faster than the software,
Using the constant WQE set layout can initialize most of the WQE
segments in advanced, and software only needs to configure very
limited segments in datapath. This accelerates the software WQE
organize in datapath.

This commit initializes the fixed WQE set segments.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 83 +++++++++++++++++++++++++++++--
 drivers/crypto/mlx5/mlx5_crypto.h | 10 +++-
 2 files changed, 87 insertions(+), 6 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index fc251f3f4c..e5f8d96ff7 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -297,6 +297,69 @@ mlx5_crypto_qp2rts(struct mlx5_crypto_qp *qp)
 	return 0;
 }
 
+static void
+mlx5_crypto_qp_init(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp)
+{
+	uint32_t i;
+
+	for (i = 0 ; i < qp->entries_n; i++) {
+		struct mlx5_wqe_cseg *cseg = RTE_PTR_ADD(qp->umem_buf, i *
+							 priv->wqe_set_size);
+		struct mlx5_wqe_umr_cseg *ucseg = (struct mlx5_wqe_umr_cseg *)
+								     (cseg + 1);
+		struct mlx5_wqe_umr_bsf_seg *bsf =
+			(struct mlx5_wqe_umr_bsf_seg *)(RTE_PTR_ADD(cseg,
+						       priv->umr_wqe_size)) - 1;
+		struct mlx5_wqe_rseg *rseg;
+
+		/* Init UMR WQE. */
+		cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj->id << 8) |
+					 (priv->umr_wqe_size / MLX5_WSEG_SIZE));
+		cseg->flags = RTE_BE32(MLX5_COMP_ONLY_FIRST_ERR <<
+				       MLX5_COMP_MODE_OFFSET);
+		cseg->misc = rte_cpu_to_be_32(qp->mkey[i]->id);
+		ucseg->if_cf_toe_cq_res = RTE_BE32(1u << MLX5_UMRC_IF_OFFSET);
+		ucseg->mkey_mask = RTE_BE64(1u << 0); /* Mkey length bit. */
+		ucseg->ko_to_bs = rte_cpu_to_be_32
+			((RTE_ALIGN(priv->max_segs_num, 4u) <<
+			 MLX5_UMRC_KO_OFFSET) | (4 << MLX5_UMRC_TO_BS_OFFSET));
+		bsf->keytag = priv->keytag;
+		/* Init RDMA WRITE WQE. */
+		cseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size);
+		cseg->flags = RTE_BE32((MLX5_COMP_ALWAYS <<
+				      MLX5_COMP_MODE_OFFSET) |
+				      MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE);
+		rseg = (struct mlx5_wqe_rseg *)(cseg + 1);
+		rseg->rkey = rte_cpu_to_be_32(qp->mkey[i]->id);
+	}
+}
+
+static int
+mlx5_crypto_indirect_mkeys_prepare(struct mlx5_crypto_priv *priv,
+				  struct mlx5_crypto_qp *qp)
+{
+	struct mlx5_umr_wqe *umr;
+	uint32_t i;
+	struct mlx5_devx_mkey_attr attr = {
+		.pd = priv->pdn,
+		.umr_en = 1,
+		.crypto_en = 1,
+		.set_remote_rw = 1,
+		.klm_num = RTE_ALIGN(priv->max_segs_num, 4),
+	};
+
+	for (umr = (struct mlx5_umr_wqe *)qp->umem_buf, i = 0;
+	   i < qp->entries_n; i++, umr = RTE_PTR_ADD(umr, priv->wqe_set_size)) {
+		attr.klm_array = (struct mlx5_klm *)&umr->kseg[0];
+		qp->mkey[i] = mlx5_devx_cmd_mkey_create(priv->ctx, &attr);
+		if (!qp->mkey[i]) {
+			DRV_LOG(ERR, "Failed to allocate indirect mkey.");
+			return -1;
+		}
+	}
+	return 0;
+}
+
 static int
 mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 			     const struct rte_cryptodev_qp_conf *qp_conf,
@@ -307,7 +370,7 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	struct mlx5_crypto_qp *qp;
 	uint16_t log_nb_desc = rte_log2_u32(qp_conf->nb_descriptors);
 	uint32_t umem_size = RTE_BIT32(log_nb_desc) *
-			      MLX5_CRYPTO_WQE_SET_SIZE +
+			      priv->wqe_set_size +
 			      sizeof(*qp->db_rec) * 2;
 	uint32_t alloc_size = sizeof(*qp);
 	struct mlx5_devx_cq_attr cq_attr = {
@@ -317,7 +380,9 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	if (dev->data->queue_pairs[qp_id] != NULL)
 		mlx5_crypto_queue_pair_release(dev, qp_id);
 	alloc_size = RTE_ALIGN(alloc_size, RTE_CACHE_LINE_SIZE);
-	alloc_size += sizeof(struct rte_crypto_op *) * RTE_BIT32(log_nb_desc);
+	alloc_size += (sizeof(struct rte_crypto_op *) +
+		       sizeof(struct mlx5_devx_obj *)) *
+		       RTE_BIT32(log_nb_desc);
 	qp = rte_zmalloc_socket(__func__, alloc_size, RTE_CACHE_LINE_SIZE,
 				socket_id);
 	if (qp == NULL) {
@@ -362,8 +427,7 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	attr.wq_umem_id = qp->umem_obj->umem_id;
 	attr.wq_umem_offset = 0;
 	attr.dbr_umem_id = qp->umem_obj->umem_id;
-	attr.dbr_address = RTE_BIT64(log_nb_desc) *
-			   MLX5_CRYPTO_WQE_SET_SIZE;
+	attr.dbr_address = RTE_BIT64(log_nb_desc) * priv->wqe_set_size;
 	qp->qp_obj = mlx5_devx_cmd_create_qp(priv->ctx, &attr);
 	if (qp->qp_obj == NULL) {
 		DRV_LOG(ERR, "Failed to create QP(%u).", rte_errno);
@@ -372,8 +436,17 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	qp->db_rec = RTE_PTR_ADD(qp->umem_buf, (uintptr_t)attr.dbr_address);
 	if (mlx5_crypto_qp2rts(qp))
 		goto error;
-	qp->ops = (struct rte_crypto_op **)RTE_ALIGN((uintptr_t)(qp + 1),
+	qp->mkey = (struct mlx5_devx_obj **)RTE_ALIGN((uintptr_t)(qp + 1),
 							   RTE_CACHE_LINE_SIZE);
+	qp->ops = (struct rte_crypto_op **)(qp->mkey + RTE_BIT32(log_nb_desc));
+	qp->entries_n = 1 << log_nb_desc;
+	if (mlx5_crypto_indirect_mkeys_prepare(priv, qp)) {
+		DRV_LOG(ERR, "Cannot allocate indirect memory regions.");
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	mlx5_crypto_qp_init(priv, qp);
+	qp->priv = priv;
 	dev->data->queue_pairs[qp_id] = qp;
 	return 0;
 error:
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 7a386fe76a..d9b1ff8e99 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -16,7 +16,6 @@
 
 #define MLX5_CRYPTO_DEK_HTABLE_SZ (1 << 11)
 #define MLX5_CRYPTO_KEY_LENGTH 80
-#define MLX5_CRYPTO_WQE_SET_SIZE 1024
 
 struct mlx5_crypto_priv {
 	TAILQ_ENTRY(mlx5_crypto_priv) next;
@@ -24,6 +23,7 @@ struct mlx5_crypto_priv {
 	struct rte_pci_device *pci_dev;
 	struct rte_cryptodev *crypto_dev;
 	void *uar; /* User Access Region. */
+	volatile uint64_t *uar_addr;
 	uint32_t pdn; /* Protection Domain number. */
 	uint32_t max_segs_num; /* Maximum supported data segs. */
 	struct ibv_pd *pd;
@@ -39,13 +39,21 @@ struct mlx5_crypto_priv {
 };
 
 struct mlx5_crypto_qp {
+	struct mlx5_crypto_priv *priv;
 	struct mlx5_devx_cq cq_obj;
 	struct mlx5_devx_obj *qp_obj;
+	struct rte_cryptodev_stats stats;
 	struct mlx5dv_devx_umem *umem_obj;
 	void *umem_buf;
 	volatile uint32_t *db_rec;
 	struct rte_crypto_op **ops;
+	struct mlx5_devx_obj **mkey; /* WQE's indirect mekys. */
 	struct mlx5_mr_ctrl mr_ctrl;
+	uint8_t *wqe;
+	uint16_t entries_n;
+	uint16_t pi;
+	uint16_t ci;
+	uint16_t db_pi;
 };
 
 struct mlx5_crypto_dek {
-- 
2.27.0

[dpdk-dev] [PATCH v7 12/16] crypto/mlx5: add enqueue and dequeue operations

[Shiri Kuzin]

From: Suanming Mou <suanmingm@nvidia.com>

The crypto operations are done with the WQE set which contains
one UMR WQE and one rdma write WQE. Most segments of the WQE
set are initialized properly during queue setup, only limited
segments are initialized according to the crypto detail in the
datapath process.

This commit adds the enquue and dequeue operations and updates
the WQE set segments accordingly.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
Signed-off-by: Michael Baum <michaelba@nvidia.com>
---
 doc/guides/cryptodevs/features/mlx5.ini |   5 +
 drivers/crypto/mlx5/mlx5_crypto.c       | 286 +++++++++++++++++++++++-
 drivers/crypto/mlx5/mlx5_crypto.h       |   3 +
 3 files changed, 290 insertions(+), 4 deletions(-)

diff --git a/doc/guides/cryptodevs/features/mlx5.ini b/doc/guides/cryptodevs/features/mlx5.ini
index bd757b5211..a89526add0 100644
--- a/doc/guides/cryptodevs/features/mlx5.ini
+++ b/doc/guides/cryptodevs/features/mlx5.ini
@@ -6,6 +6,11 @@
 [Features]
 Symmetric crypto       = Y
 HW Accelerated         = Y
+In Place SGL           = Y
+OOP SGL In SGL Out     = Y
+OOP SGL In LB  Out     = Y
+OOP LB  In SGL Out     = Y
+OOP LB  In LB  Out     = Y
 Cipher multiple data units = Y
 Cipher wrapped key     = Y
 
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index e5f8d96ff7..b467739a3c 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -25,6 +25,10 @@
 
 #define MLX5_CRYPTO_FEATURE_FLAGS \
 	(RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | \
+	 RTE_CRYPTODEV_FF_IN_PLACE_SGL | RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT | \
+	 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | \
+	 RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT | \
+	 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | \
 	 RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY | \
 	 RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS)
 
@@ -297,6 +301,279 @@ mlx5_crypto_qp2rts(struct mlx5_crypto_qp *qp)
 	return 0;
 }
 
+static __rte_noinline uint32_t
+mlx5_crypto_get_block_size(struct rte_crypto_op *op)
+{
+	uint32_t bl = op->sym->cipher.data.length;
+
+	switch (bl) {
+	case (1 << 20):
+		return RTE_BE32(MLX5_BLOCK_SIZE_1MB << MLX5_BLOCK_SIZE_OFFSET);
+	case (1 << 12):
+		return RTE_BE32(MLX5_BLOCK_SIZE_4096B <<
+				MLX5_BLOCK_SIZE_OFFSET);
+	case (1 << 9):
+		return RTE_BE32(MLX5_BLOCK_SIZE_512B << MLX5_BLOCK_SIZE_OFFSET);
+	default:
+		DRV_LOG(ERR, "Unknown block size: %u.", bl);
+		return UINT32_MAX;
+	}
+}
+
+/**
+ * Query LKey from a packet buffer for QP. If not found, add the mempool.
+ *
+ * @param priv
+ *   Pointer to the priv object.
+ * @param addr
+ *   Search key.
+ * @param mr_ctrl
+ *   Pointer to per-queue MR control structure.
+ * @param ol_flags
+ *   Mbuf offload features.
+ *
+ * @return
+ *   Searched LKey on success, UINT32_MAX on no match.
+ */
+static __rte_always_inline uint32_t
+mlx5_crypto_addr2mr(struct mlx5_crypto_priv *priv, uintptr_t addr,
+		    struct mlx5_mr_ctrl *mr_ctrl, uint64_t ol_flags)
+{
+	uint32_t lkey;
+
+	/* Check generation bit to see if there's any change on existing MRs. */
+	if (unlikely(*mr_ctrl->dev_gen_ptr != mr_ctrl->cur_gen))
+		mlx5_mr_flush_local_cache(mr_ctrl);
+	/* Linear search on MR cache array. */
+	lkey = mlx5_mr_lookup_lkey(mr_ctrl->cache, &mr_ctrl->mru,
+				   MLX5_MR_CACHE_N, addr);
+	if (likely(lkey != UINT32_MAX))
+		return lkey;
+	/* Take slower bottom-half on miss. */
+	return mlx5_mr_addr2mr_bh(priv->pd, 0, &priv->mr_scache, mr_ctrl, addr,
+				  !!(ol_flags & EXT_ATTACHED_MBUF));
+}
+
+static __rte_always_inline uint32_t
+mlx5_crypto_klm_set(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp,
+		      struct rte_mbuf *mbuf, struct mlx5_wqe_dseg *klm,
+		      uint32_t offset, uint32_t *remain)
+{
+	uint32_t data_len = (rte_pktmbuf_data_len(mbuf) - offset);
+	uintptr_t addr = rte_pktmbuf_mtod_offset(mbuf, uintptr_t, offset);
+
+	if (data_len > *remain)
+		data_len = *remain;
+	*remain -= data_len;
+	klm->bcount = rte_cpu_to_be_32(data_len);
+	klm->pbuf = rte_cpu_to_be_64(addr);
+	klm->lkey = mlx5_crypto_addr2mr(priv, addr, &qp->mr_ctrl,
+					mbuf->ol_flags);
+	return klm->lkey;
+
+}
+
+static __rte_always_inline uint32_t
+mlx5_crypto_klms_set(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp,
+		     struct rte_crypto_op *op, struct rte_mbuf *mbuf,
+		     struct mlx5_wqe_dseg *klm)
+{
+	uint32_t remain_len = op->sym->cipher.data.length;
+	uint32_t nb_segs = mbuf->nb_segs;
+	uint32_t klm_n = 1u;
+
+	/* First mbuf needs to take the cipher offset. */
+	if (unlikely(mlx5_crypto_klm_set(priv, qp, mbuf, klm,
+		     op->sym->cipher.data.offset, &remain_len) == UINT32_MAX)) {
+		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return 0;
+	}
+	while (remain_len) {
+		nb_segs--;
+		mbuf = mbuf->next;
+		if (unlikely(mbuf == NULL || nb_segs == 0)) {
+			op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+			return 0;
+		}
+		if (unlikely(mlx5_crypto_klm_set(priv, qp, mbuf, ++klm, 0,
+						 &remain_len) == UINT32_MAX)) {
+			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			return 0;
+		}
+		klm_n++;
+	}
+	return klm_n;
+}
+
+static __rte_always_inline int
+mlx5_crypto_wqe_set(struct mlx5_crypto_priv *priv,
+			 struct mlx5_crypto_qp *qp,
+			 struct rte_crypto_op *op,
+			 struct mlx5_umr_wqe *umr)
+{
+	struct mlx5_crypto_session *sess = get_sym_session_private_data
+				(op->sym->session, mlx5_crypto_driver_id);
+	struct mlx5_wqe_cseg *cseg = &umr->ctr;
+	struct mlx5_wqe_mkey_cseg *mkc = &umr->mkc;
+	struct mlx5_wqe_dseg *klms = &umr->kseg[0];
+	struct mlx5_wqe_umr_bsf_seg *bsf = ((struct mlx5_wqe_umr_bsf_seg *)
+				      RTE_PTR_ADD(umr, priv->umr_wqe_size)) - 1;
+	uint32_t ds;
+	bool ipl = op->sym->m_dst == NULL || op->sym->m_dst == op->sym->m_src;
+	/* Set UMR WQE. */
+	uint32_t klm_n = mlx5_crypto_klms_set(priv, qp, op,
+				   ipl ? op->sym->m_src : op->sym->m_dst, klms);
+
+	if (unlikely(klm_n == 0))
+		return 0;
+	bsf->bs_bpt_eo_es = sess->bs_bpt_eo_es;
+	if (unlikely(!sess->bsp_res)) {
+		bsf->bsp_res = mlx5_crypto_get_block_size(op);
+		if (unlikely(bsf->bsp_res == UINT32_MAX)) {
+			op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+			return 0;
+		}
+	} else {
+		bsf->bsp_res = sess->bsp_res;
+	}
+	bsf->raw_data_size = rte_cpu_to_be_32(op->sym->cipher.data.length);
+	memcpy(bsf->xts_initial_tweak,
+	       rte_crypto_op_ctod_offset(op, uint8_t *, sess->iv_offset), 16);
+	bsf->res_dp = sess->dek_id;
+	mkc->len = rte_cpu_to_be_64(op->sym->cipher.data.length);
+	cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) | MLX5_OPCODE_UMR);
+	qp->db_pi += priv->umr_wqe_stride;
+	/* Set RDMA_WRITE WQE. */
+	cseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size);
+	klms = RTE_PTR_ADD(cseg, sizeof(struct mlx5_rdma_write_wqe));
+	if (!ipl) {
+		klm_n = mlx5_crypto_klms_set(priv, qp, op, op->sym->m_src,
+					     klms);
+		if (unlikely(klm_n == 0))
+			return 0;
+	} else {
+		memcpy(klms, &umr->kseg[0], sizeof(*klms) * klm_n);
+	}
+	ds = 2 + klm_n;
+	cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj->id << 8) | ds);
+	cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) |
+							MLX5_OPCODE_RDMA_WRITE);
+	ds = RTE_ALIGN(ds, 4);
+	qp->db_pi += ds >> 2;
+	/* Set NOP WQE if needed. */
+	if (priv->max_rdmar_ds > ds) {
+		cseg += ds;
+		ds = priv->max_rdmar_ds - ds;
+		cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj->id << 8) | ds);
+		cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) |
+							       MLX5_OPCODE_NOP);
+		qp->db_pi += ds >> 2; /* Here, DS is 4 aligned for sure. */
+	}
+	qp->wqe = (uint8_t *)cseg;
+	return 1;
+}
+
+static __rte_always_inline void
+mlx5_crypto_uar_write(uint64_t val, struct mlx5_crypto_priv *priv)
+{
+#ifdef RTE_ARCH_64
+	*priv->uar_addr = val;
+#else /* !RTE_ARCH_64 */
+	rte_spinlock_lock(&priv->uar32_sl);
+	*(volatile uint32_t *)priv->uar_addr = val;
+	rte_io_wmb();
+	*((volatile uint32_t *)priv->uar_addr + 1) = val >> 32;
+	rte_spinlock_unlock(&priv->uar32_sl);
+#endif
+}
+
+static uint16_t
+mlx5_crypto_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
+			  uint16_t nb_ops)
+{
+	struct mlx5_crypto_qp *qp = queue_pair;
+	struct mlx5_crypto_priv *priv = qp->priv;
+	struct mlx5_umr_wqe *umr;
+	struct rte_crypto_op *op;
+	uint16_t mask = qp->entries_n - 1;
+	uint16_t remain = qp->entries_n - (qp->pi - qp->ci);
+
+	if (remain < nb_ops)
+		nb_ops = remain;
+	else
+		remain = nb_ops;
+	if (unlikely(remain == 0))
+		return 0;
+	do {
+		op = *ops++;
+		umr = RTE_PTR_ADD(qp->umem_buf, priv->wqe_set_size * qp->pi);
+		if (unlikely(mlx5_crypto_wqe_set(priv, qp, op, umr) == 0)) {
+			if (remain != nb_ops)
+				break;
+			return 0;
+		}
+		qp->ops[qp->pi] = op;
+		qp->pi = (qp->pi + 1) & mask;
+	} while (--remain);
+	rte_io_wmb();
+	qp->db_rec[MLX5_SND_DBR] = rte_cpu_to_be_32(qp->db_pi);
+	rte_wmb();
+	mlx5_crypto_uar_write(*(volatile uint64_t *)qp->wqe, qp->priv);
+	rte_wmb();
+	return nb_ops;
+}
+
+static __rte_noinline void
+mlx5_crypto_cqe_err_handle(struct mlx5_crypto_qp *qp, struct rte_crypto_op *op)
+{
+	const uint32_t idx = qp->ci & (qp->entries_n - 1);
+	volatile struct mlx5_err_cqe *cqe = (volatile struct mlx5_err_cqe *)
+							&qp->cq_obj.cqes[idx];
+
+	op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+	DRV_LOG(ERR, "CQE ERR:%x.\n", rte_be_to_cpu_32(cqe->syndrome));
+}
+
+static uint16_t
+mlx5_crypto_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
+			  uint16_t nb_ops)
+{
+	struct mlx5_crypto_qp *qp = queue_pair;
+	volatile struct mlx5_cqe *restrict cqe;
+	struct rte_crypto_op *restrict op;
+	const unsigned int cq_size = qp->entries_n;
+	const unsigned int mask = cq_size - 1;
+	uint32_t idx;
+	uint32_t next_idx = qp->ci & mask;
+	const uint16_t max = RTE_MIN((uint16_t)(qp->pi - qp->ci), nb_ops);
+	uint16_t i = 0;
+	int ret;
+
+	if (unlikely(max == 0))
+		return 0;
+	do {
+		idx = next_idx;
+		next_idx = (qp->ci + 1) & mask;
+		op = qp->ops[idx];
+		cqe = &qp->cq_obj.cqes[idx];
+		ret = check_cqe(cqe, cq_size, qp->ci);
+		rte_io_rmb();
+		if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) {
+			if (unlikely(ret != MLX5_CQE_STATUS_HW_OWN))
+				mlx5_crypto_cqe_err_handle(qp, op);
+			break;
+		}
+		op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+		ops[i++] = op;
+		qp->ci++;
+	} while (i < max);
+	if (likely(i != 0)) {
+		rte_io_wmb();
+		qp->cq_obj.db_rec[0] = rte_cpu_to_be_32(qp->ci);
+	}
+	return i;
+}
+
 static void
 mlx5_crypto_qp_init(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp)
 {
@@ -521,8 +798,9 @@ mlx5_crypto_hw_global_prepare(struct mlx5_crypto_priv *priv)
 	if (mlx5_crypto_pd_create(priv) != 0)
 		return -1;
 	priv->uar = mlx5_devx_alloc_uar(priv->ctx, -1);
-	if (priv->uar == NULL || mlx5_os_get_devx_uar_reg_addr(priv->uar) ==
-	    NULL) {
+	if (priv->uar)
+		priv->uar_addr = mlx5_os_get_devx_uar_reg_addr(priv->uar);
+	if (priv->uar == NULL || priv->uar_addr == NULL) {
 		rte_errno = errno;
 		claim_zero(mlx5_glue->dealloc_pd(priv->pd));
 		DRV_LOG(ERR, "Failed to allocate UAR.");
@@ -752,8 +1030,8 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	DRV_LOG(INFO,
 		"Crypto device %s was created successfully.", ibv->name);
 	crypto_dev->dev_ops = &mlx5_crypto_ops;
-	crypto_dev->dequeue_burst = NULL;
-	crypto_dev->enqueue_burst = NULL;
+	crypto_dev->dequeue_burst = mlx5_crypto_dequeue_burst;
+	crypto_dev->enqueue_burst = mlx5_crypto_enqueue_burst;
 	crypto_dev->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
 	crypto_dev->driver_id = mlx5_crypto_driver_id;
 	priv = crypto_dev->data->dev_private;
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index d9b1ff8e99..1350513b9e 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -36,6 +36,9 @@ struct mlx5_crypto_priv {
 	uint16_t umr_wqe_size;
 	uint16_t umr_wqe_stride;
 	uint16_t max_rdmar_ds;
+#ifndef RTE_ARCH_64
+	rte_spinlock_t uar32_sl;
+#endif /* RTE_ARCH_64 */
 };
 
 struct mlx5_crypto_qp {
-- 
2.27.0

[dpdk-dev] [PATCH v7 13/16] crypto/mlx5: add statistic get and reset operations

[Shiri Kuzin]

From: Suanming Mou <suanmingm@nvidia.com>

This commit adds mlx5 crypto statistic get and reset operations.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 40 ++++++++++++++++++++++++++++---
 1 file changed, 37 insertions(+), 3 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index b467739a3c..0ecca32cce 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -508,13 +508,17 @@ mlx5_crypto_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
 		op = *ops++;
 		umr = RTE_PTR_ADD(qp->umem_buf, priv->wqe_set_size * qp->pi);
 		if (unlikely(mlx5_crypto_wqe_set(priv, qp, op, umr) == 0)) {
-			if (remain != nb_ops)
+			qp->stats.enqueue_err_count++;
+			if (remain != nb_ops) {
+				qp->stats.enqueued_count -= remain;
 				break;
+			}
 			return 0;
 		}
 		qp->ops[qp->pi] = op;
 		qp->pi = (qp->pi + 1) & mask;
 	} while (--remain);
+	qp->stats.enqueued_count += nb_ops;
 	rte_io_wmb();
 	qp->db_rec[MLX5_SND_DBR] = rte_cpu_to_be_32(qp->db_pi);
 	rte_wmb();
@@ -531,6 +535,7 @@ mlx5_crypto_cqe_err_handle(struct mlx5_crypto_qp *qp, struct rte_crypto_op *op)
 							&qp->cq_obj.cqes[idx];
 
 	op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+	qp->stats.dequeue_err_count++;
 	DRV_LOG(ERR, "CQE ERR:%x.\n", rte_be_to_cpu_32(cqe->syndrome));
 }
 
@@ -570,6 +575,7 @@ mlx5_crypto_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
 	if (likely(i != 0)) {
 		rte_io_wmb();
 		qp->cq_obj.db_rec[0] = rte_cpu_to_be_32(qp->ci);
+		qp->stats.dequeued_count += i;
 	}
 	return i;
 }
@@ -731,14 +737,42 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	return -1;
 }
 
+static void
+mlx5_crypto_stats_get(struct rte_cryptodev *dev,
+		      struct rte_cryptodev_stats *stats)
+{
+	int qp_id;
+
+	for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
+		struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
+
+		stats->enqueued_count += qp->stats.enqueued_count;
+		stats->dequeued_count += qp->stats.dequeued_count;
+		stats->enqueue_err_count += qp->stats.enqueue_err_count;
+		stats->dequeue_err_count += qp->stats.dequeue_err_count;
+	}
+}
+
+static void
+mlx5_crypto_stats_reset(struct rte_cryptodev *dev)
+{
+	int qp_id;
+
+	for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
+		struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
+
+		memset(&qp->stats, 0, sizeof(qp->stats));
+	}
+}
+
 static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_configure			= mlx5_crypto_dev_configure,
 	.dev_start			= mlx5_crypto_dev_start,
 	.dev_stop			= mlx5_crypto_dev_stop,
 	.dev_close			= mlx5_crypto_dev_close,
 	.dev_infos_get			= mlx5_crypto_dev_infos_get,
-	.stats_get			= NULL,
-	.stats_reset			= NULL,
+	.stats_get			= mlx5_crypto_stats_get,
+	.stats_reset			= mlx5_crypto_stats_reset,
 	.queue_pair_setup		= mlx5_crypto_queue_pair_setup,
 	.queue_pair_release		= mlx5_crypto_queue_pair_release,
 	.sym_session_get_size		= mlx5_crypto_sym_session_get_size,
-- 
2.27.0

[dpdk-dev] [PATCH v7 14/16] test/crypto: add mlx5 crypto driver

[Shiri Kuzin]

In order to test the new mlx5 crypto PMD, the driver is added to the
crypto test application.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 app/test/test_cryptodev.c      | 7 +++++++
 app/test/test_cryptodev.h      | 2 ++
 doc/guides/cryptodevs/mlx5.rst | 3 +++
 3 files changed, 12 insertions(+)

diff --git a/app/test/test_cryptodev.c b/app/test/test_cryptodev.c
index 31201d93e1..e46668a1e0 100644
--- a/app/test/test_cryptodev.c
+++ b/app/test/test_cryptodev.c
@@ -14569,6 +14569,12 @@ test_cryptodev_cpu_aesni_gcm(void)
 	return rc;
 }
 
+static int
+test_cryptodev_mlx5(void /*argv __rte_unused, int argc __rte_unused*/)
+{
+	return run_cryptodev_testsuite(RTE_STR(CRYPTODEV_NAME_MLX5_PMD));
+}
+
 static int
 test_cryptodev_null(void)
 {
@@ -14797,6 +14803,7 @@ REGISTER_TEST_COMMAND(cryptodev_openssl_autotest, test_cryptodev_openssl);
 REGISTER_TEST_COMMAND(cryptodev_aesni_gcm_autotest, test_cryptodev_aesni_gcm);
 REGISTER_TEST_COMMAND(cryptodev_cpu_aesni_gcm_autotest,
 	test_cryptodev_cpu_aesni_gcm);
+REGISTER_TEST_COMMAND(cryptodev_mlx5_autotest, test_cryptodev_mlx5);
 REGISTER_TEST_COMMAND(cryptodev_null_autotest, test_cryptodev_null);
 REGISTER_TEST_COMMAND(cryptodev_sw_snow3g_autotest, test_cryptodev_sw_snow3g);
 REGISTER_TEST_COMMAND(cryptodev_sw_kasumi_autotest, test_cryptodev_sw_kasumi);
diff --git a/app/test/test_cryptodev.h b/app/test/test_cryptodev.h
index 5bf1e8818b..262e545ee6 100644
--- a/app/test/test_cryptodev.h
+++ b/app/test/test_cryptodev.h
@@ -73,6 +73,8 @@
 #define CRYPTODEV_NAME_BCMFS_PMD	crypto_bcmfs
 #define CRYPTODEV_NAME_CN9K_PMD		crypto_cn9k
 #define CRYPTODEV_NAME_CN10K_PMD	crypto_cn10k
+#define CRYPTODEV_NAME_MLX5_PMD		mlx5_crypto
+
 
 enum cryptodev_api_test_type {
 	CRYPTODEV_API_TEST = 0,
diff --git a/doc/guides/cryptodevs/mlx5.rst b/doc/guides/cryptodevs/mlx5.rst
index dd4705b744..2023dd1b3b 100644
--- a/doc/guides/cryptodevs/mlx5.rst
+++ b/doc/guides/cryptodevs/mlx5.rst
@@ -92,6 +92,9 @@ The mlxreg dedicated tool should be used as follows:
   The "wrapped_crypto_operational" value will be "0x00000001" if the mode was
   successfully changed to operational mode.
 
+  The mlx5 crypto PMD can be verfied by running the test application:
+  ./build/app/test/dpdk-test -c 1 -n 1 -w <dev>,class=crypto,wcs_file=<file_path>
+  RTE>>cryptodev_mlx5_autotest
 
 Driver options
 --------------
-- 
2.27.0

[dpdk-dev] [PATCH v7 15/16] test/crypto: add data-unit and wrapped vectors

[Shiri Kuzin]

The AES-XTS algorithm supports using a wrapped key.
In AES-XTS the data-unit defines the data block size to be
encrypted\decrypted.

Add AES-XTS vectors with a wrapped key.
Add a variable stating whether the key is wrapped or not.
Add the AES-XTS data-unit.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 app/test/test_cryptodev.h                  |    2 +-
 app/test/test_cryptodev_aes_test_vectors.h | 1340 ++++++++++++++++++++
 app/test/test_cryptodev_blockcipher.c      |   10 +-
 app/test/test_cryptodev_blockcipher.h      |    2 +
 4 files changed, 1352 insertions(+), 2 deletions(-)

diff --git a/app/test/test_cryptodev.h b/app/test/test_cryptodev.h
index 262e545ee6..f35dba6837 100644
--- a/app/test/test_cryptodev.h
+++ b/app/test/test_cryptodev.h
@@ -19,7 +19,7 @@
 #define DEFAULT_NUM_XFORMS              (2)
 #define NUM_MBUFS                       (8191)
 #define MBUF_CACHE_SIZE                 (256)
-#define MBUF_DATAPAYLOAD_SIZE		(2048 + DIGEST_BYTE_LENGTH_SHA512)
+#define MBUF_DATAPAYLOAD_SIZE		(4096 + DIGEST_BYTE_LENGTH_SHA512)
 #define MBUF_SIZE			(sizeof(struct rte_mbuf) + \
 		RTE_PKTMBUF_HEADROOM + MBUF_DATAPAYLOAD_SIZE)
 
diff --git a/app/test/test_cryptodev_aes_test_vectors.h b/app/test/test_cryptodev_aes_test_vectors.h
index e404050701..dd7ffde685 100644
--- a/app/test/test_cryptodev_aes_test_vectors.h
+++ b/app/test/test_cryptodev_aes_test_vectors.h
@@ -2609,6 +2609,1298 @@ blockcipher_test_data aes_test_data_xts_key_64_pt_48 = {
 	}
 };
 
+static const uint8_t plaintext_aes256xts_512bytes[] = {
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79
+};
+static const uint8_t ciphertext_aes256xts_512bytes[] = {
+	0xD3, 0x94, 0x3E, 0xC9, 0xD3, 0x43, 0x13, 0xD3,
+	0x07, 0x7E, 0x51, 0x11, 0x97, 0xA5, 0xB1, 0xB2,
+	0xB0, 0x55, 0xF8, 0xD6, 0xA2, 0x10, 0x78, 0x76,
+	0x37, 0x95, 0x4F, 0x25, 0x99, 0xCC, 0x12, 0xD6,
+	0xBB, 0x2A, 0x24, 0xF4, 0x2D, 0x82, 0xFF, 0x56,
+	0xEC, 0x55, 0x3B, 0xDD, 0xF9, 0xE1, 0xC8, 0x48,
+	0x68, 0x16, 0xD2, 0x2B, 0x95, 0x7C, 0x1F, 0xCB,
+	0x32, 0xD6, 0x8F, 0x9A, 0x2E, 0xF7, 0x9B, 0xBE,
+	0x72, 0xFC, 0x55, 0x33, 0x27, 0x21, 0x2F, 0x69,
+	0xCA, 0x76, 0xA5, 0x9B, 0x21, 0x2E, 0x40, 0x57,
+	0x2C, 0x1C, 0x98, 0x41, 0x9A, 0x5E, 0x55, 0x38,
+	0xDE, 0xC2, 0x09, 0x57, 0x32, 0xA3, 0x34, 0x40,
+	0xC1, 0x8D, 0xCF, 0x70, 0x15, 0xF2, 0x6F, 0x4F,
+	0x6A, 0x04, 0xA6, 0x6D, 0xFF, 0x53, 0x25, 0x6A,
+	0x0E, 0xD0, 0x87, 0x63, 0xA1, 0x6C, 0xB1, 0x99,
+	0x4A, 0x42, 0xF5, 0xF6, 0xEA, 0xA4, 0xEB, 0x6D,
+	0x70, 0x9B, 0x0F, 0x85, 0xE2, 0x43, 0x6C, 0x27,
+	0x54, 0x57, 0x52, 0x1C, 0xCF, 0x72, 0x11, 0x83,
+	0xC3, 0xF5, 0xC6, 0xB6, 0x07, 0xEC, 0x1A, 0xF5,
+	0xAC, 0xA1, 0xF2, 0x3A, 0x01, 0x53, 0x0C, 0xA4,
+	0x40, 0x19, 0xAF, 0x7B, 0x2D, 0xE7, 0x56, 0x8B,
+	0x71, 0x5F, 0x8D, 0x96, 0xF1, 0x5D, 0x33, 0xAC,
+	0xE7, 0xAD, 0x96, 0xDB, 0xBB, 0xF7, 0xF5, 0x3F,
+	0x0E, 0x02, 0x2E, 0x80, 0xE9, 0xA2, 0x30, 0xD2,
+	0x82, 0x65, 0xEC, 0x1A, 0xB2, 0xE8, 0x98, 0x23,
+	0x42, 0x79, 0x43, 0x85, 0x3B, 0xF9, 0xFB, 0xBB,
+	0xC5, 0x29, 0xAD, 0x95, 0xAE, 0x22, 0x5B, 0x26,
+	0xDF, 0x76, 0x65, 0x37, 0x17, 0xBC, 0x58, 0xBB,
+	0x1B, 0x0A, 0x71, 0xDC, 0x68, 0xFF, 0x90, 0x36,
+	0x37, 0xB7, 0x49, 0x5E, 0x49, 0x4E, 0xE4, 0x1E,
+	0x24, 0x39, 0x48, 0xC7, 0x68, 0x19, 0xED, 0x94,
+	0xB5, 0xF6, 0x85, 0xFA, 0xE6, 0xB8, 0x2B, 0x9E,
+	0x07, 0x9E, 0xFB, 0x1D, 0x61, 0x06, 0x47, 0x2A,
+	0x3E, 0x1E, 0xD8, 0x52, 0xFB, 0xB7, 0xE3, 0xB4,
+	0x0D, 0xA8, 0x15, 0x1E, 0x98, 0x02, 0xBD, 0x89,
+	0x89, 0xE3, 0x38, 0x2C, 0xAB, 0x50, 0x25, 0x30,
+	0xB4, 0x5E, 0xA5, 0xCD, 0xA8, 0x9B, 0xA4, 0x2A,
+	0xED, 0x19, 0x3B, 0xC3, 0x05, 0x07, 0x57, 0xB5,
+	0x52, 0x11, 0x74, 0x95, 0x51, 0x5A, 0xD8, 0xED,
+	0xDF, 0x37, 0x91, 0x83, 0x27, 0xD5, 0x08, 0x82,
+	0xB3, 0x42, 0x08, 0xC7, 0x81, 0x35, 0x5F, 0x58,
+	0x28, 0x69, 0x0D, 0x97, 0x1D, 0x28, 0xE6, 0xB0,
+	0x58, 0x93, 0xCE, 0x2A, 0xB4, 0x7D, 0x4B, 0x83,
+	0x20, 0x1E, 0x08, 0xEF, 0x68, 0x51, 0xEB, 0xB4,
+	0xFA, 0x78, 0xB5, 0xE5, 0x2D, 0x93, 0x07, 0x99,
+	0xBB, 0xDD, 0x9A, 0x4E, 0xD6, 0xE7, 0x31, 0x9F,
+	0x4D, 0xB4, 0x05, 0x45, 0x89, 0x59, 0x42, 0xFF,
+	0x0C, 0xD4, 0xC2, 0xF3, 0xA0, 0xF7, 0xD3, 0xCE,
+	0x26, 0x2F, 0x9B, 0xCB, 0x98, 0x24, 0xDE, 0x50,
+	0xF6, 0x81, 0x24, 0xA3, 0xB4, 0x99, 0x94, 0xDB,
+	0xF5, 0xF4, 0x74, 0xAC, 0xED, 0xF1, 0xD6, 0x2F,
+	0x89, 0x48, 0x41, 0x24, 0xBB, 0xA7, 0x64, 0x48,
+	0xAE, 0x36, 0xE3, 0xF3, 0xA8, 0x37, 0x4F, 0xE6,
+	0x24, 0x92, 0xC4, 0x27, 0x86, 0x68, 0xC4, 0xA2,
+	0x2F, 0xB2, 0xA8, 0xFF, 0x27, 0x5F, 0x59, 0xB9,
+	0x86, 0x89, 0x5A, 0x09, 0xB8, 0xDA, 0xD6, 0xBA,
+	0x26, 0x86, 0x51, 0xC2, 0xDF, 0xCF, 0x07, 0x49,
+	0x1C, 0xB7, 0x1A, 0x80, 0xD4, 0x97, 0x13, 0xAA,
+	0xE8, 0x38, 0xA7, 0x2C, 0xB7, 0xE2, 0xE0, 0x33,
+	0xF9, 0xC3, 0x21, 0xF6, 0x2A, 0xAA, 0x1A, 0xF7,
+	0x39, 0x1F, 0x2F, 0x7E, 0x38, 0xC8, 0x82, 0x3F,
+	0xCC, 0xD8, 0xE2, 0xD9, 0x02, 0xE6, 0x8E, 0x31,
+	0xA7, 0x37, 0xE8, 0x59, 0x3F, 0xE1, 0x60, 0x1C,
+	0x74, 0x52, 0xE4, 0x49, 0xF7, 0x21, 0x60, 0x11
+};
+static const struct
+blockcipher_test_data aes_test_data_xts_wrapped_key_48_pt_512_du_512 = {
+	.crypto_algo = RTE_CRYPTO_CIPHER_AES_XTS,
+	.cipher_key = {
+		.data = {
+			0x94, 0xD8, 0xAE, 0xD7, 0x74, 0xBF, 0x51, 0xF1,
+			0x2D, 0x19, 0x0B, 0x9A, 0xB3, 0xDC, 0x1E, 0xA9,
+			0x5C, 0xC5, 0x47, 0xC1, 0xBB, 0x94, 0xDB, 0xFA,
+			0x2B, 0x1E, 0x54, 0xA5, 0x76, 0xAD, 0x07, 0x4B,
+			0x46, 0xE0, 0xAA, 0xF9, 0xF7, 0x5C, 0x64, 0x2D,
+			0x8F, 0x04, 0x17, 0x72, 0xE5, 0x8D, 0xFE, 0x9C
+		},
+		.len = 48
+	},
+	.iv = {
+		.data = {
+			0x4F, 0xAE, 0xF7, 0x11, 0x7C, 0xDA, 0x59, 0xC6,
+			0x6E, 0x4B, 0x92, 0x01, 0x3E, 0x76, 0x8A, 0xD5
+		},
+		.len = 16
+	},
+	.plaintext = {
+		.data = plaintext_aes256xts_512bytes,
+		.len = 512
+	},
+	.ciphertext = {
+		.data = ciphertext_aes256xts_512bytes,
+		.len = 512
+	},
+	.xts_dataunit_len = 512,
+	.wrapped_key = true
+};
+static const uint8_t plaintext_aes256xts_4096bytes[] = {
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79
+};
+static const uint8_t ciphertext_aes256xts_4096bytes[] = {
+	0xD3, 0x94, 0x3E, 0xC9, 0xD3, 0x43, 0x13, 0xD3,
+	0x07, 0x7E, 0x51, 0x11, 0x97, 0xA5, 0xB1, 0xB2,
+	0xB0, 0x55, 0xF8, 0xD6, 0xA2, 0x10, 0x78, 0x76,
+	0x37, 0x95, 0x4F, 0x25, 0x99, 0xCC, 0x12, 0xD6,
+	0xBB, 0x2A, 0x24, 0xF4, 0x2D, 0x82, 0xFF, 0x56,
+	0xEC, 0x55, 0x3B, 0xDD, 0xF9, 0xE1, 0xC8, 0x48,
+	0x68, 0x16, 0xD2, 0x2B, 0x95, 0x7C, 0x1F, 0xCB,
+	0x32, 0xD6, 0x8F, 0x9A, 0x2E, 0xF7, 0x9B, 0xBE,
+	0x72, 0xFC, 0x55, 0x33, 0x27, 0x21, 0x2F, 0x69,
+	0xCA, 0x76, 0xA5, 0x9B, 0x21, 0x2E, 0x40, 0x57,
+	0x2C, 0x1C, 0x98, 0x41, 0x9A, 0x5E, 0x55, 0x38,
+	0xDE, 0xC2, 0x09, 0x57, 0x32, 0xA3, 0x34, 0x40,
+	0xC1, 0x8D, 0xCF, 0x70, 0x15, 0xF2, 0x6F, 0x4F,
+	0x6A, 0x04, 0xA6, 0x6D, 0xFF, 0x53, 0x25, 0x6A,
+	0x0E, 0xD0, 0x87, 0x63, 0xA1, 0x6C, 0xB1, 0x99,
+	0x4A, 0x42, 0xF5, 0xF6, 0xEA, 0xA4, 0xEB, 0x6D,
+	0x70, 0x9B, 0x0F, 0x85, 0xE2, 0x43, 0x6C, 0x27,
+	0x54, 0x57, 0x52, 0x1C, 0xCF, 0x72, 0x11, 0x83,
+	0xC3, 0xF5, 0xC6, 0xB6, 0x07, 0xEC, 0x1A, 0xF5,
+	0xAC, 0xA1, 0xF2, 0x3A, 0x01, 0x53, 0x0C, 0xA4,
+	0x40, 0x19, 0xAF, 0x7B, 0x2D, 0xE7, 0x56, 0x8B,
+	0x71, 0x5F, 0x8D, 0x96, 0xF1, 0x5D, 0x33, 0xAC,
+	0xE7, 0xAD, 0x96, 0xDB, 0xBB, 0xF7, 0xF5, 0x3F,
+	0x0E, 0x02, 0x2E, 0x80, 0xE9, 0xA2, 0x30, 0xD2,
+	0x82, 0x65, 0xEC, 0x1A, 0xB2, 0xE8, 0x98, 0x23,
+	0x42, 0x79, 0x43, 0x85, 0x3B, 0xF9, 0xFB, 0xBB,
+	0xC5, 0x29, 0xAD, 0x95, 0xAE, 0x22, 0x5B, 0x26,
+	0xDF, 0x76, 0x65, 0x37, 0x17, 0xBC, 0x58, 0xBB,
+	0x1B, 0x0A, 0x71, 0xDC, 0x68, 0xFF, 0x90, 0x36,
+	0x37, 0xB7, 0x49, 0x5E, 0x49, 0x4E, 0xE4, 0x1E,
+	0x24, 0x39, 0x48, 0xC7, 0x68, 0x19, 0xED, 0x94,
+	0xB5, 0xF6, 0x85, 0xFA, 0xE6, 0xB8, 0x2B, 0x9E,
+	0x07, 0x9E, 0xFB, 0x1D, 0x61, 0x06, 0x47, 0x2A,
+	0x3E, 0x1E, 0xD8, 0x52, 0xFB, 0xB7, 0xE3, 0xB4,
+	0x0D, 0xA8, 0x15, 0x1E, 0x98, 0x02, 0xBD, 0x89,
+	0x89, 0xE3, 0x38, 0x2C, 0xAB, 0x50, 0x25, 0x30,
+	0xB4, 0x5E, 0xA5, 0xCD, 0xA8, 0x9B, 0xA4, 0x2A,
+	0xED, 0x19, 0x3B, 0xC3, 0x05, 0x07, 0x57, 0xB5,
+	0x52, 0x11, 0x74, 0x95, 0x51, 0x5A, 0xD8, 0xED,
+	0xDF, 0x37, 0x91, 0x83, 0x27, 0xD5, 0x08, 0x82,
+	0xB3, 0x42, 0x08, 0xC7, 0x81, 0x35, 0x5F, 0x58,
+	0x28, 0x69, 0x0D, 0x97, 0x1D, 0x28, 0xE6, 0xB0,
+	0x58, 0x93, 0xCE, 0x2A, 0xB4, 0x7D, 0x4B, 0x83,
+	0x20, 0x1E, 0x08, 0xEF, 0x68, 0x51, 0xEB, 0xB4,
+	0xFA, 0x78, 0xB5, 0xE5, 0x2D, 0x93, 0x07, 0x99,
+	0xBB, 0xDD, 0x9A, 0x4E, 0xD6, 0xE7, 0x31, 0x9F,
+	0x4D, 0xB4, 0x05, 0x45, 0x89, 0x59, 0x42, 0xFF,
+	0x0C, 0xD4, 0xC2, 0xF3, 0xA0, 0xF7, 0xD3, 0xCE,
+	0x26, 0x2F, 0x9B, 0xCB, 0x98, 0x24, 0xDE, 0x50,
+	0xF6, 0x81, 0x24, 0xA3, 0xB4, 0x99, 0x94, 0xDB,
+	0xF5, 0xF4, 0x74, 0xAC, 0xED, 0xF1, 0xD6, 0x2F,
+	0x89, 0x48, 0x41, 0x24, 0xBB, 0xA7, 0x64, 0x48,
+	0xAE, 0x36, 0xE3, 0xF3, 0xA8, 0x37, 0x4F, 0xE6,
+	0x24, 0x92, 0xC4, 0x27, 0x86, 0x68, 0xC4, 0xA2,
+	0x2F, 0xB2, 0xA8, 0xFF, 0x27, 0x5F, 0x59, 0xB9,
+	0x86, 0x89, 0x5A, 0x09, 0xB8, 0xDA, 0xD6, 0xBA,
+	0x26, 0x86, 0x51, 0xC2, 0xDF, 0xCF, 0x07, 0x49,
+	0x1C, 0xB7, 0x1A, 0x80, 0xD4, 0x97, 0x13, 0xAA,
+	0xE8, 0x38, 0xA7, 0x2C, 0xB7, 0xE2, 0xE0, 0x33,
+	0xF9, 0xC3, 0x21, 0xF6, 0x2A, 0xAA, 0x1A, 0xF7,
+	0x39, 0x1F, 0x2F, 0x7E, 0x38, 0xC8, 0x82, 0x3F,
+	0xCC, 0xD8, 0xE2, 0xD9, 0x02, 0xE6, 0x8E, 0x31,
+	0xA7, 0x37, 0xE8, 0x59, 0x3F, 0xE1, 0x60, 0x1C,
+	0x74, 0x52, 0xE4, 0x49, 0xF7, 0x21, 0x60, 0x11,
+	0x8F, 0xD0, 0xA1, 0x4B, 0xE3, 0xF1, 0x18, 0x3D,
+	0x90, 0x89, 0x54, 0x27, 0xA0, 0xF9, 0x32, 0x09,
+	0x3D, 0x9D, 0x9A, 0x09, 0x53, 0xC6, 0x7E, 0x95,
+	0x85, 0x53, 0x98, 0x4C, 0x23, 0xEA, 0x54, 0xBD,
+	0x6F, 0x50, 0xBC, 0x4C, 0xCF, 0x37, 0xC5, 0x7B,
+	0x4B, 0xCE, 0x84, 0xAF, 0xE2, 0xE2, 0x55, 0x49,
+	0xBC, 0xBF, 0x92, 0xCA, 0x1E, 0x5E, 0x10, 0xDF,
+	0x60, 0x87, 0x09, 0xA1, 0x4C, 0x1D, 0x7E, 0x1E,
+	0x59, 0xE9, 0xCF, 0xDA, 0x45, 0x3F, 0xE2, 0x0F,
+	0x53, 0x8D, 0x8B, 0x79, 0xBD, 0xD8, 0xB0, 0xE3,
+	0x5B, 0x7C, 0x55, 0x4A, 0x84, 0xF0, 0x1E, 0xF9,
+	0xE8, 0xF3, 0x09, 0x4D, 0x0B, 0xD7, 0x77, 0xCC,
+	0x3F, 0x70, 0x22, 0x7D, 0x17, 0x27, 0x48, 0x57,
+	0xE2, 0x36, 0xA0, 0x84, 0x3B, 0xDE, 0x05, 0x34,
+	0xEF, 0x55, 0x12, 0xF4, 0x9A, 0x99, 0x0D, 0x28,
+	0x86, 0x28, 0x99, 0x6B, 0x22, 0xEE, 0x63, 0xF0,
+	0x68, 0x9C, 0xE1, 0x70, 0xF6, 0x26, 0xD8, 0x3B,
+	0xF9, 0x57, 0x18, 0x3D, 0xAD, 0x66, 0xF0, 0xCF,
+	0x7B, 0x0C, 0x28, 0x4D, 0xB8, 0xEB, 0x7B, 0x04,
+	0x1E, 0x7D, 0x40, 0x5F, 0x5A, 0x1E, 0x7E, 0x08,
+	0x7F, 0x4C, 0x1E, 0x18, 0xE5, 0x3E, 0x6C, 0x90,
+	0x3C, 0x89, 0x13, 0x2A, 0xC4, 0x2A, 0x94, 0xB5,
+	0x3E, 0x18, 0x1C, 0x4C, 0xBA, 0xEA, 0x86, 0xD2,
+	0x05, 0xA9, 0x59, 0x9C, 0x80, 0xC2, 0x45, 0xAD,
+	0x30, 0x99, 0x18, 0x6A, 0x2F, 0x73, 0x8C, 0xF0,
+	0xFE, 0xA4, 0xBD, 0x44, 0x3E, 0xEB, 0x98, 0x75,
+	0x48, 0x08, 0x57, 0x45, 0xD8, 0x41, 0xDE, 0x61,
+	0x6D, 0x06, 0x93, 0xC4, 0x99, 0x1B, 0x23, 0xB5,
+	0x12, 0x22, 0x5C, 0xC7, 0x9E, 0x18, 0xEA, 0x64,
+	0xDB, 0xCE, 0x1A, 0xAC, 0x5D, 0x9B, 0x80, 0xE2,
+	0xBF, 0x3E, 0xC2, 0xA4, 0x78, 0x4F, 0xF1, 0xE3,
+	0x7D, 0x2A, 0x20, 0x94, 0x13, 0xCD, 0xF3, 0x1C,
+	0x33, 0x9C, 0xC1, 0x59, 0x85, 0x52, 0xCB, 0xDB,
+	0x03, 0xDF, 0x11, 0xE6, 0xAB, 0x95, 0x82, 0x65,
+	0x7A, 0x88, 0x73, 0xEE, 0xBA, 0x21, 0x1C, 0x2F,
+	0xCD, 0xD7, 0xC6, 0xE5, 0x13, 0xDE, 0x7A, 0x9E,
+	0xEE, 0x83, 0x8D, 0xC6, 0x47, 0x63, 0xE0, 0xC7,
+	0xC4, 0xBE, 0x19, 0x25, 0xEE, 0xCC, 0x0A, 0x13,
+	0x18, 0x9D, 0x34, 0x5B, 0x55, 0x6C, 0xC1, 0x6E,
+	0xBF, 0x5A, 0xC5, 0x61, 0x75, 0x77, 0x49, 0x8C,
+	0x67, 0x61, 0xE8, 0x72, 0x87, 0xE8, 0xCA, 0xBE,
+	0x6E, 0xC3, 0xD7, 0x81, 0x8C, 0x78, 0x79, 0xC8,
+	0x72, 0xDA, 0x1A, 0x40, 0x7D, 0x60, 0xE2, 0x5A,
+	0x47, 0x38, 0xA1, 0x21, 0x22, 0x6B, 0x54, 0x74,
+	0xDD, 0xF3, 0xBC, 0x96, 0x28, 0x7E, 0xC2, 0x8B,
+	0x13, 0xEE, 0x53, 0xBC, 0x34, 0x67, 0x07, 0x87,
+	0xD5, 0x6B, 0x93, 0x22, 0x21, 0xB9, 0xED, 0x17,
+	0xE4, 0xA1, 0x96, 0xB2, 0xC6, 0xFF, 0x79, 0xA0,
+	0xA7, 0xF9, 0xDD, 0x92, 0x78, 0xF7, 0xE3, 0x16,
+	0x79, 0xEF, 0xEF, 0x31, 0x4D, 0x1D, 0x75, 0xF9,
+	0xCF, 0x5A, 0x1B, 0x68, 0x16, 0x7F, 0xAF, 0x5F,
+	0x30, 0xB7, 0xEF, 0xF8, 0x94, 0x63, 0x73, 0x3D,
+	0xB3, 0x63, 0xE4, 0xE6, 0xD8, 0xAD, 0xF4, 0x80,
+	0x5E, 0x82, 0xA3, 0xFB, 0x3D, 0x0F, 0xCF, 0x59,
+	0xB8, 0x76, 0xAF, 0x27, 0x83, 0xE3, 0x2D, 0x6F,
+	0xE0, 0xF3, 0x11, 0xD5, 0xAE, 0x82, 0x14, 0x1D,
+	0x78, 0x95, 0xBF, 0x31, 0x22, 0x1B, 0x80, 0x12,
+	0x02, 0xD7, 0x4D, 0x1B, 0x92, 0xE3, 0x15, 0xBA,
+	0x67, 0xD6, 0x8F, 0xD4, 0xDA, 0xBF, 0xD4, 0x62,
+	0xAD, 0x76, 0xDA, 0x04, 0xA2, 0xEA, 0x98, 0xD3,
+	0xC3, 0x6E, 0x5F, 0x26, 0x3C, 0x5E, 0xD9, 0xEA,
+	0x09, 0xF0, 0x02, 0xFC, 0xD2, 0x11, 0xF8, 0xA8,
+	0x7E, 0xFF, 0x06, 0x28, 0x5B, 0xE5, 0x6F, 0x9A,
+	0x00, 0xE7, 0x7F, 0xB9, 0xFB, 0x59, 0xBB, 0xDD,
+	0x85, 0xF3, 0x40, 0xCE, 0xA3, 0x5E, 0x2E, 0x2E,
+	0x34, 0x91, 0x58, 0x41, 0x00, 0xB5, 0xE1, 0x88,
+	0x24, 0x51, 0xC7, 0xB6, 0xF3, 0x21, 0x52, 0x6E,
+	0xE7, 0xFC, 0x42, 0xE7, 0x9C, 0xCC, 0x1E, 0x51,
+	0x45, 0x39, 0xBE, 0x09, 0xFE, 0x1A, 0xC4, 0xF0,
+	0x79, 0xF4, 0x05, 0xC7, 0xA7, 0xF8, 0x0F, 0xB6,
+	0x5A, 0x7B, 0xD7, 0xE1, 0x6F, 0xF0, 0x9D, 0x67,
+	0xA3, 0xE3, 0x3E, 0x2E, 0xB9, 0x8C, 0x83, 0x9E,
+	0xFD, 0x2E, 0xA2, 0x92, 0x99, 0x3C, 0xC0, 0x99,
+	0x01, 0xAB, 0x0D, 0xFA, 0x55, 0x96, 0x04, 0x60,
+	0x1A, 0xAD, 0x4C, 0xBB, 0x3D, 0xBB, 0x7D, 0x8B,
+	0x9F, 0x28, 0x85, 0x7D, 0xB9, 0xE4, 0x05, 0x79,
+	0x7B, 0x63, 0xDD, 0x7F, 0x4D, 0xE7, 0x50, 0xD9,
+	0x41, 0xFF, 0x53, 0xB1, 0xCE, 0x42, 0x7B, 0xD6,
+	0x05, 0x1B, 0x4E, 0xAF, 0xC4, 0x8C, 0x17, 0xC8,
+	0x52, 0xBD, 0x03, 0x3B, 0x92, 0x57, 0x4E, 0xA8,
+	0x15, 0xC3, 0x26, 0x1C, 0x55, 0xC1, 0xFF, 0xAE,
+	0xA9, 0x26, 0x2D, 0xA7, 0x8E, 0x3A, 0x7F, 0xA3,
+	0x48, 0xA5, 0xBC, 0x14, 0x84, 0xF2, 0x90, 0xCE,
+	0x35, 0x0F, 0x64, 0x6B, 0xD8, 0x1C, 0x12, 0xFE,
+	0x5A, 0x4F, 0x0E, 0xCE, 0x81, 0x4E, 0x79, 0x6B,
+	0xCF, 0x56, 0xA7, 0xDB, 0x24, 0xBC, 0xB0, 0x84,
+	0x4C, 0xB0, 0xDA, 0xBE, 0xE6, 0x8F, 0xD7, 0x8E,
+	0x0E, 0xA0, 0xD3, 0x55, 0xC2, 0x4A, 0x34, 0x1C,
+	0xF9, 0xC7, 0x3D, 0x29, 0x70, 0x8B, 0xF0, 0x99,
+	0x61, 0xF5, 0x11, 0xFB, 0x82, 0xE2, 0x67, 0x35,
+	0x60, 0x78, 0x47, 0x81, 0x2A, 0x74, 0x5E, 0x4D,
+	0x48, 0xD3, 0x7C, 0x32, 0xCA, 0x1B, 0xD2, 0xA2,
+	0x5C, 0x3A, 0x2F, 0xCE, 0xB4, 0x6C, 0x3A, 0x6A,
+	0x8F, 0x67, 0x46, 0x12, 0xE7, 0xAE, 0x6A, 0x3B,
+	0x99, 0x04, 0x5E, 0x96, 0xD0, 0xB9, 0x84, 0xF6,
+	0xA7, 0x64, 0x11, 0xE8, 0x0C, 0x51, 0xFD, 0x3F,
+	0x18, 0xFA, 0xE8, 0x52, 0xD9, 0x4B, 0x99, 0x7A,
+	0x25, 0x2B, 0x1B, 0x21, 0xAD, 0x8C, 0xFE, 0x0D,
+	0x34, 0x51, 0x91, 0x75, 0x55, 0x6F, 0xEB, 0x9F,
+	0x42, 0xDC, 0x73, 0x7D, 0x31, 0x0A, 0x74, 0x13,
+	0x80, 0xB8, 0xC3, 0xED, 0x73, 0x9D, 0x79, 0x42,
+	0xC0, 0x33, 0xAB, 0xC1, 0xCB, 0xB9, 0xD0, 0xBE,
+	0xA0, 0x78, 0xB8, 0x3B, 0xEB, 0x3D, 0x1A, 0x3F,
+	0xFB, 0x9B, 0xAA, 0x8F, 0x89, 0xF9, 0xD1, 0x22,
+	0x82, 0xE6, 0x66, 0xEE, 0x2A, 0xFD, 0x9F, 0xF8,
+	0x92, 0x7E, 0x10, 0xF5, 0xD5, 0x23, 0x0B, 0xB1,
+	0xD6, 0xF1, 0x7A, 0x3D, 0x73, 0xE9, 0xCE, 0x7F,
+	0xE6, 0x0B, 0x17, 0xBC, 0x23, 0xAE, 0x72, 0xB6,
+	0xFA, 0x19, 0x46, 0xBB, 0xFE, 0xA4, 0xC1, 0x64,
+	0xA8, 0x5E, 0xE8, 0xBB, 0x63, 0x58, 0x19, 0x50,
+	0xAA, 0x36, 0xC2, 0x4B, 0x38, 0x24, 0xD1, 0x2E,
+	0xAE, 0xAD, 0x6E, 0x34, 0x64, 0xA8, 0xC8, 0xF2,
+	0x4E, 0x74, 0x5C, 0x98, 0xE8, 0xDF, 0x99, 0x8C,
+	0x41, 0x79, 0x60, 0x2D, 0xD5, 0xF4, 0xE3, 0xE9,
+	0x1D, 0xF6, 0x5A, 0xA9, 0x69, 0x8E, 0xA1, 0x4F,
+	0xD3, 0x1B, 0x09, 0xA8, 0x7A, 0xD1, 0xE1, 0xCF,
+	0xAC, 0xBA, 0xD0, 0xD1, 0x34, 0x34, 0x8F, 0xC1,
+	0xD1, 0xA8, 0xAF, 0x6E, 0x92, 0xE0, 0xB0, 0xF6,
+	0xF9, 0x08, 0xA0, 0xCB, 0x58, 0x34, 0xF6, 0x68,
+	0xA2, 0xBF, 0x05, 0x39, 0x63, 0xBA, 0x4F, 0xEF,
+	0xE3, 0x95, 0x69, 0xD5, 0x89, 0x7C, 0x64, 0x07,
+	0x13, 0x42, 0x14, 0xF1, 0xA6, 0x9B, 0x87, 0xE5,
+	0xF4, 0x49, 0xAE, 0x67, 0x65, 0xCC, 0xF2, 0x26,
+	0xF8, 0x31, 0xBD, 0x33, 0x6A, 0x87, 0x77, 0x4E,
+	0xB1, 0xEE, 0xA4, 0xA2, 0xC8, 0xA0, 0x4A, 0xC1,
+	0xDF, 0x55, 0xE0, 0xDE, 0x53, 0x15, 0x3B, 0xEC,
+	0x55, 0x32, 0xCA, 0x06, 0xE4, 0x78, 0x59, 0x63,
+	0x10, 0x68, 0xA9, 0x46, 0x1B, 0xEF, 0x73, 0x6D,
+	0x1A, 0x02, 0x64, 0x12, 0x76, 0x9B, 0xDB, 0x7C,
+	0x03, 0x35, 0x19, 0xE1, 0x58, 0x7A, 0x87, 0x0C,
+	0x76, 0xDC, 0xFC, 0xC0, 0x28, 0xE4, 0xA2, 0x07,
+	0x9C, 0x28, 0x05, 0x21, 0x13, 0x58, 0xEF, 0x05,
+	0xBB, 0xAB, 0x94, 0xA2, 0x93, 0xBC, 0x31, 0x61,
+	0x26, 0x39, 0x38, 0x0C, 0xC4, 0x67, 0xDA, 0xA5,
+	0xE4, 0x1E, 0x1B, 0xB6, 0xE5, 0x73, 0xD6, 0x6C,
+	0xEE, 0xBC, 0x9D, 0xB9, 0xE7, 0xD9, 0x45, 0x2F,
+	0xF2, 0xB6, 0x92, 0x54, 0x41, 0x05, 0xB7, 0xB7,
+	0xFC, 0x37, 0x63, 0x6A, 0xB4, 0xBE, 0xB8, 0x3E,
+	0xD8, 0x53, 0x3B, 0xF8, 0x7D, 0x9A, 0x05, 0xDF,
+	0x20, 0x02, 0x27, 0x64, 0x38, 0xFA, 0x7D, 0xAF,
+	0x7F, 0xFA, 0xD1, 0xB7, 0x32, 0xC5, 0x74, 0x3E,
+	0x04, 0xA2, 0x67, 0x79, 0x02, 0x2E, 0x6F, 0xA1,
+	0x27, 0x87, 0x07, 0xB5, 0x9F, 0x0A, 0x7D, 0x5E,
+	0x14, 0xA0, 0x31, 0x46, 0x3F, 0xA9, 0xDE, 0x98,
+	0xB9, 0x89, 0xA0, 0x4A, 0x7A, 0xBD, 0x15, 0xAE,
+	0x2D, 0x0B, 0x38, 0x9A, 0xD8, 0x0E, 0xD2, 0xBA,
+	0x6D, 0xA1, 0x04, 0x1E, 0x4E, 0x39, 0x87, 0x4B,
+	0xC8, 0x3C, 0x74, 0x35, 0x4D, 0xC8, 0x1B, 0x42,
+	0x06, 0x5B, 0x73, 0xB7, 0x33, 0x86, 0x4A, 0x10,
+	0x2A, 0x10, 0x16, 0x28, 0x6F, 0x2A, 0xE3, 0x86,
+	0xDE, 0xA3, 0x44, 0x23, 0xE2, 0x90, 0xC4, 0x20,
+	0x90, 0xE0, 0xB8, 0xE6, 0xA7, 0xB6, 0xD6, 0x92,
+	0xF4, 0xF8, 0x8A, 0xBC, 0xAC, 0x31, 0x47, 0x8F,
+	0xAA, 0xE0, 0xD9, 0xF7, 0xE3, 0xCB, 0x11, 0xA4,
+	0x6B, 0x05, 0xB3, 0xB8, 0x72, 0x69, 0xE6, 0xDD,
+	0x75, 0x0F, 0x20, 0x1D, 0x3F, 0xC6, 0x96, 0xA0,
+	0x18, 0xB6, 0x24, 0xA1, 0xA6, 0xFD, 0x0C, 0x80,
+	0x1E, 0xD2, 0x28, 0xA2, 0x1A, 0x27, 0xF4, 0x23,
+	0x59, 0x1A, 0xCC, 0x0F, 0xD4, 0x99, 0xD0, 0xB4,
+	0x1E, 0x91, 0xC7, 0xD8, 0x8F, 0x8C, 0x5B, 0xEB,
+	0xB5, 0x9F, 0xFF, 0x4F, 0xD0, 0xD5, 0xB7, 0x60,
+	0xCC, 0x0A, 0x10, 0x38, 0xBF, 0xA8, 0x2E, 0xCC,
+	0xEB, 0x26, 0xB0, 0x78, 0xB3, 0xE0, 0x40, 0xAF,
+	0xCD, 0x12, 0xC5, 0x3A, 0x24, 0xD8, 0xEE, 0x3A,
+	0x64, 0x83, 0x2E, 0xD9, 0x25, 0x21, 0x66, 0xA5,
+	0x28, 0xD1, 0xE1, 0x84, 0x25, 0x1B, 0x20, 0xB8,
+	0xF5, 0x76, 0xB6, 0x3E, 0x4B, 0xC6, 0xEC, 0xC1,
+	0xC7, 0xAC, 0xC4, 0xAD, 0xCE, 0xF0, 0xB4, 0x0F,
+	0x35, 0x1E, 0xCE, 0x4E, 0xE3, 0x57, 0x30, 0xFC,
+	0xF4, 0x9B, 0x86, 0xB0, 0xDD, 0x3F, 0x2F, 0xB6,
+	0x10, 0x20, 0xE4, 0x24, 0x17, 0x1C, 0x24, 0xC6,
+	0x89, 0xE4, 0x14, 0xAD, 0x2E, 0x41, 0x08, 0x33,
+	0x88, 0xB1, 0x6F, 0x11, 0x85, 0xAF, 0x58, 0x17,
+	0xE3, 0x91, 0xB4, 0x72, 0xA2, 0x7F, 0xA3, 0x98,
+	0xAF, 0xB7, 0x6B, 0x58, 0x76, 0xA3, 0x11, 0x1C,
+	0x8A, 0x1A, 0xE6, 0x58, 0x54, 0xB0, 0xB9, 0x6E,
+	0x46, 0xCB, 0x16, 0xC0, 0x63, 0x0C, 0xEE, 0xA2,
+	0xAE, 0xF6, 0x71, 0xEF, 0xD1, 0xB9, 0x3D, 0xB7,
+	0x76, 0xCE, 0x5B, 0x84, 0x66, 0x7C, 0x7D, 0xF1,
+	0x96, 0x60, 0x34, 0xF6, 0xD1, 0x64, 0x27, 0xD9,
+	0xF3, 0x78, 0x8B, 0xF4, 0xC3, 0x1D, 0x37, 0xC0,
+	0xF4, 0x4A, 0xD0, 0xA5, 0x9A, 0xEB, 0xDD, 0x79,
+	0x54, 0x5D, 0xEB, 0x04, 0xC1, 0xA4, 0xBC, 0xED,
+	0xE3, 0x74, 0xC3, 0xB9, 0x9A, 0x6A, 0xAA, 0x06,
+	0xD1, 0xF0, 0x0F, 0xC5, 0xEF, 0x7E, 0x0B, 0xC8,
+	0xF4, 0x94, 0x4E, 0x69, 0x0E, 0x36, 0x00, 0x13,
+	0x45, 0xCE, 0x68, 0x13, 0xFE, 0x7F, 0x29, 0xA2,
+	0x1D, 0x79, 0xDF, 0xF2, 0x27, 0xFB, 0xAE, 0x52,
+	0x05, 0x78, 0xD7, 0xB9, 0xF7, 0x38, 0x68, 0xD5,
+	0xBA, 0xD7, 0xCF, 0x09, 0xC6, 0xD2, 0x5B, 0xC6,
+	0x98, 0xE4, 0xEC, 0xD5, 0xE9, 0xC2, 0xA5, 0x1A,
+	0x52, 0xC8, 0xA7, 0xBA, 0x3D, 0x74, 0x75, 0x00,
+	0xAA, 0xDD, 0x6A, 0x3F, 0xB6, 0x2F, 0x08, 0xB7,
+	0x1C, 0x6B, 0x52, 0x0C, 0xC9, 0xE4, 0x4D, 0xF4,
+	0xC5, 0x26, 0x1F, 0x35, 0x41, 0x25, 0x68, 0x17,
+	0xA8, 0x81, 0x75, 0xF4, 0x66, 0x41, 0xB5, 0xE4,
+	0x1D, 0x92, 0xEE, 0xDA, 0x0F, 0x56, 0x76, 0xC6,
+	0xAA, 0x0F, 0xA8, 0x63, 0x8D, 0xF0, 0x69, 0x63,
+	0x93, 0x45, 0xBC, 0x76, 0x40, 0xBE, 0xA9, 0x96,
+	0x36, 0xAF, 0x2F, 0x6B, 0x3E, 0xAB, 0xF3, 0xC0,
+	0xD7, 0xD5, 0xB1, 0x23, 0x23, 0xA2, 0xA0, 0xC4,
+	0xC5, 0x70, 0xEF, 0x66, 0x79, 0x15, 0xF0, 0xD0,
+	0x40, 0x0A, 0x33, 0x0C, 0xF3, 0x32, 0x6D, 0x8D,
+	0xB4, 0x44, 0x46, 0x78, 0x3F, 0x8D, 0x75, 0x40,
+	0xA5, 0x60, 0xBC, 0x9B, 0x76, 0xDF, 0x25, 0xF4,
+	0xE9, 0xED, 0xAC, 0x74, 0x2F, 0x9A, 0x00, 0xC4,
+	0x2B, 0x52, 0x26, 0x79, 0x09, 0x19, 0x57, 0x89,
+	0x60, 0x14, 0xBE, 0x65, 0xBD, 0x7B, 0x4D, 0x7D,
+	0x9B, 0x8B, 0x9E, 0x72, 0x6C, 0x0C, 0x57, 0xC7,
+	0x00, 0x08, 0x38, 0x7C, 0x37, 0x45, 0x9D, 0x55,
+	0xA2, 0x62, 0x5E, 0x34, 0x19, 0x99, 0x31, 0x16,
+	0xF1, 0x14, 0x44, 0x2D, 0xE3, 0x7E, 0x22, 0xE1,
+	0xA2, 0xB8, 0x9A, 0x9F, 0xE0, 0x37, 0x29, 0xBB,
+	0xCD, 0x46, 0xEE, 0x0A, 0x62, 0x2B, 0x98, 0x34,
+	0xBA, 0x9E, 0x54, 0x1B, 0xB1, 0x5C, 0x4F, 0xE9,
+	0xAA, 0xE4, 0x95, 0x8C, 0xA4, 0xEF, 0xC2, 0xB1,
+	0x7F, 0xF9, 0x80, 0xDA, 0x55, 0x95, 0x92, 0xC0,
+	0x86, 0xF4, 0x2D, 0x99, 0x3E, 0x17, 0xDC, 0x55,
+	0xA6, 0x33, 0x85, 0x90, 0x31, 0xC8, 0xFF, 0x58,
+	0x83, 0xC5, 0xBA, 0x60, 0x20, 0x5F, 0x87, 0x29,
+	0x20, 0x5A, 0x7D, 0x44, 0x2B, 0xA0, 0xE2, 0x99,
+	0xC8, 0x70, 0xBE, 0x89, 0xC5, 0xBC, 0xF6, 0x0D,
+	0x04, 0xC0, 0x96, 0xD1, 0x5C, 0xD1, 0x90, 0x43,
+	0xD3, 0x7B, 0x73, 0x52, 0x30, 0xB6, 0xA9, 0x7C,
+	0x0A, 0xA3, 0x24, 0x0E, 0x80, 0xFE, 0xBE, 0x31,
+	0xFD, 0xB5, 0x96, 0x04, 0x2B, 0xCF, 0x0B, 0x28,
+	0x1F, 0x7A, 0xCF, 0xC4, 0x82, 0x78, 0x52, 0x30,
+	0xB1, 0x34, 0x12, 0x50, 0x03, 0x09, 0x1C, 0x8B,
+	0x80, 0x60, 0xE3, 0xA1, 0xE5, 0x61, 0xF7, 0xD7,
+	0xB6, 0x74, 0xBE, 0xD6, 0x58, 0x03, 0xD3, 0xE5,
+	0xF7, 0xAC, 0x07, 0x60, 0xB7, 0x8A, 0xEC, 0xFA,
+	0xC6, 0x0F, 0xF0, 0x20, 0x04, 0x6B, 0x8F, 0x61,
+	0x09, 0x92, 0x03, 0xFB, 0x85, 0x99, 0x94, 0x9D,
+	0x2E, 0x6A, 0xC2, 0x9F, 0x20, 0x46, 0x2A, 0x96,
+	0xED, 0x42, 0x7D, 0x64, 0xA9, 0xE4, 0x1B, 0xDE,
+	0x11, 0x20, 0x12, 0x93, 0xE6, 0x2B, 0xE5, 0x93,
+	0x48, 0x37, 0x8C, 0x5A, 0x54, 0x0D, 0xEB, 0xF0,
+	0x9F, 0x9D, 0xE4, 0xA5, 0xC4, 0x93, 0x6F, 0x6A,
+	0xE3, 0x99, 0x69, 0xD9, 0xFE, 0x0C, 0x4E, 0xEC,
+	0x8B, 0x30, 0x1F, 0x7A, 0xB8, 0xC8, 0x5B, 0x61,
+	0x8E, 0xC2, 0x10, 0x90, 0x57, 0xB4, 0x72, 0x58,
+	0x7F, 0x41, 0x29, 0x7E, 0xF9, 0xBE, 0x40, 0xC3,
+	0x6F, 0xA9, 0xE3, 0x00, 0xE9, 0xC8, 0xFD, 0x4B,
+	0xFD, 0x3F, 0xE3, 0x3F, 0x25, 0x22, 0xFD, 0xB7,
+	0x2D, 0x57, 0xEF, 0x91, 0x08, 0xF0, 0x20, 0x56,
+	0x30, 0xFA, 0x83, 0x69, 0xFD, 0x56, 0x5A, 0x9B,
+	0xCE, 0xF8, 0x28, 0x02, 0xB4, 0x91, 0x35, 0x75,
+	0x9E, 0x63, 0x99, 0x48, 0xCF, 0x35, 0xF5, 0x58,
+	0x0C, 0x48, 0x8F, 0x0A, 0x2D, 0x9A, 0xE6, 0x40,
+	0xF6, 0x21, 0xB5, 0x69, 0xC1, 0x09, 0x31, 0x00,
+	0xA3, 0xC1, 0x4C, 0x99, 0x70, 0x4F, 0x5A, 0x63,
+	0x17, 0x90, 0xB8, 0xF8, 0x3A, 0x0E, 0xFD, 0x67,
+	0xEA, 0x0E, 0xBA, 0x7B, 0x1E, 0xEF, 0x37, 0x84,
+	0xD5, 0x51, 0x37, 0x01, 0xD6, 0x93, 0x15, 0xDF,
+	0x56, 0x89, 0x0E, 0x54, 0xF5, 0x1D, 0xF7, 0xE5,
+	0xB7, 0xC1, 0xF1, 0xC2, 0xD9, 0x14, 0x6F, 0x40,
+	0x55, 0x67, 0x50, 0x7C, 0x58, 0x35, 0x8B, 0x39,
+	0xCB, 0xB5, 0x87, 0xF5, 0x55, 0x5E, 0x26, 0x8C,
+	0x5B, 0x73, 0x0E, 0xBB, 0x25, 0x51, 0x0E, 0xAD,
+	0x57, 0x72, 0x7B, 0x68, 0x83, 0x11, 0x1E, 0x3A,
+	0x3D, 0xA4, 0x7C, 0x18, 0xB7, 0x70, 0x18, 0xBC,
+	0x72, 0x03, 0x4A, 0xA1, 0xD2, 0xF9, 0xA9, 0x8A,
+	0x25, 0x45, 0x19, 0xEE, 0x93, 0x06, 0xB5, 0x09,
+	0x71, 0xC9, 0x2D, 0xFD, 0x2B, 0xF3, 0xC7, 0x64,
+	0x5F, 0xCE, 0x71, 0x1D, 0x81, 0x96, 0x67, 0xBF,
+	0x01, 0x39, 0x8C, 0xE7, 0xA2, 0xD0, 0x98, 0x57,
+	0x5A, 0xFD, 0x21, 0xC7, 0x46, 0xAA, 0xB4, 0xE4,
+	0x0E, 0xBE, 0xC6, 0x68, 0x3E, 0x38, 0xF5, 0xA2,
+	0xED, 0x73, 0xCC, 0x53, 0x7E, 0x7E, 0x03, 0x32,
+	0xDC, 0xB6, 0xC1, 0x03, 0x9E, 0xB3, 0x2A, 0xAD,
+	0xC0, 0xC3, 0x6E, 0x47, 0xFB, 0x1E, 0xB7, 0x0D,
+	0x86, 0x95, 0x09, 0xA6, 0x9D, 0x6F, 0x92, 0xFC,
+	0xFF, 0x2C, 0x7D, 0x09, 0x16, 0x68, 0x50, 0x3E,
+	0x4F, 0x23, 0x4C, 0x93, 0x95, 0x2A, 0xE1, 0x9B,
+	0x16, 0xF0, 0x0F, 0xFF, 0x79, 0xA8, 0x06, 0xF9,
+	0x70, 0x61, 0x72, 0x2C, 0xE8, 0x91, 0x01, 0x6D,
+	0x45, 0xE5, 0x82, 0x5D, 0x26, 0x21, 0xAD, 0x3D,
+	0x77, 0x73, 0x23, 0x04, 0x84, 0x27, 0xA3, 0x5D,
+	0x6D, 0xA8, 0x99, 0xC1, 0xCE, 0x4F, 0xA9, 0xF7,
+	0xAB, 0x5C, 0xDE, 0x01, 0xE6, 0x1E, 0xEF, 0xE6,
+	0xFD, 0xE0, 0x68, 0x85, 0x3E, 0xEE, 0xBF, 0xF1,
+	0x0D, 0x79, 0xF4, 0xA2, 0xB4, 0x14, 0xBC, 0x0C,
+	0x49, 0x77, 0x03, 0x71, 0x08, 0x3E, 0x40, 0xA6,
+	0xD7, 0x03, 0xFA, 0xE2, 0xFB, 0xC7, 0x59, 0x30,
+	0x6E, 0x07, 0x06, 0x1C, 0x7C, 0x47, 0xE5, 0x4C,
+	0x57, 0x0A, 0x91, 0x4A, 0x43, 0xE4, 0x8A, 0xCD,
+	0x6E, 0x92, 0x01, 0xE2, 0x52, 0xC1, 0x92, 0x34,
+	0x8E, 0x64, 0x0F, 0x39, 0x63, 0x53, 0xAB, 0xE5,
+	0x44, 0xD5, 0xAA, 0xAA, 0xF6, 0x03, 0x89, 0xB9,
+	0xDD, 0xB2, 0x2D, 0x56, 0x1A, 0xE0, 0x72, 0x5A,
+	0x52, 0x19, 0x46, 0xEA, 0xB3, 0xCE, 0xB3, 0x59,
+	0x46, 0x7A, 0xA7, 0x48, 0x37, 0x0C, 0x09, 0xBA,
+	0x92, 0x70, 0x17, 0x7F, 0xF5, 0xD3, 0x60, 0x44,
+	0xC4, 0xC6, 0xC6, 0x7D, 0xD2, 0x21, 0xAC, 0x3F,
+	0x62, 0x6C, 0xE9, 0xBA, 0x4C, 0xF3, 0x82, 0x7E,
+	0x6D, 0x3A, 0x92, 0xDC, 0x94, 0xE4, 0x5F, 0xA6,
+	0x8B, 0x66, 0xA0, 0xDD, 0xE2, 0x97, 0x83, 0xED,
+	0xF5, 0x9D, 0xDF, 0x74, 0x77, 0x23, 0x7D, 0xDA,
+	0xC4, 0xFB, 0x92, 0x1A, 0xD9, 0x37, 0x36, 0xD2,
+	0x88, 0xC9, 0xEA, 0x0F, 0x98, 0xBD, 0xC5, 0xF8,
+	0xAA, 0x19, 0x75, 0x12, 0x6A, 0x41, 0xB5, 0xB3,
+	0xB5, 0xA4, 0x96, 0xDC, 0x2B, 0x49, 0x86, 0x66,
+	0x35, 0xD8, 0x4A, 0x62, 0xB4, 0xCB, 0x1E, 0x27,
+	0xC1, 0xAD, 0x34, 0x0E, 0x26, 0x16, 0xF2, 0xC2,
+	0x22, 0x52, 0x84, 0xD8, 0xD1, 0x32, 0xB8, 0x9C,
+	0xFE, 0x64, 0x42, 0x9F, 0xE4, 0x69, 0xF0, 0xAE,
+	0x3B, 0xD9, 0x2C, 0xA0, 0x14, 0xEB, 0x69, 0x74,
+	0x7C, 0xE2, 0xA6, 0x60, 0xE1, 0x52, 0x1C, 0xCC,
+	0xBF, 0xE6, 0xA1, 0x83, 0x20, 0x5D, 0x9E, 0xA3,
+	0xFB, 0x84, 0x8B, 0x33, 0xE6, 0xC9, 0x32, 0x83,
+	0xC0, 0x3F, 0x98, 0x1D, 0x6E, 0xC0, 0x50, 0x71,
+	0x29, 0x60, 0x5F, 0x36, 0xB4, 0x68, 0x1D, 0xB9,
+	0x76, 0x73, 0xC3, 0x80, 0xC5, 0xBC, 0x59, 0x7B,
+	0x59, 0xB4, 0xE0, 0x6A, 0x80, 0xCD, 0x4D, 0x8C,
+	0x9E, 0xE0, 0x0B, 0x45, 0x7D, 0x54, 0xD4, 0xC4,
+	0x97, 0x6C, 0x54, 0xEF, 0x14, 0x64, 0xBD, 0x3B,
+	0xD7, 0xEE, 0xF4, 0xD1, 0x41, 0x76, 0x3A, 0x24,
+	0x7A, 0xC2, 0xCA, 0x68, 0x28, 0x53, 0x46, 0xF7,
+	0x1B, 0xDA, 0x4B, 0x7A, 0x56, 0x75, 0x86, 0xFB,
+	0x31, 0x2C, 0x27, 0xF9, 0x4D, 0x35, 0xA4, 0x82,
+	0xE7, 0x2F, 0x41, 0xB4, 0xCA, 0xCE, 0x75, 0x94,
+	0x08, 0x54, 0xE2, 0x9E, 0x99, 0xC9, 0x85, 0xDE,
+	0x6F, 0x80, 0x95, 0x59, 0x3E, 0x54, 0x9F, 0x31,
+	0xF8, 0xDE, 0xD0, 0xD7, 0xA6, 0xD4, 0xD3, 0xBB,
+	0xD9, 0xC7, 0x55, 0xDD, 0xAE, 0xAD, 0x9E, 0x57,
+	0x4A, 0x33, 0x5D, 0x7A, 0xA6, 0xA3, 0xCA, 0xF9,
+	0x4C, 0x5B, 0x51, 0xCC, 0x22, 0xBB, 0x76, 0x44,
+	0x17, 0xDE, 0x22, 0xA1, 0xDF, 0x80, 0x13, 0x7D,
+	0xE5, 0x34, 0x7E, 0x75, 0x73, 0x10, 0x40, 0xFB,
+	0x9A, 0x21, 0xCD, 0xD3, 0xD3, 0x84, 0xB6, 0x0C,
+	0x31, 0x1E, 0xB5, 0x42, 0xF4, 0x34, 0x11, 0x7F,
+	0x4A, 0x23, 0xA8, 0xA5, 0x8F, 0x20, 0xCD, 0xA9,
+	0xF2, 0xE4, 0xEE, 0xFA, 0x57, 0xD1, 0x22, 0x1C,
+	0xA5, 0xDC, 0x0B, 0x25, 0xFE, 0xC2, 0xA7, 0x7E,
+	0x09, 0x2E, 0xDA, 0x40, 0x9F, 0x6C, 0xC8, 0x71,
+	0x58, 0x91, 0x04, 0x25, 0xD0, 0x06, 0xEA, 0x1B,
+	0xCD, 0x9D, 0x50, 0xD8, 0x40, 0x24, 0xAC, 0xC3,
+	0xB4, 0x07, 0x6E, 0x76, 0xF4, 0x4C, 0xD8, 0x80,
+	0xD0, 0x20, 0xF5, 0x15, 0x6A, 0x0A, 0x12, 0xF8,
+	0x6B, 0x67, 0x77, 0x34, 0xAE, 0x60, 0x68, 0x13,
+	0x5B, 0x8E, 0xFF, 0x5E, 0x7A, 0x77, 0x67, 0x0D,
+	0xE6, 0x96, 0x43, 0x9F, 0x8F, 0x77, 0x5F, 0x97,
+	0x23, 0x91, 0x33, 0x72, 0xC7, 0x8A, 0xC7, 0x80,
+	0xCF, 0xE7, 0x71, 0x06, 0x25, 0xB7, 0x4B, 0x89,
+	0x6A, 0x46, 0x67, 0x19, 0x49, 0x44, 0x03, 0x52,
+	0x32, 0xB1, 0x8F, 0xE7, 0x9E, 0xDA, 0x03, 0x41,
+	0xA3, 0xAC, 0xE5, 0xF3, 0x96, 0xE6, 0xAC, 0xFF,
+	0xEC, 0x35, 0x4D, 0x83, 0xA9, 0xCE, 0x7C, 0x52,
+	0xF2, 0x36, 0x97, 0xF0, 0x28, 0x36, 0x54, 0x59,
+	0x96, 0xEA, 0xEE, 0xB2, 0xC1, 0xAB, 0xA4, 0x96,
+	0x62, 0xD4, 0x3C, 0xF0, 0x1F, 0x2D, 0x65, 0x0E,
+	0x46, 0x7E, 0x12, 0x31, 0x8F, 0xA7, 0x8D, 0x7A,
+	0x4A, 0x41, 0x15, 0x57, 0x90, 0xF6, 0xF1, 0xE8,
+	0xE8, 0xE3, 0x57, 0x7B, 0x55, 0x85, 0x95, 0x97,
+	0xB3, 0x29, 0x3D, 0x02, 0x73, 0x1E, 0x87, 0x1F,
+	0x01, 0x89, 0x06, 0x88, 0x9E, 0x8A, 0x2E, 0xE0,
+	0x99, 0xFC, 0xF0, 0x48, 0x60, 0x37, 0x65, 0x25,
+	0xDB, 0x89, 0xC9, 0x7A, 0x51, 0x7E, 0x35, 0x92,
+	0x00, 0xC9, 0x61, 0x3F, 0x88, 0xE3, 0x20, 0x01,
+	0x46, 0x5A, 0x2C, 0x37, 0x02, 0xC9, 0x15, 0x0F,
+	0xB2, 0xEB, 0xC3, 0x55, 0x18, 0xF0, 0x15, 0x1A,
+	0x08, 0x8E, 0xB8, 0x9D, 0x18, 0xE4, 0x9F, 0x34,
+	0x10, 0x67, 0x68, 0x57, 0x60, 0x61, 0xEC, 0xD5,
+	0xD9, 0xA8, 0x3A, 0xAB, 0xD6, 0xD2, 0x7A, 0x47,
+	0x3D, 0xA4, 0x08, 0x7C, 0x3E, 0x4D, 0x76, 0x08,
+	0x19, 0x22, 0xB2, 0x89, 0x57, 0x84, 0xC2, 0x98,
+	0x72, 0xB8, 0x8B, 0xE0, 0x85, 0xA1, 0x3A, 0xC2,
+	0xA0, 0x06, 0x43, 0x03, 0xCF, 0x4F, 0x27, 0x80,
+	0x48, 0x9A, 0xBC, 0xB3, 0x3C, 0xC4, 0x5E, 0xAC,
+	0x8B, 0x85, 0x6F, 0x21, 0xD6, 0xFE, 0x12, 0x90,
+	0x53, 0x2F, 0x65, 0x32, 0x8D, 0x03, 0xFE, 0xFE,
+	0x61, 0x04, 0x47, 0x24, 0x6A, 0xB5, 0x01, 0x98,
+	0xB9, 0x27, 0x10, 0xE1, 0x32, 0x3D, 0x2A, 0xA0,
+	0xC5, 0x70, 0xDE, 0x1E, 0x10, 0xD7, 0x01, 0x50,
+	0x4F, 0x87, 0xCA, 0xD9, 0xBF, 0x12, 0xEA, 0x38,
+	0x4C, 0x43, 0xD5, 0x5A, 0xEF, 0x3E, 0x21, 0x8E,
+	0x59, 0x77, 0x23, 0xED, 0x51, 0x09, 0x99, 0x73,
+	0xD2, 0x56, 0x04, 0xDD, 0x8F, 0x5F, 0xDF, 0x79,
+	0xFF, 0x16, 0x8C, 0xB0, 0xBA, 0x8A, 0x1A, 0x56,
+	0xAF, 0xCA, 0x19, 0xF2, 0x64, 0x1A, 0xF5, 0x1E,
+	0xA7, 0xA7, 0x84, 0x3D, 0xAD, 0xC1, 0x0E, 0x22,
+	0xA1, 0x45, 0xFC, 0xB4, 0x13, 0x91, 0x34, 0xB7,
+	0x48, 0xEF, 0x9E, 0xD9, 0x0B, 0xE3, 0x82, 0x75,
+	0x80, 0xC5, 0xD9, 0xA0, 0x77, 0xA3, 0xF9, 0xCC,
+	0x67, 0xDD, 0xCB, 0x28, 0xC6, 0xE8, 0x2C, 0xB8,
+	0xAC, 0x63, 0xBD, 0x3B, 0x28, 0x4A, 0xE9, 0x2D,
+	0x29, 0x84, 0xD7, 0x4F, 0x61, 0x06, 0xE3, 0x37,
+	0xC1, 0x58, 0x20, 0x5D, 0x0A, 0xE7, 0x45, 0x29,
+	0x7D, 0xED, 0x0F, 0xCE, 0x00, 0x95, 0x2A, 0x62,
+	0x38, 0xA8, 0x1A, 0x3A, 0x96, 0x0E, 0x56, 0xD9,
+	0x18, 0xC2, 0x25, 0xA5, 0xAA, 0x27, 0x0A, 0x6E,
+	0xDD, 0x1C, 0x35, 0x6C, 0xC1, 0x26, 0x90, 0xF6,
+	0x43, 0x1B, 0x34, 0xDA, 0xE1, 0x5D, 0x09, 0x7C,
+	0xBE, 0x0F, 0x40, 0xD3, 0x24, 0x82, 0x0B, 0xFF,
+	0xE6, 0xB7, 0x10, 0xD6, 0x36, 0xD0, 0xE6, 0xC0,
+	0xBE, 0x65, 0x4C, 0x83, 0xF1, 0xDA, 0xDE, 0xCE,
+	0xE8, 0x5A, 0x80, 0x88, 0xFE, 0x9B, 0x79, 0x54,
+	0xA3, 0xA4, 0x5A, 0x76, 0xD0, 0xE2, 0xCE, 0x92,
+	0x53, 0x7D, 0x9C, 0xDA, 0xA1, 0xED, 0x9F, 0x56,
+	0x05, 0x0A, 0xA4, 0x81, 0xC7, 0x82, 0x5B, 0xB8,
+	0xC7, 0xA8, 0x95, 0x21, 0x99, 0x0B, 0x0F, 0xD2,
+	0x66, 0x68, 0xC3, 0x07, 0x53, 0x93, 0x8C, 0x68,
+	0x5A, 0xF5, 0x6F, 0x5E, 0x07, 0x68, 0x70, 0xF2,
+	0x6A, 0x78, 0xA8, 0xDB, 0x24, 0x6F, 0xD9, 0x74,
+	0x38, 0x29, 0xBF, 0x50, 0xCC, 0xC8, 0x34, 0x82,
+	0x69, 0x1A, 0xF4, 0x1A, 0x95, 0xE1, 0x31, 0x39,
+	0x00, 0xBE, 0xF6, 0x25, 0x19, 0x5A, 0xF6, 0xA7,
+	0xB7, 0xC7, 0xAC, 0xFA, 0x52, 0x6D, 0xC7, 0xEA,
+	0xFF, 0xC4, 0xB0, 0x84, 0x41, 0x2D, 0x6B, 0x22,
+	0x57, 0x1F, 0x8A, 0x97, 0x71, 0xEF, 0x54, 0xA7,
+	0xA9, 0xB5, 0xA0, 0x7A, 0xFF, 0x52, 0xBC, 0x78,
+	0x56, 0x8C, 0x9C, 0x2A, 0xFB, 0x31, 0xA6, 0xC8,
+	0x87, 0xC8, 0x82, 0x72, 0x00, 0x68, 0x27, 0xDE,
+	0x54, 0x67, 0x4C, 0x36, 0x4E, 0xE2, 0x77, 0x7F,
+	0xAD, 0x15, 0x1B, 0xC9, 0x07, 0xC4, 0x84, 0x50,
+	0x84, 0x45, 0xB5, 0x1D, 0xCE, 0x3F, 0x7C, 0xDF,
+	0x73, 0x88, 0x3E, 0x6D, 0xAC, 0x18, 0x67, 0x2C,
+	0x1D, 0x31, 0xD5, 0x41, 0x6E, 0xFC, 0x3D, 0xFE,
+	0x5C, 0x6D, 0x3B, 0xCB, 0x67, 0x05, 0x44, 0x8B,
+	0x02, 0xB7, 0xF5, 0x05, 0xFD, 0x1D, 0x7E, 0x13,
+	0x90, 0xCE, 0xED, 0xD2, 0xAB, 0x08, 0xFF, 0xC3,
+	0x91, 0x2C, 0x79, 0x06, 0xDB, 0x54, 0xAB, 0xFF,
+	0xF6, 0x9D, 0xBB, 0xDC, 0x3C, 0xCD, 0x03, 0xE7,
+	0xD8, 0x1A, 0x4E, 0x7F, 0xCB, 0x1B, 0xA3, 0xCA,
+	0xDC, 0x14, 0xC5, 0xFE, 0x46, 0x38, 0x07, 0x82,
+	0x5E, 0x9A, 0x77, 0x9C, 0xB4, 0x44, 0x26, 0xBA,
+	0xC2, 0x27, 0xD0, 0xF4, 0x75, 0x67, 0x4A, 0x15,
+	0x2A, 0x55, 0x2A, 0x61, 0x87, 0x55, 0xA0, 0xFB,
+	0xE0, 0x93, 0xA5, 0xD7, 0xDF, 0x0D, 0x97, 0xD3,
+	0x93, 0x39, 0x0B, 0x5A, 0xC6, 0x86, 0x17, 0x7D,
+	0x6A, 0xA4, 0x07, 0x9C, 0xB9, 0x0F, 0x42, 0x15,
+	0x46, 0x71, 0x45, 0x70, 0x75, 0x4B, 0xD5, 0x80,
+	0x64, 0x62, 0x92, 0x50, 0xBC, 0x10, 0xA1, 0x68,
+	0x60, 0x5A, 0x6F, 0x31, 0x26, 0xCA, 0xB1, 0x48,
+	0xFB, 0xC6, 0xF4, 0x3A, 0xCA, 0x52, 0x20, 0x2F,
+	0x7D, 0xEC, 0x19, 0xF8, 0xAA, 0x27, 0xC2, 0x23,
+	0xD0, 0x8F, 0x60, 0xFC, 0x7F, 0xA0, 0xCB, 0xDC,
+	0xA2, 0xC6, 0xC4, 0xC5, 0x13, 0x00, 0xF3, 0x43,
+	0x48, 0x6D, 0xFD, 0x7D, 0xB9, 0xA8, 0x14, 0xB0,
+	0x0C, 0x72, 0x82, 0x2F, 0x99, 0x64, 0x41, 0x2B,
+	0xB3, 0x34, 0x73, 0x73, 0xF4, 0x26, 0x1D, 0x06,
+	0xDF, 0x6E, 0xF4, 0x20, 0x1E, 0x31, 0xE3, 0x8A,
+	0x01, 0x6C, 0xDB, 0x3C, 0xE3, 0xC6, 0xC4, 0xC5,
+	0xB8, 0x20, 0x51, 0xF1, 0xD6, 0xB0, 0x30, 0xB7,
+	0x2D, 0xDA, 0x95, 0x01, 0x0D, 0xED, 0xEE, 0x6F,
+	0x69, 0xFD, 0xCF, 0x9D, 0xDD, 0x05, 0xD6, 0xC0,
+	0xFE, 0x11, 0x67, 0xAF, 0x53, 0x94, 0x60, 0xFC,
+	0x56, 0xBA, 0x0C, 0x5F, 0xA7, 0x7E, 0xDA, 0x65
+};
+static const struct
+blockcipher_test_data aes_test_data_xts_wrapped_key_48_pt_4096_du_4096 = {
+	.crypto_algo = RTE_CRYPTO_CIPHER_AES_XTS,
+	.cipher_key = {
+		.data = {
+			0x94, 0xD8, 0xAE, 0xD7, 0x74, 0xBF, 0x51, 0xF1,
+			0x2D, 0x19, 0x0B, 0x9A, 0xB3, 0xDC, 0x1E, 0xA9,
+			0x5C, 0xC5, 0x47, 0xC1, 0xBB, 0x94, 0xDB, 0xFA,
+			0x2B, 0x1E, 0x54, 0xA5, 0x76, 0xAD, 0x07, 0x4B,
+			0x46, 0xE0, 0xAA, 0xF9, 0xF7, 0x5C, 0x64, 0x2D,
+			0x8F, 0x04, 0x17, 0x72, 0xE5, 0x8D, 0xFE, 0x9C
+		},
+		.len = 48
+	},
+	.iv = {
+		.data = {
+			0x4F, 0xAE, 0xF7, 0x11, 0x7C, 0xDA, 0x59, 0xC6,
+			0x6E, 0x4B, 0x92, 0x01, 0x3E, 0x76, 0x8A, 0xD5
+		},
+		.len = 16
+	},
+	.plaintext = {
+		.data = plaintext_aes256xts_4096bytes,
+		.len = 4096
+	},
+	.ciphertext = {
+		.data = ciphertext_aes256xts_4096bytes,
+		.len = 4096
+	},
+	.xts_dataunit_len = 4096,
+	.wrapped_key = true
+};
+
+static const struct
+blockcipher_test_data aes_test_data_xts_wrapped_key_48_pt_4096_du_0 = {
+	.crypto_algo = RTE_CRYPTO_CIPHER_AES_XTS,
+	.cipher_key = {
+		.data = {
+			0x94, 0xD8, 0xAE, 0xD7, 0x74, 0xBF, 0x51, 0xF1,
+			0x2D, 0x19, 0x0B, 0x9A, 0xB3, 0xDC, 0x1E, 0xA9,
+			0x5C, 0xC5, 0x47, 0xC1, 0xBB, 0x94, 0xDB, 0xFA,
+			0x2B, 0x1E, 0x54, 0xA5, 0x76, 0xAD, 0x07, 0x4B,
+			0x46, 0xE0, 0xAA, 0xF9, 0xF7, 0x5C, 0x64, 0x2D,
+			0x8F, 0x04, 0x17, 0x72, 0xE5, 0x8D, 0xFE, 0x9C
+		},
+		.len = 48
+	},
+	.iv = {
+		.data = {
+			0x4F, 0xAE, 0xF7, 0x11, 0x7C, 0xDA, 0x59, 0xC6,
+			0x6E, 0x4B, 0x92, 0x01, 0x3E, 0x76, 0x8A, 0xD5
+		},
+		.len = 16
+	},
+	.plaintext = {
+		.data = plaintext_aes256xts_4096bytes,
+		.len = 4096
+	},
+	.ciphertext = {
+		.data = ciphertext_aes256xts_4096bytes,
+		.len = 4096
+	},
+	.xts_dataunit_len = 0,
+	.wrapped_key = true
+};
+
+static const struct
+blockcipher_test_data aes_test_data_xts_wrapped_key_48_pt_512_du_0 = {
+	.crypto_algo = RTE_CRYPTO_CIPHER_AES_XTS,
+	.cipher_key = {
+		.data = {
+			0x94, 0xD8, 0xAE, 0xD7, 0x74, 0xBF, 0x51, 0xF1,
+			0x2D, 0x19, 0x0B, 0x9A, 0xB3, 0xDC, 0x1E, 0xA9,
+			0x5C, 0xC5, 0x47, 0xC1, 0xBB, 0x94, 0xDB, 0xFA,
+			0x2B, 0x1E, 0x54, 0xA5, 0x76, 0xAD, 0x07, 0x4B,
+			0x46, 0xE0, 0xAA, 0xF9, 0xF7, 0x5C, 0x64, 0x2D,
+			0x8F, 0x04, 0x17, 0x72, 0xE5, 0x8D, 0xFE, 0x9C
+		},
+		.len = 48
+	},
+	.iv = {
+		.data = {
+			0x4F, 0xAE, 0xF7, 0x11, 0x7C, 0xDA, 0x59, 0xC6,
+			0x6E, 0x4B, 0x92, 0x01, 0x3E, 0x76, 0x8A, 0xD5
+		},
+		.len = 16
+	},
+	.plaintext = {
+		.data = plaintext_aes256xts_512bytes,
+		.len = 512
+	},
+	.ciphertext = {
+		.data = ciphertext_aes256xts_512bytes,
+		.len = 512
+	},
+	.xts_dataunit_len = 0,
+	.wrapped_key = true
+};
+
+
 /* AES-DOCSIS-BPI test vectors */
 
 /* Multiple of AES block size */
@@ -3347,6 +4639,54 @@ static const struct blockcipher_test_case aes_cipheronly_test_cases[] = {
 		.test_data = &aes_test_data_xts_key_64_pt_48,
 		.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
 	},
+	{
+		.test_descr = "AES-256-XTS Encryption (512-byte plaintext"
+			      " Dataunit 512)",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_512_du_512,
+		.op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT,
+	},
+	{
+		.test_descr = "AES-256-XTS Decryption (512-byte plaintext"
+			      " Dataunit 512)",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_512_du_512,
+		.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
+	},
+	{
+		.test_descr = "AES-256-XTS Encryption (512-byte plaintext"
+			      " Dataunit 0)",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_512_du_0,
+		.op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT,
+	},
+	{
+		.test_descr = "AES-256-XTS Decryption (512-byte plaintext"
+			      " Dataunit 0)",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_512_du_0,
+		.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
+	},
+	{
+		.test_descr = "AES-256-XTS Encryption (4096-byte plaintext"
+			      " Dataunit 4096)",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_4096_du_4096,
+		.op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT,
+	},
+	{
+		.test_descr = "AES-256-XTS Decryption (4096-byte plaintext"
+			      " Dataunit 4096)",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_4096_du_4096,
+		.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
+	},
+	{
+		.test_descr = "AES-256-XTS Encryption (4096-byte plaintext"
+			      " Dataunit 0)",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_4096_du_0,
+		.op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT,
+	},
+	{
+		.test_descr = "AES-256-XTS Decryption (4096-byte plaintext"
+			      " Dataunit 0)",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_4096_du_0,
+		.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
+	},
 	{
 		.test_descr = "cipher-only - NULL algo - x8 - encryption",
 		.test_data = &null_test_data_chain_x8_multiple,
diff --git a/app/test/test_cryptodev_blockcipher.c b/app/test/test_cryptodev_blockcipher.c
index d342c7b859..53fd4718af 100644
--- a/app/test/test_cryptodev_blockcipher.c
+++ b/app/test/test_cryptodev_blockcipher.c
@@ -36,7 +36,8 @@ verify_algo_support(const struct blockcipher_test_case *t,
 		if (capability == NULL)
 			return -1;
 
-		if (cap_idx.algo.cipher != RTE_CRYPTO_CIPHER_NULL)
+		if (cap_idx.algo.cipher != RTE_CRYPTO_CIPHER_NULL &&
+				!(t->test_data->wrapped_key))
 			ret = rte_cryptodev_sym_capability_check_cipher(capability,
 							tdata->cipher_key.len,
 							tdata->iv.len);
@@ -145,6 +146,12 @@ test_blockcipher_one_case(const struct blockcipher_test_case *t,
 
 		nb_segs = 3;
 	}
+	if (!!(feat_flags & RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY) ^
+		tdata->wrapped_key) {
+		snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN,
+			"SKIPPED");
+		return TEST_SKIPPED;
+	}
 
 	if (global_api_test_type == CRYPTODEV_RAW_API_TEST &&
 		!(feat_flags & RTE_CRYPTODEV_FF_SYM_RAW_DP)) {
@@ -450,6 +457,7 @@ test_blockcipher_one_case(const struct blockcipher_test_case *t,
 		cipher_xform->cipher.key.data = cipher_key;
 		cipher_xform->cipher.key.length = tdata->cipher_key.len;
 		cipher_xform->cipher.iv.offset = IV_OFFSET;
+		cipher_xform->cipher.dataunit_len = tdata->xts_dataunit_len;
 
 		if (tdata->crypto_algo == RTE_CRYPTO_CIPHER_NULL)
 			cipher_xform->cipher.iv.length = 0;
diff --git a/app/test/test_cryptodev_blockcipher.h b/app/test/test_cryptodev_blockcipher.h
index a06241b06d..dcaa08ae22 100644
--- a/app/test/test_cryptodev_blockcipher.h
+++ b/app/test/test_cryptodev_blockcipher.h
@@ -97,6 +97,8 @@ struct blockcipher_test_data {
 
 	unsigned int cipher_offset;
 	unsigned int auth_offset;
+	uint16_t xts_dataunit_len;
+	bool wrapped_key;
 };
 
 struct unit_test_suite *
-- 
2.27.0

[dpdk-dev] [PATCH v7 16/16] test/crypto: add AES-XTS multi segment OOP tests

[Shiri Kuzin]

The AES-XTS algorithm can supports wrapped key and data-unit.
The encryption/decryption can be done out of place and using multi
segments.

Add multi segment and out of place tests to the recently added AES-XTS
vectors, which support using data-unit and a wrapped key.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 app/test/test_cryptodev_aes_test_vectors.h | 63 +++++++++++++++++++++-
 1 file changed, 61 insertions(+), 2 deletions(-)

diff --git a/app/test/test_cryptodev_aes_test_vectors.h b/app/test/test_cryptodev_aes_test_vectors.h
index dd7ffde685..f0a7376cac 100644
--- a/app/test/test_cryptodev_aes_test_vectors.h
+++ b/app/test/test_cryptodev_aes_test_vectors.h
@@ -4688,9 +4688,68 @@ static const struct blockcipher_test_case aes_cipheronly_test_cases[] = {
 		.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
 	},
 	{
-		.test_descr = "cipher-only - NULL algo - x8 - encryption",
-		.test_data = &null_test_data_chain_x8_multiple,
+		.test_descr = "AES-256-XTS Encryption (512-byte plaintext"
+			      " Dataunit 512) Scater gather OOP",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_512_du_512,
+		.op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT,
+		.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP |
+				BLOCKCIPHER_TEST_FEATURE_SG,
+	},
+	{
+		.test_descr = "AES-256-XTS Decryption (512-byte plaintext"
+			      " Dataunit 512) Scater gather OOP",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_512_du_512,
+		.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
+		.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP |
+				BLOCKCIPHER_TEST_FEATURE_SG,
+	},
+	{
+		.test_descr = "AES-256-XTS Encryption (512-byte plaintext"
+			      " Dataunit 0) Scater gather OOP",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_512_du_0,
+		.op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT,
+		.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP |
+				BLOCKCIPHER_TEST_FEATURE_SG,
+	},
+	{
+		.test_descr = "AES-256-XTS Decryption (512-byte plaintext"
+			      " Dataunit 0) Scater gather OOP",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_512_du_0,
+		.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
+		.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP |
+				BLOCKCIPHER_TEST_FEATURE_SG,
+	},
+	{
+		.test_descr = "AES-256-XTS Encryption (4096-byte plaintext"
+			      " Dataunit 4096) Scater gather OOP",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_4096_du_4096,
 		.op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT,
+		.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP |
+				BLOCKCIPHER_TEST_FEATURE_SG,
+	},
+	{
+		.test_descr = "AES-256-XTS Decryption (4096-byte plaintext"
+			      " Dataunit 4096) Scater gather OOP",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_4096_du_4096,
+		.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
+		.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP |
+				BLOCKCIPHER_TEST_FEATURE_SG,
+	},
+	{
+		.test_descr = "AES-256-XTS Encryption (4096-byte plaintext"
+			      " Dataunit 0) Scater gather OOP",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_4096_du_0,
+		.op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT,
+		.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP |
+				BLOCKCIPHER_TEST_FEATURE_SG,
+	},
+	{
+		.test_descr = "AES-256-XTS Decryption (4096-byte plaintext"
+			      " Dataunit 0) Scater gather OOP",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_4096_du_0,
+		.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
+		.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP |
+				BLOCKCIPHER_TEST_FEATURE_SG,
 	},
 	{
 		.test_descr = "cipher-only - NULL algo - x8 - decryption",
-- 
2.27.0

[dpdk-dev] [PATCH v8 00/16] drivers: introduce mlx5 crypto PMD

[Shiri Kuzin]

v2:
- Add data-path part.

v3:
- Rebase.

v4:
- Rebase + Address the following Akhil comments:
- Set HW feature flag in the capability patch.
- Fix mp object release in session clear.
- Some spelling and word missing in doc.
- Squash data-unit adjustment to the session operations commit.
- Wording: device argument -> devarg.

v5: 
- Add mlx5 crypto tests into test library.
- Update documentation according to Akhil comments.
- Fix memory region management.
- Fix multi segment case in data-path code.
- Split documentation to the correct commits according to Akhil comments.
- Rebase to new version.
- Change license to Nvidia license.

v6:
- Rebase to new version.
- Address David's comment and update log accordingly. 
- Update testing app commits as suggested by Akhil.

v7:
- Rebase to new version.
- Add scatter gather and OOP test cases.

v8:
- Add a test case that was removed by error.



Shiri Kuzin (11):
  drivers: introduce mlx5 crypto PMD
  crypto/mlx5: add DEK object management
  crypto/mlx5: add session operations
  crypto/mlx5: add basic operations
  crypto/mlx5: add queue pairs operations
  crypto/mlx5: add dev stop and start operations
  crypto/mlx5: add memory region management
  crypto/mlx5: create login object using DevX
  test/crypto: add mlx5 crypto driver
  test/crypto: add data-unit and wrapped vectors
  test/crypto: add AES-XTS multi segment OOP tests

Suanming Mou (5):
  crypto/mlx5: add keytag devarg
  crypto/mlx5: add maximum segments devarg
  crypto/mlx5: add WQE set initialization
  crypto/mlx5: add enqueue and dequeue operations
  crypto/mlx5: add statistic get and reset operations

 MAINTAINERS                                |    4 +
 app/test/test_cryptodev.c                  |    7 +
 app/test/test_cryptodev.h                  |    4 +-
 app/test/test_cryptodev_aes_test_vectors.h | 1404 ++++++++++++++++++++
 app/test/test_cryptodev_blockcipher.c      |   10 +-
 app/test/test_cryptodev_blockcipher.h      |    2 +
 doc/guides/cryptodevs/features/mlx5.ini    |   37 +
 doc/guides/cryptodevs/index.rst            |    1 +
 doc/guides/cryptodevs/mlx5.rst             |  153 +++
 doc/guides/rel_notes/release_21_08.rst     |    5 +
 drivers/common/mlx5/mlx5_common.h          |    1 +
 drivers/common/mlx5/mlx5_common_pci.c      |   14 +
 drivers/common/mlx5/mlx5_common_pci.h      |   21 +-
 drivers/crypto/meson.build                 |    1 +
 drivers/crypto/mlx5/meson.build            |   27 +
 drivers/crypto/mlx5/mlx5_crypto.c          | 1177 ++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto.h          |   91 ++
 drivers/crypto/mlx5/mlx5_crypto_dek.c      |  161 +++
 drivers/crypto/mlx5/mlx5_crypto_utils.h    |   19 +
 drivers/crypto/mlx5/version.map            |    3 +
 20 files changed, 3130 insertions(+), 12 deletions(-)
 create mode 100644 doc/guides/cryptodevs/features/mlx5.ini
 create mode 100644 doc/guides/cryptodevs/mlx5.rst
 create mode 100644 drivers/crypto/mlx5/meson.build
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto.c
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto.h
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_dek.c
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_utils.h
 create mode 100644 drivers/crypto/mlx5/version.map

-- 
2.27.0

[dpdk-dev] [PATCH v8 01/16] drivers: introduce mlx5 crypto PMD

[Shiri Kuzin]

Add a new PMD for Mellanox devices- crypto PMD.

The crypto PMD will be supported starting Nvidia ConnectX6 and
BlueField2.

The crypto PMD will add the support of encryption and decryption using
the AES-XTS symmetric algorithm.

The crypto PMD requires rdma-core and uses mlx5 DevX.

This patch adds the PCI probing, basic functions, build files and
log utility.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 MAINTAINERS                             |   4 +
 doc/guides/cryptodevs/features/mlx5.ini |  27 +++
 doc/guides/cryptodevs/index.rst         |   1 +
 doc/guides/cryptodevs/mlx5.rst          |  63 ++++++
 doc/guides/rel_notes/release_21_08.rst  |   5 +
 drivers/common/mlx5/mlx5_common.h       |   1 +
 drivers/common/mlx5/mlx5_common_pci.c   |  14 ++
 drivers/common/mlx5/mlx5_common_pci.h   |  21 +-
 drivers/crypto/meson.build              |   1 +
 drivers/crypto/mlx5/meson.build         |  26 +++
 drivers/crypto/mlx5/mlx5_crypto.c       | 275 ++++++++++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto_utils.h |  19 ++
 drivers/crypto/mlx5/version.map         |   3 +
 13 files changed, 450 insertions(+), 10 deletions(-)
 create mode 100644 doc/guides/cryptodevs/features/mlx5.ini
 create mode 100644 doc/guides/cryptodevs/mlx5.rst
 create mode 100644 drivers/crypto/mlx5/meson.build
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto.c
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_utils.h
 create mode 100644 drivers/crypto/mlx5/version.map

diff --git a/MAINTAINERS b/MAINTAINERS
index af2a91d7c4..ba2cb4ef42 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -1102,6 +1102,10 @@ F: drivers/crypto/octeontx2/
 F: doc/guides/cryptodevs/octeontx2.rst
 F: doc/guides/cryptodevs/features/octeontx2.ini
 
+Mellanox mlx5
+M: Matan Azrad <matan@nvidia.com>
+F: drivers/crypto/mlx5/
+
 Null Crypto
 M: Declan Doherty <declan.doherty@intel.com>
 F: drivers/crypto/null/
diff --git a/doc/guides/cryptodevs/features/mlx5.ini b/doc/guides/cryptodevs/features/mlx5.ini
new file mode 100644
index 0000000000..ceadd967b6
--- /dev/null
+++ b/doc/guides/cryptodevs/features/mlx5.ini
@@ -0,0 +1,27 @@
+;
+; Features of a mlx5 crypto driver.
+;
+; Refer to default.ini for the full list of available PMD features.
+;
+[Features]
+HW Accelerated         = Y
+
+;
+; Supported crypto algorithms of a mlx5 crypto driver.
+;
+[Cipher]
+
+;
+; Supported authentication algorithms of a mlx5 crypto driver.
+;
+[Auth]
+
+;
+; Supported AEAD algorithms of a mlx5 crypto driver.
+;
+[AEAD]
+
+;
+; Supported Asymmetric algorithms of a mlx5 crypto driver.
+;
+[Asymmetric]
diff --git a/doc/guides/cryptodevs/index.rst b/doc/guides/cryptodevs/index.rst
index 067e7d784e..0f981c77b5 100644
--- a/doc/guides/cryptodevs/index.rst
+++ b/doc/guides/cryptodevs/index.rst
@@ -23,6 +23,7 @@ Crypto Device Drivers
     octeontx
     octeontx2
     openssl
+    mlx5
     mvsam
     nitrox
     null
diff --git a/doc/guides/cryptodevs/mlx5.rst b/doc/guides/cryptodevs/mlx5.rst
new file mode 100644
index 0000000000..05a0a449e2
--- /dev/null
+++ b/doc/guides/cryptodevs/mlx5.rst
@@ -0,0 +1,63 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+   Copyright (c) 2021 NVIDIA Corporation & Affiliates
+
+.. include:: <isonum.txt>
+
+MLX5 Crypto Driver
+==================
+
+The MLX5 crypto driver library
+(**librte_crypto_mlx5**) provides support for **Mellanox ConnectX-6**
+family adapters.
+
+Overview
+--------
+
+The device can provide disk encryption services, allowing data encryption
+and decryption towards a disk. Having all encryption/decryption
+operations done in a single device can reduce cost and overheads of the related
+FIPS certification, as ConnectX-6 is FIPS 140-2 level-2 ready.
+The encryption cipher is AES-XTS of 256/512 bit key size.
+
+MKEY is a memory region object in the hardware, that holds address translation information and
+attributes per memory area. Its ID must be tied to addresses provided to the hardware.
+The encryption operations are performed with MKEY read/write transactions, when
+the MKEY is configured to perform crypto operations.
+
+The encryption does not require text to be aligned to the AES block size (128b).
+
+The PMD uses libibverbs and libmlx5 to access the device firmware or to
+access the hardware components directly.
+There are different levels of objects and bypassing abilities.
+To get the best performances:
+
+- Verbs is a complete high-level generic API.
+- Direct Verbs is a device-specific API.
+- DevX allows to access firmware objects.
+
+Enabling librte_crypto_mlx5 causes DPDK applications to be linked against
+libibverbs.
+
+
+Driver options
+--------------
+
+- ``class`` parameter [string]
+
+  Select the class of the driver that should probe the device.
+  `crypto` for the mlx5 crypto driver.
+
+
+Supported NICs
+--------------
+
+* Mellanox\ |reg| ConnectX\ |reg|-6 200G MCX654106A-HCAT (2x200G)
+
+Prerequisites
+-------------
+
+- Mellanox OFED version: **5.3**
+  see :doc:`../../nics/mlx5` guide for more Mellanox OFED details.
+
+- Compilation can be done also with rdma-core v15+.
+  see :doc:`../../nics/mlx5` guide for more rdma-core details.
diff --git a/doc/guides/rel_notes/release_21_08.rst b/doc/guides/rel_notes/release_21_08.rst
index 7d289e07e3..2bf4ce7a73 100644
--- a/doc/guides/rel_notes/release_21_08.rst
+++ b/doc/guides/rel_notes/release_21_08.rst
@@ -125,6 +125,11 @@ New Features
   The experimental PMD power management API now supports managing
   multiple Ethernet Rx queues per lcore.
 
+* **Added support for Nvidia crypto device driver.**
+
+  * Added mlx5 crypto driver to support AES-XTS cipher operations.
+    The first device to support it is ConnectX-6.
+
 
 Removed Items
 -------------
diff --git a/drivers/common/mlx5/mlx5_common.h b/drivers/common/mlx5/mlx5_common.h
index 7fb7d40b38..962179a5a5 100644
--- a/drivers/common/mlx5/mlx5_common.h
+++ b/drivers/common/mlx5/mlx5_common.h
@@ -216,6 +216,7 @@ enum mlx5_class {
 	MLX5_CLASS_VDPA = RTE_BIT64(1),
 	MLX5_CLASS_REGEX = RTE_BIT64(2),
 	MLX5_CLASS_COMPRESS = RTE_BIT64(3),
+	MLX5_CLASS_CRYPTO = RTE_BIT64(4),
 };
 
 #define MLX5_DBR_SIZE RTE_CACHE_LINE_SIZE
diff --git a/drivers/common/mlx5/mlx5_common_pci.c b/drivers/common/mlx5/mlx5_common_pci.c
index 34747c4e07..5547e62d6b 100644
--- a/drivers/common/mlx5/mlx5_common_pci.c
+++ b/drivers/common/mlx5/mlx5_common_pci.c
@@ -31,6 +31,7 @@ static const struct {
 	{ .name = "net", .driver_class = MLX5_CLASS_NET },
 	{ .name = "regex", .driver_class = MLX5_CLASS_REGEX },
 	{ .name = "compress", .driver_class = MLX5_CLASS_COMPRESS },
+	{ .name = "crypto", .driver_class = MLX5_CLASS_CRYPTO },
 };
 
 static const unsigned int mlx5_class_combinations[] = {
@@ -38,13 +39,26 @@ static const unsigned int mlx5_class_combinations[] = {
 	MLX5_CLASS_VDPA,
 	MLX5_CLASS_REGEX,
 	MLX5_CLASS_COMPRESS,
+	MLX5_CLASS_CRYPTO,
 	MLX5_CLASS_NET | MLX5_CLASS_REGEX,
 	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX,
 	MLX5_CLASS_NET | MLX5_CLASS_COMPRESS,
 	MLX5_CLASS_VDPA | MLX5_CLASS_COMPRESS,
 	MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
+	MLX5_CLASS_NET | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_VDPA | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO,
 	MLX5_CLASS_NET | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
 	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
+	MLX5_CLASS_NET | MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_NET | MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_VDPA | MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_NET | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS |
+	MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS |
+	MLX5_CLASS_CRYPTO,
 	/* New class combination should be added here. */
 };
 
diff --git a/drivers/common/mlx5/mlx5_common_pci.h b/drivers/common/mlx5/mlx5_common_pci.h
index de89bb98bc..cb8d2f5f87 100644
--- a/drivers/common/mlx5/mlx5_common_pci.h
+++ b/drivers/common/mlx5/mlx5_common_pci.h
@@ -9,17 +9,18 @@
  * @file
  *
  * RTE Mellanox PCI Driver Interface
- * Mellanox ConnectX PCI device supports multiple class: net,vdpa,regex and
- * compress devices. This layer enables creating such multiple class of devices
- * on a single PCI device by allowing to bind multiple class specific device
- * driver to attach to mlx5_pci driver.
+ * Mellanox ConnectX PCI device supports multiple class: net,vdpa,regex,compress
+ * and crypto devices. This layer enables creating such multiple class of
+ * devices on a single PCI device by allowing to bind multiple class specific
+ * device driver to attach to mlx5_pci driver.
  *
- * -----------    ------------    -------------    ----------------
- * |   mlx5  |    |   mlx5   |    |   mlx5    |    |     mlx5     |
- * | net pmd |    | vdpa pmd |    | regex pmd |    | compress pmd |
- * -----------    ------------    -------------    ----------------
- *      \              \                    /              /
- *       \              \                  /              /
+ * --------    --------    ---------    ------------    ----------
+ * | mlx5 |    | mlx5 |    | mlx5  |    |   mlx5   |    |  mlx5  |
+ * | net  |    | vdpa |    | regex |    | compress |    | crypto |
+ * | pmd  |    | pmd  |    |  pmd  |    |   pmd    |    |  pmd   |
+ * --------    --------    ---------    ------------    ----------
+ *      \              \         |          /              /
+ *       \              \        |         /              /
  *        \              \_--------------_/              /
  *         \_______________|   mlx5     |_______________/
  *                         | pci common |
diff --git a/drivers/crypto/meson.build b/drivers/crypto/meson.build
index cb865aa0d5..ea239f4c56 100644
--- a/drivers/crypto/meson.build
+++ b/drivers/crypto/meson.build
@@ -16,6 +16,7 @@ drivers = [
         'dpaa_sec',
         'dpaa2_sec',
         'kasumi',
+        'mlx5',
         'mvsam',
         'nitrox',
         'null',
diff --git a/drivers/crypto/mlx5/meson.build b/drivers/crypto/mlx5/meson.build
new file mode 100644
index 0000000000..6fd70bc477
--- /dev/null
+++ b/drivers/crypto/mlx5/meson.build
@@ -0,0 +1,26 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2021 NVIDIA Corporation & Affiliates
+
+if not is_linux
+	build = false
+	reason = 'only supported on Linux'
+	subdir_done()
+endif
+
+fmt_name = 'mlx5_crypto'
+deps += ['common_mlx5', 'eal', 'cryptodev']
+sources = files(
+	'mlx5_crypto.c',
+)
+cflags_options = [
+	'-std=c11',
+	'-Wno-strict-prototypes',
+	'-D_BSD_SOURCE',
+	'-D_DEFAULT_SOURCE',
+	'-D_XOPEN_SOURCE=600',
+]
+foreach option:cflags_options
+	if cc.has_argument(option)
+		cflags += option
+	endif
+endforeach
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
new file mode 100644
index 0000000000..fbe3c21aae
--- /dev/null
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -0,0 +1,275 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2021 NVIDIA Corporation & Affiliates
+ */
+
+#include <rte_malloc.h>
+#include <rte_log.h>
+#include <rte_errno.h>
+#include <rte_pci.h>
+#include <rte_crypto.h>
+#include <rte_cryptodev.h>
+#include <rte_cryptodev_pmd.h>
+
+#include <mlx5_glue.h>
+#include <mlx5_common.h>
+#include <mlx5_common_pci.h>
+#include <mlx5_devx_cmds.h>
+#include <mlx5_common_os.h>
+
+#include "mlx5_crypto_utils.h"
+
+#define MLX5_CRYPTO_DRIVER_NAME mlx5_crypto
+#define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
+
+#define MLX5_CRYPTO_FEATURE_FLAGS \
+	RTE_CRYPTODEV_FF_HW_ACCELERATED
+
+struct mlx5_crypto_priv {
+	TAILQ_ENTRY(mlx5_crypto_priv) next;
+	struct ibv_context *ctx; /* Device context. */
+	struct rte_pci_device *pci_dev;
+	struct rte_cryptodev *crypto_dev;
+	void *uar; /* User Access Region. */
+	uint32_t pdn; /* Protection Domain number. */
+	struct ibv_pd *pd;
+};
+
+TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
+				TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
+static pthread_mutex_t priv_list_lock = PTHREAD_MUTEX_INITIALIZER;
+
+int mlx5_crypto_logtype;
+
+uint8_t mlx5_crypto_driver_id;
+
+static const char mlx5_crypto_drv_name[] = RTE_STR(MLX5_CRYPTO_DRIVER_NAME);
+
+static const struct rte_driver mlx5_drv = {
+	.name = mlx5_crypto_drv_name,
+	.alias = mlx5_crypto_drv_name
+};
+
+static struct cryptodev_driver mlx5_cryptodev_driver;
+
+static struct rte_cryptodev_ops mlx5_crypto_ops = {
+	.dev_configure			= NULL,
+	.dev_start			= NULL,
+	.dev_stop			= NULL,
+	.dev_close			= NULL,
+	.dev_infos_get			= NULL,
+	.stats_get			= NULL,
+	.stats_reset			= NULL,
+	.queue_pair_setup		= NULL,
+	.queue_pair_release		= NULL,
+	.sym_session_get_size		= NULL,
+	.sym_session_configure		= NULL,
+	.sym_session_clear		= NULL,
+	.sym_get_raw_dp_ctx_size	= NULL,
+	.sym_configure_raw_dp_ctx	= NULL,
+};
+
+static void
+mlx5_crypto_hw_global_release(struct mlx5_crypto_priv *priv)
+{
+	if (priv->pd != NULL) {
+		claim_zero(mlx5_glue->dealloc_pd(priv->pd));
+		priv->pd = NULL;
+	}
+	if (priv->uar != NULL) {
+		mlx5_glue->devx_free_uar(priv->uar);
+		priv->uar = NULL;
+	}
+}
+
+static int
+mlx5_crypto_pd_create(struct mlx5_crypto_priv *priv)
+{
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+	struct mlx5dv_obj obj;
+	struct mlx5dv_pd pd_info;
+	int ret;
+
+	priv->pd = mlx5_glue->alloc_pd(priv->ctx);
+	if (priv->pd == NULL) {
+		DRV_LOG(ERR, "Failed to allocate PD.");
+		return errno ? -errno : -ENOMEM;
+	}
+	obj.pd.in = priv->pd;
+	obj.pd.out = &pd_info;
+	ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_PD);
+	if (ret != 0) {
+		DRV_LOG(ERR, "Fail to get PD object info.");
+		mlx5_glue->dealloc_pd(priv->pd);
+		priv->pd = NULL;
+		return -errno;
+	}
+	priv->pdn = pd_info.pdn;
+	return 0;
+#else
+	(void)priv;
+	DRV_LOG(ERR, "Cannot get pdn - no DV support.");
+	return -ENOTSUP;
+#endif /* HAVE_IBV_FLOW_DV_SUPPORT */
+}
+
+static int
+mlx5_crypto_hw_global_prepare(struct mlx5_crypto_priv *priv)
+{
+	if (mlx5_crypto_pd_create(priv) != 0)
+		return -1;
+	priv->uar = mlx5_devx_alloc_uar(priv->ctx, -1);
+	if (priv->uar == NULL || mlx5_os_get_devx_uar_reg_addr(priv->uar) ==
+	    NULL) {
+		rte_errno = errno;
+		claim_zero(mlx5_glue->dealloc_pd(priv->pd));
+		DRV_LOG(ERR, "Failed to allocate UAR.");
+		return -1;
+	}
+	return 0;
+}
+
+/**
+ * DPDK callback to register a PCI device.
+ *
+ * This function spawns crypto device out of a given PCI device.
+ *
+ * @param[in] pci_drv
+ *   PCI driver structure (mlx5_crypto_driver).
+ * @param[in] pci_dev
+ *   PCI device information.
+ *
+ * @return
+ *   0 on success, 1 to skip this driver, a negative errno value otherwise
+ *   and rte_errno is set.
+ */
+static int
+mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
+			struct rte_pci_device *pci_dev)
+{
+	struct ibv_device *ibv;
+	struct rte_cryptodev *crypto_dev;
+	struct ibv_context *ctx;
+	struct mlx5_crypto_priv *priv;
+	struct mlx5_hca_attr attr = { 0 };
+	struct rte_cryptodev_pmd_init_params init_params = {
+		.name = "",
+		.private_data_size = sizeof(struct mlx5_crypto_priv),
+		.socket_id = pci_dev->device.numa_node,
+		.max_nb_queue_pairs =
+				RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
+	};
+	RTE_SET_USED(pci_drv);
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+		DRV_LOG(ERR, "Non-primary process type is not supported.");
+		rte_errno = ENOTSUP;
+		return -rte_errno;
+	}
+	ibv = mlx5_os_get_ibv_device(&pci_dev->addr);
+	if (ibv == NULL) {
+		DRV_LOG(ERR, "No matching IB device for PCI slot "
+			PCI_PRI_FMT ".", pci_dev->addr.domain,
+			pci_dev->addr.bus, pci_dev->addr.devid,
+			pci_dev->addr.function);
+		return -rte_errno;
+	}
+	DRV_LOG(INFO, "PCI information matches for device \"%s\".", ibv->name);
+	ctx = mlx5_glue->dv_open_device(ibv);
+	if (ctx == NULL) {
+		DRV_LOG(ERR, "Failed to open IB device \"%s\".", ibv->name);
+		rte_errno = ENODEV;
+		return -rte_errno;
+	}
+	if (mlx5_devx_cmd_query_hca_attr(ctx, &attr) != 0 ||
+	    attr.crypto == 0 || attr.aes_xts == 0) {
+		DRV_LOG(ERR, "Not enough capabilities to support crypto "
+			"operations, maybe old FW/OFED version?");
+		claim_zero(mlx5_glue->close_device(ctx));
+		rte_errno = ENOTSUP;
+		return -ENOTSUP;
+	}
+	crypto_dev = rte_cryptodev_pmd_create(ibv->name, &pci_dev->device,
+					&init_params);
+	if (crypto_dev == NULL) {
+		DRV_LOG(ERR, "Failed to create device \"%s\".", ibv->name);
+		claim_zero(mlx5_glue->close_device(ctx));
+		return -ENODEV;
+	}
+	DRV_LOG(INFO,
+		"Crypto device %s was created successfully.", ibv->name);
+	crypto_dev->dev_ops = &mlx5_crypto_ops;
+	crypto_dev->dequeue_burst = NULL;
+	crypto_dev->enqueue_burst = NULL;
+	crypto_dev->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
+	crypto_dev->driver_id = mlx5_crypto_driver_id;
+	priv = crypto_dev->data->dev_private;
+	priv->ctx = ctx;
+	priv->pci_dev = pci_dev;
+	priv->crypto_dev = crypto_dev;
+	if (mlx5_crypto_hw_global_prepare(priv) != 0) {
+		rte_cryptodev_pmd_destroy(priv->crypto_dev);
+		claim_zero(mlx5_glue->close_device(priv->ctx));
+		return -1;
+	}
+	pthread_mutex_lock(&priv_list_lock);
+	TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next);
+	pthread_mutex_unlock(&priv_list_lock);
+	return 0;
+}
+
+static int
+mlx5_crypto_pci_remove(struct rte_pci_device *pdev)
+{
+	struct mlx5_crypto_priv *priv = NULL;
+
+	pthread_mutex_lock(&priv_list_lock);
+	TAILQ_FOREACH(priv, &mlx5_crypto_priv_list, next)
+		if (rte_pci_addr_cmp(&priv->pci_dev->addr, &pdev->addr) != 0)
+			break;
+	if (priv)
+		TAILQ_REMOVE(&mlx5_crypto_priv_list, priv, next);
+	pthread_mutex_unlock(&priv_list_lock);
+	if (priv) {
+		mlx5_crypto_hw_global_release(priv);
+		rte_cryptodev_pmd_destroy(priv->crypto_dev);
+		claim_zero(mlx5_glue->close_device(priv->ctx));
+	}
+	return 0;
+}
+
+static const struct rte_pci_id mlx5_crypto_pci_id_map[] = {
+		{
+			RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
+					PCI_DEVICE_ID_MELLANOX_CONNECTX6)
+		},
+		{
+			.vendor_id = 0
+		}
+	};
+
+static struct mlx5_pci_driver mlx5_crypto_driver = {
+	.driver_class = MLX5_CLASS_CRYPTO,
+	.pci_driver = {
+		.driver = {
+			.name = RTE_STR(MLX5_CRYPTO_DRIVER_NAME),
+		},
+		.id_table = mlx5_crypto_pci_id_map,
+		.probe = mlx5_crypto_pci_probe,
+		.remove = mlx5_crypto_pci_remove,
+		.drv_flags = 0,
+	},
+};
+
+RTE_INIT(rte_mlx5_crypto_init)
+{
+	mlx5_common_init();
+	if (mlx5_glue != NULL)
+		mlx5_pci_driver_register(&mlx5_crypto_driver);
+}
+
+RTE_PMD_REGISTER_CRYPTO_DRIVER(mlx5_cryptodev_driver, mlx5_drv,
+			       mlx5_crypto_driver_id);
+
+RTE_LOG_REGISTER_DEFAULT(mlx5_crypto_logtype, NOTICE)
+RTE_PMD_EXPORT_NAME(MLX5_CRYPTO_DRIVER_NAME, __COUNTER__);
+RTE_PMD_REGISTER_PCI_TABLE(MLX5_CRYPTO_DRIVER_NAME, mlx5_crypto_pci_id_map);
+RTE_PMD_REGISTER_KMOD_DEP(MLX5_CRYPTO_DRIVER_NAME, "* ib_uverbs & mlx5_core & mlx5_ib");
diff --git a/drivers/crypto/mlx5/mlx5_crypto_utils.h b/drivers/crypto/mlx5/mlx5_crypto_utils.h
new file mode 100644
index 0000000000..b6c60ca782
--- /dev/null
+++ b/drivers/crypto/mlx5/mlx5_crypto_utils.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2021 NVIDIA Corporation & Affiliates
+ */
+
+#ifndef RTE_PMD_MLX5_CRYPTO_UTILS_H_
+#define RTE_PMD_MLX5_CRYPTO_UTILS_H_
+
+#include <mlx5_common.h>
+
+extern int mlx5_crypto_logtype;
+
+#define MLX5_CRYPTO_LOG_PREFIX "mlx5_crypto"
+/* Generic printf()-like logging macro with automatic line feed. */
+#define DRV_LOG(level, ...) \
+	PMD_DRV_LOG_(level, mlx5_crypto_logtype, MLX5_CRYPTO_LOG_PREFIX, \
+		__VA_ARGS__ PMD_DRV_LOG_STRIP PMD_DRV_LOG_OPAREN, \
+		PMD_DRV_LOG_CPAREN)
+
+#endif /* RTE_PMD_MLX5_CRYPTO_UTILS_H_ */
diff --git a/drivers/crypto/mlx5/version.map b/drivers/crypto/mlx5/version.map
new file mode 100644
index 0000000000..4a76d1d52d
--- /dev/null
+++ b/drivers/crypto/mlx5/version.map
@@ -0,0 +1,3 @@
+DPDK_21 {
+	local: *;
+};
-- 
2.27.0

[dpdk-dev] [PATCH v8 02/16] crypto/mlx5: add DEK object management

[Shiri Kuzin]

A DEK(Data encryption Key) is an mlx5 HW object which represents the
cipher algorithm key.
The DEKs are used during data encryption/decryption operations.

In symmetric algorithms like AES-STS, we use the same DEK for both
encryption and decryption.

Use the mlx5 hash-list tool to manage the DEK objects in the PMD.

Provide the compare, create and destroy functions to manage DEKs in
hash-list and introduce an internal API to setup and unset the DEK
management and to prepare and destroy specific DEK object.

The DEK hash-list will be created in dev_configure routine and
destroyed in dev_close routine.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/meson.build       |   1 +
 drivers/crypto/mlx5/mlx5_crypto.c     |  42 ++++---
 drivers/crypto/mlx5/mlx5_crypto.h     |  51 ++++++++
 drivers/crypto/mlx5/mlx5_crypto_dek.c | 161 ++++++++++++++++++++++++++
 4 files changed, 239 insertions(+), 16 deletions(-)
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto.h
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_dek.c

diff --git a/drivers/crypto/mlx5/meson.build b/drivers/crypto/mlx5/meson.build
index 6fd70bc477..d55cdbfe6f 100644
--- a/drivers/crypto/mlx5/meson.build
+++ b/drivers/crypto/mlx5/meson.build
@@ -11,6 +11,7 @@ fmt_name = 'mlx5_crypto'
 deps += ['common_mlx5', 'eal', 'cryptodev']
 sources = files(
 	'mlx5_crypto.c',
+	'mlx5_crypto_dek.c',
 )
 cflags_options = [
 	'-std=c11',
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index fbe3c21aae..d2d82c7b15 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -3,12 +3,9 @@
  */
 
 #include <rte_malloc.h>
-#include <rte_log.h>
 #include <rte_errno.h>
+#include <rte_log.h>
 #include <rte_pci.h>
-#include <rte_crypto.h>
-#include <rte_cryptodev.h>
-#include <rte_cryptodev_pmd.h>
 
 #include <mlx5_glue.h>
 #include <mlx5_common.h>
@@ -17,6 +14,7 @@
 #include <mlx5_common_os.h>
 
 #include "mlx5_crypto_utils.h"
+#include "mlx5_crypto.h"
 
 #define MLX5_CRYPTO_DRIVER_NAME mlx5_crypto
 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
@@ -24,16 +22,6 @@
 #define MLX5_CRYPTO_FEATURE_FLAGS \
 	RTE_CRYPTODEV_FF_HW_ACCELERATED
 
-struct mlx5_crypto_priv {
-	TAILQ_ENTRY(mlx5_crypto_priv) next;
-	struct ibv_context *ctx; /* Device context. */
-	struct rte_pci_device *pci_dev;
-	struct rte_cryptodev *crypto_dev;
-	void *uar; /* User Access Region. */
-	uint32_t pdn; /* Protection Domain number. */
-	struct ibv_pd *pd;
-};
-
 TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
 				TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
 static pthread_mutex_t priv_list_lock = PTHREAD_MUTEX_INITIALIZER;
@@ -51,11 +39,33 @@ static const struct rte_driver mlx5_drv = {
 
 static struct cryptodev_driver mlx5_cryptodev_driver;
 
+static int
+mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
+		struct rte_cryptodev_config *config __rte_unused)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+
+	if (mlx5_crypto_dek_setup(priv) != 0) {
+		DRV_LOG(ERR, "Dek hash list creation has failed.");
+		return -ENOMEM;
+	}
+	return 0;
+}
+
+static int
+mlx5_crypto_dev_close(struct rte_cryptodev *dev)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+
+	mlx5_crypto_dek_unset(priv);
+	return 0;
+}
+
 static struct rte_cryptodev_ops mlx5_crypto_ops = {
-	.dev_configure			= NULL,
+	.dev_configure			= mlx5_crypto_dev_configure,
 	.dev_start			= NULL,
 	.dev_stop			= NULL,
-	.dev_close			= NULL,
+	.dev_close			= mlx5_crypto_dev_close,
 	.dev_infos_get			= NULL,
 	.stats_get			= NULL,
 	.stats_reset			= NULL,
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
new file mode 100644
index 0000000000..167e9e57ad
--- /dev/null
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -0,0 +1,51 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2021 NVIDIA Corporation & Affiliates
+ */
+
+#ifndef MLX5_CRYPTO_H_
+#define MLX5_CRYPTO_H_
+
+#include <stdbool.h>
+
+#include <rte_cryptodev.h>
+#include <rte_cryptodev_pmd.h>
+
+#include <mlx5_common_utils.h>
+
+#define MLX5_CRYPTO_DEK_HTABLE_SZ (1 << 11)
+#define MLX5_CRYPTO_KEY_LENGTH 80
+
+struct mlx5_crypto_priv {
+	TAILQ_ENTRY(mlx5_crypto_priv) next;
+	struct ibv_context *ctx; /* Device context. */
+	struct rte_pci_device *pci_dev;
+	struct rte_cryptodev *crypto_dev;
+	void *uar; /* User Access Region. */
+	uint32_t pdn; /* Protection Domain number. */
+	struct ibv_pd *pd;
+	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
+};
+
+struct mlx5_crypto_dek {
+	struct mlx5_list_entry entry; /* Pointer to DEK hash list entry. */
+	struct mlx5_devx_obj *obj; /* Pointer to DEK DevX object. */
+	uint8_t data[MLX5_CRYPTO_KEY_LENGTH]; /* DEK key data. */
+	bool size_is_48; /* Whether the key\data size is 48 bytes or not. */
+} __rte_cache_aligned;
+
+int
+mlx5_crypto_dek_destroy(struct mlx5_crypto_priv *priv,
+			struct mlx5_crypto_dek *dek);
+
+struct mlx5_crypto_dek *
+mlx5_crypto_dek_prepare(struct mlx5_crypto_priv *priv,
+			struct rte_crypto_cipher_xform *cipher);
+
+int
+mlx5_crypto_dek_setup(struct mlx5_crypto_priv *priv);
+
+void
+mlx5_crypto_dek_unset(struct mlx5_crypto_priv *priv);
+
+#endif /* MLX5_CRYPTO_H_ */
+
diff --git a/drivers/crypto/mlx5/mlx5_crypto_dek.c b/drivers/crypto/mlx5/mlx5_crypto_dek.c
new file mode 100644
index 0000000000..43d1bcc9f8
--- /dev/null
+++ b/drivers/crypto/mlx5/mlx5_crypto_dek.c
@@ -0,0 +1,161 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2021 NVIDIA Corporation & Affiliates
+ */
+
+#include <rte_ip.h>
+#include <rte_common.h>
+#include <rte_errno.h>
+#include <rte_log.h>
+
+#include <mlx5_prm.h>
+#include <mlx5_devx_cmds.h>
+
+#include "mlx5_crypto_utils.h"
+#include "mlx5_crypto.h"
+
+struct mlx5_crypto_dek_ctx {
+	struct rte_crypto_cipher_xform *cipher;
+	struct mlx5_crypto_priv *priv;
+};
+
+int
+mlx5_crypto_dek_destroy(struct mlx5_crypto_priv *priv,
+			struct mlx5_crypto_dek *dek)
+{
+	return mlx5_hlist_unregister(priv->dek_hlist, &dek->entry);
+}
+
+struct mlx5_crypto_dek *
+mlx5_crypto_dek_prepare(struct mlx5_crypto_priv *priv,
+			struct rte_crypto_cipher_xform *cipher)
+{
+	struct mlx5_hlist *dek_hlist = priv->dek_hlist;
+	struct mlx5_crypto_dek_ctx dek_ctx = {
+		.cipher = cipher,
+		.priv = priv,
+	};
+	struct rte_crypto_cipher_xform *cipher_ctx = cipher;
+	uint64_t key64 = __rte_raw_cksum(cipher_ctx->key.data,
+					 cipher_ctx->key.length, 0);
+	struct mlx5_list_entry *entry = mlx5_hlist_register(dek_hlist,
+							     key64, &dek_ctx);
+
+	return entry == NULL ? NULL :
+			     container_of(entry, struct mlx5_crypto_dek, entry);
+}
+
+static struct mlx5_list_entry *
+mlx5_crypto_dek_clone_cb(void *tool_ctx __rte_unused,
+			 struct mlx5_list_entry *oentry,
+			 void *cb_ctx __rte_unused)
+{
+	struct mlx5_crypto_dek *entry = rte_zmalloc(__func__, sizeof(*entry),
+						    RTE_CACHE_LINE_SIZE);
+
+	if (!entry) {
+		DRV_LOG(ERR, "Cannot allocate dek resource memory.");
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+	memcpy(entry, oentry, sizeof(*entry));
+	return &entry->entry;
+}
+
+static void
+mlx5_crypto_dek_clone_free_cb(void *tool_ctx __rte_unused,
+			      struct mlx5_list_entry *entry)
+{
+	struct mlx5_crypto_dek *dek = container_of(entry,
+						struct mlx5_crypto_dek, entry);
+
+	rte_free(dek);
+}
+
+static int
+mlx5_crypto_dek_match_cb(void *tool_ctx __rte_unused,
+			 struct mlx5_list_entry *entry, void *cb_ctx)
+{
+	struct mlx5_crypto_dek_ctx *ctx = cb_ctx;
+	struct rte_crypto_cipher_xform *cipher_ctx = ctx->cipher;
+	struct mlx5_crypto_dek *dek =
+			container_of(entry, typeof(*dek), entry);
+	uint32_t key_len = dek->size_is_48 ? 48 : 80;
+
+	if (key_len != cipher_ctx->key.length)
+		return -1;
+	return memcmp(cipher_ctx->key.data, dek->data, key_len);
+}
+
+static struct mlx5_list_entry *
+mlx5_crypto_dek_create_cb(void *tool_ctx __rte_unused, void *cb_ctx)
+{
+	struct mlx5_crypto_dek_ctx *ctx = cb_ctx;
+	struct rte_crypto_cipher_xform *cipher_ctx = ctx->cipher;
+	struct mlx5_crypto_dek *dek = rte_zmalloc(__func__, sizeof(*dek),
+						  RTE_CACHE_LINE_SIZE);
+	struct mlx5_devx_dek_attr dek_attr = {
+		.pd = ctx->priv->pdn,
+		.key_purpose = MLX5_CRYPTO_KEY_PURPOSE_AES_XTS,
+		.has_keytag = 1,
+	};
+
+	if (dek == NULL) {
+		DRV_LOG(ERR, "Failed to allocate dek memory.");
+		return NULL;
+	}
+	switch (cipher_ctx->key.length) {
+	case 48:
+		dek->size_is_48 = true;
+		dek_attr.key_size = MLX5_CRYPTO_KEY_SIZE_128b;
+		break;
+	case 80:
+		dek->size_is_48 = false;
+		dek_attr.key_size = MLX5_CRYPTO_KEY_SIZE_256b;
+		break;
+	default:
+		DRV_LOG(ERR, "Key size not supported.");
+		return NULL;
+	}
+	memcpy(&dek_attr.key, cipher_ctx->key.data, cipher_ctx->key.length);
+	dek->obj = mlx5_devx_cmd_create_dek_obj(ctx->priv->ctx, &dek_attr);
+	if (dek->obj == NULL) {
+		rte_free(dek);
+		return NULL;
+	}
+	memcpy(&dek->data, cipher_ctx->key.data, cipher_ctx->key.length);
+	return &dek->entry;
+}
+
+static void
+mlx5_crypto_dek_remove_cb(void *tool_ctx __rte_unused,
+			  struct mlx5_list_entry *entry)
+{
+	struct mlx5_crypto_dek *dek =
+		container_of(entry, typeof(*dek), entry);
+
+	claim_zero(mlx5_devx_cmd_destroy(dek->obj));
+	rte_free(dek);
+}
+
+
+int
+mlx5_crypto_dek_setup(struct mlx5_crypto_priv *priv)
+{
+	priv->dek_hlist = mlx5_hlist_create("dek_hlist",
+				 MLX5_CRYPTO_DEK_HTABLE_SZ,
+				 0, 1, NULL, mlx5_crypto_dek_create_cb,
+				 mlx5_crypto_dek_match_cb,
+				 mlx5_crypto_dek_remove_cb,
+				 mlx5_crypto_dek_clone_cb,
+				 mlx5_crypto_dek_clone_free_cb);
+	if (priv->dek_hlist == NULL)
+		return -1;
+	return 0;
+}
+
+void
+mlx5_crypto_dek_unset(struct mlx5_crypto_priv *priv)
+{
+	mlx5_hlist_destroy(priv->dek_hlist);
+	priv->dek_hlist = NULL;
+}
-- 
2.27.0

[dpdk-dev] [PATCH v8 03/16] crypto/mlx5: add session operations

[Shiri Kuzin]

Sessions are used in symmetric transformations in order to prepare
objects and data for packet processing stage.

A mlx5 session includes iv_offset, pointer to mlx5_crypto_dek struct,
bsf_size, bsf_p_type, block size index, encryption_order and encryption
standard.

Implement the next session operations:
        mlx5_crypto_sym_session_get_size- returns the size of the mlx5
	session struct.
	mlx5_crypto_sym_session_configure- prepares the DEK hash-list
	and saves all the session data.
	mlx5_crypto_sym_session_clear - destroys the DEK hash-list.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 doc/guides/cryptodevs/features/mlx5.ini |   5 +
 doc/guides/cryptodevs/mlx5.rst          |  10 ++
 drivers/crypto/mlx5/mlx5_crypto.c       | 172 +++++++++++++++++++++++-
 3 files changed, 182 insertions(+), 5 deletions(-)

diff --git a/doc/guides/cryptodevs/features/mlx5.ini b/doc/guides/cryptodevs/features/mlx5.ini
index ceadd967b6..bd757b5211 100644
--- a/doc/guides/cryptodevs/features/mlx5.ini
+++ b/doc/guides/cryptodevs/features/mlx5.ini
@@ -4,12 +4,17 @@
 ; Refer to default.ini for the full list of available PMD features.
 ;
 [Features]
+Symmetric crypto       = Y
 HW Accelerated         = Y
+Cipher multiple data units = Y
+Cipher wrapped key     = Y
 
 ;
 ; Supported crypto algorithms of a mlx5 crypto driver.
 ;
 [Cipher]
+AES XTS (128)  = Y
+AES XTS (256)  = Y
 
 ;
 ; Supported authentication algorithms of a mlx5 crypto driver.
diff --git a/doc/guides/cryptodevs/mlx5.rst b/doc/guides/cryptodevs/mlx5.rst
index 05a0a449e2..dd1d1a615d 100644
--- a/doc/guides/cryptodevs/mlx5.rst
+++ b/doc/guides/cryptodevs/mlx5.rst
@@ -53,6 +53,16 @@ Supported NICs
 
 * Mellanox\ |reg| ConnectX\ |reg|-6 200G MCX654106A-HCAT (2x200G)
 
+
+Limitations
+-----------
+
+- AES-XTS keys provided in xform must include keytag and should be wrappend.
+- The supported data-unit lengths are 512B and 1KB. In case the `dataunit_len`
+  is not provided in the cipher xform, the OP length is limited to the above
+  values and 1MB.
+
+
 Prerequisites
 -------------
 
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index d2d82c7b15..3f0c97d081 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -3,6 +3,7 @@
  */
 
 #include <rte_malloc.h>
+#include <rte_mempool.h>
 #include <rte_errno.h>
 #include <rte_log.h>
 #include <rte_pci.h>
@@ -20,7 +21,9 @@
 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
 
 #define MLX5_CRYPTO_FEATURE_FLAGS \
-	RTE_CRYPTODEV_FF_HW_ACCELERATED
+	(RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | \
+	 RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY | \
+	 RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS)
 
 TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
 				TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
@@ -30,6 +33,32 @@ int mlx5_crypto_logtype;
 
 uint8_t mlx5_crypto_driver_id;
 
+const struct rte_cryptodev_capabilities mlx5_crypto_caps[] = {
+	{		/* AES XTS */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+			{.cipher = {
+				.algo = RTE_CRYPTO_CIPHER_AES_XTS,
+				.block_size = 16,
+				.key_size = {
+					.min = 32,
+					.max = 64,
+					.increment = 32
+				},
+				.iv_size = {
+					.min = 16,
+					.max = 16,
+					.increment = 0
+				},
+				.dataunit_set =
+				RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_512_BYTES |
+				RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_4096_BYTES,
+			}, }
+		}, }
+	},
+};
+
 static const char mlx5_crypto_drv_name[] = RTE_STR(MLX5_CRYPTO_DRIVER_NAME);
 
 static const struct rte_driver mlx5_drv = {
@@ -39,6 +68,49 @@ static const struct rte_driver mlx5_drv = {
 
 static struct cryptodev_driver mlx5_cryptodev_driver;
 
+struct mlx5_crypto_session {
+	uint32_t bs_bpt_eo_es;
+	/*
+	 * bsf_size, bsf_p_type, encryption_order and encryption standard,
+	 * saved in big endian format.
+	 */
+	uint32_t bsp_res;
+	/*
+	 * crypto_block_size_pointer and reserved 24 bits saved in big endian
+	 * format.
+	 */
+	uint32_t iv_offset:16;
+	/* Starting point for Initialisation Vector. */
+	struct mlx5_crypto_dek *dek; /* Pointer to dek struct. */
+	uint32_t dek_id; /* DEK ID */
+} __rte_packed;
+
+static void
+mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev,
+			  struct rte_cryptodev_info *dev_info)
+{
+	RTE_SET_USED(dev);
+	if (dev_info != NULL) {
+		dev_info->driver_id = mlx5_crypto_driver_id;
+		dev_info->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
+		dev_info->capabilities = mlx5_crypto_caps;
+		dev_info->max_nb_queue_pairs = 0;
+		dev_info->min_mbuf_headroom_req = 0;
+		dev_info->min_mbuf_tailroom_req = 0;
+		dev_info->sym.max_nb_sessions = 0;
+		/*
+		 * If 0, the device does not have any limitation in number of
+		 * sessions that can be used.
+		 */
+	}
+}
+
+static unsigned int
+mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
+{
+	return sizeof(struct mlx5_crypto_session);
+}
+
 static int
 mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
 		struct rte_cryptodev_config *config __rte_unused)
@@ -61,19 +133,109 @@ mlx5_crypto_dev_close(struct rte_cryptodev *dev)
 	return 0;
 }
 
+static int
+mlx5_crypto_sym_session_configure(struct rte_cryptodev *dev,
+				  struct rte_crypto_sym_xform *xform,
+				  struct rte_cryptodev_sym_session *session,
+				  struct rte_mempool *mp)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+	struct mlx5_crypto_session *sess_private_data;
+	struct rte_crypto_cipher_xform *cipher;
+	uint8_t encryption_order;
+	int ret;
+
+	if (unlikely(xform->next != NULL)) {
+		DRV_LOG(ERR, "Xform next is not supported.");
+		return -ENOTSUP;
+	}
+	if (unlikely((xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) ||
+		     (xform->cipher.algo != RTE_CRYPTO_CIPHER_AES_XTS))) {
+		DRV_LOG(ERR, "Only AES-XTS algorithm is supported.");
+		return -ENOTSUP;
+	}
+	ret = rte_mempool_get(mp, (void *)&sess_private_data);
+	if (ret != 0) {
+		DRV_LOG(ERR,
+			"Failed to get session %p private data from mempool.",
+			sess_private_data);
+		return -ENOMEM;
+	}
+	cipher = &xform->cipher;
+	sess_private_data->dek = mlx5_crypto_dek_prepare(priv, cipher);
+	if (sess_private_data->dek == NULL) {
+		rte_mempool_put(mp, sess_private_data);
+		DRV_LOG(ERR, "Failed to prepare dek.");
+		return -ENOMEM;
+	}
+	if (cipher->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
+		encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_MEMORY;
+	else
+		encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_WIRE;
+	sess_private_data->bs_bpt_eo_es = rte_cpu_to_be_32
+			(MLX5_BSF_SIZE_64B << MLX5_BSF_SIZE_OFFSET |
+			 MLX5_BSF_P_TYPE_CRYPTO << MLX5_BSF_P_TYPE_OFFSET |
+			 encryption_order << MLX5_ENCRYPTION_ORDER_OFFSET |
+			 MLX5_ENCRYPTION_STANDARD_AES_XTS);
+	switch (xform->cipher.dataunit_len) {
+	case 0:
+		sess_private_data->bsp_res = 0;
+		break;
+	case 512:
+		sess_private_data->bsp_res = rte_cpu_to_be_32
+					     ((uint32_t)MLX5_BLOCK_SIZE_512B <<
+					     MLX5_BLOCK_SIZE_OFFSET);
+		break;
+	case 4096:
+		sess_private_data->bsp_res = rte_cpu_to_be_32
+					     ((uint32_t)MLX5_BLOCK_SIZE_4096B <<
+					     MLX5_BLOCK_SIZE_OFFSET);
+		break;
+	default:
+		DRV_LOG(ERR, "Cipher data unit length is not supported.");
+		return -ENOTSUP;
+	}
+	sess_private_data->iv_offset = cipher->iv.offset;
+	sess_private_data->dek_id =
+			rte_cpu_to_be_32(sess_private_data->dek->obj->id &
+					 0xffffff);
+	set_sym_session_private_data(session, dev->driver_id,
+				     sess_private_data);
+	DRV_LOG(DEBUG, "Session %p was configured.", sess_private_data);
+	return 0;
+}
+
+static void
+mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev,
+			      struct rte_cryptodev_sym_session *sess)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+	struct mlx5_crypto_session *spriv = get_sym_session_private_data(sess,
+								dev->driver_id);
+
+	if (unlikely(spriv == NULL)) {
+		DRV_LOG(ERR, "Failed to get session %p private data.", spriv);
+		return;
+	}
+	mlx5_crypto_dek_destroy(priv, spriv->dek);
+	set_sym_session_private_data(sess, dev->driver_id, NULL);
+	rte_mempool_put(rte_mempool_from_obj(spriv), spriv);
+	DRV_LOG(DEBUG, "Session %p was cleared.", spriv);
+}
+
 static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_configure			= mlx5_crypto_dev_configure,
 	.dev_start			= NULL,
 	.dev_stop			= NULL,
 	.dev_close			= mlx5_crypto_dev_close,
-	.dev_infos_get			= NULL,
+	.dev_infos_get			= mlx5_crypto_dev_infos_get,
 	.stats_get			= NULL,
 	.stats_reset			= NULL,
 	.queue_pair_setup		= NULL,
 	.queue_pair_release		= NULL,
-	.sym_session_get_size		= NULL,
-	.sym_session_configure		= NULL,
-	.sym_session_clear		= NULL,
+	.sym_session_get_size		= mlx5_crypto_sym_session_get_size,
+	.sym_session_configure		= mlx5_crypto_sym_session_configure,
+	.sym_session_clear		= mlx5_crypto_sym_session_clear,
 	.sym_get_raw_dp_ctx_size	= NULL,
 	.sym_configure_raw_dp_ctx	= NULL,
 };
-- 
2.27.0

[dpdk-dev] [PATCH v8 04/16] crypto/mlx5: add basic operations

[Shiri Kuzin]

The basic dev control operations are configure, close and get info.

Extended the existing support of configure and close:
	-mlx5_crypto_dev_configure- function used to configure device.
	-mlx5_crypto_dev_close-  function used to close a configured
	 device.

Added config struct to user private data with the fields socket id,
number of queue pairs and feature flags to be disabled.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 26 +++++++++++++++++++-------
 drivers/crypto/mlx5/mlx5_crypto.h |  1 +
 2 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 3f0c97d081..a7e44deb9e 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -105,22 +105,27 @@ mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev,
 	}
 }
 
-static unsigned int
-mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
-{
-	return sizeof(struct mlx5_crypto_session);
-}
-
 static int
 mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
-		struct rte_cryptodev_config *config __rte_unused)
+			  struct rte_cryptodev_config *config)
 {
 	struct mlx5_crypto_priv *priv = dev->data->dev_private;
 
+	if (config == NULL) {
+		DRV_LOG(ERR, "Invalid crypto dev configure parameters.");
+		return -EINVAL;
+	}
+	if ((config->ff_disable & RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO) != 0) {
+		DRV_LOG(ERR,
+			"Disabled symmetric crypto feature is not supported.");
+		return -ENOTSUP;
+	}
 	if (mlx5_crypto_dek_setup(priv) != 0) {
 		DRV_LOG(ERR, "Dek hash list creation has failed.");
 		return -ENOMEM;
 	}
+	priv->dev_config = *config;
+	DRV_LOG(DEBUG, "Device %u was configured.", dev->driver_id);
 	return 0;
 }
 
@@ -130,9 +135,16 @@ mlx5_crypto_dev_close(struct rte_cryptodev *dev)
 	struct mlx5_crypto_priv *priv = dev->data->dev_private;
 
 	mlx5_crypto_dek_unset(priv);
+	DRV_LOG(DEBUG, "Device %u was closed.", dev->driver_id);
 	return 0;
 }
 
+static unsigned int
+mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
+{
+	return sizeof(struct mlx5_crypto_session);
+}
+
 static int
 mlx5_crypto_sym_session_configure(struct rte_cryptodev *dev,
 				  struct rte_crypto_sym_xform *xform,
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 167e9e57ad..a0df775407 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -24,6 +24,7 @@ struct mlx5_crypto_priv {
 	uint32_t pdn; /* Protection Domain number. */
 	struct ibv_pd *pd;
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
+	struct rte_cryptodev_config dev_config;
 };
 
 struct mlx5_crypto_dek {
-- 
2.27.0

[dpdk-dev] [PATCH v8 05/16] crypto/mlx5: add queue pairs operations

[Shiri Kuzin]

The HW queue pairs are a pair of send queue and receive queue of
independent work queues packed together in one object for the purpose
of transferring data between nodes of a network.

Completion Queue is a FIFO queue of completed work requests.

In crypto driver we use one QP in loopback in order to encrypt and
decrypt data locally without sending it to the wire.
In the configured QP we only use the SQ to perform the encryption and
decryption operations.

Added implementation for the QP setup function which creates the CQ,
creates the QP and changes its state to RTS (ready to send).

Added implementation for the release QP function to release all the QP
resources.

Added the ops structure that contains any operation which is supported
by the cryptodev.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 129 +++++++++++++++++++++++++++++-
 drivers/crypto/mlx5/mlx5_crypto.h |  11 +++
 2 files changed, 137 insertions(+), 3 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index a7e44deb9e..ebaa65c7a9 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -7,6 +7,7 @@
 #include <rte_errno.h>
 #include <rte_log.h>
 #include <rte_pci.h>
+#include <rte_memory.h>
 
 #include <mlx5_glue.h>
 #include <mlx5_common.h>
@@ -19,6 +20,7 @@
 
 #define MLX5_CRYPTO_DRIVER_NAME mlx5_crypto
 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
+#define MLX5_CRYPTO_MAX_QPS 1024
 
 #define MLX5_CRYPTO_FEATURE_FLAGS \
 	(RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | \
@@ -94,7 +96,7 @@ mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev,
 		dev_info->driver_id = mlx5_crypto_driver_id;
 		dev_info->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
 		dev_info->capabilities = mlx5_crypto_caps;
-		dev_info->max_nb_queue_pairs = 0;
+		dev_info->max_nb_queue_pairs = MLX5_CRYPTO_MAX_QPS;
 		dev_info->min_mbuf_headroom_req = 0;
 		dev_info->min_mbuf_tailroom_req = 0;
 		dev_info->sym.max_nb_sessions = 0;
@@ -235,6 +237,127 @@ mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev,
 	DRV_LOG(DEBUG, "Session %p was cleared.", spriv);
 }
 
+static int
+mlx5_crypto_queue_pair_release(struct rte_cryptodev *dev, uint16_t qp_id)
+{
+	struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
+
+	if (qp->qp_obj != NULL)
+		claim_zero(mlx5_devx_cmd_destroy(qp->qp_obj));
+	if (qp->umem_obj != NULL)
+		claim_zero(mlx5_glue->devx_umem_dereg(qp->umem_obj));
+	if (qp->umem_buf != NULL)
+		rte_free(qp->umem_buf);
+	mlx5_devx_cq_destroy(&qp->cq_obj);
+	rte_free(qp);
+	dev->data->queue_pairs[qp_id] = NULL;
+	return 0;
+}
+
+static int
+mlx5_crypto_qp2rts(struct mlx5_crypto_qp *qp)
+{
+	/*
+	 * In Order to configure self loopback, when calling these functions the
+	 * remote QP id that is used is the id of the same QP.
+	 */
+	if (mlx5_devx_cmd_modify_qp_state(qp->qp_obj, MLX5_CMD_OP_RST2INIT_QP,
+					  qp->qp_obj->id)) {
+		DRV_LOG(ERR, "Failed to modify QP to INIT state(%u).",
+			rte_errno);
+		return -1;
+	}
+	if (mlx5_devx_cmd_modify_qp_state(qp->qp_obj, MLX5_CMD_OP_INIT2RTR_QP,
+					  qp->qp_obj->id)) {
+		DRV_LOG(ERR, "Failed to modify QP to RTR state(%u).",
+			rte_errno);
+		return -1;
+	}
+	if (mlx5_devx_cmd_modify_qp_state(qp->qp_obj, MLX5_CMD_OP_RTR2RTS_QP,
+					  qp->qp_obj->id)) {
+		DRV_LOG(ERR, "Failed to modify QP to RTS state(%u).",
+			rte_errno);
+		return -1;
+	}
+	return 0;
+}
+
+static int
+mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
+			     const struct rte_cryptodev_qp_conf *qp_conf,
+			     int socket_id)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+	struct mlx5_devx_qp_attr attr = {0};
+	struct mlx5_crypto_qp *qp;
+	uint16_t log_nb_desc = rte_log2_u32(qp_conf->nb_descriptors);
+	uint32_t umem_size = RTE_BIT32(log_nb_desc) *
+			      MLX5_CRYPTO_WQE_SET_SIZE +
+			      sizeof(*qp->db_rec) * 2;
+	uint32_t alloc_size = sizeof(*qp);
+	struct mlx5_devx_cq_attr cq_attr = {
+		.uar_page_id = mlx5_os_get_devx_uar_page_id(priv->uar),
+	};
+
+	if (dev->data->queue_pairs[qp_id] != NULL)
+		mlx5_crypto_queue_pair_release(dev, qp_id);
+	alloc_size = RTE_ALIGN(alloc_size, RTE_CACHE_LINE_SIZE);
+	alloc_size += sizeof(struct rte_crypto_op *) * RTE_BIT32(log_nb_desc);
+	qp = rte_zmalloc_socket(__func__, alloc_size, RTE_CACHE_LINE_SIZE,
+				socket_id);
+	if (qp == NULL) {
+		DRV_LOG(ERR, "Failed to allocate QP memory.");
+		rte_errno = ENOMEM;
+		return -rte_errno;
+	}
+	if (mlx5_devx_cq_create(priv->ctx, &qp->cq_obj, log_nb_desc,
+				&cq_attr, socket_id) != 0) {
+		DRV_LOG(ERR, "Failed to create CQ.");
+		goto error;
+	}
+	qp->umem_buf = rte_zmalloc_socket(__func__, umem_size, 4096, socket_id);
+	if (qp->umem_buf == NULL) {
+		DRV_LOG(ERR, "Failed to allocate QP umem.");
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	qp->umem_obj = mlx5_glue->devx_umem_reg(priv->ctx,
+					       (void *)(uintptr_t)qp->umem_buf,
+					       umem_size,
+					       IBV_ACCESS_LOCAL_WRITE);
+	if (qp->umem_obj == NULL) {
+		DRV_LOG(ERR, "Failed to register QP umem.");
+		goto error;
+	}
+	attr.pd = priv->pdn;
+	attr.uar_index = mlx5_os_get_devx_uar_page_id(priv->uar);
+	attr.cqn = qp->cq_obj.cq->id;
+	attr.log_page_size = rte_log2_u32(sysconf(_SC_PAGESIZE));
+	attr.rq_size =  0;
+	attr.sq_size = RTE_BIT32(log_nb_desc);
+	attr.dbr_umem_valid = 1;
+	attr.wq_umem_id = qp->umem_obj->umem_id;
+	attr.wq_umem_offset = 0;
+	attr.dbr_umem_id = qp->umem_obj->umem_id;
+	attr.dbr_address = RTE_BIT64(log_nb_desc) *
+			   MLX5_CRYPTO_WQE_SET_SIZE;
+	qp->qp_obj = mlx5_devx_cmd_create_qp(priv->ctx, &attr);
+	if (qp->qp_obj == NULL) {
+		DRV_LOG(ERR, "Failed to create QP(%u).", rte_errno);
+		goto error;
+	}
+	qp->db_rec = RTE_PTR_ADD(qp->umem_buf, (uintptr_t)attr.dbr_address);
+	if (mlx5_crypto_qp2rts(qp))
+		goto error;
+	qp->ops = (struct rte_crypto_op **)RTE_ALIGN((uintptr_t)(qp + 1),
+							   RTE_CACHE_LINE_SIZE);
+	dev->data->queue_pairs[qp_id] = qp;
+	return 0;
+error:
+	mlx5_crypto_queue_pair_release(dev, qp_id);
+	return -1;
+}
+
 static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_configure			= mlx5_crypto_dev_configure,
 	.dev_start			= NULL,
@@ -243,8 +366,8 @@ static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_infos_get			= mlx5_crypto_dev_infos_get,
 	.stats_get			= NULL,
 	.stats_reset			= NULL,
-	.queue_pair_setup		= NULL,
-	.queue_pair_release		= NULL,
+	.queue_pair_setup		= mlx5_crypto_queue_pair_setup,
+	.queue_pair_release		= mlx5_crypto_queue_pair_release,
 	.sym_session_get_size		= mlx5_crypto_sym_session_get_size,
 	.sym_session_configure		= mlx5_crypto_sym_session_configure,
 	.sym_session_clear		= mlx5_crypto_sym_session_clear,
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index a0df775407..1c3c4855ec 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -11,9 +11,11 @@
 #include <rte_cryptodev_pmd.h>
 
 #include <mlx5_common_utils.h>
+#include <mlx5_common_devx.h>
 
 #define MLX5_CRYPTO_DEK_HTABLE_SZ (1 << 11)
 #define MLX5_CRYPTO_KEY_LENGTH 80
+#define MLX5_CRYPTO_WQE_SET_SIZE 1024
 
 struct mlx5_crypto_priv {
 	TAILQ_ENTRY(mlx5_crypto_priv) next;
@@ -27,6 +29,15 @@ struct mlx5_crypto_priv {
 	struct rte_cryptodev_config dev_config;
 };
 
+struct mlx5_crypto_qp {
+	struct mlx5_devx_cq cq_obj;
+	struct mlx5_devx_obj *qp_obj;
+	struct mlx5dv_devx_umem *umem_obj;
+	void *umem_buf;
+	volatile uint32_t *db_rec;
+	struct rte_crypto_op **ops;
+};
+
 struct mlx5_crypto_dek {
 	struct mlx5_list_entry entry; /* Pointer to DEK hash list entry. */
 	struct mlx5_devx_obj *obj; /* Pointer to DEK DevX object. */
-- 
2.27.0

[dpdk-dev] [PATCH v8 06/16] crypto/mlx5: add dev stop and start operations

[Shiri Kuzin]

Add the dev_start function that is used to start a configured device.
Add the dev_stop function that is used to stop a configured device.

Both functions set the dev parameter as used and return 0.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index ebaa65c7a9..37e66cf57b 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -131,6 +131,19 @@ mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
 	return 0;
 }
 
+static void
+mlx5_crypto_dev_stop(struct rte_cryptodev *dev)
+{
+	RTE_SET_USED(dev);
+}
+
+static int
+mlx5_crypto_dev_start(struct rte_cryptodev *dev)
+{
+	RTE_SET_USED(dev);
+	return 0;
+}
+
 static int
 mlx5_crypto_dev_close(struct rte_cryptodev *dev)
 {
@@ -360,8 +373,8 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 
 static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_configure			= mlx5_crypto_dev_configure,
-	.dev_start			= NULL,
-	.dev_stop			= NULL,
+	.dev_start			= mlx5_crypto_dev_start,
+	.dev_stop			= mlx5_crypto_dev_stop,
 	.dev_close			= mlx5_crypto_dev_close,
 	.dev_infos_get			= mlx5_crypto_dev_infos_get,
 	.stats_get			= NULL,
-- 
2.27.0

[dpdk-dev] [PATCH v8 07/16] crypto/mlx5: add memory region management

[Shiri Kuzin]

Mellanox user space drivers don't deal with physical addresses as part
of a memory protection mechanism.
The device translates the given virtual address to a physical address
using the given memory key as an address space identifier.
That's why any mbuf virtual address is moved directly to the HW
descriptor(WQE).

The mapping between the virtual address to the physical address is saved
in MR configured by the kernel to the HW.

Each MR has a key that should also be moved to the WQE by the SW.

When the SW sees an unmapped address, it extends the address range and
creates a MR using a system call.

Add memory region cache management:
	- 2 level cache per queue-pair - no locks.
	- 1 shared cache between all the queues using a lock.

Using this way, the MR key search per data-path address is optimized.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Signed-off-by: Michael Baum <michaelba@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 doc/guides/cryptodevs/mlx5.rst    |  6 +++
 drivers/crypto/mlx5/mlx5_crypto.c | 63 +++++++++++++++++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto.h |  3 ++
 3 files changed, 72 insertions(+)

diff --git a/doc/guides/cryptodevs/mlx5.rst b/doc/guides/cryptodevs/mlx5.rst
index dd1d1a615d..ece881220e 100644
--- a/doc/guides/cryptodevs/mlx5.rst
+++ b/doc/guides/cryptodevs/mlx5.rst
@@ -26,6 +26,12 @@ the MKEY is configured to perform crypto operations.
 
 The encryption does not require text to be aligned to the AES block size (128b).
 
+For security reasons and to increase robustness, this driver only deals with virtual
+memory addresses. The way resources allocations are handled by the kernel,
+combined with hardware specifications that allow handling virtual memory
+addresses directly, ensure that DPDK applications cannot access random
+physical memory (or memory that does not belong to the current process).
+
 The PMD uses libibverbs and libmlx5 to access the device firmware or to
 access the hardware components directly.
 There are different levels of objects and bypassing abilities.
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 37e66cf57b..38008dcb28 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -261,6 +261,7 @@ mlx5_crypto_queue_pair_release(struct rte_cryptodev *dev, uint16_t qp_id)
 		claim_zero(mlx5_glue->devx_umem_dereg(qp->umem_obj));
 	if (qp->umem_buf != NULL)
 		rte_free(qp->umem_buf);
+	mlx5_mr_btree_free(&qp->mr_ctrl.cache_bh);
 	mlx5_devx_cq_destroy(&qp->cq_obj);
 	rte_free(qp);
 	dev->data->queue_pairs[qp_id] = NULL;
@@ -342,6 +343,14 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 		DRV_LOG(ERR, "Failed to register QP umem.");
 		goto error;
 	}
+	if (mlx5_mr_btree_init(&qp->mr_ctrl.cache_bh, MLX5_MR_BTREE_CACHE_N,
+			       priv->dev_config.socket_id) != 0) {
+		DRV_LOG(ERR, "Cannot allocate MR Btree for qp %u.",
+			(uint32_t)qp_id);
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	qp->mr_ctrl.dev_gen_ptr = &priv->mr_scache.dev_gen;
 	attr.pd = priv->pdn;
 	attr.uar_index = mlx5_os_get_devx_uar_page_id(priv->uar);
 	attr.cqn = qp->cq_obj.cq->id;
@@ -448,6 +457,40 @@ mlx5_crypto_hw_global_prepare(struct mlx5_crypto_priv *priv)
 	return 0;
 }
 
+/**
+ * Callback for memory event.
+ *
+ * @param event_type
+ *   Memory event type.
+ * @param addr
+ *   Address of memory.
+ * @param len
+ *   Size of memory.
+ */
+static void
+mlx5_crypto_mr_mem_event_cb(enum rte_mem_event event_type, const void *addr,
+			    size_t len, void *arg __rte_unused)
+{
+	struct mlx5_crypto_priv *priv;
+
+	/* Must be called from the primary process. */
+	MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);
+	switch (event_type) {
+	case RTE_MEM_EVENT_FREE:
+		pthread_mutex_lock(&priv_list_lock);
+		/* Iterate all the existing mlx5 devices. */
+		TAILQ_FOREACH(priv, &mlx5_crypto_priv_list, next)
+			mlx5_free_mr_by_addr(&priv->mr_scache,
+					     priv->ctx->device->name,
+					     addr, len);
+		pthread_mutex_unlock(&priv_list_lock);
+		break;
+	case RTE_MEM_EVENT_ALLOC:
+	default:
+		break;
+	}
+}
+
 /**
  * DPDK callback to register a PCI device.
  *
@@ -530,6 +573,22 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 		claim_zero(mlx5_glue->close_device(priv->ctx));
 		return -1;
 	}
+	if (mlx5_mr_btree_init(&priv->mr_scache.cache,
+			     MLX5_MR_BTREE_CACHE_N * 2, rte_socket_id()) != 0) {
+		DRV_LOG(ERR, "Failed to allocate shared cache MR memory.");
+		mlx5_crypto_hw_global_release(priv);
+		rte_cryptodev_pmd_destroy(priv->crypto_dev);
+		claim_zero(mlx5_glue->close_device(priv->ctx));
+		rte_errno = ENOMEM;
+		return -rte_errno;
+	}
+	priv->mr_scache.reg_mr_cb = mlx5_common_verbs_reg_mr;
+	priv->mr_scache.dereg_mr_cb = mlx5_common_verbs_dereg_mr;
+	/* Register callback function for global shared MR cache management. */
+	if (TAILQ_EMPTY(&mlx5_crypto_priv_list))
+		rte_mem_event_callback_register("MLX5_MEM_EVENT_CB",
+						mlx5_crypto_mr_mem_event_cb,
+						NULL);
 	pthread_mutex_lock(&priv_list_lock);
 	TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next);
 	pthread_mutex_unlock(&priv_list_lock);
@@ -549,6 +608,10 @@ mlx5_crypto_pci_remove(struct rte_pci_device *pdev)
 		TAILQ_REMOVE(&mlx5_crypto_priv_list, priv, next);
 	pthread_mutex_unlock(&priv_list_lock);
 	if (priv) {
+		if (TAILQ_EMPTY(&mlx5_crypto_priv_list))
+			rte_mem_event_callback_unregister("MLX5_MEM_EVENT_CB",
+							  NULL);
+		mlx5_mr_release_cache(&priv->mr_scache);
 		mlx5_crypto_hw_global_release(priv);
 		rte_cryptodev_pmd_destroy(priv->crypto_dev);
 		claim_zero(mlx5_glue->close_device(priv->ctx));
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 1c3c4855ec..a9fbc0aa82 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -12,6 +12,7 @@
 
 #include <mlx5_common_utils.h>
 #include <mlx5_common_devx.h>
+#include <mlx5_common_mr.h>
 
 #define MLX5_CRYPTO_DEK_HTABLE_SZ (1 << 11)
 #define MLX5_CRYPTO_KEY_LENGTH 80
@@ -27,6 +28,7 @@ struct mlx5_crypto_priv {
 	struct ibv_pd *pd;
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
 	struct rte_cryptodev_config dev_config;
+	struct mlx5_mr_share_cache mr_scache; /* Global shared MR cache. */
 };
 
 struct mlx5_crypto_qp {
@@ -36,6 +38,7 @@ struct mlx5_crypto_qp {
 	void *umem_buf;
 	volatile uint32_t *db_rec;
 	struct rte_crypto_op **ops;
+	struct mlx5_mr_ctrl mr_ctrl;
 };
 
 struct mlx5_crypto_dek {
-- 
2.27.0

[dpdk-dev] [PATCH v8 08/16] crypto/mlx5: create login object using DevX

[Shiri Kuzin]

To work with crypto engines that are marked with wrapped_import_method,
a login session is required.
A crypto login object needs to be created using DevX.

The crypto login object contains:
	- The credential pointer.
	- The import_KEK pointer to be used for all secured information
	  communicated in crypto commands (key fields), including the
	  provided credential in this command.
	- The credential secret, wrapped by the import_KEK indicated in
	  this command. Size includes 8 bytes IV for wrapping.

Added devargs for the required login values:
	- wcs_file - path to the file containing the credential.
	- import_kek_id - the import KEK pointer.
	- credential_id - the credential pointer.

Create the login DevX object in pci_probe function and destroy it in
pci_remove.
Destroying the crypto login object means logout.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 doc/guides/cryptodevs/mlx5.rst    |  60 +++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto.c | 103 ++++++++++++++++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto.h |   7 ++
 3 files changed, 170 insertions(+)

diff --git a/doc/guides/cryptodevs/mlx5.rst b/doc/guides/cryptodevs/mlx5.rst
index ece881220e..6cae5affbd 100644
--- a/doc/guides/cryptodevs/mlx5.rst
+++ b/doc/guides/cryptodevs/mlx5.rst
@@ -44,6 +44,51 @@ To get the best performances:
 Enabling librte_crypto_mlx5 causes DPDK applications to be linked against
 libibverbs.
 
+In order to move the device to crypto operational mode, credential and KEK
+(Key Encrypting Key) should be set as the first step.
+The credential will be used by the software in order to perform crypto login, and the KEK is
+the AES Key Wrap Algorithm (rfc3394) key that will be used for sensitive data
+wrapping.
+The credential and the AES-XTS keys should be provided to the hardware, as ciphertext
+encrypted by the KEK.
+
+When crypto engines are defined to work in wrapped import method, they come out
+of the factory in Commissioning mode, and thus, cannot be used for crypto operations
+yet. A dedicated tool is used for changing the mode from Commissioning to
+Operational, while setting the first import_KEK and credential in plaintext.
+The mlxreg dedicated tool should be used as follows:
+
+- Set CRYPTO_OPERATIONAL register to set the device in crypto operational mode.
+
+  The input to this tool is:
+    The first credential in plaintext, 40B.
+    The first import_KEK in plaintext: kek size 0 for 16B or 1 for 32B, kek data.
+
+  Example:
+  mlxreg -d /dev/mst/mt4123_pciconf0 --reg_name CRYPTO_OPERATIONAL --get
+
+  The "wrapped_crypto_operational" value will be "0x00000000".
+  The command to set the register should be executed only once, and all the
+  values mentioned above should be specified in the same command.
+
+  Example:
+  mlxreg -d /dev/mst/mt4123_pciconf0 --reg_name CRYPTO_OPERATIONAL
+  --set "credential[0]=0x10000000, credential[1]=0x10000000, kek[0]=0x00000000"
+
+  All values not specified will remain 0.
+  "wrapped_crypto_going_to_commissioning" and  "wrapped_crypto_operational"
+  should not be specified.
+
+  All the device ports should set it in order to move to operational mode.
+
+- Query CRYPTO_OPERATIONAL register to make sure the device is in Operational
+  mode.
+
+  Example:
+  mlxreg -d /dev/mst/mt4123_pciconf0 --reg_name CRYPTO_OPERATIONAL --get
+  The "wrapped_crypto_operational" value will be "0x00000001" if the mode was
+  successfully changed to operational mode.
+
 
 Driver options
 --------------
@@ -53,6 +98,21 @@ Driver options
   Select the class of the driver that should probe the device.
   `crypto` for the mlx5 crypto driver.
 
+- ``wcs_file`` parameter [string] - mandatory
+
+  File path including only the wrapped credential in string format of hexadecimal
+  numbers, represent 48 bytes (8 bytes IV added by the AES key wrap algorithm).
+
+- ``import_kek_id`` parameter [int]
+
+  The identifier of the KEK, default value is 0 represents the operational
+  register import_kek..
+
+- ``credential_id`` parameter [int]
+
+  The identifier of the credential, default value is 0 represents the operational
+  register credential.
+
 
 Supported NICs
 --------------
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 38008dcb28..cbaa2e52ff 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -457,6 +457,101 @@ mlx5_crypto_hw_global_prepare(struct mlx5_crypto_priv *priv)
 	return 0;
 }
 
+
+static int
+mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque)
+{
+	struct mlx5_crypto_devarg_params *devarg_prms = opaque;
+	struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr;
+	unsigned long tmp;
+	FILE *file;
+	int ret;
+	int i;
+
+	if (strcmp(key, "class") == 0)
+		return 0;
+	if (strcmp(key, "wcs_file") == 0) {
+		file = fopen(val, "rb");
+		if (file == NULL) {
+			rte_errno = ENOTSUP;
+			return -rte_errno;
+		}
+		for (i = 0 ; i < MLX5_CRYPTO_CREDENTIAL_SIZE ; i++) {
+			ret = fscanf(file, "%02hhX", &attr->credential[i]);
+			if (ret <= 0) {
+				fclose(file);
+				DRV_LOG(ERR,
+					"Failed to read credential from file.");
+				rte_errno = EINVAL;
+				return -rte_errno;
+			}
+		}
+		fclose(file);
+		devarg_prms->login_devarg = true;
+		return 0;
+	}
+	errno = 0;
+	tmp = strtoul(val, NULL, 0);
+	if (errno) {
+		DRV_LOG(WARNING, "%s: \"%s\" is an invalid integer.", key, val);
+		return -errno;
+	}
+	if (strcmp(key, "import_kek_id") == 0)
+		attr->session_import_kek_ptr = (uint32_t)tmp;
+	else if (strcmp(key, "credential_id") == 0)
+		attr->credential_pointer = (uint32_t)tmp;
+	else
+		DRV_LOG(WARNING, "Invalid key %s.", key);
+	return 0;
+}
+
+static struct mlx5_devx_obj *
+mlx5_crypto_config_login(struct rte_devargs *devargs,
+			 struct ibv_context *ctx)
+{
+	/*
+	 * Set credential pointer and session import KEK pointer to a default
+	 * value of 0.
+	 */
+	struct mlx5_crypto_devarg_params login = {
+			.login_devarg = false,
+			.login_attr = {
+					.credential_pointer = 0,
+					.session_import_kek_ptr = 0,
+			}
+	};
+	struct rte_kvargs *kvlist;
+
+	if (devargs == NULL) {
+		DRV_LOG(ERR,
+	"No login devargs in order to enable crypto operations in the device.");
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	kvlist = rte_kvargs_parse(devargs->args, NULL);
+	if (kvlist == NULL) {
+		DRV_LOG(ERR, "Failed to parse devargs.");
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	if (rte_kvargs_process(kvlist, NULL, mlx5_crypto_args_check_handler,
+			   &login) != 0) {
+		DRV_LOG(ERR, "Devargs handler function Failed.");
+		rte_kvargs_free(kvlist);
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	rte_kvargs_free(kvlist);
+	if (login.login_devarg == false) {
+		DRV_LOG(ERR,
+	"No login credential devarg in order to enable crypto operations "
+	"in the device.");
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	return mlx5_devx_cmd_create_crypto_login_obj(ctx, &login.login_attr);
+}
+
 /**
  * Callback for memory event.
  *
@@ -512,6 +607,7 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	struct ibv_device *ibv;
 	struct rte_cryptodev *crypto_dev;
 	struct ibv_context *ctx;
+	struct mlx5_devx_obj *login;
 	struct mlx5_crypto_priv *priv;
 	struct mlx5_hca_attr attr = { 0 };
 	struct rte_cryptodev_pmd_init_params init_params = {
@@ -550,6 +646,11 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 		rte_errno = ENOTSUP;
 		return -ENOTSUP;
 	}
+	login = mlx5_crypto_config_login(pci_dev->device.devargs, ctx);
+	if (login == NULL) {
+		DRV_LOG(ERR, "Failed to configure login.");
+		return -rte_errno;
+	}
 	crypto_dev = rte_cryptodev_pmd_create(ibv->name, &pci_dev->device,
 					&init_params);
 	if (crypto_dev == NULL) {
@@ -566,6 +667,7 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	crypto_dev->driver_id = mlx5_crypto_driver_id;
 	priv = crypto_dev->data->dev_private;
 	priv->ctx = ctx;
+	priv->login_obj = login;
 	priv->pci_dev = pci_dev;
 	priv->crypto_dev = crypto_dev;
 	if (mlx5_crypto_hw_global_prepare(priv) != 0) {
@@ -614,6 +716,7 @@ mlx5_crypto_pci_remove(struct rte_pci_device *pdev)
 		mlx5_mr_release_cache(&priv->mr_scache);
 		mlx5_crypto_hw_global_release(priv);
 		rte_cryptodev_pmd_destroy(priv->crypto_dev);
+		claim_zero(mlx5_devx_cmd_destroy(priv->login_obj));
 		claim_zero(mlx5_glue->close_device(priv->ctx));
 	}
 	return 0;
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index a9fbc0aa82..b98c621c23 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -29,6 +29,7 @@ struct mlx5_crypto_priv {
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
 	struct rte_cryptodev_config dev_config;
 	struct mlx5_mr_share_cache mr_scache; /* Global shared MR cache. */
+	struct mlx5_devx_obj *login_obj;
 };
 
 struct mlx5_crypto_qp {
@@ -48,6 +49,12 @@ struct mlx5_crypto_dek {
 	bool size_is_48; /* Whether the key\data size is 48 bytes or not. */
 } __rte_cache_aligned;
 
+
+struct mlx5_crypto_devarg_params {
+	bool login_devarg;
+	struct mlx5_devx_crypto_login_attr login_attr;
+};
+
 int
 mlx5_crypto_dek_destroy(struct mlx5_crypto_priv *priv,
 			struct mlx5_crypto_dek *dek);
-- 
2.27.0

[dpdk-dev] [PATCH v8 09/16] crypto/mlx5: add keytag devarg

[Shiri Kuzin]

From: Suanming Mou <suanmingm@nvidia.com>

A keytag is a piece of data encrypted together with a DEK.

When a DEK is referenced by an MKEY.bsf through its index, the keytag is
also supplied in the BSF as plaintext. The HW will decrypt the DEK (and
the attached keytag) and will fail the operation if the keytags don't
match.

This commit adds the configuration of the keytag with devargs.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
 doc/guides/cryptodevs/mlx5.rst    |  7 +++++
 drivers/crypto/mlx5/mlx5_crypto.c | 50 +++++++++++++++++--------------
 drivers/crypto/mlx5/mlx5_crypto.h |  3 +-
 3 files changed, 37 insertions(+), 23 deletions(-)

diff --git a/doc/guides/cryptodevs/mlx5.rst b/doc/guides/cryptodevs/mlx5.rst
index 6cae5affbd..c3632484a5 100644
--- a/doc/guides/cryptodevs/mlx5.rst
+++ b/doc/guides/cryptodevs/mlx5.rst
@@ -52,6 +52,9 @@ wrapping.
 The credential and the AES-XTS keys should be provided to the hardware, as ciphertext
 encrypted by the KEK.
 
+A keytag (64 bits) should be appended to the AES-XTS keys (before wrapping),
+and will be validated when the hardware attempts to access it.
+
 When crypto engines are defined to work in wrapped import method, they come out
 of the factory in Commissioning mode, and thus, cannot be used for crypto operations
 yet. A dedicated tool is used for changing the mode from Commissioning to
@@ -113,6 +116,10 @@ Driver options
   The identifier of the credential, default value is 0 represents the operational
   register credential.
 
+- ``keytag`` parameter [int]
+
+  The plaintext of the keytag appanded to the AES-XTS keys, default value is 0.
+
 
 Supported NICs
 --------------
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index cbaa2e52ff..0f786ac4ca 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -500,56 +500,52 @@ mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque)
 		attr->session_import_kek_ptr = (uint32_t)tmp;
 	else if (strcmp(key, "credential_id") == 0)
 		attr->credential_pointer = (uint32_t)tmp;
+	else if (strcmp(key, "keytag") == 0)
+		devarg_prms->keytag = tmp;
 	else
 		DRV_LOG(WARNING, "Invalid key %s.", key);
 	return 0;
 }
 
-static struct mlx5_devx_obj *
-mlx5_crypto_config_login(struct rte_devargs *devargs,
-			 struct ibv_context *ctx)
+static int
+mlx5_crypto_parse_devargs(struct rte_devargs *devargs,
+			  struct mlx5_crypto_devarg_params *devarg_prms)
 {
-	/*
-	 * Set credential pointer and session import KEK pointer to a default
-	 * value of 0.
-	 */
-	struct mlx5_crypto_devarg_params login = {
-			.login_devarg = false,
-			.login_attr = {
-					.credential_pointer = 0,
-					.session_import_kek_ptr = 0,
-			}
-	};
+	struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr;
 	struct rte_kvargs *kvlist;
 
+	/* Default values. */
+	attr->credential_pointer = 0;
+	attr->session_import_kek_ptr = 0;
+	devarg_prms->keytag = 0;
 	if (devargs == NULL) {
 		DRV_LOG(ERR,
 	"No login devargs in order to enable crypto operations in the device.");
 		rte_errno = EINVAL;
-		return NULL;
+		return -1;
 	}
 	kvlist = rte_kvargs_parse(devargs->args, NULL);
 	if (kvlist == NULL) {
 		DRV_LOG(ERR, "Failed to parse devargs.");
 		rte_errno = EINVAL;
-		return NULL;
+		return -1;
 	}
 	if (rte_kvargs_process(kvlist, NULL, mlx5_crypto_args_check_handler,
-			   &login) != 0) {
+			   devarg_prms) != 0) {
 		DRV_LOG(ERR, "Devargs handler function Failed.");
 		rte_kvargs_free(kvlist);
 		rte_errno = EINVAL;
-		return NULL;
+		return -1;
 	}
 	rte_kvargs_free(kvlist);
-	if (login.login_devarg == false) {
+	if (devarg_prms->login_devarg == false) {
 		DRV_LOG(ERR,
 	"No login credential devarg in order to enable crypto operations "
 	"in the device.");
 		rte_errno = EINVAL;
-		return NULL;
+		return -1;
 	}
-	return mlx5_devx_cmd_create_crypto_login_obj(ctx, &login.login_attr);
+	return 0;
 }
 
 /**
@@ -609,6 +605,7 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	struct ibv_context *ctx;
 	struct mlx5_devx_obj *login;
 	struct mlx5_crypto_priv *priv;
+	struct mlx5_crypto_devarg_params devarg_prms = { 0 };
 	struct mlx5_hca_attr attr = { 0 };
 	struct rte_cryptodev_pmd_init_params init_params = {
 		.name = "",
@@ -617,6 +614,8 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 		.max_nb_queue_pairs =
 				RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
 	};
+	int ret;
+
 	RTE_SET_USED(pci_drv);
 	if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
 		DRV_LOG(ERR, "Non-primary process type is not supported.");
@@ -646,7 +645,13 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 		rte_errno = ENOTSUP;
 		return -ENOTSUP;
 	}
-	login = mlx5_crypto_config_login(pci_dev->device.devargs, ctx);
+	ret = mlx5_crypto_parse_devargs(pci_dev->device.devargs, &devarg_prms);
+	if (ret) {
+		DRV_LOG(ERR, "Failed to parse devargs.");
+		return -rte_errno;
+	}
+	login = mlx5_devx_cmd_create_crypto_login_obj(ctx,
+						      &devarg_prms.login_attr);
 	if (login == NULL) {
 		DRV_LOG(ERR, "Failed to configure login.");
 		return -rte_errno;
@@ -686,6 +691,7 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	}
 	priv->mr_scache.reg_mr_cb = mlx5_common_verbs_reg_mr;
 	priv->mr_scache.dereg_mr_cb = mlx5_common_verbs_dereg_mr;
+	priv->keytag = rte_cpu_to_be_64(devarg_prms.keytag);
 	/* Register callback function for global shared MR cache management. */
 	if (TAILQ_EMPTY(&mlx5_crypto_priv_list))
 		rte_mem_event_callback_register("MLX5_MEM_EVENT_CB",
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index b98c621c23..d374d3c4e9 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -30,6 +30,7 @@ struct mlx5_crypto_priv {
 	struct rte_cryptodev_config dev_config;
 	struct mlx5_mr_share_cache mr_scache; /* Global shared MR cache. */
 	struct mlx5_devx_obj *login_obj;
+	uint64_t keytag;
 };
 
 struct mlx5_crypto_qp {
@@ -49,10 +50,10 @@ struct mlx5_crypto_dek {
 	bool size_is_48; /* Whether the key\data size is 48 bytes or not. */
 } __rte_cache_aligned;
 
-
 struct mlx5_crypto_devarg_params {
 	bool login_devarg;
 	struct mlx5_devx_crypto_login_attr login_attr;
+	uint64_t keytag;
 };
 
 int
-- 
2.27.0

[dpdk-dev] [PATCH v8 10/16] crypto/mlx5: add maximum segments devarg

[Shiri Kuzin]

From: Suanming Mou <suanmingm@nvidia.com>

The mlx5 HW crypto operations are done by attaching crypto property
to a memory region. Once done, every access to the memory via the
crypto-enabled memory region will result with in-line encryption or
decryption of the data.

As a result, the design choice is to provide two types of WQEs. One
is UMR WQE which sets the crypto property and the other is rdma write
WQE which sends DMA command to copy data from local MR to remote MR.

The size of the WQEs will be defined by a new devarg called
max_segs_num.

This devarg also defines the maximum segments in mbuf chain that will be
supported for crypto operations.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
 doc/guides/cryptodevs/mlx5.rst    |  4 ++++
 drivers/crypto/mlx5/mlx5_crypto.c | 33 +++++++++++++++++++++++++++----
 drivers/crypto/mlx5/mlx5_crypto.h |  6 ++++++
 3 files changed, 39 insertions(+), 4 deletions(-)

diff --git a/doc/guides/cryptodevs/mlx5.rst b/doc/guides/cryptodevs/mlx5.rst
index c3632484a5..dd4705b744 100644
--- a/doc/guides/cryptodevs/mlx5.rst
+++ b/doc/guides/cryptodevs/mlx5.rst
@@ -120,6 +120,10 @@ Driver options
 
   The plaintext of the keytag appanded to the AES-XTS keys, default value is 0.
 
+- ``max_segs_num`` parameter [int]
+
+  Maximum number of mbuf chain segments(src or dest), default value is 8.
+
 
 Supported NICs
 --------------
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 0f786ac4ca..fc251f3f4c 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -21,6 +21,7 @@
 #define MLX5_CRYPTO_DRIVER_NAME mlx5_crypto
 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
 #define MLX5_CRYPTO_MAX_QPS 1024
+#define MLX5_CRYPTO_MAX_SEGS 56
 
 #define MLX5_CRYPTO_FEATURE_FLAGS \
 	(RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | \
@@ -496,14 +497,24 @@ mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque)
 		DRV_LOG(WARNING, "%s: \"%s\" is an invalid integer.", key, val);
 		return -errno;
 	}
-	if (strcmp(key, "import_kek_id") == 0)
+	if (strcmp(key, "max_segs_num") == 0) {
+		if (!tmp || tmp > MLX5_CRYPTO_MAX_SEGS) {
+			DRV_LOG(WARNING, "Invalid max_segs_num: %d, should"
+				" be less than %d.",
+				(uint32_t)tmp, MLX5_CRYPTO_MAX_SEGS);
+			rte_errno = EINVAL;
+			return -rte_errno;
+		}
+		devarg_prms->max_segs_num = (uint32_t)tmp;
+	} else if (strcmp(key, "import_kek_id") == 0) {
 		attr->session_import_kek_ptr = (uint32_t)tmp;
-	else if (strcmp(key, "credential_id") == 0)
+	} else if (strcmp(key, "credential_id") == 0) {
 		attr->credential_pointer = (uint32_t)tmp;
-	else if (strcmp(key, "keytag") == 0)
+	} else if (strcmp(key, "keytag") == 0) {
 		devarg_prms->keytag = tmp;
-	else
+	} else {
 		DRV_LOG(WARNING, "Invalid key %s.", key);
+	}
 	return 0;
 }
 
@@ -518,6 +529,7 @@ mlx5_crypto_parse_devargs(struct rte_devargs *devargs,
 	attr->credential_pointer = 0;
 	attr->session_import_kek_ptr = 0;
 	devarg_prms->keytag = 0;
+	devarg_prms->max_segs_num = 8;
 	if (devargs == NULL) {
 		DRV_LOG(ERR,
 	"No login devargs in order to enable crypto operations in the device.");
@@ -614,6 +626,7 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 		.max_nb_queue_pairs =
 				RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
 	};
+	uint16_t rdmw_wqe_size;
 	int ret;
 
 	RTE_SET_USED(pci_drv);
@@ -692,6 +705,18 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	priv->mr_scache.reg_mr_cb = mlx5_common_verbs_reg_mr;
 	priv->mr_scache.dereg_mr_cb = mlx5_common_verbs_dereg_mr;
 	priv->keytag = rte_cpu_to_be_64(devarg_prms.keytag);
+	priv->max_segs_num = devarg_prms.max_segs_num;
+	priv->umr_wqe_size = sizeof(struct mlx5_wqe_umr_bsf_seg) +
+			     sizeof(struct mlx5_umr_wqe) +
+			     RTE_ALIGN(priv->max_segs_num, 4) *
+			     sizeof(struct mlx5_wqe_dseg);
+	rdmw_wqe_size = sizeof(struct mlx5_rdma_write_wqe) +
+			      sizeof(struct mlx5_wqe_dseg) *
+			      (priv->max_segs_num <= 2 ? 2 : 2 +
+			       RTE_ALIGN(priv->max_segs_num - 2, 4));
+	priv->wqe_set_size = priv->umr_wqe_size + rdmw_wqe_size;
+	priv->umr_wqe_stride = priv->umr_wqe_size / MLX5_SEND_WQE_BB;
+	priv->max_rdmar_ds = rdmw_wqe_size / sizeof(struct mlx5_wqe_dseg);
 	/* Register callback function for global shared MR cache management. */
 	if (TAILQ_EMPTY(&mlx5_crypto_priv_list))
 		rte_mem_event_callback_register("MLX5_MEM_EVENT_CB",
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index d374d3c4e9..7a386fe76a 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -25,12 +25,17 @@ struct mlx5_crypto_priv {
 	struct rte_cryptodev *crypto_dev;
 	void *uar; /* User Access Region. */
 	uint32_t pdn; /* Protection Domain number. */
+	uint32_t max_segs_num; /* Maximum supported data segs. */
 	struct ibv_pd *pd;
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
 	struct rte_cryptodev_config dev_config;
 	struct mlx5_mr_share_cache mr_scache; /* Global shared MR cache. */
 	struct mlx5_devx_obj *login_obj;
 	uint64_t keytag;
+	uint16_t wqe_set_size;
+	uint16_t umr_wqe_size;
+	uint16_t umr_wqe_stride;
+	uint16_t max_rdmar_ds;
 };
 
 struct mlx5_crypto_qp {
@@ -54,6 +59,7 @@ struct mlx5_crypto_devarg_params {
 	bool login_devarg;
 	struct mlx5_devx_crypto_login_attr login_attr;
 	uint64_t keytag;
+	uint32_t max_segs_num;
 };
 
 int
-- 
2.27.0

[dpdk-dev] [PATCH v8 11/16] crypto/mlx5: add WQE set initialization

[Shiri Kuzin]

From: Suanming Mou <suanmingm@nvidia.com>

Currently, HW handles the WQEs much faster than the software,
Using the constant WQE set layout can initialize most of the WQE
segments in advanced, and software only needs to configure very
limited segments in datapath. This accelerates the software WQE
organize in datapath.

This commit initializes the fixed WQE set segments.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 83 +++++++++++++++++++++++++++++--
 drivers/crypto/mlx5/mlx5_crypto.h | 10 +++-
 2 files changed, 87 insertions(+), 6 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index fc251f3f4c..e5f8d96ff7 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -297,6 +297,69 @@ mlx5_crypto_qp2rts(struct mlx5_crypto_qp *qp)
 	return 0;
 }
 
+static void
+mlx5_crypto_qp_init(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp)
+{
+	uint32_t i;
+
+	for (i = 0 ; i < qp->entries_n; i++) {
+		struct mlx5_wqe_cseg *cseg = RTE_PTR_ADD(qp->umem_buf, i *
+							 priv->wqe_set_size);
+		struct mlx5_wqe_umr_cseg *ucseg = (struct mlx5_wqe_umr_cseg *)
+								     (cseg + 1);
+		struct mlx5_wqe_umr_bsf_seg *bsf =
+			(struct mlx5_wqe_umr_bsf_seg *)(RTE_PTR_ADD(cseg,
+						       priv->umr_wqe_size)) - 1;
+		struct mlx5_wqe_rseg *rseg;
+
+		/* Init UMR WQE. */
+		cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj->id << 8) |
+					 (priv->umr_wqe_size / MLX5_WSEG_SIZE));
+		cseg->flags = RTE_BE32(MLX5_COMP_ONLY_FIRST_ERR <<
+				       MLX5_COMP_MODE_OFFSET);
+		cseg->misc = rte_cpu_to_be_32(qp->mkey[i]->id);
+		ucseg->if_cf_toe_cq_res = RTE_BE32(1u << MLX5_UMRC_IF_OFFSET);
+		ucseg->mkey_mask = RTE_BE64(1u << 0); /* Mkey length bit. */
+		ucseg->ko_to_bs = rte_cpu_to_be_32
+			((RTE_ALIGN(priv->max_segs_num, 4u) <<
+			 MLX5_UMRC_KO_OFFSET) | (4 << MLX5_UMRC_TO_BS_OFFSET));
+		bsf->keytag = priv->keytag;
+		/* Init RDMA WRITE WQE. */
+		cseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size);
+		cseg->flags = RTE_BE32((MLX5_COMP_ALWAYS <<
+				      MLX5_COMP_MODE_OFFSET) |
+				      MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE);
+		rseg = (struct mlx5_wqe_rseg *)(cseg + 1);
+		rseg->rkey = rte_cpu_to_be_32(qp->mkey[i]->id);
+	}
+}
+
+static int
+mlx5_crypto_indirect_mkeys_prepare(struct mlx5_crypto_priv *priv,
+				  struct mlx5_crypto_qp *qp)
+{
+	struct mlx5_umr_wqe *umr;
+	uint32_t i;
+	struct mlx5_devx_mkey_attr attr = {
+		.pd = priv->pdn,
+		.umr_en = 1,
+		.crypto_en = 1,
+		.set_remote_rw = 1,
+		.klm_num = RTE_ALIGN(priv->max_segs_num, 4),
+	};
+
+	for (umr = (struct mlx5_umr_wqe *)qp->umem_buf, i = 0;
+	   i < qp->entries_n; i++, umr = RTE_PTR_ADD(umr, priv->wqe_set_size)) {
+		attr.klm_array = (struct mlx5_klm *)&umr->kseg[0];
+		qp->mkey[i] = mlx5_devx_cmd_mkey_create(priv->ctx, &attr);
+		if (!qp->mkey[i]) {
+			DRV_LOG(ERR, "Failed to allocate indirect mkey.");
+			return -1;
+		}
+	}
+	return 0;
+}
+
 static int
 mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 			     const struct rte_cryptodev_qp_conf *qp_conf,
@@ -307,7 +370,7 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	struct mlx5_crypto_qp *qp;
 	uint16_t log_nb_desc = rte_log2_u32(qp_conf->nb_descriptors);
 	uint32_t umem_size = RTE_BIT32(log_nb_desc) *
-			      MLX5_CRYPTO_WQE_SET_SIZE +
+			      priv->wqe_set_size +
 			      sizeof(*qp->db_rec) * 2;
 	uint32_t alloc_size = sizeof(*qp);
 	struct mlx5_devx_cq_attr cq_attr = {
@@ -317,7 +380,9 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	if (dev->data->queue_pairs[qp_id] != NULL)
 		mlx5_crypto_queue_pair_release(dev, qp_id);
 	alloc_size = RTE_ALIGN(alloc_size, RTE_CACHE_LINE_SIZE);
-	alloc_size += sizeof(struct rte_crypto_op *) * RTE_BIT32(log_nb_desc);
+	alloc_size += (sizeof(struct rte_crypto_op *) +
+		       sizeof(struct mlx5_devx_obj *)) *
+		       RTE_BIT32(log_nb_desc);
 	qp = rte_zmalloc_socket(__func__, alloc_size, RTE_CACHE_LINE_SIZE,
 				socket_id);
 	if (qp == NULL) {
@@ -362,8 +427,7 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	attr.wq_umem_id = qp->umem_obj->umem_id;
 	attr.wq_umem_offset = 0;
 	attr.dbr_umem_id = qp->umem_obj->umem_id;
-	attr.dbr_address = RTE_BIT64(log_nb_desc) *
-			   MLX5_CRYPTO_WQE_SET_SIZE;
+	attr.dbr_address = RTE_BIT64(log_nb_desc) * priv->wqe_set_size;
 	qp->qp_obj = mlx5_devx_cmd_create_qp(priv->ctx, &attr);
 	if (qp->qp_obj == NULL) {
 		DRV_LOG(ERR, "Failed to create QP(%u).", rte_errno);
@@ -372,8 +436,17 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	qp->db_rec = RTE_PTR_ADD(qp->umem_buf, (uintptr_t)attr.dbr_address);
 	if (mlx5_crypto_qp2rts(qp))
 		goto error;
-	qp->ops = (struct rte_crypto_op **)RTE_ALIGN((uintptr_t)(qp + 1),
+	qp->mkey = (struct mlx5_devx_obj **)RTE_ALIGN((uintptr_t)(qp + 1),
 							   RTE_CACHE_LINE_SIZE);
+	qp->ops = (struct rte_crypto_op **)(qp->mkey + RTE_BIT32(log_nb_desc));
+	qp->entries_n = 1 << log_nb_desc;
+	if (mlx5_crypto_indirect_mkeys_prepare(priv, qp)) {
+		DRV_LOG(ERR, "Cannot allocate indirect memory regions.");
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	mlx5_crypto_qp_init(priv, qp);
+	qp->priv = priv;
 	dev->data->queue_pairs[qp_id] = qp;
 	return 0;
 error:
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 7a386fe76a..d9b1ff8e99 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -16,7 +16,6 @@
 
 #define MLX5_CRYPTO_DEK_HTABLE_SZ (1 << 11)
 #define MLX5_CRYPTO_KEY_LENGTH 80
-#define MLX5_CRYPTO_WQE_SET_SIZE 1024
 
 struct mlx5_crypto_priv {
 	TAILQ_ENTRY(mlx5_crypto_priv) next;
@@ -24,6 +23,7 @@ struct mlx5_crypto_priv {
 	struct rte_pci_device *pci_dev;
 	struct rte_cryptodev *crypto_dev;
 	void *uar; /* User Access Region. */
+	volatile uint64_t *uar_addr;
 	uint32_t pdn; /* Protection Domain number. */
 	uint32_t max_segs_num; /* Maximum supported data segs. */
 	struct ibv_pd *pd;
@@ -39,13 +39,21 @@ struct mlx5_crypto_priv {
 };
 
 struct mlx5_crypto_qp {
+	struct mlx5_crypto_priv *priv;
 	struct mlx5_devx_cq cq_obj;
 	struct mlx5_devx_obj *qp_obj;
+	struct rte_cryptodev_stats stats;
 	struct mlx5dv_devx_umem *umem_obj;
 	void *umem_buf;
 	volatile uint32_t *db_rec;
 	struct rte_crypto_op **ops;
+	struct mlx5_devx_obj **mkey; /* WQE's indirect mekys. */
 	struct mlx5_mr_ctrl mr_ctrl;
+	uint8_t *wqe;
+	uint16_t entries_n;
+	uint16_t pi;
+	uint16_t ci;
+	uint16_t db_pi;
 };
 
 struct mlx5_crypto_dek {
-- 
2.27.0

[dpdk-dev] [PATCH v8 12/16] crypto/mlx5: add enqueue and dequeue operations

[Shiri Kuzin]

From: Suanming Mou <suanmingm@nvidia.com>

The crypto operations are done with the WQE set which contains
one UMR WQE and one rdma write WQE. Most segments of the WQE
set are initialized properly during queue setup, only limited
segments are initialized according to the crypto detail in the
datapath process.

This commit adds the enquue and dequeue operations and updates
the WQE set segments accordingly.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
Signed-off-by: Michael Baum <michaelba@nvidia.com>
---
 doc/guides/cryptodevs/features/mlx5.ini |   5 +
 drivers/crypto/mlx5/mlx5_crypto.c       | 286 +++++++++++++++++++++++-
 drivers/crypto/mlx5/mlx5_crypto.h       |   3 +
 3 files changed, 290 insertions(+), 4 deletions(-)

diff --git a/doc/guides/cryptodevs/features/mlx5.ini b/doc/guides/cryptodevs/features/mlx5.ini
index bd757b5211..a89526add0 100644
--- a/doc/guides/cryptodevs/features/mlx5.ini
+++ b/doc/guides/cryptodevs/features/mlx5.ini
@@ -6,6 +6,11 @@
 [Features]
 Symmetric crypto       = Y
 HW Accelerated         = Y
+In Place SGL           = Y
+OOP SGL In SGL Out     = Y
+OOP SGL In LB  Out     = Y
+OOP LB  In SGL Out     = Y
+OOP LB  In LB  Out     = Y
 Cipher multiple data units = Y
 Cipher wrapped key     = Y
 
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index e5f8d96ff7..b467739a3c 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -25,6 +25,10 @@
 
 #define MLX5_CRYPTO_FEATURE_FLAGS \
 	(RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | \
+	 RTE_CRYPTODEV_FF_IN_PLACE_SGL | RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT | \
+	 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | \
+	 RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT | \
+	 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | \
 	 RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY | \
 	 RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS)
 
@@ -297,6 +301,279 @@ mlx5_crypto_qp2rts(struct mlx5_crypto_qp *qp)
 	return 0;
 }
 
+static __rte_noinline uint32_t
+mlx5_crypto_get_block_size(struct rte_crypto_op *op)
+{
+	uint32_t bl = op->sym->cipher.data.length;
+
+	switch (bl) {
+	case (1 << 20):
+		return RTE_BE32(MLX5_BLOCK_SIZE_1MB << MLX5_BLOCK_SIZE_OFFSET);
+	case (1 << 12):
+		return RTE_BE32(MLX5_BLOCK_SIZE_4096B <<
+				MLX5_BLOCK_SIZE_OFFSET);
+	case (1 << 9):
+		return RTE_BE32(MLX5_BLOCK_SIZE_512B << MLX5_BLOCK_SIZE_OFFSET);
+	default:
+		DRV_LOG(ERR, "Unknown block size: %u.", bl);
+		return UINT32_MAX;
+	}
+}
+
+/**
+ * Query LKey from a packet buffer for QP. If not found, add the mempool.
+ *
+ * @param priv
+ *   Pointer to the priv object.
+ * @param addr
+ *   Search key.
+ * @param mr_ctrl
+ *   Pointer to per-queue MR control structure.
+ * @param ol_flags
+ *   Mbuf offload features.
+ *
+ * @return
+ *   Searched LKey on success, UINT32_MAX on no match.
+ */
+static __rte_always_inline uint32_t
+mlx5_crypto_addr2mr(struct mlx5_crypto_priv *priv, uintptr_t addr,
+		    struct mlx5_mr_ctrl *mr_ctrl, uint64_t ol_flags)
+{
+	uint32_t lkey;
+
+	/* Check generation bit to see if there's any change on existing MRs. */
+	if (unlikely(*mr_ctrl->dev_gen_ptr != mr_ctrl->cur_gen))
+		mlx5_mr_flush_local_cache(mr_ctrl);
+	/* Linear search on MR cache array. */
+	lkey = mlx5_mr_lookup_lkey(mr_ctrl->cache, &mr_ctrl->mru,
+				   MLX5_MR_CACHE_N, addr);
+	if (likely(lkey != UINT32_MAX))
+		return lkey;
+	/* Take slower bottom-half on miss. */
+	return mlx5_mr_addr2mr_bh(priv->pd, 0, &priv->mr_scache, mr_ctrl, addr,
+				  !!(ol_flags & EXT_ATTACHED_MBUF));
+}
+
+static __rte_always_inline uint32_t
+mlx5_crypto_klm_set(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp,
+		      struct rte_mbuf *mbuf, struct mlx5_wqe_dseg *klm,
+		      uint32_t offset, uint32_t *remain)
+{
+	uint32_t data_len = (rte_pktmbuf_data_len(mbuf) - offset);
+	uintptr_t addr = rte_pktmbuf_mtod_offset(mbuf, uintptr_t, offset);
+
+	if (data_len > *remain)
+		data_len = *remain;
+	*remain -= data_len;
+	klm->bcount = rte_cpu_to_be_32(data_len);
+	klm->pbuf = rte_cpu_to_be_64(addr);
+	klm->lkey = mlx5_crypto_addr2mr(priv, addr, &qp->mr_ctrl,
+					mbuf->ol_flags);
+	return klm->lkey;
+
+}
+
+static __rte_always_inline uint32_t
+mlx5_crypto_klms_set(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp,
+		     struct rte_crypto_op *op, struct rte_mbuf *mbuf,
+		     struct mlx5_wqe_dseg *klm)
+{
+	uint32_t remain_len = op->sym->cipher.data.length;
+	uint32_t nb_segs = mbuf->nb_segs;
+	uint32_t klm_n = 1u;
+
+	/* First mbuf needs to take the cipher offset. */
+	if (unlikely(mlx5_crypto_klm_set(priv, qp, mbuf, klm,
+		     op->sym->cipher.data.offset, &remain_len) == UINT32_MAX)) {
+		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return 0;
+	}
+	while (remain_len) {
+		nb_segs--;
+		mbuf = mbuf->next;
+		if (unlikely(mbuf == NULL || nb_segs == 0)) {
+			op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+			return 0;
+		}
+		if (unlikely(mlx5_crypto_klm_set(priv, qp, mbuf, ++klm, 0,
+						 &remain_len) == UINT32_MAX)) {
+			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			return 0;
+		}
+		klm_n++;
+	}
+	return klm_n;
+}
+
+static __rte_always_inline int
+mlx5_crypto_wqe_set(struct mlx5_crypto_priv *priv,
+			 struct mlx5_crypto_qp *qp,
+			 struct rte_crypto_op *op,
+			 struct mlx5_umr_wqe *umr)
+{
+	struct mlx5_crypto_session *sess = get_sym_session_private_data
+				(op->sym->session, mlx5_crypto_driver_id);
+	struct mlx5_wqe_cseg *cseg = &umr->ctr;
+	struct mlx5_wqe_mkey_cseg *mkc = &umr->mkc;
+	struct mlx5_wqe_dseg *klms = &umr->kseg[0];
+	struct mlx5_wqe_umr_bsf_seg *bsf = ((struct mlx5_wqe_umr_bsf_seg *)
+				      RTE_PTR_ADD(umr, priv->umr_wqe_size)) - 1;
+	uint32_t ds;
+	bool ipl = op->sym->m_dst == NULL || op->sym->m_dst == op->sym->m_src;
+	/* Set UMR WQE. */
+	uint32_t klm_n = mlx5_crypto_klms_set(priv, qp, op,
+				   ipl ? op->sym->m_src : op->sym->m_dst, klms);
+
+	if (unlikely(klm_n == 0))
+		return 0;
+	bsf->bs_bpt_eo_es = sess->bs_bpt_eo_es;
+	if (unlikely(!sess->bsp_res)) {
+		bsf->bsp_res = mlx5_crypto_get_block_size(op);
+		if (unlikely(bsf->bsp_res == UINT32_MAX)) {
+			op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+			return 0;
+		}
+	} else {
+		bsf->bsp_res = sess->bsp_res;
+	}
+	bsf->raw_data_size = rte_cpu_to_be_32(op->sym->cipher.data.length);
+	memcpy(bsf->xts_initial_tweak,
+	       rte_crypto_op_ctod_offset(op, uint8_t *, sess->iv_offset), 16);
+	bsf->res_dp = sess->dek_id;
+	mkc->len = rte_cpu_to_be_64(op->sym->cipher.data.length);
+	cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) | MLX5_OPCODE_UMR);
+	qp->db_pi += priv->umr_wqe_stride;
+	/* Set RDMA_WRITE WQE. */
+	cseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size);
+	klms = RTE_PTR_ADD(cseg, sizeof(struct mlx5_rdma_write_wqe));
+	if (!ipl) {
+		klm_n = mlx5_crypto_klms_set(priv, qp, op, op->sym->m_src,
+					     klms);
+		if (unlikely(klm_n == 0))
+			return 0;
+	} else {
+		memcpy(klms, &umr->kseg[0], sizeof(*klms) * klm_n);
+	}
+	ds = 2 + klm_n;
+	cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj->id << 8) | ds);
+	cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) |
+							MLX5_OPCODE_RDMA_WRITE);
+	ds = RTE_ALIGN(ds, 4);
+	qp->db_pi += ds >> 2;
+	/* Set NOP WQE if needed. */
+	if (priv->max_rdmar_ds > ds) {
+		cseg += ds;
+		ds = priv->max_rdmar_ds - ds;
+		cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj->id << 8) | ds);
+		cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) |
+							       MLX5_OPCODE_NOP);
+		qp->db_pi += ds >> 2; /* Here, DS is 4 aligned for sure. */
+	}
+	qp->wqe = (uint8_t *)cseg;
+	return 1;
+}
+
+static __rte_always_inline void
+mlx5_crypto_uar_write(uint64_t val, struct mlx5_crypto_priv *priv)
+{
+#ifdef RTE_ARCH_64
+	*priv->uar_addr = val;
+#else /* !RTE_ARCH_64 */
+	rte_spinlock_lock(&priv->uar32_sl);
+	*(volatile uint32_t *)priv->uar_addr = val;
+	rte_io_wmb();
+	*((volatile uint32_t *)priv->uar_addr + 1) = val >> 32;
+	rte_spinlock_unlock(&priv->uar32_sl);
+#endif
+}
+
+static uint16_t
+mlx5_crypto_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
+			  uint16_t nb_ops)
+{
+	struct mlx5_crypto_qp *qp = queue_pair;
+	struct mlx5_crypto_priv *priv = qp->priv;
+	struct mlx5_umr_wqe *umr;
+	struct rte_crypto_op *op;
+	uint16_t mask = qp->entries_n - 1;
+	uint16_t remain = qp->entries_n - (qp->pi - qp->ci);
+
+	if (remain < nb_ops)
+		nb_ops = remain;
+	else
+		remain = nb_ops;
+	if (unlikely(remain == 0))
+		return 0;
+	do {
+		op = *ops++;
+		umr = RTE_PTR_ADD(qp->umem_buf, priv->wqe_set_size * qp->pi);
+		if (unlikely(mlx5_crypto_wqe_set(priv, qp, op, umr) == 0)) {
+			if (remain != nb_ops)
+				break;
+			return 0;
+		}
+		qp->ops[qp->pi] = op;
+		qp->pi = (qp->pi + 1) & mask;
+	} while (--remain);
+	rte_io_wmb();
+	qp->db_rec[MLX5_SND_DBR] = rte_cpu_to_be_32(qp->db_pi);
+	rte_wmb();
+	mlx5_crypto_uar_write(*(volatile uint64_t *)qp->wqe, qp->priv);
+	rte_wmb();
+	return nb_ops;
+}
+
+static __rte_noinline void
+mlx5_crypto_cqe_err_handle(struct mlx5_crypto_qp *qp, struct rte_crypto_op *op)
+{
+	const uint32_t idx = qp->ci & (qp->entries_n - 1);
+	volatile struct mlx5_err_cqe *cqe = (volatile struct mlx5_err_cqe *)
+							&qp->cq_obj.cqes[idx];
+
+	op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+	DRV_LOG(ERR, "CQE ERR:%x.\n", rte_be_to_cpu_32(cqe->syndrome));
+}
+
+static uint16_t
+mlx5_crypto_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
+			  uint16_t nb_ops)
+{
+	struct mlx5_crypto_qp *qp = queue_pair;
+	volatile struct mlx5_cqe *restrict cqe;
+	struct rte_crypto_op *restrict op;
+	const unsigned int cq_size = qp->entries_n;
+	const unsigned int mask = cq_size - 1;
+	uint32_t idx;
+	uint32_t next_idx = qp->ci & mask;
+	const uint16_t max = RTE_MIN((uint16_t)(qp->pi - qp->ci), nb_ops);
+	uint16_t i = 0;
+	int ret;
+
+	if (unlikely(max == 0))
+		return 0;
+	do {
+		idx = next_idx;
+		next_idx = (qp->ci + 1) & mask;
+		op = qp->ops[idx];
+		cqe = &qp->cq_obj.cqes[idx];
+		ret = check_cqe(cqe, cq_size, qp->ci);
+		rte_io_rmb();
+		if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) {
+			if (unlikely(ret != MLX5_CQE_STATUS_HW_OWN))
+				mlx5_crypto_cqe_err_handle(qp, op);
+			break;
+		}
+		op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+		ops[i++] = op;
+		qp->ci++;
+	} while (i < max);
+	if (likely(i != 0)) {
+		rte_io_wmb();
+		qp->cq_obj.db_rec[0] = rte_cpu_to_be_32(qp->ci);
+	}
+	return i;
+}
+
 static void
 mlx5_crypto_qp_init(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp)
 {
@@ -521,8 +798,9 @@ mlx5_crypto_hw_global_prepare(struct mlx5_crypto_priv *priv)
 	if (mlx5_crypto_pd_create(priv) != 0)
 		return -1;
 	priv->uar = mlx5_devx_alloc_uar(priv->ctx, -1);
-	if (priv->uar == NULL || mlx5_os_get_devx_uar_reg_addr(priv->uar) ==
-	    NULL) {
+	if (priv->uar)
+		priv->uar_addr = mlx5_os_get_devx_uar_reg_addr(priv->uar);
+	if (priv->uar == NULL || priv->uar_addr == NULL) {
 		rte_errno = errno;
 		claim_zero(mlx5_glue->dealloc_pd(priv->pd));
 		DRV_LOG(ERR, "Failed to allocate UAR.");
@@ -752,8 +1030,8 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	DRV_LOG(INFO,
 		"Crypto device %s was created successfully.", ibv->name);
 	crypto_dev->dev_ops = &mlx5_crypto_ops;
-	crypto_dev->dequeue_burst = NULL;
-	crypto_dev->enqueue_burst = NULL;
+	crypto_dev->dequeue_burst = mlx5_crypto_dequeue_burst;
+	crypto_dev->enqueue_burst = mlx5_crypto_enqueue_burst;
 	crypto_dev->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
 	crypto_dev->driver_id = mlx5_crypto_driver_id;
 	priv = crypto_dev->data->dev_private;
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index d9b1ff8e99..1350513b9e 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -36,6 +36,9 @@ struct mlx5_crypto_priv {
 	uint16_t umr_wqe_size;
 	uint16_t umr_wqe_stride;
 	uint16_t max_rdmar_ds;
+#ifndef RTE_ARCH_64
+	rte_spinlock_t uar32_sl;
+#endif /* RTE_ARCH_64 */
 };
 
 struct mlx5_crypto_qp {
-- 
2.27.0

[dpdk-dev] [PATCH v8 13/16] crypto/mlx5: add statistic get and reset operations

[Shiri Kuzin]

From: Suanming Mou <suanmingm@nvidia.com>

This commit adds mlx5 crypto statistic get and reset operations.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 40 ++++++++++++++++++++++++++++---
 1 file changed, 37 insertions(+), 3 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index b467739a3c..0ecca32cce 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -508,13 +508,17 @@ mlx5_crypto_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
 		op = *ops++;
 		umr = RTE_PTR_ADD(qp->umem_buf, priv->wqe_set_size * qp->pi);
 		if (unlikely(mlx5_crypto_wqe_set(priv, qp, op, umr) == 0)) {
-			if (remain != nb_ops)
+			qp->stats.enqueue_err_count++;
+			if (remain != nb_ops) {
+				qp->stats.enqueued_count -= remain;
 				break;
+			}
 			return 0;
 		}
 		qp->ops[qp->pi] = op;
 		qp->pi = (qp->pi + 1) & mask;
 	} while (--remain);
+	qp->stats.enqueued_count += nb_ops;
 	rte_io_wmb();
 	qp->db_rec[MLX5_SND_DBR] = rte_cpu_to_be_32(qp->db_pi);
 	rte_wmb();
@@ -531,6 +535,7 @@ mlx5_crypto_cqe_err_handle(struct mlx5_crypto_qp *qp, struct rte_crypto_op *op)
 							&qp->cq_obj.cqes[idx];
 
 	op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+	qp->stats.dequeue_err_count++;
 	DRV_LOG(ERR, "CQE ERR:%x.\n", rte_be_to_cpu_32(cqe->syndrome));
 }
 
@@ -570,6 +575,7 @@ mlx5_crypto_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
 	if (likely(i != 0)) {
 		rte_io_wmb();
 		qp->cq_obj.db_rec[0] = rte_cpu_to_be_32(qp->ci);
+		qp->stats.dequeued_count += i;
 	}
 	return i;
 }
@@ -731,14 +737,42 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	return -1;
 }
 
+static void
+mlx5_crypto_stats_get(struct rte_cryptodev *dev,
+		      struct rte_cryptodev_stats *stats)
+{
+	int qp_id;
+
+	for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
+		struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
+
+		stats->enqueued_count += qp->stats.enqueued_count;
+		stats->dequeued_count += qp->stats.dequeued_count;
+		stats->enqueue_err_count += qp->stats.enqueue_err_count;
+		stats->dequeue_err_count += qp->stats.dequeue_err_count;
+	}
+}
+
+static void
+mlx5_crypto_stats_reset(struct rte_cryptodev *dev)
+{
+	int qp_id;
+
+	for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
+		struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
+
+		memset(&qp->stats, 0, sizeof(qp->stats));
+	}
+}
+
 static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_configure			= mlx5_crypto_dev_configure,
 	.dev_start			= mlx5_crypto_dev_start,
 	.dev_stop			= mlx5_crypto_dev_stop,
 	.dev_close			= mlx5_crypto_dev_close,
 	.dev_infos_get			= mlx5_crypto_dev_infos_get,
-	.stats_get			= NULL,
-	.stats_reset			= NULL,
+	.stats_get			= mlx5_crypto_stats_get,
+	.stats_reset			= mlx5_crypto_stats_reset,
 	.queue_pair_setup		= mlx5_crypto_queue_pair_setup,
 	.queue_pair_release		= mlx5_crypto_queue_pair_release,
 	.sym_session_get_size		= mlx5_crypto_sym_session_get_size,
-- 
2.27.0

[dpdk-dev] [PATCH v8 14/16] test/crypto: add mlx5 crypto driver

[Shiri Kuzin]

In order to test the new mlx5 crypto PMD, the driver is added to the
crypto test application.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 app/test/test_cryptodev.c      | 7 +++++++
 app/test/test_cryptodev.h      | 2 ++
 doc/guides/cryptodevs/mlx5.rst | 3 +++
 3 files changed, 12 insertions(+)

diff --git a/app/test/test_cryptodev.c b/app/test/test_cryptodev.c
index 31201d93e1..e46668a1e0 100644
--- a/app/test/test_cryptodev.c
+++ b/app/test/test_cryptodev.c
@@ -14569,6 +14569,12 @@ test_cryptodev_cpu_aesni_gcm(void)
 	return rc;
 }
 
+static int
+test_cryptodev_mlx5(void /*argv __rte_unused, int argc __rte_unused*/)
+{
+	return run_cryptodev_testsuite(RTE_STR(CRYPTODEV_NAME_MLX5_PMD));
+}
+
 static int
 test_cryptodev_null(void)
 {
@@ -14797,6 +14803,7 @@ REGISTER_TEST_COMMAND(cryptodev_openssl_autotest, test_cryptodev_openssl);
 REGISTER_TEST_COMMAND(cryptodev_aesni_gcm_autotest, test_cryptodev_aesni_gcm);
 REGISTER_TEST_COMMAND(cryptodev_cpu_aesni_gcm_autotest,
 	test_cryptodev_cpu_aesni_gcm);
+REGISTER_TEST_COMMAND(cryptodev_mlx5_autotest, test_cryptodev_mlx5);
 REGISTER_TEST_COMMAND(cryptodev_null_autotest, test_cryptodev_null);
 REGISTER_TEST_COMMAND(cryptodev_sw_snow3g_autotest, test_cryptodev_sw_snow3g);
 REGISTER_TEST_COMMAND(cryptodev_sw_kasumi_autotest, test_cryptodev_sw_kasumi);
diff --git a/app/test/test_cryptodev.h b/app/test/test_cryptodev.h
index 5bf1e8818b..262e545ee6 100644
--- a/app/test/test_cryptodev.h
+++ b/app/test/test_cryptodev.h
@@ -73,6 +73,8 @@
 #define CRYPTODEV_NAME_BCMFS_PMD	crypto_bcmfs
 #define CRYPTODEV_NAME_CN9K_PMD		crypto_cn9k
 #define CRYPTODEV_NAME_CN10K_PMD	crypto_cn10k
+#define CRYPTODEV_NAME_MLX5_PMD		mlx5_crypto
+
 
 enum cryptodev_api_test_type {
 	CRYPTODEV_API_TEST = 0,
diff --git a/doc/guides/cryptodevs/mlx5.rst b/doc/guides/cryptodevs/mlx5.rst
index dd4705b744..2023dd1b3b 100644
--- a/doc/guides/cryptodevs/mlx5.rst
+++ b/doc/guides/cryptodevs/mlx5.rst
@@ -92,6 +92,9 @@ The mlxreg dedicated tool should be used as follows:
   The "wrapped_crypto_operational" value will be "0x00000001" if the mode was
   successfully changed to operational mode.
 
+  The mlx5 crypto PMD can be verfied by running the test application:
+  ./build/app/test/dpdk-test -c 1 -n 1 -w <dev>,class=crypto,wcs_file=<file_path>
+  RTE>>cryptodev_mlx5_autotest
 
 Driver options
 --------------
-- 
2.27.0

[dpdk-dev] [PATCH v8 15/16] test/crypto: add data-unit and wrapped vectors

[Shiri Kuzin]

The AES-XTS algorithm supports using a wrapped key.
In AES-XTS the data-unit defines the data block size to be
encrypted\decrypted.

Add AES-XTS vectors with a wrapped key.
Add a variable stating whether the key is wrapped or not.
Add the AES-XTS data-unit.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 app/test/test_cryptodev.h                  |    2 +-
 app/test/test_cryptodev_aes_test_vectors.h | 1340 ++++++++++++++++++++
 app/test/test_cryptodev_blockcipher.c      |   10 +-
 app/test/test_cryptodev_blockcipher.h      |    2 +
 4 files changed, 1352 insertions(+), 2 deletions(-)

diff --git a/app/test/test_cryptodev.h b/app/test/test_cryptodev.h
index 262e545ee6..f35dba6837 100644
--- a/app/test/test_cryptodev.h
+++ b/app/test/test_cryptodev.h
@@ -19,7 +19,7 @@
 #define DEFAULT_NUM_XFORMS              (2)
 #define NUM_MBUFS                       (8191)
 #define MBUF_CACHE_SIZE                 (256)
-#define MBUF_DATAPAYLOAD_SIZE		(2048 + DIGEST_BYTE_LENGTH_SHA512)
+#define MBUF_DATAPAYLOAD_SIZE		(4096 + DIGEST_BYTE_LENGTH_SHA512)
 #define MBUF_SIZE			(sizeof(struct rte_mbuf) + \
 		RTE_PKTMBUF_HEADROOM + MBUF_DATAPAYLOAD_SIZE)
 
diff --git a/app/test/test_cryptodev_aes_test_vectors.h b/app/test/test_cryptodev_aes_test_vectors.h
index e404050701..dd7ffde685 100644
--- a/app/test/test_cryptodev_aes_test_vectors.h
+++ b/app/test/test_cryptodev_aes_test_vectors.h
@@ -2609,6 +2609,1298 @@ blockcipher_test_data aes_test_data_xts_key_64_pt_48 = {
 	}
 };
 
+static const uint8_t plaintext_aes256xts_512bytes[] = {
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79
+};
+static const uint8_t ciphertext_aes256xts_512bytes[] = {
+	0xD3, 0x94, 0x3E, 0xC9, 0xD3, 0x43, 0x13, 0xD3,
+	0x07, 0x7E, 0x51, 0x11, 0x97, 0xA5, 0xB1, 0xB2,
+	0xB0, 0x55, 0xF8, 0xD6, 0xA2, 0x10, 0x78, 0x76,
+	0x37, 0x95, 0x4F, 0x25, 0x99, 0xCC, 0x12, 0xD6,
+	0xBB, 0x2A, 0x24, 0xF4, 0x2D, 0x82, 0xFF, 0x56,
+	0xEC, 0x55, 0x3B, 0xDD, 0xF9, 0xE1, 0xC8, 0x48,
+	0x68, 0x16, 0xD2, 0x2B, 0x95, 0x7C, 0x1F, 0xCB,
+	0x32, 0xD6, 0x8F, 0x9A, 0x2E, 0xF7, 0x9B, 0xBE,
+	0x72, 0xFC, 0x55, 0x33, 0x27, 0x21, 0x2F, 0x69,
+	0xCA, 0x76, 0xA5, 0x9B, 0x21, 0x2E, 0x40, 0x57,
+	0x2C, 0x1C, 0x98, 0x41, 0x9A, 0x5E, 0x55, 0x38,
+	0xDE, 0xC2, 0x09, 0x57, 0x32, 0xA3, 0x34, 0x40,
+	0xC1, 0x8D, 0xCF, 0x70, 0x15, 0xF2, 0x6F, 0x4F,
+	0x6A, 0x04, 0xA6, 0x6D, 0xFF, 0x53, 0x25, 0x6A,
+	0x0E, 0xD0, 0x87, 0x63, 0xA1, 0x6C, 0xB1, 0x99,
+	0x4A, 0x42, 0xF5, 0xF6, 0xEA, 0xA4, 0xEB, 0x6D,
+	0x70, 0x9B, 0x0F, 0x85, 0xE2, 0x43, 0x6C, 0x27,
+	0x54, 0x57, 0x52, 0x1C, 0xCF, 0x72, 0x11, 0x83,
+	0xC3, 0xF5, 0xC6, 0xB6, 0x07, 0xEC, 0x1A, 0xF5,
+	0xAC, 0xA1, 0xF2, 0x3A, 0x01, 0x53, 0x0C, 0xA4,
+	0x40, 0x19, 0xAF, 0x7B, 0x2D, 0xE7, 0x56, 0x8B,
+	0x71, 0x5F, 0x8D, 0x96, 0xF1, 0x5D, 0x33, 0xAC,
+	0xE7, 0xAD, 0x96, 0xDB, 0xBB, 0xF7, 0xF5, 0x3F,
+	0x0E, 0x02, 0x2E, 0x80, 0xE9, 0xA2, 0x30, 0xD2,
+	0x82, 0x65, 0xEC, 0x1A, 0xB2, 0xE8, 0x98, 0x23,
+	0x42, 0x79, 0x43, 0x85, 0x3B, 0xF9, 0xFB, 0xBB,
+	0xC5, 0x29, 0xAD, 0x95, 0xAE, 0x22, 0x5B, 0x26,
+	0xDF, 0x76, 0x65, 0x37, 0x17, 0xBC, 0x58, 0xBB,
+	0x1B, 0x0A, 0x71, 0xDC, 0x68, 0xFF, 0x90, 0x36,
+	0x37, 0xB7, 0x49, 0x5E, 0x49, 0x4E, 0xE4, 0x1E,
+	0x24, 0x39, 0x48, 0xC7, 0x68, 0x19, 0xED, 0x94,
+	0xB5, 0xF6, 0x85, 0xFA, 0xE6, 0xB8, 0x2B, 0x9E,
+	0x07, 0x9E, 0xFB, 0x1D, 0x61, 0x06, 0x47, 0x2A,
+	0x3E, 0x1E, 0xD8, 0x52, 0xFB, 0xB7, 0xE3, 0xB4,
+	0x0D, 0xA8, 0x15, 0x1E, 0x98, 0x02, 0xBD, 0x89,
+	0x89, 0xE3, 0x38, 0x2C, 0xAB, 0x50, 0x25, 0x30,
+	0xB4, 0x5E, 0xA5, 0xCD, 0xA8, 0x9B, 0xA4, 0x2A,
+	0xED, 0x19, 0x3B, 0xC3, 0x05, 0x07, 0x57, 0xB5,
+	0x52, 0x11, 0x74, 0x95, 0x51, 0x5A, 0xD8, 0xED,
+	0xDF, 0x37, 0x91, 0x83, 0x27, 0xD5, 0x08, 0x82,
+	0xB3, 0x42, 0x08, 0xC7, 0x81, 0x35, 0x5F, 0x58,
+	0x28, 0x69, 0x0D, 0x97, 0x1D, 0x28, 0xE6, 0xB0,
+	0x58, 0x93, 0xCE, 0x2A, 0xB4, 0x7D, 0x4B, 0x83,
+	0x20, 0x1E, 0x08, 0xEF, 0x68, 0x51, 0xEB, 0xB4,
+	0xFA, 0x78, 0xB5, 0xE5, 0x2D, 0x93, 0x07, 0x99,
+	0xBB, 0xDD, 0x9A, 0x4E, 0xD6, 0xE7, 0x31, 0x9F,
+	0x4D, 0xB4, 0x05, 0x45, 0x89, 0x59, 0x42, 0xFF,
+	0x0C, 0xD4, 0xC2, 0xF3, 0xA0, 0xF7, 0xD3, 0xCE,
+	0x26, 0x2F, 0x9B, 0xCB, 0x98, 0x24, 0xDE, 0x50,
+	0xF6, 0x81, 0x24, 0xA3, 0xB4, 0x99, 0x94, 0xDB,
+	0xF5, 0xF4, 0x74, 0xAC, 0xED, 0xF1, 0xD6, 0x2F,
+	0x89, 0x48, 0x41, 0x24, 0xBB, 0xA7, 0x64, 0x48,
+	0xAE, 0x36, 0xE3, 0xF3, 0xA8, 0x37, 0x4F, 0xE6,
+	0x24, 0x92, 0xC4, 0x27, 0x86, 0x68, 0xC4, 0xA2,
+	0x2F, 0xB2, 0xA8, 0xFF, 0x27, 0x5F, 0x59, 0xB9,
+	0x86, 0x89, 0x5A, 0x09, 0xB8, 0xDA, 0xD6, 0xBA,
+	0x26, 0x86, 0x51, 0xC2, 0xDF, 0xCF, 0x07, 0x49,
+	0x1C, 0xB7, 0x1A, 0x80, 0xD4, 0x97, 0x13, 0xAA,
+	0xE8, 0x38, 0xA7, 0x2C, 0xB7, 0xE2, 0xE0, 0x33,
+	0xF9, 0xC3, 0x21, 0xF6, 0x2A, 0xAA, 0x1A, 0xF7,
+	0x39, 0x1F, 0x2F, 0x7E, 0x38, 0xC8, 0x82, 0x3F,
+	0xCC, 0xD8, 0xE2, 0xD9, 0x02, 0xE6, 0x8E, 0x31,
+	0xA7, 0x37, 0xE8, 0x59, 0x3F, 0xE1, 0x60, 0x1C,
+	0x74, 0x52, 0xE4, 0x49, 0xF7, 0x21, 0x60, 0x11
+};
+static const struct
+blockcipher_test_data aes_test_data_xts_wrapped_key_48_pt_512_du_512 = {
+	.crypto_algo = RTE_CRYPTO_CIPHER_AES_XTS,
+	.cipher_key = {
+		.data = {
+			0x94, 0xD8, 0xAE, 0xD7, 0x74, 0xBF, 0x51, 0xF1,
+			0x2D, 0x19, 0x0B, 0x9A, 0xB3, 0xDC, 0x1E, 0xA9,
+			0x5C, 0xC5, 0x47, 0xC1, 0xBB, 0x94, 0xDB, 0xFA,
+			0x2B, 0x1E, 0x54, 0xA5, 0x76, 0xAD, 0x07, 0x4B,
+			0x46, 0xE0, 0xAA, 0xF9, 0xF7, 0x5C, 0x64, 0x2D,
+			0x8F, 0x04, 0x17, 0x72, 0xE5, 0x8D, 0xFE, 0x9C
+		},
+		.len = 48
+	},
+	.iv = {
+		.data = {
+			0x4F, 0xAE, 0xF7, 0x11, 0x7C, 0xDA, 0x59, 0xC6,
+			0x6E, 0x4B, 0x92, 0x01, 0x3E, 0x76, 0x8A, 0xD5
+		},
+		.len = 16
+	},
+	.plaintext = {
+		.data = plaintext_aes256xts_512bytes,
+		.len = 512
+	},
+	.ciphertext = {
+		.data = ciphertext_aes256xts_512bytes,
+		.len = 512
+	},
+	.xts_dataunit_len = 512,
+	.wrapped_key = true
+};
+static const uint8_t plaintext_aes256xts_4096bytes[] = {
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79
+};
+static const uint8_t ciphertext_aes256xts_4096bytes[] = {
+	0xD3, 0x94, 0x3E, 0xC9, 0xD3, 0x43, 0x13, 0xD3,
+	0x07, 0x7E, 0x51, 0x11, 0x97, 0xA5, 0xB1, 0xB2,
+	0xB0, 0x55, 0xF8, 0xD6, 0xA2, 0x10, 0x78, 0x76,
+	0x37, 0x95, 0x4F, 0x25, 0x99, 0xCC, 0x12, 0xD6,
+	0xBB, 0x2A, 0x24, 0xF4, 0x2D, 0x82, 0xFF, 0x56,
+	0xEC, 0x55, 0x3B, 0xDD, 0xF9, 0xE1, 0xC8, 0x48,
+	0x68, 0x16, 0xD2, 0x2B, 0x95, 0x7C, 0x1F, 0xCB,
+	0x32, 0xD6, 0x8F, 0x9A, 0x2E, 0xF7, 0x9B, 0xBE,
+	0x72, 0xFC, 0x55, 0x33, 0x27, 0x21, 0x2F, 0x69,
+	0xCA, 0x76, 0xA5, 0x9B, 0x21, 0x2E, 0x40, 0x57,
+	0x2C, 0x1C, 0x98, 0x41, 0x9A, 0x5E, 0x55, 0x38,
+	0xDE, 0xC2, 0x09, 0x57, 0x32, 0xA3, 0x34, 0x40,
+	0xC1, 0x8D, 0xCF, 0x70, 0x15, 0xF2, 0x6F, 0x4F,
+	0x6A, 0x04, 0xA6, 0x6D, 0xFF, 0x53, 0x25, 0x6A,
+	0x0E, 0xD0, 0x87, 0x63, 0xA1, 0x6C, 0xB1, 0x99,
+	0x4A, 0x42, 0xF5, 0xF6, 0xEA, 0xA4, 0xEB, 0x6D,
+	0x70, 0x9B, 0x0F, 0x85, 0xE2, 0x43, 0x6C, 0x27,
+	0x54, 0x57, 0x52, 0x1C, 0xCF, 0x72, 0x11, 0x83,
+	0xC3, 0xF5, 0xC6, 0xB6, 0x07, 0xEC, 0x1A, 0xF5,
+	0xAC, 0xA1, 0xF2, 0x3A, 0x01, 0x53, 0x0C, 0xA4,
+	0x40, 0x19, 0xAF, 0x7B, 0x2D, 0xE7, 0x56, 0x8B,
+	0x71, 0x5F, 0x8D, 0x96, 0xF1, 0x5D, 0x33, 0xAC,
+	0xE7, 0xAD, 0x96, 0xDB, 0xBB, 0xF7, 0xF5, 0x3F,
+	0x0E, 0x02, 0x2E, 0x80, 0xE9, 0xA2, 0x30, 0xD2,
+	0x82, 0x65, 0xEC, 0x1A, 0xB2, 0xE8, 0x98, 0x23,
+	0x42, 0x79, 0x43, 0x85, 0x3B, 0xF9, 0xFB, 0xBB,
+	0xC5, 0x29, 0xAD, 0x95, 0xAE, 0x22, 0x5B, 0x26,
+	0xDF, 0x76, 0x65, 0x37, 0x17, 0xBC, 0x58, 0xBB,
+	0x1B, 0x0A, 0x71, 0xDC, 0x68, 0xFF, 0x90, 0x36,
+	0x37, 0xB7, 0x49, 0x5E, 0x49, 0x4E, 0xE4, 0x1E,
+	0x24, 0x39, 0x48, 0xC7, 0x68, 0x19, 0xED, 0x94,
+	0xB5, 0xF6, 0x85, 0xFA, 0xE6, 0xB8, 0x2B, 0x9E,
+	0x07, 0x9E, 0xFB, 0x1D, 0x61, 0x06, 0x47, 0x2A,
+	0x3E, 0x1E, 0xD8, 0x52, 0xFB, 0xB7, 0xE3, 0xB4,
+	0x0D, 0xA8, 0x15, 0x1E, 0x98, 0x02, 0xBD, 0x89,
+	0x89, 0xE3, 0x38, 0x2C, 0xAB, 0x50, 0x25, 0x30,
+	0xB4, 0x5E, 0xA5, 0xCD, 0xA8, 0x9B, 0xA4, 0x2A,
+	0xED, 0x19, 0x3B, 0xC3, 0x05, 0x07, 0x57, 0xB5,
+	0x52, 0x11, 0x74, 0x95, 0x51, 0x5A, 0xD8, 0xED,
+	0xDF, 0x37, 0x91, 0x83, 0x27, 0xD5, 0x08, 0x82,
+	0xB3, 0x42, 0x08, 0xC7, 0x81, 0x35, 0x5F, 0x58,
+	0x28, 0x69, 0x0D, 0x97, 0x1D, 0x28, 0xE6, 0xB0,
+	0x58, 0x93, 0xCE, 0x2A, 0xB4, 0x7D, 0x4B, 0x83,
+	0x20, 0x1E, 0x08, 0xEF, 0x68, 0x51, 0xEB, 0xB4,
+	0xFA, 0x78, 0xB5, 0xE5, 0x2D, 0x93, 0x07, 0x99,
+	0xBB, 0xDD, 0x9A, 0x4E, 0xD6, 0xE7, 0x31, 0x9F,
+	0x4D, 0xB4, 0x05, 0x45, 0x89, 0x59, 0x42, 0xFF,
+	0x0C, 0xD4, 0xC2, 0xF3, 0xA0, 0xF7, 0xD3, 0xCE,
+	0x26, 0x2F, 0x9B, 0xCB, 0x98, 0x24, 0xDE, 0x50,
+	0xF6, 0x81, 0x24, 0xA3, 0xB4, 0x99, 0x94, 0xDB,
+	0xF5, 0xF4, 0x74, 0xAC, 0xED, 0xF1, 0xD6, 0x2F,
+	0x89, 0x48, 0x41, 0x24, 0xBB, 0xA7, 0x64, 0x48,
+	0xAE, 0x36, 0xE3, 0xF3, 0xA8, 0x37, 0x4F, 0xE6,
+	0x24, 0x92, 0xC4, 0x27, 0x86, 0x68, 0xC4, 0xA2,
+	0x2F, 0xB2, 0xA8, 0xFF, 0x27, 0x5F, 0x59, 0xB9,
+	0x86, 0x89, 0x5A, 0x09, 0xB8, 0xDA, 0xD6, 0xBA,
+	0x26, 0x86, 0x51, 0xC2, 0xDF, 0xCF, 0x07, 0x49,
+	0x1C, 0xB7, 0x1A, 0x80, 0xD4, 0x97, 0x13, 0xAA,
+	0xE8, 0x38, 0xA7, 0x2C, 0xB7, 0xE2, 0xE0, 0x33,
+	0xF9, 0xC3, 0x21, 0xF6, 0x2A, 0xAA, 0x1A, 0xF7,
+	0x39, 0x1F, 0x2F, 0x7E, 0x38, 0xC8, 0x82, 0x3F,
+	0xCC, 0xD8, 0xE2, 0xD9, 0x02, 0xE6, 0x8E, 0x31,
+	0xA7, 0x37, 0xE8, 0x59, 0x3F, 0xE1, 0x60, 0x1C,
+	0x74, 0x52, 0xE4, 0x49, 0xF7, 0x21, 0x60, 0x11,
+	0x8F, 0xD0, 0xA1, 0x4B, 0xE3, 0xF1, 0x18, 0x3D,
+	0x90, 0x89, 0x54, 0x27, 0xA0, 0xF9, 0x32, 0x09,
+	0x3D, 0x9D, 0x9A, 0x09, 0x53, 0xC6, 0x7E, 0x95,
+	0x85, 0x53, 0x98, 0x4C, 0x23, 0xEA, 0x54, 0xBD,
+	0x6F, 0x50, 0xBC, 0x4C, 0xCF, 0x37, 0xC5, 0x7B,
+	0x4B, 0xCE, 0x84, 0xAF, 0xE2, 0xE2, 0x55, 0x49,
+	0xBC, 0xBF, 0x92, 0xCA, 0x1E, 0x5E, 0x10, 0xDF,
+	0x60, 0x87, 0x09, 0xA1, 0x4C, 0x1D, 0x7E, 0x1E,
+	0x59, 0xE9, 0xCF, 0xDA, 0x45, 0x3F, 0xE2, 0x0F,
+	0x53, 0x8D, 0x8B, 0x79, 0xBD, 0xD8, 0xB0, 0xE3,
+	0x5B, 0x7C, 0x55, 0x4A, 0x84, 0xF0, 0x1E, 0xF9,
+	0xE8, 0xF3, 0x09, 0x4D, 0x0B, 0xD7, 0x77, 0xCC,
+	0x3F, 0x70, 0x22, 0x7D, 0x17, 0x27, 0x48, 0x57,
+	0xE2, 0x36, 0xA0, 0x84, 0x3B, 0xDE, 0x05, 0x34,
+	0xEF, 0x55, 0x12, 0xF4, 0x9A, 0x99, 0x0D, 0x28,
+	0x86, 0x28, 0x99, 0x6B, 0x22, 0xEE, 0x63, 0xF0,
+	0x68, 0x9C, 0xE1, 0x70, 0xF6, 0x26, 0xD8, 0x3B,
+	0xF9, 0x57, 0x18, 0x3D, 0xAD, 0x66, 0xF0, 0xCF,
+	0x7B, 0x0C, 0x28, 0x4D, 0xB8, 0xEB, 0x7B, 0x04,
+	0x1E, 0x7D, 0x40, 0x5F, 0x5A, 0x1E, 0x7E, 0x08,
+	0x7F, 0x4C, 0x1E, 0x18, 0xE5, 0x3E, 0x6C, 0x90,
+	0x3C, 0x89, 0x13, 0x2A, 0xC4, 0x2A, 0x94, 0xB5,
+	0x3E, 0x18, 0x1C, 0x4C, 0xBA, 0xEA, 0x86, 0xD2,
+	0x05, 0xA9, 0x59, 0x9C, 0x80, 0xC2, 0x45, 0xAD,
+	0x30, 0x99, 0x18, 0x6A, 0x2F, 0x73, 0x8C, 0xF0,
+	0xFE, 0xA4, 0xBD, 0x44, 0x3E, 0xEB, 0x98, 0x75,
+	0x48, 0x08, 0x57, 0x45, 0xD8, 0x41, 0xDE, 0x61,
+	0x6D, 0x06, 0x93, 0xC4, 0x99, 0x1B, 0x23, 0xB5,
+	0x12, 0x22, 0x5C, 0xC7, 0x9E, 0x18, 0xEA, 0x64,
+	0xDB, 0xCE, 0x1A, 0xAC, 0x5D, 0x9B, 0x80, 0xE2,
+	0xBF, 0x3E, 0xC2, 0xA4, 0x78, 0x4F, 0xF1, 0xE3,
+	0x7D, 0x2A, 0x20, 0x94, 0x13, 0xCD, 0xF3, 0x1C,
+	0x33, 0x9C, 0xC1, 0x59, 0x85, 0x52, 0xCB, 0xDB,
+	0x03, 0xDF, 0x11, 0xE6, 0xAB, 0x95, 0x82, 0x65,
+	0x7A, 0x88, 0x73, 0xEE, 0xBA, 0x21, 0x1C, 0x2F,
+	0xCD, 0xD7, 0xC6, 0xE5, 0x13, 0xDE, 0x7A, 0x9E,
+	0xEE, 0x83, 0x8D, 0xC6, 0x47, 0x63, 0xE0, 0xC7,
+	0xC4, 0xBE, 0x19, 0x25, 0xEE, 0xCC, 0x0A, 0x13,
+	0x18, 0x9D, 0x34, 0x5B, 0x55, 0x6C, 0xC1, 0x6E,
+	0xBF, 0x5A, 0xC5, 0x61, 0x75, 0x77, 0x49, 0x8C,
+	0x67, 0x61, 0xE8, 0x72, 0x87, 0xE8, 0xCA, 0xBE,
+	0x6E, 0xC3, 0xD7, 0x81, 0x8C, 0x78, 0x79, 0xC8,
+	0x72, 0xDA, 0x1A, 0x40, 0x7D, 0x60, 0xE2, 0x5A,
+	0x47, 0x38, 0xA1, 0x21, 0x22, 0x6B, 0x54, 0x74,
+	0xDD, 0xF3, 0xBC, 0x96, 0x28, 0x7E, 0xC2, 0x8B,
+	0x13, 0xEE, 0x53, 0xBC, 0x34, 0x67, 0x07, 0x87,
+	0xD5, 0x6B, 0x93, 0x22, 0x21, 0xB9, 0xED, 0x17,
+	0xE4, 0xA1, 0x96, 0xB2, 0xC6, 0xFF, 0x79, 0xA0,
+	0xA7, 0xF9, 0xDD, 0x92, 0x78, 0xF7, 0xE3, 0x16,
+	0x79, 0xEF, 0xEF, 0x31, 0x4D, 0x1D, 0x75, 0xF9,
+	0xCF, 0x5A, 0x1B, 0x68, 0x16, 0x7F, 0xAF, 0x5F,
+	0x30, 0xB7, 0xEF, 0xF8, 0x94, 0x63, 0x73, 0x3D,
+	0xB3, 0x63, 0xE4, 0xE6, 0xD8, 0xAD, 0xF4, 0x80,
+	0x5E, 0x82, 0xA3, 0xFB, 0x3D, 0x0F, 0xCF, 0x59,
+	0xB8, 0x76, 0xAF, 0x27, 0x83, 0xE3, 0x2D, 0x6F,
+	0xE0, 0xF3, 0x11, 0xD5, 0xAE, 0x82, 0x14, 0x1D,
+	0x78, 0x95, 0xBF, 0x31, 0x22, 0x1B, 0x80, 0x12,
+	0x02, 0xD7, 0x4D, 0x1B, 0x92, 0xE3, 0x15, 0xBA,
+	0x67, 0xD6, 0x8F, 0xD4, 0xDA, 0xBF, 0xD4, 0x62,
+	0xAD, 0x76, 0xDA, 0x04, 0xA2, 0xEA, 0x98, 0xD3,
+	0xC3, 0x6E, 0x5F, 0x26, 0x3C, 0x5E, 0xD9, 0xEA,
+	0x09, 0xF0, 0x02, 0xFC, 0xD2, 0x11, 0xF8, 0xA8,
+	0x7E, 0xFF, 0x06, 0x28, 0x5B, 0xE5, 0x6F, 0x9A,
+	0x00, 0xE7, 0x7F, 0xB9, 0xFB, 0x59, 0xBB, 0xDD,
+	0x85, 0xF3, 0x40, 0xCE, 0xA3, 0x5E, 0x2E, 0x2E,
+	0x34, 0x91, 0x58, 0x41, 0x00, 0xB5, 0xE1, 0x88,
+	0x24, 0x51, 0xC7, 0xB6, 0xF3, 0x21, 0x52, 0x6E,
+	0xE7, 0xFC, 0x42, 0xE7, 0x9C, 0xCC, 0x1E, 0x51,
+	0x45, 0x39, 0xBE, 0x09, 0xFE, 0x1A, 0xC4, 0xF0,
+	0x79, 0xF4, 0x05, 0xC7, 0xA7, 0xF8, 0x0F, 0xB6,
+	0x5A, 0x7B, 0xD7, 0xE1, 0x6F, 0xF0, 0x9D, 0x67,
+	0xA3, 0xE3, 0x3E, 0x2E, 0xB9, 0x8C, 0x83, 0x9E,
+	0xFD, 0x2E, 0xA2, 0x92, 0x99, 0x3C, 0xC0, 0x99,
+	0x01, 0xAB, 0x0D, 0xFA, 0x55, 0x96, 0x04, 0x60,
+	0x1A, 0xAD, 0x4C, 0xBB, 0x3D, 0xBB, 0x7D, 0x8B,
+	0x9F, 0x28, 0x85, 0x7D, 0xB9, 0xE4, 0x05, 0x79,
+	0x7B, 0x63, 0xDD, 0x7F, 0x4D, 0xE7, 0x50, 0xD9,
+	0x41, 0xFF, 0x53, 0xB1, 0xCE, 0x42, 0x7B, 0xD6,
+	0x05, 0x1B, 0x4E, 0xAF, 0xC4, 0x8C, 0x17, 0xC8,
+	0x52, 0xBD, 0x03, 0x3B, 0x92, 0x57, 0x4E, 0xA8,
+	0x15, 0xC3, 0x26, 0x1C, 0x55, 0xC1, 0xFF, 0xAE,
+	0xA9, 0x26, 0x2D, 0xA7, 0x8E, 0x3A, 0x7F, 0xA3,
+	0x48, 0xA5, 0xBC, 0x14, 0x84, 0xF2, 0x90, 0xCE,
+	0x35, 0x0F, 0x64, 0x6B, 0xD8, 0x1C, 0x12, 0xFE,
+	0x5A, 0x4F, 0x0E, 0xCE, 0x81, 0x4E, 0x79, 0x6B,
+	0xCF, 0x56, 0xA7, 0xDB, 0x24, 0xBC, 0xB0, 0x84,
+	0x4C, 0xB0, 0xDA, 0xBE, 0xE6, 0x8F, 0xD7, 0x8E,
+	0x0E, 0xA0, 0xD3, 0x55, 0xC2, 0x4A, 0x34, 0x1C,
+	0xF9, 0xC7, 0x3D, 0x29, 0x70, 0x8B, 0xF0, 0x99,
+	0x61, 0xF5, 0x11, 0xFB, 0x82, 0xE2, 0x67, 0x35,
+	0x60, 0x78, 0x47, 0x81, 0x2A, 0x74, 0x5E, 0x4D,
+	0x48, 0xD3, 0x7C, 0x32, 0xCA, 0x1B, 0xD2, 0xA2,
+	0x5C, 0x3A, 0x2F, 0xCE, 0xB4, 0x6C, 0x3A, 0x6A,
+	0x8F, 0x67, 0x46, 0x12, 0xE7, 0xAE, 0x6A, 0x3B,
+	0x99, 0x04, 0x5E, 0x96, 0xD0, 0xB9, 0x84, 0xF6,
+	0xA7, 0x64, 0x11, 0xE8, 0x0C, 0x51, 0xFD, 0x3F,
+	0x18, 0xFA, 0xE8, 0x52, 0xD9, 0x4B, 0x99, 0x7A,
+	0x25, 0x2B, 0x1B, 0x21, 0xAD, 0x8C, 0xFE, 0x0D,
+	0x34, 0x51, 0x91, 0x75, 0x55, 0x6F, 0xEB, 0x9F,
+	0x42, 0xDC, 0x73, 0x7D, 0x31, 0x0A, 0x74, 0x13,
+	0x80, 0xB8, 0xC3, 0xED, 0x73, 0x9D, 0x79, 0x42,
+	0xC0, 0x33, 0xAB, 0xC1, 0xCB, 0xB9, 0xD0, 0xBE,
+	0xA0, 0x78, 0xB8, 0x3B, 0xEB, 0x3D, 0x1A, 0x3F,
+	0xFB, 0x9B, 0xAA, 0x8F, 0x89, 0xF9, 0xD1, 0x22,
+	0x82, 0xE6, 0x66, 0xEE, 0x2A, 0xFD, 0x9F, 0xF8,
+	0x92, 0x7E, 0x10, 0xF5, 0xD5, 0x23, 0x0B, 0xB1,
+	0xD6, 0xF1, 0x7A, 0x3D, 0x73, 0xE9, 0xCE, 0x7F,
+	0xE6, 0x0B, 0x17, 0xBC, 0x23, 0xAE, 0x72, 0xB6,
+	0xFA, 0x19, 0x46, 0xBB, 0xFE, 0xA4, 0xC1, 0x64,
+	0xA8, 0x5E, 0xE8, 0xBB, 0x63, 0x58, 0x19, 0x50,
+	0xAA, 0x36, 0xC2, 0x4B, 0x38, 0x24, 0xD1, 0x2E,
+	0xAE, 0xAD, 0x6E, 0x34, 0x64, 0xA8, 0xC8, 0xF2,
+	0x4E, 0x74, 0x5C, 0x98, 0xE8, 0xDF, 0x99, 0x8C,
+	0x41, 0x79, 0x60, 0x2D, 0xD5, 0xF4, 0xE3, 0xE9,
+	0x1D, 0xF6, 0x5A, 0xA9, 0x69, 0x8E, 0xA1, 0x4F,
+	0xD3, 0x1B, 0x09, 0xA8, 0x7A, 0xD1, 0xE1, 0xCF,
+	0xAC, 0xBA, 0xD0, 0xD1, 0x34, 0x34, 0x8F, 0xC1,
+	0xD1, 0xA8, 0xAF, 0x6E, 0x92, 0xE0, 0xB0, 0xF6,
+	0xF9, 0x08, 0xA0, 0xCB, 0x58, 0x34, 0xF6, 0x68,
+	0xA2, 0xBF, 0x05, 0x39, 0x63, 0xBA, 0x4F, 0xEF,
+	0xE3, 0x95, 0x69, 0xD5, 0x89, 0x7C, 0x64, 0x07,
+	0x13, 0x42, 0x14, 0xF1, 0xA6, 0x9B, 0x87, 0xE5,
+	0xF4, 0x49, 0xAE, 0x67, 0x65, 0xCC, 0xF2, 0x26,
+	0xF8, 0x31, 0xBD, 0x33, 0x6A, 0x87, 0x77, 0x4E,
+	0xB1, 0xEE, 0xA4, 0xA2, 0xC8, 0xA0, 0x4A, 0xC1,
+	0xDF, 0x55, 0xE0, 0xDE, 0x53, 0x15, 0x3B, 0xEC,
+	0x55, 0x32, 0xCA, 0x06, 0xE4, 0x78, 0x59, 0x63,
+	0x10, 0x68, 0xA9, 0x46, 0x1B, 0xEF, 0x73, 0x6D,
+	0x1A, 0x02, 0x64, 0x12, 0x76, 0x9B, 0xDB, 0x7C,
+	0x03, 0x35, 0x19, 0xE1, 0x58, 0x7A, 0x87, 0x0C,
+	0x76, 0xDC, 0xFC, 0xC0, 0x28, 0xE4, 0xA2, 0x07,
+	0x9C, 0x28, 0x05, 0x21, 0x13, 0x58, 0xEF, 0x05,
+	0xBB, 0xAB, 0x94, 0xA2, 0x93, 0xBC, 0x31, 0x61,
+	0x26, 0x39, 0x38, 0x0C, 0xC4, 0x67, 0xDA, 0xA5,
+	0xE4, 0x1E, 0x1B, 0xB6, 0xE5, 0x73, 0xD6, 0x6C,
+	0xEE, 0xBC, 0x9D, 0xB9, 0xE7, 0xD9, 0x45, 0x2F,
+	0xF2, 0xB6, 0x92, 0x54, 0x41, 0x05, 0xB7, 0xB7,
+	0xFC, 0x37, 0x63, 0x6A, 0xB4, 0xBE, 0xB8, 0x3E,
+	0xD8, 0x53, 0x3B, 0xF8, 0x7D, 0x9A, 0x05, 0xDF,
+	0x20, 0x02, 0x27, 0x64, 0x38, 0xFA, 0x7D, 0xAF,
+	0x7F, 0xFA, 0xD1, 0xB7, 0x32, 0xC5, 0x74, 0x3E,
+	0x04, 0xA2, 0x67, 0x79, 0x02, 0x2E, 0x6F, 0xA1,
+	0x27, 0x87, 0x07, 0xB5, 0x9F, 0x0A, 0x7D, 0x5E,
+	0x14, 0xA0, 0x31, 0x46, 0x3F, 0xA9, 0xDE, 0x98,
+	0xB9, 0x89, 0xA0, 0x4A, 0x7A, 0xBD, 0x15, 0xAE,
+	0x2D, 0x0B, 0x38, 0x9A, 0xD8, 0x0E, 0xD2, 0xBA,
+	0x6D, 0xA1, 0x04, 0x1E, 0x4E, 0x39, 0x87, 0x4B,
+	0xC8, 0x3C, 0x74, 0x35, 0x4D, 0xC8, 0x1B, 0x42,
+	0x06, 0x5B, 0x73, 0xB7, 0x33, 0x86, 0x4A, 0x10,
+	0x2A, 0x10, 0x16, 0x28, 0x6F, 0x2A, 0xE3, 0x86,
+	0xDE, 0xA3, 0x44, 0x23, 0xE2, 0x90, 0xC4, 0x20,
+	0x90, 0xE0, 0xB8, 0xE6, 0xA7, 0xB6, 0xD6, 0x92,
+	0xF4, 0xF8, 0x8A, 0xBC, 0xAC, 0x31, 0x47, 0x8F,
+	0xAA, 0xE0, 0xD9, 0xF7, 0xE3, 0xCB, 0x11, 0xA4,
+	0x6B, 0x05, 0xB3, 0xB8, 0x72, 0x69, 0xE6, 0xDD,
+	0x75, 0x0F, 0x20, 0x1D, 0x3F, 0xC6, 0x96, 0xA0,
+	0x18, 0xB6, 0x24, 0xA1, 0xA6, 0xFD, 0x0C, 0x80,
+	0x1E, 0xD2, 0x28, 0xA2, 0x1A, 0x27, 0xF4, 0x23,
+	0x59, 0x1A, 0xCC, 0x0F, 0xD4, 0x99, 0xD0, 0xB4,
+	0x1E, 0x91, 0xC7, 0xD8, 0x8F, 0x8C, 0x5B, 0xEB,
+	0xB5, 0x9F, 0xFF, 0x4F, 0xD0, 0xD5, 0xB7, 0x60,
+	0xCC, 0x0A, 0x10, 0x38, 0xBF, 0xA8, 0x2E, 0xCC,
+	0xEB, 0x26, 0xB0, 0x78, 0xB3, 0xE0, 0x40, 0xAF,
+	0xCD, 0x12, 0xC5, 0x3A, 0x24, 0xD8, 0xEE, 0x3A,
+	0x64, 0x83, 0x2E, 0xD9, 0x25, 0x21, 0x66, 0xA5,
+	0x28, 0xD1, 0xE1, 0x84, 0x25, 0x1B, 0x20, 0xB8,
+	0xF5, 0x76, 0xB6, 0x3E, 0x4B, 0xC6, 0xEC, 0xC1,
+	0xC7, 0xAC, 0xC4, 0xAD, 0xCE, 0xF0, 0xB4, 0x0F,
+	0x35, 0x1E, 0xCE, 0x4E, 0xE3, 0x57, 0x30, 0xFC,
+	0xF4, 0x9B, 0x86, 0xB0, 0xDD, 0x3F, 0x2F, 0xB6,
+	0x10, 0x20, 0xE4, 0x24, 0x17, 0x1C, 0x24, 0xC6,
+	0x89, 0xE4, 0x14, 0xAD, 0x2E, 0x41, 0x08, 0x33,
+	0x88, 0xB1, 0x6F, 0x11, 0x85, 0xAF, 0x58, 0x17,
+	0xE3, 0x91, 0xB4, 0x72, 0xA2, 0x7F, 0xA3, 0x98,
+	0xAF, 0xB7, 0x6B, 0x58, 0x76, 0xA3, 0x11, 0x1C,
+	0x8A, 0x1A, 0xE6, 0x58, 0x54, 0xB0, 0xB9, 0x6E,
+	0x46, 0xCB, 0x16, 0xC0, 0x63, 0x0C, 0xEE, 0xA2,
+	0xAE, 0xF6, 0x71, 0xEF, 0xD1, 0xB9, 0x3D, 0xB7,
+	0x76, 0xCE, 0x5B, 0x84, 0x66, 0x7C, 0x7D, 0xF1,
+	0x96, 0x60, 0x34, 0xF6, 0xD1, 0x64, 0x27, 0xD9,
+	0xF3, 0x78, 0x8B, 0xF4, 0xC3, 0x1D, 0x37, 0xC0,
+	0xF4, 0x4A, 0xD0, 0xA5, 0x9A, 0xEB, 0xDD, 0x79,
+	0x54, 0x5D, 0xEB, 0x04, 0xC1, 0xA4, 0xBC, 0xED,
+	0xE3, 0x74, 0xC3, 0xB9, 0x9A, 0x6A, 0xAA, 0x06,
+	0xD1, 0xF0, 0x0F, 0xC5, 0xEF, 0x7E, 0x0B, 0xC8,
+	0xF4, 0x94, 0x4E, 0x69, 0x0E, 0x36, 0x00, 0x13,
+	0x45, 0xCE, 0x68, 0x13, 0xFE, 0x7F, 0x29, 0xA2,
+	0x1D, 0x79, 0xDF, 0xF2, 0x27, 0xFB, 0xAE, 0x52,
+	0x05, 0x78, 0xD7, 0xB9, 0xF7, 0x38, 0x68, 0xD5,
+	0xBA, 0xD7, 0xCF, 0x09, 0xC6, 0xD2, 0x5B, 0xC6,
+	0x98, 0xE4, 0xEC, 0xD5, 0xE9, 0xC2, 0xA5, 0x1A,
+	0x52, 0xC8, 0xA7, 0xBA, 0x3D, 0x74, 0x75, 0x00,
+	0xAA, 0xDD, 0x6A, 0x3F, 0xB6, 0x2F, 0x08, 0xB7,
+	0x1C, 0x6B, 0x52, 0x0C, 0xC9, 0xE4, 0x4D, 0xF4,
+	0xC5, 0x26, 0x1F, 0x35, 0x41, 0x25, 0x68, 0x17,
+	0xA8, 0x81, 0x75, 0xF4, 0x66, 0x41, 0xB5, 0xE4,
+	0x1D, 0x92, 0xEE, 0xDA, 0x0F, 0x56, 0x76, 0xC6,
+	0xAA, 0x0F, 0xA8, 0x63, 0x8D, 0xF0, 0x69, 0x63,
+	0x93, 0x45, 0xBC, 0x76, 0x40, 0xBE, 0xA9, 0x96,
+	0x36, 0xAF, 0x2F, 0x6B, 0x3E, 0xAB, 0xF3, 0xC0,
+	0xD7, 0xD5, 0xB1, 0x23, 0x23, 0xA2, 0xA0, 0xC4,
+	0xC5, 0x70, 0xEF, 0x66, 0x79, 0x15, 0xF0, 0xD0,
+	0x40, 0x0A, 0x33, 0x0C, 0xF3, 0x32, 0x6D, 0x8D,
+	0xB4, 0x44, 0x46, 0x78, 0x3F, 0x8D, 0x75, 0x40,
+	0xA5, 0x60, 0xBC, 0x9B, 0x76, 0xDF, 0x25, 0xF4,
+	0xE9, 0xED, 0xAC, 0x74, 0x2F, 0x9A, 0x00, 0xC4,
+	0x2B, 0x52, 0x26, 0x79, 0x09, 0x19, 0x57, 0x89,
+	0x60, 0x14, 0xBE, 0x65, 0xBD, 0x7B, 0x4D, 0x7D,
+	0x9B, 0x8B, 0x9E, 0x72, 0x6C, 0x0C, 0x57, 0xC7,
+	0x00, 0x08, 0x38, 0x7C, 0x37, 0x45, 0x9D, 0x55,
+	0xA2, 0x62, 0x5E, 0x34, 0x19, 0x99, 0x31, 0x16,
+	0xF1, 0x14, 0x44, 0x2D, 0xE3, 0x7E, 0x22, 0xE1,
+	0xA2, 0xB8, 0x9A, 0x9F, 0xE0, 0x37, 0x29, 0xBB,
+	0xCD, 0x46, 0xEE, 0x0A, 0x62, 0x2B, 0x98, 0x34,
+	0xBA, 0x9E, 0x54, 0x1B, 0xB1, 0x5C, 0x4F, 0xE9,
+	0xAA, 0xE4, 0x95, 0x8C, 0xA4, 0xEF, 0xC2, 0xB1,
+	0x7F, 0xF9, 0x80, 0xDA, 0x55, 0x95, 0x92, 0xC0,
+	0x86, 0xF4, 0x2D, 0x99, 0x3E, 0x17, 0xDC, 0x55,
+	0xA6, 0x33, 0x85, 0x90, 0x31, 0xC8, 0xFF, 0x58,
+	0x83, 0xC5, 0xBA, 0x60, 0x20, 0x5F, 0x87, 0x29,
+	0x20, 0x5A, 0x7D, 0x44, 0x2B, 0xA0, 0xE2, 0x99,
+	0xC8, 0x70, 0xBE, 0x89, 0xC5, 0xBC, 0xF6, 0x0D,
+	0x04, 0xC0, 0x96, 0xD1, 0x5C, 0xD1, 0x90, 0x43,
+	0xD3, 0x7B, 0x73, 0x52, 0x30, 0xB6, 0xA9, 0x7C,
+	0x0A, 0xA3, 0x24, 0x0E, 0x80, 0xFE, 0xBE, 0x31,
+	0xFD, 0xB5, 0x96, 0x04, 0x2B, 0xCF, 0x0B, 0x28,
+	0x1F, 0x7A, 0xCF, 0xC4, 0x82, 0x78, 0x52, 0x30,
+	0xB1, 0x34, 0x12, 0x50, 0x03, 0x09, 0x1C, 0x8B,
+	0x80, 0x60, 0xE3, 0xA1, 0xE5, 0x61, 0xF7, 0xD7,
+	0xB6, 0x74, 0xBE, 0xD6, 0x58, 0x03, 0xD3, 0xE5,
+	0xF7, 0xAC, 0x07, 0x60, 0xB7, 0x8A, 0xEC, 0xFA,
+	0xC6, 0x0F, 0xF0, 0x20, 0x04, 0x6B, 0x8F, 0x61,
+	0x09, 0x92, 0x03, 0xFB, 0x85, 0x99, 0x94, 0x9D,
+	0x2E, 0x6A, 0xC2, 0x9F, 0x20, 0x46, 0x2A, 0x96,
+	0xED, 0x42, 0x7D, 0x64, 0xA9, 0xE4, 0x1B, 0xDE,
+	0x11, 0x20, 0x12, 0x93, 0xE6, 0x2B, 0xE5, 0x93,
+	0x48, 0x37, 0x8C, 0x5A, 0x54, 0x0D, 0xEB, 0xF0,
+	0x9F, 0x9D, 0xE4, 0xA5, 0xC4, 0x93, 0x6F, 0x6A,
+	0xE3, 0x99, 0x69, 0xD9, 0xFE, 0x0C, 0x4E, 0xEC,
+	0x8B, 0x30, 0x1F, 0x7A, 0xB8, 0xC8, 0x5B, 0x61,
+	0x8E, 0xC2, 0x10, 0x90, 0x57, 0xB4, 0x72, 0x58,
+	0x7F, 0x41, 0x29, 0x7E, 0xF9, 0xBE, 0x40, 0xC3,
+	0x6F, 0xA9, 0xE3, 0x00, 0xE9, 0xC8, 0xFD, 0x4B,
+	0xFD, 0x3F, 0xE3, 0x3F, 0x25, 0x22, 0xFD, 0xB7,
+	0x2D, 0x57, 0xEF, 0x91, 0x08, 0xF0, 0x20, 0x56,
+	0x30, 0xFA, 0x83, 0x69, 0xFD, 0x56, 0x5A, 0x9B,
+	0xCE, 0xF8, 0x28, 0x02, 0xB4, 0x91, 0x35, 0x75,
+	0x9E, 0x63, 0x99, 0x48, 0xCF, 0x35, 0xF5, 0x58,
+	0x0C, 0x48, 0x8F, 0x0A, 0x2D, 0x9A, 0xE6, 0x40,
+	0xF6, 0x21, 0xB5, 0x69, 0xC1, 0x09, 0x31, 0x00,
+	0xA3, 0xC1, 0x4C, 0x99, 0x70, 0x4F, 0x5A, 0x63,
+	0x17, 0x90, 0xB8, 0xF8, 0x3A, 0x0E, 0xFD, 0x67,
+	0xEA, 0x0E, 0xBA, 0x7B, 0x1E, 0xEF, 0x37, 0x84,
+	0xD5, 0x51, 0x37, 0x01, 0xD6, 0x93, 0x15, 0xDF,
+	0x56, 0x89, 0x0E, 0x54, 0xF5, 0x1D, 0xF7, 0xE5,
+	0xB7, 0xC1, 0xF1, 0xC2, 0xD9, 0x14, 0x6F, 0x40,
+	0x55, 0x67, 0x50, 0x7C, 0x58, 0x35, 0x8B, 0x39,
+	0xCB, 0xB5, 0x87, 0xF5, 0x55, 0x5E, 0x26, 0x8C,
+	0x5B, 0x73, 0x0E, 0xBB, 0x25, 0x51, 0x0E, 0xAD,
+	0x57, 0x72, 0x7B, 0x68, 0x83, 0x11, 0x1E, 0x3A,
+	0x3D, 0xA4, 0x7C, 0x18, 0xB7, 0x70, 0x18, 0xBC,
+	0x72, 0x03, 0x4A, 0xA1, 0xD2, 0xF9, 0xA9, 0x8A,
+	0x25, 0x45, 0x19, 0xEE, 0x93, 0x06, 0xB5, 0x09,
+	0x71, 0xC9, 0x2D, 0xFD, 0x2B, 0xF3, 0xC7, 0x64,
+	0x5F, 0xCE, 0x71, 0x1D, 0x81, 0x96, 0x67, 0xBF,
+	0x01, 0x39, 0x8C, 0xE7, 0xA2, 0xD0, 0x98, 0x57,
+	0x5A, 0xFD, 0x21, 0xC7, 0x46, 0xAA, 0xB4, 0xE4,
+	0x0E, 0xBE, 0xC6, 0x68, 0x3E, 0x38, 0xF5, 0xA2,
+	0xED, 0x73, 0xCC, 0x53, 0x7E, 0x7E, 0x03, 0x32,
+	0xDC, 0xB6, 0xC1, 0x03, 0x9E, 0xB3, 0x2A, 0xAD,
+	0xC0, 0xC3, 0x6E, 0x47, 0xFB, 0x1E, 0xB7, 0x0D,
+	0x86, 0x95, 0x09, 0xA6, 0x9D, 0x6F, 0x92, 0xFC,
+	0xFF, 0x2C, 0x7D, 0x09, 0x16, 0x68, 0x50, 0x3E,
+	0x4F, 0x23, 0x4C, 0x93, 0x95, 0x2A, 0xE1, 0x9B,
+	0x16, 0xF0, 0x0F, 0xFF, 0x79, 0xA8, 0x06, 0xF9,
+	0x70, 0x61, 0x72, 0x2C, 0xE8, 0x91, 0x01, 0x6D,
+	0x45, 0xE5, 0x82, 0x5D, 0x26, 0x21, 0xAD, 0x3D,
+	0x77, 0x73, 0x23, 0x04, 0x84, 0x27, 0xA3, 0x5D,
+	0x6D, 0xA8, 0x99, 0xC1, 0xCE, 0x4F, 0xA9, 0xF7,
+	0xAB, 0x5C, 0xDE, 0x01, 0xE6, 0x1E, 0xEF, 0xE6,
+	0xFD, 0xE0, 0x68, 0x85, 0x3E, 0xEE, 0xBF, 0xF1,
+	0x0D, 0x79, 0xF4, 0xA2, 0xB4, 0x14, 0xBC, 0x0C,
+	0x49, 0x77, 0x03, 0x71, 0x08, 0x3E, 0x40, 0xA6,
+	0xD7, 0x03, 0xFA, 0xE2, 0xFB, 0xC7, 0x59, 0x30,
+	0x6E, 0x07, 0x06, 0x1C, 0x7C, 0x47, 0xE5, 0x4C,
+	0x57, 0x0A, 0x91, 0x4A, 0x43, 0xE4, 0x8A, 0xCD,
+	0x6E, 0x92, 0x01, 0xE2, 0x52, 0xC1, 0x92, 0x34,
+	0x8E, 0x64, 0x0F, 0x39, 0x63, 0x53, 0xAB, 0xE5,
+	0x44, 0xD5, 0xAA, 0xAA, 0xF6, 0x03, 0x89, 0xB9,
+	0xDD, 0xB2, 0x2D, 0x56, 0x1A, 0xE0, 0x72, 0x5A,
+	0x52, 0x19, 0x46, 0xEA, 0xB3, 0xCE, 0xB3, 0x59,
+	0x46, 0x7A, 0xA7, 0x48, 0x37, 0x0C, 0x09, 0xBA,
+	0x92, 0x70, 0x17, 0x7F, 0xF5, 0xD3, 0x60, 0x44,
+	0xC4, 0xC6, 0xC6, 0x7D, 0xD2, 0x21, 0xAC, 0x3F,
+	0x62, 0x6C, 0xE9, 0xBA, 0x4C, 0xF3, 0x82, 0x7E,
+	0x6D, 0x3A, 0x92, 0xDC, 0x94, 0xE4, 0x5F, 0xA6,
+	0x8B, 0x66, 0xA0, 0xDD, 0xE2, 0x97, 0x83, 0xED,
+	0xF5, 0x9D, 0xDF, 0x74, 0x77, 0x23, 0x7D, 0xDA,
+	0xC4, 0xFB, 0x92, 0x1A, 0xD9, 0x37, 0x36, 0xD2,
+	0x88, 0xC9, 0xEA, 0x0F, 0x98, 0xBD, 0xC5, 0xF8,
+	0xAA, 0x19, 0x75, 0x12, 0x6A, 0x41, 0xB5, 0xB3,
+	0xB5, 0xA4, 0x96, 0xDC, 0x2B, 0x49, 0x86, 0x66,
+	0x35, 0xD8, 0x4A, 0x62, 0xB4, 0xCB, 0x1E, 0x27,
+	0xC1, 0xAD, 0x34, 0x0E, 0x26, 0x16, 0xF2, 0xC2,
+	0x22, 0x52, 0x84, 0xD8, 0xD1, 0x32, 0xB8, 0x9C,
+	0xFE, 0x64, 0x42, 0x9F, 0xE4, 0x69, 0xF0, 0xAE,
+	0x3B, 0xD9, 0x2C, 0xA0, 0x14, 0xEB, 0x69, 0x74,
+	0x7C, 0xE2, 0xA6, 0x60, 0xE1, 0x52, 0x1C, 0xCC,
+	0xBF, 0xE6, 0xA1, 0x83, 0x20, 0x5D, 0x9E, 0xA3,
+	0xFB, 0x84, 0x8B, 0x33, 0xE6, 0xC9, 0x32, 0x83,
+	0xC0, 0x3F, 0x98, 0x1D, 0x6E, 0xC0, 0x50, 0x71,
+	0x29, 0x60, 0x5F, 0x36, 0xB4, 0x68, 0x1D, 0xB9,
+	0x76, 0x73, 0xC3, 0x80, 0xC5, 0xBC, 0x59, 0x7B,
+	0x59, 0xB4, 0xE0, 0x6A, 0x80, 0xCD, 0x4D, 0x8C,
+	0x9E, 0xE0, 0x0B, 0x45, 0x7D, 0x54, 0xD4, 0xC4,
+	0x97, 0x6C, 0x54, 0xEF, 0x14, 0x64, 0xBD, 0x3B,
+	0xD7, 0xEE, 0xF4, 0xD1, 0x41, 0x76, 0x3A, 0x24,
+	0x7A, 0xC2, 0xCA, 0x68, 0x28, 0x53, 0x46, 0xF7,
+	0x1B, 0xDA, 0x4B, 0x7A, 0x56, 0x75, 0x86, 0xFB,
+	0x31, 0x2C, 0x27, 0xF9, 0x4D, 0x35, 0xA4, 0x82,
+	0xE7, 0x2F, 0x41, 0xB4, 0xCA, 0xCE, 0x75, 0x94,
+	0x08, 0x54, 0xE2, 0x9E, 0x99, 0xC9, 0x85, 0xDE,
+	0x6F, 0x80, 0x95, 0x59, 0x3E, 0x54, 0x9F, 0x31,
+	0xF8, 0xDE, 0xD0, 0xD7, 0xA6, 0xD4, 0xD3, 0xBB,
+	0xD9, 0xC7, 0x55, 0xDD, 0xAE, 0xAD, 0x9E, 0x57,
+	0x4A, 0x33, 0x5D, 0x7A, 0xA6, 0xA3, 0xCA, 0xF9,
+	0x4C, 0x5B, 0x51, 0xCC, 0x22, 0xBB, 0x76, 0x44,
+	0x17, 0xDE, 0x22, 0xA1, 0xDF, 0x80, 0x13, 0x7D,
+	0xE5, 0x34, 0x7E, 0x75, 0x73, 0x10, 0x40, 0xFB,
+	0x9A, 0x21, 0xCD, 0xD3, 0xD3, 0x84, 0xB6, 0x0C,
+	0x31, 0x1E, 0xB5, 0x42, 0xF4, 0x34, 0x11, 0x7F,
+	0x4A, 0x23, 0xA8, 0xA5, 0x8F, 0x20, 0xCD, 0xA9,
+	0xF2, 0xE4, 0xEE, 0xFA, 0x57, 0xD1, 0x22, 0x1C,
+	0xA5, 0xDC, 0x0B, 0x25, 0xFE, 0xC2, 0xA7, 0x7E,
+	0x09, 0x2E, 0xDA, 0x40, 0x9F, 0x6C, 0xC8, 0x71,
+	0x58, 0x91, 0x04, 0x25, 0xD0, 0x06, 0xEA, 0x1B,
+	0xCD, 0x9D, 0x50, 0xD8, 0x40, 0x24, 0xAC, 0xC3,
+	0xB4, 0x07, 0x6E, 0x76, 0xF4, 0x4C, 0xD8, 0x80,
+	0xD0, 0x20, 0xF5, 0x15, 0x6A, 0x0A, 0x12, 0xF8,
+	0x6B, 0x67, 0x77, 0x34, 0xAE, 0x60, 0x68, 0x13,
+	0x5B, 0x8E, 0xFF, 0x5E, 0x7A, 0x77, 0x67, 0x0D,
+	0xE6, 0x96, 0x43, 0x9F, 0x8F, 0x77, 0x5F, 0x97,
+	0x23, 0x91, 0x33, 0x72, 0xC7, 0x8A, 0xC7, 0x80,
+	0xCF, 0xE7, 0x71, 0x06, 0x25, 0xB7, 0x4B, 0x89,
+	0x6A, 0x46, 0x67, 0x19, 0x49, 0x44, 0x03, 0x52,
+	0x32, 0xB1, 0x8F, 0xE7, 0x9E, 0xDA, 0x03, 0x41,
+	0xA3, 0xAC, 0xE5, 0xF3, 0x96, 0xE6, 0xAC, 0xFF,
+	0xEC, 0x35, 0x4D, 0x83, 0xA9, 0xCE, 0x7C, 0x52,
+	0xF2, 0x36, 0x97, 0xF0, 0x28, 0x36, 0x54, 0x59,
+	0x96, 0xEA, 0xEE, 0xB2, 0xC1, 0xAB, 0xA4, 0x96,
+	0x62, 0xD4, 0x3C, 0xF0, 0x1F, 0x2D, 0x65, 0x0E,
+	0x46, 0x7E, 0x12, 0x31, 0x8F, 0xA7, 0x8D, 0x7A,
+	0x4A, 0x41, 0x15, 0x57, 0x90, 0xF6, 0xF1, 0xE8,
+	0xE8, 0xE3, 0x57, 0x7B, 0x55, 0x85, 0x95, 0x97,
+	0xB3, 0x29, 0x3D, 0x02, 0x73, 0x1E, 0x87, 0x1F,
+	0x01, 0x89, 0x06, 0x88, 0x9E, 0x8A, 0x2E, 0xE0,
+	0x99, 0xFC, 0xF0, 0x48, 0x60, 0x37, 0x65, 0x25,
+	0xDB, 0x89, 0xC9, 0x7A, 0x51, 0x7E, 0x35, 0x92,
+	0x00, 0xC9, 0x61, 0x3F, 0x88, 0xE3, 0x20, 0x01,
+	0x46, 0x5A, 0x2C, 0x37, 0x02, 0xC9, 0x15, 0x0F,
+	0xB2, 0xEB, 0xC3, 0x55, 0x18, 0xF0, 0x15, 0x1A,
+	0x08, 0x8E, 0xB8, 0x9D, 0x18, 0xE4, 0x9F, 0x34,
+	0x10, 0x67, 0x68, 0x57, 0x60, 0x61, 0xEC, 0xD5,
+	0xD9, 0xA8, 0x3A, 0xAB, 0xD6, 0xD2, 0x7A, 0x47,
+	0x3D, 0xA4, 0x08, 0x7C, 0x3E, 0x4D, 0x76, 0x08,
+	0x19, 0x22, 0xB2, 0x89, 0x57, 0x84, 0xC2, 0x98,
+	0x72, 0xB8, 0x8B, 0xE0, 0x85, 0xA1, 0x3A, 0xC2,
+	0xA0, 0x06, 0x43, 0x03, 0xCF, 0x4F, 0x27, 0x80,
+	0x48, 0x9A, 0xBC, 0xB3, 0x3C, 0xC4, 0x5E, 0xAC,
+	0x8B, 0x85, 0x6F, 0x21, 0xD6, 0xFE, 0x12, 0x90,
+	0x53, 0x2F, 0x65, 0x32, 0x8D, 0x03, 0xFE, 0xFE,
+	0x61, 0x04, 0x47, 0x24, 0x6A, 0xB5, 0x01, 0x98,
+	0xB9, 0x27, 0x10, 0xE1, 0x32, 0x3D, 0x2A, 0xA0,
+	0xC5, 0x70, 0xDE, 0x1E, 0x10, 0xD7, 0x01, 0x50,
+	0x4F, 0x87, 0xCA, 0xD9, 0xBF, 0x12, 0xEA, 0x38,
+	0x4C, 0x43, 0xD5, 0x5A, 0xEF, 0x3E, 0x21, 0x8E,
+	0x59, 0x77, 0x23, 0xED, 0x51, 0x09, 0x99, 0x73,
+	0xD2, 0x56, 0x04, 0xDD, 0x8F, 0x5F, 0xDF, 0x79,
+	0xFF, 0x16, 0x8C, 0xB0, 0xBA, 0x8A, 0x1A, 0x56,
+	0xAF, 0xCA, 0x19, 0xF2, 0x64, 0x1A, 0xF5, 0x1E,
+	0xA7, 0xA7, 0x84, 0x3D, 0xAD, 0xC1, 0x0E, 0x22,
+	0xA1, 0x45, 0xFC, 0xB4, 0x13, 0x91, 0x34, 0xB7,
+	0x48, 0xEF, 0x9E, 0xD9, 0x0B, 0xE3, 0x82, 0x75,
+	0x80, 0xC5, 0xD9, 0xA0, 0x77, 0xA3, 0xF9, 0xCC,
+	0x67, 0xDD, 0xCB, 0x28, 0xC6, 0xE8, 0x2C, 0xB8,
+	0xAC, 0x63, 0xBD, 0x3B, 0x28, 0x4A, 0xE9, 0x2D,
+	0x29, 0x84, 0xD7, 0x4F, 0x61, 0x06, 0xE3, 0x37,
+	0xC1, 0x58, 0x20, 0x5D, 0x0A, 0xE7, 0x45, 0x29,
+	0x7D, 0xED, 0x0F, 0xCE, 0x00, 0x95, 0x2A, 0x62,
+	0x38, 0xA8, 0x1A, 0x3A, 0x96, 0x0E, 0x56, 0xD9,
+	0x18, 0xC2, 0x25, 0xA5, 0xAA, 0x27, 0x0A, 0x6E,
+	0xDD, 0x1C, 0x35, 0x6C, 0xC1, 0x26, 0x90, 0xF6,
+	0x43, 0x1B, 0x34, 0xDA, 0xE1, 0x5D, 0x09, 0x7C,
+	0xBE, 0x0F, 0x40, 0xD3, 0x24, 0x82, 0x0B, 0xFF,
+	0xE6, 0xB7, 0x10, 0xD6, 0x36, 0xD0, 0xE6, 0xC0,
+	0xBE, 0x65, 0x4C, 0x83, 0xF1, 0xDA, 0xDE, 0xCE,
+	0xE8, 0x5A, 0x80, 0x88, 0xFE, 0x9B, 0x79, 0x54,
+	0xA3, 0xA4, 0x5A, 0x76, 0xD0, 0xE2, 0xCE, 0x92,
+	0x53, 0x7D, 0x9C, 0xDA, 0xA1, 0xED, 0x9F, 0x56,
+	0x05, 0x0A, 0xA4, 0x81, 0xC7, 0x82, 0x5B, 0xB8,
+	0xC7, 0xA8, 0x95, 0x21, 0x99, 0x0B, 0x0F, 0xD2,
+	0x66, 0x68, 0xC3, 0x07, 0x53, 0x93, 0x8C, 0x68,
+	0x5A, 0xF5, 0x6F, 0x5E, 0x07, 0x68, 0x70, 0xF2,
+	0x6A, 0x78, 0xA8, 0xDB, 0x24, 0x6F, 0xD9, 0x74,
+	0x38, 0x29, 0xBF, 0x50, 0xCC, 0xC8, 0x34, 0x82,
+	0x69, 0x1A, 0xF4, 0x1A, 0x95, 0xE1, 0x31, 0x39,
+	0x00, 0xBE, 0xF6, 0x25, 0x19, 0x5A, 0xF6, 0xA7,
+	0xB7, 0xC7, 0xAC, 0xFA, 0x52, 0x6D, 0xC7, 0xEA,
+	0xFF, 0xC4, 0xB0, 0x84, 0x41, 0x2D, 0x6B, 0x22,
+	0x57, 0x1F, 0x8A, 0x97, 0x71, 0xEF, 0x54, 0xA7,
+	0xA9, 0xB5, 0xA0, 0x7A, 0xFF, 0x52, 0xBC, 0x78,
+	0x56, 0x8C, 0x9C, 0x2A, 0xFB, 0x31, 0xA6, 0xC8,
+	0x87, 0xC8, 0x82, 0x72, 0x00, 0x68, 0x27, 0xDE,
+	0x54, 0x67, 0x4C, 0x36, 0x4E, 0xE2, 0x77, 0x7F,
+	0xAD, 0x15, 0x1B, 0xC9, 0x07, 0xC4, 0x84, 0x50,
+	0x84, 0x45, 0xB5, 0x1D, 0xCE, 0x3F, 0x7C, 0xDF,
+	0x73, 0x88, 0x3E, 0x6D, 0xAC, 0x18, 0x67, 0x2C,
+	0x1D, 0x31, 0xD5, 0x41, 0x6E, 0xFC, 0x3D, 0xFE,
+	0x5C, 0x6D, 0x3B, 0xCB, 0x67, 0x05, 0x44, 0x8B,
+	0x02, 0xB7, 0xF5, 0x05, 0xFD, 0x1D, 0x7E, 0x13,
+	0x90, 0xCE, 0xED, 0xD2, 0xAB, 0x08, 0xFF, 0xC3,
+	0x91, 0x2C, 0x79, 0x06, 0xDB, 0x54, 0xAB, 0xFF,
+	0xF6, 0x9D, 0xBB, 0xDC, 0x3C, 0xCD, 0x03, 0xE7,
+	0xD8, 0x1A, 0x4E, 0x7F, 0xCB, 0x1B, 0xA3, 0xCA,
+	0xDC, 0x14, 0xC5, 0xFE, 0x46, 0x38, 0x07, 0x82,
+	0x5E, 0x9A, 0x77, 0x9C, 0xB4, 0x44, 0x26, 0xBA,
+	0xC2, 0x27, 0xD0, 0xF4, 0x75, 0x67, 0x4A, 0x15,
+	0x2A, 0x55, 0x2A, 0x61, 0x87, 0x55, 0xA0, 0xFB,
+	0xE0, 0x93, 0xA5, 0xD7, 0xDF, 0x0D, 0x97, 0xD3,
+	0x93, 0x39, 0x0B, 0x5A, 0xC6, 0x86, 0x17, 0x7D,
+	0x6A, 0xA4, 0x07, 0x9C, 0xB9, 0x0F, 0x42, 0x15,
+	0x46, 0x71, 0x45, 0x70, 0x75, 0x4B, 0xD5, 0x80,
+	0x64, 0x62, 0x92, 0x50, 0xBC, 0x10, 0xA1, 0x68,
+	0x60, 0x5A, 0x6F, 0x31, 0x26, 0xCA, 0xB1, 0x48,
+	0xFB, 0xC6, 0xF4, 0x3A, 0xCA, 0x52, 0x20, 0x2F,
+	0x7D, 0xEC, 0x19, 0xF8, 0xAA, 0x27, 0xC2, 0x23,
+	0xD0, 0x8F, 0x60, 0xFC, 0x7F, 0xA0, 0xCB, 0xDC,
+	0xA2, 0xC6, 0xC4, 0xC5, 0x13, 0x00, 0xF3, 0x43,
+	0x48, 0x6D, 0xFD, 0x7D, 0xB9, 0xA8, 0x14, 0xB0,
+	0x0C, 0x72, 0x82, 0x2F, 0x99, 0x64, 0x41, 0x2B,
+	0xB3, 0x34, 0x73, 0x73, 0xF4, 0x26, 0x1D, 0x06,
+	0xDF, 0x6E, 0xF4, 0x20, 0x1E, 0x31, 0xE3, 0x8A,
+	0x01, 0x6C, 0xDB, 0x3C, 0xE3, 0xC6, 0xC4, 0xC5,
+	0xB8, 0x20, 0x51, 0xF1, 0xD6, 0xB0, 0x30, 0xB7,
+	0x2D, 0xDA, 0x95, 0x01, 0x0D, 0xED, 0xEE, 0x6F,
+	0x69, 0xFD, 0xCF, 0x9D, 0xDD, 0x05, 0xD6, 0xC0,
+	0xFE, 0x11, 0x67, 0xAF, 0x53, 0x94, 0x60, 0xFC,
+	0x56, 0xBA, 0x0C, 0x5F, 0xA7, 0x7E, 0xDA, 0x65
+};
+static const struct
+blockcipher_test_data aes_test_data_xts_wrapped_key_48_pt_4096_du_4096 = {
+	.crypto_algo = RTE_CRYPTO_CIPHER_AES_XTS,
+	.cipher_key = {
+		.data = {
+			0x94, 0xD8, 0xAE, 0xD7, 0x74, 0xBF, 0x51, 0xF1,
+			0x2D, 0x19, 0x0B, 0x9A, 0xB3, 0xDC, 0x1E, 0xA9,
+			0x5C, 0xC5, 0x47, 0xC1, 0xBB, 0x94, 0xDB, 0xFA,
+			0x2B, 0x1E, 0x54, 0xA5, 0x76, 0xAD, 0x07, 0x4B,
+			0x46, 0xE0, 0xAA, 0xF9, 0xF7, 0x5C, 0x64, 0x2D,
+			0x8F, 0x04, 0x17, 0x72, 0xE5, 0x8D, 0xFE, 0x9C
+		},
+		.len = 48
+	},
+	.iv = {
+		.data = {
+			0x4F, 0xAE, 0xF7, 0x11, 0x7C, 0xDA, 0x59, 0xC6,
+			0x6E, 0x4B, 0x92, 0x01, 0x3E, 0x76, 0x8A, 0xD5
+		},
+		.len = 16
+	},
+	.plaintext = {
+		.data = plaintext_aes256xts_4096bytes,
+		.len = 4096
+	},
+	.ciphertext = {
+		.data = ciphertext_aes256xts_4096bytes,
+		.len = 4096
+	},
+	.xts_dataunit_len = 4096,
+	.wrapped_key = true
+};
+
+static const struct
+blockcipher_test_data aes_test_data_xts_wrapped_key_48_pt_4096_du_0 = {
+	.crypto_algo = RTE_CRYPTO_CIPHER_AES_XTS,
+	.cipher_key = {
+		.data = {
+			0x94, 0xD8, 0xAE, 0xD7, 0x74, 0xBF, 0x51, 0xF1,
+			0x2D, 0x19, 0x0B, 0x9A, 0xB3, 0xDC, 0x1E, 0xA9,
+			0x5C, 0xC5, 0x47, 0xC1, 0xBB, 0x94, 0xDB, 0xFA,
+			0x2B, 0x1E, 0x54, 0xA5, 0x76, 0xAD, 0x07, 0x4B,
+			0x46, 0xE0, 0xAA, 0xF9, 0xF7, 0x5C, 0x64, 0x2D,
+			0x8F, 0x04, 0x17, 0x72, 0xE5, 0x8D, 0xFE, 0x9C
+		},
+		.len = 48
+	},
+	.iv = {
+		.data = {
+			0x4F, 0xAE, 0xF7, 0x11, 0x7C, 0xDA, 0x59, 0xC6,
+			0x6E, 0x4B, 0x92, 0x01, 0x3E, 0x76, 0x8A, 0xD5
+		},
+		.len = 16
+	},
+	.plaintext = {
+		.data = plaintext_aes256xts_4096bytes,
+		.len = 4096
+	},
+	.ciphertext = {
+		.data = ciphertext_aes256xts_4096bytes,
+		.len = 4096
+	},
+	.xts_dataunit_len = 0,
+	.wrapped_key = true
+};
+
+static const struct
+blockcipher_test_data aes_test_data_xts_wrapped_key_48_pt_512_du_0 = {
+	.crypto_algo = RTE_CRYPTO_CIPHER_AES_XTS,
+	.cipher_key = {
+		.data = {
+			0x94, 0xD8, 0xAE, 0xD7, 0x74, 0xBF, 0x51, 0xF1,
+			0x2D, 0x19, 0x0B, 0x9A, 0xB3, 0xDC, 0x1E, 0xA9,
+			0x5C, 0xC5, 0x47, 0xC1, 0xBB, 0x94, 0xDB, 0xFA,
+			0x2B, 0x1E, 0x54, 0xA5, 0x76, 0xAD, 0x07, 0x4B,
+			0x46, 0xE0, 0xAA, 0xF9, 0xF7, 0x5C, 0x64, 0x2D,
+			0x8F, 0x04, 0x17, 0x72, 0xE5, 0x8D, 0xFE, 0x9C
+		},
+		.len = 48
+	},
+	.iv = {
+		.data = {
+			0x4F, 0xAE, 0xF7, 0x11, 0x7C, 0xDA, 0x59, 0xC6,
+			0x6E, 0x4B, 0x92, 0x01, 0x3E, 0x76, 0x8A, 0xD5
+		},
+		.len = 16
+	},
+	.plaintext = {
+		.data = plaintext_aes256xts_512bytes,
+		.len = 512
+	},
+	.ciphertext = {
+		.data = ciphertext_aes256xts_512bytes,
+		.len = 512
+	},
+	.xts_dataunit_len = 0,
+	.wrapped_key = true
+};
+
+
 /* AES-DOCSIS-BPI test vectors */
 
 /* Multiple of AES block size */
@@ -3347,6 +4639,54 @@ static const struct blockcipher_test_case aes_cipheronly_test_cases[] = {
 		.test_data = &aes_test_data_xts_key_64_pt_48,
 		.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
 	},
+	{
+		.test_descr = "AES-256-XTS Encryption (512-byte plaintext"
+			      " Dataunit 512)",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_512_du_512,
+		.op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT,
+	},
+	{
+		.test_descr = "AES-256-XTS Decryption (512-byte plaintext"
+			      " Dataunit 512)",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_512_du_512,
+		.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
+	},
+	{
+		.test_descr = "AES-256-XTS Encryption (512-byte plaintext"
+			      " Dataunit 0)",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_512_du_0,
+		.op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT,
+	},
+	{
+		.test_descr = "AES-256-XTS Decryption (512-byte plaintext"
+			      " Dataunit 0)",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_512_du_0,
+		.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
+	},
+	{
+		.test_descr = "AES-256-XTS Encryption (4096-byte plaintext"
+			      " Dataunit 4096)",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_4096_du_4096,
+		.op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT,
+	},
+	{
+		.test_descr = "AES-256-XTS Decryption (4096-byte plaintext"
+			      " Dataunit 4096)",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_4096_du_4096,
+		.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
+	},
+	{
+		.test_descr = "AES-256-XTS Encryption (4096-byte plaintext"
+			      " Dataunit 0)",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_4096_du_0,
+		.op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT,
+	},
+	{
+		.test_descr = "AES-256-XTS Decryption (4096-byte plaintext"
+			      " Dataunit 0)",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_4096_du_0,
+		.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
+	},
 	{
 		.test_descr = "cipher-only - NULL algo - x8 - encryption",
 		.test_data = &null_test_data_chain_x8_multiple,
diff --git a/app/test/test_cryptodev_blockcipher.c b/app/test/test_cryptodev_blockcipher.c
index d342c7b859..53fd4718af 100644
--- a/app/test/test_cryptodev_blockcipher.c
+++ b/app/test/test_cryptodev_blockcipher.c
@@ -36,7 +36,8 @@ verify_algo_support(const struct blockcipher_test_case *t,
 		if (capability == NULL)
 			return -1;
 
-		if (cap_idx.algo.cipher != RTE_CRYPTO_CIPHER_NULL)
+		if (cap_idx.algo.cipher != RTE_CRYPTO_CIPHER_NULL &&
+				!(t->test_data->wrapped_key))
 			ret = rte_cryptodev_sym_capability_check_cipher(capability,
 							tdata->cipher_key.len,
 							tdata->iv.len);
@@ -145,6 +146,12 @@ test_blockcipher_one_case(const struct blockcipher_test_case *t,
 
 		nb_segs = 3;
 	}
+	if (!!(feat_flags & RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY) ^
+		tdata->wrapped_key) {
+		snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN,
+			"SKIPPED");
+		return TEST_SKIPPED;
+	}
 
 	if (global_api_test_type == CRYPTODEV_RAW_API_TEST &&
 		!(feat_flags & RTE_CRYPTODEV_FF_SYM_RAW_DP)) {
@@ -450,6 +457,7 @@ test_blockcipher_one_case(const struct blockcipher_test_case *t,
 		cipher_xform->cipher.key.data = cipher_key;
 		cipher_xform->cipher.key.length = tdata->cipher_key.len;
 		cipher_xform->cipher.iv.offset = IV_OFFSET;
+		cipher_xform->cipher.dataunit_len = tdata->xts_dataunit_len;
 
 		if (tdata->crypto_algo == RTE_CRYPTO_CIPHER_NULL)
 			cipher_xform->cipher.iv.length = 0;
diff --git a/app/test/test_cryptodev_blockcipher.h b/app/test/test_cryptodev_blockcipher.h
index a06241b06d..dcaa08ae22 100644
--- a/app/test/test_cryptodev_blockcipher.h
+++ b/app/test/test_cryptodev_blockcipher.h
@@ -97,6 +97,8 @@ struct blockcipher_test_data {
 
 	unsigned int cipher_offset;
 	unsigned int auth_offset;
+	uint16_t xts_dataunit_len;
+	bool wrapped_key;
 };
 
 struct unit_test_suite *
-- 
2.27.0

[dpdk-dev] [PATCH v8 16/16] test/crypto: add AES-XTS multi segment OOP tests

[Shiri Kuzin]

The AES-XTS algorithm can supports wrapped key and data-unit.
The encryption/decryption can be done out of place and using multi
segments.

Add multi segment and out of place tests to the recently added AES-XTS
vectors, which support using data-unit and a wrapped key.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 app/test/test_cryptodev_aes_test_vectors.h | 64 ++++++++++++++++++++++
 1 file changed, 64 insertions(+)

diff --git a/app/test/test_cryptodev_aes_test_vectors.h b/app/test/test_cryptodev_aes_test_vectors.h
index dd7ffde685..a797af1b00 100644
--- a/app/test/test_cryptodev_aes_test_vectors.h
+++ b/app/test/test_cryptodev_aes_test_vectors.h
@@ -4687,6 +4687,70 @@ static const struct blockcipher_test_case aes_cipheronly_test_cases[] = {
 		.test_data = &aes_test_data_xts_wrapped_key_48_pt_4096_du_0,
 		.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
 	},
+	{
+		.test_descr = "AES-256-XTS Encryption (512-byte plaintext"
+			      " Dataunit 512) Scater gather OOP",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_512_du_512,
+		.op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT,
+		.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP |
+				BLOCKCIPHER_TEST_FEATURE_SG,
+	},
+	{
+		.test_descr = "AES-256-XTS Decryption (512-byte plaintext"
+			      " Dataunit 512) Scater gather OOP",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_512_du_512,
+		.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
+		.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP |
+				BLOCKCIPHER_TEST_FEATURE_SG,
+	},
+	{
+		.test_descr = "AES-256-XTS Encryption (512-byte plaintext"
+			      " Dataunit 0) Scater gather OOP",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_512_du_0,
+		.op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT,
+		.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP |
+				BLOCKCIPHER_TEST_FEATURE_SG,
+	},
+	{
+		.test_descr = "AES-256-XTS Decryption (512-byte plaintext"
+			      " Dataunit 0) Scater gather OOP",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_512_du_0,
+		.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
+		.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP |
+				BLOCKCIPHER_TEST_FEATURE_SG,
+	},
+	{
+		.test_descr = "AES-256-XTS Encryption (4096-byte plaintext"
+			      " Dataunit 4096) Scater gather OOP",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_4096_du_4096,
+		.op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT,
+		.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP |
+				BLOCKCIPHER_TEST_FEATURE_SG,
+	},
+	{
+		.test_descr = "AES-256-XTS Decryption (4096-byte plaintext"
+			      " Dataunit 4096) Scater gather OOP",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_4096_du_4096,
+		.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
+		.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP |
+				BLOCKCIPHER_TEST_FEATURE_SG,
+	},
+	{
+		.test_descr = "AES-256-XTS Encryption (4096-byte plaintext"
+			      " Dataunit 0) Scater gather OOP",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_4096_du_0,
+		.op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT,
+		.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP |
+				BLOCKCIPHER_TEST_FEATURE_SG,
+	},
+	{
+		.test_descr = "AES-256-XTS Decryption (4096-byte plaintext"
+			      " Dataunit 0) Scater gather OOP",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_4096_du_0,
+		.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
+		.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP |
+				BLOCKCIPHER_TEST_FEATURE_SG,
+	},
 	{
 		.test_descr = "cipher-only - NULL algo - x8 - encryption",
 		.test_data = &null_test_data_chain_x8_multiple,
-- 
2.27.0

Re: [dpdk-dev] [EXT] [PATCH v8 00/16] drivers: introduce mlx5 crypto PMD

[Akhil Goyal]

> v2:
> - Add data-path part.
> 
> v3:
> - Rebase.
> 
> v4:
> - Rebase + Address the following Akhil comments:
> - Set HW feature flag in the capability patch.
> - Fix mp object release in session clear.
> - Some spelling and word missing in doc.
> - Squash data-unit adjustment to the session operations commit.
> - Wording: device argument -> devarg.
> 
> v5:
> - Add mlx5 crypto tests into test library.
> - Update documentation according to Akhil comments.
> - Fix memory region management.
> - Fix multi segment case in data-path code.
> - Split documentation to the correct commits according to Akhil comments.
> - Rebase to new version.
> - Change license to Nvidia license.
> 
> v6:
> - Rebase to new version.
> - Address David's comment and update log accordingly.
> - Update testing app commits as suggested by Akhil.
> 
> v7:
> - Rebase to new version.
> - Add scatter gather and OOP test cases.
> 
> v8:
> - Add a test case that was removed by error.
> 
> 
> 
> Shiri Kuzin (11):
>   drivers: introduce mlx5 crypto PMD
>   crypto/mlx5: add DEK object management

This is really bad, I was planning to apply this series today,
It had small issues, so was planning to fix them while applying.
But when I applied the 2/16, it does not compile.
Please fix this, we are into v8 and compilation is broken.

I am adding few more comments, please fix them too in your next version
As soon as possible. I was planning to close RC2 during weekend, but it looks
It will fall over next week.

  147 |      mlx5_crypto_dek_match_cb,
      |      ^~~~~~~~~~~~~~~~~~~~~~~~
      |      |
      |      int (*)(void *, struct mlx5_list_entry *, void *)
In file included from ../drivers/crypto/mlx5/mlx5_crypto.h:13,
                 from ../drivers/crypto/mlx5/mlx5_crypto_dek.c:14:
../drivers/common/mlx5/mlx5_common_utils.h:138:20: note: expected 'mlx5_hlist_remove_cb' {aka 'void (*)(struct mlx5_hlist *, struct mlx5_hlist_entry *)'} but argument is of type 'int (*)(void *, struct mlx5_list_entry *, void *)'
  138 | struct mlx5_hlist *mlx5_hlist_create(const char *name, uint32_t size,
      |                    ^~~~~~~~~~~~~~~~~
../drivers/crypto/mlx5/mlx5_crypto_dek.c:144:20: error: too many arguments to function 'mlx5_hlist_create'
  144 |  priv->dek_hlist = mlx5_hlist_create("dek_hlist",
      |                    ^~~~~~~~~~~~~~~~~
In file included from ../drivers/crypto/mlx5/mlx5_crypto.h:13,
                 from ../drivers/crypto/mlx5/mlx5_crypto_dek.c:14:
../drivers/common/mlx5/mlx5_common_utils.h:138:20: note: declared here
  138 | struct mlx5_hlist *mlx5_hlist_create(const char *name, uint32_t size,
      |                    ^~~~~~~~~~~~~~~~~
../drivers/crypto/mlx5/mlx5_crypto_dek.c: In function 'mlx5_crypto_dek_destroy':
../drivers/crypto/mlx5/mlx5_crypto_dek.c:26:1: error: control reaches end of non-void function [-Werror=return-type]
   26 | }
      | ^
../drivers/crypto/mlx5/mlx5_crypto_dek.c: In function 'mlx5_crypto_dek_prepare':
../drivers/crypto/mlx5/mlx5_crypto_dek.c:45:1: error: control reaches end of non-void function [-Werror=return-type]
   45 | }
      | ^
../drivers/crypto/mlx5/mlx5_crypto_dek.c: In function 'mlx5_crypto_dek_create_cb':
../drivers/crypto/mlx5/mlx5_crypto_dek.c:127:1: error: control reaches end of non-void function [-Werror=return-type]
  127 | }
      | ^
cc1: all warnings being treated as errors

Re: [dpdk-dev] [EXT] [PATCH v8 01/16] drivers: introduce mlx5 crypto PMD

[Akhil Goyal]

> diff --git a/doc/guides/rel_notes/release_21_08.rst
> b/doc/guides/rel_notes/release_21_08.rst
> index 7d289e07e3..2bf4ce7a73 100644
> --- a/doc/guides/rel_notes/release_21_08.rst
> +++ b/doc/guides/rel_notes/release_21_08.rst
> @@ -125,6 +125,11 @@ New Features
>    The experimental PMD power management API now supports managing
>    multiple Ethernet Rx queues per lcore.
> 
> +* **Added support for Nvidia crypto device driver.**
> +
> +  * Added mlx5 crypto driver to support AES-XTS cipher operations.
> +    The first device to support it is ConnectX-6.
> +

Update release notes when the feature is complete.
Probably in your enq/deq patch (12/16)


> +static const struct rte_pci_id mlx5_crypto_pci_id_map[] = {
> +		{
> +			RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
> +
> 	PCI_DEVICE_ID_MELLANOX_CONNECTX6)
> +		},
> +		{
> +			.vendor_id = 0
> +		}
> +	};

Fix indentation.

> +

Re: [dpdk-dev] [EXT] [PATCH v8 02/16] crypto/mlx5: add DEK object management

[Akhil Goyal]

> A DEK(Data encryption Key) is an mlx5 HW object which represents the
> cipher algorithm key.
> The DEKs are used during data encryption/decryption operations.
> 
> In symmetric algorithms like AES-STS, we use the same DEK for both
> encryption and decryption.
> 
> Use the mlx5 hash-list tool to manage the DEK objects in the PMD.
> 
> Provide the compare, create and destroy functions to manage DEKs in
> hash-list and introduce an internal API to setup and unset the DEK
> management and to prepare and destroy specific DEK object.
> 
> The DEK hash-list will be created in dev_configure routine and
> destroyed in dev_close routine.
> 
> Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
> Acked-by: Matan Azrad <matan@nvidia.com>
> ---
>  drivers/crypto/mlx5/meson.build       |   1 +
>  drivers/crypto/mlx5/mlx5_crypto.c     |  42 ++++---
>  drivers/crypto/mlx5/mlx5_crypto.h     |  51 ++++++++
>  drivers/crypto/mlx5/mlx5_crypto_dek.c | 161
> ++++++++++++++++++++++++++
>  4 files changed, 239 insertions(+), 16 deletions(-)
>  create mode 100644 drivers/crypto/mlx5/mlx5_crypto.h
>  create mode 100644 drivers/crypto/mlx5/mlx5_crypto_dek.c
> 
> diff --git a/drivers/crypto/mlx5/meson.build
> b/drivers/crypto/mlx5/meson.build
> index 6fd70bc477..d55cdbfe6f 100644
> --- a/drivers/crypto/mlx5/meson.build
> +++ b/drivers/crypto/mlx5/meson.build
> @@ -11,6 +11,7 @@ fmt_name = 'mlx5_crypto'
>  deps += ['common_mlx5', 'eal', 'cryptodev']
>  sources = files(
>  	'mlx5_crypto.c',
> +	'mlx5_crypto_dek.c',
>  )
>  cflags_options = [
>  	'-std=c11',
> diff --git a/drivers/crypto/mlx5/mlx5_crypto.c
> b/drivers/crypto/mlx5/mlx5_crypto.c
> index fbe3c21aae..d2d82c7b15 100644
> --- a/drivers/crypto/mlx5/mlx5_crypto.c
> +++ b/drivers/crypto/mlx5/mlx5_crypto.c
> @@ -3,12 +3,9 @@
>   */
> 
>  #include <rte_malloc.h>
> -#include <rte_log.h>
>  #include <rte_errno.h>
> +#include <rte_log.h>
>  #include <rte_pci.h>
> -#include <rte_crypto.h>
> -#include <rte_cryptodev.h>
> -#include <rte_cryptodev_pmd.h>

There is some issue in the splitting of the patches,
The above headers are added in first patch and moved to a header file in this patch.
Take reference of the cnxk crypto driver which got merged recently.

> 
>  #include <mlx5_glue.h>
>  #include <mlx5_common.h>
> @@ -17,6 +14,7 @@
>  #include <mlx5_common_os.h>
> 
>  #include "mlx5_crypto_utils.h"
> +#include "mlx5_crypto.h"
> 
>  #define MLX5_CRYPTO_DRIVER_NAME mlx5_crypto
>  #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
> @@ -24,16 +22,6 @@
>  #define MLX5_CRYPTO_FEATURE_FLAGS \
>  	RTE_CRYPTODEV_FF_HW_ACCELERATED
> 
> -struct mlx5_crypto_priv {
> -	TAILQ_ENTRY(mlx5_crypto_priv) next;
> -	struct ibv_context *ctx; /* Device context. */
> -	struct rte_pci_device *pci_dev;
> -	struct rte_cryptodev *crypto_dev;
> -	void *uar; /* User Access Region. */
> -	uint32_t pdn; /* Protection Domain number. */
> -	struct ibv_pd *pd;
> -};
> -
>  TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
> 
> 	TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
>  static pthread_mutex_t priv_list_lock = PTHREAD_MUTEX_INITIALIZER;
> @@ -51,11 +39,33 @@ static const struct rte_driver mlx5_drv = {
> 
>  static struct cryptodev_driver mlx5_cryptodev_driver;
> 
> +static int
> +mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
> +		struct rte_cryptodev_config *config __rte_unused)
> +{
> +	struct mlx5_crypto_priv *priv = dev->data->dev_private;
> +
> +	if (mlx5_crypto_dek_setup(priv) != 0) {
> +		DRV_LOG(ERR, "Dek hash list creation has failed.");
> +		return -ENOMEM;
> +	}
> +	return 0;
> +}
> +
> +static int
> +mlx5_crypto_dev_close(struct rte_cryptodev *dev)
> +{
> +	struct mlx5_crypto_priv *priv = dev->data->dev_private;
> +
> +	mlx5_crypto_dek_unset(priv);
> +	return 0;
> +}
> +
>  static struct rte_cryptodev_ops mlx5_crypto_ops = {
> -	.dev_configure			= NULL,
> +	.dev_configure			= mlx5_crypto_dev_configure,
>  	.dev_start			= NULL,
>  	.dev_stop			= NULL,
> -	.dev_close			= NULL,
> +	.dev_close			= mlx5_crypto_dev_close,
>  	.dev_infos_get			= NULL,
>  	.stats_get			= NULL,
>  	.stats_reset			= NULL,
> diff --git a/drivers/crypto/mlx5/mlx5_crypto.h
> b/drivers/crypto/mlx5/mlx5_crypto.h
> new file mode 100644
> index 0000000000..167e9e57ad
> --- /dev/null
> +++ b/drivers/crypto/mlx5/mlx5_crypto.h
> @@ -0,0 +1,51 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright (c) 2021 NVIDIA Corporation & Affiliates
> + */
> +
> +#ifndef MLX5_CRYPTO_H_
> +#define MLX5_CRYPTO_H_
> +
> +#include <stdbool.h>
> +
> +#include <rte_cryptodev.h>
> +#include <rte_cryptodev_pmd.h>
> +
> +#include <mlx5_common_utils.h>
> +
> +#define MLX5_CRYPTO_DEK_HTABLE_SZ (1 << 11)
> +#define MLX5_CRYPTO_KEY_LENGTH 80
> +
> +struct mlx5_crypto_priv {
> +	TAILQ_ENTRY(mlx5_crypto_priv) next;
> +	struct ibv_context *ctx; /* Device context. */
> +	struct rte_pci_device *pci_dev;
> +	struct rte_cryptodev *crypto_dev;
> +	void *uar; /* User Access Region. */
> +	uint32_t pdn; /* Protection Domain number. */
> +	struct ibv_pd *pd;
> +	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
> +};
> +
> +struct mlx5_crypto_dek {
> +	struct mlx5_list_entry entry; /* Pointer to DEK hash list entry. */
> +	struct mlx5_devx_obj *obj; /* Pointer to DEK DevX object. */
> +	uint8_t data[MLX5_CRYPTO_KEY_LENGTH]; /* DEK key data. */
> +	bool size_is_48; /* Whether the key\data size is 48 bytes or not. */
> +} __rte_cache_aligned;
> +
> +int
> +mlx5_crypto_dek_destroy(struct mlx5_crypto_priv *priv,
> +			struct mlx5_crypto_dek *dek);
> +
> +struct mlx5_crypto_dek *
> +mlx5_crypto_dek_prepare(struct mlx5_crypto_priv *priv,
> +			struct rte_crypto_cipher_xform *cipher);
> +
> +int
> +mlx5_crypto_dek_setup(struct mlx5_crypto_priv *priv);
> +
> +void
> +mlx5_crypto_dek_unset(struct mlx5_crypto_priv *priv);
> +
> +#endif /* MLX5_CRYPTO_H_ */
> +
> diff --git a/drivers/crypto/mlx5/mlx5_crypto_dek.c
> b/drivers/crypto/mlx5/mlx5_crypto_dek.c
> new file mode 100644
> index 0000000000..43d1bcc9f8
> --- /dev/null
> +++ b/drivers/crypto/mlx5/mlx5_crypto_dek.c
> @@ -0,0 +1,161 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright (c) 2021 NVIDIA Corporation & Affiliates
> + */
> +
> +#include <rte_ip.h>
> +#include <rte_common.h>
> +#include <rte_errno.h>
> +#include <rte_log.h>
> +
> +#include <mlx5_prm.h>
> +#include <mlx5_devx_cmds.h>
> +
> +#include "mlx5_crypto_utils.h"
> +#include "mlx5_crypto.h"
> +
> +struct mlx5_crypto_dek_ctx {
> +	struct rte_crypto_cipher_xform *cipher;
> +	struct mlx5_crypto_priv *priv;
> +};
> +
> +int
> +mlx5_crypto_dek_destroy(struct mlx5_crypto_priv *priv,
> +			struct mlx5_crypto_dek *dek)
> +{
> +	return mlx5_hlist_unregister(priv->dek_hlist, &dek->entry);
> +}
> +
> +struct mlx5_crypto_dek *
> +mlx5_crypto_dek_prepare(struct mlx5_crypto_priv *priv,
> +			struct rte_crypto_cipher_xform *cipher)
> +{
> +	struct mlx5_hlist *dek_hlist = priv->dek_hlist;
> +	struct mlx5_crypto_dek_ctx dek_ctx = {
> +		.cipher = cipher,
> +		.priv = priv,
> +	};
> +	struct rte_crypto_cipher_xform *cipher_ctx = cipher;
> +	uint64_t key64 = __rte_raw_cksum(cipher_ctx->key.data,
> +					 cipher_ctx->key.length, 0);
> +	struct mlx5_list_entry *entry = mlx5_hlist_register(dek_hlist,
> +							     key64, &dek_ctx);
> +
> +	return entry == NULL ? NULL :
> +			     container_of(entry, struct mlx5_crypto_dek,
> entry);
> +}
> +
> +static struct mlx5_list_entry *
> +mlx5_crypto_dek_clone_cb(void *tool_ctx __rte_unused,
> +			 struct mlx5_list_entry *oentry,
> +			 void *cb_ctx __rte_unused)
> +{
> +	struct mlx5_crypto_dek *entry = rte_zmalloc(__func__,
> sizeof(*entry),
> +						    RTE_CACHE_LINE_SIZE);
> +
> +	if (!entry) {
> +		DRV_LOG(ERR, "Cannot allocate dek resource memory.");
> +		rte_errno = ENOMEM;
> +		return NULL;
> +	}
> +	memcpy(entry, oentry, sizeof(*entry));
> +	return &entry->entry;
> +}
> +
> +static void
> +mlx5_crypto_dek_clone_free_cb(void *tool_ctx __rte_unused,
> +			      struct mlx5_list_entry *entry)
> +{
> +	struct mlx5_crypto_dek *dek = container_of(entry,
> +						struct mlx5_crypto_dek,
> entry);
> +
> +	rte_free(dek);
> +}
> +
> +static int
> +mlx5_crypto_dek_match_cb(void *tool_ctx __rte_unused,
> +			 struct mlx5_list_entry *entry, void *cb_ctx)
> +{
> +	struct mlx5_crypto_dek_ctx *ctx = cb_ctx;
> +	struct rte_crypto_cipher_xform *cipher_ctx = ctx->cipher;
> +	struct mlx5_crypto_dek *dek =
> +			container_of(entry, typeof(*dek), entry);
> +	uint32_t key_len = dek->size_is_48 ? 48 : 80;
> +
> +	if (key_len != cipher_ctx->key.length)
> +		return -1;
> +	return memcmp(cipher_ctx->key.data, dek->data, key_len);
> +}
> +
> +static struct mlx5_list_entry *
> +mlx5_crypto_dek_create_cb(void *tool_ctx __rte_unused, void *cb_ctx)
> +{
> +	struct mlx5_crypto_dek_ctx *ctx = cb_ctx;
> +	struct rte_crypto_cipher_xform *cipher_ctx = ctx->cipher;
> +	struct mlx5_crypto_dek *dek = rte_zmalloc(__func__, sizeof(*dek),
> +						  RTE_CACHE_LINE_SIZE);
> +	struct mlx5_devx_dek_attr dek_attr = {
> +		.pd = ctx->priv->pdn,
> +		.key_purpose = MLX5_CRYPTO_KEY_PURPOSE_AES_XTS,
> +		.has_keytag = 1,
> +	};
> +
> +	if (dek == NULL) {
> +		DRV_LOG(ERR, "Failed to allocate dek memory.");
> +		return NULL;
> +	}
> +	switch (cipher_ctx->key.length) {
> +	case 48:
> +		dek->size_is_48 = true;
> +		dek_attr.key_size = MLX5_CRYPTO_KEY_SIZE_128b;
> +		break;
> +	case 80:
> +		dek->size_is_48 = false;
> +		dek_attr.key_size = MLX5_CRYPTO_KEY_SIZE_256b;
> +		break;
> +	default:
> +		DRV_LOG(ERR, "Key size not supported.");
> +		return NULL;
> +	}
> +	memcpy(&dek_attr.key, cipher_ctx->key.data, cipher_ctx-
> >key.length);
> +	dek->obj = mlx5_devx_cmd_create_dek_obj(ctx->priv->ctx,
> &dek_attr);
> +	if (dek->obj == NULL) {
> +		rte_free(dek);
> +		return NULL;
> +	}
> +	memcpy(&dek->data, cipher_ctx->key.data, cipher_ctx->key.length);
> +	return &dek->entry;
> +}
> +
> +static void
> +mlx5_crypto_dek_remove_cb(void *tool_ctx __rte_unused,
> +			  struct mlx5_list_entry *entry)
> +{
> +	struct mlx5_crypto_dek *dek =
> +		container_of(entry, typeof(*dek), entry);
> +
> +	claim_zero(mlx5_devx_cmd_destroy(dek->obj));
> +	rte_free(dek);
> +}
> +
> +

Extra line...

> +int
> +mlx5_crypto_dek_setup(struct mlx5_crypto_priv *priv)
> +{
> +	priv->dek_hlist = mlx5_hlist_create("dek_hlist",
> +				 MLX5_CRYPTO_DEK_HTABLE_SZ,
> +				 0, 1, NULL, mlx5_crypto_dek_create_cb,
> +				 mlx5_crypto_dek_match_cb,
> +				 mlx5_crypto_dek_remove_cb,
> +				 mlx5_crypto_dek_clone_cb,
> +				 mlx5_crypto_dek_clone_free_cb);
> +	if (priv->dek_hlist == NULL)
> +		return -1;
> +	return 0;
> +}
> +
> +void
> +mlx5_crypto_dek_unset(struct mlx5_crypto_priv *priv)
> +{
> +	mlx5_hlist_destroy(priv->dek_hlist);
> +	priv->dek_hlist = NULL;
> +}
> --
> 2.27.0

Re: [dpdk-dev] [EXT] [PATCH v8 04/16] crypto/mlx5: add basic operations

[Akhil Goyal]

> The basic dev control operations are configure, close and get info.
> 
> Extended the existing support of configure and close:
> 	-mlx5_crypto_dev_configure- function used to configure device.
> 	-mlx5_crypto_dev_close-  function used to close a configured
> 	 device.
> 
> Added config struct to user private data with the fields socket id,
> number of queue pairs and feature flags to be disabled.
> 
> Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
> Acked-by: Matan Azrad <matan@nvidia.com>
> ---
>  drivers/crypto/mlx5/mlx5_crypto.c | 26 +++++++++++++++++++-------
>  drivers/crypto/mlx5/mlx5_crypto.h |  1 +
>  2 files changed, 20 insertions(+), 7 deletions(-)
> 
> diff --git a/drivers/crypto/mlx5/mlx5_crypto.c
> b/drivers/crypto/mlx5/mlx5_crypto.c
> index 3f0c97d081..a7e44deb9e 100644
> --- a/drivers/crypto/mlx5/mlx5_crypto.c
> +++ b/drivers/crypto/mlx5/mlx5_crypto.c
> @@ -105,22 +105,27 @@ mlx5_crypto_dev_infos_get(struct rte_cryptodev
> *dev,
>  	}
>  }
> 
> -static unsigned int
> -mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev
> __rte_unused)
> -{
> -	return sizeof(struct mlx5_crypto_session);
> -}
> -

I do not get the reason to remove above function, it was introduced in the previous patch.
It looks the patches are not properly split.

>  static int
>  mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
> -		struct rte_cryptodev_config *config __rte_unused)
> +			  struct rte_cryptodev_config *config)
>  {
>  	struct mlx5_crypto_priv *priv = dev->data->dev_private;
> 
> +	if (config == NULL) {
> +		DRV_LOG(ERR, "Invalid crypto dev configure parameters.");
> +		return -EINVAL;
> +	}
> +	if ((config->ff_disable & RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO)
> != 0) {
> +		DRV_LOG(ERR,
> +			"Disabled symmetric crypto feature is not
> supported.");
> +		return -ENOTSUP;
> +	}
>  	if (mlx5_crypto_dek_setup(priv) != 0) {
>  		DRV_LOG(ERR, "Dek hash list creation has failed.");
>  		return -ENOMEM;
>  	}
> +	priv->dev_config = *config;
> +	DRV_LOG(DEBUG, "Device %u was configured.", dev->driver_id);
>  	return 0;
>  }

The patch title and the patch do not match.
Title says, add basic operations, which should introduce the
Configure and close ops. But here the configure and close ops
Were already there and you are introducing some new checks
In them.

> 
> @@ -130,9 +135,16 @@ mlx5_crypto_dev_close(struct rte_cryptodev *dev)
>  	struct mlx5_crypto_priv *priv = dev->data->dev_private;
> 
>  	mlx5_crypto_dek_unset(priv);
> +	DRV_LOG(DEBUG, "Device %u was closed.", dev->driver_id);
>  	return 0;
>  }
Logging could have been added in the patch where dev_close was added.

> 
> +static unsigned int
> +mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev
> __rte_unused)
> +{
> +	return sizeof(struct mlx5_crypto_session);
> +}
> +
>  static int
>  mlx5_crypto_sym_session_configure(struct rte_cryptodev *dev,
>  				  struct rte_crypto_sym_xform *xform,
> diff --git a/drivers/crypto/mlx5/mlx5_crypto.h
> b/drivers/crypto/mlx5/mlx5_crypto.h
> index 167e9e57ad..a0df775407 100644
> --- a/drivers/crypto/mlx5/mlx5_crypto.h
> +++ b/drivers/crypto/mlx5/mlx5_crypto.h
> @@ -24,6 +24,7 @@ struct mlx5_crypto_priv {
>  	uint32_t pdn; /* Protection Domain number. */
>  	struct ibv_pd *pd;
>  	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
> +	struct rte_cryptodev_config dev_config;
>  };
> 
>  struct mlx5_crypto_dek {
> --
> 2.27.0

Re: [dpdk-dev] [EXT] [PATCH v8 03/16] crypto/mlx5: add session operations

[Akhil Goyal]

> Sessions are used in symmetric transformations in order to prepare
> objects and data for packet processing stage.
> 
> A mlx5 session includes iv_offset, pointer to mlx5_crypto_dek struct,
> bsf_size, bsf_p_type, block size index, encryption_order and encryption
> standard.
> 
> Implement the next session operations:
>         mlx5_crypto_sym_session_get_size- returns the size of the mlx5
> 	session struct.
> 	mlx5_crypto_sym_session_configure- prepares the DEK hash-list
> 	and saves all the session data.
> 	mlx5_crypto_sym_session_clear - destroys the DEK hash-list.
> 
> Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
> Acked-by: Matan Azrad <matan@nvidia.com>
> ---
>  doc/guides/cryptodevs/features/mlx5.ini |   5 +
>  doc/guides/cryptodevs/mlx5.rst          |  10 ++
>  drivers/crypto/mlx5/mlx5_crypto.c       | 172 +++++++++++++++++++++++-
>  3 files changed, 182 insertions(+), 5 deletions(-)
> 
> diff --git a/doc/guides/cryptodevs/features/mlx5.ini
> b/doc/guides/cryptodevs/features/mlx5.ini
> index ceadd967b6..bd757b5211 100644
> --- a/doc/guides/cryptodevs/features/mlx5.ini
> +++ b/doc/guides/cryptodevs/features/mlx5.ini
> @@ -4,12 +4,17 @@
>  ; Refer to default.ini for the full list of available PMD features.
>  ;
>  [Features]
> +Symmetric crypto       = Y
>  HW Accelerated         = Y
> +Cipher multiple data units = Y
> +Cipher wrapped key     = Y
> 
>  ;
>  ; Supported crypto algorithms of a mlx5 crypto driver.
>  ;
>  [Cipher]
> +AES XTS (128)  = Y
> +AES XTS (256)  = Y
> 
>  ;
>  ; Supported authentication algorithms of a mlx5 crypto driver.
> diff --git a/doc/guides/cryptodevs/mlx5.rst
> b/doc/guides/cryptodevs/mlx5.rst
> index 05a0a449e2..dd1d1a615d 100644
> --- a/doc/guides/cryptodevs/mlx5.rst
> +++ b/doc/guides/cryptodevs/mlx5.rst
> @@ -53,6 +53,16 @@ Supported NICs
> 
>  * Mellanox\ |reg| ConnectX\ |reg|-6 200G MCX654106A-HCAT (2x200G)
> 
> +
> +Limitations
> +-----------
> +
> +- AES-XTS keys provided in xform must include keytag and should be
> wrappend.

wrapped

> +- The supported data-unit lengths are 512B and 1KB. In case the
> `dataunit_len`
> +  is not provided in the cipher xform, the OP length is limited to the above
> +  values and 1MB.
> +
> +
>  Prerequisites
>  -------------
> 
> diff --git a/drivers/crypto/mlx5/mlx5_crypto.c
> b/drivers/crypto/mlx5/mlx5_crypto.c
> index d2d82c7b15..3f0c97d081 100644
> --- a/drivers/crypto/mlx5/mlx5_crypto.c
> +++ b/drivers/crypto/mlx5/mlx5_crypto.c
> @@ -3,6 +3,7 @@
>   */
> 
>  #include <rte_malloc.h>
> +#include <rte_mempool.h>
>  #include <rte_errno.h>
>  #include <rte_log.h>
>  #include <rte_pci.h>
> @@ -20,7 +21,9 @@
>  #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
> 
>  #define MLX5_CRYPTO_FEATURE_FLAGS \
> -	RTE_CRYPTODEV_FF_HW_ACCELERATED
> +	(RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
> RTE_CRYPTODEV_FF_HW_ACCELERATED | \
> +	 RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY | \
> +	 RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS)
> 
>  TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
> 
> 	TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
> @@ -30,6 +33,32 @@ int mlx5_crypto_logtype;
> 
>  uint8_t mlx5_crypto_driver_id;
> 
> +const struct rte_cryptodev_capabilities mlx5_crypto_caps[] = {
> +	{		/* AES XTS */
> +		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
> +		{.sym = {
> +			.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
> +			{.cipher = {
> +				.algo = RTE_CRYPTO_CIPHER_AES_XTS,
> +				.block_size = 16,
> +				.key_size = {
> +					.min = 32,
> +					.max = 64,
> +					.increment = 32
> +				},
> +				.iv_size = {
> +					.min = 16,
> +					.max = 16,
> +					.increment = 0
> +				},
> +				.dataunit_set =
> +
> 	RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_512_BYTES |
> +
> 	RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_4096_BYTES,
> +			}, }
> +		}, }
> +	},
> +};
> +
>  static const char mlx5_crypto_drv_name[] =
> RTE_STR(MLX5_CRYPTO_DRIVER_NAME);
> 
>  static const struct rte_driver mlx5_drv = {
> @@ -39,6 +68,49 @@ static const struct rte_driver mlx5_drv = {
> 
>  static struct cryptodev_driver mlx5_cryptodev_driver;
> 
> +struct mlx5_crypto_session {
> +	uint32_t bs_bpt_eo_es;
> +	/*
> +	 * bsf_size, bsf_p_type, encryption_order and encryption standard,
> +	 * saved in big endian format.
> +	 */

Normally the comments are added before the variable. Or add
/**< for post comment.

> +	uint32_t bsp_res;
> +	/*
> +	 * crypto_block_size_pointer and reserved 24 bits saved in big endian
> +	 * format.
> +	 */
> +	uint32_t iv_offset:16;
> +	/* Starting point for Initialisation Vector. */
> +	struct mlx5_crypto_dek *dek; /* Pointer to dek struct. */
> +	uint32_t dek_id; /* DEK ID */
> +} __rte_packed;
> +
> +static void
> +mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev,
> +			  struct rte_cryptodev_info *dev_info)
> +{
> +	RTE_SET_USED(dev);
> +	if (dev_info != NULL) {
> +		dev_info->driver_id = mlx5_crypto_driver_id;
> +		dev_info->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
> +		dev_info->capabilities = mlx5_crypto_caps;
> +		dev_info->max_nb_queue_pairs = 0;
> +		dev_info->min_mbuf_headroom_req = 0;
> +		dev_info->min_mbuf_tailroom_req = 0;
> +		dev_info->sym.max_nb_sessions = 0;
> +		/*
> +		 * If 0, the device does not have any limitation in number of
> +		 * sessions that can be used.
> +		 */
> +	}
> +}
> +
> +static unsigned int
> +mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev
> __rte_unused)
> +{
> +	return sizeof(struct mlx5_crypto_session);
> +}
> +
>  static int
>  mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
>  		struct rte_cryptodev_config *config __rte_unused)
> @@ -61,19 +133,109 @@ mlx5_crypto_dev_close(struct rte_cryptodev *dev)
>  	return 0;
>  }
> 
> +static int
> +mlx5_crypto_sym_session_configure(struct rte_cryptodev *dev,
> +				  struct rte_crypto_sym_xform *xform,
> +				  struct rte_cryptodev_sym_session *session,
> +				  struct rte_mempool *mp)
> +{
> +	struct mlx5_crypto_priv *priv = dev->data->dev_private;
> +	struct mlx5_crypto_session *sess_private_data;
> +	struct rte_crypto_cipher_xform *cipher;
> +	uint8_t encryption_order;
> +	int ret;
> +
> +	if (unlikely(xform->next != NULL)) {
> +		DRV_LOG(ERR, "Xform next is not supported.");
> +		return -ENOTSUP;
> +	}
> +	if (unlikely((xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) ||
> +		     (xform->cipher.algo != RTE_CRYPTO_CIPHER_AES_XTS))) {
> +		DRV_LOG(ERR, "Only AES-XTS algorithm is supported.");
> +		return -ENOTSUP;
> +	}
> +	ret = rte_mempool_get(mp, (void *)&sess_private_data);
> +	if (ret != 0) {
> +		DRV_LOG(ERR,
> +			"Failed to get session %p private data from
> mempool.",
> +			sess_private_data);
> +		return -ENOMEM;
> +	}
> +	cipher = &xform->cipher;
> +	sess_private_data->dek = mlx5_crypto_dek_prepare(priv, cipher);
> +	if (sess_private_data->dek == NULL) {
> +		rte_mempool_put(mp, sess_private_data);
> +		DRV_LOG(ERR, "Failed to prepare dek.");
> +		return -ENOMEM;
> +	}
> +	if (cipher->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
> +		encryption_order =
> MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_MEMORY;
> +	else
> +		encryption_order =
> MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_WIRE;
> +	sess_private_data->bs_bpt_eo_es = rte_cpu_to_be_32
> +			(MLX5_BSF_SIZE_64B << MLX5_BSF_SIZE_OFFSET |
> +			 MLX5_BSF_P_TYPE_CRYPTO <<
> MLX5_BSF_P_TYPE_OFFSET |
> +			 encryption_order <<
> MLX5_ENCRYPTION_ORDER_OFFSET |
> +			 MLX5_ENCRYPTION_STANDARD_AES_XTS);
> +	switch (xform->cipher.dataunit_len) {
> +	case 0:
> +		sess_private_data->bsp_res = 0;
> +		break;
> +	case 512:
> +		sess_private_data->bsp_res = rte_cpu_to_be_32
> +
> ((uint32_t)MLX5_BLOCK_SIZE_512B <<
> +					     MLX5_BLOCK_SIZE_OFFSET);
> +		break;
> +	case 4096:
> +		sess_private_data->bsp_res = rte_cpu_to_be_32
> +
> ((uint32_t)MLX5_BLOCK_SIZE_4096B <<
> +					     MLX5_BLOCK_SIZE_OFFSET);
> +		break;
> +	default:
> +		DRV_LOG(ERR, "Cipher data unit length is not supported.");
> +		return -ENOTSUP;
> +	}
> +	sess_private_data->iv_offset = cipher->iv.offset;
> +	sess_private_data->dek_id =
> +			rte_cpu_to_be_32(sess_private_data->dek->obj->id
> &
> +					 0xffffff);
> +	set_sym_session_private_data(session, dev->driver_id,
> +				     sess_private_data);
> +	DRV_LOG(DEBUG, "Session %p was configured.", sess_private_data);
> +	return 0;
> +}
> +
> +static void
> +mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev,
> +			      struct rte_cryptodev_sym_session *sess)
> +{
> +	struct mlx5_crypto_priv *priv = dev->data->dev_private;
> +	struct mlx5_crypto_session *spriv =
> get_sym_session_private_data(sess,
> +								dev-
> >driver_id);
> +
> +	if (unlikely(spriv == NULL)) {
> +		DRV_LOG(ERR, "Failed to get session %p private data.",
> spriv);
> +		return;
> +	}
> +	mlx5_crypto_dek_destroy(priv, spriv->dek);
> +	set_sym_session_private_data(sess, dev->driver_id, NULL);
> +	rte_mempool_put(rte_mempool_from_obj(spriv), spriv);
> +	DRV_LOG(DEBUG, "Session %p was cleared.", spriv);
> +}
> +
>  static struct rte_cryptodev_ops mlx5_crypto_ops = {
>  	.dev_configure			= mlx5_crypto_dev_configure,
>  	.dev_start			= NULL,
>  	.dev_stop			= NULL,
>  	.dev_close			= mlx5_crypto_dev_close,
> -	.dev_infos_get			= NULL,
> +	.dev_infos_get			= mlx5_crypto_dev_infos_get,
>  	.stats_get			= NULL,
>  	.stats_reset			= NULL,
>  	.queue_pair_setup		= NULL,
>  	.queue_pair_release		= NULL,
> -	.sym_session_get_size		= NULL,
> -	.sym_session_configure		= NULL,
> -	.sym_session_clear		= NULL,
> +	.sym_session_get_size		= mlx5_crypto_sym_session_get_size,
> +	.sym_session_configure		=
> mlx5_crypto_sym_session_configure,
> +	.sym_session_clear		= mlx5_crypto_sym_session_clear,
>  	.sym_get_raw_dp_ctx_size	= NULL,
>  	.sym_configure_raw_dp_ctx	= NULL,
>  };
> --
> 2.27.0

Re: [dpdk-dev] [EXT] [PATCH v8 03/16] crypto/mlx5: add session operations

[Akhil Goyal]

> Sessions are used in symmetric transformations in order to prepare
> objects and data for packet processing stage.
> 
> A mlx5 session includes iv_offset, pointer to mlx5_crypto_dek struct,
> bsf_size, bsf_p_type, block size index, encryption_order and encryption
> standard.
> 
> Implement the next session operations:
>         mlx5_crypto_sym_session_get_size- returns the size of the mlx5
> 	session struct.
> 	mlx5_crypto_sym_session_configure- prepares the DEK hash-list
> 	and saves all the session data.
> 	mlx5_crypto_sym_session_clear - destroys the DEK hash-list.
> 
> Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
> Acked-by: Matan Azrad <matan@nvidia.com>
> ---
>  doc/guides/cryptodevs/features/mlx5.ini |   5 +
>  doc/guides/cryptodevs/mlx5.rst          |  10 ++

Documentation update is done in 3/16 which means the feature is complete.
But basic operations and queue pair initialization is done later.
This is not good. Please move this patch after QP additions.
Please take reference of other drivers for sequence of patches.

>  drivers/crypto/mlx5/mlx5_crypto.c       | 172 +++++++++++++++++++++++-
>  3 files changed, 182 insertions(+), 5 deletions(-)
> 

Re: [dpdk-dev] [EXT] [PATCH v8 06/16] crypto/mlx5: add dev stop and start operations

[Akhil Goyal]

> Add the dev_start function that is used to start a configured device.
> Add the dev_stop function that is used to stop a configured device.
> 
> Both functions set the dev parameter as used and return 0.
> 
> Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
> Acked-by: Matan Azrad <matan@nvidia.com>
> ---
>  drivers/crypto/mlx5/mlx5_crypto.c | 17 +++++++++++++++--
>  1 file changed, 15 insertions(+), 2 deletions(-)
> 
> diff --git a/drivers/crypto/mlx5/mlx5_crypto.c
> b/drivers/crypto/mlx5/mlx5_crypto.c
> index ebaa65c7a9..37e66cf57b 100644
> --- a/drivers/crypto/mlx5/mlx5_crypto.c
> +++ b/drivers/crypto/mlx5/mlx5_crypto.c
> @@ -131,6 +131,19 @@ mlx5_crypto_dev_configure(struct rte_cryptodev
> *dev,
>  	return 0;
>  }
> 
> +static void
> +mlx5_crypto_dev_stop(struct rte_cryptodev *dev)
> +{
> +	RTE_SET_USED(dev);
> +}
> +
> +static int
> +mlx5_crypto_dev_start(struct rte_cryptodev *dev)
> +{
> +	RTE_SET_USED(dev);
> +	return 0;
> +}
> +
>  static int
>  mlx5_crypto_dev_close(struct rte_cryptodev *dev)
>  {
> @@ -360,8 +373,8 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev
> *dev, uint16_t qp_id,
> 
>  static struct rte_cryptodev_ops mlx5_crypto_ops = {
>  	.dev_configure			= mlx5_crypto_dev_configure,
> -	.dev_start			= NULL,
> -	.dev_stop			= NULL,
> +	.dev_start			= mlx5_crypto_dev_start,
> +	.dev_stop			= mlx5_crypto_dev_stop,
>  	.dev_close			= mlx5_crypto_dev_close,
>  	.dev_infos_get			= mlx5_crypto_dev_infos_get,
>  	.stats_get			= NULL,


This patch could be squashed in your basic operations patch as these ops are not doing 
Anything.

Re: [dpdk-dev] [EXT] [PATCH v8 14/16] test/crypto: add mlx5 crypto driver

[Akhil Goyal]

> diff --git a/app/test/test_cryptodev.c b/app/test/test_cryptodev.c
> index 31201d93e1..e46668a1e0 100644
> --- a/app/test/test_cryptodev.c
> +++ b/app/test/test_cryptodev.c
> @@ -14569,6 +14569,12 @@ test_cryptodev_cpu_aesni_gcm(void)
>  	return rc;
>  }
> 
> +static int
> +test_cryptodev_mlx5(void /*argv __rte_unused, int argc __rte_unused*/)

Remove unwanted comments.
Rebase over current TOT

> +{
> +	return
> run_cryptodev_testsuite(RTE_STR(CRYPTODEV_NAME_MLX5_PMD));
> +}



> diff --git a/app/test/test_cryptodev.h b/app/test/test_cryptodev.h
> index 5bf1e8818b..262e545ee6 100644
> --- a/app/test/test_cryptodev.h
> +++ b/app/test/test_cryptodev.h
> @@ -73,6 +73,8 @@
>  #define CRYPTODEV_NAME_BCMFS_PMD	crypto_bcmfs
>  #define CRYPTODEV_NAME_CN9K_PMD		crypto_cn9k
>  #define CRYPTODEV_NAME_CN10K_PMD	crypto_cn10k
> +#define CRYPTODEV_NAME_MLX5_PMD		mlx5_crypto

Follow convention
It should be crypto_mlx5

Re: [dpdk-dev] [EXT] [PATCH v8 15/16] test/crypto: add data-unit and wrapped vectors

[Akhil Goyal]

> The AES-XTS algorithm supports using a wrapped key.
> In AES-XTS the data-unit defines the data block size to be
> encrypted\decrypted.
> 
> Add AES-XTS vectors with a wrapped key.
> Add a variable stating whether the key is wrapped or not.
> Add the AES-XTS data-unit.
> 
> Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
> Acked-by: Matan Azrad <matan@nvidia.com>
> ---
>  app/test/test_cryptodev.h                  |    2 +-
>  app/test/test_cryptodev_aes_test_vectors.h | 1340 ++++++++++++++++++++
>  app/test/test_cryptodev_blockcipher.c      |   10 +-
>  app/test/test_cryptodev_blockcipher.h      |    2 +
>  4 files changed, 1352 insertions(+), 2 deletions(-)
> 
> diff --git a/app/test/test_cryptodev.h b/app/test/test_cryptodev.h
> index 262e545ee6..f35dba6837 100644
> --- a/app/test/test_cryptodev.h
> +++ b/app/test/test_cryptodev.h
> @@ -19,7 +19,7 @@
>  #define DEFAULT_NUM_XFORMS              (2)
>  #define NUM_MBUFS                       (8191)
>  #define MBUF_CACHE_SIZE                 (256)
> -#define MBUF_DATAPAYLOAD_SIZE		(2048 +
> DIGEST_BYTE_LENGTH_SHA512)
> +#define MBUF_DATAPAYLOAD_SIZE		(4096 +
> DIGEST_BYTE_LENGTH_SHA512)

Any specific reason this size is increased. It may negatively impact on other platforms
Which have lesser memory available.
We need a reason to change the existing MACROS which affect all drivers.

Please rebase, I will need to test these patches on other platforms to ensure it is not
Broken.

Re: [dpdk-dev] [EXT] [PATCH v8 16/16] test/crypto: add AES-XTS multi segment OOP tests

[Akhil Goyal]

> The AES-XTS algorithm can supports wrapped key and data-unit.
> The encryption/decryption can be done out of place and using multi
> segments.
> 
> Add multi segment and out of place tests to the recently added AES-XTS
> vectors, which support using data-unit and a wrapped key.
> 
> Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
> Acked-by: Matan Azrad <matan@nvidia.com>

@Fan/Ciara: Can you test these test app patches on intel platforms and provide
Feedback on these patches.

> ---
>  app/test/test_cryptodev_aes_test_vectors.h | 64 ++++++++++++++++++++++
>  1 file changed, 64 insertions(+)
> 
> diff --git a/app/test/test_cryptodev_aes_test_vectors.h
> b/app/test/test_cryptodev_aes_test_vectors.h
> index dd7ffde685..a797af1b00 100644
> --- a/app/test/test_cryptodev_aes_test_vectors.h
> +++ b/app/test/test_cryptodev_aes_test_vectors.h
> @@ -4687,6 +4687,70 @@ static const struct blockcipher_test_case
> aes_cipheronly_test_cases[] = {
>  		.test_data =
> &aes_test_data_xts_wrapped_key_48_pt_4096_du_0,
>  		.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
>  	},
> +	{
> +		.test_descr = "AES-256-XTS Encryption (512-byte plaintext"
> +			      " Dataunit 512) Scater gather OOP",

Spell check for scatter at all places.

> +		.test_data =
> &aes_test_data_xts_wrapped_key_48_pt_512_du_512,
> +		.op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT,
> +		.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP |
> +				BLOCKCIPHER_TEST_FEATURE_SG,
> +	},
> +	{
> +		.test_descr = "AES-256-XTS Decryption (512-byte plaintext"
> +			      " Dataunit 512) Scater gather OOP",
> +		.test_data =
> &aes_test_data_xts_wrapped_key_48_pt_512_du_512,
> +		.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
> +		.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP |
> +				BLOCKCIPHER_TEST_FEATURE_SG,
> +	},
> +	{
> +		.test_descr = "AES-256-XTS Encryption (512-byte plaintext"
> +			      " Dataunit 0) Scater gather OOP",
> +		.test_data =
> &aes_test_data_xts_wrapped_key_48_pt_512_du_0,
> +		.op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT,
> +		.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP |
> +				BLOCKCIPHER_TEST_FEATURE_SG,
> +	},
> +	{
> +		.test_descr = "AES-256-XTS Decryption (512-byte plaintext"
> +			      " Dataunit 0) Scater gather OOP",
> +		.test_data =
> &aes_test_data_xts_wrapped_key_48_pt_512_du_0,
> +		.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
> +		.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP |
> +				BLOCKCIPHER_TEST_FEATURE_SG,
> +	},
> +	{
> +		.test_descr = "AES-256-XTS Encryption (4096-byte plaintext"
> +			      " Dataunit 4096) Scater gather OOP",
> +		.test_data =
> &aes_test_data_xts_wrapped_key_48_pt_4096_du_4096,
> +		.op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT,
> +		.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP |
> +				BLOCKCIPHER_TEST_FEATURE_SG,
> +	},
> +	{
> +		.test_descr = "AES-256-XTS Decryption (4096-byte plaintext"
> +			      " Dataunit 4096) Scater gather OOP",
> +		.test_data =
> &aes_test_data_xts_wrapped_key_48_pt_4096_du_4096,
> +		.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
> +		.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP |
> +				BLOCKCIPHER_TEST_FEATURE_SG,
> +	},
> +	{
> +		.test_descr = "AES-256-XTS Encryption (4096-byte plaintext"
> +			      " Dataunit 0) Scater gather OOP",
> +		.test_data =
> &aes_test_data_xts_wrapped_key_48_pt_4096_du_0,
> +		.op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT,
> +		.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP |
> +				BLOCKCIPHER_TEST_FEATURE_SG,
> +	},
> +	{
> +		.test_descr = "AES-256-XTS Decryption (4096-byte plaintext"
> +			      " Dataunit 0) Scater gather OOP",
> +		.test_data =
> &aes_test_data_xts_wrapped_key_48_pt_4096_du_0,
> +		.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
> +		.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP |
> +				BLOCKCIPHER_TEST_FEATURE_SG,
> +	},
>  	{
>  		.test_descr = "cipher-only - NULL algo - x8 - encryption",
>  		.test_data = &null_test_data_chain_x8_multiple,
> --
> 2.27.0

Re: [dpdk-dev] [EXT] [PATCH v8 00/16] drivers: introduce mlx5 crypto PMD

[Akhil Goyal]

> > v2:
> > - Add data-path part.
> >
> > v3:
> > - Rebase.
> >
> > v4:
> > - Rebase + Address the following Akhil comments:
> > - Set HW feature flag in the capability patch.
> > - Fix mp object release in session clear.
> > - Some spelling and word missing in doc.
> > - Squash data-unit adjustment to the session operations commit.
> > - Wording: device argument -> devarg.
> >
> > v5:
> > - Add mlx5 crypto tests into test library.
> > - Update documentation according to Akhil comments.
> > - Fix memory region management.
> > - Fix multi segment case in data-path code.
> > - Split documentation to the correct commits according to Akhil comments.
> > - Rebase to new version.
> > - Change license to Nvidia license.
> >
> > v6:
> > - Rebase to new version.
> > - Address David's comment and update log accordingly.
> > - Update testing app commits as suggested by Akhil.
> >
> > v7:
> > - Rebase to new version.
> > - Add scatter gather and OOP test cases.
> >
> > v8:
> > - Add a test case that was removed by error.
> >
> >
> >
> > Shiri Kuzin (11):
> >   drivers: introduce mlx5 crypto PMD
> >   crypto/mlx5: add DEK object management
> 
> This is really bad, I was planning to apply this series today,
> It had small issues, so was planning to fix them while applying.
> But when I applied the 2/16, it does not compile.
> Please fix this, we are into v8 and compilation is broken.
> 
> I am adding few more comments, please fix them too in your next version
> As soon as possible. I was planning to close RC2 during weekend, but it looks
> It will fall over next week.
> 
Please review this patchset thoroughly within Nvidia before sending next version.
Take reference to other crypto PMDs for patch splitting and sequencing.

Re: [dpdk-dev] [EXT] [PATCH v8 00/16] drivers: introduce mlx5 crypto PMD

[Akhil Goyal]

> 
> > > v2:
> > > - Add data-path part.
> > >
> > > v3:
> > > - Rebase.
> > >
> > > v4:
> > > - Rebase + Address the following Akhil comments:
> > > - Set HW feature flag in the capability patch.
> > > - Fix mp object release in session clear.
> > > - Some spelling and word missing in doc.
> > > - Squash data-unit adjustment to the session operations commit.
> > > - Wording: device argument -> devarg.
> > >
> > > v5:
> > > - Add mlx5 crypto tests into test library.
> > > - Update documentation according to Akhil comments.
> > > - Fix memory region management.
> > > - Fix multi segment case in data-path code.
> > > - Split documentation to the correct commits according to Akhil
> comments.
> > > - Rebase to new version.
> > > - Change license to Nvidia license.
> > >
> > > v6:
> > > - Rebase to new version.
> > > - Address David's comment and update log accordingly.
> > > - Update testing app commits as suggested by Akhil.
> > >
> > > v7:
> > > - Rebase to new version.
> > > - Add scatter gather and OOP test cases.
> > >
> > > v8:
> > > - Add a test case that was removed by error.
> > >
> > >
> > >
> > > Shiri Kuzin (11):
> > >   drivers: introduce mlx5 crypto PMD
> > >   crypto/mlx5: add DEK object management
> >
> > This is really bad, I was planning to apply this series today,
> > It had small issues, so was planning to fix them while applying.
> > But when I applied the 2/16, it does not compile.
> > Please fix this, we are into v8 and compilation is broken.
> >
> > I am adding few more comments, please fix them too in your next version
> > As soon as possible. I was planning to close RC2 during weekend, but it
> looks
> > It will fall over next week.
> >
> Please review this patchset thoroughly within Nvidia before sending next
> version.
> Take reference to other crypto PMDs for patch splitting and sequencing.

Are we expecting next version today?
This is the only major patchset left for RC2.
Any update?

Re: [dpdk-dev] [EXT] [PATCH v8 00/16] drivers: introduce mlx5 crypto PMD

[Suanming Mou]

Hi Akhil,

> -----Original Message-----
> From: Akhil Goyal <gakhil@marvell.com>
> Sent: Tuesday, July 20, 2021 4:14 PM
> To: Shiri Kuzin <shirik@nvidia.com>; NBU-Contact-Thomas Monjalon
> <thomas@monjalon.net>; Matan Azrad <matan@nvidia.com>; Suanming Mou
> <suanmingm@nvidia.com>
> Cc: david.marchand@redhat.com; dev@dpdk.org
> Subject: RE: [EXT] [PATCH v8 00/16] drivers: introduce mlx5 crypto PMD
> 
> 
> >
> > > > v2:
> > > > - Add data-path part.
> > > >
> > > > v3:
> > > > - Rebase.
> > > >
> > > > v4:
> > > > - Rebase + Address the following Akhil comments:
> > > > - Set HW feature flag in the capability patch.
> > > > - Fix mp object release in session clear.
> > > > - Some spelling and word missing in doc.
> > > > - Squash data-unit adjustment to the session operations commit.
> > > > - Wording: device argument -> devarg.
> > > >
> > > > v5:
> > > > - Add mlx5 crypto tests into test library.
> > > > - Update documentation according to Akhil comments.
> > > > - Fix memory region management.
> > > > - Fix multi segment case in data-path code.
> > > > - Split documentation to the correct commits according to Akhil
> > comments.
> > > > - Rebase to new version.
> > > > - Change license to Nvidia license.
> > > >
> > > > v6:
> > > > - Rebase to new version.
> > > > - Address David's comment and update log accordingly.
> > > > - Update testing app commits as suggested by Akhil.
> > > >
> > > > v7:
> > > > - Rebase to new version.
> > > > - Add scatter gather and OOP test cases.
> > > >
> > > > v8:
> > > > - Add a test case that was removed by error.
> > > >
> > > >
> > > >
> > > > Shiri Kuzin (11):
> > > >   drivers: introduce mlx5 crypto PMD
> > > >   crypto/mlx5: add DEK object management
> > >
> > > This is really bad, I was planning to apply this series today, It
> > > had small issues, so was planning to fix them while applying.
> > > But when I applied the 2/16, it does not compile.
> > > Please fix this, we are into v8 and compilation is broken.
> > >
> > > I am adding few more comments, please fix them too in your next
> > > version As soon as possible. I was planning to close RC2 during
> > > weekend, but it
> > looks
> > > It will fall over next week.
> > >
> > Please review this patchset thoroughly within Nvidia before sending
> > next version.
> > Take reference to other crypto PMDs for patch splitting and sequencing.
> 
> Are we expecting next version today?
> This is the only major patchset left for RC2.
> Any update?

Sorry for the late reply, Shiri is not available these days,  so I'm trying to follow up the status and update the new version later today.
Regarding the compilation failure, it's due to the crypto patch series depends on some mlx5 common code. 
The mlx5 common code has some changes and I assume Shiri rebased the version with the dpdk-next-mlx and meanwhile the dpdk-next-crypto branch still did not get the latest mlx5 common code.

I just checked the dpdk-next-crypto branch, seems the mlx5 part is still not updated yet. So will you also sync up the crypto branch today? If no, then the failure will come again.
I'm not so familiar with the branch update procedure, but we will need the branch to be  updated to the version below:
http://git.dpdk.org/next/dpdk-next-net-mlx/commit/?id=7e1cf892711b9ba237563b6c876382caaec434f7

Thanks,
SuanmingMou

Re: [dpdk-dev] [EXT] [PATCH v8 01/16] drivers: introduce mlx5 crypto PMD

[Suanming Mou]

> -----Original Message-----
> From: Akhil Goyal <gakhil@marvell.com>
> Sent: Saturday, July 17, 2021 3:19 AM
> To: Shiri Kuzin <shirik@nvidia.com>; dev@dpdk.org
> Cc: Matan Azrad <matan@nvidia.com>; Suanming Mou
> <suanmingm@nvidia.com>; david.marchand@redhat.com
> Subject: RE: [EXT] [PATCH v8 01/16] drivers: introduce mlx5 crypto PMD
> 
> > diff --git a/doc/guides/rel_notes/release_21_08.rst
> > b/doc/guides/rel_notes/release_21_08.rst
> > index 7d289e07e3..2bf4ce7a73 100644
> > --- a/doc/guides/rel_notes/release_21_08.rst
> > +++ b/doc/guides/rel_notes/release_21_08.rst
> > @@ -125,6 +125,11 @@ New Features
> >    The experimental PMD power management API now supports managing
> >    multiple Ethernet Rx queues per lcore.
> >
> > +* **Added support for Nvidia crypto device driver.**
> > +
> > +  * Added mlx5 crypto driver to support AES-XTS cipher operations.
> > +    The first device to support it is ConnectX-6.
> > +
> 
> Update release notes when the feature is complete.
> Probably in your enq/deq patch (12/16)

I assume it should  be (13/16)? Since this patch finishes implementing all the callbacks.

> 
> 
> > +static const struct rte_pci_id mlx5_crypto_pci_id_map[] = {
> > +		{
> > +			RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
> > +
> > 	PCI_DEVICE_ID_MELLANOX_CONNECTX6)
> > +		},
> > +		{
> > +			.vendor_id = 0
> > +		}
> > +	};
> 
> Fix indentation.

Good catch.

> 
> > +

Re: [dpdk-dev] [EXT] [PATCH v8 02/16] crypto/mlx5: add DEK object management

[Suanming Mou]

> -----Original Message-----
> From: Akhil Goyal <gakhil@marvell.com>
> Sent: Saturday, July 17, 2021 3:26 AM
> To: Shiri Kuzin <shirik@nvidia.com>; dev@dpdk.org
> Cc: Matan Azrad <matan@nvidia.com>; Suanming Mou
> <suanmingm@nvidia.com>; david.marchand@redhat.com
> Subject: RE: [EXT] [PATCH v8 02/16] crypto/mlx5: add DEK object management
> 
> > A DEK(Data encryption Key) is an mlx5 HW object which represents the
> > cipher algorithm key.
> > The DEKs are used during data encryption/decryption operations.
> >
> > In symmetric algorithms like AES-STS, we use the same DEK for both
> > encryption and decryption.
> >
> > Use the mlx5 hash-list tool to manage the DEK objects in the PMD.
> >
> > Provide the compare, create and destroy functions to manage DEKs in
> > hash-list and introduce an internal API to setup and unset the DEK
> > management and to prepare and destroy specific DEK object.
> >
> > The DEK hash-list will be created in dev_configure routine and
> > destroyed in dev_close routine.
> >
> > Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
> > Acked-by: Matan Azrad <matan@nvidia.com>
> > ---
> >  drivers/crypto/mlx5/meson.build       |   1 +
> >  drivers/crypto/mlx5/mlx5_crypto.c     |  42 ++++---
> >  drivers/crypto/mlx5/mlx5_crypto.h     |  51 ++++++++
> >  drivers/crypto/mlx5/mlx5_crypto_dek.c | 161
> > ++++++++++++++++++++++++++
> >  4 files changed, 239 insertions(+), 16 deletions(-)  create mode
> > 100644 drivers/crypto/mlx5/mlx5_crypto.h  create mode 100644
> > drivers/crypto/mlx5/mlx5_crypto_dek.c
> >
> > diff --git a/drivers/crypto/mlx5/meson.build
> > b/drivers/crypto/mlx5/meson.build index 6fd70bc477..d55cdbfe6f 100644
> > --- a/drivers/crypto/mlx5/meson.build
> > +++ b/drivers/crypto/mlx5/meson.build
> > @@ -11,6 +11,7 @@ fmt_name = 'mlx5_crypto'
> >  deps += ['common_mlx5', 'eal', 'cryptodev']  sources = files(
> >  	'mlx5_crypto.c',
> > +	'mlx5_crypto_dek.c',
> >  )
> >  cflags_options = [
> >  	'-std=c11',
> > diff --git a/drivers/crypto/mlx5/mlx5_crypto.c
> > b/drivers/crypto/mlx5/mlx5_crypto.c
> > index fbe3c21aae..d2d82c7b15 100644
> > --- a/drivers/crypto/mlx5/mlx5_crypto.c
> > +++ b/drivers/crypto/mlx5/mlx5_crypto.c
> > @@ -3,12 +3,9 @@
> >   */
> >
> >  #include <rte_malloc.h>
> > -#include <rte_log.h>
> >  #include <rte_errno.h>
> > +#include <rte_log.h>
> >  #include <rte_pci.h>
> > -#include <rte_crypto.h>
> > -#include <rte_cryptodev.h>
> > -#include <rte_cryptodev_pmd.h>
> 
> There is some issue in the splitting of the patches, The above headers are added
> in first patch and moved to a header file in this patch.
> Take reference of the cnxk crypto driver which got merged recently.

The main reason is that in the patch we add a new c file: drivers/crypto/mlx5/mlx5_crypto_dek.c
Now mlx5_crypto.c and that new c file both share the new h file mlx5_crypto.h, so all the common includes are moved to the mlx5_crypto.h file.
The header files include are changed due to the new add codes.

> 
> >

Re: [dpdk-dev] [EXT] [PATCH v8 04/16] crypto/mlx5: add basic operations

[Suanming Mou]

> -----Original Message-----
> From: Akhil Goyal <gakhil@marvell.com>
> Sent: Saturday, July 17, 2021 3:34 AM
> To: Shiri Kuzin <shirik@nvidia.com>; dev@dpdk.org
> Cc: Matan Azrad <matan@nvidia.com>; Suanming Mou
> <suanmingm@nvidia.com>; david.marchand@redhat.com
> Subject: RE: [EXT] [PATCH v8 04/16] crypto/mlx5: add basic operations
> 
> > The basic dev control operations are configure, close and get info.
> >
> > Extended the existing support of configure and close:
> > 	-mlx5_crypto_dev_configure- function used to configure device.
> > 	-mlx5_crypto_dev_close-  function used to close a configured
> > 	 device.
> >
> > Added config struct to user private data with the fields socket id,
> > number of queue pairs and feature flags to be disabled.
> >
> > Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
> > Acked-by: Matan Azrad <matan@nvidia.com>
> > ---
> >  drivers/crypto/mlx5/mlx5_crypto.c | 26 +++++++++++++++++++-------
> > drivers/crypto/mlx5/mlx5_crypto.h |  1 +
> >  2 files changed, 20 insertions(+), 7 deletions(-)
> >
> > diff --git a/drivers/crypto/mlx5/mlx5_crypto.c
> > b/drivers/crypto/mlx5/mlx5_crypto.c
> > index 3f0c97d081..a7e44deb9e 100644
> > --- a/drivers/crypto/mlx5/mlx5_crypto.c
> > +++ b/drivers/crypto/mlx5/mlx5_crypto.c
> > @@ -105,22 +105,27 @@ mlx5_crypto_dev_infos_get(struct rte_cryptodev
> > *dev,
> >  	}
> >  }
> >
> > -static unsigned int
> > -mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev
> > __rte_unused)
> > -{
> > -	return sizeof(struct mlx5_crypto_session);
> > -}
> > -
> 
> I do not get the reason to remove above function, it was introduced in the
> previous patch.
> It looks the patches are not properly split.

We are not removing the function here, in fact at the last of this patch, you can say it is still there.
It's just some code movement.

> 
> >  static int
> >  mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
> > -		struct rte_cryptodev_config *config __rte_unused)
> > +			  struct rte_cryptodev_config *config)
> >  {
> >  	struct mlx5_crypto_priv *priv = dev->data->dev_private;
> >
> > +	if (config == NULL) {
> > +		DRV_LOG(ERR, "Invalid crypto dev configure parameters.");
> > +		return -EINVAL;
> > +	}
> > +	if ((config->ff_disable & RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO)
> > != 0) {
> > +		DRV_LOG(ERR,
> > +			"Disabled symmetric crypto feature is not
> > supported.");
> > +		return -ENOTSUP;
> > +	}
> >  	if (mlx5_crypto_dek_setup(priv) != 0) {
> >  		DRV_LOG(ERR, "Dek hash list creation has failed.");
> >  		return -ENOMEM;
> >  	}
> > +	priv->dev_config = *config;
> > +	DRV_LOG(DEBUG, "Device %u was configured.", dev->driver_id);
> >  	return 0;
> >  }
> 
> The patch title and the patch do not match.
> Title says, add basic operations, which should introduce the Configure and close
> ops. But here the configure and close ops Were already there and you are
> introducing some new checks In them.

Good suggestion, will try to implement the configure and close dev in this patch.

> 
> >
> > @@ -130,9 +135,16 @@ mlx5_crypto_dev_close(struct rte_cryptodev *dev)
> >  	struct mlx5_crypto_priv *priv = dev->data->dev_private;
> >
> >  	mlx5_crypto_dek_unset(priv);
> > +	DRV_LOG(DEBUG, "Device %u was closed.", dev->driver_id);
> >  	return 0;
> >  }
> Logging could have been added in the patch where dev_close was added.
> 
> >
> > +static unsigned int
> > +mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev
> > __rte_unused)
> > +{
> > +	return sizeof(struct mlx5_crypto_session); }
> > +
> >  static int
> >  mlx5_crypto_sym_session_configure(struct rte_cryptodev *dev,
> >  				  struct rte_crypto_sym_xform *xform, diff --git
> > a/drivers/crypto/mlx5/mlx5_crypto.h
> > b/drivers/crypto/mlx5/mlx5_crypto.h
> > index 167e9e57ad..a0df775407 100644
> > --- a/drivers/crypto/mlx5/mlx5_crypto.h
> > +++ b/drivers/crypto/mlx5/mlx5_crypto.h
> > @@ -24,6 +24,7 @@ struct mlx5_crypto_priv {
> >  	uint32_t pdn; /* Protection Domain number. */
> >  	struct ibv_pd *pd;
> >  	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
> > +	struct rte_cryptodev_config dev_config;
> >  };
> >
> >  struct mlx5_crypto_dek {
> > --
> > 2.27.0

Re: [dpdk-dev] [EXT] [PATCH v8 02/16] crypto/mlx5: add DEK object management

[Akhil Goyal]

> > > A DEK(Data encryption Key) is an mlx5 HW object which represents the
> > > cipher algorithm key.
> > > The DEKs are used during data encryption/decryption operations.
> > >
> > > In symmetric algorithms like AES-STS, we use the same DEK for both
> > > encryption and decryption.
> > >
> > > Use the mlx5 hash-list tool to manage the DEK objects in the PMD.
> > >
> > > Provide the compare, create and destroy functions to manage DEKs in
> > > hash-list and introduce an internal API to setup and unset the DEK
> > > management and to prepare and destroy specific DEK object.
> > >
> > > The DEK hash-list will be created in dev_configure routine and
> > > destroyed in dev_close routine.
> > >
> > > Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
> > > Acked-by: Matan Azrad <matan@nvidia.com>
> > > ---
> > >  drivers/crypto/mlx5/meson.build       |   1 +
> > >  drivers/crypto/mlx5/mlx5_crypto.c     |  42 ++++---
> > >  drivers/crypto/mlx5/mlx5_crypto.h     |  51 ++++++++
> > >  drivers/crypto/mlx5/mlx5_crypto_dek.c | 161
> > > ++++++++++++++++++++++++++
> > >  4 files changed, 239 insertions(+), 16 deletions(-)  create mode
> > > 100644 drivers/crypto/mlx5/mlx5_crypto.h  create mode 100644
> > > drivers/crypto/mlx5/mlx5_crypto_dek.c
> > >
> > > diff --git a/drivers/crypto/mlx5/meson.build
> > > b/drivers/crypto/mlx5/meson.build index 6fd70bc477..d55cdbfe6f
> 100644
> > > --- a/drivers/crypto/mlx5/meson.build
> > > +++ b/drivers/crypto/mlx5/meson.build
> > > @@ -11,6 +11,7 @@ fmt_name = 'mlx5_crypto'
> > >  deps += ['common_mlx5', 'eal', 'cryptodev']  sources = files(
> > >  	'mlx5_crypto.c',
> > > +	'mlx5_crypto_dek.c',
> > >  )
> > >  cflags_options = [
> > >  	'-std=c11',
> > > diff --git a/drivers/crypto/mlx5/mlx5_crypto.c
> > > b/drivers/crypto/mlx5/mlx5_crypto.c
> > > index fbe3c21aae..d2d82c7b15 100644
> > > --- a/drivers/crypto/mlx5/mlx5_crypto.c
> > > +++ b/drivers/crypto/mlx5/mlx5_crypto.c
> > > @@ -3,12 +3,9 @@
> > >   */
> > >
> > >  #include <rte_malloc.h>
> > > -#include <rte_log.h>
> > >  #include <rte_errno.h>
> > > +#include <rte_log.h>
> > >  #include <rte_pci.h>
> > > -#include <rte_crypto.h>
> > > -#include <rte_cryptodev.h>
> > > -#include <rte_cryptodev_pmd.h>
> >
> > There is some issue in the splitting of the patches, The above headers are
> added
> > in first patch and moved to a header file in this patch.
> > Take reference of the cnxk crypto driver which got merged recently.
> 
> The main reason is that in the patch we add a new c file:
> drivers/crypto/mlx5/mlx5_crypto_dek.c
> Now mlx5_crypto.c and that new c file both share the new h file
> mlx5_crypto.h, so all the common includes are moved to the mlx5_crypto.h
> file.
> The header files include are changed due to the new add codes.
> 
Is it not good to add these headers to mlx5_crypto.h in the first place?

Re: [dpdk-dev] [EXT] [PATCH v8 00/16] drivers: introduce mlx5 crypto PMD

[Akhil Goyal]

> > > > > v2:
> > > > > - Add data-path part.
> > > > >
> > > > > v3:
> > > > > - Rebase.
> > > > >
> > > > > v4:
> > > > > - Rebase + Address the following Akhil comments:
> > > > > - Set HW feature flag in the capability patch.
> > > > > - Fix mp object release in session clear.
> > > > > - Some spelling and word missing in doc.
> > > > > - Squash data-unit adjustment to the session operations commit.
> > > > > - Wording: device argument -> devarg.
> > > > >
> > > > > v5:
> > > > > - Add mlx5 crypto tests into test library.
> > > > > - Update documentation according to Akhil comments.
> > > > > - Fix memory region management.
> > > > > - Fix multi segment case in data-path code.
> > > > > - Split documentation to the correct commits according to Akhil
> > > comments.
> > > > > - Rebase to new version.
> > > > > - Change license to Nvidia license.
> > > > >
> > > > > v6:
> > > > > - Rebase to new version.
> > > > > - Address David's comment and update log accordingly.
> > > > > - Update testing app commits as suggested by Akhil.
> > > > >
> > > > > v7:
> > > > > - Rebase to new version.
> > > > > - Add scatter gather and OOP test cases.
> > > > >
> > > > > v8:
> > > > > - Add a test case that was removed by error.
> > > > >
> > > > >
> > > > >
> > > > > Shiri Kuzin (11):
> > > > >   drivers: introduce mlx5 crypto PMD
> > > > >   crypto/mlx5: add DEK object management
> > > >
> > > > This is really bad, I was planning to apply this series today, It
> > > > had small issues, so was planning to fix them while applying.
> > > > But when I applied the 2/16, it does not compile.
> > > > Please fix this, we are into v8 and compilation is broken.
> > > >
> > > > I am adding few more comments, please fix them too in your next
> > > > version As soon as possible. I was planning to close RC2 during
> > > > weekend, but it
> > > looks
> > > > It will fall over next week.
> > > >
> > > Please review this patchset thoroughly within Nvidia before sending
> > > next version.
> > > Take reference to other crypto PMDs for patch splitting and sequencing.
> >
> > Are we expecting next version today?
> > This is the only major patchset left for RC2.
> > Any update?
> 
> Sorry for the late reply, Shiri is not available these days,  so I'm trying to
> follow up the status and update the new version later today.
> Regarding the compilation failure, it's due to the crypto patch series depends
> on some mlx5 common code.
> The mlx5 common code has some changes and I assume Shiri rebased the
> version with the dpdk-next-mlx and meanwhile the dpdk-next-crypto branch
> still did not get the latest mlx5 common code.
> 
> I just checked the dpdk-next-crypto branch, seems the mlx5 part is still not
> updated yet. So will you also sync up the crypto branch today? If no, then the
> failure will come again.
> I'm not so familiar with the branch update procedure, but we will need the
> branch to be  updated to the version below:
> http://git.dpdk.org/next/dpdk-next-net-mlx/commit/?id=7e1cf892711b9ba237563b6c876382caaec434f7

The tree is in sync with main branch now.

Re: [dpdk-dev] [EXT] [PATCH v8 00/16] drivers: introduce mlx5 crypto PMD

[Akhil Goyal]

> > Sorry for the late reply, Shiri is not available these days,  so I'm trying to
> > follow up the status and update the new version later today.
> > Regarding the compilation failure, it's due to the crypto patch series
> depends
> > on some mlx5 common code.
> > The mlx5 common code has some changes and I assume Shiri rebased the
> > version with the dpdk-next-mlx and meanwhile the dpdk-next-crypto
> branch
> > still did not get the latest mlx5 common code.
> >
> > I just checked the dpdk-next-crypto branch, seems the mlx5 part is still not
> > updated yet. So will you also sync up the crypto branch today? If no, then
> the
> > failure will come again.
> > I'm not so familiar with the branch update procedure, but we will need the
> > branch to be  updated to the version below:
> > http://git.dpdk.org/next/dpdk-next-net-
> mlx/commit/?id=7e1cf892711b9ba237563b6c876382caaec434f7
> 
> The tree is in sync with main branch now.

BTW, it was not mentioned in the cover note that it has dependency on other patchset.

Re: [dpdk-dev] [EXT] [PATCH v8 00/16] drivers: introduce mlx5 crypto PMD

[Suanming Mou]

> -----Original Message-----
> From: Akhil Goyal <gakhil@marvell.com>
> Sent: Tuesday, July 20, 2021 4:42 PM
> To: Suanming Mou <suanmingm@nvidia.com>; Shiri Kuzin <shirik@nvidia.com>;
> NBU-Contact-Thomas Monjalon <thomas@monjalon.net>; Matan Azrad
> <matan@nvidia.com>
> Cc: david.marchand@redhat.com; dev@dpdk.org
> Subject: RE: [EXT] [PATCH v8 00/16] drivers: introduce mlx5 crypto PMD
> 
> > > Sorry for the late reply, Shiri is not available these days,  so I'm
> > > trying to follow up the status and update the new version later today.
> > > Regarding the compilation failure, it's due to the crypto patch
> > > series
> > depends
> > > on some mlx5 common code.
> > > The mlx5 common code has some changes and I assume Shiri rebased the
> > > version with the dpdk-next-mlx and meanwhile the dpdk-next-crypto
> > branch
> > > still did not get the latest mlx5 common code.
> > >
> > > I just checked the dpdk-next-crypto branch, seems the mlx5 part is
> > > still not updated yet. So will you also sync up the crypto branch
> > > today? If no, then
> > the
> > > failure will come again.
> > > I'm not so familiar with the branch update procedure, but we will
> > > need the branch to be  updated to the version below:
> > > http://git.dpdk.org/next/dpdk-next-net-
> > mlx/commit/?id=7e1cf892711b9ba237563b6c876382caaec434f7
> >
> > The tree is in sync with main branch now.
> 
> BTW, it was not mentioned in the cover note that it has dependency on other
> patchset.

Thank you for the branch syncing. 
I assume the main reason is that the required patchset was already merged when the crypto patch v8 was sent, we are not so familiar with the branch syncing.
Will keep an eye on that next time.

Re: [dpdk-dev] [EXT] [PATCH v8 02/16] crypto/mlx5: add DEK object management

[Suanming Mou]

> -----Original Message-----
> From: Akhil Goyal <gakhil@marvell.com>
> Sent: Tuesday, July 20, 2021 4:36 PM
> To: Suanming Mou <suanmingm@nvidia.com>; Shiri Kuzin <shirik@nvidia.com>;
> dev@dpdk.org
> Cc: Matan Azrad <matan@nvidia.com>; david.marchand@redhat.com
> Subject: RE: [EXT] [PATCH v8 02/16] crypto/mlx5: add DEK object management
> 
> > > > A DEK(Data encryption Key) is an mlx5 HW object which represents
> > > > the cipher algorithm key.
> > > > The DEKs are used during data encryption/decryption operations.
> > > >
> > > > In symmetric algorithms like AES-STS, we use the same DEK for both
> > > > encryption and decryption.
> > > >
> > > > Use the mlx5 hash-list tool to manage the DEK objects in the PMD.
> > > >
> > > > Provide the compare, create and destroy functions to manage DEKs
> > > > in hash-list and introduce an internal API to setup and unset the
> > > > DEK management and to prepare and destroy specific DEK object.
> > > >
> > > > The DEK hash-list will be created in dev_configure routine and
> > > > destroyed in dev_close routine.
> > > >
> > > > Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
> > > > Acked-by: Matan Azrad <matan@nvidia.com>
> > > > ---
> > > >  drivers/crypto/mlx5/meson.build       |   1 +
> > > >  drivers/crypto/mlx5/mlx5_crypto.c     |  42 ++++---
> > > >  drivers/crypto/mlx5/mlx5_crypto.h     |  51 ++++++++
> > > >  drivers/crypto/mlx5/mlx5_crypto_dek.c | 161
> > > > ++++++++++++++++++++++++++
> > > >  4 files changed, 239 insertions(+), 16 deletions(-)  create mode
> > > > 100644 drivers/crypto/mlx5/mlx5_crypto.h  create mode 100644
> > > > drivers/crypto/mlx5/mlx5_crypto_dek.c
> > > >
> > > > diff --git a/drivers/crypto/mlx5/meson.build
> > > > b/drivers/crypto/mlx5/meson.build index 6fd70bc477..d55cdbfe6f
> > 100644
> > > > --- a/drivers/crypto/mlx5/meson.build
> > > > +++ b/drivers/crypto/mlx5/meson.build
> > > > @@ -11,6 +11,7 @@ fmt_name = 'mlx5_crypto'
> > > >  deps += ['common_mlx5', 'eal', 'cryptodev']  sources = files(
> > > >  	'mlx5_crypto.c',
> > > > +	'mlx5_crypto_dek.c',
> > > >  )
> > > >  cflags_options = [
> > > >  	'-std=c11',
> > > > diff --git a/drivers/crypto/mlx5/mlx5_crypto.c
> > > > b/drivers/crypto/mlx5/mlx5_crypto.c
> > > > index fbe3c21aae..d2d82c7b15 100644
> > > > --- a/drivers/crypto/mlx5/mlx5_crypto.c
> > > > +++ b/drivers/crypto/mlx5/mlx5_crypto.c
> > > > @@ -3,12 +3,9 @@
> > > >   */
> > > >
> > > >  #include <rte_malloc.h>
> > > > -#include <rte_log.h>
> > > >  #include <rte_errno.h>
> > > > +#include <rte_log.h>
> > > >  #include <rte_pci.h>
> > > > -#include <rte_crypto.h>
> > > > -#include <rte_cryptodev.h>
> > > > -#include <rte_cryptodev_pmd.h>
> > >
> > > There is some issue in the splitting of the patches, The above
> > > headers are
> > added
> > > in first patch and moved to a header file in this patch.
> > > Take reference of the cnxk crypto driver which got merged recently.
> >
> > The main reason is that in the patch we add a new c file:
> > drivers/crypto/mlx5/mlx5_crypto_dek.c
> > Now mlx5_crypto.c and that new c file both share the new h file
> > mlx5_crypto.h, so all the common includes are moved to the
> > mlx5_crypto.h file.
> > The header files include are changed due to the new add codes.
> >
> Is it not good to add these headers to mlx5_crypto.h in the first place?

As we can see the single c file satisfied everything in the first patch, add an extra h file with only with includes seems weird in the first patch.
Then in this patch, we have two c file, we adjust the includes to the new shared h file. Everything is changed based on the new added codes.
Please let us know if you insist on that or not.

Re: [dpdk-dev] [EXT] [PATCH v8 02/16] crypto/mlx5: add DEK object management

[Akhil Goyal]

> > > > > A DEK(Data encryption Key) is an mlx5 HW object which represents
> > > > > the cipher algorithm key.
> > > > > The DEKs are used during data encryption/decryption operations.
> > > > >
> > > > > In symmetric algorithms like AES-STS, we use the same DEK for both
> > > > > encryption and decryption.
> > > > >
> > > > > Use the mlx5 hash-list tool to manage the DEK objects in the PMD.
> > > > >
> > > > > Provide the compare, create and destroy functions to manage DEKs
> > > > > in hash-list and introduce an internal API to setup and unset the
> > > > > DEK management and to prepare and destroy specific DEK object.
> > > > >
> > > > > The DEK hash-list will be created in dev_configure routine and
> > > > > destroyed in dev_close routine.
> > > > >
> > > > > Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
> > > > > Acked-by: Matan Azrad <matan@nvidia.com>
> > > > > ---
> > > > >  drivers/crypto/mlx5/meson.build       |   1 +
> > > > >  drivers/crypto/mlx5/mlx5_crypto.c     |  42 ++++---
> > > > >  drivers/crypto/mlx5/mlx5_crypto.h     |  51 ++++++++
> > > > >  drivers/crypto/mlx5/mlx5_crypto_dek.c | 161
> > > > > ++++++++++++++++++++++++++
> > > > >  4 files changed, 239 insertions(+), 16 deletions(-)  create mode
> > > > > 100644 drivers/crypto/mlx5/mlx5_crypto.h  create mode 100644
> > > > > drivers/crypto/mlx5/mlx5_crypto_dek.c
> > > > >
> > > > > diff --git a/drivers/crypto/mlx5/meson.build
> > > > > b/drivers/crypto/mlx5/meson.build index 6fd70bc477..d55cdbfe6f
> > > 100644
> > > > > --- a/drivers/crypto/mlx5/meson.build
> > > > > +++ b/drivers/crypto/mlx5/meson.build
> > > > > @@ -11,6 +11,7 @@ fmt_name = 'mlx5_crypto'
> > > > >  deps += ['common_mlx5', 'eal', 'cryptodev']  sources = files(
> > > > >  	'mlx5_crypto.c',
> > > > > +	'mlx5_crypto_dek.c',
> > > > >  )
> > > > >  cflags_options = [
> > > > >  	'-std=c11',
> > > > > diff --git a/drivers/crypto/mlx5/mlx5_crypto.c
> > > > > b/drivers/crypto/mlx5/mlx5_crypto.c
> > > > > index fbe3c21aae..d2d82c7b15 100644
> > > > > --- a/drivers/crypto/mlx5/mlx5_crypto.c
> > > > > +++ b/drivers/crypto/mlx5/mlx5_crypto.c
> > > > > @@ -3,12 +3,9 @@
> > > > >   */
> > > > >
> > > > >  #include <rte_malloc.h>
> > > > > -#include <rte_log.h>
> > > > >  #include <rte_errno.h>
> > > > > +#include <rte_log.h>
> > > > >  #include <rte_pci.h>
> > > > > -#include <rte_crypto.h>
> > > > > -#include <rte_cryptodev.h>
> > > > > -#include <rte_cryptodev_pmd.h>
> > > >
> > > > There is some issue in the splitting of the patches, The above
> > > > headers are
> > > added
> > > > in first patch and moved to a header file in this patch.
> > > > Take reference of the cnxk crypto driver which got merged recently.
> > >
> > > The main reason is that in the patch we add a new c file:
> > > drivers/crypto/mlx5/mlx5_crypto_dek.c
> > > Now mlx5_crypto.c and that new c file both share the new h file
> > > mlx5_crypto.h, so all the common includes are moved to the
> > > mlx5_crypto.h file.
> > > The header files include are changed due to the new add codes.
> > >
> > Is it not good to add these headers to mlx5_crypto.h in the first place?
> 
> As we can see the single c file satisfied everything in the first patch, add an
> extra h file with only with includes seems weird in the first patch.
> Then in this patch, we have two c file, we adjust the includes to the new
> shared h file. Everything is changed based on the new added codes.
> Please let us know if you insist on that or not.

When you add the base infrastructure patch, it places all the necessary pieces which are
required for the subsequent patches so that when needed, code can be added and
unnecessary pieces of code are not moved again and again which is happening in the
current case.
This looks cleaner and easy to understand while reviewing. Frequent movement of code
Looks complex and difficult to understand.

Re: [dpdk-dev] [EXT] [PATCH v8 03/16] crypto/mlx5: add session operations

[Suanming Mou]

> -----Original Message-----
> From: Akhil Goyal <gakhil@marvell.com>
> Sent: Saturday, July 17, 2021 3:40 AM
> To: Shiri Kuzin <shirik@nvidia.com>; dev@dpdk.org
> Cc: Matan Azrad <matan@nvidia.com>; Suanming Mou
> <suanmingm@nvidia.com>; david.marchand@redhat.com
> Subject: RE: [EXT] [PATCH v8 03/16] crypto/mlx5: add session operations
> >  doc/guides/cryptodevs/mlx5.rst          |  10 ++
> >  drivers/crypto/mlx5/mlx5_crypto.c       | 172 +++++++++++++++++++++++-
> >  3 files changed, 182 insertions(+), 5 deletions(-)
> >
> >  static const struct rte_driver mlx5_drv = { @@ -39,6 +68,49 @@ static
> > const struct rte_driver mlx5_drv = {
> >
> >  static struct cryptodev_driver mlx5_cryptodev_driver;
> >
> > +struct mlx5_crypto_session {
> > +	uint32_t bs_bpt_eo_es;
> > +	/*
> > +	 * bsf_size, bsf_p_type, encryption_order and encryption standard,
> > +	 * saved in big endian format.
> > +	 */
> 
> Normally the comments are added before the variable. Or add /**< for post
> comment.

Yes, you are right for the "normal" case, I think the main reason here is that all the mlx related PMDs are putting the comment after the variable, so in fact put the comments after the variable is the normal case for mlx PMDs.
Let's try to keep it in normal case to not make other mlx PMD familiar developers confusing.

> 
> > +	uint32_t bsp_res;
> > +	/*
> > +	 * crypto_block_size_pointer and reserved 24 bits saved in big endian
> > +	 * format.
> > +	 */
> > +	uint32_t iv_offset:16;
> > +	/* Starting point for Initialisation Vector. */
> > +	struct mlx5_crypto_dek *dek; /* Pointer to dek struct. */
> > +	uint32_t dek_id; /* DEK ID */
> > +} __rte_packed;
> > +
> > --
> > 2.27.0

Re: [dpdk-dev] [EXT] [PATCH v8 03/16] crypto/mlx5: add session operations

[Akhil Goyal]

> -----Original Message-----
> From: Suanming Mou <suanmingm@nvidia.com>
> Sent: Tuesday, July 20, 2021 2:29 PM
> To: Akhil Goyal <gakhil@marvell.com>; Shiri Kuzin <shirik@nvidia.com>;
> dev@dpdk.org
> Cc: Matan Azrad <matan@nvidia.com>; david.marchand@redhat.com
> Subject: RE: [EXT] [PATCH v8 03/16] crypto/mlx5: add session operations
> 
> 
> 
> > -----Original Message-----
> > From: Akhil Goyal <gakhil@marvell.com>
> > Sent: Saturday, July 17, 2021 3:40 AM
> > To: Shiri Kuzin <shirik@nvidia.com>; dev@dpdk.org
> > Cc: Matan Azrad <matan@nvidia.com>; Suanming Mou
> > <suanmingm@nvidia.com>; david.marchand@redhat.com
> > Subject: RE: [EXT] [PATCH v8 03/16] crypto/mlx5: add session operations
> > >  doc/guides/cryptodevs/mlx5.rst          |  10 ++
> > >  drivers/crypto/mlx5/mlx5_crypto.c       | 172 +++++++++++++++++++++++-
> > >  3 files changed, 182 insertions(+), 5 deletions(-)
> > >
> > >  static const struct rte_driver mlx5_drv = { @@ -39,6 +68,49 @@ static
> > > const struct rte_driver mlx5_drv = {
> > >
> > >  static struct cryptodev_driver mlx5_cryptodev_driver;
> > >
> > > +struct mlx5_crypto_session {
> > > +	uint32_t bs_bpt_eo_es;
> > > +	/*
> > > +	 * bsf_size, bsf_p_type, encryption_order and encryption standard,
> > > +	 * saved in big endian format.
> > > +	 */
> >
> > Normally the comments are added before the variable. Or add /**< for
> post
> > comment.
> 
> Yes, you are right for the "normal" case, I think the main reason here is that
> all the mlx related PMDs are putting the comment after the variable, so in
> fact put the comments after the variable is the normal case for mlx PMDs.
> Let's try to keep it in normal case to not make other mlx PMD familiar
> developers confusing.
> 
Please use "/**<" for post commenting.

Re: [dpdk-dev] [EXT] [PATCH v8 03/16] crypto/mlx5: add session operations

[Suanming Mou]

> -----Original Message-----
> From: Akhil Goyal <gakhil@marvell.com>
> Sent: Saturday, July 17, 2021 3:44 AM
> To: Shiri Kuzin <shirik@nvidia.com>; dev@dpdk.org
> Cc: Matan Azrad <matan@nvidia.com>; Suanming Mou
> <suanmingm@nvidia.com>; david.marchand@redhat.com
> Subject: RE: [EXT] [PATCH v8 03/16] crypto/mlx5: add session operations
> 
> > Sessions are used in symmetric transformations in order to prepare
> > objects and data for packet processing stage.
> >
> > A mlx5 session includes iv_offset, pointer to mlx5_crypto_dek struct,
> > bsf_size, bsf_p_type, block size index, encryption_order and
> > encryption standard.
> >
> > Implement the next session operations:
> >         mlx5_crypto_sym_session_get_size- returns the size of the mlx5
> > 	session struct.
> > 	mlx5_crypto_sym_session_configure- prepares the DEK hash-list
> > 	and saves all the session data.
> > 	mlx5_crypto_sym_session_clear - destroys the DEK hash-list.
> >
> > Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
> > Acked-by: Matan Azrad <matan@nvidia.com>
> > ---
> >  doc/guides/cryptodevs/features/mlx5.ini |   5 +
> >  doc/guides/cryptodevs/mlx5.rst          |  10 ++
> 
> Documentation update is done in 3/16 which means the feature is complete.
> But basic operations and queue pair initialization is done later.
> This is not good. Please move this patch after QP additions.
> Please take reference of other drivers for sequence of patches.

I'm not sure what was discussed before for suitable documentation places, but as you suggested here and in other patches, can we be aligned to put all this documentation related pieces to the (13/16) final PMD patch?

> 
> >  drivers/crypto/mlx5/mlx5_crypto.c       | 172 +++++++++++++++++++++++-
> >  3 files changed, 182 insertions(+), 5 deletions(-)
> >

Re: [dpdk-dev] [EXT] [PATCH v8 06/16] crypto/mlx5: add dev stop and start operations

[Suanming Mou]

> -----Original Message-----
> From: Akhil Goyal <gakhil@marvell.com>
> Sent: Saturday, July 17, 2021 3:45 AM
> To: Shiri Kuzin <shirik@nvidia.com>; dev@dpdk.org
> Cc: Matan Azrad <matan@nvidia.com>; Suanming Mou
> <suanmingm@nvidia.com>; david.marchand@redhat.com
> Subject: RE: [EXT] [PATCH v8 06/16] crypto/mlx5: add dev stop and start
> operations
> 
> > Add the dev_start function that is used to start a configured device.
> > Add the dev_stop function that is used to stop a configured device.
> >
> > Both functions set the dev parameter as used and return 0.
> >
> > Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
> > Acked-by: Matan Azrad <matan@nvidia.com>
> > ---
> >  drivers/crypto/mlx5/mlx5_crypto.c | 17 +++++++++++++++--
> >  1 file changed, 15 insertions(+), 2 deletions(-)
> >
> > diff --git a/drivers/crypto/mlx5/mlx5_crypto.c
> > b/drivers/crypto/mlx5/mlx5_crypto.c
> > index ebaa65c7a9..37e66cf57b 100644
> > --- a/drivers/crypto/mlx5/mlx5_crypto.c
> > +++ b/drivers/crypto/mlx5/mlx5_crypto.c
> > @@ -131,6 +131,19 @@ mlx5_crypto_dev_configure(struct rte_cryptodev
> > *dev,
> >  	return 0;
> >  }
> >
> > +static void
> > +mlx5_crypto_dev_stop(struct rte_cryptodev *dev) {
> > +	RTE_SET_USED(dev);
> > +}
> > +
> > +static int
> > +mlx5_crypto_dev_start(struct rte_cryptodev *dev) {
> > +	RTE_SET_USED(dev);
> > +	return 0;
> > +}
> > +
> >  static int
> >  mlx5_crypto_dev_close(struct rte_cryptodev *dev)  { @@ -360,8 +373,8
> > @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t
> > qp_id,
> >
> >  static struct rte_cryptodev_ops mlx5_crypto_ops = {
> >  	.dev_configure			= mlx5_crypto_dev_configure,
> > -	.dev_start			= NULL,
> > -	.dev_stop			= NULL,
> > +	.dev_start			= mlx5_crypto_dev_start,
> > +	.dev_stop			= mlx5_crypto_dev_stop,
> >  	.dev_close			= mlx5_crypto_dev_close,
> >  	.dev_infos_get			= mlx5_crypto_dev_infos_get,
> >  	.stats_get			= NULL,
> 
> 
> This patch could be squashed in your basic operations patch as these ops are not
> doing Anything.

OK, good suggestion.

Re: [dpdk-dev] [EXT] [PATCH v8 15/16] test/crypto: add data-unit and wrapped vectors

[Suanming Mou]

> -----Original Message-----
> From: Akhil Goyal <gakhil@marvell.com>
> Sent: Saturday, July 17, 2021 4:06 AM
> To: Shiri Kuzin <shirik@nvidia.com>; dev@dpdk.org
> Cc: Matan Azrad <matan@nvidia.com>; Suanming Mou
> <suanmingm@nvidia.com>; david.marchand@redhat.com
> Subject: RE: [EXT] [PATCH v8 15/16] test/crypto: add data-unit and wrapped
> vectors
> 
> > The AES-XTS algorithm supports using a wrapped key.
> > In AES-XTS the data-unit defines the data block size to be
> > encrypted\decrypted.
> >
> > Add AES-XTS vectors with a wrapped key.
> > Add a variable stating whether the key is wrapped or not.
> > Add the AES-XTS data-unit.
> >
> > Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
> > Acked-by: Matan Azrad <matan@nvidia.com>
> > ---
> >  app/test/test_cryptodev.h                  |    2 +-
> >  app/test/test_cryptodev_aes_test_vectors.h | 1340 ++++++++++++++++++++
> >  app/test/test_cryptodev_blockcipher.c      |   10 +-
> >  app/test/test_cryptodev_blockcipher.h      |    2 +
> >  4 files changed, 1352 insertions(+), 2 deletions(-)
> >
> > diff --git a/app/test/test_cryptodev.h b/app/test/test_cryptodev.h
> > index 262e545ee6..f35dba6837 100644
> > --- a/app/test/test_cryptodev.h
> > +++ b/app/test/test_cryptodev.h
> > @@ -19,7 +19,7 @@
> >  #define DEFAULT_NUM_XFORMS              (2)
> >  #define NUM_MBUFS                       (8191)
> >  #define MBUF_CACHE_SIZE                 (256)
> > -#define MBUF_DATAPAYLOAD_SIZE		(2048 +
> > DIGEST_BYTE_LENGTH_SHA512)
> > +#define MBUF_DATAPAYLOAD_SIZE		(4096 +
> > DIGEST_BYTE_LENGTH_SHA512)
> 
> Any specific reason this size is increased. It may negatively impact on other
> platforms Which have lesser memory available.
> We need a reason to change the existing MACROS which affect all drivers.

The reason here is that mlx device can support up to 4K data payload size,  I assume even we haven't changed it, there will still be other vendor can support more big value later.
After the rebase today, please try the patches on other platforms in case everything is still good.

> 
> Please rebase, I will need to test these patches on other platforms to ensure it is
> not Broken.

Re: [dpdk-dev] [EXT] [PATCH v8 03/16] crypto/mlx5: add session operations

[Akhil Goyal]

> > > Sessions are used in symmetric transformations in order to prepare
> > > objects and data for packet processing stage.
> > >
> > > A mlx5 session includes iv_offset, pointer to mlx5_crypto_dek struct,
> > > bsf_size, bsf_p_type, block size index, encryption_order and
> > > encryption standard.
> > >
> > > Implement the next session operations:
> > >         mlx5_crypto_sym_session_get_size- returns the size of the mlx5
> > > 	session struct.
> > > 	mlx5_crypto_sym_session_configure- prepares the DEK hash-list
> > > 	and saves all the session data.
> > > 	mlx5_crypto_sym_session_clear - destroys the DEK hash-list.
> > >
> > > Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
> > > Acked-by: Matan Azrad <matan@nvidia.com>
> > > ---
> > >  doc/guides/cryptodevs/features/mlx5.ini |   5 +
> > >  doc/guides/cryptodevs/mlx5.rst          |  10 ++
> >
> > Documentation update is done in 3/16 which means the feature is
> complete.
> > But basic operations and queue pair initialization is done later.
> > This is not good. Please move this patch after QP additions.
> > Please take reference of other drivers for sequence of patches.
> 
> I'm not sure what was discussed before for suitable documentation places,
> but as you suggested here and in other patches, can we be aligned to put all
> this documentation related pieces to the (13/16) final PMD patch?
> 
Introduction of the PMD in mlx5.rst can be part of initial patch.
You can also add empty .ini file also in the initial patch.
Now as and when the feature/algo is added, .ini file need to be updated.
And while adding the test app changes in the last patch, you can update
mlx5.rst.
For reference see cnxk patches.

Re: [dpdk-dev] [EXT] [PATCH v8 15/16] test/crypto: add data-unit and wrapped vectors

[Akhil Goyal]

> >
> > > The AES-XTS algorithm supports using a wrapped key.
> > > In AES-XTS the data-unit defines the data block size to be
> > > encrypted\decrypted.
> > >
> > > Add AES-XTS vectors with a wrapped key.
> > > Add a variable stating whether the key is wrapped or not.
> > > Add the AES-XTS data-unit.
> > >
> > > Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
> > > Acked-by: Matan Azrad <matan@nvidia.com>
> > > ---
> > >  app/test/test_cryptodev.h                  |    2 +-
> > >  app/test/test_cryptodev_aes_test_vectors.h | 1340
> ++++++++++++++++++++
> > >  app/test/test_cryptodev_blockcipher.c      |   10 +-
> > >  app/test/test_cryptodev_blockcipher.h      |    2 +
> > >  4 files changed, 1352 insertions(+), 2 deletions(-)
> > >
> > > diff --git a/app/test/test_cryptodev.h b/app/test/test_cryptodev.h
> > > index 262e545ee6..f35dba6837 100644
> > > --- a/app/test/test_cryptodev.h
> > > +++ b/app/test/test_cryptodev.h
> > > @@ -19,7 +19,7 @@
> > >  #define DEFAULT_NUM_XFORMS              (2)
> > >  #define NUM_MBUFS                       (8191)
> > >  #define MBUF_CACHE_SIZE                 (256)
> > > -#define MBUF_DATAPAYLOAD_SIZE		(2048 +
> > > DIGEST_BYTE_LENGTH_SHA512)
> > > +#define MBUF_DATAPAYLOAD_SIZE		(4096 +
> > > DIGEST_BYTE_LENGTH_SHA512)
> >
> > Any specific reason this size is increased. It may negatively impact on other
> > platforms Which have lesser memory available.
> > We need a reason to change the existing MACROS which affect all drivers.
> 
> The reason here is that mlx device can support up to 4K data payload size,  I
> assume even we haven't changed it, there will still be other vendor can
> support more big value later.
> After the rebase today, please try the patches on other platforms in case
> everything is still good.
> 
Can you make this particular change as a separate patch outside your series?
This would need ack from other PMD owners as well.

[dpdk-dev] [PATCH v9 00/15] drivers: introduce mlx5 crypto PMD

[Suanming Mou]

v2:
- Add data-path part.

v3:
- Rebase.

v4:
- Rebase + Address the following Akhil comments:
- Set HW feature flag in the capability patch.
- Fix mp object release in session clear.
- Some spelling and word missing in doc.
- Squash data-unit adjustment to the session operations commit.
- Wording: device argument -> devarg.

v5: 
- Add mlx5 crypto tests into test library.
- Update documentation according to Akhil comments.
- Fix memory region management.
- Fix multi segment case in data-path code.
- Split documentation to the correct commits according to Akhil
  comments.
- Rebase to new version.
- Change license to Nvidia license.

v6:
- Rebase to new version.
- Address David's comment and update log accordingly. 
- Update testing app commits as suggested by Akhil.

v7:
- Rebase to new version.
- Add scatter gather and OOP test cases.

v8:
- Add a test case that was removed by error.

v9:
- Reorganize the basic operation commit.
- Move the seesion operation commit after QP patch.
- Fix coding style.
- Move the rel_notes to the last PMD change.
- Move the mlx5 test code to last.
- Rename the driver name to crypto_mlx5.
- Split the MBUF_DATAPAYLOAD_SIZE update out of the series.

Shiri Kuzin (10):
  drivers: introduce mlx5 crypto PMD
  crypto/mlx5: add DEK object management
  crypto/mlx5: add basic operations
  crypto/mlx5: add queue pairs operations
  crypto/mlx5: add session operations
  crypto/mlx5: add memory region management
  crypto/mlx5: create login object using DevX
  test/crypto: add data-unit and wrapped vectors
  test/crypto: add AES-XTS multi segment OOP tests
  test/crypto: add mlx5 crypto driver

Suanming Mou (5):
  crypto/mlx5: add keytag devarg
  crypto/mlx5: add maximum segments devarg
  crypto/mlx5: add WQE set initialization
  crypto/mlx5: add enqueue and dequeue operations
  crypto/mlx5: add statistic get and reset operations

 MAINTAINERS                                |    4 +
 app/test/test_cryptodev.c                  |    7 +
 app/test/test_cryptodev.h                  |    2 +
 app/test/test_cryptodev_aes_test_vectors.h | 1404 ++++++++++++++++++++
 app/test/test_cryptodev_blockcipher.c      |   10 +-
 app/test/test_cryptodev_blockcipher.h      |    2 +
 doc/guides/cryptodevs/features/mlx5.ini    |   37 +
 doc/guides/cryptodevs/index.rst            |    1 +
 doc/guides/cryptodevs/mlx5.rst             |  153 +++
 doc/guides/rel_notes/release_21_08.rst     |    5 +
 drivers/common/mlx5/mlx5_common.h          |    1 +
 drivers/common/mlx5/mlx5_common_pci.c      |   14 +
 drivers/common/mlx5/mlx5_common_pci.h      |   21 +-
 drivers/crypto/meson.build                 |    1 +
 drivers/crypto/mlx5/meson.build            |   27 +
 drivers/crypto/mlx5/mlx5_crypto.c          | 1175 ++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto.h          |   90 ++
 drivers/crypto/mlx5/mlx5_crypto_dek.c      |  160 +++
 drivers/crypto/mlx5/mlx5_crypto_utils.h    |   19 +
 drivers/crypto/mlx5/version.map            |    3 +
 20 files changed, 3125 insertions(+), 11 deletions(-)
 create mode 100644 doc/guides/cryptodevs/features/mlx5.ini
 create mode 100644 doc/guides/cryptodevs/mlx5.rst
 create mode 100644 drivers/crypto/mlx5/meson.build
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto.c
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto.h
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_dek.c
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_utils.h
 create mode 100644 drivers/crypto/mlx5/version.map

-- 
2.25.1

[dpdk-dev] [PATCH v9 01/15] drivers: introduce mlx5 crypto PMD

[Suanming Mou]

From: Shiri Kuzin <shirik@nvidia.com>

Add a new PMD for Mellanox devices- crypto PMD.

The crypto PMD will be supported starting Nvidia ConnectX6 and
BlueField2.

The crypto PMD will add the support of encryption and decryption using
the AES-XTS symmetric algorithm.

The crypto PMD requires rdma-core and uses mlx5 DevX.

This patch adds the PCI probing, basic functions, build files and
log utility.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 MAINTAINERS                             |   4 +
 doc/guides/cryptodevs/features/mlx5.ini |  27 +++
 doc/guides/cryptodevs/index.rst         |   1 +
 doc/guides/cryptodevs/mlx5.rst          |  63 ++++++
 drivers/common/mlx5/mlx5_common.h       |   1 +
 drivers/common/mlx5/mlx5_common_pci.c   |  14 ++
 drivers/common/mlx5/mlx5_common_pci.h   |  21 +-
 drivers/crypto/meson.build              |   1 +
 drivers/crypto/mlx5/meson.build         |  26 +++
 drivers/crypto/mlx5/mlx5_crypto.c       | 263 ++++++++++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto.h       |  25 +++
 drivers/crypto/mlx5/mlx5_crypto_utils.h |  19 ++
 drivers/crypto/mlx5/version.map         |   3 +
 13 files changed, 458 insertions(+), 10 deletions(-)
 create mode 100644 doc/guides/cryptodevs/features/mlx5.ini
 create mode 100644 doc/guides/cryptodevs/mlx5.rst
 create mode 100644 drivers/crypto/mlx5/meson.build
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto.c
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto.h
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_utils.h
 create mode 100644 drivers/crypto/mlx5/version.map

diff --git a/MAINTAINERS b/MAINTAINERS
index af2a91d7c4..ba2cb4ef42 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -1102,6 +1102,10 @@ F: drivers/crypto/octeontx2/
 F: doc/guides/cryptodevs/octeontx2.rst
 F: doc/guides/cryptodevs/features/octeontx2.ini
 
+Mellanox mlx5
+M: Matan Azrad <matan@nvidia.com>
+F: drivers/crypto/mlx5/
+
 Null Crypto
 M: Declan Doherty <declan.doherty@intel.com>
 F: drivers/crypto/null/
diff --git a/doc/guides/cryptodevs/features/mlx5.ini b/doc/guides/cryptodevs/features/mlx5.ini
new file mode 100644
index 0000000000..ceadd967b6
--- /dev/null
+++ b/doc/guides/cryptodevs/features/mlx5.ini
@@ -0,0 +1,27 @@
+;
+; Features of a mlx5 crypto driver.
+;
+; Refer to default.ini for the full list of available PMD features.
+;
+[Features]
+HW Accelerated         = Y
+
+;
+; Supported crypto algorithms of a mlx5 crypto driver.
+;
+[Cipher]
+
+;
+; Supported authentication algorithms of a mlx5 crypto driver.
+;
+[Auth]
+
+;
+; Supported AEAD algorithms of a mlx5 crypto driver.
+;
+[AEAD]
+
+;
+; Supported Asymmetric algorithms of a mlx5 crypto driver.
+;
+[Asymmetric]
diff --git a/doc/guides/cryptodevs/index.rst b/doc/guides/cryptodevs/index.rst
index 067e7d784e..0f981c77b5 100644
--- a/doc/guides/cryptodevs/index.rst
+++ b/doc/guides/cryptodevs/index.rst
@@ -23,6 +23,7 @@ Crypto Device Drivers
     octeontx
     octeontx2
     openssl
+    mlx5
     mvsam
     nitrox
     null
diff --git a/doc/guides/cryptodevs/mlx5.rst b/doc/guides/cryptodevs/mlx5.rst
new file mode 100644
index 0000000000..05a0a449e2
--- /dev/null
+++ b/doc/guides/cryptodevs/mlx5.rst
@@ -0,0 +1,63 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+   Copyright (c) 2021 NVIDIA Corporation & Affiliates
+
+.. include:: <isonum.txt>
+
+MLX5 Crypto Driver
+==================
+
+The MLX5 crypto driver library
+(**librte_crypto_mlx5**) provides support for **Mellanox ConnectX-6**
+family adapters.
+
+Overview
+--------
+
+The device can provide disk encryption services, allowing data encryption
+and decryption towards a disk. Having all encryption/decryption
+operations done in a single device can reduce cost and overheads of the related
+FIPS certification, as ConnectX-6 is FIPS 140-2 level-2 ready.
+The encryption cipher is AES-XTS of 256/512 bit key size.
+
+MKEY is a memory region object in the hardware, that holds address translation information and
+attributes per memory area. Its ID must be tied to addresses provided to the hardware.
+The encryption operations are performed with MKEY read/write transactions, when
+the MKEY is configured to perform crypto operations.
+
+The encryption does not require text to be aligned to the AES block size (128b).
+
+The PMD uses libibverbs and libmlx5 to access the device firmware or to
+access the hardware components directly.
+There are different levels of objects and bypassing abilities.
+To get the best performances:
+
+- Verbs is a complete high-level generic API.
+- Direct Verbs is a device-specific API.
+- DevX allows to access firmware objects.
+
+Enabling librte_crypto_mlx5 causes DPDK applications to be linked against
+libibverbs.
+
+
+Driver options
+--------------
+
+- ``class`` parameter [string]
+
+  Select the class of the driver that should probe the device.
+  `crypto` for the mlx5 crypto driver.
+
+
+Supported NICs
+--------------
+
+* Mellanox\ |reg| ConnectX\ |reg|-6 200G MCX654106A-HCAT (2x200G)
+
+Prerequisites
+-------------
+
+- Mellanox OFED version: **5.3**
+  see :doc:`../../nics/mlx5` guide for more Mellanox OFED details.
+
+- Compilation can be done also with rdma-core v15+.
+  see :doc:`../../nics/mlx5` guide for more rdma-core details.
diff --git a/drivers/common/mlx5/mlx5_common.h b/drivers/common/mlx5/mlx5_common.h
index 7fb7d40b38..962179a5a5 100644
--- a/drivers/common/mlx5/mlx5_common.h
+++ b/drivers/common/mlx5/mlx5_common.h
@@ -216,6 +216,7 @@ enum mlx5_class {
 	MLX5_CLASS_VDPA = RTE_BIT64(1),
 	MLX5_CLASS_REGEX = RTE_BIT64(2),
 	MLX5_CLASS_COMPRESS = RTE_BIT64(3),
+	MLX5_CLASS_CRYPTO = RTE_BIT64(4),
 };
 
 #define MLX5_DBR_SIZE RTE_CACHE_LINE_SIZE
diff --git a/drivers/common/mlx5/mlx5_common_pci.c b/drivers/common/mlx5/mlx5_common_pci.c
index 34747c4e07..5547e62d6b 100644
--- a/drivers/common/mlx5/mlx5_common_pci.c
+++ b/drivers/common/mlx5/mlx5_common_pci.c
@@ -31,6 +31,7 @@ static const struct {
 	{ .name = "net", .driver_class = MLX5_CLASS_NET },
 	{ .name = "regex", .driver_class = MLX5_CLASS_REGEX },
 	{ .name = "compress", .driver_class = MLX5_CLASS_COMPRESS },
+	{ .name = "crypto", .driver_class = MLX5_CLASS_CRYPTO },
 };
 
 static const unsigned int mlx5_class_combinations[] = {
@@ -38,13 +39,26 @@ static const unsigned int mlx5_class_combinations[] = {
 	MLX5_CLASS_VDPA,
 	MLX5_CLASS_REGEX,
 	MLX5_CLASS_COMPRESS,
+	MLX5_CLASS_CRYPTO,
 	MLX5_CLASS_NET | MLX5_CLASS_REGEX,
 	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX,
 	MLX5_CLASS_NET | MLX5_CLASS_COMPRESS,
 	MLX5_CLASS_VDPA | MLX5_CLASS_COMPRESS,
 	MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
+	MLX5_CLASS_NET | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_VDPA | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO,
 	MLX5_CLASS_NET | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
 	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
+	MLX5_CLASS_NET | MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_NET | MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_VDPA | MLX5_CLASS_COMPRESS | MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_NET | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS |
+	MLX5_CLASS_CRYPTO,
+	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS |
+	MLX5_CLASS_CRYPTO,
 	/* New class combination should be added here. */
 };
 
diff --git a/drivers/common/mlx5/mlx5_common_pci.h b/drivers/common/mlx5/mlx5_common_pci.h
index de89bb98bc..cb8d2f5f87 100644
--- a/drivers/common/mlx5/mlx5_common_pci.h
+++ b/drivers/common/mlx5/mlx5_common_pci.h
@@ -9,17 +9,18 @@
  * @file
  *
  * RTE Mellanox PCI Driver Interface
- * Mellanox ConnectX PCI device supports multiple class: net,vdpa,regex and
- * compress devices. This layer enables creating such multiple class of devices
- * on a single PCI device by allowing to bind multiple class specific device
- * driver to attach to mlx5_pci driver.
+ * Mellanox ConnectX PCI device supports multiple class: net,vdpa,regex,compress
+ * and crypto devices. This layer enables creating such multiple class of
+ * devices on a single PCI device by allowing to bind multiple class specific
+ * device driver to attach to mlx5_pci driver.
  *
- * -----------    ------------    -------------    ----------------
- * |   mlx5  |    |   mlx5   |    |   mlx5    |    |     mlx5     |
- * | net pmd |    | vdpa pmd |    | regex pmd |    | compress pmd |
- * -----------    ------------    -------------    ----------------
- *      \              \                    /              /
- *       \              \                  /              /
+ * --------    --------    ---------    ------------    ----------
+ * | mlx5 |    | mlx5 |    | mlx5  |    |   mlx5   |    |  mlx5  |
+ * | net  |    | vdpa |    | regex |    | compress |    | crypto |
+ * | pmd  |    | pmd  |    |  pmd  |    |   pmd    |    |  pmd   |
+ * --------    --------    ---------    ------------    ----------
+ *      \              \         |          /              /
+ *       \              \        |         /              /
  *        \              \_--------------_/              /
  *         \_______________|   mlx5     |_______________/
  *                         | pci common |
diff --git a/drivers/crypto/meson.build b/drivers/crypto/meson.build
index cb865aa0d5..ea239f4c56 100644
--- a/drivers/crypto/meson.build
+++ b/drivers/crypto/meson.build
@@ -16,6 +16,7 @@ drivers = [
         'dpaa_sec',
         'dpaa2_sec',
         'kasumi',
+        'mlx5',
         'mvsam',
         'nitrox',
         'null',
diff --git a/drivers/crypto/mlx5/meson.build b/drivers/crypto/mlx5/meson.build
new file mode 100644
index 0000000000..6fd70bc477
--- /dev/null
+++ b/drivers/crypto/mlx5/meson.build
@@ -0,0 +1,26 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2021 NVIDIA Corporation & Affiliates
+
+if not is_linux
+	build = false
+	reason = 'only supported on Linux'
+	subdir_done()
+endif
+
+fmt_name = 'mlx5_crypto'
+deps += ['common_mlx5', 'eal', 'cryptodev']
+sources = files(
+	'mlx5_crypto.c',
+)
+cflags_options = [
+	'-std=c11',
+	'-Wno-strict-prototypes',
+	'-D_BSD_SOURCE',
+	'-D_DEFAULT_SOURCE',
+	'-D_XOPEN_SOURCE=600',
+]
+foreach option:cflags_options
+	if cc.has_argument(option)
+		cflags += option
+	endif
+endforeach
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
new file mode 100644
index 0000000000..34b4b52b04
--- /dev/null
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -0,0 +1,263 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2021 NVIDIA Corporation & Affiliates
+ */
+
+#include <rte_malloc.h>
+#include <rte_errno.h>
+#include <rte_log.h>
+#include <rte_pci.h>
+
+#include <mlx5_glue.h>
+#include <mlx5_common.h>
+#include <mlx5_common_pci.h>
+#include <mlx5_devx_cmds.h>
+#include <mlx5_common_os.h>
+
+#include "mlx5_crypto_utils.h"
+#include "mlx5_crypto.h"
+
+#define MLX5_CRYPTO_DRIVER_NAME crypto_mlx5
+#define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
+
+#define MLX5_CRYPTO_FEATURE_FLAGS \
+	RTE_CRYPTODEV_FF_HW_ACCELERATED
+
+TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
+				TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
+static pthread_mutex_t priv_list_lock = PTHREAD_MUTEX_INITIALIZER;
+
+int mlx5_crypto_logtype;
+
+uint8_t mlx5_crypto_driver_id;
+
+static const char mlx5_crypto_drv_name[] = RTE_STR(MLX5_CRYPTO_DRIVER_NAME);
+
+static const struct rte_driver mlx5_drv = {
+	.name = mlx5_crypto_drv_name,
+	.alias = mlx5_crypto_drv_name
+};
+
+static struct cryptodev_driver mlx5_cryptodev_driver;
+
+static struct rte_cryptodev_ops mlx5_crypto_ops = {
+	.dev_configure			= NULL,
+	.dev_start			= NULL,
+	.dev_stop			= NULL,
+	.dev_close			= NULL,
+	.dev_infos_get			= NULL,
+	.stats_get			= NULL,
+	.stats_reset			= NULL,
+	.queue_pair_setup		= NULL,
+	.queue_pair_release		= NULL,
+	.sym_session_get_size		= NULL,
+	.sym_session_configure		= NULL,
+	.sym_session_clear		= NULL,
+	.sym_get_raw_dp_ctx_size	= NULL,
+	.sym_configure_raw_dp_ctx	= NULL,
+};
+
+static void
+mlx5_crypto_hw_global_release(struct mlx5_crypto_priv *priv)
+{
+	if (priv->pd != NULL) {
+		claim_zero(mlx5_glue->dealloc_pd(priv->pd));
+		priv->pd = NULL;
+	}
+	if (priv->uar != NULL) {
+		mlx5_glue->devx_free_uar(priv->uar);
+		priv->uar = NULL;
+	}
+}
+
+static int
+mlx5_crypto_pd_create(struct mlx5_crypto_priv *priv)
+{
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+	struct mlx5dv_obj obj;
+	struct mlx5dv_pd pd_info;
+	int ret;
+
+	priv->pd = mlx5_glue->alloc_pd(priv->ctx);
+	if (priv->pd == NULL) {
+		DRV_LOG(ERR, "Failed to allocate PD.");
+		return errno ? -errno : -ENOMEM;
+	}
+	obj.pd.in = priv->pd;
+	obj.pd.out = &pd_info;
+	ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_PD);
+	if (ret != 0) {
+		DRV_LOG(ERR, "Fail to get PD object info.");
+		mlx5_glue->dealloc_pd(priv->pd);
+		priv->pd = NULL;
+		return -errno;
+	}
+	priv->pdn = pd_info.pdn;
+	return 0;
+#else
+	(void)priv;
+	DRV_LOG(ERR, "Cannot get pdn - no DV support.");
+	return -ENOTSUP;
+#endif /* HAVE_IBV_FLOW_DV_SUPPORT */
+}
+
+static int
+mlx5_crypto_hw_global_prepare(struct mlx5_crypto_priv *priv)
+{
+	if (mlx5_crypto_pd_create(priv) != 0)
+		return -1;
+	priv->uar = mlx5_devx_alloc_uar(priv->ctx, -1);
+	if (priv->uar == NULL || mlx5_os_get_devx_uar_reg_addr(priv->uar) ==
+	    NULL) {
+		rte_errno = errno;
+		claim_zero(mlx5_glue->dealloc_pd(priv->pd));
+		DRV_LOG(ERR, "Failed to allocate UAR.");
+		return -1;
+	}
+	return 0;
+}
+
+/**
+ * DPDK callback to register a PCI device.
+ *
+ * This function spawns crypto device out of a given PCI device.
+ *
+ * @param[in] pci_drv
+ *   PCI driver structure (mlx5_crypto_driver).
+ * @param[in] pci_dev
+ *   PCI device information.
+ *
+ * @return
+ *   0 on success, 1 to skip this driver, a negative errno value otherwise
+ *   and rte_errno is set.
+ */
+static int
+mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
+			struct rte_pci_device *pci_dev)
+{
+	struct ibv_device *ibv;
+	struct rte_cryptodev *crypto_dev;
+	struct ibv_context *ctx;
+	struct mlx5_crypto_priv *priv;
+	struct mlx5_hca_attr attr = { 0 };
+	struct rte_cryptodev_pmd_init_params init_params = {
+		.name = "",
+		.private_data_size = sizeof(struct mlx5_crypto_priv),
+		.socket_id = pci_dev->device.numa_node,
+		.max_nb_queue_pairs =
+				RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
+	};
+	RTE_SET_USED(pci_drv);
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+		DRV_LOG(ERR, "Non-primary process type is not supported.");
+		rte_errno = ENOTSUP;
+		return -rte_errno;
+	}
+	ibv = mlx5_os_get_ibv_device(&pci_dev->addr);
+	if (ibv == NULL) {
+		DRV_LOG(ERR, "No matching IB device for PCI slot "
+			PCI_PRI_FMT ".", pci_dev->addr.domain,
+			pci_dev->addr.bus, pci_dev->addr.devid,
+			pci_dev->addr.function);
+		return -rte_errno;
+	}
+	DRV_LOG(INFO, "PCI information matches for device \"%s\".", ibv->name);
+	ctx = mlx5_glue->dv_open_device(ibv);
+	if (ctx == NULL) {
+		DRV_LOG(ERR, "Failed to open IB device \"%s\".", ibv->name);
+		rte_errno = ENODEV;
+		return -rte_errno;
+	}
+	if (mlx5_devx_cmd_query_hca_attr(ctx, &attr) != 0 ||
+	    attr.crypto == 0 || attr.aes_xts == 0) {
+		DRV_LOG(ERR, "Not enough capabilities to support crypto "
+			"operations, maybe old FW/OFED version?");
+		claim_zero(mlx5_glue->close_device(ctx));
+		rte_errno = ENOTSUP;
+		return -ENOTSUP;
+	}
+	crypto_dev = rte_cryptodev_pmd_create(ibv->name, &pci_dev->device,
+					&init_params);
+	if (crypto_dev == NULL) {
+		DRV_LOG(ERR, "Failed to create device \"%s\".", ibv->name);
+		claim_zero(mlx5_glue->close_device(ctx));
+		return -ENODEV;
+	}
+	DRV_LOG(INFO,
+		"Crypto device %s was created successfully.", ibv->name);
+	crypto_dev->dev_ops = &mlx5_crypto_ops;
+	crypto_dev->dequeue_burst = NULL;
+	crypto_dev->enqueue_burst = NULL;
+	crypto_dev->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
+	crypto_dev->driver_id = mlx5_crypto_driver_id;
+	priv = crypto_dev->data->dev_private;
+	priv->ctx = ctx;
+	priv->pci_dev = pci_dev;
+	priv->crypto_dev = crypto_dev;
+	if (mlx5_crypto_hw_global_prepare(priv) != 0) {
+		rte_cryptodev_pmd_destroy(priv->crypto_dev);
+		claim_zero(mlx5_glue->close_device(priv->ctx));
+		return -1;
+	}
+	pthread_mutex_lock(&priv_list_lock);
+	TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next);
+	pthread_mutex_unlock(&priv_list_lock);
+	return 0;
+}
+
+static int
+mlx5_crypto_pci_remove(struct rte_pci_device *pdev)
+{
+	struct mlx5_crypto_priv *priv = NULL;
+
+	pthread_mutex_lock(&priv_list_lock);
+	TAILQ_FOREACH(priv, &mlx5_crypto_priv_list, next)
+		if (rte_pci_addr_cmp(&priv->pci_dev->addr, &pdev->addr) != 0)
+			break;
+	if (priv)
+		TAILQ_REMOVE(&mlx5_crypto_priv_list, priv, next);
+	pthread_mutex_unlock(&priv_list_lock);
+	if (priv) {
+		mlx5_crypto_hw_global_release(priv);
+		rte_cryptodev_pmd_destroy(priv->crypto_dev);
+		claim_zero(mlx5_glue->close_device(priv->ctx));
+	}
+	return 0;
+}
+
+static const struct rte_pci_id mlx5_crypto_pci_id_map[] = {
+		{
+			RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
+					PCI_DEVICE_ID_MELLANOX_CONNECTX6)
+		},
+		{
+			.vendor_id = 0
+		}
+};
+
+static struct mlx5_pci_driver mlx5_crypto_driver = {
+	.driver_class = MLX5_CLASS_CRYPTO,
+	.pci_driver = {
+		.driver = {
+			.name = RTE_STR(MLX5_CRYPTO_DRIVER_NAME),
+		},
+		.id_table = mlx5_crypto_pci_id_map,
+		.probe = mlx5_crypto_pci_probe,
+		.remove = mlx5_crypto_pci_remove,
+		.drv_flags = 0,
+	},
+};
+
+RTE_INIT(rte_mlx5_crypto_init)
+{
+	mlx5_common_init();
+	if (mlx5_glue != NULL)
+		mlx5_pci_driver_register(&mlx5_crypto_driver);
+}
+
+RTE_PMD_REGISTER_CRYPTO_DRIVER(mlx5_cryptodev_driver, mlx5_drv,
+			       mlx5_crypto_driver_id);
+
+RTE_LOG_REGISTER_DEFAULT(mlx5_crypto_logtype, NOTICE)
+RTE_PMD_EXPORT_NAME(MLX5_CRYPTO_DRIVER_NAME, __COUNTER__);
+RTE_PMD_REGISTER_PCI_TABLE(MLX5_CRYPTO_DRIVER_NAME, mlx5_crypto_pci_id_map);
+RTE_PMD_REGISTER_KMOD_DEP(MLX5_CRYPTO_DRIVER_NAME, "* ib_uverbs & mlx5_core & mlx5_ib");
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
new file mode 100644
index 0000000000..6c187b7bc1
--- /dev/null
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2021 NVIDIA Corporation & Affiliates
+ */
+
+#ifndef MLX5_CRYPTO_H_
+#define MLX5_CRYPTO_H_
+
+#include <stdbool.h>
+
+#include <rte_cryptodev.h>
+#include <rte_cryptodev_pmd.h>
+
+#include <mlx5_common_utils.h>
+
+struct mlx5_crypto_priv {
+	TAILQ_ENTRY(mlx5_crypto_priv) next;
+	struct ibv_context *ctx; /* Device context. */
+	struct rte_pci_device *pci_dev;
+	struct rte_cryptodev *crypto_dev;
+	void *uar; /* User Access Region. */
+	uint32_t pdn; /* Protection Domain number. */
+	struct ibv_pd *pd;
+};
+
+#endif /* MLX5_CRYPTO_H_ */
diff --git a/drivers/crypto/mlx5/mlx5_crypto_utils.h b/drivers/crypto/mlx5/mlx5_crypto_utils.h
new file mode 100644
index 0000000000..b6c60ca782
--- /dev/null
+++ b/drivers/crypto/mlx5/mlx5_crypto_utils.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2021 NVIDIA Corporation & Affiliates
+ */
+
+#ifndef RTE_PMD_MLX5_CRYPTO_UTILS_H_
+#define RTE_PMD_MLX5_CRYPTO_UTILS_H_
+
+#include <mlx5_common.h>
+
+extern int mlx5_crypto_logtype;
+
+#define MLX5_CRYPTO_LOG_PREFIX "mlx5_crypto"
+/* Generic printf()-like logging macro with automatic line feed. */
+#define DRV_LOG(level, ...) \
+	PMD_DRV_LOG_(level, mlx5_crypto_logtype, MLX5_CRYPTO_LOG_PREFIX, \
+		__VA_ARGS__ PMD_DRV_LOG_STRIP PMD_DRV_LOG_OPAREN, \
+		PMD_DRV_LOG_CPAREN)
+
+#endif /* RTE_PMD_MLX5_CRYPTO_UTILS_H_ */
diff --git a/drivers/crypto/mlx5/version.map b/drivers/crypto/mlx5/version.map
new file mode 100644
index 0000000000..4a76d1d52d
--- /dev/null
+++ b/drivers/crypto/mlx5/version.map
@@ -0,0 +1,3 @@
+DPDK_21 {
+	local: *;
+};
-- 
2.25.1

[dpdk-dev] [PATCH v9 02/15] crypto/mlx5: add DEK object management

[Suanming Mou]

From: Shiri Kuzin <shirik@nvidia.com>

A DEK(Data encryption Key) is an mlx5 HW object which represents the
cipher algorithm key.
The DEKs are used during data encryption/decryption operations.

In symmetric algorithms like AES-STS, we use the same DEK for both
encryption and decryption.

Use the mlx5 hash-list tool to manage the DEK objects in the PMD.

Provide the compare, create and destroy functions to manage DEKs in
hash-list and introduce an internal API to setup and unset the DEK
management and to prepare and destroy specific DEK object.

The DEK hash-list will be created in dev_configure routine and
destroyed in dev_close routine.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/meson.build       |   1 +
 drivers/crypto/mlx5/mlx5_crypto.h     |  25 ++++
 drivers/crypto/mlx5/mlx5_crypto_dek.c | 160 ++++++++++++++++++++++++++
 3 files changed, 186 insertions(+)
 create mode 100644 drivers/crypto/mlx5/mlx5_crypto_dek.c

diff --git a/drivers/crypto/mlx5/meson.build b/drivers/crypto/mlx5/meson.build
index 6fd70bc477..d55cdbfe6f 100644
--- a/drivers/crypto/mlx5/meson.build
+++ b/drivers/crypto/mlx5/meson.build
@@ -11,6 +11,7 @@ fmt_name = 'mlx5_crypto'
 deps += ['common_mlx5', 'eal', 'cryptodev']
 sources = files(
 	'mlx5_crypto.c',
+	'mlx5_crypto_dek.c',
 )
 cflags_options = [
 	'-std=c11',
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 6c187b7bc1..3f783fc956 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -12,6 +12,9 @@
 
 #include <mlx5_common_utils.h>
 
+#define MLX5_CRYPTO_DEK_HTABLE_SZ (1 << 11)
+#define MLX5_CRYPTO_KEY_LENGTH 80
+
 struct mlx5_crypto_priv {
 	TAILQ_ENTRY(mlx5_crypto_priv) next;
 	struct ibv_context *ctx; /* Device context. */
@@ -20,6 +23,28 @@ struct mlx5_crypto_priv {
 	void *uar; /* User Access Region. */
 	uint32_t pdn; /* Protection Domain number. */
 	struct ibv_pd *pd;
+	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
 };
 
+struct mlx5_crypto_dek {
+	struct mlx5_list_entry entry; /* Pointer to DEK hash list entry. */
+	struct mlx5_devx_obj *obj; /* Pointer to DEK DevX object. */
+	uint8_t data[MLX5_CRYPTO_KEY_LENGTH]; /* DEK key data. */
+	bool size_is_48; /* Whether the key\data size is 48 bytes or not. */
+} __rte_cache_aligned;
+
+int
+mlx5_crypto_dek_destroy(struct mlx5_crypto_priv *priv,
+			struct mlx5_crypto_dek *dek);
+
+struct mlx5_crypto_dek *
+mlx5_crypto_dek_prepare(struct mlx5_crypto_priv *priv,
+			struct rte_crypto_cipher_xform *cipher);
+
+int
+mlx5_crypto_dek_setup(struct mlx5_crypto_priv *priv);
+
+void
+mlx5_crypto_dek_unset(struct mlx5_crypto_priv *priv);
+
 #endif /* MLX5_CRYPTO_H_ */
diff --git a/drivers/crypto/mlx5/mlx5_crypto_dek.c b/drivers/crypto/mlx5/mlx5_crypto_dek.c
new file mode 100644
index 0000000000..67b1fa3819
--- /dev/null
+++ b/drivers/crypto/mlx5/mlx5_crypto_dek.c
@@ -0,0 +1,160 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2021 NVIDIA Corporation & Affiliates
+ */
+
+#include <rte_ip.h>
+#include <rte_common.h>
+#include <rte_errno.h>
+#include <rte_log.h>
+
+#include <mlx5_prm.h>
+#include <mlx5_devx_cmds.h>
+
+#include "mlx5_crypto_utils.h"
+#include "mlx5_crypto.h"
+
+struct mlx5_crypto_dek_ctx {
+	struct rte_crypto_cipher_xform *cipher;
+	struct mlx5_crypto_priv *priv;
+};
+
+int
+mlx5_crypto_dek_destroy(struct mlx5_crypto_priv *priv,
+			struct mlx5_crypto_dek *dek)
+{
+	return mlx5_hlist_unregister(priv->dek_hlist, &dek->entry);
+}
+
+struct mlx5_crypto_dek *
+mlx5_crypto_dek_prepare(struct mlx5_crypto_priv *priv,
+			struct rte_crypto_cipher_xform *cipher)
+{
+	struct mlx5_hlist *dek_hlist = priv->dek_hlist;
+	struct mlx5_crypto_dek_ctx dek_ctx = {
+		.cipher = cipher,
+		.priv = priv,
+	};
+	struct rte_crypto_cipher_xform *cipher_ctx = cipher;
+	uint64_t key64 = __rte_raw_cksum(cipher_ctx->key.data,
+					 cipher_ctx->key.length, 0);
+	struct mlx5_list_entry *entry = mlx5_hlist_register(dek_hlist,
+							     key64, &dek_ctx);
+
+	return entry == NULL ? NULL :
+			     container_of(entry, struct mlx5_crypto_dek, entry);
+}
+
+static struct mlx5_list_entry *
+mlx5_crypto_dek_clone_cb(void *tool_ctx __rte_unused,
+			 struct mlx5_list_entry *oentry,
+			 void *cb_ctx __rte_unused)
+{
+	struct mlx5_crypto_dek *entry = rte_zmalloc(__func__, sizeof(*entry),
+						    RTE_CACHE_LINE_SIZE);
+
+	if (!entry) {
+		DRV_LOG(ERR, "Cannot allocate dek resource memory.");
+		rte_errno = ENOMEM;
+		return NULL;
+	}
+	memcpy(entry, oentry, sizeof(*entry));
+	return &entry->entry;
+}
+
+static void
+mlx5_crypto_dek_clone_free_cb(void *tool_ctx __rte_unused,
+			      struct mlx5_list_entry *entry)
+{
+	struct mlx5_crypto_dek *dek = container_of(entry,
+						struct mlx5_crypto_dek, entry);
+
+	rte_free(dek);
+}
+
+static int
+mlx5_crypto_dek_match_cb(void *tool_ctx __rte_unused,
+			 struct mlx5_list_entry *entry, void *cb_ctx)
+{
+	struct mlx5_crypto_dek_ctx *ctx = cb_ctx;
+	struct rte_crypto_cipher_xform *cipher_ctx = ctx->cipher;
+	struct mlx5_crypto_dek *dek =
+			container_of(entry, typeof(*dek), entry);
+	uint32_t key_len = dek->size_is_48 ? 48 : 80;
+
+	if (key_len != cipher_ctx->key.length)
+		return -1;
+	return memcmp(cipher_ctx->key.data, dek->data, key_len);
+}
+
+static struct mlx5_list_entry *
+mlx5_crypto_dek_create_cb(void *tool_ctx __rte_unused, void *cb_ctx)
+{
+	struct mlx5_crypto_dek_ctx *ctx = cb_ctx;
+	struct rte_crypto_cipher_xform *cipher_ctx = ctx->cipher;
+	struct mlx5_crypto_dek *dek = rte_zmalloc(__func__, sizeof(*dek),
+						  RTE_CACHE_LINE_SIZE);
+	struct mlx5_devx_dek_attr dek_attr = {
+		.pd = ctx->priv->pdn,
+		.key_purpose = MLX5_CRYPTO_KEY_PURPOSE_AES_XTS,
+		.has_keytag = 1,
+	};
+
+	if (dek == NULL) {
+		DRV_LOG(ERR, "Failed to allocate dek memory.");
+		return NULL;
+	}
+	switch (cipher_ctx->key.length) {
+	case 48:
+		dek->size_is_48 = true;
+		dek_attr.key_size = MLX5_CRYPTO_KEY_SIZE_128b;
+		break;
+	case 80:
+		dek->size_is_48 = false;
+		dek_attr.key_size = MLX5_CRYPTO_KEY_SIZE_256b;
+		break;
+	default:
+		DRV_LOG(ERR, "Key size not supported.");
+		return NULL;
+	}
+	memcpy(&dek_attr.key, cipher_ctx->key.data, cipher_ctx->key.length);
+	dek->obj = mlx5_devx_cmd_create_dek_obj(ctx->priv->ctx, &dek_attr);
+	if (dek->obj == NULL) {
+		rte_free(dek);
+		return NULL;
+	}
+	memcpy(&dek->data, cipher_ctx->key.data, cipher_ctx->key.length);
+	return &dek->entry;
+}
+
+static void
+mlx5_crypto_dek_remove_cb(void *tool_ctx __rte_unused,
+			  struct mlx5_list_entry *entry)
+{
+	struct mlx5_crypto_dek *dek =
+		container_of(entry, typeof(*dek), entry);
+
+	claim_zero(mlx5_devx_cmd_destroy(dek->obj));
+	rte_free(dek);
+}
+
+int
+mlx5_crypto_dek_setup(struct mlx5_crypto_priv *priv)
+{
+	priv->dek_hlist = mlx5_hlist_create("dek_hlist",
+				 MLX5_CRYPTO_DEK_HTABLE_SZ,
+				 0, 1, NULL, mlx5_crypto_dek_create_cb,
+				 mlx5_crypto_dek_match_cb,
+				 mlx5_crypto_dek_remove_cb,
+				 mlx5_crypto_dek_clone_cb,
+				 mlx5_crypto_dek_clone_free_cb);
+	if (priv->dek_hlist == NULL)
+		return -1;
+	return 0;
+}
+
+void
+mlx5_crypto_dek_unset(struct mlx5_crypto_priv *priv)
+{
+	mlx5_hlist_destroy(priv->dek_hlist);
+	priv->dek_hlist = NULL;
+}
-- 
2.25.1

[dpdk-dev] [PATCH v9 03/15] crypto/mlx5: add basic operations

[Suanming Mou]

From: Shiri Kuzin <shirik@nvidia.com>

The basic dev control operations are configure, close, start, stop and
get info.

Extended the existing support of configure and close:
	-mlx5_crypto_dev_configure- function used to configure device.
	-mlx5_crypto_dev_close-  function used to close a configured
	 device.
	-mlx5_crypto_dev_stop- function used to stop device.
	-mlx5_crypto_dev_start- function used to start device.
	-mlx5_crypto_dev_infos_get- function used to get info.

Added config struct to user private data with the fields socket id,
number of queue pairs and feature flags to be disabled.
Add the dev_start function that is used to start a configured device.
Add the dev_stop function that is used to stop a configured device.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 103 ++++++++++++++++++++++++++++--
 drivers/crypto/mlx5/mlx5_crypto.h |   1 +
 2 files changed, 99 insertions(+), 5 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 34b4b52b04..9ad64f7244 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -30,6 +30,32 @@ int mlx5_crypto_logtype;
 
 uint8_t mlx5_crypto_driver_id;
 
+const struct rte_cryptodev_capabilities mlx5_crypto_caps[] = {
+	{		/* AES XTS */
+		.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+		{.sym = {
+			.xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+			{.cipher = {
+				.algo = RTE_CRYPTO_CIPHER_AES_XTS,
+				.block_size = 16,
+				.key_size = {
+					.min = 32,
+					.max = 64,
+					.increment = 32
+				},
+				.iv_size = {
+					.min = 16,
+					.max = 16,
+					.increment = 0
+				},
+				.dataunit_set =
+				RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_512_BYTES |
+				RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_4096_BYTES,
+			}, }
+		}, }
+	},
+};
+
 static const char mlx5_crypto_drv_name[] = RTE_STR(MLX5_CRYPTO_DRIVER_NAME);
 
 static const struct rte_driver mlx5_drv = {
@@ -39,12 +65,79 @@ static const struct rte_driver mlx5_drv = {
 
 static struct cryptodev_driver mlx5_cryptodev_driver;
 
+static void
+mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev,
+			  struct rte_cryptodev_info *dev_info)
+{
+	RTE_SET_USED(dev);
+	if (dev_info != NULL) {
+		dev_info->driver_id = mlx5_crypto_driver_id;
+		dev_info->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
+		dev_info->capabilities = mlx5_crypto_caps;
+		dev_info->max_nb_queue_pairs = 0;
+		dev_info->min_mbuf_headroom_req = 0;
+		dev_info->min_mbuf_tailroom_req = 0;
+		dev_info->sym.max_nb_sessions = 0;
+		/*
+		 * If 0, the device does not have any limitation in number of
+		 * sessions that can be used.
+		 */
+	}
+}
+
+static int
+mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
+			  struct rte_cryptodev_config *config)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+
+	if (config == NULL) {
+		DRV_LOG(ERR, "Invalid crypto dev configure parameters.");
+		return -EINVAL;
+	}
+	if ((config->ff_disable & RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO) != 0) {
+		DRV_LOG(ERR,
+			"Disabled symmetric crypto feature is not supported.");
+		return -ENOTSUP;
+	}
+	if (mlx5_crypto_dek_setup(priv) != 0) {
+		DRV_LOG(ERR, "Dek hash list creation has failed.");
+		return -ENOMEM;
+	}
+	priv->dev_config = *config;
+	DRV_LOG(DEBUG, "Device %u was configured.", dev->driver_id);
+	return 0;
+}
+
+static void
+mlx5_crypto_dev_stop(struct rte_cryptodev *dev)
+{
+	RTE_SET_USED(dev);
+}
+
+static int
+mlx5_crypto_dev_start(struct rte_cryptodev *dev)
+{
+	RTE_SET_USED(dev);
+	return 0;
+}
+
+static int
+mlx5_crypto_dev_close(struct rte_cryptodev *dev)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+
+	mlx5_crypto_dek_unset(priv);
+	DRV_LOG(DEBUG, "Device %u was closed.", dev->driver_id);
+	return 0;
+}
+
 static struct rte_cryptodev_ops mlx5_crypto_ops = {
-	.dev_configure			= NULL,
-	.dev_start			= NULL,
-	.dev_stop			= NULL,
-	.dev_close			= NULL,
-	.dev_infos_get			= NULL,
+	.dev_configure			= mlx5_crypto_dev_configure,
+	.dev_start			= mlx5_crypto_dev_start,
+	.dev_stop			= mlx5_crypto_dev_stop,
+	.dev_close			= mlx5_crypto_dev_close,
+	.dev_infos_get			= mlx5_crypto_dev_infos_get,
 	.stats_get			= NULL,
 	.stats_reset			= NULL,
 	.queue_pair_setup		= NULL,
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 3f783fc956..11772bb846 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -24,6 +24,7 @@ struct mlx5_crypto_priv {
 	uint32_t pdn; /* Protection Domain number. */
 	struct ibv_pd *pd;
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
+	struct rte_cryptodev_config dev_config;
 };
 
 struct mlx5_crypto_dek {
-- 
2.25.1

[dpdk-dev] [PATCH v9 04/15] crypto/mlx5: add queue pairs operations

[Suanming Mou]

From: Shiri Kuzin <shirik@nvidia.com>

The HW queue pairs are a pair of send queue and receive queue of
independent work queues packed together in one object for the purpose
of transferring data between nodes of a network.

Completion Queue is a FIFO queue of completed work requests.

In crypto driver we use one QP in loopback in order to encrypt and
decrypt data locally without sending it to the wire.
In the configured QP we only use the SQ to perform the encryption and
decryption operations.

Added implementation for the QP setup function which creates the CQ,
creates the QP and changes its state to RTS (ready to send).

Added implementation for the release QP function to release all the QP
resources.

Added the ops structure that contains any operation which is supported
by the cryptodev.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 129 +++++++++++++++++++++++++++++-
 drivers/crypto/mlx5/mlx5_crypto.h |  11 +++
 2 files changed, 137 insertions(+), 3 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 9ad64f7244..ad1d7b65aa 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -6,6 +6,7 @@
 #include <rte_errno.h>
 #include <rte_log.h>
 #include <rte_pci.h>
+#include <rte_memory.h>
 
 #include <mlx5_glue.h>
 #include <mlx5_common.h>
@@ -18,6 +19,7 @@
 
 #define MLX5_CRYPTO_DRIVER_NAME crypto_mlx5
 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
+#define MLX5_CRYPTO_MAX_QPS 1024
 
 #define MLX5_CRYPTO_FEATURE_FLAGS \
 	RTE_CRYPTODEV_FF_HW_ACCELERATED
@@ -74,7 +76,7 @@ mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev,
 		dev_info->driver_id = mlx5_crypto_driver_id;
 		dev_info->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
 		dev_info->capabilities = mlx5_crypto_caps;
-		dev_info->max_nb_queue_pairs = 0;
+		dev_info->max_nb_queue_pairs = MLX5_CRYPTO_MAX_QPS;
 		dev_info->min_mbuf_headroom_req = 0;
 		dev_info->min_mbuf_tailroom_req = 0;
 		dev_info->sym.max_nb_sessions = 0;
@@ -132,6 +134,127 @@ mlx5_crypto_dev_close(struct rte_cryptodev *dev)
 	return 0;
 }
 
+static int
+mlx5_crypto_queue_pair_release(struct rte_cryptodev *dev, uint16_t qp_id)
+{
+	struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
+
+	if (qp->qp_obj != NULL)
+		claim_zero(mlx5_devx_cmd_destroy(qp->qp_obj));
+	if (qp->umem_obj != NULL)
+		claim_zero(mlx5_glue->devx_umem_dereg(qp->umem_obj));
+	if (qp->umem_buf != NULL)
+		rte_free(qp->umem_buf);
+	mlx5_devx_cq_destroy(&qp->cq_obj);
+	rte_free(qp);
+	dev->data->queue_pairs[qp_id] = NULL;
+	return 0;
+}
+
+static int
+mlx5_crypto_qp2rts(struct mlx5_crypto_qp *qp)
+{
+	/*
+	 * In Order to configure self loopback, when calling these functions the
+	 * remote QP id that is used is the id of the same QP.
+	 */
+	if (mlx5_devx_cmd_modify_qp_state(qp->qp_obj, MLX5_CMD_OP_RST2INIT_QP,
+					  qp->qp_obj->id)) {
+		DRV_LOG(ERR, "Failed to modify QP to INIT state(%u).",
+			rte_errno);
+		return -1;
+	}
+	if (mlx5_devx_cmd_modify_qp_state(qp->qp_obj, MLX5_CMD_OP_INIT2RTR_QP,
+					  qp->qp_obj->id)) {
+		DRV_LOG(ERR, "Failed to modify QP to RTR state(%u).",
+			rte_errno);
+		return -1;
+	}
+	if (mlx5_devx_cmd_modify_qp_state(qp->qp_obj, MLX5_CMD_OP_RTR2RTS_QP,
+					  qp->qp_obj->id)) {
+		DRV_LOG(ERR, "Failed to modify QP to RTS state(%u).",
+			rte_errno);
+		return -1;
+	}
+	return 0;
+}
+
+static int
+mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
+			     const struct rte_cryptodev_qp_conf *qp_conf,
+			     int socket_id)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+	struct mlx5_devx_qp_attr attr = {0};
+	struct mlx5_crypto_qp *qp;
+	uint16_t log_nb_desc = rte_log2_u32(qp_conf->nb_descriptors);
+	uint32_t umem_size = RTE_BIT32(log_nb_desc) *
+			      MLX5_CRYPTO_WQE_SET_SIZE +
+			      sizeof(*qp->db_rec) * 2;
+	uint32_t alloc_size = sizeof(*qp);
+	struct mlx5_devx_cq_attr cq_attr = {
+		.uar_page_id = mlx5_os_get_devx_uar_page_id(priv->uar),
+	};
+
+	if (dev->data->queue_pairs[qp_id] != NULL)
+		mlx5_crypto_queue_pair_release(dev, qp_id);
+	alloc_size = RTE_ALIGN(alloc_size, RTE_CACHE_LINE_SIZE);
+	alloc_size += sizeof(struct rte_crypto_op *) * RTE_BIT32(log_nb_desc);
+	qp = rte_zmalloc_socket(__func__, alloc_size, RTE_CACHE_LINE_SIZE,
+				socket_id);
+	if (qp == NULL) {
+		DRV_LOG(ERR, "Failed to allocate QP memory.");
+		rte_errno = ENOMEM;
+		return -rte_errno;
+	}
+	if (mlx5_devx_cq_create(priv->ctx, &qp->cq_obj, log_nb_desc,
+				&cq_attr, socket_id) != 0) {
+		DRV_LOG(ERR, "Failed to create CQ.");
+		goto error;
+	}
+	qp->umem_buf = rte_zmalloc_socket(__func__, umem_size, 4096, socket_id);
+	if (qp->umem_buf == NULL) {
+		DRV_LOG(ERR, "Failed to allocate QP umem.");
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	qp->umem_obj = mlx5_glue->devx_umem_reg(priv->ctx,
+					       (void *)(uintptr_t)qp->umem_buf,
+					       umem_size,
+					       IBV_ACCESS_LOCAL_WRITE);
+	if (qp->umem_obj == NULL) {
+		DRV_LOG(ERR, "Failed to register QP umem.");
+		goto error;
+	}
+	attr.pd = priv->pdn;
+	attr.uar_index = mlx5_os_get_devx_uar_page_id(priv->uar);
+	attr.cqn = qp->cq_obj.cq->id;
+	attr.log_page_size = rte_log2_u32(sysconf(_SC_PAGESIZE));
+	attr.rq_size =  0;
+	attr.sq_size = RTE_BIT32(log_nb_desc);
+	attr.dbr_umem_valid = 1;
+	attr.wq_umem_id = qp->umem_obj->umem_id;
+	attr.wq_umem_offset = 0;
+	attr.dbr_umem_id = qp->umem_obj->umem_id;
+	attr.dbr_address = RTE_BIT64(log_nb_desc) *
+			   MLX5_CRYPTO_WQE_SET_SIZE;
+	qp->qp_obj = mlx5_devx_cmd_create_qp(priv->ctx, &attr);
+	if (qp->qp_obj == NULL) {
+		DRV_LOG(ERR, "Failed to create QP(%u).", rte_errno);
+		goto error;
+	}
+	qp->db_rec = RTE_PTR_ADD(qp->umem_buf, (uintptr_t)attr.dbr_address);
+	if (mlx5_crypto_qp2rts(qp))
+		goto error;
+	qp->ops = (struct rte_crypto_op **)RTE_ALIGN((uintptr_t)(qp + 1),
+							   RTE_CACHE_LINE_SIZE);
+	dev->data->queue_pairs[qp_id] = qp;
+	return 0;
+error:
+	mlx5_crypto_queue_pair_release(dev, qp_id);
+	return -1;
+}
+
 static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_configure			= mlx5_crypto_dev_configure,
 	.dev_start			= mlx5_crypto_dev_start,
@@ -140,8 +263,8 @@ static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_infos_get			= mlx5_crypto_dev_infos_get,
 	.stats_get			= NULL,
 	.stats_reset			= NULL,
-	.queue_pair_setup		= NULL,
-	.queue_pair_release		= NULL,
+	.queue_pair_setup		= mlx5_crypto_queue_pair_setup,
+	.queue_pair_release		= mlx5_crypto_queue_pair_release,
 	.sym_session_get_size		= NULL,
 	.sym_session_configure		= NULL,
 	.sym_session_clear		= NULL,
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 11772bb846..949092cd37 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -11,9 +11,11 @@
 #include <rte_cryptodev_pmd.h>
 
 #include <mlx5_common_utils.h>
+#include <mlx5_common_devx.h>
 
 #define MLX5_CRYPTO_DEK_HTABLE_SZ (1 << 11)
 #define MLX5_CRYPTO_KEY_LENGTH 80
+#define MLX5_CRYPTO_WQE_SET_SIZE 1024
 
 struct mlx5_crypto_priv {
 	TAILQ_ENTRY(mlx5_crypto_priv) next;
@@ -27,6 +29,15 @@ struct mlx5_crypto_priv {
 	struct rte_cryptodev_config dev_config;
 };
 
+struct mlx5_crypto_qp {
+	struct mlx5_devx_cq cq_obj;
+	struct mlx5_devx_obj *qp_obj;
+	struct mlx5dv_devx_umem *umem_obj;
+	void *umem_buf;
+	volatile uint32_t *db_rec;
+	struct rte_crypto_op **ops;
+};
+
 struct mlx5_crypto_dek {
 	struct mlx5_list_entry entry; /* Pointer to DEK hash list entry. */
 	struct mlx5_devx_obj *obj; /* Pointer to DEK DevX object. */
-- 
2.25.1

[dpdk-dev] [PATCH v9 05/15] crypto/mlx5: add session operations

[Suanming Mou]

From: Shiri Kuzin <shirik@nvidia.com>

Sessions are used in symmetric transformations in order to prepare
objects and data for packet processing stage.

A mlx5 session includes iv_offset, pointer to mlx5_crypto_dek struct,
bsf_size, bsf_p_type, block size index, encryption_order and encryption
standard.

Implement the next session operations:
        mlx5_crypto_sym_session_get_size- returns the size of the mlx5
	session struct.
	mlx5_crypto_sym_session_configure- prepares the DEK hash-list
	and saves all the session data.
	mlx5_crypto_sym_session_clear - destroys the DEK hash-list.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 doc/guides/cryptodevs/features/mlx5.ini |   5 +
 doc/guides/cryptodevs/mlx5.rst          |  10 ++
 drivers/crypto/mlx5/mlx5_crypto.c       | 122 +++++++++++++++++++++++-
 3 files changed, 133 insertions(+), 4 deletions(-)

diff --git a/doc/guides/cryptodevs/features/mlx5.ini b/doc/guides/cryptodevs/features/mlx5.ini
index ceadd967b6..bd757b5211 100644
--- a/doc/guides/cryptodevs/features/mlx5.ini
+++ b/doc/guides/cryptodevs/features/mlx5.ini
@@ -4,12 +4,17 @@
 ; Refer to default.ini for the full list of available PMD features.
 ;
 [Features]
+Symmetric crypto       = Y
 HW Accelerated         = Y
+Cipher multiple data units = Y
+Cipher wrapped key     = Y
 
 ;
 ; Supported crypto algorithms of a mlx5 crypto driver.
 ;
 [Cipher]
+AES XTS (128)  = Y
+AES XTS (256)  = Y
 
 ;
 ; Supported authentication algorithms of a mlx5 crypto driver.
diff --git a/doc/guides/cryptodevs/mlx5.rst b/doc/guides/cryptodevs/mlx5.rst
index 05a0a449e2..ecab385c0d 100644
--- a/doc/guides/cryptodevs/mlx5.rst
+++ b/doc/guides/cryptodevs/mlx5.rst
@@ -53,6 +53,16 @@ Supported NICs
 
 * Mellanox\ |reg| ConnectX\ |reg|-6 200G MCX654106A-HCAT (2x200G)
 
+
+Limitations
+-----------
+
+- AES-XTS keys provided in xform must include keytag and should be wrapped.
+- The supported data-unit lengths are 512B and 1KB. In case the `dataunit_len`
+  is not provided in the cipher xform, the OP length is limited to the above
+  values and 1MB.
+
+
 Prerequisites
 -------------
 
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index ad1d7b65aa..2fe2e8b871 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -3,6 +3,7 @@
  */
 
 #include <rte_malloc.h>
+#include <rte_mempool.h>
 #include <rte_errno.h>
 #include <rte_log.h>
 #include <rte_pci.h>
@@ -22,7 +23,9 @@
 #define MLX5_CRYPTO_MAX_QPS 1024
 
 #define MLX5_CRYPTO_FEATURE_FLAGS \
-	RTE_CRYPTODEV_FF_HW_ACCELERATED
+	(RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | \
+	 RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY | \
+	 RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS)
 
 TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
 				TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
@@ -67,6 +70,21 @@ static const struct rte_driver mlx5_drv = {
 
 static struct cryptodev_driver mlx5_cryptodev_driver;
 
+struct mlx5_crypto_session {
+	uint32_t bs_bpt_eo_es;
+	/**< bsf_size, bsf_p_type, encryption_order and encryption standard,
+	 * saved in big endian format.
+	 */
+	uint32_t bsp_res;
+	/**< crypto_block_size_pointer and reserved 24 bits saved in big
+	 * endian format.
+	 */
+	uint32_t iv_offset:16;
+	/**< Starting point for Initialisation Vector. */
+	struct mlx5_crypto_dek *dek; /**< Pointer to dek struct. */
+	uint32_t dek_id; /**< DEK ID */
+} __rte_packed;
+
 static void
 mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev,
 			  struct rte_cryptodev_info *dev_info)
@@ -134,6 +152,102 @@ mlx5_crypto_dev_close(struct rte_cryptodev *dev)
 	return 0;
 }
 
+static unsigned int
+mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
+{
+	return sizeof(struct mlx5_crypto_session);
+}
+
+static int
+mlx5_crypto_sym_session_configure(struct rte_cryptodev *dev,
+				  struct rte_crypto_sym_xform *xform,
+				  struct rte_cryptodev_sym_session *session,
+				  struct rte_mempool *mp)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+	struct mlx5_crypto_session *sess_private_data;
+	struct rte_crypto_cipher_xform *cipher;
+	uint8_t encryption_order;
+	int ret;
+
+	if (unlikely(xform->next != NULL)) {
+		DRV_LOG(ERR, "Xform next is not supported.");
+		return -ENOTSUP;
+	}
+	if (unlikely((xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) ||
+		     (xform->cipher.algo != RTE_CRYPTO_CIPHER_AES_XTS))) {
+		DRV_LOG(ERR, "Only AES-XTS algorithm is supported.");
+		return -ENOTSUP;
+	}
+	ret = rte_mempool_get(mp, (void *)&sess_private_data);
+	if (ret != 0) {
+		DRV_LOG(ERR,
+			"Failed to get session %p private data from mempool.",
+			sess_private_data);
+		return -ENOMEM;
+	}
+	cipher = &xform->cipher;
+	sess_private_data->dek = mlx5_crypto_dek_prepare(priv, cipher);
+	if (sess_private_data->dek == NULL) {
+		rte_mempool_put(mp, sess_private_data);
+		DRV_LOG(ERR, "Failed to prepare dek.");
+		return -ENOMEM;
+	}
+	if (cipher->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
+		encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_MEMORY;
+	else
+		encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_WIRE;
+	sess_private_data->bs_bpt_eo_es = rte_cpu_to_be_32
+			(MLX5_BSF_SIZE_64B << MLX5_BSF_SIZE_OFFSET |
+			 MLX5_BSF_P_TYPE_CRYPTO << MLX5_BSF_P_TYPE_OFFSET |
+			 encryption_order << MLX5_ENCRYPTION_ORDER_OFFSET |
+			 MLX5_ENCRYPTION_STANDARD_AES_XTS);
+	switch (xform->cipher.dataunit_len) {
+	case 0:
+		sess_private_data->bsp_res = 0;
+		break;
+	case 512:
+		sess_private_data->bsp_res = rte_cpu_to_be_32
+					     ((uint32_t)MLX5_BLOCK_SIZE_512B <<
+					     MLX5_BLOCK_SIZE_OFFSET);
+		break;
+	case 4096:
+		sess_private_data->bsp_res = rte_cpu_to_be_32
+					     ((uint32_t)MLX5_BLOCK_SIZE_4096B <<
+					     MLX5_BLOCK_SIZE_OFFSET);
+		break;
+	default:
+		DRV_LOG(ERR, "Cipher data unit length is not supported.");
+		return -ENOTSUP;
+	}
+	sess_private_data->iv_offset = cipher->iv.offset;
+	sess_private_data->dek_id =
+			rte_cpu_to_be_32(sess_private_data->dek->obj->id &
+					 0xffffff);
+	set_sym_session_private_data(session, dev->driver_id,
+				     sess_private_data);
+	DRV_LOG(DEBUG, "Session %p was configured.", sess_private_data);
+	return 0;
+}
+
+static void
+mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev,
+			      struct rte_cryptodev_sym_session *sess)
+{
+	struct mlx5_crypto_priv *priv = dev->data->dev_private;
+	struct mlx5_crypto_session *spriv = get_sym_session_private_data(sess,
+								dev->driver_id);
+
+	if (unlikely(spriv == NULL)) {
+		DRV_LOG(ERR, "Failed to get session %p private data.", spriv);
+		return;
+	}
+	mlx5_crypto_dek_destroy(priv, spriv->dek);
+	set_sym_session_private_data(sess, dev->driver_id, NULL);
+	rte_mempool_put(rte_mempool_from_obj(spriv), spriv);
+	DRV_LOG(DEBUG, "Session %p was cleared.", spriv);
+}
+
 static int
 mlx5_crypto_queue_pair_release(struct rte_cryptodev *dev, uint16_t qp_id)
 {
@@ -265,9 +379,9 @@ static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.stats_reset			= NULL,
 	.queue_pair_setup		= mlx5_crypto_queue_pair_setup,
 	.queue_pair_release		= mlx5_crypto_queue_pair_release,
-	.sym_session_get_size		= NULL,
-	.sym_session_configure		= NULL,
-	.sym_session_clear		= NULL,
+	.sym_session_get_size		= mlx5_crypto_sym_session_get_size,
+	.sym_session_configure		= mlx5_crypto_sym_session_configure,
+	.sym_session_clear		= mlx5_crypto_sym_session_clear,
 	.sym_get_raw_dp_ctx_size	= NULL,
 	.sym_configure_raw_dp_ctx	= NULL,
 };
-- 
2.25.1

[dpdk-dev] [PATCH v9 06/15] crypto/mlx5: add memory region management

[Suanming Mou]

From: Shiri Kuzin <shirik@nvidia.com>

Mellanox user space drivers don't deal with physical addresses as part
of a memory protection mechanism.
The device translates the given virtual address to a physical address
using the given memory key as an address space identifier.
That's why any mbuf virtual address is moved directly to the HW
descriptor(WQE).

The mapping between the virtual address to the physical address is saved
in MR configured by the kernel to the HW.

Each MR has a key that should also be moved to the WQE by the SW.

When the SW sees an unmapped address, it extends the address range and
creates a MR using a system call.

Add memory region cache management:
	- 2 level cache per queue-pair - no locks.
	- 1 shared cache between all the queues using a lock.

Using this way, the MR key search per data-path address is optimized.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Signed-off-by: Michael Baum <michaelba@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 doc/guides/cryptodevs/mlx5.rst    |  6 +++
 drivers/crypto/mlx5/mlx5_crypto.c | 63 +++++++++++++++++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto.h |  3 ++
 3 files changed, 72 insertions(+)

diff --git a/doc/guides/cryptodevs/mlx5.rst b/doc/guides/cryptodevs/mlx5.rst
index ecab385c0d..c41db95d40 100644
--- a/doc/guides/cryptodevs/mlx5.rst
+++ b/doc/guides/cryptodevs/mlx5.rst
@@ -26,6 +26,12 @@ the MKEY is configured to perform crypto operations.
 
 The encryption does not require text to be aligned to the AES block size (128b).
 
+For security reasons and to increase robustness, this driver only deals with virtual
+memory addresses. The way resources allocations are handled by the kernel,
+combined with hardware specifications that allow handling virtual memory
+addresses directly, ensure that DPDK applications cannot access random
+physical memory (or memory that does not belong to the current process).
+
 The PMD uses libibverbs and libmlx5 to access the device firmware or to
 access the hardware components directly.
 There are different levels of objects and bypassing abilities.
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 2fe2e8b871..9416590aba 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -259,6 +259,7 @@ mlx5_crypto_queue_pair_release(struct rte_cryptodev *dev, uint16_t qp_id)
 		claim_zero(mlx5_glue->devx_umem_dereg(qp->umem_obj));
 	if (qp->umem_buf != NULL)
 		rte_free(qp->umem_buf);
+	mlx5_mr_btree_free(&qp->mr_ctrl.cache_bh);
 	mlx5_devx_cq_destroy(&qp->cq_obj);
 	rte_free(qp);
 	dev->data->queue_pairs[qp_id] = NULL;
@@ -340,6 +341,14 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 		DRV_LOG(ERR, "Failed to register QP umem.");
 		goto error;
 	}
+	if (mlx5_mr_btree_init(&qp->mr_ctrl.cache_bh, MLX5_MR_BTREE_CACHE_N,
+			       priv->dev_config.socket_id) != 0) {
+		DRV_LOG(ERR, "Cannot allocate MR Btree for qp %u.",
+			(uint32_t)qp_id);
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	qp->mr_ctrl.dev_gen_ptr = &priv->mr_scache.dev_gen;
 	attr.pd = priv->pdn;
 	attr.uar_index = mlx5_os_get_devx_uar_page_id(priv->uar);
 	attr.cqn = qp->cq_obj.cq->id;
@@ -446,6 +455,40 @@ mlx5_crypto_hw_global_prepare(struct mlx5_crypto_priv *priv)
 	return 0;
 }
 
+/**
+ * Callback for memory event.
+ *
+ * @param event_type
+ *   Memory event type.
+ * @param addr
+ *   Address of memory.
+ * @param len
+ *   Size of memory.
+ */
+static void
+mlx5_crypto_mr_mem_event_cb(enum rte_mem_event event_type, const void *addr,
+			    size_t len, void *arg __rte_unused)
+{
+	struct mlx5_crypto_priv *priv;
+
+	/* Must be called from the primary process. */
+	MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);
+	switch (event_type) {
+	case RTE_MEM_EVENT_FREE:
+		pthread_mutex_lock(&priv_list_lock);
+		/* Iterate all the existing mlx5 devices. */
+		TAILQ_FOREACH(priv, &mlx5_crypto_priv_list, next)
+			mlx5_free_mr_by_addr(&priv->mr_scache,
+					     priv->ctx->device->name,
+					     addr, len);
+		pthread_mutex_unlock(&priv_list_lock);
+		break;
+	case RTE_MEM_EVENT_ALLOC:
+	default:
+		break;
+	}
+}
+
 /**
  * DPDK callback to register a PCI device.
  *
@@ -528,6 +571,22 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 		claim_zero(mlx5_glue->close_device(priv->ctx));
 		return -1;
 	}
+	if (mlx5_mr_btree_init(&priv->mr_scache.cache,
+			     MLX5_MR_BTREE_CACHE_N * 2, rte_socket_id()) != 0) {
+		DRV_LOG(ERR, "Failed to allocate shared cache MR memory.");
+		mlx5_crypto_hw_global_release(priv);
+		rte_cryptodev_pmd_destroy(priv->crypto_dev);
+		claim_zero(mlx5_glue->close_device(priv->ctx));
+		rte_errno = ENOMEM;
+		return -rte_errno;
+	}
+	priv->mr_scache.reg_mr_cb = mlx5_common_verbs_reg_mr;
+	priv->mr_scache.dereg_mr_cb = mlx5_common_verbs_dereg_mr;
+	/* Register callback function for global shared MR cache management. */
+	if (TAILQ_EMPTY(&mlx5_crypto_priv_list))
+		rte_mem_event_callback_register("MLX5_MEM_EVENT_CB",
+						mlx5_crypto_mr_mem_event_cb,
+						NULL);
 	pthread_mutex_lock(&priv_list_lock);
 	TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next);
 	pthread_mutex_unlock(&priv_list_lock);
@@ -547,6 +606,10 @@ mlx5_crypto_pci_remove(struct rte_pci_device *pdev)
 		TAILQ_REMOVE(&mlx5_crypto_priv_list, priv, next);
 	pthread_mutex_unlock(&priv_list_lock);
 	if (priv) {
+		if (TAILQ_EMPTY(&mlx5_crypto_priv_list))
+			rte_mem_event_callback_unregister("MLX5_MEM_EVENT_CB",
+							  NULL);
+		mlx5_mr_release_cache(&priv->mr_scache);
 		mlx5_crypto_hw_global_release(priv);
 		rte_cryptodev_pmd_destroy(priv->crypto_dev);
 		claim_zero(mlx5_glue->close_device(priv->ctx));
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 949092cd37..af292ed19f 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -12,6 +12,7 @@
 
 #include <mlx5_common_utils.h>
 #include <mlx5_common_devx.h>
+#include <mlx5_common_mr.h>
 
 #define MLX5_CRYPTO_DEK_HTABLE_SZ (1 << 11)
 #define MLX5_CRYPTO_KEY_LENGTH 80
@@ -27,6 +28,7 @@ struct mlx5_crypto_priv {
 	struct ibv_pd *pd;
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
 	struct rte_cryptodev_config dev_config;
+	struct mlx5_mr_share_cache mr_scache; /* Global shared MR cache. */
 };
 
 struct mlx5_crypto_qp {
@@ -36,6 +38,7 @@ struct mlx5_crypto_qp {
 	void *umem_buf;
 	volatile uint32_t *db_rec;
 	struct rte_crypto_op **ops;
+	struct mlx5_mr_ctrl mr_ctrl;
 };
 
 struct mlx5_crypto_dek {
-- 
2.25.1

[dpdk-dev] [PATCH v9 07/15] crypto/mlx5: create login object using DevX

[Suanming Mou]

From: Shiri Kuzin <shirik@nvidia.com>

To work with crypto engines that are marked with wrapped_import_method,
a login session is required.
A crypto login object needs to be created using DevX.

The crypto login object contains:
	- The credential pointer.
	- The import_KEK pointer to be used for all secured information
	  communicated in crypto commands (key fields), including the
	  provided credential in this command.
	- The credential secret, wrapped by the import_KEK indicated in
	  this command. Size includes 8 bytes IV for wrapping.

Added devargs for the required login values:
	- wcs_file - path to the file containing the credential.
	- import_kek_id - the import KEK pointer.
	- credential_id - the credential pointer.

Create the login DevX object in pci_probe function and destroy it in
pci_remove.
Destroying the crypto login object means logout.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 doc/guides/cryptodevs/mlx5.rst    |  60 +++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto.c | 103 ++++++++++++++++++++++++++++++
 drivers/crypto/mlx5/mlx5_crypto.h |   7 ++
 3 files changed, 170 insertions(+)

diff --git a/doc/guides/cryptodevs/mlx5.rst b/doc/guides/cryptodevs/mlx5.rst
index c41db95d40..c316bfdc58 100644
--- a/doc/guides/cryptodevs/mlx5.rst
+++ b/doc/guides/cryptodevs/mlx5.rst
@@ -44,6 +44,51 @@ To get the best performances:
 Enabling librte_crypto_mlx5 causes DPDK applications to be linked against
 libibverbs.
 
+In order to move the device to crypto operational mode, credential and KEK
+(Key Encrypting Key) should be set as the first step.
+The credential will be used by the software in order to perform crypto login, and the KEK is
+the AES Key Wrap Algorithm (rfc3394) key that will be used for sensitive data
+wrapping.
+The credential and the AES-XTS keys should be provided to the hardware, as ciphertext
+encrypted by the KEK.
+
+When crypto engines are defined to work in wrapped import method, they come out
+of the factory in Commissioning mode, and thus, cannot be used for crypto operations
+yet. A dedicated tool is used for changing the mode from Commissioning to
+Operational, while setting the first import_KEK and credential in plaintext.
+The mlxreg dedicated tool should be used as follows:
+
+- Set CRYPTO_OPERATIONAL register to set the device in crypto operational mode.
+
+  The input to this tool is:
+    The first credential in plaintext, 40B.
+    The first import_KEK in plaintext: kek size 0 for 16B or 1 for 32B, kek data.
+
+  Example:
+  mlxreg -d /dev/mst/mt4123_pciconf0 --reg_name CRYPTO_OPERATIONAL --get
+
+  The "wrapped_crypto_operational" value will be "0x00000000".
+  The command to set the register should be executed only once, and all the
+  values mentioned above should be specified in the same command.
+
+  Example:
+  mlxreg -d /dev/mst/mt4123_pciconf0 --reg_name CRYPTO_OPERATIONAL
+  --set "credential[0]=0x10000000, credential[1]=0x10000000, kek[0]=0x00000000"
+
+  All values not specified will remain 0.
+  "wrapped_crypto_going_to_commissioning" and  "wrapped_crypto_operational"
+  should not be specified.
+
+  All the device ports should set it in order to move to operational mode.
+
+- Query CRYPTO_OPERATIONAL register to make sure the device is in Operational
+  mode.
+
+  Example:
+  mlxreg -d /dev/mst/mt4123_pciconf0 --reg_name CRYPTO_OPERATIONAL --get
+  The "wrapped_crypto_operational" value will be "0x00000001" if the mode was
+  successfully changed to operational mode.
+
 
 Driver options
 --------------
@@ -53,6 +98,21 @@ Driver options
   Select the class of the driver that should probe the device.
   `crypto` for the mlx5 crypto driver.
 
+- ``wcs_file`` parameter [string] - mandatory
+
+  File path including only the wrapped credential in string format of hexadecimal
+  numbers, represent 48 bytes (8 bytes IV added by the AES key wrap algorithm).
+
+- ``import_kek_id`` parameter [int]
+
+  The identifier of the KEK, default value is 0 represents the operational
+  register import_kek..
+
+- ``credential_id`` parameter [int]
+
+  The identifier of the credential, default value is 0 represents the operational
+  register credential.
+
 
 Supported NICs
 --------------
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 9416590aba..a16578b3af 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -455,6 +455,101 @@ mlx5_crypto_hw_global_prepare(struct mlx5_crypto_priv *priv)
 	return 0;
 }
 
+
+static int
+mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque)
+{
+	struct mlx5_crypto_devarg_params *devarg_prms = opaque;
+	struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr;
+	unsigned long tmp;
+	FILE *file;
+	int ret;
+	int i;
+
+	if (strcmp(key, "class") == 0)
+		return 0;
+	if (strcmp(key, "wcs_file") == 0) {
+		file = fopen(val, "rb");
+		if (file == NULL) {
+			rte_errno = ENOTSUP;
+			return -rte_errno;
+		}
+		for (i = 0 ; i < MLX5_CRYPTO_CREDENTIAL_SIZE ; i++) {
+			ret = fscanf(file, "%02hhX", &attr->credential[i]);
+			if (ret <= 0) {
+				fclose(file);
+				DRV_LOG(ERR,
+					"Failed to read credential from file.");
+				rte_errno = EINVAL;
+				return -rte_errno;
+			}
+		}
+		fclose(file);
+		devarg_prms->login_devarg = true;
+		return 0;
+	}
+	errno = 0;
+	tmp = strtoul(val, NULL, 0);
+	if (errno) {
+		DRV_LOG(WARNING, "%s: \"%s\" is an invalid integer.", key, val);
+		return -errno;
+	}
+	if (strcmp(key, "import_kek_id") == 0)
+		attr->session_import_kek_ptr = (uint32_t)tmp;
+	else if (strcmp(key, "credential_id") == 0)
+		attr->credential_pointer = (uint32_t)tmp;
+	else
+		DRV_LOG(WARNING, "Invalid key %s.", key);
+	return 0;
+}
+
+static struct mlx5_devx_obj *
+mlx5_crypto_config_login(struct rte_devargs *devargs,
+			 struct ibv_context *ctx)
+{
+	/*
+	 * Set credential pointer and session import KEK pointer to a default
+	 * value of 0.
+	 */
+	struct mlx5_crypto_devarg_params login = {
+			.login_devarg = false,
+			.login_attr = {
+					.credential_pointer = 0,
+					.session_import_kek_ptr = 0,
+			}
+	};
+	struct rte_kvargs *kvlist;
+
+	if (devargs == NULL) {
+		DRV_LOG(ERR,
+	"No login devargs in order to enable crypto operations in the device.");
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	kvlist = rte_kvargs_parse(devargs->args, NULL);
+	if (kvlist == NULL) {
+		DRV_LOG(ERR, "Failed to parse devargs.");
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	if (rte_kvargs_process(kvlist, NULL, mlx5_crypto_args_check_handler,
+			   &login) != 0) {
+		DRV_LOG(ERR, "Devargs handler function Failed.");
+		rte_kvargs_free(kvlist);
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	rte_kvargs_free(kvlist);
+	if (login.login_devarg == false) {
+		DRV_LOG(ERR,
+	"No login credential devarg in order to enable crypto operations "
+	"in the device.");
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	return mlx5_devx_cmd_create_crypto_login_obj(ctx, &login.login_attr);
+}
+
 /**
  * Callback for memory event.
  *
@@ -510,6 +605,7 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	struct ibv_device *ibv;
 	struct rte_cryptodev *crypto_dev;
 	struct ibv_context *ctx;
+	struct mlx5_devx_obj *login;
 	struct mlx5_crypto_priv *priv;
 	struct mlx5_hca_attr attr = { 0 };
 	struct rte_cryptodev_pmd_init_params init_params = {
@@ -548,6 +644,11 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 		rte_errno = ENOTSUP;
 		return -ENOTSUP;
 	}
+	login = mlx5_crypto_config_login(pci_dev->device.devargs, ctx);
+	if (login == NULL) {
+		DRV_LOG(ERR, "Failed to configure login.");
+		return -rte_errno;
+	}
 	crypto_dev = rte_cryptodev_pmd_create(ibv->name, &pci_dev->device,
 					&init_params);
 	if (crypto_dev == NULL) {
@@ -564,6 +665,7 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	crypto_dev->driver_id = mlx5_crypto_driver_id;
 	priv = crypto_dev->data->dev_private;
 	priv->ctx = ctx;
+	priv->login_obj = login;
 	priv->pci_dev = pci_dev;
 	priv->crypto_dev = crypto_dev;
 	if (mlx5_crypto_hw_global_prepare(priv) != 0) {
@@ -612,6 +714,7 @@ mlx5_crypto_pci_remove(struct rte_pci_device *pdev)
 		mlx5_mr_release_cache(&priv->mr_scache);
 		mlx5_crypto_hw_global_release(priv);
 		rte_cryptodev_pmd_destroy(priv->crypto_dev);
+		claim_zero(mlx5_devx_cmd_destroy(priv->login_obj));
 		claim_zero(mlx5_glue->close_device(priv->ctx));
 	}
 	return 0;
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index af292ed19f..9df982b23e 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -29,6 +29,7 @@ struct mlx5_crypto_priv {
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
 	struct rte_cryptodev_config dev_config;
 	struct mlx5_mr_share_cache mr_scache; /* Global shared MR cache. */
+	struct mlx5_devx_obj *login_obj;
 };
 
 struct mlx5_crypto_qp {
@@ -48,6 +49,12 @@ struct mlx5_crypto_dek {
 	bool size_is_48; /* Whether the key\data size is 48 bytes or not. */
 } __rte_cache_aligned;
 
+
+struct mlx5_crypto_devarg_params {
+	bool login_devarg;
+	struct mlx5_devx_crypto_login_attr login_attr;
+};
+
 int
 mlx5_crypto_dek_destroy(struct mlx5_crypto_priv *priv,
 			struct mlx5_crypto_dek *dek);
-- 
2.25.1

[dpdk-dev] [PATCH v9 08/15] crypto/mlx5: add keytag devarg

[Suanming Mou]

A keytag is a piece of data encrypted together with a DEK.

When a DEK is referenced by an MKEY.bsf through its index, the keytag is
also supplied in the BSF as plaintext. The HW will decrypt the DEK (and
the attached keytag) and will fail the operation if the keytags don't
match.

This commit adds the configuration of the keytag with devargs.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
 doc/guides/cryptodevs/mlx5.rst    |  7 +++++
 drivers/crypto/mlx5/mlx5_crypto.c | 50 +++++++++++++++++--------------
 drivers/crypto/mlx5/mlx5_crypto.h |  3 +-
 3 files changed, 37 insertions(+), 23 deletions(-)

diff --git a/doc/guides/cryptodevs/mlx5.rst b/doc/guides/cryptodevs/mlx5.rst
index c316bfdc58..5b874824db 100644
--- a/doc/guides/cryptodevs/mlx5.rst
+++ b/doc/guides/cryptodevs/mlx5.rst
@@ -52,6 +52,9 @@ wrapping.
 The credential and the AES-XTS keys should be provided to the hardware, as ciphertext
 encrypted by the KEK.
 
+A keytag (64 bits) should be appended to the AES-XTS keys (before wrapping),
+and will be validated when the hardware attempts to access it.
+
 When crypto engines are defined to work in wrapped import method, they come out
 of the factory in Commissioning mode, and thus, cannot be used for crypto operations
 yet. A dedicated tool is used for changing the mode from Commissioning to
@@ -113,6 +116,10 @@ Driver options
   The identifier of the credential, default value is 0 represents the operational
   register credential.
 
+- ``keytag`` parameter [int]
+
+  The plaintext of the keytag appanded to the AES-XTS keys, default value is 0.
+
 
 Supported NICs
 --------------
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index a16578b3af..b24e68532c 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -498,56 +498,52 @@ mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque)
 		attr->session_import_kek_ptr = (uint32_t)tmp;
 	else if (strcmp(key, "credential_id") == 0)
 		attr->credential_pointer = (uint32_t)tmp;
+	else if (strcmp(key, "keytag") == 0)
+		devarg_prms->keytag = tmp;
 	else
 		DRV_LOG(WARNING, "Invalid key %s.", key);
 	return 0;
 }
 
-static struct mlx5_devx_obj *
-mlx5_crypto_config_login(struct rte_devargs *devargs,
-			 struct ibv_context *ctx)
+static int
+mlx5_crypto_parse_devargs(struct rte_devargs *devargs,
+			  struct mlx5_crypto_devarg_params *devarg_prms)
 {
-	/*
-	 * Set credential pointer and session import KEK pointer to a default
-	 * value of 0.
-	 */
-	struct mlx5_crypto_devarg_params login = {
-			.login_devarg = false,
-			.login_attr = {
-					.credential_pointer = 0,
-					.session_import_kek_ptr = 0,
-			}
-	};
+	struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr;
 	struct rte_kvargs *kvlist;
 
+	/* Default values. */
+	attr->credential_pointer = 0;
+	attr->session_import_kek_ptr = 0;
+	devarg_prms->keytag = 0;
 	if (devargs == NULL) {
 		DRV_LOG(ERR,
 	"No login devargs in order to enable crypto operations in the device.");
 		rte_errno = EINVAL;
-		return NULL;
+		return -1;
 	}
 	kvlist = rte_kvargs_parse(devargs->args, NULL);
 	if (kvlist == NULL) {
 		DRV_LOG(ERR, "Failed to parse devargs.");
 		rte_errno = EINVAL;
-		return NULL;
+		return -1;
 	}
 	if (rte_kvargs_process(kvlist, NULL, mlx5_crypto_args_check_handler,
-			   &login) != 0) {
+			   devarg_prms) != 0) {
 		DRV_LOG(ERR, "Devargs handler function Failed.");
 		rte_kvargs_free(kvlist);
 		rte_errno = EINVAL;
-		return NULL;
+		return -1;
 	}
 	rte_kvargs_free(kvlist);
-	if (login.login_devarg == false) {
+	if (devarg_prms->login_devarg == false) {
 		DRV_LOG(ERR,
 	"No login credential devarg in order to enable crypto operations "
 	"in the device.");
 		rte_errno = EINVAL;
-		return NULL;
+		return -1;
 	}
-	return mlx5_devx_cmd_create_crypto_login_obj(ctx, &login.login_attr);
+	return 0;
 }
 
 /**
@@ -607,6 +603,7 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	struct ibv_context *ctx;
 	struct mlx5_devx_obj *login;
 	struct mlx5_crypto_priv *priv;
+	struct mlx5_crypto_devarg_params devarg_prms = { 0 };
 	struct mlx5_hca_attr attr = { 0 };
 	struct rte_cryptodev_pmd_init_params init_params = {
 		.name = "",
@@ -615,6 +612,8 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 		.max_nb_queue_pairs =
 				RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
 	};
+	int ret;
+
 	RTE_SET_USED(pci_drv);
 	if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
 		DRV_LOG(ERR, "Non-primary process type is not supported.");
@@ -644,7 +643,13 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 		rte_errno = ENOTSUP;
 		return -ENOTSUP;
 	}
-	login = mlx5_crypto_config_login(pci_dev->device.devargs, ctx);
+	ret = mlx5_crypto_parse_devargs(pci_dev->device.devargs, &devarg_prms);
+	if (ret) {
+		DRV_LOG(ERR, "Failed to parse devargs.");
+		return -rte_errno;
+	}
+	login = mlx5_devx_cmd_create_crypto_login_obj(ctx,
+						      &devarg_prms.login_attr);
 	if (login == NULL) {
 		DRV_LOG(ERR, "Failed to configure login.");
 		return -rte_errno;
@@ -684,6 +689,7 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	}
 	priv->mr_scache.reg_mr_cb = mlx5_common_verbs_reg_mr;
 	priv->mr_scache.dereg_mr_cb = mlx5_common_verbs_dereg_mr;
+	priv->keytag = rte_cpu_to_be_64(devarg_prms.keytag);
 	/* Register callback function for global shared MR cache management. */
 	if (TAILQ_EMPTY(&mlx5_crypto_priv_list))
 		rte_mem_event_callback_register("MLX5_MEM_EVENT_CB",
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 9df982b23e..a513e9ee36 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -30,6 +30,7 @@ struct mlx5_crypto_priv {
 	struct rte_cryptodev_config dev_config;
 	struct mlx5_mr_share_cache mr_scache; /* Global shared MR cache. */
 	struct mlx5_devx_obj *login_obj;
+	uint64_t keytag;
 };
 
 struct mlx5_crypto_qp {
@@ -49,10 +50,10 @@ struct mlx5_crypto_dek {
 	bool size_is_48; /* Whether the key\data size is 48 bytes or not. */
 } __rte_cache_aligned;
 
-
 struct mlx5_crypto_devarg_params {
 	bool login_devarg;
 	struct mlx5_devx_crypto_login_attr login_attr;
+	uint64_t keytag;
 };
 
 int
-- 
2.25.1

[dpdk-dev] [PATCH v9 09/15] crypto/mlx5: add maximum segments devarg

[Suanming Mou]

The mlx5 HW crypto operations are done by attaching crypto property
to a memory region. Once done, every access to the memory via the
crypto-enabled memory region will result with in-line encryption or
decryption of the data.

As a result, the design choice is to provide two types of WQEs. One
is UMR WQE which sets the crypto property and the other is rdma write
WQE which sends DMA command to copy data from local MR to remote MR.

The size of the WQEs will be defined by a new devarg called
max_segs_num.

This devarg also defines the maximum segments in mbuf chain that will be
supported for crypto operations.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
 doc/guides/cryptodevs/mlx5.rst    |  4 ++++
 drivers/crypto/mlx5/mlx5_crypto.c | 33 +++++++++++++++++++++++++++----
 drivers/crypto/mlx5/mlx5_crypto.h |  6 ++++++
 3 files changed, 39 insertions(+), 4 deletions(-)

diff --git a/doc/guides/cryptodevs/mlx5.rst b/doc/guides/cryptodevs/mlx5.rst
index 5b874824db..b20f593549 100644
--- a/doc/guides/cryptodevs/mlx5.rst
+++ b/doc/guides/cryptodevs/mlx5.rst
@@ -120,6 +120,10 @@ Driver options
 
   The plaintext of the keytag appanded to the AES-XTS keys, default value is 0.
 
+- ``max_segs_num`` parameter [int]
+
+  Maximum number of mbuf chain segments(src or dest), default value is 8.
+
 
 Supported NICs
 --------------
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index b24e68532c..f7e23d16e9 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -21,6 +21,7 @@
 #define MLX5_CRYPTO_DRIVER_NAME crypto_mlx5
 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
 #define MLX5_CRYPTO_MAX_QPS 1024
+#define MLX5_CRYPTO_MAX_SEGS 56
 
 #define MLX5_CRYPTO_FEATURE_FLAGS \
 	(RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | \
@@ -494,14 +495,24 @@ mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque)
 		DRV_LOG(WARNING, "%s: \"%s\" is an invalid integer.", key, val);
 		return -errno;
 	}
-	if (strcmp(key, "import_kek_id") == 0)
+	if (strcmp(key, "max_segs_num") == 0) {
+		if (!tmp || tmp > MLX5_CRYPTO_MAX_SEGS) {
+			DRV_LOG(WARNING, "Invalid max_segs_num: %d, should"
+				" be less than %d.",
+				(uint32_t)tmp, MLX5_CRYPTO_MAX_SEGS);
+			rte_errno = EINVAL;
+			return -rte_errno;
+		}
+		devarg_prms->max_segs_num = (uint32_t)tmp;
+	} else if (strcmp(key, "import_kek_id") == 0) {
 		attr->session_import_kek_ptr = (uint32_t)tmp;
-	else if (strcmp(key, "credential_id") == 0)
+	} else if (strcmp(key, "credential_id") == 0) {
 		attr->credential_pointer = (uint32_t)tmp;
-	else if (strcmp(key, "keytag") == 0)
+	} else if (strcmp(key, "keytag") == 0) {
 		devarg_prms->keytag = tmp;
-	else
+	} else {
 		DRV_LOG(WARNING, "Invalid key %s.", key);
+	}
 	return 0;
 }
 
@@ -516,6 +527,7 @@ mlx5_crypto_parse_devargs(struct rte_devargs *devargs,
 	attr->credential_pointer = 0;
 	attr->session_import_kek_ptr = 0;
 	devarg_prms->keytag = 0;
+	devarg_prms->max_segs_num = 8;
 	if (devargs == NULL) {
 		DRV_LOG(ERR,
 	"No login devargs in order to enable crypto operations in the device.");
@@ -612,6 +624,7 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 		.max_nb_queue_pairs =
 				RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
 	};
+	uint16_t rdmw_wqe_size;
 	int ret;
 
 	RTE_SET_USED(pci_drv);
@@ -690,6 +703,18 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	priv->mr_scache.reg_mr_cb = mlx5_common_verbs_reg_mr;
 	priv->mr_scache.dereg_mr_cb = mlx5_common_verbs_dereg_mr;
 	priv->keytag = rte_cpu_to_be_64(devarg_prms.keytag);
+	priv->max_segs_num = devarg_prms.max_segs_num;
+	priv->umr_wqe_size = sizeof(struct mlx5_wqe_umr_bsf_seg) +
+			     sizeof(struct mlx5_umr_wqe) +
+			     RTE_ALIGN(priv->max_segs_num, 4) *
+			     sizeof(struct mlx5_wqe_dseg);
+	rdmw_wqe_size = sizeof(struct mlx5_rdma_write_wqe) +
+			      sizeof(struct mlx5_wqe_dseg) *
+			      (priv->max_segs_num <= 2 ? 2 : 2 +
+			       RTE_ALIGN(priv->max_segs_num - 2, 4));
+	priv->wqe_set_size = priv->umr_wqe_size + rdmw_wqe_size;
+	priv->umr_wqe_stride = priv->umr_wqe_size / MLX5_SEND_WQE_BB;
+	priv->max_rdmar_ds = rdmw_wqe_size / sizeof(struct mlx5_wqe_dseg);
 	/* Register callback function for global shared MR cache management. */
 	if (TAILQ_EMPTY(&mlx5_crypto_priv_list))
 		rte_mem_event_callback_register("MLX5_MEM_EVENT_CB",
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index a513e9ee36..c0fa8ad4d4 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -25,12 +25,17 @@ struct mlx5_crypto_priv {
 	struct rte_cryptodev *crypto_dev;
 	void *uar; /* User Access Region. */
 	uint32_t pdn; /* Protection Domain number. */
+	uint32_t max_segs_num; /* Maximum supported data segs. */
 	struct ibv_pd *pd;
 	struct mlx5_hlist *dek_hlist; /* Dek hash list. */
 	struct rte_cryptodev_config dev_config;
 	struct mlx5_mr_share_cache mr_scache; /* Global shared MR cache. */
 	struct mlx5_devx_obj *login_obj;
 	uint64_t keytag;
+	uint16_t wqe_set_size;
+	uint16_t umr_wqe_size;
+	uint16_t umr_wqe_stride;
+	uint16_t max_rdmar_ds;
 };
 
 struct mlx5_crypto_qp {
@@ -54,6 +59,7 @@ struct mlx5_crypto_devarg_params {
 	bool login_devarg;
 	struct mlx5_devx_crypto_login_attr login_attr;
 	uint64_t keytag;
+	uint32_t max_segs_num;
 };
 
 int
-- 
2.25.1

[dpdk-dev] [PATCH v9 10/15] crypto/mlx5: add WQE set initialization

[Suanming Mou]

Currently, HW handles the WQEs much faster than the software,
Using the constant WQE set layout can initialize most of the WQE
segments in advanced, and software only needs to configure very
limited segments in datapath. This accelerates the software WQE
organize in datapath.

This commit initializes the fixed WQE set segments.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
 drivers/crypto/mlx5/mlx5_crypto.c | 83 +++++++++++++++++++++++++++++--
 drivers/crypto/mlx5/mlx5_crypto.h | 10 +++-
 2 files changed, 87 insertions(+), 6 deletions(-)

diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index f7e23d16e9..caad94d2f5 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -295,6 +295,69 @@ mlx5_crypto_qp2rts(struct mlx5_crypto_qp *qp)
 	return 0;
 }
 
+static void
+mlx5_crypto_qp_init(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp)
+{
+	uint32_t i;
+
+	for (i = 0 ; i < qp->entries_n; i++) {
+		struct mlx5_wqe_cseg *cseg = RTE_PTR_ADD(qp->umem_buf, i *
+							 priv->wqe_set_size);
+		struct mlx5_wqe_umr_cseg *ucseg = (struct mlx5_wqe_umr_cseg *)
+								     (cseg + 1);
+		struct mlx5_wqe_umr_bsf_seg *bsf =
+			(struct mlx5_wqe_umr_bsf_seg *)(RTE_PTR_ADD(cseg,
+						       priv->umr_wqe_size)) - 1;
+		struct mlx5_wqe_rseg *rseg;
+
+		/* Init UMR WQE. */
+		cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj->id << 8) |
+					 (priv->umr_wqe_size / MLX5_WSEG_SIZE));
+		cseg->flags = RTE_BE32(MLX5_COMP_ONLY_FIRST_ERR <<
+				       MLX5_COMP_MODE_OFFSET);
+		cseg->misc = rte_cpu_to_be_32(qp->mkey[i]->id);
+		ucseg->if_cf_toe_cq_res = RTE_BE32(1u << MLX5_UMRC_IF_OFFSET);
+		ucseg->mkey_mask = RTE_BE64(1u << 0); /* Mkey length bit. */
+		ucseg->ko_to_bs = rte_cpu_to_be_32
+			((RTE_ALIGN(priv->max_segs_num, 4u) <<
+			 MLX5_UMRC_KO_OFFSET) | (4 << MLX5_UMRC_TO_BS_OFFSET));
+		bsf->keytag = priv->keytag;
+		/* Init RDMA WRITE WQE. */
+		cseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size);
+		cseg->flags = RTE_BE32((MLX5_COMP_ALWAYS <<
+				      MLX5_COMP_MODE_OFFSET) |
+				      MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE);
+		rseg = (struct mlx5_wqe_rseg *)(cseg + 1);
+		rseg->rkey = rte_cpu_to_be_32(qp->mkey[i]->id);
+	}
+}
+
+static int
+mlx5_crypto_indirect_mkeys_prepare(struct mlx5_crypto_priv *priv,
+				  struct mlx5_crypto_qp *qp)
+{
+	struct mlx5_umr_wqe *umr;
+	uint32_t i;
+	struct mlx5_devx_mkey_attr attr = {
+		.pd = priv->pdn,
+		.umr_en = 1,
+		.crypto_en = 1,
+		.set_remote_rw = 1,
+		.klm_num = RTE_ALIGN(priv->max_segs_num, 4),
+	};
+
+	for (umr = (struct mlx5_umr_wqe *)qp->umem_buf, i = 0;
+	   i < qp->entries_n; i++, umr = RTE_PTR_ADD(umr, priv->wqe_set_size)) {
+		attr.klm_array = (struct mlx5_klm *)&umr->kseg[0];
+		qp->mkey[i] = mlx5_devx_cmd_mkey_create(priv->ctx, &attr);
+		if (!qp->mkey[i]) {
+			DRV_LOG(ERR, "Failed to allocate indirect mkey.");
+			return -1;
+		}
+	}
+	return 0;
+}
+
 static int
 mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 			     const struct rte_cryptodev_qp_conf *qp_conf,
@@ -305,7 +368,7 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	struct mlx5_crypto_qp *qp;
 	uint16_t log_nb_desc = rte_log2_u32(qp_conf->nb_descriptors);
 	uint32_t umem_size = RTE_BIT32(log_nb_desc) *
-			      MLX5_CRYPTO_WQE_SET_SIZE +
+			      priv->wqe_set_size +
 			      sizeof(*qp->db_rec) * 2;
 	uint32_t alloc_size = sizeof(*qp);
 	struct mlx5_devx_cq_attr cq_attr = {
@@ -315,7 +378,9 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	if (dev->data->queue_pairs[qp_id] != NULL)
 		mlx5_crypto_queue_pair_release(dev, qp_id);
 	alloc_size = RTE_ALIGN(alloc_size, RTE_CACHE_LINE_SIZE);
-	alloc_size += sizeof(struct rte_crypto_op *) * RTE_BIT32(log_nb_desc);
+	alloc_size += (sizeof(struct rte_crypto_op *) +
+		       sizeof(struct mlx5_devx_obj *)) *
+		       RTE_BIT32(log_nb_desc);
 	qp = rte_zmalloc_socket(__func__, alloc_size, RTE_CACHE_LINE_SIZE,
 				socket_id);
 	if (qp == NULL) {
@@ -360,8 +425,7 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	attr.wq_umem_id = qp->umem_obj->umem_id;
 	attr.wq_umem_offset = 0;
 	attr.dbr_umem_id = qp->umem_obj->umem_id;
-	attr.dbr_address = RTE_BIT64(log_nb_desc) *
-			   MLX5_CRYPTO_WQE_SET_SIZE;
+	attr.dbr_address = RTE_BIT64(log_nb_desc) * priv->wqe_set_size;
 	qp->qp_obj = mlx5_devx_cmd_create_qp(priv->ctx, &attr);
 	if (qp->qp_obj == NULL) {
 		DRV_LOG(ERR, "Failed to create QP(%u).", rte_errno);
@@ -370,8 +434,17 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	qp->db_rec = RTE_PTR_ADD(qp->umem_buf, (uintptr_t)attr.dbr_address);
 	if (mlx5_crypto_qp2rts(qp))
 		goto error;
-	qp->ops = (struct rte_crypto_op **)RTE_ALIGN((uintptr_t)(qp + 1),
+	qp->mkey = (struct mlx5_devx_obj **)RTE_ALIGN((uintptr_t)(qp + 1),
 							   RTE_CACHE_LINE_SIZE);
+	qp->ops = (struct rte_crypto_op **)(qp->mkey + RTE_BIT32(log_nb_desc));
+	qp->entries_n = 1 << log_nb_desc;
+	if (mlx5_crypto_indirect_mkeys_prepare(priv, qp)) {
+		DRV_LOG(ERR, "Cannot allocate indirect memory regions.");
+		rte_errno = ENOMEM;
+		goto error;
+	}
+	mlx5_crypto_qp_init(priv, qp);
+	qp->priv = priv;
 	dev->data->queue_pairs[qp_id] = qp;
 	return 0;
 error:
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index c0fa8ad4d4..235a43a72a 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -16,7 +16,6 @@
 
 #define MLX5_CRYPTO_DEK_HTABLE_SZ (1 << 11)
 #define MLX5_CRYPTO_KEY_LENGTH 80
-#define MLX5_CRYPTO_WQE_SET_SIZE 1024
 
 struct mlx5_crypto_priv {
 	TAILQ_ENTRY(mlx5_crypto_priv) next;
@@ -24,6 +23,7 @@ struct mlx5_crypto_priv {
 	struct rte_pci_device *pci_dev;
 	struct rte_cryptodev *crypto_dev;
 	void *uar; /* User Access Region. */
+	volatile uint64_t *uar_addr;
 	uint32_t pdn; /* Protection Domain number. */
 	uint32_t max_segs_num; /* Maximum supported data segs. */
 	struct ibv_pd *pd;
@@ -39,13 +39,21 @@ struct mlx5_crypto_priv {
 };
 
 struct mlx5_crypto_qp {
+	struct mlx5_crypto_priv *priv;
 	struct mlx5_devx_cq cq_obj;
 	struct mlx5_devx_obj *qp_obj;
+	struct rte_cryptodev_stats stats;
 	struct mlx5dv_devx_umem *umem_obj;
 	void *umem_buf;
 	volatile uint32_t *db_rec;
 	struct rte_crypto_op **ops;
+	struct mlx5_devx_obj **mkey; /* WQE's indirect mekys. */
 	struct mlx5_mr_ctrl mr_ctrl;
+	uint8_t *wqe;
+	uint16_t entries_n;
+	uint16_t pi;
+	uint16_t ci;
+	uint16_t db_pi;
 };
 
 struct mlx5_crypto_dek {
-- 
2.25.1

[dpdk-dev] [PATCH v9 11/15] crypto/mlx5: add enqueue and dequeue operations

[Suanming Mou]

The crypto operations are done with the WQE set which contains
one UMR WQE and one rdma write WQE. Most segments of the WQE
set are initialized properly during queue setup, only limited
segments are initialized according to the crypto detail in the
datapath process.

This commit adds the enquue and dequeue operations and updates
the WQE set segments accordingly.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
Signed-off-by: Michael Baum <michaelba@nvidia.com>
---
 doc/guides/cryptodevs/features/mlx5.ini |   5 +
 drivers/crypto/mlx5/mlx5_crypto.c       | 286 +++++++++++++++++++++++-
 drivers/crypto/mlx5/mlx5_crypto.h       |   3 +
 3 files changed, 290 insertions(+), 4 deletions(-)

diff --git a/doc/guides/cryptodevs/features/mlx5.ini b/doc/guides/cryptodevs/features/mlx5.ini
index bd757b5211..a89526add0 100644
--- a/doc/guides/cryptodevs/features/mlx5.ini
+++ b/doc/guides/cryptodevs/features/mlx5.ini
@@ -6,6 +6,11 @@
 [Features]
 Symmetric crypto       = Y
 HW Accelerated         = Y
+In Place SGL           = Y
+OOP SGL In SGL Out     = Y
+OOP SGL In LB  Out     = Y
+OOP LB  In SGL Out     = Y
+OOP LB  In LB  Out     = Y
 Cipher multiple data units = Y
 Cipher wrapped key     = Y
 
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index caad94d2f5..4cd1d41588 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -25,6 +25,10 @@
 
 #define MLX5_CRYPTO_FEATURE_FLAGS \
 	(RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | \
+	 RTE_CRYPTODEV_FF_IN_PLACE_SGL | RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT | \
+	 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | \
+	 RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT | \
+	 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | \
 	 RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY | \
 	 RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS)
 
@@ -295,6 +299,279 @@ mlx5_crypto_qp2rts(struct mlx5_crypto_qp *qp)
 	return 0;
 }
 
+static __rte_noinline uint32_t
+mlx5_crypto_get_block_size(struct rte_crypto_op *op)
+{
+	uint32_t bl = op->sym->cipher.data.length;
+
+	switch (bl) {
+	case (1 << 20):
+		return RTE_BE32(MLX5_BLOCK_SIZE_1MB << MLX5_BLOCK_SIZE_OFFSET);
+	case (1 << 12):
+		return RTE_BE32(MLX5_BLOCK_SIZE_4096B <<
+				MLX5_BLOCK_SIZE_OFFSET);
+	case (1 << 9):
+		return RTE_BE32(MLX5_BLOCK_SIZE_512B << MLX5_BLOCK_SIZE_OFFSET);
+	default:
+		DRV_LOG(ERR, "Unknown block size: %u.", bl);
+		return UINT32_MAX;
+	}
+}
+
+/**
+ * Query LKey from a packet buffer for QP. If not found, add the mempool.
+ *
+ * @param priv
+ *   Pointer to the priv object.
+ * @param addr
+ *   Search key.
+ * @param mr_ctrl
+ *   Pointer to per-queue MR control structure.
+ * @param ol_flags
+ *   Mbuf offload features.
+ *
+ * @return
+ *   Searched LKey on success, UINT32_MAX on no match.
+ */
+static __rte_always_inline uint32_t
+mlx5_crypto_addr2mr(struct mlx5_crypto_priv *priv, uintptr_t addr,
+		    struct mlx5_mr_ctrl *mr_ctrl, uint64_t ol_flags)
+{
+	uint32_t lkey;
+
+	/* Check generation bit to see if there's any change on existing MRs. */
+	if (unlikely(*mr_ctrl->dev_gen_ptr != mr_ctrl->cur_gen))
+		mlx5_mr_flush_local_cache(mr_ctrl);
+	/* Linear search on MR cache array. */
+	lkey = mlx5_mr_lookup_lkey(mr_ctrl->cache, &mr_ctrl->mru,
+				   MLX5_MR_CACHE_N, addr);
+	if (likely(lkey != UINT32_MAX))
+		return lkey;
+	/* Take slower bottom-half on miss. */
+	return mlx5_mr_addr2mr_bh(priv->pd, 0, &priv->mr_scache, mr_ctrl, addr,
+				  !!(ol_flags & EXT_ATTACHED_MBUF));
+}
+
+static __rte_always_inline uint32_t
+mlx5_crypto_klm_set(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp,
+		      struct rte_mbuf *mbuf, struct mlx5_wqe_dseg *klm,
+		      uint32_t offset, uint32_t *remain)
+{
+	uint32_t data_len = (rte_pktmbuf_data_len(mbuf) - offset);
+	uintptr_t addr = rte_pktmbuf_mtod_offset(mbuf, uintptr_t, offset);
+
+	if (data_len > *remain)
+		data_len = *remain;
+	*remain -= data_len;
+	klm->bcount = rte_cpu_to_be_32(data_len);
+	klm->pbuf = rte_cpu_to_be_64(addr);
+	klm->lkey = mlx5_crypto_addr2mr(priv, addr, &qp->mr_ctrl,
+					mbuf->ol_flags);
+	return klm->lkey;
+
+}
+
+static __rte_always_inline uint32_t
+mlx5_crypto_klms_set(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp,
+		     struct rte_crypto_op *op, struct rte_mbuf *mbuf,
+		     struct mlx5_wqe_dseg *klm)
+{
+	uint32_t remain_len = op->sym->cipher.data.length;
+	uint32_t nb_segs = mbuf->nb_segs;
+	uint32_t klm_n = 1u;
+
+	/* First mbuf needs to take the cipher offset. */
+	if (unlikely(mlx5_crypto_klm_set(priv, qp, mbuf, klm,
+		     op->sym->cipher.data.offset, &remain_len) == UINT32_MAX)) {
+		op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		return 0;
+	}
+	while (remain_len) {
+		nb_segs--;
+		mbuf = mbuf->next;
+		if (unlikely(mbuf == NULL || nb_segs == 0)) {
+			op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+			return 0;
+		}
+		if (unlikely(mlx5_crypto_klm_set(priv, qp, mbuf, ++klm, 0,
+						 &remain_len) == UINT32_MAX)) {
+			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+			return 0;
+		}
+		klm_n++;
+	}
+	return klm_n;
+}
+
+static __rte_always_inline int
+mlx5_crypto_wqe_set(struct mlx5_crypto_priv *priv,
+			 struct mlx5_crypto_qp *qp,
+			 struct rte_crypto_op *op,
+			 struct mlx5_umr_wqe *umr)
+{
+	struct mlx5_crypto_session *sess = get_sym_session_private_data
+				(op->sym->session, mlx5_crypto_driver_id);
+	struct mlx5_wqe_cseg *cseg = &umr->ctr;
+	struct mlx5_wqe_mkey_cseg *mkc = &umr->mkc;
+	struct mlx5_wqe_dseg *klms = &umr->kseg[0];
+	struct mlx5_wqe_umr_bsf_seg *bsf = ((struct mlx5_wqe_umr_bsf_seg *)
+				      RTE_PTR_ADD(umr, priv->umr_wqe_size)) - 1;
+	uint32_t ds;
+	bool ipl = op->sym->m_dst == NULL || op->sym->m_dst == op->sym->m_src;
+	/* Set UMR WQE. */
+	uint32_t klm_n = mlx5_crypto_klms_set(priv, qp, op,
+				   ipl ? op->sym->m_src : op->sym->m_dst, klms);
+
+	if (unlikely(klm_n == 0))
+		return 0;
+	bsf->bs_bpt_eo_es = sess->bs_bpt_eo_es;
+	if (unlikely(!sess->bsp_res)) {
+		bsf->bsp_res = mlx5_crypto_get_block_size(op);
+		if (unlikely(bsf->bsp_res == UINT32_MAX)) {
+			op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+			return 0;
+		}
+	} else {
+		bsf->bsp_res = sess->bsp_res;
+	}
+	bsf->raw_data_size = rte_cpu_to_be_32(op->sym->cipher.data.length);
+	memcpy(bsf->xts_initial_tweak,
+	       rte_crypto_op_ctod_offset(op, uint8_t *, sess->iv_offset), 16);
+	bsf->res_dp = sess->dek_id;
+	mkc->len = rte_cpu_to_be_64(op->sym->cipher.data.length);
+	cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) | MLX5_OPCODE_UMR);
+	qp->db_pi += priv->umr_wqe_stride;
+	/* Set RDMA_WRITE WQE. */
+	cseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size);
+	klms = RTE_PTR_ADD(cseg, sizeof(struct mlx5_rdma_write_wqe));
+	if (!ipl) {
+		klm_n = mlx5_crypto_klms_set(priv, qp, op, op->sym->m_src,
+					     klms);
+		if (unlikely(klm_n == 0))
+			return 0;
+	} else {
+		memcpy(klms, &umr->kseg[0], sizeof(*klms) * klm_n);
+	}
+	ds = 2 + klm_n;
+	cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj->id << 8) | ds);
+	cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) |
+							MLX5_OPCODE_RDMA_WRITE);
+	ds = RTE_ALIGN(ds, 4);
+	qp->db_pi += ds >> 2;
+	/* Set NOP WQE if needed. */
+	if (priv->max_rdmar_ds > ds) {
+		cseg += ds;
+		ds = priv->max_rdmar_ds - ds;
+		cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj->id << 8) | ds);
+		cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) |
+							       MLX5_OPCODE_NOP);
+		qp->db_pi += ds >> 2; /* Here, DS is 4 aligned for sure. */
+	}
+	qp->wqe = (uint8_t *)cseg;
+	return 1;
+}
+
+static __rte_always_inline void
+mlx5_crypto_uar_write(uint64_t val, struct mlx5_crypto_priv *priv)
+{
+#ifdef RTE_ARCH_64
+	*priv->uar_addr = val;
+#else /* !RTE_ARCH_64 */
+	rte_spinlock_lock(&priv->uar32_sl);
+	*(volatile uint32_t *)priv->uar_addr = val;
+	rte_io_wmb();
+	*((volatile uint32_t *)priv->uar_addr + 1) = val >> 32;
+	rte_spinlock_unlock(&priv->uar32_sl);
+#endif
+}
+
+static uint16_t
+mlx5_crypto_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
+			  uint16_t nb_ops)
+{
+	struct mlx5_crypto_qp *qp = queue_pair;
+	struct mlx5_crypto_priv *priv = qp->priv;
+	struct mlx5_umr_wqe *umr;
+	struct rte_crypto_op *op;
+	uint16_t mask = qp->entries_n - 1;
+	uint16_t remain = qp->entries_n - (qp->pi - qp->ci);
+
+	if (remain < nb_ops)
+		nb_ops = remain;
+	else
+		remain = nb_ops;
+	if (unlikely(remain == 0))
+		return 0;
+	do {
+		op = *ops++;
+		umr = RTE_PTR_ADD(qp->umem_buf, priv->wqe_set_size * qp->pi);
+		if (unlikely(mlx5_crypto_wqe_set(priv, qp, op, umr) == 0)) {
+			if (remain != nb_ops)
+				break;
+			return 0;
+		}
+		qp->ops[qp->pi] = op;
+		qp->pi = (qp->pi + 1) & mask;
+	} while (--remain);
+	rte_io_wmb();
+	qp->db_rec[MLX5_SND_DBR] = rte_cpu_to_be_32(qp->db_pi);
+	rte_wmb();
+	mlx5_crypto_uar_write(*(volatile uint64_t *)qp->wqe, qp->priv);
+	rte_wmb();
+	return nb_ops;
+}
+
+static __rte_noinline void
+mlx5_crypto_cqe_err_handle(struct mlx5_crypto_qp *qp, struct rte_crypto_op *op)
+{
+	const uint32_t idx = qp->ci & (qp->entries_n - 1);
+	volatile struct mlx5_err_cqe *cqe = (volatile struct mlx5_err_cqe *)
+							&qp->cq_obj.cqes[idx];
+
+	op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+	DRV_LOG(ERR, "CQE ERR:%x.\n", rte_be_to_cpu_32(cqe->syndrome));
+}
+
+static uint16_t
+mlx5_crypto_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
+			  uint16_t nb_ops)
+{
+	struct mlx5_crypto_qp *qp = queue_pair;
+	volatile struct mlx5_cqe *restrict cqe;
+	struct rte_crypto_op *restrict op;
+	const unsigned int cq_size = qp->entries_n;
+	const unsigned int mask = cq_size - 1;
+	uint32_t idx;
+	uint32_t next_idx = qp->ci & mask;
+	const uint16_t max = RTE_MIN((uint16_t)(qp->pi - qp->ci), nb_ops);
+	uint16_t i = 0;
+	int ret;
+
+	if (unlikely(max == 0))
+		return 0;
+	do {
+		idx = next_idx;
+		next_idx = (qp->ci + 1) & mask;
+		op = qp->ops[idx];
+		cqe = &qp->cq_obj.cqes[idx];
+		ret = check_cqe(cqe, cq_size, qp->ci);
+		rte_io_rmb();
+		if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) {
+			if (unlikely(ret != MLX5_CQE_STATUS_HW_OWN))
+				mlx5_crypto_cqe_err_handle(qp, op);
+			break;
+		}
+		op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+		ops[i++] = op;
+		qp->ci++;
+	} while (i < max);
+	if (likely(i != 0)) {
+		rte_io_wmb();
+		qp->cq_obj.db_rec[0] = rte_cpu_to_be_32(qp->ci);
+	}
+	return i;
+}
+
 static void
 mlx5_crypto_qp_init(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp)
 {
@@ -519,8 +796,9 @@ mlx5_crypto_hw_global_prepare(struct mlx5_crypto_priv *priv)
 	if (mlx5_crypto_pd_create(priv) != 0)
 		return -1;
 	priv->uar = mlx5_devx_alloc_uar(priv->ctx, -1);
-	if (priv->uar == NULL || mlx5_os_get_devx_uar_reg_addr(priv->uar) ==
-	    NULL) {
+	if (priv->uar)
+		priv->uar_addr = mlx5_os_get_devx_uar_reg_addr(priv->uar);
+	if (priv->uar == NULL || priv->uar_addr == NULL) {
 		rte_errno = errno;
 		claim_zero(mlx5_glue->dealloc_pd(priv->pd));
 		DRV_LOG(ERR, "Failed to allocate UAR.");
@@ -750,8 +1028,8 @@ mlx5_crypto_pci_probe(struct rte_pci_driver *pci_drv,
 	DRV_LOG(INFO,
 		"Crypto device %s was created successfully.", ibv->name);
 	crypto_dev->dev_ops = &mlx5_crypto_ops;
-	crypto_dev->dequeue_burst = NULL;
-	crypto_dev->enqueue_burst = NULL;
+	crypto_dev->dequeue_burst = mlx5_crypto_dequeue_burst;
+	crypto_dev->enqueue_burst = mlx5_crypto_enqueue_burst;
 	crypto_dev->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
 	crypto_dev->driver_id = mlx5_crypto_driver_id;
 	priv = crypto_dev->data->dev_private;
diff --git a/drivers/crypto/mlx5/mlx5_crypto.h b/drivers/crypto/mlx5/mlx5_crypto.h
index 235a43a72a..e751c9c202 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.h
+++ b/drivers/crypto/mlx5/mlx5_crypto.h
@@ -36,6 +36,9 @@ struct mlx5_crypto_priv {
 	uint16_t umr_wqe_size;
 	uint16_t umr_wqe_stride;
 	uint16_t max_rdmar_ds;
+#ifndef RTE_ARCH_64
+	rte_spinlock_t uar32_sl;
+#endif /* RTE_ARCH_64 */
 };
 
 struct mlx5_crypto_qp {
-- 
2.25.1

[dpdk-dev] [PATCH v9 12/15] crypto/mlx5: add statistic get and reset operations

[Suanming Mou]

This commit adds mlx5 crypto statistic get and reset operations.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Signed-off-by: Matan Azrad <matan@nvidia.com>
---
 doc/guides/rel_notes/release_21_08.rst |  5 ++++
 drivers/crypto/mlx5/mlx5_crypto.c      | 40 ++++++++++++++++++++++++--
 2 files changed, 42 insertions(+), 3 deletions(-)

diff --git a/doc/guides/rel_notes/release_21_08.rst b/doc/guides/rel_notes/release_21_08.rst
index 7d289e07e3..2bf4ce7a73 100644
--- a/doc/guides/rel_notes/release_21_08.rst
+++ b/doc/guides/rel_notes/release_21_08.rst
@@ -125,6 +125,11 @@ New Features
   The experimental PMD power management API now supports managing
   multiple Ethernet Rx queues per lcore.
 
+* **Added support for Nvidia crypto device driver.**
+
+  * Added mlx5 crypto driver to support AES-XTS cipher operations.
+    The first device to support it is ConnectX-6.
+
 
 Removed Items
 -------------
diff --git a/drivers/crypto/mlx5/mlx5_crypto.c b/drivers/crypto/mlx5/mlx5_crypto.c
index 4cd1d41588..fc05bb7d46 100644
--- a/drivers/crypto/mlx5/mlx5_crypto.c
+++ b/drivers/crypto/mlx5/mlx5_crypto.c
@@ -506,13 +506,17 @@ mlx5_crypto_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
 		op = *ops++;
 		umr = RTE_PTR_ADD(qp->umem_buf, priv->wqe_set_size * qp->pi);
 		if (unlikely(mlx5_crypto_wqe_set(priv, qp, op, umr) == 0)) {
-			if (remain != nb_ops)
+			qp->stats.enqueue_err_count++;
+			if (remain != nb_ops) {
+				qp->stats.enqueued_count -= remain;
 				break;
+			}
 			return 0;
 		}
 		qp->ops[qp->pi] = op;
 		qp->pi = (qp->pi + 1) & mask;
 	} while (--remain);
+	qp->stats.enqueued_count += nb_ops;
 	rte_io_wmb();
 	qp->db_rec[MLX5_SND_DBR] = rte_cpu_to_be_32(qp->db_pi);
 	rte_wmb();
@@ -529,6 +533,7 @@ mlx5_crypto_cqe_err_handle(struct mlx5_crypto_qp *qp, struct rte_crypto_op *op)
 							&qp->cq_obj.cqes[idx];
 
 	op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+	qp->stats.dequeue_err_count++;
 	DRV_LOG(ERR, "CQE ERR:%x.\n", rte_be_to_cpu_32(cqe->syndrome));
 }
 
@@ -568,6 +573,7 @@ mlx5_crypto_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
 	if (likely(i != 0)) {
 		rte_io_wmb();
 		qp->cq_obj.db_rec[0] = rte_cpu_to_be_32(qp->ci);
+		qp->stats.dequeued_count += i;
 	}
 	return i;
 }
@@ -729,14 +735,42 @@ mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 	return -1;
 }
 
+static void
+mlx5_crypto_stats_get(struct rte_cryptodev *dev,
+		      struct rte_cryptodev_stats *stats)
+{
+	int qp_id;
+
+	for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
+		struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
+
+		stats->enqueued_count += qp->stats.enqueued_count;
+		stats->dequeued_count += qp->stats.dequeued_count;
+		stats->enqueue_err_count += qp->stats.enqueue_err_count;
+		stats->dequeue_err_count += qp->stats.dequeue_err_count;
+	}
+}
+
+static void
+mlx5_crypto_stats_reset(struct rte_cryptodev *dev)
+{
+	int qp_id;
+
+	for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
+		struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
+
+		memset(&qp->stats, 0, sizeof(qp->stats));
+	}
+}
+
 static struct rte_cryptodev_ops mlx5_crypto_ops = {
 	.dev_configure			= mlx5_crypto_dev_configure,
 	.dev_start			= mlx5_crypto_dev_start,
 	.dev_stop			= mlx5_crypto_dev_stop,
 	.dev_close			= mlx5_crypto_dev_close,
 	.dev_infos_get			= mlx5_crypto_dev_infos_get,
-	.stats_get			= NULL,
-	.stats_reset			= NULL,
+	.stats_get			= mlx5_crypto_stats_get,
+	.stats_reset			= mlx5_crypto_stats_reset,
 	.queue_pair_setup		= mlx5_crypto_queue_pair_setup,
 	.queue_pair_release		= mlx5_crypto_queue_pair_release,
 	.sym_session_get_size		= mlx5_crypto_sym_session_get_size,
-- 
2.25.1

[dpdk-dev] [PATCH v9 13/15] test/crypto: add data-unit and wrapped vectors

[Suanming Mou]

From: Shiri Kuzin <shirik@nvidia.com>

The AES-XTS algorithm supports using a wrapped key.
In AES-XTS the data-unit defines the data block size to be
encrypted\decrypted.

Add AES-XTS vectors with a wrapped key.
Add a variable stating whether the key is wrapped or not.
Add the AES-XTS data-unit.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 app/test/test_cryptodev_aes_test_vectors.h | 1340 ++++++++++++++++++++
 app/test/test_cryptodev_blockcipher.c      |   10 +-
 app/test/test_cryptodev_blockcipher.h      |    2 +
 3 files changed, 1351 insertions(+), 1 deletion(-)

diff --git a/app/test/test_cryptodev_aes_test_vectors.h b/app/test/test_cryptodev_aes_test_vectors.h
index e404050701..dd7ffde685 100644
--- a/app/test/test_cryptodev_aes_test_vectors.h
+++ b/app/test/test_cryptodev_aes_test_vectors.h
@@ -2609,6 +2609,1298 @@ blockcipher_test_data aes_test_data_xts_key_64_pt_48 = {
 	}
 };
 
+static const uint8_t plaintext_aes256xts_512bytes[] = {
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79
+};
+static const uint8_t ciphertext_aes256xts_512bytes[] = {
+	0xD3, 0x94, 0x3E, 0xC9, 0xD3, 0x43, 0x13, 0xD3,
+	0x07, 0x7E, 0x51, 0x11, 0x97, 0xA5, 0xB1, 0xB2,
+	0xB0, 0x55, 0xF8, 0xD6, 0xA2, 0x10, 0x78, 0x76,
+	0x37, 0x95, 0x4F, 0x25, 0x99, 0xCC, 0x12, 0xD6,
+	0xBB, 0x2A, 0x24, 0xF4, 0x2D, 0x82, 0xFF, 0x56,
+	0xEC, 0x55, 0x3B, 0xDD, 0xF9, 0xE1, 0xC8, 0x48,
+	0x68, 0x16, 0xD2, 0x2B, 0x95, 0x7C, 0x1F, 0xCB,
+	0x32, 0xD6, 0x8F, 0x9A, 0x2E, 0xF7, 0x9B, 0xBE,
+	0x72, 0xFC, 0x55, 0x33, 0x27, 0x21, 0x2F, 0x69,
+	0xCA, 0x76, 0xA5, 0x9B, 0x21, 0x2E, 0x40, 0x57,
+	0x2C, 0x1C, 0x98, 0x41, 0x9A, 0x5E, 0x55, 0x38,
+	0xDE, 0xC2, 0x09, 0x57, 0x32, 0xA3, 0x34, 0x40,
+	0xC1, 0x8D, 0xCF, 0x70, 0x15, 0xF2, 0x6F, 0x4F,
+	0x6A, 0x04, 0xA6, 0x6D, 0xFF, 0x53, 0x25, 0x6A,
+	0x0E, 0xD0, 0x87, 0x63, 0xA1, 0x6C, 0xB1, 0x99,
+	0x4A, 0x42, 0xF5, 0xF6, 0xEA, 0xA4, 0xEB, 0x6D,
+	0x70, 0x9B, 0x0F, 0x85, 0xE2, 0x43, 0x6C, 0x27,
+	0x54, 0x57, 0x52, 0x1C, 0xCF, 0x72, 0x11, 0x83,
+	0xC3, 0xF5, 0xC6, 0xB6, 0x07, 0xEC, 0x1A, 0xF5,
+	0xAC, 0xA1, 0xF2, 0x3A, 0x01, 0x53, 0x0C, 0xA4,
+	0x40, 0x19, 0xAF, 0x7B, 0x2D, 0xE7, 0x56, 0x8B,
+	0x71, 0x5F, 0x8D, 0x96, 0xF1, 0x5D, 0x33, 0xAC,
+	0xE7, 0xAD, 0x96, 0xDB, 0xBB, 0xF7, 0xF5, 0x3F,
+	0x0E, 0x02, 0x2E, 0x80, 0xE9, 0xA2, 0x30, 0xD2,
+	0x82, 0x65, 0xEC, 0x1A, 0xB2, 0xE8, 0x98, 0x23,
+	0x42, 0x79, 0x43, 0x85, 0x3B, 0xF9, 0xFB, 0xBB,
+	0xC5, 0x29, 0xAD, 0x95, 0xAE, 0x22, 0x5B, 0x26,
+	0xDF, 0x76, 0x65, 0x37, 0x17, 0xBC, 0x58, 0xBB,
+	0x1B, 0x0A, 0x71, 0xDC, 0x68, 0xFF, 0x90, 0x36,
+	0x37, 0xB7, 0x49, 0x5E, 0x49, 0x4E, 0xE4, 0x1E,
+	0x24, 0x39, 0x48, 0xC7, 0x68, 0x19, 0xED, 0x94,
+	0xB5, 0xF6, 0x85, 0xFA, 0xE6, 0xB8, 0x2B, 0x9E,
+	0x07, 0x9E, 0xFB, 0x1D, 0x61, 0x06, 0x47, 0x2A,
+	0x3E, 0x1E, 0xD8, 0x52, 0xFB, 0xB7, 0xE3, 0xB4,
+	0x0D, 0xA8, 0x15, 0x1E, 0x98, 0x02, 0xBD, 0x89,
+	0x89, 0xE3, 0x38, 0x2C, 0xAB, 0x50, 0x25, 0x30,
+	0xB4, 0x5E, 0xA5, 0xCD, 0xA8, 0x9B, 0xA4, 0x2A,
+	0xED, 0x19, 0x3B, 0xC3, 0x05, 0x07, 0x57, 0xB5,
+	0x52, 0x11, 0x74, 0x95, 0x51, 0x5A, 0xD8, 0xED,
+	0xDF, 0x37, 0x91, 0x83, 0x27, 0xD5, 0x08, 0x82,
+	0xB3, 0x42, 0x08, 0xC7, 0x81, 0x35, 0x5F, 0x58,
+	0x28, 0x69, 0x0D, 0x97, 0x1D, 0x28, 0xE6, 0xB0,
+	0x58, 0x93, 0xCE, 0x2A, 0xB4, 0x7D, 0x4B, 0x83,
+	0x20, 0x1E, 0x08, 0xEF, 0x68, 0x51, 0xEB, 0xB4,
+	0xFA, 0x78, 0xB5, 0xE5, 0x2D, 0x93, 0x07, 0x99,
+	0xBB, 0xDD, 0x9A, 0x4E, 0xD6, 0xE7, 0x31, 0x9F,
+	0x4D, 0xB4, 0x05, 0x45, 0x89, 0x59, 0x42, 0xFF,
+	0x0C, 0xD4, 0xC2, 0xF3, 0xA0, 0xF7, 0xD3, 0xCE,
+	0x26, 0x2F, 0x9B, 0xCB, 0x98, 0x24, 0xDE, 0x50,
+	0xF6, 0x81, 0x24, 0xA3, 0xB4, 0x99, 0x94, 0xDB,
+	0xF5, 0xF4, 0x74, 0xAC, 0xED, 0xF1, 0xD6, 0x2F,
+	0x89, 0x48, 0x41, 0x24, 0xBB, 0xA7, 0x64, 0x48,
+	0xAE, 0x36, 0xE3, 0xF3, 0xA8, 0x37, 0x4F, 0xE6,
+	0x24, 0x92, 0xC4, 0x27, 0x86, 0x68, 0xC4, 0xA2,
+	0x2F, 0xB2, 0xA8, 0xFF, 0x27, 0x5F, 0x59, 0xB9,
+	0x86, 0x89, 0x5A, 0x09, 0xB8, 0xDA, 0xD6, 0xBA,
+	0x26, 0x86, 0x51, 0xC2, 0xDF, 0xCF, 0x07, 0x49,
+	0x1C, 0xB7, 0x1A, 0x80, 0xD4, 0x97, 0x13, 0xAA,
+	0xE8, 0x38, 0xA7, 0x2C, 0xB7, 0xE2, 0xE0, 0x33,
+	0xF9, 0xC3, 0x21, 0xF6, 0x2A, 0xAA, 0x1A, 0xF7,
+	0x39, 0x1F, 0x2F, 0x7E, 0x38, 0xC8, 0x82, 0x3F,
+	0xCC, 0xD8, 0xE2, 0xD9, 0x02, 0xE6, 0x8E, 0x31,
+	0xA7, 0x37, 0xE8, 0x59, 0x3F, 0xE1, 0x60, 0x1C,
+	0x74, 0x52, 0xE4, 0x49, 0xF7, 0x21, 0x60, 0x11
+};
+static const struct
+blockcipher_test_data aes_test_data_xts_wrapped_key_48_pt_512_du_512 = {
+	.crypto_algo = RTE_CRYPTO_CIPHER_AES_XTS,
+	.cipher_key = {
+		.data = {
+			0x94, 0xD8, 0xAE, 0xD7, 0x74, 0xBF, 0x51, 0xF1,
+			0x2D, 0x19, 0x0B, 0x9A, 0xB3, 0xDC, 0x1E, 0xA9,
+			0x5C, 0xC5, 0x47, 0xC1, 0xBB, 0x94, 0xDB, 0xFA,
+			0x2B, 0x1E, 0x54, 0xA5, 0x76, 0xAD, 0x07, 0x4B,
+			0x46, 0xE0, 0xAA, 0xF9, 0xF7, 0x5C, 0x64, 0x2D,
+			0x8F, 0x04, 0x17, 0x72, 0xE5, 0x8D, 0xFE, 0x9C
+		},
+		.len = 48
+	},
+	.iv = {
+		.data = {
+			0x4F, 0xAE, 0xF7, 0x11, 0x7C, 0xDA, 0x59, 0xC6,
+			0x6E, 0x4B, 0x92, 0x01, 0x3E, 0x76, 0x8A, 0xD5
+		},
+		.len = 16
+	},
+	.plaintext = {
+		.data = plaintext_aes256xts_512bytes,
+		.len = 512
+	},
+	.ciphertext = {
+		.data = ciphertext_aes256xts_512bytes,
+		.len = 512
+	},
+	.xts_dataunit_len = 512,
+	.wrapped_key = true
+};
+static const uint8_t plaintext_aes256xts_4096bytes[] = {
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
+	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79
+};
+static const uint8_t ciphertext_aes256xts_4096bytes[] = {
+	0xD3, 0x94, 0x3E, 0xC9, 0xD3, 0x43, 0x13, 0xD3,
+	0x07, 0x7E, 0x51, 0x11, 0x97, 0xA5, 0xB1, 0xB2,
+	0xB0, 0x55, 0xF8, 0xD6, 0xA2, 0x10, 0x78, 0x76,
+	0x37, 0x95, 0x4F, 0x25, 0x99, 0xCC, 0x12, 0xD6,
+	0xBB, 0x2A, 0x24, 0xF4, 0x2D, 0x82, 0xFF, 0x56,
+	0xEC, 0x55, 0x3B, 0xDD, 0xF9, 0xE1, 0xC8, 0x48,
+	0x68, 0x16, 0xD2, 0x2B, 0x95, 0x7C, 0x1F, 0xCB,
+	0x32, 0xD6, 0x8F, 0x9A, 0x2E, 0xF7, 0x9B, 0xBE,
+	0x72, 0xFC, 0x55, 0x33, 0x27, 0x21, 0x2F, 0x69,
+	0xCA, 0x76, 0xA5, 0x9B, 0x21, 0x2E, 0x40, 0x57,
+	0x2C, 0x1C, 0x98, 0x41, 0x9A, 0x5E, 0x55, 0x38,
+	0xDE, 0xC2, 0x09, 0x57, 0x32, 0xA3, 0x34, 0x40,
+	0xC1, 0x8D, 0xCF, 0x70, 0x15, 0xF2, 0x6F, 0x4F,
+	0x6A, 0x04, 0xA6, 0x6D, 0xFF, 0x53, 0x25, 0x6A,
+	0x0E, 0xD0, 0x87, 0x63, 0xA1, 0x6C, 0xB1, 0x99,
+	0x4A, 0x42, 0xF5, 0xF6, 0xEA, 0xA4, 0xEB, 0x6D,
+	0x70, 0x9B, 0x0F, 0x85, 0xE2, 0x43, 0x6C, 0x27,
+	0x54, 0x57, 0x52, 0x1C, 0xCF, 0x72, 0x11, 0x83,
+	0xC3, 0xF5, 0xC6, 0xB6, 0x07, 0xEC, 0x1A, 0xF5,
+	0xAC, 0xA1, 0xF2, 0x3A, 0x01, 0x53, 0x0C, 0xA4,
+	0x40, 0x19, 0xAF, 0x7B, 0x2D, 0xE7, 0x56, 0x8B,
+	0x71, 0x5F, 0x8D, 0x96, 0xF1, 0x5D, 0x33, 0xAC,
+	0xE7, 0xAD, 0x96, 0xDB, 0xBB, 0xF7, 0xF5, 0x3F,
+	0x0E, 0x02, 0x2E, 0x80, 0xE9, 0xA2, 0x30, 0xD2,
+	0x82, 0x65, 0xEC, 0x1A, 0xB2, 0xE8, 0x98, 0x23,
+	0x42, 0x79, 0x43, 0x85, 0x3B, 0xF9, 0xFB, 0xBB,
+	0xC5, 0x29, 0xAD, 0x95, 0xAE, 0x22, 0x5B, 0x26,
+	0xDF, 0x76, 0x65, 0x37, 0x17, 0xBC, 0x58, 0xBB,
+	0x1B, 0x0A, 0x71, 0xDC, 0x68, 0xFF, 0x90, 0x36,
+	0x37, 0xB7, 0x49, 0x5E, 0x49, 0x4E, 0xE4, 0x1E,
+	0x24, 0x39, 0x48, 0xC7, 0x68, 0x19, 0xED, 0x94,
+	0xB5, 0xF6, 0x85, 0xFA, 0xE6, 0xB8, 0x2B, 0x9E,
+	0x07, 0x9E, 0xFB, 0x1D, 0x61, 0x06, 0x47, 0x2A,
+	0x3E, 0x1E, 0xD8, 0x52, 0xFB, 0xB7, 0xE3, 0xB4,
+	0x0D, 0xA8, 0x15, 0x1E, 0x98, 0x02, 0xBD, 0x89,
+	0x89, 0xE3, 0x38, 0x2C, 0xAB, 0x50, 0x25, 0x30,
+	0xB4, 0x5E, 0xA5, 0xCD, 0xA8, 0x9B, 0xA4, 0x2A,
+	0xED, 0x19, 0x3B, 0xC3, 0x05, 0x07, 0x57, 0xB5,
+	0x52, 0x11, 0x74, 0x95, 0x51, 0x5A, 0xD8, 0xED,
+	0xDF, 0x37, 0x91, 0x83, 0x27, 0xD5, 0x08, 0x82,
+	0xB3, 0x42, 0x08, 0xC7, 0x81, 0x35, 0x5F, 0x58,
+	0x28, 0x69, 0x0D, 0x97, 0x1D, 0x28, 0xE6, 0xB0,
+	0x58, 0x93, 0xCE, 0x2A, 0xB4, 0x7D, 0x4B, 0x83,
+	0x20, 0x1E, 0x08, 0xEF, 0x68, 0x51, 0xEB, 0xB4,
+	0xFA, 0x78, 0xB5, 0xE5, 0x2D, 0x93, 0x07, 0x99,
+	0xBB, 0xDD, 0x9A, 0x4E, 0xD6, 0xE7, 0x31, 0x9F,
+	0x4D, 0xB4, 0x05, 0x45, 0x89, 0x59, 0x42, 0xFF,
+	0x0C, 0xD4, 0xC2, 0xF3, 0xA0, 0xF7, 0xD3, 0xCE,
+	0x26, 0x2F, 0x9B, 0xCB, 0x98, 0x24, 0xDE, 0x50,
+	0xF6, 0x81, 0x24, 0xA3, 0xB4, 0x99, 0x94, 0xDB,
+	0xF5, 0xF4, 0x74, 0xAC, 0xED, 0xF1, 0xD6, 0x2F,
+	0x89, 0x48, 0x41, 0x24, 0xBB, 0xA7, 0x64, 0x48,
+	0xAE, 0x36, 0xE3, 0xF3, 0xA8, 0x37, 0x4F, 0xE6,
+	0x24, 0x92, 0xC4, 0x27, 0x86, 0x68, 0xC4, 0xA2,
+	0x2F, 0xB2, 0xA8, 0xFF, 0x27, 0x5F, 0x59, 0xB9,
+	0x86, 0x89, 0x5A, 0x09, 0xB8, 0xDA, 0xD6, 0xBA,
+	0x26, 0x86, 0x51, 0xC2, 0xDF, 0xCF, 0x07, 0x49,
+	0x1C, 0xB7, 0x1A, 0x80, 0xD4, 0x97, 0x13, 0xAA,
+	0xE8, 0x38, 0xA7, 0x2C, 0xB7, 0xE2, 0xE0, 0x33,
+	0xF9, 0xC3, 0x21, 0xF6, 0x2A, 0xAA, 0x1A, 0xF7,
+	0x39, 0x1F, 0x2F, 0x7E, 0x38, 0xC8, 0x82, 0x3F,
+	0xCC, 0xD8, 0xE2, 0xD9, 0x02, 0xE6, 0x8E, 0x31,
+	0xA7, 0x37, 0xE8, 0x59, 0x3F, 0xE1, 0x60, 0x1C,
+	0x74, 0x52, 0xE4, 0x49, 0xF7, 0x21, 0x60, 0x11,
+	0x8F, 0xD0, 0xA1, 0x4B, 0xE3, 0xF1, 0x18, 0x3D,
+	0x90, 0x89, 0x54, 0x27, 0xA0, 0xF9, 0x32, 0x09,
+	0x3D, 0x9D, 0x9A, 0x09, 0x53, 0xC6, 0x7E, 0x95,
+	0x85, 0x53, 0x98, 0x4C, 0x23, 0xEA, 0x54, 0xBD,
+	0x6F, 0x50, 0xBC, 0x4C, 0xCF, 0x37, 0xC5, 0x7B,
+	0x4B, 0xCE, 0x84, 0xAF, 0xE2, 0xE2, 0x55, 0x49,
+	0xBC, 0xBF, 0x92, 0xCA, 0x1E, 0x5E, 0x10, 0xDF,
+	0x60, 0x87, 0x09, 0xA1, 0x4C, 0x1D, 0x7E, 0x1E,
+	0x59, 0xE9, 0xCF, 0xDA, 0x45, 0x3F, 0xE2, 0x0F,
+	0x53, 0x8D, 0x8B, 0x79, 0xBD, 0xD8, 0xB0, 0xE3,
+	0x5B, 0x7C, 0x55, 0x4A, 0x84, 0xF0, 0x1E, 0xF9,
+	0xE8, 0xF3, 0x09, 0x4D, 0x0B, 0xD7, 0x77, 0xCC,
+	0x3F, 0x70, 0x22, 0x7D, 0x17, 0x27, 0x48, 0x57,
+	0xE2, 0x36, 0xA0, 0x84, 0x3B, 0xDE, 0x05, 0x34,
+	0xEF, 0x55, 0x12, 0xF4, 0x9A, 0x99, 0x0D, 0x28,
+	0x86, 0x28, 0x99, 0x6B, 0x22, 0xEE, 0x63, 0xF0,
+	0x68, 0x9C, 0xE1, 0x70, 0xF6, 0x26, 0xD8, 0x3B,
+	0xF9, 0x57, 0x18, 0x3D, 0xAD, 0x66, 0xF0, 0xCF,
+	0x7B, 0x0C, 0x28, 0x4D, 0xB8, 0xEB, 0x7B, 0x04,
+	0x1E, 0x7D, 0x40, 0x5F, 0x5A, 0x1E, 0x7E, 0x08,
+	0x7F, 0x4C, 0x1E, 0x18, 0xE5, 0x3E, 0x6C, 0x90,
+	0x3C, 0x89, 0x13, 0x2A, 0xC4, 0x2A, 0x94, 0xB5,
+	0x3E, 0x18, 0x1C, 0x4C, 0xBA, 0xEA, 0x86, 0xD2,
+	0x05, 0xA9, 0x59, 0x9C, 0x80, 0xC2, 0x45, 0xAD,
+	0x30, 0x99, 0x18, 0x6A, 0x2F, 0x73, 0x8C, 0xF0,
+	0xFE, 0xA4, 0xBD, 0x44, 0x3E, 0xEB, 0x98, 0x75,
+	0x48, 0x08, 0x57, 0x45, 0xD8, 0x41, 0xDE, 0x61,
+	0x6D, 0x06, 0x93, 0xC4, 0x99, 0x1B, 0x23, 0xB5,
+	0x12, 0x22, 0x5C, 0xC7, 0x9E, 0x18, 0xEA, 0x64,
+	0xDB, 0xCE, 0x1A, 0xAC, 0x5D, 0x9B, 0x80, 0xE2,
+	0xBF, 0x3E, 0xC2, 0xA4, 0x78, 0x4F, 0xF1, 0xE3,
+	0x7D, 0x2A, 0x20, 0x94, 0x13, 0xCD, 0xF3, 0x1C,
+	0x33, 0x9C, 0xC1, 0x59, 0x85, 0x52, 0xCB, 0xDB,
+	0x03, 0xDF, 0x11, 0xE6, 0xAB, 0x95, 0x82, 0x65,
+	0x7A, 0x88, 0x73, 0xEE, 0xBA, 0x21, 0x1C, 0x2F,
+	0xCD, 0xD7, 0xC6, 0xE5, 0x13, 0xDE, 0x7A, 0x9E,
+	0xEE, 0x83, 0x8D, 0xC6, 0x47, 0x63, 0xE0, 0xC7,
+	0xC4, 0xBE, 0x19, 0x25, 0xEE, 0xCC, 0x0A, 0x13,
+	0x18, 0x9D, 0x34, 0x5B, 0x55, 0x6C, 0xC1, 0x6E,
+	0xBF, 0x5A, 0xC5, 0x61, 0x75, 0x77, 0x49, 0x8C,
+	0x67, 0x61, 0xE8, 0x72, 0x87, 0xE8, 0xCA, 0xBE,
+	0x6E, 0xC3, 0xD7, 0x81, 0x8C, 0x78, 0x79, 0xC8,
+	0x72, 0xDA, 0x1A, 0x40, 0x7D, 0x60, 0xE2, 0x5A,
+	0x47, 0x38, 0xA1, 0x21, 0x22, 0x6B, 0x54, 0x74,
+	0xDD, 0xF3, 0xBC, 0x96, 0x28, 0x7E, 0xC2, 0x8B,
+	0x13, 0xEE, 0x53, 0xBC, 0x34, 0x67, 0x07, 0x87,
+	0xD5, 0x6B, 0x93, 0x22, 0x21, 0xB9, 0xED, 0x17,
+	0xE4, 0xA1, 0x96, 0xB2, 0xC6, 0xFF, 0x79, 0xA0,
+	0xA7, 0xF9, 0xDD, 0x92, 0x78, 0xF7, 0xE3, 0x16,
+	0x79, 0xEF, 0xEF, 0x31, 0x4D, 0x1D, 0x75, 0xF9,
+	0xCF, 0x5A, 0x1B, 0x68, 0x16, 0x7F, 0xAF, 0x5F,
+	0x30, 0xB7, 0xEF, 0xF8, 0x94, 0x63, 0x73, 0x3D,
+	0xB3, 0x63, 0xE4, 0xE6, 0xD8, 0xAD, 0xF4, 0x80,
+	0x5E, 0x82, 0xA3, 0xFB, 0x3D, 0x0F, 0xCF, 0x59,
+	0xB8, 0x76, 0xAF, 0x27, 0x83, 0xE3, 0x2D, 0x6F,
+	0xE0, 0xF3, 0x11, 0xD5, 0xAE, 0x82, 0x14, 0x1D,
+	0x78, 0x95, 0xBF, 0x31, 0x22, 0x1B, 0x80, 0x12,
+	0x02, 0xD7, 0x4D, 0x1B, 0x92, 0xE3, 0x15, 0xBA,
+	0x67, 0xD6, 0x8F, 0xD4, 0xDA, 0xBF, 0xD4, 0x62,
+	0xAD, 0x76, 0xDA, 0x04, 0xA2, 0xEA, 0x98, 0xD3,
+	0xC3, 0x6E, 0x5F, 0x26, 0x3C, 0x5E, 0xD9, 0xEA,
+	0x09, 0xF0, 0x02, 0xFC, 0xD2, 0x11, 0xF8, 0xA8,
+	0x7E, 0xFF, 0x06, 0x28, 0x5B, 0xE5, 0x6F, 0x9A,
+	0x00, 0xE7, 0x7F, 0xB9, 0xFB, 0x59, 0xBB, 0xDD,
+	0x85, 0xF3, 0x40, 0xCE, 0xA3, 0x5E, 0x2E, 0x2E,
+	0x34, 0x91, 0x58, 0x41, 0x00, 0xB5, 0xE1, 0x88,
+	0x24, 0x51, 0xC7, 0xB6, 0xF3, 0x21, 0x52, 0x6E,
+	0xE7, 0xFC, 0x42, 0xE7, 0x9C, 0xCC, 0x1E, 0x51,
+	0x45, 0x39, 0xBE, 0x09, 0xFE, 0x1A, 0xC4, 0xF0,
+	0x79, 0xF4, 0x05, 0xC7, 0xA7, 0xF8, 0x0F, 0xB6,
+	0x5A, 0x7B, 0xD7, 0xE1, 0x6F, 0xF0, 0x9D, 0x67,
+	0xA3, 0xE3, 0x3E, 0x2E, 0xB9, 0x8C, 0x83, 0x9E,
+	0xFD, 0x2E, 0xA2, 0x92, 0x99, 0x3C, 0xC0, 0x99,
+	0x01, 0xAB, 0x0D, 0xFA, 0x55, 0x96, 0x04, 0x60,
+	0x1A, 0xAD, 0x4C, 0xBB, 0x3D, 0xBB, 0x7D, 0x8B,
+	0x9F, 0x28, 0x85, 0x7D, 0xB9, 0xE4, 0x05, 0x79,
+	0x7B, 0x63, 0xDD, 0x7F, 0x4D, 0xE7, 0x50, 0xD9,
+	0x41, 0xFF, 0x53, 0xB1, 0xCE, 0x42, 0x7B, 0xD6,
+	0x05, 0x1B, 0x4E, 0xAF, 0xC4, 0x8C, 0x17, 0xC8,
+	0x52, 0xBD, 0x03, 0x3B, 0x92, 0x57, 0x4E, 0xA8,
+	0x15, 0xC3, 0x26, 0x1C, 0x55, 0xC1, 0xFF, 0xAE,
+	0xA9, 0x26, 0x2D, 0xA7, 0x8E, 0x3A, 0x7F, 0xA3,
+	0x48, 0xA5, 0xBC, 0x14, 0x84, 0xF2, 0x90, 0xCE,
+	0x35, 0x0F, 0x64, 0x6B, 0xD8, 0x1C, 0x12, 0xFE,
+	0x5A, 0x4F, 0x0E, 0xCE, 0x81, 0x4E, 0x79, 0x6B,
+	0xCF, 0x56, 0xA7, 0xDB, 0x24, 0xBC, 0xB0, 0x84,
+	0x4C, 0xB0, 0xDA, 0xBE, 0xE6, 0x8F, 0xD7, 0x8E,
+	0x0E, 0xA0, 0xD3, 0x55, 0xC2, 0x4A, 0x34, 0x1C,
+	0xF9, 0xC7, 0x3D, 0x29, 0x70, 0x8B, 0xF0, 0x99,
+	0x61, 0xF5, 0x11, 0xFB, 0x82, 0xE2, 0x67, 0x35,
+	0x60, 0x78, 0x47, 0x81, 0x2A, 0x74, 0x5E, 0x4D,
+	0x48, 0xD3, 0x7C, 0x32, 0xCA, 0x1B, 0xD2, 0xA2,
+	0x5C, 0x3A, 0x2F, 0xCE, 0xB4, 0x6C, 0x3A, 0x6A,
+	0x8F, 0x67, 0x46, 0x12, 0xE7, 0xAE, 0x6A, 0x3B,
+	0x99, 0x04, 0x5E, 0x96, 0xD0, 0xB9, 0x84, 0xF6,
+	0xA7, 0x64, 0x11, 0xE8, 0x0C, 0x51, 0xFD, 0x3F,
+	0x18, 0xFA, 0xE8, 0x52, 0xD9, 0x4B, 0x99, 0x7A,
+	0x25, 0x2B, 0x1B, 0x21, 0xAD, 0x8C, 0xFE, 0x0D,
+	0x34, 0x51, 0x91, 0x75, 0x55, 0x6F, 0xEB, 0x9F,
+	0x42, 0xDC, 0x73, 0x7D, 0x31, 0x0A, 0x74, 0x13,
+	0x80, 0xB8, 0xC3, 0xED, 0x73, 0x9D, 0x79, 0x42,
+	0xC0, 0x33, 0xAB, 0xC1, 0xCB, 0xB9, 0xD0, 0xBE,
+	0xA0, 0x78, 0xB8, 0x3B, 0xEB, 0x3D, 0x1A, 0x3F,
+	0xFB, 0x9B, 0xAA, 0x8F, 0x89, 0xF9, 0xD1, 0x22,
+	0x82, 0xE6, 0x66, 0xEE, 0x2A, 0xFD, 0x9F, 0xF8,
+	0x92, 0x7E, 0x10, 0xF5, 0xD5, 0x23, 0x0B, 0xB1,
+	0xD6, 0xF1, 0x7A, 0x3D, 0x73, 0xE9, 0xCE, 0x7F,
+	0xE6, 0x0B, 0x17, 0xBC, 0x23, 0xAE, 0x72, 0xB6,
+	0xFA, 0x19, 0x46, 0xBB, 0xFE, 0xA4, 0xC1, 0x64,
+	0xA8, 0x5E, 0xE8, 0xBB, 0x63, 0x58, 0x19, 0x50,
+	0xAA, 0x36, 0xC2, 0x4B, 0x38, 0x24, 0xD1, 0x2E,
+	0xAE, 0xAD, 0x6E, 0x34, 0x64, 0xA8, 0xC8, 0xF2,
+	0x4E, 0x74, 0x5C, 0x98, 0xE8, 0xDF, 0x99, 0x8C,
+	0x41, 0x79, 0x60, 0x2D, 0xD5, 0xF4, 0xE3, 0xE9,
+	0x1D, 0xF6, 0x5A, 0xA9, 0x69, 0x8E, 0xA1, 0x4F,
+	0xD3, 0x1B, 0x09, 0xA8, 0x7A, 0xD1, 0xE1, 0xCF,
+	0xAC, 0xBA, 0xD0, 0xD1, 0x34, 0x34, 0x8F, 0xC1,
+	0xD1, 0xA8, 0xAF, 0x6E, 0x92, 0xE0, 0xB0, 0xF6,
+	0xF9, 0x08, 0xA0, 0xCB, 0x58, 0x34, 0xF6, 0x68,
+	0xA2, 0xBF, 0x05, 0x39, 0x63, 0xBA, 0x4F, 0xEF,
+	0xE3, 0x95, 0x69, 0xD5, 0x89, 0x7C, 0x64, 0x07,
+	0x13, 0x42, 0x14, 0xF1, 0xA6, 0x9B, 0x87, 0xE5,
+	0xF4, 0x49, 0xAE, 0x67, 0x65, 0xCC, 0xF2, 0x26,
+	0xF8, 0x31, 0xBD, 0x33, 0x6A, 0x87, 0x77, 0x4E,
+	0xB1, 0xEE, 0xA4, 0xA2, 0xC8, 0xA0, 0x4A, 0xC1,
+	0xDF, 0x55, 0xE0, 0xDE, 0x53, 0x15, 0x3B, 0xEC,
+	0x55, 0x32, 0xCA, 0x06, 0xE4, 0x78, 0x59, 0x63,
+	0x10, 0x68, 0xA9, 0x46, 0x1B, 0xEF, 0x73, 0x6D,
+	0x1A, 0x02, 0x64, 0x12, 0x76, 0x9B, 0xDB, 0x7C,
+	0x03, 0x35, 0x19, 0xE1, 0x58, 0x7A, 0x87, 0x0C,
+	0x76, 0xDC, 0xFC, 0xC0, 0x28, 0xE4, 0xA2, 0x07,
+	0x9C, 0x28, 0x05, 0x21, 0x13, 0x58, 0xEF, 0x05,
+	0xBB, 0xAB, 0x94, 0xA2, 0x93, 0xBC, 0x31, 0x61,
+	0x26, 0x39, 0x38, 0x0C, 0xC4, 0x67, 0xDA, 0xA5,
+	0xE4, 0x1E, 0x1B, 0xB6, 0xE5, 0x73, 0xD6, 0x6C,
+	0xEE, 0xBC, 0x9D, 0xB9, 0xE7, 0xD9, 0x45, 0x2F,
+	0xF2, 0xB6, 0x92, 0x54, 0x41, 0x05, 0xB7, 0xB7,
+	0xFC, 0x37, 0x63, 0x6A, 0xB4, 0xBE, 0xB8, 0x3E,
+	0xD8, 0x53, 0x3B, 0xF8, 0x7D, 0x9A, 0x05, 0xDF,
+	0x20, 0x02, 0x27, 0x64, 0x38, 0xFA, 0x7D, 0xAF,
+	0x7F, 0xFA, 0xD1, 0xB7, 0x32, 0xC5, 0x74, 0x3E,
+	0x04, 0xA2, 0x67, 0x79, 0x02, 0x2E, 0x6F, 0xA1,
+	0x27, 0x87, 0x07, 0xB5, 0x9F, 0x0A, 0x7D, 0x5E,
+	0x14, 0xA0, 0x31, 0x46, 0x3F, 0xA9, 0xDE, 0x98,
+	0xB9, 0x89, 0xA0, 0x4A, 0x7A, 0xBD, 0x15, 0xAE,
+	0x2D, 0x0B, 0x38, 0x9A, 0xD8, 0x0E, 0xD2, 0xBA,
+	0x6D, 0xA1, 0x04, 0x1E, 0x4E, 0x39, 0x87, 0x4B,
+	0xC8, 0x3C, 0x74, 0x35, 0x4D, 0xC8, 0x1B, 0x42,
+	0x06, 0x5B, 0x73, 0xB7, 0x33, 0x86, 0x4A, 0x10,
+	0x2A, 0x10, 0x16, 0x28, 0x6F, 0x2A, 0xE3, 0x86,
+	0xDE, 0xA3, 0x44, 0x23, 0xE2, 0x90, 0xC4, 0x20,
+	0x90, 0xE0, 0xB8, 0xE6, 0xA7, 0xB6, 0xD6, 0x92,
+	0xF4, 0xF8, 0x8A, 0xBC, 0xAC, 0x31, 0x47, 0x8F,
+	0xAA, 0xE0, 0xD9, 0xF7, 0xE3, 0xCB, 0x11, 0xA4,
+	0x6B, 0x05, 0xB3, 0xB8, 0x72, 0x69, 0xE6, 0xDD,
+	0x75, 0x0F, 0x20, 0x1D, 0x3F, 0xC6, 0x96, 0xA0,
+	0x18, 0xB6, 0x24, 0xA1, 0xA6, 0xFD, 0x0C, 0x80,
+	0x1E, 0xD2, 0x28, 0xA2, 0x1A, 0x27, 0xF4, 0x23,
+	0x59, 0x1A, 0xCC, 0x0F, 0xD4, 0x99, 0xD0, 0xB4,
+	0x1E, 0x91, 0xC7, 0xD8, 0x8F, 0x8C, 0x5B, 0xEB,
+	0xB5, 0x9F, 0xFF, 0x4F, 0xD0, 0xD5, 0xB7, 0x60,
+	0xCC, 0x0A, 0x10, 0x38, 0xBF, 0xA8, 0x2E, 0xCC,
+	0xEB, 0x26, 0xB0, 0x78, 0xB3, 0xE0, 0x40, 0xAF,
+	0xCD, 0x12, 0xC5, 0x3A, 0x24, 0xD8, 0xEE, 0x3A,
+	0x64, 0x83, 0x2E, 0xD9, 0x25, 0x21, 0x66, 0xA5,
+	0x28, 0xD1, 0xE1, 0x84, 0x25, 0x1B, 0x20, 0xB8,
+	0xF5, 0x76, 0xB6, 0x3E, 0x4B, 0xC6, 0xEC, 0xC1,
+	0xC7, 0xAC, 0xC4, 0xAD, 0xCE, 0xF0, 0xB4, 0x0F,
+	0x35, 0x1E, 0xCE, 0x4E, 0xE3, 0x57, 0x30, 0xFC,
+	0xF4, 0x9B, 0x86, 0xB0, 0xDD, 0x3F, 0x2F, 0xB6,
+	0x10, 0x20, 0xE4, 0x24, 0x17, 0x1C, 0x24, 0xC6,
+	0x89, 0xE4, 0x14, 0xAD, 0x2E, 0x41, 0x08, 0x33,
+	0x88, 0xB1, 0x6F, 0x11, 0x85, 0xAF, 0x58, 0x17,
+	0xE3, 0x91, 0xB4, 0x72, 0xA2, 0x7F, 0xA3, 0x98,
+	0xAF, 0xB7, 0x6B, 0x58, 0x76, 0xA3, 0x11, 0x1C,
+	0x8A, 0x1A, 0xE6, 0x58, 0x54, 0xB0, 0xB9, 0x6E,
+	0x46, 0xCB, 0x16, 0xC0, 0x63, 0x0C, 0xEE, 0xA2,
+	0xAE, 0xF6, 0x71, 0xEF, 0xD1, 0xB9, 0x3D, 0xB7,
+	0x76, 0xCE, 0x5B, 0x84, 0x66, 0x7C, 0x7D, 0xF1,
+	0x96, 0x60, 0x34, 0xF6, 0xD1, 0x64, 0x27, 0xD9,
+	0xF3, 0x78, 0x8B, 0xF4, 0xC3, 0x1D, 0x37, 0xC0,
+	0xF4, 0x4A, 0xD0, 0xA5, 0x9A, 0xEB, 0xDD, 0x79,
+	0x54, 0x5D, 0xEB, 0x04, 0xC1, 0xA4, 0xBC, 0xED,
+	0xE3, 0x74, 0xC3, 0xB9, 0x9A, 0x6A, 0xAA, 0x06,
+	0xD1, 0xF0, 0x0F, 0xC5, 0xEF, 0x7E, 0x0B, 0xC8,
+	0xF4, 0x94, 0x4E, 0x69, 0x0E, 0x36, 0x00, 0x13,
+	0x45, 0xCE, 0x68, 0x13, 0xFE, 0x7F, 0x29, 0xA2,
+	0x1D, 0x79, 0xDF, 0xF2, 0x27, 0xFB, 0xAE, 0x52,
+	0x05, 0x78, 0xD7, 0xB9, 0xF7, 0x38, 0x68, 0xD5,
+	0xBA, 0xD7, 0xCF, 0x09, 0xC6, 0xD2, 0x5B, 0xC6,
+	0x98, 0xE4, 0xEC, 0xD5, 0xE9, 0xC2, 0xA5, 0x1A,
+	0x52, 0xC8, 0xA7, 0xBA, 0x3D, 0x74, 0x75, 0x00,
+	0xAA, 0xDD, 0x6A, 0x3F, 0xB6, 0x2F, 0x08, 0xB7,
+	0x1C, 0x6B, 0x52, 0x0C, 0xC9, 0xE4, 0x4D, 0xF4,
+	0xC5, 0x26, 0x1F, 0x35, 0x41, 0x25, 0x68, 0x17,
+	0xA8, 0x81, 0x75, 0xF4, 0x66, 0x41, 0xB5, 0xE4,
+	0x1D, 0x92, 0xEE, 0xDA, 0x0F, 0x56, 0x76, 0xC6,
+	0xAA, 0x0F, 0xA8, 0x63, 0x8D, 0xF0, 0x69, 0x63,
+	0x93, 0x45, 0xBC, 0x76, 0x40, 0xBE, 0xA9, 0x96,
+	0x36, 0xAF, 0x2F, 0x6B, 0x3E, 0xAB, 0xF3, 0xC0,
+	0xD7, 0xD5, 0xB1, 0x23, 0x23, 0xA2, 0xA0, 0xC4,
+	0xC5, 0x70, 0xEF, 0x66, 0x79, 0x15, 0xF0, 0xD0,
+	0x40, 0x0A, 0x33, 0x0C, 0xF3, 0x32, 0x6D, 0x8D,
+	0xB4, 0x44, 0x46, 0x78, 0x3F, 0x8D, 0x75, 0x40,
+	0xA5, 0x60, 0xBC, 0x9B, 0x76, 0xDF, 0x25, 0xF4,
+	0xE9, 0xED, 0xAC, 0x74, 0x2F, 0x9A, 0x00, 0xC4,
+	0x2B, 0x52, 0x26, 0x79, 0x09, 0x19, 0x57, 0x89,
+	0x60, 0x14, 0xBE, 0x65, 0xBD, 0x7B, 0x4D, 0x7D,
+	0x9B, 0x8B, 0x9E, 0x72, 0x6C, 0x0C, 0x57, 0xC7,
+	0x00, 0x08, 0x38, 0x7C, 0x37, 0x45, 0x9D, 0x55,
+	0xA2, 0x62, 0x5E, 0x34, 0x19, 0x99, 0x31, 0x16,
+	0xF1, 0x14, 0x44, 0x2D, 0xE3, 0x7E, 0x22, 0xE1,
+	0xA2, 0xB8, 0x9A, 0x9F, 0xE0, 0x37, 0x29, 0xBB,
+	0xCD, 0x46, 0xEE, 0x0A, 0x62, 0x2B, 0x98, 0x34,
+	0xBA, 0x9E, 0x54, 0x1B, 0xB1, 0x5C, 0x4F, 0xE9,
+	0xAA, 0xE4, 0x95, 0x8C, 0xA4, 0xEF, 0xC2, 0xB1,
+	0x7F, 0xF9, 0x80, 0xDA, 0x55, 0x95, 0x92, 0xC0,
+	0x86, 0xF4, 0x2D, 0x99, 0x3E, 0x17, 0xDC, 0x55,
+	0xA6, 0x33, 0x85, 0x90, 0x31, 0xC8, 0xFF, 0x58,
+	0x83, 0xC5, 0xBA, 0x60, 0x20, 0x5F, 0x87, 0x29,
+	0x20, 0x5A, 0x7D, 0x44, 0x2B, 0xA0, 0xE2, 0x99,
+	0xC8, 0x70, 0xBE, 0x89, 0xC5, 0xBC, 0xF6, 0x0D,
+	0x04, 0xC0, 0x96, 0xD1, 0x5C, 0xD1, 0x90, 0x43,
+	0xD3, 0x7B, 0x73, 0x52, 0x30, 0xB6, 0xA9, 0x7C,
+	0x0A, 0xA3, 0x24, 0x0E, 0x80, 0xFE, 0xBE, 0x31,
+	0xFD, 0xB5, 0x96, 0x04, 0x2B, 0xCF, 0x0B, 0x28,
+	0x1F, 0x7A, 0xCF, 0xC4, 0x82, 0x78, 0x52, 0x30,
+	0xB1, 0x34, 0x12, 0x50, 0x03, 0x09, 0x1C, 0x8B,
+	0x80, 0x60, 0xE3, 0xA1, 0xE5, 0x61, 0xF7, 0xD7,
+	0xB6, 0x74, 0xBE, 0xD6, 0x58, 0x03, 0xD3, 0xE5,
+	0xF7, 0xAC, 0x07, 0x60, 0xB7, 0x8A, 0xEC, 0xFA,
+	0xC6, 0x0F, 0xF0, 0x20, 0x04, 0x6B, 0x8F, 0x61,
+	0x09, 0x92, 0x03, 0xFB, 0x85, 0x99, 0x94, 0x9D,
+	0x2E, 0x6A, 0xC2, 0x9F, 0x20, 0x46, 0x2A, 0x96,
+	0xED, 0x42, 0x7D, 0x64, 0xA9, 0xE4, 0x1B, 0xDE,
+	0x11, 0x20, 0x12, 0x93, 0xE6, 0x2B, 0xE5, 0x93,
+	0x48, 0x37, 0x8C, 0x5A, 0x54, 0x0D, 0xEB, 0xF0,
+	0x9F, 0x9D, 0xE4, 0xA5, 0xC4, 0x93, 0x6F, 0x6A,
+	0xE3, 0x99, 0x69, 0xD9, 0xFE, 0x0C, 0x4E, 0xEC,
+	0x8B, 0x30, 0x1F, 0x7A, 0xB8, 0xC8, 0x5B, 0x61,
+	0x8E, 0xC2, 0x10, 0x90, 0x57, 0xB4, 0x72, 0x58,
+	0x7F, 0x41, 0x29, 0x7E, 0xF9, 0xBE, 0x40, 0xC3,
+	0x6F, 0xA9, 0xE3, 0x00, 0xE9, 0xC8, 0xFD, 0x4B,
+	0xFD, 0x3F, 0xE3, 0x3F, 0x25, 0x22, 0xFD, 0xB7,
+	0x2D, 0x57, 0xEF, 0x91, 0x08, 0xF0, 0x20, 0x56,
+	0x30, 0xFA, 0x83, 0x69, 0xFD, 0x56, 0x5A, 0x9B,
+	0xCE, 0xF8, 0x28, 0x02, 0xB4, 0x91, 0x35, 0x75,
+	0x9E, 0x63, 0x99, 0x48, 0xCF, 0x35, 0xF5, 0x58,
+	0x0C, 0x48, 0x8F, 0x0A, 0x2D, 0x9A, 0xE6, 0x40,
+	0xF6, 0x21, 0xB5, 0x69, 0xC1, 0x09, 0x31, 0x00,
+	0xA3, 0xC1, 0x4C, 0x99, 0x70, 0x4F, 0x5A, 0x63,
+	0x17, 0x90, 0xB8, 0xF8, 0x3A, 0x0E, 0xFD, 0x67,
+	0xEA, 0x0E, 0xBA, 0x7B, 0x1E, 0xEF, 0x37, 0x84,
+	0xD5, 0x51, 0x37, 0x01, 0xD6, 0x93, 0x15, 0xDF,
+	0x56, 0x89, 0x0E, 0x54, 0xF5, 0x1D, 0xF7, 0xE5,
+	0xB7, 0xC1, 0xF1, 0xC2, 0xD9, 0x14, 0x6F, 0x40,
+	0x55, 0x67, 0x50, 0x7C, 0x58, 0x35, 0x8B, 0x39,
+	0xCB, 0xB5, 0x87, 0xF5, 0x55, 0x5E, 0x26, 0x8C,
+	0x5B, 0x73, 0x0E, 0xBB, 0x25, 0x51, 0x0E, 0xAD,
+	0x57, 0x72, 0x7B, 0x68, 0x83, 0x11, 0x1E, 0x3A,
+	0x3D, 0xA4, 0x7C, 0x18, 0xB7, 0x70, 0x18, 0xBC,
+	0x72, 0x03, 0x4A, 0xA1, 0xD2, 0xF9, 0xA9, 0x8A,
+	0x25, 0x45, 0x19, 0xEE, 0x93, 0x06, 0xB5, 0x09,
+	0x71, 0xC9, 0x2D, 0xFD, 0x2B, 0xF3, 0xC7, 0x64,
+	0x5F, 0xCE, 0x71, 0x1D, 0x81, 0x96, 0x67, 0xBF,
+	0x01, 0x39, 0x8C, 0xE7, 0xA2, 0xD0, 0x98, 0x57,
+	0x5A, 0xFD, 0x21, 0xC7, 0x46, 0xAA, 0xB4, 0xE4,
+	0x0E, 0xBE, 0xC6, 0x68, 0x3E, 0x38, 0xF5, 0xA2,
+	0xED, 0x73, 0xCC, 0x53, 0x7E, 0x7E, 0x03, 0x32,
+	0xDC, 0xB6, 0xC1, 0x03, 0x9E, 0xB3, 0x2A, 0xAD,
+	0xC0, 0xC3, 0x6E, 0x47, 0xFB, 0x1E, 0xB7, 0x0D,
+	0x86, 0x95, 0x09, 0xA6, 0x9D, 0x6F, 0x92, 0xFC,
+	0xFF, 0x2C, 0x7D, 0x09, 0x16, 0x68, 0x50, 0x3E,
+	0x4F, 0x23, 0x4C, 0x93, 0x95, 0x2A, 0xE1, 0x9B,
+	0x16, 0xF0, 0x0F, 0xFF, 0x79, 0xA8, 0x06, 0xF9,
+	0x70, 0x61, 0x72, 0x2C, 0xE8, 0x91, 0x01, 0x6D,
+	0x45, 0xE5, 0x82, 0x5D, 0x26, 0x21, 0xAD, 0x3D,
+	0x77, 0x73, 0x23, 0x04, 0x84, 0x27, 0xA3, 0x5D,
+	0x6D, 0xA8, 0x99, 0xC1, 0xCE, 0x4F, 0xA9, 0xF7,
+	0xAB, 0x5C, 0xDE, 0x01, 0xE6, 0x1E, 0xEF, 0xE6,
+	0xFD, 0xE0, 0x68, 0x85, 0x3E, 0xEE, 0xBF, 0xF1,
+	0x0D, 0x79, 0xF4, 0xA2, 0xB4, 0x14, 0xBC, 0x0C,
+	0x49, 0x77, 0x03, 0x71, 0x08, 0x3E, 0x40, 0xA6,
+	0xD7, 0x03, 0xFA, 0xE2, 0xFB, 0xC7, 0x59, 0x30,
+	0x6E, 0x07, 0x06, 0x1C, 0x7C, 0x47, 0xE5, 0x4C,
+	0x57, 0x0A, 0x91, 0x4A, 0x43, 0xE4, 0x8A, 0xCD,
+	0x6E, 0x92, 0x01, 0xE2, 0x52, 0xC1, 0x92, 0x34,
+	0x8E, 0x64, 0x0F, 0x39, 0x63, 0x53, 0xAB, 0xE5,
+	0x44, 0xD5, 0xAA, 0xAA, 0xF6, 0x03, 0x89, 0xB9,
+	0xDD, 0xB2, 0x2D, 0x56, 0x1A, 0xE0, 0x72, 0x5A,
+	0x52, 0x19, 0x46, 0xEA, 0xB3, 0xCE, 0xB3, 0x59,
+	0x46, 0x7A, 0xA7, 0x48, 0x37, 0x0C, 0x09, 0xBA,
+	0x92, 0x70, 0x17, 0x7F, 0xF5, 0xD3, 0x60, 0x44,
+	0xC4, 0xC6, 0xC6, 0x7D, 0xD2, 0x21, 0xAC, 0x3F,
+	0x62, 0x6C, 0xE9, 0xBA, 0x4C, 0xF3, 0x82, 0x7E,
+	0x6D, 0x3A, 0x92, 0xDC, 0x94, 0xE4, 0x5F, 0xA6,
+	0x8B, 0x66, 0xA0, 0xDD, 0xE2, 0x97, 0x83, 0xED,
+	0xF5, 0x9D, 0xDF, 0x74, 0x77, 0x23, 0x7D, 0xDA,
+	0xC4, 0xFB, 0x92, 0x1A, 0xD9, 0x37, 0x36, 0xD2,
+	0x88, 0xC9, 0xEA, 0x0F, 0x98, 0xBD, 0xC5, 0xF8,
+	0xAA, 0x19, 0x75, 0x12, 0x6A, 0x41, 0xB5, 0xB3,
+	0xB5, 0xA4, 0x96, 0xDC, 0x2B, 0x49, 0x86, 0x66,
+	0x35, 0xD8, 0x4A, 0x62, 0xB4, 0xCB, 0x1E, 0x27,
+	0xC1, 0xAD, 0x34, 0x0E, 0x26, 0x16, 0xF2, 0xC2,
+	0x22, 0x52, 0x84, 0xD8, 0xD1, 0x32, 0xB8, 0x9C,
+	0xFE, 0x64, 0x42, 0x9F, 0xE4, 0x69, 0xF0, 0xAE,
+	0x3B, 0xD9, 0x2C, 0xA0, 0x14, 0xEB, 0x69, 0x74,
+	0x7C, 0xE2, 0xA6, 0x60, 0xE1, 0x52, 0x1C, 0xCC,
+	0xBF, 0xE6, 0xA1, 0x83, 0x20, 0x5D, 0x9E, 0xA3,
+	0xFB, 0x84, 0x8B, 0x33, 0xE6, 0xC9, 0x32, 0x83,
+	0xC0, 0x3F, 0x98, 0x1D, 0x6E, 0xC0, 0x50, 0x71,
+	0x29, 0x60, 0x5F, 0x36, 0xB4, 0x68, 0x1D, 0xB9,
+	0x76, 0x73, 0xC3, 0x80, 0xC5, 0xBC, 0x59, 0x7B,
+	0x59, 0xB4, 0xE0, 0x6A, 0x80, 0xCD, 0x4D, 0x8C,
+	0x9E, 0xE0, 0x0B, 0x45, 0x7D, 0x54, 0xD4, 0xC4,
+	0x97, 0x6C, 0x54, 0xEF, 0x14, 0x64, 0xBD, 0x3B,
+	0xD7, 0xEE, 0xF4, 0xD1, 0x41, 0x76, 0x3A, 0x24,
+	0x7A, 0xC2, 0xCA, 0x68, 0x28, 0x53, 0x46, 0xF7,
+	0x1B, 0xDA, 0x4B, 0x7A, 0x56, 0x75, 0x86, 0xFB,
+	0x31, 0x2C, 0x27, 0xF9, 0x4D, 0x35, 0xA4, 0x82,
+	0xE7, 0x2F, 0x41, 0xB4, 0xCA, 0xCE, 0x75, 0x94,
+	0x08, 0x54, 0xE2, 0x9E, 0x99, 0xC9, 0x85, 0xDE,
+	0x6F, 0x80, 0x95, 0x59, 0x3E, 0x54, 0x9F, 0x31,
+	0xF8, 0xDE, 0xD0, 0xD7, 0xA6, 0xD4, 0xD3, 0xBB,
+	0xD9, 0xC7, 0x55, 0xDD, 0xAE, 0xAD, 0x9E, 0x57,
+	0x4A, 0x33, 0x5D, 0x7A, 0xA6, 0xA3, 0xCA, 0xF9,
+	0x4C, 0x5B, 0x51, 0xCC, 0x22, 0xBB, 0x76, 0x44,
+	0x17, 0xDE, 0x22, 0xA1, 0xDF, 0x80, 0x13, 0x7D,
+	0xE5, 0x34, 0x7E, 0x75, 0x73, 0x10, 0x40, 0xFB,
+	0x9A, 0x21, 0xCD, 0xD3, 0xD3, 0x84, 0xB6, 0x0C,
+	0x31, 0x1E, 0xB5, 0x42, 0xF4, 0x34, 0x11, 0x7F,
+	0x4A, 0x23, 0xA8, 0xA5, 0x8F, 0x20, 0xCD, 0xA9,
+	0xF2, 0xE4, 0xEE, 0xFA, 0x57, 0xD1, 0x22, 0x1C,
+	0xA5, 0xDC, 0x0B, 0x25, 0xFE, 0xC2, 0xA7, 0x7E,
+	0x09, 0x2E, 0xDA, 0x40, 0x9F, 0x6C, 0xC8, 0x71,
+	0x58, 0x91, 0x04, 0x25, 0xD0, 0x06, 0xEA, 0x1B,
+	0xCD, 0x9D, 0x50, 0xD8, 0x40, 0x24, 0xAC, 0xC3,
+	0xB4, 0x07, 0x6E, 0x76, 0xF4, 0x4C, 0xD8, 0x80,
+	0xD0, 0x20, 0xF5, 0x15, 0x6A, 0x0A, 0x12, 0xF8,
+	0x6B, 0x67, 0x77, 0x34, 0xAE, 0x60, 0x68, 0x13,
+	0x5B, 0x8E, 0xFF, 0x5E, 0x7A, 0x77, 0x67, 0x0D,
+	0xE6, 0x96, 0x43, 0x9F, 0x8F, 0x77, 0x5F, 0x97,
+	0x23, 0x91, 0x33, 0x72, 0xC7, 0x8A, 0xC7, 0x80,
+	0xCF, 0xE7, 0x71, 0x06, 0x25, 0xB7, 0x4B, 0x89,
+	0x6A, 0x46, 0x67, 0x19, 0x49, 0x44, 0x03, 0x52,
+	0x32, 0xB1, 0x8F, 0xE7, 0x9E, 0xDA, 0x03, 0x41,
+	0xA3, 0xAC, 0xE5, 0xF3, 0x96, 0xE6, 0xAC, 0xFF,
+	0xEC, 0x35, 0x4D, 0x83, 0xA9, 0xCE, 0x7C, 0x52,
+	0xF2, 0x36, 0x97, 0xF0, 0x28, 0x36, 0x54, 0x59,
+	0x96, 0xEA, 0xEE, 0xB2, 0xC1, 0xAB, 0xA4, 0x96,
+	0x62, 0xD4, 0x3C, 0xF0, 0x1F, 0x2D, 0x65, 0x0E,
+	0x46, 0x7E, 0x12, 0x31, 0x8F, 0xA7, 0x8D, 0x7A,
+	0x4A, 0x41, 0x15, 0x57, 0x90, 0xF6, 0xF1, 0xE8,
+	0xE8, 0xE3, 0x57, 0x7B, 0x55, 0x85, 0x95, 0x97,
+	0xB3, 0x29, 0x3D, 0x02, 0x73, 0x1E, 0x87, 0x1F,
+	0x01, 0x89, 0x06, 0x88, 0x9E, 0x8A, 0x2E, 0xE0,
+	0x99, 0xFC, 0xF0, 0x48, 0x60, 0x37, 0x65, 0x25,
+	0xDB, 0x89, 0xC9, 0x7A, 0x51, 0x7E, 0x35, 0x92,
+	0x00, 0xC9, 0x61, 0x3F, 0x88, 0xE3, 0x20, 0x01,
+	0x46, 0x5A, 0x2C, 0x37, 0x02, 0xC9, 0x15, 0x0F,
+	0xB2, 0xEB, 0xC3, 0x55, 0x18, 0xF0, 0x15, 0x1A,
+	0x08, 0x8E, 0xB8, 0x9D, 0x18, 0xE4, 0x9F, 0x34,
+	0x10, 0x67, 0x68, 0x57, 0x60, 0x61, 0xEC, 0xD5,
+	0xD9, 0xA8, 0x3A, 0xAB, 0xD6, 0xD2, 0x7A, 0x47,
+	0x3D, 0xA4, 0x08, 0x7C, 0x3E, 0x4D, 0x76, 0x08,
+	0x19, 0x22, 0xB2, 0x89, 0x57, 0x84, 0xC2, 0x98,
+	0x72, 0xB8, 0x8B, 0xE0, 0x85, 0xA1, 0x3A, 0xC2,
+	0xA0, 0x06, 0x43, 0x03, 0xCF, 0x4F, 0x27, 0x80,
+	0x48, 0x9A, 0xBC, 0xB3, 0x3C, 0xC4, 0x5E, 0xAC,
+	0x8B, 0x85, 0x6F, 0x21, 0xD6, 0xFE, 0x12, 0x90,
+	0x53, 0x2F, 0x65, 0x32, 0x8D, 0x03, 0xFE, 0xFE,
+	0x61, 0x04, 0x47, 0x24, 0x6A, 0xB5, 0x01, 0x98,
+	0xB9, 0x27, 0x10, 0xE1, 0x32, 0x3D, 0x2A, 0xA0,
+	0xC5, 0x70, 0xDE, 0x1E, 0x10, 0xD7, 0x01, 0x50,
+	0x4F, 0x87, 0xCA, 0xD9, 0xBF, 0x12, 0xEA, 0x38,
+	0x4C, 0x43, 0xD5, 0x5A, 0xEF, 0x3E, 0x21, 0x8E,
+	0x59, 0x77, 0x23, 0xED, 0x51, 0x09, 0x99, 0x73,
+	0xD2, 0x56, 0x04, 0xDD, 0x8F, 0x5F, 0xDF, 0x79,
+	0xFF, 0x16, 0x8C, 0xB0, 0xBA, 0x8A, 0x1A, 0x56,
+	0xAF, 0xCA, 0x19, 0xF2, 0x64, 0x1A, 0xF5, 0x1E,
+	0xA7, 0xA7, 0x84, 0x3D, 0xAD, 0xC1, 0x0E, 0x22,
+	0xA1, 0x45, 0xFC, 0xB4, 0x13, 0x91, 0x34, 0xB7,
+	0x48, 0xEF, 0x9E, 0xD9, 0x0B, 0xE3, 0x82, 0x75,
+	0x80, 0xC5, 0xD9, 0xA0, 0x77, 0xA3, 0xF9, 0xCC,
+	0x67, 0xDD, 0xCB, 0x28, 0xC6, 0xE8, 0x2C, 0xB8,
+	0xAC, 0x63, 0xBD, 0x3B, 0x28, 0x4A, 0xE9, 0x2D,
+	0x29, 0x84, 0xD7, 0x4F, 0x61, 0x06, 0xE3, 0x37,
+	0xC1, 0x58, 0x20, 0x5D, 0x0A, 0xE7, 0x45, 0x29,
+	0x7D, 0xED, 0x0F, 0xCE, 0x00, 0x95, 0x2A, 0x62,
+	0x38, 0xA8, 0x1A, 0x3A, 0x96, 0x0E, 0x56, 0xD9,
+	0x18, 0xC2, 0x25, 0xA5, 0xAA, 0x27, 0x0A, 0x6E,
+	0xDD, 0x1C, 0x35, 0x6C, 0xC1, 0x26, 0x90, 0xF6,
+	0x43, 0x1B, 0x34, 0xDA, 0xE1, 0x5D, 0x09, 0x7C,
+	0xBE, 0x0F, 0x40, 0xD3, 0x24, 0x82, 0x0B, 0xFF,
+	0xE6, 0xB7, 0x10, 0xD6, 0x36, 0xD0, 0xE6, 0xC0,
+	0xBE, 0x65, 0x4C, 0x83, 0xF1, 0xDA, 0xDE, 0xCE,
+	0xE8, 0x5A, 0x80, 0x88, 0xFE, 0x9B, 0x79, 0x54,
+	0xA3, 0xA4, 0x5A, 0x76, 0xD0, 0xE2, 0xCE, 0x92,
+	0x53, 0x7D, 0x9C, 0xDA, 0xA1, 0xED, 0x9F, 0x56,
+	0x05, 0x0A, 0xA4, 0x81, 0xC7, 0x82, 0x5B, 0xB8,
+	0xC7, 0xA8, 0x95, 0x21, 0x99, 0x0B, 0x0F, 0xD2,
+	0x66, 0x68, 0xC3, 0x07, 0x53, 0x93, 0x8C, 0x68,
+	0x5A, 0xF5, 0x6F, 0x5E, 0x07, 0x68, 0x70, 0xF2,
+	0x6A, 0x78, 0xA8, 0xDB, 0x24, 0x6F, 0xD9, 0x74,
+	0x38, 0x29, 0xBF, 0x50, 0xCC, 0xC8, 0x34, 0x82,
+	0x69, 0x1A, 0xF4, 0x1A, 0x95, 0xE1, 0x31, 0x39,
+	0x00, 0xBE, 0xF6, 0x25, 0x19, 0x5A, 0xF6, 0xA7,
+	0xB7, 0xC7, 0xAC, 0xFA, 0x52, 0x6D, 0xC7, 0xEA,
+	0xFF, 0xC4, 0xB0, 0x84, 0x41, 0x2D, 0x6B, 0x22,
+	0x57, 0x1F, 0x8A, 0x97, 0x71, 0xEF, 0x54, 0xA7,
+	0xA9, 0xB5, 0xA0, 0x7A, 0xFF, 0x52, 0xBC, 0x78,
+	0x56, 0x8C, 0x9C, 0x2A, 0xFB, 0x31, 0xA6, 0xC8,
+	0x87, 0xC8, 0x82, 0x72, 0x00, 0x68, 0x27, 0xDE,
+	0x54, 0x67, 0x4C, 0x36, 0x4E, 0xE2, 0x77, 0x7F,
+	0xAD, 0x15, 0x1B, 0xC9, 0x07, 0xC4, 0x84, 0x50,
+	0x84, 0x45, 0xB5, 0x1D, 0xCE, 0x3F, 0x7C, 0xDF,
+	0x73, 0x88, 0x3E, 0x6D, 0xAC, 0x18, 0x67, 0x2C,
+	0x1D, 0x31, 0xD5, 0x41, 0x6E, 0xFC, 0x3D, 0xFE,
+	0x5C, 0x6D, 0x3B, 0xCB, 0x67, 0x05, 0x44, 0x8B,
+	0x02, 0xB7, 0xF5, 0x05, 0xFD, 0x1D, 0x7E, 0x13,
+	0x90, 0xCE, 0xED, 0xD2, 0xAB, 0x08, 0xFF, 0xC3,
+	0x91, 0x2C, 0x79, 0x06, 0xDB, 0x54, 0xAB, 0xFF,
+	0xF6, 0x9D, 0xBB, 0xDC, 0x3C, 0xCD, 0x03, 0xE7,
+	0xD8, 0x1A, 0x4E, 0x7F, 0xCB, 0x1B, 0xA3, 0xCA,
+	0xDC, 0x14, 0xC5, 0xFE, 0x46, 0x38, 0x07, 0x82,
+	0x5E, 0x9A, 0x77, 0x9C, 0xB4, 0x44, 0x26, 0xBA,
+	0xC2, 0x27, 0xD0, 0xF4, 0x75, 0x67, 0x4A, 0x15,
+	0x2A, 0x55, 0x2A, 0x61, 0x87, 0x55, 0xA0, 0xFB,
+	0xE0, 0x93, 0xA5, 0xD7, 0xDF, 0x0D, 0x97, 0xD3,
+	0x93, 0x39, 0x0B, 0x5A, 0xC6, 0x86, 0x17, 0x7D,
+	0x6A, 0xA4, 0x07, 0x9C, 0xB9, 0x0F, 0x42, 0x15,
+	0x46, 0x71, 0x45, 0x70, 0x75, 0x4B, 0xD5, 0x80,
+	0x64, 0x62, 0x92, 0x50, 0xBC, 0x10, 0xA1, 0x68,
+	0x60, 0x5A, 0x6F, 0x31, 0x26, 0xCA, 0xB1, 0x48,
+	0xFB, 0xC6, 0xF4, 0x3A, 0xCA, 0x52, 0x20, 0x2F,
+	0x7D, 0xEC, 0x19, 0xF8, 0xAA, 0x27, 0xC2, 0x23,
+	0xD0, 0x8F, 0x60, 0xFC, 0x7F, 0xA0, 0xCB, 0xDC,
+	0xA2, 0xC6, 0xC4, 0xC5, 0x13, 0x00, 0xF3, 0x43,
+	0x48, 0x6D, 0xFD, 0x7D, 0xB9, 0xA8, 0x14, 0xB0,
+	0x0C, 0x72, 0x82, 0x2F, 0x99, 0x64, 0x41, 0x2B,
+	0xB3, 0x34, 0x73, 0x73, 0xF4, 0x26, 0x1D, 0x06,
+	0xDF, 0x6E, 0xF4, 0x20, 0x1E, 0x31, 0xE3, 0x8A,
+	0x01, 0x6C, 0xDB, 0x3C, 0xE3, 0xC6, 0xC4, 0xC5,
+	0xB8, 0x20, 0x51, 0xF1, 0xD6, 0xB0, 0x30, 0xB7,
+	0x2D, 0xDA, 0x95, 0x01, 0x0D, 0xED, 0xEE, 0x6F,
+	0x69, 0xFD, 0xCF, 0x9D, 0xDD, 0x05, 0xD6, 0xC0,
+	0xFE, 0x11, 0x67, 0xAF, 0x53, 0x94, 0x60, 0xFC,
+	0x56, 0xBA, 0x0C, 0x5F, 0xA7, 0x7E, 0xDA, 0x65
+};
+static const struct
+blockcipher_test_data aes_test_data_xts_wrapped_key_48_pt_4096_du_4096 = {
+	.crypto_algo = RTE_CRYPTO_CIPHER_AES_XTS,
+	.cipher_key = {
+		.data = {
+			0x94, 0xD8, 0xAE, 0xD7, 0x74, 0xBF, 0x51, 0xF1,
+			0x2D, 0x19, 0x0B, 0x9A, 0xB3, 0xDC, 0x1E, 0xA9,
+			0x5C, 0xC5, 0x47, 0xC1, 0xBB, 0x94, 0xDB, 0xFA,
+			0x2B, 0x1E, 0x54, 0xA5, 0x76, 0xAD, 0x07, 0x4B,
+			0x46, 0xE0, 0xAA, 0xF9, 0xF7, 0x5C, 0x64, 0x2D,
+			0x8F, 0x04, 0x17, 0x72, 0xE5, 0x8D, 0xFE, 0x9C
+		},
+		.len = 48
+	},
+	.iv = {
+		.data = {
+			0x4F, 0xAE, 0xF7, 0x11, 0x7C, 0xDA, 0x59, 0xC6,
+			0x6E, 0x4B, 0x92, 0x01, 0x3E, 0x76, 0x8A, 0xD5
+		},
+		.len = 16
+	},
+	.plaintext = {
+		.data = plaintext_aes256xts_4096bytes,
+		.len = 4096
+	},
+	.ciphertext = {
+		.data = ciphertext_aes256xts_4096bytes,
+		.len = 4096
+	},
+	.xts_dataunit_len = 4096,
+	.wrapped_key = true
+};
+
+static const struct
+blockcipher_test_data aes_test_data_xts_wrapped_key_48_pt_4096_du_0 = {
+	.crypto_algo = RTE_CRYPTO_CIPHER_AES_XTS,
+	.cipher_key = {
+		.data = {
+			0x94, 0xD8, 0xAE, 0xD7, 0x74, 0xBF, 0x51, 0xF1,
+			0x2D, 0x19, 0x0B, 0x9A, 0xB3, 0xDC, 0x1E, 0xA9,
+			0x5C, 0xC5, 0x47, 0xC1, 0xBB, 0x94, 0xDB, 0xFA,
+			0x2B, 0x1E, 0x54, 0xA5, 0x76, 0xAD, 0x07, 0x4B,
+			0x46, 0xE0, 0xAA, 0xF9, 0xF7, 0x5C, 0x64, 0x2D,
+			0x8F, 0x04, 0x17, 0x72, 0xE5, 0x8D, 0xFE, 0x9C
+		},
+		.len = 48
+	},
+	.iv = {
+		.data = {
+			0x4F, 0xAE, 0xF7, 0x11, 0x7C, 0xDA, 0x59, 0xC6,
+			0x6E, 0x4B, 0x92, 0x01, 0x3E, 0x76, 0x8A, 0xD5
+		},
+		.len = 16
+	},
+	.plaintext = {
+		.data = plaintext_aes256xts_4096bytes,
+		.len = 4096
+	},
+	.ciphertext = {
+		.data = ciphertext_aes256xts_4096bytes,
+		.len = 4096
+	},
+	.xts_dataunit_len = 0,
+	.wrapped_key = true
+};
+
+static const struct
+blockcipher_test_data aes_test_data_xts_wrapped_key_48_pt_512_du_0 = {
+	.crypto_algo = RTE_CRYPTO_CIPHER_AES_XTS,
+	.cipher_key = {
+		.data = {
+			0x94, 0xD8, 0xAE, 0xD7, 0x74, 0xBF, 0x51, 0xF1,
+			0x2D, 0x19, 0x0B, 0x9A, 0xB3, 0xDC, 0x1E, 0xA9,
+			0x5C, 0xC5, 0x47, 0xC1, 0xBB, 0x94, 0xDB, 0xFA,
+			0x2B, 0x1E, 0x54, 0xA5, 0x76, 0xAD, 0x07, 0x4B,
+			0x46, 0xE0, 0xAA, 0xF9, 0xF7, 0x5C, 0x64, 0x2D,
+			0x8F, 0x04, 0x17, 0x72, 0xE5, 0x8D, 0xFE, 0x9C
+		},
+		.len = 48
+	},
+	.iv = {
+		.data = {
+			0x4F, 0xAE, 0xF7, 0x11, 0x7C, 0xDA, 0x59, 0xC6,
+			0x6E, 0x4B, 0x92, 0x01, 0x3E, 0x76, 0x8A, 0xD5
+		},
+		.len = 16
+	},
+	.plaintext = {
+		.data = plaintext_aes256xts_512bytes,
+		.len = 512
+	},
+	.ciphertext = {
+		.data = ciphertext_aes256xts_512bytes,
+		.len = 512
+	},
+	.xts_dataunit_len = 0,
+	.wrapped_key = true
+};
+
+
 /* AES-DOCSIS-BPI test vectors */
 
 /* Multiple of AES block size */
@@ -3347,6 +4639,54 @@ static const struct blockcipher_test_case aes_cipheronly_test_cases[] = {
 		.test_data = &aes_test_data_xts_key_64_pt_48,
 		.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
 	},
+	{
+		.test_descr = "AES-256-XTS Encryption (512-byte plaintext"
+			      " Dataunit 512)",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_512_du_512,
+		.op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT,
+	},
+	{
+		.test_descr = "AES-256-XTS Decryption (512-byte plaintext"
+			      " Dataunit 512)",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_512_du_512,
+		.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
+	},
+	{
+		.test_descr = "AES-256-XTS Encryption (512-byte plaintext"
+			      " Dataunit 0)",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_512_du_0,
+		.op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT,
+	},
+	{
+		.test_descr = "AES-256-XTS Decryption (512-byte plaintext"
+			      " Dataunit 0)",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_512_du_0,
+		.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
+	},
+	{
+		.test_descr = "AES-256-XTS Encryption (4096-byte plaintext"
+			      " Dataunit 4096)",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_4096_du_4096,
+		.op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT,
+	},
+	{
+		.test_descr = "AES-256-XTS Decryption (4096-byte plaintext"
+			      " Dataunit 4096)",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_4096_du_4096,
+		.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
+	},
+	{
+		.test_descr = "AES-256-XTS Encryption (4096-byte plaintext"
+			      " Dataunit 0)",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_4096_du_0,
+		.op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT,
+	},
+	{
+		.test_descr = "AES-256-XTS Decryption (4096-byte plaintext"
+			      " Dataunit 0)",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_4096_du_0,
+		.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
+	},
 	{
 		.test_descr = "cipher-only - NULL algo - x8 - encryption",
 		.test_data = &null_test_data_chain_x8_multiple,
diff --git a/app/test/test_cryptodev_blockcipher.c b/app/test/test_cryptodev_blockcipher.c
index d342c7b859..53fd4718af 100644
--- a/app/test/test_cryptodev_blockcipher.c
+++ b/app/test/test_cryptodev_blockcipher.c
@@ -36,7 +36,8 @@ verify_algo_support(const struct blockcipher_test_case *t,
 		if (capability == NULL)
 			return -1;
 
-		if (cap_idx.algo.cipher != RTE_CRYPTO_CIPHER_NULL)
+		if (cap_idx.algo.cipher != RTE_CRYPTO_CIPHER_NULL &&
+				!(t->test_data->wrapped_key))
 			ret = rte_cryptodev_sym_capability_check_cipher(capability,
 							tdata->cipher_key.len,
 							tdata->iv.len);
@@ -145,6 +146,12 @@ test_blockcipher_one_case(const struct blockcipher_test_case *t,
 
 		nb_segs = 3;
 	}
+	if (!!(feat_flags & RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY) ^
+		tdata->wrapped_key) {
+		snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN,
+			"SKIPPED");
+		return TEST_SKIPPED;
+	}
 
 	if (global_api_test_type == CRYPTODEV_RAW_API_TEST &&
 		!(feat_flags & RTE_CRYPTODEV_FF_SYM_RAW_DP)) {
@@ -450,6 +457,7 @@ test_blockcipher_one_case(const struct blockcipher_test_case *t,
 		cipher_xform->cipher.key.data = cipher_key;
 		cipher_xform->cipher.key.length = tdata->cipher_key.len;
 		cipher_xform->cipher.iv.offset = IV_OFFSET;
+		cipher_xform->cipher.dataunit_len = tdata->xts_dataunit_len;
 
 		if (tdata->crypto_algo == RTE_CRYPTO_CIPHER_NULL)
 			cipher_xform->cipher.iv.length = 0;
diff --git a/app/test/test_cryptodev_blockcipher.h b/app/test/test_cryptodev_blockcipher.h
index a06241b06d..dcaa08ae22 100644
--- a/app/test/test_cryptodev_blockcipher.h
+++ b/app/test/test_cryptodev_blockcipher.h
@@ -97,6 +97,8 @@ struct blockcipher_test_data {
 
 	unsigned int cipher_offset;
 	unsigned int auth_offset;
+	uint16_t xts_dataunit_len;
+	bool wrapped_key;
 };
 
 struct unit_test_suite *
-- 
2.25.1

[dpdk-dev] [PATCH v9 14/15] test/crypto: add AES-XTS multi segment OOP tests

[Suanming Mou]

From: Shiri Kuzin <shirik@nvidia.com>

The AES-XTS algorithm can supports wrapped key and data-unit.
The encryption/decryption can be done out of place and using multi
segments.

Add multi segment and out of place tests to the recently added AES-XTS
vectors, which support using data-unit and a wrapped key.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 app/test/test_cryptodev_aes_test_vectors.h | 64 ++++++++++++++++++++++
 1 file changed, 64 insertions(+)

diff --git a/app/test/test_cryptodev_aes_test_vectors.h b/app/test/test_cryptodev_aes_test_vectors.h
index dd7ffde685..a797af1b00 100644
--- a/app/test/test_cryptodev_aes_test_vectors.h
+++ b/app/test/test_cryptodev_aes_test_vectors.h
@@ -4687,6 +4687,70 @@ static const struct blockcipher_test_case aes_cipheronly_test_cases[] = {
 		.test_data = &aes_test_data_xts_wrapped_key_48_pt_4096_du_0,
 		.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
 	},
+	{
+		.test_descr = "AES-256-XTS Encryption (512-byte plaintext"
+			      " Dataunit 512) Scater gather OOP",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_512_du_512,
+		.op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT,
+		.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP |
+				BLOCKCIPHER_TEST_FEATURE_SG,
+	},
+	{
+		.test_descr = "AES-256-XTS Decryption (512-byte plaintext"
+			      " Dataunit 512) Scater gather OOP",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_512_du_512,
+		.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
+		.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP |
+				BLOCKCIPHER_TEST_FEATURE_SG,
+	},
+	{
+		.test_descr = "AES-256-XTS Encryption (512-byte plaintext"
+			      " Dataunit 0) Scater gather OOP",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_512_du_0,
+		.op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT,
+		.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP |
+				BLOCKCIPHER_TEST_FEATURE_SG,
+	},
+	{
+		.test_descr = "AES-256-XTS Decryption (512-byte plaintext"
+			      " Dataunit 0) Scater gather OOP",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_512_du_0,
+		.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
+		.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP |
+				BLOCKCIPHER_TEST_FEATURE_SG,
+	},
+	{
+		.test_descr = "AES-256-XTS Encryption (4096-byte plaintext"
+			      " Dataunit 4096) Scater gather OOP",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_4096_du_4096,
+		.op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT,
+		.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP |
+				BLOCKCIPHER_TEST_FEATURE_SG,
+	},
+	{
+		.test_descr = "AES-256-XTS Decryption (4096-byte plaintext"
+			      " Dataunit 4096) Scater gather OOP",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_4096_du_4096,
+		.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
+		.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP |
+				BLOCKCIPHER_TEST_FEATURE_SG,
+	},
+	{
+		.test_descr = "AES-256-XTS Encryption (4096-byte plaintext"
+			      " Dataunit 0) Scater gather OOP",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_4096_du_0,
+		.op_mask = BLOCKCIPHER_TEST_OP_ENCRYPT,
+		.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP |
+				BLOCKCIPHER_TEST_FEATURE_SG,
+	},
+	{
+		.test_descr = "AES-256-XTS Decryption (4096-byte plaintext"
+			      " Dataunit 0) Scater gather OOP",
+		.test_data = &aes_test_data_xts_wrapped_key_48_pt_4096_du_0,
+		.op_mask = BLOCKCIPHER_TEST_OP_DECRYPT,
+		.feature_mask = BLOCKCIPHER_TEST_FEATURE_OOP |
+				BLOCKCIPHER_TEST_FEATURE_SG,
+	},
 	{
 		.test_descr = "cipher-only - NULL algo - x8 - encryption",
 		.test_data = &null_test_data_chain_x8_multiple,
-- 
2.25.1

[dpdk-dev] [PATCH v9 15/15] test/crypto: add mlx5 crypto driver

[Suanming Mou]

From: Shiri Kuzin <shirik@nvidia.com>

In order to test the new mlx5 crypto PMD, the driver is added to the
crypto test application.

Signed-off-by: Shiri Kuzin <shirik@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 app/test/test_cryptodev.c      | 7 +++++++
 app/test/test_cryptodev.h      | 2 ++
 doc/guides/cryptodevs/mlx5.rst | 3 +++
 3 files changed, 12 insertions(+)

diff --git a/app/test/test_cryptodev.c b/app/test/test_cryptodev.c
index 31201d93e1..c752c1e8f2 100644
--- a/app/test/test_cryptodev.c
+++ b/app/test/test_cryptodev.c
@@ -14569,6 +14569,12 @@ test_cryptodev_cpu_aesni_gcm(void)
 	return rc;
 }
 
+static int
+test_cryptodev_mlx5(void)
+{
+	return run_cryptodev_testsuite(RTE_STR(CRYPTODEV_NAME_MLX5_PMD));
+}
+
 static int
 test_cryptodev_null(void)
 {
@@ -14797,6 +14803,7 @@ REGISTER_TEST_COMMAND(cryptodev_openssl_autotest, test_cryptodev_openssl);
 REGISTER_TEST_COMMAND(cryptodev_aesni_gcm_autotest, test_cryptodev_aesni_gcm);
 REGISTER_TEST_COMMAND(cryptodev_cpu_aesni_gcm_autotest,
 	test_cryptodev_cpu_aesni_gcm);
+REGISTER_TEST_COMMAND(cryptodev_mlx5_autotest, test_cryptodev_mlx5);
 REGISTER_TEST_COMMAND(cryptodev_null_autotest, test_cryptodev_null);
 REGISTER_TEST_COMMAND(cryptodev_sw_snow3g_autotest, test_cryptodev_sw_snow3g);
 REGISTER_TEST_COMMAND(cryptodev_sw_kasumi_autotest, test_cryptodev_sw_kasumi);
diff --git a/app/test/test_cryptodev.h b/app/test/test_cryptodev.h
index 5bf1e8818b..31e278261a 100644
--- a/app/test/test_cryptodev.h
+++ b/app/test/test_cryptodev.h
@@ -73,6 +73,8 @@
 #define CRYPTODEV_NAME_BCMFS_PMD	crypto_bcmfs
 #define CRYPTODEV_NAME_CN9K_PMD		crypto_cn9k
 #define CRYPTODEV_NAME_CN10K_PMD	crypto_cn10k
+#define CRYPTODEV_NAME_MLX5_PMD		crypto_mlx5
+
 
 enum cryptodev_api_test_type {
 	CRYPTODEV_API_TEST = 0,
diff --git a/doc/guides/cryptodevs/mlx5.rst b/doc/guides/cryptodevs/mlx5.rst
index b20f593549..39d89b2b95 100644
--- a/doc/guides/cryptodevs/mlx5.rst
+++ b/doc/guides/cryptodevs/mlx5.rst
@@ -92,6 +92,9 @@ The mlxreg dedicated tool should be used as follows:
   The "wrapped_crypto_operational" value will be "0x00000001" if the mode was
   successfully changed to operational mode.
 
+  The mlx5 crypto PMD can be verfied by running the test application:
+  ./build/app/test/dpdk-test -c 1 -n 1 -w <dev>,class=crypto,wcs_file=<file_path>
+  RTE>>cryptodev_mlx5_autotest
 
 Driver options
 --------------
-- 
2.25.1

Re: [dpdk-dev] [EXT] [PATCH v9 00/15] drivers: introduce mlx5 crypto PMD

[Akhil Goyal]

> v2:
> - Add data-path part.
> 
> v3:
> - Rebase.
> 
> v4:
> - Rebase + Address the following Akhil comments:
> - Set HW feature flag in the capability patch.
> - Fix mp object release in session clear.
> - Some spelling and word missing in doc.
> - Squash data-unit adjustment to the session operations commit.
> - Wording: device argument -> devarg.
> 
> v5:
> - Add mlx5 crypto tests into test library.
> - Update documentation according to Akhil comments.
> - Fix memory region management.
> - Fix multi segment case in data-path code.
> - Split documentation to the correct commits according to Akhil
>   comments.
> - Rebase to new version.
> - Change license to Nvidia license.
> 
> v6:
> - Rebase to new version.
> - Address David's comment and update log accordingly.
> - Update testing app commits as suggested by Akhil.
> 
> v7:
> - Rebase to new version.
> - Add scatter gather and OOP test cases.
> 
> v8:
> - Add a test case that was removed by error.
> 
> v9:
> - Reorganize the basic operation commit.
> - Move the seesion operation commit after QP patch.
> - Fix coding style.
> - Move the rel_notes to the last PMD change.
> - Move the mlx5 test code to last.
> - Rename the driver name to crypto_mlx5.
> - Split the MBUF_DATAPAYLOAD_SIZE update out of the series.
> 
Series Acked-by: Akhil Goyal <gakhil@marvell.com>

Applied to dpdk-next-crypto
Updated MAINTAINERS file for ownership of  mlx5.rst and mlx5.ini files
Fixed some typos while applying patches.

Re: [dpdk-dev] [EXT] [PATCH v9 00/15] drivers: introduce mlx5 crypto PMD

[Akhil Goyal]

> > v2:
> > - Add data-path part.
> >
> > v3:
> > - Rebase.
> >
> > v4:
> > - Rebase + Address the following Akhil comments:
> > - Set HW feature flag in the capability patch.
> > - Fix mp object release in session clear.
> > - Some spelling and word missing in doc.
> > - Squash data-unit adjustment to the session operations commit.
> > - Wording: device argument -> devarg.
> >
> > v5:
> > - Add mlx5 crypto tests into test library.
> > - Update documentation according to Akhil comments.
> > - Fix memory region management.
> > - Fix multi segment case in data-path code.
> > - Split documentation to the correct commits according to Akhil
> >   comments.
> > - Rebase to new version.
> > - Change license to Nvidia license.
> >
> > v6:
> > - Rebase to new version.
> > - Address David's comment and update log accordingly.
> > - Update testing app commits as suggested by Akhil.
> >
> > v7:
> > - Rebase to new version.
> > - Add scatter gather and OOP test cases.
> >
> > v8:
> > - Add a test case that was removed by error.
> >
> > v9:
> > - Reorganize the basic operation commit.
> > - Move the seesion operation commit after QP patch.
> > - Fix coding style.
> > - Move the rel_notes to the last PMD change.
> > - Move the mlx5 test code to last.
> > - Rename the driver name to crypto_mlx5.
> > - Split the MBUF_DATAPAYLOAD_SIZE update out of the series.
> >
> Series Acked-by: Akhil Goyal <gakhil@marvell.com>
> 
> Applied to dpdk-next-crypto
> Updated MAINTAINERS file for ownership of  mlx5.rst and mlx5.ini files
> Fixed some typos while applying patches.

I am getting error while cross-compile arm on x86. Can you check. Please send a fix if it is a genuine issue, I will squash in the original commit.

meson build.octeontx2 --cross-file config/arm/arm64_octeontx2_linux_gcc -Dexamples=all --buildtype=debug
ninja -C build.octeontx2
[906/2513] Linking target drivers/librte_common_mlx5.so.21.3.
FAILED: drivers/librte_common_mlx5.so.21.3
aarch64-linux-gnu-gcc  -o drivers/librte_common_mlx5.so.21.3 'drivers/a715181@@rte_common_mlx5@sha/meson-generated_.._rte_common_mlx5.pmd.c.o' 'drivers/a715181@@tmp_rte_common_mlx5@sta/common_mlx5_mlx5_devx_cmds.c.o' 'drivers/a715181@@tmp_rte_common_mlx5@sta/common_mlx5_mlx5_common.c.o' 'drivers/a715181@@tmp_rte_common_mlx5@sta/common_mlx5_mlx5_common_mp.c.o' 'drivers/a715181@@tmp_rte_common_mlx5@sta/common_mlx5_mlx5_common_mr.c.o' 'drivers/a715181@@tmp_rte_common_mlx5@sta/common_mlx5_mlx5_malloc.c.o' 'drivers/a715181@@tmp_rte_common_mlx5@sta/common_mlx5_mlx5_common_pci.c.o' 'drivers/a715181@@tmp_rte_common_mlx5@sta/common_mlx5_mlx5_common_devx.c.o' 'drivers/a715181@@tmp_rte_common_mlx5@sta/common_mlx5_mlx5_common_utils.c.o' 'drivers/a715181@@tmp_rte_common_mlx5@sta/common_mlx5_linux_mlx5_nl.c.o' 'drivers/a715181@@tmp_rte_common_mlx5@sta/common_mlx5_linux_mlx5_common_os.c.o' 'drivers/a715181@@tmp_rte_common_mlx5@sta/common_mlx5_linux_mlx5_common_verbs.c.o' 'drivers/a715181@@tmp_rte_common_mlx5@sta/common_mlx5_linux_mlx5_glue.c.o' -Wl,--as-needed -Wl,--no-undefined -shared -fPIC -Wl,--start-group -Wl,-soname,librte_common_mlx5.so.21 -Wl,--no-as-needed -pthread -lm -ldl lib/librte_hash.so.21.3 lib/librte_eal.so.21.3 lib/librte_kvargs.so.21.3 lib/librte_telemetry.so.21.3 lib/librte_net.so.21.3 lib/librte_mbuf.so.21.3 lib/librte_mempool.so.21.3 lib/librte_ring.so.21.3 lib/librte_rcu.so.21.3 lib/librte_pci.so.21.3 drivers/librte_bus_pci.so.21.3 -Wl,--version-script=/home/cavium/up/dpdk-next-crypto/drivers/common/mlx5/version.map /home/cavium/up/mlx/rdma-core/build/lib/libmlx5.so -Wl,-rpath,/home/cavium/up/mlx/rdma-core/build/lib /home/cavium/up/mlx/rdma-core/build/lib/libibverbs.so -Wl,--end-group -Wl,-rpath,/home/cavium/up/mlx/rdma-core/build/lib '-Wl,-rpath,$ORIGIN/../lib:$ORIGIN/' -Wl,-rpath-link,/home/cavium/up/dpdk-next-crypto/build.octeontx2/lib -Wl,-rpath-link,/home/cavium/up/dpdk-next-crypto/build.octeontx2/drivers
/home/cavium/tools/marvell-tools-1013.0/bin/../lib/gcc/aarch64-marvell-linux-gnu/10.1.0/../../../../aarch64-marvell-linux-gnu/bin/ld: /home/cavium/up/mlx/rdma-core/build/lib/libmlx5.so: error adding symbols: file in wrong format
collect2: error: ld returned 1 exit status
[915/2513] Generating rte_common_sfc_efx.sym_chk with a meson_exe.py custom command.

Re: [dpdk-dev] [EXT] [PATCH v9 00/15] drivers: introduce mlx5 crypto PMD

[Suanming Mou]

Hi Akhil,

> -----Original Message-----
> From: Akhil Goyal <gakhil@marvell.com>
> Sent: Wednesday, July 21, 2021 5:02 AM
> To: Suanming Mou <suanmingm@nvidia.com>; Shiri Kuzin <shirik@nvidia.com>
> Cc: Matan Azrad <matan@nvidia.com>; david.marchand@redhat.com;
> dev@dpdk.org
> Subject: RE: [EXT] [PATCH v9 00/15] drivers: introduce mlx5 crypto PMD
> 
> > > v2:
> > > - Add data-path part.
> > >
> > > v3:
> > > - Rebase.
> > >
> > > v4:
> > > - Rebase + Address the following Akhil comments:
> > > - Set HW feature flag in the capability patch.
> > > - Fix mp object release in session clear.
> > > - Some spelling and word missing in doc.
> > > - Squash data-unit adjustment to the session operations commit.
> > > - Wording: device argument -> devarg.
> > >
> > > v5:
> > > - Add mlx5 crypto tests into test library.
> > > - Update documentation according to Akhil comments.
> > > - Fix memory region management.
> > > - Fix multi segment case in data-path code.
> > > - Split documentation to the correct commits according to Akhil
> > >   comments.
> > > - Rebase to new version.
> > > - Change license to Nvidia license.
> > >
> > > v6:
> > > - Rebase to new version.
> > > - Address David's comment and update log accordingly.
> > > - Update testing app commits as suggested by Akhil.
> > >
> > > v7:
> > > - Rebase to new version.
> > > - Add scatter gather and OOP test cases.
> > >
> > > v8:
> > > - Add a test case that was removed by error.
> > >
> > > v9:
> > > - Reorganize the basic operation commit.
> > > - Move the seesion operation commit after QP patch.
> > > - Fix coding style.
> > > - Move the rel_notes to the last PMD change.
> > > - Move the mlx5 test code to last.
> > > - Rename the driver name to crypto_mlx5.
> > > - Split the MBUF_DATAPAYLOAD_SIZE update out of the series.
> > >
> > Series Acked-by: Akhil Goyal <gakhil@marvell.com>
> >
> > Applied to dpdk-next-crypto
> > Updated MAINTAINERS file for ownership of  mlx5.rst and mlx5.ini files
> > Fixed some typos while applying patches.
> 
> I am getting error while cross-compile arm on x86. Can you check. Please send a
> fix if it is a genuine issue, I will squash in the original commit.
> 
> meson build.octeontx2 --cross-file config/arm/arm64_octeontx2_linux_gcc -
> Dexamples=all --buildtype=debug ninja -C build.octeontx2 [906/2513] Linking
> target drivers/librte_common_mlx5.so.21.3.
> FAILED: drivers/librte_common_mlx5.so.21.3
> aarch64-linux-gnu-gcc  -o drivers/librte_common_mlx5.so.21.3
> 'drivers/a715181@@rte_common_mlx5@sha/meson-
> generated_.._rte_common_mlx5.pmd.c.o'
> 'drivers/a715181@@tmp_rte_common_mlx5@sta/common_mlx5_mlx5_devx_
> cmds.c.o'
> 'drivers/a715181@@tmp_rte_common_mlx5@sta/common_mlx5_mlx5_comm
> on.c.o'
> 'drivers/a715181@@tmp_rte_common_mlx5@sta/common_mlx5_mlx5_comm
> on_mp.c.o'
> 'drivers/a715181@@tmp_rte_common_mlx5@sta/common_mlx5_mlx5_comm
> on_mr.c.o'
> 'drivers/a715181@@tmp_rte_common_mlx5@sta/common_mlx5_mlx5_malloc
> .c.o'
> 'drivers/a715181@@tmp_rte_common_mlx5@sta/common_mlx5_mlx5_comm
> on_pci.c.o'
> 'drivers/a715181@@tmp_rte_common_mlx5@sta/common_mlx5_mlx5_comm
> on_devx.c.o'
> 'drivers/a715181@@tmp_rte_common_mlx5@sta/common_mlx5_mlx5_comm
> on_utils.c.o'
> 'drivers/a715181@@tmp_rte_common_mlx5@sta/common_mlx5_linux_mlx5_
> nl.c.o'
> 'drivers/a715181@@tmp_rte_common_mlx5@sta/common_mlx5_linux_mlx5_
> common_os.c.o'
> 'drivers/a715181@@tmp_rte_common_mlx5@sta/common_mlx5_linux_mlx5_
> common_verbs.c.o'
> 'drivers/a715181@@tmp_rte_common_mlx5@sta/common_mlx5_linux_mlx5_
> glue.c.o' -Wl,--as-needed -Wl,--no-undefined -shared -fPIC -Wl,--start-group -
> Wl,-soname,librte_common_mlx5.so.21 -Wl,--no-as-needed -pthread -lm -ldl
> lib/librte_hash.so.21.3 lib/librte_eal.so.21.3 lib/librte_kvargs.so.21.3
> lib/librte_telemetry.so.21.3 lib/librte_net.so.21.3 lib/librte_mbuf.so.21.3
> lib/librte_mempool.so.21.3 lib/librte_ring.so.21.3 lib/librte_rcu.so.21.3
> lib/librte_pci.so.21.3 drivers/librte_bus_pci.so.21.3 -Wl,--version-
> script=/home/cavium/up/dpdk-next-crypto/drivers/common/mlx5/version.map
> /home/cavium/up/mlx/rdma-core/build/lib/libmlx5.so -Wl,-
> rpath,/home/cavium/up/mlx/rdma-core/build/lib /home/cavium/up/mlx/rdma-
> core/build/lib/libibverbs.so -Wl,--end-group -Wl,-
> rpath,/home/cavium/up/mlx/rdma-core/build/lib '-Wl,-
> rpath,$ORIGIN/../lib:$ORIGIN/' -Wl,-rpath-link,/home/cavium/up/dpdk-next-
> crypto/build.octeontx2/lib -Wl,-rpath-link,/home/cavium/up/dpdk-next-
> crypto/build.octeontx2/drivers
> /home/cavium/tools/marvell-tools-1013.0/bin/../lib/gcc/aarch64-marvell-linux-
> gnu/10.1.0/../../../../aarch64-marvell-linux-gnu/bin/ld:
> /home/cavium/up/mlx/rdma-core/build/lib/libmlx5.so: error adding symbols:
> file in wrong format
> collect2: error: ld returned 1 exit status [915/2513] Generating
> rte_common_sfc_efx.sym_chk with a meson_exe.py custom command.

Not sure what's the real issue here, I'm trying to find a local setup to reproduce it.

Re: [dpdk-dev] [EXT] [PATCH v9 00/15] drivers: introduce mlx5 crypto PMD

[Suanming Mou]

Hi Akhil,

> -----Original Message-----
> From: Suanming Mou
> Sent: Wednesday, July 21, 2021 8:27 AM
> To: Akhil Goyal <gakhil@marvell.com>; Shiri Kuzin <shirik@nvidia.com>
> Cc: Matan Azrad <matan@nvidia.com>; david.marchand@redhat.com;
> dev@dpdk.org
> Subject: RE: [EXT] [PATCH v9 00/15] drivers: introduce mlx5 crypto PMD
> 
> Hi Akhil,
> 
> > -----Original Message-----
> > From: Akhil Goyal <gakhil@marvell.com>
> > Sent: Wednesday, July 21, 2021 5:02 AM
> > To: Suanming Mou <suanmingm@nvidia.com>; Shiri Kuzin
> > <shirik@nvidia.com>
> > Cc: Matan Azrad <matan@nvidia.com>; david.marchand@redhat.com;
> > dev@dpdk.org
> > Subject: RE: [EXT] [PATCH v9 00/15] drivers: introduce mlx5 crypto PMD
> >
> > > > v2:
> > > > - Add data-path part.
> > > >
> > > > v3:
> > > > - Rebase.
> > > >
> > > > v4:
> > > > - Rebase + Address the following Akhil comments:
> > > > - Set HW feature flag in the capability patch.
> > > > - Fix mp object release in session clear.
> > > > - Some spelling and word missing in doc.
> > > > - Squash data-unit adjustment to the session operations commit.
> > > > - Wording: device argument -> devarg.
> > > >
> > > > v5:
> > > > - Add mlx5 crypto tests into test library.
> > > > - Update documentation according to Akhil comments.
> > > > - Fix memory region management.
> > > > - Fix multi segment case in data-path code.
> > > > - Split documentation to the correct commits according to Akhil
> > > >   comments.
> > > > - Rebase to new version.
> > > > - Change license to Nvidia license.
> > > >
> > > > v6:
> > > > - Rebase to new version.
> > > > - Address David's comment and update log accordingly.
> > > > - Update testing app commits as suggested by Akhil.
> > > >
> > > > v7:
> > > > - Rebase to new version.
> > > > - Add scatter gather and OOP test cases.
> > > >
> > > > v8:
> > > > - Add a test case that was removed by error.
> > > >
> > > > v9:
> > > > - Reorganize the basic operation commit.
> > > > - Move the seesion operation commit after QP patch.
> > > > - Fix coding style.
> > > > - Move the rel_notes to the last PMD change.
> > > > - Move the mlx5 test code to last.
> > > > - Rename the driver name to crypto_mlx5.
> > > > - Split the MBUF_DATAPAYLOAD_SIZE update out of the series.
> > > >
> > > Series Acked-by: Akhil Goyal <gakhil@marvell.com>
> > >
> > > Applied to dpdk-next-crypto
> > > Updated MAINTAINERS file for ownership of  mlx5.rst and mlx5.ini
> > > files Fixed some typos while applying patches.
> >
> > I am getting error while cross-compile arm on x86. Can you check.
> > Please send a fix if it is a genuine issue, I will squash in the original commit.
> >
> > meson build.octeontx2 --cross-file
> > config/arm/arm64_octeontx2_linux_gcc - Dexamples=all --buildtype=debug
> > ninja -C build.octeontx2 [906/2513] Linking target
> drivers/librte_common_mlx5.so.21.3.
> > FAILED: drivers/librte_common_mlx5.so.21.3
> > aarch64-linux-gnu-gcc  -o drivers/librte_common_mlx5.so.21.3
> > 'drivers/a715181@@rte_common_mlx5@sha/meson-
> > generated_.._rte_common_mlx5.pmd.c.o'
> >
> 'drivers/a715181@@tmp_rte_common_mlx5@sta/common_mlx5_mlx5_devx_
> > cmds.c.o'
> >
> 'drivers/a715181@@tmp_rte_common_mlx5@sta/common_mlx5_mlx5_comm
> > on.c.o'
> >
> 'drivers/a715181@@tmp_rte_common_mlx5@sta/common_mlx5_mlx5_comm
> > on_mp.c.o'
> >
> 'drivers/a715181@@tmp_rte_common_mlx5@sta/common_mlx5_mlx5_comm
> > on_mr.c.o'
> >
> 'drivers/a715181@@tmp_rte_common_mlx5@sta/common_mlx5_mlx5_malloc
> > .c.o'
> >
> 'drivers/a715181@@tmp_rte_common_mlx5@sta/common_mlx5_mlx5_comm
> > on_pci.c.o'
> >
> 'drivers/a715181@@tmp_rte_common_mlx5@sta/common_mlx5_mlx5_comm
> > on_devx.c.o'
> >
> 'drivers/a715181@@tmp_rte_common_mlx5@sta/common_mlx5_mlx5_comm
> > on_utils.c.o'
> >
> 'drivers/a715181@@tmp_rte_common_mlx5@sta/common_mlx5_linux_mlx5_
> > nl.c.o'
> >
> 'drivers/a715181@@tmp_rte_common_mlx5@sta/common_mlx5_linux_mlx5_
> > common_os.c.o'
> >
> 'drivers/a715181@@tmp_rte_common_mlx5@sta/common_mlx5_linux_mlx5_
> > common_verbs.c.o'
> >
> 'drivers/a715181@@tmp_rte_common_mlx5@sta/common_mlx5_linux_mlx5_
> > glue.c.o' -Wl,--as-needed -Wl,--no-undefined -shared -fPIC
> > -Wl,--start-group -
> > Wl,-soname,librte_common_mlx5.so.21 -Wl,--no-as-needed -pthread -lm
> > -ldl
> > lib/librte_hash.so.21.3 lib/librte_eal.so.21.3
> > lib/librte_kvargs.so.21.3
> > lib/librte_telemetry.so.21.3 lib/librte_net.so.21.3
> > lib/librte_mbuf.so.21.3
> > lib/librte_mempool.so.21.3 lib/librte_ring.so.21.3
> > lib/librte_rcu.so.21.3
> > lib/librte_pci.so.21.3 drivers/librte_bus_pci.so.21.3 -Wl,--version-
> > script=/home/cavium/up/dpdk-next-crypto/drivers/common/mlx5/version.ma
> > p /home/cavium/up/mlx/rdma-core/build/lib/libmlx5.so -Wl,-
> > rpath,/home/cavium/up/mlx/rdma-core/build/lib
> > /home/cavium/up/mlx/rdma- core/build/lib/libibverbs.so -Wl,--end-group
> > -Wl,- rpath,/home/cavium/up/mlx/rdma-core/build/lib '-Wl,-
> > rpath,$ORIGIN/../lib:$ORIGIN/'
> > -Wl,-rpath-link,/home/cavium/up/dpdk-next-
> > crypto/build.octeontx2/lib -Wl,-rpath-link,/home/cavium/up/dpdk-next-
> > crypto/build.octeontx2/drivers
> > /home/cavium/tools/marvell-tools-1013.0/bin/../lib/gcc/aarch64-marvell
> > -linux-
> > gnu/10.1.0/../../../../aarch64-marvell-linux-gnu/bin/ld:
> > /home/cavium/up/mlx/rdma-core/build/lib/libmlx5.so: error adding symbols:
> > file in wrong format
> > collect2: error: ld returned 1 exit status [915/2513] Generating
> > rte_common_sfc_efx.sym_chk with a meson_exe.py custom command.
> 
> Not sure what's the real issue here, I'm trying to find a local setup to reproduce
> it.

I can't be reproduced the issue on my local setup while cross-compiling.
And double checked the log here, seems the rdma-core libmlx5.so is not correct:
/home/cavium/up/mlx/rdma-core/build/lib/libmlx5.so: error adding symbols: file in wrong format
I assume you will see the same issue even when you don't have the crypto patch set?
Do you get the error only when you applied the crypto patch set?

Thanks,
SuanmingMou