[dpdk-dev] [PATCH 02/10] drivers: introduce mlx5 compress PMD

[Matan Azrad <matan@nvidia.com>]

Add a new compress PMD for Mellanox devices.

The MLX5 compress driver library provides support for Mellanox
BlueField 2 families of 25/50/100/200 Gb/s adapters.

Using the BlueField 2 device, the compress class operations can be run in
parallel to the net, vdpa, and regex class operations.

This driver is depending on rdma-core like the other mlx5 PMDs, also it
is going to use mlx5 DevX to create HW objects directly by the FW.

Add the probing functions, PCI bus connectivity, HW capabilities checks
and some basic objects preparations.

Signed-off-by: Matan Azrad <matan@nvidia.com>
---
 MAINTAINERS                                 |   4 +
 drivers/common/mlx5/mlx5_common.h           |   1 +
 drivers/common/mlx5/mlx5_common_pci.c       |   7 +
 drivers/common/mlx5/mlx5_common_pci.h       |  36 ++--
 drivers/compress/meson.build                |   2 +-
 drivers/compress/mlx5/meson.build           |  26 +++
 drivers/compress/mlx5/mlx5_compress.c       | 297 ++++++++++++++++++++++++++++
 drivers/compress/mlx5/mlx5_compress_utils.h |  20 ++
 drivers/compress/mlx5/version.map           |   3 +
 9 files changed, 377 insertions(+), 19 deletions(-)
 create mode 100644 drivers/compress/mlx5/meson.build
 create mode 100644 drivers/compress/mlx5/mlx5_compress.c
 create mode 100644 drivers/compress/mlx5/mlx5_compress_utils.h
 create mode 100644 drivers/compress/mlx5/version.map

diff --git a/MAINTAINERS b/MAINTAINERS
index eafe9f8..f7527f0 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -1132,6 +1132,10 @@ F: drivers/compress/zlib/
 F: doc/guides/compressdevs/zlib.rst
 F: doc/guides/compressdevs/features/zlib.ini
 
+Mellanox mlx5
+M: Matan Azrad <matan@nvidia.com>
+F: drivers/compress/mlx5/
+
 
 RegEx Drivers
 -------------
diff --git a/drivers/common/mlx5/mlx5_common.h b/drivers/common/mlx5/mlx5_common.h
index e35188d..3855582 100644
--- a/drivers/common/mlx5/mlx5_common.h
+++ b/drivers/common/mlx5/mlx5_common.h
@@ -217,6 +217,7 @@ enum mlx5_class {
 	MLX5_CLASS_NET = RTE_BIT64(0),
 	MLX5_CLASS_VDPA = RTE_BIT64(1),
 	MLX5_CLASS_REGEX = RTE_BIT64(2),
+	MLX5_CLASS_COMPRESS = RTE_BIT64(3),
 };
 
 #define MLX5_DBR_SIZE RTE_CACHE_LINE_SIZE
diff --git a/drivers/common/mlx5/mlx5_common_pci.c b/drivers/common/mlx5/mlx5_common_pci.c
index 5208972..2b65768 100644
--- a/drivers/common/mlx5/mlx5_common_pci.c
+++ b/drivers/common/mlx5/mlx5_common_pci.c
@@ -28,14 +28,21 @@ static TAILQ_HEAD(mlx5_pci_devices_head, mlx5_pci_device) devices_list =
 	{ .name = "vdpa", .driver_class = MLX5_CLASS_VDPA },
 	{ .name = "net", .driver_class = MLX5_CLASS_NET },
 	{ .name = "regex", .driver_class = MLX5_CLASS_REGEX },
+	{ .name = "compress", .driver_class = MLX5_CLASS_COMPRESS },
 };
 
 static const unsigned int mlx5_class_combinations[] = {
 	MLX5_CLASS_NET,
 	MLX5_CLASS_VDPA,
 	MLX5_CLASS_REGEX,
+	MLX5_CLASS_COMPRESS,
 	MLX5_CLASS_NET | MLX5_CLASS_REGEX,
 	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX,
+	MLX5_CLASS_NET | MLX5_CLASS_COMPRESS,
+	MLX5_CLASS_VDPA | MLX5_CLASS_COMPRESS,
+	MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
+	MLX5_CLASS_NET | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
+	MLX5_CLASS_VDPA | MLX5_CLASS_REGEX | MLX5_CLASS_COMPRESS,
 	/* New class combination should be added here. */
 };
 
diff --git a/drivers/common/mlx5/mlx5_common_pci.h b/drivers/common/mlx5/mlx5_common_pci.h
index 41b73e1..de89bb9 100644
--- a/drivers/common/mlx5/mlx5_common_pci.h
+++ b/drivers/common/mlx5/mlx5_common_pci.h
@@ -9,26 +9,26 @@
  * @file
  *
  * RTE Mellanox PCI Driver Interface
- * Mellanox ConnectX PCI device supports multiple class (net/vdpa/regex)
- * devices. This layer enables creating such multiple class of devices on a
- * single PCI device by allowing to bind multiple class specific device
+ * Mellanox ConnectX PCI device supports multiple class: net,vdpa,regex and
+ * compress devices. This layer enables creating such multiple class of devices
+ * on a single PCI device by allowing to bind multiple class specific device
  * driver to attach to mlx5_pci driver.
  *
- * -----------    ------------    -------------
- * |   mlx5  |    |   mlx5   |    |   mlx5    |
- * | net pmd |    | vdpa pmd |    | regex pmd |
- * -----------    ------------    -------------
- *      \              |                 /
- *       \             |                /
- *        \       --------------       /
- *         \______|   mlx5     |_____ /
- *                | pci common |
- *                --------------
- *                     |
- *                 -----------
- *                 |   mlx5  |
- *                 | pci dev |
- *                 -----------
+ * -----------    ------------    -------------    ----------------
+ * |   mlx5  |    |   mlx5   |    |   mlx5    |    |     mlx5     |
+ * | net pmd |    | vdpa pmd |    | regex pmd |    | compress pmd |
+ * -----------    ------------    -------------    ----------------
+ *      \              \                    /              /
+ *       \              \                  /              /
+ *        \              \_--------------_/              /
+ *         \_______________|   mlx5     |_______________/
+ *                         | pci common |
+ *                         --------------
+ *                               |
+ *                           -----------
+ *                           |   mlx5  |
+ *                           | pci dev |
+ *                           -----------
  *
  * - mlx5 pci driver binds to mlx5 PCI devices defined by PCI
  *   ID table of all related mlx5 PCI devices.
diff --git a/drivers/compress/meson.build b/drivers/compress/meson.build
index 33f5e33..d8f3ddb 100644
--- a/drivers/compress/meson.build
+++ b/drivers/compress/meson.build
@@ -5,7 +5,7 @@ if is_windows
 	subdir_done()
 endif
 
-drivers = ['isal', 'octeontx', 'zlib']
+drivers = ['isal', 'octeontx', 'zlib', 'mlx5']
 
 std_deps = ['compressdev'] # compressdev pulls in all other needed deps
 config_flag_fmt = 'RTE_LIBRTE_PMD_@0@'
diff --git a/drivers/compress/mlx5/meson.build b/drivers/compress/mlx5/meson.build
new file mode 100644
index 0000000..2a6dc3f
--- /dev/null
+++ b/drivers/compress/mlx5/meson.build
@@ -0,0 +1,26 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2021 Mellanox Technologies, Ltd
+
+if not is_linux
+	build = false
+	reason = 'only supported on Linux'
+	subdir_done()
+endif
+
+fmt_name = 'mlx5_compress'
+deps += ['common_mlx5', 'eal', 'compressdev']
+sources = files(
+	'mlx5_compress.c',
+)
+cflags_options = [
+	'-std=c11',
+	'-Wno-strict-prototypes',
+	'-D_BSD_SOURCE',
+	'-D_DEFAULT_SOURCE',
+	'-D_XOPEN_SOURCE=600'
+]
+foreach option:cflags_options
+	if cc.has_argument(option)
+		cflags += option
+	endif
+endforeach
diff --git a/drivers/compress/mlx5/mlx5_compress.c b/drivers/compress/mlx5/mlx5_compress.c
new file mode 100644
index 0000000..639dd61
--- /dev/null
+++ b/drivers/compress/mlx5/mlx5_compress.c
@@ -0,0 +1,297 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#include <rte_malloc.h>
+#include <rte_log.h>
+#include <rte_errno.h>
+#include <rte_pci.h>
+#include <rte_comp.h>
+#include <rte_compressdev.h>
+#include <rte_compressdev_pmd.h>
+
+#include <mlx5_glue.h>
+#include <mlx5_common.h>
+#include <mlx5_common_pci.h>
+#include <mlx5_devx_cmds.h>
+#include <mlx5_common_os.h>
+#include <mlx5_prm.h>
+
+#include "mlx5_compress_utils.h"
+
+#define MLX5_COMPRESS_DRIVER_NAME mlx5_compress
+#define MLX5_COMPRESS_LOG_NAME    pmd.compress.mlx5
+
+struct mlx5_compress_priv {
+	TAILQ_ENTRY(mlx5_compress_priv) next;
+	struct ibv_context *ctx; /* Device context. */
+	struct rte_pci_device *pci_dev;
+	struct rte_compressdev *cdev;
+	void *uar;
+	uint32_t pdn; /* Protection Domain number. */
+	uint8_t min_block_size;
+	/* Minimum huffman block size supported by the device. */
+	struct ibv_pd *pd;
+};
+
+TAILQ_HEAD(mlx5_compress_privs, mlx5_compress_priv) mlx5_compress_priv_list =
+				TAILQ_HEAD_INITIALIZER(mlx5_compress_priv_list);
+static pthread_mutex_t priv_list_lock = PTHREAD_MUTEX_INITIALIZER;
+
+int mlx5_compress_logtype;
+
+static struct rte_compressdev_ops mlx5_compress_ops = {
+	.dev_configure		= NULL,
+	.dev_start		= NULL,
+	.dev_stop		= NULL,
+	.dev_close		= NULL,
+	.dev_infos_get		= NULL,
+	.stats_get		= NULL,
+	.stats_reset		= NULL,
+	.queue_pair_setup	= NULL,
+	.queue_pair_release	= NULL,
+	.private_xform_create	= NULL,
+	.private_xform_free	= NULL,
+	.stream_create		= NULL,
+	.stream_free		= NULL,
+};
+
+static struct ibv_device *
+mlx5_compress_get_ib_device_match(struct rte_pci_addr *addr)
+{
+	int n;
+	struct ibv_device **ibv_list = mlx5_glue->get_device_list(&n);
+	struct ibv_device *ibv_match = NULL;
+
+	if (ibv_list == NULL) {
+		rte_errno = ENOSYS;
+		return NULL;
+	}
+	while (n-- > 0) {
+		struct rte_pci_addr paddr;
+
+		DRV_LOG(DEBUG, "Checking device \"%s\"..", ibv_list[n]->name);
+		if (mlx5_dev_to_pci_addr(ibv_list[n]->ibdev_path, &paddr) != 0)
+			continue;
+		if (rte_pci_addr_cmp(addr, &paddr) != 0)
+			continue;
+		ibv_match = ibv_list[n];
+		break;
+	}
+	if (ibv_match == NULL)
+		rte_errno = ENOENT;
+	mlx5_glue->free_device_list(ibv_list);
+	return ibv_match;
+}
+
+static void
+mlx5_compress_hw_global_release(struct mlx5_compress_priv *priv)
+{
+	if (priv->pd != NULL) {
+		claim_zero(mlx5_glue->dealloc_pd(priv->pd));
+		priv->pd = NULL;
+	}
+	if (priv->uar != NULL) {
+		mlx5_glue->devx_free_uar(priv->uar);
+		priv->uar = NULL;
+	}
+}
+
+static int
+mlx5_compress_pd_create(struct mlx5_compress_priv *priv)
+{
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+	struct mlx5dv_obj obj;
+	struct mlx5dv_pd pd_info;
+	int ret;
+
+	priv->pd = mlx5_glue->alloc_pd(priv->ctx);
+	if (priv->pd == NULL) {
+		DRV_LOG(ERR, "Failed to allocate PD.");
+		return errno ? -errno : -ENOMEM;
+	}
+	obj.pd.in = priv->pd;
+	obj.pd.out = &pd_info;
+	ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_PD);
+	if (ret != 0) {
+		DRV_LOG(ERR, "Fail to get PD object info.");
+		mlx5_glue->dealloc_pd(priv->pd);
+		priv->pd = NULL;
+		return -errno;
+	}
+	priv->pdn = pd_info.pdn;
+	return 0;
+#else
+	(void)priv;
+	DRV_LOG(ERR, "Cannot get pdn - no DV support.");
+	return -ENOTSUP;
+#endif /* HAVE_IBV_FLOW_DV_SUPPORT */
+}
+
+static int
+mlx5_compress_hw_global_prepare(struct mlx5_compress_priv *priv)
+{
+	if (mlx5_compress_pd_create(priv) != 0)
+		return -1;
+	priv->uar = mlx5_devx_alloc_uar(priv->ctx, -1);
+	if (priv->uar == NULL || mlx5_os_get_devx_uar_reg_addr(priv->uar) ==
+	    NULL) {
+		rte_errno = errno;
+		claim_zero(mlx5_glue->dealloc_pd(priv->pd));
+		DRV_LOG(ERR, "Failed to allocate UAR.");
+		return -1;
+	}
+	return 0;
+}
+
+/**
+ * DPDK callback to register a PCI device.
+ *
+ * This function spawns compress device out of a given PCI device.
+ *
+ * @param[in] pci_drv
+ *   PCI driver structure (mlx5_compress_driver).
+ * @param[in] pci_dev
+ *   PCI device information.
+ *
+ * @return
+ *   0 on success, 1 to skip this driver, a negative errno value otherwise
+ *   and rte_errno is set.
+ */
+static int
+mlx5_compress_pci_probe(struct rte_pci_driver *pci_drv,
+			struct rte_pci_device *pci_dev)
+{
+	struct ibv_device *ibv;
+	struct rte_compressdev *cdev;
+	struct ibv_context *ctx;
+	struct mlx5_compress_priv *priv;
+	struct mlx5_hca_attr att = { 0 };
+	struct rte_compressdev_pmd_init_params init_params = {
+		.name = "",
+		.socket_id = pci_dev->device.numa_node,
+	};
+
+	RTE_SET_USED(pci_drv);
+	ibv = mlx5_compress_get_ib_device_match(&pci_dev->addr);
+	if (ibv == NULL) {
+		DRV_LOG(ERR, "No matching IB device for PCI slot "
+			PCI_PRI_FMT ".", pci_dev->addr.domain,
+			pci_dev->addr.bus, pci_dev->addr.devid,
+			pci_dev->addr.function);
+		return -rte_errno;
+	}
+	DRV_LOG(INFO, "PCI information matches for device \"%s\".", ibv->name);
+	ctx = mlx5_glue->dv_open_device(ibv);
+	if (ctx == NULL) {
+		DRV_LOG(ERR, "Failed to open IB device \"%s\".", ibv->name);
+		rte_errno = ENODEV;
+		return -rte_errno;
+	}
+	if (mlx5_devx_cmd_query_hca_attr(ctx, &att) != 0 ||
+	    att.mmo_compress_en == 0 || att.mmo_decompress_en == 0 ||
+	    att.mmo_dma_en == 0) {
+		DRV_LOG(ERR, "Not enough capabilities to support compress "
+			"operations, maybe old FW/OFED version?");
+		claim_zero(mlx5_glue->close_device(ctx));
+		rte_errno = ENOTSUP;
+		return -ENOTSUP;
+	}
+	cdev = rte_compressdev_pmd_create(ibv->name, &pci_dev->device,
+					  sizeof(*priv), &init_params);
+	if (cdev == NULL) {
+		DRV_LOG(ERR, "Failed to create device \"%s\".", ibv->name);
+		claim_zero(mlx5_glue->close_device(ctx));
+		return -ENODEV;
+	}
+	DRV_LOG(INFO,
+		"Compress device %s was created successfully.", ibv->name);
+	cdev->dev_ops = &mlx5_compress_ops;
+	cdev->dequeue_burst = NULL;
+	cdev->enqueue_burst = NULL;
+	cdev->feature_flags = RTE_COMPDEV_FF_HW_ACCELERATED;
+	priv = cdev->data->dev_private;
+	priv->ctx = ctx;
+	priv->pci_dev = pci_dev;
+	priv->cdev = cdev;
+	priv->min_block_size = att.compress_min_block_size;
+	if (mlx5_compress_hw_global_prepare(priv) != 0) {
+		rte_compressdev_pmd_destroy(priv->cdev);
+		claim_zero(mlx5_glue->close_device(priv->ctx));
+		return -1;
+	}
+	pthread_mutex_lock(&priv_list_lock);
+	TAILQ_INSERT_TAIL(&mlx5_compress_priv_list, priv, next);
+	pthread_mutex_unlock(&priv_list_lock);
+	return 0;
+}
+
+/**
+ * DPDK callback to remove a PCI device.
+ *
+ * This function removes all compress devices belong to a given PCI device.
+ *
+ * @param[in] pci_dev
+ *   Pointer to the PCI device.
+ *
+ * @return
+ *   0 on success, the function cannot fail.
+ */
+static int
+mlx5_compress_pci_remove(struct rte_pci_device *pdev)
+{
+	struct mlx5_compress_priv *priv = NULL;
+	int found = 0;
+
+	pthread_mutex_lock(&priv_list_lock);
+	TAILQ_FOREACH(priv, &mlx5_compress_priv_list, next) {
+		if (rte_pci_addr_cmp(&priv->pci_dev->addr, &pdev->addr) != 0) {
+			found = 1;
+			break;
+		}
+	}
+	if (found != 0)
+		TAILQ_REMOVE(&mlx5_compress_priv_list, priv, next);
+	pthread_mutex_unlock(&priv_list_lock);
+	if (found != 0) {
+		mlx5_compress_hw_global_release(priv);
+		rte_compressdev_pmd_destroy(priv->cdev);
+		claim_zero(mlx5_glue->close_device(priv->ctx));
+	}
+	return 0;
+}
+
+static const struct rte_pci_id mlx5_compress_pci_id_map[] = {
+	{
+		RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
+				PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF)
+	},
+	{
+		.vendor_id = 0
+	}
+};
+
+static struct mlx5_pci_driver mlx5_compress_driver = {
+	.driver_class = MLX5_CLASS_COMPRESS,
+	.pci_driver = {
+		.driver = {
+			.name = RTE_STR(MLX5_COMPRESS_DRIVER_NAME),
+		},
+		.id_table = mlx5_compress_pci_id_map,
+		.probe = mlx5_compress_pci_probe,
+		.remove = mlx5_compress_pci_remove,
+		.drv_flags = 0,
+	},
+};
+
+RTE_INIT(rte_mlx5_compress_init)
+{
+	mlx5_common_init();
+	if (mlx5_glue != NULL)
+		mlx5_pci_driver_register(&mlx5_compress_driver);
+}
+
+RTE_LOG_REGISTER(mlx5_compress_logtype, MLX5_COMPRESS_LOG_NAME, NOTICE)
+RTE_PMD_EXPORT_NAME(MLX5_COMPRESS_DRIVER_NAME, __COUNTER__);
+RTE_PMD_REGISTER_PCI_TABLE(MLX5_COMPRESS_DRIVER_NAME, mlx5_compress_pci_id_map);
+RTE_PMD_REGISTER_KMOD_DEP(MLX5_COMPRESS_DRIVER_NAME, "* ib_uverbs & mlx5_core & mlx5_ib");
diff --git a/drivers/compress/mlx5/mlx5_compress_utils.h b/drivers/compress/mlx5/mlx5_compress_utils.h
new file mode 100644
index 0000000..f93244f
--- /dev/null
+++ b/drivers/compress/mlx5/mlx5_compress_utils.h
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021 Mellanox Technologies, Ltd
+ */
+
+#ifndef RTE_PMD_MLX5_COMPRESS_UTILS_H_
+#define RTE_PMD_MLX5_COMPRESS_UTILS_H_
+
+#include <mlx5_common.h>
+
+
+extern int mlx5_compress_logtype;
+
+#define MLX5_COMPRESS_LOG_PREFIX "mlx5_compress"
+/* Generic printf()-like logging macro with automatic line feed. */
+#define DRV_LOG(level, ...) \
+	PMD_DRV_LOG_(level, mlx5_compress_logtype, MLX5_COMPRESS_LOG_PREFIX, \
+		__VA_ARGS__ PMD_DRV_LOG_STRIP PMD_DRV_LOG_OPAREN, \
+		PMD_DRV_LOG_CPAREN)
+
+#endif /* RTE_PMD_MLX5_COMPRESS_UTILS_H_ */
diff --git a/drivers/compress/mlx5/version.map b/drivers/compress/mlx5/version.map
new file mode 100644
index 0000000..4a76d1d
--- /dev/null
+++ b/drivers/compress/mlx5/version.map
@@ -0,0 +1,3 @@
+DPDK_21 {
+	local: *;
+};
-- 
1.8.3.1