[dpdk-dev] [PATCH v5 19/32] common/mlx5/windows: add glue functions APIs

[Tal Shnaiderman <talshn@nvidia.com>]

Windows glue functions are added to file mlx5/windows/mlx5_glue.c.
The following APIs are supported:
get_device_list, free_device_list, open_device, close_device,
query_device, query_hca_iseg, devx_obj_create, devx_obj_destroy,
devx_obj_query, devx_obj_modify, devx_general_cmd, devx_umem_reg,
devx_umem_dereg, devx_alloc_uar, devx_free_uar, devx_fs_rule_add,
devx_fs_rule_del, devx_query_eqn
New added files:
mlx5_win_defs.h - this file imports missing definitions from Linux
rdma-core library and Linux OS.
mlx5_win_ext.h - this file contains structs that enable a unified
Linux/Windows API. Each struct has an equivalent (but different) Linux
struct. By calling with 'void *' pointers - the Linux/Windows API is
identical.

Signed-off-by: Tal Shnaiderman <talshn@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/common/mlx5/rte_common_mlx5_exports.def |   1 +
 drivers/common/mlx5/windows/mlx5_glue.c         | 304 ++++++++++++++++++++++++
 drivers/common/mlx5/windows/mlx5_glue.h         |  58 +++++
 drivers/common/mlx5/windows/mlx5_win_defs.h     |  25 ++
 drivers/common/mlx5/windows/mlx5_win_ext.h      |  34 +++
 5 files changed, 422 insertions(+)
 create mode 100644 drivers/common/mlx5/windows/mlx5_glue.c
 create mode 100644 drivers/common/mlx5/windows/mlx5_glue.h
 create mode 100644 drivers/common/mlx5/windows/mlx5_win_defs.h
 create mode 100644 drivers/common/mlx5/windows/mlx5_win_ext.h

diff --git a/drivers/common/mlx5/rte_common_mlx5_exports.def b/drivers/common/mlx5/rte_common_mlx5_exports.def
index 15aafc3809..2f6aab133b 100644
--- a/drivers/common/mlx5/rte_common_mlx5_exports.def
+++ b/drivers/common/mlx5/rte_common_mlx5_exports.def
@@ -36,6 +36,7 @@ EXPORTS
 	mlx5_devx_cmd_create_flow_hit_aso_obj
 
 	mlx5_get_dbr
+	mlx5_glue
 
 	mlx5_malloc_mem_select
 	mlx5_mr_btree_init
diff --git a/drivers/common/mlx5/windows/mlx5_glue.c b/drivers/common/mlx5/windows/mlx5_glue.c
new file mode 100644
index 0000000000..7f8a00aaa7
--- /dev/null
+++ b/drivers/common/mlx5/windows/mlx5_glue.c
@@ -0,0 +1,304 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2020 Mellanox Technologies, Ltd
+ */
+
+#include <errno.h>
+#include <stdalign.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include <rte_malloc.h>
+
+#include "mlx5_glue.h"
+#include "mlx5_common_utils.h"
+#include "mlx5_win_ext.h"
+
+/*
+ * The returned value of this API is an array of pointers to mlx5
+ * devices under Windows. The interesting parameters of a device:
+ * Device PCI parameters: domain, bus, device id, function.
+ * Device name.
+ */
+static void *
+mlx5_glue_devx_get_device_list(int *num_devices)
+{
+	struct devx_device_bdf *devx_bdf_devs = NULL;
+	size_t n_devx_devx = 0;
+	int32_t ret = devx_get_device_list(&n_devx_devx, &devx_bdf_devs);
+
+	if (ret) {
+		errno = ret;
+		*num_devices = 0;
+		return NULL;
+	}
+	*num_devices = (int)n_devx_devx;
+	return devx_bdf_devs;
+}
+
+static void
+mlx5_glue_devx_free_device_list(void *list)
+{
+	if (!list) {
+		errno = EINVAL;
+		return;
+	}
+	devx_free_device_list(list);
+}
+
+static int
+mlx5_glue_devx_close_device(void *ctx)
+{
+	mlx5_context_st *mlx5_ctx;
+	int rc;
+
+	if (!ctx)
+		return -EINVAL;
+	mlx5_ctx = (mlx5_context_st *)ctx;
+	rc = devx_close_device(mlx5_ctx->devx_ctx);
+	free(mlx5_ctx);
+	return rc;
+}
+
+static void *
+mlx5_glue_devx_open_device(void *device)
+{
+	struct mlx5_context *mlx5_ctx;
+
+	if (!device) {
+		errno = EINVAL;
+		return NULL;
+	}
+	mlx5_ctx = malloc((sizeof(struct mlx5_context)));
+	if (!mlx5_ctx) {
+		errno = ENOMEM;
+		return NULL;
+	}
+	memset(mlx5_ctx, 0, sizeof(*mlx5_ctx));
+	mlx5_ctx->devx_ctx = devx_open_device(device);
+	if (DEVX_IS_ERR(mlx5_ctx->devx_ctx)) {
+		errno = -DEVX_PTR_ERR(mlx5_ctx->devx_ctx);
+		free(mlx5_ctx);
+		return NULL;
+	}
+	return mlx5_ctx;
+}
+
+static int
+mlx5_glue_devx_query_device(void *device_bdf, void *dev_inf)
+{
+	struct devx_device_bdf *dev_bdf;
+	struct devx_device *mlx5_dev;
+
+	if (!device_bdf)
+		return -EINVAL;
+	dev_bdf = (struct devx_device_bdf *)device_bdf;
+	mlx5_dev = (struct devx_device *)dev_inf;
+	int err = devx_query_device(dev_bdf, mlx5_dev);
+	if (err)
+		return -E_FAIL;
+	return 0;
+}
+
+static void *
+mlx5_glue_devx_query_hca_iseg_mapping(void *ctx, uint32_t *cb_iseg)
+{
+	struct mlx5_context *mlx5_ctx;
+	void *pv_iseg;
+	int err;
+
+	if (!ctx) {
+		errno = EINVAL;
+		return NULL;
+	}
+	mlx5_ctx = (struct mlx5_context *)ctx;
+	err = devx_query_hca_iseg_mapping(mlx5_ctx->devx_ctx,
+						cb_iseg, &pv_iseg);
+	if (err) {
+		errno = err;
+		return NULL;
+	}
+	return pv_iseg;
+}
+
+static void *
+mlx5_glue_devx_obj_create(void *ctx,
+			      void *in, size_t inlen,
+			      void *out, size_t outlen)
+{
+	mlx5_devx_obj_st *devx_obj;
+
+	if (!ctx) {
+		errno = EINVAL;
+		return NULL;
+	}
+	devx_obj = malloc((sizeof(*devx_obj)));
+	if (!devx_obj) {
+		errno = ENOMEM;
+		return NULL;
+	}
+	memset(devx_obj, 0, sizeof(*devx_obj));
+	devx_obj->devx_ctx = GET_DEVX_CTX(ctx);
+	devx_obj->obj = devx_obj_create(devx_obj->devx_ctx,
+					in, inlen, out, outlen);
+	if (DEVX_IS_ERR(devx_obj->obj)) {
+		errno = -DEVX_PTR_ERR(devx_obj->obj);
+		free(devx_obj);
+		return NULL;
+	}
+	return devx_obj;
+}
+
+static int
+mlx5_glue_devx_obj_destroy(void *obj)
+{
+	mlx5_devx_obj_st *devx_obj;
+
+	if (!obj)
+		return -EINVAL;
+	devx_obj = obj;
+	int rc = devx_obj_destroy(devx_obj->obj);
+	free(devx_obj);
+	return rc;
+}
+
+static int
+mlx5_glue_devx_general_cmd(void *ctx,
+			   void *in, size_t inlen,
+			   void *out, size_t outlen)
+{
+	if (!ctx)
+		return -EINVAL;
+	return devx_cmd(GET_DEVX_CTX(ctx), in, inlen, out, outlen);
+}
+
+static int
+mlx5_glue_devx_obj_query(void *obj,
+			    void *in, size_t inlen,
+			    void *out, size_t outlen)
+{
+	return devx_cmd(GET_OBJ_CTX(obj), in, inlen, out, outlen);
+}
+
+static int
+mlx5_glue_devx_obj_modify(void *obj,
+			    void *in, size_t inlen,
+			    void *out, size_t outlen)
+{
+	return devx_cmd(GET_OBJ_CTX(obj), in, inlen, out, outlen);
+}
+
+static int
+mlx5_glue_devx_umem_dereg(void *pumem)
+{
+	struct devx_obj_handle *umem;
+
+	if (!pumem)
+		return -EINVAL;
+	umem = pumem;
+	return devx_umem_unreg(umem);
+}
+
+static void *
+mlx5_glue_devx_umem_reg(void *ctx, void *addr, size_t size,
+				  uint32_t access, uint32_t *id)
+{
+	struct devx_obj_handle *umem_hdl;
+	int w_access = DEVX_UMEM_ACCESS_READ;
+
+	if (!ctx) {
+		errno = EINVAL;
+		return NULL;
+	}
+	if (access)
+		w_access |= DEVX_UMEM_ACCESS_WRITE;
+
+	umem_hdl = devx_umem_reg(GET_DEVX_CTX(ctx), addr,
+					size, w_access, id);
+	if (DEVX_IS_ERR(umem_hdl)) {
+		errno = -DEVX_PTR_ERR(umem_hdl);
+		return NULL;
+	}
+	return umem_hdl;
+}
+
+static void *
+mlx5_glue_devx_alloc_uar(void *ctx,
+		uint32_t flags)
+{
+	devx_uar_handle *uar;
+
+	if (!ctx) {
+		errno = EINVAL;
+		return NULL;
+	}
+	uar = devx_alloc_uar(GET_DEVX_CTX(ctx), flags);
+	if (DEVX_IS_ERR(uar)) {
+		errno = -DEVX_PTR_ERR(uar);
+		return NULL;
+	}
+	return uar;
+}
+
+static int
+mlx5_glue_devx_query_eqn(void *ctx,
+		uint32_t cpus, uint32_t *eqn)
+{
+	if (!ctx)
+		return -EINVAL;
+	return devx_query_eqn(GET_DEVX_CTX(ctx), cpus, eqn);
+}
+
+static void
+mlx5_glue_devx_free_uar(void *uar)
+{
+	devx_free_uar((devx_uar_handle *)uar);
+}
+
+static void*
+mlx5_glue_devx_fs_rule_add(void *ctx, void *in, uint32_t inlen)
+
+{
+	struct devx_obj_handle *rule_hdl = NULL;
+
+	if (!ctx) {
+		errno = EINVAL;
+		return NULL;
+	}
+	rule_hdl = devx_fs_rule_add(GET_DEVX_CTX(ctx), in, inlen);
+	if (DEVX_IS_ERR(rule_hdl)) {
+		errno = -DEVX_PTR_ERR(rule_hdl);
+		return NULL;
+	}
+	return rule_hdl;
+}
+
+static int
+mlx5_glue_devx_fs_rule_del(void *flow)
+{
+	return devx_fs_rule_del(flow);
+}
+
+alignas(RTE_CACHE_LINE_SIZE)
+const struct mlx5_glue *mlx5_glue = &(const struct mlx5_glue){
+	.version = MLX5_GLUE_VERSION,
+	.get_device_list = mlx5_glue_devx_get_device_list,
+	.free_device_list = mlx5_glue_devx_free_device_list,
+	.open_device = mlx5_glue_devx_open_device,
+	.close_device = mlx5_glue_devx_close_device,
+	.query_device = mlx5_glue_devx_query_device,
+	.query_hca_iseg = mlx5_glue_devx_query_hca_iseg_mapping,
+	.devx_obj_create = mlx5_glue_devx_obj_create,
+	.devx_obj_destroy = mlx5_glue_devx_obj_destroy,
+	.devx_obj_query = mlx5_glue_devx_obj_query,
+	.devx_obj_modify = mlx5_glue_devx_obj_modify,
+	.devx_general_cmd = mlx5_glue_devx_general_cmd,
+	.devx_umem_reg = mlx5_glue_devx_umem_reg,
+	.devx_umem_dereg = mlx5_glue_devx_umem_dereg,
+	.devx_alloc_uar = mlx5_glue_devx_alloc_uar,
+	.devx_free_uar = mlx5_glue_devx_free_uar,
+	.devx_fs_rule_add = mlx5_glue_devx_fs_rule_add,
+	.devx_fs_rule_del = mlx5_glue_devx_fs_rule_del,
+	.devx_query_eqn = mlx5_glue_devx_query_eqn,
+};
diff --git a/drivers/common/mlx5/windows/mlx5_glue.h b/drivers/common/mlx5/windows/mlx5_glue.h
new file mode 100644
index 0000000000..f2261ec7ac
--- /dev/null
+++ b/drivers/common/mlx5/windows/mlx5_glue.h
@@ -0,0 +1,58 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2020 Mellanox Technologies, Ltd
+ */
+
+#ifndef MLX5_GLUE_H_
+#define MLX5_GLUE_H_
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <rte_byteorder.h>
+#include <mlx5_win_defs.h>
+
+#ifndef MLX5_GLUE_VERSION
+#define MLX5_GLUE_VERSION ""
+#endif
+
+/* LIB_GLUE_VERSION must be updated every time this structure is modified. */
+struct mlx5_glue {
+	const char *version;
+	void *(*devx_obj_create)(void *ctx,
+				 void *in, size_t inlen,
+				 void *out, size_t outlen);
+	int (*devx_obj_destroy)(void *obj);
+	int (*devx_obj_query)(void *obj,
+			      void *in, size_t inlen,
+			      void *out, size_t outlen);
+	int (*devx_obj_modify)(void *obj,
+			       void *in, size_t inlen,
+			       void *out, size_t outlen);
+	int (*devx_general_cmd)(void *ctx,
+			       void *in, size_t inlen,
+			       void *out, size_t outlen);
+	int (*devx_umem_dereg)(void *umem);
+	void *(*devx_umem_reg)(void *ctx,
+			void *addr, size_t size,
+			uint32_t access, uint32_t *id);
+	void *(*devx_alloc_uar)(void *ctx,
+			uint32_t flags);
+	void (*devx_free_uar)(void *uar);
+	void *(*get_device_list)(int *num_devices);
+	void (*free_device_list)(void *list);
+	void *(*open_device)(void *device);
+	int (*close_device)(void *ctx);
+	int (*query_device)(void *device_bdf, void *dev_inf);
+	void* (*query_hca_iseg)(void *ctx, uint32_t *cb_iseg);
+	int (*devx_obj_query_async)(void *obj,
+				    const void *in, size_t inlen,
+				    size_t outlen, uint64_t wr_id,
+				    void *cmd_comp);
+	void *(*devx_fs_rule_add)(void *ctx, void *in, uint32_t inlen);
+	int (*devx_fs_rule_del)(void *flow);
+	int (*devx_query_eqn)(void *context, uint32_t cpus, uint32_t *eqn);
+};
+
+extern const struct mlx5_glue *mlx5_glue;
+
+#endif /* MLX5_GLUE_H_ */
diff --git a/drivers/common/mlx5/windows/mlx5_win_defs.h b/drivers/common/mlx5/windows/mlx5_win_defs.h
new file mode 100644
index 0000000000..72a3131f5e
--- /dev/null
+++ b/drivers/common/mlx5/windows/mlx5_win_defs.h
@@ -0,0 +1,25 @@
+/*
+ * Copyright (C) Mellanox Technologies, Ltd. 2001-2020.
+ *
+ */
+#ifndef __MLX5_WIN_DEFS_H__
+#define __MLX5_WIN_DEFS_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+enum {
+	MLX5_CQE_OWNER_MASK	= 1,
+	MLX5_CQE_REQ		= 0,
+	MLX5_CQE_RESP_WR_IMM	= 1,
+	MLX5_CQE_RESP_SEND	= 2,
+	MLX5_CQE_RESP_SEND_IMM	= 3,
+	MLX5_CQE_RESP_SEND_INV	= 4,
+	MLX5_CQE_RESIZE_CQ	= 5,
+	MLX5_CQE_NO_PACKET	= 6,
+	MLX5_CQE_REQ_ERR	= 13,
+	MLX5_CQE_RESP_ERR	= 14,
+	MLX5_CQE_INVALID	= 15,
+};
+#endif /* __MLX5_WIN_DEFS_H__ */
diff --git a/drivers/common/mlx5/windows/mlx5_win_ext.h b/drivers/common/mlx5/windows/mlx5_win_ext.h
new file mode 100644
index 0000000000..0e74910e9d
--- /dev/null
+++ b/drivers/common/mlx5/windows/mlx5_win_ext.h
@@ -0,0 +1,34 @@
+/*
+ * Copyright (C) Mellanox Technologies, Ltd. 2001-2020.
+ *
+ */
+#ifndef __MLX5_WIN_ETX_H__
+#define __MLX5_WIN_ETX_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "mlx5devx.h"
+
+typedef struct mlx5_context {
+	devx_device_ctx        *devx_ctx;
+	struct devx_device mlx5_dev;
+
+} mlx5_context_st;
+
+typedef struct {
+	devx_device_ctx *devx_ctx;
+	struct devx_obj_handle *obj;
+} mlx5_devx_obj_st;
+
+struct mlx5_devx_umem {
+	void                   *addr;
+	struct devx_obj_handle *umem_hdl;
+	uint32_t                umem_id;
+};
+
+#define GET_DEVX_CTX(ctx) (((mlx5_context_st *)ctx)->devx_ctx)
+#define GET_OBJ_CTX(obj)  (((mlx5_devx_obj_st *)obj)->devx_ctx)
+
+#endif /* __MLX5_WIN_ETX_H__ */
-- 
2.16.1.windows.4