From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id D46BDA04B5; Sun, 13 Dec 2020 11:28:44 +0100 (CET) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 8E64ECA63; Sun, 13 Dec 2020 11:22:20 +0100 (CET) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by dpdk.org (Postfix) with ESMTP id B6A2EC9AC for ; Sun, 13 Dec 2020 11:21:48 +0100 (CET) Received: from Internal Mail-Server by MTLPINE1 (envelope-from talshn@nvidia.com) with SMTP; 13 Dec 2020 12:21:43 +0200 Received: from nvidia.com (l-wincomp04-vm.mtl.labs.mlnx [10.237.1.5]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 0BDALe1j018562; Sun, 13 Dec 2020 12:21:42 +0200 From: Tal Shnaiderman To: dev@dpdk.org Cc: thomas@monjalon.net, matan@nvidia.com, rasland@nvidia.com, ophirmu@nvidia.com Date: Sun, 13 Dec 2020 12:20:43 +0200 Message-Id: <20201213102056.11380-20-talshn@nvidia.com> X-Mailer: git-send-email 2.16.1.windows.4 In-Reply-To: <20201213102056.11380-1-talshn@nvidia.com> References: <20201210150648.8784-2-talshn@nvidia.com> <20201213102056.11380-1-talshn@nvidia.com> Subject: [dpdk-dev] [PATCH v3 19/32] common/mlx5/windows: add glue functions APIs X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Windows glue functions are added to file mlx5/windows/mlx5_glue.c. The following APIs are supported: get_device_list, free_device_list, open_device, close_device, query_device, query_hca_iseg, devx_obj_create, devx_obj_destroy, devx_obj_query, devx_obj_modify, devx_general_cmd, devx_umem_reg, devx_umem_dereg, devx_alloc_uar, devx_free_uar, devx_fs_rule_add, devx_fs_rule_del, devx_query_eqn New added files: mlx5_win_defs.h - this file imports missing definitions from Linux rdma-core library and Linux OS. mlx5_win_ext.h - this file contains structs that enable a unified Linux/Windows API. Each struct has an equivalent (but different) Linux struct. By calling with 'void *' pointers - the Linux/Windows API is identical. Signed-off-by: Tal Shnaiderman Acked-by: Matan Azrad --- drivers/common/mlx5/rte_common_mlx5_exports.def | 1 + drivers/common/mlx5/windows/mlx5_glue.c | 304 ++++++++++++++++++++++++ drivers/common/mlx5/windows/mlx5_glue.h | 58 +++++ drivers/common/mlx5/windows/mlx5_win_defs.h | 25 ++ drivers/common/mlx5/windows/mlx5_win_ext.h | 34 +++ 5 files changed, 422 insertions(+) create mode 100644 drivers/common/mlx5/windows/mlx5_glue.c create mode 100644 drivers/common/mlx5/windows/mlx5_glue.h create mode 100644 drivers/common/mlx5/windows/mlx5_win_defs.h create mode 100644 drivers/common/mlx5/windows/mlx5_win_ext.h diff --git a/drivers/common/mlx5/rte_common_mlx5_exports.def b/drivers/common/mlx5/rte_common_mlx5_exports.def index 15aafc3809..2f6aab133b 100644 --- a/drivers/common/mlx5/rte_common_mlx5_exports.def +++ b/drivers/common/mlx5/rte_common_mlx5_exports.def @@ -36,6 +36,7 @@ EXPORTS mlx5_devx_cmd_create_flow_hit_aso_obj mlx5_get_dbr + mlx5_glue mlx5_malloc_mem_select mlx5_mr_btree_init diff --git a/drivers/common/mlx5/windows/mlx5_glue.c b/drivers/common/mlx5/windows/mlx5_glue.c new file mode 100644 index 0000000000..7f8a00aaa7 --- /dev/null +++ b/drivers/common/mlx5/windows/mlx5_glue.c @@ -0,0 +1,304 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2020 Mellanox Technologies, Ltd + */ + +#include +#include +#include +#include +#include +#include + +#include + +#include "mlx5_glue.h" +#include "mlx5_common_utils.h" +#include "mlx5_win_ext.h" + +/* + * The returned value of this API is an array of pointers to mlx5 + * devices under Windows. The interesting parameters of a device: + * Device PCI parameters: domain, bus, device id, function. + * Device name. + */ +static void * +mlx5_glue_devx_get_device_list(int *num_devices) +{ + struct devx_device_bdf *devx_bdf_devs = NULL; + size_t n_devx_devx = 0; + int32_t ret = devx_get_device_list(&n_devx_devx, &devx_bdf_devs); + + if (ret) { + errno = ret; + *num_devices = 0; + return NULL; + } + *num_devices = (int)n_devx_devx; + return devx_bdf_devs; +} + +static void +mlx5_glue_devx_free_device_list(void *list) +{ + if (!list) { + errno = EINVAL; + return; + } + devx_free_device_list(list); +} + +static int +mlx5_glue_devx_close_device(void *ctx) +{ + mlx5_context_st *mlx5_ctx; + int rc; + + if (!ctx) + return -EINVAL; + mlx5_ctx = (mlx5_context_st *)ctx; + rc = devx_close_device(mlx5_ctx->devx_ctx); + free(mlx5_ctx); + return rc; +} + +static void * +mlx5_glue_devx_open_device(void *device) +{ + struct mlx5_context *mlx5_ctx; + + if (!device) { + errno = EINVAL; + return NULL; + } + mlx5_ctx = malloc((sizeof(struct mlx5_context))); + if (!mlx5_ctx) { + errno = ENOMEM; + return NULL; + } + memset(mlx5_ctx, 0, sizeof(*mlx5_ctx)); + mlx5_ctx->devx_ctx = devx_open_device(device); + if (DEVX_IS_ERR(mlx5_ctx->devx_ctx)) { + errno = -DEVX_PTR_ERR(mlx5_ctx->devx_ctx); + free(mlx5_ctx); + return NULL; + } + return mlx5_ctx; +} + +static int +mlx5_glue_devx_query_device(void *device_bdf, void *dev_inf) +{ + struct devx_device_bdf *dev_bdf; + struct devx_device *mlx5_dev; + + if (!device_bdf) + return -EINVAL; + dev_bdf = (struct devx_device_bdf *)device_bdf; + mlx5_dev = (struct devx_device *)dev_inf; + int err = devx_query_device(dev_bdf, mlx5_dev); + if (err) + return -E_FAIL; + return 0; +} + +static void * +mlx5_glue_devx_query_hca_iseg_mapping(void *ctx, uint32_t *cb_iseg) +{ + struct mlx5_context *mlx5_ctx; + void *pv_iseg; + int err; + + if (!ctx) { + errno = EINVAL; + return NULL; + } + mlx5_ctx = (struct mlx5_context *)ctx; + err = devx_query_hca_iseg_mapping(mlx5_ctx->devx_ctx, + cb_iseg, &pv_iseg); + if (err) { + errno = err; + return NULL; + } + return pv_iseg; +} + +static void * +mlx5_glue_devx_obj_create(void *ctx, + void *in, size_t inlen, + void *out, size_t outlen) +{ + mlx5_devx_obj_st *devx_obj; + + if (!ctx) { + errno = EINVAL; + return NULL; + } + devx_obj = malloc((sizeof(*devx_obj))); + if (!devx_obj) { + errno = ENOMEM; + return NULL; + } + memset(devx_obj, 0, sizeof(*devx_obj)); + devx_obj->devx_ctx = GET_DEVX_CTX(ctx); + devx_obj->obj = devx_obj_create(devx_obj->devx_ctx, + in, inlen, out, outlen); + if (DEVX_IS_ERR(devx_obj->obj)) { + errno = -DEVX_PTR_ERR(devx_obj->obj); + free(devx_obj); + return NULL; + } + return devx_obj; +} + +static int +mlx5_glue_devx_obj_destroy(void *obj) +{ + mlx5_devx_obj_st *devx_obj; + + if (!obj) + return -EINVAL; + devx_obj = obj; + int rc = devx_obj_destroy(devx_obj->obj); + free(devx_obj); + return rc; +} + +static int +mlx5_glue_devx_general_cmd(void *ctx, + void *in, size_t inlen, + void *out, size_t outlen) +{ + if (!ctx) + return -EINVAL; + return devx_cmd(GET_DEVX_CTX(ctx), in, inlen, out, outlen); +} + +static int +mlx5_glue_devx_obj_query(void *obj, + void *in, size_t inlen, + void *out, size_t outlen) +{ + return devx_cmd(GET_OBJ_CTX(obj), in, inlen, out, outlen); +} + +static int +mlx5_glue_devx_obj_modify(void *obj, + void *in, size_t inlen, + void *out, size_t outlen) +{ + return devx_cmd(GET_OBJ_CTX(obj), in, inlen, out, outlen); +} + +static int +mlx5_glue_devx_umem_dereg(void *pumem) +{ + struct devx_obj_handle *umem; + + if (!pumem) + return -EINVAL; + umem = pumem; + return devx_umem_unreg(umem); +} + +static void * +mlx5_glue_devx_umem_reg(void *ctx, void *addr, size_t size, + uint32_t access, uint32_t *id) +{ + struct devx_obj_handle *umem_hdl; + int w_access = DEVX_UMEM_ACCESS_READ; + + if (!ctx) { + errno = EINVAL; + return NULL; + } + if (access) + w_access |= DEVX_UMEM_ACCESS_WRITE; + + umem_hdl = devx_umem_reg(GET_DEVX_CTX(ctx), addr, + size, w_access, id); + if (DEVX_IS_ERR(umem_hdl)) { + errno = -DEVX_PTR_ERR(umem_hdl); + return NULL; + } + return umem_hdl; +} + +static void * +mlx5_glue_devx_alloc_uar(void *ctx, + uint32_t flags) +{ + devx_uar_handle *uar; + + if (!ctx) { + errno = EINVAL; + return NULL; + } + uar = devx_alloc_uar(GET_DEVX_CTX(ctx), flags); + if (DEVX_IS_ERR(uar)) { + errno = -DEVX_PTR_ERR(uar); + return NULL; + } + return uar; +} + +static int +mlx5_glue_devx_query_eqn(void *ctx, + uint32_t cpus, uint32_t *eqn) +{ + if (!ctx) + return -EINVAL; + return devx_query_eqn(GET_DEVX_CTX(ctx), cpus, eqn); +} + +static void +mlx5_glue_devx_free_uar(void *uar) +{ + devx_free_uar((devx_uar_handle *)uar); +} + +static void* +mlx5_glue_devx_fs_rule_add(void *ctx, void *in, uint32_t inlen) + +{ + struct devx_obj_handle *rule_hdl = NULL; + + if (!ctx) { + errno = EINVAL; + return NULL; + } + rule_hdl = devx_fs_rule_add(GET_DEVX_CTX(ctx), in, inlen); + if (DEVX_IS_ERR(rule_hdl)) { + errno = -DEVX_PTR_ERR(rule_hdl); + return NULL; + } + return rule_hdl; +} + +static int +mlx5_glue_devx_fs_rule_del(void *flow) +{ + return devx_fs_rule_del(flow); +} + +alignas(RTE_CACHE_LINE_SIZE) +const struct mlx5_glue *mlx5_glue = &(const struct mlx5_glue){ + .version = MLX5_GLUE_VERSION, + .get_device_list = mlx5_glue_devx_get_device_list, + .free_device_list = mlx5_glue_devx_free_device_list, + .open_device = mlx5_glue_devx_open_device, + .close_device = mlx5_glue_devx_close_device, + .query_device = mlx5_glue_devx_query_device, + .query_hca_iseg = mlx5_glue_devx_query_hca_iseg_mapping, + .devx_obj_create = mlx5_glue_devx_obj_create, + .devx_obj_destroy = mlx5_glue_devx_obj_destroy, + .devx_obj_query = mlx5_glue_devx_obj_query, + .devx_obj_modify = mlx5_glue_devx_obj_modify, + .devx_general_cmd = mlx5_glue_devx_general_cmd, + .devx_umem_reg = mlx5_glue_devx_umem_reg, + .devx_umem_dereg = mlx5_glue_devx_umem_dereg, + .devx_alloc_uar = mlx5_glue_devx_alloc_uar, + .devx_free_uar = mlx5_glue_devx_free_uar, + .devx_fs_rule_add = mlx5_glue_devx_fs_rule_add, + .devx_fs_rule_del = mlx5_glue_devx_fs_rule_del, + .devx_query_eqn = mlx5_glue_devx_query_eqn, +}; diff --git a/drivers/common/mlx5/windows/mlx5_glue.h b/drivers/common/mlx5/windows/mlx5_glue.h new file mode 100644 index 0000000000..f2261ec7ac --- /dev/null +++ b/drivers/common/mlx5/windows/mlx5_glue.h @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2020 Mellanox Technologies, Ltd + */ + +#ifndef MLX5_GLUE_H_ +#define MLX5_GLUE_H_ + +#include +#include + +#include +#include + +#ifndef MLX5_GLUE_VERSION +#define MLX5_GLUE_VERSION "" +#endif + +/* LIB_GLUE_VERSION must be updated every time this structure is modified. */ +struct mlx5_glue { + const char *version; + void *(*devx_obj_create)(void *ctx, + void *in, size_t inlen, + void *out, size_t outlen); + int (*devx_obj_destroy)(void *obj); + int (*devx_obj_query)(void *obj, + void *in, size_t inlen, + void *out, size_t outlen); + int (*devx_obj_modify)(void *obj, + void *in, size_t inlen, + void *out, size_t outlen); + int (*devx_general_cmd)(void *ctx, + void *in, size_t inlen, + void *out, size_t outlen); + int (*devx_umem_dereg)(void *umem); + void *(*devx_umem_reg)(void *ctx, + void *addr, size_t size, + uint32_t access, uint32_t *id); + void *(*devx_alloc_uar)(void *ctx, + uint32_t flags); + void (*devx_free_uar)(void *uar); + void *(*get_device_list)(int *num_devices); + void (*free_device_list)(void *list); + void *(*open_device)(void *device); + int (*close_device)(void *ctx); + int (*query_device)(void *device_bdf, void *dev_inf); + void* (*query_hca_iseg)(void *ctx, uint32_t *cb_iseg); + int (*devx_obj_query_async)(void *obj, + const void *in, size_t inlen, + size_t outlen, uint64_t wr_id, + void *cmd_comp); + void *(*devx_fs_rule_add)(void *ctx, void *in, uint32_t inlen); + int (*devx_fs_rule_del)(void *flow); + int (*devx_query_eqn)(void *context, uint32_t cpus, uint32_t *eqn); +}; + +extern const struct mlx5_glue *mlx5_glue; + +#endif /* MLX5_GLUE_H_ */ diff --git a/drivers/common/mlx5/windows/mlx5_win_defs.h b/drivers/common/mlx5/windows/mlx5_win_defs.h new file mode 100644 index 0000000000..72a3131f5e --- /dev/null +++ b/drivers/common/mlx5/windows/mlx5_win_defs.h @@ -0,0 +1,25 @@ +/* + * Copyright (C) Mellanox Technologies, Ltd. 2001-2020. + * + */ +#ifndef __MLX5_WIN_DEFS_H__ +#define __MLX5_WIN_DEFS_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +enum { + MLX5_CQE_OWNER_MASK = 1, + MLX5_CQE_REQ = 0, + MLX5_CQE_RESP_WR_IMM = 1, + MLX5_CQE_RESP_SEND = 2, + MLX5_CQE_RESP_SEND_IMM = 3, + MLX5_CQE_RESP_SEND_INV = 4, + MLX5_CQE_RESIZE_CQ = 5, + MLX5_CQE_NO_PACKET = 6, + MLX5_CQE_REQ_ERR = 13, + MLX5_CQE_RESP_ERR = 14, + MLX5_CQE_INVALID = 15, +}; +#endif /* __MLX5_WIN_DEFS_H__ */ diff --git a/drivers/common/mlx5/windows/mlx5_win_ext.h b/drivers/common/mlx5/windows/mlx5_win_ext.h new file mode 100644 index 0000000000..0e74910e9d --- /dev/null +++ b/drivers/common/mlx5/windows/mlx5_win_ext.h @@ -0,0 +1,34 @@ +/* + * Copyright (C) Mellanox Technologies, Ltd. 2001-2020. + * + */ +#ifndef __MLX5_WIN_ETX_H__ +#define __MLX5_WIN_ETX_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "mlx5devx.h" + +typedef struct mlx5_context { + devx_device_ctx *devx_ctx; + struct devx_device mlx5_dev; + +} mlx5_context_st; + +typedef struct { + devx_device_ctx *devx_ctx; + struct devx_obj_handle *obj; +} mlx5_devx_obj_st; + +struct mlx5_devx_umem { + void *addr; + struct devx_obj_handle *umem_hdl; + uint32_t umem_id; +}; + +#define GET_DEVX_CTX(ctx) (((mlx5_context_st *)ctx)->devx_ctx) +#define GET_OBJ_CTX(obj) (((mlx5_devx_obj_st *)obj)->devx_ctx) + +#endif /* __MLX5_WIN_ETX_H__ */ -- 2.16.1.windows.4