From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id BCCACA04B5; Wed, 28 Oct 2020 00:33:53 +0100 (CET) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 7176FACAA; Wed, 28 Oct 2020 00:24:59 +0100 (CET) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by dpdk.org (Postfix) with ESMTP id 3B56B34EE for ; Wed, 28 Oct 2020 00:24:01 +0100 (CET) Received: from Internal Mail-Server by MTLPINE1 (envelope-from ophirmu@nvidia.com) with SMTP; 28 Oct 2020 01:23:56 +0200 Received: from nvidia.com (pegasus05.mtr.labs.mlnx [10.210.16.100]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 09RNNrrg026642; Wed, 28 Oct 2020 01:23:55 +0200 From: Ophir Munk To: dev@dpdk.org, Raslan Darawsheh Cc: Ophir Munk , Matan Azrad , Tal Shnaiderman , Thomas Monjalon Date: Tue, 27 Oct 2020 23:22:53 +0000 Message-Id: <20201027232335.31427-31-ophirmu@nvidia.com> X-Mailer: git-send-email 2.8.4 In-Reply-To: <20201027232335.31427-1-ophirmu@nvidia.com> References: <20201027232335.31427-1-ophirmu@nvidia.com> Subject: [dpdk-dev] [PATCH v1 30/72] common/mlx5/windows: add OS reg/dereg MR X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" This commits implements Windows API for MR registration and deregistration. It is based on DevX. Is support the relaxed ordering flow in Windows by checking the capabilities and machine type. Signed-off-by: Ophir Munk Signed-off-by: Tal Shnaiderman Acked-by: Matan Azrad --- drivers/common/mlx5/mlx5_common_mr.h | 1 + drivers/common/mlx5/rte_common_mlx5_exports.def | 2 + drivers/common/mlx5/windows/mlx5_common_os.c | 78 +++++++++++++++++++++++++ drivers/common/mlx5/windows/mlx5_common_os.h | 7 +++ 4 files changed, 88 insertions(+) diff --git a/drivers/common/mlx5/mlx5_common_mr.h b/drivers/common/mlx5/mlx5_common_mr.h index a2c426d..c4f87a5 100644 --- a/drivers/common/mlx5/mlx5_common_mr.h +++ b/drivers/common/mlx5/mlx5_common_mr.h @@ -28,6 +28,7 @@ struct mlx5_pmd_mr { void *addr; size_t len; void *obj; /* verbs mr object or devx umem object. */ + struct mlx5_devx_obj *mkey; /* devx mkey object. */ }; /** diff --git a/drivers/common/mlx5/rte_common_mlx5_exports.def b/drivers/common/mlx5/rte_common_mlx5_exports.def index 1d12e71..23eee59 100644 --- a/drivers/common/mlx5/rte_common_mlx5_exports.def +++ b/drivers/common/mlx5/rte_common_mlx5_exports.def @@ -61,5 +61,7 @@ EXPORTS mlx5_free mlx5_os_alloc_pd mlx5_os_dealloc_pd + mlx5_os_dereg_mr + mlx5_os_reg_mr mlx5_os_umem_reg mlx5_os_umem_dereg diff --git a/drivers/common/mlx5/windows/mlx5_common_os.c b/drivers/common/mlx5/windows/mlx5_common_os.c index c8343ad..52d609a 100644 --- a/drivers/common/mlx5/windows/mlx5_common_os.c +++ b/drivers/common/mlx5/windows/mlx5_common_os.c @@ -14,6 +14,8 @@ #include "mlx5_devx_cmds.h" #include "mlx5_common_utils.h" #include +#include "mlx5_common_os.h" +#include "mlx5_malloc.h" /** * Initialization routine for run-time dependency on external lib @@ -129,3 +131,79 @@ mlx5_os_umem_dereg(void *pumem) mlx5_free(umem); return err; } + +/** + * Register mr. Given protection doamin pointer, pointer to addr and length + * register the memory region. + * + * @param[in] pd + * Pointer to protection domain context (type mlx5_pd). + * @param[in] addr + * Pointer to memory start address (type devx_device_ctx). + * @param[in] length + * Lengtoh of the memory to register. + * @param[out] pmd_mr + * pmd_mr struct set with lkey, address, length, pointer to mr object, mkey + * + * @return + * 0 on successful registration, -1 otherwise + */ +int +mlx5_os_reg_mr(void *pd, + void *addr, size_t length, struct mlx5_pmd_mr *pmd_mr) +{ + struct mlx5_devx_mkey_attr mkey_attr; + struct mlx5_pd *mlx5_pd = (struct mlx5_pd *)pd; + struct mlx5_hca_attr attr; + + if (!pd || !addr) { + rte_errno = EINVAL; + return -1; + } + memset(pmd_mr, 0, sizeof(*pmd_mr)); + if (mlx5_devx_cmd_query_hca_attr(mlx5_pd->devx_ctx, &attr)) + return -1; + pmd_mr->addr = addr; + pmd_mr->len = length; + pmd_mr->obj = mlx5_os_umem_reg(mlx5_pd->devx_ctx, pmd_mr->addr, + pmd_mr->len, IBV_ACCESS_LOCAL_WRITE); + if (!pmd_mr->obj) + return -1; + mkey_attr.addr = (uintptr_t)addr; + mkey_attr.size = length; + mkey_attr.umem_id = ((struct mlx5_devx_umem *)(pmd_mr->obj))->umem_id; + mkey_attr.pd = mlx5_pd->pdn; + mkey_attr.log_entity_size = 0; + mkey_attr.pg_access = 0; + mkey_attr.klm_array = NULL; + mkey_attr.klm_num = 0; + mkey_attr.relaxed_ordering_read = 0; + mkey_attr.relaxed_ordering_write = 0; + if (!haswell_broadwell_cpu) { + mkey_attr.relaxed_ordering_write = attr.relaxed_ordering_write; + mkey_attr.relaxed_ordering_read = attr.relaxed_ordering_read; + } + pmd_mr->mkey = mlx5_devx_cmd_mkey_create(mlx5_pd->devx_ctx, &mkey_attr); + if (!pmd_mr->mkey) { + claim_zero(mlx5_os_umem_dereg(pmd_mr->obj)); + return -1; + } + pmd_mr->lkey = pmd_mr->mkey->id; + return 0; +} + +/** + * De-register mr. + * + * @param[in] pmd_mr + * Pointer to PMD mr object + */ +void +mlx5_os_dereg_mr(struct mlx5_pmd_mr *pmd_mr) +{ + if (pmd_mr && pmd_mr->mkey) + claim_zero(mlx5_glue->devx_obj_destroy(pmd_mr->mkey->obj)); + if (pmd_mr && pmd_mr->obj) + claim_zero(mlx5_os_umem_dereg(pmd_mr->obj)); + memset(pmd_mr, 0, sizeof(*pmd_mr)); +} diff --git a/drivers/common/mlx5/windows/mlx5_common_os.h b/drivers/common/mlx5/windows/mlx5_common_os.h index decb5ac..ba16641 100644 --- a/drivers/common/mlx5/windows/mlx5_common_os.h +++ b/drivers/common/mlx5/windows/mlx5_common_os.h @@ -7,9 +7,13 @@ #include +#include + #include "mlx5_autoconf.h" #include "mlx5_glue.h" #include "mlx5_malloc.h" +#include "mlx5_common_mr.h" +#include "mlx5_win_ext.h" /** * This API allocates aligned or non-aligned memory. The free can be on either @@ -144,4 +148,7 @@ void *mlx5_os_alloc_pd(void *ctx); int mlx5_os_dealloc_pd(void *pd); void *mlx5_os_umem_reg(void *ctx, void *addr, size_t size, uint32_t access); int mlx5_os_umem_dereg(void *pumem); +int mlx5_os_reg_mr(void *pd, + void *addr, size_t length, struct mlx5_pmd_mr *pmd_mr); +void mlx5_os_dereg_mr(struct mlx5_pmd_mr *pmd_mr); #endif /* RTE_PMD_MLX5_COMMON_OS_H_ */ -- 2.8.4