From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 04FFFA0526; Mon, 20 Jan 2020 18:05:01 +0100 (CET) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 37C301BFD6; Mon, 20 Jan 2020 18:03:37 +0100 (CET) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by dpdk.org (Postfix) with ESMTP id 2C0D91BF9A for ; Mon, 20 Jan 2020 18:03:14 +0100 (CET) Received: from Internal Mail-Server by MTLPINE1 (envelope-from asafp@mellanox.com) with ESMTPS (AES256-SHA encrypted); 20 Jan 2020 19:03:12 +0200 Received: from pegasus07.mtr.labs.mlnx (pegasus07.mtr.labs.mlnx [10.210.16.112]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 00KH3BGU024424; Mon, 20 Jan 2020 19:03:12 +0200 From: Matan Azrad To: dev@dpdk.org Cc: Maxime Coquelin , Thomas Monjalon Date: Mon, 20 Jan 2020 17:02:43 +0000 Message-Id: <1579539790-3882-12-git-send-email-matan@mellanox.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1579539790-3882-1-git-send-email-matan@mellanox.com> References: <1579539790-3882-1-git-send-email-matan@mellanox.com> Subject: [dpdk-dev] [PATCH v1 11/38] common/mlx5: support DevX indirect mkey creation X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Add option to create an indirect mkey by the current mlx5_devx_cmd_mkey_create command. Align the net/mlx5 driver usage in the above command. Signed-off-by: Matan Azrad --- drivers/common/mlx5/mlx5_devx_cmds.c | 43 ++++++++++++++++++++++++++++++------ drivers/common/mlx5/mlx5_devx_cmds.h | 12 ++++++++++ drivers/common/mlx5/mlx5_prm.h | 12 ++++++++++ drivers/net/mlx5/mlx5_flow_dv.c | 4 ++++ 4 files changed, 64 insertions(+), 7 deletions(-) diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c index c6965da..aa2feeb 100644 --- a/drivers/common/mlx5/mlx5_devx_cmds.c +++ b/drivers/common/mlx5/mlx5_devx_cmds.c @@ -142,7 +142,11 @@ struct mlx5_devx_obj * mlx5_devx_cmd_mkey_create(struct ibv_context *ctx, struct mlx5_devx_mkey_attr *attr) { - uint32_t in[MLX5_ST_SZ_DW(create_mkey_in)] = {0}; + struct mlx5_klm *klm_array = attr->klm_array; + int klm_num = attr->klm_num; + int in_size_dw = MLX5_ST_SZ_DW(create_mkey_in) + + (klm_num ? RTE_ALIGN(klm_num, 4) : 0) * MLX5_ST_SZ_DW(klm); + uint32_t in[in_size_dw]; uint32_t out[MLX5_ST_SZ_DW(create_mkey_out)] = {0}; void *mkc; struct mlx5_devx_obj *mkey = rte_zmalloc("mkey", sizeof(*mkey), 0); @@ -153,27 +157,52 @@ struct mlx5_devx_obj * rte_errno = ENOMEM; return NULL; } + memset(in, 0, in_size_dw * 4); pgsize = sysconf(_SC_PAGESIZE); - translation_size = (RTE_ALIGN(attr->size, pgsize) * 8) / 16; MLX5_SET(create_mkey_in, in, opcode, MLX5_CMD_OP_CREATE_MKEY); + mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); + if (klm_num > 0) { + int i; + uint8_t *klm = (uint8_t *)MLX5_ADDR_OF(create_mkey_in, in, + klm_pas_mtt); + translation_size = RTE_ALIGN(klm_num, 4); + for (i = 0; i < klm_num; i++) { + MLX5_SET(klm, klm, byte_count, klm_array[i].byte_count); + MLX5_SET(klm, klm, mkey, klm_array[i].mkey); + MLX5_SET64(klm, klm, address, klm_array[i].address); + klm += MLX5_ST_SZ_BYTES(klm); + } + for (; i < (int)translation_size; i++) { + MLX5_SET(klm, klm, mkey, 0x0); + MLX5_SET64(klm, klm, address, 0x0); + klm += MLX5_ST_SZ_BYTES(klm); + } + MLX5_SET(mkc, mkc, access_mode_1_0, attr->log_entity_size ? + MLX5_MKC_ACCESS_MODE_KLM_FBS : + MLX5_MKC_ACCESS_MODE_KLM); + MLX5_SET(mkc, mkc, log_page_size, attr->log_entity_size); + } else { + translation_size = (RTE_ALIGN(attr->size, pgsize) * 8) / 16; + MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT); + MLX5_SET(mkc, mkc, log_page_size, rte_log2_u32(pgsize)); + } MLX5_SET(create_mkey_in, in, translations_octword_actual_size, translation_size); MLX5_SET(create_mkey_in, in, mkey_umem_id, attr->umem_id); - mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); + MLX5_SET(create_mkey_in, in, pg_access, attr->pg_access); MLX5_SET(mkc, mkc, lw, 0x1); MLX5_SET(mkc, mkc, lr, 0x1); - MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT); MLX5_SET(mkc, mkc, qpn, 0xffffff); MLX5_SET(mkc, mkc, pd, attr->pd); MLX5_SET(mkc, mkc, mkey_7_0, attr->umem_id & 0xFF); MLX5_SET(mkc, mkc, translations_octword_size, translation_size); MLX5_SET64(mkc, mkc, start_addr, attr->addr); MLX5_SET64(mkc, mkc, len, attr->size); - MLX5_SET(mkc, mkc, log_page_size, rte_log2_u32(pgsize)); - mkey->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in), out, + mkey->obj = mlx5_glue->devx_obj_create(ctx, in, in_size_dw * 4, out, sizeof(out)); if (!mkey->obj) { - DRV_LOG(ERR, "Can't create mkey - error %d", errno); + DRV_LOG(ERR, "Can't create %sdirect mkey - error %d\n", + klm_num ? "an in" : "a ", errno); rte_errno = errno; rte_free(mkey); return NULL; diff --git a/drivers/common/mlx5/mlx5_devx_cmds.h b/drivers/common/mlx5/mlx5_devx_cmds.h index dea1597..ceeca64 100644 --- a/drivers/common/mlx5/mlx5_devx_cmds.h +++ b/drivers/common/mlx5/mlx5_devx_cmds.h @@ -13,11 +13,22 @@ struct mlx5_devx_obj { int id; /* The object ID. */ }; +/* UMR memory buffer used to define 1 entry in indirect mkey. */ +struct mlx5_klm { + uint32_t byte_count; + uint32_t mkey; + uint64_t address; +}; + struct mlx5_devx_mkey_attr { uint64_t addr; uint64_t size; uint32_t umem_id; uint32_t pd; + uint32_t log_entity_size; + uint32_t pg_access:1; + struct mlx5_klm *klm_array; + int klm_num; }; /* HCA qos attributes. */ @@ -212,6 +223,7 @@ struct mlx5_devx_modify_sq_attr { uint32_t hairpin_peer_vhca:16; }; + /* mlx5_devx_cmds.c */ struct mlx5_devx_obj *mlx5_devx_cmd_flow_counter_alloc(struct ibv_context *ctx, diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h index b0b8ab9..058bd8c 100644 --- a/drivers/common/mlx5/mlx5_prm.h +++ b/drivers/common/mlx5/mlx5_prm.h @@ -726,6 +726,8 @@ enum { enum { MLX5_MKC_ACCESS_MODE_MTT = 0x1, + MLX5_MKC_ACCESS_MODE_KLM = 0x2, + MLX5_MKC_ACCESS_MODE_KLM_FBS = 0x3, }; /* Flow counters. */ @@ -790,6 +792,16 @@ struct mlx5_ifc_query_flow_counter_in_bits { u8 flow_counter_id[0x20]; }; +#define MLX5_MAX_KLM_BYTE_COUNT 0x80000000u +#define MLX5_MIN_KLM_FIXED_BUFFER_SIZE 0x1000u + + +struct mlx5_ifc_klm_bits { + u8 byte_count[0x20]; + u8 mkey[0x20]; + u8 address[0x40]; +}; + struct mlx5_ifc_mkc_bits { u8 reserved_at_0[0x1]; u8 free[0x1]; diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index 50d1078..cfdfdf4 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -3933,6 +3933,10 @@ struct field_modify_info modify_tcp[] = { mkey_attr.size = size; mkey_attr.umem_id = mem_mng->umem->umem_id; mkey_attr.pd = sh->pdn; + mkey_attr.log_entity_size = 0; + mkey_attr.pg_access = 0; + mkey_attr.klm_array = NULL; + mkey_attr.klm_num = 0; mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx, &mkey_attr); if (!mem_mng->dm) { mlx5_glue->devx_umem_dereg(mem_mng->umem); -- 1.8.3.1