From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 12516A0540; Sun, 19 Jul 2020 20:11:21 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id F29261C1B4; Sun, 19 Jul 2020 20:10:13 +0200 (CEST) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by dpdk.org (Postfix) with ESMTP id ABBDC1C1A2 for ; Sun, 19 Jul 2020 20:10:03 +0200 (CEST) Received: from Internal Mail-Server by MTLPINE1 (envelope-from orika@mellanox.com) with SMTP; 19 Jul 2020 21:10:02 +0300 Received: from pegasus04.mtr.labs.mlnx. (pegasus04.mtr.labs.mlnx [10.210.16.126]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 06JI9ISN009919; Sun, 19 Jul 2020 21:10:02 +0300 From: Ori Kam To: jerinj@marvell.com, xiang.w.wang@intel.com, matan@mellanox.com, viacheslavo@mellanox.com, Shahaf Shuler Cc: guyk@marvell.com, dev@dpdk.org, pbhagavatula@marvell.com, hemant.agrawal@nxp.com, opher@mellanox.com, alexr@mellanox.com, dovrat@marvell.com, pkapoor@marvell.com, nipun.gupta@nxp.com, bruce.richardson@intel.com, yang.a.hong@intel.com, harry.chang@intel.com, gu.jian1@zte.com.cn, shanjiangh@chinatelecom.cn, zhangy.yun@chinatelecom.cn, lixingfu@huachentel.com, wushuai@inspur.com, yuyingxia@yxlink.com, fanchenggang@sunyainfo.com, davidfgao@tencent.com, liuzhong1@chinaunicom.cn, zhaoyong11@huawei.com, oc@yunify.com, jim@netgate.com, hongjun.ni@intel.com, deri@ntop.org, fc@napatech.com, arthur.su@lionic.com, thomas@monjalon.net, orika@mellanox.com, rasland@mellanox.com, Yuval Avnery Date: Sun, 19 Jul 2020 18:09:10 +0000 Message-Id: <1595182154-36508-11-git-send-email-orika@mellanox.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1595182154-36508-1-git-send-email-orika@mellanox.com> References: <1593941027-86651-1-git-send-email-orika@mellanox.com> <1595182154-36508-1-git-send-email-orika@mellanox.com> Subject: [dpdk-dev] [PATCH v5 10/13] regex/mlx5: fastpath setup X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" From: Yuval Avnery Allocated and register input/output buffers and metadata. Signed-off-by: Yuval Avnery Acked-by: Ori Kam --- drivers/common/mlx5/mlx5_prm.h | 36 ++++++ drivers/regex/mlx5/Makefile | 1 + drivers/regex/mlx5/meson.build | 1 + drivers/regex/mlx5/mlx5_regex.h | 8 ++ drivers/regex/mlx5/mlx5_regex_control.c | 2 + drivers/regex/mlx5/mlx5_regex_fastpath.c | 200 +++++++++++++++++++++++++++++++ 6 files changed, 248 insertions(+) create mode 100644 drivers/regex/mlx5/mlx5_regex_fastpath.c diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h index a59acc7..6a3f22e 100644 --- a/drivers/common/mlx5/mlx5_prm.h +++ b/drivers/common/mlx5/mlx5_prm.h @@ -405,6 +405,42 @@ struct mlx5_cqe_ts { uint8_t op_own; }; +/* MMO metadata segment */ + +#define MLX5_OPCODE_MMO 0x2f +#define MLX5_OPC_MOD_MMO_REGEX 0x4 + +struct mlx5_wqe_metadata_seg { + uint32_t mmo_control_31_0; /* mmo_control_63_32 is in ctrl_seg.imm */ + uint32_t lkey; + uint64_t addr; +}; + +struct mlx5_ifc_regexp_mmo_control_bits { + uint8_t reserved_at_31[0x2]; + uint8_t le[0x1]; + uint8_t reserved_at_28[0x1]; + uint8_t subset_id_0[0xc]; + uint8_t reserved_at_16[0x4]; + uint8_t subset_id_1[0xc]; + uint8_t ctrl[0x4]; + uint8_t subset_id_2[0xc]; + uint8_t reserved_at_16_1[0x4]; + uint8_t subset_id_3[0xc]; +}; + +struct mlx5_ifc_regexp_metadata_bits { + uint8_t rof_version[0x10]; + uint8_t latency_count[0x10]; + uint8_t instruction_count[0x10]; + uint8_t primary_thread_count[0x10]; + uint8_t match_count[0x8]; + uint8_t detected_match_count[0x8]; + uint8_t status[0x10]; + uint8_t job_id[0x20]; + uint8_t reserved[0x80]; +}; + /* Adding direct verbs to data-path. */ /* CQ sequence number mask. */ diff --git a/drivers/regex/mlx5/Makefile b/drivers/regex/mlx5/Makefile index 8918ebe..9e11926 100644 --- a/drivers/regex/mlx5/Makefile +++ b/drivers/regex/mlx5/Makefile @@ -11,6 +11,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX5_REGEX_PMD) += mlx5_regex.c SRCS-$(CONFIG_RTE_LIBRTE_MLX5_REGEX_PMD) += mlx5_rxp.c SRCS-$(CONFIG_RTE_LIBRTE_MLX5_REGEX_PMD) += mlx5_regex_devx.c SRCS-$(CONFIG_RTE_LIBRTE_MLX5_REGEX_PMD) += mlx5_regex_control.c +SRCS-$(CONFIG_RTE_LIBRTE_MLX5_REGEX_PMD) += mlx5_regex_fastpath.c # Basic CFLAGS. CFLAGS += -O3 diff --git a/drivers/regex/mlx5/meson.build b/drivers/regex/mlx5/meson.build index 4495fa7..d0f8fcf 100644 --- a/drivers/regex/mlx5/meson.build +++ b/drivers/regex/mlx5/meson.build @@ -14,6 +14,7 @@ sources = files( 'mlx5_rxp.c', 'mlx5_regex_devx.c', 'mlx5_regex_control.c', + 'mlx5_regex_fastpath.c', ) cflags_options = [ '-std=c11', diff --git a/drivers/regex/mlx5/mlx5_regex.h b/drivers/regex/mlx5/mlx5_regex.h index bf285a1..21bb02b 100644 --- a/drivers/regex/mlx5/mlx5_regex.h +++ b/drivers/regex/mlx5/mlx5_regex.h @@ -40,6 +40,11 @@ struct mlx5_regex_qp { struct mlx5_regex_sq *sqs; /* Pointer to sq array. */ uint16_t nb_obj; /* Number of sq objects. */ struct mlx5_regex_cq cq; /* CQ struct. */ + uint32_t free_sqs; + struct mlx5_regex_job *jobs; + struct ibv_mr *metadata; + struct ibv_mr *inputs; + struct ibv_mr *outputs; }; struct mlx5_regex_db { @@ -91,4 +96,7 @@ int mlx5_devx_regex_database_program(void *ctx, uint8_t engine, /* mlx5_regex_control.c */ int mlx5_regex_qp_setup(struct rte_regexdev *dev, uint16_t qp_ind, const struct rte_regexdev_qp_conf *cfg); + +/* mlx5_regex_fastpath.c */ +int mlx5_regexdev_setup_fastpath(struct mlx5_regex_priv *priv, uint32_t qp_id); #endif /* MLX5_REGEX_H */ diff --git a/drivers/regex/mlx5/mlx5_regex_control.c b/drivers/regex/mlx5/mlx5_regex_control.c index 83d390a..ec57e24 100644 --- a/drivers/regex/mlx5/mlx5_regex_control.c +++ b/drivers/regex/mlx5/mlx5_regex_control.c @@ -363,6 +363,8 @@ goto error; } } + + mlx5_regexdev_setup_fastpath(priv, qp_ind); return 0; error: diff --git a/drivers/regex/mlx5/mlx5_regex_fastpath.c b/drivers/regex/mlx5/mlx5_regex_fastpath.c new file mode 100644 index 0000000..8d55bbc --- /dev/null +++ b/drivers/regex/mlx5/mlx5_regex_fastpath.c @@ -0,0 +1,200 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2020 Mellanox Technologies, Ltd + */ +#include +#include + +#include +#include +#include +#include +#include +#include +#include + + +#include +#include +#include +#include +#include + +#include "mlx5_regex_utils.h" +#include "mlx5_rxp.h" +#include "mlx5_regex.h" + +/* Verbs header. */ +/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ +#include + +#define MAX_WQE_INDEX 0xffff +#define MLX5_REGEX_METADATA_SIZE 64 +#define MLX5_REGEX_MAX_INPUT (1 << 14) +#define MLX5_REGEX_MAX_OUTPUT (1 << 11) + +#define MLX5_REGEX_WQE_METADATA_OFFSET 16 +#define MLX5_REGEX_WQE_GATHER_OFFSET 32 +#define MLX5_REGEX_WQE_SCATTER_OFFSET 48 + +static inline uint32_t +sq_size_get(struct mlx5_regex_sq *sq) +{ + return (1U << sq->log_nb_desc); +} +static inline uint32_t +cq_size_get(struct mlx5_regex_cq *cq) +{ + return (1U << cq->log_nb_desc); +} + +struct mlx5_regex_job { + uint64_t user_id; + uint8_t *input; + volatile uint8_t *output; + volatile uint8_t *metadata; +} __rte_cached_aligned; + +static inline void +set_data_seg(struct mlx5_wqe_data_seg *seg, + uint32_t length, uint32_t lkey, + uintptr_t address) +{ + seg->byte_count = rte_cpu_to_be_32(length); + seg->lkey = rte_cpu_to_be_32(lkey); + seg->addr = rte_cpu_to_be_64(address); +} + +static inline void +set_metadata_seg(struct mlx5_wqe_metadata_seg *seg, + uint32_t mmo_control_31_0, uint32_t lkey, + uintptr_t address) +{ + seg->mmo_control_31_0 = htobe32(mmo_control_31_0); + seg->lkey = rte_cpu_to_be_32(lkey); + seg->addr = rte_cpu_to_be_64(address); +} + +static void +setup_sqs(struct mlx5_regex_qp *queue) +{ + size_t sqid, entry; + uint32_t job_id; + for (sqid = 0; sqid < queue->nb_obj; sqid++) { + struct mlx5_regex_sq *sq = &queue->sqs[sqid]; + uint8_t *wqe = (uint8_t *)sq->wqe; + for (entry = 0 ; entry < sq_size_get(sq); entry++) { + job_id = sqid * sq_size_get(sq) + entry; + struct mlx5_regex_job *job = &queue->jobs[job_id]; + + set_metadata_seg((struct mlx5_wqe_metadata_seg *) + (wqe + MLX5_REGEX_WQE_METADATA_OFFSET), + 0, queue->metadata->lkey, + (uintptr_t)job->metadata); + set_data_seg((struct mlx5_wqe_data_seg *) + (wqe + MLX5_REGEX_WQE_GATHER_OFFSET), + 0, queue->inputs->lkey, + (uintptr_t)job->input); + set_data_seg((struct mlx5_wqe_data_seg *) + (wqe + MLX5_REGEX_WQE_SCATTER_OFFSET), + MLX5_REGEX_MAX_OUTPUT, + queue->outputs->lkey, + (uintptr_t)job->output); + wqe += 64; + } + queue->free_sqs |= 1 << sqid; + } +} + +static int +setup_buffers(struct mlx5_regex_qp *qp, struct ibv_pd *pd) +{ + uint32_t i; + int err; + + void *ptr = rte_calloc(__func__, qp->nb_desc, + MLX5_REGEX_METADATA_SIZE, + MLX5_REGEX_METADATA_SIZE); + if (!ptr) + return -ENOMEM; + + qp->metadata = mlx5_glue->reg_mr(pd, ptr, + MLX5_REGEX_METADATA_SIZE*qp->nb_desc, + IBV_ACCESS_LOCAL_WRITE); + if (!qp->metadata) { + rte_free(ptr); + return -EINVAL; + } + ptr = rte_calloc(__func__, qp->nb_desc, + MLX5_REGEX_MAX_INPUT, + MLX5_REGEX_MAX_INPUT); + + if (!ptr) { + err = -ENOMEM; + goto err_input; + } + qp->inputs = mlx5_glue->reg_mr(pd, ptr, + MLX5_REGEX_MAX_INPUT*qp->nb_desc, + IBV_ACCESS_LOCAL_WRITE); + if (!qp->inputs) { + rte_free(ptr); + err = -EINVAL; + goto err_input; + } + + ptr = rte_calloc(__func__, qp->nb_desc, + MLX5_REGEX_MAX_OUTPUT, + MLX5_REGEX_MAX_OUTPUT); + if (!ptr) { + err = -ENOMEM; + goto err_output; + } + qp->outputs = mlx5_glue->reg_mr(pd, ptr, + MLX5_REGEX_MAX_OUTPUT*qp->nb_desc, + IBV_ACCESS_LOCAL_WRITE); + if (!qp->outputs) { + rte_free(ptr); + err = -EINVAL; + goto err_output; + } + + /* distribute buffers to jobs */ + for (i = 0; i < qp->nb_desc; i++) { + qp->jobs[i].input = + (uint8_t *)qp->inputs->addr + + (i % qp->nb_desc) * MLX5_REGEX_MAX_INPUT; + qp->jobs[i].output = + (uint8_t *)qp->outputs->addr + + (i % qp->nb_desc) * MLX5_REGEX_MAX_OUTPUT; + qp->jobs[i].metadata = + (uint8_t *)qp->metadata->addr + + (i % qp->nb_desc) * MLX5_REGEX_METADATA_SIZE; + } + return 0; + +err_output: + ptr = qp->inputs->addr; + rte_free(ptr); + mlx5_glue->dereg_mr(qp->inputs); +err_input: + ptr = qp->metadata->addr; + rte_free(ptr); + mlx5_glue->dereg_mr(qp->metadata); + return err; +} + +int +mlx5_regexdev_setup_fastpath(struct mlx5_regex_priv *priv, uint32_t qp_id) +{ + struct mlx5_regex_qp *qp = &priv->qps[qp_id]; + int err; + + qp->jobs = rte_calloc(__func__, qp->nb_desc, sizeof(*qp->jobs), + sizeof(*qp->jobs)); + if (!qp->jobs) + return -ENOMEM; + err = setup_buffers(qp, priv->pd); + if (err) + return err; + setup_sqs(qp); + return 0; +} -- 1.8.3.1