From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id E017BA0540; Sun, 19 Jul 2020 20:11:42 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 1FEE91C1BA; Sun, 19 Jul 2020 20:10:31 +0200 (CEST) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by dpdk.org (Postfix) with ESMTP id CC02B1C1D0 for ; Sun, 19 Jul 2020 20:10:13 +0200 (CEST) Received: from Internal Mail-Server by MTLPINE1 (envelope-from orika@mellanox.com) with SMTP; 19 Jul 2020 21:10:10 +0300 Received: from pegasus04.mtr.labs.mlnx. (pegasus04.mtr.labs.mlnx [10.210.16.126]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 06JI9ISP009919; Sun, 19 Jul 2020 21:10:10 +0300 From: Ori Kam To: jerinj@marvell.com, xiang.w.wang@intel.com, matan@mellanox.com, viacheslavo@mellanox.com, Shahaf Shuler Cc: guyk@marvell.com, dev@dpdk.org, pbhagavatula@marvell.com, hemant.agrawal@nxp.com, opher@mellanox.com, alexr@mellanox.com, dovrat@marvell.com, pkapoor@marvell.com, nipun.gupta@nxp.com, bruce.richardson@intel.com, yang.a.hong@intel.com, harry.chang@intel.com, gu.jian1@zte.com.cn, shanjiangh@chinatelecom.cn, zhangy.yun@chinatelecom.cn, lixingfu@huachentel.com, wushuai@inspur.com, yuyingxia@yxlink.com, fanchenggang@sunyainfo.com, davidfgao@tencent.com, liuzhong1@chinaunicom.cn, zhaoyong11@huawei.com, oc@yunify.com, jim@netgate.com, hongjun.ni@intel.com, deri@ntop.org, fc@napatech.com, arthur.su@lionic.com, thomas@monjalon.net, orika@mellanox.com, rasland@mellanox.com, Yuval Avnery Date: Sun, 19 Jul 2020 18:09:12 +0000 Message-Id: <1595182154-36508-13-git-send-email-orika@mellanox.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1595182154-36508-1-git-send-email-orika@mellanox.com> References: <1593941027-86651-1-git-send-email-orika@mellanox.com> <1595182154-36508-1-git-send-email-orika@mellanox.com> Subject: [dpdk-dev] [PATCH v5 12/13] regex/mlx5: implement dequeue function X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" From: Yuval Avnery Implement dequeue function for the regex API. Signed-off-by: Yuval Avnery Acked-by: Ori Kam --- drivers/common/mlx5/mlx5_prm.h | 10 +++ drivers/regex/mlx5/mlx5_regex.c | 1 + drivers/regex/mlx5/mlx5_regex.h | 4 ++ drivers/regex/mlx5/mlx5_regex_control.c | 1 + drivers/regex/mlx5/mlx5_regex_fastpath.c | 108 ++++++++++++++++++++++++++++++- 5 files changed, 123 insertions(+), 1 deletion(-) diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h index 6a3f22e..8fab962 100644 --- a/drivers/common/mlx5/mlx5_prm.h +++ b/drivers/common/mlx5/mlx5_prm.h @@ -441,6 +441,12 @@ struct mlx5_ifc_regexp_metadata_bits { uint8_t reserved[0x80]; }; +struct mlx5_ifc_regexp_match_tuple_bits { + uint8_t length[0x10]; + uint8_t start_ptr[0x10]; + uint8_t rule_id[0x20]; +}; + /* Adding direct verbs to data-path. */ /* CQ sequence number mask. */ @@ -637,6 +643,10 @@ struct mlx5_modification_cmd { __mlx5_16_bit_off(typ, fld))); \ } while (0) +#define MLX5_GET_VOLATILE(typ, p, fld) \ + ((rte_be_to_cpu_32(*((volatile __be32 *)(p) +\ + __mlx5_dw_off(typ, fld))) >> __mlx5_dw_bit_off(typ, fld)) & \ + __mlx5_mask(typ, fld)) #define MLX5_GET(typ, p, fld) \ ((rte_be_to_cpu_32(*((__be32 *)(p) +\ __mlx5_dw_off(typ, fld))) >> __mlx5_dw_bit_off(typ, fld)) & \ diff --git a/drivers/regex/mlx5/mlx5_regex.c b/drivers/regex/mlx5/mlx5_regex.c index 607ca4d..fd457bf 100644 --- a/drivers/regex/mlx5/mlx5_regex.c +++ b/drivers/regex/mlx5/mlx5_regex.c @@ -168,6 +168,7 @@ } priv->regexdev->dev_ops = &mlx5_regexdev_ops; priv->regexdev->enqueue = mlx5_regexdev_enqueue; + priv->regexdev->dequeue = mlx5_regexdev_dequeue; priv->regexdev->device = (struct rte_device *)pci_dev; priv->regexdev->data->dev_private = priv; priv->regexdev->state = RTE_REGEXDEV_READY; diff --git a/drivers/regex/mlx5/mlx5_regex.h b/drivers/regex/mlx5/mlx5_regex.h index fb81cb8..217a1a4 100644 --- a/drivers/regex/mlx5/mlx5_regex.h +++ b/drivers/regex/mlx5/mlx5_regex.h @@ -34,6 +34,7 @@ struct mlx5_regex_cq { uint32_t dbr_umem; /* Door bell record umem id. */ volatile struct mlx5_cqe *cqe; /* The CQ ring buffer. */ struct mlx5dv_devx_umem *cqe_umem; /* CQ buffer umem. */ + size_t ci; uint32_t *dbr; }; @@ -105,4 +106,7 @@ int mlx5_regex_qp_setup(struct rte_regexdev *dev, uint16_t qp_ind, int mlx5_regexdev_setup_fastpath(struct mlx5_regex_priv *priv, uint32_t qp_id); uint16_t mlx5_regexdev_enqueue(struct rte_regexdev *dev, uint16_t qp_id, struct rte_regex_ops **ops, uint16_t nb_ops); +uint16_t mlx5_regexdev_dequeue(struct rte_regexdev *dev, uint16_t qp_id, + struct rte_regex_ops **ops, uint16_t nb_ops); + #endif /* MLX5_REGEX_H */ diff --git a/drivers/regex/mlx5/mlx5_regex_control.c b/drivers/regex/mlx5/mlx5_regex_control.c index f144c69..faafb76 100644 --- a/drivers/regex/mlx5/mlx5_regex_control.c +++ b/drivers/regex/mlx5/mlx5_regex_control.c @@ -120,6 +120,7 @@ cq->cqe_umem = mlx5_glue->devx_umem_reg(priv->ctx, buf, sizeof(struct mlx5_cqe) * cq_size, 7); + cq->ci = 0; if (!cq->cqe_umem) { DRV_LOG(ERR, "Can't register cqe mem."); rte_errno = ENOMEM; diff --git a/drivers/regex/mlx5/mlx5_regex_fastpath.c b/drivers/regex/mlx5/mlx5_regex_fastpath.c index ff6386b..fc5a7fc 100644 --- a/drivers/regex/mlx5/mlx5_regex_fastpath.c +++ b/drivers/regex/mlx5/mlx5_regex_fastpath.c @@ -35,6 +35,7 @@ #define MLX5_REGEX_WQE_METADATA_OFFSET 16 #define MLX5_REGEX_WQE_GATHER_OFFSET 32 #define MLX5_REGEX_WQE_SCATTER_OFFSET 48 +#define MLX5_REGEX_METADATA_OFF 32 #define MLX5_REGEX_WQE_METADATA_OFFSET 16 #define MLX5_REGEX_WQE_GATHER_OFFSET 32 @@ -159,7 +160,7 @@ struct mlx5_regex_job { static inline uint32_t job_id_get(uint32_t qid, size_t sq_size, size_t index) { - return qid*sq_size + index % sq_size; + return qid * sq_size + (index & (sq_size - 1)); } uint16_t @@ -192,6 +193,111 @@ struct mlx5_regex_job { return i; } +#define MLX5_REGEX_RESP_SZ 8 + +static inline void +extract_result(struct rte_regex_ops *op, struct mlx5_regex_job *job) +{ + size_t j, offset; + op->user_id = job->user_id; + op->nb_matches = MLX5_GET_VOLATILE(regexp_metadata, job->metadata + + MLX5_REGEX_METADATA_OFF, + match_count); + op->nb_actual_matches = MLX5_GET_VOLATILE(regexp_metadata, + job->metadata + + MLX5_REGEX_METADATA_OFF, + detected_match_count); + for (j = 0; j < op->nb_matches; j++) { + offset = MLX5_REGEX_RESP_SZ * j; + op->matches[j].rule_id = + MLX5_GET_VOLATILE(regexp_match_tuple, + (job->output + offset), rule_id); + op->matches[j].start_offset = + MLX5_GET_VOLATILE(regexp_match_tuple, + (job->output + offset), start_ptr); + op->matches[j].len = + MLX5_GET_VOLATILE(regexp_match_tuple, + (job->output + offset), length); + } +} + +static inline volatile struct mlx5_cqe * +poll_one(struct mlx5_regex_cq *cq) +{ + volatile struct mlx5_cqe *cqe; + size_t next_cqe_offset; + + next_cqe_offset = (cq->ci & (cq_size_get(cq) - 1)); + cqe = (volatile struct mlx5_cqe *)(cq->cqe + next_cqe_offset); + rte_cio_wmb(); + + int ret = check_cqe(cqe, cq_size_get(cq), cq->ci); + + if (unlikely(ret == MLX5_CQE_STATUS_ERR)) { + DRV_LOG(ERR, "Completion with error on qp 0x%x", 0); + return NULL; + } + + if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) + return NULL; + + return cqe; +} + + +/** + * DPDK callback for dequeue. + * + * @param dev + * Pointer to the regex dev structure. + * @param qp_id + * The queue to enqueue the traffic to. + * @param ops + * List of regex ops to dequeue. + * @param nb_ops + * Number of ops in ops parameter. + * + * @return + * Number of packets successfully dequeued (<= pkts_n). + */ +uint16_t +mlx5_regexdev_dequeue(struct rte_regexdev *dev, uint16_t qp_id, + struct rte_regex_ops **ops, uint16_t nb_ops) +{ + struct mlx5_regex_priv *priv = dev->data->dev_private; + struct mlx5_regex_qp *queue = &priv->qps[qp_id]; + struct mlx5_regex_cq *cq = &queue->cq; + volatile struct mlx5_cqe *cqe; + size_t i = 0; + + while ((cqe = poll_one(cq))) { + uint16_t wq_counter + = (rte_be_to_cpu_16(cqe->wqe_counter) + 1) & + MLX5_REGEX_MAX_WQE_INDEX; + size_t sqid = cqe->rsvd3[2]; + struct mlx5_regex_sq *sq = &queue->sqs[sqid]; + while (sq->ci != wq_counter) { + if (unlikely(i == nb_ops)) { + /* Return without updating cq->ci */ + goto out; + } + uint32_t job_id = job_id_get(sqid, sq_size_get(sq), + sq->ci); + extract_result(ops[i], &queue->jobs[job_id]); + sq->ci = (sq->ci + 1) & MLX5_REGEX_MAX_WQE_INDEX; + i++; + } + cq->ci = (cq->ci + 1) & 0xffffff; + rte_wmb(); + cq->dbr[0] = rte_cpu_to_be_32(cq->ci); + queue->free_sqs |= (1 << sqid); + } + +out: + queue->ci += i; + return i; +} + static void setup_sqs(struct mlx5_regex_qp *queue) { -- 1.8.3.1