From: Volodymyr Fialko <vfialko@marvell.com>
To: <dev@dpdk.org>, Ankur Dwivedi <adwivedi@marvell.com>,
Anoob Joseph <anoobj@marvell.com>,
Tejasree Kondoj <ktejasree@marvell.com>,
"Pavan Nikhilesh" <pbhagavatula@marvell.com>,
Shijith Thotton <sthotton@marvell.com>
Cc: <jerinj@marvell.com>, <gakhil@marvell.com>,
Volodymyr Fialko <vfialko@marvell.com>
Subject: [PATCH 2/3] crypto/cnxk: add burst enqueue for event crypto
Date: Thu, 4 Aug 2022 11:59:06 +0200 [thread overview]
Message-ID: <20220804095907.97895-3-vfialko@marvell.com> (raw)
In-Reply-To: <20220804095907.97895-1-vfialko@marvell.com>
Added support for burst enqueue for cn10k event crypto adapter.
Instruction will be grouped based on the queue pair and sent in a burst.
Signed-off-by: Volodymyr Fialko <vfialko@marvell.com>
---
drivers/crypto/cnxk/cn10k_cryptodev_ops.c | 141 +++++++++++++++-------
drivers/crypto/cnxk/cn10k_cryptodev_ops.h | 7 +-
drivers/crypto/cnxk/meson.build | 2 +-
drivers/event/cnxk/cn10k_eventdev.c | 2 +-
drivers/event/cnxk/cn10k_worker.c | 10 --
drivers/event/cnxk/cn10k_worker.h | 2 -
6 files changed, 105 insertions(+), 59 deletions(-)
diff --git a/drivers/crypto/cnxk/cn10k_cryptodev_ops.c b/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
index f761ba36e2..bfa6374005 100644
--- a/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
@@ -9,11 +9,12 @@
#include "cn10k_cryptodev.h"
#include "cn10k_cryptodev_ops.h"
-#include "cn10k_ipsec_la_ops.h"
#include "cn10k_ipsec.h"
+#include "cn10k_ipsec_la_ops.h"
#include "cnxk_ae.h"
#include "cnxk_cryptodev.h"
#include "cnxk_cryptodev_ops.h"
+#include "cnxk_eventdev.h"
#include "cnxk_se.h"
#include "roc_api.h"
@@ -391,79 +392,135 @@ cn10k_ca_meta_info_extract(struct rte_crypto_op *op,
return 0;
}
-uint16_t
-cn10k_cpt_crypto_adapter_enqueue(uintptr_t base, struct rte_crypto_op *op)
+static inline uint16_t
+ca_lmtst_burst_submit(struct cn10k_sso_hws *ws, uint64_t w2[], struct cnxk_cpt_qp *qp,
+ struct rte_crypto_op *op[], uint16_t nb_ops)
{
+ struct cpt_inflight_req *infl_reqs[PKTS_PER_LOOP];
+ uint64_t lmt_base, lmt_arg, io_addr;
+ struct cpt_inst_s *inst, *inst_base;
struct cpt_inflight_req *infl_req;
- uint64_t lmt_base, lmt_arg, w2;
- struct cpt_inst_s *inst;
union cpt_fc_write_s fc;
- struct cnxk_cpt_qp *qp;
uint64_t *fc_addr;
uint16_t lmt_id;
- int ret;
+ int ret, i;
- ret = cn10k_ca_meta_info_extract(op, &qp, &w2);
- if (unlikely(ret)) {
- rte_errno = EINVAL;
- return 0;
- }
+ lmt_base = qp->lmtline.lmt_base;
+ io_addr = qp->lmtline.io_addr;
+ fc_addr = qp->lmtline.fc_addr;
+
+ const uint32_t fc_thresh = qp->lmtline.fc_thresh;
+
+ ROC_LMT_BASE_ID_GET(lmt_base, lmt_id);
+ inst_base = (struct cpt_inst_s *)lmt_base;
if (unlikely(!qp->ca.enabled)) {
rte_errno = EINVAL;
return 0;
}
- if (unlikely(rte_mempool_get(qp->ca.req_mp, (void **)&infl_req))) {
+ if (unlikely(rte_mempool_get_bulk(qp->ca.req_mp, (void **)infl_reqs, nb_ops))) {
rte_errno = ENOMEM;
return 0;
}
- infl_req->op_flags = 0;
-
- lmt_base = qp->lmtline.lmt_base;
- fc_addr = qp->lmtline.fc_addr;
- const uint32_t fc_thresh = qp->lmtline.fc_thresh;
+ for (i = 0; i < nb_ops; i++) {
+ inst = &inst_base[2 * i];
+ infl_req = infl_reqs[i];
+ infl_req->op_flags = 0;
- ROC_LMT_BASE_ID_GET(lmt_base, lmt_id);
- inst = (struct cpt_inst_s *)lmt_base;
+ ret = cn10k_cpt_fill_inst(qp, &op[i], inst, infl_req);
+ if (unlikely(ret != 1)) {
+ plt_dp_err("Could not process op: %p", op[i]);
+ if (i != 0)
+ goto submit;
+ else
+ goto put;
+ }
- ret = cn10k_cpt_fill_inst(qp, &op, inst, infl_req);
- if (unlikely(ret != 1)) {
- plt_dp_err("Could not process op: %p", op);
- rte_mempool_put(qp->ca.req_mp, infl_req);
- return 0;
+ infl_req->res.cn10k.compcode = CPT_COMP_NOT_DONE;
+ infl_req->qp = qp;
+ inst->w0.u64 = 0;
+ inst->res_addr = (uint64_t)&infl_req->res;
+ inst->w2.u64 = w2[i];
+ inst->w3.u64 = CNXK_CPT_INST_W3(1, infl_req);
}
- infl_req->cop = op;
- infl_req->res.cn10k.compcode = CPT_COMP_NOT_DONE;
- infl_req->qp = qp;
- inst->w0.u64 = 0;
- inst->res_addr = (uint64_t)&infl_req->res;
- inst->w2.u64 = w2;
- inst->w3.u64 = CNXK_CPT_INST_W3(1, infl_req);
-
fc.u64[0] = __atomic_load_n(fc_addr, __ATOMIC_RELAXED);
if (unlikely(fc.s.qsize > fc_thresh)) {
- rte_mempool_put(qp->ca.req_mp, infl_req);
rte_errno = EAGAIN;
- return 0;
+ i = 0;
+ goto put;
}
- if (inst->w2.s.tt == RTE_SCHED_TYPE_ORDERED)
- roc_sso_hws_head_wait(base);
+submit:
+ if (CNXK_TT_FROM_TAG(ws->gw_rdata) == SSO_TT_ORDERED)
+ roc_sso_hws_head_wait(ws->base);
- lmt_arg = ROC_CN10K_CPT_LMT_ARG | (uint64_t)lmt_id;
- roc_lmt_submit_steorl(lmt_arg, qp->lmtline.io_addr);
+ if (i > PKTS_PER_STEORL) {
+ lmt_arg = ROC_CN10K_CPT_LMT_ARG | (PKTS_PER_STEORL - 1) << 12 | (uint64_t)lmt_id;
+ roc_lmt_submit_steorl(lmt_arg, io_addr);
+ lmt_arg = ROC_CN10K_CPT_LMT_ARG | (i - PKTS_PER_STEORL - 1) << 12 |
+ (uint64_t)(lmt_id + PKTS_PER_STEORL);
+ roc_lmt_submit_steorl(lmt_arg, io_addr);
+ } else {
+ lmt_arg = ROC_CN10K_CPT_LMT_ARG | (i - 1) << 12 | (uint64_t)lmt_id;
+ roc_lmt_submit_steorl(lmt_arg, io_addr);
+ }
rte_io_wmb();
- return 1;
+put:
+ if (unlikely(i != nb_ops))
+ rte_mempool_put_bulk(qp->ca.req_mp, (void *)&infl_reqs[i], nb_ops - i);
+
+ return i;
+}
+
+uint16_t __rte_hot
+cn10k_cpt_crypto_adapter_enqueue(void *ws, struct rte_event ev[], uint16_t nb_events)
+{
+ struct rte_crypto_op *ops[PKTS_PER_LOOP], *op;
+ struct cnxk_cpt_qp *qp, *curr_qp = NULL;
+ uint64_t w2s[PKTS_PER_LOOP], w2;
+ uint16_t submitted, count = 0;
+ int ret, i, ops_len = 0;
+
+ for (i = 0; i < nb_events; i++) {
+ op = ev[i].event_ptr;
+ ret = cn10k_ca_meta_info_extract(op, &qp, &w2);
+ if (unlikely(ret)) {
+ rte_errno = EINVAL;
+ return count;
+ }
+
+ if (qp != curr_qp) {
+ if (ops_len) {
+ submitted = ca_lmtst_burst_submit(ws, w2s, curr_qp, ops, ops_len);
+ count += submitted;
+ if (unlikely(submitted != ops_len))
+ return count;
+ ops_len = 0;
+ }
+ curr_qp = qp;
+ }
+ w2s[ops_len] = w2;
+ ops[ops_len] = op;
+ if (++ops_len == PKTS_PER_LOOP) {
+ submitted = ca_lmtst_burst_submit(ws, w2s, curr_qp, ops, ops_len);
+ count += submitted;
+ if (unlikely(submitted != ops_len))
+ return count;
+ ops_len = 0;
+ }
+ }
+ if (ops_len)
+ count += ca_lmtst_burst_submit(ws, w2s, curr_qp, ops, ops_len);
+ return count;
}
static inline void
-cn10k_cpt_sec_post_process(struct rte_crypto_op *cop,
- struct cpt_cn10k_res_s *res)
+cn10k_cpt_sec_post_process(struct rte_crypto_op *cop, struct cpt_cn10k_res_s *res)
{
struct rte_mbuf *mbuf = cop->sym->m_src;
const uint16_t m_len = res->rlen;
diff --git a/drivers/crypto/cnxk/cn10k_cryptodev_ops.h b/drivers/crypto/cnxk/cn10k_cryptodev_ops.h
index 1ad4c16873..628d6a567c 100644
--- a/drivers/crypto/cnxk/cn10k_cryptodev_ops.h
+++ b/drivers/crypto/cnxk/cn10k_cryptodev_ops.h
@@ -5,16 +5,17 @@
#ifndef _CN10K_CRYPTODEV_OPS_H_
#define _CN10K_CRYPTODEV_OPS_H_
-#include <rte_cryptodev.h>
#include <cryptodev_pmd.h>
+#include <rte_cryptodev.h>
+#include <rte_eventdev.h>
extern struct rte_cryptodev_ops cn10k_cpt_ops;
void cn10k_cpt_set_enqdeq_fns(struct rte_cryptodev *dev);
__rte_internal
-uint16_t cn10k_cpt_crypto_adapter_enqueue(uintptr_t base,
- struct rte_crypto_op *op);
+uint16_t __rte_hot cn10k_cpt_crypto_adapter_enqueue(void *ws, struct rte_event ev[],
+ uint16_t nb_events);
__rte_internal
uintptr_t cn10k_cpt_crypto_adapter_dequeue(uintptr_t get_work1);
diff --git a/drivers/crypto/cnxk/meson.build b/drivers/crypto/cnxk/meson.build
index 23a1cc3aac..952554ac12 100644
--- a/drivers/crypto/cnxk/meson.build
+++ b/drivers/crypto/cnxk/meson.build
@@ -24,7 +24,7 @@ sources = files(
deps += ['bus_pci', 'common_cnxk', 'security', 'eventdev']
-includes += include_directories('../../../lib/net')
+includes += include_directories('../../../lib/net', '../../event/cnxk')
if get_option('buildtype').contains('debug')
cflags += [ '-DLA_IPSEC_DEBUG' ]
diff --git a/drivers/event/cnxk/cn10k_eventdev.c b/drivers/event/cnxk/cn10k_eventdev.c
index e74ec57382..3651af9ca8 100644
--- a/drivers/event/cnxk/cn10k_eventdev.c
+++ b/drivers/event/cnxk/cn10k_eventdev.c
@@ -454,7 +454,7 @@ cn10k_sso_fp_fns_set(struct rte_eventdev *event_dev)
sso_hws_deq_tmo_ca_burst);
}
}
- event_dev->ca_enqueue = cn10k_sso_hws_ca_enq;
+ event_dev->ca_enqueue = cn10k_cpt_crypto_adapter_enqueue;
if (dev->tx_offloads & NIX_TX_MULTI_SEG_F)
CN10K_SET_EVDEV_ENQ_OP(dev, event_dev->txa_enqueue,
diff --git a/drivers/event/cnxk/cn10k_worker.c b/drivers/event/cnxk/cn10k_worker.c
index f953e19dd0..4581c41233 100644
--- a/drivers/event/cnxk/cn10k_worker.c
+++ b/drivers/event/cnxk/cn10k_worker.c
@@ -64,13 +64,3 @@ cn10k_sso_hws_enq_fwd_burst(void *port, const struct rte_event ev[],
return 1;
}
-
-uint16_t __rte_hot
-cn10k_sso_hws_ca_enq(void *port, struct rte_event ev[], uint16_t nb_events)
-{
- struct cn10k_sso_hws *ws = port;
-
- RTE_SET_USED(nb_events);
-
- return cn10k_cpt_crypto_adapter_enqueue(ws->base, ev->event_ptr);
-}
diff --git a/drivers/event/cnxk/cn10k_worker.h b/drivers/event/cnxk/cn10k_worker.h
index 0915f404e0..65bb08c0a1 100644
--- a/drivers/event/cnxk/cn10k_worker.h
+++ b/drivers/event/cnxk/cn10k_worker.h
@@ -353,8 +353,6 @@ uint16_t __rte_hot cn10k_sso_hws_enq_new_burst(void *port,
uint16_t __rte_hot cn10k_sso_hws_enq_fwd_burst(void *port,
const struct rte_event ev[],
uint16_t nb_events);
-uint16_t __rte_hot cn10k_sso_hws_ca_enq(void *port, struct rte_event ev[],
- uint16_t nb_events);
#define R(name, flags) \
uint16_t __rte_hot cn10k_sso_hws_deq_##name( \
--
2.25.1
next prev parent reply other threads:[~2022-08-04 10:00 UTC|newest]
Thread overview: 36+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-06-22 1:38 [PATCH v2] app/test: add event inline security tests Volodymyr Fialko
2022-06-22 11:33 ` [PATCH v3] " Volodymyr Fialko
2022-06-22 16:32 ` Anoob Joseph
2022-06-28 8:29 ` Akhil Goyal
2022-06-28 12:09 ` [PATCH] doc: announce change in crypto adapter queue add Volodymyr Fialko
2022-06-28 12:40 ` Akhil Goyal
2022-07-11 14:56 ` Jerin Jacob
2022-07-12 5:31 ` [EXT] " Akhil Goyal
2022-07-13 6:49 ` Gujjar, Abhinandan S
2022-07-14 9:04 ` Hemant Agrawal
2022-07-17 11:32 ` Thomas Monjalon
2022-08-04 9:59 ` [PATCH 0/3] Vector support for event crypto adapter Volodymyr Fialko
2022-08-04 9:59 ` [PATCH 1/3] eventdev: introduce event cryptodev vector type Volodymyr Fialko
2022-09-21 18:32 ` Akhil Goyal
2022-09-22 4:53 ` Gujjar, Abhinandan S
2022-09-24 8:43 ` Gujjar, Abhinandan S
2022-09-26 11:02 ` Volodymyr Fialko
2022-09-27 9:05 ` Gujjar, Abhinandan S
2022-09-27 9:24 ` Volodymyr Fialko
2022-09-27 9:38 ` Gujjar, Abhinandan S
2022-09-27 13:26 ` Jerin Jacob
2022-09-28 14:43 ` Gujjar, Abhinandan S
2022-09-28 16:13 ` Jerin Jacob
2022-08-04 9:59 ` Volodymyr Fialko [this message]
2022-08-04 9:59 ` [PATCH 3/3] crypto/cnxk: add vectorization for event crypto Volodymyr Fialko
2022-09-26 11:36 ` [PATCH v2 0/3] Vector support for event crypto adapter Volodymyr Fialko
2022-09-26 11:36 ` [PATCH v2 1/3] eventdev: introduce event cryptodev vector type Volodymyr Fialko
2022-09-26 11:36 ` [PATCH v2 2/3] crypto/cnxk: add burst enqueue for event crypto Volodymyr Fialko
2022-09-26 11:36 ` [PATCH v2 3/3] crypto/cnxk: add vectorization " Volodymyr Fialko
2022-10-01 0:42 ` [PATCH v3 0/2] Vector support for event crypto adapter Volodymyr Fialko
2022-10-01 0:42 ` [PATCH v3 1/2] eventdev: introduce event cryptodev vector type Volodymyr Fialko
2022-10-01 0:42 ` [PATCH v3 2/2] crypto/cnxk: add vectorization for event crypto Volodymyr Fialko
2022-10-01 3:42 ` [PATCH v3 0/2] Vector support for event crypto adapter Akhil Goyal
2022-10-01 8:00 ` Gujjar, Abhinandan S
2022-10-01 8:47 ` Akhil Goyal
2022-10-02 1:56 ` Gujjar, Abhinandan S
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20220804095907.97895-3-vfialko@marvell.com \
--to=vfialko@marvell.com \
--cc=adwivedi@marvell.com \
--cc=anoobj@marvell.com \
--cc=dev@dpdk.org \
--cc=gakhil@marvell.com \
--cc=jerinj@marvell.com \
--cc=ktejasree@marvell.com \
--cc=pbhagavatula@marvell.com \
--cc=sthotton@marvell.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).