From: Tejasree Kondoj <ktejasree@marvell.com>
To: Akhil Goyal <gakhil@marvell.com>
Cc: Anoob Joseph <anoobj@marvell.com>,
Gowrishankar Muthukrishnan <gmuthukrishn@marvell.com>,
<dev@dpdk.org>
Subject: [PATCH v2 09/11] crypto/cnxk: support cn10k IPsec SG mode
Date: Fri, 24 Feb 2023 15:10:12 +0530 [thread overview]
Message-ID: <20230224094014.3246764-10-ktejasree@marvell.com> (raw)
In-Reply-To: <20230224094014.3246764-1-ktejasree@marvell.com>
Adding support for scatter-gather mode in 103XX and
106XX lookaside IPsec.
Signed-off-by: Tejasree Kondoj <ktejasree@marvell.com>
---
drivers/crypto/cnxk/cn10k_cryptodev_ops.c | 21 +-
drivers/crypto/cnxk/cn10k_ipsec_la_ops.h | 222 ++++++++++++++++++++--
drivers/crypto/cnxk/cnxk_sg.h | 23 +++
3 files changed, 239 insertions(+), 27 deletions(-)
diff --git a/drivers/crypto/cnxk/cn10k_cryptodev_ops.c b/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
index 9f6fd4e411..e405a2ad9f 100644
--- a/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
@@ -77,8 +77,8 @@ cn10k_cpt_sym_temp_sess_create(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op)
}
static __rte_always_inline int __rte_hot
-cpt_sec_inst_fill(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,
- struct cn10k_sec_session *sess, struct cpt_inst_s *inst)
+cpt_sec_inst_fill(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op, struct cn10k_sec_session *sess,
+ struct cpt_inst_s *inst, struct cpt_inflight_req *infl_req, const bool is_sg_ver2)
{
struct rte_crypto_sym_op *sym_op = op->sym;
int ret;
@@ -88,15 +88,11 @@ cpt_sec_inst_fill(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,
return -ENOTSUP;
}
- if (unlikely(!rte_pktmbuf_is_contiguous(sym_op->m_src))) {
- plt_dp_err("Scatter Gather mode is not supported");
- return -ENOTSUP;
- }
-
if (sess->is_outbound)
- ret = process_outb_sa(&qp->lf, op, sess, inst);
+ ret = process_outb_sa(&qp->lf, op, sess, &qp->meta_info, infl_req, inst,
+ is_sg_ver2);
else
- ret = process_inb_sa(op, sess, inst);
+ ret = process_inb_sa(op, sess, inst, &qp->meta_info, infl_req, is_sg_ver2);
return ret;
}
@@ -129,7 +125,7 @@ cn10k_cpt_fill_inst(struct cnxk_cpt_qp *qp, struct rte_crypto_op *ops[], struct
if (op->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
sec_sess = (struct cn10k_sec_session *)sym_op->session;
- ret = cpt_sec_inst_fill(qp, op, sec_sess, &inst[0]);
+ ret = cpt_sec_inst_fill(qp, op, sec_sess, &inst[0], infl_req, is_sg_ver2);
if (unlikely(ret))
return 0;
w7 = sec_sess->inst.w7;
@@ -827,7 +823,10 @@ cn10k_cpt_sec_post_process(struct rte_crypto_op *cop, struct cpt_cn10k_res_s *re
cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
return;
}
- mbuf->data_len = m_len;
+
+ if (mbuf->next == NULL)
+ mbuf->data_len = m_len;
+
mbuf->pkt_len = m_len;
}
diff --git a/drivers/crypto/cnxk/cn10k_ipsec_la_ops.h b/drivers/crypto/cnxk/cn10k_ipsec_la_ops.h
index f2761a55a5..8e208eb2ca 100644
--- a/drivers/crypto/cnxk/cn10k_ipsec_la_ops.h
+++ b/drivers/crypto/cnxk/cn10k_ipsec_la_ops.h
@@ -8,9 +8,13 @@
#include <rte_crypto_sym.h>
#include <rte_security.h>
+#include "roc_ie.h"
+
#include "cn10k_cryptodev.h"
#include "cn10k_ipsec.h"
#include "cnxk_cryptodev.h"
+#include "cnxk_cryptodev_ops.h"
+#include "cnxk_sg.h"
static inline void
ipsec_po_sa_iv_set(struct cn10k_sec_session *sess, struct rte_crypto_op *cop)
@@ -44,18 +48,14 @@ ipsec_po_sa_aes_gcm_iv_set(struct cn10k_sec_session *sess, struct rte_crypto_op
static __rte_always_inline int
process_outb_sa(struct roc_cpt_lf *lf, struct rte_crypto_op *cop, struct cn10k_sec_session *sess,
- struct cpt_inst_s *inst)
+ struct cpt_qp_meta_info *m_info, struct cpt_inflight_req *infl_req,
+ struct cpt_inst_s *inst, const bool is_sg_ver2)
{
struct rte_crypto_sym_op *sym_op = cop->sym;
struct rte_mbuf *m_src = sym_op->m_src;
uint64_t inst_w4_u64 = sess->inst.w4;
uint64_t dptr;
- if (unlikely(rte_pktmbuf_tailroom(m_src) < sess->max_extended_len)) {
- plt_dp_err("Not enough tail room");
- return -ENOMEM;
- }
-
RTE_SET_USED(lf);
#ifdef LA_IPSEC_DEBUG
@@ -79,27 +79,217 @@ process_outb_sa(struct roc_cpt_lf *lf, struct rte_crypto_op *cop, struct cn10k_s
if (m_src->ol_flags & RTE_MBUF_F_TX_L4_MASK)
inst_w4_u64 &= ~BIT_ULL(32);
- /* Prepare CPT instruction */
- inst->w4.u64 = inst_w4_u64 | rte_pktmbuf_pkt_len(m_src);
- dptr = rte_pktmbuf_mtod(m_src, uint64_t);
- inst->dptr = dptr;
+ if (likely(m_src->next == NULL)) {
+ if (unlikely(rte_pktmbuf_tailroom(m_src) < sess->max_extended_len)) {
+ plt_dp_err("Not enough tail room");
+ return -ENOMEM;
+ }
+
+ /* Prepare CPT instruction */
+ inst->w4.u64 = inst_w4_u64 | rte_pktmbuf_pkt_len(m_src);
+ dptr = rte_pktmbuf_mtod(m_src, uint64_t);
+ inst->dptr = dptr;
+ } else if (is_sg_ver2 == false) {
+ struct roc_sglist_comp *scatter_comp, *gather_comp;
+ uint32_t g_size_bytes, s_size_bytes;
+ struct rte_mbuf *last_seg;
+ uint8_t *in_buffer;
+ uint32_t dlen;
+ void *m_data;
+ int i;
+
+ last_seg = rte_pktmbuf_lastseg(m_src);
+
+ if (unlikely(rte_pktmbuf_tailroom(last_seg) < sess->max_extended_len)) {
+ plt_dp_err("Not enough tail room (required: %d, available: %d)",
+ sess->max_extended_len, rte_pktmbuf_tailroom(last_seg));
+ return -ENOMEM;
+ }
+
+ m_data = alloc_op_meta(NULL, m_info->mlen, m_info->pool, infl_req);
+ if (unlikely(m_data == NULL)) {
+ plt_dp_err("Error allocating meta buffer for request");
+ return -ENOMEM;
+ }
+
+ in_buffer = m_data;
+
+ ((uint16_t *)in_buffer)[0] = 0;
+ ((uint16_t *)in_buffer)[1] = 0;
+
+ /* Input Gather List */
+ i = 0;
+ gather_comp = (struct roc_sglist_comp *)((uint8_t *)m_data + 8);
+
+ i = fill_ipsec_sg_comp_from_pkt(gather_comp, i, m_src);
+ ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
+
+ g_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);
+
+ /* Output Scatter List */
+ last_seg->data_len += sess->max_extended_len;
+
+ i = 0;
+ scatter_comp = (struct roc_sglist_comp *)((uint8_t *)gather_comp + g_size_bytes);
+
+ i = fill_ipsec_sg_comp_from_pkt(scatter_comp, i, m_src);
+ ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
+
+ s_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);
+
+ dlen = g_size_bytes + s_size_bytes + ROC_SG_LIST_HDR_SIZE;
+
+ inst->dptr = (uint64_t)in_buffer;
+
+ inst->w4.u64 = sess->inst.w4 | dlen;
+ inst->w4.s.opcode_major |= (uint64_t)ROC_DMA_MODE_SG;
+ } else {
+ struct roc_sg2list_comp *scatter_comp, *gather_comp;
+ union cpt_inst_w5 cpt_inst_w5;
+ union cpt_inst_w6 cpt_inst_w6;
+ struct rte_mbuf *last_seg;
+ uint32_t g_size_bytes;
+ void *m_data;
+ int i;
+
+ last_seg = rte_pktmbuf_lastseg(m_src);
+
+ if (unlikely(rte_pktmbuf_tailroom(last_seg) < sess->max_extended_len)) {
+ plt_dp_err("Not enough tail room (required: %d, available: %d)",
+ sess->max_extended_len, rte_pktmbuf_tailroom(last_seg));
+ return -ENOMEM;
+ }
+
+ m_data = alloc_op_meta(NULL, m_info->mlen, m_info->pool, infl_req);
+ if (unlikely(m_data == NULL)) {
+ plt_dp_err("Error allocating meta buffer for request");
+ return -ENOMEM;
+ }
+
+ /* Input Gather List */
+ i = 0;
+ gather_comp = (struct roc_sg2list_comp *)((uint8_t *)m_data);
+
+ i = fill_ipsec_sg2_comp_from_pkt(gather_comp, i, m_src);
+
+ cpt_inst_w5.s.gather_sz = ((i + 2) / 3);
+ g_size_bytes = ((i + 2) / 3) * sizeof(struct roc_sg2list_comp);
+
+ /* Output Scatter List */
+ last_seg->data_len += sess->max_extended_len;
+
+ i = 0;
+ scatter_comp = (struct roc_sg2list_comp *)((uint8_t *)gather_comp + g_size_bytes);
+
+ i = fill_ipsec_sg2_comp_from_pkt(scatter_comp, i, m_src);
+
+ cpt_inst_w6.s.scatter_sz = ((i + 2) / 3);
+
+ cpt_inst_w5.s.dptr = (uint64_t)gather_comp;
+ cpt_inst_w6.s.rptr = (uint64_t)scatter_comp;
+
+ inst->w5.u64 = cpt_inst_w5.u64;
+ inst->w6.u64 = cpt_inst_w6.u64;
+ inst->w4.u64 = sess->inst.w4 | rte_pktmbuf_pkt_len(m_src);
+ inst->w4.s.opcode_major &= (~(ROC_IE_OT_INPLACE_BIT));
+ }
return 0;
}
static __rte_always_inline int
-process_inb_sa(struct rte_crypto_op *cop, struct cn10k_sec_session *sess, struct cpt_inst_s *inst)
+process_inb_sa(struct rte_crypto_op *cop, struct cn10k_sec_session *sess, struct cpt_inst_s *inst,
+ struct cpt_qp_meta_info *m_info, struct cpt_inflight_req *infl_req,
+ const bool is_sg_ver2)
{
struct rte_crypto_sym_op *sym_op = cop->sym;
struct rte_mbuf *m_src = sym_op->m_src;
uint64_t dptr;
- /* Prepare CPT instruction */
- inst->w4.u64 = sess->inst.w4 | rte_pktmbuf_pkt_len(m_src);
- dptr = rte_pktmbuf_mtod(m_src, uint64_t);
- inst->dptr = dptr;
- m_src->ol_flags |= (uint64_t)sess->ip_csum;
+ if (likely(m_src->next == NULL)) {
+ /* Prepare CPT instruction */
+ inst->w4.u64 = sess->inst.w4 | rte_pktmbuf_pkt_len(m_src);
+ dptr = rte_pktmbuf_mtod(m_src, uint64_t);
+ inst->dptr = dptr;
+ m_src->ol_flags |= (uint64_t)sess->ip_csum;
+ } else if (is_sg_ver2 == false) {
+ struct roc_sglist_comp *scatter_comp, *gather_comp;
+ uint32_t g_size_bytes, s_size_bytes;
+ uint8_t *in_buffer;
+ uint32_t dlen;
+ void *m_data;
+ int i;
+
+ m_data = alloc_op_meta(NULL, m_info->mlen, m_info->pool, infl_req);
+ if (unlikely(m_data == NULL)) {
+ plt_dp_err("Error allocating meta buffer for request");
+ return -ENOMEM;
+ }
+
+ in_buffer = m_data;
+
+ ((uint16_t *)in_buffer)[0] = 0;
+ ((uint16_t *)in_buffer)[1] = 0;
+
+ /* Input Gather List */
+ i = 0;
+ gather_comp = (struct roc_sglist_comp *)((uint8_t *)m_data + 8);
+ i = fill_ipsec_sg_comp_from_pkt(gather_comp, i, m_src);
+ ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
+
+ g_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);
+ /* Output Scatter List */
+ i = 0;
+ scatter_comp = (struct roc_sglist_comp *)((uint8_t *)gather_comp + g_size_bytes);
+ i = fill_ipsec_sg_comp_from_pkt(scatter_comp, i, m_src);
+ ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
+
+ s_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);
+
+ dlen = g_size_bytes + s_size_bytes + ROC_SG_LIST_HDR_SIZE;
+
+ inst->dptr = (uint64_t)in_buffer;
+ inst->w4.u64 = sess->inst.w4 | dlen;
+ inst->w4.s.opcode_major |= (uint64_t)ROC_DMA_MODE_SG;
+ } else {
+ struct roc_sg2list_comp *scatter_comp, *gather_comp;
+ union cpt_inst_w5 cpt_inst_w5;
+ union cpt_inst_w6 cpt_inst_w6;
+ uint32_t g_size_bytes;
+ void *m_data;
+ int i;
+
+ m_data = alloc_op_meta(NULL, m_info->mlen, m_info->pool, infl_req);
+ if (unlikely(m_data == NULL)) {
+ plt_dp_err("Error allocating meta buffer for request");
+ return -ENOMEM;
+ }
+
+ /* Input Gather List */
+ i = 0;
+ gather_comp = (struct roc_sg2list_comp *)((uint8_t *)m_data);
+
+ i = fill_ipsec_sg2_comp_from_pkt(gather_comp, i, m_src);
+
+ cpt_inst_w5.s.gather_sz = ((i + 2) / 3);
+ g_size_bytes = ((i + 2) / 3) * sizeof(struct roc_sg2list_comp);
+
+ /* Output Scatter List */
+ i = 0;
+ scatter_comp = (struct roc_sg2list_comp *)((uint8_t *)gather_comp + g_size_bytes);
+ i = fill_ipsec_sg2_comp_from_pkt(scatter_comp, i, m_src);
+
+ cpt_inst_w6.s.scatter_sz = ((i + 2) / 3);
+
+ cpt_inst_w5.s.dptr = (uint64_t)gather_comp;
+ cpt_inst_w6.s.rptr = (uint64_t)scatter_comp;
+
+ inst->w5.u64 = cpt_inst_w5.u64;
+ inst->w6.u64 = cpt_inst_w6.u64;
+ inst->w4.u64 = sess->inst.w4 | rte_pktmbuf_pkt_len(m_src);
+ inst->w4.s.opcode_major &= (~(ROC_IE_OT_INPLACE_BIT));
+ }
return 0;
}
diff --git a/drivers/crypto/cnxk/cnxk_sg.h b/drivers/crypto/cnxk/cnxk_sg.h
index ead2886e99..65244199bd 100644
--- a/drivers/crypto/cnxk/cnxk_sg.h
+++ b/drivers/crypto/cnxk/cnxk_sg.h
@@ -6,6 +6,7 @@
#define _CNXK_SG_H_
#include "roc_cpt_sg.h"
+#include "roc_se.h"
static __rte_always_inline uint32_t
fill_sg_comp(struct roc_sglist_comp *list, uint32_t i, phys_addr_t dma_addr, uint32_t size)
@@ -148,6 +149,28 @@ fill_ipsec_sg_comp_from_pkt(struct roc_sglist_comp *list, uint32_t i, struct rte
return i;
}
+static __rte_always_inline uint32_t
+fill_ipsec_sg2_comp_from_pkt(struct roc_sg2list_comp *list, uint32_t i, struct rte_mbuf *pkt)
+{
+ uint32_t buf_sz;
+ void *vaddr;
+
+ while (unlikely(pkt != NULL)) {
+ struct roc_sg2list_comp *to = &list[i / 3];
+ buf_sz = pkt->data_len;
+ vaddr = rte_pktmbuf_mtod(pkt, void *);
+
+ to->u.s.len[i % 3] = buf_sz;
+ to->ptr[i % 3] = (uint64_t)vaddr;
+ to->u.s.valid_segs = (i % 3) + 1;
+
+ pkt = pkt->next;
+ i++;
+ }
+
+ return i;
+}
+
static __rte_always_inline uint32_t
fill_sg2_comp(struct roc_sg2list_comp *list, uint32_t i, phys_addr_t dma_addr, uint32_t size)
{
--
2.25.1
next prev parent reply other threads:[~2023-02-24 9:41 UTC|newest]
Thread overview: 15+ messages / expand[flat|nested] mbox.gz Atom feed top
2023-02-24 9:40 [PATCH v2 00/11] fixes and improvements to cnxk crypto PMD Tejasree Kondoj
2023-02-24 9:40 ` [PATCH v2 01/11] common/cnxk: fix incorrect auth key length Tejasree Kondoj
2023-02-24 9:40 ` [PATCH v2 02/11] crypto/cnxk: make sg version check const Tejasree Kondoj
2023-02-24 9:40 ` [PATCH v2 03/11] crypto/cnxk: use version field directly Tejasree Kondoj
2023-02-24 9:40 ` [PATCH v2 04/11] crypto/cnxk: use direct mode for zero aad length Tejasree Kondoj
2023-02-24 9:40 ` [PATCH v2 05/11] crypto/cnxk: set ctx for AE Tejasree Kondoj
2023-02-27 17:38 ` Akhil Goyal
2023-02-28 5:40 ` Tejasree Kondoj
2023-02-24 9:40 ` [PATCH v2 06/11] common/cnxk: ensure flush inval completion with CSR read Tejasree Kondoj
2023-02-24 9:40 ` [PATCH v2 07/11] common/cnxk: add errata function for CPT set ctx Tejasree Kondoj
2023-02-24 9:40 ` [PATCH v2 08/11] common/cnxk: replace CPT revision check with caps Tejasree Kondoj
2023-02-24 9:40 ` Tejasree Kondoj [this message]
2023-02-24 9:40 ` [PATCH v2 10/11] crypto/cnxk: fix order of ECFPM params Tejasree Kondoj
2023-02-24 9:40 ` [PATCH v2 11/11] crypto/cnxk: add model check for pdcp chain Tejasree Kondoj
2023-03-01 12:09 ` [PATCH v2 00/11] fixes and improvements to cnxk crypto PMD Akhil Goyal
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20230224094014.3246764-10-ktejasree@marvell.com \
--to=ktejasree@marvell.com \
--cc=anoobj@marvell.com \
--cc=dev@dpdk.org \
--cc=gakhil@marvell.com \
--cc=gmuthukrishn@marvell.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).