[PATCH v3 16/24] crypto/cnxk: add TLS record datapath handling

[Anoob Joseph <anoobj@marvell.com>]

From: Vidya Sagar Velumuri <vvelumuri@marvell.com>

Add support for TLS record handling in datapath.

Signed-off-by: Anoob Joseph <anoobj@marvell.com>
Signed-off-by: Vidya Sagar Velumuri <vvelumuri@marvell.com>
---
 drivers/crypto/cnxk/cn10k_cryptodev_ops.c |  57 +++-
 drivers/crypto/cnxk/cn10k_cryptodev_sec.c |   7 +
 drivers/crypto/cnxk/cn10k_tls_ops.h       | 322 ++++++++++++++++++++++
 3 files changed, 380 insertions(+), 6 deletions(-)
 create mode 100644 drivers/crypto/cnxk/cn10k_tls_ops.h

diff --git a/drivers/crypto/cnxk/cn10k_cryptodev_ops.c b/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
index 084c8d3a24..843a111b0e 100644
--- a/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
@@ -20,11 +20,14 @@
 #include "roc_sso_dp.h"
 
 #include "cn10k_cryptodev.h"
-#include "cn10k_cryptodev_ops.h"
 #include "cn10k_cryptodev_event_dp.h"
+#include "cn10k_cryptodev_ops.h"
+#include "cn10k_cryptodev_sec.h"
 #include "cn10k_eventdev.h"
 #include "cn10k_ipsec.h"
 #include "cn10k_ipsec_la_ops.h"
+#include "cn10k_tls.h"
+#include "cn10k_tls_ops.h"
 #include "cnxk_ae.h"
 #include "cnxk_cryptodev.h"
 #include "cnxk_cryptodev_ops.h"
@@ -101,6 +104,18 @@ cpt_sec_ipsec_inst_fill(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,
 	return ret;
 }
 
+static __rte_always_inline int __rte_hot
+cpt_sec_tls_inst_fill(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,
+		      struct cn10k_sec_session *sess, struct cpt_inst_s *inst,
+		      struct cpt_inflight_req *infl_req, const bool is_sg_ver2)
+{
+	if (sess->tls.is_write)
+		return process_tls_write(&qp->lf, op, sess, &qp->meta_info, infl_req, inst,
+					 is_sg_ver2);
+	else
+		return process_tls_read(op, sess, &qp->meta_info, infl_req, inst, is_sg_ver2);
+}
+
 static __rte_always_inline int __rte_hot
 cpt_sec_inst_fill(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op, struct cn10k_sec_session *sess,
 		  struct cpt_inst_s *inst, struct cpt_inflight_req *infl_req, const bool is_sg_ver2)
@@ -108,6 +123,8 @@ cpt_sec_inst_fill(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op, struct cn10k
 
 	if (sess->proto == RTE_SECURITY_PROTOCOL_IPSEC)
 		return cpt_sec_ipsec_inst_fill(qp, op, sess, &inst[0], infl_req, is_sg_ver2);
+	else if (sess->proto == RTE_SECURITY_PROTOCOL_TLS_RECORD)
+		return cpt_sec_tls_inst_fill(qp, op, sess, &inst[0], infl_req, is_sg_ver2);
 
 	return 0;
 }
@@ -812,7 +829,7 @@ cn10k_cpt_sg_ver2_crypto_adapter_enqueue(void *ws, struct rte_event ev[], uint16
 }
 
 static inline void
-cn10k_cpt_sec_post_process(struct rte_crypto_op *cop, struct cpt_cn10k_res_s *res)
+cn10k_cpt_ipsec_post_process(struct rte_crypto_op *cop, struct cpt_cn10k_res_s *res)
 {
 	struct rte_mbuf *mbuf = cop->sym->m_src;
 	const uint16_t m_len = res->rlen;
@@ -849,10 +866,38 @@ cn10k_cpt_sec_post_process(struct rte_crypto_op *cop, struct cpt_cn10k_res_s *re
 }
 
 static inline void
-cn10k_cpt_dequeue_post_process(struct cnxk_cpt_qp *qp,
-			       struct rte_crypto_op *cop,
-			       struct cpt_inflight_req *infl_req,
-			       struct cpt_cn10k_res_s *res)
+cn10k_cpt_tls_post_process(struct rte_crypto_op *cop, struct cpt_cn10k_res_s *res)
+{
+	struct rte_mbuf *mbuf = cop->sym->m_src;
+	const uint16_t m_len = res->rlen;
+
+	if (!res->uc_compcode) {
+		if (mbuf->next == NULL)
+			mbuf->data_len = m_len;
+		mbuf->pkt_len = m_len;
+	} else {
+		cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		cop->aux_flags = res->uc_compcode;
+		plt_err("crypto op failed with UC compcode: 0x%x", res->uc_compcode);
+	}
+}
+
+static inline void
+cn10k_cpt_sec_post_process(struct rte_crypto_op *cop, struct cpt_cn10k_res_s *res)
+{
+	struct rte_crypto_sym_op *sym_op = cop->sym;
+	struct cn10k_sec_session *sess;
+
+	sess = sym_op->session;
+	if (sess->proto == RTE_SECURITY_PROTOCOL_IPSEC)
+		cn10k_cpt_ipsec_post_process(cop, res);
+	else if (sess->proto == RTE_SECURITY_PROTOCOL_TLS_RECORD)
+		cn10k_cpt_tls_post_process(cop, res);
+}
+
+static inline void
+cn10k_cpt_dequeue_post_process(struct cnxk_cpt_qp *qp, struct rte_crypto_op *cop,
+			       struct cpt_inflight_req *infl_req, struct cpt_cn10k_res_s *res)
 {
 	const uint8_t uc_compcode = res->uc_compcode;
 	const uint8_t compcode = res->compcode;
diff --git a/drivers/crypto/cnxk/cn10k_cryptodev_sec.c b/drivers/crypto/cnxk/cn10k_cryptodev_sec.c
index 12e53f18db..cb013986c4 100644
--- a/drivers/crypto/cnxk/cn10k_cryptodev_sec.c
+++ b/drivers/crypto/cnxk/cn10k_cryptodev_sec.c
@@ -32,6 +32,10 @@ cn10k_sec_session_create(void *dev, struct rte_security_session_conf *conf,
 		return cn10k_ipsec_session_create(vf, qp, &conf->ipsec, conf->crypto_xform, sess);
 	}
 
+	if (conf->protocol == RTE_SECURITY_PROTOCOL_TLS_RECORD)
+		return cn10k_tls_record_session_create(vf, qp, &conf->tls_record,
+						       conf->crypto_xform, sess);
+
 	return -ENOTSUP;
 }
 
@@ -54,6 +58,9 @@ cn10k_sec_session_destroy(void *dev, struct rte_security_session *sec_sess)
 	if (cn10k_sec_sess->proto == RTE_SECURITY_PROTOCOL_IPSEC)
 		return cn10k_sec_ipsec_session_destroy(qp, cn10k_sec_sess);
 
+	if (cn10k_sec_sess->proto == RTE_SECURITY_PROTOCOL_TLS_RECORD)
+		return cn10k_sec_tls_session_destroy(qp, cn10k_sec_sess);
+
 	return -EINVAL;
 }
 
diff --git a/drivers/crypto/cnxk/cn10k_tls_ops.h b/drivers/crypto/cnxk/cn10k_tls_ops.h
new file mode 100644
index 0000000000..7c8ac14ab2
--- /dev/null
+++ b/drivers/crypto/cnxk/cn10k_tls_ops.h
@@ -0,0 +1,322 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2024 Marvell.
+ */
+
+#ifndef __CN10K_TLS_OPS_H__
+#define __CN10K_TLS_OPS_H__
+
+#include <rte_crypto_sym.h>
+#include <rte_security.h>
+
+#include "roc_ie.h"
+
+#include "cn10k_cryptodev.h"
+#include "cn10k_cryptodev_sec.h"
+#include "cnxk_cryptodev.h"
+#include "cnxk_cryptodev_ops.h"
+#include "cnxk_sg.h"
+
+static __rte_always_inline int
+process_tls_write(struct roc_cpt_lf *lf, struct rte_crypto_op *cop, struct cn10k_sec_session *sess,
+		  struct cpt_qp_meta_info *m_info, struct cpt_inflight_req *infl_req,
+		  struct cpt_inst_s *inst, const bool is_sg_ver2)
+{
+	struct rte_crypto_sym_op *sym_op = cop->sym;
+#ifdef LA_IPSEC_DEBUG
+	struct roc_ie_ot_tls_write_sa *write_sa;
+#endif
+	struct rte_mbuf *m_src = sym_op->m_src;
+	struct rte_mbuf *last_seg;
+	union cpt_inst_w4 w4;
+	void *m_data = NULL;
+	uint8_t *in_buffer;
+
+#ifdef LA_IPSEC_DEBUG
+	write_sa = &sess->tls_rec.write_sa;
+	if (write_sa->w2.s.iv_at_cptr == ROC_IE_OT_TLS_IV_SRC_FROM_SA) {
+
+		uint8_t *iv = PLT_PTR_ADD(write_sa->cipher_key, 32);
+
+		if (write_sa->w2.s.cipher_select == ROC_IE_OT_TLS_CIPHER_AES_GCM) {
+			uint32_t *tmp;
+
+			/* For GCM, the IV and salt format will be like below:
+			 * iv[0-3]: lower bytes of IV in BE format.
+			 * iv[4-7]: salt / nonce.
+			 * iv[12-15]: upper bytes of IV in BE format.
+			 */
+			memcpy(iv, rte_crypto_op_ctod_offset(cop, uint8_t *, sess->iv_offset), 4);
+			tmp = (uint32_t *)iv;
+			*tmp = rte_be_to_cpu_32(*tmp);
+
+			memcpy(iv + 12,
+			       rte_crypto_op_ctod_offset(cop, uint8_t *, sess->iv_offset + 4), 4);
+			tmp = (uint32_t *)(iv + 12);
+			*tmp = rte_be_to_cpu_32(*tmp);
+		} else if (write_sa->w2.s.cipher_select == ROC_IE_OT_TLS_CIPHER_AES_CBC) {
+			uint64_t *tmp;
+
+			memcpy(iv, rte_crypto_op_ctod_offset(cop, uint8_t *, sess->iv_offset), 16);
+			tmp = (uint64_t *)iv;
+			*tmp = rte_be_to_cpu_64(*tmp);
+			tmp = (uint64_t *)(iv + 8);
+			*tmp = rte_be_to_cpu_64(*tmp);
+		} else if (write_sa->w2.s.cipher_select == ROC_IE_OT_TLS_CIPHER_3DES) {
+			uint64_t *tmp;
+
+			memcpy(iv, rte_crypto_op_ctod_offset(cop, uint8_t *, sess->iv_offset), 8);
+			tmp = (uint64_t *)iv;
+			*tmp = rte_be_to_cpu_64(*tmp);
+		}
+
+		/* Trigger CTX reload to fetch new data from DRAM */
+		roc_cpt_lf_ctx_reload(lf, write_sa);
+		rte_delay_ms(1);
+	}
+#else
+	RTE_SET_USED(lf);
+#endif
+	/* Single buffer direct mode */
+	if (likely(m_src->next == NULL)) {
+		void *vaddr;
+
+		if (unlikely(rte_pktmbuf_tailroom(m_src) < sess->max_extended_len)) {
+			plt_dp_err("Not enough tail room");
+			return -ENOMEM;
+		}
+
+		vaddr = rte_pktmbuf_mtod(m_src, void *);
+		inst->dptr = (uint64_t)vaddr;
+		inst->rptr = (uint64_t)vaddr;
+
+		w4.u64 = sess->inst.w4;
+		w4.s.param1 = m_src->data_len;
+		w4.s.dlen = m_src->data_len;
+
+		w4.s.param2 = cop->param1.tls_record.content_type;
+		w4.s.opcode_minor = sess->tls.enable_padding * cop->aux_flags * 8;
+
+		inst->w4.u64 = w4.u64;
+	} else if (is_sg_ver2 == false) {
+		struct roc_sglist_comp *scatter_comp, *gather_comp;
+		uint32_t g_size_bytes, s_size_bytes;
+		uint32_t dlen;
+		int i;
+
+		last_seg = rte_pktmbuf_lastseg(m_src);
+
+		if (unlikely(rte_pktmbuf_tailroom(last_seg) < sess->max_extended_len)) {
+			plt_dp_err("Not enough tail room (required: %d, available: %d)",
+				   sess->max_extended_len, rte_pktmbuf_tailroom(last_seg));
+			return -ENOMEM;
+		}
+
+		m_data = alloc_op_meta(NULL, m_info->mlen, m_info->pool, infl_req);
+		if (unlikely(m_data == NULL)) {
+			plt_dp_err("Error allocating meta buffer for request");
+			return -ENOMEM;
+		}
+
+		in_buffer = (uint8_t *)m_data;
+		((uint16_t *)in_buffer)[0] = 0;
+		((uint16_t *)in_buffer)[1] = 0;
+
+		/* Input Gather List */
+		i = 0;
+		gather_comp = (struct roc_sglist_comp *)((uint8_t *)in_buffer + 8);
+
+		i = fill_sg_comp_from_pkt(gather_comp, i, m_src);
+		((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
+
+		g_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);
+
+		i = 0;
+		scatter_comp = (struct roc_sglist_comp *)((uint8_t *)gather_comp + g_size_bytes);
+
+		i = fill_sg_comp_from_pkt(scatter_comp, i, m_src);
+		((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
+
+		s_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);
+
+		dlen = g_size_bytes + s_size_bytes + ROC_SG_LIST_HDR_SIZE;
+
+		inst->dptr = (uint64_t)in_buffer;
+		inst->rptr = (uint64_t)in_buffer;
+
+		w4.u64 = sess->inst.w4;
+		w4.s.dlen = dlen;
+		w4.s.param1 = rte_pktmbuf_pkt_len(m_src);
+		w4.s.param2 = cop->param1.tls_record.content_type;
+		w4.s.opcode_major |= (uint64_t)ROC_DMA_MODE_SG;
+		w4.s.opcode_minor = sess->tls.enable_padding * cop->aux_flags * 8;
+
+		/* Output Scatter List */
+		last_seg->data_len += sess->max_extended_len;
+		inst->w4.u64 = w4.u64;
+	} else {
+		struct roc_sg2list_comp *scatter_comp, *gather_comp;
+		union cpt_inst_w5 cpt_inst_w5;
+		union cpt_inst_w6 cpt_inst_w6;
+		uint32_t g_size_bytes;
+		int i;
+
+		last_seg = rte_pktmbuf_lastseg(m_src);
+
+		if (unlikely(rte_pktmbuf_tailroom(last_seg) < sess->max_extended_len)) {
+			plt_dp_err("Not enough tail room (required: %d, available: %d)",
+				   sess->max_extended_len, rte_pktmbuf_tailroom(last_seg));
+			return -ENOMEM;
+		}
+
+		m_data = alloc_op_meta(NULL, m_info->mlen, m_info->pool, infl_req);
+		if (unlikely(m_data == NULL)) {
+			plt_dp_err("Error allocating meta buffer for request");
+			return -ENOMEM;
+		}
+
+		in_buffer = (uint8_t *)m_data;
+		/* Input Gather List */
+		i = 0;
+		gather_comp = (struct roc_sg2list_comp *)((uint8_t *)in_buffer);
+		i = fill_sg2_comp_from_pkt(gather_comp, i, m_src);
+
+		cpt_inst_w5.s.gather_sz = ((i + 2) / 3);
+		g_size_bytes = ((i + 2) / 3) * sizeof(struct roc_sg2list_comp);
+
+		i = 0;
+		scatter_comp = (struct roc_sg2list_comp *)((uint8_t *)gather_comp + g_size_bytes);
+
+		i = fill_sg2_comp_from_pkt(scatter_comp, i, m_src);
+
+		cpt_inst_w6.s.scatter_sz = ((i + 2) / 3);
+
+		cpt_inst_w5.s.dptr = (uint64_t)gather_comp;
+		cpt_inst_w6.s.rptr = (uint64_t)scatter_comp;
+
+		inst->w5.u64 = cpt_inst_w5.u64;
+		inst->w6.u64 = cpt_inst_w6.u64;
+		w4.u64 = sess->inst.w4;
+		w4.s.dlen = rte_pktmbuf_pkt_len(m_src);
+		w4.s.opcode_major &= (~(ROC_IE_OT_INPLACE_BIT));
+		w4.s.opcode_minor = sess->tls.enable_padding * cop->aux_flags * 8;
+		w4.s.param1 = w4.s.dlen;
+		w4.s.param2 = cop->param1.tls_record.content_type;
+		/* Output Scatter List */
+		last_seg->data_len += sess->max_extended_len;
+		inst->w4.u64 = w4.u64;
+	}
+
+	return 0;
+}
+
+static __rte_always_inline int
+process_tls_read(struct rte_crypto_op *cop, struct cn10k_sec_session *sess,
+		 struct cpt_qp_meta_info *m_info, struct cpt_inflight_req *infl_req,
+		 struct cpt_inst_s *inst, const bool is_sg_ver2)
+{
+	struct rte_crypto_sym_op *sym_op = cop->sym;
+	struct rte_mbuf *m_src = sym_op->m_src;
+	union cpt_inst_w4 w4;
+	uint8_t *in_buffer;
+	void *m_data;
+
+	if (likely(m_src->next == NULL)) {
+		void *vaddr;
+
+		vaddr = rte_pktmbuf_mtod(m_src, void *);
+
+		inst->dptr = (uint64_t)vaddr;
+		inst->rptr = (uint64_t)vaddr;
+
+		w4.u64 = sess->inst.w4;
+		w4.s.dlen = m_src->data_len;
+		w4.s.param1 = m_src->data_len;
+		inst->w4.u64 = w4.u64;
+	} else if (is_sg_ver2 == false) {
+		struct roc_sglist_comp *scatter_comp, *gather_comp;
+		uint32_t g_size_bytes, s_size_bytes;
+		uint32_t dlen;
+		int i;
+
+		m_data = alloc_op_meta(NULL, m_info->mlen, m_info->pool, infl_req);
+		if (unlikely(m_data == NULL)) {
+			plt_dp_err("Error allocating meta buffer for request");
+			return -ENOMEM;
+		}
+
+		in_buffer = (uint8_t *)m_data;
+		((uint16_t *)in_buffer)[0] = 0;
+		((uint16_t *)in_buffer)[1] = 0;
+
+		/* Input Gather List */
+		i = 0;
+		gather_comp = (struct roc_sglist_comp *)((uint8_t *)in_buffer + 8);
+
+		i = fill_sg_comp_from_pkt(gather_comp, i, m_src);
+		((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
+
+		g_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);
+
+		i = 0;
+		scatter_comp = (struct roc_sglist_comp *)((uint8_t *)gather_comp + g_size_bytes);
+
+		i = fill_sg_comp_from_pkt(scatter_comp, i, m_src);
+		((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
+
+		s_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);
+
+		dlen = g_size_bytes + s_size_bytes + ROC_SG_LIST_HDR_SIZE;
+
+		inst->dptr = (uint64_t)in_buffer;
+		inst->rptr = (uint64_t)in_buffer;
+
+		w4.u64 = sess->inst.w4;
+		w4.s.dlen = dlen;
+		w4.s.opcode_major |= (uint64_t)ROC_DMA_MODE_SG;
+		w4.s.param1 = rte_pktmbuf_pkt_len(m_src);
+		inst->w4.u64 = w4.u64;
+	} else {
+		struct roc_sg2list_comp *scatter_comp, *gather_comp;
+		union cpt_inst_w5 cpt_inst_w5;
+		union cpt_inst_w6 cpt_inst_w6;
+		uint32_t g_size_bytes;
+		int i;
+
+		m_data = alloc_op_meta(NULL, m_info->mlen, m_info->pool, infl_req);
+		if (unlikely(m_data == NULL)) {
+			plt_dp_err("Error allocating meta buffer for request");
+			return -ENOMEM;
+		}
+
+		in_buffer = (uint8_t *)m_data;
+		/* Input Gather List */
+		i = 0;
+
+		gather_comp = (struct roc_sg2list_comp *)((uint8_t *)in_buffer);
+		i = fill_sg2_comp_from_pkt(gather_comp, i, m_src);
+
+		cpt_inst_w5.s.gather_sz = ((i + 2) / 3);
+		g_size_bytes = ((i + 2) / 3) * sizeof(struct roc_sg2list_comp);
+
+		i = 0;
+		scatter_comp = (struct roc_sg2list_comp *)((uint8_t *)gather_comp + g_size_bytes);
+
+		i = fill_sg2_comp_from_pkt(scatter_comp, i, m_src);
+
+		cpt_inst_w6.s.scatter_sz = ((i + 2) / 3);
+
+		cpt_inst_w5.s.dptr = (uint64_t)gather_comp;
+		cpt_inst_w6.s.rptr = (uint64_t)scatter_comp;
+
+		inst->w5.u64 = cpt_inst_w5.u64;
+		inst->w6.u64 = cpt_inst_w6.u64;
+		w4.u64 = sess->inst.w4;
+		w4.s.dlen = rte_pktmbuf_pkt_len(m_src);
+		w4.s.param1 = w4.s.dlen;
+		w4.s.opcode_major &= (~(ROC_IE_OT_INPLACE_BIT));
+		inst->w4.u64 = w4.u64;
+	}
+
+	return 0;
+}
+#endif /* __CN10K_TLS_OPS_H__ */
-- 
2.25.1