[PATCH v2 2/2] crypto/cnxk: fix failure from session rework

[Anoob Joseph <anoobj@marvell.com>]

Post security session rework, CPTR got changed affecting cn9k IPsec
functionality. Address the same. Also, move all s/w accessible
fast path fields to rte_security_session cacheline for better perf.

Fixes: 3f3fc3308bd0 ("security: remove private mempool usage")

Signed-off-by: Anoob Joseph <anoobj@marvell.com>
---
v2:
* Fixed debug build failure

 drivers/crypto/cnxk/cn9k_cryptodev_ops.c | 46 +++++++---------
 drivers/crypto/cnxk/cn9k_ipsec.c         | 70 ++++++++++++------------
 drivers/crypto/cnxk/cn9k_ipsec.h         | 44 ++++++++-------
 drivers/crypto/cnxk/cn9k_ipsec_la_ops.h  | 35 ++++++------
 4 files changed, 93 insertions(+), 102 deletions(-)

diff --git a/drivers/crypto/cnxk/cn9k_cryptodev_ops.c b/drivers/crypto/cnxk/cn9k_cryptodev_ops.c
index 2ed298e01f..289601330e 100644
--- a/drivers/crypto/cnxk/cn9k_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cn9k_cryptodev_ops.c
@@ -23,12 +23,10 @@ cn9k_cpt_sec_inst_fill(struct rte_crypto_op *op,
 		       struct cpt_inst_s *inst)
 {
 	struct rte_crypto_sym_op *sym_op = op->sym;
-	struct cn9k_sec_session *priv;
-	struct cn9k_ipsec_sa *sa;
+	struct cn9k_sec_session *sec_sess;
 	int ret;

-	priv = SECURITY_GET_SESS_PRIV(op->sym->session);
-	sa = &priv->sa;
+	sec_sess = (struct cn9k_sec_session *)(op->sym->session);

 	if (unlikely(sym_op->m_dst && sym_op->m_dst != sym_op->m_src)) {
 		plt_dp_err("Out of place is not supported");
@@ -40,12 +38,12 @@ cn9k_cpt_sec_inst_fill(struct rte_crypto_op *op,
 		return -ENOTSUP;
 	}

-	if (sa->dir == RTE_SECURITY_IPSEC_SA_DIR_EGRESS)
-		ret = process_outb_sa(op, sa, inst);
+	if (sec_sess->is_outbound)
+		ret = process_outb_sa(op, sec_sess, inst);
 	else {
 		infl_req->op_flags |= CPT_OP_FLAGS_IPSEC_DIR_INBOUND;
-		process_inb_sa(op, sa, inst);
-		if (unlikely(sa->replay_win_sz))
+		process_inb_sa(op, sec_sess, inst);
+		if (unlikely(sec_sess->replay_win_sz))
 			infl_req->op_flags |= CPT_OP_FLAGS_IPSEC_INB_REPLAY;
 		ret = 0;
 	}
@@ -335,12 +333,10 @@ cn9k_cpt_crypto_adapter_ev_mdata_set(struct rte_cryptodev *dev __rte_unused,
 	if (op_type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
 		if (sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
 			struct cn9k_sec_session *priv;
-			struct cn9k_ipsec_sa *sa;

-			priv = SECURITY_GET_SESS_PRIV(sess);
-			sa = &priv->sa;
-			sa->qp = qp;
-			sa->inst.w2 = w2;
+			priv = (struct cn9k_sec_session *)sess;
+			priv->qp = qp;
+			priv->inst.w2 = w2;
 		} else if (sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
 			struct cnxk_se_sess *priv;

@@ -372,12 +368,10 @@ cn9k_ca_meta_info_extract(struct rte_crypto_op *op,
 	if (op->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
 		if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
 			struct cn9k_sec_session *priv;
-			struct cn9k_ipsec_sa *sa;

-			priv = SECURITY_GET_SESS_PRIV(op->sym->session);
-			sa = &priv->sa;
-			*qp = sa->qp;
-			inst->w2.u64 = sa->inst.w2;
+			priv = (struct cn9k_sec_session *)(op->sym->session);
+			*qp = priv->qp;
+			inst->w2.u64 = priv->inst.w2;
 		} else if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
 			struct cnxk_se_sess *priv;

@@ -480,7 +474,8 @@ cn9k_cpt_crypto_adapter_enqueue(uintptr_t base, struct rte_crypto_op *op)
 }

 static inline int
-ipsec_antireplay_check(struct cn9k_ipsec_sa *sa, uint32_t win_sz, struct roc_ie_on_inb_hdr *data)
+ipsec_antireplay_check(struct cn9k_sec_session *sess, uint32_t win_sz,
+		       struct roc_ie_on_inb_hdr *data)
 {
 	uint32_t esn_low, esn_hi, seql, seqh = 0;
 	struct roc_ie_on_common_sa *common_sa;
@@ -489,7 +484,7 @@ ipsec_antireplay_check(struct cn9k_ipsec_sa *sa, uint32_t win_sz, struct roc_ie_
 	uint8_t esn;
 	int ret;

-	in_sa = &sa->in_sa;
+	in_sa = &sess->sa.in_sa;
 	common_sa = &in_sa->common_sa;

 	esn = common_sa->ctl.esn_en;
@@ -505,7 +500,7 @@ ipsec_antireplay_check(struct cn9k_ipsec_sa *sa, uint32_t win_sz, struct roc_ie_
 	if (unlikely(seq == 0))
 		return IPSEC_ANTI_REPLAY_FAILED;

-	ret = cnxk_on_anti_replay_check(seq, &sa->ar, win_sz);
+	ret = cnxk_on_anti_replay_check(seq, &sess->ar, win_sz);
 	if (esn && !ret) {
 		esn_low = rte_be_to_cpu_32(common_sa->seq_t.tl);
 		esn_hi = rte_be_to_cpu_32(common_sa->seq_t.th);
@@ -526,7 +521,6 @@ cn9k_cpt_sec_post_process(struct rte_crypto_op *cop,
 	struct rte_crypto_sym_op *sym_op = cop->sym;
 	struct rte_mbuf *m = sym_op->m_src;
 	struct cn9k_sec_session *priv;
-	struct cn9k_ipsec_sa *sa;
 	struct rte_ipv6_hdr *ip6;
 	struct rte_ipv4_hdr *ip;
 	uint16_t m_len = 0;
@@ -539,12 +533,10 @@ cn9k_cpt_sec_post_process(struct rte_crypto_op *cop,
 			     CPT_OP_FLAGS_IPSEC_INB_REPLAY)) {
 			int ret;

-			priv = SECURITY_GET_SESS_PRIV(sym_op->session);
-			sa = &priv->sa;
+			priv = (struct cn9k_sec_session *)(sym_op->session);

-			ret = ipsec_antireplay_check(
-				sa, sa->replay_win_sz,
-				(struct roc_ie_on_inb_hdr *)data);
+			ret = ipsec_antireplay_check(priv, priv->replay_win_sz,
+						     (struct roc_ie_on_inb_hdr *)data);
 			if (unlikely(ret)) {
 				cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
 				return;
diff --git a/drivers/crypto/cnxk/cn9k_ipsec.c b/drivers/crypto/cnxk/cn9k_ipsec.c
index b56843f49b..5f3a74107b 100644
--- a/drivers/crypto/cnxk/cn9k_ipsec.c
+++ b/drivers/crypto/cnxk/cn9k_ipsec.c
@@ -32,38 +32,39 @@ cn9k_ipsec_outb_sa_create(struct cnxk_cpt_qp *qp,
 	uint8_t egrp;
 	int ret;

-	sess = SECURITY_GET_SESS_PRIV(sec_sess);
+	sess = (struct cn9k_sec_session *)sec_sess;
 	sa = &sess->sa;

+	/* Initialize lookaside IPsec private data */
+
 	memset(sa, 0, sizeof(struct cn9k_ipsec_sa));

-	/* Initialize lookaside IPsec private data */
-	sa->dir = RTE_SECURITY_IPSEC_SA_DIR_EGRESS;
+	sess->is_outbound = 1;

 	if (ipsec->esn.value)
-		sa->esn = ipsec->esn.value - 1;
+		sess->esn = ipsec->esn.value - 1;

-	ret = cnxk_ipsec_outb_rlens_get(&sa->rlens, ipsec, crypto_xform);
+	ret = cnxk_ipsec_outb_rlens_get(&sess->rlens, ipsec, crypto_xform);
 	if (ret)
 		return ret;

-	sa->custom_hdr_len =
+	sess->custom_hdr_len =
 		sizeof(struct roc_ie_on_outb_hdr) - ROC_IE_ON_MAX_IV_LEN;

 #ifdef LA_IPSEC_DEBUG
 	/* Use IV from application in debug mode */
 	if (ipsec->options.iv_gen_disable == 1) {
-		sa->custom_hdr_len = sizeof(struct roc_ie_on_outb_hdr);
+		sess->custom_hdr_len = sizeof(struct roc_ie_on_outb_hdr);

 		if (crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
-			sa->cipher_iv_off = crypto_xform->aead.iv.offset;
-			sa->cipher_iv_len = crypto_xform->aead.iv.length;
+			sess->cipher_iv_off = crypto_xform->aead.iv.offset;
+			sess->cipher_iv_len = crypto_xform->aead.iv.length;
 		} else if (crypto_xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
-			sa->cipher_iv_off = crypto_xform->cipher.iv.offset;
-			sa->cipher_iv_len = crypto_xform->cipher.iv.length;
+			sess->cipher_iv_off = crypto_xform->cipher.iv.offset;
+			sess->cipher_iv_len = crypto_xform->cipher.iv.length;
 		} else {
-			sa->cipher_iv_off = crypto_xform->auth.iv.offset;
-			sa->cipher_iv_len = crypto_xform->auth.iv.length;
+			sess->cipher_iv_off = crypto_xform->auth.iv.offset;
+			sess->cipher_iv_len = crypto_xform->auth.iv.length;
 		}
 	}
 #else
@@ -80,8 +81,7 @@ cn9k_ipsec_outb_sa_create(struct cnxk_cpt_qp *qp,

 	ctx_len = ret;
 	egrp = roc_cpt->eng_grp[CPT_ENG_TYPE_IE];
-	ret = roc_on_cpt_ctx_write(&qp->lf, SECURITY_GET_SESS_PRIV_IOVA(sec_sess),
-				   false, ctx_len, egrp);
+	ret = roc_on_cpt_ctx_write(&qp->lf, (uintptr_t)sa, false, ctx_len, egrp);

 	if (ret)
 		return ret;
@@ -108,9 +108,9 @@ cn9k_ipsec_outb_sa_create(struct cnxk_cpt_qp *qp,

 	w7.u64 = 0;
 	w7.s.egrp = egrp;
-	w7.s.cptr = SECURITY_GET_SESS_PRIV_IOVA(sec_sess);
+	w7.s.cptr = (uintptr_t)&sess->sa;

-	inst_tmpl = &sa->inst;
+	inst_tmpl = &sess->inst;
 	inst_tmpl->w4 = w4.u64;
 	inst_tmpl->w7 = w7.u64;

@@ -134,31 +134,30 @@ cn9k_ipsec_inb_sa_create(struct cnxk_cpt_qp *qp,
 	uint8_t egrp;
 	int ret = 0;

-	sess = SECURITY_GET_SESS_PRIV(sec_sess);
+	sess = (struct cn9k_sec_session *)sec_sess;
 	sa = &sess->sa;

 	memset(sa, 0, sizeof(struct cn9k_ipsec_sa));

-	sa->dir = RTE_SECURITY_IPSEC_SA_DIR_INGRESS;
-	sa->replay_win_sz = ipsec->replay_win_sz;
+	sess->is_outbound = 0;
+	sess->replay_win_sz = ipsec->replay_win_sz;

-	if (sa->replay_win_sz) {
-		if (sa->replay_win_sz > CNXK_ON_AR_WIN_SIZE_MAX) {
-			plt_err("Replay window size:%u is not supported",
-				sa->replay_win_sz);
+	if (sess->replay_win_sz) {
+		if (sess->replay_win_sz > CNXK_ON_AR_WIN_SIZE_MAX) {
+			plt_err("Replay window size:%u is not supported", sess->replay_win_sz);
 			return -ENOTSUP;
 		}

 		/* Set window bottom to 1, base and top to size of window */
-		sa->ar.winb = 1;
-		sa->ar.wint = sa->replay_win_sz;
-		sa->ar.base = sa->replay_win_sz;
+		sess->ar.winb = 1;
+		sess->ar.wint = sess->replay_win_sz;
+		sess->ar.base = sess->replay_win_sz;

-		sa->seq_lo = ipsec->esn.low;
-		sa->seq_hi = ipsec->esn.hi;
+		sess->seq_lo = ipsec->esn.low;
+		sess->seq_hi = ipsec->esn.hi;

-		sa->in_sa.common_sa.seq_t.tl = sa->seq_lo;
-		sa->in_sa.common_sa.seq_t.th = sa->seq_hi;
+		sess->sa.in_sa.common_sa.seq_t.tl = sess->seq_lo;
+		sess->sa.in_sa.common_sa.seq_t.th = sess->seq_hi;
 	}

 	ret = cnxk_on_ipsec_inb_sa_create(ipsec, crypto_xform, &sa->in_sa);
@@ -166,12 +165,11 @@ cn9k_ipsec_inb_sa_create(struct cnxk_cpt_qp *qp,
 		return ret;

 	if (sa->in_sa.common_sa.ctl.esn_en)
-		sa->esn_en = 1;
+		sess->esn_en = 1;

 	ctx_len = ret;
 	egrp = roc_cpt->eng_grp[CPT_ENG_TYPE_IE];
-	ret = roc_on_cpt_ctx_write(&qp->lf, SECURITY_GET_SESS_PRIV_IOVA(sec_sess),
-				   true, ctx_len, egrp);
+	ret = roc_on_cpt_ctx_write(&qp->lf, (uint64_t)sa, true, ctx_len, egrp);
 	if (ret)
 		return ret;

@@ -184,9 +182,9 @@ cn9k_ipsec_inb_sa_create(struct cnxk_cpt_qp *qp,
 	w4.s.param2 = param2.u16;

 	w7.s.egrp = egrp;
-	w7.s.cptr = SECURITY_GET_SESS_PRIV_IOVA(sec_sess);
+	w7.s.cptr = (uintptr_t)&sess->sa;

-	inst_tmpl = &sa->inst;
+	inst_tmpl = &sess->inst;
 	inst_tmpl->w4 = w4.u64;
 	inst_tmpl->w7 = w7.u64;

diff --git a/drivers/crypto/cnxk/cn9k_ipsec.h b/drivers/crypto/cnxk/cn9k_ipsec.h
index bed5976096..1ea946afd8 100644
--- a/drivers/crypto/cnxk/cn9k_ipsec.h
+++ b/drivers/crypto/cnxk/cn9k_ipsec.h
@@ -5,6 +5,8 @@
 #ifndef __CN9K_IPSEC_H__
 #define __CN9K_IPSEC_H__

+#include <rte_security_driver.h>
+
 #include "cnxk_ipsec.h"
 #include "cnxk_security.h"
 #include "cnxk_security_ar.h"
@@ -16,40 +18,42 @@ struct cn9k_ipsec_sa {
 		/** Outbound SA */
 		struct roc_ie_on_outb_sa out_sa;
 	};
+} __rte_aligned(8);
+
+struct cn9k_sec_session {
+	struct rte_security_session rte_sess;
+
+	/** PMD private space */
+
+	/** ESN */
+	union {
+		uint64_t esn;
+		struct {
+			uint32_t seq_lo;
+			uint32_t seq_hi;
+		};
+	};
 	/** IPsec SA direction */
-	enum rte_security_ipsec_sa_direction dir;
+	uint8_t is_outbound;
+	/* ESN enable flag */
+	uint8_t esn_en;
 	/** Pre-populated CPT inst words */
 	struct cnxk_cpt_inst_tmpl inst;
+	/** Response length calculation data */
+	struct cnxk_ipsec_outb_rlens rlens;
+	/** Anti replay window size */
+	uint32_t replay_win_sz;
 	/** Cipher IV offset in bytes */
 	uint16_t cipher_iv_off;
 	/** Cipher IV length in bytes */
 	uint8_t cipher_iv_len;
 	/** Outbound custom header length */
 	uint8_t custom_hdr_len;
-	/** Response length calculation data */
-	struct cnxk_ipsec_outb_rlens rlens;
-	/** ESN */
-	union {
-		uint64_t esn;
-		struct {
-			uint32_t seq_lo;
-			uint32_t seq_hi;
-		};
-	};
 	/** Anti replay */
 	struct cnxk_on_ipsec_ar ar;
-	/** Anti replay window size */
-	uint32_t replay_win_sz;
-	/*
-	 * ESN enable flag. Copy of in_sa ctl.esn_en to have single cache line
-	 * access in the non-esn fastpath.
-	 */
-	uint8_t esn_en;
 	/** Queue pair */
 	struct cnxk_cpt_qp *qp;
-};

-struct cn9k_sec_session {
 	struct cn9k_ipsec_sa sa;
 } __rte_cache_aligned;

diff --git a/drivers/crypto/cnxk/cn9k_ipsec_la_ops.h b/drivers/crypto/cnxk/cn9k_ipsec_la_ops.h
index 8b68e4c728..8b4e636c70 100644
--- a/drivers/crypto/cnxk/cn9k_ipsec_la_ops.h
+++ b/drivers/crypto/cnxk/cn9k_ipsec_la_ops.h
@@ -13,21 +13,20 @@
 #include "cnxk_security_ar.h"

 static __rte_always_inline int32_t
-ipsec_po_out_rlen_get(struct cn9k_ipsec_sa *sa, uint32_t plen)
+ipsec_po_out_rlen_get(struct cn9k_sec_session *sess, uint32_t plen)
 {
 	uint32_t enc_payload_len;

-	enc_payload_len = RTE_ALIGN_CEIL(plen + sa->rlens.roundup_len,
-					 sa->rlens.roundup_byte);
+	enc_payload_len = RTE_ALIGN_CEIL(plen + sess->rlens.roundup_len,
+					 sess->rlens.roundup_byte);

-	return sa->custom_hdr_len + sa->rlens.partial_len + enc_payload_len;
+	return sess->custom_hdr_len + sess->rlens.partial_len + enc_payload_len;
 }

 static __rte_always_inline int
-process_outb_sa(struct rte_crypto_op *cop, struct cn9k_ipsec_sa *sa,
-		struct cpt_inst_s *inst)
+process_outb_sa(struct rte_crypto_op *cop, struct cn9k_sec_session *sess, struct cpt_inst_s *inst)
 {
-	const unsigned int hdr_len = sa->custom_hdr_len;
+	const unsigned int hdr_len = sess->custom_hdr_len;
 	struct rte_crypto_sym_op *sym_op = cop->sym;
 	struct rte_mbuf *m_src = sym_op->m_src;
 	uint32_t dlen, rlen, pkt_len, seq_lo;
@@ -38,7 +37,7 @@ process_outb_sa(struct rte_crypto_op *cop, struct cn9k_ipsec_sa *sa,

 	pkt_len = rte_pktmbuf_pkt_len(m_src);
 	dlen = pkt_len + hdr_len;
-	rlen = ipsec_po_out_rlen_get(sa, pkt_len);
+	rlen = ipsec_po_out_rlen_get(sess, pkt_len);

 	extend_tail = rlen - dlen;
 	if (unlikely(extend_tail > rte_pktmbuf_tailroom(m_src))) {
@@ -61,15 +60,14 @@ process_outb_sa(struct rte_crypto_op *cop, struct cn9k_ipsec_sa *sa,
 	hdr = PLT_PTR_ADD(m_src->buf_addr, data_off - hdr_len);

 #ifdef LA_IPSEC_DEBUG
-	if (sa->inst.w4 & ROC_IE_ON_PER_PKT_IV) {
+	if (sess->inst.w4 & ROC_IE_ON_PER_PKT_IV) {
 		memcpy(&hdr->iv[0],
-		       rte_crypto_op_ctod_offset(cop, uint8_t *,
-						 sa->cipher_iv_off),
-		       sa->cipher_iv_len);
+		       rte_crypto_op_ctod_offset(cop, uint8_t *, sess->cipher_iv_off),
+		       sess->cipher_iv_len);
 	}
 #endif

-	esn = ++sa->esn;
+	esn = ++sess->esn;

 	/* Set ESN seq hi */
 	hdr->esn = rte_cpu_to_be_32(esn >> 32);
@@ -82,24 +80,23 @@ process_outb_sa(struct rte_crypto_op *cop, struct cn9k_ipsec_sa *sa,
 	hdr->ip_id = seq_lo;

 	/* Prepare CPT instruction */
-	inst->w4.u64 = sa->inst.w4 | dlen;
+	inst->w4.u64 = sess->inst.w4 | dlen;
 	inst->dptr = PLT_U64_CAST(hdr);
 	inst->rptr = PLT_U64_CAST(hdr);
-	inst->w7.u64 = sa->inst.w7;
+	inst->w7.u64 = sess->inst.w7;

 	return 0;
 }

 static __rte_always_inline void
-process_inb_sa(struct rte_crypto_op *cop, struct cn9k_ipsec_sa *sa,
-	       struct cpt_inst_s *inst)
+process_inb_sa(struct rte_crypto_op *cop, struct cn9k_sec_session *sess, struct cpt_inst_s *inst)
 {
 	struct rte_crypto_sym_op *sym_op = cop->sym;
 	struct rte_mbuf *m_src = sym_op->m_src;

 	/* Prepare CPT instruction */
-	inst->w4.u64 = sa->inst.w4 | rte_pktmbuf_pkt_len(m_src);
+	inst->w4.u64 = sess->inst.w4 | rte_pktmbuf_pkt_len(m_src);
 	inst->dptr = inst->rptr = rte_pktmbuf_mtod(m_src, uint64_t);
-	inst->w7.u64 = sa->inst.w7;
+	inst->w7.u64 = sess->inst.w7;
 }
 #endif /* __CN9K_IPSEC_LA_OPS_H__ */
--
2.25.1