* [PATCH 01/17] common/cnxk: perform LF fini ops only when allocated
2022-12-20 14:32 [PATCH 00/17] fixes and improvements to cnxk crytpo PMD Tejasree Kondoj
@ 2022-12-20 14:32 ` Tejasree Kondoj
2022-12-20 14:32 ` [PATCH 02/17] common/cnxk: generate opad and ipad in driver Tejasree Kondoj
` (16 subsequent siblings)
17 siblings, 0 replies; 19+ messages in thread
From: Tejasree Kondoj @ 2022-12-20 14:32 UTC (permalink / raw)
To: Akhil Goyal
Cc: Anoob Joseph, Vidya Sagar Velumuri, Gowrishankar Muthukrishnan,
Volodymyr Fialko, Aakash Sasidharan, dev
From: Anoob Joseph <anoobj@marvell.com>
If LFs are not attached, LF detach and free ops should not be attempted.
Signed-off-by: Anoob Joseph <anoobj@marvell.com>
---
drivers/common/cnxk/roc_cpt.c | 3 +++
1 file changed, 3 insertions(+)
diff --git a/drivers/common/cnxk/roc_cpt.c b/drivers/common/cnxk/roc_cpt.c
index fb97ec89b2..bf216b5545 100644
--- a/drivers/common/cnxk/roc_cpt.c
+++ b/drivers/common/cnxk/roc_cpt.c
@@ -861,6 +861,9 @@ roc_cpt_dev_clear(struct roc_cpt *roc_cpt)
if (cpt == NULL)
return;
+ if (roc_cpt->nb_lf == 0)
+ return;
+
for (i = 0; i < roc_cpt->nb_lf; i++)
cpt->lf_msix_off[i] = 0;
--
2.25.1
^ permalink raw reply [flat|nested] 19+ messages in thread* [PATCH 02/17] common/cnxk: generate opad and ipad in driver
2022-12-20 14:32 [PATCH 00/17] fixes and improvements to cnxk crytpo PMD Tejasree Kondoj
2022-12-20 14:32 ` [PATCH 01/17] common/cnxk: perform LF fini ops only when allocated Tejasree Kondoj
@ 2022-12-20 14:32 ` Tejasree Kondoj
2022-12-20 14:32 ` [PATCH 03/17] crypto/cnxk: update resp len calculation for IPv6 Tejasree Kondoj
` (15 subsequent siblings)
17 siblings, 0 replies; 19+ messages in thread
From: Tejasree Kondoj @ 2022-12-20 14:32 UTC (permalink / raw)
To: Akhil Goyal
Cc: Aakash Sasidharan, Anoob Joseph, Vidya Sagar Velumuri,
Gowrishankar Muthukrishnan, Volodymyr Fialko, dev
From: Aakash Sasidharan <asasidharan@marvell.com>
Generate opad and ipad in driver for SHA based crypto algos
Signed-off-by: Aakash Sasidharan <asasidharan@marvell.com>
---
drivers/common/cnxk/cnxk_security.c | 4 +--
drivers/common/cnxk/roc_hash.c | 12 ++++---
drivers/common/cnxk/roc_hash.h | 2 +-
drivers/common/cnxk/roc_se.c | 52 +++++++++++++++++++++++++++--
4 files changed, 60 insertions(+), 10 deletions(-)
diff --git a/drivers/common/cnxk/cnxk_security.c b/drivers/common/cnxk/cnxk_security.c
index 85105472a1..5034c76938 100644
--- a/drivers/common/cnxk/cnxk_security.c
+++ b/drivers/common/cnxk/cnxk_security.c
@@ -37,8 +37,8 @@ ipsec_hmac_opad_ipad_gen(struct rte_crypto_sym_xform *auth_xform,
roc_hash_sha1_gen(ipad, (uint32_t *)&hmac_opad_ipad[24]);
break;
case RTE_CRYPTO_AUTH_SHA256_HMAC:
- roc_hash_sha256_gen(opad, (uint32_t *)&hmac_opad_ipad[0]);
- roc_hash_sha256_gen(ipad, (uint32_t *)&hmac_opad_ipad[64]);
+ roc_hash_sha256_gen(opad, (uint32_t *)&hmac_opad_ipad[0], 256);
+ roc_hash_sha256_gen(ipad, (uint32_t *)&hmac_opad_ipad[64], 256);
break;
case RTE_CRYPTO_AUTH_SHA384_HMAC:
roc_hash_sha512_gen(opad, (uint64_t *)&hmac_opad_ipad[0], 384);
diff --git a/drivers/common/cnxk/roc_hash.c b/drivers/common/cnxk/roc_hash.c
index 1b9030e693..8c451440b1 100644
--- a/drivers/common/cnxk/roc_hash.c
+++ b/drivers/common/cnxk/roc_hash.c
@@ -232,7 +232,7 @@ roc_hash_sha1_gen(uint8_t *msg, uint32_t *hash)
* Based on implementation from RFC 3174
*/
void
-roc_hash_sha256_gen(uint8_t *msg, uint32_t *hash)
+roc_hash_sha256_gen(uint8_t *msg, uint32_t *hash, int hash_size)
{
const uint32_t _K[] = {
/* Round Constants defined in SHA-256 */
@@ -250,13 +250,17 @@ roc_hash_sha256_gen(uint8_t *msg, uint32_t *hash)
0x682e6ff3, 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208,
0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2};
- const uint32_t _H[] = {/* Initial Hash constants defined in SHA-256 */
- 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a,
- 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19};
+ const uint32_t _H224[] = {/* Initial Hash constants defined in SHA-224 */
+ 0xc1059ed8, 0x367cd507, 0x3070dd17, 0xf70e5939,
+ 0xffc00b31, 0x68581511, 0x64f98fa7, 0xbefa4fa4};
+ const uint32_t _H256[] = {/* Initial Hash constants defined in SHA-256 */
+ 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a,
+ 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19};
int i;
uint32_t temp[4], S0, S1; /* Temporary word value */
uint32_t W[64]; /* Word sequence */
uint32_t A, B, C, D, E, F, G, H; /* Word buffers */
+ const uint32_t *_H = (hash_size == 224) ? _H224 : _H256;
/* Initialize the first 16 words in the array W */
memcpy(&W[0], msg, 16 * sizeof(W[0]));
diff --git a/drivers/common/cnxk/roc_hash.h b/drivers/common/cnxk/roc_hash.h
index 8940faa6eb..c3ddb9407b 100644
--- a/drivers/common/cnxk/roc_hash.h
+++ b/drivers/common/cnxk/roc_hash.h
@@ -11,7 +11,7 @@
*/
void __roc_api roc_hash_md5_gen(uint8_t *msg, uint32_t *hash);
void __roc_api roc_hash_sha1_gen(uint8_t *msg, uint32_t *hash);
-void __roc_api roc_hash_sha256_gen(uint8_t *msg, uint32_t *hash);
+void __roc_api roc_hash_sha256_gen(uint8_t *msg, uint32_t *hash, int hash_size);
void __roc_api roc_hash_sha512_gen(uint8_t *msg, uint64_t *hash, int hash_size);
#endif /* _ROC_HASH_H_ */
diff --git a/drivers/common/cnxk/roc_se.c b/drivers/common/cnxk/roc_se.c
index 2663480099..22df61f5f0 100644
--- a/drivers/common/cnxk/roc_se.c
+++ b/drivers/common/cnxk/roc_se.c
@@ -149,6 +149,53 @@ cpt_ciph_aes_key_type_set(struct roc_se_context *fctx, uint16_t key_len)
fctx->enc.aes_key = aes_key_type;
}
+static void
+cpt_hmac_opad_ipad_gen(roc_se_auth_type auth_type, const uint8_t *key, uint16_t length,
+ struct roc_se_hmac_context *hmac)
+{
+ uint8_t opad[128] = {[0 ... 127] = 0x5c};
+ uint8_t ipad[128] = {[0 ... 127] = 0x36};
+ uint32_t i;
+
+ /* HMAC OPAD and IPAD */
+ for (i = 0; i < 128 && i < length; i++) {
+ opad[i] = opad[i] ^ key[i];
+ ipad[i] = ipad[i] ^ key[i];
+ }
+
+ /* Precompute hash of HMAC OPAD and IPAD to avoid
+ * per packet computation
+ */
+ switch (auth_type) {
+ case ROC_SE_MD5_TYPE:
+ roc_hash_md5_gen(opad, (uint32_t *)hmac->opad);
+ roc_hash_md5_gen(ipad, (uint32_t *)hmac->ipad);
+ break;
+ case ROC_SE_SHA1_TYPE:
+ roc_hash_sha1_gen(opad, (uint32_t *)hmac->opad);
+ roc_hash_sha1_gen(ipad, (uint32_t *)hmac->ipad);
+ break;
+ case ROC_SE_SHA2_SHA224:
+ roc_hash_sha256_gen(opad, (uint32_t *)hmac->opad, 224);
+ roc_hash_sha256_gen(ipad, (uint32_t *)hmac->ipad, 224);
+ break;
+ case ROC_SE_SHA2_SHA256:
+ roc_hash_sha256_gen(opad, (uint32_t *)hmac->opad, 256);
+ roc_hash_sha256_gen(ipad, (uint32_t *)hmac->ipad, 256);
+ break;
+ case ROC_SE_SHA2_SHA384:
+ roc_hash_sha512_gen(opad, (uint64_t *)hmac->opad, 384);
+ roc_hash_sha512_gen(ipad, (uint64_t *)hmac->ipad, 384);
+ break;
+ case ROC_SE_SHA2_SHA512:
+ roc_hash_sha512_gen(opad, (uint64_t *)hmac->opad, 512);
+ roc_hash_sha512_gen(ipad, (uint64_t *)hmac->ipad, 512);
+ break;
+ default:
+ break;
+ }
+}
+
static int
cpt_pdcp_key_type_set(struct roc_se_zuc_snow3g_ctx *zs_ctx, uint16_t key_len)
{
@@ -434,9 +481,8 @@ roc_se_auth_key_set(struct roc_se_ctx *se_ctx, roc_se_auth_type type,
memset(fctx->hmac.ipad, 0, sizeof(fctx->hmac.ipad));
memset(fctx->hmac.opad, 0, sizeof(fctx->hmac.opad));
- if (key_len <= 64)
- memcpy(fctx->hmac.opad, key, key_len);
- fctx->enc.auth_input_type = 1;
+ cpt_hmac_opad_ipad_gen(type, key, key_len, &fctx->hmac);
+ fctx->enc.auth_input_type = 0;
}
return 0;
}
--
2.25.1
^ permalink raw reply [flat|nested] 19+ messages in thread* [PATCH 03/17] crypto/cnxk: update resp len calculation for IPv6
2022-12-20 14:32 [PATCH 00/17] fixes and improvements to cnxk crytpo PMD Tejasree Kondoj
2022-12-20 14:32 ` [PATCH 01/17] common/cnxk: perform LF fini ops only when allocated Tejasree Kondoj
2022-12-20 14:32 ` [PATCH 02/17] common/cnxk: generate opad and ipad in driver Tejasree Kondoj
@ 2022-12-20 14:32 ` Tejasree Kondoj
2022-12-20 14:32 ` [PATCH 04/17] crypto/cnxk: add context to passthrough instruction Tejasree Kondoj
` (14 subsequent siblings)
17 siblings, 0 replies; 19+ messages in thread
From: Tejasree Kondoj @ 2022-12-20 14:32 UTC (permalink / raw)
To: Akhil Goyal
Cc: Vidya Sagar Velumuri, Anoob Joseph, Gowrishankar Muthukrishnan,
Volodymyr Fialko, Aakash Sasidharan, dev
From: Vidya Sagar Velumuri <vvelumuri@marvell.com>
In IPsec lookaside path, response length for an encryption is pre
calculated and updated in the mbuf as the packet length.
Response length dpepends on the layer-3 header length.
As IPsec security context does not provide any information about the IP
version, the layer-3 header length is calculated in data path based on IP
version and extension headers.
Signed-off-by: Vidya Sagar Velumuri <vvelumuri@marvell.com>
---
drivers/common/cnxk/cnxk_security.c | 4 +--
drivers/crypto/cnxk/cn9k_ipsec_la_ops.h | 36 ++++++++++++++++++++++---
2 files changed, 35 insertions(+), 5 deletions(-)
diff --git a/drivers/common/cnxk/cnxk_security.c b/drivers/common/cnxk/cnxk_security.c
index 5034c76938..79427d48fe 100644
--- a/drivers/common/cnxk/cnxk_security.c
+++ b/drivers/common/cnxk/cnxk_security.c
@@ -19,7 +19,7 @@ ipsec_hmac_opad_ipad_gen(struct rte_crypto_sym_xform *auth_xform,
uint32_t i;
/* HMAC OPAD and IPAD */
- for (i = 0; i < 127 && i < length; i++) {
+ for (i = 0; i < 128 && i < length; i++) {
opad[i] = opad[i] ^ key[i];
ipad[i] = ipad[i] ^ key[i];
}
@@ -1344,7 +1344,7 @@ cnxk_on_ipsec_outb_sa_create(struct rte_security_ipsec_xform *ipsec,
} else
ctx_len += sizeof(template->ip4);
- ctx_len += RTE_ALIGN_CEIL(ctx_len, 8);
+ ctx_len = RTE_ALIGN_CEIL(ctx_len, 8);
if (crypto_xform->type != RTE_CRYPTO_SYM_XFORM_AEAD) {
uint8_t *hmac_opad_ipad = (uint8_t *)&out_sa->sha2;
diff --git a/drivers/crypto/cnxk/cn9k_ipsec_la_ops.h b/drivers/crypto/cnxk/cn9k_ipsec_la_ops.h
index 724fc525ad..f1298017ce 100644
--- a/drivers/crypto/cnxk/cn9k_ipsec_la_ops.h
+++ b/drivers/crypto/cnxk/cn9k_ipsec_la_ops.h
@@ -13,14 +13,44 @@
#include "cnxk_security_ar.h"
static __rte_always_inline int32_t
-ipsec_po_out_rlen_get(struct cn9k_sec_session *sess, uint32_t plen)
+ipsec_po_out_rlen_get(struct cn9k_sec_session *sess, uint32_t plen, struct rte_mbuf *m_src)
{
uint32_t enc_payload_len;
int adj_len = 0;
- if (sess->sa.out_sa.common_sa.ctl.ipsec_mode == ROC_IE_SA_MODE_TRANSPORT)
+ if (sess->sa.out_sa.common_sa.ctl.ipsec_mode == ROC_IE_SA_MODE_TRANSPORT) {
adj_len = ROC_CPT_TUNNEL_IPV4_HDR_LEN;
+ uintptr_t data = (uintptr_t)m_src->buf_addr + m_src->data_off;
+ struct rte_ipv4_hdr *ip = (struct rte_ipv4_hdr *)data;
+
+ if (unlikely(((ip->version_ihl & 0xf0) >> RTE_IPV4_IHL_MULTIPLIER) != IPVERSION)) {
+ struct rte_ipv6_hdr *ip6 = (struct rte_ipv6_hdr *)ip;
+ uint8_t *nxt_hdr = (uint8_t *)ip6;
+ uint8_t dest_op_cnt = 0;
+ int nh = ip6->proto;
+
+ PLT_ASSERT(((ip->version_ihl & 0xf0) >> RTE_IPV4_IHL_MULTIPLIER) == 6);
+
+ adj_len = ROC_CPT_TUNNEL_IPV6_HDR_LEN;
+ nxt_hdr += ROC_CPT_TUNNEL_IPV6_HDR_LEN;
+ while (nh != -EINVAL) {
+ size_t ext_len = 0;
+
+ nh = rte_ipv6_get_next_ext(nxt_hdr, nh, &ext_len);
+ /* With multiple dest ops headers, the ESP hdr will be before
+ * the 2nd dest ops and after the first dest ops header
+ */
+ if ((nh == IPPROTO_DSTOPTS) && dest_op_cnt)
+ break;
+ else if (nh == IPPROTO_DSTOPTS)
+ dest_op_cnt++;
+ adj_len += ext_len;
+ nxt_hdr += ext_len;
+ }
+ }
+ }
+
enc_payload_len =
RTE_ALIGN_CEIL(plen + sess->rlens.roundup_len - adj_len, sess->rlens.roundup_byte);
@@ -41,7 +71,7 @@ process_outb_sa(struct rte_crypto_op *cop, struct cn9k_sec_session *sess, struct
pkt_len = rte_pktmbuf_pkt_len(m_src);
dlen = pkt_len + hdr_len;
- rlen = ipsec_po_out_rlen_get(sess, pkt_len);
+ rlen = ipsec_po_out_rlen_get(sess, pkt_len, m_src);
extend_tail = rlen - dlen;
if (unlikely(extend_tail > rte_pktmbuf_tailroom(m_src))) {
--
2.25.1
^ permalink raw reply [flat|nested] 19+ messages in thread* [PATCH 04/17] crypto/cnxk: add context to passthrough instruction
2022-12-20 14:32 [PATCH 00/17] fixes and improvements to cnxk crytpo PMD Tejasree Kondoj
` (2 preceding siblings ...)
2022-12-20 14:32 ` [PATCH 03/17] crypto/cnxk: update resp len calculation for IPv6 Tejasree Kondoj
@ 2022-12-20 14:32 ` Tejasree Kondoj
2022-12-20 14:32 ` [PATCH 05/17] crypto/cnxk: support truncated digest length Tejasree Kondoj
` (13 subsequent siblings)
17 siblings, 0 replies; 19+ messages in thread
From: Tejasree Kondoj @ 2022-12-20 14:32 UTC (permalink / raw)
To: Akhil Goyal
Cc: Volodymyr Fialko, Anoob Joseph, Vidya Sagar Velumuri,
Gowrishankar Muthukrishnan, Aakash Sasidharan, dev
From: Volodymyr Fialko <vfialko@marvell.com>
Attach valid context of last packet instruction to the passthrough
instruction to match hardware requirements.
Signed-off-by: Volodymyr Fialko <vfialko@marvell.com>
---
drivers/crypto/cnxk/cn10k_cryptodev_ops.c | 13 ++++++++++---
1 file changed, 10 insertions(+), 3 deletions(-)
diff --git a/drivers/crypto/cnxk/cn10k_cryptodev_ops.c b/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
index 7dad370047..2c750d19bf 100644
--- a/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
@@ -36,6 +36,7 @@ struct ops_burst {
struct vec_request {
struct cpt_inflight_req *req;
struct rte_event_vector *vec;
+ union cpt_inst_w7 w7;
uint64_t w2;
};
@@ -387,7 +388,7 @@ cn10k_ca_meta_info_extract(struct rte_crypto_op *op, struct cnxk_cpt_qp **qp, ui
static inline void
cn10k_cpt_vec_inst_fill(struct vec_request *vec_req, struct cpt_inst_s *inst,
- struct cnxk_cpt_qp *qp)
+ struct cnxk_cpt_qp *qp, union cpt_inst_w7 w7)
{
const union cpt_res_s res = {.cn10k.compcode = CPT_COMP_NOT_DONE};
struct cpt_inflight_req *infl_req = vec_req->req;
@@ -400,6 +401,8 @@ cn10k_cpt_vec_inst_fill(struct vec_request *vec_req, struct cpt_inst_s *inst,
.s.dlen = 0,
};
+ w7.s.egrp = ROC_CPT_DFLT_ENG_GRP_SE;
+
infl_req->vec = vec_req->vec;
infl_req->qp = qp;
@@ -410,7 +413,7 @@ cn10k_cpt_vec_inst_fill(struct vec_request *vec_req, struct cpt_inst_s *inst,
inst->w2.u64 = vec_req->w2;
inst->w3.u64 = CNXK_CPT_INST_W3(1, infl_req);
inst->w4.u64 = w4.u64;
- inst->w7.u64 = ROC_CPT_DFLT_ENG_GRP_SE << 61;
+ inst->w7.u64 = w7.u64;
}
static void
@@ -451,7 +454,7 @@ cn10k_cpt_vec_submit(struct vec_request vec_tbl[], uint16_t vec_tbl_len, struct
again:
burst_size = RTE_MIN(PKTS_PER_STEORL, vec_tbl_len);
for (i = 0; i < burst_size; i++)
- cn10k_cpt_vec_inst_fill(&vec_tbl[i], &inst[i * 2], qp);
+ cn10k_cpt_vec_inst_fill(&vec_tbl[i], &inst[i * 2], qp, vec_tbl[0].w7);
do {
fc.u64[0] = __atomic_load_n(fc_addr, __ATOMIC_RELAXED);
@@ -590,6 +593,10 @@ next_op:;
roc_lmt_submit_steorl(lmt_arg, io_addr);
}
+ /* Store w7 of last successfully filled instruction */
+ inst = &inst_base[2 * (i - 1)];
+ vec_tbl[0].w7 = inst->w7;
+
rte_io_wmb();
put:
--
2.25.1
^ permalink raw reply [flat|nested] 19+ messages in thread* [PATCH 05/17] crypto/cnxk: support truncated digest length
2022-12-20 14:32 [PATCH 00/17] fixes and improvements to cnxk crytpo PMD Tejasree Kondoj
` (3 preceding siblings ...)
2022-12-20 14:32 ` [PATCH 04/17] crypto/cnxk: add context to passthrough instruction Tejasree Kondoj
@ 2022-12-20 14:32 ` Tejasree Kondoj
2022-12-20 14:32 ` [PATCH 06/17] crypto/cnxk: add queue pair check to meta set Tejasree Kondoj
` (12 subsequent siblings)
17 siblings, 0 replies; 19+ messages in thread
From: Tejasree Kondoj @ 2022-12-20 14:32 UTC (permalink / raw)
To: Akhil Goyal
Cc: Anoob Joseph, Vidya Sagar Velumuri, Gowrishankar Muthukrishnan,
Volodymyr Fialko, Aakash Sasidharan, dev
Adding truncated digest length support.
Signed-off-by: Tejasree Kondoj <ktejasree@marvell.com>
---
.../crypto/cnxk/cnxk_cryptodev_capabilities.c | 48 +++++++++----------
drivers/crypto/cnxk/cnxk_se.h | 19 ++++----
2 files changed, 35 insertions(+), 32 deletions(-)
diff --git a/drivers/crypto/cnxk/cnxk_cryptodev_capabilities.c b/drivers/crypto/cnxk/cnxk_cryptodev_capabilities.c
index 6c28f8942e..9dfbf875ec 100644
--- a/drivers/crypto/cnxk/cnxk_cryptodev_capabilities.c
+++ b/drivers/crypto/cnxk/cnxk_cryptodev_capabilities.c
@@ -108,9 +108,9 @@ static const struct rte_cryptodev_capabilities caps_sha1_sha2[] = {
.increment = 0
},
.digest_size = {
- .min = 20,
+ .min = 1,
.max = 20,
- .increment = 0
+ .increment = 1
},
}, }
}, }
@@ -128,9 +128,9 @@ static const struct rte_cryptodev_capabilities caps_sha1_sha2[] = {
.increment = 1
},
.digest_size = {
- .min = 12,
+ .min = 1,
.max = 20,
- .increment = 8
+ .increment = 1
},
}, }
}, }
@@ -148,9 +148,9 @@ static const struct rte_cryptodev_capabilities caps_sha1_sha2[] = {
.increment = 0
},
.digest_size = {
- .min = 28,
+ .min = 1,
.max = 28,
- .increment = 0
+ .increment = 1
},
}, }
}, }
@@ -168,9 +168,9 @@ static const struct rte_cryptodev_capabilities caps_sha1_sha2[] = {
.increment = 1
},
.digest_size = {
- .min = 28,
+ .min = 1,
.max = 28,
- .increment = 0
+ .increment = 1
},
}, }
}, }
@@ -188,9 +188,9 @@ static const struct rte_cryptodev_capabilities caps_sha1_sha2[] = {
.increment = 0
},
.digest_size = {
- .min = 32,
+ .min = 1,
.max = 32,
- .increment = 0
+ .increment = 1
},
}, }
}, }
@@ -208,9 +208,9 @@ static const struct rte_cryptodev_capabilities caps_sha1_sha2[] = {
.increment = 1
},
.digest_size = {
- .min = 16,
+ .min = 1,
.max = 32,
- .increment = 16
+ .increment = 1
},
}, }
}, }
@@ -228,9 +228,9 @@ static const struct rte_cryptodev_capabilities caps_sha1_sha2[] = {
.increment = 0
},
.digest_size = {
- .min = 48,
+ .min = 1,
.max = 48,
- .increment = 0
+ .increment = 1
},
}, }
}, }
@@ -248,9 +248,9 @@ static const struct rte_cryptodev_capabilities caps_sha1_sha2[] = {
.increment = 1
},
.digest_size = {
- .min = 24,
+ .min = 1,
.max = 48,
- .increment = 24
+ .increment = 1
},
}, }
}, }
@@ -268,9 +268,9 @@ static const struct rte_cryptodev_capabilities caps_sha1_sha2[] = {
.increment = 0
},
.digest_size = {
- .min = 64,
+ .min = 1,
.max = 64,
- .increment = 0
+ .increment = 1
},
}, }
}, }
@@ -288,9 +288,9 @@ static const struct rte_cryptodev_capabilities caps_sha1_sha2[] = {
.increment = 1
},
.digest_size = {
- .min = 32,
+ .min = 1,
.max = 64,
- .increment = 32
+ .increment = 1
},
}, }
}, }
@@ -308,9 +308,9 @@ static const struct rte_cryptodev_capabilities caps_sha1_sha2[] = {
.increment = 0
},
.digest_size = {
- .min = 16,
+ .min = 1,
.max = 16,
- .increment = 0
+ .increment = 1
},
}, }
}, }
@@ -328,9 +328,9 @@ static const struct rte_cryptodev_capabilities caps_sha1_sha2[] = {
.increment = 8
},
.digest_size = {
- .min = 12,
+ .min = 1,
.max = 16,
- .increment = 4
+ .increment = 1
},
}, }
}, }
diff --git a/drivers/crypto/cnxk/cnxk_se.h b/drivers/crypto/cnxk/cnxk_se.h
index b07fc22858..ce4c2bb585 100644
--- a/drivers/crypto/cnxk/cnxk_se.h
+++ b/drivers/crypto/cnxk/cnxk_se.h
@@ -148,30 +148,33 @@ cpt_mac_len_verify(struct rte_crypto_auth_xform *auth)
uint16_t mac_len = auth->digest_length;
int ret;
+ if ((auth->algo != RTE_CRYPTO_AUTH_NULL) && (mac_len == 0))
+ return -1;
+
switch (auth->algo) {
case RTE_CRYPTO_AUTH_MD5:
case RTE_CRYPTO_AUTH_MD5_HMAC:
- ret = (mac_len == 16) ? 0 : -1;
+ ret = (mac_len <= 16) ? 0 : -1;
break;
case RTE_CRYPTO_AUTH_SHA1:
case RTE_CRYPTO_AUTH_SHA1_HMAC:
- ret = (mac_len == 20) ? 0 : -1;
+ ret = (mac_len <= 20) ? 0 : -1;
break;
case RTE_CRYPTO_AUTH_SHA224:
case RTE_CRYPTO_AUTH_SHA224_HMAC:
- ret = (mac_len == 28) ? 0 : -1;
+ ret = (mac_len <= 28) ? 0 : -1;
break;
case RTE_CRYPTO_AUTH_SHA256:
case RTE_CRYPTO_AUTH_SHA256_HMAC:
- ret = (mac_len == 32) ? 0 : -1;
+ ret = (mac_len <= 32) ? 0 : -1;
break;
case RTE_CRYPTO_AUTH_SHA384:
case RTE_CRYPTO_AUTH_SHA384_HMAC:
- ret = (mac_len == 48) ? 0 : -1;
+ ret = (mac_len <= 48) ? 0 : -1;
break;
case RTE_CRYPTO_AUTH_SHA512:
case RTE_CRYPTO_AUTH_SHA512_HMAC:
- ret = (mac_len == 64) ? 0 : -1;
+ ret = (mac_len <= 64) ? 0 : -1;
break;
case RTE_CRYPTO_AUTH_NULL:
ret = 0;
@@ -838,7 +841,7 @@ cpt_digest_gen_sg_ver1_prep(uint32_t flags, uint64_t d_lens, struct roc_se_fc_pa
/*GP op header */
cpt_inst_w4.s.opcode_minor = 0;
- cpt_inst_w4.s.param2 = ((uint16_t)hash_type << 8);
+ cpt_inst_w4.s.param2 = ((uint16_t)hash_type << 8) | mac_len;
if (ctx->hmac) {
cpt_inst_w4.s.opcode_major =
ROC_SE_MAJOR_OP_HMAC | ROC_SE_DMA_MODE;
@@ -969,7 +972,7 @@ cpt_digest_gen_sg_ver2_prep(uint32_t flags, uint64_t d_lens, struct roc_se_fc_pa
/*GP op header */
cpt_inst_w4.s.opcode_minor = 0;
- cpt_inst_w4.s.param2 = ((uint16_t)hash_type << 8);
+ cpt_inst_w4.s.param2 = ((uint16_t)hash_type << 8) | mac_len;
if (ctx->hmac) {
cpt_inst_w4.s.opcode_major = ROC_SE_MAJOR_OP_HMAC;
cpt_inst_w4.s.param1 = key_len;
--
2.25.1
^ permalink raw reply [flat|nested] 19+ messages in thread* [PATCH 06/17] crypto/cnxk: add queue pair check to meta set
2022-12-20 14:32 [PATCH 00/17] fixes and improvements to cnxk crytpo PMD Tejasree Kondoj
` (4 preceding siblings ...)
2022-12-20 14:32 ` [PATCH 05/17] crypto/cnxk: support truncated digest length Tejasree Kondoj
@ 2022-12-20 14:32 ` Tejasree Kondoj
2022-12-20 14:32 ` [PATCH 07/17] crypto/cnxk: update crypto completion code handling Tejasree Kondoj
` (11 subsequent siblings)
17 siblings, 0 replies; 19+ messages in thread
From: Tejasree Kondoj @ 2022-12-20 14:32 UTC (permalink / raw)
To: Akhil Goyal
Cc: Volodymyr Fialko, Anoob Joseph, Vidya Sagar Velumuri,
Gowrishankar Muthukrishnan, Aakash Sasidharan, dev
From: Volodymyr Fialko <vfialko@marvell.com>
Check that queue pair provided by metadata information is enabled (added
to crypto adapter).
Signed-off-by: Volodymyr Fialko <vfialko@marvell.com>
---
drivers/crypto/cnxk/cn10k_cryptodev_ops.c | 3 +++
1 file changed, 3 insertions(+)
diff --git a/drivers/crypto/cnxk/cn10k_cryptodev_ops.c b/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
index 2c750d19bf..1caa321112 100644
--- a/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
@@ -296,6 +296,9 @@ cn10k_cpt_crypto_adapter_ev_mdata_set(struct rte_cryptodev *dev __rte_unused, vo
qp_id = ec_mdata->request_info.queue_pair_id;
qp = rte_cryptodevs[cdev_id].data->queue_pairs[qp_id];
+ if (!qp->ca.enabled)
+ return -EINVAL;
+
/* Prepare w2 */
tag_type = qp->ca.vector_sz ? RTE_EVENT_TYPE_CRYPTODEV_VECTOR : RTE_EVENT_TYPE_CRYPTODEV;
rsp_info = &ec_mdata->response_info;
--
2.25.1
^ permalink raw reply [flat|nested] 19+ messages in thread* [PATCH 07/17] crypto/cnxk: update crypto completion code handling
2022-12-20 14:32 [PATCH 00/17] fixes and improvements to cnxk crytpo PMD Tejasree Kondoj
` (5 preceding siblings ...)
2022-12-20 14:32 ` [PATCH 06/17] crypto/cnxk: add queue pair check to meta set Tejasree Kondoj
@ 2022-12-20 14:32 ` Tejasree Kondoj
2022-12-20 14:32 ` [PATCH 08/17] crypto/cnxk: fix incorrect digest for an empty input data Tejasree Kondoj
` (10 subsequent siblings)
17 siblings, 0 replies; 19+ messages in thread
From: Tejasree Kondoj @ 2022-12-20 14:32 UTC (permalink / raw)
To: Akhil Goyal
Cc: Vidya Sagar Velumuri, Anoob Joseph, Gowrishankar Muthukrishnan,
Volodymyr Fialko, Aakash Sasidharan, dev
From: Vidya Sagar Velumuri <vvelumuri@marvell.com>
Update crypto and IPsec completion handling as per microcode version
OCPT-04-IE-IPSEC-MC-30-01-28-00
ci: skip_checkformat
Signed-off-by: Vidya Sagar Velumuri <vvelumuri@marvell.com>
---
drivers/crypto/cnxk/cn10k_cryptodev_ops.c | 18 +++++++++---------
drivers/crypto/cnxk/cn10k_ipsec.c | 7 +++++--
drivers/crypto/cnxk/cn10k_ipsec.h | 1 +
drivers/crypto/cnxk/cn10k_ipsec_la_ops.h | 1 +
4 files changed, 16 insertions(+), 11 deletions(-)
diff --git a/drivers/crypto/cnxk/cn10k_cryptodev_ops.c b/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
index 1caa321112..5a098ffcf2 100644
--- a/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
@@ -801,13 +801,11 @@ cn10k_cpt_sec_post_process(struct rte_crypto_op *cop, struct cpt_cn10k_res_s *re
struct rte_mbuf *mbuf = cop->sym->m_src;
const uint16_t m_len = res->rlen;
- mbuf->data_len = m_len;
- mbuf->pkt_len = m_len;
-
switch (res->uc_compcode) {
case ROC_IE_OT_UCC_SUCCESS:
break;
case ROC_IE_OT_UCC_SUCCESS_PKT_IP_BADCSUM:
+ mbuf->ol_flags &= ~RTE_MBUF_F_RX_IP_CKSUM_GOOD;
mbuf->ol_flags |= RTE_MBUF_F_RX_IP_CKSUM_BAD;
break;
case ROC_IE_OT_UCC_SUCCESS_PKT_L4_GOODCSUM:
@@ -819,15 +817,17 @@ cn10k_cpt_sec_post_process(struct rte_crypto_op *cop, struct cpt_cn10k_res_s *re
RTE_MBUF_F_RX_IP_CKSUM_GOOD;
break;
case ROC_IE_OT_UCC_SUCCESS_PKT_IP_GOODCSUM:
- mbuf->ol_flags |= RTE_MBUF_F_RX_IP_CKSUM_GOOD;
break;
case ROC_IE_OT_UCC_SUCCESS_SA_SOFTEXP_FIRST:
+ case ROC_IE_OT_UCC_SUCCESS_SA_SOFTEXP_AGAIN:
cop->aux_flags = RTE_CRYPTO_OP_AUX_FLAGS_IPSEC_SOFT_EXPIRY;
break;
default:
- plt_dp_err("Success with unknown microcode completion code");
- break;
+ cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
+ return;
}
+ mbuf->data_len = m_len;
+ mbuf->pkt_len = m_len;
}
static inline void
@@ -843,7 +843,7 @@ cn10k_cpt_dequeue_post_process(struct cnxk_cpt_qp *qp,
if (cop->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC &&
cop->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
- if (likely(compcode == CPT_COMP_WARN)) {
+ if (likely(compcode == CPT_COMP_GOOD || compcode == CPT_COMP_WARN)) {
/* Success with additional info */
cn10k_cpt_sec_post_process(cop, res);
} else {
@@ -860,7 +860,7 @@ cn10k_cpt_dequeue_post_process(struct cnxk_cpt_qp *qp,
return;
}
- if (likely(compcode == CPT_COMP_GOOD || compcode == CPT_COMP_WARN)) {
+ if (likely(compcode == CPT_COMP_GOOD)) {
if (unlikely(uc_compcode)) {
if (uc_compcode == ROC_SE_ERR_GC_ICV_MISCOMPARE)
cop->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
@@ -964,7 +964,7 @@ cn10k_cpt_crypto_adapter_vector_dequeue(uintptr_t get_work1)
#ifdef CNXK_CRYPTODEV_DEBUG
res.u64[0] = __atomic_load_n(&vec_infl_req->res.u64[0], __ATOMIC_RELAXED);
- PLT_ASSERT(res.cn10k.compcode == CPT_COMP_WARN);
+ PLT_ASSERT(res.cn10k.compcode == CPT_COMP_GOOD);
PLT_ASSERT(res.cn10k.uc_compcode == 0);
#endif
diff --git a/drivers/crypto/cnxk/cn10k_ipsec.c b/drivers/crypto/cnxk/cn10k_ipsec.c
index 1740a73c36..aafd461436 100644
--- a/drivers/crypto/cnxk/cn10k_ipsec.c
+++ b/drivers/crypto/cnxk/cn10k_ipsec.c
@@ -200,9 +200,12 @@ cn10k_ipsec_inb_sa_create(struct roc_cpt *roc_cpt, struct roc_cpt_lf *lf,
/* Disable IP checksum verification by default */
param1.s.ip_csum_disable = ROC_IE_OT_SA_INNER_PKT_IP_CSUM_DISABLE;
+ /* Set the ip chksum flag in mbuf before enqueue.
+ * Reset the flag in post process in case of errors
+ */
if (ipsec_xfrm->options.ip_csum_enable) {
- param1.s.ip_csum_disable =
- ROC_IE_OT_SA_INNER_PKT_IP_CSUM_ENABLE;
+ param1.s.ip_csum_disable = ROC_IE_OT_SA_INNER_PKT_IP_CSUM_ENABLE;
+ sec_sess->ip_csum = RTE_MBUF_F_RX_IP_CKSUM_GOOD;
}
/* Disable L4 checksum verification by default */
diff --git a/drivers/crypto/cnxk/cn10k_ipsec.h b/drivers/crypto/cnxk/cn10k_ipsec.h
index 044fe33046..23d7a4fac4 100644
--- a/drivers/crypto/cnxk/cn10k_ipsec.h
+++ b/drivers/crypto/cnxk/cn10k_ipsec.h
@@ -33,6 +33,7 @@ struct cn10k_sec_session {
uint16_t max_extended_len;
uint16_t iv_offset;
uint8_t iv_length;
+ uint8_t ip_csum;
bool is_outbound;
/** Queue pair */
struct cnxk_cpt_qp *qp;
diff --git a/drivers/crypto/cnxk/cn10k_ipsec_la_ops.h b/drivers/crypto/cnxk/cn10k_ipsec_la_ops.h
index 084198b5bb..f2761a55a5 100644
--- a/drivers/crypto/cnxk/cn10k_ipsec_la_ops.h
+++ b/drivers/crypto/cnxk/cn10k_ipsec_la_ops.h
@@ -98,6 +98,7 @@ process_inb_sa(struct rte_crypto_op *cop, struct cn10k_sec_session *sess, struct
inst->w4.u64 = sess->inst.w4 | rte_pktmbuf_pkt_len(m_src);
dptr = rte_pktmbuf_mtod(m_src, uint64_t);
inst->dptr = dptr;
+ m_src->ol_flags |= (uint64_t)sess->ip_csum;
return 0;
}
--
2.25.1
^ permalink raw reply [flat|nested] 19+ messages in thread* [PATCH 08/17] crypto/cnxk: fix incorrect digest for an empty input data
2022-12-20 14:32 [PATCH 00/17] fixes and improvements to cnxk crytpo PMD Tejasree Kondoj
` (6 preceding siblings ...)
2022-12-20 14:32 ` [PATCH 07/17] crypto/cnxk: update crypto completion code handling Tejasree Kondoj
@ 2022-12-20 14:32 ` Tejasree Kondoj
2022-12-20 14:32 ` [PATCH 09/17] crypto/cnxk: add CN9K IPsec SG support Tejasree Kondoj
` (9 subsequent siblings)
17 siblings, 0 replies; 19+ messages in thread
From: Tejasree Kondoj @ 2022-12-20 14:32 UTC (permalink / raw)
To: Akhil Goyal
Cc: Gowrishankar Muthukrishnan, Anoob Joseph, Vidya Sagar Velumuri,
Volodymyr Fialko, Aakash Sasidharan, dev
From: Gowrishankar Muthukrishnan <gmuthukrishn@marvell.com>
For an empty input data, digest value is incorrectly generated from
previous auth op, in case executed. This is due to incorrect gather
component used.
Fixes: 7c19abdd0cf1 ("common/cnxk: support 103XX CPT")
Signed-off-by: Anoob Joseph <anoobj@marvell.com>
Signed-off-by: Gowrishankar Muthukrishnan <gmuthukrishn@marvell.com>
---
drivers/crypto/cnxk/cnxk_se.h | 46 ++++++++++++++---------------------
1 file changed, 18 insertions(+), 28 deletions(-)
diff --git a/drivers/crypto/cnxk/cnxk_se.h b/drivers/crypto/cnxk/cnxk_se.h
index ce4c2bb585..2944d3c287 100644
--- a/drivers/crypto/cnxk/cnxk_se.h
+++ b/drivers/crypto/cnxk/cnxk_se.h
@@ -247,7 +247,7 @@ fill_sg_comp_from_iov(struct roc_se_sglist_comp *list, uint32_t i,
uint32_t extra_len = extra_buf ? extra_buf->size : 0;
uint32_t size = *psize;
- for (j = 0; (j < from->buf_cnt) && size; j++) {
+ for (j = 0; j < from->buf_cnt; j++) {
struct roc_se_sglist_comp *to = &list[i >> 2];
uint32_t buf_sz = from->bufs[j].size;
void *vaddr = from->bufs[j].vaddr;
@@ -314,6 +314,9 @@ fill_sg_comp_from_iov(struct roc_se_sglist_comp *list, uint32_t i,
if (extra_offset)
extra_offset -= size;
i++;
+
+ if (unlikely(!size))
+ break;
}
*psize = size;
@@ -370,7 +373,9 @@ fill_sg2_comp_from_iov(struct roc_se_sg2list_comp *list, uint32_t i, struct roc_
uint32_t extra_len = extra_buf ? extra_buf->size : 0;
uint32_t size = *psize;
- for (j = 0; (j < from->buf_cnt) && size; j++) {
+ rte_prefetch2(psize);
+
+ for (j = 0; j < from->buf_cnt; j++) {
struct roc_se_sg2list_comp *to = &list[i / 3];
uint32_t buf_sz = from->bufs[j].size;
void *vaddr = from->bufs[j].vaddr;
@@ -436,6 +441,9 @@ fill_sg2_comp_from_iov(struct roc_se_sg2list_comp *list, uint32_t i, struct roc_
if (extra_offset)
extra_offset -= size;
i++;
+
+ if (unlikely(!size))
+ break;
}
*psize = size;
@@ -887,20 +895,10 @@ cpt_digest_gen_sg_ver1_prep(uint32_t flags, uint64_t d_lens, struct roc_se_fc_pa
/* input data */
size = data_len;
- if (size) {
- i = fill_sg_comp_from_iov(gather_comp, i, params->src_iov, 0,
- &size, NULL, 0);
- if (unlikely(size)) {
- plt_dp_err("Insufficient dst IOV size, short by %dB",
- size);
- return -1;
- }
- } else {
- /*
- * Looks like we need to support zero data
- * gather ptr in case of hash & hmac
- */
- i++;
+ i = fill_sg_comp_from_iov(gather_comp, i, params->src_iov, 0, &size, NULL, 0);
+ if (unlikely(size)) {
+ plt_dp_err("Insufficient dst IOV size, short by %dB", size);
+ return -1;
}
((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
g_size_bytes = ((i + 3) / 4) * sizeof(struct roc_se_sglist_comp);
@@ -1011,18 +1009,10 @@ cpt_digest_gen_sg_ver2_prep(uint32_t flags, uint64_t d_lens, struct roc_se_fc_pa
/* input data */
size = data_len;
- if (size) {
- i = fill_sg2_comp_from_iov(gather_comp, i, params->src_iov, 0, &size, NULL, 0);
- if (unlikely(size)) {
- plt_dp_err("Insufficient dst IOV size, short by %dB", size);
- return -1;
- }
- } else {
- /*
- * Looks like we need to support zero data
- * gather ptr in case of hash & hmac
- */
- i++;
+ i = fill_sg2_comp_from_iov(gather_comp, i, params->src_iov, 0, &size, NULL, 0);
+ if (unlikely(size)) {
+ plt_dp_err("Insufficient dst IOV size, short by %dB", size);
+ return -1;
}
cpt_inst_w5.s.gather_sz = ((i + 2) / 3);
--
2.25.1
^ permalink raw reply [flat|nested] 19+ messages in thread* [PATCH 09/17] crypto/cnxk: add CN9K IPsec SG support
2022-12-20 14:32 [PATCH 00/17] fixes and improvements to cnxk crytpo PMD Tejasree Kondoj
` (7 preceding siblings ...)
2022-12-20 14:32 ` [PATCH 08/17] crypto/cnxk: fix incorrect digest for an empty input data Tejasree Kondoj
@ 2022-12-20 14:32 ` Tejasree Kondoj
2022-12-20 14:32 ` [PATCH 10/17] crypto/cnxk: add support for SHA3 hash Tejasree Kondoj
` (8 subsequent siblings)
17 siblings, 0 replies; 19+ messages in thread
From: Tejasree Kondoj @ 2022-12-20 14:32 UTC (permalink / raw)
To: Akhil Goyal
Cc: Archana Muniganti, Anoob Joseph, Vidya Sagar Velumuri,
Gowrishankar Muthukrishnan, Volodymyr Fialko, Aakash Sasidharan,
dev
From: Archana Muniganti <marchana@marvell.com>
Added IPsec SG support in CN9K
Signed-off-by: Archana Muniganti <marchana@marvell.com>
Signed-off-by: Tejasree Kondoj <ktejasree@marvell.com>
---
drivers/common/cnxk/roc_api.h | 1 +
drivers/common/cnxk/roc_cpt_sg.h | 37 +++
drivers/common/cnxk/roc_ie_on.h | 9 +-
drivers/common/cnxk/roc_se.h | 28 --
drivers/crypto/cnxk/cn9k_cryptodev_ops.c | 75 ++---
drivers/crypto/cnxk/cn9k_ipsec_la_ops.h | 171 ++++++++--
drivers/crypto/cnxk/cnxk_cryptodev_ops.c | 25 +-
drivers/crypto/cnxk/cnxk_cryptodev_ops.h | 19 ++
drivers/crypto/cnxk/cnxk_se.h | 387 +++--------------------
drivers/crypto/cnxk/cnxk_sg.h | 273 ++++++++++++++++
10 files changed, 594 insertions(+), 431 deletions(-)
create mode 100644 drivers/common/cnxk/roc_cpt_sg.h
create mode 100644 drivers/crypto/cnxk/cnxk_sg.h
diff --git a/drivers/common/cnxk/roc_api.h b/drivers/common/cnxk/roc_api.h
index 072f16d77d..14a11321e0 100644
--- a/drivers/common/cnxk/roc_api.h
+++ b/drivers/common/cnxk/roc_api.h
@@ -86,6 +86,7 @@
/* CPT microcode */
#include "roc_ae.h"
#include "roc_ae_fpm_tables.h"
+#include "roc_cpt_sg.h"
#include "roc_ie.h"
#include "roc_ie_on.h"
#include "roc_ie_ot.h"
diff --git a/drivers/common/cnxk/roc_cpt_sg.h b/drivers/common/cnxk/roc_cpt_sg.h
new file mode 100644
index 0000000000..8a97e1aa5b
--- /dev/null
+++ b/drivers/common/cnxk/roc_cpt_sg.h
@@ -0,0 +1,37 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2022 Marvell.
+ */
+
+#ifndef _ROC_CPT_SG_H_
+#define _ROC_CPT_SG_H_
+
+#define ROC_DMA_MODE_SG (1 << 7)
+
+#define ROC_MAX_SG_IN_OUT_CNT 32
+#define ROC_MAX_SG_CNT (ROC_MAX_SG_IN_OUT_CNT / 2)
+
+#define ROC_SG_LIST_HDR_SIZE (8u)
+#define ROC_SG_ENTRY_SIZE sizeof(struct roc_sglist_comp)
+
+struct roc_sglist_comp {
+ union {
+ uint64_t len;
+ struct {
+ uint16_t len[4];
+ } s;
+ } u;
+ uint64_t ptr[4];
+};
+
+struct roc_sg2list_comp {
+ union {
+ uint64_t len;
+ struct {
+ uint16_t len[3];
+ uint16_t valid_segs;
+ } s;
+ } u;
+ uint64_t ptr[3];
+};
+
+#endif /* _ROC_CPT_SG_H_ */
diff --git a/drivers/common/cnxk/roc_ie_on.h b/drivers/common/cnxk/roc_ie_on.h
index 057ff95362..9933ffa148 100644
--- a/drivers/common/cnxk/roc_ie_on.h
+++ b/drivers/common/cnxk/roc_ie_on.h
@@ -25,10 +25,11 @@ enum roc_ie_on_ucc_ipsec {
};
/* Helper macros */
-#define ROC_IE_ON_INB_RPTR_HDR 16
-#define ROC_IE_ON_MAX_IV_LEN 16
-#define ROC_IE_ON_PER_PKT_IV BIT(43)
-#define ROC_IE_ON_INPLACE_BIT BIT(6)
+#define ROC_IE_ON_OUTB_DPTR_HDR 16
+#define ROC_IE_ON_INB_RPTR_HDR 16
+#define ROC_IE_ON_MAX_IV_LEN 16
+#define ROC_IE_ON_PER_PKT_IV BIT(43)
+#define ROC_IE_ON_INPLACE_BIT BIT(6)
enum {
ROC_IE_ON_SA_ENC_NULL = 0,
diff --git a/drivers/common/cnxk/roc_se.h b/drivers/common/cnxk/roc_se.h
index c357c19c0b..a8f0f49479 100644
--- a/drivers/common/cnxk/roc_se.h
+++ b/drivers/common/cnxk/roc_se.h
@@ -27,13 +27,6 @@
#define ROC_SE_MAX_MAC_LEN 64
#define ROC_SE_OFF_CTRL_LEN 8
-#define ROC_SE_DMA_MODE (1 << 7)
-
-#define ROC_SE_MAX_SG_IN_OUT_CNT 32
-#define ROC_SE_MAX_SG_CNT (ROC_SE_MAX_SG_IN_OUT_CNT / 2)
-
-#define ROC_SE_SG_LIST_HDR_SIZE (8u)
-#define ROC_SE_SG_ENTRY_SIZE sizeof(struct roc_se_sglist_comp)
#define ROC_SE_ZS_EA 0x1
#define ROC_SE_ZS_IA 0x2
@@ -173,27 +166,6 @@ typedef enum {
ROC_SE_PDCP_MAC_LEN_128_BIT = 0x3
} roc_se_pdcp_mac_len_type;
-struct roc_se_sglist_comp {
- union {
- uint64_t len;
- struct {
- uint16_t len[4];
- } s;
- } u;
- uint64_t ptr[4];
-};
-
-struct roc_se_sg2list_comp {
- union {
- uint64_t len;
- struct {
- uint16_t len[3];
- uint16_t valid_segs;
- } s;
- } u;
- uint64_t ptr[3];
-};
-
struct roc_se_enc_context {
uint64_t iv_source : 1;
uint64_t aes_key : 2;
diff --git a/drivers/crypto/cnxk/cn9k_cryptodev_ops.c b/drivers/crypto/cnxk/cn9k_cryptodev_ops.c
index 04c004bc7a..cfe1e08dff 100644
--- a/drivers/crypto/cnxk/cn9k_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cn9k_cryptodev_ops.c
@@ -18,13 +18,11 @@
#include "cnxk_se.h"
static __rte_always_inline int __rte_hot
-cn9k_cpt_sec_inst_fill(struct rte_crypto_op *op,
- struct cpt_inflight_req *infl_req,
- struct cpt_inst_s *inst)
+cn9k_cpt_sec_inst_fill(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,
+ struct cpt_inflight_req *infl_req, struct cpt_inst_s *inst)
{
struct rte_crypto_sym_op *sym_op = op->sym;
struct cn9k_sec_session *sec_sess;
- int ret;
sec_sess = (struct cn9k_sec_session *)(op->sym->session);
@@ -33,22 +31,10 @@ cn9k_cpt_sec_inst_fill(struct rte_crypto_op *op,
return -ENOTSUP;
}
- if (unlikely(!rte_pktmbuf_is_contiguous(sym_op->m_src))) {
- plt_dp_err("Scatter Gather mode is not supported");
- return -ENOTSUP;
- }
-
if (sec_sess->is_outbound)
- ret = process_outb_sa(op, sec_sess, inst);
- else {
- infl_req->op_flags |= CPT_OP_FLAGS_IPSEC_DIR_INBOUND;
- process_inb_sa(op, sec_sess, inst);
- if (unlikely(sec_sess->replay_win_sz))
- infl_req->op_flags |= CPT_OP_FLAGS_IPSEC_INB_REPLAY;
- ret = 0;
- }
-
- return ret;
+ return process_outb_sa(&qp->meta_info, op, sec_sess, inst, infl_req);
+ else
+ return process_inb_sa(&qp->meta_info, op, sec_sess, inst, infl_req);
}
static inline struct cnxk_se_sess *
@@ -94,7 +80,7 @@ cn9k_cpt_inst_prep(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,
ret = cpt_sym_inst_fill(qp, op, sess, infl_req, inst, false);
inst->w7.u64 = sess->cpt_inst_w7;
} else if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION)
- ret = cn9k_cpt_sec_inst_fill(op, infl_req, inst);
+ ret = cn9k_cpt_sec_inst_fill(qp, op, infl_req, inst);
else {
sess = cn9k_cpt_sym_temp_sess_create(qp, op);
if (unlikely(sess == NULL)) {
@@ -522,45 +508,60 @@ cn9k_cpt_sec_post_process(struct rte_crypto_op *cop,
{
struct rte_crypto_sym_op *sym_op = cop->sym;
struct rte_mbuf *m = sym_op->m_src;
+ struct roc_ie_on_inb_hdr *hdr;
struct cn9k_sec_session *priv;
struct rte_ipv6_hdr *ip6;
struct rte_ipv4_hdr *ip;
uint16_t m_len = 0;
- char *data;
if (infl_req->op_flags & CPT_OP_FLAGS_IPSEC_DIR_INBOUND) {
- data = rte_pktmbuf_mtod(m, char *);
- if (unlikely(infl_req->op_flags &
- CPT_OP_FLAGS_IPSEC_INB_REPLAY)) {
+ hdr = (struct roc_ie_on_inb_hdr *)rte_pktmbuf_mtod(m, char *);
+
+ if (likely(m->next == NULL)) {
+ ip = PLT_PTR_ADD(hdr, ROC_IE_ON_INB_RPTR_HDR);
+ } else {
+ ip = (struct rte_ipv4_hdr *)hdr;
+ hdr = infl_req->mdata;
+ }
+
+ if (unlikely(infl_req->op_flags & CPT_OP_FLAGS_IPSEC_INB_REPLAY)) {
int ret;
priv = (struct cn9k_sec_session *)(sym_op->session);
- ret = ipsec_antireplay_check(priv, priv->replay_win_sz,
- (struct roc_ie_on_inb_hdr *)data);
+ ret = ipsec_antireplay_check(priv, priv->replay_win_sz, hdr);
if (unlikely(ret)) {
cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
return;
}
}
- ip = (struct rte_ipv4_hdr *)(data + ROC_IE_ON_INB_RPTR_HDR);
-
- if (((ip->version_ihl & 0xf0) >> RTE_IPV4_IHL_MULTIPLIER) ==
- IPVERSION) {
+ if (((ip->version_ihl & 0xf0) >> RTE_IPV4_IHL_MULTIPLIER) == IPVERSION) {
m_len = rte_be_to_cpu_16(ip->total_length);
} else {
- PLT_ASSERT(((ip->version_ihl & 0xf0) >>
- RTE_IPV4_IHL_MULTIPLIER) == 6);
+ PLT_ASSERT(((ip->version_ihl & 0xf0) >> RTE_IPV4_IHL_MULTIPLIER) == 6);
ip6 = (struct rte_ipv6_hdr *)ip;
- m_len = rte_be_to_cpu_16(ip6->payload_len) +
- sizeof(struct rte_ipv6_hdr);
+ m_len = rte_be_to_cpu_16(ip6->payload_len) + sizeof(struct rte_ipv6_hdr);
}
- m->data_len = m_len;
- m->pkt_len = m_len;
- m->data_off += ROC_IE_ON_INB_RPTR_HDR;
+ if (likely(m->next == NULL)) {
+ m->data_len = m_len;
+ m->pkt_len = m_len;
+
+ m->data_off += ROC_IE_ON_INB_RPTR_HDR;
+ } else {
+ struct rte_mbuf *temp = m;
+ uint8_t m_len_s = m_len;
+
+ while (m_len_s - temp->data_len > 0) {
+ m_len_s -= temp->data_len;
+ temp = temp->next;
+ }
+
+ temp->data_len = m_len_s;
+ m->pkt_len = m_len;
+ }
}
}
diff --git a/drivers/crypto/cnxk/cn9k_ipsec_la_ops.h b/drivers/crypto/cnxk/cn9k_ipsec_la_ops.h
index f1298017ce..3d9c851f10 100644
--- a/drivers/crypto/cnxk/cn9k_ipsec_la_ops.h
+++ b/drivers/crypto/cnxk/cn9k_ipsec_la_ops.h
@@ -10,7 +10,9 @@
#include <rte_security.h>
#include "cn9k_ipsec.h"
+#include "cnxk_cryptodev_ops.h"
#include "cnxk_security_ar.h"
+#include "cnxk_sg.h"
static __rte_always_inline int32_t
ipsec_po_out_rlen_get(struct cn9k_sec_session *sess, uint32_t plen, struct rte_mbuf *m_src)
@@ -58,7 +60,9 @@ ipsec_po_out_rlen_get(struct cn9k_sec_session *sess, uint32_t plen, struct rte_m
}
static __rte_always_inline int
-process_outb_sa(struct rte_crypto_op *cop, struct cn9k_sec_session *sess, struct cpt_inst_s *inst)
+process_outb_sa(struct cpt_qp_meta_info *m_info, struct rte_crypto_op *cop,
+ struct cn9k_sec_session *sess, struct cpt_inst_s *inst,
+ struct cpt_inflight_req *infl_req)
{
const unsigned int hdr_len = sess->custom_hdr_len;
struct rte_crypto_sym_op *sym_op = cop->sym;
@@ -74,24 +78,90 @@ process_outb_sa(struct rte_crypto_op *cop, struct cn9k_sec_session *sess, struct
rlen = ipsec_po_out_rlen_get(sess, pkt_len, m_src);
extend_tail = rlen - dlen;
- if (unlikely(extend_tail > rte_pktmbuf_tailroom(m_src))) {
- plt_dp_err("Not enough tail room (required: %d, available: %d)",
- extend_tail, rte_pktmbuf_tailroom(m_src));
- return -ENOMEM;
- }
+ pkt_len += extend_tail;
- if (unlikely(hdr_len > data_off)) {
- plt_dp_err("Not enough head room (required: %d, available: %d)",
- hdr_len, rte_pktmbuf_headroom(m_src));
- return -ENOMEM;
- }
+ if (likely(m_src->next == NULL)) {
+ if (unlikely(extend_tail > rte_pktmbuf_tailroom(m_src))) {
+ plt_dp_err("Not enough tail room (required: %d, available: %d)",
+ extend_tail, rte_pktmbuf_tailroom(m_src));
+ return -ENOMEM;
+ }
- pkt_len += extend_tail;
+ if (unlikely(hdr_len > data_off)) {
+ plt_dp_err("Not enough head room (required: %d, available: %d)", hdr_len,
+ rte_pktmbuf_headroom(m_src));
+ return -ENOMEM;
+ }
- m_src->data_len = pkt_len;
- m_src->pkt_len = pkt_len;
+ m_src->data_len = pkt_len;
+
+ hdr = PLT_PTR_ADD(m_src->buf_addr, data_off - hdr_len);
+
+ inst->dptr = PLT_U64_CAST(hdr);
+ inst->w4.u64 = sess->inst.w4 | dlen;
+ } else {
+ struct roc_sglist_comp *scatter_comp, *gather_comp;
+ uint32_t g_size_bytes, s_size_bytes;
+ struct rte_mbuf *last_seg;
+ uint8_t *in_buffer;
+ void *m_data;
+ int i;
+
+ last_seg = rte_pktmbuf_lastseg(m_src);
+
+ if (unlikely(extend_tail > rte_pktmbuf_tailroom(last_seg))) {
+ plt_dp_err("Not enough tail room (required: %d, available: %d)",
+ extend_tail, rte_pktmbuf_tailroom(last_seg));
+ return -ENOMEM;
+ }
+
+ m_data = alloc_op_meta(NULL, m_info->mlen, m_info->pool, infl_req);
+ if (unlikely(m_data == NULL)) {
+ plt_dp_err("Error allocating meta buffer for request");
+ return -ENOMEM;
+ }
+
+ hdr = m_data;
+
+ m_data = (uint8_t *)m_data + hdr_len;
+ in_buffer = m_data;
+
+ ((uint16_t *)in_buffer)[0] = 0;
+ ((uint16_t *)in_buffer)[1] = 0;
+
+ /*
+ * Input Gather List
+ */
+ i = 0;
+ gather_comp = (struct roc_sglist_comp *)((uint8_t *)m_data + 8);
+
+ i = fill_sg_comp(gather_comp, i, (uint64_t)hdr, hdr_len);
+ i = fill_ipsec_sg_comp_from_pkt(gather_comp, i, m_src);
+ ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
+
+ g_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);
+
+ /*
+ * output Scatter List
+ */
+ last_seg->data_len += extend_tail;
+
+ i = 0;
+ scatter_comp = (struct roc_sglist_comp *)((uint8_t *)gather_comp + g_size_bytes);
+
+ i = fill_sg_comp(scatter_comp, i, (uint64_t)hdr, hdr_len);
+ i = fill_ipsec_sg_comp_from_pkt(scatter_comp, i, m_src);
+ ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
- hdr = PLT_PTR_ADD(m_src->buf_addr, data_off - hdr_len);
+ s_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);
+
+ dlen = g_size_bytes + s_size_bytes + ROC_SG_LIST_HDR_SIZE;
+
+ inst->dptr = (uint64_t)in_buffer;
+
+ inst->w4.u64 = sess->inst.w4 | dlen;
+ inst->w4.s.opcode_major |= (uint64_t)ROC_DMA_MODE_SG;
+ }
#ifdef LA_IPSEC_DEBUG
if (sess->inst.w4 & ROC_IE_ON_PER_PKT_IV) {
@@ -101,6 +171,7 @@ process_outb_sa(struct rte_crypto_op *cop, struct cn9k_sec_session *sess, struct
}
#endif
+ m_src->pkt_len = pkt_len;
esn = ++sess->esn;
/* Set ESN seq hi */
@@ -114,22 +185,80 @@ process_outb_sa(struct rte_crypto_op *cop, struct cn9k_sec_session *sess, struct
hdr->ip_id = seq_lo;
/* Prepare CPT instruction */
- inst->w4.u64 = sess->inst.w4 | dlen;
- inst->dptr = PLT_U64_CAST(hdr);
inst->w7.u64 = sess->inst.w7;
return 0;
}
-static __rte_always_inline void
-process_inb_sa(struct rte_crypto_op *cop, struct cn9k_sec_session *sess, struct cpt_inst_s *inst)
+static __rte_always_inline int
+process_inb_sa(struct cpt_qp_meta_info *m_info, struct rte_crypto_op *cop, struct cn9k_sec_session *sess, struct cpt_inst_s *inst, struct cpt_inflight_req *infl_req)
{
+ const unsigned int hdr_len = ROC_IE_ON_INB_RPTR_HDR;
struct rte_crypto_sym_op *sym_op = cop->sym;
struct rte_mbuf *m_src = sym_op->m_src;
+ struct roc_ie_on_inb_hdr *hdr;
+ uint32_t dlen;
+
+ infl_req->op_flags |= CPT_OP_FLAGS_IPSEC_DIR_INBOUND;
+ if (likely(m_src->next == NULL)) {
+ dlen = rte_pktmbuf_pkt_len(m_src);
+ inst->dptr = rte_pktmbuf_mtod(m_src, uint64_t);
+ inst->w4.u64 = sess->inst.w4 | dlen;
+ } else {
+ struct roc_sglist_comp *scatter_comp, *gather_comp;
+ uint32_t g_size_bytes, s_size_bytes;
+ uint8_t *in_buffer;
+ void *m_data;
+ int i;
+
+ m_data = alloc_op_meta(NULL, m_info->mlen, m_info->pool, infl_req);
+ if (unlikely(m_data == NULL)) {
+ plt_dp_err("Error allocating meta buffer for request");
+ return -ENOMEM;
+ }
+
+ hdr = m_data;
+
+ m_data = (uint8_t *)m_data + hdr_len;
+ in_buffer = m_data;
+
+ ((uint16_t *)in_buffer)[0] = 0;
+ ((uint16_t *)in_buffer)[1] = 0;
+
+ /*
+ * Input Gather List
+ */
+ i = 0;
+ gather_comp = (struct roc_sglist_comp *)((uint8_t *)m_data + 8);
+ i = fill_ipsec_sg_comp_from_pkt(gather_comp, i, m_src);
+ ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
+
+ g_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);
+
+ /*
+ * Output Scatter List
+ */
+ i = 0;
+ scatter_comp = (struct roc_sglist_comp *)((uint8_t *)gather_comp + g_size_bytes);
+ i = fill_sg_comp(scatter_comp, i, (uint64_t)hdr, hdr_len);
+ i = fill_ipsec_sg_comp_from_pkt(scatter_comp, i, m_src);
+ ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
+
+ s_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);
+
+ dlen = g_size_bytes + s_size_bytes + ROC_SG_LIST_HDR_SIZE;
+
+ inst->dptr = (uint64_t)in_buffer;
+ inst->w4.u64 = sess->inst.w4 | dlen;
+ inst->w4.s.opcode_major |= (uint64_t)ROC_DMA_MODE_SG;
+ }
/* Prepare CPT instruction */
- inst->w4.u64 = sess->inst.w4 | rte_pktmbuf_pkt_len(m_src);
- inst->dptr = rte_pktmbuf_mtod(m_src, uint64_t);
inst->w7.u64 = sess->inst.w7;
+
+ if (unlikely(sess->replay_win_sz))
+ infl_req->op_flags |= CPT_OP_FLAGS_IPSEC_INB_REPLAY;
+
+ return 0;
}
#endif /* __CN9K_IPSEC_LA_OPS_H__ */
diff --git a/drivers/crypto/cnxk/cnxk_cryptodev_ops.c b/drivers/crypto/cnxk/cnxk_cryptodev_ops.c
index a9c42205e6..eb2ed0d103 100644
--- a/drivers/crypto/cnxk/cnxk_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cnxk_cryptodev_ops.c
@@ -30,10 +30,22 @@ cnxk_cpt_get_mlen(void)
/* For PDCP_CHAIN passthrough alignment */
len += 8;
len += ROC_SE_OFF_CTRL_LEN + ROC_CPT_AES_CBC_IV_LEN;
- len += RTE_ALIGN_CEIL(
- (ROC_SE_SG_LIST_HDR_SIZE +
- (RTE_ALIGN_CEIL(ROC_SE_MAX_SG_IN_OUT_CNT, 4) >> 2) * ROC_SE_SG_ENTRY_SIZE),
- 8);
+ len += RTE_ALIGN_CEIL((ROC_SG_LIST_HDR_SIZE +
+ (RTE_ALIGN_CEIL(ROC_MAX_SG_IN_OUT_CNT, 4) >> 2) * ROC_SG_ENTRY_SIZE),
+ 8);
+
+ return len;
+}
+
+static int
+cnxk_cpt_sec_get_mlen(void)
+{
+ uint32_t len;
+
+ len = ROC_IE_ON_OUTB_DPTR_HDR + ROC_IE_ON_MAX_IV_LEN;
+ len += RTE_ALIGN_CEIL((ROC_SG_LIST_HDR_SIZE +
+ (RTE_ALIGN_CEIL(ROC_MAX_SG_IN_OUT_CNT, 4) >> 2) * ROC_SG_ENTRY_SIZE),
+ 8);
return len;
}
@@ -196,6 +208,11 @@ cnxk_cpt_metabuf_mempool_create(const struct rte_cryptodev *dev,
mlen = cnxk_cpt_get_mlen();
}
+ if (dev->feature_flags & RTE_CRYPTODEV_FF_SECURITY) {
+ /* Get meta len for security operations */
+ mlen = cnxk_cpt_sec_get_mlen();
+ }
+
if (dev->feature_flags & RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO) {
/* Get meta len required for asymmetric operations */
diff --git a/drivers/crypto/cnxk/cnxk_cryptodev_ops.h b/drivers/crypto/cnxk/cnxk_cryptodev_ops.h
index 13c90444d6..5153d334ba 100644
--- a/drivers/crypto/cnxk/cnxk_cryptodev_ops.h
+++ b/drivers/crypto/cnxk/cnxk_cryptodev_ops.h
@@ -158,4 +158,23 @@ pending_queue_free_cnt(uint64_t head, uint64_t tail, const uint64_t mask)
return mask - pending_queue_infl_cnt(head, tail, mask);
}
+static __rte_always_inline void *
+alloc_op_meta(struct roc_se_buf_ptr *buf, int32_t len, struct rte_mempool *cpt_meta_pool,
+ struct cpt_inflight_req *infl_req)
+{
+ uint8_t *mdata;
+
+ if (unlikely(rte_mempool_get(cpt_meta_pool, (void **)&mdata) < 0))
+ return NULL;
+
+ if (likely(buf)) {
+ buf->vaddr = mdata;
+ buf->size = len;
+ }
+
+ infl_req->mdata = mdata;
+ infl_req->op_flags |= CPT_OP_FLAGS_METABUF;
+
+ return mdata;
+}
#endif /* _CNXK_CRYPTODEV_OPS_H_ */
diff --git a/drivers/crypto/cnxk/cnxk_se.h b/drivers/crypto/cnxk/cnxk_se.h
index 2944d3c287..88049ac431 100644
--- a/drivers/crypto/cnxk/cnxk_se.h
+++ b/drivers/crypto/cnxk/cnxk_se.h
@@ -8,11 +8,12 @@
#include "cnxk_cryptodev.h"
#include "cnxk_cryptodev_ops.h"
+#include "cnxk_sg.h"
#define SRC_IOV_SIZE \
- (sizeof(struct roc_se_iov_ptr) + (sizeof(struct roc_se_buf_ptr) * ROC_SE_MAX_SG_CNT))
+ (sizeof(struct roc_se_iov_ptr) + (sizeof(struct roc_se_buf_ptr) * ROC_MAX_SG_CNT))
#define DST_IOV_SIZE \
- (sizeof(struct roc_se_iov_ptr) + (sizeof(struct roc_se_buf_ptr) * ROC_SE_MAX_SG_CNT))
+ (sizeof(struct roc_se_iov_ptr) + (sizeof(struct roc_se_buf_ptr) * ROC_MAX_SG_CNT))
enum cpt_dp_thread_type {
CPT_DP_THREAD_TYPE_FC_CHAIN = 0x1,
@@ -193,272 +194,14 @@ cpt_fc_salt_update(struct roc_se_ctx *se_ctx, uint8_t *salt)
memcpy(fctx->enc.encr_iv, salt, 4);
}
-static __rte_always_inline uint32_t
-fill_sg_comp(struct roc_se_sglist_comp *list, uint32_t i, phys_addr_t dma_addr,
- uint32_t size)
-{
- struct roc_se_sglist_comp *to = &list[i >> 2];
-
- to->u.s.len[i % 4] = rte_cpu_to_be_16(size);
- to->ptr[i % 4] = rte_cpu_to_be_64(dma_addr);
- i++;
- return i;
-}
-
-static __rte_always_inline uint32_t
-fill_sg_comp_from_buf(struct roc_se_sglist_comp *list, uint32_t i,
- struct roc_se_buf_ptr *from)
-{
- struct roc_se_sglist_comp *to = &list[i >> 2];
-
- to->u.s.len[i % 4] = rte_cpu_to_be_16(from->size);
- to->ptr[i % 4] = rte_cpu_to_be_64((uint64_t)from->vaddr);
- i++;
- return i;
-}
-
-static __rte_always_inline uint32_t
-fill_sg_comp_from_buf_min(struct roc_se_sglist_comp *list, uint32_t i,
- struct roc_se_buf_ptr *from, uint32_t *psize)
-{
- struct roc_se_sglist_comp *to = &list[i >> 2];
- uint32_t size = *psize;
- uint32_t e_len;
-
- e_len = (size > from->size) ? from->size : size;
- to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len);
- to->ptr[i % 4] = rte_cpu_to_be_64((uint64_t)from->vaddr);
- *psize -= e_len;
- i++;
- return i;
-}
-
-/*
- * This fills the MC expected SGIO list
- * from IOV given by user.
- */
-static __rte_always_inline uint32_t
-fill_sg_comp_from_iov(struct roc_se_sglist_comp *list, uint32_t i,
- struct roc_se_iov_ptr *from, uint32_t from_offset,
- uint32_t *psize, struct roc_se_buf_ptr *extra_buf,
- uint32_t extra_offset)
-{
- int32_t j;
- uint32_t extra_len = extra_buf ? extra_buf->size : 0;
- uint32_t size = *psize;
-
- for (j = 0; j < from->buf_cnt; j++) {
- struct roc_se_sglist_comp *to = &list[i >> 2];
- uint32_t buf_sz = from->bufs[j].size;
- void *vaddr = from->bufs[j].vaddr;
- uint64_t e_vaddr;
- uint32_t e_len;
-
- if (unlikely(from_offset)) {
- if (from_offset >= buf_sz) {
- from_offset -= buf_sz;
- continue;
- }
- e_vaddr = (uint64_t)vaddr + from_offset;
- e_len = (size > (buf_sz - from_offset)) ?
- (buf_sz - from_offset) :
- size;
- from_offset = 0;
- } else {
- e_vaddr = (uint64_t)vaddr;
- e_len = (size > buf_sz) ? buf_sz : size;
- }
-
- to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len);
- to->ptr[i % 4] = rte_cpu_to_be_64(e_vaddr);
-
- if (extra_len && (e_len >= extra_offset)) {
- /* Break the data at given offset */
- uint32_t next_len = e_len - extra_offset;
- uint64_t next_vaddr = e_vaddr + extra_offset;
-
- if (!extra_offset) {
- i--;
- } else {
- e_len = extra_offset;
- size -= e_len;
- to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len);
- }
-
- extra_len = RTE_MIN(extra_len, size);
- /* Insert extra data ptr */
- if (extra_len) {
- i++;
- to = &list[i >> 2];
- to->u.s.len[i % 4] =
- rte_cpu_to_be_16(extra_len);
- to->ptr[i % 4] = rte_cpu_to_be_64(
- (uint64_t)extra_buf->vaddr);
- size -= extra_len;
- }
-
- next_len = RTE_MIN(next_len, size);
- /* insert the rest of the data */
- if (next_len) {
- i++;
- to = &list[i >> 2];
- to->u.s.len[i % 4] = rte_cpu_to_be_16(next_len);
- to->ptr[i % 4] = rte_cpu_to_be_64(next_vaddr);
- size -= next_len;
- }
- extra_len = 0;
-
- } else {
- size -= e_len;
- }
- if (extra_offset)
- extra_offset -= size;
- i++;
-
- if (unlikely(!size))
- break;
- }
-
- *psize = size;
- return (uint32_t)i;
-}
-
-static __rte_always_inline uint32_t
-fill_sg2_comp(struct roc_se_sg2list_comp *list, uint32_t i, phys_addr_t dma_addr, uint32_t size)
-{
- struct roc_se_sg2list_comp *to = &list[i / 3];
-
- to->u.s.len[i % 3] = (size);
- to->ptr[i % 3] = (dma_addr);
- to->u.s.valid_segs = (i % 3) + 1;
- i++;
- return i;
-}
-
-static __rte_always_inline uint32_t
-fill_sg2_comp_from_buf(struct roc_se_sg2list_comp *list, uint32_t i, struct roc_se_buf_ptr *from)
-{
- struct roc_se_sg2list_comp *to = &list[i / 3];
-
- to->u.s.len[i % 3] = (from->size);
- to->ptr[i % 3] = ((uint64_t)from->vaddr);
- to->u.s.valid_segs = (i % 3) + 1;
- i++;
- return i;
-}
-
-static __rte_always_inline uint32_t
-fill_sg2_comp_from_buf_min(struct roc_se_sg2list_comp *list, uint32_t i,
- struct roc_se_buf_ptr *from, uint32_t *psize)
-{
- struct roc_se_sg2list_comp *to = &list[i / 3];
- uint32_t size = *psize;
- uint32_t e_len;
-
- e_len = (size > from->size) ? from->size : size;
- to->u.s.len[i % 3] = (e_len);
- to->ptr[i % 3] = ((uint64_t)from->vaddr);
- to->u.s.valid_segs = (i % 3) + 1;
- *psize -= e_len;
- i++;
- return i;
-}
-
-static __rte_always_inline uint32_t
-fill_sg2_comp_from_iov(struct roc_se_sg2list_comp *list, uint32_t i, struct roc_se_iov_ptr *from,
- uint32_t from_offset, uint32_t *psize, struct roc_se_buf_ptr *extra_buf,
- uint32_t extra_offset)
-{
- int32_t j;
- uint32_t extra_len = extra_buf ? extra_buf->size : 0;
- uint32_t size = *psize;
-
- rte_prefetch2(psize);
-
- for (j = 0; j < from->buf_cnt; j++) {
- struct roc_se_sg2list_comp *to = &list[i / 3];
- uint32_t buf_sz = from->bufs[j].size;
- void *vaddr = from->bufs[j].vaddr;
- uint64_t e_vaddr;
- uint32_t e_len;
-
- if (unlikely(from_offset)) {
- if (from_offset >= buf_sz) {
- from_offset -= buf_sz;
- continue;
- }
- e_vaddr = (uint64_t)vaddr + from_offset;
- e_len = (size > (buf_sz - from_offset)) ? (buf_sz - from_offset) : size;
- from_offset = 0;
- } else {
- e_vaddr = (uint64_t)vaddr;
- e_len = (size > buf_sz) ? buf_sz : size;
- }
-
- to->u.s.len[i % 3] = (e_len);
- to->ptr[i % 3] = (e_vaddr);
- to->u.s.valid_segs = (i % 3) + 1;
-
- if (extra_len && (e_len >= extra_offset)) {
- /* Break the data at given offset */
- uint32_t next_len = e_len - extra_offset;
- uint64_t next_vaddr = e_vaddr + extra_offset;
-
- if (!extra_offset) {
- i--;
- } else {
- e_len = extra_offset;
- size -= e_len;
- to->u.s.len[i % 3] = (e_len);
- }
-
- extra_len = RTE_MIN(extra_len, size);
- /* Insert extra data ptr */
- if (extra_len) {
- i++;
- to = &list[i / 3];
- to->u.s.len[i % 3] = (extra_len);
- to->ptr[i % 3] = ((uint64_t)extra_buf->vaddr);
- to->u.s.valid_segs = (i % 3) + 1;
- size -= extra_len;
- }
-
- next_len = RTE_MIN(next_len, size);
- /* insert the rest of the data */
- if (next_len) {
- i++;
- to = &list[i / 3];
- to->u.s.len[i % 3] = (next_len);
- to->ptr[i % 3] = (next_vaddr);
- to->u.s.valid_segs = (i % 3) + 1;
- size -= next_len;
- }
- extra_len = 0;
-
- } else {
- size -= e_len;
- }
- if (extra_offset)
- extra_offset -= size;
- i++;
-
- if (unlikely(!size))
- break;
- }
-
- *psize = size;
- return (uint32_t)i;
-}
-
static __rte_always_inline int
sg_inst_prep(struct roc_se_fc_params *params, struct cpt_inst_s *inst, uint64_t offset_ctrl,
uint8_t *iv_s, int iv_len, uint8_t pack_iv, uint8_t pdcp_alg_type, int32_t inputlen,
int32_t outputlen, uint32_t passthrough_len, uint32_t req_flags, int pdcp_flag,
int decrypt)
{
+ struct roc_sglist_comp *gather_comp, *scatter_comp;
void *m_vaddr = params->meta_buf.vaddr;
- struct roc_se_sglist_comp *gather_comp;
- struct roc_se_sglist_comp *scatter_comp;
struct roc_se_buf_ptr *aad_buf = NULL;
uint32_t mac_len = 0, aad_len = 0;
struct roc_se_ctx *se_ctx;
@@ -485,7 +228,7 @@ sg_inst_prep(struct roc_se_fc_params *params, struct cpt_inst_s *inst, uint64_t
m_vaddr = (uint8_t *)m_vaddr + ROC_SE_OFF_CTRL_LEN + RTE_ALIGN_CEIL(iv_len, 8);
- inst->w4.s.opcode_major |= (uint64_t)ROC_SE_DMA_MODE;
+ inst->w4.s.opcode_major |= (uint64_t)ROC_DMA_MODE_SG;
/* iv offset is 0 */
*offset_vaddr = offset_ctrl;
@@ -503,7 +246,7 @@ sg_inst_prep(struct roc_se_fc_params *params, struct cpt_inst_s *inst, uint64_t
/* DPTR has SG list */
/* TODO Add error check if space will be sufficient */
- gather_comp = (struct roc_se_sglist_comp *)((uint8_t *)m_vaddr + 8);
+ gather_comp = (struct roc_sglist_comp *)((uint8_t *)m_vaddr + 8);
/*
* Input Gather List
@@ -562,13 +305,13 @@ sg_inst_prep(struct roc_se_fc_params *params, struct cpt_inst_s *inst, uint64_t
((uint16_t *)in_buffer)[1] = 0;
((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
- g_size_bytes = ((i + 3) / 4) * sizeof(struct roc_se_sglist_comp);
+ g_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);
/*
* Output Scatter List
*/
i = 0;
- scatter_comp = (struct roc_se_sglist_comp *)((uint8_t *)gather_comp + g_size_bytes);
+ scatter_comp = (struct roc_sglist_comp *)((uint8_t *)gather_comp + g_size_bytes);
if (zsk_flags == 0x1) {
/* IV in SLIST only for EEA3 & UEA2 or for F8 */
@@ -626,9 +369,9 @@ sg_inst_prep(struct roc_se_fc_params *params, struct cpt_inst_s *inst, uint64_t
}
}
((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
- s_size_bytes = ((i + 3) / 4) * sizeof(struct roc_se_sglist_comp);
+ s_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);
- size = g_size_bytes + s_size_bytes + ROC_SE_SG_LIST_HDR_SIZE;
+ size = g_size_bytes + s_size_bytes + ROC_SG_LIST_HDR_SIZE;
/* This is DPTR len in case of SG mode */
inst->w4.s.dlen = size;
@@ -643,18 +386,17 @@ sg2_inst_prep(struct roc_se_fc_params *params, struct cpt_inst_s *inst, uint64_t
int32_t outputlen, uint32_t passthrough_len, uint32_t req_flags, int pdcp_flag,
int decrypt)
{
+ struct roc_sg2list_comp *gather_comp, *scatter_comp;
void *m_vaddr = params->meta_buf.vaddr;
- uint32_t i, g_size_bytes;
- struct roc_se_sg2list_comp *gather_comp;
- struct roc_se_sg2list_comp *scatter_comp;
struct roc_se_buf_ptr *aad_buf = NULL;
+ uint32_t mac_len = 0, aad_len = 0;
+ union cpt_inst_w5 cpt_inst_w5;
+ union cpt_inst_w6 cpt_inst_w6;
struct roc_se_ctx *se_ctx;
+ uint32_t i, g_size_bytes;
uint64_t *offset_vaddr;
- uint32_t mac_len = 0, aad_len = 0;
int zsk_flags;
uint32_t size;
- union cpt_inst_w5 cpt_inst_w5;
- union cpt_inst_w6 cpt_inst_w6;
uint8_t *iv_d;
se_ctx = params->ctx;
@@ -672,7 +414,7 @@ sg2_inst_prep(struct roc_se_fc_params *params, struct cpt_inst_s *inst, uint64_t
m_vaddr = (uint8_t *)m_vaddr + ROC_SE_OFF_CTRL_LEN + RTE_ALIGN_CEIL(iv_len, 8);
- inst->w4.s.opcode_major |= (uint64_t)ROC_SE_DMA_MODE;
+ inst->w4.s.opcode_major |= (uint64_t)ROC_DMA_MODE_SG;
/* iv offset is 0 */
*offset_vaddr = offset_ctrl;
@@ -689,7 +431,7 @@ sg2_inst_prep(struct roc_se_fc_params *params, struct cpt_inst_s *inst, uint64_t
/* DPTR has SG list */
/* TODO Add error check if space will be sufficient */
- gather_comp = (struct roc_se_sg2list_comp *)((uint8_t *)m_vaddr);
+ gather_comp = (struct roc_sg2list_comp *)((uint8_t *)m_vaddr);
/*
* Input Gather List
@@ -746,13 +488,13 @@ sg2_inst_prep(struct roc_se_fc_params *params, struct cpt_inst_s *inst, uint64_t
cpt_inst_w5.s.gather_sz = ((i + 2) / 3);
- g_size_bytes = ((i + 2) / 3) * sizeof(struct roc_se_sg2list_comp);
+ g_size_bytes = ((i + 2) / 3) * sizeof(struct roc_sg2list_comp);
/*
* Output Scatter List
*/
i = 0;
- scatter_comp = (struct roc_se_sg2list_comp *)((uint8_t *)gather_comp + g_size_bytes);
+ scatter_comp = (struct roc_sg2list_comp *)((uint8_t *)gather_comp + g_size_bytes);
if (zsk_flags == 0x1) {
/* IV in SLIST only for EEA3 & UEA2 or for F8 */
@@ -829,16 +571,15 @@ static __rte_always_inline int
cpt_digest_gen_sg_ver1_prep(uint32_t flags, uint64_t d_lens, struct roc_se_fc_params *params,
struct cpt_inst_s *inst)
{
+ struct roc_sglist_comp *gather_comp, *scatter_comp;
void *m_vaddr = params->meta_buf.vaddr;
- uint32_t size, i;
+ uint32_t g_size_bytes, s_size_bytes;
uint16_t data_len, mac_len, key_len;
+ union cpt_inst_w4 cpt_inst_w4;
roc_se_auth_type hash_type;
struct roc_se_ctx *ctx;
- struct roc_se_sglist_comp *gather_comp;
- struct roc_se_sglist_comp *scatter_comp;
uint8_t *in_buffer;
- uint32_t g_size_bytes, s_size_bytes;
- union cpt_inst_w4 cpt_inst_w4;
+ uint32_t size, i;
ctx = params->ctx;
@@ -851,13 +592,11 @@ cpt_digest_gen_sg_ver1_prep(uint32_t flags, uint64_t d_lens, struct roc_se_fc_pa
cpt_inst_w4.s.opcode_minor = 0;
cpt_inst_w4.s.param2 = ((uint16_t)hash_type << 8) | mac_len;
if (ctx->hmac) {
- cpt_inst_w4.s.opcode_major =
- ROC_SE_MAJOR_OP_HMAC | ROC_SE_DMA_MODE;
+ cpt_inst_w4.s.opcode_major = ROC_SE_MAJOR_OP_HMAC | ROC_DMA_MODE_SG;
cpt_inst_w4.s.param1 = key_len;
cpt_inst_w4.s.dlen = data_len + RTE_ALIGN_CEIL(key_len, 8);
} else {
- cpt_inst_w4.s.opcode_major =
- ROC_SE_MAJOR_OP_HASH | ROC_SE_DMA_MODE;
+ cpt_inst_w4.s.opcode_major = ROC_SE_MAJOR_OP_HASH | ROC_DMA_MODE_SG;
cpt_inst_w4.s.param1 = 0;
cpt_inst_w4.s.dlen = data_len;
}
@@ -878,7 +617,7 @@ cpt_digest_gen_sg_ver1_prep(uint32_t flags, uint64_t d_lens, struct roc_se_fc_pa
((uint16_t *)in_buffer)[1] = 0;
/* TODO Add error check if space will be sufficient */
- gather_comp = (struct roc_se_sglist_comp *)((uint8_t *)m_vaddr + 8);
+ gather_comp = (struct roc_sglist_comp *)((uint8_t *)m_vaddr + 8);
/*
* Input gather list
@@ -901,15 +640,14 @@ cpt_digest_gen_sg_ver1_prep(uint32_t flags, uint64_t d_lens, struct roc_se_fc_pa
return -1;
}
((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
- g_size_bytes = ((i + 3) / 4) * sizeof(struct roc_se_sglist_comp);
+ g_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);
/*
* Output Gather list
*/
i = 0;
- scatter_comp = (struct roc_se_sglist_comp *)((uint8_t *)gather_comp +
- g_size_bytes);
+ scatter_comp = (struct roc_sglist_comp *)((uint8_t *)gather_comp + g_size_bytes);
if (flags & ROC_SE_VALID_MAC_BUF) {
if (unlikely(params->mac_buf.size < mac_len)) {
@@ -932,9 +670,9 @@ cpt_digest_gen_sg_ver1_prep(uint32_t flags, uint64_t d_lens, struct roc_se_fc_pa
}
((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
- s_size_bytes = ((i + 3) / 4) * sizeof(struct roc_se_sglist_comp);
+ s_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);
- size = g_size_bytes + s_size_bytes + ROC_SE_SG_LIST_HDR_SIZE;
+ size = g_size_bytes + s_size_bytes + ROC_SG_LIST_HDR_SIZE;
/* This is DPTR len in case of SG mode */
cpt_inst_w4.s.dlen = size;
@@ -949,17 +687,16 @@ static __rte_always_inline int
cpt_digest_gen_sg_ver2_prep(uint32_t flags, uint64_t d_lens, struct roc_se_fc_params *params,
struct cpt_inst_s *inst)
{
+ struct roc_sg2list_comp *gather_comp, *scatter_comp;
void *m_vaddr = params->meta_buf.vaddr;
- uint32_t size, i;
uint16_t data_len, mac_len, key_len;
- roc_se_auth_type hash_type;
- struct roc_se_ctx *ctx;
- struct roc_se_sg2list_comp *gather_comp;
- struct roc_se_sg2list_comp *scatter_comp;
+ union cpt_inst_w4 cpt_inst_w4;
union cpt_inst_w5 cpt_inst_w5;
union cpt_inst_w6 cpt_inst_w6;
+ roc_se_auth_type hash_type;
+ struct roc_se_ctx *ctx;
uint32_t g_size_bytes;
- union cpt_inst_w4 cpt_inst_w4;
+ uint32_t size, i;
ctx = params->ctx;
@@ -993,7 +730,7 @@ cpt_digest_gen_sg_ver2_prep(uint32_t flags, uint64_t d_lens, struct roc_se_fc_pa
/* DPTR has SG list */
/* TODO Add error check if space will be sufficient */
- gather_comp = (struct roc_se_sg2list_comp *)((uint8_t *)m_vaddr + 0);
+ gather_comp = (struct roc_sg2list_comp *)((uint8_t *)m_vaddr + 0);
/*
* Input gather list
@@ -1016,14 +753,14 @@ cpt_digest_gen_sg_ver2_prep(uint32_t flags, uint64_t d_lens, struct roc_se_fc_pa
}
cpt_inst_w5.s.gather_sz = ((i + 2) / 3);
- g_size_bytes = ((i + 2) / 3) * sizeof(struct roc_se_sg2list_comp);
+ g_size_bytes = ((i + 2) / 3) * sizeof(struct roc_sg2list_comp);
/*
* Output Gather list
*/
i = 0;
- scatter_comp = (struct roc_se_sg2list_comp *)((uint8_t *)gather_comp + g_size_bytes);
+ scatter_comp = (struct roc_sg2list_comp *)((uint8_t *)gather_comp + g_size_bytes);
if (flags & ROC_SE_VALID_MAC_BUF) {
if (unlikely(params->mac_buf.size < mac_len)) {
@@ -1482,8 +1219,7 @@ cpt_pdcp_chain_alg_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
pdcp_iv_copy(iv_d, auth_iv, pdcp_auth_alg, pack_iv);
} else {
-
- struct roc_se_sglist_comp *scatter_comp, *gather_comp;
+ struct roc_sglist_comp *scatter_comp, *gather_comp;
void *m_vaddr = params->meta_buf.vaddr;
uint32_t i, g_size_bytes, s_size_bytes;
uint8_t *in_buffer;
@@ -1494,7 +1230,7 @@ cpt_pdcp_chain_alg_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
m_vaddr = PLT_PTR_ADD(m_vaddr, ROC_SE_OFF_CTRL_LEN + RTE_ALIGN_CEIL(hdr_len, 8));
- cpt_inst_w4.s.opcode_major |= (uint64_t)ROC_SE_DMA_MODE;
+ cpt_inst_w4.s.opcode_major |= (uint64_t)ROC_DMA_MODE_SG;
/* DPTR has SG list */
in_buffer = m_vaddr;
@@ -1502,8 +1238,7 @@ cpt_pdcp_chain_alg_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
((uint16_t *)in_buffer)[0] = 0;
((uint16_t *)in_buffer)[1] = 0;
- gather_comp =
- (struct roc_se_sglist_comp *)((uint8_t *)m_vaddr + 8);
+ gather_comp = (struct roc_sglist_comp *)((uint8_t *)m_vaddr + 8);
/* Input Gather List */
i = 0;
@@ -1536,15 +1271,14 @@ cpt_pdcp_chain_alg_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
}
}
((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
- g_size_bytes =
- ((i + 3) / 4) * sizeof(struct roc_se_sglist_comp);
+ g_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);
/*
* Output Scatter List
*/
i = 0;
- scatter_comp = (struct roc_se_sglist_comp *)((uint8_t *)gather_comp + g_size_bytes);
+ scatter_comp = (struct roc_sglist_comp *)((uint8_t *)gather_comp + g_size_bytes);
if ((hdr_len)) {
i = fill_sg_comp(scatter_comp, i,
@@ -1573,10 +1307,9 @@ cpt_pdcp_chain_alg_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
}
((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
- s_size_bytes =
- ((i + 3) / 4) * sizeof(struct roc_se_sglist_comp);
+ s_size_bytes = ((i + 3) / 4) * sizeof(struct roc_sglist_comp);
- size = g_size_bytes + s_size_bytes + ROC_SE_SG_LIST_HDR_SIZE;
+ size = g_size_bytes + s_size_bytes + ROC_SG_LIST_HDR_SIZE;
/* This is DPTR len in case of SG mode */
cpt_inst_w4.s.dlen = size;
@@ -1800,7 +1533,7 @@ cpt_kasumi_enc_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
}
}
- cpt_inst_w4.s.opcode_major = ROC_SE_MAJOR_OP_KASUMI | ROC_SE_DMA_MODE;
+ cpt_inst_w4.s.opcode_major = ROC_SE_MAJOR_OP_KASUMI | ROC_DMA_MODE_SG;
/* Indicate ECB/CBC, direction, CTX from CPTR, IV from DPTR */
cpt_inst_w4.s.opcode_minor =
@@ -1841,11 +1574,11 @@ cpt_kasumi_dec_prep(uint64_t d_offs, uint64_t d_lens, struct roc_se_fc_params *p
flags = se_ctx->zsk_flags;
cpt_inst_w4.u64 = 0;
- cpt_inst_w4.s.opcode_major = ROC_SE_MAJOR_OP_KASUMI | ROC_SE_DMA_MODE;
+ cpt_inst_w4.s.opcode_major = ROC_SE_MAJOR_OP_KASUMI | ROC_DMA_MODE_SG;
/* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */
- cpt_inst_w4.s.opcode_minor = ((1 << 6) | (se_ctx->k_ecb << 5) |
- (dir << 4) | (0 << 3) | (flags & 0x7));
+ cpt_inst_w4.s.opcode_minor =
+ ((1 << 6) | (se_ctx->k_ecb << 5) | (dir << 4) | (0 << 3) | (flags & 0x7));
/*
* GP op header, lengths are expected in bits.
@@ -2290,28 +2023,8 @@ fill_sess_gmac(struct rte_crypto_sym_xform *xform, struct cnxk_se_sess *sess)
return 0;
}
-static __rte_always_inline void *
-alloc_op_meta(struct roc_se_buf_ptr *buf, int32_t len,
- struct rte_mempool *cpt_meta_pool,
- struct cpt_inflight_req *infl_req)
-{
- uint8_t *mdata;
-
- if (unlikely(rte_mempool_get(cpt_meta_pool, (void **)&mdata) < 0))
- return NULL;
-
- buf->vaddr = mdata;
- buf->size = len;
-
- infl_req->mdata = mdata;
- infl_req->op_flags |= CPT_OP_FLAGS_METABUF;
-
- return mdata;
-}
-
static __rte_always_inline uint32_t
-prepare_iov_from_pkt(struct rte_mbuf *pkt, struct roc_se_iov_ptr *iovec,
- uint32_t start_offset)
+prepare_iov_from_pkt(struct rte_mbuf *pkt, struct roc_se_iov_ptr *iovec, uint32_t start_offset)
{
uint16_t index = 0;
void *seg_data = NULL;
diff --git a/drivers/crypto/cnxk/cnxk_sg.h b/drivers/crypto/cnxk/cnxk_sg.h
new file mode 100644
index 0000000000..1dfca261cf
--- /dev/null
+++ b/drivers/crypto/cnxk/cnxk_sg.h
@@ -0,0 +1,273 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2022 Marvell.
+ */
+
+#ifndef _CNXK_SG_H_
+#define _CNXK_SG_H_
+
+static __rte_always_inline uint32_t
+fill_sg_comp(struct roc_sglist_comp *list, uint32_t i, phys_addr_t dma_addr, uint32_t size)
+{
+ struct roc_sglist_comp *to = &list[i >> 2];
+
+ to->u.s.len[i % 4] = rte_cpu_to_be_16(size);
+ to->ptr[i % 4] = rte_cpu_to_be_64(dma_addr);
+ return ++i;
+}
+
+static __rte_always_inline uint32_t
+fill_sg_comp_from_buf(struct roc_sglist_comp *list, uint32_t i, struct roc_se_buf_ptr *from)
+{
+ struct roc_sglist_comp *to = &list[i >> 2];
+
+ to->u.s.len[i % 4] = rte_cpu_to_be_16(from->size);
+ to->ptr[i % 4] = rte_cpu_to_be_64((uint64_t)from->vaddr);
+ return ++i;
+}
+
+static __rte_always_inline uint32_t
+fill_sg_comp_from_buf_min(struct roc_sglist_comp *list, uint32_t i, struct roc_se_buf_ptr *from,
+ uint32_t *psize)
+{
+ struct roc_sglist_comp *to = &list[i >> 2];
+ uint32_t size = *psize;
+ uint32_t e_len;
+
+ e_len = RTE_MIN(from->size, size);
+ to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len);
+ to->ptr[i % 4] = rte_cpu_to_be_64((uint64_t)from->vaddr);
+ *psize -= e_len;
+ return ++i;
+}
+
+/*
+ * This fills the MC expected SGIO list
+ * from IOV given by user.
+ */
+static __rte_always_inline uint32_t
+fill_sg_comp_from_iov(struct roc_sglist_comp *list, uint32_t i, struct roc_se_iov_ptr *from,
+ uint32_t from_offset, uint32_t *psize, struct roc_se_buf_ptr *extra_buf,
+ uint32_t extra_offset)
+{
+ uint32_t extra_len = extra_buf ? extra_buf->size : 0;
+ uint32_t size = *psize;
+ int32_t j;
+
+ for (j = 0; j < from->buf_cnt; j++) {
+ struct roc_sglist_comp *to = &list[i >> 2];
+ uint32_t buf_sz = from->bufs[j].size;
+ void *vaddr = from->bufs[j].vaddr;
+ uint64_t e_vaddr;
+ uint32_t e_len;
+
+ if (unlikely(from_offset)) {
+ if (from_offset >= buf_sz) {
+ from_offset -= buf_sz;
+ continue;
+ }
+ e_vaddr = (uint64_t)vaddr + from_offset;
+ e_len = RTE_MIN((buf_sz - from_offset), size);
+ from_offset = 0;
+ } else {
+ e_vaddr = (uint64_t)vaddr;
+ e_len = RTE_MIN(buf_sz, size);
+ }
+
+ to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len);
+ to->ptr[i % 4] = rte_cpu_to_be_64(e_vaddr);
+
+ if (extra_len && (e_len >= extra_offset)) {
+ /* Break the data at given offset */
+ uint32_t next_len = e_len - extra_offset;
+ uint64_t next_vaddr = e_vaddr + extra_offset;
+
+ if (!extra_offset) {
+ i--;
+ } else {
+ e_len = extra_offset;
+ size -= e_len;
+ to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len);
+ }
+
+ extra_len = RTE_MIN(extra_len, size);
+ /* Insert extra data ptr */
+ if (extra_len) {
+ i++;
+ to = &list[i >> 2];
+ to->u.s.len[i % 4] = rte_cpu_to_be_16(extra_len);
+ to->ptr[i % 4] = rte_cpu_to_be_64((uint64_t)extra_buf->vaddr);
+ size -= extra_len;
+ }
+
+ next_len = RTE_MIN(next_len, size);
+ /* insert the rest of the data */
+ if (next_len) {
+ i++;
+ to = &list[i >> 2];
+ to->u.s.len[i % 4] = rte_cpu_to_be_16(next_len);
+ to->ptr[i % 4] = rte_cpu_to_be_64(next_vaddr);
+ size -= next_len;
+ }
+ extra_len = 0;
+
+ } else {
+ size -= e_len;
+ }
+ if (extra_offset)
+ extra_offset -= size;
+ i++;
+
+ if (unlikely(!size))
+ break;
+ }
+
+ *psize = size;
+ return (uint32_t)i;
+}
+
+static __rte_always_inline uint32_t
+fill_ipsec_sg_comp_from_pkt(struct roc_sglist_comp *list, uint32_t i, struct rte_mbuf *pkt)
+{
+ uint32_t buf_sz;
+ void *vaddr;
+
+ while (unlikely(pkt != NULL)) {
+ struct roc_sglist_comp *to = &list[i >> 2];
+ buf_sz = pkt->data_len;
+ vaddr = rte_pktmbuf_mtod(pkt, void *);
+
+ to->u.s.len[i % 4] = rte_cpu_to_be_16(buf_sz);
+ to->ptr[i % 4] = rte_cpu_to_be_64((uint64_t)vaddr);
+
+ pkt = pkt->next;
+ i++;
+ }
+
+ return i;
+}
+
+static __rte_always_inline uint32_t
+fill_sg2_comp(struct roc_sg2list_comp *list, uint32_t i, phys_addr_t dma_addr, uint32_t size)
+{
+ struct roc_sg2list_comp *to = &list[i / 3];
+
+ to->u.s.len[i % 3] = (size);
+ to->ptr[i % 3] = (dma_addr);
+ to->u.s.valid_segs = (i % 3) + 1;
+ return ++i;
+}
+
+static __rte_always_inline uint32_t
+fill_sg2_comp_from_buf(struct roc_sg2list_comp *list, uint32_t i, struct roc_se_buf_ptr *from)
+{
+ struct roc_sg2list_comp *to = &list[i / 3];
+
+ to->u.s.len[i % 3] = (from->size);
+ to->ptr[i % 3] = ((uint64_t)from->vaddr);
+ to->u.s.valid_segs = (i % 3) + 1;
+ return ++i;
+}
+
+static __rte_always_inline uint32_t
+fill_sg2_comp_from_buf_min(struct roc_sg2list_comp *list, uint32_t i, struct roc_se_buf_ptr *from,
+ uint32_t *psize)
+{
+ struct roc_sg2list_comp *to = &list[i / 3];
+ uint32_t size = *psize;
+ uint32_t e_len;
+
+ e_len = RTE_MIN(from->size, size);
+ to->u.s.len[i % 3] = (e_len);
+ to->ptr[i % 3] = ((uint64_t)from->vaddr);
+ to->u.s.valid_segs = (i % 3) + 1;
+ *psize -= e_len;
+ return ++i;
+}
+
+static __rte_always_inline uint32_t
+fill_sg2_comp_from_iov(struct roc_sg2list_comp *list, uint32_t i, struct roc_se_iov_ptr *from,
+ uint32_t from_offset, uint32_t *psize, struct roc_se_buf_ptr *extra_buf,
+ uint32_t extra_offset)
+{
+ uint32_t extra_len = extra_buf ? extra_buf->size : 0;
+ uint32_t size = *psize;
+ int32_t j;
+
+ rte_prefetch2(psize);
+
+ for (j = 0; j < from->buf_cnt; j++) {
+ struct roc_sg2list_comp *to = &list[i / 3];
+ uint32_t buf_sz = from->bufs[j].size;
+ void *vaddr = from->bufs[j].vaddr;
+ uint64_t e_vaddr;
+ uint32_t e_len;
+
+ if (unlikely(from_offset)) {
+ if (from_offset >= buf_sz) {
+ from_offset -= buf_sz;
+ continue;
+ }
+ e_vaddr = (uint64_t)vaddr + from_offset;
+ e_len = RTE_MIN((buf_sz - from_offset), size);
+ from_offset = 0;
+ } else {
+ e_vaddr = (uint64_t)vaddr;
+ e_len = RTE_MIN(buf_sz, size);
+ }
+
+ to->u.s.len[i % 3] = (e_len);
+ to->ptr[i % 3] = (e_vaddr);
+ to->u.s.valid_segs = (i % 3) + 1;
+
+ if (extra_len && (e_len >= extra_offset)) {
+ /* Break the data at given offset */
+ uint32_t next_len = e_len - extra_offset;
+ uint64_t next_vaddr = e_vaddr + extra_offset;
+
+ if (!extra_offset) {
+ i--;
+ } else {
+ e_len = extra_offset;
+ size -= e_len;
+ to->u.s.len[i % 3] = (e_len);
+ }
+
+ extra_len = RTE_MIN(extra_len, size);
+ /* Insert extra data ptr */
+ if (extra_len) {
+ i++;
+ to = &list[i / 3];
+ to->u.s.len[i % 3] = (extra_len);
+ to->ptr[i % 3] = ((uint64_t)extra_buf->vaddr);
+ to->u.s.valid_segs = (i % 3) + 1;
+ size -= extra_len;
+ }
+
+ next_len = RTE_MIN(next_len, size);
+ /* insert the rest of the data */
+ if (next_len) {
+ i++;
+ to = &list[i / 3];
+ to->u.s.len[i % 3] = (next_len);
+ to->ptr[i % 3] = (next_vaddr);
+ to->u.s.valid_segs = (i % 3) + 1;
+ size -= next_len;
+ }
+ extra_len = 0;
+
+ } else {
+ size -= e_len;
+ }
+ if (extra_offset)
+ extra_offset -= size;
+ i++;
+
+ if (unlikely(!size))
+ break;
+ }
+
+ *psize = size;
+ return (uint32_t)i;
+}
+
+#endif /*_CNXK_SG_H_ */
--
2.25.1
^ permalink raw reply [flat|nested] 19+ messages in thread* [PATCH 10/17] crypto/cnxk: add support for SHA3 hash
2022-12-20 14:32 [PATCH 00/17] fixes and improvements to cnxk crytpo PMD Tejasree Kondoj
` (8 preceding siblings ...)
2022-12-20 14:32 ` [PATCH 09/17] crypto/cnxk: add CN9K IPsec SG support Tejasree Kondoj
@ 2022-12-20 14:32 ` Tejasree Kondoj
2022-12-20 14:32 ` [PATCH 11/17] common/cnxk: skip hmac hash precomputation Tejasree Kondoj
` (7 subsequent siblings)
17 siblings, 0 replies; 19+ messages in thread
From: Tejasree Kondoj @ 2022-12-20 14:32 UTC (permalink / raw)
To: Akhil Goyal
Cc: Volodymyr Fialko, Anoob Joseph, Vidya Sagar Velumuri,
Gowrishankar Muthukrishnan, Aakash Sasidharan, dev
From: Volodymyr Fialko <vfialko@marvell.com>
Add support for SHA3 family hash and hmac operations.
Signed-off-by: Volodymyr Fialko <vfialko@marvell.com>
---
doc/guides/cryptodevs/cnxk.rst | 8 +
doc/guides/cryptodevs/features/cn10k.ini | 8 +
doc/guides/cryptodevs/features/cn9k.ini | 8 +
drivers/crypto/cnxk/cnxk_cryptodev.h | 2 +-
.../crypto/cnxk/cnxk_cryptodev_capabilities.c | 164 ++++++++++++++++++
drivers/crypto/cnxk/cnxk_cryptodev_ops.c | 2 +-
drivers/crypto/cnxk/cnxk_se.h | 33 +++-
7 files changed, 222 insertions(+), 3 deletions(-)
diff --git a/doc/guides/cryptodevs/cnxk.rst b/doc/guides/cryptodevs/cnxk.rst
index baf0e3c4fd..9b01e04e5f 100644
--- a/doc/guides/cryptodevs/cnxk.rst
+++ b/doc/guides/cryptodevs/cnxk.rst
@@ -59,6 +59,14 @@ Hash algorithms:
* ``RTE_CRYPTO_AUTH_SHA384_HMAC``
* ``RTE_CRYPTO_AUTH_SHA512``
* ``RTE_CRYPTO_AUTH_SHA512_HMAC``
+* ``RTE_CRYPTO_AUTH_SHA3_224``
+* ``RTE_CRYPTO_AUTH_SHA3_224_HMAC``
+* ``RTE_CRYPTO_AUTH_SHA3_256``
+* ``RTE_CRYPTO_AUTH_SHA3_256_HMAC``
+* ``RTE_CRYPTO_AUTH_SHA3_384``
+* ``RTE_CRYPTO_AUTH_SHA3_384_HMAC``
+* ``RTE_CRYPTO_AUTH_SHA3_512``
+* ``RTE_CRYPTO_AUTH_SHA3_512_HMAC``
* ``RTE_CRYPTO_AUTH_SNOW3G_UIA2``
* ``RTE_CRYPTO_AUTH_ZUC_EIA3``
* ``RTE_CRYPTO_AUTH_AES_CMAC``
diff --git a/doc/guides/cryptodevs/features/cn10k.ini b/doc/guides/cryptodevs/features/cn10k.ini
index 6e4e0e0095..44b61663fc 100644
--- a/doc/guides/cryptodevs/features/cn10k.ini
+++ b/doc/guides/cryptodevs/features/cn10k.ini
@@ -63,6 +63,14 @@ ZUC EIA3 = Y
AES CMAC (128) = Y
AES CMAC (192) = Y
AES CMAC (256) = Y
+SHA3_224 = Y
+SHA3_224 HMAC = Y
+SHA3_256 = Y
+SHA3_256 HMAC = Y
+SHA3_384 = Y
+SHA3_384 HMAC = Y
+SHA3_512 = Y
+SHA3_512 HMAC = Y
;
; Supported AEAD algorithms of 'cn10k' crypto driver.
diff --git a/doc/guides/cryptodevs/features/cn9k.ini b/doc/guides/cryptodevs/features/cn9k.ini
index f9c896f6bd..e7b287db26 100644
--- a/doc/guides/cryptodevs/features/cn9k.ini
+++ b/doc/guides/cryptodevs/features/cn9k.ini
@@ -64,6 +64,14 @@ ZUC EIA3 = Y
AES CMAC (128) = Y
AES CMAC (192) = Y
AES CMAC (256) = Y
+SHA3_224 = Y
+SHA3_224 HMAC = Y
+SHA3_256 = Y
+SHA3_256 HMAC = Y
+SHA3_384 = Y
+SHA3_384 HMAC = Y
+SHA3_512 = Y
+SHA3_512 HMAC = Y
;
; Supported AEAD algorithms of 'cn9k' crypto driver.
diff --git a/drivers/crypto/cnxk/cnxk_cryptodev.h b/drivers/crypto/cnxk/cnxk_cryptodev.h
index 48bd6e144c..8241ee67d0 100644
--- a/drivers/crypto/cnxk/cnxk_cryptodev.h
+++ b/drivers/crypto/cnxk/cnxk_cryptodev.h
@@ -10,7 +10,7 @@
#include "roc_cpt.h"
-#define CNXK_CPT_MAX_CAPS 37
+#define CNXK_CPT_MAX_CAPS 45
#define CNXK_SEC_CRYPTO_MAX_CAPS 16
#define CNXK_SEC_MAX_CAPS 9
#define CNXK_AE_EC_ID_MAX 8
diff --git a/drivers/crypto/cnxk/cnxk_cryptodev_capabilities.c b/drivers/crypto/cnxk/cnxk_cryptodev_capabilities.c
index 9dfbf875ec..b2197a12be 100644
--- a/drivers/crypto/cnxk/cnxk_cryptodev_capabilities.c
+++ b/drivers/crypto/cnxk/cnxk_cryptodev_capabilities.c
@@ -337,6 +337,169 @@ static const struct rte_cryptodev_capabilities caps_sha1_sha2[] = {
},
};
+static const struct rte_cryptodev_capabilities caps_sha3[] = {
+ { /* SHA3_224 */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+ {.auth = {
+ .algo = RTE_CRYPTO_AUTH_SHA3_224,
+ .block_size = 144,
+ .key_size = {
+ .min = 0,
+ .max = 0,
+ .increment = 0
+ },
+ .digest_size = {
+ .min = 1,
+ .max = 28,
+ .increment = 1
+ },
+ }, }
+ }, }
+ },
+ { /* SHA3_224 HMAC */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+ {.auth = {
+ .algo = RTE_CRYPTO_AUTH_SHA3_224_HMAC,
+ .block_size = 144,
+ .key_size = {
+ .min = 1,
+ .max = 1024,
+ .increment = 1
+ },
+ .digest_size = {
+ .min = 1,
+ .max = 28,
+ .increment = 1
+ },
+ }, }
+ }, }
+ },
+ { /* SHA3_256 */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+ {.auth = {
+ .algo = RTE_CRYPTO_AUTH_SHA3_256,
+ .block_size = 136,
+ .key_size = {
+ .min = 0,
+ .max = 0,
+ .increment = 0
+ },
+ .digest_size = {
+ .min = 1,
+ .max = 32,
+ .increment = 1
+ },
+ }, }
+ }, }
+ },
+ { /* SHA3_256 HMAC */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+ {.auth = {
+ .algo = RTE_CRYPTO_AUTH_SHA3_256_HMAC,
+ .block_size = 136,
+ .key_size = {
+ .min = 1,
+ .max = 1024,
+ .increment = 1
+ },
+ .digest_size = {
+ .min = 1,
+ .max = 32,
+ .increment = 1
+ },
+ }, }
+ }, }
+ },
+ { /* SHA3_384 */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+ {.auth = {
+ .algo = RTE_CRYPTO_AUTH_SHA3_384,
+ .block_size = 104,
+ .key_size = {
+ .min = 0,
+ .max = 0,
+ .increment = 0
+ },
+ .digest_size = {
+ .min = 1,
+ .max = 48,
+ .increment = 0
+ },
+ }, }
+ }, }
+ },
+ { /* SHA3_384 HMAC */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+ {.auth = {
+ .algo = RTE_CRYPTO_AUTH_SHA3_384_HMAC,
+ .block_size = 104,
+ .key_size = {
+ .min = 1,
+ .max = 1024,
+ .increment = 1
+ },
+ .digest_size = {
+ .min = 1,
+ .max = 48,
+ .increment = 1
+ },
+ }, }
+ }, }
+ },
+ { /* SHA3_512 */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+ {.auth = {
+ .algo = RTE_CRYPTO_AUTH_SHA3_512,
+ .block_size = 72,
+ .key_size = {
+ .min = 0,
+ .max = 0,
+ .increment = 0
+ },
+ .digest_size = {
+ .min = 1,
+ .max = 64,
+ .increment = 1
+ },
+ }, }
+ }, }
+ },
+ { /* SHA3_512 HMAC */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+ {.auth = {
+ .algo = RTE_CRYPTO_AUTH_SHA3_512_HMAC,
+ .block_size = 72,
+ .key_size = {
+ .min = 1,
+ .max = 1024,
+ .increment = 1
+ },
+ .digest_size = {
+ .min = 1,
+ .max = 64,
+ .increment = 1
+ },
+ }, }
+ }, }
+ },
+};
+
static const struct rte_cryptodev_capabilities caps_chacha20[] = {
{ /* Chacha20-Poly1305 */
.op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
@@ -1265,6 +1428,7 @@ crypto_caps_populate(struct rte_cryptodev_capabilities cnxk_caps[],
CPT_CAPS_ADD(cnxk_caps, &cur_pos, hw_caps, mul);
CPT_CAPS_ADD(cnxk_caps, &cur_pos, hw_caps, sha1_sha2);
+ CPT_CAPS_ADD(cnxk_caps, &cur_pos, hw_caps, sha3);
CPT_CAPS_ADD(cnxk_caps, &cur_pos, hw_caps, chacha20);
CPT_CAPS_ADD(cnxk_caps, &cur_pos, hw_caps, zuc_snow3g);
CPT_CAPS_ADD(cnxk_caps, &cur_pos, hw_caps, aes);
diff --git a/drivers/crypto/cnxk/cnxk_cryptodev_ops.c b/drivers/crypto/cnxk/cnxk_cryptodev_ops.c
index eb2ed0d103..92e8755671 100644
--- a/drivers/crypto/cnxk/cnxk_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cnxk_cryptodev_ops.c
@@ -627,7 +627,7 @@ cnxk_cpt_inst_w7_get(struct cnxk_se_sess *sess, struct roc_cpt *roc_cpt)
inst_w7.s.cptr = (uint64_t)&sess->roc_se_ctx.se_ctx;
/* Set the engine group */
- if (sess->zsk_flag || sess->aes_ctr_eea2)
+ if (sess->zsk_flag || sess->aes_ctr_eea2 || sess->is_sha3)
inst_w7.s.egrp = roc_cpt->eng_grp[CPT_ENG_TYPE_SE];
else
inst_w7.s.egrp = roc_cpt->eng_grp[CPT_ENG_TYPE_IE];
diff --git a/drivers/crypto/cnxk/cnxk_se.h b/drivers/crypto/cnxk/cnxk_se.h
index 88049ac431..092cdd88e7 100644
--- a/drivers/crypto/cnxk/cnxk_se.h
+++ b/drivers/crypto/cnxk/cnxk_se.h
@@ -39,6 +39,8 @@ struct cnxk_se_sess {
uint16_t zs_auth : 4;
uint16_t dp_thr_type : 8;
uint16_t aad_length;
+ uint8_t is_sha3 : 1;
+ uint8_t rsvd : 7;
uint8_t mac_len;
uint8_t iv_length;
uint8_t auth_iv_length;
@@ -163,18 +165,26 @@ cpt_mac_len_verify(struct rte_crypto_auth_xform *auth)
break;
case RTE_CRYPTO_AUTH_SHA224:
case RTE_CRYPTO_AUTH_SHA224_HMAC:
+ case RTE_CRYPTO_AUTH_SHA3_224:
+ case RTE_CRYPTO_AUTH_SHA3_224_HMAC:
ret = (mac_len <= 28) ? 0 : -1;
break;
case RTE_CRYPTO_AUTH_SHA256:
case RTE_CRYPTO_AUTH_SHA256_HMAC:
+ case RTE_CRYPTO_AUTH_SHA3_256:
+ case RTE_CRYPTO_AUTH_SHA3_256_HMAC:
ret = (mac_len <= 32) ? 0 : -1;
break;
case RTE_CRYPTO_AUTH_SHA384:
case RTE_CRYPTO_AUTH_SHA384_HMAC:
+ case RTE_CRYPTO_AUTH_SHA3_384:
+ case RTE_CRYPTO_AUTH_SHA3_384_HMAC:
ret = (mac_len <= 48) ? 0 : -1;
break;
case RTE_CRYPTO_AUTH_SHA512:
case RTE_CRYPTO_AUTH_SHA512_HMAC:
+ case RTE_CRYPTO_AUTH_SHA3_512:
+ case RTE_CRYPTO_AUTH_SHA3_512_HMAC:
ret = (mac_len <= 64) ? 0 : -1;
break;
case RTE_CRYPTO_AUTH_NULL:
@@ -1848,7 +1858,7 @@ fill_sess_cipher(struct rte_crypto_sym_xform *xform, struct cnxk_se_sess *sess)
static __rte_always_inline int
fill_sess_auth(struct rte_crypto_sym_xform *xform, struct cnxk_se_sess *sess)
{
- uint8_t zsk_flag = 0, zs_auth = 0, aes_gcm = 0, is_null = 0;
+ uint8_t zsk_flag = 0, zs_auth = 0, aes_gcm = 0, is_null = 0, is_sha3 = 0;
struct rte_crypto_auth_xform *a_form;
roc_se_auth_type auth_type = 0; /* NULL Auth type */
@@ -1900,6 +1910,26 @@ fill_sess_auth(struct rte_crypto_sym_xform *xform, struct cnxk_se_sess *sess)
case RTE_CRYPTO_AUTH_SHA384:
auth_type = ROC_SE_SHA2_SHA384;
break;
+ case RTE_CRYPTO_AUTH_SHA3_224_HMAC:
+ case RTE_CRYPTO_AUTH_SHA3_224:
+ is_sha3 = 1;
+ auth_type = ROC_SE_SHA3_SHA224;
+ break;
+ case RTE_CRYPTO_AUTH_SHA3_256_HMAC:
+ case RTE_CRYPTO_AUTH_SHA3_256:
+ is_sha3 = 1;
+ auth_type = ROC_SE_SHA3_SHA256;
+ break;
+ case RTE_CRYPTO_AUTH_SHA3_384_HMAC:
+ case RTE_CRYPTO_AUTH_SHA3_384:
+ is_sha3 = 1;
+ auth_type = ROC_SE_SHA3_SHA384;
+ break;
+ case RTE_CRYPTO_AUTH_SHA3_512_HMAC:
+ case RTE_CRYPTO_AUTH_SHA3_512:
+ is_sha3 = 1;
+ auth_type = ROC_SE_SHA3_SHA512;
+ break;
case RTE_CRYPTO_AUTH_MD5_HMAC:
case RTE_CRYPTO_AUTH_MD5:
auth_type = ROC_SE_MD5_TYPE;
@@ -1959,6 +1989,7 @@ fill_sess_auth(struct rte_crypto_sym_xform *xform, struct cnxk_se_sess *sess)
sess->aes_gcm = aes_gcm;
sess->mac_len = a_form->digest_length;
sess->is_null = is_null;
+ sess->is_sha3 = is_sha3;
if (zsk_flag) {
sess->auth_iv_offset = a_form->iv.offset;
sess->auth_iv_length = a_form->iv.length;
--
2.25.1
^ permalink raw reply [flat|nested] 19+ messages in thread* [PATCH 11/17] common/cnxk: skip hmac hash precomputation
2022-12-20 14:32 [PATCH 00/17] fixes and improvements to cnxk crytpo PMD Tejasree Kondoj
` (9 preceding siblings ...)
2022-12-20 14:32 ` [PATCH 10/17] crypto/cnxk: add support for SHA3 hash Tejasree Kondoj
@ 2022-12-20 14:32 ` Tejasree Kondoj
2022-12-20 14:32 ` [PATCH 12/17] crypto/octeontx: support truncated digest size Tejasree Kondoj
` (6 subsequent siblings)
17 siblings, 0 replies; 19+ messages in thread
From: Tejasree Kondoj @ 2022-12-20 14:32 UTC (permalink / raw)
To: Akhil Goyal
Cc: Volodymyr Fialko, Anoob Joseph, Vidya Sagar Velumuri,
Gowrishankar Muthukrishnan, Aakash Sasidharan, dev
From: Volodymyr Fialko <vfialko@marvell.com>
Operations with FC opcode requires precomputed ipad and opad hashes, but
for auth only (HMAC opcode) this is not required, thus could be skipped.
Signed-off-by: Volodymyr Fialko <vfialko@marvell.com>
---
drivers/common/cnxk/roc_se.c | 33 ++++++++++++++++++++-------------
1 file changed, 20 insertions(+), 13 deletions(-)
diff --git a/drivers/common/cnxk/roc_se.c b/drivers/common/cnxk/roc_se.c
index 22df61f5f0..aba7f9416d 100644
--- a/drivers/common/cnxk/roc_se.c
+++ b/drivers/common/cnxk/roc_se.c
@@ -308,6 +308,7 @@ roc_se_auth_key_set(struct roc_se_ctx *se_ctx, roc_se_auth_type type,
struct roc_se_context *fctx;
uint8_t opcode_minor;
uint8_t pdcp_alg;
+ bool chained_op;
int ret;
if (se_ctx == NULL)
@@ -318,12 +319,12 @@ roc_se_auth_key_set(struct roc_se_ctx *se_ctx, roc_se_auth_type type,
k_ctx = &se_ctx->se_ctx.k_ctx;
fctx = &se_ctx->se_ctx.fctx;
+ chained_op = se_ctx->ciph_then_auth || se_ctx->auth_then_ciph;
+
if ((type >= ROC_SE_ZUC_EIA3) && (type <= ROC_SE_KASUMI_F9_ECB)) {
uint8_t *zuc_const;
uint32_t keyx[4];
uint8_t *ci_key;
- bool chained_op =
- se_ctx->ciph_then_auth || se_ctx->auth_then_ciph;
if (!key_len)
return -1;
@@ -470,19 +471,25 @@ roc_se_auth_key_set(struct roc_se_ctx *se_ctx, roc_se_auth_type type,
se_ctx->mac_len = mac_len;
if (key_len) {
- se_ctx->hmac = 1;
-
- se_ctx->auth_key = plt_zmalloc(key_len, 8);
- if (se_ctx->auth_key == NULL)
- return -1;
+ /*
+ * Chained operation (FC opcode) requires precomputed ipad and opad hashes, but for
+ * auth only (HMAC opcode) this is not required
+ */
+ if (chained_op) {
+ memset(fctx->hmac.ipad, 0, sizeof(fctx->hmac.ipad));
+ memset(fctx->hmac.opad, 0, sizeof(fctx->hmac.opad));
+ cpt_hmac_opad_ipad_gen(type, key, key_len, &fctx->hmac);
+ fctx->enc.auth_input_type = 0;
+ } else {
+ se_ctx->hmac = 1;
- memcpy(se_ctx->auth_key, key, key_len);
- se_ctx->auth_key_len = key_len;
- memset(fctx->hmac.ipad, 0, sizeof(fctx->hmac.ipad));
- memset(fctx->hmac.opad, 0, sizeof(fctx->hmac.opad));
+ se_ctx->auth_key = plt_zmalloc(key_len, 8);
+ if (se_ctx->auth_key == NULL)
+ return -1;
- cpt_hmac_opad_ipad_gen(type, key, key_len, &fctx->hmac);
- fctx->enc.auth_input_type = 0;
+ memcpy(se_ctx->auth_key, key, key_len);
+ se_ctx->auth_key_len = key_len;
+ }
}
return 0;
}
--
2.25.1
^ permalink raw reply [flat|nested] 19+ messages in thread* [PATCH 12/17] crypto/octeontx: support truncated digest size
2022-12-20 14:32 [PATCH 00/17] fixes and improvements to cnxk crytpo PMD Tejasree Kondoj
` (10 preceding siblings ...)
2022-12-20 14:32 ` [PATCH 11/17] common/cnxk: skip hmac hash precomputation Tejasree Kondoj
@ 2022-12-20 14:32 ` Tejasree Kondoj
2022-12-20 14:32 ` [PATCH 13/17] crypto/cnxk: set device ops to null in PCI remove Tejasree Kondoj
` (5 subsequent siblings)
17 siblings, 0 replies; 19+ messages in thread
From: Tejasree Kondoj @ 2022-12-20 14:32 UTC (permalink / raw)
To: Akhil Goyal
Cc: Anoob Joseph, Vidya Sagar Velumuri, Gowrishankar Muthukrishnan,
Volodymyr Fialko, Aakash Sasidharan, dev
Support truncated digest size for auth only mode.
Signed-off-by: Tejasree Kondoj <ktejasree@marvell.com>
---
drivers/common/cpt/cpt_ucode.h | 17 ++++---
.../octeontx/otx_cryptodev_capabilities.c | 48 +++++++++----------
2 files changed, 34 insertions(+), 31 deletions(-)
diff --git a/drivers/common/cpt/cpt_ucode.h b/drivers/common/cpt/cpt_ucode.h
index 22aabab6ac..b393be4cf6 100644
--- a/drivers/common/cpt/cpt_ucode.h
+++ b/drivers/common/cpt/cpt_ucode.h
@@ -41,30 +41,33 @@ cpt_mac_len_verify(struct rte_crypto_auth_xform *auth)
uint16_t mac_len = auth->digest_length;
int ret;
+ if ((auth->algo != RTE_CRYPTO_AUTH_NULL) && (mac_len == 0))
+ return -1;
+
switch (auth->algo) {
case RTE_CRYPTO_AUTH_MD5:
case RTE_CRYPTO_AUTH_MD5_HMAC:
- ret = (mac_len == 16) ? 0 : -1;
+ ret = (mac_len <= 16) ? 0 : -1;
break;
case RTE_CRYPTO_AUTH_SHA1:
case RTE_CRYPTO_AUTH_SHA1_HMAC:
- ret = (mac_len == 20) ? 0 : -1;
+ ret = (mac_len <= 20) ? 0 : -1;
break;
case RTE_CRYPTO_AUTH_SHA224:
case RTE_CRYPTO_AUTH_SHA224_HMAC:
- ret = (mac_len == 28) ? 0 : -1;
+ ret = (mac_len <= 28) ? 0 : -1;
break;
case RTE_CRYPTO_AUTH_SHA256:
case RTE_CRYPTO_AUTH_SHA256_HMAC:
- ret = (mac_len == 32) ? 0 : -1;
+ ret = (mac_len <= 32) ? 0 : -1;
break;
case RTE_CRYPTO_AUTH_SHA384:
case RTE_CRYPTO_AUTH_SHA384_HMAC:
- ret = (mac_len == 48) ? 0 : -1;
+ ret = (mac_len <= 48) ? 0 : -1;
break;
case RTE_CRYPTO_AUTH_SHA512:
case RTE_CRYPTO_AUTH_SHA512_HMAC:
- ret = (mac_len == 64) ? 0 : -1;
+ ret = (mac_len <= 64) ? 0 : -1;
break;
case RTE_CRYPTO_AUTH_NULL:
ret = 0;
@@ -523,7 +526,7 @@ cpt_digest_gen_prep(uint32_t flags,
/*GP op header */
vq_cmd_w0.s.opcode.minor = 0;
- vq_cmd_w0.s.param2 = ((uint16_t)hash_type << 8);
+ vq_cmd_w0.s.param2 = ((uint16_t)hash_type << 8) | mac_len;
if (ctx->hmac) {
vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_HMAC | CPT_DMA_MODE;
vq_cmd_w0.s.param1 = key_len;
diff --git a/drivers/crypto/octeontx/otx_cryptodev_capabilities.c b/drivers/crypto/octeontx/otx_cryptodev_capabilities.c
index 3f734b232c..80a9fe2123 100644
--- a/drivers/crypto/octeontx/otx_cryptodev_capabilities.c
+++ b/drivers/crypto/octeontx/otx_cryptodev_capabilities.c
@@ -86,9 +86,9 @@ static const struct rte_cryptodev_capabilities otx_sym_capabilities[] = {
.increment = 0
},
.digest_size = {
- .min = 16,
+ .min = 1,
.max = 16,
- .increment = 0
+ .increment = 1
},
}, }
}, }
@@ -106,9 +106,9 @@ static const struct rte_cryptodev_capabilities otx_sym_capabilities[] = {
.increment = 8
},
.digest_size = {
- .min = 16,
+ .min = 1,
.max = 16,
- .increment = 0
+ .increment = 1
},
}, }
}, }
@@ -126,9 +126,9 @@ static const struct rte_cryptodev_capabilities otx_sym_capabilities[] = {
.increment = 0
},
.digest_size = {
- .min = 20,
+ .min = 1,
.max = 20,
- .increment = 0
+ .increment = 1
},
}, }
}, }
@@ -146,9 +146,9 @@ static const struct rte_cryptodev_capabilities otx_sym_capabilities[] = {
.increment = 1
},
.digest_size = {
- .min = 20,
+ .min = 1,
.max = 20,
- .increment = 0
+ .increment = 1
},
}, }
}, }
@@ -166,9 +166,9 @@ static const struct rte_cryptodev_capabilities otx_sym_capabilities[] = {
.increment = 0
},
.digest_size = {
- .min = 28,
+ .min = 1,
.max = 28,
- .increment = 0
+ .increment = 1
},
}, }
}, }
@@ -186,9 +186,9 @@ static const struct rte_cryptodev_capabilities otx_sym_capabilities[] = {
.increment = 1
},
.digest_size = {
- .min = 28,
+ .min = 1,
.max = 28,
- .increment = 0
+ .increment = 1
},
}, }
}, }
@@ -206,9 +206,9 @@ static const struct rte_cryptodev_capabilities otx_sym_capabilities[] = {
.increment = 0
},
.digest_size = {
- .min = 32,
+ .min = 1,
.max = 32,
- .increment = 0
+ .increment = 1
},
}, }
}, }
@@ -226,9 +226,9 @@ static const struct rte_cryptodev_capabilities otx_sym_capabilities[] = {
.increment = 1
},
.digest_size = {
- .min = 32,
+ .min = 1,
.max = 32,
- .increment = 0
+ .increment = 1
},
}, }
}, }
@@ -246,9 +246,9 @@ static const struct rte_cryptodev_capabilities otx_sym_capabilities[] = {
.increment = 0
},
.digest_size = {
- .min = 48,
+ .min = 1,
.max = 48,
- .increment = 0
+ .increment = 1
},
}, }
}, }
@@ -266,9 +266,9 @@ static const struct rte_cryptodev_capabilities otx_sym_capabilities[] = {
.increment = 1
},
.digest_size = {
- .min = 48,
+ .min = 1,
.max = 48,
- .increment = 0
+ .increment = 1
},
}, }
}, }
@@ -286,9 +286,9 @@ static const struct rte_cryptodev_capabilities otx_sym_capabilities[] = {
.increment = 0
},
.digest_size = {
- .min = 64,
+ .min = 1,
.max = 64,
- .increment = 0
+ .increment = 1
},
}, }
}, }
@@ -306,9 +306,9 @@ static const struct rte_cryptodev_capabilities otx_sym_capabilities[] = {
.increment = 1
},
.digest_size = {
- .min = 64,
+ .min = 1,
.max = 64,
- .increment = 0
+ .increment = 1
},
}, }
}, }
--
2.25.1
^ permalink raw reply [flat|nested] 19+ messages in thread* [PATCH 13/17] crypto/cnxk: set device ops to null in PCI remove
2022-12-20 14:32 [PATCH 00/17] fixes and improvements to cnxk crytpo PMD Tejasree Kondoj
` (11 preceding siblings ...)
2022-12-20 14:32 ` [PATCH 12/17] crypto/octeontx: support truncated digest size Tejasree Kondoj
@ 2022-12-20 14:32 ` Tejasree Kondoj
2022-12-20 14:32 ` [PATCH 14/17] crypto/cnxk: add CTX for non IPsec operations Tejasree Kondoj
` (4 subsequent siblings)
17 siblings, 0 replies; 19+ messages in thread
From: Tejasree Kondoj @ 2022-12-20 14:32 UTC (permalink / raw)
To: Akhil Goyal
Cc: Anoob Joseph, Vidya Sagar Velumuri, Gowrishankar Muthukrishnan,
Volodymyr Fialko, Aakash Sasidharan, dev
Setting dev_ops to NULL in pci_remove to avoid
device close after removal.
Signed-off-by: Tejasree Kondoj <ktejasree@marvell.com>
---
drivers/crypto/cnxk/cn10k_cryptodev.c | 1 +
drivers/crypto/cnxk/cn9k_cryptodev.c | 1 +
2 files changed, 2 insertions(+)
diff --git a/drivers/crypto/cnxk/cn10k_cryptodev.c b/drivers/crypto/cnxk/cn10k_cryptodev.c
index 52de9b9657..2fd4df3c5d 100644
--- a/drivers/crypto/cnxk/cn10k_cryptodev.c
+++ b/drivers/crypto/cnxk/cn10k_cryptodev.c
@@ -138,6 +138,7 @@ cn10k_cpt_pci_remove(struct rte_pci_device *pci_dev)
cnxk_crypto_sec_ctx_destroy(dev);
if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+ dev->dev_ops = NULL;
vf = dev->data->dev_private;
ret = roc_cpt_dev_fini(&vf->cpt);
if (ret)
diff --git a/drivers/crypto/cnxk/cn9k_cryptodev.c b/drivers/crypto/cnxk/cn9k_cryptodev.c
index 4cfc1f2150..47b0874185 100644
--- a/drivers/crypto/cnxk/cn9k_cryptodev.c
+++ b/drivers/crypto/cnxk/cn9k_cryptodev.c
@@ -138,6 +138,7 @@ cn9k_cpt_pci_remove(struct rte_pci_device *pci_dev)
cnxk_crypto_sec_ctx_destroy(dev);
if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+ dev->dev_ops = NULL;
vf = dev->data->dev_private;
ret = roc_cpt_dev_fini(&vf->cpt);
if (ret)
--
2.25.1
^ permalink raw reply [flat|nested] 19+ messages in thread* [PATCH 14/17] crypto/cnxk: add CTX for non IPsec operations
2022-12-20 14:32 [PATCH 00/17] fixes and improvements to cnxk crytpo PMD Tejasree Kondoj
` (12 preceding siblings ...)
2022-12-20 14:32 ` [PATCH 13/17] crypto/cnxk: set device ops to null in PCI remove Tejasree Kondoj
@ 2022-12-20 14:32 ` Tejasree Kondoj
2022-12-20 14:32 ` [PATCH 15/17] crypto/cnxk: set salt in dptr as part of IV Tejasree Kondoj
` (3 subsequent siblings)
17 siblings, 0 replies; 19+ messages in thread
From: Tejasree Kondoj @ 2022-12-20 14:32 UTC (permalink / raw)
To: Akhil Goyal
Cc: Anoob Joseph, Vidya Sagar Velumuri, Gowrishankar Muthukrishnan,
Volodymyr Fialko, Aakash Sasidharan, dev
Support context cache with non IPsec operations.
Signed-off-by: Tejasree Kondoj <ktejasree@marvell.com>
Signed-off-by: Anoob Joseph <anoobj@marvell.com>
---
drivers/common/cnxk/roc_platform.h | 3 +-
drivers/common/cnxk/roc_se.c | 47 +++++++++++++++++++++++
drivers/common/cnxk/roc_se.h | 42 +++++++++++++-------
drivers/common/cnxk/version.map | 1 +
drivers/crypto/cnxk/cn10k_cryptodev_ops.c | 17 ++++----
drivers/crypto/cnxk/cn9k_cryptodev_ops.c | 14 +++----
drivers/crypto/cnxk/cnxk_cryptodev_ops.c | 38 +++++++++++++-----
drivers/crypto/cnxk/cnxk_cryptodev_ops.h | 13 +++----
drivers/crypto/cnxk/cnxk_se.h | 10 +++--
9 files changed, 134 insertions(+), 51 deletions(-)
diff --git a/drivers/common/cnxk/roc_platform.h b/drivers/common/cnxk/roc_platform.h
index 1a48ff3db4..8ba28e69fa 100644
--- a/drivers/common/cnxk/roc_platform.h
+++ b/drivers/common/cnxk/roc_platform.h
@@ -57,9 +57,10 @@
#define PLT_ALIGN RTE_ALIGN
#define PLT_ALIGN_MUL_CEIL RTE_ALIGN_MUL_CEIL
#define PLT_MODEL_MZ_NAME "roc_model_mz"
-#define PLT_CACHE_LINE_SIZE RTE_CACHE_LINE_SIZE
+#define PLT_CACHE_LINE_SIZE RTE_CACHE_LINE_SIZE
#define BITMASK_ULL GENMASK_ULL
#define PLT_ALIGN_CEIL RTE_ALIGN_CEIL
+#define PLT_ALIGN_FLOOR RTE_ALIGN_FLOOR
#define PLT_INIT RTE_INIT
#define PLT_MAX_ETHPORTS RTE_MAX_ETHPORTS
#define PLT_TAILQ_FOREACH_SAFE RTE_TAILQ_FOREACH_SAFE
diff --git a/drivers/common/cnxk/roc_se.c b/drivers/common/cnxk/roc_se.c
index aba7f9416d..8c19c5fccc 100644
--- a/drivers/common/cnxk/roc_se.c
+++ b/drivers/common/cnxk/roc_se.c
@@ -726,3 +726,50 @@ roc_se_ctx_swap(struct roc_se_ctx *se_ctx)
zs_ctx->zuc.otk_ctx.w0.u64 = htobe64(zs_ctx->zuc.otk_ctx.w0.u64);
}
+
+void
+roc_se_ctx_init(struct roc_se_ctx *roc_se_ctx)
+{
+ struct se_ctx_s *ctx = &roc_se_ctx->se_ctx;
+ uint64_t ctx_len, *uc_ctx;
+ uint8_t i;
+
+ switch (roc_se_ctx->fc_type) {
+ case ROC_SE_FC_GEN:
+ ctx_len = sizeof(struct roc_se_context);
+ break;
+ case ROC_SE_PDCP:
+ ctx_len = sizeof(struct roc_se_zuc_snow3g_ctx);
+ break;
+ case ROC_SE_KASUMI:
+ ctx_len = sizeof(struct roc_se_kasumi_ctx);
+ break;
+ case ROC_SE_PDCP_CHAIN:
+ ctx_len = sizeof(struct roc_se_zuc_snow3g_chain_ctx);
+ break;
+ default:
+ ctx_len = 0;
+ }
+
+ ctx_len = PLT_ALIGN_CEIL(ctx_len, 8);
+
+ /* Skip w0 for swap */
+ uc_ctx = PLT_PTR_ADD(ctx, sizeof(ctx->w0));
+ for (i = 0; i < (ctx_len / 8); i++)
+ uc_ctx[i] = plt_cpu_to_be_64(((uint64_t *)uc_ctx)[i]);
+
+ /* Include w0 */
+ ctx_len += sizeof(ctx->w0);
+ ctx_len = PLT_ALIGN_CEIL(ctx_len, 8);
+
+ ctx->w0.s.aop_valid = 1;
+ ctx->w0.s.ctx_hdr_size = 0;
+
+ ctx->w0.s.ctx_size = PLT_ALIGN_FLOOR(ctx_len, 128);
+ if (ctx->w0.s.ctx_size == 0)
+ ctx->w0.s.ctx_size = 1;
+
+ ctx->w0.s.ctx_push_size = ctx_len / 8;
+ if (ctx->w0.s.ctx_push_size > 32)
+ ctx->w0.s.ctx_push_size = 32;
+}
diff --git a/drivers/common/cnxk/roc_se.h b/drivers/common/cnxk/roc_se.h
index a8f0f49479..d1ef71577c 100644
--- a/drivers/common/cnxk/roc_se.h
+++ b/drivers/common/cnxk/roc_se.h
@@ -300,14 +300,31 @@ struct roc_se_ctx {
uint64_t rsvd : 17;
union cpt_inst_w4 template_w4;
/* Below fields are accessed by hardware */
- union {
- struct roc_se_context fctx;
- struct roc_se_zuc_snow3g_ctx zs_ctx;
- struct roc_se_zuc_snow3g_chain_ctx zs_ch_ctx;
- struct roc_se_kasumi_ctx k_ctx;
- } se_ctx;
+ struct se_ctx_s {
+ /* Word0 */
+ union {
+ struct {
+ uint64_t rsvd : 48;
+
+ uint64_t ctx_push_size : 7;
+ uint64_t rsvd1 : 1;
+
+ uint64_t ctx_hdr_size : 2;
+ uint64_t aop_valid : 1;
+ uint64_t rsvd2 : 1;
+ uint64_t ctx_size : 4;
+ } s;
+ uint64_t u64;
+ } w0;
+ union {
+ struct roc_se_context fctx;
+ struct roc_se_zuc_snow3g_ctx zs_ctx;
+ struct roc_se_zuc_snow3g_chain_ctx zs_ch_ctx;
+ struct roc_se_kasumi_ctx k_ctx;
+ };
+ } se_ctx __plt_aligned(ROC_ALIGN);
uint8_t *auth_key;
-};
+} __plt_aligned(ROC_ALIGN);
struct roc_se_fc_params {
union {
@@ -349,14 +366,13 @@ roc_se_zuc_bytes_swap(uint8_t *arr, int len)
}
}
-int __roc_api roc_se_auth_key_set(struct roc_se_ctx *se_ctx,
- roc_se_auth_type type, const uint8_t *key,
- uint16_t key_len, uint16_t mac_len);
+int __roc_api roc_se_auth_key_set(struct roc_se_ctx *se_ctx, roc_se_auth_type type,
+ const uint8_t *key, uint16_t key_len, uint16_t mac_len);
-int __roc_api roc_se_ciph_key_set(struct roc_se_ctx *se_ctx,
- roc_se_cipher_type type, const uint8_t *key,
- uint16_t key_len, uint8_t *salt);
+int __roc_api roc_se_ciph_key_set(struct roc_se_ctx *se_ctx, roc_se_cipher_type type,
+ const uint8_t *key, uint16_t key_len, uint8_t *salt);
void __roc_api roc_se_ctx_swap(struct roc_se_ctx *se_ctx);
+void __roc_api roc_se_ctx_init(struct roc_se_ctx *se_ctx);
#endif /* __ROC_SE_H__ */
diff --git a/drivers/common/cnxk/version.map b/drivers/common/cnxk/version.map
index 17f0ec6b48..ee283d2392 100644
--- a/drivers/common/cnxk/version.map
+++ b/drivers/common/cnxk/version.map
@@ -99,6 +99,7 @@ INTERNAL {
roc_model;
roc_se_auth_key_set;
roc_se_ciph_key_set;
+ roc_se_ctx_init;
roc_nix_bpf_alloc;
roc_nix_bpf_config;
roc_nix_bpf_connect;
diff --git a/drivers/crypto/cnxk/cn10k_cryptodev_ops.c b/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
index 5a098ffcf2..57b44210c2 100644
--- a/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
@@ -52,14 +52,13 @@ cn10k_cpt_sym_temp_sess_create(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op)
if (rte_mempool_get(qp->sess_mp, (void **)&sess) < 0)
return NULL;
- ret = sym_session_configure(qp->lf.roc_cpt, sym_op->xform,
- sess);
+ ret = sym_session_configure(qp->lf.roc_cpt, sym_op->xform, sess, true);
if (ret) {
rte_mempool_put(qp->sess_mp, (void *)sess);
goto sess_put;
}
- priv = (void *)sess->driver_priv_data;
+ priv = (void *)sess;
sym_op->session = sess;
return priv;
@@ -121,13 +120,13 @@ cn10k_cpt_fill_inst(struct cnxk_cpt_qp *qp, struct rte_crypto_op *ops[], struct
if (op->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
- sec_sess = (struct cn10k_sec_session *)(sym_op->session);
+ sec_sess = (struct cn10k_sec_session *)sym_op->session;
ret = cpt_sec_inst_fill(qp, op, sec_sess, &inst[0]);
if (unlikely(ret))
return 0;
w7 = sec_sess->inst.w7;
} else if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
- sess = CRYPTODEV_GET_SYM_SESS_PRIV(sym_op->session);
+ sess = (struct cnxk_se_sess *)(sym_op->session);
ret = cpt_sym_inst_fill(qp, op, sess, infl_req, &inst[0], is_sg_ver2);
if (unlikely(ret))
return 0;
@@ -141,7 +140,7 @@ cn10k_cpt_fill_inst(struct cnxk_cpt_qp *qp, struct rte_crypto_op *ops[], struct
ret = cpt_sym_inst_fill(qp, op, sess, infl_req, &inst[0], is_sg_ver2);
if (unlikely(ret)) {
- sym_session_clear(op->sym->session);
+ sym_session_clear(op->sym->session, true);
rte_mempool_put(qp->sess_mp, op->sym->session);
return 0;
}
@@ -316,7 +315,7 @@ cn10k_cpt_crypto_adapter_ev_mdata_set(struct rte_cryptodev *dev __rte_unused, vo
} else if (sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
struct cnxk_se_sess *priv;
- priv = CRYPTODEV_GET_SYM_SESS_PRIV(sess);
+ priv = (struct cnxk_se_sess *)sess;
priv->qp = qp;
priv->cpt_inst_w2 = w2;
} else
@@ -351,7 +350,7 @@ cn10k_ca_meta_info_extract(struct rte_crypto_op *op, struct cnxk_cpt_qp **qp, ui
} else if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
struct cnxk_se_sess *priv;
- priv = CRYPTODEV_GET_SYM_SESS_PRIV(op->sym->session);
+ priv = (struct cnxk_se_sess *)op->sym->session;
*qp = priv->qp;
*w2 = priv->cpt_inst_w2;
} else {
@@ -914,7 +913,7 @@ cn10k_cpt_dequeue_post_process(struct cnxk_cpt_qp *qp,
temp_sess_free:
if (unlikely(cop->sess_type == RTE_CRYPTO_OP_SESSIONLESS)) {
if (cop->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
- sym_session_clear(cop->sym->session);
+ sym_session_clear(cop->sym->session, true);
rte_mempool_put(qp->sess_mp, cop->sym->session);
cop->sym->session = NULL;
}
diff --git a/drivers/crypto/cnxk/cn9k_cryptodev_ops.c b/drivers/crypto/cnxk/cn9k_cryptodev_ops.c
index cfe1e08dff..e3784e34c9 100644
--- a/drivers/crypto/cnxk/cn9k_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cn9k_cryptodev_ops.c
@@ -49,11 +49,11 @@ cn9k_cpt_sym_temp_sess_create(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op)
if (rte_mempool_get(qp->sess_mp, (void **)&sess) < 0)
return NULL;
- ret = sym_session_configure(qp->lf.roc_cpt, sym_op->xform, sess);
+ ret = sym_session_configure(qp->lf.roc_cpt, sym_op->xform, sess, true);
if (ret)
goto sess_put;
- priv = CRYPTODEV_GET_SYM_SESS_PRIV(sess);
+ priv = (struct cnxk_se_sess *)sess;
sym_op->session = sess;
@@ -76,7 +76,7 @@ cn9k_cpt_inst_prep(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,
if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
sym_op = op->sym;
- sess = CRYPTODEV_GET_SYM_SESS_PRIV(sym_op->session);
+ sess = (struct cnxk_se_sess *)sym_op->session;
ret = cpt_sym_inst_fill(qp, op, sess, infl_req, inst, false);
inst->w7.u64 = sess->cpt_inst_w7;
} else if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION)
@@ -90,7 +90,7 @@ cn9k_cpt_inst_prep(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,
ret = cpt_sym_inst_fill(qp, op, sess, infl_req, inst, false);
if (unlikely(ret)) {
- sym_session_clear(op->sym->session);
+ sym_session_clear(op->sym->session, true);
rte_mempool_put(qp->sess_mp, op->sym->session);
}
inst->w7.u64 = sess->cpt_inst_w7;
@@ -326,7 +326,7 @@ cn9k_cpt_crypto_adapter_ev_mdata_set(struct rte_cryptodev *dev __rte_unused,
} else if (sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
struct cnxk_se_sess *priv;
- priv = CRYPTODEV_GET_SYM_SESS_PRIV(sess);
+ priv = (struct cnxk_se_sess *)sess;
priv->qp = qp;
priv->cpt_inst_w2 = w2;
} else
@@ -361,7 +361,7 @@ cn9k_ca_meta_info_extract(struct rte_crypto_op *op,
} else if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
struct cnxk_se_sess *priv;
- priv = CRYPTODEV_GET_SYM_SESS_PRIV(op->sym->session);
+ priv = (struct cnxk_se_sess *)op->sym->session;
*qp = priv->qp;
inst->w2.u64 = priv->cpt_inst_w2;
} else {
@@ -630,7 +630,7 @@ cn9k_cpt_dequeue_post_process(struct cnxk_cpt_qp *qp, struct rte_crypto_op *cop,
temp_sess_free:
if (unlikely(cop->sess_type == RTE_CRYPTO_OP_SESSIONLESS)) {
if (cop->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
- sym_session_clear(cop->sym->session);
+ sym_session_clear(cop->sym->session, true);
rte_mempool_put(qp->sess_mp, cop->sym->session);
cop->sym->session = NULL;
}
diff --git a/drivers/crypto/cnxk/cnxk_cryptodev_ops.c b/drivers/crypto/cnxk/cnxk_cryptodev_ops.c
index 92e8755671..2e845afce9 100644
--- a/drivers/crypto/cnxk/cnxk_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cnxk_cryptodev_ops.c
@@ -626,6 +626,11 @@ cnxk_cpt_inst_w7_get(struct cnxk_se_sess *sess, struct roc_cpt *roc_cpt)
inst_w7.s.cptr = (uint64_t)&sess->roc_se_ctx.se_ctx;
+ if (roc_cpt->cpt_revision == ROC_CPT_REVISION_ID_106XX)
+ inst_w7.s.ctx_val = 1;
+ else
+ inst_w7.s.cptr += 8;
+
/* Set the engine group */
if (sess->zsk_flag || sess->aes_ctr_eea2 || sess->is_sha3)
inst_w7.s.egrp = roc_cpt->eng_grp[CPT_ENG_TYPE_SE];
@@ -636,19 +641,22 @@ cnxk_cpt_inst_w7_get(struct cnxk_se_sess *sess, struct roc_cpt *roc_cpt)
}
int
-sym_session_configure(struct roc_cpt *roc_cpt,
- struct rte_crypto_sym_xform *xform,
- struct rte_cryptodev_sym_session *sess)
+sym_session_configure(struct roc_cpt *roc_cpt, struct rte_crypto_sym_xform *xform,
+ struct rte_cryptodev_sym_session *sess, bool is_session_less)
{
enum cpt_dp_thread_type thr_type;
- struct cnxk_se_sess *sess_priv = CRYPTODEV_GET_SYM_SESS_PRIV(sess);
+ struct cnxk_se_sess *sess_priv = (struct cnxk_se_sess *)sess;
int ret;
- memset(sess_priv, 0, sizeof(struct cnxk_se_sess));
+ if (is_session_less)
+ memset(sess_priv, 0, sizeof(struct cnxk_se_sess));
+
ret = cnxk_sess_fill(roc_cpt, xform, sess_priv);
if (ret)
goto priv_put;
+ sess_priv->lf = roc_cpt->lf[0];
+
if (sess_priv->cpt_op & ROC_SE_OP_CIPHER_MASK) {
switch (sess_priv->roc_se_ctx.fc_type) {
case ROC_SE_FC_GEN:
@@ -690,6 +698,10 @@ sym_session_configure(struct roc_cpt *roc_cpt,
}
sess_priv->cpt_inst_w7 = cnxk_cpt_inst_w7_get(sess_priv, roc_cpt);
+
+ if (roc_cpt->cpt_revision == ROC_CPT_REVISION_ID_106XX)
+ roc_se_ctx_init(&sess_priv->roc_se_ctx);
+
return 0;
priv_put:
@@ -704,25 +716,31 @@ cnxk_cpt_sym_session_configure(struct rte_cryptodev *dev,
struct cnxk_cpt_vf *vf = dev->data->dev_private;
struct roc_cpt *roc_cpt = &vf->cpt;
- return sym_session_configure(roc_cpt, xform, sess);
+ return sym_session_configure(roc_cpt, xform, sess, false);
}
void
-sym_session_clear(struct rte_cryptodev_sym_session *sess)
+sym_session_clear(struct rte_cryptodev_sym_session *sess, bool is_session_less)
{
- struct cnxk_se_sess *sess_priv = CRYPTODEV_GET_SYM_SESS_PRIV(sess);
+ struct cnxk_se_sess *sess_priv = (struct cnxk_se_sess *)sess;
+
+ /* Trigger CTX flush + invalidate to remove from CTX_CACHE */
+ roc_cpt_lf_ctx_flush(sess_priv->lf, &sess_priv->roc_se_ctx.se_ctx, true);
+
+ plt_delay_ms(1);
if (sess_priv->roc_se_ctx.auth_key != NULL)
plt_free(sess_priv->roc_se_ctx.auth_key);
- memset(sess_priv, 0, cnxk_cpt_sym_session_get_size(NULL));
+ if (is_session_less)
+ memset(sess_priv, 0, cnxk_cpt_sym_session_get_size(NULL));
}
void
cnxk_cpt_sym_session_clear(struct rte_cryptodev *dev __rte_unused,
struct rte_cryptodev_sym_session *sess)
{
- return sym_session_clear(sess);
+ return sym_session_clear(sess, false);
}
unsigned int
diff --git a/drivers/crypto/cnxk/cnxk_cryptodev_ops.h b/drivers/crypto/cnxk/cnxk_cryptodev_ops.h
index 5153d334ba..f91ad368ea 100644
--- a/drivers/crypto/cnxk/cnxk_cryptodev_ops.h
+++ b/drivers/crypto/cnxk/cnxk_cryptodev_ops.h
@@ -107,18 +107,15 @@ int cnxk_cpt_queue_pair_release(struct rte_cryptodev *dev, uint16_t qp_id);
unsigned int cnxk_cpt_sym_session_get_size(struct rte_cryptodev *dev);
-int cnxk_cpt_sym_session_configure(struct rte_cryptodev *dev,
- struct rte_crypto_sym_xform *xform,
+int cnxk_cpt_sym_session_configure(struct rte_cryptodev *dev, struct rte_crypto_sym_xform *xform,
struct rte_cryptodev_sym_session *sess);
-int sym_session_configure(struct roc_cpt *roc_cpt,
- struct rte_crypto_sym_xform *xform,
- struct rte_cryptodev_sym_session *sess);
+int sym_session_configure(struct roc_cpt *roc_cpt, struct rte_crypto_sym_xform *xform,
+ struct rte_cryptodev_sym_session *sess, bool is_session_less);
-void cnxk_cpt_sym_session_clear(struct rte_cryptodev *dev,
- struct rte_cryptodev_sym_session *sess);
+void cnxk_cpt_sym_session_clear(struct rte_cryptodev *dev, struct rte_cryptodev_sym_session *sess);
-void sym_session_clear(struct rte_cryptodev_sym_session *sess);
+void sym_session_clear(struct rte_cryptodev_sym_session *sess, bool is_session_less);
unsigned int cnxk_ae_session_size_get(struct rte_cryptodev *dev __rte_unused);
diff --git a/drivers/crypto/cnxk/cnxk_se.h b/drivers/crypto/cnxk/cnxk_se.h
index 092cdd88e7..774efd5879 100644
--- a/drivers/crypto/cnxk/cnxk_se.h
+++ b/drivers/crypto/cnxk/cnxk_se.h
@@ -6,6 +6,8 @@
#define _CNXK_SE_H_
#include <stdbool.h>
+#include <rte_cryptodev.h>
+
#include "cnxk_cryptodev.h"
#include "cnxk_cryptodev_ops.h"
#include "cnxk_sg.h"
@@ -25,6 +27,7 @@ enum cpt_dp_thread_type {
};
struct cnxk_se_sess {
+ struct rte_cryptodev_sym_session rte_sess;
uint16_t cpt_op : 4;
uint16_t zsk_flag : 4;
uint16_t aes_gcm : 1;
@@ -51,10 +54,11 @@ struct cnxk_se_sess {
uint64_t cpt_inst_w2;
struct cnxk_cpt_qp *qp;
struct roc_se_ctx roc_se_ctx;
-} __rte_cache_aligned;
+ struct roc_cpt_lf *lf;
+} __rte_aligned(ROC_ALIGN);
-static __rte_always_inline int
-fill_sess_gmac(struct rte_crypto_sym_xform *xform, struct cnxk_se_sess *sess);
+static __rte_always_inline int fill_sess_gmac(struct rte_crypto_sym_xform *xform,
+ struct cnxk_se_sess *sess);
static inline void
cpt_pack_iv(uint8_t *iv_src, uint8_t *iv_dst)
--
2.25.1
^ permalink raw reply [flat|nested] 19+ messages in thread* [PATCH 15/17] crypto/cnxk: set salt in dptr as part of IV
2022-12-20 14:32 [PATCH 00/17] fixes and improvements to cnxk crytpo PMD Tejasree Kondoj
` (13 preceding siblings ...)
2022-12-20 14:32 ` [PATCH 14/17] crypto/cnxk: add CTX for non IPsec operations Tejasree Kondoj
@ 2022-12-20 14:32 ` Tejasree Kondoj
2022-12-20 14:32 ` [PATCH 16/17] crypto/cnxk: remove null check of session priv Tejasree Kondoj
` (2 subsequent siblings)
17 siblings, 0 replies; 19+ messages in thread
From: Tejasree Kondoj @ 2022-12-20 14:32 UTC (permalink / raw)
To: Akhil Goyal
Cc: Anoob Joseph, Vidya Sagar Velumuri, Gowrishankar Muthukrishnan,
Volodymyr Fialko, Aakash Sasidharan, dev
Set salt as part of 16B IV in dptr to avoid race
condition.
Signed-off-by: Tejasree Kondoj <ktejasree@marvell.com>
---
drivers/common/cnxk/roc_se.c | 7 +++-
drivers/crypto/cnxk/cnxk_se.h | 76 +++++++++++++++++++----------------
2 files changed, 47 insertions(+), 36 deletions(-)
diff --git a/drivers/common/cnxk/roc_se.c b/drivers/common/cnxk/roc_se.c
index 8c19c5fccc..f335c2367f 100644
--- a/drivers/common/cnxk/roc_se.c
+++ b/drivers/common/cnxk/roc_se.c
@@ -462,8 +462,10 @@ roc_se_auth_key_set(struct roc_se_ctx *se_ctx, roc_se_auth_type type,
return -1;
/* For GMAC auth, cipher must be NULL */
- if (type == ROC_SE_GMAC_TYPE)
+ if (type == ROC_SE_GMAC_TYPE) {
fctx->enc.enc_cipher = 0;
+ se_ctx->template_w4.s.opcode_minor = BIT(5);
+ }
fctx->enc.hash_type = type;
se_ctx->hash_type = type;
@@ -530,6 +532,9 @@ roc_se_ciph_key_set(struct roc_se_ctx *se_ctx, roc_se_cipher_type type,
return 0;
}
+ if (type == ROC_SE_AES_GCM)
+ se_ctx->template_w4.s.opcode_minor = BIT(5);
+
ret = cpt_ciph_type_set(type, se_ctx, key_len);
if (unlikely(ret))
return -1;
diff --git a/drivers/crypto/cnxk/cnxk_se.h b/drivers/crypto/cnxk/cnxk_se.h
index 774efd5879..1827b0d7b3 100644
--- a/drivers/crypto/cnxk/cnxk_se.h
+++ b/drivers/crypto/cnxk/cnxk_se.h
@@ -43,7 +43,8 @@ struct cnxk_se_sess {
uint16_t dp_thr_type : 8;
uint16_t aad_length;
uint8_t is_sha3 : 1;
- uint8_t rsvd : 7;
+ uint8_t short_iv : 1;
+ uint8_t rsvd : 6;
uint8_t mac_len;
uint8_t iv_length;
uint8_t auth_iv_length;
@@ -201,13 +202,6 @@ cpt_mac_len_verify(struct rte_crypto_auth_xform *auth)
return ret;
}
-static __rte_always_inline void
-cpt_fc_salt_update(struct roc_se_ctx *se_ctx, uint8_t *salt)
-{
- struct roc_se_context *fctx = &se_ctx->se_ctx.fctx;
- memcpy(fctx->enc.encr_iv, salt, 4);
-}
-
static __rte_always_inline int
sg_inst_prep(struct roc_se_fc_params *params, struct cpt_inst_s *inst, uint64_t offset_ctrl,
uint8_t *iv_s, int iv_len, uint8_t pack_iv, uint8_t pdcp_alg_type, int32_t inputlen,
@@ -1703,8 +1697,17 @@ fill_sess_aead(struct rte_crypto_sym_xform *xform, struct cnxk_se_sess *sess)
sess->iv_length = aead_form->iv.length;
sess->aad_length = aead_form->aad_length;
- if (unlikely(roc_se_ciph_key_set(&sess->roc_se_ctx, enc_type,
- aead_form->key.data,
+ switch (sess->iv_length) {
+ case 12:
+ sess->short_iv = 1;
+ case 16:
+ break;
+ default:
+ plt_dp_err("Crypto: Unsupported IV length %u", sess->iv_length);
+ return -1;
+ }
+
+ if (unlikely(roc_se_ciph_key_set(&sess->roc_se_ctx, enc_type, aead_form->key.data,
aead_form->key.length, NULL)))
return -1;
@@ -1712,6 +1715,8 @@ fill_sess_aead(struct rte_crypto_sym_xform *xform, struct cnxk_se_sess *sess)
aead_form->digest_length)))
return -1;
+ if (enc_type == ROC_SE_CHACHA20)
+ sess->roc_se_ctx.template_w4.s.opcode_minor |= BIT(5);
return 0;
}
@@ -1849,9 +1854,19 @@ fill_sess_cipher(struct rte_crypto_sym_xform *xform, struct cnxk_se_sess *sess)
sess->iv_length = c_form->iv.length;
sess->is_null = is_null;
- if (unlikely(roc_se_ciph_key_set(&sess->roc_se_ctx, enc_type,
- c_form->key.data, c_form->key.length,
- NULL)))
+ if (aes_ctr)
+ switch (sess->iv_length) {
+ case 12:
+ sess->short_iv = 1;
+ case 16:
+ break;
+ default:
+ plt_dp_err("Crypto: Unsupported IV length %u", sess->iv_length);
+ return -1;
+ }
+
+ if (unlikely(roc_se_ciph_key_set(&sess->roc_se_ctx, enc_type, c_form->key.data,
+ c_form->key.length, NULL)))
return -1;
if ((enc_type >= ROC_SE_ZUC_EEA3) && (enc_type <= ROC_SE_AES_CTR_EEA2))
@@ -2046,9 +2061,18 @@ fill_sess_gmac(struct rte_crypto_sym_xform *xform, struct cnxk_se_sess *sess)
sess->iv_length = a_form->iv.length;
sess->mac_len = a_form->digest_length;
- if (unlikely(roc_se_ciph_key_set(&sess->roc_se_ctx, enc_type,
- a_form->key.data, a_form->key.length,
- NULL)))
+ switch (sess->iv_length) {
+ case 12:
+ sess->short_iv = 1;
+ case 16:
+ break;
+ default:
+ plt_dp_err("Crypto: Unsupported IV length %u", sess->iv_length);
+ return -1;
+ }
+
+ if (unlikely(roc_se_ciph_key_set(&sess->roc_se_ctx, enc_type, a_form->key.data,
+ a_form->key.length, NULL)))
return -1;
if (unlikely(roc_se_auth_key_set(&sess->roc_se_ctx, auth_type, NULL, 0,
@@ -2192,7 +2216,7 @@ fill_fc_params(struct rte_crypto_op *cop, struct cnxk_se_sess *sess,
if (likely(is_kasumi || sess->iv_length)) {
flags |= ROC_SE_VALID_IV_BUF;
fc_params.iv_buf = rte_crypto_op_ctod_offset(cop, uint8_t *, sess->iv_offset);
- if (!is_aead && sess->aes_ctr && unlikely(sess->iv_length != 16)) {
+ if (sess->short_iv) {
memcpy((uint8_t *)iv_buf,
rte_crypto_op_ctod_offset(cop, uint8_t *, sess->iv_offset), 12);
iv_buf[3] = rte_cpu_to_be_32(0x1);
@@ -2209,7 +2233,6 @@ fill_fc_params(struct rte_crypto_op *cop, struct cnxk_se_sess *sess,
if (is_aead) {
struct rte_mbuf *m;
- uint8_t *salt;
uint8_t *aad_data;
uint16_t aad_len;
@@ -2234,13 +2257,6 @@ fill_fc_params(struct rte_crypto_op *cop, struct cnxk_se_sess *sess,
d_lens = d_lens << 32;
}
- salt = fc_params.iv_buf;
- if (unlikely(*(uint32_t *)salt != sess->salt)) {
- cpt_fc_salt_update(&sess->roc_se_ctx, salt);
- sess->salt = *(uint32_t *)salt;
- }
-
- fc_params.iv_buf = PLT_PTR_ADD(salt, 4);
m = cpt_m_dst_get(cpt_op, m_src, m_dst);
/* Digest immediately following data is best case */
@@ -2266,16 +2282,6 @@ fill_fc_params(struct rte_crypto_op *cop, struct cnxk_se_sess *sess,
d_lens = ci_data_length;
d_lens = (d_lens << 32) | a_data_length;
- /* for gmac, salt should be updated like in gcm */
- if (unlikely(sess->is_gmac)) {
- uint8_t *salt;
- salt = fc_params.iv_buf;
- if (unlikely(*(uint32_t *)salt != sess->salt)) {
- cpt_fc_salt_update(&sess->roc_se_ctx, salt);
- sess->salt = *(uint32_t *)salt;
- }
- fc_params.iv_buf = salt + 4;
- }
if (likely(sess->mac_len)) {
struct rte_mbuf *m = cpt_m_dst_get(cpt_op, m_src, m_dst);
--
2.25.1
^ permalink raw reply [flat|nested] 19+ messages in thread* [PATCH 16/17] crypto/cnxk: remove null check of session priv
2022-12-20 14:32 [PATCH 00/17] fixes and improvements to cnxk crytpo PMD Tejasree Kondoj
` (14 preceding siblings ...)
2022-12-20 14:32 ` [PATCH 15/17] crypto/cnxk: set salt in dptr as part of IV Tejasree Kondoj
@ 2022-12-20 14:32 ` Tejasree Kondoj
2022-12-20 14:32 ` [PATCH 17/17] common/cnxk: remove salt from session Tejasree Kondoj
2023-01-04 10:11 ` [PATCH 00/17] fixes and improvements to cnxk crytpo PMD Akhil Goyal
17 siblings, 0 replies; 19+ messages in thread
From: Tejasree Kondoj @ 2022-12-20 14:32 UTC (permalink / raw)
To: Akhil Goyal
Cc: Anoob Joseph, Vidya Sagar Velumuri, Gowrishankar Muthukrishnan,
Volodymyr Fialko, Aakash Sasidharan, dev
From: Anoob Joseph <anoobj@marvell.com>
Session private data is at an offset of session. This can never be
NULL. Remove check for the same.
Signed-off-by: Anoob Joseph <anoobj@marvell.com>
---
drivers/crypto/cnxk/cn10k_ipsec.c | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/drivers/crypto/cnxk/cn10k_ipsec.c b/drivers/crypto/cnxk/cn10k_ipsec.c
index aafd461436..ffd3f50eed 100644
--- a/drivers/crypto/cnxk/cn10k_ipsec.c
+++ b/drivers/crypto/cnxk/cn10k_ipsec.c
@@ -422,14 +422,12 @@ cn10k_sec_session_update(void *device, struct rte_security_session *sess,
struct rte_security_session_conf *conf)
{
struct rte_cryptodev *crypto_dev = device;
- struct cn10k_sec_session *priv;
struct roc_cpt *roc_cpt;
struct cnxk_cpt_qp *qp;
struct cnxk_cpt_vf *vf;
int ret;
- priv = SECURITY_GET_SESS_PRIV(sess);
- if (priv == NULL)
+ if (sess == NULL)
return -EINVAL;
qp = crypto_dev->data->queue_pairs[0];
--
2.25.1
^ permalink raw reply [flat|nested] 19+ messages in thread* [PATCH 17/17] common/cnxk: remove salt from session
2022-12-20 14:32 [PATCH 00/17] fixes and improvements to cnxk crytpo PMD Tejasree Kondoj
` (15 preceding siblings ...)
2022-12-20 14:32 ` [PATCH 16/17] crypto/cnxk: remove null check of session priv Tejasree Kondoj
@ 2022-12-20 14:32 ` Tejasree Kondoj
2023-01-04 10:11 ` [PATCH 00/17] fixes and improvements to cnxk crytpo PMD Akhil Goyal
17 siblings, 0 replies; 19+ messages in thread
From: Tejasree Kondoj @ 2022-12-20 14:32 UTC (permalink / raw)
To: Akhil Goyal
Cc: Anoob Joseph, Vidya Sagar Velumuri, Gowrishankar Muthukrishnan,
Volodymyr Fialko, Aakash Sasidharan, dev
From: Anoob Joseph <anoobj@marvell.com>
Complete 16B IV can be passed in DPTR to microcode. Passing salt in CPTR
cannot be supported as the salt need to be given in different formats
when CTX is enabled.
Signed-off-by: Anoob Joseph <anoobj@marvell.com>
---
drivers/common/cnxk/roc_se.c | 17 +++--------------
drivers/common/cnxk/roc_se.h | 2 +-
drivers/crypto/cnxk/cnxk_se.h | 6 +++---
3 files changed, 7 insertions(+), 18 deletions(-)
diff --git a/drivers/common/cnxk/roc_se.c b/drivers/common/cnxk/roc_se.c
index f335c2367f..6f0a81b22c 100644
--- a/drivers/common/cnxk/roc_se.c
+++ b/drivers/common/cnxk/roc_se.c
@@ -497,8 +497,9 @@ roc_se_auth_key_set(struct roc_se_ctx *se_ctx, roc_se_auth_type type,
}
int
-roc_se_ciph_key_set(struct roc_se_ctx *se_ctx, roc_se_cipher_type type,
- const uint8_t *key, uint16_t key_len, uint8_t *salt)
+roc_se_ciph_key_set(struct roc_se_ctx *se_ctx, roc_se_cipher_type type, const uint8_t *key,
+ uint16_t key_len)
+
{
bool chained_op = se_ctx->ciph_then_auth || se_ctx->auth_then_ciph;
struct roc_se_zuc_snow3g_ctx *zs_ctx = &se_ctx->se_ctx.zs_ctx;
@@ -520,18 +521,6 @@ roc_se_ciph_key_set(struct roc_se_ctx *se_ctx, roc_se_cipher_type type,
zuc_const = zs_ctx->zuc.otk_ctx.zuc_const;
}
- /* For AES-GCM, salt is taken from ctx even if IV source
- * is from DPTR
- */
- if ((salt != NULL) && (type == ROC_SE_AES_GCM)) {
- memcpy(fctx->enc.encr_iv, salt, 4);
- /* Assuming it was just salt update
- * and nothing else
- */
- if (key == NULL)
- return 0;
- }
-
if (type == ROC_SE_AES_GCM)
se_ctx->template_w4.s.opcode_minor = BIT(5);
diff --git a/drivers/common/cnxk/roc_se.h b/drivers/common/cnxk/roc_se.h
index d1ef71577c..e9415f21a5 100644
--- a/drivers/common/cnxk/roc_se.h
+++ b/drivers/common/cnxk/roc_se.h
@@ -370,7 +370,7 @@ int __roc_api roc_se_auth_key_set(struct roc_se_ctx *se_ctx, roc_se_auth_type ty
const uint8_t *key, uint16_t key_len, uint16_t mac_len);
int __roc_api roc_se_ciph_key_set(struct roc_se_ctx *se_ctx, roc_se_cipher_type type,
- const uint8_t *key, uint16_t key_len, uint8_t *salt);
+ const uint8_t *key, uint16_t key_len);
void __roc_api roc_se_ctx_swap(struct roc_se_ctx *se_ctx);
void __roc_api roc_se_ctx_init(struct roc_se_ctx *se_ctx);
diff --git a/drivers/crypto/cnxk/cnxk_se.h b/drivers/crypto/cnxk/cnxk_se.h
index 1827b0d7b3..0e5d2dde39 100644
--- a/drivers/crypto/cnxk/cnxk_se.h
+++ b/drivers/crypto/cnxk/cnxk_se.h
@@ -1708,7 +1708,7 @@ fill_sess_aead(struct rte_crypto_sym_xform *xform, struct cnxk_se_sess *sess)
}
if (unlikely(roc_se_ciph_key_set(&sess->roc_se_ctx, enc_type, aead_form->key.data,
- aead_form->key.length, NULL)))
+ aead_form->key.length)))
return -1;
if (unlikely(roc_se_auth_key_set(&sess->roc_se_ctx, auth_type, NULL, 0,
@@ -1866,7 +1866,7 @@ fill_sess_cipher(struct rte_crypto_sym_xform *xform, struct cnxk_se_sess *sess)
}
if (unlikely(roc_se_ciph_key_set(&sess->roc_se_ctx, enc_type, c_form->key.data,
- c_form->key.length, NULL)))
+ c_form->key.length)))
return -1;
if ((enc_type >= ROC_SE_ZUC_EEA3) && (enc_type <= ROC_SE_AES_CTR_EEA2))
@@ -2072,7 +2072,7 @@ fill_sess_gmac(struct rte_crypto_sym_xform *xform, struct cnxk_se_sess *sess)
}
if (unlikely(roc_se_ciph_key_set(&sess->roc_se_ctx, enc_type, a_form->key.data,
- a_form->key.length, NULL)))
+ a_form->key.length)))
return -1;
if (unlikely(roc_se_auth_key_set(&sess->roc_se_ctx, auth_type, NULL, 0,
--
2.25.1
^ permalink raw reply [flat|nested] 19+ messages in thread* RE: [PATCH 00/17] fixes and improvements to cnxk crytpo PMD
2022-12-20 14:32 [PATCH 00/17] fixes and improvements to cnxk crytpo PMD Tejasree Kondoj
` (16 preceding siblings ...)
2022-12-20 14:32 ` [PATCH 17/17] common/cnxk: remove salt from session Tejasree Kondoj
@ 2023-01-04 10:11 ` Akhil Goyal
17 siblings, 0 replies; 19+ messages in thread
From: Akhil Goyal @ 2023-01-04 10:11 UTC (permalink / raw)
To: Tejasree Kondoj
Cc: Anoob Joseph, Jerin Jacob Kollanukkaran, Nithin Kumar Dabilpuram,
Rahul Bhansali, Vidya Sagar Velumuri, Gowrishankar Muthukrishnan,
Volodymyr Fialko, Aakash Sasidharan, dev
> Subject: [PATCH 00/17] fixes and improvements to cnxk crytpo PMD
>
> This series adds improvements and support for SHA3,
> IPsec scatter gather mode in cnxk crypto PMD.
>
> Aakash Sasidharan (1):
> common/cnxk: generate opad and ipad in driver
>
> Anoob Joseph (3):
> common/cnxk: perform LF fini ops only when allocated
> crypto/cnxk: remove null check of session priv
> common/cnxk: remove salt from session
>
> Archana Muniganti (1):
> crypto/cnxk: add CN9K IPsec SG support
>
> Gowrishankar Muthukrishnan (1):
> crypto/cnxk: fix incorrect digest for an empty input data
>
> Tejasree Kondoj (5):
> crypto/cnxk: support truncated digest length
> crypto/octeontx: support truncated digest size
> crypto/cnxk: set device ops to null in PCI remove
> crypto/cnxk: add CTX for non IPsec operations
> crypto/cnxk: set salt in dptr as part of IV
>
> Vidya Sagar Velumuri (2):
> crypto/cnxk: update resp len calculation for IPv6
> crypto/cnxk: update crypto completion code handling
>
> Volodymyr Fialko (4):
> crypto/cnxk: add context to passthrough instruction
> crypto/cnxk: add queue pair check to meta set
> crypto/cnxk: add support for SHA3 hash
> common/cnxk: skip hmac hash precomputation
>
Series Acked-by: Akhil Goyal <gakhil@marvell.com>
Fixed couple of formatting issues while applying.
Applied to dpdk-next-crypto
^ permalink raw reply [flat|nested] 19+ messages in thread