[PATCH 0/4] ipsec_mb fixes for ZUC algorithm

[PATCH 0/4] ipsec_mb fixes for ZUC algorithm

[Pablo de Lara]

This patchset fixes various issues affecting ZUC and AESNI MB PMD,
when ZUC algorithm is used.

Pablo de Lara (4):
  crypto/ipsec_mb: check for missing operation types
  crypto/ipsec_mb: fix ZUC authentication verify
  crypto/ipsec_mb: fix crypto operation overwrite
  crypto/ipsec_mb: fix length and offset settings

 drivers/crypto/ipsec_mb/pmd_aesni_mb.c | 120 +++++++++++++++++--------
 drivers/crypto/ipsec_mb/pmd_zuc.c      |  22 +++--
 drivers/crypto/ipsec_mb/pmd_zuc_priv.h |   2 +-
 3 files changed, 99 insertions(+), 45 deletions(-)

-- 
2.25.1

[PATCH 1/4] crypto/ipsec_mb: check for missing operation types

[Pablo de Lara]

When processing crypto operations in ZUC PMD,
there were two operation types that were set at session level,
but not checked when the operations are enqueued and processed,
leaving the buffers untouched silently.

Fixes: cde8df1bda9d ("crypto/ipsec_mb: move zuc PMD")
Cc: piotrx.bronowski@intel.com
Cc: stable@dpdk.org

Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
 drivers/crypto/ipsec_mb/pmd_zuc.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/drivers/crypto/ipsec_mb/pmd_zuc.c b/drivers/crypto/ipsec_mb/pmd_zuc.c
index 2eae1d1ec7..ec83d96dfc 100644
--- a/drivers/crypto/ipsec_mb/pmd_zuc.c
+++ b/drivers/crypto/ipsec_mb/pmd_zuc.c
@@ -198,7 +198,7 @@ process_ops(struct rte_crypto_op **ops, enum ipsec_mb_operation op_type,
 		struct ipsec_mb_qp *qp, uint8_t num_ops)
 {
 	unsigned int i;
-	unsigned int processed_ops;
+	unsigned int processed_ops = 0;
 
 	switch (op_type) {
 	case IPSEC_MB_OP_ENCRYPT_ONLY:
@@ -212,18 +212,21 @@ process_ops(struct rte_crypto_op **ops, enum ipsec_mb_operation op_type,
 				num_ops);
 		break;
 	case IPSEC_MB_OP_ENCRYPT_THEN_HASH_GEN:
+	case IPSEC_MB_OP_DECRYPT_THEN_HASH_VERIFY:
 		processed_ops = process_zuc_cipher_op(qp, ops, sessions,
 				num_ops);
 		process_zuc_hash_op(qp, ops, sessions, processed_ops);
 		break;
 	case IPSEC_MB_OP_HASH_VERIFY_THEN_DECRYPT:
+	case IPSEC_MB_OP_HASH_GEN_THEN_ENCRYPT:
 		processed_ops = process_zuc_hash_op(qp, ops, sessions,
 				num_ops);
 		process_zuc_cipher_op(qp, ops, sessions, processed_ops);
 		break;
 	default:
 		/* Operation not supported. */
-		processed_ops = 0;
+		for (i = 0; i < num_ops; i++)
+			ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
 	}
 
 	for (i = 0; i < num_ops; i++) {
-- 
2.25.1

[PATCH 2/4] crypto/ipsec_mb: fix ZUC authentication verify

[Pablo de Lara]

ZUC authentication is done over multiple buffers at a time.
When authentication verification is done, multiple scratch buffers
are using to generate the tags that will be compared afterwards.
However, the same scratch buffer was used always, instead of having
different ones for each crypto operation.

Fixes: 0b133c36ad7d ("crypto/zuc: support IPsec Multi-buffer lib v0.54")
Cc: pablo.de.lara.guarch@intel.com
Cc: stable@dpdk.org

Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
 drivers/crypto/ipsec_mb/pmd_zuc.c      | 2 +-
 drivers/crypto/ipsec_mb/pmd_zuc_priv.h | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/drivers/crypto/ipsec_mb/pmd_zuc.c b/drivers/crypto/ipsec_mb/pmd_zuc.c
index ec83d96dfc..07cf1462d2 100644
--- a/drivers/crypto/ipsec_mb/pmd_zuc.c
+++ b/drivers/crypto/ipsec_mb/pmd_zuc.c
@@ -166,7 +166,7 @@ process_zuc_hash_op(struct ipsec_mb_qp *qp, struct rte_crypto_op **ops,
 
 		hash_keys[i] = sess->pKey_hash;
 		if (sess->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY)
-			dst[i] = (uint32_t *)qp_data->temp_digest;
+			dst[i] = (uint32_t *)qp_data->temp_digest[i];
 		else
 			dst[i] = (uint32_t *)ops[i]->sym->auth.digest.data;
 
diff --git a/drivers/crypto/ipsec_mb/pmd_zuc_priv.h b/drivers/crypto/ipsec_mb/pmd_zuc_priv.h
index 46d5bfae37..76fd6758c2 100644
--- a/drivers/crypto/ipsec_mb/pmd_zuc_priv.h
+++ b/drivers/crypto/ipsec_mb/pmd_zuc_priv.h
@@ -75,7 +75,7 @@ struct zuc_session {
 
 struct zuc_qp_data {
 
-	uint8_t temp_digest[ZUC_DIGEST_LENGTH];
+	uint8_t temp_digest[ZUC_MAX_BURST][ZUC_DIGEST_LENGTH];
 	/* *< Buffers used to store the digest generated
 	 * by the driver when verifying a digest provided
 	 * by the user (using authentication verify operation)
-- 
2.25.1

[PATCH 3/4] crypto/ipsec_mb: fix crypto operation overwrite

[Pablo de Lara]

ZUC PMD batches crypto operations depending on their type
(encryption + tag generation, tag verification + decryption, etc),
to allow parallelization.
The array used to store the pointers to these operations was
always the same array provided by dequeue_burst() function,
and it was looping around the same positions (from 0 to ZUC_MAX_BURST - 1).

A new internal array is used to avoid overwriting the pointers
of the array provided by dequeue_burst() function.

Fixes: cf7685d68f00 ("crypto/zuc: add driver for ZUC library")
Cc: pablo.de.lara.guarch@intel.com
Cc: stable@dpdk.org

Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
 drivers/crypto/ipsec_mb/pmd_zuc.c | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/drivers/crypto/ipsec_mb/pmd_zuc.c b/drivers/crypto/ipsec_mb/pmd_zuc.c
index 07cf1462d2..e36c7092d6 100644
--- a/drivers/crypto/ipsec_mb/pmd_zuc.c
+++ b/drivers/crypto/ipsec_mb/pmd_zuc.c
@@ -259,6 +259,7 @@ zuc_pmd_dequeue_burst(void *queue_pair,
 
 	struct zuc_session *curr_sess;
 	struct zuc_session *sessions[ZUC_MAX_BURST];
+	struct rte_crypto_op *int_c_ops[ZUC_MAX_BURST];
 	enum ipsec_mb_operation prev_zuc_op = IPSEC_MB_OP_NOT_SUPPORTED;
 	enum ipsec_mb_operation curr_zuc_op;
 	struct ipsec_mb_qp *qp = queue_pair;
@@ -290,11 +291,11 @@ zuc_pmd_dequeue_burst(void *queue_pair,
 		 */
 		if (burst_size == 0) {
 			prev_zuc_op = curr_zuc_op;
-			c_ops[0] = curr_c_op;
+			int_c_ops[0] = curr_c_op;
 			sessions[0] = curr_sess;
 			burst_size++;
 		} else if (curr_zuc_op == prev_zuc_op) {
-			c_ops[burst_size] = curr_c_op;
+			int_c_ops[burst_size] = curr_c_op;
 			sessions[burst_size] = curr_sess;
 			burst_size++;
 			/*
@@ -302,7 +303,7 @@ zuc_pmd_dequeue_burst(void *queue_pair,
 			 * process them, and start a new batch.
 			 */
 			if (burst_size == ZUC_MAX_BURST) {
-				processed_ops = process_ops(c_ops, curr_zuc_op,
+				processed_ops = process_ops(int_c_ops, curr_zuc_op,
 						sessions, qp, burst_size);
 				if (processed_ops < burst_size) {
 					burst_size = 0;
@@ -316,7 +317,7 @@ zuc_pmd_dequeue_burst(void *queue_pair,
 			 * Different operation type, process the ops
 			 * of the previous type.
 			 */
-			processed_ops = process_ops(c_ops, prev_zuc_op,
+			processed_ops = process_ops(int_c_ops, prev_zuc_op,
 					sessions, qp, burst_size);
 			if (processed_ops < burst_size) {
 				burst_size = 0;
@@ -326,7 +327,7 @@ zuc_pmd_dequeue_burst(void *queue_pair,
 			burst_size = 0;
 			prev_zuc_op = curr_zuc_op;
 
-			c_ops[0] = curr_c_op;
+			int_c_ops[0] = curr_c_op;
 			sessions[0] = curr_sess;
 			burst_size++;
 		}
@@ -334,7 +335,7 @@ zuc_pmd_dequeue_burst(void *queue_pair,
 
 	if (burst_size != 0) {
 		/* Process the crypto ops of the last operation type. */
-		processed_ops = process_ops(c_ops, prev_zuc_op,
+		processed_ops = process_ops(int_c_ops, prev_zuc_op,
 				sessions, qp, burst_size);
 	}
 
-- 
2.25.1

[PATCH 4/4] crypto/ipsec_mb: fix length and offset settings

[Pablo de Lara]

KASUMI, SNOW3G and ZUC require lengths and offsets to
be set in bits or bytes depending on the algorithm.
There were some algorithms that were mixing these two,
so this commit is fixing this issue.

Fixes: ae8e085c608d ("crypto/aesni_mb: support KASUMI F8/F9")
Fixes: 6c42e0cf4d12 ("crypto/aesni_mb: support SNOW3G-UEA2/UIA2")
Fixes: fd8df85487c4 ("crypto/aesni_mb: support ZUC-EEA3/EIA3")
Fixes: 8c835018de84 ("crypto/ipsec_mb: support ZUC-256 for aesni_mb")
Cc: pablo.de.lara.guarch@intel.com
Cc: stable@dpdk.org

Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
 drivers/crypto/ipsec_mb/pmd_aesni_mb.c | 120 +++++++++++++++++--------
 1 file changed, 85 insertions(+), 35 deletions(-)

diff --git a/drivers/crypto/ipsec_mb/pmd_aesni_mb.c b/drivers/crypto/ipsec_mb/pmd_aesni_mb.c
index a308d42ffa..eac8f1bd28 100644
--- a/drivers/crypto/ipsec_mb/pmd_aesni_mb.c
+++ b/drivers/crypto/ipsec_mb/pmd_aesni_mb.c
@@ -918,7 +918,9 @@ aesni_mb_set_docsis_sec_session_parameters(
 
 static inline uint64_t
 auth_start_offset(struct rte_crypto_op *op, struct aesni_mb_session *session,
-		uint32_t oop)
+		uint32_t oop, const uint32_t auth_offset,
+		const uint32_t cipher_offset, const uint32_t auth_length,
+		const uint32_t cipher_length)
 {
 	struct rte_mbuf *m_src, *m_dst;
 	uint8_t *p_src, *p_dst;
@@ -927,7 +929,7 @@ auth_start_offset(struct rte_crypto_op *op, struct aesni_mb_session *session,
 
 	/* Only cipher then hash needs special calculation. */
 	if (!oop || session->chain_order != IMB_ORDER_CIPHER_HASH)
-		return op->sym->auth.data.offset;
+		return auth_offset;
 
 	m_src = op->sym->m_src;
 	m_dst = op->sym->m_dst;
@@ -935,24 +937,24 @@ auth_start_offset(struct rte_crypto_op *op, struct aesni_mb_session *session,
 	p_src = rte_pktmbuf_mtod(m_src, uint8_t *);
 	p_dst = rte_pktmbuf_mtod(m_dst, uint8_t *);
 	u_src = (uintptr_t)p_src;
-	u_dst = (uintptr_t)p_dst + op->sym->auth.data.offset;
+	u_dst = (uintptr_t)p_dst + auth_offset;
 
 	/**
 	 * Copy the content between cipher offset and auth offset for generating
 	 * correct digest.
 	 */
-	if (op->sym->cipher.data.offset > op->sym->auth.data.offset)
-		memcpy(p_dst + op->sym->auth.data.offset,
-				p_src + op->sym->auth.data.offset,
-				op->sym->cipher.data.offset -
-				op->sym->auth.data.offset);
+	if (cipher_offset > auth_offset)
+		memcpy(p_dst + auth_offset,
+				p_src + auth_offset,
+				cipher_offset -
+				auth_offset);
 
 	/**
 	 * Copy the content between (cipher offset + length) and (auth offset +
 	 * length) for generating correct digest
 	 */
-	cipher_end = op->sym->cipher.data.offset + op->sym->cipher.data.length;
-	auth_end = op->sym->auth.data.offset + op->sym->auth.data.length;
+	cipher_end = cipher_offset + cipher_length;
+	auth_end = auth_offset + auth_length;
 	if (cipher_end < auth_end)
 		memcpy(p_dst + cipher_end, p_src + cipher_end,
 				auth_end - cipher_end);
@@ -1099,6 +1101,10 @@ set_mb_job_params(IMB_JOB *job, struct ipsec_mb_qp *qp,
 	struct aesni_mb_qp_data *qp_data = ipsec_mb_get_qp_private_data(qp);
 	struct aesni_mb_session *session;
 	uint32_t m_offset, oop;
+	uint32_t auth_off_in_bytes;
+	uint32_t ciph_off_in_bytes;
+	uint32_t auth_len_in_bytes;
+	uint32_t ciph_len_in_bytes;
 
 	session = ipsec_mb_get_session_private(qp, op);
 	if (session == NULL) {
@@ -1207,6 +1213,7 @@ set_mb_job_params(IMB_JOB *job, struct ipsec_mb_qp *qp,
 	if (job->cipher_mode == IMB_CIPHER_ZUC_EEA3) {
 		job->enc_keys = session->cipher.zuc_cipher_key;
 		job->dec_keys = session->cipher.zuc_cipher_key;
+		m_offset >>= 3;
 	} else if (job->cipher_mode == IMB_CIPHER_SNOW3G_UEA2_BITLEN) {
 		job->enc_keys = &session->cipher.pKeySched_snow3g_cipher;
 		m_offset = 0;
@@ -1264,9 +1271,6 @@ set_mb_job_params(IMB_JOB *job, struct ipsec_mb_qp *qp,
 
 	switch (job->hash_alg) {
 	case IMB_AUTH_AES_CCM:
-		job->cipher_start_src_offset_in_bytes =
-				op->sym->aead.data.offset;
-		job->msg_len_to_cipher_in_bytes = op->sym->aead.data.length;
 		job->hash_start_src_offset_in_bytes = op->sym->aead.data.offset;
 		job->msg_len_to_hash_in_bytes = op->sym->aead.data.length;
 
@@ -1276,19 +1280,11 @@ set_mb_job_params(IMB_JOB *job, struct ipsec_mb_qp *qp,
 
 	case IMB_AUTH_AES_GMAC:
 		if (session->cipher.mode == IMB_CIPHER_GCM) {
-			job->cipher_start_src_offset_in_bytes =
-					op->sym->aead.data.offset;
 			job->hash_start_src_offset_in_bytes =
 					op->sym->aead.data.offset;
-			job->msg_len_to_cipher_in_bytes =
-					op->sym->aead.data.length;
 			job->msg_len_to_hash_in_bytes =
 					op->sym->aead.data.length;
 		} else {
-			job->cipher_start_src_offset_in_bytes =
-					op->sym->auth.data.offset;
-			job->hash_start_src_offset_in_bytes =
-					op->sym->auth.data.offset;
 			job->msg_len_to_cipher_in_bytes = 0;
 			job->msg_len_to_hash_in_bytes = 0;
 		}
@@ -1298,36 +1294,90 @@ set_mb_job_params(IMB_JOB *job, struct ipsec_mb_qp *qp,
 		break;
 
 	case IMB_AUTH_CHACHA20_POLY1305:
-		job->cipher_start_src_offset_in_bytes =
-			op->sym->aead.data.offset;
 		job->hash_start_src_offset_in_bytes =
 			op->sym->aead.data.offset;
-		job->msg_len_to_cipher_in_bytes =
-				op->sym->aead.data.length;
 		job->msg_len_to_hash_in_bytes =
 					op->sym->aead.data.length;
 
 		job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
 				session->iv.offset);
 		break;
-	default:
-		/* For SNOW3G, length and offsets are already in bits */
-		job->cipher_start_src_offset_in_bytes =
-				op->sym->cipher.data.offset;
-		job->msg_len_to_cipher_in_bytes = op->sym->cipher.data.length;
 
+	/* ZUC and SNOW3G require length in bits and offset in bytes */
+	case IMB_AUTH_ZUC_EIA3_BITLEN:
+	case IMB_AUTH_ZUC256_EIA3_BITLEN:
+	case IMB_AUTH_SNOW3G_UIA2_BITLEN:
+		auth_off_in_bytes = op->sym->auth.data.offset >> 3;
+		ciph_off_in_bytes = op->sym->cipher.data.offset >> 3;
+		auth_len_in_bytes = op->sym->auth.data.length >> 3;
+		ciph_len_in_bytes = op->sym->cipher.data.length >> 3;
+
+		job->hash_start_src_offset_in_bytes = auth_start_offset(op,
+				session, oop, auth_off_in_bytes,
+				ciph_off_in_bytes, auth_len_in_bytes,
+				ciph_len_in_bytes);
+		job->msg_len_to_hash_in_bits = op->sym->auth.data.length;
+
+		job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
+			session->iv.offset);
+		break;
+
+	/* KASUMI requires lengths and offset in bytes */
+	case IMB_AUTH_KASUMI_UIA1:
+		auth_off_in_bytes = op->sym->auth.data.offset >> 3;
+		ciph_off_in_bytes = op->sym->cipher.data.offset >> 3;
+		auth_len_in_bytes = op->sym->auth.data.length >> 3;
+		ciph_len_in_bytes = op->sym->cipher.data.length >> 3;
+
+		job->hash_start_src_offset_in_bytes = auth_start_offset(op,
+				session, oop, auth_off_in_bytes,
+				ciph_off_in_bytes, auth_len_in_bytes,
+				ciph_len_in_bytes);
+		job->msg_len_to_hash_in_bytes = auth_len_in_bytes;
+
+		job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
+			session->iv.offset);
+	break;
+
+	default:
 		job->hash_start_src_offset_in_bytes = auth_start_offset(op,
-				session, oop);
+				session, oop, op->sym->auth.data.offset,
+		op->sym->cipher.data.offset,
+		op->sym->auth.data.length, op->sym->cipher.data.length);
 		job->msg_len_to_hash_in_bytes = op->sym->auth.data.length;
 
 		job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
 			session->iv.offset);
 	}
 
-	if (job->cipher_mode == IMB_CIPHER_ZUC_EEA3)
-		job->msg_len_to_cipher_in_bytes >>= 3;
-	else if (job->hash_alg == IMB_AUTH_KASUMI_UIA1)
-		job->msg_len_to_hash_in_bytes >>= 3;
+	switch (job->cipher_mode) {
+	/* ZUC requires length and offset in bytes */
+	case IMB_CIPHER_ZUC_EEA3:
+		job->cipher_start_src_offset_in_bytes =
+					op->sym->cipher.data.offset >> 3;
+		job->msg_len_to_cipher_in_bytes =
+					op->sym->cipher.data.length >> 3;
+	break;
+	/* ZUC and SNOW3G require length and offset in bits */
+	case IMB_CIPHER_SNOW3G_UEA2_BITLEN:
+	case IMB_CIPHER_KASUMI_UEA1_BITLEN:
+		job->cipher_start_src_offset_in_bits =
+					op->sym->cipher.data.offset;
+		job->msg_len_to_cipher_in_bits =
+					op->sym->cipher.data.length;
+	break;
+	case IMB_CIPHER_CCM:
+	case IMB_CIPHER_GCM:
+	case IMB_CIPHER_CHACHA20_POLY1305:
+		job->cipher_start_src_offset_in_bytes =
+				op->sym->aead.data.offset;
+		job->msg_len_to_cipher_in_bytes = op->sym->aead.data.length;
+	break;
+	default:
+		job->cipher_start_src_offset_in_bits =
+					op->sym->cipher.data.offset;
+		job->msg_len_to_cipher_in_bits = op->sym->cipher.data.length;
+	}
 
 	/* Set user data to be crypto operation data struct */
 	job->user_data = op;
-- 
2.25.1

RE: [EXT] [PATCH 4/4] crypto/ipsec_mb: fix length and offset settings

[Akhil Goyal]

Hi Pablo,

> KASUMI, SNOW3G and ZUC require lengths and offsets to
> be set in bits or bytes depending on the algorithm.
> There were some algorithms that were mixing these two,
> so this commit is fixing this issue.
> 
> Fixes: ae8e085c608d ("crypto/aesni_mb: support KASUMI F8/F9")
> Fixes: 6c42e0cf4d12 ("crypto/aesni_mb: support SNOW3G-UEA2/UIA2")
> Fixes: fd8df85487c4 ("crypto/aesni_mb: support ZUC-EEA3/EIA3")
> Fixes: 8c835018de84 ("crypto/ipsec_mb: support ZUC-256 for aesni_mb")
> Cc: pablo.de.lara.guarch@intel.com
> Cc: stable@dpdk.org
> 
> Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
> ---
Could you please rebase this patchset. It does not apply cleanly.

[PATCH v2 0/4] ipsec_mb fixes for ZUC algorithm

[Pablo de Lara]

This patchset fixes various issues affecting ZUC and AESNI MB PMD,
when ZUC algorithm is used.

-v2: rebased on main branch

Pablo de Lara (4):
  crypto/ipsec_mb: check for missing operation types
  crypto/ipsec_mb: fix ZUC authentication verify
  crypto/ipsec_mb: fix crypto operation overwrite
  crypto/ipsec_mb: fix length and offset settings

 drivers/crypto/ipsec_mb/pmd_aesni_mb.c | 122 +++++++++++++++++--------
 drivers/crypto/ipsec_mb/pmd_zuc.c      |  22 +++--
 drivers/crypto/ipsec_mb/pmd_zuc_priv.h |   2 +-
 3 files changed, 100 insertions(+), 46 deletions(-)

-- 
2.25.1

[PATCH v2 1/4] crypto/ipsec_mb: check for missing operation types

[Pablo de Lara]

When processing crypto operations in ZUC PMD,
there were two operation types that were set at session level,
but not checked when the operations are enqueued and processed,
leaving the buffers untouched silently.

Fixes: cde8df1bda9d ("crypto/ipsec_mb: move zuc PMD")
Cc: piotrx.bronowski@intel.com
Cc: stable@dpdk.org

Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
 drivers/crypto/ipsec_mb/pmd_zuc.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/drivers/crypto/ipsec_mb/pmd_zuc.c b/drivers/crypto/ipsec_mb/pmd_zuc.c
index 2eae1d1ec7..ec83d96dfc 100644
--- a/drivers/crypto/ipsec_mb/pmd_zuc.c
+++ b/drivers/crypto/ipsec_mb/pmd_zuc.c
@@ -198,7 +198,7 @@ process_ops(struct rte_crypto_op **ops, enum ipsec_mb_operation op_type,
 		struct ipsec_mb_qp *qp, uint8_t num_ops)
 {
 	unsigned int i;
-	unsigned int processed_ops;
+	unsigned int processed_ops = 0;
 
 	switch (op_type) {
 	case IPSEC_MB_OP_ENCRYPT_ONLY:
@@ -212,18 +212,21 @@ process_ops(struct rte_crypto_op **ops, enum ipsec_mb_operation op_type,
 				num_ops);
 		break;
 	case IPSEC_MB_OP_ENCRYPT_THEN_HASH_GEN:
+	case IPSEC_MB_OP_DECRYPT_THEN_HASH_VERIFY:
 		processed_ops = process_zuc_cipher_op(qp, ops, sessions,
 				num_ops);
 		process_zuc_hash_op(qp, ops, sessions, processed_ops);
 		break;
 	case IPSEC_MB_OP_HASH_VERIFY_THEN_DECRYPT:
+	case IPSEC_MB_OP_HASH_GEN_THEN_ENCRYPT:
 		processed_ops = process_zuc_hash_op(qp, ops, sessions,
 				num_ops);
 		process_zuc_cipher_op(qp, ops, sessions, processed_ops);
 		break;
 	default:
 		/* Operation not supported. */
-		processed_ops = 0;
+		for (i = 0; i < num_ops; i++)
+			ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
 	}
 
 	for (i = 0; i < num_ops; i++) {
-- 
2.25.1

[PATCH v2 2/4] crypto/ipsec_mb: fix ZUC authentication verify

[Pablo de Lara]

ZUC authentication is done over multiple buffers at a time.
When authentication verification is done, multiple scratch buffers
are using to generate the tags that will be compared afterwards.
However, the same scratch buffer was used always, instead of having
different ones for each crypto operation.

Fixes: 0b133c36ad7d ("crypto/zuc: support IPsec Multi-buffer lib v0.54")
Cc: pablo.de.lara.guarch@intel.com
Cc: stable@dpdk.org

Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
 drivers/crypto/ipsec_mb/pmd_zuc.c      | 2 +-
 drivers/crypto/ipsec_mb/pmd_zuc_priv.h | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/drivers/crypto/ipsec_mb/pmd_zuc.c b/drivers/crypto/ipsec_mb/pmd_zuc.c
index ec83d96dfc..07cf1462d2 100644
--- a/drivers/crypto/ipsec_mb/pmd_zuc.c
+++ b/drivers/crypto/ipsec_mb/pmd_zuc.c
@@ -166,7 +166,7 @@ process_zuc_hash_op(struct ipsec_mb_qp *qp, struct rte_crypto_op **ops,
 
 		hash_keys[i] = sess->pKey_hash;
 		if (sess->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY)
-			dst[i] = (uint32_t *)qp_data->temp_digest;
+			dst[i] = (uint32_t *)qp_data->temp_digest[i];
 		else
 			dst[i] = (uint32_t *)ops[i]->sym->auth.digest.data;
 
diff --git a/drivers/crypto/ipsec_mb/pmd_zuc_priv.h b/drivers/crypto/ipsec_mb/pmd_zuc_priv.h
index 46d5bfae37..76fd6758c2 100644
--- a/drivers/crypto/ipsec_mb/pmd_zuc_priv.h
+++ b/drivers/crypto/ipsec_mb/pmd_zuc_priv.h
@@ -75,7 +75,7 @@ struct zuc_session {
 
 struct zuc_qp_data {
 
-	uint8_t temp_digest[ZUC_DIGEST_LENGTH];
+	uint8_t temp_digest[ZUC_MAX_BURST][ZUC_DIGEST_LENGTH];
 	/* *< Buffers used to store the digest generated
 	 * by the driver when verifying a digest provided
 	 * by the user (using authentication verify operation)
-- 
2.25.1

[PATCH v2 3/4] crypto/ipsec_mb: fix crypto operation overwrite

[Pablo de Lara]

ZUC PMD batches crypto operations depending on their type
(encryption + tag generation, tag verification + decryption, etc),
to allow parallelization.
The array used to store the pointers to these operations was
always the same array provided by dequeue_burst() function,
and it was looping around the same positions (from 0 to ZUC_MAX_BURST - 1).

A new internal array is used to avoid overwriting the pointers
of the array provided by dequeue_burst() function.

Fixes: cf7685d68f00 ("crypto/zuc: add driver for ZUC library")
Cc: pablo.de.lara.guarch@intel.com
Cc: stable@dpdk.org

Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
 drivers/crypto/ipsec_mb/pmd_zuc.c | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/drivers/crypto/ipsec_mb/pmd_zuc.c b/drivers/crypto/ipsec_mb/pmd_zuc.c
index 07cf1462d2..e36c7092d6 100644
--- a/drivers/crypto/ipsec_mb/pmd_zuc.c
+++ b/drivers/crypto/ipsec_mb/pmd_zuc.c
@@ -259,6 +259,7 @@ zuc_pmd_dequeue_burst(void *queue_pair,
 
 	struct zuc_session *curr_sess;
 	struct zuc_session *sessions[ZUC_MAX_BURST];
+	struct rte_crypto_op *int_c_ops[ZUC_MAX_BURST];
 	enum ipsec_mb_operation prev_zuc_op = IPSEC_MB_OP_NOT_SUPPORTED;
 	enum ipsec_mb_operation curr_zuc_op;
 	struct ipsec_mb_qp *qp = queue_pair;
@@ -290,11 +291,11 @@ zuc_pmd_dequeue_burst(void *queue_pair,
 		 */
 		if (burst_size == 0) {
 			prev_zuc_op = curr_zuc_op;
-			c_ops[0] = curr_c_op;
+			int_c_ops[0] = curr_c_op;
 			sessions[0] = curr_sess;
 			burst_size++;
 		} else if (curr_zuc_op == prev_zuc_op) {
-			c_ops[burst_size] = curr_c_op;
+			int_c_ops[burst_size] = curr_c_op;
 			sessions[burst_size] = curr_sess;
 			burst_size++;
 			/*
@@ -302,7 +303,7 @@ zuc_pmd_dequeue_burst(void *queue_pair,
 			 * process them, and start a new batch.
 			 */
 			if (burst_size == ZUC_MAX_BURST) {
-				processed_ops = process_ops(c_ops, curr_zuc_op,
+				processed_ops = process_ops(int_c_ops, curr_zuc_op,
 						sessions, qp, burst_size);
 				if (processed_ops < burst_size) {
 					burst_size = 0;
@@ -316,7 +317,7 @@ zuc_pmd_dequeue_burst(void *queue_pair,
 			 * Different operation type, process the ops
 			 * of the previous type.
 			 */
-			processed_ops = process_ops(c_ops, prev_zuc_op,
+			processed_ops = process_ops(int_c_ops, prev_zuc_op,
 					sessions, qp, burst_size);
 			if (processed_ops < burst_size) {
 				burst_size = 0;
@@ -326,7 +327,7 @@ zuc_pmd_dequeue_burst(void *queue_pair,
 			burst_size = 0;
 			prev_zuc_op = curr_zuc_op;
 
-			c_ops[0] = curr_c_op;
+			int_c_ops[0] = curr_c_op;
 			sessions[0] = curr_sess;
 			burst_size++;
 		}
@@ -334,7 +335,7 @@ zuc_pmd_dequeue_burst(void *queue_pair,
 
 	if (burst_size != 0) {
 		/* Process the crypto ops of the last operation type. */
-		processed_ops = process_ops(c_ops, prev_zuc_op,
+		processed_ops = process_ops(int_c_ops, prev_zuc_op,
 				sessions, qp, burst_size);
 	}
 
-- 
2.25.1

[PATCH v2 4/4] crypto/ipsec_mb: fix length and offset settings

[Pablo de Lara]

KASUMI, SNOW3G and ZUC require lengths and offsets to
be set in bits or bytes depending on the algorithm.
There were some algorithms that were mixing these two,
so this commit is fixing this issue.

Fixes: ae8e085c608d ("crypto/aesni_mb: support KASUMI F8/F9")
Fixes: 6c42e0cf4d12 ("crypto/aesni_mb: support SNOW3G-UEA2/UIA2")
Fixes: fd8df85487c4 ("crypto/aesni_mb: support ZUC-EEA3/EIA3")
Fixes: 8c835018de84 ("crypto/ipsec_mb: support ZUC-256 for aesni_mb")
Cc: pablo.de.lara.guarch@intel.com
Cc: stable@dpdk.org

Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
---
 drivers/crypto/ipsec_mb/pmd_aesni_mb.c | 122 +++++++++++++++++--------
 1 file changed, 86 insertions(+), 36 deletions(-)

diff --git a/drivers/crypto/ipsec_mb/pmd_aesni_mb.c b/drivers/crypto/ipsec_mb/pmd_aesni_mb.c
index 0111c6f540..c974886032 100644
--- a/drivers/crypto/ipsec_mb/pmd_aesni_mb.c
+++ b/drivers/crypto/ipsec_mb/pmd_aesni_mb.c
@@ -930,7 +930,9 @@ aesni_mb_set_docsis_sec_session_parameters(
 
 static inline uint64_t
 auth_start_offset(struct rte_crypto_op *op, struct aesni_mb_session *session,
-		uint32_t oop)
+		uint32_t oop, const uint32_t auth_offset,
+		const uint32_t cipher_offset, const uint32_t auth_length,
+		const uint32_t cipher_length)
 {
 	struct rte_mbuf *m_src, *m_dst;
 	uint8_t *p_src, *p_dst;
@@ -939,7 +941,7 @@ auth_start_offset(struct rte_crypto_op *op, struct aesni_mb_session *session,
 
 	/* Only cipher then hash needs special calculation. */
 	if (!oop || session->chain_order != IMB_ORDER_CIPHER_HASH)
-		return op->sym->auth.data.offset;
+		return auth_offset;
 
 	m_src = op->sym->m_src;
 	m_dst = op->sym->m_dst;
@@ -947,24 +949,24 @@ auth_start_offset(struct rte_crypto_op *op, struct aesni_mb_session *session,
 	p_src = rte_pktmbuf_mtod(m_src, uint8_t *);
 	p_dst = rte_pktmbuf_mtod(m_dst, uint8_t *);
 	u_src = (uintptr_t)p_src;
-	u_dst = (uintptr_t)p_dst + op->sym->auth.data.offset;
+	u_dst = (uintptr_t)p_dst + auth_offset;
 
 	/**
 	 * Copy the content between cipher offset and auth offset for generating
 	 * correct digest.
 	 */
-	if (op->sym->cipher.data.offset > op->sym->auth.data.offset)
-		memcpy(p_dst + op->sym->auth.data.offset,
-				p_src + op->sym->auth.data.offset,
-				op->sym->cipher.data.offset -
-				op->sym->auth.data.offset);
+	if (cipher_offset > auth_offset)
+		memcpy(p_dst + auth_offset,
+				p_src + auth_offset,
+				cipher_offset -
+				auth_offset);
 
 	/**
 	 * Copy the content between (cipher offset + length) and (auth offset +
 	 * length) for generating correct digest
 	 */
-	cipher_end = op->sym->cipher.data.offset + op->sym->cipher.data.length;
-	auth_end = op->sym->auth.data.offset + op->sym->auth.data.length;
+	cipher_end = cipher_offset + cipher_length;
+	auth_end = auth_offset + auth_length;
 	if (cipher_end < auth_end)
 		memcpy(p_dst + cipher_end, p_src + cipher_end,
 				auth_end - cipher_end);
@@ -1111,6 +1113,10 @@ set_mb_job_params(IMB_JOB *job, struct ipsec_mb_qp *qp,
 	struct aesni_mb_qp_data *qp_data = ipsec_mb_get_qp_private_data(qp);
 	struct aesni_mb_session *session;
 	uint32_t m_offset, oop;
+	uint32_t auth_off_in_bytes;
+	uint32_t ciph_off_in_bytes;
+	uint32_t auth_len_in_bytes;
+	uint32_t ciph_len_in_bytes;
 
 	session = ipsec_mb_get_session_private(qp, op);
 	if (session == NULL) {
@@ -1219,6 +1225,7 @@ set_mb_job_params(IMB_JOB *job, struct ipsec_mb_qp *qp,
 	if (job->cipher_mode == IMB_CIPHER_ZUC_EEA3) {
 		job->enc_keys = session->cipher.zuc_cipher_key;
 		job->dec_keys = session->cipher.zuc_cipher_key;
+		m_offset >>= 3;
 	} else if (job->cipher_mode == IMB_CIPHER_SNOW3G_UEA2_BITLEN) {
 		job->enc_keys = &session->cipher.pKeySched_snow3g_cipher;
 		m_offset = 0;
@@ -1276,9 +1283,6 @@ set_mb_job_params(IMB_JOB *job, struct ipsec_mb_qp *qp,
 
 	switch (job->hash_alg) {
 	case IMB_AUTH_AES_CCM:
-		job->cipher_start_src_offset_in_bytes =
-				op->sym->aead.data.offset;
-		job->msg_len_to_cipher_in_bytes = op->sym->aead.data.length;
 		job->hash_start_src_offset_in_bytes = op->sym->aead.data.offset;
 		job->msg_len_to_hash_in_bytes = op->sym->aead.data.length;
 
@@ -1288,19 +1292,11 @@ set_mb_job_params(IMB_JOB *job, struct ipsec_mb_qp *qp,
 
 	case IMB_AUTH_AES_GMAC:
 		if (session->cipher.mode == IMB_CIPHER_GCM) {
-			job->cipher_start_src_offset_in_bytes =
-					op->sym->aead.data.offset;
 			job->hash_start_src_offset_in_bytes =
 					op->sym->aead.data.offset;
-			job->msg_len_to_cipher_in_bytes =
-					op->sym->aead.data.length;
 			job->msg_len_to_hash_in_bytes =
 					op->sym->aead.data.length;
 		} else {
-			job->cipher_start_src_offset_in_bytes =
-					op->sym->auth.data.offset;
-			job->hash_start_src_offset_in_bytes =
-					op->sym->auth.data.offset;
 			job->msg_len_to_cipher_in_bytes = 0;
 			job->msg_len_to_hash_in_bytes = 0;
 		}
@@ -1310,43 +1306,97 @@ set_mb_job_params(IMB_JOB *job, struct ipsec_mb_qp *qp,
 		break;
 
 	case IMB_AUTH_CHACHA20_POLY1305:
-		job->cipher_start_src_offset_in_bytes =
-			op->sym->aead.data.offset;
 		job->hash_start_src_offset_in_bytes =
 			op->sym->aead.data.offset;
-		job->msg_len_to_cipher_in_bytes =
-				op->sym->aead.data.length;
 		job->msg_len_to_hash_in_bytes =
 					op->sym->aead.data.length;
 
 		job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
 				session->iv.offset);
 		break;
-	default:
-		/* For SNOW3G, length and offsets are already in bits */
-		job->cipher_start_src_offset_in_bytes =
-				op->sym->cipher.data.offset;
-		job->msg_len_to_cipher_in_bytes = op->sym->cipher.data.length;
+	/* ZUC and SNOW3G require length in bits and offset in bytes */
+	case IMB_AUTH_ZUC_EIA3_BITLEN:
+	case IMB_AUTH_ZUC256_EIA3_BITLEN:
+	case IMB_AUTH_SNOW3G_UIA2_BITLEN:
+		auth_off_in_bytes = op->sym->auth.data.offset >> 3;
+		ciph_off_in_bytes = op->sym->cipher.data.offset >> 3;
+		auth_len_in_bytes = op->sym->auth.data.length >> 3;
+		ciph_len_in_bytes = op->sym->cipher.data.length >> 3;
+
+		job->hash_start_src_offset_in_bytes = auth_start_offset(op,
+				session, oop, auth_off_in_bytes,
+				ciph_off_in_bytes, auth_len_in_bytes,
+				ciph_len_in_bytes);
+		job->msg_len_to_hash_in_bits = op->sym->auth.data.length;
+
+		job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
+			session->iv.offset);
+		break;
+
+	/* KASUMI requires lengths and offset in bytes */
+	case IMB_AUTH_KASUMI_UIA1:
+		auth_off_in_bytes = op->sym->auth.data.offset >> 3;
+		ciph_off_in_bytes = op->sym->cipher.data.offset >> 3;
+		auth_len_in_bytes = op->sym->auth.data.length >> 3;
+		ciph_len_in_bytes = op->sym->cipher.data.length >> 3;
 
 		job->hash_start_src_offset_in_bytes = auth_start_offset(op,
-				session, oop);
+				session, oop, auth_off_in_bytes,
+				ciph_off_in_bytes, auth_len_in_bytes,
+				ciph_len_in_bytes);
+		job->msg_len_to_hash_in_bytes = auth_len_in_bytes;
+
+		job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
+			session->iv.offset);
+		break;
+
+	default:
+		job->hash_start_src_offset_in_bytes = auth_start_offset(op,
+				session, oop, op->sym->auth.data.offset,
+				op->sym->cipher.data.offset,
+				op->sym->auth.data.length,
+				op->sym->cipher.data.length);
 		job->msg_len_to_hash_in_bytes = op->sym->auth.data.length;
 
 		job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
 			session->iv.offset);
 	}
 
+	switch (job->cipher_mode) {
+	/* ZUC requires length and offset in bytes */
+	case IMB_CIPHER_ZUC_EEA3:
+		job->cipher_start_src_offset_in_bytes =
+					op->sym->cipher.data.offset >> 3;
+		job->msg_len_to_cipher_in_bytes =
+					op->sym->cipher.data.length >> 3;
+		break;
+	/* ZUC and SNOW3G require length and offset in bits */
+	case IMB_CIPHER_SNOW3G_UEA2_BITLEN:
+	case IMB_CIPHER_KASUMI_UEA1_BITLEN:
+		job->cipher_start_src_offset_in_bits =
+					op->sym->cipher.data.offset;
+		job->msg_len_to_cipher_in_bits =
+					op->sym->cipher.data.length;
+		break;
+	case IMB_CIPHER_CCM:
+	case IMB_CIPHER_GCM:
+	case IMB_CIPHER_CHACHA20_POLY1305:
+		job->cipher_start_src_offset_in_bytes =
+				op->sym->aead.data.offset;
+		job->msg_len_to_cipher_in_bytes = op->sym->aead.data.length;
+		break;
+	default:
+		job->cipher_start_src_offset_in_bytes =
+					op->sym->cipher.data.offset;
+		job->msg_len_to_cipher_in_bytes = op->sym->cipher.data.length;
+	}
+
 	if (job->cipher_mode == IMB_CIPHER_NULL && oop) {
 		memcpy(job->dst + job->cipher_start_src_offset_in_bytes,
 			job->src + job->cipher_start_src_offset_in_bytes,
 			job->msg_len_to_cipher_in_bytes);
 	}
 
-	if (job->cipher_mode == IMB_CIPHER_ZUC_EEA3)
-		job->msg_len_to_cipher_in_bytes >>= 3;
-	else if (job->hash_alg == IMB_AUTH_KASUMI_UIA1)
-		job->msg_len_to_hash_in_bytes >>= 3;
-
 	/* Set user data to be crypto operation data struct */
 	job->user_data = op;
 
-- 
2.25.1

RE: [EXT] [PATCH v2 0/4] ipsec_mb fixes for ZUC algorithm

[Akhil Goyal]

> This patchset fixes various issues affecting ZUC and AESNI MB PMD,
> when ZUC algorithm is used.
> 
> -v2: rebased on main branch
> 
> Pablo de Lara (4):
>   crypto/ipsec_mb: check for missing operation types
>   crypto/ipsec_mb: fix ZUC authentication verify
>   crypto/ipsec_mb: fix crypto operation overwrite
>   crypto/ipsec_mb: fix length and offset settings
> 
>  drivers/crypto/ipsec_mb/pmd_aesni_mb.c | 122 +++++++++++++++++--------
>  drivers/crypto/ipsec_mb/pmd_zuc.c      |  22 +++--
>  drivers/crypto/ipsec_mb/pmd_zuc_priv.h |   2 +-
>  3 files changed, 100 insertions(+), 46 deletions(-)
> 
Series Applied to dpdk-next-crypto

Thanks.