[dpdk-dev] [PATCH 0/7] Improvements and fixes in crypto/cnxk PMDs

[dpdk-dev] [PATCH 0/7] Improvements and fixes in crypto/cnxk PMDs

[Anoob Joseph]

Improvements and fixes to crypto_cn9k & crypto_cn10k PMDs.
- Updated IPsec completion codes to match latest firmware
- Updated ZUC API to match latest firmware
- Added dual submission in crypto_cn9k PMD

Depends on:
http://patches.dpdk.org/project/dpdk/list/?series=18627

Anoob Joseph (5):
  common/cnxk: update to v1.16 ucc codes
  crypto/cnxk: remove redundant snow3g dec
  crypto/cnxk: remove redundant memcpy of IV for ZUC
  crypto/cnxk: remove redundant assignment
  crypto/cnxk: add dual submission

Vidya Sagar Velumuri (2):
  common/cnxk: update to v1.13 ZUC API
  crypto/cnxk: support for ucode API change

 drivers/common/cnxk/roc_ie_ot.h           |  65 +++----
 drivers/common/cnxk/roc_se.c              | 150 +++++++++++++--
 drivers/common/cnxk/roc_se.h              |  37 +++-
 drivers/common/cnxk/version.map           |   1 +
 drivers/crypto/cnxk/cn10k_cryptodev_ops.c |   2 +-
 drivers/crypto/cnxk/cn9k_cryptodev_ops.c  | 157 +++++++++++++---
 drivers/crypto/cnxk/cnxk_se.h             | 298 ++++++------------------------
 7 files changed, 389 insertions(+), 321 deletions(-)

-- 
2.7.4

[dpdk-dev] [PATCH 1/7] common/cnxk: update to v1.13 ZUC API

[Anoob Joseph]

From: Vidya Sagar Velumuri <vvelumuri@marvell.com>

Add support for ZUC API change in ucode 1.13

Signed-off-by: Anoob Joseph <anoobj@marvell.com>
Signed-off-by: Vidya Sagar Velumuri <vvelumuri@marvell.com>
---
 drivers/common/cnxk/roc_se.c    | 150 ++++++++++++++++++++++++++++++++++++----
 drivers/common/cnxk/roc_se.h    |  37 +++++++++-
 drivers/common/cnxk/version.map |   1 +
 3 files changed, 172 insertions(+), 16 deletions(-)

diff --git a/drivers/common/cnxk/roc_se.c b/drivers/common/cnxk/roc_se.c
index b04de79..03fbc5f 100644
--- a/drivers/common/cnxk/roc_se.c
+++ b/drivers/common/cnxk/roc_se.c
@@ -72,6 +72,11 @@ cpt_ciph_type_set(roc_se_cipher_type type, struct roc_se_ctx *ctx,
 		fc_type = ROC_SE_FC_GEN;
 		break;
 	case ROC_SE_ZUC_EEA3:
+		/* No support for chained operations */
+		if (unlikely(ctx->hash_type))
+			return -1;
+		fc_type = ROC_SE_PDCP;
+		break;
 	case ROC_SE_SNOW3G_UEA2:
 		if (unlikely(key_len != 16))
 			return -1;
@@ -123,6 +128,56 @@ cpt_ciph_aes_key_type_set(struct roc_se_context *fctx, uint16_t key_len)
 	fctx->enc.aes_key = aes_key_type;
 }
 
+static int
+cpt_pdcp_key_type_set(struct roc_se_zuc_snow3g_ctx *zs_ctx, uint16_t key_len)
+{
+	roc_se_aes_type key_type = 0;
+
+	if (roc_model_is_cn9k()) {
+		if (key_len != 16) {
+			plt_err("Only key len 16 is supported on cn9k");
+			return -ENOTSUP;
+		}
+	}
+
+	switch (key_len) {
+	case 16:
+		key_type = ROC_SE_AES_128_BIT;
+		break;
+	case 32:
+		key_type = ROC_SE_AES_256_BIT;
+		break;
+	default:
+		plt_err("Invalid AES key len");
+		return -ENOTSUP;
+	}
+	zs_ctx->zuc.otk_ctx.w0.s.key_len = key_type;
+	return 0;
+}
+
+static int
+cpt_pdcp_mac_len_set(struct roc_se_zuc_snow3g_ctx *zs_ctx, uint16_t mac_len)
+{
+	roc_se_pdcp_mac_len_type mac_type = 0;
+
+	switch (mac_len) {
+	case 4:
+		mac_type = ROC_SE_PDCP_MAC_LEN_32_BIT;
+		break;
+	case 8:
+		mac_type = ROC_SE_PDCP_MAC_LEN_64_BIT;
+		break;
+	case 16:
+		mac_type = ROC_SE_PDCP_MAC_LEN_128_BIT;
+		break;
+	default:
+		plt_err("Invalid ZUC MAC len");
+		return -ENOTSUP;
+	}
+	zs_ctx->zuc.otk_ctx.w0.s.mac_len = mac_type;
+	return 0;
+}
+
 int
 roc_se_auth_key_set(struct roc_se_ctx *se_ctx, roc_se_auth_type type,
 		    const uint8_t *key, uint16_t key_len, uint16_t mac_len)
@@ -130,6 +185,7 @@ roc_se_auth_key_set(struct roc_se_ctx *se_ctx, roc_se_auth_type type,
 	struct roc_se_zuc_snow3g_ctx *zs_ctx;
 	struct roc_se_kasumi_ctx *k_ctx;
 	struct roc_se_context *fctx;
+	int ret;
 
 	if (se_ctx == NULL)
 		return -1;
@@ -139,32 +195,57 @@ roc_se_auth_key_set(struct roc_se_ctx *se_ctx, roc_se_auth_type type,
 	fctx = &se_ctx->se_ctx.fctx;
 
 	if ((type >= ROC_SE_ZUC_EIA3) && (type <= ROC_SE_KASUMI_F9_ECB)) {
+		uint8_t *zuc_const;
 		uint32_t keyx[4];
+		uint8_t *ci_key;
 
-		if (key_len != 16)
+		if (!key_len)
 			return -1;
-		/* No support for AEAD yet */
+
+		/* No support for chained operations yet */
 		if (se_ctx->enc_cipher)
 			return -1;
+
+		if (roc_model_is_cn9k()) {
+			ci_key = zs_ctx->zuc.onk_ctx.ci_key;
+			zuc_const = zs_ctx->zuc.onk_ctx.zuc_const;
+		} else {
+			ci_key = zs_ctx->zuc.otk_ctx.ci_key;
+			zuc_const = zs_ctx->zuc.otk_ctx.zuc_const;
+		}
+
 		/* For ZUC/SNOW3G/Kasumi */
 		switch (type) {
 		case ROC_SE_SNOW3G_UIA2:
+			zs_ctx->zuc.otk_ctx.w0.s.alg_type =
+				ROC_SE_PDCP_ALG_TYPE_SNOW3G;
+			zs_ctx->zuc.otk_ctx.w0.s.mac_len =
+				ROC_SE_PDCP_MAC_LEN_32_BIT;
 			se_ctx->pdcp_alg_type = ROC_SE_PDCP_ALG_TYPE_SNOW3G;
 			cpt_snow3g_key_gen(key, keyx);
-			memcpy(zs_ctx->ci_key, keyx, key_len);
+			memcpy(ci_key, keyx, key_len);
 			se_ctx->fc_type = ROC_SE_PDCP;
 			se_ctx->zsk_flags = 0x1;
 			break;
 		case ROC_SE_ZUC_EIA3:
+			zs_ctx->zuc.otk_ctx.w0.s.alg_type =
+				ROC_SE_PDCP_ALG_TYPE_ZUC;
+			ret = cpt_pdcp_mac_len_set(zs_ctx, mac_len);
+			if (ret)
+				return ret;
 			se_ctx->pdcp_alg_type = ROC_SE_PDCP_ALG_TYPE_ZUC;
-			memcpy(zs_ctx->ci_key, key, key_len);
-			memcpy(zs_ctx->zuc_const, zuc_d, 32);
+			memcpy(ci_key, key, key_len);
+			memcpy(zuc_const, zuc_d, 32);
 			se_ctx->fc_type = ROC_SE_PDCP;
 			se_ctx->zsk_flags = 0x1;
 			break;
 		case ROC_SE_AES_CMAC_EIA2:
+			zs_ctx->zuc.otk_ctx.w0.s.alg_type =
+				ROC_SE_PDCP_ALG_TYPE_AES_CTR;
+			zs_ctx->zuc.otk_ctx.w0.s.mac_len =
+				ROC_SE_PDCP_MAC_LEN_32_BIT;
 			se_ctx->pdcp_alg_type = ROC_SE_PDCP_ALG_TYPE_AES_CTR;
-			memcpy(zs_ctx->ci_key, key, key_len);
+			memcpy(ci_key, key, key_len);
 			se_ctx->fc_type = ROC_SE_PDCP;
 			se_ctx->zsk_flags = 0x1;
 			break;
@@ -183,8 +264,13 @@ roc_se_auth_key_set(struct roc_se_ctx *se_ctx, roc_se_auth_type type,
 		default:
 			return -1;
 		}
-		se_ctx->mac_len = 4;
+		se_ctx->mac_len = mac_len;
 		se_ctx->hash_type = type;
+		if (roc_model_is_cn9k())
+			se_ctx->template_w4.s.opcode_minor =
+				((1 << 7) | (se_ctx->pdcp_alg_type << 5) | 1);
+		else
+			se_ctx->template_w4.s.opcode_minor = ((1 << 4) | 1);
 		return 0;
 	}
 
@@ -227,11 +313,21 @@ int
 roc_se_ciph_key_set(struct roc_se_ctx *se_ctx, roc_se_cipher_type type,
 		    const uint8_t *key, uint16_t key_len, uint8_t *salt)
 {
+	struct roc_se_zuc_snow3g_ctx *zs_ctx = &se_ctx->se_ctx.zs_ctx;
 	struct roc_se_context *fctx = &se_ctx->se_ctx.fctx;
-	struct roc_se_zuc_snow3g_ctx *zs_ctx;
+	uint8_t *zuc_const;
 	uint32_t keyx[4];
+	uint8_t *ci_key;
 	int ret;
 
+	if (roc_model_is_cn9k()) {
+		ci_key = zs_ctx->zuc.onk_ctx.ci_key;
+		zuc_const = zs_ctx->zuc.onk_ctx.zuc_const;
+	} else {
+		ci_key = zs_ctx->zuc.otk_ctx.ci_key;
+		zuc_const = zs_ctx->zuc.otk_ctx.zuc_const;
+	}
+
 	/* For AES-GCM, salt is taken from ctx even if IV source
 	 * is from DPTR
 	 */
@@ -301,21 +397,29 @@ roc_se_ciph_key_set(struct roc_se_ctx *se_ctx, roc_se_cipher_type type,
 		memcpy(fctx->hmac.ipad, &key[key_len], key_len);
 		break;
 	case ROC_SE_SNOW3G_UEA2:
+		zs_ctx->zuc.otk_ctx.w0.s.key_len = ROC_SE_AES_128_BIT;
+		zs_ctx->zuc.otk_ctx.w0.s.alg_type = ROC_SE_PDCP_ALG_TYPE_SNOW3G;
 		se_ctx->pdcp_alg_type = ROC_SE_PDCP_ALG_TYPE_SNOW3G;
 		cpt_snow3g_key_gen(key, keyx);
-		memcpy(se_ctx->se_ctx.zs_ctx.ci_key, keyx, key_len);
+		memcpy(ci_key, keyx, key_len);
 		se_ctx->zsk_flags = 0;
 		goto success;
 	case ROC_SE_ZUC_EEA3:
-		zs_ctx = &se_ctx->se_ctx.zs_ctx;
+		ret = cpt_pdcp_key_type_set(zs_ctx, key_len);
+		if (ret)
+			return ret;
+		zs_ctx->zuc.otk_ctx.w0.s.alg_type = ROC_SE_PDCP_ALG_TYPE_ZUC;
 		se_ctx->pdcp_alg_type = ROC_SE_PDCP_ALG_TYPE_ZUC;
-		memcpy(zs_ctx->ci_key, key, key_len);
-		memcpy(zs_ctx->zuc_const, zuc_d, 32);
+		memcpy(ci_key, key, key_len);
+		memcpy(zuc_const, zuc_d, 32);
 		se_ctx->zsk_flags = 0;
 		goto success;
 	case ROC_SE_AES_CTR_EEA2:
+		zs_ctx->zuc.otk_ctx.w0.s.key_len = ROC_SE_AES_128_BIT;
+		zs_ctx->zuc.otk_ctx.w0.s.alg_type =
+			ROC_SE_PDCP_ALG_TYPE_AES_CTR;
 		se_ctx->pdcp_alg_type = ROC_SE_PDCP_ALG_TYPE_AES_CTR;
-		memcpy(se_ctx->se_ctx.zs_ctx.ci_key, key, key_len);
+		memcpy(ci_key, key, key_len);
 		se_ctx->zsk_flags = 0;
 		goto success;
 	case ROC_SE_KASUMI_F8_ECB:
@@ -341,6 +445,24 @@ roc_se_ciph_key_set(struct roc_se_ctx *se_ctx, roc_se_cipher_type type,
 
 success:
 	se_ctx->enc_cipher = type;
-
+	if (se_ctx->fc_type == ROC_SE_PDCP) {
+		if (roc_model_is_cn9k())
+			se_ctx->template_w4.s.opcode_minor =
+				((1 << 7) | (se_ctx->pdcp_alg_type << 5) |
+				 (se_ctx->zsk_flags & 0x7));
+		else
+			se_ctx->template_w4.s.opcode_minor = ((1 << 4));
+	}
 	return 0;
 }
+
+void
+roc_se_ctx_swap(struct roc_se_ctx *se_ctx)
+{
+	struct roc_se_zuc_snow3g_ctx *zs_ctx = &se_ctx->se_ctx.zs_ctx;
+
+	if (roc_model_is_cn9k())
+		return;
+
+	zs_ctx->zuc.otk_ctx.w0.u64 = htobe64(zs_ctx->zuc.otk_ctx.w0.u64);
+}
diff --git a/drivers/common/cnxk/roc_se.h b/drivers/common/cnxk/roc_se.h
index 5c7e2ca..051d496 100644
--- a/drivers/common/cnxk/roc_se.h
+++ b/drivers/common/cnxk/roc_se.h
@@ -158,6 +158,12 @@ typedef enum {
 	ROC_SE_AES_256_BIT = 0x3
 } roc_se_aes_type;
 
+typedef enum {
+	ROC_SE_PDCP_MAC_LEN_32_BIT = 0x1,
+	ROC_SE_PDCP_MAC_LEN_64_BIT = 0x2,
+	ROC_SE_PDCP_MAC_LEN_128_BIT = 0x3
+} roc_se_pdcp_mac_len_type;
+
 struct roc_se_sglist_comp {
 	union {
 		uint64_t len;
@@ -192,12 +198,35 @@ struct roc_se_context {
 	struct roc_se_hmac_context hmac;
 };
 
-struct roc_se_zuc_snow3g_ctx {
+struct roc_se_otk_zuc_ctx {
+	union {
+		uint64_t u64;
+		struct {
+			uint64_t rsvd_56 : 57;
+			uint64_t mac_len : 2;
+			uint64_t key_len : 2;
+			uint64_t lfsr_state : 1;
+			uint64_t alg_type : 2;
+		} s;
+	} w0;
+	uint8_t ci_key[32];
+	uint8_t encr_auth_iv[24];
+	uint8_t zuc_const[32];
+};
+
+struct roc_se_onk_zuc_ctx {
 	uint8_t encr_auth_iv[16];
 	uint8_t ci_key[16];
 	uint8_t zuc_const[32];
 };
 
+struct roc_se_zuc_snow3g_ctx {
+	union {
+		struct roc_se_onk_zuc_ctx onk_ctx;
+		struct roc_se_otk_zuc_ctx otk_ctx;
+	} zuc;
+};
+
 struct roc_se_kasumi_ctx {
 	uint8_t reg_A[8];
 	uint8_t ci_key[16];
@@ -229,7 +258,10 @@ struct roc_se_fc_params {
 	void *auth_iv_buf;
 	struct roc_se_buf_ptr meta_buf;
 	struct roc_se_buf_ptr ctx_buf;
-	uint64_t rsvd2;
+	uint32_t rsvd2;
+	uint16_t rsvd3;
+	uint8_t cipher_iv_len;
+	uint8_t auth_iv_len;
 
 	/* 1st cache line */
 	struct roc_se_buf_ptr aad_buf __plt_cache_aligned;
@@ -272,4 +304,5 @@ int __roc_api roc_se_ciph_key_set(struct roc_se_ctx *se_ctx,
 				  roc_se_cipher_type type, const uint8_t *key,
 				  uint16_t key_len, uint8_t *salt);
 
+void __roc_api roc_se_ctx_swap(struct roc_se_ctx *se_ctx);
 #endif /* __ROC_SE_H__ */
diff --git a/drivers/common/cnxk/version.map b/drivers/common/cnxk/version.map
index 34a844b..5865fc6 100644
--- a/drivers/common/cnxk/version.map
+++ b/drivers/common/cnxk/version.map
@@ -267,6 +267,7 @@ INTERNAL {
 	roc_tim_lf_disable;
 	roc_tim_lf_enable;
 	roc_tim_lf_free;
+	roc_se_ctx_swap;
 
 	local: *;
 };
-- 
2.7.4

[dpdk-dev] [PATCH 2/7] common/cnxk: update to v1.16 ucc codes

[Anoob Joseph]

Update to v1.16 microcode completion codes.

Signed-off-by: Anoob Joseph <anoobj@marvell.com>
---
 drivers/common/cnxk/roc_ie_ot.h | 65 +++++++++++++++++++++--------------------
 1 file changed, 34 insertions(+), 31 deletions(-)

diff --git a/drivers/common/cnxk/roc_ie_ot.h b/drivers/common/cnxk/roc_ie_ot.h
index 3987a08..1ff4688 100644
--- a/drivers/common/cnxk/roc_ie_ot.h
+++ b/drivers/common/cnxk/roc_ie_ot.h
@@ -14,37 +14,40 @@
 
 enum roc_ie_ot_ucc_ipsec {
 	ROC_IE_OT_UCC_SUCCESS = 0x00,
-	ROC_IE_OT_UCC_SUCCESS_PKT_IP_GOODCSUM = 0x02,
-	ROC_IE_OT_UCC_ERR_SA_INVAL = 0x03,
-	ROC_IE_OT_UCC_SUCCESS_PKT_IP_BADCSUM = 0x04,
-	ROC_IE_OT_UCC_ERR_SA_EXPIRED = 0x05,
-	ROC_IE_OT_UCC_SUCCESS_PKT_L4_GOODCSUM = 0x06,
-	ROC_IE_OT_UCC_ERR_SA_OVERFLOW = 0x07,
-	ROC_IE_OT_UCC_SUCCESS_PKT_L4_BADCSUM = 0x08,
-	ROC_IE_OT_UCC_ERR_SA_ESP_BAD_ALGO = 0x09,
-	ROC_IE_OT_UCC_SUCCESS_PKT_UDPESP_NZCSUM = 0x0a,
-	ROC_IE_OT_UCC_ERR_SA_ESP_BAD_KEYS = 0x0b,
-	ROC_IE_OT_UCC_SUCCESS_SA_SOFTEXP_FIRST = 0x0c,
-	ROC_IE_OT_UCC_ERR_SA_AH_BAD_ALGO = 0x0d,
-	ROC_IE_OT_UCC_SUCCESS_SA_SOFTEXP_AGAIN = 0x0e,
-	ROC_IE_OT_UCC_ERR_SA_AH_BAD_KEYS = 0x0f,
-	ROC_IE_OT_UCC_ERR_SA_BAD_IP = 0x11,
-	ROC_IE_OT_UCC_ERR_SA_BAD_CTX = 0x13,
-	ROC_IE_OT_UCC_ERR_AOP_IPSEC = 0x17,
-	ROC_IE_OT_UCC_ERR_PKT_IP = 0x23,
-	ROC_IE_OT_UCC_ERR_PKT_IP6_BAD_EXT = 0x25,
-	ROC_IE_OT_UCC_ERR_PKT_IP6_HBH = 0x27,
-	ROC_IE_OT_UCC_ERR_PKT_IP6_BIGEXT = 0x29,
-	ROC_IE_OT_UCC_ERR_PKT_IP_FRAG = 0x2b,
-	ROC_IE_OT_UCC_ERR_PKT_IP_ULP = 0x2d,
-	ROC_IE_OT_UCC_ERR_PKT_SA_MISMATCH = 0x2f,
-	ROC_IE_OT_UCC_ERR_PKT_SPI_MISMATCH = 0x31,
-	ROC_IE_OT_UCC_ERR_PKT_ESP_BADPAD = 0x33,
-	ROC_IE_OT_UCC_ERR_PKT_BADICV = 0x35,
-	ROC_IE_OT_UCC_ERR_PKT_REPLAY_SEQ = 0x37,
-	ROC_IE_OT_UCC_ERR_PKT_REPLAY_WINDOW = 0x39,
-	ROC_IE_OT_UCC_ERR_PKT_BADNH = 0x3b,
-	ROC_IE_OT_UCC_ERR_PKT_SA_PORT_MISMATCH = 0x3d,
+	ROC_IE_OT_UCC_ERR_SA_INVAL = 0xb0,
+	ROC_IE_OT_UCC_ERR_SA_EXPIRED = 0xb1,
+	ROC_IE_OT_UCC_ERR_SA_OVERFLOW = 0xb2,
+	ROC_IE_OT_UCC_ERR_SA_ESP_BAD_ALGO = 0xb3,
+	ROC_IE_OT_UCC_ERR_SA_AH_BAD_ALGO = 0xb4,
+	ROC_IE_OT_UCC_ERR_SA_BAD_CTX = 0xb5,
+	ROC_IE_OT_UCC_SA_CTX_FLAG_MISMATCH = 0xb6,
+	ROC_IE_OT_UCC_ERR_AOP_IPSEC = 0xb7,
+	ROC_IE_OT_UCC_ERR_PKT_IP = 0xb8,
+	ROC_IE_OT_UCC_ERR_PKT_IP6_BAD_EXT = 0xb9,
+	ROC_IE_OT_UCC_ERR_PKT_IP6_HBH = 0xba,
+	ROC_IE_OT_UCC_ERR_PKT_IP6_BIGEXT = 0xbb,
+	ROC_IE_OT_UCC_ERR_PKT_IP_ULP = 0xbc,
+	ROC_IE_OT_UCC_ERR_PKT_SA_MISMATCH = 0xbd,
+	ROC_IE_OT_UCC_ERR_PKT_SPI_MISMATCH = 0xbe,
+	ROC_IE_OT_UCC_ERR_PKT_ESP_BADPAD = 0xbf,
+	ROC_IE_OT_UCC_ERR_PKT_BADICV = 0xc0,
+	ROC_IE_OT_UCC_ERR_PKT_REPLAY_SEQ = 0xc1,
+	ROC_IE_OT_UCC_ERR_PKT_BADNH = 0xc2,
+	ROC_IE_OT_UCC_ERR_PKT_SA_PORT_MISMATCH = 0xc3,
+	ROC_IE_OT_UCC_ERR_PKT_BAD_DLEN = 0xc4,
+	ROC_IE_OT_UCC_ERR_SA_ESP_BAD_KEYS = 0xc5,
+	ROC_IE_OT_UCC_ERR_SA_AH_BAD_KEYS = 0xc6,
+	ROC_IE_OT_UCC_ERR_SA_BAD_IP = 0xc7,
+	ROC_IE_OT_UCC_ERR_PKT_REPLAY_WINDOW = 0xc8,
+	ROC_IE_OT_UCC_ERR_PKT_IP_FRAG = 0xc9,
+	ROC_IE_OT_UCC_SUCCESS_SA_SOFTEXP_FIRST = 0xf0,
+	ROC_IE_OT_UCC_SUCCESS_PKT_IP_BADCSUM = 0xf1,
+	ROC_IE_OT_UCC_SUCCESS_SA_SOFTEXP_AGAIN = 0xf2,
+	ROC_IE_OT_UCC_SUCCESS_PKT_L4_GOODCSUM = 0xf3,
+	ROC_IE_OT_UCC_SUCCESS_PKT_L4_BADCSUM = 0xf4,
+	ROC_IE_OT_UCC_SUCCESS_PKT_UDPESP_NZCSUM = 0xf5,
+	ROC_IE_OT_UCC_SUCCESS_PKT_UDP_ZEROCSUM = 0xf6,
+	ROC_IE_OT_UCC_SUCCESS_PKT_IP_GOODCSUM = 0xf7,
 };
 
 enum {
-- 
2.7.4

[dpdk-dev] [PATCH 3/7] crypto/cnxk: remove redundant snow3g dec

[Anoob Joseph]

The opcode for encryption & decryption is the same and single routine
would be able to handle both encryption and decryption operations.

Signed-off-by: Anoob Joseph <anoobj@marvell.com>
---
 drivers/crypto/cnxk/cnxk_se.h | 232 ++++--------------------------------------
 1 file changed, 17 insertions(+), 215 deletions(-)

diff --git a/drivers/crypto/cnxk/cnxk_se.h b/drivers/crypto/cnxk/cnxk_se.h
index 3ed6b90..9d1ce09 100644
--- a/drivers/crypto/cnxk/cnxk_se.h
+++ b/drivers/crypto/cnxk/cnxk_se.h
@@ -947,17 +947,16 @@ cpt_dec_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens,
 }
 
 static __rte_always_inline int
-cpt_zuc_snow3g_enc_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
-			struct roc_se_fc_params *params,
-			struct cpt_inst_s *inst)
+cpt_zuc_snow3g_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
+		    struct roc_se_fc_params *params, struct cpt_inst_s *inst)
 {
 	uint32_t size;
 	int32_t inputlen, outputlen;
 	struct roc_se_ctx *se_ctx;
 	uint32_t mac_len = 0;
 	uint8_t pdcp_alg_type, j;
-	uint32_t encr_offset = 0, auth_offset = 0;
-	uint32_t encr_data_len = 0, auth_data_len = 0;
+	uint32_t encr_offset, auth_offset;
+	uint32_t encr_data_len, auth_data_len;
 	int flags, iv_len = 16;
 	uint64_t offset_ctrl;
 	uint64_t *offset_vaddr;
@@ -995,6 +994,10 @@ cpt_zuc_snow3g_enc_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
 
 		offset_ctrl = rte_cpu_to_be_64((uint64_t)auth_offset);
 
+		encr_data_len = 0;
+		encr_offset = 0;
+
+		iv_s = params->auth_iv_buf;
 	} else {
 		/* EEA3 or UEA2 */
 		/*
@@ -1013,6 +1016,11 @@ cpt_zuc_snow3g_enc_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
 
 		/* iv offset is 0 */
 		offset_ctrl = rte_cpu_to_be_64((uint64_t)encr_offset << 16);
+
+		auth_data_len = 0;
+		auth_offset = 0;
+
+		iv_s = params->iv_buf;
 	}
 
 	if (unlikely((encr_offset >> 16) || (auth_offset >> 8))) {
@@ -1022,9 +1030,6 @@ cpt_zuc_snow3g_enc_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
 		return -1;
 	}
 
-	/* IV */
-	iv_s = (flags == 0x1) ? params->auth_iv_buf : params->iv_buf;
-
 	if (pdcp_alg_type == ROC_SE_PDCP_ALG_TYPE_SNOW3G) {
 		/*
 		 * DPDK seems to provide it in form of IV3 IV2 IV1 IV0
@@ -1209,209 +1214,6 @@ cpt_zuc_snow3g_enc_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
 }
 
 static __rte_always_inline int
-cpt_zuc_snow3g_dec_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
-			struct roc_se_fc_params *params,
-			struct cpt_inst_s *inst)
-{
-	uint32_t size;
-	int32_t inputlen = 0, outputlen;
-	struct roc_se_ctx *se_ctx;
-	uint8_t pdcp_alg_type, iv_len = 16;
-	uint32_t encr_offset;
-	uint32_t encr_data_len;
-	int flags;
-	uint64_t *offset_vaddr;
-	uint32_t *iv_s, iv[4], j;
-	union cpt_inst_w4 cpt_inst_w4;
-
-	/*
-	 * Microcode expects offsets in bytes
-	 * TODO: Rounding off
-	 */
-	encr_offset = ROC_SE_ENCR_OFFSET(d_offs) / 8;
-	encr_data_len = ROC_SE_ENCR_DLEN(d_lens);
-
-	se_ctx = params->ctx_buf.vaddr;
-	flags = se_ctx->zsk_flags;
-	pdcp_alg_type = se_ctx->pdcp_alg_type;
-
-	cpt_inst_w4.u64 = 0;
-	cpt_inst_w4.s.opcode_major = ROC_SE_MAJOR_OP_ZUC_SNOW3G;
-
-	/* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */
-
-	cpt_inst_w4.s.opcode_minor = ((1 << 7) | (pdcp_alg_type << 5) |
-				      (0 << 4) | (0 << 3) | (flags & 0x7));
-
-	/* consider iv len */
-	encr_offset += iv_len;
-
-	inputlen = encr_offset + (RTE_ALIGN(encr_data_len, 8) / 8);
-	outputlen = inputlen;
-
-	/* IV */
-	iv_s = params->iv_buf;
-	if (pdcp_alg_type == ROC_SE_PDCP_ALG_TYPE_SNOW3G) {
-		/*
-		 * DPDK seems to provide it in form of IV3 IV2 IV1 IV0
-		 * and BigEndian, MC needs it as IV0 IV1 IV2 IV3
-		 */
-
-		for (j = 0; j < 4; j++)
-			iv[j] = iv_s[3 - j];
-	} else {
-		/* ZUC doesn't need a swap */
-		for (j = 0; j < 4; j++)
-			iv[j] = iv_s[j];
-	}
-
-	/*
-	 * GP op header, lengths are expected in bits.
-	 */
-	cpt_inst_w4.s.param1 = encr_data_len;
-
-	/*
-	 * In cn9k, cn10k since we have a limitation of
-	 * IV & Offset control word not part of instruction
-	 * and need to be part of Data Buffer, we check if
-	 * head room is there and then only do the Direct mode processing
-	 */
-	if (likely((req_flags & ROC_SE_SINGLE_BUF_INPLACE) &&
-		   (req_flags & ROC_SE_SINGLE_BUF_HEADROOM))) {
-		void *dm_vaddr = params->bufs[0].vaddr;
-
-		/* Use Direct mode */
-
-		offset_vaddr = (uint64_t *)((uint8_t *)dm_vaddr -
-					    ROC_SE_OFF_CTRL_LEN - iv_len);
-
-		/* DPTR */
-		inst->dptr = (uint64_t)offset_vaddr;
-
-		/* RPTR should just exclude offset control word */
-		inst->rptr = (uint64_t)dm_vaddr - iv_len;
-
-		cpt_inst_w4.s.dlen = inputlen + ROC_SE_OFF_CTRL_LEN;
-
-		if (likely(iv_len)) {
-			uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr +
-						      ROC_SE_OFF_CTRL_LEN);
-			memcpy(iv_d, iv, 16);
-		}
-
-		/* iv offset is 0 */
-		*offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16);
-	} else {
-		void *m_vaddr = params->meta_buf.vaddr;
-		uint32_t i, g_size_bytes, s_size_bytes;
-		struct roc_se_sglist_comp *gather_comp;
-		struct roc_se_sglist_comp *scatter_comp;
-		uint8_t *in_buffer;
-		uint32_t *iv_d;
-
-		/* save space for offset and iv... */
-		offset_vaddr = m_vaddr;
-
-		m_vaddr = (uint8_t *)m_vaddr + ROC_SE_OFF_CTRL_LEN + iv_len;
-
-		cpt_inst_w4.s.opcode_major |= (uint64_t)ROC_SE_DMA_MODE;
-
-		/* DPTR has SG list */
-		in_buffer = m_vaddr;
-
-		((uint16_t *)in_buffer)[0] = 0;
-		((uint16_t *)in_buffer)[1] = 0;
-
-		/* TODO Add error check if space will be sufficient */
-		gather_comp =
-			(struct roc_se_sglist_comp *)((uint8_t *)m_vaddr + 8);
-
-		/*
-		 * Input Gather List
-		 */
-		i = 0;
-
-		/* Offset control word */
-
-		/* iv offset is 0 */
-		*offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16);
-
-		i = fill_sg_comp(gather_comp, i, (uint64_t)offset_vaddr,
-				 ROC_SE_OFF_CTRL_LEN + iv_len);
-
-		iv_d = (uint32_t *)((uint8_t *)offset_vaddr +
-				    ROC_SE_OFF_CTRL_LEN);
-		memcpy(iv_d, iv, 16);
-
-		/* Add input data */
-		size = inputlen - iv_len;
-		if (size) {
-			i = fill_sg_comp_from_iov(gather_comp, i,
-						  params->src_iov, 0, &size,
-						  NULL, 0);
-			if (unlikely(size)) {
-				plt_dp_err("Insufficient buffer space,"
-					   " size %d needed",
-					   size);
-				return -1;
-			}
-		}
-		((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
-		g_size_bytes =
-			((i + 3) / 4) * sizeof(struct roc_se_sglist_comp);
-
-		/*
-		 * Output Scatter List
-		 */
-
-		i = 0;
-		scatter_comp =
-			(struct roc_se_sglist_comp *)((uint8_t *)gather_comp +
-						      g_size_bytes);
-
-		/* IV */
-		i = fill_sg_comp(scatter_comp, i,
-				 (uint64_t)offset_vaddr + ROC_SE_OFF_CTRL_LEN,
-				 iv_len);
-
-		/* Add output data */
-		size = outputlen - iv_len;
-		if (size) {
-			i = fill_sg_comp_from_iov(scatter_comp, i,
-						  params->dst_iov, 0, &size,
-						  NULL, 0);
-
-			if (unlikely(size)) {
-				plt_dp_err("Insufficient buffer space,"
-					   " size %d needed",
-					   size);
-				return -1;
-			}
-		}
-		((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
-		s_size_bytes =
-			((i + 3) / 4) * sizeof(struct roc_se_sglist_comp);
-
-		size = g_size_bytes + s_size_bytes + ROC_SE_SG_LIST_HDR_SIZE;
-
-		/* This is DPTR len in case of SG mode */
-		cpt_inst_w4.s.dlen = size;
-
-		inst->dptr = (uint64_t)in_buffer;
-	}
-
-	if (unlikely((encr_offset >> 16))) {
-		plt_dp_err("Offset not supported");
-		plt_dp_err("enc_offset: %d", encr_offset);
-		return -1;
-	}
-
-	inst->w4.u64 = cpt_inst_w4.u64;
-
-	return 0;
-}
-
-static __rte_always_inline int
 cpt_kasumi_enc_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
 		    struct roc_se_fc_params *params, struct cpt_inst_s *inst)
 {
@@ -1749,8 +1551,8 @@ cpt_fc_dec_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens,
 	if (likely(fc_type == ROC_SE_FC_GEN)) {
 		ret = cpt_dec_hmac_prep(flags, d_offs, d_lens, fc_params, inst);
 	} else if (fc_type == ROC_SE_PDCP) {
-		ret = cpt_zuc_snow3g_dec_prep(flags, d_offs, d_lens, fc_params,
-					      inst);
+		ret = cpt_zuc_snow3g_prep(flags, d_offs, d_lens, fc_params,
+					  inst);
 	} else if (fc_type == ROC_SE_KASUMI) {
 		ret = cpt_kasumi_dec_prep(d_offs, d_lens, fc_params, inst);
 	}
@@ -1778,8 +1580,8 @@ cpt_fc_enc_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens,
 	if (likely(fc_type == ROC_SE_FC_GEN)) {
 		ret = cpt_enc_hmac_prep(flags, d_offs, d_lens, fc_params, inst);
 	} else if (fc_type == ROC_SE_PDCP) {
-		ret = cpt_zuc_snow3g_enc_prep(flags, d_offs, d_lens, fc_params,
-					      inst);
+		ret = cpt_zuc_snow3g_prep(flags, d_offs, d_lens, fc_params,
+					  inst);
 	} else if (fc_type == ROC_SE_KASUMI) {
 		ret = cpt_kasumi_enc_prep(flags, d_offs, d_lens, fc_params,
 					  inst);
-- 
2.7.4

[dpdk-dev] [PATCH 4/7] crypto/cnxk: remove redundant memcpy of IV for ZUC

[Anoob Joseph]

Swap is not required for ZUC. Update IV updation sequence to remove the
redundant swap in case of ZUC.

Signed-off-by: Anoob Joseph <anoobj@marvell.com>
---
 drivers/crypto/cnxk/cnxk_se.h | 55 +++++++++++++++++++++++--------------------
 1 file changed, 30 insertions(+), 25 deletions(-)

diff --git a/drivers/crypto/cnxk/cnxk_se.h b/drivers/crypto/cnxk/cnxk_se.h
index 9d1ce09..bbad289 100644
--- a/drivers/crypto/cnxk/cnxk_se.h
+++ b/drivers/crypto/cnxk/cnxk_se.h
@@ -36,6 +36,29 @@ struct cnxk_se_sess {
 	struct roc_se_ctx roc_se_ctx;
 } __rte_cache_aligned;
 
+static inline void
+pdcp_iv_copy(uint8_t *iv_d, uint8_t *iv_s, const uint8_t pdcp_alg_type)
+{
+	uint32_t *iv_s_temp, iv_temp[4];
+	int j;
+
+	if (pdcp_alg_type == ROC_SE_PDCP_ALG_TYPE_SNOW3G) {
+		/*
+		 * DPDK seems to provide it in form of IV3 IV2 IV1 IV0
+		 * and BigEndian, MC needs it as IV0 IV1 IV2 IV3
+		 */
+
+		iv_s_temp = (uint32_t *)iv_s;
+
+		for (j = 0; j < 4; j++)
+			iv_temp[j] = iv_s_temp[3 - j];
+		memcpy(iv_d, iv_temp, 16);
+	} else {
+		/* ZUC doesn't need a swap */
+		memcpy(iv_d, iv_s, 16);
+	}
+}
+
 static __rte_always_inline int
 cpt_mac_len_verify(struct rte_crypto_auth_xform *auth)
 {
@@ -954,13 +977,13 @@ cpt_zuc_snow3g_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
 	int32_t inputlen, outputlen;
 	struct roc_se_ctx *se_ctx;
 	uint32_t mac_len = 0;
-	uint8_t pdcp_alg_type, j;
+	uint8_t pdcp_alg_type;
 	uint32_t encr_offset, auth_offset;
 	uint32_t encr_data_len, auth_data_len;
 	int flags, iv_len = 16;
 	uint64_t offset_ctrl;
 	uint64_t *offset_vaddr;
-	uint32_t *iv_s, iv[4];
+	uint8_t *iv_s;
 	union cpt_inst_w4 cpt_inst_w4;
 
 	se_ctx = params->ctx_buf.vaddr;
@@ -1030,20 +1053,6 @@ cpt_zuc_snow3g_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
 		return -1;
 	}
 
-	if (pdcp_alg_type == ROC_SE_PDCP_ALG_TYPE_SNOW3G) {
-		/*
-		 * DPDK seems to provide it in form of IV3 IV2 IV1 IV0
-		 * and BigEndian, MC needs it as IV0 IV1 IV2 IV3
-		 */
-
-		for (j = 0; j < 4; j++)
-			iv[j] = iv_s[3 - j];
-	} else {
-		/* ZUC doesn't need a swap */
-		for (j = 0; j < 4; j++)
-			iv[j] = iv_s[j];
-	}
-
 	/*
 	 * GP op header, lengths are expected in bits.
 	 */
@@ -1072,11 +1081,8 @@ cpt_zuc_snow3g_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
 
 		cpt_inst_w4.s.dlen = inputlen + ROC_SE_OFF_CTRL_LEN;
 
-		if (likely(iv_len)) {
-			uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr +
-						      ROC_SE_OFF_CTRL_LEN);
-			memcpy(iv_d, iv, 16);
-		}
+		uint8_t *iv_d = ((uint8_t *)offset_vaddr + ROC_SE_OFF_CTRL_LEN);
+		pdcp_iv_copy(iv_d, iv_s, pdcp_alg_type);
 
 		*offset_vaddr = offset_ctrl;
 	} else {
@@ -1085,7 +1091,7 @@ cpt_zuc_snow3g_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
 		struct roc_se_sglist_comp *gather_comp;
 		struct roc_se_sglist_comp *scatter_comp;
 		uint8_t *in_buffer;
-		uint32_t *iv_d;
+		uint8_t *iv_d;
 
 		/* save space for iv */
 		offset_vaddr = m_vaddr;
@@ -1117,9 +1123,8 @@ cpt_zuc_snow3g_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
 		/* iv offset is 0 */
 		*offset_vaddr = offset_ctrl;
 
-		iv_d = (uint32_t *)((uint8_t *)offset_vaddr +
-				    ROC_SE_OFF_CTRL_LEN);
-		memcpy(iv_d, iv, 16);
+		iv_d = ((uint8_t *)offset_vaddr + ROC_SE_OFF_CTRL_LEN);
+		pdcp_iv_copy(iv_d, iv_s, pdcp_alg_type);
 
 		/* input data */
 		size = inputlen - iv_len;
-- 
2.7.4

[dpdk-dev] [PATCH 5/7] crypto/cnxk: remove redundant assignment

[Anoob Joseph]

The assignment to -1 is not required. Remove the same.

Signed-off-by: Anoob Joseph <anoobj@marvell.com>
---
 drivers/crypto/cnxk/cn10k_cryptodev_ops.c | 2 +-
 drivers/crypto/cnxk/cn9k_cryptodev_ops.c  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/drivers/crypto/cnxk/cn10k_cryptodev_ops.c b/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
index 28055ac..cccca77 100644
--- a/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
@@ -84,7 +84,7 @@ cpt_sym_inst_fill(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,
 		  struct cpt_inst_s *inst)
 {
 	uint64_t cpt_op;
-	int ret = -1;
+	int ret;
 
 	cpt_op = sess->cpt_op;
 
diff --git a/drivers/crypto/cnxk/cn9k_cryptodev_ops.c b/drivers/crypto/cnxk/cn9k_cryptodev_ops.c
index 08f08c8..4c0eb12 100644
--- a/drivers/crypto/cnxk/cn9k_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cn9k_cryptodev_ops.c
@@ -20,7 +20,7 @@ cn9k_cpt_sym_inst_fill(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,
 		       struct cpt_inst_s *inst)
 {
 	uint64_t cpt_op;
-	int ret = -1;
+	int ret;
 
 	cpt_op = sess->cpt_op;
 
-- 
2.7.4

[dpdk-dev] [PATCH 6/7] crypto/cnxk: support for ucode API change

[Anoob Joseph]

From: Vidya Sagar Velumuri <vvelumuri@marvell.com>

Add support for API changes in ucode 1.13

Signed-off-by: Vidya Sagar Velumuri <vvelumuri@marvell.com>
---
 drivers/crypto/cnxk/cnxk_se.h | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/drivers/crypto/cnxk/cnxk_se.h b/drivers/crypto/cnxk/cnxk_se.h
index bbad289..aedc4bc 100644
--- a/drivers/crypto/cnxk/cnxk_se.h
+++ b/drivers/crypto/cnxk/cnxk_se.h
@@ -993,10 +993,7 @@ cpt_zuc_snow3g_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens,
 
 	cpt_inst_w4.s.opcode_major = ROC_SE_MAJOR_OP_ZUC_SNOW3G;
 
-	/* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */
-
-	cpt_inst_w4.s.opcode_minor = ((1 << 7) | (pdcp_alg_type << 5) |
-				      (0 << 4) | (0 << 3) | (flags & 0x7));
+	cpt_inst_w4.s.opcode_minor = se_ctx->template_w4.s.opcode_minor;
 
 	if (flags == 0x1) {
 		/*
@@ -1766,6 +1763,8 @@ fill_sess_cipher(struct rte_crypto_sym_xform *xform, struct cnxk_se_sess *sess)
 					 NULL)))
 		return -1;
 
+	if ((enc_type >= ROC_SE_ZUC_EEA3) && (enc_type <= ROC_SE_AES_CTR_EEA2))
+		roc_se_ctx_swap(&sess->roc_se_ctx);
 	return 0;
 }
 
@@ -1869,6 +1868,10 @@ fill_sess_auth(struct rte_crypto_sym_xform *xform, struct cnxk_se_sess *sess)
 					 a_form->digest_length)))
 		return -1;
 
+	if ((auth_type >= ROC_SE_ZUC_EIA3) &&
+	    (auth_type <= ROC_SE_AES_CMAC_EIA2))
+		roc_se_ctx_swap(&sess->roc_se_ctx);
+
 	return 0;
 }
 
-- 
2.7.4

[dpdk-dev] [PATCH 7/7] crypto/cnxk: add dual submission in crypto_cn9k

[Anoob Joseph]

Submit two instructions with one LMTST operation. Also updated dequeue
path to have local var for constants.

Signed-off-by: Anoob Joseph <anoobj@marvell.com>
---
 drivers/crypto/cnxk/cn9k_cryptodev_ops.c | 155 +++++++++++++++++++++++++------
 1 file changed, 127 insertions(+), 28 deletions(-)

diff --git a/drivers/crypto/cnxk/cn9k_cryptodev_ops.c b/drivers/crypto/cnxk/cn9k_cryptodev_ops.c
index 4c0eb12..8ade197 100644
--- a/drivers/crypto/cnxk/cn9k_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cn9k_cryptodev_ops.c
@@ -5,6 +5,7 @@
 #include <rte_cryptodev.h>
 #include <rte_cryptodev_pmd.h>
 #include <rte_event_crypto_adapter.h>
+#include <rte_vect.h>
 
 #include "cn9k_cryptodev.h"
 #include "cn9k_cryptodev_ops.h"
@@ -64,9 +65,8 @@ cn9k_cpt_sym_temp_sess_create(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op)
 }
 
 static inline int
-cn9k_cpt_prepare_instruction(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,
-			     struct cpt_inflight_req *infl_req,
-			     struct cpt_inst_s *inst)
+cn9k_cpt_inst_prep(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,
+		   struct cpt_inflight_req *infl_req, struct cpt_inst_s *inst)
 {
 	int ret;
 
@@ -118,8 +118,8 @@ cn9k_cpt_prepare_instruction(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,
 }
 
 static inline void
-cn9k_cpt_submit_instruction(struct cpt_inst_s *inst, uint64_t lmtline,
-			    uint64_t io_addr)
+cn9k_cpt_inst_submit(struct cpt_inst_s *inst, uint64_t lmtline,
+		     uint64_t io_addr)
 {
 	uint64_t lmt_status;
 
@@ -138,46 +138,144 @@ cn9k_cpt_submit_instruction(struct cpt_inst_s *inst, uint64_t lmtline,
 	} while (lmt_status == 0);
 }
 
+static __plt_always_inline void
+cn9k_cpt_inst_submit_dual(struct cpt_inst_s *inst, uint64_t lmtline,
+			  uint64_t io_addr)
+{
+	uint64_t lmt_status;
+
+	do {
+		/* Copy 2 CPT inst_s to LMTLINE */
+#if defined(RTE_ARCH_ARM64)
+		uint64_t *s = (uint64_t *)inst;
+		uint64_t *d = (uint64_t *)lmtline;
+
+		vst1q_u64(&d[0], vld1q_u64(&s[0]));
+		vst1q_u64(&d[2], vld1q_u64(&s[2]));
+		vst1q_u64(&d[4], vld1q_u64(&s[4]));
+		vst1q_u64(&d[6], vld1q_u64(&s[6]));
+		vst1q_u64(&d[8], vld1q_u64(&s[8]));
+		vst1q_u64(&d[10], vld1q_u64(&s[10]));
+		vst1q_u64(&d[12], vld1q_u64(&s[12]));
+		vst1q_u64(&d[14], vld1q_u64(&s[14]));
+#else
+		roc_lmt_mov_seg((void *)lmtline, inst, 8);
+#endif
+
+		/*
+		 * Make sure compiler does not reorder memcpy and ldeor.
+		 * LMTST transactions are always flushed from the write
+		 * buffer immediately, a DMB is not required to push out
+		 * LMTSTs.
+		 */
+		rte_io_wmb();
+		lmt_status = roc_lmt_submit_ldeor(io_addr);
+	} while (lmt_status == 0);
+}
+
 static uint16_t
 cn9k_cpt_enqueue_burst(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
 {
-	struct cpt_inflight_req *infl_req;
+	struct cpt_inflight_req *infl_req_1, *infl_req_2;
+	struct cpt_inst_s inst[2] __rte_cache_aligned;
+	struct rte_crypto_op *op_1, *op_2;
 	uint16_t nb_allowed, count = 0;
 	struct cnxk_cpt_qp *qp = qptr;
 	struct pending_queue *pend_q;
-	struct rte_crypto_op *op;
-	struct cpt_inst_s inst;
+	uint64_t enq_tail;
 	int ret;
 
+	const uint32_t nb_desc = qp->lf.nb_desc;
+	const uint64_t lmt_base = qp->lf.lmt_base;
+	const uint64_t io_addr = qp->lf.io_addr;
+
 	pend_q = &qp->pend_q;
 
-	inst.w0.u64 = 0;
-	inst.w2.u64 = 0;
-	inst.w3.u64 = 0;
+	/* Clear w0, w2, w3 of both inst */
+
+	inst[0].w0.u64 = 0;
+	inst[0].w2.u64 = 0;
+	inst[0].w3.u64 = 0;
+	inst[1].w0.u64 = 0;
+	inst[1].w2.u64 = 0;
+	inst[1].w3.u64 = 0;
 
 	nb_allowed = qp->lf.nb_desc - pend_q->pending_count;
 	nb_ops = RTE_MIN(nb_ops, nb_allowed);
 
-	for (count = 0; count < nb_ops; count++) {
-		op = ops[count];
-		infl_req = &pend_q->req_queue[pend_q->enq_tail];
-		infl_req->op_flags = 0;
+	enq_tail = pend_q->enq_tail;
+
+	if (unlikely(nb_ops & 1)) {
+		op_1 = ops[0];
+		infl_req_1 = &pend_q->req_queue[enq_tail];
+		infl_req_1->op_flags = 0;
 
-		ret = cn9k_cpt_prepare_instruction(qp, op, infl_req, &inst);
+		ret = cn9k_cpt_inst_prep(qp, op_1, infl_req_1, &inst[0]);
 		if (unlikely(ret)) {
-			plt_dp_err("Could not process op: %p", op);
+			plt_dp_err("Could not process op: %p", op_1);
+			return 0;
+		}
+
+		infl_req_1->cop = op_1;
+		infl_req_1->res.cn9k.compcode = CPT_COMP_NOT_DONE;
+		inst[0].res_addr = (uint64_t)&infl_req_1->res;
+
+		cn9k_cpt_inst_submit(&inst[0], lmt_base, io_addr);
+		MOD_INC(enq_tail, nb_desc);
+		count++;
+	}
+
+	while (count < nb_ops) {
+		op_1 = ops[count];
+		op_2 = ops[count + 1];
+
+		infl_req_1 = &pend_q->req_queue[enq_tail];
+		MOD_INC(enq_tail, nb_desc);
+		infl_req_2 = &pend_q->req_queue[enq_tail];
+		MOD_INC(enq_tail, nb_desc);
+
+		infl_req_1->cop = op_1;
+		infl_req_2->cop = op_2;
+		infl_req_1->op_flags = 0;
+		infl_req_2->op_flags = 0;
+
+		infl_req_1->res.cn9k.compcode = CPT_COMP_NOT_DONE;
+		inst[0].res_addr = (uint64_t)&infl_req_1->res;
+
+		infl_req_2->res.cn9k.compcode = CPT_COMP_NOT_DONE;
+		inst[1].res_addr = (uint64_t)&infl_req_2->res;
+
+		ret = cn9k_cpt_inst_prep(qp, op_1, infl_req_1, &inst[0]);
+		if (unlikely(ret)) {
+			plt_dp_err("Could not process op: %p", op_1);
+			if (enq_tail == 0)
+				enq_tail = nb_desc - 2;
+			else if (enq_tail == 1)
+				enq_tail = nb_desc - 1;
+			else
+				enq_tail--;
+			break;
+		}
+
+		ret = cn9k_cpt_inst_prep(qp, op_2, infl_req_2, &inst[1]);
+		if (unlikely(ret)) {
+			plt_dp_err("Could not process op: %p", op_2);
+			if (enq_tail == 0)
+				enq_tail = nb_desc - 1;
+			else
+				enq_tail--;
+
+			cn9k_cpt_inst_submit(&inst[0], lmt_base, io_addr);
+			count++;
 			break;
 		}
 
-		infl_req->cop = op;
-		infl_req->res.cn9k.compcode = CPT_COMP_NOT_DONE;
-		inst.res_addr = (uint64_t)&infl_req->res;
+		cn9k_cpt_inst_submit_dual(&inst[0], lmt_base, io_addr);
 
-		cn9k_cpt_submit_instruction(&inst, qp->lmtline.lmt_base,
-					    qp->lmtline.io_addr);
-		MOD_INC(pend_q->enq_tail, qp->lf.nb_desc);
+		count += 2;
 	}
 
+	pend_q->enq_tail = enq_tail;
 	pend_q->pending_count += count;
 	pend_q->time_out = rte_get_timer_cycles() +
 			   DEFAULT_COMMAND_TIMEOUT * rte_get_timer_hz();
@@ -219,7 +317,7 @@ cn9k_cpt_crypto_adapter_enqueue(uintptr_t tag_op, struct rte_crypto_op *op)
 	}
 	infl_req->op_flags = 0;
 
-	ret = cn9k_cpt_prepare_instruction(qp, op, infl_req, &inst);
+	ret = cn9k_cpt_inst_prep(qp, op, infl_req, &inst);
 	if (unlikely(ret)) {
 		plt_dp_err("Could not process op: %p", op);
 		rte_mempool_put(qp->ca.req_mp, infl_req);
@@ -245,8 +343,7 @@ cn9k_cpt_crypto_adapter_enqueue(uintptr_t tag_op, struct rte_crypto_op *op)
 	if (!rsp_info->sched_type)
 		roc_sso_hws_head_wait(tag_op);
 
-	cn9k_cpt_submit_instruction(&inst, qp->lmtline.lmt_base,
-				    qp->lmtline.io_addr);
+	cn9k_cpt_inst_submit(&inst, qp->lmtline.lmt_base, qp->lmtline.io_addr);
 
 	return 1;
 }
@@ -347,14 +444,16 @@ cn9k_cpt_crypto_adapter_dequeue(uintptr_t get_work1)
 static uint16_t
 cn9k_cpt_dequeue_burst(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
 {
+	struct cpt_inflight_req *infl_req;
 	struct cnxk_cpt_qp *qp = qptr;
 	struct pending_queue *pend_q;
-	struct cpt_inflight_req *infl_req;
 	struct cpt_cn9k_res_s *res;
 	struct rte_crypto_op *cop;
 	uint32_t pq_deq_head;
 	int i;
 
+	const uint32_t nb_desc = qp->lf.nb_desc;
+
 	pend_q = &qp->pend_q;
 
 	nb_ops = RTE_MIN(nb_ops, pend_q->pending_count);
@@ -377,7 +476,7 @@ cn9k_cpt_dequeue_burst(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
 			break;
 		}
 
-		MOD_INC(pq_deq_head, qp->lf.nb_desc);
+		MOD_INC(pq_deq_head, nb_desc);
 
 		cop = infl_req->cop;
 
-- 
2.7.4

Re: [dpdk-dev] [PATCH 0/7] Improvements and fixes in crypto/cnxk PMDs

[Akhil Goyal]

> Improvements and fixes to crypto_cn9k & crypto_cn10k PMDs.
> - Updated IPsec completion codes to match latest firmware
> - Updated ZUC API to match latest firmware
> - Added dual submission in crypto_cn9k PMD
> 
> Depends on:
> http://patches.dpdk.org/project/dpdk/list/?series=18627
> 
> Anoob Joseph (5):
>   common/cnxk: update to v1.16 ucc codes
>   crypto/cnxk: remove redundant snow3g dec
>   crypto/cnxk: remove redundant memcpy of IV for ZUC
>   crypto/cnxk: remove redundant assignment
>   crypto/cnxk: add dual submission
> 
> Vidya Sagar Velumuri (2):
>   common/cnxk: update to v1.13 ZUC API
>   crypto/cnxk: support for ucode API change
> 
>  drivers/common/cnxk/roc_ie_ot.h           |  65 +++----
>  drivers/common/cnxk/roc_se.c              | 150 +++++++++++++--
>  drivers/common/cnxk/roc_se.h              |  37 +++-
>  drivers/common/cnxk/version.map           |   1 +
>  drivers/crypto/cnxk/cn10k_cryptodev_ops.c |   2 +-
>  drivers/crypto/cnxk/cn9k_cryptodev_ops.c  | 157 +++++++++++++---
>  drivers/crypto/cnxk/cnxk_se.h             | 298 ++++++------------------------
>  7 files changed, 389 insertions(+), 321 deletions(-)
> 
Series Acked-by: Akhil Goyal <gakhil@marvell.com>

Applied to dpdk-next-crypto

Thanks.