[dpdk-dev] [PATCH 09/11] crypto/octeontx2: add dequeue burst

[Anoob Joseph <anoobj@marvell.com>]

From: Ankur Dwivedi <adwivedi@marvell.com>

This patch adds the dequeue burst callbacks for the OCTEON TX2
crypto driver. The completion code is checked during the dequeue
and the respective status is set in the crypto operation.

Signed-off-by: Ankur Dwivedi <adwivedi@marvell.com>
Signed-off-by: Anoob Joseph <anoobj@marvell.com>
Signed-off-by: Tejasree Kondoj <ktejasree@marvell.com>
---
 drivers/common/cpt/cpt_hw_types.h                  |  14 +++
 drivers/crypto/octeontx2/Makefile                  |   1 +
 drivers/crypto/octeontx2/meson.build               |   1 +
 .../crypto/octeontx2/otx2_cryptodev_hw_access.h    |  16 +++
 drivers/crypto/octeontx2/otx2_cryptodev_ops.c      | 128 +++++++++++++++++++++
 5 files changed, 160 insertions(+)

diff --git a/drivers/common/cpt/cpt_hw_types.h b/drivers/common/cpt/cpt_hw_types.h
index 4286512..e2b127d 100644
--- a/drivers/common/cpt/cpt_hw_types.h
+++ b/drivers/common/cpt/cpt_hw_types.h
@@ -281,6 +281,20 @@ typedef union cpt_res_s {
 		uint64_t reserved_64_127       : 64;
 #endif /* Word 1 - End */
 	} s8x;
+	struct cpt_res_s_9s {
+#if (RTE_BYTE_ORDER == RTE_BIG_ENDIAN) /* Word 0 - Big Endian */
+		uint64_t reserved_17_63:47;
+		uint64_t doneint:1;
+		uint64_t uc_compcode:8;
+		uint64_t compcode:8;
+#else /* Word 0 - Little Endian */
+		uint64_t compcode:8;
+		uint64_t uc_compcode:8;
+		uint64_t doneint:1;
+		uint64_t reserved_17_63:47;
+#endif /* Word 0 - End */
+		uint64_t reserved_64_127;
+	} s9x;
 } cpt_res_s_t;
 
 /**
diff --git a/drivers/crypto/octeontx2/Makefile b/drivers/crypto/octeontx2/Makefile
index 968efac..ce116b8 100644
--- a/drivers/crypto/octeontx2/Makefile
+++ b/drivers/crypto/octeontx2/Makefile
@@ -24,6 +24,7 @@ CFLAGS += -O3
 CFLAGS += -I$(RTE_SDK)/drivers/common/cpt
 CFLAGS += -I$(RTE_SDK)/drivers/common/octeontx2
 CFLAGS += -I$(RTE_SDK)/drivers/mempool/octeontx2
+CFLAGS += -DALLOW_EXPERIMENTAL_API
 
 ifneq ($(CONFIG_RTE_ARCH_64),y)
 CFLAGS += -Wno-int-to-pointer-cast
diff --git a/drivers/crypto/octeontx2/meson.build b/drivers/crypto/octeontx2/meson.build
index ee2e907..b6e5b73 100644
--- a/drivers/crypto/octeontx2/meson.build
+++ b/drivers/crypto/octeontx2/meson.build
@@ -10,6 +10,7 @@ deps += ['common_cpt']
 deps += ['common_octeontx2']
 name = 'octeontx2_crypto'
 
+allow_experimental_apis = true
 sources = files('otx2_cryptodev.c',
 		'otx2_cryptodev_capabilities.c',
 		'otx2_cryptodev_hw_access.c',
diff --git a/drivers/crypto/octeontx2/otx2_cryptodev_hw_access.h b/drivers/crypto/octeontx2/otx2_cryptodev_hw_access.h
index 82718df..d787d74 100644
--- a/drivers/crypto/octeontx2/otx2_cryptodev_hw_access.h
+++ b/drivers/crypto/octeontx2/otx2_cryptodev_hw_access.h
@@ -12,6 +12,7 @@
 
 #include "cpt_common.h"
 #include "cpt_hw_types.h"
+#include "cpt_mcode_defines.h"
 
 #include "otx2_dev.h"
 
@@ -173,6 +174,21 @@ union otx2_cpt_lf_q_grp_ptr {
 	} s;
 };
 
+/*
+ * Enumeration cpt_9x_comp_e
+ *
+ * CPT 9X Completion Enumeration
+ * Enumerates the values of CPT_RES_S[COMPCODE].
+ */
+enum cpt_9x_comp_e {
+	CPT_9X_COMP_E_NOTDONE = 0x00,
+	CPT_9X_COMP_E_GOOD = 0x01,
+	CPT_9X_COMP_E_FAULT = 0x02,
+	CPT_9X_COMP_E_HWERR = 0x04,
+	CPT_9X_COMP_E_INSTERR = 0x05,
+	CPT_9X_COMP_E_LAST_ENTRY = 0x06
+};
+
 struct otx2_cpt_qp {
 	uint32_t id;
 	/**< Queue pair id */
diff --git a/drivers/crypto/octeontx2/otx2_cryptodev_ops.c b/drivers/crypto/octeontx2/otx2_cryptodev_ops.c
index 74e6f1c..b21714c 100644
--- a/drivers/crypto/octeontx2/otx2_cryptodev_ops.c
+++ b/drivers/crypto/octeontx2/otx2_cryptodev_ops.c
@@ -486,6 +486,133 @@ otx2_cpt_enqueue_burst(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
 	return count;
 }
 
+static inline void
+otx2_cpt_dequeue_post_process(struct otx2_cpt_qp *qp, struct rte_crypto_op *cop,
+			      uintptr_t *rsp, uint8_t cc)
+{
+	if (cop->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
+		if (likely(cc == NO_ERR)) {
+			/* Verify authentication data if required */
+			if (unlikely(rsp[2]))
+				compl_auth_verify(cop, (uint8_t *)rsp[2],
+						 rsp[3]);
+			else
+				cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+		} else {
+			if (cc == ERR_GC_ICV_MISCOMPARE)
+				cop->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
+			else
+				cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		}
+
+		if (unlikely(cop->sess_type == RTE_CRYPTO_OP_SESSIONLESS)) {
+			sym_session_clear(otx2_cryptodev_driver_id,
+					  cop->sym->session);
+			rte_mempool_put(qp->sess_mp, cop->sym->session);
+			cop->sym->session = NULL;
+		}
+	}
+}
+
+static __rte_always_inline uint8_t
+otx2_cpt_compcode_get(struct cpt_request_info *req)
+{
+	volatile struct cpt_res_s_9s *res;
+	uint8_t ret;
+
+	res = (volatile struct cpt_res_s_9s *)req->completion_addr;
+
+	if (unlikely(res->compcode == CPT_9X_COMP_E_NOTDONE)) {
+		if (rte_get_timer_cycles() < req->time_out)
+			return ERR_REQ_PENDING;
+
+		CPT_LOG_DP_ERR("Request timed out");
+		return ERR_REQ_TIMEOUT;
+	}
+
+	if (likely(res->compcode == CPT_9X_COMP_E_GOOD)) {
+		ret = NO_ERR;
+		if (unlikely(res->uc_compcode)) {
+			ret = res->uc_compcode;
+			CPT_LOG_DP_DEBUG("Request failed with microcode error");
+			CPT_LOG_DP_DEBUG("MC completion code 0x%x",
+					 res->uc_compcode);
+		}
+	} else {
+		CPT_LOG_DP_DEBUG("HW completion code 0x%x", res->compcode);
+
+		ret = res->compcode;
+		switch (res->compcode) {
+		case CPT_9X_COMP_E_INSTERR:
+			CPT_LOG_DP_ERR("Request failed with instruction error");
+			break;
+		case CPT_9X_COMP_E_FAULT:
+			CPT_LOG_DP_ERR("Request failed with DMA fault");
+			break;
+		case CPT_9X_COMP_E_HWERR:
+			CPT_LOG_DP_ERR("Request failed with hardware error");
+			break;
+		default:
+			CPT_LOG_DP_ERR("Request failed with unknown completion code");
+		}
+	}
+
+	return ret;
+}
+
+static uint16_t
+otx2_cpt_dequeue_burst(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
+{
+	int i, nb_pending, nb_completed;
+	struct otx2_cpt_qp *qp = qptr;
+	struct pending_queue *pend_q;
+	struct cpt_request_info *req;
+	struct rte_crypto_op *cop;
+	uint8_t cc[nb_ops];
+	struct rid *rid;
+	uintptr_t *rsp;
+	void *metabuf;
+
+	pend_q = &qp->pend_q;
+
+	nb_pending = pend_q->pending_count;
+
+	if (nb_ops > nb_pending)
+		nb_ops = nb_pending;
+
+	for (i = 0; i < nb_ops; i++) {
+		rid = &pend_q->rid_queue[pend_q->deq_head];
+		req = (struct cpt_request_info *)(rid->rid);
+
+		cc[i] = otx2_cpt_compcode_get(req);
+
+		if (unlikely(cc[i] == ERR_REQ_PENDING))
+			break;
+
+		ops[i] = req->op;
+
+		MOD_INC(pend_q->deq_head, OTX2_CPT_DEFAULT_CMD_QLEN);
+		pend_q->pending_count -= 1;
+	}
+
+	nb_completed = i;
+
+	for (i = 0; i < nb_completed; i++) {
+		rsp = (void *)ops[i];
+
+		metabuf = (void *)rsp[0];
+		cop = (void *)rsp[1];
+
+		ops[i] = cop;
+
+		otx2_cpt_dequeue_post_process(qp, cop, rsp, cc[i]);
+
+		free_op_meta(metabuf, qp->meta_info.pool);
+	}
+
+	return nb_completed;
+}
+
 /* PMD ops */
 
 static int
@@ -536,6 +663,7 @@ otx2_cpt_dev_config(struct rte_cryptodev *dev,
 	}
 
 	dev->enqueue_burst = otx2_cpt_enqueue_burst;
+	dev->dequeue_burst = otx2_cpt_dequeue_burst;
 
 	rte_mb();
 	return 0;
-- 
2.7.4