[dpdk-dev] [PATCH 08/20] crypto/cnxk: add dequeue burst op

[Anoob Joseph <anoobj@marvell.com>]

Add dequeue_burst op in cn9k & cn10k.

Signed-off-by: Ankur Dwivedi <adwivedi@marvell.com>
Signed-off-by: Anoob Joseph <anoobj@marvell.com>
Signed-off-by: Archana Muniganti <marchana@marvell.com>
Signed-off-by: Tejasree Kondoj <ktejasree@marvell.com>
---
 doc/guides/cryptodevs/features/cn10k.ini  |   3 +
 doc/guides/cryptodevs/features/cn9k.ini   |   3 +
 drivers/crypto/cnxk/cn10k_cryptodev.c     |   4 ++
 drivers/crypto/cnxk/cn10k_cryptodev_ops.c | 105 ++++++++++++++++++++++++++++++
 drivers/crypto/cnxk/cn9k_cryptodev.c      |   4 ++
 drivers/crypto/cnxk/cn9k_cryptodev_ops.c  | 103 +++++++++++++++++++++++++++++
 6 files changed, 222 insertions(+)

diff --git a/doc/guides/cryptodevs/features/cn10k.ini b/doc/guides/cryptodevs/features/cn10k.ini
index 0aa097d..7f433fa 100644
--- a/doc/guides/cryptodevs/features/cn10k.ini
+++ b/doc/guides/cryptodevs/features/cn10k.ini
@@ -4,6 +4,9 @@
 ; Refer to default.ini for the full list of available PMD features.
 ;
 [Features]
+Symmetric crypto       = Y
+HW Accelerated         = Y
+Symmetric sessionless  = Y
 
 ;
 ; Supported crypto algorithms of 'cn10k' crypto driver.
diff --git a/doc/guides/cryptodevs/features/cn9k.ini b/doc/guides/cryptodevs/features/cn9k.ini
index 64ee929..9c9d54d 100644
--- a/doc/guides/cryptodevs/features/cn9k.ini
+++ b/doc/guides/cryptodevs/features/cn9k.ini
@@ -4,6 +4,9 @@
 ; Refer to default.ini for the full list of available PMD features.
 ;
 [Features]
+Symmetric crypto       = Y
+HW Accelerated         = Y
+Symmetric sessionless  = Y
 
 ;
 ; Supported crypto algorithms of 'cn9k' crypto driver.
diff --git a/drivers/crypto/cnxk/cn10k_cryptodev.c b/drivers/crypto/cnxk/cn10k_cryptodev.c
index a34dbbf..2abd396 100644
--- a/drivers/crypto/cnxk/cn10k_cryptodev.c
+++ b/drivers/crypto/cnxk/cn10k_cryptodev.c
@@ -79,6 +79,10 @@ cn10k_cpt_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 
 	cnxk_cpt_caps_populate(vf);
 
+	dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
+			     RTE_CRYPTODEV_FF_HW_ACCELERATED |
+			     RTE_CRYPTODEV_FF_SYM_SESSIONLESS;
+
 	cn10k_cpt_set_enqdeq_fns(dev);
 
 	return 0;
diff --git a/drivers/crypto/cnxk/cn10k_cryptodev_ops.c b/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
index afdd43c..83b24c9 100644
--- a/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cn10k_cryptodev_ops.c
@@ -190,10 +190,115 @@ cn10k_cpt_enqueue_burst(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
 	return count + i;
 }
 
+static inline void
+cn10k_cpt_dequeue_post_process(struct cnxk_cpt_qp *qp,
+			       struct rte_crypto_op *cop,
+			       struct cpt_inflight_req *infl_req)
+{
+	struct cpt_cn10k_res_s *res = (struct cpt_cn10k_res_s *)&infl_req->res;
+	unsigned int sz;
+
+	if (likely(res->compcode == CPT_COMP_GOOD ||
+		   res->compcode == CPT_COMP_WARN)) {
+		if (unlikely(res->uc_compcode)) {
+			cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
+
+			CPT_LOG_DP_DEBUG("Request failed with microcode error");
+			CPT_LOG_DP_DEBUG("MC completion code 0x%x",
+					 res->uc_compcode);
+			goto temp_sess_free;
+		}
+
+		cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+	} else {
+		cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		CPT_LOG_DP_DEBUG("HW completion code 0x%x", res->compcode);
+
+		switch (res->compcode) {
+		case CPT_COMP_INSTERR:
+			CPT_LOG_DP_ERR("Request failed with instruction error");
+			break;
+		case CPT_COMP_FAULT:
+			CPT_LOG_DP_ERR("Request failed with DMA fault");
+			break;
+		case CPT_COMP_HWERR:
+			CPT_LOG_DP_ERR("Request failed with hardware error");
+			break;
+		default:
+			CPT_LOG_DP_ERR(
+				"Request failed with unknown completion code");
+		}
+	}
+
+temp_sess_free:
+	if (unlikely(cop->sess_type == RTE_CRYPTO_OP_SESSIONLESS)) {
+		if (cop->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
+			sym_session_clear(cn10k_cryptodev_driver_id,
+					  cop->sym->session);
+			sz = rte_cryptodev_sym_get_existing_header_session_size(
+				cop->sym->session);
+			memset(cop->sym->session, 0, sz);
+			rte_mempool_put(qp->sess_mp, cop->sym->session);
+			cop->sym->session = NULL;
+		}
+	}
+}
+
+static uint16_t
+cn10k_cpt_dequeue_burst(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
+{
+	struct cpt_inflight_req *infl_req;
+	struct cnxk_cpt_qp *qp = qptr;
+	struct pending_queue *pend_q;
+	struct cpt_cn10k_res_s *res;
+	struct rte_crypto_op *cop;
+	int i, nb_pending;
+
+	pend_q = &qp->pend_q;
+
+	nb_pending = pend_q->pending_count;
+
+	if (nb_ops > nb_pending)
+		nb_ops = nb_pending;
+
+	for (i = 0; i < nb_ops; i++) {
+		infl_req = &pend_q->req_queue[pend_q->deq_head];
+
+		res = (struct cpt_cn10k_res_s *)&infl_req->res;
+
+		if (unlikely(res->compcode == CPT_COMP_NOT_DONE)) {
+			if (unlikely(rte_get_timer_cycles() >
+				     pend_q->time_out)) {
+				plt_err("Request timed out");
+				pend_q->time_out = rte_get_timer_cycles() +
+						   DEFAULT_COMMAND_TIMEOUT *
+							   rte_get_timer_hz();
+			}
+			break;
+		}
+
+		MOD_INC(pend_q->deq_head, qp->lf.nb_desc);
+
+		cop = infl_req->cop;
+
+		ops[i] = cop;
+
+		cn10k_cpt_dequeue_post_process(qp, cop, infl_req);
+
+		if (unlikely(infl_req->op_flags & CPT_OP_FLAGS_METABUF))
+			rte_mempool_put(qp->meta_info.pool, infl_req->mdata);
+	}
+
+	pend_q->pending_count -= i;
+
+	return i;
+}
+
 void
 cn10k_cpt_set_enqdeq_fns(struct rte_cryptodev *dev)
 {
 	dev->enqueue_burst = cn10k_cpt_enqueue_burst;
+	dev->dequeue_burst = cn10k_cpt_dequeue_burst;
 
 	rte_mb();
 }
diff --git a/drivers/crypto/cnxk/cn9k_cryptodev.c b/drivers/crypto/cnxk/cn9k_cryptodev.c
index 7470397..db61175 100644
--- a/drivers/crypto/cnxk/cn9k_cryptodev.c
+++ b/drivers/crypto/cnxk/cn9k_cryptodev.c
@@ -77,6 +77,10 @@ cn9k_cpt_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 
 	cnxk_cpt_caps_populate(vf);
 
+	dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
+			     RTE_CRYPTODEV_FF_HW_ACCELERATED |
+			     RTE_CRYPTODEV_FF_SYM_SESSIONLESS;
+
 	cn9k_cpt_set_enqdeq_fns(dev);
 
 	return 0;
diff --git a/drivers/crypto/cnxk/cn9k_cryptodev_ops.c b/drivers/crypto/cnxk/cn9k_cryptodev_ops.c
index 59e3cb0..41c411b 100644
--- a/drivers/crypto/cnxk/cn9k_cryptodev_ops.c
+++ b/drivers/crypto/cnxk/cn9k_cryptodev_ops.c
@@ -155,10 +155,113 @@ cn9k_cpt_enqueue_burst(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
 	return count;
 }
 
+static inline void
+cn9k_cpt_dequeue_post_process(struct cnxk_cpt_qp *qp, struct rte_crypto_op *cop,
+			      struct cpt_inflight_req *infl_req)
+{
+	struct cpt_cn9k_res_s *res = (struct cpt_cn9k_res_s *)&infl_req->res;
+	unsigned int sz;
+
+	if (likely(res->compcode == CPT_COMP_GOOD)) {
+		if (unlikely(res->uc_compcode)) {
+			cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
+
+			CPT_LOG_DP_DEBUG("Request failed with microcode error");
+			CPT_LOG_DP_DEBUG("MC completion code 0x%x",
+					 res->uc_compcode);
+			goto temp_sess_free;
+		}
+
+		cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+	} else {
+		cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
+		CPT_LOG_DP_DEBUG("HW completion code 0x%x", res->compcode);
+
+		switch (res->compcode) {
+		case CPT_COMP_INSTERR:
+			CPT_LOG_DP_ERR("Request failed with instruction error");
+			break;
+		case CPT_COMP_FAULT:
+			CPT_LOG_DP_ERR("Request failed with DMA fault");
+			break;
+		case CPT_COMP_HWERR:
+			CPT_LOG_DP_ERR("Request failed with hardware error");
+			break;
+		default:
+			CPT_LOG_DP_ERR(
+				"Request failed with unknown completion code");
+		}
+	}
+
+temp_sess_free:
+	if (unlikely(cop->sess_type == RTE_CRYPTO_OP_SESSIONLESS)) {
+		if (cop->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
+			sym_session_clear(cn9k_cryptodev_driver_id,
+					  cop->sym->session);
+			sz = rte_cryptodev_sym_get_existing_header_session_size(
+				cop->sym->session);
+			memset(cop->sym->session, 0, sz);
+			rte_mempool_put(qp->sess_mp, cop->sym->session);
+			cop->sym->session = NULL;
+		}
+	}
+}
+
+static uint16_t
+cn9k_cpt_dequeue_burst(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
+{
+	struct cnxk_cpt_qp *qp = qptr;
+	struct pending_queue *pend_q;
+	struct cpt_inflight_req *infl_req;
+	struct cpt_cn9k_res_s *res;
+	struct rte_crypto_op *cop;
+	uint32_t pq_deq_head;
+	int i;
+
+	pend_q = &qp->pend_q;
+
+	nb_ops = RTE_MIN(nb_ops, pend_q->pending_count);
+
+	pq_deq_head = pend_q->deq_head;
+
+	for (i = 0; i < nb_ops; i++) {
+		infl_req = &pend_q->req_queue[pq_deq_head];
+
+		res = (struct cpt_cn9k_res_s *)&infl_req->res;
+
+		if (unlikely(res->compcode == CPT_COMP_NOT_DONE)) {
+			if (unlikely(rte_get_timer_cycles() >
+				     pend_q->time_out)) {
+				plt_err("Request timed out");
+				pend_q->time_out = rte_get_timer_cycles() +
+						   DEFAULT_COMMAND_TIMEOUT *
+							   rte_get_timer_hz();
+			}
+			break;
+		}
+
+		MOD_INC(pq_deq_head, qp->lf.nb_desc);
+
+		cop = infl_req->cop;
+
+		ops[i] = cop;
+
+		cn9k_cpt_dequeue_post_process(qp, cop, infl_req);
+
+		if (unlikely(infl_req->op_flags & CPT_OP_FLAGS_METABUF))
+			rte_mempool_put(qp->meta_info.pool, infl_req->mdata);
+	}
+
+	pend_q->pending_count -= i;
+	pend_q->deq_head = pq_deq_head;
+
+	return i;
+}
 void
 cn9k_cpt_set_enqdeq_fns(struct rte_cryptodev *dev)
 {
 	dev->enqueue_burst = cn9k_cpt_enqueue_burst;
+	dev->dequeue_burst = cn9k_cpt_dequeue_burst;
 
 	rte_mb();
 }
-- 
2.7.4