[PATCH 23/30] dma/dpaa: block dequeue

[Gagandeep Singh <g.singh@nxp.com>]

From: Jun Yang <jun.yang@nxp.com>

Perform block dequeue to identify which queue of this block is completed.

Signed-off-by: Jun Yang <jun.yang@nxp.com>
---
 drivers/dma/dpaa/dpaa_qdma.c | 252 ++++++++++++++++-------------------
 1 file changed, 116 insertions(+), 136 deletions(-)

diff --git a/drivers/dma/dpaa/dpaa_qdma.c b/drivers/dma/dpaa/dpaa_qdma.c
index 75e92a1b0c..32d8a5b2be 100644
--- a/drivers/dma/dpaa/dpaa_qdma.c
+++ b/drivers/dma/dpaa/dpaa_qdma.c
@@ -442,86 +442,6 @@ fsl_qdma_data_validation(struct fsl_qdma_desc *desc[],
 	}
 }
 
-static int
-fsl_qdma_queue_drain(struct fsl_qdma_queue *fsl_queue)
-{
-	uint32_t reg;
-	int count = 0, ret;
-	uint8_t *block = fsl_queue->block_vir;
-	uint16_t *dq_complete, drain_num = 0;
-	struct fsl_qdma_desc *desc[FSL_QDMA_SG_MAX_ENTRY];
-
-	while (1) {
-		if (rte_ring_free_count(fsl_queue->complete_pool) <
-			(FSL_QDMA_SG_MAX_ENTRY * 2))
-			break;
-		reg = qdma_readl_be(block + FSL_QDMA_BSQSR);
-		if (reg & FSL_QDMA_BSQSR_QE_BE)
-			break;
-
-		qdma_writel_be(FSL_QDMA_BSQMR_DI, block + FSL_QDMA_BSQMR);
-		ret = rte_ring_dequeue(fsl_queue->complete_burst,
-			(void **)&dq_complete);
-		if (ret)
-			DPAA_QDMA_ERR("DQ desc number failed!\n");
-
-		ret = rte_ring_dequeue_bulk(fsl_queue->complete_desc,
-			(void **)desc, *dq_complete, NULL);
-		if (ret != (*dq_complete)) {
-			DPAA_QDMA_ERR("DQ %d descs failed!(%d)\n",
-				*dq_complete, ret);
-		}
-
-		fsl_qdma_data_validation(desc, *dq_complete, fsl_queue);
-
-		ret = rte_ring_enqueue_bulk(fsl_queue->complete_pool,
-			(void **)desc, (*dq_complete), NULL);
-		if (ret != (*dq_complete)) {
-			DPAA_QDMA_ERR("EQ %d descs to return queue failed!(%d)\n",
-				*dq_complete, ret);
-		}
-
-		drain_num += *dq_complete;
-		fsl_queue->complete_start =
-			(fsl_queue->complete_start + (*dq_complete)) &
-			(fsl_queue->pending_max - 1);
-		fsl_queue->stats.completed++;
-
-		count++;
-	}
-
-	return drain_num;
-}
-
-static int
-fsl_qdma_queue_transfer_complete(struct fsl_qdma_queue *fsl_queue,
-	const uint16_t nb_cpls, uint16_t *last_idx,
-	enum rte_dma_status_code *status)
-{
-	int ret;
-	uint16_t dq_num = 0, i;
-	struct fsl_qdma_desc *desc_complete[nb_cpls];
-
-	ret = fsl_qdma_queue_drain(fsl_queue);
-	if (ret < 0) {
-		DPAA_QDMA_ERR("Drain TX%d/Q%d failed!(%d)",
-			fsl_queue->block_id, fsl_queue->queue_id,
-			ret);
-	}
-
-	dq_num = rte_ring_dequeue_burst(fsl_queue->complete_pool,
-		(void **)desc_complete, nb_cpls, NULL);
-	for (i = 0; i < dq_num; i++)
-		last_idx[i] = desc_complete[i]->flag;
-
-	if (status) {
-		for (i = 0; i < dq_num; i++)
-			status[i] = RTE_DMA_STATUS_SUCCESSFUL;
-	}
-
-	return dq_num;
-}
-
 static int
 fsl_qdma_reg_init(struct fsl_qdma_engine *fsl_qdma)
 {
@@ -682,13 +602,90 @@ fsl_qdma_enqueue_desc_single(struct fsl_qdma_queue *fsl_queue,
 	return 0;
 }
 
+static uint16_t
+dpaa_qdma_block_dequeue(struct fsl_qdma_engine *fsl_qdma,
+	uint8_t block_id)
+{
+	struct fsl_qdma_status_queue *stat_queue;
+	struct fsl_qdma_queue *cmd_queue;
+	struct fsl_qdma_comp_cmd_desc *cq;
+	uint16_t start, count = 0;
+	uint8_t qid = 0;
+	uint32_t reg;
+	int ret;
+	uint8_t *block;
+	uint16_t *dq_complete;
+	struct fsl_qdma_desc *desc[FSL_QDMA_SG_MAX_ENTRY];
+
+	stat_queue = &fsl_qdma->stat_queues[block_id];
+	cq = stat_queue->cq;
+	start = stat_queue->complete;
+
+	block = fsl_qdma->block_base +
+		FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, block_id);
+
+	do {
+		reg = qdma_readl_be(block + FSL_QDMA_BSQSR);
+		if (reg & FSL_QDMA_BSQSR_QE_BE)
+			break;
+
+		qdma_writel_be(FSL_QDMA_BSQMR_DI, block + FSL_QDMA_BSQMR);
+		ret = qdma_ccdf_get_queue(&cq[start], &qid);
+		if (ret == true) {
+			cmd_queue = &fsl_qdma->cmd_queues[block_id][qid];
+			cmd_queue->stats.completed++;
+
+			ret = rte_ring_dequeue(cmd_queue->complete_burst,
+				(void **)&dq_complete);
+			if (ret)
+				DPAA_QDMA_ERR("DQ desc number failed!\n");
+
+			ret = rte_ring_dequeue_bulk(cmd_queue->complete_desc,
+				(void **)desc, *dq_complete, NULL);
+			if (ret != (*dq_complete)) {
+				DPAA_QDMA_ERR("DQ %d descs failed!(%d)\n",
+					*dq_complete, ret);
+			}
+
+			fsl_qdma_data_validation(desc, *dq_complete, cmd_queue);
+
+			ret = rte_ring_enqueue_bulk(cmd_queue->complete_pool,
+				(void **)desc, (*dq_complete), NULL);
+			if (ret != (*dq_complete)) {
+				DPAA_QDMA_ERR("Failed desc eq %d!=%d to %s\n",
+					ret, *dq_complete,
+					cmd_queue->complete_pool->name);
+			}
+
+			cmd_queue->complete_start =
+				(cmd_queue->complete_start + (*dq_complete)) &
+				(cmd_queue->pending_max - 1);
+			cmd_queue->stats.completed++;
+
+			start++;
+			if (unlikely(start == stat_queue->n_cq))
+				start = 0;
+			count++;
+		} else {
+			DPAA_QDMA_ERR("Block%d not empty but dq-queue failed!",
+				block_id);
+			break;
+		}
+	} while (1);
+	stat_queue->complete = start;
+
+	return count;
+}
+
 static int
 fsl_qdma_enqueue_overflow(struct fsl_qdma_queue *fsl_queue)
 {
-	int overflow = 0, drain;
-	uint32_t reg, check_num, drain_num;
+	int overflow = 0;
+	uint32_t reg;
+	uint16_t blk_drain, check_num, drain_num;
 	uint8_t *block = fsl_queue->block_vir;
 	const struct rte_dma_stats *st = &fsl_queue->stats;
+	struct fsl_qdma_engine *fsl_qdma = fsl_queue->engine;
 
 	check_num = 0;
 overflow_check:
@@ -711,11 +708,12 @@ fsl_qdma_enqueue_overflow(struct fsl_qdma_queue *fsl_queue)
 	drain_num = 0;
 
 drain_again:
-	drain = fsl_qdma_queue_drain(fsl_queue);
-	if (drain <= 0) {
+	blk_drain = dpaa_qdma_block_dequeue(fsl_qdma,
+		fsl_queue->block_id);
+	if (!blk_drain) {
 		drain_num++;
 		if (drain_num > 100) {
-			DPAA_QDMA_ERR("TC%d/Q%d failed drain, %"PRIu64" bd in HW.",
+			DPAA_QDMA_ERR("TC%d failed drain, Q%d's %"PRIu64" bd in HW.",
 				fsl_queue->block_id, fsl_queue->queue_id,
 				st->submitted - st->completed);
 			return -ENOSPC;
@@ -724,7 +722,7 @@ fsl_qdma_enqueue_overflow(struct fsl_qdma_queue *fsl_queue)
 	}
 	check_num++;
 	if (check_num > 10) {
-		DPAA_QDMA_ERR("TC%d/Q%d failed drain, %"PRIu64" bd in HW.",
+		DPAA_QDMA_ERR("TC%d failed drain, Q%d's %"PRIu64" bd in HW.",
 			fsl_queue->block_id, fsl_queue->queue_id,
 			st->submitted - st->completed);
 		return -ENOSPC;
@@ -1059,39 +1057,6 @@ dpaa_qdma_copy_sg(void *dev_private,
 	return ret;
 }
 
-static uint16_t
-dpaa_qdma_block_dequeue(struct fsl_qdma_engine *fsl_qdma,
-	uint8_t block_id)
-{
-	struct fsl_qdma_status_queue *stat_queue;
-	struct fsl_qdma_queue *cmd_queue;
-	struct fsl_qdma_comp_cmd_desc *cq;
-	uint16_t start, count = 0;
-	uint8_t qid;
-	int ret;
-
-	stat_queue = &fsl_qdma->stat_queues[block_id];
-	cq = stat_queue->cq;
-	start = stat_queue->complete;
-
-	do {
-		ret = qdma_ccdf_get_queue(&cq[start], &qid);
-		if (ret == true) {
-			cmd_queue = &fsl_qdma->cmd_queues[block_id][qid];
-			cmd_queue->stats.completed++;
-			start++;
-			if (unlikely(start == stat_queue->n_cq))
-				start = 0;
-			count++;
-		} else {
-			break;
-		}
-	} while (1);
-	stat_queue->complete = start;
-
-	return count;
-}
-
 static int
 dpaa_qdma_err_handle(struct fsl_qdma_err_reg *reg)
 {
@@ -1164,22 +1129,32 @@ dpaa_qdma_dequeue_status(void *dev_private, uint16_t vchan,
 	enum rte_dma_status_code *st)
 {
 	struct fsl_qdma_engine *fsl_qdma = dev_private;
-	int ret, err;
+	int err;
 	struct fsl_qdma_queue *fsl_queue = fsl_qdma->chan[vchan];
 	void *status = fsl_qdma->status_base;
+	struct fsl_qdma_desc *desc_complete[nb_cpls];
+	uint16_t i, dq_num;
 
 	if (unlikely(fsl_qdma->is_slient)) {
 		DPAA_QDMA_WARN("Can't dq in silent mode\n");
 		return 0;
 	}
 
-	if (fsl_qdma->block_queues[fsl_queue->block_id] > 1) {
-		ret = dpaa_qdma_block_dequeue(fsl_qdma,
-				fsl_queue->block_id);
-	} else {
-		ret = fsl_qdma_queue_transfer_complete(fsl_queue,
-				nb_cpls, last_idx, st);
+	dq_num = dpaa_qdma_block_dequeue(fsl_qdma,
+			fsl_queue->block_id);
+	DPAA_QDMA_DP_DEBUG("%s: block dq(%d)\n",
+		__func__, dq_num);
+
+	dq_num = rte_ring_dequeue_burst(fsl_queue->complete_pool,
+			(void **)desc_complete, nb_cpls, NULL);
+	for (i = 0; i < dq_num; i++)
+		last_idx[i] = desc_complete[i]->flag;
+
+	if (st) {
+		for (i = 0; i < dq_num; i++)
+			st[i] = RTE_DMA_STATUS_SUCCESSFUL;
 	}
+
 	if (s_hw_err_check) {
 		err = dpaa_qdma_err_handle(status +
 			FSL_QDMA_ERR_REG_STATUS_OFFSET);
@@ -1187,7 +1162,7 @@ dpaa_qdma_dequeue_status(void *dev_private, uint16_t vchan,
 			fsl_queue->stats.errors++;
 	}
 
-	return ret;
+	return dq_num;
 }
 
 static uint16_t
@@ -1196,9 +1171,11 @@ dpaa_qdma_dequeue(void *dev_private,
 	uint16_t *last_idx, bool *has_error)
 {
 	struct fsl_qdma_engine *fsl_qdma = dev_private;
-	int ret, err;
+	int err;
 	struct fsl_qdma_queue *fsl_queue = fsl_qdma->chan[vchan];
 	void *status = fsl_qdma->status_base;
+	struct fsl_qdma_desc *desc_complete[nb_cpls];
+	uint16_t i, dq_num;
 
 	if (unlikely(fsl_qdma->is_slient)) {
 		DPAA_QDMA_WARN("Can't dq in silent mode\n");
@@ -1207,13 +1184,16 @@ dpaa_qdma_dequeue(void *dev_private,
 	}
 
 	*has_error = false;
-	if (fsl_qdma->block_queues[fsl_queue->block_id] > 1) {
-		ret = dpaa_qdma_block_dequeue(fsl_qdma,
-				fsl_queue->block_id);
-	} else {
-		ret = fsl_qdma_queue_transfer_complete(fsl_queue,
-				nb_cpls, last_idx, NULL);
-	}
+	dq_num = dpaa_qdma_block_dequeue(fsl_qdma,
+		fsl_queue->block_id);
+	DPAA_QDMA_DP_DEBUG("%s: block dq(%d)\n",
+		__func__, dq_num);
+
+	dq_num = rte_ring_dequeue_burst(fsl_queue->complete_pool,
+			(void **)desc_complete, nb_cpls, NULL);
+	for (i = 0; i < dq_num; i++)
+		last_idx[i] = desc_complete[i]->flag;
+
 	if (s_hw_err_check) {
 		err = dpaa_qdma_err_handle(status +
 			FSL_QDMA_ERR_REG_STATUS_OFFSET);
@@ -1224,7 +1204,7 @@ dpaa_qdma_dequeue(void *dev_private,
 		}
 	}
 
-	return ret;
+	return dq_num;
 }
 
 static int
-- 
2.25.1