[v3 24/30] dma/dpaa: improve congestion handling

[Gagandeep Singh <g.singh@nxp.com>]

From: Jun Yang <jun.yang@nxp.com>

The congestion issue occurs frequently on low speed device(PCIe).
We should drain the command queue to make dma work when congestion occurs.

Signed-off-by: Jun Yang <jun.yang@nxp.com>
---
 drivers/dma/dpaa/dpaa_qdma.c | 157 +++++++++++++++++++----------------
 1 file changed, 85 insertions(+), 72 deletions(-)

diff --git a/drivers/dma/dpaa/dpaa_qdma.c b/drivers/dma/dpaa/dpaa_qdma.c
index de5ecc7d0b..eaa5f81f6d 100644
--- a/drivers/dma/dpaa/dpaa_qdma.c
+++ b/drivers/dma/dpaa/dpaa_qdma.c
@@ -535,73 +535,6 @@ fsl_qdma_reg_init(struct fsl_qdma_engine *fsl_qdma)
 	return 0;
 }
 
-static int
-fsl_qdma_enqueue_desc_to_ring(struct fsl_qdma_queue *fsl_queue,
-	int is_burst)
-{
-	uint16_t i, num = fsl_queue->pending_num, idx, start;
-	int ret;
-
-	num = is_burst ? fsl_queue->pending_num : 1;
-
-	fsl_queue->desc_in_hw[fsl_queue->ci] = num;
-	ret = rte_ring_enqueue(fsl_queue->complete_burst,
-			&fsl_queue->desc_in_hw[fsl_queue->ci]);
-	if (ret) {
-		DPAA_QDMA_ERR("%s: Queue is full, try dequeue first",
-			__func__);
-		DPAA_QDMA_ERR("%s: submitted:%"PRIu64", completed:%"PRIu64"",
-			__func__, fsl_queue->stats.submitted,
-			fsl_queue->stats.completed);
-		return ret;
-	}
-	start = fsl_queue->pending_start;
-	for (i = 0; i < num; i++) {
-		idx = (start + i) & (fsl_queue->pending_max - 1);
-		ret = rte_ring_enqueue(fsl_queue->complete_desc,
-				&fsl_queue->pending_desc[idx]);
-		if (ret) {
-			DPAA_QDMA_ERR("Descriptors eq failed!\r\n");
-			return ret;
-		}
-	}
-
-	return 0;
-}
-
-static int
-fsl_qdma_enqueue_desc_single(struct fsl_qdma_queue *fsl_queue,
-	dma_addr_t dst, dma_addr_t src, size_t len)
-{
-	uint8_t *block = fsl_queue->block_vir;
-	struct fsl_qdma_comp_sg_desc *csgf_src, *csgf_dest;
-	struct fsl_qdma_cmpd_ft *ft;
-	int ret;
-
-	ft = fsl_queue->ft[fsl_queue->ci];
-	csgf_src = &ft->desc_sbuf;
-	csgf_dest = &ft->desc_dbuf;
-	qdma_desc_sge_addr_set64(csgf_src, src);
-	csgf_src->length = len;
-	csgf_src->extion = 0;
-	qdma_desc_sge_addr_set64(csgf_dest, dst);
-	csgf_dest->length = len;
-	csgf_dest->extion = 0;
-	/* This entry is the last entry. */
-	csgf_dest->final = 1;
-
-	ret = fsl_qdma_enqueue_desc_to_ring(fsl_queue, 0);
-	if (ret)
-		return ret;
-	fsl_queue->ci = (fsl_queue->ci + 1) & (fsl_queue->n_cq - 1);
-
-	qdma_writel(fsl_queue->le_cqmr | FSL_QDMA_BCQMR_EI,
-		block + FSL_QDMA_BCQMR(fsl_queue->queue_id));
-	fsl_queue->stats.submitted++;
-
-	return 0;
-}
-
 static uint16_t
 dpaa_qdma_block_dequeue(struct fsl_qdma_engine *fsl_qdma,
 	uint8_t block_id)
@@ -633,7 +566,6 @@ dpaa_qdma_block_dequeue(struct fsl_qdma_engine *fsl_qdma,
 		ret = qdma_ccdf_get_queue(&cq[start], &qid);
 		if (ret == true) {
 			cmd_queue = &fsl_qdma->cmd_queues[block_id][qid];
-			cmd_queue->stats.completed++;
 
 			ret = rte_ring_dequeue(cmd_queue->complete_burst,
 				(void **)&dq_complete);
@@ -677,6 +609,87 @@ dpaa_qdma_block_dequeue(struct fsl_qdma_engine *fsl_qdma,
 	return count;
 }
 
+static int
+fsl_qdma_enqueue_desc_to_ring(struct fsl_qdma_queue *fsl_queue,
+	int is_burst)
+{
+	uint16_t i, num = fsl_queue->pending_num, idx, start, dq;
+	int ret, dq_cnt;
+
+	num = is_burst ? fsl_queue->pending_num : 1;
+
+	fsl_queue->desc_in_hw[fsl_queue->ci] = num;
+eq_again:
+	ret = rte_ring_enqueue(fsl_queue->complete_burst,
+			&fsl_queue->desc_in_hw[fsl_queue->ci]);
+	if (ret) {
+		DPAA_QDMA_DP_DEBUG("%s: Queue is full, try dequeue first",
+			__func__);
+		DPAA_QDMA_DP_DEBUG("%s: submitted:%"PRIu64", completed:%"PRIu64"",
+			__func__, fsl_queue->stats.submitted,
+			fsl_queue->stats.completed);
+		dq_cnt = 0;
+dq_again:
+		dq = dpaa_qdma_block_dequeue(fsl_queue->engine,
+			fsl_queue->block_id);
+		dq_cnt++;
+		if (dq > 0) {
+			goto eq_again;
+		} else {
+			if (dq_cnt < 100)
+				goto dq_again;
+			DPAA_QDMA_ERR("%s: Dq block%d failed!",
+				__func__, fsl_queue->block_id);
+		}
+		return ret;
+	}
+	start = fsl_queue->pending_start;
+	for (i = 0; i < num; i++) {
+		idx = (start + i) & (fsl_queue->pending_max - 1);
+		ret = rte_ring_enqueue(fsl_queue->complete_desc,
+				&fsl_queue->pending_desc[idx]);
+		if (ret) {
+			DPAA_QDMA_ERR("Descriptors eq failed!\r\n");
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static int
+fsl_qdma_enqueue_desc_single(struct fsl_qdma_queue *fsl_queue,
+	dma_addr_t dst, dma_addr_t src, size_t len)
+{
+	uint8_t *block = fsl_queue->block_vir;
+	struct fsl_qdma_comp_sg_desc *csgf_src, *csgf_dest;
+	struct fsl_qdma_cmpd_ft *ft;
+	int ret;
+
+	ft = fsl_queue->ft[fsl_queue->ci];
+	csgf_src = &ft->desc_sbuf;
+	csgf_dest = &ft->desc_dbuf;
+	qdma_desc_sge_addr_set64(csgf_src, src);
+	csgf_src->length = len;
+	csgf_src->extion = 0;
+	qdma_desc_sge_addr_set64(csgf_dest, dst);
+	csgf_dest->length = len;
+	csgf_dest->extion = 0;
+	/* This entry is the last entry. */
+	csgf_dest->final = 1;
+
+	ret = fsl_qdma_enqueue_desc_to_ring(fsl_queue, 0);
+	if (ret)
+		return ret;
+	fsl_queue->ci = (fsl_queue->ci + 1) & (fsl_queue->n_cq - 1);
+
+	qdma_writel(fsl_queue->le_cqmr | FSL_QDMA_BCQMR_EI,
+		block + FSL_QDMA_BCQMR(fsl_queue->queue_id));
+	fsl_queue->stats.submitted++;
+
+	return 0;
+}
+
 static int
 fsl_qdma_enqueue_overflow(struct fsl_qdma_queue *fsl_queue)
 {
@@ -702,7 +715,7 @@ fsl_qdma_enqueue_overflow(struct fsl_qdma_queue *fsl_queue)
 	if (likely(!overflow))
 		return 0;
 
-	DPAA_QDMA_ERR("TC%d/Q%d submitted(%"PRIu64")-completed(%"PRIu64") >= %d",
+	DPAA_QDMA_DP_DEBUG("TC%d/Q%d submitted(%"PRIu64")-completed(%"PRIu64") >= %d",
 		fsl_queue->block_id, fsl_queue->queue_id,
 		st->submitted, st->completed, QDMA_QUEUE_CR_WM);
 	drain_num = 0;
@@ -712,7 +725,7 @@ fsl_qdma_enqueue_overflow(struct fsl_qdma_queue *fsl_queue)
 		fsl_queue->block_id);
 	if (!blk_drain) {
 		drain_num++;
-		if (drain_num > 100) {
+		if (drain_num > 1000) {
 			DPAA_QDMA_ERR("TC%d failed drain, Q%d's %"PRIu64" bd in HW.",
 				fsl_queue->block_id, fsl_queue->queue_id,
 				st->submitted - st->completed);
@@ -721,8 +734,8 @@ fsl_qdma_enqueue_overflow(struct fsl_qdma_queue *fsl_queue)
 		goto drain_again;
 	}
 	check_num++;
-	if (check_num > 10) {
-		DPAA_QDMA_ERR("TC%d failed drain, Q%d's %"PRIu64" bd in HW.",
+	if (check_num > 1000) {
+		DPAA_QDMA_ERR("TC%d failed check, Q%d's %"PRIu64" bd in HW.",
 			fsl_queue->block_id, fsl_queue->queue_id,
 			st->submitted - st->completed);
 		return -ENOSPC;
-- 
2.25.1