[v4 08/15] dma/dpaa: refactor driver

[Gagandeep Singh <g.singh@nxp.com>]

From: Jun Yang <jun.yang@nxp.com>

This patch refactor the DPAA DMA driver code with changes:
 - HW descriptors rename and update with details.
 - update qdma engine and queue structures
 - using rte_ring APIs for enqueue and dequeue.

Signed-off-by: Jun Yang <jun.yang@nxp.com>
Signed-off-by: Gagandeep Singh <g.singh@nxp.com>
---
 drivers/dma/dpaa/dpaa_qdma.c | 1330 ++++++++++++++++++----------------
 drivers/dma/dpaa/dpaa_qdma.h |  222 +++---
 2 files changed, 864 insertions(+), 688 deletions(-)

diff --git a/drivers/dma/dpaa/dpaa_qdma.c b/drivers/dma/dpaa/dpaa_qdma.c
index 3d4fd818f8..a10a867580 100644
--- a/drivers/dma/dpaa/dpaa_qdma.c
+++ b/drivers/dma/dpaa/dpaa_qdma.c
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright 2021 NXP
+ * Copyright 2021-2024 NXP
  */
 
 #include <bus_dpaa_driver.h>
@@ -8,69 +8,71 @@
 #include "dpaa_qdma.h"
 #include "dpaa_qdma_logs.h"
 
+static uint32_t s_sg_max_entry_sz = 2000;
+
 static inline void
-qdma_desc_addr_set64(struct fsl_qdma_format *ccdf, u64 addr)
+qdma_desc_addr_set64(struct fsl_qdma_comp_cmd_desc *ccdf, u64 addr)
 {
 	ccdf->addr_hi = upper_32_bits(addr);
 	ccdf->addr_lo = rte_cpu_to_le_32(lower_32_bits(addr));
 }
 
-static inline u64
-qdma_ccdf_get_queue(const struct fsl_qdma_format *ccdf)
+static inline void
+qdma_desc_sge_addr_set64(struct fsl_qdma_comp_sg_desc *sge, u64 addr)
 {
-	return ccdf->cfg8b_w1 & 0xff;
+	sge->addr_hi = upper_32_bits(addr);
+	sge->addr_lo = rte_cpu_to_le_32(lower_32_bits(addr));
 }
 
 static inline int
-qdma_ccdf_get_offset(const struct fsl_qdma_format *ccdf)
+qdma_ccdf_get_queue(struct fsl_qdma_comp_cmd_desc *ccdf,
+	uint8_t *queue_idx)
 {
-	return (rte_le_to_cpu_32(ccdf->cfg) & QDMA_CCDF_MASK)
-		>> QDMA_CCDF_OFFSET;
-}
+	uint64_t addr = ((uint64_t)ccdf->addr_hi) << 32 | ccdf->addr_lo;
+
+	if (addr && queue_idx)
+		*queue_idx = ccdf->queue;
+	if (addr) {
+		ccdf->addr_hi = 0;
+		ccdf->addr_lo = 0;
+		return true;
+	}
 
-static inline void
-qdma_ccdf_set_format(struct fsl_qdma_format *ccdf, int offset)
-{
-	ccdf->cfg = rte_cpu_to_le_32(QDMA_CCDF_FOTMAT | offset);
+	return false;
 }
 
 static inline int
-qdma_ccdf_get_status(const struct fsl_qdma_format *ccdf)
+ilog2(int x)
 {
-	return (rte_le_to_cpu_32(ccdf->status) & QDMA_CCDF_MASK)
-		>> QDMA_CCDF_STATUS;
-}
+	int log = 0;
 
-static inline void
-qdma_ccdf_set_ser(struct fsl_qdma_format *ccdf, int status)
-{
-	ccdf->status = rte_cpu_to_le_32(QDMA_CCDF_SER | status);
+	x >>= 1;
+
+	while (x) {
+		log++;
+		x >>= 1;
+	}
+	return log;
 }
 
-static inline void
-qdma_csgf_set_len(struct fsl_qdma_format *csgf, int len)
+static inline int
+ilog2_qsize(uint32_t q_size)
 {
-	csgf->cfg = rte_cpu_to_le_32(len & QDMA_SG_LEN_MASK);
+	return (ilog2(q_size) - ilog2(64));
 }
 
-static inline void
-qdma_csgf_set_f(struct fsl_qdma_format *csgf, int len)
+static inline int
+ilog2_qthld(uint32_t q_thld)
 {
-	csgf->cfg = rte_cpu_to_le_32(QDMA_SG_FIN | (len & QDMA_SG_LEN_MASK));
+	return (ilog2(q_thld) - ilog2(16));
 }
 
 static inline int
-ilog2(int x)
+fsl_qdma_queue_bd_in_hw(struct fsl_qdma_queue *fsl_queue)
 {
-	int log = 0;
-
-	x >>= 1;
+	struct rte_dma_stats *stats = &fsl_queue->stats;
 
-	while (x) {
-		log++;
-		x >>= 1;
-	}
-	return log;
+	return (stats->submitted - stats->completed);
 }
 
 static u32
@@ -97,12 +99,12 @@ qdma_writel_be(u32 val, void *addr)
 	QDMA_OUT_BE(addr, val);
 }
 
-static void
-*dma_pool_alloc(int size, int aligned, dma_addr_t *phy_addr)
+static void *
+dma_pool_alloc(char *nm, int size, int aligned, dma_addr_t *phy_addr)
 {
 	void *virt_addr;
 
-	virt_addr = rte_malloc("dma pool alloc", size, aligned);
+	virt_addr = rte_zmalloc(nm, size, aligned);
 	if (!virt_addr)
 		return NULL;
 
@@ -111,268 +113,221 @@ static void
 	return virt_addr;
 }
 
-static void
-dma_pool_free(void *addr)
-{
-	rte_free(addr);
-}
-
-static void
-fsl_qdma_free_chan_resources(struct fsl_qdma_chan *fsl_chan)
-{
-	struct fsl_qdma_queue *fsl_queue = fsl_chan->queue;
-	struct fsl_qdma_engine *fsl_qdma = fsl_chan->qdma;
-	struct fsl_qdma_comp *comp_temp, *_comp_temp;
-	int id;
-
-	if (--fsl_queue->count)
-		goto finally;
-
-	id = (fsl_qdma->block_base - fsl_queue->block_base) /
-	      fsl_qdma->block_offset;
-
-	while (rte_atomic32_read(&wait_task[id]) == 1)
-		rte_delay_us(QDMA_DELAY);
-
-	list_for_each_entry_safe(comp_temp, _comp_temp,
-				 &fsl_queue->comp_used,	list) {
-		list_del(&comp_temp->list);
-		dma_pool_free(comp_temp->virt_addr);
-		dma_pool_free(comp_temp->desc_virt_addr);
-		rte_free(comp_temp);
-	}
-
-	list_for_each_entry_safe(comp_temp, _comp_temp,
-				 &fsl_queue->comp_free, list) {
-		list_del(&comp_temp->list);
-		dma_pool_free(comp_temp->virt_addr);
-		dma_pool_free(comp_temp->desc_virt_addr);
-		rte_free(comp_temp);
-	}
-
-finally:
-	fsl_qdma->desc_allocated--;
-}
-
-static void
-fsl_qdma_comp_fill_memcpy(struct fsl_qdma_comp *fsl_comp,
-				      dma_addr_t dst, dma_addr_t src, u32 len)
-{
-	struct fsl_qdma_format *csgf_src, *csgf_dest;
-
-	/* Note: command table (fsl_comp->virt_addr) is getting filled
-	 * directly in cmd descriptors of queues while enqueuing the descriptor
-	 * please refer fsl_qdma_enqueue_desc
-	 * frame list table (virt_addr) + 1) and source,
-	 * destination descriptor table
-	 * (fsl_comp->desc_virt_addr and fsl_comp->desc_virt_addr+1) move to
-	 * the control path to fsl_qdma_pre_request_enqueue_comp_sd_desc
-	 */
-	csgf_src = (struct fsl_qdma_format *)fsl_comp->virt_addr + 2;
-	csgf_dest = (struct fsl_qdma_format *)fsl_comp->virt_addr + 3;
-
-	/* Status notification is enqueued to status queue. */
-	qdma_desc_addr_set64(csgf_src, src);
-	qdma_csgf_set_len(csgf_src, len);
-	qdma_desc_addr_set64(csgf_dest, dst);
-	qdma_csgf_set_len(csgf_dest, len);
-	/* This entry is the last entry. */
-	qdma_csgf_set_f(csgf_dest, len);
-}
-
 /*
  * Pre-request command descriptor and compound S/G for enqueue.
  */
 static int
-fsl_qdma_pre_request_enqueue_comp_sd_desc(
-					struct fsl_qdma_queue *queue,
-					int size, int aligned)
+fsl_qdma_pre_comp_sd_desc(struct fsl_qdma_queue *queue)
 {
-	struct fsl_qdma_comp *comp_temp, *_comp_temp;
 	struct fsl_qdma_sdf *sdf;
 	struct fsl_qdma_ddf *ddf;
-	struct fsl_qdma_format *csgf_desc;
-	int i;
-
-	for (i = 0; i < (int)(queue->n_cq + COMMAND_QUEUE_OVERFLOW); i++) {
-		comp_temp = rte_zmalloc("qdma: comp temp",
-					sizeof(*comp_temp), 0);
-		if (!comp_temp)
-			return -ENOMEM;
-
-		comp_temp->virt_addr =
-		dma_pool_alloc(size, aligned, &comp_temp->bus_addr);
-		if (!comp_temp->virt_addr) {
-			rte_free(comp_temp);
+	struct fsl_qdma_comp_cmd_desc *ccdf;
+	uint16_t i, j;
+	struct fsl_qdma_cmpd_ft *ft;
+
+	for (i = 0; i < queue->n_cq; i++) {
+		dma_addr_t phy_ft = 0;
+
+		queue->ft[i] = dma_pool_alloc(NULL,
+			sizeof(struct fsl_qdma_cmpd_ft),
+			RTE_CACHE_LINE_SIZE, &phy_ft);
+		if (!queue->ft[i])
+			goto fail;
+		if (((uint64_t)queue->ft[i]) &
+			(RTE_CACHE_LINE_SIZE - 1)) {
+			DPAA_QDMA_ERR("FD[%d] addr(%p) not cache aligned",
+				i, queue->ft[i]);
+			rte_free(queue->ft[i]);
+			queue->ft[i] = NULL;
 			goto fail;
 		}
-
-		comp_temp->desc_virt_addr =
-		dma_pool_alloc(size, aligned, &comp_temp->desc_bus_addr);
-		if (!comp_temp->desc_virt_addr) {
-			rte_free(comp_temp->virt_addr);
-			rte_free(comp_temp);
+		if (((uint64_t)(&queue->ft[i]->desc_ssge[0])) &
+			(RTE_CACHE_LINE_SIZE - 1)) {
+			DPAA_QDMA_ERR("FD[%d] SGE addr(%p) not cache aligned",
+				i, &queue->ft[i]->desc_ssge[0]);
+			rte_free(queue->ft[i]);
+			queue->ft[i] = NULL;
 			goto fail;
 		}
-
-		memset(comp_temp->virt_addr, 0, FSL_QDMA_COMMAND_BUFFER_SIZE);
-		memset(comp_temp->desc_virt_addr, 0,
-		       FSL_QDMA_DESCRIPTOR_BUFFER_SIZE);
-
-		csgf_desc = (struct fsl_qdma_format *)comp_temp->virt_addr + 1;
-		sdf = (struct fsl_qdma_sdf *)comp_temp->desc_virt_addr;
-		ddf = (struct fsl_qdma_ddf *)comp_temp->desc_virt_addr + 1;
+		queue->ft[i]->phy_ssge = phy_ft +
+			offsetof(struct fsl_qdma_cmpd_ft, desc_ssge);
+		queue->ft[i]->phy_dsge = phy_ft +
+			offsetof(struct fsl_qdma_cmpd_ft, desc_dsge);
+		queue->ft[i]->phy_df = phy_ft +
+			offsetof(struct fsl_qdma_cmpd_ft, df);
+
+		ft = queue->ft[i];
+		sdf = &ft->df.sdf;
+		ddf = &ft->df.ddf;
 		/* Compound Command Descriptor(Frame List Table) */
-		qdma_desc_addr_set64(csgf_desc, comp_temp->desc_bus_addr);
+		qdma_desc_sge_addr_set64(&ft->desc_buf, ft->phy_df);
 		/* It must be 32 as Compound S/G Descriptor */
-		qdma_csgf_set_len(csgf_desc, 32);
+		ft->desc_buf.length = sizeof(struct fsl_qdma_df);
+
 		/* Descriptor Buffer */
-		sdf->cmd = rte_cpu_to_le_32(FSL_QDMA_CMD_RWTTYPE <<
-			       FSL_QDMA_CMD_RWTTYPE_OFFSET);
-		ddf->cmd = rte_cpu_to_le_32(FSL_QDMA_CMD_RWTTYPE <<
-			       FSL_QDMA_CMD_RWTTYPE_OFFSET);
-		ddf->cmd |= rte_cpu_to_le_32(FSL_QDMA_CMD_LWC <<
-				FSL_QDMA_CMD_LWC_OFFSET);
-
-		list_add_tail(&comp_temp->list, &queue->comp_free);
+		sdf->srttype = FSL_QDMA_CMD_RWTTYPE;
+
+		ddf->dwttype = FSL_QDMA_CMD_RWTTYPE;
+		ddf->lwc = FSL_QDMA_CMD_LWC;
+
+		ccdf = &queue->cq[i];
+		qdma_desc_addr_set64(ccdf, phy_ft);
+		ccdf->format = FSL_QDMA_COMP_SG_FORMAT;
+
+		ccdf->queue = queue->queue_id;
 	}
+	queue->ci = 0;
 
 	return 0;
 
 fail:
-	list_for_each_entry_safe(comp_temp, _comp_temp,
-				 &queue->comp_free, list) {
-		list_del(&comp_temp->list);
-		rte_free(comp_temp->virt_addr);
-		rte_free(comp_temp->desc_virt_addr);
-		rte_free(comp_temp);
-	}
+	for (j = 0; j < i; j++)
+		rte_free(queue->ft[j]);
 
 	return -ENOMEM;
 }
 
-/*
- * Request a command descriptor for enqueue.
- */
-static struct fsl_qdma_comp *
-fsl_qdma_request_enqueue_desc(struct fsl_qdma_chan *fsl_chan)
+static int
+fsl_qdma_alloc_queue_resources(struct fsl_qdma_engine *fsl_qdma,
+	int queue_id, int block_id)
 {
-	struct fsl_qdma_queue *queue = fsl_chan->queue;
-	struct fsl_qdma_comp *comp_temp;
-
-	if (!list_empty(&queue->comp_free)) {
-		comp_temp = list_first_entry(&queue->comp_free,
-					     struct fsl_qdma_comp,
-					     list);
-		list_del(&comp_temp->list);
-		return comp_temp;
+	struct fsl_qdma_queue *cmd_queue;
+	uint32_t queue_size;
+	char nm[RTE_MEMZONE_NAMESIZE];
+
+	cmd_queue = &fsl_qdma->cmd_queues[block_id][queue_id];
+	cmd_queue->engine = fsl_qdma;
+
+	queue_size = sizeof(struct fsl_qdma_comp_cmd_desc) *
+		QDMA_QUEUE_SIZE;
+
+	sprintf(nm, "Command queue_%d_%d",
+		block_id, queue_id);
+	cmd_queue->cq = dma_pool_alloc(nm, queue_size,
+		queue_size, &cmd_queue->bus_addr);
+	if (!cmd_queue->cq) {
+		DPAA_QDMA_ERR("%s alloc failed!", nm);
+		return -ENOMEM;
 	}
 
-	return NULL;
-}
-
-static struct fsl_qdma_queue
-*fsl_qdma_alloc_queue_resources(struct fsl_qdma_engine *fsl_qdma)
-{
-	struct fsl_qdma_queue *queue_head, *queue_temp;
-	int len, i, j;
-	int queue_num;
-	int blocks;
-	unsigned int queue_size[FSL_QDMA_QUEUE_MAX];
-
-	queue_num = fsl_qdma->n_queues;
-	blocks = fsl_qdma->num_blocks;
-
-	len = sizeof(*queue_head) * queue_num * blocks;
-	queue_head = rte_zmalloc("qdma: queue head", len, 0);
-	if (!queue_head)
-		return NULL;
+	cmd_queue->block_vir = fsl_qdma->block_base +
+		FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, block_id);
+	cmd_queue->n_cq = QDMA_QUEUE_SIZE;
+	cmd_queue->queue_id = queue_id;
+	cmd_queue->block_id = block_id;
+	cmd_queue->pending_start = 0;
+	cmd_queue->pending_num = 0;
+	cmd_queue->complete_start = 0;
+
+	sprintf(nm, "Compound Table_%d_%d",
+		block_id, queue_id);
+	cmd_queue->ft = rte_zmalloc(nm,
+			sizeof(void *) * QDMA_QUEUE_SIZE, 0);
+	if (!cmd_queue->ft) {
+		DPAA_QDMA_ERR("%s zmalloc failed!", nm);
+		rte_free(cmd_queue->cq);
+		return -ENOMEM;
+	}
+	sprintf(nm, "Pending_desc_%d_%d",
+		block_id, queue_id);
+	cmd_queue->pending_desc = rte_zmalloc(nm,
+		sizeof(struct fsl_qdma_desc) * FSL_QDMA_MAX_DESC_NUM, 0);
+	if (!cmd_queue->pending_desc) {
+		DPAA_QDMA_ERR("%s zmalloc failed!", nm);
+		rte_free(cmd_queue->ft);
+		rte_free(cmd_queue->cq);
+		return -ENOMEM;
+	}
+	sprintf(nm, "complete-burst_ring_%d_%d",
+		block_id, queue_id);
+	cmd_queue->complete_burst = rte_ring_create(nm,
+		QDMA_QUEUE_SIZE * 2, 0,
+		RING_F_SP_ENQ | RING_F_SC_DEQ);
+	if (!cmd_queue->complete_burst) {
+		DPAA_QDMA_ERR("%s create failed!", nm);
+		rte_free(cmd_queue->pending_desc);
+		rte_free(cmd_queue->ft);
+		rte_free(cmd_queue->cq);
+		return -ENOMEM;
+	}
+	sprintf(nm, "complete-desc_ring_%d_%d",
+		block_id, queue_id);
+	cmd_queue->complete_desc = rte_ring_create(nm,
+		FSL_QDMA_MAX_DESC_NUM * 2, 0,
+		RING_F_SP_ENQ | RING_F_SC_DEQ);
+	if (!cmd_queue->complete_desc) {
+		DPAA_QDMA_ERR("%s create failed!", nm);
+		rte_ring_free(cmd_queue->complete_burst);
+		rte_free(cmd_queue->pending_desc);
+		rte_free(cmd_queue->ft);
+		rte_free(cmd_queue->cq);
+		return -ENOMEM;
+	}
+	sprintf(nm, "complete-pool-desc_ring_%d_%d",
+		block_id, queue_id);
+	cmd_queue->complete_pool = rte_ring_create(nm,
+		FSL_QDMA_MAX_DESC_NUM * 2, 0,
+		RING_F_SP_ENQ | RING_F_SC_DEQ);
+	if (!cmd_queue->complete_pool) {
+		DPAA_QDMA_ERR("%s create failed!", nm);
+		rte_ring_free(cmd_queue->complete_desc);
+		rte_ring_free(cmd_queue->complete_burst);
+		rte_free(cmd_queue->pending_desc);
+		rte_free(cmd_queue->ft);
+		rte_free(cmd_queue->cq);
+		return -ENOMEM;
+	}
 
-	for (i = 0; i < FSL_QDMA_QUEUE_MAX; i++)
-		queue_size[i] = QDMA_QUEUE_SIZE;
+	memset(&cmd_queue->stats, 0, sizeof(struct rte_dma_stats));
+	cmd_queue->pending_max = FSL_QDMA_MAX_DESC_NUM;
 
-	for (j = 0; j < blocks; j++) {
-		for (i = 0; i < queue_num; i++) {
-			if (queue_size[i] > FSL_QDMA_CIRCULAR_DESC_SIZE_MAX ||
-			    queue_size[i] < FSL_QDMA_CIRCULAR_DESC_SIZE_MIN) {
-				DPAA_QDMA_ERR("Get wrong queue-sizes.");
-				goto fail;
-			}
-			queue_temp = queue_head + i + (j * queue_num);
-
-			queue_temp->cq =
-			dma_pool_alloc(sizeof(struct fsl_qdma_format) *
-				       queue_size[i],
-				       sizeof(struct fsl_qdma_format) *
-				       queue_size[i], &queue_temp->bus_addr);
-
-			if (!queue_temp->cq)
-				goto fail;
-
-			memset(queue_temp->cq, 0x0, queue_size[i] *
-			       sizeof(struct fsl_qdma_format));
-
-			queue_temp->block_base = fsl_qdma->block_base +
-				FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, j);
-			queue_temp->n_cq = queue_size[i];
-			queue_temp->id = i;
-			queue_temp->count = 0;
-			queue_temp->pending = 0;
-			queue_temp->virt_head = queue_temp->cq;
-			queue_temp->stats = (struct rte_dma_stats){0};
-		}
-	}
-	return queue_head;
+	return 0;
+}
 
-fail:
-	for (j = 0; j < blocks; j++) {
-		for (i = 0; i < queue_num; i++) {
-			queue_temp = queue_head + i + (j * queue_num);
-			dma_pool_free(queue_temp->cq);
-		}
-	}
-	rte_free(queue_head);
+static void
+fsl_qdma_free_cmdq_res(struct fsl_qdma_queue *queue)
+{
+	rte_free(queue->ft);
+	rte_free(queue->cq);
+	rte_free(queue->pending_desc);
+	rte_ring_free(queue->complete_burst);
+	rte_ring_free(queue->complete_desc);
+	rte_ring_free(queue->complete_pool);
+}
 
-	return NULL;
+static void
+fsl_qdma_free_stq_res(struct fsl_qdma_status_queue *queue)
+{
+	rte_free(queue->cq);
 }
 
-static struct
-fsl_qdma_queue *fsl_qdma_prep_status_queue(void)
+static int
+fsl_qdma_prep_status_queue(struct fsl_qdma_engine *fsl_qdma,
+	uint32_t block_id)
 {
-	struct fsl_qdma_queue *status_head;
-	unsigned int status_size;
+	struct fsl_qdma_status_queue *status;
+	uint32_t status_size;
 
-	status_size = QDMA_STATUS_SIZE;
-	if (status_size > FSL_QDMA_CIRCULAR_DESC_SIZE_MAX ||
-	    status_size < FSL_QDMA_CIRCULAR_DESC_SIZE_MIN) {
-		DPAA_QDMA_ERR("Get wrong status_size.");
-		return NULL;
-	}
+	status = &fsl_qdma->stat_queues[block_id];
+	status->engine = fsl_qdma;
 
-	status_head = rte_zmalloc("qdma: status head", sizeof(*status_head), 0);
-	if (!status_head)
-		return NULL;
+	status_size = QDMA_STATUS_SIZE *
+		sizeof(struct fsl_qdma_comp_cmd_desc);
 
-	/*
-	 * Buffer for queue command
-	 */
-	status_head->cq = dma_pool_alloc(sizeof(struct fsl_qdma_format) *
-					 status_size,
-					 sizeof(struct fsl_qdma_format) *
-					 status_size,
-					 &status_head->bus_addr);
-
-	if (!status_head->cq) {
-		rte_free(status_head);
-		return NULL;
-	}
+	status->cq = dma_pool_alloc(NULL, status_size,
+		status_size, &status->bus_addr);
+
+	if (!status->cq)
+		return -ENOMEM;
 
-	memset(status_head->cq, 0x0, status_size *
-	       sizeof(struct fsl_qdma_format));
-	status_head->n_cq = status_size;
-	status_head->virt_head = status_head->cq;
+	memset(status->cq, 0x0, status_size);
+	status->n_cq = QDMA_STATUS_SIZE;
+	status->complete = 0;
+	status->block_id = block_id;
+	status->block_vir = fsl_qdma->block_base +
+		FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, block_id);
 
-	return status_head;
+	return 0;
 }
 
 static int
@@ -420,59 +375,41 @@ fsl_qdma_halt(struct fsl_qdma_engine *fsl_qdma)
 	return 0;
 }
 
-static int
-fsl_qdma_queue_transfer_complete(struct fsl_qdma_engine *fsl_qdma,
-				 void *block, int id, const uint16_t nb_cpls,
-				 uint16_t *last_idx,
-				 enum rte_dma_status_code *status)
+static void
+fsl_qdma_data_validation(struct fsl_qdma_desc *desc[],
+	uint8_t num, struct fsl_qdma_queue *fsl_queue)
 {
-	struct fsl_qdma_queue *fsl_queue = fsl_qdma->queue;
-	struct fsl_qdma_queue *fsl_status = fsl_qdma->status[id];
-	struct fsl_qdma_queue *temp_queue;
-	struct fsl_qdma_format *status_addr;
-	struct fsl_qdma_comp *fsl_comp = NULL;
-	u32 reg, i;
-	int count = 0;
-
-	while (count < nb_cpls) {
-		reg = qdma_readl_be(block + FSL_QDMA_BSQSR);
-		if (reg & FSL_QDMA_BSQSR_QE_BE)
-			return count;
-
-		status_addr = fsl_status->virt_head;
-
-		i = qdma_ccdf_get_queue(status_addr) +
-			id * fsl_qdma->n_queues;
-		temp_queue = fsl_queue + i;
-		fsl_comp = list_first_entry(&temp_queue->comp_used,
-					    struct fsl_qdma_comp,
-					    list);
-		list_del(&fsl_comp->list);
-
-		reg = qdma_readl_be(block + FSL_QDMA_BSQMR);
-		reg |= FSL_QDMA_BSQMR_DI_BE;
-
-		qdma_desc_addr_set64(status_addr, 0x0);
-		fsl_status->virt_head++;
-		if (fsl_status->virt_head == fsl_status->cq + fsl_status->n_cq)
-			fsl_status->virt_head = fsl_status->cq;
-		qdma_writel_be(reg, block + FSL_QDMA_BSQMR);
-		*last_idx = fsl_comp->index;
-		if (status != NULL)
-			status[count] = RTE_DMA_STATUS_SUCCESSFUL;
-
-		list_add_tail(&fsl_comp->list, &temp_queue->comp_free);
-		count++;
-
+	uint32_t i, j;
+	uint8_t *v_src, *v_dst;
+	char err_msg[512];
+	int offset;
+
+
+	offset = sprintf(err_msg, "Fatal TC%d/queue%d: ",
+		fsl_queue->block_id,
+		fsl_queue->queue_id);
+	for (i = 0; i < num; i++) {
+		v_src = rte_mem_iova2virt(desc[i]->src);
+		v_dst = rte_mem_iova2virt(desc[i]->dst);
+		for (j = 0; j < desc[i]->len; j++) {
+			if (v_src[j] != v_dst[j]) {
+				sprintf(&err_msg[offset],
+					"job[%"PRIu64"]:src(%p)[%d](%d)!=dst(%p)[%d](%d)",
+					desc[i]->flag, v_src, j, v_src[j],
+					v_dst, j, v_dst[j]);
+				DPAA_QDMA_ERR("%s, stop validating!",
+					err_msg);
+				return;
+			}
+		}
 	}
-	return count;
 }
 
 static int
 fsl_qdma_reg_init(struct fsl_qdma_engine *fsl_qdma)
 {
-	struct fsl_qdma_queue *fsl_queue = fsl_qdma->queue;
 	struct fsl_qdma_queue *temp;
+	struct fsl_qdma_status_queue *temp_stat;
 	void *ctrl = fsl_qdma->ctrl_base;
 	void *block;
 	u32 i, j;
@@ -489,8 +426,8 @@ fsl_qdma_reg_init(struct fsl_qdma_engine *fsl_qdma)
 	for (j = 0; j < fsl_qdma->num_blocks; j++) {
 		block = fsl_qdma->block_base +
 			FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, j);
-		for (i = 0; i < fsl_qdma->n_queues; i++) {
-			temp = fsl_queue + i + (j * fsl_qdma->n_queues);
+		for (i = 0; i < QDMA_QUEUES; i++) {
+			temp = &fsl_qdma->cmd_queues[j][i];
 			/*
 			 * Initialize Command Queue registers to
 			 * point to the first
@@ -531,18 +468,15 @@ fsl_qdma_reg_init(struct fsl_qdma_engine *fsl_qdma)
 		 * Enqueue Pointer Address Registers
 		 */
 
-		qdma_writel(
-			    upper_32_bits(fsl_qdma->status[j]->bus_addr),
-			    block + FSL_QDMA_SQEEPAR);
-		qdma_writel(
-			    lower_32_bits(fsl_qdma->status[j]->bus_addr),
-			    block + FSL_QDMA_SQEPAR);
-		qdma_writel(
-			    upper_32_bits(fsl_qdma->status[j]->bus_addr),
-			    block + FSL_QDMA_SQEDPAR);
-		qdma_writel(
-			    lower_32_bits(fsl_qdma->status[j]->bus_addr),
-			    block + FSL_QDMA_SQDPAR);
+		temp_stat = &fsl_qdma->stat_queues[j];
+		qdma_writel(upper_32_bits(temp_stat->bus_addr),
+			block + FSL_QDMA_SQEEPAR);
+		qdma_writel(lower_32_bits(temp_stat->bus_addr),
+			block + FSL_QDMA_SQEPAR);
+		qdma_writel(upper_32_bits(temp_stat->bus_addr),
+			block + FSL_QDMA_SQEDPAR);
+		qdma_writel(lower_32_bits(temp_stat->bus_addr),
+			block + FSL_QDMA_SQDPAR);
 		/* Desiable status queue interrupt. */
 
 		qdma_writel(0x0, block + FSL_QDMA_BCQIER(0));
@@ -551,7 +485,7 @@ fsl_qdma_reg_init(struct fsl_qdma_engine *fsl_qdma)
 
 		/* Initialize the status queue mode. */
 		reg = FSL_QDMA_BSQMR_EN;
-		val = ilog2(fsl_qdma->status[j]->n_cq) - 6;
+		val = ilog2_qsize(temp_stat->n_cq);
 		reg |= FSL_QDMA_BSQMR_CQ_SIZE(val);
 		qdma_writel(reg, block + FSL_QDMA_BSQMR);
 	}
@@ -563,158 +497,389 @@ fsl_qdma_reg_init(struct fsl_qdma_engine *fsl_qdma)
 	return 0;
 }
 
-static void *
-fsl_qdma_prep_memcpy(void *fsl_chan, dma_addr_t dst,
-			   dma_addr_t src, size_t len,
-			   void *call_back,
-			   void *param)
+static uint16_t
+dpaa_qdma_block_dequeue(struct fsl_qdma_engine *fsl_qdma,
+	uint8_t block_id)
 {
-	struct fsl_qdma_comp *fsl_comp;
+	struct fsl_qdma_status_queue *stat_queue;
+	struct fsl_qdma_queue *cmd_queue;
+	struct fsl_qdma_comp_cmd_desc *cq;
+	uint16_t start, count = 0;
+	uint8_t qid = 0;
+	uint32_t reg;
+	int ret;
+	uint8_t *block;
+	uint16_t *dq_complete;
+	struct fsl_qdma_desc *desc[FSL_QDMA_SG_MAX_ENTRY];
 
-	fsl_comp =
-	fsl_qdma_request_enqueue_desc((struct fsl_qdma_chan *)fsl_chan);
-	if (!fsl_comp)
-		return NULL;
+	stat_queue = &fsl_qdma->stat_queues[block_id];
+	cq = stat_queue->cq;
+	start = stat_queue->complete;
+
+	block = fsl_qdma->block_base +
+		FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, block_id);
 
-	fsl_comp->qchan = fsl_chan;
-	fsl_comp->call_back_func = call_back;
-	fsl_comp->params = param;
+	do {
+		reg = qdma_readl_be(block + FSL_QDMA_BSQSR);
+		if (reg & FSL_QDMA_BSQSR_QE_BE)
+			break;
 
-	fsl_qdma_comp_fill_memcpy(fsl_comp, dst, src, len);
-	return (void *)fsl_comp;
+		qdma_writel_be(FSL_QDMA_BSQMR_DI, block + FSL_QDMA_BSQMR);
+		ret = qdma_ccdf_get_queue(&cq[start], &qid);
+		if (ret == true) {
+			cmd_queue = &fsl_qdma->cmd_queues[block_id][qid];
+
+			ret = rte_ring_dequeue(cmd_queue->complete_burst,
+				(void **)&dq_complete);
+			if (ret)
+				rte_panic("DQ desc number failed!\n");
+
+			ret = rte_ring_dequeue_bulk(cmd_queue->complete_desc,
+				(void **)desc, *dq_complete, NULL);
+			if (ret != (*dq_complete)) {
+				rte_panic("DQ %d descs failed!(%d)\n",
+					*dq_complete, ret);
+			}
+
+			fsl_qdma_data_validation(desc, *dq_complete, cmd_queue);
+
+			ret = rte_ring_enqueue_bulk(cmd_queue->complete_pool,
+				(void **)desc, (*dq_complete), NULL);
+			if (ret != (*dq_complete)) {
+				rte_panic("Failed desc eq %d!=%d to %s\n",
+					ret, *dq_complete,
+					cmd_queue->complete_pool->name);
+			}
+
+			cmd_queue->complete_start =
+				(cmd_queue->complete_start + (*dq_complete)) &
+				(cmd_queue->pending_max - 1);
+			cmd_queue->stats.completed++;
+
+			start++;
+			if (unlikely(start == stat_queue->n_cq))
+				start = 0;
+			count++;
+		} else {
+			DPAA_QDMA_ERR("Block%d not empty but dq-queue failed!",
+				block_id);
+			break;
+		}
+	} while (1);
+	stat_queue->complete = start;
+
+	return count;
 }
 
 static int
-fsl_qdma_enqueue_desc(struct fsl_qdma_chan *fsl_chan,
-				  struct fsl_qdma_comp *fsl_comp,
-				  uint64_t flags)
+fsl_qdma_enqueue_desc_to_ring(struct fsl_qdma_queue *fsl_queue,
+	uint16_t num)
 {
-	struct fsl_qdma_queue *fsl_queue = fsl_chan->queue;
-	void *block = fsl_queue->block_base;
-	struct fsl_qdma_format *ccdf;
-	u32 reg;
+	uint16_t i, idx, start, dq;
+	int ret, dq_cnt;
+
 
-	/* retrieve and store the register value in big endian
-	 * to avoid bits swap
-	 */
-	reg = qdma_readl_be(block +
-			 FSL_QDMA_BCQSR(fsl_queue->id));
-	if (reg & (FSL_QDMA_BCQSR_QF_XOFF_BE))
-		return -1;
-
-	/* filling descriptor  command table */
-	ccdf = (struct fsl_qdma_format *)fsl_queue->virt_head;
-	qdma_desc_addr_set64(ccdf, fsl_comp->bus_addr + 16);
-	qdma_ccdf_set_format(ccdf, qdma_ccdf_get_offset(fsl_comp->virt_addr));
-	qdma_ccdf_set_ser(ccdf, qdma_ccdf_get_status(fsl_comp->virt_addr));
-	fsl_comp->index = fsl_queue->virt_head - fsl_queue->cq;
-	fsl_queue->virt_head++;
-
-	if (fsl_queue->virt_head == fsl_queue->cq + fsl_queue->n_cq)
-		fsl_queue->virt_head = fsl_queue->cq;
-
-	list_add_tail(&fsl_comp->list, &fsl_queue->comp_used);
-
-	if (flags == RTE_DMA_OP_FLAG_SUBMIT) {
-		reg = qdma_readl_be(block + FSL_QDMA_BCQMR(fsl_queue->id));
-		reg |= FSL_QDMA_BCQMR_EI_BE;
-		qdma_writel_be(reg, block + FSL_QDMA_BCQMR(fsl_queue->id));
-		fsl_queue->stats.submitted++;
-	} else {
-		fsl_queue->pending++;
+	fsl_queue->desc_in_hw[fsl_queue->ci] = num;
+eq_again:
+	ret = rte_ring_enqueue(fsl_queue->complete_burst,
+			&fsl_queue->desc_in_hw[fsl_queue->ci]);
+	if (ret) {
+		DPAA_QDMA_DP_DEBUG("%s: Queue is full, try dequeue first",
+			__func__);
+		DPAA_QDMA_DP_DEBUG("%s: submitted:%"PRIu64", completed:%"PRIu64"",
+			__func__, fsl_queue->stats.submitted,
+			fsl_queue->stats.completed);
+		dq_cnt = 0;
+dq_again:
+		dq = dpaa_qdma_block_dequeue(fsl_queue->engine,
+			fsl_queue->block_id);
+		dq_cnt++;
+		if (dq > 0) {
+			goto eq_again;
+		} else {
+			if (dq_cnt < 100)
+				goto dq_again;
+			DPAA_QDMA_ERR("%s: Dq block%d failed!",
+				__func__, fsl_queue->block_id);
+		}
+		return ret;
+	}
+	start = fsl_queue->pending_start;
+	for (i = 0; i < num; i++) {
+		idx = (start + i) & (fsl_queue->pending_max - 1);
+		ret = rte_ring_enqueue(fsl_queue->complete_desc,
+				&fsl_queue->pending_desc[idx]);
+		if (ret) {
+			rte_panic("Descriptors eq failed!\r\n");
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static int
+fsl_qdma_enqueue_overflow(struct fsl_qdma_queue *fsl_queue)
+{
+	int overflow = 0;
+	uint16_t blk_drain, check_num, drain_num;
+	const struct rte_dma_stats *st = &fsl_queue->stats;
+	struct fsl_qdma_engine *fsl_qdma = fsl_queue->engine;
+
+	check_num = 0;
+overflow_check:
+	overflow = (fsl_qdma_queue_bd_in_hw(fsl_queue) >=
+			QDMA_QUEUE_CR_WM) ? 1 : 0;
+
+	if (likely(!overflow))
+		return 0;
+
+	DPAA_QDMA_DP_DEBUG("TC%d/Q%d submitted(%"PRIu64")-completed(%"PRIu64") >= %d",
+		fsl_queue->block_id, fsl_queue->queue_id,
+		st->submitted, st->completed, QDMA_QUEUE_CR_WM);
+	drain_num = 0;
+
+drain_again:
+	blk_drain = dpaa_qdma_block_dequeue(fsl_qdma,
+		fsl_queue->block_id);
+	if (!blk_drain) {
+		drain_num++;
+		if (drain_num >= 10000) {
+			DPAA_QDMA_WARN("TC%d failed drain, Q%d's %"PRIu64" bd in HW.",
+				fsl_queue->block_id, fsl_queue->queue_id,
+				st->submitted - st->completed);
+			drain_num = 0;
+		}
+		goto drain_again;
+	}
+	check_num++;
+	if (check_num >= 1000) {
+		DPAA_QDMA_WARN("TC%d failed check, Q%d's %"PRIu64" bd in HW.",
+			fsl_queue->block_id, fsl_queue->queue_id,
+			st->submitted - st->completed);
+		check_num = 0;
 	}
-	return fsl_comp->index;
+	goto overflow_check;
+
+	return 0;
 }
 
 static int
-fsl_qdma_alloc_chan_resources(struct fsl_qdma_chan *fsl_chan)
+fsl_qdma_enqueue_desc_single(struct fsl_qdma_queue *fsl_queue,
+	dma_addr_t dst, dma_addr_t src, size_t len)
 {
-	struct fsl_qdma_queue *fsl_queue = fsl_chan->queue;
-	struct fsl_qdma_engine *fsl_qdma = fsl_chan->qdma;
+	uint8_t *block = fsl_queue->block_vir;
+	struct fsl_qdma_comp_sg_desc *csgf_src, *csgf_dest;
+	struct fsl_qdma_cmpd_ft *ft;
 	int ret;
 
-	if (fsl_queue->count++)
-		goto finally;
+	ret = fsl_qdma_enqueue_overflow(fsl_queue);
+	if (unlikely(ret))
+		return ret;
 
-	INIT_LIST_HEAD(&fsl_queue->comp_free);
-	INIT_LIST_HEAD(&fsl_queue->comp_used);
+	ft = fsl_queue->ft[fsl_queue->ci];
 
-	ret = fsl_qdma_pre_request_enqueue_comp_sd_desc(fsl_queue,
-				FSL_QDMA_COMMAND_BUFFER_SIZE, 64);
-	if (ret) {
-		DPAA_QDMA_ERR(
-			"failed to alloc dma buffer for comp descriptor");
-		goto exit;
-	}
+	csgf_src = &ft->desc_sbuf;
+	csgf_dest = &ft->desc_dbuf;
+	qdma_desc_sge_addr_set64(csgf_src, src);
+	csgf_src->length = len;
+	csgf_src->extion = 0;
+	qdma_desc_sge_addr_set64(csgf_dest, dst);
+	csgf_dest->length = len;
+	csgf_dest->extion = 0;
+	/* This entry is the last entry. */
+	csgf_dest->final = 1;
+
+	ret = fsl_qdma_enqueue_desc_to_ring(fsl_queue, 1);
+	if (ret)
+		return ret;
+	fsl_queue->ci = (fsl_queue->ci + 1) & (fsl_queue->n_cq - 1);
 
-finally:
-	return fsl_qdma->desc_allocated++;
+	qdma_writel(fsl_queue->le_cqmr | FSL_QDMA_BCQMR_EI,
+		block + FSL_QDMA_BCQMR(fsl_queue->queue_id));
+	fsl_queue->stats.submitted++;
 
-exit:
-	return -ENOMEM;
+	return 0;
 }
 
 static int
-dpaa_info_get(const struct rte_dma_dev *dev, struct rte_dma_info *dev_info,
-	      uint32_t info_sz)
+fsl_qdma_enqueue_desc_sg(struct fsl_qdma_queue *fsl_queue)
 {
-#define DPAADMA_MAX_DESC        64
-#define DPAADMA_MIN_DESC        64
+	uint8_t *block = fsl_queue->block_vir;
+	struct fsl_qdma_comp_sg_desc *csgf_src, *csgf_dest;
+	struct fsl_qdma_cmpd_ft *ft;
+	uint32_t total_len;
+	uint16_t start, idx, num, i, next_idx;
+	int ret;
 
-	RTE_SET_USED(dev);
-	RTE_SET_USED(info_sz);
+eq_sg:
+	total_len = 0;
+	start = fsl_queue->pending_start;
+	if (fsl_queue->pending_desc[start].len > s_sg_max_entry_sz ||
+		fsl_queue->pending_num == 1) {
+		ret = fsl_qdma_enqueue_desc_single(fsl_queue,
+			fsl_queue->pending_desc[start].dst,
+			fsl_queue->pending_desc[start].src,
+			fsl_queue->pending_desc[start].len);
+		if (!ret) {
+			fsl_queue->pending_start =
+				(start + 1) & (fsl_queue->pending_max - 1);
+			fsl_queue->pending_num--;
+		}
+		if (fsl_queue->pending_num > 0)
+			goto eq_sg;
 
-	dev_info->dev_capa = RTE_DMA_CAPA_MEM_TO_MEM |
-			     RTE_DMA_CAPA_MEM_TO_DEV |
-			     RTE_DMA_CAPA_DEV_TO_DEV |
-			     RTE_DMA_CAPA_DEV_TO_MEM |
-			     RTE_DMA_CAPA_SILENT |
-			     RTE_DMA_CAPA_OPS_COPY;
-	dev_info->max_vchans = 1;
-	dev_info->max_desc = DPAADMA_MAX_DESC;
-	dev_info->min_desc = DPAADMA_MIN_DESC;
+		return ret;
+	}
+
+	ret = fsl_qdma_enqueue_overflow(fsl_queue);
+	if (unlikely(ret))
+		return ret;
+
+	if (fsl_queue->pending_num > FSL_QDMA_SG_MAX_ENTRY)
+		num = FSL_QDMA_SG_MAX_ENTRY;
+	else
+		num = fsl_queue->pending_num;
+
+	ft = fsl_queue->ft[fsl_queue->ci];
+	csgf_src = &ft->desc_sbuf;
+	csgf_dest = &ft->desc_dbuf;
+
+	qdma_desc_sge_addr_set64(csgf_src, ft->phy_ssge);
+	csgf_src->extion = 1;
+	qdma_desc_sge_addr_set64(csgf_dest, ft->phy_dsge);
+	csgf_dest->extion = 1;
+	/* This entry is the last entry. */
+	csgf_dest->final = 1;
+	for (i = 0; i < num; i++) {
+		idx = (start + i) & (fsl_queue->pending_max - 1);
+		qdma_desc_sge_addr_set64(&ft->desc_ssge[i],
+			fsl_queue->pending_desc[idx].src);
+		ft->desc_ssge[i].length = fsl_queue->pending_desc[idx].len;
+		ft->desc_ssge[i].final = 0;
+		qdma_desc_sge_addr_set64(&ft->desc_dsge[i],
+			fsl_queue->pending_desc[idx].dst);
+		ft->desc_dsge[i].length = fsl_queue->pending_desc[idx].len;
+		ft->desc_dsge[i].final = 0;
+		total_len += fsl_queue->pending_desc[idx].len;
+		if ((i + 1) != num) {
+			next_idx = (idx + 1) & (fsl_queue->pending_max - 1);
+			if (fsl_queue->pending_desc[next_idx].len >
+				s_sg_max_entry_sz) {
+				num = i + 1;
+				break;
+			}
+		}
+	}
+
+	ft->desc_ssge[num - 1].final = 1;
+	ft->desc_dsge[num - 1].final = 1;
+	csgf_src->length = total_len;
+	csgf_dest->length = total_len;
+	ret = fsl_qdma_enqueue_desc_to_ring(fsl_queue, num);
+	if (ret)
+		return ret;
+
+	fsl_queue->ci = (fsl_queue->ci + 1) & (fsl_queue->n_cq - 1);
+
+	qdma_writel(fsl_queue->le_cqmr | FSL_QDMA_BCQMR_EI,
+		block + FSL_QDMA_BCQMR(fsl_queue->queue_id));
+	fsl_queue->stats.submitted++;
+
+	fsl_queue->pending_start =
+		(start + num) & (fsl_queue->pending_max - 1);
+	fsl_queue->pending_num -= num;
+	if (fsl_queue->pending_num > 0)
+		goto eq_sg;
 
 	return 0;
 }
 
 static int
-dpaa_get_channel(struct fsl_qdma_engine *fsl_qdma,  uint16_t vchan)
+fsl_qdma_enqueue_desc(struct fsl_qdma_queue *fsl_queue)
 {
-	u32 i, start, end;
+	uint16_t start = fsl_queue->pending_start;
 	int ret;
 
-	start = fsl_qdma->free_block_id * QDMA_QUEUES;
-	fsl_qdma->free_block_id++;
+	if (fsl_queue->pending_num == 1) {
+		ret = fsl_qdma_enqueue_desc_single(fsl_queue,
+			fsl_queue->pending_desc[start].dst,
+			fsl_queue->pending_desc[start].src,
+			fsl_queue->pending_desc[start].len);
+		if (!ret) {
+			fsl_queue->pending_start =
+				(start + 1) & (fsl_queue->pending_max - 1);
+			fsl_queue->pending_num = 0;
+		}
+		return ret;
+	}
+
+	return fsl_qdma_enqueue_desc_sg(fsl_queue);
+}
 
-	end = start + 1;
-	for (i = start; i < end; i++) {
-		struct fsl_qdma_chan *fsl_chan = &fsl_qdma->chans[i];
+static int
+dpaa_qdma_info_get(const struct rte_dma_dev *dev,
+	struct rte_dma_info *dev_info, __rte_unused uint32_t info_sz)
+{
+	struct fsl_qdma_engine *fsl_qdma = dev->data->dev_private;
 
-		if (fsl_chan->free) {
-			fsl_chan->free = false;
-			ret = fsl_qdma_alloc_chan_resources(fsl_chan);
-			if (ret)
-				return ret;
+	dev_info->dev_capa = RTE_DMA_CAPA_MEM_TO_MEM |
+		RTE_DMA_CAPA_SILENT | RTE_DMA_CAPA_OPS_COPY |
+		RTE_DMA_CAPA_OPS_COPY_SG;
+	dev_info->dev_capa |= DPAA_QDMA_FLAGS_INDEX;
+	dev_info->max_vchans = fsl_qdma->n_queues;
+	dev_info->max_desc = FSL_QDMA_MAX_DESC_NUM;
+	dev_info->min_desc = QDMA_QUEUE_SIZE;
+	dev_info->max_sges = FSL_QDMA_SG_MAX_ENTRY;
 
-			fsl_qdma->vchan_map[vchan] = i;
-			return 0;
+	return 0;
+}
+
+static int
+dpaa_get_channel(struct fsl_qdma_engine *fsl_qdma,
+	uint16_t vchan)
+{
+	int ret, i, j, found = 0;
+	struct fsl_qdma_queue *fsl_queue = fsl_qdma->chan[vchan];
+
+	if (fsl_queue) {
+		found = 1;
+		goto queue_found;
+	}
+
+	for (i = 0; i < QDMA_BLOCKS; i++) {
+		for (j = 0; j < QDMA_QUEUES; j++) {
+			fsl_queue = &fsl_qdma->cmd_queues[i][j];
+
+			if (fsl_queue->channel_id == vchan) {
+				found = 1;
+				fsl_qdma->chan[vchan] = fsl_queue;
+				goto queue_found;
+			}
 		}
 	}
 
-	return -1;
-}
+queue_found:
+	if (!found)
+		return -ENXIO;
 
-static void
-dma_release(void *fsl_chan)
-{
-	((struct fsl_qdma_chan *)fsl_chan)->free = true;
-	fsl_qdma_free_chan_resources((struct fsl_qdma_chan *)fsl_chan);
+	if (fsl_queue->used)
+		return 0;
+
+	ret = fsl_qdma_pre_comp_sd_desc(fsl_queue);
+	if (ret)
+		return ret;
+
+	fsl_queue->used = 1;
+	fsl_qdma->block_queues[fsl_queue->block_id]++;
+
+	return 0;
 }
 
 static int
 dpaa_qdma_configure(__rte_unused struct rte_dma_dev *dmadev,
-		    __rte_unused const struct rte_dma_conf *dev_conf,
-		    __rte_unused uint32_t conf_sz)
+	__rte_unused const struct rte_dma_conf *dev_conf,
+	__rte_unused uint32_t conf_sz)
 {
 	return 0;
 }
@@ -745,148 +910,112 @@ dpaa_qdma_queue_setup(struct rte_dma_dev *dmadev,
 static int
 dpaa_qdma_submit(void *dev_private, uint16_t vchan)
 {
-	struct fsl_qdma_engine *fsl_qdma = (struct fsl_qdma_engine *)dev_private;
-	struct fsl_qdma_chan *fsl_chan =
-		&fsl_qdma->chans[fsl_qdma->vchan_map[vchan]];
-	struct fsl_qdma_queue *fsl_queue = fsl_chan->queue;
-	void *block = fsl_queue->block_base;
-	u32 reg;
+	struct fsl_qdma_engine *fsl_qdma = dev_private;
+	struct fsl_qdma_queue *fsl_queue = fsl_qdma->chan[vchan];
 
-	while (fsl_queue->pending) {
-		reg = qdma_readl_be(block + FSL_QDMA_BCQMR(fsl_queue->id));
-		reg |= FSL_QDMA_BCQMR_EI_BE;
-		qdma_writel_be(reg, block + FSL_QDMA_BCQMR(fsl_queue->id));
-		fsl_queue->pending--;
-		fsl_queue->stats.submitted++;
-	}
+	if (!fsl_queue->pending_num)
+		return 0;
 
-	return 0;
+	return fsl_qdma_enqueue_desc(fsl_queue);
 }
 
 static int
 dpaa_qdma_enqueue(void *dev_private, uint16_t vchan,
-		  rte_iova_t src, rte_iova_t dst,
-		  uint32_t length, uint64_t flags)
+	rte_iova_t src, rte_iova_t dst,
+	uint32_t length, uint64_t flags)
 {
-	struct fsl_qdma_engine *fsl_qdma = (struct fsl_qdma_engine *)dev_private;
-	struct fsl_qdma_chan *fsl_chan =
-		&fsl_qdma->chans[fsl_qdma->vchan_map[vchan]];
+	struct fsl_qdma_engine *fsl_qdma = dev_private;
+	struct fsl_qdma_queue *fsl_queue = fsl_qdma->chan[vchan];
+	uint16_t start = fsl_queue->pending_start;
+	uint8_t pending = fsl_queue->pending_num;
+	uint16_t idx;
 	int ret;
 
-	void *fsl_comp = NULL;
-
-	fsl_comp = fsl_qdma_prep_memcpy(fsl_chan,
-			(dma_addr_t)dst, (dma_addr_t)src,
-			length, NULL, NULL);
-	if (!fsl_comp) {
-		DPAA_QDMA_DP_DEBUG("fsl_comp is NULL");
-		return -1;
+	if (pending >= fsl_queue->pending_max) {
+		DPAA_QDMA_ERR("Too many pending jobs(%d) on queue%d",
+			pending, vchan);
+		return -ENOSPC;
 	}
-	ret = fsl_qdma_enqueue_desc(fsl_chan, fsl_comp, flags);
+	idx = (start + pending) & (fsl_queue->pending_max - 1);
+
+	fsl_queue->pending_desc[idx].src = src;
+	fsl_queue->pending_desc[idx].dst = dst;
+	fsl_queue->pending_desc[idx].flag =
+		DPAA_QDMA_IDX_FROM_FLAG(flags);
+	fsl_queue->pending_desc[idx].len = length;
+	fsl_queue->pending_num++;
+
+	if (!(flags & RTE_DMA_OP_FLAG_SUBMIT))
+		return idx;
+
+	ret = fsl_qdma_enqueue_desc(fsl_queue);
+	if (!ret)
+		return fsl_queue->pending_start;
 
 	return ret;
 }
 
+
 static uint16_t
 dpaa_qdma_dequeue_status(void *dev_private, uint16_t vchan,
-			 const uint16_t nb_cpls, uint16_t *last_idx,
-			 enum rte_dma_status_code *st)
+	const uint16_t nb_cpls, uint16_t *last_idx,
+	enum rte_dma_status_code *st)
 {
-	struct fsl_qdma_engine *fsl_qdma = (struct fsl_qdma_engine *)dev_private;
-	int id = (int)((fsl_qdma->vchan_map[vchan]) / QDMA_QUEUES);
-	void *block;
-	int intr;
-	void *status = fsl_qdma->status_base;
-	struct fsl_qdma_chan *fsl_chan =
-		&fsl_qdma->chans[fsl_qdma->vchan_map[vchan]];
-	struct fsl_qdma_queue *fsl_queue = fsl_chan->queue;
-
-	intr = qdma_readl_be(status + FSL_QDMA_DEDR);
-	if (intr) {
-		DPAA_QDMA_ERR("DMA transaction error! %x", intr);
-		intr = qdma_readl(status + FSL_QDMA_DECFDW0R);
-		DPAA_QDMA_INFO("reg FSL_QDMA_DECFDW0R %x", intr);
-		intr = qdma_readl(status + FSL_QDMA_DECFDW1R);
-		DPAA_QDMA_INFO("reg FSL_QDMA_DECFDW1R %x", intr);
-		intr = qdma_readl(status + FSL_QDMA_DECFDW2R);
-		DPAA_QDMA_INFO("reg FSL_QDMA_DECFDW2R %x", intr);
-		intr = qdma_readl(status + FSL_QDMA_DECFDW3R);
-		DPAA_QDMA_INFO("reg FSL_QDMA_DECFDW3R %x", intr);
-		intr = qdma_readl(status + FSL_QDMA_DECFQIDR);
-		DPAA_QDMA_INFO("reg FSL_QDMA_DECFQIDR %x", intr);
-		intr = qdma_readl(status + FSL_QDMA_DECBR);
-		DPAA_QDMA_INFO("reg FSL_QDMA_DECBR %x", intr);
-		qdma_writel(0xffffffff,
-			    status + FSL_QDMA_DEDR);
-		intr = qdma_readl(status + FSL_QDMA_DEDR);
-		fsl_queue->stats.errors++;
+	struct fsl_qdma_engine *fsl_qdma = dev_private;
+	struct fsl_qdma_queue *fsl_queue = fsl_qdma->chan[vchan];
+	struct fsl_qdma_desc *desc_complete[nb_cpls];
+	uint16_t i, dq_num;
+
+	dq_num = dpaa_qdma_block_dequeue(fsl_qdma,
+			fsl_queue->block_id);
+	DPAA_QDMA_DP_DEBUG("%s: block dq(%d)",
+		__func__, dq_num);
+
+	dq_num = rte_ring_dequeue_burst(fsl_queue->complete_pool,
+			(void **)desc_complete, nb_cpls, NULL);
+	for (i = 0; i < dq_num; i++)
+		last_idx[i] = desc_complete[i]->flag;
+
+	if (st) {
+		for (i = 0; i < dq_num; i++)
+			st[i] = RTE_DMA_STATUS_SUCCESSFUL;
 	}
 
-	block = fsl_qdma->block_base +
-		FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, id);
-
-	intr = fsl_qdma_queue_transfer_complete(fsl_qdma, block, id, nb_cpls,
-						last_idx, st);
-	fsl_queue->stats.completed += intr;
 
-	return intr;
+	return dq_num;
 }
 
-
 static uint16_t
 dpaa_qdma_dequeue(void *dev_private,
-		  uint16_t vchan, const uint16_t nb_cpls,
-		  uint16_t *last_idx, bool *has_error)
+	uint16_t vchan, const uint16_t nb_cpls,
+	uint16_t *last_idx, bool *has_error)
 {
-	struct fsl_qdma_engine *fsl_qdma = (struct fsl_qdma_engine *)dev_private;
-	int id = (int)((fsl_qdma->vchan_map[vchan]) / QDMA_QUEUES);
-	void *block;
-	int intr;
-	void *status = fsl_qdma->status_base;
-	struct fsl_qdma_chan *fsl_chan =
-		&fsl_qdma->chans[fsl_qdma->vchan_map[vchan]];
-	struct fsl_qdma_queue *fsl_queue = fsl_chan->queue;
-
-	intr = qdma_readl_be(status + FSL_QDMA_DEDR);
-	if (intr) {
-		DPAA_QDMA_ERR("DMA transaction error! %x", intr);
-		intr = qdma_readl(status + FSL_QDMA_DECFDW0R);
-		DPAA_QDMA_INFO("reg FSL_QDMA_DECFDW0R %x", intr);
-		intr = qdma_readl(status + FSL_QDMA_DECFDW1R);
-		DPAA_QDMA_INFO("reg FSL_QDMA_DECFDW1R %x", intr);
-		intr = qdma_readl(status + FSL_QDMA_DECFDW2R);
-		DPAA_QDMA_INFO("reg FSL_QDMA_DECFDW2R %x", intr);
-		intr = qdma_readl(status + FSL_QDMA_DECFDW3R);
-		DPAA_QDMA_INFO("reg FSL_QDMA_DECFDW3R %x", intr);
-		intr = qdma_readl(status + FSL_QDMA_DECFQIDR);
-		DPAA_QDMA_INFO("reg FSL_QDMA_DECFQIDR %x", intr);
-		intr = qdma_readl(status + FSL_QDMA_DECBR);
-		DPAA_QDMA_INFO("reg FSL_QDMA_DECBR %x", intr);
-		qdma_writel(0xffffffff,
-			    status + FSL_QDMA_DEDR);
-		intr = qdma_readl(status + FSL_QDMA_DEDR);
-		*has_error = true;
-		fsl_queue->stats.errors++;
-	}
+	struct fsl_qdma_engine *fsl_qdma = dev_private;
+	struct fsl_qdma_queue *fsl_queue = fsl_qdma->chan[vchan];
+	struct fsl_qdma_desc *desc_complete[nb_cpls];
+	uint16_t i, dq_num;
 
-	block = fsl_qdma->block_base +
-		FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, id);
 
-	intr = fsl_qdma_queue_transfer_complete(fsl_qdma, block, id, nb_cpls,
-						last_idx, NULL);
-	fsl_queue->stats.completed += intr;
+	*has_error = false;
+	dq_num = dpaa_qdma_block_dequeue(fsl_qdma,
+		fsl_queue->block_id);
+	DPAA_QDMA_DP_DEBUG("%s: block dq(%d)",
+		__func__, dq_num);
 
-	return intr;
+	dq_num = rte_ring_dequeue_burst(fsl_queue->complete_pool,
+			(void **)desc_complete, nb_cpls, NULL);
+	for (i = 0; i < dq_num; i++)
+		last_idx[i] = desc_complete[i]->flag;
+
+	return dq_num;
 }
 
 static int
-dpaa_qdma_stats_get(const struct rte_dma_dev *dmadev, uint16_t vchan,
-		    struct rte_dma_stats *rte_stats, uint32_t size)
+dpaa_qdma_stats_get(const struct rte_dma_dev *dmadev,
+	uint16_t vchan, struct rte_dma_stats *rte_stats, uint32_t size)
 {
 	struct fsl_qdma_engine *fsl_qdma = dmadev->data->dev_private;
-	struct fsl_qdma_chan *fsl_chan =
-		&fsl_qdma->chans[fsl_qdma->vchan_map[vchan]];
-	struct fsl_qdma_queue *fsl_queue = fsl_chan->queue;
+	struct fsl_qdma_queue *fsl_queue = fsl_qdma->chan[vchan];
 	struct rte_dma_stats *stats = &fsl_queue->stats;
 
 	if (size < sizeof(rte_stats))
@@ -903,17 +1032,15 @@ static int
 dpaa_qdma_stats_reset(struct rte_dma_dev *dmadev, uint16_t vchan)
 {
 	struct fsl_qdma_engine *fsl_qdma = dmadev->data->dev_private;
-	struct fsl_qdma_chan *fsl_chan =
-		&fsl_qdma->chans[fsl_qdma->vchan_map[vchan]];
-	struct fsl_qdma_queue *fsl_queue = fsl_chan->queue;
+	struct fsl_qdma_queue *fsl_queue = fsl_qdma->chan[vchan];
 
-	fsl_queue->stats = (struct rte_dma_stats){0};
+	memset(&fsl_queue->stats, 0, sizeof(struct rte_dma_stats));
 
 	return 0;
 }
 
 static struct rte_dma_dev_ops dpaa_qdma_ops = {
-	.dev_info_get		  = dpaa_info_get,
+	.dev_info_get		  = dpaa_qdma_info_get,
 	.dev_configure            = dpaa_qdma_configure,
 	.dev_start                = dpaa_qdma_start,
 	.dev_close                = dpaa_qdma_close,
@@ -926,90 +1053,80 @@ static int
 dpaa_qdma_init(struct rte_dma_dev *dmadev)
 {
 	struct fsl_qdma_engine *fsl_qdma = dmadev->data->dev_private;
-	struct fsl_qdma_chan *fsl_chan;
 	uint64_t phys_addr;
-	unsigned int len;
 	int ccsr_qdma_fd;
 	int regs_size;
 	int ret;
-	u32 i;
+	uint32_t i, j, k;
 
-	fsl_qdma->desc_allocated = 0;
-	fsl_qdma->n_chans = VIRT_CHANNELS;
-	fsl_qdma->n_queues = QDMA_QUEUES;
+	fsl_qdma->n_queues = QDMA_QUEUES * QDMA_BLOCKS;
 	fsl_qdma->num_blocks = QDMA_BLOCKS;
 	fsl_qdma->block_offset = QDMA_BLOCK_OFFSET;
 
-	len = sizeof(*fsl_chan) * fsl_qdma->n_chans;
-	fsl_qdma->chans = rte_zmalloc("qdma: fsl chans", len, 0);
-	if (!fsl_qdma->chans)
-		return -1;
-
-	len = sizeof(struct fsl_qdma_queue *) * fsl_qdma->num_blocks;
-	fsl_qdma->status = rte_zmalloc("qdma: fsl status", len, 0);
-	if (!fsl_qdma->status) {
-		rte_free(fsl_qdma->chans);
-		return -1;
-	}
-
-	for (i = 0; i < fsl_qdma->num_blocks; i++) {
-		rte_atomic32_init(&wait_task[i]);
-		fsl_qdma->status[i] = fsl_qdma_prep_status_queue();
-		if (!fsl_qdma->status[i])
-			goto err;
-	}
-
 	ccsr_qdma_fd = open("/dev/mem", O_RDWR);
 	if (unlikely(ccsr_qdma_fd < 0)) {
 		DPAA_QDMA_ERR("Can not open /dev/mem for qdma CCSR map");
-		goto err;
+		return ccsr_qdma_fd;
 	}
 
-	regs_size = fsl_qdma->block_offset * (fsl_qdma->num_blocks + 2);
+	regs_size = fsl_qdma->block_offset * fsl_qdma->num_blocks;
+	regs_size += (QDMA_CTRL_REGION_SIZE + QDMA_STATUS_REGION_SIZE);
 	phys_addr = QDMA_CCSR_BASE;
-	fsl_qdma->ctrl_base = mmap(NULL, regs_size, PROT_READ |
-					 PROT_WRITE, MAP_SHARED,
-					 ccsr_qdma_fd, phys_addr);
+	fsl_qdma->reg_base = mmap(NULL, regs_size,
+		PROT_READ | PROT_WRITE, MAP_SHARED,
+		ccsr_qdma_fd, phys_addr);
 
 	close(ccsr_qdma_fd);
-	if (fsl_qdma->ctrl_base == MAP_FAILED) {
-		DPAA_QDMA_ERR("Can not map CCSR base qdma: Phys: %08" PRIx64
-		       "size %d", phys_addr, regs_size);
-		goto err;
+	if (fsl_qdma->reg_base == MAP_FAILED) {
+		DPAA_QDMA_ERR("Map qdma reg: Phys(0x%"PRIx64"), size(%d)",
+			phys_addr, regs_size);
+		return -ENOMEM;
 	}
 
-	fsl_qdma->status_base = fsl_qdma->ctrl_base + QDMA_BLOCK_OFFSET;
-	fsl_qdma->block_base = fsl_qdma->status_base + QDMA_BLOCK_OFFSET;
-
-	fsl_qdma->queue = fsl_qdma_alloc_queue_resources(fsl_qdma);
-	if (!fsl_qdma->queue) {
-		munmap(fsl_qdma->ctrl_base, regs_size);
-		goto err;
+	fsl_qdma->ctrl_base =
+		fsl_qdma->reg_base + QDMA_CTRL_REGION_OFFSET;
+	fsl_qdma->status_base =
+		fsl_qdma->reg_base + QDMA_STATUS_REGION_OFFSET;
+	fsl_qdma->block_base =
+		fsl_qdma->status_base + QDMA_STATUS_REGION_SIZE;
+
+	for (i = 0; i < QDMA_BLOCKS; i++) {
+		ret = fsl_qdma_prep_status_queue(fsl_qdma, i);
+		if (ret)
+			goto mem_free;
 	}
 
-	for (i = 0; i < fsl_qdma->n_chans; i++) {
-		struct fsl_qdma_chan *fsl_chan = &fsl_qdma->chans[i];
-
-		fsl_chan->qdma = fsl_qdma;
-		fsl_chan->queue = fsl_qdma->queue + i % (fsl_qdma->n_queues *
-							fsl_qdma->num_blocks);
-		fsl_chan->free = true;
+	k = 0;
+	for (i = 0; i < QDMA_QUEUES; i++) {
+		for (j = 0; j < QDMA_BLOCKS; j++) {
+			ret = fsl_qdma_alloc_queue_resources(fsl_qdma, i, j);
+			if (ret)
+				goto mem_free;
+			fsl_qdma->cmd_queues[j][i].channel_id = k;
+			k++;
+		}
 	}
 
 	ret = fsl_qdma_reg_init(fsl_qdma);
 	if (ret) {
 		DPAA_QDMA_ERR("Can't Initialize the qDMA engine.");
-		munmap(fsl_qdma->ctrl_base, regs_size);
-		goto err;
+		goto mem_free;
 	}
 
 	return 0;
 
-err:
-	rte_free(fsl_qdma->chans);
-	rte_free(fsl_qdma->status);
+mem_free:
+	for (i = 0; i < fsl_qdma->num_blocks; i++)
+		fsl_qdma_free_stq_res(&fsl_qdma->stat_queues[i]);
+
+	for (i = 0; i < fsl_qdma->num_blocks; i++) {
+		for (j = 0; j < QDMA_QUEUES; j++)
+			fsl_qdma_free_cmdq_res(&fsl_qdma->cmd_queues[i][j]);
+	}
 
-	return -1;
+	munmap(fsl_qdma->ctrl_base, regs_size);
+
+	return ret;
 }
 
 static int
@@ -1052,17 +1169,20 @@ dpaa_qdma_remove(struct rte_dpaa_device *dpaa_dev)
 {
 	struct rte_dma_dev *dmadev = dpaa_dev->dmadev;
 	struct fsl_qdma_engine *fsl_qdma = dmadev->data->dev_private;
-	int i = 0, max = QDMA_QUEUES * QDMA_BLOCKS;
+	uint32_t i, j, regs_size;
+
+	regs_size = fsl_qdma->block_offset * fsl_qdma->num_blocks;
+	regs_size += (QDMA_CTRL_REGION_SIZE + QDMA_STATUS_REGION_SIZE);
 
-	for (i = 0; i < max; i++) {
-		struct fsl_qdma_chan *fsl_chan = &fsl_qdma->chans[i];
+	for (i = 0; i < QDMA_BLOCKS; i++)
+		fsl_qdma_free_stq_res(&fsl_qdma->stat_queues[i]);
 
-		if (fsl_chan->free == false)
-			dma_release(fsl_chan);
+	for (i = 0; i < QDMA_BLOCKS; i++) {
+		for (j = 0; j < QDMA_QUEUES; j++)
+			fsl_qdma_free_cmdq_res(&fsl_qdma->cmd_queues[i][j]);
 	}
 
-	rte_free(fsl_qdma->status);
-	rte_free(fsl_qdma->chans);
+	munmap(fsl_qdma->ctrl_base, regs_size);
 
 	(void)rte_dma_pmd_release(dpaa_dev->device.name);
 
diff --git a/drivers/dma/dpaa/dpaa_qdma.h b/drivers/dma/dpaa/dpaa_qdma.h
index 7e9e76e21a..75c014f32f 100644
--- a/drivers/dma/dpaa/dpaa_qdma.h
+++ b/drivers/dma/dpaa/dpaa_qdma.h
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright 2021 NXP
+ * Copyright 2021-2024 NXP
  */
 
 #ifndef _DPAA_QDMA_H_
@@ -11,7 +11,6 @@
 #define BIT(nr)		(1UL << (nr))
 #endif
 
-#define CORE_NUMBER 4
 #define RETRIES	5
 
 #ifndef GENMASK
@@ -20,6 +19,14 @@
 		(((~0UL) << (l)) & (~0UL >> (BITS_PER_LONG - 1 - (h))))
 #endif
 
+#define QDMA_CTRL_REGION_OFFSET 0
+#define QDMA_CTRL_REGION_SIZE 0x10000
+#define QDMA_STATUS_REGION_OFFSET \
+	(QDMA_CTRL_REGION_OFFSET + QDMA_CTRL_REGION_SIZE)
+#define QDMA_STATUS_REGION_SIZE 0x10000
+#define DPAA_QDMA_COPY_IDX_OFFSET 8
+#define DPAA_QDMA_FLAGS_INDEX RTE_BIT64(63)
+
 #define FSL_QDMA_DMR			0x0
 #define FSL_QDMA_DSR			0x4
 #define FSL_QDMA_DEDR			0xe04
@@ -54,15 +61,16 @@
 #define FSL_QDMA_QUEUE_MAX		8
 
 #define FSL_QDMA_BCQMR_EN		0x80000000
-#define FSL_QDMA_BCQMR_EI_BE		0x40
+#define FSL_QDMA_BCQMR_EI		0x40000000
+
 #define FSL_QDMA_BCQMR_CD_THLD(x)	((x) << 20)
 #define FSL_QDMA_BCQMR_CQ_SIZE(x)	((x) << 16)
 
 #define FSL_QDMA_BCQSR_QF_XOFF_BE	0x1000100
 
 #define FSL_QDMA_BSQMR_EN		0x80000000
-#define FSL_QDMA_BSQMR_DI_BE		0x40
 #define FSL_QDMA_BSQMR_CQ_SIZE(x)	((x) << 16)
+#define FSL_QDMA_BSQMR_DI		0xc0
 
 #define FSL_QDMA_BSQSR_QE_BE		0x200
 
@@ -75,23 +83,14 @@
 #define FSL_QDMA_CIRCULAR_DESC_SIZE_MAX	16384
 #define FSL_QDMA_QUEUE_NUM_MAX		8
 
+#define FSL_QDMA_COMP_SG_FORMAT		0x1
+
 #define FSL_QDMA_CMD_RWTTYPE		0x4
 #define FSL_QDMA_CMD_LWC		0x2
 
 #define FSL_QDMA_CMD_RWTTYPE_OFFSET	28
 #define FSL_QDMA_CMD_LWC_OFFSET		16
 
-#define QDMA_CCDF_STATUS		20
-#define QDMA_CCDF_OFFSET		20
-#define QDMA_CCDF_MASK			GENMASK(28, 20)
-#define QDMA_CCDF_FOTMAT		BIT(29)
-#define QDMA_CCDF_SER			BIT(30)
-
-#define QDMA_SG_FIN			BIT(30)
-#define QDMA_SG_LEN_MASK		GENMASK(29, 0)
-
-#define COMMAND_QUEUE_OVERFLOW		10
-
 /* qdma engine attribute */
 #define QDMA_QUEUE_SIZE			64
 #define QDMA_STATUS_SIZE		64
@@ -101,6 +100,7 @@
 #define QDMA_BLOCKS			4
 #define QDMA_QUEUES			8
 #define QDMA_DELAY			1000
+#define QDMA_QUEUE_CR_WM 32
 
 #define QDMA_BIG_ENDIAN			1
 #ifdef QDMA_BIG_ENDIAN
@@ -118,89 +118,145 @@
 #define FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma_engine, x)			\
 	(((fsl_qdma_engine)->block_offset) * (x))
 
-typedef void (*dma_call_back)(void *params);
-
 /* qDMA Command Descriptor Formats */
-struct fsl_qdma_format {
-	__le32 status; /* ser, status */
-	__le32 cfg;	/* format, offset */
-	union {
-		struct {
-			__le32 addr_lo;	/* low 32-bits of 40-bit address */
-			u8 addr_hi;	/* high 8-bits of 40-bit address */
-			u8 __reserved1[2];
-			u8 cfg8b_w1; /* dd, queue */
-		};
-		__le64 data;
-	};
-};
+struct fsl_qdma_comp_cmd_desc {
+	uint8_t status;
+	uint32_t rsv0:22;
+	uint32_t ser:1;
+	uint32_t rsv1:21;
+	uint32_t offset:9;
+	uint32_t format:3;
+	uint32_t addr_lo;
+	uint8_t addr_hi;
+	uint16_t rsv3;
+	uint8_t queue:3;
+	uint8_t rsv4:3;
+	uint8_t dd:2;
+} __rte_packed;
+
+struct fsl_qdma_comp_sg_desc {
+	uint32_t offset:13;
+	uint32_t rsv0:19;
+	uint32_t length:30;
+	uint32_t final:1;
+	uint32_t extion:1;
+	uint32_t addr_lo;
+	uint8_t addr_hi;
+	uint32_t rsv1:24;
+} __rte_packed;
 
-/* qDMA Source Descriptor Format */
 struct fsl_qdma_sdf {
-	__le32 rev3;
-	__le32 cfg; /* rev4, bit[0-11] - ssd, bit[12-23] sss */
-	__le32 rev5;
-	__le32 cmd;
-};
+	uint32_t rsv0;
+	uint32_t ssd:12;
+	uint32_t sss:12;
+	uint32_t rsv1:8;
+	uint32_t rsv2;
+
+	uint32_t rsv3:17;
+	uint32_t prefetch:1;
+	uint32_t rsv4:1;
+	uint32_t ssen:1;
+	uint32_t rthrotl:4;
+	uint32_t sqos:3;
+	uint32_t ns:1;
+	uint32_t srttype:4;
+} __rte_packed;
 
-/* qDMA Destination Descriptor Format */
 struct fsl_qdma_ddf {
-	__le32 rev1;
-	__le32 cfg; /* rev2, bit[0-11] - dsd, bit[12-23] - dss */
-	__le32 rev3;
-	__le32 cmd;
+	uint32_t rsv0;
+	uint32_t dsd:12;
+	uint32_t dss:12;
+	uint32_t rsv1:8;
+	uint32_t rsv2;
+
+	uint16_t rsv3;
+	uint32_t lwc:2;
+	uint32_t rsv4:1;
+	uint32_t dsen:1;
+	uint32_t wthrotl:4;
+	uint32_t dqos:3;
+	uint32_t ns:1;
+	uint32_t dwttype:4;
+} __rte_packed;
+
+struct fsl_qdma_df {
+	struct fsl_qdma_sdf sdf;
+	struct fsl_qdma_ddf ddf;
 };
 
-struct fsl_qdma_chan {
-	struct fsl_qdma_engine	*qdma;
-	struct fsl_qdma_queue	*queue;
-	bool			free;
-	struct list_head	list;
+#define FSL_QDMA_SG_MAX_ENTRY 64
+#define FSL_QDMA_MAX_DESC_NUM (FSL_QDMA_SG_MAX_ENTRY * QDMA_QUEUE_SIZE)
+struct fsl_qdma_cmpd_ft {
+	struct fsl_qdma_comp_sg_desc desc_buf;
+	struct fsl_qdma_comp_sg_desc desc_sbuf;
+	struct fsl_qdma_comp_sg_desc desc_dbuf;
+	uint64_t cache_align[2];
+	struct fsl_qdma_comp_sg_desc desc_ssge[FSL_QDMA_SG_MAX_ENTRY];
+	struct fsl_qdma_comp_sg_desc desc_dsge[FSL_QDMA_SG_MAX_ENTRY];
+	struct fsl_qdma_df df;
+	uint64_t phy_ssge;
+	uint64_t phy_dsge;
+	uint64_t phy_df;
+} __rte_packed;
+
+#define DPAA_QDMA_IDX_FROM_FLAG(flag) \
+	((flag) >> DPAA_QDMA_COPY_IDX_OFFSET)
+
+struct fsl_qdma_desc {
+	rte_iova_t src;
+	rte_iova_t dst;
+	uint64_t flag;
+	uint64_t len;
 };
 
 struct fsl_qdma_queue {
-	struct fsl_qdma_format	*virt_head;
-	struct list_head	comp_used;
-	struct list_head	comp_free;
-	dma_addr_t		bus_addr;
-	u32			n_cq;
-	u32			id;
-	u32			count;
-	u32			pending;
-	struct fsl_qdma_format	*cq;
-	void			*block_base;
-	struct rte_dma_stats	stats;
+	int used;
+	struct fsl_qdma_cmpd_ft **ft;
+	uint16_t ci;
+	struct rte_ring *complete_burst;
+	struct rte_ring *complete_desc;
+	struct rte_ring *complete_pool;
+	uint16_t n_cq;
+	uint8_t block_id;
+	uint8_t queue_id;
+	uint8_t channel_id;
+	void *block_vir;
+	uint32_t le_cqmr;
+	struct fsl_qdma_comp_cmd_desc *cq;
+	uint16_t desc_in_hw[QDMA_QUEUE_SIZE];
+	struct rte_dma_stats stats;
+	struct fsl_qdma_desc *pending_desc;
+	uint16_t pending_max;
+	uint16_t pending_start;
+	uint16_t pending_num;
+	uint16_t complete_start;
+	dma_addr_t bus_addr;
+	void *engine;
 };
 
-struct fsl_qdma_comp {
-	dma_addr_t		bus_addr;
-	dma_addr_t		desc_bus_addr;
-	void			*virt_addr;
-	int			index;
-	void			*desc_virt_addr;
-	struct fsl_qdma_chan	*qchan;
-	dma_call_back		call_back_func;
-	void			*params;
-	struct list_head	list;
+struct fsl_qdma_status_queue {
+	uint16_t n_cq;
+	uint16_t complete;
+	uint8_t block_id;
+	void *block_vir;
+	struct fsl_qdma_comp_cmd_desc *cq;
+	struct rte_dma_stats stats;
+	dma_addr_t bus_addr;
+	void *engine;
 };
 
 struct fsl_qdma_engine {
-	int			desc_allocated;
-	void			*ctrl_base;
-	void			*status_base;
-	void			*block_base;
-	u32			n_chans;
-	u32			n_queues;
-	int			error_irq;
-	struct fsl_qdma_queue	*queue;
-	struct fsl_qdma_queue	**status;
-	struct fsl_qdma_chan	*chans;
-	u32			num_blocks;
-	u8			free_block_id;
-	u32			vchan_map[4];
-	int			block_offset;
+	void *reg_base;
+	void *ctrl_base;
+	void *status_base;
+	void *block_base;
+	uint32_t n_queues;
+	uint8_t block_queues[QDMA_BLOCKS];
+	struct fsl_qdma_queue cmd_queues[QDMA_BLOCKS][QDMA_QUEUES];
+	struct fsl_qdma_status_queue stat_queues[QDMA_BLOCKS];
+	struct fsl_qdma_queue *chan[QDMA_BLOCKS * QDMA_QUEUES];
+	uint32_t num_blocks;
+	int block_offset;
 };
 
-static rte_atomic32_t wait_task[CORE_NUMBER];
-
 #endif /* _DPAA_QDMA_H_ */
-- 
2.25.1