[v1 16/43] bus/fslmc: dynamic IOVA mode configuration

[vanshika.shukla@nxp.com]

From: Jun Yang <jun.yang@nxp.com>

IOVA mode should not be configured with CFLAGS because
1) User can perform "--iova-mode" to configure IOVA.
2) IOVA mode is determined by negotiation between multiple devices.
   Eal is in VA mode only when all devices support VA mode.

Hence:
1) Remove RTE_LIBRTE_DPAA2_USE_PHYS_IOVA cflags.
   Instead, use rte_eal_iova_mode API to identify VA or PA mode.
2) Support memory IOMMU mapping and I/O IOMMU mapping(PCI space).
3) For memory IOMMU, in VA mode, IOVA:VA = 1:1;
   in PA mode, IOVA:VA = PA:VA. The mapping policy is determined by
   EAL memory driver.
4) For I/O IOMMU, IOVA:VA is up to I/O driver configuration.
   In general, it's aligned with memory IOMMU mapping.
5) Memory and I/O IOVA tables are created and update when DMA
   mapping is setup, which takes place of dpaax IOVA table.

Signed-off-by: Jun Yang <jun.yang@nxp.com>
Signed-off-by: Vanshika Shukla <vanshika.shukla@nxp.com>
---
 drivers/bus/fslmc/bus_fslmc_driver.h     |  29 +-
 drivers/bus/fslmc/fslmc_bus.c            |  33 +-
 drivers/bus/fslmc/fslmc_logs.h           |   5 +-
 drivers/bus/fslmc/fslmc_vfio.c           | 666 ++++++++++++++++++-----
 drivers/bus/fslmc/fslmc_vfio.h           |   4 +
 drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c |   3 +-
 drivers/bus/fslmc/portal/dpaa2_hw_dpio.c |  10 +-
 drivers/bus/fslmc/portal/dpaa2_hw_dpio.h |   3 +-
 drivers/bus/fslmc/portal/dpaa2_hw_pvt.h  | 111 ++--
 drivers/bus/fslmc/version.map            |   7 +-
 drivers/dma/dpaa2/dpaa2_qdma.c           |   1 +
 11 files changed, 617 insertions(+), 255 deletions(-)

diff --git a/drivers/bus/fslmc/bus_fslmc_driver.h b/drivers/bus/fslmc/bus_fslmc_driver.h
index dc2f395f60..11eebd560c 100644
--- a/drivers/bus/fslmc/bus_fslmc_driver.h
+++ b/drivers/bus/fslmc/bus_fslmc_driver.h
@@ -37,9 +37,6 @@ extern "C" {
 
 #include <fslmc_vfio.h>
 
-#include "portal/dpaa2_hw_pvt.h"
-#include "portal/dpaa2_hw_dpio.h"
-
 #define FSLMC_OBJECT_MAX_LEN 32   /**< Length of each device on bus */
 
 #define DPAA2_INVALID_MBUF_SEQN        0
@@ -149,6 +146,32 @@ struct rte_dpaa2_driver {
 	rte_dpaa2_remove_t remove;
 };
 
+int
+rte_fslmc_vfio_mem_dmamap(uint64_t vaddr, uint64_t iova, uint64_t size);
+__rte_internal
+int
+rte_fslmc_vfio_mem_dmaunmap(uint64_t iova, uint64_t size);
+__rte_internal
+uint64_t
+rte_fslmc_cold_mem_vaddr_to_iova(void *vaddr,
+	uint64_t size);
+__rte_internal
+void *
+rte_fslmc_cold_mem_iova_to_vaddr(uint64_t iova,
+	uint64_t size);
+__rte_internal
+__hot uint64_t
+rte_fslmc_mem_vaddr_to_iova(void *vaddr);
+__rte_internal
+__hot void *
+rte_fslmc_mem_iova_to_vaddr(uint64_t iova);
+__rte_internal
+uint64_t
+rte_fslmc_io_vaddr_to_iova(void *vaddr);
+__rte_internal
+void *
+rte_fslmc_io_iova_to_vaddr(uint64_t iova);
+
 /**
  * Register a DPAA2 driver.
  *
diff --git a/drivers/bus/fslmc/fslmc_bus.c b/drivers/bus/fslmc/fslmc_bus.c
index 654726dbe6..ce87b4ddbd 100644
--- a/drivers/bus/fslmc/fslmc_bus.c
+++ b/drivers/bus/fslmc/fslmc_bus.c
@@ -27,7 +27,6 @@
 #define FSLMC_BUS_NAME	fslmc
 
 struct rte_fslmc_bus rte_fslmc_bus;
-uint8_t dpaa2_virt_mode;
 
 #define DPAA2_SEQN_DYNFIELD_NAME "dpaa2_seqn_dynfield"
 int dpaa2_seqn_dynfield_offset = -1;
@@ -457,22 +456,6 @@ rte_fslmc_probe(void)
 
 	probe_all = rte_fslmc_bus.bus.conf.scan_mode != RTE_BUS_SCAN_ALLOWLIST;
 
-	/* In case of PA, the FD addresses returned by qbman APIs are physical
-	 * addresses, which need conversion into equivalent VA address for
-	 * rte_mbuf. For that, a table (a serial array, in memory) is used to
-	 * increase translation efficiency.
-	 * This has to be done before probe as some device initialization
-	 * (during) probe allocate memory (dpaa2_sec) which needs to be pinned
-	 * to this table.
-	 *
-	 * Error is ignored as relevant logs are handled within dpaax and
-	 * handling for unavailable dpaax table too is transparent to caller.
-	 *
-	 * And, the IOVA table is only applicable in case of PA mode.
-	 */
-	if (rte_eal_iova_mode() == RTE_IOVA_PA)
-		dpaax_iova_table_populate();
-
 	TAILQ_FOREACH(dev, &rte_fslmc_bus.device_list, next) {
 		TAILQ_FOREACH(drv, &rte_fslmc_bus.driver_list, next) {
 			ret = rte_fslmc_match(drv, dev);
@@ -507,9 +490,6 @@ rte_fslmc_probe(void)
 		}
 	}
 
-	if (rte_eal_iova_mode() == RTE_IOVA_VA)
-		dpaa2_virt_mode = 1;
-
 	return 0;
 }
 
@@ -558,12 +538,6 @@ rte_fslmc_driver_register(struct rte_dpaa2_driver *driver)
 void
 rte_fslmc_driver_unregister(struct rte_dpaa2_driver *driver)
 {
-	/* Cleanup the PA->VA Translation table; From wherever this function
-	 * is called from.
-	 */
-	if (rte_eal_iova_mode() == RTE_IOVA_PA)
-		dpaax_iova_table_depopulate();
-
 	TAILQ_REMOVE(&rte_fslmc_bus.driver_list, driver, next);
 }
 
@@ -599,13 +573,12 @@ rte_dpaa2_get_iommu_class(void)
 	bool is_vfio_noiommu_enabled = 1;
 	bool has_iova_va;
 
+	if (rte_eal_iova_mode() == RTE_IOVA_PA)
+		return RTE_IOVA_PA;
+
 	if (TAILQ_EMPTY(&rte_fslmc_bus.device_list))
 		return RTE_IOVA_DC;
 
-#ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
-	return RTE_IOVA_PA;
-#endif
-
 	/* check if all devices on the bus support Virtual addressing or not */
 	has_iova_va = fslmc_all_device_support_iova();
 
diff --git a/drivers/bus/fslmc/fslmc_logs.h b/drivers/bus/fslmc/fslmc_logs.h
index e15c603426..d6abffc566 100644
--- a/drivers/bus/fslmc/fslmc_logs.h
+++ b/drivers/bus/fslmc/fslmc_logs.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
- *   Copyright 2016 NXP
+ *   Copyright 2016-2023 NXP
  *
  */
 
@@ -10,7 +10,8 @@
 extern int dpaa2_logtype_bus;
 
 #define DPAA2_BUS_LOG(level, fmt, args...) \
-	rte_log(RTE_LOG_ ## level, dpaa2_logtype_bus, "fslmc: " fmt "\n", \
+	rte_log(RTE_LOG_ ## level, dpaa2_logtype_bus, \
+		"fslmc " # level ": " fmt "\n", \
 		##args)
 
 /* Debug logs are with Function names */
diff --git a/drivers/bus/fslmc/fslmc_vfio.c b/drivers/bus/fslmc/fslmc_vfio.c
index b550066183..31011b8532 100644
--- a/drivers/bus/fslmc/fslmc_vfio.c
+++ b/drivers/bus/fslmc/fslmc_vfio.c
@@ -19,6 +19,7 @@
 #include <libgen.h>
 #include <dirent.h>
 #include <sys/eventfd.h>
+#include <ctype.h>
 
 #include <eal_filesystem.h>
 #include <rte_mbuf.h>
@@ -49,9 +50,41 @@
  */
 static struct fslmc_vfio_container s_vfio_container;
 /* Currently we only support single group/process. */
-const char *fslmc_group; /* dprc.x*/
+static const char *fslmc_group; /* dprc.x*/
 static uint32_t *msi_intr_vaddr;
-void *(*rte_mcp_ptr_list);
+static void *(*rte_mcp_ptr_list);
+
+struct fslmc_dmaseg {
+	uint64_t vaddr;
+	uint64_t iova;
+	uint64_t size;
+
+	TAILQ_ENTRY(fslmc_dmaseg) next;
+};
+
+TAILQ_HEAD(fslmc_dmaseg_list, fslmc_dmaseg);
+
+struct fslmc_dmaseg_list fslmc_memsegs =
+		TAILQ_HEAD_INITIALIZER(fslmc_memsegs);
+struct fslmc_dmaseg_list fslmc_iosegs =
+		TAILQ_HEAD_INITIALIZER(fslmc_iosegs);
+
+static uint64_t fslmc_mem_va2iova = RTE_BAD_IOVA;
+static int fslmc_mem_map_num;
+
+struct fslmc_mem_param {
+	struct vfio_mp_param mp_param;
+	struct fslmc_dmaseg_list memsegs;
+	struct fslmc_dmaseg_list iosegs;
+	uint64_t mem_va2iova;
+	int mem_map_num;
+};
+
+enum {
+	FSLMC_VFIO_SOCKET_REQ_CONTAINER = 0x100,
+	FSLMC_VFIO_SOCKET_REQ_GROUP,
+	FSLMC_VFIO_SOCKET_REQ_MEM
+};
 
 void *
 dpaa2_get_mcp_ptr(int portal_idx)
@@ -65,6 +98,64 @@ dpaa2_get_mcp_ptr(int portal_idx)
 static struct rte_dpaa2_object_list dpaa2_obj_list =
 	TAILQ_HEAD_INITIALIZER(dpaa2_obj_list);
 
+static uint64_t
+fslmc_io_virt2phy(const void *virtaddr)
+{
+	FILE *fp = fopen("/proc/self/maps", "r");
+	char *line = NULL;
+	size_t linesz;
+	uint64_t start, end, phy;
+	const uint64_t va = (const uint64_t)virtaddr;
+	char tmp[1024];
+	int ret;
+
+	if (!fp)
+		return RTE_BAD_IOVA;
+	while (getdelim(&line, &linesz, '\n', fp) > 0) {
+		char *ptr = line;
+		int n;
+
+		/** Parse virtual address range.*/
+		n = 0;
+		while (*ptr && !isspace(*ptr)) {
+			tmp[n] = *ptr;
+			ptr++;
+			n++;
+		}
+		tmp[n] = 0;
+		ret = sscanf(tmp, "%" SCNx64 "-%" SCNx64, &start, &end);
+		if (ret != 2)
+			continue;
+		if (va < start || va >= end)
+			continue;
+
+		/** This virtual address is in this segment.*/
+		while (*ptr == ' ' || *ptr == 'r' ||
+			*ptr == 'w' || *ptr == 's' ||
+			*ptr == 'p' || *ptr == 'x' ||
+			*ptr == '-')
+			ptr++;
+
+		/** Extract phy address*/
+		n = 0;
+		while (*ptr && !isspace(*ptr)) {
+			tmp[n] = *ptr;
+			ptr++;
+			n++;
+		}
+		tmp[n] = 0;
+		phy = strtoul(tmp, 0, 16);
+		if (!phy)
+			continue;
+
+		fclose(fp);
+		return phy + va - start;
+	}
+
+	fclose(fp);
+	return RTE_BAD_IOVA;
+}
+
 /*register a fslmc bus based dpaa2 driver */
 void
 rte_fslmc_object_register(struct rte_dpaa2_object *object)
@@ -271,7 +362,7 @@ fslmc_get_group_id(const char *group_name,
 	ret = rte_vfio_get_group_num(SYSFS_FSL_MC_DEVICES,
 			group_name, groupid);
 	if (ret <= 0) {
-		DPAA2_BUS_ERR("Unable to find %s IOMMU group", group_name);
+		DPAA2_BUS_ERR("Find %s IOMMU group", group_name);
 		if (ret < 0)
 			return ret;
 
@@ -314,7 +405,7 @@ fslmc_vfio_open_group_fd(const char *group_name)
 	/* if we're in a secondary process, request group fd from the primary
 	 * process via mp channel.
 	 */
-	p->req = SOCKET_REQ_GROUP;
+	p->req = FSLMC_VFIO_SOCKET_REQ_GROUP;
 	p->group_num = iommu_group_num;
 	strcpy(mp_req.name, FSLMC_VFIO_MP);
 	mp_req.len_param = sizeof(*p);
@@ -408,7 +499,7 @@ fslmc_vfio_open_container_fd(void)
 	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 		vfio_container_fd = open(VFIO_CONTAINER_PATH, O_RDWR);
 		if (vfio_container_fd < 0) {
-			DPAA2_BUS_ERR("Cannot open VFIO container(%s), err(%d)",
+			DPAA2_BUS_ERR("Open VFIO container(%s), err(%d)",
 				VFIO_CONTAINER_PATH, vfio_container_fd);
 			ret = vfio_container_fd;
 			goto err_exit;
@@ -417,7 +508,7 @@ fslmc_vfio_open_container_fd(void)
 		/* check VFIO API version */
 		ret = ioctl(vfio_container_fd, VFIO_GET_API_VERSION);
 		if (ret < 0) {
-			DPAA2_BUS_ERR("Could not get VFIO API version(%d)",
+			DPAA2_BUS_ERR("Get VFIO API version(%d)",
 				ret);
 		} else if (ret != VFIO_API_VERSION) {
 			DPAA2_BUS_ERR("Unsupported VFIO API version(%d)",
@@ -431,7 +522,7 @@ fslmc_vfio_open_container_fd(void)
 
 		ret = fslmc_vfio_check_extensions(vfio_container_fd);
 		if (ret) {
-			DPAA2_BUS_ERR("No supported IOMMU extensions found(%d)",
+			DPAA2_BUS_ERR("Unsupported IOMMU extensions found(%d)",
 				ret);
 			close(vfio_container_fd);
 			goto err_exit;
@@ -443,7 +534,7 @@ fslmc_vfio_open_container_fd(void)
 	 * if we're in a secondary process, request container fd from the
 	 * primary process via mp channel
 	 */
-	p->req = SOCKET_REQ_CONTAINER;
+	p->req = FSLMC_VFIO_SOCKET_REQ_CONTAINER;
 	strcpy(mp_req.name, FSLMC_VFIO_MP);
 	mp_req.len_param = sizeof(*p);
 	mp_req.num_fds = 0;
@@ -473,7 +564,7 @@ fslmc_vfio_open_container_fd(void)
 err_exit:
 	if (mp_reply.msgs)
 		free(mp_reply.msgs);
-	DPAA2_BUS_ERR("Cannot request container fd err(%d)", ret);
+	DPAA2_BUS_ERR("Open container fd err(%d)", ret);
 	return ret;
 }
 
@@ -506,17 +597,19 @@ fslmc_vfio_mp_primary(const struct rte_mp_msg *msg,
 	struct rte_mp_msg reply;
 	struct vfio_mp_param *r = (void *)reply.param;
 	const struct vfio_mp_param *m = (const void *)msg->param;
+	struct fslmc_mem_param *map;
 
 	if (msg->len_param != sizeof(*m)) {
-		DPAA2_BUS_ERR("fslmc vfio received invalid message!");
+		DPAA2_BUS_ERR("Invalid msg size(%d) for req(%d)",
+			msg->len_param, m->req);
 		return -EINVAL;
 	}
 
 	memset(&reply, 0, sizeof(reply));
 
 	switch (m->req) {
-	case SOCKET_REQ_GROUP:
-		r->req = SOCKET_REQ_GROUP;
+	case FSLMC_VFIO_SOCKET_REQ_GROUP:
+		r->req = FSLMC_VFIO_SOCKET_REQ_GROUP;
 		r->group_num = m->group_num;
 		fd = fslmc_vfio_group_fd_by_id(m->group_num);
 		if (fd < 0) {
@@ -530,9 +623,10 @@ fslmc_vfio_mp_primary(const struct rte_mp_msg *msg,
 			reply.num_fds = 1;
 			reply.fds[0] = fd;
 		}
+		reply.len_param = sizeof(*r);
 		break;
-	case SOCKET_REQ_CONTAINER:
-		r->req = SOCKET_REQ_CONTAINER;
+	case FSLMC_VFIO_SOCKET_REQ_CONTAINER:
+		r->req = FSLMC_VFIO_SOCKET_REQ_CONTAINER;
 		fd = fslmc_vfio_container_fd();
 		if (fd <= 0) {
 			r->result = SOCKET_ERR;
@@ -541,20 +635,73 @@ fslmc_vfio_mp_primary(const struct rte_mp_msg *msg,
 			reply.num_fds = 1;
 			reply.fds[0] = fd;
 		}
+		reply.len_param = sizeof(*r);
+		break;
+	case FSLMC_VFIO_SOCKET_REQ_MEM:
+		map = (void *)reply.param;
+		r = &map->mp_param;
+		r->req = FSLMC_VFIO_SOCKET_REQ_MEM;
+		r->result = SOCKET_OK;
+		rte_memcpy(&map->memsegs, &fslmc_memsegs,
+			sizeof(struct fslmc_dmaseg_list));
+		rte_memcpy(&map->iosegs, &fslmc_iosegs,
+			sizeof(struct fslmc_dmaseg_list));
+		map->mem_va2iova = fslmc_mem_va2iova;
+		map->mem_map_num = fslmc_mem_map_num;
+		reply.len_param = sizeof(struct fslmc_mem_param);
 		break;
 	default:
-		DPAA2_BUS_ERR("fslmc vfio received invalid message(%08x)",
+		DPAA2_BUS_ERR("VFIO received invalid message(%08x)",
 			m->req);
 		return -ENOTSUP;
 	}
 
 	strcpy(reply.name, FSLMC_VFIO_MP);
-	reply.len_param = sizeof(*r);
 	ret = rte_mp_reply(&reply, peer);
 
 	return ret;
 }
 
+static int
+fslmc_vfio_mp_sync_mem_req(void)
+{
+	struct rte_mp_msg mp_req, *mp_rep;
+	struct rte_mp_reply mp_reply = {0};
+	struct timespec ts = {.tv_sec = 5, .tv_nsec = 0};
+	int ret = 0;
+	struct vfio_mp_param *mp_param;
+	struct fslmc_mem_param *mem_rsp;
+
+	mp_param = (void *)mp_req.param;
+	memset(&mp_req, 0, sizeof(struct rte_mp_msg));
+	mp_param->req = FSLMC_VFIO_SOCKET_REQ_MEM;
+	strcpy(mp_req.name, FSLMC_VFIO_MP);
+	mp_req.len_param = sizeof(struct vfio_mp_param);
+	if (rte_mp_request_sync(&mp_req, &mp_reply, &ts) == 0 &&
+		mp_reply.nb_received == 1) {
+		mp_rep = &mp_reply.msgs[0];
+		mem_rsp = (struct fslmc_mem_param *)mp_rep->param;
+		if (mem_rsp->mp_param.result == SOCKET_OK) {
+			rte_memcpy(&fslmc_memsegs,
+				&mem_rsp->memsegs,
+				sizeof(struct fslmc_dmaseg_list));
+			rte_memcpy(&fslmc_memsegs,
+				&mem_rsp->memsegs,
+				sizeof(struct fslmc_dmaseg_list));
+			fslmc_mem_va2iova = mem_rsp->mem_va2iova;
+			fslmc_mem_map_num = mem_rsp->mem_map_num;
+		} else {
+			DPAA2_BUS_ERR("Bad MEM SEG");
+			ret = -EINVAL;
+		}
+	} else {
+		ret = -EINVAL;
+	}
+	free(mp_reply.msgs);
+
+	return ret;
+}
+
 static int
 fslmc_vfio_mp_sync_setup(void)
 {
@@ -565,6 +712,10 @@ fslmc_vfio_mp_sync_setup(void)
 			fslmc_vfio_mp_primary);
 		if (ret && rte_errno != ENOTSUP)
 			return ret;
+	} else {
+		ret = fslmc_vfio_mp_sync_mem_req();
+		if (ret)
+			return ret;
 	}
 
 	return 0;
@@ -585,30 +736,34 @@ vfio_connect_container(int vfio_container_fd,
 
 	iommu_type = fslmc_vfio_iommu_type(vfio_group_fd);
 	if (iommu_type < 0) {
-		DPAA2_BUS_ERR("Failed to get iommu type(%d)",
-			iommu_type);
+		DPAA2_BUS_ERR("Get iommu type(%d)", iommu_type);
 
 		return iommu_type;
 	}
 
 	/* Check whether support for SMMU type IOMMU present or not */
-	if (ioctl(vfio_container_fd, VFIO_CHECK_EXTENSION, iommu_type)) {
-		/* Connect group to container */
-		ret = ioctl(vfio_group_fd, VFIO_GROUP_SET_CONTAINER,
+	ret = ioctl(vfio_container_fd, VFIO_CHECK_EXTENSION, iommu_type);
+	if (ret <= 0) {
+		DPAA2_BUS_ERR("Unsupport IOMMU type(%d) ret(%d), err(%d)",
+			iommu_type, ret, -errno);
+		return -EINVAL;
+	}
+
+	ret = ioctl(vfio_group_fd, VFIO_GROUP_SET_CONTAINER,
 			&vfio_container_fd);
-		if (ret) {
-			DPAA2_BUS_ERR("Failed to setup group container");
-			return -errno;
-		}
+	if (ret) {
+		DPAA2_BUS_ERR("Set group container ret(%d), err(%d)",
+			ret, -errno);
 
-		ret = ioctl(vfio_container_fd, VFIO_SET_IOMMU, iommu_type);
-		if (ret) {
-			DPAA2_BUS_ERR("Failed to setup VFIO iommu");
-			return -errno;
-		}
-	} else {
-		DPAA2_BUS_ERR("No supported IOMMU available");
-		return -EINVAL;
+		return ret;
+	}
+
+	ret = ioctl(vfio_container_fd, VFIO_SET_IOMMU, iommu_type);
+	if (ret) {
+		DPAA2_BUS_ERR("Set iommu ret(%d), err(%d)",
+			ret, -errno);
+
+		return ret;
 	}
 
 	return fslmc_vfio_connect_container(vfio_group_fd);
@@ -629,11 +784,11 @@ static int vfio_map_irq_region(void)
 
 	fd = fslmc_vfio_group_fd_by_name(group_name);
 	if (fd <= 0) {
-		DPAA2_BUS_ERR("%s failed to open group fd(%d)",
-			__func__, fd);
+		DPAA2_BUS_ERR("%s: Get fd by name(%s) failed(%d)",
+			__func__, group_name, fd);
 		if (fd < 0)
 			return fd;
-		return -rte_errno;
+		return -EIO;
 	}
 	if (!fslmc_vfio_container_connected(fd)) {
 		DPAA2_BUS_ERR("Container is not connected");
@@ -643,8 +798,8 @@ static int vfio_map_irq_region(void)
 	vaddr = (unsigned long *)mmap(NULL, 0x1000, PROT_WRITE |
 		PROT_READ, MAP_SHARED, fd, 0x6030000);
 	if (vaddr == MAP_FAILED) {
-		DPAA2_BUS_INFO("Unable to map region (errno = %d)", errno);
-		return -errno;
+		DPAA2_BUS_ERR("Unable to map region (errno = %d)", errno);
+		return -ENOMEM;
 	}
 
 	msi_intr_vaddr = (uint32_t *)((char *)(vaddr) + 64);
@@ -654,141 +809,200 @@ static int vfio_map_irq_region(void)
 		return 0;
 
 	DPAA2_BUS_ERR("Unable to map DMA address (errno = %d)", errno);
-	return -errno;
-}
-
-static int fslmc_map_dma(uint64_t vaddr, rte_iova_t iovaddr, size_t len);
-static int fslmc_unmap_dma(uint64_t vaddr, rte_iova_t iovaddr, size_t len);
-
-static void
-fslmc_memevent_cb(enum rte_mem_event type, const void *addr,
-	size_t len, void *arg __rte_unused)
-{
-	struct rte_memseg_list *msl;
-	struct rte_memseg *ms;
-	size_t cur_len = 0, map_len = 0;
-	uint64_t virt_addr;
-	rte_iova_t iova_addr;
-	int ret;
-
-	msl = rte_mem_virt2memseg_list(addr);
-
-	while (cur_len < len) {
-		const void *va = RTE_PTR_ADD(addr, cur_len);
-
-		ms = rte_mem_virt2memseg(va, msl);
-		iova_addr = ms->iova;
-		virt_addr = ms->addr_64;
-		map_len = ms->len;
-
-		DPAA2_BUS_DEBUG("Request for %s, va=%p, "
-				"virt_addr=0x%" PRIx64 ", "
-				"iova=0x%" PRIx64 ", map_len=%zu",
-				type == RTE_MEM_EVENT_ALLOC ?
-					"alloc" : "dealloc",
-				va, virt_addr, iova_addr, map_len);
-
-		/* iova_addr may be set to RTE_BAD_IOVA */
-		if (iova_addr == RTE_BAD_IOVA) {
-			DPAA2_BUS_DEBUG("Segment has invalid iova, skipping\n");
-			cur_len += map_len;
-			continue;
-		}
-
-		if (type == RTE_MEM_EVENT_ALLOC)
-			ret = fslmc_map_dma(virt_addr, iova_addr, map_len);
-		else
-			ret = fslmc_unmap_dma(virt_addr, iova_addr, map_len);
-
-		if (ret != 0) {
-			DPAA2_BUS_ERR("DMA Mapping/Unmapping failed. "
-					"Map=%d, addr=%p, len=%zu, err:(%d)",
-					type, va, map_len, ret);
-			return;
-		}
-
-		cur_len += map_len;
-	}
-
-	if (type == RTE_MEM_EVENT_ALLOC)
-		DPAA2_BUS_DEBUG("Total Mapped: addr=%p, len=%zu",
-				addr, len);
-	else
-		DPAA2_BUS_DEBUG("Total Unmapped: addr=%p, len=%zu",
-				addr, len);
+	return ret;
 }
 
 static int
-fslmc_map_dma(uint64_t vaddr, rte_iova_t iovaddr,
-	size_t len)
+fslmc_map_dma(uint64_t vaddr, rte_iova_t iovaddr, size_t len)
 {
 	struct vfio_iommu_type1_dma_map dma_map = {
 		.argsz = sizeof(struct vfio_iommu_type1_dma_map),
 		.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE,
 	};
-	int ret, fd;
+	int ret, fd, is_io = 0;
 	const char *group_name = fslmc_vfio_get_group_name();
+	struct fslmc_dmaseg *dmaseg = NULL;
+	uint64_t phy = 0;
+
+	if (rte_eal_iova_mode() == RTE_IOVA_VA) {
+		if (vaddr != iovaddr) {
+			DPAA2_BUS_ERR("IOVA:VA(%" PRIx64 " : %" PRIx64 ") %s",
+				iovaddr, vaddr,
+				"should be 1:1 for VA mode");
+
+			return -EINVAL;
+		}
+	}
 
+	phy = rte_mem_virt2phy((const void *)(uintptr_t)vaddr);
+	if (phy == RTE_BAD_IOVA) {
+		phy = fslmc_io_virt2phy((const void *)(uintptr_t)vaddr);
+		if (phy == RTE_BAD_IOVA)
+			return -ENOMEM;
+		is_io = 1;
+	} else if (fslmc_mem_va2iova != RTE_BAD_IOVA &&
+		fslmc_mem_va2iova != (iovaddr - vaddr)) {
+		DPAA2_BUS_WARN("Multiple MEM PA<->VA conversions.");
+	}
+	DPAA2_BUS_DEBUG("%s(%zu): VA(%" PRIx64 "):IOVA(%" PRIx64 "):PHY(%" PRIx64 ")",
+		is_io ? "DMA IO map size" : "DMA MEM map size",
+		len, vaddr, iovaddr, phy);
+
+	if (is_io)
+		goto io_mapping_check;
+
+	TAILQ_FOREACH(dmaseg, &fslmc_memsegs, next) {
+		if (!((vaddr + len) <= dmaseg->vaddr ||
+			(dmaseg->vaddr + dmaseg->size) <= vaddr)) {
+			DPAA2_BUS_ERR("MEM: New VA Range(%" PRIx64 " ~ %" PRIx64 ")",
+				vaddr, vaddr + len);
+			DPAA2_BUS_ERR("MEM: Overlap with (%" PRIx64 " ~ %" PRIx64 ")",
+				dmaseg->vaddr,
+				dmaseg->vaddr + dmaseg->size);
+			return -EEXIST;
+		}
+		if (!((iovaddr + len) <= dmaseg->iova ||
+			(dmaseg->iova + dmaseg->size) <= iovaddr)) {
+			DPAA2_BUS_ERR("MEM: New IOVA Range(%" PRIx64 " ~ %" PRIx64 ")",
+				iovaddr, iovaddr + len);
+			DPAA2_BUS_ERR("MEM: Overlap with (%" PRIx64 " ~ %" PRIx64 ")",
+				dmaseg->iova,
+				dmaseg->iova + dmaseg->size);
+			return -EEXIST;
+		}
+	}
+	goto start_mapping;
+
+io_mapping_check:
+	TAILQ_FOREACH(dmaseg, &fslmc_iosegs, next) {
+		if (!((vaddr + len) <= dmaseg->vaddr ||
+			(dmaseg->vaddr + dmaseg->size) <= vaddr)) {
+			DPAA2_BUS_ERR("IO: New VA Range (%" PRIx64 " ~ %" PRIx64 ")",
+				vaddr, vaddr + len);
+			DPAA2_BUS_ERR("IO: Overlap with (%" PRIx64 " ~ %" PRIx64 ")",
+				dmaseg->vaddr,
+				dmaseg->vaddr + dmaseg->size);
+			return -EEXIST;
+		}
+		if (!((iovaddr + len) <= dmaseg->iova ||
+			(dmaseg->iova + dmaseg->size) <= iovaddr)) {
+			DPAA2_BUS_ERR("IO: New IOVA Range(%" PRIx64 " ~ %" PRIx64 ")",
+				iovaddr, iovaddr + len);
+			DPAA2_BUS_ERR("IO: Overlap with (%" PRIx64 " ~ %" PRIx64 ")",
+				dmaseg->iova,
+				dmaseg->iova + dmaseg->size);
+			return -EEXIST;
+		}
+	}
+
+start_mapping:
 	fd = fslmc_vfio_group_fd_by_name(group_name);
 	if (fd <= 0) {
-		DPAA2_BUS_ERR("%s failed to open group fd(%d)",
-			__func__, fd);
+		DPAA2_BUS_ERR("%s: Get fd by name(%s) failed(%d)",
+			__func__, group_name, fd);
 		if (fd < 0)
 			return fd;
-		return -rte_errno;
+		return -EIO;
 	}
 	if (fslmc_vfio_iommu_type(fd) == RTE_VFIO_NOIOMMU) {
 		DPAA2_BUS_DEBUG("Running in NOIOMMU mode");
-		return 0;
+		if (phy != iovaddr) {
+			DPAA2_BUS_ERR("IOVA should support with IOMMU");
+			return -EIO;
+		}
+		goto end_mapping;
 	}
 
 	dma_map.size = len;
 	dma_map.vaddr = vaddr;
 	dma_map.iova = iovaddr;
 
-#ifndef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
-	if (vaddr != iovaddr) {
-		DPAA2_BUS_WARN("vaddr(0x%lx) != iovaddr(0x%lx)",
-			vaddr, iovaddr);
-	}
-#endif
-
 	/* SET DMA MAP for IOMMU */
 	if (!fslmc_vfio_container_connected(fd)) {
-		DPAA2_BUS_ERR("Container is not connected ");
+		DPAA2_BUS_ERR("Container is not connected");
 		return -EIO;
 	}
 
-	DPAA2_BUS_DEBUG("--> Map address: 0x%"PRIx64", size: %"PRIu64"",
-			(uint64_t)dma_map.vaddr, (uint64_t)dma_map.size);
 	ret = ioctl(fslmc_vfio_container_fd(), VFIO_IOMMU_MAP_DMA,
 		&dma_map);
 	if (ret) {
-		DPAA2_BUS_ERR("VFIO_IOMMU_MAP_DMA API(errno = %d)",
-				errno);
+		DPAA2_BUS_ERR("%s(%d) VA(%" PRIx64 "):IOVA(%" PRIx64 "):PHY(%" PRIx64 ")",
+			is_io ? "DMA IO map err" : "DMA MEM map err",
+			errno, vaddr, iovaddr, phy);
 		return ret;
 	}
 
+end_mapping:
+	dmaseg = malloc(sizeof(struct fslmc_dmaseg));
+	if (!dmaseg) {
+		DPAA2_BUS_ERR("DMA segment malloc failed!");
+		return -ENOMEM;
+	}
+	dmaseg->vaddr = vaddr;
+	dmaseg->iova = iovaddr;
+	dmaseg->size = len;
+	if (is_io) {
+		TAILQ_INSERT_TAIL(&fslmc_iosegs, dmaseg, next);
+	} else {
+		fslmc_mem_map_num++;
+		if (fslmc_mem_map_num == 1)
+			fslmc_mem_va2iova = iovaddr - vaddr;
+		else
+			fslmc_mem_va2iova = RTE_BAD_IOVA;
+		TAILQ_INSERT_TAIL(&fslmc_memsegs, dmaseg, next);
+	}
+	DPAA2_BUS_LOG(NOTICE,
+		"%s(%zx): VA(%" PRIx64 "):IOVA(%" PRIx64 "):PHY(%" PRIx64 ")",
+		is_io ? "DMA I/O map size" : "DMA MEM map size",
+		len, vaddr, iovaddr, phy);
+
 	return 0;
 }
 
 static int
-fslmc_unmap_dma(uint64_t vaddr, uint64_t iovaddr __rte_unused, size_t len)
+fslmc_unmap_dma(uint64_t vaddr, uint64_t iovaddr, size_t len)
 {
 	struct vfio_iommu_type1_dma_unmap dma_unmap = {
 		.argsz = sizeof(struct vfio_iommu_type1_dma_unmap),
 		.flags = 0,
 	};
-	int ret, fd;
+	int ret, fd, is_io = 0;
 	const char *group_name = fslmc_vfio_get_group_name();
+	struct fslmc_dmaseg *dmaseg = NULL;
+
+	TAILQ_FOREACH(dmaseg, &fslmc_memsegs, next) {
+		if (((vaddr && dmaseg->vaddr == vaddr) || !vaddr) &&
+			dmaseg->iova == iovaddr &&
+			dmaseg->size == len) {
+			is_io = 0;
+			break;
+		}
+	}
+
+	if (!dmaseg) {
+		TAILQ_FOREACH(dmaseg, &fslmc_iosegs, next) {
+			if (((vaddr && dmaseg->vaddr == vaddr) || !vaddr) &&
+				dmaseg->iova == iovaddr &&
+				dmaseg->size == len) {
+				is_io = 1;
+				break;
+			}
+		}
+	}
+
+	if (!dmaseg) {
+		DPAA2_BUS_ERR("IOVA(%" PRIx64 ") with length(%zx) not mapped",
+			iovaddr, len);
+		return 0;
+	}
 
 	fd = fslmc_vfio_group_fd_by_name(group_name);
 	if (fd <= 0) {
-		DPAA2_BUS_ERR("%s failed to open group fd(%d)",
-			__func__, fd);
+		DPAA2_BUS_ERR("%s: Get fd by name(%s) failed(%d)",
+			__func__, group_name, fd);
 		if (fd < 0)
 			return fd;
-		return -rte_errno;
+		return -EIO;
 	}
 	if (fslmc_vfio_iommu_type(fd) == RTE_VFIO_NOIOMMU) {
 		DPAA2_BUS_DEBUG("Running in NOIOMMU mode");
@@ -796,7 +1010,7 @@ fslmc_unmap_dma(uint64_t vaddr, uint64_t iovaddr __rte_unused, size_t len)
 	}
 
 	dma_unmap.size = len;
-	dma_unmap.iova = vaddr;
+	dma_unmap.iova = iovaddr;
 
 	/* SET DMA MAP for IOMMU */
 	if (!fslmc_vfio_container_connected(fd)) {
@@ -804,19 +1018,162 @@ fslmc_unmap_dma(uint64_t vaddr, uint64_t iovaddr __rte_unused, size_t len)
 		return -EIO;
 	}
 
-	DPAA2_BUS_DEBUG("--> Unmap address: 0x%"PRIx64", size: %"PRIu64"",
-			(uint64_t)dma_unmap.iova, (uint64_t)dma_unmap.size);
 	ret = ioctl(fslmc_vfio_container_fd(), VFIO_IOMMU_UNMAP_DMA,
 		&dma_unmap);
 	if (ret) {
-		DPAA2_BUS_ERR("VFIO_IOMMU_UNMAP_DMA API(errno = %d)",
-				errno);
-		return -1;
+		DPAA2_BUS_ERR("DMA un-map IOVA(%" PRIx64 " ~ %" PRIx64 ") err(%d)",
+			iovaddr, iovaddr + len, errno);
+		return ret;
+	}
+
+	if (is_io) {
+		TAILQ_REMOVE(&fslmc_iosegs, dmaseg, next);
+	} else {
+		TAILQ_REMOVE(&fslmc_memsegs, dmaseg, next);
+		fslmc_mem_map_num--;
+		if (TAILQ_EMPTY(&fslmc_memsegs))
+			fslmc_mem_va2iova = RTE_BAD_IOVA;
 	}
 
+	free(dmaseg);
+
 	return 0;
 }
 
+uint64_t
+rte_fslmc_cold_mem_vaddr_to_iova(void *vaddr,
+	uint64_t size)
+{
+	struct fslmc_dmaseg *dmaseg;
+	uint64_t va;
+
+	va = (uint64_t)vaddr;
+	TAILQ_FOREACH(dmaseg, &fslmc_memsegs, next) {
+		if (va >= dmaseg->vaddr &&
+			(va + size) < (dmaseg->vaddr + dmaseg->size)) {
+			return dmaseg->iova + va - dmaseg->vaddr;
+		}
+	}
+
+	return RTE_BAD_IOVA;
+}
+
+void *
+rte_fslmc_cold_mem_iova_to_vaddr(uint64_t iova,
+	uint64_t size)
+{
+	struct fslmc_dmaseg *dmaseg;
+
+	TAILQ_FOREACH(dmaseg, &fslmc_memsegs, next) {
+		if (iova >= dmaseg->iova &&
+			(iova + size) < (dmaseg->iova + dmaseg->size))
+			return (void *)((uintptr_t)dmaseg->vaddr + (uintptr_t)(iova - dmaseg->iova));
+	}
+
+	return NULL;
+}
+
+__hot uint64_t
+rte_fslmc_mem_vaddr_to_iova(void *vaddr)
+{
+	if (likely(fslmc_mem_va2iova != RTE_BAD_IOVA))
+		return (uint64_t)vaddr + fslmc_mem_va2iova;
+
+	return rte_fslmc_cold_mem_vaddr_to_iova(vaddr, 0);
+}
+
+__hot void *
+rte_fslmc_mem_iova_to_vaddr(uint64_t iova)
+{
+	if (likely(fslmc_mem_va2iova != RTE_BAD_IOVA))
+		return (void *)((uintptr_t)iova - (uintptr_t)fslmc_mem_va2iova);
+
+	return rte_fslmc_cold_mem_iova_to_vaddr(iova, 0);
+}
+
+uint64_t
+rte_fslmc_io_vaddr_to_iova(void *vaddr)
+{
+	struct fslmc_dmaseg *dmaseg = NULL;
+	uint64_t va = (uint64_t)vaddr;
+
+	TAILQ_FOREACH(dmaseg, &fslmc_iosegs, next) {
+		if ((va >= dmaseg->vaddr) &&
+			va < dmaseg->vaddr + dmaseg->size)
+			return dmaseg->iova + va - dmaseg->vaddr;
+	}
+
+	return RTE_BAD_IOVA;
+}
+
+void *
+rte_fslmc_io_iova_to_vaddr(uint64_t iova)
+{
+	struct fslmc_dmaseg *dmaseg = NULL;
+
+	TAILQ_FOREACH(dmaseg, &fslmc_iosegs, next) {
+		if ((iova >= dmaseg->iova) &&
+			iova < dmaseg->iova + dmaseg->size)
+			return (void *)((uintptr_t)dmaseg->vaddr + (uintptr_t)(iova - dmaseg->iova));
+	}
+
+	return NULL;
+}
+
+static void
+fslmc_memevent_cb(enum rte_mem_event type, const void *addr,
+	size_t len, void *arg __rte_unused)
+{
+	struct rte_memseg_list *msl;
+	struct rte_memseg *ms;
+	size_t cur_len = 0, map_len = 0;
+	uint64_t virt_addr;
+	rte_iova_t iova_addr;
+	int ret;
+
+	msl = rte_mem_virt2memseg_list(addr);
+
+	while (cur_len < len) {
+		const void *va = RTE_PTR_ADD(addr, cur_len);
+
+		ms = rte_mem_virt2memseg(va, msl);
+		iova_addr = ms->iova;
+		virt_addr = ms->addr_64;
+		map_len = ms->len;
+
+		DPAA2_BUS_DEBUG("%s, va=%p, virt=%" PRIx64 ", iova=%" PRIx64 ", len=%zu",
+			type == RTE_MEM_EVENT_ALLOC ? "alloc" : "dealloc",
+			va, virt_addr, iova_addr, map_len);
+
+		/* iova_addr may be set to RTE_BAD_IOVA */
+		if (iova_addr == RTE_BAD_IOVA) {
+			DPAA2_BUS_DEBUG("Segment has invalid iova, skipping\n");
+			cur_len += map_len;
+			continue;
+		}
+
+		if (type == RTE_MEM_EVENT_ALLOC)
+			ret = fslmc_map_dma(virt_addr, iova_addr, map_len);
+		else
+			ret = fslmc_unmap_dma(virt_addr, iova_addr, map_len);
+
+		if (ret != 0) {
+			DPAA2_BUS_ERR("%s: Map=%d, addr=%p, len=%zu, err:(%d)",
+				type == RTE_MEM_EVENT_ALLOC ?
+				"DMA Mapping failed. " :
+				"DMA Unmapping failed. ",
+				type, va, map_len, ret);
+			return;
+		}
+
+		cur_len += map_len;
+	}
+
+	DPAA2_BUS_DEBUG("Total %s: addr=%p, len=%zu",
+		type == RTE_MEM_EVENT_ALLOC ? "Mapped" : "Unmapped",
+		addr, len);
+}
+
 static int
 fslmc_dmamap_seg(const struct rte_memseg_list *msl __rte_unused,
 		const struct rte_memseg *ms, void *arg)
@@ -848,7 +1205,7 @@ __rte_internal
 int
 rte_fslmc_vfio_mem_dmaunmap(uint64_t iova, uint64_t size)
 {
-	return fslmc_unmap_dma(iova, 0, size);
+	return fslmc_unmap_dma(0, iova, size);
 }
 
 int rte_fslmc_vfio_dmamap(void)
@@ -858,9 +1215,10 @@ int rte_fslmc_vfio_dmamap(void)
 	/* Lock before parsing and registering callback to memory subsystem */
 	rte_mcfg_mem_read_lock();
 
-	if (rte_memseg_walk(fslmc_dmamap_seg, &i) < 0) {
+	ret = rte_memseg_walk(fslmc_dmamap_seg, &i);
+	if (ret) {
 		rte_mcfg_mem_read_unlock();
-		return -1;
+		return ret;
 	}
 
 	ret = rte_mem_event_callback_register("fslmc_memevent_clb",
@@ -899,6 +1257,14 @@ fslmc_vfio_setup_device(const char *dev_addr,
 	const char *group_name = fslmc_vfio_get_group_name();
 
 	vfio_group_fd = fslmc_vfio_group_fd_by_name(group_name);
+	if (vfio_group_fd <= 0) {
+		DPAA2_BUS_ERR("%s: Get fd by name(%s) failed(%d)",
+			__func__, group_name, vfio_group_fd);
+		if (vfio_group_fd < 0)
+			return vfio_group_fd;
+		return -EIO;
+	}
+
 	if (!fslmc_vfio_container_connected(vfio_group_fd)) {
 		DPAA2_BUS_ERR("Container is not connected");
 		return -EIO;
@@ -1007,8 +1373,7 @@ int rte_dpaa2_intr_disable(struct rte_intr_handle *intr_handle, int index)
 	vfio_dev_fd = rte_intr_dev_fd_get(intr_handle);
 	ret = ioctl(vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
 	if (ret)
-		DPAA2_BUS_ERR(
-			"Error disabling dpaa2 interrupts for fd %d",
+		DPAA2_BUS_ERR("Error disabling dpaa2 interrupts for fd %d",
 			rte_intr_fd_get(intr_handle));
 
 	return ret;
@@ -1033,7 +1398,7 @@ rte_dpaa2_vfio_setup_intr(struct rte_intr_handle *intr_handle,
 		if (ret < 0) {
 			DPAA2_BUS_ERR("Cannot get IRQ(%d) info, error %i (%s)",
 				      i, errno, strerror(errno));
-			return -1;
+			return ret;
 		}
 
 		/* if this vector cannot be used with eventfd,
@@ -1047,8 +1412,8 @@ rte_dpaa2_vfio_setup_intr(struct rte_intr_handle *intr_handle,
 		fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
 		if (fd < 0) {
 			DPAA2_BUS_ERR("Cannot set up eventfd, error %i (%s)",
-				      errno, strerror(errno));
-			return -1;
+				errno, strerror(errno));
+			return fd;
 		}
 
 		if (rte_intr_fd_set(intr_handle, fd))
@@ -1064,7 +1429,7 @@ rte_dpaa2_vfio_setup_intr(struct rte_intr_handle *intr_handle,
 	}
 
 	/* if we're here, we haven't found a suitable interrupt vector */
-	return -1;
+	return -EIO;
 }
 
 static void
@@ -1238,6 +1603,13 @@ fslmc_vfio_close_group(void)
 	const char *group_name = fslmc_vfio_get_group_name();
 
 	vfio_group_fd = fslmc_vfio_group_fd_by_name(group_name);
+	if (vfio_group_fd <= 0) {
+		DPAA2_BUS_ERR("%s: Get fd by name(%s) failed(%d)",
+			__func__, group_name, vfio_group_fd);
+		if (vfio_group_fd < 0)
+			return vfio_group_fd;
+		return -EIO;
+	}
 
 	RTE_TAILQ_FOREACH_SAFE(dev, &rte_fslmc_bus.device_list, next, dev_temp) {
 		if (dev->device.devargs &&
@@ -1329,7 +1701,7 @@ fslmc_vfio_process_group(void)
 				ret = fslmc_process_mcp(dev);
 				if (ret) {
 					DPAA2_BUS_ERR("Unable to map MC Portal");
-					return -1;
+					return ret;
 				}
 				found_mportal = 1;
 			}
@@ -1346,7 +1718,7 @@ fslmc_vfio_process_group(void)
 	/* Cannot continue if there is not even a single mportal */
 	if (!found_mportal) {
 		DPAA2_BUS_ERR("No MC Portal device found. Not continuing");
-		return -1;
+		return -EIO;
 	}
 
 	/* Search for DPRC device next as it updates endpoint of
@@ -1358,7 +1730,7 @@ fslmc_vfio_process_group(void)
 			ret = fslmc_process_iodevices(dev);
 			if (ret) {
 				DPAA2_BUS_ERR("Unable to process dprc");
-				return -1;
+				return ret;
 			}
 			TAILQ_REMOVE(&rte_fslmc_bus.device_list, dev, next);
 		}
@@ -1415,7 +1787,7 @@ fslmc_vfio_process_group(void)
 			if (ret) {
 				DPAA2_BUS_DEBUG("Dev (%s) init failed",
 						dev->device.name);
-				return -1;
+				return ret;
 			}
 
 			break;
@@ -1439,7 +1811,7 @@ fslmc_vfio_process_group(void)
 			if (ret) {
 				DPAA2_BUS_DEBUG("Dev (%s) init failed",
 						dev->device.name);
-				return -1;
+				return ret;
 			}
 
 			break;
@@ -1468,9 +1840,9 @@ fslmc_vfio_setup_group(void)
 	vfio_container_fd = fslmc_vfio_container_fd();
 	if (vfio_container_fd <= 0) {
 		vfio_container_fd = fslmc_vfio_open_container_fd();
-		if (vfio_container_fd <= 0) {
+		if (vfio_container_fd < 0) {
 			DPAA2_BUS_ERR("Failed to create MC VFIO container");
-			return -rte_errno;
+			return vfio_container_fd;
 		}
 	}
 
@@ -1483,6 +1855,8 @@ fslmc_vfio_setup_group(void)
 	if (vfio_group_fd <= 0) {
 		vfio_group_fd = fslmc_vfio_open_group_fd(group_name);
 		if (vfio_group_fd <= 0) {
+			DPAA2_BUS_ERR("%s: open group name(%s) failed(%d)",
+				__func__, group_name, vfio_group_fd);
 			if (!vfio_group_fd)
 				close(vfio_group_fd);
 			DPAA2_BUS_ERR("Failed to create MC VFIO group");
diff --git a/drivers/bus/fslmc/fslmc_vfio.h b/drivers/bus/fslmc/fslmc_vfio.h
index 1695b6c078..408b35680d 100644
--- a/drivers/bus/fslmc/fslmc_vfio.h
+++ b/drivers/bus/fslmc/fslmc_vfio.h
@@ -11,6 +11,10 @@
 #include <rte_compat.h>
 #include <rte_vfio.h>
 
+#ifndef __hot
+#define __hot __attribute__((hot))
+#endif
+
 /* Pathname of FSL-MC devices directory. */
 #define SYSFS_FSL_MC_DEVICES	"/sys/bus/fsl-mc/devices"
 #define DPAA2_MC_DPNI_DEVID	7
diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c
index bc36607e64..85e4c16c03 100644
--- a/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c
+++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
  *   Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
- *   Copyright 2016,2020 NXP
+ *   Copyright 2016,2020-2023 NXP
  *
  */
 
@@ -28,7 +28,6 @@
 #include "portal/dpaa2_hw_pvt.h"
 #include "portal/dpaa2_hw_dpio.h"
 
-
 TAILQ_HEAD(dpbp_dev_list, dpaa2_dpbp_dev);
 static struct dpbp_dev_list dpbp_dev_list
 	= TAILQ_HEAD_INITIALIZER(dpbp_dev_list); /*!< DPBP device list */
diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
index 8265fee497..b52a8c8ba5 100644
--- a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
+++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
@@ -332,9 +332,8 @@ dpaa2_affine_qbman_swp(void)
 		}
 		RTE_PER_LCORE(_dpaa2_io).dpio_dev = dpio_dev;
 
-		DPAA2_BUS_INFO(
-			"DPAA Portal=%p (%d) is affined to thread %" PRIu64,
-			dpio_dev, dpio_dev->index, tid);
+		DPAA2_BUS_DEBUG("Portal[%d] is affined to thread %" PRIu64,
+			dpio_dev->index, tid);
 	}
 	return 0;
 }
@@ -354,9 +353,8 @@ dpaa2_affine_qbman_ethrx_swp(void)
 		}
 		RTE_PER_LCORE(_dpaa2_io).ethrx_dpio_dev = dpio_dev;
 
-		DPAA2_BUS_INFO(
-			"DPAA Portal=%p (%d) is affined for eth rx to thread %"
-			PRIu64, dpio_dev, dpio_dev->index, tid);
+		DPAA2_BUS_DEBUG("Portal_eth_rx[%d] is affined to thread %" PRIu64,
+			dpio_dev->index, tid);
 	}
 	return 0;
 }
diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.h b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.h
index 7407f8d38d..328e1e788a 100644
--- a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.h
+++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
  *   Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
- *   Copyright 2016-2019 NXP
+ *   Copyright 2016-2023 NXP
  *
  */
 
@@ -12,6 +12,7 @@
 #include <mc/fsl_mc_sys.h>
 
 #include <rte_compat.h>
+#include <dpaa2_hw_pvt.h>
 
 struct dpaa2_io_portal_t {
 	struct dpaa2_dpio_dev *dpio_dev;
diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h b/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h
index 169c7917ea..c5900bd06a 100644
--- a/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h
+++ b/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h
@@ -14,6 +14,7 @@
 
 #include <mc/fsl_mc_sys.h>
 #include <fsl_qbman_portal.h>
+#include <bus_fslmc_driver.h>
 
 #ifndef false
 #define false      0
@@ -80,6 +81,8 @@
 #define DPAA2_PACKET_LAYOUT_ALIGN	64 /*changing from 256 */
 
 #define DPAA2_DPCI_MAX_QUEUES 2
+#define DPAA2_INVALID_FLOW_ID 0xffff
+#define DPAA2_INVALID_CGID 0xff
 
 struct dpaa2_queue;
 
@@ -365,83 +368,63 @@ enum qbman_fd_format {
  */
 #define DPAA2_EQ_RESP_ALWAYS		1
 
-/* Various structures representing contiguous memory maps */
-struct dpaa2_memseg {
-	TAILQ_ENTRY(dpaa2_memseg) next;
-	char *vaddr;
-	rte_iova_t iova;
-	size_t len;
-};
-
-#ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
-extern uint8_t dpaa2_virt_mode;
-static void *dpaa2_mem_ptov(phys_addr_t paddr) __rte_unused;
-
-static void *dpaa2_mem_ptov(phys_addr_t paddr)
+static inline uint64_t
+dpaa2_mem_va_to_iova(void *va)
 {
-	void *va;
-
-	if (dpaa2_virt_mode)
-		return (void *)(size_t)paddr;
-
-	va = (void *)dpaax_iova_table_get_va(paddr);
-	if (likely(va != NULL))
-		return va;
-
-	/* If not, Fallback to full memseg list searching */
-	va = rte_mem_iova2virt(paddr);
+	if (likely(rte_eal_iova_mode() == RTE_IOVA_VA))
+		return (uint64_t)va;
 
-	return va;
+	return rte_fslmc_mem_vaddr_to_iova(va);
 }
 
-static phys_addr_t dpaa2_mem_vtop(uint64_t vaddr) __rte_unused;
-
-static phys_addr_t dpaa2_mem_vtop(uint64_t vaddr)
+static inline void *
+dpaa2_mem_iova_to_va(uint64_t iova)
 {
-	const struct rte_memseg *memseg;
-
-	if (dpaa2_virt_mode)
-		return vaddr;
+	if (likely(rte_eal_iova_mode() == RTE_IOVA_VA))
+		return (void *)(uintptr_t)iova;
 
-	memseg = rte_mem_virt2memseg((void *)(uintptr_t)vaddr, NULL);
-	if (memseg)
-		return memseg->iova + RTE_PTR_DIFF(vaddr, memseg->addr);
-	return (size_t)NULL;
+	return rte_fslmc_mem_iova_to_vaddr(iova);
 }
 
-/**
- * When we are using Physical addresses as IO Virtual Addresses,
- * Need to call conversion routines dpaa2_mem_vtop & dpaa2_mem_ptov
- * wherever required.
- * These routines are called with help of below MACRO's
- */
-
 #define DPAA2_MBUF_VADDR_TO_IOVA(mbuf) ((mbuf)->buf_iova)
-
-/**
- * macro to convert Virtual address to IOVA
- */
-#define DPAA2_VADDR_TO_IOVA(_vaddr) dpaa2_mem_vtop((size_t)(_vaddr))
-
-/**
- * macro to convert IOVA to Virtual address
- */
-#define DPAA2_IOVA_TO_VADDR(_iova) dpaa2_mem_ptov((size_t)(_iova))
-
-/**
- * macro to convert modify the memory containing IOVA to Virtual address
- */
+#define DPAA2_VADDR_TO_IOVA(_vaddr) \
+	dpaa2_mem_va_to_iova((void *)(uintptr_t)_vaddr)
+#define DPAA2_IOVA_TO_VADDR(_iova) \
+	dpaa2_mem_iova_to_va((uint64_t)_iova)
 #define DPAA2_MODIFY_IOVA_TO_VADDR(_mem, _type) \
-	{_mem = (_type)(dpaa2_mem_ptov((size_t)(_mem))); }
+	{_mem = (_type)DPAA2_IOVA_TO_VADDR(_mem); }
+
+#define DPAA2_VAMODE_VADDR_TO_IOVA(_vaddr) ((uint64_t)_vaddr)
+#define DPAA2_VAMODE_IOVA_TO_VADDR(_iova) ((void *)_iova)
+#define DPAA2_VAMODE_MODIFY_IOVA_TO_VADDR(_mem, _type) \
+	{_mem = (_type)(_mem); }
+
+#define DPAA2_PAMODE_VADDR_TO_IOVA(_vaddr) \
+	rte_fslmc_mem_vaddr_to_iova((void *)_vaddr)
+#define DPAA2_PAMODE_IOVA_TO_VADDR(_iova) \
+	rte_fslmc_mem_iova_to_vaddr((uint64_t)_iova)
+#define DPAA2_PAMODE_MODIFY_IOVA_TO_VADDR(_mem, _type) \
+	{_mem = (_type)rte_fslmc_mem_iova_to_vaddr(_mem); }
+
+static inline uint64_t
+dpaa2_mem_va_to_iova_check(void *va, uint64_t size)
+{
+	uint64_t iova = rte_fslmc_cold_mem_vaddr_to_iova(va, size);
 
-#else	/* RTE_LIBRTE_DPAA2_USE_PHYS_IOVA */
+	if (iova == RTE_BAD_IOVA)
+		return RTE_BAD_IOVA;
 
-#define DPAA2_MBUF_VADDR_TO_IOVA(mbuf) ((mbuf)->buf_addr)
-#define DPAA2_VADDR_TO_IOVA(_vaddr) (phys_addr_t)(_vaddr)
-#define DPAA2_IOVA_TO_VADDR(_iova) (void *)(_iova)
-#define DPAA2_MODIFY_IOVA_TO_VADDR(_mem, _type)
+	/** Double check the iova is valid.*/
+	if (iova != rte_mem_virt2iova(va))
+		return RTE_BAD_IOVA;
+
+	return iova;
+}
 
-#endif /* RTE_LIBRTE_DPAA2_USE_PHYS_IOVA */
+#define DPAA2_VADDR_TO_IOVA_AND_CHECK(_vaddr, size) \
+	dpaa2_mem_va_to_iova_check(_vaddr, size)
+#define DPAA2_IOVA_TO_VADDR_AND_CHECK(_iova, size) \
+	rte_fslmc_cold_mem_iova_to_vaddr(_iova, size)
 
 static inline
 int check_swp_active_dqs(uint16_t dpio_index)
diff --git a/drivers/bus/fslmc/version.map b/drivers/bus/fslmc/version.map
index b49bc0a62c..2c36895285 100644
--- a/drivers/bus/fslmc/version.map
+++ b/drivers/bus/fslmc/version.map
@@ -24,7 +24,6 @@ INTERNAL {
 	dpaa2_seqn_dynfield_offset;
 	dpaa2_seqn;
 	dpaa2_svr_family;
-	dpaa2_virt_mode;
 	dpbp_disable;
 	dpbp_enable;
 	dpbp_get_attributes;
@@ -119,6 +118,12 @@ INTERNAL {
 	rte_fslmc_object_register;
 	rte_global_active_dqs_list;
 	rte_fslmc_vfio_mem_dmaunmap;
+	rte_fslmc_cold_mem_vaddr_to_iova;
+	rte_fslmc_cold_mem_iova_to_vaddr;
+	rte_fslmc_mem_vaddr_to_iova;
+	rte_fslmc_mem_iova_to_vaddr;
+	rte_fslmc_io_vaddr_to_iova;
+	rte_fslmc_io_iova_to_vaddr;
 
 	local: *;
 };
diff --git a/drivers/dma/dpaa2/dpaa2_qdma.c b/drivers/dma/dpaa2/dpaa2_qdma.c
index 2c91ceec13..99b8881c5d 100644
--- a/drivers/dma/dpaa2/dpaa2_qdma.c
+++ b/drivers/dma/dpaa2/dpaa2_qdma.c
@@ -10,6 +10,7 @@
 
 #include <mc/fsl_dpdmai.h>
 
+#include <dpaa2_hw_dpio.h>
 #include "rte_pmd_dpaa2_qdma.h"
 #include "dpaa2_qdma.h"
 #include "dpaa2_qdma_logs.h"
-- 
2.25.1