[PATCH] bus/pci: fix secondary process PCI uio resource map problem

[PATCH] bus/pci: fix secondary process PCI uio resource map problem

[Chaoyong He]

From: Zerun Fu <zerun.fu@corigine.com>

For the primary process, the logic loops all BARs and will skip
the map of BAR with an invalid physical address (0), also will
assign 'uio_res->nb_maps' with the real mapped BARs number. But
for the secondary process, instead of loops all BARs, the logic
using the 'uio_res->nb_map' as index. If the device uses continuous
BARs there will be no problem, whereas if it uses discrete BARs,
it will lead to mapping errors.

Fix this problem by also loops all BARs and skip the map of BAR
with an invalid physical address in secondary process.

Fixes: 9b957f378abf ("pci: merge uio functions for linux and bsd")
Cc: mukawa@igel.co.jp
Cc: stable@dpdk.org

Signed-off-by: Zerun Fu <zerun.fu@corigine.com>
Reviewed-by: Chaoyong He <chaoyong.he@corigine.com>
Reviewed-by: Long Wu <long.wu@corigine.com>
Reviewed-by: Peng Zhang <peng.zhang@corigine.com>
---
 drivers/bus/pci/pci_common_uio.c | 94 ++++++++++++++++++++------------
 1 file changed, 60 insertions(+), 34 deletions(-)

diff --git a/drivers/bus/pci/pci_common_uio.c b/drivers/bus/pci/pci_common_uio.c
index 76c661f054..fcd8a49daf 100644
--- a/drivers/bus/pci/pci_common_uio.c
+++ b/drivers/bus/pci/pci_common_uio.c
@@ -23,10 +23,57 @@ static struct rte_tailq_elem rte_uio_tailq = {
 };
 EAL_REGISTER_TAILQ(rte_uio_tailq)
 
+static int
+pci_uio_map_secondary_resource_by_index(struct rte_pci_device *dev,
+		int res_idx, struct mapped_pci_resource *uio_res, int map_idx)
+{
+	int fd, i;
+
+	if (map_idx >= uio_res->nb_maps)
+		return -1;
+
+	/*
+	 * open devname, to mmap it
+	 */
+	fd = open(uio_res->maps[map_idx].path, O_RDWR);
+	if (fd < 0) {
+		RTE_LOG(ERR, EAL, "Cannot open %s: %s\n",
+			uio_res->maps[map_idx].path, strerror(errno));
+		return -1;
+	}
+
+	void *mapaddr = pci_map_resource(uio_res->maps[map_idx].addr,
+			fd, (off_t)uio_res->maps[map_idx].offset,
+			(size_t)uio_res->maps[map_idx].size, 0);
+
+	/* fd is not needed in secondary process, close it */
+	close(fd);
+	if (mapaddr != uio_res->maps[map_idx].addr) {
+		RTE_LOG(ERR, EAL,
+			"Cannot mmap device resource file %s to address: %p\n",
+			uio_res->maps[map_idx].path,
+			uio_res->maps[map_idx].addr);
+		if (mapaddr != NULL) {
+			/* unmap addrs correctly mapped */
+			for (i = 0; i < map_idx; i++)
+				pci_unmap_resource(
+					uio_res->maps[i].addr,
+					(size_t)uio_res->maps[i].size);
+			/* unmap addr wrongly mapped */
+			pci_unmap_resource(mapaddr,
+				(size_t)uio_res->maps[map_idx].size);
+		}
+		return -1;
+	}
+	dev->mem_resource[res_idx].addr = mapaddr;
+
+	return 0;
+}
+
 static int
 pci_uio_map_secondary(struct rte_pci_device *dev)
 {
-	int fd, i, j;
+	int map_idx = 0, res_idx, ret;
 	struct mapped_pci_resource *uio_res;
 	struct mapped_pci_res_list *uio_res_list =
 			RTE_TAILQ_CAST(rte_uio_tailq.head, mapped_pci_res_list);
@@ -37,41 +84,20 @@ pci_uio_map_secondary(struct rte_pci_device *dev)
 		if (rte_pci_addr_cmp(&uio_res->pci_addr, &dev->addr))
 			continue;
 
-		for (i = 0; i != uio_res->nb_maps; i++) {
-			/*
-			 * open devname, to mmap it
-			 */
-			fd = open(uio_res->maps[i].path, O_RDWR);
-			if (fd < 0) {
-				RTE_LOG(ERR, EAL, "Cannot open %s: %s\n",
-					uio_res->maps[i].path, strerror(errno));
-				return -1;
+		/* Map all BARs */
+		for (res_idx = 0; res_idx != PCI_MAX_RESOURCE; res_idx++) {
+			 /* skip empty BAR */
+			if (dev->mem_resource[res_idx].phys_addr == 0)
+				continue;
+
+			ret = pci_uio_map_secondary_resource_by_index(dev, res_idx,
+					uio_res, map_idx);
+			if (ret < 0) {
+				RTE_LOG(ERR, EAL, "Failed to map resources\n");
+				return ret;
 			}
 
-			void *mapaddr = pci_map_resource(uio_res->maps[i].addr,
-					fd, (off_t)uio_res->maps[i].offset,
-					(size_t)uio_res->maps[i].size, 0);
-
-			/* fd is not needed in secondary process, close it */
-			close(fd);
-			if (mapaddr != uio_res->maps[i].addr) {
-				RTE_LOG(ERR, EAL,
-					"Cannot mmap device resource file %s to address: %p\n",
-					uio_res->maps[i].path,
-					uio_res->maps[i].addr);
-				if (mapaddr != NULL) {
-					/* unmap addrs correctly mapped */
-					for (j = 0; j < i; j++)
-						pci_unmap_resource(
-							uio_res->maps[j].addr,
-							(size_t)uio_res->maps[j].size);
-					/* unmap addr wrongly mapped */
-					pci_unmap_resource(mapaddr,
-						(size_t)uio_res->maps[i].size);
-				}
-				return -1;
-			}
-			dev->mem_resource[i].addr = mapaddr;
+			map_idx++;
 		}
 		return 0;
 	}
-- 
2.39.1

[PATCH v2 1/2] bus/pci: fix secondary process PCI uio resource map problem

[Chaoyong He]

From: Zerun Fu <zerun.fu@corigine.com>

For the primary process, the logic loops all BARs and will skip
the map of BAR with an invalid physical address (0), also will
assign 'uio_res->nb_maps' with the real mapped BARs number. But
for the secondary process, instead of loops all BARs, the logic
using the 'uio_res->nb_map' as index. If the device uses continuous
BARs there will be no problem, whereas if it uses discrete BARs,
it will lead to mapping errors.

Fix this problem by also loops all BARs and skip the map of BAR
with an invalid physical address in secondary process.

Fixes: 9b957f378abf ("pci: merge uio functions for linux and bsd")
Cc: mukawa@igel.co.jp
Cc: stable@dpdk.org

Signed-off-by: Zerun Fu <zerun.fu@corigine.com>
Reviewed-by: Chaoyong He <chaoyong.he@corigine.com>
Reviewed-by: Long Wu <long.wu@corigine.com>
Reviewed-by: Peng Zhang <peng.zhang@corigine.com>
---
 drivers/bus/pci/pci_common_uio.c | 94 ++++++++++++++++++++------------
 1 file changed, 60 insertions(+), 34 deletions(-)

diff --git a/drivers/bus/pci/pci_common_uio.c b/drivers/bus/pci/pci_common_uio.c
index 76c661f054..fcd8a49daf 100644
--- a/drivers/bus/pci/pci_common_uio.c
+++ b/drivers/bus/pci/pci_common_uio.c
@@ -23,10 +23,57 @@ static struct rte_tailq_elem rte_uio_tailq = {
 };
 EAL_REGISTER_TAILQ(rte_uio_tailq)
 
+static int
+pci_uio_map_secondary_resource_by_index(struct rte_pci_device *dev,
+		int res_idx, struct mapped_pci_resource *uio_res, int map_idx)
+{
+	int fd, i;
+
+	if (map_idx >= uio_res->nb_maps)
+		return -1;
+
+	/*
+	 * open devname, to mmap it
+	 */
+	fd = open(uio_res->maps[map_idx].path, O_RDWR);
+	if (fd < 0) {
+		RTE_LOG(ERR, EAL, "Cannot open %s: %s\n",
+			uio_res->maps[map_idx].path, strerror(errno));
+		return -1;
+	}
+
+	void *mapaddr = pci_map_resource(uio_res->maps[map_idx].addr,
+			fd, (off_t)uio_res->maps[map_idx].offset,
+			(size_t)uio_res->maps[map_idx].size, 0);
+
+	/* fd is not needed in secondary process, close it */
+	close(fd);
+	if (mapaddr != uio_res->maps[map_idx].addr) {
+		RTE_LOG(ERR, EAL,
+			"Cannot mmap device resource file %s to address: %p\n",
+			uio_res->maps[map_idx].path,
+			uio_res->maps[map_idx].addr);
+		if (mapaddr != NULL) {
+			/* unmap addrs correctly mapped */
+			for (i = 0; i < map_idx; i++)
+				pci_unmap_resource(
+					uio_res->maps[i].addr,
+					(size_t)uio_res->maps[i].size);
+			/* unmap addr wrongly mapped */
+			pci_unmap_resource(mapaddr,
+				(size_t)uio_res->maps[map_idx].size);
+		}
+		return -1;
+	}
+	dev->mem_resource[res_idx].addr = mapaddr;
+
+	return 0;
+}
+
 static int
 pci_uio_map_secondary(struct rte_pci_device *dev)
 {
-	int fd, i, j;
+	int map_idx = 0, res_idx, ret;
 	struct mapped_pci_resource *uio_res;
 	struct mapped_pci_res_list *uio_res_list =
 			RTE_TAILQ_CAST(rte_uio_tailq.head, mapped_pci_res_list);
@@ -37,41 +84,20 @@ pci_uio_map_secondary(struct rte_pci_device *dev)
 		if (rte_pci_addr_cmp(&uio_res->pci_addr, &dev->addr))
 			continue;
 
-		for (i = 0; i != uio_res->nb_maps; i++) {
-			/*
-			 * open devname, to mmap it
-			 */
-			fd = open(uio_res->maps[i].path, O_RDWR);
-			if (fd < 0) {
-				RTE_LOG(ERR, EAL, "Cannot open %s: %s\n",
-					uio_res->maps[i].path, strerror(errno));
-				return -1;
+		/* Map all BARs */
+		for (res_idx = 0; res_idx != PCI_MAX_RESOURCE; res_idx++) {
+			 /* skip empty BAR */
+			if (dev->mem_resource[res_idx].phys_addr == 0)
+				continue;
+
+			ret = pci_uio_map_secondary_resource_by_index(dev, res_idx,
+					uio_res, map_idx);
+			if (ret < 0) {
+				RTE_LOG(ERR, EAL, "Failed to map resources\n");
+				return ret;
 			}
 
-			void *mapaddr = pci_map_resource(uio_res->maps[i].addr,
-					fd, (off_t)uio_res->maps[i].offset,
-					(size_t)uio_res->maps[i].size, 0);
-
-			/* fd is not needed in secondary process, close it */
-			close(fd);
-			if (mapaddr != uio_res->maps[i].addr) {
-				RTE_LOG(ERR, EAL,
-					"Cannot mmap device resource file %s to address: %p\n",
-					uio_res->maps[i].path,
-					uio_res->maps[i].addr);
-				if (mapaddr != NULL) {
-					/* unmap addrs correctly mapped */
-					for (j = 0; j < i; j++)
-						pci_unmap_resource(
-							uio_res->maps[j].addr,
-							(size_t)uio_res->maps[j].size);
-					/* unmap addr wrongly mapped */
-					pci_unmap_resource(mapaddr,
-						(size_t)uio_res->maps[i].size);
-				}
-				return -1;
-			}
-			dev->mem_resource[i].addr = mapaddr;
+			map_idx++;
 		}
 		return 0;
 	}
-- 
2.39.1

[PATCH v2 2/2] bus/pci: fix secondary process save 'FD' problem

[Chaoyong He]

From: Zerun Fu <zerun.fu@corigine.com>

In the previous logic the 'fd' was only saved in the primary process,
but for some devices this value is also used in the secondary logic.

For example, the call of 'rte_pci_find_ext_capability()' will fail in
the secondary process.

Fix this problem by getting and saving the value of 'fd' also in the
secondary process logic.

Fixes: 9b957f378abf ("pci: merge uio functions for linux and bsd")
Cc: mukawa@igel.co.jp
Cc: stable@dpdk.org

Signed-off-by: Zerun Fu <zerun.fu@corigine.com>
Reviewed-by: Chaoyong He <chaoyong.he@corigine.com>
Reviewed-by: Long Wu <long.wu@corigine.com>
Reviewed-by: Peng Zhang <peng.zhang@corigine.com>
---
 drivers/bus/pci/linux/pci_uio.c  | 5 ++++-
 drivers/bus/pci/pci_common_uio.c | 8 ++++----
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/drivers/bus/pci/linux/pci_uio.c b/drivers/bus/pci/linux/pci_uio.c
index 97d740dfe5..6680e42efb 100644
--- a/drivers/bus/pci/linux/pci_uio.c
+++ b/drivers/bus/pci/linux/pci_uio.c
@@ -237,7 +237,7 @@ pci_uio_alloc_resource(struct rte_pci_device *dev,
 	}
 	snprintf(devname, sizeof(devname), "/dev/uio%u", uio_num);
 
-	/* save fd if in primary process */
+	/* save fd */
 	fd = open(devname, O_RDWR);
 	if (fd < 0) {
 		RTE_LOG(ERR, EAL, "Cannot open %s: %s\n",
@@ -261,6 +261,9 @@ pci_uio_alloc_resource(struct rte_pci_device *dev,
 	if (rte_intr_dev_fd_set(dev->intr_handle, uio_cfg_fd))
 		goto error;
 
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+		return 0;
+
 	if (dev->kdrv == RTE_PCI_KDRV_IGB_UIO) {
 		if (rte_intr_type_set(dev->intr_handle, RTE_INTR_HANDLE_UIO))
 			goto error;
diff --git a/drivers/bus/pci/pci_common_uio.c b/drivers/bus/pci/pci_common_uio.c
index fcd8a49daf..b6f79b067d 100644
--- a/drivers/bus/pci/pci_common_uio.c
+++ b/drivers/bus/pci/pci_common_uio.c
@@ -122,15 +122,15 @@ pci_uio_map_resource(struct rte_pci_device *dev)
 	if (rte_intr_dev_fd_set(dev->intr_handle, -1))
 		return -1;
 
-	/* secondary processes - use already recorded details */
-	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
-		return pci_uio_map_secondary(dev);
-
 	/* allocate uio resource */
 	ret = pci_uio_alloc_resource(dev, &uio_res);
 	if (ret)
 		return ret;
 
+	/* secondary processes - use already recorded details */
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+		return pci_uio_map_secondary(dev);
+
 	/* Map all BARs */
 	for (i = 0; i != PCI_MAX_RESOURCE; i++) {
 		/* skip empty BAR */
-- 
2.39.1

Re: [PATCH v2 1/2] bus/pci: fix secondary process PCI uio resource map problem

[fengchengwen]

Hi,

Nice to see this fix.
ps: our team also founds this problem, and the bugfix is still being reviewed internally.

As for this patch, I prefer not extract subfunction, because the father is simple and just
need do some minor adjustment.

		map_idx = 0;
		for (i = 0; i != PCI_MAX_RESOURCE; i++) {
			/* skip empty BAR */
			if (dev->mem_resource[i].phys_addr == 0)
				continue;
			... // use map_idx instead i
			map_idx++;
			dev->mem_resource[i].addr = mapaddr;
		}

Also suggest use pci_uio_find_resource() to refactor pci_uio_map_secondary() function.

Thanks

On 2024/1/29 17:22, Chaoyong He wrote:
> From: Zerun Fu <zerun.fu@corigine.com>
> 
> For the primary process, the logic loops all BARs and will skip
> the map of BAR with an invalid physical address (0), also will
> assign 'uio_res->nb_maps' with the real mapped BARs number. But
> for the secondary process, instead of loops all BARs, the logic
> using the 'uio_res->nb_map' as index. If the device uses continuous
> BARs there will be no problem, whereas if it uses discrete BARs,
> it will lead to mapping errors.
> 
> Fix this problem by also loops all BARs and skip the map of BAR
> with an invalid physical address in secondary process.
> 
> Fixes: 9b957f378abf ("pci: merge uio functions for linux and bsd")
> Cc: mukawa@igel.co.jp
> Cc: stable@dpdk.org
> 
> Signed-off-by: Zerun Fu <zerun.fu@corigine.com>
> Reviewed-by: Chaoyong He <chaoyong.he@corigine.com>
> Reviewed-by: Long Wu <long.wu@corigine.com>
> Reviewed-by: Peng Zhang <peng.zhang@corigine.com>
> ---
>  drivers/bus/pci/pci_common_uio.c | 94 ++++++++++++++++++++------------
>  1 file changed, 60 insertions(+), 34 deletions(-)
> 
> diff --git a/drivers/bus/pci/pci_common_uio.c b/drivers/bus/pci/pci_common_uio.c
> index 76c661f054..fcd8a49daf 100644
> --- a/drivers/bus/pci/pci_common_uio.c
> +++ b/drivers/bus/pci/pci_common_uio.c
> @@ -23,10 +23,57 @@ static struct rte_tailq_elem rte_uio_tailq = {
>  };
>  EAL_REGISTER_TAILQ(rte_uio_tailq)
>  
> +static int
> +pci_uio_map_secondary_resource_by_index(struct rte_pci_device *dev,
> +		int res_idx, struct mapped_pci_resource *uio_res, int map_idx)
> +{
> +	int fd, i;
> +
> +	if (map_idx >= uio_res->nb_maps)
> +		return -1;
> +
> +	/*
> +	 * open devname, to mmap it
> +	 */
> +	fd = open(uio_res->maps[map_idx].path, O_RDWR);
> +	if (fd < 0) {
> +		RTE_LOG(ERR, EAL, "Cannot open %s: %s\n",
> +			uio_res->maps[map_idx].path, strerror(errno));
> +		return -1;
> +	}
> +
> +	void *mapaddr = pci_map_resource(uio_res->maps[map_idx].addr,
> +			fd, (off_t)uio_res->maps[map_idx].offset,
> +			(size_t)uio_res->maps[map_idx].size, 0);
> +
> +	/* fd is not needed in secondary process, close it */
> +	close(fd);
> +	if (mapaddr != uio_res->maps[map_idx].addr) {
> +		RTE_LOG(ERR, EAL,
> +			"Cannot mmap device resource file %s to address: %p\n",
> +			uio_res->maps[map_idx].path,
> +			uio_res->maps[map_idx].addr);
> +		if (mapaddr != NULL) {
> +			/* unmap addrs correctly mapped */
> +			for (i = 0; i < map_idx; i++)
> +				pci_unmap_resource(
> +					uio_res->maps[i].addr,
> +					(size_t)uio_res->maps[i].size);
> +			/* unmap addr wrongly mapped */
> +			pci_unmap_resource(mapaddr,
> +				(size_t)uio_res->maps[map_idx].size);
> +		}
> +		return -1;
> +	}
> +	dev->mem_resource[res_idx].addr = mapaddr;
> +
> +	return 0;
> +}
> +
>  static int
>  pci_uio_map_secondary(struct rte_pci_device *dev)
>  {
> -	int fd, i, j;
> +	int map_idx = 0, res_idx, ret;
>  	struct mapped_pci_resource *uio_res;
>  	struct mapped_pci_res_list *uio_res_list =
>  			RTE_TAILQ_CAST(rte_uio_tailq.head, mapped_pci_res_list);
> @@ -37,41 +84,20 @@ pci_uio_map_secondary(struct rte_pci_device *dev)
>  		if (rte_pci_addr_cmp(&uio_res->pci_addr, &dev->addr))
>  			continue;
>  
> -		for (i = 0; i != uio_res->nb_maps; i++) {
> -			/*
> -			 * open devname, to mmap it
> -			 */
> -			fd = open(uio_res->maps[i].path, O_RDWR);
> -			if (fd < 0) {
> -				RTE_LOG(ERR, EAL, "Cannot open %s: %s\n",
> -					uio_res->maps[i].path, strerror(errno));
> -				return -1;
> +		/* Map all BARs */
> +		for (res_idx = 0; res_idx != PCI_MAX_RESOURCE; res_idx++) {
> +			 /* skip empty BAR */
> +			if (dev->mem_resource[res_idx].phys_addr == 0)
> +				continue;
> +
> +			ret = pci_uio_map_secondary_resource_by_index(dev, res_idx,
> +					uio_res, map_idx);
> +			if (ret < 0) {
> +				RTE_LOG(ERR, EAL, "Failed to map resources\n");
> +				return ret;
>  			}
>  
> -			void *mapaddr = pci_map_resource(uio_res->maps[i].addr,
> -					fd, (off_t)uio_res->maps[i].offset,
> -					(size_t)uio_res->maps[i].size, 0);
> -
> -			/* fd is not needed in secondary process, close it */
> -			close(fd);
> -			if (mapaddr != uio_res->maps[i].addr) {
> -				RTE_LOG(ERR, EAL,
> -					"Cannot mmap device resource file %s to address: %p\n",
> -					uio_res->maps[i].path,
> -					uio_res->maps[i].addr);
> -				if (mapaddr != NULL) {
> -					/* unmap addrs correctly mapped */
> -					for (j = 0; j < i; j++)
> -						pci_unmap_resource(
> -							uio_res->maps[j].addr,
> -							(size_t)uio_res->maps[j].size);
> -					/* unmap addr wrongly mapped */
> -					pci_unmap_resource(mapaddr,
> -						(size_t)uio_res->maps[i].size);
> -				}
> -				return -1;
> -			}
> -			dev->mem_resource[i].addr = mapaddr;
> +			map_idx++;
>  		}
>  		return 0;
>  	}
> 

Re: [PATCH v2 2/2] bus/pci: fix secondary process save 'FD' problem

[fengchengwen]

Hi,

The pci_uio_alloc_resource in drivers/bus/pci/bsd should also modified.

How about add subfunction: pci_uio_init_intr() which includes access/rte_intr_fd_set/rte_intr_fd_get/rte_intr_type_set?

Thanks

On 2024/1/29 17:22, Chaoyong He wrote:
> From: Zerun Fu <zerun.fu@corigine.com>
> 
> In the previous logic the 'fd' was only saved in the primary process,
> but for some devices this value is also used in the secondary logic.
> 
> For example, the call of 'rte_pci_find_ext_capability()' will fail in
> the secondary process.
> 
> Fix this problem by getting and saving the value of 'fd' also in the
> secondary process logic.
> 
> Fixes: 9b957f378abf ("pci: merge uio functions for linux and bsd")
> Cc: mukawa@igel.co.jp
> Cc: stable@dpdk.org
> 
> Signed-off-by: Zerun Fu <zerun.fu@corigine.com>
> Reviewed-by: Chaoyong He <chaoyong.he@corigine.com>
> Reviewed-by: Long Wu <long.wu@corigine.com>
> Reviewed-by: Peng Zhang <peng.zhang@corigine.com>
> ---
>  drivers/bus/pci/linux/pci_uio.c  | 5 ++++-
>  drivers/bus/pci/pci_common_uio.c | 8 ++++----
>  2 files changed, 8 insertions(+), 5 deletions(-)
> 
> diff --git a/drivers/bus/pci/linux/pci_uio.c b/drivers/bus/pci/linux/pci_uio.c
> index 97d740dfe5..6680e42efb 100644
> --- a/drivers/bus/pci/linux/pci_uio.c
> +++ b/drivers/bus/pci/linux/pci_uio.c
> @@ -237,7 +237,7 @@ pci_uio_alloc_resource(struct rte_pci_device *dev,
>  	}
>  	snprintf(devname, sizeof(devname), "/dev/uio%u", uio_num);
>  
> -	/* save fd if in primary process */
> +	/* save fd */
>  	fd = open(devname, O_RDWR);
>  	if (fd < 0) {
>  		RTE_LOG(ERR, EAL, "Cannot open %s: %s\n",
> @@ -261,6 +261,9 @@ pci_uio_alloc_resource(struct rte_pci_device *dev,
>  	if (rte_intr_dev_fd_set(dev->intr_handle, uio_cfg_fd))
>  		goto error;
>  
> +	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
> +		return 0;
> +
>  	if (dev->kdrv == RTE_PCI_KDRV_IGB_UIO) {
>  		if (rte_intr_type_set(dev->intr_handle, RTE_INTR_HANDLE_UIO))
>  			goto error;
> diff --git a/drivers/bus/pci/pci_common_uio.c b/drivers/bus/pci/pci_common_uio.c
> index fcd8a49daf..b6f79b067d 100644
> --- a/drivers/bus/pci/pci_common_uio.c
> +++ b/drivers/bus/pci/pci_common_uio.c
> @@ -122,15 +122,15 @@ pci_uio_map_resource(struct rte_pci_device *dev)
>  	if (rte_intr_dev_fd_set(dev->intr_handle, -1))
>  		return -1;
>  
> -	/* secondary processes - use already recorded details */
> -	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
> -		return pci_uio_map_secondary(dev);
> -
>  	/* allocate uio resource */
>  	ret = pci_uio_alloc_resource(dev, &uio_res);
>  	if (ret)
>  		return ret;
>  
> +	/* secondary processes - use already recorded details */
> +	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
> +		return pci_uio_map_secondary(dev);
> +
>  	/* Map all BARs */
>  	for (i = 0; i != PCI_MAX_RESOURCE; i++) {
>  		/* skip empty BAR */
> 

Re: [PATCH v2 1/2] bus/pci: fix secondary process PCI uio resource map problem

[huangdengdui]

On 2024/1/29 17:22, Chaoyong He wrote:
> From: Zerun Fu <zerun.fu@corigine.com>
> 
> For the primary process, the logic loops all BARs and will skip
> the map of BAR with an invalid physical address (0), also will
> assign 'uio_res->nb_maps' with the real mapped BARs number. But
> for the secondary process, instead of loops all BARs, the logic
> using the 'uio_res->nb_map' as index. If the device uses continuous
> BARs there will be no problem, whereas if it uses discrete BARs,
> it will lead to mapping errors.
> 
> Fix this problem by also loops all BARs and skip the map of BAR
> with an invalid physical address in secondary process.
> 
> Fixes: 9b957f378abf ("pci: merge uio functions for linux and bsd")Hi, Chaoyong
This patch is just a code refactoring, this bug is caused by the following patch:

Fixes: e6cf7bee1c77 ("bus/pci: fix UIO resource access from secondary process")

> Cc: mukawa@igel.co.jp
> Cc: stable@dpdk.org
> 
> Signed-off-by: Zerun Fu <zerun.fu@corigine.com>
> Reviewed-by: Chaoyong He <chaoyong.he@corigine.com>
> Reviewed-by: Long Wu <long.wu@corigine.com>
> Reviewed-by: Peng Zhang <peng.zhang@corigine.com>

test by the igb_uio driver on the ARM.
Tested-by: Dengdui Huang <huangdengdui@huawei.com>

> ---
>  drivers/bus/pci/pci_common_uio.c | 94 ++++++++++++++++++++------------
>  1 file changed, 60 insertions(+), 34 deletions(-)
> 
> diff --git a/drivers/bus/pci/pci_common_uio.c b/drivers/bus/pci/pci_common_uio.c
> index 76c661f054..fcd8a49daf 100644
> --- a/drivers/bus/pci/pci_common_uio.c
> +++ b/drivers/bus/pci/pci_common_uio.c
> @@ -23,10 +23,57 @@ static struct rte_tailq_elem rte_uio_tailq = {
>  };
>  EAL_REGISTER_TAILQ(rte_uio_tailq)
>  
> +static int
> +pci_uio_map_secondary_resource_by_index(struct rte_pci_device *dev,
> +		int res_idx, struct mapped_pci_resource *uio_res, int map_idx)
> +{
> +	int fd, i;
> +
> +	if (map_idx >= uio_res->nb_maps)
> +		return -1;
> +
> +	/*
> +	 * open devname, to mmap it
> +	 */
> +	fd = open(uio_res->maps[map_idx].path, O_RDWR);
> +	if (fd < 0) {
> +		RTE_LOG(ERR, EAL, "Cannot open %s: %s\n",
> +			uio_res->maps[map_idx].path, strerror(errno));
> +		return -1;
> +	}
> +
> +	void *mapaddr = pci_map_resource(uio_res->maps[map_idx].addr,
> +			fd, (off_t)uio_res->maps[map_idx].offset,
> +			(size_t)uio_res->maps[map_idx].size, 0);
> +
> +	/* fd is not needed in secondary process, close it */
> +	close(fd);
> +	if (mapaddr != uio_res->maps[map_idx].addr) {
> +		RTE_LOG(ERR, EAL,
> +			"Cannot mmap device resource file %s to address: %p\n",
> +			uio_res->maps[map_idx].path,
> +			uio_res->maps[map_idx].addr);
> +		if (mapaddr != NULL) {
> +			/* unmap addrs correctly mapped */
> +			for (i = 0; i < map_idx; i++)
> +				pci_unmap_resource(
> +					uio_res->maps[i].addr,
> +					(size_t)uio_res->maps[i].size);
> +			/* unmap addr wrongly mapped */
> +			pci_unmap_resource(mapaddr,
> +				(size_t)uio_res->maps[map_idx].size);
> +		}
When mapaddr == NULL, the addrs correctly mapped should also be unmaped
It may be better to make the following changes

for (i = 0; i < map_idx; i++)
	pci_unmap_resource(uio_res->maps[i].addr, (size_t)uio_res->maps[i].size);
if (mapaddr != NULL)
	pci_unmap_resource(mapaddr, (size_t)uio_res->maps[map_idx].size);


> +		return -1;
> +	}
> +	dev->mem_resource[res_idx].addr = mapaddr;
> +
> +	return 0;
> +}
> +
>  static int
>  pci_uio_map_secondary(struct rte_pci_device *dev)
>  {
> -	int fd, i, j;
> +	int map_idx = 0, res_idx, ret;
>  	struct mapped_pci_resource *uio_res;
>  	struct mapped_pci_res_list *uio_res_list =
>  			RTE_TAILQ_CAST(rte_uio_tailq.head, mapped_pci_res_list);
> @@ -37,41 +84,20 @@ pci_uio_map_secondary(struct rte_pci_device *dev)
>  		if (rte_pci_addr_cmp(&uio_res->pci_addr, &dev->addr))
>  			continue;
>  
> -		for (i = 0; i != uio_res->nb_maps; i++) {
> -			/*
> -			 * open devname, to mmap it
> -			 */
> -			fd = open(uio_res->maps[i].path, O_RDWR);
> -			if (fd < 0) {
> -				RTE_LOG(ERR, EAL, "Cannot open %s: %s\n",
> -					uio_res->maps[i].path, strerror(errno));
> -				return -1;
> +		/* Map all BARs */
> +		for (res_idx = 0; res_idx != PCI_MAX_RESOURCE; res_idx++) {
> +			 /* skip empty BAR */
> +			if (dev->mem_resource[res_idx].phys_addr == 0)
> +				continue;
> +
> +			ret = pci_uio_map_secondary_resource_by_index(dev, res_idx,
> +					uio_res, map_idx);
> +			if (ret < 0) {
> +				RTE_LOG(ERR, EAL, "Failed to map resources\n");
> +				return ret;
>  			}
>  
> -			void *mapaddr = pci_map_resource(uio_res->maps[i].addr,
> -					fd, (off_t)uio_res->maps[i].offset,
> -					(size_t)uio_res->maps[i].size, 0);
> -
> -			/* fd is not needed in secondary process, close it */
> -			close(fd);
> -			if (mapaddr != uio_res->maps[i].addr) {
> -				RTE_LOG(ERR, EAL,
> -					"Cannot mmap device resource file %s to address: %p\n",
> -					uio_res->maps[i].path,
> -					uio_res->maps[i].addr);
> -				if (mapaddr != NULL) {
> -					/* unmap addrs correctly mapped */
> -					for (j = 0; j < i; j++)
> -						pci_unmap_resource(
> -							uio_res->maps[j].addr,
> -							(size_t)uio_res->maps[j].size);
> -					/* unmap addr wrongly mapped */
> -					pci_unmap_resource(mapaddr,
> -						(size_t)uio_res->maps[i].size);
> -				}
> -				return -1;
> -			}
> -			dev->mem_resource[i].addr = mapaddr;
> +			map_idx++;
>  		}
>  		return 0;
>  	}

Re: [PATCH v2 1/2] bus/pci: fix secondary process PCI uio resource map problem

[Burakov, Anatoly]

On 1/29/2024 10:22 AM, Chaoyong He wrote:
> From: Zerun Fu <zerun.fu@corigine.com>
> 
> For the primary process, the logic loops all BARs and will skip
> the map of BAR with an invalid physical address (0), also will
> assign 'uio_res->nb_maps' with the real mapped BARs number. But
> for the secondary process, instead of loops all BARs, the logic
> using the 'uio_res->nb_map' as index. If the device uses continuous
> BARs there will be no problem, whereas if it uses discrete BARs,
> it will lead to mapping errors.
> 
> Fix this problem by also loops all BARs and skip the map of BAR
> with an invalid physical address in secondary process.
> 
> Fixes: 9b957f378abf ("pci: merge uio functions for linux and bsd")
> Cc: mukawa@igel.co.jp
> Cc: stable@dpdk.org
> 
> Signed-off-by: Zerun Fu <zerun.fu@corigine.com>
> Reviewed-by: Chaoyong He <chaoyong.he@corigine.com>
> Reviewed-by: Long Wu <long.wu@corigine.com>
> Reviewed-by: Peng Zhang <peng.zhang@corigine.com>
> ---
>   drivers/bus/pci/pci_common_uio.c | 94 ++++++++++++++++++++------------
>   1 file changed, 60 insertions(+), 34 deletions(-)
> 
> diff --git a/drivers/bus/pci/pci_common_uio.c b/drivers/bus/pci/pci_common_uio.c
> index 76c661f054..fcd8a49daf 100644
> --- a/drivers/bus/pci/pci_common_uio.c
> +++ b/drivers/bus/pci/pci_common_uio.c
> @@ -23,10 +23,57 @@ static struct rte_tailq_elem rte_uio_tailq = {
>   };
>   EAL_REGISTER_TAILQ(rte_uio_tailq)
>   
> +static int
> +pci_uio_map_secondary_resource_by_index(struct rte_pci_device *dev,
> +		int res_idx, struct mapped_pci_resource *uio_res, int map_idx)
> +{
> +	int fd, i;
> +
> +	if (map_idx >= uio_res->nb_maps)
> +		return -1;
> +
> +	/*
> +	 * open devname, to mmap it
> +	 */
> +	fd = open(uio_res->maps[map_idx].path, O_RDWR);
> +	if (fd < 0) {
> +		RTE_LOG(ERR, EAL, "Cannot open %s: %s\n",
> +			uio_res->maps[map_idx].path, strerror(errno));
> +		return -1;
> +	}
> +
> +	void *mapaddr = pci_map_resource(uio_res->maps[map_idx].addr,
> +			fd, (off_t)uio_res->maps[map_idx].offset,
> +			(size_t)uio_res->maps[map_idx].size, 0);
> +
> +	/* fd is not needed in secondary process, close it */
> +	close(fd);
> +	if (mapaddr != uio_res->maps[map_idx].addr) {
> +		RTE_LOG(ERR, EAL,
> +			"Cannot mmap device resource file %s to address: %p\n",
> +			uio_res->maps[map_idx].path,
> +			uio_res->maps[map_idx].addr);
> +		if (mapaddr != NULL) {
> +			/* unmap addrs correctly mapped */
> +			for (i = 0; i < map_idx; i++)
> +				pci_unmap_resource(
> +					uio_res->maps[i].addr,
> +					(size_t)uio_res->maps[i].size);
> +			/* unmap addr wrongly mapped */
> +			pci_unmap_resource(mapaddr,
> +				(size_t)uio_res->maps[map_idx].size);
> +		}
> +		return -1;
> +	}
> +	dev->mem_resource[res_idx].addr = mapaddr;
> +
> +	return 0;
> +}
> +
>   static int
>   pci_uio_map_secondary(struct rte_pci_device *dev)
>   {
> -	int fd, i, j;
> +	int map_idx = 0, res_idx, ret;
>   	struct mapped_pci_resource *uio_res;
>   	struct mapped_pci_res_list *uio_res_list =
>   			RTE_TAILQ_CAST(rte_uio_tailq.head, mapped_pci_res_list);
> @@ -37,41 +84,20 @@ pci_uio_map_secondary(struct rte_pci_device *dev)
>   		if (rte_pci_addr_cmp(&uio_res->pci_addr, &dev->addr))
>   			continue;
>   
> -		for (i = 0; i != uio_res->nb_maps; i++) {
> -			/*
> -			 * open devname, to mmap it
> -			 */
> -			fd = open(uio_res->maps[i].path, O_RDWR);
> -			if (fd < 0) {
> -				RTE_LOG(ERR, EAL, "Cannot open %s: %s\n",
> -					uio_res->maps[i].path, strerror(errno));
> -				return -1;
> +		/* Map all BARs */
> +		for (res_idx = 0; res_idx != PCI_MAX_RESOURCE; res_idx++) {
> +			 /* skip empty BAR */
> +			if (dev->mem_resource[res_idx].phys_addr == 0)
> +				continue;
> +
> +			ret = pci_uio_map_secondary_resource_by_index(dev, res_idx,
> +					uio_res, map_idx);
> +			if (ret < 0) {
> +				RTE_LOG(ERR, EAL, "Failed to map resources\n");
> +				return ret;
>   			}
>   
> -			void *mapaddr = pci_map_resource(uio_res->maps[i].addr,
> -					fd, (off_t)uio_res->maps[i].offset,
> -					(size_t)uio_res->maps[i].size, 0);
> -
> -			/* fd is not needed in secondary process, close it */
> -			close(fd);
> -			if (mapaddr != uio_res->maps[i].addr) {
> -				RTE_LOG(ERR, EAL,
> -					"Cannot mmap device resource file %s to address: %p\n",
> -					uio_res->maps[i].path,
> -					uio_res->maps[i].addr);
> -				if (mapaddr != NULL) {
> -					/* unmap addrs correctly mapped */
> -					for (j = 0; j < i; j++)
> -						pci_unmap_resource(
> -							uio_res->maps[j].addr,
> -							(size_t)uio_res->maps[j].size);
> -					/* unmap addr wrongly mapped */
> -					pci_unmap_resource(mapaddr,
> -						(size_t)uio_res->maps[i].size);

Nitpicking, but I feel like this would've been better done from the 
caller, not here. This is how it's done in primary process.

> -				}
> -				return -1;
> -			}
> -			dev->mem_resource[i].addr = mapaddr;
> +			map_idx++;
>   		}
>   		return 0;
>   	}

With fix above,

Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>

-- 
Thanks,
Anatoly

Re: [PATCH v2 2/2] bus/pci: fix secondary process save 'FD' problem

[Burakov, Anatoly]

On 1/29/2024 10:22 AM, Chaoyong He wrote:
> From: Zerun Fu <zerun.fu@corigine.com>
> 
> In the previous logic the 'fd' was only saved in the primary process,
> but for some devices this value is also used in the secondary logic.
> 
> For example, the call of 'rte_pci_find_ext_capability()' will fail in
> the secondary process.
> 
> Fix this problem by getting and saving the value of 'fd' also in the
> secondary process logic.
> 
> Fixes: 9b957f378abf ("pci: merge uio functions for linux and bsd")
> Cc: mukawa@igel.co.jp
> Cc: stable@dpdk.org
> 
> Signed-off-by: Zerun Fu <zerun.fu@corigine.com>
> Reviewed-by: Chaoyong He <chaoyong.he@corigine.com>
> Reviewed-by: Long Wu <long.wu@corigine.com>
> Reviewed-by: Peng Zhang <peng.zhang@corigine.com>
> ---
Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>

-- 
Thanks,
Anatoly

[PATCH v3 1/2] bus/pci: fix secondary process PCI uio resource map problem

[Chaoyong He]

From: Zerun Fu <zerun.fu@corigine.com>

For the primary process, the logic loops all BARs and will skip
the map of BAR with an invalid physical address (0), also will
assign 'uio_res->nb_maps' with the real mapped BARs number. But
for the secondary process, instead of loops all BARs, the logic
using the 'uio_res->nb_map' as index. If the device uses continuous
BARs there will be no problem, whereas if it uses discrete BARs,
it will lead to mapping errors.

Fix this problem by also loops all BARs and skip the map of BAR
with an invalid physical address in secondary process.

Fixes: 9b957f378abf ("pci: merge uio functions for linux and bsd")
Cc: mukawa@igel.co.jp
Cc: stable@dpdk.org

Signed-off-by: Zerun Fu <zerun.fu@corigine.com>
Reviewed-by: Chaoyong He <chaoyong.he@corigine.com>
Reviewed-by: Long Wu <long.wu@corigine.com>
Reviewed-by: Peng Zhang <peng.zhang@corigine.com>
---
 drivers/bus/pci/pci_common_uio.c | 40 ++++++++++++++++++++------------
 1 file changed, 25 insertions(+), 15 deletions(-)

diff --git a/drivers/bus/pci/pci_common_uio.c b/drivers/bus/pci/pci_common_uio.c
index 76c661f054..432d68b12c 100644
--- a/drivers/bus/pci/pci_common_uio.c
+++ b/drivers/bus/pci/pci_common_uio.c
@@ -26,7 +26,7 @@ EAL_REGISTER_TAILQ(rte_uio_tailq)
 static int
 pci_uio_map_secondary(struct rte_pci_device *dev)
 {
-	int fd, i, j;
+	int fd, i, map_idx = 0, res_idx;
 	struct mapped_pci_resource *uio_res;
 	struct mapped_pci_res_list *uio_res_list =
 			RTE_TAILQ_CAST(rte_uio_tailq.head, mapped_pci_res_list);
@@ -37,41 +37,51 @@ pci_uio_map_secondary(struct rte_pci_device *dev)
 		if (rte_pci_addr_cmp(&uio_res->pci_addr, &dev->addr))
 			continue;
 
-		for (i = 0; i != uio_res->nb_maps; i++) {
+		/* Map all BARs */
+		for (res_idx = 0; res_idx != PCI_MAX_RESOURCE; res_idx++) {
+			 /* skip empty BAR */
+			if (dev->mem_resource[res_idx].phys_addr == 0)
+				continue;
+
+			if (map_idx >= uio_res->nb_maps)
+				return -1;
+
 			/*
 			 * open devname, to mmap it
 			 */
-			fd = open(uio_res->maps[i].path, O_RDWR);
+			fd = open(uio_res->maps[map_idx].path, O_RDWR);
 			if (fd < 0) {
 				RTE_LOG(ERR, EAL, "Cannot open %s: %s\n",
-					uio_res->maps[i].path, strerror(errno));
+					uio_res->maps[map_idx].path, strerror(errno));
 				return -1;
 			}
 
-			void *mapaddr = pci_map_resource(uio_res->maps[i].addr,
-					fd, (off_t)uio_res->maps[i].offset,
-					(size_t)uio_res->maps[i].size, 0);
+			void *mapaddr = pci_map_resource(uio_res->maps[map_idx].addr,
+					fd, (off_t)uio_res->maps[map_idx].offset,
+					(size_t)uio_res->maps[map_idx].size, 0);
 
 			/* fd is not needed in secondary process, close it */
 			close(fd);
-			if (mapaddr != uio_res->maps[i].addr) {
+			if (mapaddr != uio_res->maps[map_idx].addr) {
 				RTE_LOG(ERR, EAL,
 					"Cannot mmap device resource file %s to address: %p\n",
-					uio_res->maps[i].path,
-					uio_res->maps[i].addr);
+					uio_res->maps[map_idx].path,
+					uio_res->maps[map_idx].addr);
 				if (mapaddr != NULL) {
 					/* unmap addrs correctly mapped */
-					for (j = 0; j < i; j++)
+					for (i = 0; i < map_idx; i++)
 						pci_unmap_resource(
-							uio_res->maps[j].addr,
-							(size_t)uio_res->maps[j].size);
+							uio_res->maps[i].addr,
+							(size_t)uio_res->maps[i].size);
 					/* unmap addr wrongly mapped */
 					pci_unmap_resource(mapaddr,
-						(size_t)uio_res->maps[i].size);
+						(size_t)uio_res->maps[map_idx].size);
 				}
 				return -1;
 			}
-			dev->mem_resource[i].addr = mapaddr;
+			dev->mem_resource[res_idx].addr = mapaddr;
+
+			map_idx++;
 		}
 		return 0;
 	}
-- 
2.39.1

[PATCH v3 2/2] bus/pci: fix secondary process save 'FD' problem

[Chaoyong He]

From: Zerun Fu <zerun.fu@corigine.com>

In the previous logic the 'fd' was only saved in the primary process,
but for some devices this value is also used in the secondary logic.

For example, the call of 'rte_pci_find_ext_capability()' will fail in
the secondary process.

Fix this problem by getting and saving the value of 'fd' also in the
secondary process logic.

Fixes: 9b957f378abf ("pci: merge uio functions for linux and bsd")
Cc: mukawa@igel.co.jp
Cc: stable@dpdk.org

Signed-off-by: Zerun Fu <zerun.fu@corigine.com>
Reviewed-by: Chaoyong He <chaoyong.he@corigine.com>
Reviewed-by: Long Wu <long.wu@corigine.com>
Reviewed-by: Peng Zhang <peng.zhang@corigine.com>
---
 drivers/bus/pci/linux/pci_uio.c  | 5 ++++-
 drivers/bus/pci/pci_common_uio.c | 8 ++++----
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/drivers/bus/pci/linux/pci_uio.c b/drivers/bus/pci/linux/pci_uio.c
index 97d740dfe5..6680e42efb 100644
--- a/drivers/bus/pci/linux/pci_uio.c
+++ b/drivers/bus/pci/linux/pci_uio.c
@@ -237,7 +237,7 @@ pci_uio_alloc_resource(struct rte_pci_device *dev,
 	}
 	snprintf(devname, sizeof(devname), "/dev/uio%u", uio_num);
 
-	/* save fd if in primary process */
+	/* save fd */
 	fd = open(devname, O_RDWR);
 	if (fd < 0) {
 		RTE_LOG(ERR, EAL, "Cannot open %s: %s\n",
@@ -261,6 +261,9 @@ pci_uio_alloc_resource(struct rte_pci_device *dev,
 	if (rte_intr_dev_fd_set(dev->intr_handle, uio_cfg_fd))
 		goto error;
 
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+		return 0;
+
 	if (dev->kdrv == RTE_PCI_KDRV_IGB_UIO) {
 		if (rte_intr_type_set(dev->intr_handle, RTE_INTR_HANDLE_UIO))
 			goto error;
diff --git a/drivers/bus/pci/pci_common_uio.c b/drivers/bus/pci/pci_common_uio.c
index 432d68b12c..8688862738 100644
--- a/drivers/bus/pci/pci_common_uio.c
+++ b/drivers/bus/pci/pci_common_uio.c
@@ -106,15 +106,15 @@ pci_uio_map_resource(struct rte_pci_device *dev)
 	if (rte_intr_dev_fd_set(dev->intr_handle, -1))
 		return -1;
 
-	/* secondary processes - use already recorded details */
-	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
-		return pci_uio_map_secondary(dev);
-
 	/* allocate uio resource */
 	ret = pci_uio_alloc_resource(dev, &uio_res);
 	if (ret)
 		return ret;
 
+	/* secondary processes - use already recorded details */
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+		return pci_uio_map_secondary(dev);
+
 	/* Map all BARs */
 	for (i = 0; i != PCI_MAX_RESOURCE; i++) {
 		/* skip empty BAR */
-- 
2.39.1