[dpdk-dev] [PATCH v2 1/3] eal: add API for bus close

[dpdk-dev] [PATCH v2 1/3] eal: add API for bus close

[rohit.raj]

From: Rohit Raj <rohit.raj@nxp.com>

As per the current code we have API for bus probe, but the
bus close API is missing. This breaks the multi process
scenarios as objects are not cleaned while terminating the
secondary processes.

This patch adds a new API rte_bus_close() for cleanup of
bus objects which were acquired during probe.

Signed-off-by: Rohit Raj <rohit.raj@nxp.com>
---

v2:
* Moved rte_bus_close call to rte_eal_cleanup path.

 lib/librte_eal/common/eal_common_bus.c | 34 +++++++++++++++++++++++++-
 lib/librte_eal/include/rte_bus.h       | 25 ++++++++++++++++++-
 lib/librte_eal/linux/eal.c             |  1 +
 lib/librte_eal/rte_eal_version.map     |  3 +++
 4 files changed, 61 insertions(+), 2 deletions(-)

diff --git a/lib/librte_eal/common/eal_common_bus.c b/lib/librte_eal/common/eal_common_bus.c
index baa5b532a..4c8c7eecc 100644
--- a/lib/librte_eal/common/eal_common_bus.c
+++ b/lib/librte_eal/common/eal_common_bus.c
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright 2016 NXP
+ * Copyright 2016,2020 NXP
  */
 
 #include <stdio.h>
@@ -56,6 +56,38 @@ rte_bus_scan(void)
 	return 0;
 }
 
+int
+rte_bus_close(void)
+{
+	int ret;
+	struct rte_bus *bus, *vbus = NULL;
+
+	TAILQ_FOREACH(bus, &rte_bus_list, next) {
+		if (!strcmp(bus->name, "vdev")) {
+			vbus = bus;
+			continue;
+		}
+
+		if (bus->close) {
+			ret = bus->close();
+			if (ret)
+				RTE_LOG(ERR, EAL, "Bus (%s) close failed.\n",
+					bus->name);
+		}
+	}
+
+	if (vbus) {
+		if (vbus->close) {
+			ret = vbus->close();
+			if (ret)
+				RTE_LOG(ERR, EAL, "Bus (%s) close failed.\n",
+					vbus->name);
+		}
+	}
+
+	return 0;
+}
+
 /* Probe all devices of all buses */
 int
 rte_bus_probe(void)
diff --git a/lib/librte_eal/include/rte_bus.h b/lib/librte_eal/include/rte_bus.h
index d3034d0ed..af4787b18 100644
--- a/lib/librte_eal/include/rte_bus.h
+++ b/lib/librte_eal/include/rte_bus.h
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright 2016 NXP
+ * Copyright 2016,2020 NXP
  */
 
 #ifndef _RTE_BUS_H_
@@ -67,6 +67,18 @@ typedef int (*rte_bus_scan_t)(void);
  */
 typedef int (*rte_bus_probe_t)(void);
 
+/**
+ * Implementation specific close function which is responsible for closing
+ * devices on that bus.
+ *
+ * This is called while iterating over each registered bus.
+ *
+ * @return
+ *	0 for successful close
+ *	!0 for any error while closing
+ */
+typedef int (*rte_bus_close_t)(void);
+
 /**
  * Device iterator to find a device on a bus.
  *
@@ -248,6 +260,7 @@ struct rte_bus {
 	const char *name;            /**< Name of the bus */
 	rte_bus_scan_t scan;         /**< Scan for devices attached to bus */
 	rte_bus_probe_t probe;       /**< Probe devices on bus */
+	rte_bus_close_t close;       /**< Close devices on bus */
 	rte_bus_find_device_t find_device; /**< Find a device on the bus */
 	rte_bus_plug_t plug;         /**< Probe single device for drivers */
 	rte_bus_unplug_t unplug;     /**< Remove single device from driver */
@@ -301,6 +314,16 @@ int rte_bus_scan(void);
  */
 int rte_bus_probe(void);
 
+/**
+ * For each device on the buses, call the device specific close.
+ *
+ * @return
+ *	 0 for successful close
+ *	!0 otherwise
+ */
+__rte_experimental
+int rte_bus_close(void);
+
 /**
  * Dump information of all the buses registered with EAL.
  *
diff --git a/lib/librte_eal/linux/eal.c b/lib/librte_eal/linux/eal.c
index 0960f01d0..a8a63d8ca 100644
--- a/lib/librte_eal/linux/eal.c
+++ b/lib/librte_eal/linux/eal.c
@@ -1357,6 +1357,7 @@ rte_eal_cleanup(void)
 
 	if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 		rte_memseg_walk(mark_freeable, NULL);
+	rte_bus_close();
 	rte_service_finalize();
 	rte_mp_channel_cleanup();
 	rte_trace_save();
diff --git a/lib/librte_eal/rte_eal_version.map b/lib/librte_eal/rte_eal_version.map
index bf0c17c23..6a6d993a1 100644
--- a/lib/librte_eal/rte_eal_version.map
+++ b/lib/librte_eal/rte_eal_version.map
@@ -403,6 +403,9 @@ EXPERIMENTAL {
 	rte_mp_disable;
 	rte_thread_register;
 	rte_thread_unregister;
+
+	# added in 20.11
+	rte_bus_close;
 };
 
 INTERNAL {
-- 
2.17.1

[dpdk-dev] [PATCH v2 2/3] examples/multi_process: cleanup bus objects while terminating app

[rohit.raj]

From: Rohit Raj <rohit.raj@nxp.com>

Certain bus objects may need to be closed and re-acquired
while terminating and rerunning the client application.
Hence a signal handler is required to catch the termination
of the App and hence closing the bus objects.

This patch adds the missing signal handler in the client
app and closes the Bus objects in both client and server
applications when the signal Handler is called.

Signed-off-by: Rohit Raj <rohit.raj@nxp.com>
---
 .../multi_process/client_server_mp/mp_client/client.c | 11 +++++++++++
 .../multi_process/client_server_mp/mp_server/main.c   |  4 +++-
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/examples/multi_process/client_server_mp/mp_client/client.c b/examples/multi_process/client_server_mp/mp_client/client.c
index 361d90b54..c37516b4c 100644
--- a/examples/multi_process/client_server_mp/mp_client/client.c
+++ b/examples/multi_process/client_server_mp/mp_client/client.c
@@ -11,6 +11,7 @@
 #include <stdlib.h>
 #include <getopt.h>
 #include <string.h>
+#include <signal.h>
 
 #include <rte_common.h>
 #include <rte_malloc.h>
@@ -196,6 +197,14 @@ handle_packet(struct rte_mbuf *buf)
 
 }
 
+static void
+signal_handler(int signal)
+{
+	if (signal == SIGINT)
+		rte_eal_cleanup();
+	exit(0);
+}
+
 /*
  * Application main function - loops through
  * receiving and processing packets. Never returns
@@ -217,6 +226,8 @@ main(int argc, char *argv[])
 	argc -= retval;
 	argv += retval;
 
+	signal(SIGINT, signal_handler);
+
 	if (parse_app_args(argc, argv) < 0)
 		rte_exit(EXIT_FAILURE, "Invalid command-line arguments\n");
 
diff --git a/examples/multi_process/client_server_mp/mp_server/main.c b/examples/multi_process/client_server_mp/mp_server/main.c
index 280dab867..b0241cc20 100644
--- a/examples/multi_process/client_server_mp/mp_server/main.c
+++ b/examples/multi_process/client_server_mp/mp_server/main.c
@@ -275,11 +275,13 @@ signal_handler(int signal)
 {
 	uint16_t port_id;
 
-	if (signal == SIGINT)
+	if (signal == SIGINT) {
 		RTE_ETH_FOREACH_DEV(port_id) {
 			rte_eth_dev_stop(port_id);
 			rte_eth_dev_close(port_id);
 		}
+		rte_eal_cleanup();
+	}
 	exit(0);
 }
 
-- 
2.17.1

[dpdk-dev] [PATCH v2 3/3] bus/fslmc: support bus close API

[rohit.raj]

From: Rohit Raj <rohit.raj@nxp.com>

This patch add support for closing the bus objects which
were acquired In the bus probe.

Some devices need to be cleaned while in both primary and
secondary process and while some need to be cleaned only in
case of primary process.

The devices are closed as per the white list used while
creating the objects in a particular process.

Signed-off-by: Rohit Raj <rohit.raj@nxp.com>
Acked-by: Sachin Saxena <sachin.saxena@oss.nxp.com>
---
 drivers/bus/fslmc/fslmc_bus.c               | 15 +++-
 drivers/bus/fslmc/fslmc_vfio.c              | 89 ++++++++++++++++++++-
 drivers/bus/fslmc/fslmc_vfio.h              |  3 +-
 drivers/bus/fslmc/mc/fsl_dpcon.h            |  1 +
 drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c    | 31 ++++++-
 drivers/bus/fslmc/portal/dpaa2_hw_dpci.c    | 32 +++++++-
 drivers/bus/fslmc/portal/dpaa2_hw_dpio.c    | 34 ++++++++
 drivers/bus/fslmc/rte_bus_fslmc_version.map |  1 +
 drivers/bus/fslmc/rte_fslmc.h               |  5 +-
 drivers/event/dpaa2/dpaa2_hw_dpcon.c        | 32 +++++++-
 drivers/net/dpaa2/dpaa2_mux.c               | 18 ++++-
 11 files changed, 253 insertions(+), 8 deletions(-)

diff --git a/drivers/bus/fslmc/fslmc_bus.c b/drivers/bus/fslmc/fslmc_bus.c
index beb3dd008..9d3da6be3 100644
--- a/drivers/bus/fslmc/fslmc_bus.c
+++ b/drivers/bus/fslmc/fslmc_bus.c
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
- *   Copyright 2016,2018-2019 NXP
+ *   Copyright 2016,2018-2020 NXP
  *
  */
 
@@ -365,6 +365,18 @@ rte_fslmc_match(struct rte_dpaa2_driver *dpaa2_drv,
 	return 1;
 }
 
+static int
+rte_fslmc_close(void)
+{
+	int ret = 0;
+
+	ret = fslmc_vfio_close_group();
+	if (ret)
+		DPAA2_BUS_ERR("Unable to close devices %d", ret);
+
+	return 0;
+}
+
 static int
 rte_fslmc_probe(void)
 {
@@ -637,6 +649,7 @@ struct rte_fslmc_bus rte_fslmc_bus = {
 	.bus = {
 		.scan = rte_fslmc_scan,
 		.probe = rte_fslmc_probe,
+		.close = rte_fslmc_close,
 		.parse = rte_fslmc_parse,
 		.find_device = rte_fslmc_find_device,
 		.get_iommu_class = rte_dpaa2_get_iommu_class,
diff --git a/drivers/bus/fslmc/fslmc_vfio.c b/drivers/bus/fslmc/fslmc_vfio.c
index 9134ec552..34ccc9db3 100644
--- a/drivers/bus/fslmc/fslmc_vfio.c
+++ b/drivers/bus/fslmc/fslmc_vfio.c
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
  *   Copyright (c) 2015-2016 Freescale Semiconductor, Inc. All rights reserved.
- *   Copyright 2016-2019 NXP
+ *   Copyright 2016-2020 NXP
  *
  */
 
@@ -692,6 +692,54 @@ rte_dpaa2_vfio_setup_intr(struct rte_intr_handle *intr_handle,
 	return -1;
 }
 
+static void
+fslmc_close_iodevices(struct rte_dpaa2_device *dev)
+{
+	struct rte_dpaa2_object *object = NULL;
+	struct rte_dpaa2_driver *drv;
+	int ret, probe_all;
+
+	switch (dev->dev_type) {
+	case DPAA2_IO:
+	case DPAA2_CON:
+	case DPAA2_CI:
+	case DPAA2_BPOOL:
+	case DPAA2_MUX:
+		TAILQ_FOREACH(object, &dpaa2_obj_list, next) {
+			if (dev->dev_type == object->dev_type)
+				object->close(dev->object_id);
+			else
+				continue;
+		}
+		break;
+	case DPAA2_ETH:
+	case DPAA2_CRYPTO:
+	case DPAA2_QDMA:
+		probe_all = rte_fslmc_bus.bus.conf.scan_mode !=
+			    RTE_BUS_SCAN_WHITELIST;
+		TAILQ_FOREACH(drv, &rte_fslmc_bus.driver_list, next) {
+			if (drv->drv_type != dev->dev_type)
+				continue;
+			if (rte_dev_is_probed(&dev->device))
+				continue;
+			if (probe_all ||
+			    (dev->device.devargs &&
+			     dev->device.devargs->policy ==
+			     RTE_DEV_WHITELISTED)) {
+				ret = drv->remove(dev);
+				if (ret)
+					DPAA2_BUS_ERR("Unable to remove");
+			}
+		}
+		break;
+	default:
+		break;
+	}
+
+	DPAA2_BUS_LOG(DEBUG, "Device (%s) Closed",
+		      dev->device.name);
+}
+
 /*
  * fslmc_process_iodevices for processing only IO (ETH, CRYPTO, and possibly
  * EVENT) devices.
@@ -796,6 +844,45 @@ fslmc_process_mcp(struct rte_dpaa2_device *dev)
 	return ret;
 }
 
+int
+fslmc_vfio_close_group(void)
+{
+	struct rte_dpaa2_device *dev, *dev_temp;
+
+	TAILQ_FOREACH_SAFE(dev, &rte_fslmc_bus.device_list, next, dev_temp) {
+		if (dev->device.devargs &&
+		    dev->device.devargs->policy == RTE_DEV_BLACKLISTED) {
+			DPAA2_BUS_LOG(DEBUG, "%s Blacklisted, skipping",
+				      dev->device.name);
+			TAILQ_REMOVE(&rte_fslmc_bus.device_list, dev, next);
+				continue;
+		}
+		switch (dev->dev_type) {
+		case DPAA2_ETH:
+		case DPAA2_CRYPTO:
+		case DPAA2_QDMA:
+		case DPAA2_IO:
+			fslmc_close_iodevices(dev);
+			break;
+		case DPAA2_CON:
+		case DPAA2_CI:
+		case DPAA2_BPOOL:
+		case DPAA2_MUX:
+			if (rte_eal_process_type() == RTE_PROC_SECONDARY)
+				continue;
+
+			fslmc_close_iodevices(dev);
+			break;
+		case DPAA2_DPRTC:
+		default:
+			DPAA2_BUS_DEBUG("Device cannot be closed: Not supported (%s)",
+					dev->device.name);
+		}
+	}
+
+	return 0;
+}
+
 int
 fslmc_vfio_process_group(void)
 {
diff --git a/drivers/bus/fslmc/fslmc_vfio.h b/drivers/bus/fslmc/fslmc_vfio.h
index bc7c6f62d..0a29fe739 100644
--- a/drivers/bus/fslmc/fslmc_vfio.h
+++ b/drivers/bus/fslmc/fslmc_vfio.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
  *   Copyright (c) 2015-2016 Freescale Semiconductor, Inc. All rights reserved.
- *   Copyright 2016,2019 NXP
+ *   Copyright 2016,2019-2020 NXP
  *
  */
 
@@ -55,6 +55,7 @@ int rte_dpaa2_vfio_setup_intr(struct rte_intr_handle *intr_handle,
 
 int fslmc_vfio_setup_group(void);
 int fslmc_vfio_process_group(void);
+int fslmc_vfio_close_group(void);
 char *fslmc_get_container(void);
 int fslmc_get_container_group(int *gropuid);
 int rte_fslmc_vfio_dmamap(void);
diff --git a/drivers/bus/fslmc/mc/fsl_dpcon.h b/drivers/bus/fslmc/mc/fsl_dpcon.h
index 7caa6c68a..8053a6db1 100644
--- a/drivers/bus/fslmc/mc/fsl_dpcon.h
+++ b/drivers/bus/fslmc/mc/fsl_dpcon.h
@@ -26,6 +26,7 @@ int dpcon_open(struct fsl_mc_io *mc_io,
 	       int dpcon_id,
 	       uint16_t *token);
 
+__rte_internal
 int dpcon_close(struct fsl_mc_io *mc_io,
 		uint32_t cmd_flags,
 		uint16_t token);
diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c
index d9619848d..9a4cdfd7a 100644
--- a/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c
+++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
  *   Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
- *   Copyright 2016 NXP
+ *   Copyright 2016,2020 NXP
  *
  */
 
@@ -33,6 +33,19 @@ TAILQ_HEAD(dpbp_dev_list, dpaa2_dpbp_dev);
 static struct dpbp_dev_list dpbp_dev_list
 	= TAILQ_HEAD_INITIALIZER(dpbp_dev_list); /*!< DPBP device list */
 
+static struct dpaa2_dpbp_dev *get_dpbp_from_id(uint32_t dpbp_id)
+{
+	struct dpaa2_dpbp_dev *dpbp_dev = NULL;
+
+	/* Get DPBP dev handle from list using index */
+	TAILQ_FOREACH(dpbp_dev, &dpbp_dev_list, next) {
+		if (dpbp_dev->dpbp_id == dpbp_id)
+			break;
+	}
+
+	return dpbp_dev;
+}
+
 static int
 dpaa2_create_dpbp_device(int vdev_fd __rte_unused,
 			 struct vfio_device_info *obj_info __rte_unused,
@@ -116,9 +129,25 @@ int dpaa2_dpbp_supported(void)
 	return 0;
 }
 
+static void
+dpaa2_close_dpbp_device(int object_id)
+{
+	struct dpaa2_dpbp_dev *dpbp_dev = NULL;
+
+	dpbp_dev = get_dpbp_from_id((uint32_t)object_id);
+
+	if (dpbp_dev) {
+		dpaa2_free_dpbp_dev(dpbp_dev);
+		dpbp_close(&dpbp_dev->dpbp, CMD_PRI_LOW, dpbp_dev->token);
+		TAILQ_REMOVE(&dpbp_dev_list, dpbp_dev, next);
+		rte_free(dpbp_dev);
+	}
+}
+
 static struct rte_dpaa2_object rte_dpaa2_dpbp_obj = {
 	.dev_type = DPAA2_BPOOL,
 	.create = dpaa2_create_dpbp_device,
+	.close = dpaa2_close_dpbp_device,
 };
 
 RTE_PMD_REGISTER_DPAA2_OBJECT(dpbp, rte_dpaa2_dpbp_obj);
diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c
index d393ce618..9902f38ce 100644
--- a/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c
+++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
- *   Copyright 2017 NXP
+ *   Copyright 2017,2020 NXP
  *
  */
 
@@ -30,6 +30,19 @@ TAILQ_HEAD(dpci_dev_list, dpaa2_dpci_dev);
 static struct dpci_dev_list dpci_dev_list
 	= TAILQ_HEAD_INITIALIZER(dpci_dev_list); /*!< DPCI device list */
 
+static struct dpaa2_dpci_dev *get_dpci_from_id(uint32_t dpci_id)
+{
+	struct dpaa2_dpci_dev *dpci_dev = NULL;
+
+	/* Get DPCI dev handle from list using index */
+	TAILQ_FOREACH(dpci_dev, &dpci_dev_list, next) {
+		if (dpci_dev->dpci_id == dpci_id)
+			break;
+	}
+
+	return dpci_dev;
+}
+
 static int
 rte_dpaa2_create_dpci_device(int vdev_fd __rte_unused,
 			     struct vfio_device_info *obj_info __rte_unused,
@@ -179,9 +192,26 @@ void rte_dpaa2_free_dpci_dev(struct dpaa2_dpci_dev *dpci)
 	}
 }
 
+
+static void
+rte_dpaa2_close_dpci_device(int object_id)
+{
+	struct dpaa2_dpci_dev *dpci_dev = NULL;
+
+	dpci_dev = get_dpci_from_id((uint32_t)object_id);
+
+	if (dpci_dev) {
+		rte_dpaa2_free_dpci_dev(dpci_dev);
+		dpci_close(&dpci_dev->dpci, CMD_PRI_LOW, dpci_dev->token);
+		TAILQ_REMOVE(&dpci_dev_list, dpci_dev, next);
+		rte_free(dpci_dev);
+	}
+}
+
 static struct rte_dpaa2_object rte_dpaa2_dpci_obj = {
 	.dev_type = DPAA2_CI,
 	.create = rte_dpaa2_create_dpci_device,
+	.close = rte_dpaa2_close_dpci_device,
 };
 
 RTE_PMD_REGISTER_DPAA2_OBJECT(dpci, rte_dpaa2_dpci_obj);
diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
index 97be76116..9563cd620 100644
--- a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
+++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
@@ -86,6 +86,19 @@ static int dpaa2_cluster_sz = 2;
  * Cluster 4 (ID = x07) : CPU14, CPU15;
  */
 
+static struct dpaa2_dpio_dev *get_dpio_dev_from_id(int32_t dpio_id)
+{
+	struct dpaa2_dpio_dev *dpio_dev = NULL;
+
+	/* Get DPIO dev handle from list using index */
+	TAILQ_FOREACH(dpio_dev, &dpio_dev_list, next) {
+		if (dpio_dev->hw_id == dpio_id)
+			break;
+	}
+
+	return dpio_dev;
+}
+
 static int
 dpaa2_get_core_id(void)
 {
@@ -360,6 +373,26 @@ static void dpaa2_portal_finish(void *arg)
 	pthread_setspecific(dpaa2_portal_key, NULL);
 }
 
+static void
+dpaa2_close_dpio_device(int object_id)
+{
+	struct dpaa2_dpio_dev *dpio_dev = NULL;
+
+	dpio_dev = get_dpio_dev_from_id((int32_t)object_id);
+
+	if (dpio_dev) {
+		if (dpio_dev->dpio) {
+			dpio_disable(dpio_dev->dpio, CMD_PRI_LOW,
+				     dpio_dev->token);
+			dpio_close(dpio_dev->dpio, CMD_PRI_LOW,
+				   dpio_dev->token);
+			rte_free(dpio_dev->dpio);
+		}
+		TAILQ_REMOVE(&dpio_dev_list, dpio_dev, next);
+		rte_free(dpio_dev);
+	}
+}
+
 static int
 dpaa2_create_dpio_device(int vdev_fd,
 			 struct vfio_device_info *obj_info,
@@ -623,6 +656,7 @@ dpaa2_free_eq_descriptors(void)
 static struct rte_dpaa2_object rte_dpaa2_dpio_obj = {
 	.dev_type = DPAA2_IO,
 	.create = dpaa2_create_dpio_device,
+	.close = dpaa2_close_dpio_device,
 };
 
 RTE_PMD_REGISTER_DPAA2_OBJECT(dpio, rte_dpaa2_dpio_obj);
diff --git a/drivers/bus/fslmc/rte_bus_fslmc_version.map b/drivers/bus/fslmc/rte_bus_fslmc_version.map
index b169f5228..ac4ee2bf8 100644
--- a/drivers/bus/fslmc/rte_bus_fslmc_version.map
+++ b/drivers/bus/fslmc/rte_bus_fslmc_version.map
@@ -30,6 +30,7 @@ INTERNAL {
 	dpci_get_opr;
 	dpci_set_opr;
 	dpci_set_rx_queue;
+	dpcon_close;
 	dpcon_get_attributes;
 	dpcon_open;
 	dpdmai_close;
diff --git a/drivers/bus/fslmc/rte_fslmc.h b/drivers/bus/fslmc/rte_fslmc.h
index 80873fffc..a938ab8f9 100644
--- a/drivers/bus/fslmc/rte_fslmc.h
+++ b/drivers/bus/fslmc/rte_fslmc.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
- *   Copyright 2016,2019 NXP
+ *   Copyright 2016,2019-2020 NXP
  *
  */
 
@@ -79,6 +79,8 @@ typedef int (*rte_dpaa2_obj_create_t)(int vdev_fd,
 				      struct vfio_device_info *obj_info,
 				      int object_id);
 
+typedef void (*rte_dpaa2_obj_close_t)(int object_id);
+
 /**
  * A structure describing a DPAA2 object.
  */
@@ -87,6 +89,7 @@ struct rte_dpaa2_object {
 	const char *name;                   /**< Name of Object. */
 	enum rte_dpaa2_dev_type dev_type;   /**< Type of device */
 	rte_dpaa2_obj_create_t create;
+	rte_dpaa2_obj_close_t close;
 };
 
 /**
diff --git a/drivers/event/dpaa2/dpaa2_hw_dpcon.c b/drivers/event/dpaa2/dpaa2_hw_dpcon.c
index 200b71640..86844f49f 100644
--- a/drivers/event/dpaa2/dpaa2_hw_dpcon.c
+++ b/drivers/event/dpaa2/dpaa2_hw_dpcon.c
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
- *   Copyright 2017 NXP
+ *   Copyright 2017,2020 NXP
  *
  */
 
@@ -30,6 +30,19 @@ TAILQ_HEAD(dpcon_dev_list, dpaa2_dpcon_dev);
 static struct dpcon_dev_list dpcon_dev_list
 	= TAILQ_HEAD_INITIALIZER(dpcon_dev_list); /*!< DPCON device list */
 
+static struct dpaa2_dpcon_dev *get_dpcon_from_id(uint32_t dpcon_id)
+{
+	struct dpaa2_dpcon_dev *dpcon_dev = NULL;
+
+	/* Get DPCONC dev handle from list using index */
+	TAILQ_FOREACH(dpcon_dev, &dpcon_dev_list, next) {
+		if (dpcon_dev->dpcon_id == dpcon_id)
+			break;
+	}
+
+	return dpcon_dev;
+}
+
 static int
 rte_dpaa2_create_dpcon_device(int dev_fd __rte_unused,
 			      struct vfio_device_info *obj_info __rte_unused,
@@ -105,9 +118,26 @@ void rte_dpaa2_free_dpcon_dev(struct dpaa2_dpcon_dev *dpcon)
 	}
 }
 
+
+static void
+rte_dpaa2_close_dpcon_device(int object_id)
+{
+	struct dpaa2_dpcon_dev *dpcon_dev = NULL;
+
+	dpcon_dev = get_dpcon_from_id((uint32_t)object_id);
+
+	if (dpcon_dev) {
+		rte_dpaa2_free_dpcon_dev(dpcon_dev);
+		dpcon_close(&dpcon_dev->dpcon, CMD_PRI_LOW, dpcon_dev->token);
+		TAILQ_REMOVE(&dpcon_dev_list, dpcon_dev, next);
+		rte_free(dpcon_dev);
+	}
+}
+
 static struct rte_dpaa2_object rte_dpaa2_dpcon_obj = {
 	.dev_type = DPAA2_CON,
 	.create = rte_dpaa2_create_dpcon_device,
+	.close = rte_dpaa2_close_dpcon_device,
 };
 
 RTE_PMD_REGISTER_DPAA2_OBJECT(dpcon, rte_dpaa2_dpcon_obj);
diff --git a/drivers/net/dpaa2/dpaa2_mux.c b/drivers/net/dpaa2/dpaa2_mux.c
index f8366e839..15114dac9 100644
--- a/drivers/net/dpaa2/dpaa2_mux.c
+++ b/drivers/net/dpaa2/dpaa2_mux.c
@@ -44,7 +44,7 @@ static struct dpaa2_dpdmux_dev *get_dpdmux_from_id(uint32_t dpdmux_id)
 {
 	struct dpaa2_dpdmux_dev *dpdmux_dev = NULL;
 
-	/* Get DPBP dev handle from list using index */
+	/* Get DPDMUX dev handle from list using index */
 	TAILQ_FOREACH(dpdmux_dev, &dpdmux_dev_list, next) {
 		if (dpdmux_dev->dpdmux_id == dpdmux_id)
 			break;
@@ -261,9 +261,25 @@ dpaa2_create_dpdmux_device(int vdev_fd __rte_unused,
 	return -1;
 }
 
+static void
+dpaa2_close_dpdmux_device(int object_id)
+{
+	struct dpaa2_dpdmux_dev *dpdmux_dev;
+
+	dpdmux_dev = get_dpdmux_from_id((uint32_t)object_id);
+
+	if (dpdmux_dev) {
+		dpdmux_close(&dpdmux_dev->dpdmux, CMD_PRI_LOW,
+			     dpdmux_dev->token);
+		TAILQ_REMOVE(&dpdmux_dev_list, dpdmux_dev, next);
+		rte_free(dpdmux_dev);
+	}
+}
+
 static struct rte_dpaa2_object rte_dpaa2_dpdmux_obj = {
 	.dev_type = DPAA2_MUX,
 	.create = dpaa2_create_dpdmux_device,
+	.close = dpaa2_close_dpdmux_device,
 };
 
 RTE_PMD_REGISTER_DPAA2_OBJECT(dpdmux, rte_dpaa2_dpdmux_obj);
-- 
2.17.1

Re: [dpdk-dev] [PATCH v2 1/3] eal: add API for bus close

[Stephen Hemminger]

On Mon, 24 Aug 2020 13:54:12 +0530
rohit.raj@nxp.com wrote:

> +	if (vbus) {
> +		if (vbus->close) {

Looks good, minor nit.
This could be:
	if (vbus && vbus->close) {

[dpdk-dev] [PATCH v3 1/3] eal: add API for bus close

[rohit.raj]

From: Rohit Raj <rohit.raj@nxp.com>

As per the current code we have API for bus probe, but the
bus close API is missing. This breaks the multi process
scenarios as objects are not cleaned while terminating the
secondary processes.

This patch adds a new API rte_bus_close() for cleanup of
bus objects which were acquired during probe.

Signed-off-by: Rohit Raj <rohit.raj@nxp.com>
---

v3:
* nit: combined nested if statements

v2:
* Moved rte_bus_close call to rte_eal_cleanup path.

 lib/librte_eal/common/eal_common_bus.c | 32 +++++++++++++++++++++++++-
 lib/librte_eal/include/rte_bus.h       | 25 +++++++++++++++++++-
 lib/librte_eal/linux/eal.c             |  1 +
 lib/librte_eal/rte_eal_version.map     |  3 +++
 4 files changed, 59 insertions(+), 2 deletions(-)

diff --git a/lib/librte_eal/common/eal_common_bus.c b/lib/librte_eal/common/eal_common_bus.c
index baa5b532a..5fd7cf6c5 100644
--- a/lib/librte_eal/common/eal_common_bus.c
+++ b/lib/librte_eal/common/eal_common_bus.c
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright 2016 NXP
+ * Copyright 2016,2020 NXP
  */
 
 #include <stdio.h>
@@ -56,6 +56,36 @@ rte_bus_scan(void)
 	return 0;
 }
 
+int
+rte_bus_close(void)
+{
+	int ret;
+	struct rte_bus *bus, *vbus = NULL;
+
+	TAILQ_FOREACH(bus, &rte_bus_list, next) {
+		if (!strcmp(bus->name, "vdev")) {
+			vbus = bus;
+			continue;
+		}
+
+		if (bus->close) {
+			ret = bus->close();
+			if (ret)
+				RTE_LOG(ERR, EAL, "Bus (%s) close failed.\n",
+					bus->name);
+		}
+	}
+
+	if (vbus && vbus->close) {
+		ret = vbus->close();
+		if (ret)
+			RTE_LOG(ERR, EAL, "Bus (%s) close failed.\n",
+				vbus->name);
+	}
+
+	return 0;
+}
+
 /* Probe all devices of all buses */
 int
 rte_bus_probe(void)
diff --git a/lib/librte_eal/include/rte_bus.h b/lib/librte_eal/include/rte_bus.h
index d3034d0ed..af4787b18 100644
--- a/lib/librte_eal/include/rte_bus.h
+++ b/lib/librte_eal/include/rte_bus.h
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright 2016 NXP
+ * Copyright 2016,2020 NXP
  */
 
 #ifndef _RTE_BUS_H_
@@ -67,6 +67,18 @@ typedef int (*rte_bus_scan_t)(void);
  */
 typedef int (*rte_bus_probe_t)(void);
 
+/**
+ * Implementation specific close function which is responsible for closing
+ * devices on that bus.
+ *
+ * This is called while iterating over each registered bus.
+ *
+ * @return
+ *	0 for successful close
+ *	!0 for any error while closing
+ */
+typedef int (*rte_bus_close_t)(void);
+
 /**
  * Device iterator to find a device on a bus.
  *
@@ -248,6 +260,7 @@ struct rte_bus {
 	const char *name;            /**< Name of the bus */
 	rte_bus_scan_t scan;         /**< Scan for devices attached to bus */
 	rte_bus_probe_t probe;       /**< Probe devices on bus */
+	rte_bus_close_t close;       /**< Close devices on bus */
 	rte_bus_find_device_t find_device; /**< Find a device on the bus */
 	rte_bus_plug_t plug;         /**< Probe single device for drivers */
 	rte_bus_unplug_t unplug;     /**< Remove single device from driver */
@@ -301,6 +314,16 @@ int rte_bus_scan(void);
  */
 int rte_bus_probe(void);
 
+/**
+ * For each device on the buses, call the device specific close.
+ *
+ * @return
+ *	 0 for successful close
+ *	!0 otherwise
+ */
+__rte_experimental
+int rte_bus_close(void);
+
 /**
  * Dump information of all the buses registered with EAL.
  *
diff --git a/lib/librte_eal/linux/eal.c b/lib/librte_eal/linux/eal.c
index 0960f01d0..a8a63d8ca 100644
--- a/lib/librte_eal/linux/eal.c
+++ b/lib/librte_eal/linux/eal.c
@@ -1357,6 +1357,7 @@ rte_eal_cleanup(void)
 
 	if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 		rte_memseg_walk(mark_freeable, NULL);
+	rte_bus_close();
 	rte_service_finalize();
 	rte_mp_channel_cleanup();
 	rte_trace_save();
diff --git a/lib/librte_eal/rte_eal_version.map b/lib/librte_eal/rte_eal_version.map
index 0b18e2ef8..dc5266f22 100644
--- a/lib/librte_eal/rte_eal_version.map
+++ b/lib/librte_eal/rte_eal_version.map
@@ -397,6 +397,9 @@ EXPERIMENTAL {
 	rte_mp_disable;
 	rte_thread_register;
 	rte_thread_unregister;
+
+	# added in 20.11
+	rte_bus_close;
 };
 
 INTERNAL {
-- 
2.17.1

[dpdk-dev] [PATCH v3 2/3] examples/multi_process: cleanup bus objects while terminating app

[rohit.raj]

From: Rohit Raj <rohit.raj@nxp.com>

Certain bus objects may need to be closed and re-acquired
while terminating and rerunning the client application.
Hence a signal handler is required to catch the termination
of the App and hence closing the bus objects.

This patch adds the missing signal handler in the client
app and closes the Bus objects in both client and server
applications when the signal Handler is called.

Signed-off-by: Rohit Raj <rohit.raj@nxp.com>
---
 .../multi_process/client_server_mp/mp_client/client.c | 11 +++++++++++
 .../multi_process/client_server_mp/mp_server/main.c   |  4 +++-
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/examples/multi_process/client_server_mp/mp_client/client.c b/examples/multi_process/client_server_mp/mp_client/client.c
index 361d90b54..c37516b4c 100644
--- a/examples/multi_process/client_server_mp/mp_client/client.c
+++ b/examples/multi_process/client_server_mp/mp_client/client.c
@@ -11,6 +11,7 @@
 #include <stdlib.h>
 #include <getopt.h>
 #include <string.h>
+#include <signal.h>
 
 #include <rte_common.h>
 #include <rte_malloc.h>
@@ -196,6 +197,14 @@ handle_packet(struct rte_mbuf *buf)
 
 }
 
+static void
+signal_handler(int signal)
+{
+	if (signal == SIGINT)
+		rte_eal_cleanup();
+	exit(0);
+}
+
 /*
  * Application main function - loops through
  * receiving and processing packets. Never returns
@@ -217,6 +226,8 @@ main(int argc, char *argv[])
 	argc -= retval;
 	argv += retval;
 
+	signal(SIGINT, signal_handler);
+
 	if (parse_app_args(argc, argv) < 0)
 		rte_exit(EXIT_FAILURE, "Invalid command-line arguments\n");
 
diff --git a/examples/multi_process/client_server_mp/mp_server/main.c b/examples/multi_process/client_server_mp/mp_server/main.c
index 280dab867..b0241cc20 100644
--- a/examples/multi_process/client_server_mp/mp_server/main.c
+++ b/examples/multi_process/client_server_mp/mp_server/main.c
@@ -275,11 +275,13 @@ signal_handler(int signal)
 {
 	uint16_t port_id;
 
-	if (signal == SIGINT)
+	if (signal == SIGINT) {
 		RTE_ETH_FOREACH_DEV(port_id) {
 			rte_eth_dev_stop(port_id);
 			rte_eth_dev_close(port_id);
 		}
+		rte_eal_cleanup();
+	}
 	exit(0);
 }
 
-- 
2.17.1

[dpdk-dev] [PATCH v3 3/3] bus/fslmc: support bus close API

[rohit.raj]

From: Rohit Raj <rohit.raj@nxp.com>

This patch add support for closing the bus objects which
were acquired In the bus probe.

Some devices need to be cleaned while in both primary and
secondary process and while some need to be cleaned only in
case of primary process.

The devices are closed as per the white list used while
creating the objects in a particular process.

Signed-off-by: Rohit Raj <rohit.raj@nxp.com>
Acked-by: Sachin Saxena <sachin.saxena@oss.nxp.com>
---
 drivers/bus/fslmc/fslmc_bus.c               | 15 +++-
 drivers/bus/fslmc/fslmc_vfio.c              | 89 ++++++++++++++++++++-
 drivers/bus/fslmc/fslmc_vfio.h              |  3 +-
 drivers/bus/fslmc/mc/fsl_dpcon.h            |  1 +
 drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c    | 31 ++++++-
 drivers/bus/fslmc/portal/dpaa2_hw_dpci.c    | 32 +++++++-
 drivers/bus/fslmc/portal/dpaa2_hw_dpio.c    | 34 ++++++++
 drivers/bus/fslmc/rte_bus_fslmc_version.map |  1 +
 drivers/bus/fslmc/rte_fslmc.h               |  5 +-
 drivers/event/dpaa2/dpaa2_hw_dpcon.c        | 32 +++++++-
 drivers/net/dpaa2/dpaa2_mux.c               | 18 ++++-
 11 files changed, 253 insertions(+), 8 deletions(-)

diff --git a/drivers/bus/fslmc/fslmc_bus.c b/drivers/bus/fslmc/fslmc_bus.c
index beb3dd008..9d3da6be3 100644
--- a/drivers/bus/fslmc/fslmc_bus.c
+++ b/drivers/bus/fslmc/fslmc_bus.c
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
- *   Copyright 2016,2018-2019 NXP
+ *   Copyright 2016,2018-2020 NXP
  *
  */
 
@@ -365,6 +365,18 @@ rte_fslmc_match(struct rte_dpaa2_driver *dpaa2_drv,
 	return 1;
 }
 
+static int
+rte_fslmc_close(void)
+{
+	int ret = 0;
+
+	ret = fslmc_vfio_close_group();
+	if (ret)
+		DPAA2_BUS_ERR("Unable to close devices %d", ret);
+
+	return 0;
+}
+
 static int
 rte_fslmc_probe(void)
 {
@@ -637,6 +649,7 @@ struct rte_fslmc_bus rte_fslmc_bus = {
 	.bus = {
 		.scan = rte_fslmc_scan,
 		.probe = rte_fslmc_probe,
+		.close = rte_fslmc_close,
 		.parse = rte_fslmc_parse,
 		.find_device = rte_fslmc_find_device,
 		.get_iommu_class = rte_dpaa2_get_iommu_class,
diff --git a/drivers/bus/fslmc/fslmc_vfio.c b/drivers/bus/fslmc/fslmc_vfio.c
index 9134ec552..34ccc9db3 100644
--- a/drivers/bus/fslmc/fslmc_vfio.c
+++ b/drivers/bus/fslmc/fslmc_vfio.c
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
  *   Copyright (c) 2015-2016 Freescale Semiconductor, Inc. All rights reserved.
- *   Copyright 2016-2019 NXP
+ *   Copyright 2016-2020 NXP
  *
  */
 
@@ -692,6 +692,54 @@ rte_dpaa2_vfio_setup_intr(struct rte_intr_handle *intr_handle,
 	return -1;
 }
 
+static void
+fslmc_close_iodevices(struct rte_dpaa2_device *dev)
+{
+	struct rte_dpaa2_object *object = NULL;
+	struct rte_dpaa2_driver *drv;
+	int ret, probe_all;
+
+	switch (dev->dev_type) {
+	case DPAA2_IO:
+	case DPAA2_CON:
+	case DPAA2_CI:
+	case DPAA2_BPOOL:
+	case DPAA2_MUX:
+		TAILQ_FOREACH(object, &dpaa2_obj_list, next) {
+			if (dev->dev_type == object->dev_type)
+				object->close(dev->object_id);
+			else
+				continue;
+		}
+		break;
+	case DPAA2_ETH:
+	case DPAA2_CRYPTO:
+	case DPAA2_QDMA:
+		probe_all = rte_fslmc_bus.bus.conf.scan_mode !=
+			    RTE_BUS_SCAN_WHITELIST;
+		TAILQ_FOREACH(drv, &rte_fslmc_bus.driver_list, next) {
+			if (drv->drv_type != dev->dev_type)
+				continue;
+			if (rte_dev_is_probed(&dev->device))
+				continue;
+			if (probe_all ||
+			    (dev->device.devargs &&
+			     dev->device.devargs->policy ==
+			     RTE_DEV_WHITELISTED)) {
+				ret = drv->remove(dev);
+				if (ret)
+					DPAA2_BUS_ERR("Unable to remove");
+			}
+		}
+		break;
+	default:
+		break;
+	}
+
+	DPAA2_BUS_LOG(DEBUG, "Device (%s) Closed",
+		      dev->device.name);
+}
+
 /*
  * fslmc_process_iodevices for processing only IO (ETH, CRYPTO, and possibly
  * EVENT) devices.
@@ -796,6 +844,45 @@ fslmc_process_mcp(struct rte_dpaa2_device *dev)
 	return ret;
 }
 
+int
+fslmc_vfio_close_group(void)
+{
+	struct rte_dpaa2_device *dev, *dev_temp;
+
+	TAILQ_FOREACH_SAFE(dev, &rte_fslmc_bus.device_list, next, dev_temp) {
+		if (dev->device.devargs &&
+		    dev->device.devargs->policy == RTE_DEV_BLACKLISTED) {
+			DPAA2_BUS_LOG(DEBUG, "%s Blacklisted, skipping",
+				      dev->device.name);
+			TAILQ_REMOVE(&rte_fslmc_bus.device_list, dev, next);
+				continue;
+		}
+		switch (dev->dev_type) {
+		case DPAA2_ETH:
+		case DPAA2_CRYPTO:
+		case DPAA2_QDMA:
+		case DPAA2_IO:
+			fslmc_close_iodevices(dev);
+			break;
+		case DPAA2_CON:
+		case DPAA2_CI:
+		case DPAA2_BPOOL:
+		case DPAA2_MUX:
+			if (rte_eal_process_type() == RTE_PROC_SECONDARY)
+				continue;
+
+			fslmc_close_iodevices(dev);
+			break;
+		case DPAA2_DPRTC:
+		default:
+			DPAA2_BUS_DEBUG("Device cannot be closed: Not supported (%s)",
+					dev->device.name);
+		}
+	}
+
+	return 0;
+}
+
 int
 fslmc_vfio_process_group(void)
 {
diff --git a/drivers/bus/fslmc/fslmc_vfio.h b/drivers/bus/fslmc/fslmc_vfio.h
index bc7c6f62d..0a29fe739 100644
--- a/drivers/bus/fslmc/fslmc_vfio.h
+++ b/drivers/bus/fslmc/fslmc_vfio.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
  *   Copyright (c) 2015-2016 Freescale Semiconductor, Inc. All rights reserved.
- *   Copyright 2016,2019 NXP
+ *   Copyright 2016,2019-2020 NXP
  *
  */
 
@@ -55,6 +55,7 @@ int rte_dpaa2_vfio_setup_intr(struct rte_intr_handle *intr_handle,
 
 int fslmc_vfio_setup_group(void);
 int fslmc_vfio_process_group(void);
+int fslmc_vfio_close_group(void);
 char *fslmc_get_container(void);
 int fslmc_get_container_group(int *gropuid);
 int rte_fslmc_vfio_dmamap(void);
diff --git a/drivers/bus/fslmc/mc/fsl_dpcon.h b/drivers/bus/fslmc/mc/fsl_dpcon.h
index 7caa6c68a..8053a6db1 100644
--- a/drivers/bus/fslmc/mc/fsl_dpcon.h
+++ b/drivers/bus/fslmc/mc/fsl_dpcon.h
@@ -26,6 +26,7 @@ int dpcon_open(struct fsl_mc_io *mc_io,
 	       int dpcon_id,
 	       uint16_t *token);
 
+__rte_internal
 int dpcon_close(struct fsl_mc_io *mc_io,
 		uint32_t cmd_flags,
 		uint16_t token);
diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c
index d9619848d..9a4cdfd7a 100644
--- a/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c
+++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
  *   Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
- *   Copyright 2016 NXP
+ *   Copyright 2016,2020 NXP
  *
  */
 
@@ -33,6 +33,19 @@ TAILQ_HEAD(dpbp_dev_list, dpaa2_dpbp_dev);
 static struct dpbp_dev_list dpbp_dev_list
 	= TAILQ_HEAD_INITIALIZER(dpbp_dev_list); /*!< DPBP device list */
 
+static struct dpaa2_dpbp_dev *get_dpbp_from_id(uint32_t dpbp_id)
+{
+	struct dpaa2_dpbp_dev *dpbp_dev = NULL;
+
+	/* Get DPBP dev handle from list using index */
+	TAILQ_FOREACH(dpbp_dev, &dpbp_dev_list, next) {
+		if (dpbp_dev->dpbp_id == dpbp_id)
+			break;
+	}
+
+	return dpbp_dev;
+}
+
 static int
 dpaa2_create_dpbp_device(int vdev_fd __rte_unused,
 			 struct vfio_device_info *obj_info __rte_unused,
@@ -116,9 +129,25 @@ int dpaa2_dpbp_supported(void)
 	return 0;
 }
 
+static void
+dpaa2_close_dpbp_device(int object_id)
+{
+	struct dpaa2_dpbp_dev *dpbp_dev = NULL;
+
+	dpbp_dev = get_dpbp_from_id((uint32_t)object_id);
+
+	if (dpbp_dev) {
+		dpaa2_free_dpbp_dev(dpbp_dev);
+		dpbp_close(&dpbp_dev->dpbp, CMD_PRI_LOW, dpbp_dev->token);
+		TAILQ_REMOVE(&dpbp_dev_list, dpbp_dev, next);
+		rte_free(dpbp_dev);
+	}
+}
+
 static struct rte_dpaa2_object rte_dpaa2_dpbp_obj = {
 	.dev_type = DPAA2_BPOOL,
 	.create = dpaa2_create_dpbp_device,
+	.close = dpaa2_close_dpbp_device,
 };
 
 RTE_PMD_REGISTER_DPAA2_OBJECT(dpbp, rte_dpaa2_dpbp_obj);
diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c
index d393ce618..9902f38ce 100644
--- a/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c
+++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
- *   Copyright 2017 NXP
+ *   Copyright 2017,2020 NXP
  *
  */
 
@@ -30,6 +30,19 @@ TAILQ_HEAD(dpci_dev_list, dpaa2_dpci_dev);
 static struct dpci_dev_list dpci_dev_list
 	= TAILQ_HEAD_INITIALIZER(dpci_dev_list); /*!< DPCI device list */
 
+static struct dpaa2_dpci_dev *get_dpci_from_id(uint32_t dpci_id)
+{
+	struct dpaa2_dpci_dev *dpci_dev = NULL;
+
+	/* Get DPCI dev handle from list using index */
+	TAILQ_FOREACH(dpci_dev, &dpci_dev_list, next) {
+		if (dpci_dev->dpci_id == dpci_id)
+			break;
+	}
+
+	return dpci_dev;
+}
+
 static int
 rte_dpaa2_create_dpci_device(int vdev_fd __rte_unused,
 			     struct vfio_device_info *obj_info __rte_unused,
@@ -179,9 +192,26 @@ void rte_dpaa2_free_dpci_dev(struct dpaa2_dpci_dev *dpci)
 	}
 }
 
+
+static void
+rte_dpaa2_close_dpci_device(int object_id)
+{
+	struct dpaa2_dpci_dev *dpci_dev = NULL;
+
+	dpci_dev = get_dpci_from_id((uint32_t)object_id);
+
+	if (dpci_dev) {
+		rte_dpaa2_free_dpci_dev(dpci_dev);
+		dpci_close(&dpci_dev->dpci, CMD_PRI_LOW, dpci_dev->token);
+		TAILQ_REMOVE(&dpci_dev_list, dpci_dev, next);
+		rte_free(dpci_dev);
+	}
+}
+
 static struct rte_dpaa2_object rte_dpaa2_dpci_obj = {
 	.dev_type = DPAA2_CI,
 	.create = rte_dpaa2_create_dpci_device,
+	.close = rte_dpaa2_close_dpci_device,
 };
 
 RTE_PMD_REGISTER_DPAA2_OBJECT(dpci, rte_dpaa2_dpci_obj);
diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
index 97be76116..9563cd620 100644
--- a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
+++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
@@ -86,6 +86,19 @@ static int dpaa2_cluster_sz = 2;
  * Cluster 4 (ID = x07) : CPU14, CPU15;
  */
 
+static struct dpaa2_dpio_dev *get_dpio_dev_from_id(int32_t dpio_id)
+{
+	struct dpaa2_dpio_dev *dpio_dev = NULL;
+
+	/* Get DPIO dev handle from list using index */
+	TAILQ_FOREACH(dpio_dev, &dpio_dev_list, next) {
+		if (dpio_dev->hw_id == dpio_id)
+			break;
+	}
+
+	return dpio_dev;
+}
+
 static int
 dpaa2_get_core_id(void)
 {
@@ -360,6 +373,26 @@ static void dpaa2_portal_finish(void *arg)
 	pthread_setspecific(dpaa2_portal_key, NULL);
 }
 
+static void
+dpaa2_close_dpio_device(int object_id)
+{
+	struct dpaa2_dpio_dev *dpio_dev = NULL;
+
+	dpio_dev = get_dpio_dev_from_id((int32_t)object_id);
+
+	if (dpio_dev) {
+		if (dpio_dev->dpio) {
+			dpio_disable(dpio_dev->dpio, CMD_PRI_LOW,
+				     dpio_dev->token);
+			dpio_close(dpio_dev->dpio, CMD_PRI_LOW,
+				   dpio_dev->token);
+			rte_free(dpio_dev->dpio);
+		}
+		TAILQ_REMOVE(&dpio_dev_list, dpio_dev, next);
+		rte_free(dpio_dev);
+	}
+}
+
 static int
 dpaa2_create_dpio_device(int vdev_fd,
 			 struct vfio_device_info *obj_info,
@@ -623,6 +656,7 @@ dpaa2_free_eq_descriptors(void)
 static struct rte_dpaa2_object rte_dpaa2_dpio_obj = {
 	.dev_type = DPAA2_IO,
 	.create = dpaa2_create_dpio_device,
+	.close = dpaa2_close_dpio_device,
 };
 
 RTE_PMD_REGISTER_DPAA2_OBJECT(dpio, rte_dpaa2_dpio_obj);
diff --git a/drivers/bus/fslmc/rte_bus_fslmc_version.map b/drivers/bus/fslmc/rte_bus_fslmc_version.map
index b169f5228..ac4ee2bf8 100644
--- a/drivers/bus/fslmc/rte_bus_fslmc_version.map
+++ b/drivers/bus/fslmc/rte_bus_fslmc_version.map
@@ -30,6 +30,7 @@ INTERNAL {
 	dpci_get_opr;
 	dpci_set_opr;
 	dpci_set_rx_queue;
+	dpcon_close;
 	dpcon_get_attributes;
 	dpcon_open;
 	dpdmai_close;
diff --git a/drivers/bus/fslmc/rte_fslmc.h b/drivers/bus/fslmc/rte_fslmc.h
index 80873fffc..a938ab8f9 100644
--- a/drivers/bus/fslmc/rte_fslmc.h
+++ b/drivers/bus/fslmc/rte_fslmc.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
- *   Copyright 2016,2019 NXP
+ *   Copyright 2016,2019-2020 NXP
  *
  */
 
@@ -79,6 +79,8 @@ typedef int (*rte_dpaa2_obj_create_t)(int vdev_fd,
 				      struct vfio_device_info *obj_info,
 				      int object_id);
 
+typedef void (*rte_dpaa2_obj_close_t)(int object_id);
+
 /**
  * A structure describing a DPAA2 object.
  */
@@ -87,6 +89,7 @@ struct rte_dpaa2_object {
 	const char *name;                   /**< Name of Object. */
 	enum rte_dpaa2_dev_type dev_type;   /**< Type of device */
 	rte_dpaa2_obj_create_t create;
+	rte_dpaa2_obj_close_t close;
 };
 
 /**
diff --git a/drivers/event/dpaa2/dpaa2_hw_dpcon.c b/drivers/event/dpaa2/dpaa2_hw_dpcon.c
index 200b71640..86844f49f 100644
--- a/drivers/event/dpaa2/dpaa2_hw_dpcon.c
+++ b/drivers/event/dpaa2/dpaa2_hw_dpcon.c
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
- *   Copyright 2017 NXP
+ *   Copyright 2017,2020 NXP
  *
  */
 
@@ -30,6 +30,19 @@ TAILQ_HEAD(dpcon_dev_list, dpaa2_dpcon_dev);
 static struct dpcon_dev_list dpcon_dev_list
 	= TAILQ_HEAD_INITIALIZER(dpcon_dev_list); /*!< DPCON device list */
 
+static struct dpaa2_dpcon_dev *get_dpcon_from_id(uint32_t dpcon_id)
+{
+	struct dpaa2_dpcon_dev *dpcon_dev = NULL;
+
+	/* Get DPCONC dev handle from list using index */
+	TAILQ_FOREACH(dpcon_dev, &dpcon_dev_list, next) {
+		if (dpcon_dev->dpcon_id == dpcon_id)
+			break;
+	}
+
+	return dpcon_dev;
+}
+
 static int
 rte_dpaa2_create_dpcon_device(int dev_fd __rte_unused,
 			      struct vfio_device_info *obj_info __rte_unused,
@@ -105,9 +118,26 @@ void rte_dpaa2_free_dpcon_dev(struct dpaa2_dpcon_dev *dpcon)
 	}
 }
 
+
+static void
+rte_dpaa2_close_dpcon_device(int object_id)
+{
+	struct dpaa2_dpcon_dev *dpcon_dev = NULL;
+
+	dpcon_dev = get_dpcon_from_id((uint32_t)object_id);
+
+	if (dpcon_dev) {
+		rte_dpaa2_free_dpcon_dev(dpcon_dev);
+		dpcon_close(&dpcon_dev->dpcon, CMD_PRI_LOW, dpcon_dev->token);
+		TAILQ_REMOVE(&dpcon_dev_list, dpcon_dev, next);
+		rte_free(dpcon_dev);
+	}
+}
+
 static struct rte_dpaa2_object rte_dpaa2_dpcon_obj = {
 	.dev_type = DPAA2_CON,
 	.create = rte_dpaa2_create_dpcon_device,
+	.close = rte_dpaa2_close_dpcon_device,
 };
 
 RTE_PMD_REGISTER_DPAA2_OBJECT(dpcon, rte_dpaa2_dpcon_obj);
diff --git a/drivers/net/dpaa2/dpaa2_mux.c b/drivers/net/dpaa2/dpaa2_mux.c
index f8366e839..15114dac9 100644
--- a/drivers/net/dpaa2/dpaa2_mux.c
+++ b/drivers/net/dpaa2/dpaa2_mux.c
@@ -44,7 +44,7 @@ static struct dpaa2_dpdmux_dev *get_dpdmux_from_id(uint32_t dpdmux_id)
 {
 	struct dpaa2_dpdmux_dev *dpdmux_dev = NULL;
 
-	/* Get DPBP dev handle from list using index */
+	/* Get DPDMUX dev handle from list using index */
 	TAILQ_FOREACH(dpdmux_dev, &dpdmux_dev_list, next) {
 		if (dpdmux_dev->dpdmux_id == dpdmux_id)
 			break;
@@ -261,9 +261,25 @@ dpaa2_create_dpdmux_device(int vdev_fd __rte_unused,
 	return -1;
 }
 
+static void
+dpaa2_close_dpdmux_device(int object_id)
+{
+	struct dpaa2_dpdmux_dev *dpdmux_dev;
+
+	dpdmux_dev = get_dpdmux_from_id((uint32_t)object_id);
+
+	if (dpdmux_dev) {
+		dpdmux_close(&dpdmux_dev->dpdmux, CMD_PRI_LOW,
+			     dpdmux_dev->token);
+		TAILQ_REMOVE(&dpdmux_dev_list, dpdmux_dev, next);
+		rte_free(dpdmux_dev);
+	}
+}
+
 static struct rte_dpaa2_object rte_dpaa2_dpdmux_obj = {
 	.dev_type = DPAA2_MUX,
 	.create = dpaa2_create_dpdmux_device,
+	.close = dpaa2_close_dpdmux_device,
 };
 
 RTE_PMD_REGISTER_DPAA2_OBJECT(dpdmux, rte_dpaa2_dpdmux_obj);
-- 
2.17.1

Re: [dpdk-dev] [PATCH v3 1/3] eal: add API for bus close

[Hemant Agrawal]

 -----Original Message-----
> From: dev <dev-bounces@dpdk.org> On Behalf Of rohit.raj@nxp.com
> Sent: Wednesday, August 26, 2020 11:23 AM
> To: Ray Kinsella <mdr@ashroe.eu>; Neil Horman <nhorman@tuxdriver.com>
> Cc: dev@dpdk.org; Rohit Raj <rohit.raj@nxp.com>
> Subject: [dpdk-dev] [PATCH v3 1/3] eal: add API for bus close
> 
> From: Rohit Raj <rohit.raj@nxp.com>
> 
> As per the current code we have API for bus probe, but the bus close API is
> missing. This breaks the multi process scenarios as objects are not cleaned
> while terminating the secondary processes.
> 
> This patch adds a new API rte_bus_close() for cleanup of bus objects which
> were acquired during probe.
> 
> Signed-off-by: Rohit Raj <rohit.raj@nxp.com>

Acked-by: Hemant Agrawal <hemant.agrawal@nxp.com>

Re: [dpdk-dev] [PATCH v2 2/3] examples/multi_process: cleanup bus objects while terminating app

[Hemant Agrawal]

Acked-by: Hemant Agrawal <hemant.agrawal@nxp.com>

Re: [dpdk-dev] [PATCH v3 2/3] examples/multi_process: cleanup bus objects while terminating app

[Hemant Agrawal]

Acked-by: Hemant Agrawal <hemant.agrawal@nxp.com>

Re: [dpdk-dev] [PATCH v3 3/3] bus/fslmc: support bus close API

[Hemant Agrawal (OSS)]

Acked-by: Hemant Agrawal <hemant.agrawal@nxp.com>

Re: [dpdk-dev] [PATCH v3 1/3] eal: add API for bus close

[Kinsella, Ray]

On 26/08/2020 06:52, rohit.raj@nxp.com wrote:
> From: Rohit Raj <rohit.raj@nxp.com>
> 
> As per the current code we have API for bus probe, but the
> bus close API is missing. This breaks the multi process
> scenarios as objects are not cleaned while terminating the
> secondary processes.
> 
> This patch adds a new API rte_bus_close() for cleanup of
> bus objects which were acquired during probe.
> 
> Signed-off-by: Rohit Raj <rohit.raj@nxp.com>
> ---
> 
> v3:
> * nit: combined nested if statements
> 
> v2:
> * Moved rte_bus_close call to rte_eal_cleanup path.
> 
>  lib/librte_eal/common/eal_common_bus.c | 32 +++++++++++++++++++++++++-
>  lib/librte_eal/include/rte_bus.h       | 25 +++++++++++++++++++-
>  lib/librte_eal/linux/eal.c             |  1 +
>  lib/librte_eal/rte_eal_version.map     |  3 +++
>  4 files changed, 59 insertions(+), 2 deletions(-)
> 
> diff --git a/lib/librte_eal/common/eal_common_bus.c b/lib/librte_eal/common/eal_common_bus.c
> index baa5b532a..5fd7cf6c5 100644
> --- a/lib/librte_eal/common/eal_common_bus.c
> +++ b/lib/librte_eal/common/eal_common_bus.c
> @@ -1,5 +1,5 @@
>  /* SPDX-License-Identifier: BSD-3-Clause
> - * Copyright 2016 NXP
> + * Copyright 2016,2020 NXP
>   */
>  
>  #include <stdio.h>
> @@ -56,6 +56,36 @@ rte_bus_scan(void)
>  	return 0;
>  }
>  
> +int
> +rte_bus_close(void)
> +{
> +	int ret;
> +	struct rte_bus *bus, *vbus = NULL;
> +
> +	TAILQ_FOREACH(bus, &rte_bus_list, next) {
> +		if (!strcmp(bus->name, "vdev")) {
> +			vbus = bus;
> +			continue;
> +		}
> +
> +		if (bus->close) {
> +			ret = bus->close();
> +			if (ret)
> +				RTE_LOG(ERR, EAL, "Bus (%s) close failed.\n",
> +					bus->name);
> +		}
> +	}
> +
> +	if (vbus && vbus->close) {
> +		ret = vbus->close();
> +		if (ret)
> +			RTE_LOG(ERR, EAL, "Bus (%s) close failed.\n",
> +				vbus->name);
> +	}
> +
> +	return 0;
> +}
> +
>  /* Probe all devices of all buses */
>  int
>  rte_bus_probe(void)
> diff --git a/lib/librte_eal/include/rte_bus.h b/lib/librte_eal/include/rte_bus.h
> index d3034d0ed..af4787b18 100644
> --- a/lib/librte_eal/include/rte_bus.h
> +++ b/lib/librte_eal/include/rte_bus.h
> @@ -1,5 +1,5 @@
>  /* SPDX-License-Identifier: BSD-3-Clause
> - * Copyright 2016 NXP
> + * Copyright 2016,2020 NXP
>   */
>  
>  #ifndef _RTE_BUS_H_
> @@ -67,6 +67,18 @@ typedef int (*rte_bus_scan_t)(void);
>   */
>  typedef int (*rte_bus_probe_t)(void);
>  
> +/**
> + * Implementation specific close function which is responsible for closing
> + * devices on that bus.
> + *
> + * This is called while iterating over each registered bus.
> + *
> + * @return
> + *	0 for successful close
> + *	!0 for any error while closing
> + */
> +typedef int (*rte_bus_close_t)(void);
> +
>  /**
>   * Device iterator to find a device on a bus.
>   *
> @@ -248,6 +260,7 @@ struct rte_bus {
>  	const char *name;            /**< Name of the bus */
>  	rte_bus_scan_t scan;         /**< Scan for devices attached to bus */
>  	rte_bus_probe_t probe;       /**< Probe devices on bus */
> +	rte_bus_close_t close;       /**< Close devices on bus */
>  	rte_bus_find_device_t find_device; /**< Find a device on the bus */
>  	rte_bus_plug_t plug;         /**< Probe single device for drivers */
>  	rte_bus_unplug_t unplug;     /**< Remove single device from driver */
> @@ -301,6 +314,16 @@ int rte_bus_scan(void);
>   */
>  int rte_bus_probe(void);
>  
> +/**
> + * For each device on the buses, call the device specific close.
> + *
> + * @return
> + *	 0 for successful close
> + *	!0 otherwise
> + */
> +__rte_experimental
> +int rte_bus_close(void);
> +
>  /**
>   * Dump information of all the buses registered with EAL.
>   *
> diff --git a/lib/librte_eal/linux/eal.c b/lib/librte_eal/linux/eal.c
> index 0960f01d0..a8a63d8ca 100644
> --- a/lib/librte_eal/linux/eal.c
> +++ b/lib/librte_eal/linux/eal.c
> @@ -1357,6 +1357,7 @@ rte_eal_cleanup(void)
>  
>  	if (rte_eal_process_type() == RTE_PROC_PRIMARY)
>  		rte_memseg_walk(mark_freeable, NULL);
> +	rte_bus_close();
>  	rte_service_finalize();
>  	rte_mp_channel_cleanup();
>  	rte_trace_save();
> diff --git a/lib/librte_eal/rte_eal_version.map b/lib/librte_eal/rte_eal_version.map
> index 0b18e2ef8..dc5266f22 100644
> --- a/lib/librte_eal/rte_eal_version.map
> +++ b/lib/librte_eal/rte_eal_version.map
> @@ -397,6 +397,9 @@ EXPERIMENTAL {
>  	rte_mp_disable;
>  	rte_thread_register;
>  	rte_thread_unregister;
> +
> +	# added in 20.11
> +	rte_bus_close;
>  };
>  
>  INTERNAL {
> 

Putting EXPERITMENAL on such a trivial API feels like overkill, however ...

Acked-by: Ray Kinsella <mdr@ashroe.eu>

Re: [dpdk-dev] [PATCH v3 3/3] bus/fslmc: support bus close API

[Kinsella, Ray]

On 26/08/2020 06:52, rohit.raj@nxp.com wrote:
> From: Rohit Raj <rohit.raj@nxp.com>
> 
> This patch add support for closing the bus objects which
> were acquired In the bus probe.
> 
> Some devices need to be cleaned while in both primary and
> secondary process and while some need to be cleaned only in
> case of primary process.
> 
> The devices are closed as per the white list used while
> creating the objects in a particular process.
> 
> Signed-off-by: Rohit Raj <rohit.raj@nxp.com>
> Acked-by: Sachin Saxena <sachin.saxena@oss.nxp.com>
> ---
>  drivers/bus/fslmc/fslmc_bus.c               | 15 +++-
>  drivers/bus/fslmc/fslmc_vfio.c              | 89 ++++++++++++++++++++-
>  drivers/bus/fslmc/fslmc_vfio.h              |  3 +-
>  drivers/bus/fslmc/mc/fsl_dpcon.h            |  1 +
>  drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c    | 31 ++++++-
>  drivers/bus/fslmc/portal/dpaa2_hw_dpci.c    | 32 +++++++-
>  drivers/bus/fslmc/portal/dpaa2_hw_dpio.c    | 34 ++++++++
>  drivers/bus/fslmc/rte_bus_fslmc_version.map |  1 +
>  drivers/bus/fslmc/rte_fslmc.h               |  5 +-
>  drivers/event/dpaa2/dpaa2_hw_dpcon.c        | 32 +++++++-
>  drivers/net/dpaa2/dpaa2_mux.c               | 18 ++++-
>  11 files changed, 253 insertions(+), 8 deletions(-)
> 
> diff --git a/drivers/bus/fslmc/fslmc_bus.c b/drivers/bus/fslmc/fslmc_bus.c
> index beb3dd008..9d3da6be3 100644
> --- a/drivers/bus/fslmc/fslmc_bus.c
> +++ b/drivers/bus/fslmc/fslmc_bus.c
> @@ -1,6 +1,6 @@
>  /* SPDX-License-Identifier: BSD-3-Clause
>   *
> - *   Copyright 2016,2018-2019 NXP
> + *   Copyright 2016,2018-2020 NXP
>   *
>   */
>  
> @@ -365,6 +365,18 @@ rte_fslmc_match(struct rte_dpaa2_driver *dpaa2_drv,
>  	return 1;
>  }
>  
> +static int
> +rte_fslmc_close(void)
> +{
> +	int ret = 0;
> +
> +	ret = fslmc_vfio_close_group();
> +	if (ret)
> +		DPAA2_BUS_ERR("Unable to close devices %d", ret);
> +
> +	return 0;
> +}
> +
>  static int
>  rte_fslmc_probe(void)
>  {
> @@ -637,6 +649,7 @@ struct rte_fslmc_bus rte_fslmc_bus = {
>  	.bus = {
>  		.scan = rte_fslmc_scan,
>  		.probe = rte_fslmc_probe,
> +		.close = rte_fslmc_close,
>  		.parse = rte_fslmc_parse,
>  		.find_device = rte_fslmc_find_device,
>  		.get_iommu_class = rte_dpaa2_get_iommu_class,
> diff --git a/drivers/bus/fslmc/fslmc_vfio.c b/drivers/bus/fslmc/fslmc_vfio.c
> index 9134ec552..34ccc9db3 100644
> --- a/drivers/bus/fslmc/fslmc_vfio.c
> +++ b/drivers/bus/fslmc/fslmc_vfio.c
> @@ -1,7 +1,7 @@
>  /* SPDX-License-Identifier: BSD-3-Clause
>   *
>   *   Copyright (c) 2015-2016 Freescale Semiconductor, Inc. All rights reserved.
> - *   Copyright 2016-2019 NXP
> + *   Copyright 2016-2020 NXP
>   *
>   */
>  
> @@ -692,6 +692,54 @@ rte_dpaa2_vfio_setup_intr(struct rte_intr_handle *intr_handle,
>  	return -1;
>  }
>  
> +static void
> +fslmc_close_iodevices(struct rte_dpaa2_device *dev)
> +{
> +	struct rte_dpaa2_object *object = NULL;
> +	struct rte_dpaa2_driver *drv;
> +	int ret, probe_all;
> +
> +	switch (dev->dev_type) {
> +	case DPAA2_IO:
> +	case DPAA2_CON:
> +	case DPAA2_CI:
> +	case DPAA2_BPOOL:
> +	case DPAA2_MUX:
> +		TAILQ_FOREACH(object, &dpaa2_obj_list, next) {
> +			if (dev->dev_type == object->dev_type)
> +				object->close(dev->object_id);
> +			else
> +				continue;
> +		}
> +		break;
> +	case DPAA2_ETH:
> +	case DPAA2_CRYPTO:
> +	case DPAA2_QDMA:
> +		probe_all = rte_fslmc_bus.bus.conf.scan_mode !=
> +			    RTE_BUS_SCAN_WHITELIST;
> +		TAILQ_FOREACH(drv, &rte_fslmc_bus.driver_list, next) {
> +			if (drv->drv_type != dev->dev_type)
> +				continue;
> +			if (rte_dev_is_probed(&dev->device))
> +				continue;
> +			if (probe_all ||
> +			    (dev->device.devargs &&
> +			     dev->device.devargs->policy ==
> +			     RTE_DEV_WHITELISTED)) {
> +				ret = drv->remove(dev);
> +				if (ret)
> +					DPAA2_BUS_ERR("Unable to remove");
> +			}
> +		}
> +		break;
> +	default:
> +		break;
> +	}
> +
> +	DPAA2_BUS_LOG(DEBUG, "Device (%s) Closed",
> +		      dev->device.name);
> +}
> +
>  /*
>   * fslmc_process_iodevices for processing only IO (ETH, CRYPTO, and possibly
>   * EVENT) devices.
> @@ -796,6 +844,45 @@ fslmc_process_mcp(struct rte_dpaa2_device *dev)
>  	return ret;
>  }
>  
> +int
> +fslmc_vfio_close_group(void)
> +{
> +	struct rte_dpaa2_device *dev, *dev_temp;
> +
> +	TAILQ_FOREACH_SAFE(dev, &rte_fslmc_bus.device_list, next, dev_temp) {
> +		if (dev->device.devargs &&
> +		    dev->device.devargs->policy == RTE_DEV_BLACKLISTED) {
> +			DPAA2_BUS_LOG(DEBUG, "%s Blacklisted, skipping",
> +				      dev->device.name);
> +			TAILQ_REMOVE(&rte_fslmc_bus.device_list, dev, next);
> +				continue;
> +		}
> +		switch (dev->dev_type) {
> +		case DPAA2_ETH:
> +		case DPAA2_CRYPTO:
> +		case DPAA2_QDMA:
> +		case DPAA2_IO:
> +			fslmc_close_iodevices(dev);
> +			break;
> +		case DPAA2_CON:
> +		case DPAA2_CI:
> +		case DPAA2_BPOOL:
> +		case DPAA2_MUX:
> +			if (rte_eal_process_type() == RTE_PROC_SECONDARY)
> +				continue;
> +
> +			fslmc_close_iodevices(dev);
> +			break;
> +		case DPAA2_DPRTC:
> +		default:
> +			DPAA2_BUS_DEBUG("Device cannot be closed: Not supported (%s)",
> +					dev->device.name);
> +		}
> +	}
> +
> +	return 0;
> +}
> +
>  int
>  fslmc_vfio_process_group(void)
>  {
> diff --git a/drivers/bus/fslmc/fslmc_vfio.h b/drivers/bus/fslmc/fslmc_vfio.h
> index bc7c6f62d..0a29fe739 100644
> --- a/drivers/bus/fslmc/fslmc_vfio.h
> +++ b/drivers/bus/fslmc/fslmc_vfio.h
> @@ -1,7 +1,7 @@
>  /* SPDX-License-Identifier: BSD-3-Clause
>   *
>   *   Copyright (c) 2015-2016 Freescale Semiconductor, Inc. All rights reserved.
> - *   Copyright 2016,2019 NXP
> + *   Copyright 2016,2019-2020 NXP
>   *
>   */
>  
> @@ -55,6 +55,7 @@ int rte_dpaa2_vfio_setup_intr(struct rte_intr_handle *intr_handle,
>  
>  int fslmc_vfio_setup_group(void);
>  int fslmc_vfio_process_group(void);
> +int fslmc_vfio_close_group(void);
>  char *fslmc_get_container(void);
>  int fslmc_get_container_group(int *gropuid);
>  int rte_fslmc_vfio_dmamap(void);
> diff --git a/drivers/bus/fslmc/mc/fsl_dpcon.h b/drivers/bus/fslmc/mc/fsl_dpcon.h
> index 7caa6c68a..8053a6db1 100644
> --- a/drivers/bus/fslmc/mc/fsl_dpcon.h
> +++ b/drivers/bus/fslmc/mc/fsl_dpcon.h
> @@ -26,6 +26,7 @@ int dpcon_open(struct fsl_mc_io *mc_io,
>  	       int dpcon_id,
>  	       uint16_t *token);
>  
> +__rte_internal
>  int dpcon_close(struct fsl_mc_io *mc_io,
>  		uint32_t cmd_flags,
>  		uint16_t token);
> diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c
> index d9619848d..9a4cdfd7a 100644
> --- a/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c
> +++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c
> @@ -1,7 +1,7 @@
>  /* SPDX-License-Identifier: BSD-3-Clause
>   *
>   *   Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
> - *   Copyright 2016 NXP
> + *   Copyright 2016,2020 NXP
>   *
>   */
>  
> @@ -33,6 +33,19 @@ TAILQ_HEAD(dpbp_dev_list, dpaa2_dpbp_dev);
>  static struct dpbp_dev_list dpbp_dev_list
>  	= TAILQ_HEAD_INITIALIZER(dpbp_dev_list); /*!< DPBP device list */
>  
> +static struct dpaa2_dpbp_dev *get_dpbp_from_id(uint32_t dpbp_id)
> +{
> +	struct dpaa2_dpbp_dev *dpbp_dev = NULL;
> +
> +	/* Get DPBP dev handle from list using index */
> +	TAILQ_FOREACH(dpbp_dev, &dpbp_dev_list, next) {
> +		if (dpbp_dev->dpbp_id == dpbp_id)
> +			break;
> +	}
> +
> +	return dpbp_dev;
> +}
> +
>  static int
>  dpaa2_create_dpbp_device(int vdev_fd __rte_unused,
>  			 struct vfio_device_info *obj_info __rte_unused,
> @@ -116,9 +129,25 @@ int dpaa2_dpbp_supported(void)
>  	return 0;
>  }
>  
> +static void
> +dpaa2_close_dpbp_device(int object_id)
> +{
> +	struct dpaa2_dpbp_dev *dpbp_dev = NULL;
> +
> +	dpbp_dev = get_dpbp_from_id((uint32_t)object_id);
> +
> +	if (dpbp_dev) {
> +		dpaa2_free_dpbp_dev(dpbp_dev);
> +		dpbp_close(&dpbp_dev->dpbp, CMD_PRI_LOW, dpbp_dev->token);
> +		TAILQ_REMOVE(&dpbp_dev_list, dpbp_dev, next);
> +		rte_free(dpbp_dev);
> +	}
> +}
> +
>  static struct rte_dpaa2_object rte_dpaa2_dpbp_obj = {
>  	.dev_type = DPAA2_BPOOL,
>  	.create = dpaa2_create_dpbp_device,
> +	.close = dpaa2_close_dpbp_device,
>  };
>  
>  RTE_PMD_REGISTER_DPAA2_OBJECT(dpbp, rte_dpaa2_dpbp_obj);
> diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c
> index d393ce618..9902f38ce 100644
> --- a/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c
> +++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c
> @@ -1,6 +1,6 @@
>  /* SPDX-License-Identifier: BSD-3-Clause
>   *
> - *   Copyright 2017 NXP
> + *   Copyright 2017,2020 NXP
>   *
>   */
>  
> @@ -30,6 +30,19 @@ TAILQ_HEAD(dpci_dev_list, dpaa2_dpci_dev);
>  static struct dpci_dev_list dpci_dev_list
>  	= TAILQ_HEAD_INITIALIZER(dpci_dev_list); /*!< DPCI device list */
>  
> +static struct dpaa2_dpci_dev *get_dpci_from_id(uint32_t dpci_id)
> +{
> +	struct dpaa2_dpci_dev *dpci_dev = NULL;
> +
> +	/* Get DPCI dev handle from list using index */
> +	TAILQ_FOREACH(dpci_dev, &dpci_dev_list, next) {
> +		if (dpci_dev->dpci_id == dpci_id)
> +			break;
> +	}
> +
> +	return dpci_dev;
> +}
> +
>  static int
>  rte_dpaa2_create_dpci_device(int vdev_fd __rte_unused,
>  			     struct vfio_device_info *obj_info __rte_unused,
> @@ -179,9 +192,26 @@ void rte_dpaa2_free_dpci_dev(struct dpaa2_dpci_dev *dpci)
>  	}
>  }
>  
> +
> +static void
> +rte_dpaa2_close_dpci_device(int object_id)
> +{
> +	struct dpaa2_dpci_dev *dpci_dev = NULL;
> +
> +	dpci_dev = get_dpci_from_id((uint32_t)object_id);
> +
> +	if (dpci_dev) {
> +		rte_dpaa2_free_dpci_dev(dpci_dev);
> +		dpci_close(&dpci_dev->dpci, CMD_PRI_LOW, dpci_dev->token);
> +		TAILQ_REMOVE(&dpci_dev_list, dpci_dev, next);
> +		rte_free(dpci_dev);
> +	}
> +}
> +
>  static struct rte_dpaa2_object rte_dpaa2_dpci_obj = {
>  	.dev_type = DPAA2_CI,
>  	.create = rte_dpaa2_create_dpci_device,
> +	.close = rte_dpaa2_close_dpci_device,
>  };
>  
>  RTE_PMD_REGISTER_DPAA2_OBJECT(dpci, rte_dpaa2_dpci_obj);
> diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
> index 97be76116..9563cd620 100644
> --- a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
> +++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
> @@ -86,6 +86,19 @@ static int dpaa2_cluster_sz = 2;
>   * Cluster 4 (ID = x07) : CPU14, CPU15;
>   */
>  
> +static struct dpaa2_dpio_dev *get_dpio_dev_from_id(int32_t dpio_id)
> +{
> +	struct dpaa2_dpio_dev *dpio_dev = NULL;
> +
> +	/* Get DPIO dev handle from list using index */
> +	TAILQ_FOREACH(dpio_dev, &dpio_dev_list, next) {
> +		if (dpio_dev->hw_id == dpio_id)
> +			break;
> +	}
> +
> +	return dpio_dev;
> +}
> +
>  static int
>  dpaa2_get_core_id(void)
>  {
> @@ -360,6 +373,26 @@ static void dpaa2_portal_finish(void *arg)
>  	pthread_setspecific(dpaa2_portal_key, NULL);
>  }
>  
> +static void
> +dpaa2_close_dpio_device(int object_id)
> +{
> +	struct dpaa2_dpio_dev *dpio_dev = NULL;
> +
> +	dpio_dev = get_dpio_dev_from_id((int32_t)object_id);
> +
> +	if (dpio_dev) {
> +		if (dpio_dev->dpio) {
> +			dpio_disable(dpio_dev->dpio, CMD_PRI_LOW,
> +				     dpio_dev->token);
> +			dpio_close(dpio_dev->dpio, CMD_PRI_LOW,
> +				   dpio_dev->token);
> +			rte_free(dpio_dev->dpio);
> +		}
> +		TAILQ_REMOVE(&dpio_dev_list, dpio_dev, next);
> +		rte_free(dpio_dev);
> +	}
> +}
> +
>  static int
>  dpaa2_create_dpio_device(int vdev_fd,
>  			 struct vfio_device_info *obj_info,
> @@ -623,6 +656,7 @@ dpaa2_free_eq_descriptors(void)
>  static struct rte_dpaa2_object rte_dpaa2_dpio_obj = {
>  	.dev_type = DPAA2_IO,
>  	.create = dpaa2_create_dpio_device,
> +	.close = dpaa2_close_dpio_device,
>  };
>  
>  RTE_PMD_REGISTER_DPAA2_OBJECT(dpio, rte_dpaa2_dpio_obj);
> diff --git a/drivers/bus/fslmc/rte_bus_fslmc_version.map b/drivers/bus/fslmc/rte_bus_fslmc_version.map
> index b169f5228..ac4ee2bf8 100644
> --- a/drivers/bus/fslmc/rte_bus_fslmc_version.map
> +++ b/drivers/bus/fslmc/rte_bus_fslmc_version.map
> @@ -30,6 +30,7 @@ INTERNAL {
>  	dpci_get_opr;
>  	dpci_set_opr;
>  	dpci_set_rx_queue;
> +	dpcon_close;
>  	dpcon_get_attributes;
>  	dpcon_open;
>  	dpdmai_close;
> diff --git a/drivers/bus/fslmc/rte_fslmc.h b/drivers/bus/fslmc/rte_fslmc.h
> index 80873fffc..a938ab8f9 100644
> --- a/drivers/bus/fslmc/rte_fslmc.h
> +++ b/drivers/bus/fslmc/rte_fslmc.h
> @@ -1,6 +1,6 @@
>  /* SPDX-License-Identifier: BSD-3-Clause
>   *
> - *   Copyright 2016,2019 NXP
> + *   Copyright 2016,2019-2020 NXP
>   *
>   */
>  
> @@ -79,6 +79,8 @@ typedef int (*rte_dpaa2_obj_create_t)(int vdev_fd,
>  				      struct vfio_device_info *obj_info,
>  				      int object_id);
>  
> +typedef void (*rte_dpaa2_obj_close_t)(int object_id);
> +
>  /**
>   * A structure describing a DPAA2 object.
>   */
> @@ -87,6 +89,7 @@ struct rte_dpaa2_object {
>  	const char *name;                   /**< Name of Object. */
>  	enum rte_dpaa2_dev_type dev_type;   /**< Type of device */
>  	rte_dpaa2_obj_create_t create;
> +	rte_dpaa2_obj_close_t close;
>  };
>  
>  /**
> diff --git a/drivers/event/dpaa2/dpaa2_hw_dpcon.c b/drivers/event/dpaa2/dpaa2_hw_dpcon.c
> index 200b71640..86844f49f 100644
> --- a/drivers/event/dpaa2/dpaa2_hw_dpcon.c
> +++ b/drivers/event/dpaa2/dpaa2_hw_dpcon.c
> @@ -1,6 +1,6 @@
>  /* SPDX-License-Identifier: BSD-3-Clause
>   *
> - *   Copyright 2017 NXP
> + *   Copyright 2017,2020 NXP
>   *
>   */
>  
> @@ -30,6 +30,19 @@ TAILQ_HEAD(dpcon_dev_list, dpaa2_dpcon_dev);
>  static struct dpcon_dev_list dpcon_dev_list
>  	= TAILQ_HEAD_INITIALIZER(dpcon_dev_list); /*!< DPCON device list */
>  
> +static struct dpaa2_dpcon_dev *get_dpcon_from_id(uint32_t dpcon_id)
> +{
> +	struct dpaa2_dpcon_dev *dpcon_dev = NULL;
> +
> +	/* Get DPCONC dev handle from list using index */
> +	TAILQ_FOREACH(dpcon_dev, &dpcon_dev_list, next) {
> +		if (dpcon_dev->dpcon_id == dpcon_id)
> +			break;
> +	}
> +
> +	return dpcon_dev;
> +}
> +
>  static int
>  rte_dpaa2_create_dpcon_device(int dev_fd __rte_unused,
>  			      struct vfio_device_info *obj_info __rte_unused,
> @@ -105,9 +118,26 @@ void rte_dpaa2_free_dpcon_dev(struct dpaa2_dpcon_dev *dpcon)
>  	}
>  }
>  
> +
> +static void
> +rte_dpaa2_close_dpcon_device(int object_id)
> +{
> +	struct dpaa2_dpcon_dev *dpcon_dev = NULL;
> +
> +	dpcon_dev = get_dpcon_from_id((uint32_t)object_id);
> +
> +	if (dpcon_dev) {
> +		rte_dpaa2_free_dpcon_dev(dpcon_dev);
> +		dpcon_close(&dpcon_dev->dpcon, CMD_PRI_LOW, dpcon_dev->token);
> +		TAILQ_REMOVE(&dpcon_dev_list, dpcon_dev, next);
> +		rte_free(dpcon_dev);
> +	}
> +}
> +
>  static struct rte_dpaa2_object rte_dpaa2_dpcon_obj = {
>  	.dev_type = DPAA2_CON,
>  	.create = rte_dpaa2_create_dpcon_device,
> +	.close = rte_dpaa2_close_dpcon_device,
>  };
>  
>  RTE_PMD_REGISTER_DPAA2_OBJECT(dpcon, rte_dpaa2_dpcon_obj);
> diff --git a/drivers/net/dpaa2/dpaa2_mux.c b/drivers/net/dpaa2/dpaa2_mux.c
> index f8366e839..15114dac9 100644
> --- a/drivers/net/dpaa2/dpaa2_mux.c
> +++ b/drivers/net/dpaa2/dpaa2_mux.c
> @@ -44,7 +44,7 @@ static struct dpaa2_dpdmux_dev *get_dpdmux_from_id(uint32_t dpdmux_id)
>  {
>  	struct dpaa2_dpdmux_dev *dpdmux_dev = NULL;
>  
> -	/* Get DPBP dev handle from list using index */
> +	/* Get DPDMUX dev handle from list using index */
>  	TAILQ_FOREACH(dpdmux_dev, &dpdmux_dev_list, next) {
>  		if (dpdmux_dev->dpdmux_id == dpdmux_id)
>  			break;
> @@ -261,9 +261,25 @@ dpaa2_create_dpdmux_device(int vdev_fd __rte_unused,
>  	return -1;
>  }
>  
> +static void
> +dpaa2_close_dpdmux_device(int object_id)
> +{
> +	struct dpaa2_dpdmux_dev *dpdmux_dev;
> +
> +	dpdmux_dev = get_dpdmux_from_id((uint32_t)object_id);
> +
> +	if (dpdmux_dev) {
> +		dpdmux_close(&dpdmux_dev->dpdmux, CMD_PRI_LOW,
> +			     dpdmux_dev->token);
> +		TAILQ_REMOVE(&dpdmux_dev_list, dpdmux_dev, next);
> +		rte_free(dpdmux_dev);
> +	}
> +}
> +
>  static struct rte_dpaa2_object rte_dpaa2_dpdmux_obj = {
>  	.dev_type = DPAA2_MUX,
>  	.create = dpaa2_create_dpdmux_device,
> +	.close = dpaa2_close_dpdmux_device,
>  };
>  
>  RTE_PMD_REGISTER_DPAA2_OBJECT(dpdmux, rte_dpaa2_dpdmux_obj);
> 

Acked-by: Ray Kinsella <mdr@ashroe.eu>

Re: [dpdk-dev] [PATCH v3 1/3] eal: add API for bus close

[Thomas Monjalon]

> > Signed-off-by: Rohit Raj <rohit.raj@nxp.com>
> Acked-by: Hemant Agrawal <hemant.agrawal@nxp.com>

Please could you look urgently to the close of a port?
All NXP drivers are late to migrate to the new behaviour:
http://inbox.dpdk.org/dev/20200913220711.3768597-17-thomas@monjalon.net/
http://inbox.dpdk.org/dev/80837131.n3tlCtGQ32@thomas/

Re: [dpdk-dev] [PATCH v3 1/3] eal: add API for bus close

[Ferruh Yigit]

On 8/26/2020 6:52 AM, rohit.raj@nxp.com wrote:
> From: Rohit Raj <rohit.raj@nxp.com>
> 
> As per the current code we have API for bus probe, but the
> bus close API is missing. This breaks the multi process
> scenarios as objects are not cleaned while terminating the
> secondary processes.
> 
> This patch adds a new API rte_bus_close() for cleanup of
> bus objects which were acquired during probe.
> 
> Signed-off-by: Rohit Raj <rohit.raj@nxp.com>

<...>

> @@ -67,6 +67,18 @@ typedef int (*rte_bus_scan_t)(void);
>    */
>   typedef int (*rte_bus_probe_t)(void);
>   
> +/**
> + * Implementation specific close function which is responsible for closing
> + * devices on that bus.
> + *
> + * This is called while iterating over each registered bus.
> + *
> + * @return
> + *	0 for successful close
> + *	!0 for any error while closing
> + */
> +typedef int (*rte_bus_close_t)(void);
> +

Can you please put more comments to clarify what does "closing devices 
on bus" mean, what is the responsibility of that function, what is the 
expected state after that function, etc ?

Since 'pci' & 'vdev' bus implementations are missing, this comment 
should help them to clarify what exactly needs to be implemented for 
those buses.

Re: [dpdk-dev] [PATCH v3 1/3] eal: add API for bus close

[Sachin Saxena (OSS)]

On 26-Aug-20 11:22 AM, rohit.raj@nxp.com wrote:
> From: Rohit Raj <rohit.raj@nxp.com>
>
> As per the current code we have API for bus probe, but the
> bus close API is missing. This breaks the multi process
> scenarios as objects are not cleaned while terminating the
> secondary processes.
>
> This patch adds a new API rte_bus_close() for cleanup of
> bus objects which were acquired during probe.
>
> Signed-off-by: Rohit Raj <rohit.raj@nxp.com>
> ---
>
> v3:
> * nit: combined nested if statements
>
> v2:
> * Moved rte_bus_close call to rte_eal_cleanup path.
>
>   lib/librte_eal/common/eal_common_bus.c | 32 +++++++++++++++++++++++++-
>   lib/librte_eal/include/rte_bus.h       | 25 +++++++++++++++++++-
>   lib/librte_eal/linux/eal.c             |  1 +
>   lib/librte_eal/rte_eal_version.map     |  3 +++
>   4 files changed, 59 insertions(+), 2 deletions(-)
>
> diff --git a/lib/librte_eal/common/eal_common_bus.c b/lib/librte_eal/common/eal_common_bus.c
> index baa5b532a..5fd7cf6c5 100644
> --- a/lib/librte_eal/common/eal_common_bus.c
> +++ b/lib/librte_eal/common/eal_common_bus.c
> @@ -1,5 +1,5 @@
>   /* SPDX-License-Identifier: BSD-3-Clause
> - * Copyright 2016 NXP
> + * Copyright 2016,2020 NXP
>    */
>   
>   #include <stdio.h>
> @@ -56,6 +56,36 @@ rte_bus_scan(void)
>   	return 0;
>   }
>   
> +int
> +rte_bus_close(void)
> +{
> +	int ret;
> +	struct rte_bus *bus, *vbus = NULL;
> +
> +	TAILQ_FOREACH(bus, &rte_bus_list, next) {
> +		if (!strcmp(bus->name, "vdev")) {
> +			vbus = bus;
> +			continue;
> +		}
> +
> +		if (bus->close) {
> +			ret = bus->close();
> +			if (ret)
> +				RTE_LOG(ERR, EAL, "Bus (%s) close failed.\n",
> +					bus->name);
> +		}
> +	}
> +
> +	if (vbus && vbus->close) {
> +		ret = vbus->close();
> +		if (ret)
> +			RTE_LOG(ERR, EAL, "Bus (%s) close failed.\n",
> +				vbus->name);
> +	}
> +
> +	return 0;
> +}
> +
>   /* Probe all devices of all buses */
>   int
>   rte_bus_probe(void)
> diff --git a/lib/librte_eal/include/rte_bus.h b/lib/librte_eal/include/rte_bus.h
> index d3034d0ed..af4787b18 100644
> --- a/lib/librte_eal/include/rte_bus.h
> +++ b/lib/librte_eal/include/rte_bus.h
> @@ -1,5 +1,5 @@
>   /* SPDX-License-Identifier: BSD-3-Clause
> - * Copyright 2016 NXP
> + * Copyright 2016,2020 NXP
>    */
>   
>   #ifndef _RTE_BUS_H_
> @@ -67,6 +67,18 @@ typedef int (*rte_bus_scan_t)(void);
>    */
>   typedef int (*rte_bus_probe_t)(void);
>   
> +/**
> + * Implementation specific close function which is responsible for closing
> + * devices on that bus.
> + *
> + * This is called while iterating over each registered bus.
> + *
> + * @return
> + *	0 for successful close
> + *	!0 for any error while closing
> + */
> +typedef int (*rte_bus_close_t)(void);
> +
>   /**
>    * Device iterator to find a device on a bus.
>    *
> @@ -248,6 +260,7 @@ struct rte_bus {
>   	const char *name;            /**< Name of the bus */
>   	rte_bus_scan_t scan;         /**< Scan for devices attached to bus */
>   	rte_bus_probe_t probe;       /**< Probe devices on bus */
> +	rte_bus_close_t close;       /**< Close devices on bus */
>   	rte_bus_find_device_t find_device; /**< Find a device on the bus */
>   	rte_bus_plug_t plug;         /**< Probe single device for drivers */
>   	rte_bus_unplug_t unplug;     /**< Remove single device from driver */
> @@ -301,6 +314,16 @@ int rte_bus_scan(void);
>    */
>   int rte_bus_probe(void);
>   
> +/**
> + * For each device on the buses, call the device specific close.
> + *
> + * @return
> + *	 0 for successful close
> + *	!0 otherwise
> + */
> +__rte_experimental
> +int rte_bus_close(void);
> +
>   /**
>    * Dump information of all the buses registered with EAL.
>    *
> diff --git a/lib/librte_eal/linux/eal.c b/lib/librte_eal/linux/eal.c
> index 0960f01d0..a8a63d8ca 100644
> --- a/lib/librte_eal/linux/eal.c
> +++ b/lib/librte_eal/linux/eal.c
> @@ -1357,6 +1357,7 @@ rte_eal_cleanup(void)
>   
>   	if (rte_eal_process_type() == RTE_PROC_PRIMARY)
>   		rte_memseg_walk(mark_freeable, NULL);
> +	rte_bus_close();
Since this is common framework change, we should introduce similar call 
for "rte_bus_close" in FreeBSD and Windows version also.

>   	rte_service_finalize();
>   	rte_mp_channel_cleanup();
>   	rte_trace_save();
> diff --git a/lib/librte_eal/rte_eal_version.map b/lib/librte_eal/rte_eal_version.map
> index 0b18e2ef8..dc5266f22 100644
> --- a/lib/librte_eal/rte_eal_version.map
> +++ b/lib/librte_eal/rte_eal_version.map
> @@ -397,6 +397,9 @@ EXPERIMENTAL {
>   	rte_mp_disable;
>   	rte_thread_register;
>   	rte_thread_unregister;
> +
> +	# added in 20.11
> +	rte_bus_close;
>   };
>   
>   INTERNAL {

Re: [dpdk-dev] [PATCH v3 1/3] eal: add API for bus close

[Ferruh Yigit]

On 8/26/2020 6:52 AM, rohit.raj@nxp.com wrote:
> From: Rohit Raj <rohit.raj@nxp.com>
> 
> As per the current code we have API for bus probe, but the
> bus close API is missing. This breaks the multi process
> scenarios as objects are not cleaned while terminating the
> secondary processes.
> 
> This patch adds a new API rte_bus_close() for cleanup of
> bus objects which were acquired during probe.
> 
> Signed-off-by: Rohit Raj <rohit.raj@nxp.com>
> ---
> 
> v3:
> * nit: combined nested if statements
> 
> v2:
> * Moved rte_bus_close call to rte_eal_cleanup path.
> 
>   lib/librte_eal/common/eal_common_bus.c | 32 +++++++++++++++++++++++++-
>   lib/librte_eal/include/rte_bus.h       | 25 +++++++++++++++++++-
>   lib/librte_eal/linux/eal.c             |  1 +
>   lib/librte_eal/rte_eal_version.map     |  3 +++
>   4 files changed, 59 insertions(+), 2 deletions(-)
> 
> diff --git a/lib/librte_eal/common/eal_common_bus.c b/lib/librte_eal/common/eal_common_bus.c
> index baa5b532a..5fd7cf6c5 100644
> --- a/lib/librte_eal/common/eal_common_bus.c
> +++ b/lib/librte_eal/common/eal_common_bus.c
> @@ -1,5 +1,5 @@
>   /* SPDX-License-Identifier: BSD-3-Clause
> - * Copyright 2016 NXP
> + * Copyright 2016,2020 NXP
>    */
>   
>   #include <stdio.h>
> @@ -56,6 +56,36 @@ rte_bus_scan(void)
>   	return 0;
>   }
>   
> +int
> +rte_bus_close(void)
> +{
> +	int ret;
> +	struct rte_bus *bus, *vbus = NULL;
> +
> +	TAILQ_FOREACH(bus, &rte_bus_list, next) {
> +		if (!strcmp(bus->name, "vdev")) {
> +			vbus = bus;
> +			continue;
> +		}

This special treatment for 'vdev' bus is done in probe to be sure physically 
device port ids start from '0',  I guess we don't need to do this for 'close'.

> +
> +		if (bus->close) {
> +			ret = bus->close();
> +			if (ret)
> +				RTE_LOG(ERR, EAL, "Bus (%s) close failed.\n",
> +					bus->name);
> +		}
> +	}
> +
> +	if (vbus && vbus->close) {
> +		ret = vbus->close();
> +		if (ret)
> +			RTE_LOG(ERR, EAL, "Bus (%s) close failed.\n",
> +				vbus->name);
> +	}
> +
> +	return 0;
> +}
> +
>   /* Probe all devices of all buses */
>   int
>   rte_bus_probe(void)
> diff --git a/lib/librte_eal/include/rte_bus.h b/lib/librte_eal/include/rte_bus.h
> index d3034d0ed..af4787b18 100644
> --- a/lib/librte_eal/include/rte_bus.h
> +++ b/lib/librte_eal/include/rte_bus.h
> @@ -1,5 +1,5 @@
>   /* SPDX-License-Identifier: BSD-3-Clause
> - * Copyright 2016 NXP
> + * Copyright 2016,2020 NXP
>    */
>   
>   #ifndef _RTE_BUS_H_
> @@ -67,6 +67,18 @@ typedef int (*rte_bus_scan_t)(void);
>    */
>   typedef int (*rte_bus_probe_t)(void);
>   
> +/**
> + * Implementation specific close function which is responsible for closing
> + * devices on that bus.
> + *
> + * This is called while iterating over each registered bus.
> + *
> + * @return
> + *	0 for successful close
> + *	!0 for any error while closing
> + */
> +typedef int (*rte_bus_close_t)(void);
> +

As I checked the 'rte_fslmc_bus->close()' ops, it iterates on all devices in the 
bus instead of doing a bus level close, in that case
instead of adding a new 'close' bus operations, will it work if existing 
'bus->unplug(dev)' used?
Whatever done in the 'rte_fslmc_bus->close()' per device, can it be done under 
the 'fslmc_bus_unplug()'?

And in that case a 'rte_bus_remove()' API can be added which can call 
'bus->unplug(dev)' for all buses and it will be beneficial for all buses, and it 
can fit well into the 'rte_eal_cleanup()'.

What do you think?

[dpdk-dev] [PATCH v4 1/5] eal: add API for bus close

[rohit.raj]

From: Rohit Raj <rohit.raj@nxp.com>

As per the current code we have API for bus probe, but the
bus close API is missing. This breaks the multi process
scenarios as objects are not cleaned while terminating the
secondary processes.

This patch adds a new API rte_bus_close() for cleanup of
bus objects which were acquired during probe.

Signed-off-by: Rohit Raj <rohit.raj@nxp.com>
---
 lib/librte_eal/common/eal_common_bus.c | 32 +++++++++++++++++++++++++-
 lib/librte_eal/include/rte_bus.h       | 27 +++++++++++++++++++++-
 lib/librte_eal/linux/eal.c             |  1 +
 lib/librte_eal/rte_eal_version.map     |  1 +
 4 files changed, 59 insertions(+), 2 deletions(-)

diff --git a/lib/librte_eal/common/eal_common_bus.c b/lib/librte_eal/common/eal_common_bus.c
index baa5b532a..5fd7cf6c5 100644
--- a/lib/librte_eal/common/eal_common_bus.c
+++ b/lib/librte_eal/common/eal_common_bus.c
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright 2016 NXP
+ * Copyright 2016,2020 NXP
  */
 
 #include <stdio.h>
@@ -56,6 +56,36 @@ rte_bus_scan(void)
 	return 0;
 }
 
+int
+rte_bus_close(void)
+{
+	int ret;
+	struct rte_bus *bus, *vbus = NULL;
+
+	TAILQ_FOREACH(bus, &rte_bus_list, next) {
+		if (!strcmp(bus->name, "vdev")) {
+			vbus = bus;
+			continue;
+		}
+
+		if (bus->close) {
+			ret = bus->close();
+			if (ret)
+				RTE_LOG(ERR, EAL, "Bus (%s) close failed.\n",
+					bus->name);
+		}
+	}
+
+	if (vbus && vbus->close) {
+		ret = vbus->close();
+		if (ret)
+			RTE_LOG(ERR, EAL, "Bus (%s) close failed.\n",
+				vbus->name);
+	}
+
+	return 0;
+}
+
 /* Probe all devices of all buses */
 int
 rte_bus_probe(void)
diff --git a/lib/librte_eal/include/rte_bus.h b/lib/librte_eal/include/rte_bus.h
index d3034d0ed..7bf7e81b2 100644
--- a/lib/librte_eal/include/rte_bus.h
+++ b/lib/librte_eal/include/rte_bus.h
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright 2016 NXP
+ * Copyright 2016,2020 NXP
  */
 
 #ifndef _RTE_BUS_H_
@@ -67,6 +67,20 @@ typedef int (*rte_bus_scan_t)(void);
  */
 typedef int (*rte_bus_probe_t)(void);
 
+/**
+ * Implementation specific close function which is responsible for resetting all
+ * detected devices on the bus to a default state, closing UIO nodes or VFIO
+ * groups and also freeing any memory allocated during rte_bus_probe like
+ * private resources for device list.
+ *
+ * This is called while iterating over each registered bus.
+ *
+ * @return
+ *	0 for successful close
+ *	!0 for any error while closing
+ */
+typedef int (*rte_bus_close_t)(void);
+
 /**
  * Device iterator to find a device on a bus.
  *
@@ -248,6 +262,7 @@ struct rte_bus {
 	const char *name;            /**< Name of the bus */
 	rte_bus_scan_t scan;         /**< Scan for devices attached to bus */
 	rte_bus_probe_t probe;       /**< Probe devices on bus */
+	rte_bus_close_t close;       /**< Close devices on bus */
 	rte_bus_find_device_t find_device; /**< Find a device on the bus */
 	rte_bus_plug_t plug;         /**< Probe single device for drivers */
 	rte_bus_unplug_t unplug;     /**< Remove single device from driver */
@@ -301,6 +316,16 @@ int rte_bus_scan(void);
  */
 int rte_bus_probe(void);
 
+/**
+ * For each device on the buses, call the device specific close.
+ *
+ * @return
+ *	 0 for successful close
+ *	!0 otherwise
+ */
+__rte_experimental
+int rte_bus_close(void);
+
 /**
  * Dump information of all the buses registered with EAL.
  *
diff --git a/lib/librte_eal/linux/eal.c b/lib/librte_eal/linux/eal.c
index 9cf0e2ec0..1e1256f07 100644
--- a/lib/librte_eal/linux/eal.c
+++ b/lib/librte_eal/linux/eal.c
@@ -1358,6 +1358,7 @@ rte_eal_cleanup(void)
 
 	if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 		rte_memseg_walk(mark_freeable, NULL);
+	rte_bus_close();
 	rte_service_finalize();
 	rte_mp_channel_cleanup();
 	rte_trace_save();
diff --git a/lib/librte_eal/rte_eal_version.map b/lib/librte_eal/rte_eal_version.map
index a93dea9fe..8e226b297 100644
--- a/lib/librte_eal/rte_eal_version.map
+++ b/lib/librte_eal/rte_eal_version.map
@@ -400,6 +400,7 @@ EXPERIMENTAL {
 	# added in 20.11
 	__rte_eal_trace_generic_size_t;
 	rte_service_lcore_may_be_active;
+	rte_bus_close;
 };
 
 INTERNAL {
-- 
2.17.1

[dpdk-dev] [PATCH v4 2/5] examples/multi_process: cleanup bus objects while terminating app

[rohit.raj]

From: Rohit Raj <rohit.raj@nxp.com>

Certain bus objects may need to be closed and re-acquired
while terminating and rerunning the client application.
Hence a signal handler is required to catch the termination
of the App and hence closing the bus objects.

This patch adds the missing signal handler in the client
app and closes the Bus objects in both client and server
applications when the signal Handler is called.

Signed-off-by: Rohit Raj <rohit.raj@nxp.com>
---
 .../multi_process/client_server_mp/mp_client/client.c | 11 +++++++++++
 .../multi_process/client_server_mp/mp_server/main.c   |  4 +++-
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/examples/multi_process/client_server_mp/mp_client/client.c b/examples/multi_process/client_server_mp/mp_client/client.c
index 361d90b54..c37516b4c 100644
--- a/examples/multi_process/client_server_mp/mp_client/client.c
+++ b/examples/multi_process/client_server_mp/mp_client/client.c
@@ -11,6 +11,7 @@
 #include <stdlib.h>
 #include <getopt.h>
 #include <string.h>
+#include <signal.h>
 
 #include <rte_common.h>
 #include <rte_malloc.h>
@@ -196,6 +197,14 @@ handle_packet(struct rte_mbuf *buf)
 
 }
 
+static void
+signal_handler(int signal)
+{
+	if (signal == SIGINT)
+		rte_eal_cleanup();
+	exit(0);
+}
+
 /*
  * Application main function - loops through
  * receiving and processing packets. Never returns
@@ -217,6 +226,8 @@ main(int argc, char *argv[])
 	argc -= retval;
 	argv += retval;
 
+	signal(SIGINT, signal_handler);
+
 	if (parse_app_args(argc, argv) < 0)
 		rte_exit(EXIT_FAILURE, "Invalid command-line arguments\n");
 
diff --git a/examples/multi_process/client_server_mp/mp_server/main.c b/examples/multi_process/client_server_mp/mp_server/main.c
index 280dab867..b0241cc20 100644
--- a/examples/multi_process/client_server_mp/mp_server/main.c
+++ b/examples/multi_process/client_server_mp/mp_server/main.c
@@ -275,11 +275,13 @@ signal_handler(int signal)
 {
 	uint16_t port_id;
 
-	if (signal == SIGINT)
+	if (signal == SIGINT) {
 		RTE_ETH_FOREACH_DEV(port_id) {
 			rte_eth_dev_stop(port_id);
 			rte_eth_dev_close(port_id);
 		}
+		rte_eal_cleanup();
+	}
 	exit(0);
 }
 
-- 
2.17.1

[dpdk-dev] [PATCH v4 3/5] bus/fslmc: support bus close API

[rohit.raj]

From: Rohit Raj <rohit.raj@nxp.com>

This patch add support for closing the bus objects which
were acquired In the bus probe.

Some devices need to be cleaned while in both primary and
secondary process and while some need to be cleaned only in
case of primary process.

The devices are closed as per the white list used while
creating the objects in a particular process.

Signed-off-by: Rohit Raj <rohit.raj@nxp.com>
Acked-by: Sachin Saxena <sachin.saxena@oss.nxp.com>
---
 drivers/bus/fslmc/fslmc_bus.c               | 15 +++-
 drivers/bus/fslmc/fslmc_vfio.c              | 89 ++++++++++++++++++++-
 drivers/bus/fslmc/fslmc_vfio.h              |  3 +-
 drivers/bus/fslmc/mc/fsl_dpcon.h            |  1 +
 drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c    | 31 ++++++-
 drivers/bus/fslmc/portal/dpaa2_hw_dpci.c    | 32 +++++++-
 drivers/bus/fslmc/portal/dpaa2_hw_dpio.c    | 34 ++++++++
 drivers/bus/fslmc/rte_bus_fslmc_version.map |  1 +
 drivers/bus/fslmc/rte_fslmc.h               |  5 +-
 drivers/event/dpaa2/dpaa2_hw_dpcon.c        | 32 +++++++-
 drivers/net/dpaa2/dpaa2_mux.c               | 18 ++++-
 11 files changed, 253 insertions(+), 8 deletions(-)

diff --git a/drivers/bus/fslmc/fslmc_bus.c b/drivers/bus/fslmc/fslmc_bus.c
index beb3dd008..9d3da6be3 100644
--- a/drivers/bus/fslmc/fslmc_bus.c
+++ b/drivers/bus/fslmc/fslmc_bus.c
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
- *   Copyright 2016,2018-2019 NXP
+ *   Copyright 2016,2018-2020 NXP
  *
  */
 
@@ -365,6 +365,18 @@ rte_fslmc_match(struct rte_dpaa2_driver *dpaa2_drv,
 	return 1;
 }
 
+static int
+rte_fslmc_close(void)
+{
+	int ret = 0;
+
+	ret = fslmc_vfio_close_group();
+	if (ret)
+		DPAA2_BUS_ERR("Unable to close devices %d", ret);
+
+	return 0;
+}
+
 static int
 rte_fslmc_probe(void)
 {
@@ -637,6 +649,7 @@ struct rte_fslmc_bus rte_fslmc_bus = {
 	.bus = {
 		.scan = rte_fslmc_scan,
 		.probe = rte_fslmc_probe,
+		.close = rte_fslmc_close,
 		.parse = rte_fslmc_parse,
 		.find_device = rte_fslmc_find_device,
 		.get_iommu_class = rte_dpaa2_get_iommu_class,
diff --git a/drivers/bus/fslmc/fslmc_vfio.c b/drivers/bus/fslmc/fslmc_vfio.c
index aba55b46d..055e8b764 100644
--- a/drivers/bus/fslmc/fslmc_vfio.c
+++ b/drivers/bus/fslmc/fslmc_vfio.c
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
  *   Copyright (c) 2015-2016 Freescale Semiconductor, Inc. All rights reserved.
- *   Copyright 2016-2019 NXP
+ *   Copyright 2016-2020 NXP
  *
  */
 
@@ -695,6 +695,54 @@ rte_dpaa2_vfio_setup_intr(struct rte_intr_handle *intr_handle,
 	return -1;
 }
 
+static void
+fslmc_close_iodevices(struct rte_dpaa2_device *dev)
+{
+	struct rte_dpaa2_object *object = NULL;
+	struct rte_dpaa2_driver *drv;
+	int ret, probe_all;
+
+	switch (dev->dev_type) {
+	case DPAA2_IO:
+	case DPAA2_CON:
+	case DPAA2_CI:
+	case DPAA2_BPOOL:
+	case DPAA2_MUX:
+		TAILQ_FOREACH(object, &dpaa2_obj_list, next) {
+			if (dev->dev_type == object->dev_type)
+				object->close(dev->object_id);
+			else
+				continue;
+		}
+		break;
+	case DPAA2_ETH:
+	case DPAA2_CRYPTO:
+	case DPAA2_QDMA:
+		probe_all = rte_fslmc_bus.bus.conf.scan_mode !=
+			    RTE_BUS_SCAN_WHITELIST;
+		TAILQ_FOREACH(drv, &rte_fslmc_bus.driver_list, next) {
+			if (drv->drv_type != dev->dev_type)
+				continue;
+			if (rte_dev_is_probed(&dev->device))
+				continue;
+			if (probe_all ||
+			    (dev->device.devargs &&
+			     dev->device.devargs->policy ==
+			     RTE_DEV_WHITELISTED)) {
+				ret = drv->remove(dev);
+				if (ret)
+					DPAA2_BUS_ERR("Unable to remove");
+			}
+		}
+		break;
+	default:
+		break;
+	}
+
+	DPAA2_BUS_LOG(DEBUG, "Device (%s) Closed",
+		      dev->device.name);
+}
+
 /*
  * fslmc_process_iodevices for processing only IO (ETH, CRYPTO, and possibly
  * EVENT) devices.
@@ -799,6 +847,45 @@ fslmc_process_mcp(struct rte_dpaa2_device *dev)
 	return ret;
 }
 
+int
+fslmc_vfio_close_group(void)
+{
+	struct rte_dpaa2_device *dev, *dev_temp;
+
+	TAILQ_FOREACH_SAFE(dev, &rte_fslmc_bus.device_list, next, dev_temp) {
+		if (dev->device.devargs &&
+		    dev->device.devargs->policy == RTE_DEV_BLACKLISTED) {
+			DPAA2_BUS_LOG(DEBUG, "%s Blacklisted, skipping",
+				      dev->device.name);
+			TAILQ_REMOVE(&rte_fslmc_bus.device_list, dev, next);
+				continue;
+		}
+		switch (dev->dev_type) {
+		case DPAA2_ETH:
+		case DPAA2_CRYPTO:
+		case DPAA2_QDMA:
+		case DPAA2_IO:
+			fslmc_close_iodevices(dev);
+			break;
+		case DPAA2_CON:
+		case DPAA2_CI:
+		case DPAA2_BPOOL:
+		case DPAA2_MUX:
+			if (rte_eal_process_type() == RTE_PROC_SECONDARY)
+				continue;
+
+			fslmc_close_iodevices(dev);
+			break;
+		case DPAA2_DPRTC:
+		default:
+			DPAA2_BUS_DEBUG("Device cannot be closed: Not supported (%s)",
+					dev->device.name);
+		}
+	}
+
+	return 0;
+}
+
 int
 fslmc_vfio_process_group(void)
 {
diff --git a/drivers/bus/fslmc/fslmc_vfio.h b/drivers/bus/fslmc/fslmc_vfio.h
index bc7c6f62d..0a29fe739 100644
--- a/drivers/bus/fslmc/fslmc_vfio.h
+++ b/drivers/bus/fslmc/fslmc_vfio.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
  *   Copyright (c) 2015-2016 Freescale Semiconductor, Inc. All rights reserved.
- *   Copyright 2016,2019 NXP
+ *   Copyright 2016,2019-2020 NXP
  *
  */
 
@@ -55,6 +55,7 @@ int rte_dpaa2_vfio_setup_intr(struct rte_intr_handle *intr_handle,
 
 int fslmc_vfio_setup_group(void);
 int fslmc_vfio_process_group(void);
+int fslmc_vfio_close_group(void);
 char *fslmc_get_container(void);
 int fslmc_get_container_group(int *gropuid);
 int rte_fslmc_vfio_dmamap(void);
diff --git a/drivers/bus/fslmc/mc/fsl_dpcon.h b/drivers/bus/fslmc/mc/fsl_dpcon.h
index 7caa6c68a..8053a6db1 100644
--- a/drivers/bus/fslmc/mc/fsl_dpcon.h
+++ b/drivers/bus/fslmc/mc/fsl_dpcon.h
@@ -26,6 +26,7 @@ int dpcon_open(struct fsl_mc_io *mc_io,
 	       int dpcon_id,
 	       uint16_t *token);
 
+__rte_internal
 int dpcon_close(struct fsl_mc_io *mc_io,
 		uint32_t cmd_flags,
 		uint16_t token);
diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c
index d9619848d..9a4cdfd7a 100644
--- a/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c
+++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
  *   Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
- *   Copyright 2016 NXP
+ *   Copyright 2016,2020 NXP
  *
  */
 
@@ -33,6 +33,19 @@ TAILQ_HEAD(dpbp_dev_list, dpaa2_dpbp_dev);
 static struct dpbp_dev_list dpbp_dev_list
 	= TAILQ_HEAD_INITIALIZER(dpbp_dev_list); /*!< DPBP device list */
 
+static struct dpaa2_dpbp_dev *get_dpbp_from_id(uint32_t dpbp_id)
+{
+	struct dpaa2_dpbp_dev *dpbp_dev = NULL;
+
+	/* Get DPBP dev handle from list using index */
+	TAILQ_FOREACH(dpbp_dev, &dpbp_dev_list, next) {
+		if (dpbp_dev->dpbp_id == dpbp_id)
+			break;
+	}
+
+	return dpbp_dev;
+}
+
 static int
 dpaa2_create_dpbp_device(int vdev_fd __rte_unused,
 			 struct vfio_device_info *obj_info __rte_unused,
@@ -116,9 +129,25 @@ int dpaa2_dpbp_supported(void)
 	return 0;
 }
 
+static void
+dpaa2_close_dpbp_device(int object_id)
+{
+	struct dpaa2_dpbp_dev *dpbp_dev = NULL;
+
+	dpbp_dev = get_dpbp_from_id((uint32_t)object_id);
+
+	if (dpbp_dev) {
+		dpaa2_free_dpbp_dev(dpbp_dev);
+		dpbp_close(&dpbp_dev->dpbp, CMD_PRI_LOW, dpbp_dev->token);
+		TAILQ_REMOVE(&dpbp_dev_list, dpbp_dev, next);
+		rte_free(dpbp_dev);
+	}
+}
+
 static struct rte_dpaa2_object rte_dpaa2_dpbp_obj = {
 	.dev_type = DPAA2_BPOOL,
 	.create = dpaa2_create_dpbp_device,
+	.close = dpaa2_close_dpbp_device,
 };
 
 RTE_PMD_REGISTER_DPAA2_OBJECT(dpbp, rte_dpaa2_dpbp_obj);
diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c
index d393ce618..9902f38ce 100644
--- a/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c
+++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
- *   Copyright 2017 NXP
+ *   Copyright 2017,2020 NXP
  *
  */
 
@@ -30,6 +30,19 @@ TAILQ_HEAD(dpci_dev_list, dpaa2_dpci_dev);
 static struct dpci_dev_list dpci_dev_list
 	= TAILQ_HEAD_INITIALIZER(dpci_dev_list); /*!< DPCI device list */
 
+static struct dpaa2_dpci_dev *get_dpci_from_id(uint32_t dpci_id)
+{
+	struct dpaa2_dpci_dev *dpci_dev = NULL;
+
+	/* Get DPCI dev handle from list using index */
+	TAILQ_FOREACH(dpci_dev, &dpci_dev_list, next) {
+		if (dpci_dev->dpci_id == dpci_id)
+			break;
+	}
+
+	return dpci_dev;
+}
+
 static int
 rte_dpaa2_create_dpci_device(int vdev_fd __rte_unused,
 			     struct vfio_device_info *obj_info __rte_unused,
@@ -179,9 +192,26 @@ void rte_dpaa2_free_dpci_dev(struct dpaa2_dpci_dev *dpci)
 	}
 }
 
+
+static void
+rte_dpaa2_close_dpci_device(int object_id)
+{
+	struct dpaa2_dpci_dev *dpci_dev = NULL;
+
+	dpci_dev = get_dpci_from_id((uint32_t)object_id);
+
+	if (dpci_dev) {
+		rte_dpaa2_free_dpci_dev(dpci_dev);
+		dpci_close(&dpci_dev->dpci, CMD_PRI_LOW, dpci_dev->token);
+		TAILQ_REMOVE(&dpci_dev_list, dpci_dev, next);
+		rte_free(dpci_dev);
+	}
+}
+
 static struct rte_dpaa2_object rte_dpaa2_dpci_obj = {
 	.dev_type = DPAA2_CI,
 	.create = rte_dpaa2_create_dpci_device,
+	.close = rte_dpaa2_close_dpci_device,
 };
 
 RTE_PMD_REGISTER_DPAA2_OBJECT(dpci, rte_dpaa2_dpci_obj);
diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
index b0055b164..b7caaea31 100644
--- a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
+++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
@@ -86,6 +86,19 @@ static int dpaa2_cluster_sz = 2;
  * Cluster 4 (ID = x07) : CPU14, CPU15;
  */
 
+static struct dpaa2_dpio_dev *get_dpio_dev_from_id(int32_t dpio_id)
+{
+	struct dpaa2_dpio_dev *dpio_dev = NULL;
+
+	/* Get DPIO dev handle from list using index */
+	TAILQ_FOREACH(dpio_dev, &dpio_dev_list, next) {
+		if (dpio_dev->hw_id == dpio_id)
+			break;
+	}
+
+	return dpio_dev;
+}
+
 static int
 dpaa2_get_core_id(void)
 {
@@ -360,6 +373,26 @@ static void dpaa2_portal_finish(void *arg)
 	pthread_setspecific(dpaa2_portal_key, NULL);
 }
 
+static void
+dpaa2_close_dpio_device(int object_id)
+{
+	struct dpaa2_dpio_dev *dpio_dev = NULL;
+
+	dpio_dev = get_dpio_dev_from_id((int32_t)object_id);
+
+	if (dpio_dev) {
+		if (dpio_dev->dpio) {
+			dpio_disable(dpio_dev->dpio, CMD_PRI_LOW,
+				     dpio_dev->token);
+			dpio_close(dpio_dev->dpio, CMD_PRI_LOW,
+				   dpio_dev->token);
+			rte_free(dpio_dev->dpio);
+		}
+		TAILQ_REMOVE(&dpio_dev_list, dpio_dev, next);
+		rte_free(dpio_dev);
+	}
+}
+
 static int
 dpaa2_create_dpio_device(int vdev_fd,
 			 struct vfio_device_info *obj_info,
@@ -628,6 +661,7 @@ dpaa2_free_eq_descriptors(void)
 static struct rte_dpaa2_object rte_dpaa2_dpio_obj = {
 	.dev_type = DPAA2_IO,
 	.create = dpaa2_create_dpio_device,
+	.close = dpaa2_close_dpio_device,
 };
 
 RTE_PMD_REGISTER_DPAA2_OBJECT(dpio, rte_dpaa2_dpio_obj);
diff --git a/drivers/bus/fslmc/rte_bus_fslmc_version.map b/drivers/bus/fslmc/rte_bus_fslmc_version.map
index b169f5228..ac4ee2bf8 100644
--- a/drivers/bus/fslmc/rte_bus_fslmc_version.map
+++ b/drivers/bus/fslmc/rte_bus_fslmc_version.map
@@ -30,6 +30,7 @@ INTERNAL {
 	dpci_get_opr;
 	dpci_set_opr;
 	dpci_set_rx_queue;
+	dpcon_close;
 	dpcon_get_attributes;
 	dpcon_open;
 	dpdmai_close;
diff --git a/drivers/bus/fslmc/rte_fslmc.h b/drivers/bus/fslmc/rte_fslmc.h
index 80873fffc..a938ab8f9 100644
--- a/drivers/bus/fslmc/rte_fslmc.h
+++ b/drivers/bus/fslmc/rte_fslmc.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
- *   Copyright 2016,2019 NXP
+ *   Copyright 2016,2019-2020 NXP
  *
  */
 
@@ -79,6 +79,8 @@ typedef int (*rte_dpaa2_obj_create_t)(int vdev_fd,
 				      struct vfio_device_info *obj_info,
 				      int object_id);
 
+typedef void (*rte_dpaa2_obj_close_t)(int object_id);
+
 /**
  * A structure describing a DPAA2 object.
  */
@@ -87,6 +89,7 @@ struct rte_dpaa2_object {
 	const char *name;                   /**< Name of Object. */
 	enum rte_dpaa2_dev_type dev_type;   /**< Type of device */
 	rte_dpaa2_obj_create_t create;
+	rte_dpaa2_obj_close_t close;
 };
 
 /**
diff --git a/drivers/event/dpaa2/dpaa2_hw_dpcon.c b/drivers/event/dpaa2/dpaa2_hw_dpcon.c
index 200b71640..86844f49f 100644
--- a/drivers/event/dpaa2/dpaa2_hw_dpcon.c
+++ b/drivers/event/dpaa2/dpaa2_hw_dpcon.c
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
- *   Copyright 2017 NXP
+ *   Copyright 2017,2020 NXP
  *
  */
 
@@ -30,6 +30,19 @@ TAILQ_HEAD(dpcon_dev_list, dpaa2_dpcon_dev);
 static struct dpcon_dev_list dpcon_dev_list
 	= TAILQ_HEAD_INITIALIZER(dpcon_dev_list); /*!< DPCON device list */
 
+static struct dpaa2_dpcon_dev *get_dpcon_from_id(uint32_t dpcon_id)
+{
+	struct dpaa2_dpcon_dev *dpcon_dev = NULL;
+
+	/* Get DPCONC dev handle from list using index */
+	TAILQ_FOREACH(dpcon_dev, &dpcon_dev_list, next) {
+		if (dpcon_dev->dpcon_id == dpcon_id)
+			break;
+	}
+
+	return dpcon_dev;
+}
+
 static int
 rte_dpaa2_create_dpcon_device(int dev_fd __rte_unused,
 			      struct vfio_device_info *obj_info __rte_unused,
@@ -105,9 +118,26 @@ void rte_dpaa2_free_dpcon_dev(struct dpaa2_dpcon_dev *dpcon)
 	}
 }
 
+
+static void
+rte_dpaa2_close_dpcon_device(int object_id)
+{
+	struct dpaa2_dpcon_dev *dpcon_dev = NULL;
+
+	dpcon_dev = get_dpcon_from_id((uint32_t)object_id);
+
+	if (dpcon_dev) {
+		rte_dpaa2_free_dpcon_dev(dpcon_dev);
+		dpcon_close(&dpcon_dev->dpcon, CMD_PRI_LOW, dpcon_dev->token);
+		TAILQ_REMOVE(&dpcon_dev_list, dpcon_dev, next);
+		rte_free(dpcon_dev);
+	}
+}
+
 static struct rte_dpaa2_object rte_dpaa2_dpcon_obj = {
 	.dev_type = DPAA2_CON,
 	.create = rte_dpaa2_create_dpcon_device,
+	.close = rte_dpaa2_close_dpcon_device,
 };
 
 RTE_PMD_REGISTER_DPAA2_OBJECT(dpcon, rte_dpaa2_dpcon_obj);
diff --git a/drivers/net/dpaa2/dpaa2_mux.c b/drivers/net/dpaa2/dpaa2_mux.c
index f8366e839..15114dac9 100644
--- a/drivers/net/dpaa2/dpaa2_mux.c
+++ b/drivers/net/dpaa2/dpaa2_mux.c
@@ -44,7 +44,7 @@ static struct dpaa2_dpdmux_dev *get_dpdmux_from_id(uint32_t dpdmux_id)
 {
 	struct dpaa2_dpdmux_dev *dpdmux_dev = NULL;
 
-	/* Get DPBP dev handle from list using index */
+	/* Get DPDMUX dev handle from list using index */
 	TAILQ_FOREACH(dpdmux_dev, &dpdmux_dev_list, next) {
 		if (dpdmux_dev->dpdmux_id == dpdmux_id)
 			break;
@@ -261,9 +261,25 @@ dpaa2_create_dpdmux_device(int vdev_fd __rte_unused,
 	return -1;
 }
 
+static void
+dpaa2_close_dpdmux_device(int object_id)
+{
+	struct dpaa2_dpdmux_dev *dpdmux_dev;
+
+	dpdmux_dev = get_dpdmux_from_id((uint32_t)object_id);
+
+	if (dpdmux_dev) {
+		dpdmux_close(&dpdmux_dev->dpdmux, CMD_PRI_LOW,
+			     dpdmux_dev->token);
+		TAILQ_REMOVE(&dpdmux_dev_list, dpdmux_dev, next);
+		rte_free(dpdmux_dev);
+	}
+}
+
 static struct rte_dpaa2_object rte_dpaa2_dpdmux_obj = {
 	.dev_type = DPAA2_MUX,
 	.create = dpaa2_create_dpdmux_device,
+	.close = dpaa2_close_dpdmux_device,
 };
 
 RTE_PMD_REGISTER_DPAA2_OBJECT(dpdmux, rte_dpaa2_dpdmux_obj);
-- 
2.17.1

[dpdk-dev] [PATCH v4 4/5] eal/freebsd: added support for rte_bus_close API

[rohit.raj]

From: Rohit Raj <rohit.raj@nxp.com>

Added support for rte_bus_close API on freebsd to reset the devices on
the bus to default state.

Signed-off-by: Rohit Raj <rohit.raj@nxp.com>
---
 lib/librte_eal/freebsd/eal.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/librte_eal/freebsd/eal.c b/lib/librte_eal/freebsd/eal.c
index ccea60afe..e78bde22e 100644
--- a/lib/librte_eal/freebsd/eal.c
+++ b/lib/librte_eal/freebsd/eal.c
@@ -962,6 +962,7 @@ rte_eal_cleanup(void)
 {
 	struct internal_config *internal_conf =
 		eal_get_internal_configuration();
+	rte_bus_close();
 	rte_service_finalize();
 	rte_mp_channel_cleanup();
 	rte_trace_save();
-- 
2.17.1

[dpdk-dev] [PATCH v4 5/5] eal/windows: added support for rte_bus_close API

[rohit.raj]

From: Rohit Raj <rohit.raj@nxp.com>

Added support for rte_bus_close API on windows to reset the devices on
the bus to default state.

Signed-off-by: Rohit Raj <rohit.raj@nxp.com>
---
 lib/librte_eal/windows/eal.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/librte_eal/windows/eal.c b/lib/librte_eal/windows/eal.c
index bc48f27ab..9eb32077b 100644
--- a/lib/librte_eal/windows/eal.c
+++ b/lib/librte_eal/windows/eal.c
@@ -251,7 +251,7 @@ rte_eal_cleanup(void)
 {
 	struct internal_config *internal_conf =
 		eal_get_internal_configuration();
-
+	rte_bus_close();
 	eal_cleanup_config(internal_conf);
 	return 0;
 }
-- 
2.17.1

Re: [dpdk-dev] [PATCH v3 1/3] eal: add API for bus close

[Ferruh Yigit]

On 9/30/2020 12:50 PM, Ferruh Yigit wrote:
> On 8/26/2020 6:52 AM, rohit.raj@nxp.com wrote:
>> From: Rohit Raj <rohit.raj@nxp.com>
>>
>> As per the current code we have API for bus probe, but the
>> bus close API is missing. This breaks the multi process
>> scenarios as objects are not cleaned while terminating the
>> secondary processes.
>>
>> This patch adds a new API rte_bus_close() for cleanup of
>> bus objects which were acquired during probe.
>>
>> Signed-off-by: Rohit Raj <rohit.raj@nxp.com>
>> ---
>>
>> v3:
>> * nit: combined nested if statements
>>
>> v2:
>> * Moved rte_bus_close call to rte_eal_cleanup path.
>>
>>   lib/librte_eal/common/eal_common_bus.c | 32 +++++++++++++++++++++++++-
>>   lib/librte_eal/include/rte_bus.h       | 25 +++++++++++++++++++-
>>   lib/librte_eal/linux/eal.c             |  1 +
>>   lib/librte_eal/rte_eal_version.map     |  3 +++
>>   4 files changed, 59 insertions(+), 2 deletions(-)
>>
>> diff --git a/lib/librte_eal/common/eal_common_bus.c 
>> b/lib/librte_eal/common/eal_common_bus.c
>> index baa5b532a..5fd7cf6c5 100644
>> --- a/lib/librte_eal/common/eal_common_bus.c
>> +++ b/lib/librte_eal/common/eal_common_bus.c
>> @@ -1,5 +1,5 @@
>>   /* SPDX-License-Identifier: BSD-3-Clause
>> - * Copyright 2016 NXP
>> + * Copyright 2016,2020 NXP
>>    */
>>   #include <stdio.h>
>> @@ -56,6 +56,36 @@ rte_bus_scan(void)
>>       return 0;
>>   }
>> +int
>> +rte_bus_close(void)
>> +{
>> +    int ret;
>> +    struct rte_bus *bus, *vbus = NULL;
>> +
>> +    TAILQ_FOREACH(bus, &rte_bus_list, next) {
>> +        if (!strcmp(bus->name, "vdev")) {
>> +            vbus = bus;
>> +            continue;
>> +        }
> 
> This special treatment for 'vdev' bus is done in probe to be sure physically 
> device port ids start from '0',  I guess we don't need to do this for 'close'.
> 
>> +
>> +        if (bus->close) {
>> +            ret = bus->close();
>> +            if (ret)
>> +                RTE_LOG(ERR, EAL, "Bus (%s) close failed.\n",
>> +                    bus->name);
>> +        }
>> +    }
>> +
>> +    if (vbus && vbus->close) {
>> +        ret = vbus->close();
>> +        if (ret)
>> +            RTE_LOG(ERR, EAL, "Bus (%s) close failed.\n",
>> +                vbus->name);
>> +    }
>> +
>> +    return 0;
>> +}
>> +
>>   /* Probe all devices of all buses */
>>   int
>>   rte_bus_probe(void)
>> diff --git a/lib/librte_eal/include/rte_bus.h b/lib/librte_eal/include/rte_bus.h
>> index d3034d0ed..af4787b18 100644
>> --- a/lib/librte_eal/include/rte_bus.h
>> +++ b/lib/librte_eal/include/rte_bus.h
>> @@ -1,5 +1,5 @@
>>   /* SPDX-License-Identifier: BSD-3-Clause
>> - * Copyright 2016 NXP
>> + * Copyright 2016,2020 NXP
>>    */
>>   #ifndef _RTE_BUS_H_
>> @@ -67,6 +67,18 @@ typedef int (*rte_bus_scan_t)(void);
>>    */
>>   typedef int (*rte_bus_probe_t)(void);
>> +/**
>> + * Implementation specific close function which is responsible for closing
>> + * devices on that bus.
>> + *
>> + * This is called while iterating over each registered bus.
>> + *
>> + * @return
>> + *    0 for successful close
>> + *    !0 for any error while closing
>> + */
>> +typedef int (*rte_bus_close_t)(void);
>> +
> 
> As I checked the 'rte_fslmc_bus->close()' ops, it iterates on all devices in the 
> bus instead of doing a bus level close, in that case
> instead of adding a new 'close' bus operations, will it work if existing 
> 'bus->unplug(dev)' used?
> Whatever done in the 'rte_fslmc_bus->close()' per device, can it be done under 
> the 'fslmc_bus_unplug()'?
> 
> And in that case a 'rte_bus_remove()' API can be added which can call 
> 'bus->unplug(dev)' for all buses and it will be beneficial for all buses, and it 
> can fit well into the 'rte_eal_cleanup()'.
> 
> What do you think?

Hi Rohit,

I have seen new version has been sent today, I want to remind above question, 
can you please check it?

Re: [dpdk-dev] [EXT] Re: [PATCH v3 1/3] eal: add API for bus close

[Rohit Raj]

Hi Ferruh,

Sorry, I missed you comment last time. I will send an updated patch series after taking care of your suggested changes.

Regards,
Rohit Raj
Software Engineer | NXP India Private Limited

Sent from Outlook Mobile
________________________________
From: Ferruh Yigit <ferruh.yigit@intel.com>
Sent: Thursday, October 8, 2020 9:54:58 PM
To: Rohit Raj <rohit.raj@nxp.com>; Ray Kinsella <mdr@ashroe.eu>; Neil Horman <nhorman@tuxdriver.com>
Cc: dev@dpdk.org <dev@dpdk.org>
Subject: [EXT] Re: [dpdk-dev] [PATCH v3 1/3] eal: add API for bus close

Caution: EXT Email

On 9/30/2020 12:50 PM, Ferruh Yigit wrote:
> On 8/26/2020 6:52 AM, rohit.raj@nxp.com wrote:
>> From: Rohit Raj <rohit.raj@nxp.com>
>>
>> As per the current code we have API for bus probe, but the
>> bus close API is missing. This breaks the multi process
>> scenarios as objects are not cleaned while terminating the
>> secondary processes.
>>
>> This patch adds a new API rte_bus_close() for cleanup of
>> bus objects which were acquired during probe.
>>
>> Signed-off-by: Rohit Raj <rohit.raj@nxp.com>
>> ---
>>
>> v3:
>> * nit: combined nested if statements
>>
>> v2:
>> * Moved rte_bus_close call to rte_eal_cleanup path.
>>
>>   lib/librte_eal/common/eal_common_bus.c | 32 +++++++++++++++++++++++++-
>>   lib/librte_eal/include/rte_bus.h       | 25 +++++++++++++++++++-
>>   lib/librte_eal/linux/eal.c             |  1 +
>>   lib/librte_eal/rte_eal_version.map     |  3 +++
>>   4 files changed, 59 insertions(+), 2 deletions(-)
>>
>> diff --git a/lib/librte_eal/common/eal_common_bus.c
>> b/lib/librte_eal/common/eal_common_bus.c
>> index baa5b532a..5fd7cf6c5 100644
>> --- a/lib/librte_eal/common/eal_common_bus.c
>> +++ b/lib/librte_eal/common/eal_common_bus.c
>> @@ -1,5 +1,5 @@
>>   /* SPDX-License-Identifier: BSD-3-Clause
>> - * Copyright 2016 NXP
>> + * Copyright 2016,2020 NXP
>>    */
>>   #include <stdio.h>
>> @@ -56,6 +56,36 @@ rte_bus_scan(void)
>>       return 0;
>>   }
>> +int
>> +rte_bus_close(void)
>> +{
>> +    int ret;
>> +    struct rte_bus *bus, *vbus = NULL;
>> +
>> +    TAILQ_FOREACH(bus, &rte_bus_list, next) {
>> +        if (!strcmp(bus->name, "vdev")) {
>> +            vbus = bus;
>> +            continue;
>> +        }
>
> This special treatment for 'vdev' bus is done in probe to be sure physically
> device port ids start from '0',  I guess we don't need to do this for 'close'.
>
>> +
>> +        if (bus->close) {
>> +            ret = bus->close();
>> +            if (ret)
>> +                RTE_LOG(ERR, EAL, "Bus (%s) close failed.\n",
>> +                    bus->name);
>> +        }
>> +    }
>> +
>> +    if (vbus && vbus->close) {
>> +        ret = vbus->close();
>> +        if (ret)
>> +            RTE_LOG(ERR, EAL, "Bus (%s) close failed.\n",
>> +                vbus->name);
>> +    }
>> +
>> +    return 0;
>> +}
>> +
>>   /* Probe all devices of all buses */
>>   int
>>   rte_bus_probe(void)
>> diff --git a/lib/librte_eal/include/rte_bus.h b/lib/librte_eal/include/rte_bus.h
>> index d3034d0ed..af4787b18 100644
>> --- a/lib/librte_eal/include/rte_bus.h
>> +++ b/lib/librte_eal/include/rte_bus.h
>> @@ -1,5 +1,5 @@
>>   /* SPDX-License-Identifier: BSD-3-Clause
>> - * Copyright 2016 NXP
>> + * Copyright 2016,2020 NXP
>>    */
>>   #ifndef _RTE_BUS_H_
>> @@ -67,6 +67,18 @@ typedef int (*rte_bus_scan_t)(void);
>>    */
>>   typedef int (*rte_bus_probe_t)(void);
>> +/**
>> + * Implementation specific close function which is responsible for closing
>> + * devices on that bus.
>> + *
>> + * This is called while iterating over each registered bus.
>> + *
>> + * @return
>> + *    0 for successful close
>> + *    !0 for any error while closing
>> + */
>> +typedef int (*rte_bus_close_t)(void);
>> +
>
> As I checked the 'rte_fslmc_bus->close()' ops, it iterates on all devices in the
> bus instead of doing a bus level close, in that case
> instead of adding a new 'close' bus operations, will it work if existing
> 'bus->unplug(dev)' used?
> Whatever done in the 'rte_fslmc_bus->close()' per device, can it be done under
> the 'fslmc_bus_unplug()'?
>
> And in that case a 'rte_bus_remove()' API can be added which can call
> 'bus->unplug(dev)' for all buses and it will be beneficial for all buses, and it
> can fit well into the 'rte_eal_cleanup()'.
>
> What do you think?

Hi Rohit,

I have seen new version has been sent today, I want to remind above question,
can you please check it?

Re: [dpdk-dev] [PATCH v4 5/5] eal/windows: added support for rte_bus_close API

[Tal Shnaiderman]

> Subject: [dpdk-dev] [PATCH v4 5/5] eal/windows: added support for
> rte_bus_close API
> 
> External email: Use caution opening links or attachments
> 
> 
> From: Rohit Raj <rohit.raj@nxp.com>
> 
> Added support for rte_bus_close API on windows to reset the devices on
> the bus to default state.
> 
> Signed-off-by: Rohit Raj <rohit.raj@nxp.com>
> ---
>  lib/librte_eal/windows/eal.c | 2 +-
>  1 file changed, 1 insertion(+), 1 deletion(-)
> 
> diff --git a/lib/librte_eal/windows/eal.c b/lib/librte_eal/windows/eal.c index
> bc48f27ab..9eb32077b 100644
> --- a/lib/librte_eal/windows/eal.c
> +++ b/lib/librte_eal/windows/eal.c
> @@ -251,7 +251,7 @@ rte_eal_cleanup(void)  {
>         struct internal_config *internal_conf =
>                 eal_get_internal_configuration();
> -
> +       rte_bus_close();
>         eal_cleanup_config(internal_conf);
>         return 0;
>  }
> --
> 2.17.1

Hi Rohit,

You need to add rte_bus_close to lib/librte_eal/rte_eal_exports.def as it is currently the way to export functions from eal in Windows clang build.

Thanks,

Tal.

Re: [dpdk-dev] [PATCH v4 1/5] eal: add API for bus close

[David Marchand]

On Thu, Oct 8, 2020 at 5:31 PM <rohit.raj@nxp.com> wrote:
>
> From: Rohit Raj <rohit.raj@nxp.com>
>
> As per the current code we have API for bus probe, but the
> bus close API is missing. This breaks the multi process
> scenarios as objects are not cleaned while terminating the
> secondary processes.
>
> This patch adds a new API rte_bus_close() for cleanup of
> bus objects which were acquired during probe.
>
> Signed-off-by: Rohit Raj <rohit.raj@nxp.com>
> ---
>  lib/librte_eal/common/eal_common_bus.c | 32 +++++++++++++++++++++++++-
>  lib/librte_eal/include/rte_bus.h       | 27 +++++++++++++++++++++-
>  lib/librte_eal/linux/eal.c             |  1 +
>  lib/librte_eal/rte_eal_version.map     |  1 +
>  4 files changed, 59 insertions(+), 2 deletions(-)

This is a new EAL API, please update the release notes.


>
> diff --git a/lib/librte_eal/common/eal_common_bus.c b/lib/librte_eal/common/eal_common_bus.c
> index baa5b532a..5fd7cf6c5 100644
> --- a/lib/librte_eal/common/eal_common_bus.c
> +++ b/lib/librte_eal/common/eal_common_bus.c
> @@ -1,5 +1,5 @@
>  /* SPDX-License-Identifier: BSD-3-Clause
> - * Copyright 2016 NXP
> + * Copyright 2016,2020 NXP
>   */
>
>  #include <stdio.h>
> @@ -56,6 +56,36 @@ rte_bus_scan(void)
>         return 0;
>  }
>
> +int
> +rte_bus_close(void)
> +{
> +       int ret;
> +       struct rte_bus *bus, *vbus = NULL;
> +
> +       TAILQ_FOREACH(bus, &rte_bus_list, next) {
> +               if (!strcmp(bus->name, "vdev")) {
> +                       vbus = bus;
> +                       continue;
> +               }
> +
> +               if (bus->close) {
> +                       ret = bus->close();
> +                       if (ret)
> +                               RTE_LOG(ERR, EAL, "Bus (%s) close failed.\n",
> +                                       bus->name);
> +               }
> +       }
> +
> +       if (vbus && vbus->close) {
> +               ret = vbus->close();
> +               if (ret)
> +                       RTE_LOG(ERR, EAL, "Bus (%s) close failed.\n",
> +                               vbus->name);
> +       }
> +
> +       return 0;

We should propagate that an error occurred so that the caller is aware
something went wrong.


> +}
> +
>  /* Probe all devices of all buses */
>  int
>  rte_bus_probe(void)
> diff --git a/lib/librte_eal/include/rte_bus.h b/lib/librte_eal/include/rte_bus.h
> index d3034d0ed..7bf7e81b2 100644
> --- a/lib/librte_eal/include/rte_bus.h
> +++ b/lib/librte_eal/include/rte_bus.h
> @@ -1,5 +1,5 @@
>  /* SPDX-License-Identifier: BSD-3-Clause
> - * Copyright 2016 NXP
> + * Copyright 2016,2020 NXP
>   */
>
>  #ifndef _RTE_BUS_H_
> @@ -67,6 +67,20 @@ typedef int (*rte_bus_scan_t)(void);
>   */
>  typedef int (*rte_bus_probe_t)(void);
>
> +/**
> + * Implementation specific close function which is responsible for resetting all
> + * detected devices on the bus to a default state, closing UIO nodes or VFIO
> + * groups and also freeing any memory allocated during rte_bus_probe like
> + * private resources for device list.
> + *
> + * This is called while iterating over each registered bus.
> + *
> + * @return
> + *     0 for successful close
> + *     !0 for any error while closing
> + */
> +typedef int (*rte_bus_close_t)(void);
> +
>  /**
>   * Device iterator to find a device on a bus.
>   *
> @@ -248,6 +262,7 @@ struct rte_bus {
>         const char *name;            /**< Name of the bus */
>         rte_bus_scan_t scan;         /**< Scan for devices attached to bus */
>         rte_bus_probe_t probe;       /**< Probe devices on bus */
> +       rte_bus_close_t close;       /**< Close devices on bus */
>         rte_bus_find_device_t find_device; /**< Find a device on the bus */
>         rte_bus_plug_t plug;         /**< Probe single device for drivers */
>         rte_bus_unplug_t unplug;     /**< Remove single device from driver */
> @@ -301,6 +316,16 @@ int rte_bus_scan(void);
>   */
>  int rte_bus_probe(void);
>
> +/**

Missing a banner:

+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice

> + * For each device on the buses, call the device specific close.
> + *
> + * @return
> + *      0 for successful close
> + *     !0 otherwise
> + */
> +__rte_experimental
> +int rte_bus_close(void);
> +
>  /**
>   * Dump information of all the buses registered with EAL.
>   *
> diff --git a/lib/librte_eal/linux/eal.c b/lib/librte_eal/linux/eal.c
> index 9cf0e2ec0..1e1256f07 100644
> --- a/lib/librte_eal/linux/eal.c
> +++ b/lib/librte_eal/linux/eal.c
> @@ -1358,6 +1358,7 @@ rte_eal_cleanup(void)
>
>         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
>                 rte_memseg_walk(mark_freeable, NULL);
> +       rte_bus_close();
>         rte_service_finalize();
>         rte_mp_channel_cleanup();
>         rte_trace_save();

You can squash patch 4 and patch 5 in this patch so that we update all
OS-specific bits at once.


> diff --git a/lib/librte_eal/rte_eal_version.map b/lib/librte_eal/rte_eal_version.map
> index a93dea9fe..8e226b297 100644
> --- a/lib/librte_eal/rte_eal_version.map
> +++ b/lib/librte_eal/rte_eal_version.map
> @@ -400,6 +400,7 @@ EXPERIMENTAL {
>         # added in 20.11
>         __rte_eal_trace_generic_size_t;
>         rte_service_lcore_may_be_active;
> +       rte_bus_close;

Please, alphabetical order.


>  };
>
>  INTERNAL {
> --
> 2.17.1
>


-- 
David Marchand

Re: [dpdk-dev] [PATCH v4 2/5] examples/multi_process: cleanup bus objects while terminating app

[David Marchand]

On Thu, Oct 8, 2020 at 5:31 PM <rohit.raj@nxp.com> wrote:
>
> From: Rohit Raj <rohit.raj@nxp.com>
>
> Certain bus objects may need to be closed and re-acquired
> while terminating and rerunning the client application.
> Hence a signal handler is required to catch the termination
> of the App and hence closing the bus objects.
>
> This patch adds the missing signal handler in the client
> app and closes the Bus objects in both client and server
> applications when the signal Handler is called.
>
> Signed-off-by: Rohit Raj <rohit.raj@nxp.com>
> ---
>  .../multi_process/client_server_mp/mp_client/client.c | 11 +++++++++++
>  .../multi_process/client_server_mp/mp_server/main.c   |  4 +++-
>  2 files changed, 14 insertions(+), 1 deletion(-)
>
> diff --git a/examples/multi_process/client_server_mp/mp_client/client.c b/examples/multi_process/client_server_mp/mp_client/client.c
> index 361d90b54..c37516b4c 100644
> --- a/examples/multi_process/client_server_mp/mp_client/client.c
> +++ b/examples/multi_process/client_server_mp/mp_client/client.c
> @@ -11,6 +11,7 @@
>  #include <stdlib.h>
>  #include <getopt.h>
>  #include <string.h>
> +#include <signal.h>
>
>  #include <rte_common.h>
>  #include <rte_malloc.h>
> @@ -196,6 +197,14 @@ handle_packet(struct rte_mbuf *buf)
>
>  }
>
> +static void
> +signal_handler(int signal)
> +{
> +       if (signal == SIGINT)
> +               rte_eal_cleanup();
> +       exit(0);
> +}

Calling rte_eal_cleanup from a signal handler is a bad idea.
In most cases, you are racing with other threads still using DPDK resources.
https://git.dpdk.org/dpdk/commit?id=2c434431f4
https://git.dpdk.org/dpdk/commit?id=613ce6691c

This might not be a problem in this multi_process example, but let's
keep a consistent way across all examples.
Thanks.

-- 
David Marchand

Re: [dpdk-dev] [EXT] Re: [PATCH v3 1/3] eal: add API for bus close

[David Marchand]

Hello Rohit,

On Fri, Oct 9, 2020 at 6:53 AM Rohit Raj <rohit.raj@nxp.com> wrote:
> Sorry, I missed you comment last time. I will send an updated patch series after taking care of your suggested changes.

Added few more comments.
Waiting for a respin.

Thanks.


-- 
David Marchand

Re: [dpdk-dev] [PATCH v4 5/5] eal/windows: added support for rte_bus_close API

[Narcisa Ana Maria Vasile]

On Thu, Oct 08, 2020 at 09:00:47PM +0530, rohit.raj@nxp.com wrote:
> From: Rohit Raj <rohit.raj@nxp.com>
> 
> Added support for rte_bus_close API on windows to reset the devices on
> the bus to default state.
> 
> Signed-off-by: Rohit Raj <rohit.raj@nxp.com>
> ---
>  lib/librte_eal/windows/eal.c | 2 +-
>  1 file changed, 1 insertion(+), 1 deletion(-)
> 
> diff --git a/lib/librte_eal/windows/eal.c b/lib/librte_eal/windows/eal.c
> index bc48f27ab..9eb32077b 100644
> --- a/lib/librte_eal/windows/eal.c
> +++ b/lib/librte_eal/windows/eal.c
> @@ -251,7 +251,7 @@ rte_eal_cleanup(void)
>  {
>  	struct internal_config *internal_conf =
>  		eal_get_internal_configuration();
> -
> +	rte_bus_close();

Should we verify the return code here?

>  	eal_cleanup_config(internal_conf);
>  	return 0;
>  }

Re: [dpdk-dev] [PATCH v4 2/5] examples/multi_process: cleanup bus objects while terminating app

[oulijun]

在 2020/10/18 17:25, David Marchand 写道:
> On Thu, Oct 8, 2020 at 5:31 PM <rohit.raj@nxp.com> wrote:
>>
>> From: Rohit Raj <rohit.raj@nxp.com>
>>
>> Certain bus objects may need to be closed and re-acquired
>> while terminating and rerunning the client application.
>> Hence a signal handler is required to catch the termination
>> of the App and hence closing the bus objects.
>>
>> This patch adds the missing signal handler in the client
>> app and closes the Bus objects in both client and server
>> applications when the signal Handler is called.
>>
>> Signed-off-by: Rohit Raj <rohit.raj@nxp.com>
>> ---
>>   .../multi_process/client_server_mp/mp_client/client.c | 11 +++++++++++
>>   .../multi_process/client_server_mp/mp_server/main.c   |  4 +++-
>>   2 files changed, 14 insertions(+), 1 deletion(-)
>>
>> diff --git a/examples/multi_process/client_server_mp/mp_client/client.c b/examples/multi_process/client_server_mp/mp_client/client.c
>> index 361d90b54..c37516b4c 100644
>> --- a/examples/multi_process/client_server_mp/mp_client/client.c
>> +++ b/examples/multi_process/client_server_mp/mp_client/client.c
>> @@ -11,6 +11,7 @@
>>   #include <stdlib.h>
>>   #include <getopt.h>
>>   #include <string.h>
>> +#include <signal.h>
>>
>>   #include <rte_common.h>
>>   #include <rte_malloc.h>
>> @@ -196,6 +197,14 @@ handle_packet(struct rte_mbuf *buf)
>>
>>   }
>>
>> +static void
>> +signal_handler(int signal)
>> +{
>> +       if (signal == SIGINT)
>> +               rte_eal_cleanup();
>> +       exit(0);
>> +}
> 
> Calling rte_eal_cleanup from a signal handler is a bad idea.
> In most cases, you are racing with other threads still using DPDK resources.
> https://git.dpdk.org/dpdk/commit?id=2c434431f4
> https://git.dpdk.org/dpdk/commit?id=613ce6691c
> 
> This might not be a problem in this multi_process example, but let's
> keep a consistent way across all examples.
> Thanks.
Hi，
   I want to know why you don't think he's a problem. recently, we use 
the patch
https://patchwork.dpdk.org/patch/86988/
startup with multiprocess. Use the tester to send traffic and kill the 
slave process repeatedly.
The coredump exception occurs. According to my analysis, the cause is 
that resources are not released after the slave process is killed.

Thanks
Lijun Ou
> 

Re: [dpdk-dev] [PATCH v4 2/5] examples/multi_process: cleanup bus objects while terminating app

[David Marchand]

On Mon, Jan 25, 2021 at 12:07 PM oulijun <oulijun@huawei.com> wrote:
> >> +static void
> >> +signal_handler(int signal)
> >> +{
> >> +       if (signal == SIGINT)
> >> +               rte_eal_cleanup();
> >> +       exit(0);
> >> +}
> >
> > Calling rte_eal_cleanup from a signal handler is a bad idea.
> > In most cases, you are racing with other threads still using DPDK resources.
> > https://git.dpdk.org/dpdk/commit?id=2c434431f4
> > https://git.dpdk.org/dpdk/commit?id=613ce6691c
> >
> > This might not be a problem in this multi_process example, but let's
> > keep a consistent way across all examples.
> > Thanks.
> Hi，
>    I want to know why you don't think he's a problem. recently, we use
> the patch
> https://patchwork.dpdk.org/patch/86988/
> startup with multiprocess. Use the tester to send traffic and kill the
> slave process repeatedly.
> The coredump exception occurs. According to my analysis, the cause is
> that resources are not released after the slave process is killed.
>

Not sure I get your question and I don't see the relation with the
testpmd patch.
I did not say we must not release resources.

Just to rephrase my previous mail:

Generally speaking, calling rte_eal_cleanup() from a signal handler is
wrong since it creates races with other threads.
I recommend putting in place a synchronisation mechanism so that all
worker threads break out of their processing loop and the main thread
synchronizes/joins with them before calling rte_eal_cleanup() in its
exit path.

Now, for this patch, in
examples/multi_process/client_server_mp/mp_client/client.c, this
secondary process code seems to only have one thread (but I might be
wrong).
If this is true, then no race in theory => my comment "This might not
be a problem in this multi_process example".


-- 
David Marchand

[PATCH v5 1/2] eal: add API for bus close

[rohit.raj]

From: Rohit Raj <rohit.raj@nxp.com>

As per the current code we have API for bus probe, but the
bus close API is missing. This breaks the multi process
scenarios as objects are not cleaned while terminating the
secondary processes.

This patch adds a new API rte_bus_close() for cleanup of
bus objects which were acquired during probe.

Signed-off-by: Rohit Raj <rohit.raj@nxp.com>
---

Rebased on this patch series:
https://patches.dpdk.org/project/dpdk/list/?series=21049

v5:
* Updated release notes for new feature and API change.
* Added support for error checking while closing bus.
* Added experimental banner for new API.
* Squashed changes related to freebsd and windows into single patch.
* Discarded patch to fix a bug which is already fixed on latest
  release.

v4:
* Added comments to clarify responsibility of rte_bus_close.
* Added support for rte_bus_close on freebsd.
* Added support for rte_bus_close on windows.

v3:
* nit: combined nested if statements.

v2:
* Moved rte_bus_close call to rte_eal_cleanup path.

 doc/guides/rel_notes/release_22_03.rst |  8 +++++++
 lib/eal/common/eal_common_bus.c        | 33 +++++++++++++++++++++++++-
 lib/eal/freebsd/eal.c                  |  1 +
 lib/eal/include/rte_bus.h              | 30 ++++++++++++++++++++++-
 lib/eal/linux/eal.c                    |  8 +++++++
 lib/eal/version.map                    |  3 +++
 lib/eal/windows/eal.c                  |  1 +
 7 files changed, 82 insertions(+), 2 deletions(-)

diff --git a/doc/guides/rel_notes/release_22_03.rst b/doc/guides/rel_notes/release_22_03.rst
index 6d99d1eaa9..7417606a2a 100644
--- a/doc/guides/rel_notes/release_22_03.rst
+++ b/doc/guides/rel_notes/release_22_03.rst
@@ -55,6 +55,11 @@ New Features
      Also, make sure to start the actual text at the margin.
      =======================================================
 
+    * **Added support to close bus.**
+
+      Added capability to allow a user to do cleanup of bus objects which
+      were acquired during bus probe.
+
 
 Removed Items
 -------------
@@ -84,6 +89,9 @@ API Changes
    Also, make sure to start the actual text at the margin.
    =======================================================
 
+   * eal: Added new API ``rte_bus_close`` to perform cleanup bus objects which
+     were acquired during bus probe.
+
 
 ABI Changes
 -----------
diff --git a/lib/eal/common/eal_common_bus.c b/lib/eal/common/eal_common_bus.c
index baa5b532af..2c3c0a90d2 100644
--- a/lib/eal/common/eal_common_bus.c
+++ b/lib/eal/common/eal_common_bus.c
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright 2016 NXP
+ * Copyright 2016,2022 NXP
  */
 
 #include <stdio.h>
@@ -85,6 +85,37 @@ rte_bus_probe(void)
 	return 0;
 }
 
+/* Close all devices of all buses */
+int
+rte_bus_close(void)
+{
+	int ret;
+	struct rte_bus *bus, *vbus = NULL;
+
+	TAILQ_FOREACH(bus, &rte_bus_list, next) {
+		if (!strcmp(bus->name, "vdev")) {
+			vbus = bus;
+			continue;
+		}
+
+		if (bus->close) {
+			ret = bus->close();
+			if (ret)
+				RTE_LOG(ERR, EAL, "Bus (%s) close failed.\n",
+					bus->name);
+		}
+	}
+
+	if (vbus && vbus->close) {
+		ret = vbus->close();
+		if (ret)
+			RTE_LOG(ERR, EAL, "Bus (%s) close failed.\n",
+				vbus->name);
+	}
+
+	return 0;
+}
+
 /* Dump information of a single bus */
 static int
 bus_dump_one(FILE *f, struct rte_bus *bus)
diff --git a/lib/eal/freebsd/eal.c b/lib/eal/freebsd/eal.c
index a1cd2462db..87d70c6898 100644
--- a/lib/eal/freebsd/eal.c
+++ b/lib/eal/freebsd/eal.c
@@ -984,6 +984,7 @@ rte_eal_cleanup(void)
 {
 	struct internal_config *internal_conf =
 		eal_get_internal_configuration();
+	rte_bus_close();
 	rte_service_finalize();
 	rte_mp_channel_cleanup();
 	/* after this point, any DPDK pointers will become dangling */
diff --git a/lib/eal/include/rte_bus.h b/lib/eal/include/rte_bus.h
index bbbb6efd28..c6211bbd95 100644
--- a/lib/eal/include/rte_bus.h
+++ b/lib/eal/include/rte_bus.h
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright 2016 NXP
+ * Copyright 2016,2022 NXP
  */
 
 #ifndef _RTE_BUS_H_
@@ -66,6 +66,23 @@ typedef int (*rte_bus_scan_t)(void);
  */
 typedef int (*rte_bus_probe_t)(void);
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Implementation specific close function which is responsible for resetting all
+ * detected devices on the bus to a default state, closing UIO nodes or VFIO
+ * groups and also freeing any memory allocated during rte_bus_probe like
+ * private resources for device list.
+ *
+ * This is called while iterating over each registered bus.
+ *
+ * @return
+ *	0 for successful close
+ *	!0 for any error while closing
+ */
+typedef int (*rte_bus_close_t)(void);
+
 /**
  * Device iterator to find a device on a bus.
  *
@@ -263,6 +280,7 @@ struct rte_bus {
 	const char *name;            /**< Name of the bus */
 	rte_bus_scan_t scan;         /**< Scan for devices attached to bus */
 	rte_bus_probe_t probe;       /**< Probe devices on bus */
+	rte_bus_close_t close;       /**< Close devices on bus */
 	rte_bus_find_device_t find_device; /**< Find a device on the bus */
 	rte_bus_plug_t plug;         /**< Probe single device for drivers */
 	rte_bus_unplug_t unplug;     /**< Remove single device from driver */
@@ -317,6 +335,16 @@ int rte_bus_scan(void);
  */
 int rte_bus_probe(void);
 
+/**
+ * For each device on the buses, call the device specific close.
+ *
+ * @return
+ *	 0 for successful close
+ *	!0 otherwise
+ */
+__rte_experimental
+int rte_bus_close(void);
+
 /**
  * Dump information of all the buses registered with EAL.
  *
diff --git a/lib/eal/linux/eal.c b/lib/eal/linux/eal.c
index 60b4924838..5c60131e46 100644
--- a/lib/eal/linux/eal.c
+++ b/lib/eal/linux/eal.c
@@ -1362,6 +1362,14 @@ rte_eal_cleanup(void)
 
 	if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 		rte_memseg_walk(mark_freeable, NULL);
+
+	/* Close all the buses and devices/drivers on them */
+	if (rte_bus_close()) {
+		rte_eal_init_alert("Cannot close devices");
+		rte_errno = ENOTSUP;
+		return -1;
+	}
+
 	rte_service_finalize();
 	rte_mp_channel_cleanup();
 	/* after this point, any DPDK pointers will become dangling */
diff --git a/lib/eal/version.map b/lib/eal/version.map
index ab28c22791..39882dbbd5 100644
--- a/lib/eal/version.map
+++ b/lib/eal/version.map
@@ -420,6 +420,9 @@ EXPERIMENTAL {
 	rte_intr_instance_free;
 	rte_intr_type_get;
 	rte_intr_type_set;
+
+	# added in 22.03
+	rte_bus_close;
 };
 
 INTERNAL {
diff --git a/lib/eal/windows/eal.c b/lib/eal/windows/eal.c
index 67db7f099a..5915ab6291 100644
--- a/lib/eal/windows/eal.c
+++ b/lib/eal/windows/eal.c
@@ -260,6 +260,7 @@ rte_eal_cleanup(void)
 	struct internal_config *internal_conf =
 		eal_get_internal_configuration();
 
+	rte_bus_close();
 	eal_intr_thread_cancel();
 	eal_mem_virt2iova_cleanup();
 	/* after this point, any DPDK pointers will become dangling */
-- 
2.17.1

[PATCH v5 2/2] bus/fslmc: support bus close API

[rohit.raj]

From: Rohit Raj <rohit.raj@nxp.com>

This patch add support for closing the bus objects which
were acquired In the bus probe.

Some devices need to be cleaned while in both primary and
secondary process and while some need to be cleaned only in
case of primary process.

The devices are closed as per the white list used while
creating the objects in a particular process.

Signed-off-by: Rohit Raj <rohit.raj@nxp.com>
---
 drivers/bus/fslmc/fslmc_bus.c            | 15 +++-
 drivers/bus/fslmc/fslmc_vfio.c           | 89 +++++++++++++++++++++++-
 drivers/bus/fslmc/fslmc_vfio.h           |  3 +-
 drivers/bus/fslmc/mc/fsl_dpcon.h         |  3 +-
 drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c | 30 +++++++-
 drivers/bus/fslmc/portal/dpaa2_hw_dpci.c | 30 +++++++-
 drivers/bus/fslmc/portal/dpaa2_hw_dpio.c | 36 +++++++++-
 drivers/bus/fslmc/portal/dpaa2_hw_dprc.c | 31 ++++++++-
 drivers/bus/fslmc/rte_fslmc.h            |  5 +-
 drivers/bus/fslmc/version.map            |  1 +
 drivers/event/dpaa2/dpaa2_hw_dpcon.c     | 31 ++++++++-
 drivers/net/dpaa2/dpaa2_mux.c            | 20 +++++-
 12 files changed, 282 insertions(+), 12 deletions(-)

diff --git a/drivers/bus/fslmc/fslmc_bus.c b/drivers/bus/fslmc/fslmc_bus.c
index a3c0d838c4..79053233a1 100644
--- a/drivers/bus/fslmc/fslmc_bus.c
+++ b/drivers/bus/fslmc/fslmc_bus.c
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
- *   Copyright 2016,2018-2021 NXP
+ *   Copyright 2016,2018-2022 NXP
  *
  */
 
@@ -493,6 +493,18 @@ rte_fslmc_probe(void)
 	return 0;
 }
 
+static int
+rte_fslmc_close(void)
+{
+	int ret = 0;
+
+	ret = fslmc_vfio_close_group();
+	if (ret)
+		DPAA2_BUS_ERR("Unable to close devices %d", ret);
+
+	return 0;
+}
+
 static struct rte_device *
 rte_fslmc_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
 		      const void *data)
@@ -668,6 +680,7 @@ struct rte_fslmc_bus rte_fslmc_bus = {
 	.bus = {
 		.scan = rte_fslmc_scan,
 		.probe = rte_fslmc_probe,
+		.close = rte_fslmc_close,
 		.parse = rte_fslmc_parse,
 		.find_device = rte_fslmc_find_device,
 		.get_iommu_class = rte_dpaa2_get_iommu_class,
diff --git a/drivers/bus/fslmc/fslmc_vfio.c b/drivers/bus/fslmc/fslmc_vfio.c
index 1b89a56bbc..59bb0f0171 100644
--- a/drivers/bus/fslmc/fslmc_vfio.c
+++ b/drivers/bus/fslmc/fslmc_vfio.c
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
  *   Copyright (c) 2015-2016 Freescale Semiconductor, Inc. All rights reserved.
- *   Copyright 2016-2021 NXP
+ *   Copyright 2016-2022 NXP
  *
  */
 
@@ -1066,3 +1066,90 @@ fslmc_vfio_setup_group(void)
 
 	return 0;
 }
+
+static void
+fslmc_close_iodevices(struct rte_dpaa2_device *dev)
+{
+	struct rte_dpaa2_object *object = NULL;
+	struct rte_dpaa2_driver *drv;
+	int ret, probe_all;
+
+	switch (dev->dev_type) {
+	case DPAA2_IO:
+	case DPAA2_CON:
+	case DPAA2_CI:
+	case DPAA2_BPOOL:
+	case DPAA2_MUX:
+		TAILQ_FOREACH(object, &dpaa2_obj_list, next) {
+			if (dev->dev_type == object->dev_type)
+				object->close(dev->object_id);
+			else
+				continue;
+		}
+		break;
+	case DPAA2_ETH:
+	case DPAA2_CRYPTO:
+	case DPAA2_QDMA:
+		probe_all = rte_fslmc_bus.bus.conf.scan_mode !=
+			    RTE_BUS_SCAN_ALLOWLIST;
+		TAILQ_FOREACH(drv, &rte_fslmc_bus.driver_list, next) {
+			if (drv->drv_type != dev->dev_type)
+				continue;
+			if (rte_dev_is_probed(&dev->device))
+				continue;
+			if (probe_all ||
+			    (dev->device.devargs &&
+			     dev->device.devargs->policy ==
+			     RTE_DEV_ALLOWED)) {
+				ret = drv->remove(dev);
+				if (ret)
+					DPAA2_BUS_ERR("Unable to remove");
+			}
+		}
+		break;
+	default:
+		break;
+	}
+
+	DPAA2_BUS_LOG(DEBUG, "Device (%s) Closed",
+		      dev->device.name);
+}
+
+int
+fslmc_vfio_close_group(void)
+{
+	struct rte_dpaa2_device *dev, *dev_temp;
+
+	RTE_TAILQ_FOREACH_SAFE(dev, &rte_fslmc_bus.device_list, next, dev_temp) {
+		if (dev->device.devargs &&
+		    dev->device.devargs->policy == RTE_DEV_BLOCKED) {
+			DPAA2_BUS_LOG(DEBUG, "%s Blacklisted, skipping",
+				      dev->device.name);
+			TAILQ_REMOVE(&rte_fslmc_bus.device_list, dev, next);
+				continue;
+		}
+		switch (dev->dev_type) {
+		case DPAA2_ETH:
+		case DPAA2_CRYPTO:
+		case DPAA2_QDMA:
+		case DPAA2_IO:
+			fslmc_close_iodevices(dev);
+			break;
+		case DPAA2_CON:
+		case DPAA2_CI:
+		case DPAA2_BPOOL:
+		case DPAA2_MUX:
+			if (rte_eal_process_type() == RTE_PROC_SECONDARY)
+				continue;
+
+			fslmc_close_iodevices(dev);
+			break;
+		case DPAA2_DPRTC:
+		default:
+			DPAA2_BUS_DEBUG("Device cannot be closed: Not supported (%s)",
+					dev->device.name);
+		}
+	}
+
+	return 0;
+}
diff --git a/drivers/bus/fslmc/fslmc_vfio.h b/drivers/bus/fslmc/fslmc_vfio.h
index 133606a9fd..5f2f8119fb 100644
--- a/drivers/bus/fslmc/fslmc_vfio.h
+++ b/drivers/bus/fslmc/fslmc_vfio.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
  *   Copyright (c) 2015-2016 Freescale Semiconductor, Inc. All rights reserved.
- *   Copyright 2016,2019 NXP
+ *   Copyright 2016,2019,2021-2022 NXP
  *
  */
 
@@ -55,6 +55,7 @@ int rte_dpaa2_vfio_setup_intr(struct rte_intr_handle *intr_handle,
 
 int fslmc_vfio_setup_group(void);
 int fslmc_vfio_process_group(void);
+int fslmc_vfio_close_group(void);
 char *fslmc_get_container(void);
 int fslmc_get_container_group(int *gropuid);
 int rte_fslmc_vfio_dmamap(void);
diff --git a/drivers/bus/fslmc/mc/fsl_dpcon.h b/drivers/bus/fslmc/mc/fsl_dpcon.h
index 7caa6c68a1..a158f576b5 100644
--- a/drivers/bus/fslmc/mc/fsl_dpcon.h
+++ b/drivers/bus/fslmc/mc/fsl_dpcon.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
  *
  * Copyright 2013-2016 Freescale Semiconductor Inc.
- * Copyright 2017-2019 NXP
+ * Copyright 2017-2019,2021-2022 NXP
  *
  */
 #ifndef __FSL_DPCON_H
@@ -26,6 +26,7 @@ int dpcon_open(struct fsl_mc_io *mc_io,
 	       int dpcon_id,
 	       uint16_t *token);
 
+__rte_internal
 int dpcon_close(struct fsl_mc_io *mc_io,
 		uint32_t cmd_flags,
 		uint16_t token);
diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c
index 122aa1740d..4cde4890d9 100644
--- a/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c
+++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
  *   Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
- *   Copyright 2016 NXP
+ *   Copyright 2016,2021-2022 NXP
  *
  */
 
@@ -33,6 +33,19 @@ TAILQ_HEAD(dpbp_dev_list, dpaa2_dpbp_dev);
 static struct dpbp_dev_list dpbp_dev_list
 	= TAILQ_HEAD_INITIALIZER(dpbp_dev_list); /*!< DPBP device list */
 
+static struct dpaa2_dpbp_dev *get_dpbp_from_id(uint32_t dpbp_id)
+{
+	struct dpaa2_dpbp_dev *dpbp_dev = NULL;
+
+	/* Get DPBP dev handle from list using index */
+	TAILQ_FOREACH(dpbp_dev, &dpbp_dev_list, next) {
+		if (dpbp_dev->dpbp_id == dpbp_id)
+			break;
+	}
+
+	return dpbp_dev;
+}
+
 static int
 dpaa2_create_dpbp_device(int vdev_fd __rte_unused,
 			 struct vfio_device_info *obj_info __rte_unused,
@@ -109,6 +122,20 @@ void dpaa2_free_dpbp_dev(struct dpaa2_dpbp_dev *dpbp)
 	}
 }
 
+static void dpaa2_close_dpbp_device(int object_id)
+{
+	struct dpaa2_dpbp_dev *dpbp_dev = NULL;
+
+	dpbp_dev = get_dpbp_from_id((uint32_t)object_id);
+
+	if (dpbp_dev) {
+		dpaa2_free_dpbp_dev(dpbp_dev);
+		dpbp_close(&dpbp_dev->dpbp, CMD_PRI_LOW, dpbp_dev->token);
+		TAILQ_REMOVE(&dpbp_dev_list, dpbp_dev, next);
+		rte_free(dpbp_dev);
+	}
+}
+
 int dpaa2_dpbp_supported(void)
 {
 	if (TAILQ_EMPTY(&dpbp_dev_list))
@@ -119,6 +146,7 @@ int dpaa2_dpbp_supported(void)
 static struct rte_dpaa2_object rte_dpaa2_dpbp_obj = {
 	.dev_type = DPAA2_BPOOL,
 	.create = dpaa2_create_dpbp_device,
+	.close = dpaa2_close_dpbp_device,
 };
 
 RTE_PMD_REGISTER_DPAA2_OBJECT(dpbp, rte_dpaa2_dpbp_obj);
diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c
index 8ed969c7c0..5eae63a6fd 100644
--- a/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c
+++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
- *   Copyright 2017 NXP
+ *   Copyright 2017,2021-2022 NXP
  *
  */
 
@@ -30,6 +30,19 @@ TAILQ_HEAD(dpci_dev_list, dpaa2_dpci_dev);
 static struct dpci_dev_list dpci_dev_list
 	= TAILQ_HEAD_INITIALIZER(dpci_dev_list); /*!< DPCI device list */
 
+static struct dpaa2_dpci_dev *get_dpci_from_id(uint32_t dpci_id)
+{
+	struct dpaa2_dpci_dev *dpci_dev = NULL;
+
+	/* Get DPCI dev handle from list using index */
+	TAILQ_FOREACH(dpci_dev, &dpci_dev_list, next) {
+		if (dpci_dev->dpci_id == dpci_id)
+			break;
+	}
+
+	return dpci_dev;
+}
+
 static int
 rte_dpaa2_create_dpci_device(int vdev_fd __rte_unused,
 			     struct vfio_device_info *obj_info __rte_unused,
@@ -179,9 +192,24 @@ void rte_dpaa2_free_dpci_dev(struct dpaa2_dpci_dev *dpci)
 	}
 }
 
+static void rte_dpaa2_close_dpci_device(int object_id)
+{
+	struct dpaa2_dpci_dev *dpci_dev = NULL;
+
+	dpci_dev = get_dpci_from_id((uint32_t)object_id);
+
+	if (dpci_dev) {
+		rte_dpaa2_free_dpci_dev(dpci_dev);
+		dpci_close(&dpci_dev->dpci, CMD_PRI_LOW, dpci_dev->token);
+		TAILQ_REMOVE(&dpci_dev_list, dpci_dev, next);
+		rte_free(dpci_dev);
+	}
+}
+
 static struct rte_dpaa2_object rte_dpaa2_dpci_obj = {
 	.dev_type = DPAA2_CI,
 	.create = rte_dpaa2_create_dpci_device,
+	.close = rte_dpaa2_close_dpci_device,
 };
 
 RTE_PMD_REGISTER_DPAA2_OBJECT(dpci, rte_dpaa2_dpci_obj);
diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
index 2210a0fa4a..f8676ec53b 100644
--- a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
+++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
  *   Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
- *   Copyright 2016-2019 NXP
+ *   Copyright 2016-2019,2021-2022 NXP
  *
  */
 #include <unistd.h>
@@ -86,6 +86,19 @@ static int dpaa2_cluster_sz = 2;
  * Cluster 4 (ID = x07) : CPU14, CPU15;
  */
 
+static struct dpaa2_dpio_dev *get_dpio_dev_from_id(int32_t dpio_id)
+{
+	struct dpaa2_dpio_dev *dpio_dev = NULL;
+
+	/* Get DPIO dev handle from list using index */
+	TAILQ_FOREACH(dpio_dev, &dpio_dev_list, next) {
+		if (dpio_dev->hw_id == dpio_id)
+			break;
+	}
+
+	return dpio_dev;
+}
+
 static int
 dpaa2_get_core_id(void)
 {
@@ -634,9 +647,30 @@ dpaa2_free_eq_descriptors(void)
 			MAX_EQ_RESP_ENTRIES - 1;
 }
 
+static void
+dpaa2_close_dpio_device(int object_id)
+{
+	struct dpaa2_dpio_dev *dpio_dev = NULL;
+
+	dpio_dev = get_dpio_dev_from_id((int32_t)object_id);
+
+	if (dpio_dev) {
+		if (dpio_dev->dpio) {
+			dpio_disable(dpio_dev->dpio, CMD_PRI_LOW,
+				     dpio_dev->token);
+			dpio_close(dpio_dev->dpio, CMD_PRI_LOW,
+				   dpio_dev->token);
+			rte_free(dpio_dev->dpio);
+		}
+		TAILQ_REMOVE(&dpio_dev_list, dpio_dev, next);
+		rte_free(dpio_dev);
+	}
+}
+
 static struct rte_dpaa2_object rte_dpaa2_dpio_obj = {
 	.dev_type = DPAA2_IO,
 	.create = dpaa2_create_dpio_device,
+	.close = dpaa2_close_dpio_device,
 };
 
 RTE_PMD_REGISTER_DPAA2_OBJECT(dpio, rte_dpaa2_dpio_obj);
diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dprc.c b/drivers/bus/fslmc/portal/dpaa2_hw_dprc.c
index ca1d0304d5..ec35fdc65a 100644
--- a/drivers/bus/fslmc/portal/dpaa2_hw_dprc.c
+++ b/drivers/bus/fslmc/portal/dpaa2_hw_dprc.c
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
- *   Copyright 2021 NXP
+ *   Copyright 2021-2022 NXP
  *
  */
 
@@ -92,9 +92,38 @@ rte_dpaa2_create_dprc_device(int vdev_fd __rte_unused,
 	return 0;
 }
 
+static struct dpaa2_dprc_dev *
+get_dprc_from_id(uint32_t dprc_id)
+{
+	struct dpaa2_dprc_dev *dprc_dev = NULL;
+
+	/* Get DPRC dev handle from list using index */
+	TAILQ_FOREACH(dprc_dev, &dprc_dev_list, next) {
+		if (dprc_dev->dprc_id == dprc_id)
+			break;
+	}
+
+	return dprc_dev;
+}
+
+static void
+rte_dpaa2_close_dprc_device(int object_id)
+{
+	struct dpaa2_dprc_dev *dprc_dev = NULL;
+
+	dprc_dev = get_dprc_from_id((uint32_t)object_id);
+
+	if (dprc_dev) {
+		dprc_close(&dprc_dev->dprc, CMD_PRI_LOW, dprc_dev->token);
+		TAILQ_REMOVE(&dprc_dev_list, dprc_dev, next);
+		rte_free(dprc_dev);
+	}
+}
+
 static struct rte_dpaa2_object rte_dpaa2_dprc_obj = {
 	.dev_type = DPAA2_DPRC,
 	.create = rte_dpaa2_create_dprc_device,
+	.close = rte_dpaa2_close_dprc_device,
 };
 
 RTE_PMD_REGISTER_DPAA2_OBJECT(dprc, rte_dpaa2_dprc_obj);
diff --git a/drivers/bus/fslmc/rte_fslmc.h b/drivers/bus/fslmc/rte_fslmc.h
index 12b586b13b..f9753d8af2 100644
--- a/drivers/bus/fslmc/rte_fslmc.h
+++ b/drivers/bus/fslmc/rte_fslmc.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
- *   Copyright 2016,2021 NXP
+ *   Copyright 2016,2021-2022 NXP
  *
  */
 
@@ -104,6 +104,8 @@ typedef int (*rte_dpaa2_obj_create_t)(int vdev_fd,
 				      struct vfio_device_info *obj_info,
 				      int object_id);
 
+typedef void (*rte_dpaa2_obj_close_t)(int object_id);
+
 /**
  * A structure describing a DPAA2 object.
  */
@@ -112,6 +114,7 @@ struct rte_dpaa2_object {
 	const char *name;                   /**< Name of Object. */
 	enum rte_dpaa2_dev_type dev_type;   /**< Type of device */
 	rte_dpaa2_obj_create_t create;
+	rte_dpaa2_obj_close_t close;
 };
 
 /**
diff --git a/drivers/bus/fslmc/version.map b/drivers/bus/fslmc/version.map
index 79b29746b6..465c4d10e6 100644
--- a/drivers/bus/fslmc/version.map
+++ b/drivers/bus/fslmc/version.map
@@ -34,6 +34,7 @@ INTERNAL {
 	dpci_get_opr;
 	dpci_set_opr;
 	dpci_set_rx_queue;
+	dpcon_close;
 	dpcon_get_attributes;
 	dpcon_open;
 	dpdmai_close;
diff --git a/drivers/event/dpaa2/dpaa2_hw_dpcon.c b/drivers/event/dpaa2/dpaa2_hw_dpcon.c
index acc1fde771..1de3ec9a81 100644
--- a/drivers/event/dpaa2/dpaa2_hw_dpcon.c
+++ b/drivers/event/dpaa2/dpaa2_hw_dpcon.c
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
- *   Copyright 2017 NXP
+ *   Copyright 2017,2022 NXP
  *
  */
 
@@ -30,6 +30,19 @@ TAILQ_HEAD(dpcon_dev_list, dpaa2_dpcon_dev);
 static struct dpcon_dev_list dpcon_dev_list
 	= TAILQ_HEAD_INITIALIZER(dpcon_dev_list); /*!< DPCON device list */
 
+static struct dpaa2_dpcon_dev *get_dpcon_from_id(uint32_t dpcon_id)
+{
+	struct dpaa2_dpcon_dev *dpcon_dev = NULL;
+
+	/* Get DPCONC dev handle from list using index */
+	TAILQ_FOREACH(dpcon_dev, &dpcon_dev_list, next) {
+		if (dpcon_dev->dpcon_id == dpcon_id)
+			break;
+	}
+
+	return dpcon_dev;
+}
+
 static int
 rte_dpaa2_create_dpcon_device(int dev_fd __rte_unused,
 			      struct vfio_device_info *obj_info __rte_unused,
@@ -105,9 +118,25 @@ void rte_dpaa2_free_dpcon_dev(struct dpaa2_dpcon_dev *dpcon)
 	}
 }
 
+static void
+rte_dpaa2_close_dpcon_device(int object_id)
+{
+	struct dpaa2_dpcon_dev *dpcon_dev = NULL;
+
+	dpcon_dev = get_dpcon_from_id((uint32_t)object_id);
+
+	if (dpcon_dev) {
+		rte_dpaa2_free_dpcon_dev(dpcon_dev);
+		dpcon_close(&dpcon_dev->dpcon, CMD_PRI_LOW, dpcon_dev->token);
+		TAILQ_REMOVE(&dpcon_dev_list, dpcon_dev, next);
+		rte_free(dpcon_dev);
+	}
+}
+
 static struct rte_dpaa2_object rte_dpaa2_dpcon_obj = {
 	.dev_type = DPAA2_CON,
 	.create = rte_dpaa2_create_dpcon_device,
+	.close = rte_dpaa2_close_dpcon_device,
 };
 
 RTE_PMD_REGISTER_DPAA2_OBJECT(dpcon, rte_dpaa2_dpcon_obj);
diff --git a/drivers/net/dpaa2/dpaa2_mux.c b/drivers/net/dpaa2/dpaa2_mux.c
index d347f4df51..e9289ca808 100644
--- a/drivers/net/dpaa2/dpaa2_mux.c
+++ b/drivers/net/dpaa2/dpaa2_mux.c
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright 2018-2021 NXP
+ * Copyright 2018-2022 NXP
  */
 
 #include <sys/queue.h>
@@ -44,7 +44,7 @@ static struct dpaa2_dpdmux_dev *get_dpdmux_from_id(uint32_t dpdmux_id)
 {
 	struct dpaa2_dpdmux_dev *dpdmux_dev = NULL;
 
-	/* Get DPBP dev handle from list using index */
+	/* Get DPDMUX dev handle from list using index */
 	TAILQ_FOREACH(dpdmux_dev, &dpdmux_dev_list, next) {
 		if (dpdmux_dev->dpdmux_id == dpdmux_id)
 			break;
@@ -359,9 +359,25 @@ dpaa2_create_dpdmux_device(int vdev_fd __rte_unused,
 	return -1;
 }
 
+static void
+dpaa2_close_dpdmux_device(int object_id)
+{
+	struct dpaa2_dpdmux_dev *dpdmux_dev;
+
+	dpdmux_dev = get_dpdmux_from_id((uint32_t)object_id);
+
+	if (dpdmux_dev) {
+		dpdmux_close(&dpdmux_dev->dpdmux, CMD_PRI_LOW,
+			     dpdmux_dev->token);
+		TAILQ_REMOVE(&dpdmux_dev_list, dpdmux_dev, next);
+		rte_free(dpdmux_dev);
+	}
+}
+
 static struct rte_dpaa2_object rte_dpaa2_dpdmux_obj = {
 	.dev_type = DPAA2_MUX,
 	.create = dpaa2_create_dpdmux_device,
+	.close = dpaa2_close_dpdmux_device,
 };
 
 RTE_PMD_REGISTER_DPAA2_OBJECT(dpdmux, rte_dpaa2_dpdmux_obj);
-- 
2.17.1

Re: [PATCH v5 1/2] eal: add API for bus close

[Thomas Monjalon]

Hi,

10/01/2022 06:26, rohit.raj@nxp.com:
> From: Rohit Raj <rohit.raj@nxp.com>
> 
> As per the current code we have API for bus probe, but the
> bus close API is missing. This breaks the multi process
> scenarios as objects are not cleaned while terminating the
> secondary processes.
> 
> This patch adds a new API rte_bus_close() for cleanup of
> bus objects which were acquired during probe.

I don't understand how closing all devices of a bus will help better
than just closing all devices.

As Ferruh already suggested in the past,
we could force closing all devices in rte_eal_cleanup().
And we already have the function rte_dev_remove().

RE: [EXT] Re: [PATCH v5 1/2] eal: add API for bus close

[Rohit Raj]

Hi Thomas,

This "rte_bus_close" API is introduced to do the opposite of what "rte_bus_probe" does. Just like there are plug and unplug APIs for plugging and unplugging a single device.

The API you mentioned, "rte_dev_remove" supports only rte_device.  But we also need to close/remove private devices of dpaa and fslmc buses which are not exposed directly to user (eg: mempool device). 
Note that these private devices/bus objects are not associated with a particular rte_device but they are available as a resource to be used by any of the device based on these hardware specific buses. 
So, to close these devices, we need a new API which can do this for us. That is why "rte_bus_close" is required.

Regards,
Rohit Raj

> -----Original Message-----
> From: Thomas Monjalon <thomas@monjalon.net>
> Sent: Wednesday, January 19, 2022 4:02 PM
> To: Rohit Raj <rohit.raj@nxp.com>
> Cc: Bruce Richardson <bruce.richardson@intel.com>; Ray Kinsella
> <mdr@ashroe.eu>; Dmitry Kozlyuk <dmitry.kozliuk@gmail.com>; Narcisa Ana
> Maria Vasile <navasile@linux.microsoft.com>; Dmitry Malloy
> <dmitrym@microsoft.com>; Pallavi Kadam <pallavi.kadam@intel.com>;
> dev@dpdk.org; Nipun Gupta <nipun.gupta@nxp.com>; Sachin Saxena
> <sachin.saxena@nxp.com>; Hemant Agrawal <hemant.agrawal@nxp.com>;
> ferruh.yigit@intel.com; david.marchand@redhat.com
> Subject: [EXT] Re: [PATCH v5 1/2] eal: add API for bus close
> 
> Caution: EXT Email
> 
> Hi,
> 
> 10/01/2022 06:26, rohit.raj@nxp.com:
> > From: Rohit Raj <rohit.raj@nxp.com>
> >
> > As per the current code we have API for bus probe, but the bus close
> > API is missing. This breaks the multi process scenarios as objects are
> > not cleaned while terminating the secondary processes.
> >
> > This patch adds a new API rte_bus_close() for cleanup of bus objects
> > which were acquired during probe.
> 
> I don't understand how closing all devices of a bus will help better than just
> closing all devices.
> 
> As Ferruh already suggested in the past, we could force closing all devices in
> rte_eal_cleanup().
> And we already have the function rte_dev_remove().
> 
> 

Re: [EXT] Re: [PATCH v5 1/2] eal: add API for bus close

[Thomas Monjalon]

20/01/2022 15:51, Rohit Raj:
> Hi Thomas,
> 
> This "rte_bus_close" API is introduced to do the opposite of what "rte_bus_probe" does. Just like there are plug and unplug APIs for plugging and unplugging a single device.
> 
> The API you mentioned, "rte_dev_remove" supports only rte_device.  But we also need to close/remove private devices of dpaa and fslmc buses which are not exposed directly to user (eg: mempool device). 
> Note that these private devices/bus objects are not associated with a particular rte_device but they are available as a resource to be used by any of the device based on these hardware specific buses. 
> So, to close these devices, we need a new API which can do this for us. That is why "rte_bus_close" is required.

You mean some resources are associated to a bus but not to a device?
It lools very weird. A resource on a bus *is* a device.

PS: please avoid top-post

> From: Thomas Monjalon <thomas@monjalon.net>
> > 10/01/2022 06:26, rohit.raj@nxp.com:
> > > From: Rohit Raj <rohit.raj@nxp.com>
> > >
> > > As per the current code we have API for bus probe, but the bus close
> > > API is missing. This breaks the multi process scenarios as objects are
> > > not cleaned while terminating the secondary processes.
> > >
> > > This patch adds a new API rte_bus_close() for cleanup of bus objects
> > > which were acquired during probe.
> > 
> > I don't understand how closing all devices of a bus will help better than just
> > closing all devices.
> > 
> > As Ferruh already suggested in the past, we could force closing all devices in
> > rte_eal_cleanup().
> > And we already have the function rte_dev_remove().

RE: [EXT] Re: [PATCH v5 1/2] eal: add API for bus close

[Rohit Raj]

> -----Original Message-----
> From: Thomas Monjalon <thomas@monjalon.net>
> Sent: Thursday, January 20, 2022 8:28 PM
> To: Rohit Raj <rohit.raj@nxp.com>
> Cc: Bruce Richardson <bruce.richardson@intel.com>; Ray Kinsella
> <mdr@ashroe.eu>; Dmitry Kozlyuk <dmitry.kozliuk@gmail.com>; Narcisa Ana
> Maria Vasile <navasile@linux.microsoft.com>; Dmitry Malloy
> <dmitrym@microsoft.com>; Pallavi Kadam <pallavi.kadam@intel.com>;
> dev@dpdk.org; Nipun Gupta <nipun.gupta@nxp.com>; Sachin Saxena
> <sachin.saxena@nxp.com>; Hemant Agrawal <hemant.agrawal@nxp.com>;
> ferruh.yigit@intel.com; david.marchand@redhat.com
> Subject: Re: [EXT] Re: [PATCH v5 1/2] eal: add API for bus close
> 
> Caution: EXT Email
> 
> 20/01/2022 15:51, Rohit Raj:
> > Hi Thomas,
> >
> > This "rte_bus_close" API is introduced to do the opposite of what
> "rte_bus_probe" does. Just like there are plug and unplug APIs for plugging and
> unplugging a single device.
> >
> > The API you mentioned, "rte_dev_remove" supports only rte_device.  But we
> also need to close/remove private devices of dpaa and fslmc buses which are
> not exposed directly to user (eg: mempool device).
> > Note that these private devices/bus objects are not associated with a
> particular rte_device but they are available as a resource to be used by any of
> the device based on these hardware specific buses.
> > So, to close these devices, we need a new API which can do this for us. That is
> why "rte_bus_close" is required.
> 
> You mean some resources are associated to a bus but not to a device?
> It lools very weird. A resource on a bus *is* a device.
> 
> PS: please avoid top-post

FSLMC bus has hardware resources for memory pools, queues, hardware access lock(called portal). 
These are common resources, which can be associated with any device. So they don't belong to a specific device. 
Eg: mempool resource can be used by both eth and crypto device. So, we cannot close mempool while closing just one of the device(It can happen in multiprocess applications). So, these resources should be specifically closed/freed with the bus instead with a device.

There is no need to expose these devices to users and their usages is limited to other devices on the bus. There is no reason to create yet another type of device in DPDK to expose these internal only resources.


> > From: Thomas Monjalon <thomas@monjalon.net>
> > > 10/01/2022 06:26, rohit.raj@nxp.com:
> > > > From: Rohit Raj <rohit.raj@nxp.com>
> > > >
> > > > As per the current code we have API for bus probe, but the bus
> > > > close API is missing. This breaks the multi process scenarios as
> > > > objects are not cleaned while terminating the secondary processes.
> > > >
> > > > This patch adds a new API rte_bus_close() for cleanup of bus
> > > > objects which were acquired during probe.
> > >
> > > I don't understand how closing all devices of a bus will help better
> > > than just closing all devices.
> > >
> > > As Ferruh already suggested in the past, we could force closing all
> > > devices in rte_eal_cleanup().
> > > And we already have the function rte_dev_remove().
> 
> 

Re: [EXT] Re: [PATCH v5 1/2] eal: add API for bus close

[Thomas Monjalon]

01/02/2022 06:40, Rohit Raj:
> From: Thomas Monjalon <thomas@monjalon.net>
> > 20/01/2022 15:51, Rohit Raj:
> > > Hi Thomas,
> > >
> > > This "rte_bus_close" API is introduced to do the opposite of what
> > "rte_bus_probe" does. Just like there are plug and unplug APIs for plugging and
> > unplugging a single device.
> > >
> > > The API you mentioned, "rte_dev_remove" supports only rte_device.  But we
> > also need to close/remove private devices of dpaa and fslmc buses which are
> > not exposed directly to user (eg: mempool device).
> > > Note that these private devices/bus objects are not associated with a
> > particular rte_device but they are available as a resource to be used by any of
> > the device based on these hardware specific buses.
> > > So, to close these devices, we need a new API which can do this for us. That is
> > why "rte_bus_close" is required.
> > 
> > You mean some resources are associated to a bus but not to a device?
> > It lools very weird. A resource on a bus *is* a device.
> > 
> > PS: please avoid top-post
> 
> FSLMC bus has hardware resources for memory pools, queues, hardware access lock(called portal). 
> These are common resources, which can be associated with any device. So they don't belong to a specific device. 
> Eg: mempool resource can be used by both eth and crypto device. So, we cannot close mempool while closing just one of the device(It can happen in multiprocess applications). So, these resources should be specifically closed/freed with the bus instead with a device.
> 
> There is no need to expose these devices to users and their usages is limited to other devices on the bus. There is no reason to create yet another type of device in DPDK to expose these internal only resources.


OK I understand better now, thanks.

 
> > > From: Thomas Monjalon <thomas@monjalon.net>
> > > > 10/01/2022 06:26, rohit.raj@nxp.com:
> > > > > From: Rohit Raj <rohit.raj@nxp.com>
> > > > >
> > > > > As per the current code we have API for bus probe, but the bus
> > > > > close API is missing. This breaks the multi process scenarios as
> > > > > objects are not cleaned while terminating the secondary processes.
> > > > >
> > > > > This patch adds a new API rte_bus_close() for cleanup of bus
> > > > > objects which were acquired during probe.
> > > >
> > > > I don't understand how closing all devices of a bus will help better
> > > > than just closing all devices.
> > > >
> > > > As Ferruh already suggested in the past, we could force closing all
> > > > devices in rte_eal_cleanup().
> > > > And we already have the function rte_dev_remove().

Re: [PATCH v5 1/2] eal: add API for bus close

[David Marchand]

On Mon, Jan 10, 2022 at 6:26 AM <rohit.raj@nxp.com> wrote:
>
> From: Rohit Raj <rohit.raj@nxp.com>
>
> As per the current code we have API for bus probe, but the
> bus close API is missing. This breaks the multi process
> scenarios as objects are not cleaned while terminating the
> secondary processes.

After an application crash, how does this bus resets the associated resources?


>
> This patch adds a new API rte_bus_close() for cleanup of
> bus objects which were acquired during probe.

The patch in its current form breaks the ABI on rte_bus object.
This can be seen in GHA, or calling DPDK_ABI_REF_VERSION=v21.11
./devtools/test-meson-builds.sh.


[snip]

> diff --git a/doc/guides/rel_notes/release_22_03.rst b/doc/guides/rel_notes/release_22_03.rst
> index 6d99d1eaa9..7417606a2a 100644
> --- a/doc/guides/rel_notes/release_22_03.rst
> +++ b/doc/guides/rel_notes/release_22_03.rst
> @@ -55,6 +55,11 @@ New Features
>       Also, make sure to start the actual text at the margin.
>       =======================================================
>
> +    * **Added support to close bus.**
> +
> +      Added capability to allow a user to do cleanup of bus objects which
> +      were acquired during bus probe.
> +

Wrongly indented.


>
>  Removed Items
>  -------------
> @@ -84,6 +89,9 @@ API Changes
>     Also, make sure to start the actual text at the margin.
>     =======================================================
>
> +   * eal: Added new API ``rte_bus_close`` to perform cleanup bus objects which
> +     were acquired during bus probe.
> +
>
>  ABI Changes
>  -----------
> diff --git a/lib/eal/common/eal_common_bus.c b/lib/eal/common/eal_common_bus.c
> index baa5b532af..2c3c0a90d2 100644
> --- a/lib/eal/common/eal_common_bus.c
> +++ b/lib/eal/common/eal_common_bus.c
> @@ -1,5 +1,5 @@
>  /* SPDX-License-Identifier: BSD-3-Clause
> - * Copyright 2016 NXP
> + * Copyright 2016,2022 NXP
>   */
>
>  #include <stdio.h>
> @@ -85,6 +85,37 @@ rte_bus_probe(void)
>         return 0;
>  }
>
> +/* Close all devices of all buses */
> +int
> +rte_bus_close(void)
> +{
> +       int ret;
> +       struct rte_bus *bus, *vbus = NULL;
> +
> +       TAILQ_FOREACH(bus, &rte_bus_list, next) {
> +               if (!strcmp(bus->name, "vdev")) {
> +                       vbus = bus;
> +                       continue;
> +               }
> +
> +               if (bus->close) {
> +                       ret = bus->close();
> +                       if (ret)
> +                               RTE_LOG(ERR, EAL, "Bus (%s) close failed.\n",
> +                                       bus->name);
> +               }
> +       }
> +
> +       if (vbus && vbus->close) {
> +               ret = vbus->close();
> +               if (ret)
> +                       RTE_LOG(ERR, EAL, "Bus (%s) close failed.\n",
> +                               vbus->name);
> +       }

The vdev bus is special in that some drivers can reference objects
from other buses (see f4ce209a8ce5 ("eal: postpone vdev
initialization") and da76cc02342b ("eal: probe new virtual bus after
other bus devices")).
For this reason, I would expect that the vdev bus is closed before the
other buses.


> +
> +       return 0;
> +}
> +
>  /* Dump information of a single bus */
>  static int
>  bus_dump_one(FILE *f, struct rte_bus *bus)
> diff --git a/lib/eal/freebsd/eal.c b/lib/eal/freebsd/eal.c
> index a1cd2462db..87d70c6898 100644
> --- a/lib/eal/freebsd/eal.c
> +++ b/lib/eal/freebsd/eal.c
> @@ -984,6 +984,7 @@ rte_eal_cleanup(void)
>  {
>         struct internal_config *internal_conf =
>                 eal_get_internal_configuration();
> +       rte_bus_close();
>         rte_service_finalize();
>         rte_mp_channel_cleanup();
>         /* after this point, any DPDK pointers will become dangling */
> diff --git a/lib/eal/include/rte_bus.h b/lib/eal/include/rte_bus.h
> index bbbb6efd28..c6211bbd95 100644
> --- a/lib/eal/include/rte_bus.h
> +++ b/lib/eal/include/rte_bus.h
> @@ -1,5 +1,5 @@
>  /* SPDX-License-Identifier: BSD-3-Clause
> - * Copyright 2016 NXP
> + * Copyright 2016,2022 NXP
>   */
>
>  #ifndef _RTE_BUS_H_
> @@ -66,6 +66,23 @@ typedef int (*rte_bus_scan_t)(void);
>   */
>  typedef int (*rte_bus_probe_t)(void);
>
> +/**
> + * @warning
> + * @b EXPERIMENTAL: this API may change without prior notice
> + *
> + * Implementation specific close function which is responsible for resetting all
> + * detected devices on the bus to a default state, closing UIO nodes or VFIO
> + * groups and also freeing any memory allocated during rte_bus_probe like
> + * private resources for device list.
> + *
> + * This is called while iterating over each registered bus.
> + *
> + * @return
> + *     0 for successful close
> + *     !0 for any error while closing
> + */
> +typedef int (*rte_bus_close_t)(void);
> +
>  /**
>   * Device iterator to find a device on a bus.
>   *
> @@ -263,6 +280,7 @@ struct rte_bus {
>         const char *name;            /**< Name of the bus */
>         rte_bus_scan_t scan;         /**< Scan for devices attached to bus */
>         rte_bus_probe_t probe;       /**< Probe devices on bus */
> +       rte_bus_close_t close;       /**< Close devices on bus */
>         rte_bus_find_device_t find_device; /**< Find a device on the bus */
>         rte_bus_plug_t plug;         /**< Probe single device for drivers */
>         rte_bus_unplug_t unplug;     /**< Remove single device from driver */
> @@ -317,6 +335,16 @@ int rte_bus_scan(void);
>   */
>  int rte_bus_probe(void);
>
> +/**
> + * For each device on the buses, call the device specific close.
> + *
> + * @return
> + *      0 for successful close
> + *     !0 otherwise
> + */
> +__rte_experimental
> +int rte_bus_close(void);
> +
>  /**
>   * Dump information of all the buses registered with EAL.
>   *
> diff --git a/lib/eal/linux/eal.c b/lib/eal/linux/eal.c
> index 60b4924838..5c60131e46 100644
> --- a/lib/eal/linux/eal.c
> +++ b/lib/eal/linux/eal.c
> @@ -1362,6 +1362,14 @@ rte_eal_cleanup(void)
>
>         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
>                 rte_memseg_walk(mark_freeable, NULL);
> +
> +       /* Close all the buses and devices/drivers on them */
> +       if (rte_bus_close()) {
> +               rte_eal_init_alert("Cannot close devices");

You can't call rte_eal_*init*_alert in rte_eal_cleanup.

There is not much to do if the bus close fails, I'd rather leave the
cleanup continue.


> +               rte_errno = ENOTSUP;
> +               return -1;
> +       }
> +
>         rte_service_finalize();
>         rte_mp_channel_cleanup();
>         /* after this point, any DPDK pointers will become dangling */
> diff --git a/lib/eal/version.map b/lib/eal/version.map
> index ab28c22791..39882dbbd5 100644
> --- a/lib/eal/version.map
> +++ b/lib/eal/version.map
> @@ -420,6 +420,9 @@ EXPERIMENTAL {
>         rte_intr_instance_free;
>         rte_intr_type_get;
>         rte_intr_type_set;
> +
> +       # added in 22.03
> +       rte_bus_close;
>  };
>
>  INTERNAL {
> diff --git a/lib/eal/windows/eal.c b/lib/eal/windows/eal.c
> index 67db7f099a..5915ab6291 100644
> --- a/lib/eal/windows/eal.c
> +++ b/lib/eal/windows/eal.c
> @@ -260,6 +260,7 @@ rte_eal_cleanup(void)
>         struct internal_config *internal_conf =
>                 eal_get_internal_configuration();
>
> +       rte_bus_close();
>         eal_intr_thread_cancel();
>         eal_mem_virt2iova_cleanup();
>         /* after this point, any DPDK pointers will become dangling */
> --
> 2.17.1
>


-- 
David Marchand

Re: [PATCH v5 1/2] eal: add API for bus close

[Thomas Monjalon]

09/02/2022 12:04, David Marchand:
> On Mon, Jan 10, 2022 at 6:26 AM <rohit.raj@nxp.com> wrote:
> >
> > From: Rohit Raj <rohit.raj@nxp.com>
> >
> > As per the current code we have API for bus probe, but the
> > bus close API is missing. This breaks the multi process
> > scenarios as objects are not cleaned while terminating the
> > secondary processes.
> 
> After an application crash, how does this bus resets the associated resources?
> 
> > This patch adds a new API rte_bus_close() for cleanup of
> > bus objects which were acquired during probe.
> 
> The patch in its current form breaks the ABI on rte_bus object.
> This can be seen in GHA, or calling DPDK_ABI_REF_VERSION=v21.11
> ./devtools/test-meson-builds.sh.

[...]
> > +/* Close all devices of all buses */
> > +int
> > +rte_bus_close(void)
> > +{
> > +       int ret;
> > +       struct rte_bus *bus, *vbus = NULL;
> > +
> > +       TAILQ_FOREACH(bus, &rte_bus_list, next) {
> > +               if (!strcmp(bus->name, "vdev")) {

Please do an explicit comparison "== 0".

> > +                       vbus = bus;
> > +                       continue;
> > +               }
> > +
> > +               if (bus->close) {

Please do an explicit comparison with "!= NULL".
We can also completely remove this check and implement the callback
in all buses. It should loop in all remaining devices and remove them.

> > +                       ret = bus->close();
> > +                       if (ret)
> > +                               RTE_LOG(ERR, EAL, "Bus (%s) close failed.\n",
> > +                                       bus->name);
> > +               }
> > +       }
> > +
> > +       if (vbus && vbus->close) {
> > +               ret = vbus->close();
> > +               if (ret)
> > +                       RTE_LOG(ERR, EAL, "Bus (%s) close failed.\n",
> > +                               vbus->name);
> > +       }
> 
> The vdev bus is special in that some drivers can reference objects
> from other buses (see f4ce209a8ce5 ("eal: postpone vdev
> initialization") and da76cc02342b ("eal: probe new virtual bus after
> other bus devices")).
> For this reason, I would expect that the vdev bus is closed before the
> other buses.

Yes, good catch.

We don't have to expose this function as API.
This can be an internal function called only in rte_eal_cleanup().

Instead, it would be more useful to have a public function
to close a single bus by its name.

> > @@ -263,6 +280,7 @@ struct rte_bus {
> >         const char *name;            /**< Name of the bus */
> >         rte_bus_scan_t scan;         /**< Scan for devices attached to bus */
> >         rte_bus_probe_t probe;       /**< Probe devices on bus */
> > +       rte_bus_close_t close;       /**< Close devices on bus */

As David said, it is breaking the ABI.

[...]
> > @@ -1362,6 +1362,14 @@ rte_eal_cleanup(void)
> >
> >         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
> >                 rte_memseg_walk(mark_freeable, NULL);
> > +
> > +       /* Close all the buses and devices/drivers on them */
> > +       if (rte_bus_close()) {
> > +               rte_eal_init_alert("Cannot close devices");
> 
> You can't call rte_eal_*init*_alert in rte_eal_cleanup.
> 
> There is not much to do if the bus close fails, I'd rather leave the
> cleanup continue.

+1, just log and save the error code for return at the end.

Re: [dpdk-dev] [PATCH v4 2/5] examples/multi_process: cleanup bus objects while terminating app

[Stephen Hemminger]

On Thu,  8 Oct 2020 21:00:44 +0530
rohit.raj@nxp.com wrote:

> +static void
> +signal_handler(int signal)
> +{
> +	if (signal == SIGINT)
> +		rte_eal_cleanup();

NAK
Call rte_eal_cleanup in signal handler is not safe.

Need to set a flag and handle it in main code.

Re: [dpdk-dev] [PATCH v2 1/3] eal: add API for bus close

[Stephen Hemminger]

On Mon, 24 Aug 2020 13:54:12 +0530
rohit.raj@nxp.com wrote:

> From: Rohit Raj <rohit.raj@nxp.com>
> 
> As per the current code we have API for bus probe, but the
> bus close API is missing. This breaks the multi process
> scenarios as objects are not cleaned while terminating the
> secondary processes.
> 
> This patch adds a new API rte_bus_close() for cleanup of
> bus objects which were acquired during probe.
> 
> Signed-off-by: Rohit Raj <rohit.raj@nxp.com>

This patchset got some good feedback but stalled at version 5.
Marking it as "Changes requested".
Please rebase and resubmit