DPDK patches and discussions
 help / color / Atom feed
* [dpdk-dev] [PATCH v1 1/3] eal: add API for bus close
@ 2020-07-28 16:42 rohit.raj
  2020-07-28 16:42 ` [dpdk-dev] [PATCH v1 2/3] examples/multi_process: cleanup bus objects while terminating app rohit.raj
                   ` (2 more replies)
  0 siblings, 3 replies; 7+ messages in thread
From: rohit.raj @ 2020-07-28 16:42 UTC (permalink / raw)
  To: Ray Kinsella, Neil Horman; +Cc: dev, Rohit Raj

From: Rohit Raj <rohit.raj@nxp.com>

As per the current code we have API for bus probe, but the
bus close API is missing. This breaks the multi process
scenarios as objects are not cleaned while terminating the
secondary processes.

This patch adds a new API rte_bus_close() for cleanup of
bus objects which were acquired during probe.

Signed-off-by: Rohit Raj <rohit.raj@nxp.com>
---
 lib/librte_eal/common/eal_common_bus.c | 34 +++++++++++++++++++++++++-
 lib/librte_eal/include/rte_bus.h       | 25 ++++++++++++++++++-
 lib/librte_eal/rte_eal_version.map     |  3 +++
 3 files changed, 60 insertions(+), 2 deletions(-)

diff --git a/lib/librte_eal/common/eal_common_bus.c b/lib/librte_eal/common/eal_common_bus.c
index baa5b532a..4c8c7eecc 100644
--- a/lib/librte_eal/common/eal_common_bus.c
+++ b/lib/librte_eal/common/eal_common_bus.c
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright 2016 NXP
+ * Copyright 2016,2020 NXP
  */
 
 #include <stdio.h>
@@ -56,6 +56,38 @@ rte_bus_scan(void)
 	return 0;
 }
 
+int
+rte_bus_close(void)
+{
+	int ret;
+	struct rte_bus *bus, *vbus = NULL;
+
+	TAILQ_FOREACH(bus, &rte_bus_list, next) {
+		if (!strcmp(bus->name, "vdev")) {
+			vbus = bus;
+			continue;
+		}
+
+		if (bus->close) {
+			ret = bus->close();
+			if (ret)
+				RTE_LOG(ERR, EAL, "Bus (%s) close failed.\n",
+					bus->name);
+		}
+	}
+
+	if (vbus) {
+		if (vbus->close) {
+			ret = vbus->close();
+			if (ret)
+				RTE_LOG(ERR, EAL, "Bus (%s) close failed.\n",
+					vbus->name);
+		}
+	}
+
+	return 0;
+}
+
 /* Probe all devices of all buses */
 int
 rte_bus_probe(void)
diff --git a/lib/librte_eal/include/rte_bus.h b/lib/librte_eal/include/rte_bus.h
index d3034d0ed..af4787b18 100644
--- a/lib/librte_eal/include/rte_bus.h
+++ b/lib/librte_eal/include/rte_bus.h
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright 2016 NXP
+ * Copyright 2016,2020 NXP
  */
 
 #ifndef _RTE_BUS_H_
@@ -67,6 +67,18 @@ typedef int (*rte_bus_scan_t)(void);
  */
 typedef int (*rte_bus_probe_t)(void);
 
+/**
+ * Implementation specific close function which is responsible for closing
+ * devices on that bus.
+ *
+ * This is called while iterating over each registered bus.
+ *
+ * @return
+ *	0 for successful close
+ *	!0 for any error while closing
+ */
+typedef int (*rte_bus_close_t)(void);
+
 /**
  * Device iterator to find a device on a bus.
  *
@@ -248,6 +260,7 @@ struct rte_bus {
 	const char *name;            /**< Name of the bus */
 	rte_bus_scan_t scan;         /**< Scan for devices attached to bus */
 	rte_bus_probe_t probe;       /**< Probe devices on bus */
+	rte_bus_close_t close;       /**< Close devices on bus */
 	rte_bus_find_device_t find_device; /**< Find a device on the bus */
 	rte_bus_plug_t plug;         /**< Probe single device for drivers */
 	rte_bus_unplug_t unplug;     /**< Remove single device from driver */
@@ -301,6 +314,16 @@ int rte_bus_scan(void);
  */
 int rte_bus_probe(void);
 
+/**
+ * For each device on the buses, call the device specific close.
+ *
+ * @return
+ *	 0 for successful close
+ *	!0 otherwise
+ */
+__rte_experimental
+int rte_bus_close(void);
+
 /**
  * Dump information of all the buses registered with EAL.
  *
diff --git a/lib/librte_eal/rte_eal_version.map b/lib/librte_eal/rte_eal_version.map
index bf0c17c23..6a6d993a1 100644
--- a/lib/librte_eal/rte_eal_version.map
+++ b/lib/librte_eal/rte_eal_version.map
@@ -403,6 +403,9 @@ EXPERIMENTAL {
 	rte_mp_disable;
 	rte_thread_register;
 	rte_thread_unregister;
+
+	# added in 20.11
+	rte_bus_close;
 };
 
 INTERNAL {
-- 
2.17.1


^ permalink raw reply	[flat|nested] 7+ messages in thread

* [dpdk-dev] [PATCH v1 2/3] examples/multi_process: cleanup bus objects while terminating app
  2020-07-28 16:42 [dpdk-dev] [PATCH v1 1/3] eal: add API for bus close rohit.raj
@ 2020-07-28 16:42 ` rohit.raj
  2020-08-05  9:28   ` Sachin Saxena (OSS)
  2020-07-28 16:42 ` [dpdk-dev] [PATCH v1 3/3] bus/fslmc: support bus close API rohit.raj
  2020-07-28 17:23 ` [dpdk-dev] [PATCH v1 1/3] eal: add API for bus close Stephen Hemminger
  2 siblings, 1 reply; 7+ messages in thread
From: rohit.raj @ 2020-07-28 16:42 UTC (permalink / raw)
  To: Anatoly Burakov; +Cc: dev, Rohit Raj

From: Rohit Raj <rohit.raj@nxp.com>

Certain bus objects may need to be closed and re-acquired
while terminating and rerunning the client application.
Hence a signal handler is required to catch the termination
of the App and hence closing the bus objects.

This patch adds the missing signal handler in the client
app and closes the Bus objects in both client and server
applications when the signal Handler is called.

Signed-off-by: Rohit Raj <rohit.raj@nxp.com>
---
 .../multi_process/client_server_mp/mp_client/Makefile |  1 +
 .../multi_process/client_server_mp/mp_client/client.c | 11 +++++++++++
 .../multi_process/client_server_mp/mp_server/main.c   |  4 +++-
 3 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/examples/multi_process/client_server_mp/mp_client/Makefile b/examples/multi_process/client_server_mp/mp_client/Makefile
index 7c447feba..9fdc74e61 100644
--- a/examples/multi_process/client_server_mp/mp_client/Makefile
+++ b/examples/multi_process/client_server_mp/mp_client/Makefile
@@ -14,6 +14,7 @@ APP = mp_client
 # all source are stored in SRCS-y
 SRCS-y := client.c
 
+CFLAGS += -DALLOW_EXPERIMENTAL_API
 CFLAGS += $(WERROR_FLAGS) -O3
 CFLAGS += -I$(SRCDIR)/../shared
 CFLAGS += -DALLOW_EXPERIMENTAL_API
diff --git a/examples/multi_process/client_server_mp/mp_client/client.c b/examples/multi_process/client_server_mp/mp_client/client.c
index 361d90b54..a81be8849 100644
--- a/examples/multi_process/client_server_mp/mp_client/client.c
+++ b/examples/multi_process/client_server_mp/mp_client/client.c
@@ -11,6 +11,7 @@
 #include <stdlib.h>
 #include <getopt.h>
 #include <string.h>
+#include <signal.h>
 
 #include <rte_common.h>
 #include <rte_malloc.h>
@@ -196,6 +197,14 @@ handle_packet(struct rte_mbuf *buf)
 
 }
 
+static void
+signal_handler(int signal)
+{
+	if (signal == SIGINT)
+		rte_bus_close();
+	exit(0);
+}
+
 /*
  * Application main function - loops through
  * receiving and processing packets. Never returns
@@ -217,6 +226,8 @@ main(int argc, char *argv[])
 	argc -= retval;
 	argv += retval;
 
+	signal(SIGINT, signal_handler);
+
 	if (parse_app_args(argc, argv) < 0)
 		rte_exit(EXIT_FAILURE, "Invalid command-line arguments\n");
 
diff --git a/examples/multi_process/client_server_mp/mp_server/main.c b/examples/multi_process/client_server_mp/mp_server/main.c
index 280dab867..a5ac84d85 100644
--- a/examples/multi_process/client_server_mp/mp_server/main.c
+++ b/examples/multi_process/client_server_mp/mp_server/main.c
@@ -275,11 +275,13 @@ signal_handler(int signal)
 {
 	uint16_t port_id;
 
-	if (signal == SIGINT)
+	if (signal == SIGINT) {
 		RTE_ETH_FOREACH_DEV(port_id) {
 			rte_eth_dev_stop(port_id);
 			rte_eth_dev_close(port_id);
 		}
+		rte_bus_close();
+	}
 	exit(0);
 }
 
-- 
2.17.1


^ permalink raw reply	[flat|nested] 7+ messages in thread

* [dpdk-dev] [PATCH v1 3/3] bus/fslmc: support bus close API
  2020-07-28 16:42 [dpdk-dev] [PATCH v1 1/3] eal: add API for bus close rohit.raj
  2020-07-28 16:42 ` [dpdk-dev] [PATCH v1 2/3] examples/multi_process: cleanup bus objects while terminating app rohit.raj
@ 2020-07-28 16:42 ` rohit.raj
  2020-08-05  9:57   ` Sachin Saxena (OSS)
  2020-07-28 17:23 ` [dpdk-dev] [PATCH v1 1/3] eal: add API for bus close Stephen Hemminger
  2 siblings, 1 reply; 7+ messages in thread
From: rohit.raj @ 2020-07-28 16:42 UTC (permalink / raw)
  To: Hemant Agrawal, Sachin Saxena, Ray Kinsella, Neil Horman,
	Nipun Gupta, Anatoly Burakov
  Cc: dev, Rohit Raj

From: Rohit Raj <rohit.raj@nxp.com>

This patch add support for closing the bus objects which
were acquired In the bus probe.

Some devices need to be cleaned while in both primary and
secondary process and while some need to be cleaned only in
case of primary process.

The devices are closed as per the white list used while
creating the objects in a particular process.

Signed-off-by: Rohit Raj <rohit.raj@nxp.com>
---
 drivers/bus/fslmc/fslmc_bus.c               | 15 +++-
 drivers/bus/fslmc/fslmc_vfio.c              | 89 ++++++++++++++++++++-
 drivers/bus/fslmc/fslmc_vfio.h              |  3 +-
 drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c    | 31 ++++++-
 drivers/bus/fslmc/portal/dpaa2_hw_dpci.c    | 32 +++++++-
 drivers/bus/fslmc/portal/dpaa2_hw_dpio.c    | 34 ++++++++
 drivers/bus/fslmc/rte_bus_fslmc_version.map |  1 +
 drivers/bus/fslmc/rte_fslmc.h               |  5 +-
 drivers/event/dpaa2/dpaa2_hw_dpcon.c        | 32 +++++++-
 drivers/net/dpaa2/dpaa2_mux.c               | 18 ++++-
 10 files changed, 252 insertions(+), 8 deletions(-)

diff --git a/drivers/bus/fslmc/fslmc_bus.c b/drivers/bus/fslmc/fslmc_bus.c
index beb3dd008..9d3da6be3 100644
--- a/drivers/bus/fslmc/fslmc_bus.c
+++ b/drivers/bus/fslmc/fslmc_bus.c
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
- *   Copyright 2016,2018-2019 NXP
+ *   Copyright 2016,2018-2020 NXP
  *
  */
 
@@ -365,6 +365,18 @@ rte_fslmc_match(struct rte_dpaa2_driver *dpaa2_drv,
 	return 1;
 }
 
+static int
+rte_fslmc_close(void)
+{
+	int ret = 0;
+
+	ret = fslmc_vfio_close_group();
+	if (ret)
+		DPAA2_BUS_ERR("Unable to close devices %d", ret);
+
+	return 0;
+}
+
 static int
 rte_fslmc_probe(void)
 {
@@ -637,6 +649,7 @@ struct rte_fslmc_bus rte_fslmc_bus = {
 	.bus = {
 		.scan = rte_fslmc_scan,
 		.probe = rte_fslmc_probe,
+		.close = rte_fslmc_close,
 		.parse = rte_fslmc_parse,
 		.find_device = rte_fslmc_find_device,
 		.get_iommu_class = rte_dpaa2_get_iommu_class,
diff --git a/drivers/bus/fslmc/fslmc_vfio.c b/drivers/bus/fslmc/fslmc_vfio.c
index 9134ec552..34ccc9db3 100644
--- a/drivers/bus/fslmc/fslmc_vfio.c
+++ b/drivers/bus/fslmc/fslmc_vfio.c
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
  *   Copyright (c) 2015-2016 Freescale Semiconductor, Inc. All rights reserved.
- *   Copyright 2016-2019 NXP
+ *   Copyright 2016-2020 NXP
  *
  */
 
@@ -692,6 +692,54 @@ rte_dpaa2_vfio_setup_intr(struct rte_intr_handle *intr_handle,
 	return -1;
 }
 
+static void
+fslmc_close_iodevices(struct rte_dpaa2_device *dev)
+{
+	struct rte_dpaa2_object *object = NULL;
+	struct rte_dpaa2_driver *drv;
+	int ret, probe_all;
+
+	switch (dev->dev_type) {
+	case DPAA2_IO:
+	case DPAA2_CON:
+	case DPAA2_CI:
+	case DPAA2_BPOOL:
+	case DPAA2_MUX:
+		TAILQ_FOREACH(object, &dpaa2_obj_list, next) {
+			if (dev->dev_type == object->dev_type)
+				object->close(dev->object_id);
+			else
+				continue;
+		}
+		break;
+	case DPAA2_ETH:
+	case DPAA2_CRYPTO:
+	case DPAA2_QDMA:
+		probe_all = rte_fslmc_bus.bus.conf.scan_mode !=
+			    RTE_BUS_SCAN_WHITELIST;
+		TAILQ_FOREACH(drv, &rte_fslmc_bus.driver_list, next) {
+			if (drv->drv_type != dev->dev_type)
+				continue;
+			if (rte_dev_is_probed(&dev->device))
+				continue;
+			if (probe_all ||
+			    (dev->device.devargs &&
+			     dev->device.devargs->policy ==
+			     RTE_DEV_WHITELISTED)) {
+				ret = drv->remove(dev);
+				if (ret)
+					DPAA2_BUS_ERR("Unable to remove");
+			}
+		}
+		break;
+	default:
+		break;
+	}
+
+	DPAA2_BUS_LOG(DEBUG, "Device (%s) Closed",
+		      dev->device.name);
+}
+
 /*
  * fslmc_process_iodevices for processing only IO (ETH, CRYPTO, and possibly
  * EVENT) devices.
@@ -796,6 +844,45 @@ fslmc_process_mcp(struct rte_dpaa2_device *dev)
 	return ret;
 }
 
+int
+fslmc_vfio_close_group(void)
+{
+	struct rte_dpaa2_device *dev, *dev_temp;
+
+	TAILQ_FOREACH_SAFE(dev, &rte_fslmc_bus.device_list, next, dev_temp) {
+		if (dev->device.devargs &&
+		    dev->device.devargs->policy == RTE_DEV_BLACKLISTED) {
+			DPAA2_BUS_LOG(DEBUG, "%s Blacklisted, skipping",
+				      dev->device.name);
+			TAILQ_REMOVE(&rte_fslmc_bus.device_list, dev, next);
+				continue;
+		}
+		switch (dev->dev_type) {
+		case DPAA2_ETH:
+		case DPAA2_CRYPTO:
+		case DPAA2_QDMA:
+		case DPAA2_IO:
+			fslmc_close_iodevices(dev);
+			break;
+		case DPAA2_CON:
+		case DPAA2_CI:
+		case DPAA2_BPOOL:
+		case DPAA2_MUX:
+			if (rte_eal_process_type() == RTE_PROC_SECONDARY)
+				continue;
+
+			fslmc_close_iodevices(dev);
+			break;
+		case DPAA2_DPRTC:
+		default:
+			DPAA2_BUS_DEBUG("Device cannot be closed: Not supported (%s)",
+					dev->device.name);
+		}
+	}
+
+	return 0;
+}
+
 int
 fslmc_vfio_process_group(void)
 {
diff --git a/drivers/bus/fslmc/fslmc_vfio.h b/drivers/bus/fslmc/fslmc_vfio.h
index bc7c6f62d..0a29fe739 100644
--- a/drivers/bus/fslmc/fslmc_vfio.h
+++ b/drivers/bus/fslmc/fslmc_vfio.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
  *   Copyright (c) 2015-2016 Freescale Semiconductor, Inc. All rights reserved.
- *   Copyright 2016,2019 NXP
+ *   Copyright 2016,2019-2020 NXP
  *
  */
 
@@ -55,6 +55,7 @@ int rte_dpaa2_vfio_setup_intr(struct rte_intr_handle *intr_handle,
 
 int fslmc_vfio_setup_group(void);
 int fslmc_vfio_process_group(void);
+int fslmc_vfio_close_group(void);
 char *fslmc_get_container(void);
 int fslmc_get_container_group(int *gropuid);
 int rte_fslmc_vfio_dmamap(void);
diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c
index d9619848d..9a4cdfd7a 100644
--- a/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c
+++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
  *   Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
- *   Copyright 2016 NXP
+ *   Copyright 2016,2020 NXP
  *
  */
 
@@ -33,6 +33,19 @@ TAILQ_HEAD(dpbp_dev_list, dpaa2_dpbp_dev);
 static struct dpbp_dev_list dpbp_dev_list
 	= TAILQ_HEAD_INITIALIZER(dpbp_dev_list); /*!< DPBP device list */
 
+static struct dpaa2_dpbp_dev *get_dpbp_from_id(uint32_t dpbp_id)
+{
+	struct dpaa2_dpbp_dev *dpbp_dev = NULL;
+
+	/* Get DPBP dev handle from list using index */
+	TAILQ_FOREACH(dpbp_dev, &dpbp_dev_list, next) {
+		if (dpbp_dev->dpbp_id == dpbp_id)
+			break;
+	}
+
+	return dpbp_dev;
+}
+
 static int
 dpaa2_create_dpbp_device(int vdev_fd __rte_unused,
 			 struct vfio_device_info *obj_info __rte_unused,
@@ -116,9 +129,25 @@ int dpaa2_dpbp_supported(void)
 	return 0;
 }
 
+static void
+dpaa2_close_dpbp_device(int object_id)
+{
+	struct dpaa2_dpbp_dev *dpbp_dev = NULL;
+
+	dpbp_dev = get_dpbp_from_id((uint32_t)object_id);
+
+	if (dpbp_dev) {
+		dpaa2_free_dpbp_dev(dpbp_dev);
+		dpbp_close(&dpbp_dev->dpbp, CMD_PRI_LOW, dpbp_dev->token);
+		TAILQ_REMOVE(&dpbp_dev_list, dpbp_dev, next);
+		rte_free(dpbp_dev);
+	}
+}
+
 static struct rte_dpaa2_object rte_dpaa2_dpbp_obj = {
 	.dev_type = DPAA2_BPOOL,
 	.create = dpaa2_create_dpbp_device,
+	.close = dpaa2_close_dpbp_device,
 };
 
 RTE_PMD_REGISTER_DPAA2_OBJECT(dpbp, rte_dpaa2_dpbp_obj);
diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c
index d393ce618..9902f38ce 100644
--- a/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c
+++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
- *   Copyright 2017 NXP
+ *   Copyright 2017,2020 NXP
  *
  */
 
@@ -30,6 +30,19 @@ TAILQ_HEAD(dpci_dev_list, dpaa2_dpci_dev);
 static struct dpci_dev_list dpci_dev_list
 	= TAILQ_HEAD_INITIALIZER(dpci_dev_list); /*!< DPCI device list */
 
+static struct dpaa2_dpci_dev *get_dpci_from_id(uint32_t dpci_id)
+{
+	struct dpaa2_dpci_dev *dpci_dev = NULL;
+
+	/* Get DPCI dev handle from list using index */
+	TAILQ_FOREACH(dpci_dev, &dpci_dev_list, next) {
+		if (dpci_dev->dpci_id == dpci_id)
+			break;
+	}
+
+	return dpci_dev;
+}
+
 static int
 rte_dpaa2_create_dpci_device(int vdev_fd __rte_unused,
 			     struct vfio_device_info *obj_info __rte_unused,
@@ -179,9 +192,26 @@ void rte_dpaa2_free_dpci_dev(struct dpaa2_dpci_dev *dpci)
 	}
 }
 
+
+static void
+rte_dpaa2_close_dpci_device(int object_id)
+{
+	struct dpaa2_dpci_dev *dpci_dev = NULL;
+
+	dpci_dev = get_dpci_from_id((uint32_t)object_id);
+
+	if (dpci_dev) {
+		rte_dpaa2_free_dpci_dev(dpci_dev);
+		dpci_close(&dpci_dev->dpci, CMD_PRI_LOW, dpci_dev->token);
+		TAILQ_REMOVE(&dpci_dev_list, dpci_dev, next);
+		rte_free(dpci_dev);
+	}
+}
+
 static struct rte_dpaa2_object rte_dpaa2_dpci_obj = {
 	.dev_type = DPAA2_CI,
 	.create = rte_dpaa2_create_dpci_device,
+	.close = rte_dpaa2_close_dpci_device,
 };
 
 RTE_PMD_REGISTER_DPAA2_OBJECT(dpci, rte_dpaa2_dpci_obj);
diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
index b0055b164..b7caaea31 100644
--- a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
+++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
@@ -86,6 +86,19 @@ static int dpaa2_cluster_sz = 2;
  * Cluster 4 (ID = x07) : CPU14, CPU15;
  */
 
+static struct dpaa2_dpio_dev *get_dpio_dev_from_id(int32_t dpio_id)
+{
+	struct dpaa2_dpio_dev *dpio_dev = NULL;
+
+	/* Get DPIO dev handle from list using index */
+	TAILQ_FOREACH(dpio_dev, &dpio_dev_list, next) {
+		if (dpio_dev->hw_id == dpio_id)
+			break;
+	}
+
+	return dpio_dev;
+}
+
 static int
 dpaa2_get_core_id(void)
 {
@@ -360,6 +373,26 @@ static void dpaa2_portal_finish(void *arg)
 	pthread_setspecific(dpaa2_portal_key, NULL);
 }
 
+static void
+dpaa2_close_dpio_device(int object_id)
+{
+	struct dpaa2_dpio_dev *dpio_dev = NULL;
+
+	dpio_dev = get_dpio_dev_from_id((int32_t)object_id);
+
+	if (dpio_dev) {
+		if (dpio_dev->dpio) {
+			dpio_disable(dpio_dev->dpio, CMD_PRI_LOW,
+				     dpio_dev->token);
+			dpio_close(dpio_dev->dpio, CMD_PRI_LOW,
+				   dpio_dev->token);
+			rte_free(dpio_dev->dpio);
+		}
+		TAILQ_REMOVE(&dpio_dev_list, dpio_dev, next);
+		rte_free(dpio_dev);
+	}
+}
+
 static int
 dpaa2_create_dpio_device(int vdev_fd,
 			 struct vfio_device_info *obj_info,
@@ -628,6 +661,7 @@ dpaa2_free_eq_descriptors(void)
 static struct rte_dpaa2_object rte_dpaa2_dpio_obj = {
 	.dev_type = DPAA2_IO,
 	.create = dpaa2_create_dpio_device,
+	.close = dpaa2_close_dpio_device,
 };
 
 RTE_PMD_REGISTER_DPAA2_OBJECT(dpio, rte_dpaa2_dpio_obj);
diff --git a/drivers/bus/fslmc/rte_bus_fslmc_version.map b/drivers/bus/fslmc/rte_bus_fslmc_version.map
index b169f5228..ac4ee2bf8 100644
--- a/drivers/bus/fslmc/rte_bus_fslmc_version.map
+++ b/drivers/bus/fslmc/rte_bus_fslmc_version.map
@@ -30,6 +30,7 @@ INTERNAL {
 	dpci_get_opr;
 	dpci_set_opr;
 	dpci_set_rx_queue;
+	dpcon_close;
 	dpcon_get_attributes;
 	dpcon_open;
 	dpdmai_close;
diff --git a/drivers/bus/fslmc/rte_fslmc.h b/drivers/bus/fslmc/rte_fslmc.h
index 80873fffc..a938ab8f9 100644
--- a/drivers/bus/fslmc/rte_fslmc.h
+++ b/drivers/bus/fslmc/rte_fslmc.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
- *   Copyright 2016,2019 NXP
+ *   Copyright 2016,2019-2020 NXP
  *
  */
 
@@ -79,6 +79,8 @@ typedef int (*rte_dpaa2_obj_create_t)(int vdev_fd,
 				      struct vfio_device_info *obj_info,
 				      int object_id);
 
+typedef void (*rte_dpaa2_obj_close_t)(int object_id);
+
 /**
  * A structure describing a DPAA2 object.
  */
@@ -87,6 +89,7 @@ struct rte_dpaa2_object {
 	const char *name;                   /**< Name of Object. */
 	enum rte_dpaa2_dev_type dev_type;   /**< Type of device */
 	rte_dpaa2_obj_create_t create;
+	rte_dpaa2_obj_close_t close;
 };
 
 /**
diff --git a/drivers/event/dpaa2/dpaa2_hw_dpcon.c b/drivers/event/dpaa2/dpaa2_hw_dpcon.c
index 200b71640..86844f49f 100644
--- a/drivers/event/dpaa2/dpaa2_hw_dpcon.c
+++ b/drivers/event/dpaa2/dpaa2_hw_dpcon.c
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  *
- *   Copyright 2017 NXP
+ *   Copyright 2017,2020 NXP
  *
  */
 
@@ -30,6 +30,19 @@ TAILQ_HEAD(dpcon_dev_list, dpaa2_dpcon_dev);
 static struct dpcon_dev_list dpcon_dev_list
 	= TAILQ_HEAD_INITIALIZER(dpcon_dev_list); /*!< DPCON device list */
 
+static struct dpaa2_dpcon_dev *get_dpcon_from_id(uint32_t dpcon_id)
+{
+	struct dpaa2_dpcon_dev *dpcon_dev = NULL;
+
+	/* Get DPCONC dev handle from list using index */
+	TAILQ_FOREACH(dpcon_dev, &dpcon_dev_list, next) {
+		if (dpcon_dev->dpcon_id == dpcon_id)
+			break;
+	}
+
+	return dpcon_dev;
+}
+
 static int
 rte_dpaa2_create_dpcon_device(int dev_fd __rte_unused,
 			      struct vfio_device_info *obj_info __rte_unused,
@@ -105,9 +118,26 @@ void rte_dpaa2_free_dpcon_dev(struct dpaa2_dpcon_dev *dpcon)
 	}
 }
 
+
+static void
+rte_dpaa2_close_dpcon_device(int object_id)
+{
+	struct dpaa2_dpcon_dev *dpcon_dev = NULL;
+
+	dpcon_dev = get_dpcon_from_id((uint32_t)object_id);
+
+	if (dpcon_dev) {
+		rte_dpaa2_free_dpcon_dev(dpcon_dev);
+		dpcon_close(&dpcon_dev->dpcon, CMD_PRI_LOW, dpcon_dev->token);
+		TAILQ_REMOVE(&dpcon_dev_list, dpcon_dev, next);
+		rte_free(dpcon_dev);
+	}
+}
+
 static struct rte_dpaa2_object rte_dpaa2_dpcon_obj = {
 	.dev_type = DPAA2_CON,
 	.create = rte_dpaa2_create_dpcon_device,
+	.close = rte_dpaa2_close_dpcon_device,
 };
 
 RTE_PMD_REGISTER_DPAA2_OBJECT(dpcon, rte_dpaa2_dpcon_obj);
diff --git a/drivers/net/dpaa2/dpaa2_mux.c b/drivers/net/dpaa2/dpaa2_mux.c
index f8366e839..15114dac9 100644
--- a/drivers/net/dpaa2/dpaa2_mux.c
+++ b/drivers/net/dpaa2/dpaa2_mux.c
@@ -44,7 +44,7 @@ static struct dpaa2_dpdmux_dev *get_dpdmux_from_id(uint32_t dpdmux_id)
 {
 	struct dpaa2_dpdmux_dev *dpdmux_dev = NULL;
 
-	/* Get DPBP dev handle from list using index */
+	/* Get DPDMUX dev handle from list using index */
 	TAILQ_FOREACH(dpdmux_dev, &dpdmux_dev_list, next) {
 		if (dpdmux_dev->dpdmux_id == dpdmux_id)
 			break;
@@ -261,9 +261,25 @@ dpaa2_create_dpdmux_device(int vdev_fd __rte_unused,
 	return -1;
 }
 
+static void
+dpaa2_close_dpdmux_device(int object_id)
+{
+	struct dpaa2_dpdmux_dev *dpdmux_dev;
+
+	dpdmux_dev = get_dpdmux_from_id((uint32_t)object_id);
+
+	if (dpdmux_dev) {
+		dpdmux_close(&dpdmux_dev->dpdmux, CMD_PRI_LOW,
+			     dpdmux_dev->token);
+		TAILQ_REMOVE(&dpdmux_dev_list, dpdmux_dev, next);
+		rte_free(dpdmux_dev);
+	}
+}
+
 static struct rte_dpaa2_object rte_dpaa2_dpdmux_obj = {
 	.dev_type = DPAA2_MUX,
 	.create = dpaa2_create_dpdmux_device,
+	.close = dpaa2_close_dpdmux_device,
 };
 
 RTE_PMD_REGISTER_DPAA2_OBJECT(dpdmux, rte_dpaa2_dpdmux_obj);
-- 
2.17.1


^ permalink raw reply	[flat|nested] 7+ messages in thread

* Re: [dpdk-dev] [PATCH v1 1/3] eal: add API for bus close
  2020-07-28 16:42 [dpdk-dev] [PATCH v1 1/3] eal: add API for bus close rohit.raj
  2020-07-28 16:42 ` [dpdk-dev] [PATCH v1 2/3] examples/multi_process: cleanup bus objects while terminating app rohit.raj
  2020-07-28 16:42 ` [dpdk-dev] [PATCH v1 3/3] bus/fslmc: support bus close API rohit.raj
@ 2020-07-28 17:23 ` Stephen Hemminger
  2020-07-30 15:53   ` [dpdk-dev] [EXT] " Rohit Raj
  2 siblings, 1 reply; 7+ messages in thread
From: Stephen Hemminger @ 2020-07-28 17:23 UTC (permalink / raw)
  To: rohit.raj; +Cc: Ray Kinsella, Neil Horman, dev

On Tue, 28 Jul 2020 22:12:50 +0530
rohit.raj@nxp.com wrote:

> From: Rohit Raj <rohit.raj@nxp.com>
> 
> As per the current code we have API for bus probe, but the
> bus close API is missing. This breaks the multi process
> scenarios as objects are not cleaned while terminating the
> secondary processes.
> 
> This patch adds a new API rte_bus_close() for cleanup of
> bus objects which were acquired during probe.
> 
> Signed-off-by: Rohit Raj <rohit.raj@nxp.com>
> ---
>  lib/librte_eal/common/eal_common_bus.c | 34 +++++++++++++++++++++++++-
>  lib/librte_eal/include/rte_bus.h       | 25 ++++++++++++++++++-
>  lib/librte_eal/rte_eal_version.map     |  3 +++
>  3 files changed, 60 insertions(+), 2 deletions(-)
> 
> diff --git a/lib/librte_eal/common/eal_common_bus.c b/lib/librte_eal/common/eal_common_bus.c
> index baa5b532a..4c8c7eecc 100644
> --- a/lib/librte_eal/common/eal_common_bus.c
> +++ b/lib/librte_eal/common/eal_common_bus.c
> @@ -1,5 +1,5 @@
>  /* SPDX-License-Identifier: BSD-3-Clause
> - * Copyright 2016 NXP
> + * Copyright 2016,2020 NXP
>   */
>  
>  #include <stdio.h>
> @@ -56,6 +56,38 @@ rte_bus_scan(void)
>  	return 0;
>  }
>  
> +int
> +rte_bus_close(void)
> +{
> +	int ret;
> +	struct rte_bus *bus, *vbus = NULL;
> +
> +	TAILQ_FOREACH(bus, &rte_bus_list, next) {
> +		if (!strcmp(bus->name, "vdev")) {
> +			vbus = bus;
> +			continue;
> +		}
> +
> +		if (bus->close) {
> +			ret = bus->close();
> +			if (ret)
> +				RTE_LOG(ERR, EAL, "Bus (%s) close failed.\n",
> +					bus->name);
> +		}
> +	}
> +
> +	if (vbus) {
> +		if (vbus->close) {
> +			ret = vbus->close();
> +			if (ret)
> +				RTE_LOG(ERR, EAL, "Bus (%s) close failed.\n",
> +					vbus->name);
> +		}
> +	}
> +
> +	return 0;
> +}
> +
>  /* Probe all devices of all buses */
>  int
>  rte_bus_probe(void)
> diff --git a/lib/librte_eal/include/rte_bus.h b/lib/librte_eal/include/rte_bus.h
> index d3034d0ed..af4787b18 100644
> --- a/lib/librte_eal/include/rte_bus.h
> +++ b/lib/librte_eal/include/rte_bus.h
> @@ -1,5 +1,5 @@
>  /* SPDX-License-Identifier: BSD-3-Clause
> - * Copyright 2016 NXP
> + * Copyright 2016,2020 NXP
>   */
>  
>  #ifndef _RTE_BUS_H_
> @@ -67,6 +67,18 @@ typedef int (*rte_bus_scan_t)(void);
>   */
>  typedef int (*rte_bus_probe_t)(void);
>  
> +/**
> + * Implementation specific close function which is responsible for closing
> + * devices on that bus.
> + *
> + * This is called while iterating over each registered bus.
> + *
> + * @return
> + *	0 for successful close
> + *	!0 for any error while closing
> + */
> +typedef int (*rte_bus_close_t)(void);
> +
>  /**
>   * Device iterator to find a device on a bus.
>   *
> @@ -248,6 +260,7 @@ struct rte_bus {
>  	const char *name;            /**< Name of the bus */
>  	rte_bus_scan_t scan;         /**< Scan for devices attached to bus */
>  	rte_bus_probe_t probe;       /**< Probe devices on bus */
> +	rte_bus_close_t close;       /**< Close devices on bus */
>  	rte_bus_find_device_t find_device; /**< Find a device on the bus */
>  	rte_bus_plug_t plug;         /**< Probe single device for drivers */
>  	rte_bus_unplug_t unplug;     /**< Remove single device from driver */
> @@ -301,6 +314,16 @@ int rte_bus_scan(void);
>   */
>  int rte_bus_probe(void);
>  
> +/**
> + * For each device on the buses, call the device specific close.
> + *
> + * @return
> + *	 0 for successful close
> + *	!0 otherwise
> + */
> +__rte_experimental
> +int rte_bus_close(void);
> +
>  /**
>   * Dump information of all the buses registered with EAL.
>   *
> diff --git a/lib/librte_eal/rte_eal_version.map b/lib/librte_eal/rte_eal_version.map
> index bf0c17c23..6a6d993a1 100644
> --- a/lib/librte_eal/rte_eal_version.map
> +++ b/lib/librte_eal/rte_eal_version.map
> @@ -403,6 +403,9 @@ EXPERIMENTAL {
>  	rte_mp_disable;
>  	rte_thread_register;
>  	rte_thread_unregister;
> +
> +	# added in 20.11
> +	rte_bus_close;
>  };
>  
>  INTERNAL {

This be called in the rte_eal_cleanup path?

^ permalink raw reply	[flat|nested] 7+ messages in thread

* Re: [dpdk-dev] [EXT] Re: [PATCH v1 1/3] eal: add API for bus close
  2020-07-28 17:23 ` [dpdk-dev] [PATCH v1 1/3] eal: add API for bus close Stephen Hemminger
@ 2020-07-30 15:53   ` " Rohit Raj
  0 siblings, 0 replies; 7+ messages in thread
From: Rohit Raj @ 2020-07-30 15:53 UTC (permalink / raw)
  To: Stephen Hemminger; +Cc: Ray Kinsella, Neil Horman, dev

Yes, I agree.
I will rework and send this patch series again.

Regards,
Rohit Raj
Software Engineer | NXP India Private Limited
Email: rohit.raj@nxp.com


-----Original Message-----
From: Stephen Hemminger <stephen@networkplumber.org> 
Sent: Tuesday, July 28, 2020 10:54 PM
To: Rohit Raj <rohit.raj@nxp.com>
Cc: Ray Kinsella <mdr@ashroe.eu>; Neil Horman <nhorman@tuxdriver.com>; dev@dpdk.org
Subject: [EXT] Re: [dpdk-dev] [PATCH v1 1/3] eal: add API for bus close

Caution: EXT Email

On Tue, 28 Jul 2020 22:12:50 +0530
rohit.raj@nxp.com wrote:

> From: Rohit Raj <rohit.raj@nxp.com>
>
> As per the current code we have API for bus probe, but the bus close 
> API is missing. This breaks the multi process scenarios as objects are 
> not cleaned while terminating the secondary processes.
>
> This patch adds a new API rte_bus_close() for cleanup of bus objects 
> which were acquired during probe.
>
> Signed-off-by: Rohit Raj <rohit.raj@nxp.com>
> ---
>  lib/librte_eal/common/eal_common_bus.c | 34 +++++++++++++++++++++++++-
>  lib/librte_eal/include/rte_bus.h       | 25 ++++++++++++++++++-
>  lib/librte_eal/rte_eal_version.map     |  3 +++
>  3 files changed, 60 insertions(+), 2 deletions(-)
>
> diff --git a/lib/librte_eal/common/eal_common_bus.c 
> b/lib/librte_eal/common/eal_common_bus.c
> index baa5b532a..4c8c7eecc 100644
> --- a/lib/librte_eal/common/eal_common_bus.c
> +++ b/lib/librte_eal/common/eal_common_bus.c
> @@ -1,5 +1,5 @@
>  /* SPDX-License-Identifier: BSD-3-Clause
> - * Copyright 2016 NXP
> + * Copyright 2016,2020 NXP
>   */
>
>  #include <stdio.h>
> @@ -56,6 +56,38 @@ rte_bus_scan(void)
>       return 0;
>  }
>
> +int
> +rte_bus_close(void)
> +{
> +     int ret;
> +     struct rte_bus *bus, *vbus = NULL;
> +
> +     TAILQ_FOREACH(bus, &rte_bus_list, next) {
> +             if (!strcmp(bus->name, "vdev")) {
> +                     vbus = bus;
> +                     continue;
> +             }
> +
> +             if (bus->close) {
> +                     ret = bus->close();
> +                     if (ret)
> +                             RTE_LOG(ERR, EAL, "Bus (%s) close failed.\n",
> +                                     bus->name);
> +             }
> +     }
> +
> +     if (vbus) {
> +             if (vbus->close) {
> +                     ret = vbus->close();
> +                     if (ret)
> +                             RTE_LOG(ERR, EAL, "Bus (%s) close failed.\n",
> +                                     vbus->name);
> +             }
> +     }
> +
> +     return 0;
> +}
> +
>  /* Probe all devices of all buses */
>  int
>  rte_bus_probe(void)
> diff --git a/lib/librte_eal/include/rte_bus.h 
> b/lib/librte_eal/include/rte_bus.h
> index d3034d0ed..af4787b18 100644
> --- a/lib/librte_eal/include/rte_bus.h
> +++ b/lib/librte_eal/include/rte_bus.h
> @@ -1,5 +1,5 @@
>  /* SPDX-License-Identifier: BSD-3-Clause
> - * Copyright 2016 NXP
> + * Copyright 2016,2020 NXP
>   */
>
>  #ifndef _RTE_BUS_H_
> @@ -67,6 +67,18 @@ typedef int (*rte_bus_scan_t)(void);
>   */
>  typedef int (*rte_bus_probe_t)(void);
>
> +/**
> + * Implementation specific close function which is responsible for 
> +closing
> + * devices on that bus.
> + *
> + * This is called while iterating over each registered bus.
> + *
> + * @return
> + *   0 for successful close
> + *   !0 for any error while closing
> + */
> +typedef int (*rte_bus_close_t)(void);
> +
>  /**
>   * Device iterator to find a device on a bus.
>   *
> @@ -248,6 +260,7 @@ struct rte_bus {
>       const char *name;            /**< Name of the bus */
>       rte_bus_scan_t scan;         /**< Scan for devices attached to bus */
>       rte_bus_probe_t probe;       /**< Probe devices on bus */
> +     rte_bus_close_t close;       /**< Close devices on bus */
>       rte_bus_find_device_t find_device; /**< Find a device on the bus */
>       rte_bus_plug_t plug;         /**< Probe single device for drivers */
>       rte_bus_unplug_t unplug;     /**< Remove single device from driver */
> @@ -301,6 +314,16 @@ int rte_bus_scan(void);
>   */
>  int rte_bus_probe(void);
>
> +/**
> + * For each device on the buses, call the device specific close.
> + *
> + * @return
> + *    0 for successful close
> + *   !0 otherwise
> + */
> +__rte_experimental
> +int rte_bus_close(void);
> +
>  /**
>   * Dump information of all the buses registered with EAL.
>   *
> diff --git a/lib/librte_eal/rte_eal_version.map 
> b/lib/librte_eal/rte_eal_version.map
> index bf0c17c23..6a6d993a1 100644
> --- a/lib/librte_eal/rte_eal_version.map
> +++ b/lib/librte_eal/rte_eal_version.map
> @@ -403,6 +403,9 @@ EXPERIMENTAL {
>       rte_mp_disable;
>       rte_thread_register;
>       rte_thread_unregister;
> +
> +     # added in 20.11
> +     rte_bus_close;
>  };
>
>  INTERNAL {

This be called in the rte_eal_cleanup path?

^ permalink raw reply	[flat|nested] 7+ messages in thread

* Re: [dpdk-dev] [PATCH v1 2/3] examples/multi_process: cleanup bus objects while terminating app
  2020-07-28 16:42 ` [dpdk-dev] [PATCH v1 2/3] examples/multi_process: cleanup bus objects while terminating app rohit.raj
@ 2020-08-05  9:28   ` Sachin Saxena (OSS)
  0 siblings, 0 replies; 7+ messages in thread
From: Sachin Saxena (OSS) @ 2020-08-05  9:28 UTC (permalink / raw)
  To: rohit.raj; +Cc: dev



On 28-Jul-20 10:12 PM, rohit.raj@nxp.com wrote:
> From: Rohit Raj <rohit.raj@nxp.com>
>
> Certain bus objects may need to be closed and re-acquired
> while terminating and rerunning the client application.
> Hence a signal handler is required to catch the termination
> of the App and hence closing the bus objects.
>
> This patch adds the missing signal handler in the client
> app and closes the Bus objects in both client and server
> applications when the signal Handler is called.
>
> Signed-off-by: Rohit Raj <rohit.raj@nxp.com>
> ---
>   .../multi_process/client_server_mp/mp_client/Makefile |  1 +
>   .../multi_process/client_server_mp/mp_client/client.c | 11 +++++++++++
>   .../multi_process/client_server_mp/mp_server/main.c   |  4 +++-
>   3 files changed, 15 insertions(+), 1 deletion(-)
>
> diff --git a/examples/multi_process/client_server_mp/mp_client/Makefile b/examples/multi_process/client_server_mp/mp_client/Makefile
> index 7c447feba..9fdc74e61 100644
> --- a/examples/multi_process/client_server_mp/mp_client/Makefile
> +++ b/examples/multi_process/client_server_mp/mp_client/Makefile
> @@ -14,6 +14,7 @@ APP = mp_client
>   # all source are stored in SRCS-y
>   SRCS-y := client.c
>   
> +CFLAGS += -DALLOW_EXPERIMENTAL_API
Duplicate flag as it is already there in list (see below)
>   CFLAGS += $(WERROR_FLAGS) -O3
>   CFLAGS += -I$(SRCDIR)/../shared
>   CFLAGS += -DALLOW_EXPERIMENTAL_API
> diff --git a/examples/multi_process/client_server_mp/mp_client/client.c b/examples/multi_process/client_server_mp/mp_client/client.c
> index 361d90b54..a81be8849 100644
> --- a/examples/multi_process/client_server_mp/mp_client/client.c
> +++ b/examples/multi_process/client_server_mp/mp_client/client.c
> @@ -11,6 +11,7 @@
>   #include <stdlib.h>
>   #include <getopt.h>
>   #include <string.h>
> +#include <signal.h>
>   
>   #include <rte_common.h>
>   #include <rte_malloc.h>
> @@ -196,6 +197,14 @@ handle_packet(struct rte_mbuf *buf)
>   
>   }
>   
> +static void
> +signal_handler(int signal)
> +{
> +	if (signal == SIGINT)
> +		rte_bus_close();
- Since rte_bus_probe() is wrapped under rte_eal_init(), rte_bus_close() 
should also be handled in rte_eal_cleanup ( Also indicated by Stephen).
- We should call rte_eal_cleanup routine here.
- What about Normal exit path of main(), we should also handle this 
there too.

> +	exit(0);
> +}
> +
>   /*
>    * Application main function - loops through
>    * receiving and processing packets. Never returns
> @@ -217,6 +226,8 @@ main(int argc, char *argv[])
>   	argc -= retval;
>   	argv += retval;
>   
> +	signal(SIGINT, signal_handler);
> +
>   	if (parse_app_args(argc, argv) < 0)
>   		rte_exit(EXIT_FAILURE, "Invalid command-line arguments\n");
>   
> diff --git a/examples/multi_process/client_server_mp/mp_server/main.c b/examples/multi_process/client_server_mp/mp_server/main.c
> index 280dab867..a5ac84d85 100644
> --- a/examples/multi_process/client_server_mp/mp_server/main.c
> +++ b/examples/multi_process/client_server_mp/mp_server/main.c
> @@ -275,11 +275,13 @@ signal_handler(int signal)
>   {
>   	uint16_t port_id;
>   
> -	if (signal == SIGINT)
> +	if (signal == SIGINT) {
>   		RTE_ETH_FOREACH_DEV(port_id) {
>   			rte_eth_dev_stop(port_id);
>   			rte_eth_dev_close(port_id);
>   		}
> +		rte_bus_close();
> +	}
>   	exit(0);
>   }
>   


^ permalink raw reply	[flat|nested] 7+ messages in thread

* Re: [dpdk-dev] [PATCH v1 3/3] bus/fslmc: support bus close API
  2020-07-28 16:42 ` [dpdk-dev] [PATCH v1 3/3] bus/fslmc: support bus close API rohit.raj
@ 2020-08-05  9:57   ` Sachin Saxena (OSS)
  0 siblings, 0 replies; 7+ messages in thread
From: Sachin Saxena (OSS) @ 2020-08-05  9:57 UTC (permalink / raw)
  To: rohit.raj, Hemant Agrawal; +Cc: dev

Acked-by: Sachin Saxena<sachin.saxena@oss.nxp.com>



On 28-Jul-20 10:12 PM, rohit.raj@nxp.com wrote:
> From: Rohit Raj <rohit.raj@nxp.com>
>
> This patch add support for closing the bus objects which
> were acquired In the bus probe.
>
> Some devices need to be cleaned while in both primary and
> secondary process and while some need to be cleaned only in
> case of primary process.
>
> The devices are closed as per the white list used while
> creating the objects in a particular process.
>
> Signed-off-by: Rohit Raj <rohit.raj@nxp.com>
> ---
>   drivers/bus/fslmc/fslmc_bus.c               | 15 +++-
>   drivers/bus/fslmc/fslmc_vfio.c              | 89 ++++++++++++++++++++-
>   drivers/bus/fslmc/fslmc_vfio.h              |  3 +-
>   drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c    | 31 ++++++-
>   drivers/bus/fslmc/portal/dpaa2_hw_dpci.c    | 32 +++++++-
>   drivers/bus/fslmc/portal/dpaa2_hw_dpio.c    | 34 ++++++++
>   drivers/bus/fslmc/rte_bus_fslmc_version.map |  1 +
>   drivers/bus/fslmc/rte_fslmc.h               |  5 +-
>   drivers/event/dpaa2/dpaa2_hw_dpcon.c        | 32 +++++++-
>   drivers/net/dpaa2/dpaa2_mux.c               | 18 ++++-
>   10 files changed, 252 insertions(+), 8 deletions(-)
>
> diff --git a/drivers/bus/fslmc/fslmc_bus.c b/drivers/bus/fslmc/fslmc_bus.c
> index beb3dd008..9d3da6be3 100644
> --- a/drivers/bus/fslmc/fslmc_bus.c
> +++ b/drivers/bus/fslmc/fslmc_bus.c
> @@ -1,6 +1,6 @@
>   /* SPDX-License-Identifier: BSD-3-Clause
>    *
> - *   Copyright 2016,2018-2019 NXP
> + *   Copyright 2016,2018-2020 NXP
>    *
>    */
>   
> @@ -365,6 +365,18 @@ rte_fslmc_match(struct rte_dpaa2_driver *dpaa2_drv,
>   	return 1;
>   }
>   
> +static int
> +rte_fslmc_close(void)
> +{
> +	int ret = 0;
> +
> +	ret = fslmc_vfio_close_group();
> +	if (ret)
> +		DPAA2_BUS_ERR("Unable to close devices %d", ret);
> +
> +	return 0;
> +}
> +
>   static int
>   rte_fslmc_probe(void)
>   {
> @@ -637,6 +649,7 @@ struct rte_fslmc_bus rte_fslmc_bus = {
>   	.bus = {
>   		.scan = rte_fslmc_scan,
>   		.probe = rte_fslmc_probe,
> +		.close = rte_fslmc_close,
>   		.parse = rte_fslmc_parse,
>   		.find_device = rte_fslmc_find_device,
>   		.get_iommu_class = rte_dpaa2_get_iommu_class,
> diff --git a/drivers/bus/fslmc/fslmc_vfio.c b/drivers/bus/fslmc/fslmc_vfio.c
> index 9134ec552..34ccc9db3 100644
> --- a/drivers/bus/fslmc/fslmc_vfio.c
> +++ b/drivers/bus/fslmc/fslmc_vfio.c
> @@ -1,7 +1,7 @@
>   /* SPDX-License-Identifier: BSD-3-Clause
>    *
>    *   Copyright (c) 2015-2016 Freescale Semiconductor, Inc. All rights reserved.
> - *   Copyright 2016-2019 NXP
> + *   Copyright 2016-2020 NXP
>    *
>    */
>   
> @@ -692,6 +692,54 @@ rte_dpaa2_vfio_setup_intr(struct rte_intr_handle *intr_handle,
>   	return -1;
>   }
>   
> +static void
> +fslmc_close_iodevices(struct rte_dpaa2_device *dev)
> +{
> +	struct rte_dpaa2_object *object = NULL;
> +	struct rte_dpaa2_driver *drv;
> +	int ret, probe_all;
> +
> +	switch (dev->dev_type) {
> +	case DPAA2_IO:
> +	case DPAA2_CON:
> +	case DPAA2_CI:
> +	case DPAA2_BPOOL:
> +	case DPAA2_MUX:
> +		TAILQ_FOREACH(object, &dpaa2_obj_list, next) {
> +			if (dev->dev_type == object->dev_type)
> +				object->close(dev->object_id);
> +			else
> +				continue;
> +		}
> +		break;
> +	case DPAA2_ETH:
> +	case DPAA2_CRYPTO:
> +	case DPAA2_QDMA:
> +		probe_all = rte_fslmc_bus.bus.conf.scan_mode !=
> +			    RTE_BUS_SCAN_WHITELIST;
> +		TAILQ_FOREACH(drv, &rte_fslmc_bus.driver_list, next) {
> +			if (drv->drv_type != dev->dev_type)
> +				continue;
> +			if (rte_dev_is_probed(&dev->device))
> +				continue;
> +			if (probe_all ||
> +			    (dev->device.devargs &&
> +			     dev->device.devargs->policy ==
> +			     RTE_DEV_WHITELISTED)) {
> +				ret = drv->remove(dev);
> +				if (ret)
> +					DPAA2_BUS_ERR("Unable to remove");
> +			}
> +		}
> +		break;
> +	default:
> +		break;
> +	}
> +
> +	DPAA2_BUS_LOG(DEBUG, "Device (%s) Closed",
> +		      dev->device.name);
> +}
> +
>   /*
>    * fslmc_process_iodevices for processing only IO (ETH, CRYPTO, and possibly
>    * EVENT) devices.
> @@ -796,6 +844,45 @@ fslmc_process_mcp(struct rte_dpaa2_device *dev)
>   	return ret;
>   }
>   
> +int
> +fslmc_vfio_close_group(void)
> +{
> +	struct rte_dpaa2_device *dev, *dev_temp;
> +
> +	TAILQ_FOREACH_SAFE(dev, &rte_fslmc_bus.device_list, next, dev_temp) {
> +		if (dev->device.devargs &&
> +		    dev->device.devargs->policy == RTE_DEV_BLACKLISTED) {
> +			DPAA2_BUS_LOG(DEBUG, "%s Blacklisted, skipping",
> +				      dev->device.name);
> +			TAILQ_REMOVE(&rte_fslmc_bus.device_list, dev, next);
> +				continue;
> +		}
> +		switch (dev->dev_type) {
> +		case DPAA2_ETH:
> +		case DPAA2_CRYPTO:
> +		case DPAA2_QDMA:
> +		case DPAA2_IO:
> +			fslmc_close_iodevices(dev);
> +			break;
> +		case DPAA2_CON:
> +		case DPAA2_CI:
> +		case DPAA2_BPOOL:
> +		case DPAA2_MUX:
> +			if (rte_eal_process_type() == RTE_PROC_SECONDARY)
> +				continue;
> +
> +			fslmc_close_iodevices(dev);
> +			break;
> +		case DPAA2_DPRTC:
> +		default:
> +			DPAA2_BUS_DEBUG("Device cannot be closed: Not supported (%s)",
> +					dev->device.name);
> +		}
> +	}
> +
> +	return 0;
> +}
> +
>   int
>   fslmc_vfio_process_group(void)
>   {
> diff --git a/drivers/bus/fslmc/fslmc_vfio.h b/drivers/bus/fslmc/fslmc_vfio.h
> index bc7c6f62d..0a29fe739 100644
> --- a/drivers/bus/fslmc/fslmc_vfio.h
> +++ b/drivers/bus/fslmc/fslmc_vfio.h
> @@ -1,7 +1,7 @@
>   /* SPDX-License-Identifier: BSD-3-Clause
>    *
>    *   Copyright (c) 2015-2016 Freescale Semiconductor, Inc. All rights reserved.
> - *   Copyright 2016,2019 NXP
> + *   Copyright 2016,2019-2020 NXP
>    *
>    */
>   
> @@ -55,6 +55,7 @@ int rte_dpaa2_vfio_setup_intr(struct rte_intr_handle *intr_handle,
>   
>   int fslmc_vfio_setup_group(void);
>   int fslmc_vfio_process_group(void);
> +int fslmc_vfio_close_group(void);
>   char *fslmc_get_container(void);
>   int fslmc_get_container_group(int *gropuid);
>   int rte_fslmc_vfio_dmamap(void);
> diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c
> index d9619848d..9a4cdfd7a 100644
> --- a/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c
> +++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c
> @@ -1,7 +1,7 @@
>   /* SPDX-License-Identifier: BSD-3-Clause
>    *
>    *   Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
> - *   Copyright 2016 NXP
> + *   Copyright 2016,2020 NXP
>    *
>    */
>   
> @@ -33,6 +33,19 @@ TAILQ_HEAD(dpbp_dev_list, dpaa2_dpbp_dev);
>   static struct dpbp_dev_list dpbp_dev_list
>   	= TAILQ_HEAD_INITIALIZER(dpbp_dev_list); /*!< DPBP device list */
>   
> +static struct dpaa2_dpbp_dev *get_dpbp_from_id(uint32_t dpbp_id)
> +{
> +	struct dpaa2_dpbp_dev *dpbp_dev = NULL;
> +
> +	/* Get DPBP dev handle from list using index */
> +	TAILQ_FOREACH(dpbp_dev, &dpbp_dev_list, next) {
> +		if (dpbp_dev->dpbp_id == dpbp_id)
> +			break;
> +	}
> +
> +	return dpbp_dev;
> +}
> +
>   static int
>   dpaa2_create_dpbp_device(int vdev_fd __rte_unused,
>   			 struct vfio_device_info *obj_info __rte_unused,
> @@ -116,9 +129,25 @@ int dpaa2_dpbp_supported(void)
>   	return 0;
>   }
>   
> +static void
> +dpaa2_close_dpbp_device(int object_id)
> +{
> +	struct dpaa2_dpbp_dev *dpbp_dev = NULL;
> +
> +	dpbp_dev = get_dpbp_from_id((uint32_t)object_id);
> +
> +	if (dpbp_dev) {
> +		dpaa2_free_dpbp_dev(dpbp_dev);
> +		dpbp_close(&dpbp_dev->dpbp, CMD_PRI_LOW, dpbp_dev->token);
> +		TAILQ_REMOVE(&dpbp_dev_list, dpbp_dev, next);
> +		rte_free(dpbp_dev);
> +	}
> +}
> +
>   static struct rte_dpaa2_object rte_dpaa2_dpbp_obj = {
>   	.dev_type = DPAA2_BPOOL,
>   	.create = dpaa2_create_dpbp_device,
> +	.close = dpaa2_close_dpbp_device,
>   };
>   
>   RTE_PMD_REGISTER_DPAA2_OBJECT(dpbp, rte_dpaa2_dpbp_obj);
> diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c
> index d393ce618..9902f38ce 100644
> --- a/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c
> +++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c
> @@ -1,6 +1,6 @@
>   /* SPDX-License-Identifier: BSD-3-Clause
>    *
> - *   Copyright 2017 NXP
> + *   Copyright 2017,2020 NXP
>    *
>    */
>   
> @@ -30,6 +30,19 @@ TAILQ_HEAD(dpci_dev_list, dpaa2_dpci_dev);
>   static struct dpci_dev_list dpci_dev_list
>   	= TAILQ_HEAD_INITIALIZER(dpci_dev_list); /*!< DPCI device list */
>   
> +static struct dpaa2_dpci_dev *get_dpci_from_id(uint32_t dpci_id)
> +{
> +	struct dpaa2_dpci_dev *dpci_dev = NULL;
> +
> +	/* Get DPCI dev handle from list using index */
> +	TAILQ_FOREACH(dpci_dev, &dpci_dev_list, next) {
> +		if (dpci_dev->dpci_id == dpci_id)
> +			break;
> +	}
> +
> +	return dpci_dev;
> +}
> +
>   static int
>   rte_dpaa2_create_dpci_device(int vdev_fd __rte_unused,
>   			     struct vfio_device_info *obj_info __rte_unused,
> @@ -179,9 +192,26 @@ void rte_dpaa2_free_dpci_dev(struct dpaa2_dpci_dev *dpci)
>   	}
>   }
>   
> +
> +static void
> +rte_dpaa2_close_dpci_device(int object_id)
> +{
> +	struct dpaa2_dpci_dev *dpci_dev = NULL;
> +
> +	dpci_dev = get_dpci_from_id((uint32_t)object_id);
> +
> +	if (dpci_dev) {
> +		rte_dpaa2_free_dpci_dev(dpci_dev);
> +		dpci_close(&dpci_dev->dpci, CMD_PRI_LOW, dpci_dev->token);
> +		TAILQ_REMOVE(&dpci_dev_list, dpci_dev, next);
> +		rte_free(dpci_dev);
> +	}
> +}
> +
>   static struct rte_dpaa2_object rte_dpaa2_dpci_obj = {
>   	.dev_type = DPAA2_CI,
>   	.create = rte_dpaa2_create_dpci_device,
> +	.close = rte_dpaa2_close_dpci_device,
>   };
>   
>   RTE_PMD_REGISTER_DPAA2_OBJECT(dpci, rte_dpaa2_dpci_obj);
> diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
> index b0055b164..b7caaea31 100644
> --- a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
> +++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
> @@ -86,6 +86,19 @@ static int dpaa2_cluster_sz = 2;
>    * Cluster 4 (ID = x07) : CPU14, CPU15;
>    */
>   
> +static struct dpaa2_dpio_dev *get_dpio_dev_from_id(int32_t dpio_id)
> +{
> +	struct dpaa2_dpio_dev *dpio_dev = NULL;
> +
> +	/* Get DPIO dev handle from list using index */
> +	TAILQ_FOREACH(dpio_dev, &dpio_dev_list, next) {
> +		if (dpio_dev->hw_id == dpio_id)
> +			break;
> +	}
> +
> +	return dpio_dev;
> +}
> +
>   static int
>   dpaa2_get_core_id(void)
>   {
> @@ -360,6 +373,26 @@ static void dpaa2_portal_finish(void *arg)
>   	pthread_setspecific(dpaa2_portal_key, NULL);
>   }
>   
> +static void
> +dpaa2_close_dpio_device(int object_id)
> +{
> +	struct dpaa2_dpio_dev *dpio_dev = NULL;
> +
> +	dpio_dev = get_dpio_dev_from_id((int32_t)object_id);
> +
> +	if (dpio_dev) {
> +		if (dpio_dev->dpio) {
> +			dpio_disable(dpio_dev->dpio, CMD_PRI_LOW,
> +				     dpio_dev->token);
> +			dpio_close(dpio_dev->dpio, CMD_PRI_LOW,
> +				   dpio_dev->token);
> +			rte_free(dpio_dev->dpio);
> +		}
> +		TAILQ_REMOVE(&dpio_dev_list, dpio_dev, next);
> +		rte_free(dpio_dev);
> +	}
> +}
> +
>   static int
>   dpaa2_create_dpio_device(int vdev_fd,
>   			 struct vfio_device_info *obj_info,
> @@ -628,6 +661,7 @@ dpaa2_free_eq_descriptors(void)
>   static struct rte_dpaa2_object rte_dpaa2_dpio_obj = {
>   	.dev_type = DPAA2_IO,
>   	.create = dpaa2_create_dpio_device,
> +	.close = dpaa2_close_dpio_device,
>   };
>   
>   RTE_PMD_REGISTER_DPAA2_OBJECT(dpio, rte_dpaa2_dpio_obj);
> diff --git a/drivers/bus/fslmc/rte_bus_fslmc_version.map b/drivers/bus/fslmc/rte_bus_fslmc_version.map
> index b169f5228..ac4ee2bf8 100644
> --- a/drivers/bus/fslmc/rte_bus_fslmc_version.map
> +++ b/drivers/bus/fslmc/rte_bus_fslmc_version.map
> @@ -30,6 +30,7 @@ INTERNAL {
>   	dpci_get_opr;
>   	dpci_set_opr;
>   	dpci_set_rx_queue;
> +	dpcon_close;
>   	dpcon_get_attributes;
>   	dpcon_open;
>   	dpdmai_close;
> diff --git a/drivers/bus/fslmc/rte_fslmc.h b/drivers/bus/fslmc/rte_fslmc.h
> index 80873fffc..a938ab8f9 100644
> --- a/drivers/bus/fslmc/rte_fslmc.h
> +++ b/drivers/bus/fslmc/rte_fslmc.h
> @@ -1,6 +1,6 @@
>   /* SPDX-License-Identifier: BSD-3-Clause
>    *
> - *   Copyright 2016,2019 NXP
> + *   Copyright 2016,2019-2020 NXP
>    *
>    */
>   
> @@ -79,6 +79,8 @@ typedef int (*rte_dpaa2_obj_create_t)(int vdev_fd,
>   				      struct vfio_device_info *obj_info,
>   				      int object_id);
>   
> +typedef void (*rte_dpaa2_obj_close_t)(int object_id);
> +
>   /**
>    * A structure describing a DPAA2 object.
>    */
> @@ -87,6 +89,7 @@ struct rte_dpaa2_object {
>   	const char *name;                   /**< Name of Object. */
>   	enum rte_dpaa2_dev_type dev_type;   /**< Type of device */
>   	rte_dpaa2_obj_create_t create;
> +	rte_dpaa2_obj_close_t close;
>   };
>   
>   /**
> diff --git a/drivers/event/dpaa2/dpaa2_hw_dpcon.c b/drivers/event/dpaa2/dpaa2_hw_dpcon.c
> index 200b71640..86844f49f 100644
> --- a/drivers/event/dpaa2/dpaa2_hw_dpcon.c
> +++ b/drivers/event/dpaa2/dpaa2_hw_dpcon.c
> @@ -1,6 +1,6 @@
>   /* SPDX-License-Identifier: BSD-3-Clause
>    *
> - *   Copyright 2017 NXP
> + *   Copyright 2017,2020 NXP
>    *
>    */
>   
> @@ -30,6 +30,19 @@ TAILQ_HEAD(dpcon_dev_list, dpaa2_dpcon_dev);
>   static struct dpcon_dev_list dpcon_dev_list
>   	= TAILQ_HEAD_INITIALIZER(dpcon_dev_list); /*!< DPCON device list */
>   
> +static struct dpaa2_dpcon_dev *get_dpcon_from_id(uint32_t dpcon_id)
> +{
> +	struct dpaa2_dpcon_dev *dpcon_dev = NULL;
> +
> +	/* Get DPCONC dev handle from list using index */
> +	TAILQ_FOREACH(dpcon_dev, &dpcon_dev_list, next) {
> +		if (dpcon_dev->dpcon_id == dpcon_id)
> +			break;
> +	}
> +
> +	return dpcon_dev;
> +}
> +
>   static int
>   rte_dpaa2_create_dpcon_device(int dev_fd __rte_unused,
>   			      struct vfio_device_info *obj_info __rte_unused,
> @@ -105,9 +118,26 @@ void rte_dpaa2_free_dpcon_dev(struct dpaa2_dpcon_dev *dpcon)
>   	}
>   }
>   
> +
> +static void
> +rte_dpaa2_close_dpcon_device(int object_id)
> +{
> +	struct dpaa2_dpcon_dev *dpcon_dev = NULL;
> +
> +	dpcon_dev = get_dpcon_from_id((uint32_t)object_id);
> +
> +	if (dpcon_dev) {
> +		rte_dpaa2_free_dpcon_dev(dpcon_dev);
> +		dpcon_close(&dpcon_dev->dpcon, CMD_PRI_LOW, dpcon_dev->token);
> +		TAILQ_REMOVE(&dpcon_dev_list, dpcon_dev, next);
> +		rte_free(dpcon_dev);
> +	}
> +}
> +
>   static struct rte_dpaa2_object rte_dpaa2_dpcon_obj = {
>   	.dev_type = DPAA2_CON,
>   	.create = rte_dpaa2_create_dpcon_device,
> +	.close = rte_dpaa2_close_dpcon_device,
>   };
>   
>   RTE_PMD_REGISTER_DPAA2_OBJECT(dpcon, rte_dpaa2_dpcon_obj);
> diff --git a/drivers/net/dpaa2/dpaa2_mux.c b/drivers/net/dpaa2/dpaa2_mux.c
> index f8366e839..15114dac9 100644
> --- a/drivers/net/dpaa2/dpaa2_mux.c
> +++ b/drivers/net/dpaa2/dpaa2_mux.c
> @@ -44,7 +44,7 @@ static struct dpaa2_dpdmux_dev *get_dpdmux_from_id(uint32_t dpdmux_id)
>   {
>   	struct dpaa2_dpdmux_dev *dpdmux_dev = NULL;
>   
> -	/* Get DPBP dev handle from list using index */
> +	/* Get DPDMUX dev handle from list using index */
>   	TAILQ_FOREACH(dpdmux_dev, &dpdmux_dev_list, next) {
>   		if (dpdmux_dev->dpdmux_id == dpdmux_id)
>   			break;
> @@ -261,9 +261,25 @@ dpaa2_create_dpdmux_device(int vdev_fd __rte_unused,
>   	return -1;
>   }
>   
> +static void
> +dpaa2_close_dpdmux_device(int object_id)
> +{
> +	struct dpaa2_dpdmux_dev *dpdmux_dev;
> +
> +	dpdmux_dev = get_dpdmux_from_id((uint32_t)object_id);
> +
> +	if (dpdmux_dev) {
> +		dpdmux_close(&dpdmux_dev->dpdmux, CMD_PRI_LOW,
> +			     dpdmux_dev->token);
> +		TAILQ_REMOVE(&dpdmux_dev_list, dpdmux_dev, next);
> +		rte_free(dpdmux_dev);
> +	}
> +}
> +
>   static struct rte_dpaa2_object rte_dpaa2_dpdmux_obj = {
>   	.dev_type = DPAA2_MUX,
>   	.create = dpaa2_create_dpdmux_device,
> +	.close = dpaa2_close_dpdmux_device,
>   };
>   
>   RTE_PMD_REGISTER_DPAA2_OBJECT(dpdmux, rte_dpaa2_dpdmux_obj);


^ permalink raw reply	[flat|nested] 7+ messages in thread

end of thread, back to index

Thread overview: 7+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2020-07-28 16:42 [dpdk-dev] [PATCH v1 1/3] eal: add API for bus close rohit.raj
2020-07-28 16:42 ` [dpdk-dev] [PATCH v1 2/3] examples/multi_process: cleanup bus objects while terminating app rohit.raj
2020-08-05  9:28   ` Sachin Saxena (OSS)
2020-07-28 16:42 ` [dpdk-dev] [PATCH v1 3/3] bus/fslmc: support bus close API rohit.raj
2020-08-05  9:57   ` Sachin Saxena (OSS)
2020-07-28 17:23 ` [dpdk-dev] [PATCH v1 1/3] eal: add API for bus close Stephen Hemminger
2020-07-30 15:53   ` [dpdk-dev] [EXT] " Rohit Raj

DPDK patches and discussions

Archives are clonable:
	git clone --mirror http://inbox.dpdk.org/dev/0 dev/git/0.git

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V2 dev dev/ http://inbox.dpdk.org/dev \
		dev@dpdk.org
	public-inbox-index dev


Newsgroup available over NNTP:
	nntp://inbox.dpdk.org/inbox.dpdk.dev


AGPL code for this site: git clone https://public-inbox.org/ public-inbox