DPDK patches and discussions
 help / color / mirror / Atom feed
From: Kevin Laatz <kevin.laatz@intel.com>
To: dev@dpdk.org
Cc: Kevin Laatz <kevin.laatz@intel.com>
Subject: [RFC] eal: add bus cleanup to eal cleanup
Date: Tue, 19 Apr 2022 17:14:38 +0100	[thread overview]
Message-ID: <20220419161438.1837860-1-kevin.laatz@intel.com> (raw)

During EAL init, all buses are probed and the devices found are
initialized. On eal_cleanup(), the inverse does not happen, meaning any
allocated memory and other configuration will not be cleaned up
appropriately on exit.

Currently, in order for device cleanup to take place, applications must
call the driver-relevant functions to ensure proper cleanup is done before
the application exits. Since initialization occurs for all devices on the
bus, not just the devices used by an application, it requires a)
application awareness of all bus devices that could have been probed on the
system, and b) code duplication across applications to ensure cleanup is
performed. An example of this is rte_eth_dev_close() which is commonly used
across the example applications.

This RFC proposes adding bus cleanup to the eal_cleanup() to make EAL's
init/exit more symmetrical, ensuring all bus devices are cleaned up
appropriately without the application needing to be aware of all bus types
that may have been probed during initialization.

Contained in this RFC are the changes required to perform cleanup for
devices on the PCI bus during eal_cleanup(). This can be expanded in
subsequent versions if these changes are desired. There would be an ask for
bus maintainers to add the relevant cleanup for their buses since they have
the domain expertise.

Signed-off-by: Kevin Laatz <kevin.laatz@intel.com>
---
 drivers/bus/pci/pci_common.c    | 29 +++++++++++++++++++++++++++++
 lib/eal/common/eal_common_bus.c | 18 ++++++++++++++++++
 lib/eal/include/rte_bus.h       | 23 +++++++++++++++++++++++
 lib/eal/linux/eal.c             |  1 +
 4 files changed, 71 insertions(+)

diff --git a/drivers/bus/pci/pci_common.c b/drivers/bus/pci/pci_common.c
index 37ab879779..ee6cce8fc6 100644
--- a/drivers/bus/pci/pci_common.c
+++ b/drivers/bus/pci/pci_common.c
@@ -394,6 +394,34 @@ pci_probe(void)
 	return (probed && probed == failed) ? -1 : 0;
 }
 
+static int
+pci_cleanup(void)
+{
+	struct rte_pci_device *dev = NULL;
+	int ret = 0;
+
+	FOREACH_DEVICE_ON_PCIBUS(dev) {
+		struct rte_pci_addr *loc = &dev->addr;
+		struct rte_pci_driver *drv = dev->driver;
+
+		RTE_LOG(INFO, EAL,
+				"Clean up PCI driver: %s (%x:%x) device: "PCI_PRI_FMT" (socket %i)\n",
+				drv->driver.name, dev->id.vendor_id, dev->id.device_id,
+				loc->domain, loc->bus, loc->devid, loc->function,
+				dev->device.numa_node);
+
+		ret = drv->remove(dev);
+		if (ret < 0) {
+			RTE_LOG(ERR, EAL, "Cleanup for device "PCI_PRI_FMT" failed\n",
+					dev->addr.domain, dev->addr.bus, dev->addr.devid,
+					dev->addr.function);
+			rte_errno = errno;
+		}
+	}
+
+	return ret;
+}
+
 /* dump one device */
 static int
 pci_dump_one_device(FILE *f, struct rte_pci_device *dev)
@@ -813,6 +841,7 @@ struct rte_pci_bus rte_pci_bus = {
 	.bus = {
 		.scan = rte_pci_scan,
 		.probe = pci_probe,
+		.cleanup = pci_cleanup,
 		.find_device = pci_find_device,
 		.plug = pci_plug,
 		.unplug = pci_unplug,
diff --git a/lib/eal/common/eal_common_bus.c b/lib/eal/common/eal_common_bus.c
index baa5b532af..046a06a2bf 100644
--- a/lib/eal/common/eal_common_bus.c
+++ b/lib/eal/common/eal_common_bus.c
@@ -85,6 +85,24 @@ rte_bus_probe(void)
 	return 0;
 }
 
+/* Clean up all devices of all buses */
+int
+rte_bus_cleanup(void)
+{
+	int ret;
+	struct rte_bus *bus;
+
+	TAILQ_FOREACH(bus, &rte_bus_list, next) {
+		if (bus->cleanup == NULL)
+			continue;
+		ret = bus->cleanup();
+		if (ret)
+			RTE_LOG(ERR, EAL, "Bus (%s) cleanup failed.\n", bus->name);
+	}
+
+	return 0;
+}
+
 /* Dump information of a single bus */
 static int
 bus_dump_one(FILE *f, struct rte_bus *bus)
diff --git a/lib/eal/include/rte_bus.h b/lib/eal/include/rte_bus.h
index bbbb6efd28..7dbc398408 100644
--- a/lib/eal/include/rte_bus.h
+++ b/lib/eal/include/rte_bus.h
@@ -66,6 +66,18 @@ typedef int (*rte_bus_scan_t)(void);
  */
 typedef int (*rte_bus_probe_t)(void);
 
+/**
+ * Implementation specific cleanup function which is responsible for cleaning up
+ * devices on that bus with applicable drivers.
+ *
+ * This is called while iterating over each registered bus.
+ *
+ * @return
+ * 0 for successful cleanup
+ * !0 for any error during cleanup
+ */
+typedef int (*rte_bus_cleanup_t)(void);
+
 /**
  * Device iterator to find a device on a bus.
  *
@@ -263,6 +275,7 @@ struct rte_bus {
 	const char *name;            /**< Name of the bus */
 	rte_bus_scan_t scan;         /**< Scan for devices attached to bus */
 	rte_bus_probe_t probe;       /**< Probe devices on bus */
+	rte_bus_cleanup_t cleanup;   /**< Cleanup devices on bus */
 	rte_bus_find_device_t find_device; /**< Find a device on the bus */
 	rte_bus_plug_t plug;         /**< Probe single device for drivers */
 	rte_bus_unplug_t unplug;     /**< Remove single device from driver */
@@ -317,6 +330,16 @@ int rte_bus_scan(void);
  */
 int rte_bus_probe(void);
 
+/**
+ * For each device on the buses, perform a driver 'match' and call the
+ * driver-specific function for device cleanup.
+ *
+ * @return
+ * 0 for successful match/cleanup
+ * !0 otherwise
+ */
+int rte_bus_cleanup(void);
+
 /**
  * Dump information of all the buses registered with EAL.
  *
diff --git a/lib/eal/linux/eal.c b/lib/eal/linux/eal.c
index 025e5cc10d..37983b98c0 100644
--- a/lib/eal/linux/eal.c
+++ b/lib/eal/linux/eal.c
@@ -1268,6 +1268,7 @@ rte_eal_cleanup(void)
 	vfio_mp_sync_cleanup();
 #endif
 	rte_mp_channel_cleanup();
+	rte_bus_cleanup();
 	/* after this point, any DPDK pointers will become dangling */
 	rte_eal_memory_detach();
 	eal_mp_dev_hotplug_cleanup();
-- 
2.31.1


             reply	other threads:[~2022-04-19 16:14 UTC|newest]

Thread overview: 36+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2022-04-19 16:14 Kevin Laatz [this message]
2022-04-19 16:36 ` Stephen Hemminger
2022-04-20  6:55 ` Morten Brørup
2022-04-22  9:18   ` Kevin Laatz
2022-04-22 12:14     ` Morten Brørup
2022-04-22 16:27 ` [RFC v2] " Kevin Laatz
2022-05-24  9:08 ` [PATCH v3] " Kevin Laatz
2022-05-24  9:25 ` [PATCH v4] " Kevin Laatz
2022-05-24  9:38   ` Bruce Richardson
2022-05-24 15:19     ` Kevin Laatz
2022-05-24 14:48   ` Stephen Hemminger
2022-05-24 15:20     ` Kevin Laatz
2022-05-25 10:39 ` [PATCH v5] " Kevin Laatz
2022-05-25 11:12   ` Bruce Richardson
2022-05-26  8:36     ` Kevin Laatz
2022-06-01 17:02 ` [PATCH v6] " Kevin Laatz
2022-06-01 17:03   ` Bruce Richardson
2022-06-02  2:06   ` lihuisong (C)
2022-06-03 14:35     ` Kevin Laatz
2022-06-03 14:36 ` [PATCH v7] " Kevin Laatz
2022-06-03 15:11   ` Stephen Hemminger
2022-06-03 15:39     ` Bruce Richardson
2022-06-04  2:07   ` lihuisong (C)
2022-06-07 11:09   ` Thomas Monjalon
2022-06-07 15:12     ` David Marchand
2022-06-13 15:58       ` Bruce Richardson
2022-10-03 12:35         ` David Marchand
2022-10-03 14:39           ` Kevin Laatz
2022-10-04 13:11 ` [PATCH v8] " Kevin Laatz
2022-10-04 15:28   ` David Marchand
2022-10-04 15:36     ` Kevin Laatz
2022-10-04 16:50 ` [PATCH v9] " Kevin Laatz
2022-10-05  7:45   ` David Marchand
2022-10-05  9:41     ` Thomas Monjalon
2022-10-05 11:03       ` Kevin Laatz
2022-10-05 12:06         ` Thomas Monjalon

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20220419161438.1837860-1-kevin.laatz@intel.com \
    --to=kevin.laatz@intel.com \
    --cc=dev@dpdk.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).