[dpdk-dev] [PATCH v2 06/14] eal/soc: implement probing of drivers

[Shreyansh Jain <shreyansh.jain@nxp.com>]

Each SoC PMD registers a set of callback for scanning its own bus/infra and
matching devices to drivers when probe is called.
This patch introduces the infra for calls to SoC scan on rte_eal_soc_init()
and match on rte_eal_soc_probe().

Patch also adds test case for scan and probe.

Signed-off-by: Jan Viktorin <viktorin@rehivetech.com>
Signed-off-by: Shreyansh Jain <shreyansh.jain@nxp.com>
Signed-off-by: Hemant Agrawal <hemant.agrawal@nxp.com>
---
 app/test/test_soc.c                             | 138 ++++++++++++++++-
 lib/librte_eal/bsdapp/eal/rte_eal_version.map   |   3 +
 lib/librte_eal/common/eal_common_soc.c          | 191 ++++++++++++++++++++++++
 lib/librte_eal/common/include/rte_soc.h         |  37 +++++
 lib/librte_eal/linuxapp/eal/eal.c               |   5 +
 lib/librte_eal/linuxapp/eal/eal_soc.c           |  16 ++
 lib/librte_eal/linuxapp/eal/rte_eal_version.map |   3 +
 7 files changed, 392 insertions(+), 1 deletion(-)

diff --git a/app/test/test_soc.c b/app/test/test_soc.c
index ac03e64..4eb8146 100644
--- a/app/test/test_soc.c
+++ b/app/test/test_soc.c
@@ -87,14 +87,45 @@ static int test_compare_addr(void)
  */
 struct test_wrapper {
 	struct rte_soc_driver soc_drv;
+	struct rte_soc_device soc_dev;
 };
 
+static int empty_pmd0_devinit(struct rte_soc_driver *drv,
+			      struct rte_soc_device *dev);
+static int empty_pmd0_devuninit(struct rte_soc_device *dev);
+static void test_soc_scan_dev0_cb(void);
+static int test_soc_match_dev0_cb(struct rte_soc_driver *drv,
+				  struct rte_soc_device *dev);
+static void test_soc_scan_dev1_cb(void);
+static int test_soc_match_dev1_cb(struct rte_soc_driver *drv,
+				  struct rte_soc_device *dev);
+
+static int
+empty_pmd0_devinit(struct rte_soc_driver *drv __rte_unused,
+		   struct rte_soc_device *dev __rte_unused)
+{
+	return 0;
+}
+
+static int
+empty_pmd0_devuninit(struct rte_soc_device *dev)
+{
+	/* Release the memory associated with dev->addr.name */
+	free(dev->addr.name);
+
+	return 0;
+}
+
 struct test_wrapper empty_pmd0 = {
 	.soc_drv = {
 		.driver = {
 			.name = "empty_pmd0"
 		},
-	},
+		.devinit = empty_pmd0_devinit,
+		.devuninit = empty_pmd0_devuninit,
+		.scan_fn = test_soc_scan_dev0_cb,
+		.match_fn = test_soc_match_dev0_cb,
+	}
 };
 
 struct test_wrapper empty_pmd1 = {
@@ -102,9 +133,54 @@ struct test_wrapper empty_pmd1 = {
 		.driver = {
 			.name = "empty_pmd1"
 		},
+		.scan_fn = test_soc_scan_dev1_cb,
+		.match_fn = test_soc_match_dev1_cb,
 	},
 };
 
+static void
+test_soc_scan_dev0_cb(void)
+{
+	/* SoC's scan would scan devices on its bus and add to
+	 * soc_device_list
+	 */
+	empty_pmd0.soc_dev.addr.name = strdup("empty_pmd0_dev");
+
+	TAILQ_INSERT_TAIL(&soc_device_list, &empty_pmd0.soc_dev, next);
+}
+
+static int
+test_soc_match_dev0_cb(struct rte_soc_driver *drv __rte_unused,
+		       struct rte_soc_device *dev)
+{
+	if (!dev->addr.name || strcmp(dev->addr.name, "empty_pmd0_dev"))
+		return 0;
+
+	return 1;
+}
+
+
+static void
+test_soc_scan_dev1_cb(void)
+{
+	/* SoC's scan would scan devices on its bus and add to
+	 * soc_device_list
+	 */
+	empty_pmd0.soc_dev.addr.name = strdup("empty_pmd1_dev");
+
+	TAILQ_INSERT_TAIL(&soc_device_list, &empty_pmd1.soc_dev, next);
+}
+
+static int
+test_soc_match_dev1_cb(struct rte_soc_driver *drv __rte_unused,
+		       struct rte_soc_device *dev)
+{
+	if (!dev->addr.name || strcmp(dev->addr.name, "empty_pmd1_dev"))
+		return 0;
+
+	return 1;
+}
+
 static int
 count_registered_socdrvs(void)
 {
@@ -148,13 +224,54 @@ test_register_unregister(void)
 	return 0;
 }
 
+/* Test Probe (scan and match) functionality */
+static int
+test_soc_init_and_probe(void)
+{
+	struct rte_soc_driver *drv;
+
+	/* Registering dummy drivers */
+	rte_eal_soc_register(&empty_pmd0.soc_drv);
+	rte_eal_soc_register(&empty_pmd1.soc_drv);
+	/* Assuming that test_register_unregister is working, not verifying
+	 * that drivers are indeed registered
+	*/
+
+	/* rte_eal_soc_init is called by rte_eal_init, which in turn calls the
+	 * scan_fn of each driver.
+	 */
+	TAILQ_FOREACH(drv, &soc_driver_list, next) {
+		if (drv && drv->scan_fn)
+			drv->scan_fn();
+	}
+
+	/* rte_eal_init() would perform other inits here */
+
+	/* Probe would link the SoC devices<=>drivers */
+	rte_eal_soc_probe();
+
+	/* Unregistering dummy drivers */
+	rte_eal_soc_unregister(&empty_pmd0.soc_drv);
+	rte_eal_soc_unregister(&empty_pmd1.soc_drv);
+
+	free(empty_pmd0.soc_dev.addr.name);
+
+	printf("%s has been successful\n", __func__);
+	return 0;
+}
+
 /* save real devices and drivers until the tests finishes */
 struct soc_driver_list real_soc_driver_list =
 	TAILQ_HEAD_INITIALIZER(real_soc_driver_list);
 
+/* save real devices and drivers until the tests finishes */
+struct soc_device_list real_soc_device_list =
+	TAILQ_HEAD_INITIALIZER(real_soc_device_list);
+
 static int test_soc_setup(void)
 {
 	struct rte_soc_driver *drv;
+	struct rte_soc_device *dev;
 
 	/* no real drivers for the test */
 	while (!TAILQ_EMPTY(&soc_driver_list)) {
@@ -163,12 +280,20 @@ static int test_soc_setup(void)
 		TAILQ_INSERT_TAIL(&real_soc_driver_list, drv, next);
 	}
 
+	/* And, no real devices for the test */
+	while (!TAILQ_EMPTY(&soc_device_list)) {
+		dev = TAILQ_FIRST(&soc_device_list);
+		/* TODO: detach the device */
+		TAILQ_INSERT_TAIL(&real_soc_device_list, dev, next);
+	}
+
 	return 0;
 }
 
 static int test_soc_cleanup(void)
 {
 	struct rte_soc_driver *drv;
+	struct rte_soc_device *dev;
 
 	/* bring back real drivers after the test */
 	while (!TAILQ_EMPTY(&real_soc_driver_list)) {
@@ -177,6 +302,13 @@ static int test_soc_cleanup(void)
 		rte_eal_soc_register(drv);
 	}
 
+	/* And, bring back real devices after the test */
+	while (!TAILQ_EMPTY(&real_soc_device_list)) {
+		dev = TAILQ_FIRST(&real_soc_device_list);
+		TAILQ_REMOVE(&real_soc_device_list, dev, next);
+		/* TODO: Attach the device */
+	}
+
 	return 0;
 }
 
@@ -192,6 +324,10 @@ test_soc(void)
 	if (test_register_unregister())
 		return -1;
 
+	/* Assuming test_register_unregister has succeeded */
+	if (test_soc_init_and_probe())
+		return -1;
+
 	if (test_soc_cleanup())
 		return -1;
 
diff --git a/lib/librte_eal/bsdapp/eal/rte_eal_version.map b/lib/librte_eal/bsdapp/eal/rte_eal_version.map
index de38848..72b42d7 100644
--- a/lib/librte_eal/bsdapp/eal/rte_eal_version.map
+++ b/lib/librte_eal/bsdapp/eal/rte_eal_version.map
@@ -173,5 +173,8 @@ DPDK_16.11 {
 	rte_eal_soc_register;
 	rte_eal_soc_unregister;
 	rte_eal_soc_dump;
+	rte_eal_soc_detach;
+	rte_eal_soc_probe;
+	rte_eal_soc_probe_one;
 
 } DPDK_16.07;
diff --git a/lib/librte_eal/common/eal_common_soc.c b/lib/librte_eal/common/eal_common_soc.c
index 5dcddc5..4797dd9 100644
--- a/lib/librte_eal/common/eal_common_soc.c
+++ b/lib/librte_eal/common/eal_common_soc.c
@@ -36,6 +36,8 @@
 #include <sys/queue.h>
 
 #include <rte_log.h>
+#include <rte_common.h>
+#include <rte_soc.h>
 
 #include "eal_private.h"
 
@@ -45,6 +47,189 @@ struct soc_driver_list soc_driver_list =
 struct soc_device_list soc_device_list =
 	TAILQ_HEAD_INITIALIZER(soc_device_list);
 
+static int
+rte_eal_soc_probe_one_driver(struct rte_soc_driver *drv,
+			     struct rte_soc_device *dev)
+{
+	int ret = 1;
+
+	ret = drv->match_fn(drv, dev);
+	if (!ret) {
+		RTE_LOG(DEBUG, EAL,
+			" match function failed, skipping\n");
+		return ret;
+	}
+
+	dev->driver = drv;
+	RTE_VERIFY(drv->devinit != NULL);
+	return drv->devinit(drv, dev);
+}
+
+static int
+soc_probe_all_drivers(struct rte_soc_device *dev)
+{
+	struct rte_soc_driver *dr = NULL;
+	int rc = 0;
+
+	if (dev == NULL)
+		return -1;
+
+	TAILQ_FOREACH(dr, &soc_driver_list, next) {
+		rc = rte_eal_soc_probe_one_driver(dr, dev);
+		if (rc < 0)
+			/* negative value is an error */
+			return -1;
+		if (rc > 0)
+			/* positive value means driver doesn't support it */
+			continue;
+		return 0;
+	}
+	return 1;
+}
+
+/* If the IDs match, call the devuninit() function of the driver. */
+static int
+rte_eal_soc_detach_dev(struct rte_soc_driver *drv,
+		       struct rte_soc_device *dev)
+{
+	int ret;
+
+	if ((drv == NULL) || (dev == NULL))
+		return -EINVAL;
+
+	ret = drv->match_fn(drv, dev);
+	if (!ret) {
+		RTE_LOG(DEBUG, EAL,
+			" match function failed, skipping\n");
+		return ret;
+	}
+
+	RTE_LOG(DEBUG, EAL, "SoC device %s\n",
+		dev->addr.name);
+
+	RTE_LOG(DEBUG, EAL, "  remove driver: %s\n", drv->driver.name);
+
+	if (drv->devuninit && (drv->devuninit(dev) < 0))
+		return -1;	/* negative value is an error */
+
+	/* clear driver structure */
+	dev->driver = NULL;
+
+	return 0;
+}
+
+/*
+ * Call the devuninit() function of all registered drivers for the given
+ * device if their IDs match.
+ *
+ * @return
+ *       0 when successful
+ *      -1 if deinitialization fails
+ *       1 if no driver is found for this device.
+ */
+static int
+soc_detach_all_drivers(struct rte_soc_device *dev)
+{
+	struct rte_soc_driver *dr = NULL;
+	int rc = 0;
+
+	if (dev == NULL)
+		return -1;
+
+	TAILQ_FOREACH(dr, &soc_driver_list, next) {
+		rc = rte_eal_soc_detach_dev(dr, dev);
+		if (rc < 0)
+			/* negative value is an error */
+			return -1;
+		if (rc > 0)
+			/* positive value means driver doesn't support it */
+			continue;
+		return 0;
+	}
+	return 1;
+}
+
+/*
+ * Detach device specified by its SoC address.
+ */
+int
+rte_eal_soc_detach(const struct rte_soc_addr *addr)
+{
+	struct rte_soc_device *dev = NULL;
+	int ret = 0;
+
+	if (addr == NULL)
+		return -1;
+
+	TAILQ_FOREACH(dev, &soc_device_list, next) {
+		if (rte_eal_compare_soc_addr(&dev->addr, addr))
+			continue;
+
+		ret = soc_detach_all_drivers(dev);
+		if (ret < 0)
+			goto err_return;
+
+		TAILQ_REMOVE(&soc_device_list, dev, next);
+		return 0;
+	}
+	return -1;
+
+err_return:
+	RTE_LOG(WARNING, EAL, "Requested device %s cannot be used\n",
+		dev->addr.name);
+	return -1;
+}
+
+int
+rte_eal_soc_probe_one(const struct rte_soc_addr *addr)
+{
+	struct rte_soc_device *dev = NULL;
+	int ret = 0;
+
+	if (addr == NULL)
+		return -1;
+
+	/* unlike pci, in case of soc, it the responsibility of the soc driver
+	 * to check during init whether device has been updated since last add.
+	 */
+
+	TAILQ_FOREACH(dev, &soc_device_list, next) {
+		if (rte_eal_compare_soc_addr(&dev->addr, addr))
+			continue;
+
+		ret = soc_probe_all_drivers(dev);
+		if (ret < 0)
+			goto err_return;
+		return 0;
+	}
+	return -1;
+
+err_return:
+	RTE_LOG(WARNING, EAL,
+		"Requested device %s cannot be used\n", addr->name);
+	return -1;
+}
+
+/*
+ * Scan the SoC devices and call the devinit() function for all registered
+ * drivers that have a matching entry in its id_table for discovered devices.
+ */
+int
+rte_eal_soc_probe(void)
+{
+	struct rte_soc_device *dev = NULL;
+	int ret = 0;
+
+	TAILQ_FOREACH(dev, &soc_device_list, next) {
+		ret = soc_probe_all_drivers(dev);
+		if (ret < 0)
+			rte_exit(EXIT_FAILURE, "Requested device %s"
+				 " cannot be used\n", dev->addr.name);
+	}
+
+	return 0;
+}
+
 /* dump one device */
 static int
 soc_dump_one_device(FILE *f, struct rte_soc_device *dev)
@@ -79,6 +264,12 @@ rte_eal_soc_dump(FILE *f)
 void
 rte_eal_soc_register(struct rte_soc_driver *driver)
 {
+	/* For a valid soc driver, match and scan function
+	 * should be provided.
+	 */
+	RTE_VERIFY(driver != NULL);
+	RTE_VERIFY(driver->match_fn != NULL);
+	RTE_VERIFY(driver->scan_fn != NULL);
 	TAILQ_INSERT_TAIL(&soc_driver_list, driver, next);
 }
 
diff --git a/lib/librte_eal/common/include/rte_soc.h b/lib/librte_eal/common/include/rte_soc.h
index c6f98eb..cc17829 100644
--- a/lib/librte_eal/common/include/rte_soc.h
+++ b/lib/librte_eal/common/include/rte_soc.h
@@ -97,6 +97,16 @@ typedef int (soc_devinit_t)(struct rte_soc_driver *, struct rte_soc_device *);
 typedef int (soc_devuninit_t)(struct rte_soc_device *);
 
 /**
+ * SoC device scan callback, called from rte_eal_soc_init.
+ */
+typedef void (soc_scan_t)(void);
+
+/**
+ * Custom device<=>driver match callback for SoC
+ */
+typedef int (soc_match_t)(struct rte_soc_driver *, struct rte_soc_device *);
+
+/**
  * A structure describing a SoC driver.
  */
 struct rte_soc_driver {
@@ -104,6 +114,8 @@ struct rte_soc_driver {
 	struct rte_driver driver;          /**< Inherit core driver. */
 	soc_devinit_t *devinit;            /**< Device initialization */
 	soc_devuninit_t *devuninit;        /**< Device uninitialization */
+	soc_scan_t *scan_fn;               /**< Callback for scanning SoC bus*/
+	soc_match_t *match_fn;             /**< Callback to match dev<->drv */
 	const struct rte_soc_id *id_table; /**< ID table, NULL terminated */
 };
 
@@ -146,6 +158,31 @@ rte_eal_compare_soc_addr(const struct rte_soc_addr *a0,
 }
 
 /**
+ * Probe SoC devices for registered drivers.
+ */
+int rte_eal_soc_probe(void);
+
+/**
+ * Probe the single SoC device.
+ */
+int rte_eal_soc_probe_one(const struct rte_soc_addr *addr);
+
+/**
+ * Close the single SoC device.
+ *
+ * Scan the SoC devices and find the SoC device specified by the SoC
+ * address, then call the devuninit() function for registered driver
+ * that has a matching entry in its id_table for discovered device.
+ *
+ * @param addr
+ *	The SoC address to close.
+ * @return
+ *   - 0 on success.
+ *   - Negative on error.
+ */
+int rte_eal_soc_detach(const struct rte_soc_addr *addr);
+
+/**
  * Dump discovered SoC devices.
  */
 void rte_eal_soc_dump(FILE *f);
diff --git a/lib/librte_eal/linuxapp/eal/eal.c b/lib/librte_eal/linuxapp/eal/eal.c
index 15c8c3d..147b601 100644
--- a/lib/librte_eal/linuxapp/eal/eal.c
+++ b/lib/librte_eal/linuxapp/eal/eal.c
@@ -70,6 +70,7 @@
 #include <rte_cpuflags.h>
 #include <rte_interrupts.h>
 #include <rte_pci.h>
+#include <rte_soc.h>
 #include <rte_dev.h>
 #include <rte_devargs.h>
 #include <rte_common.h>
@@ -881,6 +882,10 @@ rte_eal_init(int argc, char **argv)
 	if (rte_eal_pci_probe())
 		rte_panic("Cannot probe PCI\n");
 
+	/* Probe & Initialize SoC devices */
+	if (rte_eal_soc_probe())
+		rte_panic("Cannot probe SoC\n");
+
 	rte_eal_mcfg_complete();
 
 	return fctret;
diff --git a/lib/librte_eal/linuxapp/eal/eal_soc.c b/lib/librte_eal/linuxapp/eal/eal_soc.c
index 04848b9..5f961c4 100644
--- a/lib/librte_eal/linuxapp/eal/eal_soc.c
+++ b/lib/librte_eal/linuxapp/eal/eal_soc.c
@@ -52,5 +52,21 @@
 int
 rte_eal_soc_init(void)
 {
+	struct rte_soc_driver *drv;
+
+	/* for debug purposes, SoC can be disabled */
+	if (internal_config.no_soc)
+		return 0;
+
+	/* For each registered driver, call their scan routine to perform any
+	 * custom scan for devices (for example, custom buses)
+	 */
+	TAILQ_FOREACH(drv, &soc_driver_list, next) {
+		if (drv && drv->scan_fn) {
+			drv->scan_fn();
+			/* Ignore all errors from this */
+		}
+	}
+
 	return 0;
 }
diff --git a/lib/librte_eal/linuxapp/eal/rte_eal_version.map b/lib/librte_eal/linuxapp/eal/rte_eal_version.map
index b9d1932..8092e62 100644
--- a/lib/librte_eal/linuxapp/eal/rte_eal_version.map
+++ b/lib/librte_eal/linuxapp/eal/rte_eal_version.map
@@ -179,5 +179,8 @@ DPDK_16.11 {
 	rte_eal_soc_register;
 	rte_eal_soc_unregister;
 	rte_eal_soc_dump;
+	rte_eal_soc_detach;
+	rte_eal_soc_probe;
+	rte_eal_soc_probe_one;
 
 } DPDK_16.07;
-- 
2.7.4