[RFC PATCH 0/7] support vfio platform PMD

[RFC PATCH 0/7] support vfio platform PMD

[Tomasz Duszynski]

This series aims to add support for managing vfio platform
devices from userspace application conveniently i.e
without going through the configuration required by vfio
and opencoding everything. Instead convenience helpers
are provided.

vfio platform devices, from the kernel standpoint, are ones
that do not have built-in discovery capabilities and are
behind an IOMMU.

This PMD is backed by both vfio-platform and vfio
kernel drivers and ideally should give access to all
vfio capabilities. As of now, access to memory maps and
DMA are supported.

PMD requires platform bus support [1].

[1] https://lore.kernel.org/dpdk-dev/20221222000106.270619-1-tduszynski@marvell.com/

Tomasz Duszynski (7):
  lib: add helper to read strings from sysfs files
  raw/vfio_platform: add driver skeleton
  raw/vfio_platform: add platform probe and remove
  raw/vfio_platform: support rawdev close
  raw/vfio_platform: support rawdev configure
  raw/vfio_platform: support rawdev device info
  raw/vfio_platform: support DMA map/unmap

 app/test/test_eal_fs.c                        | 108 ++++-
 doc/guides/rawdevs/index.rst                  |   1 +
 doc/guides/rawdevs/vfio_platform.rst          |  24 +
 drivers/raw/meson.build                       |   1 +
 drivers/raw/vfio_platform/meson.build         |  16 +
 .../raw/vfio_platform/rte_pmd_vfio_platform.h | 129 +++++
 drivers/raw/vfio_platform/version.map         |  12 +
 drivers/raw/vfio_platform/vfio_platform.c     | 449 ++++++++++++++++++
 drivers/raw/vfio_platform/vfio_platform.h     |  35 ++
 lib/eal/common/eal_filesystem.h               |   6 +
 lib/eal/unix/eal_filesystem.c                 |  24 +-
 lib/eal/version.map                           |   3 +
 12 files changed, 790 insertions(+), 18 deletions(-)
 create mode 100644 doc/guides/rawdevs/vfio_platform.rst
 create mode 100644 drivers/raw/vfio_platform/meson.build
 create mode 100644 drivers/raw/vfio_platform/rte_pmd_vfio_platform.h
 create mode 100644 drivers/raw/vfio_platform/version.map
 create mode 100644 drivers/raw/vfio_platform/vfio_platform.c
 create mode 100644 drivers/raw/vfio_platform/vfio_platform.h

--
2.25.1

[RFC PATCH 1/7] lib: add helper to read strings from sysfs files

[Tomasz Duszynski]

Reading strings from sysfs files is a re-occurring pattern
hence add helper for doing that.

Signed-off-by: Tomasz Duszynski <tduszynski@marvell.com>
---
 app/test/test_eal_fs.c          | 108 ++++++++++++++++++++++++++++----
 lib/eal/common/eal_filesystem.h |   6 ++
 lib/eal/unix/eal_filesystem.c   |  24 ++++---
 lib/eal/version.map             |   3 +
 4 files changed, 123 insertions(+), 18 deletions(-)

diff --git a/app/test/test_eal_fs.c b/app/test/test_eal_fs.c
index b3686edcb4..6c373fc7f1 100644
--- a/app/test/test_eal_fs.c
+++ b/app/test/test_eal_fs.c
@@ -20,12 +20,33 @@ test_eal_fs(void)
 
 #else
 
+static int
+temp_create(char *filename, size_t len)
+{
+	char file_template[] = "/tmp/eal_test_XXXXXX";
+	char proc_path[PATH_MAX];
+	int fd;
+
+	fd = mkstemp(file_template);
+	if (fd == -1) {
+		perror("mkstemp() failure");
+		return -1;
+	}
+
+	snprintf(proc_path, sizeof(proc_path), "/proc/self/fd/%d", fd);
+	if (readlink(proc_path, filename, len) < 0) {
+		perror("readlink() failure");
+		close(fd);
+		return -1;
+	}
+
+	return fd;
+}
+
 static int
 test_parse_sysfs_value(void)
 {
 	char filename[PATH_MAX] = "";
-	char proc_path[PATH_MAX];
-	char file_template[] = "/tmp/eal_test_XXXXXX";
 	int tmp_file_handle = -1;
 	FILE *fd = NULL;
 	unsigned valid_number;
@@ -40,16 +61,10 @@ test_parse_sysfs_value(void)
 
 	/* get a temporary filename to use for all tests - create temp file handle and then
 	 * use /proc to get the actual file that we can open */
-	tmp_file_handle = mkstemp(file_template);
-	if (tmp_file_handle == -1) {
-		perror("mkstemp() failure");
+	tmp_file_handle = temp_create(filename, sizeof(filename));
+	if (tmp_file_handle < 0)
 		goto error;
-	}
-	snprintf(proc_path, sizeof(proc_path), "/proc/self/fd/%d", tmp_file_handle);
-	if (readlink(proc_path, filename, sizeof(filename)) < 0) {
-		perror("readlink() failure");
-		goto error;
-	}
+
 	printf("Temporary file is: %s\n", filename);
 
 	/* test we get an error value if we use file before it's created */
@@ -175,11 +190,82 @@ test_parse_sysfs_value(void)
 	return -1;
 }
 
+static int
+test_parse_sysfs_string(void)
+{
+	const char *teststr = "the quick brown dog jumps over the lazy fox\n";
+	char filename[PATH_MAX] = "";
+	char buf[BUFSIZ] = { };
+	int tmp_file_handle;
+	FILE *fd = NULL;
+
+#ifdef RTE_EXEC_ENV_FREEBSD
+	/* BSD doesn't have /proc/pid/fd */
+	return 0;
+#endif
+	printf("Testing function eal_parse_sysfs_string()\n");
+
+	/* get a temporary filename to use for all tests - create temp file handle and then
+	 * use /proc to get the actual file that we can open
+	 */
+	tmp_file_handle = temp_create(filename, sizeof(filename));
+	if (tmp_file_handle < 0)
+		goto error;
+
+	printf("Temporary file is: %s\n", filename);
+
+	/* test we get an error value if we use file before it's created */
+	printf("Test reading a missing file ...\n");
+	if (eal_parse_sysfs_string("/dev/not-quite-null", buf, sizeof(buf)) == 0) {
+		printf("Error with eal_parse_sysfs_string() - returned success on reading empty file\n");
+		goto error;
+	}
+	printf("Confirmed return error when reading empty file\n");
+
+	/* test reading a string from file */
+	printf("Test reading string ...\n");
+	fd = fopen(filename, "w");
+	if (fd == NULL) {
+		printf("line %d, Error opening %s: %s\n", __LINE__, filename, strerror(errno));
+		goto error;
+	}
+	fprintf(fd, "%s", teststr);
+	fclose(fd);
+	fd = NULL;
+	if (eal_parse_sysfs_string(filename, buf, sizeof(buf) - 1) < 0) {
+		printf("eal_parse_sysfs_string() returned error - test failed\n");
+		goto error;
+	}
+	if (strcmp(teststr, buf)) {
+		printf("Invalid string read by eal_parse_sysfs_string() - test failed\n");
+		goto error;
+	}
+	/* don't print newline */
+	buf[strlen(buf) - 1] = '\0';
+	printf("Read '%s\\n' ok\n", buf);
+
+	close(tmp_file_handle);
+	unlink(filename);
+	printf("eal_parse_sysfs_string() - OK\n");
+	return 0;
+
+error:
+	if (fd)
+		fclose(fd);
+	if (tmp_file_handle > 0)
+		close(tmp_file_handle);
+	if (filename[0] != '\0')
+		unlink(filename);
+	return -1;
+}
+
 static int
 test_eal_fs(void)
 {
 	if (test_parse_sysfs_value() < 0)
 		return -1;
+	if (test_parse_sysfs_string() < 0)
+		return -1;
 	return 0;
 }
 
diff --git a/lib/eal/common/eal_filesystem.h b/lib/eal/common/eal_filesystem.h
index 5d21f07c20..79bcc5ca1f 100644
--- a/lib/eal/common/eal_filesystem.h
+++ b/lib/eal/common/eal_filesystem.h
@@ -104,4 +104,10 @@ eal_get_hugefile_path(char *buffer, size_t buflen, const char *hugedir, int f_id
  * Used to read information from files on /sys */
 int eal_parse_sysfs_value(const char *filename, unsigned long *val);
 
+/** Function to read a string from a file on the filesystem.
+ * Used to read information for files in /sys
+ */
+__rte_experimental
+int eal_parse_sysfs_string(const char *filename, char *str, size_t size);
+
 #endif /* EAL_FILESYSTEM_H */
diff --git a/lib/eal/unix/eal_filesystem.c b/lib/eal/unix/eal_filesystem.c
index afbab9368a..8ed10094be 100644
--- a/lib/eal/unix/eal_filesystem.c
+++ b/lib/eal/unix/eal_filesystem.c
@@ -76,12 +76,9 @@ int eal_create_runtime_dir(void)
 	return 0;
 }
 
-/* parse a sysfs (or other) file containing one integer value */
-int eal_parse_sysfs_value(const char *filename, unsigned long *val)
+int eal_parse_sysfs_string(const char *filename, char *str, size_t size)
 {
 	FILE *f;
-	char buf[BUFSIZ];
-	char *end = NULL;
 
 	if ((f = fopen(filename, "r")) == NULL) {
 		RTE_LOG(ERR, EAL, "%s(): cannot open sysfs value %s\n",
@@ -89,19 +86,32 @@ int eal_parse_sysfs_value(const char *filename, unsigned long *val)
 		return -1;
 	}
 
-	if (fgets(buf, sizeof(buf), f) == NULL) {
+	if (fgets(str, size, f) == NULL) {
 		RTE_LOG(ERR, EAL, "%s(): cannot read sysfs value %s\n",
 			__func__, filename);
 		fclose(f);
 		return -1;
 	}
+	fclose(f);
+	return 0;
+}
+
+/* parse a sysfs (or other) file containing one integer value */
+int eal_parse_sysfs_value(const char *filename, unsigned long *val)
+{
+	char buf[BUFSIZ];
+	char *end = NULL;
+	int ret;
+
+	ret = eal_parse_sysfs_string(filename, buf, sizeof(buf));
+	if (ret < 0)
+		return ret;
+
 	*val = strtoul(buf, &end, 0);
 	if ((buf[0] == '\0') || (end == NULL) || (*end != '\n')) {
 		RTE_LOG(ERR, EAL, "%s(): cannot parse sysfs value %s\n",
 				__func__, filename);
-		fclose(f);
 		return -1;
 	}
-	fclose(f);
 	return 0;
 }
diff --git a/lib/eal/version.map b/lib/eal/version.map
index 7ad12a7dc9..2dbc10e6be 100644
--- a/lib/eal/version.map
+++ b/lib/eal/version.map
@@ -440,6 +440,9 @@ EXPERIMENTAL {
 	rte_thread_detach;
 	rte_thread_equal;
 	rte_thread_join;
+
+	# added in 23.03
+	eal_parse_sysfs_string; # WINDOWS_NO_EXPORT
 };
 
 INTERNAL {
-- 
2.25.1

[RFC PATCH 2/7] raw/vfio_platform: add driver skeleton

[Tomasz Duszynski]

Add driver skeleton which does not do anything beyond registering driver
itself to platform bus and providing means to create and destroy VFIO
container.

Signed-off-by: Tomasz Duszynski <tduszynski@marvell.com>
---
 doc/guides/rawdevs/index.rst                  |  1 +
 doc/guides/rawdevs/vfio_platform.rst          | 24 ++++++
 drivers/raw/meson.build                       |  1 +
 drivers/raw/vfio_platform/meson.build         | 16 ++++
 .../raw/vfio_platform/rte_pmd_vfio_platform.h | 49 +++++++++++
 drivers/raw/vfio_platform/version.map         | 10 +++
 drivers/raw/vfio_platform/vfio_platform.c     | 85 +++++++++++++++++++
 7 files changed, 186 insertions(+)
 create mode 100644 doc/guides/rawdevs/vfio_platform.rst
 create mode 100644 drivers/raw/vfio_platform/meson.build
 create mode 100644 drivers/raw/vfio_platform/rte_pmd_vfio_platform.h
 create mode 100644 drivers/raw/vfio_platform/version.map
 create mode 100644 drivers/raw/vfio_platform/vfio_platform.c

diff --git a/doc/guides/rawdevs/index.rst b/doc/guides/rawdevs/index.rst
index f34315f051..b95c155b48 100644
--- a/doc/guides/rawdevs/index.rst
+++ b/doc/guides/rawdevs/index.rst
@@ -16,3 +16,4 @@ application through rawdev API.
     dpaa2_cmdif
     ifpga
     ntb
+    vfio_platform
diff --git a/doc/guides/rawdevs/vfio_platform.rst b/doc/guides/rawdevs/vfio_platform.rst
new file mode 100644
index 0000000000..97443a85fb
--- /dev/null
+++ b/doc/guides/rawdevs/vfio_platform.rst
@@ -0,0 +1,24 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright(c) 2023 Marvell.
+
+VFIO Platform driver
+====================
+
+VFIO platform driver allows to configure and manage kernel VFIO platform devices.
+These devices do not normally have built-in bus discovery capabilities, known
+for example from PCI bus, and reside behind an IOMMU.
+
+Features
+--------
+
+Following features are available:
+
+- bind to devices managed by kernel vfio-platform driver
+- expose device memory resources
+- map/unmap DMA memory
+
+Requirements
+------------
+
+Since PMD is backed by vfio and vfio-platform kernel drivers they need to be enabled
+in kernel config.
diff --git a/drivers/raw/meson.build b/drivers/raw/meson.build
index 05cad143fe..a3e1126c98 100644
--- a/drivers/raw/meson.build
+++ b/drivers/raw/meson.build
@@ -12,5 +12,6 @@ drivers = [
         'ifpga',
         'ntb',
         'skeleton',
+        'vfio_platform',
 ]
 std_deps = ['rawdev']
diff --git a/drivers/raw/vfio_platform/meson.build b/drivers/raw/vfio_platform/meson.build
new file mode 100644
index 0000000000..a03836d890
--- /dev/null
+++ b/drivers/raw/vfio_platform/meson.build
@@ -0,0 +1,16 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(C) 2023 Marvell.
+#
+
+if not is_linux
+  build = false
+  reason = 'only supported on Linux'
+endif
+
+deps += ['bus_platform', 'rawdev']
+sources = files(
+        'vfio_platform.c',
+)
+headers = files(
+        'rte_pmd_vfio_platform.h',
+)
diff --git a/drivers/raw/vfio_platform/rte_pmd_vfio_platform.h b/drivers/raw/vfio_platform/rte_pmd_vfio_platform.h
new file mode 100644
index 0000000000..307a20d9fc
--- /dev/null
+++ b/drivers/raw/vfio_platform/rte_pmd_vfio_platform.h
@@ -0,0 +1,49 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2023 Marvell.
+ */
+
+#ifndef _RTE_PMD_VFIO_PLATFORM_H_
+#define _RTE_PMD_VFIO_PLATFORM_H_
+
+/**
+ * @file
+ *
+ * VFIO platform specific structures and interface.
+ *
+ * @b EXPERIMENTAL: this API may change or be removed without prior notice
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Create a new VFIO container.
+ *
+ * @return
+ *   container on success
+ *   <0 on failure
+ */
+__rte_experimental
+int
+rte_pmd_vfio_platform_container_create(void);
+
+/**
+ * Destroy previously created container.
+ *
+ * @param container
+ *   Container to destroy.
+ *
+ * @return
+ *   0 on success
+ *   <0 on failure
+ */
+__rte_experimental
+int
+rte_pmd_vfio_platform_container_destroy(int container);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_PMD_VFIO_PLATFORM_H_ */
diff --git a/drivers/raw/vfio_platform/version.map b/drivers/raw/vfio_platform/version.map
new file mode 100644
index 0000000000..2aea50f4c1
--- /dev/null
+++ b/drivers/raw/vfio_platform/version.map
@@ -0,0 +1,10 @@
+DPDK_23 {
+	local: *;
+};
+
+EXPERIMENTAL {
+	global:
+
+	rte_pmd_vfio_platform_container_create;
+	rte_pmd_vfio_platform_container_destroy;
+};
diff --git a/drivers/raw/vfio_platform/vfio_platform.c b/drivers/raw/vfio_platform/vfio_platform.c
new file mode 100644
index 0000000000..93558b310b
--- /dev/null
+++ b/drivers/raw/vfio_platform/vfio_platform.c
@@ -0,0 +1,85 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2023 Marvell.
+ */
+
+#include <errno.h>
+
+#include <rte_bus_platform.h>
+#include <rte_common.h>
+#include <rte_config.h>
+#include <rte_dev.h>
+#include <rte_errno.h>
+#include <rte_pmd_vfio_platform.h>
+#include <rte_vfio.h>
+
+static struct {
+	int container_fd;
+	int refcnt;
+} container_tbl[RTE_MAX_VFIO_CONTAINERS];
+
+static int
+container_next_free_index(void)
+{
+	int i;
+
+	for (i = 0; i < (int)RTE_DIM(container_tbl); i++) {
+		if (container_tbl[i].refcnt == 0)
+			return i;
+	}
+
+	return -1;
+}
+
+static void
+container_put(int index)
+{
+
+	if (container_tbl[index].refcnt == 0)
+		return;
+
+	if (--container_tbl[index].refcnt == 0)
+		rte_vfio_container_destroy(container_tbl[index].container_fd);
+}
+
+static void
+container_get(int index)
+{
+	container_tbl[index].refcnt++;
+}
+
+int
+rte_pmd_vfio_platform_container_create(void)
+{
+	int index, fd;
+
+	fd = rte_vfio_container_create();
+	if (fd < 0)
+		return -1;
+
+	index = container_next_free_index();
+	if (index < 0)
+		return -ENOSPC;
+
+	container_tbl[index].container_fd = fd;
+	container_get(index);
+
+	return index;
+}
+
+int
+rte_pmd_vfio_platform_container_destroy(int container)
+{
+	if ((unsigned int)container >= RTE_DIM(container_tbl))
+		return -EINVAL;
+
+	container_put(container);
+
+	return 0;
+}
+
+static struct rte_platform_driver vfio_platform = {
+};
+
+RTE_PMD_REGISTER_PLATFORM(vfio_platform, vfio_platform);
+RTE_PMD_REGISTER_ALIAS(vfio_platform, vfio-platform);
+RTE_PMD_REGISTER_KMOD_DEP(vfio_platform, "vfio-platform");
-- 
2.25.1

[RFC PATCH 3/7] raw/vfio_platform: add platform probe and remove

[Tomasz Duszynski]

Add platform bus probe and remove.

Signed-off-by: Tomasz Duszynski <tduszynski@marvell.com>
---
 drivers/raw/vfio_platform/vfio_platform.c | 56 +++++++++++++++++++++++
 drivers/raw/vfio_platform/vfio_platform.h | 35 ++++++++++++++
 2 files changed, 91 insertions(+)
 create mode 100644 drivers/raw/vfio_platform/vfio_platform.h

diff --git a/drivers/raw/vfio_platform/vfio_platform.c b/drivers/raw/vfio_platform/vfio_platform.c
index 93558b310b..b056041892 100644
--- a/drivers/raw/vfio_platform/vfio_platform.c
+++ b/drivers/raw/vfio_platform/vfio_platform.c
@@ -9,9 +9,14 @@
 #include <rte_config.h>
 #include <rte_dev.h>
 #include <rte_errno.h>
+#include <rte_log.h>
 #include <rte_pmd_vfio_platform.h>
+#include <rte_rawdev.h>
+#include <rte_rawdev_pmd.h>
 #include <rte_vfio.h>
 
+#include "vfio_platform.h"
+
 static struct {
 	int container_fd;
 	int refcnt;
@@ -77,9 +82,60 @@ rte_pmd_vfio_platform_container_destroy(int container)
 	return 0;
 }
 
+static const struct rte_rawdev_ops vfio_platform_rawdev_ops = {
+};
+
+static int
+vfio_platform_probe(struct rte_platform_device *pdev)
+{
+	struct rte_platform_driver *pdrv = pdev->driver;
+	struct vfio_platform *plat;
+	struct rte_rawdev *rawdev;
+
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+		return 0;
+
+	rawdev = rte_rawdev_pmd_allocate(pdev->device.name, sizeof(*plat), rte_socket_id());
+	if (!rawdev)
+		return -ENOMEM;
+
+	rawdev->dev_ops = &vfio_platform_rawdev_ops;
+	rawdev->device = &pdev->device;
+	rawdev->driver_name = pdrv->driver.name;
+
+	plat = rawdev->dev_private;
+	plat->device = pdev;
+
+	pdev->driver_data = plat;
+
+	VFIO_PLATFORM_LOG(INFO, "probed %s\n", pdev->device.name);
+
+	return 0;
+}
+
+static int
+vfio_platform_remove(struct rte_platform_device *pdev)
+{
+	struct rte_rawdev *rawdev;
+	int ret;
+
+	rawdev = rte_rawdev_pmd_get_named_dev(pdev->device.name);
+	if (!rawdev)
+		return -ENODEV;
+
+	ret = rte_rawdev_pmd_release(rawdev);
+	if (ret == 0)
+		VFIO_PLATFORM_LOG(INFO, "removed %s\n", pdev->device.name);
+
+	return ret;
+}
+
 static struct rte_platform_driver vfio_platform = {
+	.probe = vfio_platform_probe,
+	.remove = vfio_platform_remove,
 };
 
 RTE_PMD_REGISTER_PLATFORM(vfio_platform, vfio_platform);
 RTE_PMD_REGISTER_ALIAS(vfio_platform, vfio-platform);
 RTE_PMD_REGISTER_KMOD_DEP(vfio_platform, "vfio-platform");
+RTE_LOG_REGISTER_DEFAULT(vfio_platform_logtype, NOTICE);
diff --git a/drivers/raw/vfio_platform/vfio_platform.h b/drivers/raw/vfio_platform/vfio_platform.h
new file mode 100644
index 0000000000..f6b2316a26
--- /dev/null
+++ b/drivers/raw/vfio_platform/vfio_platform.h
@@ -0,0 +1,35 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2023 Marvell.
+ */
+
+#ifndef _VFIO_PLATFORM_H_
+#define _VFIO_PLATFORM_H_
+
+#include <rte_bus_platform.h>
+#include <rte_common.h>
+#include <rte_dev.h>
+#include <rte_log.h>
+
+extern int vfio_platform_logtype;
+
+#define VFIO_PLATFORM_LOG(level, ...) \
+	rte_log(RTE_LOG_ ## level, vfio_platform_logtype, \
+		RTE_FMT("vfio platform: " RTE_FMT_HEAD(__VA_ARGS__,) "\n", \
+			RTE_FMT_TAIL(__VA_ARGS__,)))
+
+struct vfio_platform_resource {
+	struct rte_mem_resource mem;
+	const char *name;
+};
+
+struct vfio_platform {
+	int dev_fd;
+	int group_fd;
+	int group;
+	int container;
+	struct rte_platform_device *device;
+	struct vfio_platform_resource *resource;
+	int num_resource;
+};
+
+#endif /* _VFIO_PLATFORM_H_ */
-- 
2.25.1

[RFC PATCH 4/7] raw/vfio_platform: support rawdev close

[Tomasz Duszynski]

Add support for closing raw device allocated during platform probe.

Signed-off-by: Tomasz Duszynski <tduszynski@marvell.com>
---
 drivers/raw/vfio_platform/vfio_platform.c | 49 +++++++++++++++++++++++
 1 file changed, 49 insertions(+)

diff --git a/drivers/raw/vfio_platform/vfio_platform.c b/drivers/raw/vfio_platform/vfio_platform.c
index b056041892..ae0572dc99 100644
--- a/drivers/raw/vfio_platform/vfio_platform.c
+++ b/drivers/raw/vfio_platform/vfio_platform.c
@@ -3,6 +3,8 @@
  */
 
 #include <errno.h>
+#include <sys/mman.h>
+#include <unistd.h>
 
 #include <rte_bus_platform.h>
 #include <rte_common.h>
@@ -17,6 +19,8 @@
 
 #include "vfio_platform.h"
 
+#define VFIO_PLATFORM_SYSFS_BASE "/sys/bus/platform/devices"
+
 static struct {
 	int container_fd;
 	int refcnt;
@@ -52,6 +56,12 @@ container_get(int index)
 	container_tbl[index].refcnt++;
 }
 
+static int
+container_fd(int index)
+{
+	return container_tbl[index].container_fd;
+}
+
 int
 rte_pmd_vfio_platform_container_create(void)
 {
@@ -82,7 +92,46 @@ rte_pmd_vfio_platform_container_destroy(int container)
 	return 0;
 }
 
+static void
+device_release_maps(struct vfio_platform *plat)
+{
+	struct vfio_platform_resource *res;
+	int i;
+
+	for (i = 0; i < plat->num_resource; i++) {
+		res = &plat->resource[i];
+		munmap(res->mem.addr, res->mem.len);
+		free((void *)(size_t)res->name);
+	}
+
+	free(plat->resource);
+	plat->resource = NULL;
+	plat->num_resource = 0;
+}
+
+static void
+device_release(struct vfio_platform *plat)
+{
+	device_release_maps(plat);
+	rte_vfio_release_device(VFIO_PLATFORM_SYSFS_BASE, plat->device->name, plat->dev_fd);
+	rte_vfio_container_group_unbind(container_fd(plat->container), plat->group);
+	close(plat->group_fd);
+	rte_vfio_clear_group(plat->group_fd);
+	container_put(plat->container);
+}
+
+static int
+vfio_rawdev_dev_close(struct rte_rawdev *dev)
+{
+	struct vfio_platform *plat = dev->dev_private;
+
+	device_release(plat);
+
+	return 0;
+}
+
 static const struct rte_rawdev_ops vfio_platform_rawdev_ops = {
+	.dev_close = vfio_rawdev_dev_close,
 };
 
 static int
-- 
2.25.1

[RFC PATCH 5/7] raw/vfio_platform: support rawdev configure

[Tomasz Duszynski]

Add support for binding device VFIO group to a VFIO container.

Signed-off-by: Tomasz Duszynski <tduszynski@marvell.com>
---
 .../raw/vfio_platform/rte_pmd_vfio_platform.h |   6 +
 drivers/raw/vfio_platform/vfio_platform.c     | 147 ++++++++++++++++++
 2 files changed, 153 insertions(+)

diff --git a/drivers/raw/vfio_platform/rte_pmd_vfio_platform.h b/drivers/raw/vfio_platform/rte_pmd_vfio_platform.h
index 307a20d9fc..b595171af6 100644
--- a/drivers/raw/vfio_platform/rte_pmd_vfio_platform.h
+++ b/drivers/raw/vfio_platform/rte_pmd_vfio_platform.h
@@ -17,6 +17,12 @@
 extern "C" {
 #endif
 
+/** vfio platform device configuration */
+struct vfio_platform_dev_config {
+	/** logical value representing the vfio container */
+	int container;
+};
+
 /**
  * Create a new VFIO container.
  *
diff --git a/drivers/raw/vfio_platform/vfio_platform.c b/drivers/raw/vfio_platform/vfio_platform.c
index ae0572dc99..ab5b96a0b0 100644
--- a/drivers/raw/vfio_platform/vfio_platform.c
+++ b/drivers/raw/vfio_platform/vfio_platform.c
@@ -3,9 +3,11 @@
  */
 
 #include <errno.h>
+#include <sys/ioctl.h>
 #include <sys/mman.h>
 #include <unistd.h>
 
+#include <eal_filesystem.h>
 #include <rte_bus_platform.h>
 #include <rte_common.h>
 #include <rte_config.h>
@@ -92,6 +94,29 @@ rte_pmd_vfio_platform_container_destroy(int container)
 	return 0;
 }
 
+static char *
+of_resource_name(const char *dev_name, int index)
+{
+	char path[PATH_MAX], buf[BUFSIZ] = { };
+	int num = 0, ret;
+	char *name;
+
+	snprintf(path, sizeof(path), VFIO_PLATFORM_SYSFS_BASE "/%s/of_node/reg-names", dev_name);
+	/* save space for null byte */
+	ret = eal_parse_sysfs_string(path, buf, sizeof(buf) - 1);
+	if (ret)
+		return NULL;
+
+	for (name = buf; name; name += strlen(name) + 2) {
+		if (num++ != index)
+			continue;
+
+		return strdup(name);
+	}
+
+	return NULL;
+}
+
 static void
 device_release_maps(struct vfio_platform *plat)
 {
@@ -109,6 +134,109 @@ device_release_maps(struct vfio_platform *plat)
 	plat->num_resource = 0;
 }
 
+static int
+device_configure_maps(struct vfio_platform *plat, struct vfio_device_info *dev_info)
+{
+	struct vfio_region_info reg_info = { .argsz = sizeof(reg_info), };
+	struct vfio_platform_resource *res;
+	unsigned int i;
+	void *map;
+	int ret;
+
+	plat->resource = calloc(dev_info->num_regions, sizeof(*plat->resource));
+	if (!plat->resource)
+		return -ENOMEM;
+
+	for (i = 0; i < dev_info->num_regions; i++) {
+		reg_info.index = i;
+		ret = ioctl(plat->dev_fd, VFIO_DEVICE_GET_REGION_INFO, &reg_info);
+		if (ret) {
+			VFIO_PLATFORM_LOG(ERR, "failed to get region info at %d\n", i);
+			ret = -errno;
+			goto out;
+		}
+
+		map = mmap(NULL, reg_info.size, PROT_READ | PROT_WRITE, MAP_SHARED, plat->dev_fd,
+			   reg_info.offset);
+		if (map == MAP_FAILED) {
+			ret = -errno;
+			goto out;
+		}
+
+		res = &plat->resource[plat->num_resource];
+		res->mem.addr = map;
+		res->mem.len = reg_info.size;
+		res->name = of_resource_name(plat->device->name, reg_info.index);
+		plat->num_resource++;
+
+		VFIO_PLATFORM_LOG(DEBUG, "adding resource va=%p len=%lu name = %s\n",
+				  res->mem.addr, res->mem.len, res->name);
+	}
+
+	return 0;
+out:
+	device_release_maps(plat);
+
+	return ret;
+}
+
+static int
+device_configure(struct vfio_platform *plat, struct vfio_platform_dev_config *config)
+{
+	struct vfio_device_info dev_info = { .argsz = sizeof(dev_info), };
+	struct rte_platform_device *pdev = plat->device;
+	const char *name = pdev->device.name;
+	int ret;
+
+	ret = rte_vfio_get_group_num(VFIO_PLATFORM_SYSFS_BASE, name, &plat->group);
+	if (ret != 1) {
+		VFIO_PLATFORM_LOG(ERR, "failed to read device %s iommu group number\n", name);
+		return -ENODEV;
+	}
+
+	container_get(config->container);
+	plat->group_fd = rte_vfio_container_group_bind(container_fd(config->container), plat->group);
+	if (plat->group_fd < 0) {
+		VFIO_PLATFORM_LOG(ERR, "failed to bind group to container\n");
+		ret = -ENODEV;
+		goto out;
+	}
+
+	ret = rte_vfio_setup_device(VFIO_PLATFORM_SYSFS_BASE, name, &plat->dev_fd, &dev_info);
+	if (ret) {
+		VFIO_PLATFORM_LOG(ERR, "failed to setup %s\n", name);
+		ret = -ENODEV;
+		goto out_unbind;
+	}
+
+	if (!(dev_info.flags & VFIO_DEVICE_FLAGS_PLATFORM)) {
+		VFIO_PLATFORM_LOG(ERR, "device not backed by vfio platform\n");
+		ret = -ENOTSUP;
+		goto out_release;
+	}
+
+	plat->container = config->container;
+
+	ret = device_configure_maps(plat, &dev_info);
+	if (ret) {
+		VFIO_PLATFORM_LOG(ERR, "failed to setup memory maps\n");
+		goto out_release;
+	}
+
+	return 0;
+
+out_release:
+	rte_vfio_release_device(VFIO_PLATFORM_SYSFS_BASE, name, plat->dev_fd);
+out_unbind:
+	rte_vfio_container_group_unbind(container_fd(config->container), plat->group);
+out:
+	close(plat->group_fd);
+	rte_vfio_clear_group(plat->group_fd);
+	container_put(config->container);
+
+	return ret;
+}
+
 static void
 device_release(struct vfio_platform *plat)
 {
@@ -120,6 +248,24 @@ device_release(struct vfio_platform *plat)
 	container_put(plat->container);
 }
 
+static int
+vfio_rawdev_dev_configure(const struct rte_rawdev *dev, rte_rawdev_obj_t config, size_t config_size)
+{
+	struct vfio_platform_dev_config *dev_config = config;
+	struct vfio_platform *plat = dev->dev_private;
+
+	if (!dev_config)
+		return -EINVAL;
+
+	if (sizeof(*dev_config) != config_size)
+		return -EINVAL;
+
+	if ((unsigned int)dev_config->container >= RTE_DIM(container_tbl))
+		return -EINVAL;
+
+	return device_configure(plat, dev_config);
+}
+
 static int
 vfio_rawdev_dev_close(struct rte_rawdev *dev)
 {
@@ -131,6 +277,7 @@ vfio_rawdev_dev_close(struct rte_rawdev *dev)
 }
 
 static const struct rte_rawdev_ops vfio_platform_rawdev_ops = {
+	.dev_configure = vfio_rawdev_dev_configure,
 	.dev_close = vfio_rawdev_dev_close,
 };
 
-- 
2.25.1

[RFC PATCH 6/7] raw/vfio_platform: support rawdev device info

[Tomasz Duszynski]

Add support for querying rawdev device info.

Signed-off-by: Tomasz Duszynski <tduszynski@marvell.com>
---
 .../raw/vfio_platform/rte_pmd_vfio_platform.h | 34 ++++++++++++++
 drivers/raw/vfio_platform/vfio_platform.c     | 45 +++++++++++++++++++
 2 files changed, 79 insertions(+)

diff --git a/drivers/raw/vfio_platform/rte_pmd_vfio_platform.h b/drivers/raw/vfio_platform/rte_pmd_vfio_platform.h
index b595171af6..377cebde08 100644
--- a/drivers/raw/vfio_platform/rte_pmd_vfio_platform.h
+++ b/drivers/raw/vfio_platform/rte_pmd_vfio_platform.h
@@ -17,6 +17,40 @@
 extern "C" {
 #endif
 
+#include <rte_compat.h>
+#include <rte_dev.h>
+
+/**
+ * vfio platform device region info
+ *
+ * Caller uses that to retrieve information about device memory regions.
+ * Caller provides either resource name or an index, both of which can
+ * be extracted from the device tree. The former is equivalent to
+ * 'reg-names' property while the latter is a 0-based index into 'reg'
+ * array.
+ */
+struct vfio_platform_dev_region_info {
+	/**< mmapped memory resource */
+	struct rte_mem_resource mem;
+	/**< name of the resource */
+	const char *name;
+	/**< index of the resource */
+	int index;
+};
+
+enum vfio_platform_info {
+	INFO_REGION_INDEX,
+	INFO_REGION_NAME,
+};
+
+/**< vfio platform device info */
+struct vfio_platform_dev_info {
+	/**< Type of device info */
+	enum vfio_platform_info type;
+	/**< Memory region info */
+	struct vfio_platform_dev_region_info reg_info;
+};
+
 /** vfio platform device configuration */
 struct vfio_platform_dev_config {
 	/** logical value representing the vfio container */
diff --git a/drivers/raw/vfio_platform/vfio_platform.c b/drivers/raw/vfio_platform/vfio_platform.c
index ab5b96a0b0..8148dce06b 100644
--- a/drivers/raw/vfio_platform/vfio_platform.c
+++ b/drivers/raw/vfio_platform/vfio_platform.c
@@ -94,6 +94,50 @@ rte_pmd_vfio_platform_container_destroy(int container)
 	return 0;
 }
 
+static int
+vfio_rawdev_dev_info_get(struct rte_rawdev *dev, rte_rawdev_obj_t dev_info,
+			 size_t dev_private_size)
+{
+	struct vfio_platform_dev_info *info = (struct vfio_platform_dev_info *)dev_info;
+	struct vfio_platform_dev_region_info *reg_info = &info->reg_info;
+	struct vfio_platform *plat = dev->dev_private;
+	int i;
+
+	if (dev_private_size != sizeof(*info))
+		return -EINVAL;
+
+	if (plat->num_resource == 0)
+		return -ENODATA;
+
+	switch (info->type) {
+	case INFO_REGION_INDEX:
+		if (reg_info->index < 0 || reg_info->index >= plat->num_resource)
+			return -EINVAL;
+
+		i = reg_info->index;
+		break;
+	case INFO_REGION_NAME:
+		if (!reg_info->name)
+			return -EINVAL;
+
+		for (i = 0; i < plat->num_resource; i++) {
+			if (!strcmp(reg_info->name, plat->resource[i].name))
+				break;
+		}
+
+		if (i == plat->num_resource)
+			return -ENODATA;
+
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	memcpy(&reg_info->mem, &plat->resource[i].mem, sizeof(reg_info->mem));
+
+	return 0;
+}
+
 static char *
 of_resource_name(const char *dev_name, int index)
 {
@@ -277,6 +321,7 @@ vfio_rawdev_dev_close(struct rte_rawdev *dev)
 }
 
 static const struct rte_rawdev_ops vfio_platform_rawdev_ops = {
+	.dev_info_get = vfio_rawdev_dev_info_get,
 	.dev_configure = vfio_rawdev_dev_configure,
 	.dev_close = vfio_rawdev_dev_close,
 };
-- 
2.25.1

[RFC PATCH 7/7] raw/vfio_platform: support DMA map/unmap

[Tomasz Duszynski]

Add support for DMA map/unmap operations.

Signed-off-by: Tomasz Duszynski <tduszynski@marvell.com>
---
 .../raw/vfio_platform/rte_pmd_vfio_platform.h | 40 +++++++++++
 drivers/raw/vfio_platform/version.map         |  2 +
 drivers/raw/vfio_platform/vfio_platform.c     | 67 +++++++++++++++++++
 3 files changed, 109 insertions(+)

diff --git a/drivers/raw/vfio_platform/rte_pmd_vfio_platform.h b/drivers/raw/vfio_platform/rte_pmd_vfio_platform.h
index 377cebde08..e1c194c46d 100644
--- a/drivers/raw/vfio_platform/rte_pmd_vfio_platform.h
+++ b/drivers/raw/vfio_platform/rte_pmd_vfio_platform.h
@@ -82,6 +82,46 @@ __rte_experimental
 int
 rte_pmd_vfio_platform_container_destroy(int container);
 
+/**
+ * Map DMA memory.
+ *
+ * @param dev_id
+ *   Device identifier.
+ * @param addr
+ *   Virtual address to map.
+ * @param iova
+ *   IOVA address to map.
+ * @param len
+ *   Length of the memory segment being mapped.
+ *
+ * @return
+ *   0 on success
+ *   <0 on failure and rte_errno is set
+ */
+__rte_experimental
+int
+rte_pmd_vfio_platform_dma_map(uint16_t dev_id, void *addr, uint64_t iova, size_t len);
+
+/**
+ * Remove DMA memory mapping.
+ *
+ * @param dev_id
+ *   Device identifier.
+ * @param addr
+ *   Virtual address to map.
+ * @param iova
+ *   IOVA address to map.
+ * @param len
+ *   Length of the memory segment being mapped.
+ *
+ * @return
+ *   0 on success
+ *   <0 on failure and rte_errno is set
+ */
+__rte_experimental
+int
+rte_pmd_vfio_platform_dma_unmap(uint16_t dev_id, void *addr, uint64_t iova, size_t len);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/drivers/raw/vfio_platform/version.map b/drivers/raw/vfio_platform/version.map
index 2aea50f4c1..1b54459183 100644
--- a/drivers/raw/vfio_platform/version.map
+++ b/drivers/raw/vfio_platform/version.map
@@ -7,4 +7,6 @@ EXPERIMENTAL {
 
 	rte_pmd_vfio_platform_container_create;
 	rte_pmd_vfio_platform_container_destroy;
+	rte_pmd_vfio_platform_dma_map;
+	rte_pmd_vfio_platform_dma_unmap;
 };
diff --git a/drivers/raw/vfio_platform/vfio_platform.c b/drivers/raw/vfio_platform/vfio_platform.c
index 8148dce06b..966f9f177e 100644
--- a/drivers/raw/vfio_platform/vfio_platform.c
+++ b/drivers/raw/vfio_platform/vfio_platform.c
@@ -94,6 +94,45 @@ rte_pmd_vfio_platform_container_destroy(int container)
 	return 0;
 }
 
+static struct rte_rawdev *
+rawdev_from_dev_id(uint16_t dev_id)
+{
+	if (!rte_rawdev_pmd_is_valid_dev(dev_id))
+		return NULL;
+
+	return &rte_rawdevs[dev_id];
+}
+
+int
+rte_pmd_vfio_platform_dma_map(uint16_t dev_id, void *addr, uint64_t iova, size_t len)
+{
+	struct rte_rawdev *rawdev;
+
+	rawdev = rawdev_from_dev_id(dev_id);
+	if (!rawdev) {
+		rte_errno = ENODEV;
+
+		return -1;
+	}
+
+	return rte_dev_dma_map(rawdev->device, addr, iova, len);
+}
+
+int
+rte_pmd_vfio_platform_dma_unmap(uint16_t dev_id, void *addr, uint64_t iova, size_t len)
+{
+	struct rte_rawdev *rawdev;
+
+	rawdev = rawdev_from_dev_id(dev_id);
+	if (!rawdev) {
+		rte_errno = ENODEV;
+
+		return -1;
+	}
+
+	return rte_dev_dma_unmap(rawdev->device, addr, iova, len);
+}
+
 static int
 vfio_rawdev_dev_info_get(struct rte_rawdev *dev, rte_rawdev_obj_t dev_info,
 			 size_t dev_private_size)
@@ -371,9 +410,37 @@ vfio_platform_remove(struct rte_platform_device *pdev)
 	return ret;
 }
 
+static int
+vfio_platform_dma_map(struct rte_platform_device *pdev, void *addr, uint64_t iova, size_t len)
+{
+	struct vfio_platform *plat = pdev->driver_data;
+	int ret;
+
+	ret = rte_vfio_container_dma_map(container_fd(plat->container), (uint64_t)addr, iova, len);
+	if (ret)
+		return -EFAULT;
+
+	return 0;
+}
+
+static int
+vfio_platform_dma_unmap(struct rte_platform_device *pdev, void *addr, uint64_t iova, size_t len)
+{
+	struct vfio_platform *plat = pdev->driver_data;
+	int ret;
+
+	ret = rte_vfio_container_dma_unmap(container_fd(plat->container), (uint64_t)addr, iova, len);
+	if (ret)
+		return -EFAULT;
+
+	return 0;
+}
+
 static struct rte_platform_driver vfio_platform = {
 	.probe = vfio_platform_probe,
 	.remove = vfio_platform_remove,
+	.dma_map = vfio_platform_dma_map,
+	.dma_unmap = vfio_platform_dma_unmap,
 };
 
 RTE_PMD_REGISTER_PLATFORM(vfio_platform, vfio_platform);
-- 
2.25.1

RE: [RFC PATCH 0/7] support vfio platform PMD

[Xia, Chenbo]

> -----Original Message-----
> From: Tomasz Duszynski <tduszynski@marvell.com>
> Sent: Friday, December 23, 2022 7:24 AM
> To: dev@dpdk.org
> Cc: thomas@monjalon.net; jerinj@marvell.com; Tomasz Duszynski
> <tduszynski@marvell.com>
> Subject: [RFC PATCH 0/7] support vfio platform PMD
> 
> This series aims to add support for managing vfio platform
> devices from userspace application conveniently i.e
> without going through the configuration required by vfio
> and opencoding everything. Instead convenience helpers
> are provided.
> 
> vfio platform devices, from the kernel standpoint, are ones
> that do not have built-in discovery capabilities and are
> behind an IOMMU.
> 
> This PMD is backed by both vfio-platform and vfio
> kernel drivers and ideally should give access to all
> vfio capabilities. As of now, access to memory maps and
> DMA are supported.
> 
> PMD requires platform bus support [1].
> 
> [1] https://lore.kernel.org/dpdk-dev/20221222000106.270619-1-
> tduszynski@marvell.com/

I have not looked into all details. But just wondering: why not making this
part of the platform bus. This seems generic and then vendors can have their
own platform drivers.

Thanks,
Chenbo

Re: [RFC PATCH 1/7] lib: add helper to read strings from sysfs files

[Stephen Hemminger]

On Fri, 23 Dec 2022 00:24:29 +0100
Tomasz Duszynski <tduszynski@marvell.com> wrote:

> diff --git a/lib/eal/version.map b/lib/eal/version.map
> index 7ad12a7dc9..2dbc10e6be 100644
> --- a/lib/eal/version.map
> +++ b/lib/eal/version.map
> @@ -440,6 +440,9 @@ EXPERIMENTAL {
>  	rte_thread_detach;
>  	rte_thread_equal;
>  	rte_thread_join;
> +
> +	# added in 23.03
> +	eal_parse_sysfs_string; # WINDOWS_NO_EXPORT
>  };

Should be internal not exported as user API.

RE: [EXT] Re: [RFC PATCH 1/7] lib: add helper to read strings from sysfs files

[Tomasz Duszynski]

Hi Stephen, 

>-----Original Message-----
>From: Stephen Hemminger <stephen@networkplumber.org>
>Sent: Friday, December 23, 2022 5:40 PM
>To: Tomasz Duszynski <tduszynski@marvell.com>
>Cc: dev@dpdk.org; thomas@monjalon.net; Jerin Jacob Kollanukkaran <jerinj@marvell.com>
>Subject: [EXT] Re: [RFC PATCH 1/7] lib: add helper to read strings from sysfs files
>
>External Email
>
>----------------------------------------------------------------------
>On Fri, 23 Dec 2022 00:24:29 +0100
>Tomasz Duszynski <tduszynski@marvell.com> wrote:
>
>> diff --git a/lib/eal/version.map b/lib/eal/version.map index
>> 7ad12a7dc9..2dbc10e6be 100644
>> --- a/lib/eal/version.map
>> +++ b/lib/eal/version.map
>> @@ -440,6 +440,9 @@ EXPERIMENTAL {
>>  	rte_thread_detach;
>>  	rte_thread_equal;
>>  	rte_thread_join;
>> +
>> +	# added in 23.03
>> +	eal_parse_sysfs_string; # WINDOWS_NO_EXPORT
>>  };
>
>Should be internal not exported as user API.

Okay. I'll put that under different section. 

RE: [RFC PATCH 0/7] support vfio platform PMD

[Tomasz Duszynski]

Hi Chenbo, 

>-----Original Message-----
>From: Xia, Chenbo <chenbo.xia@intel.com>
>Sent: Friday, December 23, 2022 8:05 AM
>To: Tomasz Duszynski <tduszynski@marvell.com>; dev@dpdk.org
>Cc: thomas@monjalon.net; Jerin Jacob Kollanukkaran <jerinj@marvell.com>
>Subject: [EXT] RE: [RFC PATCH 0/7] support vfio platform PMD
>
>External Email
>
>----------------------------------------------------------------------
>> -----Original Message-----
>> From: Tomasz Duszynski <tduszynski@marvell.com>
>> Sent: Friday, December 23, 2022 7:24 AM
>> To: dev@dpdk.org
>> Cc: thomas@monjalon.net; jerinj@marvell.com; Tomasz Duszynski
>> <tduszynski@marvell.com>
>> Subject: [RFC PATCH 0/7] support vfio platform PMD
>>
>> This series aims to add support for managing vfio platform devices
>> from userspace application conveniently i.e without going through the
>> configuration required by vfio and opencoding everything. Instead
>> convenience helpers are provided.
>>
>> vfio platform devices, from the kernel standpoint, are ones that do
>> not have built-in discovery capabilities and are behind an IOMMU.
>>
>> This PMD is backed by both vfio-platform and vfio kernel drivers and
>> ideally should give access to all vfio capabilities. As of now, access
>> to memory maps and DMA are supported.
>>
>> PMD requires platform bus support [1].
>>
>> [1]
>> https://urldefense.proofpoint.com/v2/url?u=https-3A__lore.kernel.org_d
>> pdk-2Ddev_20221222000106.270619-2D1-2D&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xt
>> fQ&r=PZNXgrbjdlXxVEEGYkxIxRndyEUwWU_ad5ce22YI6Is&m=D9rwjUu4gMDK-DA_YvF
>> eU6OBKRUXqZVQ7Nfg3O4xldhUMqQsXQx4AM9La3PpF1Bd&s=L93CKF1oLmgSm7z50nWbVf
>> 4FFDww_noCSxOENlbfXGY&e=
>> tduszynski@marvell.com/
>
>I have not looked into all details. But just wondering: why not making this part of the platform
>bus. This seems generic and then vendors can have their own platform drivers.
>

Valid point. I'll rethink current approach.  

>Thanks,
>Chenbo