[dpdk-dev] [PATCH] eal: fix linuxapp numa node detection

[dpdk-dev] [PATCH] eal: fix linuxapp numa node detection

[Bruce Richardson]

Using the "physical_package_id" as a fallback for determining the
numa node of a core tends to be unreliable. Fix this by using a
detection routine which reads the numa information from
/sys/devices/system/node and just returns a numa node of 0 on
failure.

Signed-off-by: Bruce Richardson <bruce.richardson@intel.com>
---
 lib/librte_eal/linuxapp/eal/eal_lcore.c | 60 ++++++++-------------------------
 1 file changed, 14 insertions(+), 46 deletions(-)

diff --git a/lib/librte_eal/linuxapp/eal/eal_lcore.c b/lib/librte_eal/linuxapp/eal/eal_lcore.c
index ef8c433..f25108e 100644
--- a/lib/librte_eal/linuxapp/eal/eal_lcore.c
+++ b/lib/librte_eal/linuxapp/eal/eal_lcore.c
@@ -49,7 +49,7 @@
 
 #define SYS_CPU_DIR "/sys/devices/system/cpu/cpu%u"
 #define CORE_ID_FILE "topology/core_id"
-#define PHYS_PKG_FILE "topology/physical_package_id"
+#define NUMA_NODE_PATH "/sys/devices/system/node"
 
 /* Check if a cpu is present by the presence of the cpu information for it */
 static int
@@ -66,58 +66,26 @@ cpu_detected(unsigned lcore_id)
 	return 1;
 }
 
-/* Get CPU socket id (NUMA node) by reading directory
- * /sys/devices/system/cpu/cpuX looking for symlink "nodeY"
- * which gives the NUMA topology information.
- * Note: physical package id != NUMA node, but we use it as a
- * fallback for kernels which don't create a nodeY link
+/*
+ * Get CPU socket id (NUMA node) for a logical core.
+ *
+ * This searches each nodeX directories in /sys for the symlink for the given
+ * lcore_id and returns the numa node where the lcore is found. If lcore is not
+ * found on any numa node, returns zero.
  */
 unsigned
 eal_cpu_socket_id(unsigned lcore_id)
 {
-	const char node_prefix[] = "node";
-	const size_t prefix_len = sizeof(node_prefix) - 1;
-	char path[PATH_MAX];
-	DIR *d = NULL;
-	unsigned long id = 0;
-	struct dirent *e;
-	char *endptr = NULL;
-
-	int len = snprintf(path, sizeof(path),
-			       SYS_CPU_DIR, lcore_id);
-	if (len <= 0 || (unsigned)len >= sizeof(path))
-		goto err;
+	unsigned socket;
 
-	d = opendir(path);
-	if (!d)
-		goto err;
+	for (socket = 0; socket < RTE_MAX_NUMA_NODES; socket++) {
+		char path[PATH_MAX];
 
-	while ((e = readdir(d)) != NULL) {
-		if (strncmp(e->d_name, node_prefix, prefix_len) == 0) {
-			id = strtoul(e->d_name+prefix_len, &endptr, 0);
-			break;
-		}
-	}
-	if (endptr == NULL || *endptr!='\0' || endptr == e->d_name+prefix_len) {
-		RTE_LOG(WARNING, EAL, "Cannot read numa node link "
-				"for lcore %u - using physical package id instead\n",
-				lcore_id);
-
-		len = snprintf(path, sizeof(path), SYS_CPU_DIR "/%s",
-				lcore_id, PHYS_PKG_FILE);
-		if (len <= 0 || (unsigned)len >= sizeof(path))
-			goto err;
-		if (eal_parse_sysfs_value(path, &id) != 0)
-			goto err;
+		snprintf(path, sizeof(path),"%s/node%u/cpu%u", NUMA_NODE_PATH,
+				socket, lcore_id);
+		if (access(path, F_OK) == 0)
+			return socket;
 	}
-	closedir(d);
-	return (unsigned)id;
-
-err:
-	if (d)
-		closedir(d);
-	RTE_LOG(ERR, EAL, "Error getting NUMA socket information from %s "
-			"for lcore %u - assuming NUMA socket 0\n", SYS_CPU_DIR, lcore_id);
 	return 0;
 }
 
-- 
2.1.0

Re: [dpdk-dev] [PATCH] eal: fix linuxapp numa node detection

[Stephen Hemminger]

On Wed, 22 Apr 2015 14:31:55 +0100
Bruce Richardson <bruce.richardson@intel.com> wrote:

> Using the "physical_package_id" as a fallback for determining the
> numa node of a core tends to be unreliable. Fix this by using a
> detection routine which reads the numa information from
> /sys/devices/system/node and just returns a numa node of 0 on
> failure.
> 
> Signed-off-by: Bruce Richardson <bruce.richardson@intel.com>

Acked-by: Stephen Hemminger <stephen@networkplumber.org>

Documentation/cputopology.txt

> 1) /sys/devices/system/cpu/cpuX/topology/physical_package_id:
> 
> 	physical package id of cpuX. Typically corresponds to a physical
> 	socket number, but the actual value is architecture and platform
> 	dependent.

Re: [dpdk-dev] [PATCH] eal: fix linuxapp numa node detection

[Ramia, Kannan Babu]

BTW, the BIOS fix has been posted for the NUMA node reporting issue in Linux kernel when COD is disabled. Now the /sys/devices/*/*/numa_node will provide you the correct information.

Regards
Kannan Babu

-----Original Message-----
From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Bruce Richardson
Sent: Wednesday, April 22, 2015 7:02 PM
To: dev@dpdk.org
Subject: [dpdk-dev] [PATCH] eal: fix linuxapp numa node detection

Using the "physical_package_id" as a fallback for determining the numa node of a core tends to be unreliable. Fix this by using a detection routine which reads the numa information from /sys/devices/system/node and just returns a numa node of 0 on failure.

Signed-off-by: Bruce Richardson <bruce.richardson@intel.com>
---
 lib/librte_eal/linuxapp/eal/eal_lcore.c | 60 ++++++++-------------------------
 1 file changed, 14 insertions(+), 46 deletions(-)

diff --git a/lib/librte_eal/linuxapp/eal/eal_lcore.c b/lib/librte_eal/linuxapp/eal/eal_lcore.c
index ef8c433..f25108e 100644
--- a/lib/librte_eal/linuxapp/eal/eal_lcore.c
+++ b/lib/librte_eal/linuxapp/eal/eal_lcore.c
@@ -49,7 +49,7 @@
 
 #define SYS_CPU_DIR "/sys/devices/system/cpu/cpu%u"
 #define CORE_ID_FILE "topology/core_id"
-#define PHYS_PKG_FILE "topology/physical_package_id"
+#define NUMA_NODE_PATH "/sys/devices/system/node"
 
 /* Check if a cpu is present by the presence of the cpu information for it */  static int @@ -66,58 +66,26 @@ cpu_detected(unsigned lcore_id)
 	return 1;
 }
 
-/* Get CPU socket id (NUMA node) by reading directory
- * /sys/devices/system/cpu/cpuX looking for symlink "nodeY"
- * which gives the NUMA topology information.
- * Note: physical package id != NUMA node, but we use it as a
- * fallback for kernels which don't create a nodeY link
+/*
+ * Get CPU socket id (NUMA node) for a logical core.
+ *
+ * This searches each nodeX directories in /sys for the symlink for the 
+given
+ * lcore_id and returns the numa node where the lcore is found. If 
+lcore is not
+ * found on any numa node, returns zero.
  */
 unsigned
 eal_cpu_socket_id(unsigned lcore_id)
 {
-	const char node_prefix[] = "node";
-	const size_t prefix_len = sizeof(node_prefix) - 1;
-	char path[PATH_MAX];
-	DIR *d = NULL;
-	unsigned long id = 0;
-	struct dirent *e;
-	char *endptr = NULL;
-
-	int len = snprintf(path, sizeof(path),
-			       SYS_CPU_DIR, lcore_id);
-	if (len <= 0 || (unsigned)len >= sizeof(path))
-		goto err;
+	unsigned socket;
 
-	d = opendir(path);
-	if (!d)
-		goto err;
+	for (socket = 0; socket < RTE_MAX_NUMA_NODES; socket++) {
+		char path[PATH_MAX];
 
-	while ((e = readdir(d)) != NULL) {
-		if (strncmp(e->d_name, node_prefix, prefix_len) == 0) {
-			id = strtoul(e->d_name+prefix_len, &endptr, 0);
-			break;
-		}
-	}
-	if (endptr == NULL || *endptr!='\0' || endptr == e->d_name+prefix_len) {
-		RTE_LOG(WARNING, EAL, "Cannot read numa node link "
-				"for lcore %u - using physical package id instead\n",
-				lcore_id);
-
-		len = snprintf(path, sizeof(path), SYS_CPU_DIR "/%s",
-				lcore_id, PHYS_PKG_FILE);
-		if (len <= 0 || (unsigned)len >= sizeof(path))
-			goto err;
-		if (eal_parse_sysfs_value(path, &id) != 0)
-			goto err;
+		snprintf(path, sizeof(path),"%s/node%u/cpu%u", NUMA_NODE_PATH,
+				socket, lcore_id);
+		if (access(path, F_OK) == 0)
+			return socket;
 	}
-	closedir(d);
-	return (unsigned)id;
-
-err:
-	if (d)
-		closedir(d);
-	RTE_LOG(ERR, EAL, "Error getting NUMA socket information from %s "
-			"for lcore %u - assuming NUMA socket 0\n", SYS_CPU_DIR, lcore_id);
 	return 0;
 }
 
--
2.1.0

Re: [dpdk-dev] [PATCH] eal: fix linuxapp numa node detection

[Thomas Monjalon]

2015-04-22 08:44, Stephen Hemminger:
> On Wed, 22 Apr 2015 14:31:55 +0100
> Bruce Richardson <bruce.richardson@intel.com> wrote:
> 
> > Using the "physical_package_id" as a fallback for determining the
> > numa node of a core tends to be unreliable. Fix this by using a
> > detection routine which reads the numa information from
> > /sys/devices/system/node and just returns a numa node of 0 on
> > failure.
> > 
> > Signed-off-by: Bruce Richardson <bruce.richardson@intel.com>
> 
> Acked-by: Stephen Hemminger <stephen@networkplumber.org>

Applied, thanks