[dpdk-dev] [PATCH 4/4] eal: remove limitation on cpuset with --lcores

[David Marchand <david.marchand@redhat.com>]

Contrary to the -c/-l options, where a logical core runs on the same
physical core in a 1:1 fashion (example: lcore 0 runs on core 0, lcore
16 runs on core 16), the --lcores option makes it possible to select the
physical cores on which runs a logical core.

However the current parsing code still limits the cpuset to the
[0, RTE_MAX_LCORE] range.

Example, before the patch, on a 24 cores system with RTE_MAX_LCORE == 16:
$ ./master/app/testpmd --no-huge --no-pci -m 512 --log-level *:debug \
 --lcores 0@16,1@17 -- -i --total-num-mbufs 2048
EAL: Detected lcore 0 as core 0 on socket 0
EAL: Detected lcore 1 as core 1 on socket 0
EAL: Detected lcore 2 as core 2 on socket 0
EAL: Detected lcore 3 as core 3 on socket 0
EAL: Detected lcore 4 as core 4 on socket 0
EAL: Detected lcore 5 as core 5 on socket 0
EAL: Detected lcore 6 as core 6 on socket 0
EAL: Detected lcore 7 as core 8 on socket 0
EAL: Detected lcore 8 as core 9 on socket 0
EAL: Detected lcore 9 as core 10 on socket 0
EAL: Detected lcore 10 as core 11 on socket 0
EAL: Detected lcore 11 as core 12 on socket 0
EAL: Detected lcore 12 as core 13 on socket 0
EAL: Detected lcore 13 as core 14 on socket 0
EAL: Detected lcore 14 as core 0 on socket 0
EAL: Detected lcore 15 as core 1 on socket 0
EAL: Skipped lcore 16 as core 2 on socket 0
EAL: Skipped lcore 17 as core 3 on socket 0
EAL: Skipped lcore 18 as core 4 on socket 0
EAL: Skipped lcore 19 as core 5 on socket 0
EAL: Skipped lcore 20 as core 6 on socket 0
EAL: Skipped lcore 21 as core 8 on socket 0
EAL: Skipped lcore 22 as core 9 on socket 0
EAL: Skipped lcore 23 as core 10 on socket 0
EAL: Skipped lcore 24 as core 11 on socket 0
EAL: Skipped lcore 25 as core 12 on socket 0
EAL: Skipped lcore 26 as core 13 on socket 0
EAL: Skipped lcore 27 as core 14 on socket 0
EAL: Support maximum 16 logical core(s) by configuration.
EAL: Detected 16 lcore(s)
EAL: Detected 1 NUMA nodes
EAL: invalid parameter for --lcores

We can remove this limitation by using a cpuset_t (which is a more
natural type since this is what gets passed to pthread_setaffinity*
in the end).

After the patch:
$ ./master/app/testpmd --no-huge --no-pci -m 512 --log-level *:debug \
 --lcores 0@16,1@17 -- -i --total-num-mbufs 2048
[...]
EAL: Master lcore 0 is ready (tid=7f94217bbc00;cpuset=[16])
EAL: lcore 1 is ready (tid=7f941f491700;cpuset=[17])

Signed-off-by: David Marchand <david.marchand@redhat.com>
---
 drivers/bus/fslmc/portal/dpaa2_hw_dpio.c   | 31 ++++++-----
 lib/librte_eal/common/eal_common_options.c | 63 +++++++++++-----------
 lib/librte_eal/common/eal_common_thread.c  |  4 +-
 3 files changed, 50 insertions(+), 48 deletions(-)

diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
index 3ca3ae4f51..739ce434ba 100644
--- a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
+++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
@@ -365,20 +365,25 @@ dpaa2_check_lcore_cpuset(void)
 		dpaa2_cpu[lcore_id] = 0xffffffff;
 
 	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
-		for (i = 0; i < RTE_MAX_LCORE; i++) {
-			rte_cpuset_t cpuset = rte_lcore_cpuset(lcore_id);
-
-			if (CPU_ISSET(i, &cpuset)) {
-				RTE_LOG(DEBUG, EAL, "lcore id = %u cpu=%u\n",
-					lcore_id, i);
-				if (dpaa2_cpu[lcore_id] != 0xffffffff) {
-					DPAA2_BUS_ERR(
-				    "ERR:lcore map to multi-cpu not supported");
-					ret = -1;
-				} else  {
-					dpaa2_cpu[lcore_id] = i;
-				}
+		rte_cpuset_t cpuset = rte_lcore_cpuset(lcore_id);
+
+		for (i = 0; i < CPU_SETSIZE; i++) {
+			if (!CPU_ISSET(i, &cpuset))
+				continue;
+			if (i >= RTE_MAX_LCORE) {
+				DPAA2_BUS_ERR("ERR:lcore map to core %u (>= %u) not supported",
+					i, RTE_MAX_LCORE);
+				ret = -1;
+				continue;
 			}
+			RTE_LOG(DEBUG, EAL, "lcore id = %u cpu=%u\n",
+				lcore_id, i);
+			if (dpaa2_cpu[lcore_id] != 0xffffffff) {
+				DPAA2_BUS_ERR("ERR:lcore map to multi-cpu not supported");
+				ret = -1;
+				continue;
+			}
+			dpaa2_cpu[lcore_id] = i;
 		}
 	}
 	return ret;
diff --git a/lib/librte_eal/common/eal_common_options.c b/lib/librte_eal/common/eal_common_options.c
index 68f7d1cd73..5920233bcd 100644
--- a/lib/librte_eal/common/eal_common_options.c
+++ b/lib/librte_eal/common/eal_common_options.c
@@ -658,14 +658,14 @@ eal_parse_master_lcore(const char *arg)
  *                       ',' used for a single number.
  */
 static int
-eal_parse_set(const char *input, uint16_t set[], unsigned num)
+eal_parse_set(const char *input, rte_cpuset_t *set)
 {
 	unsigned idx;
 	const char *str = input;
 	char *end = NULL;
 	unsigned min, max;
 
-	memset(set, 0, num * sizeof(uint16_t));
+	CPU_ZERO(set);
 
 	while (isblank(*str))
 		str++;
@@ -678,7 +678,7 @@ eal_parse_set(const char *input, uint16_t set[], unsigned num)
 	if (*str != '(') {
 		errno = 0;
 		idx = strtoul(str, &end, 10);
-		if (errno || end == NULL || idx >= num)
+		if (errno || end == NULL || idx >= CPU_SETSIZE)
 			return -1;
 		else {
 			while (isblank(*end))
@@ -696,7 +696,7 @@ eal_parse_set(const char *input, uint16_t set[], unsigned num)
 
 				errno = 0;
 				idx = strtoul(end, &end, 10);
-				if (errno || end == NULL || idx >= num)
+				if (errno || end == NULL || idx >= CPU_SETSIZE)
 					return -1;
 				max = idx;
 				while (isblank(*end))
@@ -711,7 +711,7 @@ eal_parse_set(const char *input, uint16_t set[], unsigned num)
 
 			for (idx = RTE_MIN(min, max);
 			     idx <= RTE_MAX(min, max); idx++)
-				set[idx] = 1;
+				CPU_SET(idx, set);
 
 			return end - input;
 		}
@@ -736,7 +736,7 @@ eal_parse_set(const char *input, uint16_t set[], unsigned num)
 		/* get the digit value */
 		errno = 0;
 		idx = strtoul(str, &end, 10);
-		if (errno || end == NULL || idx >= num)
+		if (errno || end == NULL || idx >= CPU_SETSIZE)
 			return -1;
 
 		/* go ahead to separator '-',',' and ')' */
@@ -753,7 +753,7 @@ eal_parse_set(const char *input, uint16_t set[], unsigned num)
 				min = idx;
 			for (idx = RTE_MIN(min, max);
 			     idx <= RTE_MAX(min, max); idx++)
-				set[idx] = 1;
+				CPU_SET(idx, set);
 
 			min = RTE_MAX_LCORE;
 		} else
@@ -772,17 +772,13 @@ eal_parse_set(const char *input, uint16_t set[], unsigned num)
 	return str - input;
 }
 
-/* convert from set array to cpuset bitmap */
 static int
-convert_to_cpuset(rte_cpuset_t *cpusetp,
-	      uint16_t *set, unsigned num)
+check_cpuset(rte_cpuset_t *set)
 {
-	unsigned idx;
-
-	CPU_ZERO(cpusetp);
+	unsigned int idx;
 
-	for (idx = 0; idx < num; idx++) {
-		if (!set[idx])
+	for (idx = 0; idx < CPU_SETSIZE; idx++) {
+		if (!CPU_ISSET(idx, set))
 			continue;
 
 		if (eal_cpu_detected(idx) == 0) {
@@ -790,10 +786,7 @@ convert_to_cpuset(rte_cpuset_t *cpusetp,
 				"unavailable\n", idx);
 			return -1;
 		}
-
-		CPU_SET(idx, cpusetp);
 	}
-
 	return 0;
 }
 
@@ -815,7 +808,8 @@ static int
 eal_parse_lcores(const char *lcores)
 {
 	struct rte_config *cfg = rte_eal_get_configuration();
-	static uint16_t set[RTE_MAX_LCORE];
+	rte_cpuset_t lcore_set;
+	unsigned int set_count;
 	unsigned idx = 0;
 	unsigned count = 0;
 	const char *lcore_start = NULL;
@@ -864,18 +858,13 @@ eal_parse_lcores(const char *lcores)
 		lcores += strcspn(lcores, "@,");
 
 		if (*lcores == '@') {
-			/* explicit assign cpu_set */
-			offset = eal_parse_set(lcores + 1, set, RTE_DIM(set));
+			/* explicit assign cpuset and update the end cursor */
+			offset = eal_parse_set(lcores + 1, &cpuset);
 			if (offset < 0)
 				goto err;
-
-			/* prepare cpu_set and update the end cursor */
-			if (0 > convert_to_cpuset(&cpuset,
-						  set, RTE_DIM(set)))
-				goto err;
 			end = lcores + 1 + offset;
 		} else { /* ',' or '\0' */
-			/* haven't given cpu_set, current loop done */
+			/* haven't given cpuset, current loop done */
 			end = lcores;
 
 			/* go back to check <number>-<number> */
@@ -889,18 +878,19 @@ eal_parse_lcores(const char *lcores)
 			goto err;
 
 		/* parse lcore_set from start point */
-		if (0 > eal_parse_set(lcore_start, set, RTE_DIM(set)))
+		if (eal_parse_set(lcore_start, &lcore_set) < 0)
 			goto err;
 
-		/* without '@', by default using lcore_set as cpu_set */
-		if (*lcores != '@' &&
-		    0 > convert_to_cpuset(&cpuset, set, RTE_DIM(set)))
-			goto err;
+		/* without '@', by default using lcore_set as cpuset */
+		if (*lcores != '@')
+			rte_memcpy(&cpuset, &lcore_set, sizeof(cpuset));
 
+		set_count = CPU_COUNT(&lcore_set);
 		/* start to update lcore_set */
 		for (idx = 0; idx < RTE_MAX_LCORE; idx++) {
-			if (!set[idx])
+			if (!CPU_ISSET(idx, &lcore_set))
 				continue;
+			set_count--;
 
 			if (cfg->lcore_role[idx] != ROLE_RTE) {
 				lcore_config[idx].core_index = count;
@@ -912,10 +902,17 @@ eal_parse_lcores(const char *lcores)
 				CPU_ZERO(&cpuset);
 				CPU_SET(idx, &cpuset);
 			}
+
+			if (check_cpuset(&cpuset) < 0)
+				goto err;
 			rte_memcpy(&lcore_config[idx].cpuset, &cpuset,
 				   sizeof(rte_cpuset_t));
 		}
 
+		/* some cores from the lcore_set can't be handled by EAL */
+		if (set_count != 0)
+			goto err;
+
 		lcores = end + 1;
 	} while (*end != '\0');
 
diff --git a/lib/librte_eal/common/eal_common_thread.c b/lib/librte_eal/common/eal_common_thread.c
index f9a8cf14d2..78581753c0 100644
--- a/lib/librte_eal/common/eal_common_thread.c
+++ b/lib/librte_eal/common/eal_common_thread.c
@@ -61,7 +61,7 @@ eal_cpuset_socket_id(rte_cpuset_t *cpusetp)
 			break;
 		}
 
-	} while (++cpu < RTE_MAX_LCORE);
+	} while (++cpu < CPU_SETSIZE);
 
 	return socket_id;
 }
@@ -118,7 +118,7 @@ eal_thread_dump_affinity(char *str, unsigned size)
 
 	rte_thread_get_affinity(&cpuset);
 
-	for (cpu = 0; cpu < RTE_MAX_LCORE; cpu++) {
+	for (cpu = 0; cpu < CPU_SETSIZE; cpu++) {
 		if (!CPU_ISSET(cpu, &cpuset))
 			continue;
 
-- 
2.23.0