[dpdk-dev] [PATCH] eal: fix ctrl thread affinity with --lcores

[David Marchand <david.marchand@redhat.com>]

When using -l/-c options, each lcore is mapped to a physical cpu in a
1:1 fashion.
On the contrary, when using --lcores, each lcore has its own cpuset on
which the associated EAL thread runs.

To handle those two situations, rely on the per lcore cpuset.

Introduced macros to manipulate cpusets in both Linux and FreeBSD.

Examples in a 4 cores FreeBSD vm:

$ ./build/app/testpmd --master-lcore 1 --lcores '0@(1,3),1@2' \
 --no-huge --no-pci -m 512 -- -i --total-num-mbufs=2048

  PID    TID COMM                TDNAME              CPU CSID CPU MASK
31733 100155 testpmd             -                     2    1 2
31733 100286 testpmd             eal-intr-thread       0    1 0
31733 100287 testpmd             rte_mp_handle         0    1 0
31733 100288 testpmd             lcore-slave-0         3    1 1,3

$ cpuset -l 1,2,3 \
 ./build/app/testpmd --master-lcore 1 --lcores '0@(1,3),1@2' \
 --no-huge --no-pci -m 512 -- -i --total-num-mbufs=2048

  PID    TID COMM                TDNAME              CPU CSID CPU MASK
31757 100139 testpmd             -                     2    2 2
31757 100292 testpmd             eal-intr-thread       2    2 2
31757 100293 testpmd             rte_mp_handle         2    2 2
31757 100294 testpmd             lcore-slave-0         3    2 1,3

$ cpuset -l 1,2,3 \
 ./build/app/testpmd --master-lcore 1 --lcores '0@1,1@2' \
 --no-huge --no-pci -m 512 -- -i --total-num-mbufs=2048

  PID    TID COMM                TDNAME              CPU CSID CPU MASK
31776 100166 testpmd             -                     2    2 2
31776 100295 testpmd             eal-intr-thread       3    2 3
31776 100296 testpmd             rte_mp_handle         3    2 3
31776 100297 testpmd             lcore-slave-0         1    2 1

Bugzilla ID: 322
Fixes: c3568ea37670 ("eal: restrict control threads to startup CPU affinity")
Cc: stable@dpdk.org

Signed-off-by: David Marchand <david.marchand@redhat.com>
---
 lib/librte_eal/common/eal_common_options.c | 16 +++++++++-------
 lib/librte_eal/common/include/rte_lcore.h  | 28 ++++++++++++++++++++++++++++
 2 files changed, 37 insertions(+), 7 deletions(-)

diff --git a/lib/librte_eal/common/eal_common_options.c b/lib/librte_eal/common/eal_common_options.c
index 24e36cf..d828271 100644
--- a/lib/librte_eal/common/eal_common_options.c
+++ b/lib/librte_eal/common/eal_common_options.c
@@ -1455,11 +1455,11 @@ compute_ctrl_threads_cpuset(struct internal_config *internal_cfg)
 	unsigned int lcore_id;
 
 	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
-		if (eal_cpu_detected(lcore_id) &&
-				rte_lcore_has_role(lcore_id, ROLE_OFF)) {
-			CPU_SET(lcore_id, cpuset);
-		}
+		if (rte_lcore_has_role(lcore_id, ROLE_OFF))
+			continue;
+		RTE_CPU_OR(cpuset, cpuset, &lcore_config[lcore_id].cpuset);
 	}
+	RTE_CPU_NOT(cpuset, cpuset);
 
 	if (pthread_getaffinity_np(pthread_self(), sizeof(rte_cpuset_t),
 				&default_set))
@@ -1467,9 +1467,11 @@ compute_ctrl_threads_cpuset(struct internal_config *internal_cfg)
 
 	RTE_CPU_AND(cpuset, cpuset, &default_set);
 
-	/* if no detected CPU is off, use master core */
-	if (!CPU_COUNT(cpuset))
-		CPU_SET(rte_get_master_lcore(), cpuset);
+	/* if no detected CPU is off, use master lcore cpuset */
+	if (!CPU_COUNT(cpuset)) {
+		memcpy(cpuset, &lcore_config[rte_get_master_lcore()].cpuset,
+			sizeof(*cpuset));
+	}
 }
 
 int
diff --git a/lib/librte_eal/common/include/rte_lcore.h b/lib/librte_eal/common/include/rte_lcore.h
index 411df30..9520d79 100644
--- a/lib/librte_eal/common/include/rte_lcore.h
+++ b/lib/librte_eal/common/include/rte_lcore.h
@@ -25,6 +25,19 @@ extern "C" {
 #if defined(__linux__)
 typedef	cpu_set_t rte_cpuset_t;
 #define RTE_CPU_AND(dst, src1, src2) CPU_AND(dst, src1, src2)
+#define RTE_CPU_OR(dst, src1, src2) CPU_OR(dst, src1, src2)
+#define RTE_CPU_FILL(set) do \
+{ \
+	unsigned int i; \
+	for (i = 0; i < CPU_SETSIZE; i++) \
+		CPU_SET(i, set); \
+} while (0)
+#define RTE_CPU_NOT(dst, src) do \
+{ \
+	cpu_set_t tmp; \
+	RTE_CPU_FILL(&tmp); \
+	CPU_XOR(dst, &tmp, src); \
+} while (0)
 #elif defined(__FreeBSD__)
 #include <pthread_np.h>
 typedef cpuset_t rte_cpuset_t;
@@ -35,6 +48,21 @@ typedef cpuset_t rte_cpuset_t;
 	CPU_AND(&tmp, src2); \
 	CPU_COPY(&tmp, dst); \
 } while (0)
+#define RTE_CPU_OR(dst, src1, src2) do \
+{ \
+	cpuset_t tmp; \
+	CPU_COPY(src1, &tmp); \
+	CPU_OR(&tmp, src2); \
+	CPU_COPY(&tmp, dst); \
+} while (0)
+#define RTE_CPU_FILL(set) CPU_FILL(set)
+#define RTE_CPU_NOT(dst, src) do \
+{ \
+	cpuset_t tmp; \
+	CPU_FILL(&tmp); \
+	CPU_NAND(&tmp, src); \
+	CPU_COPY(&tmp, dst); \
+} while (0)
 #endif
 
 /**
-- 
1.8.3.1