[dpdk-dev] [PATCH] eal: restrict ctrl threads to startup cpu affinity

[David Marchand <david.marchand@redhat.com>]

Spawning the ctrl threads on anything that is not part of the eal
coremask is not that polite to the rest of the system.

Rather than introduce yet another eal options for this, let's take
the startup cpu affinity as a reference and remove the eal coremask
from it.
If no cpu is left, then we default to the master core.

The cpuset is computed once at init before the original cpu affinity.

Fixes: d651ee4919cd ("eal: set affinity for control threads")
Signed-off-by: David Marchand <david.marchand@redhat.com>
---
 lib/librte_eal/common/eal_common_options.c | 28 ++++++++++++++++++++++++++++
 lib/librte_eal/common/eal_common_thread.c  | 21 ++++-----------------
 lib/librte_eal/common/eal_internal_cfg.h   |  3 +++
 3 files changed, 35 insertions(+), 17 deletions(-)

diff --git a/lib/librte_eal/common/eal_common_options.c b/lib/librte_eal/common/eal_common_options.c
index 6c96f45..b766252 100644
--- a/lib/librte_eal/common/eal_common_options.c
+++ b/lib/librte_eal/common/eal_common_options.c
@@ -217,6 +217,7 @@ struct device_option {
 	internal_cfg->create_uio_dev = 0;
 	internal_cfg->iova_mode = RTE_IOVA_DC;
 	internal_cfg->user_mbuf_pool_ops_name = NULL;
+	CPU_ZERO(&internal_cfg->ctrl_cpuset);
 	internal_cfg->init_complete = 0;
 }
 
@@ -1360,6 +1361,31 @@ static int xdigit2val(unsigned char c)
 	cfg->lcore_count -= removed;
 }
 
+static void
+compute_ctrl_threads_cpuset(struct internal_config *internal_cfg)
+{
+	rte_cpuset_t *cpuset = &internal_cfg->ctrl_cpuset;
+	rte_cpuset_t default_set;
+	unsigned int lcore_id;
+
+	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
+		if (eal_cpu_detected(lcore_id) &&
+				rte_lcore_has_role(lcore_id, ROLE_OFF)) {
+			CPU_SET(lcore_id, cpuset);
+		}
+	}
+
+	if (pthread_getaffinity_np(pthread_self(), sizeof(rte_cpuset_t),
+				&default_set) < 0)
+		CPU_ZERO(&default_set);
+
+	CPU_AND(cpuset, cpuset, &default_set);
+
+	/* if no detected cpu is off, use master core */
+	if (!CPU_COUNT(cpuset))
+		CPU_SET(rte_get_master_lcore(), cpuset);
+}
+
 int
 eal_cleanup_config(struct internal_config *internal_cfg)
 {
@@ -1393,6 +1419,8 @@ static int xdigit2val(unsigned char c)
 		lcore_config[cfg->master_lcore].core_role = ROLE_RTE;
 	}
 
+	compute_ctrl_threads_cpuset(internal_cfg);
+
 	/* if no memory amounts were requested, this will result in 0 and
 	 * will be overridden later, right after eal_hugepage_info_init() */
 	for (i = 0; i < RTE_MAX_NUMA_NODES; i++)
diff --git a/lib/librte_eal/common/eal_common_thread.c b/lib/librte_eal/common/eal_common_thread.c
index 48ef4d6..893b0c1 100644
--- a/lib/librte_eal/common/eal_common_thread.c
+++ b/lib/librte_eal/common/eal_common_thread.c
@@ -16,6 +16,7 @@
 #include <rte_memory.h>
 #include <rte_log.h>
 
+#include "eal_internal_cfg.h"
 #include "eal_private.h"
 #include "eal_thread.h"
 
@@ -168,10 +169,9 @@ static void *rte_thread_init(void *arg)
 		const pthread_attr_t *attr,
 		void *(*start_routine)(void *), void *arg)
 {
+	rte_cpuset_t *cpuset = &internal_config.ctrl_cpuset;
 	struct rte_thread_ctrl_params *params;
-	unsigned int lcore_id;
-	rte_cpuset_t cpuset;
-	int cpu_found, ret;
+	int ret;
 
 	params = malloc(sizeof(*params));
 	if (!params)
@@ -195,20 +195,7 @@ static void *rte_thread_init(void *arg)
 				"Cannot set name for ctrl thread\n");
 	}
 
-	cpu_found = 0;
-	CPU_ZERO(&cpuset);
-	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
-		if (eal_cpu_detected(lcore_id) &&
-				rte_lcore_has_role(lcore_id, ROLE_OFF)) {
-			CPU_SET(lcore_id, &cpuset);
-			cpu_found = 1;
-		}
-	}
-	/* if no detected cpu is off, use master core */
-	if (!cpu_found)
-		CPU_SET(rte_get_master_lcore(), &cpuset);
-
-	ret = pthread_setaffinity_np(*thread, sizeof(cpuset), &cpuset);
+	ret = pthread_setaffinity_np(*thread, sizeof(*cpuset), cpuset);
 	if (ret < 0)
 		goto fail;
 
diff --git a/lib/librte_eal/common/eal_internal_cfg.h b/lib/librte_eal/common/eal_internal_cfg.h
index 60eaead..edff09d 100644
--- a/lib/librte_eal/common/eal_internal_cfg.h
+++ b/lib/librte_eal/common/eal_internal_cfg.h
@@ -13,6 +13,8 @@
 #include <rte_eal.h>
 #include <rte_pci_dev_feature_defs.h>
 
+#include "eal_thread.h"
+
 #define MAX_HUGEPAGE_SIZES 3  /**< support up to 3 page sizes */
 
 /*
@@ -73,6 +75,7 @@ struct internal_config {
 	unsigned num_hugepage_sizes;      /**< how many sizes on this system */
 	struct hugepage_info hugepage_info[MAX_HUGEPAGE_SIZES];
 	enum rte_iova_mode iova_mode ;    /**< Set IOVA mode on this system  */
+	rte_cpuset_t ctrl_cpuset;         /**< cpuset for ctrl threads */
 	volatile unsigned int init_complete;
 	/**< indicates whether EAL has completed initialization */
 };
-- 
1.8.3.1