From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mga14.intel.com (mga14.intel.com [192.55.52.115]) by dpdk.org (Postfix) with ESMTP id 113DF5A6D for ; Thu, 12 Feb 2015 09:17:25 +0100 (CET) Received: from orsmga003.jf.intel.com ([10.7.209.27]) by fmsmga103.fm.intel.com with ESMTP; 12 Feb 2015 00:10:16 -0800 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.09,564,1418112000"; d="scan'208";a="526437405" Received: from shvmail01.sh.intel.com ([10.239.29.42]) by orsmga003.jf.intel.com with ESMTP; 12 Feb 2015 00:09:11 -0800 Received: from shecgisg004.sh.intel.com (shecgisg004.sh.intel.com [10.239.29.89]) by shvmail01.sh.intel.com with ESMTP id t1C8HL7h012208; Thu, 12 Feb 2015 16:17:21 +0800 Received: from shecgisg004.sh.intel.com (localhost [127.0.0.1]) by shecgisg004.sh.intel.com (8.13.6/8.13.6/SuSE Linux 0.8) with ESMTP id t1C8HILk003133; Thu, 12 Feb 2015 16:17:20 +0800 Received: (from cliang18@localhost) by shecgisg004.sh.intel.com (8.13.6/8.13.6/Submit) id t1C8HIhh003129; Thu, 12 Feb 2015 16:17:18 +0800 From: Cunming Liang To: dev@dpdk.org Date: Thu, 12 Feb 2015 16:16:27 +0800 Message-Id: <1423728996-3004-11-git-send-email-cunming.liang@intel.com> X-Mailer: git-send-email 1.7.4.1 In-Reply-To: <1423728996-3004-1-git-send-email-cunming.liang@intel.com> References: <1422842559-13617-1-git-send-email-cunming.liang@intel.com> <1423728996-3004-1-git-send-email-cunming.liang@intel.com> Subject: [dpdk-dev] [PATCH v5 10/19] eal: apply affinity of EAL thread by assigned cpuset X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: patches and discussions about DPDK List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 12 Feb 2015 08:17:26 -0000 EAL threads use assigned cpuset to set core affinity during startup. It keeps 1:1 mapping, if no '--lcores' option is used. Signed-off-by: Cunming Liang --- v5 changes: add return check for dump_affinity call rte_thread_set_affinity() directly during EAL thread set lib/librte_eal/bsdapp/eal/eal.c | 10 +++-- lib/librte_eal/bsdapp/eal/eal_thread.c | 64 ++++++------------------------ lib/librte_eal/common/include/rte_lcore.h | 2 +- lib/librte_eal/linuxapp/eal/eal.c | 8 +++- lib/librte_eal/linuxapp/eal/eal_thread.c | 66 ++++++------------------------- 5 files changed, 37 insertions(+), 113 deletions(-) diff --git a/lib/librte_eal/bsdapp/eal/eal.c b/lib/librte_eal/bsdapp/eal/eal.c index cb11b5c..b66f6c6 100644 --- a/lib/librte_eal/bsdapp/eal/eal.c +++ b/lib/librte_eal/bsdapp/eal/eal.c @@ -432,6 +432,7 @@ rte_eal_init(int argc, char **argv) int i, fctret, ret; pthread_t thread_id; static rte_atomic32_t run_once = RTE_ATOMIC32_INIT(0); + char cpuset[RTE_CPU_AFFINITY_STR_LEN]; if (!rte_atomic32_test_and_set(&run_once)) return -1; @@ -502,15 +503,18 @@ rte_eal_init(int argc, char **argv) if (rte_eal_pci_init() < 0) rte_panic("Cannot init PCI\n"); - RTE_LOG(DEBUG, EAL, "Master core %u is ready (tid=%p)\n", - rte_config.master_lcore, thread_id); - eal_check_mem_on_local_socket(); rte_eal_mcfg_complete(); eal_thread_init_master(rte_config.master_lcore); + ret = eal_thread_dump_affinity(cpuset, RTE_CPU_AFFINITY_STR_LEN); + + RTE_LOG(DEBUG, EAL, "Master lcore %u is ready (tid=%p;cpuset=[%s%s])\n", + rte_config.master_lcore, thread_id, cpuset, + ret == 0 ? "" : "..."); + if (rte_eal_dev_init() < 0) rte_panic("Cannot init pmd devices\n"); diff --git a/lib/librte_eal/bsdapp/eal/eal_thread.c b/lib/librte_eal/bsdapp/eal/eal_thread.c index d0c077b..e16f685 100644 --- a/lib/librte_eal/bsdapp/eal/eal_thread.c +++ b/lib/librte_eal/bsdapp/eal/eal_thread.c @@ -101,58 +101,13 @@ rte_eal_remote_launch(int (*f)(void *), void *arg, unsigned slave_id) static int eal_thread_set_affinity(void) { - int s; - pthread_t thread; + unsigned lcore_id = rte_lcore_id(); -/* - * According to the section VERSIONS of the CPU_ALLOC man page: - * - * The CPU_ZERO(), CPU_SET(), CPU_CLR(), and CPU_ISSET() macros were added - * in glibc 2.3.3. - * - * CPU_COUNT() first appeared in glibc 2.6. - * - * CPU_AND(), CPU_OR(), CPU_XOR(), CPU_EQUAL(), CPU_ALLOC(), - * CPU_ALLOC_SIZE(), CPU_FREE(), CPU_ZERO_S(), CPU_SET_S(), CPU_CLR_S(), - * CPU_ISSET_S(), CPU_AND_S(), CPU_OR_S(), CPU_XOR_S(), and CPU_EQUAL_S() - * first appeared in glibc 2.7. - */ -#if defined(CPU_ALLOC) - size_t size; - cpu_set_t *cpusetp; - - cpusetp = CPU_ALLOC(RTE_MAX_LCORE); - if (cpusetp == NULL) { - RTE_LOG(ERR, EAL, "CPU_ALLOC failed\n"); - return -1; - } - - size = CPU_ALLOC_SIZE(RTE_MAX_LCORE); - CPU_ZERO_S(size, cpusetp); - CPU_SET_S(rte_lcore_id(), size, cpusetp); + /* acquire system unique id */ + rte_gettid(); - thread = pthread_self(); - s = pthread_setaffinity_np(thread, size, cpusetp); - if (s != 0) { - RTE_LOG(ERR, EAL, "pthread_setaffinity_np failed\n"); - CPU_FREE(cpusetp); - return -1; - } - - CPU_FREE(cpusetp); -#else /* CPU_ALLOC */ - cpuset_t cpuset; - CPU_ZERO( &cpuset ); - CPU_SET( rte_lcore_id(), &cpuset ); - - thread = pthread_self(); - s = pthread_setaffinity_np(thread, sizeof( cpuset ), &cpuset); - if (s != 0) { - RTE_LOG(ERR, EAL, "pthread_setaffinity_np failed\n"); - return -1; - } -#endif - return 0; + /* update EAL thread core affinity */ + return rte_thread_set_affinity(&lcore_config[lcore_id].cpuset); } void eal_thread_init_master(unsigned lcore_id) @@ -174,6 +129,7 @@ eal_thread_loop(__attribute__((unused)) void *arg) unsigned lcore_id; pthread_t thread_id; int m2s, s2m; + char cpuset[RTE_CPU_AFFINITY_STR_LEN]; thread_id = pthread_self(); @@ -185,9 +141,6 @@ eal_thread_loop(__attribute__((unused)) void *arg) if (lcore_id == RTE_MAX_LCORE) rte_panic("cannot retrieve lcore id\n"); - RTE_LOG(DEBUG, EAL, "Core %u is ready (tid=%p)\n", - lcore_id, thread_id); - m2s = lcore_config[lcore_id].pipe_master2slave[0]; s2m = lcore_config[lcore_id].pipe_slave2master[1]; @@ -198,6 +151,11 @@ eal_thread_loop(__attribute__((unused)) void *arg) if (eal_thread_set_affinity() < 0) rte_panic("cannot set affinity\n"); + ret = eal_thread_dump_affinity(cpuset, RTE_CPU_AFFINITY_STR_LEN); + + RTE_LOG(DEBUG, EAL, "lcore %u is ready (tid=%p;cpuset=[%s%s])\n", + lcore_id, thread_id, cpuset, ret == 0 ? "" : "..."); + /* read on our pipe to get commands */ while (1) { void *fct_arg; diff --git a/lib/librte_eal/common/include/rte_lcore.h b/lib/librte_eal/common/include/rte_lcore.h index 33f558e..6a5bcbc 100644 --- a/lib/librte_eal/common/include/rte_lcore.h +++ b/lib/librte_eal/common/include/rte_lcore.h @@ -148,7 +148,7 @@ rte_lcore_index(int lcore_id) static inline unsigned rte_socket_id(void) { - return lcore_config[rte_lcore_id()].socket_id; + return RTE_PER_LCORE(_socket_id); } /** diff --git a/lib/librte_eal/linuxapp/eal/eal.c b/lib/librte_eal/linuxapp/eal/eal.c index f99e158..137a16c 100644 --- a/lib/librte_eal/linuxapp/eal/eal.c +++ b/lib/librte_eal/linuxapp/eal/eal.c @@ -702,6 +702,7 @@ rte_eal_init(int argc, char **argv) static rte_atomic32_t run_once = RTE_ATOMIC32_INIT(0); struct shared_driver *solib = NULL; const char *logid; + char cpuset[RTE_CPU_AFFINITY_STR_LEN]; if (!rte_atomic32_test_and_set(&run_once)) return -1; @@ -802,8 +803,11 @@ rte_eal_init(int argc, char **argv) eal_thread_init_master(rte_config.master_lcore); - RTE_LOG(DEBUG, EAL, "Master core %u is ready (tid=%x)\n", - rte_config.master_lcore, (int)thread_id); + ret = eal_thread_dump_affinity(cpuset, RTE_CPU_AFFINITY_STR_LEN); + + RTE_LOG(DEBUG, EAL, "Master lcore %u is ready (tid=%x;cpuset=[%s%s])\n", + rte_config.master_lcore, (int)thread_id, cpuset, + ret == 0 ? "" : "..."); if (rte_eal_dev_init() < 0) rte_panic("Cannot init pmd devices\n"); diff --git a/lib/librte_eal/linuxapp/eal/eal_thread.c b/lib/librte_eal/linuxapp/eal/eal_thread.c index ed20c93..57b0515 100644 --- a/lib/librte_eal/linuxapp/eal/eal_thread.c +++ b/lib/librte_eal/linuxapp/eal/eal_thread.c @@ -97,62 +97,17 @@ rte_eal_remote_launch(int (*f)(void *), void *arg, unsigned slave_id) return 0; } -/* set affinity for current thread */ +/* set affinity for current EAL thread */ static int eal_thread_set_affinity(void) { - int s; - pthread_t thread; + unsigned lcore_id = rte_lcore_id(); -/* - * According to the section VERSIONS of the CPU_ALLOC man page: - * - * The CPU_ZERO(), CPU_SET(), CPU_CLR(), and CPU_ISSET() macros were added - * in glibc 2.3.3. - * - * CPU_COUNT() first appeared in glibc 2.6. - * - * CPU_AND(), CPU_OR(), CPU_XOR(), CPU_EQUAL(), CPU_ALLOC(), - * CPU_ALLOC_SIZE(), CPU_FREE(), CPU_ZERO_S(), CPU_SET_S(), CPU_CLR_S(), - * CPU_ISSET_S(), CPU_AND_S(), CPU_OR_S(), CPU_XOR_S(), and CPU_EQUAL_S() - * first appeared in glibc 2.7. - */ -#if defined(CPU_ALLOC) - size_t size; - cpu_set_t *cpusetp; - - cpusetp = CPU_ALLOC(RTE_MAX_LCORE); - if (cpusetp == NULL) { - RTE_LOG(ERR, EAL, "CPU_ALLOC failed\n"); - return -1; - } - - size = CPU_ALLOC_SIZE(RTE_MAX_LCORE); - CPU_ZERO_S(size, cpusetp); - CPU_SET_S(rte_lcore_id(), size, cpusetp); + /* acquire system unique id */ + rte_gettid(); - thread = pthread_self(); - s = pthread_setaffinity_np(thread, size, cpusetp); - if (s != 0) { - RTE_LOG(ERR, EAL, "pthread_setaffinity_np failed\n"); - CPU_FREE(cpusetp); - return -1; - } - - CPU_FREE(cpusetp); -#else /* CPU_ALLOC */ - cpu_set_t cpuset; - CPU_ZERO( &cpuset ); - CPU_SET( rte_lcore_id(), &cpuset ); - - thread = pthread_self(); - s = pthread_setaffinity_np(thread, sizeof( cpuset ), &cpuset); - if (s != 0) { - RTE_LOG(ERR, EAL, "pthread_setaffinity_np failed\n"); - return -1; - } -#endif - return 0; + /* update EAL thread core affinity */ + return rte_thread_set_affinity(&lcore_config[lcore_id].cpuset); } void eal_thread_init_master(unsigned lcore_id) @@ -174,6 +129,7 @@ eal_thread_loop(__attribute__((unused)) void *arg) unsigned lcore_id; pthread_t thread_id; int m2s, s2m; + char cpuset[RTE_CPU_AFFINITY_STR_LEN]; thread_id = pthread_self(); @@ -185,9 +141,6 @@ eal_thread_loop(__attribute__((unused)) void *arg) if (lcore_id == RTE_MAX_LCORE) rte_panic("cannot retrieve lcore id\n"); - RTE_LOG(DEBUG, EAL, "Core %u is ready (tid=%x)\n", - lcore_id, (int)thread_id); - m2s = lcore_config[lcore_id].pipe_master2slave[0]; s2m = lcore_config[lcore_id].pipe_slave2master[1]; @@ -198,6 +151,11 @@ eal_thread_loop(__attribute__((unused)) void *arg) if (eal_thread_set_affinity() < 0) rte_panic("cannot set affinity\n"); + ret = eal_thread_dump_affinity(cpuset, RTE_CPU_AFFINITY_STR_LEN); + + RTE_LOG(DEBUG, EAL, "lcore %u is ready (tid=%x;cpuset=[%s%s])\n", + lcore_id, (int)thread_id, cpuset, ret == 0 ? "" : "..."); + /* read on our pipe to get commands */ while (1) { void *fct_arg; -- 1.8.1.4