From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mga02.intel.com (mga02.intel.com [134.134.136.20]) by dpdk.org (Postfix) with ESMTP id 00E1D37B7 for ; Mon, 2 Feb 2015 03:03:13 +0100 (CET) Received: from fmsmga003.fm.intel.com ([10.253.24.29]) by orsmga101.jf.intel.com with ESMTP; 01 Feb 2015 18:03:12 -0800 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="4.97,862,1389772800"; d="scan'208";a="448478070" Received: from shvmail01.sh.intel.com ([10.239.29.42]) by FMSMGA003.fm.intel.com with ESMTP; 01 Feb 2015 17:49:09 -0800 Received: from shecgisg004.sh.intel.com (shecgisg004.sh.intel.com [10.239.29.89]) by shvmail01.sh.intel.com with ESMTP id t12238bN013389; Mon, 2 Feb 2015 10:03:08 +0800 Received: from shecgisg004.sh.intel.com (localhost [127.0.0.1]) by shecgisg004.sh.intel.com (8.13.6/8.13.6/SuSE Linux 0.8) with ESMTP id t12234mI013728; Mon, 2 Feb 2015 10:03:06 +0800 Received: (from cliang18@localhost) by shecgisg004.sh.intel.com (8.13.6/8.13.6/Submit) id t12234K6013724; Mon, 2 Feb 2015 10:03:04 +0800 From: Cunming Liang To: dev@dpdk.org Date: Mon, 2 Feb 2015 10:02:30 +0800 Message-Id: <1422842559-13617-9-git-send-email-cunming.liang@intel.com> X-Mailer: git-send-email 1.7.4.1 In-Reply-To: <1422842559-13617-1-git-send-email-cunming.liang@intel.com> References: <1422491072-5114-1-git-send-email-cunming.liang@intel.com> <1422842559-13617-1-git-send-email-cunming.liang@intel.com> Subject: [dpdk-dev] [PATCH v4 08/17] eal: apply affinity of EAL thread by assigned cpuset X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: patches and discussions about DPDK List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 02 Feb 2015 02:03:15 -0000 EAL threads use assigned cpuset to set core affinity during startup. It keeps 1:1 mapping, if no '--lcores' option is used. Signed-off-by: Cunming Liang --- lib/librte_eal/bsdapp/eal/eal.c | 13 ++++--- lib/librte_eal/bsdapp/eal/eal_thread.c | 63 +++++++++--------------------- lib/librte_eal/linuxapp/eal/eal.c | 7 +++- lib/librte_eal/linuxapp/eal/eal_thread.c | 67 +++++++++++--------------------- 4 files changed, 54 insertions(+), 96 deletions(-) diff --git a/lib/librte_eal/bsdapp/eal/eal.c b/lib/librte_eal/bsdapp/eal/eal.c index 69f3c03..98c5a83 100644 --- a/lib/librte_eal/bsdapp/eal/eal.c +++ b/lib/librte_eal/bsdapp/eal/eal.c @@ -432,6 +432,7 @@ rte_eal_init(int argc, char **argv) int i, fctret, ret; pthread_t thread_id; static rte_atomic32_t run_once = RTE_ATOMIC32_INIT(0); + char cpuset[CPU_STR_LEN]; if (!rte_atomic32_test_and_set(&run_once)) return -1; @@ -502,13 +503,17 @@ rte_eal_init(int argc, char **argv) if (rte_eal_pci_init() < 0) rte_panic("Cannot init PCI\n"); - RTE_LOG(DEBUG, EAL, "Master core %u is ready (tid=%p)\n", - rte_config.master_lcore, thread_id); - eal_check_mem_on_local_socket(); rte_eal_mcfg_complete(); + eal_thread_init_master(rte_config.master_lcore); + + eal_thread_dump_affinity(cpuset, CPU_STR_LEN); + + RTE_LOG(DEBUG, EAL, "Master lcore %u is ready (tid=%p;cpuset=[%s])\n", + rte_config.master_lcore, thread_id, cpuset); + if (rte_eal_dev_init() < 0) rte_panic("Cannot init pmd devices\n"); @@ -532,8 +537,6 @@ rte_eal_init(int argc, char **argv) rte_panic("Cannot create thread\n"); } - eal_thread_init_master(rte_config.master_lcore); - /* * Launch a dummy function on all slave lcores, so that master lcore * knows they are all ready when this function returns. diff --git a/lib/librte_eal/bsdapp/eal/eal_thread.c b/lib/librte_eal/bsdapp/eal/eal_thread.c index d0c077b..5b16302 100644 --- a/lib/librte_eal/bsdapp/eal/eal_thread.c +++ b/lib/librte_eal/bsdapp/eal/eal_thread.c @@ -103,55 +103,27 @@ eal_thread_set_affinity(void) { int s; pthread_t thread; - -/* - * According to the section VERSIONS of the CPU_ALLOC man page: - * - * The CPU_ZERO(), CPU_SET(), CPU_CLR(), and CPU_ISSET() macros were added - * in glibc 2.3.3. - * - * CPU_COUNT() first appeared in glibc 2.6. - * - * CPU_AND(), CPU_OR(), CPU_XOR(), CPU_EQUAL(), CPU_ALLOC(), - * CPU_ALLOC_SIZE(), CPU_FREE(), CPU_ZERO_S(), CPU_SET_S(), CPU_CLR_S(), - * CPU_ISSET_S(), CPU_AND_S(), CPU_OR_S(), CPU_XOR_S(), and CPU_EQUAL_S() - * first appeared in glibc 2.7. - */ -#if defined(CPU_ALLOC) - size_t size; - cpu_set_t *cpusetp; - - cpusetp = CPU_ALLOC(RTE_MAX_LCORE); - if (cpusetp == NULL) { - RTE_LOG(ERR, EAL, "CPU_ALLOC failed\n"); - return -1; - } - - size = CPU_ALLOC_SIZE(RTE_MAX_LCORE); - CPU_ZERO_S(size, cpusetp); - CPU_SET_S(rte_lcore_id(), size, cpusetp); + unsigned lcore_id = rte_lcore_id(); thread = pthread_self(); - s = pthread_setaffinity_np(thread, size, cpusetp); + s = pthread_setaffinity_np(thread, sizeof(cpuset_t), + &lcore_config[lcore_id].cpuset); if (s != 0) { RTE_LOG(ERR, EAL, "pthread_setaffinity_np failed\n"); - CPU_FREE(cpusetp); return -1; } - CPU_FREE(cpusetp); -#else /* CPU_ALLOC */ - cpuset_t cpuset; - CPU_ZERO( &cpuset ); - CPU_SET( rte_lcore_id(), &cpuset ); + /* acquire system unique id */ + rte_gettid(); + + /* store socket_id in TLS for quick access */ + RTE_PER_LCORE(_socket_id) = + eal_cpuset_socket_id(&lcore_config[lcore_id].cpuset); + + CPU_COPY(&lcore_config[lcore_id].cpuset, &RTE_PER_LCORE(_cpuset)); + + lcore_config[lcore_id].socket_id = RTE_PER_LCORE(_socket_id); - thread = pthread_self(); - s = pthread_setaffinity_np(thread, sizeof( cpuset ), &cpuset); - if (s != 0) { - RTE_LOG(ERR, EAL, "pthread_setaffinity_np failed\n"); - return -1; - } -#endif return 0; } @@ -174,6 +146,7 @@ eal_thread_loop(__attribute__((unused)) void *arg) unsigned lcore_id; pthread_t thread_id; int m2s, s2m; + char cpuset[CPU_STR_LEN]; thread_id = pthread_self(); @@ -185,9 +158,6 @@ eal_thread_loop(__attribute__((unused)) void *arg) if (lcore_id == RTE_MAX_LCORE) rte_panic("cannot retrieve lcore id\n"); - RTE_LOG(DEBUG, EAL, "Core %u is ready (tid=%p)\n", - lcore_id, thread_id); - m2s = lcore_config[lcore_id].pipe_master2slave[0]; s2m = lcore_config[lcore_id].pipe_slave2master[1]; @@ -198,6 +168,11 @@ eal_thread_loop(__attribute__((unused)) void *arg) if (eal_thread_set_affinity() < 0) rte_panic("cannot set affinity\n"); + eal_thread_dump_affinity(cpuset, CPU_STR_LEN); + + RTE_LOG(DEBUG, EAL, "lcore %u is ready (tid=%p;cpuset=[%s])\n", + lcore_id, thread_id, cpuset); + /* read on our pipe to get commands */ while (1) { void *fct_arg; diff --git a/lib/librte_eal/linuxapp/eal/eal.c b/lib/librte_eal/linuxapp/eal/eal.c index f99e158..c95adec 100644 --- a/lib/librte_eal/linuxapp/eal/eal.c +++ b/lib/librte_eal/linuxapp/eal/eal.c @@ -702,6 +702,7 @@ rte_eal_init(int argc, char **argv) static rte_atomic32_t run_once = RTE_ATOMIC32_INIT(0); struct shared_driver *solib = NULL; const char *logid; + char cpuset[CPU_STR_LEN]; if (!rte_atomic32_test_and_set(&run_once)) return -1; @@ -802,8 +803,10 @@ rte_eal_init(int argc, char **argv) eal_thread_init_master(rte_config.master_lcore); - RTE_LOG(DEBUG, EAL, "Master core %u is ready (tid=%x)\n", - rte_config.master_lcore, (int)thread_id); + eal_thread_dump_affinity(cpuset, CPU_STR_LEN); + + RTE_LOG(DEBUG, EAL, "Master lcore %u is ready (tid=%x;cpuset=[%s])\n", + rte_config.master_lcore, (int)thread_id, cpuset); if (rte_eal_dev_init() < 0) rte_panic("Cannot init pmd devices\n"); diff --git a/lib/librte_eal/linuxapp/eal/eal_thread.c b/lib/librte_eal/linuxapp/eal/eal_thread.c index ed20c93..6eb1525 100644 --- a/lib/librte_eal/linuxapp/eal/eal_thread.c +++ b/lib/librte_eal/linuxapp/eal/eal_thread.c @@ -52,6 +52,7 @@ #include #include #include +#include #include "eal_private.h" #include "eal_thread.h" @@ -97,61 +98,34 @@ rte_eal_remote_launch(int (*f)(void *), void *arg, unsigned slave_id) return 0; } -/* set affinity for current thread */ +/* set affinity for current EAL thread */ static int eal_thread_set_affinity(void) { int s; pthread_t thread; - -/* - * According to the section VERSIONS of the CPU_ALLOC man page: - * - * The CPU_ZERO(), CPU_SET(), CPU_CLR(), and CPU_ISSET() macros were added - * in glibc 2.3.3. - * - * CPU_COUNT() first appeared in glibc 2.6. - * - * CPU_AND(), CPU_OR(), CPU_XOR(), CPU_EQUAL(), CPU_ALLOC(), - * CPU_ALLOC_SIZE(), CPU_FREE(), CPU_ZERO_S(), CPU_SET_S(), CPU_CLR_S(), - * CPU_ISSET_S(), CPU_AND_S(), CPU_OR_S(), CPU_XOR_S(), and CPU_EQUAL_S() - * first appeared in glibc 2.7. - */ -#if defined(CPU_ALLOC) - size_t size; - cpu_set_t *cpusetp; - - cpusetp = CPU_ALLOC(RTE_MAX_LCORE); - if (cpusetp == NULL) { - RTE_LOG(ERR, EAL, "CPU_ALLOC failed\n"); - return -1; - } - - size = CPU_ALLOC_SIZE(RTE_MAX_LCORE); - CPU_ZERO_S(size, cpusetp); - CPU_SET_S(rte_lcore_id(), size, cpusetp); + unsigned lcore_id = rte_lcore_id(); thread = pthread_self(); - s = pthread_setaffinity_np(thread, size, cpusetp); + s = pthread_setaffinity_np(thread, sizeof(cpu_set_t), + &lcore_config[lcore_id].cpuset); if (s != 0) { RTE_LOG(ERR, EAL, "pthread_setaffinity_np failed\n"); - CPU_FREE(cpusetp); return -1; } - CPU_FREE(cpusetp); -#else /* CPU_ALLOC */ - cpu_set_t cpuset; - CPU_ZERO( &cpuset ); - CPU_SET( rte_lcore_id(), &cpuset ); + /* acquire system unique id */ + rte_gettid(); + + /* store socket_id in TLS for quick access */ + RTE_PER_LCORE(_socket_id) = + eal_cpuset_socket_id(&lcore_config[lcore_id].cpuset); + + rte_memcpy(&RTE_PER_LCORE(_cpuset), + &lcore_config[lcore_id].cpuset, sizeof(rte_cpuset_t)); + + lcore_config[lcore_id].socket_id = RTE_PER_LCORE(_socket_id); - thread = pthread_self(); - s = pthread_setaffinity_np(thread, sizeof( cpuset ), &cpuset); - if (s != 0) { - RTE_LOG(ERR, EAL, "pthread_setaffinity_np failed\n"); - return -1; - } -#endif return 0; } @@ -174,6 +148,7 @@ eal_thread_loop(__attribute__((unused)) void *arg) unsigned lcore_id; pthread_t thread_id; int m2s, s2m; + char cpuset[CPU_STR_LEN]; thread_id = pthread_self(); @@ -185,9 +160,6 @@ eal_thread_loop(__attribute__((unused)) void *arg) if (lcore_id == RTE_MAX_LCORE) rte_panic("cannot retrieve lcore id\n"); - RTE_LOG(DEBUG, EAL, "Core %u is ready (tid=%x)\n", - lcore_id, (int)thread_id); - m2s = lcore_config[lcore_id].pipe_master2slave[0]; s2m = lcore_config[lcore_id].pipe_slave2master[1]; @@ -198,6 +170,11 @@ eal_thread_loop(__attribute__((unused)) void *arg) if (eal_thread_set_affinity() < 0) rte_panic("cannot set affinity\n"); + eal_thread_dump_affinity(cpuset, CPU_STR_LEN); + + RTE_LOG(DEBUG, EAL, "lcore %u is ready (tid=%x;cpuset=[%s])\n", + lcore_id, (int)thread_id, cpuset); + /* read on our pipe to get commands */ while (1) { void *fct_arg; -- 1.8.1.4