From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mail-wg0-f48.google.com (mail-wg0-f48.google.com [74.125.82.48]) by dpdk.org (Postfix) with ESMTP id A535A58D0 for ; Tue, 28 Oct 2014 13:42:34 +0100 (CET) Received: by mail-wg0-f48.google.com with SMTP id m15so766055wgh.7 for ; Tue, 28 Oct 2014 05:51:21 -0700 (PDT) X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20130820; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references; bh=3QdtHpSiek/bgSvHbqxaohlwVe4hV0ub/XEDZXRr90g=; b=kkTFnrMLLOLbBXDUF1eJJ+H2TyVVfwHy5/qBZhUOwTIR7wcFNbYMMjkV3N99AWekfO 2I0zWIGKivSuNPJrQngtR6dx1xOiUev6YBEdQzBBe/4jTdJzQucep2Xen6ONnBsYCaVY mj82jQysKNsHZqsx21zrdW1GxVOmcDUK9FIbI6PxkjeGM8vOThvsiaCMy8nsfF/hDfvS g6FlwGqhJEE3pff9p+tHS/l4CjTcIOQxioTbgQBoD77eu9oFi5Qx0kyHtklVQ3m1KWJa O54zSf+yT40OCGLtzAl1iwj6KhxiA7rADIbcDc5d87KKtegTIERi9FdlAUiz9NJuqIS7 ndig== X-Gm-Message-State: ALoCoQnT1qS9exuVa1kmRXrvA3piEPxhsZP8fnPQh9k1obWXBX5qETjldwX70UwSacAjyP44viwc X-Received: by 10.180.95.41 with SMTP id dh9mr4706837wib.0.1414500681742; Tue, 28 Oct 2014 05:51:21 -0700 (PDT) Received: from alcyon.dev.6wind.com (guy78-3-82-239-227-177.fbx.proxad.net. [82.239.227.177]) by mx.google.com with ESMTPSA id ht9sm15314425wib.8.2014.10.28.05.51.18 for (version=TLSv1.2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128); Tue, 28 Oct 2014 05:51:20 -0700 (PDT) From: David Marchand To: dev@dpdk.org Date: Tue, 28 Oct 2014 13:50:57 +0100 Message-Id: <1414500657-23774-11-git-send-email-david.marchand@6wind.com> X-Mailer: git-send-email 1.7.10.4 In-Reply-To: <1414500657-23774-1-git-send-email-david.marchand@6wind.com> References: <1414500657-23774-1-git-send-email-david.marchand@6wind.com> Cc: bjzhuc@cn.ibm.com Subject: [dpdk-dev] [PATCH v3 10/10] eal: factorize x86 headers X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: patches and discussions about DPDK List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 28 Oct 2014 12:42:35 -0000 No need to keep the same code duplicated for 32 and 64bits x86. Signed-off-by: David Marchand --- .../common/include/arch/i686/rte_atomic.h | 393 -------------------- .../common/include/arch/i686/rte_byteorder.h | 129 ------- .../common/include/arch/i686/rte_cpuflags.h | 310 --------------- .../common/include/arch/i686/rte_cycles.h | 121 ------ .../common/include/arch/i686/rte_memcpy.h | 297 --------------- .../common/include/arch/i686/rte_prefetch.h | 62 --- .../common/include/arch/i686/rte_spinlock.h | 94 ----- .../common/include/arch/x86/rte_atomic.h | 216 +++++++++++ .../common/include/arch/x86/rte_atomic_32.h | 222 +++++++++++ .../common/include/arch/x86/rte_atomic_64.h | 191 ++++++++++ .../common/include/arch/x86/rte_byteorder.h | 121 ++++++ .../common/include/arch/x86/rte_byteorder_32.h | 51 +++ .../common/include/arch/x86/rte_byteorder_64.h | 52 +++ .../common/include/arch/x86/rte_cpuflags.h | 310 +++++++++++++++ .../common/include/arch/x86/rte_cycles.h | 121 ++++++ .../common/include/arch/x86/rte_memcpy.h | 297 +++++++++++++++ .../common/include/arch/x86/rte_prefetch.h | 62 +++ .../common/include/arch/x86/rte_spinlock.h | 94 +++++ .../common/include/arch/x86_64/rte_atomic.h | 362 ------------------ .../common/include/arch/x86_64/rte_byteorder.h | 130 ------- .../common/include/arch/x86_64/rte_cpuflags.h | 310 --------------- .../common/include/arch/x86_64/rte_cycles.h | 121 ------ .../common/include/arch/x86_64/rte_memcpy.h | 297 --------------- .../common/include/arch/x86_64/rte_prefetch.h | 62 --- .../common/include/arch/x86_64/rte_spinlock.h | 94 ----- mk/arch/i686/rte.vars.mk | 2 + mk/arch/x86_64/rte.vars.mk | 2 + 27 files changed, 1741 insertions(+), 2782 deletions(-) delete mode 100644 lib/librte_eal/common/include/arch/i686/rte_atomic.h delete mode 100644 lib/librte_eal/common/include/arch/i686/rte_byteorder.h delete mode 100644 lib/librte_eal/common/include/arch/i686/rte_cpuflags.h delete mode 100644 lib/librte_eal/common/include/arch/i686/rte_cycles.h delete mode 100644 lib/librte_eal/common/include/arch/i686/rte_memcpy.h delete mode 100644 lib/librte_eal/common/include/arch/i686/rte_prefetch.h delete mode 100644 lib/librte_eal/common/include/arch/i686/rte_spinlock.h create mode 100644 lib/librte_eal/common/include/arch/x86/rte_atomic.h create mode 100644 lib/librte_eal/common/include/arch/x86/rte_atomic_32.h create mode 100644 lib/librte_eal/common/include/arch/x86/rte_atomic_64.h create mode 100644 lib/librte_eal/common/include/arch/x86/rte_byteorder.h create mode 100644 lib/librte_eal/common/include/arch/x86/rte_byteorder_32.h create mode 100644 lib/librte_eal/common/include/arch/x86/rte_byteorder_64.h create mode 100644 lib/librte_eal/common/include/arch/x86/rte_cpuflags.h create mode 100644 lib/librte_eal/common/include/arch/x86/rte_cycles.h create mode 100644 lib/librte_eal/common/include/arch/x86/rte_memcpy.h create mode 100644 lib/librte_eal/common/include/arch/x86/rte_prefetch.h create mode 100644 lib/librte_eal/common/include/arch/x86/rte_spinlock.h delete mode 100644 lib/librte_eal/common/include/arch/x86_64/rte_atomic.h delete mode 100644 lib/librte_eal/common/include/arch/x86_64/rte_byteorder.h delete mode 100644 lib/librte_eal/common/include/arch/x86_64/rte_cpuflags.h delete mode 100644 lib/librte_eal/common/include/arch/x86_64/rte_cycles.h delete mode 100644 lib/librte_eal/common/include/arch/x86_64/rte_memcpy.h delete mode 100644 lib/librte_eal/common/include/arch/x86_64/rte_prefetch.h delete mode 100644 lib/librte_eal/common/include/arch/x86_64/rte_spinlock.h diff --git a/lib/librte_eal/common/include/arch/i686/rte_atomic.h b/lib/librte_eal/common/include/arch/i686/rte_atomic.h deleted file mode 100644 index 8330250..0000000 --- a/lib/librte_eal/common/include/arch/i686/rte_atomic.h +++ /dev/null @@ -1,393 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/* - * Inspired from FreeBSD src/sys/i386/include/atomic.h - * Copyright (c) 1998 Doug Rabson - * All rights reserved. - */ - -#ifndef _RTE_ATOMIC_I686_H_ -#define _RTE_ATOMIC_I686_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include "generic/rte_atomic.h" - -#if RTE_MAX_LCORE == 1 -#define MPLOCKED /**< No need to insert MP lock prefix. */ -#else -#define MPLOCKED "lock ; " /**< Insert MP lock prefix. */ -#endif - -#define rte_mb() _mm_mfence() - -#define rte_wmb() _mm_sfence() - -#define rte_rmb() _mm_lfence() - -/*------------------------- 16 bit atomic operations -------------------------*/ - -#ifndef RTE_FORCE_INTRINSICS -static inline int -rte_atomic16_cmpset(volatile uint16_t *dst, uint16_t exp, uint16_t src) -{ - uint8_t res; - - asm volatile( - MPLOCKED - "cmpxchgw %[src], %[dst];" - "sete %[res];" - : [res] "=a" (res), /* output */ - [dst] "=m" (*dst) - : [src] "r" (src), /* input */ - "a" (exp), - "m" (*dst) - : "memory"); /* no-clobber list */ - return res; -} - -static inline int rte_atomic16_test_and_set(rte_atomic16_t *v) -{ - return rte_atomic16_cmpset((volatile uint16_t *)&v->cnt, 0, 1); -} - -static inline void -rte_atomic16_inc(rte_atomic16_t *v) -{ - asm volatile( - MPLOCKED - "incw %[cnt]" - : [cnt] "=m" (v->cnt) /* output */ - : "m" (v->cnt) /* input */ - ); -} - -static inline void -rte_atomic16_dec(rte_atomic16_t *v) -{ - asm volatile( - MPLOCKED - "decw %[cnt]" - : [cnt] "=m" (v->cnt) /* output */ - : "m" (v->cnt) /* input */ - ); -} - -static inline int rte_atomic16_inc_and_test(rte_atomic16_t *v) -{ - uint8_t ret; - - asm volatile( - MPLOCKED - "incw %[cnt] ; " - "sete %[ret]" - : [cnt] "+m" (v->cnt), /* output */ - [ret] "=qm" (ret) - ); - return (ret != 0); -} - -static inline int rte_atomic16_dec_and_test(rte_atomic16_t *v) -{ - uint8_t ret; - - asm volatile(MPLOCKED - "decw %[cnt] ; " - "sete %[ret]" - : [cnt] "+m" (v->cnt), /* output */ - [ret] "=qm" (ret) - ); - return (ret != 0); -} - -/*------------------------- 32 bit atomic operations -------------------------*/ - -static inline int -rte_atomic32_cmpset(volatile uint32_t *dst, uint32_t exp, uint32_t src) -{ - uint8_t res; - - asm volatile( - MPLOCKED - "cmpxchgl %[src], %[dst];" - "sete %[res];" - : [res] "=a" (res), /* output */ - [dst] "=m" (*dst) - : [src] "r" (src), /* input */ - "a" (exp), - "m" (*dst) - : "memory"); /* no-clobber list */ - return res; -} - -static inline int rte_atomic32_test_and_set(rte_atomic32_t *v) -{ - return rte_atomic32_cmpset((volatile uint32_t *)&v->cnt, 0, 1); -} - -static inline void -rte_atomic32_inc(rte_atomic32_t *v) -{ - asm volatile( - MPLOCKED - "incl %[cnt]" - : [cnt] "=m" (v->cnt) /* output */ - : "m" (v->cnt) /* input */ - ); -} - -static inline void -rte_atomic32_dec(rte_atomic32_t *v) -{ - asm volatile( - MPLOCKED - "decl %[cnt]" - : [cnt] "=m" (v->cnt) /* output */ - : "m" (v->cnt) /* input */ - ); -} - -static inline int rte_atomic32_inc_and_test(rte_atomic32_t *v) -{ - uint8_t ret; - - asm volatile( - MPLOCKED - "incl %[cnt] ; " - "sete %[ret]" - : [cnt] "+m" (v->cnt), /* output */ - [ret] "=qm" (ret) - ); - return (ret != 0); -} - -static inline int rte_atomic32_dec_and_test(rte_atomic32_t *v) -{ - uint8_t ret; - - asm volatile(MPLOCKED - "decl %[cnt] ; " - "sete %[ret]" - : [cnt] "+m" (v->cnt), /* output */ - [ret] "=qm" (ret) - ); - return (ret != 0); -} - -/*------------------------- 64 bit atomic operations -------------------------*/ - -static inline int -rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src) -{ - uint8_t res; - union { - struct { - uint32_t l32; - uint32_t h32; - }; - uint64_t u64; - } _exp, _src; - - _exp.u64 = exp; - _src.u64 = src; - -#ifndef __PIC__ - asm volatile ( - MPLOCKED - "cmpxchg8b (%[dst]);" - "setz %[res];" - : [res] "=a" (res) /* result in eax */ - : [dst] "S" (dst), /* esi */ - "b" (_src.l32), /* ebx */ - "c" (_src.h32), /* ecx */ - "a" (_exp.l32), /* eax */ - "d" (_exp.h32) /* edx */ - : "memory" ); /* no-clobber list */ -#else - asm volatile ( - "mov %%ebx, %%edi\n" - MPLOCKED - "cmpxchg8b (%[dst]);" - "setz %[res];" - "xchgl %%ebx, %%edi;\n" - : [res] "=a" (res) /* result in eax */ - : [dst] "S" (dst), /* esi */ - "D" (_src.l32), /* ebx */ - "c" (_src.h32), /* ecx */ - "a" (_exp.l32), /* eax */ - "d" (_exp.h32) /* edx */ - : "memory" ); /* no-clobber list */ -#endif - - return res; -} - -static inline void -rte_atomic64_init(rte_atomic64_t *v) -{ - int success = 0; - uint64_t tmp; - - while (success == 0) { - tmp = v->cnt; - success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt, - tmp, 0); - } -} - -static inline int64_t -rte_atomic64_read(rte_atomic64_t *v) -{ - int success = 0; - uint64_t tmp; - - while (success == 0) { - tmp = v->cnt; - /* replace the value by itself */ - success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt, - tmp, tmp); - } - return tmp; -} - -static inline void -rte_atomic64_set(rte_atomic64_t *v, int64_t new_value) -{ - int success = 0; - uint64_t tmp; - - while (success == 0) { - tmp = v->cnt; - success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt, - tmp, new_value); - } -} - -static inline void -rte_atomic64_add(rte_atomic64_t *v, int64_t inc) -{ - int success = 0; - uint64_t tmp; - - while (success == 0) { - tmp = v->cnt; - success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt, - tmp, tmp + inc); - } -} - -static inline void -rte_atomic64_sub(rte_atomic64_t *v, int64_t dec) -{ - int success = 0; - uint64_t tmp; - - while (success == 0) { - tmp = v->cnt; - success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt, - tmp, tmp - dec); - } -} - -static inline void -rte_atomic64_inc(rte_atomic64_t *v) -{ - rte_atomic64_add(v, 1); -} - -static inline void -rte_atomic64_dec(rte_atomic64_t *v) -{ - rte_atomic64_sub(v, 1); -} - -static inline int64_t -rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc) -{ - int success = 0; - uint64_t tmp; - - while (success == 0) { - tmp = v->cnt; - success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt, - tmp, tmp + inc); - } - - return tmp + inc; -} - -static inline int64_t -rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec) -{ - int success = 0; - uint64_t tmp; - - while (success == 0) { - tmp = v->cnt; - success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt, - tmp, tmp - dec); - } - - return tmp - dec; -} - -static inline int rte_atomic64_inc_and_test(rte_atomic64_t *v) -{ - return rte_atomic64_add_return(v, 1) == 0; -} - -static inline int rte_atomic64_dec_and_test(rte_atomic64_t *v) -{ - return rte_atomic64_sub_return(v, 1) == 0; -} - -static inline int rte_atomic64_test_and_set(rte_atomic64_t *v) -{ - return rte_atomic64_cmpset((volatile uint64_t *)&v->cnt, 0, 1); -} - -static inline void rte_atomic64_clear(rte_atomic64_t *v) -{ - rte_atomic64_set(v, 0); -} -#endif - -#ifdef __cplusplus -} -#endif - -#endif /* _RTE_ATOMIC_I686_H_ */ diff --git a/lib/librte_eal/common/include/arch/i686/rte_byteorder.h b/lib/librte_eal/common/include/arch/i686/rte_byteorder.h deleted file mode 100644 index 6d5b23e..0000000 --- a/lib/librte_eal/common/include/arch/i686/rte_byteorder.h +++ /dev/null @@ -1,129 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef _RTE_BYTEORDER_I686_H_ -#define _RTE_BYTEORDER_I686_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include "generic/rte_byteorder.h" - -/* - * An architecture-optimized byte swap for a 16-bit value. - * - * Do not use this function directly. The preferred function is rte_bswap16(). - */ -static inline uint16_t rte_arch_bswap16(uint16_t _x) -{ - register uint16_t x = _x; - asm volatile ("xchgb %b[x1],%h[x2]" - : [x1] "=Q" (x) - : [x2] "0" (x) - ); - return x; -} - -/* - * An architecture-optimized byte swap for a 32-bit value. - * - * Do not use this function directly. The preferred function is rte_bswap32(). - */ -static inline uint32_t rte_arch_bswap32(uint32_t _x) -{ - register uint32_t x = _x; - asm volatile ("bswap %[x]" - : [x] "+r" (x) - ); - return x; -} - -/* - * An architecture-optimized byte swap for a 64-bit value. - * - * Do not use this function directly. The preferred function is rte_bswap64(). - */ -/* Compat./Leg. mode */ -static inline uint64_t rte_arch_bswap64(uint64_t x) -{ - uint64_t ret = 0; - ret |= ((uint64_t)rte_arch_bswap32(x & 0xffffffffUL) << 32); - ret |= ((uint64_t)rte_arch_bswap32((x >> 32) & 0xffffffffUL)); - return ret; -} - -#ifndef RTE_FORCE_INTRINSICS -#define rte_bswap16(x) ((uint16_t)(__builtin_constant_p(x) ? \ - rte_constant_bswap16(x) : \ - rte_arch_bswap16(x))) - -#define rte_bswap32(x) ((uint32_t)(__builtin_constant_p(x) ? \ - rte_constant_bswap32(x) : \ - rte_arch_bswap32(x))) - -#define rte_bswap64(x) ((uint64_t)(__builtin_constant_p(x) ? \ - rte_constant_bswap64(x) : \ - rte_arch_bswap64(x))) -#else -/* - * __builtin_bswap16 is only available gcc 4.8 and upwards - */ -#if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 8) -#define rte_bswap16(x) ((uint16_t)(__builtin_constant_p(x) ? \ - rte_constant_bswap16(x) : \ - rte_arch_bswap16(x))) -#endif -#endif - -#define rte_cpu_to_le_16(x) (x) -#define rte_cpu_to_le_32(x) (x) -#define rte_cpu_to_le_64(x) (x) - -#define rte_cpu_to_be_16(x) rte_bswap16(x) -#define rte_cpu_to_be_32(x) rte_bswap32(x) -#define rte_cpu_to_be_64(x) rte_bswap64(x) - -#define rte_le_to_cpu_16(x) (x) -#define rte_le_to_cpu_32(x) (x) -#define rte_le_to_cpu_64(x) (x) - -#define rte_be_to_cpu_16(x) rte_bswap16(x) -#define rte_be_to_cpu_32(x) rte_bswap32(x) -#define rte_be_to_cpu_64(x) rte_bswap64(x) - -#ifdef __cplusplus -} -#endif - -#endif /* _RTE_BYTEORDER_I686_H_ */ diff --git a/lib/librte_eal/common/include/arch/i686/rte_cpuflags.h b/lib/librte_eal/common/include/arch/i686/rte_cpuflags.h deleted file mode 100644 index fd27e8f..0000000 --- a/lib/librte_eal/common/include/arch/i686/rte_cpuflags.h +++ /dev/null @@ -1,310 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef _RTE_CPUFLAGS_I686_H_ -#define _RTE_CPUFLAGS_I686_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include -#include - -#include "generic/rte_cpuflags.h" - -enum rte_cpu_flag_t { - /* (EAX 01h) ECX features*/ - RTE_CPUFLAG_SSE3 = 0, /**< SSE3 */ - RTE_CPUFLAG_PCLMULQDQ, /**< PCLMULQDQ */ - RTE_CPUFLAG_DTES64, /**< DTES64 */ - RTE_CPUFLAG_MONITOR, /**< MONITOR */ - RTE_CPUFLAG_DS_CPL, /**< DS_CPL */ - RTE_CPUFLAG_VMX, /**< VMX */ - RTE_CPUFLAG_SMX, /**< SMX */ - RTE_CPUFLAG_EIST, /**< EIST */ - RTE_CPUFLAG_TM2, /**< TM2 */ - RTE_CPUFLAG_SSSE3, /**< SSSE3 */ - RTE_CPUFLAG_CNXT_ID, /**< CNXT_ID */ - RTE_CPUFLAG_FMA, /**< FMA */ - RTE_CPUFLAG_CMPXCHG16B, /**< CMPXCHG16B */ - RTE_CPUFLAG_XTPR, /**< XTPR */ - RTE_CPUFLAG_PDCM, /**< PDCM */ - RTE_CPUFLAG_PCID, /**< PCID */ - RTE_CPUFLAG_DCA, /**< DCA */ - RTE_CPUFLAG_SSE4_1, /**< SSE4_1 */ - RTE_CPUFLAG_SSE4_2, /**< SSE4_2 */ - RTE_CPUFLAG_X2APIC, /**< X2APIC */ - RTE_CPUFLAG_MOVBE, /**< MOVBE */ - RTE_CPUFLAG_POPCNT, /**< POPCNT */ - RTE_CPUFLAG_TSC_DEADLINE, /**< TSC_DEADLINE */ - RTE_CPUFLAG_AES, /**< AES */ - RTE_CPUFLAG_XSAVE, /**< XSAVE */ - RTE_CPUFLAG_OSXSAVE, /**< OSXSAVE */ - RTE_CPUFLAG_AVX, /**< AVX */ - RTE_CPUFLAG_F16C, /**< F16C */ - RTE_CPUFLAG_RDRAND, /**< RDRAND */ - - /* (EAX 01h) EDX features */ - RTE_CPUFLAG_FPU, /**< FPU */ - RTE_CPUFLAG_VME, /**< VME */ - RTE_CPUFLAG_DE, /**< DE */ - RTE_CPUFLAG_PSE, /**< PSE */ - RTE_CPUFLAG_TSC, /**< TSC */ - RTE_CPUFLAG_MSR, /**< MSR */ - RTE_CPUFLAG_PAE, /**< PAE */ - RTE_CPUFLAG_MCE, /**< MCE */ - RTE_CPUFLAG_CX8, /**< CX8 */ - RTE_CPUFLAG_APIC, /**< APIC */ - RTE_CPUFLAG_SEP, /**< SEP */ - RTE_CPUFLAG_MTRR, /**< MTRR */ - RTE_CPUFLAG_PGE, /**< PGE */ - RTE_CPUFLAG_MCA, /**< MCA */ - RTE_CPUFLAG_CMOV, /**< CMOV */ - RTE_CPUFLAG_PAT, /**< PAT */ - RTE_CPUFLAG_PSE36, /**< PSE36 */ - RTE_CPUFLAG_PSN, /**< PSN */ - RTE_CPUFLAG_CLFSH, /**< CLFSH */ - RTE_CPUFLAG_DS, /**< DS */ - RTE_CPUFLAG_ACPI, /**< ACPI */ - RTE_CPUFLAG_MMX, /**< MMX */ - RTE_CPUFLAG_FXSR, /**< FXSR */ - RTE_CPUFLAG_SSE, /**< SSE */ - RTE_CPUFLAG_SSE2, /**< SSE2 */ - RTE_CPUFLAG_SS, /**< SS */ - RTE_CPUFLAG_HTT, /**< HTT */ - RTE_CPUFLAG_TM, /**< TM */ - RTE_CPUFLAG_PBE, /**< PBE */ - - /* (EAX 06h) EAX features */ - RTE_CPUFLAG_DIGTEMP, /**< DIGTEMP */ - RTE_CPUFLAG_TRBOBST, /**< TRBOBST */ - RTE_CPUFLAG_ARAT, /**< ARAT */ - RTE_CPUFLAG_PLN, /**< PLN */ - RTE_CPUFLAG_ECMD, /**< ECMD */ - RTE_CPUFLAG_PTM, /**< PTM */ - - /* (EAX 06h) ECX features */ - RTE_CPUFLAG_MPERF_APERF_MSR, /**< MPERF_APERF_MSR */ - RTE_CPUFLAG_ACNT2, /**< ACNT2 */ - RTE_CPUFLAG_ENERGY_EFF, /**< ENERGY_EFF */ - - /* (EAX 07h, ECX 0h) EBX features */ - RTE_CPUFLAG_FSGSBASE, /**< FSGSBASE */ - RTE_CPUFLAG_BMI1, /**< BMI1 */ - RTE_CPUFLAG_HLE, /**< Hardware Lock elision */ - RTE_CPUFLAG_AVX2, /**< AVX2 */ - RTE_CPUFLAG_SMEP, /**< SMEP */ - RTE_CPUFLAG_BMI2, /**< BMI2 */ - RTE_CPUFLAG_ERMS, /**< ERMS */ - RTE_CPUFLAG_INVPCID, /**< INVPCID */ - RTE_CPUFLAG_RTM, /**< Transactional memory */ - - /* (EAX 80000001h) ECX features */ - RTE_CPUFLAG_LAHF_SAHF, /**< LAHF_SAHF */ - RTE_CPUFLAG_LZCNT, /**< LZCNT */ - - /* (EAX 80000001h) EDX features */ - RTE_CPUFLAG_SYSCALL, /**< SYSCALL */ - RTE_CPUFLAG_XD, /**< XD */ - RTE_CPUFLAG_1GB_PG, /**< 1GB_PG */ - RTE_CPUFLAG_RDTSCP, /**< RDTSCP */ - RTE_CPUFLAG_EM64T, /**< EM64T */ - - /* (EAX 80000007h) EDX features */ - RTE_CPUFLAG_INVTSC, /**< INVTSC */ - - /* The last item */ - RTE_CPUFLAG_NUMFLAGS, /**< This should always be the last! */ -}; - -enum cpu_register_t { - REG_EAX = 0, - REG_EBX, - REG_ECX, - REG_EDX, -}; - -static const struct feature_entry cpu_feature_table[] = { - FEAT_DEF(SSE3, 0x00000001, 0, REG_ECX, 0) - FEAT_DEF(PCLMULQDQ, 0x00000001, 0, REG_ECX, 1) - FEAT_DEF(DTES64, 0x00000001, 0, REG_ECX, 2) - FEAT_DEF(MONITOR, 0x00000001, 0, REG_ECX, 3) - FEAT_DEF(DS_CPL, 0x00000001, 0, REG_ECX, 4) - FEAT_DEF(VMX, 0x00000001, 0, REG_ECX, 5) - FEAT_DEF(SMX, 0x00000001, 0, REG_ECX, 6) - FEAT_DEF(EIST, 0x00000001, 0, REG_ECX, 7) - FEAT_DEF(TM2, 0x00000001, 0, REG_ECX, 8) - FEAT_DEF(SSSE3, 0x00000001, 0, REG_ECX, 9) - FEAT_DEF(CNXT_ID, 0x00000001, 0, REG_ECX, 10) - FEAT_DEF(FMA, 0x00000001, 0, REG_ECX, 12) - FEAT_DEF(CMPXCHG16B, 0x00000001, 0, REG_ECX, 13) - FEAT_DEF(XTPR, 0x00000001, 0, REG_ECX, 14) - FEAT_DEF(PDCM, 0x00000001, 0, REG_ECX, 15) - FEAT_DEF(PCID, 0x00000001, 0, REG_ECX, 17) - FEAT_DEF(DCA, 0x00000001, 0, REG_ECX, 18) - FEAT_DEF(SSE4_1, 0x00000001, 0, REG_ECX, 19) - FEAT_DEF(SSE4_2, 0x00000001, 0, REG_ECX, 20) - FEAT_DEF(X2APIC, 0x00000001, 0, REG_ECX, 21) - FEAT_DEF(MOVBE, 0x00000001, 0, REG_ECX, 22) - FEAT_DEF(POPCNT, 0x00000001, 0, REG_ECX, 23) - FEAT_DEF(TSC_DEADLINE, 0x00000001, 0, REG_ECX, 24) - FEAT_DEF(AES, 0x00000001, 0, REG_ECX, 25) - FEAT_DEF(XSAVE, 0x00000001, 0, REG_ECX, 26) - FEAT_DEF(OSXSAVE, 0x00000001, 0, REG_ECX, 27) - FEAT_DEF(AVX, 0x00000001, 0, REG_ECX, 28) - FEAT_DEF(F16C, 0x00000001, 0, REG_ECX, 29) - FEAT_DEF(RDRAND, 0x00000001, 0, REG_ECX, 30) - - FEAT_DEF(FPU, 0x00000001, 0, REG_EDX, 0) - FEAT_DEF(VME, 0x00000001, 0, REG_EDX, 1) - FEAT_DEF(DE, 0x00000001, 0, REG_EDX, 2) - FEAT_DEF(PSE, 0x00000001, 0, REG_EDX, 3) - FEAT_DEF(TSC, 0x00000001, 0, REG_EDX, 4) - FEAT_DEF(MSR, 0x00000001, 0, REG_EDX, 5) - FEAT_DEF(PAE, 0x00000001, 0, REG_EDX, 6) - FEAT_DEF(MCE, 0x00000001, 0, REG_EDX, 7) - FEAT_DEF(CX8, 0x00000001, 0, REG_EDX, 8) - FEAT_DEF(APIC, 0x00000001, 0, REG_EDX, 9) - FEAT_DEF(SEP, 0x00000001, 0, REG_EDX, 11) - FEAT_DEF(MTRR, 0x00000001, 0, REG_EDX, 12) - FEAT_DEF(PGE, 0x00000001, 0, REG_EDX, 13) - FEAT_DEF(MCA, 0x00000001, 0, REG_EDX, 14) - FEAT_DEF(CMOV, 0x00000001, 0, REG_EDX, 15) - FEAT_DEF(PAT, 0x00000001, 0, REG_EDX, 16) - FEAT_DEF(PSE36, 0x00000001, 0, REG_EDX, 17) - FEAT_DEF(PSN, 0x00000001, 0, REG_EDX, 18) - FEAT_DEF(CLFSH, 0x00000001, 0, REG_EDX, 19) - FEAT_DEF(DS, 0x00000001, 0, REG_EDX, 21) - FEAT_DEF(ACPI, 0x00000001, 0, REG_EDX, 22) - FEAT_DEF(MMX, 0x00000001, 0, REG_EDX, 23) - FEAT_DEF(FXSR, 0x00000001, 0, REG_EDX, 24) - FEAT_DEF(SSE, 0x00000001, 0, REG_EDX, 25) - FEAT_DEF(SSE2, 0x00000001, 0, REG_EDX, 26) - FEAT_DEF(SS, 0x00000001, 0, REG_EDX, 27) - FEAT_DEF(HTT, 0x00000001, 0, REG_EDX, 28) - FEAT_DEF(TM, 0x00000001, 0, REG_EDX, 29) - FEAT_DEF(PBE, 0x00000001, 0, REG_EDX, 31) - - FEAT_DEF(DIGTEMP, 0x00000006, 0, REG_EAX, 0) - FEAT_DEF(TRBOBST, 0x00000006, 0, REG_EAX, 1) - FEAT_DEF(ARAT, 0x00000006, 0, REG_EAX, 2) - FEAT_DEF(PLN, 0x00000006, 0, REG_EAX, 4) - FEAT_DEF(ECMD, 0x00000006, 0, REG_EAX, 5) - FEAT_DEF(PTM, 0x00000006, 0, REG_EAX, 6) - - FEAT_DEF(MPERF_APERF_MSR, 0x00000006, 0, REG_ECX, 0) - FEAT_DEF(ACNT2, 0x00000006, 0, REG_ECX, 1) - FEAT_DEF(ENERGY_EFF, 0x00000006, 0, REG_ECX, 3) - - FEAT_DEF(FSGSBASE, 0x00000007, 0, REG_EBX, 0) - FEAT_DEF(BMI1, 0x00000007, 0, REG_EBX, 2) - FEAT_DEF(HLE, 0x00000007, 0, REG_EBX, 4) - FEAT_DEF(AVX2, 0x00000007, 0, REG_EBX, 5) - FEAT_DEF(SMEP, 0x00000007, 0, REG_EBX, 6) - FEAT_DEF(BMI2, 0x00000007, 0, REG_EBX, 7) - FEAT_DEF(ERMS, 0x00000007, 0, REG_EBX, 8) - FEAT_DEF(INVPCID, 0x00000007, 0, REG_EBX, 10) - FEAT_DEF(RTM, 0x00000007, 0, REG_EBX, 11) - - FEAT_DEF(LAHF_SAHF, 0x80000001, 0, REG_ECX, 0) - FEAT_DEF(LZCNT, 0x80000001, 0, REG_ECX, 4) - - FEAT_DEF(SYSCALL, 0x80000001, 0, REG_EDX, 11) - FEAT_DEF(XD, 0x80000001, 0, REG_EDX, 20) - FEAT_DEF(1GB_PG, 0x80000001, 0, REG_EDX, 26) - FEAT_DEF(RDTSCP, 0x80000001, 0, REG_EDX, 27) - FEAT_DEF(EM64T, 0x80000001, 0, REG_EDX, 29) - - FEAT_DEF(INVTSC, 0x80000007, 0, REG_EDX, 8) -}; - -static inline void -rte_cpu_get_features(uint32_t leaf, uint32_t subleaf, cpuid_registers_t out) -{ -#if defined(__i386__) && defined(__PIC__) - /* %ebx is a forbidden register if we compile with -fPIC or -fPIE */ - asm volatile("movl %%ebx,%0 ; cpuid ; xchgl %%ebx,%0" - : "=r" (out[REG_EBX]), - "=a" (out[REG_EAX]), - "=c" (out[REG_ECX]), - "=d" (out[REG_EDX]) - : "a" (leaf), "c" (subleaf)); -#else - - asm volatile("cpuid" - : "=a" (out[REG_EAX]), - "=b" (out[REG_EBX]), - "=c" (out[REG_ECX]), - "=d" (out[REG_EDX]) - : "a" (leaf), "c" (subleaf)); - -#endif -} - -static inline int -rte_cpu_get_flag_enabled(enum rte_cpu_flag_t feature) -{ - const struct feature_entry *feat; - cpuid_registers_t regs; - - - if (feature >= RTE_CPUFLAG_NUMFLAGS) - /* Flag does not match anything in the feature tables */ - return -ENOENT; - - feat = &cpu_feature_table[feature]; - - if (!feat->leaf) - /* This entry in the table wasn't filled out! */ - return -EFAULT; - - rte_cpu_get_features(feat->leaf & 0xffff0000, 0, regs); - if (((regs[REG_EAX] ^ feat->leaf) & 0xffff0000) || - regs[REG_EAX] < feat->leaf) - return 0; - - /* get the cpuid leaf containing the desired feature */ - rte_cpu_get_features(feat->leaf, feat->subleaf, regs); - - /* check if the feature is enabled */ - return (regs[feat->reg] >> feat->bit) & 1; -} - -#ifdef __cplusplus -} -#endif - -#endif /* _RTE_CPUFLAGS_I686_H_ */ diff --git a/lib/librte_eal/common/include/arch/i686/rte_cycles.h b/lib/librte_eal/common/include/arch/i686/rte_cycles.h deleted file mode 100644 index 6e47040..0000000 --- a/lib/librte_eal/common/include/arch/i686/rte_cycles.h +++ /dev/null @@ -1,121 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -/* BSD LICENSE - * - * Copyright(c) 2013 6WIND. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of 6WIND S.A. nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef _RTE_CYCLES_I686_H_ -#define _RTE_CYCLES_I686_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include "generic/rte_cycles.h" - -#ifdef RTE_LIBRTE_EAL_VMWARE_TSC_MAP_SUPPORT -/* Global switch to use VMWARE mapping of TSC instead of RDTSC */ -extern int rte_cycles_vmware_tsc_map; -#include -#endif - -static inline uint64_t -rte_rdtsc(void) -{ - union { - uint64_t tsc_64; - struct { - uint32_t lo_32; - uint32_t hi_32; - }; - } tsc; - -#ifdef RTE_LIBRTE_EAL_VMWARE_TSC_MAP_SUPPORT - if (unlikely(rte_cycles_vmware_tsc_map)) { - /* ecx = 0x10000 corresponds to the physical TSC for VMware */ - asm volatile("rdpmc" : - "=a" (tsc.lo_32), - "=d" (tsc.hi_32) : - "c"(0x10000)); - return tsc.tsc_64; - } -#endif - - asm volatile("rdtsc" : - "=a" (tsc.lo_32), - "=d" (tsc.hi_32)); - return tsc.tsc_64; -} - -static inline uint64_t -rte_rdtsc_precise(void) -{ - rte_mb(); - return rte_rdtsc(); -} - -static inline uint64_t -rte_get_tsc_cycles(void) { return rte_rdtsc(); } - -#ifdef __cplusplus -} -#endif - -#endif /* _RTE_CYCLES_I686_H_ */ diff --git a/lib/librte_eal/common/include/arch/i686/rte_memcpy.h b/lib/librte_eal/common/include/arch/i686/rte_memcpy.h deleted file mode 100644 index b8513f6..0000000 --- a/lib/librte_eal/common/include/arch/i686/rte_memcpy.h +++ /dev/null @@ -1,297 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef _RTE_MEMCPY_I686_H_ -#define _RTE_MEMCPY_I686_H_ - -#include -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -#include "generic/rte_memcpy.h" - -#ifdef __INTEL_COMPILER -#pragma warning(disable:593) /* Stop unused variable warning (reg_a etc). */ -#endif - -static inline void -rte_mov16(uint8_t *dst, const uint8_t *src) -{ - __m128i reg_a; - asm volatile ( - "movdqu (%[src]), %[reg_a]\n\t" - "movdqu %[reg_a], (%[dst])\n\t" - : [reg_a] "=x" (reg_a) - : [src] "r" (src), - [dst] "r"(dst) - : "memory" - ); -} - -static inline void -rte_mov32(uint8_t *dst, const uint8_t *src) -{ - __m128i reg_a, reg_b; - asm volatile ( - "movdqu (%[src]), %[reg_a]\n\t" - "movdqu 16(%[src]), %[reg_b]\n\t" - "movdqu %[reg_a], (%[dst])\n\t" - "movdqu %[reg_b], 16(%[dst])\n\t" - : [reg_a] "=x" (reg_a), - [reg_b] "=x" (reg_b) - : [src] "r" (src), - [dst] "r"(dst) - : "memory" - ); -} - -static inline void -rte_mov48(uint8_t *dst, const uint8_t *src) -{ - __m128i reg_a, reg_b, reg_c; - asm volatile ( - "movdqu (%[src]), %[reg_a]\n\t" - "movdqu 16(%[src]), %[reg_b]\n\t" - "movdqu 32(%[src]), %[reg_c]\n\t" - "movdqu %[reg_a], (%[dst])\n\t" - "movdqu %[reg_b], 16(%[dst])\n\t" - "movdqu %[reg_c], 32(%[dst])\n\t" - : [reg_a] "=x" (reg_a), - [reg_b] "=x" (reg_b), - [reg_c] "=x" (reg_c) - : [src] "r" (src), - [dst] "r"(dst) - : "memory" - ); -} - -static inline void -rte_mov64(uint8_t *dst, const uint8_t *src) -{ - __m128i reg_a, reg_b, reg_c, reg_d; - asm volatile ( - "movdqu (%[src]), %[reg_a]\n\t" - "movdqu 16(%[src]), %[reg_b]\n\t" - "movdqu 32(%[src]), %[reg_c]\n\t" - "movdqu 48(%[src]), %[reg_d]\n\t" - "movdqu %[reg_a], (%[dst])\n\t" - "movdqu %[reg_b], 16(%[dst])\n\t" - "movdqu %[reg_c], 32(%[dst])\n\t" - "movdqu %[reg_d], 48(%[dst])\n\t" - : [reg_a] "=x" (reg_a), - [reg_b] "=x" (reg_b), - [reg_c] "=x" (reg_c), - [reg_d] "=x" (reg_d) - : [src] "r" (src), - [dst] "r"(dst) - : "memory" - ); -} - -static inline void -rte_mov128(uint8_t *dst, const uint8_t *src) -{ - __m128i reg_a, reg_b, reg_c, reg_d, reg_e, reg_f, reg_g, reg_h; - asm volatile ( - "movdqu (%[src]), %[reg_a]\n\t" - "movdqu 16(%[src]), %[reg_b]\n\t" - "movdqu 32(%[src]), %[reg_c]\n\t" - "movdqu 48(%[src]), %[reg_d]\n\t" - "movdqu 64(%[src]), %[reg_e]\n\t" - "movdqu 80(%[src]), %[reg_f]\n\t" - "movdqu 96(%[src]), %[reg_g]\n\t" - "movdqu 112(%[src]), %[reg_h]\n\t" - "movdqu %[reg_a], (%[dst])\n\t" - "movdqu %[reg_b], 16(%[dst])\n\t" - "movdqu %[reg_c], 32(%[dst])\n\t" - "movdqu %[reg_d], 48(%[dst])\n\t" - "movdqu %[reg_e], 64(%[dst])\n\t" - "movdqu %[reg_f], 80(%[dst])\n\t" - "movdqu %[reg_g], 96(%[dst])\n\t" - "movdqu %[reg_h], 112(%[dst])\n\t" - : [reg_a] "=x" (reg_a), - [reg_b] "=x" (reg_b), - [reg_c] "=x" (reg_c), - [reg_d] "=x" (reg_d), - [reg_e] "=x" (reg_e), - [reg_f] "=x" (reg_f), - [reg_g] "=x" (reg_g), - [reg_h] "=x" (reg_h) - : [src] "r" (src), - [dst] "r"(dst) - : "memory" - ); -} - -#ifdef __INTEL_COMPILER -#pragma warning(enable:593) -#endif - -static inline void -rte_mov256(uint8_t *dst, const uint8_t *src) -{ - rte_mov128(dst, src); - rte_mov128(dst + 128, src + 128); -} - -#define rte_memcpy(dst, src, n) \ - ((__builtin_constant_p(n)) ? \ - memcpy((dst), (src), (n)) : \ - rte_memcpy_func((dst), (src), (n))) - -static inline void * -rte_memcpy_func(void *dst, const void *src, size_t n) -{ - void *ret = dst; - - /* We can't copy < 16 bytes using XMM registers so do it manually. */ - if (n < 16) { - if (n & 0x01) { - *(uint8_t *)dst = *(const uint8_t *)src; - dst = (uint8_t *)dst + 1; - src = (const uint8_t *)src + 1; - } - if (n & 0x02) { - *(uint16_t *)dst = *(const uint16_t *)src; - dst = (uint16_t *)dst + 1; - src = (const uint16_t *)src + 1; - } - if (n & 0x04) { - *(uint32_t *)dst = *(const uint32_t *)src; - dst = (uint32_t *)dst + 1; - src = (const uint32_t *)src + 1; - } - if (n & 0x08) { - *(uint64_t *)dst = *(const uint64_t *)src; - } - return ret; - } - - /* Special fast cases for <= 128 bytes */ - if (n <= 32) { - rte_mov16((uint8_t *)dst, (const uint8_t *)src); - rte_mov16((uint8_t *)dst - 16 + n, (const uint8_t *)src - 16 + n); - return ret; - } - - if (n <= 64) { - rte_mov32((uint8_t *)dst, (const uint8_t *)src); - rte_mov32((uint8_t *)dst - 32 + n, (const uint8_t *)src - 32 + n); - return ret; - } - - if (n <= 128) { - rte_mov64((uint8_t *)dst, (const uint8_t *)src); - rte_mov64((uint8_t *)dst - 64 + n, (const uint8_t *)src - 64 + n); - return ret; - } - - /* - * For large copies > 128 bytes. This combination of 256, 64 and 16 byte - * copies was found to be faster than doing 128 and 32 byte copies as - * well. - */ - for ( ; n >= 256; n -= 256) { - rte_mov256((uint8_t *)dst, (const uint8_t *)src); - dst = (uint8_t *)dst + 256; - src = (const uint8_t *)src + 256; - } - - /* - * We split the remaining bytes (which will be less than 256) into - * 64byte (2^6) chunks. - * Using incrementing integers in the case labels of a switch statement - * enourages the compiler to use a jump table. To get incrementing - * integers, we shift the 2 relevant bits to the LSB position to first - * get decrementing integers, and then subtract. - */ - switch (3 - (n >> 6)) { - case 0x00: - rte_mov64((uint8_t *)dst, (const uint8_t *)src); - n -= 64; - dst = (uint8_t *)dst + 64; - src = (const uint8_t *)src + 64; /* fallthrough */ - case 0x01: - rte_mov64((uint8_t *)dst, (const uint8_t *)src); - n -= 64; - dst = (uint8_t *)dst + 64; - src = (const uint8_t *)src + 64; /* fallthrough */ - case 0x02: - rte_mov64((uint8_t *)dst, (const uint8_t *)src); - n -= 64; - dst = (uint8_t *)dst + 64; - src = (const uint8_t *)src + 64; /* fallthrough */ - default: - ; - } - - /* - * We split the remaining bytes (which will be less than 64) into - * 16byte (2^4) chunks, using the same switch structure as above. - */ - switch (3 - (n >> 4)) { - case 0x00: - rte_mov16((uint8_t *)dst, (const uint8_t *)src); - n -= 16; - dst = (uint8_t *)dst + 16; - src = (const uint8_t *)src + 16; /* fallthrough */ - case 0x01: - rte_mov16((uint8_t *)dst, (const uint8_t *)src); - n -= 16; - dst = (uint8_t *)dst + 16; - src = (const uint8_t *)src + 16; /* fallthrough */ - case 0x02: - rte_mov16((uint8_t *)dst, (const uint8_t *)src); - n -= 16; - dst = (uint8_t *)dst + 16; - src = (const uint8_t *)src + 16; /* fallthrough */ - default: - ; - } - - /* Copy any remaining bytes, without going beyond end of buffers */ - if (n != 0) { - rte_mov16((uint8_t *)dst - 16 + n, (const uint8_t *)src - 16 + n); - } - return ret; -} - -#ifdef __cplusplus -} -#endif - -#endif /* _RTE_MEMCPY_I686_H_ */ diff --git a/lib/librte_eal/common/include/arch/i686/rte_prefetch.h b/lib/librte_eal/common/include/arch/i686/rte_prefetch.h deleted file mode 100644 index 5fbd98e..0000000 --- a/lib/librte_eal/common/include/arch/i686/rte_prefetch.h +++ /dev/null @@ -1,62 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef _RTE_PREFETCH_I686_H_ -#define _RTE_PREFETCH_I686_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include "generic/rte_prefetch.h" - -static inline void rte_prefetch0(volatile void *p) -{ - asm volatile ("prefetcht0 %[p]" : [p] "+m" (*(volatile char *)p)); -} - -static inline void rte_prefetch1(volatile void *p) -{ - asm volatile ("prefetcht1 %[p]" : [p] "+m" (*(volatile char *)p)); -} - -static inline void rte_prefetch2(volatile void *p) -{ - asm volatile ("prefetcht2 %[p]" : [p] "+m" (*(volatile char *)p)); -} - -#ifdef __cplusplus -} -#endif - -#endif /* _RTE_PREFETCH_I686_H_ */ diff --git a/lib/librte_eal/common/include/arch/i686/rte_spinlock.h b/lib/librte_eal/common/include/arch/i686/rte_spinlock.h deleted file mode 100644 index 60cfd4d..0000000 --- a/lib/librte_eal/common/include/arch/i686/rte_spinlock.h +++ /dev/null @@ -1,94 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef _RTE_SPINLOCK_I686_H_ -#define _RTE_SPINLOCK_I686_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include "generic/rte_spinlock.h" - -#ifndef RTE_FORCE_INTRINSICS -static inline void -rte_spinlock_lock(rte_spinlock_t *sl) -{ - int lock_val = 1; - asm volatile ( - "1:\n" - "xchg %[locked], %[lv]\n" - "test %[lv], %[lv]\n" - "jz 3f\n" - "2:\n" - "pause\n" - "cmpl $0, %[locked]\n" - "jnz 2b\n" - "jmp 1b\n" - "3:\n" - : [locked] "=m" (sl->locked), [lv] "=q" (lock_val) - : "[lv]" (lock_val) - : "memory"); -} - -static inline void -rte_spinlock_unlock (rte_spinlock_t *sl) -{ - int unlock_val = 0; - asm volatile ( - "xchg %[locked], %[ulv]\n" - : [locked] "=m" (sl->locked), [ulv] "=q" (unlock_val) - : "[ulv]" (unlock_val) - : "memory"); -} - -static inline int -rte_spinlock_trylock (rte_spinlock_t *sl) -{ - int lockval = 1; - - asm volatile ( - "xchg %[locked], %[lockval]" - : [locked] "=m" (sl->locked), [lockval] "=q" (lockval) - : "[lockval]" (lockval) - : "memory"); - - return (lockval == 0); -} -#endif - -#ifdef __cplusplus -} -#endif - -#endif /* _RTE_SPINLOCK_I686_H_ */ diff --git a/lib/librte_eal/common/include/arch/x86/rte_atomic.h b/lib/librte_eal/common/include/arch/x86/rte_atomic.h new file mode 100644 index 0000000..e93e8ee --- /dev/null +++ b/lib/librte_eal/common/include/arch/x86/rte_atomic.h @@ -0,0 +1,216 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_ATOMIC_X86_H_ +#define _RTE_ATOMIC_X86_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include "generic/rte_atomic.h" + +#if RTE_MAX_LCORE == 1 +#define MPLOCKED /**< No need to insert MP lock prefix. */ +#else +#define MPLOCKED "lock ; " /**< Insert MP lock prefix. */ +#endif + +#define rte_mb() _mm_mfence() + +#define rte_wmb() _mm_sfence() + +#define rte_rmb() _mm_lfence() + +/*------------------------- 16 bit atomic operations -------------------------*/ + +#ifndef RTE_FORCE_INTRINSICS +static inline int +rte_atomic16_cmpset(volatile uint16_t *dst, uint16_t exp, uint16_t src) +{ + uint8_t res; + + asm volatile( + MPLOCKED + "cmpxchgw %[src], %[dst];" + "sete %[res];" + : [res] "=a" (res), /* output */ + [dst] "=m" (*dst) + : [src] "r" (src), /* input */ + "a" (exp), + "m" (*dst) + : "memory"); /* no-clobber list */ + return res; +} + +static inline int rte_atomic16_test_and_set(rte_atomic16_t *v) +{ + return rte_atomic16_cmpset((volatile uint16_t *)&v->cnt, 0, 1); +} + +static inline void +rte_atomic16_inc(rte_atomic16_t *v) +{ + asm volatile( + MPLOCKED + "incw %[cnt]" + : [cnt] "=m" (v->cnt) /* output */ + : "m" (v->cnt) /* input */ + ); +} + +static inline void +rte_atomic16_dec(rte_atomic16_t *v) +{ + asm volatile( + MPLOCKED + "decw %[cnt]" + : [cnt] "=m" (v->cnt) /* output */ + : "m" (v->cnt) /* input */ + ); +} + +static inline int rte_atomic16_inc_and_test(rte_atomic16_t *v) +{ + uint8_t ret; + + asm volatile( + MPLOCKED + "incw %[cnt] ; " + "sete %[ret]" + : [cnt] "+m" (v->cnt), /* output */ + [ret] "=qm" (ret) + ); + return (ret != 0); +} + +static inline int rte_atomic16_dec_and_test(rte_atomic16_t *v) +{ + uint8_t ret; + + asm volatile(MPLOCKED + "decw %[cnt] ; " + "sete %[ret]" + : [cnt] "+m" (v->cnt), /* output */ + [ret] "=qm" (ret) + ); + return (ret != 0); +} + +/*------------------------- 32 bit atomic operations -------------------------*/ + +static inline int +rte_atomic32_cmpset(volatile uint32_t *dst, uint32_t exp, uint32_t src) +{ + uint8_t res; + + asm volatile( + MPLOCKED + "cmpxchgl %[src], %[dst];" + "sete %[res];" + : [res] "=a" (res), /* output */ + [dst] "=m" (*dst) + : [src] "r" (src), /* input */ + "a" (exp), + "m" (*dst) + : "memory"); /* no-clobber list */ + return res; +} + +static inline int rte_atomic32_test_and_set(rte_atomic32_t *v) +{ + return rte_atomic32_cmpset((volatile uint32_t *)&v->cnt, 0, 1); +} + +static inline void +rte_atomic32_inc(rte_atomic32_t *v) +{ + asm volatile( + MPLOCKED + "incl %[cnt]" + : [cnt] "=m" (v->cnt) /* output */ + : "m" (v->cnt) /* input */ + ); +} + +static inline void +rte_atomic32_dec(rte_atomic32_t *v) +{ + asm volatile( + MPLOCKED + "decl %[cnt]" + : [cnt] "=m" (v->cnt) /* output */ + : "m" (v->cnt) /* input */ + ); +} + +static inline int rte_atomic32_inc_and_test(rte_atomic32_t *v) +{ + uint8_t ret; + + asm volatile( + MPLOCKED + "incl %[cnt] ; " + "sete %[ret]" + : [cnt] "+m" (v->cnt), /* output */ + [ret] "=qm" (ret) + ); + return (ret != 0); +} + +static inline int rte_atomic32_dec_and_test(rte_atomic32_t *v) +{ + uint8_t ret; + + asm volatile(MPLOCKED + "decl %[cnt] ; " + "sete %[ret]" + : [cnt] "+m" (v->cnt), /* output */ + [ret] "=qm" (ret) + ); + return (ret != 0); +} +#endif + +#ifdef RTE_ARCH_I686 +#include "rte_atomic_32.h" +#else +#include "rte_atomic_64.h" +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_ATOMIC_X86_H_ */ diff --git a/lib/librte_eal/common/include/arch/x86/rte_atomic_32.h b/lib/librte_eal/common/include/arch/x86/rte_atomic_32.h new file mode 100644 index 0000000..400d8a9 --- /dev/null +++ b/lib/librte_eal/common/include/arch/x86/rte_atomic_32.h @@ -0,0 +1,222 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Inspired from FreeBSD src/sys/i386/include/atomic.h + * Copyright (c) 1998 Doug Rabson + * All rights reserved. + */ + +#ifndef _RTE_ATOMIC_I686_H_ +#define _RTE_ATOMIC_I686_H_ + +/*------------------------- 64 bit atomic operations -------------------------*/ + +#ifndef RTE_FORCE_INTRINSICS +static inline int +rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src) +{ + uint8_t res; + union { + struct { + uint32_t l32; + uint32_t h32; + }; + uint64_t u64; + } _exp, _src; + + _exp.u64 = exp; + _src.u64 = src; + +#ifndef __PIC__ + asm volatile ( + MPLOCKED + "cmpxchg8b (%[dst]);" + "setz %[res];" + : [res] "=a" (res) /* result in eax */ + : [dst] "S" (dst), /* esi */ + "b" (_src.l32), /* ebx */ + "c" (_src.h32), /* ecx */ + "a" (_exp.l32), /* eax */ + "d" (_exp.h32) /* edx */ + : "memory" ); /* no-clobber list */ +#else + asm volatile ( + "mov %%ebx, %%edi\n" + MPLOCKED + "cmpxchg8b (%[dst]);" + "setz %[res];" + "xchgl %%ebx, %%edi;\n" + : [res] "=a" (res) /* result in eax */ + : [dst] "S" (dst), /* esi */ + "D" (_src.l32), /* ebx */ + "c" (_src.h32), /* ecx */ + "a" (_exp.l32), /* eax */ + "d" (_exp.h32) /* edx */ + : "memory" ); /* no-clobber list */ +#endif + + return res; +} + +static inline void +rte_atomic64_init(rte_atomic64_t *v) +{ + int success = 0; + uint64_t tmp; + + while (success == 0) { + tmp = v->cnt; + success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt, + tmp, 0); + } +} + +static inline int64_t +rte_atomic64_read(rte_atomic64_t *v) +{ + int success = 0; + uint64_t tmp; + + while (success == 0) { + tmp = v->cnt; + /* replace the value by itself */ + success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt, + tmp, tmp); + } + return tmp; +} + +static inline void +rte_atomic64_set(rte_atomic64_t *v, int64_t new_value) +{ + int success = 0; + uint64_t tmp; + + while (success == 0) { + tmp = v->cnt; + success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt, + tmp, new_value); + } +} + +static inline void +rte_atomic64_add(rte_atomic64_t *v, int64_t inc) +{ + int success = 0; + uint64_t tmp; + + while (success == 0) { + tmp = v->cnt; + success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt, + tmp, tmp + inc); + } +} + +static inline void +rte_atomic64_sub(rte_atomic64_t *v, int64_t dec) +{ + int success = 0; + uint64_t tmp; + + while (success == 0) { + tmp = v->cnt; + success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt, + tmp, tmp - dec); + } +} + +static inline void +rte_atomic64_inc(rte_atomic64_t *v) +{ + rte_atomic64_add(v, 1); +} + +static inline void +rte_atomic64_dec(rte_atomic64_t *v) +{ + rte_atomic64_sub(v, 1); +} + +static inline int64_t +rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc) +{ + int success = 0; + uint64_t tmp; + + while (success == 0) { + tmp = v->cnt; + success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt, + tmp, tmp + inc); + } + + return tmp + inc; +} + +static inline int64_t +rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec) +{ + int success = 0; + uint64_t tmp; + + while (success == 0) { + tmp = v->cnt; + success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt, + tmp, tmp - dec); + } + + return tmp - dec; +} + +static inline int rte_atomic64_inc_and_test(rte_atomic64_t *v) +{ + return rte_atomic64_add_return(v, 1) == 0; +} + +static inline int rte_atomic64_dec_and_test(rte_atomic64_t *v) +{ + return rte_atomic64_sub_return(v, 1) == 0; +} + +static inline int rte_atomic64_test_and_set(rte_atomic64_t *v) +{ + return rte_atomic64_cmpset((volatile uint64_t *)&v->cnt, 0, 1); +} + +static inline void rte_atomic64_clear(rte_atomic64_t *v) +{ + rte_atomic64_set(v, 0); +} +#endif + +#endif /* _RTE_ATOMIC_I686_H_ */ diff --git a/lib/librte_eal/common/include/arch/x86/rte_atomic_64.h b/lib/librte_eal/common/include/arch/x86/rte_atomic_64.h new file mode 100644 index 0000000..4de6600 --- /dev/null +++ b/lib/librte_eal/common/include/arch/x86/rte_atomic_64.h @@ -0,0 +1,191 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Inspired from FreeBSD src/sys/amd64/include/atomic.h + * Copyright (c) 1998 Doug Rabson + * All rights reserved. + */ + +#ifndef _RTE_ATOMIC_X86_64_H_ +#define _RTE_ATOMIC_X86_64_H_ + +/*------------------------- 64 bit atomic operations -------------------------*/ + +#ifndef RTE_FORCE_INTRINSICS +static inline int +rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src) +{ + uint8_t res; + + + asm volatile( + MPLOCKED + "cmpxchgq %[src], %[dst];" + "sete %[res];" + : [res] "=a" (res), /* output */ + [dst] "=m" (*dst) + : [src] "r" (src), /* input */ + "a" (exp), + "m" (*dst) + : "memory"); /* no-clobber list */ + + return res; +} + +static inline void +rte_atomic64_init(rte_atomic64_t *v) +{ + v->cnt = 0; +} + +static inline int64_t +rte_atomic64_read(rte_atomic64_t *v) +{ + return v->cnt; +} + +static inline void +rte_atomic64_set(rte_atomic64_t *v, int64_t new_value) +{ + v->cnt = new_value; +} + +static inline void +rte_atomic64_add(rte_atomic64_t *v, int64_t inc) +{ + asm volatile( + MPLOCKED + "addq %[inc], %[cnt]" + : [cnt] "=m" (v->cnt) /* output */ + : [inc] "ir" (inc), /* input */ + "m" (v->cnt) + ); +} + +static inline void +rte_atomic64_sub(rte_atomic64_t *v, int64_t dec) +{ + asm volatile( + MPLOCKED + "subq %[dec], %[cnt]" + : [cnt] "=m" (v->cnt) /* output */ + : [dec] "ir" (dec), /* input */ + "m" (v->cnt) + ); +} + +static inline void +rte_atomic64_inc(rte_atomic64_t *v) +{ + asm volatile( + MPLOCKED + "incq %[cnt]" + : [cnt] "=m" (v->cnt) /* output */ + : "m" (v->cnt) /* input */ + ); +} + +static inline void +rte_atomic64_dec(rte_atomic64_t *v) +{ + asm volatile( + MPLOCKED + "decq %[cnt]" + : [cnt] "=m" (v->cnt) /* output */ + : "m" (v->cnt) /* input */ + ); +} + +static inline int64_t +rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc) +{ + int64_t prev = inc; + + asm volatile( + MPLOCKED + "xaddq %[prev], %[cnt]" + : [prev] "+r" (prev), /* output */ + [cnt] "=m" (v->cnt) + : "m" (v->cnt) /* input */ + ); + return prev + inc; +} + +static inline int64_t +rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec) +{ + return rte_atomic64_add_return(v, -dec); +} + +static inline int rte_atomic64_inc_and_test(rte_atomic64_t *v) +{ + uint8_t ret; + + asm volatile( + MPLOCKED + "incq %[cnt] ; " + "sete %[ret]" + : [cnt] "+m" (v->cnt), /* output */ + [ret] "=qm" (ret) + ); + + return ret != 0; +} + +static inline int rte_atomic64_dec_and_test(rte_atomic64_t *v) +{ + uint8_t ret; + + asm volatile( + MPLOCKED + "decq %[cnt] ; " + "sete %[ret]" + : [cnt] "+m" (v->cnt), /* output */ + [ret] "=qm" (ret) + ); + return ret != 0; +} + +static inline int rte_atomic64_test_and_set(rte_atomic64_t *v) +{ + return rte_atomic64_cmpset((volatile uint64_t *)&v->cnt, 0, 1); +} + +static inline void rte_atomic64_clear(rte_atomic64_t *v) +{ + v->cnt = 0; +} +#endif + +#endif /* _RTE_ATOMIC_X86_64_H_ */ diff --git a/lib/librte_eal/common/include/arch/x86/rte_byteorder.h b/lib/librte_eal/common/include/arch/x86/rte_byteorder.h new file mode 100644 index 0000000..1aa6985 --- /dev/null +++ b/lib/librte_eal/common/include/arch/x86/rte_byteorder.h @@ -0,0 +1,121 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_BYTEORDER_X86_H_ +#define _RTE_BYTEORDER_X86_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "generic/rte_byteorder.h" + +/* + * An architecture-optimized byte swap for a 16-bit value. + * + * Do not use this function directly. The preferred function is rte_bswap16(). + */ +static inline uint16_t rte_arch_bswap16(uint16_t _x) +{ + register uint16_t x = _x; + asm volatile ("xchgb %b[x1],%h[x2]" + : [x1] "=Q" (x) + : [x2] "0" (x) + ); + return x; +} + +/* + * An architecture-optimized byte swap for a 32-bit value. + * + * Do not use this function directly. The preferred function is rte_bswap32(). + */ +static inline uint32_t rte_arch_bswap32(uint32_t _x) +{ + register uint32_t x = _x; + asm volatile ("bswap %[x]" + : [x] "+r" (x) + ); + return x; +} + +#ifndef RTE_FORCE_INTRINSICS +#define rte_bswap16(x) ((uint16_t)(__builtin_constant_p(x) ? \ + rte_constant_bswap16(x) : \ + rte_arch_bswap16(x))) + +#define rte_bswap32(x) ((uint32_t)(__builtin_constant_p(x) ? \ + rte_constant_bswap32(x) : \ + rte_arch_bswap32(x))) + +#define rte_bswap64(x) ((uint64_t)(__builtin_constant_p(x) ? \ + rte_constant_bswap64(x) : \ + rte_arch_bswap64(x))) +#else +/* + * __builtin_bswap16 is only available gcc 4.8 and upwards + */ +#if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 8) +#define rte_bswap16(x) ((uint16_t)(__builtin_constant_p(x) ? \ + rte_constant_bswap16(x) : \ + rte_arch_bswap16(x))) +#endif +#endif + +#define rte_cpu_to_le_16(x) (x) +#define rte_cpu_to_le_32(x) (x) +#define rte_cpu_to_le_64(x) (x) + +#define rte_cpu_to_be_16(x) rte_bswap16(x) +#define rte_cpu_to_be_32(x) rte_bswap32(x) +#define rte_cpu_to_be_64(x) rte_bswap64(x) + +#define rte_le_to_cpu_16(x) (x) +#define rte_le_to_cpu_32(x) (x) +#define rte_le_to_cpu_64(x) (x) + +#define rte_be_to_cpu_16(x) rte_bswap16(x) +#define rte_be_to_cpu_32(x) rte_bswap32(x) +#define rte_be_to_cpu_64(x) rte_bswap64(x) + +#ifdef RTE_ARCH_I686 +#include "rte_byteorder_32.h" +#else +#include "rte_byteorder_64.h" +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_BYTEORDER_X86_H_ */ diff --git a/lib/librte_eal/common/include/arch/x86/rte_byteorder_32.h b/lib/librte_eal/common/include/arch/x86/rte_byteorder_32.h new file mode 100644 index 0000000..51c306f --- /dev/null +++ b/lib/librte_eal/common/include/arch/x86/rte_byteorder_32.h @@ -0,0 +1,51 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_BYTEORDER_I686_H_ +#define _RTE_BYTEORDER_I686_H_ + +/* + * An architecture-optimized byte swap for a 64-bit value. + * + * Do not use this function directly. The preferred function is rte_bswap64(). + */ +/* Compat./Leg. mode */ +static inline uint64_t rte_arch_bswap64(uint64_t x) +{ + uint64_t ret = 0; + ret |= ((uint64_t)rte_arch_bswap32(x & 0xffffffffUL) << 32); + ret |= ((uint64_t)rte_arch_bswap32((x >> 32) & 0xffffffffUL)); + return ret; +} + +#endif /* _RTE_BYTEORDER_I686_H_ */ diff --git a/lib/librte_eal/common/include/arch/x86/rte_byteorder_64.h b/lib/librte_eal/common/include/arch/x86/rte_byteorder_64.h new file mode 100644 index 0000000..dda572b --- /dev/null +++ b/lib/librte_eal/common/include/arch/x86/rte_byteorder_64.h @@ -0,0 +1,52 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_BYTEORDER_X86_64_H_ +#define _RTE_BYTEORDER_X86_64_H_ + +/* + * An architecture-optimized byte swap for a 64-bit value. + * + * Do not use this function directly. The preferred function is rte_bswap64(). + */ +/* 64-bit mode */ +static inline uint64_t rte_arch_bswap64(uint64_t _x) +{ + register uint64_t x = _x; + asm volatile ("bswap %[x]" + : [x] "+r" (x) + ); + return x; +} + +#endif /* _RTE_BYTEORDER_X86_64_H_ */ diff --git a/lib/librte_eal/common/include/arch/x86/rte_cpuflags.h b/lib/librte_eal/common/include/arch/x86/rte_cpuflags.h new file mode 100644 index 0000000..98906c8 --- /dev/null +++ b/lib/librte_eal/common/include/arch/x86/rte_cpuflags.h @@ -0,0 +1,310 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_CPUFLAGS_X86_64_H_ +#define _RTE_CPUFLAGS_X86_64_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include + +#include "generic/rte_cpuflags.h" + +enum rte_cpu_flag_t { + /* (EAX 01h) ECX features*/ + RTE_CPUFLAG_SSE3 = 0, /**< SSE3 */ + RTE_CPUFLAG_PCLMULQDQ, /**< PCLMULQDQ */ + RTE_CPUFLAG_DTES64, /**< DTES64 */ + RTE_CPUFLAG_MONITOR, /**< MONITOR */ + RTE_CPUFLAG_DS_CPL, /**< DS_CPL */ + RTE_CPUFLAG_VMX, /**< VMX */ + RTE_CPUFLAG_SMX, /**< SMX */ + RTE_CPUFLAG_EIST, /**< EIST */ + RTE_CPUFLAG_TM2, /**< TM2 */ + RTE_CPUFLAG_SSSE3, /**< SSSE3 */ + RTE_CPUFLAG_CNXT_ID, /**< CNXT_ID */ + RTE_CPUFLAG_FMA, /**< FMA */ + RTE_CPUFLAG_CMPXCHG16B, /**< CMPXCHG16B */ + RTE_CPUFLAG_XTPR, /**< XTPR */ + RTE_CPUFLAG_PDCM, /**< PDCM */ + RTE_CPUFLAG_PCID, /**< PCID */ + RTE_CPUFLAG_DCA, /**< DCA */ + RTE_CPUFLAG_SSE4_1, /**< SSE4_1 */ + RTE_CPUFLAG_SSE4_2, /**< SSE4_2 */ + RTE_CPUFLAG_X2APIC, /**< X2APIC */ + RTE_CPUFLAG_MOVBE, /**< MOVBE */ + RTE_CPUFLAG_POPCNT, /**< POPCNT */ + RTE_CPUFLAG_TSC_DEADLINE, /**< TSC_DEADLINE */ + RTE_CPUFLAG_AES, /**< AES */ + RTE_CPUFLAG_XSAVE, /**< XSAVE */ + RTE_CPUFLAG_OSXSAVE, /**< OSXSAVE */ + RTE_CPUFLAG_AVX, /**< AVX */ + RTE_CPUFLAG_F16C, /**< F16C */ + RTE_CPUFLAG_RDRAND, /**< RDRAND */ + + /* (EAX 01h) EDX features */ + RTE_CPUFLAG_FPU, /**< FPU */ + RTE_CPUFLAG_VME, /**< VME */ + RTE_CPUFLAG_DE, /**< DE */ + RTE_CPUFLAG_PSE, /**< PSE */ + RTE_CPUFLAG_TSC, /**< TSC */ + RTE_CPUFLAG_MSR, /**< MSR */ + RTE_CPUFLAG_PAE, /**< PAE */ + RTE_CPUFLAG_MCE, /**< MCE */ + RTE_CPUFLAG_CX8, /**< CX8 */ + RTE_CPUFLAG_APIC, /**< APIC */ + RTE_CPUFLAG_SEP, /**< SEP */ + RTE_CPUFLAG_MTRR, /**< MTRR */ + RTE_CPUFLAG_PGE, /**< PGE */ + RTE_CPUFLAG_MCA, /**< MCA */ + RTE_CPUFLAG_CMOV, /**< CMOV */ + RTE_CPUFLAG_PAT, /**< PAT */ + RTE_CPUFLAG_PSE36, /**< PSE36 */ + RTE_CPUFLAG_PSN, /**< PSN */ + RTE_CPUFLAG_CLFSH, /**< CLFSH */ + RTE_CPUFLAG_DS, /**< DS */ + RTE_CPUFLAG_ACPI, /**< ACPI */ + RTE_CPUFLAG_MMX, /**< MMX */ + RTE_CPUFLAG_FXSR, /**< FXSR */ + RTE_CPUFLAG_SSE, /**< SSE */ + RTE_CPUFLAG_SSE2, /**< SSE2 */ + RTE_CPUFLAG_SS, /**< SS */ + RTE_CPUFLAG_HTT, /**< HTT */ + RTE_CPUFLAG_TM, /**< TM */ + RTE_CPUFLAG_PBE, /**< PBE */ + + /* (EAX 06h) EAX features */ + RTE_CPUFLAG_DIGTEMP, /**< DIGTEMP */ + RTE_CPUFLAG_TRBOBST, /**< TRBOBST */ + RTE_CPUFLAG_ARAT, /**< ARAT */ + RTE_CPUFLAG_PLN, /**< PLN */ + RTE_CPUFLAG_ECMD, /**< ECMD */ + RTE_CPUFLAG_PTM, /**< PTM */ + + /* (EAX 06h) ECX features */ + RTE_CPUFLAG_MPERF_APERF_MSR, /**< MPERF_APERF_MSR */ + RTE_CPUFLAG_ACNT2, /**< ACNT2 */ + RTE_CPUFLAG_ENERGY_EFF, /**< ENERGY_EFF */ + + /* (EAX 07h, ECX 0h) EBX features */ + RTE_CPUFLAG_FSGSBASE, /**< FSGSBASE */ + RTE_CPUFLAG_BMI1, /**< BMI1 */ + RTE_CPUFLAG_HLE, /**< Hardware Lock elision */ + RTE_CPUFLAG_AVX2, /**< AVX2 */ + RTE_CPUFLAG_SMEP, /**< SMEP */ + RTE_CPUFLAG_BMI2, /**< BMI2 */ + RTE_CPUFLAG_ERMS, /**< ERMS */ + RTE_CPUFLAG_INVPCID, /**< INVPCID */ + RTE_CPUFLAG_RTM, /**< Transactional memory */ + + /* (EAX 80000001h) ECX features */ + RTE_CPUFLAG_LAHF_SAHF, /**< LAHF_SAHF */ + RTE_CPUFLAG_LZCNT, /**< LZCNT */ + + /* (EAX 80000001h) EDX features */ + RTE_CPUFLAG_SYSCALL, /**< SYSCALL */ + RTE_CPUFLAG_XD, /**< XD */ + RTE_CPUFLAG_1GB_PG, /**< 1GB_PG */ + RTE_CPUFLAG_RDTSCP, /**< RDTSCP */ + RTE_CPUFLAG_EM64T, /**< EM64T */ + + /* (EAX 80000007h) EDX features */ + RTE_CPUFLAG_INVTSC, /**< INVTSC */ + + /* The last item */ + RTE_CPUFLAG_NUMFLAGS, /**< This should always be the last! */ +}; + +enum cpu_register_t { + REG_EAX = 0, + REG_EBX, + REG_ECX, + REG_EDX, +}; + +static const struct feature_entry cpu_feature_table[] = { + FEAT_DEF(SSE3, 0x00000001, 0, REG_ECX, 0) + FEAT_DEF(PCLMULQDQ, 0x00000001, 0, REG_ECX, 1) + FEAT_DEF(DTES64, 0x00000001, 0, REG_ECX, 2) + FEAT_DEF(MONITOR, 0x00000001, 0, REG_ECX, 3) + FEAT_DEF(DS_CPL, 0x00000001, 0, REG_ECX, 4) + FEAT_DEF(VMX, 0x00000001, 0, REG_ECX, 5) + FEAT_DEF(SMX, 0x00000001, 0, REG_ECX, 6) + FEAT_DEF(EIST, 0x00000001, 0, REG_ECX, 7) + FEAT_DEF(TM2, 0x00000001, 0, REG_ECX, 8) + FEAT_DEF(SSSE3, 0x00000001, 0, REG_ECX, 9) + FEAT_DEF(CNXT_ID, 0x00000001, 0, REG_ECX, 10) + FEAT_DEF(FMA, 0x00000001, 0, REG_ECX, 12) + FEAT_DEF(CMPXCHG16B, 0x00000001, 0, REG_ECX, 13) + FEAT_DEF(XTPR, 0x00000001, 0, REG_ECX, 14) + FEAT_DEF(PDCM, 0x00000001, 0, REG_ECX, 15) + FEAT_DEF(PCID, 0x00000001, 0, REG_ECX, 17) + FEAT_DEF(DCA, 0x00000001, 0, REG_ECX, 18) + FEAT_DEF(SSE4_1, 0x00000001, 0, REG_ECX, 19) + FEAT_DEF(SSE4_2, 0x00000001, 0, REG_ECX, 20) + FEAT_DEF(X2APIC, 0x00000001, 0, REG_ECX, 21) + FEAT_DEF(MOVBE, 0x00000001, 0, REG_ECX, 22) + FEAT_DEF(POPCNT, 0x00000001, 0, REG_ECX, 23) + FEAT_DEF(TSC_DEADLINE, 0x00000001, 0, REG_ECX, 24) + FEAT_DEF(AES, 0x00000001, 0, REG_ECX, 25) + FEAT_DEF(XSAVE, 0x00000001, 0, REG_ECX, 26) + FEAT_DEF(OSXSAVE, 0x00000001, 0, REG_ECX, 27) + FEAT_DEF(AVX, 0x00000001, 0, REG_ECX, 28) + FEAT_DEF(F16C, 0x00000001, 0, REG_ECX, 29) + FEAT_DEF(RDRAND, 0x00000001, 0, REG_ECX, 30) + + FEAT_DEF(FPU, 0x00000001, 0, REG_EDX, 0) + FEAT_DEF(VME, 0x00000001, 0, REG_EDX, 1) + FEAT_DEF(DE, 0x00000001, 0, REG_EDX, 2) + FEAT_DEF(PSE, 0x00000001, 0, REG_EDX, 3) + FEAT_DEF(TSC, 0x00000001, 0, REG_EDX, 4) + FEAT_DEF(MSR, 0x00000001, 0, REG_EDX, 5) + FEAT_DEF(PAE, 0x00000001, 0, REG_EDX, 6) + FEAT_DEF(MCE, 0x00000001, 0, REG_EDX, 7) + FEAT_DEF(CX8, 0x00000001, 0, REG_EDX, 8) + FEAT_DEF(APIC, 0x00000001, 0, REG_EDX, 9) + FEAT_DEF(SEP, 0x00000001, 0, REG_EDX, 11) + FEAT_DEF(MTRR, 0x00000001, 0, REG_EDX, 12) + FEAT_DEF(PGE, 0x00000001, 0, REG_EDX, 13) + FEAT_DEF(MCA, 0x00000001, 0, REG_EDX, 14) + FEAT_DEF(CMOV, 0x00000001, 0, REG_EDX, 15) + FEAT_DEF(PAT, 0x00000001, 0, REG_EDX, 16) + FEAT_DEF(PSE36, 0x00000001, 0, REG_EDX, 17) + FEAT_DEF(PSN, 0x00000001, 0, REG_EDX, 18) + FEAT_DEF(CLFSH, 0x00000001, 0, REG_EDX, 19) + FEAT_DEF(DS, 0x00000001, 0, REG_EDX, 21) + FEAT_DEF(ACPI, 0x00000001, 0, REG_EDX, 22) + FEAT_DEF(MMX, 0x00000001, 0, REG_EDX, 23) + FEAT_DEF(FXSR, 0x00000001, 0, REG_EDX, 24) + FEAT_DEF(SSE, 0x00000001, 0, REG_EDX, 25) + FEAT_DEF(SSE2, 0x00000001, 0, REG_EDX, 26) + FEAT_DEF(SS, 0x00000001, 0, REG_EDX, 27) + FEAT_DEF(HTT, 0x00000001, 0, REG_EDX, 28) + FEAT_DEF(TM, 0x00000001, 0, REG_EDX, 29) + FEAT_DEF(PBE, 0x00000001, 0, REG_EDX, 31) + + FEAT_DEF(DIGTEMP, 0x00000006, 0, REG_EAX, 0) + FEAT_DEF(TRBOBST, 0x00000006, 0, REG_EAX, 1) + FEAT_DEF(ARAT, 0x00000006, 0, REG_EAX, 2) + FEAT_DEF(PLN, 0x00000006, 0, REG_EAX, 4) + FEAT_DEF(ECMD, 0x00000006, 0, REG_EAX, 5) + FEAT_DEF(PTM, 0x00000006, 0, REG_EAX, 6) + + FEAT_DEF(MPERF_APERF_MSR, 0x00000006, 0, REG_ECX, 0) + FEAT_DEF(ACNT2, 0x00000006, 0, REG_ECX, 1) + FEAT_DEF(ENERGY_EFF, 0x00000006, 0, REG_ECX, 3) + + FEAT_DEF(FSGSBASE, 0x00000007, 0, REG_EBX, 0) + FEAT_DEF(BMI1, 0x00000007, 0, REG_EBX, 2) + FEAT_DEF(HLE, 0x00000007, 0, REG_EBX, 4) + FEAT_DEF(AVX2, 0x00000007, 0, REG_EBX, 5) + FEAT_DEF(SMEP, 0x00000007, 0, REG_EBX, 6) + FEAT_DEF(BMI2, 0x00000007, 0, REG_EBX, 7) + FEAT_DEF(ERMS, 0x00000007, 0, REG_EBX, 8) + FEAT_DEF(INVPCID, 0x00000007, 0, REG_EBX, 10) + FEAT_DEF(RTM, 0x00000007, 0, REG_EBX, 11) + + FEAT_DEF(LAHF_SAHF, 0x80000001, 0, REG_ECX, 0) + FEAT_DEF(LZCNT, 0x80000001, 0, REG_ECX, 4) + + FEAT_DEF(SYSCALL, 0x80000001, 0, REG_EDX, 11) + FEAT_DEF(XD, 0x80000001, 0, REG_EDX, 20) + FEAT_DEF(1GB_PG, 0x80000001, 0, REG_EDX, 26) + FEAT_DEF(RDTSCP, 0x80000001, 0, REG_EDX, 27) + FEAT_DEF(EM64T, 0x80000001, 0, REG_EDX, 29) + + FEAT_DEF(INVTSC, 0x80000007, 0, REG_EDX, 8) +}; + +static inline void +rte_cpu_get_features(uint32_t leaf, uint32_t subleaf, cpuid_registers_t out) +{ +#if defined(__i386__) && defined(__PIC__) + /* %ebx is a forbidden register if we compile with -fPIC or -fPIE */ + asm volatile("movl %%ebx,%0 ; cpuid ; xchgl %%ebx,%0" + : "=r" (out[REG_EBX]), + "=a" (out[REG_EAX]), + "=c" (out[REG_ECX]), + "=d" (out[REG_EDX]) + : "a" (leaf), "c" (subleaf)); +#else + + asm volatile("cpuid" + : "=a" (out[REG_EAX]), + "=b" (out[REG_EBX]), + "=c" (out[REG_ECX]), + "=d" (out[REG_EDX]) + : "a" (leaf), "c" (subleaf)); + +#endif +} + +static inline int +rte_cpu_get_flag_enabled(enum rte_cpu_flag_t feature) +{ + const struct feature_entry *feat; + cpuid_registers_t regs; + + + if (feature >= RTE_CPUFLAG_NUMFLAGS) + /* Flag does not match anything in the feature tables */ + return -ENOENT; + + feat = &cpu_feature_table[feature]; + + if (!feat->leaf) + /* This entry in the table wasn't filled out! */ + return -EFAULT; + + rte_cpu_get_features(feat->leaf & 0xffff0000, 0, regs); + if (((regs[REG_EAX] ^ feat->leaf) & 0xffff0000) || + regs[REG_EAX] < feat->leaf) + return 0; + + /* get the cpuid leaf containing the desired feature */ + rte_cpu_get_features(feat->leaf, feat->subleaf, regs); + + /* check if the feature is enabled */ + return (regs[feat->reg] >> feat->bit) & 1; +} + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_CPUFLAGS_X86_64_H_ */ diff --git a/lib/librte_eal/common/include/arch/x86/rte_cycles.h b/lib/librte_eal/common/include/arch/x86/rte_cycles.h new file mode 100644 index 0000000..6e3c7d8 --- /dev/null +++ b/lib/librte_eal/common/include/arch/x86/rte_cycles.h @@ -0,0 +1,121 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +/* BSD LICENSE + * + * Copyright(c) 2013 6WIND. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_CYCLES_X86_64_H_ +#define _RTE_CYCLES_X86_64_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "generic/rte_cycles.h" + +#ifdef RTE_LIBRTE_EAL_VMWARE_TSC_MAP_SUPPORT +/* Global switch to use VMWARE mapping of TSC instead of RDTSC */ +extern int rte_cycles_vmware_tsc_map; +#include +#endif + +static inline uint64_t +rte_rdtsc(void) +{ + union { + uint64_t tsc_64; + struct { + uint32_t lo_32; + uint32_t hi_32; + }; + } tsc; + +#ifdef RTE_LIBRTE_EAL_VMWARE_TSC_MAP_SUPPORT + if (unlikely(rte_cycles_vmware_tsc_map)) { + /* ecx = 0x10000 corresponds to the physical TSC for VMware */ + asm volatile("rdpmc" : + "=a" (tsc.lo_32), + "=d" (tsc.hi_32) : + "c"(0x10000)); + return tsc.tsc_64; + } +#endif + + asm volatile("rdtsc" : + "=a" (tsc.lo_32), + "=d" (tsc.hi_32)); + return tsc.tsc_64; +} + +static inline uint64_t +rte_rdtsc_precise(void) +{ + rte_mb(); + return rte_rdtsc(); +} + +static inline uint64_t +rte_get_tsc_cycles(void) { return rte_rdtsc(); } + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_CYCLES_X86_64_H_ */ diff --git a/lib/librte_eal/common/include/arch/x86/rte_memcpy.h b/lib/librte_eal/common/include/arch/x86/rte_memcpy.h new file mode 100644 index 0000000..290c5cd --- /dev/null +++ b/lib/librte_eal/common/include/arch/x86/rte_memcpy.h @@ -0,0 +1,297 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_MEMCPY_X86_64_H_ +#define _RTE_MEMCPY_X86_64_H_ + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#include "generic/rte_memcpy.h" + +#ifdef __INTEL_COMPILER +#pragma warning(disable:593) /* Stop unused variable warning (reg_a etc). */ +#endif + +static inline void +rte_mov16(uint8_t *dst, const uint8_t *src) +{ + __m128i reg_a; + asm volatile ( + "movdqu (%[src]), %[reg_a]\n\t" + "movdqu %[reg_a], (%[dst])\n\t" + : [reg_a] "=x" (reg_a) + : [src] "r" (src), + [dst] "r"(dst) + : "memory" + ); +} + +static inline void +rte_mov32(uint8_t *dst, const uint8_t *src) +{ + __m128i reg_a, reg_b; + asm volatile ( + "movdqu (%[src]), %[reg_a]\n\t" + "movdqu 16(%[src]), %[reg_b]\n\t" + "movdqu %[reg_a], (%[dst])\n\t" + "movdqu %[reg_b], 16(%[dst])\n\t" + : [reg_a] "=x" (reg_a), + [reg_b] "=x" (reg_b) + : [src] "r" (src), + [dst] "r"(dst) + : "memory" + ); +} + +static inline void +rte_mov48(uint8_t *dst, const uint8_t *src) +{ + __m128i reg_a, reg_b, reg_c; + asm volatile ( + "movdqu (%[src]), %[reg_a]\n\t" + "movdqu 16(%[src]), %[reg_b]\n\t" + "movdqu 32(%[src]), %[reg_c]\n\t" + "movdqu %[reg_a], (%[dst])\n\t" + "movdqu %[reg_b], 16(%[dst])\n\t" + "movdqu %[reg_c], 32(%[dst])\n\t" + : [reg_a] "=x" (reg_a), + [reg_b] "=x" (reg_b), + [reg_c] "=x" (reg_c) + : [src] "r" (src), + [dst] "r"(dst) + : "memory" + ); +} + +static inline void +rte_mov64(uint8_t *dst, const uint8_t *src) +{ + __m128i reg_a, reg_b, reg_c, reg_d; + asm volatile ( + "movdqu (%[src]), %[reg_a]\n\t" + "movdqu 16(%[src]), %[reg_b]\n\t" + "movdqu 32(%[src]), %[reg_c]\n\t" + "movdqu 48(%[src]), %[reg_d]\n\t" + "movdqu %[reg_a], (%[dst])\n\t" + "movdqu %[reg_b], 16(%[dst])\n\t" + "movdqu %[reg_c], 32(%[dst])\n\t" + "movdqu %[reg_d], 48(%[dst])\n\t" + : [reg_a] "=x" (reg_a), + [reg_b] "=x" (reg_b), + [reg_c] "=x" (reg_c), + [reg_d] "=x" (reg_d) + : [src] "r" (src), + [dst] "r"(dst) + : "memory" + ); +} + +static inline void +rte_mov128(uint8_t *dst, const uint8_t *src) +{ + __m128i reg_a, reg_b, reg_c, reg_d, reg_e, reg_f, reg_g, reg_h; + asm volatile ( + "movdqu (%[src]), %[reg_a]\n\t" + "movdqu 16(%[src]), %[reg_b]\n\t" + "movdqu 32(%[src]), %[reg_c]\n\t" + "movdqu 48(%[src]), %[reg_d]\n\t" + "movdqu 64(%[src]), %[reg_e]\n\t" + "movdqu 80(%[src]), %[reg_f]\n\t" + "movdqu 96(%[src]), %[reg_g]\n\t" + "movdqu 112(%[src]), %[reg_h]\n\t" + "movdqu %[reg_a], (%[dst])\n\t" + "movdqu %[reg_b], 16(%[dst])\n\t" + "movdqu %[reg_c], 32(%[dst])\n\t" + "movdqu %[reg_d], 48(%[dst])\n\t" + "movdqu %[reg_e], 64(%[dst])\n\t" + "movdqu %[reg_f], 80(%[dst])\n\t" + "movdqu %[reg_g], 96(%[dst])\n\t" + "movdqu %[reg_h], 112(%[dst])\n\t" + : [reg_a] "=x" (reg_a), + [reg_b] "=x" (reg_b), + [reg_c] "=x" (reg_c), + [reg_d] "=x" (reg_d), + [reg_e] "=x" (reg_e), + [reg_f] "=x" (reg_f), + [reg_g] "=x" (reg_g), + [reg_h] "=x" (reg_h) + : [src] "r" (src), + [dst] "r"(dst) + : "memory" + ); +} + +#ifdef __INTEL_COMPILER +#pragma warning(enable:593) +#endif + +static inline void +rte_mov256(uint8_t *dst, const uint8_t *src) +{ + rte_mov128(dst, src); + rte_mov128(dst + 128, src + 128); +} + +#define rte_memcpy(dst, src, n) \ + ((__builtin_constant_p(n)) ? \ + memcpy((dst), (src), (n)) : \ + rte_memcpy_func((dst), (src), (n))) + +static inline void * +rte_memcpy_func(void *dst, const void *src, size_t n) +{ + void *ret = dst; + + /* We can't copy < 16 bytes using XMM registers so do it manually. */ + if (n < 16) { + if (n & 0x01) { + *(uint8_t *)dst = *(const uint8_t *)src; + dst = (uint8_t *)dst + 1; + src = (const uint8_t *)src + 1; + } + if (n & 0x02) { + *(uint16_t *)dst = *(const uint16_t *)src; + dst = (uint16_t *)dst + 1; + src = (const uint16_t *)src + 1; + } + if (n & 0x04) { + *(uint32_t *)dst = *(const uint32_t *)src; + dst = (uint32_t *)dst + 1; + src = (const uint32_t *)src + 1; + } + if (n & 0x08) { + *(uint64_t *)dst = *(const uint64_t *)src; + } + return ret; + } + + /* Special fast cases for <= 128 bytes */ + if (n <= 32) { + rte_mov16((uint8_t *)dst, (const uint8_t *)src); + rte_mov16((uint8_t *)dst - 16 + n, (const uint8_t *)src - 16 + n); + return ret; + } + + if (n <= 64) { + rte_mov32((uint8_t *)dst, (const uint8_t *)src); + rte_mov32((uint8_t *)dst - 32 + n, (const uint8_t *)src - 32 + n); + return ret; + } + + if (n <= 128) { + rte_mov64((uint8_t *)dst, (const uint8_t *)src); + rte_mov64((uint8_t *)dst - 64 + n, (const uint8_t *)src - 64 + n); + return ret; + } + + /* + * For large copies > 128 bytes. This combination of 256, 64 and 16 byte + * copies was found to be faster than doing 128 and 32 byte copies as + * well. + */ + for ( ; n >= 256; n -= 256) { + rte_mov256((uint8_t *)dst, (const uint8_t *)src); + dst = (uint8_t *)dst + 256; + src = (const uint8_t *)src + 256; + } + + /* + * We split the remaining bytes (which will be less than 256) into + * 64byte (2^6) chunks. + * Using incrementing integers in the case labels of a switch statement + * enourages the compiler to use a jump table. To get incrementing + * integers, we shift the 2 relevant bits to the LSB position to first + * get decrementing integers, and then subtract. + */ + switch (3 - (n >> 6)) { + case 0x00: + rte_mov64((uint8_t *)dst, (const uint8_t *)src); + n -= 64; + dst = (uint8_t *)dst + 64; + src = (const uint8_t *)src + 64; /* fallthrough */ + case 0x01: + rte_mov64((uint8_t *)dst, (const uint8_t *)src); + n -= 64; + dst = (uint8_t *)dst + 64; + src = (const uint8_t *)src + 64; /* fallthrough */ + case 0x02: + rte_mov64((uint8_t *)dst, (const uint8_t *)src); + n -= 64; + dst = (uint8_t *)dst + 64; + src = (const uint8_t *)src + 64; /* fallthrough */ + default: + ; + } + + /* + * We split the remaining bytes (which will be less than 64) into + * 16byte (2^4) chunks, using the same switch structure as above. + */ + switch (3 - (n >> 4)) { + case 0x00: + rte_mov16((uint8_t *)dst, (const uint8_t *)src); + n -= 16; + dst = (uint8_t *)dst + 16; + src = (const uint8_t *)src + 16; /* fallthrough */ + case 0x01: + rte_mov16((uint8_t *)dst, (const uint8_t *)src); + n -= 16; + dst = (uint8_t *)dst + 16; + src = (const uint8_t *)src + 16; /* fallthrough */ + case 0x02: + rte_mov16((uint8_t *)dst, (const uint8_t *)src); + n -= 16; + dst = (uint8_t *)dst + 16; + src = (const uint8_t *)src + 16; /* fallthrough */ + default: + ; + } + + /* Copy any remaining bytes, without going beyond end of buffers */ + if (n != 0) { + rte_mov16((uint8_t *)dst - 16 + n, (const uint8_t *)src - 16 + n); + } + return ret; +} + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_MEMCPY_X86_64_H_ */ diff --git a/lib/librte_eal/common/include/arch/x86/rte_prefetch.h b/lib/librte_eal/common/include/arch/x86/rte_prefetch.h new file mode 100644 index 0000000..ec2454d --- /dev/null +++ b/lib/librte_eal/common/include/arch/x86/rte_prefetch.h @@ -0,0 +1,62 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_PREFETCH_X86_64_H_ +#define _RTE_PREFETCH_X86_64_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "generic/rte_prefetch.h" + +static inline void rte_prefetch0(volatile void *p) +{ + asm volatile ("prefetcht0 %[p]" : [p] "+m" (*(volatile char *)p)); +} + +static inline void rte_prefetch1(volatile void *p) +{ + asm volatile ("prefetcht1 %[p]" : [p] "+m" (*(volatile char *)p)); +} + +static inline void rte_prefetch2(volatile void *p) +{ + asm volatile ("prefetcht2 %[p]" : [p] "+m" (*(volatile char *)p)); +} + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_PREFETCH_X86_64_H_ */ diff --git a/lib/librte_eal/common/include/arch/x86/rte_spinlock.h b/lib/librte_eal/common/include/arch/x86/rte_spinlock.h new file mode 100644 index 0000000..54fba95 --- /dev/null +++ b/lib/librte_eal/common/include/arch/x86/rte_spinlock.h @@ -0,0 +1,94 @@ +/*- + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_SPINLOCK_X86_64_H_ +#define _RTE_SPINLOCK_X86_64_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "generic/rte_spinlock.h" + +#ifndef RTE_FORCE_INTRINSICS +static inline void +rte_spinlock_lock(rte_spinlock_t *sl) +{ + int lock_val = 1; + asm volatile ( + "1:\n" + "xchg %[locked], %[lv]\n" + "test %[lv], %[lv]\n" + "jz 3f\n" + "2:\n" + "pause\n" + "cmpl $0, %[locked]\n" + "jnz 2b\n" + "jmp 1b\n" + "3:\n" + : [locked] "=m" (sl->locked), [lv] "=q" (lock_val) + : "[lv]" (lock_val) + : "memory"); +} + +static inline void +rte_spinlock_unlock (rte_spinlock_t *sl) +{ + int unlock_val = 0; + asm volatile ( + "xchg %[locked], %[ulv]\n" + : [locked] "=m" (sl->locked), [ulv] "=q" (unlock_val) + : "[ulv]" (unlock_val) + : "memory"); +} + +static inline int +rte_spinlock_trylock (rte_spinlock_t *sl) +{ + int lockval = 1; + + asm volatile ( + "xchg %[locked], %[lockval]" + : [locked] "=m" (sl->locked), [lockval] "=q" (lockval) + : "[lockval]" (lockval) + : "memory"); + + return (lockval == 0); +} +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_SPINLOCK_X86_64_H_ */ diff --git a/lib/librte_eal/common/include/arch/x86_64/rte_atomic.h b/lib/librte_eal/common/include/arch/x86_64/rte_atomic.h deleted file mode 100644 index 9138328..0000000 --- a/lib/librte_eal/common/include/arch/x86_64/rte_atomic.h +++ /dev/null @@ -1,362 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/* - * Inspired from FreeBSD src/sys/amd64/include/atomic.h - * Copyright (c) 1998 Doug Rabson - * All rights reserved. - */ - -#ifndef _RTE_ATOMIC_X86_64_H_ -#define _RTE_ATOMIC_X86_64_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include "generic/rte_atomic.h" - -#if RTE_MAX_LCORE == 1 -#define MPLOCKED /**< No need to insert MP lock prefix. */ -#else -#define MPLOCKED "lock ; " /**< Insert MP lock prefix. */ -#endif - -#define rte_mb() _mm_mfence() - -#define rte_wmb() _mm_sfence() - -#define rte_rmb() _mm_lfence() - -/*------------------------- 16 bit atomic operations -------------------------*/ - -#ifndef RTE_FORCE_INTRINSICS -static inline int -rte_atomic16_cmpset(volatile uint16_t *dst, uint16_t exp, uint16_t src) -{ - uint8_t res; - - asm volatile( - MPLOCKED - "cmpxchgw %[src], %[dst];" - "sete %[res];" - : [res] "=a" (res), /* output */ - [dst] "=m" (*dst) - : [src] "r" (src), /* input */ - "a" (exp), - "m" (*dst) - : "memory"); /* no-clobber list */ - return res; -} - -static inline int rte_atomic16_test_and_set(rte_atomic16_t *v) -{ - return rte_atomic16_cmpset((volatile uint16_t *)&v->cnt, 0, 1); -} - -static inline void -rte_atomic16_inc(rte_atomic16_t *v) -{ - asm volatile( - MPLOCKED - "incw %[cnt]" - : [cnt] "=m" (v->cnt) /* output */ - : "m" (v->cnt) /* input */ - ); -} - -static inline void -rte_atomic16_dec(rte_atomic16_t *v) -{ - asm volatile( - MPLOCKED - "decw %[cnt]" - : [cnt] "=m" (v->cnt) /* output */ - : "m" (v->cnt) /* input */ - ); -} - -static inline int rte_atomic16_inc_and_test(rte_atomic16_t *v) -{ - uint8_t ret; - - asm volatile( - MPLOCKED - "incw %[cnt] ; " - "sete %[ret]" - : [cnt] "+m" (v->cnt), /* output */ - [ret] "=qm" (ret) - ); - return (ret != 0); -} - -static inline int rte_atomic16_dec_and_test(rte_atomic16_t *v) -{ - uint8_t ret; - - asm volatile(MPLOCKED - "decw %[cnt] ; " - "sete %[ret]" - : [cnt] "+m" (v->cnt), /* output */ - [ret] "=qm" (ret) - ); - return (ret != 0); -} - -/*------------------------- 32 bit atomic operations -------------------------*/ - -static inline int -rte_atomic32_cmpset(volatile uint32_t *dst, uint32_t exp, uint32_t src) -{ - uint8_t res; - - asm volatile( - MPLOCKED - "cmpxchgl %[src], %[dst];" - "sete %[res];" - : [res] "=a" (res), /* output */ - [dst] "=m" (*dst) - : [src] "r" (src), /* input */ - "a" (exp), - "m" (*dst) - : "memory"); /* no-clobber list */ - return res; -} - -static inline int rte_atomic32_test_and_set(rte_atomic32_t *v) -{ - return rte_atomic32_cmpset((volatile uint32_t *)&v->cnt, 0, 1); -} - -static inline void -rte_atomic32_inc(rte_atomic32_t *v) -{ - asm volatile( - MPLOCKED - "incl %[cnt]" - : [cnt] "=m" (v->cnt) /* output */ - : "m" (v->cnt) /* input */ - ); -} - -static inline void -rte_atomic32_dec(rte_atomic32_t *v) -{ - asm volatile( - MPLOCKED - "decl %[cnt]" - : [cnt] "=m" (v->cnt) /* output */ - : "m" (v->cnt) /* input */ - ); -} - -static inline int rte_atomic32_inc_and_test(rte_atomic32_t *v) -{ - uint8_t ret; - - asm volatile( - MPLOCKED - "incl %[cnt] ; " - "sete %[ret]" - : [cnt] "+m" (v->cnt), /* output */ - [ret] "=qm" (ret) - ); - return (ret != 0); -} - -static inline int rte_atomic32_dec_and_test(rte_atomic32_t *v) -{ - uint8_t ret; - - asm volatile(MPLOCKED - "decl %[cnt] ; " - "sete %[ret]" - : [cnt] "+m" (v->cnt), /* output */ - [ret] "=qm" (ret) - ); - return (ret != 0); -} - -/*------------------------- 64 bit atomic operations -------------------------*/ - -static inline int -rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src) -{ - uint8_t res; - - - asm volatile( - MPLOCKED - "cmpxchgq %[src], %[dst];" - "sete %[res];" - : [res] "=a" (res), /* output */ - [dst] "=m" (*dst) - : [src] "r" (src), /* input */ - "a" (exp), - "m" (*dst) - : "memory"); /* no-clobber list */ - - return res; -} - -static inline void -rte_atomic64_init(rte_atomic64_t *v) -{ - v->cnt = 0; -} - -static inline int64_t -rte_atomic64_read(rte_atomic64_t *v) -{ - return v->cnt; -} - -static inline void -rte_atomic64_set(rte_atomic64_t *v, int64_t new_value) -{ - v->cnt = new_value; -} - -static inline void -rte_atomic64_add(rte_atomic64_t *v, int64_t inc) -{ - asm volatile( - MPLOCKED - "addq %[inc], %[cnt]" - : [cnt] "=m" (v->cnt) /* output */ - : [inc] "ir" (inc), /* input */ - "m" (v->cnt) - ); -} - -static inline void -rte_atomic64_sub(rte_atomic64_t *v, int64_t dec) -{ - asm volatile( - MPLOCKED - "subq %[dec], %[cnt]" - : [cnt] "=m" (v->cnt) /* output */ - : [dec] "ir" (dec), /* input */ - "m" (v->cnt) - ); -} - -static inline void -rte_atomic64_inc(rte_atomic64_t *v) -{ - asm volatile( - MPLOCKED - "incq %[cnt]" - : [cnt] "=m" (v->cnt) /* output */ - : "m" (v->cnt) /* input */ - ); -} - -static inline void -rte_atomic64_dec(rte_atomic64_t *v) -{ - asm volatile( - MPLOCKED - "decq %[cnt]" - : [cnt] "=m" (v->cnt) /* output */ - : "m" (v->cnt) /* input */ - ); -} - -static inline int64_t -rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc) -{ - int64_t prev = inc; - - asm volatile( - MPLOCKED - "xaddq %[prev], %[cnt]" - : [prev] "+r" (prev), /* output */ - [cnt] "=m" (v->cnt) - : "m" (v->cnt) /* input */ - ); - return prev + inc; -} - -static inline int64_t -rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec) -{ - return rte_atomic64_add_return(v, -dec); -} - -static inline int rte_atomic64_inc_and_test(rte_atomic64_t *v) -{ - uint8_t ret; - - asm volatile( - MPLOCKED - "incq %[cnt] ; " - "sete %[ret]" - : [cnt] "+m" (v->cnt), /* output */ - [ret] "=qm" (ret) - ); - - return ret != 0; -} - -static inline int rte_atomic64_dec_and_test(rte_atomic64_t *v) -{ - uint8_t ret; - - asm volatile( - MPLOCKED - "decq %[cnt] ; " - "sete %[ret]" - : [cnt] "+m" (v->cnt), /* output */ - [ret] "=qm" (ret) - ); - return ret != 0; -} - -static inline int rte_atomic64_test_and_set(rte_atomic64_t *v) -{ - return rte_atomic64_cmpset((volatile uint64_t *)&v->cnt, 0, 1); -} - -static inline void rte_atomic64_clear(rte_atomic64_t *v) -{ - v->cnt = 0; -} -#endif - -#ifdef __cplusplus -} -#endif - -#endif /* _RTE_ATOMIC_X86_64_H_ */ diff --git a/lib/librte_eal/common/include/arch/x86_64/rte_byteorder.h b/lib/librte_eal/common/include/arch/x86_64/rte_byteorder.h deleted file mode 100644 index 825e576..0000000 --- a/lib/librte_eal/common/include/arch/x86_64/rte_byteorder.h +++ /dev/null @@ -1,130 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef _RTE_BYTEORDER_X86_64_H_ -#define _RTE_BYTEORDER_X86_64_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include "generic/rte_byteorder.h" - -/* - * An architecture-optimized byte swap for a 16-bit value. - * - * Do not use this function directly. The preferred function is rte_bswap16(). - */ -static inline uint16_t rte_arch_bswap16(uint16_t _x) -{ - register uint16_t x = _x; - asm volatile ("xchgb %b[x1],%h[x2]" - : [x1] "=Q" (x) - : [x2] "0" (x) - ); - return x; -} - -/* - * An architecture-optimized byte swap for a 32-bit value. - * - * Do not use this function directly. The preferred function is rte_bswap32(). - */ -static inline uint32_t rte_arch_bswap32(uint32_t _x) -{ - register uint32_t x = _x; - asm volatile ("bswap %[x]" - : [x] "+r" (x) - ); - return x; -} - -/* - * An architecture-optimized byte swap for a 64-bit value. - * - * Do not use this function directly. The preferred function is rte_bswap64(). - */ -/* 64-bit mode */ -static inline uint64_t rte_arch_bswap64(uint64_t _x) -{ - register uint64_t x = _x; - asm volatile ("bswap %[x]" - : [x] "+r" (x) - ); - return x; -} - -#ifndef RTE_FORCE_INTRINSICS -#define rte_bswap16(x) ((uint16_t)(__builtin_constant_p(x) ? \ - rte_constant_bswap16(x) : \ - rte_arch_bswap16(x))) - -#define rte_bswap32(x) ((uint32_t)(__builtin_constant_p(x) ? \ - rte_constant_bswap32(x) : \ - rte_arch_bswap32(x))) - -#define rte_bswap64(x) ((uint64_t)(__builtin_constant_p(x) ? \ - rte_constant_bswap64(x) : \ - rte_arch_bswap64(x))) -#else -/* - * __builtin_bswap16 is only available gcc 4.8 and upwards - */ -#if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 8) -#define rte_bswap16(x) ((uint16_t)(__builtin_constant_p(x) ? \ - rte_constant_bswap16(x) : \ - rte_arch_bswap16(x))) -#endif -#endif - -#define rte_cpu_to_le_16(x) (x) -#define rte_cpu_to_le_32(x) (x) -#define rte_cpu_to_le_64(x) (x) - -#define rte_cpu_to_be_16(x) rte_bswap16(x) -#define rte_cpu_to_be_32(x) rte_bswap32(x) -#define rte_cpu_to_be_64(x) rte_bswap64(x) - -#define rte_le_to_cpu_16(x) (x) -#define rte_le_to_cpu_32(x) (x) -#define rte_le_to_cpu_64(x) (x) - -#define rte_be_to_cpu_16(x) rte_bswap16(x) -#define rte_be_to_cpu_32(x) rte_bswap32(x) -#define rte_be_to_cpu_64(x) rte_bswap64(x) - -#ifdef __cplusplus -} -#endif - -#endif /* _RTE_BYTEORDER_X86_64_H_ */ diff --git a/lib/librte_eal/common/include/arch/x86_64/rte_cpuflags.h b/lib/librte_eal/common/include/arch/x86_64/rte_cpuflags.h deleted file mode 100644 index 98906c8..0000000 --- a/lib/librte_eal/common/include/arch/x86_64/rte_cpuflags.h +++ /dev/null @@ -1,310 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef _RTE_CPUFLAGS_X86_64_H_ -#define _RTE_CPUFLAGS_X86_64_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include -#include - -#include "generic/rte_cpuflags.h" - -enum rte_cpu_flag_t { - /* (EAX 01h) ECX features*/ - RTE_CPUFLAG_SSE3 = 0, /**< SSE3 */ - RTE_CPUFLAG_PCLMULQDQ, /**< PCLMULQDQ */ - RTE_CPUFLAG_DTES64, /**< DTES64 */ - RTE_CPUFLAG_MONITOR, /**< MONITOR */ - RTE_CPUFLAG_DS_CPL, /**< DS_CPL */ - RTE_CPUFLAG_VMX, /**< VMX */ - RTE_CPUFLAG_SMX, /**< SMX */ - RTE_CPUFLAG_EIST, /**< EIST */ - RTE_CPUFLAG_TM2, /**< TM2 */ - RTE_CPUFLAG_SSSE3, /**< SSSE3 */ - RTE_CPUFLAG_CNXT_ID, /**< CNXT_ID */ - RTE_CPUFLAG_FMA, /**< FMA */ - RTE_CPUFLAG_CMPXCHG16B, /**< CMPXCHG16B */ - RTE_CPUFLAG_XTPR, /**< XTPR */ - RTE_CPUFLAG_PDCM, /**< PDCM */ - RTE_CPUFLAG_PCID, /**< PCID */ - RTE_CPUFLAG_DCA, /**< DCA */ - RTE_CPUFLAG_SSE4_1, /**< SSE4_1 */ - RTE_CPUFLAG_SSE4_2, /**< SSE4_2 */ - RTE_CPUFLAG_X2APIC, /**< X2APIC */ - RTE_CPUFLAG_MOVBE, /**< MOVBE */ - RTE_CPUFLAG_POPCNT, /**< POPCNT */ - RTE_CPUFLAG_TSC_DEADLINE, /**< TSC_DEADLINE */ - RTE_CPUFLAG_AES, /**< AES */ - RTE_CPUFLAG_XSAVE, /**< XSAVE */ - RTE_CPUFLAG_OSXSAVE, /**< OSXSAVE */ - RTE_CPUFLAG_AVX, /**< AVX */ - RTE_CPUFLAG_F16C, /**< F16C */ - RTE_CPUFLAG_RDRAND, /**< RDRAND */ - - /* (EAX 01h) EDX features */ - RTE_CPUFLAG_FPU, /**< FPU */ - RTE_CPUFLAG_VME, /**< VME */ - RTE_CPUFLAG_DE, /**< DE */ - RTE_CPUFLAG_PSE, /**< PSE */ - RTE_CPUFLAG_TSC, /**< TSC */ - RTE_CPUFLAG_MSR, /**< MSR */ - RTE_CPUFLAG_PAE, /**< PAE */ - RTE_CPUFLAG_MCE, /**< MCE */ - RTE_CPUFLAG_CX8, /**< CX8 */ - RTE_CPUFLAG_APIC, /**< APIC */ - RTE_CPUFLAG_SEP, /**< SEP */ - RTE_CPUFLAG_MTRR, /**< MTRR */ - RTE_CPUFLAG_PGE, /**< PGE */ - RTE_CPUFLAG_MCA, /**< MCA */ - RTE_CPUFLAG_CMOV, /**< CMOV */ - RTE_CPUFLAG_PAT, /**< PAT */ - RTE_CPUFLAG_PSE36, /**< PSE36 */ - RTE_CPUFLAG_PSN, /**< PSN */ - RTE_CPUFLAG_CLFSH, /**< CLFSH */ - RTE_CPUFLAG_DS, /**< DS */ - RTE_CPUFLAG_ACPI, /**< ACPI */ - RTE_CPUFLAG_MMX, /**< MMX */ - RTE_CPUFLAG_FXSR, /**< FXSR */ - RTE_CPUFLAG_SSE, /**< SSE */ - RTE_CPUFLAG_SSE2, /**< SSE2 */ - RTE_CPUFLAG_SS, /**< SS */ - RTE_CPUFLAG_HTT, /**< HTT */ - RTE_CPUFLAG_TM, /**< TM */ - RTE_CPUFLAG_PBE, /**< PBE */ - - /* (EAX 06h) EAX features */ - RTE_CPUFLAG_DIGTEMP, /**< DIGTEMP */ - RTE_CPUFLAG_TRBOBST, /**< TRBOBST */ - RTE_CPUFLAG_ARAT, /**< ARAT */ - RTE_CPUFLAG_PLN, /**< PLN */ - RTE_CPUFLAG_ECMD, /**< ECMD */ - RTE_CPUFLAG_PTM, /**< PTM */ - - /* (EAX 06h) ECX features */ - RTE_CPUFLAG_MPERF_APERF_MSR, /**< MPERF_APERF_MSR */ - RTE_CPUFLAG_ACNT2, /**< ACNT2 */ - RTE_CPUFLAG_ENERGY_EFF, /**< ENERGY_EFF */ - - /* (EAX 07h, ECX 0h) EBX features */ - RTE_CPUFLAG_FSGSBASE, /**< FSGSBASE */ - RTE_CPUFLAG_BMI1, /**< BMI1 */ - RTE_CPUFLAG_HLE, /**< Hardware Lock elision */ - RTE_CPUFLAG_AVX2, /**< AVX2 */ - RTE_CPUFLAG_SMEP, /**< SMEP */ - RTE_CPUFLAG_BMI2, /**< BMI2 */ - RTE_CPUFLAG_ERMS, /**< ERMS */ - RTE_CPUFLAG_INVPCID, /**< INVPCID */ - RTE_CPUFLAG_RTM, /**< Transactional memory */ - - /* (EAX 80000001h) ECX features */ - RTE_CPUFLAG_LAHF_SAHF, /**< LAHF_SAHF */ - RTE_CPUFLAG_LZCNT, /**< LZCNT */ - - /* (EAX 80000001h) EDX features */ - RTE_CPUFLAG_SYSCALL, /**< SYSCALL */ - RTE_CPUFLAG_XD, /**< XD */ - RTE_CPUFLAG_1GB_PG, /**< 1GB_PG */ - RTE_CPUFLAG_RDTSCP, /**< RDTSCP */ - RTE_CPUFLAG_EM64T, /**< EM64T */ - - /* (EAX 80000007h) EDX features */ - RTE_CPUFLAG_INVTSC, /**< INVTSC */ - - /* The last item */ - RTE_CPUFLAG_NUMFLAGS, /**< This should always be the last! */ -}; - -enum cpu_register_t { - REG_EAX = 0, - REG_EBX, - REG_ECX, - REG_EDX, -}; - -static const struct feature_entry cpu_feature_table[] = { - FEAT_DEF(SSE3, 0x00000001, 0, REG_ECX, 0) - FEAT_DEF(PCLMULQDQ, 0x00000001, 0, REG_ECX, 1) - FEAT_DEF(DTES64, 0x00000001, 0, REG_ECX, 2) - FEAT_DEF(MONITOR, 0x00000001, 0, REG_ECX, 3) - FEAT_DEF(DS_CPL, 0x00000001, 0, REG_ECX, 4) - FEAT_DEF(VMX, 0x00000001, 0, REG_ECX, 5) - FEAT_DEF(SMX, 0x00000001, 0, REG_ECX, 6) - FEAT_DEF(EIST, 0x00000001, 0, REG_ECX, 7) - FEAT_DEF(TM2, 0x00000001, 0, REG_ECX, 8) - FEAT_DEF(SSSE3, 0x00000001, 0, REG_ECX, 9) - FEAT_DEF(CNXT_ID, 0x00000001, 0, REG_ECX, 10) - FEAT_DEF(FMA, 0x00000001, 0, REG_ECX, 12) - FEAT_DEF(CMPXCHG16B, 0x00000001, 0, REG_ECX, 13) - FEAT_DEF(XTPR, 0x00000001, 0, REG_ECX, 14) - FEAT_DEF(PDCM, 0x00000001, 0, REG_ECX, 15) - FEAT_DEF(PCID, 0x00000001, 0, REG_ECX, 17) - FEAT_DEF(DCA, 0x00000001, 0, REG_ECX, 18) - FEAT_DEF(SSE4_1, 0x00000001, 0, REG_ECX, 19) - FEAT_DEF(SSE4_2, 0x00000001, 0, REG_ECX, 20) - FEAT_DEF(X2APIC, 0x00000001, 0, REG_ECX, 21) - FEAT_DEF(MOVBE, 0x00000001, 0, REG_ECX, 22) - FEAT_DEF(POPCNT, 0x00000001, 0, REG_ECX, 23) - FEAT_DEF(TSC_DEADLINE, 0x00000001, 0, REG_ECX, 24) - FEAT_DEF(AES, 0x00000001, 0, REG_ECX, 25) - FEAT_DEF(XSAVE, 0x00000001, 0, REG_ECX, 26) - FEAT_DEF(OSXSAVE, 0x00000001, 0, REG_ECX, 27) - FEAT_DEF(AVX, 0x00000001, 0, REG_ECX, 28) - FEAT_DEF(F16C, 0x00000001, 0, REG_ECX, 29) - FEAT_DEF(RDRAND, 0x00000001, 0, REG_ECX, 30) - - FEAT_DEF(FPU, 0x00000001, 0, REG_EDX, 0) - FEAT_DEF(VME, 0x00000001, 0, REG_EDX, 1) - FEAT_DEF(DE, 0x00000001, 0, REG_EDX, 2) - FEAT_DEF(PSE, 0x00000001, 0, REG_EDX, 3) - FEAT_DEF(TSC, 0x00000001, 0, REG_EDX, 4) - FEAT_DEF(MSR, 0x00000001, 0, REG_EDX, 5) - FEAT_DEF(PAE, 0x00000001, 0, REG_EDX, 6) - FEAT_DEF(MCE, 0x00000001, 0, REG_EDX, 7) - FEAT_DEF(CX8, 0x00000001, 0, REG_EDX, 8) - FEAT_DEF(APIC, 0x00000001, 0, REG_EDX, 9) - FEAT_DEF(SEP, 0x00000001, 0, REG_EDX, 11) - FEAT_DEF(MTRR, 0x00000001, 0, REG_EDX, 12) - FEAT_DEF(PGE, 0x00000001, 0, REG_EDX, 13) - FEAT_DEF(MCA, 0x00000001, 0, REG_EDX, 14) - FEAT_DEF(CMOV, 0x00000001, 0, REG_EDX, 15) - FEAT_DEF(PAT, 0x00000001, 0, REG_EDX, 16) - FEAT_DEF(PSE36, 0x00000001, 0, REG_EDX, 17) - FEAT_DEF(PSN, 0x00000001, 0, REG_EDX, 18) - FEAT_DEF(CLFSH, 0x00000001, 0, REG_EDX, 19) - FEAT_DEF(DS, 0x00000001, 0, REG_EDX, 21) - FEAT_DEF(ACPI, 0x00000001, 0, REG_EDX, 22) - FEAT_DEF(MMX, 0x00000001, 0, REG_EDX, 23) - FEAT_DEF(FXSR, 0x00000001, 0, REG_EDX, 24) - FEAT_DEF(SSE, 0x00000001, 0, REG_EDX, 25) - FEAT_DEF(SSE2, 0x00000001, 0, REG_EDX, 26) - FEAT_DEF(SS, 0x00000001, 0, REG_EDX, 27) - FEAT_DEF(HTT, 0x00000001, 0, REG_EDX, 28) - FEAT_DEF(TM, 0x00000001, 0, REG_EDX, 29) - FEAT_DEF(PBE, 0x00000001, 0, REG_EDX, 31) - - FEAT_DEF(DIGTEMP, 0x00000006, 0, REG_EAX, 0) - FEAT_DEF(TRBOBST, 0x00000006, 0, REG_EAX, 1) - FEAT_DEF(ARAT, 0x00000006, 0, REG_EAX, 2) - FEAT_DEF(PLN, 0x00000006, 0, REG_EAX, 4) - FEAT_DEF(ECMD, 0x00000006, 0, REG_EAX, 5) - FEAT_DEF(PTM, 0x00000006, 0, REG_EAX, 6) - - FEAT_DEF(MPERF_APERF_MSR, 0x00000006, 0, REG_ECX, 0) - FEAT_DEF(ACNT2, 0x00000006, 0, REG_ECX, 1) - FEAT_DEF(ENERGY_EFF, 0x00000006, 0, REG_ECX, 3) - - FEAT_DEF(FSGSBASE, 0x00000007, 0, REG_EBX, 0) - FEAT_DEF(BMI1, 0x00000007, 0, REG_EBX, 2) - FEAT_DEF(HLE, 0x00000007, 0, REG_EBX, 4) - FEAT_DEF(AVX2, 0x00000007, 0, REG_EBX, 5) - FEAT_DEF(SMEP, 0x00000007, 0, REG_EBX, 6) - FEAT_DEF(BMI2, 0x00000007, 0, REG_EBX, 7) - FEAT_DEF(ERMS, 0x00000007, 0, REG_EBX, 8) - FEAT_DEF(INVPCID, 0x00000007, 0, REG_EBX, 10) - FEAT_DEF(RTM, 0x00000007, 0, REG_EBX, 11) - - FEAT_DEF(LAHF_SAHF, 0x80000001, 0, REG_ECX, 0) - FEAT_DEF(LZCNT, 0x80000001, 0, REG_ECX, 4) - - FEAT_DEF(SYSCALL, 0x80000001, 0, REG_EDX, 11) - FEAT_DEF(XD, 0x80000001, 0, REG_EDX, 20) - FEAT_DEF(1GB_PG, 0x80000001, 0, REG_EDX, 26) - FEAT_DEF(RDTSCP, 0x80000001, 0, REG_EDX, 27) - FEAT_DEF(EM64T, 0x80000001, 0, REG_EDX, 29) - - FEAT_DEF(INVTSC, 0x80000007, 0, REG_EDX, 8) -}; - -static inline void -rte_cpu_get_features(uint32_t leaf, uint32_t subleaf, cpuid_registers_t out) -{ -#if defined(__i386__) && defined(__PIC__) - /* %ebx is a forbidden register if we compile with -fPIC or -fPIE */ - asm volatile("movl %%ebx,%0 ; cpuid ; xchgl %%ebx,%0" - : "=r" (out[REG_EBX]), - "=a" (out[REG_EAX]), - "=c" (out[REG_ECX]), - "=d" (out[REG_EDX]) - : "a" (leaf), "c" (subleaf)); -#else - - asm volatile("cpuid" - : "=a" (out[REG_EAX]), - "=b" (out[REG_EBX]), - "=c" (out[REG_ECX]), - "=d" (out[REG_EDX]) - : "a" (leaf), "c" (subleaf)); - -#endif -} - -static inline int -rte_cpu_get_flag_enabled(enum rte_cpu_flag_t feature) -{ - const struct feature_entry *feat; - cpuid_registers_t regs; - - - if (feature >= RTE_CPUFLAG_NUMFLAGS) - /* Flag does not match anything in the feature tables */ - return -ENOENT; - - feat = &cpu_feature_table[feature]; - - if (!feat->leaf) - /* This entry in the table wasn't filled out! */ - return -EFAULT; - - rte_cpu_get_features(feat->leaf & 0xffff0000, 0, regs); - if (((regs[REG_EAX] ^ feat->leaf) & 0xffff0000) || - regs[REG_EAX] < feat->leaf) - return 0; - - /* get the cpuid leaf containing the desired feature */ - rte_cpu_get_features(feat->leaf, feat->subleaf, regs); - - /* check if the feature is enabled */ - return (regs[feat->reg] >> feat->bit) & 1; -} - -#ifdef __cplusplus -} -#endif - -#endif /* _RTE_CPUFLAGS_X86_64_H_ */ diff --git a/lib/librte_eal/common/include/arch/x86_64/rte_cycles.h b/lib/librte_eal/common/include/arch/x86_64/rte_cycles.h deleted file mode 100644 index 6e3c7d8..0000000 --- a/lib/librte_eal/common/include/arch/x86_64/rte_cycles.h +++ /dev/null @@ -1,121 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -/* BSD LICENSE - * - * Copyright(c) 2013 6WIND. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of 6WIND S.A. nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef _RTE_CYCLES_X86_64_H_ -#define _RTE_CYCLES_X86_64_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include "generic/rte_cycles.h" - -#ifdef RTE_LIBRTE_EAL_VMWARE_TSC_MAP_SUPPORT -/* Global switch to use VMWARE mapping of TSC instead of RDTSC */ -extern int rte_cycles_vmware_tsc_map; -#include -#endif - -static inline uint64_t -rte_rdtsc(void) -{ - union { - uint64_t tsc_64; - struct { - uint32_t lo_32; - uint32_t hi_32; - }; - } tsc; - -#ifdef RTE_LIBRTE_EAL_VMWARE_TSC_MAP_SUPPORT - if (unlikely(rte_cycles_vmware_tsc_map)) { - /* ecx = 0x10000 corresponds to the physical TSC for VMware */ - asm volatile("rdpmc" : - "=a" (tsc.lo_32), - "=d" (tsc.hi_32) : - "c"(0x10000)); - return tsc.tsc_64; - } -#endif - - asm volatile("rdtsc" : - "=a" (tsc.lo_32), - "=d" (tsc.hi_32)); - return tsc.tsc_64; -} - -static inline uint64_t -rte_rdtsc_precise(void) -{ - rte_mb(); - return rte_rdtsc(); -} - -static inline uint64_t -rte_get_tsc_cycles(void) { return rte_rdtsc(); } - -#ifdef __cplusplus -} -#endif - -#endif /* _RTE_CYCLES_X86_64_H_ */ diff --git a/lib/librte_eal/common/include/arch/x86_64/rte_memcpy.h b/lib/librte_eal/common/include/arch/x86_64/rte_memcpy.h deleted file mode 100644 index 290c5cd..0000000 --- a/lib/librte_eal/common/include/arch/x86_64/rte_memcpy.h +++ /dev/null @@ -1,297 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef _RTE_MEMCPY_X86_64_H_ -#define _RTE_MEMCPY_X86_64_H_ - -#include -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -#include "generic/rte_memcpy.h" - -#ifdef __INTEL_COMPILER -#pragma warning(disable:593) /* Stop unused variable warning (reg_a etc). */ -#endif - -static inline void -rte_mov16(uint8_t *dst, const uint8_t *src) -{ - __m128i reg_a; - asm volatile ( - "movdqu (%[src]), %[reg_a]\n\t" - "movdqu %[reg_a], (%[dst])\n\t" - : [reg_a] "=x" (reg_a) - : [src] "r" (src), - [dst] "r"(dst) - : "memory" - ); -} - -static inline void -rte_mov32(uint8_t *dst, const uint8_t *src) -{ - __m128i reg_a, reg_b; - asm volatile ( - "movdqu (%[src]), %[reg_a]\n\t" - "movdqu 16(%[src]), %[reg_b]\n\t" - "movdqu %[reg_a], (%[dst])\n\t" - "movdqu %[reg_b], 16(%[dst])\n\t" - : [reg_a] "=x" (reg_a), - [reg_b] "=x" (reg_b) - : [src] "r" (src), - [dst] "r"(dst) - : "memory" - ); -} - -static inline void -rte_mov48(uint8_t *dst, const uint8_t *src) -{ - __m128i reg_a, reg_b, reg_c; - asm volatile ( - "movdqu (%[src]), %[reg_a]\n\t" - "movdqu 16(%[src]), %[reg_b]\n\t" - "movdqu 32(%[src]), %[reg_c]\n\t" - "movdqu %[reg_a], (%[dst])\n\t" - "movdqu %[reg_b], 16(%[dst])\n\t" - "movdqu %[reg_c], 32(%[dst])\n\t" - : [reg_a] "=x" (reg_a), - [reg_b] "=x" (reg_b), - [reg_c] "=x" (reg_c) - : [src] "r" (src), - [dst] "r"(dst) - : "memory" - ); -} - -static inline void -rte_mov64(uint8_t *dst, const uint8_t *src) -{ - __m128i reg_a, reg_b, reg_c, reg_d; - asm volatile ( - "movdqu (%[src]), %[reg_a]\n\t" - "movdqu 16(%[src]), %[reg_b]\n\t" - "movdqu 32(%[src]), %[reg_c]\n\t" - "movdqu 48(%[src]), %[reg_d]\n\t" - "movdqu %[reg_a], (%[dst])\n\t" - "movdqu %[reg_b], 16(%[dst])\n\t" - "movdqu %[reg_c], 32(%[dst])\n\t" - "movdqu %[reg_d], 48(%[dst])\n\t" - : [reg_a] "=x" (reg_a), - [reg_b] "=x" (reg_b), - [reg_c] "=x" (reg_c), - [reg_d] "=x" (reg_d) - : [src] "r" (src), - [dst] "r"(dst) - : "memory" - ); -} - -static inline void -rte_mov128(uint8_t *dst, const uint8_t *src) -{ - __m128i reg_a, reg_b, reg_c, reg_d, reg_e, reg_f, reg_g, reg_h; - asm volatile ( - "movdqu (%[src]), %[reg_a]\n\t" - "movdqu 16(%[src]), %[reg_b]\n\t" - "movdqu 32(%[src]), %[reg_c]\n\t" - "movdqu 48(%[src]), %[reg_d]\n\t" - "movdqu 64(%[src]), %[reg_e]\n\t" - "movdqu 80(%[src]), %[reg_f]\n\t" - "movdqu 96(%[src]), %[reg_g]\n\t" - "movdqu 112(%[src]), %[reg_h]\n\t" - "movdqu %[reg_a], (%[dst])\n\t" - "movdqu %[reg_b], 16(%[dst])\n\t" - "movdqu %[reg_c], 32(%[dst])\n\t" - "movdqu %[reg_d], 48(%[dst])\n\t" - "movdqu %[reg_e], 64(%[dst])\n\t" - "movdqu %[reg_f], 80(%[dst])\n\t" - "movdqu %[reg_g], 96(%[dst])\n\t" - "movdqu %[reg_h], 112(%[dst])\n\t" - : [reg_a] "=x" (reg_a), - [reg_b] "=x" (reg_b), - [reg_c] "=x" (reg_c), - [reg_d] "=x" (reg_d), - [reg_e] "=x" (reg_e), - [reg_f] "=x" (reg_f), - [reg_g] "=x" (reg_g), - [reg_h] "=x" (reg_h) - : [src] "r" (src), - [dst] "r"(dst) - : "memory" - ); -} - -#ifdef __INTEL_COMPILER -#pragma warning(enable:593) -#endif - -static inline void -rte_mov256(uint8_t *dst, const uint8_t *src) -{ - rte_mov128(dst, src); - rte_mov128(dst + 128, src + 128); -} - -#define rte_memcpy(dst, src, n) \ - ((__builtin_constant_p(n)) ? \ - memcpy((dst), (src), (n)) : \ - rte_memcpy_func((dst), (src), (n))) - -static inline void * -rte_memcpy_func(void *dst, const void *src, size_t n) -{ - void *ret = dst; - - /* We can't copy < 16 bytes using XMM registers so do it manually. */ - if (n < 16) { - if (n & 0x01) { - *(uint8_t *)dst = *(const uint8_t *)src; - dst = (uint8_t *)dst + 1; - src = (const uint8_t *)src + 1; - } - if (n & 0x02) { - *(uint16_t *)dst = *(const uint16_t *)src; - dst = (uint16_t *)dst + 1; - src = (const uint16_t *)src + 1; - } - if (n & 0x04) { - *(uint32_t *)dst = *(const uint32_t *)src; - dst = (uint32_t *)dst + 1; - src = (const uint32_t *)src + 1; - } - if (n & 0x08) { - *(uint64_t *)dst = *(const uint64_t *)src; - } - return ret; - } - - /* Special fast cases for <= 128 bytes */ - if (n <= 32) { - rte_mov16((uint8_t *)dst, (const uint8_t *)src); - rte_mov16((uint8_t *)dst - 16 + n, (const uint8_t *)src - 16 + n); - return ret; - } - - if (n <= 64) { - rte_mov32((uint8_t *)dst, (const uint8_t *)src); - rte_mov32((uint8_t *)dst - 32 + n, (const uint8_t *)src - 32 + n); - return ret; - } - - if (n <= 128) { - rte_mov64((uint8_t *)dst, (const uint8_t *)src); - rte_mov64((uint8_t *)dst - 64 + n, (const uint8_t *)src - 64 + n); - return ret; - } - - /* - * For large copies > 128 bytes. This combination of 256, 64 and 16 byte - * copies was found to be faster than doing 128 and 32 byte copies as - * well. - */ - for ( ; n >= 256; n -= 256) { - rte_mov256((uint8_t *)dst, (const uint8_t *)src); - dst = (uint8_t *)dst + 256; - src = (const uint8_t *)src + 256; - } - - /* - * We split the remaining bytes (which will be less than 256) into - * 64byte (2^6) chunks. - * Using incrementing integers in the case labels of a switch statement - * enourages the compiler to use a jump table. To get incrementing - * integers, we shift the 2 relevant bits to the LSB position to first - * get decrementing integers, and then subtract. - */ - switch (3 - (n >> 6)) { - case 0x00: - rte_mov64((uint8_t *)dst, (const uint8_t *)src); - n -= 64; - dst = (uint8_t *)dst + 64; - src = (const uint8_t *)src + 64; /* fallthrough */ - case 0x01: - rte_mov64((uint8_t *)dst, (const uint8_t *)src); - n -= 64; - dst = (uint8_t *)dst + 64; - src = (const uint8_t *)src + 64; /* fallthrough */ - case 0x02: - rte_mov64((uint8_t *)dst, (const uint8_t *)src); - n -= 64; - dst = (uint8_t *)dst + 64; - src = (const uint8_t *)src + 64; /* fallthrough */ - default: - ; - } - - /* - * We split the remaining bytes (which will be less than 64) into - * 16byte (2^4) chunks, using the same switch structure as above. - */ - switch (3 - (n >> 4)) { - case 0x00: - rte_mov16((uint8_t *)dst, (const uint8_t *)src); - n -= 16; - dst = (uint8_t *)dst + 16; - src = (const uint8_t *)src + 16; /* fallthrough */ - case 0x01: - rte_mov16((uint8_t *)dst, (const uint8_t *)src); - n -= 16; - dst = (uint8_t *)dst + 16; - src = (const uint8_t *)src + 16; /* fallthrough */ - case 0x02: - rte_mov16((uint8_t *)dst, (const uint8_t *)src); - n -= 16; - dst = (uint8_t *)dst + 16; - src = (const uint8_t *)src + 16; /* fallthrough */ - default: - ; - } - - /* Copy any remaining bytes, without going beyond end of buffers */ - if (n != 0) { - rte_mov16((uint8_t *)dst - 16 + n, (const uint8_t *)src - 16 + n); - } - return ret; -} - -#ifdef __cplusplus -} -#endif - -#endif /* _RTE_MEMCPY_X86_64_H_ */ diff --git a/lib/librte_eal/common/include/arch/x86_64/rte_prefetch.h b/lib/librte_eal/common/include/arch/x86_64/rte_prefetch.h deleted file mode 100644 index ec2454d..0000000 --- a/lib/librte_eal/common/include/arch/x86_64/rte_prefetch.h +++ /dev/null @@ -1,62 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef _RTE_PREFETCH_X86_64_H_ -#define _RTE_PREFETCH_X86_64_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include "generic/rte_prefetch.h" - -static inline void rte_prefetch0(volatile void *p) -{ - asm volatile ("prefetcht0 %[p]" : [p] "+m" (*(volatile char *)p)); -} - -static inline void rte_prefetch1(volatile void *p) -{ - asm volatile ("prefetcht1 %[p]" : [p] "+m" (*(volatile char *)p)); -} - -static inline void rte_prefetch2(volatile void *p) -{ - asm volatile ("prefetcht2 %[p]" : [p] "+m" (*(volatile char *)p)); -} - -#ifdef __cplusplus -} -#endif - -#endif /* _RTE_PREFETCH_X86_64_H_ */ diff --git a/lib/librte_eal/common/include/arch/x86_64/rte_spinlock.h b/lib/librte_eal/common/include/arch/x86_64/rte_spinlock.h deleted file mode 100644 index 54fba95..0000000 --- a/lib/librte_eal/common/include/arch/x86_64/rte_spinlock.h +++ /dev/null @@ -1,94 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef _RTE_SPINLOCK_X86_64_H_ -#define _RTE_SPINLOCK_X86_64_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include "generic/rte_spinlock.h" - -#ifndef RTE_FORCE_INTRINSICS -static inline void -rte_spinlock_lock(rte_spinlock_t *sl) -{ - int lock_val = 1; - asm volatile ( - "1:\n" - "xchg %[locked], %[lv]\n" - "test %[lv], %[lv]\n" - "jz 3f\n" - "2:\n" - "pause\n" - "cmpl $0, %[locked]\n" - "jnz 2b\n" - "jmp 1b\n" - "3:\n" - : [locked] "=m" (sl->locked), [lv] "=q" (lock_val) - : "[lv]" (lock_val) - : "memory"); -} - -static inline void -rte_spinlock_unlock (rte_spinlock_t *sl) -{ - int unlock_val = 0; - asm volatile ( - "xchg %[locked], %[ulv]\n" - : [locked] "=m" (sl->locked), [ulv] "=q" (unlock_val) - : "[ulv]" (unlock_val) - : "memory"); -} - -static inline int -rte_spinlock_trylock (rte_spinlock_t *sl) -{ - int lockval = 1; - - asm volatile ( - "xchg %[locked], %[lockval]" - : [locked] "=m" (sl->locked), [lockval] "=q" (lockval) - : "[lockval]" (lockval) - : "memory"); - - return (lockval == 0); -} -#endif - -#ifdef __cplusplus -} -#endif - -#endif /* _RTE_SPINLOCK_X86_64_H_ */ diff --git a/mk/arch/i686/rte.vars.mk b/mk/arch/i686/rte.vars.mk index 8d56ca7..8ba9a23 100644 --- a/mk/arch/i686/rte.vars.mk +++ b/mk/arch/i686/rte.vars.mk @@ -48,6 +48,8 @@ # ARCH ?= i386 +# common arch dir in eal headers +ARCH_DIR := x86 CROSS ?= CPU_CFLAGS ?= -m32 diff --git a/mk/arch/x86_64/rte.vars.mk b/mk/arch/x86_64/rte.vars.mk index 51bd477..b986f04 100644 --- a/mk/arch/x86_64/rte.vars.mk +++ b/mk/arch/x86_64/rte.vars.mk @@ -48,6 +48,8 @@ # ARCH ?= x86_64 +# common arch dir in eal headers +ARCH_DIR := x86 CROSS ?= CPU_CFLAGS ?= -m64 -- 1.7.10.4