From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id C4591A0C4D; Thu, 2 Sep 2021 07:33:12 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 9BF3D40698; Thu, 2 Sep 2021 07:33:08 +0200 (CEST) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mails.dpdk.org (Postfix) with ESMTP id 97B2B40696 for ; Thu, 2 Sep 2021 07:33:07 +0200 (CEST) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 1ECCF1FB; Wed, 1 Sep 2021 22:33:07 -0700 (PDT) Received: from net-x86-dell-8268.shanghai.arm.com (net-x86-dell-8268.shanghai.arm.com [10.169.210.115]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 495603F5A1; Wed, 1 Sep 2021 22:33:05 -0700 (PDT) From: Feifei Wang To: Ruifeng Wang Cc: dev@dpdk.org, nd@arm.com, Feifei Wang Date: Thu, 2 Sep 2021 13:32:49 +0800 Message-Id: <20210902053253.3017858-2-feifei.wang2@arm.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20210902053253.3017858-1-feifei.wang2@arm.com> References: <20210902053253.3017858-1-feifei.wang2@arm.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Subject: [dpdk-dev] [RFC PATCH v1 1/5] eal: add new API for wait until scheme X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" For 'wait until' scheme, add new APIs for more cases: 1. add wait_until_unequal API 2. add wait_until_part_equal API Signed-off-by: Feifei Wang Reviewed-by: Ruifeng Wang --- lib/eal/arm/include/rte_pause_64.h | 271 +++++++++++++++++++----- lib/eal/include/generic/rte_pause.h | 309 ++++++++++++++++++++++++++++ 2 files changed, 526 insertions(+), 54 deletions(-) diff --git a/lib/eal/arm/include/rte_pause_64.h b/lib/eal/arm/include/rte_pause_64.h index e87d10b8cc..19716276fc 100644 --- a/lib/eal/arm/include/rte_pause_64.h +++ b/lib/eal/arm/include/rte_pause_64.h @@ -31,20 +31,12 @@ static inline void rte_pause(void) /* Put processor into low power WFE(Wait For Event) state. */ #define __WFE() { asm volatile("wfe" : : : "memory"); } -static __rte_always_inline void -rte_wait_until_equal_16(volatile uint16_t *addr, uint16_t expected, - int memorder) -{ - uint16_t value; - - assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); - - /* - * Atomic exclusive load from addr, it returns the 16-bit content of - * *addr while making it 'monitored',when it is written by someone - * else, the 'monitored' state is cleared and a event is generated - * implicitly to exit WFE. - */ +/* + * Atomic exclusive load from addr, it returns the 16-bit content of + * *addr while making it 'monitored', when it is written by someone + * else, the 'monitored' state is cleared and a event is generated + * implicitly to exit WFE. + */ #define __LOAD_EXC_16(src, dst, memorder) { \ if (memorder == __ATOMIC_RELAXED) { \ asm volatile("ldxrh %w[tmp], [%x[addr]]" \ @@ -58,6 +50,52 @@ rte_wait_until_equal_16(volatile uint16_t *addr, uint16_t expected, : "memory"); \ } } +/* + * Atomic exclusive load from addr, it returns the 32-bit content of + * *addr while making it 'monitored', when it is written by someone + * else, the 'monitored' state is cleared and a event is generated + * implicitly to exit WFE. + */ +#define __LOAD_EXC_32(src, dst, memorder) { \ + if (memorder == __ATOMIC_RELAXED) { \ + asm volatile("ldxr %w[tmp], [%x[addr]]" \ + : [tmp] "=&r" (dst) \ + : [addr] "r"(src) \ + : "memory"); \ + } else { \ + asm volatile("ldaxr %w[tmp], [%x[addr]]" \ + : [tmp] "=&r" (dst) \ + : [addr] "r"(src) \ + : "memory"); \ + } } + +/* + * Atomic exclusive load from addr, it returns the 64-bit content of + * *addr while making it 'monitored', when it is written by someone + * else, the 'monitored' state is cleared and a event is generated + * implicitly to exit WFE. + */ +#define __LOAD_EXC_64(src, dst, memorder) { \ + if (memorder == __ATOMIC_RELAXED) { \ + asm volatile("ldxr %x[tmp], [%x[addr]]" \ + : [tmp] "=&r" (dst) \ + : [addr] "r"(src) \ + : "memory"); \ + } else { \ + asm volatile("ldaxr %x[tmp], [%x[addr]]" \ + : [tmp] "=&r" (dst) \ + : [addr] "r"(src) \ + : "memory"); \ + } } + +static __rte_always_inline void +rte_wait_until_equal_16(volatile uint16_t *addr, uint16_t expected, + int memorder) +{ + uint16_t value; + + assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); + __LOAD_EXC_16(addr, value, memorder) if (value != expected) { __SEVL() @@ -66,7 +104,6 @@ rte_wait_until_equal_16(volatile uint16_t *addr, uint16_t expected, __LOAD_EXC_16(addr, value, memorder) } while (value != expected); } -#undef __LOAD_EXC_16 } static __rte_always_inline void @@ -77,25 +114,6 @@ rte_wait_until_equal_32(volatile uint32_t *addr, uint32_t expected, assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); - /* - * Atomic exclusive load from addr, it returns the 32-bit content of - * *addr while making it 'monitored',when it is written by someone - * else, the 'monitored' state is cleared and a event is generated - * implicitly to exit WFE. - */ -#define __LOAD_EXC_32(src, dst, memorder) { \ - if (memorder == __ATOMIC_RELAXED) { \ - asm volatile("ldxr %w[tmp], [%x[addr]]" \ - : [tmp] "=&r" (dst) \ - : [addr] "r"(src) \ - : "memory"); \ - } else { \ - asm volatile("ldaxr %w[tmp], [%x[addr]]" \ - : [tmp] "=&r" (dst) \ - : [addr] "r"(src) \ - : "memory"); \ - } } - __LOAD_EXC_32(addr, value, memorder) if (value != expected) { __SEVL() @@ -104,7 +122,6 @@ rte_wait_until_equal_32(volatile uint32_t *addr, uint32_t expected, __LOAD_EXC_32(addr, value, memorder) } while (value != expected); } -#undef __LOAD_EXC_32 } static __rte_always_inline void @@ -115,25 +132,6 @@ rte_wait_until_equal_64(volatile uint64_t *addr, uint64_t expected, assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); - /* - * Atomic exclusive load from addr, it returns the 64-bit content of - * *addr while making it 'monitored',when it is written by someone - * else, the 'monitored' state is cleared and a event is generated - * implicitly to exit WFE. - */ -#define __LOAD_EXC_64(src, dst, memorder) { \ - if (memorder == __ATOMIC_RELAXED) { \ - asm volatile("ldxr %x[tmp], [%x[addr]]" \ - : [tmp] "=&r" (dst) \ - : [addr] "r"(src) \ - : "memory"); \ - } else { \ - asm volatile("ldaxr %x[tmp], [%x[addr]]" \ - : [tmp] "=&r" (dst) \ - : [addr] "r"(src) \ - : "memory"); \ - } } - __LOAD_EXC_64(addr, value, memorder) if (value != expected) { __SEVL() @@ -143,6 +141,171 @@ rte_wait_until_equal_64(volatile uint64_t *addr, uint64_t expected, } while (value != expected); } } + +static __rte_always_inline void +rte_wait_until_part_equal_16(volatile uint16_t *addr, uint16_t mask, + uint16_t expected, int memorder) +{ + uint16_t value; + + assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); + + __LOAD_EXC_16(addr, value, memorder) + if ((value & mask) != expected) { + __SEVL() + do { + __WFE() + __LOAD_EXC_16(addr, value, memorder) + } while ((value & mask) != expected); + } +} + +static __rte_always_inline void +rte_wait_until_part_equal_32(volatile uint32_t *addr, uint32_t mask, + uint32_t expected, int memorder) +{ + uint32_t value; + + assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); + + __LOAD_EXC_32(addr, value, memorder) + if ((value & mask) != expected) { + __SEVL() + do { + __WFE() + __LOAD_EXC_32(addr, value, memorder) + } while ((value & mask) != expected); + } +} + +static __rte_always_inline void +rte_wait_until_part_equal_64(volatile uint64_t *addr, uint64_t mask, + uint64_t expected, int memorder) +{ + uint64_t value; + + assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); + + __LOAD_EXC_64(addr, value, memorder) + if ((value & mask) != expected) { + __SEVL() + do { + __WFE() + __LOAD_EXC_64(addr, value, memorder) + } while ((value & mask) != expected); + } +} + +static __rte_always_inline void +rte_wait_until_unequal_16(volatile uint16_t *addr, uint16_t original, + int memorder) +{ + uint16_t value; + + assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); + + __LOAD_EXC_16(addr, value, memorder) + if (value == original) { + __SEVL() + do { + __WFE() + __LOAD_EXC_16(addr, value, memorder) + } while (value == original); + } +} + +static __rte_always_inline void +rte_wait_until_unequal_32(volatile uint32_t *addr, uint32_t original, + int memorder) +{ + uint32_t value; + + assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); + + __LOAD_EXC_32(addr, value, memorder) + if (value == original) { + __SEVL() + do { + __WFE() + __LOAD_EXC_32(addr, value, memorder) + } while (value == original); + } +} + +static __rte_always_inline void +rte_wait_until_unequal_64(volatile uint64_t *addr, uint64_t original, + int memorder) +{ + uint64_t value; + + assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); + + __LOAD_EXC_64(addr, value, memorder) + if (value == original) { + __SEVL() + do { + __WFE() + __LOAD_EXC_64(addr, value, memorder) + } while (value == original); + } +} + +static __rte_always_inline void +rte_wait_until_part_unequal_16(volatile uint16_t *addr, uint16_t mask, + uint16_t original, int memorder) +{ + uint16_t value; + + assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); + + __LOAD_EXC_16(addr, value, memorder) + if ((value & mask) == original) { + __SEVL() + do { + __WFE() + __LOAD_EXC_16(addr, value, memorder) + } while ((value & mask) == original); + } +} + +static __rte_always_inline void +rte_wait_until_part_unequal_32(volatile uint32_t *addr, uint32_t mask, + uint32_t original, int memorder) +{ + uint32_t value; + + assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); + + __LOAD_EXC_32(addr, value, memorder) + if ((value & mask) == original) { + __SEVL() + do { + __WFE() + __LOAD_EXC_32(addr, value, memorder) + } while ((value & mask) == original); + } +} + +static __rte_always_inline void +rte_wait_until_part_unequal_64(volatile uint64_t *addr, uint64_t mask, + uint64_t original, int memorder) +{ + uint64_t value; + + assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); + + __LOAD_EXC_64(addr, value, memorder) + if ((value & mask) == original) { + __SEVL() + do { + __WFE() + __LOAD_EXC_64(addr, value, memorder) + } while ((value & mask) == original); + } +} + +#undef __LOAD_EXC_16 +#undef __LOAD_EXC_32 #undef __LOAD_EXC_64 #undef __SEVL diff --git a/lib/eal/include/generic/rte_pause.h b/lib/eal/include/generic/rte_pause.h index 668ee4a184..943a886f01 100644 --- a/lib/eal/include/generic/rte_pause.h +++ b/lib/eal/include/generic/rte_pause.h @@ -81,6 +81,222 @@ static __rte_always_inline void rte_wait_until_equal_64(volatile uint64_t *addr, uint64_t expected, int memorder); +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Wait for part bits of *addr to be equal with a 16-bit expected value, with + * a relaxed memory ordering model meaning the loads around this API can be + * reordered. + * + * @param addr + * A pointer to the memory location. + * @param mask + * value mask of a specific location + * @param expected + * A 16-bit expected value to be in the memory location. + * @param memorder + * Two different memory orders that can be specified: + * __ATOMIC_ACQUIRE and __ATOMIC_RELAXED. These map to + * C++11 memory orders with the same names, see the C++11 standard or + * the GCC wiki on atomic synchronization for detailed definition. + */ +__rte_experimental +static __rte_always_inline void +rte_wait_until_part_equal_16(volatile uint16_t *addr, uint16_t mask, + uint16_t expected, int memorder); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Wait for part bits of *addr to be equal with a 32-bit expected value, with + * a relaxed memory ordering model meaning the loads around this API can be + * reordered. + * + * @param addr + * A pointer to the memory location. + * @param mask + * value mask of a specific location + * @param expected + * A 32-bit expected value to be in the memory location. + * @param memorder + * Two different memory orders that can be specified: + * __ATOMIC_ACQUIRE and __ATOMIC_RELAXED. These map to + * C++11 memory orders with the same names, see the C++11 standard or + * the GCC wiki on atomic synchronization for detailed definition. + */ +__rte_experimental +static __rte_always_inline void +rte_wait_until_part_equal_32(volatile uint32_t *addr, uint32_t mask, + uint32_t expected, int memorder); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Wait for part bits of *addr to be equal with a 64-bit expected value, with + * a relaxed memory ordering model meaning the loads around this API can be + * reordered. + * + * @param addr + * A pointer to the memory location. + * @param mask + * value mask of a specific location + * @param expected + * A 64-bit expected value to be in the memory location. + * @param memorder + * Two different memory orders that can be specified: + * __ATOMIC_ACQUIRE and __ATOMIC_RELAXED. These map to + * C++11 memory orders with the same names, see the C++11 standard or + * the GCC wiki on atomic synchronization for detailed definition. + */ +__rte_experimental +static __rte_always_inline void +rte_wait_until_part_equal_64(volatile uint64_t *addr, uint64_t mask, + uint64_t expected, int memorder); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Wait for *addr to be unequal with a 16-bit original value, with a relaxed + * memory ordering model meaning the loads around this API can be reordered. + * + * @param addr + * A pointer to the memory location. + * @param original + * A 16-bit original value to be in the memory location. + * @param memorder + * Two different memory orders that can be specified: + * __ATOMIC_ACQUIRE and __ATOMIC_RELAXED. These map to + * C++11 memory orders with the same names, see the C++11 standard or + * the GCC wiki on atomic synchronization for detailed definition. + */ +__rte_experimental +static __rte_always_inline void +rte_wait_until_unequal_16(volatile uint16_t *addr, uint16_t original, + int memorder); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Wait for *addr to be unequal with a 32-bit original value, with a relaxed + * memory ordering model meaning the loads around this API can be reordered. + * + * @param addr + * A pointer to the memory location. + * @param original + * A 32-bit original value to be in the memory location. + * @param memorder + * Two different memory orders that can be specified: + * __ATOMIC_ACQUIRE and __ATOMIC_RELAXED. These map to + * C++11 memory orders with the same names, see the C++11 standard or + * the GCC wiki on atomic synchronization for detailed definition. + */ +__rte_experimental +static __rte_always_inline void +rte_wait_until_unequal_32(volatile uint32_t *addr, uint32_t original, + int memorder); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Wait for *addr to be unequal with a 64-bit original value, with a relaxed + * memory ordering model meaning the loads around this API can be reordered. + * + * @param addr + * A pointer to the memory location. + * @param original + * A 64-bit original value to be in the memory location. + * @param memorder + * Two different memory orders that can be specified: + * __ATOMIC_ACQUIRE and __ATOMIC_RELAXED. These map to + * C++11 memory orders with the same names, see the C++11 standard or + * the GCC wiki on atomic synchronization for detailed definition. + */ +__rte_experimental +static __rte_always_inline void +rte_wait_until_unequal_64(volatile uint64_t *addr, uint64_t original, + int memorder); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Wait for part bits of *addr to be unequal with a 16-bit original value, with + * a relaxed memory ordering model meaning the loads around this API can be + * reordered. + * + * @param addr + * A pointer to the memory location. + * @param mask + * value mask of a specific location + * @param original + * A 16-bit original value to be in the memory location. + * @param memorder + * Two different memory orders that can be specified: + * __ATOMIC_ACQUIRE and __ATOMIC_RELAXED. These map to + * C++11 memory orders with the same names, see the C++11 standard or + * the GCC wiki on atomic synchronization for detailed definition. + */ +__rte_experimental +static __rte_always_inline void +rte_wait_until_part_unequal_16(volatile uint16_t *addr, uint16_t mask, + uint16_t original, int memorder); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Wait for part bits of *addr to be unequal with a 32-bit original value, with + * a relaxed memory ordering model meaning the loads around this API can be + * reordered. + * + * @param addr + * A pointer to the memory location. + * @param mask + * value mask of a specific location + * @param original + * A 32-bit original value to be in the memory location. + * @param memorder + * Two different memory orders that can be specified: + * __ATOMIC_ACQUIRE and __ATOMIC_RELAXED. These map to + * C++11 memory orders with the same names, see the C++11 standard or + * the GCC wiki on atomic synchronization for detailed definition. + */ +__rte_experimental +static __rte_always_inline void +rte_wait_until_part_unequal_32(volatile uint32_t *addr, uint32_t mask, + uint32_t original, int memorder); + +/** + * @warning + * @b EXPERIMENTAL: this API may change without prior notice + * + * Wait for part bits of *addr to be unequal with a 64-bit original value, with + * a relaxed memory ordering model meaning the loads around this API can be + * reordered. + * + * @param addr + * A pointer to the memory location. + * @param mask + * value mask of a specific location + * @param original + * A 64-bit original value to be in the memory location. + * @param memorder + * Two different memory orders that can be specified: + * __ATOMIC_ACQUIRE and __ATOMIC_RELAXED. These map to + * C++11 memory orders with the same names, see the C++11 standard or + * the GCC wiki on atomic synchronization for detailed definition. + */ +__rte_experimental +static __rte_always_inline void +rte_wait_until_part_unequal_64(volatile uint64_t *addr, uint64_t mask, + uint64_t original, int memorder); + #ifndef RTE_WAIT_UNTIL_EQUAL_ARCH_DEFINED static __rte_always_inline void rte_wait_until_equal_16(volatile uint16_t *addr, uint16_t expected, @@ -111,6 +327,99 @@ rte_wait_until_equal_64(volatile uint64_t *addr, uint64_t expected, while (__atomic_load_n(addr, memorder) != expected) rte_pause(); } + +static __rte_always_inline void +rte_wait_until_part_equal_16(volatile uint16_t *addr, uint16_t mask, + uint16_t expected, int memorder) +{ + assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); + + while ((__atomic_load_n(addr, memorder) & mask) != expected) + rte_pause(); + +} + +static __rte_always_inline void +rte_wait_until_part_equal_32(volatile uint32_t *addr, uint32_t mask, + uint32_t expected, int memorder) +{ + assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); + + while ((__atomic_load_n(addr, memorder) & mask) != expected) + rte_pause(); + +} + +static __rte_always_inline void +rte_wait_until_part_equal_64(volatile uint64_t *addr, uint64_t mask, + uint64_t expected, int memorder) +{ + assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); + + while ((__atomic_load_n(addr, memorder) & mask) != expected) + rte_pause(); + +} + +static __rte_always_inline void +rte_wait_until_unequal_16(volatile uint16_t *addr, uint16_t original, + int memorder) +{ + assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); + + while (__atomic_load_n(addr, memorder) == original) + rte_pause(); +} + +static __rte_always_inline void +rte_wait_until_unequal_32(volatile uint32_t *addr, uint32_t original, + int memorder) +{ + assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); + + while (__atomic_load_n(addr, memorder) == original) + rte_pause(); +} + +static __rte_always_inline void +rte_wait_until_unequal_64(volatile uint64_t *addr, uint64_t original, + int memorder) +{ + assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); + + while (__atomic_load_n(addr, memorder) == original) + rte_pause(); +} + +static __rte_always_inline void +rte_wait_until_part_unequal_16(volatile uint16_t *addr, uint16_t mask, + uint16_t original, int memorder) +{ + assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); + + while ((__atomic_load_n(addr, memorder) & mask) == original) + rte_pause(); +} + +static __rte_always_inline void +rte_wait_until_part_unequal_32(volatile uint32_t *addr, uint32_t mask, + uint32_t original, int memorder) +{ + assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); + + while ((__atomic_load_n(addr, memorder) & mask) == original) + rte_pause(); +} + +static __rte_always_inline void +rte_wait_until_part_unequal_64(volatile uint64_t *addr, uint64_t mask, + uint64_t original, int memorder) +{ + assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); + + while ((__atomic_load_n(addr, memorder) & mask) == original) + rte_pause(); +} #endif #endif /* _RTE_PAUSE_H_ */ -- 2.25.1