From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 77A0EA0487 for ; Wed, 3 Jul 2019 10:59:04 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id EEAE11BC07; Wed, 3 Jul 2019 10:58:58 +0200 (CEST) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by dpdk.org (Postfix) with ESMTP id 83DD01B964 for ; Wed, 3 Jul 2019 10:58:55 +0200 (CEST) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 08B1E360; Wed, 3 Jul 2019 01:58:55 -0700 (PDT) Received: from net-arm-thunderx2.shanghai.arm.com (net-arm-thunderx2.shanghai.arm.com [10.169.40.40]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 505703F718; Wed, 3 Jul 2019 01:58:54 -0700 (PDT) From: Gavin Hu To: dev@dpdk.org Cc: nd@arm.com Date: Wed, 3 Jul 2019 16:58:32 +0800 Message-Id: <1562144316-14687-2-git-send-email-gavin.hu@arm.com> X-Mailer: git-send-email 2.7.4 In-Reply-To: <1562144316-14687-1-git-send-email-gavin.hu@arm.com> References: <1562144316-14687-1-git-send-email-gavin.hu@arm.com> In-Reply-To: <1561911676-37718-1-git-send-email-gavin.hu@arm.com> References: <1561911676-37718-1-git-send-email-gavin.hu@arm.com> Subject: [dpdk-dev] [RFC v2 1/5] eal: add the APIs to wait until equal X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" The rte_wait_until_equalxx APIs abstract the functionality of 'polling for a memory location to become equal to a given value'. Signed-off-by: Gavin Hu Reviewed-by: Ruifeng Wang Reviewed-by: Steve Capper Reviewed-by: Ola Liljedahl Reviewed-by: Honnappa Nagarahalli --- .../common/include/arch/arm/rte_atomic_64.h | 4 + .../common/include/arch/arm/rte_pause_64.h | 106 +++++++++++++++++++++ lib/librte_eal/common/include/generic/rte_pause.h | 39 +++++++- 3 files changed, 148 insertions(+), 1 deletion(-) diff --git a/lib/librte_eal/common/include/arch/arm/rte_atomic_64.h b/lib/librte_eal/common/include/arch/arm/rte_atomic_64.h index 97060e4..8d742c6 100644 --- a/lib/librte_eal/common/include/arch/arm/rte_atomic_64.h +++ b/lib/librte_eal/common/include/arch/arm/rte_atomic_64.h @@ -15,8 +15,12 @@ extern "C" { #include "generic/rte_atomic.h" +#ifndef dsb #define dsb(opt) asm volatile("dsb " #opt : : : "memory") +#endif +#ifndef dmb #define dmb(opt) asm volatile("dmb " #opt : : : "memory") +#endif #define rte_mb() dsb(sy) diff --git a/lib/librte_eal/common/include/arch/arm/rte_pause_64.h b/lib/librte_eal/common/include/arch/arm/rte_pause_64.h index 93895d3..1f7be0a 100644 --- a/lib/librte_eal/common/include/arch/arm/rte_pause_64.h +++ b/lib/librte_eal/common/include/arch/arm/rte_pause_64.h @@ -17,6 +17,112 @@ static inline void rte_pause(void) asm volatile("yield" ::: "memory"); } +#ifdef RTE_USE_WFE +/* Wait for *addr to be updated with expected value */ +static __rte_always_inline void +rte_wait_until_equal16(volatile uint16_t *addr, uint16_t expected, int memorder) +{ + uint16_t tmp; + if (memorder == __ATOMIC_RELAXED) + asm volatile( + "ldxrh %w[tmp], %w[addr]\n" + "cmp %w[tmp], %w[expected]\n" + "b.eq 2f\n" + "sevl\n" + "1: wfe\n" + "ldxrh %w[tmp], %w[addr]\n" + "cmp %w[tmp], %w[expected]\n" + "bne 1b\n" + "2:\n" + : [tmp] "=&r" (tmp) + : [addr] "Q"(*addr), [expected] "r"(expected) + : "cc", "memory"); + else + asm volatile( + "ldaxrh %w[tmp], %w[addr]\n" + "cmp %w[tmp], %w[expected]\n" + "b.eq 2f\n" + "sevl\n" + "1: wfe\n" + "ldaxrh %w[tmp], %w[addr]\n" + "cmp %w[tmp], %w[expected]\n" + "bne 1b\n" + "2:\n" + : [tmp] "=&r" (tmp) + : [addr] "Q"(*addr), [expected] "r"(expected) + : "cc", "memory"); +} + +static __rte_always_inline void +rte_wait_until_equal32(volatile uint32_t *addr, uint32_t expected, int memorder) +{ + uint32_t tmp; + if (memorder == __ATOMIC_RELAXED) + asm volatile( + "ldxr %w[tmp], %w[addr]\n" + "cmp %w[tmp], %w[expected]\n" + "b.eq 2f\n" + "sevl\n" + "1: wfe\n" + "ldxr %w[tmp], %w[addr]\n" + "cmp %w[tmp], %w[expected]\n" + "bne 1b\n" + "2:\n" + : [tmp] "=&r" (tmp) + : [addr] "Q"(*addr), [expected] "r"(expected) + : "cc", "memory"); + else + asm volatile( + "ldaxr %w[tmp], %w[addr]\n" + "cmp %w[tmp], %w[expected]\n" + "b.eq 2f\n" + "sevl\n" + "1: wfe\n" + "ldaxr %w[tmp], %w[addr]\n" + "cmp %w[tmp], %w[expected]\n" + "bne 1b\n" + "2:\n" + : [tmp] "=&r" (tmp) + : [addr] "Q"(*addr), [expected] "r"(expected) + : "cc", "memory"); +} + +static __rte_always_inline void +rte_wait_until_equal64(volatile uint64_t *addr, uint64_t expected, int memorder) +{ + uint64_t tmp; + if (memorder == __ATOMIC_RELAXED) + asm volatile( + "ldxr %x[tmp], %x[addr]\n" + "cmp %x[tmp], %x[expected]\n" + "b.eq 2f\n" + "sevl\n" + "1: wfe\n" + "ldxr %x[tmp], %x[addr]\n" + "cmp %x[tmp], %x[expected]\n" + "bne 1b\n" + "2:\n" + : [tmp] "=&r" (tmp) + : [addr] "Q"(*addr), [expected] "r"(expected) + : "cc", "memory"); + else + asm volatile( + "ldaxr %x[tmp], %x[addr]\n" + "cmp %x[tmp], %x[expected]\n" + "b.eq 2f\n" + "sevl\n" + "1: wfe\n" + "ldaxr %x[tmp], %x[addr]\n" + "cmp %x[tmp], %x[expected]\n" + "bne 1b\n" + "2:\n" + : [tmp] "=&r" (tmp) + : [addr] "Q"(*addr), [expected] "r"(expected) + : "cc", "memory"); +} + +#endif + #ifdef __cplusplus } #endif diff --git a/lib/librte_eal/common/include/generic/rte_pause.h b/lib/librte_eal/common/include/generic/rte_pause.h index 52bd4db..8f5f025 100644 --- a/lib/librte_eal/common/include/generic/rte_pause.h +++ b/lib/librte_eal/common/include/generic/rte_pause.h @@ -4,7 +4,6 @@ #ifndef _RTE_PAUSE_H_ #define _RTE_PAUSE_H_ - /** * @file * @@ -12,6 +11,10 @@ * */ +#include +#include +#include + /** * Pause CPU execution for a short while * @@ -20,4 +23,38 @@ */ static inline void rte_pause(void); +#if !defined(RTE_USE_WFE) +#ifdef RTE_USE_C11_MEM_MODEL +#define __rte_wait_until_equal(addr, expected, memorder) do {\ + while (__atomic_load_n(addr, memorder) != expected) \ + rte_pause();\ +} while (0) +#else +#define __rte_wait_until_equal(addr, expected, memorder) do {\ + while (*addr != expected)\ + rte_pause();\ + if (memorder != __ATOMIC_RELAXED)\ + rte_smp_rmb();\ +} while (0) +#endif + +static __rte_always_inline void +rte_wait_until_equal16(volatile uint16_t *addr, uint16_t expected, int memorder) +{ + __rte_wait_until_equal(addr, expected, memorder); +} + +static __rte_always_inline void +rte_wait_until_equal32(volatile uint32_t *addr, uint32_t expected, int memorder) +{ + __rte_wait_until_equal(addr, expected, memorder); +} + +static __rte_always_inline void +rte_wait_until_equal64(volatile uint64_t *addr, uint64_t expected, int memorder) +{ + __rte_wait_until_equal(addr, expected, memorder); +} +#endif /* RTE_USE_WFE */ + #endif /* _RTE_PAUSE_H_ */ -- 2.7.4