From: Jerin Jacob <jerin.jacob@caviumnetworks.com>
To: David Hunt <david.hunt@intel.com>
Cc: dev@dpdk.org
Subject: Re: [dpdk-dev] [PATCH v3 1/6] eal/arm: add 64-bit armv8 version of rte_memcpy.h
Date: Mon, 2 Nov 2015 10:27:29 +0530 [thread overview]
Message-ID: <20151102045728.GB16413@localhost.localdomain> (raw)
In-Reply-To: <1446212959-19832-2-git-send-email-david.hunt@intel.com>
On Fri, Oct 30, 2015 at 01:49:14PM +0000, David Hunt wrote:
> Signed-off-by: David Hunt <david.hunt@intel.com>
> ---
> .../common/include/arch/arm/rte_memcpy.h | 4 +
> .../common/include/arch/arm/rte_memcpy_64.h | 308 +++++++++++++++++++++
> 2 files changed, 312 insertions(+)
> create mode 100644 lib/librte_eal/common/include/arch/arm/rte_memcpy_64.h
>
> diff --git a/lib/librte_eal/common/include/arch/arm/rte_memcpy.h b/lib/librte_eal/common/include/arch/arm/rte_memcpy.h
> index d9f5bf1..1d562c3 100644
> --- a/lib/librte_eal/common/include/arch/arm/rte_memcpy.h
> +++ b/lib/librte_eal/common/include/arch/arm/rte_memcpy.h
> @@ -33,6 +33,10 @@
> #ifndef _RTE_MEMCPY_ARM_H_
> #define _RTE_MEMCPY_ARM_H_
>
> +#ifdef RTE_ARCH_64
> +#include <rte_memcpy_64.h>
> +#else
> #include <rte_memcpy_32.h>
> +#endif
>
> #endif /* _RTE_MEMCPY_ARM_H_ */
> diff --git a/lib/librte_eal/common/include/arch/arm/rte_memcpy_64.h b/lib/librte_eal/common/include/arch/arm/rte_memcpy_64.h
> new file mode 100644
> index 0000000..6d85113
> --- /dev/null
> +++ b/lib/librte_eal/common/include/arch/arm/rte_memcpy_64.h
> @@ -0,0 +1,308 @@
> +/*
> + * BSD LICENSE
> + *
> + * Copyright (C) IBM Corporation 2014.
> + *
> + * Redistribution and use in source and binary forms, with or without
> + * modification, are permitted provided that the following conditions
> + * are met:
> + *
> + * * Redistributions of source code must retain the above copyright
> + * notice, this list of conditions and the following disclaimer.
> + * * Redistributions in binary form must reproduce the above copyright
> + * notice, this list of conditions and the following disclaimer in
> + * the documentation and/or other materials provided with the
> + * distribution.
> + * * Neither the name of IBM Corporation nor the names of its
> + * contributors may be used to endorse or promote products derived
> + * from this software without specific prior written permission.
> + *
> + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> +*/
> +
> +#ifndef _RTE_MEMCPY_ARM_64_H_
> +#define _RTE_MEMCPY_ARM_64_H_
> +
> +#include <stdint.h>
> +#include <string.h>
> +
> +#ifdef __cplusplus
> +extern "C" {
> +#endif
> +
> +#include "generic/rte_memcpy.h"
> +
> +#ifdef __ARM_NEON_FP
SIMD is not optional in armv8 spec.So every armv8 machine will have
SIMD instruction unlike armv7.More over LDP/STP instruction is
not part of SIMD.So this check is not required or it can
be replaced with a check that select memcpy from either libc or this specific
implementation
> +
> +/* ARM NEON Intrinsics are used to copy data */
> +#include <arm_neon.h>
> +
> +static inline void
> +rte_mov16(uint8_t *dst, const uint8_t *src)
> +{
> + asm volatile("LDP d0, d1, [%0]\n\t"
> + "STP d0, d1, [%1]\n\t"
> + : : "r" (src), "r" (dst) :
> + );
> +}
IMO, no need to hardcode registers used for the mem move(d0, d1).
Let compiler schedule the registers for better performance.
> +
> +static inline void
> +rte_mov32(uint8_t *dst, const uint8_t *src)
> +{
> + asm volatile("LDP q0, q1, [%0]\n\t"
> + "STP q0, q1, [%1]\n\t"
> + : : "r" (src), "r" (dst) :
> + );
> +}
> +
> +static inline void
> +rte_mov48(uint8_t *dst, const uint8_t *src)
> +{
> + asm volatile("LDP q0, q1, [%0]\n\t"
> + "STP q0, q1, [%1]\n\t"
> + "LDP d0, d1, [%0 , #32]\n\t"
> + "STP d0, d1, [%1 , #32]\n\t"
> + : : "r" (src), "r" (dst) :
> + );
> +}
> +
> +static inline void
> +rte_mov64(uint8_t *dst, const uint8_t *src)
> +{
> + asm volatile("LDP q0, q1, [%0]\n\t"
> + "STP q0, q1, [%1]\n\t"
> + "LDP q0, q1, [%0 , #32]\n\t"
> + "STP q0, q1, [%1 , #32]\n\t"
> + : : "r" (src), "r" (dst) :
> + );
> +}
> +
> +static inline void
> +rte_mov128(uint8_t *dst, const uint8_t *src)
> +{
> + asm volatile("LDP q0, q1, [%0]\n\t"
> + "STP q0, q1, [%1]\n\t"
> + "LDP q0, q1, [%0 , #32]\n\t"
> + "STP q0, q1, [%1 , #32]\n\t"
> + "LDP q0, q1, [%0 , #64]\n\t"
> + "STP q0, q1, [%1 , #64]\n\t"
> + "LDP q0, q1, [%0 , #96]\n\t"
> + "STP q0, q1, [%1 , #96]\n\t"
> + : : "r" (src), "r" (dst) :
> + );
> +}
> +
> +static inline void
> +rte_mov256(uint8_t *dst, const uint8_t *src)
> +{
> + asm volatile("LDP q0, q1, [%0]\n\t"
> + "STP q0, q1, [%1]\n\t"
> + "LDP q0, q1, [%0 , #32]\n\t"
> + "STP q0, q1, [%1 , #32]\n\t"
> + "LDP q0, q1, [%0 , #64]\n\t"
> + "STP q0, q1, [%1 , #64]\n\t"
> + "LDP q0, q1, [%0 , #96]\n\t"
> + "STP q0, q1, [%1 , #96]\n\t"
> + "LDP q0, q1, [%0 , #128]\n\t"
> + "STP q0, q1, [%1 , #128]\n\t"
> + "LDP q0, q1, [%0 , #160]\n\t"
> + "STP q0, q1, [%1 , #160]\n\t"
> + "LDP q0, q1, [%0 , #192]\n\t"
> + "STP q0, q1, [%1 , #192]\n\t"
> + "LDP q0, q1, [%0 , #224]\n\t"
> + "STP q0, q1, [%1 , #224]\n\t"
> + : : "r" (src), "r" (dst) :
> + );
> +}
> +
> +#define rte_memcpy(dst, src, n) \
> + ({ (__builtin_constant_p(n)) ? \
> + memcpy((dst), (src), (n)) : \
> + rte_memcpy_func((dst), (src), (n)); })
> +
> +static inline void *
> +rte_memcpy_func(void *dst, const void *src, size_t n)
> +{
> + void *ret = dst;
> +
> + /* We can't copy < 16 bytes using XMM registers so do it manually. */
> + if (n < 16) {
> + if (n & 0x01) {
> + *(uint8_t *)dst = *(const uint8_t *)src;
> + dst = (uint8_t *)dst + 1;
> + src = (const uint8_t *)src + 1;
> + }
> + if (n & 0x02) {
> + *(uint16_t *)dst = *(const uint16_t *)src;
> + dst = (uint16_t *)dst + 1;
> + src = (const uint16_t *)src + 1;
> + }
> + if (n & 0x04) {
> + *(uint32_t *)dst = *(const uint32_t *)src;
> + dst = (uint32_t *)dst + 1;
> + src = (const uint32_t *)src + 1;
> + }
> + if (n & 0x08)
> + *(uint64_t *)dst = *(const uint64_t *)src;
> + return ret;
> + }
> +
> + /* Special fast cases for <= 128 bytes */
> + if (n <= 32) {
> + rte_mov16((uint8_t *)dst, (const uint8_t *)src);
> + rte_mov16((uint8_t *)dst - 16 + n,
> + (const uint8_t *)src - 16 + n);
> + return ret;
> + }
> +
> + if (n <= 64) {
> + rte_mov32((uint8_t *)dst, (const uint8_t *)src);
> + rte_mov32((uint8_t *)dst - 32 + n,
> + (const uint8_t *)src - 32 + n);
> + return ret;
> + }
> +
> + if (n <= 128) {
> + rte_mov64((uint8_t *)dst, (const uint8_t *)src);
> + rte_mov64((uint8_t *)dst - 64 + n,
> + (const uint8_t *)src - 64 + n);
> + return ret;
> + }
> +
> + /*
> + * For large copies > 128 bytes. This combination of 256, 64 and 16 byte
> + * copies was found to be faster than doing 128 and 32 byte copies as
> + * well.
> + */
> + for ( ; n >= 256; n -= 256) {
There is room for prefetching the next cacheline based on the cache line
size.
> + rte_mov256((uint8_t *)dst, (const uint8_t *)src);
> + dst = (uint8_t *)dst + 256;
> + src = (const uint8_t *)src + 256;
> + }
> +
> + /*
> + * We split the remaining bytes (which will be less than 256) into
> + * 64byte (2^6) chunks.
> + * Using incrementing integers in the case labels of a switch statement
> + * enourages the compiler to use a jump table. To get incrementing
> + * integers, we shift the 2 relevant bits to the LSB position to first
> + * get decrementing integers, and then subtract.
> + */
> + switch (3 - (n >> 6)) {
> + case 0x00:
> + rte_mov64((uint8_t *)dst, (const uint8_t *)src);
> + n -= 64;
> + dst = (uint8_t *)dst + 64;
> + src = (const uint8_t *)src + 64; /* fallthrough */
> + case 0x01:
> + rte_mov64((uint8_t *)dst, (const uint8_t *)src);
> + n -= 64;
> + dst = (uint8_t *)dst + 64;
> + src = (const uint8_t *)src + 64; /* fallthrough */
> + case 0x02:
> + rte_mov64((uint8_t *)dst, (const uint8_t *)src);
> + n -= 64;
> + dst = (uint8_t *)dst + 64;
> + src = (const uint8_t *)src + 64; /* fallthrough */
> + default:
> + break;
> + }
> +
> + /*
> + * We split the remaining bytes (which will be less than 64) into
> + * 16byte (2^4) chunks, using the same switch structure as above.
> + */
> + switch (3 - (n >> 4)) {
> + case 0x00:
> + rte_mov16((uint8_t *)dst, (const uint8_t *)src);
> + n -= 16;
> + dst = (uint8_t *)dst + 16;
> + src = (const uint8_t *)src + 16; /* fallthrough */
> + case 0x01:
> + rte_mov16((uint8_t *)dst, (const uint8_t *)src);
> + n -= 16;
> + dst = (uint8_t *)dst + 16;
> + src = (const uint8_t *)src + 16; /* fallthrough */
> + case 0x02:
> + rte_mov16((uint8_t *)dst, (const uint8_t *)src);
> + n -= 16;
> + dst = (uint8_t *)dst + 16;
> + src = (const uint8_t *)src + 16; /* fallthrough */
> + default:
> + break;
> + }
> +
> + /* Copy any remaining bytes, without going beyond end of buffers */
> + if (n != 0)
> + rte_mov16((uint8_t *)dst - 16 + n,
> + (const uint8_t *)src - 16 + n);
> + return ret;
> +}
> +
> +#else
> +
> +static inline void
> +rte_mov16(uint8_t *dst, const uint8_t *src)
> +{
> + memcpy(dst, src, 16);
> +}
> +
> +static inline void
> +rte_mov32(uint8_t *dst, const uint8_t *src)
> +{
> + memcpy(dst, src, 32);
> +}
> +
> +static inline void
> +rte_mov48(uint8_t *dst, const uint8_t *src)
> +{
> + memcpy(dst, src, 48);
> +}
> +
> +static inline void
> +rte_mov64(uint8_t *dst, const uint8_t *src)
> +{
> + memcpy(dst, src, 64);
> +}
> +
> +static inline void
> +rte_mov128(uint8_t *dst, const uint8_t *src)
> +{
> + memcpy(dst, src, 128);
> +}
> +
> +static inline void
> +rte_mov256(uint8_t *dst, const uint8_t *src)
> +{
> + memcpy(dst, src, 256);
> +}
> +
> +static inline void *
> +rte_memcpy(void *dst, const void *src, size_t n)
> +{
> + return memcpy(dst, src, n);
> +}
> +
> +static inline void *
> +rte_memcpy_func(void *dst, const void *src, size_t n)
> +{
> + return memcpy(dst, src, n);
> +}
> +
> +#endif /* __ARM_NEON_FP */
> +
> +#ifdef __cplusplus
> +}
> +#endif
> +
> +#endif /* _RTE_MEMCPY_ARM_64_H_ */
> --
> 1.9.1
>
next prev parent reply other threads:[~2015-11-02 4:58 UTC|newest]
Thread overview: 18+ messages / expand[flat|nested] mbox.gz Atom feed top
2015-10-30 13:49 [dpdk-dev] [PATCH v3 0/6] ARMv8 additions to ARMv7 support David Hunt
2015-10-30 13:49 ` [dpdk-dev] [PATCH v3 1/6] eal/arm: add 64-bit armv8 version of rte_memcpy.h David Hunt
2015-11-02 4:57 ` Jerin Jacob [this message]
2015-11-02 12:22 ` Hunt, David
2015-11-02 12:45 ` Jan Viktorin
2015-11-02 12:57 ` Jerin Jacob
2015-11-02 15:26 ` Hunt, David
2015-11-02 15:36 ` Jan Viktorin
2015-11-02 15:49 ` Hunt, David
2015-11-02 16:29 ` Jerin Jacob
2015-11-02 17:29 ` Jan Viktorin
2015-10-30 13:49 ` [dpdk-dev] [PATCH v3 2/6] eal/arm: add 64-bit armv8 version of rte_prefetch.h David Hunt
2015-10-30 13:49 ` [dpdk-dev] [PATCH v3 3/6] eal/arm: add 64-bit armv8 version of rte_cycles.h David Hunt
2015-11-02 5:15 ` Jerin Jacob
2015-10-30 13:49 ` [dpdk-dev] [PATCH v3 4/6] eal/arm: fix 64-bit armv8 compilation of rte_cpuflags.h David Hunt
2015-10-30 13:49 ` [dpdk-dev] [PATCH v3 5/6] mk: add support for armv8 on top of armv7 David Hunt
2015-11-02 4:43 ` Jerin Jacob
2015-10-30 13:49 ` [dpdk-dev] [PATCH v3 6/6] test: add checks for cpu flags on armv8 David Hunt
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20151102045728.GB16413@localhost.localdomain \
--to=jerin.jacob@caviumnetworks.com \
--cc=david.hunt@intel.com \
--cc=dev@dpdk.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).