DPDK patches and discussions
 help / color / mirror / Atom feed
* Re: [dpdk-dev] [PATCH v1 07/12] eal/arm: vector memcpy for ARM
@ 2015-10-03  9:12 Jan Viktorin
  0 siblings, 0 replies; 2+ messages in thread
From: Jan Viktorin @ 2015-10-03  9:12 UTC (permalink / raw)
  To: dev

‎I am missing the percent sign (%) for the measurments...

So it's eg. by 4.9 %‎ faster then memcpy...

Regards
Jan Viktorin
  Původní zpráva  
Od: Jan Viktorin
Odesláno: sobota, 3. října 2015 9:58
Komu: dev@dpdk.org
Kopie: Vlastimil Kosar; Jan Viktorin
Předmět: [PATCH v1 07/12] eal/arm: vector memcpy for ARM

From: Vlastimil Kosar <kosar@rehivetech.com>

The SSE based memory copy in DPDK only support x86. This patch
adds ARM NEON based memory copy functions for ARM architecture.

The implementation improves memory copy of short or well aligned
data buffers. The following measurements show improvements over
the libc memcpy on Cortex CPUs.

Length (B) a15 a7 a9
1 4.9 15.2 3.2
7 56.9 48.2 40.3
8 37.3 39.8 29.6
9 69.3 38.7 33.9
15 60.8 35.3 23.7
16 50.6 35.9 35.0
17 57.7 35.7 31.1
31 16.0 23.3 9.0
32 65.9 13.5 21.4
33 3.9 10.3 -3.7
63 2.0 12.9 -2.0
64 66.5 0.0 16.5
65 2.7 7.6 -35.6
127 0.1 4.5 -18.9
128 66.2 1.5 -51.4
129 -0.8 3.2 -35.8
255 -3.1 -0.9 -69.1
256 67.9 1.2 7.2
257 -3.6 -1.9 -36.9
320 67.7 1.4 0.0
384 66.8 1.4 -14.2
511 -44.9 -2.3 -41.9
512 67.3 1.4 -6.8
513 -41.7 -3.0 -36.2
1023 -82.4 -2.8 -41.2
1024 68.3 1.4 -11.6
1025 -80.1 -3.3 -38.1
1518 -47.3 -5.0 -38.3
1522 -48.3 -6.0 -37.9
1600 65.4 1.3 -27.3
2048 59.5 1.5 -10.9
3072 52.3 1.5 -12.2
4096 45.3 1.4 -12.5
5120 40.6 1.5 -14.5
6144 35.4 1.4 -13.4
7168 32.9 1.4 -13.9
8192 28.2 1.4 -15.1

Signed-off-by: Vlastimil Kosar <kosar@rehivetech.com>
Signed-off-by: Jan Viktorin <viktorin@rehivetech.com>
---
.../common/include/arch/arm/rte_memcpy.h | 270 +++++++++++++++++++++
1 file changed, 270 insertions(+)
create mode 100644 lib/librte_eal/common/include/arch/arm/rte_memcpy.h

diff --git a/lib/librte_eal/common/include/arch/arm/rte_memcpy.h b/lib/librte_eal/common/include/arch/arm/rte_memcpy.h
new file mode 100644
index 0000000..ac885e9
--- /dev/null
+++ b/lib/librte_eal/common/include/arch/arm/rte_memcpy.h
@@ -0,0 +1,270 @@
+/*
+ * BSD LICENSE
+ *
+ * Copyright(c) 2015 RehiveTech. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of RehiveTech nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RTE_MEMCPY_ARM_H_
+#define _RTE_MEMCPY_ARM_H_
+
+#include <stdint.h>
+#include <string.h>
+/* ARM NEON Intrinsics are used to copy data */
+#include <arm_neon.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "generic/rte_memcpy.h"
+
+static inline void
+rte_mov16(uint8_t *dst, const uint8_t *src)
+{
+	vst1q_u8(dst, vld1q_u8(src));
+}
+
+static inline void
+rte_mov32(uint8_t *dst, const uint8_t *src)
+{
+	asm volatile ("vld1.8 {d0-d3}, [%[src]]\n\t"
+	 "vst1.8 {d0-d3}, [%[dst]]\n\t"
+	 : [src] "+r" (src), [dst] "+r" (dst)
+	 : : "memory", "d0", "d1", "d2", "d3");
+}
+
+static inline void
+rte_mov48(uint8_t *dst, const uint8_t *src)
+{
+	asm volatile ("vld1.8 {d0-d3}, [%[src]]!\n\t"
+	 "vld1.8 {d4-d5}, [%[src]]\n\t"
+	 "vst1.8 {d0-d3}, [%[dst]]!\n\t"
+	 "vst1.8 {d4-d5}, [%[dst]]\n\t"
+	 : [src] "+r" (src), [dst] "+r" (dst)
+	 : : "memory", "d0", "d1", "d2", "d3", "d4", "d5");
+}
+
+static inline void
+rte_mov64(uint8_t *dst, const uint8_t *src)
+{
+	asm volatile ("vld1.8 {d0-d3}, [%[src]]!\n\t"
+	 "vld1.8 {d4-d7}, [%[src]]\n\t"
+	 "vst1.8 {d0-d3}, [%[dst]]!\n\t"
+	 "vst1.8 {d4-d7}, [%[dst]]\n\t"
+	 : [src] "+r" (src), [dst] "+r" (dst)
+	 : : "memory", "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7");
+}
+
+static inline void
+rte_mov128(uint8_t *dst, const uint8_t *src)
+{
+	asm volatile ("pld [%[src], #64]" : : [src] "r" (src));
+	asm volatile ("vld1.8 {d0-d3}, [%[src]]!\n\t"
+	 "vld1.8 {d4-d7}, [%[src]]!\n\t"
+	 "vld1.8 {d8-d11}, [%[src]]!\n\t"
+	 "vld1.8 {d12-d15}, [%[src]]\n\t"
+	 "vst1.8 {d0-d3}, [%[dst]]!\n\t"
+	 "vst1.8 {d4-d7}, [%[dst]]!\n\t"
+	 "vst1.8 {d8-d11}, [%[dst]]!\n\t"
+	 "vst1.8 {d12-d15}, [%[dst]]\n\t"
+	 : [src] "+r" (src), [dst] "+r" (dst)
+	 : : "memory", "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7",
+	 "d8", "d9", "d10", "d11", "d12", "d13", "d14", "d15");
+}
+
+static inline void
+rte_mov256(uint8_t *dst, const uint8_t *src)
+{
+	asm volatile ("pld [%[src], #64]" : : [src] "r" (src));
+	asm volatile ("pld [%[src], #128]" : : [src] "r" (src));
+	asm volatile ("pld [%[src], #192]" : : [src] "r" (src));
+	asm volatile ("pld [%[src], #256]" : : [src] "r" (src));
+	asm volatile ("pld [%[src], #320]" : : [src] "r" (src));
+	asm volatile ("pld [%[src], #384]" : : [src] "r" (src));
+	asm volatile ("pld [%[src], #448]" : : [src] "r" (src));
+	asm volatile ("vld1.8 {d0-d3}, [%[src]]!\n\t"
+	 "vld1.8 {d4-d7}, [%[src]]!\n\t"
+	 "vld1.8 {d8-d11}, [%[src]]!\n\t"
+	 "vld1.8 {d12-d15}, [%[src]]!\n\t"
+	 "vld1.8 {d16-d19}, [%[src]]!\n\t"
+	 "vld1.8 {d20-d23}, [%[src]]!\n\t"
+	 "vld1.8 {d24-d27}, [%[src]]!\n\t"
+	 "vld1.8 {d28-d31}, [%[src]]\n\t"
+	 "vst1.8 {d0-d3}, [%[dst]]!\n\t"
+	 "vst1.8 {d4-d7}, [%[dst]]!\n\t"
+	 "vst1.8 {d8-d11}, [%[dst]]!\n\t"
+	 "vst1.8 {d12-d15}, [%[dst]]!\n\t"
+	 "vst1.8 {d16-d19}, [%[dst]]!\n\t"
+	 "vst1.8 {d20-d23}, [%[dst]]!\n\t"
+	 "vst1.8 {d24-d27}, [%[dst]]!\n\t"
+	 "vst1.8 {d28-d31}, [%[dst]]!\n\t"
+	 : [src] "+r" (src), [dst] "+r" (dst)
+	 : : "memory", "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7",
+	 "d8", "d9", "d10", "d11", "d12", "d13", "d14", "d15",
+	 "d16", "d17", "d18", "d19", "d20", "d21", "d22", "d23",
+	 "d24", "d25", "d26", "d27", "d28", "d29", "d30", "d31");
+}
+
+#define rte_memcpy(dst, src, n) \
+	({ (__builtin_constant_p(n)) ? \
+	memcpy((dst), (src), (n)) : \
+	rte_memcpy_func((dst), (src), (n)); })
+
+static inline void *
+rte_memcpy_func(void *dst, const void *src, size_t n)
+{
+	void *ret = dst;
+
+	/* We can't copy < 16 bytes using XMM registers so do it manually. */
+	if (n < 16) {
+	 if (n & 0x01) {
+	 *(uint8_t *)dst = *(const uint8_t *)src;
+	 dst = (uint8_t *)dst + 1;
+	 src = (const uint8_t *)src + 1;
+	 }
+	 if (n & 0x02) {
+	 *(uint16_t *)dst = *(const uint16_t *)src;
+	 dst = (uint16_t *)dst + 1;
+	 src = (const uint16_t *)src + 1;
+	 }
+	 if (n & 0x04) {
+	 *(uint32_t *)dst = *(const uint32_t *)src;
+	 dst = (uint32_t *)dst + 1;
+	 src = (const uint32_t *)src + 1;
+	 }
+	 if (n & 0x08) {
+	 /* ARMv7 can not handle unaligned access to long long
+	 * (uint64_t). Therefore two uint32_t operations are used.
+	 * TODO: use NEON too?
+	 */
+	 *(uint32_t *)dst = *(const uint32_t *)src;
+	 dst = (uint32_t *)dst + 1;
+	 src = (const uint32_t *)src + 1;
+	 *(uint32_t *)dst = *(const uint32_t *)src;
+	 }
+	 return ret;
+	}
+
+	/* Special fast cases for <= 128 bytes */
+	if (n <= 32) {
+	 rte_mov16((uint8_t *)dst, (const uint8_t *)src);
+	 rte_mov16((uint8_t *)dst - 16 + n,
+	 (const uint8_t *)src - 16 + n);
+	 return ret;
+	}
+
+	if (n <= 64) {
+	 rte_mov32((uint8_t *)dst, (const uint8_t *)src);
+	 rte_mov32((uint8_t *)dst - 32 + n,
+	 (const uint8_t *)src - 32 + n);
+	 return ret;
+	}
+
+	if (n <= 128) {
+	 rte_mov64((uint8_t *)dst, (const uint8_t *)src);
+	 rte_mov64((uint8_t *)dst - 64 + n,
+	 (const uint8_t *)src - 64 + n);
+	 return ret;
+	}
+
+	/*
+	 * For large copies > 128 bytes. This combination of 256, 64 and 16 byte
+	 * copies was found to be faster than doing 128 and 32 byte copies as
+	 * well.
+	 */
+	for ( ; n >= 256; n -= 256) {
+	 rte_mov256((uint8_t *)dst, (const uint8_t *)src);
+	 dst = (uint8_t *)dst + 256;
+	 src = (const uint8_t *)src + 256;
+	}
+
+	/*
+	 * We split the remaining bytes (which will be less than 256) into
+	 * 64byte (2^6) chunks.
+	 * Using incrementing integers in the case labels of a switch statement
+	 * enourages the compiler to use a jump table. To get incrementing
+	 * integers, we shift the 2 relevant bits to the LSB position to first
+	 * get decrementing integers, and then subtract.
+	 */
+	switch (3 - (n >> 6)) {
+	case 0x00:
+	 rte_mov64((uint8_t *)dst, (const uint8_t *)src);
+	 n -= 64;
+	 dst = (uint8_t *)dst + 64;
+	 src = (const uint8_t *)src + 64; /* fallthrough */
+	case 0x01:
+	 rte_mov64((uint8_t *)dst, (const uint8_t *)src);
+	 n -= 64;
+	 dst = (uint8_t *)dst + 64;
+	 src = (const uint8_t *)src + 64; /* fallthrough */
+	case 0x02:
+	 rte_mov64((uint8_t *)dst, (const uint8_t *)src);
+	 n -= 64;
+	 dst = (uint8_t *)dst + 64;
+	 src = (const uint8_t *)src + 64; /* fallthrough */
+	default:
+	 ;
+	}
+
+	/*
+	 * We split the remaining bytes (which will be less than 64) into
+	 * 16byte (2^4) chunks, using the same switch structure as above.
+	 */
+	switch (3 - (n >> 4)) {
+	case 0x00:
+	 rte_mov16((uint8_t *)dst, (const uint8_t *)src);
+	 n -= 16;
+	 dst = (uint8_t *)dst + 16;
+	 src = (const uint8_t *)src + 16; /* fallthrough */
+	case 0x01:
+	 rte_mov16((uint8_t *)dst, (const uint8_t *)src);
+	 n -= 16;
+	 dst = (uint8_t *)dst + 16;
+	 src = (const uint8_t *)src + 16; /* fallthrough */
+	case 0x02:
+	 rte_mov16((uint8_t *)dst, (const uint8_t *)src);
+	 n -= 16;
+	 dst = (uint8_t *)dst + 16;
+	 src = (const uint8_t *)src + 16; /* fallthrough */
+	default:
+	 ;
+	}
+
+	/* Copy any remaining bytes, without going beyond end of buffers */
+	if (n != 0)
+	 rte_mov16((uint8_t *)dst - 16 + n,
+	 (const uint8_t *)src - 16 + n);
+	return ret;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_MEMCPY_ARM_H_ */
-- 
2.5.2

^ permalink raw reply	[flat|nested] 2+ messages in thread
* [dpdk-dev] [PATCH v1 00/12] Support for ARM(v7)
@ 2015-10-03  8:58 Jan Viktorin
  2015-10-03  8:58 ` [dpdk-dev] [PATCH v1 07/12] eal/arm: vector memcpy for ARM Jan Viktorin
  0 siblings, 1 reply; 2+ messages in thread
From: Jan Viktorin @ 2015-10-03  8:58 UTC (permalink / raw)
  To: dev; +Cc: Jan Viktorin

Dear DPDK community,

I am proposing a patch series with support of the ARMv7 architecture
for DPDK. The patch series does not introduce any PMD driver. It is
possible to compile it, boot it and test it with some virtual PMD (eg.
pcap). It is rebased on top of v2.1.0.

All but the last two patches (11, 12) are quite staightforward
and usually based on the ppc_64 architecture. Notes:

* we test on Cortex-A9 (mostly Xilinx Zynq at the moment)
* atomic operations and spinlocks are implemented by (GCC) intrinsics
* cpu cycle is implemented by clock_gettime because there is no
  standard 64-bit counter available
* we have to set -Wno-error to pass the build process because there are
  quite a lot of alignment problems reported (we didn't find any real issues
  so far)

The last two patches (11, 12) are not to be merged into mainline. They
are just a temporary workaround for the two libraries (ACL, LPM) which
heavily utilizes the SSE... It is not possible to easily convert the
SSE calls to the NEON SIMD operations.

============

It is important to note that the current Linux Kernel does not contain
the support for huge tables for non-LPAE ARM architectures (Cortex-A9).
There is a patch available on the Internet but it is not going to be
merged for now (4/2014):

 http://thread.gmane.org/gmane.linux.kernel.mm/115788

We ported this patch to 3.18 and it can improve the performance. Here
follow results for our tests of several algorithms showing the execution
time reduction:

CPU median 3x3        -  0.2 %
NEON median 3x3       - 19.5 %
Random read           -  0.0 %
Random write          -  6.2 %
Matrix multiplication - 31.0 %
NEON copy             -  4.2 %

============

We are working on the PMD + kernel-support part. At the moment, we have
a working PMD for Xilinx Zynq's EMAC. However, it uses some dirty features.
We have to rethink it a bit before going to the mainline. We are facing some
problems during the implementation (some are already being solved in the
mailing-list):

* rte_eth_dev is defined as a PCI device. As ARMs are SoCs with integrated
  EMAC on the chip and an external phyter, we need a different approach.
  There can be an ARM computer with PCI-E but then you put there a network
  card and use a different kind of driver (but this is not very common
  at the moment).
* ARM does not have coherent memory for DMA transfers. It is possible to
  allocate non-cachable memory (DMA transfers can be as fast as possible)
  but it slows down the payload processing on CPU. For this purpose, we
  have to call dma_map/unmap_* in kernel. A custom kernel driver is needed
  and it should not be the UIO because it is quite limited (almost
  non-extendable mmap, no support for custom ioctl and write).
* We are not going to put the PHY layer into userspace, so it will stay
  in the kernel. There is also a need for the CLK control (clock gating)
  in the PMD.

Regards
Jan Viktorin


Jan Viktorin (2):
  eal/arm: rwlock support for ARM
  gcc/arm: avoid alignment errors to break build

Vlastimil Kosar (10):
  mk: Introduce ARMv7 architecture
  eal/arm: atomic operations for ARM
  eal/arm: byte order operations for ARM
  eal/arm: cpu cycle operations for ARM
  eal/arm: prefetch operations for ARM
  eal/arm: spinlock operations for ARM (without HTM)
  eal/arm: vector memcpy for ARM
  eal/arm: cpu flag checks for ARM
  lpm/arm: implement rte_lpm_lookupx4 using rte_lpm_lookup_bulk on
    for-x86
  arm: Disable usage of SSE optimized code in librte_acl

 app/test/test_cpuflags.c                           |   5 +
 config/defconfig_arm-armv7-a-linuxapp-gcc          |  72 ++++++
 lib/librte_acl/acl.h                               |   2 +
 lib/librte_acl/rte_acl.c                           |   8 +-
 lib/librte_acl/rte_acl_osdep.h                     |   2 +
 .../common/include/arch/arm/rte_atomic.h           | 257 ++++++++++++++++++++
 .../common/include/arch/arm/rte_byteorder.h        | 148 +++++++++++
 .../common/include/arch/arm/rte_cpuflags.h         | 169 +++++++++++++
 .../common/include/arch/arm/rte_cycles.h           |  85 +++++++
 .../common/include/arch/arm/rte_memcpy.h           | 270 +++++++++++++++++++++
 .../common/include/arch/arm/rte_prefetch.h         |  61 +++++
 .../common/include/arch/arm/rte_rwlock.h           |  40 +++
 .../common/include/arch/arm/rte_spinlock.h         | 114 +++++++++
 lib/librte_lpm/rte_lpm.h                           |  71 ++++++
 mk/arch/arm/rte.vars.mk                            |  39 +++
 mk/machine/armv7-a/rte.vars.mk                     |  60 +++++
 mk/rte.cpuflags.mk                                 |   6 +
 mk/toolchain/gcc/rte.vars.mk                       |   6 +
 18 files changed, 1414 insertions(+), 1 deletion(-)
 create mode 100644 config/defconfig_arm-armv7-a-linuxapp-gcc
 create mode 100644 lib/librte_eal/common/include/arch/arm/rte_atomic.h
 create mode 100644 lib/librte_eal/common/include/arch/arm/rte_byteorder.h
 create mode 100644 lib/librte_eal/common/include/arch/arm/rte_cpuflags.h
 create mode 100644 lib/librte_eal/common/include/arch/arm/rte_cycles.h
 create mode 100644 lib/librte_eal/common/include/arch/arm/rte_memcpy.h
 create mode 100644 lib/librte_eal/common/include/arch/arm/rte_prefetch.h
 create mode 100644 lib/librte_eal/common/include/arch/arm/rte_rwlock.h
 create mode 100644 lib/librte_eal/common/include/arch/arm/rte_spinlock.h
 create mode 100644 mk/arch/arm/rte.vars.mk
 create mode 100644 mk/machine/armv7-a/rte.vars.mk

-- 
2.5.2

^ permalink raw reply	[flat|nested] 2+ messages in thread
end of thread, other threads:[~2015-10-03  9:12 UTC | newest]

Thread overview: 2+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2015-10-03  9:12 [dpdk-dev] [PATCH v1 07/12] eal/arm: vector memcpy for ARM Jan Viktorin
  -- strict thread matches above, loose matches on Subject: below --
2015-10-03  8:58 [dpdk-dev] [PATCH v1 00/12] Support for ARM(v7) Jan Viktorin
2015-10-03  8:58 ` [dpdk-dev] [PATCH v1 07/12] eal/arm: vector memcpy for ARM Jan Viktorin

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).