[dpdk-dev] [PATCH 1/2] lib/eal: add amd epyc2 memcpy routine to eal

[dpdk-dev] [PATCH 1/2] lib/eal: add amd epyc2 memcpy routine to eal

[Aman Kumar]

This patch provides rte_memcpy* calls optimized for
AMD EPYC Gen2 platforms. This option is disabled by
default and can be enabled by defining 'rte_memcpy_amdepyc2'
in the meson build.

Signed-off-by: Aman Kumar <aman.kumar@vvdntech.in>
---
 lib/eal/x86/include/meson.build  |   1 +
 lib/eal/x86/include/rte_memcpy.h | 502 +++++++++++++++++++++++++++++++
 meson_options.txt                |   2 +
 3 files changed, 505 insertions(+)

diff --git a/lib/eal/x86/include/meson.build b/lib/eal/x86/include/meson.build
index 12c2e00035..a03683779d 100644
--- a/lib/eal/x86/include/meson.build
+++ b/lib/eal/x86/include/meson.build
@@ -27,3 +27,4 @@ arch_indirect_headers = files(
 )
 install_headers(arch_headers + arch_indirect_headers, subdir: get_option('include_subdir_arch'))
 dpdk_chkinc_headers += arch_headers
+dpdk_conf.set('RTE_MEMCPY_AMDEPYC2', get_option('rte_memcpy_amdepyc2'))
diff --git a/lib/eal/x86/include/rte_memcpy.h b/lib/eal/x86/include/rte_memcpy.h
index 79f381dd9b..47dda9cb87 100644
--- a/lib/eal/x86/include/rte_memcpy.h
+++ b/lib/eal/x86/include/rte_memcpy.h
@@ -368,6 +368,498 @@ rte_mov128blocks(uint8_t *dst, const uint8_t *src, size_t n)
 	}
 }
 
+#if defined RTE_MEMCPY_AMDEPYC2
+
+/**
+ * Copy 16 bytes from one location to another,
+ * with temporal stores
+ */
+static __rte_always_inline void
+rte_copy16_ts(uint8_t *dst, uint8_t *src)
+{
+	__m128i var128;
+
+	var128 = _mm_stream_load_si128((__m128i *)src);
+	_mm_storeu_si128((__m128i *)dst, var128);
+}
+
+/**
+ * Copy 32 bytes from one location to another,
+ * with temporal stores
+ */
+static __rte_always_inline void
+rte_copy32_ts(uint8_t *dst, uint8_t *src)
+{
+	__m256i ymm0;
+
+	ymm0 = _mm256_stream_load_si256((const __m256i *)src);
+	_mm256_storeu_si256((__m256i *)dst, ymm0);
+}
+
+/**
+ * Copy 64 bytes from one location to another,
+ * with temporal stores
+ */
+static __rte_always_inline void
+rte_copy64_ts(uint8_t *dst, uint8_t *src)
+{
+	rte_copy32_ts(dst + 0 * 32, src + 0 * 32);
+	rte_copy32_ts(dst + 1 * 32, src + 1 * 32);
+}
+
+/**
+ * Copy 128 bytes from one location to another,
+ * with temporal stores
+ */
+static __rte_always_inline void
+rte_copy128_ts(uint8_t *dst, uint8_t *src)
+{
+	rte_copy32_ts(dst + 0 * 32, src + 0 * 32);
+	rte_copy32_ts(dst + 1 * 32, src + 1 * 32);
+	rte_copy32_ts(dst + 2 * 32, src + 2 * 32);
+	rte_copy32_ts(dst + 3 * 32, src + 3 * 32);
+}
+
+/**
+ * Copy len bytes from one location to another,
+ * with temporal stores 16B aligned
+ */
+static __rte_always_inline void *
+rte_memcpy_aligned_tstore16_generic(void *dst, void *src, int len)
+{
+	void *dest = dst;
+
+	while (len >= 128) {
+		rte_copy128_ts((uint8_t *)dst, (uint8_t *)src);
+		dst = (uint8_t *)dst + 128;
+		src = (uint8_t *)src + 128;
+		len -= 128;
+	}
+	while (len >= 64) {
+		rte_copy64_ts((uint8_t *)dst, (uint8_t *)src);
+		dst = (uint8_t *)dst + 64;
+		src = (uint8_t *)src + 64;
+		len -= 64;
+	}
+	while (len >= 32) {
+		rte_copy32_ts((uint8_t *)dst, (uint8_t *)src);
+		dst = (uint8_t *)dst + 32;
+		src = (uint8_t *)src + 32;
+		len -= 32;
+	}
+	if (len >= 16) {
+		rte_copy16_ts((uint8_t *)dst, (uint8_t *)src);
+		dst = (uint8_t *)dst + 16;
+		src = (uint8_t *)src + 16;
+		len -= 16;
+	}
+	if (len >= 8) {
+		*(uint64_t *)dst = *(const uint64_t *)src;
+		dst = (uint8_t *)dst + 8;
+		src = (uint8_t *)src + 8;
+		len -= 8;
+	}
+	if (len >= 4) {
+		*(uint32_t *)dst = *(const uint32_t *)src;
+		dst = (uint8_t *)dst + 4;
+		src = (uint8_t *)src + 4;
+		len -= 4;
+	}
+	if (len != 0) {
+		dst = (uint8_t *)dst - (4 - len);
+		src = (uint8_t *)src - (4 - len);
+		*(uint32_t *)dst = *(const uint32_t *)src;
+	}
+
+	return dest;
+}
+
+static __rte_always_inline void *
+rte_memcpy_aligned_ntload_tstore16_amdepyc2(void *dst,
+					    const void *src,
+					    size_t size)
+{
+	asm volatile goto("movq %0, %%rsi\n\t"
+	"movq %1, %%rdi\n\t"
+	"movq %2, %%rdx\n\t"
+	"cmpq   $(128), %%rdx\n\t"
+	"jb     202f\n\t"
+	"201:\n\t"
+	"vmovntdqa (%%rsi), %%ymm0\n\t"
+	"vmovntdqa 32(%%rsi), %%ymm1\n\t"
+	"vmovntdqa 64(%%rsi), %%ymm2\n\t"
+	"vmovntdqa 96(%%rsi), %%ymm3\n\t"
+	"vmovdqu  %%ymm0, (%%rdi)\n\t"
+	"vmovdqu  %%ymm1, 32(%%rdi)\n\t"
+	"vmovdqu  %%ymm2, 64(%%rdi)\n\t"
+	"vmovdqu  %%ymm3, 96(%%rdi)\n\t"
+	"addq   $128, %%rsi\n\t"
+	"addq   $128, %%rdi\n\t"
+	"subq   $128, %%rdx\n\t"
+	"jz     %l[done]\n\t"
+	"cmpq   $128, %%rdx\n\t" /*Vector Size 32B.  */
+	"jae    201b\n\t"
+	"202:\n\t"
+	"cmpq   $64, %%rdx\n\t"
+	"jb     203f\n\t"
+	"vmovntdqa (%%rsi), %%ymm0\n\t"
+	"vmovntdqa 32(%%rsi), %%ymm1\n\t"
+	"vmovdqu  %%ymm0, (%%rdi)\n\t"
+	"vmovdqu  %%ymm1, 32(%%rdi)\n\t"
+	"addq   $64, %%rsi\n\t"
+	"addq   $64, %%rdi\n\t"
+	"subq   $64, %%rdx\n\t"
+	"jz     %l[done]\n\t"
+	"203:\n\t"
+	"cmpq   $32, %%rdx\n\t"
+	"jb     204f\n\t"
+	"vmovntdqa (%%rsi), %%ymm0\n\t"
+	"vmovdqu  %%ymm0, (%%rdi)\n\t"
+	"addq   $32, %%rsi\n\t"
+	"addq   $32, %%rdi\n\t"
+	"subq   $32, %%rdx\n\t"
+	"jz     %l[done]\n\t"
+	"204:\n\t"
+	"cmpb   $16, %%dl\n\t"
+	"jb     205f\n\t"
+	"vmovntdqa (%%rsi), %%xmm0\n\t"
+	"vmovdqu  %%xmm0, (%%rdi)\n\t"
+	"addq   $16, %%rsi\n\t"
+	"addq   $16, %%rdi\n\t"
+	"subq   $16, %%rdx\n\t"
+	"jz     %l[done]\n\t"
+	"205:\n\t"
+	"cmpb   $2, %%dl\n\t"
+	"jb     208f\n\t"
+	"cmpb   $4, %%dl\n\t"
+	"jbe    207f\n\t"
+	"cmpb   $8, %%dl\n\t"
+	"jbe    206f\n\t"
+	"movq   -8(%%rsi,%%rdx), %%rcx\n\t"
+	"movq   (%%rsi), %%rsi\n\t"
+	"movq   %%rcx, -8(%%rdi,%%rdx)\n\t"
+	"movq   %%rsi, (%%rdi)\n\t"
+	"jmp    %l[done]\n\t"
+	"206:\n\t"
+	"movl   -4(%%rsi,%%rdx), %%ecx\n\t"
+	"movl   (%%rsi), %%esi\n\t"
+	"movl   %%ecx, -4(%%rdi,%%rdx)\n\t"
+	"movl   %%esi, (%%rdi)\n\t"
+	"jmp    %l[done]\n\t"
+	"207:\n\t"
+	"movzwl -2(%%rsi,%%rdx), %%ecx\n\t"
+	"movzwl (%%rsi), %%esi\n\t"
+	"movw   %%cx, -2(%%rdi,%%rdx)\n\t"
+	"movw   %%si, (%%rdi)\n\t"
+	"jmp    %l[done]\n\t"
+	"208:\n\t"
+	"movzbl (%%rsi), %%ecx\n\t"
+	"movb   %%cl, (%%rdi)"
+	:
+	: "r"(src), "r"(dst), "r"(size)
+	: "rcx", "rdx", "rsi", "rdi", "ymm0", "ymm1", "ymm2", "ymm3", "memory"
+	: done
+	);
+done:
+	return dst;
+}
+
+static __rte_always_inline void *
+rte_memcpy_generic(void *dst, const void *src, size_t len)
+{
+	asm goto("movq	%0, %%rsi\n\t"
+	"movq	%1, %%rdi\n\t"
+	"movq	%2, %%rdx\n\t"
+	"movq    %%rdi, %%rax\n\t"
+	"cmp     $32, %%rdx\n\t"
+	"jb      101f\n\t"
+	"cmp     $(32 * 2), %%rdx\n\t"
+	"ja      108f\n\t"
+	"vmovdqu   (%%rsi), %%ymm0\n\t"
+	"vmovdqu   -32(%%rsi,%%rdx), %%ymm1\n\t"
+	"vmovdqu   %%ymm0, (%%rdi)\n\t"
+	"vmovdqu   %%ymm1, -32(%%rdi,%%rdx)\n\t"
+	"vzeroupper\n\t"
+	"jmp %l[done]\n\t"
+	"101:\n\t"
+	/* Less than 1 VEC.  */
+	"cmpb    $32, %%dl\n\t"
+	"jae     103f\n\t"
+	"cmpb    $16, %%dl\n\t"
+	"jae     104f\n\t"
+	"cmpb    $8, %%dl\n\t"
+	"jae     105f\n\t"
+	"cmpb    $4, %%dl\n\t"
+	"jae     106f\n\t"
+	"cmpb    $1, %%dl\n\t"
+	"ja      107f\n\t"
+	"jb      102f\n\t"
+	"movzbl  (%%rsi), %%ecx\n\t"
+	"movb    %%cl, (%%rdi)\n\t"
+	"102:\n\t"
+	"jmp %l[done]\n\t"
+	"103:\n\t"
+	/* From 32 to 63.  No branch when size == 32.  */
+	"vmovdqu (%%rsi), %%ymm0\n\t"
+	"vmovdqu -32(%%rsi,%%rdx), %%ymm1\n\t"
+	"vmovdqu %%ymm0, (%%rdi)\n\t"
+	"vmovdqu %%ymm1, -32(%%rdi,%%rdx)\n\t"
+	"vzeroupper\n\t"
+	"jmp %l[done]\n\t"
+	/* From 16 to 31.  No branch when size == 16.  */
+	"104:\n\t"
+	"vmovdqu (%%rsi), %%xmm0\n\t"
+	"vmovdqu -16(%%rsi,%%rdx), %%xmm1\n\t"
+	"vmovdqu %%xmm0, (%%rdi)\n\t"
+	"vmovdqu %%xmm1, -16(%%rdi,%%rdx)\n\t"
+	"jmp %l[done]\n\t"
+	"105:\n\t"
+	/* From 8 to 15.  No branch when size == 8.  */
+	"movq    -8(%%rsi,%%rdx), %%rcx\n\t"
+	"movq    (%%rsi), %%rsi\n\t"
+	"movq    %%rcx, -8(%%rdi,%%rdx)\n\t"
+	"movq    %%rsi, (%%rdi)\n\t"
+	"jmp %l[done]\n\t"
+	"106:\n\t"
+	/* From 4 to 7.  No branch when size == 4.  */
+	"movl    -4(%%rsi,%%rdx), %%ecx\n\t"
+	"movl    (%%rsi), %%esi\n\t"
+	"movl    %%ecx, -4(%%rdi,%%rdx)\n\t"
+	"movl    %%esi, (%%rdi)\n\t"
+	"jmp %l[done]\n\t"
+	"107:\n\t"
+	/* From 2 to 3.  No branch when size == 2.  */
+	"movzwl  -2(%%rsi,%%rdx), %%ecx\n\t"
+	"movzwl  (%%rsi), %%esi\n\t"
+	"movw    %%cx, -2(%%rdi,%%rdx)\n\t"
+	"movw    %%si, (%%rdi)\n\t"
+	"jmp %l[done]\n\t"
+	"108:\n\t"
+	/* More than 2 * VEC and there may be overlap between destination */
+	/* and source.  */
+	"cmpq    $(32 * 8), %%rdx\n\t"
+	"ja      111f\n\t"
+	"cmpq    $(32 * 4), %%rdx\n\t"
+	"jb      109f\n\t"
+	/* Copy from 4 * VEC to 8 * VEC, inclusively. */
+	"vmovdqu   (%%rsi), %%ymm0\n\t"
+	"vmovdqu   32(%%rsi), %%ymm1\n\t"
+	"vmovdqu   (32 * 2)(%%rsi), %%ymm2\n\t"
+	"vmovdqu   (32 * 3)(%%rsi), %%ymm3\n\t"
+	"vmovdqu   -32(%%rsi,%%rdx), %%ymm4\n\t"
+	"vmovdqu   -(32 * 2)(%%rsi,%%rdx), %%ymm5\n\t"
+	"vmovdqu   -(32 * 3)(%%rsi,%%rdx), %%ymm6\n\t"
+	"vmovdqu   -(32 * 4)(%%rsi,%%rdx), %%ymm7\n\t"
+	"vmovdqu   %%ymm0, (%%rdi)\n\t"
+	"vmovdqu   %%ymm1, 32(%%rdi)\n\t"
+	"vmovdqu   %%ymm2, (32 * 2)(%%rdi)\n\t"
+	"vmovdqu   %%ymm3, (32 * 3)(%%rdi)\n\t"
+	"vmovdqu   %%ymm4, -32(%%rdi,%%rdx)\n\t"
+	"vmovdqu   %%ymm5, -(32 * 2)(%%rdi,%%rdx)\n\t"
+	"vmovdqu   %%ymm6, -(32 * 3)(%%rdi,%%rdx)\n\t"
+	"vmovdqu   %%ymm7, -(32 * 4)(%%rdi,%%rdx)\n\t"
+	"vzeroupper\n\t"
+	"jmp %l[done]\n\t"
+	"109:\n\t"
+	/* Copy from 2 * VEC to 4 * VEC. */
+	"vmovdqu   (%%rsi), %%ymm0\n\t"
+	"vmovdqu   32(%%rsi), %%ymm1\n\t"
+	"vmovdqu   -32(%%rsi,%%rdx), %%ymm2\n\t"
+	"vmovdqu   -(32 * 2)(%%rsi,%%rdx), %%ymm3\n\t"
+	"vmovdqu   %%ymm0, (%%rdi)\n\t"
+	"vmovdqu   %%ymm1, 32(%%rdi)\n\t"
+	"vmovdqu   %%ymm2, -32(%%rdi,%%rdx)\n\t"
+	"vmovdqu   %%ymm3, -(32 * 2)(%%rdi,%%rdx)\n\t"
+	"vzeroupper\n\t"
+	"110:\n\t"
+	"jmp %l[done]\n\t"
+	"111:\n\t"
+	"cmpq    %%rsi, %%rdi\n\t"
+	"ja      113f\n\t"
+	/* Source == destination is less common.  */
+	"je      110b\n\t"
+	/* Load the first VEC and last 4 * VEC to
+	 * support overlapping addresses.
+	 */
+	"vmovdqu   (%%rsi), %%ymm4\n\t"
+	"vmovdqu   -32(%%rsi, %%rdx), %%ymm5\n\t"
+	"vmovdqu   -(32 * 2)(%%rsi, %%rdx), %%ymm6\n\t"
+	"vmovdqu   -(32 * 3)(%%rsi, %%rdx), %%ymm7\n\t"
+	"vmovdqu   -(32 * 4)(%%rsi, %%rdx), %%ymm8\n\t"
+	/* Save start and stop of the destination buffer.  */
+	"movq    %%rdi, %%r11\n\t"
+	"leaq    -32(%%rdi, %%rdx), %%rcx\n\t"
+	/* Align destination for aligned stores in the loop.  Compute */
+	/* how much destination is misaligned.  */
+	"movq    %%rdi, %%r8\n\t"
+	"andq    $(32 - 1), %%r8\n\t"
+	/* Get the negative of offset for alignment.  */
+	"subq    $32, %%r8\n\t"
+	/* Adjust source.  */
+	"subq    %%r8, %%rsi\n\t"
+	/* Adjust destination which should be aligned now.  */
+	"subq    %%r8, %%rdi\n\t"
+	/* Adjust length.  */
+	"addq    %%r8, %%rdx\n\t"
+	/* Check non-temporal store threshold.  */
+	"cmpq	 $(1024*1024), %%rdx\n\t"
+	"ja      115f\n\t"
+	"112:\n\t"
+	/* Copy 4 * VEC a time forward.  */
+	"vmovdqu   (%%rsi), %%ymm0\n\t"
+	"vmovdqu   32(%%rsi), %%ymm1\n\t"
+	"vmovdqu   (32 * 2)(%%rsi), %%ymm2\n\t"
+	"vmovdqu   (32 * 3)(%%rsi), %%ymm3\n\t"
+	"addq    $(32 * 4), %%rsi\n\t"
+	"subq    $(32 * 4), %%rdx\n\t"
+	"vmovdqa   %%ymm0, (%%rdi)\n\t"
+	"vmovdqa   %%ymm1, 32(%%rdi)\n\t"
+	"vmovdqa   %%ymm2, (32 * 2)(%%rdi)\n\t"
+	"vmovdqa   %%ymm3, (32 * 3)(%%rdi)\n\t"
+	"addq    $(32 * 4), %%rdi\n\t"
+	"cmpq    $(32 * 4), %%rdx\n\t"
+	"ja      112b\n\t"
+	/* Store the last 4 * VEC.  */
+	"vmovdqu   %%ymm5, (%%rcx)\n\t"
+	"vmovdqu   %%ymm6, -32(%%rcx)\n\t"
+	"vmovdqu   %%ymm7, -(32 * 2)(%%rcx)\n\t"
+	"vmovdqu   %%ymm8, -(32 * 3)(%%rcx)\n\t"
+	/* Store the first VEC.  */
+	"vmovdqu   %%ymm4, (%%r11)\n\t"
+	"vzeroupper\n\t"
+	"jmp %l[done]\n\t"
+	"113:\n\t"
+	/* Load the first 4*VEC and last VEC to support overlapping addresses.*/
+	"vmovdqu   (%%rsi), %%ymm4\n\t"
+	"vmovdqu   32(%%rsi), %%ymm5\n\t"
+	"vmovdqu   (32 * 2)(%%rsi), %%ymm6\n\t"
+	"vmovdqu   (32 * 3)(%%rsi), %%ymm7\n\t"
+	"vmovdqu   -32(%%rsi,%%rdx), %%ymm8\n\t"
+	/* Save stop of the destination buffer.  */
+	"leaq    -32(%%rdi, %%rdx), %%r11\n\t"
+	/* Align destination end for aligned stores in the loop.  Compute */
+	/* how much destination end is misaligned.  */
+	"leaq    -32(%%rsi, %%rdx), %%rcx\n\t"
+	"movq    %%r11, %%r9\n\t"
+	"movq    %%r11, %%r8\n\t"
+	"andq    $(32 - 1), %%r8\n\t"
+	/* Adjust source.  */
+	"subq    %%r8, %%rcx\n\t"
+	/* Adjust the end of destination which should be aligned now.  */
+	"subq    %%r8, %%r9\n\t"
+	/* Adjust length.  */
+	"subq    %%r8, %%rdx\n\t"
+	 /* Check non-temporal store threshold.  */
+	"cmpq	 $(1024*1024), %%rdx\n\t"
+	"ja      117f\n\t"
+	"114:\n\t"
+	/* Copy 4 * VEC a time backward.  */
+	"vmovdqu   (%%rcx), %%ymm0\n\t"
+	"vmovdqu   -32(%%rcx), %%ymm1\n\t"
+	"vmovdqu   -(32 * 2)(%%rcx), %%ymm2\n\t"
+	"vmovdqu   -(32 * 3)(%%rcx), %%ymm3\n\t"
+	"subq    $(32 * 4), %%rcx\n\t"
+	"subq    $(32 * 4), %%rdx\n\t"
+	"vmovdqa   %%ymm0, (%%r9)\n\t"
+	"vmovdqa   %%ymm1, -32(%%r9)\n\t"
+	"vmovdqa   %%ymm2, -(32 * 2)(%%r9)\n\t"
+	"vmovdqa   %%ymm3, -(32 * 3)(%%r9)\n\t"
+	"subq    $(32 * 4), %%r9\n\t"
+	"cmpq    $(32 * 4), %%rdx\n\t"
+	"ja      114b\n\t"
+	/* Store the first 4 * VEC. */
+	"vmovdqu   %%ymm4, (%%rdi)\n\t"
+	"vmovdqu   %%ymm5, 32(%%rdi)\n\t"
+	"vmovdqu   %%ymm6, (32 * 2)(%%rdi)\n\t"
+	"vmovdqu   %%ymm7, (32 * 3)(%%rdi)\n\t"
+	/* Store the last VEC. */
+	"vmovdqu   %%ymm8, (%%r11)\n\t"
+	"vzeroupper\n\t"
+	"jmp %l[done]\n\t"
+
+	"115:\n\t"
+	/* Don't use non-temporal store if there is overlap between */
+	/* destination and source since destination may be in cache */
+	/* when source is loaded. */
+	"leaq    (%%rdi, %%rdx), %%r10\n\t"
+	"cmpq    %%r10, %%rsi\n\t"
+	"jb      112b\n\t"
+	"116:\n\t"
+	/* Copy 4 * VEC a time forward with non-temporal stores.  */
+	"prefetcht0 (32*4*2)(%%rsi)\n\t"
+	"prefetcht0 (32*4*2 + 64)(%%rsi)\n\t"
+	"prefetcht0 (32*4*3)(%%rsi)\n\t"
+	"prefetcht0 (32*4*3 + 64)(%%rsi)\n\t"
+	"vmovdqu   (%%rsi), %%ymm0\n\t"
+	"vmovdqu   32(%%rsi), %%ymm1\n\t"
+	"vmovdqu   (32 * 2)(%%rsi), %%ymm2\n\t"
+	"vmovdqu   (32 * 3)(%%rsi), %%ymm3\n\t"
+	"addq    $(32*4), %%rsi\n\t"
+	"subq    $(32*4), %%rdx\n\t"
+	"vmovntdq  %%ymm0, (%%rdi)\n\t"
+	"vmovntdq  %%ymm1, 32(%%rdi)\n\t"
+	"vmovntdq  %%ymm2, (32 * 2)(%%rdi)\n\t"
+	"vmovntdq  %%ymm3, (32 * 3)(%%rdi)\n\t"
+	"addq    $(32*4), %%rdi\n\t"
+	"cmpq    $(32*4), %%rdx\n\t"
+	"ja      116b\n\t"
+	"sfence\n\t"
+	/* Store the last 4 * VEC.  */
+	"vmovdqu   %%ymm5, (%%rcx)\n\t"
+	"vmovdqu   %%ymm6, -32(%%rcx)\n\t"
+	"vmovdqu   %%ymm7, -(32 * 2)(%%rcx)\n\t"
+	"vmovdqu   %%ymm8, -(32 * 3)(%%rcx)\n\t"
+	/* Store the first VEC.  */
+	"vmovdqu   %%ymm4, (%%r11)\n\t"
+	"vzeroupper\n\t"
+	"jmp %l[done]\n\t"
+	"117:\n\t"
+	/* Don't use non-temporal store if there is overlap between */
+	/* destination and source since destination may be in cache */
+	/* when source is loaded.  */
+	"leaq    (%%rcx, %%rdx), %%r10\n\t"
+	"cmpq    %%r10, %%r9\n\t"
+	"jb      114b\n\t"
+	"118:\n\t"
+	/* Copy 4 * VEC a time backward with non-temporal stores. */
+	"prefetcht0 (-32 * 4 * 2)(%%rcx)\n\t"
+	"prefetcht0 (-32 * 4 * 2 - 64)(%%rcx)\n\t"
+	"prefetcht0 (-32 * 4 * 3)(%%rcx)\n\t"
+	"prefetcht0 (-32 * 4 * 3 - 64)(%%rcx)\n\t"
+	"vmovdqu   (%%rcx), %%ymm0\n\t"
+	"vmovdqu   -32(%%rcx), %%ymm1\n\t"
+	"vmovdqu   -(32 * 2)(%%rcx), %%ymm2\n\t"
+	"vmovdqu   -(32 * 3)(%%rcx), %%ymm3\n\t"
+	"subq    $(32*4), %%rcx\n\t"
+	"subq    $(32*4), %%rdx\n\t"
+	"vmovntdq  %%ymm0, (%%r9)\n\t"
+	"vmovntdq  %%ymm1, -32(%%r9)\n\t"
+	"vmovntdq  %%ymm2, -(32 * 2)(%%r9)\n\t"
+	"vmovntdq  %%ymm3, -(32 * 3)(%%r9)\n\t"
+	"subq    $(32 * 4), %%r9\n\t"
+	"cmpq    $(32 * 4), %%rdx\n\t"
+	"ja      118b\n\t"
+	"sfence\n\t"
+	/* Store the first 4 * VEC.  */
+	"vmovdqu   %%ymm4, (%%rdi)\n\t"
+	"vmovdqu   %%ymm5, 32(%%rdi)\n\t"
+	"vmovdqu   %%ymm6, (32 * 2)(%%rdi)\n\t"
+	"vmovdqu   %%ymm7, (32 * 3)(%%rdi)\n\t"
+	/* Store the last VEC.  */
+	"vmovdqu   %%ymm8, (%%r11)\n\t"
+	"vzeroupper\n\t"
+	"jmp %l[done]"
+	:
+	: "r"(src), "r"(dst), "r"(len)
+	: "rax", "rcx", "rdx", "rdi", "rsi", "r8", "r9", "r10", "r11", "r12", "ymm0",
+	"ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", "ymm8", "memory"
+	: done
+	);
+done:
+	return dst;
+}
+
+#else
 static __rte_always_inline void *
 rte_memcpy_generic(void *dst, const void *src, size_t n)
 {
@@ -479,6 +971,8 @@ rte_memcpy_generic(void *dst, const void *src, size_t n)
 	goto COPY_BLOCK_128_BACK31;
 }
 
+#endif /* RTE_MEMCPY_AMDEPYC2 */
+
 #else /* __AVX512F__ */
 
 #define ALIGNMENT_MASK 0x0F
@@ -874,6 +1368,14 @@ rte_memcpy(void *dst, const void *src, size_t n)
 		return rte_memcpy_generic(dst, src, n);
 }
 
+#if defined __AVX2__ && defined(RTE_MEMCPY_AMDEPYC2)
+static __rte_always_inline void *
+rte_memcpy_aligned_tstore16(void *dst, void *src, int len)
+{
+	return rte_memcpy_aligned_ntload_tstore16_amdepyc2(dst, src, len);
+}
+#endif
+
 #if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION >= 100000)
 #pragma GCC diagnostic pop
 #endif
diff --git a/meson_options.txt b/meson_options.txt
index 0e92734c49..e232c9c340 100644
--- a/meson_options.txt
+++ b/meson_options.txt
@@ -42,6 +42,8 @@ option('platform', type: 'string', value: 'native', description:
        'Platform to build, either "native", "generic" or a SoC. Please refer to the Linux build guide for more information.')
 option('enable_trace_fp', type: 'boolean', value: false, description:
        'enable fast path trace points.')
+option('rte_memcpy_amdepyc2', type: 'boolean', value: false, description:
+       'to enable amd epyc memcpy routines')
 option('tests', type: 'boolean', value: true, description:
        'build unit tests')
 option('use_hpet', type: 'boolean', value: false, description:
-- 
2.25.1

[dpdk-dev] [PATCH 2/2] net/mlx5: optimize mprq memcpy for AMD EPYC2 platforms

[Aman Kumar]

add non temporal load and temporal store for mprq memcpy.
define mlx5_ntload_tstore in meson build configuration to
enable this optimization. This utilizes AMD EPYC2 optimized
rte_memcpy* routines.

Signed-off-by: Aman Kumar <aman.kumar@vvdntech.in>
---
 drivers/net/mlx5/meson.build |  1 +
 drivers/net/mlx5/mlx5.c      | 12 ++++++++++++
 drivers/net/mlx5/mlx5.h      |  3 +++
 drivers/net/mlx5/mlx5_rx.h   | 24 ++++++++++++++++++++++++
 drivers/net/mlx5/mlx5_rxq.c  |  4 ++++
 meson_options.txt            |  2 ++
 6 files changed, 46 insertions(+)

diff --git a/drivers/net/mlx5/meson.build b/drivers/net/mlx5/meson.build
index dac7f1fabf..0d2888742c 100644
--- a/drivers/net/mlx5/meson.build
+++ b/drivers/net/mlx5/meson.build
@@ -61,6 +61,7 @@ foreach option:cflags_options
         cflags += option
     endif
 endforeach
+dpdk_conf.set('RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY', get_option('mlx5_ntload_tstore'))
 if get_option('buildtype').contains('debug')
     cflags += [ '-pedantic', '-DPEDANTIC' ]
 else
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index f84e061fe7..cf57867c25 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -161,6 +161,11 @@
 /* Configure timeout of LRO session (in microseconds). */
 #define MLX5_LRO_TIMEOUT_USEC "lro_timeout_usec"
 
+#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
+/* mprq_tstore_memcpy */
+#define MLX5_MPRQ_TSTORE_MEMCPY "mprq_tstore_memcpy"
+#endif
+
 /*
  * Device parameter to configure the total data buffer size for a single
  * hairpin queue (logarithm value).
@@ -1991,6 +1996,10 @@ mlx5_args_check(const char *key, const char *val, void *opaque)
 		config->decap_en = !!tmp;
 	} else if (strcmp(MLX5_ALLOW_DUPLICATE_PATTERN, key) == 0) {
 		config->allow_duplicate_pattern = !!tmp;
+#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
+	} else if (strcmp(MLX5_MPRQ_TSTORE_MEMCPY, key) == 0) {
+		config->mprq_tstore_memcpy = tmp;
+#endif
 	} else {
 		DRV_LOG(WARNING, "%s: unknown parameter", key);
 		rte_errno = EINVAL;
@@ -2051,6 +2060,9 @@ mlx5_args(struct mlx5_dev_config *config, struct rte_devargs *devargs)
 		MLX5_SYS_MEM_EN,
 		MLX5_DECAP_EN,
 		MLX5_ALLOW_DUPLICATE_PATTERN,
+#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
+		MLX5_MPRQ_TSTORE_MEMCPY,
+#endif
 		NULL,
 	};
 	struct rte_kvargs *kvlist;
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index e02714e231..7d5617f5ca 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -298,6 +298,9 @@ struct mlx5_dev_config {
 	int tx_skew; /* Tx scheduling skew between WQE and data on wire. */
 	struct mlx5_hca_attr hca_attr; /* HCA attributes. */
 	struct mlx5_lro_config lro; /* LRO configuration. */
+#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
+	unsigned int mprq_tstore_memcpy:1;
+#endif
 };
 
 
diff --git a/drivers/net/mlx5/mlx5_rx.h b/drivers/net/mlx5/mlx5_rx.h
index 3f2b99fb65..19318bdd1b 100644
--- a/drivers/net/mlx5/mlx5_rx.h
+++ b/drivers/net/mlx5/mlx5_rx.h
@@ -148,6 +148,9 @@ struct mlx5_rxq_data {
 	uint32_t rxseg_n; /* Number of split segment descriptions. */
 	struct mlx5_eth_rxseg rxseg[MLX5_MAX_RXQ_NSEG];
 	/* Buffer split segment descriptions - sizes, offsets, pools. */
+#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
+	unsigned int mprq_tstore_memcpy:1;
+#endif
 } __rte_cache_aligned;
 
 enum mlx5_rxq_type {
@@ -422,6 +425,15 @@ mprq_buf_to_pkt(struct mlx5_rxq_data *rxq, struct rte_mbuf *pkt, uint32_t len,
 	const uint32_t offset = strd_idx * strd_sz + strd_shift;
 	void *addr = RTE_PTR_ADD(mlx5_mprq_buf_addr(buf, strd_n), offset);
 
+#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
+	if (unlikely(!rxq->mprq_tstore_memcpy) &&
+	    len <= rxq->mprq_max_memcpy_len) {
+		rte_prefetch1(addr);
+		if (len > RTE_CACHE_LINE_SIZE)
+			rte_prefetch2((void *)((uintptr_t)addr +
+					       RTE_CACHE_LINE_SIZE));
+	}
+#endif
 	/*
 	 * Memcpy packets to the target mbuf if:
 	 * - The size of packet is smaller than mprq_max_memcpy_len.
@@ -433,8 +445,20 @@ mprq_buf_to_pkt(struct mlx5_rxq_data *rxq, struct rte_mbuf *pkt, uint32_t len,
 	    (hdrm_overlap > 0 && !rxq->strd_scatter_en)) {
 		if (likely(len <=
 			   (uint32_t)(pkt->buf_len - RTE_PKTMBUF_HEADROOM))) {
+#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
+			uintptr_t data_addr;
+
+			data_addr = (uintptr_t)rte_pktmbuf_mtod(pkt, void *);
+			if (!((data_addr | (uintptr_t)addr) & ALIGNMENT_MASK) &&
+				rxq->mprq_tstore_memcpy)
+				rte_memcpy_aligned_tstore16((void *)data_addr,
+					   addr, len);
+			else
+				rte_memcpy((void *)data_addr, addr, len);
+#else
 			rte_memcpy(rte_pktmbuf_mtod(pkt, void *),
 				   addr, len);
+#endif
 			DATA_LEN(pkt) = len;
 		} else if (rxq->strd_scatter_en) {
 			struct rte_mbuf *prev = pkt;
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index abd8ce7989..a1b0fa6455 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -1449,6 +1449,10 @@ mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 	tmpl->socket = socket;
 	if (dev->data->dev_conf.intr_conf.rxq)
 		tmpl->irq = 1;
+#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
+	tmpl->rxq.mprq_tstore_memcpy = config->mprq_tstore_memcpy;
+#endif
+
 	/*
 	 * This Rx queue can be configured as a Multi-Packet RQ if all of the
 	 * following conditions are met:
diff --git a/meson_options.txt b/meson_options.txt
index e232c9c340..cc7b629d17 100644
--- a/meson_options.txt
+++ b/meson_options.txt
@@ -38,6 +38,8 @@ option('max_lcores', type: 'integer', value: 128, description:
        'maximum number of cores/threads supported by EAL')
 option('max_numa_nodes', type: 'integer', value: 32, description:
        'maximum number of NUMA nodes supported by EAL')
+option('mlx5_ntload_tstore', type: 'boolean', value: false, description:
+       'to enable optimized MPRQ in RX datapath')
 option('platform', type: 'string', value: 'native', description:
        'Platform to build, either "native", "generic" or a SoC. Please refer to the Linux build guide for more information.')
 option('enable_trace_fp', type: 'boolean', value: false, description:
-- 
2.25.1

Re: [dpdk-dev] [PATCH 2/2] net/mlx5: optimize mprq memcpy for AMD EPYC2 platforms

[Thomas Monjalon]

23/08/2021 10:44, Aman Kumar:
> add non temporal load and temporal store for mprq memcpy.
> define mlx5_ntload_tstore in meson build configuration to
> enable this optimization. This utilizes AMD EPYC2 optimized
> rte_memcpy* routines.
[...]
> +option('mlx5_ntload_tstore', type: 'boolean', value: false, description:
> +       'to enable optimized MPRQ in RX datapath')

Please don't make it a compilation option.
Why isn't it always enabled?

There was a comment on the first patch.
Do you plan to make a new version?

Re: [dpdk-dev] [PATCH 2/2] net/mlx5: optimize mprq memcpy for AMD EPYC2 platforms

[Aman Kumar]

Dear Thomas,

Please check V2 patchset
<https://patchwork.dpdk.org/project/dpdk/patch/20211019104724.19416-1-aman.kumar@vvdntech.in/>.
Upon suggestions from Jerin, we've moved this build option using
config/x86/x86_amd_epyc_linux_gcc cross-file.
This option is not a compilation option and enabled by default in V2, once
it is cross built. Please let us know your comments.


*With Best Regards*
Aman Kumar
VVDN Technologies Pvt. Ltd.
*web:* www.vvdntech.com




On Wed, Oct 13, 2021 at 10:23 PM Thomas Monjalon <thomas@monjalon.net>
wrote:

> 23/08/2021 10:44, Aman Kumar:
> > add non temporal load and temporal store for mprq memcpy.
> > define mlx5_ntload_tstore in meson build configuration to
> > enable this optimization. This utilizes AMD EPYC2 optimized
> > rte_memcpy* routines.
> [...]
> > +option('mlx5_ntload_tstore', type: 'boolean', value: false, description:
> > +       'to enable optimized MPRQ in RX datapath')
>
> Please don't make it a compilation option.
> Why isn't it always enabled?
>
> There was a comment on the first patch.
> Do you plan to make a new version?
>
>
>

Re: [dpdk-dev] [PATCH 1/2] lib/eal: add amd epyc2 memcpy routine to eal

[Jerin Jacob]

On Mon, Aug 23, 2021 at 2:14 PM Aman Kumar <aman.kumar@vvdntech.in> wrote:
>
> This patch provides rte_memcpy* calls optimized for
> AMD EPYC Gen2 platforms. This option is disabled by
> default and can be enabled by defining 'rte_memcpy_amdepyc2'

Generic options should support all the architectures.
Another more scalable alternative is to introduce config/x86/x86_amd_epyc2
and new parameter under [properties]


> in the meson build.
>
> Signed-off-by: Aman Kumar <aman.kumar@vvdntech.in>
> ---
>  lib/eal/x86/include/meson.build  |   1 +
>  lib/eal/x86/include/rte_memcpy.h | 502 +++++++++++++++++++++++++++++++
>  meson_options.txt                |   2 +
>  3 files changed, 505 insertions(+)
>
> diff --git a/lib/eal/x86/include/meson.build b/lib/eal/x86/include/meson.build
> index 12c2e00035..a03683779d 100644
> --- a/lib/eal/x86/include/meson.build
> +++ b/lib/eal/x86/include/meson.build
> @@ -27,3 +27,4 @@ arch_indirect_headers = files(
>  )
>  install_headers(arch_headers + arch_indirect_headers, subdir: get_option('include_subdir_arch'))
>  dpdk_chkinc_headers += arch_headers
> +dpdk_conf.set('RTE_MEMCPY_AMDEPYC2', get_option('rte_memcpy_amdepyc2'))
> diff --git a/lib/eal/x86/include/rte_memcpy.h b/lib/eal/x86/include/rte_memcpy.h
> index 79f381dd9b..47dda9cb87 100644
> --- a/lib/eal/x86/include/rte_memcpy.h
> +++ b/lib/eal/x86/include/rte_memcpy.h
> @@ -368,6 +368,498 @@ rte_mov128blocks(uint8_t *dst, const uint8_t *src, size_t n)
>         }
>  }
>
> +#if defined RTE_MEMCPY_AMDEPYC2
> +
> +/**
> + * Copy 16 bytes from one location to another,
> + * with temporal stores
> + */
> +static __rte_always_inline void
> +rte_copy16_ts(uint8_t *dst, uint8_t *src)
> +{
> +       __m128i var128;
> +
> +       var128 = _mm_stream_load_si128((__m128i *)src);
> +       _mm_storeu_si128((__m128i *)dst, var128);
> +}
> +
> +/**
> + * Copy 32 bytes from one location to another,
> + * with temporal stores
> + */
> +static __rte_always_inline void
> +rte_copy32_ts(uint8_t *dst, uint8_t *src)
> +{
> +       __m256i ymm0;
> +
> +       ymm0 = _mm256_stream_load_si256((const __m256i *)src);
> +       _mm256_storeu_si256((__m256i *)dst, ymm0);
> +}
> +
> +/**
> + * Copy 64 bytes from one location to another,
> + * with temporal stores
> + */
> +static __rte_always_inline void
> +rte_copy64_ts(uint8_t *dst, uint8_t *src)
> +{
> +       rte_copy32_ts(dst + 0 * 32, src + 0 * 32);
> +       rte_copy32_ts(dst + 1 * 32, src + 1 * 32);
> +}
> +
> +/**
> + * Copy 128 bytes from one location to another,
> + * with temporal stores
> + */
> +static __rte_always_inline void
> +rte_copy128_ts(uint8_t *dst, uint8_t *src)
> +{
> +       rte_copy32_ts(dst + 0 * 32, src + 0 * 32);
> +       rte_copy32_ts(dst + 1 * 32, src + 1 * 32);
> +       rte_copy32_ts(dst + 2 * 32, src + 2 * 32);
> +       rte_copy32_ts(dst + 3 * 32, src + 3 * 32);
> +}
> +
> +/**
> + * Copy len bytes from one location to another,
> + * with temporal stores 16B aligned
> + */
> +static __rte_always_inline void *
> +rte_memcpy_aligned_tstore16_generic(void *dst, void *src, int len)
> +{
> +       void *dest = dst;
> +
> +       while (len >= 128) {
> +               rte_copy128_ts((uint8_t *)dst, (uint8_t *)src);
> +               dst = (uint8_t *)dst + 128;
> +               src = (uint8_t *)src + 128;
> +               len -= 128;
> +       }
> +       while (len >= 64) {
> +               rte_copy64_ts((uint8_t *)dst, (uint8_t *)src);
> +               dst = (uint8_t *)dst + 64;
> +               src = (uint8_t *)src + 64;
> +               len -= 64;
> +       }
> +       while (len >= 32) {
> +               rte_copy32_ts((uint8_t *)dst, (uint8_t *)src);
> +               dst = (uint8_t *)dst + 32;
> +               src = (uint8_t *)src + 32;
> +               len -= 32;
> +       }
> +       if (len >= 16) {
> +               rte_copy16_ts((uint8_t *)dst, (uint8_t *)src);
> +               dst = (uint8_t *)dst + 16;
> +               src = (uint8_t *)src + 16;
> +               len -= 16;
> +       }
> +       if (len >= 8) {
> +               *(uint64_t *)dst = *(const uint64_t *)src;
> +               dst = (uint8_t *)dst + 8;
> +               src = (uint8_t *)src + 8;
> +               len -= 8;
> +       }
> +       if (len >= 4) {
> +               *(uint32_t *)dst = *(const uint32_t *)src;
> +               dst = (uint8_t *)dst + 4;
> +               src = (uint8_t *)src + 4;
> +               len -= 4;
> +       }
> +       if (len != 0) {
> +               dst = (uint8_t *)dst - (4 - len);
> +               src = (uint8_t *)src - (4 - len);
> +               *(uint32_t *)dst = *(const uint32_t *)src;
> +       }
> +
> +       return dest;
> +}
> +
> +static __rte_always_inline void *
> +rte_memcpy_aligned_ntload_tstore16_amdepyc2(void *dst,
> +                                           const void *src,
> +                                           size_t size)
> +{
> +       asm volatile goto("movq %0, %%rsi\n\t"
> +       "movq %1, %%rdi\n\t"
> +       "movq %2, %%rdx\n\t"
> +       "cmpq   $(128), %%rdx\n\t"
> +       "jb     202f\n\t"
> +       "201:\n\t"
> +       "vmovntdqa (%%rsi), %%ymm0\n\t"
> +       "vmovntdqa 32(%%rsi), %%ymm1\n\t"
> +       "vmovntdqa 64(%%rsi), %%ymm2\n\t"
> +       "vmovntdqa 96(%%rsi), %%ymm3\n\t"
> +       "vmovdqu  %%ymm0, (%%rdi)\n\t"
> +       "vmovdqu  %%ymm1, 32(%%rdi)\n\t"
> +       "vmovdqu  %%ymm2, 64(%%rdi)\n\t"
> +       "vmovdqu  %%ymm3, 96(%%rdi)\n\t"
> +       "addq   $128, %%rsi\n\t"
> +       "addq   $128, %%rdi\n\t"
> +       "subq   $128, %%rdx\n\t"
> +       "jz     %l[done]\n\t"
> +       "cmpq   $128, %%rdx\n\t" /*Vector Size 32B.  */
> +       "jae    201b\n\t"
> +       "202:\n\t"
> +       "cmpq   $64, %%rdx\n\t"
> +       "jb     203f\n\t"
> +       "vmovntdqa (%%rsi), %%ymm0\n\t"
> +       "vmovntdqa 32(%%rsi), %%ymm1\n\t"
> +       "vmovdqu  %%ymm0, (%%rdi)\n\t"
> +       "vmovdqu  %%ymm1, 32(%%rdi)\n\t"
> +       "addq   $64, %%rsi\n\t"
> +       "addq   $64, %%rdi\n\t"
> +       "subq   $64, %%rdx\n\t"
> +       "jz     %l[done]\n\t"
> +       "203:\n\t"
> +       "cmpq   $32, %%rdx\n\t"
> +       "jb     204f\n\t"
> +       "vmovntdqa (%%rsi), %%ymm0\n\t"
> +       "vmovdqu  %%ymm0, (%%rdi)\n\t"
> +       "addq   $32, %%rsi\n\t"
> +       "addq   $32, %%rdi\n\t"
> +       "subq   $32, %%rdx\n\t"
> +       "jz     %l[done]\n\t"
> +       "204:\n\t"
> +       "cmpb   $16, %%dl\n\t"
> +       "jb     205f\n\t"
> +       "vmovntdqa (%%rsi), %%xmm0\n\t"
> +       "vmovdqu  %%xmm0, (%%rdi)\n\t"
> +       "addq   $16, %%rsi\n\t"
> +       "addq   $16, %%rdi\n\t"
> +       "subq   $16, %%rdx\n\t"
> +       "jz     %l[done]\n\t"
> +       "205:\n\t"
> +       "cmpb   $2, %%dl\n\t"
> +       "jb     208f\n\t"
> +       "cmpb   $4, %%dl\n\t"
> +       "jbe    207f\n\t"
> +       "cmpb   $8, %%dl\n\t"
> +       "jbe    206f\n\t"
> +       "movq   -8(%%rsi,%%rdx), %%rcx\n\t"
> +       "movq   (%%rsi), %%rsi\n\t"
> +       "movq   %%rcx, -8(%%rdi,%%rdx)\n\t"
> +       "movq   %%rsi, (%%rdi)\n\t"
> +       "jmp    %l[done]\n\t"
> +       "206:\n\t"
> +       "movl   -4(%%rsi,%%rdx), %%ecx\n\t"
> +       "movl   (%%rsi), %%esi\n\t"
> +       "movl   %%ecx, -4(%%rdi,%%rdx)\n\t"
> +       "movl   %%esi, (%%rdi)\n\t"
> +       "jmp    %l[done]\n\t"
> +       "207:\n\t"
> +       "movzwl -2(%%rsi,%%rdx), %%ecx\n\t"
> +       "movzwl (%%rsi), %%esi\n\t"
> +       "movw   %%cx, -2(%%rdi,%%rdx)\n\t"
> +       "movw   %%si, (%%rdi)\n\t"
> +       "jmp    %l[done]\n\t"
> +       "208:\n\t"
> +       "movzbl (%%rsi), %%ecx\n\t"
> +       "movb   %%cl, (%%rdi)"
> +       :
> +       : "r"(src), "r"(dst), "r"(size)
> +       : "rcx", "rdx", "rsi", "rdi", "ymm0", "ymm1", "ymm2", "ymm3", "memory"
> +       : done
> +       );
> +done:
> +       return dst;
> +}
> +
> +static __rte_always_inline void *
> +rte_memcpy_generic(void *dst, const void *src, size_t len)
> +{
> +       asm goto("movq  %0, %%rsi\n\t"
> +       "movq   %1, %%rdi\n\t"
> +       "movq   %2, %%rdx\n\t"
> +       "movq    %%rdi, %%rax\n\t"
> +       "cmp     $32, %%rdx\n\t"
> +       "jb      101f\n\t"
> +       "cmp     $(32 * 2), %%rdx\n\t"
> +       "ja      108f\n\t"
> +       "vmovdqu   (%%rsi), %%ymm0\n\t"
> +       "vmovdqu   -32(%%rsi,%%rdx), %%ymm1\n\t"
> +       "vmovdqu   %%ymm0, (%%rdi)\n\t"
> +       "vmovdqu   %%ymm1, -32(%%rdi,%%rdx)\n\t"
> +       "vzeroupper\n\t"
> +       "jmp %l[done]\n\t"
> +       "101:\n\t"
> +       /* Less than 1 VEC.  */
> +       "cmpb    $32, %%dl\n\t"
> +       "jae     103f\n\t"
> +       "cmpb    $16, %%dl\n\t"
> +       "jae     104f\n\t"
> +       "cmpb    $8, %%dl\n\t"
> +       "jae     105f\n\t"
> +       "cmpb    $4, %%dl\n\t"
> +       "jae     106f\n\t"
> +       "cmpb    $1, %%dl\n\t"
> +       "ja      107f\n\t"
> +       "jb      102f\n\t"
> +       "movzbl  (%%rsi), %%ecx\n\t"
> +       "movb    %%cl, (%%rdi)\n\t"
> +       "102:\n\t"
> +       "jmp %l[done]\n\t"
> +       "103:\n\t"
> +       /* From 32 to 63.  No branch when size == 32.  */
> +       "vmovdqu (%%rsi), %%ymm0\n\t"
> +       "vmovdqu -32(%%rsi,%%rdx), %%ymm1\n\t"
> +       "vmovdqu %%ymm0, (%%rdi)\n\t"
> +       "vmovdqu %%ymm1, -32(%%rdi,%%rdx)\n\t"
> +       "vzeroupper\n\t"
> +       "jmp %l[done]\n\t"
> +       /* From 16 to 31.  No branch when size == 16.  */
> +       "104:\n\t"
> +       "vmovdqu (%%rsi), %%xmm0\n\t"
> +       "vmovdqu -16(%%rsi,%%rdx), %%xmm1\n\t"
> +       "vmovdqu %%xmm0, (%%rdi)\n\t"
> +       "vmovdqu %%xmm1, -16(%%rdi,%%rdx)\n\t"
> +       "jmp %l[done]\n\t"
> +       "105:\n\t"
> +       /* From 8 to 15.  No branch when size == 8.  */
> +       "movq    -8(%%rsi,%%rdx), %%rcx\n\t"
> +       "movq    (%%rsi), %%rsi\n\t"
> +       "movq    %%rcx, -8(%%rdi,%%rdx)\n\t"
> +       "movq    %%rsi, (%%rdi)\n\t"
> +       "jmp %l[done]\n\t"
> +       "106:\n\t"
> +       /* From 4 to 7.  No branch when size == 4.  */
> +       "movl    -4(%%rsi,%%rdx), %%ecx\n\t"
> +       "movl    (%%rsi), %%esi\n\t"
> +       "movl    %%ecx, -4(%%rdi,%%rdx)\n\t"
> +       "movl    %%esi, (%%rdi)\n\t"
> +       "jmp %l[done]\n\t"
> +       "107:\n\t"
> +       /* From 2 to 3.  No branch when size == 2.  */
> +       "movzwl  -2(%%rsi,%%rdx), %%ecx\n\t"
> +       "movzwl  (%%rsi), %%esi\n\t"
> +       "movw    %%cx, -2(%%rdi,%%rdx)\n\t"
> +       "movw    %%si, (%%rdi)\n\t"
> +       "jmp %l[done]\n\t"
> +       "108:\n\t"
> +       /* More than 2 * VEC and there may be overlap between destination */
> +       /* and source.  */
> +       "cmpq    $(32 * 8), %%rdx\n\t"
> +       "ja      111f\n\t"
> +       "cmpq    $(32 * 4), %%rdx\n\t"
> +       "jb      109f\n\t"
> +       /* Copy from 4 * VEC to 8 * VEC, inclusively. */
> +       "vmovdqu   (%%rsi), %%ymm0\n\t"
> +       "vmovdqu   32(%%rsi), %%ymm1\n\t"
> +       "vmovdqu   (32 * 2)(%%rsi), %%ymm2\n\t"
> +       "vmovdqu   (32 * 3)(%%rsi), %%ymm3\n\t"
> +       "vmovdqu   -32(%%rsi,%%rdx), %%ymm4\n\t"
> +       "vmovdqu   -(32 * 2)(%%rsi,%%rdx), %%ymm5\n\t"
> +       "vmovdqu   -(32 * 3)(%%rsi,%%rdx), %%ymm6\n\t"
> +       "vmovdqu   -(32 * 4)(%%rsi,%%rdx), %%ymm7\n\t"
> +       "vmovdqu   %%ymm0, (%%rdi)\n\t"
> +       "vmovdqu   %%ymm1, 32(%%rdi)\n\t"
> +       "vmovdqu   %%ymm2, (32 * 2)(%%rdi)\n\t"
> +       "vmovdqu   %%ymm3, (32 * 3)(%%rdi)\n\t"
> +       "vmovdqu   %%ymm4, -32(%%rdi,%%rdx)\n\t"
> +       "vmovdqu   %%ymm5, -(32 * 2)(%%rdi,%%rdx)\n\t"
> +       "vmovdqu   %%ymm6, -(32 * 3)(%%rdi,%%rdx)\n\t"
> +       "vmovdqu   %%ymm7, -(32 * 4)(%%rdi,%%rdx)\n\t"
> +       "vzeroupper\n\t"
> +       "jmp %l[done]\n\t"
> +       "109:\n\t"
> +       /* Copy from 2 * VEC to 4 * VEC. */
> +       "vmovdqu   (%%rsi), %%ymm0\n\t"
> +       "vmovdqu   32(%%rsi), %%ymm1\n\t"
> +       "vmovdqu   -32(%%rsi,%%rdx), %%ymm2\n\t"
> +       "vmovdqu   -(32 * 2)(%%rsi,%%rdx), %%ymm3\n\t"
> +       "vmovdqu   %%ymm0, (%%rdi)\n\t"
> +       "vmovdqu   %%ymm1, 32(%%rdi)\n\t"
> +       "vmovdqu   %%ymm2, -32(%%rdi,%%rdx)\n\t"
> +       "vmovdqu   %%ymm3, -(32 * 2)(%%rdi,%%rdx)\n\t"
> +       "vzeroupper\n\t"
> +       "110:\n\t"
> +       "jmp %l[done]\n\t"
> +       "111:\n\t"
> +       "cmpq    %%rsi, %%rdi\n\t"
> +       "ja      113f\n\t"
> +       /* Source == destination is less common.  */
> +       "je      110b\n\t"
> +       /* Load the first VEC and last 4 * VEC to
> +        * support overlapping addresses.
> +        */
> +       "vmovdqu   (%%rsi), %%ymm4\n\t"
> +       "vmovdqu   -32(%%rsi, %%rdx), %%ymm5\n\t"
> +       "vmovdqu   -(32 * 2)(%%rsi, %%rdx), %%ymm6\n\t"
> +       "vmovdqu   -(32 * 3)(%%rsi, %%rdx), %%ymm7\n\t"
> +       "vmovdqu   -(32 * 4)(%%rsi, %%rdx), %%ymm8\n\t"
> +       /* Save start and stop of the destination buffer.  */
> +       "movq    %%rdi, %%r11\n\t"
> +       "leaq    -32(%%rdi, %%rdx), %%rcx\n\t"
> +       /* Align destination for aligned stores in the loop.  Compute */
> +       /* how much destination is misaligned.  */
> +       "movq    %%rdi, %%r8\n\t"
> +       "andq    $(32 - 1), %%r8\n\t"
> +       /* Get the negative of offset for alignment.  */
> +       "subq    $32, %%r8\n\t"
> +       /* Adjust source.  */
> +       "subq    %%r8, %%rsi\n\t"
> +       /* Adjust destination which should be aligned now.  */
> +       "subq    %%r8, %%rdi\n\t"
> +       /* Adjust length.  */
> +       "addq    %%r8, %%rdx\n\t"
> +       /* Check non-temporal store threshold.  */
> +       "cmpq    $(1024*1024), %%rdx\n\t"
> +       "ja      115f\n\t"
> +       "112:\n\t"
> +       /* Copy 4 * VEC a time forward.  */
> +       "vmovdqu   (%%rsi), %%ymm0\n\t"
> +       "vmovdqu   32(%%rsi), %%ymm1\n\t"
> +       "vmovdqu   (32 * 2)(%%rsi), %%ymm2\n\t"
> +       "vmovdqu   (32 * 3)(%%rsi), %%ymm3\n\t"
> +       "addq    $(32 * 4), %%rsi\n\t"
> +       "subq    $(32 * 4), %%rdx\n\t"
> +       "vmovdqa   %%ymm0, (%%rdi)\n\t"
> +       "vmovdqa   %%ymm1, 32(%%rdi)\n\t"
> +       "vmovdqa   %%ymm2, (32 * 2)(%%rdi)\n\t"
> +       "vmovdqa   %%ymm3, (32 * 3)(%%rdi)\n\t"
> +       "addq    $(32 * 4), %%rdi\n\t"
> +       "cmpq    $(32 * 4), %%rdx\n\t"
> +       "ja      112b\n\t"
> +       /* Store the last 4 * VEC.  */
> +       "vmovdqu   %%ymm5, (%%rcx)\n\t"
> +       "vmovdqu   %%ymm6, -32(%%rcx)\n\t"
> +       "vmovdqu   %%ymm7, -(32 * 2)(%%rcx)\n\t"
> +       "vmovdqu   %%ymm8, -(32 * 3)(%%rcx)\n\t"
> +       /* Store the first VEC.  */
> +       "vmovdqu   %%ymm4, (%%r11)\n\t"
> +       "vzeroupper\n\t"
> +       "jmp %l[done]\n\t"
> +       "113:\n\t"
> +       /* Load the first 4*VEC and last VEC to support overlapping addresses.*/
> +       "vmovdqu   (%%rsi), %%ymm4\n\t"
> +       "vmovdqu   32(%%rsi), %%ymm5\n\t"
> +       "vmovdqu   (32 * 2)(%%rsi), %%ymm6\n\t"
> +       "vmovdqu   (32 * 3)(%%rsi), %%ymm7\n\t"
> +       "vmovdqu   -32(%%rsi,%%rdx), %%ymm8\n\t"
> +       /* Save stop of the destination buffer.  */
> +       "leaq    -32(%%rdi, %%rdx), %%r11\n\t"
> +       /* Align destination end for aligned stores in the loop.  Compute */
> +       /* how much destination end is misaligned.  */
> +       "leaq    -32(%%rsi, %%rdx), %%rcx\n\t"
> +       "movq    %%r11, %%r9\n\t"
> +       "movq    %%r11, %%r8\n\t"
> +       "andq    $(32 - 1), %%r8\n\t"
> +       /* Adjust source.  */
> +       "subq    %%r8, %%rcx\n\t"
> +       /* Adjust the end of destination which should be aligned now.  */
> +       "subq    %%r8, %%r9\n\t"
> +       /* Adjust length.  */
> +       "subq    %%r8, %%rdx\n\t"
> +        /* Check non-temporal store threshold.  */
> +       "cmpq    $(1024*1024), %%rdx\n\t"
> +       "ja      117f\n\t"
> +       "114:\n\t"
> +       /* Copy 4 * VEC a time backward.  */
> +       "vmovdqu   (%%rcx), %%ymm0\n\t"
> +       "vmovdqu   -32(%%rcx), %%ymm1\n\t"
> +       "vmovdqu   -(32 * 2)(%%rcx), %%ymm2\n\t"
> +       "vmovdqu   -(32 * 3)(%%rcx), %%ymm3\n\t"
> +       "subq    $(32 * 4), %%rcx\n\t"
> +       "subq    $(32 * 4), %%rdx\n\t"
> +       "vmovdqa   %%ymm0, (%%r9)\n\t"
> +       "vmovdqa   %%ymm1, -32(%%r9)\n\t"
> +       "vmovdqa   %%ymm2, -(32 * 2)(%%r9)\n\t"
> +       "vmovdqa   %%ymm3, -(32 * 3)(%%r9)\n\t"
> +       "subq    $(32 * 4), %%r9\n\t"
> +       "cmpq    $(32 * 4), %%rdx\n\t"
> +       "ja      114b\n\t"
> +       /* Store the first 4 * VEC. */
> +       "vmovdqu   %%ymm4, (%%rdi)\n\t"
> +       "vmovdqu   %%ymm5, 32(%%rdi)\n\t"
> +       "vmovdqu   %%ymm6, (32 * 2)(%%rdi)\n\t"
> +       "vmovdqu   %%ymm7, (32 * 3)(%%rdi)\n\t"
> +       /* Store the last VEC. */
> +       "vmovdqu   %%ymm8, (%%r11)\n\t"
> +       "vzeroupper\n\t"
> +       "jmp %l[done]\n\t"
> +
> +       "115:\n\t"
> +       /* Don't use non-temporal store if there is overlap between */
> +       /* destination and source since destination may be in cache */
> +       /* when source is loaded. */
> +       "leaq    (%%rdi, %%rdx), %%r10\n\t"
> +       "cmpq    %%r10, %%rsi\n\t"
> +       "jb      112b\n\t"
> +       "116:\n\t"
> +       /* Copy 4 * VEC a time forward with non-temporal stores.  */
> +       "prefetcht0 (32*4*2)(%%rsi)\n\t"
> +       "prefetcht0 (32*4*2 + 64)(%%rsi)\n\t"
> +       "prefetcht0 (32*4*3)(%%rsi)\n\t"
> +       "prefetcht0 (32*4*3 + 64)(%%rsi)\n\t"
> +       "vmovdqu   (%%rsi), %%ymm0\n\t"
> +       "vmovdqu   32(%%rsi), %%ymm1\n\t"
> +       "vmovdqu   (32 * 2)(%%rsi), %%ymm2\n\t"
> +       "vmovdqu   (32 * 3)(%%rsi), %%ymm3\n\t"
> +       "addq    $(32*4), %%rsi\n\t"
> +       "subq    $(32*4), %%rdx\n\t"
> +       "vmovntdq  %%ymm0, (%%rdi)\n\t"
> +       "vmovntdq  %%ymm1, 32(%%rdi)\n\t"
> +       "vmovntdq  %%ymm2, (32 * 2)(%%rdi)\n\t"
> +       "vmovntdq  %%ymm3, (32 * 3)(%%rdi)\n\t"
> +       "addq    $(32*4), %%rdi\n\t"
> +       "cmpq    $(32*4), %%rdx\n\t"
> +       "ja      116b\n\t"
> +       "sfence\n\t"
> +       /* Store the last 4 * VEC.  */
> +       "vmovdqu   %%ymm5, (%%rcx)\n\t"
> +       "vmovdqu   %%ymm6, -32(%%rcx)\n\t"
> +       "vmovdqu   %%ymm7, -(32 * 2)(%%rcx)\n\t"
> +       "vmovdqu   %%ymm8, -(32 * 3)(%%rcx)\n\t"
> +       /* Store the first VEC.  */
> +       "vmovdqu   %%ymm4, (%%r11)\n\t"
> +       "vzeroupper\n\t"
> +       "jmp %l[done]\n\t"
> +       "117:\n\t"
> +       /* Don't use non-temporal store if there is overlap between */
> +       /* destination and source since destination may be in cache */
> +       /* when source is loaded.  */
> +       "leaq    (%%rcx, %%rdx), %%r10\n\t"
> +       "cmpq    %%r10, %%r9\n\t"
> +       "jb      114b\n\t"
> +       "118:\n\t"
> +       /* Copy 4 * VEC a time backward with non-temporal stores. */
> +       "prefetcht0 (-32 * 4 * 2)(%%rcx)\n\t"
> +       "prefetcht0 (-32 * 4 * 2 - 64)(%%rcx)\n\t"
> +       "prefetcht0 (-32 * 4 * 3)(%%rcx)\n\t"
> +       "prefetcht0 (-32 * 4 * 3 - 64)(%%rcx)\n\t"
> +       "vmovdqu   (%%rcx), %%ymm0\n\t"
> +       "vmovdqu   -32(%%rcx), %%ymm1\n\t"
> +       "vmovdqu   -(32 * 2)(%%rcx), %%ymm2\n\t"
> +       "vmovdqu   -(32 * 3)(%%rcx), %%ymm3\n\t"
> +       "subq    $(32*4), %%rcx\n\t"
> +       "subq    $(32*4), %%rdx\n\t"
> +       "vmovntdq  %%ymm0, (%%r9)\n\t"
> +       "vmovntdq  %%ymm1, -32(%%r9)\n\t"
> +       "vmovntdq  %%ymm2, -(32 * 2)(%%r9)\n\t"
> +       "vmovntdq  %%ymm3, -(32 * 3)(%%r9)\n\t"
> +       "subq    $(32 * 4), %%r9\n\t"
> +       "cmpq    $(32 * 4), %%rdx\n\t"
> +       "ja      118b\n\t"
> +       "sfence\n\t"
> +       /* Store the first 4 * VEC.  */
> +       "vmovdqu   %%ymm4, (%%rdi)\n\t"
> +       "vmovdqu   %%ymm5, 32(%%rdi)\n\t"
> +       "vmovdqu   %%ymm6, (32 * 2)(%%rdi)\n\t"
> +       "vmovdqu   %%ymm7, (32 * 3)(%%rdi)\n\t"
> +       /* Store the last VEC.  */
> +       "vmovdqu   %%ymm8, (%%r11)\n\t"
> +       "vzeroupper\n\t"
> +       "jmp %l[done]"
> +       :
> +       : "r"(src), "r"(dst), "r"(len)
> +       : "rax", "rcx", "rdx", "rdi", "rsi", "r8", "r9", "r10", "r11", "r12", "ymm0",
> +       "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", "ymm8", "memory"
> +       : done
> +       );
> +done:
> +       return dst;
> +}
> +
> +#else
>  static __rte_always_inline void *
>  rte_memcpy_generic(void *dst, const void *src, size_t n)
>  {
> @@ -479,6 +971,8 @@ rte_memcpy_generic(void *dst, const void *src, size_t n)
>         goto COPY_BLOCK_128_BACK31;
>  }
>
> +#endif /* RTE_MEMCPY_AMDEPYC2 */
> +
>  #else /* __AVX512F__ */
>
>  #define ALIGNMENT_MASK 0x0F
> @@ -874,6 +1368,14 @@ rte_memcpy(void *dst, const void *src, size_t n)
>                 return rte_memcpy_generic(dst, src, n);
>  }
>
> +#if defined __AVX2__ && defined(RTE_MEMCPY_AMDEPYC2)
> +static __rte_always_inline void *
> +rte_memcpy_aligned_tstore16(void *dst, void *src, int len)
> +{
> +       return rte_memcpy_aligned_ntload_tstore16_amdepyc2(dst, src, len);
> +}
> +#endif
> +
>  #if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION >= 100000)
>  #pragma GCC diagnostic pop
>  #endif
> diff --git a/meson_options.txt b/meson_options.txt
> index 0e92734c49..e232c9c340 100644
> --- a/meson_options.txt
> +++ b/meson_options.txt
> @@ -42,6 +42,8 @@ option('platform', type: 'string', value: 'native', description:
>         'Platform to build, either "native", "generic" or a SoC. Please refer to the Linux build guide for more information.')
>  option('enable_trace_fp', type: 'boolean', value: false, description:
>         'enable fast path trace points.')
> +option('rte_memcpy_amdepyc2', type: 'boolean', value: false, description:
> +       'to enable amd epyc memcpy routines')
>  option('tests', type: 'boolean', value: true, description:
>         'build unit tests')
>  option('use_hpet', type: 'boolean', value: false, description:
> --
> 2.25.1
>

Re: [dpdk-dev] [PATCH 1/2] lib/eal: add amd epyc2 memcpy routine to eal

[Aman Kumar]

Hi Jerin,
Thanks for your comments and suggestions.
I will try to update this in V2.

On Mon, Aug 23, 2021 at 8:51 PM Jerin Jacob <jerinjacobk@gmail.com> wrote:

> On Mon, Aug 23, 2021 at 2:14 PM Aman Kumar <aman.kumar@vvdntech.in> wrote:
> >
> > This patch provides rte_memcpy* calls optimized for
> > AMD EPYC Gen2 platforms. This option is disabled by
> > default and can be enabled by defining 'rte_memcpy_amdepyc2'
>
> Generic options should support all the architectures.
> Another more scalable alternative is to introduce config/x86/x86_amd_epyc2
> and new parameter under [properties]
>
>
> > in the meson build.
> >
> > Signed-off-by: Aman Kumar <aman.kumar@vvdntech.in>
> > ---
> >  lib/eal/x86/include/meson.build  |   1 +
> >  lib/eal/x86/include/rte_memcpy.h | 502 +++++++++++++++++++++++++++++++
> >  meson_options.txt                |   2 +
> >  3 files changed, 505 insertions(+)
> >
> > diff --git a/lib/eal/x86/include/meson.build
> b/lib/eal/x86/include/meson.build
> > index 12c2e00035..a03683779d 100644
> > --- a/lib/eal/x86/include/meson.build
> > +++ b/lib/eal/x86/include/meson.build
> > @@ -27,3 +27,4 @@ arch_indirect_headers = files(
> >  )
> >  install_headers(arch_headers + arch_indirect_headers, subdir:
> get_option('include_subdir_arch'))
> >  dpdk_chkinc_headers += arch_headers
> > +dpdk_conf.set('RTE_MEMCPY_AMDEPYC2', get_option('rte_memcpy_amdepyc2'))
> > diff --git a/lib/eal/x86/include/rte_memcpy.h
> b/lib/eal/x86/include/rte_memcpy.h
> > index 79f381dd9b..47dda9cb87 100644
> > --- a/lib/eal/x86/include/rte_memcpy.h
> > +++ b/lib/eal/x86/include/rte_memcpy.h
> > @@ -368,6 +368,498 @@ rte_mov128blocks(uint8_t *dst, const uint8_t *src,
> size_t n)
> >         }
> >  }
> >
> > +#if defined RTE_MEMCPY_AMDEPYC2
> > +
> > +/**
> > + * Copy 16 bytes from one location to another,
> > + * with temporal stores
> > + */
> > +static __rte_always_inline void
> > +rte_copy16_ts(uint8_t *dst, uint8_t *src)
> > +{
> > +       __m128i var128;
> > +
> > +       var128 = _mm_stream_load_si128((__m128i *)src);
> > +       _mm_storeu_si128((__m128i *)dst, var128);
> > +}
> > +
> > +/**
> > + * Copy 32 bytes from one location to another,
> > + * with temporal stores
> > + */
> > +static __rte_always_inline void
> > +rte_copy32_ts(uint8_t *dst, uint8_t *src)
> > +{
> > +       __m256i ymm0;
> > +
> > +       ymm0 = _mm256_stream_load_si256((const __m256i *)src);
> > +       _mm256_storeu_si256((__m256i *)dst, ymm0);
> > +}
> > +
> > +/**
> > + * Copy 64 bytes from one location to another,
> > + * with temporal stores
> > + */
> > +static __rte_always_inline void
> > +rte_copy64_ts(uint8_t *dst, uint8_t *src)
> > +{
> > +       rte_copy32_ts(dst + 0 * 32, src + 0 * 32);
> > +       rte_copy32_ts(dst + 1 * 32, src + 1 * 32);
> > +}
> > +
> > +/**
> > + * Copy 128 bytes from one location to another,
> > + * with temporal stores
> > + */
> > +static __rte_always_inline void
> > +rte_copy128_ts(uint8_t *dst, uint8_t *src)
> > +{
> > +       rte_copy32_ts(dst + 0 * 32, src + 0 * 32);
> > +       rte_copy32_ts(dst + 1 * 32, src + 1 * 32);
> > +       rte_copy32_ts(dst + 2 * 32, src + 2 * 32);
> > +       rte_copy32_ts(dst + 3 * 32, src + 3 * 32);
> > +}
> > +
> > +/**
> > + * Copy len bytes from one location to another,
> > + * with temporal stores 16B aligned
> > + */
> > +static __rte_always_inline void *
> > +rte_memcpy_aligned_tstore16_generic(void *dst, void *src, int len)
> > +{
> > +       void *dest = dst;
> > +
> > +       while (len >= 128) {
> > +               rte_copy128_ts((uint8_t *)dst, (uint8_t *)src);
> > +               dst = (uint8_t *)dst + 128;
> > +               src = (uint8_t *)src + 128;
> > +               len -= 128;
> > +       }
> > +       while (len >= 64) {
> > +               rte_copy64_ts((uint8_t *)dst, (uint8_t *)src);
> > +               dst = (uint8_t *)dst + 64;
> > +               src = (uint8_t *)src + 64;
> > +               len -= 64;
> > +       }
> > +       while (len >= 32) {
> > +               rte_copy32_ts((uint8_t *)dst, (uint8_t *)src);
> > +               dst = (uint8_t *)dst + 32;
> > +               src = (uint8_t *)src + 32;
> > +               len -= 32;
> > +       }
> > +       if (len >= 16) {
> > +               rte_copy16_ts((uint8_t *)dst, (uint8_t *)src);
> > +               dst = (uint8_t *)dst + 16;
> > +               src = (uint8_t *)src + 16;
> > +               len -= 16;
> > +       }
> > +       if (len >= 8) {
> > +               *(uint64_t *)dst = *(const uint64_t *)src;
> > +               dst = (uint8_t *)dst + 8;
> > +               src = (uint8_t *)src + 8;
> > +               len -= 8;
> > +       }
> > +       if (len >= 4) {
> > +               *(uint32_t *)dst = *(const uint32_t *)src;
> > +               dst = (uint8_t *)dst + 4;
> > +               src = (uint8_t *)src + 4;
> > +               len -= 4;
> > +       }
> > +       if (len != 0) {
> > +               dst = (uint8_t *)dst - (4 - len);
> > +               src = (uint8_t *)src - (4 - len);
> > +               *(uint32_t *)dst = *(const uint32_t *)src;
> > +       }
> > +
> > +       return dest;
> > +}
> > +
> > +static __rte_always_inline void *
> > +rte_memcpy_aligned_ntload_tstore16_amdepyc2(void *dst,
> > +                                           const void *src,
> > +                                           size_t size)
> > +{
> > +       asm volatile goto("movq %0, %%rsi\n\t"
> > +       "movq %1, %%rdi\n\t"
> > +       "movq %2, %%rdx\n\t"
> > +       "cmpq   $(128), %%rdx\n\t"
> > +       "jb     202f\n\t"
> > +       "201:\n\t"
> > +       "vmovntdqa (%%rsi), %%ymm0\n\t"
> > +       "vmovntdqa 32(%%rsi), %%ymm1\n\t"
> > +       "vmovntdqa 64(%%rsi), %%ymm2\n\t"
> > +       "vmovntdqa 96(%%rsi), %%ymm3\n\t"
> > +       "vmovdqu  %%ymm0, (%%rdi)\n\t"
> > +       "vmovdqu  %%ymm1, 32(%%rdi)\n\t"
> > +       "vmovdqu  %%ymm2, 64(%%rdi)\n\t"
> > +       "vmovdqu  %%ymm3, 96(%%rdi)\n\t"
> > +       "addq   $128, %%rsi\n\t"
> > +       "addq   $128, %%rdi\n\t"
> > +       "subq   $128, %%rdx\n\t"
> > +       "jz     %l[done]\n\t"
> > +       "cmpq   $128, %%rdx\n\t" /*Vector Size 32B.  */
> > +       "jae    201b\n\t"
> > +       "202:\n\t"
> > +       "cmpq   $64, %%rdx\n\t"
> > +       "jb     203f\n\t"
> > +       "vmovntdqa (%%rsi), %%ymm0\n\t"
> > +       "vmovntdqa 32(%%rsi), %%ymm1\n\t"
> > +       "vmovdqu  %%ymm0, (%%rdi)\n\t"
> > +       "vmovdqu  %%ymm1, 32(%%rdi)\n\t"
> > +       "addq   $64, %%rsi\n\t"
> > +       "addq   $64, %%rdi\n\t"
> > +       "subq   $64, %%rdx\n\t"
> > +       "jz     %l[done]\n\t"
> > +       "203:\n\t"
> > +       "cmpq   $32, %%rdx\n\t"
> > +       "jb     204f\n\t"
> > +       "vmovntdqa (%%rsi), %%ymm0\n\t"
> > +       "vmovdqu  %%ymm0, (%%rdi)\n\t"
> > +       "addq   $32, %%rsi\n\t"
> > +       "addq   $32, %%rdi\n\t"
> > +       "subq   $32, %%rdx\n\t"
> > +       "jz     %l[done]\n\t"
> > +       "204:\n\t"
> > +       "cmpb   $16, %%dl\n\t"
> > +       "jb     205f\n\t"
> > +       "vmovntdqa (%%rsi), %%xmm0\n\t"
> > +       "vmovdqu  %%xmm0, (%%rdi)\n\t"
> > +       "addq   $16, %%rsi\n\t"
> > +       "addq   $16, %%rdi\n\t"
> > +       "subq   $16, %%rdx\n\t"
> > +       "jz     %l[done]\n\t"
> > +       "205:\n\t"
> > +       "cmpb   $2, %%dl\n\t"
> > +       "jb     208f\n\t"
> > +       "cmpb   $4, %%dl\n\t"
> > +       "jbe    207f\n\t"
> > +       "cmpb   $8, %%dl\n\t"
> > +       "jbe    206f\n\t"
> > +       "movq   -8(%%rsi,%%rdx), %%rcx\n\t"
> > +       "movq   (%%rsi), %%rsi\n\t"
> > +       "movq   %%rcx, -8(%%rdi,%%rdx)\n\t"
> > +       "movq   %%rsi, (%%rdi)\n\t"
> > +       "jmp    %l[done]\n\t"
> > +       "206:\n\t"
> > +       "movl   -4(%%rsi,%%rdx), %%ecx\n\t"
> > +       "movl   (%%rsi), %%esi\n\t"
> > +       "movl   %%ecx, -4(%%rdi,%%rdx)\n\t"
> > +       "movl   %%esi, (%%rdi)\n\t"
> > +       "jmp    %l[done]\n\t"
> > +       "207:\n\t"
> > +       "movzwl -2(%%rsi,%%rdx), %%ecx\n\t"
> > +       "movzwl (%%rsi), %%esi\n\t"
> > +       "movw   %%cx, -2(%%rdi,%%rdx)\n\t"
> > +       "movw   %%si, (%%rdi)\n\t"
> > +       "jmp    %l[done]\n\t"
> > +       "208:\n\t"
> > +       "movzbl (%%rsi), %%ecx\n\t"
> > +       "movb   %%cl, (%%rdi)"
> > +       :
> > +       : "r"(src), "r"(dst), "r"(size)
> > +       : "rcx", "rdx", "rsi", "rdi", "ymm0", "ymm1", "ymm2", "ymm3",
> "memory"
> > +       : done
> > +       );
> > +done:
> > +       return dst;
> > +}
> > +
> > +static __rte_always_inline void *
> > +rte_memcpy_generic(void *dst, const void *src, size_t len)
> > +{
> > +       asm goto("movq  %0, %%rsi\n\t"
> > +       "movq   %1, %%rdi\n\t"
> > +       "movq   %2, %%rdx\n\t"
> > +       "movq    %%rdi, %%rax\n\t"
> > +       "cmp     $32, %%rdx\n\t"
> > +       "jb      101f\n\t"
> > +       "cmp     $(32 * 2), %%rdx\n\t"
> > +       "ja      108f\n\t"
> > +       "vmovdqu   (%%rsi), %%ymm0\n\t"
> > +       "vmovdqu   -32(%%rsi,%%rdx), %%ymm1\n\t"
> > +       "vmovdqu   %%ymm0, (%%rdi)\n\t"
> > +       "vmovdqu   %%ymm1, -32(%%rdi,%%rdx)\n\t"
> > +       "vzeroupper\n\t"
> > +       "jmp %l[done]\n\t"
> > +       "101:\n\t"
> > +       /* Less than 1 VEC.  */
> > +       "cmpb    $32, %%dl\n\t"
> > +       "jae     103f\n\t"
> > +       "cmpb    $16, %%dl\n\t"
> > +       "jae     104f\n\t"
> > +       "cmpb    $8, %%dl\n\t"
> > +       "jae     105f\n\t"
> > +       "cmpb    $4, %%dl\n\t"
> > +       "jae     106f\n\t"
> > +       "cmpb    $1, %%dl\n\t"
> > +       "ja      107f\n\t"
> > +       "jb      102f\n\t"
> > +       "movzbl  (%%rsi), %%ecx\n\t"
> > +       "movb    %%cl, (%%rdi)\n\t"
> > +       "102:\n\t"
> > +       "jmp %l[done]\n\t"
> > +       "103:\n\t"
> > +       /* From 32 to 63.  No branch when size == 32.  */
> > +       "vmovdqu (%%rsi), %%ymm0\n\t"
> > +       "vmovdqu -32(%%rsi,%%rdx), %%ymm1\n\t"
> > +       "vmovdqu %%ymm0, (%%rdi)\n\t"
> > +       "vmovdqu %%ymm1, -32(%%rdi,%%rdx)\n\t"
> > +       "vzeroupper\n\t"
> > +       "jmp %l[done]\n\t"
> > +       /* From 16 to 31.  No branch when size == 16.  */
> > +       "104:\n\t"
> > +       "vmovdqu (%%rsi), %%xmm0\n\t"
> > +       "vmovdqu -16(%%rsi,%%rdx), %%xmm1\n\t"
> > +       "vmovdqu %%xmm0, (%%rdi)\n\t"
> > +       "vmovdqu %%xmm1, -16(%%rdi,%%rdx)\n\t"
> > +       "jmp %l[done]\n\t"
> > +       "105:\n\t"
> > +       /* From 8 to 15.  No branch when size == 8.  */
> > +       "movq    -8(%%rsi,%%rdx), %%rcx\n\t"
> > +       "movq    (%%rsi), %%rsi\n\t"
> > +       "movq    %%rcx, -8(%%rdi,%%rdx)\n\t"
> > +       "movq    %%rsi, (%%rdi)\n\t"
> > +       "jmp %l[done]\n\t"
> > +       "106:\n\t"
> > +       /* From 4 to 7.  No branch when size == 4.  */
> > +       "movl    -4(%%rsi,%%rdx), %%ecx\n\t"
> > +       "movl    (%%rsi), %%esi\n\t"
> > +       "movl    %%ecx, -4(%%rdi,%%rdx)\n\t"
> > +       "movl    %%esi, (%%rdi)\n\t"
> > +       "jmp %l[done]\n\t"
> > +       "107:\n\t"
> > +       /* From 2 to 3.  No branch when size == 2.  */
> > +       "movzwl  -2(%%rsi,%%rdx), %%ecx\n\t"
> > +       "movzwl  (%%rsi), %%esi\n\t"
> > +       "movw    %%cx, -2(%%rdi,%%rdx)\n\t"
> > +       "movw    %%si, (%%rdi)\n\t"
> > +       "jmp %l[done]\n\t"
> > +       "108:\n\t"
> > +       /* More than 2 * VEC and there may be overlap between
> destination */
> > +       /* and source.  */
> > +       "cmpq    $(32 * 8), %%rdx\n\t"
> > +       "ja      111f\n\t"
> > +       "cmpq    $(32 * 4), %%rdx\n\t"
> > +       "jb      109f\n\t"
> > +       /* Copy from 4 * VEC to 8 * VEC, inclusively. */
> > +       "vmovdqu   (%%rsi), %%ymm0\n\t"
> > +       "vmovdqu   32(%%rsi), %%ymm1\n\t"
> > +       "vmovdqu   (32 * 2)(%%rsi), %%ymm2\n\t"
> > +       "vmovdqu   (32 * 3)(%%rsi), %%ymm3\n\t"
> > +       "vmovdqu   -32(%%rsi,%%rdx), %%ymm4\n\t"
> > +       "vmovdqu   -(32 * 2)(%%rsi,%%rdx), %%ymm5\n\t"
> > +       "vmovdqu   -(32 * 3)(%%rsi,%%rdx), %%ymm6\n\t"
> > +       "vmovdqu   -(32 * 4)(%%rsi,%%rdx), %%ymm7\n\t"
> > +       "vmovdqu   %%ymm0, (%%rdi)\n\t"
> > +       "vmovdqu   %%ymm1, 32(%%rdi)\n\t"
> > +       "vmovdqu   %%ymm2, (32 * 2)(%%rdi)\n\t"
> > +       "vmovdqu   %%ymm3, (32 * 3)(%%rdi)\n\t"
> > +       "vmovdqu   %%ymm4, -32(%%rdi,%%rdx)\n\t"
> > +       "vmovdqu   %%ymm5, -(32 * 2)(%%rdi,%%rdx)\n\t"
> > +       "vmovdqu   %%ymm6, -(32 * 3)(%%rdi,%%rdx)\n\t"
> > +       "vmovdqu   %%ymm7, -(32 * 4)(%%rdi,%%rdx)\n\t"
> > +       "vzeroupper\n\t"
> > +       "jmp %l[done]\n\t"
> > +       "109:\n\t"
> > +       /* Copy from 2 * VEC to 4 * VEC. */
> > +       "vmovdqu   (%%rsi), %%ymm0\n\t"
> > +       "vmovdqu   32(%%rsi), %%ymm1\n\t"
> > +       "vmovdqu   -32(%%rsi,%%rdx), %%ymm2\n\t"
> > +       "vmovdqu   -(32 * 2)(%%rsi,%%rdx), %%ymm3\n\t"
> > +       "vmovdqu   %%ymm0, (%%rdi)\n\t"
> > +       "vmovdqu   %%ymm1, 32(%%rdi)\n\t"
> > +       "vmovdqu   %%ymm2, -32(%%rdi,%%rdx)\n\t"
> > +       "vmovdqu   %%ymm3, -(32 * 2)(%%rdi,%%rdx)\n\t"
> > +       "vzeroupper\n\t"
> > +       "110:\n\t"
> > +       "jmp %l[done]\n\t"
> > +       "111:\n\t"
> > +       "cmpq    %%rsi, %%rdi\n\t"
> > +       "ja      113f\n\t"
> > +       /* Source == destination is less common.  */
> > +       "je      110b\n\t"
> > +       /* Load the first VEC and last 4 * VEC to
> > +        * support overlapping addresses.
> > +        */
> > +       "vmovdqu   (%%rsi), %%ymm4\n\t"
> > +       "vmovdqu   -32(%%rsi, %%rdx), %%ymm5\n\t"
> > +       "vmovdqu   -(32 * 2)(%%rsi, %%rdx), %%ymm6\n\t"
> > +       "vmovdqu   -(32 * 3)(%%rsi, %%rdx), %%ymm7\n\t"
> > +       "vmovdqu   -(32 * 4)(%%rsi, %%rdx), %%ymm8\n\t"
> > +       /* Save start and stop of the destination buffer.  */
> > +       "movq    %%rdi, %%r11\n\t"
> > +       "leaq    -32(%%rdi, %%rdx), %%rcx\n\t"
> > +       /* Align destination for aligned stores in the loop.  Compute */
> > +       /* how much destination is misaligned.  */
> > +       "movq    %%rdi, %%r8\n\t"
> > +       "andq    $(32 - 1), %%r8\n\t"
> > +       /* Get the negative of offset for alignment.  */
> > +       "subq    $32, %%r8\n\t"
> > +       /* Adjust source.  */
> > +       "subq    %%r8, %%rsi\n\t"
> > +       /* Adjust destination which should be aligned now.  */
> > +       "subq    %%r8, %%rdi\n\t"
> > +       /* Adjust length.  */
> > +       "addq    %%r8, %%rdx\n\t"
> > +       /* Check non-temporal store threshold.  */
> > +       "cmpq    $(1024*1024), %%rdx\n\t"
> > +       "ja      115f\n\t"
> > +       "112:\n\t"
> > +       /* Copy 4 * VEC a time forward.  */
> > +       "vmovdqu   (%%rsi), %%ymm0\n\t"
> > +       "vmovdqu   32(%%rsi), %%ymm1\n\t"
> > +       "vmovdqu   (32 * 2)(%%rsi), %%ymm2\n\t"
> > +       "vmovdqu   (32 * 3)(%%rsi), %%ymm3\n\t"
> > +       "addq    $(32 * 4), %%rsi\n\t"
> > +       "subq    $(32 * 4), %%rdx\n\t"
> > +       "vmovdqa   %%ymm0, (%%rdi)\n\t"
> > +       "vmovdqa   %%ymm1, 32(%%rdi)\n\t"
> > +       "vmovdqa   %%ymm2, (32 * 2)(%%rdi)\n\t"
> > +       "vmovdqa   %%ymm3, (32 * 3)(%%rdi)\n\t"
> > +       "addq    $(32 * 4), %%rdi\n\t"
> > +       "cmpq    $(32 * 4), %%rdx\n\t"
> > +       "ja      112b\n\t"
> > +       /* Store the last 4 * VEC.  */
> > +       "vmovdqu   %%ymm5, (%%rcx)\n\t"
> > +       "vmovdqu   %%ymm6, -32(%%rcx)\n\t"
> > +       "vmovdqu   %%ymm7, -(32 * 2)(%%rcx)\n\t"
> > +       "vmovdqu   %%ymm8, -(32 * 3)(%%rcx)\n\t"
> > +       /* Store the first VEC.  */
> > +       "vmovdqu   %%ymm4, (%%r11)\n\t"
> > +       "vzeroupper\n\t"
> > +       "jmp %l[done]\n\t"
> > +       "113:\n\t"
> > +       /* Load the first 4*VEC and last VEC to support overlapping
> addresses.*/
> > +       "vmovdqu   (%%rsi), %%ymm4\n\t"
> > +       "vmovdqu   32(%%rsi), %%ymm5\n\t"
> > +       "vmovdqu   (32 * 2)(%%rsi), %%ymm6\n\t"
> > +       "vmovdqu   (32 * 3)(%%rsi), %%ymm7\n\t"
> > +       "vmovdqu   -32(%%rsi,%%rdx), %%ymm8\n\t"
> > +       /* Save stop of the destination buffer.  */
> > +       "leaq    -32(%%rdi, %%rdx), %%r11\n\t"
> > +       /* Align destination end for aligned stores in the loop.
> Compute */
> > +       /* how much destination end is misaligned.  */
> > +       "leaq    -32(%%rsi, %%rdx), %%rcx\n\t"
> > +       "movq    %%r11, %%r9\n\t"
> > +       "movq    %%r11, %%r8\n\t"
> > +       "andq    $(32 - 1), %%r8\n\t"
> > +       /* Adjust source.  */
> > +       "subq    %%r8, %%rcx\n\t"
> > +       /* Adjust the end of destination which should be aligned now.  */
> > +       "subq    %%r8, %%r9\n\t"
> > +       /* Adjust length.  */
> > +       "subq    %%r8, %%rdx\n\t"
> > +        /* Check non-temporal store threshold.  */
> > +       "cmpq    $(1024*1024), %%rdx\n\t"
> > +       "ja      117f\n\t"
> > +       "114:\n\t"
> > +       /* Copy 4 * VEC a time backward.  */
> > +       "vmovdqu   (%%rcx), %%ymm0\n\t"
> > +       "vmovdqu   -32(%%rcx), %%ymm1\n\t"
> > +       "vmovdqu   -(32 * 2)(%%rcx), %%ymm2\n\t"
> > +       "vmovdqu   -(32 * 3)(%%rcx), %%ymm3\n\t"
> > +       "subq    $(32 * 4), %%rcx\n\t"
> > +       "subq    $(32 * 4), %%rdx\n\t"
> > +       "vmovdqa   %%ymm0, (%%r9)\n\t"
> > +       "vmovdqa   %%ymm1, -32(%%r9)\n\t"
> > +       "vmovdqa   %%ymm2, -(32 * 2)(%%r9)\n\t"
> > +       "vmovdqa   %%ymm3, -(32 * 3)(%%r9)\n\t"
> > +       "subq    $(32 * 4), %%r9\n\t"
> > +       "cmpq    $(32 * 4), %%rdx\n\t"
> > +       "ja      114b\n\t"
> > +       /* Store the first 4 * VEC. */
> > +       "vmovdqu   %%ymm4, (%%rdi)\n\t"
> > +       "vmovdqu   %%ymm5, 32(%%rdi)\n\t"
> > +       "vmovdqu   %%ymm6, (32 * 2)(%%rdi)\n\t"
> > +       "vmovdqu   %%ymm7, (32 * 3)(%%rdi)\n\t"
> > +       /* Store the last VEC. */
> > +       "vmovdqu   %%ymm8, (%%r11)\n\t"
> > +       "vzeroupper\n\t"
> > +       "jmp %l[done]\n\t"
> > +
> > +       "115:\n\t"
> > +       /* Don't use non-temporal store if there is overlap between */
> > +       /* destination and source since destination may be in cache */
> > +       /* when source is loaded. */
> > +       "leaq    (%%rdi, %%rdx), %%r10\n\t"
> > +       "cmpq    %%r10, %%rsi\n\t"
> > +       "jb      112b\n\t"
> > +       "116:\n\t"
> > +       /* Copy 4 * VEC a time forward with non-temporal stores.  */
> > +       "prefetcht0 (32*4*2)(%%rsi)\n\t"
> > +       "prefetcht0 (32*4*2 + 64)(%%rsi)\n\t"
> > +       "prefetcht0 (32*4*3)(%%rsi)\n\t"
> > +       "prefetcht0 (32*4*3 + 64)(%%rsi)\n\t"
> > +       "vmovdqu   (%%rsi), %%ymm0\n\t"
> > +       "vmovdqu   32(%%rsi), %%ymm1\n\t"
> > +       "vmovdqu   (32 * 2)(%%rsi), %%ymm2\n\t"
> > +       "vmovdqu   (32 * 3)(%%rsi), %%ymm3\n\t"
> > +       "addq    $(32*4), %%rsi\n\t"
> > +       "subq    $(32*4), %%rdx\n\t"
> > +       "vmovntdq  %%ymm0, (%%rdi)\n\t"
> > +       "vmovntdq  %%ymm1, 32(%%rdi)\n\t"
> > +       "vmovntdq  %%ymm2, (32 * 2)(%%rdi)\n\t"
> > +       "vmovntdq  %%ymm3, (32 * 3)(%%rdi)\n\t"
> > +       "addq    $(32*4), %%rdi\n\t"
> > +       "cmpq    $(32*4), %%rdx\n\t"
> > +       "ja      116b\n\t"
> > +       "sfence\n\t"
> > +       /* Store the last 4 * VEC.  */
> > +       "vmovdqu   %%ymm5, (%%rcx)\n\t"
> > +       "vmovdqu   %%ymm6, -32(%%rcx)\n\t"
> > +       "vmovdqu   %%ymm7, -(32 * 2)(%%rcx)\n\t"
> > +       "vmovdqu   %%ymm8, -(32 * 3)(%%rcx)\n\t"
> > +       /* Store the first VEC.  */
> > +       "vmovdqu   %%ymm4, (%%r11)\n\t"
> > +       "vzeroupper\n\t"
> > +       "jmp %l[done]\n\t"
> > +       "117:\n\t"
> > +       /* Don't use non-temporal store if there is overlap between */
> > +       /* destination and source since destination may be in cache */
> > +       /* when source is loaded.  */
> > +       "leaq    (%%rcx, %%rdx), %%r10\n\t"
> > +       "cmpq    %%r10, %%r9\n\t"
> > +       "jb      114b\n\t"
> > +       "118:\n\t"
> > +       /* Copy 4 * VEC a time backward with non-temporal stores. */
> > +       "prefetcht0 (-32 * 4 * 2)(%%rcx)\n\t"
> > +       "prefetcht0 (-32 * 4 * 2 - 64)(%%rcx)\n\t"
> > +       "prefetcht0 (-32 * 4 * 3)(%%rcx)\n\t"
> > +       "prefetcht0 (-32 * 4 * 3 - 64)(%%rcx)\n\t"
> > +       "vmovdqu   (%%rcx), %%ymm0\n\t"
> > +       "vmovdqu   -32(%%rcx), %%ymm1\n\t"
> > +       "vmovdqu   -(32 * 2)(%%rcx), %%ymm2\n\t"
> > +       "vmovdqu   -(32 * 3)(%%rcx), %%ymm3\n\t"
> > +       "subq    $(32*4), %%rcx\n\t"
> > +       "subq    $(32*4), %%rdx\n\t"
> > +       "vmovntdq  %%ymm0, (%%r9)\n\t"
> > +       "vmovntdq  %%ymm1, -32(%%r9)\n\t"
> > +       "vmovntdq  %%ymm2, -(32 * 2)(%%r9)\n\t"
> > +       "vmovntdq  %%ymm3, -(32 * 3)(%%r9)\n\t"
> > +       "subq    $(32 * 4), %%r9\n\t"
> > +       "cmpq    $(32 * 4), %%rdx\n\t"
> > +       "ja      118b\n\t"
> > +       "sfence\n\t"
> > +       /* Store the first 4 * VEC.  */
> > +       "vmovdqu   %%ymm4, (%%rdi)\n\t"
> > +       "vmovdqu   %%ymm5, 32(%%rdi)\n\t"
> > +       "vmovdqu   %%ymm6, (32 * 2)(%%rdi)\n\t"
> > +       "vmovdqu   %%ymm7, (32 * 3)(%%rdi)\n\t"
> > +       /* Store the last VEC.  */
> > +       "vmovdqu   %%ymm8, (%%r11)\n\t"
> > +       "vzeroupper\n\t"
> > +       "jmp %l[done]"
> > +       :
> > +       : "r"(src), "r"(dst), "r"(len)
> > +       : "rax", "rcx", "rdx", "rdi", "rsi", "r8", "r9", "r10", "r11",
> "r12", "ymm0",
> > +       "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", "ymm8",
> "memory"
> > +       : done
> > +       );
> > +done:
> > +       return dst;
> > +}
> > +
> > +#else
> >  static __rte_always_inline void *
> >  rte_memcpy_generic(void *dst, const void *src, size_t n)
> >  {
> > @@ -479,6 +971,8 @@ rte_memcpy_generic(void *dst, const void *src,
> size_t n)
> >         goto COPY_BLOCK_128_BACK31;
> >  }
> >
> > +#endif /* RTE_MEMCPY_AMDEPYC2 */
> > +
> >  #else /* __AVX512F__ */
> >
> >  #define ALIGNMENT_MASK 0x0F
> > @@ -874,6 +1368,14 @@ rte_memcpy(void *dst, const void *src, size_t n)
> >                 return rte_memcpy_generic(dst, src, n);
> >  }
> >
> > +#if defined __AVX2__ && defined(RTE_MEMCPY_AMDEPYC2)
> > +static __rte_always_inline void *
> > +rte_memcpy_aligned_tstore16(void *dst, void *src, int len)
> > +{
> > +       return rte_memcpy_aligned_ntload_tstore16_amdepyc2(dst, src,
> len);
> > +}
> > +#endif
> > +
> >  #if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION >= 100000)
> >  #pragma GCC diagnostic pop
> >  #endif
> > diff --git a/meson_options.txt b/meson_options.txt
> > index 0e92734c49..e232c9c340 100644
> > --- a/meson_options.txt
> > +++ b/meson_options.txt
> > @@ -42,6 +42,8 @@ option('platform', type: 'string', value: 'native',
> description:
> >         'Platform to build, either "native", "generic" or a SoC. Please
> refer to the Linux build guide for more information.')
> >  option('enable_trace_fp', type: 'boolean', value: false, description:
> >         'enable fast path trace points.')
> > +option('rte_memcpy_amdepyc2', type: 'boolean', value: false,
> description:
> > +       'to enable amd epyc memcpy routines')
> >  option('tests', type: 'boolean', value: true, description:
> >         'build unit tests')
> >  option('use_hpet', type: 'boolean', value: false, description:
> > --
> > 2.25.1
> >
>

[dpdk-dev] [PATCH v2 1/2] lib/eal: add amd epyc2 memcpy routine to eal

[Aman Kumar]

This patch provides rte_memcpy* calls optimized for
AMD EPYC platforms. Use config/x86/x86_amd_epyc_linux_gcc
as cross-file with meson to build dpdk for AMD EPYC platforms.

Signed-off-by: Aman Kumar <aman.kumar@vvdntech.in>
---
 config/x86/meson.build            |   7 +
 config/x86/x86_amd_epyc_linux_gcc |  16 +
 lib/eal/x86/include/rte_memcpy.h  | 502 ++++++++++++++++++++++++++++++
 3 files changed, 525 insertions(+)
 create mode 100644 config/x86/x86_amd_epyc_linux_gcc

diff --git a/config/x86/meson.build b/config/x86/meson.build
index 29f3dea181..598b0e62ce 100644
--- a/config/x86/meson.build
+++ b/config/x86/meson.build
@@ -72,3 +72,10 @@ endif
 dpdk_conf.set('RTE_CACHE_LINE_SIZE', 64)
 dpdk_conf.set('RTE_MAX_LCORE', 128)
 dpdk_conf.set('RTE_MAX_NUMA_NODES', 32)
+
+if meson.is_cross_build()
+	if meson.get_cross_property('platform') == 'amd-epyc'
+	    dpdk_conf.set('RTE_MAX_LCORE', 512)
+	    dpdk_conf.set('RTE_MEMCPY_AMDEPYC', 1)
+	endif
+endif
diff --git a/config/x86/x86_amd_epyc_linux_gcc b/config/x86/x86_amd_epyc_linux_gcc
new file mode 100644
index 0000000000..0b2453135f
--- /dev/null
+++ b/config/x86/x86_amd_epyc_linux_gcc
@@ -0,0 +1,16 @@
+[binaries]
+c = 'x86_64-linux-gnu-gcc'
+cpp = 'x86_64-linux-gnu-g++'
+ld = 'x86_64-linux-gnu-ld'
+ar = 'x86_64-linux-gnu-ar'
+strip = 'x86_64-linux-gnu-strip'
+pkgconfig = 'x86_64-linux-gnu-pkg-config'
+
+[host_machine]
+system = 'linux'
+cpu_family = 'x86_64'
+cpu = 'native'
+endian = 'little'
+
+[properties]
+platform = 'amd-epyc'
diff --git a/lib/eal/x86/include/rte_memcpy.h b/lib/eal/x86/include/rte_memcpy.h
index 79f381dd9b..8f66c115e3 100644
--- a/lib/eal/x86/include/rte_memcpy.h
+++ b/lib/eal/x86/include/rte_memcpy.h
@@ -368,6 +368,498 @@ rte_mov128blocks(uint8_t *dst, const uint8_t *src, size_t n)
 	}
 }
 
+#if defined RTE_MEMCPY_AMDEPYC
+
+/**
+ * Copy 16 bytes from one location to another,
+ * with temporal stores
+ */
+static __rte_always_inline void
+rte_copy16_ts(uint8_t *dst, uint8_t *src)
+{
+	__m128i var128;
+
+	var128 = _mm_stream_load_si128((__m128i *)src);
+	_mm_storeu_si128((__m128i *)dst, var128);
+}
+
+/**
+ * Copy 32 bytes from one location to another,
+ * with temporal stores
+ */
+static __rte_always_inline void
+rte_copy32_ts(uint8_t *dst, uint8_t *src)
+{
+	__m256i ymm0;
+
+	ymm0 = _mm256_stream_load_si256((const __m256i *)src);
+	_mm256_storeu_si256((__m256i *)dst, ymm0);
+}
+
+/**
+ * Copy 64 bytes from one location to another,
+ * with temporal stores
+ */
+static __rte_always_inline void
+rte_copy64_ts(uint8_t *dst, uint8_t *src)
+{
+	rte_copy32_ts(dst + 0 * 32, src + 0 * 32);
+	rte_copy32_ts(dst + 1 * 32, src + 1 * 32);
+}
+
+/**
+ * Copy 128 bytes from one location to another,
+ * with temporal stores
+ */
+static __rte_always_inline void
+rte_copy128_ts(uint8_t *dst, uint8_t *src)
+{
+	rte_copy32_ts(dst + 0 * 32, src + 0 * 32);
+	rte_copy32_ts(dst + 1 * 32, src + 1 * 32);
+	rte_copy32_ts(dst + 2 * 32, src + 2 * 32);
+	rte_copy32_ts(dst + 3 * 32, src + 3 * 32);
+}
+
+/**
+ * Copy len bytes from one location to another,
+ * with temporal stores 16B aligned
+ */
+static __rte_always_inline void *
+rte_memcpy_aligned_tstore16_generic(void *dst, void *src, int len)
+{
+	void *dest = dst;
+
+	while (len >= 128) {
+		rte_copy128_ts((uint8_t *)dst, (uint8_t *)src);
+		dst = (uint8_t *)dst + 128;
+		src = (uint8_t *)src + 128;
+		len -= 128;
+	}
+	while (len >= 64) {
+		rte_copy64_ts((uint8_t *)dst, (uint8_t *)src);
+		dst = (uint8_t *)dst + 64;
+		src = (uint8_t *)src + 64;
+		len -= 64;
+	}
+	while (len >= 32) {
+		rte_copy32_ts((uint8_t *)dst, (uint8_t *)src);
+		dst = (uint8_t *)dst + 32;
+		src = (uint8_t *)src + 32;
+		len -= 32;
+	}
+	if (len >= 16) {
+		rte_copy16_ts((uint8_t *)dst, (uint8_t *)src);
+		dst = (uint8_t *)dst + 16;
+		src = (uint8_t *)src + 16;
+		len -= 16;
+	}
+	if (len >= 8) {
+		*(uint64_t *)dst = *(const uint64_t *)src;
+		dst = (uint8_t *)dst + 8;
+		src = (uint8_t *)src + 8;
+		len -= 8;
+	}
+	if (len >= 4) {
+		*(uint32_t *)dst = *(const uint32_t *)src;
+		dst = (uint8_t *)dst + 4;
+		src = (uint8_t *)src + 4;
+		len -= 4;
+	}
+	if (len != 0) {
+		dst = (uint8_t *)dst - (4 - len);
+		src = (uint8_t *)src - (4 - len);
+		*(uint32_t *)dst = *(const uint32_t *)src;
+	}
+
+	return dest;
+}
+
+static __rte_always_inline void *
+rte_memcpy_aligned_ntload_tstore16_amdepyc2(void *dst,
+					    const void *src,
+					    size_t size)
+{
+	asm volatile goto("movq %0, %%rsi\n\t"
+	"movq %1, %%rdi\n\t"
+	"movq %2, %%rdx\n\t"
+	"cmpq   $(128), %%rdx\n\t"
+	"jb     202f\n\t"
+	"201:\n\t"
+	"vmovntdqa (%%rsi), %%ymm0\n\t"
+	"vmovntdqa 32(%%rsi), %%ymm1\n\t"
+	"vmovntdqa 64(%%rsi), %%ymm2\n\t"
+	"vmovntdqa 96(%%rsi), %%ymm3\n\t"
+	"vmovdqu  %%ymm0, (%%rdi)\n\t"
+	"vmovdqu  %%ymm1, 32(%%rdi)\n\t"
+	"vmovdqu  %%ymm2, 64(%%rdi)\n\t"
+	"vmovdqu  %%ymm3, 96(%%rdi)\n\t"
+	"addq   $128, %%rsi\n\t"
+	"addq   $128, %%rdi\n\t"
+	"subq   $128, %%rdx\n\t"
+	"jz     %l[done]\n\t"
+	"cmpq   $128, %%rdx\n\t" /*Vector Size 32B.  */
+	"jae    201b\n\t"
+	"202:\n\t"
+	"cmpq   $64, %%rdx\n\t"
+	"jb     203f\n\t"
+	"vmovntdqa (%%rsi), %%ymm0\n\t"
+	"vmovntdqa 32(%%rsi), %%ymm1\n\t"
+	"vmovdqu  %%ymm0, (%%rdi)\n\t"
+	"vmovdqu  %%ymm1, 32(%%rdi)\n\t"
+	"addq   $64, %%rsi\n\t"
+	"addq   $64, %%rdi\n\t"
+	"subq   $64, %%rdx\n\t"
+	"jz     %l[done]\n\t"
+	"203:\n\t"
+	"cmpq   $32, %%rdx\n\t"
+	"jb     204f\n\t"
+	"vmovntdqa (%%rsi), %%ymm0\n\t"
+	"vmovdqu  %%ymm0, (%%rdi)\n\t"
+	"addq   $32, %%rsi\n\t"
+	"addq   $32, %%rdi\n\t"
+	"subq   $32, %%rdx\n\t"
+	"jz     %l[done]\n\t"
+	"204:\n\t"
+	"cmpb   $16, %%dl\n\t"
+	"jb     205f\n\t"
+	"vmovntdqa (%%rsi), %%xmm0\n\t"
+	"vmovdqu  %%xmm0, (%%rdi)\n\t"
+	"addq   $16, %%rsi\n\t"
+	"addq   $16, %%rdi\n\t"
+	"subq   $16, %%rdx\n\t"
+	"jz     %l[done]\n\t"
+	"205:\n\t"
+	"cmpb   $2, %%dl\n\t"
+	"jb     208f\n\t"
+	"cmpb   $4, %%dl\n\t"
+	"jbe    207f\n\t"
+	"cmpb   $8, %%dl\n\t"
+	"jbe    206f\n\t"
+	"movq   -8(%%rsi,%%rdx), %%rcx\n\t"
+	"movq   (%%rsi), %%rsi\n\t"
+	"movq   %%rcx, -8(%%rdi,%%rdx)\n\t"
+	"movq   %%rsi, (%%rdi)\n\t"
+	"jmp    %l[done]\n\t"
+	"206:\n\t"
+	"movl   -4(%%rsi,%%rdx), %%ecx\n\t"
+	"movl   (%%rsi), %%esi\n\t"
+	"movl   %%ecx, -4(%%rdi,%%rdx)\n\t"
+	"movl   %%esi, (%%rdi)\n\t"
+	"jmp    %l[done]\n\t"
+	"207:\n\t"
+	"movzwl -2(%%rsi,%%rdx), %%ecx\n\t"
+	"movzwl (%%rsi), %%esi\n\t"
+	"movw   %%cx, -2(%%rdi,%%rdx)\n\t"
+	"movw   %%si, (%%rdi)\n\t"
+	"jmp    %l[done]\n\t"
+	"208:\n\t"
+	"movzbl (%%rsi), %%ecx\n\t"
+	"movb   %%cl, (%%rdi)"
+	:
+	: "r"(src), "r"(dst), "r"(size)
+	: "rcx", "rdx", "rsi", "rdi", "ymm0", "ymm1", "ymm2", "ymm3", "memory"
+	: done
+	);
+done:
+	return dst;
+}
+
+static __rte_always_inline void *
+rte_memcpy_generic(void *dst, const void *src, size_t len)
+{
+	asm goto("movq	%0, %%rsi\n\t"
+	"movq	%1, %%rdi\n\t"
+	"movq	%2, %%rdx\n\t"
+	"movq    %%rdi, %%rax\n\t"
+	"cmp     $32, %%rdx\n\t"
+	"jb      101f\n\t"
+	"cmp     $(32 * 2), %%rdx\n\t"
+	"ja      108f\n\t"
+	"vmovdqu   (%%rsi), %%ymm0\n\t"
+	"vmovdqu   -32(%%rsi,%%rdx), %%ymm1\n\t"
+	"vmovdqu   %%ymm0, (%%rdi)\n\t"
+	"vmovdqu   %%ymm1, -32(%%rdi,%%rdx)\n\t"
+	"vzeroupper\n\t"
+	"jmp %l[done]\n\t"
+	"101:\n\t"
+	/* Less than 1 VEC.  */
+	"cmpb    $32, %%dl\n\t"
+	"jae     103f\n\t"
+	"cmpb    $16, %%dl\n\t"
+	"jae     104f\n\t"
+	"cmpb    $8, %%dl\n\t"
+	"jae     105f\n\t"
+	"cmpb    $4, %%dl\n\t"
+	"jae     106f\n\t"
+	"cmpb    $1, %%dl\n\t"
+	"ja      107f\n\t"
+	"jb      102f\n\t"
+	"movzbl  (%%rsi), %%ecx\n\t"
+	"movb    %%cl, (%%rdi)\n\t"
+	"102:\n\t"
+	"jmp %l[done]\n\t"
+	"103:\n\t"
+	/* From 32 to 63.  No branch when size == 32.  */
+	"vmovdqu (%%rsi), %%ymm0\n\t"
+	"vmovdqu -32(%%rsi,%%rdx), %%ymm1\n\t"
+	"vmovdqu %%ymm0, (%%rdi)\n\t"
+	"vmovdqu %%ymm1, -32(%%rdi,%%rdx)\n\t"
+	"vzeroupper\n\t"
+	"jmp %l[done]\n\t"
+	/* From 16 to 31.  No branch when size == 16.  */
+	"104:\n\t"
+	"vmovdqu (%%rsi), %%xmm0\n\t"
+	"vmovdqu -16(%%rsi,%%rdx), %%xmm1\n\t"
+	"vmovdqu %%xmm0, (%%rdi)\n\t"
+	"vmovdqu %%xmm1, -16(%%rdi,%%rdx)\n\t"
+	"jmp %l[done]\n\t"
+	"105:\n\t"
+	/* From 8 to 15.  No branch when size == 8.  */
+	"movq    -8(%%rsi,%%rdx), %%rcx\n\t"
+	"movq    (%%rsi), %%rsi\n\t"
+	"movq    %%rcx, -8(%%rdi,%%rdx)\n\t"
+	"movq    %%rsi, (%%rdi)\n\t"
+	"jmp %l[done]\n\t"
+	"106:\n\t"
+	/* From 4 to 7.  No branch when size == 4.  */
+	"movl    -4(%%rsi,%%rdx), %%ecx\n\t"
+	"movl    (%%rsi), %%esi\n\t"
+	"movl    %%ecx, -4(%%rdi,%%rdx)\n\t"
+	"movl    %%esi, (%%rdi)\n\t"
+	"jmp %l[done]\n\t"
+	"107:\n\t"
+	/* From 2 to 3.  No branch when size == 2.  */
+	"movzwl  -2(%%rsi,%%rdx), %%ecx\n\t"
+	"movzwl  (%%rsi), %%esi\n\t"
+	"movw    %%cx, -2(%%rdi,%%rdx)\n\t"
+	"movw    %%si, (%%rdi)\n\t"
+	"jmp %l[done]\n\t"
+	"108:\n\t"
+	/* More than 2 * VEC and there may be overlap between destination */
+	/* and source.  */
+	"cmpq    $(32 * 8), %%rdx\n\t"
+	"ja      111f\n\t"
+	"cmpq    $(32 * 4), %%rdx\n\t"
+	"jb      109f\n\t"
+	/* Copy from 4 * VEC to 8 * VEC, inclusively. */
+	"vmovdqu   (%%rsi), %%ymm0\n\t"
+	"vmovdqu   32(%%rsi), %%ymm1\n\t"
+	"vmovdqu   (32 * 2)(%%rsi), %%ymm2\n\t"
+	"vmovdqu   (32 * 3)(%%rsi), %%ymm3\n\t"
+	"vmovdqu   -32(%%rsi,%%rdx), %%ymm4\n\t"
+	"vmovdqu   -(32 * 2)(%%rsi,%%rdx), %%ymm5\n\t"
+	"vmovdqu   -(32 * 3)(%%rsi,%%rdx), %%ymm6\n\t"
+	"vmovdqu   -(32 * 4)(%%rsi,%%rdx), %%ymm7\n\t"
+	"vmovdqu   %%ymm0, (%%rdi)\n\t"
+	"vmovdqu   %%ymm1, 32(%%rdi)\n\t"
+	"vmovdqu   %%ymm2, (32 * 2)(%%rdi)\n\t"
+	"vmovdqu   %%ymm3, (32 * 3)(%%rdi)\n\t"
+	"vmovdqu   %%ymm4, -32(%%rdi,%%rdx)\n\t"
+	"vmovdqu   %%ymm5, -(32 * 2)(%%rdi,%%rdx)\n\t"
+	"vmovdqu   %%ymm6, -(32 * 3)(%%rdi,%%rdx)\n\t"
+	"vmovdqu   %%ymm7, -(32 * 4)(%%rdi,%%rdx)\n\t"
+	"vzeroupper\n\t"
+	"jmp %l[done]\n\t"
+	"109:\n\t"
+	/* Copy from 2 * VEC to 4 * VEC. */
+	"vmovdqu   (%%rsi), %%ymm0\n\t"
+	"vmovdqu   32(%%rsi), %%ymm1\n\t"
+	"vmovdqu   -32(%%rsi,%%rdx), %%ymm2\n\t"
+	"vmovdqu   -(32 * 2)(%%rsi,%%rdx), %%ymm3\n\t"
+	"vmovdqu   %%ymm0, (%%rdi)\n\t"
+	"vmovdqu   %%ymm1, 32(%%rdi)\n\t"
+	"vmovdqu   %%ymm2, -32(%%rdi,%%rdx)\n\t"
+	"vmovdqu   %%ymm3, -(32 * 2)(%%rdi,%%rdx)\n\t"
+	"vzeroupper\n\t"
+	"110:\n\t"
+	"jmp %l[done]\n\t"
+	"111:\n\t"
+	"cmpq    %%rsi, %%rdi\n\t"
+	"ja      113f\n\t"
+	/* Source == destination is less common.  */
+	"je      110b\n\t"
+	/* Load the first VEC and last 4 * VEC to
+	 * support overlapping addresses.
+	 */
+	"vmovdqu   (%%rsi), %%ymm4\n\t"
+	"vmovdqu   -32(%%rsi, %%rdx), %%ymm5\n\t"
+	"vmovdqu   -(32 * 2)(%%rsi, %%rdx), %%ymm6\n\t"
+	"vmovdqu   -(32 * 3)(%%rsi, %%rdx), %%ymm7\n\t"
+	"vmovdqu   -(32 * 4)(%%rsi, %%rdx), %%ymm8\n\t"
+	/* Save start and stop of the destination buffer.  */
+	"movq    %%rdi, %%r11\n\t"
+	"leaq    -32(%%rdi, %%rdx), %%rcx\n\t"
+	/* Align destination for aligned stores in the loop.  Compute */
+	/* how much destination is misaligned.  */
+	"movq    %%rdi, %%r8\n\t"
+	"andq    $(32 - 1), %%r8\n\t"
+	/* Get the negative of offset for alignment.  */
+	"subq    $32, %%r8\n\t"
+	/* Adjust source.  */
+	"subq    %%r8, %%rsi\n\t"
+	/* Adjust destination which should be aligned now.  */
+	"subq    %%r8, %%rdi\n\t"
+	/* Adjust length.  */
+	"addq    %%r8, %%rdx\n\t"
+	/* Check non-temporal store threshold.  */
+	"cmpq	 $(1024*1024), %%rdx\n\t"
+	"ja      115f\n\t"
+	"112:\n\t"
+	/* Copy 4 * VEC a time forward.  */
+	"vmovdqu   (%%rsi), %%ymm0\n\t"
+	"vmovdqu   32(%%rsi), %%ymm1\n\t"
+	"vmovdqu   (32 * 2)(%%rsi), %%ymm2\n\t"
+	"vmovdqu   (32 * 3)(%%rsi), %%ymm3\n\t"
+	"addq    $(32 * 4), %%rsi\n\t"
+	"subq    $(32 * 4), %%rdx\n\t"
+	"vmovdqa   %%ymm0, (%%rdi)\n\t"
+	"vmovdqa   %%ymm1, 32(%%rdi)\n\t"
+	"vmovdqa   %%ymm2, (32 * 2)(%%rdi)\n\t"
+	"vmovdqa   %%ymm3, (32 * 3)(%%rdi)\n\t"
+	"addq    $(32 * 4), %%rdi\n\t"
+	"cmpq    $(32 * 4), %%rdx\n\t"
+	"ja      112b\n\t"
+	/* Store the last 4 * VEC.  */
+	"vmovdqu   %%ymm5, (%%rcx)\n\t"
+	"vmovdqu   %%ymm6, -32(%%rcx)\n\t"
+	"vmovdqu   %%ymm7, -(32 * 2)(%%rcx)\n\t"
+	"vmovdqu   %%ymm8, -(32 * 3)(%%rcx)\n\t"
+	/* Store the first VEC.  */
+	"vmovdqu   %%ymm4, (%%r11)\n\t"
+	"vzeroupper\n\t"
+	"jmp %l[done]\n\t"
+	"113:\n\t"
+	/* Load the first 4*VEC and last VEC to support overlapping addresses.*/
+	"vmovdqu   (%%rsi), %%ymm4\n\t"
+	"vmovdqu   32(%%rsi), %%ymm5\n\t"
+	"vmovdqu   (32 * 2)(%%rsi), %%ymm6\n\t"
+	"vmovdqu   (32 * 3)(%%rsi), %%ymm7\n\t"
+	"vmovdqu   -32(%%rsi,%%rdx), %%ymm8\n\t"
+	/* Save stop of the destination buffer.  */
+	"leaq    -32(%%rdi, %%rdx), %%r11\n\t"
+	/* Align destination end for aligned stores in the loop.  Compute */
+	/* how much destination end is misaligned.  */
+	"leaq    -32(%%rsi, %%rdx), %%rcx\n\t"
+	"movq    %%r11, %%r9\n\t"
+	"movq    %%r11, %%r8\n\t"
+	"andq    $(32 - 1), %%r8\n\t"
+	/* Adjust source.  */
+	"subq    %%r8, %%rcx\n\t"
+	/* Adjust the end of destination which should be aligned now.  */
+	"subq    %%r8, %%r9\n\t"
+	/* Adjust length.  */
+	"subq    %%r8, %%rdx\n\t"
+	 /* Check non-temporal store threshold.  */
+	"cmpq	 $(1024*1024), %%rdx\n\t"
+	"ja      117f\n\t"
+	"114:\n\t"
+	/* Copy 4 * VEC a time backward.  */
+	"vmovdqu   (%%rcx), %%ymm0\n\t"
+	"vmovdqu   -32(%%rcx), %%ymm1\n\t"
+	"vmovdqu   -(32 * 2)(%%rcx), %%ymm2\n\t"
+	"vmovdqu   -(32 * 3)(%%rcx), %%ymm3\n\t"
+	"subq    $(32 * 4), %%rcx\n\t"
+	"subq    $(32 * 4), %%rdx\n\t"
+	"vmovdqa   %%ymm0, (%%r9)\n\t"
+	"vmovdqa   %%ymm1, -32(%%r9)\n\t"
+	"vmovdqa   %%ymm2, -(32 * 2)(%%r9)\n\t"
+	"vmovdqa   %%ymm3, -(32 * 3)(%%r9)\n\t"
+	"subq    $(32 * 4), %%r9\n\t"
+	"cmpq    $(32 * 4), %%rdx\n\t"
+	"ja      114b\n\t"
+	/* Store the first 4 * VEC. */
+	"vmovdqu   %%ymm4, (%%rdi)\n\t"
+	"vmovdqu   %%ymm5, 32(%%rdi)\n\t"
+	"vmovdqu   %%ymm6, (32 * 2)(%%rdi)\n\t"
+	"vmovdqu   %%ymm7, (32 * 3)(%%rdi)\n\t"
+	/* Store the last VEC. */
+	"vmovdqu   %%ymm8, (%%r11)\n\t"
+	"vzeroupper\n\t"
+	"jmp %l[done]\n\t"
+
+	"115:\n\t"
+	/* Don't use non-temporal store if there is overlap between */
+	/* destination and source since destination may be in cache */
+	/* when source is loaded. */
+	"leaq    (%%rdi, %%rdx), %%r10\n\t"
+	"cmpq    %%r10, %%rsi\n\t"
+	"jb      112b\n\t"
+	"116:\n\t"
+	/* Copy 4 * VEC a time forward with non-temporal stores.  */
+	"prefetcht0 (32*4*2)(%%rsi)\n\t"
+	"prefetcht0 (32*4*2 + 64)(%%rsi)\n\t"
+	"prefetcht0 (32*4*3)(%%rsi)\n\t"
+	"prefetcht0 (32*4*3 + 64)(%%rsi)\n\t"
+	"vmovdqu   (%%rsi), %%ymm0\n\t"
+	"vmovdqu   32(%%rsi), %%ymm1\n\t"
+	"vmovdqu   (32 * 2)(%%rsi), %%ymm2\n\t"
+	"vmovdqu   (32 * 3)(%%rsi), %%ymm3\n\t"
+	"addq    $(32*4), %%rsi\n\t"
+	"subq    $(32*4), %%rdx\n\t"
+	"vmovntdq  %%ymm0, (%%rdi)\n\t"
+	"vmovntdq  %%ymm1, 32(%%rdi)\n\t"
+	"vmovntdq  %%ymm2, (32 * 2)(%%rdi)\n\t"
+	"vmovntdq  %%ymm3, (32 * 3)(%%rdi)\n\t"
+	"addq    $(32*4), %%rdi\n\t"
+	"cmpq    $(32*4), %%rdx\n\t"
+	"ja      116b\n\t"
+	"sfence\n\t"
+	/* Store the last 4 * VEC.  */
+	"vmovdqu   %%ymm5, (%%rcx)\n\t"
+	"vmovdqu   %%ymm6, -32(%%rcx)\n\t"
+	"vmovdqu   %%ymm7, -(32 * 2)(%%rcx)\n\t"
+	"vmovdqu   %%ymm8, -(32 * 3)(%%rcx)\n\t"
+	/* Store the first VEC.  */
+	"vmovdqu   %%ymm4, (%%r11)\n\t"
+	"vzeroupper\n\t"
+	"jmp %l[done]\n\t"
+	"117:\n\t"
+	/* Don't use non-temporal store if there is overlap between */
+	/* destination and source since destination may be in cache */
+	/* when source is loaded.  */
+	"leaq    (%%rcx, %%rdx), %%r10\n\t"
+	"cmpq    %%r10, %%r9\n\t"
+	"jb      114b\n\t"
+	"118:\n\t"
+	/* Copy 4 * VEC a time backward with non-temporal stores. */
+	"prefetcht0 (-32 * 4 * 2)(%%rcx)\n\t"
+	"prefetcht0 (-32 * 4 * 2 - 64)(%%rcx)\n\t"
+	"prefetcht0 (-32 * 4 * 3)(%%rcx)\n\t"
+	"prefetcht0 (-32 * 4 * 3 - 64)(%%rcx)\n\t"
+	"vmovdqu   (%%rcx), %%ymm0\n\t"
+	"vmovdqu   -32(%%rcx), %%ymm1\n\t"
+	"vmovdqu   -(32 * 2)(%%rcx), %%ymm2\n\t"
+	"vmovdqu   -(32 * 3)(%%rcx), %%ymm3\n\t"
+	"subq    $(32*4), %%rcx\n\t"
+	"subq    $(32*4), %%rdx\n\t"
+	"vmovntdq  %%ymm0, (%%r9)\n\t"
+	"vmovntdq  %%ymm1, -32(%%r9)\n\t"
+	"vmovntdq  %%ymm2, -(32 * 2)(%%r9)\n\t"
+	"vmovntdq  %%ymm3, -(32 * 3)(%%r9)\n\t"
+	"subq    $(32 * 4), %%r9\n\t"
+	"cmpq    $(32 * 4), %%rdx\n\t"
+	"ja      118b\n\t"
+	"sfence\n\t"
+	/* Store the first 4 * VEC.  */
+	"vmovdqu   %%ymm4, (%%rdi)\n\t"
+	"vmovdqu   %%ymm5, 32(%%rdi)\n\t"
+	"vmovdqu   %%ymm6, (32 * 2)(%%rdi)\n\t"
+	"vmovdqu   %%ymm7, (32 * 3)(%%rdi)\n\t"
+	/* Store the last VEC.  */
+	"vmovdqu   %%ymm8, (%%r11)\n\t"
+	"vzeroupper\n\t"
+	"jmp %l[done]"
+	:
+	: "r"(src), "r"(dst), "r"(len)
+	: "rax", "rcx", "rdx", "rdi", "rsi", "r8", "r9", "r10", "r11", "r12", "ymm0",
+	"ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", "ymm8", "memory"
+	: done
+	);
+done:
+	return dst;
+}
+
+#else
 static __rte_always_inline void *
 rte_memcpy_generic(void *dst, const void *src, size_t n)
 {
@@ -479,6 +971,8 @@ rte_memcpy_generic(void *dst, const void *src, size_t n)
 	goto COPY_BLOCK_128_BACK31;
 }
 
+#endif /* RTE_MEMCPY_AMDEPYC */
+
 #else /* __AVX512F__ */
 
 #define ALIGNMENT_MASK 0x0F
@@ -874,6 +1368,14 @@ rte_memcpy(void *dst, const void *src, size_t n)
 		return rte_memcpy_generic(dst, src, n);
 }
 
+#if defined __AVX2__ && defined(RTE_MEMCPY_AMDEPYC)
+static __rte_always_inline void *
+rte_memcpy_aligned_tstore16(void *dst, void *src, int len)
+{
+	return rte_memcpy_aligned_ntload_tstore16_amdepyc2(dst, src, len);
+}
+#endif
+
 #if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION >= 100000)
 #pragma GCC diagnostic pop
 #endif
-- 
2.25.1

[dpdk-dev] [PATCH v2 2/2] net/mlx5: optimize mprq memcpy for AMD EPYC2 plaform

[Aman Kumar]

add non temporal load and temporal store for mprq memcpy.
This utilizes AMD EPYC2 optimized rte_memcpy* routines and
only enabled if config/x86/x86_amd_epyc_linux_gcc is build.

Signed-off-by: Aman Kumar <aman.kumar@vvdntech.in>
---
 drivers/net/mlx5/mlx5_rx.h | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/drivers/net/mlx5/mlx5_rx.h b/drivers/net/mlx5/mlx5_rx.h
index 2b7ad3e48b..cda6aa02f2 100644
--- a/drivers/net/mlx5/mlx5_rx.h
+++ b/drivers/net/mlx5/mlx5_rx.h
@@ -422,6 +422,14 @@ mprq_buf_to_pkt(struct mlx5_rxq_data *rxq, struct rte_mbuf *pkt, uint32_t len,
 	const uint32_t offset = strd_idx * strd_sz + strd_shift;
 	void *addr = RTE_PTR_ADD(mlx5_mprq_buf_addr(buf, strd_n), offset);
 
+#ifdef RTE_MEMCPY_AMDEPYC
+	if (len <= rxq->mprq_max_memcpy_len) {
+		rte_prefetch1(addr);
+		if (len > RTE_CACHE_LINE_SIZE)
+			rte_prefetch2((void *)((uintptr_t)addr +
+					       RTE_CACHE_LINE_SIZE));
+	}
+#endif
 	/*
 	 * Memcpy packets to the target mbuf if:
 	 * - The size of packet is smaller than mprq_max_memcpy_len.
@@ -433,8 +441,19 @@ mprq_buf_to_pkt(struct mlx5_rxq_data *rxq, struct rte_mbuf *pkt, uint32_t len,
 	    (hdrm_overlap > 0 && !rxq->strd_scatter_en)) {
 		if (likely(len <=
 			   (uint32_t)(pkt->buf_len - RTE_PKTMBUF_HEADROOM))) {
+#ifdef RTE_MEMCPY_AMDEPYC
+			uintptr_t data_addr;
+
+			data_addr = (uintptr_t)rte_pktmbuf_mtod(pkt, void *);
+			if (!((data_addr | (uintptr_t)addr) & ALIGNMENT_MASK))
+				rte_memcpy_aligned_tstore16((void *)data_addr,
+					   addr, len);
+			else
+				rte_memcpy((void *)data_addr, addr, len);
+#else
 			rte_memcpy(rte_pktmbuf_mtod(pkt, void *),
 				   addr, len);
+#endif
 			DATA_LEN(pkt) = len;
 		} else if (rxq->strd_scatter_en) {
 			struct rte_mbuf *prev = pkt;
-- 
2.25.1

Re: [dpdk-dev] [PATCH v2 1/2] lib/eal: add amd epyc2 memcpy routine to eal

[Thomas Monjalon]

19/10/2021 12:47, Aman Kumar:
> This patch provides rte_memcpy* calls optimized for
> AMD EPYC platforms. Use config/x86/x86_amd_epyc_linux_gcc
> as cross-file with meson to build dpdk for AMD EPYC platforms.

Please split in 2 patches: platform & memcpy.

What optimization is specific to EPYC?

I dislike the asm code below.
What is AMD specific inside?
Can it use compiler intrinsics as it is done elsewhere?

> +static __rte_always_inline void *
> +rte_memcpy_aligned_ntload_tstore16_amdepyc2(void *dst,
> +					    const void *src,
> +					    size_t size)
> +{
> +	asm volatile goto("movq %0, %%rsi\n\t"
> +	"movq %1, %%rdi\n\t"
> +	"movq %2, %%rdx\n\t"
> +	"cmpq   $(128), %%rdx\n\t"
> +	"jb     202f\n\t"
> +	"201:\n\t"
> +	"vmovntdqa (%%rsi), %%ymm0\n\t"
> +	"vmovntdqa 32(%%rsi), %%ymm1\n\t"
> +	"vmovntdqa 64(%%rsi), %%ymm2\n\t"
> +	"vmovntdqa 96(%%rsi), %%ymm3\n\t"
> +	"vmovdqu  %%ymm0, (%%rdi)\n\t"
> +	"vmovdqu  %%ymm1, 32(%%rdi)\n\t"
> +	"vmovdqu  %%ymm2, 64(%%rdi)\n\t"
> +	"vmovdqu  %%ymm3, 96(%%rdi)\n\t"
> +	"addq   $128, %%rsi\n\t"
> +	"addq   $128, %%rdi\n\t"
> +	"subq   $128, %%rdx\n\t"
> +	"jz     %l[done]\n\t"
> +	"cmpq   $128, %%rdx\n\t" /*Vector Size 32B.  */
> +	"jae    201b\n\t"
> +	"202:\n\t"
> +	"cmpq   $64, %%rdx\n\t"
> +	"jb     203f\n\t"
> +	"vmovntdqa (%%rsi), %%ymm0\n\t"
> +	"vmovntdqa 32(%%rsi), %%ymm1\n\t"
> +	"vmovdqu  %%ymm0, (%%rdi)\n\t"
> +	"vmovdqu  %%ymm1, 32(%%rdi)\n\t"
> +	"addq   $64, %%rsi\n\t"
> +	"addq   $64, %%rdi\n\t"
> +	"subq   $64, %%rdx\n\t"
> +	"jz     %l[done]\n\t"
> +	"203:\n\t"
> +	"cmpq   $32, %%rdx\n\t"
> +	"jb     204f\n\t"
> +	"vmovntdqa (%%rsi), %%ymm0\n\t"
> +	"vmovdqu  %%ymm0, (%%rdi)\n\t"
> +	"addq   $32, %%rsi\n\t"
> +	"addq   $32, %%rdi\n\t"
> +	"subq   $32, %%rdx\n\t"
> +	"jz     %l[done]\n\t"
> +	"204:\n\t"
> +	"cmpb   $16, %%dl\n\t"
> +	"jb     205f\n\t"
> +	"vmovntdqa (%%rsi), %%xmm0\n\t"
> +	"vmovdqu  %%xmm0, (%%rdi)\n\t"
> +	"addq   $16, %%rsi\n\t"
> +	"addq   $16, %%rdi\n\t"
> +	"subq   $16, %%rdx\n\t"
> +	"jz     %l[done]\n\t"
> +	"205:\n\t"
> +	"cmpb   $2, %%dl\n\t"
> +	"jb     208f\n\t"
> +	"cmpb   $4, %%dl\n\t"
> +	"jbe    207f\n\t"
> +	"cmpb   $8, %%dl\n\t"
> +	"jbe    206f\n\t"
> +	"movq   -8(%%rsi,%%rdx), %%rcx\n\t"
> +	"movq   (%%rsi), %%rsi\n\t"
> +	"movq   %%rcx, -8(%%rdi,%%rdx)\n\t"
> +	"movq   %%rsi, (%%rdi)\n\t"
> +	"jmp    %l[done]\n\t"
> +	"206:\n\t"
> +	"movl   -4(%%rsi,%%rdx), %%ecx\n\t"
> +	"movl   (%%rsi), %%esi\n\t"
> +	"movl   %%ecx, -4(%%rdi,%%rdx)\n\t"
> +	"movl   %%esi, (%%rdi)\n\t"
> +	"jmp    %l[done]\n\t"
> +	"207:\n\t"
> +	"movzwl -2(%%rsi,%%rdx), %%ecx\n\t"
> +	"movzwl (%%rsi), %%esi\n\t"
> +	"movw   %%cx, -2(%%rdi,%%rdx)\n\t"
> +	"movw   %%si, (%%rdi)\n\t"
> +	"jmp    %l[done]\n\t"
> +	"208:\n\t"
> +	"movzbl (%%rsi), %%ecx\n\t"
> +	"movb   %%cl, (%%rdi)"
> +	:
> +	: "r"(src), "r"(dst), "r"(size)
> +	: "rcx", "rdx", "rsi", "rdi", "ymm0", "ymm1", "ymm2", "ymm3", "memory"
> +	: done
> +	);
> +done:
> +	return dst;
> +}
> +
> +static __rte_always_inline void *
> +rte_memcpy_generic(void *dst, const void *src, size_t len)
> +{
> +	asm goto("movq	%0, %%rsi\n\t"
> +	"movq	%1, %%rdi\n\t"
> +	"movq	%2, %%rdx\n\t"
> +	"movq    %%rdi, %%rax\n\t"
> +	"cmp     $32, %%rdx\n\t"
> +	"jb      101f\n\t"
> +	"cmp     $(32 * 2), %%rdx\n\t"
> +	"ja      108f\n\t"
> +	"vmovdqu   (%%rsi), %%ymm0\n\t"
> +	"vmovdqu   -32(%%rsi,%%rdx), %%ymm1\n\t"
> +	"vmovdqu   %%ymm0, (%%rdi)\n\t"
> +	"vmovdqu   %%ymm1, -32(%%rdi,%%rdx)\n\t"
> +	"vzeroupper\n\t"
> +	"jmp %l[done]\n\t"
> +	"101:\n\t"
> +	/* Less than 1 VEC.  */
> +	"cmpb    $32, %%dl\n\t"
> +	"jae     103f\n\t"
> +	"cmpb    $16, %%dl\n\t"
> +	"jae     104f\n\t"
> +	"cmpb    $8, %%dl\n\t"
> +	"jae     105f\n\t"
> +	"cmpb    $4, %%dl\n\t"
> +	"jae     106f\n\t"
> +	"cmpb    $1, %%dl\n\t"
> +	"ja      107f\n\t"
> +	"jb      102f\n\t"
> +	"movzbl  (%%rsi), %%ecx\n\t"
> +	"movb    %%cl, (%%rdi)\n\t"
> +	"102:\n\t"
> +	"jmp %l[done]\n\t"
> +	"103:\n\t"
> +	/* From 32 to 63.  No branch when size == 32.  */
> +	"vmovdqu (%%rsi), %%ymm0\n\t"
> +	"vmovdqu -32(%%rsi,%%rdx), %%ymm1\n\t"
> +	"vmovdqu %%ymm0, (%%rdi)\n\t"
> +	"vmovdqu %%ymm1, -32(%%rdi,%%rdx)\n\t"
> +	"vzeroupper\n\t"
> +	"jmp %l[done]\n\t"
> +	/* From 16 to 31.  No branch when size == 16.  */
> +	"104:\n\t"
> +	"vmovdqu (%%rsi), %%xmm0\n\t"
> +	"vmovdqu -16(%%rsi,%%rdx), %%xmm1\n\t"
> +	"vmovdqu %%xmm0, (%%rdi)\n\t"
> +	"vmovdqu %%xmm1, -16(%%rdi,%%rdx)\n\t"
> +	"jmp %l[done]\n\t"
> +	"105:\n\t"
> +	/* From 8 to 15.  No branch when size == 8.  */
> +	"movq    -8(%%rsi,%%rdx), %%rcx\n\t"
> +	"movq    (%%rsi), %%rsi\n\t"
> +	"movq    %%rcx, -8(%%rdi,%%rdx)\n\t"
> +	"movq    %%rsi, (%%rdi)\n\t"
> +	"jmp %l[done]\n\t"
> +	"106:\n\t"
> +	/* From 4 to 7.  No branch when size == 4.  */
> +	"movl    -4(%%rsi,%%rdx), %%ecx\n\t"
> +	"movl    (%%rsi), %%esi\n\t"
> +	"movl    %%ecx, -4(%%rdi,%%rdx)\n\t"
> +	"movl    %%esi, (%%rdi)\n\t"
> +	"jmp %l[done]\n\t"
> +	"107:\n\t"
> +	/* From 2 to 3.  No branch when size == 2.  */
> +	"movzwl  -2(%%rsi,%%rdx), %%ecx\n\t"
> +	"movzwl  (%%rsi), %%esi\n\t"
> +	"movw    %%cx, -2(%%rdi,%%rdx)\n\t"
> +	"movw    %%si, (%%rdi)\n\t"
> +	"jmp %l[done]\n\t"
> +	"108:\n\t"
> +	/* More than 2 * VEC and there may be overlap between destination */
> +	/* and source.  */
> +	"cmpq    $(32 * 8), %%rdx\n\t"
> +	"ja      111f\n\t"
> +	"cmpq    $(32 * 4), %%rdx\n\t"
> +	"jb      109f\n\t"
> +	/* Copy from 4 * VEC to 8 * VEC, inclusively. */
> +	"vmovdqu   (%%rsi), %%ymm0\n\t"
> +	"vmovdqu   32(%%rsi), %%ymm1\n\t"
> +	"vmovdqu   (32 * 2)(%%rsi), %%ymm2\n\t"
> +	"vmovdqu   (32 * 3)(%%rsi), %%ymm3\n\t"
> +	"vmovdqu   -32(%%rsi,%%rdx), %%ymm4\n\t"
> +	"vmovdqu   -(32 * 2)(%%rsi,%%rdx), %%ymm5\n\t"
> +	"vmovdqu   -(32 * 3)(%%rsi,%%rdx), %%ymm6\n\t"
> +	"vmovdqu   -(32 * 4)(%%rsi,%%rdx), %%ymm7\n\t"
> +	"vmovdqu   %%ymm0, (%%rdi)\n\t"
> +	"vmovdqu   %%ymm1, 32(%%rdi)\n\t"
> +	"vmovdqu   %%ymm2, (32 * 2)(%%rdi)\n\t"
> +	"vmovdqu   %%ymm3, (32 * 3)(%%rdi)\n\t"
> +	"vmovdqu   %%ymm4, -32(%%rdi,%%rdx)\n\t"
> +	"vmovdqu   %%ymm5, -(32 * 2)(%%rdi,%%rdx)\n\t"
> +	"vmovdqu   %%ymm6, -(32 * 3)(%%rdi,%%rdx)\n\t"
> +	"vmovdqu   %%ymm7, -(32 * 4)(%%rdi,%%rdx)\n\t"
> +	"vzeroupper\n\t"
> +	"jmp %l[done]\n\t"
> +	"109:\n\t"
> +	/* Copy from 2 * VEC to 4 * VEC. */
> +	"vmovdqu   (%%rsi), %%ymm0\n\t"
> +	"vmovdqu   32(%%rsi), %%ymm1\n\t"
> +	"vmovdqu   -32(%%rsi,%%rdx), %%ymm2\n\t"
> +	"vmovdqu   -(32 * 2)(%%rsi,%%rdx), %%ymm3\n\t"
> +	"vmovdqu   %%ymm0, (%%rdi)\n\t"
> +	"vmovdqu   %%ymm1, 32(%%rdi)\n\t"
> +	"vmovdqu   %%ymm2, -32(%%rdi,%%rdx)\n\t"
> +	"vmovdqu   %%ymm3, -(32 * 2)(%%rdi,%%rdx)\n\t"
> +	"vzeroupper\n\t"
> +	"110:\n\t"
> +	"jmp %l[done]\n\t"
> +	"111:\n\t"
> +	"cmpq    %%rsi, %%rdi\n\t"
> +	"ja      113f\n\t"
> +	/* Source == destination is less common.  */
> +	"je      110b\n\t"
> +	/* Load the first VEC and last 4 * VEC to
> +	 * support overlapping addresses.
> +	 */
> +	"vmovdqu   (%%rsi), %%ymm4\n\t"
> +	"vmovdqu   -32(%%rsi, %%rdx), %%ymm5\n\t"
> +	"vmovdqu   -(32 * 2)(%%rsi, %%rdx), %%ymm6\n\t"
> +	"vmovdqu   -(32 * 3)(%%rsi, %%rdx), %%ymm7\n\t"
> +	"vmovdqu   -(32 * 4)(%%rsi, %%rdx), %%ymm8\n\t"
> +	/* Save start and stop of the destination buffer.  */
> +	"movq    %%rdi, %%r11\n\t"
> +	"leaq    -32(%%rdi, %%rdx), %%rcx\n\t"
> +	/* Align destination for aligned stores in the loop.  Compute */
> +	/* how much destination is misaligned.  */
> +	"movq    %%rdi, %%r8\n\t"
> +	"andq    $(32 - 1), %%r8\n\t"
> +	/* Get the negative of offset for alignment.  */
> +	"subq    $32, %%r8\n\t"
> +	/* Adjust source.  */
> +	"subq    %%r8, %%rsi\n\t"
> +	/* Adjust destination which should be aligned now.  */
> +	"subq    %%r8, %%rdi\n\t"
> +	/* Adjust length.  */
> +	"addq    %%r8, %%rdx\n\t"
> +	/* Check non-temporal store threshold.  */
> +	"cmpq	 $(1024*1024), %%rdx\n\t"
> +	"ja      115f\n\t"
> +	"112:\n\t"
> +	/* Copy 4 * VEC a time forward.  */
> +	"vmovdqu   (%%rsi), %%ymm0\n\t"
> +	"vmovdqu   32(%%rsi), %%ymm1\n\t"
> +	"vmovdqu   (32 * 2)(%%rsi), %%ymm2\n\t"
> +	"vmovdqu   (32 * 3)(%%rsi), %%ymm3\n\t"
> +	"addq    $(32 * 4), %%rsi\n\t"
> +	"subq    $(32 * 4), %%rdx\n\t"
> +	"vmovdqa   %%ymm0, (%%rdi)\n\t"
> +	"vmovdqa   %%ymm1, 32(%%rdi)\n\t"
> +	"vmovdqa   %%ymm2, (32 * 2)(%%rdi)\n\t"
> +	"vmovdqa   %%ymm3, (32 * 3)(%%rdi)\n\t"
> +	"addq    $(32 * 4), %%rdi\n\t"
> +	"cmpq    $(32 * 4), %%rdx\n\t"
> +	"ja      112b\n\t"
> +	/* Store the last 4 * VEC.  */
> +	"vmovdqu   %%ymm5, (%%rcx)\n\t"
> +	"vmovdqu   %%ymm6, -32(%%rcx)\n\t"
> +	"vmovdqu   %%ymm7, -(32 * 2)(%%rcx)\n\t"
> +	"vmovdqu   %%ymm8, -(32 * 3)(%%rcx)\n\t"
> +	/* Store the first VEC.  */
> +	"vmovdqu   %%ymm4, (%%r11)\n\t"
> +	"vzeroupper\n\t"
> +	"jmp %l[done]\n\t"
> +	"113:\n\t"
> +	/* Load the first 4*VEC and last VEC to support overlapping addresses.*/
> +	"vmovdqu   (%%rsi), %%ymm4\n\t"
> +	"vmovdqu   32(%%rsi), %%ymm5\n\t"
> +	"vmovdqu   (32 * 2)(%%rsi), %%ymm6\n\t"
> +	"vmovdqu   (32 * 3)(%%rsi), %%ymm7\n\t"
> +	"vmovdqu   -32(%%rsi,%%rdx), %%ymm8\n\t"
> +	/* Save stop of the destination buffer.  */
> +	"leaq    -32(%%rdi, %%rdx), %%r11\n\t"
> +	/* Align destination end for aligned stores in the loop.  Compute */
> +	/* how much destination end is misaligned.  */
> +	"leaq    -32(%%rsi, %%rdx), %%rcx\n\t"
> +	"movq    %%r11, %%r9\n\t"
> +	"movq    %%r11, %%r8\n\t"
> +	"andq    $(32 - 1), %%r8\n\t"
> +	/* Adjust source.  */
> +	"subq    %%r8, %%rcx\n\t"
> +	/* Adjust the end of destination which should be aligned now.  */
> +	"subq    %%r8, %%r9\n\t"
> +	/* Adjust length.  */
> +	"subq    %%r8, %%rdx\n\t"
> +	 /* Check non-temporal store threshold.  */
> +	"cmpq	 $(1024*1024), %%rdx\n\t"
> +	"ja      117f\n\t"
> +	"114:\n\t"
> +	/* Copy 4 * VEC a time backward.  */
> +	"vmovdqu   (%%rcx), %%ymm0\n\t"
> +	"vmovdqu   -32(%%rcx), %%ymm1\n\t"
> +	"vmovdqu   -(32 * 2)(%%rcx), %%ymm2\n\t"
> +	"vmovdqu   -(32 * 3)(%%rcx), %%ymm3\n\t"
> +	"subq    $(32 * 4), %%rcx\n\t"
> +	"subq    $(32 * 4), %%rdx\n\t"
> +	"vmovdqa   %%ymm0, (%%r9)\n\t"
> +	"vmovdqa   %%ymm1, -32(%%r9)\n\t"
> +	"vmovdqa   %%ymm2, -(32 * 2)(%%r9)\n\t"
> +	"vmovdqa   %%ymm3, -(32 * 3)(%%r9)\n\t"
> +	"subq    $(32 * 4), %%r9\n\t"
> +	"cmpq    $(32 * 4), %%rdx\n\t"
> +	"ja      114b\n\t"
> +	/* Store the first 4 * VEC. */
> +	"vmovdqu   %%ymm4, (%%rdi)\n\t"
> +	"vmovdqu   %%ymm5, 32(%%rdi)\n\t"
> +	"vmovdqu   %%ymm6, (32 * 2)(%%rdi)\n\t"
> +	"vmovdqu   %%ymm7, (32 * 3)(%%rdi)\n\t"
> +	/* Store the last VEC. */
> +	"vmovdqu   %%ymm8, (%%r11)\n\t"
> +	"vzeroupper\n\t"
> +	"jmp %l[done]\n\t"
> +
> +	"115:\n\t"
> +	/* Don't use non-temporal store if there is overlap between */
> +	/* destination and source since destination may be in cache */
> +	/* when source is loaded. */
> +	"leaq    (%%rdi, %%rdx), %%r10\n\t"
> +	"cmpq    %%r10, %%rsi\n\t"
> +	"jb      112b\n\t"
> +	"116:\n\t"
> +	/* Copy 4 * VEC a time forward with non-temporal stores.  */
> +	"prefetcht0 (32*4*2)(%%rsi)\n\t"
> +	"prefetcht0 (32*4*2 + 64)(%%rsi)\n\t"
> +	"prefetcht0 (32*4*3)(%%rsi)\n\t"
> +	"prefetcht0 (32*4*3 + 64)(%%rsi)\n\t"
> +	"vmovdqu   (%%rsi), %%ymm0\n\t"
> +	"vmovdqu   32(%%rsi), %%ymm1\n\t"
> +	"vmovdqu   (32 * 2)(%%rsi), %%ymm2\n\t"
> +	"vmovdqu   (32 * 3)(%%rsi), %%ymm3\n\t"
> +	"addq    $(32*4), %%rsi\n\t"
> +	"subq    $(32*4), %%rdx\n\t"
> +	"vmovntdq  %%ymm0, (%%rdi)\n\t"
> +	"vmovntdq  %%ymm1, 32(%%rdi)\n\t"
> +	"vmovntdq  %%ymm2, (32 * 2)(%%rdi)\n\t"
> +	"vmovntdq  %%ymm3, (32 * 3)(%%rdi)\n\t"
> +	"addq    $(32*4), %%rdi\n\t"
> +	"cmpq    $(32*4), %%rdx\n\t"
> +	"ja      116b\n\t"
> +	"sfence\n\t"
> +	/* Store the last 4 * VEC.  */
> +	"vmovdqu   %%ymm5, (%%rcx)\n\t"
> +	"vmovdqu   %%ymm6, -32(%%rcx)\n\t"
> +	"vmovdqu   %%ymm7, -(32 * 2)(%%rcx)\n\t"
> +	"vmovdqu   %%ymm8, -(32 * 3)(%%rcx)\n\t"
> +	/* Store the first VEC.  */
> +	"vmovdqu   %%ymm4, (%%r11)\n\t"
> +	"vzeroupper\n\t"
> +	"jmp %l[done]\n\t"
> +	"117:\n\t"
> +	/* Don't use non-temporal store if there is overlap between */
> +	/* destination and source since destination may be in cache */
> +	/* when source is loaded.  */
> +	"leaq    (%%rcx, %%rdx), %%r10\n\t"
> +	"cmpq    %%r10, %%r9\n\t"
> +	"jb      114b\n\t"
> +	"118:\n\t"
> +	/* Copy 4 * VEC a time backward with non-temporal stores. */
> +	"prefetcht0 (-32 * 4 * 2)(%%rcx)\n\t"
> +	"prefetcht0 (-32 * 4 * 2 - 64)(%%rcx)\n\t"
> +	"prefetcht0 (-32 * 4 * 3)(%%rcx)\n\t"
> +	"prefetcht0 (-32 * 4 * 3 - 64)(%%rcx)\n\t"
> +	"vmovdqu   (%%rcx), %%ymm0\n\t"
> +	"vmovdqu   -32(%%rcx), %%ymm1\n\t"
> +	"vmovdqu   -(32 * 2)(%%rcx), %%ymm2\n\t"
> +	"vmovdqu   -(32 * 3)(%%rcx), %%ymm3\n\t"
> +	"subq    $(32*4), %%rcx\n\t"
> +	"subq    $(32*4), %%rdx\n\t"
> +	"vmovntdq  %%ymm0, (%%r9)\n\t"
> +	"vmovntdq  %%ymm1, -32(%%r9)\n\t"
> +	"vmovntdq  %%ymm2, -(32 * 2)(%%r9)\n\t"
> +	"vmovntdq  %%ymm3, -(32 * 3)(%%r9)\n\t"
> +	"subq    $(32 * 4), %%r9\n\t"
> +	"cmpq    $(32 * 4), %%rdx\n\t"
> +	"ja      118b\n\t"
> +	"sfence\n\t"
> +	/* Store the first 4 * VEC.  */
> +	"vmovdqu   %%ymm4, (%%rdi)\n\t"
> +	"vmovdqu   %%ymm5, 32(%%rdi)\n\t"
> +	"vmovdqu   %%ymm6, (32 * 2)(%%rdi)\n\t"
> +	"vmovdqu   %%ymm7, (32 * 3)(%%rdi)\n\t"
> +	/* Store the last VEC.  */
> +	"vmovdqu   %%ymm8, (%%r11)\n\t"
> +	"vzeroupper\n\t"
> +	"jmp %l[done]"
> +	:
> +	: "r"(src), "r"(dst), "r"(len)
> +	: "rax", "rcx", "rdx", "rdi", "rsi", "r8", "r9", "r10", "r11", "r12", "ymm0",
> +	"ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", "ymm8", "memory"
> +	: done
> +	);
> +done:
> +	return dst;
> +}

Re: [dpdk-dev] [PATCH v2 1/2] lib/eal: add amd epyc2 memcpy routine to eal

[Stephen Hemminger]

On Tue, 19 Oct 2021 14:31:01 +0200
Thomas Monjalon <thomas@monjalon.net> wrote:

> 19/10/2021 12:47, Aman Kumar:
> > This patch provides rte_memcpy* calls optimized for
> > AMD EPYC platforms. Use config/x86/x86_amd_epyc_linux_gcc
> > as cross-file with meson to build dpdk for AMD EPYC platforms.  
> 
> Please split in 2 patches: platform & memcpy.
> 
> What optimization is specific to EPYC?
> 
> I dislike the asm code below.
> What is AMD specific inside?
> Can it use compiler intrinsics as it is done elsewhere?

And why is this not done by Gcc?

Re: [dpdk-dev] [PATCH v2 1/2] lib/eal: add amd epyc2 memcpy routine to eal

[Song, Keesang]

[AMD Official Use Only]

Hi Thomas,

I hope this can make some explanation to your question.
We(AMD Linux library support team) have implemented the custom tailored memcpy solution which is a close match with DPDK use case requirements like the below.
1)      Min 64B length data packet with cache aligned Source and Destination.
2)      Non-Temporal load and temporal store for cache aligned source for both RX and TX paths. Could not implement the non-temporal store for TX_PATH, as non-Temporal load/stores works only with 32B aligned addresses for AVX2
3)      This solution works for all AVX2 supported AMD machines.

Internally we have completed the integrity testing and benchmarking of the solution and found gains of 8.4% to 14.5% specifically on Milan CPU(3rd Gen of EPYC Processor)

Thanks for your support,
Keesang

-----Original Message-----
From: Thomas Monjalon <thomas@monjalon.net>
Sent: Tuesday, October 19, 2021 5:31 AM
To: Aman Kumar <aman.kumar@vvdntech.in>
Cc: dev@dpdk.org; rasland@nvidia.com; asafp@nvidia.com; shys@nvidia.com; viacheslavo@nvidia.com; akozyrev@nvidia.com; matan@nvidia.com; anatoly.burakov@intel.com; Song, Keesang <Keesang.Song@amd.com>; aman.kumar@vvdntech.in; jerinjacobk@gmail.com; bruce.richardson@intel.com; konstantin.ananyev@intel.com; david.marchand@redhat.com
Subject: Re: [dpdk-dev] [PATCH v2 1/2] lib/eal: add amd epyc2 memcpy routine to eal

[CAUTION: External Email]

19/10/2021 12:47, Aman Kumar:
> This patch provides rte_memcpy* calls optimized for AMD EPYC
> platforms. Use config/x86/x86_amd_epyc_linux_gcc as cross-file with
> meson to build dpdk for AMD EPYC platforms.

Please split in 2 patches: platform & memcpy.

What optimization is specific to EPYC?

I dislike the asm code below.
What is AMD specific inside?
Can it use compiler intrinsics as it is done elsewhere?

> +static __rte_always_inline void *
> +rte_memcpy_aligned_ntload_tstore16_amdepyc2(void *dst,
> +                                         const void *src,
> +                                         size_t size) {
> +     asm volatile goto("movq %0, %%rsi\n\t"
> +     "movq %1, %%rdi\n\t"
> +     "movq %2, %%rdx\n\t"
> +     "cmpq   $(128), %%rdx\n\t"
> +     "jb     202f\n\t"
> +     "201:\n\t"
> +     "vmovntdqa (%%rsi), %%ymm0\n\t"
> +     "vmovntdqa 32(%%rsi), %%ymm1\n\t"
> +     "vmovntdqa 64(%%rsi), %%ymm2\n\t"
> +     "vmovntdqa 96(%%rsi), %%ymm3\n\t"
> +     "vmovdqu  %%ymm0, (%%rdi)\n\t"
> +     "vmovdqu  %%ymm1, 32(%%rdi)\n\t"
> +     "vmovdqu  %%ymm2, 64(%%rdi)\n\t"
> +     "vmovdqu  %%ymm3, 96(%%rdi)\n\t"
> +     "addq   $128, %%rsi\n\t"
> +     "addq   $128, %%rdi\n\t"
> +     "subq   $128, %%rdx\n\t"
> +     "jz     %l[done]\n\t"
> +     "cmpq   $128, %%rdx\n\t" /*Vector Size 32B.  */
> +     "jae    201b\n\t"
> +     "202:\n\t"
> +     "cmpq   $64, %%rdx\n\t"
> +     "jb     203f\n\t"
> +     "vmovntdqa (%%rsi), %%ymm0\n\t"
> +     "vmovntdqa 32(%%rsi), %%ymm1\n\t"
> +     "vmovdqu  %%ymm0, (%%rdi)\n\t"
> +     "vmovdqu  %%ymm1, 32(%%rdi)\n\t"
> +     "addq   $64, %%rsi\n\t"
> +     "addq   $64, %%rdi\n\t"
> +     "subq   $64, %%rdx\n\t"
> +     "jz     %l[done]\n\t"
> +     "203:\n\t"
> +     "cmpq   $32, %%rdx\n\t"
> +     "jb     204f\n\t"
> +     "vmovntdqa (%%rsi), %%ymm0\n\t"
> +     "vmovdqu  %%ymm0, (%%rdi)\n\t"
> +     "addq   $32, %%rsi\n\t"
> +     "addq   $32, %%rdi\n\t"
> +     "subq   $32, %%rdx\n\t"
> +     "jz     %l[done]\n\t"
> +     "204:\n\t"
> +     "cmpb   $16, %%dl\n\t"
> +     "jb     205f\n\t"
> +     "vmovntdqa (%%rsi), %%xmm0\n\t"
> +     "vmovdqu  %%xmm0, (%%rdi)\n\t"
> +     "addq   $16, %%rsi\n\t"
> +     "addq   $16, %%rdi\n\t"
> +     "subq   $16, %%rdx\n\t"
> +     "jz     %l[done]\n\t"
> +     "205:\n\t"
> +     "cmpb   $2, %%dl\n\t"
> +     "jb     208f\n\t"
> +     "cmpb   $4, %%dl\n\t"
> +     "jbe    207f\n\t"
> +     "cmpb   $8, %%dl\n\t"
> +     "jbe    206f\n\t"
> +     "movq   -8(%%rsi,%%rdx), %%rcx\n\t"
> +     "movq   (%%rsi), %%rsi\n\t"
> +     "movq   %%rcx, -8(%%rdi,%%rdx)\n\t"
> +     "movq   %%rsi, (%%rdi)\n\t"
> +     "jmp    %l[done]\n\t"
> +     "206:\n\t"
> +     "movl   -4(%%rsi,%%rdx), %%ecx\n\t"
> +     "movl   (%%rsi), %%esi\n\t"
> +     "movl   %%ecx, -4(%%rdi,%%rdx)\n\t"
> +     "movl   %%esi, (%%rdi)\n\t"
> +     "jmp    %l[done]\n\t"
> +     "207:\n\t"
> +     "movzwl -2(%%rsi,%%rdx), %%ecx\n\t"
> +     "movzwl (%%rsi), %%esi\n\t"
> +     "movw   %%cx, -2(%%rdi,%%rdx)\n\t"
> +     "movw   %%si, (%%rdi)\n\t"
> +     "jmp    %l[done]\n\t"
> +     "208:\n\t"
> +     "movzbl (%%rsi), %%ecx\n\t"
> +     "movb   %%cl, (%%rdi)"
> +     :
> +     : "r"(src), "r"(dst), "r"(size)
> +     : "rcx", "rdx", "rsi", "rdi", "ymm0", "ymm1", "ymm2", "ymm3", "memory"
> +     : done
> +     );
> +done:
> +     return dst;
> +}
> +
> +static __rte_always_inline void *
> +rte_memcpy_generic(void *dst, const void *src, size_t len) {
> +     asm goto("movq  %0, %%rsi\n\t"
> +     "movq   %1, %%rdi\n\t"
> +     "movq   %2, %%rdx\n\t"
> +     "movq    %%rdi, %%rax\n\t"
> +     "cmp     $32, %%rdx\n\t"
> +     "jb      101f\n\t"
> +     "cmp     $(32 * 2), %%rdx\n\t"
> +     "ja      108f\n\t"
> +     "vmovdqu   (%%rsi), %%ymm0\n\t"
> +     "vmovdqu   -32(%%rsi,%%rdx), %%ymm1\n\t"
> +     "vmovdqu   %%ymm0, (%%rdi)\n\t"
> +     "vmovdqu   %%ymm1, -32(%%rdi,%%rdx)\n\t"
> +     "vzeroupper\n\t"
> +     "jmp %l[done]\n\t"
> +     "101:\n\t"
> +     /* Less than 1 VEC.  */
> +     "cmpb    $32, %%dl\n\t"
> +     "jae     103f\n\t"
> +     "cmpb    $16, %%dl\n\t"
> +     "jae     104f\n\t"
> +     "cmpb    $8, %%dl\n\t"
> +     "jae     105f\n\t"
> +     "cmpb    $4, %%dl\n\t"
> +     "jae     106f\n\t"
> +     "cmpb    $1, %%dl\n\t"
> +     "ja      107f\n\t"
> +     "jb      102f\n\t"
> +     "movzbl  (%%rsi), %%ecx\n\t"
> +     "movb    %%cl, (%%rdi)\n\t"
> +     "102:\n\t"
> +     "jmp %l[done]\n\t"
> +     "103:\n\t"
> +     /* From 32 to 63.  No branch when size == 32.  */
> +     "vmovdqu (%%rsi), %%ymm0\n\t"
> +     "vmovdqu -32(%%rsi,%%rdx), %%ymm1\n\t"
> +     "vmovdqu %%ymm0, (%%rdi)\n\t"
> +     "vmovdqu %%ymm1, -32(%%rdi,%%rdx)\n\t"
> +     "vzeroupper\n\t"
> +     "jmp %l[done]\n\t"
> +     /* From 16 to 31.  No branch when size == 16.  */
> +     "104:\n\t"
> +     "vmovdqu (%%rsi), %%xmm0\n\t"
> +     "vmovdqu -16(%%rsi,%%rdx), %%xmm1\n\t"
> +     "vmovdqu %%xmm0, (%%rdi)\n\t"
> +     "vmovdqu %%xmm1, -16(%%rdi,%%rdx)\n\t"
> +     "jmp %l[done]\n\t"
> +     "105:\n\t"
> +     /* From 8 to 15.  No branch when size == 8.  */
> +     "movq    -8(%%rsi,%%rdx), %%rcx\n\t"
> +     "movq    (%%rsi), %%rsi\n\t"
> +     "movq    %%rcx, -8(%%rdi,%%rdx)\n\t"
> +     "movq    %%rsi, (%%rdi)\n\t"
> +     "jmp %l[done]\n\t"
> +     "106:\n\t"
> +     /* From 4 to 7.  No branch when size == 4.  */
> +     "movl    -4(%%rsi,%%rdx), %%ecx\n\t"
> +     "movl    (%%rsi), %%esi\n\t"
> +     "movl    %%ecx, -4(%%rdi,%%rdx)\n\t"
> +     "movl    %%esi, (%%rdi)\n\t"
> +     "jmp %l[done]\n\t"
> +     "107:\n\t"
> +     /* From 2 to 3.  No branch when size == 2.  */
> +     "movzwl  -2(%%rsi,%%rdx), %%ecx\n\t"
> +     "movzwl  (%%rsi), %%esi\n\t"
> +     "movw    %%cx, -2(%%rdi,%%rdx)\n\t"
> +     "movw    %%si, (%%rdi)\n\t"
> +     "jmp %l[done]\n\t"
> +     "108:\n\t"
> +     /* More than 2 * VEC and there may be overlap between destination */
> +     /* and source.  */
> +     "cmpq    $(32 * 8), %%rdx\n\t"
> +     "ja      111f\n\t"
> +     "cmpq    $(32 * 4), %%rdx\n\t"
> +     "jb      109f\n\t"
> +     /* Copy from 4 * VEC to 8 * VEC, inclusively. */
> +     "vmovdqu   (%%rsi), %%ymm0\n\t"
> +     "vmovdqu   32(%%rsi), %%ymm1\n\t"
> +     "vmovdqu   (32 * 2)(%%rsi), %%ymm2\n\t"
> +     "vmovdqu   (32 * 3)(%%rsi), %%ymm3\n\t"
> +     "vmovdqu   -32(%%rsi,%%rdx), %%ymm4\n\t"
> +     "vmovdqu   -(32 * 2)(%%rsi,%%rdx), %%ymm5\n\t"
> +     "vmovdqu   -(32 * 3)(%%rsi,%%rdx), %%ymm6\n\t"
> +     "vmovdqu   -(32 * 4)(%%rsi,%%rdx), %%ymm7\n\t"
> +     "vmovdqu   %%ymm0, (%%rdi)\n\t"
> +     "vmovdqu   %%ymm1, 32(%%rdi)\n\t"
> +     "vmovdqu   %%ymm2, (32 * 2)(%%rdi)\n\t"
> +     "vmovdqu   %%ymm3, (32 * 3)(%%rdi)\n\t"
> +     "vmovdqu   %%ymm4, -32(%%rdi,%%rdx)\n\t"
> +     "vmovdqu   %%ymm5, -(32 * 2)(%%rdi,%%rdx)\n\t"
> +     "vmovdqu   %%ymm6, -(32 * 3)(%%rdi,%%rdx)\n\t"
> +     "vmovdqu   %%ymm7, -(32 * 4)(%%rdi,%%rdx)\n\t"
> +     "vzeroupper\n\t"
> +     "jmp %l[done]\n\t"
> +     "109:\n\t"
> +     /* Copy from 2 * VEC to 4 * VEC. */
> +     "vmovdqu   (%%rsi), %%ymm0\n\t"
> +     "vmovdqu   32(%%rsi), %%ymm1\n\t"
> +     "vmovdqu   -32(%%rsi,%%rdx), %%ymm2\n\t"
> +     "vmovdqu   -(32 * 2)(%%rsi,%%rdx), %%ymm3\n\t"
> +     "vmovdqu   %%ymm0, (%%rdi)\n\t"
> +     "vmovdqu   %%ymm1, 32(%%rdi)\n\t"
> +     "vmovdqu   %%ymm2, -32(%%rdi,%%rdx)\n\t"
> +     "vmovdqu   %%ymm3, -(32 * 2)(%%rdi,%%rdx)\n\t"
> +     "vzeroupper\n\t"
> +     "110:\n\t"
> +     "jmp %l[done]\n\t"
> +     "111:\n\t"
> +     "cmpq    %%rsi, %%rdi\n\t"
> +     "ja      113f\n\t"
> +     /* Source == destination is less common.  */
> +     "je      110b\n\t"
> +     /* Load the first VEC and last 4 * VEC to
> +      * support overlapping addresses.
> +      */
> +     "vmovdqu   (%%rsi), %%ymm4\n\t"
> +     "vmovdqu   -32(%%rsi, %%rdx), %%ymm5\n\t"
> +     "vmovdqu   -(32 * 2)(%%rsi, %%rdx), %%ymm6\n\t"
> +     "vmovdqu   -(32 * 3)(%%rsi, %%rdx), %%ymm7\n\t"
> +     "vmovdqu   -(32 * 4)(%%rsi, %%rdx), %%ymm8\n\t"
> +     /* Save start and stop of the destination buffer.  */
> +     "movq    %%rdi, %%r11\n\t"
> +     "leaq    -32(%%rdi, %%rdx), %%rcx\n\t"
> +     /* Align destination for aligned stores in the loop.  Compute */
> +     /* how much destination is misaligned.  */
> +     "movq    %%rdi, %%r8\n\t"
> +     "andq    $(32 - 1), %%r8\n\t"
> +     /* Get the negative of offset for alignment.  */
> +     "subq    $32, %%r8\n\t"
> +     /* Adjust source.  */
> +     "subq    %%r8, %%rsi\n\t"
> +     /* Adjust destination which should be aligned now.  */
> +     "subq    %%r8, %%rdi\n\t"
> +     /* Adjust length.  */
> +     "addq    %%r8, %%rdx\n\t"
> +     /* Check non-temporal store threshold.  */
> +     "cmpq    $(1024*1024), %%rdx\n\t"
> +     "ja      115f\n\t"
> +     "112:\n\t"
> +     /* Copy 4 * VEC a time forward.  */
> +     "vmovdqu   (%%rsi), %%ymm0\n\t"
> +     "vmovdqu   32(%%rsi), %%ymm1\n\t"
> +     "vmovdqu   (32 * 2)(%%rsi), %%ymm2\n\t"
> +     "vmovdqu   (32 * 3)(%%rsi), %%ymm3\n\t"
> +     "addq    $(32 * 4), %%rsi\n\t"
> +     "subq    $(32 * 4), %%rdx\n\t"
> +     "vmovdqa   %%ymm0, (%%rdi)\n\t"
> +     "vmovdqa   %%ymm1, 32(%%rdi)\n\t"
> +     "vmovdqa   %%ymm2, (32 * 2)(%%rdi)\n\t"
> +     "vmovdqa   %%ymm3, (32 * 3)(%%rdi)\n\t"
> +     "addq    $(32 * 4), %%rdi\n\t"
> +     "cmpq    $(32 * 4), %%rdx\n\t"
> +     "ja      112b\n\t"
> +     /* Store the last 4 * VEC.  */
> +     "vmovdqu   %%ymm5, (%%rcx)\n\t"
> +     "vmovdqu   %%ymm6, -32(%%rcx)\n\t"
> +     "vmovdqu   %%ymm7, -(32 * 2)(%%rcx)\n\t"
> +     "vmovdqu   %%ymm8, -(32 * 3)(%%rcx)\n\t"
> +     /* Store the first VEC.  */
> +     "vmovdqu   %%ymm4, (%%r11)\n\t"
> +     "vzeroupper\n\t"
> +     "jmp %l[done]\n\t"
> +     "113:\n\t"
> +     /* Load the first 4*VEC and last VEC to support overlapping addresses.*/
> +     "vmovdqu   (%%rsi), %%ymm4\n\t"
> +     "vmovdqu   32(%%rsi), %%ymm5\n\t"
> +     "vmovdqu   (32 * 2)(%%rsi), %%ymm6\n\t"
> +     "vmovdqu   (32 * 3)(%%rsi), %%ymm7\n\t"
> +     "vmovdqu   -32(%%rsi,%%rdx), %%ymm8\n\t"
> +     /* Save stop of the destination buffer.  */
> +     "leaq    -32(%%rdi, %%rdx), %%r11\n\t"
> +     /* Align destination end for aligned stores in the loop.  Compute */
> +     /* how much destination end is misaligned.  */
> +     "leaq    -32(%%rsi, %%rdx), %%rcx\n\t"
> +     "movq    %%r11, %%r9\n\t"
> +     "movq    %%r11, %%r8\n\t"
> +     "andq    $(32 - 1), %%r8\n\t"
> +     /* Adjust source.  */
> +     "subq    %%r8, %%rcx\n\t"
> +     /* Adjust the end of destination which should be aligned now.  */
> +     "subq    %%r8, %%r9\n\t"
> +     /* Adjust length.  */
> +     "subq    %%r8, %%rdx\n\t"
> +      /* Check non-temporal store threshold.  */
> +     "cmpq    $(1024*1024), %%rdx\n\t"
> +     "ja      117f\n\t"
> +     "114:\n\t"
> +     /* Copy 4 * VEC a time backward.  */
> +     "vmovdqu   (%%rcx), %%ymm0\n\t"
> +     "vmovdqu   -32(%%rcx), %%ymm1\n\t"
> +     "vmovdqu   -(32 * 2)(%%rcx), %%ymm2\n\t"
> +     "vmovdqu   -(32 * 3)(%%rcx), %%ymm3\n\t"
> +     "subq    $(32 * 4), %%rcx\n\t"
> +     "subq    $(32 * 4), %%rdx\n\t"
> +     "vmovdqa   %%ymm0, (%%r9)\n\t"
> +     "vmovdqa   %%ymm1, -32(%%r9)\n\t"
> +     "vmovdqa   %%ymm2, -(32 * 2)(%%r9)\n\t"
> +     "vmovdqa   %%ymm3, -(32 * 3)(%%r9)\n\t"
> +     "subq    $(32 * 4), %%r9\n\t"
> +     "cmpq    $(32 * 4), %%rdx\n\t"
> +     "ja      114b\n\t"
> +     /* Store the first 4 * VEC. */
> +     "vmovdqu   %%ymm4, (%%rdi)\n\t"
> +     "vmovdqu   %%ymm5, 32(%%rdi)\n\t"
> +     "vmovdqu   %%ymm6, (32 * 2)(%%rdi)\n\t"
> +     "vmovdqu   %%ymm7, (32 * 3)(%%rdi)\n\t"
> +     /* Store the last VEC. */
> +     "vmovdqu   %%ymm8, (%%r11)\n\t"
> +     "vzeroupper\n\t"
> +     "jmp %l[done]\n\t"
> +
> +     "115:\n\t"
> +     /* Don't use non-temporal store if there is overlap between */
> +     /* destination and source since destination may be in cache */
> +     /* when source is loaded. */
> +     "leaq    (%%rdi, %%rdx), %%r10\n\t"
> +     "cmpq    %%r10, %%rsi\n\t"
> +     "jb      112b\n\t"
> +     "116:\n\t"
> +     /* Copy 4 * VEC a time forward with non-temporal stores.  */
> +     "prefetcht0 (32*4*2)(%%rsi)\n\t"
> +     "prefetcht0 (32*4*2 + 64)(%%rsi)\n\t"
> +     "prefetcht0 (32*4*3)(%%rsi)\n\t"
> +     "prefetcht0 (32*4*3 + 64)(%%rsi)\n\t"
> +     "vmovdqu   (%%rsi), %%ymm0\n\t"
> +     "vmovdqu   32(%%rsi), %%ymm1\n\t"
> +     "vmovdqu   (32 * 2)(%%rsi), %%ymm2\n\t"
> +     "vmovdqu   (32 * 3)(%%rsi), %%ymm3\n\t"
> +     "addq    $(32*4), %%rsi\n\t"
> +     "subq    $(32*4), %%rdx\n\t"
> +     "vmovntdq  %%ymm0, (%%rdi)\n\t"
> +     "vmovntdq  %%ymm1, 32(%%rdi)\n\t"
> +     "vmovntdq  %%ymm2, (32 * 2)(%%rdi)\n\t"
> +     "vmovntdq  %%ymm3, (32 * 3)(%%rdi)\n\t"
> +     "addq    $(32*4), %%rdi\n\t"
> +     "cmpq    $(32*4), %%rdx\n\t"
> +     "ja      116b\n\t"
> +     "sfence\n\t"
> +     /* Store the last 4 * VEC.  */
> +     "vmovdqu   %%ymm5, (%%rcx)\n\t"
> +     "vmovdqu   %%ymm6, -32(%%rcx)\n\t"
> +     "vmovdqu   %%ymm7, -(32 * 2)(%%rcx)\n\t"
> +     "vmovdqu   %%ymm8, -(32 * 3)(%%rcx)\n\t"
> +     /* Store the first VEC.  */
> +     "vmovdqu   %%ymm4, (%%r11)\n\t"
> +     "vzeroupper\n\t"
> +     "jmp %l[done]\n\t"
> +     "117:\n\t"
> +     /* Don't use non-temporal store if there is overlap between */
> +     /* destination and source since destination may be in cache */
> +     /* when source is loaded.  */
> +     "leaq    (%%rcx, %%rdx), %%r10\n\t"
> +     "cmpq    %%r10, %%r9\n\t"
> +     "jb      114b\n\t"
> +     "118:\n\t"
> +     /* Copy 4 * VEC a time backward with non-temporal stores. */
> +     "prefetcht0 (-32 * 4 * 2)(%%rcx)\n\t"
> +     "prefetcht0 (-32 * 4 * 2 - 64)(%%rcx)\n\t"
> +     "prefetcht0 (-32 * 4 * 3)(%%rcx)\n\t"
> +     "prefetcht0 (-32 * 4 * 3 - 64)(%%rcx)\n\t"
> +     "vmovdqu   (%%rcx), %%ymm0\n\t"
> +     "vmovdqu   -32(%%rcx), %%ymm1\n\t"
> +     "vmovdqu   -(32 * 2)(%%rcx), %%ymm2\n\t"
> +     "vmovdqu   -(32 * 3)(%%rcx), %%ymm3\n\t"
> +     "subq    $(32*4), %%rcx\n\t"
> +     "subq    $(32*4), %%rdx\n\t"
> +     "vmovntdq  %%ymm0, (%%r9)\n\t"
> +     "vmovntdq  %%ymm1, -32(%%r9)\n\t"
> +     "vmovntdq  %%ymm2, -(32 * 2)(%%r9)\n\t"
> +     "vmovntdq  %%ymm3, -(32 * 3)(%%r9)\n\t"
> +     "subq    $(32 * 4), %%r9\n\t"
> +     "cmpq    $(32 * 4), %%rdx\n\t"
> +     "ja      118b\n\t"
> +     "sfence\n\t"
> +     /* Store the first 4 * VEC.  */
> +     "vmovdqu   %%ymm4, (%%rdi)\n\t"
> +     "vmovdqu   %%ymm5, 32(%%rdi)\n\t"
> +     "vmovdqu   %%ymm6, (32 * 2)(%%rdi)\n\t"
> +     "vmovdqu   %%ymm7, (32 * 3)(%%rdi)\n\t"
> +     /* Store the last VEC.  */
> +     "vmovdqu   %%ymm8, (%%r11)\n\t"
> +     "vzeroupper\n\t"
> +     "jmp %l[done]"
> +     :
> +     : "r"(src), "r"(dst), "r"(len)
> +     : "rax", "rcx", "rdx", "rdi", "rsi", "r8", "r9", "r10", "r11", "r12", "ymm0",
> +     "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", "ymm8", "memory"
> +     : done
> +     );
> +done:
> +     return dst;
> +}

Re: [dpdk-dev] [PATCH v2 1/2] lib/eal: add amd epyc2 memcpy routine to eal

[Ananyev, Konstantin]

> 
> Hi Thomas,
> 
> I hope this can make some explanation to your question.
> We(AMD Linux library support team) have implemented the custom tailored memcpy solution which is a close match with DPDK use case
> requirements like the below.
> 1)      Min 64B length data packet with cache aligned Source and Destination.
> 2)      Non-Temporal load and temporal store for cache aligned source for both RX and TX paths. Could not implement the non-temporal
> store for TX_PATH, as non-Temporal load/stores works only with 32B aligned addresses for AVX2
> 3)      This solution works for all AVX2 supported AMD machines.
> 
> Internally we have completed the integrity testing and benchmarking of the solution and found gains of 8.4% to 14.5% specifically on Milan
> CPU(3rd Gen of EPYC Processor)

It still not clear to me why it has to be written in assembler.
Why similar stuff can't be written in C with instincts, as rest of rte_memcpy.h does?

> 
> Thanks for your support,
> Keesang
> 
> -----Original Message-----
> From: Thomas Monjalon <thomas@monjalon.net>
> Sent: Tuesday, October 19, 2021 5:31 AM
> To: Aman Kumar <aman.kumar@vvdntech.in>
> Cc: dev@dpdk.org; rasland@nvidia.com; asafp@nvidia.com; shys@nvidia.com; viacheslavo@nvidia.com; akozyrev@nvidia.com;
> matan@nvidia.com; anatoly.burakov@intel.com; Song, Keesang <Keesang.Song@amd.com>; aman.kumar@vvdntech.in;
> jerinjacobk@gmail.com; bruce.richardson@intel.com; konstantin.ananyev@intel.com; david.marchand@redhat.com
> Subject: Re: [dpdk-dev] [PATCH v2 1/2] lib/eal: add amd epyc2 memcpy routine to eal
> 
> [CAUTION: External Email]
> 
> 19/10/2021 12:47, Aman Kumar:
> > This patch provides rte_memcpy* calls optimized for AMD EPYC
> > platforms. Use config/x86/x86_amd_epyc_linux_gcc as cross-file with
> > meson to build dpdk for AMD EPYC platforms.
> 
> Please split in 2 patches: platform & memcpy.
> 
> What optimization is specific to EPYC?
> 
> I dislike the asm code below.
> What is AMD specific inside?
> Can it use compiler intrinsics as it is done elsewhere?
> 
> > +static __rte_always_inline void *
> > +rte_memcpy_aligned_ntload_tstore16_amdepyc2(void *dst,
> > +                                         const void *src,
> > +                                         size_t size) {
> > +     asm volatile goto("movq %0, %%rsi\n\t"
> > +     "movq %1, %%rdi\n\t"
> > +     "movq %2, %%rdx\n\t"
> > +     "cmpq   $(128), %%rdx\n\t"
> > +     "jb     202f\n\t"
> > +     "201:\n\t"
> > +     "vmovntdqa (%%rsi), %%ymm0\n\t"
> > +     "vmovntdqa 32(%%rsi), %%ymm1\n\t"
> > +     "vmovntdqa 64(%%rsi), %%ymm2\n\t"
> > +     "vmovntdqa 96(%%rsi), %%ymm3\n\t"
> > +     "vmovdqu  %%ymm0, (%%rdi)\n\t"
> > +     "vmovdqu  %%ymm1, 32(%%rdi)\n\t"
> > +     "vmovdqu  %%ymm2, 64(%%rdi)\n\t"
> > +     "vmovdqu  %%ymm3, 96(%%rdi)\n\t"
> > +     "addq   $128, %%rsi\n\t"
> > +     "addq   $128, %%rdi\n\t"
> > +     "subq   $128, %%rdx\n\t"
> > +     "jz     %l[done]\n\t"
> > +     "cmpq   $128, %%rdx\n\t" /*Vector Size 32B.  */
> > +     "jae    201b\n\t"
> > +     "202:\n\t"
> > +     "cmpq   $64, %%rdx\n\t"
> > +     "jb     203f\n\t"
> > +     "vmovntdqa (%%rsi), %%ymm0\n\t"
> > +     "vmovntdqa 32(%%rsi), %%ymm1\n\t"
> > +     "vmovdqu  %%ymm0, (%%rdi)\n\t"
> > +     "vmovdqu  %%ymm1, 32(%%rdi)\n\t"
> > +     "addq   $64, %%rsi\n\t"
> > +     "addq   $64, %%rdi\n\t"
> > +     "subq   $64, %%rdx\n\t"
> > +     "jz     %l[done]\n\t"
> > +     "203:\n\t"
> > +     "cmpq   $32, %%rdx\n\t"
> > +     "jb     204f\n\t"
> > +     "vmovntdqa (%%rsi), %%ymm0\n\t"
> > +     "vmovdqu  %%ymm0, (%%rdi)\n\t"
> > +     "addq   $32, %%rsi\n\t"
> > +     "addq   $32, %%rdi\n\t"
> > +     "subq   $32, %%rdx\n\t"
> > +     "jz     %l[done]\n\t"
> > +     "204:\n\t"
> > +     "cmpb   $16, %%dl\n\t"
> > +     "jb     205f\n\t"
> > +     "vmovntdqa (%%rsi), %%xmm0\n\t"
> > +     "vmovdqu  %%xmm0, (%%rdi)\n\t"
> > +     "addq   $16, %%rsi\n\t"
> > +     "addq   $16, %%rdi\n\t"
> > +     "subq   $16, %%rdx\n\t"
> > +     "jz     %l[done]\n\t"
> > +     "205:\n\t"
> > +     "cmpb   $2, %%dl\n\t"
> > +     "jb     208f\n\t"
> > +     "cmpb   $4, %%dl\n\t"
> > +     "jbe    207f\n\t"
> > +     "cmpb   $8, %%dl\n\t"
> > +     "jbe    206f\n\t"
> > +     "movq   -8(%%rsi,%%rdx), %%rcx\n\t"
> > +     "movq   (%%rsi), %%rsi\n\t"
> > +     "movq   %%rcx, -8(%%rdi,%%rdx)\n\t"
> > +     "movq   %%rsi, (%%rdi)\n\t"
> > +     "jmp    %l[done]\n\t"
> > +     "206:\n\t"
> > +     "movl   -4(%%rsi,%%rdx), %%ecx\n\t"
> > +     "movl   (%%rsi), %%esi\n\t"
> > +     "movl   %%ecx, -4(%%rdi,%%rdx)\n\t"
> > +     "movl   %%esi, (%%rdi)\n\t"
> > +     "jmp    %l[done]\n\t"
> > +     "207:\n\t"
> > +     "movzwl -2(%%rsi,%%rdx), %%ecx\n\t"
> > +     "movzwl (%%rsi), %%esi\n\t"
> > +     "movw   %%cx, -2(%%rdi,%%rdx)\n\t"
> > +     "movw   %%si, (%%rdi)\n\t"
> > +     "jmp    %l[done]\n\t"
> > +     "208:\n\t"
> > +     "movzbl (%%rsi), %%ecx\n\t"
> > +     "movb   %%cl, (%%rdi)"
> > +     :
> > +     : "r"(src), "r"(dst), "r"(size)
> > +     : "rcx", "rdx", "rsi", "rdi", "ymm0", "ymm1", "ymm2", "ymm3", "memory"
> > +     : done
> > +     );
> > +done:
> > +     return dst;
> > +}
> > +
> > +static __rte_always_inline void *
> > +rte_memcpy_generic(void *dst, const void *src, size_t len) {
> > +     asm goto("movq  %0, %%rsi\n\t"
> > +     "movq   %1, %%rdi\n\t"
> > +     "movq   %2, %%rdx\n\t"
> > +     "movq    %%rdi, %%rax\n\t"
> > +     "cmp     $32, %%rdx\n\t"
> > +     "jb      101f\n\t"
> > +     "cmp     $(32 * 2), %%rdx\n\t"
> > +     "ja      108f\n\t"
> > +     "vmovdqu   (%%rsi), %%ymm0\n\t"
> > +     "vmovdqu   -32(%%rsi,%%rdx), %%ymm1\n\t"
> > +     "vmovdqu   %%ymm0, (%%rdi)\n\t"
> > +     "vmovdqu   %%ymm1, -32(%%rdi,%%rdx)\n\t"
> > +     "vzeroupper\n\t"
> > +     "jmp %l[done]\n\t"
> > +     "101:\n\t"
> > +     /* Less than 1 VEC.  */
> > +     "cmpb    $32, %%dl\n\t"
> > +     "jae     103f\n\t"
> > +     "cmpb    $16, %%dl\n\t"
> > +     "jae     104f\n\t"
> > +     "cmpb    $8, %%dl\n\t"
> > +     "jae     105f\n\t"
> > +     "cmpb    $4, %%dl\n\t"
> > +     "jae     106f\n\t"
> > +     "cmpb    $1, %%dl\n\t"
> > +     "ja      107f\n\t"
> > +     "jb      102f\n\t"
> > +     "movzbl  (%%rsi), %%ecx\n\t"
> > +     "movb    %%cl, (%%rdi)\n\t"
> > +     "102:\n\t"
> > +     "jmp %l[done]\n\t"
> > +     "103:\n\t"
> > +     /* From 32 to 63.  No branch when size == 32.  */
> > +     "vmovdqu (%%rsi), %%ymm0\n\t"
> > +     "vmovdqu -32(%%rsi,%%rdx), %%ymm1\n\t"
> > +     "vmovdqu %%ymm0, (%%rdi)\n\t"
> > +     "vmovdqu %%ymm1, -32(%%rdi,%%rdx)\n\t"
> > +     "vzeroupper\n\t"
> > +     "jmp %l[done]\n\t"
> > +     /* From 16 to 31.  No branch when size == 16.  */
> > +     "104:\n\t"
> > +     "vmovdqu (%%rsi), %%xmm0\n\t"
> > +     "vmovdqu -16(%%rsi,%%rdx), %%xmm1\n\t"
> > +     "vmovdqu %%xmm0, (%%rdi)\n\t"
> > +     "vmovdqu %%xmm1, -16(%%rdi,%%rdx)\n\t"
> > +     "jmp %l[done]\n\t"
> > +     "105:\n\t"
> > +     /* From 8 to 15.  No branch when size == 8.  */
> > +     "movq    -8(%%rsi,%%rdx), %%rcx\n\t"
> > +     "movq    (%%rsi), %%rsi\n\t"
> > +     "movq    %%rcx, -8(%%rdi,%%rdx)\n\t"
> > +     "movq    %%rsi, (%%rdi)\n\t"
> > +     "jmp %l[done]\n\t"
> > +     "106:\n\t"
> > +     /* From 4 to 7.  No branch when size == 4.  */
> > +     "movl    -4(%%rsi,%%rdx), %%ecx\n\t"
> > +     "movl    (%%rsi), %%esi\n\t"
> > +     "movl    %%ecx, -4(%%rdi,%%rdx)\n\t"
> > +     "movl    %%esi, (%%rdi)\n\t"
> > +     "jmp %l[done]\n\t"
> > +     "107:\n\t"
> > +     /* From 2 to 3.  No branch when size == 2.  */
> > +     "movzwl  -2(%%rsi,%%rdx), %%ecx\n\t"
> > +     "movzwl  (%%rsi), %%esi\n\t"
> > +     "movw    %%cx, -2(%%rdi,%%rdx)\n\t"
> > +     "movw    %%si, (%%rdi)\n\t"
> > +     "jmp %l[done]\n\t"
> > +     "108:\n\t"
> > +     /* More than 2 * VEC and there may be overlap between destination */
> > +     /* and source.  */
> > +     "cmpq    $(32 * 8), %%rdx\n\t"
> > +     "ja      111f\n\t"
> > +     "cmpq    $(32 * 4), %%rdx\n\t"
> > +     "jb      109f\n\t"
> > +     /* Copy from 4 * VEC to 8 * VEC, inclusively. */
> > +     "vmovdqu   (%%rsi), %%ymm0\n\t"
> > +     "vmovdqu   32(%%rsi), %%ymm1\n\t"
> > +     "vmovdqu   (32 * 2)(%%rsi), %%ymm2\n\t"
> > +     "vmovdqu   (32 * 3)(%%rsi), %%ymm3\n\t"
> > +     "vmovdqu   -32(%%rsi,%%rdx), %%ymm4\n\t"
> > +     "vmovdqu   -(32 * 2)(%%rsi,%%rdx), %%ymm5\n\t"
> > +     "vmovdqu   -(32 * 3)(%%rsi,%%rdx), %%ymm6\n\t"
> > +     "vmovdqu   -(32 * 4)(%%rsi,%%rdx), %%ymm7\n\t"
> > +     "vmovdqu   %%ymm0, (%%rdi)\n\t"
> > +     "vmovdqu   %%ymm1, 32(%%rdi)\n\t"
> > +     "vmovdqu   %%ymm2, (32 * 2)(%%rdi)\n\t"
> > +     "vmovdqu   %%ymm3, (32 * 3)(%%rdi)\n\t"
> > +     "vmovdqu   %%ymm4, -32(%%rdi,%%rdx)\n\t"
> > +     "vmovdqu   %%ymm5, -(32 * 2)(%%rdi,%%rdx)\n\t"
> > +     "vmovdqu   %%ymm6, -(32 * 3)(%%rdi,%%rdx)\n\t"
> > +     "vmovdqu   %%ymm7, -(32 * 4)(%%rdi,%%rdx)\n\t"
> > +     "vzeroupper\n\t"
> > +     "jmp %l[done]\n\t"
> > +     "109:\n\t"
> > +     /* Copy from 2 * VEC to 4 * VEC. */
> > +     "vmovdqu   (%%rsi), %%ymm0\n\t"
> > +     "vmovdqu   32(%%rsi), %%ymm1\n\t"
> > +     "vmovdqu   -32(%%rsi,%%rdx), %%ymm2\n\t"
> > +     "vmovdqu   -(32 * 2)(%%rsi,%%rdx), %%ymm3\n\t"
> > +     "vmovdqu   %%ymm0, (%%rdi)\n\t"
> > +     "vmovdqu   %%ymm1, 32(%%rdi)\n\t"
> > +     "vmovdqu   %%ymm2, -32(%%rdi,%%rdx)\n\t"
> > +     "vmovdqu   %%ymm3, -(32 * 2)(%%rdi,%%rdx)\n\t"
> > +     "vzeroupper\n\t"
> > +     "110:\n\t"
> > +     "jmp %l[done]\n\t"
> > +     "111:\n\t"
> > +     "cmpq    %%rsi, %%rdi\n\t"
> > +     "ja      113f\n\t"
> > +     /* Source == destination is less common.  */
> > +     "je      110b\n\t"
> > +     /* Load the first VEC and last 4 * VEC to
> > +      * support overlapping addresses.
> > +      */
> > +     "vmovdqu   (%%rsi), %%ymm4\n\t"
> > +     "vmovdqu   -32(%%rsi, %%rdx), %%ymm5\n\t"
> > +     "vmovdqu   -(32 * 2)(%%rsi, %%rdx), %%ymm6\n\t"
> > +     "vmovdqu   -(32 * 3)(%%rsi, %%rdx), %%ymm7\n\t"
> > +     "vmovdqu   -(32 * 4)(%%rsi, %%rdx), %%ymm8\n\t"
> > +     /* Save start and stop of the destination buffer.  */
> > +     "movq    %%rdi, %%r11\n\t"
> > +     "leaq    -32(%%rdi, %%rdx), %%rcx\n\t"
> > +     /* Align destination for aligned stores in the loop.  Compute */
> > +     /* how much destination is misaligned.  */
> > +     "movq    %%rdi, %%r8\n\t"
> > +     "andq    $(32 - 1), %%r8\n\t"
> > +     /* Get the negative of offset for alignment.  */
> > +     "subq    $32, %%r8\n\t"
> > +     /* Adjust source.  */
> > +     "subq    %%r8, %%rsi\n\t"
> > +     /* Adjust destination which should be aligned now.  */
> > +     "subq    %%r8, %%rdi\n\t"
> > +     /* Adjust length.  */
> > +     "addq    %%r8, %%rdx\n\t"
> > +     /* Check non-temporal store threshold.  */
> > +     "cmpq    $(1024*1024), %%rdx\n\t"
> > +     "ja      115f\n\t"
> > +     "112:\n\t"
> > +     /* Copy 4 * VEC a time forward.  */
> > +     "vmovdqu   (%%rsi), %%ymm0\n\t"
> > +     "vmovdqu   32(%%rsi), %%ymm1\n\t"
> > +     "vmovdqu   (32 * 2)(%%rsi), %%ymm2\n\t"
> > +     "vmovdqu   (32 * 3)(%%rsi), %%ymm3\n\t"
> > +     "addq    $(32 * 4), %%rsi\n\t"
> > +     "subq    $(32 * 4), %%rdx\n\t"
> > +     "vmovdqa   %%ymm0, (%%rdi)\n\t"
> > +     "vmovdqa   %%ymm1, 32(%%rdi)\n\t"
> > +     "vmovdqa   %%ymm2, (32 * 2)(%%rdi)\n\t"
> > +     "vmovdqa   %%ymm3, (32 * 3)(%%rdi)\n\t"
> > +     "addq    $(32 * 4), %%rdi\n\t"
> > +     "cmpq    $(32 * 4), %%rdx\n\t"
> > +     "ja      112b\n\t"
> > +     /* Store the last 4 * VEC.  */
> > +     "vmovdqu   %%ymm5, (%%rcx)\n\t"
> > +     "vmovdqu   %%ymm6, -32(%%rcx)\n\t"
> > +     "vmovdqu   %%ymm7, -(32 * 2)(%%rcx)\n\t"
> > +     "vmovdqu   %%ymm8, -(32 * 3)(%%rcx)\n\t"
> > +     /* Store the first VEC.  */
> > +     "vmovdqu   %%ymm4, (%%r11)\n\t"
> > +     "vzeroupper\n\t"
> > +     "jmp %l[done]\n\t"
> > +     "113:\n\t"
> > +     /* Load the first 4*VEC and last VEC to support overlapping addresses.*/
> > +     "vmovdqu   (%%rsi), %%ymm4\n\t"
> > +     "vmovdqu   32(%%rsi), %%ymm5\n\t"
> > +     "vmovdqu   (32 * 2)(%%rsi), %%ymm6\n\t"
> > +     "vmovdqu   (32 * 3)(%%rsi), %%ymm7\n\t"
> > +     "vmovdqu   -32(%%rsi,%%rdx), %%ymm8\n\t"
> > +     /* Save stop of the destination buffer.  */
> > +     "leaq    -32(%%rdi, %%rdx), %%r11\n\t"
> > +     /* Align destination end for aligned stores in the loop.  Compute */
> > +     /* how much destination end is misaligned.  */
> > +     "leaq    -32(%%rsi, %%rdx), %%rcx\n\t"
> > +     "movq    %%r11, %%r9\n\t"
> > +     "movq    %%r11, %%r8\n\t"
> > +     "andq    $(32 - 1), %%r8\n\t"
> > +     /* Adjust source.  */
> > +     "subq    %%r8, %%rcx\n\t"
> > +     /* Adjust the end of destination which should be aligned now.  */
> > +     "subq    %%r8, %%r9\n\t"
> > +     /* Adjust length.  */
> > +     "subq    %%r8, %%rdx\n\t"
> > +      /* Check non-temporal store threshold.  */
> > +     "cmpq    $(1024*1024), %%rdx\n\t"
> > +     "ja      117f\n\t"
> > +     "114:\n\t"
> > +     /* Copy 4 * VEC a time backward.  */
> > +     "vmovdqu   (%%rcx), %%ymm0\n\t"
> > +     "vmovdqu   -32(%%rcx), %%ymm1\n\t"
> > +     "vmovdqu   -(32 * 2)(%%rcx), %%ymm2\n\t"
> > +     "vmovdqu   -(32 * 3)(%%rcx), %%ymm3\n\t"
> > +     "subq    $(32 * 4), %%rcx\n\t"
> > +     "subq    $(32 * 4), %%rdx\n\t"
> > +     "vmovdqa   %%ymm0, (%%r9)\n\t"
> > +     "vmovdqa   %%ymm1, -32(%%r9)\n\t"
> > +     "vmovdqa   %%ymm2, -(32 * 2)(%%r9)\n\t"
> > +     "vmovdqa   %%ymm3, -(32 * 3)(%%r9)\n\t"
> > +     "subq    $(32 * 4), %%r9\n\t"
> > +     "cmpq    $(32 * 4), %%rdx\n\t"
> > +     "ja      114b\n\t"
> > +     /* Store the first 4 * VEC. */
> > +     "vmovdqu   %%ymm4, (%%rdi)\n\t"
> > +     "vmovdqu   %%ymm5, 32(%%rdi)\n\t"
> > +     "vmovdqu   %%ymm6, (32 * 2)(%%rdi)\n\t"
> > +     "vmovdqu   %%ymm7, (32 * 3)(%%rdi)\n\t"
> > +     /* Store the last VEC. */
> > +     "vmovdqu   %%ymm8, (%%r11)\n\t"
> > +     "vzeroupper\n\t"
> > +     "jmp %l[done]\n\t"
> > +
> > +     "115:\n\t"
> > +     /* Don't use non-temporal store if there is overlap between */
> > +     /* destination and source since destination may be in cache */
> > +     /* when source is loaded. */
> > +     "leaq    (%%rdi, %%rdx), %%r10\n\t"
> > +     "cmpq    %%r10, %%rsi\n\t"
> > +     "jb      112b\n\t"
> > +     "116:\n\t"
> > +     /* Copy 4 * VEC a time forward with non-temporal stores.  */
> > +     "prefetcht0 (32*4*2)(%%rsi)\n\t"
> > +     "prefetcht0 (32*4*2 + 64)(%%rsi)\n\t"
> > +     "prefetcht0 (32*4*3)(%%rsi)\n\t"
> > +     "prefetcht0 (32*4*3 + 64)(%%rsi)\n\t"
> > +     "vmovdqu   (%%rsi), %%ymm0\n\t"
> > +     "vmovdqu   32(%%rsi), %%ymm1\n\t"
> > +     "vmovdqu   (32 * 2)(%%rsi), %%ymm2\n\t"
> > +     "vmovdqu   (32 * 3)(%%rsi), %%ymm3\n\t"
> > +     "addq    $(32*4), %%rsi\n\t"
> > +     "subq    $(32*4), %%rdx\n\t"
> > +     "vmovntdq  %%ymm0, (%%rdi)\n\t"
> > +     "vmovntdq  %%ymm1, 32(%%rdi)\n\t"
> > +     "vmovntdq  %%ymm2, (32 * 2)(%%rdi)\n\t"
> > +     "vmovntdq  %%ymm3, (32 * 3)(%%rdi)\n\t"
> > +     "addq    $(32*4), %%rdi\n\t"
> > +     "cmpq    $(32*4), %%rdx\n\t"
> > +     "ja      116b\n\t"
> > +     "sfence\n\t"
> > +     /* Store the last 4 * VEC.  */
> > +     "vmovdqu   %%ymm5, (%%rcx)\n\t"
> > +     "vmovdqu   %%ymm6, -32(%%rcx)\n\t"
> > +     "vmovdqu   %%ymm7, -(32 * 2)(%%rcx)\n\t"
> > +     "vmovdqu   %%ymm8, -(32 * 3)(%%rcx)\n\t"
> > +     /* Store the first VEC.  */
> > +     "vmovdqu   %%ymm4, (%%r11)\n\t"
> > +     "vzeroupper\n\t"
> > +     "jmp %l[done]\n\t"
> > +     "117:\n\t"
> > +     /* Don't use non-temporal store if there is overlap between */
> > +     /* destination and source since destination may be in cache */
> > +     /* when source is loaded.  */
> > +     "leaq    (%%rcx, %%rdx), %%r10\n\t"
> > +     "cmpq    %%r10, %%r9\n\t"
> > +     "jb      114b\n\t"
> > +     "118:\n\t"
> > +     /* Copy 4 * VEC a time backward with non-temporal stores. */
> > +     "prefetcht0 (-32 * 4 * 2)(%%rcx)\n\t"
> > +     "prefetcht0 (-32 * 4 * 2 - 64)(%%rcx)\n\t"
> > +     "prefetcht0 (-32 * 4 * 3)(%%rcx)\n\t"
> > +     "prefetcht0 (-32 * 4 * 3 - 64)(%%rcx)\n\t"
> > +     "vmovdqu   (%%rcx), %%ymm0\n\t"
> > +     "vmovdqu   -32(%%rcx), %%ymm1\n\t"
> > +     "vmovdqu   -(32 * 2)(%%rcx), %%ymm2\n\t"
> > +     "vmovdqu   -(32 * 3)(%%rcx), %%ymm3\n\t"
> > +     "subq    $(32*4), %%rcx\n\t"
> > +     "subq    $(32*4), %%rdx\n\t"
> > +     "vmovntdq  %%ymm0, (%%r9)\n\t"
> > +     "vmovntdq  %%ymm1, -32(%%r9)\n\t"
> > +     "vmovntdq  %%ymm2, -(32 * 2)(%%r9)\n\t"
> > +     "vmovntdq  %%ymm3, -(32 * 3)(%%r9)\n\t"
> > +     "subq    $(32 * 4), %%r9\n\t"
> > +     "cmpq    $(32 * 4), %%rdx\n\t"
> > +     "ja      118b\n\t"
> > +     "sfence\n\t"
> > +     /* Store the first 4 * VEC.  */
> > +     "vmovdqu   %%ymm4, (%%rdi)\n\t"
> > +     "vmovdqu   %%ymm5, 32(%%rdi)\n\t"
> > +     "vmovdqu   %%ymm6, (32 * 2)(%%rdi)\n\t"
> > +     "vmovdqu   %%ymm7, (32 * 3)(%%rdi)\n\t"
> > +     /* Store the last VEC.  */
> > +     "vmovdqu   %%ymm8, (%%r11)\n\t"
> > +     "vzeroupper\n\t"
> > +     "jmp %l[done]"
> > +     :
> > +     : "r"(src), "r"(dst), "r"(len)
> > +     : "rax", "rcx", "rdx", "rdi", "rsi", "r8", "r9", "r10", "r11", "r12", "ymm0",
> > +     "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", "ymm8", "memory"
> > +     : done
> > +     );
> > +done:
> > +     return dst;
> > +}
> 
> 

Re: [dpdk-dev] [PATCH v2 1/2] lib/eal: add amd epyc2 memcpy routine to eal

[Song, Keesang]

[AMD Official Use Only]

Hi Ananyev,

The current memcpy implementation in Glibc is based out of assembly coding.
Although memcpy could have been implemented with intrinsic, but since our AMD library developers are working on the Glibc functions, they have provided a tailored implementation based out of inline assembly coding.

Thanks for your support,
Keesang

-----Original Message-----
From: Ananyev, Konstantin <konstantin.ananyev@intel.com>
Sent: Thursday, October 21, 2021 10:40 AM
To: Song, Keesang <Keesang.Song@amd.com>; Thomas Monjalon <thomas@monjalon.net>; Aman Kumar <aman.kumar@vvdntech.in>
Cc: dev@dpdk.org; rasland@nvidia.com; asafp@nvidia.com; shys@nvidia.com; viacheslavo@nvidia.com; akozyrev@nvidia.com; matan@nvidia.com; Burakov, Anatoly <anatoly.burakov@intel.com>; aman.kumar@vvdntech.in; jerinjacobk@gmail.com; Richardson, Bruce <bruce.richardson@intel.com>; david.marchand@redhat.com
Subject: RE: [dpdk-dev] [PATCH v2 1/2] lib/eal: add amd epyc2 memcpy routine to eal

[AMD Official Use Only]

[CAUTION: External Email]

>
> Hi Thomas,
>
> I hope this can make some explanation to your question.
> We(AMD Linux library support team) have implemented the custom
> tailored memcpy solution which is a close match with DPDK use case requirements like the below.
> 1)      Min 64B length data packet with cache aligned Source and Destination.
> 2)      Non-Temporal load and temporal store for cache aligned source for both RX and TX paths. Could not implement the non-temporal
> store for TX_PATH, as non-Temporal load/stores works only with 32B aligned addresses for AVX2
> 3)      This solution works for all AVX2 supported AMD machines.
>
> Internally we have completed the integrity testing and benchmarking of
> the solution and found gains of 8.4% to 14.5% specifically on Milan
> CPU(3rd Gen of EPYC Processor)

It still not clear to me why it has to be written in assembler.
Why similar stuff can't be written in C with instincts, as rest of rte_memcpy.h does?

>
> Thanks for your support,
> Keesang
>
> -----Original Message-----
> From: Thomas Monjalon <thomas@monjalon.net>
> Sent: Tuesday, October 19, 2021 5:31 AM
> To: Aman Kumar <aman.kumar@vvdntech.in>
> Cc: dev@dpdk.org; rasland@nvidia.com; asafp@nvidia.com;
> shys@nvidia.com; viacheslavo@nvidia.com; akozyrev@nvidia.com;
> matan@nvidia.com; anatoly.burakov@intel.com; Song, Keesang
> <Keesang.Song@amd.com>; aman.kumar@vvdntech.in; jerinjacobk@gmail.com;
> bruce.richardson@intel.com; konstantin.ananyev@intel.com;
> david.marchand@redhat.com
> Subject: Re: [dpdk-dev] [PATCH v2 1/2] lib/eal: add amd epyc2 memcpy
> routine to eal
>
> [CAUTION: External Email]
>
> 19/10/2021 12:47, Aman Kumar:
> > This patch provides rte_memcpy* calls optimized for AMD EPYC
> > platforms. Use config/x86/x86_amd_epyc_linux_gcc as cross-file with
> > meson to build dpdk for AMD EPYC platforms.
>
> Please split in 2 patches: platform & memcpy.
>
> What optimization is specific to EPYC?
>
> I dislike the asm code below.
> What is AMD specific inside?
> Can it use compiler intrinsics as it is done elsewhere?
>
> > +static __rte_always_inline void *
> > +rte_memcpy_aligned_ntload_tstore16_amdepyc2(void *dst,
> > +                                         const void *src,
> > +                                         size_t size) {
> > +     asm volatile goto("movq %0, %%rsi\n\t"
> > +     "movq %1, %%rdi\n\t"
> > +     "movq %2, %%rdx\n\t"
> > +     "cmpq   $(128), %%rdx\n\t"
> > +     "jb     202f\n\t"
> > +     "201:\n\t"
> > +     "vmovntdqa (%%rsi), %%ymm0\n\t"
> > +     "vmovntdqa 32(%%rsi), %%ymm1\n\t"
> > +     "vmovntdqa 64(%%rsi), %%ymm2\n\t"
> > +     "vmovntdqa 96(%%rsi), %%ymm3\n\t"
> > +     "vmovdqu  %%ymm0, (%%rdi)\n\t"
> > +     "vmovdqu  %%ymm1, 32(%%rdi)\n\t"
> > +     "vmovdqu  %%ymm2, 64(%%rdi)\n\t"
> > +     "vmovdqu  %%ymm3, 96(%%rdi)\n\t"
> > +     "addq   $128, %%rsi\n\t"
> > +     "addq   $128, %%rdi\n\t"
> > +     "subq   $128, %%rdx\n\t"
> > +     "jz     %l[done]\n\t"
> > +     "cmpq   $128, %%rdx\n\t" /*Vector Size 32B.  */
> > +     "jae    201b\n\t"
> > +     "202:\n\t"
> > +     "cmpq   $64, %%rdx\n\t"
> > +     "jb     203f\n\t"
> > +     "vmovntdqa (%%rsi), %%ymm0\n\t"
> > +     "vmovntdqa 32(%%rsi), %%ymm1\n\t"
> > +     "vmovdqu  %%ymm0, (%%rdi)\n\t"
> > +     "vmovdqu  %%ymm1, 32(%%rdi)\n\t"
> > +     "addq   $64, %%rsi\n\t"
> > +     "addq   $64, %%rdi\n\t"
> > +     "subq   $64, %%rdx\n\t"
> > +     "jz     %l[done]\n\t"
> > +     "203:\n\t"
> > +     "cmpq   $32, %%rdx\n\t"
> > +     "jb     204f\n\t"
> > +     "vmovntdqa (%%rsi), %%ymm0\n\t"
> > +     "vmovdqu  %%ymm0, (%%rdi)\n\t"
> > +     "addq   $32, %%rsi\n\t"
> > +     "addq   $32, %%rdi\n\t"
> > +     "subq   $32, %%rdx\n\t"
> > +     "jz     %l[done]\n\t"
> > +     "204:\n\t"
> > +     "cmpb   $16, %%dl\n\t"
> > +     "jb     205f\n\t"
> > +     "vmovntdqa (%%rsi), %%xmm0\n\t"
> > +     "vmovdqu  %%xmm0, (%%rdi)\n\t"
> > +     "addq   $16, %%rsi\n\t"
> > +     "addq   $16, %%rdi\n\t"
> > +     "subq   $16, %%rdx\n\t"
> > +     "jz     %l[done]\n\t"
> > +     "205:\n\t"
> > +     "cmpb   $2, %%dl\n\t"
> > +     "jb     208f\n\t"
> > +     "cmpb   $4, %%dl\n\t"
> > +     "jbe    207f\n\t"
> > +     "cmpb   $8, %%dl\n\t"
> > +     "jbe    206f\n\t"
> > +     "movq   -8(%%rsi,%%rdx), %%rcx\n\t"
> > +     "movq   (%%rsi), %%rsi\n\t"
> > +     "movq   %%rcx, -8(%%rdi,%%rdx)\n\t"
> > +     "movq   %%rsi, (%%rdi)\n\t"
> > +     "jmp    %l[done]\n\t"
> > +     "206:\n\t"
> > +     "movl   -4(%%rsi,%%rdx), %%ecx\n\t"
> > +     "movl   (%%rsi), %%esi\n\t"
> > +     "movl   %%ecx, -4(%%rdi,%%rdx)\n\t"
> > +     "movl   %%esi, (%%rdi)\n\t"
> > +     "jmp    %l[done]\n\t"
> > +     "207:\n\t"
> > +     "movzwl -2(%%rsi,%%rdx), %%ecx\n\t"
> > +     "movzwl (%%rsi), %%esi\n\t"
> > +     "movw   %%cx, -2(%%rdi,%%rdx)\n\t"
> > +     "movw   %%si, (%%rdi)\n\t"
> > +     "jmp    %l[done]\n\t"
> > +     "208:\n\t"
> > +     "movzbl (%%rsi), %%ecx\n\t"
> > +     "movb   %%cl, (%%rdi)"
> > +     :
> > +     : "r"(src), "r"(dst), "r"(size)
> > +     : "rcx", "rdx", "rsi", "rdi", "ymm0", "ymm1", "ymm2", "ymm3", "memory"
> > +     : done
> > +     );
> > +done:
> > +     return dst;
> > +}
> > +
> > +static __rte_always_inline void *
> > +rte_memcpy_generic(void *dst, const void *src, size_t len) {
> > +     asm goto("movq  %0, %%rsi\n\t"
> > +     "movq   %1, %%rdi\n\t"
> > +     "movq   %2, %%rdx\n\t"
> > +     "movq    %%rdi, %%rax\n\t"
> > +     "cmp     $32, %%rdx\n\t"
> > +     "jb      101f\n\t"
> > +     "cmp     $(32 * 2), %%rdx\n\t"
> > +     "ja      108f\n\t"
> > +     "vmovdqu   (%%rsi), %%ymm0\n\t"
> > +     "vmovdqu   -32(%%rsi,%%rdx), %%ymm1\n\t"
> > +     "vmovdqu   %%ymm0, (%%rdi)\n\t"
> > +     "vmovdqu   %%ymm1, -32(%%rdi,%%rdx)\n\t"
> > +     "vzeroupper\n\t"
> > +     "jmp %l[done]\n\t"
> > +     "101:\n\t"
> > +     /* Less than 1 VEC.  */
> > +     "cmpb    $32, %%dl\n\t"
> > +     "jae     103f\n\t"
> > +     "cmpb    $16, %%dl\n\t"
> > +     "jae     104f\n\t"
> > +     "cmpb    $8, %%dl\n\t"
> > +     "jae     105f\n\t"
> > +     "cmpb    $4, %%dl\n\t"
> > +     "jae     106f\n\t"
> > +     "cmpb    $1, %%dl\n\t"
> > +     "ja      107f\n\t"
> > +     "jb      102f\n\t"
> > +     "movzbl  (%%rsi), %%ecx\n\t"
> > +     "movb    %%cl, (%%rdi)\n\t"
> > +     "102:\n\t"
> > +     "jmp %l[done]\n\t"
> > +     "103:\n\t"
> > +     /* From 32 to 63.  No branch when size == 32.  */
> > +     "vmovdqu (%%rsi), %%ymm0\n\t"
> > +     "vmovdqu -32(%%rsi,%%rdx), %%ymm1\n\t"
> > +     "vmovdqu %%ymm0, (%%rdi)\n\t"
> > +     "vmovdqu %%ymm1, -32(%%rdi,%%rdx)\n\t"
> > +     "vzeroupper\n\t"
> > +     "jmp %l[done]\n\t"
> > +     /* From 16 to 31.  No branch when size == 16.  */
> > +     "104:\n\t"
> > +     "vmovdqu (%%rsi), %%xmm0\n\t"
> > +     "vmovdqu -16(%%rsi,%%rdx), %%xmm1\n\t"
> > +     "vmovdqu %%xmm0, (%%rdi)\n\t"
> > +     "vmovdqu %%xmm1, -16(%%rdi,%%rdx)\n\t"
> > +     "jmp %l[done]\n\t"
> > +     "105:\n\t"
> > +     /* From 8 to 15.  No branch when size == 8.  */
> > +     "movq    -8(%%rsi,%%rdx), %%rcx\n\t"
> > +     "movq    (%%rsi), %%rsi\n\t"
> > +     "movq    %%rcx, -8(%%rdi,%%rdx)\n\t"
> > +     "movq    %%rsi, (%%rdi)\n\t"
> > +     "jmp %l[done]\n\t"
> > +     "106:\n\t"
> > +     /* From 4 to 7.  No branch when size == 4.  */
> > +     "movl    -4(%%rsi,%%rdx), %%ecx\n\t"
> > +     "movl    (%%rsi), %%esi\n\t"
> > +     "movl    %%ecx, -4(%%rdi,%%rdx)\n\t"
> > +     "movl    %%esi, (%%rdi)\n\t"
> > +     "jmp %l[done]\n\t"
> > +     "107:\n\t"
> > +     /* From 2 to 3.  No branch when size == 2.  */
> > +     "movzwl  -2(%%rsi,%%rdx), %%ecx\n\t"
> > +     "movzwl  (%%rsi), %%esi\n\t"
> > +     "movw    %%cx, -2(%%rdi,%%rdx)\n\t"
> > +     "movw    %%si, (%%rdi)\n\t"
> > +     "jmp %l[done]\n\t"
> > +     "108:\n\t"
> > +     /* More than 2 * VEC and there may be overlap between destination */
> > +     /* and source.  */
> > +     "cmpq    $(32 * 8), %%rdx\n\t"
> > +     "ja      111f\n\t"
> > +     "cmpq    $(32 * 4), %%rdx\n\t"
> > +     "jb      109f\n\t"
> > +     /* Copy from 4 * VEC to 8 * VEC, inclusively. */
> > +     "vmovdqu   (%%rsi), %%ymm0\n\t"
> > +     "vmovdqu   32(%%rsi), %%ymm1\n\t"
> > +     "vmovdqu   (32 * 2)(%%rsi), %%ymm2\n\t"
> > +     "vmovdqu   (32 * 3)(%%rsi), %%ymm3\n\t"
> > +     "vmovdqu   -32(%%rsi,%%rdx), %%ymm4\n\t"
> > +     "vmovdqu   -(32 * 2)(%%rsi,%%rdx), %%ymm5\n\t"
> > +     "vmovdqu   -(32 * 3)(%%rsi,%%rdx), %%ymm6\n\t"
> > +     "vmovdqu   -(32 * 4)(%%rsi,%%rdx), %%ymm7\n\t"
> > +     "vmovdqu   %%ymm0, (%%rdi)\n\t"
> > +     "vmovdqu   %%ymm1, 32(%%rdi)\n\t"
> > +     "vmovdqu   %%ymm2, (32 * 2)(%%rdi)\n\t"
> > +     "vmovdqu   %%ymm3, (32 * 3)(%%rdi)\n\t"
> > +     "vmovdqu   %%ymm4, -32(%%rdi,%%rdx)\n\t"
> > +     "vmovdqu   %%ymm5, -(32 * 2)(%%rdi,%%rdx)\n\t"
> > +     "vmovdqu   %%ymm6, -(32 * 3)(%%rdi,%%rdx)\n\t"
> > +     "vmovdqu   %%ymm7, -(32 * 4)(%%rdi,%%rdx)\n\t"
> > +     "vzeroupper\n\t"
> > +     "jmp %l[done]\n\t"
> > +     "109:\n\t"
> > +     /* Copy from 2 * VEC to 4 * VEC. */
> > +     "vmovdqu   (%%rsi), %%ymm0\n\t"
> > +     "vmovdqu   32(%%rsi), %%ymm1\n\t"
> > +     "vmovdqu   -32(%%rsi,%%rdx), %%ymm2\n\t"
> > +     "vmovdqu   -(32 * 2)(%%rsi,%%rdx), %%ymm3\n\t"
> > +     "vmovdqu   %%ymm0, (%%rdi)\n\t"
> > +     "vmovdqu   %%ymm1, 32(%%rdi)\n\t"
> > +     "vmovdqu   %%ymm2, -32(%%rdi,%%rdx)\n\t"
> > +     "vmovdqu   %%ymm3, -(32 * 2)(%%rdi,%%rdx)\n\t"
> > +     "vzeroupper\n\t"
> > +     "110:\n\t"
> > +     "jmp %l[done]\n\t"
> > +     "111:\n\t"
> > +     "cmpq    %%rsi, %%rdi\n\t"
> > +     "ja      113f\n\t"
> > +     /* Source == destination is less common.  */
> > +     "je      110b\n\t"
> > +     /* Load the first VEC and last 4 * VEC to
> > +      * support overlapping addresses.
> > +      */
> > +     "vmovdqu   (%%rsi), %%ymm4\n\t"
> > +     "vmovdqu   -32(%%rsi, %%rdx), %%ymm5\n\t"
> > +     "vmovdqu   -(32 * 2)(%%rsi, %%rdx), %%ymm6\n\t"
> > +     "vmovdqu   -(32 * 3)(%%rsi, %%rdx), %%ymm7\n\t"
> > +     "vmovdqu   -(32 * 4)(%%rsi, %%rdx), %%ymm8\n\t"
> > +     /* Save start and stop of the destination buffer.  */
> > +     "movq    %%rdi, %%r11\n\t"
> > +     "leaq    -32(%%rdi, %%rdx), %%rcx\n\t"
> > +     /* Align destination for aligned stores in the loop.  Compute */
> > +     /* how much destination is misaligned.  */
> > +     "movq    %%rdi, %%r8\n\t"
> > +     "andq    $(32 - 1), %%r8\n\t"
> > +     /* Get the negative of offset for alignment.  */
> > +     "subq    $32, %%r8\n\t"
> > +     /* Adjust source.  */
> > +     "subq    %%r8, %%rsi\n\t"
> > +     /* Adjust destination which should be aligned now.  */
> > +     "subq    %%r8, %%rdi\n\t"
> > +     /* Adjust length.  */
> > +     "addq    %%r8, %%rdx\n\t"
> > +     /* Check non-temporal store threshold.  */
> > +     "cmpq    $(1024*1024), %%rdx\n\t"
> > +     "ja      115f\n\t"
> > +     "112:\n\t"
> > +     /* Copy 4 * VEC a time forward.  */
> > +     "vmovdqu   (%%rsi), %%ymm0\n\t"
> > +     "vmovdqu   32(%%rsi), %%ymm1\n\t"
> > +     "vmovdqu   (32 * 2)(%%rsi), %%ymm2\n\t"
> > +     "vmovdqu   (32 * 3)(%%rsi), %%ymm3\n\t"
> > +     "addq    $(32 * 4), %%rsi\n\t"
> > +     "subq    $(32 * 4), %%rdx\n\t"
> > +     "vmovdqa   %%ymm0, (%%rdi)\n\t"
> > +     "vmovdqa   %%ymm1, 32(%%rdi)\n\t"
> > +     "vmovdqa   %%ymm2, (32 * 2)(%%rdi)\n\t"
> > +     "vmovdqa   %%ymm3, (32 * 3)(%%rdi)\n\t"
> > +     "addq    $(32 * 4), %%rdi\n\t"
> > +     "cmpq    $(32 * 4), %%rdx\n\t"
> > +     "ja      112b\n\t"
> > +     /* Store the last 4 * VEC.  */
> > +     "vmovdqu   %%ymm5, (%%rcx)\n\t"
> > +     "vmovdqu   %%ymm6, -32(%%rcx)\n\t"
> > +     "vmovdqu   %%ymm7, -(32 * 2)(%%rcx)\n\t"
> > +     "vmovdqu   %%ymm8, -(32 * 3)(%%rcx)\n\t"
> > +     /* Store the first VEC.  */
> > +     "vmovdqu   %%ymm4, (%%r11)\n\t"
> > +     "vzeroupper\n\t"
> > +     "jmp %l[done]\n\t"
> > +     "113:\n\t"
> > +     /* Load the first 4*VEC and last VEC to support overlapping addresses.*/
> > +     "vmovdqu   (%%rsi), %%ymm4\n\t"
> > +     "vmovdqu   32(%%rsi), %%ymm5\n\t"
> > +     "vmovdqu   (32 * 2)(%%rsi), %%ymm6\n\t"
> > +     "vmovdqu   (32 * 3)(%%rsi), %%ymm7\n\t"
> > +     "vmovdqu   -32(%%rsi,%%rdx), %%ymm8\n\t"
> > +     /* Save stop of the destination buffer.  */
> > +     "leaq    -32(%%rdi, %%rdx), %%r11\n\t"
> > +     /* Align destination end for aligned stores in the loop.  Compute */
> > +     /* how much destination end is misaligned.  */
> > +     "leaq    -32(%%rsi, %%rdx), %%rcx\n\t"
> > +     "movq    %%r11, %%r9\n\t"
> > +     "movq    %%r11, %%r8\n\t"
> > +     "andq    $(32 - 1), %%r8\n\t"
> > +     /* Adjust source.  */
> > +     "subq    %%r8, %%rcx\n\t"
> > +     /* Adjust the end of destination which should be aligned now.  */
> > +     "subq    %%r8, %%r9\n\t"
> > +     /* Adjust length.  */
> > +     "subq    %%r8, %%rdx\n\t"
> > +      /* Check non-temporal store threshold.  */
> > +     "cmpq    $(1024*1024), %%rdx\n\t"
> > +     "ja      117f\n\t"
> > +     "114:\n\t"
> > +     /* Copy 4 * VEC a time backward.  */
> > +     "vmovdqu   (%%rcx), %%ymm0\n\t"
> > +     "vmovdqu   -32(%%rcx), %%ymm1\n\t"
> > +     "vmovdqu   -(32 * 2)(%%rcx), %%ymm2\n\t"
> > +     "vmovdqu   -(32 * 3)(%%rcx), %%ymm3\n\t"
> > +     "subq    $(32 * 4), %%rcx\n\t"
> > +     "subq    $(32 * 4), %%rdx\n\t"
> > +     "vmovdqa   %%ymm0, (%%r9)\n\t"
> > +     "vmovdqa   %%ymm1, -32(%%r9)\n\t"
> > +     "vmovdqa   %%ymm2, -(32 * 2)(%%r9)\n\t"
> > +     "vmovdqa   %%ymm3, -(32 * 3)(%%r9)\n\t"
> > +     "subq    $(32 * 4), %%r9\n\t"
> > +     "cmpq    $(32 * 4), %%rdx\n\t"
> > +     "ja      114b\n\t"
> > +     /* Store the first 4 * VEC. */
> > +     "vmovdqu   %%ymm4, (%%rdi)\n\t"
> > +     "vmovdqu   %%ymm5, 32(%%rdi)\n\t"
> > +     "vmovdqu   %%ymm6, (32 * 2)(%%rdi)\n\t"
> > +     "vmovdqu   %%ymm7, (32 * 3)(%%rdi)\n\t"
> > +     /* Store the last VEC. */
> > +     "vmovdqu   %%ymm8, (%%r11)\n\t"
> > +     "vzeroupper\n\t"
> > +     "jmp %l[done]\n\t"
> > +
> > +     "115:\n\t"
> > +     /* Don't use non-temporal store if there is overlap between */
> > +     /* destination and source since destination may be in cache */
> > +     /* when source is loaded. */
> > +     "leaq    (%%rdi, %%rdx), %%r10\n\t"
> > +     "cmpq    %%r10, %%rsi\n\t"
> > +     "jb      112b\n\t"
> > +     "116:\n\t"
> > +     /* Copy 4 * VEC a time forward with non-temporal stores.  */
> > +     "prefetcht0 (32*4*2)(%%rsi)\n\t"
> > +     "prefetcht0 (32*4*2 + 64)(%%rsi)\n\t"
> > +     "prefetcht0 (32*4*3)(%%rsi)\n\t"
> > +     "prefetcht0 (32*4*3 + 64)(%%rsi)\n\t"
> > +     "vmovdqu   (%%rsi), %%ymm0\n\t"
> > +     "vmovdqu   32(%%rsi), %%ymm1\n\t"
> > +     "vmovdqu   (32 * 2)(%%rsi), %%ymm2\n\t"
> > +     "vmovdqu   (32 * 3)(%%rsi), %%ymm3\n\t"
> > +     "addq    $(32*4), %%rsi\n\t"
> > +     "subq    $(32*4), %%rdx\n\t"
> > +     "vmovntdq  %%ymm0, (%%rdi)\n\t"
> > +     "vmovntdq  %%ymm1, 32(%%rdi)\n\t"
> > +     "vmovntdq  %%ymm2, (32 * 2)(%%rdi)\n\t"
> > +     "vmovntdq  %%ymm3, (32 * 3)(%%rdi)\n\t"
> > +     "addq    $(32*4), %%rdi\n\t"
> > +     "cmpq    $(32*4), %%rdx\n\t"
> > +     "ja      116b\n\t"
> > +     "sfence\n\t"
> > +     /* Store the last 4 * VEC.  */
> > +     "vmovdqu   %%ymm5, (%%rcx)\n\t"
> > +     "vmovdqu   %%ymm6, -32(%%rcx)\n\t"
> > +     "vmovdqu   %%ymm7, -(32 * 2)(%%rcx)\n\t"
> > +     "vmovdqu   %%ymm8, -(32 * 3)(%%rcx)\n\t"
> > +     /* Store the first VEC.  */
> > +     "vmovdqu   %%ymm4, (%%r11)\n\t"
> > +     "vzeroupper\n\t"
> > +     "jmp %l[done]\n\t"
> > +     "117:\n\t"
> > +     /* Don't use non-temporal store if there is overlap between */
> > +     /* destination and source since destination may be in cache */
> > +     /* when source is loaded.  */
> > +     "leaq    (%%rcx, %%rdx), %%r10\n\t"
> > +     "cmpq    %%r10, %%r9\n\t"
> > +     "jb      114b\n\t"
> > +     "118:\n\t"
> > +     /* Copy 4 * VEC a time backward with non-temporal stores. */
> > +     "prefetcht0 (-32 * 4 * 2)(%%rcx)\n\t"
> > +     "prefetcht0 (-32 * 4 * 2 - 64)(%%rcx)\n\t"
> > +     "prefetcht0 (-32 * 4 * 3)(%%rcx)\n\t"
> > +     "prefetcht0 (-32 * 4 * 3 - 64)(%%rcx)\n\t"
> > +     "vmovdqu   (%%rcx), %%ymm0\n\t"
> > +     "vmovdqu   -32(%%rcx), %%ymm1\n\t"
> > +     "vmovdqu   -(32 * 2)(%%rcx), %%ymm2\n\t"
> > +     "vmovdqu   -(32 * 3)(%%rcx), %%ymm3\n\t"
> > +     "subq    $(32*4), %%rcx\n\t"
> > +     "subq    $(32*4), %%rdx\n\t"
> > +     "vmovntdq  %%ymm0, (%%r9)\n\t"
> > +     "vmovntdq  %%ymm1, -32(%%r9)\n\t"
> > +     "vmovntdq  %%ymm2, -(32 * 2)(%%r9)\n\t"
> > +     "vmovntdq  %%ymm3, -(32 * 3)(%%r9)\n\t"
> > +     "subq    $(32 * 4), %%r9\n\t"
> > +     "cmpq    $(32 * 4), %%rdx\n\t"
> > +     "ja      118b\n\t"
> > +     "sfence\n\t"
> > +     /* Store the first 4 * VEC.  */
> > +     "vmovdqu   %%ymm4, (%%rdi)\n\t"
> > +     "vmovdqu   %%ymm5, 32(%%rdi)\n\t"
> > +     "vmovdqu   %%ymm6, (32 * 2)(%%rdi)\n\t"
> > +     "vmovdqu   %%ymm7, (32 * 3)(%%rdi)\n\t"
> > +     /* Store the last VEC.  */
> > +     "vmovdqu   %%ymm8, (%%r11)\n\t"
> > +     "vzeroupper\n\t"
> > +     "jmp %l[done]"
> > +     :
> > +     : "r"(src), "r"(dst), "r"(len)
> > +     : "rax", "rcx", "rdx", "rdi", "rsi", "r8", "r9", "r10", "r11", "r12", "ymm0",
> > +     "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", "ymm8", "memory"
> > +     : done
> > +     );
> > +done:
> > +     return dst;
> > +}
>
>

Re: [dpdk-dev] [PATCH v2 1/2] lib/eal: add amd epyc2 memcpy routine to eal

[Thomas Monjalon]

Please convert it to C code, thanks.


21/10/2021 20:12, Song, Keesang:
> [AMD Official Use Only]
> 
> Hi Ananyev,
> 
> The current memcpy implementation in Glibc is based out of assembly coding.
> Although memcpy could have been implemented with intrinsic, but since our AMD library developers are working on the Glibc functions, they have provided a tailored implementation based out of inline assembly coding.
> 
> Thanks for your support,
> Keesang
> 
> -----Original Message-----
> From: Ananyev, Konstantin <konstantin.ananyev@intel.com>
> Sent: Thursday, October 21, 2021 10:40 AM
> To: Song, Keesang <Keesang.Song@amd.com>; Thomas Monjalon <thomas@monjalon.net>; Aman Kumar <aman.kumar@vvdntech.in>
> Cc: dev@dpdk.org; rasland@nvidia.com; asafp@nvidia.com; shys@nvidia.com; viacheslavo@nvidia.com; akozyrev@nvidia.com; matan@nvidia.com; Burakov, Anatoly <anatoly.burakov@intel.com>; aman.kumar@vvdntech.in; jerinjacobk@gmail.com; Richardson, Bruce <bruce.richardson@intel.com>; david.marchand@redhat.com
> Subject: RE: [dpdk-dev] [PATCH v2 1/2] lib/eal: add amd epyc2 memcpy routine to eal
> 
> [AMD Official Use Only]
> 
> [CAUTION: External Email]
> 
> >
> > Hi Thomas,
> >
> > I hope this can make some explanation to your question.
> > We(AMD Linux library support team) have implemented the custom
> > tailored memcpy solution which is a close match with DPDK use case requirements like the below.
> > 1)      Min 64B length data packet with cache aligned Source and Destination.
> > 2)      Non-Temporal load and temporal store for cache aligned source for both RX and TX paths. Could not implement the non-temporal
> > store for TX_PATH, as non-Temporal load/stores works only with 32B aligned addresses for AVX2
> > 3)      This solution works for all AVX2 supported AMD machines.
> >
> > Internally we have completed the integrity testing and benchmarking of
> > the solution and found gains of 8.4% to 14.5% specifically on Milan
> > CPU(3rd Gen of EPYC Processor)
> 
> It still not clear to me why it has to be written in assembler.
> Why similar stuff can't be written in C with instincts, as rest of rte_memcpy.h does?
> 
> >
> > Thanks for your support,
> > Keesang
> >
> > -----Original Message-----
> > From: Thomas Monjalon <thomas@monjalon.net>
> > Sent: Tuesday, October 19, 2021 5:31 AM
> > To: Aman Kumar <aman.kumar@vvdntech.in>
> > Cc: dev@dpdk.org; rasland@nvidia.com; asafp@nvidia.com;
> > shys@nvidia.com; viacheslavo@nvidia.com; akozyrev@nvidia.com;
> > matan@nvidia.com; anatoly.burakov@intel.com; Song, Keesang
> > <Keesang.Song@amd.com>; aman.kumar@vvdntech.in; jerinjacobk@gmail.com;
> > bruce.richardson@intel.com; konstantin.ananyev@intel.com;
> > david.marchand@redhat.com
> > Subject: Re: [dpdk-dev] [PATCH v2 1/2] lib/eal: add amd epyc2 memcpy
> > routine to eal
> >
> > [CAUTION: External Email]
> >
> > 19/10/2021 12:47, Aman Kumar:
> > > This patch provides rte_memcpy* calls optimized for AMD EPYC
> > > platforms. Use config/x86/x86_amd_epyc_linux_gcc as cross-file with
> > > meson to build dpdk for AMD EPYC platforms.
> >
> > Please split in 2 patches: platform & memcpy.
> >
> > What optimization is specific to EPYC?
> >
> > I dislike the asm code below.
> > What is AMD specific inside?
> > Can it use compiler intrinsics as it is done elsewhere?
> >
> > > +static __rte_always_inline void *
> > > +rte_memcpy_aligned_ntload_tstore16_amdepyc2(void *dst,
> > > +                                         const void *src,
> > > +                                         size_t size) {
> > > +     asm volatile goto("movq %0, %%rsi\n\t"
> > > +     "movq %1, %%rdi\n\t"
> > > +     "movq %2, %%rdx\n\t"
> > > +     "cmpq   $(128), %%rdx\n\t"
> > > +     "jb     202f\n\t"
> > > +     "201:\n\t"
> > > +     "vmovntdqa (%%rsi), %%ymm0\n\t"
> > > +     "vmovntdqa 32(%%rsi), %%ymm1\n\t"
> > > +     "vmovntdqa 64(%%rsi), %%ymm2\n\t"
> > > +     "vmovntdqa 96(%%rsi), %%ymm3\n\t"
> > > +     "vmovdqu  %%ymm0, (%%rdi)\n\t"
> > > +     "vmovdqu  %%ymm1, 32(%%rdi)\n\t"
> > > +     "vmovdqu  %%ymm2, 64(%%rdi)\n\t"
> > > +     "vmovdqu  %%ymm3, 96(%%rdi)\n\t"
> > > +     "addq   $128, %%rsi\n\t"
> > > +     "addq   $128, %%rdi\n\t"
> > > +     "subq   $128, %%rdx\n\t"
> > > +     "jz     %l[done]\n\t"
> > > +     "cmpq   $128, %%rdx\n\t" /*Vector Size 32B.  */
> > > +     "jae    201b\n\t"
> > > +     "202:\n\t"
> > > +     "cmpq   $64, %%rdx\n\t"
> > > +     "jb     203f\n\t"
> > > +     "vmovntdqa (%%rsi), %%ymm0\n\t"
> > > +     "vmovntdqa 32(%%rsi), %%ymm1\n\t"
> > > +     "vmovdqu  %%ymm0, (%%rdi)\n\t"
> > > +     "vmovdqu  %%ymm1, 32(%%rdi)\n\t"
> > > +     "addq   $64, %%rsi\n\t"
> > > +     "addq   $64, %%rdi\n\t"
> > > +     "subq   $64, %%rdx\n\t"
> > > +     "jz     %l[done]\n\t"
> > > +     "203:\n\t"
> > > +     "cmpq   $32, %%rdx\n\t"
> > > +     "jb     204f\n\t"
> > > +     "vmovntdqa (%%rsi), %%ymm0\n\t"
> > > +     "vmovdqu  %%ymm0, (%%rdi)\n\t"
> > > +     "addq   $32, %%rsi\n\t"
> > > +     "addq   $32, %%rdi\n\t"
> > > +     "subq   $32, %%rdx\n\t"
> > > +     "jz     %l[done]\n\t"
> > > +     "204:\n\t"
> > > +     "cmpb   $16, %%dl\n\t"
> > > +     "jb     205f\n\t"
> > > +     "vmovntdqa (%%rsi), %%xmm0\n\t"
> > > +     "vmovdqu  %%xmm0, (%%rdi)\n\t"
> > > +     "addq   $16, %%rsi\n\t"
> > > +     "addq   $16, %%rdi\n\t"
> > > +     "subq   $16, %%rdx\n\t"
> > > +     "jz     %l[done]\n\t"
> > > +     "205:\n\t"
> > > +     "cmpb   $2, %%dl\n\t"
> > > +     "jb     208f\n\t"
> > > +     "cmpb   $4, %%dl\n\t"
> > > +     "jbe    207f\n\t"
> > > +     "cmpb   $8, %%dl\n\t"
> > > +     "jbe    206f\n\t"
> > > +     "movq   -8(%%rsi,%%rdx), %%rcx\n\t"
> > > +     "movq   (%%rsi), %%rsi\n\t"
> > > +     "movq   %%rcx, -8(%%rdi,%%rdx)\n\t"
> > > +     "movq   %%rsi, (%%rdi)\n\t"
> > > +     "jmp    %l[done]\n\t"
> > > +     "206:\n\t"
> > > +     "movl   -4(%%rsi,%%rdx), %%ecx\n\t"
> > > +     "movl   (%%rsi), %%esi\n\t"
> > > +     "movl   %%ecx, -4(%%rdi,%%rdx)\n\t"
> > > +     "movl   %%esi, (%%rdi)\n\t"
> > > +     "jmp    %l[done]\n\t"
> > > +     "207:\n\t"
> > > +     "movzwl -2(%%rsi,%%rdx), %%ecx\n\t"
> > > +     "movzwl (%%rsi), %%esi\n\t"
> > > +     "movw   %%cx, -2(%%rdi,%%rdx)\n\t"
> > > +     "movw   %%si, (%%rdi)\n\t"
> > > +     "jmp    %l[done]\n\t"
> > > +     "208:\n\t"
> > > +     "movzbl (%%rsi), %%ecx\n\t"
> > > +     "movb   %%cl, (%%rdi)"
> > > +     :
> > > +     : "r"(src), "r"(dst), "r"(size)
> > > +     : "rcx", "rdx", "rsi", "rdi", "ymm0", "ymm1", "ymm2", "ymm3", "memory"
> > > +     : done
> > > +     );
> > > +done:
> > > +     return dst;
> > > +}
> > > +
> > > +static __rte_always_inline void *
> > > +rte_memcpy_generic(void *dst, const void *src, size_t len) {
> > > +     asm goto("movq  %0, %%rsi\n\t"
> > > +     "movq   %1, %%rdi\n\t"
> > > +     "movq   %2, %%rdx\n\t"
> > > +     "movq    %%rdi, %%rax\n\t"
> > > +     "cmp     $32, %%rdx\n\t"
> > > +     "jb      101f\n\t"
> > > +     "cmp     $(32 * 2), %%rdx\n\t"
> > > +     "ja      108f\n\t"
> > > +     "vmovdqu   (%%rsi), %%ymm0\n\t"
> > > +     "vmovdqu   -32(%%rsi,%%rdx), %%ymm1\n\t"
> > > +     "vmovdqu   %%ymm0, (%%rdi)\n\t"
> > > +     "vmovdqu   %%ymm1, -32(%%rdi,%%rdx)\n\t"
> > > +     "vzeroupper\n\t"
> > > +     "jmp %l[done]\n\t"
> > > +     "101:\n\t"
> > > +     /* Less than 1 VEC.  */
> > > +     "cmpb    $32, %%dl\n\t"
> > > +     "jae     103f\n\t"
> > > +     "cmpb    $16, %%dl\n\t"
> > > +     "jae     104f\n\t"
> > > +     "cmpb    $8, %%dl\n\t"
> > > +     "jae     105f\n\t"
> > > +     "cmpb    $4, %%dl\n\t"
> > > +     "jae     106f\n\t"
> > > +     "cmpb    $1, %%dl\n\t"
> > > +     "ja      107f\n\t"
> > > +     "jb      102f\n\t"
> > > +     "movzbl  (%%rsi), %%ecx\n\t"
> > > +     "movb    %%cl, (%%rdi)\n\t"
> > > +     "102:\n\t"
> > > +     "jmp %l[done]\n\t"
> > > +     "103:\n\t"
> > > +     /* From 32 to 63.  No branch when size == 32.  */
> > > +     "vmovdqu (%%rsi), %%ymm0\n\t"
> > > +     "vmovdqu -32(%%rsi,%%rdx), %%ymm1\n\t"
> > > +     "vmovdqu %%ymm0, (%%rdi)\n\t"
> > > +     "vmovdqu %%ymm1, -32(%%rdi,%%rdx)\n\t"
> > > +     "vzeroupper\n\t"
> > > +     "jmp %l[done]\n\t"
> > > +     /* From 16 to 31.  No branch when size == 16.  */
> > > +     "104:\n\t"
> > > +     "vmovdqu (%%rsi), %%xmm0\n\t"
> > > +     "vmovdqu -16(%%rsi,%%rdx), %%xmm1\n\t"
> > > +     "vmovdqu %%xmm0, (%%rdi)\n\t"
> > > +     "vmovdqu %%xmm1, -16(%%rdi,%%rdx)\n\t"
> > > +     "jmp %l[done]\n\t"
> > > +     "105:\n\t"
> > > +     /* From 8 to 15.  No branch when size == 8.  */
> > > +     "movq    -8(%%rsi,%%rdx), %%rcx\n\t"
> > > +     "movq    (%%rsi), %%rsi\n\t"
> > > +     "movq    %%rcx, -8(%%rdi,%%rdx)\n\t"
> > > +     "movq    %%rsi, (%%rdi)\n\t"
> > > +     "jmp %l[done]\n\t"
> > > +     "106:\n\t"
> > > +     /* From 4 to 7.  No branch when size == 4.  */
> > > +     "movl    -4(%%rsi,%%rdx), %%ecx\n\t"
> > > +     "movl    (%%rsi), %%esi\n\t"
> > > +     "movl    %%ecx, -4(%%rdi,%%rdx)\n\t"
> > > +     "movl    %%esi, (%%rdi)\n\t"
> > > +     "jmp %l[done]\n\t"
> > > +     "107:\n\t"
> > > +     /* From 2 to 3.  No branch when size == 2.  */
> > > +     "movzwl  -2(%%rsi,%%rdx), %%ecx\n\t"
> > > +     "movzwl  (%%rsi), %%esi\n\t"
> > > +     "movw    %%cx, -2(%%rdi,%%rdx)\n\t"
> > > +     "movw    %%si, (%%rdi)\n\t"
> > > +     "jmp %l[done]\n\t"
> > > +     "108:\n\t"
> > > +     /* More than 2 * VEC and there may be overlap between destination */
> > > +     /* and source.  */
> > > +     "cmpq    $(32 * 8), %%rdx\n\t"
> > > +     "ja      111f\n\t"
> > > +     "cmpq    $(32 * 4), %%rdx\n\t"
> > > +     "jb      109f\n\t"
> > > +     /* Copy from 4 * VEC to 8 * VEC, inclusively. */
> > > +     "vmovdqu   (%%rsi), %%ymm0\n\t"
> > > +     "vmovdqu   32(%%rsi), %%ymm1\n\t"
> > > +     "vmovdqu   (32 * 2)(%%rsi), %%ymm2\n\t"
> > > +     "vmovdqu   (32 * 3)(%%rsi), %%ymm3\n\t"
> > > +     "vmovdqu   -32(%%rsi,%%rdx), %%ymm4\n\t"
> > > +     "vmovdqu   -(32 * 2)(%%rsi,%%rdx), %%ymm5\n\t"
> > > +     "vmovdqu   -(32 * 3)(%%rsi,%%rdx), %%ymm6\n\t"
> > > +     "vmovdqu   -(32 * 4)(%%rsi,%%rdx), %%ymm7\n\t"
> > > +     "vmovdqu   %%ymm0, (%%rdi)\n\t"
> > > +     "vmovdqu   %%ymm1, 32(%%rdi)\n\t"
> > > +     "vmovdqu   %%ymm2, (32 * 2)(%%rdi)\n\t"
> > > +     "vmovdqu   %%ymm3, (32 * 3)(%%rdi)\n\t"
> > > +     "vmovdqu   %%ymm4, -32(%%rdi,%%rdx)\n\t"
> > > +     "vmovdqu   %%ymm5, -(32 * 2)(%%rdi,%%rdx)\n\t"
> > > +     "vmovdqu   %%ymm6, -(32 * 3)(%%rdi,%%rdx)\n\t"
> > > +     "vmovdqu   %%ymm7, -(32 * 4)(%%rdi,%%rdx)\n\t"
> > > +     "vzeroupper\n\t"
> > > +     "jmp %l[done]\n\t"
> > > +     "109:\n\t"
> > > +     /* Copy from 2 * VEC to 4 * VEC. */
> > > +     "vmovdqu   (%%rsi), %%ymm0\n\t"
> > > +     "vmovdqu   32(%%rsi), %%ymm1\n\t"
> > > +     "vmovdqu   -32(%%rsi,%%rdx), %%ymm2\n\t"
> > > +     "vmovdqu   -(32 * 2)(%%rsi,%%rdx), %%ymm3\n\t"
> > > +     "vmovdqu   %%ymm0, (%%rdi)\n\t"
> > > +     "vmovdqu   %%ymm1, 32(%%rdi)\n\t"
> > > +     "vmovdqu   %%ymm2, -32(%%rdi,%%rdx)\n\t"
> > > +     "vmovdqu   %%ymm3, -(32 * 2)(%%rdi,%%rdx)\n\t"
> > > +     "vzeroupper\n\t"
> > > +     "110:\n\t"
> > > +     "jmp %l[done]\n\t"
> > > +     "111:\n\t"
> > > +     "cmpq    %%rsi, %%rdi\n\t"
> > > +     "ja      113f\n\t"
> > > +     /* Source == destination is less common.  */
> > > +     "je      110b\n\t"
> > > +     /* Load the first VEC and last 4 * VEC to
> > > +      * support overlapping addresses.
> > > +      */
> > > +     "vmovdqu   (%%rsi), %%ymm4\n\t"
> > > +     "vmovdqu   -32(%%rsi, %%rdx), %%ymm5\n\t"
> > > +     "vmovdqu   -(32 * 2)(%%rsi, %%rdx), %%ymm6\n\t"
> > > +     "vmovdqu   -(32 * 3)(%%rsi, %%rdx), %%ymm7\n\t"
> > > +     "vmovdqu   -(32 * 4)(%%rsi, %%rdx), %%ymm8\n\t"
> > > +     /* Save start and stop of the destination buffer.  */
> > > +     "movq    %%rdi, %%r11\n\t"
> > > +     "leaq    -32(%%rdi, %%rdx), %%rcx\n\t"
> > > +     /* Align destination for aligned stores in the loop.  Compute */
> > > +     /* how much destination is misaligned.  */
> > > +     "movq    %%rdi, %%r8\n\t"
> > > +     "andq    $(32 - 1), %%r8\n\t"
> > > +     /* Get the negative of offset for alignment.  */
> > > +     "subq    $32, %%r8\n\t"
> > > +     /* Adjust source.  */
> > > +     "subq    %%r8, %%rsi\n\t"
> > > +     /* Adjust destination which should be aligned now.  */
> > > +     "subq    %%r8, %%rdi\n\t"
> > > +     /* Adjust length.  */
> > > +     "addq    %%r8, %%rdx\n\t"
> > > +     /* Check non-temporal store threshold.  */
> > > +     "cmpq    $(1024*1024), %%rdx\n\t"
> > > +     "ja      115f\n\t"
> > > +     "112:\n\t"
> > > +     /* Copy 4 * VEC a time forward.  */
> > > +     "vmovdqu   (%%rsi), %%ymm0\n\t"
> > > +     "vmovdqu   32(%%rsi), %%ymm1\n\t"
> > > +     "vmovdqu   (32 * 2)(%%rsi), %%ymm2\n\t"
> > > +     "vmovdqu   (32 * 3)(%%rsi), %%ymm3\n\t"
> > > +     "addq    $(32 * 4), %%rsi\n\t"
> > > +     "subq    $(32 * 4), %%rdx\n\t"
> > > +     "vmovdqa   %%ymm0, (%%rdi)\n\t"
> > > +     "vmovdqa   %%ymm1, 32(%%rdi)\n\t"
> > > +     "vmovdqa   %%ymm2, (32 * 2)(%%rdi)\n\t"
> > > +     "vmovdqa   %%ymm3, (32 * 3)(%%rdi)\n\t"
> > > +     "addq    $(32 * 4), %%rdi\n\t"
> > > +     "cmpq    $(32 * 4), %%rdx\n\t"
> > > +     "ja      112b\n\t"
> > > +     /* Store the last 4 * VEC.  */
> > > +     "vmovdqu   %%ymm5, (%%rcx)\n\t"
> > > +     "vmovdqu   %%ymm6, -32(%%rcx)\n\t"
> > > +     "vmovdqu   %%ymm7, -(32 * 2)(%%rcx)\n\t"
> > > +     "vmovdqu   %%ymm8, -(32 * 3)(%%rcx)\n\t"
> > > +     /* Store the first VEC.  */
> > > +     "vmovdqu   %%ymm4, (%%r11)\n\t"
> > > +     "vzeroupper\n\t"
> > > +     "jmp %l[done]\n\t"
> > > +     "113:\n\t"
> > > +     /* Load the first 4*VEC and last VEC to support overlapping addresses.*/
> > > +     "vmovdqu   (%%rsi), %%ymm4\n\t"
> > > +     "vmovdqu   32(%%rsi), %%ymm5\n\t"
> > > +     "vmovdqu   (32 * 2)(%%rsi), %%ymm6\n\t"
> > > +     "vmovdqu   (32 * 3)(%%rsi), %%ymm7\n\t"
> > > +     "vmovdqu   -32(%%rsi,%%rdx), %%ymm8\n\t"
> > > +     /* Save stop of the destination buffer.  */
> > > +     "leaq    -32(%%rdi, %%rdx), %%r11\n\t"
> > > +     /* Align destination end for aligned stores in the loop.  Compute */
> > > +     /* how much destination end is misaligned.  */
> > > +     "leaq    -32(%%rsi, %%rdx), %%rcx\n\t"
> > > +     "movq    %%r11, %%r9\n\t"
> > > +     "movq    %%r11, %%r8\n\t"
> > > +     "andq    $(32 - 1), %%r8\n\t"
> > > +     /* Adjust source.  */
> > > +     "subq    %%r8, %%rcx\n\t"
> > > +     /* Adjust the end of destination which should be aligned now.  */
> > > +     "subq    %%r8, %%r9\n\t"
> > > +     /* Adjust length.  */
> > > +     "subq    %%r8, %%rdx\n\t"
> > > +      /* Check non-temporal store threshold.  */
> > > +     "cmpq    $(1024*1024), %%rdx\n\t"
> > > +     "ja      117f\n\t"
> > > +     "114:\n\t"
> > > +     /* Copy 4 * VEC a time backward.  */
> > > +     "vmovdqu   (%%rcx), %%ymm0\n\t"
> > > +     "vmovdqu   -32(%%rcx), %%ymm1\n\t"
> > > +     "vmovdqu   -(32 * 2)(%%rcx), %%ymm2\n\t"
> > > +     "vmovdqu   -(32 * 3)(%%rcx), %%ymm3\n\t"
> > > +     "subq    $(32 * 4), %%rcx\n\t"
> > > +     "subq    $(32 * 4), %%rdx\n\t"
> > > +     "vmovdqa   %%ymm0, (%%r9)\n\t"
> > > +     "vmovdqa   %%ymm1, -32(%%r9)\n\t"
> > > +     "vmovdqa   %%ymm2, -(32 * 2)(%%r9)\n\t"
> > > +     "vmovdqa   %%ymm3, -(32 * 3)(%%r9)\n\t"
> > > +     "subq    $(32 * 4), %%r9\n\t"
> > > +     "cmpq    $(32 * 4), %%rdx\n\t"
> > > +     "ja      114b\n\t"
> > > +     /* Store the first 4 * VEC. */
> > > +     "vmovdqu   %%ymm4, (%%rdi)\n\t"
> > > +     "vmovdqu   %%ymm5, 32(%%rdi)\n\t"
> > > +     "vmovdqu   %%ymm6, (32 * 2)(%%rdi)\n\t"
> > > +     "vmovdqu   %%ymm7, (32 * 3)(%%rdi)\n\t"
> > > +     /* Store the last VEC. */
> > > +     "vmovdqu   %%ymm8, (%%r11)\n\t"
> > > +     "vzeroupper\n\t"
> > > +     "jmp %l[done]\n\t"
> > > +
> > > +     "115:\n\t"
> > > +     /* Don't use non-temporal store if there is overlap between */
> > > +     /* destination and source since destination may be in cache */
> > > +     /* when source is loaded. */
> > > +     "leaq    (%%rdi, %%rdx), %%r10\n\t"
> > > +     "cmpq    %%r10, %%rsi\n\t"
> > > +     "jb      112b\n\t"
> > > +     "116:\n\t"
> > > +     /* Copy 4 * VEC a time forward with non-temporal stores.  */
> > > +     "prefetcht0 (32*4*2)(%%rsi)\n\t"
> > > +     "prefetcht0 (32*4*2 + 64)(%%rsi)\n\t"
> > > +     "prefetcht0 (32*4*3)(%%rsi)\n\t"
> > > +     "prefetcht0 (32*4*3 + 64)(%%rsi)\n\t"
> > > +     "vmovdqu   (%%rsi), %%ymm0\n\t"
> > > +     "vmovdqu   32(%%rsi), %%ymm1\n\t"
> > > +     "vmovdqu   (32 * 2)(%%rsi), %%ymm2\n\t"
> > > +     "vmovdqu   (32 * 3)(%%rsi), %%ymm3\n\t"
> > > +     "addq    $(32*4), %%rsi\n\t"
> > > +     "subq    $(32*4), %%rdx\n\t"
> > > +     "vmovntdq  %%ymm0, (%%rdi)\n\t"
> > > +     "vmovntdq  %%ymm1, 32(%%rdi)\n\t"
> > > +     "vmovntdq  %%ymm2, (32 * 2)(%%rdi)\n\t"
> > > +     "vmovntdq  %%ymm3, (32 * 3)(%%rdi)\n\t"
> > > +     "addq    $(32*4), %%rdi\n\t"
> > > +     "cmpq    $(32*4), %%rdx\n\t"
> > > +     "ja      116b\n\t"
> > > +     "sfence\n\t"
> > > +     /* Store the last 4 * VEC.  */
> > > +     "vmovdqu   %%ymm5, (%%rcx)\n\t"
> > > +     "vmovdqu   %%ymm6, -32(%%rcx)\n\t"
> > > +     "vmovdqu   %%ymm7, -(32 * 2)(%%rcx)\n\t"
> > > +     "vmovdqu   %%ymm8, -(32 * 3)(%%rcx)\n\t"
> > > +     /* Store the first VEC.  */
> > > +     "vmovdqu   %%ymm4, (%%r11)\n\t"
> > > +     "vzeroupper\n\t"
> > > +     "jmp %l[done]\n\t"
> > > +     "117:\n\t"
> > > +     /* Don't use non-temporal store if there is overlap between */
> > > +     /* destination and source since destination may be in cache */
> > > +     /* when source is loaded.  */
> > > +     "leaq    (%%rcx, %%rdx), %%r10\n\t"
> > > +     "cmpq    %%r10, %%r9\n\t"
> > > +     "jb      114b\n\t"
> > > +     "118:\n\t"
> > > +     /* Copy 4 * VEC a time backward with non-temporal stores. */
> > > +     "prefetcht0 (-32 * 4 * 2)(%%rcx)\n\t"
> > > +     "prefetcht0 (-32 * 4 * 2 - 64)(%%rcx)\n\t"
> > > +     "prefetcht0 (-32 * 4 * 3)(%%rcx)\n\t"
> > > +     "prefetcht0 (-32 * 4 * 3 - 64)(%%rcx)\n\t"
> > > +     "vmovdqu   (%%rcx), %%ymm0\n\t"
> > > +     "vmovdqu   -32(%%rcx), %%ymm1\n\t"
> > > +     "vmovdqu   -(32 * 2)(%%rcx), %%ymm2\n\t"
> > > +     "vmovdqu   -(32 * 3)(%%rcx), %%ymm3\n\t"
> > > +     "subq    $(32*4), %%rcx\n\t"
> > > +     "subq    $(32*4), %%rdx\n\t"
> > > +     "vmovntdq  %%ymm0, (%%r9)\n\t"
> > > +     "vmovntdq  %%ymm1, -32(%%r9)\n\t"
> > > +     "vmovntdq  %%ymm2, -(32 * 2)(%%r9)\n\t"
> > > +     "vmovntdq  %%ymm3, -(32 * 3)(%%r9)\n\t"
> > > +     "subq    $(32 * 4), %%r9\n\t"
> > > +     "cmpq    $(32 * 4), %%rdx\n\t"
> > > +     "ja      118b\n\t"
> > > +     "sfence\n\t"
> > > +     /* Store the first 4 * VEC.  */
> > > +     "vmovdqu   %%ymm4, (%%rdi)\n\t"
> > > +     "vmovdqu   %%ymm5, 32(%%rdi)\n\t"
> > > +     "vmovdqu   %%ymm6, (32 * 2)(%%rdi)\n\t"
> > > +     "vmovdqu   %%ymm7, (32 * 3)(%%rdi)\n\t"
> > > +     /* Store the last VEC.  */
> > > +     "vmovdqu   %%ymm8, (%%r11)\n\t"
> > > +     "vzeroupper\n\t"
> > > +     "jmp %l[done]"
> > > +     :
> > > +     : "r"(src), "r"(dst), "r"(len)
> > > +     : "rax", "rcx", "rdx", "rdi", "rsi", "r8", "r9", "r10", "r11", "r12", "ymm0",
> > > +     "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", "ymm8", "memory"
> > > +     : done
> > > +     );
> > > +done:
> > > +     return dst;
> > > +}
> >
> >
> 
> 

Re: [dpdk-dev] [PATCH v2 1/2] lib/eal: add amd epyc2 memcpy routine to eal

[Song, Keesang]

[AMD Official Use Only]

Hi Thomas,

I've already asked our AMD tools team, but they're saying they are not really familiar with C code implementation. We need your approval for now since we really need to get this patch submitted to 21.11 LTS.

Thanks,
Keesang

-----Original Message-----
From: Thomas Monjalon <thomas@monjalon.net>
Sent: Thursday, October 21, 2021 11:42 AM
To: Aman Kumar <aman.kumar@vvdntech.in>; Song, Keesang <Keesang.Song@amd.com>
Cc: Ananyev, Konstantin <konstantin.ananyev@intel.com>; dev@dpdk.org; rasland@nvidia.com; asafp@nvidia.com; shys@nvidia.com; viacheslavo@nvidia.com; akozyrev@nvidia.com; matan@nvidia.com; Burakov, Anatoly <anatoly.burakov@intel.com>; aman.kumar@vvdntech.in; jerinjacobk@gmail.com; Richardson, Bruce <bruce.richardson@intel.com>; david.marchand@redhat.com
Subject: Re: [dpdk-dev] [PATCH v2 1/2] lib/eal: add amd epyc2 memcpy routine to eal

[CAUTION: External Email]

Please convert it to C code, thanks.


21/10/2021 20:12, Song, Keesang:
> [AMD Official Use Only]
>
> Hi Ananyev,
>
> The current memcpy implementation in Glibc is based out of assembly coding.
> Although memcpy could have been implemented with intrinsic, but since our AMD library developers are working on the Glibc functions, they have provided a tailored implementation based out of inline assembly coding.
>
> Thanks for your support,
> Keesang
>
> -----Original Message-----
> From: Ananyev, Konstantin <konstantin.ananyev@intel.com>
> Sent: Thursday, October 21, 2021 10:40 AM
> To: Song, Keesang <Keesang.Song@amd.com>; Thomas Monjalon
> <thomas@monjalon.net>; Aman Kumar <aman.kumar@vvdntech.in>
> Cc: dev@dpdk.org; rasland@nvidia.com; asafp@nvidia.com;
> shys@nvidia.com; viacheslavo@nvidia.com; akozyrev@nvidia.com;
> matan@nvidia.com; Burakov, Anatoly <anatoly.burakov@intel.com>;
> aman.kumar@vvdntech.in; jerinjacobk@gmail.com; Richardson, Bruce
> <bruce.richardson@intel.com>; david.marchand@redhat.com
> Subject: RE: [dpdk-dev] [PATCH v2 1/2] lib/eal: add amd epyc2 memcpy
> routine to eal
>
> [AMD Official Use Only]
>
> [CAUTION: External Email]
>
> >
> > Hi Thomas,
> >
> > I hope this can make some explanation to your question.
> > We(AMD Linux library support team) have implemented the custom
> > tailored memcpy solution which is a close match with DPDK use case requirements like the below.
> > 1)      Min 64B length data packet with cache aligned Source and Destination.
> > 2)      Non-Temporal load and temporal store for cache aligned source for both RX and TX paths. Could not implement the non-temporal
> > store for TX_PATH, as non-Temporal load/stores works only with 32B aligned addresses for AVX2
> > 3)      This solution works for all AVX2 supported AMD machines.
> >
> > Internally we have completed the integrity testing and benchmarking
> > of the solution and found gains of 8.4% to 14.5% specifically on
> > Milan CPU(3rd Gen of EPYC Processor)
>
> It still not clear to me why it has to be written in assembler.
> Why similar stuff can't be written in C with instincts, as rest of rte_memcpy.h does?
>
> >
> > Thanks for your support,
> > Keesang
> >
> > -----Original Message-----
> > From: Thomas Monjalon <thomas@monjalon.net>
> > Sent: Tuesday, October 19, 2021 5:31 AM
> > To: Aman Kumar <aman.kumar@vvdntech.in>
> > Cc: dev@dpdk.org; rasland@nvidia.com; asafp@nvidia.com;
> > shys@nvidia.com; viacheslavo@nvidia.com; akozyrev@nvidia.com;
> > matan@nvidia.com; anatoly.burakov@intel.com; Song, Keesang
> > <Keesang.Song@amd.com>; aman.kumar@vvdntech.in;
> > jerinjacobk@gmail.com; bruce.richardson@intel.com;
> > konstantin.ananyev@intel.com; david.marchand@redhat.com
> > Subject: Re: [dpdk-dev] [PATCH v2 1/2] lib/eal: add amd epyc2 memcpy
> > routine to eal
> >
> > [CAUTION: External Email]
> >
> > 19/10/2021 12:47, Aman Kumar:
> > > This patch provides rte_memcpy* calls optimized for AMD EPYC
> > > platforms. Use config/x86/x86_amd_epyc_linux_gcc as cross-file
> > > with meson to build dpdk for AMD EPYC platforms.
> >
> > Please split in 2 patches: platform & memcpy.
> >
> > What optimization is specific to EPYC?
> >
> > I dislike the asm code below.
> > What is AMD specific inside?
> > Can it use compiler intrinsics as it is done elsewhere?
> >
> > > +static __rte_always_inline void *
> > > +rte_memcpy_aligned_ntload_tstore16_amdepyc2(void *dst,
> > > +                                         const void *src,
> > > +                                         size_t size) {
> > > +     asm volatile goto("movq %0, %%rsi\n\t"
> > > +     "movq %1, %%rdi\n\t"
> > > +     "movq %2, %%rdx\n\t"
> > > +     "cmpq   $(128), %%rdx\n\t"
> > > +     "jb     202f\n\t"
> > > +     "201:\n\t"
> > > +     "vmovntdqa (%%rsi), %%ymm0\n\t"
> > > +     "vmovntdqa 32(%%rsi), %%ymm1\n\t"
> > > +     "vmovntdqa 64(%%rsi), %%ymm2\n\t"
> > > +     "vmovntdqa 96(%%rsi), %%ymm3\n\t"
> > > +     "vmovdqu  %%ymm0, (%%rdi)\n\t"
> > > +     "vmovdqu  %%ymm1, 32(%%rdi)\n\t"
> > > +     "vmovdqu  %%ymm2, 64(%%rdi)\n\t"
> > > +     "vmovdqu  %%ymm3, 96(%%rdi)\n\t"
> > > +     "addq   $128, %%rsi\n\t"
> > > +     "addq   $128, %%rdi\n\t"
> > > +     "subq   $128, %%rdx\n\t"
> > > +     "jz     %l[done]\n\t"
> > > +     "cmpq   $128, %%rdx\n\t" /*Vector Size 32B.  */
> > > +     "jae    201b\n\t"
> > > +     "202:\n\t"
> > > +     "cmpq   $64, %%rdx\n\t"
> > > +     "jb     203f\n\t"
> > > +     "vmovntdqa (%%rsi), %%ymm0\n\t"
> > > +     "vmovntdqa 32(%%rsi), %%ymm1\n\t"
> > > +     "vmovdqu  %%ymm0, (%%rdi)\n\t"
> > > +     "vmovdqu  %%ymm1, 32(%%rdi)\n\t"
> > > +     "addq   $64, %%rsi\n\t"
> > > +     "addq   $64, %%rdi\n\t"
> > > +     "subq   $64, %%rdx\n\t"
> > > +     "jz     %l[done]\n\t"
> > > +     "203:\n\t"
> > > +     "cmpq   $32, %%rdx\n\t"
> > > +     "jb     204f\n\t"
> > > +     "vmovntdqa (%%rsi), %%ymm0\n\t"
> > > +     "vmovdqu  %%ymm0, (%%rdi)\n\t"
> > > +     "addq   $32, %%rsi\n\t"
> > > +     "addq   $32, %%rdi\n\t"
> > > +     "subq   $32, %%rdx\n\t"
> > > +     "jz     %l[done]\n\t"
> > > +     "204:\n\t"
> > > +     "cmpb   $16, %%dl\n\t"
> > > +     "jb     205f\n\t"
> > > +     "vmovntdqa (%%rsi), %%xmm0\n\t"
> > > +     "vmovdqu  %%xmm0, (%%rdi)\n\t"
> > > +     "addq   $16, %%rsi\n\t"
> > > +     "addq   $16, %%rdi\n\t"
> > > +     "subq   $16, %%rdx\n\t"
> > > +     "jz     %l[done]\n\t"
> > > +     "205:\n\t"
> > > +     "cmpb   $2, %%dl\n\t"
> > > +     "jb     208f\n\t"
> > > +     "cmpb   $4, %%dl\n\t"
> > > +     "jbe    207f\n\t"
> > > +     "cmpb   $8, %%dl\n\t"
> > > +     "jbe    206f\n\t"
> > > +     "movq   -8(%%rsi,%%rdx), %%rcx\n\t"
> > > +     "movq   (%%rsi), %%rsi\n\t"
> > > +     "movq   %%rcx, -8(%%rdi,%%rdx)\n\t"
> > > +     "movq   %%rsi, (%%rdi)\n\t"
> > > +     "jmp    %l[done]\n\t"
> > > +     "206:\n\t"
> > > +     "movl   -4(%%rsi,%%rdx), %%ecx\n\t"
> > > +     "movl   (%%rsi), %%esi\n\t"
> > > +     "movl   %%ecx, -4(%%rdi,%%rdx)\n\t"
> > > +     "movl   %%esi, (%%rdi)\n\t"
> > > +     "jmp    %l[done]\n\t"
> > > +     "207:\n\t"
> > > +     "movzwl -2(%%rsi,%%rdx), %%ecx\n\t"
> > > +     "movzwl (%%rsi), %%esi\n\t"
> > > +     "movw   %%cx, -2(%%rdi,%%rdx)\n\t"
> > > +     "movw   %%si, (%%rdi)\n\t"
> > > +     "jmp    %l[done]\n\t"
> > > +     "208:\n\t"
> > > +     "movzbl (%%rsi), %%ecx\n\t"
> > > +     "movb   %%cl, (%%rdi)"
> > > +     :
> > > +     : "r"(src), "r"(dst), "r"(size)
> > > +     : "rcx", "rdx", "rsi", "rdi", "ymm0", "ymm1", "ymm2", "ymm3", "memory"
> > > +     : done
> > > +     );
> > > +done:
> > > +     return dst;
> > > +}
> > > +
> > > +static __rte_always_inline void * rte_memcpy_generic(void *dst,
> > > +const void *src, size_t len) {
> > > +     asm goto("movq  %0, %%rsi\n\t"
> > > +     "movq   %1, %%rdi\n\t"
> > > +     "movq   %2, %%rdx\n\t"
> > > +     "movq    %%rdi, %%rax\n\t"
> > > +     "cmp     $32, %%rdx\n\t"
> > > +     "jb      101f\n\t"
> > > +     "cmp     $(32 * 2), %%rdx\n\t"
> > > +     "ja      108f\n\t"
> > > +     "vmovdqu   (%%rsi), %%ymm0\n\t"
> > > +     "vmovdqu   -32(%%rsi,%%rdx), %%ymm1\n\t"
> > > +     "vmovdqu   %%ymm0, (%%rdi)\n\t"
> > > +     "vmovdqu   %%ymm1, -32(%%rdi,%%rdx)\n\t"
> > > +     "vzeroupper\n\t"
> > > +     "jmp %l[done]\n\t"
> > > +     "101:\n\t"
> > > +     /* Less than 1 VEC.  */
> > > +     "cmpb    $32, %%dl\n\t"
> > > +     "jae     103f\n\t"
> > > +     "cmpb    $16, %%dl\n\t"
> > > +     "jae     104f\n\t"
> > > +     "cmpb    $8, %%dl\n\t"
> > > +     "jae     105f\n\t"
> > > +     "cmpb    $4, %%dl\n\t"
> > > +     "jae     106f\n\t"
> > > +     "cmpb    $1, %%dl\n\t"
> > > +     "ja      107f\n\t"
> > > +     "jb      102f\n\t"
> > > +     "movzbl  (%%rsi), %%ecx\n\t"
> > > +     "movb    %%cl, (%%rdi)\n\t"
> > > +     "102:\n\t"
> > > +     "jmp %l[done]\n\t"
> > > +     "103:\n\t"
> > > +     /* From 32 to 63.  No branch when size == 32.  */
> > > +     "vmovdqu (%%rsi), %%ymm0\n\t"
> > > +     "vmovdqu -32(%%rsi,%%rdx), %%ymm1\n\t"
> > > +     "vmovdqu %%ymm0, (%%rdi)\n\t"
> > > +     "vmovdqu %%ymm1, -32(%%rdi,%%rdx)\n\t"
> > > +     "vzeroupper\n\t"
> > > +     "jmp %l[done]\n\t"
> > > +     /* From 16 to 31.  No branch when size == 16.  */
> > > +     "104:\n\t"
> > > +     "vmovdqu (%%rsi), %%xmm0\n\t"
> > > +     "vmovdqu -16(%%rsi,%%rdx), %%xmm1\n\t"
> > > +     "vmovdqu %%xmm0, (%%rdi)\n\t"
> > > +     "vmovdqu %%xmm1, -16(%%rdi,%%rdx)\n\t"
> > > +     "jmp %l[done]\n\t"
> > > +     "105:\n\t"
> > > +     /* From 8 to 15.  No branch when size == 8.  */
> > > +     "movq    -8(%%rsi,%%rdx), %%rcx\n\t"
> > > +     "movq    (%%rsi), %%rsi\n\t"
> > > +     "movq    %%rcx, -8(%%rdi,%%rdx)\n\t"
> > > +     "movq    %%rsi, (%%rdi)\n\t"
> > > +     "jmp %l[done]\n\t"
> > > +     "106:\n\t"
> > > +     /* From 4 to 7.  No branch when size == 4.  */
> > > +     "movl    -4(%%rsi,%%rdx), %%ecx\n\t"
> > > +     "movl    (%%rsi), %%esi\n\t"
> > > +     "movl    %%ecx, -4(%%rdi,%%rdx)\n\t"
> > > +     "movl    %%esi, (%%rdi)\n\t"
> > > +     "jmp %l[done]\n\t"
> > > +     "107:\n\t"
> > > +     /* From 2 to 3.  No branch when size == 2.  */
> > > +     "movzwl  -2(%%rsi,%%rdx), %%ecx\n\t"
> > > +     "movzwl  (%%rsi), %%esi\n\t"
> > > +     "movw    %%cx, -2(%%rdi,%%rdx)\n\t"
> > > +     "movw    %%si, (%%rdi)\n\t"
> > > +     "jmp %l[done]\n\t"
> > > +     "108:\n\t"
> > > +     /* More than 2 * VEC and there may be overlap between destination */
> > > +     /* and source.  */
> > > +     "cmpq    $(32 * 8), %%rdx\n\t"
> > > +     "ja      111f\n\t"
> > > +     "cmpq    $(32 * 4), %%rdx\n\t"
> > > +     "jb      109f\n\t"
> > > +     /* Copy from 4 * VEC to 8 * VEC, inclusively. */
> > > +     "vmovdqu   (%%rsi), %%ymm0\n\t"
> > > +     "vmovdqu   32(%%rsi), %%ymm1\n\t"
> > > +     "vmovdqu   (32 * 2)(%%rsi), %%ymm2\n\t"
> > > +     "vmovdqu   (32 * 3)(%%rsi), %%ymm3\n\t"
> > > +     "vmovdqu   -32(%%rsi,%%rdx), %%ymm4\n\t"
> > > +     "vmovdqu   -(32 * 2)(%%rsi,%%rdx), %%ymm5\n\t"
> > > +     "vmovdqu   -(32 * 3)(%%rsi,%%rdx), %%ymm6\n\t"
> > > +     "vmovdqu   -(32 * 4)(%%rsi,%%rdx), %%ymm7\n\t"
> > > +     "vmovdqu   %%ymm0, (%%rdi)\n\t"
> > > +     "vmovdqu   %%ymm1, 32(%%rdi)\n\t"
> > > +     "vmovdqu   %%ymm2, (32 * 2)(%%rdi)\n\t"
> > > +     "vmovdqu   %%ymm3, (32 * 3)(%%rdi)\n\t"
> > > +     "vmovdqu   %%ymm4, -32(%%rdi,%%rdx)\n\t"
> > > +     "vmovdqu   %%ymm5, -(32 * 2)(%%rdi,%%rdx)\n\t"
> > > +     "vmovdqu   %%ymm6, -(32 * 3)(%%rdi,%%rdx)\n\t"
> > > +     "vmovdqu   %%ymm7, -(32 * 4)(%%rdi,%%rdx)\n\t"
> > > +     "vzeroupper\n\t"
> > > +     "jmp %l[done]\n\t"
> > > +     "109:\n\t"
> > > +     /* Copy from 2 * VEC to 4 * VEC. */
> > > +     "vmovdqu   (%%rsi), %%ymm0\n\t"
> > > +     "vmovdqu   32(%%rsi), %%ymm1\n\t"
> > > +     "vmovdqu   -32(%%rsi,%%rdx), %%ymm2\n\t"
> > > +     "vmovdqu   -(32 * 2)(%%rsi,%%rdx), %%ymm3\n\t"
> > > +     "vmovdqu   %%ymm0, (%%rdi)\n\t"
> > > +     "vmovdqu   %%ymm1, 32(%%rdi)\n\t"
> > > +     "vmovdqu   %%ymm2, -32(%%rdi,%%rdx)\n\t"
> > > +     "vmovdqu   %%ymm3, -(32 * 2)(%%rdi,%%rdx)\n\t"
> > > +     "vzeroupper\n\t"
> > > +     "110:\n\t"
> > > +     "jmp %l[done]\n\t"
> > > +     "111:\n\t"
> > > +     "cmpq    %%rsi, %%rdi\n\t"
> > > +     "ja      113f\n\t"
> > > +     /* Source == destination is less common.  */
> > > +     "je      110b\n\t"
> > > +     /* Load the first VEC and last 4 * VEC to
> > > +      * support overlapping addresses.
> > > +      */
> > > +     "vmovdqu   (%%rsi), %%ymm4\n\t"
> > > +     "vmovdqu   -32(%%rsi, %%rdx), %%ymm5\n\t"
> > > +     "vmovdqu   -(32 * 2)(%%rsi, %%rdx), %%ymm6\n\t"
> > > +     "vmovdqu   -(32 * 3)(%%rsi, %%rdx), %%ymm7\n\t"
> > > +     "vmovdqu   -(32 * 4)(%%rsi, %%rdx), %%ymm8\n\t"
> > > +     /* Save start and stop of the destination buffer.  */
> > > +     "movq    %%rdi, %%r11\n\t"
> > > +     "leaq    -32(%%rdi, %%rdx), %%rcx\n\t"
> > > +     /* Align destination for aligned stores in the loop.  Compute */
> > > +     /* how much destination is misaligned.  */
> > > +     "movq    %%rdi, %%r8\n\t"
> > > +     "andq    $(32 - 1), %%r8\n\t"
> > > +     /* Get the negative of offset for alignment.  */
> > > +     "subq    $32, %%r8\n\t"
> > > +     /* Adjust source.  */
> > > +     "subq    %%r8, %%rsi\n\t"
> > > +     /* Adjust destination which should be aligned now.  */
> > > +     "subq    %%r8, %%rdi\n\t"
> > > +     /* Adjust length.  */
> > > +     "addq    %%r8, %%rdx\n\t"
> > > +     /* Check non-temporal store threshold.  */
> > > +     "cmpq    $(1024*1024), %%rdx\n\t"
> > > +     "ja      115f\n\t"
> > > +     "112:\n\t"
> > > +     /* Copy 4 * VEC a time forward.  */
> > > +     "vmovdqu   (%%rsi), %%ymm0\n\t"
> > > +     "vmovdqu   32(%%rsi), %%ymm1\n\t"
> > > +     "vmovdqu   (32 * 2)(%%rsi), %%ymm2\n\t"
> > > +     "vmovdqu   (32 * 3)(%%rsi), %%ymm3\n\t"
> > > +     "addq    $(32 * 4), %%rsi\n\t"
> > > +     "subq    $(32 * 4), %%rdx\n\t"
> > > +     "vmovdqa   %%ymm0, (%%rdi)\n\t"
> > > +     "vmovdqa   %%ymm1, 32(%%rdi)\n\t"
> > > +     "vmovdqa   %%ymm2, (32 * 2)(%%rdi)\n\t"
> > > +     "vmovdqa   %%ymm3, (32 * 3)(%%rdi)\n\t"
> > > +     "addq    $(32 * 4), %%rdi\n\t"
> > > +     "cmpq    $(32 * 4), %%rdx\n\t"
> > > +     "ja      112b\n\t"
> > > +     /* Store the last 4 * VEC.  */
> > > +     "vmovdqu   %%ymm5, (%%rcx)\n\t"
> > > +     "vmovdqu   %%ymm6, -32(%%rcx)\n\t"
> > > +     "vmovdqu   %%ymm7, -(32 * 2)(%%rcx)\n\t"
> > > +     "vmovdqu   %%ymm8, -(32 * 3)(%%rcx)\n\t"
> > > +     /* Store the first VEC.  */
> > > +     "vmovdqu   %%ymm4, (%%r11)\n\t"
> > > +     "vzeroupper\n\t"
> > > +     "jmp %l[done]\n\t"
> > > +     "113:\n\t"
> > > +     /* Load the first 4*VEC and last VEC to support overlapping addresses.*/
> > > +     "vmovdqu   (%%rsi), %%ymm4\n\t"
> > > +     "vmovdqu   32(%%rsi), %%ymm5\n\t"
> > > +     "vmovdqu   (32 * 2)(%%rsi), %%ymm6\n\t"
> > > +     "vmovdqu   (32 * 3)(%%rsi), %%ymm7\n\t"
> > > +     "vmovdqu   -32(%%rsi,%%rdx), %%ymm8\n\t"
> > > +     /* Save stop of the destination buffer.  */
> > > +     "leaq    -32(%%rdi, %%rdx), %%r11\n\t"
> > > +     /* Align destination end for aligned stores in the loop.  Compute */
> > > +     /* how much destination end is misaligned.  */
> > > +     "leaq    -32(%%rsi, %%rdx), %%rcx\n\t"
> > > +     "movq    %%r11, %%r9\n\t"
> > > +     "movq    %%r11, %%r8\n\t"
> > > +     "andq    $(32 - 1), %%r8\n\t"
> > > +     /* Adjust source.  */
> > > +     "subq    %%r8, %%rcx\n\t"
> > > +     /* Adjust the end of destination which should be aligned now.  */
> > > +     "subq    %%r8, %%r9\n\t"
> > > +     /* Adjust length.  */
> > > +     "subq    %%r8, %%rdx\n\t"
> > > +      /* Check non-temporal store threshold.  */
> > > +     "cmpq    $(1024*1024), %%rdx\n\t"
> > > +     "ja      117f\n\t"
> > > +     "114:\n\t"
> > > +     /* Copy 4 * VEC a time backward.  */
> > > +     "vmovdqu   (%%rcx), %%ymm0\n\t"
> > > +     "vmovdqu   -32(%%rcx), %%ymm1\n\t"
> > > +     "vmovdqu   -(32 * 2)(%%rcx), %%ymm2\n\t"
> > > +     "vmovdqu   -(32 * 3)(%%rcx), %%ymm3\n\t"
> > > +     "subq    $(32 * 4), %%rcx\n\t"
> > > +     "subq    $(32 * 4), %%rdx\n\t"
> > > +     "vmovdqa   %%ymm0, (%%r9)\n\t"
> > > +     "vmovdqa   %%ymm1, -32(%%r9)\n\t"
> > > +     "vmovdqa   %%ymm2, -(32 * 2)(%%r9)\n\t"
> > > +     "vmovdqa   %%ymm3, -(32 * 3)(%%r9)\n\t"
> > > +     "subq    $(32 * 4), %%r9\n\t"
> > > +     "cmpq    $(32 * 4), %%rdx\n\t"
> > > +     "ja      114b\n\t"
> > > +     /* Store the first 4 * VEC. */
> > > +     "vmovdqu   %%ymm4, (%%rdi)\n\t"
> > > +     "vmovdqu   %%ymm5, 32(%%rdi)\n\t"
> > > +     "vmovdqu   %%ymm6, (32 * 2)(%%rdi)\n\t"
> > > +     "vmovdqu   %%ymm7, (32 * 3)(%%rdi)\n\t"
> > > +     /* Store the last VEC. */
> > > +     "vmovdqu   %%ymm8, (%%r11)\n\t"
> > > +     "vzeroupper\n\t"
> > > +     "jmp %l[done]\n\t"
> > > +
> > > +     "115:\n\t"
> > > +     /* Don't use non-temporal store if there is overlap between */
> > > +     /* destination and source since destination may be in cache */
> > > +     /* when source is loaded. */
> > > +     "leaq    (%%rdi, %%rdx), %%r10\n\t"
> > > +     "cmpq    %%r10, %%rsi\n\t"
> > > +     "jb      112b\n\t"
> > > +     "116:\n\t"
> > > +     /* Copy 4 * VEC a time forward with non-temporal stores.  */
> > > +     "prefetcht0 (32*4*2)(%%rsi)\n\t"
> > > +     "prefetcht0 (32*4*2 + 64)(%%rsi)\n\t"
> > > +     "prefetcht0 (32*4*3)(%%rsi)\n\t"
> > > +     "prefetcht0 (32*4*3 + 64)(%%rsi)\n\t"
> > > +     "vmovdqu   (%%rsi), %%ymm0\n\t"
> > > +     "vmovdqu   32(%%rsi), %%ymm1\n\t"
> > > +     "vmovdqu   (32 * 2)(%%rsi), %%ymm2\n\t"
> > > +     "vmovdqu   (32 * 3)(%%rsi), %%ymm3\n\t"
> > > +     "addq    $(32*4), %%rsi\n\t"
> > > +     "subq    $(32*4), %%rdx\n\t"
> > > +     "vmovntdq  %%ymm0, (%%rdi)\n\t"
> > > +     "vmovntdq  %%ymm1, 32(%%rdi)\n\t"
> > > +     "vmovntdq  %%ymm2, (32 * 2)(%%rdi)\n\t"
> > > +     "vmovntdq  %%ymm3, (32 * 3)(%%rdi)\n\t"
> > > +     "addq    $(32*4), %%rdi\n\t"
> > > +     "cmpq    $(32*4), %%rdx\n\t"
> > > +     "ja      116b\n\t"
> > > +     "sfence\n\t"
> > > +     /* Store the last 4 * VEC.  */
> > > +     "vmovdqu   %%ymm5, (%%rcx)\n\t"
> > > +     "vmovdqu   %%ymm6, -32(%%rcx)\n\t"
> > > +     "vmovdqu   %%ymm7, -(32 * 2)(%%rcx)\n\t"
> > > +     "vmovdqu   %%ymm8, -(32 * 3)(%%rcx)\n\t"
> > > +     /* Store the first VEC.  */
> > > +     "vmovdqu   %%ymm4, (%%r11)\n\t"
> > > +     "vzeroupper\n\t"
> > > +     "jmp %l[done]\n\t"
> > > +     "117:\n\t"
> > > +     /* Don't use non-temporal store if there is overlap between */
> > > +     /* destination and source since destination may be in cache */
> > > +     /* when source is loaded.  */
> > > +     "leaq    (%%rcx, %%rdx), %%r10\n\t"
> > > +     "cmpq    %%r10, %%r9\n\t"
> > > +     "jb      114b\n\t"
> > > +     "118:\n\t"
> > > +     /* Copy 4 * VEC a time backward with non-temporal stores. */
> > > +     "prefetcht0 (-32 * 4 * 2)(%%rcx)\n\t"
> > > +     "prefetcht0 (-32 * 4 * 2 - 64)(%%rcx)\n\t"
> > > +     "prefetcht0 (-32 * 4 * 3)(%%rcx)\n\t"
> > > +     "prefetcht0 (-32 * 4 * 3 - 64)(%%rcx)\n\t"
> > > +     "vmovdqu   (%%rcx), %%ymm0\n\t"
> > > +     "vmovdqu   -32(%%rcx), %%ymm1\n\t"
> > > +     "vmovdqu   -(32 * 2)(%%rcx), %%ymm2\n\t"
> > > +     "vmovdqu   -(32 * 3)(%%rcx), %%ymm3\n\t"
> > > +     "subq    $(32*4), %%rcx\n\t"
> > > +     "subq    $(32*4), %%rdx\n\t"
> > > +     "vmovntdq  %%ymm0, (%%r9)\n\t"
> > > +     "vmovntdq  %%ymm1, -32(%%r9)\n\t"
> > > +     "vmovntdq  %%ymm2, -(32 * 2)(%%r9)\n\t"
> > > +     "vmovntdq  %%ymm3, -(32 * 3)(%%r9)\n\t"
> > > +     "subq    $(32 * 4), %%r9\n\t"
> > > +     "cmpq    $(32 * 4), %%rdx\n\t"
> > > +     "ja      118b\n\t"
> > > +     "sfence\n\t"
> > > +     /* Store the first 4 * VEC.  */
> > > +     "vmovdqu   %%ymm4, (%%rdi)\n\t"
> > > +     "vmovdqu   %%ymm5, 32(%%rdi)\n\t"
> > > +     "vmovdqu   %%ymm6, (32 * 2)(%%rdi)\n\t"
> > > +     "vmovdqu   %%ymm7, (32 * 3)(%%rdi)\n\t"
> > > +     /* Store the last VEC.  */
> > > +     "vmovdqu   %%ymm8, (%%r11)\n\t"
> > > +     "vzeroupper\n\t"
> > > +     "jmp %l[done]"
> > > +     :
> > > +     : "r"(src), "r"(dst), "r"(len)
> > > +     : "rax", "rcx", "rdx", "rdi", "rsi", "r8", "r9", "r10", "r11", "r12", "ymm0",
> > > +     "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", "ymm8", "memory"
> > > +     : done
> > > +     );
> > > +done:
> > > +     return dst;
> > > +}
> >
> >
>
>

Re: [dpdk-dev] [PATCH v2 1/2] lib/eal: add amd epyc2 memcpy routine to eal

[Thomas Monjalon]

21/10/2021 21:03, Song, Keesang:
> From: Thomas Monjalon <thomas@monjalon.net>
> > 21/10/2021 20:12, Song, Keesang:
> > > From: Ananyev, Konstantin <konstantin.ananyev@intel.com>
> > > > 21/10/2021 19:10, Song, Keesang:
> > > > > 19/10/2021 17:35, Stephen Hemminger:
> > > > > > From: Thomas Monjalon <thomas@monjalon.net>
> > > > > > > 19/10/2021 12:47, Aman Kumar:
> > > > > > > > This patch provides rte_memcpy* calls optimized for AMD EPYC
> > > > > > > > platforms. Use config/x86/x86_amd_epyc_linux_gcc as cross-file
> > > > > > > > with meson to build dpdk for AMD EPYC platforms.
> > > > > > > 
> > > > > > > Please split in 2 patches: platform & memcpy.
> > > > > > > 
> > > > > > > What optimization is specific to EPYC?
> > > > > > > 
> > > > > > > I dislike the asm code below.
> > > > > > > What is AMD specific inside?
> > > > > > > Can it use compiler intrinsics as it is done elsewhere?
> > > > > > 
> > > > > > And why is this not done by Gcc?
> > > > >
> > > > > I hope this can make some explanation to your question.
> > > > > We(AMD Linux library support team) have implemented the custom
> > > > > tailored memcpy solution which is a close match with DPDK use case
> > > > > requirements like the below.
> > > > > 1)      Min 64B length data packet with cache aligned
> > > > > Source and Destination.
> > > > > 2)      Non-Temporal load and temporal store for cache aligned
> > > > > source for both RX and TX paths.
> > > > > Could not implement the non-temporal store for TX_PATH,
> > > > > as non-Temporal load/stores works only with 32B aligned addresses
> > > > > for AVX2
> > > > > 3)      This solution works for all AVX2 supported AMD machines.
> > > > > 
> > > > > Internally we have completed the integrity testing and benchmarking
> > > > > of the solution and found gains of 8.4% to 14.5% specifically on
> > > > > Milan CPU(3rd Gen of EPYC Processor)
> > > > 
> > > > It still not clear to me why it has to be written in assembler.
> > > > Why similar stuff can't be written in C with instincts, as rest of
> > > > rte_memcpy.h does?
> > > 
> > > The current memcpy implementation in Glibc is based out of assembly
> > > coding.
> > > Although memcpy could have been implemented with intrinsic,
> > > but since our AMD library developers are working on the Glibc
> > > functions, they have provided a tailored implementation based
> > > out of inline assembly coding.
> > 
> > Please convert it to C code, thanks.
> 
> I've already asked our AMD tools team, but they're saying
> they are not really familiar with C code implementation.
> We need your approval for now since we really need to get
> this patch submitted to 21.11 LTS.

Not sure it is urgent given that v2 came after the planned -rc1 date,
after 6 weeks of silence.
About the approval, there are already 3 technical board members
(Konstantin, Stephen and me) objecting against this patch.
Not being familiar with C code when working on CPU optimization
in 2021 is a strange argument.

In general, I don't really understand why we should maintain memcpy
functions in DPDK instead of relying on libc optimizations.
Having big asm code to maintain and debug is not helping.

I think this case shows that AMD needs to become more familiar
with DPDK schedule and expectations.
I would encourage you to contribute more in the project,
so such misunderstanding won't happen in future.

Hope that's all understandable


PS: discussion is more readable with replies below

Re: [dpdk-dev] [PATCH v2 1/2] lib/eal: add amd epyc2 memcpy routine to eal

[Thomas Monjalon]

19/10/2021 12:47, Aman Kumar:
> This patch provides rte_memcpy* calls optimized for
> AMD EPYC platforms. Use config/x86/x86_amd_epyc_linux_gcc
> as cross-file with meson to build dpdk for AMD EPYC platforms.
[...]
> --- a/config/x86/meson.build
> +++ b/config/x86/meson.build
> @@ -72,3 +72,10 @@ endif
>  dpdk_conf.set('RTE_CACHE_LINE_SIZE', 64)
>  dpdk_conf.set('RTE_MAX_LCORE', 128)
>  dpdk_conf.set('RTE_MAX_NUMA_NODES', 32)
> +
> +if meson.is_cross_build()
> +	if meson.get_cross_property('platform') == 'amd-epyc'
> +	    dpdk_conf.set('RTE_MAX_LCORE', 512)
> +	    dpdk_conf.set('RTE_MEMCPY_AMDEPYC', 1)
> +	endif
> +endif

Thinking again about the cross file.
Why not using the meson option "cpu_instruction_set"
to define RTE_MACHINE as "epyc" and tune other compilation options
without using artificial cross build?

Reminder, the default in config/meson.build is:
if cpu_instruction_set == 'generic'
    if host_machine.cpu_family().startswith('x86')
        cpu_instruction_set = 'corei7'

Cc Bruce who maintains this meson code.

Re: [dpdk-dev] [PATCH v2 1/2] lib/eal: add amd epyc2 memcpy routine to eal

[Bruce Richardson]

On Thu, Oct 21, 2021 at 10:14:47PM +0200, Thomas Monjalon wrote:
> 19/10/2021 12:47, Aman Kumar:
> > This patch provides rte_memcpy* calls optimized for
> > AMD EPYC platforms. Use config/x86/x86_amd_epyc_linux_gcc
> > as cross-file with meson to build dpdk for AMD EPYC platforms.
> [...]
> > --- a/config/x86/meson.build
> > +++ b/config/x86/meson.build
> > @@ -72,3 +72,10 @@ endif
> >  dpdk_conf.set('RTE_CACHE_LINE_SIZE', 64)
> >  dpdk_conf.set('RTE_MAX_LCORE', 128)
> >  dpdk_conf.set('RTE_MAX_NUMA_NODES', 32)
> > +
> > +if meson.is_cross_build()
> > +	if meson.get_cross_property('platform') == 'amd-epyc'
> > +	    dpdk_conf.set('RTE_MAX_LCORE', 512)
> > +	    dpdk_conf.set('RTE_MEMCPY_AMDEPYC', 1)
> > +	endif
> > +endif
> 
> Thinking again about the cross file.
> Why not using the meson option "cpu_instruction_set"
> to define RTE_MACHINE as "epyc" and tune other compilation options
> without using artificial cross build?
> 
> Reminder, the default in config/meson.build is:
> if cpu_instruction_set == 'generic'
>     if host_machine.cpu_family().startswith('x86')
>         cpu_instruction_set = 'corei7'
> 
> Cc Bruce who maintains this meson code.
>
Yes, that is a good suggestion. You could detect a particular instruction
set value and set additional defines based on it.

/Bruce

[dpdk-dev] [PATCH v3 1/3] config/x86: add support for AMD platform

[Aman Kumar]

-Dcpu_instruction_set=znverX meson option can be used
to build dpdk for AMD platform. Supported options are
znver1, znver2 and znver3.

Signed-off-by: Aman Kumar <aman.kumar@vvdntech.in>
---
 config/x86/meson.build | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/config/x86/meson.build b/config/x86/meson.build
index 29f3dea181..abb1bafb6e 100644
--- a/config/x86/meson.build
+++ b/config/x86/meson.build
@@ -72,3 +72,12 @@ endif
 dpdk_conf.set('RTE_CACHE_LINE_SIZE', 64)
 dpdk_conf.set('RTE_MAX_LCORE', 128)
 dpdk_conf.set('RTE_MAX_NUMA_NODES', 32)
+
+# AMD platform support
+if get_option('cpu_instruction_set') == 'znver3'
+    dpdk_conf.set('RTE_MAX_LCORE', 512)
+elif get_option('cpu_instruction_set') == 'znver2'
+    dpdk_conf.set('RTE_MAX_LCORE', 512)
+elif get_option('cpu_instruction_set') == 'znver1'
+    dpdk_conf.set('RTE_MAX_LCORE', 256)
+endif
-- 
2.25.1

[dpdk-dev] [PATCH v3 2/3] doc/guides: add dpdk build instruction for AMD platforms

[Aman Kumar]

linux guide updated with AMD platform related build
instructions.

Signed-off-by: Aman Kumar <aman.kumar@vvdntech.in>
---
 doc/guides/linux_gsg/build_dpdk.rst | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/doc/guides/linux_gsg/build_dpdk.rst b/doc/guides/linux_gsg/build_dpdk.rst
index 0b08492ca2..3110448467 100644
--- a/doc/guides/linux_gsg/build_dpdk.rst
+++ b/doc/guides/linux_gsg/build_dpdk.rst
@@ -113,6 +113,10 @@ The instruction set will be set automatically by default according to these rule
 To override what instruction set will be used, set the ``cpu_instruction_set``
 parameter to the instruction set of your choice (such as ``corei7``, ``power8``, etc.).
 
+To build dpdk for AMD Zen CPU based platforms, pass ``-Dcpu_instruction_set=<znverX>``
+to meson when configuring the build folder initially. Supported options are znver1,
+znver2 and znver3.
+
 ``cpu_instruction_set`` is not used in Arm builds, as setting the instruction set
 without other parameters leads to inferior builds. The way to tailor Arm builds
 is to build for a SoC using ``-Dplatform=<SoC>`` mentioned above.
-- 
2.25.1

Re: [dpdk-dev] [PATCH v3 2/3] doc/guides: add dpdk build instruction for AMD platforms

[Thomas Monjalon]

26/10/2021 17:56, Aman Kumar:
> linux guide updated with AMD platform related build
> instructions.
> 
> Signed-off-by: Aman Kumar <aman.kumar@vvdntech.in>
> ---
>  doc/guides/linux_gsg/build_dpdk.rst | 4 ++++
>  1 file changed, 4 insertions(+)
> 
> diff --git a/doc/guides/linux_gsg/build_dpdk.rst b/doc/guides/linux_gsg/build_dpdk.rst
> index 0b08492ca2..3110448467 100644
> --- a/doc/guides/linux_gsg/build_dpdk.rst
> +++ b/doc/guides/linux_gsg/build_dpdk.rst
> @@ -113,6 +113,10 @@ The instruction set will be set automatically by default according to these rule
>  To override what instruction set will be used, set the ``cpu_instruction_set``
>  parameter to the instruction set of your choice (such as ``corei7``, ``power8``, etc.).

All possible values are not documented here.
If you want to mention znverX, I suggest just adding znver3 after corei7 above,
and remove the lines below.
Note: such change should be squashed with the meson change.

> +To build dpdk for AMD Zen CPU based platforms, pass ``-Dcpu_instruction_set=<znverX>``

s/dpdk/DPDK/

> +to meson when configuring the build folder initially. Supported options are znver1,
> +znver2 and znver3.

You can start a new line when starting a new sentence.

If you want to document AMD platform, you can add a file in
	doc/guides/platform/
For describing tuning on Linux you can do the same as
	doc/guides/linux_gsg/nic_perf_intel_platform.rst

Re: [dpdk-dev] [PATCH v3 2/3] doc/guides: add dpdk build instruction for AMD platforms

[Aman Kumar]

On Tue, Oct 26, 2021 at 9:37 PM Thomas Monjalon <thomas@monjalon.net> wrote:

> 26/10/2021 17:56, Aman Kumar:
> > linux guide updated with AMD platform related build
> > instructions.
> >
> > Signed-off-by: Aman Kumar <aman.kumar@vvdntech.in>
> > ---
> >  doc/guides/linux_gsg/build_dpdk.rst | 4 ++++
> >  1 file changed, 4 insertions(+)
> >
> > diff --git a/doc/guides/linux_gsg/build_dpdk.rst
> b/doc/guides/linux_gsg/build_dpdk.rst
> > index 0b08492ca2..3110448467 100644
> > --- a/doc/guides/linux_gsg/build_dpdk.rst
> > +++ b/doc/guides/linux_gsg/build_dpdk.rst
> > @@ -113,6 +113,10 @@ The instruction set will be set automatically by
> default according to these rule
> >  To override what instruction set will be used, set the
> ``cpu_instruction_set``
> >  parameter to the instruction set of your choice (such as ``corei7``,
> ``power8``, etc.).
>
> All possible values are not documented here.
> If you want to mention znverX, I suggest just adding znver3 after corei7
> above,
> and remove the lines below.
> Note: such change should be squashed with the meson change.
>

Noted. Will squash this with meson changes.

>
> > +To build dpdk for AMD Zen CPU based platforms, pass
> ``-Dcpu_instruction_set=<znverX>``
>
> s/dpdk/DPDK/
>
> > +to meson when configuring the build folder initially. Supported options
> are znver1,
> > +znver2 and znver3.
>
> You can start a new line when starting a new sentence.
>
> If you want to document AMD platform, you can add a file in
>         doc/guides/platform/
> For describing tuning on Linux you can do the same as
>         doc/guides/linux_gsg/nic_perf_intel_platform.rst
>
> For now, the above entry in the build_dpdk.rst will suffice.
We will add a separate platform doc once we have more platform specific
items.

[dpdk-dev] [PATCH v3 3/3] lib/eal: add temporal store memcpy support on AMD platform

[Aman Kumar]

This patch provides a rte_memcpy* call with temporal stores.
Use -Dcpu_instruction_set=znverX with build to enable this API.

Signed-off-by: Aman Kumar <aman.kumar@vvdntech.in>
---
 config/x86/meson.build           |   2 +
 lib/eal/x86/include/rte_memcpy.h | 114 +++++++++++++++++++++++++++++++
 2 files changed, 116 insertions(+)

diff --git a/config/x86/meson.build b/config/x86/meson.build
index abb1bafb6e..7ef7f32cd4 100644
--- a/config/x86/meson.build
+++ b/config/x86/meson.build
@@ -76,8 +76,10 @@ dpdk_conf.set('RTE_MAX_NUMA_NODES', 32)
 # AMD platform support
 if get_option('cpu_instruction_set') == 'znver3'
     dpdk_conf.set('RTE_MAX_LCORE', 512)
+    dpdk_conf.set('RTE_MEMCPY_AMDEPYC', 1)
 elif get_option('cpu_instruction_set') == 'znver2'
     dpdk_conf.set('RTE_MAX_LCORE', 512)
+    dpdk_conf.set('RTE_MEMCPY_AMDEPYC', 1)
 elif get_option('cpu_instruction_set') == 'znver1'
     dpdk_conf.set('RTE_MAX_LCORE', 256)
 endif
diff --git a/lib/eal/x86/include/rte_memcpy.h b/lib/eal/x86/include/rte_memcpy.h
index 1b6c6e585f..8fe7822cb4 100644
--- a/lib/eal/x86/include/rte_memcpy.h
+++ b/lib/eal/x86/include/rte_memcpy.h
@@ -376,6 +376,120 @@ rte_mov128blocks(uint8_t *dst, const uint8_t *src, size_t n)
 	}
 }
 
+#if defined RTE_MEMCPY_AMDEPYC
+
+/**
+ * Copy 16 bytes from one location to another,
+ * with temporal stores
+ */
+static __rte_always_inline void
+rte_copy16_ts(uint8_t *dst, uint8_t *src)
+{
+	__m128i var128;
+
+	var128 = _mm_stream_load_si128((__m128i *)src);
+	_mm_storeu_si128((__m128i *)dst, var128);
+}
+
+/**
+ * Copy 32 bytes from one location to another,
+ * with temporal stores
+ */
+static __rte_always_inline void
+rte_copy32_ts(uint8_t *dst, uint8_t *src)
+{
+	__m256i ymm0;
+
+	ymm0 = _mm256_stream_load_si256((const __m256i *)src);
+	_mm256_storeu_si256((__m256i *)dst, ymm0);
+}
+
+/**
+ * Copy 64 bytes from one location to another,
+ * with temporal stores
+ */
+static __rte_always_inline void
+rte_copy64_ts(uint8_t *dst, uint8_t *src)
+{
+	rte_copy32_ts(dst + 0 * 32, src + 0 * 32);
+	rte_copy32_ts(dst + 1 * 32, src + 1 * 32);
+}
+
+/**
+ * Copy 128 bytes from one location to another,
+ * with temporal stores
+ */
+static __rte_always_inline void
+rte_copy128_ts(uint8_t *dst, uint8_t *src)
+{
+	rte_copy32_ts(dst + 0 * 32, src + 0 * 32);
+	rte_copy32_ts(dst + 1 * 32, src + 1 * 32);
+	rte_copy32_ts(dst + 2 * 32, src + 2 * 32);
+	rte_copy32_ts(dst + 3 * 32, src + 3 * 32);
+}
+
+/**
+ * Copy len bytes from one location to another,
+ * with temporal stores 16B aligned
+ */
+static __rte_always_inline void *
+rte_memcpy_aligned_tstore16_generic(void *dst, void *src, int len)
+{
+	void *dest = dst;
+
+	while (len >= 128) {
+		rte_copy128_ts((uint8_t *)dst, (uint8_t *)src);
+		dst = (uint8_t *)dst + 128;
+		src = (uint8_t *)src + 128;
+		len -= 128;
+	}
+	while (len >= 64) {
+		rte_copy64_ts((uint8_t *)dst, (uint8_t *)src);
+		dst = (uint8_t *)dst + 64;
+		src = (uint8_t *)src + 64;
+		len -= 64;
+	}
+	while (len >= 32) {
+		rte_copy32_ts((uint8_t *)dst, (uint8_t *)src);
+		dst = (uint8_t *)dst + 32;
+		src = (uint8_t *)src + 32;
+		len -= 32;
+	}
+	if (len >= 16) {
+		rte_copy16_ts((uint8_t *)dst, (uint8_t *)src);
+		dst = (uint8_t *)dst + 16;
+		src = (uint8_t *)src + 16;
+		len -= 16;
+	}
+	if (len >= 8) {
+		*(uint64_t *)dst = *(const uint64_t *)src;
+		dst = (uint8_t *)dst + 8;
+		src = (uint8_t *)src + 8;
+		len -= 8;
+	}
+	if (len >= 4) {
+		*(uint32_t *)dst = *(const uint32_t *)src;
+		dst = (uint8_t *)dst + 4;
+		src = (uint8_t *)src + 4;
+		len -= 4;
+	}
+	if (len != 0) {
+		dst = (uint8_t *)dst - (4 - len);
+		src = (uint8_t *)src - (4 - len);
+		*(uint32_t *)dst = *(const uint32_t *)src;
+	}
+
+	return dest;
+}
+
+static __rte_always_inline void *
+rte_memcpy_aligned_tstore16(void *dst, void *src, int len)
+{
+	return rte_memcpy_aligned_tstore16_generic(dst, src, len);
+}
+
+#endif /* RTE_MEMCPY_AMDEPYC */
+
 static __rte_always_inline void *
 rte_memcpy_generic(void *dst, const void *src, size_t n)
 {
-- 
2.25.1

Re: [dpdk-dev] [PATCH v3 3/3] lib/eal: add temporal store memcpy support on AMD platform

[Thomas Monjalon]

26/10/2021 17:56, Aman Kumar:
> This patch provides a rte_memcpy* call with temporal stores.
> Use -Dcpu_instruction_set=znverX with build to enable this API.
> 
> Signed-off-by: Aman Kumar <aman.kumar@vvdntech.in>
> ---
>  config/x86/meson.build           |   2 +
>  lib/eal/x86/include/rte_memcpy.h | 114 +++++++++++++++++++++++++++++++

It looks better as C code.
Do you achieve the same performance as the asm version?

> +#if defined RTE_MEMCPY_AMDEPYC
[...]
> +static __rte_always_inline void *
> +rte_memcpy_aligned_tstore16_generic(void *dst, void *src, int len)

So to be clear, an application will benefit of this optimization if
1/ DPDK is specifically compiled for AMD
2/ the application is compiled with above DPDK build (because of inlinining)

I guess there is no good way to benefit from the optimization
without specific compilation, because of inlining constraint.
Another design, with less constraint but less performance,
would be to have a function pointer assigned at runtime based on the CPU.

Re: [dpdk-dev] [PATCH v3 3/3] lib/eal: add temporal store memcpy support on AMD platform

[Aman Kumar]

On Tue, Oct 26, 2021 at 9:44 PM Thomas Monjalon <thomas@monjalon.net> wrote:

> 26/10/2021 17:56, Aman Kumar:
> > This patch provides a rte_memcpy* call with temporal stores.
> > Use -Dcpu_instruction_set=znverX with build to enable this API.
> >
> > Signed-off-by: Aman Kumar <aman.kumar@vvdntech.in>
> > ---
> >  config/x86/meson.build           |   2 +
> >  lib/eal/x86/include/rte_memcpy.h | 114 +++++++++++++++++++++++++++++++
>
> It looks better as C code.
> Do you achieve the same performance as the asm version?
>

In a few corner cases assembly performed better, but overall we have very
similar perf observations.

> > +#if defined RTE_MEMCPY_AMDEPYC
> [...]
> > +static __rte_always_inline void *
> > +rte_memcpy_aligned_tstore16_generic(void *dst, void *src, int len)
>
> So to be clear, an application will benefit of this optimization if
> 1/ DPDK is specifically compiled for AMD
> 2/ the application is compiled with above DPDK build (because of
> inlinining)
>
> I guess there is no good way to benefit from the optimization
> without specific compilation, because of inlining constraint.
> Another design, with less constraint but less performance,
> would be to have a function pointer assigned at runtime based on the CPU.
>

You're right. We need to build DPDK and apps with this flag enabled to get
the benefit.
In future versions, we will try to adapt in a more dynamic way. Thanks.

Re: [dpdk-dev] [PATCH v3 3/3] lib/eal: add temporal store memcpy support on AMD platform

[Thomas Monjalon]

27/10/2021 08:34, Aman Kumar:
> On Tue, Oct 26, 2021 at 9:44 PM Thomas Monjalon <thomas@monjalon.net> wrote:
> 
> > 26/10/2021 17:56, Aman Kumar:
> > > This patch provides a rte_memcpy* call with temporal stores.
> > > Use -Dcpu_instruction_set=znverX with build to enable this API.
> > >
> > > Signed-off-by: Aman Kumar <aman.kumar@vvdntech.in>
> > > ---
> > >  config/x86/meson.build           |   2 +
> > >  lib/eal/x86/include/rte_memcpy.h | 114 +++++++++++++++++++++++++++++++
> >
> > It looks better as C code.
> > Do you achieve the same performance as the asm version?
> >
> 
> In a few corner cases assembly performed better, but overall we have very
> similar perf observations.
> 
> > > +#if defined RTE_MEMCPY_AMDEPYC
> > [...]
> > > +static __rte_always_inline void *
> > > +rte_memcpy_aligned_tstore16_generic(void *dst, void *src, int len)
> >
> > So to be clear, an application will benefit of this optimization if
> > 1/ DPDK is specifically compiled for AMD
> > 2/ the application is compiled with above DPDK build (because of
> > inlinining)
> >
> > I guess there is no good way to benefit from the optimization
> > without specific compilation, because of inlining constraint.
> > Another design, with less constraint but less performance,
> > would be to have a function pointer assigned at runtime based on the CPU.
> >
> 
> You're right. We need to build DPDK and apps with this flag enabled to get
> the benefit.

So the x86 packages, as in Linux distributions, won't have this optimization.

> In future versions, we will try to adapt in a more dynamic way. Thanks.

No, I was trying to say that unfortunately there is probably no solution.

Re: [dpdk-dev] [PATCH v3 3/3] lib/eal: add temporal store memcpy support on AMD platform

[Stephen Hemminger]

On Tue, 26 Oct 2021 21:26:45 +0530
Aman Kumar <aman.kumar@vvdntech.in> wrote:

> This patch provides a rte_memcpy* call with temporal stores.
> Use -Dcpu_instruction_set=znverX with build to enable this API.
> 
> Signed-off-by: Aman Kumar <aman.kumar@vvdntech.in>

Ok, but would be better to get it into glibc.
Would benefit wider array of platforms and get more testing.

Re: [dpdk-dev] [PATCH v3 3/3] lib/eal: add temporal store memcpy support on AMD platform

[Aman Kumar]

On Wed, Oct 27, 2021 at 2:41 AM Stephen Hemminger <
stephen@networkplumber.org> wrote:

> On Tue, 26 Oct 2021 21:26:45 +0530
> Aman Kumar <aman.kumar@vvdntech.in> wrote:
>
> > This patch provides a rte_memcpy* call with temporal stores.
> > Use -Dcpu_instruction_set=znverX with build to enable this API.
> >
> > Signed-off-by: Aman Kumar <aman.kumar@vvdntech.in>
>
> Ok, but would be better to get it into glibc.
> Would benefit wider array of platforms and get more testing.
>

Yes, we've considered this. This may go into glibc in future.

Re: [dpdk-dev] [PATCH v3 1/3] config/x86: add support for AMD platform

[Thomas Monjalon]

26/10/2021 17:56, Aman Kumar:
> -Dcpu_instruction_set=znverX meson option can be used
> to build dpdk for AMD platform. Supported options are
> znver1, znver2 and znver3.

OK that approach looks good.

> +# AMD platform support
> +if get_option('cpu_instruction_set') == 'znver3'
> +    dpdk_conf.set('RTE_MAX_LCORE', 512)
> +elif get_option('cpu_instruction_set') == 'znver2'
> +    dpdk_conf.set('RTE_MAX_LCORE', 512)
> +elif get_option('cpu_instruction_set') == 'znver1'
> +    dpdk_conf.set('RTE_MAX_LCORE', 256)
> +endif

Maybe sort it in the reverse order, starting with 1?

Re: [dpdk-dev] [PATCH v3 1/3] config/x86: add support for AMD platform

[Aman Kumar]

On Tue, Oct 26, 2021 at 9:31 PM Thomas Monjalon <thomas@monjalon.net> wrote:

> 26/10/2021 17:56, Aman Kumar:
> > -Dcpu_instruction_set=znverX meson option can be used
> > to build dpdk for AMD platform. Supported options are
> > znver1, znver2 and znver3.
>
> OK that approach looks good.
>
> > +# AMD platform support
> > +if get_option('cpu_instruction_set') == 'znver3'
> > +    dpdk_conf.set('RTE_MAX_LCORE', 512)
> > +elif get_option('cpu_instruction_set') == 'znver2'
> > +    dpdk_conf.set('RTE_MAX_LCORE', 512)
> > +elif get_option('cpu_instruction_set') == 'znver1'
> > +    dpdk_conf.set('RTE_MAX_LCORE', 256)
> > +endif
>
> Maybe sort it in the reverse order, starting with 1?
>
> OK. Will update this.

[dpdk-dev] [PATCH v4 1/2] config/x86: add support for AMD platform

[Aman Kumar]

-Dcpu_instruction_set=znverX meson option can be used
to build dpdk for AMD platforms. Supported options are
znver1, znver2 and znver3.

Signed-off-by: Aman Kumar <aman.kumar@vvdntech.in>
---
 config/x86/meson.build              | 9 +++++++++
 doc/guides/linux_gsg/build_dpdk.rst | 2 +-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/config/x86/meson.build b/config/x86/meson.build
index 29f3dea181..21cda6fd33 100644
--- a/config/x86/meson.build
+++ b/config/x86/meson.build
@@ -72,3 +72,12 @@ endif
 dpdk_conf.set('RTE_CACHE_LINE_SIZE', 64)
 dpdk_conf.set('RTE_MAX_LCORE', 128)
 dpdk_conf.set('RTE_MAX_NUMA_NODES', 32)
+
+# AMD platform support
+if get_option('cpu_instruction_set') == 'znver1'
+    dpdk_conf.set('RTE_MAX_LCORE', 256)
+elif get_option('cpu_instruction_set') == 'znver2'
+    dpdk_conf.set('RTE_MAX_LCORE', 512)
+elif get_option('cpu_instruction_set') == 'znver3'
+    dpdk_conf.set('RTE_MAX_LCORE', 512)
+endif
diff --git a/doc/guides/linux_gsg/build_dpdk.rst b/doc/guides/linux_gsg/build_dpdk.rst
index 0b08492ca2..e224a06cbd 100644
--- a/doc/guides/linux_gsg/build_dpdk.rst
+++ b/doc/guides/linux_gsg/build_dpdk.rst
@@ -111,7 +111,7 @@ The instruction set will be set automatically by default according to these rule
   a common minimal baseline needed for DPDK.
 
 To override what instruction set will be used, set the ``cpu_instruction_set``
-parameter to the instruction set of your choice (such as ``corei7``, ``power8``, etc.).
+parameter to the instruction set of your choice (such as ``corei7``, ``power8``, ``znver3``, etc.).
 
 ``cpu_instruction_set`` is not used in Arm builds, as setting the instruction set
 without other parameters leads to inferior builds. The way to tailor Arm builds
-- 
2.25.1

[dpdk-dev] [PATCH v4 2/2] lib/eal: add temporal store memcpy support for AMD platform

[Aman Kumar]

This patch provides a rte_memcpy* call with temporal stores.
Use -Dcpu_instruction_set=znverX with build to enable this API.

Signed-off-by: Aman Kumar <aman.kumar@vvdntech.in>
---
 config/x86/meson.build           |   2 +
 lib/eal/x86/include/rte_memcpy.h | 114 +++++++++++++++++++++++++++++++
 2 files changed, 116 insertions(+)

diff --git a/config/x86/meson.build b/config/x86/meson.build
index 21cda6fd33..56dae4aca7 100644
--- a/config/x86/meson.build
+++ b/config/x86/meson.build
@@ -78,6 +78,8 @@ if get_option('cpu_instruction_set') == 'znver1'
     dpdk_conf.set('RTE_MAX_LCORE', 256)
 elif get_option('cpu_instruction_set') == 'znver2'
     dpdk_conf.set('RTE_MAX_LCORE', 512)
+    dpdk_conf.set('RTE_MEMCPY_AMDEPYC', 1)
 elif get_option('cpu_instruction_set') == 'znver3'
     dpdk_conf.set('RTE_MAX_LCORE', 512)
+    dpdk_conf.set('RTE_MEMCPY_AMDEPYC', 1)
 endif
diff --git a/lib/eal/x86/include/rte_memcpy.h b/lib/eal/x86/include/rte_memcpy.h
index 1b6c6e585f..8fe7822cb4 100644
--- a/lib/eal/x86/include/rte_memcpy.h
+++ b/lib/eal/x86/include/rte_memcpy.h
@@ -376,6 +376,120 @@ rte_mov128blocks(uint8_t *dst, const uint8_t *src, size_t n)
 	}
 }
 
+#if defined RTE_MEMCPY_AMDEPYC
+
+/**
+ * Copy 16 bytes from one location to another,
+ * with temporal stores
+ */
+static __rte_always_inline void
+rte_copy16_ts(uint8_t *dst, uint8_t *src)
+{
+	__m128i var128;
+
+	var128 = _mm_stream_load_si128((__m128i *)src);
+	_mm_storeu_si128((__m128i *)dst, var128);
+}
+
+/**
+ * Copy 32 bytes from one location to another,
+ * with temporal stores
+ */
+static __rte_always_inline void
+rte_copy32_ts(uint8_t *dst, uint8_t *src)
+{
+	__m256i ymm0;
+
+	ymm0 = _mm256_stream_load_si256((const __m256i *)src);
+	_mm256_storeu_si256((__m256i *)dst, ymm0);
+}
+
+/**
+ * Copy 64 bytes from one location to another,
+ * with temporal stores
+ */
+static __rte_always_inline void
+rte_copy64_ts(uint8_t *dst, uint8_t *src)
+{
+	rte_copy32_ts(dst + 0 * 32, src + 0 * 32);
+	rte_copy32_ts(dst + 1 * 32, src + 1 * 32);
+}
+
+/**
+ * Copy 128 bytes from one location to another,
+ * with temporal stores
+ */
+static __rte_always_inline void
+rte_copy128_ts(uint8_t *dst, uint8_t *src)
+{
+	rte_copy32_ts(dst + 0 * 32, src + 0 * 32);
+	rte_copy32_ts(dst + 1 * 32, src + 1 * 32);
+	rte_copy32_ts(dst + 2 * 32, src + 2 * 32);
+	rte_copy32_ts(dst + 3 * 32, src + 3 * 32);
+}
+
+/**
+ * Copy len bytes from one location to another,
+ * with temporal stores 16B aligned
+ */
+static __rte_always_inline void *
+rte_memcpy_aligned_tstore16_generic(void *dst, void *src, int len)
+{
+	void *dest = dst;
+
+	while (len >= 128) {
+		rte_copy128_ts((uint8_t *)dst, (uint8_t *)src);
+		dst = (uint8_t *)dst + 128;
+		src = (uint8_t *)src + 128;
+		len -= 128;
+	}
+	while (len >= 64) {
+		rte_copy64_ts((uint8_t *)dst, (uint8_t *)src);
+		dst = (uint8_t *)dst + 64;
+		src = (uint8_t *)src + 64;
+		len -= 64;
+	}
+	while (len >= 32) {
+		rte_copy32_ts((uint8_t *)dst, (uint8_t *)src);
+		dst = (uint8_t *)dst + 32;
+		src = (uint8_t *)src + 32;
+		len -= 32;
+	}
+	if (len >= 16) {
+		rte_copy16_ts((uint8_t *)dst, (uint8_t *)src);
+		dst = (uint8_t *)dst + 16;
+		src = (uint8_t *)src + 16;
+		len -= 16;
+	}
+	if (len >= 8) {
+		*(uint64_t *)dst = *(const uint64_t *)src;
+		dst = (uint8_t *)dst + 8;
+		src = (uint8_t *)src + 8;
+		len -= 8;
+	}
+	if (len >= 4) {
+		*(uint32_t *)dst = *(const uint32_t *)src;
+		dst = (uint8_t *)dst + 4;
+		src = (uint8_t *)src + 4;
+		len -= 4;
+	}
+	if (len != 0) {
+		dst = (uint8_t *)dst - (4 - len);
+		src = (uint8_t *)src - (4 - len);
+		*(uint32_t *)dst = *(const uint32_t *)src;
+	}
+
+	return dest;
+}
+
+static __rte_always_inline void *
+rte_memcpy_aligned_tstore16(void *dst, void *src, int len)
+{
+	return rte_memcpy_aligned_tstore16_generic(dst, src, len);
+}
+
+#endif /* RTE_MEMCPY_AMDEPYC */
+
 static __rte_always_inline void *
 rte_memcpy_generic(void *dst, const void *src, size_t n)
 {
-- 
2.25.1

Re: [dpdk-dev] [PATCH v4 2/2] lib/eal: add temporal store memcpy support for AMD platform

[Thomas Monjalon]

27/10/2021 09:28, Aman Kumar:
> This patch provides a rte_memcpy* call with temporal stores.
> Use -Dcpu_instruction_set=znverX with build to enable this API.
> 
> Signed-off-by: Aman Kumar <aman.kumar@vvdntech.in>

For the series, Acked-by: Thomas Monjalon <thomas@monjalon.net>
With the hope that such optimization will go in libc in a near future.

If there is no objection, I will merge this AMD-specific series in 21.11-rc2.
It should not affect other platforms.

Re: [dpdk-dev] [PATCH v4 2/2] lib/eal: add temporal store memcpy support for AMD platform

[Van Haaren, Harry]

> -----Original Message-----
> From: dev <dev-bounces@dpdk.org> On Behalf Of Thomas Monjalon
> Sent: Wednesday, October 27, 2021 9:13 AM
> To: Aman Kumar <aman.kumar@vvdntech.in>
> Cc: dev@dpdk.org; viacheslavo@nvidia.com; Burakov, Anatoly
> <anatoly.burakov@intel.com>; keesang.song@amd.com;
> aman.kumar@vvdntech.in; jerinjacobk@gmail.com; Ananyev, Konstantin
> <konstantin.ananyev@intel.com>; Richardson, Bruce
> <bruce.richardson@intel.com>; honnappa.nagarahalli@arm.com; Ruifeng Wang
> <ruifeng.wang@arm.com>; David Christensen <drc@linux.vnet.ibm.com>;
> david.marchand@redhat.com; stephen@networkplumber.org
> Subject: Re: [dpdk-dev] [PATCH v4 2/2] lib/eal: add temporal store memcpy
> support for AMD platform
> 
> 27/10/2021 09:28, Aman Kumar:
> > This patch provides a rte_memcpy* call with temporal stores.
> > Use -Dcpu_instruction_set=znverX with build to enable this API.
> >
> > Signed-off-by: Aman Kumar <aman.kumar@vvdntech.in>
> 
> For the series, Acked-by: Thomas Monjalon <thomas@monjalon.net>
> With the hope that such optimization will go in libc in a near future.
> 
> If there is no objection, I will merge this AMD-specific series in 21.11-rc2.
> It should not affect other platforms.

Hi Folks,

This patchset was brought to my attention, and I have a few concerns.
I'll add short snippets of context from the patch here so I can refer to it below;

+/**
+ * Copy 16 bytes from one location to another,
+ * with temporal stores
+ */
+static __rte_always_inline void
+rte_copy16_ts(uint8_t *dst, uint8_t *src)
+{
+	__m128i var128;
+
+	var128 = _mm_stream_load_si128((__m128i *)src);
+	_mm_storeu_si128((__m128i *)dst, var128);
+}

1) What is fundamentally specific to the znverX CPU? Is there any reason this can not just be enabled for x86-64 generic with SSE4.1 ISA requirements?
_mm_stream_load_si128() is part of SSE4.1
_mm_storeu_si128() is SSE2. 
Using the intrinsics guide for lookup of intrinsics to ISA level: https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html?wapkw=intrinsics%20guide#text=_mm_stream_load&ig_expand=6884

2) Are -D options allowed to change/break API/ABI?
By allowing -Dcpu_instruction_set= to change available functions, any application using it is no longer source-code (API) compatible with "DPDK" proper.
This patch essentially splits a "DPDK" app to depend on "DPDK + CPU version -D flag", in an incompatible way (no fallback?).

3) The stream load instruction used here *requires* 16-byte alignment for its operand.
This is not documented, and worse, a uint8_t* is accepted, which is cast to (__m128i *).
This cast hides the compiler warning for expanding type-alignments.
And the code itself is broken - passing a "src" parameter that is not 16-byte aligned will segfault.

4) Temporal and Non-temporal are not logically presented here.
Temporal loads/stores are normal loads/stores. They use the L1/L2 caches.
Non-temporal loads/stores indicate that the data will *not* be used again in a short space of time.
Non-temporal means "having no relation to time" according to my internet search.

5) The *store* here uses a normal store (temporal, targets cache). The *load* however is a streaming (non-temporal, no cache) load.
It is not clearly documented that A) stream load will be used.
The inverse is documented "copy with ts" aka, copy with temporal store.
Is documenting the store as temporal meant to imply that the load is non-temporal?

6) What is the use-case for this? When would a user *want* to use this instead of rte_memcpy()?
If the data being loaded is relevant to datapath/packets, presumably other packets might require the
loaded data, so temporal (normal) loads should be used to cache the source data?

7) Why is streaming (non-temporal) loads & stores not used? I guess maybe this is regarding the use-case,
but its not clear to me right now why loads are NT, and stores are T.

All in all, I do not think merging this patch is a good idea. I would like to understand the motivation for adding
this type of function, and then see it being done in a way that is clearly documented regarding temporal loads/stores,
and not changing/adding APIs for specific CPUs.

So apologies for late feedback, but this is not of high enough quality to be merged to DPDK right now, NACK.

Re: [dpdk-dev] [PATCH v4 2/2] lib/eal: add temporal store memcpy support for AMD platform

[Mattias Rönnblom]

On 2021-10-27 13:03, Van Haaren, Harry wrote:
>> -----Original Message-----
>> From: dev <dev-bounces@dpdk.org> On Behalf Of Thomas Monjalon
>> Sent: Wednesday, October 27, 2021 9:13 AM
>> To: Aman Kumar <aman.kumar@vvdntech.in>
>> Cc: dev@dpdk.org; viacheslavo@nvidia.com; Burakov, Anatoly
>> <anatoly.burakov@intel.com>; keesang.song@amd.com;
>> aman.kumar@vvdntech.in; jerinjacobk@gmail.com; Ananyev, Konstantin
>> <konstantin.ananyev@intel.com>; Richardson, Bruce
>> <bruce.richardson@intel.com>; honnappa.nagarahalli@arm.com; Ruifeng Wang
>> <ruifeng.wang@arm.com>; David Christensen <drc@linux.vnet.ibm.com>;
>> david.marchand@redhat.com; stephen@networkplumber.org
>> Subject: Re: [dpdk-dev] [PATCH v4 2/2] lib/eal: add temporal store memcpy
>> support for AMD platform
>>
>> 27/10/2021 09:28, Aman Kumar:
>>> This patch provides a rte_memcpy* call with temporal stores.
>>> Use -Dcpu_instruction_set=znverX with build to enable this API.
>>>
>>> Signed-off-by: Aman Kumar <aman.kumar@vvdntech.in>
>> For the series, Acked-by: Thomas Monjalon <thomas@monjalon.net>
>> With the hope that such optimization will go in libc in a near future.
>>
>> If there is no objection, I will merge this AMD-specific series in 21.11-rc2.
>> It should not affect other platforms.
> Hi Folks,
>
> This patchset was brought to my attention, and I have a few concerns.
> I'll add short snippets of context from the patch here so I can refer to it below;
>
> +/**
> + * Copy 16 bytes from one location to another,
> + * with temporal stores
> + */
> +static __rte_always_inline void
> +rte_copy16_ts(uint8_t *dst, uint8_t *src)
> +{
> +	__m128i var128;
> +
> +	var128 = _mm_stream_load_si128((__m128i *)src);
> +	_mm_storeu_si128((__m128i *)dst, var128);
> +}
>
> 1) What is fundamentally specific to the znverX CPU? Is there any reason this can not just be enabled for x86-64 generic with SSE4.1 ISA requirements?
> _mm_stream_load_si128() is part of SSE4.1
> _mm_storeu_si128() is SSE2.
> Using the intrinsics guide for lookup of intrinsics to ISA level: https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html?wapkw=intrinsics%20guide#text=_mm_stream_load&ig_expand=6884
>
> 2) Are -D options allowed to change/break API/ABI?
> By allowing -Dcpu_instruction_set= to change available functions, any application using it is no longer source-code (API) compatible with "DPDK" proper.
> This patch essentially splits a "DPDK" app to depend on "DPDK + CPU version -D flag", in an incompatible way (no fallback?).
>
> 3) The stream load instruction used here *requires* 16-byte alignment for its operand.
> This is not documented, and worse, a uint8_t* is accepted, which is cast to (__m128i *).
> This cast hides the compiler warning for expanding type-alignments.
> And the code itself is broken - passing a "src" parameter that is not 16-byte aligned will segfault.
>
> 4) Temporal and Non-temporal are not logically presented here.
> Temporal loads/stores are normal loads/stores. They use the L1/L2 caches.
> Non-temporal loads/stores indicate that the data will *not* be used again in a short space of time.
> Non-temporal means "having no relation to time" according to my internet search.
>
> 5) The *store* here uses a normal store (temporal, targets cache). The *load* however is a streaming (non-temporal, no cache) load.
> It is not clearly documented that A) stream load will be used.
> The inverse is documented "copy with ts" aka, copy with temporal store.
> Is documenting the store as temporal meant to imply that the load is non-temporal?
>
> 6) What is the use-case for this? When would a user *want* to use this instead of rte_memcpy()?
> If the data being loaded is relevant to datapath/packets, presumably other packets might require the
> loaded data, so temporal (normal) loads should be used to cache the source data?


I'm not sure if your first question is rhetorical or not, but a memcpy() 
in a NT variant is certainly useful. One use case for a memcpy() with 
temporal loads and non-temporal stores is if you need to archive packet 
payload for (distant, potential) future use, and want to avoid causing 
unnecessary LLC evictions while doing so.


> 7) Why is streaming (non-temporal) loads & stores not used? I guess maybe this is regarding the use-case,
> but its not clear to me right now why loads are NT, and stores are T.
>
> All in all, I do not think merging this patch is a good idea. I would like to understand the motivation for adding
> this type of function, and then see it being done in a way that is clearly documented regarding temporal loads/stores,
> and not changing/adding APIs for specific CPUs.
>
> So apologies for late feedback, but this is not of high enough quality to be merged to DPDK right now, NACK.

Re: [dpdk-dev] [PATCH v4 2/2] lib/eal: add temporal store memcpy support for AMD platform

[Van Haaren, Harry]

> -----Original Message-----
> From: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
> Sent: Wednesday, October 27, 2021 12:42 PM
> To: Van Haaren, Harry <harry.van.haaren@intel.com>; Thomas Monjalon
> <thomas@monjalon.net>; Aman Kumar <aman.kumar@vvdntech.in>
> Cc: dev@dpdk.org; viacheslavo@nvidia.com; Burakov, Anatoly
> <anatoly.burakov@intel.com>; Song, Keesang <Keesang.Song@amd.com>;
> jerinjacobk@gmail.com; Ananyev, Konstantin <konstantin.ananyev@intel.com>;
> Richardson, Bruce <bruce.richardson@intel.com>;
> honnappa.nagarahalli@arm.com; Ruifeng Wang <ruifeng.wang@arm.com>;
> David Christensen <drc@linux.vnet.ibm.com>; david.marchand@redhat.com;
> stephen@networkplumber.org
> Subject: Re: [dpdk-dev] [PATCH v4 2/2] lib/eal: add temporal store memcpy
> support for AMD platform
> 
> On 2021-10-27 13:03, Van Haaren, Harry wrote:
> >> -----Original Message-----

<snip>

Hi Mattias,

> > 6) What is the use-case for this? When would a user *want* to use this instead
> of rte_memcpy()?
> > If the data being loaded is relevant to datapath/packets, presumably other
> packets might require the
> > loaded data, so temporal (normal) loads should be used to cache the source
> data?
> 
> 
> I'm not sure if your first question is rhetorical or not, but a memcpy()
> in a NT variant is certainly useful. One use case for a memcpy() with
> temporal loads and non-temporal stores is if you need to archive packet
> payload for (distant, potential) future use, and want to avoid causing
> unnecessary LLC evictions while doing so.

Yes I agree that there are certainly benefits in using cache-locality hints.
There is an open question around if the src or dst or both are non-temporal.

In the implementation of this patch, the NT/T type of store is reversed from your use-case:
1) Loads are NT (so loaded data is not cached for future packets)
2) Stores are T (so copied/dst data is now resident in L1/L2)

In theory there might even be valid uses for this type of memcpy where loaded
data is not needed again soon and stored data is referenced again soon,
although I cannot think of any here while typing this mail..

I think some use-case examples, and clear documentation on when/how to choose
between rte_memcpy() or any (potential future) rte_memcpy_nt() variants is required
to progress this patch.

Assuming a strong use-case exists, and it can be clearly indicators to users of DPDK APIs which
rte_memcpy() to use, we can look at technical details around enabling the implementation.

-Harry

<snip remaining points>

Re: [dpdk-dev] [PATCH v4 2/2] lib/eal: add temporal store memcpy support for AMD platform

[Ananyev, Konstantin]

 
> 
> Hi Mattias,
> 
> > > 6) What is the use-case for this? When would a user *want* to use this instead
> > of rte_memcpy()?
> > > If the data being loaded is relevant to datapath/packets, presumably other
> > packets might require the
> > > loaded data, so temporal (normal) loads should be used to cache the source
> > data?
> >
> >
> > I'm not sure if your first question is rhetorical or not, but a memcpy()
> > in a NT variant is certainly useful. One use case for a memcpy() with
> > temporal loads and non-temporal stores is if you need to archive packet
> > payload for (distant, potential) future use, and want to avoid causing
> > unnecessary LLC evictions while doing so.
> 
> Yes I agree that there are certainly benefits in using cache-locality hints.
> There is an open question around if the src or dst or both are non-temporal.
> 
> In the implementation of this patch, the NT/T type of store is reversed from your use-case:
> 1) Loads are NT (so loaded data is not cached for future packets)
> 2) Stores are T (so copied/dst data is now resident in L1/L2)
> 
> In theory there might even be valid uses for this type of memcpy where loaded
> data is not needed again soon and stored data is referenced again soon,
> although I cannot think of any here while typing this mail..
> 
> I think some use-case examples, and clear documentation on when/how to choose
> between rte_memcpy() or any (potential future) rte_memcpy_nt() variants is required
> to progress this patch.
> 
> Assuming a strong use-case exists, and it can be clearly indicators to users of DPDK APIs which
> rte_memcpy() to use, we can look at technical details around enabling the implementation.
> 

+1 here.
Function behaviour and restrictions (src parameter needs to be 16/32 B aligned, etc.),
along with expected usage scenarios have to be documented properly.
Again, as Harry pointed out, I don't see any AMD specific instructions in this function,
so presumably such function can go into __AVX2__ code block and no new defines will
be required. 

 

Re: [dpdk-dev] [PATCH v4 2/2] lib/eal: add temporal store memcpy support for AMD platform

[Aman Kumar]

On Wed, Oct 27, 2021 at 5:53 PM Ananyev, Konstantin <
konstantin.ananyev@intel.com> wrote

> >
> > Hi Mattias,
> >
> > > > 6) What is the use-case for this? When would a user *want* to use
> this instead
> > > of rte_memcpy()?
> > > > If the data being loaded is relevant to datapath/packets, presumably
> other
> > > packets might require the
> > > > loaded data, so temporal (normal) loads should be used to cache the
> source
> > > data?
> > >
> > >
> > > I'm not sure if your first question is rhetorical or not, but a
> memcpy()
> > > in a NT variant is certainly useful. One use case for a memcpy() with
> > > temporal loads and non-temporal stores is if you need to archive packet
> > > payload for (distant, potential) future use, and want to avoid causing
> > > unnecessary LLC evictions while doing so.
> >
> > Yes I agree that there are certainly benefits in using cache-locality
> hints.
> > There is an open question around if the src or dst or both are
> non-temporal.
> >
> > In the implementation of this patch, the NT/T type of store is reversed
> from your use-case:
> > 1) Loads are NT (so loaded data is not cached for future packets)
> > 2) Stores are T (so copied/dst data is now resident in L1/L2)
> >
> > In theory there might even be valid uses for this type of memcpy where
> loaded
> > data is not needed again soon and stored data is referenced again soon,
> > although I cannot think of any here while typing this mail..
> >
> > I think some use-case examples, and clear documentation on when/how to
> choose
> > between rte_memcpy() or any (potential future) rte_memcpy_nt() variants
> is required
> > to progress this patch.
> >
> > Assuming a strong use-case exists, and it can be clearly indicators to
> users of DPDK APIs which
> > rte_memcpy() to use, we can look at technical details around enabling
> the implementation.
> >
>
> +1 here.
> Function behaviour and restrictions (src parameter needs to be 16/32 B
> aligned, etc.),
> along with expected usage scenarios have to be documented properly.
> Again, as Harry pointed out, I don't see any AMD specific instructions in
> this function,
> so presumably such function can go into __AVX2__ code block and no new
> defines will
> be required.
>
> Agreed that APIs are generic but we've kept under an AMD flag for a
simple reason that it is NOT tested on any other platform.
A use-case on how to use this was planned earlier for mlx5 pmd but dropped
in this version of patch as the data path of mlx5 is going to be refactored
soon and may not be useful for future versions of mlx5 (>22.02).
Ref link: adaptation to mlx5 mprq
<https://patchwork.dpdk.org/project/dpdk/patch/20211019104724.19416-2-aman.kumar@vvdntech.in/>
(*we've plan to adapt this into future version*)
The patch in the link basically enhances mlx5 mprq implementation for our
specific use-case and with 128B packet size, we achieve ~60% better perf.
We understand the use of this copy function should be documented which we
shall plan along with few other platform specific optimizations in future
versions of DPDK. As this does not conflict with other platforms, can we still
keep under AMD flag for now as suggested by Thomas?

Re: [dpdk-dev] [PATCH v4 2/2] lib/eal: add temporal store memcpy support for AMD platform

[Van Haaren, Harry]

From: Aman Kumar <aman.kumar@vvdntech.in> 
Sent: Wednesday, October 27, 2021 2:35 PM
To: Ananyev, Konstantin <konstantin.ananyev@intel.com>
Cc: Van Haaren, Harry <harry.van.haaren@intel.com>; mattias.ronnblom <mattias.ronnblom@ericsson.com>; Thomas Monjalon <thomas@monjalon.net>; dev@dpdk.org; viacheslavo@nvidia.com; Burakov, Anatoly <anatoly.burakov@intel.com>; Song, Keesang <Keesang.Song@amd.com>; jerinjacobk@gmail.com; Richardson, Bruce <bruce.richardson@intel.com>; honnappa.nagarahalli@arm.com; Ruifeng Wang <ruifeng.wang@arm.com>; David Christensen <drc@linux.vnet.ibm.com>; david.marchand@redhat.com; stephen@networkplumber.org
Subject: Re: [dpdk-dev] [PATCH v4 2/2] lib/eal: add temporal store memcpy support for AMD platform

Hi Aman,

Please sent plain-text email, converting to other formats it makes writing inline replies difficult.
I've converted this reply email back to plain-text, and will annotate email below with [<author> wrote]:

On Wed, Oct 27, 2021 at 5:53 PM Ananyev, Konstantin <mailto:konstantin.ananyev@intel.com> wrote
> 
> Hi Mattias,
> 
> > > 6) What is the use-case for this? When would a user *want* to use this instead
> > of rte_memcpy()?
> > > If the data being loaded is relevant to datapath/packets, presumably other
> > packets might require the
> > > loaded data, so temporal (normal) loads should be used to cache the source
> > data?
> >
> >
> > I'm not sure if your first question is rhetorical or not, but a memcpy()
> > in a NT variant is certainly useful. One use case for a memcpy() with
> > temporal loads and non-temporal stores is if you need to archive packet
> > payload for (distant, potential) future use, and want to avoid causing
> > unnecessary LLC evictions while doing so.
> 
> Yes I agree that there are certainly benefits in using cache-locality hints.
> There is an open question around if the src or dst or both are non-temporal.
> 
> In the implementation of this patch, the NT/T type of store is reversed from your use-case:
> 1) Loads are NT (so loaded data is not cached for future packets)
> 2) Stores are T (so copied/dst data is now resident in L1/L2)
> 
> In theory there might even be valid uses for this type of memcpy where loaded
> data is not needed again soon and stored data is referenced again soon,
> although I cannot think of any here while typing this mail..
> 
> I think some use-case examples, and clear documentation on when/how to choose
> between rte_memcpy() or any (potential future) rte_memcpy_nt() variants is required
> to progress this patch.
> 
> Assuming a strong use-case exists, and it can be clearly indicators to users of DPDK APIs which
> rte_memcpy() to use, we can look at technical details around enabling the implementation.
> 

[Konstantin wrote]:
+1 here.
Function behaviour and restrictions (src parameter needs to be 16/32 B aligned, etc.),
along with expected usage scenarios have to be documented properly.
Again, as Harry pointed out, I don't see any AMD specific instructions in this function,
so presumably such function can go into __AVX2__ code block and no new defines will
be required. 


[Aman wrote]:
Agreed that APIs are generic but we've kept under an AMD flag for a simple reason that it is NOT tested on any other platform.
A use-case on how to use this was planned earlier for mlx5 pmd but dropped in this version of patch as the data path of mlx5 is going to be refactored soon and may not be useful for future versions of mlx5 (>22.02). 
Ref link: https://patchwork.dpdk.org/project/dpdk/patch/20211019104724.19416-2-aman.kumar@vvdntech.in/(we've plan to adapt this into future version)
The patch in the link basically enhances mlx5 mprq implementation for our specific use-case and with 128B packet size, we achieve ~60% better perf. We understand the use of this copy function should be documented which we shall plan along with few other platform specific optimizations in future versions of DPDK. As this does not conflict with other platforms, can we still keep under AMD flag for now as suggested by Thomas?


[HvH wrote]:
As an open-source community, any contributions should aim to improve the whole.
In the past, numerous improvements have been merged to DPDK that improve performance.
Sometimes these are architecture specific (x86/arm/ppc) sometimes the are ISA specific (SSE, AVX512, NEON).

I am not familiar with any cases in DPDK, where there is a #ifdef based on a *specific platform*.
A quick "grep" through the "dpdk/lib" directory does not show any place where PMD or generic code
has been explicitly optimized for a *specific platform*.

Obviously, in cases where ISA either exists or does not exist, yes there is an optimization to enable it.
But this is not exposed as a top-level compile-time option, it uses runtime CPU ISA detection.

Please take a step back from the code, and look at what this patch asks of DPDK:
"Please accept & maintain these changes upstream, which benefit only platform X, even though these ISA features are also available on other platforms".

Other patches that enhance performance of DPDK ask this:
"Please accept & maintain these changes upstream, which benefit all platforms which have ISA capability X".


=== Question "As this does not conflict with other platforms, can we still keep under AMD flag for now"?
I feel the contribution is too specific to a platform. Make it generic by enabling it at an ISA capability level.

Please yes, contribute to the DPDK community by improving performance of a PMD by enabling/leveraging ISA.
But do so in a way that does not benefit only a specific platform - do so in a way that enhances all of DPDK, as
other patches have done for the DPDK that this patch is built on.

If you have concerns that the PMD maintainers will not accept the changes due to potential regressions on
other platforms, then discuss those, make a plan on how to performance validate, and work to a solution.


=== Regarding specifically the request for "can we still keep under AMD flag for now"?
I do not believe we should introduce APIs for specific platforms. DPDK's EAL is an abstraction layer.
The value of EAL is to provide a common abstraction. This platform-specific flag breaks the abstraction,
and results in packaging issues, as well as API/ABI instability based on -Dcpu_instruction_set choice.
So, no, we should not introduce APIs based on any compile-time flag.

Re: [dpdk-dev] [PATCH v4 2/2] lib/eal: add temporal store memcpy support for AMD platform

[Thomas Monjalon]

27/10/2021 16:10, Van Haaren, Harry:
> From: Aman Kumar <aman.kumar@vvdntech.in> 
> On Wed, Oct 27, 2021 at 5:53 PM Ananyev, Konstantin <mailto:konstantin.ananyev@intel.com> wrote
> > 
> > Hi Mattias,
> > 
> > > > 6) What is the use-case for this? When would a user *want* to use this instead
> > > of rte_memcpy()?
> > > > If the data being loaded is relevant to datapath/packets, presumably other
> > > packets might require the
> > > > loaded data, so temporal (normal) loads should be used to cache the source
> > > data?
> > >
> > >
> > > I'm not sure if your first question is rhetorical or not, but a memcpy()
> > > in a NT variant is certainly useful. One use case for a memcpy() with
> > > temporal loads and non-temporal stores is if you need to archive packet
> > > payload for (distant, potential) future use, and want to avoid causing
> > > unnecessary LLC evictions while doing so.
> > 
> > Yes I agree that there are certainly benefits in using cache-locality hints.
> > There is an open question around if the src or dst or both are non-temporal.
> > 
> > In the implementation of this patch, the NT/T type of store is reversed from your use-case:
> > 1) Loads are NT (so loaded data is not cached for future packets)
> > 2) Stores are T (so copied/dst data is now resident in L1/L2)
> > 
> > In theory there might even be valid uses for this type of memcpy where loaded
> > data is not needed again soon and stored data is referenced again soon,
> > although I cannot think of any here while typing this mail..
> > 
> > I think some use-case examples, and clear documentation on when/how to choose
> > between rte_memcpy() or any (potential future) rte_memcpy_nt() variants is required
> > to progress this patch.
> > 
> > Assuming a strong use-case exists, and it can be clearly indicators to users of DPDK APIs which
> > rte_memcpy() to use, we can look at technical details around enabling the implementation.
> > 
> 
> [Konstantin wrote]:
> +1 here.
> Function behaviour and restrictions (src parameter needs to be 16/32 B aligned, etc.),
> along with expected usage scenarios have to be documented properly.
> Again, as Harry pointed out, I don't see any AMD specific instructions in this function,
> so presumably such function can go into __AVX2__ code block and no new defines will
> be required. 
> 
> 
> [Aman wrote]:
> Agreed that APIs are generic but we've kept under an AMD flag for a simple reason that it is NOT tested on any other platform.
> A use-case on how to use this was planned earlier for mlx5 pmd but dropped in this version of patch as the data path of mlx5 is going to be refactored soon and may not be useful for future versions of mlx5 (>22.02). 
> Ref link: https://patchwork.dpdk.org/project/dpdk/patch/20211019104724.19416-2-aman.kumar@vvdntech.in/(we've plan to adapt this into future version)
> The patch in the link basically enhances mlx5 mprq implementation for our specific use-case and with 128B packet size, we achieve ~60% better perf. We understand the use of this copy function should be documented which we shall plan along with few other platform specific optimizations in future versions of DPDK. As this does not conflict with other platforms, can we still keep under AMD flag for now as suggested by Thomas?

I said I could merge if there is no objection.
I've overlooked that it's adding completely new functions in the API.
And the comments go in the direction of what I asked in previous version:
what is specific to AMD here?
Now seeing the valid objections, I agree it should be reworked.
We must provide API to applications which is generic, stable and well documented.


> [HvH wrote]:
> As an open-source community, any contributions should aim to improve the whole.
> In the past, numerous improvements have been merged to DPDK that improve performance.
> Sometimes these are architecture specific (x86/arm/ppc) sometimes the are ISA specific (SSE, AVX512, NEON).
> 
> I am not familiar with any cases in DPDK, where there is a #ifdef based on a *specific platform*.
> A quick "grep" through the "dpdk/lib" directory does not show any place where PMD or generic code
> has been explicitly optimized for a *specific platform*.
> 
> Obviously, in cases where ISA either exists or does not exist, yes there is an optimization to enable it.
> But this is not exposed as a top-level compile-time option, it uses runtime CPU ISA detection.
> 
> Please take a step back from the code, and look at what this patch asks of DPDK:
> "Please accept & maintain these changes upstream, which benefit only platform X, even though these ISA features are also available on other platforms".
> 
> Other patches that enhance performance of DPDK ask this:
> "Please accept & maintain these changes upstream, which benefit all platforms which have ISA capability X".
> 
> 
> === Question "As this does not conflict with other platforms, can we still keep under AMD flag for now"?
> I feel the contribution is too specific to a platform. Make it generic by enabling it at an ISA capability level.
> 
> Please yes, contribute to the DPDK community by improving performance of a PMD by enabling/leveraging ISA.
> But do so in a way that does not benefit only a specific platform - do so in a way that enhances all of DPDK, as
> other patches have done for the DPDK that this patch is built on.
> 
> If you have concerns that the PMD maintainers will not accept the changes due to potential regressions on
> other platforms, then discuss those, make a plan on how to performance validate, and work to a solution.
> 
> 
> === Regarding specifically the request for "can we still keep under AMD flag for now"?
> I do not believe we should introduce APIs for specific platforms. DPDK's EAL is an abstraction layer.
> The value of EAL is to provide a common abstraction. This platform-specific flag breaks the abstraction,
> and results in packaging issues, as well as API/ABI instability based on -Dcpu_instruction_set choice.
> So, no, we should not introduce APIs based on any compile-time flag.

I agree

Re: [dpdk-dev] [PATCH v4 2/2] lib/eal: add temporal store memcpy support for AMD platform

[Song, Keesang]

[AMD Official Use Only]

Hi Thomas,

There are some gaps among us, so I think we really need another quick meeting call to discuss. I will set up a call like the last time on Monday.
Please join in the call if possible.

Thanks,
Keesang

-----Original Message-----
From: Thomas Monjalon <thomas@monjalon.net>
Sent: Wednesday, October 27, 2021 7:31 AM
To: Aman Kumar <aman.kumar@vvdntech.in>; Ananyev, Konstantin <konstantin.ananyev@intel.com>; Van Haaren, Harry <harry.van.haaren@intel.com>
Cc: mattias. ronnblom <mattias.ronnblom@ericsson.com>; dev@dpdk.org; viacheslavo@nvidia.com; Burakov, Anatoly <anatoly.burakov@intel.com>; Song, Keesang <Keesang.Song@amd.com>; jerinjacobk@gmail.com; Richardson, Bruce <bruce.richardson@intel.com>; honnappa.nagarahalli@arm.com; Ruifeng Wang <ruifeng.wang@arm.com>; David Christensen <drc@linux.vnet.ibm.com>; david.marchand@redhat.com; stephen@networkplumber.org
Subject: Re: [dpdk-dev] [PATCH v4 2/2] lib/eal: add temporal store memcpy support for AMD platform

[CAUTION: External Email]

27/10/2021 16:10, Van Haaren, Harry:
> From: Aman Kumar <aman.kumar@vvdntech.in> On Wed, Oct 27, 2021 at 5:53
> PM Ananyev, Konstantin <mailto:konstantin.ananyev@intel.com> wrote
> >
> > Hi Mattias,
> >
> > > > 6) What is the use-case for this? When would a user *want* to
> > > > use this instead
> > > of rte_memcpy()?
> > > > If the data being loaded is relevant to datapath/packets,
> > > > presumably other
> > > packets might require the
> > > > loaded data, so temporal (normal) loads should be used to cache
> > > > the source
> > > data?
> > >
> > >
> > > I'm not sure if your first question is rhetorical or not, but a
> > > memcpy() in a NT variant is certainly useful. One use case for a
> > > memcpy() with temporal loads and non-temporal stores is if you
> > > need to archive packet payload for (distant, potential) future
> > > use, and want to avoid causing unnecessary LLC evictions while doing so.
> >
> > Yes I agree that there are certainly benefits in using cache-locality hints.
> > There is an open question around if the src or dst or both are non-temporal.
> >
> > In the implementation of this patch, the NT/T type of store is reversed from your use-case:
> > 1) Loads are NT (so loaded data is not cached for future packets)
> > 2) Stores are T (so copied/dst data is now resident in L1/L2)
> >
> > In theory there might even be valid uses for this type of memcpy
> > where loaded data is not needed again soon and stored data is
> > referenced again soon, although I cannot think of any here while typing this mail..
> >
> > I think some use-case examples, and clear documentation on when/how
> > to choose between rte_memcpy() or any (potential future)
> > rte_memcpy_nt() variants is required to progress this patch.
> >
> > Assuming a strong use-case exists, and it can be clearly indicators
> > to users of DPDK APIs which
> > rte_memcpy() to use, we can look at technical details around enabling the implementation.
> >
>
> [Konstantin wrote]:
> +1 here.
> Function behaviour and restrictions (src parameter needs to be 16/32 B
> aligned, etc.), along with expected usage scenarios have to be documented properly.
> Again, as Harry pointed out, I don't see any AMD specific instructions
> in this function, so presumably such function can go into __AVX2__
> code block and no new defines will be required.
>
>
> [Aman wrote]:
> Agreed that APIs are generic but we've kept under an AMD flag for a simple reason that it is NOT tested on any other platform.
> A use-case on how to use this was planned earlier for mlx5 pmd but dropped in this version of patch as the data path of mlx5 is going to be refactored soon and may not be useful for future versions of mlx5 (>22.02).
> Ref link:
> https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fpatchwork.dpdk.org%2Fproject%2Fdpdk%2Fpatch%2F20211019104724.19416-2-aman.kumar%40vvdntech.in%2F&amp;data=04%7C01%7CKeesang.Song%40amd.com%7C1988237087f74375caf808d9995678f0%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637709418976849481%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=FErr0cuni6WLxpq5z2KKjAx2StGTlGuN4QaXoXFE%2BKI%3D&amp;reserved=0(we've plan to adapt this into future version) The patch in the link basically enhances mlx5 mprq implementation for our specific use-case and with 128B packet size, we achieve ~60% better perf. We understand the use of this copy function should be documented which we shall plan along with few other platform specific optimizations in future versions of DPDK. As this does not conflict with other platforms, can we still keep under AMD flag for now as suggested by Thomas?

I said I could merge if there is no objection.
I've overlooked that it's adding completely new functions in the API.
And the comments go in the direction of what I asked in previous version:
what is specific to AMD here?
Now seeing the valid objections, I agree it should be reworked.
We must provide API to applications which is generic, stable and well documented.


> [HvH wrote]:
> As an open-source community, any contributions should aim to improve the whole.
> In the past, numerous improvements have been merged to DPDK that improve performance.
> Sometimes these are architecture specific (x86/arm/ppc) sometimes the are ISA specific (SSE, AVX512, NEON).
>
> I am not familiar with any cases in DPDK, where there is a #ifdef based on a *specific platform*.
> A quick "grep" through the "dpdk/lib" directory does not show any
> place where PMD or generic code has been explicitly optimized for a *specific platform*.
>
> Obviously, in cases where ISA either exists or does not exist, yes there is an optimization to enable it.
> But this is not exposed as a top-level compile-time option, it uses runtime CPU ISA detection.
>
> Please take a step back from the code, and look at what this patch asks of DPDK:
> "Please accept & maintain these changes upstream, which benefit only platform X, even though these ISA features are also available on other platforms".
>
> Other patches that enhance performance of DPDK ask this:
> "Please accept & maintain these changes upstream, which benefit all platforms which have ISA capability X".
>
>
> === Question "As this does not conflict with other platforms, can we still keep under AMD flag for now"?
> I feel the contribution is too specific to a platform. Make it generic by enabling it at an ISA capability level.
>
> Please yes, contribute to the DPDK community by improving performance of a PMD by enabling/leveraging ISA.
> But do so in a way that does not benefit only a specific platform - do
> so in a way that enhances all of DPDK, as other patches have done for the DPDK that this patch is built on.
>
> If you have concerns that the PMD maintainers will not accept the
> changes due to potential regressions on other platforms, then discuss those, make a plan on how to performance validate, and work to a solution.
>
>
> === Regarding specifically the request for "can we still keep under AMD flag for now"?
> I do not believe we should introduce APIs for specific platforms. DPDK's EAL is an abstraction layer.
> The value of EAL is to provide a common abstraction. This
> platform-specific flag breaks the abstraction, and results in packaging issues, as well as API/ABI instability based on -Dcpu_instruction_set choice.
> So, no, we should not introduce APIs based on any compile-time flag.

I agree

Re: [dpdk-dev] [PATCH v4 2/2] lib/eal: add temporal store memcpy support for AMD platform

[Ananyev, Konstantin]

From: Aman Kumar <aman.kumar@vvdntech.in> 
Sent: Wednesday, October 27, 2021 2:35 PM
To: Ananyev, Konstantin <konstantin.ananyev@intel.com>
Cc: Van Haaren, Harry <harry.van.haaren@intel.com>; mattias.ronnblom <mattias.ronnblom@ericsson.com>; Thomas Monjalon <thomas@monjalon.net>; dev@dpdk.org; viacheslavo@nvidia.com; Burakov, Anatoly <anatoly.burakov@intel.com>; Song, Keesang <Keesang.Song@amd.com>; jerinjacobk@gmail.com; Richardson, Bruce <bruce.richardson@intel.com>; honnappa.nagarahalli@arm.com; Ruifeng Wang <ruifeng.wang@arm.com>; David Christensen <drc@linux.vnet.ibm.com>; david.marchand@redhat.com; stephen@networkplumber.org
Subject: Re: [dpdk-dev] [PATCH v4 2/2] lib/eal: add temporal store memcpy support for AMD platform

>> 
>> Hi Mattias,
>> 
>> > > 6) What is the use-case for this? When would a user *want* to use this instead
>> > of rte_memcpy()?
>> > > If the data being loaded is relevant to datapath/packets, presumably other
>> > packets might require the
>> > > loaded data, so temporal (normal) loads should be used to cache the source
>> > data?
>> >
>> >
>> > I'm not sure if your first question is rhetorical or not, but a memcpy()
>> > in a NT variant is certainly useful. One use case for a memcpy() with
>> > temporal loads and non-temporal stores is if you need to archive packet
>> > payload for (distant, potential) future use, and want to avoid causing
>> > unnecessary LLC evictions while doing so.
>> 
>> Yes I agree that there are certainly benefits in using cache-locality hints.
>> There is an open question around if the src or dst or both are non-temporal.
>> 
>> In the implementation of this patch, the NT/T type of store is reversed from your use-case:
>> 1) Loads are NT (so loaded data is not cached for future packets)
>> 2) Stores are T (so copied/dst data is now resident in L1/L2)
>> 
>> In theory there might even be valid uses for this type of memcpy where loaded
>> data is not needed again soon and stored data is referenced again soon,
>> although I cannot think of any here while typing this mail..
>> 
>> I think some use-case examples, and clear documentation on when/how to choose
>> between rte_memcpy() or any (potential future) rte_memcpy_nt() variants is required
>> to progress this patch.
>> 
>> Assuming a strong use-case exists, and it can be clearly indicators to users of DPDK APIs which
>> rte_memcpy() to use, we can look at technical details around enabling the implementation.
>> 
>
> +1 here.
> Function behaviour and restrictions (src parameter needs to be 16/32 B aligned, etc.),
> along with expected usage scenarios have to be documented properly.
> Again, as Harry pointed out, I don't see any AMD specific instructions in this function,
> so presumably such function can go into __AVX2__ code block and no new defines will
> be required. 
> Agreed that APIs are generic but we've kept under an AMD flag for a simple reason that it is NOT tested on any other platform.
> A use-case on how to use this was planned earlier for mlx5 pmd but dropped in this version of patch as the data path of mlx5 is going to be refactored soon and may not be useful for > future versions of mlx5 (>22.02). 
> Ref link: https://patchwork.dpdk.org/project/dpdk/patch/20211019104724.19416-2-aman.kumar@vvdntech.in/(we've plan to adapt this into future version)
> The patch in the link basically enhances mlx5 mprq implementation for our specific use-case and with 128B packet size, we achieve ~60% better perf. We understand the use of this
> copy function should be documented which we shall plan along with few other platform specific optimizations in future versions of DPDK. As this does not conflict with other  >platforms, can we still keep under AMD flag for now as suggested by Thomas?

From what I read above the patch is sort of in the half-ready stage.
Why to rush here and try to push to the DPDK things that doesn't full-fill DPDK policy?
Probably better to do all missing parts first (docs, tests, etc.) and then come-up with updated version.
 

Re: [dpdk-dev] [PATCH v4 2/2] lib/eal: add temporal store memcpy support for AMD platform

[Mattias Rönnblom]

On 2021-10-27 09:28, Aman Kumar wrote:
> This patch provides a rte_memcpy* call with temporal stores.
> Use -Dcpu_instruction_set=znverX with build to enable this API.
>
> Signed-off-by: Aman Kumar <aman.kumar@vvdntech.in>
> ---
>   config/x86/meson.build           |   2 +
>   lib/eal/x86/include/rte_memcpy.h | 114 +++++++++++++++++++++++++++++++
>   2 files changed, 116 insertions(+)
>
> diff --git a/config/x86/meson.build b/config/x86/meson.build
> index 21cda6fd33..56dae4aca7 100644
> --- a/config/x86/meson.build
> +++ b/config/x86/meson.build
> @@ -78,6 +78,8 @@ if get_option('cpu_instruction_set') == 'znver1'
>       dpdk_conf.set('RTE_MAX_LCORE', 256)
>   elif get_option('cpu_instruction_set') == 'znver2'
>       dpdk_conf.set('RTE_MAX_LCORE', 512)
> +    dpdk_conf.set('RTE_MEMCPY_AMDEPYC', 1)
>   elif get_option('cpu_instruction_set') == 'znver3'
>       dpdk_conf.set('RTE_MAX_LCORE', 512)
> +    dpdk_conf.set('RTE_MEMCPY_AMDEPYC', 1)
>   endif
> diff --git a/lib/eal/x86/include/rte_memcpy.h b/lib/eal/x86/include/rte_memcpy.h
> index 1b6c6e585f..8fe7822cb4 100644
> --- a/lib/eal/x86/include/rte_memcpy.h
> +++ b/lib/eal/x86/include/rte_memcpy.h
> @@ -376,6 +376,120 @@ rte_mov128blocks(uint8_t *dst, const uint8_t *src, size_t n)
>   	}
>   }
>   
> +#if defined RTE_MEMCPY_AMDEPYC
> +
> +/**
> + * Copy 16 bytes from one location to another,
> + * with temporal stores
> + */
> +static __rte_always_inline void
> +rte_copy16_ts(uint8_t *dst, uint8_t *src)
> +{
> +	__m128i var128;
> +
> +	var128 = _mm_stream_load_si128((__m128i *)src);
> +	_mm_storeu_si128((__m128i *)dst, var128);
> +}
> +
> +/**
> + * Copy 32 bytes from one location to another,
> + * with temporal stores
> + */
> +static __rte_always_inline void
> +rte_copy32_ts(uint8_t *dst, uint8_t *src)
> +{
> +	__m256i ymm0;
> +
> +	ymm0 = _mm256_stream_load_si256((const __m256i *)src);
> +	_mm256_storeu_si256((__m256i *)dst, ymm0);
> +}
> +
> +/**
> + * Copy 64 bytes from one location to another,
> + * with temporal stores
> + */
> +static __rte_always_inline void
> +rte_copy64_ts(uint8_t *dst, uint8_t *src)
> +{
> +	rte_copy32_ts(dst + 0 * 32, src + 0 * 32);
> +	rte_copy32_ts(dst + 1 * 32, src + 1 * 32);
> +}
> +
> +/**
> + * Copy 128 bytes from one location to another,
> + * with temporal stores
> + */
> +static __rte_always_inline void
> +rte_copy128_ts(uint8_t *dst, uint8_t *src)
> +{
> +	rte_copy32_ts(dst + 0 * 32, src + 0 * 32);
> +	rte_copy32_ts(dst + 1 * 32, src + 1 * 32);
> +	rte_copy32_ts(dst + 2 * 32, src + 2 * 32);
> +	rte_copy32_ts(dst + 3 * 32, src + 3 * 32);
> +}
> +
> +/**
> + * Copy len bytes from one location to another,
> + * with temporal stores 16B aligned
> + */
> +static __rte_always_inline void *
> +rte_memcpy_aligned_tstore16_generic(void *dst, void *src, int len)
> +{
> +	void *dest = dst;
> +
> +	while (len >= 128) {
> +		rte_copy128_ts((uint8_t *)dst, (uint8_t *)src);
> +		dst = (uint8_t *)dst + 128;
> +		src = (uint8_t *)src + 128;
> +		len -= 128;
> +	}
> +	while (len >= 64) {
> +		rte_copy64_ts((uint8_t *)dst, (uint8_t *)src);
> +		dst = (uint8_t *)dst + 64;
> +		src = (uint8_t *)src + 64;
> +		len -= 64;
> +	}
> +	while (len >= 32) {
> +		rte_copy32_ts((uint8_t *)dst, (uint8_t *)src);
> +		dst = (uint8_t *)dst + 32;
> +		src = (uint8_t *)src + 32;
> +		len -= 32;
> +	}
> +	if (len >= 16) {
> +		rte_copy16_ts((uint8_t *)dst, (uint8_t *)src);
> +		dst = (uint8_t *)dst + 16;
> +		src = (uint8_t *)src + 16;
> +		len -= 16;
> +	}
> +	if (len >= 8) {
> +		*(uint64_t *)dst = *(const uint64_t *)src;
> +		dst = (uint8_t *)dst + 8;
> +		src = (uint8_t *)src + 8;
> +		len -= 8;
> +	}
> +	if (len >= 4) {
> +		*(uint32_t *)dst = *(const uint32_t *)src;
> +		dst = (uint8_t *)dst + 4;
> +		src = (uint8_t *)src + 4;
> +		len -= 4;
> +	}
> +	if (len != 0) {
> +		dst = (uint8_t *)dst - (4 - len);
> +		src = (uint8_t *)src - (4 - len);
> +		*(uint32_t *)dst = *(const uint32_t *)src;
> +	}
> +
> +	return dest;


You don't need a _mm_sfence after the NT stores to avoid surprises (e.g, 
if you use this NT memcpy() in combination with DPDK rings)? NT stores 
are weakly ordered on x86_64, from what I understand.


> +}
> +
> +static __rte_always_inline void *
> +rte_memcpy_aligned_tstore16(void *dst, void *src, int len)


Shouldn't both dst and src be marked __restrict? Goes for all these 
functions.

> +{
> +	return rte_memcpy_aligned_tstore16_generic(dst, src, len);
> +}
> +
> +#endif /* RTE_MEMCPY_AMDEPYC */


What does x86_64 NT stores have to do with EPYC?


> +
>   static __rte_always_inline void *
>   rte_memcpy_generic(void *dst, const void *src, size_t n)
>   {