[PATCH v1 1/2] riscv support rte_memcpy in vector

[Qiguo Chen <chen.qiguo@zte.com.cn>]

[-- Attachment #1.1.1: Type: text/plain, Size: 9838 bytes --]

This patch implements RISC-V vector intrinsics
to accelerate memory copy operations for byte range (129~1600).

Signed-off-by: Qiguo Chen <chen.qiguo@zte.com.cn>
---
 .mailmap                           |   1 +
 config/riscv/meson.build           |  14 ++
 lib/eal/riscv/include/rte_memcpy.h | 310 ++++++++++++++++++++++++++++-
 3 files changed, 323 insertions(+), 2 deletions(-)

diff --git a/.mailmap b/.mailmap
index 08e5ec8560..178c5f44f4 100644
--- a/.mailmap
+++ b/.mailmap
@@ -1285,6 +1285,7 @@ Qian Hao <qi_an_hao@126.com>
 Qian Xu <qian.q.xu@intel.com>
 Qiao Liu <qiao.liu@intel.com>
 Qi Fu <qi.fu@intel.com>
+Qiguo Chen <chen.qiguo@zte.com.cn>
 Qimai Xiao <qimaix.xiao@intel.com>
 Qiming Chen <chenqiming_huawei@163.com>
 Qiming Yang <qiming.yang@intel.com>
diff --git a/config/riscv/meson.build b/config/riscv/meson.build
index f3daea0c0e..abba474b5e 100644
--- a/config/riscv/meson.build
+++ b/config/riscv/meson.build
@@ -146,6 +146,20 @@ if (riscv_extension_macros and
     endif
 endif
 
+# detect extensions
+# Requires intrinsics available in GCC 14.1.0+ and Clang 18.1.0+
+if (riscv_extension_macros and
+    (cc.get_define('__riscv_zicbop', args: machine_args) != ''))
+  if ((cc.get_id() == 'gcc' and cc.version().version_compare('>=14.1.0'))
+      or (cc.get_id() == 'clang' and cc.version().version_compare('>=18.1.0')))
+      message('Compiling with the zicbop extension')
+      machine_args += ['-DRTE_RISCV_FEATURE_PREFETCH']
+  else
+    warning('Detected zicbop extension but cannot use because intrinsics are not available (present in GCC 14.1.0+ and Clang 18.1.0+)')
+  endif
+endif
+
+
 # apply flags
 foreach flag: dpdk_flags
     if flag.length() > 0
diff --git a/lib/eal/riscv/include/rte_memcpy.h b/lib/eal/riscv/include/rte_memcpy.h
index d8a942c5d2..6f8cb0d4a4 100644
--- a/lib/eal/riscv/include/rte_memcpy.h
+++ b/lib/eal/riscv/include/rte_memcpy.h
@@ -11,6 +11,7 @@
 #include <string.h>
 
 #include "rte_common.h"
+#include <rte_branch_prediction.h>
 
 #include "generic/rte_memcpy.h"
 
@@ -18,6 +19,290 @@
 extern "C" {
 #endif
 
+
+#if defined(RTE_RISCV_FEATURE_V) && !(defined(RTE_RISCV_FEATURE_PREFETCH))
+#undef RTE_RISCV_FEATURE_V
+#endif
+
+
+#if defined(RTE_RISCV_FEATURE_V)
+
+#include "rte_cpuflags.h"
+
+#define RISCV_VLENB   16
+#define MEMCPY_GLIBC       (1U << 0)
+#define MEMCPY_RISCV       (1U << 1)
+#define ALIGNMENT_MASK_128   0x7F
+#define ALIGNMENT_MASK_64    0x3F
+#define ALIGNMENT_MASK_16    0xF
+
+static uint8_t memcpy_alg = MEMCPY_GLIBC;
+
+
+static __rte_always_inline void
+memcpy_prefetch64_1(const uint8_t *src, uint8_t *dst)
+{
+	__asm__ (
+		"prefetch.r 64(%0)\n"
+		"prefetch.w 64(%1)"
+		:: "r"(src), "r"(dst)
+	);
+}
+
+static __rte_always_inline void
+memcpy_prefetch128_1(const uint8_t *src, uint8_t *dst)
+{
+	__asm__ (
+		"prefetch.r 128(%0)\n"
+		"prefetch.w 128(%1)"
+		:: "r"(src), "r"(dst)
+	);
+}
+
+static __rte_always_inline void
+memcpy_prefetch128_2(const uint8_t *src, uint8_t *dst)
+{
+	__asm__ (
+		"prefetch.r 128(%0);"
+		"prefetch.w 128(%1);"
+		"prefetch.r 192(%0);"
+		"prefetch.w 192(%1)"
+		:: "r"(src), "r"(dst)
+	);
+}
+
+
+static __rte_always_inline void
+_rte_mov32(uint8_t *dst, const uint8_t *src)
+{
+	uint32_t n = 32;
+	asm volatile (
+	     "vsetvli t1, %2, e8, m2, ta, ma\n"
+	     "vle8.v v2, (%1)\n"
+	     "vse8.v v2, (%0)"
+	     :: "r"(dst), "r"(src), "r"(n)
+	     : "v2", "v3", "t1", "memory"
+	 );
+}
+
+static __rte_always_inline void
+_rte_mov64(uint8_t *dst, const uint8_t *src)
+{
+	uint32_t n = 64;
+	asm volatile (
+		"vsetvli t3, %2, e8, m4, ta, ma\n"
+		"vle8.v v8, (%1)\n"
+		"vse8.v v8, (%0)"
+		:: "r"(dst), "r"(src), "r"(n)
+		:  "v8", "v9", "v10", "v11", "t3", "memory"
+	 );
+}
+
+static __rte_always_inline void
+_rte_mov128(uint8_t *dst, const uint8_t *src)
+{
+	uint32_t n = 128;
+	asm volatile (
+		"vsetvli t4, %2, e8, m8, ta, ma\n"
+		"vle8.v v16, (%1)\n"
+		"vse8.v v16, (%0)"
+		:: "r"(dst), "r"(src), "r"(n)
+		: "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "t4", "memory"
+	 );
+}
+
+static __rte_always_inline void
+_rte_mov256(uint8_t *dst, const uint8_t *src)
+{
+	memcpy_prefetch128_2(src, dst);
+	_rte_mov128(dst, src);
+	_rte_mov128(dst + 128, src + 128);
+}
+
+static __rte_always_inline void
+_rte_mov128blocks(uint8_t *dst, const uint8_t *src, size_t n)
+{
+	asm volatile (
+		"prefetch.r 64(%1)\n"
+		"prefetch.w 64(%0)\n"
+		"prefetch.r 128(%1)\n"
+		"prefetch.w 128(%0)\n"
+		"prefetch.r 192(%1)\n"
+		"prefetch.w 192(%0)\n"
+		"prefetch.r 256(%1)\n"
+		"prefetch.w 256(%0)\n"
+		"prefetch.r 320(%1)\n"
+		"prefetch.w 320(%0)\n"
+		"prefetch.r 384(%1)\n"
+		"prefetch.w 384(%0)\n"
+		"prefetch.r 448(%1)\n"
+		"prefetch.w 448(%0)\n"
+		"prefetch.r 512(%1)\n"
+		"li t6, 512\n"
+		"3:\n"
+		"li t5, 128;"
+		"vsetvli zero, t5, e8, m8, ta, ma\n"
+		"1:;"
+		"bgt %2, t6, 4f\n"
+		"j 2f\n"
+		"4:\n"
+		"prefetch.r 576(%1)\n"
+		"prefetch.r 640(%1)\n"
+		"2:\n"
+		"vle8.v   v16, (%1)\n"
+		"add      %1, %1, t5\n"
+		"vse8.v   v16, (%0)\n"
+		"add      %0, %0, t5\n"
+		"sub      %2, %2, t5\n"
+		"bnez     %2, 1b"
+		: "+r"(dst), "+r"(src), "+r"(n)
+		:
+		: "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "t5", "t6", "memory"
+	);
+}
+
+static __rte_always_inline void
+_rte_mov(uint8_t *dst, const uint8_t *src, uint32_t n)
+{
+	asm volatile (
+		"1:\n"
+		"vsetvli t4, %2, e8, m8, ta, ma\n"
+		"vle8.v v16, (%1)\n"
+		"add %1, %1, t4\n"
+		"vse8.v v16, (%0)\n"
+		"add %0, %0, t4\n"
+		"sub %2, %2, t4\n"
+		"bnez %2, 1b"
+		: "+r"(dst), "+r"(src), "+r"(n)
+		:
+		: "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "t4", "memory"
+	 );
+}
+
+static __rte_always_inline void
+_rte_mov_aligned(uint8_t *dst, const uint8_t *src, uint32_t n)
+{
+	asm volatile (
+		"prefetch.r 128(%1)\n"
+		"prefetch.r 192(%1)\n"
+		"prefetch.r 256(%1)\n"
+		"prefetch.r 320(%1)\n"
+		"prefetch.r 384(%1)\n"
+		"prefetch.r 448(%1)\n"
+		"prefetch.r 512(%1)\n"
+		"prefetch.r 576(%1)\n"
+		"li t6, 640\n"
+		"1:\n"
+		"vsetvli t4, %2, e8, m8, ta, ma\n"
+		"vle8.v v16, (%1)\n"
+		"add %1, %1, t4\n"
+		"vse8.v v16, (%0)\n"
+		"add %0, %0, t4\n"
+		"sub %2, %2, t4\n"
+		"blt %2, t6, 3f\n"
+		"prefetch.r 512(%1)\n"
+		"prefetch.r 576(%1)\n"
+		"3:\n"
+		"bnez %2, 1b"
+		: "+r"(dst), "+r"(src), "+r"(n)
+		:
+		: "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "t4", "t6", "memory"
+	 );
+}
+
+static __rte_always_inline void *
+_rte_memcpy_generic(uint8_t       *dst, const uint8_t *src, size_t n)
+{
+	void *ret = dst;
+	size_t dstofss;
+	uint32_t bn;
+
+	if (n <= 384) {
+		if (n >= 256) {
+			memcpy_prefetch128_2(src, dst);
+			n -= 256;
+			_rte_mov128(dst, src);
+			_rte_mov128((uint8_t *)dst + 128, (const uint8_t *)src + 128);
+			src = (const uint8_t *)src + 256;
+			dst = (uint8_t *)dst + 256;
+		}
+		if (n >= 128) {
+			memcpy_prefetch128_1(src, dst);
+			n -= 128;
+			_rte_mov128(dst, src);
+			src = (const uint8_t *)src + 128;
+			dst = (uint8_t *)dst + 128;
+		}
+
+		if (n >= 64) {
+			memcpy_prefetch64_1(src, dst);
+			n -= 64;
+			_rte_mov64(dst, src);
+			src = (const uint8_t *)src + 64;
+			dst = (uint8_t *)dst + 64;
+		}
+
+		if (n > 32) {
+			_rte_mov32(dst, src);
+			_rte_mov32((uint8_t *)dst - 32 + n,
+					(const uint8_t *)src - 32 + n);
+			return ret;
+		}
+
+		if (n > 0) {
+			_rte_mov32((uint8_t *)dst - 32 + n,
+					(const uint8_t *)src - 32 + n);
+		}
+		return ret;
+	}
+
+	/**
+	 * Make store aligned when copy size exceeds 256 bytes.
+	 */
+	dstofss = (uintptr_t)dst & ALIGNMENT_MASK_64;
+	if (dstofss > 0) {
+		dstofss = 64 - dstofss;
+		n -= dstofss;
+		_rte_mov64(dst, src);
+		src = (const uint8_t *)src + dstofss;
+		dst = (uint8_t *)dst + dstofss;
+	}
+
+	/**
+	 * Copy 128-byte blocks
+	 */
+	if ((uintptr_t)src & ALIGNMENT_MASK_64)	{
+		bn = n - (n & ALIGNMENT_MASK_128);
+		_rte_mov128blocks(dst, src, bn);
+		n = n & ALIGNMENT_MASK_128;
+		src = (const uint8_t *)src + bn;
+		dst = (uint8_t *)dst + bn;
+		_rte_mov(dst, src, n);
+	} else
+		_rte_mov_aligned(dst, src, n);
+
+	return ret;
+}
+
+static __rte_always_inline void *
+_rte_memcpy(void *dst, const void *src, size_t n)
+{
+	return _rte_memcpy_generic((uint8_t *)dst, (const uint8_t *)src, n);
+}
+#endif
+
+/*----------------------api---------------------------------------------------*/
+static __rte_always_inline void *
+rte_memcpy(void *dst, const void *src, size_t n)
+{
+#if defined(RTE_RISCV_FEATURE_V)
+	if (likely((memcpy_alg == MEMCPY_RISCV) && (n >= 128) && (n < 2048)))
+		return _rte_memcpy(dst, src, n);
+	/*else*/
+#endif
+		return memcpy(dst, src, n);
+}
+
 static inline void
 rte_mov16(uint8_t *dst, const uint8_t *src)
 {
@@ -51,10 +336,31 @@ rte_mov128(uint8_t *dst, const uint8_t *src)
 static inline void
 rte_mov256(uint8_t *dst, const uint8_t *src)
 {
-	memcpy(dst, src, 256);
+#if defined(RTE_RISCV_FEATURE_V)
+	if (likely(memcpy_alg == MEMCPY_RISCV))
+		_rte_mov256(dst, src);
+	else
+#endif
+		memcpy(dst, src, 256);
+}
+/*----------------------------------------------------------------------------*/
+#if defined(RTE_RISCV_FEATURE_V)
+static inline long
+riscv_vlenb(void)
+{
+	long vlenb;
+	asm ("csrr %0, 0xc22" : "=r"(vlenb));
+	return vlenb;
 }
 
-#define rte_memcpy(d, s, n)	memcpy((d), (s), (n))
+RTE_INIT(rte_vect_memcpy_init)
+{
+	long vlenb = riscv_vlenb();
+	if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_RISCV_ISA_V) && (vlenb >= RISCV_VLENB))
+		memcpy_alg = MEMCPY_RISCV;
+}
+#endif
+
 
 #ifdef __cplusplus
 }
-- 
2.21.0.windows.1

[-- Attachment #1.1.2: Type: text/html , Size: 26648 bytes --]