[RFC] eal: provide option to use compiler memcpy instead of RTE

[RFC] eal: provide option to use compiler memcpy instead of RTE

[Mattias Rönnblom]

Provide build option to have functions in <rte_memcpy.h> delegate to
the standard compiler/libc memcpy(), instead of using the various
traditional, handcrafted, per-architecture rte_memcpy()
implementations.

A new meson build option 'use_cc_memcpy' is added. The default is
true. It's not obvious what should be the default, but compiler
memcpy() is enabled by default in this RFC so any tests run with this
patch use the new approach.

One purpose of this RFC is to make it easy to evaluate the costs and
benefits of a switch.

Only ARM and x86 is implemented.

Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
---
 config/meson.build                   |  1 +
 lib/eal/arm/include/rte_memcpy.h     | 10 +++++
 lib/eal/include/generic/rte_memcpy.h | 62 ++++++++++++++++++++++++----
 lib/eal/x86/include/meson.build      |  6 ++-
 lib/eal/x86/include/rte_memcpy.h     | 11 ++++-
 meson_options.txt                    |  2 +
 6 files changed, 83 insertions(+), 9 deletions(-)

diff --git a/config/meson.build b/config/meson.build
index 8c8b019c25..456056628e 100644
--- a/config/meson.build
+++ b/config/meson.build
@@ -353,6 +353,7 @@ endforeach
 # set other values pulled from the build options
 dpdk_conf.set('RTE_MAX_ETHPORTS', get_option('max_ethports'))
 dpdk_conf.set('RTE_LIBEAL_USE_HPET', get_option('use_hpet'))
+dpdk_conf.set('RTE_USE_CC_MEMCPY', get_option('use_cc_memcpy'))
 dpdk_conf.set('RTE_ENABLE_STDATOMIC', get_option('enable_stdatomic'))
 dpdk_conf.set('RTE_ENABLE_TRACE_FP', get_option('enable_trace_fp'))
 dpdk_conf.set('RTE_PKTMBUF_HEADROOM', get_option('pkt_mbuf_headroom'))
diff --git a/lib/eal/arm/include/rte_memcpy.h b/lib/eal/arm/include/rte_memcpy.h
index 47dea9a8cc..e8aff722df 100644
--- a/lib/eal/arm/include/rte_memcpy.h
+++ b/lib/eal/arm/include/rte_memcpy.h
@@ -5,10 +5,20 @@
 #ifndef _RTE_MEMCPY_ARM_H_
 #define _RTE_MEMCPY_ARM_H_
 
+#include <rte_config.h>
+
+#ifdef RTE_USE_CC_MEMCPY
+
+#include <generic/rte_memcpy.h>
+
+#else
+
 #ifdef RTE_ARCH_64
 #include <rte_memcpy_64.h>
 #else
 #include <rte_memcpy_32.h>
 #endif
 
+#endif /* RTE_USE_CC_MEMCPY */
+
 #endif /* _RTE_MEMCPY_ARM_H_ */
diff --git a/lib/eal/include/generic/rte_memcpy.h b/lib/eal/include/generic/rte_memcpy.h
index e7f0f8eaa9..f2f66f372d 100644
--- a/lib/eal/include/generic/rte_memcpy.h
+++ b/lib/eal/include/generic/rte_memcpy.h
@@ -5,12 +5,20 @@
 #ifndef _RTE_MEMCPY_H_
 #define _RTE_MEMCPY_H_
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 /**
  * @file
  *
  * Functions for vectorised implementation of memcpy().
  */
 
+#include <stdint.h>
+#include <string.h>
+#include <rte_vect.h>
+
 /**
  * Copy 16 bytes from one location to another using optimised
  * instructions. The locations should not overlap.
@@ -35,8 +43,6 @@ rte_mov16(uint8_t *dst, const uint8_t *src);
 static inline void
 rte_mov32(uint8_t *dst, const uint8_t *src);
 
-#ifdef __DOXYGEN__
-
 /**
  * Copy 48 bytes from one location to another using optimised
  * instructions. The locations should not overlap.
@@ -49,8 +55,6 @@ rte_mov32(uint8_t *dst, const uint8_t *src);
 static inline void
 rte_mov48(uint8_t *dst, const uint8_t *src);
 
-#endif /* __DOXYGEN__ */
-
 /**
  * Copy 64 bytes from one location to another using optimised
  * instructions. The locations should not overlap.
@@ -87,8 +91,6 @@ rte_mov128(uint8_t *dst, const uint8_t *src);
 static inline void
 rte_mov256(uint8_t *dst, const uint8_t *src);
 
-#ifdef __DOXYGEN__
-
 /**
  * Copy bytes from one location to another. The locations must not overlap.
  *
@@ -111,6 +113,52 @@ rte_mov256(uint8_t *dst, const uint8_t *src);
 static void *
 rte_memcpy(void *dst, const void *src, size_t n);
 
-#endif /* __DOXYGEN__ */
+#ifdef RTE_USE_CC_MEMCPY
+static inline void
+rte_mov16(uint8_t *dst, const uint8_t *src)
+{
+	memcpy(dst, src, 16);
+}
+
+static inline void
+rte_mov32(uint8_t *dst, const uint8_t *src)
+{
+	memcpy(dst, src, 32);
+}
+
+static inline void
+rte_mov48(uint8_t *dst, const uint8_t *src)
+{
+	memcpy(dst, src, 48);
+}
+
+static inline void
+rte_mov64(uint8_t *dst, const uint8_t *src)
+{
+	memcpy(dst, src, 64);
+}
+
+static inline void
+rte_mov128(uint8_t *dst, const uint8_t *src)
+{
+	memcpy(dst, src, 128);
+}
+
+static inline void
+rte_mov256(uint8_t *dst, const uint8_t *src)
+{
+	memcpy(dst, src, 256);
+}
+
+static inline void *
+rte_memcpy(void *dst, const void *src, size_t n)
+{
+	return memcpy(dst, src, n);
+}
+#endif /* RTE_USE_CC_MEMCPY */
+
+#ifdef __cplusplus
+}
+#endif
 
 #endif /* _RTE_MEMCPY_H_ */
diff --git a/lib/eal/x86/include/meson.build b/lib/eal/x86/include/meson.build
index 52d2f8e969..cf851df60d 100644
--- a/lib/eal/x86/include/meson.build
+++ b/lib/eal/x86/include/meson.build
@@ -7,7 +7,6 @@ arch_headers = files(
         'rte_cpuflags.h',
         'rte_cycles.h',
         'rte_io.h',
-        'rte_memcpy.h',
         'rte_pause.h',
         'rte_power_intrinsics.h',
         'rte_prefetch.h',
@@ -16,6 +15,11 @@ arch_headers = files(
         'rte_spinlock.h',
         'rte_vect.h',
 )
+
+if not get_option('use_cc_memcpy')
+        arch_headers += 'rte_memcpy.h'
+endif
+
 arch_indirect_headers = files(
         'rte_atomic_32.h',
         'rte_atomic_64.h',
diff --git a/lib/eal/x86/include/rte_memcpy.h b/lib/eal/x86/include/rte_memcpy.h
index 72a92290e0..c5ba74d2ed 100644
--- a/lib/eal/x86/include/rte_memcpy.h
+++ b/lib/eal/x86/include/rte_memcpy.h
@@ -11,12 +11,19 @@
  * Functions for SSE/AVX/AVX2/AVX512 implementation of memcpy().
  */
 
+#include <rte_config.h>
+
+#ifdef RTE_USE_CC_MEMCPY
+
+#include <generic/rte_memcpy.h>
+
+#else
+
 #include <stdio.h>
 #include <stdint.h>
 #include <string.h>
 #include <rte_vect.h>
 #include <rte_common.h>
-#include <rte_config.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -878,4 +885,6 @@ rte_memcpy(void *dst, const void *src, size_t n)
 }
 #endif
 
+#endif /* RTE_USE_CC_MEMCPY */
+
 #endif /* _RTE_MEMCPY_X86_64_H_ */
diff --git a/meson_options.txt b/meson_options.txt
index e49b2fc089..263b0e7882 100644
--- a/meson_options.txt
+++ b/meson_options.txt
@@ -60,3 +60,5 @@ option('tests', type: 'boolean', value: true, description:
        'build unit tests')
 option('use_hpet', type: 'boolean', value: false, description:
        'use HPET timer in EAL')
+option('use_cc_memcpy', type: 'boolean', value: true, description:
+       'Have rte_memcpy() delegate to compiler/libc memcpy() instead of using custom implementation.')
-- 
2.34.1

[RFC v2] eal: provide option to use compiler memcpy instead of RTE

[Mattias Rönnblom]

Provide build option to have functions in <rte_memcpy.h> delegate to
the standard compiler/libc memcpy(), instead of using the various
traditional, handcrafted, per-architecture rte_memcpy()
implementations.

A new meson build option 'use_cc_memcpy' is added. The default is
true. It's not obvious what should be the default, but compiler
memcpy() is enabled by default in this RFC so any tests run with this
patch use the new approach.

One purpose of this RFC is to make it easy to evaluate the costs and
benefits of a switch.

Only Loongarch, ARM and x86 is implemented. Only x86 is tested.

RFC v2:
 * Fix bug where rte_memcpy.h was not installed on x86.
 * Made attempt to make Loongarch compile.

Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
---
 config/meson.build                     |  1 +
 lib/eal/arm/include/rte_memcpy.h       | 10 +++++
 lib/eal/include/generic/rte_memcpy.h   | 62 +++++++++++++++++++++++---
 lib/eal/loongarch/include/rte_memcpy.h | 52 ++-------------------
 lib/eal/x86/include/meson.build        |  1 +
 lib/eal/x86/include/rte_memcpy.h       | 11 ++++-
 meson_options.txt                      |  2 +
 7 files changed, 82 insertions(+), 57 deletions(-)

diff --git a/config/meson.build b/config/meson.build
index 8c8b019c25..456056628e 100644
--- a/config/meson.build
+++ b/config/meson.build
@@ -353,6 +353,7 @@ endforeach
 # set other values pulled from the build options
 dpdk_conf.set('RTE_MAX_ETHPORTS', get_option('max_ethports'))
 dpdk_conf.set('RTE_LIBEAL_USE_HPET', get_option('use_hpet'))
+dpdk_conf.set('RTE_USE_CC_MEMCPY', get_option('use_cc_memcpy'))
 dpdk_conf.set('RTE_ENABLE_STDATOMIC', get_option('enable_stdatomic'))
 dpdk_conf.set('RTE_ENABLE_TRACE_FP', get_option('enable_trace_fp'))
 dpdk_conf.set('RTE_PKTMBUF_HEADROOM', get_option('pkt_mbuf_headroom'))
diff --git a/lib/eal/arm/include/rte_memcpy.h b/lib/eal/arm/include/rte_memcpy.h
index 47dea9a8cc..e8aff722df 100644
--- a/lib/eal/arm/include/rte_memcpy.h
+++ b/lib/eal/arm/include/rte_memcpy.h
@@ -5,10 +5,20 @@
 #ifndef _RTE_MEMCPY_ARM_H_
 #define _RTE_MEMCPY_ARM_H_
 
+#include <rte_config.h>
+
+#ifdef RTE_USE_CC_MEMCPY
+
+#include <generic/rte_memcpy.h>
+
+#else
+
 #ifdef RTE_ARCH_64
 #include <rte_memcpy_64.h>
 #else
 #include <rte_memcpy_32.h>
 #endif
 
+#endif /* RTE_USE_CC_MEMCPY */
+
 #endif /* _RTE_MEMCPY_ARM_H_ */
diff --git a/lib/eal/include/generic/rte_memcpy.h b/lib/eal/include/generic/rte_memcpy.h
index e7f0f8eaa9..f2f66f372d 100644
--- a/lib/eal/include/generic/rte_memcpy.h
+++ b/lib/eal/include/generic/rte_memcpy.h
@@ -5,12 +5,20 @@
 #ifndef _RTE_MEMCPY_H_
 #define _RTE_MEMCPY_H_
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 /**
  * @file
  *
  * Functions for vectorised implementation of memcpy().
  */
 
+#include <stdint.h>
+#include <string.h>
+#include <rte_vect.h>
+
 /**
  * Copy 16 bytes from one location to another using optimised
  * instructions. The locations should not overlap.
@@ -35,8 +43,6 @@ rte_mov16(uint8_t *dst, const uint8_t *src);
 static inline void
 rte_mov32(uint8_t *dst, const uint8_t *src);
 
-#ifdef __DOXYGEN__
-
 /**
  * Copy 48 bytes from one location to another using optimised
  * instructions. The locations should not overlap.
@@ -49,8 +55,6 @@ rte_mov32(uint8_t *dst, const uint8_t *src);
 static inline void
 rte_mov48(uint8_t *dst, const uint8_t *src);
 
-#endif /* __DOXYGEN__ */
-
 /**
  * Copy 64 bytes from one location to another using optimised
  * instructions. The locations should not overlap.
@@ -87,8 +91,6 @@ rte_mov128(uint8_t *dst, const uint8_t *src);
 static inline void
 rte_mov256(uint8_t *dst, const uint8_t *src);
 
-#ifdef __DOXYGEN__
-
 /**
  * Copy bytes from one location to another. The locations must not overlap.
  *
@@ -111,6 +113,52 @@ rte_mov256(uint8_t *dst, const uint8_t *src);
 static void *
 rte_memcpy(void *dst, const void *src, size_t n);
 
-#endif /* __DOXYGEN__ */
+#ifdef RTE_USE_CC_MEMCPY
+static inline void
+rte_mov16(uint8_t *dst, const uint8_t *src)
+{
+	memcpy(dst, src, 16);
+}
+
+static inline void
+rte_mov32(uint8_t *dst, const uint8_t *src)
+{
+	memcpy(dst, src, 32);
+}
+
+static inline void
+rte_mov48(uint8_t *dst, const uint8_t *src)
+{
+	memcpy(dst, src, 48);
+}
+
+static inline void
+rte_mov64(uint8_t *dst, const uint8_t *src)
+{
+	memcpy(dst, src, 64);
+}
+
+static inline void
+rte_mov128(uint8_t *dst, const uint8_t *src)
+{
+	memcpy(dst, src, 128);
+}
+
+static inline void
+rte_mov256(uint8_t *dst, const uint8_t *src)
+{
+	memcpy(dst, src, 256);
+}
+
+static inline void *
+rte_memcpy(void *dst, const void *src, size_t n)
+{
+	return memcpy(dst, src, n);
+}
+#endif /* RTE_USE_CC_MEMCPY */
+
+#ifdef __cplusplus
+}
+#endif
 
 #endif /* _RTE_MEMCPY_H_ */
diff --git a/lib/eal/loongarch/include/rte_memcpy.h b/lib/eal/loongarch/include/rte_memcpy.h
index 22578d40f4..159420d3b7 100644
--- a/lib/eal/loongarch/include/rte_memcpy.h
+++ b/lib/eal/loongarch/include/rte_memcpy.h
@@ -5,57 +5,11 @@
 #ifndef RTE_MEMCPY_LOONGARCH_H
 #define RTE_MEMCPY_LOONGARCH_H
 
-#include <stdint.h>
-#include <string.h>
+#include "rte_config.h"
 
-#include "rte_common.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
+#ifndef RTE_USE_CC_MEMCPY
+#define RTE_USE_CC_MEMCPY
 
 #include "generic/rte_memcpy.h"
 
-static inline void
-rte_mov16(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 16);
-}
-
-static inline void
-rte_mov32(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 32);
-}
-
-static inline void
-rte_mov48(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 48);
-}
-
-static inline void
-rte_mov64(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 64);
-}
-
-static inline void
-rte_mov128(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 128);
-}
-
-static inline void
-rte_mov256(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 256);
-}
-
-#define rte_memcpy(d, s, n)	memcpy((d), (s), (n))
-
-#ifdef __cplusplus
-}
-#endif
-
 #endif /* RTE_MEMCPY_LOONGARCH_H */
diff --git a/lib/eal/x86/include/meson.build b/lib/eal/x86/include/meson.build
index 52d2f8e969..09c2fe2485 100644
--- a/lib/eal/x86/include/meson.build
+++ b/lib/eal/x86/include/meson.build
@@ -16,6 +16,7 @@ arch_headers = files(
         'rte_spinlock.h',
         'rte_vect.h',
 )
+
 arch_indirect_headers = files(
         'rte_atomic_32.h',
         'rte_atomic_64.h',
diff --git a/lib/eal/x86/include/rte_memcpy.h b/lib/eal/x86/include/rte_memcpy.h
index 72a92290e0..c5ba74d2ed 100644
--- a/lib/eal/x86/include/rte_memcpy.h
+++ b/lib/eal/x86/include/rte_memcpy.h
@@ -11,12 +11,19 @@
  * Functions for SSE/AVX/AVX2/AVX512 implementation of memcpy().
  */
 
+#include <rte_config.h>
+
+#ifdef RTE_USE_CC_MEMCPY
+
+#include <generic/rte_memcpy.h>
+
+#else
+
 #include <stdio.h>
 #include <stdint.h>
 #include <string.h>
 #include <rte_vect.h>
 #include <rte_common.h>
-#include <rte_config.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -878,4 +885,6 @@ rte_memcpy(void *dst, const void *src, size_t n)
 }
 #endif
 
+#endif /* RTE_USE_CC_MEMCPY */
+
 #endif /* _RTE_MEMCPY_X86_64_H_ */
diff --git a/meson_options.txt b/meson_options.txt
index e49b2fc089..263b0e7882 100644
--- a/meson_options.txt
+++ b/meson_options.txt
@@ -60,3 +60,5 @@ option('tests', type: 'boolean', value: true, description:
        'build unit tests')
 option('use_hpet', type: 'boolean', value: false, description:
        'use HPET timer in EAL')
+option('use_cc_memcpy', type: 'boolean', value: true, description:
+       'Have rte_memcpy() delegate to compiler/libc memcpy() instead of using custom implementation.')
-- 
2.34.1

Re: [RFC v2] eal: provide option to use compiler memcpy instead of RTE

[Mattias Rönnblom]

On 2024-05-28 09:43, Mattias Rönnblom wrote:
> Provide build option to have functions in <rte_memcpy.h> delegate to
> the standard compiler/libc memcpy(), instead of using the various
> traditional, handcrafted, per-architecture rte_memcpy()
> implementations.
> 
> A new meson build option 'use_cc_memcpy' is added. The default is
> true. It's not obvious what should be the default, but compiler
> memcpy() is enabled by default in this RFC so any tests run with this
> patch use the new approach.
> 
> One purpose of this RFC is to make it easy to evaluate the costs and
> benefits of a switch.
> 

I've tested this patch some with DSW micro benchmarks, and the result is 
a 2.5% reduction of the DSW+testapp overhead with cc/libc memcpy. GCC 11.4.

We've also run characteristic test suite of a large, real world app. 
Here, we saw no effect. GCC 10.5.

x86_64 in both cases (Skylake and Raptor Lake).

Last time we did the same, there were a noticeable performance 
degradation in both the above cases.

This is not a lot of data points, but I think it we should consider 
making the custom RTE memcpy() implementations optional in the next 
release, and if no-one complains, remove the implementations in the next 
release.

(Whether or not [or how long] to keep the wrapper API is another question.)

<snip>

Re: [RFC v2] eal: provide option to use compiler memcpy instead of RTE

[Bruce Richardson]

On Tue, May 28, 2024 at 09:43:54AM +0200, Mattias Rönnblom wrote:
> Provide build option to have functions in <rte_memcpy.h> delegate to
> the standard compiler/libc memcpy(), instead of using the various
> traditional, handcrafted, per-architecture rte_memcpy()
> implementations.
> 
> A new meson build option 'use_cc_memcpy' is added. The default is
> true. It's not obvious what should be the default, but compiler
> memcpy() is enabled by default in this RFC so any tests run with this
> patch use the new approach.
> 
> One purpose of this RFC is to make it easy to evaluate the costs and
> benefits of a switch.
> 
> Only Loongarch, ARM and x86 is implemented. Only x86 is tested.
> 
> RFC v2:
>  * Fix bug where rte_memcpy.h was not installed on x86.
>  * Made attempt to make Loongarch compile.
> 
> Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
> ---
>  config/meson.build                     |  1 +
>  lib/eal/arm/include/rte_memcpy.h       | 10 +++++
>  lib/eal/include/generic/rte_memcpy.h   | 62 +++++++++++++++++++++++---
>  lib/eal/loongarch/include/rte_memcpy.h | 52 ++-------------------
>  lib/eal/x86/include/meson.build        |  1 +
>  lib/eal/x86/include/rte_memcpy.h       | 11 ++++-
>  meson_options.txt                      |  2 +
>  7 files changed, 82 insertions(+), 57 deletions(-)
> 

I really support the long-term goal here of eliminating the need for us to
maintain our own memcpy. This looks a good idea to see how things perform.
If we do decide to take this patch, having the default be regular memcpy
should help with static analysis and other tooling, which would be aware of
memcpy but not rte_memcpy.

/Bruce

Re: [RFC v2] eal: provide option to use compiler memcpy instead of RTE

[Bruce Richardson]

On Tue, May 28, 2024 at 10:19:15AM +0200, Mattias Rönnblom wrote:
> On 2024-05-28 09:43, Mattias Rönnblom wrote:
> > Provide build option to have functions in <rte_memcpy.h> delegate to
> > the standard compiler/libc memcpy(), instead of using the various
> > traditional, handcrafted, per-architecture rte_memcpy()
> > implementations.
> > 
> > A new meson build option 'use_cc_memcpy' is added. The default is
> > true. It's not obvious what should be the default, but compiler
> > memcpy() is enabled by default in this RFC so any tests run with this
> > patch use the new approach.
> > 
> > One purpose of this RFC is to make it easy to evaluate the costs and
> > benefits of a switch.
> > 
> 
> I've tested this patch some with DSW micro benchmarks, and the result is a
> 2.5% reduction of the DSW+testapp overhead with cc/libc memcpy. GCC 11.4.
> 
> We've also run characteristic test suite of a large, real world app. Here,
> we saw no effect. GCC 10.5.
> 
> x86_64 in both cases (Skylake and Raptor Lake).
> 
> Last time we did the same, there were a noticeable performance degradation
> in both the above cases.
> 
> This is not a lot of data points, but I think it we should consider making
> the custom RTE memcpy() implementations optional in the next release, and if
> no-one complains, remove the implementations in the next release.
> 
> (Whether or not [or how long] to keep the wrapper API is another question.)
> 
> <snip>

The other instance I've heard mention of in the past is virtio/vhost, which
used to have a speedup from the custom memcpy.

My own thinking on these cases, is that for targetted settings like these,
we should look to have local memcpy functions written - taking account of
the specifics of each usecase. For virtio/vhost for example, we can have
assumptions around host buffer alignment, and we also can be pretty
confident we are copying to another CPU. For DSW, or other eventdev cases,
we would only be looking at copies of multiples of 16, with guaranteed
8-byte alignment on both source and destination. Writing efficient copy fns
for specific scenarios can be faster and more effective than trying to
write a general, optimized in all cases, memcpy. It also discourages the
use of non-libc memcpy except where really necessary.

Naturally, if we find there are a lot of cases where use of libc memcpy
slows us down, we will want to keep a general rte_memcpy. However, I'd hope
the slowdown cases are very few.

/Bruce

Re: [RFC v2] eal: provide option to use compiler memcpy instead of RTE

[Mattias Rönnblom]

On 2024-05-28 10:27, Bruce Richardson wrote:
> On Tue, May 28, 2024 at 10:19:15AM +0200, Mattias Rönnblom wrote:
>> On 2024-05-28 09:43, Mattias Rönnblom wrote:
>>> Provide build option to have functions in <rte_memcpy.h> delegate to
>>> the standard compiler/libc memcpy(), instead of using the various
>>> traditional, handcrafted, per-architecture rte_memcpy()
>>> implementations.
>>>
>>> A new meson build option 'use_cc_memcpy' is added. The default is
>>> true. It's not obvious what should be the default, but compiler
>>> memcpy() is enabled by default in this RFC so any tests run with this
>>> patch use the new approach.
>>>
>>> One purpose of this RFC is to make it easy to evaluate the costs and
>>> benefits of a switch.
>>>
>>
>> I've tested this patch some with DSW micro benchmarks, and the result is a
>> 2.5% reduction of the DSW+testapp overhead with cc/libc memcpy. GCC 11.4.
>>
>> We've also run characteristic test suite of a large, real world app. Here,
>> we saw no effect. GCC 10.5.
>>
>> x86_64 in both cases (Skylake and Raptor Lake).
>>
>> Last time we did the same, there were a noticeable performance degradation
>> in both the above cases.
>>
>> This is not a lot of data points, but I think it we should consider making
>> the custom RTE memcpy() implementations optional in the next release, and if
>> no-one complains, remove the implementations in the next release.
>>
>> (Whether or not [or how long] to keep the wrapper API is another question.)
>>
>> <snip>
> 
> The other instance I've heard mention of in the past is virtio/vhost, which
> used to have a speedup from the custom memcpy.
> 
> My own thinking on these cases, is that for targetted settings like these,
> we should look to have local memcpy functions written - taking account of
> the specifics of each usecase. For virtio/vhost for example, we can have
> assumptions around host buffer alignment, and we also can be pretty
> confident we are copying to another CPU. For DSW, or other eventdev cases,
> we would only be looking at copies of multiples of 16, with guaranteed
> 8-byte alignment on both source and destination. Writing efficient copy fns

In such cases, you should first try to tell the compiler that it's safe 
to assume that the pointers have a certain alignment.

void copy256(void *dst, const void *src)
{
     memcpy(dst, src, 256);
}

void copy256_a(void *dst, const void *src)
{
     void *dst_a = __builtin_assume_aligned(dst, 32);
     const void *src_a = __builtin_assume_aligned(src, 32);
     memcpy(dst_a, src_a, 256);
}

The first will generate loads/stores without alignment restrictions, 
while the latter will use things like vmovdqa or vmovaps.

(I doubt there's much of a performance difference though, if any at all.)

> for specific scenarios can be faster and more effective than trying to
> write a general, optimized in all cases, memcpy. It also discourages the
> use of non-libc memcpy except where really necessary.
> 
> Naturally, if we find there are a lot of cases where use of libc memcpy
> slows us down, we will want to keep a general rte_memcpy. However, I'd hope
> the slowdown cases are very few.
> 
> /Bruce

RE: [RFC v2] eal: provide option to use compiler memcpy instead of RTE

[Morten Brørup]

> From: Mattias Rönnblom [mailto:hofors@lysator.liu.se]
> Sent: Tuesday, 28 May 2024 11.00
> 
> On 2024-05-28 10:27, Bruce Richardson wrote:
> > On Tue, May 28, 2024 at 10:19:15AM +0200, Mattias Rönnblom wrote:
> >> On 2024-05-28 09:43, Mattias Rönnblom wrote:
> >>> Provide build option to have functions in <rte_memcpy.h> delegate to
> >>> the standard compiler/libc memcpy(), instead of using the various
> >>> traditional, handcrafted, per-architecture rte_memcpy()
> >>> implementations.
> >>>
> >>> A new meson build option 'use_cc_memcpy' is added. The default is
> >>> true. It's not obvious what should be the default, but compiler
> >>> memcpy() is enabled by default in this RFC so any tests run with this
> >>> patch use the new approach.
> >>>
> >>> One purpose of this RFC is to make it easy to evaluate the costs and
> >>> benefits of a switch.
> >>>
> >>
> >> I've tested this patch some with DSW micro benchmarks, and the result is a
> >> 2.5% reduction of the DSW+testapp overhead with cc/libc memcpy. GCC 11.4.
> >>
> >> We've also run characteristic test suite of a large, real world app. Here,
> >> we saw no effect. GCC 10.5.
> >>
> >> x86_64 in both cases (Skylake and Raptor Lake).
> >>
> >> Last time we did the same, there were a noticeable performance degradation
> >> in both the above cases.

Mattias, which compiler was that?

As previously mentioned in another thread, I'm worried about memcpy performance with older compilers.
DPDK officially supports GCC 4.9 and clang 3.4 [1].
I don't think degrading performance when using supported compilers is considered acceptable.

Alternatively, we could change the DPDK compiler policy from "supported" to "works with (but might not perform optimally)".

[1]: https://doc.dpdk.org/guides-21.11/linux_gsg/sys_reqs.html#compilation-of-the-dpdk

> >>
> >> This is not a lot of data points, but I think it we should consider making
> >> the custom RTE memcpy() implementations optional in the next release, and
> if
> >> no-one complains, remove the implementations in the next release.
> >>
> >> (Whether or not [or how long] to keep the wrapper API is another question.)
> >>
> >> <snip>
> >
> > The other instance I've heard mention of in the past is virtio/vhost, which
> > used to have a speedup from the custom memcpy.
> >
> > My own thinking on these cases, is that for targetted settings like these,
> > we should look to have local memcpy functions written - taking account of
> > the specifics of each usecase. For virtio/vhost for example, we can have
> > assumptions around host buffer alignment, and we also can be pretty
> > confident we are copying to another CPU. For DSW, or other eventdev cases,
> > we would only be looking at copies of multiples of 16, with guaranteed
> > 8-byte alignment on both source and destination. Writing efficient copy fns
> 
> In such cases, you should first try to tell the compiler that it's safe
> to assume that the pointers have a certain alignment.
> 
> void copy256(void *dst, const void *src)
> {
>      memcpy(dst, src, 256);
> }
> 
> void copy256_a(void *dst, const void *src)
> {
>      void *dst_a = __builtin_assume_aligned(dst, 32);
>      const void *src_a = __builtin_assume_aligned(src, 32);
>      memcpy(dst_a, src_a, 256);
> }
> 
> The first will generate loads/stores without alignment restrictions,
> while the latter will use things like vmovdqa or vmovaps.
> 
> (I doubt there's much of a performance difference though, if any at all.)

Interesting.

> 
> > for specific scenarios can be faster and more effective than trying to
> > write a general, optimized in all cases, memcpy. It also discourages the
> > use of non-libc memcpy except where really necessary.

Good idea, Bruce.
I have previously worked on an optimized memcpy, where information about alignment, multiples, non-temporal source/destination, etc. is passed as flags to the function [2]. But it turned into too much work, so I never finished it.

If we start with local memcpy functions optimized for each specific use case, we still have the option of consolidating them into a common rte_memcpy function later. It will also reveal which flags/features such a common function needs to support.

[2]: https://inbox.dpdk.org/dev/20221010064600.16495-1-mb@smartsharesystems.com/

> >
> > Naturally, if we find there are a lot of cases where use of libc memcpy
> > slows us down, we will want to keep a general rte_memcpy. However, I'd hope
> > the slowdown cases are very few.
> >
> > /Bruce

Re: [RFC v2] eal: provide option to use compiler memcpy instead of RTE

[Stephen Hemminger]

On Tue, 28 May 2024 10:19:15 +0200
Mattias Rönnblom <hofors@lysator.liu.se> wrote:

> >   
> 
> I've tested this patch some with DSW micro benchmarks, and the result is 
> a 2.5% reduction of the DSW+testapp overhead with cc/libc memcpy. GCC 11.4.
> 
> We've also run characteristic test suite of a large, real world app. 
> Here, we saw no effect. GCC 10.5.
> 
> x86_64 in both cases (Skylake and Raptor Lake).
> 
> Last time we did the same, there were a noticeable performance 
> degradation in both the above cases.
> 
> This is not a lot of data points, but I think it we should consider 
> making the custom RTE memcpy() implementations optional in the next 
> release, and if no-one complains, remove the implementations in the next 
> release.

Lets go farther.

1. Announce that rte_memcpy will be marked deprecated in 24.11 release

2. In 24.11 do a global replace of rte_memcpy on the tree.
   And mark rte_memcpy as deprecated.

3. In 25.11 it can go away.

Re: [RFC v2] eal: provide option to use compiler memcpy instead of RTE

[Bruce Richardson]

On Tue, May 28, 2024 at 07:59:36AM -0700, Stephen Hemminger wrote:
> On Tue, 28 May 2024 10:19:15 +0200
> Mattias Rönnblom <hofors@lysator.liu.se> wrote:
> 
> > >   
> > 
> > I've tested this patch some with DSW micro benchmarks, and the result is 
> > a 2.5% reduction of the DSW+testapp overhead with cc/libc memcpy. GCC 11.4.
> > 
> > We've also run characteristic test suite of a large, real world app. 
> > Here, we saw no effect. GCC 10.5.
> > 
> > x86_64 in both cases (Skylake and Raptor Lake).
> > 
> > Last time we did the same, there were a noticeable performance 
> > degradation in both the above cases.
> > 
> > This is not a lot of data points, but I think it we should consider 
> > making the custom RTE memcpy() implementations optional in the next 
> > release, and if no-one complains, remove the implementations in the next 
> > release.
> 
> Lets go farther.
> 
> 1. Announce that rte_memcpy will be marked deprecated in 24.11 release
> 
> 2. In 24.11 do a global replace of rte_memcpy on the tree.
>    And mark rte_memcpy as deprecated.
> 
> 3. In 25.11 it can go away.

While I'd like us to be able to do so, I believe that to be premature. We
need to see where/if there are regressions first, and see about fixing
them.

/Bruce

Re: [RFC v2] eal: provide option to use compiler memcpy instead of RTE

[Mattias Rönnblom]

On 2024-05-28 16:59, Stephen Hemminger wrote:
> On Tue, 28 May 2024 10:19:15 +0200
> Mattias Rönnblom <hofors@lysator.liu.se> wrote:
> 
>>>    
>>
>> I've tested this patch some with DSW micro benchmarks, and the result is
>> a 2.5% reduction of the DSW+testapp overhead with cc/libc memcpy. GCC 11.4.
>>
>> We've also run characteristic test suite of a large, real world app.
>> Here, we saw no effect. GCC 10.5.
>>
>> x86_64 in both cases (Skylake and Raptor Lake).
>>
>> Last time we did the same, there were a noticeable performance
>> degradation in both the above cases.
>>
>> This is not a lot of data points, but I think it we should consider
>> making the custom RTE memcpy() implementations optional in the next
>> release, and if no-one complains, remove the implementations in the next
>> release.
> 
> Lets go farther.
> 
> 1. Announce that rte_memcpy will be marked deprecated in 24.11 release
> 
> 2. In 24.11 do a global replace of rte_memcpy on the tree.
>     And mark rte_memcpy as deprecated.
> 
> 3. In 25.11 it can go away.

If/when rte_memcpy.h is just a tiny memcpy() wrapper, the maintenance 
burden is pretty much eliminated.

Keeping it around will allow for older applications to compile against 
newer DPDK version.

You can always discourage its use in the API documentation.

Also, hopefully, some day, we will have a non-temporal memcpy(), and 
those functions needs a home.

Re: [RFC v2] eal: provide option to use compiler memcpy instead of RTE

[Mattias Rönnblom]

On 2024-05-28 11:07, Morten Brørup wrote:
>> From: Mattias Rönnblom [mailto:hofors@lysator.liu.se]
>> Sent: Tuesday, 28 May 2024 11.00
>>
>> On 2024-05-28 10:27, Bruce Richardson wrote:
>>> On Tue, May 28, 2024 at 10:19:15AM +0200, Mattias Rönnblom wrote:
>>>> On 2024-05-28 09:43, Mattias Rönnblom wrote:
>>>>> Provide build option to have functions in <rte_memcpy.h> delegate to
>>>>> the standard compiler/libc memcpy(), instead of using the various
>>>>> traditional, handcrafted, per-architecture rte_memcpy()
>>>>> implementations.
>>>>>
>>>>> A new meson build option 'use_cc_memcpy' is added. The default is
>>>>> true. It's not obvious what should be the default, but compiler
>>>>> memcpy() is enabled by default in this RFC so any tests run with this
>>>>> patch use the new approach.
>>>>>
>>>>> One purpose of this RFC is to make it easy to evaluate the costs and
>>>>> benefits of a switch.
>>>>>
>>>>
>>>> I've tested this patch some with DSW micro benchmarks, and the result is a
>>>> 2.5% reduction of the DSW+testapp overhead with cc/libc memcpy. GCC 11.4.
>>>>
>>>> We've also run characteristic test suite of a large, real world app. Here,
>>>> we saw no effect. GCC 10.5.
>>>>
>>>> x86_64 in both cases (Skylake and Raptor Lake).
>>>>
>>>> Last time we did the same, there were a noticeable performance degradation
>>>> in both the above cases.
> 
> Mattias, which compiler was that?
> 

GCC 9, I think.

Not only the compiler changed between those two test runs.

It would be interesting with some ARM data points as well.

> As previously mentioned in another thread, I'm worried about memcpy performance with older compilers.
> DPDK officially supports GCC 4.9 and clang 3.4 [1].
> I don't think degrading performance when using supported compilers is considered acceptable.
> 
> Alternatively, we could change the DPDK compiler policy from "supported" to "works with (but might not perform optimally)".
> 

GCC 4.9 is ten years old.

If you are using an old compiler, odds are you don't really care too 
much about squeezing out max performance, considering how much better 
code generation is in newer compilers.

That said, we obviously don't want to cause large performance 
regressions for no good reason, even for old compilers.

> [1]: https://doc.dpdk.org/guides-21.11/linux_gsg/sys_reqs.html#compilation-of-the-dpdk
> 
>>>>
>>>> This is not a lot of data points, but I think it we should consider making
>>>> the custom RTE memcpy() implementations optional in the next release, and
>> if
>>>> no-one complains, remove the implementations in the next release.
>>>>
>>>> (Whether or not [or how long] to keep the wrapper API is another question.)
>>>>
>>>> <snip>
>>>
>>> The other instance I've heard mention of in the past is virtio/vhost, which
>>> used to have a speedup from the custom memcpy.
>>>
>>> My own thinking on these cases, is that for targetted settings like these,
>>> we should look to have local memcpy functions written - taking account of
>>> the specifics of each usecase. For virtio/vhost for example, we can have
>>> assumptions around host buffer alignment, and we also can be pretty
>>> confident we are copying to another CPU. For DSW, or other eventdev cases,
>>> we would only be looking at copies of multiples of 16, with guaranteed
>>> 8-byte alignment on both source and destination. Writing efficient copy fns
>>
>> In such cases, you should first try to tell the compiler that it's safe
>> to assume that the pointers have a certain alignment.
>>
>> void copy256(void *dst, const void *src)
>> {
>>       memcpy(dst, src, 256);
>> }
>>
>> void copy256_a(void *dst, const void *src)
>> {
>>       void *dst_a = __builtin_assume_aligned(dst, 32);
>>       const void *src_a = __builtin_assume_aligned(src, 32);
>>       memcpy(dst_a, src_a, 256);
>> }
>>
>> The first will generate loads/stores without alignment restrictions,
>> while the latter will use things like vmovdqa or vmovaps.
>>
>> (I doubt there's much of a performance difference though, if any at all.)
> 
> Interesting.
> 
>>
>>> for specific scenarios can be faster and more effective than trying to
>>> write a general, optimized in all cases, memcpy. It also discourages the
>>> use of non-libc memcpy except where really necessary.
> 
> Good idea, Bruce.
> I have previously worked on an optimized memcpy, where information about alignment, multiples, non-temporal source/destination, etc. is passed as flags to the function [2]. But it turned into too much work, so I never finished it.
> 
> If we start with local memcpy functions optimized for each specific use case, we still have the option of consolidating them into a common rte_memcpy function later. It will also reveal which flags/features such a common function needs to support.
> 
> [2]: https://inbox.dpdk.org/dev/20221010064600.16495-1-mb@smartsharesystems.com/
> 
>>>
>>> Naturally, if we find there are a lot of cases where use of libc memcpy
>>> slows us down, we will want to keep a general rte_memcpy. However, I'd hope
>>> the slowdown cases are very few.
>>>
>>> /Bruce

Re: [RFC v2] eal: provide option to use compiler memcpy instead of RTE

[Stephen Hemminger]

On Tue, 28 May 2024 18:03:27 +0200
Mattias Rönnblom <hofors@lysator.liu.se> wrote:

> > Lets go farther.
> > 
> > 1. Announce that rte_memcpy will be marked deprecated in 24.11 release
> > 
> > 2. In 24.11 do a global replace of rte_memcpy on the tree.
> >     And mark rte_memcpy as deprecated.
> > 
> > 3. In 25.11 it can go away.  
> 
> If/when rte_memcpy.h is just a tiny memcpy() wrapper, the maintenance 
> burden is pretty much eliminated.
> 
> Keeping it around will allow for older applications to compile against 
> newer DPDK version.
> 
> You can always discourage its use in the API documentation.
> 
> Also, hopefully, some day, we will have a non-temporal memcpy(), and 
> those functions needs a home.

I was thinking a wrapper but have it marked __rte_deprecated so that
warnings result and people are motivated to fix.

Re: [RFC] eal: provide option to use compiler memcpy instead of RTE

[Stephen Hemminger]

On Mon, 27 May 2024 13:11:51 +0200
Mattias Rönnblom <mattias.ronnblom@ericsson.com> wrote:

> #ifdef RTE_USE_CC_MEMCPY
> +static inline void
> +rte_mov16(uint8_t *dst, const uint8_t *src)
> +{
> +	memcpy(dst, src, 16);
> +}
> +
> +static inline void
> +rte_mov32(uint8_t *dst, const uint8_t *src)
> +{
> +	memcpy(dst, src, 32);
> +}
> +
> +static inline void
> +rte_mov48(uint8_t *dst, const uint8_t *src)
> +{
> +	memcpy(dst, src, 48);
> +}
> +
> +static inline void
> +rte_mov64(uint8_t *dst, const uint8_t *src)
> +{
> +	memcpy(dst, src, 64);
> +}
> +
> +static inline void
> +rte_mov128(uint8_t *dst, const uint8_t *src)
> +{
> +	memcpy(dst, src, 128);
> +}
> +
> +static inline void
> +rte_mov256(uint8_t *dst, const uint8_t *src)
> +{
> +	memcpy(dst, src, 256);
> +}
> +
> +static inline void *
> +rte_memcpy(void *dst, const void *src, size_t n)
> +{
> +	return memcpy(dst, src, n);
> +}
> +#endif /* RTE_USE_CC_MEMCPY */
> +
> +#ifdef __cplusplus
> +}
> +#endif

You may need to make these macros to fully engage the checking
options of GCC, fortify, coverity etc. Not sure if all the tools
are smart enough to see through an inline.

Re: [RFC v2] eal: provide option to use compiler memcpy instead of RTE

[Mattias Rönnblom]

On 2024-05-28 17:09, Bruce Richardson wrote:
> On Tue, May 28, 2024 at 07:59:36AM -0700, Stephen Hemminger wrote:
>> On Tue, 28 May 2024 10:19:15 +0200
>> Mattias Rönnblom <hofors@lysator.liu.se> wrote:
>>
>>>>    
>>>
>>> I've tested this patch some with DSW micro benchmarks, and the result is
>>> a 2.5% reduction of the DSW+testapp overhead with cc/libc memcpy. GCC 11.4.
>>>
>>> We've also run characteristic test suite of a large, real world app.
>>> Here, we saw no effect. GCC 10.5.
>>>
>>> x86_64 in both cases (Skylake and Raptor Lake).
>>>
>>> Last time we did the same, there were a noticeable performance
>>> degradation in both the above cases.
>>>
>>> This is not a lot of data points, but I think it we should consider
>>> making the custom RTE memcpy() implementations optional in the next
>>> release, and if no-one complains, remove the implementations in the next
>>> release.
>>
>> Lets go farther.
>>
>> 1. Announce that rte_memcpy will be marked deprecated in 24.11 release
>>
>> 2. In 24.11 do a global replace of rte_memcpy on the tree.
>>     And mark rte_memcpy as deprecated.
>>
>> 3. In 25.11 it can go away.
> 
> While I'd like us to be able to do so, I believe that to be premature. We
> need to see where/if there are regressions first, and see about fixing
> them.
> 
> /Bruce

Should I turn this RFC into a PATCH?

Is use_cc_memcpy a good name for the configuration parameter?

Re: [RFC v2] eal: provide option to use compiler memcpy instead of RTE

[Stephen Hemminger]

On Fri, 31 May 2024 07:19:41 +0200
Mattias Rönnblom <hofors@lysator.liu.se> wrote:

> On 2024-05-28 17:09, Bruce Richardson wrote:
> > On Tue, May 28, 2024 at 07:59:36AM -0700, Stephen Hemminger wrote:  
> >> On Tue, 28 May 2024 10:19:15 +0200
> >> Mattias Rönnblom <hofors@lysator.liu.se> wrote:
> >>  
> >>>>      
> >>>
> >>> I've tested this patch some with DSW micro benchmarks, and the result is
> >>> a 2.5% reduction of the DSW+testapp overhead with cc/libc memcpy. GCC 11.4.
> >>>
> >>> We've also run characteristic test suite of a large, real world app.
> >>> Here, we saw no effect. GCC 10.5.
> >>>
> >>> x86_64 in both cases (Skylake and Raptor Lake).
> >>>
> >>> Last time we did the same, there were a noticeable performance
> >>> degradation in both the above cases.
> >>>
> >>> This is not a lot of data points, but I think it we should consider
> >>> making the custom RTE memcpy() implementations optional in the next
> >>> release, and if no-one complains, remove the implementations in the next
> >>> release.  
> >>
> >> Lets go farther.
> >>
> >> 1. Announce that rte_memcpy will be marked deprecated in 24.11 release
> >>
> >> 2. In 24.11 do a global replace of rte_memcpy on the tree.
> >>     And mark rte_memcpy as deprecated.
> >>
> >> 3. In 25.11 it can go away.  
> > 
> > While I'd like us to be able to do so, I believe that to be premature. We
> > need to see where/if there are regressions first, and see about fixing
> > them.
> > 
> > /Bruce  
> 
> Should I turn this RFC into a PATCH?
> 
> Is use_cc_memcpy a good name for the configuration parameter?
> 

I did a slightly more direct test and found a couple of things:
   1. Ena driver is redefining memcpy as rte_memcpy, this should be removed and should have
      been blocked during code review.
   2. A couple of drivers are implicitly expecting simd vector routines to be available.
      This works because rte_memcpy.h includes rte_vect.h.  The fix is to have these
      places include rte_vect.h

Re: [RFC] eal: provide option to use compiler memcpy instead of RTE

[Mattias Rönnblom]

On 2024-05-29 23:56, Stephen Hemminger wrote:
> On Mon, 27 May 2024 13:11:51 +0200
> Mattias Rönnblom <mattias.ronnblom@ericsson.com> wrote:
> 
>> #ifdef RTE_USE_CC_MEMCPY
>> +static inline void
>> +rte_mov16(uint8_t *dst, const uint8_t *src)
>> +{
>> +	memcpy(dst, src, 16);
>> +}
>> +
>> +static inline void
>> +rte_mov32(uint8_t *dst, const uint8_t *src)
>> +{
>> +	memcpy(dst, src, 32);
>> +}
>> +
>> +static inline void
>> +rte_mov48(uint8_t *dst, const uint8_t *src)
>> +{
>> +	memcpy(dst, src, 48);
>> +}
>> +
>> +static inline void
>> +rte_mov64(uint8_t *dst, const uint8_t *src)
>> +{
>> +	memcpy(dst, src, 64);
>> +}
>> +
>> +static inline void
>> +rte_mov128(uint8_t *dst, const uint8_t *src)
>> +{
>> +	memcpy(dst, src, 128);
>> +}
>> +
>> +static inline void
>> +rte_mov256(uint8_t *dst, const uint8_t *src)
>> +{
>> +	memcpy(dst, src, 256);
>> +}
>> +
>> +static inline void *
>> +rte_memcpy(void *dst, const void *src, size_t n)
>> +{
>> +	return memcpy(dst, src, n);
>> +}
>> +#endif /* RTE_USE_CC_MEMCPY */
>> +
>> +#ifdef __cplusplus
>> +}
>> +#endif
> 
> You may need to make these macros to fully engage the checking
> options of GCC, fortify, coverity etc. Not sure if all the tools
> are smart enough to see through an inline.

At least GCC is, provided you compile with optimization enabled. That 
goes for both overlapping memcpy() warning and static buffer overruns.

clang doesn't warn about overlapping memcpy() and fails to follow 
function calls, even with optimization enabled, seemingly. Same for ICX.

Static analysis tools that can't beat the compiler seems like they would 
be of limited use.

With macros you'll lose the type checking, which in the rte_memcpy() 
case doesn't matter, but rte_mov*() case it does.

I'll sure it break something to change this to macros, although it would 
be good if clang would also generate a warning (for application code 
using <rte_memcpy.h>).

Re: [RFC v2] eal: provide option to use compiler memcpy instead of RTE

[Mattias Rönnblom]

On 2024-05-31 18:50, Stephen Hemminger wrote:
> On Fri, 31 May 2024 07:19:41 +0200
> Mattias Rönnblom <hofors@lysator.liu.se> wrote:
> 
>> On 2024-05-28 17:09, Bruce Richardson wrote:
>>> On Tue, May 28, 2024 at 07:59:36AM -0700, Stephen Hemminger wrote:
>>>> On Tue, 28 May 2024 10:19:15 +0200
>>>> Mattias Rönnblom <hofors@lysator.liu.se> wrote:
>>>>   
>>>>>>       
>>>>>
>>>>> I've tested this patch some with DSW micro benchmarks, and the result is
>>>>> a 2.5% reduction of the DSW+testapp overhead with cc/libc memcpy. GCC 11.4.
>>>>>
>>>>> We've also run characteristic test suite of a large, real world app.
>>>>> Here, we saw no effect. GCC 10.5.
>>>>>
>>>>> x86_64 in both cases (Skylake and Raptor Lake).
>>>>>
>>>>> Last time we did the same, there were a noticeable performance
>>>>> degradation in both the above cases.
>>>>>
>>>>> This is not a lot of data points, but I think it we should consider
>>>>> making the custom RTE memcpy() implementations optional in the next
>>>>> release, and if no-one complains, remove the implementations in the next
>>>>> release.
>>>>
>>>> Lets go farther.
>>>>
>>>> 1. Announce that rte_memcpy will be marked deprecated in 24.11 release
>>>>
>>>> 2. In 24.11 do a global replace of rte_memcpy on the tree.
>>>>      And mark rte_memcpy as deprecated.
>>>>
>>>> 3. In 25.11 it can go away.
>>>
>>> While I'd like us to be able to do so, I believe that to be premature. We
>>> need to see where/if there are regressions first, and see about fixing
>>> them.
>>>
>>> /Bruce
>>
>> Should I turn this RFC into a PATCH?
>>
>> Is use_cc_memcpy a good name for the configuration parameter?
>>
> 
> I did a slightly more direct test and found a couple of things:
>     1. Ena driver is redefining memcpy as rte_memcpy, this should be removed and should have
>        been blocked during code review.

Wouldn't that hack continue to work? Provided rte_memcpy() is a 
function, and the <rte_memcpy.h> header is included prior to the memcpy 
redefinition.

>     2. A couple of drivers are implicitly expecting simd vector routines to be available.
>        This works because rte_memcpy.h includes rte_vect.h.  The fix is to have these
>        places include rte_vect.h
> 

I noticed this as well. I'll add patches for those drivers.

[RFC v3 0/5] Optionally have rte_memcpy delegate to compiler memcpy

[Mattias Rönnblom]

This patch set make DPDK library, driver, and application code use the
compiler/libc memcpy() by default when functions in <rte_memcpy.h> are
invoked.

The various custom DPDK rte_memcpy() implementations may be retained
by means of a build-time option.

This patch set only make a difference on x86, PPC and ARM. Loongarch
and RISCV already used compiler/libc memcpy().

Mattias Rönnblom (5):
  event/dlb2: include headers for vector and memory copy APIs
  net/octeon_ep: properly include vector API header file
  distributor: properly include vector API header file
  fib: properly include vector API header file
  eal: provide option to use compiler memcpy instead of RTE

 config/meson.build                     |  1 +
 drivers/event/dlb2/dlb2.c              |  2 +
 drivers/net/octeon_ep/otx_ep_ethdev.c  |  2 +
 lib/distributor/rte_distributor.c      |  1 +
 lib/eal/arm/include/rte_memcpy.h       | 10 +++++
 lib/eal/include/generic/rte_memcpy.h   | 61 +++++++++++++++++++++++---
 lib/eal/loongarch/include/rte_memcpy.h | 53 ++--------------------
 lib/eal/ppc/include/rte_memcpy.h       | 10 +++++
 lib/eal/riscv/include/rte_memcpy.h     | 53 ++--------------------
 lib/eal/x86/include/meson.build        |  1 +
 lib/eal/x86/include/rte_memcpy.h       | 11 ++++-
 lib/fib/trie.c                         |  1 +
 meson_options.txt                      |  2 +
 13 files changed, 102 insertions(+), 106 deletions(-)

-- 
2.34.1

[RFC v3 1/5] event/dlb2: include headers for vector and memory copy APIs

[Mattias Rönnblom]

The DLB2 PMD depended on <rte_vect.h> being included as a side-effect
of <rte_memcpy.h> being included.

In addition, DLB2 used rte_memcpy() but did not include <rte_memcpy.h>,
but rather depended on other include files to do so.

This patch addresses both of those issues.

Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
---
 drivers/event/dlb2/dlb2.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/drivers/event/dlb2/dlb2.c b/drivers/event/dlb2/dlb2.c
index 0b91f03956..19f90b8f8d 100644
--- a/drivers/event/dlb2/dlb2.c
+++ b/drivers/event/dlb2/dlb2.c
@@ -25,11 +25,13 @@
 #include <rte_kvargs.h>
 #include <rte_log.h>
 #include <rte_malloc.h>
+#include <rte_memcpy.h>
 #include <rte_mbuf.h>
 #include <rte_power_intrinsics.h>
 #include <rte_prefetch.h>
 #include <rte_ring.h>
 #include <rte_string_fns.h>
+#include <rte_vect.h>
 
 #include "dlb2_priv.h"
 #include "dlb2_iface.h"
-- 
2.34.1

[RFC v3 2/5] net/octeon_ep: properly include vector API header file

[Mattias Rönnblom]

The octeon_ip driver relied on <rte_vect.h>, but failed to provide a
direct include of this file.

Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
---
 drivers/net/octeon_ep/otx_ep_ethdev.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/drivers/net/octeon_ep/otx_ep_ethdev.c b/drivers/net/octeon_ep/otx_ep_ethdev.c
index 46211361a0..b069216629 100644
--- a/drivers/net/octeon_ep/otx_ep_ethdev.c
+++ b/drivers/net/octeon_ep/otx_ep_ethdev.c
@@ -5,6 +5,8 @@
 #include <inttypes.h>
 #include <ethdev_pci.h>
 
+#include <rte_vect.h>
+
 #include "otx_ep_common.h"
 #include "otx_ep_vf.h"
 #include "otx2_ep_vf.h"
-- 
2.34.1

[RFC v3 3/5] distributor: properly include vector API header file

[Mattias Rönnblom]

The distributor library relied on <rte_vect.h>, but failed to provide
a direct include of this file.

Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
---
 lib/distributor/rte_distributor.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/distributor/rte_distributor.c b/lib/distributor/rte_distributor.c
index e58727cdc2..1389efc03f 100644
--- a/lib/distributor/rte_distributor.c
+++ b/lib/distributor/rte_distributor.c
@@ -15,6 +15,7 @@
 #include <rte_eal_memconfig.h>
 #include <rte_pause.h>
 #include <rte_tailq.h>
+#include <rte_vect.h>
 
 #include "rte_distributor.h"
 #include "rte_distributor_single.h"
-- 
2.34.1

[RFC v3 4/5] fib: properly include vector API header file

[Mattias Rönnblom]

The trie implementation of the fib library relied on <rte_vect.h>, but
failed to provide a direct include of this file.

Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
---
 lib/fib/trie.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/fib/trie.c b/lib/fib/trie.c
index 09470e7287..74db8863df 100644
--- a/lib/fib/trie.c
+++ b/lib/fib/trie.c
@@ -9,6 +9,7 @@
 #include <rte_debug.h>
 #include <rte_malloc.h>
 #include <rte_errno.h>
+#include <rte_vect.h>
 
 #include <rte_rib6.h>
 #include <rte_fib6.h>
-- 
2.34.1

[RFC v3 5/5] eal: provide option to use compiler memcpy instead of RTE

[Mattias Rönnblom]

Provide build option to have functions in <rte_memcpy.h> delegate to
the standard compiler/libc memcpy(), instead of using the various
custom DPDK, handcrafted, per-architecture rte_memcpy()
implementations.

A new meson build option 'use_cc_memcpy' is added. By default,
the compiler/libc memcpy() is used.

The performance benefits of the custom DPDK rte_memcpy()
implementations have been diminishing with every compiler release, and
with current toolchains the usage of a custom memcpy() implementation
may even be a liability.

An additional benefit of this change is that compilers and static
analysis tools have an easier time detecting incorrect usage of
memcpy() (e.g., buffer overruns, or overlapping source and destination
buffers).

This patch makes DPDK and DPDK applications using <rte_memcpy.h> use
compiler/libc memcpy() by default, but leaves an option to stay on the
custom DPDK implementations, would that prove beneficial for certain
applications or architectures.

RFC v3:
 o Fix missing #endif on loongarch.
 o PPC and RISCV now implemented, meaning all architectures are supported.
 o Unnecessary <rte_vect.h> include is removed from <rte_memcpy.h>.

RFC v2:
 * Fix bug where rte_memcpy.h was not installed on x86.
 * Made attempt to make Loongarch compile.

Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
---
 config/meson.build                     |  1 +
 lib/eal/arm/include/rte_memcpy.h       | 10 +++++
 lib/eal/include/generic/rte_memcpy.h   | 61 +++++++++++++++++++++++---
 lib/eal/loongarch/include/rte_memcpy.h | 53 ++--------------------
 lib/eal/ppc/include/rte_memcpy.h       | 10 +++++
 lib/eal/riscv/include/rte_memcpy.h     | 53 ++--------------------
 lib/eal/x86/include/meson.build        |  1 +
 lib/eal/x86/include/rte_memcpy.h       | 11 ++++-
 meson_options.txt                      |  2 +
 9 files changed, 96 insertions(+), 106 deletions(-)

diff --git a/config/meson.build b/config/meson.build
index 8c8b019c25..456056628e 100644
--- a/config/meson.build
+++ b/config/meson.build
@@ -353,6 +353,7 @@ endforeach
 # set other values pulled from the build options
 dpdk_conf.set('RTE_MAX_ETHPORTS', get_option('max_ethports'))
 dpdk_conf.set('RTE_LIBEAL_USE_HPET', get_option('use_hpet'))
+dpdk_conf.set('RTE_USE_CC_MEMCPY', get_option('use_cc_memcpy'))
 dpdk_conf.set('RTE_ENABLE_STDATOMIC', get_option('enable_stdatomic'))
 dpdk_conf.set('RTE_ENABLE_TRACE_FP', get_option('enable_trace_fp'))
 dpdk_conf.set('RTE_PKTMBUF_HEADROOM', get_option('pkt_mbuf_headroom'))
diff --git a/lib/eal/arm/include/rte_memcpy.h b/lib/eal/arm/include/rte_memcpy.h
index 47dea9a8cc..e8aff722df 100644
--- a/lib/eal/arm/include/rte_memcpy.h
+++ b/lib/eal/arm/include/rte_memcpy.h
@@ -5,10 +5,20 @@
 #ifndef _RTE_MEMCPY_ARM_H_
 #define _RTE_MEMCPY_ARM_H_
 
+#include <rte_config.h>
+
+#ifdef RTE_USE_CC_MEMCPY
+
+#include <generic/rte_memcpy.h>
+
+#else
+
 #ifdef RTE_ARCH_64
 #include <rte_memcpy_64.h>
 #else
 #include <rte_memcpy_32.h>
 #endif
 
+#endif /* RTE_USE_CC_MEMCPY */
+
 #endif /* _RTE_MEMCPY_ARM_H_ */
diff --git a/lib/eal/include/generic/rte_memcpy.h b/lib/eal/include/generic/rte_memcpy.h
index e7f0f8eaa9..cae06117fb 100644
--- a/lib/eal/include/generic/rte_memcpy.h
+++ b/lib/eal/include/generic/rte_memcpy.h
@@ -5,12 +5,19 @@
 #ifndef _RTE_MEMCPY_H_
 #define _RTE_MEMCPY_H_
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 /**
  * @file
  *
  * Functions for vectorised implementation of memcpy().
  */
 
+#include <stdint.h>
+#include <string.h>
+
 /**
  * Copy 16 bytes from one location to another using optimised
  * instructions. The locations should not overlap.
@@ -35,8 +42,6 @@ rte_mov16(uint8_t *dst, const uint8_t *src);
 static inline void
 rte_mov32(uint8_t *dst, const uint8_t *src);
 
-#ifdef __DOXYGEN__
-
 /**
  * Copy 48 bytes from one location to another using optimised
  * instructions. The locations should not overlap.
@@ -49,8 +54,6 @@ rte_mov32(uint8_t *dst, const uint8_t *src);
 static inline void
 rte_mov48(uint8_t *dst, const uint8_t *src);
 
-#endif /* __DOXYGEN__ */
-
 /**
  * Copy 64 bytes from one location to another using optimised
  * instructions. The locations should not overlap.
@@ -87,8 +90,6 @@ rte_mov128(uint8_t *dst, const uint8_t *src);
 static inline void
 rte_mov256(uint8_t *dst, const uint8_t *src);
 
-#ifdef __DOXYGEN__
-
 /**
  * Copy bytes from one location to another. The locations must not overlap.
  *
@@ -111,6 +112,52 @@ rte_mov256(uint8_t *dst, const uint8_t *src);
 static void *
 rte_memcpy(void *dst, const void *src, size_t n);
 
-#endif /* __DOXYGEN__ */
+#ifdef RTE_USE_CC_MEMCPY
+static inline void
+rte_mov16(uint8_t *dst, const uint8_t *src)
+{
+	memcpy(dst, src, 16);
+}
+
+static inline void
+rte_mov32(uint8_t *dst, const uint8_t *src)
+{
+	memcpy(dst, src, 32);
+}
+
+static inline void
+rte_mov48(uint8_t *dst, const uint8_t *src)
+{
+	memcpy(dst, src, 48);
+}
+
+static inline void
+rte_mov64(uint8_t *dst, const uint8_t *src)
+{
+	memcpy(dst, src, 64);
+}
+
+static inline void
+rte_mov128(uint8_t *dst, const uint8_t *src)
+{
+	memcpy(dst, src, 128);
+}
+
+static inline void
+rte_mov256(uint8_t *dst, const uint8_t *src)
+{
+	memcpy(dst, src, 256);
+}
+
+static inline void *
+rte_memcpy(void *dst, const void *src, size_t n)
+{
+	return memcpy(dst, src, n);
+}
+#endif /* RTE_USE_CC_MEMCPY */
+
+#ifdef __cplusplus
+}
+#endif
 
 #endif /* _RTE_MEMCPY_H_ */
diff --git a/lib/eal/loongarch/include/rte_memcpy.h b/lib/eal/loongarch/include/rte_memcpy.h
index 22578d40f4..344b4416b5 100644
--- a/lib/eal/loongarch/include/rte_memcpy.h
+++ b/lib/eal/loongarch/include/rte_memcpy.h
@@ -5,57 +5,12 @@
 #ifndef RTE_MEMCPY_LOONGARCH_H
 #define RTE_MEMCPY_LOONGARCH_H
 
-#include <stdint.h>
-#include <string.h>
+#include <rte_config.h>
 
-#include "rte_common.h"
-
-#ifdef __cplusplus
-extern "C" {
+#ifndef RTE_USE_CC_MEMCPY
+#define RTE_USE_CC_MEMCPY
 #endif
 
-#include "generic/rte_memcpy.h"
-
-static inline void
-rte_mov16(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 16);
-}
-
-static inline void
-rte_mov32(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 32);
-}
-
-static inline void
-rte_mov48(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 48);
-}
-
-static inline void
-rte_mov64(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 64);
-}
-
-static inline void
-rte_mov128(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 128);
-}
-
-static inline void
-rte_mov256(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 256);
-}
-
-#define rte_memcpy(d, s, n)	memcpy((d), (s), (n))
-
-#ifdef __cplusplus
-}
-#endif
+#include <generic/rte_memcpy.h>
 
 #endif /* RTE_MEMCPY_LOONGARCH_H */
diff --git a/lib/eal/ppc/include/rte_memcpy.h b/lib/eal/ppc/include/rte_memcpy.h
index 6f388c0234..645fd83986 100644
--- a/lib/eal/ppc/include/rte_memcpy.h
+++ b/lib/eal/ppc/include/rte_memcpy.h
@@ -6,6 +6,14 @@
 #ifndef _RTE_MEMCPY_PPC_64_H_
 #define _RTE_MEMCPY_PPC_64_H_
 
+#include <rte_config.h>
+
+#ifdef RTE_USE_CC_MEMCPY
+
+#include <generic/rte_memcpy.h>
+
+#else
+
 #include <stdint.h>
 #include <string.h>
 
@@ -215,4 +223,6 @@ rte_memcpy_func(void *dst, const void *src, size_t n)
 }
 #endif
 
+#endif /* RTE_USE_CC_MEMCPY */
+
 #endif /* _RTE_MEMCPY_PPC_64_H_ */
diff --git a/lib/eal/riscv/include/rte_memcpy.h b/lib/eal/riscv/include/rte_memcpy.h
index e34f19396e..4acdc4af5f 100644
--- a/lib/eal/riscv/include/rte_memcpy.h
+++ b/lib/eal/riscv/include/rte_memcpy.h
@@ -7,57 +7,12 @@
 #ifndef RTE_MEMCPY_RISCV_H
 #define RTE_MEMCPY_RISCV_H
 
-#include <stdint.h>
-#include <string.h>
+#include <rte_config.h>
 
-#include "rte_common.h"
-
-#ifdef __cplusplus
-extern "C" {
+#ifndef RTE_USE_CC_MEMCPY
+#define RTE_USE_CC_MEMCPY
 #endif
 
-#include "generic/rte_memcpy.h"
-
-static inline void
-rte_mov16(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 16);
-}
-
-static inline void
-rte_mov32(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 32);
-}
-
-static inline void
-rte_mov48(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 48);
-}
-
-static inline void
-rte_mov64(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 64);
-}
-
-static inline void
-rte_mov128(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 128);
-}
-
-static inline void
-rte_mov256(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 256);
-}
-
-#define rte_memcpy(d, s, n)	memcpy((d), (s), (n))
-
-#ifdef __cplusplus
-}
-#endif
+#include <generic/rte_memcpy.h>
 
 #endif /* RTE_MEMCPY_RISCV_H */
diff --git a/lib/eal/x86/include/meson.build b/lib/eal/x86/include/meson.build
index 52d2f8e969..09c2fe2485 100644
--- a/lib/eal/x86/include/meson.build
+++ b/lib/eal/x86/include/meson.build
@@ -16,6 +16,7 @@ arch_headers = files(
         'rte_spinlock.h',
         'rte_vect.h',
 )
+
 arch_indirect_headers = files(
         'rte_atomic_32.h',
         'rte_atomic_64.h',
diff --git a/lib/eal/x86/include/rte_memcpy.h b/lib/eal/x86/include/rte_memcpy.h
index 72a92290e0..c5ba74d2ed 100644
--- a/lib/eal/x86/include/rte_memcpy.h
+++ b/lib/eal/x86/include/rte_memcpy.h
@@ -11,12 +11,19 @@
  * Functions for SSE/AVX/AVX2/AVX512 implementation of memcpy().
  */
 
+#include <rte_config.h>
+
+#ifdef RTE_USE_CC_MEMCPY
+
+#include <generic/rte_memcpy.h>
+
+#else
+
 #include <stdio.h>
 #include <stdint.h>
 #include <string.h>
 #include <rte_vect.h>
 #include <rte_common.h>
-#include <rte_config.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -878,4 +885,6 @@ rte_memcpy(void *dst, const void *src, size_t n)
 }
 #endif
 
+#endif /* RTE_USE_CC_MEMCPY */
+
 #endif /* _RTE_MEMCPY_X86_64_H_ */
diff --git a/meson_options.txt b/meson_options.txt
index e49b2fc089..263b0e7882 100644
--- a/meson_options.txt
+++ b/meson_options.txt
@@ -60,3 +60,5 @@ option('tests', type: 'boolean', value: true, description:
        'build unit tests')
 option('use_hpet', type: 'boolean', value: false, description:
        'use HPET timer in EAL')
+option('use_cc_memcpy', type: 'boolean', value: true, description:
+       'Have rte_memcpy() delegate to compiler/libc memcpy() instead of using custom implementation.')
-- 
2.34.1

RE: [RFC v3 5/5] eal: provide option to use compiler memcpy instead of RTE

[Morten Brørup]

> From: Mattias Rönnblom [mailto:mattias.ronnblom@ericsson.com]
> Sent: Sunday, 2 June 2024 14.39
> 
> Provide build option to have functions in <rte_memcpy.h> delegate to
> the standard compiler/libc memcpy(), instead of using the various
> custom DPDK, handcrafted, per-architecture rte_memcpy()
> implementations.
> 
> A new meson build option 'use_cc_memcpy' is added. By default,
> the compiler/libc memcpy() is used.
> 
> The performance benefits of the custom DPDK rte_memcpy()
> implementations have been diminishing with every compiler release, and
> with current toolchains the usage of a custom memcpy() implementation
> may even be a liability.
> 
> An additional benefit of this change is that compilers and static
> analysis tools have an easier time detecting incorrect usage of
> memcpy() (e.g., buffer overruns, or overlapping source and destination
> buffers).
> 
> This patch makes DPDK and DPDK applications using <rte_memcpy.h> use
> compiler/libc memcpy() by default, but leaves an option to stay on the
> custom DPDK implementations, would that prove beneficial for certain
> applications or architectures.
> 
> RFC v3:
>  o Fix missing #endif on loongarch.
>  o PPC and RISCV now implemented, meaning all architectures are
> supported.
>  o Unnecessary <rte_vect.h> include is removed from <rte_memcpy.h>.
> 
> RFC v2:
>  * Fix bug where rte_memcpy.h was not installed on x86.
>  * Made attempt to make Loongarch compile.
> 
> Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
> ---

We should keep pushing DPDK forward and cleaning up old cruft along the way.

The memcpy discussion has convinced me that:
1. This change is a good idea, and
2. Mainstream compilers are sufficiently mature to do it now.

So, for the series,
Acked-by: Morten Brørup <mb@smartsharesystems.com>

>  static inline void
>  rte_mov32(uint8_t *dst, const uint8_t *src);

While at it, would it be somehow beneficial to change these from uint8_t* to void* or char* (keeping const where relevant)?

Re: [RFC v3 5/5] eal: provide option to use compiler memcpy instead of RTE

[Mattias Rönnblom]

On 2024-06-02 22:58, Morten Brørup wrote:
>> From: Mattias Rönnblom [mailto:mattias.ronnblom@ericsson.com]
>> Sent: Sunday, 2 June 2024 14.39
>>
>> Provide build option to have functions in <rte_memcpy.h> delegate to
>> the standard compiler/libc memcpy(), instead of using the various
>> custom DPDK, handcrafted, per-architecture rte_memcpy()
>> implementations.
>>
>> A new meson build option 'use_cc_memcpy' is added. By default,
>> the compiler/libc memcpy() is used.
>>
>> The performance benefits of the custom DPDK rte_memcpy()
>> implementations have been diminishing with every compiler release, and
>> with current toolchains the usage of a custom memcpy() implementation
>> may even be a liability.
>>
>> An additional benefit of this change is that compilers and static
>> analysis tools have an easier time detecting incorrect usage of
>> memcpy() (e.g., buffer overruns, or overlapping source and destination
>> buffers).
>>
>> This patch makes DPDK and DPDK applications using <rte_memcpy.h> use
>> compiler/libc memcpy() by default, but leaves an option to stay on the
>> custom DPDK implementations, would that prove beneficial for certain
>> applications or architectures.
>>
>> RFC v3:
>>   o Fix missing #endif on loongarch.
>>   o PPC and RISCV now implemented, meaning all architectures are
>> supported.
>>   o Unnecessary <rte_vect.h> include is removed from <rte_memcpy.h>.
>>
>> RFC v2:
>>   * Fix bug where rte_memcpy.h was not installed on x86.
>>   * Made attempt to make Loongarch compile.
>>
>> Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
>> ---
> 
> We should keep pushing DPDK forward and cleaning up old cruft along the way.
> 
> The memcpy discussion has convinced me that:
> 1. This change is a good idea, and
> 2. Mainstream compilers are sufficiently mature to do it now.
> 
> So, for the series,
> Acked-by: Morten Brørup <mb@smartsharesystems.com>
> 
>>   static inline void
>>   rte_mov32(uint8_t *dst, const uint8_t *src);
> 
> While at it, would it be somehow beneficial to change these from uint8_t* to void* or char* (keeping const where relevant)?
> 

Something to consider before doing such a change would be if it may 
cause any strict aliasing issue for existing users.

If we should break the API, I think we are better off removing 
rte_mov*() altogether.

Re: [RFC v3 5/5] eal: provide option to use compiler memcpy instead of RTE

[Stephen Hemminger]

On Mon, 3 Jun 2024 19:04:49 +0200
Mattias Rönnblom <hofors@lysator.liu.se> wrote:

> >>   static inline void
> >>   rte_mov32(uint8_t *dst, const uint8_t *src);  
> > 
> > While at it, would it be somehow beneficial to change these from uint8_t* to void* or char* (keeping const where relevant)?
> >   
> 
> Something to consider before doing such a change would be if it may 
> cause any strict aliasing issue for existing users.
> 
> If we should break the API, I think we are better off removing 
> rte_mov*() altogether.

Why modify soon to be dead code.
These should marked as deprecated in 24.11

[PATCH 0/5] Optionally have rte_memcpy delegate to compiler memcpy

[Mattias Rönnblom]

This patch set make DPDK library, driver, and application code use the
compiler/libc memcpy() by default when functions in <rte_memcpy.h> are
invoked.

The various custom DPDK rte_memcpy() implementations may be retained
by means of a build-time option.

This patch set only make a difference on x86, PPC and ARM. Loongarch
and RISCV already used compiler/libc memcpy().

Mattias Rönnblom (5):
  event/dlb2: include headers for vector and memory copy APIs
  net/octeon_ep: properly include vector API header file
  distributor: properly include vector API header file
  fib: properly include vector API header file
  eal: provide option to use compiler memcpy instead of RTE

 config/meson.build                     |  1 +
 doc/guides/rel_notes/release_24_07.rst | 21 +++++++++
 drivers/event/dlb2/dlb2.c              |  2 +
 drivers/net/octeon_ep/otx_ep_ethdev.c  |  2 +
 lib/distributor/rte_distributor.c      |  1 +
 lib/eal/arm/include/rte_memcpy.h       | 10 +++++
 lib/eal/include/generic/rte_memcpy.h   | 61 +++++++++++++++++++++++---
 lib/eal/loongarch/include/rte_memcpy.h | 53 ++--------------------
 lib/eal/ppc/include/rte_memcpy.h       | 10 +++++
 lib/eal/riscv/include/rte_memcpy.h     | 53 ++--------------------
 lib/eal/x86/include/meson.build        |  1 +
 lib/eal/x86/include/rte_memcpy.h       | 11 ++++-
 lib/fib/trie.c                         |  1 +
 meson_options.txt                      |  2 +
 14 files changed, 123 insertions(+), 106 deletions(-)

-- 
2.34.1

[PATCH 1/5] event/dlb2: include headers for vector and memory copy APIs

[Mattias Rönnblom]

The DLB2 PMD depended on <rte_vect.h> being included as a side-effect
of <rte_memcpy.h> being included.

In addition, DLB2 used rte_memcpy() but did not include <rte_memcpy.h>,
but rather depended on other include files to do so.

This patch addresses both of those issues.

Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
---
 drivers/event/dlb2/dlb2.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/drivers/event/dlb2/dlb2.c b/drivers/event/dlb2/dlb2.c
index 0b91f03956..19f90b8f8d 100644
--- a/drivers/event/dlb2/dlb2.c
+++ b/drivers/event/dlb2/dlb2.c
@@ -25,11 +25,13 @@
 #include <rte_kvargs.h>
 #include <rte_log.h>
 #include <rte_malloc.h>
+#include <rte_memcpy.h>
 #include <rte_mbuf.h>
 #include <rte_power_intrinsics.h>
 #include <rte_prefetch.h>
 #include <rte_ring.h>
 #include <rte_string_fns.h>
+#include <rte_vect.h>
 
 #include "dlb2_priv.h"
 #include "dlb2_iface.h"
-- 
2.34.1

[PATCH 2/5] net/octeon_ep: properly include vector API header file

[Mattias Rönnblom]

The octeon_ip driver relied on <rte_vect.h>, but failed to provide a
direct include of this file.

Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
---
 drivers/net/octeon_ep/otx_ep_ethdev.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/drivers/net/octeon_ep/otx_ep_ethdev.c b/drivers/net/octeon_ep/otx_ep_ethdev.c
index 46211361a0..b069216629 100644
--- a/drivers/net/octeon_ep/otx_ep_ethdev.c
+++ b/drivers/net/octeon_ep/otx_ep_ethdev.c
@@ -5,6 +5,8 @@
 #include <inttypes.h>
 #include <ethdev_pci.h>
 
+#include <rte_vect.h>
+
 #include "otx_ep_common.h"
 #include "otx_ep_vf.h"
 #include "otx2_ep_vf.h"
-- 
2.34.1

[PATCH 3/5] distributor: properly include vector API header file

[Mattias Rönnblom]

The distributor library relied on <rte_vect.h>, but failed to provide
a direct include of this file.

Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
---
 lib/distributor/rte_distributor.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/distributor/rte_distributor.c b/lib/distributor/rte_distributor.c
index e58727cdc2..1389efc03f 100644
--- a/lib/distributor/rte_distributor.c
+++ b/lib/distributor/rte_distributor.c
@@ -15,6 +15,7 @@
 #include <rte_eal_memconfig.h>
 #include <rte_pause.h>
 #include <rte_tailq.h>
+#include <rte_vect.h>
 
 #include "rte_distributor.h"
 #include "rte_distributor_single.h"
-- 
2.34.1

[PATCH 4/5] fib: properly include vector API header file

[Mattias Rönnblom]

The trie implementation of the fib library relied on <rte_vect.h>, but
failed to provide a direct include of this file.

Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
---
 lib/fib/trie.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/fib/trie.c b/lib/fib/trie.c
index 09470e7287..74db8863df 100644
--- a/lib/fib/trie.c
+++ b/lib/fib/trie.c
@@ -9,6 +9,7 @@
 #include <rte_debug.h>
 #include <rte_malloc.h>
 #include <rte_errno.h>
+#include <rte_vect.h>
 
 #include <rte_rib6.h>
 #include <rte_fib6.h>
-- 
2.34.1

[PATCH 5/5] eal: provide option to use compiler memcpy instead of RTE

[Mattias Rönnblom]

Provide build option to have functions in <rte_memcpy.h> delegate to
the standard compiler/libc memcpy(), instead of using the various
custom DPDK, handcrafted, per-architecture rte_memcpy()
implementations.

A new meson build option 'use_cc_memcpy' is added. By default,
the compiler/libc memcpy() is used.

The performance benefits of the custom DPDK rte_memcpy()
implementations have been diminishing with every compiler release, and
with current toolchains the use of a custom memcpy() implementation
may even be a liability.

This patch leaves an option to stay on the custom DPDK implementations,
would that prove beneficial for certain applications or architectures.

An additional benefit of this change is that compilers and static
analysis tools have an easier time detecting incorrect usage of
rte_memcpy() (e.g., buffer overruns, or overlapping source and
destination buffers).

PATCH:
 o Add entry in release notes.
 o Update meson help text.

RFC v3:
 o Fix missing #endif on loongarch.
 o PPC and RISCV now implemented, meaning all architectures are supported.
 o Unnecessary <rte_vect.h> include is removed from <rte_memcpy.h>.

RFC v2:
 * Fix bug where rte_memcpy.h was not installed on x86.
 * Made attempt to make Loongarch compile.

Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
Acked-by: Morten Brørup <mb@smartsharesystems.com>
---
 config/meson.build                     |  1 +
 doc/guides/rel_notes/release_24_07.rst | 21 +++++++++
 lib/eal/arm/include/rte_memcpy.h       | 10 +++++
 lib/eal/include/generic/rte_memcpy.h   | 61 +++++++++++++++++++++++---
 lib/eal/loongarch/include/rte_memcpy.h | 53 ++--------------------
 lib/eal/ppc/include/rte_memcpy.h       | 10 +++++
 lib/eal/riscv/include/rte_memcpy.h     | 53 ++--------------------
 lib/eal/x86/include/meson.build        |  1 +
 lib/eal/x86/include/rte_memcpy.h       | 11 ++++-
 meson_options.txt                      |  2 +
 10 files changed, 117 insertions(+), 106 deletions(-)

diff --git a/config/meson.build b/config/meson.build
index 8c8b019c25..456056628e 100644
--- a/config/meson.build
+++ b/config/meson.build
@@ -353,6 +353,7 @@ endforeach
 # set other values pulled from the build options
 dpdk_conf.set('RTE_MAX_ETHPORTS', get_option('max_ethports'))
 dpdk_conf.set('RTE_LIBEAL_USE_HPET', get_option('use_hpet'))
+dpdk_conf.set('RTE_USE_CC_MEMCPY', get_option('use_cc_memcpy'))
 dpdk_conf.set('RTE_ENABLE_STDATOMIC', get_option('enable_stdatomic'))
 dpdk_conf.set('RTE_ENABLE_TRACE_FP', get_option('enable_trace_fp'))
 dpdk_conf.set('RTE_PKTMBUF_HEADROOM', get_option('pkt_mbuf_headroom'))
diff --git a/doc/guides/rel_notes/release_24_07.rst b/doc/guides/rel_notes/release_24_07.rst
index a69f24cf99..4b6eafa86e 100644
--- a/doc/guides/rel_notes/release_24_07.rst
+++ b/doc/guides/rel_notes/release_24_07.rst
@@ -24,6 +24,27 @@ DPDK Release 24.07
 New Features
 ------------
 
+* **Compiler memcpy replaces custom DPDK implementation.**
+
+  The memory copy functions of ``<rte_memcpy.h>`` now delegates to the
+  standard memcpy() function, implemented by the compiler and the C
+  runtime (e.g., libc).
+
+  In this release of DPDK, the handcrafted, per-architecture memory
+  copy implementations are still available, and may be reactivated by
+  setting the new ``use_cc_memcpy`` build option to false.
+
+  The performance benefits of the custom DPDK rte_memcpy()
+  implementations have been diminishing with every new compiler
+  release, and with current toolchains the use of a custom memcpy()
+  implementation may even result in worse performance than the
+  standard memcpy().
+
+  An additional benefit of this change is that compilers and static
+  analysis tools have an easier time detecting incorrect usage of
+  rte_memcpy() (e.g., buffer overruns, or overlapping source and
+  destination buffers).
+
 .. This section should contain new features added in this release.
    Sample format:
 
diff --git a/lib/eal/arm/include/rte_memcpy.h b/lib/eal/arm/include/rte_memcpy.h
index 47dea9a8cc..e8aff722df 100644
--- a/lib/eal/arm/include/rte_memcpy.h
+++ b/lib/eal/arm/include/rte_memcpy.h
@@ -5,10 +5,20 @@
 #ifndef _RTE_MEMCPY_ARM_H_
 #define _RTE_MEMCPY_ARM_H_
 
+#include <rte_config.h>
+
+#ifdef RTE_USE_CC_MEMCPY
+
+#include <generic/rte_memcpy.h>
+
+#else
+
 #ifdef RTE_ARCH_64
 #include <rte_memcpy_64.h>
 #else
 #include <rte_memcpy_32.h>
 #endif
 
+#endif /* RTE_USE_CC_MEMCPY */
+
 #endif /* _RTE_MEMCPY_ARM_H_ */
diff --git a/lib/eal/include/generic/rte_memcpy.h b/lib/eal/include/generic/rte_memcpy.h
index e7f0f8eaa9..cae06117fb 100644
--- a/lib/eal/include/generic/rte_memcpy.h
+++ b/lib/eal/include/generic/rte_memcpy.h
@@ -5,12 +5,19 @@
 #ifndef _RTE_MEMCPY_H_
 #define _RTE_MEMCPY_H_
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 /**
  * @file
  *
  * Functions for vectorised implementation of memcpy().
  */
 
+#include <stdint.h>
+#include <string.h>
+
 /**
  * Copy 16 bytes from one location to another using optimised
  * instructions. The locations should not overlap.
@@ -35,8 +42,6 @@ rte_mov16(uint8_t *dst, const uint8_t *src);
 static inline void
 rte_mov32(uint8_t *dst, const uint8_t *src);
 
-#ifdef __DOXYGEN__
-
 /**
  * Copy 48 bytes from one location to another using optimised
  * instructions. The locations should not overlap.
@@ -49,8 +54,6 @@ rte_mov32(uint8_t *dst, const uint8_t *src);
 static inline void
 rte_mov48(uint8_t *dst, const uint8_t *src);
 
-#endif /* __DOXYGEN__ */
-
 /**
  * Copy 64 bytes from one location to another using optimised
  * instructions. The locations should not overlap.
@@ -87,8 +90,6 @@ rte_mov128(uint8_t *dst, const uint8_t *src);
 static inline void
 rte_mov256(uint8_t *dst, const uint8_t *src);
 
-#ifdef __DOXYGEN__
-
 /**
  * Copy bytes from one location to another. The locations must not overlap.
  *
@@ -111,6 +112,52 @@ rte_mov256(uint8_t *dst, const uint8_t *src);
 static void *
 rte_memcpy(void *dst, const void *src, size_t n);
 
-#endif /* __DOXYGEN__ */
+#ifdef RTE_USE_CC_MEMCPY
+static inline void
+rte_mov16(uint8_t *dst, const uint8_t *src)
+{
+	memcpy(dst, src, 16);
+}
+
+static inline void
+rte_mov32(uint8_t *dst, const uint8_t *src)
+{
+	memcpy(dst, src, 32);
+}
+
+static inline void
+rte_mov48(uint8_t *dst, const uint8_t *src)
+{
+	memcpy(dst, src, 48);
+}
+
+static inline void
+rte_mov64(uint8_t *dst, const uint8_t *src)
+{
+	memcpy(dst, src, 64);
+}
+
+static inline void
+rte_mov128(uint8_t *dst, const uint8_t *src)
+{
+	memcpy(dst, src, 128);
+}
+
+static inline void
+rte_mov256(uint8_t *dst, const uint8_t *src)
+{
+	memcpy(dst, src, 256);
+}
+
+static inline void *
+rte_memcpy(void *dst, const void *src, size_t n)
+{
+	return memcpy(dst, src, n);
+}
+#endif /* RTE_USE_CC_MEMCPY */
+
+#ifdef __cplusplus
+}
+#endif
 
 #endif /* _RTE_MEMCPY_H_ */
diff --git a/lib/eal/loongarch/include/rte_memcpy.h b/lib/eal/loongarch/include/rte_memcpy.h
index 22578d40f4..344b4416b5 100644
--- a/lib/eal/loongarch/include/rte_memcpy.h
+++ b/lib/eal/loongarch/include/rte_memcpy.h
@@ -5,57 +5,12 @@
 #ifndef RTE_MEMCPY_LOONGARCH_H
 #define RTE_MEMCPY_LOONGARCH_H
 
-#include <stdint.h>
-#include <string.h>
+#include <rte_config.h>
 
-#include "rte_common.h"
-
-#ifdef __cplusplus
-extern "C" {
+#ifndef RTE_USE_CC_MEMCPY
+#define RTE_USE_CC_MEMCPY
 #endif
 
-#include "generic/rte_memcpy.h"
-
-static inline void
-rte_mov16(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 16);
-}
-
-static inline void
-rte_mov32(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 32);
-}
-
-static inline void
-rte_mov48(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 48);
-}
-
-static inline void
-rte_mov64(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 64);
-}
-
-static inline void
-rte_mov128(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 128);
-}
-
-static inline void
-rte_mov256(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 256);
-}
-
-#define rte_memcpy(d, s, n)	memcpy((d), (s), (n))
-
-#ifdef __cplusplus
-}
-#endif
+#include <generic/rte_memcpy.h>
 
 #endif /* RTE_MEMCPY_LOONGARCH_H */
diff --git a/lib/eal/ppc/include/rte_memcpy.h b/lib/eal/ppc/include/rte_memcpy.h
index 6f388c0234..645fd83986 100644
--- a/lib/eal/ppc/include/rte_memcpy.h
+++ b/lib/eal/ppc/include/rte_memcpy.h
@@ -6,6 +6,14 @@
 #ifndef _RTE_MEMCPY_PPC_64_H_
 #define _RTE_MEMCPY_PPC_64_H_
 
+#include <rte_config.h>
+
+#ifdef RTE_USE_CC_MEMCPY
+
+#include <generic/rte_memcpy.h>
+
+#else
+
 #include <stdint.h>
 #include <string.h>
 
@@ -215,4 +223,6 @@ rte_memcpy_func(void *dst, const void *src, size_t n)
 }
 #endif
 
+#endif /* RTE_USE_CC_MEMCPY */
+
 #endif /* _RTE_MEMCPY_PPC_64_H_ */
diff --git a/lib/eal/riscv/include/rte_memcpy.h b/lib/eal/riscv/include/rte_memcpy.h
index e34f19396e..4acdc4af5f 100644
--- a/lib/eal/riscv/include/rte_memcpy.h
+++ b/lib/eal/riscv/include/rte_memcpy.h
@@ -7,57 +7,12 @@
 #ifndef RTE_MEMCPY_RISCV_H
 #define RTE_MEMCPY_RISCV_H
 
-#include <stdint.h>
-#include <string.h>
+#include <rte_config.h>
 
-#include "rte_common.h"
-
-#ifdef __cplusplus
-extern "C" {
+#ifndef RTE_USE_CC_MEMCPY
+#define RTE_USE_CC_MEMCPY
 #endif
 
-#include "generic/rte_memcpy.h"
-
-static inline void
-rte_mov16(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 16);
-}
-
-static inline void
-rte_mov32(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 32);
-}
-
-static inline void
-rte_mov48(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 48);
-}
-
-static inline void
-rte_mov64(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 64);
-}
-
-static inline void
-rte_mov128(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 128);
-}
-
-static inline void
-rte_mov256(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 256);
-}
-
-#define rte_memcpy(d, s, n)	memcpy((d), (s), (n))
-
-#ifdef __cplusplus
-}
-#endif
+#include <generic/rte_memcpy.h>
 
 #endif /* RTE_MEMCPY_RISCV_H */
diff --git a/lib/eal/x86/include/meson.build b/lib/eal/x86/include/meson.build
index 52d2f8e969..09c2fe2485 100644
--- a/lib/eal/x86/include/meson.build
+++ b/lib/eal/x86/include/meson.build
@@ -16,6 +16,7 @@ arch_headers = files(
         'rte_spinlock.h',
         'rte_vect.h',
 )
+
 arch_indirect_headers = files(
         'rte_atomic_32.h',
         'rte_atomic_64.h',
diff --git a/lib/eal/x86/include/rte_memcpy.h b/lib/eal/x86/include/rte_memcpy.h
index 72a92290e0..c5ba74d2ed 100644
--- a/lib/eal/x86/include/rte_memcpy.h
+++ b/lib/eal/x86/include/rte_memcpy.h
@@ -11,12 +11,19 @@
  * Functions for SSE/AVX/AVX2/AVX512 implementation of memcpy().
  */
 
+#include <rte_config.h>
+
+#ifdef RTE_USE_CC_MEMCPY
+
+#include <generic/rte_memcpy.h>
+
+#else
+
 #include <stdio.h>
 #include <stdint.h>
 #include <string.h>
 #include <rte_vect.h>
 #include <rte_common.h>
-#include <rte_config.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -878,4 +885,6 @@ rte_memcpy(void *dst, const void *src, size_t n)
 }
 #endif
 
+#endif /* RTE_USE_CC_MEMCPY */
+
 #endif /* _RTE_MEMCPY_X86_64_H_ */
diff --git a/meson_options.txt b/meson_options.txt
index e49b2fc089..06f544b631 100644
--- a/meson_options.txt
+++ b/meson_options.txt
@@ -60,3 +60,5 @@ option('tests', type: 'boolean', value: true, description:
        'build unit tests')
 option('use_hpet', type: 'boolean', value: false, description:
        'use HPET timer in EAL')
+option('use_cc_memcpy', type: 'boolean', value: true, description:
+       'Have the functions of <rte_memcpy.h> delegate to compiler/libc memcpy() instead of using custom implementation.')
-- 
2.34.1

Re: [PATCH 3/5] distributor: properly include vector API header file

[Tyler Retzlaff]

On Wed, Jun 05, 2024 at 08:49:43AM +0200, Mattias Rönnblom wrote:
> The distributor library relied on <rte_vect.h>, but failed to provide
> a direct include of this file.
> 
> Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
> ---
Acked-by: Tyler Retzlaff <roretzla@linux.microsoft.com>

Re: [PATCH 4/5] fib: properly include vector API header file

[Tyler Retzlaff]

On Wed, Jun 05, 2024 at 08:49:44AM +0200, Mattias Rönnblom wrote:
> The trie implementation of the fib library relied on <rte_vect.h>, but
> failed to provide a direct include of this file.
> 
> Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
> ---
Acked-by: Tyler Retzlaff <roretzla@linux.microsoft.com>

[PATCH v2 0/6] Optionally have rte_memcpy delegate to compiler memcpy

[Mattias Rönnblom]

This patch set make DPDK library, driver, and application code use the
compiler/libc memcpy() by default when functions in <rte_memcpy.h> are
invoked.

The various custom DPDK rte_memcpy() implementations may be retained
by means of a build-time option.

This patch set only make a difference on x86, PPC and ARM. Loongarch
and RISCV already used compiler/libc memcpy().

This patch set includes a number of fixes in drivers and libraries
which errornously relied on <rte_memcpy.h> including other header
files (e.g., <rte_vect.h>) required by its implementation.

Mattias Rönnblom (6):
  net/fm10k: add missing intrinsic include
  event/dlb2: include headers for vector and memory copy APIs
  net/octeon_ep: properly include vector API header file
  distributor: properly include vector API header file
  fib: properly include vector API header file
  eal: provide option to use compiler memcpy instead of RTE

 config/meson.build                     |  1 +
 doc/guides/rel_notes/release_24_07.rst | 21 +++++++++
 drivers/event/dlb2/dlb2.c              |  2 +
 drivers/net/fm10k/fm10k_rxtx_vec.c     |  1 +
 drivers/net/octeon_ep/otx_ep_ethdev.c  |  2 +
 lib/distributor/rte_distributor.c      |  1 +
 lib/eal/arm/include/rte_memcpy.h       | 10 +++++
 lib/eal/include/generic/rte_memcpy.h   | 61 +++++++++++++++++++++++---
 lib/eal/loongarch/include/rte_memcpy.h | 53 ++--------------------
 lib/eal/ppc/include/rte_memcpy.h       | 10 +++++
 lib/eal/riscv/include/rte_memcpy.h     | 53 ++--------------------
 lib/eal/x86/include/meson.build        |  1 +
 lib/eal/x86/include/rte_memcpy.h       | 11 ++++-
 lib/fib/trie.c                         |  1 +
 meson_options.txt                      |  2 +
 15 files changed, 124 insertions(+), 106 deletions(-)

-- 
2.34.1

[PATCH v2 1/6] net/fm10k: add missing intrinsic include

[Mattias Rönnblom]

Add missing <emmintrin.h> include, to get the _mm_cvtsi128_si64
prototype.

Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
---
 drivers/net/fm10k/fm10k_rxtx_vec.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/drivers/net/fm10k/fm10k_rxtx_vec.c b/drivers/net/fm10k/fm10k_rxtx_vec.c
index 2b6914b1da..d417b31bbb 100644
--- a/drivers/net/fm10k/fm10k_rxtx_vec.c
+++ b/drivers/net/fm10k/fm10k_rxtx_vec.c
@@ -10,6 +10,7 @@
 #include "base/fm10k_type.h"
 
 #include <tmmintrin.h>
+#include <emmintrin.h>
 
 #ifndef __INTEL_COMPILER
 #pragma GCC diagnostic ignored "-Wcast-qual"
-- 
2.34.1

[PATCH v2 2/6] event/dlb2: include headers for vector and memory copy APIs

[Mattias Rönnblom]

The DLB2 PMD depended on <rte_vect.h> being included as a side-effect
of <rte_memcpy.h> being included.

In addition, DLB2 used rte_memcpy() but did not include <rte_memcpy.h>,
but rather depended on other include files to do so.

This patch addresses both of those issues.

Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
---
 drivers/event/dlb2/dlb2.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/drivers/event/dlb2/dlb2.c b/drivers/event/dlb2/dlb2.c
index 0b91f03956..19f90b8f8d 100644
--- a/drivers/event/dlb2/dlb2.c
+++ b/drivers/event/dlb2/dlb2.c
@@ -25,11 +25,13 @@
 #include <rte_kvargs.h>
 #include <rte_log.h>
 #include <rte_malloc.h>
+#include <rte_memcpy.h>
 #include <rte_mbuf.h>
 #include <rte_power_intrinsics.h>
 #include <rte_prefetch.h>
 #include <rte_ring.h>
 #include <rte_string_fns.h>
+#include <rte_vect.h>
 
 #include "dlb2_priv.h"
 #include "dlb2_iface.h"
-- 
2.34.1

[PATCH v2 3/6] net/octeon_ep: properly include vector API header file

[Mattias Rönnblom]

The octeon_ip driver relied on <rte_vect.h>, but failed to provide a
direct include of this file.

Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
---
 drivers/net/octeon_ep/otx_ep_ethdev.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/drivers/net/octeon_ep/otx_ep_ethdev.c b/drivers/net/octeon_ep/otx_ep_ethdev.c
index 46211361a0..b069216629 100644
--- a/drivers/net/octeon_ep/otx_ep_ethdev.c
+++ b/drivers/net/octeon_ep/otx_ep_ethdev.c
@@ -5,6 +5,8 @@
 #include <inttypes.h>
 #include <ethdev_pci.h>
 
+#include <rte_vect.h>
+
 #include "otx_ep_common.h"
 #include "otx_ep_vf.h"
 #include "otx2_ep_vf.h"
-- 
2.34.1

[PATCH v2 4/6] distributor: properly include vector API header file

[Mattias Rönnblom]

The distributor library relied on <rte_vect.h>, but failed to provide
a direct include of this file.

Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
---
 lib/distributor/rte_distributor.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/distributor/rte_distributor.c b/lib/distributor/rte_distributor.c
index e58727cdc2..1389efc03f 100644
--- a/lib/distributor/rte_distributor.c
+++ b/lib/distributor/rte_distributor.c
@@ -15,6 +15,7 @@
 #include <rte_eal_memconfig.h>
 #include <rte_pause.h>
 #include <rte_tailq.h>
+#include <rte_vect.h>
 
 #include "rte_distributor.h"
 #include "rte_distributor_single.h"
-- 
2.34.1

[PATCH v2 5/6] fib: properly include vector API header file

[Mattias Rönnblom]

The trie implementation of the fib library relied on <rte_vect.h>, but
failed to provide a direct include of this file.

Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
---
 lib/fib/trie.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/fib/trie.c b/lib/fib/trie.c
index 09470e7287..74db8863df 100644
--- a/lib/fib/trie.c
+++ b/lib/fib/trie.c
@@ -9,6 +9,7 @@
 #include <rte_debug.h>
 #include <rte_malloc.h>
 #include <rte_errno.h>
+#include <rte_vect.h>
 
 #include <rte_rib6.h>
 #include <rte_fib6.h>
-- 
2.34.1

[PATCH v2 6/6] eal: provide option to use compiler memcpy instead of RTE

[Mattias Rönnblom]

Provide build option to have functions in <rte_memcpy.h> delegate to
the standard compiler/libc memcpy(), instead of using the various
custom DPDK, handcrafted, per-architecture rte_memcpy()
implementations.

A new meson build option 'use_cc_memcpy' is added. By default,
the compiler/libc memcpy() is used.

The performance benefits of the custom DPDK rte_memcpy()
implementations have been diminishing with every compiler release, and
with current toolchains the use of a custom memcpy() implementation
may even be a liability.

This patch leaves an option to stay on the custom DPDK implementations,
would that prove beneficial for certain applications or architectures.

An additional benefit of this change is that compilers and static
analysis tools have an easier time detecting incorrect usage of
rte_memcpy() (e.g., buffer overruns, or overlapping source and
destination buffers).

Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
Acked-by: Morten Brørup <mb@smartsharesystems.com>

---

PATCH:
 o Add entry in release notes.
 o Update meson help text.

RFC v3:
 o Fix missing #endif on loongarch.
 o PPC and RISCV now implemented, meaning all architectures are supported.
 o Unnecessary <rte_vect.h> include is removed from <rte_memcpy.h>.

RFC v2:
 * Fix bug where rte_memcpy.h was not installed on x86.
 * Made attempt to make Loongarch compile.
---
 config/meson.build                     |  1 +
 doc/guides/rel_notes/release_24_07.rst | 21 +++++++++
 lib/eal/arm/include/rte_memcpy.h       | 10 +++++
 lib/eal/include/generic/rte_memcpy.h   | 61 +++++++++++++++++++++++---
 lib/eal/loongarch/include/rte_memcpy.h | 53 ++--------------------
 lib/eal/ppc/include/rte_memcpy.h       | 10 +++++
 lib/eal/riscv/include/rte_memcpy.h     | 53 ++--------------------
 lib/eal/x86/include/meson.build        |  1 +
 lib/eal/x86/include/rte_memcpy.h       | 11 ++++-
 meson_options.txt                      |  2 +
 10 files changed, 117 insertions(+), 106 deletions(-)

diff --git a/config/meson.build b/config/meson.build
index 8c8b019c25..456056628e 100644
--- a/config/meson.build
+++ b/config/meson.build
@@ -353,6 +353,7 @@ endforeach
 # set other values pulled from the build options
 dpdk_conf.set('RTE_MAX_ETHPORTS', get_option('max_ethports'))
 dpdk_conf.set('RTE_LIBEAL_USE_HPET', get_option('use_hpet'))
+dpdk_conf.set('RTE_USE_CC_MEMCPY', get_option('use_cc_memcpy'))
 dpdk_conf.set('RTE_ENABLE_STDATOMIC', get_option('enable_stdatomic'))
 dpdk_conf.set('RTE_ENABLE_TRACE_FP', get_option('enable_trace_fp'))
 dpdk_conf.set('RTE_PKTMBUF_HEADROOM', get_option('pkt_mbuf_headroom'))
diff --git a/doc/guides/rel_notes/release_24_07.rst b/doc/guides/rel_notes/release_24_07.rst
index a69f24cf99..4b6eafa86e 100644
--- a/doc/guides/rel_notes/release_24_07.rst
+++ b/doc/guides/rel_notes/release_24_07.rst
@@ -24,6 +24,27 @@ DPDK Release 24.07
 New Features
 ------------
 
+* **Compiler memcpy replaces custom DPDK implementation.**
+
+  The memory copy functions of ``<rte_memcpy.h>`` now delegates to the
+  standard memcpy() function, implemented by the compiler and the C
+  runtime (e.g., libc).
+
+  In this release of DPDK, the handcrafted, per-architecture memory
+  copy implementations are still available, and may be reactivated by
+  setting the new ``use_cc_memcpy`` build option to false.
+
+  The performance benefits of the custom DPDK rte_memcpy()
+  implementations have been diminishing with every new compiler
+  release, and with current toolchains the use of a custom memcpy()
+  implementation may even result in worse performance than the
+  standard memcpy().
+
+  An additional benefit of this change is that compilers and static
+  analysis tools have an easier time detecting incorrect usage of
+  rte_memcpy() (e.g., buffer overruns, or overlapping source and
+  destination buffers).
+
 .. This section should contain new features added in this release.
    Sample format:
 
diff --git a/lib/eal/arm/include/rte_memcpy.h b/lib/eal/arm/include/rte_memcpy.h
index 47dea9a8cc..e8aff722df 100644
--- a/lib/eal/arm/include/rte_memcpy.h
+++ b/lib/eal/arm/include/rte_memcpy.h
@@ -5,10 +5,20 @@
 #ifndef _RTE_MEMCPY_ARM_H_
 #define _RTE_MEMCPY_ARM_H_
 
+#include <rte_config.h>
+
+#ifdef RTE_USE_CC_MEMCPY
+
+#include <generic/rte_memcpy.h>
+
+#else
+
 #ifdef RTE_ARCH_64
 #include <rte_memcpy_64.h>
 #else
 #include <rte_memcpy_32.h>
 #endif
 
+#endif /* RTE_USE_CC_MEMCPY */
+
 #endif /* _RTE_MEMCPY_ARM_H_ */
diff --git a/lib/eal/include/generic/rte_memcpy.h b/lib/eal/include/generic/rte_memcpy.h
index e7f0f8eaa9..cae06117fb 100644
--- a/lib/eal/include/generic/rte_memcpy.h
+++ b/lib/eal/include/generic/rte_memcpy.h
@@ -5,12 +5,19 @@
 #ifndef _RTE_MEMCPY_H_
 #define _RTE_MEMCPY_H_
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 /**
  * @file
  *
  * Functions for vectorised implementation of memcpy().
  */
 
+#include <stdint.h>
+#include <string.h>
+
 /**
  * Copy 16 bytes from one location to another using optimised
  * instructions. The locations should not overlap.
@@ -35,8 +42,6 @@ rte_mov16(uint8_t *dst, const uint8_t *src);
 static inline void
 rte_mov32(uint8_t *dst, const uint8_t *src);
 
-#ifdef __DOXYGEN__
-
 /**
  * Copy 48 bytes from one location to another using optimised
  * instructions. The locations should not overlap.
@@ -49,8 +54,6 @@ rte_mov32(uint8_t *dst, const uint8_t *src);
 static inline void
 rte_mov48(uint8_t *dst, const uint8_t *src);
 
-#endif /* __DOXYGEN__ */
-
 /**
  * Copy 64 bytes from one location to another using optimised
  * instructions. The locations should not overlap.
@@ -87,8 +90,6 @@ rte_mov128(uint8_t *dst, const uint8_t *src);
 static inline void
 rte_mov256(uint8_t *dst, const uint8_t *src);
 
-#ifdef __DOXYGEN__
-
 /**
  * Copy bytes from one location to another. The locations must not overlap.
  *
@@ -111,6 +112,52 @@ rte_mov256(uint8_t *dst, const uint8_t *src);
 static void *
 rte_memcpy(void *dst, const void *src, size_t n);
 
-#endif /* __DOXYGEN__ */
+#ifdef RTE_USE_CC_MEMCPY
+static inline void
+rte_mov16(uint8_t *dst, const uint8_t *src)
+{
+	memcpy(dst, src, 16);
+}
+
+static inline void
+rte_mov32(uint8_t *dst, const uint8_t *src)
+{
+	memcpy(dst, src, 32);
+}
+
+static inline void
+rte_mov48(uint8_t *dst, const uint8_t *src)
+{
+	memcpy(dst, src, 48);
+}
+
+static inline void
+rte_mov64(uint8_t *dst, const uint8_t *src)
+{
+	memcpy(dst, src, 64);
+}
+
+static inline void
+rte_mov128(uint8_t *dst, const uint8_t *src)
+{
+	memcpy(dst, src, 128);
+}
+
+static inline void
+rte_mov256(uint8_t *dst, const uint8_t *src)
+{
+	memcpy(dst, src, 256);
+}
+
+static inline void *
+rte_memcpy(void *dst, const void *src, size_t n)
+{
+	return memcpy(dst, src, n);
+}
+#endif /* RTE_USE_CC_MEMCPY */
+
+#ifdef __cplusplus
+}
+#endif
 
 #endif /* _RTE_MEMCPY_H_ */
diff --git a/lib/eal/loongarch/include/rte_memcpy.h b/lib/eal/loongarch/include/rte_memcpy.h
index 22578d40f4..344b4416b5 100644
--- a/lib/eal/loongarch/include/rte_memcpy.h
+++ b/lib/eal/loongarch/include/rte_memcpy.h
@@ -5,57 +5,12 @@
 #ifndef RTE_MEMCPY_LOONGARCH_H
 #define RTE_MEMCPY_LOONGARCH_H
 
-#include <stdint.h>
-#include <string.h>
+#include <rte_config.h>
 
-#include "rte_common.h"
-
-#ifdef __cplusplus
-extern "C" {
+#ifndef RTE_USE_CC_MEMCPY
+#define RTE_USE_CC_MEMCPY
 #endif
 
-#include "generic/rte_memcpy.h"
-
-static inline void
-rte_mov16(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 16);
-}
-
-static inline void
-rte_mov32(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 32);
-}
-
-static inline void
-rte_mov48(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 48);
-}
-
-static inline void
-rte_mov64(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 64);
-}
-
-static inline void
-rte_mov128(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 128);
-}
-
-static inline void
-rte_mov256(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 256);
-}
-
-#define rte_memcpy(d, s, n)	memcpy((d), (s), (n))
-
-#ifdef __cplusplus
-}
-#endif
+#include <generic/rte_memcpy.h>
 
 #endif /* RTE_MEMCPY_LOONGARCH_H */
diff --git a/lib/eal/ppc/include/rte_memcpy.h b/lib/eal/ppc/include/rte_memcpy.h
index 6f388c0234..645fd83986 100644
--- a/lib/eal/ppc/include/rte_memcpy.h
+++ b/lib/eal/ppc/include/rte_memcpy.h
@@ -6,6 +6,14 @@
 #ifndef _RTE_MEMCPY_PPC_64_H_
 #define _RTE_MEMCPY_PPC_64_H_
 
+#include <rte_config.h>
+
+#ifdef RTE_USE_CC_MEMCPY
+
+#include <generic/rte_memcpy.h>
+
+#else
+
 #include <stdint.h>
 #include <string.h>
 
@@ -215,4 +223,6 @@ rte_memcpy_func(void *dst, const void *src, size_t n)
 }
 #endif
 
+#endif /* RTE_USE_CC_MEMCPY */
+
 #endif /* _RTE_MEMCPY_PPC_64_H_ */
diff --git a/lib/eal/riscv/include/rte_memcpy.h b/lib/eal/riscv/include/rte_memcpy.h
index e34f19396e..4acdc4af5f 100644
--- a/lib/eal/riscv/include/rte_memcpy.h
+++ b/lib/eal/riscv/include/rte_memcpy.h
@@ -7,57 +7,12 @@
 #ifndef RTE_MEMCPY_RISCV_H
 #define RTE_MEMCPY_RISCV_H
 
-#include <stdint.h>
-#include <string.h>
+#include <rte_config.h>
 
-#include "rte_common.h"
-
-#ifdef __cplusplus
-extern "C" {
+#ifndef RTE_USE_CC_MEMCPY
+#define RTE_USE_CC_MEMCPY
 #endif
 
-#include "generic/rte_memcpy.h"
-
-static inline void
-rte_mov16(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 16);
-}
-
-static inline void
-rte_mov32(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 32);
-}
-
-static inline void
-rte_mov48(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 48);
-}
-
-static inline void
-rte_mov64(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 64);
-}
-
-static inline void
-rte_mov128(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 128);
-}
-
-static inline void
-rte_mov256(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 256);
-}
-
-#define rte_memcpy(d, s, n)	memcpy((d), (s), (n))
-
-#ifdef __cplusplus
-}
-#endif
+#include <generic/rte_memcpy.h>
 
 #endif /* RTE_MEMCPY_RISCV_H */
diff --git a/lib/eal/x86/include/meson.build b/lib/eal/x86/include/meson.build
index 52d2f8e969..09c2fe2485 100644
--- a/lib/eal/x86/include/meson.build
+++ b/lib/eal/x86/include/meson.build
@@ -16,6 +16,7 @@ arch_headers = files(
         'rte_spinlock.h',
         'rte_vect.h',
 )
+
 arch_indirect_headers = files(
         'rte_atomic_32.h',
         'rte_atomic_64.h',
diff --git a/lib/eal/x86/include/rte_memcpy.h b/lib/eal/x86/include/rte_memcpy.h
index 72a92290e0..c5ba74d2ed 100644
--- a/lib/eal/x86/include/rte_memcpy.h
+++ b/lib/eal/x86/include/rte_memcpy.h
@@ -11,12 +11,19 @@
  * Functions for SSE/AVX/AVX2/AVX512 implementation of memcpy().
  */
 
+#include <rte_config.h>
+
+#ifdef RTE_USE_CC_MEMCPY
+
+#include <generic/rte_memcpy.h>
+
+#else
+
 #include <stdio.h>
 #include <stdint.h>
 #include <string.h>
 #include <rte_vect.h>
 #include <rte_common.h>
-#include <rte_config.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -878,4 +885,6 @@ rte_memcpy(void *dst, const void *src, size_t n)
 }
 #endif
 
+#endif /* RTE_USE_CC_MEMCPY */
+
 #endif /* _RTE_MEMCPY_X86_64_H_ */
diff --git a/meson_options.txt b/meson_options.txt
index e49b2fc089..06f544b631 100644
--- a/meson_options.txt
+++ b/meson_options.txt
@@ -60,3 +60,5 @@ option('tests', type: 'boolean', value: true, description:
        'build unit tests')
 option('use_hpet', type: 'boolean', value: false, description:
        'use HPET timer in EAL')
+option('use_cc_memcpy', type: 'boolean', value: true, description:
+       'Have the functions of <rte_memcpy.h> delegate to compiler/libc memcpy() instead of using custom implementation.')
-- 
2.34.1

Re: [PATCH v2 1/6] net/fm10k: add missing intrinsic include

[Bruce Richardson]

On Thu, Jun 20, 2024 at 09:24:47AM +0200, Mattias Rönnblom wrote:
> Add missing <emmintrin.h> include, to get the _mm_cvtsi128_si64
> prototype.
> 
> Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
> ---
Acked-by: Bruce Richardson <bruce.richardson@intel.com>

Re: [PATCH v2 2/6] event/dlb2: include headers for vector and memory copy APIs

[Bruce Richardson]

On Thu, Jun 20, 2024 at 09:24:48AM +0200, Mattias Rönnblom wrote:
> The DLB2 PMD depended on <rte_vect.h> being included as a side-effect
> of <rte_memcpy.h> being included.
> 
> In addition, DLB2 used rte_memcpy() but did not include <rte_memcpy.h>,
> but rather depended on other include files to do so.
> 
> This patch addresses both of those issues.
> 
> Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
> ---
Acked-by: Bruce Richardson <bruce.richardson@intel.com>

Re: [PATCH v2 4/6] distributor: properly include vector API header file

[Bruce Richardson]

On Thu, Jun 20, 2024 at 09:24:50AM +0200, Mattias Rönnblom wrote:
> The distributor library relied on <rte_vect.h>, but failed to provide
> a direct include of this file.
> 
> Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
> ---
Acked-by: Bruce Richardson <bruce.richardson@intel.com>

Re: [PATCH v2 5/6] fib: properly include vector API header file

[Bruce Richardson]

On Thu, Jun 20, 2024 at 09:24:51AM +0200, Mattias Rönnblom wrote:
> The trie implementation of the fib library relied on <rte_vect.h>, but
> failed to provide a direct include of this file.
> 
> Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
> ---
Acked-by: Bruce Richardson <bruce.richardson@intel.com>

Re: [PATCH v2 1/6] net/fm10k: add missing intrinsic include

[Bruce Richardson]

On Thu, Jun 20, 2024 at 09:24:47AM +0200, Mattias Rönnblom wrote:
> Add missing <emmintrin.h> include, to get the _mm_cvtsi128_si64
> prototype.
> 
> Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
> ---
>  drivers/net/fm10k/fm10k_rxtx_vec.c | 1 +
>  1 file changed, 1 insertion(+)
> 
> diff --git a/drivers/net/fm10k/fm10k_rxtx_vec.c b/drivers/net/fm10k/fm10k_rxtx_vec.c
> index 2b6914b1da..d417b31bbb 100644
> --- a/drivers/net/fm10k/fm10k_rxtx_vec.c
> +++ b/drivers/net/fm10k/fm10k_rxtx_vec.c
> @@ -10,6 +10,7 @@
>  #include "base/fm10k_type.h"
>  
>  #include <tmmintrin.h>
> +#include <emmintrin.h>
>  
Beyond my ack of this patch, a small suggestion is to just include
rte_vect.h rather than trying to include specific x86-intrinsics headers.

My ack remains with or without taking on board this suggestion.

/Bruce

Re: [PATCH v2 1/6] net/fm10k: add missing intrinsic include

[Mattias Rönnblom]

On 2024-06-20 11:28, Bruce Richardson wrote:
> On Thu, Jun 20, 2024 at 09:24:47AM +0200, Mattias Rönnblom wrote:
>> Add missing <emmintrin.h> include, to get the _mm_cvtsi128_si64
>> prototype.
>>
>> Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
>> ---
>>   drivers/net/fm10k/fm10k_rxtx_vec.c | 1 +
>>   1 file changed, 1 insertion(+)
>>
>> diff --git a/drivers/net/fm10k/fm10k_rxtx_vec.c b/drivers/net/fm10k/fm10k_rxtx_vec.c
>> index 2b6914b1da..d417b31bbb 100644
>> --- a/drivers/net/fm10k/fm10k_rxtx_vec.c
>> +++ b/drivers/net/fm10k/fm10k_rxtx_vec.c
>> @@ -10,6 +10,7 @@
>>   #include "base/fm10k_type.h"
>>   
>>   #include <tmmintrin.h>
>> +#include <emmintrin.h>
>>   
> Beyond my ack of this patch, a small suggestion is to just include
> rte_vect.h rather than trying to include specific x86-intrinsics headers.
> 
> My ack remains with or without taking on board this suggestion.
> 
> /Bruce

I will do that, and hope it will magically solve the 
_mm_cvtsi128_si64-on-32-bit-x86 issue.

[PATCH v3 0/6] Optionally have rte_memcpy delegate to compiler memcpy

[Mattias Rönnblom]

This patch set make DPDK library, driver, and application code use the
compiler/libc memcpy() by default when functions in <rte_memcpy.h> are
invoked.

The various custom DPDK rte_memcpy() implementations may be retained
by means of a build-time option.

This patch set only make a difference on x86, PPC and ARM. Loongarch
and RISCV already used compiler/libc memcpy().

This patch set includes a number of fixes in drivers and libraries
which errornously relied on <rte_memcpy.h> including header files
(i.e., <rte_vect.h>) required by its implementation.

Mattias Rönnblom (6):
  net/fm10k: add missing vector API header include
  event/dlb2: include headers for vector and memory copy APIs
  net/octeon_ep: add missing vector API header include
  distributor: add missing vector API header include
  fib: add missing vector API header include
  eal: provide option to use compiler memcpy instead of RTE

 config/meson.build                     |  1 +
 doc/guides/rel_notes/release_24_07.rst | 21 +++++++++
 drivers/event/dlb2/dlb2.c              |  2 +
 drivers/net/fm10k/fm10k_rxtx_vec.c     |  1 +
 drivers/net/octeon_ep/otx_ep_ethdev.c  |  2 +
 lib/distributor/rte_distributor.c      |  1 +
 lib/eal/arm/include/rte_memcpy.h       | 10 +++++
 lib/eal/include/generic/rte_memcpy.h   | 61 +++++++++++++++++++++++---
 lib/eal/loongarch/include/rte_memcpy.h | 53 ++--------------------
 lib/eal/ppc/include/rte_memcpy.h       | 10 +++++
 lib/eal/riscv/include/rte_memcpy.h     | 53 ++--------------------
 lib/eal/x86/include/meson.build        |  1 +
 lib/eal/x86/include/rte_memcpy.h       | 11 ++++-
 lib/fib/trie.c                         |  1 +
 meson_options.txt                      |  2 +
 15 files changed, 124 insertions(+), 106 deletions(-)

-- 
2.34.1

[PATCH v3 1/6] net/fm10k: add missing vector API header include

[Mattias Rönnblom]

The fm10k PMD relied on <rte_vect.h>, but failed to provide a direct
include of this file.

Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
Acked-by: Bruce Richardson <bruce.richardson@intel.com>
---
 drivers/net/fm10k/fm10k_rxtx_vec.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/drivers/net/fm10k/fm10k_rxtx_vec.c b/drivers/net/fm10k/fm10k_rxtx_vec.c
index 2b6914b1da..62119de373 100644
--- a/drivers/net/fm10k/fm10k_rxtx_vec.c
+++ b/drivers/net/fm10k/fm10k_rxtx_vec.c
@@ -6,6 +6,7 @@
 
 #include <ethdev_driver.h>
 #include <rte_common.h>
+#include <rte_vect.h>
 #include "fm10k.h"
 #include "base/fm10k_type.h"
 
-- 
2.34.1

[PATCH v3 2/6] event/dlb2: include headers for vector and memory copy APIs

[Mattias Rönnblom]

The DLB2 PMD depended on <rte_vect.h> being included as a side-effect
of <rte_memcpy.h> being included.

In addition, DLB2 used rte_memcpy() but did not include <rte_memcpy.h>,
but rather depended on other include files to do so.

This patch addresses both of those issues.

Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
Acked-by: Bruce Richardson <bruce.richardson@intel.com>
---
 drivers/event/dlb2/dlb2.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/drivers/event/dlb2/dlb2.c b/drivers/event/dlb2/dlb2.c
index 0b91f03956..19f90b8f8d 100644
--- a/drivers/event/dlb2/dlb2.c
+++ b/drivers/event/dlb2/dlb2.c
@@ -25,11 +25,13 @@
 #include <rte_kvargs.h>
 #include <rte_log.h>
 #include <rte_malloc.h>
+#include <rte_memcpy.h>
 #include <rte_mbuf.h>
 #include <rte_power_intrinsics.h>
 #include <rte_prefetch.h>
 #include <rte_ring.h>
 #include <rte_string_fns.h>
+#include <rte_vect.h>
 
 #include "dlb2_priv.h"
 #include "dlb2_iface.h"
-- 
2.34.1

[PATCH v3 3/6] net/octeon_ep: add missing vector API header include

[Mattias Rönnblom]

The octeon_ip driver relied on <rte_vect.h>, but failed to provide a
direct include of this file.

Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
---
 drivers/net/octeon_ep/otx_ep_ethdev.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/drivers/net/octeon_ep/otx_ep_ethdev.c b/drivers/net/octeon_ep/otx_ep_ethdev.c
index 46211361a0..b069216629 100644
--- a/drivers/net/octeon_ep/otx_ep_ethdev.c
+++ b/drivers/net/octeon_ep/otx_ep_ethdev.c
@@ -5,6 +5,8 @@
 #include <inttypes.h>
 #include <ethdev_pci.h>
 
+#include <rte_vect.h>
+
 #include "otx_ep_common.h"
 #include "otx_ep_vf.h"
 #include "otx2_ep_vf.h"
-- 
2.34.1

[PATCH v3 4/6] distributor: add missing vector API header include

[Mattias Rönnblom]

The distributor library relied on <rte_vect.h>, but failed to provide
a direct include of this file.

Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
Acked-by: Bruce Richardson <bruce.richardson@intel.com>
---
 lib/distributor/rte_distributor.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/distributor/rte_distributor.c b/lib/distributor/rte_distributor.c
index e58727cdc2..1389efc03f 100644
--- a/lib/distributor/rte_distributor.c
+++ b/lib/distributor/rte_distributor.c
@@ -15,6 +15,7 @@
 #include <rte_eal_memconfig.h>
 #include <rte_pause.h>
 #include <rte_tailq.h>
+#include <rte_vect.h>
 
 #include "rte_distributor.h"
 #include "rte_distributor_single.h"
-- 
2.34.1

[PATCH v3 5/6] fib: add missing vector API header include

[Mattias Rönnblom]

The trie implementation of the fib library relied on <rte_vect.h>, but
failed to provide a direct include of this file.

Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
Acked-by: Bruce Richardson <bruce.richardson@intel.com>
---
 lib/fib/trie.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/fib/trie.c b/lib/fib/trie.c
index 09470e7287..74db8863df 100644
--- a/lib/fib/trie.c
+++ b/lib/fib/trie.c
@@ -9,6 +9,7 @@
 #include <rte_debug.h>
 #include <rte_malloc.h>
 #include <rte_errno.h>
+#include <rte_vect.h>
 
 #include <rte_rib6.h>
 #include <rte_fib6.h>
-- 
2.34.1

[PATCH v3 6/6] eal: provide option to use compiler memcpy instead of RTE

[Mattias Rönnblom]

Provide build option to have functions in <rte_memcpy.h> delegate to
the standard compiler/libc memcpy(), instead of using the various
custom DPDK, handcrafted, per-architecture rte_memcpy()
implementations.

A new meson build option 'use_cc_memcpy' is added. By default,
the compiler/libc memcpy() is used.

The performance benefits of the custom DPDK rte_memcpy()
implementations have been diminishing with every compiler release, and
with current toolchains the use of a custom memcpy() implementation
may even be a liability.

This patch leaves an option to stay on the custom DPDK implementations,
would that prove beneficial for certain applications or architectures.

An additional benefit of this change is that compilers and static
analysis tools have an easier time detecting incorrect usage of
rte_memcpy() (e.g., buffer overruns, or overlapping source and
destination buffers).

Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
Acked-by: Morten Brørup <mb@smartsharesystems.com>

---

PATCH:
 o Add entry in release notes.
 o Update meson help text.

RFC v3:
 o Fix missing #endif on loongarch.
 o PPC and RISCV now implemented, meaning all architectures are supported.
 o Unnecessary <rte_vect.h> include is removed from <rte_memcpy.h>.

RFC v2:
 * Fix bug where rte_memcpy.h was not installed on x86.
 * Made attempt to make Loongarch compile.
---
 config/meson.build                     |  1 +
 doc/guides/rel_notes/release_24_07.rst | 21 +++++++++
 lib/eal/arm/include/rte_memcpy.h       | 10 +++++
 lib/eal/include/generic/rte_memcpy.h   | 61 +++++++++++++++++++++++---
 lib/eal/loongarch/include/rte_memcpy.h | 53 ++--------------------
 lib/eal/ppc/include/rte_memcpy.h       | 10 +++++
 lib/eal/riscv/include/rte_memcpy.h     | 53 ++--------------------
 lib/eal/x86/include/meson.build        |  1 +
 lib/eal/x86/include/rte_memcpy.h       | 11 ++++-
 meson_options.txt                      |  2 +
 10 files changed, 117 insertions(+), 106 deletions(-)

diff --git a/config/meson.build b/config/meson.build
index 8c8b019c25..456056628e 100644
--- a/config/meson.build
+++ b/config/meson.build
@@ -353,6 +353,7 @@ endforeach
 # set other values pulled from the build options
 dpdk_conf.set('RTE_MAX_ETHPORTS', get_option('max_ethports'))
 dpdk_conf.set('RTE_LIBEAL_USE_HPET', get_option('use_hpet'))
+dpdk_conf.set('RTE_USE_CC_MEMCPY', get_option('use_cc_memcpy'))
 dpdk_conf.set('RTE_ENABLE_STDATOMIC', get_option('enable_stdatomic'))
 dpdk_conf.set('RTE_ENABLE_TRACE_FP', get_option('enable_trace_fp'))
 dpdk_conf.set('RTE_PKTMBUF_HEADROOM', get_option('pkt_mbuf_headroom'))
diff --git a/doc/guides/rel_notes/release_24_07.rst b/doc/guides/rel_notes/release_24_07.rst
index 7c88de381b..ebe0085d8b 100644
--- a/doc/guides/rel_notes/release_24_07.rst
+++ b/doc/guides/rel_notes/release_24_07.rst
@@ -24,6 +24,27 @@ DPDK Release 24.07
 New Features
 ------------
 
+* **Compiler memcpy replaces custom DPDK implementation.**
+
+  The memory copy functions of ``<rte_memcpy.h>`` now delegates to the
+  standard memcpy() function, implemented by the compiler and the C
+  runtime (e.g., libc).
+
+  In this release of DPDK, the handcrafted, per-architecture memory
+  copy implementations are still available, and may be reactivated by
+  setting the new ``use_cc_memcpy`` build option to false.
+
+  The performance benefits of the custom DPDK rte_memcpy()
+  implementations have been diminishing with every new compiler
+  release, and with current toolchains the use of a custom memcpy()
+  implementation may even result in worse performance than the
+  standard memcpy().
+
+  An additional benefit of this change is that compilers and static
+  analysis tools have an easier time detecting incorrect usage of
+  rte_memcpy() (e.g., buffer overruns, or overlapping source and
+  destination buffers).
+
 .. This section should contain new features added in this release.
    Sample format:
 
diff --git a/lib/eal/arm/include/rte_memcpy.h b/lib/eal/arm/include/rte_memcpy.h
index 47dea9a8cc..e8aff722df 100644
--- a/lib/eal/arm/include/rte_memcpy.h
+++ b/lib/eal/arm/include/rte_memcpy.h
@@ -5,10 +5,20 @@
 #ifndef _RTE_MEMCPY_ARM_H_
 #define _RTE_MEMCPY_ARM_H_
 
+#include <rte_config.h>
+
+#ifdef RTE_USE_CC_MEMCPY
+
+#include <generic/rte_memcpy.h>
+
+#else
+
 #ifdef RTE_ARCH_64
 #include <rte_memcpy_64.h>
 #else
 #include <rte_memcpy_32.h>
 #endif
 
+#endif /* RTE_USE_CC_MEMCPY */
+
 #endif /* _RTE_MEMCPY_ARM_H_ */
diff --git a/lib/eal/include/generic/rte_memcpy.h b/lib/eal/include/generic/rte_memcpy.h
index e7f0f8eaa9..cae06117fb 100644
--- a/lib/eal/include/generic/rte_memcpy.h
+++ b/lib/eal/include/generic/rte_memcpy.h
@@ -5,12 +5,19 @@
 #ifndef _RTE_MEMCPY_H_
 #define _RTE_MEMCPY_H_
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 /**
  * @file
  *
  * Functions for vectorised implementation of memcpy().
  */
 
+#include <stdint.h>
+#include <string.h>
+
 /**
  * Copy 16 bytes from one location to another using optimised
  * instructions. The locations should not overlap.
@@ -35,8 +42,6 @@ rte_mov16(uint8_t *dst, const uint8_t *src);
 static inline void
 rte_mov32(uint8_t *dst, const uint8_t *src);
 
-#ifdef __DOXYGEN__
-
 /**
  * Copy 48 bytes from one location to another using optimised
  * instructions. The locations should not overlap.
@@ -49,8 +54,6 @@ rte_mov32(uint8_t *dst, const uint8_t *src);
 static inline void
 rte_mov48(uint8_t *dst, const uint8_t *src);
 
-#endif /* __DOXYGEN__ */
-
 /**
  * Copy 64 bytes from one location to another using optimised
  * instructions. The locations should not overlap.
@@ -87,8 +90,6 @@ rte_mov128(uint8_t *dst, const uint8_t *src);
 static inline void
 rte_mov256(uint8_t *dst, const uint8_t *src);
 
-#ifdef __DOXYGEN__
-
 /**
  * Copy bytes from one location to another. The locations must not overlap.
  *
@@ -111,6 +112,52 @@ rte_mov256(uint8_t *dst, const uint8_t *src);
 static void *
 rte_memcpy(void *dst, const void *src, size_t n);
 
-#endif /* __DOXYGEN__ */
+#ifdef RTE_USE_CC_MEMCPY
+static inline void
+rte_mov16(uint8_t *dst, const uint8_t *src)
+{
+	memcpy(dst, src, 16);
+}
+
+static inline void
+rte_mov32(uint8_t *dst, const uint8_t *src)
+{
+	memcpy(dst, src, 32);
+}
+
+static inline void
+rte_mov48(uint8_t *dst, const uint8_t *src)
+{
+	memcpy(dst, src, 48);
+}
+
+static inline void
+rte_mov64(uint8_t *dst, const uint8_t *src)
+{
+	memcpy(dst, src, 64);
+}
+
+static inline void
+rte_mov128(uint8_t *dst, const uint8_t *src)
+{
+	memcpy(dst, src, 128);
+}
+
+static inline void
+rte_mov256(uint8_t *dst, const uint8_t *src)
+{
+	memcpy(dst, src, 256);
+}
+
+static inline void *
+rte_memcpy(void *dst, const void *src, size_t n)
+{
+	return memcpy(dst, src, n);
+}
+#endif /* RTE_USE_CC_MEMCPY */
+
+#ifdef __cplusplus
+}
+#endif
 
 #endif /* _RTE_MEMCPY_H_ */
diff --git a/lib/eal/loongarch/include/rte_memcpy.h b/lib/eal/loongarch/include/rte_memcpy.h
index 22578d40f4..344b4416b5 100644
--- a/lib/eal/loongarch/include/rte_memcpy.h
+++ b/lib/eal/loongarch/include/rte_memcpy.h
@@ -5,57 +5,12 @@
 #ifndef RTE_MEMCPY_LOONGARCH_H
 #define RTE_MEMCPY_LOONGARCH_H
 
-#include <stdint.h>
-#include <string.h>
+#include <rte_config.h>
 
-#include "rte_common.h"
-
-#ifdef __cplusplus
-extern "C" {
+#ifndef RTE_USE_CC_MEMCPY
+#define RTE_USE_CC_MEMCPY
 #endif
 
-#include "generic/rte_memcpy.h"
-
-static inline void
-rte_mov16(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 16);
-}
-
-static inline void
-rte_mov32(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 32);
-}
-
-static inline void
-rte_mov48(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 48);
-}
-
-static inline void
-rte_mov64(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 64);
-}
-
-static inline void
-rte_mov128(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 128);
-}
-
-static inline void
-rte_mov256(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 256);
-}
-
-#define rte_memcpy(d, s, n)	memcpy((d), (s), (n))
-
-#ifdef __cplusplus
-}
-#endif
+#include <generic/rte_memcpy.h>
 
 #endif /* RTE_MEMCPY_LOONGARCH_H */
diff --git a/lib/eal/ppc/include/rte_memcpy.h b/lib/eal/ppc/include/rte_memcpy.h
index 6f388c0234..645fd83986 100644
--- a/lib/eal/ppc/include/rte_memcpy.h
+++ b/lib/eal/ppc/include/rte_memcpy.h
@@ -6,6 +6,14 @@
 #ifndef _RTE_MEMCPY_PPC_64_H_
 #define _RTE_MEMCPY_PPC_64_H_
 
+#include <rte_config.h>
+
+#ifdef RTE_USE_CC_MEMCPY
+
+#include <generic/rte_memcpy.h>
+
+#else
+
 #include <stdint.h>
 #include <string.h>
 
@@ -215,4 +223,6 @@ rte_memcpy_func(void *dst, const void *src, size_t n)
 }
 #endif
 
+#endif /* RTE_USE_CC_MEMCPY */
+
 #endif /* _RTE_MEMCPY_PPC_64_H_ */
diff --git a/lib/eal/riscv/include/rte_memcpy.h b/lib/eal/riscv/include/rte_memcpy.h
index e34f19396e..4acdc4af5f 100644
--- a/lib/eal/riscv/include/rte_memcpy.h
+++ b/lib/eal/riscv/include/rte_memcpy.h
@@ -7,57 +7,12 @@
 #ifndef RTE_MEMCPY_RISCV_H
 #define RTE_MEMCPY_RISCV_H
 
-#include <stdint.h>
-#include <string.h>
+#include <rte_config.h>
 
-#include "rte_common.h"
-
-#ifdef __cplusplus
-extern "C" {
+#ifndef RTE_USE_CC_MEMCPY
+#define RTE_USE_CC_MEMCPY
 #endif
 
-#include "generic/rte_memcpy.h"
-
-static inline void
-rte_mov16(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 16);
-}
-
-static inline void
-rte_mov32(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 32);
-}
-
-static inline void
-rte_mov48(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 48);
-}
-
-static inline void
-rte_mov64(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 64);
-}
-
-static inline void
-rte_mov128(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 128);
-}
-
-static inline void
-rte_mov256(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 256);
-}
-
-#define rte_memcpy(d, s, n)	memcpy((d), (s), (n))
-
-#ifdef __cplusplus
-}
-#endif
+#include <generic/rte_memcpy.h>
 
 #endif /* RTE_MEMCPY_RISCV_H */
diff --git a/lib/eal/x86/include/meson.build b/lib/eal/x86/include/meson.build
index 52d2f8e969..09c2fe2485 100644
--- a/lib/eal/x86/include/meson.build
+++ b/lib/eal/x86/include/meson.build
@@ -16,6 +16,7 @@ arch_headers = files(
         'rte_spinlock.h',
         'rte_vect.h',
 )
+
 arch_indirect_headers = files(
         'rte_atomic_32.h',
         'rte_atomic_64.h',
diff --git a/lib/eal/x86/include/rte_memcpy.h b/lib/eal/x86/include/rte_memcpy.h
index 72a92290e0..c5ba74d2ed 100644
--- a/lib/eal/x86/include/rte_memcpy.h
+++ b/lib/eal/x86/include/rte_memcpy.h
@@ -11,12 +11,19 @@
  * Functions for SSE/AVX/AVX2/AVX512 implementation of memcpy().
  */
 
+#include <rte_config.h>
+
+#ifdef RTE_USE_CC_MEMCPY
+
+#include <generic/rte_memcpy.h>
+
+#else
+
 #include <stdio.h>
 #include <stdint.h>
 #include <string.h>
 #include <rte_vect.h>
 #include <rte_common.h>
-#include <rte_config.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -878,4 +885,6 @@ rte_memcpy(void *dst, const void *src, size_t n)
 }
 #endif
 
+#endif /* RTE_USE_CC_MEMCPY */
+
 #endif /* _RTE_MEMCPY_X86_64_H_ */
diff --git a/meson_options.txt b/meson_options.txt
index e49b2fc089..06f544b631 100644
--- a/meson_options.txt
+++ b/meson_options.txt
@@ -60,3 +60,5 @@ option('tests', type: 'boolean', value: true, description:
        'build unit tests')
 option('use_hpet', type: 'boolean', value: false, description:
        'use HPET timer in EAL')
+option('use_cc_memcpy', type: 'boolean', value: true, description:
+       'Have the functions of <rte_memcpy.h> delegate to compiler/libc memcpy() instead of using custom implementation.')
-- 
2.34.1

Re: [PATCH v2 1/6] net/fm10k: add missing intrinsic include

[Bruce Richardson]

On Thu, Jun 20, 2024 at 01:40:42PM +0200, Mattias Rönnblom wrote:
> On 2024-06-20 11:28, Bruce Richardson wrote:
> > On Thu, Jun 20, 2024 at 09:24:47AM +0200, Mattias Rönnblom wrote:
> > > Add missing <emmintrin.h> include, to get the _mm_cvtsi128_si64
> > > prototype.
> > > 
> > > Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
> > > ---
> > >   drivers/net/fm10k/fm10k_rxtx_vec.c | 1 +
> > >   1 file changed, 1 insertion(+)
> > > 
> > > diff --git a/drivers/net/fm10k/fm10k_rxtx_vec.c b/drivers/net/fm10k/fm10k_rxtx_vec.c
> > > index 2b6914b1da..d417b31bbb 100644
> > > --- a/drivers/net/fm10k/fm10k_rxtx_vec.c
> > > +++ b/drivers/net/fm10k/fm10k_rxtx_vec.c
> > > @@ -10,6 +10,7 @@
> > >   #include "base/fm10k_type.h"
> > >   #include <tmmintrin.h>
> > > +#include <emmintrin.h>
> > Beyond my ack of this patch, a small suggestion is to just include
> > rte_vect.h rather than trying to include specific x86-intrinsics headers.
> > 
> > My ack remains with or without taking on board this suggestion.
> > 
> > /Bruce
> 
> I will do that, and hope it will magically solve the
> _mm_cvtsi128_si64-on-32-bit-x86 issue.

I was looking at that, and it does solve it in my testing. There are a lot
of drivers that have just "tmmintrin.h" included. Changing all of those to
rte_vect.h allows 32bit to build with your other changes applied.

/Bruce

Re: [PATCH v3 1/6] net/fm10k: add missing vector API header include

[Bruce Richardson]

On Thu, Jun 20, 2024 at 01:50:22PM +0200, Mattias Rönnblom wrote:
> The fm10k PMD relied on <rte_vect.h>, but failed to provide a direct
> include of this file.
> 
> Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
> Acked-by: Bruce Richardson <bruce.richardson@intel.com>
> ---
>  drivers/net/fm10k/fm10k_rxtx_vec.c | 1 +
>  1 file changed, 1 insertion(+)
> 
To fix 32-bit builds, more than just this driver needs to be fixed. See
https://patches.dpdk.org/project/dpdk/patch/20240620123218.1936250-1-bruce.richardson@intel.com/

Feel free to include this patch in new revisions of your patchset, if it
simplifies things for you.

/Bruce

Re: [PATCH v2 5/6] fib: properly include vector API header file

[Stephen Hemminger]

On Thu, 20 Jun 2024 10:14:18 +0100
Bruce Richardson <bruce.richardson@intel.com> wrote:

> On Thu, Jun 20, 2024 at 09:24:51AM +0200, Mattias Rönnblom wrote:
> > The trie implementation of the fib library relied on <rte_vect.h>, but
> > failed to provide a direct include of this file.
> > 
> > Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
> > ---  
> Acked-by: Bruce Richardson <bruce.richardson@intel.com>

Acked-by: Stephen Hemminger <stephen@networkplumber.org>

Re: [PATCH v2 3/6] net/octeon_ep: properly include vector API header file

[Stephen Hemminger]

On Thu, 20 Jun 2024 09:24:49 +0200
Mattias Rönnblom <mattias.ronnblom@ericsson.com> wrote:

> The octeon_ip driver relied on <rte_vect.h>, but failed to provide a
> direct include of this file.
> 
> Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
> ---

Acked-by: Stephen Hemminger <stephen@networkplumber.org>

[PATCH v4 00/13] Optionally have rte_memcpy delegate to compiler memcpy

[Mattias Rönnblom]

This patch set make DPDK library, driver, and application code use the
compiler/libc memcpy() by default when functions in <rte_memcpy.h> are
invoked.

The various custom DPDK rte_memcpy() implementations may be retained
by means of a build-time option.

This patch set only make a difference on x86, PPC and ARM. Loongarch
and RISCV already used compiler/libc memcpy().

This patch set includes a number of fixes in drivers and libraries
which errornously relied on <rte_memcpy.h> including header files
(i.e., <rte_vect.h>) required by its implementation.

Mattias Rönnblom (13):
  net/i40e: add missing vector API header include
  net/iavf: add missing vector API header include
  net/ice: add missing vector API header include
  net/ixgbe: add missing vector API header include
  net/ngbe: add missing vector API header include
  net/txgbe: add missing vector API header include
  net/virtio: add missing vector API header include
  net/fm10k: add missing vector API header include
  event/dlb2: include headers for vector and memory copy APIs
  net/octeon_ep: add missing vector API header include
  distributor: add missing vector API header include
  fib: add missing vector API header include
  eal: provide option to use compiler memcpy instead of RTE

 config/meson.build                          |  1 +
 doc/guides/rel_notes/release_24_07.rst      | 21 +++++++
 drivers/event/dlb2/dlb2.c                   |  2 +
 drivers/net/fm10k/fm10k_rxtx_vec.c          |  3 +-
 drivers/net/i40e/i40e_rxtx_vec_sse.c        |  3 +-
 drivers/net/iavf/iavf_rxtx_vec_sse.c        |  3 +-
 drivers/net/ice/ice_rxtx_vec_sse.c          |  2 +-
 drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c      |  3 +-
 drivers/net/ngbe/ngbe_rxtx_vec_sse.c        |  3 +-
 drivers/net/octeon_ep/otx_ep_ethdev.c       |  2 +
 drivers/net/txgbe/txgbe_rxtx_vec_sse.c      |  3 +-
 drivers/net/virtio/virtio_rxtx_simple_sse.c |  3 +-
 lib/distributor/rte_distributor.c           |  1 +
 lib/eal/arm/include/rte_memcpy.h            | 10 ++++
 lib/eal/include/generic/rte_memcpy.h        | 61 ++++++++++++++++++---
 lib/eal/loongarch/include/rte_memcpy.h      | 53 ++----------------
 lib/eal/ppc/include/rte_memcpy.h            | 10 ++++
 lib/eal/riscv/include/rte_memcpy.h          | 53 ++----------------
 lib/eal/x86/include/meson.build             |  1 +
 lib/eal/x86/include/rte_memcpy.h            | 11 +++-
 lib/fib/trie.c                              |  1 +
 meson_options.txt                           |  2 +
 22 files changed, 131 insertions(+), 121 deletions(-)

-- 
2.34.1

[PATCH v4 01/13] net/i40e: add missing vector API header include

[Mattias Rönnblom]

The i40e driver relied on <rte_vect.h>, but failed to provide a direct
include of this file.

Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
---
 drivers/net/i40e/i40e_rxtx_vec_sse.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/drivers/net/i40e/i40e_rxtx_vec_sse.c b/drivers/net/i40e/i40e_rxtx_vec_sse.c
index 2d4480a765..0a0448544f 100644
--- a/drivers/net/i40e/i40e_rxtx_vec_sse.c
+++ b/drivers/net/i40e/i40e_rxtx_vec_sse.c
@@ -5,6 +5,7 @@
 #include <stdint.h>
 #include <ethdev_driver.h>
 #include <rte_malloc.h>
+#include <rte_vect.h>
 
 #include "base/i40e_prototype.h"
 #include "base/i40e_type.h"
@@ -12,8 +13,6 @@
 #include "i40e_rxtx.h"
 #include "i40e_rxtx_vec_common.h"
 
-#include <tmmintrin.h>
-
 #ifndef __INTEL_COMPILER
 #pragma GCC diagnostic ignored "-Wcast-qual"
 #endif
-- 
2.34.1

[PATCH v4 02/13] net/iavf: add missing vector API header include

[Mattias Rönnblom]

The iavf driver relied on <rte_vect.h>, but failed to provide a direct
include of this file.

Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
---
 drivers/net/iavf/iavf_rxtx_vec_sse.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/drivers/net/iavf/iavf_rxtx_vec_sse.c b/drivers/net/iavf/iavf_rxtx_vec_sse.c
index 96f187f511..75270876c1 100644
--- a/drivers/net/iavf/iavf_rxtx_vec_sse.c
+++ b/drivers/net/iavf/iavf_rxtx_vec_sse.c
@@ -5,13 +5,12 @@
 #include <stdint.h>
 #include <ethdev_driver.h>
 #include <rte_malloc.h>
+#include <rte_vect.h>
 
 #include "iavf.h"
 #include "iavf_rxtx.h"
 #include "iavf_rxtx_vec_common.h"
 
-#include <tmmintrin.h>
-
 #ifndef __INTEL_COMPILER
 #pragma GCC diagnostic ignored "-Wcast-qual"
 #endif
-- 
2.34.1

[PATCH v4 03/13] net/ice: add missing vector API header include

[Mattias Rönnblom]

The ice driver relied on <rte_vect.h>, but failed to provide a direct
include of this file.

Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
---
 drivers/net/ice/ice_rxtx_vec_sse.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/drivers/net/ice/ice_rxtx_vec_sse.c b/drivers/net/ice/ice_rxtx_vec_sse.c
index 9a1b7e3e51..c01d8ede29 100644
--- a/drivers/net/ice/ice_rxtx_vec_sse.c
+++ b/drivers/net/ice/ice_rxtx_vec_sse.c
@@ -4,7 +4,7 @@
 
 #include "ice_rxtx_vec_common.h"
 
-#include <tmmintrin.h>
+#include <rte_vect.h>
 
 #ifndef __INTEL_COMPILER
 #pragma GCC diagnostic ignored "-Wcast-qual"
-- 
2.34.1

[PATCH v4 04/13] net/ixgbe: add missing vector API header include

[Mattias Rönnblom]

The ixgbe driver relied on <rte_vect.h>, but failed to provide a
direct include of this file.

Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
---
 drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c b/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c
index f60808d576..0f93f58745 100644
--- a/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c
+++ b/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c
@@ -5,13 +5,12 @@
 #include <stdint.h>
 #include <ethdev_driver.h>
 #include <rte_malloc.h>
+#include <rte_vect.h>
 
 #include "ixgbe_ethdev.h"
 #include "ixgbe_rxtx.h"
 #include "ixgbe_rxtx_vec_common.h"
 
-#include <tmmintrin.h>
-
 #ifndef __INTEL_COMPILER
 #pragma GCC diagnostic ignored "-Wcast-qual"
 #endif
-- 
2.34.1

[PATCH v4 05/13] net/ngbe: add missing vector API header include

[Mattias Rönnblom]

The ngbe driver relied on <rte_vect.h>, but failed to provide a direct
include of this file.

Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
---
 drivers/net/ngbe/ngbe_rxtx_vec_sse.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/drivers/net/ngbe/ngbe_rxtx_vec_sse.c b/drivers/net/ngbe/ngbe_rxtx_vec_sse.c
index f703d0ea15..80d0bedcdd 100644
--- a/drivers/net/ngbe/ngbe_rxtx_vec_sse.c
+++ b/drivers/net/ngbe/ngbe_rxtx_vec_sse.c
@@ -5,14 +5,13 @@
 
 #include <ethdev_driver.h>
 #include <rte_malloc.h>
+#include <rte_vect.h>
 
 #include "ngbe_type.h"
 #include "ngbe_ethdev.h"
 #include "ngbe_rxtx.h"
 #include "ngbe_rxtx_vec_common.h"
 
-#include <tmmintrin.h>
-
 static inline void
 ngbe_rxq_rearm(struct ngbe_rx_queue *rxq)
 {
-- 
2.34.1

[PATCH v4 06/13] net/txgbe: add missing vector API header include

[Mattias Rönnblom]

The txgbe driver relied on <rte_vect.h>, but failed to provide a
direct include of this file.

Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
---
 drivers/net/txgbe/txgbe_rxtx_vec_sse.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/drivers/net/txgbe/txgbe_rxtx_vec_sse.c b/drivers/net/txgbe/txgbe_rxtx_vec_sse.c
index 12eb4aeef5..d5f60ec92e 100644
--- a/drivers/net/txgbe/txgbe_rxtx_vec_sse.c
+++ b/drivers/net/txgbe/txgbe_rxtx_vec_sse.c
@@ -5,13 +5,12 @@
 
 #include <ethdev_driver.h>
 #include <rte_malloc.h>
+#include <rte_vect.h>
 
 #include "txgbe_ethdev.h"
 #include "txgbe_rxtx.h"
 #include "txgbe_rxtx_vec_common.h"
 
-#include <tmmintrin.h>
-
 static inline void
 txgbe_rxq_rearm(struct txgbe_rx_queue *rxq)
 {
-- 
2.34.1

[PATCH v4 07/13] net/virtio: add missing vector API header include

[Mattias Rönnblom]

The virtio driver relied on <rte_vect.h>, but failed to provide a
direct include of this file.

Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
---
 drivers/net/virtio/virtio_rxtx_simple_sse.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/drivers/net/virtio/virtio_rxtx_simple_sse.c b/drivers/net/virtio/virtio_rxtx_simple_sse.c
index 6a18741b6d..db84a308e4 100644
--- a/drivers/net/virtio/virtio_rxtx_simple_sse.c
+++ b/drivers/net/virtio/virtio_rxtx_simple_sse.c
@@ -8,8 +8,6 @@
 #include <string.h>
 #include <errno.h>
 
-#include <tmmintrin.h>
-
 #include <rte_byteorder.h>
 #include <rte_branch_prediction.h>
 #include <rte_cycles.h>
@@ -22,6 +20,7 @@
 #include <rte_mbuf.h>
 #include <rte_prefetch.h>
 #include <rte_string_fns.h>
+#include <rte_vect.h>
 
 #include "virtio_rxtx_simple.h"
 
-- 
2.34.1

[PATCH v4 08/13] net/fm10k: add missing vector API header include

[Mattias Rönnblom]

The fm10k PMD relied on <rte_vect.h>, but failed to provide a direct
include of this file.

Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
Acked-by: Bruce Richardson <bruce.richardson@intel.com>
---
 drivers/net/fm10k/fm10k_rxtx_vec.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/drivers/net/fm10k/fm10k_rxtx_vec.c b/drivers/net/fm10k/fm10k_rxtx_vec.c
index 2b6914b1da..6be8822284 100644
--- a/drivers/net/fm10k/fm10k_rxtx_vec.c
+++ b/drivers/net/fm10k/fm10k_rxtx_vec.c
@@ -6,11 +6,10 @@
 
 #include <ethdev_driver.h>
 #include <rte_common.h>
+#include <rte_vect.h>
 #include "fm10k.h"
 #include "base/fm10k_type.h"
 
-#include <tmmintrin.h>
-
 #ifndef __INTEL_COMPILER
 #pragma GCC diagnostic ignored "-Wcast-qual"
 #endif
-- 
2.34.1

[PATCH v4 09/13] event/dlb2: include headers for vector and memory copy APIs

[Mattias Rönnblom]

The DLB2 PMD depended on <rte_vect.h> being included as a side-effect
of <rte_memcpy.h> being included.

In addition, DLB2 used rte_memcpy() but did not include <rte_memcpy.h>,
but rather depended on other include files to do so.

This patch addresses both of those issues.

Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
Acked-by: Bruce Richardson <bruce.richardson@intel.com>
---
 drivers/event/dlb2/dlb2.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/drivers/event/dlb2/dlb2.c b/drivers/event/dlb2/dlb2.c
index 0b91f03956..19f90b8f8d 100644
--- a/drivers/event/dlb2/dlb2.c
+++ b/drivers/event/dlb2/dlb2.c
@@ -25,11 +25,13 @@
 #include <rte_kvargs.h>
 #include <rte_log.h>
 #include <rte_malloc.h>
+#include <rte_memcpy.h>
 #include <rte_mbuf.h>
 #include <rte_power_intrinsics.h>
 #include <rte_prefetch.h>
 #include <rte_ring.h>
 #include <rte_string_fns.h>
+#include <rte_vect.h>
 
 #include "dlb2_priv.h"
 #include "dlb2_iface.h"
-- 
2.34.1

[PATCH v4 10/13] net/octeon_ep: add missing vector API header include

[Mattias Rönnblom]

The octeon_ip driver relied on <rte_vect.h>, but failed to provide a
direct include of this file.

Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
Acked-by: Stephen Hemminger <stephen@networkplumber.org>
---
 drivers/net/octeon_ep/otx_ep_ethdev.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/drivers/net/octeon_ep/otx_ep_ethdev.c b/drivers/net/octeon_ep/otx_ep_ethdev.c
index 46211361a0..b069216629 100644
--- a/drivers/net/octeon_ep/otx_ep_ethdev.c
+++ b/drivers/net/octeon_ep/otx_ep_ethdev.c
@@ -5,6 +5,8 @@
 #include <inttypes.h>
 #include <ethdev_pci.h>
 
+#include <rte_vect.h>
+
 #include "otx_ep_common.h"
 #include "otx_ep_vf.h"
 #include "otx2_ep_vf.h"
-- 
2.34.1

[PATCH v4 11/13] distributor: add missing vector API header include

[Mattias Rönnblom]

The distributor library relied on <rte_vect.h>, but failed to provide
a direct include of this file.

Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
Acked-by: Bruce Richardson <bruce.richardson@intel.com>
---
 lib/distributor/rte_distributor.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/distributor/rte_distributor.c b/lib/distributor/rte_distributor.c
index e58727cdc2..1389efc03f 100644
--- a/lib/distributor/rte_distributor.c
+++ b/lib/distributor/rte_distributor.c
@@ -15,6 +15,7 @@
 #include <rte_eal_memconfig.h>
 #include <rte_pause.h>
 #include <rte_tailq.h>
+#include <rte_vect.h>
 
 #include "rte_distributor.h"
 #include "rte_distributor_single.h"
-- 
2.34.1

[PATCH v4 12/13] fib: add missing vector API header include

[Mattias Rönnblom]

The trie implementation of the fib library relied on <rte_vect.h>, but
failed to provide a direct include of this file.

Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
Acked-by: Bruce Richardson <bruce.richardson@intel.com>
Acked-by: Stephen Hemminger <stephen@networkplumber.org>
---
 lib/fib/trie.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/fib/trie.c b/lib/fib/trie.c
index 09470e7287..74db8863df 100644
--- a/lib/fib/trie.c
+++ b/lib/fib/trie.c
@@ -9,6 +9,7 @@
 #include <rte_debug.h>
 #include <rte_malloc.h>
 #include <rte_errno.h>
+#include <rte_vect.h>
 
 #include <rte_rib6.h>
 #include <rte_fib6.h>
-- 
2.34.1

[PATCH v4 13/13] eal: provide option to use compiler memcpy instead of RTE

[Mattias Rönnblom]

Provide build option to have functions in <rte_memcpy.h> delegate to
the standard compiler/libc memcpy(), instead of using the various
custom DPDK, handcrafted, per-architecture rte_memcpy()
implementations.

A new meson build option 'use_cc_memcpy' is added. By default,
the compiler/libc memcpy() is used.

The performance benefits of the custom DPDK rte_memcpy()
implementations have been diminishing with every compiler release, and
with current toolchains the use of a custom memcpy() implementation
may even be a liability.

This patch leaves an option to stay on the custom DPDK implementations,
would that prove beneficial for certain applications or architectures.

An additional benefit of this change is that compilers and static
analysis tools have an easier time detecting incorrect usage of
rte_memcpy() (e.g., buffer overruns, or overlapping source and
destination buffers).

Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
Acked-by: Morten Brørup <mb@smartsharesystems.com>

---

PATCH:
 o Add entry in release notes.
 o Update meson help text.

RFC v3:
 o Fix missing #endif on loongarch.
 o PPC and RISCV now implemented, meaning all architectures are supported.
 o Unnecessary <rte_vect.h> include is removed from <rte_memcpy.h>.

RFC v2:
 * Fix bug where rte_memcpy.h was not installed on x86.
 * Made attempt to make Loongarch compile.
---
 config/meson.build                     |  1 +
 doc/guides/rel_notes/release_24_07.rst | 21 +++++++++
 lib/eal/arm/include/rte_memcpy.h       | 10 +++++
 lib/eal/include/generic/rte_memcpy.h   | 61 +++++++++++++++++++++++---
 lib/eal/loongarch/include/rte_memcpy.h | 53 ++--------------------
 lib/eal/ppc/include/rte_memcpy.h       | 10 +++++
 lib/eal/riscv/include/rte_memcpy.h     | 53 ++--------------------
 lib/eal/x86/include/meson.build        |  1 +
 lib/eal/x86/include/rte_memcpy.h       | 11 ++++-
 meson_options.txt                      |  2 +
 10 files changed, 117 insertions(+), 106 deletions(-)

diff --git a/config/meson.build b/config/meson.build
index 8c8b019c25..456056628e 100644
--- a/config/meson.build
+++ b/config/meson.build
@@ -353,6 +353,7 @@ endforeach
 # set other values pulled from the build options
 dpdk_conf.set('RTE_MAX_ETHPORTS', get_option('max_ethports'))
 dpdk_conf.set('RTE_LIBEAL_USE_HPET', get_option('use_hpet'))
+dpdk_conf.set('RTE_USE_CC_MEMCPY', get_option('use_cc_memcpy'))
 dpdk_conf.set('RTE_ENABLE_STDATOMIC', get_option('enable_stdatomic'))
 dpdk_conf.set('RTE_ENABLE_TRACE_FP', get_option('enable_trace_fp'))
 dpdk_conf.set('RTE_PKTMBUF_HEADROOM', get_option('pkt_mbuf_headroom'))
diff --git a/doc/guides/rel_notes/release_24_07.rst b/doc/guides/rel_notes/release_24_07.rst
index 7c88de381b..ebe0085d8b 100644
--- a/doc/guides/rel_notes/release_24_07.rst
+++ b/doc/guides/rel_notes/release_24_07.rst
@@ -24,6 +24,27 @@ DPDK Release 24.07
 New Features
 ------------
 
+* **Compiler memcpy replaces custom DPDK implementation.**
+
+  The memory copy functions of ``<rte_memcpy.h>`` now delegates to the
+  standard memcpy() function, implemented by the compiler and the C
+  runtime (e.g., libc).
+
+  In this release of DPDK, the handcrafted, per-architecture memory
+  copy implementations are still available, and may be reactivated by
+  setting the new ``use_cc_memcpy`` build option to false.
+
+  The performance benefits of the custom DPDK rte_memcpy()
+  implementations have been diminishing with every new compiler
+  release, and with current toolchains the use of a custom memcpy()
+  implementation may even result in worse performance than the
+  standard memcpy().
+
+  An additional benefit of this change is that compilers and static
+  analysis tools have an easier time detecting incorrect usage of
+  rte_memcpy() (e.g., buffer overruns, or overlapping source and
+  destination buffers).
+
 .. This section should contain new features added in this release.
    Sample format:
 
diff --git a/lib/eal/arm/include/rte_memcpy.h b/lib/eal/arm/include/rte_memcpy.h
index 47dea9a8cc..e8aff722df 100644
--- a/lib/eal/arm/include/rte_memcpy.h
+++ b/lib/eal/arm/include/rte_memcpy.h
@@ -5,10 +5,20 @@
 #ifndef _RTE_MEMCPY_ARM_H_
 #define _RTE_MEMCPY_ARM_H_
 
+#include <rte_config.h>
+
+#ifdef RTE_USE_CC_MEMCPY
+
+#include <generic/rte_memcpy.h>
+
+#else
+
 #ifdef RTE_ARCH_64
 #include <rte_memcpy_64.h>
 #else
 #include <rte_memcpy_32.h>
 #endif
 
+#endif /* RTE_USE_CC_MEMCPY */
+
 #endif /* _RTE_MEMCPY_ARM_H_ */
diff --git a/lib/eal/include/generic/rte_memcpy.h b/lib/eal/include/generic/rte_memcpy.h
index e7f0f8eaa9..cae06117fb 100644
--- a/lib/eal/include/generic/rte_memcpy.h
+++ b/lib/eal/include/generic/rte_memcpy.h
@@ -5,12 +5,19 @@
 #ifndef _RTE_MEMCPY_H_
 #define _RTE_MEMCPY_H_
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 /**
  * @file
  *
  * Functions for vectorised implementation of memcpy().
  */
 
+#include <stdint.h>
+#include <string.h>
+
 /**
  * Copy 16 bytes from one location to another using optimised
  * instructions. The locations should not overlap.
@@ -35,8 +42,6 @@ rte_mov16(uint8_t *dst, const uint8_t *src);
 static inline void
 rte_mov32(uint8_t *dst, const uint8_t *src);
 
-#ifdef __DOXYGEN__
-
 /**
  * Copy 48 bytes from one location to another using optimised
  * instructions. The locations should not overlap.
@@ -49,8 +54,6 @@ rte_mov32(uint8_t *dst, const uint8_t *src);
 static inline void
 rte_mov48(uint8_t *dst, const uint8_t *src);
 
-#endif /* __DOXYGEN__ */
-
 /**
  * Copy 64 bytes from one location to another using optimised
  * instructions. The locations should not overlap.
@@ -87,8 +90,6 @@ rte_mov128(uint8_t *dst, const uint8_t *src);
 static inline void
 rte_mov256(uint8_t *dst, const uint8_t *src);
 
-#ifdef __DOXYGEN__
-
 /**
  * Copy bytes from one location to another. The locations must not overlap.
  *
@@ -111,6 +112,52 @@ rte_mov256(uint8_t *dst, const uint8_t *src);
 static void *
 rte_memcpy(void *dst, const void *src, size_t n);
 
-#endif /* __DOXYGEN__ */
+#ifdef RTE_USE_CC_MEMCPY
+static inline void
+rte_mov16(uint8_t *dst, const uint8_t *src)
+{
+	memcpy(dst, src, 16);
+}
+
+static inline void
+rte_mov32(uint8_t *dst, const uint8_t *src)
+{
+	memcpy(dst, src, 32);
+}
+
+static inline void
+rte_mov48(uint8_t *dst, const uint8_t *src)
+{
+	memcpy(dst, src, 48);
+}
+
+static inline void
+rte_mov64(uint8_t *dst, const uint8_t *src)
+{
+	memcpy(dst, src, 64);
+}
+
+static inline void
+rte_mov128(uint8_t *dst, const uint8_t *src)
+{
+	memcpy(dst, src, 128);
+}
+
+static inline void
+rte_mov256(uint8_t *dst, const uint8_t *src)
+{
+	memcpy(dst, src, 256);
+}
+
+static inline void *
+rte_memcpy(void *dst, const void *src, size_t n)
+{
+	return memcpy(dst, src, n);
+}
+#endif /* RTE_USE_CC_MEMCPY */
+
+#ifdef __cplusplus
+}
+#endif
 
 #endif /* _RTE_MEMCPY_H_ */
diff --git a/lib/eal/loongarch/include/rte_memcpy.h b/lib/eal/loongarch/include/rte_memcpy.h
index 22578d40f4..344b4416b5 100644
--- a/lib/eal/loongarch/include/rte_memcpy.h
+++ b/lib/eal/loongarch/include/rte_memcpy.h
@@ -5,57 +5,12 @@
 #ifndef RTE_MEMCPY_LOONGARCH_H
 #define RTE_MEMCPY_LOONGARCH_H
 
-#include <stdint.h>
-#include <string.h>
+#include <rte_config.h>
 
-#include "rte_common.h"
-
-#ifdef __cplusplus
-extern "C" {
+#ifndef RTE_USE_CC_MEMCPY
+#define RTE_USE_CC_MEMCPY
 #endif
 
-#include "generic/rte_memcpy.h"
-
-static inline void
-rte_mov16(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 16);
-}
-
-static inline void
-rte_mov32(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 32);
-}
-
-static inline void
-rte_mov48(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 48);
-}
-
-static inline void
-rte_mov64(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 64);
-}
-
-static inline void
-rte_mov128(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 128);
-}
-
-static inline void
-rte_mov256(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 256);
-}
-
-#define rte_memcpy(d, s, n)	memcpy((d), (s), (n))
-
-#ifdef __cplusplus
-}
-#endif
+#include <generic/rte_memcpy.h>
 
 #endif /* RTE_MEMCPY_LOONGARCH_H */
diff --git a/lib/eal/ppc/include/rte_memcpy.h b/lib/eal/ppc/include/rte_memcpy.h
index 6f388c0234..645fd83986 100644
--- a/lib/eal/ppc/include/rte_memcpy.h
+++ b/lib/eal/ppc/include/rte_memcpy.h
@@ -6,6 +6,14 @@
 #ifndef _RTE_MEMCPY_PPC_64_H_
 #define _RTE_MEMCPY_PPC_64_H_
 
+#include <rte_config.h>
+
+#ifdef RTE_USE_CC_MEMCPY
+
+#include <generic/rte_memcpy.h>
+
+#else
+
 #include <stdint.h>
 #include <string.h>
 
@@ -215,4 +223,6 @@ rte_memcpy_func(void *dst, const void *src, size_t n)
 }
 #endif
 
+#endif /* RTE_USE_CC_MEMCPY */
+
 #endif /* _RTE_MEMCPY_PPC_64_H_ */
diff --git a/lib/eal/riscv/include/rte_memcpy.h b/lib/eal/riscv/include/rte_memcpy.h
index e34f19396e..4acdc4af5f 100644
--- a/lib/eal/riscv/include/rte_memcpy.h
+++ b/lib/eal/riscv/include/rte_memcpy.h
@@ -7,57 +7,12 @@
 #ifndef RTE_MEMCPY_RISCV_H
 #define RTE_MEMCPY_RISCV_H
 
-#include <stdint.h>
-#include <string.h>
+#include <rte_config.h>
 
-#include "rte_common.h"
-
-#ifdef __cplusplus
-extern "C" {
+#ifndef RTE_USE_CC_MEMCPY
+#define RTE_USE_CC_MEMCPY
 #endif
 
-#include "generic/rte_memcpy.h"
-
-static inline void
-rte_mov16(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 16);
-}
-
-static inline void
-rte_mov32(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 32);
-}
-
-static inline void
-rte_mov48(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 48);
-}
-
-static inline void
-rte_mov64(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 64);
-}
-
-static inline void
-rte_mov128(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 128);
-}
-
-static inline void
-rte_mov256(uint8_t *dst, const uint8_t *src)
-{
-	memcpy(dst, src, 256);
-}
-
-#define rte_memcpy(d, s, n)	memcpy((d), (s), (n))
-
-#ifdef __cplusplus
-}
-#endif
+#include <generic/rte_memcpy.h>
 
 #endif /* RTE_MEMCPY_RISCV_H */
diff --git a/lib/eal/x86/include/meson.build b/lib/eal/x86/include/meson.build
index 52d2f8e969..09c2fe2485 100644
--- a/lib/eal/x86/include/meson.build
+++ b/lib/eal/x86/include/meson.build
@@ -16,6 +16,7 @@ arch_headers = files(
         'rte_spinlock.h',
         'rte_vect.h',
 )
+
 arch_indirect_headers = files(
         'rte_atomic_32.h',
         'rte_atomic_64.h',
diff --git a/lib/eal/x86/include/rte_memcpy.h b/lib/eal/x86/include/rte_memcpy.h
index 72a92290e0..c5ba74d2ed 100644
--- a/lib/eal/x86/include/rte_memcpy.h
+++ b/lib/eal/x86/include/rte_memcpy.h
@@ -11,12 +11,19 @@
  * Functions for SSE/AVX/AVX2/AVX512 implementation of memcpy().
  */
 
+#include <rte_config.h>
+
+#ifdef RTE_USE_CC_MEMCPY
+
+#include <generic/rte_memcpy.h>
+
+#else
+
 #include <stdio.h>
 #include <stdint.h>
 #include <string.h>
 #include <rte_vect.h>
 #include <rte_common.h>
-#include <rte_config.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -878,4 +885,6 @@ rte_memcpy(void *dst, const void *src, size_t n)
 }
 #endif
 
+#endif /* RTE_USE_CC_MEMCPY */
+
 #endif /* _RTE_MEMCPY_X86_64_H_ */
diff --git a/meson_options.txt b/meson_options.txt
index e49b2fc089..06f544b631 100644
--- a/meson_options.txt
+++ b/meson_options.txt
@@ -60,3 +60,5 @@ option('tests', type: 'boolean', value: true, description:
        'build unit tests')
 option('use_hpet', type: 'boolean', value: false, description:
        'use HPET timer in EAL')
+option('use_cc_memcpy', type: 'boolean', value: true, description:
+       'Have the functions of <rte_memcpy.h> delegate to compiler/libc memcpy() instead of using custom implementation.')
-- 
2.34.1

RE: [PATCH v4 00/13] Optionally have rte_memcpy delegate to compiler memcpy

[Morten Brørup]

> From: Mattias Rönnblom [mailto:mattias.ronnblom@ericsson.com]
> 
> This patch set make DPDK library, driver, and application code use the
> compiler/libc memcpy() by default when functions in <rte_memcpy.h> are
> invoked.
> 
> The various custom DPDK rte_memcpy() implementations may be retained
> by means of a build-time option.
> 
> This patch set only make a difference on x86, PPC and ARM. Loongarch
> and RISCV already used compiler/libc memcpy().
> 
> This patch set includes a number of fixes in drivers and libraries
> which errornously relied on <rte_memcpy.h> including header files
> (i.e., <rte_vect.h>) required by its implementation.

Series-acked-by: Morten Brørup <mb@smartsharesystems.com>

Re: [PATCH v4 00/13] Optionally have rte_memcpy delegate to compiler memcpy

[Mattias Rönnblom]

It seems like patchwork didn't realize this v4 patch set supersedes v3.

I'm trying to figure out what I did wrong. Must you use the cover 
letter's message id in order for patchwork to recognize the history?

Re: [PATCH v4 00/13] Optionally have rte_memcpy delegate to compiler memcpy

[David Marchand]

On Fri, Jun 21, 2024 at 8:57 AM Mattias Rönnblom <hofors@lysator.liu.se> wrote:
>
> It seems like patchwork didn't realize this v4 patch set supersedes v3.

There is nothing automatic in patchwork.
Author is responsible for marking as superseded in patchwork.

I usually go around in patchwork in the morning and mark superseded
stuff on my own.
So please wait for the robot David to do the job... :-).

>
> I'm trying to figure out what I did wrong. Must you use the cover
> letter's message id in order for patchwork to recognize the history?
>


-- 
David Marchand

Re: [PATCH v4 00/13] Optionally have rte_memcpy delegate to compiler memcpy

[Mattias Rönnblom]

On 2024-06-21 09:04, David Marchand wrote:
> On Fri, Jun 21, 2024 at 8:57 AM Mattias Rönnblom <hofors@lysator.liu.se> wrote:
>>
>> It seems like patchwork didn't realize this v4 patch set supersedes v3.
> 
> There is nothing automatic in patchwork.

Oh, I see. :)

> Author is responsible for marking as superseded in patchwork.
> 
> I usually go around in patchwork in the morning and mark superseded
> stuff on my own.
> So please wait for the robot David to do the job... :-).
> 

Hmm. Author is responsible, but robot David does the actual work?

Do you prefer authors doing the work?

>>
>> I'm trying to figure out what I did wrong. Must you use the cover
>> letter's message id in order for patchwork to recognize the history?
>>
> 
> 

Re: [PATCH v4 00/13] Optionally have rte_memcpy delegate to compiler memcpy

[David Marchand]

On Fri, Jun 21, 2024 at 9:36 AM Mattias Rönnblom <hofors@lysator.liu.se> wrote:
>
> On 2024-06-21 09:04, David Marchand wrote:
> > On Fri, Jun 21, 2024 at 8:57 AM Mattias Rönnblom <hofors@lysator.liu.se> wrote:
> >>
> >> It seems like patchwork didn't realize this v4 patch set supersedes v3.
> >
> > There is nothing automatic in patchwork.
>
> Oh, I see. :)
>
> > Author is responsible for marking as superseded in patchwork.
> >
> > I usually go around in patchwork in the morning and mark superseded
> > stuff on my own.
> > So please wait for the robot David to do the job... :-).
> >
>
> Hmm. Author is responsible, but robot David does the actual work?

I am pretty sure other subtree maintainers do the same for their part
of patchwork.

>
> Do you prefer authors doing the work?

Unfortunately robot David can not read in people's mind (an AI would
be most likely better at it but I did not invest time in it).
And I also do some errors at this job because it involves clicking in
a webui (not often, but it does happen).

So yes, I prefer people do this part of the contribution job.
https://doc.dpdk.org/guides/contributing/patches.html#steps-to-getting-your-patch-merged


-- 
David Marchand

Re: [PATCH v4 13/13] eal: provide option to use compiler memcpy instead of RTE

[Stephen Hemminger]

On Thu, 20 Jun 2024 19:57:31 +0200
Mattias Rönnblom <mattias.ronnblom@ericsson.com> wrote:

> Provide build option to have functions in <rte_memcpy.h> delegate to
> the standard compiler/libc memcpy(), instead of using the various
> custom DPDK, handcrafted, per-architecture rte_memcpy()
> implementations.
> 
> A new meson build option 'use_cc_memcpy' is added. By default,
> the compiler/libc memcpy() is used.
> 
> The performance benefits of the custom DPDK rte_memcpy()
> implementations have been diminishing with every compiler release, and
> with current toolchains the use of a custom memcpy() implementation
> may even be a liability.
> 
> This patch leaves an option to stay on the custom DPDK implementations,
> would that prove beneficial for certain applications or architectures.
> 
> An additional benefit of this change is that compilers and static
> analysis tools have an easier time detecting incorrect usage of
> rte_memcpy() (e.g., buffer overruns, or overlapping source and
> destination buffers).
> 
> Signed-off-by: Mattias Rönnblom <mattias.ronnblom@ericsson.com>
> Acked-by: Morten Brørup <mb@smartsharesystems.com>

Would like to mark rte_memcpy as deprecated in a future release.

Acked-by: Stephen Hemminger <stephen@networkplumber.org>

Re: [PATCH v4 13/13] eal: provide option to use compiler memcpy instead of RTE

[Thomas Monjalon]

20/06/2024 19:57, Mattias Rönnblom:
> Provide build option to have functions in <rte_memcpy.h> delegate to
> the standard compiler/libc memcpy(), instead of using the various
> custom DPDK, handcrafted, per-architecture rte_memcpy()
> implementations.
> 
> A new meson build option 'use_cc_memcpy' is added. By default,
> the compiler/libc memcpy() is used.
> 
> The performance benefits of the custom DPDK rte_memcpy()
> implementations have been diminishing with every compiler release, and
> with current toolchains the use of a custom memcpy() implementation
> may even be a liability.
> 
> This patch leaves an option to stay on the custom DPDK implementations,
> would that prove beneficial for certain applications or architectures.
[...]
> --- a/meson_options.txt
> +++ b/meson_options.txt
> +option('use_cc_memcpy', type: 'boolean', value: true, description:
> +       'Have the functions of <rte_memcpy.h> delegate to compiler/libc memcpy() instead of using custom implementation.')

I suppose you propose this change for 24.11?
I would prefer getting the option disabled in 24.07,
so we can run tests during months before enabling it by default.
This period would also help to make sure it is compiling in all cases.
Please could you enable the option in our compilation scripts?

Re: [PATCH v4 13/13] eal: provide option to use compiler memcpy instead of RTE

[Mattias Rönnblom]

On Mon, Jun 24, 2024 at 12:05:53PM +0200, Thomas Monjalon wrote:
> 20/06/2024 19:57, Mattias Rönnblom:
> > Provide build option to have functions in <rte_memcpy.h> delegate to
> > the standard compiler/libc memcpy(), instead of using the various
> > custom DPDK, handcrafted, per-architecture rte_memcpy()
> > implementations.
> > 
> > A new meson build option 'use_cc_memcpy' is added. By default,
> > the compiler/libc memcpy() is used.
> > 
> > The performance benefits of the custom DPDK rte_memcpy()
> > implementations have been diminishing with every compiler release, and
> > with current toolchains the use of a custom memcpy() implementation
> > may even be a liability.
> > 
> > This patch leaves an option to stay on the custom DPDK implementations,
> > would that prove beneficial for certain applications or architectures.
> [...]
> > --- a/meson_options.txt
> > +++ b/meson_options.txt
> > +option('use_cc_memcpy', type: 'boolean', value: true, description:
> > +       'Have the functions of <rte_memcpy.h> delegate to compiler/libc memcpy() instead of using custom implementation.')
> 
> I suppose you propose this change for 24.11?
> I would prefer getting the option disabled in 24.07,
> so we can run tests during months before enabling it by default.

I think I would suggest having it included and enabled by default in
24.07, but that is too risky perhaps.

Having it included in 24.07 and disabled by default would be the
conservative option.

> This period would also help to make sure it is compiling in all cases.
> Please could you enable the option in our compilation scripts?
> 
>

I'll give it a try.

Re: [PATCH v4 13/13] eal: provide option to use compiler memcpy instead of RTE

[Mattias Rönnblom]

On Mon, Jun 24, 2024 at 12:05:53PM +0200, Thomas Monjalon wrote:

<snip>

> Please could you enable the option in our compilation scripts?
>

"Compilation scripts" is test-meson-builds.sh?
Should it be done in the same as is being done for stdatomic?
Wouldn't it be enough if this is done in CI only?

Re: [PATCH v4 13/13] eal: provide option to use compiler memcpy instead of RTE

[Thomas Monjalon]

25/06/2024 15:06, Mattias Rönnblom:
> On Mon, Jun 24, 2024 at 12:05:53PM +0200, Thomas Monjalon wrote:
> 
> <snip>
> 
> > Please could you enable the option in our compilation scripts?
> >
> 
> "Compilation scripts" is test-meson-builds.sh?
> Should it be done in the same as is being done for stdatomic?
> Wouldn't it be enough if this is done in CI only?

Should be the same as for stdatomic.
For stdatomic, there are changes in these 3 files:
	- devtools/test-meson-builds.sh
	- .ci/linux-build.sh
	- .github/workflows/build.yml

Re: [PATCH v4 00/13] Optionally have rte_memcpy delegate to compiler memcpy

[Maxime Coquelin]

Hi Mattias,

On 6/20/24 19:57, Mattias Rönnblom wrote:
> This patch set make DPDK library, driver, and application code use the
> compiler/libc memcpy() by default when functions in <rte_memcpy.h> are
> invoked.
> 
> The various custom DPDK rte_memcpy() implementations may be retained
> by means of a build-time option.
> 
> This patch set only make a difference on x86, PPC and ARM. Loongarch
> and RISCV already used compiler/libc memcpy().

It indeed makes a difference on x86!

Just tested latest main with and without your series on
Intel(R) Xeon(R) Gold 6438N.

The test is a simple IO loop between a Vhost PMD and a Virtio-user PMD:
# dpdk-testpmd -l 4-6   --file-prefix=virtio1 --no-pci --vdev 
'net_virtio_user0,mac=00:01:02:03:04:05,path=./vhost-net,server=1,mrg_rxbuf=1,in_order=1' 
   --single-file-segments -- -i
testpmd> start

# dpdk-testpmd -l 8-10   --file-prefix=vhost1 --no-pci --vdev 
'net_vhost0,iface=vhost-net,client=1'   --single-file-segments -- -i
testpmd> start tx_first 32

Latest main: 14.5Mpps
Latest main + this series: 10Mpps

So for me, it should be disabled by default.

Regards,
Maxime

> This patch set includes a number of fixes in drivers and libraries
> which errornously relied on <rte_memcpy.h> including header files
> (i.e., <rte_vect.h>) required by its implementation.
> 
> Mattias Rönnblom (13):
>    net/i40e: add missing vector API header include
>    net/iavf: add missing vector API header include
>    net/ice: add missing vector API header include
>    net/ixgbe: add missing vector API header include
>    net/ngbe: add missing vector API header include
>    net/txgbe: add missing vector API header include
>    net/virtio: add missing vector API header include
>    net/fm10k: add missing vector API header include
>    event/dlb2: include headers for vector and memory copy APIs
>    net/octeon_ep: add missing vector API header include
>    distributor: add missing vector API header include
>    fib: add missing vector API header include
>    eal: provide option to use compiler memcpy instead of RTE
> 
>   config/meson.build                          |  1 +
>   doc/guides/rel_notes/release_24_07.rst      | 21 +++++++
>   drivers/event/dlb2/dlb2.c                   |  2 +
>   drivers/net/fm10k/fm10k_rxtx_vec.c          |  3 +-
>   drivers/net/i40e/i40e_rxtx_vec_sse.c        |  3 +-
>   drivers/net/iavf/iavf_rxtx_vec_sse.c        |  3 +-
>   drivers/net/ice/ice_rxtx_vec_sse.c          |  2 +-
>   drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c      |  3 +-
>   drivers/net/ngbe/ngbe_rxtx_vec_sse.c        |  3 +-
>   drivers/net/octeon_ep/otx_ep_ethdev.c       |  2 +
>   drivers/net/txgbe/txgbe_rxtx_vec_sse.c      |  3 +-
>   drivers/net/virtio/virtio_rxtx_simple_sse.c |  3 +-
>   lib/distributor/rte_distributor.c           |  1 +
>   lib/eal/arm/include/rte_memcpy.h            | 10 ++++
>   lib/eal/include/generic/rte_memcpy.h        | 61 ++++++++++++++++++---
>   lib/eal/loongarch/include/rte_memcpy.h      | 53 ++----------------
>   lib/eal/ppc/include/rte_memcpy.h            | 10 ++++
>   lib/eal/riscv/include/rte_memcpy.h          | 53 ++----------------
>   lib/eal/x86/include/meson.build             |  1 +
>   lib/eal/x86/include/rte_memcpy.h            | 11 +++-
>   lib/fib/trie.c                              |  1 +
>   meson_options.txt                           |  2 +
>   22 files changed, 131 insertions(+), 121 deletions(-)
> 

Re: [PATCH v4 00/13] Optionally have rte_memcpy delegate to compiler memcpy

[Stephen Hemminger]

On Tue, 25 Jun 2024 17:29:35 +0200
Maxime Coquelin <maxime.coquelin@redhat.com> wrote:

> Hi Mattias,
> 
> On 6/20/24 19:57, Mattias Rönnblom wrote:
> > This patch set make DPDK library, driver, and application code use the
> > compiler/libc memcpy() by default when functions in <rte_memcpy.h> are
> > invoked.
> > 
> > The various custom DPDK rte_memcpy() implementations may be retained
> > by means of a build-time option.
> > 
> > This patch set only make a difference on x86, PPC and ARM. Loongarch
> > and RISCV already used compiler/libc memcpy().  
> 
> It indeed makes a difference on x86!
> 
> Just tested latest main with and without your series on
> Intel(R) Xeon(R) Gold 6438N.
> 
> The test is a simple IO loop between a Vhost PMD and a Virtio-user PMD:
> # dpdk-testpmd -l 4-6   --file-prefix=virtio1 --no-pci --vdev 
> 'net_virtio_user0,mac=00:01:02:03:04:05,path=./vhost-net,server=1,mrg_rxbuf=1,in_order=1' 
>    --single-file-segments -- -i
> testpmd> start  
> 
> # dpdk-testpmd -l 8-10   --file-prefix=vhost1 --no-pci --vdev 
> 'net_vhost0,iface=vhost-net,client=1'   --single-file-segments -- -i
> testpmd> start tx_first 32  
> 
> Latest main: 14.5Mpps
> Latest main + this series: 10Mpps
> 
> So for me, it should be disabled by default.
> 
> Regards,
> Maxime

What is the size of the copy being done?
Which compiler version?

Re: [PATCH v4 00/13] Optionally have rte_memcpy delegate to compiler memcpy

[Mattias Rönnblom]

On Tue, Jun 25, 2024 at 05:29:35PM +0200, Maxime Coquelin wrote:
> Hi Mattias,
> 
> On 6/20/24 19:57, Mattias Rönnblom wrote:
> > This patch set make DPDK library, driver, and application code use the
> > compiler/libc memcpy() by default when functions in <rte_memcpy.h> are
> > invoked.
> > 
> > The various custom DPDK rte_memcpy() implementations may be retained
> > by means of a build-time option.
> > 
> > This patch set only make a difference on x86, PPC and ARM. Loongarch
> > and RISCV already used compiler/libc memcpy().
> 
> It indeed makes a difference on x86!
>
> Just tested latest main with and without your series on
> Intel(R) Xeon(R) Gold 6438N.
> 
> The test is a simple IO loop between a Vhost PMD and a Virtio-user PMD:
> # dpdk-testpmd -l 4-6   --file-prefix=virtio1 --no-pci --vdev 'net_virtio_user0,mac=00:01:02:03:04:05,path=./vhost-net,server=1,mrg_rxbuf=1,in_order=1'
> --single-file-segments -- -i
> testpmd> start
> 
> # dpdk-testpmd -l 8-10   --file-prefix=vhost1 --no-pci --vdev
> 'net_vhost0,iface=vhost-net,client=1'   --single-file-segments -- -i
> testpmd> start tx_first 32
> 
> Latest main: 14.5Mpps
> Latest main + this series: 10Mpps
>

I ran the above benchmark on my Raptor Lake desktop (locked to 3,2
GHz). GCC 12.3.0.

Core use_cc_memcpy Mpps
E    false         9.5
E    true          9.7
P    false         16.4
P    true          13.5

On the P-cores, there's a significant performance regression, although
not as bad as the one you see on your Sapphire Rapids Xeon. On the
E-cores, there's actually a slight performance gain.

The virtio PMD does not directly invoke rte_memcpy() or anything else
from <rte_memcpy.h>, but rather use memcpy(), so I'm not sure I
understand what's going on here. Does the virtio driver delegate some
performance-critical task to some module that in turns uses
rte_memcpy()?

> So for me, it should be disabled by default.
> 
> Regards,
> Maxime
> 
> > This patch set includes a number of fixes in drivers and libraries
> > which errornously relied on <rte_memcpy.h> including header files
> > (i.e., <rte_vect.h>) required by its implementation.
> > 
> > Mattias Rönnblom (13):
> >    net/i40e: add missing vector API header include
> >    net/iavf: add missing vector API header include
> >    net/ice: add missing vector API header include
> >    net/ixgbe: add missing vector API header include
> >    net/ngbe: add missing vector API header include
> >    net/txgbe: add missing vector API header include
> >    net/virtio: add missing vector API header include
> >    net/fm10k: add missing vector API header include
> >    event/dlb2: include headers for vector and memory copy APIs
> >    net/octeon_ep: add missing vector API header include
> >    distributor: add missing vector API header include
> >    fib: add missing vector API header include
> >    eal: provide option to use compiler memcpy instead of RTE
> > 
> >   config/meson.build                          |  1 +
> >   doc/guides/rel_notes/release_24_07.rst      | 21 +++++++
> >   drivers/event/dlb2/dlb2.c                   |  2 +
> >   drivers/net/fm10k/fm10k_rxtx_vec.c          |  3 +-
> >   drivers/net/i40e/i40e_rxtx_vec_sse.c        |  3 +-
> >   drivers/net/iavf/iavf_rxtx_vec_sse.c        |  3 +-
> >   drivers/net/ice/ice_rxtx_vec_sse.c          |  2 +-
> >   drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c      |  3 +-
> >   drivers/net/ngbe/ngbe_rxtx_vec_sse.c        |  3 +-
> >   drivers/net/octeon_ep/otx_ep_ethdev.c       |  2 +
> >   drivers/net/txgbe/txgbe_rxtx_vec_sse.c      |  3 +-
> >   drivers/net/virtio/virtio_rxtx_simple_sse.c |  3 +-
> >   lib/distributor/rte_distributor.c           |  1 +
> >   lib/eal/arm/include/rte_memcpy.h            | 10 ++++
> >   lib/eal/include/generic/rte_memcpy.h        | 61 ++++++++++++++++++---
> >   lib/eal/loongarch/include/rte_memcpy.h      | 53 ++----------------
> >   lib/eal/ppc/include/rte_memcpy.h            | 10 ++++
> >   lib/eal/riscv/include/rte_memcpy.h          | 53 ++----------------
> >   lib/eal/x86/include/meson.build             |  1 +
> >   lib/eal/x86/include/rte_memcpy.h            | 11 +++-
> >   lib/fib/trie.c                              |  1 +
> >   meson_options.txt                           |  2 +
> >   22 files changed, 131 insertions(+), 121 deletions(-)
> > 
> 

Re: [PATCH v4 00/13] Optionally have rte_memcpy delegate to compiler memcpy

[Maxime Coquelin]

On 6/25/24 21:27, Mattias Rönnblom wrote:
> On Tue, Jun 25, 2024 at 05:29:35PM +0200, Maxime Coquelin wrote:
>> Hi Mattias,
>>
>> On 6/20/24 19:57, Mattias Rönnblom wrote:
>>> This patch set make DPDK library, driver, and application code use the
>>> compiler/libc memcpy() by default when functions in <rte_memcpy.h> are
>>> invoked.
>>>
>>> The various custom DPDK rte_memcpy() implementations may be retained
>>> by means of a build-time option.
>>>
>>> This patch set only make a difference on x86, PPC and ARM. Loongarch
>>> and RISCV already used compiler/libc memcpy().
>>
>> It indeed makes a difference on x86!
>>
>> Just tested latest main with and without your series on
>> Intel(R) Xeon(R) Gold 6438N.
>>
>> The test is a simple IO loop between a Vhost PMD and a Virtio-user PMD:
>> # dpdk-testpmd -l 4-6   --file-prefix=virtio1 --no-pci --vdev 'net_virtio_user0,mac=00:01:02:03:04:05,path=./vhost-net,server=1,mrg_rxbuf=1,in_order=1'
>> --single-file-segments -- -i
>> testpmd> start
>>
>> # dpdk-testpmd -l 8-10   --file-prefix=vhost1 --no-pci --vdev
>> 'net_vhost0,iface=vhost-net,client=1'   --single-file-segments -- -i
>> testpmd> start tx_first 32
>>
>> Latest main: 14.5Mpps
>> Latest main + this series: 10Mpps
>>
> 
> I ran the above benchmark on my Raptor Lake desktop (locked to 3,2
> GHz). GCC 12.3.0.
> 
> Core use_cc_memcpy Mpps
> E    false         9.5
> E    true          9.7
> P    false         16.4
> P    true          13.5
> 
> On the P-cores, there's a significant performance regression, although
> not as bad as the one you see on your Sapphire Rapids Xeon. On the
> E-cores, there's actually a slight performance gain.
> 
> The virtio PMD does not directly invoke rte_memcpy() or anything else
> from <rte_memcpy.h>, but rather use memcpy(), so I'm not sure I
> understand what's going on here. Does the virtio driver delegate some
> performance-critical task to some module that in turns uses
> rte_memcpy()?

This is because Vhost is the bottleneck here, not Virtio driver.
Indeed, the virtqueues memory belongs to the Virtio driver and the
descriptors buffers are Virtio's mbufs, so not much memcpy's are done
there.

Vhost however, is a heavy memcpy user, as all the descriptors buffers 
are copied to/from its mbufs.

>> So for me, it should be disabled by default.
>>
>> Regards,
>> Maxime
>>
>>> This patch set includes a number of fixes in drivers and libraries
>>> which errornously relied on <rte_memcpy.h> including header files
>>> (i.e., <rte_vect.h>) required by its implementation.
>>>
>>> Mattias Rönnblom (13):
>>>     net/i40e: add missing vector API header include
>>>     net/iavf: add missing vector API header include
>>>     net/ice: add missing vector API header include
>>>     net/ixgbe: add missing vector API header include
>>>     net/ngbe: add missing vector API header include
>>>     net/txgbe: add missing vector API header include
>>>     net/virtio: add missing vector API header include
>>>     net/fm10k: add missing vector API header include
>>>     event/dlb2: include headers for vector and memory copy APIs
>>>     net/octeon_ep: add missing vector API header include
>>>     distributor: add missing vector API header include
>>>     fib: add missing vector API header include
>>>     eal: provide option to use compiler memcpy instead of RTE
>>>
>>>    config/meson.build                          |  1 +
>>>    doc/guides/rel_notes/release_24_07.rst      | 21 +++++++
>>>    drivers/event/dlb2/dlb2.c                   |  2 +
>>>    drivers/net/fm10k/fm10k_rxtx_vec.c          |  3 +-
>>>    drivers/net/i40e/i40e_rxtx_vec_sse.c        |  3 +-
>>>    drivers/net/iavf/iavf_rxtx_vec_sse.c        |  3 +-
>>>    drivers/net/ice/ice_rxtx_vec_sse.c          |  2 +-
>>>    drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c      |  3 +-
>>>    drivers/net/ngbe/ngbe_rxtx_vec_sse.c        |  3 +-
>>>    drivers/net/octeon_ep/otx_ep_ethdev.c       |  2 +
>>>    drivers/net/txgbe/txgbe_rxtx_vec_sse.c      |  3 +-
>>>    drivers/net/virtio/virtio_rxtx_simple_sse.c |  3 +-
>>>    lib/distributor/rte_distributor.c           |  1 +
>>>    lib/eal/arm/include/rte_memcpy.h            | 10 ++++
>>>    lib/eal/include/generic/rte_memcpy.h        | 61 ++++++++++++++++++---
>>>    lib/eal/loongarch/include/rte_memcpy.h      | 53 ++----------------
>>>    lib/eal/ppc/include/rte_memcpy.h            | 10 ++++
>>>    lib/eal/riscv/include/rte_memcpy.h          | 53 ++----------------
>>>    lib/eal/x86/include/meson.build             |  1 +
>>>    lib/eal/x86/include/rte_memcpy.h            | 11 +++-
>>>    lib/fib/trie.c                              |  1 +
>>>    meson_options.txt                           |  2 +
>>>    22 files changed, 131 insertions(+), 121 deletions(-)
>>>
>>
> 

Re: [PATCH v4 00/13] Optionally have rte_memcpy delegate to compiler memcpy

[Stephen Hemminger]

On Wed, 26 Jun 2024 10:37:31 +0200
Maxime Coquelin <maxime.coquelin@redhat.com> wrote:

> On 6/25/24 21:27, Mattias Rönnblom wrote:
> > On Tue, Jun 25, 2024 at 05:29:35PM +0200, Maxime Coquelin wrote:  
> >> Hi Mattias,
> >>
> >> On 6/20/24 19:57, Mattias Rönnblom wrote:  
> >>> This patch set make DPDK library, driver, and application code use the
> >>> compiler/libc memcpy() by default when functions in <rte_memcpy.h> are
> >>> invoked.
> >>>
> >>> The various custom DPDK rte_memcpy() implementations may be retained
> >>> by means of a build-time option.
> >>>
> >>> This patch set only make a difference on x86, PPC and ARM. Loongarch
> >>> and RISCV already used compiler/libc memcpy().  
> >>
> >> It indeed makes a difference on x86!
> >>
> >> Just tested latest main with and without your series on
> >> Intel(R) Xeon(R) Gold 6438N.
> >>
> >> The test is a simple IO loop between a Vhost PMD and a Virtio-user PMD:
> >> # dpdk-testpmd -l 4-6   --file-prefix=virtio1 --no-pci --vdev 'net_virtio_user0,mac=00:01:02:03:04:05,path=./vhost-net,server=1,mrg_rxbuf=1,in_order=1'
> >> --single-file-segments -- -i  
> >> testpmd> start  
> >>
> >> # dpdk-testpmd -l 8-10   --file-prefix=vhost1 --no-pci --vdev
> >> 'net_vhost0,iface=vhost-net,client=1'   --single-file-segments -- -i  
> >> testpmd> start tx_first 32  
> >>
> >> Latest main: 14.5Mpps
> >> Latest main + this series: 10Mpps
> >>  
> > 
> > I ran the above benchmark on my Raptor Lake desktop (locked to 3,2
> > GHz). GCC 12.3.0.
> > 
> > Core use_cc_memcpy Mpps
> > E    false         9.5
> > E    true          9.7
> > P    false         16.4
> > P    true          13.5
> > 
> > On the P-cores, there's a significant performance regression, although
> > not as bad as the one you see on your Sapphire Rapids Xeon. On the
> > E-cores, there's actually a slight performance gain.
> > 
> > The virtio PMD does not directly invoke rte_memcpy() or anything else
> > from <rte_memcpy.h>, but rather use memcpy(), so I'm not sure I
> > understand what's going on here. Does the virtio driver delegate some
> > performance-critical task to some module that in turns uses
> > rte_memcpy()?  
> 
> This is because Vhost is the bottleneck here, not Virtio driver.
> Indeed, the virtqueues memory belongs to the Virtio driver and the
> descriptors buffers are Virtio's mbufs, so not much memcpy's are done
> there.
> 
> Vhost however, is a heavy memcpy user, as all the descriptors buffers 
> are copied to/from its mbufs.

Would be good to now the size (if small it is inlining that matters, or
maybe alignment matters), and have test results for multiple compiler versions.
Ideally, feed results back and update Gcc and Clang.

DPDK doesn't need to be in the optimize C library space.

Re: [PATCH v4 00/13] Optionally have rte_memcpy delegate to compiler memcpy

[Maxime Coquelin]

On 6/26/24 16:58, Stephen Hemminger wrote:
> On Wed, 26 Jun 2024 10:37:31 +0200
> Maxime Coquelin <maxime.coquelin@redhat.com> wrote:
> 
>> On 6/25/24 21:27, Mattias Rönnblom wrote:
>>> On Tue, Jun 25, 2024 at 05:29:35PM +0200, Maxime Coquelin wrote:
>>>> Hi Mattias,
>>>>
>>>> On 6/20/24 19:57, Mattias Rönnblom wrote:
>>>>> This patch set make DPDK library, driver, and application code use the
>>>>> compiler/libc memcpy() by default when functions in <rte_memcpy.h> are
>>>>> invoked.
>>>>>
>>>>> The various custom DPDK rte_memcpy() implementations may be retained
>>>>> by means of a build-time option.
>>>>>
>>>>> This patch set only make a difference on x86, PPC and ARM. Loongarch
>>>>> and RISCV already used compiler/libc memcpy().
>>>>
>>>> It indeed makes a difference on x86!
>>>>
>>>> Just tested latest main with and without your series on
>>>> Intel(R) Xeon(R) Gold 6438N.
>>>>
>>>> The test is a simple IO loop between a Vhost PMD and a Virtio-user PMD:
>>>> # dpdk-testpmd -l 4-6   --file-prefix=virtio1 --no-pci --vdev 'net_virtio_user0,mac=00:01:02:03:04:05,path=./vhost-net,server=1,mrg_rxbuf=1,in_order=1'
>>>> --single-file-segments -- -i
>>>> testpmd> start
>>>>
>>>> # dpdk-testpmd -l 8-10   --file-prefix=vhost1 --no-pci --vdev
>>>> 'net_vhost0,iface=vhost-net,client=1'   --single-file-segments -- -i
>>>> testpmd> start tx_first 32
>>>>
>>>> Latest main: 14.5Mpps
>>>> Latest main + this series: 10Mpps
>>>>   
>>>
>>> I ran the above benchmark on my Raptor Lake desktop (locked to 3,2
>>> GHz). GCC 12.3.0.
>>>
>>> Core use_cc_memcpy Mpps
>>> E    false         9.5
>>> E    true          9.7
>>> P    false         16.4
>>> P    true          13.5
>>>
>>> On the P-cores, there's a significant performance regression, although
>>> not as bad as the one you see on your Sapphire Rapids Xeon. On the
>>> E-cores, there's actually a slight performance gain.
>>>
>>> The virtio PMD does not directly invoke rte_memcpy() or anything else
>>> from <rte_memcpy.h>, but rather use memcpy(), so I'm not sure I
>>> understand what's going on here. Does the virtio driver delegate some
>>> performance-critical task to some module that in turns uses
>>> rte_memcpy()?
>>
>> This is because Vhost is the bottleneck here, not Virtio driver.
>> Indeed, the virtqueues memory belongs to the Virtio driver and the
>> descriptors buffers are Virtio's mbufs, so not much memcpy's are done
>> there.
>>
>> Vhost however, is a heavy memcpy user, as all the descriptors buffers
>> are copied to/from its mbufs.
> 
> Would be good to now the size (if small it is inlining that matters, or
> maybe alignment matters), and have test results for multiple compiler versions.
> Ideally, feed results back and update Gcc and Clang.

I was testing with GCC 11 on RHEL-9:
gcc (GCC) 11.4.1 20231218 (Red Hat 11.4.1-3)

I was using the default one, 64B packets.

I don't have time to perform these tests, but if you are willing to do
it I'll be happy to review the results.

> DPDK doesn't need to be in the optimize C library space.

Certainly, but we already have an optimized version currently, so not
much to do now on our side. When C libraries implementations will be on
par, we should definitely use them by default.

Maxime

Re: [PATCH v4 00/13] Optionally have rte_memcpy delegate to compiler memcpy

[Mattias Rönnblom]

On Wed, Jun 26, 2024 at 05:24:04PM +0200, Maxime Coquelin wrote:
> 
> 
> On 6/26/24 16:58, Stephen Hemminger wrote:
> > On Wed, 26 Jun 2024 10:37:31 +0200
> > Maxime Coquelin <maxime.coquelin@redhat.com> wrote:
> > 
> > > On 6/25/24 21:27, Mattias Rönnblom wrote:
> > > > On Tue, Jun 25, 2024 at 05:29:35PM +0200, Maxime Coquelin wrote:
> > > > > Hi Mattias,
> > > > > 
> > > > > On 6/20/24 19:57, Mattias Rönnblom wrote:
> > > > > > This patch set make DPDK library, driver, and application code use the
> > > > > > compiler/libc memcpy() by default when functions in <rte_memcpy.h> are
> > > > > > invoked.
> > > > > > 
> > > > > > The various custom DPDK rte_memcpy() implementations may be retained
> > > > > > by means of a build-time option.
> > > > > > 
> > > > > > This patch set only make a difference on x86, PPC and ARM. Loongarch
> > > > > > and RISCV already used compiler/libc memcpy().
> > > > > 
> > > > > It indeed makes a difference on x86!
> > > > > 
> > > > > Just tested latest main with and without your series on
> > > > > Intel(R) Xeon(R) Gold 6438N.
> > > > > 
> > > > > The test is a simple IO loop between a Vhost PMD and a Virtio-user PMD:
> > > > > # dpdk-testpmd -l 4-6   --file-prefix=virtio1 --no-pci --vdev 'net_virtio_user0,mac=00:01:02:03:04:05,path=./vhost-net,server=1,mrg_rxbuf=1,in_order=1'
> > > > > --single-file-segments -- -i
> > > > > testpmd> start
> > > > > 
> > > > > # dpdk-testpmd -l 8-10   --file-prefix=vhost1 --no-pci --vdev
> > > > > 'net_vhost0,iface=vhost-net,client=1'   --single-file-segments -- -i
> > > > > testpmd> start tx_first 32
> > > > > 
> > > > > Latest main: 14.5Mpps
> > > > > Latest main + this series: 10Mpps
> > > > 
> > > > I ran the above benchmark on my Raptor Lake desktop (locked to 3,2
> > > > GHz). GCC 12.3.0.
> > > > 
> > > > Core use_cc_memcpy Mpps
> > > > E    false         9.5
> > > > E    true          9.7
> > > > P    false         16.4
> > > > P    true          13.5
> > > > 
> > > > On the P-cores, there's a significant performance regression, although
> > > > not as bad as the one you see on your Sapphire Rapids Xeon. On the
> > > > E-cores, there's actually a slight performance gain.
> > > > 
> > > > The virtio PMD does not directly invoke rte_memcpy() or anything else
> > > > from <rte_memcpy.h>, but rather use memcpy(), so I'm not sure I
> > > > understand what's going on here. Does the virtio driver delegate some
> > > > performance-critical task to some module that in turns uses
> > > > rte_memcpy()?
> > > 
> > > This is because Vhost is the bottleneck here, not Virtio driver.
> > > Indeed, the virtqueues memory belongs to the Virtio driver and the
> > > descriptors buffers are Virtio's mbufs, so not much memcpy's are done
> > > there.
> > > 
> > > Vhost however, is a heavy memcpy user, as all the descriptors buffers
> > > are copied to/from its mbufs.
> > 
> > Would be good to now the size (if small it is inlining that matters, or
> > maybe alignment matters), and have test results for multiple compiler versions.
> > Ideally, feed results back and update Gcc and Clang.
> 
> I was testing with GCC 11 on RHEL-9:
> gcc (GCC) 11.4.1 20231218 (Red Hat 11.4.1-3)
> 
> I was using the default one, 64B packets.
> 
> I don't have time to perform these tests, but if you are willing to do
> it I'll be happy to review the results.
> 
> > DPDK doesn't need to be in the optimize C library space.
> 
> Certainly, but we already have an optimized version currently, so not
> much to do now on our side. When C libraries implementations will be on
> par, we should definitely use them by default.
>

I think it's not so much about optimized versus non-optimized at this
point. It's just that cc/libc memcpy sometimes performs better than
RTE memcpy, and sometimes doesn't.

For virtio, a single memory copy in
lib/vhost/virtio_net.c:do_data_copy_enqueue()
is responsible for >95% of the performance regression introduced by
the cc memcpy patch for small packets on Intel P-cores.

I'm not so sure this performance regression will go away in newer
compilers. PGO would certainly help, but PGO is a hassle.

One way to fix this issue would be to introduce a custom,
memcpy()-based packet copying routine. I tried the below patch, with
the following results:

Raptor Lake @ 3,2 GHz
GCC 12

64 bytes packets
Core  Mode              Mpps
----------------------------
E     RTE memcpy        9.5
E     cc memcpy         9.7
E     cc memcpy+pktcpy  9.0

P     RTE memcpy        16.4
P     cc memcpy         13.5
P     cc memcpy+pktcpy  16.2

1500 bytes
Core  Mode              Mpps
----------------------------
P    RTE memcpy         5.8
P    cc memcpy          5.9
P    cc memcpy+pktcpy   5.9

As you can see, most of the regression is eliminated, at the cost of
worse E-core performance. I didn't look at the generated code, but one
could suspect heavy use of wide SIMD is to blame, which E-cores don't
necessarily benefit from.

The below prototype assumes the source and destination buffers are
16-byte aligned. Does that always hold?

I'm sure one could further improve performance using context-specific
information, such as packets always being >= 64 bytes. One could also
consider having special cases, maybe for 64 bytes and MTU-sized
packets. Such are always a hassle when you try to characterize
performance though.

diff --git a/lib/vhost/virtio_net.c b/lib/vhost/virtio_net.c
index 370402d849..7b595a6622 100644
--- a/lib/vhost/virtio_net.c
+++ b/lib/vhost/virtio_net.c
@@ -231,6 +231,26 @@ vhost_async_dma_check_completed(struct virtio_net *dev, int16_t dma_id, uint16_t
 	return nr_copies;
 }
 
+static inline void
+pktcpy(void *restrict in_dst, const void *restrict in_src, size_t len)
+{
+	void *dst = __builtin_assume_aligned(in_dst, 16);
+	const void *src = __builtin_assume_aligned(in_src, 16);
+
+	if (len <= 256) {
+		size_t left;
+
+		for (left = len; left >= 32; left -= 32) {
+			memcpy(dst, src, 32);
+			dst = RTE_PTR_ADD(dst, 32);
+			src = RTE_PTR_ADD(src, 32);
+		}
+
+		memcpy(dst, src, left);
+	} else
+		memcpy(dst, src, len);
+}
+
 static inline void
 do_data_copy_enqueue(struct virtio_net *dev, struct vhost_virtqueue *vq)
 	__rte_shared_locks_required(&vq->iotlb_lock)
@@ -240,7 +260,7 @@ do_data_copy_enqueue(struct virtio_net *dev, struct vhost_virtqueue *vq)
 	int i;
 
 	for (i = 0; i < count; i++) {
-		rte_memcpy(elem[i].dst, elem[i].src, elem[i].len);
+		pktcpy(elem[i].dst, elem[i].src, elem[i].len);
 		vhost_log_cache_write_iova(dev, vq, elem[i].log_addr,
 					   elem[i].len);
 		PRINT_PACKET(dev, (uintptr_t)elem[i].dst, elem[i].len, 0);
@@ -257,7 +277,7 @@ do_data_copy_dequeue(struct vhost_virtqueue *vq)
 	int i;
 
 	for (i = 0; i < count; i++)
-		rte_memcpy(elem[i].dst, elem[i].src, elem[i].len);
+		pktcpy(elem[i].dst, elem[i].src, elem[i].len);
 
 	vq->batch_copy_nb_elems = 0;
 }



> Maxime
>