DPDK patches and discussions
 help / color / mirror / Atom feed
* [dpdk-dev] [PATCH v3 1/2] net/mlx5: optimize mprq memcpy
@ 2020-10-10  8:57 Aman Kumar
  2020-10-10  8:57 ` [dpdk-dev] [PATCH v3 2/2] net/mlx5: add non temporal store for WQE fields Aman Kumar
  0 siblings, 1 reply; 5+ messages in thread
From: Aman Kumar @ 2020-10-10  8:57 UTC (permalink / raw)
  To: dev
  Cc: rasland, keesang.song, asafp, shys, viacheslavo, akozyrev, matan,
	aman.kumar

add non temporal load and temporal store for mprq memcpy.
define RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY in build
configuration to enable this optimization.

Signed-off-by: Aman Kumar <aman.kumar@vvdntech.in>
---
 drivers/net/mlx5/meson.build |   1 +
 drivers/net/mlx5/mlx5.c      |  12 ++++
 drivers/net/mlx5/mlx5.h      |   3 +
 drivers/net/mlx5/mlx5_rxq.c  |   3 +
 drivers/net/mlx5/mlx5_rxtx.c | 116 ++++++++++++++++++++++++++++++++++-
 drivers/net/mlx5/mlx5_rxtx.h |   3 +
 meson_options.txt            |   2 +
 7 files changed, 138 insertions(+), 2 deletions(-)

diff --git a/drivers/net/mlx5/meson.build b/drivers/net/mlx5/meson.build
index 9a97bb9c8..38e93fdc1 100644
--- a/drivers/net/mlx5/meson.build
+++ b/drivers/net/mlx5/meson.build
@@ -47,6 +47,7 @@ foreach option:cflags_options
 		cflags += option
 	endif
 endforeach
+dpdk_conf.set('RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY', get_option('mlx5_ntload_tstore'))
 if get_option('buildtype').contains('debug')
 	cflags += [ '-pedantic', '-DPEDANTIC' ]
 else
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 01ead6e6a..a2796eaa5 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -160,6 +160,11 @@
 /* Configure timeout of LRO session (in microseconds). */
 #define MLX5_LRO_TIMEOUT_USEC "lro_timeout_usec"
 
+#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
+/* mprq_tstore_memcpy */
+#define MLX5_MPRQ_TSTORE_MEMCPY "mprq_tstore_memcpy"
+#endif
+
 /*
  * Device parameter to configure the total data buffer size for a single
  * hairpin queue (logarithm value).
@@ -1623,6 +1628,10 @@ mlx5_args_check(const char *key, const char *val, void *opaque)
 		config->sys_mem_en = !!tmp;
 	} else if (strcmp(MLX5_DECAP_EN, key) == 0) {
 		config->decap_en = !!tmp;
+#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
+	} else if (strcmp(MLX5_MPRQ_TSTORE_MEMCPY, key) == 0) {
+		config->mprq_tstore_memcpy = tmp;
+#endif
 	} else {
 		DRV_LOG(WARNING, "%s: unknown parameter", key);
 		rte_errno = EINVAL;
@@ -1683,6 +1692,9 @@ mlx5_args(struct mlx5_dev_config *config, struct rte_devargs *devargs)
 		MLX5_RECLAIM_MEM,
 		MLX5_SYS_MEM_EN,
 		MLX5_DECAP_EN,
+#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
+		MLX5_MPRQ_TSTORE_MEMCPY,
+#endif
 		NULL,
 	};
 	struct rte_kvargs *kvlist;
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 43da9a1fb..1eb305650 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -234,6 +234,9 @@ struct mlx5_dev_config {
 	int tx_skew; /* Tx scheduling skew between WQE and data on wire. */
 	struct mlx5_hca_attr hca_attr; /* HCA attributes. */
 	struct mlx5_lro_config lro; /* LRO configuration. */
+#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
+	unsigned int mprq_tstore_memcpy:1;
+#endif
 };
 
 
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index c059e216d..c8db59a12 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -1380,6 +1380,9 @@ mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 	tmpl->socket = socket;
 	if (dev->data->dev_conf.intr_conf.rxq)
 		tmpl->irq = 1;
+#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
+	tmpl->rxq.mprq_tstore_memcpy = config->mprq_tstore_memcpy;
+#endif
 	mprq_stride_nums = config->mprq.stride_num_n ?
 		config->mprq.stride_num_n : MLX5_MPRQ_STRIDE_NUM_N;
 	mprq_stride_size = non_scatter_min_mbuf_size <=
diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index 0b87be15b..f59e30d82 100644
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -123,6 +123,97 @@ uint8_t mlx5_swp_types_table[1 << 10] __rte_cache_aligned;
 uint64_t rte_net_mlx5_dynf_inline_mask;
 #define PKT_TX_DYNF_NOINLINE rte_net_mlx5_dynf_inline_mask
 
+#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
+static void copy16B_ts(void *dst, void *src)
+{
+	__m128i var128;
+
+	var128 = _mm_stream_load_si128((__m128i *)src);
+	_mm_storeu_si128((__m128i *)dst, var128);
+}
+
+static void copy32B_ts(void *dst, void *src)
+{
+	__m256i ymm0;
+
+	ymm0 = _mm256_stream_load_si256((const __m256i *)src);
+	_mm256_storeu_si256((__m256i *)dst, ymm0);
+}
+
+static void copy64B_ts(void *dst, void *src)
+{
+	__m256i ymm0, ymm1;
+
+	ymm0 = _mm256_stream_load_si256((const __m256i *)src);
+	ymm1 = _mm256_stream_load_si256((const __m256i *)((uint8_t *)src + 32));
+	_mm256_storeu_si256((__m256i *)dst, ymm0);
+	_mm256_storeu_si256((__m256i *)((uint8_t *)dst + 32), ymm1);
+}
+
+static void copy128B_ts(void *dst, void *src)
+{
+	__m256i ymm0, ymm1, ymm2, ymm3;
+
+	ymm0 = _mm256_stream_load_si256((const __m256i *)src);
+	ymm1 = _mm256_stream_load_si256((const __m256i *)((uint8_t *)src + 32));
+	ymm2 = _mm256_stream_load_si256((const __m256i *)((uint8_t *)src + 64));
+	ymm3 = _mm256_stream_load_si256((const __m256i *)((uint8_t *)src + 96));
+	_mm256_storeu_si256((__m256i *)dst, ymm0);
+	_mm256_storeu_si256((__m256i *)((uint8_t *)dst + 32), ymm1);
+	_mm256_storeu_si256((__m256i *)((uint8_t *)dst + 64), ymm2);
+	_mm256_storeu_si256((__m256i *)((uint8_t *)dst + 96), ymm3);
+}
+
+static void *memcpy_aligned_rx_tstore_16B(void *dst, void *src, int len)
+{
+	void *dest = dst;
+
+	while (len >= 128) {
+		copy128B_ts(dst, src);
+		dst = (uint8_t *)dst + 128;
+		src = (uint8_t *)src + 128;
+		len -= 128;
+	}
+	while (len >= 64) {
+		copy64B_ts(dst, src);
+		dst = (uint8_t *)dst + 64;
+		src = (uint8_t *)src + 64;
+		len -= 64;
+	}
+	while (len >= 32) {
+		copy32B_ts(dst, src);
+		dst = (uint8_t *)dst + 32;
+		src = (uint8_t *)src + 32;
+		len -= 32;
+	}
+	if (len >= 16) {
+		copy16B_ts(dst, src);
+		dst = (uint8_t *)dst + 16;
+		src = (uint8_t *)src + 16;
+		len -= 16;
+	}
+	if (len >= 8) {
+		*(uint64_t *)dst = *(const uint64_t *)src;
+		dst = (uint8_t *)dst + 8;
+		src = (uint8_t *)src + 8;
+		len -= 8;
+	}
+	if (len >= 4) {
+		*(uint32_t *)dst = *(const uint32_t *)src;
+		dst = (uint8_t *)dst + 4;
+		src = (uint8_t *)src + 4;
+		len -= 4;
+	}
+	if (len != 0) {
+		dst = (uint8_t *)dst - (4 - len);
+		src = (uint8_t *)src - (4 - len);
+		*(uint32_t *)dst = *(const uint32_t *)src;
+	}
+
+	return dest;
+}
+#endif
+
 /**
  * Build a table to translate Rx completion flags to packet type.
  *
@@ -1707,6 +1798,9 @@ mlx5_rx_burst_mprq(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		int32_t hdrm_overlap;
 		volatile struct mlx5_mini_cqe8 *mcqe = NULL;
 		uint32_t rss_hash_res = 0;
+#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
+		uintptr_t data_addr;
+#endif
 
 		if (consumed_strd == strd_n) {
 			/* Replace WQE only if the buffer is still in use. */
@@ -1772,12 +1866,30 @@ mlx5_rx_burst_mprq(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		 * - Out of buffer in the Mempool for Multi-Packet RQ.
 		 * - The packet's stride overlaps a headroom and scatter is off.
 		 */
+#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
+		if (unlikely(!rxq->mprq_tstore_memcpy) &&
+			len <= rxq->mprq_max_memcpy_len) {
+			rte_prefetch1(addr);
+			if (len > RTE_CACHE_LINE_SIZE)
+				rte_prefetch2((void *)((uintptr_t)addr + RTE_CACHE_LINE_SIZE));
+		}
+#endif
 		if (len <= rxq->mprq_max_memcpy_len ||
 		    rxq->mprq_repl == NULL ||
 		    (hdrm_overlap > 0 && !rxq->strd_scatter_en)) {
 			if (likely(rte_pktmbuf_tailroom(pkt) >= len)) {
-				rte_memcpy(rte_pktmbuf_mtod(pkt, void *),
-					   addr, len);
+#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
+				data_addr = (uintptr_t)rte_pktmbuf_mtod(pkt, void *);
+				if (!(rxq->mprq_tstore_memcpy))
+					rte_memcpy((void *)data_addr, addr, len);
+				else if ((rxq->mprq_tstore_memcpy) &&
+					   !((data_addr | (uintptr_t)addr) & ALIGNMENT_MASK))
+					memcpy_aligned_rx_tstore_16B((void *)data_addr,
+							addr, len);
+				else
+#endif
+					rte_memcpy(rte_pktmbuf_mtod(pkt, void *),
+							addr, len);
 				DATA_LEN(pkt) = len;
 			} else if (rxq->strd_scatter_en) {
 				struct rte_mbuf *prev = pkt;
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index 9ffa028d2..a8ea1a795 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -153,6 +153,9 @@ struct mlx5_rxq_data {
 	uint32_t tunnel; /* Tunnel information. */
 	uint64_t flow_meta_mask;
 	int32_t flow_meta_offset;
+#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
+	unsigned int mprq_tstore_memcpy:1;
+#endif
 } __rte_cache_aligned;
 
 enum mlx5_rxq_type {
diff --git a/meson_options.txt b/meson_options.txt
index 9bf18ab6b..a4bc565d2 100644
--- a/meson_options.txt
+++ b/meson_options.txt
@@ -30,6 +30,8 @@ option('max_lcores', type: 'integer', value: 128,
 	description: 'maximum number of cores/threads supported by EAL')
 option('max_numa_nodes', type: 'integer', value: 4,
 	description: 'maximum number of NUMA nodes supported by EAL')
+option('mlx5_ntload_tstore', type: 'boolean', value: false,
+	description: 'to enable optimized MPRQ in RX datapath')
 option('enable_trace_fp', type: 'boolean', value: false,
 	description: 'enable fast path trace points.')
 option('tests', type: 'boolean', value: true,
-- 
2.25.1


^ permalink raw reply	[flat|nested] 5+ messages in thread

* [dpdk-dev] [PATCH v3 2/2] net/mlx5: add non temporal store for WQE fields
  2020-10-10  8:57 [dpdk-dev] [PATCH v3 1/2] net/mlx5: optimize mprq memcpy Aman Kumar
@ 2020-10-10  8:57 ` Aman Kumar
  0 siblings, 0 replies; 5+ messages in thread
From: Aman Kumar @ 2020-10-10  8:57 UTC (permalink / raw)
  To: dev
  Cc: rasland, keesang.song, asafp, shys, viacheslavo, akozyrev, matan,
	aman.kumar

add non temporal store for few WQE fields to optimize
data path. Define RTE_LIBRTE_MLX5_NT_STORE in build
configurations to enable this optimization.

Signed-off-by: Aman Kumar <aman.kumar@vvdntech.in>
---
 drivers/net/mlx5/meson.build     |   1 +
 drivers/net/mlx5/mlx5.c          |  17 ++
 drivers/net/mlx5/mlx5.h          |   4 +
 drivers/net/mlx5/mlx5_rxq.c      |   3 +
 drivers/net/mlx5/mlx5_rxtx.c     | 322 ++++++++++++++++++++++++++++++-
 drivers/net/mlx5/mlx5_rxtx.h     |   6 +
 drivers/net/mlx5/mlx5_rxtx_vec.h |  29 ++-
 drivers/net/mlx5/mlx5_txq.c      |   3 +
 meson_options.txt                |   2 +
 9 files changed, 378 insertions(+), 9 deletions(-)

diff --git a/drivers/net/mlx5/meson.build b/drivers/net/mlx5/meson.build
index 38e93fdc1..347ca6527 100644
--- a/drivers/net/mlx5/meson.build
+++ b/drivers/net/mlx5/meson.build
@@ -48,6 +48,7 @@ foreach option:cflags_options
 	endif
 endforeach
 dpdk_conf.set('RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY', get_option('mlx5_ntload_tstore'))
+dpdk_conf.set('RTE_LIBRTE_MLX5_NT_STORE', get_option('mlx5_ntstore'))
 if get_option('buildtype').contains('debug')
 	cflags += [ '-pedantic', '-DPEDANTIC' ]
 else
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index a2796eaa5..01b25a109 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -164,6 +164,13 @@
 /* mprq_tstore_memcpy */
 #define MLX5_MPRQ_TSTORE_MEMCPY "mprq_tstore_memcpy"
 #endif
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+/* tx_wqe_field_ntstore */
+#define MLX5_TX_WQE_FIELD_NTSTORE "tx_wqe_field_ntstore"
+
+/* vec_rx_wqe_field_ntstore */
+#define MLX5_VEC_RX_WQE_FIELD_NTSTORE "vec_rx_wqe_field_ntstore"
+#endif
 
 /*
  * Device parameter to configure the total data buffer size for a single
@@ -1631,6 +1638,12 @@ mlx5_args_check(const char *key, const char *val, void *opaque)
 #ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
 	} else if (strcmp(MLX5_MPRQ_TSTORE_MEMCPY, key) == 0) {
 		config->mprq_tstore_memcpy = tmp;
+#endif
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+	} else if (strcmp(MLX5_TX_WQE_FIELD_NTSTORE, key) == 0) {
+		config->tx_wqe_field_ntstore = tmp;
+	} else if (strcmp(MLX5_VEC_RX_WQE_FIELD_NTSTORE, key) == 0) {
+		config->vec_rx_wqe_field_ntstore = tmp;
 #endif
 	} else {
 		DRV_LOG(WARNING, "%s: unknown parameter", key);
@@ -1694,6 +1707,10 @@ mlx5_args(struct mlx5_dev_config *config, struct rte_devargs *devargs)
 		MLX5_DECAP_EN,
 #ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
 		MLX5_MPRQ_TSTORE_MEMCPY,
+#endif
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+		MLX5_TX_WQE_FIELD_NTSTORE,
+		MLX5_VEC_RX_WQE_FIELD_NTSTORE,
 #endif
 		NULL,
 	};
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 1eb305650..9d192465f 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -237,6 +237,10 @@ struct mlx5_dev_config {
 #ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
 	unsigned int mprq_tstore_memcpy:1;
 #endif
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+	unsigned int tx_wqe_field_ntstore:1;
+	unsigned int vec_rx_wqe_field_ntstore:1;
+#endif
 };
 
 
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index c8db59a12..69ad9ab8c 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -1382,6 +1382,9 @@ mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 		tmpl->irq = 1;
 #ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
 	tmpl->rxq.mprq_tstore_memcpy = config->mprq_tstore_memcpy;
+#endif
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+	tmpl->rxq.vec_rx_wqe_field_ntstore = config->vec_rx_wqe_field_ntstore;
 #endif
 	mprq_stride_nums = config->mprq.stride_num_n ?
 		config->mprq.stride_num_n : MLX5_MPRQ_STRIDE_NUM_N;
diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index f59e30d82..76bf20b6f 100644
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -214,6 +214,301 @@ static void *memcpy_aligned_rx_tstore_16B(void *dst, void *src, int len)
 }
 #endif
 
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+static void *amd_memcpy(void *dest, const void *src, size_t size)
+{
+	asm goto (
+	"movq	%0, %%rsi\n\t"
+	"movq	%1, %%rdi\n\t"
+	"movq	%2, %%rdx\n\t"
+	"movq    %%rdi, %%rax\n\t"
+	"cmp     $32, %%rdx\n\t"
+	"jb      less_vec\n\t"
+	"cmp     $(32 * 2), %%rdx\n\t"
+	"ja      more_2x_vec\n\t"
+	"vmovdqu   (%%rsi), %%ymm0\n\t"
+	"vmovdqu   -32(%%rsi,%%rdx), %%ymm1\n\t"
+	"vmovdqu   %%ymm0, (%%rdi)\n\t"
+	"vmovdqu   %%ymm1, -32(%%rdi,%%rdx)\n\t"
+	"vzeroupper\n\t"
+	"jmp %l[done]\n\t"
+	"less_vec:\n\t"
+	/* Less than 1 VEC.  */
+	"cmpb    $32, %%dl\n\t"
+	"jae     between_32_63\n\t"
+	"cmpb    $16, %%dl\n\t"
+	"jae     between_16_31\n\t"
+	"cmpb    $8, %%dl\n\t"
+	"jae     between_8_15\n\t"
+	"cmpb    $4, %%dl\n\t"
+	"jae     between_4_7\n\t"
+	"cmpb    $1, %%dl\n\t"
+	"ja      between_2_3\n\t"
+	"jb      1f\n\t"
+	"movzbl  (%%rsi), %%ecx\n\t"
+	"movb    %%cl, (%%rdi)\n\t"
+	"1:\n\t"
+	"jmp %l[done]\n\t"
+	"between_32_63:\n\t"
+	/* From 32 to 63.  No branch when size == 32.  */
+	"vmovdqu (%%rsi), %%ymm0\n\t"
+	"vmovdqu -32(%%rsi,%%rdx), %%ymm1\n\t"
+	"vmovdqu %%ymm0, (%%rdi)\n\t"
+	"vmovdqu %%ymm1, -32(%%rdi,%%rdx)\n\t"
+	"vzeroupper\n\t"
+	"jmp %l[done]\n\t"
+	/* From 16 to 31.  No branch when size == 16.  */
+	"between_16_31:\n\t"
+	"vmovdqu (%%rsi), %%xmm0\n\t"
+	"vmovdqu -16(%%rsi,%%rdx), %%xmm1\n\t"
+	"vmovdqu %%xmm0, (%%rdi)\n\t"
+	"vmovdqu %%xmm1, -16(%%rdi,%%rdx)\n\t"
+	"jmp %l[done]\n\t"
+	"between_8_15:\n\t"
+	/* From 8 to 15.  No branch when size == 8.  */
+	"movq    -8(%%rsi,%%rdx), %%rcx\n\t"
+	"movq    (%%rsi), %%rsi\n\t"
+	"movq    %%rcx, -8(%%rdi,%%rdx)\n\t"
+	"movq    %%rsi, (%%rdi)\n\t"
+	"jmp %l[done]\n\t"
+	"between_4_7:\n\t"
+	/* From 4 to 7.  No branch when size == 4.  */
+	"movl    -4(%%rsi,%%rdx), %%ecx\n\t"
+	"movl    (%%rsi), %%esi\n\t"
+	"movl    %%ecx, -4(%%rdi,%%rdx)\n\t"
+	"movl    %%esi, (%%rdi)\n\t"
+	"jmp %l[done]\n\t"
+	"between_2_3:\n\t"
+	/* From 2 to 3.  No branch when size == 2.  */
+	"movzwl  -2(%%rsi,%%rdx), %%ecx\n\t"
+	"movzwl  (%%rsi), %%esi\n\t"
+	"movw    %%cx, -2(%%rdi,%%rdx)\n\t"
+	"movw    %%si, (%%rdi)\n\t"
+	"jmp %l[done]\n\t"
+	"more_2x_vec:\n\t"
+	/* More than 2 * VEC and there may be overlap between destination */
+	/* and source.  */
+	"cmpq    $(32 * 8), %%rdx\n\t"
+	"ja      more_8x_vec\n\t"
+	"cmpq    $(32 * 4), %%rdx\n\t"
+	"jb      last_4x_vec\n\t"
+	/* Copy from 4 * VEC to 8 * VEC, inclusively. */
+	"vmovdqu   (%%rsi), %%ymm0\n\t"
+	"vmovdqu   32(%%rsi), %%ymm1\n\t"
+	"vmovdqu   (32 * 2)(%%rsi), %%ymm2\n\t"
+	"vmovdqu   (32 * 3)(%%rsi), %%ymm3\n\t"
+	"vmovdqu   -32(%%rsi,%%rdx), %%ymm4\n\t"
+	"vmovdqu   -(32 * 2)(%%rsi,%%rdx), %%ymm5\n\t"
+	"vmovdqu   -(32 * 3)(%%rsi,%%rdx), %%ymm6\n\t"
+	"vmovdqu   -(32 * 4)(%%rsi,%%rdx), %%ymm7\n\t"
+	"vmovdqu   %%ymm0, (%%rdi)\n\t"
+	"vmovdqu   %%ymm1, 32(%%rdi)\n\t"
+	"vmovdqu   %%ymm2, (32 * 2)(%%rdi)\n\t"
+	"vmovdqu   %%ymm3, (32 * 3)(%%rdi)\n\t"
+	"vmovdqu   %%ymm4, -32(%%rdi,%%rdx)\n\t"
+	"vmovdqu   %%ymm5, -(32 * 2)(%%rdi,%%rdx)\n\t"
+	"vmovdqu   %%ymm6, -(32 * 3)(%%rdi,%%rdx)\n\t"
+	"vmovdqu   %%ymm7, -(32 * 4)(%%rdi,%%rdx)\n\t"
+	"vzeroupper\n\t"
+	"jmp %l[done]\n\t"
+	"last_4x_vec:\n\t"
+	/* Copy from 2 * VEC to 4 * VEC. */
+	"vmovdqu   (%%rsi), %%ymm0\n\t"
+	"vmovdqu   32(%%rsi), %%ymm1\n\t"
+	"vmovdqu   -32(%%rsi,%%rdx), %%ymm2\n\t"
+	"vmovdqu   -(32 * 2)(%%rsi,%%rdx), %%ymm3\n\t"
+	"vmovdqu   %%ymm0, (%%rdi)\n\t"
+	"vmovdqu   %%ymm1, 32(%%rdi)\n\t"
+	"vmovdqu   %%ymm2, -32(%%rdi,%%rdx)\n\t"
+	"vmovdqu   %%ymm3, -(32 * 2)(%%rdi,%%rdx)\n\t"
+	"vzeroupper\n\t"
+	"nop:\n\t"
+	"jmp %l[done]\n\t"
+	"more_8x_vec:\n\t"
+	"cmpq    %%rsi, %%rdi\n\t"
+	"ja      more_8x_vec_backward\n\t"
+	/* Source == destination is less common.  */
+	"je      nop\n\t"
+	/* Load the first VEC and last 4 * VEC to support overlapping addresses.  */
+	"vmovdqu   (%%rsi), %%ymm4\n\t"
+	"vmovdqu   -32(%%rsi, %%rdx), %%ymm5\n\t"
+	"vmovdqu   -(32 * 2)(%%rsi, %%rdx), %%ymm6\n\t"
+	"vmovdqu   -(32 * 3)(%%rsi, %%rdx), %%ymm7\n\t"
+	"vmovdqu   -(32 * 4)(%%rsi, %%rdx), %%ymm8\n\t"
+	/* Save start and stop of the destination buffer.  */
+	"movq    %%rdi, %%r11\n\t"
+	"leaq    -32(%%rdi, %%rdx), %%rcx\n\t"
+	/* Align destination for aligned stores in the loop.  Compute */
+	/* how much destination is misaligned.  */
+	"movq    %%rdi, %%r8\n\t"
+	"andq    $(32 - 1), %%r8\n\t"
+	/* Get the negative of offset for alignment.  */
+	"subq    $32, %%r8\n\t"
+	/* Adjust source.  */
+	"subq    %%r8, %%rsi\n\t"
+	/* Adjust destination which should be aligned now.  */
+	"subq    %%r8, %%rdi\n\t"
+	/* Adjust length.  */
+	"addq    %%r8, %%rdx\n\t"
+	/* Check non-temporal store threshold.  */
+	"cmpq	 $(1024*1024), %%rdx\n\t"
+	"ja      large_forward\n\t"
+	"loop_4x_vec_forward:\n\t"
+	/* Copy 4 * VEC a time forward.  */
+	"vmovdqu   (%%rsi), %%ymm0\n\t"
+	"vmovdqu   32(%%rsi), %%ymm1\n\t"
+	"vmovdqu   (32 * 2)(%%rsi), %%ymm2\n\t"
+	"vmovdqu   (32 * 3)(%%rsi), %%ymm3\n\t"
+	"addq    $(32 * 4), %%rsi\n\t"
+	"subq    $(32 * 4), %%rdx\n\t"
+	"vmovdqa   %%ymm0, (%%rdi)\n\t"
+	"vmovdqa   %%ymm1, 32(%%rdi)\n\t"
+	"vmovdqa   %%ymm2, (32 * 2)(%%rdi)\n\t"
+	"vmovdqa   %%ymm3, (32 * 3)(%%rdi)\n\t"
+	"addq    $(32 * 4), %%rdi\n\t"
+	"cmpq    $(32 * 4), %%rdx\n\t"
+	"ja      loop_4x_vec_forward\n\t"
+	/* Store the last 4 * VEC.  */
+	"vmovdqu   %%ymm5, (%%rcx)\n\t"
+	"vmovdqu   %%ymm6, -32(%%rcx)\n\t"
+	"vmovdqu   %%ymm7, -(32 * 2)(%%rcx)\n\t"
+	"vmovdqu   %%ymm8, -(32 * 3)(%%rcx)\n\t"
+	/* Store the first VEC.  */
+	"vmovdqu   %%ymm4, (%%r11)\n\t"
+	"vzeroupper\n\t"
+	"jmp %l[done]\n\t"
+	"more_8x_vec_backward:\n\t"
+	/* Load the first 4*VEC and last VEC to support overlapping addresses.*/
+	"vmovdqu   (%%rsi), %%ymm4\n\t"
+	"vmovdqu   32(%%rsi), %%ymm5\n\t"
+	"vmovdqu   (32 * 2)(%%rsi), %%ymm6\n\t"
+	"vmovdqu   (32 * 3)(%%rsi), %%ymm7\n\t"
+	"vmovdqu   -32(%%rsi,%%rdx), %%ymm8\n\t"
+	/* Save stop of the destination buffer.  */
+	"leaq    -32(%%rdi, %%rdx), %%r11\n\t"
+	/* Align destination end for aligned stores in the loop.  Compute */
+	/* how much destination end is misaligned.  */
+	"leaq    -32(%%rsi, %%rdx), %%rcx\n\t"
+	"movq    %%r11, %%r9\n\t"
+	"movq    %%r11, %%r8\n\t"
+	"andq    $(32 - 1), %%r8\n\t"
+	/* Adjust source.  */
+	"subq    %%r8, %%rcx\n\t"
+	/* Adjust the end of destination which should be aligned now.  */
+	"subq    %%r8, %%r9\n\t"
+	/* Adjust length.  */
+	"subq    %%r8, %%rdx\n\t"
+	 /* Check non-temporal store threshold.  */
+	"cmpq	 $(1024*1024), %%rdx\n\t"
+	"ja      large_backward\n\t"
+	"loop_4x_vec_backward:\n\t"
+	/* Copy 4 * VEC a time backward.  */
+	"vmovdqu   (%%rcx), %%ymm0\n\t"
+	"vmovdqu   -32(%%rcx), %%ymm1\n\t"
+	"vmovdqu   -(32 * 2)(%%rcx), %%ymm2\n\t"
+	"vmovdqu   -(32 * 3)(%%rcx), %%ymm3\n\t"
+	"subq    $(32 * 4), %%rcx\n\t"
+	"subq    $(32 * 4), %%rdx\n\t"
+	"vmovdqa   %%ymm0, (%%r9)\n\t"
+	"vmovdqa   %%ymm1, -32(%%r9)\n\t"
+	"vmovdqa   %%ymm2, -(32 * 2)(%%r9)\n\t"
+	"vmovdqa   %%ymm3, -(32 * 3)(%%r9)\n\t"
+	"subq    $(32 * 4), %%r9\n\t"
+	"cmpq    $(32 * 4), %%rdx\n\t"
+	"ja      loop_4x_vec_backward\n\t"
+	/* Store the first 4 * VEC. */
+	"vmovdqu   %%ymm4, (%%rdi)\n\t"
+	"vmovdqu   %%ymm5, 32(%%rdi)\n\t"
+	"vmovdqu   %%ymm6, (32 * 2)(%%rdi)\n\t"
+	"vmovdqu   %%ymm7, (32 * 3)(%%rdi)\n\t"
+	/* Store the last VEC. */
+	"vmovdqu   %%ymm8, (%%r11)\n\t"
+	"vzeroupper\n\t"
+	"jmp %l[done]\n\t"
+
+	"large_forward:\n\t"
+	/* Don't use non-temporal store if there is overlap between */
+	/* destination and source since destination may be in cache */
+	/* when source is loaded. */
+	"leaq    (%%rdi, %%rdx), %%r10\n\t"
+	"cmpq    %%r10, %%rsi\n\t"
+	"jb      loop_4x_vec_forward\n\t"
+	"loop_large_forward:\n\t"
+	/* Copy 4 * VEC a time forward with non-temporal stores.  */
+	"prefetcht0 (32*4*2)(%%rsi)\n\t"
+	"prefetcht0 (32*4*2 + 64)(%%rsi)\n\t"
+	"prefetcht0 (32*4*3)(%%rsi)\n\t"
+	"prefetcht0 (32*4*3 + 64)(%%rsi)\n\t"
+	"vmovdqu   (%%rsi), %%ymm0\n\t"
+	"vmovdqu   32(%%rsi), %%ymm1\n\t"
+	"vmovdqu   (32 * 2)(%%rsi), %%ymm2\n\t"
+	"vmovdqu   (32 * 3)(%%rsi), %%ymm3\n\t"
+	"addq    $(32*4), %%rsi\n\t"
+	"subq    $(32*4), %%rdx\n\t"
+	"vmovntdq  %%ymm0, (%%rdi)\n\t"
+	"vmovntdq  %%ymm1, 32(%%rdi)\n\t"
+	"vmovntdq  %%ymm2, (32 * 2)(%%rdi)\n\t"
+	"vmovntdq  %%ymm3, (32 * 3)(%%rdi)\n\t"
+	"addq    $(32*4), %%rdi\n\t"
+	"cmpq    $(32*4), %%rdx\n\t"
+	"ja      loop_large_forward\n\t"
+	"sfence\n\t"
+	/* Store the last 4 * VEC.  */
+	"vmovdqu   %%ymm5, (%%rcx)\n\t"
+	"vmovdqu   %%ymm6, -32(%%rcx)\n\t"
+	"vmovdqu   %%ymm7, -(32 * 2)(%%rcx)\n\t"
+	"vmovdqu   %%ymm8, -(32 * 3)(%%rcx)\n\t"
+	/* Store the first VEC.  */
+	"vmovdqu   %%ymm4, (%%r11)\n\t"
+	"vzeroupper\n\t"
+	"jmp %l[done]\n\t"
+	"large_backward:\n\t"
+	/* Don't use non-temporal store if there is overlap between */
+	/* destination and source since destination may be in cache */
+	/* when source is loaded.  */
+	"leaq    (%%rcx, %%rdx), %%r10\n\t"
+	"cmpq    %%r10, %%r9\n\t"
+	"jb      loop_4x_vec_backward\n\t"
+	"loop_large_backward:\n\t"
+	/* Copy 4 * VEC a time backward with non-temporal stores. */
+	"prefetcht0 (-32 * 4 * 2)(%%rcx)\n\t"
+	"prefetcht0 (-32 * 4 * 2 - 64)(%%rcx)\n\t"
+	"prefetcht0 (-32 * 4 * 3)(%%rcx)\n\t"
+	"prefetcht0 (-32 * 4 * 3 - 64)(%%rcx)\n\t"
+	"vmovdqu   (%%rcx), %%ymm0\n\t"
+	"vmovdqu   -32(%%rcx), %%ymm1\n\t"
+	"vmovdqu   -(32 * 2)(%%rcx), %%ymm2\n\t"
+	"vmovdqu   -(32 * 3)(%%rcx), %%ymm3\n\t"
+	"subq    $(32*4), %%rcx\n\t"
+	"subq    $(32*4), %%rdx\n\t"
+	"vmovntdq  %%ymm0, (%%r9)\n\t"
+	"vmovntdq  %%ymm1, -32(%%r9)\n\t"
+	"vmovntdq  %%ymm2, -(32 * 2)(%%r9)\n\t"
+	"vmovntdq  %%ymm3, -(32 * 3)(%%r9)\n\t"
+	"subq    $(32 * 4), %%r9\n\t"
+	"cmpq    $(32 * 4), %%rdx\n\t"
+	"ja      loop_large_backward\n\t"
+	"sfence\n\t"
+	/* Store the first 4 * VEC.  */
+	"vmovdqu   %%ymm4, (%%rdi)\n\t"
+	"vmovdqu   %%ymm5, 32(%%rdi)\n\t"
+	"vmovdqu   %%ymm6, (32 * 2)(%%rdi)\n\t"
+	"vmovdqu   %%ymm7, (32 * 3)(%%rdi)\n\t"
+	/* Store the last VEC.  */
+	"vmovdqu   %%ymm8, (%%r11)\n\t"
+	"vzeroupper\n\t"
+	"jmp %l[done]"
+	:
+	: "r"(src), "r"(dest), "r"(size)
+	: "rax", "rcx", "rdx", "rdi", "rsi", "r8", "r9", "r10", "r11", "r12", "ymm0",
+	"ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", "ymm8", "memory"
+	: done
+	);
+done:
+	return dest;
+}
+#endif
+
 /**
  * Build a table to translate Rx completion flags to packet type.
  *
@@ -2419,6 +2714,9 @@ mlx5_tx_request_completion(struct mlx5_txq_data *__rte_restrict txq,
 {
 	uint16_t head = txq->elts_head;
 	unsigned int part;
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+	register uint32_t flags;
+#endif
 
 	part = MLX5_TXOFF_CONFIG(INLINE) ?
 	       0 : loc->pkts_sent - loc->pkts_copy;
@@ -2432,9 +2730,20 @@ mlx5_tx_request_completion(struct mlx5_txq_data *__rte_restrict txq,
 		txq->elts_comp = head;
 		if (MLX5_TXOFF_CONFIG(INLINE))
 			txq->wqe_comp = txq->wqe_ci;
-		/* Request unconditional completion on last WQE. */
-		last->cseg.flags = RTE_BE32(MLX5_COMP_ALWAYS <<
-					    MLX5_COMP_MODE_OFFSET);
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+		if (txq->tx_wqe_field_ntstore) {
+			flags = RTE_BE32(MLX5_COMP_ALWAYS <<
+					MLX5_COMP_MODE_OFFSET);
+			_mm_stream_si32(((void *)(uintptr_t)&last->cseg.flags),
+					flags);
+		} else {
+#endif
+			/* Request unconditional completion on last WQE. */
+			last->cseg.flags = RTE_BE32(MLX5_COMP_ALWAYS <<
+					MLX5_COMP_MODE_OFFSET);
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+		}
+#endif
 		/* Save elts_head in dedicated free on completion queue. */
 #ifdef RTE_LIBRTE_MLX5_DEBUG
 		txq->fcqs[txq->cq_pi++ & txq->cqe_m] = head |
@@ -3162,7 +3471,12 @@ mlx5_tx_dseg_empw(struct mlx5_txq_data *__rte_restrict txq,
 	part = (uint8_t *)txq->wqes_end - pdst;
 	part = RTE_MIN(part, len);
 	do {
-		rte_memcpy(pdst, buf, part);
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+		if (likely(txq->tx_wqe_field_ntstore))
+			amd_memcpy(pdst, buf, part);
+		else
+#endif
+			rte_memcpy(pdst, buf, part);
 		len -= part;
 		if (likely(!len)) {
 			pdst += part;
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index a8ea1a795..f1e59a881 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -156,6 +156,9 @@ struct mlx5_rxq_data {
 #ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
 	unsigned int mprq_tstore_memcpy:1;
 #endif
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+	unsigned int vec_rx_wqe_field_ntstore:1;
+#endif
 } __rte_cache_aligned;
 
 enum mlx5_rxq_type {
@@ -256,6 +259,9 @@ struct mlx5_txq_data {
 	int32_t ts_offset; /* Timestamp field dynamic offset. */
 	struct mlx5_dev_ctx_shared *sh; /* Shared context. */
 	struct mlx5_txq_stats stats; /* TX queue counters. */
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+	unsigned int tx_wqe_field_ntstore:1;
+#endif
 #ifndef RTE_ARCH_64
 	rte_spinlock_t *uar_lock;
 	/* UAR access lock required for 32bit implementations */
diff --git a/drivers/net/mlx5/mlx5_rxtx_vec.h b/drivers/net/mlx5/mlx5_rxtx_vec.h
index a8d6c4f41..413f863ba 100644
--- a/drivers/net/mlx5/mlx5_rxtx_vec.h
+++ b/drivers/net/mlx5/mlx5_rxtx_vec.h
@@ -86,6 +86,10 @@ mlx5_rx_replenish_bulk_mbuf(struct mlx5_rxq_data *rxq, uint16_t n)
 	volatile struct mlx5_wqe_data_seg *wq =
 		&((volatile struct mlx5_wqe_data_seg *)rxq->wqes)[elts_idx];
 	unsigned int i;
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+	register uint64_t buf_addr2;
+	register uint32_t lkey_t;
+#endif
 
 	MLX5_ASSERT(n >= MLX5_VPMD_RXQ_RPLNSH_THRESH(q_n));
 	MLX5_ASSERT(n <= (uint16_t)(q_n - (rxq->rq_ci - rxq->rq_pi)));
@@ -107,11 +111,26 @@ mlx5_rx_replenish_bulk_mbuf(struct mlx5_rxq_data *rxq, uint16_t n)
 		 * impact the performance.
 		 */
 		buf_addr = elts[i]->buf_addr;
-		wq[i].addr = rte_cpu_to_be_64((uintptr_t)buf_addr +
-					      RTE_PKTMBUF_HEADROOM);
-		/* If there's only one MR, no need to replace LKey in WQE. */
-		if (unlikely(mlx5_mr_btree_len(&rxq->mr_ctrl.cache_bh) > 1))
-			wq[i].lkey = mlx5_rx_mb2mr(rxq, elts[i]);
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+		if (rxq->vec_rx_wqe_field_ntstore) {
+			buf_addr2 = (uint64_t)rte_cpu_to_be_64((uintptr_t)buf_addr +
+							       RTE_PKTMBUF_HEADROOM);
+			_mm_stream_si64(((void *)(uintptr_t)&wq[i].addr), buf_addr2);
+			/* If there's only one MR, no need to replace LKey in WQE. */
+			if (unlikely(mlx5_mr_btree_len(&rxq->mr_ctrl.cache_bh) > 1)) {
+				lkey_t = (uint32_t)mlx5_rx_mb2mr(rxq, elts[i]);
+				_mm_stream_si32(((void *)(uintptr_t)&wq[i].lkey), lkey_t);
+			}
+		} else {
+#endif
+			wq[i].addr = rte_cpu_to_be_64((uintptr_t)buf_addr +
+					RTE_PKTMBUF_HEADROOM);
+			/* If there's only one MR, no need to replace LKey in WQE. */
+			if (unlikely(mlx5_mr_btree_len(&rxq->mr_ctrl.cache_bh) > 1))
+				wq[i].lkey = mlx5_rx_mb2mr(rxq, elts[i]);
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+		}
+#endif
 	}
 	rxq->rq_ci += n;
 	/* Prevent overflowing into consumed mbufs. */
diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
index 1bb667d46..cba675f53 100644
--- a/drivers/net/mlx5/mlx5_txq.c
+++ b/drivers/net/mlx5/mlx5_txq.c
@@ -1565,6 +1565,9 @@ txq_set_params(struct mlx5_txq_ctrl *txq_ctrl)
 					    DEV_TX_OFFLOAD_UDP_TNL_TSO);
 	bool vlan_inline;
 	unsigned int temp;
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+	txq_ctrl->txq.tx_wqe_field_ntstore = config->tx_wqe_field_ntstore;
+#endif
 
 	if (config->txqs_inline == MLX5_ARG_UNSET)
 		txqs_inline =
diff --git a/meson_options.txt b/meson_options.txt
index a4bc565d2..21c31d57b 100644
--- a/meson_options.txt
+++ b/meson_options.txt
@@ -32,6 +32,8 @@ option('max_numa_nodes', type: 'integer', value: 4,
 	description: 'maximum number of NUMA nodes supported by EAL')
 option('mlx5_ntload_tstore', type: 'boolean', value: false,
 	description: 'to enable optimized MPRQ in RX datapath')
+option('mlx5_ntstore', type: 'boolean', value: false,
+	description: 'to enable optimized MLX5 TX datapath')
 option('enable_trace_fp', type: 'boolean', value: false,
 	description: 'enable fast path trace points.')
 option('tests', type: 'boolean', value: true,
-- 
2.25.1


^ permalink raw reply	[flat|nested] 5+ messages in thread

* Re: [dpdk-dev] [PATCH v3 1/2] net/mlx5: optimize mprq memcpy
  2021-02-04 14:14   ` Slava Ovsiienko
@ 2021-02-09  6:22     ` Aman Kumar
  0 siblings, 0 replies; 5+ messages in thread
From: Aman Kumar @ 2021-02-09  6:22 UTC (permalink / raw)
  To: Slava Ovsiienko
  Cc: dev, Raslan Darawsheh, keesang.song, Asaf Penso, Shy Shyman,
	Alexander Kozyrev, Matan Azrad

Hi Slava,

Thank you for your reply.
This is currently supported (and tested) on 2nd gen AMD EPYC series
processors. We're currently trying to make it more generic across other
EPYC platforms. We too believe having these available at EAL may also help
applications and other PMDs. I'll move memory copy instructions to
lib/librte_eal/* and update this patchset.
Thanks.

*Regards*
Aman Kumar

On Thu, Feb 4, 2021 at 7:45 PM Slava Ovsiienko <viacheslavo@nvidia.com>
wrote:

> Hi,
>
> I'm sorry for asking the questions very late.
> Is still this patch set actual and should it be updated and considered?
>
> As I can understand this one optimizes the memory writes in some way using
> the instructions with the hints.
> Is this specific for some CPU families? Is this more common? I suppose it
> should we considered and discussed
> more widely, possible on EAL level. I would propose to introduce these
> special memory routines on EAL level
> to give advantage to all PMDs, not specifically to mlx5.
>
> With best regards,
> Slava
>
> > -----Original Message-----
> > From: Aman Kumar <aman.kumar@vvdntech.in>
> > Sent: Saturday, October 10, 2020 12:01
> > To: dev@dpdk.org
> > Cc: Raslan Darawsheh <rasland@nvidia.com>; keesang.song@amd.com;
> > Asaf Penso <asafp@nvidia.com>; Shy Shyman <shys@nvidia.com>; Slava
> > Ovsiienko <viacheslavo@nvidia.com>; Alexander Kozyrev
> > <akozyrev@nvidia.com>; Matan Azrad <matan@nvidia.com>;
> > aman.kumar@vvdntech.in
> > Subject: [PATCH v3 1/2] net/mlx5: optimize mprq memcpy
> >
> > add non temporal load and temporal store for mprq memcpy.
> > define RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY in build
> > configuration to enable this optimization.
> >
> > Signed-off-by: Aman Kumar <aman.kumar@vvdntech.in>
> > ---
> >  drivers/net/mlx5/meson.build |   1 +
> >  drivers/net/mlx5/mlx5.c      |  12 ++++
> >  drivers/net/mlx5/mlx5.h      |   3 +
> >  drivers/net/mlx5/mlx5_rxq.c  |   3 +
> >  drivers/net/mlx5/mlx5_rxtx.c | 116
> > ++++++++++++++++++++++++++++++++++-
> >  drivers/net/mlx5/mlx5_rxtx.h |   3 +
> >  meson_options.txt            |   2 +
> >  7 files changed, 138 insertions(+), 2 deletions(-)
> >
> > diff --git a/drivers/net/mlx5/meson.build b/drivers/net/mlx5/meson.build
> > index 9a97bb9c8..38e93fdc1 100644
> > --- a/drivers/net/mlx5/meson.build
> > +++ b/drivers/net/mlx5/meson.build
> > @@ -47,6 +47,7 @@ foreach option:cflags_options
> >               cflags += option
> >       endif
> >  endforeach
> > +dpdk_conf.set('RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY',
> > +get_option('mlx5_ntload_tstore'))
> >  if get_option('buildtype').contains('debug')
> >       cflags += [ '-pedantic', '-DPEDANTIC' ]  else diff --git
> > a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index
> > 01ead6e6a..a2796eaa5 100644
> > --- a/drivers/net/mlx5/mlx5.c
> > +++ b/drivers/net/mlx5/mlx5.c
> > @@ -160,6 +160,11 @@
> >  /* Configure timeout of LRO session (in microseconds). */  #define
> > MLX5_LRO_TIMEOUT_USEC "lro_timeout_usec"
> >
> > +#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
> > +/* mprq_tstore_memcpy */
> > +#define MLX5_MPRQ_TSTORE_MEMCPY "mprq_tstore_memcpy"
> > +#endif
> > +
> >  /*
> >   * Device parameter to configure the total data buffer size for a single
> >   * hairpin queue (logarithm value).
> > @@ -1623,6 +1628,10 @@ mlx5_args_check(const char *key, const char
> > *val, void *opaque)
> >               config->sys_mem_en = !!tmp;
> >       } else if (strcmp(MLX5_DECAP_EN, key) == 0) {
> >               config->decap_en = !!tmp;
> > +#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
> > +     } else if (strcmp(MLX5_MPRQ_TSTORE_MEMCPY, key) == 0) {
> > +             config->mprq_tstore_memcpy = tmp;
> > +#endif
> >       } else {
> >               DRV_LOG(WARNING, "%s: unknown parameter", key);
> >               rte_errno = EINVAL;
> > @@ -1683,6 +1692,9 @@ mlx5_args(struct mlx5_dev_config *config, struct
> > rte_devargs *devargs)
> >               MLX5_RECLAIM_MEM,
> >               MLX5_SYS_MEM_EN,
> >               MLX5_DECAP_EN,
> > +#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
> > +             MLX5_MPRQ_TSTORE_MEMCPY,
> > +#endif
> >               NULL,
> >       };
> >       struct rte_kvargs *kvlist;
> > diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index
> > 43da9a1fb..1eb305650 100644
> > --- a/drivers/net/mlx5/mlx5.h
> > +++ b/drivers/net/mlx5/mlx5.h
> > @@ -234,6 +234,9 @@ struct mlx5_dev_config {
> >       int tx_skew; /* Tx scheduling skew between WQE and data on wire.
> > */
> >       struct mlx5_hca_attr hca_attr; /* HCA attributes. */
> >       struct mlx5_lro_config lro; /* LRO configuration. */
> > +#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
> > +     unsigned int mprq_tstore_memcpy:1;
> > +#endif
> >  };
> >
> >
> > diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
> index
> > c059e216d..c8db59a12 100644
> > --- a/drivers/net/mlx5/mlx5_rxq.c
> > +++ b/drivers/net/mlx5/mlx5_rxq.c
> > @@ -1380,6 +1380,9 @@ mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t
> > idx, uint16_t desc,
> >       tmpl->socket = socket;
> >       if (dev->data->dev_conf.intr_conf.rxq)
> >               tmpl->irq = 1;
> > +#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
> > +     tmpl->rxq.mprq_tstore_memcpy = config->mprq_tstore_memcpy;
> > #endif
> >       mprq_stride_nums = config->mprq.stride_num_n ?
> >               config->mprq.stride_num_n : MLX5_MPRQ_STRIDE_NUM_N;
> >       mprq_stride_size = non_scatter_min_mbuf_size <= diff --git
> > a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c index
> > 0b87be15b..f59e30d82 100644
> > --- a/drivers/net/mlx5/mlx5_rxtx.c
> > +++ b/drivers/net/mlx5/mlx5_rxtx.c
> > @@ -123,6 +123,97 @@ uint8_t mlx5_swp_types_table[1 << 10]
> > __rte_cache_aligned;  uint64_t rte_net_mlx5_dynf_inline_mask;  #define
> > PKT_TX_DYNF_NOINLINE rte_net_mlx5_dynf_inline_mask
> >
> > +#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
> > +static void copy16B_ts(void *dst, void *src) {
> > +     __m128i var128;
> > +
> > +     var128 = _mm_stream_load_si128((__m128i *)src);
> > +     _mm_storeu_si128((__m128i *)dst, var128); }
> > +
> > +static void copy32B_ts(void *dst, void *src) {
> > +     __m256i ymm0;
> > +
> > +     ymm0 = _mm256_stream_load_si256((const __m256i *)src);
> > +     _mm256_storeu_si256((__m256i *)dst, ymm0); }
> > +
> > +static void copy64B_ts(void *dst, void *src) {
> > +     __m256i ymm0, ymm1;
> > +
> > +     ymm0 = _mm256_stream_load_si256((const __m256i *)src);
> > +     ymm1 = _mm256_stream_load_si256((const __m256i *)((uint8_t
> > *)src + 32));
> > +     _mm256_storeu_si256((__m256i *)dst, ymm0);
> > +     _mm256_storeu_si256((__m256i *)((uint8_t *)dst + 32), ymm1); }
> > +
> > +static void copy128B_ts(void *dst, void *src) {
> > +     __m256i ymm0, ymm1, ymm2, ymm3;
> > +
> > +     ymm0 = _mm256_stream_load_si256((const __m256i *)src);
> > +     ymm1 = _mm256_stream_load_si256((const __m256i *)((uint8_t
> > *)src + 32));
> > +     ymm2 = _mm256_stream_load_si256((const __m256i *)((uint8_t
> > *)src + 64));
> > +     ymm3 = _mm256_stream_load_si256((const __m256i *)((uint8_t
> > *)src + 96));
> > +     _mm256_storeu_si256((__m256i *)dst, ymm0);
> > +     _mm256_storeu_si256((__m256i *)((uint8_t *)dst + 32), ymm1);
> > +     _mm256_storeu_si256((__m256i *)((uint8_t *)dst + 64), ymm2);
> > +     _mm256_storeu_si256((__m256i *)((uint8_t *)dst + 96), ymm3); }
> > +
> > +static void *memcpy_aligned_rx_tstore_16B(void *dst, void *src, int
> > +len) {
> > +     void *dest = dst;
> > +
> > +     while (len >= 128) {
> > +             copy128B_ts(dst, src);
> > +             dst = (uint8_t *)dst + 128;
> > +             src = (uint8_t *)src + 128;
> > +             len -= 128;
> > +     }
> > +     while (len >= 64) {
> > +             copy64B_ts(dst, src);
> > +             dst = (uint8_t *)dst + 64;
> > +             src = (uint8_t *)src + 64;
> > +             len -= 64;
> > +     }
> > +     while (len >= 32) {
> > +             copy32B_ts(dst, src);
> > +             dst = (uint8_t *)dst + 32;
> > +             src = (uint8_t *)src + 32;
> > +             len -= 32;
> > +     }
> > +     if (len >= 16) {
> > +             copy16B_ts(dst, src);
> > +             dst = (uint8_t *)dst + 16;
> > +             src = (uint8_t *)src + 16;
> > +             len -= 16;
> > +     }
> > +     if (len >= 8) {
> > +             *(uint64_t *)dst = *(const uint64_t *)src;
> > +             dst = (uint8_t *)dst + 8;
> > +             src = (uint8_t *)src + 8;
> > +             len -= 8;
> > +     }
> > +     if (len >= 4) {
> > +             *(uint32_t *)dst = *(const uint32_t *)src;
> > +             dst = (uint8_t *)dst + 4;
> > +             src = (uint8_t *)src + 4;
> > +             len -= 4;
> > +     }
> > +     if (len != 0) {
> > +             dst = (uint8_t *)dst - (4 - len);
> > +             src = (uint8_t *)src - (4 - len);
> > +             *(uint32_t *)dst = *(const uint32_t *)src;
> > +     }
> > +
> > +     return dest;
> > +}
> > +#endif
> > +
> >  /**
> >   * Build a table to translate Rx completion flags to packet type.
> >   *
> > @@ -1707,6 +1798,9 @@ mlx5_rx_burst_mprq(void *dpdk_rxq, struct
> > rte_mbuf **pkts, uint16_t pkts_n)
> >               int32_t hdrm_overlap;
> >               volatile struct mlx5_mini_cqe8 *mcqe = NULL;
> >               uint32_t rss_hash_res = 0;
> > +#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
> > +             uintptr_t data_addr;
> > +#endif
> >
> >               if (consumed_strd == strd_n) {
> >                       /* Replace WQE only if the buffer is still in use.
> */
> > @@ -1772,12 +1866,30 @@ mlx5_rx_burst_mprq(void *dpdk_rxq, struct
> > rte_mbuf **pkts, uint16_t pkts_n)
> >                * - Out of buffer in the Mempool for Multi-Packet RQ.
> >                * - The packet's stride overlaps a headroom and scatter is
> > off.
> >                */
> > +#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
> > +             if (unlikely(!rxq->mprq_tstore_memcpy) &&
> > +                     len <= rxq->mprq_max_memcpy_len) {
> > +                     rte_prefetch1(addr);
> > +                     if (len > RTE_CACHE_LINE_SIZE)
> > +                             rte_prefetch2((void *)((uintptr_t)addr +
> > RTE_CACHE_LINE_SIZE));
> > +             }
> > +#endif
> >               if (len <= rxq->mprq_max_memcpy_len ||
> >                   rxq->mprq_repl == NULL ||
> >                   (hdrm_overlap > 0 && !rxq->strd_scatter_en)) {
> >                       if (likely(rte_pktmbuf_tailroom(pkt) >= len)) {
> > -                             rte_memcpy(rte_pktmbuf_mtod(pkt, void *),
> > -                                        addr, len);
> > +#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
> > +                             data_addr =
> > (uintptr_t)rte_pktmbuf_mtod(pkt, void *);
> > +                             if (!(rxq->mprq_tstore_memcpy))
> > +                                     rte_memcpy((void *)data_addr,
> > addr, len);
> > +                             else if ((rxq->mprq_tstore_memcpy) &&
> > +                                        !((data_addr | (uintptr_t)addr)
> &
> > ALIGNMENT_MASK))
> > +
> >       memcpy_aligned_rx_tstore_16B((void *)data_addr,
> > +                                                     addr, len);
> > +                             else
> > +#endif
> > +                                     rte_memcpy(rte_pktmbuf_mtod(pkt,
> > void *),
> > +                                                     addr, len);
> >                               DATA_LEN(pkt) = len;
> >                       } else if (rxq->strd_scatter_en) {
> >                               struct rte_mbuf *prev = pkt;
> > diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
> > index 9ffa028d2..a8ea1a795 100644
> > --- a/drivers/net/mlx5/mlx5_rxtx.h
> > +++ b/drivers/net/mlx5/mlx5_rxtx.h
> > @@ -153,6 +153,9 @@ struct mlx5_rxq_data {
> >       uint32_t tunnel; /* Tunnel information. */
> >       uint64_t flow_meta_mask;
> >       int32_t flow_meta_offset;
> > +#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
> > +     unsigned int mprq_tstore_memcpy:1;
> > +#endif
> >  } __rte_cache_aligned;
> >
> >  enum mlx5_rxq_type {
> > diff --git a/meson_options.txt b/meson_options.txt index
> > 9bf18ab6b..a4bc565d2 100644
> > --- a/meson_options.txt
> > +++ b/meson_options.txt
> > @@ -30,6 +30,8 @@ option('max_lcores', type: 'integer', value: 128,
> >       description: 'maximum number of cores/threads supported by EAL')
> > option('max_numa_nodes', type: 'integer', value: 4,
> >       description: 'maximum number of NUMA nodes supported by EAL')
> > +option('mlx5_ntload_tstore', type: 'boolean', value: false,
> > +     description: 'to enable optimized MPRQ in RX datapath')
> >  option('enable_trace_fp', type: 'boolean', value: false,
> >       description: 'enable fast path trace points.')  option('tests',
> type:
> > 'boolean', value: true,
> > --
> > 2.25.1
>
>

^ permalink raw reply	[flat|nested] 5+ messages in thread

* Re: [dpdk-dev] [PATCH v3 1/2] net/mlx5: optimize mprq memcpy
  2020-10-10  9:00 ` [dpdk-dev] [PATCH v3 " Aman Kumar
@ 2021-02-04 14:14   ` Slava Ovsiienko
  2021-02-09  6:22     ` Aman Kumar
  0 siblings, 1 reply; 5+ messages in thread
From: Slava Ovsiienko @ 2021-02-04 14:14 UTC (permalink / raw)
  To: Aman Kumar, dev
  Cc: Raslan Darawsheh, keesang.song, Asaf Penso, Shy Shyman,
	Alexander Kozyrev, Matan Azrad

Hi,

I'm sorry for asking the questions very late.
Is still this patch set actual and should it be updated and considered? 

As I can understand this one optimizes the memory writes in some way using the instructions with the hints.
Is this specific for some CPU families? Is this more common? I suppose it should we considered and discussed
more widely, possible on EAL level. I would propose to introduce these special memory routines on EAL level
to give advantage to all PMDs, not specifically to mlx5.

With best regards,
Slava

> -----Original Message-----
> From: Aman Kumar <aman.kumar@vvdntech.in>
> Sent: Saturday, October 10, 2020 12:01
> To: dev@dpdk.org
> Cc: Raslan Darawsheh <rasland@nvidia.com>; keesang.song@amd.com;
> Asaf Penso <asafp@nvidia.com>; Shy Shyman <shys@nvidia.com>; Slava
> Ovsiienko <viacheslavo@nvidia.com>; Alexander Kozyrev
> <akozyrev@nvidia.com>; Matan Azrad <matan@nvidia.com>;
> aman.kumar@vvdntech.in
> Subject: [PATCH v3 1/2] net/mlx5: optimize mprq memcpy
> 
> add non temporal load and temporal store for mprq memcpy.
> define RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY in build
> configuration to enable this optimization.
> 
> Signed-off-by: Aman Kumar <aman.kumar@vvdntech.in>
> ---
>  drivers/net/mlx5/meson.build |   1 +
>  drivers/net/mlx5/mlx5.c      |  12 ++++
>  drivers/net/mlx5/mlx5.h      |   3 +
>  drivers/net/mlx5/mlx5_rxq.c  |   3 +
>  drivers/net/mlx5/mlx5_rxtx.c | 116
> ++++++++++++++++++++++++++++++++++-
>  drivers/net/mlx5/mlx5_rxtx.h |   3 +
>  meson_options.txt            |   2 +
>  7 files changed, 138 insertions(+), 2 deletions(-)
> 
> diff --git a/drivers/net/mlx5/meson.build b/drivers/net/mlx5/meson.build
> index 9a97bb9c8..38e93fdc1 100644
> --- a/drivers/net/mlx5/meson.build
> +++ b/drivers/net/mlx5/meson.build
> @@ -47,6 +47,7 @@ foreach option:cflags_options
>  		cflags += option
>  	endif
>  endforeach
> +dpdk_conf.set('RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY',
> +get_option('mlx5_ntload_tstore'))
>  if get_option('buildtype').contains('debug')
>  	cflags += [ '-pedantic', '-DPEDANTIC' ]  else diff --git
> a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index
> 01ead6e6a..a2796eaa5 100644
> --- a/drivers/net/mlx5/mlx5.c
> +++ b/drivers/net/mlx5/mlx5.c
> @@ -160,6 +160,11 @@
>  /* Configure timeout of LRO session (in microseconds). */  #define
> MLX5_LRO_TIMEOUT_USEC "lro_timeout_usec"
> 
> +#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
> +/* mprq_tstore_memcpy */
> +#define MLX5_MPRQ_TSTORE_MEMCPY "mprq_tstore_memcpy"
> +#endif
> +
>  /*
>   * Device parameter to configure the total data buffer size for a single
>   * hairpin queue (logarithm value).
> @@ -1623,6 +1628,10 @@ mlx5_args_check(const char *key, const char
> *val, void *opaque)
>  		config->sys_mem_en = !!tmp;
>  	} else if (strcmp(MLX5_DECAP_EN, key) == 0) {
>  		config->decap_en = !!tmp;
> +#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
> +	} else if (strcmp(MLX5_MPRQ_TSTORE_MEMCPY, key) == 0) {
> +		config->mprq_tstore_memcpy = tmp;
> +#endif
>  	} else {
>  		DRV_LOG(WARNING, "%s: unknown parameter", key);
>  		rte_errno = EINVAL;
> @@ -1683,6 +1692,9 @@ mlx5_args(struct mlx5_dev_config *config, struct
> rte_devargs *devargs)
>  		MLX5_RECLAIM_MEM,
>  		MLX5_SYS_MEM_EN,
>  		MLX5_DECAP_EN,
> +#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
> +		MLX5_MPRQ_TSTORE_MEMCPY,
> +#endif
>  		NULL,
>  	};
>  	struct rte_kvargs *kvlist;
> diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index
> 43da9a1fb..1eb305650 100644
> --- a/drivers/net/mlx5/mlx5.h
> +++ b/drivers/net/mlx5/mlx5.h
> @@ -234,6 +234,9 @@ struct mlx5_dev_config {
>  	int tx_skew; /* Tx scheduling skew between WQE and data on wire.
> */
>  	struct mlx5_hca_attr hca_attr; /* HCA attributes. */
>  	struct mlx5_lro_config lro; /* LRO configuration. */
> +#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
> +	unsigned int mprq_tstore_memcpy:1;
> +#endif
>  };
> 
> 
> diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c index
> c059e216d..c8db59a12 100644
> --- a/drivers/net/mlx5/mlx5_rxq.c
> +++ b/drivers/net/mlx5/mlx5_rxq.c
> @@ -1380,6 +1380,9 @@ mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t
> idx, uint16_t desc,
>  	tmpl->socket = socket;
>  	if (dev->data->dev_conf.intr_conf.rxq)
>  		tmpl->irq = 1;
> +#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
> +	tmpl->rxq.mprq_tstore_memcpy = config->mprq_tstore_memcpy;
> #endif
>  	mprq_stride_nums = config->mprq.stride_num_n ?
>  		config->mprq.stride_num_n : MLX5_MPRQ_STRIDE_NUM_N;
>  	mprq_stride_size = non_scatter_min_mbuf_size <= diff --git
> a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c index
> 0b87be15b..f59e30d82 100644
> --- a/drivers/net/mlx5/mlx5_rxtx.c
> +++ b/drivers/net/mlx5/mlx5_rxtx.c
> @@ -123,6 +123,97 @@ uint8_t mlx5_swp_types_table[1 << 10]
> __rte_cache_aligned;  uint64_t rte_net_mlx5_dynf_inline_mask;  #define
> PKT_TX_DYNF_NOINLINE rte_net_mlx5_dynf_inline_mask
> 
> +#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
> +static void copy16B_ts(void *dst, void *src) {
> +	__m128i var128;
> +
> +	var128 = _mm_stream_load_si128((__m128i *)src);
> +	_mm_storeu_si128((__m128i *)dst, var128); }
> +
> +static void copy32B_ts(void *dst, void *src) {
> +	__m256i ymm0;
> +
> +	ymm0 = _mm256_stream_load_si256((const __m256i *)src);
> +	_mm256_storeu_si256((__m256i *)dst, ymm0); }
> +
> +static void copy64B_ts(void *dst, void *src) {
> +	__m256i ymm0, ymm1;
> +
> +	ymm0 = _mm256_stream_load_si256((const __m256i *)src);
> +	ymm1 = _mm256_stream_load_si256((const __m256i *)((uint8_t
> *)src + 32));
> +	_mm256_storeu_si256((__m256i *)dst, ymm0);
> +	_mm256_storeu_si256((__m256i *)((uint8_t *)dst + 32), ymm1); }
> +
> +static void copy128B_ts(void *dst, void *src) {
> +	__m256i ymm0, ymm1, ymm2, ymm3;
> +
> +	ymm0 = _mm256_stream_load_si256((const __m256i *)src);
> +	ymm1 = _mm256_stream_load_si256((const __m256i *)((uint8_t
> *)src + 32));
> +	ymm2 = _mm256_stream_load_si256((const __m256i *)((uint8_t
> *)src + 64));
> +	ymm3 = _mm256_stream_load_si256((const __m256i *)((uint8_t
> *)src + 96));
> +	_mm256_storeu_si256((__m256i *)dst, ymm0);
> +	_mm256_storeu_si256((__m256i *)((uint8_t *)dst + 32), ymm1);
> +	_mm256_storeu_si256((__m256i *)((uint8_t *)dst + 64), ymm2);
> +	_mm256_storeu_si256((__m256i *)((uint8_t *)dst + 96), ymm3); }
> +
> +static void *memcpy_aligned_rx_tstore_16B(void *dst, void *src, int
> +len) {
> +	void *dest = dst;
> +
> +	while (len >= 128) {
> +		copy128B_ts(dst, src);
> +		dst = (uint8_t *)dst + 128;
> +		src = (uint8_t *)src + 128;
> +		len -= 128;
> +	}
> +	while (len >= 64) {
> +		copy64B_ts(dst, src);
> +		dst = (uint8_t *)dst + 64;
> +		src = (uint8_t *)src + 64;
> +		len -= 64;
> +	}
> +	while (len >= 32) {
> +		copy32B_ts(dst, src);
> +		dst = (uint8_t *)dst + 32;
> +		src = (uint8_t *)src + 32;
> +		len -= 32;
> +	}
> +	if (len >= 16) {
> +		copy16B_ts(dst, src);
> +		dst = (uint8_t *)dst + 16;
> +		src = (uint8_t *)src + 16;
> +		len -= 16;
> +	}
> +	if (len >= 8) {
> +		*(uint64_t *)dst = *(const uint64_t *)src;
> +		dst = (uint8_t *)dst + 8;
> +		src = (uint8_t *)src + 8;
> +		len -= 8;
> +	}
> +	if (len >= 4) {
> +		*(uint32_t *)dst = *(const uint32_t *)src;
> +		dst = (uint8_t *)dst + 4;
> +		src = (uint8_t *)src + 4;
> +		len -= 4;
> +	}
> +	if (len != 0) {
> +		dst = (uint8_t *)dst - (4 - len);
> +		src = (uint8_t *)src - (4 - len);
> +		*(uint32_t *)dst = *(const uint32_t *)src;
> +	}
> +
> +	return dest;
> +}
> +#endif
> +
>  /**
>   * Build a table to translate Rx completion flags to packet type.
>   *
> @@ -1707,6 +1798,9 @@ mlx5_rx_burst_mprq(void *dpdk_rxq, struct
> rte_mbuf **pkts, uint16_t pkts_n)
>  		int32_t hdrm_overlap;
>  		volatile struct mlx5_mini_cqe8 *mcqe = NULL;
>  		uint32_t rss_hash_res = 0;
> +#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
> +		uintptr_t data_addr;
> +#endif
> 
>  		if (consumed_strd == strd_n) {
>  			/* Replace WQE only if the buffer is still in use. */
> @@ -1772,12 +1866,30 @@ mlx5_rx_burst_mprq(void *dpdk_rxq, struct
> rte_mbuf **pkts, uint16_t pkts_n)
>  		 * - Out of buffer in the Mempool for Multi-Packet RQ.
>  		 * - The packet's stride overlaps a headroom and scatter is
> off.
>  		 */
> +#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
> +		if (unlikely(!rxq->mprq_tstore_memcpy) &&
> +			len <= rxq->mprq_max_memcpy_len) {
> +			rte_prefetch1(addr);
> +			if (len > RTE_CACHE_LINE_SIZE)
> +				rte_prefetch2((void *)((uintptr_t)addr +
> RTE_CACHE_LINE_SIZE));
> +		}
> +#endif
>  		if (len <= rxq->mprq_max_memcpy_len ||
>  		    rxq->mprq_repl == NULL ||
>  		    (hdrm_overlap > 0 && !rxq->strd_scatter_en)) {
>  			if (likely(rte_pktmbuf_tailroom(pkt) >= len)) {
> -				rte_memcpy(rte_pktmbuf_mtod(pkt, void *),
> -					   addr, len);
> +#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
> +				data_addr =
> (uintptr_t)rte_pktmbuf_mtod(pkt, void *);
> +				if (!(rxq->mprq_tstore_memcpy))
> +					rte_memcpy((void *)data_addr,
> addr, len);
> +				else if ((rxq->mprq_tstore_memcpy) &&
> +					   !((data_addr | (uintptr_t)addr) &
> ALIGNMENT_MASK))
> +
> 	memcpy_aligned_rx_tstore_16B((void *)data_addr,
> +							addr, len);
> +				else
> +#endif
> +					rte_memcpy(rte_pktmbuf_mtod(pkt,
> void *),
> +							addr, len);
>  				DATA_LEN(pkt) = len;
>  			} else if (rxq->strd_scatter_en) {
>  				struct rte_mbuf *prev = pkt;
> diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
> index 9ffa028d2..a8ea1a795 100644
> --- a/drivers/net/mlx5/mlx5_rxtx.h
> +++ b/drivers/net/mlx5/mlx5_rxtx.h
> @@ -153,6 +153,9 @@ struct mlx5_rxq_data {
>  	uint32_t tunnel; /* Tunnel information. */
>  	uint64_t flow_meta_mask;
>  	int32_t flow_meta_offset;
> +#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
> +	unsigned int mprq_tstore_memcpy:1;
> +#endif
>  } __rte_cache_aligned;
> 
>  enum mlx5_rxq_type {
> diff --git a/meson_options.txt b/meson_options.txt index
> 9bf18ab6b..a4bc565d2 100644
> --- a/meson_options.txt
> +++ b/meson_options.txt
> @@ -30,6 +30,8 @@ option('max_lcores', type: 'integer', value: 128,
>  	description: 'maximum number of cores/threads supported by EAL')
> option('max_numa_nodes', type: 'integer', value: 4,
>  	description: 'maximum number of NUMA nodes supported by EAL')
> +option('mlx5_ntload_tstore', type: 'boolean', value: false,
> +	description: 'to enable optimized MPRQ in RX datapath')
>  option('enable_trace_fp', type: 'boolean', value: false,
>  	description: 'enable fast path trace points.')  option('tests', type:
> 'boolean', value: true,
> --
> 2.25.1


^ permalink raw reply	[flat|nested] 5+ messages in thread

* [dpdk-dev] [PATCH v3 1/2] net/mlx5: optimize mprq memcpy
  2020-09-25  3:16 [dpdk-dev] [PATCH v2 1/2] net/mlx5: optimize mprq memcpy Aman Kumar
@ 2020-10-10  9:00 ` Aman Kumar
  2021-02-04 14:14   ` Slava Ovsiienko
  0 siblings, 1 reply; 5+ messages in thread
From: Aman Kumar @ 2020-10-10  9:00 UTC (permalink / raw)
  To: dev
  Cc: rasland, keesang.song, asafp, shys, viacheslavo, akozyrev, matan,
	aman.kumar

add non temporal load and temporal store for mprq memcpy.
define RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY in build
configuration to enable this optimization.

Signed-off-by: Aman Kumar <aman.kumar@vvdntech.in>
---
 drivers/net/mlx5/meson.build |   1 +
 drivers/net/mlx5/mlx5.c      |  12 ++++
 drivers/net/mlx5/mlx5.h      |   3 +
 drivers/net/mlx5/mlx5_rxq.c  |   3 +
 drivers/net/mlx5/mlx5_rxtx.c | 116 ++++++++++++++++++++++++++++++++++-
 drivers/net/mlx5/mlx5_rxtx.h |   3 +
 meson_options.txt            |   2 +
 7 files changed, 138 insertions(+), 2 deletions(-)

diff --git a/drivers/net/mlx5/meson.build b/drivers/net/mlx5/meson.build
index 9a97bb9c8..38e93fdc1 100644
--- a/drivers/net/mlx5/meson.build
+++ b/drivers/net/mlx5/meson.build
@@ -47,6 +47,7 @@ foreach option:cflags_options
 		cflags += option
 	endif
 endforeach
+dpdk_conf.set('RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY', get_option('mlx5_ntload_tstore'))
 if get_option('buildtype').contains('debug')
 	cflags += [ '-pedantic', '-DPEDANTIC' ]
 else
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 01ead6e6a..a2796eaa5 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -160,6 +160,11 @@
 /* Configure timeout of LRO session (in microseconds). */
 #define MLX5_LRO_TIMEOUT_USEC "lro_timeout_usec"
 
+#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
+/* mprq_tstore_memcpy */
+#define MLX5_MPRQ_TSTORE_MEMCPY "mprq_tstore_memcpy"
+#endif
+
 /*
  * Device parameter to configure the total data buffer size for a single
  * hairpin queue (logarithm value).
@@ -1623,6 +1628,10 @@ mlx5_args_check(const char *key, const char *val, void *opaque)
 		config->sys_mem_en = !!tmp;
 	} else if (strcmp(MLX5_DECAP_EN, key) == 0) {
 		config->decap_en = !!tmp;
+#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
+	} else if (strcmp(MLX5_MPRQ_TSTORE_MEMCPY, key) == 0) {
+		config->mprq_tstore_memcpy = tmp;
+#endif
 	} else {
 		DRV_LOG(WARNING, "%s: unknown parameter", key);
 		rte_errno = EINVAL;
@@ -1683,6 +1692,9 @@ mlx5_args(struct mlx5_dev_config *config, struct rte_devargs *devargs)
 		MLX5_RECLAIM_MEM,
 		MLX5_SYS_MEM_EN,
 		MLX5_DECAP_EN,
+#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
+		MLX5_MPRQ_TSTORE_MEMCPY,
+#endif
 		NULL,
 	};
 	struct rte_kvargs *kvlist;
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 43da9a1fb..1eb305650 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -234,6 +234,9 @@ struct mlx5_dev_config {
 	int tx_skew; /* Tx scheduling skew between WQE and data on wire. */
 	struct mlx5_hca_attr hca_attr; /* HCA attributes. */
 	struct mlx5_lro_config lro; /* LRO configuration. */
+#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
+	unsigned int mprq_tstore_memcpy:1;
+#endif
 };
 
 
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index c059e216d..c8db59a12 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -1380,6 +1380,9 @@ mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 	tmpl->socket = socket;
 	if (dev->data->dev_conf.intr_conf.rxq)
 		tmpl->irq = 1;
+#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
+	tmpl->rxq.mprq_tstore_memcpy = config->mprq_tstore_memcpy;
+#endif
 	mprq_stride_nums = config->mprq.stride_num_n ?
 		config->mprq.stride_num_n : MLX5_MPRQ_STRIDE_NUM_N;
 	mprq_stride_size = non_scatter_min_mbuf_size <=
diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index 0b87be15b..f59e30d82 100644
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -123,6 +123,97 @@ uint8_t mlx5_swp_types_table[1 << 10] __rte_cache_aligned;
 uint64_t rte_net_mlx5_dynf_inline_mask;
 #define PKT_TX_DYNF_NOINLINE rte_net_mlx5_dynf_inline_mask
 
+#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
+static void copy16B_ts(void *dst, void *src)
+{
+	__m128i var128;
+
+	var128 = _mm_stream_load_si128((__m128i *)src);
+	_mm_storeu_si128((__m128i *)dst, var128);
+}
+
+static void copy32B_ts(void *dst, void *src)
+{
+	__m256i ymm0;
+
+	ymm0 = _mm256_stream_load_si256((const __m256i *)src);
+	_mm256_storeu_si256((__m256i *)dst, ymm0);
+}
+
+static void copy64B_ts(void *dst, void *src)
+{
+	__m256i ymm0, ymm1;
+
+	ymm0 = _mm256_stream_load_si256((const __m256i *)src);
+	ymm1 = _mm256_stream_load_si256((const __m256i *)((uint8_t *)src + 32));
+	_mm256_storeu_si256((__m256i *)dst, ymm0);
+	_mm256_storeu_si256((__m256i *)((uint8_t *)dst + 32), ymm1);
+}
+
+static void copy128B_ts(void *dst, void *src)
+{
+	__m256i ymm0, ymm1, ymm2, ymm3;
+
+	ymm0 = _mm256_stream_load_si256((const __m256i *)src);
+	ymm1 = _mm256_stream_load_si256((const __m256i *)((uint8_t *)src + 32));
+	ymm2 = _mm256_stream_load_si256((const __m256i *)((uint8_t *)src + 64));
+	ymm3 = _mm256_stream_load_si256((const __m256i *)((uint8_t *)src + 96));
+	_mm256_storeu_si256((__m256i *)dst, ymm0);
+	_mm256_storeu_si256((__m256i *)((uint8_t *)dst + 32), ymm1);
+	_mm256_storeu_si256((__m256i *)((uint8_t *)dst + 64), ymm2);
+	_mm256_storeu_si256((__m256i *)((uint8_t *)dst + 96), ymm3);
+}
+
+static void *memcpy_aligned_rx_tstore_16B(void *dst, void *src, int len)
+{
+	void *dest = dst;
+
+	while (len >= 128) {
+		copy128B_ts(dst, src);
+		dst = (uint8_t *)dst + 128;
+		src = (uint8_t *)src + 128;
+		len -= 128;
+	}
+	while (len >= 64) {
+		copy64B_ts(dst, src);
+		dst = (uint8_t *)dst + 64;
+		src = (uint8_t *)src + 64;
+		len -= 64;
+	}
+	while (len >= 32) {
+		copy32B_ts(dst, src);
+		dst = (uint8_t *)dst + 32;
+		src = (uint8_t *)src + 32;
+		len -= 32;
+	}
+	if (len >= 16) {
+		copy16B_ts(dst, src);
+		dst = (uint8_t *)dst + 16;
+		src = (uint8_t *)src + 16;
+		len -= 16;
+	}
+	if (len >= 8) {
+		*(uint64_t *)dst = *(const uint64_t *)src;
+		dst = (uint8_t *)dst + 8;
+		src = (uint8_t *)src + 8;
+		len -= 8;
+	}
+	if (len >= 4) {
+		*(uint32_t *)dst = *(const uint32_t *)src;
+		dst = (uint8_t *)dst + 4;
+		src = (uint8_t *)src + 4;
+		len -= 4;
+	}
+	if (len != 0) {
+		dst = (uint8_t *)dst - (4 - len);
+		src = (uint8_t *)src - (4 - len);
+		*(uint32_t *)dst = *(const uint32_t *)src;
+	}
+
+	return dest;
+}
+#endif
+
 /**
  * Build a table to translate Rx completion flags to packet type.
  *
@@ -1707,6 +1798,9 @@ mlx5_rx_burst_mprq(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		int32_t hdrm_overlap;
 		volatile struct mlx5_mini_cqe8 *mcqe = NULL;
 		uint32_t rss_hash_res = 0;
+#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
+		uintptr_t data_addr;
+#endif
 
 		if (consumed_strd == strd_n) {
 			/* Replace WQE only if the buffer is still in use. */
@@ -1772,12 +1866,30 @@ mlx5_rx_burst_mprq(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
 		 * - Out of buffer in the Mempool for Multi-Packet RQ.
 		 * - The packet's stride overlaps a headroom and scatter is off.
 		 */
+#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
+		if (unlikely(!rxq->mprq_tstore_memcpy) &&
+			len <= rxq->mprq_max_memcpy_len) {
+			rte_prefetch1(addr);
+			if (len > RTE_CACHE_LINE_SIZE)
+				rte_prefetch2((void *)((uintptr_t)addr + RTE_CACHE_LINE_SIZE));
+		}
+#endif
 		if (len <= rxq->mprq_max_memcpy_len ||
 		    rxq->mprq_repl == NULL ||
 		    (hdrm_overlap > 0 && !rxq->strd_scatter_en)) {
 			if (likely(rte_pktmbuf_tailroom(pkt) >= len)) {
-				rte_memcpy(rte_pktmbuf_mtod(pkt, void *),
-					   addr, len);
+#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
+				data_addr = (uintptr_t)rte_pktmbuf_mtod(pkt, void *);
+				if (!(rxq->mprq_tstore_memcpy))
+					rte_memcpy((void *)data_addr, addr, len);
+				else if ((rxq->mprq_tstore_memcpy) &&
+					   !((data_addr | (uintptr_t)addr) & ALIGNMENT_MASK))
+					memcpy_aligned_rx_tstore_16B((void *)data_addr,
+							addr, len);
+				else
+#endif
+					rte_memcpy(rte_pktmbuf_mtod(pkt, void *),
+							addr, len);
 				DATA_LEN(pkt) = len;
 			} else if (rxq->strd_scatter_en) {
 				struct rte_mbuf *prev = pkt;
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index 9ffa028d2..a8ea1a795 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -153,6 +153,9 @@ struct mlx5_rxq_data {
 	uint32_t tunnel; /* Tunnel information. */
 	uint64_t flow_meta_mask;
 	int32_t flow_meta_offset;
+#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
+	unsigned int mprq_tstore_memcpy:1;
+#endif
 } __rte_cache_aligned;
 
 enum mlx5_rxq_type {
diff --git a/meson_options.txt b/meson_options.txt
index 9bf18ab6b..a4bc565d2 100644
--- a/meson_options.txt
+++ b/meson_options.txt
@@ -30,6 +30,8 @@ option('max_lcores', type: 'integer', value: 128,
 	description: 'maximum number of cores/threads supported by EAL')
 option('max_numa_nodes', type: 'integer', value: 4,
 	description: 'maximum number of NUMA nodes supported by EAL')
+option('mlx5_ntload_tstore', type: 'boolean', value: false,
+	description: 'to enable optimized MPRQ in RX datapath')
 option('enable_trace_fp', type: 'boolean', value: false,
 	description: 'enable fast path trace points.')
 option('tests', type: 'boolean', value: true,
-- 
2.25.1


^ permalink raw reply	[flat|nested] 5+ messages in thread

end of thread, other threads:[~2021-02-09  6:22 UTC | newest]

Thread overview: 5+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2020-10-10  8:57 [dpdk-dev] [PATCH v3 1/2] net/mlx5: optimize mprq memcpy Aman Kumar
2020-10-10  8:57 ` [dpdk-dev] [PATCH v3 2/2] net/mlx5: add non temporal store for WQE fields Aman Kumar
  -- strict thread matches above, loose matches on Subject: below --
2020-09-25  3:16 [dpdk-dev] [PATCH v2 1/2] net/mlx5: optimize mprq memcpy Aman Kumar
2020-10-10  9:00 ` [dpdk-dev] [PATCH v3 " Aman Kumar
2021-02-04 14:14   ` Slava Ovsiienko
2021-02-09  6:22     ` Aman Kumar

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).