DPDK patches and discussions
 help / color / mirror / Atom feed
From: Aman Kumar <aman.kumar@vvdntech.in>
To: dev@dpdk.org
Cc: rasland@mellanox.com, keesang.song@amd.com, aman.kumar@vvdntech.in
Subject: [dpdk-dev] [PATCH v2 2/2] net/mlx5: add non temporal store for WQE fields
Date: Fri, 25 Sep 2020 08:46:58 +0530
Message-ID: <20200925031658.50476-2-aman.kumar@vvdntech.in> (raw)
In-Reply-To: <20200925031658.50476-1-aman.kumar@vvdntech.in>

add non temporal store for few WQE fields to optimize
data path. Define RTE_LIBRTE_MLX5_NT_STORE in build
configurations to enable this optimization.

Signed-off-by: Aman Kumar <aman.kumar@vvdntech.in>
---
 drivers/net/mlx5/meson.build     |  1 +
 drivers/net/mlx5/mlx5.c          | 17 +++++++++++++++++
 drivers/net/mlx5/mlx5.h          |  4 ++++
 drivers/net/mlx5/mlx5_rxq.c      |  3 +++
 drivers/net/mlx5/mlx5_rxtx.c     | 20 +++++++++++++++++---
 drivers/net/mlx5/mlx5_rxtx.h     |  6 ++++++
 drivers/net/mlx5/mlx5_rxtx_vec.h | 29 ++++++++++++++++++++++++-----
 drivers/net/mlx5/mlx5_txq.c      |  3 +++
 meson_options.txt                |  2 ++
 9 files changed, 77 insertions(+), 8 deletions(-)

diff --git a/drivers/net/mlx5/meson.build b/drivers/net/mlx5/meson.build
index 38e93fdc1..347ca6527 100644
--- a/drivers/net/mlx5/meson.build
+++ b/drivers/net/mlx5/meson.build
@@ -48,6 +48,7 @@ foreach option:cflags_options
 	endif
 endforeach
 dpdk_conf.set('RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY', get_option('mlx5_ntload_tstore'))
+dpdk_conf.set('RTE_LIBRTE_MLX5_NT_STORE', get_option('mlx5_ntstore'))
 if get_option('buildtype').contains('debug')
 	cflags += [ '-pedantic', '-DPEDANTIC' ]
 else
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 0bb1194f7..af72f38a1 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -164,6 +164,13 @@
 /* mprq_tstore_memcpy */
 #define MLX5_MPRQ_TSTORE_MEMCPY "mprq_tstore_memcpy"
 #endif
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+/* tx_wqe_field_ntstore */
+#define MLX5_TX_WQE_FIELD_NTSTORE "tx_wqe_field_ntstore"
+
+/* vec_rx_wqe_field_ntstore */
+#define MLX5_VEC_RX_WQE_FIELD_NTSTORE "vec_rx_wqe_field_ntstore"
+#endif
 
 /*
  * Device parameter to configure the total data buffer size for a single
@@ -1630,6 +1637,12 @@ mlx5_args_check(const char *key, const char *val, void *opaque)
 #ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
 	} else if (strcmp(MLX5_MPRQ_TSTORE_MEMCPY, key) == 0) {
 		config->mprq_tstore_memcpy = tmp;
+#endif
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+	} else if (strcmp(MLX5_TX_WQE_FIELD_NTSTORE, key) == 0) {
+		config->tx_wqe_field_ntstore = tmp;
+	} else if (strcmp(MLX5_VEC_RX_WQE_FIELD_NTSTORE, key) == 0) {
+		config->vec_rx_wqe_field_ntstore = tmp;
 #endif
 	} else {
 		DRV_LOG(WARNING, "%s: unknown parameter", key);
@@ -1693,6 +1706,10 @@ mlx5_args(struct mlx5_dev_config *config, struct rte_devargs *devargs)
 		MLX5_DECAP_EN,
 #ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
 		MLX5_MPRQ_TSTORE_MEMCPY,
+#endif
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+		MLX5_TX_WQE_FIELD_NTSTORE,
+		MLX5_VEC_RX_WQE_FIELD_NTSTORE,
 #endif
 		NULL,
 	};
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 0f0165884..a941e9198 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -237,6 +237,10 @@ struct mlx5_dev_config {
 #ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
 	unsigned int mprq_tstore_memcpy:1;
 #endif
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+	unsigned int tx_wqe_field_ntstore:1;
+	unsigned int vec_rx_wqe_field_ntstore:1;
+#endif
 };
 
 
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index 2c7090c54..be67a087c 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -1382,6 +1382,9 @@ mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 		tmpl->irq = 1;
 #ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
 	tmpl->rxq.mprq_tstore_memcpy = config->mprq_tstore_memcpy;
+#endif
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+	tmpl->rxq.vec_rx_wqe_field_ntstore = config->vec_rx_wqe_field_ntstore;
 #endif
 	mprq_stride_nums = config->mprq.stride_num_n ?
 		config->mprq.stride_num_n : MLX5_MPRQ_STRIDE_NUM_N;
diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index 9b4fa9a27..99ce20871 100644
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -2410,6 +2410,9 @@ mlx5_tx_request_completion(struct mlx5_txq_data *__rte_restrict txq,
 {
 	uint16_t head = txq->elts_head;
 	unsigned int part;
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+	register uint32_t flags;
+#endif
 
 	part = MLX5_TXOFF_CONFIG(INLINE) ?
 	       0 : loc->pkts_sent - loc->pkts_copy;
@@ -2423,9 +2426,20 @@ mlx5_tx_request_completion(struct mlx5_txq_data *__rte_restrict txq,
 		txq->elts_comp = head;
 		if (MLX5_TXOFF_CONFIG(INLINE))
 			txq->wqe_comp = txq->wqe_ci;
-		/* Request unconditional completion on last WQE. */
-		last->cseg.flags = RTE_BE32(MLX5_COMP_ALWAYS <<
-					    MLX5_COMP_MODE_OFFSET);
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+		if (txq->tx_wqe_field_ntstore) {
+			flags = RTE_BE32(MLX5_COMP_ALWAYS <<
+					MLX5_COMP_MODE_OFFSET);
+			_mm_stream_si32(((void *)(uintptr_t)&last->cseg.flags),
+					flags);
+		} else {
+#endif
+			/* Request unconditional completion on last WQE. */
+			last->cseg.flags = RTE_BE32(MLX5_COMP_ALWAYS <<
+					MLX5_COMP_MODE_OFFSET);
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+		}
+#endif
 		/* Save elts_head in dedicated free on completion queue. */
 #ifdef RTE_LIBRTE_MLX5_DEBUG
 		txq->fcqs[txq->cq_pi++ & txq->cqe_m] = head |
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index b3c259774..86e39b91a 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -156,6 +156,9 @@ struct mlx5_rxq_data {
 #ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
 	unsigned int mprq_tstore_memcpy:1;
 #endif
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+	unsigned int vec_rx_wqe_field_ntstore:1;
+#endif
 } __rte_cache_aligned;
 
 enum mlx5_rxq_type {
@@ -256,6 +259,9 @@ struct mlx5_txq_data {
 	int32_t ts_offset; /* Timestamp field dynamic offset. */
 	struct mlx5_dev_ctx_shared *sh; /* Shared context. */
 	struct mlx5_txq_stats stats; /* TX queue counters. */
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+	unsigned int tx_wqe_field_ntstore:1;
+#endif
 #ifndef RTE_ARCH_64
 	rte_spinlock_t *uar_lock;
 	/* UAR access lock required for 32bit implementations */
diff --git a/drivers/net/mlx5/mlx5_rxtx_vec.h b/drivers/net/mlx5/mlx5_rxtx_vec.h
index a8d6c4f41..413f863ba 100644
--- a/drivers/net/mlx5/mlx5_rxtx_vec.h
+++ b/drivers/net/mlx5/mlx5_rxtx_vec.h
@@ -86,6 +86,10 @@ mlx5_rx_replenish_bulk_mbuf(struct mlx5_rxq_data *rxq, uint16_t n)
 	volatile struct mlx5_wqe_data_seg *wq =
 		&((volatile struct mlx5_wqe_data_seg *)rxq->wqes)[elts_idx];
 	unsigned int i;
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+	register uint64_t buf_addr2;
+	register uint32_t lkey_t;
+#endif
 
 	MLX5_ASSERT(n >= MLX5_VPMD_RXQ_RPLNSH_THRESH(q_n));
 	MLX5_ASSERT(n <= (uint16_t)(q_n - (rxq->rq_ci - rxq->rq_pi)));
@@ -107,11 +111,26 @@ mlx5_rx_replenish_bulk_mbuf(struct mlx5_rxq_data *rxq, uint16_t n)
 		 * impact the performance.
 		 */
 		buf_addr = elts[i]->buf_addr;
-		wq[i].addr = rte_cpu_to_be_64((uintptr_t)buf_addr +
-					      RTE_PKTMBUF_HEADROOM);
-		/* If there's only one MR, no need to replace LKey in WQE. */
-		if (unlikely(mlx5_mr_btree_len(&rxq->mr_ctrl.cache_bh) > 1))
-			wq[i].lkey = mlx5_rx_mb2mr(rxq, elts[i]);
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+		if (rxq->vec_rx_wqe_field_ntstore) {
+			buf_addr2 = (uint64_t)rte_cpu_to_be_64((uintptr_t)buf_addr +
+							       RTE_PKTMBUF_HEADROOM);
+			_mm_stream_si64(((void *)(uintptr_t)&wq[i].addr), buf_addr2);
+			/* If there's only one MR, no need to replace LKey in WQE. */
+			if (unlikely(mlx5_mr_btree_len(&rxq->mr_ctrl.cache_bh) > 1)) {
+				lkey_t = (uint32_t)mlx5_rx_mb2mr(rxq, elts[i]);
+				_mm_stream_si32(((void *)(uintptr_t)&wq[i].lkey), lkey_t);
+			}
+		} else {
+#endif
+			wq[i].addr = rte_cpu_to_be_64((uintptr_t)buf_addr +
+					RTE_PKTMBUF_HEADROOM);
+			/* If there's only one MR, no need to replace LKey in WQE. */
+			if (unlikely(mlx5_mr_btree_len(&rxq->mr_ctrl.cache_bh) > 1))
+				wq[i].lkey = mlx5_rx_mb2mr(rxq, elts[i]);
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+		}
+#endif
 	}
 	rxq->rq_ci += n;
 	/* Prevent overflowing into consumed mbufs. */
diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
index 1bb667d46..cba675f53 100644
--- a/drivers/net/mlx5/mlx5_txq.c
+++ b/drivers/net/mlx5/mlx5_txq.c
@@ -1565,6 +1565,9 @@ txq_set_params(struct mlx5_txq_ctrl *txq_ctrl)
 					    DEV_TX_OFFLOAD_UDP_TNL_TSO);
 	bool vlan_inline;
 	unsigned int temp;
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+	txq_ctrl->txq.tx_wqe_field_ntstore = config->tx_wqe_field_ntstore;
+#endif
 
 	if (config->txqs_inline == MLX5_ARG_UNSET)
 		txqs_inline =
diff --git a/meson_options.txt b/meson_options.txt
index a4bc565d2..21c31d57b 100644
--- a/meson_options.txt
+++ b/meson_options.txt
@@ -32,6 +32,8 @@ option('max_numa_nodes', type: 'integer', value: 4,
 	description: 'maximum number of NUMA nodes supported by EAL')
 option('mlx5_ntload_tstore', type: 'boolean', value: false,
 	description: 'to enable optimized MPRQ in RX datapath')
+option('mlx5_ntstore', type: 'boolean', value: false,
+	description: 'to enable optimized MLX5 TX datapath')
 option('enable_trace_fp', type: 'boolean', value: false,
 	description: 'enable fast path trace points.')
 option('tests', type: 'boolean', value: true,
-- 
2.25.1


  reply	other threads:[~2020-09-25  3:17 UTC|newest]

Thread overview: 12+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2020-09-07 19:32 [dpdk-dev] [PATCH 1/3] net/mlx5: optimize mprq memcpy Aman Kumar
2020-09-07 19:32 ` [dpdk-dev] [PATCH 2/3] net/mlx5: add non temporal store for WQE fields Aman Kumar
2020-09-07 19:32 ` [dpdk-dev] [PATCH 3/3] config: added build config file for AMD EPYC platform Aman Kumar
2020-09-08  9:11   ` David Marchand
2020-09-25  3:16 ` [dpdk-dev] [PATCH v2 1/2] net/mlx5: optimize mprq memcpy Aman Kumar
2020-09-25  3:16   ` Aman Kumar [this message]
2020-10-10  9:00   ` [dpdk-dev] [PATCH v3 " Aman Kumar
2020-10-10  9:00     ` [dpdk-dev] [PATCH v3 2/2] net/mlx5: add non temporal store for WQE fields Aman Kumar
2020-10-29  7:59     ` [dpdk-dev] [PATCH v4 1/2] net/mlx5: optimize mprq memcpy Aman Kumar
2020-10-29  7:59       ` [dpdk-dev] [PATCH v4 2/2] net/mlx5: add non temporal store for WQE fields Aman Kumar
2021-02-04 14:14     ` [dpdk-dev] [PATCH v3 1/2] net/mlx5: optimize mprq memcpy Slava Ovsiienko
2021-02-09  6:22       ` Aman Kumar

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20200925031658.50476-2-aman.kumar@vvdntech.in \
    --to=aman.kumar@vvdntech.in \
    --cc=dev@dpdk.org \
    --cc=keesang.song@amd.com \
    --cc=rasland@mellanox.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

DPDK patches and discussions

This inbox may be cloned and mirrored by anyone:

	git clone --mirror https://inbox.dpdk.org/dev/0 dev/git/0.git

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V2 dev dev/ https://inbox.dpdk.org/dev \
		dev@dpdk.org
	public-inbox-index dev

Example config snippet for mirrors.
Newsgroup available over NNTP:
	nntp://inbox.dpdk.org/inbox.dpdk.dev


AGPL code for this site: git clone https://public-inbox.org/public-inbox.git