From: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
To: dev@dpdk.org
Cc: matan@mellanox.com, rasland@mellanox.com, orika@mellanox.com
Subject: [dpdk-dev] [PATCH 4/4] net/mlx5: allow use allocated mbuf with external buffer
Date: Fri, 10 Jan 2020 17:57:02 +0000 [thread overview]
Message-ID: <1578679022-25344-5-git-send-email-viacheslavo@mellanox.com> (raw)
In-Reply-To: <1578679022-25344-1-git-send-email-viacheslavo@mellanox.com>
In the Rx datapath the flags in the newly allocated mbufs
are all explicitly cleared but the EXT_ATTACHED_MBUF must be
preserved. It would allow to use mbuf pools with pre-attached
external data buffers.
The vectorized rx_burst routines are updated in order to
inherit the EXT_ATTACHED_MBUF from mbuf pool private
RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF flag.
Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
---
drivers/net/mlx5/mlx5_rxq.c | 7 ++++++-
drivers/net/mlx5/mlx5_rxtx.c | 2 +-
drivers/net/mlx5/mlx5_rxtx.h | 2 +-
drivers/net/mlx5/mlx5_rxtx_vec.h | 14 ++++----------
drivers/net/mlx5/mlx5_rxtx_vec_altivec.h | 5 ++---
drivers/net/mlx5/mlx5_rxtx_vec_neon.h | 29 +++++++++++++++--------------
drivers/net/mlx5/mlx5_rxtx_vec_sse.h | 2 +-
7 files changed, 30 insertions(+), 31 deletions(-)
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index ca25e32..c87ce15 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -225,6 +225,9 @@
if (mlx5_rxq_check_vec_support(&rxq_ctrl->rxq) > 0) {
struct mlx5_rxq_data *rxq = &rxq_ctrl->rxq;
struct rte_mbuf *mbuf_init = &rxq->fake_mbuf;
+ struct rte_pktmbuf_pool_private *priv =
+ (struct rte_pktmbuf_pool_private *)
+ rte_mempool_get_priv(rxq_ctrl->rxq.mp);
int j;
/* Initialize default rearm_data for vPMD. */
@@ -232,13 +235,15 @@
rte_mbuf_refcnt_set(mbuf_init, 1);
mbuf_init->nb_segs = 1;
mbuf_init->port = rxq->port_id;
+ if (priv->flags & RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF)
+ mbuf_init->ol_flags = EXT_ATTACHED_MBUF;
/*
* prevent compiler reordering:
* rearm_data covers previous fields.
*/
rte_compiler_barrier();
rxq->mbuf_initializer =
- *(uint64_t *)&mbuf_init->rearm_data;
+ *(rte_xmm_t *)&mbuf_init->rearm_data;
/* Padding with a fake mbuf for vectorized Rx. */
for (j = 0; j < MLX5_VPMD_DESCS_PER_LOOP; ++j)
(*rxq->elts)[elts_n + j] = &rxq->fake_mbuf;
diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index 25a2952..e5a885d 100644
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -1341,7 +1341,7 @@ enum mlx5_txcmp_code {
}
pkt = seg;
assert(len >= (rxq->crc_present << 2));
- pkt->ol_flags = 0;
+ pkt->ol_flags &= EXT_ATTACHED_MBUF;
/* If compressed, take hash result from mini-CQE. */
rss_hash_res = rte_be_to_cpu_32(mcqe == NULL ?
cqe->rx_hash_res :
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index e927343..f35cc87 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -144,7 +144,7 @@ struct mlx5_rxq_data {
struct mlx5_mprq_buf *mprq_repl; /* Stashed mbuf for replenish. */
uint16_t idx; /* Queue index. */
struct mlx5_rxq_stats stats;
- uint64_t mbuf_initializer; /* Default rearm_data for vectorized Rx. */
+ rte_xmm_t mbuf_initializer; /* Default rearm/flags for vectorized Rx. */
struct rte_mbuf fake_mbuf; /* elts padding for vectorized Rx. */
void *cq_uar; /* CQ user access region. */
uint32_t cqn; /* CQ number. */
diff --git a/drivers/net/mlx5/mlx5_rxtx_vec.h b/drivers/net/mlx5/mlx5_rxtx_vec.h
index 85e0bd5..d8c07f2 100644
--- a/drivers/net/mlx5/mlx5_rxtx_vec.h
+++ b/drivers/net/mlx5/mlx5_rxtx_vec.h
@@ -97,18 +97,12 @@
void *buf_addr;
/*
- * Load the virtual address for Rx WQE. non-x86 processors
- * (mostly RISC such as ARM and Power) are more vulnerable to
- * load stall. For x86, reducing the number of instructions
- * seems to matter most.
+ * In order to support the mbufs with external attached
+ * data buffer we should use the buf_addr pointer instead of
+ * rte_mbuf_buf_addr(). It touches the mbuf itself and may
+ * impact the performance.
*/
-#ifdef RTE_ARCH_X86_64
buf_addr = elts[i]->buf_addr;
- assert(buf_addr == rte_mbuf_buf_addr(elts[i], rxq->mp));
-#else
- buf_addr = rte_mbuf_buf_addr(elts[i], rxq->mp);
- assert(buf_addr == elts[i]->buf_addr);
-#endif
wq[i].addr = rte_cpu_to_be_64((uintptr_t)buf_addr +
RTE_PKTMBUF_HEADROOM);
/* If there's only one MR, no need to replace LKey in WQE. */
diff --git a/drivers/net/mlx5/mlx5_rxtx_vec_altivec.h b/drivers/net/mlx5/mlx5_rxtx_vec_altivec.h
index 8e79883..6d4ddb5 100644
--- a/drivers/net/mlx5/mlx5_rxtx_vec_altivec.h
+++ b/drivers/net/mlx5/mlx5_rxtx_vec_altivec.h
@@ -344,9 +344,8 @@
PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD |
PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED};
const vector unsigned char mbuf_init =
- (vector unsigned char)(vector unsigned long){
- *(__attribute__((__aligned__(8))) unsigned long *)
- &rxq->mbuf_initializer, 0LL};
+ (vector unsigned char)(vector unsigned double) {
+ rxq->mbuf_initializer};
const vector unsigned short rearm_sel_mask =
(vector unsigned short){0, 0, 0, 0, 0xffff, 0xffff, 0, 0};
vector unsigned char rearm0, rearm1, rearm2, rearm3;
diff --git a/drivers/net/mlx5/mlx5_rxtx_vec_neon.h b/drivers/net/mlx5/mlx5_rxtx_vec_neon.h
index 86785c7..332e9ac 100644
--- a/drivers/net/mlx5/mlx5_rxtx_vec_neon.h
+++ b/drivers/net/mlx5/mlx5_rxtx_vec_neon.h
@@ -264,8 +264,8 @@
const uint32x4_t cv_mask =
vdupq_n_u32(PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD |
PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED);
- const uint64x1_t mbuf_init = vld1_u64(&rxq->mbuf_initializer);
- const uint64x1_t r32_mask = vcreate_u64(0xffffffff);
+ const uint64x2_t mbuf_init = vld1q_u64
+ ((const uint64_t *)&rxq->mbuf_initializer);
uint64x2_t rearm0, rearm1, rearm2, rearm3;
uint8_t pt_idx0, pt_idx1, pt_idx2, pt_idx3;
@@ -326,18 +326,19 @@
/* Merge to ol_flags. */
ol_flags = vorrq_u32(ol_flags, cv_flags);
/* Merge mbuf_init and ol_flags, and store. */
- rearm0 = vcombine_u64(mbuf_init,
- vshr_n_u64(vget_high_u64(vreinterpretq_u64_u32(
- ol_flags)), 32));
- rearm1 = vcombine_u64(mbuf_init,
- vand_u64(vget_high_u64(vreinterpretq_u64_u32(
- ol_flags)), r32_mask));
- rearm2 = vcombine_u64(mbuf_init,
- vshr_n_u64(vget_low_u64(vreinterpretq_u64_u32(
- ol_flags)), 32));
- rearm3 = vcombine_u64(mbuf_init,
- vand_u64(vget_low_u64(vreinterpretq_u64_u32(
- ol_flags)), r32_mask));
+ rearm0 = vreinterpretq_u64_u32(vsetq_lane_u32
+ (vgetq_lane_u32(ol_flags, 3),
+ vreinterpretq_u32_u64(mbuf_init), 2));
+ rearm1 = vreinterpretq_u64_u32(vsetq_lane_u32
+ (vgetq_lane_u32(ol_flags, 2),
+ vreinterpretq_u32_u64(mbuf_init), 2));
+ rearm2 = vreinterpretq_u64_u32(vsetq_lane_u32
+ (vgetq_lane_u32(ol_flags, 1),
+ vreinterpretq_u32_u64(mbuf_init), 2));
+ rearm3 = vreinterpretq_u64_u32(vsetq_lane_u32
+ (vgetq_lane_u32(ol_flags, 0),
+ vreinterpretq_u32_u64(mbuf_init), 2));
+
vst1q_u64((void *)&pkts[0]->rearm_data, rearm0);
vst1q_u64((void *)&pkts[1]->rearm_data, rearm1);
vst1q_u64((void *)&pkts[2]->rearm_data, rearm2);
diff --git a/drivers/net/mlx5/mlx5_rxtx_vec_sse.h b/drivers/net/mlx5/mlx5_rxtx_vec_sse.h
index 35b7761..07d40d5 100644
--- a/drivers/net/mlx5/mlx5_rxtx_vec_sse.h
+++ b/drivers/net/mlx5/mlx5_rxtx_vec_sse.h
@@ -259,7 +259,7 @@
PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD |
PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED);
const __m128i mbuf_init =
- _mm_loadl_epi64((__m128i *)&rxq->mbuf_initializer);
+ _mm_load_si128((__m128i *)&rxq->mbuf_initializer);
__m128i rearm0, rearm1, rearm2, rearm3;
uint8_t pt_idx0, pt_idx1, pt_idx2, pt_idx3;
--
1.8.3.1
next prev parent reply other threads:[~2020-01-10 17:57 UTC|newest]
Thread overview: 77+ messages / expand[flat|nested] mbox.gz Atom feed top
2019-11-18 9:50 [dpdk-dev] [RFC v20.20] mbuf: introduce pktmbuf pool with pinned external buffers Shahaf Shuler
2019-11-18 16:09 ` Stephen Hemminger
2020-01-10 17:56 ` [dpdk-dev] [PATCH 0/4] " Viacheslav Ovsiienko
2020-01-10 17:56 ` [dpdk-dev] [PATCH 1/4] mbuf: detach mbuf with pinned external buffer Viacheslav Ovsiienko
2020-01-10 18:23 ` Stephen Hemminger
2020-01-13 17:07 ` Slava Ovsiienko
2020-01-14 7:19 ` Slava Ovsiienko
2020-01-10 17:57 ` [dpdk-dev] [PATCH 2/4] mbuf: create packet pool with external memory buffers Viacheslav Ovsiienko
2020-01-10 17:57 ` [dpdk-dev] [PATCH 3/4] app/testpmd: add mempool with external data buffers Viacheslav Ovsiienko
2020-01-10 17:57 ` Viacheslav Ovsiienko [this message]
2020-01-14 7:49 ` [dpdk-dev] [PATCH v2 0/4] mbuf: introduce pktmbuf pool with pinned external buffers Viacheslav Ovsiienko
2020-01-14 7:49 ` [dpdk-dev] [PATCH v2 1/4] mbuf: detach mbuf with pinned external buffer Viacheslav Ovsiienko
2020-01-14 7:49 ` [dpdk-dev] [PATCH v2 2/4] mbuf: create packet pool with external memory buffers Viacheslav Ovsiienko
2020-01-14 7:49 ` [dpdk-dev] [PATCH v2 3/4] app/testpmd: add mempool with external data buffers Viacheslav Ovsiienko
2020-01-14 7:49 ` [dpdk-dev] [PATCH v2 4/4] net/mlx5: allow use allocated mbuf with external buffer Viacheslav Ovsiienko
2020-01-14 9:15 ` [dpdk-dev] [PATCH v3 0/4] mbuf: detach mbuf with pinned " Viacheslav Ovsiienko
2020-01-14 9:15 ` [dpdk-dev] [PATCH v3 1/4] " Viacheslav Ovsiienko
2020-01-14 15:27 ` Olivier Matz
2020-01-15 12:52 ` Slava Ovsiienko
2020-01-14 15:50 ` Stephen Hemminger
2020-01-14 9:15 ` [dpdk-dev] [PATCH v3 2/4] mbuf: create packet pool with external memory buffers Viacheslav Ovsiienko
2020-01-14 16:04 ` Olivier Matz
2020-01-15 18:13 ` Slava Ovsiienko
2020-01-14 9:15 ` [dpdk-dev] [PATCH v3 3/4] app/testpmd: add mempool with external data buffers Viacheslav Ovsiienko
2020-01-14 9:15 ` [dpdk-dev] [PATCH v3 4/4] net/mlx5: allow use allocated mbuf with external buffer Viacheslav Ovsiienko
2020-01-16 13:04 ` [dpdk-dev] [PATCH v4 0/5] mbuf: detach mbuf with pinned " Viacheslav Ovsiienko
2020-01-16 13:04 ` [dpdk-dev] [PATCH v4 1/5] mbuf: introduce routine to get private mbuf pool flags Viacheslav Ovsiienko
2020-01-20 12:16 ` Olivier Matz
2020-01-16 13:04 ` [dpdk-dev] [PATCH v4 2/5] mbuf: detach mbuf with pinned external buffer Viacheslav Ovsiienko
2020-01-20 13:56 ` Olivier Matz
2020-01-20 15:41 ` Slava Ovsiienko
2020-01-20 16:17 ` Olivier Matz
2020-01-16 13:04 ` [dpdk-dev] [PATCH v4 3/5] mbuf: create packet pool with external memory buffers Viacheslav Ovsiienko
2020-01-20 13:59 ` Olivier Matz
2020-01-20 17:33 ` Slava Ovsiienko
2020-01-16 13:04 ` [dpdk-dev] [PATCH v4 4/5] app/testpmd: add mempool with external data buffers Viacheslav Ovsiienko
2020-01-20 14:11 ` Olivier Matz
2020-01-16 13:04 ` [dpdk-dev] [PATCH v4 5/5] net/mlx5: allow use allocated mbuf with external buffer Viacheslav Ovsiienko
2020-01-20 17:23 ` [dpdk-dev] [PATCH v5 0/5] mbuf: detach mbuf with pinned " Viacheslav Ovsiienko
2020-01-20 17:23 ` [dpdk-dev] [PATCH v5 1/5] mbuf: introduce routine to get private mbuf pool flags Viacheslav Ovsiienko
2020-01-20 20:43 ` Stephen Hemminger
2020-01-20 22:52 ` Thomas Monjalon
2020-01-21 6:48 ` Slava Ovsiienko
2020-01-21 8:00 ` Slava Ovsiienko
2020-01-21 8:14 ` Olivier Matz
2020-01-21 8:23 ` Slava Ovsiienko
2020-01-21 9:13 ` Slava Ovsiienko
2020-01-21 14:01 ` Olivier Matz
2020-01-21 16:21 ` Stephen Hemminger
2020-01-20 17:23 ` [dpdk-dev] [PATCH v5 2/5] mbuf: detach mbuf with pinned external buffer Viacheslav Ovsiienko
2020-01-20 17:40 ` Olivier Matz
2020-01-20 17:23 ` [dpdk-dev] [PATCH v5 3/5] mbuf: create packet pool with external memory buffers Viacheslav Ovsiienko
2020-01-20 17:46 ` Olivier Matz
2020-01-20 17:23 ` [dpdk-dev] [PATCH v5 4/5] app/testpmd: add mempool with external data buffers Viacheslav Ovsiienko
2020-01-20 17:23 ` [dpdk-dev] [PATCH v5 5/5] net/mlx5: allow use allocated mbuf with external buffer Viacheslav Ovsiienko
2020-01-20 17:30 ` [dpdk-dev] [PATCH v5 0/5] mbuf: detach mbuf with pinned " Slava Ovsiienko
2020-01-20 17:41 ` Olivier Matz
2020-01-20 19:16 ` [dpdk-dev] [PATCH v6 " Viacheslav Ovsiienko
2020-01-20 19:16 ` [dpdk-dev] [PATCH v6 1/5] mbuf: introduce routine to get private mbuf pool flags Viacheslav Ovsiienko
2020-01-20 19:16 ` [dpdk-dev] [PATCH v6 2/5] mbuf: detach mbuf with pinned external buffer Viacheslav Ovsiienko
2023-12-06 10:55 ` [dpdk-dev] [PATCH v6 2/5] mbuf: detach mbuf with pinned externalbuffer Morten Brørup
2020-01-20 19:16 ` [dpdk-dev] [PATCH v6 3/5] mbuf: create packet pool with external memory buffers Viacheslav Ovsiienko
2020-01-20 20:48 ` Stephen Hemminger
2020-01-21 7:04 ` Slava Ovsiienko
2020-01-20 19:16 ` [dpdk-dev] [PATCH v6 4/5] app/testpmd: add mempool with external data buffers Viacheslav Ovsiienko
2020-01-20 19:16 ` [dpdk-dev] [PATCH v6 5/5] net/mlx5: allow use allocated mbuf with external buffer Viacheslav Ovsiienko
2020-01-20 22:55 ` [dpdk-dev] [PATCH v6 0/5] mbuf: detach mbuf with pinned " Thomas Monjalon
2020-01-22 8:50 ` [dpdk-dev] [PATCH] mbuf: fix pinned memory free routine style issue Viacheslav Ovsiienko
2020-02-06 9:46 ` Olivier Matz
2020-02-06 14:26 ` Thomas Monjalon
2020-01-24 20:25 ` [dpdk-dev] [PATCH] app/test: add test for mbuf with pinned external buffer Viacheslav Ovsiienko
2020-01-26 10:53 ` Slava Ovsiienko
2020-02-06 8:17 ` Olivier Matz
2020-02-06 8:24 ` Slava Ovsiienko
2020-02-06 9:51 ` Slava Ovsiienko
2020-02-06 9:49 ` [dpdk-dev] [PATCH v2] " Viacheslav Ovsiienko
2020-02-06 14:43 ` Thomas Monjalon
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1578679022-25344-5-git-send-email-viacheslavo@mellanox.com \
--to=viacheslavo@mellanox.com \
--cc=dev@dpdk.org \
--cc=matan@mellanox.com \
--cc=orika@mellanox.com \
--cc=rasland@mellanox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).