DPDK patches and discussions
 help / color / mirror / Atom feed
From: Joyce Kong <joyce.kong@arm.com>
To: dev@dpdk.org
Cc: nd@arm.com, maxime.coquelin@redhat.com, yinan.wang@intel.com,
	bruce.richardson@intel.com, tiwei.bie@intel.com,
	zhihong.wang@intel.com, amorenoz@redhat.com,
	xiao.w.wang@intel.com, yong.liu@intel.com, jfreimann@redhat.com,
	honnappa.nagarahalli@arm.com, gavin.hu@arm.com
Subject: [dpdk-dev] [PATCH v4 1/2] virtio: one way barrier for packed vring desc avail flags
Date: Tue, 17 Sep 2019 13:28:25 +0800	[thread overview]
Message-ID: <1568698107-36668-2-git-send-email-joyce.kong@arm.com> (raw)
In-Reply-To: <1568698107-36668-1-git-send-email-joyce.kong@arm.com>
In-Reply-To: <1566893979-3290-1-git-send-email-joyce.kong@arm.com>

In case VIRTIO_F_ORDER_PLATFORM(36) is not negotiated, then the frontend
and backend are assumed to be implemented in software, that is they can
run on identical CPUs in an SMP configuration.
Thus a weak form of memory barriers like rte_smp_r/wmb, other than
rte_cio_r/wmb, is sufficient for this case(vq->hw->weak_barriers == 1)
and yields better performance.
For the above case, this patch helps yielding even better performance
by replacing the two-way barriers with C11 one-way barriers for avail
flags in packed ring.

Meanwhile, a read barrier is required to ensure ordering between
descriptor's flags and content reads[1]. With C11, load-acquire can
enforce the ordering instead of rmb barrier.

[1]https://patchwork.dpdk.org/patch/49109/

Signed-off-by: Joyce Kong <joyce.kong@arm.com>
Reviewed-by: Gavin Hu <gavin.hu@arm.com>
Reviewed-by: Phil Yang <phil.yang@arm.com>
Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>
---
 drivers/net/virtio/virtio_rxtx.c                 | 13 +++++++------
 drivers/net/virtio/virtio_user/virtio_user_dev.c |  6 +++++-
 drivers/net/virtio/virtqueue.h                   | 21 +++++++++++++++++++++
 lib/librte_vhost/vhost.h                         |  2 +-
 lib/librte_vhost/virtio_net.c                    | 11 +++++------
 5 files changed, 39 insertions(+), 14 deletions(-)

diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c
index 27ead19..a87ffe1 100644
--- a/drivers/net/virtio/virtio_rxtx.c
+++ b/drivers/net/virtio/virtio_rxtx.c
@@ -456,8 +456,10 @@ virtqueue_enqueue_recv_refill_packed(struct virtqueue *vq,
 		vq->vq_desc_head_idx = dxp->next;
 		if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END)
 			vq->vq_desc_tail_idx = vq->vq_desc_head_idx;
-		virtio_wmb(hw->weak_barriers);
-		start_dp[idx].flags = flags;
+
+		virtqueue_store_flags_packed(&start_dp[idx], flags,
+					     hw->weak_barriers);
+
 		if (++vq->vq_avail_idx >= vq->vq_nentries) {
 			vq->vq_avail_idx -= vq->vq_nentries;
 			vq->vq_packed.cached_flags ^=
@@ -671,8 +673,7 @@ virtqueue_enqueue_xmit_packed_fast(struct virtnet_tx *txvq,
 			vq->vq_desc_tail_idx = VQ_RING_DESC_CHAIN_END;
 	}
 
-	virtio_wmb(vq->hw->weak_barriers);
-	dp->flags = flags;
+	virtqueue_store_flags_packed(dp, flags, vq->hw->weak_barriers);
 }
 
 static inline void
@@ -763,8 +764,8 @@ virtqueue_enqueue_xmit_packed(struct virtnet_tx *txvq, struct rte_mbuf *cookie,
 			vq->vq_desc_tail_idx = VQ_RING_DESC_CHAIN_END;
 	}
 
-	virtio_wmb(vq->hw->weak_barriers);
-	head_dp->flags = head_flags;
+	virtqueue_store_flags_packed(head_dp, head_flags,
+				     vq->hw->weak_barriers);
 }
 
 static inline void
diff --git a/drivers/net/virtio/virtio_user/virtio_user_dev.c b/drivers/net/virtio/virtio_user/virtio_user_dev.c
index fab87eb..7911c39 100644
--- a/drivers/net/virtio/virtio_user/virtio_user_dev.c
+++ b/drivers/net/virtio/virtio_user/virtio_user_dev.c
@@ -624,7 +624,7 @@ virtio_user_handle_ctrl_msg(struct virtio_user_dev *dev, struct vring *vring,
 static inline int
 desc_is_avail(struct vring_packed_desc *desc, bool wrap_counter)
 {
-	uint16_t flags = desc->flags;
+	uint16_t flags = __atomic_load_n(&desc->flags, __ATOMIC_ACQUIRE);
 
 	return wrap_counter == !!(flags & VRING_PACKED_DESC_F_AVAIL) &&
 		wrap_counter != !!(flags & VRING_PACKED_DESC_F_USED);
@@ -684,6 +684,10 @@ virtio_user_handle_cq_packed(struct virtio_user_dev *dev, uint16_t queue_idx)
 	struct vring_packed *vring = &dev->packed_vrings[queue_idx];
 	uint16_t n_descs, flags;
 
+	/* Perform a load-acquire barrier in desc_is_avail to
+	 * enforce the ordering between desc flags and desc
+	 * content.
+	 */
 	while (desc_is_avail(&vring->desc[vq->used_idx],
 			     vq->used_wrap_counter)) {
 
diff --git a/drivers/net/virtio/virtqueue.h b/drivers/net/virtio/virtqueue.h
index c6dd4a3..b728ff8 100644
--- a/drivers/net/virtio/virtqueue.h
+++ b/drivers/net/virtio/virtqueue.h
@@ -54,6 +54,27 @@ virtio_wmb(uint8_t weak_barriers)
 		rte_cio_wmb();
 }
 
+static inline void
+virtqueue_store_flags_packed(struct vring_packed_desc *dp,
+			      uint16_t flags, uint8_t weak_barriers)
+{
+	if (weak_barriers) {
+/* x86 prefers to using rte_smp_wmb over __atomic_store_n as it reports
+ * a better perf(~1.5%), which comes from the saved branch by the compiler.
+ * The if and else branch are identical with the smp and cio barriers both
+ * defined as compiler barriers on x86.
+ */
+#ifdef RTE_ARCH_X86_64
+		rte_smp_wmb();
+		dp->flags = flags;
+#else
+		__atomic_store_n(&dp->flags, flags, __ATOMIC_RELEASE);
+#endif
+	} else {
+		rte_cio_wmb();
+		dp->flags = flags;
+	}
+}
 #ifdef RTE_PMD_PACKET_PREFETCH
 #define rte_packet_prefetch(p)  rte_prefetch1(p)
 #else
diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h
index 884befa..d294ed1 100644
--- a/lib/librte_vhost/vhost.h
+++ b/lib/librte_vhost/vhost.h
@@ -344,7 +344,7 @@ vq_is_packed(struct virtio_net *dev)
 static inline bool
 desc_is_avail(struct vring_packed_desc *desc, bool wrap_counter)
 {
-	uint16_t flags = *((volatile uint16_t *) &desc->flags);
+	uint16_t flags = __atomic_load_n(&desc->flags, __ATOMIC_ACQUIRE);
 
 	return wrap_counter == !!(flags & VRING_DESC_F_AVAIL) &&
 		wrap_counter != !!(flags & VRING_DESC_F_USED);
diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c
index 5b85b83..e7463ff 100644
--- a/lib/librte_vhost/virtio_net.c
+++ b/lib/librte_vhost/virtio_net.c
@@ -503,14 +503,13 @@ fill_vec_buf_packed(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	if (avail_idx < vq->last_avail_idx)
 		wrap_counter ^= 1;
 
-	if (unlikely(!desc_is_avail(&descs[avail_idx], wrap_counter)))
-		return -1;
-
 	/*
-	 * The ordering between desc flags and desc
-	 * content reads need to be enforced.
+	 * Perform a load-acquire barrier in desc_is_avail to
+	 * enforce the ordering between desc flags and desc
+	 * content.
 	 */
-	rte_smp_rmb();
+	if (unlikely(!desc_is_avail(&descs[avail_idx], wrap_counter)))
+		return -1;
 
 	*desc_count = 0;
 	*len = 0;
-- 
2.7.4


  parent reply	other threads:[~2019-09-17  5:29 UTC|newest]

Thread overview: 30+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2019-08-27  8:19 [dpdk-dev] [RFC PATCH 0/2] virtio: one way barrier for packed vring flags Joyce Kong
2019-08-27  8:19 ` [dpdk-dev] [RFC PATCH 1/2] virtio: one way barrier for packed vring desc avail flags Joyce Kong
2019-08-27  8:19 ` [dpdk-dev] [RFC PATCH 2/2] virtio: one way barrier for packed vring desc used flags Joyce Kong
2019-09-06 11:34 ` [dpdk-dev] [PATCH v2 0/2] virtio: one way barrier for packed vring flags Joyce Kong
2019-09-06 11:34 ` [dpdk-dev] [PATCH v2 1/2] virtio: one way barrier for packed vring desc avail flags Joyce Kong
2019-09-06 16:01   ` Maxime Coquelin
2019-09-09  9:24     ` Joyce Kong (Arm Technology China)
2019-09-06 11:34 ` [dpdk-dev] [PATCH v2 2/2] virtio: one way barrier for packed vring desc used flags Joyce Kong
2019-09-09  9:14 ` [dpdk-dev] [PATCH v3 0/2] virtio: one way barrier for packed vring flags Joyce Kong
2019-09-09  9:14 ` [dpdk-dev] [PATCH v3 1/2] virtio: one way barrier for packed vring desc avail flags Joyce Kong
2019-09-09 10:10   ` Maxime Coquelin
2019-09-10  3:54     ` Wang, Yinan
2019-09-10  9:48       ` Gavin Hu (Arm Technology China)
2019-09-10 10:17         ` Maxime Coquelin
2019-09-11  2:39         ` Liu, Yong
2019-09-11  3:35           ` Gavin Hu (Arm Technology China)
2019-09-11  6:29             ` Liu, Yong
2019-09-11  8:32               ` Gavin Hu (Arm Technology China)
2019-09-11 10:02                 ` Bruce Richardson
2019-09-12  8:21                   ` Gavin Hu (Arm Technology China)
2019-09-09  9:14 ` [dpdk-dev] [PATCH v3 2/2] virtio: one way barrier for packed vring desc used flags Joyce Kong
2019-09-09 10:11   ` Maxime Coquelin
2019-09-17  5:28 ` [dpdk-dev] [PATCH v4 0/2] virtio: one way barrier for packed vring flags Joyce Kong
2019-10-16 11:07   ` Maxime Coquelin
2019-09-17  5:28 ` Joyce Kong [this message]
2019-10-14  7:42   ` [dpdk-dev] [PATCH v4 1/2] virtio: one way barrier for packed vring desc avail flags Maxime Coquelin
2019-09-17  5:28 ` [dpdk-dev] [PATCH v4 2/2] virtio: one way barrier for packed vring desc used flags Joyce Kong
2019-09-18  5:20   ` Wang, Yinan
2019-09-19  4:04     ` Gavin Hu (Arm Technology China)
2019-10-14  7:43   ` Maxime Coquelin

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=1568698107-36668-2-git-send-email-joyce.kong@arm.com \
    --to=joyce.kong@arm.com \
    --cc=amorenoz@redhat.com \
    --cc=bruce.richardson@intel.com \
    --cc=dev@dpdk.org \
    --cc=gavin.hu@arm.com \
    --cc=honnappa.nagarahalli@arm.com \
    --cc=jfreimann@redhat.com \
    --cc=maxime.coquelin@redhat.com \
    --cc=nd@arm.com \
    --cc=tiwei.bie@intel.com \
    --cc=xiao.w.wang@intel.com \
    --cc=yinan.wang@intel.com \
    --cc=yong.liu@intel.com \
    --cc=zhihong.wang@intel.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).