From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 3182942B2A; Wed, 17 May 2023 11:08:55 +0200 (CEST) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 1D2B541611; Wed, 17 May 2023 11:08:55 +0200 (CEST) Received: from us-smtp-delivery-124.mimecast.com (us-smtp-delivery-124.mimecast.com [170.10.133.124]) by mails.dpdk.org (Postfix) with ESMTP id 4C4B141611 for ; Wed, 17 May 2023 11:08:53 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=redhat.com; s=mimecast20190719; t=1684314532; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version:content-type:content-type: content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references; bh=8eZdsfGOuxFzpbkHjIjhQWES1VaK0PaQ8t3nUzO/PJg=; b=OINZYX8rFhrgjx0bEgv6FmldbQT5BcrWFGXnBowcuYlL7gLPX/p1C0PAiSIK5mIjymzfyV A077/ifkH1u/pOpwfcuyXF7HGv06VEZMwCb6dUtCR3oaw5qPD0JDukoI0UgGxn1H7UN3kI H8rdcE7KulslkrjWdnBSC9Y9FQQVTA4= Received: from mimecast-mx02.redhat.com (mx3-rdu2.redhat.com [66.187.233.73]) by relay.mimecast.com with ESMTP with STARTTLS (version=TLSv1.2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id us-mta-660-BLJgVy_WOiaCcJVJViY4-Q-1; Wed, 17 May 2023 05:08:49 -0400 X-MC-Unique: BLJgVy_WOiaCcJVJViY4-Q-1 Received: from smtp.corp.redhat.com (int-mx03.intmail.prod.int.rdu2.redhat.com [10.11.54.3]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mimecast-mx02.redhat.com (Postfix) with ESMTPS id 636D129AB3F1; Wed, 17 May 2023 09:08:49 +0000 (UTC) Received: from ebuild.redhat.com (unknown [10.39.195.120]) by smtp.corp.redhat.com (Postfix) with ESMTP id 7CB9E1121314; Wed, 17 May 2023 09:08:48 +0000 (UTC) From: Eelco Chaudron To: maxime.coquelin@redhat.com, chenbo.xia@intel.com, david.marchand@redhat.com Cc: dev@dpdk.org Subject: [PATCH v3 1/4] vhost: change vhost_virtqueue access lock to a read/write one Date: Wed, 17 May 2023 11:08:47 +0200 Message-Id: <168431452543.558450.14131829672896784074.stgit@ebuild.local> In-Reply-To: <168431450017.558450.16680518469610688737.stgit@ebuild.local> References: <168431450017.558450.16680518469610688737.stgit@ebuild.local> User-Agent: StGit/1.5 MIME-Version: 1.0 X-Scanned-By: MIMEDefang 3.1 on 10.11.54.3 X-Mimecast-Spam-Score: 0 X-Mimecast-Originator: redhat.com Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: 8bit X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org This change will allow the vhost interrupt datapath handling to be split between two processed without one of them holding an explicit lock. Signed-off-by: Eelco Chaudron --- lib/eal/include/generic/rte_rwlock.h | 17 ++++++ lib/vhost/vhost.c | 46 +++++++++-------- lib/vhost/vhost.h | 4 +- lib/vhost/vhost_user.c | 14 +++-- lib/vhost/virtio_net.c | 90 +++++++++++++++++----------------- 5 files changed, 94 insertions(+), 77 deletions(-) diff --git a/lib/eal/include/generic/rte_rwlock.h b/lib/eal/include/generic/rte_rwlock.h index 71e2d8d5f4..9e083bbc61 100644 --- a/lib/eal/include/generic/rte_rwlock.h +++ b/lib/eal/include/generic/rte_rwlock.h @@ -236,6 +236,23 @@ rte_rwlock_write_unlock(rte_rwlock_t *rwl) __atomic_fetch_sub(&rwl->cnt, RTE_RWLOCK_WRITE, __ATOMIC_RELEASE); } +/** + * Test if the write lock is taken. + * + * @param rwl + * A pointer to a rwlock structure. + * @return + * 1 if the write lock is currently taken; 0 otherwise. + */ +static inline int +rte_rwlock_write_is_locked(rte_rwlock_t *rwl) +{ + if (__atomic_load_n(&rwl->cnt, __ATOMIC_RELAXED) & RTE_RWLOCK_WRITE) + return 1; + + return 0; +} + /** * Try to execute critical section in a hardware memory transaction, if it * fails or not available take a read lock diff --git a/lib/vhost/vhost.c b/lib/vhost/vhost.c index ef37943817..74bdbfd810 100644 --- a/lib/vhost/vhost.c +++ b/lib/vhost/vhost.c @@ -393,9 +393,9 @@ free_vq(struct virtio_net *dev, struct vhost_virtqueue *vq) else rte_free(vq->shadow_used_split); - rte_spinlock_lock(&vq->access_lock); + rte_rwlock_write_lock(&vq->access_lock); vhost_free_async_mem(vq); - rte_spinlock_unlock(&vq->access_lock); + rte_rwlock_write_unlock(&vq->access_lock); rte_free(vq->batch_copy_elems); vhost_user_iotlb_destroy(vq); rte_free(vq->log_cache); @@ -630,7 +630,7 @@ alloc_vring_queue(struct virtio_net *dev, uint32_t vring_idx) dev->virtqueue[i] = vq; init_vring_queue(dev, vq, i); - rte_spinlock_init(&vq->access_lock); + rte_rwlock_init(&vq->access_lock); vq->avail_wrap_counter = 1; vq->used_wrap_counter = 1; vq->signalled_used_valid = false; @@ -1305,14 +1305,14 @@ rte_vhost_vring_call(int vid, uint16_t vring_idx) if (!vq) return -1; - rte_spinlock_lock(&vq->access_lock); + rte_rwlock_read_lock(&vq->access_lock); if (vq_is_packed(dev)) vhost_vring_call_packed(dev, vq); else vhost_vring_call_split(dev, vq); - rte_spinlock_unlock(&vq->access_lock); + rte_rwlock_read_unlock(&vq->access_lock); return 0; } @@ -1334,7 +1334,7 @@ rte_vhost_vring_call_nonblock(int vid, uint16_t vring_idx) if (!vq) return -1; - if (!rte_spinlock_trylock(&vq->access_lock)) + if (rte_rwlock_read_trylock(&vq->access_lock)) return -EAGAIN; if (vq_is_packed(dev)) @@ -1342,7 +1342,7 @@ rte_vhost_vring_call_nonblock(int vid, uint16_t vring_idx) else vhost_vring_call_split(dev, vq); - rte_spinlock_unlock(&vq->access_lock); + rte_rwlock_read_unlock(&vq->access_lock); return 0; } @@ -1365,7 +1365,7 @@ rte_vhost_avail_entries(int vid, uint16_t queue_id) if (!vq) return 0; - rte_spinlock_lock(&vq->access_lock); + rte_rwlock_write_lock(&vq->access_lock); if (unlikely(!vq->enabled || vq->avail == NULL)) goto out; @@ -1373,7 +1373,7 @@ rte_vhost_avail_entries(int vid, uint16_t queue_id) ret = *(volatile uint16_t *)&vq->avail->idx - vq->last_used_idx; out: - rte_spinlock_unlock(&vq->access_lock); + rte_rwlock_write_unlock(&vq->access_lock); return ret; } @@ -1457,12 +1457,12 @@ rte_vhost_enable_guest_notification(int vid, uint16_t queue_id, int enable) if (!vq) return -1; - rte_spinlock_lock(&vq->access_lock); + rte_rwlock_write_lock(&vq->access_lock); vq->notif_enable = enable; ret = vhost_enable_guest_notification(dev, vq, enable); - rte_spinlock_unlock(&vq->access_lock); + rte_rwlock_write_unlock(&vq->access_lock); return ret; } @@ -1520,7 +1520,7 @@ rte_vhost_rx_queue_count(int vid, uint16_t qid) if (vq == NULL) return 0; - rte_spinlock_lock(&vq->access_lock); + rte_rwlock_write_lock(&vq->access_lock); if (unlikely(!vq->enabled || vq->avail == NULL)) goto out; @@ -1528,7 +1528,7 @@ rte_vhost_rx_queue_count(int vid, uint16_t qid) ret = *((volatile uint16_t *)&vq->avail->idx) - vq->last_avail_idx; out: - rte_spinlock_unlock(&vq->access_lock); + rte_rwlock_write_unlock(&vq->access_lock); return ret; } @@ -1757,9 +1757,9 @@ rte_vhost_async_channel_register(int vid, uint16_t queue_id) if (unlikely(vq == NULL || !dev->async_copy || dev->vdpa_dev != NULL)) return -1; - rte_spinlock_lock(&vq->access_lock); + rte_rwlock_write_lock(&vq->access_lock); ret = async_channel_register(dev, vq); - rte_spinlock_unlock(&vq->access_lock); + rte_rwlock_write_unlock(&vq->access_lock); return ret; } @@ -1804,7 +1804,7 @@ rte_vhost_async_channel_unregister(int vid, uint16_t queue_id) if (vq == NULL) return ret; - if (!rte_spinlock_trylock(&vq->access_lock)) { + if (rte_rwlock_write_trylock(&vq->access_lock)) { VHOST_LOG_CONFIG(dev->ifname, ERR, "failed to unregister async channel, virtqueue busy.\n"); return ret; @@ -1821,7 +1821,7 @@ rte_vhost_async_channel_unregister(int vid, uint16_t queue_id) ret = 0; } - rte_spinlock_unlock(&vq->access_lock); + rte_rwlock_write_unlock(&vq->access_lock); return ret; } @@ -1954,7 +1954,7 @@ rte_vhost_async_get_inflight(int vid, uint16_t queue_id) if (vq == NULL) return ret; - if (!rte_spinlock_trylock(&vq->access_lock)) { + if (rte_rwlock_write_trylock(&vq->access_lock)) { VHOST_LOG_CONFIG(dev->ifname, DEBUG, "failed to check in-flight packets. virtqueue busy.\n"); return ret; @@ -1963,7 +1963,7 @@ rte_vhost_async_get_inflight(int vid, uint16_t queue_id) if (vq->async) ret = vq->async->pkts_inflight_n; - rte_spinlock_unlock(&vq->access_lock); + rte_rwlock_write_unlock(&vq->access_lock); return ret; } @@ -2084,13 +2084,13 @@ rte_vhost_vring_stats_get(int vid, uint16_t queue_id, vq = dev->virtqueue[queue_id]; - rte_spinlock_lock(&vq->access_lock); + rte_rwlock_write_lock(&vq->access_lock); for (i = 0; i < VHOST_NB_VQ_STATS; i++) { stats[i].value = *(uint64_t *)(((char *)vq) + vhost_vq_stat_strings[i].offset); stats[i].id = i; } - rte_spinlock_unlock(&vq->access_lock); + rte_rwlock_write_unlock(&vq->access_lock); return VHOST_NB_VQ_STATS; } @@ -2111,9 +2111,9 @@ int rte_vhost_vring_stats_reset(int vid, uint16_t queue_id) vq = dev->virtqueue[queue_id]; - rte_spinlock_lock(&vq->access_lock); + rte_rwlock_write_lock(&vq->access_lock); memset(&vq->stats, 0, sizeof(vq->stats)); - rte_spinlock_unlock(&vq->access_lock); + rte_rwlock_write_unlock(&vq->access_lock); return 0; } diff --git a/lib/vhost/vhost.h b/lib/vhost/vhost.h index 8fdab13c70..5c939ef06f 100644 --- a/lib/vhost/vhost.h +++ b/lib/vhost/vhost.h @@ -277,7 +277,7 @@ struct vhost_virtqueue { bool access_ok; bool ready; - rte_spinlock_t access_lock; + rte_rwlock_t access_lock; union { @@ -517,7 +517,7 @@ static inline void vq_assert_lock__(struct virtio_net *dev, struct vhost_virtqueue *vq, const char *func) __rte_assert_exclusive_lock(&vq->access_lock) { - if (unlikely(!rte_spinlock_is_locked(&vq->access_lock))) + if (unlikely(!rte_rwlock_write_is_locked(&vq->access_lock))) rte_panic("VHOST_CONFIG: (%s) %s() called without access lock taken.\n", dev->ifname, func); } diff --git a/lib/vhost/vhost_user.c b/lib/vhost/vhost_user.c index d60e39b6bc..c9454ce3d9 100644 --- a/lib/vhost/vhost_user.c +++ b/lib/vhost/vhost_user.c @@ -399,7 +399,7 @@ vhost_user_set_features(struct virtio_net **pdev, cleanup_vq_inflight(dev, vq); /* vhost_user_lock_all_queue_pairs locked all qps */ vq_assert_lock(dev, vq); - rte_spinlock_unlock(&vq->access_lock); + rte_rwlock_write_unlock(&vq->access_lock); free_vq(dev, vq); } } @@ -2649,10 +2649,10 @@ vhost_user_iotlb_msg(struct virtio_net **pdev, len, imsg->perm); if (is_vring_iotlb(dev, vq, imsg)) { - rte_spinlock_lock(&vq->access_lock); + rte_rwlock_write_lock(&vq->access_lock); translate_ring_addresses(&dev, &vq); *pdev = dev; - rte_spinlock_unlock(&vq->access_lock); + rte_rwlock_write_unlock(&vq->access_lock); } } break; @@ -2667,9 +2667,9 @@ vhost_user_iotlb_msg(struct virtio_net **pdev, imsg->size); if (is_vring_iotlb(dev, vq, imsg)) { - rte_spinlock_lock(&vq->access_lock); + rte_rwlock_write_lock(&vq->access_lock); vring_invalidate(dev, vq); - rte_spinlock_unlock(&vq->access_lock); + rte_rwlock_write_unlock(&vq->access_lock); } } break; @@ -3030,7 +3030,7 @@ vhost_user_lock_all_queue_pairs(struct virtio_net *dev) struct vhost_virtqueue *vq = dev->virtqueue[i]; if (vq) { - rte_spinlock_lock(&vq->access_lock); + rte_rwlock_write_lock(&vq->access_lock); vq_num++; } i++; @@ -3048,7 +3048,7 @@ vhost_user_unlock_all_queue_pairs(struct virtio_net *dev) struct vhost_virtqueue *vq = dev->virtqueue[i]; if (vq) { - rte_spinlock_unlock(&vq->access_lock); + rte_rwlock_write_unlock(&vq->access_lock); vq_num++; } i++; diff --git a/lib/vhost/virtio_net.c b/lib/vhost/virtio_net.c index fe10a78586..d7624d18c8 100644 --- a/lib/vhost/virtio_net.c +++ b/lib/vhost/virtio_net.c @@ -55,7 +55,7 @@ is_valid_virt_queue_idx(uint32_t idx, int is_tx, uint32_t nr_vring) static inline void vhost_queue_stats_update(struct virtio_net *dev, struct vhost_virtqueue *vq, struct rte_mbuf **pkts, uint16_t count) - __rte_exclusive_locks_required(&vq->access_lock) + __rte_shared_locks_required(&vq->access_lock) { struct virtqueue_stats *stats = &vq->stats; int i; @@ -102,7 +102,7 @@ static __rte_always_inline int64_t vhost_async_dma_transfer_one(struct virtio_net *dev, struct vhost_virtqueue *vq, int16_t dma_id, uint16_t vchan_id, uint16_t flag_idx, struct vhost_iov_iter *pkt) - __rte_exclusive_locks_required(&vq->access_lock) + __rte_shared_locks_required(&vq->access_lock) { struct async_dma_vchan_info *dma_info = &dma_copy_track[dma_id].vchans[vchan_id]; uint16_t ring_mask = dma_info->ring_mask; @@ -152,7 +152,7 @@ static __rte_always_inline uint16_t vhost_async_dma_transfer(struct virtio_net *dev, struct vhost_virtqueue *vq, int16_t dma_id, uint16_t vchan_id, uint16_t head_idx, struct vhost_iov_iter *pkts, uint16_t nr_pkts) - __rte_exclusive_locks_required(&vq->access_lock) + __rte_shared_locks_required(&vq->access_lock) { struct async_dma_vchan_info *dma_info = &dma_copy_track[dma_id].vchans[vchan_id]; int64_t ret, nr_copies = 0; @@ -454,7 +454,7 @@ vhost_async_shadow_enqueue_packed_batch(struct vhost_virtqueue *vq, static __rte_always_inline void vhost_async_shadow_dequeue_packed_batch(struct vhost_virtqueue *vq, uint16_t *ids) - __rte_exclusive_locks_required(&vq->access_lock) + __rte_shared_locks_required(&vq->access_lock) { uint16_t i; struct vhost_async *async = vq->async; @@ -1131,7 +1131,7 @@ static __rte_always_inline int async_fill_seg(struct virtio_net *dev, struct vhost_virtqueue *vq, struct rte_mbuf *m, uint32_t mbuf_offset, uint64_t buf_iova, uint32_t cpy_len, bool to_desc) - __rte_exclusive_locks_required(&vq->access_lock) + __rte_shared_locks_required(&vq->access_lock) __rte_shared_locks_required(&vq->iotlb_lock) { struct vhost_async *async = vq->async; @@ -1211,7 +1211,7 @@ static __rte_always_inline int mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, struct rte_mbuf *m, struct buf_vector *buf_vec, uint16_t nr_vec, uint16_t num_buffers, bool is_async) - __rte_exclusive_locks_required(&vq->access_lock) + __rte_shared_locks_required(&vq->access_lock) __rte_shared_locks_required(&vq->iotlb_lock) { uint32_t vec_idx = 0; @@ -1341,7 +1341,7 @@ vhost_enqueue_single_packed(struct virtio_net *dev, struct rte_mbuf *pkt, struct buf_vector *buf_vec, uint16_t *nr_descs) - __rte_exclusive_locks_required(&vq->access_lock) + __rte_shared_locks_required(&vq->access_lock) __rte_shared_locks_required(&vq->iotlb_lock) { uint16_t nr_vec = 0; @@ -1403,7 +1403,7 @@ vhost_enqueue_single_packed(struct virtio_net *dev, static __rte_noinline uint32_t virtio_dev_rx_split(struct virtio_net *dev, struct vhost_virtqueue *vq, struct rte_mbuf **pkts, uint32_t count) - __rte_exclusive_locks_required(&vq->access_lock) + __rte_shared_locks_required(&vq->access_lock) __rte_shared_locks_required(&vq->iotlb_lock) { uint32_t pkt_idx = 0; @@ -1635,7 +1635,7 @@ static __rte_always_inline int16_t virtio_dev_rx_single_packed(struct virtio_net *dev, struct vhost_virtqueue *vq, struct rte_mbuf *pkt) - __rte_exclusive_locks_required(&vq->access_lock) + __rte_shared_locks_required(&vq->access_lock) __rte_shared_locks_required(&vq->iotlb_lock) { struct buf_vector buf_vec[BUF_VECTOR_MAX]; @@ -1661,7 +1661,7 @@ virtio_dev_rx_packed(struct virtio_net *dev, struct vhost_virtqueue *__rte_restrict vq, struct rte_mbuf **__rte_restrict pkts, uint32_t count) - __rte_exclusive_locks_required(&vq->access_lock) + __rte_shared_locks_required(&vq->access_lock) __rte_shared_locks_required(&vq->iotlb_lock) { uint32_t pkt_idx = 0; @@ -1701,7 +1701,7 @@ virtio_dev_rx(struct virtio_net *dev, struct vhost_virtqueue *vq, uint32_t nb_tx = 0; VHOST_LOG_DATA(dev->ifname, DEBUG, "%s\n", __func__); - rte_spinlock_lock(&vq->access_lock); + rte_rwlock_read_lock(&vq->access_lock); if (unlikely(!vq->enabled)) goto out_access_unlock; @@ -1727,7 +1727,7 @@ virtio_dev_rx(struct virtio_net *dev, struct vhost_virtqueue *vq, vhost_user_iotlb_rd_unlock(vq); out_access_unlock: - rte_spinlock_unlock(&vq->access_lock); + rte_rwlock_read_unlock(&vq->access_lock); return nb_tx; } @@ -1760,7 +1760,7 @@ rte_vhost_enqueue_burst(int vid, uint16_t queue_id, static __rte_always_inline uint16_t async_get_first_inflight_pkt_idx(struct vhost_virtqueue *vq) - __rte_exclusive_locks_required(&vq->access_lock) + __rte_shared_locks_required(&vq->access_lock) { struct vhost_async *async = vq->async; @@ -2165,7 +2165,7 @@ virtio_dev_rx_async_submit_packed(struct virtio_net *dev, struct vhost_virtqueue static __rte_always_inline void write_back_completed_descs_split(struct vhost_virtqueue *vq, uint16_t n_descs) - __rte_exclusive_locks_required(&vq->access_lock) + __rte_shared_locks_required(&vq->access_lock) { struct vhost_async *async = vq->async; uint16_t nr_left = n_descs; @@ -2198,7 +2198,7 @@ write_back_completed_descs_split(struct vhost_virtqueue *vq, uint16_t n_descs) static __rte_always_inline void write_back_completed_descs_packed(struct vhost_virtqueue *vq, uint16_t n_buffers) - __rte_exclusive_locks_required(&vq->access_lock) + __rte_shared_locks_required(&vq->access_lock) { struct vhost_async *async = vq->async; uint16_t from = async->last_buffer_idx_packed; @@ -2263,7 +2263,7 @@ write_back_completed_descs_packed(struct vhost_virtqueue *vq, static __rte_always_inline uint16_t vhost_poll_enqueue_completed(struct virtio_net *dev, struct vhost_virtqueue *vq, struct rte_mbuf **pkts, uint16_t count, int16_t dma_id, uint16_t vchan_id) - __rte_exclusive_locks_required(&vq->access_lock) + __rte_shared_locks_required(&vq->access_lock) { struct vhost_async *async = vq->async; struct async_inflight_info *pkts_info = async->pkts_info; @@ -2357,7 +2357,7 @@ rte_vhost_poll_enqueue_completed(int vid, uint16_t queue_id, vq = dev->virtqueue[queue_id]; - if (!rte_spinlock_trylock(&vq->access_lock)) { + if (rte_rwlock_read_trylock(&vq->access_lock)) { VHOST_LOG_DATA(dev->ifname, DEBUG, "%s: virtqueue %u is busy.\n", __func__, queue_id); @@ -2377,7 +2377,7 @@ rte_vhost_poll_enqueue_completed(int vid, uint16_t queue_id, vq->stats.inflight_completed += n_pkts_cpl; out: - rte_spinlock_unlock(&vq->access_lock); + rte_rwlock_read_unlock(&vq->access_lock); return n_pkts_cpl; } @@ -2465,7 +2465,7 @@ rte_vhost_clear_queue(int vid, uint16_t queue_id, struct rte_mbuf **pkts, vq = dev->virtqueue[queue_id]; - if (!rte_spinlock_trylock(&vq->access_lock)) { + if (rte_rwlock_read_trylock(&vq->access_lock)) { VHOST_LOG_DATA(dev->ifname, DEBUG, "%s: virtqueue %u is busy.\n", __func__, queue_id); return 0; @@ -2495,7 +2495,7 @@ rte_vhost_clear_queue(int vid, uint16_t queue_id, struct rte_mbuf **pkts, vq->stats.inflight_completed += n_pkts_cpl; out_access_unlock: - rte_spinlock_unlock(&vq->access_lock); + rte_rwlock_read_unlock(&vq->access_lock); return n_pkts_cpl; } @@ -2516,7 +2516,7 @@ virtio_dev_rx_async_submit(struct virtio_net *dev, struct vhost_virtqueue *vq, return 0; } - rte_spinlock_lock(&vq->access_lock); + rte_rwlock_write_lock(&vq->access_lock); if (unlikely(!vq->enabled || !vq->async)) goto out_access_unlock; @@ -2544,7 +2544,7 @@ virtio_dev_rx_async_submit(struct virtio_net *dev, struct vhost_virtqueue *vq, vhost_user_iotlb_rd_unlock(vq); out_access_unlock: - rte_spinlock_unlock(&vq->access_lock); + rte_rwlock_write_unlock(&vq->access_lock); return nb_tx; } @@ -2866,7 +2866,7 @@ desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq, struct buf_vector *buf_vec, uint16_t nr_vec, struct rte_mbuf *m, struct rte_mempool *mbuf_pool, bool legacy_ol_flags, uint16_t slot_idx, bool is_async) - __rte_exclusive_locks_required(&vq->access_lock) + __rte_shared_locks_required(&vq->access_lock) __rte_shared_locks_required(&vq->iotlb_lock) { uint32_t buf_avail, buf_offset, buf_len; @@ -3073,7 +3073,7 @@ static uint16_t virtio_dev_tx_split(struct virtio_net *dev, struct vhost_virtqueue *vq, struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count, bool legacy_ol_flags) - __rte_exclusive_locks_required(&vq->access_lock) + __rte_shared_locks_required(&vq->access_lock) __rte_shared_locks_required(&vq->iotlb_lock) { uint16_t i; @@ -3178,7 +3178,7 @@ static uint16_t virtio_dev_tx_split_legacy(struct virtio_net *dev, struct vhost_virtqueue *vq, struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count) - __rte_exclusive_locks_required(&vq->access_lock) + __rte_shared_locks_required(&vq->access_lock) __rte_shared_locks_required(&vq->iotlb_lock) { return virtio_dev_tx_split(dev, vq, mbuf_pool, pkts, count, true); @@ -3189,7 +3189,7 @@ static uint16_t virtio_dev_tx_split_compliant(struct virtio_net *dev, struct vhost_virtqueue *vq, struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count) - __rte_exclusive_locks_required(&vq->access_lock) + __rte_shared_locks_required(&vq->access_lock) __rte_shared_locks_required(&vq->iotlb_lock) { return virtio_dev_tx_split(dev, vq, mbuf_pool, pkts, count, false); @@ -3393,7 +3393,7 @@ vhost_dequeue_single_packed(struct virtio_net *dev, uint16_t *buf_id, uint16_t *desc_count, bool legacy_ol_flags) - __rte_exclusive_locks_required(&vq->access_lock) + __rte_shared_locks_required(&vq->access_lock) __rte_shared_locks_required(&vq->iotlb_lock) { struct buf_vector buf_vec[BUF_VECTOR_MAX]; @@ -3443,7 +3443,7 @@ virtio_dev_tx_single_packed(struct virtio_net *dev, struct rte_mempool *mbuf_pool, struct rte_mbuf *pkts, bool legacy_ol_flags) - __rte_exclusive_locks_required(&vq->access_lock) + __rte_shared_locks_required(&vq->access_lock) __rte_shared_locks_required(&vq->iotlb_lock) { @@ -3475,7 +3475,7 @@ virtio_dev_tx_packed(struct virtio_net *dev, struct rte_mbuf **__rte_restrict pkts, uint32_t count, bool legacy_ol_flags) - __rte_exclusive_locks_required(&vq->access_lock) + __rte_shared_locks_required(&vq->access_lock) __rte_shared_locks_required(&vq->iotlb_lock) { uint32_t pkt_idx = 0; @@ -3520,7 +3520,7 @@ static uint16_t virtio_dev_tx_packed_legacy(struct virtio_net *dev, struct vhost_virtqueue *__rte_restrict vq, struct rte_mempool *mbuf_pool, struct rte_mbuf **__rte_restrict pkts, uint32_t count) - __rte_exclusive_locks_required(&vq->access_lock) + __rte_shared_locks_required(&vq->access_lock) __rte_shared_locks_required(&vq->iotlb_lock) { return virtio_dev_tx_packed(dev, vq, mbuf_pool, pkts, count, true); @@ -3531,7 +3531,7 @@ static uint16_t virtio_dev_tx_packed_compliant(struct virtio_net *dev, struct vhost_virtqueue *__rte_restrict vq, struct rte_mempool *mbuf_pool, struct rte_mbuf **__rte_restrict pkts, uint32_t count) - __rte_exclusive_locks_required(&vq->access_lock) + __rte_shared_locks_required(&vq->access_lock) __rte_shared_locks_required(&vq->iotlb_lock) { return virtio_dev_tx_packed(dev, vq, mbuf_pool, pkts, count, false); @@ -3566,7 +3566,7 @@ rte_vhost_dequeue_burst(int vid, uint16_t queue_id, vq = dev->virtqueue[queue_id]; - if (unlikely(rte_spinlock_trylock(&vq->access_lock) == 0)) + if (unlikely(rte_rwlock_read_trylock(&vq->access_lock) != 0)) return 0; if (unlikely(!vq->enabled)) { @@ -3636,7 +3636,7 @@ rte_vhost_dequeue_burst(int vid, uint16_t queue_id, vhost_user_iotlb_rd_unlock(vq); out_access_unlock: - rte_spinlock_unlock(&vq->access_lock); + rte_rwlock_read_unlock(&vq->access_lock); if (unlikely(rarp_mbuf != NULL)) count += 1; @@ -3648,7 +3648,7 @@ static __rte_always_inline uint16_t async_poll_dequeue_completed(struct virtio_net *dev, struct vhost_virtqueue *vq, struct rte_mbuf **pkts, uint16_t count, int16_t dma_id, uint16_t vchan_id, bool legacy_ol_flags) - __rte_exclusive_locks_required(&vq->access_lock) + __rte_shared_locks_required(&vq->access_lock) { uint16_t start_idx, from, i; uint16_t nr_cpl_pkts = 0; @@ -3695,7 +3695,7 @@ static __rte_always_inline uint16_t virtio_dev_tx_async_split(struct virtio_net *dev, struct vhost_virtqueue *vq, struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count, int16_t dma_id, uint16_t vchan_id, bool legacy_ol_flags) - __rte_exclusive_locks_required(&vq->access_lock) + __rte_shared_locks_required(&vq->access_lock) __rte_shared_locks_required(&vq->iotlb_lock) { static bool allocerr_warned; @@ -3844,7 +3844,7 @@ virtio_dev_tx_async_split_legacy(struct virtio_net *dev, struct vhost_virtqueue *vq, struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count, int16_t dma_id, uint16_t vchan_id) - __rte_exclusive_locks_required(&vq->access_lock) + __rte_shared_locks_required(&vq->access_lock) __rte_shared_locks_required(&vq->iotlb_lock) { return virtio_dev_tx_async_split(dev, vq, mbuf_pool, @@ -3857,7 +3857,7 @@ virtio_dev_tx_async_split_compliant(struct virtio_net *dev, struct vhost_virtqueue *vq, struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count, int16_t dma_id, uint16_t vchan_id) - __rte_exclusive_locks_required(&vq->access_lock) + __rte_shared_locks_required(&vq->access_lock) __rte_shared_locks_required(&vq->iotlb_lock) { return virtio_dev_tx_async_split(dev, vq, mbuf_pool, @@ -3867,7 +3867,7 @@ virtio_dev_tx_async_split_compliant(struct virtio_net *dev, static __rte_always_inline void vhost_async_shadow_dequeue_single_packed(struct vhost_virtqueue *vq, uint16_t buf_id, uint16_t count) - __rte_exclusive_locks_required(&vq->access_lock) + __rte_shared_locks_required(&vq->access_lock) { struct vhost_async *async = vq->async; uint16_t idx = async->buffer_idx_packed; @@ -3889,7 +3889,7 @@ virtio_dev_tx_async_single_packed(struct virtio_net *dev, struct rte_mbuf *pkts, uint16_t slot_idx, bool legacy_ol_flags) - __rte_exclusive_locks_required(&vq->access_lock) + __rte_shared_locks_required(&vq->access_lock) __rte_shared_locks_required(&vq->iotlb_lock) { int err; @@ -3942,7 +3942,7 @@ virtio_dev_tx_async_packed_batch(struct virtio_net *dev, struct vhost_virtqueue *vq, struct rte_mbuf **pkts, uint16_t slot_idx, uint16_t dma_id, uint16_t vchan_id) - __rte_exclusive_locks_required(&vq->access_lock) + __rte_shared_locks_required(&vq->access_lock) __rte_shared_locks_required(&vq->iotlb_lock) { uint16_t avail_idx = vq->last_avail_idx; @@ -4000,7 +4000,7 @@ static __rte_always_inline uint16_t virtio_dev_tx_async_packed(struct virtio_net *dev, struct vhost_virtqueue *vq, struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count, uint16_t dma_id, uint16_t vchan_id, bool legacy_ol_flags) - __rte_exclusive_locks_required(&vq->access_lock) + __rte_shared_locks_required(&vq->access_lock) __rte_shared_locks_required(&vq->iotlb_lock) { uint32_t pkt_idx = 0; @@ -4111,7 +4111,7 @@ static uint16_t virtio_dev_tx_async_packed_legacy(struct virtio_net *dev, struct vhost_virtqueue *vq, struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count, uint16_t dma_id, uint16_t vchan_id) - __rte_exclusive_locks_required(&vq->access_lock) + __rte_shared_locks_required(&vq->access_lock) __rte_shared_locks_required(&vq->iotlb_lock) { return virtio_dev_tx_async_packed(dev, vq, mbuf_pool, @@ -4123,7 +4123,7 @@ static uint16_t virtio_dev_tx_async_packed_compliant(struct virtio_net *dev, struct vhost_virtqueue *vq, struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count, uint16_t dma_id, uint16_t vchan_id) - __rte_exclusive_locks_required(&vq->access_lock) + __rte_shared_locks_required(&vq->access_lock) __rte_shared_locks_required(&vq->iotlb_lock) { return virtio_dev_tx_async_packed(dev, vq, mbuf_pool, @@ -4173,7 +4173,7 @@ rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id, vq = dev->virtqueue[queue_id]; - if (unlikely(rte_spinlock_trylock(&vq->access_lock) == 0)) + if (unlikely(rte_rwlock_read_trylock(&vq->access_lock) != 0)) return 0; if (unlikely(vq->enabled == 0)) { @@ -4255,7 +4255,7 @@ rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id, vhost_user_iotlb_rd_unlock(vq); out_access_unlock: - rte_spinlock_unlock(&vq->access_lock); + rte_rwlock_read_unlock(&vq->access_lock); if (unlikely(rarp_mbuf != NULL)) count += 1;