From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id AFE6743C03; Tue, 27 Feb 2024 06:44:25 +0100 (CET) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 9A16942F0F; Tue, 27 Feb 2024 06:42:41 +0100 (CET) Received: from linux.microsoft.com (linux.microsoft.com [13.77.154.182]) by mails.dpdk.org (Postfix) with ESMTP id 4D77E42E4B for ; Tue, 27 Feb 2024 06:41:43 +0100 (CET) Received: by linux.microsoft.com (Postfix, from userid 1086) id B88C520B74D2; Mon, 26 Feb 2024 21:41:41 -0800 (PST) DKIM-Filter: OpenDKIM Filter v2.11.0 linux.microsoft.com B88C520B74D2 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.microsoft.com; s=default; t=1709012501; bh=g5QA1cn6H2XRCnwto8cFpq1UfgeAarDzD+gzpktDtNs=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=eshULGuxLBWOn7+WkS+kjdaZWhr6Lposf16XP4P34Zg42UZiu1muLtGaIHBPlnQ6a CSR6bTGHD5uIkvaAjQmmJBlEPlXtB6tuyGdcg63K19BuCziC3vjtokajJaQ3x9PzNg nQWxJiEU0OG9Mug0fK53502UuMSbA8zrmC0E5W20= From: Tyler Retzlaff To: dev@dpdk.org Cc: Ajit Khaparde , Andrew Boyer , Andrew Rybchenko , Bruce Richardson , Chenbo Xia , Chengwen Feng , Dariusz Sosnowski , David Christensen , Hyong Youb Kim , Jerin Jacob , Jie Hai , Jingjing Wu , John Daley , Kevin Laatz , Kiran Kumar K , Konstantin Ananyev , Maciej Czekaj , Matan Azrad , Maxime Coquelin , Nithin Dabilpuram , Ori Kam , Ruifeng Wang , Satha Rao , Somnath Kotur , Suanming Mou , Sunil Kumar Kori , Viacheslav Ovsiienko , Yisen Zhuang , Yuying Zhang , mb@smartsharesystems.com, Tyler Retzlaff Subject: [PATCH v6 18/23] net/virtio: use mbuf descriptor accessors Date: Mon, 26 Feb 2024 21:41:34 -0800 Message-Id: <1709012499-12813-19-git-send-email-roretzla@linux.microsoft.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1709012499-12813-1-git-send-email-roretzla@linux.microsoft.com> References: <1706657173-26166-1-git-send-email-roretzla@linux.microsoft.com> <1709012499-12813-1-git-send-email-roretzla@linux.microsoft.com> X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org RTE_MARKER typedefs are a GCC extension unsupported by MSVC. Use new rte_mbuf_rearm_data and rte_mbuf_rx_descriptor_fields1 accessors that provide a compatible type pointer without using the marker fields. Signed-off-by: Tyler Retzlaff --- drivers/net/virtio/virtio_rxtx_packed.h | 4 ++-- drivers/net/virtio/virtio_rxtx_packed_avx.h | 16 +++++----------- drivers/net/virtio/virtio_rxtx_packed_neon.h | 18 ++++++++---------- drivers/net/virtio/virtio_rxtx_simple.c | 4 +--- drivers/net/virtio/virtio_rxtx_simple.h | 5 +---- drivers/net/virtio/virtio_rxtx_simple_altivec.c | 16 ++++++++-------- drivers/net/virtio/virtio_rxtx_simple_neon.c | 24 ++++++++---------------- drivers/net/virtio/virtio_rxtx_simple_sse.c | 16 ++++++++-------- 8 files changed, 41 insertions(+), 62 deletions(-) diff --git a/drivers/net/virtio/virtio_rxtx_packed.h b/drivers/net/virtio/virtio_rxtx_packed.h index 5361129..0a8c9f8 100644 --- a/drivers/net/virtio/virtio_rxtx_packed.h +++ b/drivers/net/virtio/virtio_rxtx_packed.h @@ -37,12 +37,12 @@ /* reference count offset in mbuf rearm data */ #define REFCNT_BITS_OFFSET ((offsetof(struct rte_mbuf, refcnt) - \ - offsetof(struct rte_mbuf, rearm_data)) * BYTE_SIZE) + offsetof(struct rte_mbuf, data_off)) * BYTE_SIZE) #ifdef CC_AVX512_SUPPORT /* segment number offset in mbuf rearm data */ #define SEG_NUM_BITS_OFFSET ((offsetof(struct rte_mbuf, nb_segs) - \ - offsetof(struct rte_mbuf, rearm_data)) * BYTE_SIZE) + offsetof(struct rte_mbuf, data_off)) * BYTE_SIZE) /* default rearm data */ #define DEFAULT_REARM_DATA (1ULL << SEG_NUM_BITS_OFFSET | \ 1ULL << REFCNT_BITS_OFFSET) diff --git a/drivers/net/virtio/virtio_rxtx_packed_avx.h b/drivers/net/virtio/virtio_rxtx_packed_avx.h index 584ac72..5d30e2a 100644 --- a/drivers/net/virtio/virtio_rxtx_packed_avx.h +++ b/drivers/net/virtio/virtio_rxtx_packed_avx.h @@ -36,10 +36,10 @@ /* Load four mbufs rearm data */ RTE_BUILD_BUG_ON(REFCNT_BITS_OFFSET >= 64); RTE_BUILD_BUG_ON(SEG_NUM_BITS_OFFSET >= 64); - __m256i mbufs = _mm256_set_epi64x(*tx_pkts[3]->rearm_data, - *tx_pkts[2]->rearm_data, - *tx_pkts[1]->rearm_data, - *tx_pkts[0]->rearm_data); + __m256i mbufs = _mm256_set_epi64x(*rte_mbuf_rearm_data(tx_pkts[3]), + *rte_mbuf_rearm_data(tx_pkts[2]), + *rte_mbuf_rearm_data(tx_pkts[1]), + *rte_mbuf_rearm_data(tx_pkts[0])); /* refcnt=1 and nb_segs=1 */ __m256i mbuf_ref = _mm256_set1_epi64x(DEFAULT_REARM_DATA); @@ -53,8 +53,6 @@ /* Check headroom is enough */ const __mmask16 data_mask = 0x1 | 0x1 << 4 | 0x1 << 8 | 0x1 << 12; - RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_off) != - offsetof(struct rte_mbuf, rearm_data)); cmp = _mm256_mask_cmplt_epu16_mask(data_mask, mbufs, head_rooms); if (unlikely(cmp)) return -1; @@ -187,7 +185,7 @@ rx_pkts[i] = (struct rte_mbuf *)vq->vq_descx[id + i].cookie; rte_packet_prefetch(rte_pktmbuf_mtod(rx_pkts[i], void *)); - addrs[i] = (uintptr_t)rx_pkts[i]->rx_descriptor_fields1; + addrs[i] = (uintptr_t)rte_mbuf_rx_descriptor_fields1(rx_pkts[i]); } /* @@ -203,10 +201,6 @@ __m512i v_value = _mm512_add_epi32(values, mbuf_len_offset); - /* assert offset of data_len */ - RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) != - offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8); - __m512i v_index = _mm512_set_epi64(addrs[3] + 8, addrs[3], addrs[2] + 8, addrs[2], addrs[1] + 8, addrs[1], diff --git a/drivers/net/virtio/virtio_rxtx_packed_neon.h b/drivers/net/virtio/virtio_rxtx_packed_neon.h index c222ebf..c172fad 100644 --- a/drivers/net/virtio/virtio_rxtx_packed_neon.h +++ b/drivers/net/virtio/virtio_rxtx_packed_neon.h @@ -59,10 +59,10 @@ uint8x16x2_t mbuf; /* Load four mbufs rearm data. */ RTE_BUILD_BUG_ON(REFCNT_BITS_OFFSET >= 64); - pkts[0] = vld1_u16((uint16_t *)&tx_pkts[0]->rearm_data); - pkts[1] = vld1_u16((uint16_t *)&tx_pkts[1]->rearm_data); - pkts[2] = vld1_u16((uint16_t *)&tx_pkts[2]->rearm_data); - pkts[3] = vld1_u16((uint16_t *)&tx_pkts[3]->rearm_data); + pkts[0] = vld1_u16((uint16_t *)rte_mbuf_rearm_data(tx_pkts[0])); + pkts[1] = vld1_u16((uint16_t *)rte_mbuf_rearm_data(tx_pkts[1])); + pkts[2] = vld1_u16((uint16_t *)rte_mbuf_rearm_data(tx_pkts[2])); + pkts[3] = vld1_u16((uint16_t *)rte_mbuf_rearm_data(tx_pkts[3])); mbuf.val[0] = vreinterpretq_u8_u16(vcombine_u16(pkts[0], pkts[1])); mbuf.val[1] = vreinterpretq_u8_u16(vcombine_u16(pkts[2], pkts[3])); @@ -77,8 +77,6 @@ /* Check headroom is enough. */ uint16x4_t head_rooms = vdup_n_u16(head_size); - RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_off) != - offsetof(struct rte_mbuf, rearm_data)); uint16x4_t data_offset = vreinterpret_u16_u8(vqtbl2_u8(mbuf, data_msk)); uint64x1_t cmp2 = vreinterpret_u64_u16(vclt_u16(data_offset, head_rooms)); if (unlikely(vget_lane_u64(cmp2, 0))) @@ -263,10 +261,10 @@ pkt_mb[3] = vreinterpretq_u64_u16(vsubq_u16( vreinterpretq_u16_u64(pkt_mb[3]), len_adjust)); - vst1q_u64((void *)&rx_pkts[0]->rx_descriptor_fields1, pkt_mb[0]); - vst1q_u64((void *)&rx_pkts[1]->rx_descriptor_fields1, pkt_mb[1]); - vst1q_u64((void *)&rx_pkts[2]->rx_descriptor_fields1, pkt_mb[2]); - vst1q_u64((void *)&rx_pkts[3]->rx_descriptor_fields1, pkt_mb[3]); + vst1q_u64(rte_mbuf_rx_descriptor_fields1(rx_pkts[0]), pkt_mb[0]); + vst1q_u64(rte_mbuf_rx_descriptor_fields1(rx_pkts[1]), pkt_mb[1]); + vst1q_u64(rte_mbuf_rx_descriptor_fields1(rx_pkts[2]), pkt_mb[2]); + vst1q_u64(rte_mbuf_rx_descriptor_fields1(rx_pkts[3]), pkt_mb[3]); if (hw->has_rx_offload) { virtio_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) { diff --git a/drivers/net/virtio/virtio_rxtx_simple.c b/drivers/net/virtio/virtio_rxtx_simple.c index 4382569..e6a82e5 100644 --- a/drivers/net/virtio/virtio_rxtx_simple.c +++ b/drivers/net/virtio/virtio_rxtx_simple.c @@ -31,7 +31,6 @@ virtio_rxq_vec_setup(struct virtnet_rx *rxq) { struct virtqueue *vq = virtnet_rxq_to_vq(rxq); - uintptr_t p; struct rte_mbuf mb_def = { .buf_addr = 0 }; /* zeroed mbuf */ mb_def.nb_segs = 1; @@ -41,8 +40,7 @@ /* prevent compiler reordering: rearm_data covers previous fields */ rte_compiler_barrier(); - p = (uintptr_t)&mb_def.rearm_data; - rxq->mbuf_initializer = *(uint64_t *)p; + rxq->mbuf_initializer = *rte_mbuf_rearm_data(&mb_def); return 0; } diff --git a/drivers/net/virtio/virtio_rxtx_simple.h b/drivers/net/virtio/virtio_rxtx_simple.h index 79196ed..16fef0e 100644 --- a/drivers/net/virtio/virtio_rxtx_simple.h +++ b/drivers/net/virtio/virtio_rxtx_simple.h @@ -39,10 +39,7 @@ } for (i = 0; i < RTE_VIRTIO_VPMD_RX_REARM_THRESH; i++) { - uintptr_t p; - - p = (uintptr_t)&sw_ring[i]->rearm_data; - *(uint64_t *)p = rxvq->mbuf_initializer; + *rte_mbuf_rearm_data(sw_ring[i]) = rxvq->mbuf_initializer; start_dp[i].addr = VIRTIO_MBUF_ADDR(sw_ring[i], vq) + RTE_PKTMBUF_HEADROOM - vq->hw->vtnet_hdr_size; diff --git a/drivers/net/virtio/virtio_rxtx_simple_altivec.c b/drivers/net/virtio/virtio_rxtx_simple_altivec.c index 542ec3d..39ec1ca 100644 --- a/drivers/net/virtio/virtio_rxtx_simple_altivec.c +++ b/drivers/net/virtio/virtio_rxtx_simple_altivec.c @@ -138,9 +138,9 @@ ((__vector unsigned short)pkt_mb[0] + len_adjust); pkt_mb[1] = (__vector unsigned char) ((__vector unsigned short)pkt_mb[1] + len_adjust); - *(__vector unsigned char *)&rx_pkts[0]->rx_descriptor_fields1 = + *(__vector unsigned char *)rte_mbuf_rx_descriptor_fields1(rx_pkts[0]) = pkt_mb[0]; - *(__vector unsigned char *)&rx_pkts[1]->rx_descriptor_fields1 = + *(__vector unsigned char *)rte_mbuf_rx_descriptor_fields1(rx_pkts[1]) = pkt_mb[1]; pkt_mb[2] = vec_perm(desc[1], zero, shuf_msk1); @@ -149,9 +149,9 @@ ((__vector unsigned short)pkt_mb[2] + len_adjust); pkt_mb[3] = (__vector unsigned char) ((__vector unsigned short)pkt_mb[3] + len_adjust); - *(__vector unsigned char *)&rx_pkts[2]->rx_descriptor_fields1 = + *(__vector unsigned char *)rte_mbuf_rx_descriptor_fields1(rx_pkts[2]) = pkt_mb[2]; - *(__vector unsigned char *)&rx_pkts[3]->rx_descriptor_fields1 = + *(__vector unsigned char *)rte_mbuf_rx_descriptor_fields1(rx_pkts[3]) = pkt_mb[3]; pkt_mb[4] = vec_perm(desc[2], zero, shuf_msk1); @@ -160,9 +160,9 @@ ((__vector unsigned short)pkt_mb[4] + len_adjust); pkt_mb[5] = (__vector unsigned char) ((__vector unsigned short)pkt_mb[5] + len_adjust); - *(__vector unsigned char *)&rx_pkts[4]->rx_descriptor_fields1 = + *(__vector unsigned char *)rte_mbuf_rx_descriptor_fields1(rx_pkts[4]) = pkt_mb[4]; - *(__vector unsigned char *)&rx_pkts[5]->rx_descriptor_fields1 = + *(__vector unsigned char *)rte_mbuf_rx_descriptor_fields1(rx_pkts[5]) = pkt_mb[5]; pkt_mb[6] = vec_perm(desc[3], zero, shuf_msk1); @@ -171,9 +171,9 @@ ((__vector unsigned short)pkt_mb[6] + len_adjust); pkt_mb[7] = (__vector unsigned char) ((__vector unsigned short)pkt_mb[7] + len_adjust); - *(__vector unsigned char *)&rx_pkts[6]->rx_descriptor_fields1 = + *(__vector unsigned char *)rte_mbuf_rx_descriptor_fields1(rx_pkts[6]) = pkt_mb[6]; - *(__vector unsigned char *)&rx_pkts[7]->rx_descriptor_fields1 = + *(__vector unsigned char *)rte_mbuf_rx_descriptor_fields1(rx_pkts[7]) = pkt_mb[7]; if (unlikely(nb_used <= RTE_VIRTIO_DESC_PER_LOOP)) { diff --git a/drivers/net/virtio/virtio_rxtx_simple_neon.c b/drivers/net/virtio/virtio_rxtx_simple_neon.c index 7139b31..873b42c 100644 --- a/drivers/net/virtio/virtio_rxtx_simple_neon.c +++ b/drivers/net/virtio/virtio_rxtx_simple_neon.c @@ -138,10 +138,8 @@ vreinterpretq_u16_u64(pkt_mb[1]), len_adjust)); pkt_mb[0] = vreinterpretq_u64_u16(vsubq_u16( vreinterpretq_u16_u64(pkt_mb[0]), len_adjust)); - vst1q_u64((void *)&rx_pkts[1]->rx_descriptor_fields1, - pkt_mb[1]); - vst1q_u64((void *)&rx_pkts[0]->rx_descriptor_fields1, - pkt_mb[0]); + vst1q_u64(rte_mbuf_rx_descriptor_fields1(rx_pkts[1]), pkt_mb[1]); + vst1q_u64(rte_mbuf_rx_descriptor_fields1(rx_pkts[0]), pkt_mb[0]); pkt_mb[3] = vreinterpretq_u64_u8(vqtbl1q_u8( vreinterpretq_u8_u64(desc[1]), shuf_msk2)); @@ -151,10 +149,8 @@ vreinterpretq_u16_u64(pkt_mb[3]), len_adjust)); pkt_mb[2] = vreinterpretq_u64_u16(vsubq_u16( vreinterpretq_u16_u64(pkt_mb[2]), len_adjust)); - vst1q_u64((void *)&rx_pkts[3]->rx_descriptor_fields1, - pkt_mb[3]); - vst1q_u64((void *)&rx_pkts[2]->rx_descriptor_fields1, - pkt_mb[2]); + vst1q_u64(rte_mbuf_rx_descriptor_fields1(rx_pkts[3]), pkt_mb[3]); + vst1q_u64(rte_mbuf_rx_descriptor_fields1(rx_pkts[2]), pkt_mb[2]); pkt_mb[5] = vreinterpretq_u64_u8(vqtbl1q_u8( vreinterpretq_u8_u64(desc[2]), shuf_msk2)); @@ -164,10 +160,8 @@ vreinterpretq_u16_u64(pkt_mb[5]), len_adjust)); pkt_mb[4] = vreinterpretq_u64_u16(vsubq_u16( vreinterpretq_u16_u64(pkt_mb[4]), len_adjust)); - vst1q_u64((void *)&rx_pkts[5]->rx_descriptor_fields1, - pkt_mb[5]); - vst1q_u64((void *)&rx_pkts[4]->rx_descriptor_fields1, - pkt_mb[4]); + vst1q_u64(rte_mbuf_rx_descriptor_fields1(rx_pkts[5]), pkt_mb[5]); + vst1q_u64(rte_mbuf_rx_descriptor_fields1(rx_pkts[4]), pkt_mb[4]); pkt_mb[7] = vreinterpretq_u64_u8(vqtbl1q_u8( vreinterpretq_u8_u64(desc[3]), shuf_msk2)); @@ -177,10 +171,8 @@ vreinterpretq_u16_u64(pkt_mb[7]), len_adjust)); pkt_mb[6] = vreinterpretq_u64_u16(vsubq_u16( vreinterpretq_u16_u64(pkt_mb[6]), len_adjust)); - vst1q_u64((void *)&rx_pkts[7]->rx_descriptor_fields1, - pkt_mb[7]); - vst1q_u64((void *)&rx_pkts[6]->rx_descriptor_fields1, - pkt_mb[6]); + vst1q_u64(rte_mbuf_rx_descriptor_fields1(rx_pkts[7]), pkt_mb[7]); + vst1q_u64(rte_mbuf_rx_descriptor_fields1(rx_pkts[6]), pkt_mb[6]); if (unlikely(nb_used <= RTE_VIRTIO_DESC_PER_LOOP)) { if (sw_ring + nb_used <= sw_ring_end) diff --git a/drivers/net/virtio/virtio_rxtx_simple_sse.c b/drivers/net/virtio/virtio_rxtx_simple_sse.c index 6a18741..25db486 100644 --- a/drivers/net/virtio/virtio_rxtx_simple_sse.c +++ b/drivers/net/virtio/virtio_rxtx_simple_sse.c @@ -134,36 +134,36 @@ pkt_mb[0] = _mm_shuffle_epi8(desc[0], shuf_msk1); pkt_mb[1] = _mm_add_epi16(pkt_mb[1], len_adjust); pkt_mb[0] = _mm_add_epi16(pkt_mb[0], len_adjust); - _mm_storeu_si128((void *)&rx_pkts[1]->rx_descriptor_fields1, + _mm_storeu_si128(rte_mbuf_rx_descriptor_fields1(rx_pkts[1]), pkt_mb[1]); - _mm_storeu_si128((void *)&rx_pkts[0]->rx_descriptor_fields1, + _mm_storeu_si128(rte_mbuf_rx_descriptor_fields1(rx_pkts[0]), pkt_mb[0]); pkt_mb[3] = _mm_shuffle_epi8(desc[1], shuf_msk2); pkt_mb[2] = _mm_shuffle_epi8(desc[1], shuf_msk1); pkt_mb[3] = _mm_add_epi16(pkt_mb[3], len_adjust); pkt_mb[2] = _mm_add_epi16(pkt_mb[2], len_adjust); - _mm_storeu_si128((void *)&rx_pkts[3]->rx_descriptor_fields1, + _mm_storeu_si128(rte_mbuf_rx_descriptor_fields1(rx_pkts[3]), pkt_mb[3]); - _mm_storeu_si128((void *)&rx_pkts[2]->rx_descriptor_fields1, + _mm_storeu_si128(rte_mbuf_rx_descriptor_fields1(rx_pkts[2]), pkt_mb[2]); pkt_mb[5] = _mm_shuffle_epi8(desc[2], shuf_msk2); pkt_mb[4] = _mm_shuffle_epi8(desc[2], shuf_msk1); pkt_mb[5] = _mm_add_epi16(pkt_mb[5], len_adjust); pkt_mb[4] = _mm_add_epi16(pkt_mb[4], len_adjust); - _mm_storeu_si128((void *)&rx_pkts[5]->rx_descriptor_fields1, + _mm_storeu_si128(rte_mbuf_rx_descriptor_fields1(rx_pkts[5]), pkt_mb[5]); - _mm_storeu_si128((void *)&rx_pkts[4]->rx_descriptor_fields1, + _mm_storeu_si128(rte_mbuf_rx_descriptor_fields1(rx_pkts[4]), pkt_mb[4]); pkt_mb[7] = _mm_shuffle_epi8(desc[3], shuf_msk2); pkt_mb[6] = _mm_shuffle_epi8(desc[3], shuf_msk1); pkt_mb[7] = _mm_add_epi16(pkt_mb[7], len_adjust); pkt_mb[6] = _mm_add_epi16(pkt_mb[6], len_adjust); - _mm_storeu_si128((void *)&rx_pkts[7]->rx_descriptor_fields1, + _mm_storeu_si128(rte_mbuf_rx_descriptor_fields1(rx_pkts[7]), pkt_mb[7]); - _mm_storeu_si128((void *)&rx_pkts[6]->rx_descriptor_fields1, + _mm_storeu_si128(rte_mbuf_rx_descriptor_fields1(rx_pkts[6]), pkt_mb[6]); if (unlikely(nb_used <= RTE_VIRTIO_DESC_PER_LOOP)) { -- 1.8.3.1