From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 36F274552F; Mon, 8 Jul 2024 14:14:39 +0200 (CEST) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id BD6E140ED3; Mon, 8 Jul 2024 14:14:25 +0200 (CEST) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mails.dpdk.org (Postfix) with ESMTP id 6EAE94025E for ; Mon, 8 Jul 2024 14:14:21 +0200 (CEST) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id DCB90139F; Mon, 8 Jul 2024 05:14:45 -0700 (PDT) Received: from ampere-altra-2-3.austin.arm.com (ampere-altra-2-3.austin.arm.com [10.118.14.97]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id B1C8C3F86F; Mon, 8 Jul 2024 05:14:20 -0700 (PDT) From: Yoan Picchi To: Thomas Monjalon , Yipeng Wang , Sameh Gobriel , Bruce Richardson , Vladimir Medvedkin Cc: dev@dpdk.org, nd@arm.com, Yoan Picchi Subject: [PATCH v12 2/7] hash: split compare signature into arch-specific files Date: Mon, 8 Jul 2024 12:14:06 +0000 Message-Id: <20240708121411.885996-3-yoan.picchi@arm.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20240708121411.885996-1-yoan.picchi@arm.com> References: <20231020165159.1649282-1-yoan.picchi@arm.com> <20240708121411.885996-1-yoan.picchi@arm.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Move the compare_signatures function into architecture-specific files They all have the default scalar option as an option if we disable vectorisation. Signed-off-by: Yoan Picchi --- .mailmap | 1 + lib/hash/compare_signatures_arm.h | 55 +++++++++++++++++++++++ lib/hash/compare_signatures_generic.h | 33 ++++++++++++++ lib/hash/compare_signatures_x86.h | 48 ++++++++++++++++++++ lib/hash/rte_cuckoo_hash.c | 65 ++++----------------------- 5 files changed, 145 insertions(+), 57 deletions(-) create mode 100644 lib/hash/compare_signatures_arm.h create mode 100644 lib/hash/compare_signatures_generic.h create mode 100644 lib/hash/compare_signatures_x86.h diff --git a/.mailmap b/.mailmap index f76037213d..ec525981fe 100644 --- a/.mailmap +++ b/.mailmap @@ -1661,6 +1661,7 @@ Yixue Wang Yi Yang Yi Zhang Yoann Desmouceaux +Yoan Picchi Yogesh Jangra Yogev Chaimovich Yongjie Gu diff --git a/lib/hash/compare_signatures_arm.h b/lib/hash/compare_signatures_arm.h new file mode 100644 index 0000000000..80b6afb7a5 --- /dev/null +++ b/lib/hash/compare_signatures_arm.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2010-2016 Intel Corporation + * Copyright(c) 2018-2024 Arm Limited + */ + +#ifndef _COMPARE_SIGNATURE_ARM_PVT_H_ +#define _COMPARE_SIGNATURE_ARM_PVT_H_ + +#include +#include +#include + +#include "rte_cuckoo_hash.h" + +static inline void +compare_signatures(uint32_t *prim_hash_matches, uint32_t *sec_hash_matches, + const struct rte_hash_bucket *prim_bkt, + const struct rte_hash_bucket *sec_bkt, + uint16_t sig, + enum rte_hash_sig_compare_function sig_cmp_fn) +{ + unsigned int i; + + /* For match mask the first bit of every two bits indicates the match */ + switch (sig_cmp_fn) { +#if defined(__ARM_NEON) + case RTE_HASH_COMPARE_NEON: { + uint16x8_t vmat, vsig, x; + int16x8_t shift = {-15, -13, -11, -9, -7, -5, -3, -1}; + + vsig = vld1q_dup_u16((uint16_t const *)&sig); + /* Compare all signatures in the primary bucket */ + vmat = vceqq_u16(vsig, + vld1q_u16((uint16_t const *)prim_bkt->sig_current)); + x = vshlq_u16(vandq_u16(vmat, vdupq_n_u16(0x8000)), shift); + *prim_hash_matches = (uint32_t)(vaddvq_u16(x)); + /* Compare all signatures in the secondary bucket */ + vmat = vceqq_u16(vsig, + vld1q_u16((uint16_t const *)sec_bkt->sig_current)); + x = vshlq_u16(vandq_u16(vmat, vdupq_n_u16(0x8000)), shift); + *sec_hash_matches = (uint32_t)(vaddvq_u16(x)); + } + break; +#endif + default: + for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) { + *prim_hash_matches |= + ((sig == prim_bkt->sig_current[i]) << (i << 1)); + *sec_hash_matches |= + ((sig == sec_bkt->sig_current[i]) << (i << 1)); + } + } +} + +#endif diff --git a/lib/hash/compare_signatures_generic.h b/lib/hash/compare_signatures_generic.h new file mode 100644 index 0000000000..43587adcef --- /dev/null +++ b/lib/hash/compare_signatures_generic.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2010-2016 Intel Corporation + * Copyright(c) 2018-2024 Arm Limited + */ + +#ifndef _COMPARE_SIGNATURE_GENERIC_PVT_H_ +#define _COMPARE_SIGNATURE_GENERIC_PVT_H_ + +#include +#include +#include + +#include "rte_cuckoo_hash.h" + +static inline void +compare_signatures(uint32_t *prim_hash_matches, uint32_t *sec_hash_matches, + const struct rte_hash_bucket *prim_bkt, + const struct rte_hash_bucket *sec_bkt, + uint16_t sig, + enum rte_hash_sig_compare_function sig_cmp_fn) +{ + unsigned int i; + + /* For match mask the first bit of every two bits indicates the match */ + for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) { + *prim_hash_matches |= + ((sig == prim_bkt->sig_current[i]) << (i << 1)); + *sec_hash_matches |= + ((sig == sec_bkt->sig_current[i]) << (i << 1)); + } +} + +#endif diff --git a/lib/hash/compare_signatures_x86.h b/lib/hash/compare_signatures_x86.h new file mode 100644 index 0000000000..11a82aced9 --- /dev/null +++ b/lib/hash/compare_signatures_x86.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2010-2016 Intel Corporation + * Copyright(c) 2018-2024 Arm Limited + */ + +#ifndef _COMPARE_SIGNATURE_X86_PVT_H_ +#define _COMPARE_SIGNATURE_X86_PVT_H_ + +#include +#include +#include + +#include "rte_cuckoo_hash.h" + +static inline void +compare_signatures(uint32_t *prim_hash_matches, uint32_t *sec_hash_matches, + const struct rte_hash_bucket *prim_bkt, + const struct rte_hash_bucket *sec_bkt, + uint16_t sig, + enum rte_hash_sig_compare_function sig_cmp_fn) +{ + unsigned int i; + + /* For match mask the first bit of every two bits indicates the match */ + switch (sig_cmp_fn) { +#if defined(__SSE2__) + case RTE_HASH_COMPARE_SSE: + /* Compare all signatures in the bucket */ + *prim_hash_matches = _mm_movemask_epi8(_mm_cmpeq_epi16(_mm_load_si128( + (__m128i const *)prim_bkt->sig_current), _mm_set1_epi16(sig))); + /* Extract the even-index bits only */ + *prim_hash_matches &= 0x5555; + /* Compare all signatures in the bucket */ + *sec_hash_matches = _mm_movemask_epi8(_mm_cmpeq_epi16(_mm_load_si128( + (__m128i const *)sec_bkt->sig_current), _mm_set1_epi16(sig))); + /* Extract the even-index bits only */ + *sec_hash_matches &= 0x5555; + break; +#endif + default: + for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) { + *prim_hash_matches |= (sig == prim_bkt->sig_current[i]) << (i << 1); + *sec_hash_matches |= (sig == sec_bkt->sig_current[i]) << (i << 1); + } + } +} + +#endif diff --git a/lib/hash/rte_cuckoo_hash.c b/lib/hash/rte_cuckoo_hash.c index e1d50e7d40..c3256dff4c 100644 --- a/lib/hash/rte_cuckoo_hash.c +++ b/lib/hash/rte_cuckoo_hash.c @@ -43,6 +43,14 @@ enum rte_hash_sig_compare_function { RTE_HASH_COMPARE_NUM }; +#if defined(__ARM_NEON) +#include "compare_signatures_arm.h" +#elif defined(__SSE2__) +#include "compare_signatures_x86.h" +#else +#include "compare_signatures_generic.h" +#endif + /* Mask of all flags supported by this version */ #define RTE_HASH_EXTRA_FLAGS_MASK (RTE_HASH_EXTRA_FLAGS_TRANS_MEM_SUPPORT | \ RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD | \ @@ -1890,63 +1898,6 @@ rte_hash_free_key_with_position(const struct rte_hash *h, } -static inline void -compare_signatures(uint32_t *prim_hash_matches, uint32_t *sec_hash_matches, - const struct rte_hash_bucket *prim_bkt, - const struct rte_hash_bucket *sec_bkt, - uint16_t sig, - enum rte_hash_sig_compare_function sig_cmp_fn) -{ - unsigned int i; - - /* For match mask the first bit of every two bits indicates the match */ - switch (sig_cmp_fn) { -#if defined(__SSE2__) - case RTE_HASH_COMPARE_SSE: - /* Compare all signatures in the bucket */ - *prim_hash_matches = _mm_movemask_epi8(_mm_cmpeq_epi16( - _mm_load_si128( - (__m128i const *)prim_bkt->sig_current), - _mm_set1_epi16(sig))); - /* Extract the even-index bits only */ - *prim_hash_matches &= 0x5555; - /* Compare all signatures in the bucket */ - *sec_hash_matches = _mm_movemask_epi8(_mm_cmpeq_epi16( - _mm_load_si128( - (__m128i const *)sec_bkt->sig_current), - _mm_set1_epi16(sig))); - /* Extract the even-index bits only */ - *sec_hash_matches &= 0x5555; - break; -#elif defined(__ARM_NEON) - case RTE_HASH_COMPARE_NEON: { - uint16x8_t vmat, vsig, x; - int16x8_t shift = {-15, -13, -11, -9, -7, -5, -3, -1}; - - vsig = vld1q_dup_u16((uint16_t const *)&sig); - /* Compare all signatures in the primary bucket */ - vmat = vceqq_u16(vsig, - vld1q_u16((uint16_t const *)prim_bkt->sig_current)); - x = vshlq_u16(vandq_u16(vmat, vdupq_n_u16(0x8000)), shift); - *prim_hash_matches = (uint32_t)(vaddvq_u16(x)); - /* Compare all signatures in the secondary bucket */ - vmat = vceqq_u16(vsig, - vld1q_u16((uint16_t const *)sec_bkt->sig_current)); - x = vshlq_u16(vandq_u16(vmat, vdupq_n_u16(0x8000)), shift); - *sec_hash_matches = (uint32_t)(vaddvq_u16(x)); - } - break; -#endif - default: - for (i = 0; i < RTE_HASH_BUCKET_ENTRIES; i++) { - *prim_hash_matches |= - ((sig == prim_bkt->sig_current[i]) << (i << 1)); - *sec_hash_matches |= - ((sig == sec_bkt->sig_current[i]) << (i << 1)); - } - } -} - static inline void __bulk_lookup_l(const struct rte_hash *h, const void **keys, const struct rte_hash_bucket **primary_bkt, -- 2.25.1