From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 47D58A04BC; Tue, 29 Sep 2020 17:36:21 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 32A221D6FD; Tue, 29 Sep 2020 17:36:14 +0200 (CEST) Received: from mga09.intel.com (mga09.intel.com [134.134.136.24]) by dpdk.org (Postfix) with ESMTP id 870331D6F9 for ; Tue, 29 Sep 2020 17:36:12 +0200 (CEST) IronPort-SDR: kcPi5F2/jFuHwSPU0gn6x9OgXCokCL2g+okEsyN0vKCgX7nnBUuLmAf30RL2U9i9+4uRw4M1GG c6kthCYaGWPQ== X-IronPort-AV: E=McAfee;i="6000,8403,9759"; a="163091592" X-IronPort-AV: E=Sophos;i="5.77,319,1596524400"; d="scan'208";a="163091592" X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga001.fm.intel.com ([10.253.24.23]) by orsmga102.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 29 Sep 2020 08:36:08 -0700 IronPort-SDR: Dyh2aqmRAVjfd06zsosonqzi81yvqAxZxNnr5XX8udUf8AheZwO/hdYJ41q+XkQ+M1rDfFQjoY wYhSXphjeTMA== X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.77,319,1596524400"; d="scan'208";a="415413406" Received: from irvmail001.ir.intel.com ([163.33.26.43]) by fmsmga001.fm.intel.com with ESMTP; 29 Sep 2020 08:36:07 -0700 Received: from sivswdev08.ir.intel.com (sivswdev08.ir.intel.com [10.237.217.47]) by irvmail001.ir.intel.com (8.14.3/8.13.6/MailSET/Hub) with ESMTP id 08TFa65a010461; Tue, 29 Sep 2020 16:36:06 +0100 Received: from sivswdev08.ir.intel.com (localhost [127.0.0.1]) by sivswdev08.ir.intel.com with ESMTP id 08TFa6KN011868; Tue, 29 Sep 2020 16:36:06 +0100 Received: (from moloings@localhost) by sivswdev08.ir.intel.com with LOCAL id 08TFa6VU011864; Tue, 29 Sep 2020 16:36:06 +0100 From: Mairtin o Loingsigh To: jasvinder.singh@intel.com, bruce.richardson@intel.com, pablo.de.lara.guarch@intel.com Cc: dev@dpdk.org, brendan.ryan@intel.com, david.coyle@intel.com, Mairtin o Loingsigh Date: Tue, 29 Sep 2020 16:36:01 +0100 Message-Id: <1601393761-11588-3-git-send-email-mairtin.oloingsigh@intel.com> X-Mailer: git-send-email 1.7.0.7 In-Reply-To: <1601393761-11588-1-git-send-email-mairtin.oloingsigh@intel.com> References: <1601393761-11588-1-git-send-email-mairtin.oloingsigh@intel.com> Subject: [dpdk-dev] [PATCH v3 2/2] net: add support for AVX512/VPCLMULQDQ based CRC X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" This patch enables the optimized calculation of CRC32-Ethernet and CRC16-CCITT using the AVX512 and VPCLMULQDQ instruction sets. This CRC implementation is built if the compiler supports the required instruction sets. It is selected at run-time if the host CPU, again, supports the required instruction sets. Signed-off-by: Mairtin o Loingsigh Signed-off-by: David Coyle --- app/test/test_crc.c | 11 +- config/x86/meson.build | 6 +- doc/guides/rel_notes/release_20_11.rst | 2 + lib/librte_net/meson.build | 55 +++++ lib/librte_net/net_crc.h | 11 + lib/librte_net/net_crc_avx512.c | 424 +++++++++++++++++++++++++++++++++ lib/librte_net/rte_net_crc.c | 33 +++ lib/librte_net/rte_net_crc.h | 4 +- 8 files changed, 542 insertions(+), 4 deletions(-) create mode 100644 lib/librte_net/net_crc_avx512.c diff --git a/app/test/test_crc.c b/app/test/test_crc.c index f8a74e04e..bf1d34435 100644 --- a/app/test/test_crc.c +++ b/app/test/test_crc.c @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause - * Copyright(c) 2017 Intel Corporation + * Copyright(c) 2017-2020 Intel Corporation */ #include "test.h" @@ -149,6 +149,15 @@ test_crc(void) return ret; } + /* set CRC avx512 mode */ + rte_net_crc_set_alg(RTE_NET_CRC_AVX512); + + ret = test_crc_calc(); + if (ret < 0) { + printf("test crc (x86_64 AVX512): failed (%d)\n", ret); + return ret; + } + /* set CRC neon mode */ rte_net_crc_set_alg(RTE_NET_CRC_NEON); diff --git a/config/x86/meson.build b/config/x86/meson.build index fea4d5403..172b72b72 100644 --- a/config/x86/meson.build +++ b/config/x86/meson.build @@ -1,5 +1,5 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright(c) 2017-2019 Intel Corporation +# Copyright(c) 2017-2020 Intel Corporation # get binutils version for the workaround of Bug 97 if not is_windows @@ -23,7 +23,9 @@ endforeach optional_flags = ['AES', 'PCLMUL', 'AVX', 'AVX2', 'AVX512F', - 'RDRND', 'RDSEED'] + 'RDRND', 'RDSEED', + 'AVX512BW', 'AVX512DQ', + 'AVX512VL', 'VPCLMULQDQ'] foreach f:optional_flags if cc.get_define('__@0@__'.format(f), args: machine_args) == '1' if f == 'PCLMUL' # special case flags with different defines diff --git a/doc/guides/rel_notes/release_20_11.rst b/doc/guides/rel_notes/release_20_11.rst index 6bd222dca..509749ebd 100644 --- a/doc/guides/rel_notes/release_20_11.rst +++ b/doc/guides/rel_notes/release_20_11.rst @@ -58,6 +58,8 @@ New Features * **Updated CRC modules of rte_net library.** * Added run-time selection of the optimal architecture-specific CRC path. + * Added optimized implementations of CRC32-Ethernet and CRC16-CCITT + using the AVX512 and VPCLMULQDQ instruction sets. * **Updated Cisco enic driver.** diff --git a/lib/librte_net/meson.build b/lib/librte_net/meson.build index b6880bd85..eeae25bc1 100644 --- a/lib/librte_net/meson.build +++ b/lib/librte_net/meson.build @@ -24,18 +24,62 @@ deps += ['mbuf'] if dpdk_conf.has('RTE_ARCH_X86_64') net_crc_sse42_cpu_support = \ cc.get_define('__PCLMUL__', args: machine_args) != '' + net_crc_avx512_cpu_support = \ + cc.get_define('__AVX512F__', args: machine_args) != '' and \ + cc.get_define('__AVX512BW__', args: machine_args) != '' and \ + cc.get_define('__AVX512DQ__', args: machine_args) != '' and \ + cc.get_define('__AVX512VL__', args: machine_args) != '' and \ + cc.get_define('__VPCLMULQDQ__', args: machine_args) != '' + net_crc_sse42_cc_support = \ cc.has_argument('-mpclmul') and cc.has_argument('-maes') + net_crc_avx512_cc_support = \ + not machine_args.contains('-mno-avx512f') and \ + cc.has_argument('-mavx512f') and \ + cc.has_argument('-mavx512bw') and \ + cc.has_argument('-mavx512dq') and \ + cc.has_argument('-mavx512vl') and \ + cc.has_argument('-mvpclmulqdq') and \ + cc.has_argument('-mavx2') and \ + cc.has_argument('-mavx') build_static_net_crc_sse42_lib = 0 + build_static_net_crc_avx512_lib = 0 if net_crc_sse42_cpu_support == true sources += files('net_crc_sse.c') cflags += ['-DCC_X86_64_SSE42_PCLMULQDQ_SUPPORT'] + if net_crc_avx512_cpu_support == true + sources += files('net_crc_avx512.c') + cflags += ['-DCC_X86_64_AVX512_VPCLMULQDQ_SUPPORT'] + elif net_crc_avx512_cc_support == true + build_static_net_crc_avx512_lib = 1 + net_crc_avx512_lib_cflags = ['-mavx512f', + '-mavx512bw', + '-mavx512dq', + '-mavx512vl', + '-mvpclmulqdq', + '-mavx2', + '-mavx'] + cflags += ['-DCC_X86_64_AVX512_VPCLMULQDQ_SUPPORT'] + endif elif net_crc_sse42_cc_support == true build_static_net_crc_sse42_lib = 1 net_crc_sse42_lib_cflags = ['-mpclmul', '-maes'] cflags += ['-DCC_X86_64_SSE42_PCLMULQDQ_SUPPORT'] + if net_crc_avx512_cc_support == true + build_static_net_crc_avx512_lib = 1 + net_crc_avx512_lib_cflags = ['-mpclmul', + '-maes', + '-mavx512f', + '-mavx512bw', + '-mavx512dq', + '-mavx512vl', + '-mvpclmulqdq', + '-mavx2', + '-mavx'] + cflags += ['-DCC_X86_64_AVX512_VPCLMULQDQ_SUPPORT'] + endif endif if build_static_net_crc_sse42_lib == 1 @@ -47,6 +91,17 @@ if dpdk_conf.has('RTE_ARCH_X86_64') net_crc_sse42_lib_cflags]) objs += net_crc_sse42_lib.extract_objects('net_crc_sse.c') endif + + if build_static_net_crc_avx512_lib == 1 + net_crc_avx512_lib = static_library( + 'net_crc_avx512_lib', + 'net_crc_avx512.c', + dependencies: static_rte_eal, + c_args: [cflags, + net_crc_avx512_lib_cflags]) + objs += net_crc_avx512_lib.extract_objects('net_crc_avx512.c') + endif + elif dpdk_conf.has('RTE_ARCH_ARM64') and \ cc.get_define('__ARM_FEATURE_CRYPTO', args: machine_args) != '' sources += files('net_crc_neon.c') diff --git a/lib/librte_net/net_crc.h b/lib/librte_net/net_crc.h index a1578a56c..7a74d5406 100644 --- a/lib/librte_net/net_crc.h +++ b/lib/librte_net/net_crc.h @@ -20,6 +20,17 @@ rte_crc16_ccitt_sse42_handler(const uint8_t *data, uint32_t data_len); uint32_t rte_crc32_eth_sse42_handler(const uint8_t *data, uint32_t data_len); +/* AVX512 */ + +void +rte_net_crc_avx512_init(void); + +uint32_t +rte_crc16_ccitt_avx512_handler(const uint8_t *data, uint32_t data_len); + +uint32_t +rte_crc32_eth_avx512_handler(const uint8_t *data, uint32_t data_len); + /* NEON */ void diff --git a/lib/librte_net/net_crc_avx512.c b/lib/librte_net/net_crc_avx512.c new file mode 100644 index 000000000..81aac6349 --- /dev/null +++ b/lib/librte_net/net_crc_avx512.c @@ -0,0 +1,424 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2020 Intel Corporation + */ + +#include + +#include +#include +#include + +#include "net_crc.h" + +#include + +/* VPCLMULQDQ CRC computation context structure */ +struct crc_vpclmulqdq_ctx { + __m512i rk1_rk2; + __m512i rk3_rk4; + __m512i fold_7x128b; + __m512i fold_3x128b; + __m128i rk5_rk6; + __m128i rk7_rk8; + __m128i fold_1x128b; +}; + +static struct crc_vpclmulqdq_ctx crc32_eth __rte_aligned(64); +static struct crc_vpclmulqdq_ctx crc16_ccitt __rte_aligned(64); + +static uint16_t byte_len_to_mask_table[] = { + 0x0000, 0x0001, 0x0003, 0x0007, + 0x000f, 0x001f, 0x003f, 0x007f, + 0x00ff, 0x01ff, 0x03ff, 0x07ff, + 0x0fff, 0x1fff, 0x3fff, 0x7fff, + 0xffff}; + +static const uint8_t shf_table[32] __rte_aligned(16) = { + 0x00, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f +}; + +static const uint32_t mask[4] __rte_aligned(16) = { + 0xffffffff, 0xffffffff, 0x00000000, 0x00000000 +}; + +static const uint32_t mask2[4] __rte_aligned(16) = { + 0x00000000, 0xffffffff, 0xffffffff, 0xffffffff +}; + +static __rte_always_inline __m512i +crcr32_folding_round(__m512i data_block, __m512i precomp, __m512i fold) +{ + __m512i tmp0, tmp1; + + tmp0 = _mm512_clmulepi64_epi128(fold, precomp, 0x01); + tmp1 = _mm512_clmulepi64_epi128(fold, precomp, 0x10); + + return _mm512_ternarylogic_epi64(tmp0, tmp1, data_block, 0x96); +} + +static __rte_always_inline __m128i +crc32_fold_128(__m512i fold0, __m512i fold1, + const struct crc_vpclmulqdq_ctx *params) +{ + __m128i res, res2; + __m256i a; + __m512i tmp0, tmp1, tmp2, tmp3; + __m512i tmp4; + + tmp0 = _mm512_clmulepi64_epi128(fold0, params->fold_7x128b, 0x01); + tmp1 = _mm512_clmulepi64_epi128(fold0, params->fold_7x128b, 0x10); + + res = _mm512_extracti64x2_epi64(fold1, 3); + tmp4 = _mm512_maskz_broadcast_i32x4(0xF, res); + + tmp2 = _mm512_clmulepi64_epi128(fold1, params->fold_3x128b, 0x01); + tmp3 = _mm512_clmulepi64_epi128(fold1, params->fold_3x128b, 0x10); + + tmp0 = _mm512_ternarylogic_epi64(tmp0, tmp1, tmp2, 0x96); + tmp0 = _mm512_ternarylogic_epi64(tmp0, tmp3, tmp4, 0x96); + + tmp1 = _mm512_shuffle_i64x2(tmp0, tmp0, 0x4e); + + a = _mm256_xor_si256(*(__m256i *)&tmp1, *(__m256i *)&tmp0); + res = _mm256_extracti64x2_epi64(a, 1); + res2 = _mm_xor_si128(res, *(__m128i *)&a); + + return res2; +} + +static __rte_always_inline __m128i +last_two_xmm(const uint8_t *data, uint32_t data_len, uint32_t n, __m128i res, + const struct crc_vpclmulqdq_ctx *params) +{ + uint32_t offset; + __m128i res2, res3, res4, pshufb_shf; + + const uint32_t mask3[4] __rte_aligned(16) = { + 0x80808080, 0x80808080, 0x80808080, 0x80808080 + }; + + res2 = res; + offset = data_len - n; + res3 = _mm_loadu_si128((const __m128i *)&data[n+offset-16]); + + pshufb_shf = _mm_loadu_si128((const __m128i *) + (shf_table + (data_len-n))); + + res = _mm_shuffle_epi8(res, pshufb_shf); + pshufb_shf = _mm_xor_si128(pshufb_shf, + _mm_load_si128((const __m128i *) mask3)); + res2 = _mm_shuffle_epi8(res2, pshufb_shf); + + res2 = _mm_blendv_epi8(res2, res3, pshufb_shf); + + res4 = _mm_clmulepi64_si128(res, params->fold_1x128b, 0x01); + res = _mm_clmulepi64_si128(res, params->fold_1x128b, 0x10); + res = _mm_ternarylogic_epi64(res, res2, res4, 0x96); + + return res; +} + +static __rte_always_inline __m128i +done_128(__m128i res, const struct crc_vpclmulqdq_ctx *params) +{ + __m128i res1; + + res1 = res; + + res = _mm_clmulepi64_si128(res, params->rk5_rk6, 0x0); + res1 = _mm_srli_si128(res1, 8); + res = _mm_xor_si128(res, res1); + + res1 = res; + res = _mm_slli_si128(res, 4); + res = _mm_clmulepi64_si128(res, params->rk5_rk6, 0x10); + res = _mm_xor_si128(res, res1); + + return res; +} + +static __rte_always_inline uint32_t +barrett_reduction(__m128i data64, const struct crc_vpclmulqdq_ctx *params) +{ + __m128i tmp0, tmp1; + + data64 = _mm_and_si128(data64, *(const __m128i *)mask2); + tmp0 = data64; + tmp1 = data64; + + data64 = _mm_clmulepi64_si128(tmp0, params->rk7_rk8, 0x0); + data64 = _mm_ternarylogic_epi64(data64, tmp1, *(const __m128i *)mask, + 0x28); + + tmp1 = data64; + data64 = _mm_clmulepi64_si128(data64, params->rk7_rk8, 0x10); + data64 = _mm_ternarylogic_epi64(data64, tmp1, tmp0, 0x96); + + return _mm_extract_epi32(data64, 2); +} + +static __rte_always_inline void +reduction_loop(__m128i *fold, int *len, const uint8_t *data, uint32_t *n, + const struct crc_vpclmulqdq_ctx *params) +{ + __m128i tmp, tmp1; + + tmp = _mm_clmulepi64_si128(*fold, params->fold_1x128b, 0x1); + *fold = _mm_clmulepi64_si128(*fold, params->fold_1x128b, 0x10); + *fold = _mm_xor_si128(*fold, tmp); + tmp1 = _mm_loadu_si128((const __m128i *)&data[*n]); + *fold = _mm_xor_si128(*fold, tmp1); + *n += 16; + *len -= 16; +} + +static __rte_always_inline uint32_t +crc32_eth_calc_vpclmulqdq(const uint8_t *data, uint32_t data_len, uint32_t crc, + const struct crc_vpclmulqdq_ctx *params) +{ + __m128i res, d; + __m256i b; + __m512i temp, k; + __m512i qw0 = _mm512_set1_epi64(0), qw1, qw2, qw3; + __m512i fold0, fold1, fold2, fold3; + __mmask16 mask; + uint32_t n = 0; + int reduction = 0; + + /* Get CRC init value */ + b = _mm256_insert_epi32(_mm256_setzero_si256(), crc, 0); + temp = _mm512_inserti32x8(_mm512_setzero_si512(), b, 0); + + if (data_len > 255) { + fold0 = _mm512_loadu_si512((const __m512i *)data); + fold1 = _mm512_loadu_si512((const __m512i *)(data+64)); + fold2 = _mm512_loadu_si512((const __m512i *)(data+128)); + fold3 = _mm512_loadu_si512((const __m512i *)(data+192)); + fold0 = _mm512_xor_si512(fold0, temp); + + /* Main folding loop */ + k = params->rk1_rk2; + for (n = 256; (n + 256) <= data_len; n += 256) { + qw0 = _mm512_loadu_si512((const __m512i *)&data[n]); + qw1 = _mm512_loadu_si512((const __m512i *) + &(data[n+64])); + qw2 = _mm512_loadu_si512((const __m512i *) + &(data[n+128])); + qw3 = _mm512_loadu_si512((const __m512i *) + &(data[n+192])); + fold0 = crcr32_folding_round(qw0, k, fold0); + fold1 = crcr32_folding_round(qw1, k, fold1); + fold2 = crcr32_folding_round(qw2, k, fold2); + fold3 = crcr32_folding_round(qw3, k, fold3); + } + + /* 256 to 128 fold */ + k = params->rk3_rk4; + fold0 = crcr32_folding_round(fold2, k, fold0); + fold1 = crcr32_folding_round(fold3, k, fold1); + + res = crc32_fold_128(fold0, fold1, params); + + reduction = 240 - ((n+256)-data_len); + + while (reduction > 0) + reduction_loop(&res, &reduction, data, &n, + params); + + reduction += 16; + + if (n != data_len) + res = last_two_xmm(data, data_len, n, res, + params); + } else { + if (data_len > 31) { + res = _mm_insert_epi32(_mm_setzero_si128(), crc, 0); + d = _mm_loadu_si128((const __m128i *)data); + res = _mm_xor_si128(res, d); + n += 16; + + reduction = 240 - ((n+256)-data_len); + + while (reduction > 0) + reduction_loop(&res, &reduction, data, &n, + params); + + if (n != data_len) + res = last_two_xmm(data, data_len, n, res, + params); + } else if (data_len > 16) { + res = _mm_insert_epi32(_mm_setzero_si128(), crc, 0); + d = _mm_loadu_si128((const __m128i *)data); + res = _mm_xor_si128(res, d); + n += 16; + + if (n != data_len) + res = last_two_xmm(data, data_len, n, res, + params); + } else if (data_len == 16) { + res = _mm_insert_epi32(_mm_setzero_si128(), crc, 0); + d = _mm_loadu_si128((const __m128i *)data); + res = _mm_xor_si128(res, d); + } else { + res = _mm_insert_epi32(_mm_setzero_si128(), crc, 0); + mask = byte_len_to_mask_table[data_len]; + d = _mm_maskz_loadu_epi8(mask, data); + res = _mm_xor_si128(res, d); + + if (data_len > 3) { + d = _mm_loadu_si128((const __m128i *) + &shf_table[data_len]); + res = _mm_shuffle_epi8(res, d); + } else if (data_len > 2) { + res = _mm_slli_si128(res, 5); + goto do_barrett_reduction; + } else if (data_len > 1) { + res = _mm_slli_si128(res, 6); + goto do_barrett_reduction; + } else if (data_len > 0) { + res = _mm_slli_si128(res, 7); + goto do_barrett_reduction; + } else { + /* zero length case */ + return crc; + } + } + } + + res = done_128(res, params); + +do_barrett_reduction: + n = barrett_reduction(res, params); + + return n; +} + +static void +crc32_load_init_constants(void) +{ + __m128i a; + /* fold constants */ + uint64_t c0 = 0x00000000e95c1271; + uint64_t c1 = 0x00000000ce3371cb; + uint64_t c2 = 0x00000000910eeec1; + uint64_t c3 = 0x0000000033fff533; + uint64_t c4 = 0x000000000cbec0ed; + uint64_t c5 = 0x0000000031f8303f; + uint64_t c6 = 0x0000000057c54819; + uint64_t c7 = 0x00000000df068dc2; + uint64_t c8 = 0x00000000ae0b5394; + uint64_t c9 = 0x000000001c279815; + uint64_t c10 = 0x000000001d9513d7; + uint64_t c11 = 0x000000008f352d95; + uint64_t c12 = 0x00000000af449247; + uint64_t c13 = 0x000000003db1ecdc; + uint64_t c14 = 0x0000000081256527; + uint64_t c15 = 0x00000000f1da05aa; + uint64_t c16 = 0x00000000ccaa009e; + uint64_t c17 = 0x00000000ae689191; + uint64_t c18 = 0x00000000ccaa009e; + uint64_t c19 = 0x00000000b8bc6765; + uint64_t c20 = 0x00000001f7011640; + uint64_t c21 = 0x00000001db710640; + + a = _mm_set_epi64x(c1, c0); + crc32_eth.rk1_rk2 = _mm512_broadcast_i32x4(a); + + a = _mm_set_epi64x(c3, c2); + crc32_eth.rk3_rk4 = _mm512_broadcast_i32x4(a); + + crc32_eth.fold_7x128b = _mm512_setr_epi64(c4, c5, c6, c7, c8, + c9, c10, c11); + crc32_eth.fold_3x128b = _mm512_setr_epi64(c12, c13, c14, c15, + c16, c17, 0, 0); + crc32_eth.fold_1x128b = _mm_setr_epi64(_mm_cvtsi64_m64(c16), + _mm_cvtsi64_m64(c17)); + + crc32_eth.rk5_rk6 = _mm_setr_epi64(_mm_cvtsi64_m64(c18), + _mm_cvtsi64_m64(c19)); + crc32_eth.rk7_rk8 = _mm_setr_epi64(_mm_cvtsi64_m64(c20), + _mm_cvtsi64_m64(c21)); +} + +static void +crc16_load_init_constants(void) +{ + __m128i a; + /* fold constants */ + uint64_t c0 = 0x0000000000009a19; + uint64_t c1 = 0x0000000000002df8; + uint64_t c2 = 0x00000000000068af; + uint64_t c3 = 0x000000000000b6c9; + uint64_t c4 = 0x000000000000c64f; + uint64_t c5 = 0x000000000000cd95; + uint64_t c6 = 0x000000000000d341; + uint64_t c7 = 0x000000000000b8f2; + uint64_t c8 = 0x0000000000000842; + uint64_t c9 = 0x000000000000b072; + uint64_t c10 = 0x00000000000047e3; + uint64_t c11 = 0x000000000000922d; + uint64_t c12 = 0x0000000000000e3a; + uint64_t c13 = 0x0000000000004d7a; + uint64_t c14 = 0x0000000000005b44; + uint64_t c15 = 0x0000000000007762; + uint64_t c16 = 0x00000000000081bf; + uint64_t c17 = 0x0000000000008e10; + uint64_t c18 = 0x00000000000081bf; + uint64_t c19 = 0x0000000000001cbb; + uint64_t c20 = 0x000000011c581910; + uint64_t c21 = 0x0000000000010810; + + a = _mm_set_epi64x(c1, c0); + crc16_ccitt.rk1_rk2 = _mm512_broadcast_i32x4(a); + + a = _mm_set_epi64x(c3, c2); + crc16_ccitt.rk3_rk4 = _mm512_broadcast_i32x4(a); + + crc16_ccitt.fold_7x128b = _mm512_setr_epi64(c4, c5, c6, c7, c8, + c9, c10, c11); + crc16_ccitt.fold_3x128b = _mm512_setr_epi64(c12, c13, c14, c15, + c16, c17, 0, 0); + crc16_ccitt.fold_1x128b = _mm_setr_epi64(_mm_cvtsi64_m64(c16), + _mm_cvtsi64_m64(c17)); + + crc16_ccitt.rk5_rk6 = _mm_setr_epi64(_mm_cvtsi64_m64(c18), + _mm_cvtsi64_m64(c19)); + crc16_ccitt.rk7_rk8 = _mm_setr_epi64(_mm_cvtsi64_m64(c20), + _mm_cvtsi64_m64(c21)); +} + +void +rte_net_crc_avx512_init(void) +{ + crc32_load_init_constants(); + crc16_load_init_constants(); + + /* + * Reset the register as following calculation may + * use other data types such as float, double, etc. + */ + _mm_empty(); +} + +uint32_t +rte_crc16_ccitt_avx512_handler(const uint8_t *data, uint32_t data_len) +{ + /* return 16-bit CRC value */ + return (uint16_t)~crc32_eth_calc_vpclmulqdq(data, + data_len, + 0xffff, + &crc16_ccitt); +} + +uint32_t +rte_crc32_eth_avx512_handler(const uint8_t *data, uint32_t data_len) +{ + /* return 32-bit CRC value */ + return ~crc32_eth_calc_vpclmulqdq(data, + data_len, + 0xffffffffUL, + &crc32_eth); +} diff --git a/lib/librte_net/rte_net_crc.c b/lib/librte_net/rte_net_crc.c index 83dccbfba..fcf9cc0ef 100644 --- a/lib/librte_net/rte_net_crc.c +++ b/lib/librte_net/rte_net_crc.c @@ -37,6 +37,12 @@ static rte_net_crc_handler handlers_scalar[] = { [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_handler, [RTE_NET_CRC32_ETH] = rte_crc32_eth_handler, }; +#ifdef CC_X86_64_AVX512_VPCLMULQDQ_SUPPORT +static rte_net_crc_handler handlers_avx512[] = { + [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_avx512_handler, + [RTE_NET_CRC32_ETH] = rte_crc32_eth_avx512_handler, +}; +#endif #ifdef CC_X86_64_SSE42_PCLMULQDQ_SUPPORT static rte_net_crc_handler handlers_sse42[] = { [RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_sse42_handler, @@ -132,6 +138,19 @@ rte_crc32_eth_handler(const uint8_t *data, uint32_t data_len) crc32_eth_lut); } +#ifdef CC_X86_64_AVX512_VPCLMULQDQ_SUPPORT +static uint8_t +avx512_vpclmulqdq_cpu_supported(void) +{ + return rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) && + rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512BW) && + rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512DQ) && + rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512VL) && + rte_cpu_get_flag_enabled(RTE_CPUFLAG_PCLMULQDQ) && + rte_cpu_get_flag_enabled(RTE_CPUFLAG_VPCLMULQDQ); +} +#endif + #ifdef CC_X86_64_SSE42_PCLMULQDQ_SUPPORT static uint8_t sse42_pclmulqdq_cpu_supported(void) @@ -153,6 +172,14 @@ rte_net_crc_set_alg(enum rte_net_crc_alg alg) { switch (alg) { #ifdef RTE_ARCH_X86_64 + case RTE_NET_CRC_AVX512: +#ifdef CC_X86_64_AVX512_VPCLMULQDQ_SUPPORT + if (avx512_vpclmulqdq_cpu_supported()) { + handlers = handlers_avx512; + break; + } +#endif + /* fall-through */ case RTE_NET_CRC_SSE42: #ifdef CC_X86_64_SSE42_PCLMULQDQ_SUPPORT if (sse42_pclmulqdq_cpu_supported()) { @@ -206,6 +233,12 @@ RTE_INIT(rte_net_crc_init) rte_net_crc_sse42_init(); } #endif +#ifdef CC_X86_64_AVX512_VPCLMULQDQ_SUPPORT + if (avx512_vpclmulqdq_cpu_supported()) { + alg = RTE_NET_CRC_AVX512; + rte_net_crc_avx512_init(); + } +#endif #ifdef CC_ARM64_NEON_PMULL_SUPPORT if (neon_pmull_cpu_supported()) { alg = RTE_NET_CRC_NEON; diff --git a/lib/librte_net/rte_net_crc.h b/lib/librte_net/rte_net_crc.h index 16e85ca97..72d3e10ff 100644 --- a/lib/librte_net/rte_net_crc.h +++ b/lib/librte_net/rte_net_crc.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause - * Copyright(c) 2017 Intel Corporation + * Copyright(c) 2017-2020 Intel Corporation */ #ifndef _RTE_NET_CRC_H_ @@ -23,6 +23,7 @@ enum rte_net_crc_alg { RTE_NET_CRC_SCALAR = 0, RTE_NET_CRC_SSE42, RTE_NET_CRC_NEON, + RTE_NET_CRC_AVX512, }; /** @@ -35,6 +36,7 @@ enum rte_net_crc_alg { * - RTE_NET_CRC_SCALAR * - RTE_NET_CRC_SSE42 (Use 64-bit SSE4.2 intrinsic) * - RTE_NET_CRC_NEON (Use ARM Neon intrinsic) + * - RTE_NET_CRC_AVX512 (Use 512-bit AVX intrinsic) */ void rte_net_crc_set_alg(enum rte_net_crc_alg alg); -- 2.12.3