From: pbronowx <piotrx.bronowski@intel.com>
To: dev@dpdk.org
Cc: roy.fan.zhang@intel.com, thomas@monjalon.net, gakhil@marvell.com,
ferruh.yigit@intel.com, declan.doherty@intel.com,
pbronowx <piotrx.bronowski@intel.com>
Subject: [dpdk-dev] [RFC 2/7] crypto/ipsec_mb: move aesni-mb PMD to ipsec_mb framework
Date: Fri, 18 Jun 2021 12:17:58 +0000 [thread overview]
Message-ID: <20210618121803.1189857-3-piotrx.bronowski@intel.com> (raw)
In-Reply-To: <20210618121803.1189857-1-piotrx.bronowski@intel.com>
This patch removes crypto/aesni_mb folder and gathers all aesni-mb PMD
implementation specific details into single file pmd_aesni_mb.c
in crypto/ipsec_mb.
Signed-off-by: pbronowx <piotrx.bronowski@intel.com>
---
.../crypto/aesni_mb/aesni_mb_pmd_private.h | 337 ----
drivers/crypto/aesni_mb/meson.build | 25 -
.../crypto/aesni_mb/rte_aesni_mb_pmd_ops.c | 1126 -----------
drivers/crypto/aesni_mb/version.map | 3 -
drivers/crypto/ipsec_mb/meson.build | 1 +
.../pmd_aesni_mb.c} | 1762 ++++++++++++-----
drivers/crypto/ipsec_mb/rte_ipsec_mb_pmd.c | 7 +-
.../ipsec_mb/rte_ipsec_mb_pmd_private.h | 145 +-
drivers/crypto/meson.build | 1 -
9 files changed, 1442 insertions(+), 1965 deletions(-)
delete mode 100644 drivers/crypto/aesni_mb/aesni_mb_pmd_private.h
delete mode 100644 drivers/crypto/aesni_mb/meson.build
delete mode 100644 drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c
delete mode 100644 drivers/crypto/aesni_mb/version.map
rename drivers/crypto/{aesni_mb/rte_aesni_mb_pmd.c => ipsec_mb/pmd_aesni_mb.c} (60%)
diff --git a/drivers/crypto/aesni_mb/aesni_mb_pmd_private.h b/drivers/crypto/aesni_mb/aesni_mb_pmd_private.h
deleted file mode 100644
index 11e7bf5d18..0000000000
--- a/drivers/crypto/aesni_mb/aesni_mb_pmd_private.h
+++ /dev/null
@@ -1,337 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2015-2016 Intel Corporation
- */
-
-#ifndef _AESNI_MB_PMD_PRIVATE_H_
-#define _AESNI_MB_PMD_PRIVATE_H_
-
-#include <intel-ipsec-mb.h>
-
-#if defined(RTE_LIB_SECURITY) && (IMB_VERSION_NUM) >= IMB_VERSION(0, 54, 0)
-#define AESNI_MB_DOCSIS_SEC_ENABLED 1
-#include <rte_security.h>
-#include <rte_security_driver.h>
-#endif
-
-enum aesni_mb_vector_mode {
- RTE_AESNI_MB_NOT_SUPPORTED = 0,
- RTE_AESNI_MB_SSE,
- RTE_AESNI_MB_AVX,
- RTE_AESNI_MB_AVX2,
- RTE_AESNI_MB_AVX512
-};
-
-#define CRYPTODEV_NAME_AESNI_MB_PMD crypto_aesni_mb
-/**< AES-NI Multi buffer PMD device name */
-
-/** AESNI_MB PMD LOGTYPE DRIVER */
-extern int aesni_mb_logtype_driver;
-
-#define AESNI_MB_LOG(level, fmt, ...) \
- rte_log(RTE_LOG_ ## level, aesni_mb_logtype_driver, \
- "%s() line %u: " fmt "\n", __func__, __LINE__, \
- ## __VA_ARGS__)
-
-
-#define HMAC_IPAD_VALUE (0x36)
-#define HMAC_OPAD_VALUE (0x5C)
-
-/* Maximum length for digest */
-#define DIGEST_LENGTH_MAX 64
-static const unsigned auth_blocksize[] = {
- [NULL_HASH] = 0,
- [MD5] = 64,
- [SHA1] = 64,
- [SHA_224] = 64,
- [SHA_256] = 64,
- [SHA_384] = 128,
- [SHA_512] = 128,
- [AES_XCBC] = 16,
- [AES_CCM] = 16,
- [AES_CMAC] = 16,
- [AES_GMAC] = 16,
- [PLAIN_SHA1] = 64,
- [PLAIN_SHA_224] = 64,
- [PLAIN_SHA_256] = 64,
- [PLAIN_SHA_384] = 128,
- [PLAIN_SHA_512] = 128,
-#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
- [IMB_AUTH_ZUC_EIA3_BITLEN] = 16,
- [IMB_AUTH_SNOW3G_UIA2_BITLEN] = 16,
- [IMB_AUTH_KASUMI_UIA1] = 16
-#endif
-};
-
-/**
- * Get the blocksize in bytes for a specified authentication algorithm
- *
- * @Note: this function will not return a valid value for a non-valid
- * authentication algorithm
- */
-static inline unsigned
-get_auth_algo_blocksize(JOB_HASH_ALG algo)
-{
- return auth_blocksize[algo];
-}
-
-static const unsigned auth_truncated_digest_byte_lengths[] = {
- [MD5] = 12,
- [SHA1] = 12,
- [SHA_224] = 14,
- [SHA_256] = 16,
- [SHA_384] = 24,
- [SHA_512] = 32,
- [AES_XCBC] = 12,
- [AES_CMAC] = 12,
- [AES_CCM] = 8,
- [NULL_HASH] = 0,
- [AES_GMAC] = 12,
- [PLAIN_SHA1] = 20,
- [PLAIN_SHA_224] = 28,
- [PLAIN_SHA_256] = 32,
- [PLAIN_SHA_384] = 48,
- [PLAIN_SHA_512] = 64,
-#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
- [IMB_AUTH_ZUC_EIA3_BITLEN] = 4,
- [IMB_AUTH_SNOW3G_UIA2_BITLEN] = 4,
- [IMB_AUTH_KASUMI_UIA1] = 4
-#endif
-};
-
-/**
- * Get the IPsec specified truncated length in bytes of the HMAC digest for a
- * specified authentication algorithm
- *
- * @Note: this function will not return a valid value for a non-valid
- * authentication algorithm
- */
-static inline unsigned
-get_truncated_digest_byte_length(JOB_HASH_ALG algo)
-{
- return auth_truncated_digest_byte_lengths[algo];
-}
-
-static const unsigned auth_digest_byte_lengths[] = {
- [MD5] = 16,
- [SHA1] = 20,
- [SHA_224] = 28,
- [SHA_256] = 32,
- [SHA_384] = 48,
- [SHA_512] = 64,
- [AES_XCBC] = 16,
- [AES_CMAC] = 16,
- [AES_CCM] = 16,
- [AES_GMAC] = 16,
- [NULL_HASH] = 0,
- [PLAIN_SHA1] = 20,
- [PLAIN_SHA_224] = 28,
- [PLAIN_SHA_256] = 32,
- [PLAIN_SHA_384] = 48,
- [PLAIN_SHA_512] = 64,
-#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
- [IMB_AUTH_ZUC_EIA3_BITLEN] = 4,
- [IMB_AUTH_SNOW3G_UIA2_BITLEN] = 4,
- [IMB_AUTH_KASUMI_UIA1] = 4
-#endif
- /**< Vector mode dependent pointer table of the multi-buffer APIs */
-
-};
-
-/**
- * Get the full digest size in bytes for a specified authentication algorithm
- * (if available in the Multi-buffer library)
- *
- * @Note: this function will not return a valid value for a non-valid
- * authentication algorithm
- */
-static inline unsigned
-get_digest_byte_length(JOB_HASH_ALG algo)
-{
- return auth_digest_byte_lengths[algo];
-}
-
-enum aesni_mb_operation {
- AESNI_MB_OP_HASH_CIPHER,
- AESNI_MB_OP_CIPHER_HASH,
- AESNI_MB_OP_HASH_ONLY,
- AESNI_MB_OP_CIPHER_ONLY,
- AESNI_MB_OP_AEAD_HASH_CIPHER,
- AESNI_MB_OP_AEAD_CIPHER_HASH,
- AESNI_MB_OP_NOT_SUPPORTED
-};
-
-/** private data structure for each virtual AESNI device */
-struct aesni_mb_private {
- enum aesni_mb_vector_mode vector_mode;
- /**< CPU vector instruction set mode */
- unsigned max_nb_queue_pairs;
- /**< Max number of queue pairs supported by device */
- MB_MGR *mb_mgr;
- /**< Multi-buffer instance */
-};
-
-/** AESNI Multi buffer queue pair */
-struct aesni_mb_qp {
- uint16_t id;
- /**< Queue Pair Identifier */
- char name[RTE_CRYPTODEV_NAME_MAX_LEN];
- /**< Unique Queue Pair Name */
- MB_MGR *mb_mgr;
- /**< Multi-buffer instance */
- struct rte_ring *ingress_queue;
- /**< Ring for placing operations ready for processing */
- struct rte_mempool *sess_mp;
- /**< Session Mempool */
- struct rte_mempool *sess_mp_priv;
- /**< Session Private Data Mempool */
- struct rte_cryptodev_stats stats;
- /**< Queue pair statistics */
- uint8_t digest_idx;
- /**< Index of the next slot to be used in temp_digests,
- * to store the digest for a given operation
- */
- uint8_t temp_digests[MAX_JOBS][DIGEST_LENGTH_MAX];
- /**< Buffers used to store the digest generated
- * by the driver when verifying a digest provided
- * by the user (using authentication verify operation)
- */
-} __rte_cache_aligned;
-
-/** AES-NI multi-buffer private session structure */
-struct aesni_mb_session {
- JOB_CHAIN_ORDER chain_order;
- struct {
- uint16_t length;
- uint16_t offset;
- } iv;
- struct {
- uint16_t length;
- uint16_t offset;
- } auth_iv;
- /**< IV parameters */
-
- /** Cipher Parameters */const struct aesni_mb_op_fns *op_fns;
- /**< Vector mode dependent pointer table of the multi-buffer APIs */
-
- struct {
- /** Cipher direction - encrypt / decrypt */
- JOB_CIPHER_DIRECTION direction;
- /** Cipher mode - CBC / Counter */
- JOB_CIPHER_MODE mode;
-
- uint64_t key_length_in_bytes;
-
- union {
- struct {
- uint32_t encode[60] __rte_aligned(16);
- /**< encode key */
- uint32_t decode[60] __rte_aligned(16);
- /**< decode key */
- } expanded_aes_keys;
- /**< Expanded AES keys - Allocating space to
- * contain the maximum expanded key size which
- * is 240 bytes for 256 bit AES, calculate by:
- * ((key size (bytes)) *
- * ((number of rounds) + 1))
- */
- struct {
- const void *ks_ptr[3];
- uint64_t key[3][16];
- } exp_3des_keys;
- /**< Expanded 3DES keys */
-
- struct gcm_key_data gcm_key;
- /**< Expanded GCM key */
- uint8_t zuc_cipher_key[16];
- /**< ZUC cipher key */
-#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
- snow3g_key_schedule_t pKeySched_snow3g_cipher;
- /**< SNOW3G scheduled cipher key */
- kasumi_key_sched_t pKeySched_kasumi_cipher;
- /**< KASUMI scheduled cipher key */
-#endif
- };
- } cipher;
-
- /** Authentication Parameters */
- struct {
- JOB_HASH_ALG algo; /**< Authentication Algorithm */
- enum rte_crypto_auth_operation operation;
- /**< auth operation generate or verify */
- union {
- struct {
- uint8_t inner[128] __rte_aligned(16);
- /**< inner pad */
- uint8_t outer[128] __rte_aligned(16);
- /**< outer pad */
- } pads;
- /**< HMAC Authentication pads -
- * allocating space for the maximum pad
- * size supported which is 128 bytes for
- * SHA512
- */
-
- struct {
- uint32_t k1_expanded[44] __rte_aligned(16);
- /**< k1 (expanded key). */
- uint8_t k2[16] __rte_aligned(16);
- /**< k2. */
- uint8_t k3[16] __rte_aligned(16);
- /**< k3. */
- } xcbc;
-
- struct {
- uint32_t expkey[60] __rte_aligned(16);
- /**< k1 (expanded key). */
- uint32_t skey1[4] __rte_aligned(16);
- /**< k2. */
- uint32_t skey2[4] __rte_aligned(16);
- /**< k3. */
- } cmac;
- /**< Expanded XCBC authentication keys */
- uint8_t zuc_auth_key[16];
- /**< ZUC authentication key */
-#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
- snow3g_key_schedule_t pKeySched_snow3g_auth;
- /**< SNOW3G scheduled authentication key */
- kasumi_key_sched_t pKeySched_kasumi_auth;
- /**< KASUMI scheduled authentication key */
-#endif
- };
- /** Generated digest size by the Multi-buffer library */
- uint16_t gen_digest_len;
- /** Requested digest size from Cryptodev */
- uint16_t req_digest_len;
-
- } auth;
- struct {
- /** AAD data length */
- uint16_t aad_len;
- } aead;
-} __rte_cache_aligned;
-
-extern int
-aesni_mb_set_session_parameters(const MB_MGR *mb_mgr,
- struct aesni_mb_session *sess,
- const struct rte_crypto_sym_xform *xform);
-
-#ifdef AESNI_MB_DOCSIS_SEC_ENABLED
-extern int
-aesni_mb_set_docsis_sec_session_parameters(
- __rte_unused struct rte_cryptodev *dev,
- struct rte_security_session_conf *conf,
- void *sess);
-#endif
-
-/** device specific operations function pointer structures */
-extern struct rte_cryptodev_ops *rte_aesni_mb_pmd_ops;
-#ifdef AESNI_MB_DOCSIS_SEC_ENABLED
-extern struct rte_security_ops *rte_aesni_mb_pmd_sec_ops;
-#endif
-
-extern uint32_t
-aesni_mb_cpu_crypto_process_bulk(struct rte_cryptodev *dev,
- struct rte_cryptodev_sym_session *sess, union rte_crypto_sym_ofs sofs,
- struct rte_crypto_sym_vec *vec);
-
-#endif /* _AESNI_MB_PMD_PRIVATE_H_ */
diff --git a/drivers/crypto/aesni_mb/meson.build b/drivers/crypto/aesni_mb/meson.build
deleted file mode 100644
index ed6b9f53e4..0000000000
--- a/drivers/crypto/aesni_mb/meson.build
+++ /dev/null
@@ -1,25 +0,0 @@
-# SPDX-License-Identifier: BSD-3-Clause
-# Copyright(c) 2018 Intel Corporation
-
-IMB_required_ver = '0.52.0'
-lib = cc.find_library('IPSec_MB', required: false)
-if not lib.found()
- build = false
- reason = 'missing dependency, "libIPSec_MB"'
-else
- ext_deps += lib
-
- # version comes with quotes, so we split based on " and take the middle
- imb_ver = cc.get_define('IMB_VERSION_STR',
- prefix : '#include<intel-ipsec-mb.h>').split('"')[1]
-
- if (imb_ver == '') or (imb_ver.version_compare('<' + IMB_required_ver))
- reason = 'IPSec_MB version >= @0@ is required, found version @1@'.format(
- IMB_required_ver, imb_ver)
- build = false
- endif
-
-endif
-
-sources = files('rte_aesni_mb_pmd.c', 'rte_aesni_mb_pmd_ops.c')
-deps += ['bus_vdev', 'net', 'security']
diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c
deleted file mode 100644
index fc7fdfec8e..0000000000
--- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c
+++ /dev/null
@@ -1,1126 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2015-2017 Intel Corporation
- */
-
-#include <string.h>
-
-#include <rte_string_fns.h>
-#include <rte_common.h>
-#include <rte_malloc.h>
-#include <rte_ether.h>
-#include <rte_cryptodev_pmd.h>
-
-#include "aesni_mb_pmd_private.h"
-
-
-static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = {
- { /* MD5 HMAC */
- .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
- {.sym = {
- .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
- {.auth = {
- .algo = RTE_CRYPTO_AUTH_MD5_HMAC,
- .block_size = 64,
- .key_size = {
- .min = 1,
- .max = 64,
- .increment = 1
- },
- .digest_size = {
- .min = 1,
- .max = 16,
- .increment = 1
- },
- .iv_size = { 0 }
- }, }
- }, }
- },
- { /* SHA1 HMAC */
- .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
- {.sym = {
- .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
- {.auth = {
- .algo = RTE_CRYPTO_AUTH_SHA1_HMAC,
- .block_size = 64,
- .key_size = {
- .min = 1,
- .max = 65535,
- .increment = 1
- },
- .digest_size = {
- .min = 1,
- .max = 20,
- .increment = 1
- },
- .iv_size = { 0 }
- }, }
- }, }
- },
- { /* SHA1 */
- .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
- {.sym = {
- .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
- {.auth = {
- .algo = RTE_CRYPTO_AUTH_SHA1,
- .block_size = 64,
- .key_size = {
- .min = 0,
- .max = 0,
- .increment = 0
- },
- .digest_size = {
- .min = 1,
- .max = 20,
- .increment = 1
- },
- .iv_size = { 0 }
- }, }
- }, }
- },
- { /* SHA224 HMAC */
- .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
- {.sym = {
- .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
- {.auth = {
- .algo = RTE_CRYPTO_AUTH_SHA224_HMAC,
- .block_size = 64,
- .key_size = {
- .min = 1,
- .max = 65535,
- .increment = 1
- },
- .digest_size = {
- .min = 1,
- .max = 28,
- .increment = 1
- },
- .iv_size = { 0 }
- }, }
- }, }
- },
- { /* SHA224 */
- .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
- {.sym = {
- .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
- {.auth = {
- .algo = RTE_CRYPTO_AUTH_SHA224,
- .block_size = 64,
- .key_size = {
- .min = 0,
- .max = 0,
- .increment = 0
- },
- .digest_size = {
- .min = 1,
- .max = 28,
- .increment = 1
- },
- .iv_size = { 0 }
- }, }
- }, }
- },
- { /* SHA256 HMAC */
- .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
- {.sym = {
- .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
- {.auth = {
- .algo = RTE_CRYPTO_AUTH_SHA256_HMAC,
- .block_size = 64,
- .key_size = {
- .min = 1,
- .max = 65535,
- .increment = 1
- },
- .digest_size = {
- .min = 1,
- .max = 32,
- .increment = 1
- },
- .iv_size = { 0 }
- }, }
- }, }
- },
- { /* SHA256 */
- .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
- {.sym = {
- .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
- {.auth = {
- .algo = RTE_CRYPTO_AUTH_SHA256,
- .block_size = 64,
- .key_size = {
- .min = 0,
- .max = 0,
- .increment = 0
- },
- .digest_size = {
- .min = 1,
- .max = 32,
- .increment = 1
- },
- .iv_size = { 0 }
- }, }
- }, }
- },
- { /* SHA384 HMAC */
- .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
- {.sym = {
- .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
- {.auth = {
- .algo = RTE_CRYPTO_AUTH_SHA384_HMAC,
- .block_size = 128,
- .key_size = {
- .min = 1,
- .max = 65535,
- .increment = 1
- },
- .digest_size = {
- .min = 1,
- .max = 48,
- .increment = 1
- },
- .iv_size = { 0 }
- }, }
- }, }
- },
- { /* SHA384 */
- .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
- {.sym = {
- .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
- {.auth = {
- .algo = RTE_CRYPTO_AUTH_SHA384,
- .block_size = 128,
- .key_size = {
- .min = 0,
- .max = 0,
- .increment = 0
- },
- .digest_size = {
- .min = 1,
- .max = 48,
- .increment = 1
- },
- .iv_size = { 0 }
- }, }
- }, }
- },
- { /* SHA512 HMAC */
- .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
- {.sym = {
- .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
- {.auth = {
- .algo = RTE_CRYPTO_AUTH_SHA512_HMAC,
- .block_size = 128,
- .key_size = {
- .min = 1,
- .max = 65535,
- .increment = 1
- },
- .digest_size = {
- .min = 1,
- .max = 64,
- .increment = 1
- },
- .iv_size = { 0 }
- }, }
- }, }
- },
- { /* SHA512 */
- .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
- {.sym = {
- .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
- {.auth = {
- .algo = RTE_CRYPTO_AUTH_SHA512,
- .block_size = 128,
- .key_size = {
- .min = 0,
- .max = 0,
- .increment = 0
- },
- .digest_size = {
- .min = 1,
- .max = 64,
- .increment = 1
- },
- .iv_size = { 0 }
- }, }
- }, }
- },
- { /* AES XCBC HMAC */
- .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
- {.sym = {
- .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
- {.auth = {
- .algo = RTE_CRYPTO_AUTH_AES_XCBC_MAC,
- .block_size = 16,
- .key_size = {
- .min = 16,
- .max = 16,
- .increment = 0
- },
- .digest_size = {
- .min = 12,
- .max = 12,
- .increment = 0
- },
- .iv_size = { 0 }
- }, }
- }, }
- },
- { /* AES CBC */
- .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
- {.sym = {
- .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
- {.cipher = {
- .algo = RTE_CRYPTO_CIPHER_AES_CBC,
- .block_size = 16,
- .key_size = {
- .min = 16,
- .max = 32,
- .increment = 8
- },
- .iv_size = {
- .min = 16,
- .max = 16,
- .increment = 0
- }
- }, }
- }, }
- },
- { /* AES CTR */
- .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
- {.sym = {
- .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
- {.cipher = {
- .algo = RTE_CRYPTO_CIPHER_AES_CTR,
- .block_size = 16,
- .key_size = {
- .min = 16,
- .max = 32,
- .increment = 8
- },
- .iv_size = {
- .min = 12,
- .max = 16,
- .increment = 4
- }
- }, }
- }, }
- },
- { /* AES DOCSIS BPI */
- .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
- {.sym = {
- .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
- {.cipher = {
- .algo = RTE_CRYPTO_CIPHER_AES_DOCSISBPI,
- .block_size = 16,
- .key_size = {
- .min = 16,
-#if IMB_VERSION_NUM >= IMB_VERSION(0, 53, 3)
- .max = 32,
- .increment = 16
-#else
- .max = 16,
- .increment = 0
-#endif
- },
- .iv_size = {
- .min = 16,
- .max = 16,
- .increment = 0
- }
- }, }
- }, }
- },
- { /* DES CBC */
- .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
- {.sym = {
- .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
- {.cipher = {
- .algo = RTE_CRYPTO_CIPHER_DES_CBC,
- .block_size = 8,
- .key_size = {
- .min = 8,
- .max = 8,
- .increment = 0
- },
- .iv_size = {
- .min = 8,
- .max = 8,
- .increment = 0
- }
- }, }
- }, }
- },
- { /* 3DES CBC */
- .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
- {.sym = {
- .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
- {.cipher = {
- .algo = RTE_CRYPTO_CIPHER_3DES_CBC,
- .block_size = 8,
- .key_size = {
- .min = 8,
- .max = 24,
- .increment = 8
- },
- .iv_size = {
- .min = 8,
- .max = 8,
- .increment = 0
- }
- }, }
- }, }
- },
- { /* DES DOCSIS BPI */
- .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
- {.sym = {
- .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
- {.cipher = {
- .algo = RTE_CRYPTO_CIPHER_DES_DOCSISBPI,
- .block_size = 8,
- .key_size = {
- .min = 8,
- .max = 8,
- .increment = 0
- },
- .iv_size = {
- .min = 8,
- .max = 8,
- .increment = 0
- }
- }, }
- }, }
- },
- { /* AES CCM */
- .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
- {.sym = {
- .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,
- {.aead = {
- .algo = RTE_CRYPTO_AEAD_AES_CCM,
- .block_size = 16,
- .key_size = {
- .min = 16,
-#if IMB_VERSION(0, 54, 2) <= IMB_VERSION_NUM
- .max = 32,
- .increment = 16
-#else
- .max = 16,
- .increment = 0
-#endif
- },
- .digest_size = {
- .min = 4,
- .max = 16,
- .increment = 2
- },
- .aad_size = {
- .min = 0,
- .max = 46,
- .increment = 1
- },
- .iv_size = {
- .min = 7,
- .max = 13,
- .increment = 1
- },
- }, }
- }, }
- },
- { /* AES CMAC */
- .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
- {.sym = {
- .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
- {.auth = {
- .algo = RTE_CRYPTO_AUTH_AES_CMAC,
- .block_size = 16,
- .key_size = {
- .min = 16,
- .max = 16,
- .increment = 0
- },
- .digest_size = {
- .min = 1,
- .max = 16,
- .increment = 1
- },
- .iv_size = { 0 }
- }, }
- }, }
- },
- { /* AES GCM */
- .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
- {.sym = {
- .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,
- {.aead = {
- .algo = RTE_CRYPTO_AEAD_AES_GCM,
- .block_size = 16,
- .key_size = {
- .min = 16,
- .max = 32,
- .increment = 8
- },
- .digest_size = {
- .min = 1,
- .max = 16,
- .increment = 1
- },
- .aad_size = {
- .min = 0,
- .max = 65535,
- .increment = 1
- },
- .iv_size = {
- .min = 12,
- .max = 12,
- .increment = 0
- }
- }, }
- }, }
- },
- { /* AES GMAC (AUTH) */
- .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
- {.sym = {
- .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
- {.auth = {
- .algo = RTE_CRYPTO_AUTH_AES_GMAC,
- .block_size = 16,
- .key_size = {
- .min = 16,
- .max = 32,
- .increment = 8
- },
- .digest_size = {
- .min = 1,
- .max = 16,
- .increment = 1
- },
- .iv_size = {
- .min = 12,
- .max = 12,
- .increment = 0
- }
- }, }
- }, }
- },
-#if IMB_VERSION(0, 53, 0) <= IMB_VERSION_NUM
- { /* AES ECB */
- .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
- {.sym = {
- .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
- {.cipher = {
- .algo = RTE_CRYPTO_CIPHER_AES_ECB,
- .block_size = 16,
- .key_size = {
- .min = 16,
- .max = 32,
- .increment = 8
- },
- .iv_size = { 0 }
- }, }
- }, }
- },
-#endif
-#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
- { /* ZUC (EIA3) */
- .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
- {.sym = {
- .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
- {.auth = {
- .algo = RTE_CRYPTO_AUTH_ZUC_EIA3,
- .block_size = 16,
- .key_size = {
- .min = 16,
- .max = 16,
- .increment = 0
- },
- .digest_size = {
- .min = 4,
- .max = 4,
- .increment = 0
- },
- .iv_size = {
- .min = 16,
- .max = 16,
- .increment = 0
- }
- }, }
- }, }
- },
- { /* ZUC (EEA3) */
- .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
- {.sym = {
- .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
- {.cipher = {
- .algo = RTE_CRYPTO_CIPHER_ZUC_EEA3,
- .block_size = 16,
- .key_size = {
- .min = 16,
- .max = 16,
- .increment = 0
- },
- .iv_size = {
- .min = 16,
- .max = 16,
- .increment = 0
- },
- }, }
- }, }
- },
- { /* SNOW 3G (UIA2) */
- .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
- {.sym = {
- .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
- {.auth = {
- .algo = RTE_CRYPTO_AUTH_SNOW3G_UIA2,
- .block_size = 16,
- .key_size = {
- .min = 16,
- .max = 16,
- .increment = 0
- },
- .digest_size = {
- .min = 4,
- .max = 4,
- .increment = 0
- },
- .iv_size = {
- .min = 16,
- .max = 16,
- .increment = 0
- }
- }, }
- }, }
- },
- { /* SNOW 3G (UEA2) */
- .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
- {.sym = {
- .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
- {.cipher = {
- .algo = RTE_CRYPTO_CIPHER_SNOW3G_UEA2,
- .block_size = 16,
- .key_size = {
- .min = 16,
- .max = 16,
- .increment = 0
- },
- .iv_size = {
- .min = 16,
- .max = 16,
- .increment = 0
- }
- }, }
- }, }
- },
- { /* KASUMI (F9) */
- .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
- {.sym = {
- .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
- {.auth = {
- .algo = RTE_CRYPTO_AUTH_KASUMI_F9,
- .block_size = 8,
- .key_size = {
- .min = 16,
- .max = 16,
- .increment = 0
- },
- .digest_size = {
- .min = 4,
- .max = 4,
- .increment = 0
- },
- .iv_size = { 0 }
- }, }
- }, }
- },
- { /* KASUMI (F8) */
- .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
- {.sym = {
- .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
- {.cipher = {
- .algo = RTE_CRYPTO_CIPHER_KASUMI_F8,
- .block_size = 8,
- .key_size = {
- .min = 16,
- .max = 16,
- .increment = 0
- },
- .iv_size = {
- .min = 8,
- .max = 8,
- .increment = 0
- }
- }, }
- }, }
- },
-#endif
-#if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM
- { /* CHACHA20-POLY1305 */
- .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
- {.sym = {
- .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,
- {.aead = {
- .algo = RTE_CRYPTO_AEAD_CHACHA20_POLY1305,
- .block_size = 64,
- .key_size = {
- .min = 32,
- .max = 32,
- .increment = 0
- },
- .digest_size = {
- .min = 16,
- .max = 16,
- .increment = 0
- },
- .aad_size = {
- .min = 0,
- .max = 240,
- .increment = 1
- },
- .iv_size = {
- .min = 12,
- .max = 12,
- .increment = 0
- },
- }, }
- }, }
- },
-#endif
- RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
-};
-
-#ifdef AESNI_MB_DOCSIS_SEC_ENABLED
-static const struct rte_cryptodev_capabilities
- aesni_mb_pmd_security_crypto_cap[] = {
- { /* AES DOCSIS BPI */
- .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
- {.sym = {
- .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
- {.cipher = {
- .algo = RTE_CRYPTO_CIPHER_AES_DOCSISBPI,
- .block_size = 16,
- .key_size = {
- .min = 16,
- .max = 32,
- .increment = 16
- },
- .iv_size = {
- .min = 16,
- .max = 16,
- .increment = 0
- }
- }, }
- }, }
- },
-
- RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
-};
-
-static const struct rte_security_capability aesni_mb_pmd_security_cap[] = {
- { /* DOCSIS Uplink */
- .action = RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL,
- .protocol = RTE_SECURITY_PROTOCOL_DOCSIS,
- .docsis = {
- .direction = RTE_SECURITY_DOCSIS_UPLINK
- },
- .crypto_capabilities = aesni_mb_pmd_security_crypto_cap
- },
- { /* DOCSIS Downlink */
- .action = RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL,
- .protocol = RTE_SECURITY_PROTOCOL_DOCSIS,
- .docsis = {
- .direction = RTE_SECURITY_DOCSIS_DOWNLINK
- },
- .crypto_capabilities = aesni_mb_pmd_security_crypto_cap
- },
- {
- .action = RTE_SECURITY_ACTION_TYPE_NONE
- }
-};
-#endif
-
-/** Configure device */
-static int
-aesni_mb_pmd_config(__rte_unused struct rte_cryptodev *dev,
- __rte_unused struct rte_cryptodev_config *config)
-{
- return 0;
-}
-
-/** Start device */
-static int
-aesni_mb_pmd_start(__rte_unused struct rte_cryptodev *dev)
-{
- return 0;
-}
-
-/** Stop device */
-static void
-aesni_mb_pmd_stop(__rte_unused struct rte_cryptodev *dev)
-{
-}
-
-/** Close device */
-static int
-aesni_mb_pmd_close(__rte_unused struct rte_cryptodev *dev)
-{
- return 0;
-}
-
-
-/** Get device statistics */
-static void
-aesni_mb_pmd_stats_get(struct rte_cryptodev *dev,
- struct rte_cryptodev_stats *stats)
-{
- int qp_id;
-
- for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
- struct aesni_mb_qp *qp = dev->data->queue_pairs[qp_id];
-
- stats->enqueued_count += qp->stats.enqueued_count;
- stats->dequeued_count += qp->stats.dequeued_count;
-
- stats->enqueue_err_count += qp->stats.enqueue_err_count;
- stats->dequeue_err_count += qp->stats.dequeue_err_count;
- }
-}
-
-/** Reset device statistics */
-static void
-aesni_mb_pmd_stats_reset(struct rte_cryptodev *dev)
-{
- int qp_id;
-
- for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
- struct aesni_mb_qp *qp = dev->data->queue_pairs[qp_id];
-
- memset(&qp->stats, 0, sizeof(qp->stats));
- }
-}
-
-
-/** Get device info */
-static void
-aesni_mb_pmd_info_get(struct rte_cryptodev *dev,
- struct rte_cryptodev_info *dev_info)
-{
- struct aesni_mb_private *internals = dev->data->dev_private;
-
- if (dev_info != NULL) {
- dev_info->driver_id = dev->driver_id;
- dev_info->feature_flags = dev->feature_flags;
- dev_info->capabilities = aesni_mb_pmd_capabilities;
- dev_info->max_nb_queue_pairs = internals->max_nb_queue_pairs;
- /* No limit of number of sessions */
- dev_info->sym.max_nb_sessions = 0;
- }
-}
-
-/** Release queue pair */
-static int
-aesni_mb_pmd_qp_release(struct rte_cryptodev *dev, uint16_t qp_id)
-{
- struct aesni_mb_qp *qp = dev->data->queue_pairs[qp_id];
- struct rte_ring *r = NULL;
-
- if (qp != NULL) {
- r = rte_ring_lookup(qp->name);
- if (r)
- rte_ring_free(r);
- if (qp->mb_mgr)
- free_mb_mgr(qp->mb_mgr);
- rte_free(qp);
- dev->data->queue_pairs[qp_id] = NULL;
- }
- return 0;
-}
-
-/** set a unique name for the queue pair based on it's name, dev_id and qp_id */
-static int
-aesni_mb_pmd_qp_set_unique_name(struct rte_cryptodev *dev,
- struct aesni_mb_qp *qp)
-{
- unsigned n = snprintf(qp->name, sizeof(qp->name),
- "aesni_mb_pmd_%u_qp_%u",
- dev->data->dev_id, qp->id);
-
- if (n >= sizeof(qp->name))
- return -1;
-
- return 0;
-}
-
-/** Create a ring to place processed operations on */
-static struct rte_ring *
-aesni_mb_pmd_qp_create_processed_ops_ring(struct aesni_mb_qp *qp,
- unsigned int ring_size, int socket_id)
-{
- struct rte_ring *r;
- char ring_name[RTE_CRYPTODEV_NAME_MAX_LEN];
-
- unsigned int n = strlcpy(ring_name, qp->name, sizeof(ring_name));
-
- if (n >= sizeof(ring_name))
- return NULL;
-
- r = rte_ring_lookup(ring_name);
- if (r) {
- if (rte_ring_get_size(r) >= ring_size) {
- AESNI_MB_LOG(INFO, "Reusing existing ring %s for processed ops",
- ring_name);
- return r;
- }
-
- AESNI_MB_LOG(ERR, "Unable to reuse existing ring %s for processed ops",
- ring_name);
- return NULL;
- }
-
- return rte_ring_create(ring_name, ring_size, socket_id,
- RING_F_SP_ENQ | RING_F_SC_DEQ);
-}
-
-/** Setup a queue pair */
-static int
-aesni_mb_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
- const struct rte_cryptodev_qp_conf *qp_conf,
- int socket_id)
-{
- struct aesni_mb_qp *qp = NULL;
- struct aesni_mb_private *internals = dev->data->dev_private;
- int ret = -1;
-
- /* Free memory prior to re-allocation if needed. */
- if (dev->data->queue_pairs[qp_id] != NULL)
- aesni_mb_pmd_qp_release(dev, qp_id);
-
- /* Allocate the queue pair data structure. */
- qp = rte_zmalloc_socket("AES-NI PMD Queue Pair", sizeof(*qp),
- RTE_CACHE_LINE_SIZE, socket_id);
- if (qp == NULL)
- return -ENOMEM;
-
- qp->id = qp_id;
- dev->data->queue_pairs[qp_id] = qp;
-
- if (aesni_mb_pmd_qp_set_unique_name(dev, qp))
- goto qp_setup_cleanup;
-
-
- qp->mb_mgr = alloc_mb_mgr(0);
- if (qp->mb_mgr == NULL) {
- ret = -ENOMEM;
- goto qp_setup_cleanup;
- }
-
- switch (internals->vector_mode) {
- case RTE_AESNI_MB_SSE:
- dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_SSE;
- init_mb_mgr_sse(qp->mb_mgr);
- break;
- case RTE_AESNI_MB_AVX:
- dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX;
- init_mb_mgr_avx(qp->mb_mgr);
- break;
- case RTE_AESNI_MB_AVX2:
- dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX2;
- init_mb_mgr_avx2(qp->mb_mgr);
- break;
- case RTE_AESNI_MB_AVX512:
- dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX512;
- init_mb_mgr_avx512(qp->mb_mgr);
- break;
- default:
- AESNI_MB_LOG(ERR, "Unsupported vector mode %u\n",
- internals->vector_mode);
- goto qp_setup_cleanup;
- }
-
- qp->ingress_queue = aesni_mb_pmd_qp_create_processed_ops_ring(qp,
- qp_conf->nb_descriptors, socket_id);
- if (qp->ingress_queue == NULL) {
- ret = -1;
- goto qp_setup_cleanup;
- }
-
- qp->sess_mp = qp_conf->mp_session;
- qp->sess_mp_priv = qp_conf->mp_session_private;
-
- memset(&qp->stats, 0, sizeof(qp->stats));
-
- char mp_name[RTE_MEMPOOL_NAMESIZE];
-
- snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
- "digest_mp_%u_%u", dev->data->dev_id, qp_id);
- return 0;
-
-qp_setup_cleanup:
- if (qp) {
- if (qp->mb_mgr)
- free_mb_mgr(qp->mb_mgr);
- rte_free(qp);
- }
-
- return ret;
-}
-
-/** Returns the size of the aesni multi-buffer session structure */
-static unsigned
-aesni_mb_pmd_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
-{
- return sizeof(struct aesni_mb_session);
-}
-
-/** Configure a aesni multi-buffer session from a crypto xform chain */
-static int
-aesni_mb_pmd_sym_session_configure(struct rte_cryptodev *dev,
- struct rte_crypto_sym_xform *xform,
- struct rte_cryptodev_sym_session *sess,
- struct rte_mempool *mempool)
-{
- void *sess_private_data;
- struct aesni_mb_private *internals = dev->data->dev_private;
- int ret;
-
- if (unlikely(sess == NULL)) {
- AESNI_MB_LOG(ERR, "invalid session struct");
- return -EINVAL;
- }
-
- if (rte_mempool_get(mempool, &sess_private_data)) {
- AESNI_MB_LOG(ERR,
- "Couldn't get object from session mempool");
- return -ENOMEM;
- }
-
- ret = aesni_mb_set_session_parameters(internals->mb_mgr,
- sess_private_data, xform);
- if (ret != 0) {
- AESNI_MB_LOG(ERR, "failed configure session parameters");
-
- /* Return session to mempool */
- rte_mempool_put(mempool, sess_private_data);
- return ret;
- }
-
- set_sym_session_private_data(sess, dev->driver_id,
- sess_private_data);
-
- return 0;
-}
-
-/** Clear the memory of session so it doesn't leave key material behind */
-static void
-aesni_mb_pmd_sym_session_clear(struct rte_cryptodev *dev,
- struct rte_cryptodev_sym_session *sess)
-{
- uint8_t index = dev->driver_id;
- void *sess_priv = get_sym_session_private_data(sess, index);
-
- /* Zero out the whole structure */
- if (sess_priv) {
- memset(sess_priv, 0, sizeof(struct aesni_mb_session));
- struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
- set_sym_session_private_data(sess, index, NULL);
- rte_mempool_put(sess_mp, sess_priv);
- }
-}
-
-struct rte_cryptodev_ops aesni_mb_pmd_ops = {
- .dev_configure = aesni_mb_pmd_config,
- .dev_start = aesni_mb_pmd_start,
- .dev_stop = aesni_mb_pmd_stop,
- .dev_close = aesni_mb_pmd_close,
-
- .stats_get = aesni_mb_pmd_stats_get,
- .stats_reset = aesni_mb_pmd_stats_reset,
-
- .dev_infos_get = aesni_mb_pmd_info_get,
-
- .queue_pair_setup = aesni_mb_pmd_qp_setup,
- .queue_pair_release = aesni_mb_pmd_qp_release,
-
- .sym_cpu_process = aesni_mb_cpu_crypto_process_bulk,
-
- .sym_session_get_size = aesni_mb_pmd_sym_session_get_size,
- .sym_session_configure = aesni_mb_pmd_sym_session_configure,
- .sym_session_clear = aesni_mb_pmd_sym_session_clear
-};
-
-struct rte_cryptodev_ops *rte_aesni_mb_pmd_ops = &aesni_mb_pmd_ops;
-
-#ifdef AESNI_MB_DOCSIS_SEC_ENABLED
-/**
- * Configure a aesni multi-buffer session from a security session
- * configuration
- */
-static int
-aesni_mb_pmd_sec_sess_create(void *dev, struct rte_security_session_conf *conf,
- struct rte_security_session *sess,
- struct rte_mempool *mempool)
-{
- void *sess_private_data;
- struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev;
- int ret;
-
- if (conf->action_type != RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL ||
- conf->protocol != RTE_SECURITY_PROTOCOL_DOCSIS) {
- AESNI_MB_LOG(ERR, "Invalid security protocol");
- return -EINVAL;
- }
-
- if (rte_mempool_get(mempool, &sess_private_data)) {
- AESNI_MB_LOG(ERR, "Couldn't get object from session mempool");
- return -ENOMEM;
- }
-
- ret = aesni_mb_set_docsis_sec_session_parameters(cdev, conf,
- sess_private_data);
-
- if (ret != 0) {
- AESNI_MB_LOG(ERR, "Failed to configure session parameters");
-
- /* Return session to mempool */
- rte_mempool_put(mempool, sess_private_data);
- return ret;
- }
-
- set_sec_session_private_data(sess, sess_private_data);
-
- return ret;
-}
-
-/** Clear the memory of session so it doesn't leave key material behind */
-static int
-aesni_mb_pmd_sec_sess_destroy(void *dev __rte_unused,
- struct rte_security_session *sess)
-{
- void *sess_priv = get_sec_session_private_data(sess);
-
- if (sess_priv) {
- struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
- memset(sess_priv, 0, sizeof(struct aesni_mb_session));
- set_sec_session_private_data(sess, NULL);
- rte_mempool_put(sess_mp, sess_priv);
- }
- return 0;
-}
-
-/** Get security capabilities for aesni multi-buffer */
-static const struct rte_security_capability *
-aesni_mb_pmd_sec_capa_get(void *device __rte_unused)
-{
- return aesni_mb_pmd_security_cap;
-}
-
-static struct rte_security_ops aesni_mb_pmd_sec_ops = {
- .session_create = aesni_mb_pmd_sec_sess_create,
- .session_update = NULL,
- .session_stats_get = NULL,
- .session_destroy = aesni_mb_pmd_sec_sess_destroy,
- .set_pkt_metadata = NULL,
- .capabilities_get = aesni_mb_pmd_sec_capa_get
-};
-
-struct rte_security_ops *rte_aesni_mb_pmd_sec_ops = &aesni_mb_pmd_sec_ops;
-#endif
diff --git a/drivers/crypto/aesni_mb/version.map b/drivers/crypto/aesni_mb/version.map
deleted file mode 100644
index 4a76d1d52d..0000000000
--- a/drivers/crypto/aesni_mb/version.map
+++ /dev/null
@@ -1,3 +0,0 @@
-DPDK_21 {
- local: *;
-};
diff --git a/drivers/crypto/ipsec_mb/meson.build b/drivers/crypto/ipsec_mb/meson.build
index 087ff98c33..f31817f45d 100644
--- a/drivers/crypto/ipsec_mb/meson.build
+++ b/drivers/crypto/ipsec_mb/meson.build
@@ -23,5 +23,6 @@ endif
sources = files('rte_ipsec_mb_pmd.c',
'rte_ipsec_mb_pmd_ops.c',
+ 'pmd_aesni_mb.c'
)
deps += ['bus_vdev', 'net', 'security']
diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c b/drivers/crypto/ipsec_mb/pmd_aesni_mb.c
similarity index 60%
rename from drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c
rename to drivers/crypto/ipsec_mb/pmd_aesni_mb.c
index a01c826a3c..e11ea9a013 100644
--- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c
+++ b/drivers/crypto/ipsec_mb/pmd_aesni_mb.c
@@ -1,39 +1,953 @@
/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2015-2017 Intel Corporation
+ * Copyright(c) 2015-2021 Intel Corporation
*/
+/**
+ * function used to process crypto workload using job API
+ */
#include <intel-ipsec-mb.h>
-#include <rte_common.h>
-#include <rte_hexdump.h>
-#include <rte_cryptodev.h>
-#include <rte_cryptodev_pmd.h>
-#include <rte_bus_vdev.h>
-#include <rte_malloc.h>
-#include <rte_cpuflags.h>
-#include <rte_per_lcore.h>
+#if defined(RTE_LIB_SECURITY) && (IMB_VERSION_NUM) >= IMB_VERSION(0, 54, 0)
+#define AESNI_MB_DOCSIS_SEC_ENABLED 1
+#include <rte_security.h>
+#include <rte_security_driver.h>
#include <rte_ether.h>
+#endif
-#include "aesni_mb_pmd_private.h"
+#include "rte_ipsec_mb_pmd_private.h"
#define AES_CCM_DIGEST_MIN_LEN 4
#define AES_CCM_DIGEST_MAX_LEN 16
#define HMAC_MAX_BLOCK_SIZE 128
-static uint8_t cryptodev_driver_id;
+#define HMAC_IPAD_VALUE (0x36)
+#define HMAC_OPAD_VALUE (0x5C)
+
+static const struct rte_cryptodev_capabilities aesni_mb_capabilities[] = {
+ { /* MD5 HMAC */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+ {.auth = {
+ .algo = RTE_CRYPTO_AUTH_MD5_HMAC,
+ .block_size = 64,
+ .key_size = {
+ .min = 1,
+ .max = 64,
+ .increment = 1
+ },
+ .digest_size = {
+ .min = 1,
+ .max = 16,
+ .increment = 1
+ },
+ .iv_size = { 0 }
+ }, }
+ }, }
+ },
+ { /* SHA1 HMAC */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+ {.auth = {
+ .algo = RTE_CRYPTO_AUTH_SHA1_HMAC,
+ .block_size = 64,
+ .key_size = {
+ .min = 1,
+ .max = 65535,
+ .increment = 1
+ },
+ .digest_size = {
+ .min = 1,
+ .max = 20,
+ .increment = 1
+ },
+ .iv_size = { 0 }
+ }, }
+ }, }
+ },
+ { /* SHA1 */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+ {.auth = {
+ .algo = RTE_CRYPTO_AUTH_SHA1,
+ .block_size = 64,
+ .key_size = {
+ .min = 0,
+ .max = 0,
+ .increment = 0
+ },
+ .digest_size = {
+ .min = 1,
+ .max = 20,
+ .increment = 1
+ },
+ .iv_size = { 0 }
+ }, }
+ }, }
+ },
+ { /* SHA224 HMAC */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+ {.auth = {
+ .algo = RTE_CRYPTO_AUTH_SHA224_HMAC,
+ .block_size = 64,
+ .key_size = {
+ .min = 1,
+ .max = 65535,
+ .increment = 1
+ },
+ .digest_size = {
+ .min = 1,
+ .max = 28,
+ .increment = 1
+ },
+ .iv_size = { 0 }
+ }, }
+ }, }
+ },
+ { /* SHA224 */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+ {.auth = {
+ .algo = RTE_CRYPTO_AUTH_SHA224,
+ .block_size = 64,
+ .key_size = {
+ .min = 0,
+ .max = 0,
+ .increment = 0
+ },
+ .digest_size = {
+ .min = 1,
+ .max = 28,
+ .increment = 1
+ },
+ .iv_size = { 0 }
+ }, }
+ }, }
+ },
+ { /* SHA256 HMAC */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+ {.auth = {
+ .algo = RTE_CRYPTO_AUTH_SHA256_HMAC,
+ .block_size = 64,
+ .key_size = {
+ .min = 1,
+ .max = 65535,
+ .increment = 1
+ },
+ .digest_size = {
+ .min = 1,
+ .max = 32,
+ .increment = 1
+ },
+ .iv_size = { 0 }
+ }, }
+ }, }
+ },
+ { /* SHA256 */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+ {.auth = {
+ .algo = RTE_CRYPTO_AUTH_SHA256,
+ .block_size = 64,
+ .key_size = {
+ .min = 0,
+ .max = 0,
+ .increment = 0
+ },
+ .digest_size = {
+ .min = 1,
+ .max = 32,
+ .increment = 1
+ },
+ .iv_size = { 0 }
+ }, }
+ }, }
+ },
+ { /* SHA384 HMAC */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+ {.auth = {
+ .algo = RTE_CRYPTO_AUTH_SHA384_HMAC,
+ .block_size = 128,
+ .key_size = {
+ .min = 1,
+ .max = 65535,
+ .increment = 1
+ },
+ .digest_size = {
+ .min = 1,
+ .max = 48,
+ .increment = 1
+ },
+ .iv_size = { 0 }
+ }, }
+ }, }
+ },
+ { /* SHA384 */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+ {.auth = {
+ .algo = RTE_CRYPTO_AUTH_SHA384,
+ .block_size = 128,
+ .key_size = {
+ .min = 0,
+ .max = 0,
+ .increment = 0
+ },
+ .digest_size = {
+ .min = 1,
+ .max = 48,
+ .increment = 1
+ },
+ .iv_size = { 0 }
+ }, }
+ }, }
+ },
+ { /* SHA512 HMAC */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+ {.auth = {
+ .algo = RTE_CRYPTO_AUTH_SHA512_HMAC,
+ .block_size = 128,
+ .key_size = {
+ .min = 1,
+ .max = 65535,
+ .increment = 1
+ },
+ .digest_size = {
+ .min = 1,
+ .max = 64,
+ .increment = 1
+ },
+ .iv_size = { 0 }
+ }, }
+ }, }
+ },
+ { /* SHA512 */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+ {.auth = {
+ .algo = RTE_CRYPTO_AUTH_SHA512,
+ .block_size = 128,
+ .key_size = {
+ .min = 0,
+ .max = 0,
+ .increment = 0
+ },
+ .digest_size = {
+ .min = 1,
+ .max = 64,
+ .increment = 1
+ },
+ .iv_size = { 0 }
+ }, }
+ }, }
+ },
+ { /* AES XCBC HMAC */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+ {.auth = {
+ .algo = RTE_CRYPTO_AUTH_AES_XCBC_MAC,
+ .block_size = 16,
+ .key_size = {
+ .min = 16,
+ .max = 16,
+ .increment = 0
+ },
+ .digest_size = {
+ .min = 12,
+ .max = 12,
+ .increment = 0
+ },
+ .iv_size = { 0 }
+ }, }
+ }, }
+ },
+ { /* AES CBC */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+ {.cipher = {
+ .algo = RTE_CRYPTO_CIPHER_AES_CBC,
+ .block_size = 16,
+ .key_size = {
+ .min = 16,
+ .max = 32,
+ .increment = 8
+ },
+ .iv_size = {
+ .min = 16,
+ .max = 16,
+ .increment = 0
+ }
+ }, }
+ }, }
+ },
+ { /* AES CTR */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+ {.cipher = {
+ .algo = RTE_CRYPTO_CIPHER_AES_CTR,
+ .block_size = 16,
+ .key_size = {
+ .min = 16,
+ .max = 32,
+ .increment = 8
+ },
+ .iv_size = {
+ .min = 12,
+ .max = 16,
+ .increment = 4
+ }
+ }, }
+ }, }
+ },
+ { /* AES DOCSIS BPI */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+ {.cipher = {
+ .algo = RTE_CRYPTO_CIPHER_AES_DOCSISBPI,
+ .block_size = 16,
+ .key_size = {
+ .min = 16,
+#if IMB_VERSION_NUM >= IMB_VERSION(0, 53, 3)
+ .max = 32,
+ .increment = 16
+#else
+ .max = 16,
+ .increment = 0
+#endif
+ },
+ .iv_size = {
+ .min = 16,
+ .max = 16,
+ .increment = 0
+ }
+ }, }
+ }, }
+ },
+ { /* DES CBC */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+ {.cipher = {
+ .algo = RTE_CRYPTO_CIPHER_DES_CBC,
+ .block_size = 8,
+ .key_size = {
+ .min = 8,
+ .max = 8,
+ .increment = 0
+ },
+ .iv_size = {
+ .min = 8,
+ .max = 8,
+ .increment = 0
+ }
+ }, }
+ }, }
+ },
+ { /* 3DES CBC */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+ {.cipher = {
+ .algo = RTE_CRYPTO_CIPHER_3DES_CBC,
+ .block_size = 8,
+ .key_size = {
+ .min = 8,
+ .max = 24,
+ .increment = 8
+ },
+ .iv_size = {
+ .min = 8,
+ .max = 8,
+ .increment = 0
+ }
+ }, }
+ }, }
+ },
+ { /* DES DOCSIS BPI */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+ {.cipher = {
+ .algo = RTE_CRYPTO_CIPHER_DES_DOCSISBPI,
+ .block_size = 8,
+ .key_size = {
+ .min = 8,
+ .max = 8,
+ .increment = 0
+ },
+ .iv_size = {
+ .min = 8,
+ .max = 8,
+ .increment = 0
+ }
+ }, }
+ }, }
+ },
+ { /* AES CCM */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,
+ {.aead = {
+ .algo = RTE_CRYPTO_AEAD_AES_CCM,
+ .block_size = 16,
+ .key_size = {
+ .min = 16,
+#if IMB_VERSION(0, 54, 2) <= IMB_VERSION_NUM
+ .max = 32,
+ .increment = 16
+#else
+ .max = 16,
+ .increment = 0
+#endif
+ },
+ .digest_size = {
+ .min = 4,
+ .max = 16,
+ .increment = 2
+ },
+ .aad_size = {
+ .min = 0,
+ .max = 46,
+ .increment = 1
+ },
+ .iv_size = {
+ .min = 7,
+ .max = 13,
+ .increment = 1
+ },
+ }, }
+ }, }
+ },
+ { /* AES CMAC */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+ {.auth = {
+ .algo = RTE_CRYPTO_AUTH_AES_CMAC,
+ .block_size = 16,
+ .key_size = {
+ .min = 16,
+ .max = 16,
+ .increment = 0
+ },
+ .digest_size = {
+ .min = 1,
+ .max = 16,
+ .increment = 1
+ },
+ .iv_size = { 0 }
+ }, }
+ }, }
+ },
+ { /* AES GCM */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,
+ {.aead = {
+ .algo = RTE_CRYPTO_AEAD_AES_GCM,
+ .block_size = 16,
+ .key_size = {
+ .min = 16,
+ .max = 32,
+ .increment = 8
+ },
+ .digest_size = {
+ .min = 1,
+ .max = 16,
+ .increment = 1
+ },
+ .aad_size = {
+ .min = 0,
+ .max = 65535,
+ .increment = 1
+ },
+ .iv_size = {
+ .min = 12,
+ .max = 12,
+ .increment = 0
+ }
+ }, }
+ }, }
+ },
+ { /* AES GMAC (AUTH) */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+ {.auth = {
+ .algo = RTE_CRYPTO_AUTH_AES_GMAC,
+ .block_size = 16,
+ .key_size = {
+ .min = 16,
+ .max = 32,
+ .increment = 8
+ },
+ .digest_size = {
+ .min = 1,
+ .max = 16,
+ .increment = 1
+ },
+ .iv_size = {
+ .min = 12,
+ .max = 12,
+ .increment = 0
+ }
+ }, }
+ }, }
+ },
+#if IMB_VERSION(0, 53, 0) <= IMB_VERSION_NUM
+ { /* AES ECB */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+ {.cipher = {
+ .algo = RTE_CRYPTO_CIPHER_AES_ECB,
+ .block_size = 16,
+ .key_size = {
+ .min = 16,
+ .max = 32,
+ .increment = 8
+ },
+ .iv_size = { 0 }
+ }, }
+ }, }
+ },
+#endif
+#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
+ { /* ZUC (EIA3) */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+ {.auth = {
+ .algo = RTE_CRYPTO_AUTH_ZUC_EIA3,
+ .block_size = 16,
+ .key_size = {
+ .min = 16,
+ .max = 16,
+ .increment = 0
+ },
+ .digest_size = {
+ .min = 4,
+ .max = 4,
+ .increment = 0
+ },
+ .iv_size = {
+ .min = 16,
+ .max = 16,
+ .increment = 0
+ }
+ }, }
+ }, }
+ },
+ { /* ZUC (EEA3) */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+ {.cipher = {
+ .algo = RTE_CRYPTO_CIPHER_ZUC_EEA3,
+ .block_size = 16,
+ .key_size = {
+ .min = 16,
+ .max = 16,
+ .increment = 0
+ },
+ .iv_size = {
+ .min = 16,
+ .max = 16,
+ .increment = 0
+ },
+ }, }
+ }, }
+ },
+ { /* SNOW 3G (UIA2) */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+ {.auth = {
+ .algo = RTE_CRYPTO_AUTH_SNOW3G_UIA2,
+ .block_size = 16,
+ .key_size = {
+ .min = 16,
+ .max = 16,
+ .increment = 0
+ },
+ .digest_size = {
+ .min = 4,
+ .max = 4,
+ .increment = 0
+ },
+ .iv_size = {
+ .min = 16,
+ .max = 16,
+ .increment = 0
+ }
+ }, }
+ }, }
+ },
+ { /* SNOW 3G (UEA2) */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+ {.cipher = {
+ .algo = RTE_CRYPTO_CIPHER_SNOW3G_UEA2,
+ .block_size = 16,
+ .key_size = {
+ .min = 16,
+ .max = 16,
+ .increment = 0
+ },
+ .iv_size = {
+ .min = 16,
+ .max = 16,
+ .increment = 0
+ }
+ }, }
+ }, }
+ },
+ { /* KASUMI (F9) */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
+ {.auth = {
+ .algo = RTE_CRYPTO_AUTH_KASUMI_F9,
+ .block_size = 8,
+ .key_size = {
+ .min = 16,
+ .max = 16,
+ .increment = 0
+ },
+ .digest_size = {
+ .min = 4,
+ .max = 4,
+ .increment = 0
+ },
+ .iv_size = { 0 }
+ }, }
+ }, }
+ },
+ { /* KASUMI (F8) */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+ {.cipher = {
+ .algo = RTE_CRYPTO_CIPHER_KASUMI_F8,
+ .block_size = 8,
+ .key_size = {
+ .min = 16,
+ .max = 16,
+ .increment = 0
+ },
+ .iv_size = {
+ .min = 8,
+ .max = 8,
+ .increment = 0
+ }
+ }, }
+ }, }
+ },
+#endif
+#if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM
+ { /* CHACHA20-POLY1305 */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,
+ {.aead = {
+ .algo = RTE_CRYPTO_AEAD_CHACHA20_POLY1305,
+ .block_size = 64,
+ .key_size = {
+ .min = 32,
+ .max = 32,
+ .increment = 0
+ },
+ .digest_size = {
+ .min = 16,
+ .max = 16,
+ .increment = 0
+ },
+ .aad_size = {
+ .min = 0,
+ .max = 240,
+ .increment = 1
+ },
+ .iv_size = {
+ .min = 12,
+ .max = 12,
+ .increment = 0
+ },
+ }, }
+ }, }
+ },
+#endif
+ RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
+};
+
+uint8_t pmd_driver_id_aesni_mb;
+
+struct aesni_mb_qp_data {
+ uint8_t temp_digests[MAX_JOBS][DIGEST_LENGTH_MAX];
+ /* *< Buffers used to store the digest generated
+ * by the driver when verifying a digest provided
+ * by the user (using authentication verify operation)
+ */
+};
+
+/* Maximum length for digest */
+#define DIGEST_LENGTH_MAX 64
+static const unsigned int auth_blocksize[] = {
+ [NULL_HASH] = 0,
+ [MD5] = 64,
+ [SHA1] = 64,
+ [SHA_224] = 64,
+ [SHA_256] = 64,
+ [SHA_384] = 128,
+ [SHA_512] = 128,
+ [AES_XCBC] = 16,
+ [AES_CCM] = 16,
+ [AES_CMAC] = 16,
+ [AES_GMAC] = 16,
+ [PLAIN_SHA1] = 64,
+ [PLAIN_SHA_224] = 64,
+ [PLAIN_SHA_256] = 64,
+ [PLAIN_SHA_384] = 128,
+ [PLAIN_SHA_512] = 128,
+#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
+ [IMB_AUTH_ZUC_EIA3_BITLEN] = 16,
+ [IMB_AUTH_SNOW3G_UIA2_BITLEN] = 16,
+ [IMB_AUTH_KASUMI_UIA1] = 16
+#endif
+};
+
+/**
+ * Get the blocksize in bytes for a specified authentication algorithm
+ *
+ * @Note: this function will not return a valid value for a non-valid
+ * authentication algorithm
+ */
+static inline unsigned int
+get_auth_algo_blocksize(JOB_HASH_ALG algo)
+{
+ return auth_blocksize[algo];
+}
+
+static const unsigned int auth_truncated_digest_byte_lengths[] = {
+ [MD5] = 12,
+ [SHA1] = 12,
+ [SHA_224] = 14,
+ [SHA_256] = 16,
+ [SHA_384] = 24,
+ [SHA_512] = 32,
+ [AES_XCBC] = 12,
+ [AES_CMAC] = 12,
+ [AES_CCM] = 8,
+ [NULL_HASH] = 0,
+ [AES_GMAC] = 12,
+ [PLAIN_SHA1] = 20,
+ [PLAIN_SHA_224] = 28,
+ [PLAIN_SHA_256] = 32,
+ [PLAIN_SHA_384] = 48,
+ [PLAIN_SHA_512] = 64,
+#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
+ [IMB_AUTH_ZUC_EIA3_BITLEN] = 4,
+ [IMB_AUTH_SNOW3G_UIA2_BITLEN] = 4,
+ [IMB_AUTH_KASUMI_UIA1] = 4
+#endif
+};
-/*
- * Needed to support CPU-CRYPTO API (rte_cryptodev_sym_cpu_crypto_process),
- * as we still use JOB based API even for synchronous processing.
+/**
+ * Get the IPsec specified truncated length in bytes of the HMAC digest for a
+ * specified authentication algorithm
+ *
+ * @Note: this function will not return a valid value for a non-valid
+ * authentication algorithm
+ */
+static inline unsigned int
+get_truncated_digest_byte_length(JOB_HASH_ALG algo)
+{
+ return auth_truncated_digest_byte_lengths[algo];
+}
+
+static const unsigned int auth_digest_byte_lengths[] = {
+ [MD5] = 16,
+ [SHA1] = 20,
+ [SHA_224] = 28,
+ [SHA_256] = 32,
+ [SHA_384] = 48,
+ [SHA_512] = 64,
+ [AES_XCBC] = 16,
+ [AES_CMAC] = 16,
+ [AES_CCM] = 16,
+ [AES_GMAC] = 16,
+ [NULL_HASH] = 0,
+ [PLAIN_SHA1] = 20,
+ [PLAIN_SHA_224] = 28,
+ [PLAIN_SHA_256] = 32,
+ [PLAIN_SHA_384] = 48,
+ [PLAIN_SHA_512] = 64,
+#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
+ [IMB_AUTH_ZUC_EIA3_BITLEN] = 4,
+ [IMB_AUTH_SNOW3G_UIA2_BITLEN] = 4,
+ [IMB_AUTH_KASUMI_UIA1] = 4
+#endif
+ /**< Vector mode dependent pointer table of the multi-buffer APIs */
+
+};
+
+/**
+ * Get the full digest size in bytes for a specified authentication algorithm
+ * (if available in the Multi-buffer library)
+ *
+ * @Note: this function will not return a valid value for a non-valid
+ * authentication algorithm
*/
-static RTE_DEFINE_PER_LCORE(MB_MGR *, sync_mb_mgr);
+static inline unsigned int
+get_digest_byte_length(JOB_HASH_ALG algo)
+{
+ return auth_digest_byte_lengths[algo];
+}
+
+/** AES-NI multi-buffer private session structure */
+struct aesni_mb_session {
+ JOB_CIPHER_MODE cipher_mode;
+ JOB_CIPHER_DIRECTION cipher_direction;
+ JOB_HASH_ALG hash_alg;
+ JOB_CHAIN_ORDER chain_order;
+ /* common job fields */
+ struct {
+ uint16_t length;
+ uint16_t offset;
+ } iv;
+ struct {
+ uint16_t length;
+ uint16_t offset;
+ } auth_iv;
+ /* *< IV parameters
+ */
+
+ /* * Cipher Parameters
+ */
+ struct {
+ /* * Cipher direction - encrypt / decrypt */
+ JOB_CIPHER_DIRECTION direction;
+ /* * Cipher mode - CBC / Counter */
+ JOB_CIPHER_MODE mode;
+
+ uint64_t key_length_in_bytes;
+
+ union {
+ struct {
+ uint32_t encode[60] __rte_aligned(16);
+ /* *< encode key */
+ uint32_t decode[60] __rte_aligned(16);
+ /* *< decode key */
+ } expanded_aes_keys;
+ /* *< Expanded AES keys - Allocating space to
+ * contain the maximum expanded key size which
+ * is 240 bytes for 256 bit AES, calculate by:
+ * ((key size (bytes)) *
+ * ((number of rounds) + 1))
+ */
+ struct {
+ const void *ks_ptr[3];
+ uint64_t key[3][16];
+ } exp_3des_keys;
+ /* *< Expanded 3DES keys */
+
+ struct gcm_key_data gcm_key;
+ /* *< Expanded GCM key */
+ uint8_t zuc_cipher_key[16];
+ /* *< ZUC cipher key */
+#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
+ snow3g_key_schedule_t pKeySched_snow3g_cipher;
+ /* *< SNOW3G scheduled cipher key */
+ kasumi_key_sched_t pKeySched_kasumi_cipher;
+ /* *< KASUMI scheduled cipher key */
+#endif
+ };
+ } cipher;
+
+ /* *< Authentication Parameters */
+ struct {
+ JOB_HASH_ALG algo; /* *< Authentication Algorithm */
+ enum rte_crypto_auth_operation operation;
+ /* *< auth operation generate or verify */
+ union {
+ struct {
+ uint8_t inner[128] __rte_aligned(16);
+ /* *< inner pad */
+ uint8_t outer[128] __rte_aligned(16);
+ /* *< outer pad */
+ } pads;
+ /* *< HMAC Authentication pads -
+ * allocating space for the maximum pad
+ * size supported which is 128 bytes for
+ * SHA512
+ */
+
+ struct {
+ uint32_t k1_expanded[44] __rte_aligned(16);
+ /* *< k1 (expanded key). */
+ uint8_t k2[16] __rte_aligned(16);
+ /* *< k2. */
+ uint8_t k3[16] __rte_aligned(16);
+ /* *< k3. */
+ } xcbc;
+
+ struct {
+ uint32_t expkey[60] __rte_aligned(16);
+ /* *< k1 (expanded key). */
+ uint32_t skey1[4] __rte_aligned(16);
+ /* *< k2. */
+ uint32_t skey2[4] __rte_aligned(16);
+ /* *< k3. */
+ } cmac;
+ /* *< Expanded XCBC authentication keys */
+ uint8_t zuc_auth_key[16];
+ /* *< ZUC authentication key */
+#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
+ snow3g_key_schedule_t pKeySched_snow3g_auth;
+ /* *< SNOW3G scheduled authentication key */
+ kasumi_key_sched_t pKeySched_kasumi_auth;
+ /* *< KASUMI scheduled authentication key */
+#endif
+ };
+ /* * Generated digest size by the Multi-buffer library */
+ uint16_t gen_digest_len;
+ /* * Requested digest size from Cryptodev */
+ uint16_t req_digest_len;
+
+ } auth;
+ struct {
+ /* * AAD data length */
+ uint16_t aad_len;
+ } aead;
+} __rte_cache_aligned;
typedef void (*hash_one_block_t)(const void *data, void *digest);
-typedef void (*aes_keyexp_t)(const void *key, void *enc_exp_keys, void *dec_exp_keys);
+typedef void (*aes_keyexp_t)(const void *key, void *enc_exp_keys,
+ void *dec_exp_keys);
+
/**
* Calculate the authentication pre-computes
*
- * @param one_block_hash Function pointer to calculate digest on ipad/opad
+ * @param one_block_hash Function pointer
+ * to calculate digest on ipad/opad
* @param ipad Inner pad output byte array
* @param opad Outer pad output byte array
* @param hkey Authentication key
@@ -46,7 +960,7 @@ calculate_auth_precomputes(hash_one_block_t one_block_hash,
const uint8_t *hkey, uint16_t hkey_len,
uint16_t blocksize)
{
- unsigned i, length;
+ uint32_t i, length;
uint8_t ipad_buf[blocksize] __rte_aligned(16);
uint8_t opad_buf[blocksize] __rte_aligned(16);
@@ -72,59 +986,6 @@ calculate_auth_precomputes(hash_one_block_t one_block_hash,
memset(opad_buf, 0, blocksize);
}
-/** Get xform chain order */
-static enum aesni_mb_operation
-aesni_mb_get_chain_order(const struct rte_crypto_sym_xform *xform)
-{
- if (xform == NULL)
- return AESNI_MB_OP_NOT_SUPPORTED;
-
- if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
- if (xform->next == NULL)
- return AESNI_MB_OP_CIPHER_ONLY;
- if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
- return AESNI_MB_OP_CIPHER_HASH;
- }
-
- if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
- if (xform->next == NULL)
- return AESNI_MB_OP_HASH_ONLY;
- if (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
- return AESNI_MB_OP_HASH_CIPHER;
- }
-#if IMB_VERSION_NUM > IMB_VERSION(0, 52, 0)
- if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
- if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) {
- /*
- * CCM requires to hash first and cipher later
- * when encrypting
- */
- if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM)
- return AESNI_MB_OP_AEAD_HASH_CIPHER;
- else
- return AESNI_MB_OP_AEAD_CIPHER_HASH;
- } else {
- if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM)
- return AESNI_MB_OP_AEAD_CIPHER_HASH;
- else
- return AESNI_MB_OP_AEAD_HASH_CIPHER;
- }
- }
-#else
- if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
- if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM ||
- xform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM) {
- if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT)
- return AESNI_MB_OP_AEAD_CIPHER_HASH;
- else
- return AESNI_MB_OP_AEAD_HASH_CIPHER;
- }
- }
-#endif
-
- return AESNI_MB_OP_NOT_SUPPORTED;
-}
-
static inline int
is_aead_algo(JOB_HASH_ALG hash_alg, JOB_CIPHER_MODE cipher_mode)
{
@@ -154,7 +1015,7 @@ aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr,
}
if (xform->type != RTE_CRYPTO_SYM_XFORM_AUTH) {
- AESNI_MB_LOG(ERR, "Crypto xform struct not of type auth");
+ IPSEC_MB_LOG(ERR, "Crypto xform struct not of type auth");
return -1;
}
@@ -175,7 +1036,7 @@ aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr,
uint16_t xcbc_mac_digest_len =
get_truncated_digest_byte_length(AES_XCBC);
if (sess->auth.req_digest_len != xcbc_mac_digest_len) {
- AESNI_MB_LOG(ERR, "Invalid digest size\n");
+ IPSEC_MB_LOG(ERR, "Invalid digest size\n");
return -EINVAL;
}
sess->auth.gen_digest_len = sess->auth.req_digest_len;
@@ -194,7 +1055,7 @@ aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr,
uint16_t cmac_digest_len = get_digest_byte_length(AES_CMAC);
if (sess->auth.req_digest_len > cmac_digest_len) {
- AESNI_MB_LOG(ERR, "Invalid digest size\n");
+ IPSEC_MB_LOG(ERR, "Invalid digest size\n");
return -EINVAL;
}
/*
@@ -225,8 +1086,9 @@ aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr,
sess->cipher.direction = DECRYPT;
sess->auth.algo = AES_GMAC;
- if (sess->auth.req_digest_len > get_digest_byte_length(AES_GMAC)) {
- AESNI_MB_LOG(ERR, "Invalid digest size\n");
+ if (sess->auth.req_digest_len >
+ get_digest_byte_length(AES_GMAC)) {
+ IPSEC_MB_LOG(ERR, "Invalid digest size\n");
return -EINVAL;
}
sess->auth.gen_digest_len = sess->auth.req_digest_len;
@@ -261,9 +1123,10 @@ aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr,
if (xform->auth.algo == RTE_CRYPTO_AUTH_ZUC_EIA3) {
sess->auth.algo = IMB_AUTH_ZUC_EIA3_BITLEN;
uint16_t zuc_eia3_digest_len =
- get_truncated_digest_byte_length(IMB_AUTH_ZUC_EIA3_BITLEN);
+ get_truncated_digest_byte_length(
+ IMB_AUTH_ZUC_EIA3_BITLEN);
if (sess->auth.req_digest_len != zuc_eia3_digest_len) {
- AESNI_MB_LOG(ERR, "Invalid digest size\n");
+ IPSEC_MB_LOG(ERR, "Invalid digest size\n");
return -EINVAL;
}
sess->auth.gen_digest_len = sess->auth.req_digest_len;
@@ -273,9 +1136,10 @@ aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr,
} else if (xform->auth.algo == RTE_CRYPTO_AUTH_SNOW3G_UIA2) {
sess->auth.algo = IMB_AUTH_SNOW3G_UIA2_BITLEN;
uint16_t snow3g_uia2_digest_len =
- get_truncated_digest_byte_length(IMB_AUTH_SNOW3G_UIA2_BITLEN);
+ get_truncated_digest_byte_length(
+ IMB_AUTH_SNOW3G_UIA2_BITLEN);
if (sess->auth.req_digest_len != snow3g_uia2_digest_len) {
- AESNI_MB_LOG(ERR, "Invalid digest size\n");
+ IPSEC_MB_LOG(ERR, "Invalid digest size\n");
return -EINVAL;
}
sess->auth.gen_digest_len = sess->auth.req_digest_len;
@@ -288,7 +1152,7 @@ aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr,
uint16_t kasumi_f9_digest_len =
get_truncated_digest_byte_length(IMB_AUTH_KASUMI_UIA1);
if (sess->auth.req_digest_len != kasumi_f9_digest_len) {
- AESNI_MB_LOG(ERR, "Invalid digest size\n");
+ IPSEC_MB_LOG(ERR, "Invalid digest size\n");
return -EINVAL;
}
sess->auth.gen_digest_len = sess->auth.req_digest_len;
@@ -380,7 +1244,8 @@ aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr,
auth_precompute = 0;
break;
default:
- AESNI_MB_LOG(ERR, "Unsupported authentication algorithm selection");
+ IPSEC_MB_LOG(ERR,
+ "Unsupported authentication algorithm selection");
return -ENOTSUP;
}
uint16_t trunc_digest_size =
@@ -390,7 +1255,7 @@ aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr,
if (sess->auth.req_digest_len > full_digest_size ||
sess->auth.req_digest_len == 0) {
- AESNI_MB_LOG(ERR, "Invalid digest size\n");
+ IPSEC_MB_LOG(ERR, "Invalid digest size\n");
return -EINVAL;
}
@@ -443,7 +1308,7 @@ aesni_mb_set_session_cipher_parameters(const MB_MGR *mb_mgr,
}
if (xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) {
- AESNI_MB_LOG(ERR, "Crypto xform struct not of type cipher");
+ IPSEC_MB_LOG(ERR, "Crypto xform struct not of type cipher");
return -EINVAL;
}
@@ -456,7 +1321,7 @@ aesni_mb_set_session_cipher_parameters(const MB_MGR *mb_mgr,
sess->cipher.direction = DECRYPT;
break;
default:
- AESNI_MB_LOG(ERR, "Invalid cipher operation parameter");
+ IPSEC_MB_LOG(ERR, "Invalid cipher operation parameter");
return -EINVAL;
}
@@ -505,7 +1370,7 @@ aesni_mb_set_session_cipher_parameters(const MB_MGR *mb_mgr,
break;
#endif
default:
- AESNI_MB_LOG(ERR, "Unsupported cipher mode parameter");
+ IPSEC_MB_LOG(ERR, "Unsupported cipher mode parameter");
return -ENOTSUP;
}
@@ -535,7 +1400,7 @@ aesni_mb_set_session_cipher_parameters(const MB_MGR *mb_mgr,
sess->cipher.expanded_aes_keys.decode);
break;
default:
- AESNI_MB_LOG(ERR, "Invalid cipher key length");
+ IPSEC_MB_LOG(ERR, "Invalid cipher key length");
return -EINVAL;
}
} else if (is_docsis) {
@@ -555,7 +1420,7 @@ aesni_mb_set_session_cipher_parameters(const MB_MGR *mb_mgr,
break;
#endif
default:
- AESNI_MB_LOG(ERR, "Invalid cipher key length");
+ IPSEC_MB_LOG(ERR, "Invalid cipher key length");
return -EINVAL;
}
} else if (is_3DES) {
@@ -597,7 +1462,7 @@ aesni_mb_set_session_cipher_parameters(const MB_MGR *mb_mgr,
sess->cipher.exp_3des_keys.ks_ptr[2] = keys[0];
break;
default:
- AESNI_MB_LOG(ERR, "Invalid cipher key length");
+ IPSEC_MB_LOG(ERR, "Invalid cipher key length");
return -EINVAL;
}
@@ -605,7 +1470,7 @@ aesni_mb_set_session_cipher_parameters(const MB_MGR *mb_mgr,
#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
} else if (is_zuc) {
if (xform->cipher.key.length != 16) {
- AESNI_MB_LOG(ERR, "Invalid cipher key length");
+ IPSEC_MB_LOG(ERR, "Invalid cipher key length");
return -EINVAL;
}
sess->cipher.key_length_in_bytes = 16;
@@ -613,7 +1478,7 @@ aesni_mb_set_session_cipher_parameters(const MB_MGR *mb_mgr,
16);
} else if (is_snow3g) {
if (xform->cipher.key.length != 16) {
- AESNI_MB_LOG(ERR, "Invalid cipher key length");
+ IPSEC_MB_LOG(ERR, "Invalid cipher key length");
return -EINVAL;
}
sess->cipher.key_length_in_bytes = 16;
@@ -621,7 +1486,7 @@ aesni_mb_set_session_cipher_parameters(const MB_MGR *mb_mgr,
&sess->cipher.pKeySched_snow3g_cipher);
} else if (is_kasumi) {
if (xform->cipher.key.length != 16) {
- AESNI_MB_LOG(ERR, "Invalid cipher key length");
+ IPSEC_MB_LOG(ERR, "Invalid cipher key length");
return -EINVAL;
}
sess->cipher.key_length_in_bytes = 16;
@@ -630,7 +1495,7 @@ aesni_mb_set_session_cipher_parameters(const MB_MGR *mb_mgr,
#endif
} else {
if (xform->cipher.key.length != 8) {
- AESNI_MB_LOG(ERR, "Invalid cipher key length");
+ IPSEC_MB_LOG(ERR, "Invalid cipher key length");
return -EINVAL;
}
sess->cipher.key_length_in_bytes = 8;
@@ -661,7 +1526,7 @@ aesni_mb_set_session_aead_parameters(const MB_MGR *mb_mgr,
sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY;
break;
default:
- AESNI_MB_LOG(ERR, "Invalid aead operation parameter");
+ IPSEC_MB_LOG(ERR, "Invalid aead operation parameter");
return -EINVAL;
}
@@ -693,15 +1558,15 @@ aesni_mb_set_session_aead_parameters(const MB_MGR *mb_mgr,
sess->cipher.expanded_aes_keys.decode);
break;
default:
- AESNI_MB_LOG(ERR, "Invalid cipher key length");
+ IPSEC_MB_LOG(ERR, "Invalid cipher key length");
return -EINVAL;
}
/* CCM digests must be between 4 and 16 and an even number */
if (sess->auth.req_digest_len < AES_CCM_DIGEST_MIN_LEN ||
- sess->auth.req_digest_len > AES_CCM_DIGEST_MAX_LEN ||
- (sess->auth.req_digest_len & 1) == 1) {
- AESNI_MB_LOG(ERR, "Invalid digest size\n");
+ sess->auth.req_digest_len > AES_CCM_DIGEST_MAX_LEN ||
+ (sess->auth.req_digest_len & 1) == 1) {
+ IPSEC_MB_LOG(ERR, "Invalid digest size\n");
return -EINVAL;
}
break;
@@ -727,14 +1592,14 @@ aesni_mb_set_session_aead_parameters(const MB_MGR *mb_mgr,
&sess->cipher.gcm_key);
break;
default:
- AESNI_MB_LOG(ERR, "Invalid cipher key length");
+ IPSEC_MB_LOG(ERR, "Invalid cipher key length");
return -EINVAL;
}
/* GCM digest size must be between 1 and 16 */
if (sess->auth.req_digest_len == 0 ||
sess->auth.req_digest_len > 16) {
- AESNI_MB_LOG(ERR, "Invalid digest size\n");
+ IPSEC_MB_LOG(ERR, "Invalid digest size\n");
return -EINVAL;
}
break;
@@ -745,81 +1610,82 @@ aesni_mb_set_session_aead_parameters(const MB_MGR *mb_mgr,
sess->auth.algo = IMB_AUTH_CHACHA20_POLY1305;
if (xform->aead.key.length != 32) {
- AESNI_MB_LOG(ERR, "Invalid key length");
+ IPSEC_MB_LOG(ERR, "Invalid key length");
return -EINVAL;
}
sess->cipher.key_length_in_bytes = 32;
memcpy(sess->cipher.expanded_aes_keys.encode,
xform->aead.key.data, 32);
if (sess->auth.req_digest_len != 16) {
- AESNI_MB_LOG(ERR, "Invalid digest size\n");
+ IPSEC_MB_LOG(ERR, "Invalid digest size\n");
return -EINVAL;
}
break;
#endif
default:
- AESNI_MB_LOG(ERR, "Unsupported aead mode parameter");
+ IPSEC_MB_LOG(ERR, "Unsupported aead mode parameter");
return -ENOTSUP;
}
return 0;
}
-/** Parse crypto xform chain and set private session parameters */
-int
-aesni_mb_set_session_parameters(const MB_MGR *mb_mgr,
- struct aesni_mb_session *sess,
+/** Configure a aesni multi-buffer session from a crypto xform chain */
+static int
+aesni_mb_session_configure(MB_MGR *mb_mgr,
+ void *priv_sess,
const struct rte_crypto_sym_xform *xform)
{
const struct rte_crypto_sym_xform *auth_xform = NULL;
const struct rte_crypto_sym_xform *cipher_xform = NULL;
const struct rte_crypto_sym_xform *aead_xform = NULL;
+ enum ipsec_mb_operation mode;
+ struct aesni_mb_session *sess = (struct aesni_mb_session *) priv_sess;
int ret;
+ ret = ipsec_mb_parse_xform(xform, &mode, &auth_xform,
+ &cipher_xform, &aead_xform);
+ if (ret)
+ return ret;
+
/* Select Crypto operation - hash then cipher / cipher then hash */
- switch (aesni_mb_get_chain_order(xform)) {
- case AESNI_MB_OP_HASH_CIPHER:
+ switch (mode) {
+ case IPSEC_MB_OP_HASH_VERIFY_THEN_DECRYPT:
sess->chain_order = HASH_CIPHER;
- auth_xform = xform;
- cipher_xform = xform->next;
break;
- case AESNI_MB_OP_CIPHER_HASH:
+ case IPSEC_MB_OP_ENCRYPT_THEN_HASH_GEN:
+ case IPSEC_MB_OP_DECRYPT_THEN_HASH_VERIFY:
sess->chain_order = CIPHER_HASH;
- auth_xform = xform->next;
- cipher_xform = xform;
break;
- case AESNI_MB_OP_HASH_ONLY:
+ case IPSEC_MB_OP_HASH_GEN_ONLY:
+ case IPSEC_MB_OP_HASH_VERIFY_ONLY:
+ case IPSEC_MB_OP_HASH_GEN_THEN_ENCRYPT:
sess->chain_order = HASH_CIPHER;
- auth_xform = xform;
- cipher_xform = NULL;
break;
- case AESNI_MB_OP_CIPHER_ONLY:
- /*
- * Multi buffer library operates only at two modes,
- * CIPHER_HASH and HASH_CIPHER. When doing ciphering only,
- * chain order depends on cipher operation: encryption is always
- * the first operation and decryption the last one.
- */
- if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
- sess->chain_order = CIPHER_HASH;
- else
- sess->chain_order = HASH_CIPHER;
- auth_xform = NULL;
- cipher_xform = xform;
+ /*
+ * Multi buffer library operates only at two modes,
+ * CIPHER_HASH and HASH_CIPHER. When doing ciphering only,
+ * chain order depends on cipher operation: encryption is always
+ * the first operation and decryption the last one.
+ */
+ case IPSEC_MB_OP_ENCRYPT_ONLY:
+ sess->chain_order = CIPHER_HASH;
+ break;
+ case IPSEC_MB_OP_DECRYPT_ONLY:
+ sess->chain_order = HASH_CIPHER;
break;
- case AESNI_MB_OP_AEAD_CIPHER_HASH:
+ case IPSEC_MB_OP_AEAD_AUTHENTICATED_ENCRYPT:
sess->chain_order = CIPHER_HASH;
sess->aead.aad_len = xform->aead.aad_length;
- aead_xform = xform;
break;
- case AESNI_MB_OP_AEAD_HASH_CIPHER:
+ case IPSEC_MB_OP_AEAD_AUTHENTICATED_DECRYPT:
sess->chain_order = HASH_CIPHER;
sess->aead.aad_len = xform->aead.aad_length;
- aead_xform = xform;
break;
- case AESNI_MB_OP_NOT_SUPPORTED:
+ case IPSEC_MB_OP_NOT_SUPPORTED:
default:
- AESNI_MB_LOG(ERR, "Unsupported operation chain order parameter");
+ IPSEC_MB_LOG(ERR,
+ "Unsupported operation chain order parameter");
return -ENOTSUP;
}
@@ -829,14 +1695,15 @@ aesni_mb_set_session_parameters(const MB_MGR *mb_mgr,
ret = aesni_mb_set_session_auth_parameters(mb_mgr, sess, auth_xform);
if (ret != 0) {
- AESNI_MB_LOG(ERR, "Invalid/unsupported authentication parameters");
+ IPSEC_MB_LOG(ERR,
+ "Invalid/unsupported authentication parameters");
return ret;
}
ret = aesni_mb_set_session_cipher_parameters(mb_mgr, sess,
cipher_xform);
if (ret != 0) {
- AESNI_MB_LOG(ERR, "Invalid/unsupported cipher parameters");
+ IPSEC_MB_LOG(ERR, "Invalid/unsupported cipher parameters");
return ret;
}
@@ -844,7 +1711,8 @@ aesni_mb_set_session_parameters(const MB_MGR *mb_mgr,
ret = aesni_mb_set_session_aead_parameters(mb_mgr, sess,
aead_xform);
if (ret != 0) {
- AESNI_MB_LOG(ERR, "Invalid/unsupported aead parameters");
+ IPSEC_MB_LOG(ERR,
+ "Invalid/unsupported aead parameters");
return ret;
}
}
@@ -899,7 +1767,7 @@ aesni_mb_set_docsis_sec_session_auth_parameters(struct aesni_mb_session *sess,
struct rte_security_docsis_xform *xform)
{
if (xform == NULL) {
- AESNI_MB_LOG(ERR, "Invalid DOCSIS xform");
+ IPSEC_MB_LOG(ERR, "Invalid DOCSIS xform");
return -EINVAL;
}
@@ -911,7 +1779,7 @@ aesni_mb_set_docsis_sec_session_auth_parameters(struct aesni_mb_session *sess,
sess->auth.algo = IMB_AUTH_DOCSIS_CRC32;
sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE;
} else {
- AESNI_MB_LOG(ERR, "Unsupported DOCSIS direction");
+ IPSEC_MB_LOG(ERR, "Unsupported DOCSIS direction");
return -ENOTSUP;
}
@@ -925,135 +1793,67 @@ aesni_mb_set_docsis_sec_session_auth_parameters(struct aesni_mb_session *sess,
* Parse DOCSIS security session configuration and set private session
* parameters
*/
-int
+static int
aesni_mb_set_docsis_sec_session_parameters(
__rte_unused struct rte_cryptodev *dev,
struct rte_security_session_conf *conf,
void *sess)
{
+ MB_MGR *mb_mgr = alloc_init_mb_mgr();
struct rte_security_docsis_xform *docsis_xform;
struct rte_crypto_sym_xform *cipher_xform;
- struct aesni_mb_session *aesni_sess = sess;
- struct aesni_mb_private *internals = dev->data->dev_private;
- int ret;
+ struct aesni_mb_session *ipsec_sess = sess;
+ int ret = 0;
+
+ if (!mb_mgr)
+ return -ENOMEM;
ret = check_docsis_sec_session(conf);
if (ret) {
- AESNI_MB_LOG(ERR, "Unsupported DOCSIS security configuration");
- return ret;
+ IPSEC_MB_LOG(ERR, "Unsupported DOCSIS security configuration");
+ goto error_exit;
}
switch (conf->docsis.direction) {
case RTE_SECURITY_DOCSIS_UPLINK:
- aesni_sess->chain_order = IMB_ORDER_CIPHER_HASH;
+ ipsec_sess->chain_order = IMB_ORDER_CIPHER_HASH;
docsis_xform = &conf->docsis;
cipher_xform = conf->crypto_xform;
break;
case RTE_SECURITY_DOCSIS_DOWNLINK:
- aesni_sess->chain_order = IMB_ORDER_HASH_CIPHER;
+ ipsec_sess->chain_order = IMB_ORDER_HASH_CIPHER;
cipher_xform = conf->crypto_xform;
docsis_xform = &conf->docsis;
break;
default:
- return -EINVAL;
+ IPSEC_MB_LOG(ERR, "Unsupported DOCSIS security configuration");
+ ret = -EINVAL;
+ goto error_exit;
}
/* Default IV length = 0 */
- aesni_sess->iv.length = 0;
+ ipsec_sess->iv.length = 0;
- ret = aesni_mb_set_docsis_sec_session_auth_parameters(aesni_sess,
+ ret = aesni_mb_set_docsis_sec_session_auth_parameters(ipsec_sess,
docsis_xform);
if (ret != 0) {
- AESNI_MB_LOG(ERR, "Invalid/unsupported DOCSIS parameters");
- return -EINVAL;
+ IPSEC_MB_LOG(ERR, "Invalid/unsupported DOCSIS parameters");
+ goto error_exit;
}
- ret = aesni_mb_set_session_cipher_parameters(internals->mb_mgr,
- aesni_sess, cipher_xform);
+ ret = aesni_mb_set_session_cipher_parameters(mb_mgr,
+ ipsec_sess, cipher_xform);
if (ret != 0) {
- AESNI_MB_LOG(ERR, "Invalid/unsupported cipher parameters");
- return -EINVAL;
+ IPSEC_MB_LOG(ERR, "Invalid/unsupported cipher parameters");
+ goto error_exit;
}
- return 0;
-}
-#endif
-
-/**
- * burst enqueue, place crypto operations on ingress queue for processing.
- *
- * @param __qp Queue Pair to process
- * @param ops Crypto operations for processing
- * @param nb_ops Number of crypto operations for processing
- *
- * @return
- * - Number of crypto operations enqueued
- */
-static uint16_t
-aesni_mb_pmd_enqueue_burst(void *__qp, struct rte_crypto_op **ops,
- uint16_t nb_ops)
-{
- struct aesni_mb_qp *qp = __qp;
-
- unsigned int nb_enqueued;
-
- nb_enqueued = rte_ring_enqueue_burst(qp->ingress_queue,
- (void **)ops, nb_ops, NULL);
-
- qp->stats.enqueued_count += nb_enqueued;
-
- return nb_enqueued;
+error_exit:
+ free_mb_mgr(mb_mgr);
+ return ret;
}
-
-/** Get multi buffer session */
-static inline struct aesni_mb_session *
-get_session(struct aesni_mb_qp *qp, struct rte_crypto_op *op)
-{
- struct aesni_mb_session *sess = NULL;
-
- if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
- if (likely(op->sym->session != NULL))
- sess = (struct aesni_mb_session *)
- get_sym_session_private_data(
- op->sym->session,
- cryptodev_driver_id);
-#ifdef AESNI_MB_DOCSIS_SEC_ENABLED
- } else if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
- if (likely(op->sym->sec_session != NULL))
- sess = (struct aesni_mb_session *)
- get_sec_session_private_data(
- op->sym->sec_session);
#endif
- } else {
- void *_sess = rte_cryptodev_sym_session_create(qp->sess_mp);
- void *_sess_private_data = NULL;
-
- if (_sess == NULL)
- return NULL;
-
- if (rte_mempool_get(qp->sess_mp_priv,
- (void **)&_sess_private_data))
- return NULL;
-
- sess = (struct aesni_mb_session *)_sess_private_data;
-
- if (unlikely(aesni_mb_set_session_parameters(qp->mb_mgr,
- sess, op->sym->xform) != 0)) {
- rte_mempool_put(qp->sess_mp, _sess);
- rte_mempool_put(qp->sess_mp_priv, _sess_private_data);
- sess = NULL;
- }
- op->sym->session = (struct rte_cryptodev_sym_session *)_sess;
- set_sym_session_private_data(op->sym->session,
- cryptodev_driver_id, _sess_private_data);
- }
-
- if (unlikely(sess == NULL))
- op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
-
- return sess;
-}
static inline uint64_t
auth_start_offset(struct rte_crypto_op *op, struct aesni_mb_session *session,
@@ -1173,9 +1973,12 @@ set_cpu_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_session *session,
#if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM
case IMB_AUTH_CHACHA20_POLY1305:
job->u.CHACHA20_POLY1305.aad = aad->va;
- job->u.CHACHA20_POLY1305.aad_len_in_bytes = session->aead.aad_len;
- job->aes_enc_key_expanded = session->cipher.expanded_aes_keys.encode;
- job->aes_dec_key_expanded = session->cipher.expanded_aes_keys.encode;
+ job->u.CHACHA20_POLY1305.aad_len_in_bytes =
+ session->aead.aad_len;
+ job->aes_enc_key_expanded =
+ session->cipher.expanded_aes_keys.encode;
+ job->aes_dec_key_expanded =
+ session->cipher.expanded_aes_keys.encode;
break;
#endif
default:
@@ -1240,14 +2043,15 @@ set_cpu_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_session *session,
* - NULL pointer if completion of JOB_AES_HMAC structure isn't possible
*/
static inline int
-set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
+set_mb_job_params(JOB_AES_HMAC *job, struct ipsec_mb_qp *qp,
struct rte_crypto_op *op, uint8_t *digest_idx)
{
struct rte_mbuf *m_src = op->sym->m_src, *m_dst;
+ struct aesni_mb_qp_data *qp_data = ipsec_mb_get_qp_private_data(qp);
struct aesni_mb_session *session;
uint32_t m_offset, oop;
- session = get_session(qp, op);
+ session = ipsec_mb_get_session_private(qp, op);
if (session == NULL) {
op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
return -1;
@@ -1319,25 +2123,33 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
session->auth_iv.offset);
break;
case IMB_AUTH_SNOW3G_UIA2_BITLEN:
- job->u.SNOW3G_UIA2._key = (void *) &session->auth.pKeySched_snow3g_auth;
- job->u.SNOW3G_UIA2._iv = rte_crypto_op_ctod_offset(op, uint8_t *,
+ job->u.SNOW3G_UIA2._key = (void *)
+ &session->auth.pKeySched_snow3g_auth;
+ job->u.SNOW3G_UIA2._iv =
+ rte_crypto_op_ctod_offset(op, uint8_t *,
session->auth_iv.offset);
break;
case IMB_AUTH_KASUMI_UIA1:
- job->u.KASUMI_UIA1._key = (void *) &session->auth.pKeySched_kasumi_auth;
+ job->u.KASUMI_UIA1._key = (void *)
+ &session->auth.pKeySched_kasumi_auth;
break;
#endif
#if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM
case IMB_AUTH_CHACHA20_POLY1305:
job->u.CHACHA20_POLY1305.aad = op->sym->aead.aad.data;
- job->u.CHACHA20_POLY1305.aad_len_in_bytes = session->aead.aad_len;
- job->aes_enc_key_expanded = session->cipher.expanded_aes_keys.encode;
- job->aes_dec_key_expanded = session->cipher.expanded_aes_keys.encode;
+ job->u.CHACHA20_POLY1305.aad_len_in_bytes =
+ session->aead.aad_len;
+ job->aes_enc_key_expanded =
+ session->cipher.expanded_aes_keys.encode;
+ job->aes_dec_key_expanded =
+ session->cipher.expanded_aes_keys.encode;
break;
#endif
default:
- job->u.HMAC._hashed_auth_key_xor_ipad = session->auth.pads.inner;
- job->u.HMAC._hashed_auth_key_xor_opad = session->auth.pads.outer;
+ job->u.HMAC._hashed_auth_key_xor_ipad =
+ session->auth.pads.inner;
+ job->u.HMAC._hashed_auth_key_xor_opad =
+ session->auth.pads.outer;
if (job->cipher_mode == DES3) {
job->aes_enc_key_expanded =
@@ -1387,7 +2199,7 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
/* Set digest output location */
if (job->hash_alg != NULL_HASH &&
session->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
- job->auth_tag_output = qp->temp_digests[*digest_idx];
+ job->auth_tag_output = qp_data->temp_digests[*digest_idx];
*digest_idx = (*digest_idx + 1) % MAX_JOBS;
} else {
if (aead)
@@ -1395,8 +2207,10 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
else
job->auth_tag_output = op->sym->auth.digest.data;
- if (session->auth.req_digest_len != session->auth.gen_digest_len) {
- job->auth_tag_output = qp->temp_digests[*digest_idx];
+ if (session->auth.req_digest_len !=
+ session->auth.gen_digest_len) {
+ job->auth_tag_output =
+ qp_data->temp_digests[*digest_idx];
*digest_idx = (*digest_idx + 1) % MAX_JOBS;
}
}
@@ -1452,8 +2266,10 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
#if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM
case IMB_AUTH_CHACHA20_POLY1305:
- job->cipher_start_src_offset_in_bytes = op->sym->aead.data.offset;
- job->hash_start_src_offset_in_bytes = op->sym->aead.data.offset;
+ job->cipher_start_src_offset_in_bytes =
+ op->sym->aead.data.offset;
+ job->hash_start_src_offset_in_bytes =
+ op->sym->aead.data.offset;
job->msg_len_to_cipher_in_bytes =
op->sym->aead.data.length;
job->msg_len_to_hash_in_bytes =
@@ -1497,19 +2313,25 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
* processing.
*/
static inline int
-set_sec_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
- struct rte_crypto_op *op, uint8_t *digest_idx)
+set_sec_mb_job_params(JOB_AES_HMAC *job, struct ipsec_mb_qp *qp,
+ struct rte_crypto_op *op, uint8_t *digest_idx)
{
+ struct aesni_mb_qp_data *qp_data = ipsec_mb_get_qp_private_data(qp);
struct rte_mbuf *m_src, *m_dst;
struct rte_crypto_sym_op *sym;
- struct aesni_mb_session *session;
+ struct aesni_mb_session *session = NULL;
- session = get_session(qp, op);
- if (unlikely(session == NULL)) {
+ if (unlikely(op->sess_type != RTE_CRYPTO_OP_SECURITY_SESSION)) {
op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
return -1;
}
+ session = (struct aesni_mb_session *)
+ get_sec_session_private_data(op->sym->sec_session);
+ if (unlikely(session == NULL)) {
+ op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
+ return -1;
+ }
/* Only DOCSIS protocol operations supported now */
if (session->cipher.mode != IMB_CIPHER_DOCSIS_SEC_BPI ||
session->auth.algo != IMB_AUTH_DOCSIS_CRC32) {
@@ -1548,7 +2370,7 @@ set_sec_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
job->hash_alg = session->auth.algo;
/* Set digest output location */
- job->auth_tag_output = qp->temp_digests[*digest_idx];
+ job->auth_tag_output = qp_data->temp_digests[*digest_idx];
*digest_idx = (*digest_idx + 1) % MAX_JOBS;
/* Set digest length */
@@ -1625,10 +2447,12 @@ generate_digest(JOB_AES_HMAC *job, struct rte_crypto_op *op,
* - Returns NULL on invalid job
*/
static inline struct rte_crypto_op *
-post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job)
+post_process_mb_job(struct ipsec_mb_qp *qp, JOB_AES_HMAC *job)
{
struct rte_crypto_op *op = (struct rte_crypto_op *)job->user_data;
struct aesni_mb_session *sess = NULL;
+ uint32_t driver_id = ipsec_mb_get_driver_id(
+ IPSEC_MB_PMD_TYPE_AESNI_MB);
#ifdef AESNI_MB_DOCSIS_SEC_ENABLED
uint8_t is_docsis_sec = 0;
@@ -1644,7 +2468,7 @@ post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job)
#endif
{
sess = get_sym_session_private_data(op->sym->session,
- cryptodev_driver_id);
+ driver_id);
}
if (unlikely(sess == NULL)) {
@@ -1661,7 +2485,8 @@ post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job)
break;
if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
- if (is_aead_algo(job->hash_alg, sess->cipher.mode))
+ if (is_aead_algo(job->hash_alg,
+ sess->cipher.mode))
verify_digest(job,
op->sym->aead.digest.data,
sess->auth.req_digest_len,
@@ -1718,11 +2543,12 @@ post_process_mb_sync_job(JOB_AES_HMAC *job)
* - Number of processed jobs
*/
static unsigned
-handle_completed_jobs(struct aesni_mb_qp *qp, JOB_AES_HMAC *job,
- struct rte_crypto_op **ops, uint16_t nb_ops)
+handle_completed_jobs(struct ipsec_mb_qp *qp, MB_MGR *mb_mgr,
+ JOB_AES_HMAC *job, struct rte_crypto_op **ops,
+ uint16_t nb_ops)
{
struct rte_crypto_op *op = NULL;
- unsigned processed_jobs = 0;
+ uint16_t processed_jobs = 0;
while (job != NULL) {
op = post_process_mb_job(qp, job);
@@ -1737,7 +2563,7 @@ handle_completed_jobs(struct aesni_mb_qp *qp, JOB_AES_HMAC *job,
if (processed_jobs == nb_ops)
break;
- job = IMB_GET_COMPLETED_JOB(qp->mb_mgr);
+ job = IMB_GET_COMPLETED_JOB(mb_mgr);
}
return processed_jobs;
@@ -1764,16 +2590,16 @@ flush_mb_sync_mgr(MB_MGR *mb_mgr)
}
static inline uint16_t
-flush_mb_mgr(struct aesni_mb_qp *qp, struct rte_crypto_op **ops,
- uint16_t nb_ops)
+flush_mb_mgr(struct ipsec_mb_qp *qp, MB_MGR *mb_mgr,
+ struct rte_crypto_op **ops, uint16_t nb_ops)
{
int processed_ops = 0;
/* Flush the remaining jobs */
- JOB_AES_HMAC *job = IMB_FLUSH_JOB(qp->mb_mgr);
+ JOB_AES_HMAC *job = IMB_FLUSH_JOB(mb_mgr);
if (job)
- processed_ops += handle_completed_jobs(qp, job,
+ processed_ops += handle_completed_jobs(qp, mb_mgr, job,
&ops[processed_ops], nb_ops - processed_ops);
return processed_ops;
@@ -1794,33 +2620,33 @@ set_job_null_op(JOB_AES_HMAC *job, struct rte_crypto_op *op)
}
static uint16_t
-aesni_mb_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
+aesni_mb_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
uint16_t nb_ops)
{
- struct aesni_mb_qp *qp = queue_pair;
-
+ struct ipsec_mb_qp *qp = queue_pair;
+ MB_MGR *mb_mgr = qp->mb_mgr;
struct rte_crypto_op *op;
JOB_AES_HMAC *job;
-
int retval, processed_jobs = 0;
- if (unlikely(nb_ops == 0))
+ if (unlikely(nb_ops == 0 || mb_mgr == NULL))
return 0;
uint8_t digest_idx = qp->digest_idx;
+
do {
/* Get next free mb job struct from mb manager */
- job = IMB_GET_NEXT_JOB(qp->mb_mgr);
+ job = IMB_GET_NEXT_JOB(mb_mgr);
if (unlikely(job == NULL)) {
/* if no free mb job structs we need to flush mb_mgr */
- processed_jobs += flush_mb_mgr(qp,
+ processed_jobs += flush_mb_mgr(qp, mb_mgr,
&ops[processed_jobs],
nb_ops - processed_jobs);
if (nb_ops == processed_jobs)
break;
- job = IMB_GET_NEXT_JOB(qp->mb_mgr);
+ job = IMB_GET_NEXT_JOB(mb_mgr);
}
/*
@@ -1839,7 +2665,8 @@ aesni_mb_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
&digest_idx);
else
#endif
- retval = set_mb_job_params(job, qp, op, &digest_idx);
+ retval = set_mb_job_params(job, qp, op,
+ &digest_idx);
if (unlikely(retval != 0)) {
qp->stats.dequeue_err_count++;
@@ -1848,17 +2675,17 @@ aesni_mb_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
/* Submit job to multi-buffer for processing */
#ifdef RTE_LIBRTE_PMD_AESNI_MB_DEBUG
- job = IMB_SUBMIT_JOB(qp->mb_mgr);
+ job = IMB_SUBMIT_JOB(mb_mgr);
#else
- job = IMB_SUBMIT_JOB_NOCHECK(qp->mb_mgr);
+ job = IMB_SUBMIT_JOB_NOCHECK(mb_mgr);
#endif
/*
* If submit returns a processed job then handle it,
* before submitting subsequent jobs
*/
if (job)
- processed_jobs += handle_completed_jobs(qp, job,
- &ops[processed_jobs],
+ processed_jobs += handle_completed_jobs(qp, mb_mgr,
+ job, &ops[processed_jobs],
nb_ops - processed_jobs);
} while (processed_jobs < nb_ops);
@@ -1866,44 +2693,16 @@ aesni_mb_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
qp->digest_idx = digest_idx;
if (processed_jobs < 1)
- processed_jobs += flush_mb_mgr(qp,
+ processed_jobs += flush_mb_mgr(qp, mb_mgr,
&ops[processed_jobs],
nb_ops - processed_jobs);
return processed_jobs;
}
-static MB_MGR *
-alloc_init_mb_mgr(enum aesni_mb_vector_mode vector_mode)
-{
- MB_MGR *mb_mgr = alloc_mb_mgr(0);
- if (mb_mgr == NULL)
- return NULL;
-
- switch (vector_mode) {
- case RTE_AESNI_MB_SSE:
- init_mb_mgr_sse(mb_mgr);
- break;
- case RTE_AESNI_MB_AVX:
- init_mb_mgr_avx(mb_mgr);
- break;
- case RTE_AESNI_MB_AVX2:
- init_mb_mgr_avx2(mb_mgr);
- break;
- case RTE_AESNI_MB_AVX512:
- init_mb_mgr_avx512(mb_mgr);
- break;
- default:
- AESNI_MB_LOG(ERR, "Unsupported vector mode %u\n", vector_mode);
- free_mb_mgr(mb_mgr);
- return NULL;
- }
-
- return mb_mgr;
-}
static inline void
-aesni_mb_fill_error_code(struct rte_crypto_sym_vec *vec, int32_t err)
+ipsec_mb_fill_error_code(struct rte_crypto_sym_vec *vec, int32_t err)
{
uint32_t i;
@@ -1916,9 +2715,9 @@ check_crypto_sgl(union rte_crypto_sym_ofs so, const struct rte_crypto_sgl *sgl)
{
/* no multi-seg support with current AESNI-MB PMD */
if (sgl->num != 1)
- return ENOTSUP;
+ return -ENOTSUP;
else if (so.ofs.cipher.head + so.ofs.cipher.tail > sgl->vec[0].len)
- return EINVAL;
+ return -EINVAL;
return 0;
}
@@ -1966,8 +2765,8 @@ verify_sync_dgst(struct rte_crypto_sym_vec *vec,
return k;
}
-uint32_t
-aesni_mb_cpu_crypto_process_bulk(struct rte_cryptodev *dev,
+static uint32_t
+aesni_mb_process_bulk(struct rte_cryptodev *dev,
struct rte_cryptodev_sym_session *sess, union rte_crypto_sym_ofs sofs,
struct rte_crypto_sym_vec *vec)
{
@@ -1976,32 +2775,21 @@ aesni_mb_cpu_crypto_process_bulk(struct rte_cryptodev *dev,
void *buf;
JOB_AES_HMAC *job;
MB_MGR *mb_mgr;
- struct aesni_mb_private *priv;
struct aesni_mb_session *s;
uint8_t tmp_dgst[vec->num][DIGEST_LENGTH_MAX];
s = get_sym_session_private_data(sess, dev->driver_id);
if (s == NULL) {
- aesni_mb_fill_error_code(vec, EINVAL);
+ ipsec_mb_fill_error_code(vec, EINVAL);
return 0;
}
/* get per-thread MB MGR, create one if needed */
- mb_mgr = RTE_PER_LCORE(sync_mb_mgr);
- if (mb_mgr == NULL) {
-
- priv = dev->data->dev_private;
- mb_mgr = alloc_init_mb_mgr(priv->vector_mode);
- if (mb_mgr == NULL) {
- aesni_mb_fill_error_code(vec, ENOMEM);
- return 0;
- }
- RTE_PER_LCORE(sync_mb_mgr) = mb_mgr;
- }
+ mb_mgr = get_per_thread_mb_mgr();
+ if (unlikely(mb_mgr == NULL))
+ return 0;
for (i = 0, j = 0, k = 0; i != vec->num; i++) {
-
-
ret = check_crypto_sgl(sofs, vec->sgl + i);
if (ret != 0) {
vec->status[i] = ret;
@@ -2047,186 +2835,220 @@ aesni_mb_cpu_crypto_process_bulk(struct rte_cryptodev *dev,
return k;
}
-static int cryptodev_aesni_mb_remove(struct rte_vdev_device *vdev);
+struct rte_cryptodev_ops aes_mb_pmd_ops = {
+ .dev_configure = ipsec_mb_pmd_config,
+ .dev_start = ipsec_mb_pmd_start,
+ .dev_stop = ipsec_mb_pmd_stop,
+ .dev_close = ipsec_mb_pmd_close,
-static uint64_t
-vec_mode_to_flags(enum aesni_mb_vector_mode mode)
-{
- switch (mode) {
- case RTE_AESNI_MB_SSE:
- return RTE_CRYPTODEV_FF_CPU_SSE;
- case RTE_AESNI_MB_AVX:
- return RTE_CRYPTODEV_FF_CPU_AVX;
- case RTE_AESNI_MB_AVX2:
- return RTE_CRYPTODEV_FF_CPU_AVX2;
- case RTE_AESNI_MB_AVX512:
- return RTE_CRYPTODEV_FF_CPU_AVX512;
- default:
- AESNI_MB_LOG(ERR, "Unsupported vector mode %u\n", mode);
- return 0;
- }
-}
+ .stats_get = ipsec_mb_pmd_stats_get,
+ .stats_reset = ipsec_mb_pmd_stats_reset,
-static int
-cryptodev_aesni_mb_create(const char *name,
- struct rte_vdev_device *vdev,
- struct rte_cryptodev_pmd_init_params *init_params)
-{
- struct rte_cryptodev *dev;
- struct aesni_mb_private *internals;
- enum aesni_mb_vector_mode vector_mode;
- MB_MGR *mb_mgr;
+ .dev_infos_get = ipsec_mb_pmd_info_get,
- dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
- if (dev == NULL) {
- AESNI_MB_LOG(ERR, "failed to create cryptodev vdev");
- return -ENODEV;
- }
+ .queue_pair_setup = ipsec_mb_pmd_qp_setup,
+ .queue_pair_release = ipsec_mb_pmd_qp_release,
- /* Check CPU for supported vector instruction set */
- if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F))
- vector_mode = RTE_AESNI_MB_AVX512;
- else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2))
- vector_mode = RTE_AESNI_MB_AVX2;
- else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX))
- vector_mode = RTE_AESNI_MB_AVX;
- else
- vector_mode = RTE_AESNI_MB_SSE;
+ .sym_cpu_process = aesni_mb_process_bulk,
- dev->driver_id = cryptodev_driver_id;
- dev->dev_ops = rte_aesni_mb_pmd_ops;
+ .sym_session_get_size = ipsec_mb_pmd_sym_session_get_size,
+ .sym_session_configure = ipsec_mb_pmd_sym_session_configure,
+ .sym_session_clear = ipsec_mb_pmd_sym_session_clear
+};
- /* register rx/tx burst functions for data path */
- dev->dequeue_burst = aesni_mb_pmd_dequeue_burst;
- dev->enqueue_burst = aesni_mb_pmd_enqueue_burst;
+#ifdef AESNI_MB_DOCSIS_SEC_ENABLED
+/**
+ * Configure a aesni multi-buffer session from a security session
+ * configuration
+ */
+static int
+aesni_mb_pmd_sec_sess_create(void *dev, struct rte_security_session_conf *conf,
+ struct rte_security_session *sess,
+ struct rte_mempool *mempool)
+{
+ void *sess_private_data;
+ struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev;
+ int ret;
- dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
- RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
- RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT |
- RTE_CRYPTODEV_FF_SYM_CPU_CRYPTO |
- RTE_CRYPTODEV_FF_NON_BYTE_ALIGNED_DATA |
- RTE_CRYPTODEV_FF_SYM_SESSIONLESS;
+ if (conf->action_type != RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL ||
+ conf->protocol != RTE_SECURITY_PROTOCOL_DOCSIS) {
+ IPSEC_MB_LOG(ERR, "Invalid security protocol");
+ return -EINVAL;
+ }
-#ifdef AESNI_MB_DOCSIS_SEC_ENABLED
- struct rte_security_ctx *security_instance;
- security_instance = rte_malloc("aesni_mb_sec",
- sizeof(struct rte_security_ctx),
- RTE_CACHE_LINE_SIZE);
- if (security_instance == NULL) {
- AESNI_MB_LOG(ERR, "rte_security_ctx memory alloc failed");
- rte_cryptodev_pmd_destroy(dev);
+ if (rte_mempool_get(mempool, &sess_private_data)) {
+ IPSEC_MB_LOG(ERR, "Couldn't get object from session mempool");
return -ENOMEM;
}
- security_instance->device = (void *)dev;
- security_instance->ops = rte_aesni_mb_pmd_sec_ops;
- security_instance->sess_cnt = 0;
- dev->security_ctx = security_instance;
- dev->feature_flags |= RTE_CRYPTODEV_FF_SECURITY;
-#endif
-
- /* Check CPU for support for AES instruction set */
- if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AES))
- dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AESNI;
- else
- AESNI_MB_LOG(WARNING, "AES instructions not supported by CPU");
+ ret = aesni_mb_set_docsis_sec_session_parameters(cdev, conf,
+ sess_private_data);
- dev->feature_flags |= vec_mode_to_flags(vector_mode);
+ if (ret != 0) {
+ IPSEC_MB_LOG(ERR, "Failed to configure session parameters");
- mb_mgr = alloc_init_mb_mgr(vector_mode);
- if (mb_mgr == NULL) {
-#ifdef AESNI_MB_DOCSIS_SEC_ENABLED
- rte_free(dev->security_ctx);
- dev->security_ctx = NULL;
-#endif
- rte_cryptodev_pmd_destroy(dev);
- return -ENOMEM;
+ /* Return session to mempool */
+ rte_mempool_put(mempool, sess_private_data);
+ return ret;
}
- /* Set vector instructions mode supported */
- internals = dev->data->dev_private;
-
- internals->vector_mode = vector_mode;
- internals->max_nb_queue_pairs = init_params->max_nb_queue_pairs;
- internals->mb_mgr = mb_mgr;
+ set_sec_session_private_data(sess, sess_private_data);
- AESNI_MB_LOG(INFO, "IPSec Multi-buffer library version used: %s\n",
- imb_get_version_str());
- return 0;
+ return ret;
}
+/** Clear the memory of session so it does not leave key material behind */
static int
-cryptodev_aesni_mb_probe(struct rte_vdev_device *vdev)
+aesni_mb_pmd_sec_sess_destroy(void *dev __rte_unused,
+ struct rte_security_session *sess)
{
- struct rte_cryptodev_pmd_init_params init_params = {
- "",
- sizeof(struct aesni_mb_private),
- rte_socket_id(),
- RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
- };
- const char *name, *args;
- int retval;
-
- name = rte_vdev_device_name(vdev);
- if (name == NULL)
- return -EINVAL;
+ void *sess_priv = get_sec_session_private_data(sess);
- args = rte_vdev_device_args(vdev);
+ if (sess_priv) {
+ struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
- retval = rte_cryptodev_pmd_parse_input_args(&init_params, args);
- if (retval) {
- AESNI_MB_LOG(ERR, "Failed to parse initialisation arguments[%s]",
- args);
- return -EINVAL;
+ memset(sess_priv, 0, sizeof(struct aesni_mb_session));
+ set_sec_session_private_data(sess, NULL);
+ rte_mempool_put(sess_mp, sess_priv);
}
-
- return cryptodev_aesni_mb_create(name, vdev, &init_params);
+ return 0;
}
-static int
-cryptodev_aesni_mb_remove(struct rte_vdev_device *vdev)
+static const struct rte_cryptodev_capabilities
+ aesni_mb_pmd_security_crypto_cap[] = {
+ { /* AES DOCSIS BPI */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
+ {.cipher = {
+ .algo = RTE_CRYPTO_CIPHER_AES_DOCSISBPI,
+ .block_size = 16,
+ .key_size = {
+ .min = 16,
+ .max = 32,
+ .increment = 16
+ },
+ .iv_size = {
+ .min = 16,
+ .max = 16,
+ .increment = 0
+ }
+ }, }
+ }, }
+ },
+
+ RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
+};
+
+static const struct rte_security_capability aesni_mb_pmd_security_cap[] = {
+ { /* DOCSIS Uplink */
+ .action = RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL,
+ .protocol = RTE_SECURITY_PROTOCOL_DOCSIS,
+ .docsis = {
+ .direction = RTE_SECURITY_DOCSIS_UPLINK
+ },
+ .crypto_capabilities = aesni_mb_pmd_security_crypto_cap
+ },
+ { /* DOCSIS Downlink */
+ .action = RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL,
+ .protocol = RTE_SECURITY_PROTOCOL_DOCSIS,
+ .docsis = {
+ .direction = RTE_SECURITY_DOCSIS_DOWNLINK
+ },
+ .crypto_capabilities = aesni_mb_pmd_security_crypto_cap
+ },
+ {
+ .action = RTE_SECURITY_ACTION_TYPE_NONE
+ }
+};
+
+/** Get security capabilities for aesni multi-buffer */
+static const struct rte_security_capability *
+aesni_mb_pmd_sec_capa_get(void *device __rte_unused)
{
- struct rte_cryptodev *cryptodev;
- struct aesni_mb_private *internals;
- const char *name;
+ return aesni_mb_pmd_security_cap;
+}
- name = rte_vdev_device_name(vdev);
- if (name == NULL)
- return -EINVAL;
+static struct rte_security_ops aesni_mb_pmd_sec_ops = {
+ .session_create = aesni_mb_pmd_sec_sess_create,
+ .session_update = NULL,
+ .session_stats_get = NULL,
+ .session_destroy = aesni_mb_pmd_sec_sess_destroy,
+ .set_pkt_metadata = NULL,
+ .capabilities_get = aesni_mb_pmd_sec_capa_get
+};
- cryptodev = rte_cryptodev_pmd_get_named_dev(name);
- if (cryptodev == NULL)
- return -ENODEV;
+struct rte_security_ops *rte_aesni_mb_pmd_sec_ops = &aesni_mb_pmd_sec_ops;
+
+static int
+aesni_mb_configure_dev(struct rte_cryptodev *dev)
+{
+ struct rte_security_ctx *security_instance;
- internals = cryptodev->data->dev_private;
+ security_instance = rte_malloc("aesni_mb_sec",
+ sizeof(struct rte_security_ctx),
+ RTE_CACHE_LINE_SIZE);
+ if (security_instance != NULL) {
+ security_instance->device = (void *)dev;
+ security_instance->ops = rte_aesni_mb_pmd_sec_ops;
+ security_instance->sess_cnt = 0;
+ dev->security_ctx = security_instance;
- free_mb_mgr(internals->mb_mgr);
- if (RTE_PER_LCORE(sync_mb_mgr)) {
- free_mb_mgr(RTE_PER_LCORE(sync_mb_mgr));
- RTE_PER_LCORE(sync_mb_mgr) = NULL;
+ return 0;
}
-#ifdef AESNI_MB_DOCSIS_SEC_ENABLED
- rte_free(cryptodev->security_ctx);
- cryptodev->security_ctx = NULL;
+ return -ENOMEM;
+}
+
#endif
- return rte_cryptodev_pmd_destroy(cryptodev);
+static int
+cryptodev_aesni_mb_probe(struct rte_vdev_device *vdev)
+{
+ return cryptodev_ipsec_mb_create(vdev, IPSEC_MB_PMD_TYPE_AESNI_MB);
}
static struct rte_vdev_driver cryptodev_aesni_mb_pmd_drv = {
.probe = cryptodev_aesni_mb_probe,
- .remove = cryptodev_aesni_mb_remove
+ .remove = cryptodev_ipsec_mb_remove
};
static struct cryptodev_driver aesni_mb_crypto_drv;
-RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_AESNI_MB_PMD, cryptodev_aesni_mb_pmd_drv);
+RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_AESNI_MB_PMD,
+ cryptodev_aesni_mb_pmd_drv);
RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_AESNI_MB_PMD, cryptodev_aesni_mb_pmd);
RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_AESNI_MB_PMD,
- "max_nb_queue_pairs=<int> "
- "socket_id=<int>");
-RTE_PMD_REGISTER_CRYPTO_DRIVER(aesni_mb_crypto_drv,
- cryptodev_aesni_mb_pmd_drv.driver,
- cryptodev_driver_id);
-RTE_LOG_REGISTER_DEFAULT(aesni_mb_logtype_driver, NOTICE);
+ "max_nb_queue_pairs=<int> socket_id=<int>");
+RTE_PMD_REGISTER_CRYPTO_DRIVER(
+ aesni_mb_crypto_drv,
+ cryptodev_aesni_mb_pmd_drv.driver,
+ pmd_driver_id_aesni_mb);
+
+/* Constructor function to register aesni-mb PMD */
+RTE_INIT(ipsec_mb_register_aesni_mb)
+{
+ struct ipsec_mb_pmd_data *aesni_mb_data =
+ &ipsec_mb_pmds[IPSEC_MB_PMD_TYPE_AESNI_MB];
+
+ aesni_mb_data->caps = aesni_mb_capabilities;
+ aesni_mb_data->dequeue_burst = aesni_mb_dequeue_burst;
+ aesni_mb_data->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
+ RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
+ RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT |
+ RTE_CRYPTODEV_FF_SYM_CPU_CRYPTO |
+ RTE_CRYPTODEV_FF_NON_BYTE_ALIGNED_DATA |
+ RTE_CRYPTODEV_FF_SYM_SESSIONLESS;
+
+ aesni_mb_data->internals_priv_size = 0;
+ aesni_mb_data->ops = &aes_mb_pmd_ops;
+ aesni_mb_data->qp_priv_size = sizeof(struct aesni_mb_qp_data);
+ aesni_mb_data->queue_pair_configure = NULL;
+#ifdef AESNI_MB_DOCSIS_SEC_ENABLED
+ aesni_mb_data->security_ops = &aesni_mb_pmd_sec_ops;
+ aesni_mb_data->dev_config = aesni_mb_configure_dev;
+ aesni_mb_data->feature_flags |= RTE_CRYPTODEV_FF_SECURITY;
+#endif
+ aesni_mb_data->session_configure = aesni_mb_session_configure;
+ aesni_mb_data->session_priv_size = sizeof(struct aesni_mb_session);
+}
diff --git a/drivers/crypto/ipsec_mb/rte_ipsec_mb_pmd.c b/drivers/crypto/ipsec_mb/rte_ipsec_mb_pmd.c
index 6ce1a9fe1d..18a66174ae 100644
--- a/drivers/crypto/ipsec_mb/rte_ipsec_mb_pmd.c
+++ b/drivers/crypto/ipsec_mb/rte_ipsec_mb_pmd.c
@@ -106,6 +106,7 @@ cryptodev_ipsec_mb_create(struct rte_vdev_device *vdev,
dev->dev_ops = ipsec_mb_pmds[pmd_type].ops;
dev->enqueue_burst = ipsec_mb_pmd_enqueue_burst;
dev->dequeue_burst = ipsec_mb_pmds[pmd_type].dequeue_burst;
+ dev->feature_flags = pmd_data->feature_flags;
if (pmd_data->dev_config) {
retval = (*pmd_data->dev_config)(dev);
@@ -117,8 +118,6 @@ cryptodev_ipsec_mb_create(struct rte_vdev_device *vdev,
}
}
- dev->feature_flags = pmd_data->feature_flags;
-
switch (vector_mode) {
case IPSEC_MB_AVX512:
dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX512;
@@ -170,6 +169,10 @@ cryptodev_ipsec_mb_remove(struct rte_vdev_device *vdev)
rte_free(cryptodev->security_ctx);
cryptodev->security_ctx = NULL;
}
+#ifdef AESNI_MB_DOCSIS_SEC_ENABLED
+ rte_free(cryptodev->security_ctx);
+ cryptodev->security_ctx = NULL;
+#endif
return rte_cryptodev_pmd_destroy(cryptodev);
}
diff --git a/drivers/crypto/ipsec_mb/rte_ipsec_mb_pmd_private.h b/drivers/crypto/ipsec_mb/rte_ipsec_mb_pmd_private.h
index 85733d328d..bcfe6c72ce 100644
--- a/drivers/crypto/ipsec_mb/rte_ipsec_mb_pmd_private.h
+++ b/drivers/crypto/ipsec_mb/rte_ipsec_mb_pmd_private.h
@@ -31,6 +31,9 @@ extern enum ipsec_mb_vector_mode vector_mode;
/** MB_MGR instances, one per thread */
extern RTE_DEFINE_PER_LCORE(MB_MGR *, mb_mgr);
+#define CRYPTODEV_NAME_AESNI_MB_PMD crypto_aesni_mb
+/**< IPSEC Multi buffer aesni_mb PMD device name */
+
/** PMD LOGTYPE DRIVER, common to all PMDs */
extern int ipsec_mb_logtype_driver;
#define IPSEC_MB_LOG(level, fmt, ...) \
@@ -39,6 +42,7 @@ extern int ipsec_mb_logtype_driver;
/** All supported device types */
enum ipsec_mb_pmd_types {
+ IPSEC_MB_PMD_TYPE_AESNI_MB = 0,
IPSEC_MB_N_PMD_TYPES
};
@@ -57,10 +61,18 @@ enum ipsec_mb_operation {
IPSEC_MB_OP_NOT_SUPPORTED
};
+extern uint8_t pmd_driver_id_aesni_mb;
+
/** Helper function. Gets driver ID based on PMD type */
static __rte_always_inline uint8_t
-ipsec_mb_get_driver_id(__rte_unused enum ipsec_mb_pmd_types pmd_type)
+ipsec_mb_get_driver_id(enum ipsec_mb_pmd_types pmd_type)
{
+ switch (pmd_type) {
+ case IPSEC_MB_PMD_TYPE_AESNI_MB:
+ return pmd_driver_id_aesni_mb;
+ default:
+ break;
+ }
return UINT8_MAX;
}
@@ -148,6 +160,137 @@ get_per_thread_mb_mgr(void)
return RTE_PER_LCORE(mb_mgr);
}
+/** Helper function. Gets mode and chained xforms from the xform */
+static __rte_always_inline int
+ipsec_mb_parse_xform(const struct rte_crypto_sym_xform *xform,
+ enum ipsec_mb_operation *mode,
+ const struct rte_crypto_sym_xform **auth_xform,
+ const struct rte_crypto_sym_xform **cipher_xform,
+ const struct rte_crypto_sym_xform **aead_xform)
+{
+ const struct rte_crypto_sym_xform *next = xform->next;
+
+ if (xform == NULL) {
+ *mode = IPSEC_MB_OP_NOT_SUPPORTED;
+ return -ENOTSUP;
+ }
+
+ if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
+ if (next == NULL) {
+ if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
+ *mode = IPSEC_MB_OP_ENCRYPT_ONLY;
+ *cipher_xform = xform;
+ *auth_xform = NULL;
+ return 0;
+ }
+ *mode = IPSEC_MB_OP_DECRYPT_ONLY;
+ *cipher_xform = xform;
+ *auth_xform = NULL;
+ return 0;
+ }
+
+ if (next->type != RTE_CRYPTO_SYM_XFORM_AUTH) {
+ *mode = IPSEC_MB_OP_NOT_SUPPORTED;
+ return -ENOTSUP;
+ }
+
+ if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
+ if (next->auth.op != RTE_CRYPTO_AUTH_OP_GENERATE) {
+ *mode = IPSEC_MB_OP_NOT_SUPPORTED;
+ return -ENOTSUP;
+ }
+
+ *mode = IPSEC_MB_OP_ENCRYPT_THEN_HASH_GEN;
+ *cipher_xform = xform;
+ *auth_xform = xform->next;
+ return 0;
+ }
+ if (next->auth.op != RTE_CRYPTO_AUTH_OP_VERIFY) {
+ *mode = IPSEC_MB_OP_NOT_SUPPORTED;
+ return -ENOTSUP;
+ }
+
+ *mode = IPSEC_MB_OP_DECRYPT_THEN_HASH_VERIFY;
+ *cipher_xform = xform;
+ *auth_xform = xform->next;
+ return 0;
+ }
+
+ if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
+ if (next == NULL) {
+ if (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) {
+ *mode = IPSEC_MB_OP_HASH_GEN_ONLY;
+ *auth_xform = xform;
+ *cipher_xform = NULL;
+ return 0;
+ }
+ *mode = IPSEC_MB_OP_HASH_VERIFY_ONLY;
+ *auth_xform = xform;
+ *cipher_xform = NULL;
+ return 0;
+ }
+
+ if (next->type != RTE_CRYPTO_SYM_XFORM_CIPHER) {
+ *mode = IPSEC_MB_OP_NOT_SUPPORTED;
+ return -ENOTSUP;
+ }
+
+ if (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) {
+ if (next->cipher.op != RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
+ *mode = IPSEC_MB_OP_NOT_SUPPORTED;
+ return -ENOTSUP;
+ }
+
+ *mode = IPSEC_MB_OP_HASH_GEN_THEN_ENCRYPT;
+ *auth_xform = xform;
+ *cipher_xform = xform->next;
+ return 0;
+ }
+ if (next->cipher.op != RTE_CRYPTO_CIPHER_OP_DECRYPT) {
+ *mode = IPSEC_MB_OP_NOT_SUPPORTED;
+ return -ENOTSUP;
+ }
+
+ *mode = IPSEC_MB_OP_HASH_VERIFY_THEN_DECRYPT;
+ *auth_xform = xform;
+ *cipher_xform = xform->next;
+ return 0;
+ }
+
+ if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
+ if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) {
+ /*
+ * CCM requires to hash first and cipher later
+ * when encrypting
+ */
+ if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM) {
+ *mode = IPSEC_MB_OP_AEAD_AUTHENTICATED_DECRYPT;
+ *aead_xform = xform;
+ return 0;
+ } else {
+ *mode =
+ IPSEC_MB_OP_AEAD_AUTHENTICATED_ENCRYPT;
+ *aead_xform = xform;
+ return 0;
+ }
+ } else {
+ if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM) {
+ *mode = IPSEC_MB_OP_AEAD_AUTHENTICATED_ENCRYPT;
+ *aead_xform = xform;
+ return 0;
+ }
+ *mode = IPSEC_MB_OP_AEAD_AUTHENTICATED_DECRYPT;
+ *aead_xform = xform;
+ return 0;
+ }
+ }
+
+ *mode = IPSEC_MB_OP_NOT_SUPPORTED;
+ return -ENOTSUP;
+
+}
+
+
/** Device creation function */
int
cryptodev_ipsec_mb_create(struct rte_vdev_device *vdev,
diff --git a/drivers/crypto/meson.build b/drivers/crypto/meson.build
index 216a77e3bc..d22b1eeaf3 100644
--- a/drivers/crypto/meson.build
+++ b/drivers/crypto/meson.build
@@ -8,7 +8,6 @@ endif
drivers = [
'ipsec_mb',
'aesni_gcm',
- 'aesni_mb',
'armv8',
'bcmfs',
'caam_jr',
--
2.25.1
--------------------------------------------------------------
Intel Research and Development Ireland Limited
Registered in Ireland
Registered Office: Collinstown Industrial Park, Leixlip, County Kildare
Registered Number: 308263
This e-mail and any attachments may contain confidential material for the sole
use of the intended recipient(s). Any review or distribution by others is
strictly prohibited. If you are not the intended recipient, please contact the
sender and delete all copies.
next prev parent reply other threads:[~2021-06-18 12:19 UTC|newest]
Thread overview: 67+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-06-18 12:17 [dpdk-dev] [RFC 0/7] crypto/ipsec_mb: introduce " pbronowx
2021-06-18 12:17 ` [dpdk-dev] [RFC 1/7] " pbronowx
2021-06-18 12:17 ` pbronowx [this message]
2021-06-18 12:17 ` [dpdk-dev] [RFC 3/7] crypto/ipsec_mb: move aesni-gcm PMD to " pbronowx
2021-06-18 12:18 ` [dpdk-dev] [RFC 4/7] crypto/ipsec_mb: move kasumi " pbronowx
2021-06-18 12:18 ` [dpdk-dev] [RFC 5/7] crypto/ipsec_mb: move snow3g " pbronowx
2021-06-18 12:18 ` [dpdk-dev] [RFC 6/7] crypto/snow3g: add support for digest appended ops pbronowx
2021-06-18 12:18 ` [dpdk-dev] [RFC 7/7] crypto/ipsec_mb: move zuc PMD to ipsec_mb framework pbronowx
2021-06-18 13:11 ` [dpdk-dev] [RFC 0/7] crypto/ipsec_mb: introduce " David Marchand
2021-06-18 16:05 ` [dpdk-dev] [EXT] " Akhil Goyal
2021-06-21 8:52 ` Zhang, Roy Fan
2021-08-26 15:16 ` [dpdk-dev] [PATCH v1 0/8] drivers/crypto: " Ciara Power
2021-08-26 15:16 ` [dpdk-dev] [PATCH v1 1/8] drivers/crypto: introduce IPsec-mb framework Ciara Power
2021-08-26 15:16 ` [dpdk-dev] [PATCH v1 2/8] drivers/crypto: move aesni-mb PMD to " Ciara Power
2021-08-26 15:16 ` [dpdk-dev] [PATCH v1 3/8] drivers/crypto: move aesni-gcm " Ciara Power
2021-08-26 15:16 ` [dpdk-dev] [PATCH v1 4/8] drivers/crypto: move kasumi " Ciara Power
2021-08-26 15:16 ` [dpdk-dev] [PATCH v1 5/8] drivers/crypto: move snow3g " Ciara Power
2021-08-26 15:16 ` [dpdk-dev] [PATCH v1 6/8] crypto/ipsec_mb: add snow3g digest appended ops support Ciara Power
2021-08-26 15:16 ` [dpdk-dev] [PATCH v1 7/8] drivers/crypto: move zuc PMD to IPsec-mb framework Ciara Power
2021-08-26 15:16 ` [dpdk-dev] [PATCH v1 8/8] crypto/ipsec_mb: add chacha20-poly1305 PMD to framework Ciara Power
2021-09-23 15:28 ` [dpdk-dev] [PATCH v2 0/9] drivers/crypto: introduce ipsec_mb framework Ciara Power
2021-09-23 15:28 ` [dpdk-dev] [PATCH v2 1/9] drivers/crypto: introduce IPsec-mb framework Ciara Power
2021-09-23 15:46 ` Thomas Monjalon
2021-09-27 10:02 ` Power, Ciara
2021-09-23 15:28 ` [dpdk-dev] [PATCH v2 2/9] drivers/crypto: move aesni-mb PMD to " Ciara Power
2021-09-23 15:28 ` [dpdk-dev] [PATCH v2 3/9] drivers/crypto: move aesni-gcm " Ciara Power
2021-09-23 15:28 ` [dpdk-dev] [PATCH v2 4/9] drivers/crypto: move kasumi " Ciara Power
2021-09-23 15:28 ` [dpdk-dev] [PATCH v2 5/9] drivers/crypto: move snow3g " Ciara Power
2021-09-23 15:28 ` [dpdk-dev] [PATCH v2 6/9] crypto/ipsec_mb: add snow3g digest appended ops support Ciara Power
2021-09-23 15:28 ` [dpdk-dev] [PATCH v2 7/9] drivers/crypto: move zuc PMD to IPsec-mb framework Ciara Power
2021-09-23 15:28 ` [dpdk-dev] [PATCH v2 8/9] crypto/ipsec_mb: add chacha20-poly1305 PMD to framework Ciara Power
2021-09-23 15:28 ` [dpdk-dev] [PATCH v2 9/9] doc/rel_notes: added note for SW Crypto PMD change Ciara Power
2021-09-28 16:43 ` [dpdk-dev] [EXT] [PATCH v2 0/9] drivers/crypto: introduce ipsec_mb framework Akhil Goyal
2021-10-15 14:39 ` [dpdk-dev] [PATCH v4 00/14] " Ciara Power
2021-10-15 14:39 ` [dpdk-dev] [PATCH v4 01/14] drivers/crypto: introduce IPsec-mb framework Ciara Power
2021-10-15 14:39 ` [dpdk-dev] [PATCH v4 02/14] crypto/ipsec_mb: add multiprocess support Ciara Power
2021-10-15 14:39 ` [dpdk-dev] [PATCH v4 03/14] drivers/crypto: move aesni-mb PMD to IPsec-mb framework Ciara Power
2021-10-18 7:38 ` Zhang, Roy Fan
2021-10-15 14:39 ` [dpdk-dev] [PATCH v4 04/14] crypto/ipsec_mb: support ZUC-256 for aesni_mb Ciara Power
2021-10-18 7:39 ` Zhang, Roy Fan
2021-10-15 14:39 ` [dpdk-dev] [PATCH v4 05/14] test/crypto: check cipher parameters Ciara Power
2021-10-18 7:40 ` Zhang, Roy Fan
2021-10-15 14:39 ` [dpdk-dev] [PATCH v4 06/14] test/crypto: check auth parameters Ciara Power
2021-10-18 7:41 ` Zhang, Roy Fan
2021-10-15 14:39 ` [dpdk-dev] [PATCH v4 07/14] test/crypto: add ZUC-256 vectors Ciara Power
2021-10-18 7:42 ` Zhang, Roy Fan
2021-10-15 14:39 ` [dpdk-dev] [PATCH v4 08/14] drivers/crypto: move aesni-gcm PMD to IPsec-mb framework Ciara Power
2021-10-18 7:43 ` Zhang, Roy Fan
2021-10-15 14:39 ` [dpdk-dev] [PATCH v4 09/14] drivers/crypto: move kasumi " Ciara Power
2021-10-18 7:42 ` Zhang, Roy Fan
2021-10-15 14:39 ` [dpdk-dev] [PATCH v4 10/14] drivers/crypto: move snow3g " Ciara Power
2021-10-18 7:42 ` Zhang, Roy Fan
2021-10-15 14:39 ` [dpdk-dev] [PATCH v4 11/14] crypto/ipsec_mb: add snow3g digest appended ops support Ciara Power
2021-10-18 7:43 ` Zhang, Roy Fan
2021-10-15 14:39 ` [dpdk-dev] [PATCH v4 12/14] drivers/crypto: move zuc PMD to IPsec-mb framework Ciara Power
2021-10-18 7:44 ` Zhang, Roy Fan
2021-10-15 14:39 ` [dpdk-dev] [PATCH v4 13/14] crypto/ipsec_mb: add chacha20-poly1305 PMD to framework Ciara Power
2021-10-18 7:44 ` Zhang, Roy Fan
2021-10-15 14:39 ` [dpdk-dev] [PATCH v4 14/14] test/crypto: add test for chacha20_poly1305 PMD Ciara Power
2021-10-18 7:44 ` Zhang, Roy Fan
2021-10-18 15:21 ` [dpdk-dev] [EXT] [PATCH v4 00/14] drivers/crypto: introduce ipsec_mb framework Akhil Goyal
2021-10-19 23:09 ` Thomas Monjalon
2021-10-20 4:23 ` Akhil Goyal
2021-10-20 8:31 ` Akhil Goyal
2021-10-20 9:01 ` Thomas Monjalon
2021-10-20 9:00 ` Zhang, Roy Fan
2021-10-20 9:04 ` Akhil Goyal
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210618121803.1189857-3-piotrx.bronowski@intel.com \
--to=piotrx.bronowski@intel.com \
--cc=declan.doherty@intel.com \
--cc=dev@dpdk.org \
--cc=ferruh.yigit@intel.com \
--cc=gakhil@marvell.com \
--cc=roy.fan.zhang@intel.com \
--cc=thomas@monjalon.net \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).