From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 55195A2EDB for ; Fri, 6 Sep 2019 15:14:30 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 4CD3C1F3A1; Fri, 6 Sep 2019 15:13:50 +0200 (CEST) Received: from mga18.intel.com (mga18.intel.com [134.134.136.126]) by dpdk.org (Postfix) with ESMTP id 0C92E1F384 for ; Fri, 6 Sep 2019 15:13:42 +0200 (CEST) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga002.fm.intel.com ([10.253.24.26]) by orsmga106.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 06 Sep 2019 06:13:42 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.64,473,1559545200"; d="scan'208";a="213140755" Received: from silpixa00398673.ir.intel.com (HELO silpixa00398673.ger.corp.intel.com) ([10.237.223.136]) by fmsmga002.fm.intel.com with ESMTP; 06 Sep 2019 06:13:40 -0700 From: Fan Zhang To: dev@dpdk.org Cc: konstantin.ananyev@intel.com, declan.doherty@intel.com, akhil.goyal@nxp.com, Fan Zhang Date: Fri, 6 Sep 2019 14:13:25 +0100 Message-Id: <20190906131330.40185-6-roy.fan.zhang@intel.com> X-Mailer: git-send-email 2.14.5 In-Reply-To: <20190906131330.40185-1-roy.fan.zhang@intel.com> References: <20190903154046.55992-1-roy.fan.zhang@intel.com> <20190906131330.40185-1-roy.fan.zhang@intel.com> Subject: [dpdk-dev] [PATCH 05/10] crypto/aesni_mb: add rte_security handler X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" This patch add rte_security support support to AESNI-MB PMD. The PMD now initialize security context instance, create/delete PMD specific security sessions, and process crypto workloads in synchronous mode. Signed-off-by: Fan Zhang --- drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c | 291 ++++++++++++++++++++- drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c | 91 ++++++- drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h | 21 +- 3 files changed, 398 insertions(+), 5 deletions(-) diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c index b495a9679..68767c04e 100644 --- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c +++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c @@ -8,6 +8,8 @@ #include #include #include +#include +#include #include #include #include @@ -789,6 +791,167 @@ auth_start_offset(struct rte_crypto_op *op, struct aesni_mb_session *session, (UINT64_MAX - u_src + u_dst + 1); } +union sec_userdata_field { + int status; + struct { + uint16_t is_gen_digest; + uint16_t digest_len; + }; +}; + +struct sec_udata_digest_field { + uint32_t is_digest_gen; + uint32_t digest_len; +}; + +static inline int +set_mb_job_params_sec(JOB_AES_HMAC *job, struct aesni_mb_sec_session *sec_sess, + void *buf, uint32_t buf_len, void *iv, void *aad, void *digest, + int *status, uint8_t *digest_idx) +{ + struct aesni_mb_session *session = &sec_sess->sess; + uint32_t cipher_offset = sec_sess->cipher_offset; + void *user_digest = NULL; + union sec_userdata_field udata; + + if (unlikely(cipher_offset > buf_len)) + return -EINVAL; + + /* Set crypto operation */ + job->chain_order = session->chain_order; + + /* Set cipher parameters */ + job->cipher_direction = session->cipher.direction; + job->cipher_mode = session->cipher.mode; + + job->aes_key_len_in_bytes = session->cipher.key_length_in_bytes; + + /* Set authentication parameters */ + job->hash_alg = session->auth.algo; + job->iv = iv; + + switch (job->hash_alg) { + case AES_XCBC: + job->u.XCBC._k1_expanded = session->auth.xcbc.k1_expanded; + job->u.XCBC._k2 = session->auth.xcbc.k2; + job->u.XCBC._k3 = session->auth.xcbc.k3; + + job->aes_enc_key_expanded = + session->cipher.expanded_aes_keys.encode; + job->aes_dec_key_expanded = + session->cipher.expanded_aes_keys.decode; + break; + + case AES_CCM: + job->u.CCM.aad = (uint8_t *)aad + 18; + job->u.CCM.aad_len_in_bytes = session->aead.aad_len; + job->aes_enc_key_expanded = + session->cipher.expanded_aes_keys.encode; + job->aes_dec_key_expanded = + session->cipher.expanded_aes_keys.decode; + job->iv++; + break; + + case AES_CMAC: + job->u.CMAC._key_expanded = session->auth.cmac.expkey; + job->u.CMAC._skey1 = session->auth.cmac.skey1; + job->u.CMAC._skey2 = session->auth.cmac.skey2; + job->aes_enc_key_expanded = + session->cipher.expanded_aes_keys.encode; + job->aes_dec_key_expanded = + session->cipher.expanded_aes_keys.decode; + break; + + case AES_GMAC: + if (session->cipher.mode == GCM) { + job->u.GCM.aad = aad; + job->u.GCM.aad_len_in_bytes = session->aead.aad_len; + } else { + /* For GMAC */ + job->u.GCM.aad = aad; + job->u.GCM.aad_len_in_bytes = buf_len; + job->cipher_mode = GCM; + } + job->aes_enc_key_expanded = &session->cipher.gcm_key; + job->aes_dec_key_expanded = &session->cipher.gcm_key; + break; + + default: + job->u.HMAC._hashed_auth_key_xor_ipad = + session->auth.pads.inner; + job->u.HMAC._hashed_auth_key_xor_opad = + session->auth.pads.outer; + + if (job->cipher_mode == DES3) { + job->aes_enc_key_expanded = + session->cipher.exp_3des_keys.ks_ptr; + job->aes_dec_key_expanded = + session->cipher.exp_3des_keys.ks_ptr; + } else { + job->aes_enc_key_expanded = + session->cipher.expanded_aes_keys.encode; + job->aes_dec_key_expanded = + session->cipher.expanded_aes_keys.decode; + } + } + + /* Set digest output location */ + if (job->hash_alg != NULL_HASH && + session->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) { + job->auth_tag_output = sec_sess->temp_digests[*digest_idx]; + *digest_idx = (*digest_idx + 1) % MAX_JOBS; + + udata.is_gen_digest = 0; + udata.digest_len = session->auth.req_digest_len; + user_digest = (void *)digest; + } else { + udata.is_gen_digest = 1; + udata.digest_len = session->auth.req_digest_len; + + if (session->auth.req_digest_len != + session->auth.gen_digest_len) { + job->auth_tag_output = + sec_sess->temp_digests[*digest_idx]; + *digest_idx = (*digest_idx + 1) % MAX_JOBS; + + user_digest = (void *)digest; + } else + job->auth_tag_output = digest; + + /* A bit of hack here, since job structure only supports + * 2 user data fields and we need 4 params to be passed + * (status, direction, digest for verify, and length of + * digest), we set the status value as digest length + + * direction here temporarily to avoid creating longer + * buffer to store all 4 params. + */ + *status = udata.status; + } + /* + * Multi-buffer library current only support returning a truncated + * digest length as specified in the relevant IPsec RFCs + */ + + /* Set digest length */ + job->auth_tag_output_len_in_bytes = session->auth.gen_digest_len; + + /* Set IV parameters */ + job->iv_len_in_bytes = session->iv.length; + + /* Data Parameters */ + job->src = buf; + job->dst = buf; + job->cipher_start_src_offset_in_bytes = cipher_offset; + job->msg_len_to_cipher_in_bytes = buf_len - cipher_offset; + job->hash_start_src_offset_in_bytes = 0; + job->msg_len_to_hash_in_bytes = buf_len; + + job->user_data = (void *)status; + job->user_data2 = user_digest; + + return 0; +} + /** * Process a crypto operation and complete a JOB_AES_HMAC job structure for * submission to the multi buffer library for processing. @@ -1081,6 +1244,37 @@ post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job) return op; } +static inline void +post_process_mb_sec_job(JOB_AES_HMAC *job) +{ + void *user_digest = job->user_data2; + int *status = job->user_data; + union sec_userdata_field udata; + + switch (job->status) { + case STS_COMPLETED: + if (user_digest) { + udata.status = *status; + + if (udata.is_gen_digest) { + *status = RTE_CRYPTO_OP_STATUS_SUCCESS; + memcpy(user_digest, job->auth_tag_output, + udata.digest_len); + } else { + verify_digest(job, user_digest, + udata.digest_len, (uint8_t *)status); + + if (*status == RTE_CRYPTO_OP_STATUS_AUTH_FAILED) + *status = -1; + } + } else + *status = RTE_CRYPTO_OP_STATUS_SUCCESS; + break; + default: + *status = RTE_CRYPTO_OP_STATUS_ERROR; + } +} + /** * Process a completed JOB_AES_HMAC job and keep processing jobs until * get_completed_job return NULL @@ -1117,6 +1311,32 @@ handle_completed_jobs(struct aesni_mb_qp *qp, JOB_AES_HMAC *job, return processed_jobs; } +static inline uint32_t +handle_completed_sec_jobs(JOB_AES_HMAC *job, MB_MGR *mb_mgr) +{ + uint32_t processed = 0; + + while (job != NULL) { + post_process_mb_sec_job(job); + job = IMB_GET_COMPLETED_JOB(mb_mgr); + processed++; + } + + return processed; +} + +static inline uint32_t +flush_mb_sec_mgr(MB_MGR *mb_mgr) +{ + JOB_AES_HMAC *job = IMB_FLUSH_JOB(mb_mgr); + uint32_t processed = 0; + + if (job) + processed = handle_completed_sec_jobs(job, mb_mgr); + + return processed; +} + static inline uint16_t flush_mb_mgr(struct aesni_mb_qp *qp, struct rte_crypto_op **ops, uint16_t nb_ops) @@ -1220,6 +1440,55 @@ aesni_mb_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, return processed_jobs; } +void +aesni_mb_sec_crypto_process_bulk(struct rte_security_session *sess, + struct rte_security_vec buf[], void *iv[], void *aad[], + void *digest[], int status[], uint32_t num) +{ + struct aesni_mb_sec_session *sec_sess = sess->sess_private_data; + JOB_AES_HMAC *job; + uint8_t digest_idx = sec_sess->digest_idx; + uint32_t i, processed = 0; + int ret; + + for (i = 0; i < num; i++) { + void *seg_buf = buf[i].vec[0].iov_base; + uint32_t buf_len = buf[i].vec[0].iov_len; + + job = IMB_GET_NEXT_JOB(sec_sess->mb_mgr); + if (unlikely(job == NULL)) { + processed += flush_mb_sec_mgr(sec_sess->mb_mgr); + + job = IMB_GET_NEXT_JOB(sec_sess->mb_mgr); + if (!job) + return; + } + + ret = set_mb_job_params_sec(job, sec_sess, seg_buf, buf_len, + iv[i], aad[i], digest[i], &status[i], + &digest_idx); + /* Submit job to multi-buffer for processing */ + if (ret) { + processed++; + status[i] = ret; + continue; + } + +#ifdef RTE_LIBRTE_PMD_AESNI_MB_DEBUG + job = IMB_SUBMIT_JOB(sec_sess->mb_mgr); +#else + job = IMB_SUBMIT_JOB_NOCHECK(sec_sess->mb_mgr); +#endif + + if (job) + processed += handle_completed_sec_jobs(job, + sec_sess->mb_mgr); + } + + while (processed < num) + processed += flush_mb_sec_mgr(sec_sess->mb_mgr); +} + static int cryptodev_aesni_mb_remove(struct rte_vdev_device *vdev); static int @@ -1229,8 +1498,10 @@ cryptodev_aesni_mb_create(const char *name, { struct rte_cryptodev *dev; struct aesni_mb_private *internals; + struct rte_security_ctx *sec_ctx; enum aesni_mb_vector_mode vector_mode; MB_MGR *mb_mgr; + char sec_name[RTE_DEV_NAME_MAX_LEN]; /* Check CPU for support for AES instruction set */ if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_AES)) { @@ -1264,7 +1535,8 @@ cryptodev_aesni_mb_create(const char *name, dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | RTE_CRYPTODEV_FF_CPU_AESNI | - RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT; + RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | + RTE_CRYPTODEV_FF_SECURITY; mb_mgr = alloc_mb_mgr(0); @@ -1303,11 +1575,28 @@ cryptodev_aesni_mb_create(const char *name, AESNI_MB_LOG(INFO, "IPSec Multi-buffer library version used: %s\n", imb_get_version_str()); + /* setup security operations */ + snprintf(sec_name, sizeof(sec_name) - 1, "aes_mb_sec_%u", + dev->driver_id); + sec_ctx = rte_zmalloc_socket(sec_name, + sizeof(struct rte_security_ctx), + RTE_CACHE_LINE_SIZE, init_params->socket_id); + if (sec_ctx == NULL) { + AESNI_MB_LOG(ERR, "memory allocation failed\n"); + goto error_exit; + } + + sec_ctx->device = (void *)dev; + sec_ctx->ops = rte_aesni_mb_pmd_security_ops; + dev->security_ctx = sec_ctx; + return 0; error_exit: if (mb_mgr) free_mb_mgr(mb_mgr); + if (sec_ctx) + rte_free(sec_ctx); rte_cryptodev_pmd_destroy(dev); diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c index 8d15b99d4..ca6cea775 100644 --- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c +++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c @@ -8,6 +8,7 @@ #include #include #include +#include #include "rte_aesni_mb_pmd_private.h" @@ -732,7 +733,8 @@ aesni_mb_pmd_qp_count(struct rte_cryptodev *dev) static unsigned aesni_mb_pmd_sym_session_get_size(struct rte_cryptodev *dev __rte_unused) { - return sizeof(struct aesni_mb_session); + return RTE_ALIGN_CEIL(sizeof(struct aesni_mb_session), + RTE_CACHE_LINE_SIZE); } /** Configure a aesni multi-buffer session from a crypto xform chain */ @@ -810,4 +812,91 @@ struct rte_cryptodev_ops aesni_mb_pmd_ops = { .sym_session_clear = aesni_mb_pmd_sym_session_clear }; +/** Set session authentication parameters */ + +static int +aesni_mb_security_session_create(void *dev, + struct rte_security_session_conf *conf, + struct rte_security_session *sess, + struct rte_mempool *mempool) +{ + struct rte_cryptodev *cdev = dev; + struct aesni_mb_private *internals = cdev->data->dev_private; + struct aesni_mb_sec_session *sess_priv; + int ret; + + if (!conf->crypto_xform) { + AESNI_MB_LOG(ERR, "Invalid security session conf"); + return -EINVAL; + } + + if (rte_mempool_get(mempool, (void **)(&sess_priv))) { + AESNI_MB_LOG(ERR, + "Couldn't get object from session mempool"); + return -ENOMEM; + } + + sess_priv->mb_mgr = internals->mb_mgr; + if (sess_priv->mb_mgr == NULL) + return -ENOMEM; + + sess_priv->cipher_offset = conf->cpucrypto.cipher_offset; + + ret = aesni_mb_set_session_parameters(sess_priv->mb_mgr, + &sess_priv->sess, conf->crypto_xform); + if (ret != 0) { + AESNI_MB_LOG(ERR, "failed configure session parameters"); + + rte_mempool_put(mempool, sess_priv); + } + + sess->sess_private_data = (void *)sess_priv; + + return ret; +} + +static int +aesni_mb_security_session_destroy(void *dev __rte_unused, + struct rte_security_session *sess) +{ + struct aesni_mb_sec_session *sess_priv = + get_sec_session_private_data(sess); + + if (sess_priv) { + struct rte_mempool *sess_mp = rte_mempool_from_obj( + (void *)sess_priv); + + memset(sess, 0, sizeof(struct aesni_mb_sec_session)); + set_sec_session_private_data(sess, NULL); + + if (sess_mp == NULL) { + AESNI_MB_LOG(ERR, "failed fetch session mempool"); + return -EINVAL; + } + + rte_mempool_put(sess_mp, sess_priv); + } + + return 0; +} + +static unsigned int +aesni_mb_sec_session_get_size(__rte_unused void *device) +{ + return RTE_ALIGN_CEIL(sizeof(struct aesni_mb_sec_session), + RTE_CACHE_LINE_SIZE); +} + +static struct rte_security_ops aesni_mb_security_ops = { + .session_create = aesni_mb_security_session_create, + .session_get_size = aesni_mb_sec_session_get_size, + .session_update = NULL, + .session_stats_get = NULL, + .session_destroy = aesni_mb_security_session_destroy, + .set_pkt_metadata = NULL, + .capabilities_get = NULL, + .process_cpu_crypto_bulk = aesni_mb_sec_crypto_process_bulk, +}; + struct rte_cryptodev_ops *rte_aesni_mb_pmd_ops = &aesni_mb_pmd_ops; +struct rte_security_ops *rte_aesni_mb_pmd_security_ops = &aesni_mb_security_ops; diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h index b794d4bc1..d1cf416ab 100644 --- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h +++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_private.h @@ -176,7 +176,6 @@ struct aesni_mb_qp { */ } __rte_cache_aligned; -/** AES-NI multi-buffer private session structure */ struct aesni_mb_session { JOB_CHAIN_ORDER chain_order; struct { @@ -265,16 +264,32 @@ struct aesni_mb_session { /** AAD data length */ uint16_t aad_len; } aead; -} __rte_cache_aligned; +}; + +/** AES-NI multi-buffer private security session structure */ +struct aesni_mb_sec_session { + /**< Unique Queue Pair Name */ + struct aesni_mb_session sess; + uint8_t temp_digests[MAX_JOBS][DIGEST_LENGTH_MAX]; + uint16_t digest_idx; + uint32_t cipher_offset; + MB_MGR *mb_mgr; +}; extern int aesni_mb_set_session_parameters(const MB_MGR *mb_mgr, struct aesni_mb_session *sess, const struct rte_crypto_sym_xform *xform); +extern void +aesni_mb_sec_crypto_process_bulk(struct rte_security_session *sess, + struct rte_security_vec buf[], void *iv[], void *aad[], + void *digest[], int status[], uint32_t num); + /** device specific operations function pointer structure */ extern struct rte_cryptodev_ops *rte_aesni_mb_pmd_ops; - +/** device specific operations function pointer structure for rte_security */ +extern struct rte_security_ops *rte_aesni_mb_pmd_security_ops; #endif /* _RTE_AESNI_MB_PMD_PRIVATE_H_ */ -- 2.14.5