From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 7DB5DA0514; Wed, 15 Jan 2020 19:30:45 +0100 (CET) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 7F8921C2A9; Wed, 15 Jan 2020 19:30:29 +0100 (CET) Received: from mga01.intel.com (mga01.intel.com [192.55.52.88]) by dpdk.org (Postfix) with ESMTP id 0DF231C29A for ; Wed, 15 Jan 2020 19:30:26 +0100 (CET) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga001.fm.intel.com ([10.253.24.23]) by fmsmga101.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 15 Jan 2020 10:30:26 -0800 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.70,323,1574150400"; d="scan'208";a="254502374" Received: from msmoczyx-mobl.ger.corp.intel.com ([10.104.125.3]) by fmsmga001.fm.intel.com with ESMTP; 15 Jan 2020 10:30:24 -0800 From: Marcin Smoczynski To: akhil.goyal@nxp.com, konstantin.ananyev@intel.com, roy.fan.zhang@intel.com, declan.doherty@intel.com, radu.nicolau@intel.com Cc: dev@dpdk.org, Marcin Smoczynski Date: Wed, 15 Jan 2020 19:28:28 +0100 Message-Id: <20200115182832.17012-3-marcinx.smoczynski@intel.com> X-Mailer: git-send-email 2.21.0.windows.1 In-Reply-To: <20200115182832.17012-1-marcinx.smoczynski@intel.com> References: <20200115182832.17012-1-marcinx.smoczynski@intel.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Subject: [dpdk-dev] [PATCH v3 2/6] crypto/aesni_gcm: cpu crypto support X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Add support for CPU crypto mode by introducing required handler. Crypto mode (sync/async) is chosen during sym session create if an appropriate flag is set in an xform type number. Authenticated encryption and decryption are supported with tag generation/verification. Signed-off-by: Marcin Smoczynski --- drivers/crypto/aesni_gcm/aesni_gcm_ops.h | 9 ++ drivers/crypto/aesni_gcm/aesni_gcm_pmd.c | 149 +++++++++++++++++- drivers/crypto/aesni_gcm/aesni_gcm_pmd_ops.c | 3 + .../crypto/aesni_gcm/aesni_gcm_pmd_private.h | 18 ++- 4 files changed, 169 insertions(+), 10 deletions(-) diff --git a/drivers/crypto/aesni_gcm/aesni_gcm_ops.h b/drivers/crypto/aesni_gcm/aesni_gcm_ops.h index e272f1067..404c0adff 100644 --- a/drivers/crypto/aesni_gcm/aesni_gcm_ops.h +++ b/drivers/crypto/aesni_gcm/aesni_gcm_ops.h @@ -65,4 +65,13 @@ struct aesni_gcm_ops { aesni_gcm_finalize_t finalize_dec; }; +/** GCM per-session operation handlers */ +struct aesni_gcm_session_ops { + aesni_gcm_t cipher; + aesni_gcm_pre_t pre; + aesni_gcm_init_t init; + aesni_gcm_update_t update; + aesni_gcm_finalize_t finalize; +}; + #endif /* _AESNI_GCM_OPS_H_ */ diff --git a/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c b/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c index 1a03be31d..860e9b369 100644 --- a/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c +++ b/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c @@ -25,9 +25,16 @@ aesni_gcm_set_session_parameters(const struct aesni_gcm_ops *gcm_ops, const struct rte_crypto_sym_xform *aead_xform; uint8_t key_length; const uint8_t *key; + uint32_t xform_type; + + /* check for CPU-crypto mode */ + xform_type = xform->type; + sess->mode = xform_type | RTE_CRYPTO_SYM_CPU_CRYPTO ? + AESNI_GCM_MODE_SYNC : AESNI_GCM_MODE_ASYNC; + xform_type &= RTE_CRYPTO_SYM_XFORM_TYPE_MASK; /* AES-GMAC */ - if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { + if (xform_type == RTE_CRYPTO_SYM_XFORM_AUTH) { auth_xform = xform; if (auth_xform->auth.algo != RTE_CRYPTO_AUTH_AES_GMAC) { AESNI_GCM_LOG(ERR, "Only AES GMAC is supported as an " @@ -49,7 +56,7 @@ aesni_gcm_set_session_parameters(const struct aesni_gcm_ops *gcm_ops, sess->req_digest_length = auth_xform->auth.digest_length; /* AES-GCM */ - } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) { + } else if (xform_type == RTE_CRYPTO_SYM_XFORM_AEAD) { aead_xform = xform; if (aead_xform->aead.algo != RTE_CRYPTO_AEAD_AES_GCM) { @@ -62,11 +69,24 @@ aesni_gcm_set_session_parameters(const struct aesni_gcm_ops *gcm_ops, sess->iv.offset = aead_xform->aead.iv.offset; sess->iv.length = aead_xform->aead.iv.length; + /* setup session handlers */ + sess->ops.pre = gcm_ops->pre; + sess->ops.init = gcm_ops->init; + /* Select Crypto operation */ - if (aead_xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) + if (aead_xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) { sess->op = AESNI_GCM_OP_AUTHENTICATED_ENCRYPTION; - else + sess->ops.cipher = gcm_ops->enc; + sess->ops.update = gcm_ops->update_enc; + sess->ops.finalize = gcm_ops->finalize_enc; + } + /* op == RTE_CRYPTO_AEAD_OP_DECRYPT */ + else { sess->op = AESNI_GCM_OP_AUTHENTICATED_DECRYPTION; + sess->ops.cipher = gcm_ops->dec; + sess->ops.update = gcm_ops->update_dec; + sess->ops.finalize = gcm_ops->finalize_dec; + } key_length = aead_xform->aead.key.length; key = aead_xform->aead.key.data; @@ -78,7 +98,6 @@ aesni_gcm_set_session_parameters(const struct aesni_gcm_ops *gcm_ops, return -ENOTSUP; } - /* IV check */ if (sess->iv.length != 16 && sess->iv.length != 12 && sess->iv.length != 0) { @@ -356,6 +375,122 @@ process_gcm_crypto_op(struct aesni_gcm_qp *qp, struct rte_crypto_op *op, return 0; } +static inline void +aesni_gcm_fill_error_code(struct rte_crypto_sym_vec *vec, int32_t errnum) +{ + uint32_t i; + + for (i = 0; i < vec->num; i++) + vec->status[i] = errnum; +} + + +static inline int32_t +aesni_gcm_sgl_op_finalize_encryption(struct aesni_gcm_session *s, + struct gcm_context_data *gdata_ctx, uint8_t *digest) +{ + if (s->req_digest_length != s->gen_digest_length) { + uint8_t tmpdigest[s->gen_digest_length]; + + s->ops.finalize(&s->gdata_key, gdata_ctx, tmpdigest, + s->gen_digest_length); + memcpy(digest, tmpdigest, s->req_digest_length); + } else { + s->ops.finalize(&s->gdata_key, gdata_ctx, digest, + s->gen_digest_length); + } + + return 0; +} + +static inline int32_t +aesni_gcm_sgl_op_finalize_decryption(struct aesni_gcm_session *s, + struct gcm_context_data *gdata_ctx, uint8_t *digest) +{ + uint8_t tmpdigest[s->gen_digest_length]; + + s->ops.finalize(&s->gdata_key, gdata_ctx, tmpdigest, + s->gen_digest_length); + + return memcmp(digest, tmpdigest, s->req_digest_length) == 0 ? 0 : + EBADMSG; +} + +static inline void +aesni_gcm_process_gcm_sgl_op(struct aesni_gcm_session *s, + struct gcm_context_data *gdata_ctx, struct rte_crypto_sgl *sgl, + void *iv, void *aad) +{ + uint32_t i; + + /* init crypto operation */ + s->ops.init(&s->gdata_key, gdata_ctx, iv, aad, + (uint64_t)s->aad_length); + + /* update with sgl data */ + for (i = 0; i < sgl->num; i++) { + struct rte_crypto_vec *vec = &sgl->vec[i]; + + s->ops.update(&s->gdata_key, gdata_ctx, vec->base, vec->base, + vec->len); + } +} + +/** Process CPU crypto bulk operations */ +uint32_t +aesni_gcm_pmd_cpu_crypto_process(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess, + __rte_unused union rte_crypto_sym_ofs ofs, + struct rte_crypto_sym_vec *vec) +{ + void *sess_priv; + struct aesni_gcm_session *s; + uint32_t processed; + uint32_t i; + + sess_priv = get_sym_session_private_data(sess, dev->driver_id); + if (unlikely(sess_priv == NULL)) { + aesni_gcm_fill_error_code(vec, EINVAL); + return 0; + } + + s = sess_priv; + if (unlikely(s->mode != AESNI_GCM_MODE_SYNC)) { + aesni_gcm_fill_error_code(vec, EINVAL); + return 0; + } + + processed = 0; + for (i = 0; i < vec->num; ++i) { + struct gcm_context_data gdata_ctx; + int32_t status; + + aesni_gcm_process_gcm_sgl_op(s, &gdata_ctx, &vec->sgl[i], + vec->iv[i], vec->aad[i]); + + switch (s->op) { + case AESNI_GCM_OP_AUTHENTICATED_ENCRYPTION: + status = aesni_gcm_sgl_op_finalize_encryption(s, + &gdata_ctx, vec->digest[i]); + break; + + case AESNI_GCM_OP_AUTHENTICATED_DECRYPTION: + status = aesni_gcm_sgl_op_finalize_decryption(s, + &gdata_ctx, vec->digest[i]); + break; + + default: + status = EINVAL; + } + + vec->status[i] = status; + if (status == 0) + processed++; + } + + return processed; +} + /** * Process a completed job and return rte_mbuf which job processed * @@ -527,7 +662,8 @@ aesni_gcm_create(const char *name, RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | RTE_CRYPTODEV_FF_IN_PLACE_SGL | RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | - RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT; + RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | + RTE_CRYPTODEV_FF_SYM_CPU_CRYPTO; /* Check CPU for support for AES instruction set */ if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AES)) @@ -672,7 +808,6 @@ RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_AESNI_GCM_PMD, RTE_PMD_REGISTER_CRYPTO_DRIVER(aesni_gcm_crypto_drv, aesni_gcm_pmd_drv.driver, cryptodev_driver_id); - RTE_INIT(aesni_gcm_init_log) { aesni_gcm_logtype_driver = rte_log_register("pmd.crypto.aesni_gcm"); diff --git a/drivers/crypto/aesni_gcm/aesni_gcm_pmd_ops.c b/drivers/crypto/aesni_gcm/aesni_gcm_pmd_ops.c index 2f66c7c58..5228d98b1 100644 --- a/drivers/crypto/aesni_gcm/aesni_gcm_pmd_ops.c +++ b/drivers/crypto/aesni_gcm/aesni_gcm_pmd_ops.c @@ -331,9 +331,12 @@ struct rte_cryptodev_ops aesni_gcm_pmd_ops = { .queue_pair_release = aesni_gcm_pmd_qp_release, .queue_pair_count = aesni_gcm_pmd_qp_count, + .sym_cpu_process = aesni_gcm_pmd_cpu_crypto_process, + .sym_session_get_size = aesni_gcm_pmd_sym_session_get_size, .sym_session_configure = aesni_gcm_pmd_sym_session_configure, .sym_session_clear = aesni_gcm_pmd_sym_session_clear }; struct rte_cryptodev_ops *rte_aesni_gcm_pmd_ops = &aesni_gcm_pmd_ops; + diff --git a/drivers/crypto/aesni_gcm/aesni_gcm_pmd_private.h b/drivers/crypto/aesni_gcm/aesni_gcm_pmd_private.h index 2039adb53..dc8d3c653 100644 --- a/drivers/crypto/aesni_gcm/aesni_gcm_pmd_private.h +++ b/drivers/crypto/aesni_gcm/aesni_gcm_pmd_private.h @@ -73,6 +73,11 @@ enum aesni_gcm_operation { AESNI_GMAC_OP_VERIFY }; +enum aesni_gcm_mode { + AESNI_GCM_MODE_ASYNC, + AESNI_GCM_MODE_SYNC +}; + /** AESNI GCM private session structure */ struct aesni_gcm_session { struct { @@ -90,8 +95,12 @@ struct aesni_gcm_session { /**< GCM operation type */ enum aesni_gcm_key key; /**< GCM key type */ + enum aesni_gcm_mode mode; + /**< Sync/async mode */ struct gcm_key_data gdata_key; /**< GCM parameters */ + struct aesni_gcm_session_ops ops; + /**< Session handlers */ }; @@ -109,10 +118,13 @@ aesni_gcm_set_session_parameters(const struct aesni_gcm_ops *ops, struct aesni_gcm_session *sess, const struct rte_crypto_sym_xform *xform); - -/** - * Device specific operations function pointer structure */ +/* Device specific operations function pointer structure */ extern struct rte_cryptodev_ops *rte_aesni_gcm_pmd_ops; +/** CPU crypto bulk process handler */ +uint32_t +aesni_gcm_pmd_cpu_crypto_process(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess, union rte_crypto_sym_ofs ofs, + struct rte_crypto_sym_vec *vec); #endif /* _AESNI_GCM_PMD_PRIVATE_H_ */ -- 2.17.1