From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mga11.intel.com (mga11.intel.com [192.55.52.93]) by dpdk.org (Postfix) with ESMTP id 1D2201CFBD for ; Fri, 6 Apr 2018 20:53:03 +0200 (CEST) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga004.jf.intel.com ([10.7.209.38]) by fmsmga102.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 06 Apr 2018 11:53:03 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.48,416,1517904000"; d="scan'208";a="189275884" Received: from sivswdev01.ir.intel.com (HELO localhost.localdomain) ([10.237.217.45]) by orsmga004.jf.intel.com with ESMTP; 06 Apr 2018 11:53:01 -0700 From: Fiona Trahe To: dev@dpdk.org, pablo.de.lara.guarch@intel.com Cc: fiona.trahe@intel.com, tomaszx.jozwiak@intel.com Date: Fri, 6 Apr 2018 19:52:12 +0100 Message-Id: <1523040732-3290-31-git-send-email-fiona.trahe@intel.com> X-Mailer: git-send-email 1.7.0.7 In-Reply-To: <1523040732-3290-1-git-send-email-fiona.trahe@intel.com> References: <1523040732-3290-1-git-send-email-fiona.trahe@intel.com> Subject: [dpdk-dev] [PATCH 30/30] crypto/qat: add performance improvement into qat crypto dev. X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 06 Apr 2018 18:53:05 -0000 From: Tomasz Jozwiak - removed unused args. from process_response function - changed adf_modulo function - changed qat_sym_process_response into static inline version Signed-off-by: Tomasz Jozwiak Signed-off-by: Fiona Trahe --- drivers/crypto/qat/qat_qp.c | 29 +++++---- drivers/crypto/qat/qat_qp.h | 10 +-- drivers/crypto/qat/qat_sym.c | 129 -------------------------------------- drivers/crypto/qat/qat_sym.h | 131 +++++++++++++++++++++++++++++++++++++-- drivers/crypto/qat/qat_sym_pmd.c | 1 - 5 files changed, 143 insertions(+), 157 deletions(-) diff --git a/drivers/crypto/qat/qat_qp.c b/drivers/crypto/qat/qat_qp.c index 06ffac47b..44a24926b 100644 --- a/drivers/crypto/qat/qat_qp.c +++ b/drivers/crypto/qat/qat_qp.c @@ -14,6 +14,7 @@ #include "qat_logs.h" #include "qat_device.h" #include "qat_qp.h" +#include "qat_sym.h" #include "adf_transport_access_macros.h" @@ -256,7 +257,7 @@ int qat_qp_setup(struct qat_pci_device *qat_dev, qp->qat_dev_gen = qat_dev->qat_dev_gen; qp->build_request = qat_qp_conf->build_request; - qp->process_response = qat_qp_conf->process_response; + qp->service_type = qat_qp_conf->hw->service_type; qp->qat_dev = qat_dev; PMD_DRV_LOG(DEBUG, "QP setup complete: id: %d, cookiepool: %s", @@ -394,7 +395,7 @@ qat_queue_create(struct qat_pci_device *qat_dev, struct qat_queue *queue, queue->max_inflights = ADF_MAX_INFLIGHTS(queue->queue_size, ADF_BYTES_TO_MSG_SIZE(desc_size)); - queue->modulo = ADF_RING_SIZE_MODULO(queue->queue_size); + queue->modulo_mask = (1 << ADF_RING_SIZE_MODULO(queue->queue_size)) - 1; if (queue->max_inflights < 2) { PMD_DRV_LOG(ERR, "Invalid num inflights"); @@ -419,11 +420,11 @@ qat_queue_create(struct qat_pci_device *qat_dev, struct qat_queue *queue, queue->hw_queue_number, queue_base); PMD_DRV_LOG(DEBUG, "RING: Name:%s, size in CSR: %u, in bytes %u," - " nb msgs %u, msg_size %u, max_inflights %u modulo %u", + " nb msgs %u, msg_size %u, max_inflights %u modulo mask %u", queue->memz_name, queue->queue_size, queue_size_bytes, qp_conf->nb_descriptors, desc_size, - queue->max_inflights, queue->modulo); + queue->max_inflights, queue->modulo_mask); return 0; @@ -512,13 +513,9 @@ static void adf_configure_queues(struct qat_qp *qp) queue->hw_queue_number, queue_config); } - -static inline uint32_t adf_modulo(uint32_t data, uint32_t shift) +static inline uint32_t adf_modulo(uint32_t data, uint32_t modulo_mask) { - uint32_t div = data >> shift; - uint32_t mult = div << shift; - - return data - mult; + return data & modulo_mask; } static inline void @@ -602,7 +599,7 @@ qat_enqueue_op_burst(void *qp, void **ops, uint16_t nb_ops) goto kick_tail; } - tail = adf_modulo(tail + queue->msg_size, queue->modulo); + tail = adf_modulo(tail + queue->msg_size, queue->modulo_mask); ops++; nb_ops_sent++; } @@ -634,11 +631,13 @@ qat_dequeue_op_burst(void *qp, void **ops, uint16_t nb_ops) while (*(uint32_t *)resp_msg != ADF_RING_EMPTY_SIG && resp_counter != nb_ops) { - tmp_qp->process_response(ops, resp_msg, - tmp_qp->op_cookies[head / rx_queue->msg_size], - tmp_qp->qat_dev_gen); + if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC) + qat_sym_process_response(ops, resp_msg); + /* add qat_asym_process_response here */ + /* add qat_comp_process_response here */ - head = adf_modulo(head + rx_queue->msg_size, rx_queue->modulo); + head = adf_modulo(head + rx_queue->msg_size, + rx_queue->modulo_mask); resp_msg = (uint8_t *)rx_queue->base_addr + head; ops++; diff --git a/drivers/crypto/qat/qat_qp.h b/drivers/crypto/qat/qat_qp.h index 6f07bd67c..59db945e7 100644 --- a/drivers/crypto/qat/qat_qp.h +++ b/drivers/crypto/qat/qat_qp.h @@ -21,11 +21,6 @@ typedef int (*build_request_t)(void *op, enum qat_device_gen qat_dev_gen); /**< Build a request from an op. */ -typedef int (*process_response_t)(void **ops, - uint8_t *resp, void *op_cookie, - enum qat_device_gen qat_dev_gen); -/**< Process a response descriptor and return the associated op. */ - /** * Structure with data needed for creation of queue pair. */ @@ -46,7 +41,6 @@ struct qat_qp_config { uint32_t cookie_size; int socket_id; build_request_t build_request; - process_response_t process_response; const char *service_str; }; @@ -59,7 +53,7 @@ struct qat_queue { rte_iova_t base_phys_addr; /* Queue physical address */ uint32_t head; /* Shadow copy of the head */ uint32_t tail; /* Shadow copy of the tail */ - uint32_t modulo; + uint32_t modulo_mask; uint32_t msg_size; uint16_t max_inflights; uint32_t queue_size; @@ -85,7 +79,7 @@ struct qat_qp { uint32_t nb_descriptors; enum qat_device_gen qat_dev_gen; build_request_t build_request; - process_response_t process_response; + enum qat_service_type service_type; struct qat_pci_device *qat_dev; /**< qat device this qp is on */ } __rte_cache_aligned; diff --git a/drivers/crypto/qat/qat_sym.c b/drivers/crypto/qat/qat_sym.c index 15244d113..887d4ebcd 100644 --- a/drivers/crypto/qat/qat_sym.c +++ b/drivers/crypto/qat/qat_sym.c @@ -12,44 +12,7 @@ #include #include "qat_logs.h" -#include "qat_sym_session.h" #include "qat_sym.h" -#include "qat_sym_pmd.h" - -#define BYTE_LENGTH 8 -/* bpi is only used for partial blocks of DES and AES - * so AES block len can be assumed as max len for iv, src and dst - */ -#define BPI_MAX_ENCR_IV_LEN ICP_QAT_HW_AES_BLK_SZ - -/** Encrypt a single partial block - * Depends on openssl libcrypto - * Uses ECB+XOR to do CFB encryption, same result, more performant - */ -static inline int -bpi_cipher_encrypt(uint8_t *src, uint8_t *dst, - uint8_t *iv, int ivlen, int srclen, - void *bpi_ctx) -{ - EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)bpi_ctx; - int encrypted_ivlen; - uint8_t encrypted_iv[BPI_MAX_ENCR_IV_LEN]; - uint8_t *encr = encrypted_iv; - - /* ECB method: encrypt the IV, then XOR this with plaintext */ - if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, iv, ivlen) - <= 0) - goto cipher_encrypt_err; - - for (; srclen != 0; --srclen, ++dst, ++src, ++encr) - *dst = *src ^ *encr; - - return 0; - -cipher_encrypt_err: - PMD_DRV_LOG(ERR, "libcrypto ECB cipher encrypt failed"); - return -EINVAL; -} /** Decrypt a single partial block * Depends on openssl libcrypto @@ -136,62 +99,6 @@ qat_bpicipher_preprocess(struct qat_sym_session *ctx, return sym_op->cipher.data.length - last_block_len; } -static inline uint32_t -qat_bpicipher_postprocess(struct qat_sym_session *ctx, - struct rte_crypto_op *op) -{ - int block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg); - struct rte_crypto_sym_op *sym_op = op->sym; - uint8_t last_block_len = block_len > 0 ? - sym_op->cipher.data.length % block_len : 0; - - if (last_block_len > 0 && - ctx->qat_dir == ICP_QAT_HW_CIPHER_ENCRYPT) { - - /* Encrypt last block */ - uint8_t *last_block, *dst, *iv; - uint32_t last_block_offset; - - last_block_offset = sym_op->cipher.data.offset + - sym_op->cipher.data.length - last_block_len; - last_block = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_src, - uint8_t *, last_block_offset); - - if (unlikely(sym_op->m_dst != NULL)) - /* out-of-place operation (OOP) */ - dst = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_dst, - uint8_t *, last_block_offset); - else - dst = last_block; - - if (last_block_len < sym_op->cipher.data.length) - /* use previous block ciphertext as IV */ - iv = dst - block_len; - else - /* runt block, i.e. less than one full block */ - iv = rte_crypto_op_ctod_offset(op, uint8_t *, - ctx->cipher_iv.offset); - -#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX - rte_hexdump(stdout, "BPI: src before post-process:", last_block, - last_block_len); - if (sym_op->m_dst != NULL) - rte_hexdump(stdout, "BPI: dst before post-process:", - dst, last_block_len); -#endif - bpi_cipher_encrypt(last_block, dst, iv, block_len, - last_block_len, ctx->bpi_ctx); -#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX - rte_hexdump(stdout, "BPI: src after post-process:", last_block, - last_block_len); - if (sym_op->m_dst != NULL) - rte_hexdump(stdout, "BPI: dst after post-process:", dst, - last_block_len); -#endif - } - return sym_op->cipher.data.length - last_block_len; -} - static inline void set_cipher_iv(uint16_t iv_length, uint16_t iv_offset, struct icp_qat_fw_la_cipher_req_params *cipher_param, @@ -659,39 +566,3 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg, #endif return 0; } - -int -qat_sym_process_response(void **op, uint8_t *resp, - __rte_unused void *op_cookie, - __rte_unused enum qat_device_gen qat_dev_gen) -{ - - struct icp_qat_fw_comn_resp *resp_msg = - (struct icp_qat_fw_comn_resp *)resp; - struct rte_crypto_op *rx_op = (struct rte_crypto_op *)(uintptr_t) - (resp_msg->opaque_data); - -#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX - rte_hexdump(stdout, "qat_response:", (uint8_t *)resp_msg, - sizeof(struct icp_qat_fw_comn_resp)); -#endif - - if (ICP_QAT_FW_COMN_STATUS_FLAG_OK != - ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET( - resp_msg->comn_hdr.comn_status)) { - - rx_op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; - } else { - struct qat_sym_session *sess = (struct qat_sym_session *) - get_session_private_data( - rx_op->sym->session, - cryptodev_qat_driver_id); - - if (sess->bpi_ctx) - qat_bpicipher_postprocess(sess, rx_op); - rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; - } - *op = (void *)rx_op; - - return 0; -} diff --git a/drivers/crypto/qat/qat_sym.h b/drivers/crypto/qat/qat_sym.h index bdb672be4..2eaba931e 100644 --- a/drivers/crypto/qat/qat_sym.h +++ b/drivers/crypto/qat/qat_sym.h @@ -6,8 +6,17 @@ #define _QAT_SYM_H_ #include +#include #include "qat_common.h" +#include "qat_sym_session.h" +#include "qat_sym_pmd.h" + +#define BYTE_LENGTH 8 +/* bpi is only used for partial blocks of DES and AES + * so AES block len can be assumed as max len for iv, src and dst + */ +#define BPI_MAX_ENCR_IV_LEN ICP_QAT_HW_AES_BLK_SZ struct qat_sym_session; @@ -21,9 +30,123 @@ struct qat_sym_op_cookie { int qat_sym_build_request(void *in_op, uint8_t *out_msg, void *op_cookie, enum qat_device_gen qat_dev_gen); -int -qat_sym_process_response(void **op, uint8_t *resp, - __rte_unused void *op_cookie, - __rte_unused enum qat_device_gen qat_dev_gen); + +/** Encrypt a single partial block + * Depends on openssl libcrypto + * Uses ECB+XOR to do CFB encryption, same result, more performant + */ +static inline int +bpi_cipher_encrypt(uint8_t *src, uint8_t *dst, + uint8_t *iv, int ivlen, int srclen, + void *bpi_ctx) +{ + EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)bpi_ctx; + int encrypted_ivlen; + uint8_t encrypted_iv[BPI_MAX_ENCR_IV_LEN]; + uint8_t *encr = encrypted_iv; + + /* ECB method: encrypt the IV, then XOR this with plaintext */ + if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, iv, ivlen) + <= 0) + goto cipher_encrypt_err; + + for (; srclen != 0; --srclen, ++dst, ++src, ++encr) + *dst = *src ^ *encr; + + return 0; + +cipher_encrypt_err: + PMD_DRV_LOG(ERR, "libcrypto ECB cipher encrypt failed"); + return -EINVAL; +} + +static inline uint32_t +qat_bpicipher_postprocess(struct qat_sym_session *ctx, + struct rte_crypto_op *op) +{ + int block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg); + struct rte_crypto_sym_op *sym_op = op->sym; + uint8_t last_block_len = block_len > 0 ? + sym_op->cipher.data.length % block_len : 0; + + if (last_block_len > 0 && + ctx->qat_dir == ICP_QAT_HW_CIPHER_ENCRYPT) { + + /* Encrypt last block */ + uint8_t *last_block, *dst, *iv; + uint32_t last_block_offset; + + last_block_offset = sym_op->cipher.data.offset + + sym_op->cipher.data.length - last_block_len; + last_block = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_src, + uint8_t *, last_block_offset); + + if (unlikely(sym_op->m_dst != NULL)) + /* out-of-place operation (OOP) */ + dst = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_dst, + uint8_t *, last_block_offset); + else + dst = last_block; + + if (last_block_len < sym_op->cipher.data.length) + /* use previous block ciphertext as IV */ + iv = dst - block_len; + else + /* runt block, i.e. less than one full block */ + iv = rte_crypto_op_ctod_offset(op, uint8_t *, + ctx->cipher_iv.offset); + +#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX + rte_hexdump(stdout, "BPI: src before post-process:", last_block, + last_block_len); + if (sym_op->m_dst != NULL) + rte_hexdump(stdout, "BPI: dst before post-process:", + dst, last_block_len); +#endif + bpi_cipher_encrypt(last_block, dst, iv, block_len, + last_block_len, ctx->bpi_ctx); +#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX + rte_hexdump(stdout, "BPI: src after post-process:", last_block, + last_block_len); + if (sym_op->m_dst != NULL) + rte_hexdump(stdout, "BPI: dst after post-process:", dst, + last_block_len); +#endif + } + return sym_op->cipher.data.length - last_block_len; +} + +static inline void +qat_sym_process_response(void **op, uint8_t *resp) +{ + + struct icp_qat_fw_comn_resp *resp_msg = + (struct icp_qat_fw_comn_resp *)resp; + struct rte_crypto_op *rx_op = (struct rte_crypto_op *)(uintptr_t) + (resp_msg->opaque_data); + +#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX + rte_hexdump(stdout, "qat_response:", (uint8_t *)resp_msg, + sizeof(struct icp_qat_fw_comn_resp)); +#endif + + if (ICP_QAT_FW_COMN_STATUS_FLAG_OK != + ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET( + resp_msg->comn_hdr.comn_status)) { + + rx_op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; + } else { + struct qat_sym_session *sess = (struct qat_sym_session *) + get_session_private_data( + rx_op->sym->session, + cryptodev_qat_driver_id); + + + if (sess->bpi_ctx) + qat_bpicipher_postprocess(sess, rx_op); + rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; + } + *op = (void *)rx_op; +} #endif /* _QAT_SYM_H_ */ diff --git a/drivers/crypto/qat/qat_sym_pmd.c b/drivers/crypto/qat/qat_sym_pmd.c index 28e579b77..6b39b32f8 100644 --- a/drivers/crypto/qat/qat_sym_pmd.c +++ b/drivers/crypto/qat/qat_sym_pmd.c @@ -160,7 +160,6 @@ static int qat_sym_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, qat_qp_conf.hw = qp_hw_data; qat_qp_conf.build_request = qat_sym_build_request; - qat_qp_conf.process_response = qat_sym_process_response; qat_qp_conf.cookie_size = sizeof(struct qat_sym_op_cookie); qat_qp_conf.nb_descriptors = qp_conf->nb_descriptors; qat_qp_conf.socket_id = socket_id; -- 2.13.6