[dpdk-dev] [PATCH v3 35/38] crypto/qat: make response process function inline

[Tomasz Jozwiak <tomaszx.jozwiak@intel.com>]

From: Fiona Trahe <fiona.trahe@intel.com>

Optimize the dequeue function by inlining the response
processing function, assuming only symmetric
operations are supported.

Signed-off-by: Tomasz Jozwiak <tomaszx.jozwiak@intel.com>
Signed-off-by: Fiona Trahe <fiona.trahe@intel.com>
---
 drivers/crypto/qat/qat_qp.c      |   4 +-
 drivers/crypto/qat/qat_qp.h      |   5 --
 drivers/crypto/qat/qat_sym.c     | 127 ------------------------------
 drivers/crypto/qat/qat_sym.h     | 131 ++++++++++++++++++++++++++++++-
 drivers/crypto/qat/qat_sym_pmd.c |   1 -
 5 files changed, 131 insertions(+), 137 deletions(-)

diff --git a/drivers/crypto/qat/qat_qp.c b/drivers/crypto/qat/qat_qp.c
index 0fdec0da0..b190f2cee 100644
--- a/drivers/crypto/qat/qat_qp.c
+++ b/drivers/crypto/qat/qat_qp.c
@@ -14,6 +14,7 @@
 #include "qat_logs.h"
 #include "qat_device.h"
 #include "qat_qp.h"
+#include "qat_sym.h"
 #include "adf_transport_access_macros.h"
 
 
@@ -238,7 +239,6 @@ int qat_qp_setup(struct qat_pci_device *qat_dev,
 
 	qp->qat_dev_gen = qat_dev->qat_dev_gen;
 	qp->build_request = qat_qp_conf->build_request;
-	qp->process_response = qat_qp_conf->process_response;
 	qp->qat_dev = qat_dev;
 
 	PMD_DRV_LOG(DEBUG, "QP setup complete: id: %d, cookiepool: %s",
@@ -612,7 +612,7 @@ qat_dequeue_op_burst(void *qp, void **ops, uint16_t nb_ops)
 	while (*(uint32_t *)resp_msg != ADF_RING_EMPTY_SIG &&
 			resp_counter != nb_ops) {
 
-		tmp_qp->process_response(ops, resp_msg);
+		qat_sym_process_response(ops, resp_msg);
 
 		head = adf_modulo(head + rx_queue->msg_size,
 				  rx_queue->modulo_mask);
diff --git a/drivers/crypto/qat/qat_qp.h b/drivers/crypto/qat/qat_qp.h
index eb9188410..0b3d6d3aa 100644
--- a/drivers/crypto/qat/qat_qp.h
+++ b/drivers/crypto/qat/qat_qp.h
@@ -21,9 +21,6 @@ typedef int (*build_request_t)(void *op,
 		enum qat_device_gen qat_dev_gen);
 /**< Build a request from an op. */
 
-typedef int (*process_response_t)(void **ops, uint8_t *resp);
-/**< Process a response descriptor and return the associated op. */
-
 /**
  * Structure with data needed for creation of queue pair.
  */
@@ -44,7 +41,6 @@ struct qat_qp_config {
 	uint32_t cookie_size;
 	int socket_id;
 	build_request_t build_request;
-	process_response_t process_response;
 	const char *service_str;
 };
 
@@ -83,7 +79,6 @@ struct qat_qp {
 	uint32_t nb_descriptors;
 	enum qat_device_gen qat_dev_gen;
 	build_request_t build_request;
-	process_response_t process_response;
 	struct qat_pci_device *qat_dev;
 	/**< qat device this qp is on */
 } __rte_cache_aligned;
diff --git a/drivers/crypto/qat/qat_sym.c b/drivers/crypto/qat/qat_sym.c
index f613adce6..887d4ebcd 100644
--- a/drivers/crypto/qat/qat_sym.c
+++ b/drivers/crypto/qat/qat_sym.c
@@ -12,44 +12,7 @@
 #include <rte_byteorder.h>
 
 #include "qat_logs.h"
-#include "qat_sym_session.h"
 #include "qat_sym.h"
-#include "qat_sym_pmd.h"
-
-#define BYTE_LENGTH    8
-/* bpi is only used for partial blocks of DES and AES
- * so AES block len can be assumed as max len for iv, src and dst
- */
-#define BPI_MAX_ENCR_IV_LEN ICP_QAT_HW_AES_BLK_SZ
-
-/** Encrypt a single partial block
- *  Depends on openssl libcrypto
- *  Uses ECB+XOR to do CFB encryption, same result, more performant
- */
-static inline int
-bpi_cipher_encrypt(uint8_t *src, uint8_t *dst,
-		uint8_t *iv, int ivlen, int srclen,
-		void *bpi_ctx)
-{
-	EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)bpi_ctx;
-	int encrypted_ivlen;
-	uint8_t encrypted_iv[BPI_MAX_ENCR_IV_LEN];
-	uint8_t *encr = encrypted_iv;
-
-	/* ECB method: encrypt the IV, then XOR this with plaintext */
-	if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, iv, ivlen)
-								<= 0)
-		goto cipher_encrypt_err;
-
-	for (; srclen != 0; --srclen, ++dst, ++src, ++encr)
-		*dst = *src ^ *encr;
-
-	return 0;
-
-cipher_encrypt_err:
-	PMD_DRV_LOG(ERR, "libcrypto ECB cipher encrypt failed");
-	return -EINVAL;
-}
 
 /** Decrypt a single partial block
  *  Depends on openssl libcrypto
@@ -136,62 +99,6 @@ qat_bpicipher_preprocess(struct qat_sym_session *ctx,
 	return sym_op->cipher.data.length - last_block_len;
 }
 
-static inline uint32_t
-qat_bpicipher_postprocess(struct qat_sym_session *ctx,
-				struct rte_crypto_op *op)
-{
-	int block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg);
-	struct rte_crypto_sym_op *sym_op = op->sym;
-	uint8_t last_block_len = block_len > 0 ?
-			sym_op->cipher.data.length % block_len : 0;
-
-	if (last_block_len > 0 &&
-			ctx->qat_dir == ICP_QAT_HW_CIPHER_ENCRYPT) {
-
-		/* Encrypt last block */
-		uint8_t *last_block, *dst, *iv;
-		uint32_t last_block_offset;
-
-		last_block_offset = sym_op->cipher.data.offset +
-				sym_op->cipher.data.length - last_block_len;
-		last_block = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_src,
-				uint8_t *, last_block_offset);
-
-		if (unlikely(sym_op->m_dst != NULL))
-			/* out-of-place operation (OOP) */
-			dst = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_dst,
-						uint8_t *, last_block_offset);
-		else
-			dst = last_block;
-
-		if (last_block_len < sym_op->cipher.data.length)
-			/* use previous block ciphertext as IV */
-			iv = dst - block_len;
-		else
-			/* runt block, i.e. less than one full block */
-			iv = rte_crypto_op_ctod_offset(op, uint8_t *,
-					ctx->cipher_iv.offset);
-
-#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX
-		rte_hexdump(stdout, "BPI: src before post-process:", last_block,
-			last_block_len);
-		if (sym_op->m_dst != NULL)
-			rte_hexdump(stdout, "BPI: dst before post-process:",
-					dst, last_block_len);
-#endif
-		bpi_cipher_encrypt(last_block, dst, iv, block_len,
-				last_block_len, ctx->bpi_ctx);
-#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX
-		rte_hexdump(stdout, "BPI: src after post-process:", last_block,
-			last_block_len);
-		if (sym_op->m_dst != NULL)
-			rte_hexdump(stdout, "BPI: dst after post-process:", dst,
-				last_block_len);
-#endif
-	}
-	return sym_op->cipher.data.length - last_block_len;
-}
-
 static inline void
 set_cipher_iv(uint16_t iv_length, uint16_t iv_offset,
 		struct icp_qat_fw_la_cipher_req_params *cipher_param,
@@ -659,37 +566,3 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg,
 #endif
 	return 0;
 }
-
-int
-qat_sym_process_response(void **op, uint8_t *resp)
-{
-
-	struct icp_qat_fw_comn_resp *resp_msg =
-			(struct icp_qat_fw_comn_resp *)resp;
-	struct rte_crypto_op *rx_op = (struct rte_crypto_op *)(uintptr_t)
-			(resp_msg->opaque_data);
-
-#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX
-	rte_hexdump(stdout, "qat_response:", (uint8_t *)resp_msg,
-			sizeof(struct icp_qat_fw_comn_resp));
-#endif
-
-	if (ICP_QAT_FW_COMN_STATUS_FLAG_OK !=
-			ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(
-			resp_msg->comn_hdr.comn_status)) {
-
-		rx_op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
-	} else {
-		struct qat_sym_session *sess = (struct qat_sym_session *)
-						get_session_private_data(
-						rx_op->sym->session,
-						cryptodev_qat_driver_id);
-
-		if (sess->bpi_ctx)
-			qat_bpicipher_postprocess(sess, rx_op);
-		rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
-	}
-	*op = (void *)rx_op;
-
-	return 0;
-}
diff --git a/drivers/crypto/qat/qat_sym.h b/drivers/crypto/qat/qat_sym.h
index dffd5f369..ddd0f473f 100644
--- a/drivers/crypto/qat/qat_sym.h
+++ b/drivers/crypto/qat/qat_sym.h
@@ -6,8 +6,19 @@
 #define _QAT_SYM_H_
 
 #include <rte_cryptodev_pmd.h>
+#include <rte_hexdump.h>
+
+#include <openssl/evp.h>
 
 #include "qat_common.h"
+#include "qat_sym_session.h"
+#include "qat_sym_pmd.h"
+
+#define BYTE_LENGTH    8
+/* bpi is only used for partial blocks of DES and AES
+ * so AES block len can be assumed as max len for iv, src and dst
+ */
+#define BPI_MAX_ENCR_IV_LEN ICP_QAT_HW_AES_BLK_SZ
 
 struct qat_sym_session;
 
@@ -21,7 +32,123 @@ struct qat_sym_op_cookie {
 int
 qat_sym_build_request(void *in_op, uint8_t *out_msg,
 		void *op_cookie, enum qat_device_gen qat_dev_gen);
-int
-qat_sym_process_response(void **op, uint8_t *resp);
 
+
+/** Encrypt a single partial block
+ *  Depends on openssl libcrypto
+ *  Uses ECB+XOR to do CFB encryption, same result, more performant
+ */
+static inline int
+bpi_cipher_encrypt(uint8_t *src, uint8_t *dst,
+		uint8_t *iv, int ivlen, int srclen,
+		void *bpi_ctx)
+{
+	EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)bpi_ctx;
+	int encrypted_ivlen;
+	uint8_t encrypted_iv[BPI_MAX_ENCR_IV_LEN];
+	uint8_t *encr = encrypted_iv;
+
+	/* ECB method: encrypt the IV, then XOR this with plaintext */
+	if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, iv, ivlen)
+								<= 0)
+		goto cipher_encrypt_err;
+
+	for (; srclen != 0; --srclen, ++dst, ++src, ++encr)
+		*dst = *src ^ *encr;
+
+	return 0;
+
+cipher_encrypt_err:
+	PMD_DRV_LOG(ERR, "libcrypto ECB cipher encrypt failed");
+	return -EINVAL;
+}
+
+static inline uint32_t
+qat_bpicipher_postprocess(struct qat_sym_session *ctx,
+				struct rte_crypto_op *op)
+{
+	int block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg);
+	struct rte_crypto_sym_op *sym_op = op->sym;
+	uint8_t last_block_len = block_len > 0 ?
+			sym_op->cipher.data.length % block_len : 0;
+
+	if (last_block_len > 0 &&
+			ctx->qat_dir == ICP_QAT_HW_CIPHER_ENCRYPT) {
+
+		/* Encrypt last block */
+		uint8_t *last_block, *dst, *iv;
+		uint32_t last_block_offset;
+
+		last_block_offset = sym_op->cipher.data.offset +
+				sym_op->cipher.data.length - last_block_len;
+		last_block = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_src,
+				uint8_t *, last_block_offset);
+
+		if (unlikely(sym_op->m_dst != NULL))
+			/* out-of-place operation (OOP) */
+			dst = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_dst,
+						uint8_t *, last_block_offset);
+		else
+			dst = last_block;
+
+		if (last_block_len < sym_op->cipher.data.length)
+			/* use previous block ciphertext as IV */
+			iv = dst - block_len;
+		else
+			/* runt block, i.e. less than one full block */
+			iv = rte_crypto_op_ctod_offset(op, uint8_t *,
+					ctx->cipher_iv.offset);
+
+#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX
+		rte_hexdump(stdout, "BPI: src before post-process:", last_block,
+			last_block_len);
+		if (sym_op->m_dst != NULL)
+			rte_hexdump(stdout, "BPI: dst before post-process:",
+					dst, last_block_len);
+#endif
+		bpi_cipher_encrypt(last_block, dst, iv, block_len,
+				last_block_len, ctx->bpi_ctx);
+#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX
+		rte_hexdump(stdout, "BPI: src after post-process:", last_block,
+			last_block_len);
+		if (sym_op->m_dst != NULL)
+			rte_hexdump(stdout, "BPI: dst after post-process:", dst,
+				last_block_len);
+#endif
+	}
+	return sym_op->cipher.data.length - last_block_len;
+}
+
+static inline void
+qat_sym_process_response(void **op, uint8_t *resp)
+{
+
+	struct icp_qat_fw_comn_resp *resp_msg =
+			(struct icp_qat_fw_comn_resp *)resp;
+	struct rte_crypto_op *rx_op = (struct rte_crypto_op *)(uintptr_t)
+			(resp_msg->opaque_data);
+
+#ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX
+	rte_hexdump(stdout, "qat_response:", (uint8_t *)resp_msg,
+			sizeof(struct icp_qat_fw_comn_resp));
+#endif
+
+	if (ICP_QAT_FW_COMN_STATUS_FLAG_OK !=
+			ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(
+			resp_msg->comn_hdr.comn_status)) {
+
+		rx_op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
+	} else {
+		struct qat_sym_session *sess = (struct qat_sym_session *)
+						get_session_private_data(
+						rx_op->sym->session,
+						cryptodev_qat_driver_id);
+
+
+		if (sess->bpi_ctx)
+			qat_bpicipher_postprocess(sess, rx_op);
+		rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+	}
+	*op = (void *)rx_op;
+}
 #endif /* _QAT_SYM_H_ */
diff --git a/drivers/crypto/qat/qat_sym_pmd.c b/drivers/crypto/qat/qat_sym_pmd.c
index 28e579b77..6b39b32f8 100644
--- a/drivers/crypto/qat/qat_sym_pmd.c
+++ b/drivers/crypto/qat/qat_sym_pmd.c
@@ -160,7 +160,6 @@ static int qat_sym_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 
 	qat_qp_conf.hw = qp_hw_data;
 	qat_qp_conf.build_request = qat_sym_build_request;
-	qat_qp_conf.process_response = qat_sym_process_response;
 	qat_qp_conf.cookie_size = sizeof(struct qat_sym_op_cookie);
 	qat_qp_conf.nb_descriptors = qp_conf->nb_descriptors;
 	qat_qp_conf.socket_id = socket_id;
-- 
2.17.0