[PATCH 2/3] crypto/ipsec_mb: add remaining SGL support

[Ciara Power <ciara.power@intel.com>]

The intel-ipsec-mb library supports SGL for GCM and ChaChaPoly
algorithms using the JOB API.
This support was added to AESNI_MB PMD previously, but the SGL feature
flags could not be added due to no SGL support for other algorithms.

This patch adds a workaround SGL approach for other algorithms
using the JOB API. The segmented input buffers are copied into a
linear buffer, which is passed as a single job to intel-ipsec-mb.
The job is processed, and on return, the linear buffer is split into the
original destination segments.

Existing AESNI_MB testcases are passing with these feature flags added.

Signed-off-by: Ciara Power <ciara.power@intel.com>
---
 drivers/crypto/ipsec_mb/pmd_aesni_mb.c | 190 ++++++++++++++++++++-----
 1 file changed, 157 insertions(+), 33 deletions(-)

diff --git a/drivers/crypto/ipsec_mb/pmd_aesni_mb.c b/drivers/crypto/ipsec_mb/pmd_aesni_mb.c
index 6d5d3ce8eb..53084f689a 100644
--- a/drivers/crypto/ipsec_mb/pmd_aesni_mb.c
+++ b/drivers/crypto/ipsec_mb/pmd_aesni_mb.c
@@ -937,7 +937,7 @@ static inline uint64_t
 auth_start_offset(struct rte_crypto_op *op, struct aesni_mb_session *session,
 		uint32_t oop, const uint32_t auth_offset,
 		const uint32_t cipher_offset, const uint32_t auth_length,
-		const uint32_t cipher_length)
+		const uint32_t cipher_length, uint8_t lb_sgl)
 {
 	struct rte_mbuf *m_src, *m_dst;
 	uint8_t *p_src, *p_dst;
@@ -945,7 +945,7 @@ auth_start_offset(struct rte_crypto_op *op, struct aesni_mb_session *session,
 	uint32_t cipher_end, auth_end;
 
 	/* Only cipher then hash needs special calculation. */
-	if (!oop || session->chain_order != IMB_ORDER_CIPHER_HASH)
+	if (!oop || session->chain_order != IMB_ORDER_CIPHER_HASH || lb_sgl)
 		return auth_offset;
 
 	m_src = op->sym->m_src;
@@ -1159,6 +1159,73 @@ handle_aead_sgl_job(IMB_JOB *job, IMB_MGR *mb_mgr,
 	return 0;
 }
 
+static int
+handle_sgl_linear(IMB_JOB *job, struct rte_crypto_op *op, uint32_t dst_offset,
+		struct aesni_mb_session *session)
+{
+	uint64_t cipher_len, auth_len;
+	uint8_t *src, *linear_buf = NULL;
+	int total_len;
+	int lb_offset = 0;
+	struct rte_mbuf *src_seg;
+	uint16_t src_len;
+
+	if (job->cipher_mode == IMB_CIPHER_SNOW3G_UEA2_BITLEN ||
+			job->cipher_mode == IMB_CIPHER_KASUMI_UEA1_BITLEN)
+		cipher_len = (job->msg_len_to_cipher_in_bits >> 3) +
+				(job->cipher_start_src_offset_in_bits >> 3);
+	else
+		cipher_len = job->msg_len_to_cipher_in_bytes +
+				job->cipher_start_src_offset_in_bytes;
+
+	if (job->hash_alg == IMB_AUTH_SNOW3G_UIA2_BITLEN ||
+			job->hash_alg == IMB_AUTH_ZUC_EIA3_BITLEN)
+		auth_len = (job->msg_len_to_hash_in_bits >> 3) +
+				job->hash_start_src_offset_in_bytes;
+	else if (job->hash_alg == IMB_AUTH_AES_GMAC)
+		auth_len = job->u.GCM.aad_len_in_bytes;
+	else
+		auth_len = job->msg_len_to_hash_in_bytes +
+				job->hash_start_src_offset_in_bytes;
+
+	total_len = RTE_MAX(auth_len, cipher_len) + job->auth_tag_output_len_in_bytes;
+	linear_buf = rte_zmalloc(NULL, total_len, 0);
+	if (linear_buf == NULL) {
+		IPSEC_MB_LOG(ERR, "Error allocating memory for SGL Linear Buffer\n");
+		return -1;
+	}
+
+	for (src_seg = op->sym->m_src; (src_seg != NULL) &&
+			(total_len - lb_offset > 0); src_seg = src_seg->next) {
+		src = rte_pktmbuf_mtod(src_seg, uint8_t *);
+		src_len =  RTE_MIN(src_seg->data_len, total_len - lb_offset);
+		rte_memcpy(linear_buf + lb_offset, src, src_len);
+		lb_offset += src_len;
+	}
+
+	job->src = linear_buf;
+	job->dst = linear_buf + dst_offset;
+	job->user_data2 = linear_buf;
+
+	if (job->hash_alg == IMB_AUTH_AES_GMAC)
+		job->u.GCM.aad = linear_buf;
+
+	if (session->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY)
+		job->auth_tag_output = linear_buf + lb_offset;
+	else
+		job->auth_tag_output = linear_buf + auth_len;
+
+	return 0;
+}
+
+static inline int
+imb_lib_support_sgl_algo(IMB_CIPHER_MODE alg)
+{
+	if (alg == IMB_CIPHER_CHACHA20_POLY1305
+			|| alg == IMB_CIPHER_GCM)
+		return 1;
+	return 0;
+}
 
 /**
  * Process a crypto operation and complete a IMB_JOB job structure for
@@ -1171,7 +1238,8 @@ handle_aead_sgl_job(IMB_JOB *job, IMB_MGR *mb_mgr,
  *
  * @return
  * - 0 on success, the IMB_JOB will be filled
- * - -1 if invalid session, IMB_JOB will not be filled
+ * - -1 if invalid session or errors allocationg SGL linear buffer,
+ *   IMB_JOB will not be filled
  */
 static inline int
 set_mb_job_params(IMB_JOB *job, struct ipsec_mb_qp *qp,
@@ -1191,6 +1259,7 @@ set_mb_job_params(IMB_JOB *job, struct ipsec_mb_qp *qp,
 	uint32_t total_len;
 	IMB_JOB base_job;
 	uint8_t sgl = 0;
+	uint8_t lb_sgl = 0;
 	int ret;
 
 	session = ipsec_mb_get_session_private(qp, op);
@@ -1199,18 +1268,6 @@ set_mb_job_params(IMB_JOB *job, struct ipsec_mb_qp *qp,
 		return -1;
 	}
 
-	if (op->sym->m_src->nb_segs > 1) {
-		if (session->cipher.mode != IMB_CIPHER_GCM
-				&& session->cipher.mode !=
-				IMB_CIPHER_CHACHA20_POLY1305) {
-			op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
-			IPSEC_MB_LOG(ERR, "Device only supports SGL for AES-GCM"
-					" or CHACHA20_POLY1305 algorithms.");
-			return -1;
-		}
-		sgl = 1;
-	}
-
 	/* Set crypto operation */
 	job->chain_order = session->chain_order;
 
@@ -1233,6 +1290,26 @@ set_mb_job_params(IMB_JOB *job, struct ipsec_mb_qp *qp,
 		job->dec_keys = session->cipher.expanded_aes_keys.decode;
 	}
 
+	if (!op->sym->m_dst) {
+		/* in-place operation */
+		m_dst = m_src;
+		oop = 0;
+	} else if (op->sym->m_dst == op->sym->m_src) {
+		/* in-place operation */
+		m_dst = m_src;
+		oop = 0;
+	} else {
+		/* out-of-place operation */
+		m_dst = op->sym->m_dst;
+		oop = 1;
+	}
+
+	if (m_src->nb_segs > 1 || m_dst->nb_segs > 1) {
+		sgl = 1;
+		if (!imb_lib_support_sgl_algo(session->cipher.mode))
+			lb_sgl = 1;
+	}
+
 	switch (job->hash_alg) {
 	case IMB_AUTH_AES_XCBC:
 		job->u.XCBC._k1_expanded = session->auth.xcbc.k1_expanded;
@@ -1331,20 +1408,6 @@ set_mb_job_params(IMB_JOB *job, struct ipsec_mb_qp *qp,
 		m_offset = 0;
 	}
 
-	if (!op->sym->m_dst) {
-		/* in-place operation */
-		m_dst = m_src;
-		oop = 0;
-	} else if (op->sym->m_dst == op->sym->m_src) {
-		/* in-place operation */
-		m_dst = m_src;
-		oop = 0;
-	} else {
-		/* out-of-place operation */
-		m_dst = op->sym->m_dst;
-		oop = 1;
-	}
-
 	/* Set digest output location */
 	if (job->hash_alg != IMB_AUTH_NULL &&
 			session->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
@@ -1435,7 +1498,7 @@ set_mb_job_params(IMB_JOB *job, struct ipsec_mb_qp *qp,
 		job->hash_start_src_offset_in_bytes = auth_start_offset(op,
 				session, oop, auth_off_in_bytes,
 				ciph_off_in_bytes, auth_len_in_bytes,
-				ciph_len_in_bytes);
+				ciph_len_in_bytes, lb_sgl);
 		job->msg_len_to_hash_in_bits = op->sym->auth.data.length;
 
 		job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
@@ -1452,7 +1515,7 @@ set_mb_job_params(IMB_JOB *job, struct ipsec_mb_qp *qp,
 		job->hash_start_src_offset_in_bytes = auth_start_offset(op,
 				session, oop, auth_off_in_bytes,
 				ciph_off_in_bytes, auth_len_in_bytes,
-				ciph_len_in_bytes);
+				ciph_len_in_bytes, lb_sgl);
 		job->msg_len_to_hash_in_bytes = auth_len_in_bytes;
 
 		job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
@@ -1464,7 +1527,7 @@ set_mb_job_params(IMB_JOB *job, struct ipsec_mb_qp *qp,
 				session, oop, op->sym->auth.data.offset,
 				op->sym->cipher.data.offset,
 				op->sym->auth.data.length,
-				op->sym->cipher.data.length);
+				op->sym->cipher.data.length, lb_sgl);
 		job->msg_len_to_hash_in_bytes = op->sym->auth.data.length;
 
 		job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
@@ -1525,6 +1588,10 @@ set_mb_job_params(IMB_JOB *job, struct ipsec_mb_qp *qp,
 	job->user_data = op;
 
 	if (sgl) {
+
+		if (lb_sgl)
+			return handle_sgl_linear(job, op, m_offset, session);
+
 		base_job = *job;
 		job->sgl_state = IMB_SGL_INIT;
 		job = IMB_SUBMIT_JOB(mb_mgr);
@@ -1695,6 +1762,49 @@ generate_digest(IMB_JOB *job, struct rte_crypto_op *op,
 			sess->auth.req_digest_len);
 }
 
+static void
+post_process_sgl_linear(struct rte_crypto_op *op, IMB_JOB *job,
+		struct aesni_mb_session *sess, uint8_t *linear_buf)
+{
+
+	int lb_offset = 0;
+	struct rte_mbuf *m_dst = op->sym->m_dst == NULL ?
+			op->sym->m_src : op->sym->m_dst;
+	uint16_t total_len, dst_len;
+	uint64_t cipher_len, auth_len;
+	uint8_t *dst;
+
+	if (job->cipher_mode == IMB_CIPHER_SNOW3G_UEA2_BITLEN ||
+			job->cipher_mode == IMB_CIPHER_KASUMI_UEA1_BITLEN)
+		cipher_len = (job->msg_len_to_cipher_in_bits >> 3) +
+				(job->cipher_start_src_offset_in_bits >> 3);
+	else
+		cipher_len = job->msg_len_to_cipher_in_bytes +
+				job->cipher_start_src_offset_in_bytes;
+
+	if (job->hash_alg == IMB_AUTH_SNOW3G_UIA2_BITLEN ||
+			job->hash_alg == IMB_AUTH_ZUC_EIA3_BITLEN)
+		auth_len = (job->msg_len_to_hash_in_bits >> 3) +
+				job->hash_start_src_offset_in_bytes;
+	else if (job->hash_alg == IMB_AUTH_AES_GMAC)
+		auth_len = job->u.GCM.aad_len_in_bytes;
+	else
+		auth_len = job->msg_len_to_hash_in_bytes +
+				job->hash_start_src_offset_in_bytes;
+
+	total_len = RTE_MAX(auth_len, cipher_len);
+
+	if (sess->auth.operation != RTE_CRYPTO_AUTH_OP_VERIFY)
+		total_len += job->auth_tag_output_len_in_bytes;
+
+	for (; (m_dst != NULL) && (total_len - lb_offset > 0); m_dst = m_dst->next) {
+		dst = rte_pktmbuf_mtod(m_dst, uint8_t *);
+		dst_len = RTE_MIN(m_dst->data_len, total_len - lb_offset);
+		rte_memcpy(dst, linear_buf + lb_offset, dst_len);
+		lb_offset += dst_len;
+	}
+}
+
 /**
  * Process a completed job and return rte_mbuf which job processed
  *
@@ -1712,6 +1822,7 @@ post_process_mb_job(struct ipsec_mb_qp *qp, IMB_JOB *job)
 	struct aesni_mb_session *sess = NULL;
 	uint32_t driver_id = ipsec_mb_get_driver_id(
 						IPSEC_MB_PMD_TYPE_AESNI_MB);
+	uint8_t *linear_buf = NULL;
 
 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
 	uint8_t is_docsis_sec = 0;
@@ -1740,6 +1851,14 @@ post_process_mb_job(struct ipsec_mb_qp *qp, IMB_JOB *job)
 		case IMB_STATUS_COMPLETED:
 			op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 
+			if ((op->sym->m_src->nb_segs > 1 ||
+					(op->sym->m_dst != NULL &&
+					op->sym->m_dst->nb_segs > 1)) &&
+					!imb_lib_support_sgl_algo(sess->cipher.mode)) {
+				linear_buf = (uint8_t *) job->user_data2;
+				post_process_sgl_linear(op, job, sess, linear_buf);
+			}
+
 			if (job->hash_alg == IMB_AUTH_NULL)
 				break;
 
@@ -1766,6 +1885,7 @@ post_process_mb_job(struct ipsec_mb_qp *qp, IMB_JOB *job)
 		default:
 			op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 		}
+		rte_free(linear_buf);
 	}
 
 	/* Free session if a session-less crypto op */
@@ -2252,7 +2372,11 @@ RTE_INIT(ipsec_mb_register_aesni_mb)
 			RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT |
 			RTE_CRYPTODEV_FF_SYM_CPU_CRYPTO |
 			RTE_CRYPTODEV_FF_NON_BYTE_ALIGNED_DATA |
-			RTE_CRYPTODEV_FF_SYM_SESSIONLESS;
+			RTE_CRYPTODEV_FF_SYM_SESSIONLESS |
+			RTE_CRYPTODEV_FF_IN_PLACE_SGL |
+			RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT |
+			RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT |
+			RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT;
 
 	aesni_mb_data->internals_priv_size = 0;
 	aesni_mb_data->ops = &aesni_mb_pmd_ops;
-- 
2.25.1