[dpdk-dev] [PATCH 1/2] compress/qat: overflow catching

[dpdk-dev] [PATCH 1/2] compress/qat: overflow catching

[Artur Trybula]

Out-of-space recoverable feature added to QuickAssist compressdev driver

Signed-off-by: Artur Trybula <arturx.trybula@intel.com>
---
 drivers/compress/qat/qat_comp.c | 25 ++++++++++++++++++++++---
 1 file changed, 22 insertions(+), 3 deletions(-)

diff --git a/drivers/compress/qat/qat_comp.c b/drivers/compress/qat/qat_comp.c
index 835aaa838..aa43df6b1 100644
--- a/drivers/compress/qat/qat_comp.c
+++ b/drivers/compress/qat/qat_comp.c
@@ -259,10 +259,29 @@ qat_comp_process_response(void **op, uint8_t *resp, void *op_cookie,
 		    (!cmp_err_code && xlat_err_code == ERR_CODE_OVERFLOW_ERROR)
 				||
 		    (cmp_err_code == ERR_CODE_OVERFLOW_ERROR &&
-		     xlat_err_code == ERR_CODE_OVERFLOW_ERROR))
-			rx_op->status =
+		     xlat_err_code == ERR_CODE_OVERFLOW_ERROR)) {
+
+			struct icp_qat_fw_resp_comp_pars *comp_resp =
+	  (struct icp_qat_fw_resp_comp_pars *)&resp_msg->comp_resp_pars;
+
+			if (comp_resp->input_byte_counter) {
+				if ((qat_xform->qat_comp_request_type
+				== QAT_COMP_REQUEST_FIXED_COMP_STATELESS) ||
+				    (qat_xform->qat_comp_request_type
+				== QAT_COMP_REQUEST_DYNAMIC_COMP_STATELESS)) {
+					rx_op->status =
+				RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE;
+					rx_op->consumed =
+						comp_resp->input_byte_counter;
+					rx_op->produced =
+						comp_resp->output_byte_counter;
+				} else
+					rx_op->status =
 				RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
-		else
+			} else
+				rx_op->status =
+				RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
+		} else
 			rx_op->status = RTE_COMP_OP_STATUS_ERROR;
 
 		++(*dequeue_err_count);
-- 
2.17.1

[dpdk-dev] [PATCH 2/2] test/compress: unit test for stateless overflow recovery

[Artur Trybula]

Added unit test to check out-of-space recoverable feature.

Signed-off-by: Artur Trybula <arturx.trybula@intel.com>
---
 app/test/test_compressdev.c | 264 +++++++++++++++++++++++++++++-------
 1 file changed, 216 insertions(+), 48 deletions(-)

diff --git a/app/test/test_compressdev.c b/app/test/test_compressdev.c
index 992eac8e0..3a655164f 100644
--- a/app/test/test_compressdev.c
+++ b/app/test/test_compressdev.c
@@ -30,6 +30,7 @@
  * due to the compress block headers
  */
 #define COMPRESS_BUF_SIZE_RATIO 1.3
+#define COMPRESS_BUF_SIZE_RATIO_OVERFLOW 0.2
 #define NUM_LARGE_MBUFS 16
 #define SMALL_SEG_SIZE 256
 #define MAX_SEGS 16
@@ -65,6 +66,12 @@ enum zlib_direction {
 	ZLIB_ALL
 };
 
+enum overflow_test {
+	OVERFLOW_DISABLED,
+	OVERFLOW_ENABLED
+};
+
+
 enum varied_buff {
 	LB_BOTH = 0,	/* both input and output are linear*/
 	SGL_BOTH,	/* both input and output are chained */
@@ -100,6 +107,7 @@ struct test_data_params {
 	enum zlib_direction zlib_dir;
 	unsigned int out_of_space;
 	unsigned int big_data;
+	enum overflow_test overflow;
 };
 
 static struct comp_testsuite_params testsuite_params = { 0 };
@@ -644,6 +652,7 @@ prepare_sgl_bufs(const char *test_buf, struct rte_mbuf *head_buf,
 		data_size = remaining_data;
 	else
 		data_size = seg_size;
+
 	buf_ptr = rte_pktmbuf_append(head_buf, data_size);
 	if (buf_ptr == NULL) {
 		RTE_LOG(ERR, USER1,
@@ -710,6 +719,50 @@ prepare_sgl_bufs(const char *test_buf, struct rte_mbuf *head_buf,
 	return 0;
 }
 
+
+static int
+test_run_enqueue_dequeue(struct rte_comp_op **ops, unsigned int num_bufs,
+		  struct rte_comp_op **ops_processed)
+{
+	uint16_t num_enqd, num_deqd, num_total_deqd;
+	unsigned int deqd_retries = 0;
+
+	/* Enqueue and dequeue all operations */
+	num_enqd = rte_compressdev_enqueue_burst(0, 0, ops, num_bufs);
+	if (num_enqd < num_bufs) {
+		RTE_LOG(ERR, USER1,
+			"Some operations could not be enqueued\n");
+		return -1;
+	}
+
+	num_total_deqd = 0;
+	do {
+		/*
+		 * If retrying a dequeue call, wait for 10 ms to allow
+		 * enough time to the driver to process the operations
+		 */
+		if (deqd_retries != 0) {
+			/*
+			 * Avoid infinite loop if not all the
+			 * operations get out of the device
+			 */
+			if (deqd_retries == MAX_DEQD_RETRIES) {
+				RTE_LOG(ERR, USER1,
+					"Not all operations could be dequeued\n");
+				return -1;
+			}
+			usleep(DEQUEUE_WAIT_TIME);
+		}
+		num_deqd = rte_compressdev_dequeue_burst(0, 0,
+				&ops_processed[num_total_deqd], num_bufs);
+		num_total_deqd += num_deqd;
+		deqd_retries++;
+
+	} while (num_total_deqd < num_enqd);
+
+	return 0;
+}
+
 /*
  * Compresses and decompresses buffer with compressdev API and Zlib API
  */
@@ -729,6 +782,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 	unsigned int out_of_space = test_data->out_of_space;
 	unsigned int big_data = test_data->big_data;
 	enum zlib_direction zlib_dir = test_data->zlib_dir;
+	enum overflow_test overflow_tst = test_data->overflow;
 	int ret_status = -1;
 	int ret;
 	struct rte_mbuf *uncomp_bufs[num_bufs];
@@ -754,6 +808,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 		rte_compressdev_capability_get(0, RTE_COMP_ALGO_DEFLATE);
 	char *contig_buf = NULL;
 	uint64_t compress_checksum[num_bufs];
+	uint32_t compressed_data_size[num_bufs];
 
 	if (capa == NULL) {
 		RTE_LOG(ERR, USER1,
@@ -767,6 +822,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 	memset(ops, 0, sizeof(struct rte_comp_op *) * num_bufs);
 	memset(ops_processed, 0, sizeof(struct rte_comp_op *) * num_bufs);
 	memset(priv_xforms, 0, sizeof(void *) * num_bufs);
+	memset(compressed_data_size, 0, sizeof(uint32_t) * num_bufs);
 
 	if (big_data)
 		buf_pool = ts_params->big_mbuf_pool;
@@ -799,7 +855,13 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 	} else {
 		for (i = 0; i < num_bufs; i++) {
 			data_size = strlen(test_bufs[i]) + 1;
+
 			buf_ptr = rte_pktmbuf_append(uncomp_bufs[i], data_size);
+			if (buf_ptr == NULL) {
+				RTE_LOG(ERR, USER1,
+					"Append extra bytes to the source mbuf failed\n");
+				goto exit;
+			}
 			strlcpy(buf_ptr, test_bufs[i], data_size);
 		}
 	}
@@ -835,11 +897,22 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 		for (i = 0; i < num_bufs; i++) {
 			if (out_of_space == 1 && oos_zlib_decompress)
 				data_size = OUT_OF_SPACE_BUF;
-			else
-				(data_size = strlen(test_bufs[i]) *
-					COMPRESS_BUF_SIZE_RATIO);
-
-			rte_pktmbuf_append(comp_bufs[i], data_size);
+			else {
+				float ratio =
+				((test_data->zlib_dir == ZLIB_DECOMPRESS ||
+				   test_data->zlib_dir == ZLIB_NONE) &&
+				  overflow_tst == OVERFLOW_ENABLED) ?
+					 COMPRESS_BUF_SIZE_RATIO_OVERFLOW :
+					 COMPRESS_BUF_SIZE_RATIO;
+
+				data_size = strlen(test_bufs[i]) * ratio;
+			}
+			buf_ptr = rte_pktmbuf_append(comp_bufs[i], data_size);
+			if (buf_ptr == NULL) {
+				RTE_LOG(ERR, USER1,
+					"Append extra bytes to the destination mbuf failed\n");
+				goto exit;
+			}
 		}
 	}
 
@@ -852,7 +925,6 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 		goto exit;
 	}
 
-
 	for (i = 0; i < num_bufs; i++) {
 		ops[i]->m_src = uncomp_bufs[i];
 		ops[i]->m_dst = comp_bufs[i];
@@ -928,41 +1000,41 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 			for (i = 0; i < num_bufs; i++)
 				ops[i]->private_xform = priv_xforms[i];
 		}
-
-		/* Enqueue and dequeue all operations */
-		num_enqd = rte_compressdev_enqueue_burst(0, 0, ops, num_bufs);
-		if (num_enqd < num_bufs) {
+recovery_lb:
+		ret = test_run_enqueue_dequeue(ops, num_bufs, ops_processed);
+		if (ret < 0) {
 			RTE_LOG(ERR, USER1,
-				"The operations could not be enqueued\n");
+				"Enqueue/dequeue operation failed\n");
 			goto exit;
 		}
 
-		num_total_deqd = 0;
-		do {
-			/*
-			 * If retrying a dequeue call, wait for 10 ms to allow
-			 * enough time to the driver to process the operations
-			 */
-			if (deqd_retries != 0) {
-				/*
-				 * Avoid infinite loop if not all the
-				 * operations get out of the device
-				 */
-				if (deqd_retries == MAX_DEQD_RETRIES) {
+		for (i = 0; i < num_bufs; i++) {
+			compressed_data_size[i] += ops_processed[i]->produced;
+
+			if (ops_processed[i]->status ==
+				RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE) {
+
+				ops[i]->status =
+					RTE_COMP_OP_STATUS_NOT_PROCESSED;
+				ops[i]->src.offset +=
+					ops_processed[i]->consumed;
+				ops[i]->src.length -=
+					ops_processed[i]->consumed;
+				ops[i]->dst.offset +=
+					ops_processed[i]->produced;
+
+				buf_ptr = rte_pktmbuf_append(
+					ops[i]->m_dst,
+					ops_processed[i]->produced);
+
+				if (buf_ptr == NULL) {
 					RTE_LOG(ERR, USER1,
-						"Not all operations could be "
-						"dequeued\n");
+						"Data recovery: append extra bytes to the current mbuf failed\n");
 					goto exit;
 				}
-				usleep(DEQUEUE_WAIT_TIME);
+				goto recovery_lb;
 			}
-			num_deqd = rte_compressdev_dequeue_burst(0, 0,
-					&ops_processed[num_total_deqd], num_bufs);
-			num_total_deqd += num_deqd;
-			deqd_retries++;
-
-		} while (num_total_deqd < num_enqd);
-
+		}
 		deqd_retries = 0;
 
 		/* Free compress private xforms */
@@ -1006,7 +1078,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 	for (i = 0; i < num_bufs; i++) {
 		if (out_of_space && oos_zlib_decompress) {
 			if (ops_processed[i]->status !=
-					RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED) {
+				RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED) {
 				ret_status = -1;
 
 				RTE_LOG(ERR, USER1,
@@ -1071,7 +1143,13 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 				data_size =
 				strlen(test_bufs[priv_data->orig_idx]) + 1;
 
-			rte_pktmbuf_append(uncomp_bufs[i], data_size);
+			buf_ptr = rte_pktmbuf_append(uncomp_bufs[i], data_size);
+			if (buf_ptr == NULL) {
+				RTE_LOG(ERR, USER1,
+					"Append extra bytes to the decompressed mbuf failed\n");
+				goto exit;
+			}
+
 		}
 	}
 
@@ -1093,7 +1171,12 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 		 * Set the length of the compressed data to the
 		 * number of bytes that were produced in the previous stage
 		 */
-		ops[i]->src.length = ops_processed[i]->produced;
+
+		if (compressed_data_size[i])
+			ops[i]->src.length = compressed_data_size[i];
+		else
+			ops[i]->src.length = ops_processed[i]->produced;
+
 		ops[i]->dst.offset = 0;
 		if (state == RTE_COMP_OP_STATELESS) {
 			ops[i]->flush_flag = RTE_COMP_FLUSH_FINAL;
@@ -1147,7 +1230,8 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 			num_priv_xforms++;
 		}
 
-		if (capa->comp_feature_flags & RTE_COMP_FF_SHAREABLE_PRIV_XFORM) {
+		if (capa->comp_feature_flags &
+				RTE_COMP_FF_SHAREABLE_PRIV_XFORM) {
 			/* Attach shareable private xform data to ops */
 			for (i = 0; i < num_bufs; i++) {
 				priv_data = (struct priv_op_data *)(ops[i] + 1);
@@ -1206,9 +1290,10 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 				usleep(DEQUEUE_WAIT_TIME);
 			}
 			num_deqd = rte_compressdev_dequeue_burst(0, 0,
-					&ops_processed[num_total_deqd], num_bufs);
+				&ops_processed[num_total_deqd], num_bufs);
 			num_total_deqd += num_deqd;
 			deqd_retries++;
+
 		} while (num_total_deqd < num_enqd);
 
 		deqd_retries = 0;
@@ -1233,7 +1318,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 	for (i = 0; i < num_bufs; i++) {
 		if (out_of_space && oos_zlib_compress) {
 			if (ops_processed[i]->status !=
-					RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED) {
+				RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED) {
 				ret_status = -1;
 
 				RTE_LOG(ERR, USER1,
@@ -1358,7 +1443,8 @@ test_compressdev_deflate_stateless_fixed(void)
 		LB_BOTH,
 		ZLIB_DECOMPRESS,
 		0,
-		0
+		0,
+		OVERFLOW_DISABLED
 	};
 
 	for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
@@ -1429,7 +1515,8 @@ test_compressdev_deflate_stateless_dynamic(void)
 		LB_BOTH,
 		ZLIB_DECOMPRESS,
 		0,
-		0
+		0,
+		OVERFLOW_DISABLED
 	};
 
 	for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
@@ -1483,7 +1570,8 @@ test_compressdev_deflate_stateless_multi_op(void)
 		LB_BOTH,
 		ZLIB_DECOMPRESS,
 		0,
-		0
+		0,
+		OVERFLOW_DISABLED
 	};
 
 	/* Compress with compressdev, decompress with Zlib */
@@ -1533,7 +1621,8 @@ test_compressdev_deflate_stateless_multi_level(void)
 		LB_BOTH,
 		ZLIB_DECOMPRESS,
 		0,
-		0
+		0,
+		OVERFLOW_DISABLED
 	};
 
 	for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
@@ -1623,7 +1712,8 @@ test_compressdev_deflate_stateless_multi_xform(void)
 		LB_BOTH,
 		ZLIB_DECOMPRESS,
 		0,
-		0
+		0,
+		OVERFLOW_DISABLED
 	};
 
 	/* Compress with compressdev, decompress with Zlib */
@@ -1669,7 +1759,8 @@ test_compressdev_deflate_stateless_sgl(void)
 		SGL_BOTH,
 		ZLIB_DECOMPRESS,
 		0,
-		0
+		0,
+		OVERFLOW_DISABLED
 	};
 
 	for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
@@ -1777,7 +1868,8 @@ test_compressdev_deflate_stateless_checksum(void)
 		LB_BOTH,
 		ZLIB_DECOMPRESS,
 		0,
-		0
+		0,
+		OVERFLOW_DISABLED
 	};
 
 	/* Check if driver supports crc32 checksum and test */
@@ -1908,7 +2000,8 @@ test_compressdev_out_of_space_buffer(void)
 		LB_BOTH,
 		ZLIB_DECOMPRESS,
 		1,  /* run out-of-space test */
-		0
+		0,
+		OVERFLOW_DISABLED
 	};
 	/* Compress with compressdev, decompress with Zlib */
 	test_data.zlib_dir = ZLIB_DECOMPRESS;
@@ -1989,7 +2082,8 @@ test_compressdev_deflate_stateless_dynamic_big(void)
 		SGL_BOTH,
 		ZLIB_DECOMPRESS,
 		0,
-		1
+		1,
+		OVERFLOW_DISABLED
 	};
 
 	ts_params->def_comp_xform->compress.deflate.huffman =
@@ -2023,6 +2117,78 @@ test_compressdev_deflate_stateless_dynamic_big(void)
 }
 
 
+static int
+test_compressdev_deflate_stateless_fixed_overflow(void)
+{
+	struct comp_testsuite_params *ts_params = &testsuite_params;
+	uint16_t i;
+	int ret;
+	const struct rte_compressdev_capabilities *capab;
+
+	capab = rte_compressdev_capability_get(0, RTE_COMP_ALGO_DEFLATE);
+	TEST_ASSERT(capab != NULL, "Failed to retrieve device capabilities");
+
+	if ((capab->comp_feature_flags & RTE_COMP_FF_HUFFMAN_FIXED) == 0)
+		return -ENOTSUP;
+
+	struct rte_comp_xform *compress_xform =
+			rte_malloc(NULL, sizeof(struct rte_comp_xform), 0);
+
+	if (compress_xform == NULL) {
+		RTE_LOG(ERR, USER1,
+			"Compress xform could not be created\n");
+		ret = TEST_FAILED;
+		goto exit;
+	}
+
+	memcpy(compress_xform, ts_params->def_comp_xform,
+			sizeof(struct rte_comp_xform));
+	compress_xform->compress.deflate.huffman = RTE_COMP_HUFFMAN_FIXED;
+
+	struct interim_data_params int_data = {
+		NULL,
+		1,
+		NULL,
+		&compress_xform,
+		&ts_params->def_decomp_xform,
+		1
+	};
+
+	struct test_data_params test_data = {
+		RTE_COMP_OP_STATELESS,
+		LB_BOTH,
+		ZLIB_DECOMPRESS,
+		0,
+		0,
+		OVERFLOW_ENABLED
+	};
+
+	for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
+		int_data.test_bufs = &compress_test_bufs[i];
+		int_data.buf_idx = &i;
+
+		/* Compress with compressdev, decompress with Zlib */
+		test_data.zlib_dir = ZLIB_DECOMPRESS;
+		if (test_deflate_comp_decomp(&int_data, &test_data) < 0) {
+			ret = TEST_FAILED;
+			goto exit;
+		}
+
+		/* Compress with Zlib, decompress with compressdev */
+		test_data.zlib_dir = ZLIB_COMPRESS;
+		if (test_deflate_comp_decomp(&int_data, &test_data) < 0) {
+			ret = TEST_FAILED;
+			goto exit;
+		}
+	}
+
+	ret = TEST_SUCCESS;
+
+exit:
+	rte_free(compress_xform);
+	return ret;
+}
+
 static struct unit_test_suite compressdev_testsuite  = {
 	.suite_name = "compressdev unit test suite",
 	.setup = testsuite_setup,
@@ -2048,6 +2214,8 @@ static struct unit_test_suite compressdev_testsuite  = {
 			test_compressdev_deflate_stateless_checksum),
 		TEST_CASE_ST(generic_ut_setup, generic_ut_teardown,
 			test_compressdev_out_of_space_buffer),
+		TEST_CASE_ST(generic_ut_setup, generic_ut_teardown,
+			test_compressdev_deflate_stateless_fixed_overflow),
 		TEST_CASES_END() /**< NULL terminate unit test array */
 	}
 };
-- 
2.17.1

[dpdk-dev] [PATCH v2 0/2] Add overflow recovery for stateless compression

[Artur Trybula]

This patch adds out-of-space recoverable feature for stateless
compression scenario to QuickAssist compressdev driver. Unit test 
was also created to verify the new feature.

v2:
	- rebasing over the external mbufs

This patch depends on http://patches.dpdk.org/cover/59680/


Artur Trybula (2):
  compress/qat: overflow catch for stateless compression
  test/compress: unit test for stateless overflow recovery

 app/test/test_compressdev.c     | 261 ++++++++++++++++++++++++++------
 drivers/compress/qat/qat_comp.c |  28 +++-
 2 files changed, 240 insertions(+), 49 deletions(-)

-- 
2.17.1

[dpdk-dev] [PATCH v2 1/2] compress/qat: overflow catch for stateless compression

[Artur Trybula]

This patch adds out-of-space recoverable feature for stateless
compression scenario to QuickAssist compressdev driver

Signed-off-by: Artur Trybula <arturx.trybula@intel.com>
---
 drivers/compress/qat/qat_comp.c | 28 +++++++++++++++++++++++++---
 1 file changed, 25 insertions(+), 3 deletions(-)

diff --git a/drivers/compress/qat/qat_comp.c b/drivers/compress/qat/qat_comp.c
index 8717b7432..533e34f6b 100644
--- a/drivers/compress/qat/qat_comp.c
+++ b/drivers/compress/qat/qat_comp.c
@@ -324,10 +324,32 @@ qat_comp_process_response(void **op, uint8_t *resp, void *op_cookie,
 		    (!cmp_err_code && xlat_err_code == ERR_CODE_OVERFLOW_ERROR)
 				||
 		    (cmp_err_code == ERR_CODE_OVERFLOW_ERROR &&
-		     xlat_err_code == ERR_CODE_OVERFLOW_ERROR))
-			rx_op->status =
+		     xlat_err_code == ERR_CODE_OVERFLOW_ERROR)){
+
+			struct icp_qat_fw_resp_comp_pars *comp_resp =
+	  (struct icp_qat_fw_resp_comp_pars *)&resp_msg->comp_resp_pars;
+
+			/* handle recoverable out-of-buffer condition */
+			/* in stateless compression scenario */
+			if (comp_resp->input_byte_counter) {
+				if ((qat_xform->qat_comp_request_type
+				== QAT_COMP_REQUEST_FIXED_COMP_STATELESS) ||
+				    (qat_xform->qat_comp_request_type
+				== QAT_COMP_REQUEST_DYNAMIC_COMP_STATELESS)) {
+
+					rx_op->status =
+				RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE;
+					rx_op->consumed =
+						comp_resp->input_byte_counter;
+					rx_op->produced =
+						comp_resp->output_byte_counter;
+				} else
+					rx_op->status =
+				RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
+			} else
+				rx_op->status =
 				RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
-		else
+		} else
 			rx_op->status = RTE_COMP_OP_STATUS_ERROR;
 
 		++(*dequeue_err_count);
-- 
2.17.1

[dpdk-dev] [PATCH v2 2/2] test/compress: unit test for stateless overflow recovery

[Artur Trybula]

Added unit test to check out-of-space recoverable feature.

Signed-off-by: Artur Trybula <arturx.trybula@intel.com>
---
 app/test/test_compressdev.c | 261 +++++++++++++++++++++++++++++-------
 1 file changed, 215 insertions(+), 46 deletions(-)

diff --git a/app/test/test_compressdev.c b/app/test/test_compressdev.c
index d470c34b9..0a5c76721 100644
--- a/app/test/test_compressdev.c
+++ b/app/test/test_compressdev.c
@@ -30,6 +30,7 @@
  * due to the compress block headers
  */
 #define COMPRESS_BUF_SIZE_RATIO 1.3
+#define COMPRESS_BUF_SIZE_RATIO_OVERFLOW 0.2
 #define NUM_LARGE_MBUFS 16
 #define SMALL_SEG_SIZE 256
 #define MAX_SEGS 16
@@ -72,6 +73,11 @@ enum varied_buff {
 	LB_TO_SGL	/* output buffer is chained */
 };
 
+enum overflow_test {
+	OVERFLOW_DISABLED,
+	OVERFLOW_ENABLED
+};
+
 struct priv_op_data {
 	uint16_t orig_idx;
 };
@@ -110,6 +116,8 @@ struct test_data_params {
 	const struct rte_memzone *inbuf_memzone;
 	const struct rte_memzone *compbuf_memzone;
 	const struct rte_memzone *uncompbuf_memzone;
+	/* overflow test activation */
+	enum overflow_test overflow;
 };
 
 static struct comp_testsuite_params testsuite_params = { 0 };
@@ -725,6 +733,49 @@ extbuf_free_callback(void *addr __rte_unused, void *opaque __rte_unused)
 {
 }
 
+static int
+test_run_enqueue_dequeue(struct rte_comp_op **ops, unsigned int num_bufs,
+		  struct rte_comp_op **ops_processed)
+{
+	uint16_t num_enqd, num_deqd, num_total_deqd;
+	unsigned int deqd_retries = 0;
+
+	/* Enqueue and dequeue all operations */
+	num_enqd = rte_compressdev_enqueue_burst(0, 0, ops, num_bufs);
+	if (num_enqd < num_bufs) {
+		RTE_LOG(ERR, USER1,
+			"Some operations could not be enqueued\n");
+		return -1;
+	}
+
+	num_total_deqd = 0;
+	do {
+		/*
+		 * If retrying a dequeue call, wait for 10 ms to allow
+		 * enough time to the driver to process the operations
+		 */
+		if (deqd_retries != 0) {
+			/*
+			 * Avoid infinite loop if not all the
+			 * operations get out of the device
+			 */
+			if (deqd_retries == MAX_DEQD_RETRIES) {
+				RTE_LOG(ERR, USER1,
+					"Not all operations could be dequeued\n");
+				return -1;
+			}
+			usleep(DEQUEUE_WAIT_TIME);
+		}
+		num_deqd = rte_compressdev_dequeue_burst(0, 0,
+				&ops_processed[num_total_deqd], num_bufs);
+		num_total_deqd += num_deqd;
+		deqd_retries++;
+
+	} while (num_total_deqd < num_enqd);
+
+	return 0;
+}
+
 /*
  * Compresses and decompresses buffer with compressdev API and Zlib API
  */
@@ -745,6 +796,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 	unsigned int out_of_space = test_data->out_of_space;
 	unsigned int big_data = test_data->big_data;
 	enum zlib_direction zlib_dir = test_data->zlib_dir;
+	enum overflow_test overflow_tst = test_data->overflow;
 	int ret_status = TEST_FAILED;
 	struct rte_mbuf_ext_shared_info inbuf_info;
 	struct rte_mbuf_ext_shared_info compbuf_info;
@@ -773,6 +825,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 		rte_compressdev_capability_get(0, RTE_COMP_ALGO_DEFLATE);
 	char *contig_buf = NULL;
 	uint64_t compress_checksum[num_bufs];
+	uint32_t compressed_data_size[num_bufs];
 	void *stream = NULL;
 	char *all_decomp_data = NULL;
 	unsigned int decomp_produced_data_size = 0;
@@ -793,6 +846,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 	memset(ops, 0, sizeof(struct rte_comp_op *) * num_bufs);
 	memset(ops_processed, 0, sizeof(struct rte_comp_op *) * num_bufs);
 	memset(priv_xforms, 0, sizeof(void *) * num_bufs);
+	memset(compressed_data_size, 0, sizeof(uint32_t) * num_bufs);
 
 	if (decompress_state == RTE_COMP_OP_STATEFUL) {
 		data_size = strlen(test_bufs[0]) + 1;
@@ -845,6 +899,11 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 		for (i = 0; i < num_bufs; i++) {
 			data_size = strlen(test_bufs[i]) + 1;
 			buf_ptr = rte_pktmbuf_append(uncomp_bufs[i], data_size);
+			if (buf_ptr == NULL) {
+				RTE_LOG(ERR, USER1,
+					"Append extra bytes to the source mbuf failed\n");
+				goto exit;
+			}
 			strlcpy(buf_ptr, test_bufs[i], data_size);
 		}
 	}
@@ -893,11 +952,23 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 		for (i = 0; i < num_bufs; i++) {
 			if (out_of_space == 1 && oos_zlib_decompress)
 				data_size = OUT_OF_SPACE_BUF;
-			else
-				(data_size = strlen(test_bufs[i]) *
-					COMPRESS_BUF_SIZE_RATIO);
+			else {
+				float ratio =
+				((test_data->zlib_dir == ZLIB_DECOMPRESS ||
+				   test_data->zlib_dir == ZLIB_NONE) &&
+				  overflow_tst == OVERFLOW_ENABLED) ?
+					 COMPRESS_BUF_SIZE_RATIO_OVERFLOW :
+					 COMPRESS_BUF_SIZE_RATIO;
+
+				data_size = strlen(test_bufs[i]) * ratio;
+			}
+			buf_ptr = rte_pktmbuf_append(comp_bufs[i], data_size);
+			if (buf_ptr == NULL) {
+				RTE_LOG(ERR, USER1,
+					"Append extra bytes to the destination mbuf failed\n");
+				goto exit;
+			}
 
-			rte_pktmbuf_append(comp_bufs[i], data_size);
 		}
 	}
 
@@ -963,7 +1034,8 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 			num_priv_xforms++;
 		}
 
-		if (capa->comp_feature_flags & RTE_COMP_FF_SHAREABLE_PRIV_XFORM) {
+		if (capa->comp_feature_flags &
+				RTE_COMP_FF_SHAREABLE_PRIV_XFORM) {
 			/* Attach shareable private xform data to ops */
 			for (i = 0; i < num_bufs; i++)
 				ops[i]->private_xform = priv_xforms[i % num_xforms];
@@ -987,40 +1059,41 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 				ops[i]->private_xform = priv_xforms[i];
 		}
 
-		/* Enqueue and dequeue all operations */
-		num_enqd = rte_compressdev_enqueue_burst(0, 0, ops, num_bufs);
-		if (num_enqd < num_bufs) {
+recovery_lb:
+		ret = test_run_enqueue_dequeue(ops, num_bufs, ops_processed);
+		if (ret < 0) {
 			RTE_LOG(ERR, USER1,
-				"The operations could not be enqueued\n");
+				"Enqueue/dequeue operation failed\n");
 			goto exit;
 		}
 
-		num_total_deqd = 0;
-		do {
-			/*
-			 * If retrying a dequeue call, wait for 10 ms to allow
-			 * enough time to the driver to process the operations
-			 */
-			if (deqd_retries != 0) {
-				/*
-				 * Avoid infinite loop if not all the
-				 * operations get out of the device
-				 */
-				if (deqd_retries == MAX_DEQD_RETRIES) {
+		for (i = 0; i < num_bufs; i++) {
+			compressed_data_size[i] += ops_processed[i]->produced;
+
+			if (ops_processed[i]->status ==
+				RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE) {
+
+				ops[i]->status =
+					RTE_COMP_OP_STATUS_NOT_PROCESSED;
+				ops[i]->src.offset +=
+					ops_processed[i]->consumed;
+				ops[i]->src.length -=
+					ops_processed[i]->consumed;
+				ops[i]->dst.offset +=
+					ops_processed[i]->produced;
+
+				buf_ptr = rte_pktmbuf_append(
+					ops[i]->m_dst,
+					ops_processed[i]->produced);
+
+				if (buf_ptr == NULL) {
 					RTE_LOG(ERR, USER1,
-						"Not all operations could be "
-						"dequeued\n");
+						"Data recovery: append extra bytes to the current mbuf failed\n");
 					goto exit;
 				}
-				usleep(DEQUEUE_WAIT_TIME);
+				goto recovery_lb;
 			}
-			num_deqd = rte_compressdev_dequeue_burst(0, 0,
-					&ops_processed[num_total_deqd], num_bufs);
-			num_total_deqd += num_deqd;
-			deqd_retries++;
-
-		} while (num_total_deqd < num_enqd);
-
+		}
 		deqd_retries = 0;
 
 		/* Free compress private xforms */
@@ -1064,7 +1137,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 	for (i = 0; i < num_bufs; i++) {
 		if (out_of_space && oos_zlib_decompress) {
 			if (ops_processed[i]->status !=
-					RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED) {
+				RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED) {
 				ret_status = TEST_FAILED;
 				RTE_LOG(ERR, USER1,
 					"Operation without expected out of "
@@ -1147,7 +1220,12 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 				data_size =
 				strlen(test_bufs[priv_data->orig_idx]) + 1;
 
-			rte_pktmbuf_append(uncomp_bufs[i], data_size);
+			buf_ptr = rte_pktmbuf_append(uncomp_bufs[i], data_size);
+			if (buf_ptr == NULL) {
+				RTE_LOG(ERR, USER1,
+					"Append extra bytes to the decompressed mbuf failed\n");
+				goto exit;
+			}
 		}
 	}
 
@@ -1169,7 +1247,10 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 		 * Set the length of the compressed data to the
 		 * number of bytes that were produced in the previous stage
 		 */
-		ops[i]->src.length = ops_processed[i]->produced;
+		if (compressed_data_size[i])
+			ops[i]->src.length = compressed_data_size[i];
+		else
+			ops[i]->src.length = ops_processed[i]->produced;
 
 		ops[i]->dst.offset = 0;
 		if (decompress_state == RTE_COMP_OP_STATELESS) {
@@ -1541,7 +1622,8 @@ test_compressdev_deflate_stateless_fixed(void)
 		.buff_type = LB_BOTH,
 		.zlib_dir = ZLIB_DECOMPRESS,
 		.out_of_space = 0,
-		.big_data = 0
+		.big_data = 0,
+		.overflow = OVERFLOW_DISABLED
 	};
 
 	for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
@@ -1611,7 +1693,8 @@ test_compressdev_deflate_stateless_dynamic(void)
 		.buff_type = LB_BOTH,
 		.zlib_dir = ZLIB_DECOMPRESS,
 		.out_of_space = 0,
-		.big_data = 0
+		.big_data = 0,
+		.overflow = OVERFLOW_DISABLED
 	};
 
 	for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
@@ -1665,7 +1748,8 @@ test_compressdev_deflate_stateless_multi_op(void)
 		.buff_type = LB_BOTH,
 		.zlib_dir = ZLIB_DECOMPRESS,
 		.out_of_space = 0,
-		.big_data = 0
+		.big_data = 0,
+		.overflow = OVERFLOW_DISABLED
 	};
 
 	/* Compress with compressdev, decompress with Zlib */
@@ -1718,7 +1802,8 @@ test_compressdev_deflate_stateless_multi_level(void)
 		.buff_type = LB_BOTH,
 		.zlib_dir = ZLIB_DECOMPRESS,
 		.out_of_space = 0,
-		.big_data = 0
+		.big_data = 0,
+		.overflow = OVERFLOW_DISABLED
 	};
 
 	for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
@@ -1807,7 +1892,8 @@ test_compressdev_deflate_stateless_multi_xform(void)
 		.buff_type = LB_BOTH,
 		.zlib_dir = ZLIB_DECOMPRESS,
 		.out_of_space = 0,
-		.big_data = 0
+		.big_data = 0,
+		.overflow = OVERFLOW_DISABLED
 	};
 
 	/* Compress with compressdev, decompress with Zlib */
@@ -1855,7 +1941,8 @@ test_compressdev_deflate_stateless_sgl(void)
 		.buff_type = SGL_BOTH,
 		.zlib_dir = ZLIB_DECOMPRESS,
 		.out_of_space = 0,
-		.big_data = 0
+		.big_data = 0,
+		.overflow = OVERFLOW_DISABLED
 	};
 
 	for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
@@ -1965,7 +2052,8 @@ test_compressdev_deflate_stateless_checksum(void)
 		.buff_type = LB_BOTH,
 		.zlib_dir = ZLIB_DECOMPRESS,
 		.out_of_space = 0,
-		.big_data = 0
+		.big_data = 0,
+		.overflow = OVERFLOW_DISABLED
 	};
 
 	/* Check if driver supports crc32 checksum and test */
@@ -2082,7 +2170,8 @@ test_compressdev_out_of_space_buffer(void)
 		.buff_type = LB_BOTH,
 		.zlib_dir = ZLIB_DECOMPRESS,
 		.out_of_space = 1,  /* run out-of-space test */
-		.big_data = 0
+		.big_data = 0,
+		.overflow = OVERFLOW_DISABLED
 	};
 	/* Compress with compressdev, decompress with Zlib */
 	test_data.zlib_dir = ZLIB_DECOMPRESS;
@@ -2159,7 +2248,8 @@ test_compressdev_deflate_stateless_dynamic_big(void)
 		.buff_type = SGL_BOTH,
 		.zlib_dir = ZLIB_DECOMPRESS,
 		.out_of_space = 0,
-		.big_data = 1
+		.big_data = 1,
+		.overflow = OVERFLOW_DISABLED
 	};
 
 	ts_params->def_comp_xform->compress.deflate.huffman =
@@ -2223,7 +2313,8 @@ test_compressdev_deflate_stateful_decomp(void)
 		.out_of_space = 0,
 		.big_data = 0,
 		.decompress_output_block_size = 2000,
-		.decompress_steps_max = 4
+		.decompress_steps_max = 4,
+		.overflow = OVERFLOW_DISABLED
 	};
 
 	/* Compress with Zlib, decompress with compressdev */
@@ -2305,7 +2396,8 @@ test_compressdev_deflate_stateful_decomp_checksum(void)
 		.out_of_space = 0,
 		.big_data = 0,
 		.decompress_output_block_size = 2000,
-		.decompress_steps_max = 4
+		.decompress_steps_max = 4,
+		.overflow = OVERFLOW_DISABLED
 	};
 
 	/* Check if driver supports crc32 checksum and test */
@@ -2386,6 +2478,7 @@ test_compressdev_deflate_stateful_decomp_checksum(void)
 	return ret;
 }
 
+
 static const struct rte_memzone *
 make_memzone(const char *name, size_t size)
 {
@@ -2441,7 +2534,8 @@ test_compressdev_external_mbufs(void)
 		.inbuf_memzone = make_memzone("inbuf", data_len),
 		.compbuf_memzone = make_memzone("compbuf", data_len *
 						COMPRESS_BUF_SIZE_RATIO),
-		.uncompbuf_memzone = make_memzone("decompbuf", data_len)
+		.uncompbuf_memzone = make_memzone("decompbuf", data_len),
+		.overflow = OVERFLOW_DISABLED
 	};
 
 	for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
@@ -2472,6 +2566,79 @@ test_compressdev_external_mbufs(void)
 	return ret;
 }
 
+static int
+test_compressdev_deflate_stateless_fixed_overflow(void)
+{
+	struct comp_testsuite_params *ts_params = &testsuite_params;
+	uint16_t i;
+	int ret;
+	const struct rte_compressdev_capabilities *capab;
+
+	capab = rte_compressdev_capability_get(0, RTE_COMP_ALGO_DEFLATE);
+	TEST_ASSERT(capab != NULL, "Failed to retrieve device capabilities");
+
+	if ((capab->comp_feature_flags & RTE_COMP_FF_HUFFMAN_FIXED) == 0)
+		return -ENOTSUP;
+
+	struct rte_comp_xform *compress_xform =
+			rte_malloc(NULL, sizeof(struct rte_comp_xform), 0);
+
+	if (compress_xform == NULL) {
+		RTE_LOG(ERR, USER1,
+			"Compress xform could not be created\n");
+		ret = TEST_FAILED;
+		goto exit;
+	}
+
+	memcpy(compress_xform, ts_params->def_comp_xform,
+			sizeof(struct rte_comp_xform));
+	compress_xform->compress.deflate.huffman = RTE_COMP_HUFFMAN_FIXED;
+
+	struct interim_data_params int_data = {
+		NULL,
+		1,
+		NULL,
+		&compress_xform,
+		&ts_params->def_decomp_xform,
+		1
+	};
+
+	struct test_data_params test_data = {
+		.compress_state = RTE_COMP_OP_STATELESS,
+		.decompress_state = RTE_COMP_OP_STATELESS,
+		.buff_type = LB_BOTH,
+		.zlib_dir = ZLIB_DECOMPRESS,
+		.out_of_space = 0,
+		.big_data = 0,
+		.overflow = OVERFLOW_ENABLED
+	};
+
+	for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
+		int_data.test_bufs = &compress_test_bufs[i];
+		int_data.buf_idx = &i;
+
+		/* Compress with compressdev, decompress with Zlib */
+		test_data.zlib_dir = ZLIB_DECOMPRESS;
+		if (test_deflate_comp_decomp(&int_data, &test_data) < 0) {
+			ret = TEST_FAILED;
+			goto exit;
+		}
+
+		/* Compress with Zlib, decompress with compressdev */
+		test_data.zlib_dir = ZLIB_COMPRESS;
+		if (test_deflate_comp_decomp(&int_data, &test_data) < 0) {
+			ret = TEST_FAILED;
+			goto exit;
+		}
+	}
+
+	ret = TEST_SUCCESS;
+
+exit:
+	rte_free(compress_xform);
+	return ret;
+}
+
 static struct unit_test_suite compressdev_testsuite  = {
 	.suite_name = "compressdev unit test suite",
 	.setup = testsuite_setup,
@@ -2503,6 +2670,8 @@ static struct unit_test_suite compressdev_testsuite  = {
 			test_compressdev_deflate_stateful_decomp_checksum),
 		TEST_CASE_ST(generic_ut_setup, generic_ut_teardown,
 			test_compressdev_external_mbufs),
+		TEST_CASE_ST(generic_ut_setup, generic_ut_teardown,
+			test_compressdev_deflate_stateless_fixed_overflow),
 		TEST_CASES_END() /**< NULL terminate unit test array */
 	}
 };
-- 
2.17.1

[dpdk-dev] [PATCH v3 0/2] Add overflow recovery for stateless compression

[Artur Trybula]

This patch adds out-of-space recoverable feature for stateless
compression scenario to QuickAssist compressdev driver. Unit test 
was also created to verify the new feature.

v3:
	- rebasing onto the recent master
v2:
	- rebasing over the external mbufs

Artur Trybula (2):
  compress/qat: overflow catch for stateless compression
  test/compress: unit test for stateless overflow recovery

 app/test/test_compressdev.c     | 261 ++++++++++++++++++++++++++------
 drivers/compress/qat/qat_comp.c |  28 +++-
 2 files changed, 239 insertions(+), 50 deletions(-)

-- 
2.17.1

[dpdk-dev] [PATCH v3 1/2] compress/qat: overflow catch for stateless compression

[Artur Trybula]

This patch adds out-of-space recoverable feature for stateless
compression scenario to QuickAssist compressdev driver

Signed-off-by: Artur Trybula <arturx.trybula@intel.com>
---
 drivers/compress/qat/qat_comp.c | 28 +++++++++++++++++++++++++---
 1 file changed, 25 insertions(+), 3 deletions(-)

diff --git a/drivers/compress/qat/qat_comp.c b/drivers/compress/qat/qat_comp.c
index 8717b7432..533e34f6b 100644
--- a/drivers/compress/qat/qat_comp.c
+++ b/drivers/compress/qat/qat_comp.c
@@ -324,10 +324,32 @@ qat_comp_process_response(void **op, uint8_t *resp, void *op_cookie,
 		    (!cmp_err_code && xlat_err_code == ERR_CODE_OVERFLOW_ERROR)
 				||
 		    (cmp_err_code == ERR_CODE_OVERFLOW_ERROR &&
-		     xlat_err_code == ERR_CODE_OVERFLOW_ERROR))
-			rx_op->status =
+		     xlat_err_code == ERR_CODE_OVERFLOW_ERROR)){
+
+			struct icp_qat_fw_resp_comp_pars *comp_resp =
+	  (struct icp_qat_fw_resp_comp_pars *)&resp_msg->comp_resp_pars;
+
+			/* handle recoverable out-of-buffer condition */
+			/* in stateless compression scenario */
+			if (comp_resp->input_byte_counter) {
+				if ((qat_xform->qat_comp_request_type
+				== QAT_COMP_REQUEST_FIXED_COMP_STATELESS) ||
+				    (qat_xform->qat_comp_request_type
+				== QAT_COMP_REQUEST_DYNAMIC_COMP_STATELESS)) {
+
+					rx_op->status =
+				RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE;
+					rx_op->consumed =
+						comp_resp->input_byte_counter;
+					rx_op->produced =
+						comp_resp->output_byte_counter;
+				} else
+					rx_op->status =
+				RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
+			} else
+				rx_op->status =
 				RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
-		else
+		} else
 			rx_op->status = RTE_COMP_OP_STATUS_ERROR;
 
 		++(*dequeue_err_count);
-- 
2.17.1

Re: [dpdk-dev] [PATCH v3 1/2] compress/qat: overflow catch for stateless compression

[Trahe, Fiona]

> -----Original Message-----
> From: Trybula, ArturX
> Sent: Friday, October 11, 2019 6:46 PM
> To: dev@dpdk.org; Trahe, Fiona <fiona.trahe@intel.com>; shallyv@marvell.com; Dybkowski, AdamX
> <adamx.dybkowski@intel.com>; Trybula, ArturX <arturx.trybula@intel.com>; akhil.goyal@nxp.com
> Subject: [PATCH v3 1/2] compress/qat: overflow catch for stateless compression
> 
> This patch adds out-of-space recoverable feature for stateless
> compression scenario to QuickAssist compressdev driver
> 
> Signed-off-by: Artur Trybula <arturx.trybula@intel.com>
Acked-by: Fiona Trahe <fiona.trahe@intel.com>

[dpdk-dev] [PATCH v3 2/2] test/compress: unit test for stateless overflow recovery

[Artur Trybula]

Added unit test to check out-of-space recoverable feature.

Signed-off-by: Artur Trybula <arturx.trybula@intel.com>
---
 app/test/test_compressdev.c | 261 +++++++++++++++++++++++++++++-------
 1 file changed, 214 insertions(+), 47 deletions(-)

diff --git a/app/test/test_compressdev.c b/app/test/test_compressdev.c
index d470c34b9..335deb263 100644
--- a/app/test/test_compressdev.c
+++ b/app/test/test_compressdev.c
@@ -30,6 +30,7 @@
  * due to the compress block headers
  */
 #define COMPRESS_BUF_SIZE_RATIO 1.3
+#define COMPRESS_BUF_SIZE_RATIO_OVERFLOW 0.2
 #define NUM_LARGE_MBUFS 16
 #define SMALL_SEG_SIZE 256
 #define MAX_SEGS 16
@@ -72,6 +73,11 @@ enum varied_buff {
 	LB_TO_SGL	/* output buffer is chained */
 };
 
+enum overflow_test {
+	OVERFLOW_DISABLED,
+	OVERFLOW_ENABLED
+};
+
 struct priv_op_data {
 	uint16_t orig_idx;
 };
@@ -110,6 +116,8 @@ struct test_data_params {
 	const struct rte_memzone *inbuf_memzone;
 	const struct rte_memzone *compbuf_memzone;
 	const struct rte_memzone *uncompbuf_memzone;
+	/* overflow test activation */
+	enum overflow_test overflow;
 };
 
 static struct comp_testsuite_params testsuite_params = { 0 };
@@ -725,6 +733,49 @@ extbuf_free_callback(void *addr __rte_unused, void *opaque __rte_unused)
 {
 }
 
+static int
+test_run_enqueue_dequeue(struct rte_comp_op **ops, unsigned int num_bufs,
+		  struct rte_comp_op **ops_processed)
+{
+	uint16_t num_enqd, num_deqd, num_total_deqd;
+	unsigned int deqd_retries = 0;
+
+	/* Enqueue and dequeue all operations */
+	num_enqd = rte_compressdev_enqueue_burst(0, 0, ops, num_bufs);
+	if (num_enqd < num_bufs) {
+		RTE_LOG(ERR, USER1,
+			"Some operations could not be enqueued\n");
+		return -1;
+	}
+
+	num_total_deqd = 0;
+	do {
+		/*
+		 * If retrying a dequeue call, wait for 10 ms to allow
+		 * enough time to the driver to process the operations
+		 */
+		if (deqd_retries != 0) {
+			/*
+			 * Avoid infinite loop if not all the
+			 * operations get out of the device
+			 */
+			if (deqd_retries == MAX_DEQD_RETRIES) {
+				RTE_LOG(ERR, USER1,
+					"Not all operations could be dequeued\n");
+				return -1;
+			}
+			usleep(DEQUEUE_WAIT_TIME);
+		}
+		num_deqd = rte_compressdev_dequeue_burst(0, 0,
+				&ops_processed[num_total_deqd], num_bufs);
+		num_total_deqd += num_deqd;
+		deqd_retries++;
+
+	} while (num_total_deqd < num_enqd);
+
+	return 0;
+}
+
 /*
  * Compresses and decompresses buffer with compressdev API and Zlib API
  */
@@ -745,6 +796,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 	unsigned int out_of_space = test_data->out_of_space;
 	unsigned int big_data = test_data->big_data;
 	enum zlib_direction zlib_dir = test_data->zlib_dir;
+	enum overflow_test overflow_tst = test_data->overflow;
 	int ret_status = TEST_FAILED;
 	struct rte_mbuf_ext_shared_info inbuf_info;
 	struct rte_mbuf_ext_shared_info compbuf_info;
@@ -773,6 +825,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 		rte_compressdev_capability_get(0, RTE_COMP_ALGO_DEFLATE);
 	char *contig_buf = NULL;
 	uint64_t compress_checksum[num_bufs];
+	uint32_t compressed_data_size[num_bufs];
 	void *stream = NULL;
 	char *all_decomp_data = NULL;
 	unsigned int decomp_produced_data_size = 0;
@@ -793,6 +846,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 	memset(ops, 0, sizeof(struct rte_comp_op *) * num_bufs);
 	memset(ops_processed, 0, sizeof(struct rte_comp_op *) * num_bufs);
 	memset(priv_xforms, 0, sizeof(void *) * num_bufs);
+	memset(compressed_data_size, 0, sizeof(uint32_t) * num_bufs);
 
 	if (decompress_state == RTE_COMP_OP_STATEFUL) {
 		data_size = strlen(test_bufs[0]) + 1;
@@ -845,6 +899,11 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 		for (i = 0; i < num_bufs; i++) {
 			data_size = strlen(test_bufs[i]) + 1;
 			buf_ptr = rte_pktmbuf_append(uncomp_bufs[i], data_size);
+			if (buf_ptr == NULL) {
+				RTE_LOG(ERR, USER1,
+					"Append extra bytes to the source mbuf failed\n");
+				goto exit;
+			}
 			strlcpy(buf_ptr, test_bufs[i], data_size);
 		}
 	}
@@ -893,11 +952,22 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 		for (i = 0; i < num_bufs; i++) {
 			if (out_of_space == 1 && oos_zlib_decompress)
 				data_size = OUT_OF_SPACE_BUF;
-			else
-				(data_size = strlen(test_bufs[i]) *
-					COMPRESS_BUF_SIZE_RATIO);
-
-			rte_pktmbuf_append(comp_bufs[i], data_size);
+			else {
+				float ratio =
+				((test_data->zlib_dir == ZLIB_DECOMPRESS ||
+				   test_data->zlib_dir == ZLIB_NONE) &&
+				  overflow_tst == OVERFLOW_ENABLED) ?
+					 COMPRESS_BUF_SIZE_RATIO_OVERFLOW :
+					 COMPRESS_BUF_SIZE_RATIO;
+
+				data_size = strlen(test_bufs[i]) * ratio;
+			}
+			buf_ptr = rte_pktmbuf_append(comp_bufs[i], data_size);
+			if (buf_ptr == NULL) {
+				RTE_LOG(ERR, USER1,
+					"Append extra bytes to the destination mbuf failed\n");
+				goto exit;
+			}
 		}
 	}
 
@@ -963,7 +1033,8 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 			num_priv_xforms++;
 		}
 
-		if (capa->comp_feature_flags & RTE_COMP_FF_SHAREABLE_PRIV_XFORM) {
+		if (capa->comp_feature_flags &
+				RTE_COMP_FF_SHAREABLE_PRIV_XFORM) {
 			/* Attach shareable private xform data to ops */
 			for (i = 0; i < num_bufs; i++)
 				ops[i]->private_xform = priv_xforms[i % num_xforms];
@@ -987,40 +1058,41 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 				ops[i]->private_xform = priv_xforms[i];
 		}
 
-		/* Enqueue and dequeue all operations */
-		num_enqd = rte_compressdev_enqueue_burst(0, 0, ops, num_bufs);
-		if (num_enqd < num_bufs) {
+recovery_lb:
+		ret = test_run_enqueue_dequeue(ops, num_bufs, ops_processed);
+		if (ret < 0) {
 			RTE_LOG(ERR, USER1,
-				"The operations could not be enqueued\n");
+				"Enqueue/dequeue operation failed\n");
 			goto exit;
 		}
 
-		num_total_deqd = 0;
-		do {
-			/*
-			 * If retrying a dequeue call, wait for 10 ms to allow
-			 * enough time to the driver to process the operations
-			 */
-			if (deqd_retries != 0) {
-				/*
-				 * Avoid infinite loop if not all the
-				 * operations get out of the device
-				 */
-				if (deqd_retries == MAX_DEQD_RETRIES) {
+		for (i = 0; i < num_bufs; i++) {
+			compressed_data_size[i] += ops_processed[i]->produced;
+
+			if (ops_processed[i]->status ==
+				RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE) {
+
+				ops[i]->status =
+					RTE_COMP_OP_STATUS_NOT_PROCESSED;
+				ops[i]->src.offset +=
+					ops_processed[i]->consumed;
+				ops[i]->src.length -=
+					ops_processed[i]->consumed;
+				ops[i]->dst.offset +=
+					ops_processed[i]->produced;
+
+				buf_ptr = rte_pktmbuf_append(
+					ops[i]->m_dst,
+					ops_processed[i]->produced);
+
+				if (buf_ptr == NULL) {
 					RTE_LOG(ERR, USER1,
-						"Not all operations could be "
-						"dequeued\n");
+						"Data recovery: append extra bytes to the current mbuf failed\n");
 					goto exit;
 				}
-				usleep(DEQUEUE_WAIT_TIME);
+				goto recovery_lb;
 			}
-			num_deqd = rte_compressdev_dequeue_burst(0, 0,
-					&ops_processed[num_total_deqd], num_bufs);
-			num_total_deqd += num_deqd;
-			deqd_retries++;
-
-		} while (num_total_deqd < num_enqd);
-
+		}
 		deqd_retries = 0;
 
 		/* Free compress private xforms */
@@ -1064,7 +1136,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 	for (i = 0; i < num_bufs; i++) {
 		if (out_of_space && oos_zlib_decompress) {
 			if (ops_processed[i]->status !=
-					RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED) {
+				RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED) {
 				ret_status = TEST_FAILED;
 				RTE_LOG(ERR, USER1,
 					"Operation without expected out of "
@@ -1147,7 +1219,12 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 				data_size =
 				strlen(test_bufs[priv_data->orig_idx]) + 1;
 
-			rte_pktmbuf_append(uncomp_bufs[i], data_size);
+			buf_ptr = rte_pktmbuf_append(uncomp_bufs[i], data_size);
+			if (buf_ptr == NULL) {
+				RTE_LOG(ERR, USER1,
+					"Append extra bytes to the decompressed mbuf failed\n");
+				goto exit;
+			}
 		}
 	}
 
@@ -1169,7 +1246,10 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 		 * Set the length of the compressed data to the
 		 * number of bytes that were produced in the previous stage
 		 */
-		ops[i]->src.length = ops_processed[i]->produced;
+		if (compressed_data_size[i])
+			ops[i]->src.length = compressed_data_size[i];
+		else
+			ops[i]->src.length = ops_processed[i]->produced;
 
 		ops[i]->dst.offset = 0;
 		if (decompress_state == RTE_COMP_OP_STATELESS) {
@@ -1541,7 +1621,8 @@ test_compressdev_deflate_stateless_fixed(void)
 		.buff_type = LB_BOTH,
 		.zlib_dir = ZLIB_DECOMPRESS,
 		.out_of_space = 0,
-		.big_data = 0
+		.big_data = 0,
+		.overflow = OVERFLOW_DISABLED
 	};
 
 	for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
@@ -1611,7 +1692,8 @@ test_compressdev_deflate_stateless_dynamic(void)
 		.buff_type = LB_BOTH,
 		.zlib_dir = ZLIB_DECOMPRESS,
 		.out_of_space = 0,
-		.big_data = 0
+		.big_data = 0,
+		.overflow = OVERFLOW_DISABLED
 	};
 
 	for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
@@ -1665,7 +1747,8 @@ test_compressdev_deflate_stateless_multi_op(void)
 		.buff_type = LB_BOTH,
 		.zlib_dir = ZLIB_DECOMPRESS,
 		.out_of_space = 0,
-		.big_data = 0
+		.big_data = 0,
+		.overflow = OVERFLOW_DISABLED
 	};
 
 	/* Compress with compressdev, decompress with Zlib */
@@ -1718,7 +1801,8 @@ test_compressdev_deflate_stateless_multi_level(void)
 		.buff_type = LB_BOTH,
 		.zlib_dir = ZLIB_DECOMPRESS,
 		.out_of_space = 0,
-		.big_data = 0
+		.big_data = 0,
+		.overflow = OVERFLOW_DISABLED
 	};
 
 	for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
@@ -1807,7 +1891,8 @@ test_compressdev_deflate_stateless_multi_xform(void)
 		.buff_type = LB_BOTH,
 		.zlib_dir = ZLIB_DECOMPRESS,
 		.out_of_space = 0,
-		.big_data = 0
+		.big_data = 0,
+		.overflow = OVERFLOW_DISABLED
 	};
 
 	/* Compress with compressdev, decompress with Zlib */
@@ -1855,7 +1940,8 @@ test_compressdev_deflate_stateless_sgl(void)
 		.buff_type = SGL_BOTH,
 		.zlib_dir = ZLIB_DECOMPRESS,
 		.out_of_space = 0,
-		.big_data = 0
+		.big_data = 0,
+		.overflow = OVERFLOW_DISABLED
 	};
 
 	for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
@@ -1965,7 +2051,8 @@ test_compressdev_deflate_stateless_checksum(void)
 		.buff_type = LB_BOTH,
 		.zlib_dir = ZLIB_DECOMPRESS,
 		.out_of_space = 0,
-		.big_data = 0
+		.big_data = 0,
+		.overflow = OVERFLOW_DISABLED
 	};
 
 	/* Check if driver supports crc32 checksum and test */
@@ -2082,7 +2169,8 @@ test_compressdev_out_of_space_buffer(void)
 		.buff_type = LB_BOTH,
 		.zlib_dir = ZLIB_DECOMPRESS,
 		.out_of_space = 1,  /* run out-of-space test */
-		.big_data = 0
+		.big_data = 0,
+		.overflow = OVERFLOW_DISABLED
 	};
 	/* Compress with compressdev, decompress with Zlib */
 	test_data.zlib_dir = ZLIB_DECOMPRESS;
@@ -2159,7 +2247,8 @@ test_compressdev_deflate_stateless_dynamic_big(void)
 		.buff_type = SGL_BOTH,
 		.zlib_dir = ZLIB_DECOMPRESS,
 		.out_of_space = 0,
-		.big_data = 1
+		.big_data = 1,
+		.overflow = OVERFLOW_DISABLED
 	};
 
 	ts_params->def_comp_xform->compress.deflate.huffman =
@@ -2223,7 +2312,8 @@ test_compressdev_deflate_stateful_decomp(void)
 		.out_of_space = 0,
 		.big_data = 0,
 		.decompress_output_block_size = 2000,
-		.decompress_steps_max = 4
+		.decompress_steps_max = 4,
+		.overflow = OVERFLOW_DISABLED
 	};
 
 	/* Compress with Zlib, decompress with compressdev */
@@ -2305,7 +2395,8 @@ test_compressdev_deflate_stateful_decomp_checksum(void)
 		.out_of_space = 0,
 		.big_data = 0,
 		.decompress_output_block_size = 2000,
-		.decompress_steps_max = 4
+		.decompress_steps_max = 4,
+		.overflow = OVERFLOW_DISABLED
 	};
 
 	/* Check if driver supports crc32 checksum and test */
@@ -2441,7 +2532,8 @@ test_compressdev_external_mbufs(void)
 		.inbuf_memzone = make_memzone("inbuf", data_len),
 		.compbuf_memzone = make_memzone("compbuf", data_len *
 						COMPRESS_BUF_SIZE_RATIO),
-		.uncompbuf_memzone = make_memzone("decompbuf", data_len)
+		.uncompbuf_memzone = make_memzone("decompbuf", data_len),
+		.overflow = OVERFLOW_DISABLED
 	};
 
 	for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
@@ -2472,6 +2564,79 @@ test_compressdev_external_mbufs(void)
 	return ret;
 }
 
+static int
+test_compressdev_deflate_stateless_fixed_overflow(void)
+{
+	struct comp_testsuite_params *ts_params = &testsuite_params;
+	uint16_t i;
+	int ret;
+	const struct rte_compressdev_capabilities *capab;
+
+	capab = rte_compressdev_capability_get(0, RTE_COMP_ALGO_DEFLATE);
+	TEST_ASSERT(capab != NULL, "Failed to retrieve device capabilities");
+
+	if ((capab->comp_feature_flags & RTE_COMP_FF_HUFFMAN_FIXED) == 0)
+		return -ENOTSUP;
+
+	struct rte_comp_xform *compress_xform =
+			rte_malloc(NULL, sizeof(struct rte_comp_xform), 0);
+
+	if (compress_xform == NULL) {
+		RTE_LOG(ERR, USER1,
+			"Compress xform could not be created\n");
+		ret = TEST_FAILED;
+		goto exit;
+	}
+
+	memcpy(compress_xform, ts_params->def_comp_xform,
+			sizeof(struct rte_comp_xform));
+	compress_xform->compress.deflate.huffman = RTE_COMP_HUFFMAN_FIXED;
+
+	struct interim_data_params int_data = {
+		NULL,
+		1,
+		NULL,
+		&compress_xform,
+		&ts_params->def_decomp_xform,
+		1
+	};
+
+	struct test_data_params test_data = {
+		.compress_state = RTE_COMP_OP_STATELESS,
+		.decompress_state = RTE_COMP_OP_STATELESS,
+		.buff_type = LB_BOTH,
+		.zlib_dir = ZLIB_DECOMPRESS,
+		.out_of_space = 0,
+		.big_data = 0,
+		.overflow = OVERFLOW_ENABLED
+	};
+
+	for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
+		int_data.test_bufs = &compress_test_bufs[i];
+		int_data.buf_idx = &i;
+
+		/* Compress with compressdev, decompress with Zlib */
+		test_data.zlib_dir = ZLIB_DECOMPRESS;
+		if (test_deflate_comp_decomp(&int_data, &test_data) < 0) {
+			ret = TEST_FAILED;
+			goto exit;
+		}
+
+		/* Compress with Zlib, decompress with compressdev */
+		test_data.zlib_dir = ZLIB_COMPRESS;
+		if (test_deflate_comp_decomp(&int_data, &test_data) < 0) {
+			ret = TEST_FAILED;
+			goto exit;
+		}
+	}
+
+	ret = TEST_SUCCESS;
+
+exit:
+	rte_free(compress_xform);
+	return ret;
+}
+
 static struct unit_test_suite compressdev_testsuite  = {
 	.suite_name = "compressdev unit test suite",
 	.setup = testsuite_setup,
@@ -2503,6 +2668,8 @@ static struct unit_test_suite compressdev_testsuite  = {
 			test_compressdev_deflate_stateful_decomp_checksum),
 		TEST_CASE_ST(generic_ut_setup, generic_ut_teardown,
 			test_compressdev_external_mbufs),
+		TEST_CASE_ST(generic_ut_setup, generic_ut_teardown,
+			test_compressdev_deflate_stateless_fixed_overflow),
 		TEST_CASES_END() /**< NULL terminate unit test array */
 	}
 };
-- 
2.17.1

Re: [dpdk-dev] [PATCH v3 2/2] test/compress: unit test for stateless overflow recovery

[Trahe, Fiona]

Hi Artur,

> -----Original Message-----
> From: Trybula, ArturX
> Sent: Friday, October 11, 2019 6:46 PM
> To: dev@dpdk.org; Trahe, Fiona <fiona.trahe@intel.com>; shallyv@marvell.com; Dybkowski, AdamX
> <adamx.dybkowski@intel.com>; Trybula, ArturX <arturx.trybula@intel.com>; akhil.goyal@nxp.com
> Subject: [PATCH v3 2/2] test/compress: unit test for stateless overflow recovery
> 
> Added unit test to check out-of-space recoverable feature.
> 
> Signed-off-by: Artur Trybula <arturx.trybula@intel.com>
> ---
>  app/test/test_compressdev.c | 261 +++++++++++++++++++++++++++++-------
>  1 file changed, 214 insertions(+), 47 deletions(-)


//snip//

> @@ -987,40 +1058,41 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
>  				ops[i]->private_xform = priv_xforms[i];
>  		}
> 
> -		/* Enqueue and dequeue all operations */
> -		num_enqd = rte_compressdev_enqueue_burst(0, 0, ops, num_bufs);
> -		if (num_enqd < num_bufs) {
> +recovery_lb:
> +		ret = test_run_enqueue_dequeue(ops, num_bufs, ops_processed);
> +		if (ret < 0) {
>  			RTE_LOG(ERR, USER1,
> -				"The operations could not be enqueued\n");
> +				"Enqueue/dequeue operation failed\n");
>  			goto exit;
>  		}
> 
> -		num_total_deqd = 0;
> -		do {
> -			/*
> -			 * If retrying a dequeue call, wait for 10 ms to allow
> -			 * enough time to the driver to process the operations
> -			 */
> -			if (deqd_retries != 0) {
> -				/*
> -				 * Avoid infinite loop if not all the
> -				 * operations get out of the device
> -				 */
> -				if (deqd_retries == MAX_DEQD_RETRIES) {
> +		for (i = 0; i < num_bufs; i++) {
> +			compressed_data_size[i] += ops_processed[i]->produced;
> +
> +			if (ops_processed[i]->status ==
> +				RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE) {
> +
> +				ops[i]->status =
> +					RTE_COMP_OP_STATUS_NOT_PROCESSED;
> +				ops[i]->src.offset +=
> +					ops_processed[i]->consumed;
> +				ops[i]->src.length -=
> +					ops_processed[i]->consumed;
> +				ops[i]->dst.offset +=
> +					ops_processed[i]->produced;
> +
> +				buf_ptr = rte_pktmbuf_append(
> +					ops[i]->m_dst,
> +					ops_processed[i]->produced);
> +
> +				if (buf_ptr == NULL) {
>  					RTE_LOG(ERR, USER1,
> -						"Not all operations could be "
> -						"dequeued\n");
> +						"Data recovery: append extra bytes to the current
> mbuf failed\n");
>  					goto exit;
>  				}
> -				usleep(DEQUEUE_WAIT_TIME);
> +				goto recovery_lb;
>  			}
> -			num_deqd = rte_compressdev_dequeue_burst(0, 0,
> -					&ops_processed[num_total_deqd], num_bufs);
> -			num_total_deqd += num_deqd;
> -			deqd_retries++;
> -
> -		} while (num_total_deqd < num_enqd);
> -
> +		}
>  		deqd_retries = 0;
//snip//

Not all drivers support OUT_OF_SPACE_RECOVERABLE - some may return OUT_OF_SPACE_TERMINATED.
There's no capability feature-flag which can be checked before running the test.
How would this test behave in this case?
Ideally it should recognise that OUT_OF_SPACE_TERMINATED is a valid response, but indicates that the
driver doesn't support RECOVERABLE status and so return unsupported rather than failed.

Re: [dpdk-dev] [PATCH v3 2/2] test/compress: unit test for stateless overflow recovery

[Trybula, ArturX]

Hi Fiona,

Please find my answer below,

-----Original Message-----
From: Trahe, Fiona 
Sent: Monday, October 14, 2019 15:49
To: Trybula, ArturX <arturx.trybula@intel.com>; dev@dpdk.org; shallyv@marvell.com; Dybkowski, AdamX <adamx.dybkowski@intel.com>; akhil.goyal@nxp.com
Cc: Trahe, Fiona <fiona.trahe@intel.com>
Subject: RE: [PATCH v3 2/2] test/compress: unit test for stateless overflow recovery

Hi Artur,

> -----Original Message-----
> From: Trybula, ArturX
> Sent: Friday, October 11, 2019 6:46 PM
> To: dev@dpdk.org; Trahe, Fiona <fiona.trahe@intel.com>; 
> shallyv@marvell.com; Dybkowski, AdamX <adamx.dybkowski@intel.com>; 
> Trybula, ArturX <arturx.trybula@intel.com>; akhil.goyal@nxp.com
> Subject: [PATCH v3 2/2] test/compress: unit test for stateless 
> overflow recovery
> 
> Added unit test to check out-of-space recoverable feature.
> 
> Signed-off-by: Artur Trybula <arturx.trybula@intel.com>
> ---
>  app/test/test_compressdev.c | 261 
> +++++++++++++++++++++++++++++-------
>  1 file changed, 214 insertions(+), 47 deletions(-)


//snip//

> @@ -987,40 +1058,41 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
>  				ops[i]->private_xform = priv_xforms[i];
>  		}
> 
> -		/* Enqueue and dequeue all operations */
> -		num_enqd = rte_compressdev_enqueue_burst(0, 0, ops, num_bufs);
> -		if (num_enqd < num_bufs) {
> +recovery_lb:
> +		ret = test_run_enqueue_dequeue(ops, num_bufs, ops_processed);
> +		if (ret < 0) {
>  			RTE_LOG(ERR, USER1,
> -				"The operations could not be enqueued\n");
> +				"Enqueue/dequeue operation failed\n");
>  			goto exit;
>  		}
> 
> -		num_total_deqd = 0;
> -		do {
> -			/*
> -			 * If retrying a dequeue call, wait for 10 ms to allow
> -			 * enough time to the driver to process the operations
> -			 */
> -			if (deqd_retries != 0) {
> -				/*
> -				 * Avoid infinite loop if not all the
> -				 * operations get out of the device
> -				 */
> -				if (deqd_retries == MAX_DEQD_RETRIES) {
> +		for (i = 0; i < num_bufs; i++) {
> +			compressed_data_size[i] += ops_processed[i]->produced;
> +
> +			if (ops_processed[i]->status ==
> +				RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE) {
> +
> +				ops[i]->status =
> +					RTE_COMP_OP_STATUS_NOT_PROCESSED;
> +				ops[i]->src.offset +=
> +					ops_processed[i]->consumed;
> +				ops[i]->src.length -=
> +					ops_processed[i]->consumed;
> +				ops[i]->dst.offset +=
> +					ops_processed[i]->produced;
> +
> +				buf_ptr = rte_pktmbuf_append(
> +					ops[i]->m_dst,
> +					ops_processed[i]->produced);
> +
> +				if (buf_ptr == NULL) {
>  					RTE_LOG(ERR, USER1,
> -						"Not all operations could be "
> -						"dequeued\n");
> +						"Data recovery: append extra bytes to the current
> mbuf failed\n");
>  					goto exit;
>  				}
> -				usleep(DEQUEUE_WAIT_TIME);
> +				goto recovery_lb;
>  			}
> -			num_deqd = rte_compressdev_dequeue_burst(0, 0,
> -					&ops_processed[num_total_deqd], num_bufs);
> -			num_total_deqd += num_deqd;
> -			deqd_retries++;
> -
> -		} while (num_total_deqd < num_enqd);
> -
> +		}
>  		deqd_retries = 0;
//snip//

Not all drivers support OUT_OF_SPACE_RECOVERABLE - some may return OUT_OF_SPACE_TERMINATED.
There's no capability feature-flag which can be checked before running the test.
How would this test behave in this case?
Ideally it should recognise that OUT_OF_SPACE_TERMINATED is a valid response, but indicates that the driver doesn't support RECOVERABLE status and so return unsupported rather than failed.
[Artur] It can fail if a device doesn't support overflow capability. Tomorrow I will run tests using ISAL. 

[dpdk-dev] [PATCH v4 0/2] Add overflow recovery for stateless compression

[Artur Trybula]

This patch adds out-of-space recoverable feature for stateless
compression scenario to QuickAssist compressdev driver. Unit test
was also created to verify the new feature.

v4:	- if a device doesn't support out-of-space recoverable feature,
	then instead of "failed", "unsupported" result will be
	printed out on the screen

v3:	- rebasing onto the recent master

v2:	- rebasing over the external mbufs

Artur Trybula (2):
  compress/qat: overflow catch for stateless compression
  test/compress: unit test for stateless overflow recovery

 app/test/test_compressdev.c     | 280 ++++++++++++++++++++++++++------
 drivers/compress/qat/qat_comp.c |  28 +++-
 2 files changed, 258 insertions(+), 50 deletions(-)

-- 
2.17.1

[dpdk-dev] [PATCH v4 1/2] compress/qat: overflow catch for stateless compression

[Artur Trybula]

This patch adds out-of-space recoverable feature for stateless
compression scenario to QuickAssist compressdev driver

Signed-off-by: Artur Trybula <arturx.trybula@intel.com>
---
 drivers/compress/qat/qat_comp.c | 28 +++++++++++++++++++++++++---
 1 file changed, 25 insertions(+), 3 deletions(-)

diff --git a/drivers/compress/qat/qat_comp.c b/drivers/compress/qat/qat_comp.c
index 8717b7432..533e34f6b 100644
--- a/drivers/compress/qat/qat_comp.c
+++ b/drivers/compress/qat/qat_comp.c
@@ -324,10 +324,32 @@ qat_comp_process_response(void **op, uint8_t *resp, void *op_cookie,
 		    (!cmp_err_code && xlat_err_code == ERR_CODE_OVERFLOW_ERROR)
 				||
 		    (cmp_err_code == ERR_CODE_OVERFLOW_ERROR &&
-		     xlat_err_code == ERR_CODE_OVERFLOW_ERROR))
-			rx_op->status =
+		     xlat_err_code == ERR_CODE_OVERFLOW_ERROR)){
+
+			struct icp_qat_fw_resp_comp_pars *comp_resp =
+	  (struct icp_qat_fw_resp_comp_pars *)&resp_msg->comp_resp_pars;
+
+			/* handle recoverable out-of-buffer condition */
+			/* in stateless compression scenario */
+			if (comp_resp->input_byte_counter) {
+				if ((qat_xform->qat_comp_request_type
+				== QAT_COMP_REQUEST_FIXED_COMP_STATELESS) ||
+				    (qat_xform->qat_comp_request_type
+				== QAT_COMP_REQUEST_DYNAMIC_COMP_STATELESS)) {
+
+					rx_op->status =
+				RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE;
+					rx_op->consumed =
+						comp_resp->input_byte_counter;
+					rx_op->produced =
+						comp_resp->output_byte_counter;
+				} else
+					rx_op->status =
+				RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
+			} else
+				rx_op->status =
 				RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
-		else
+		} else
 			rx_op->status = RTE_COMP_OP_STATUS_ERROR;
 
 		++(*dequeue_err_count);
-- 
2.17.1

[dpdk-dev] [PATCH v4 2/2] test/compress: unit test for stateless overflow recovery

[Artur Trybula]

Added unit test to check out-of-space recoverable feature.

Signed-off-by: Artur Trybula <arturx.trybula@intel.com>
---
 app/test/test_compressdev.c | 280 ++++++++++++++++++++++++++++++------
 1 file changed, 233 insertions(+), 47 deletions(-)

diff --git a/app/test/test_compressdev.c b/app/test/test_compressdev.c
index d470c34b9..283c64971 100644
--- a/app/test/test_compressdev.c
+++ b/app/test/test_compressdev.c
@@ -30,6 +30,7 @@
  * due to the compress block headers
  */
 #define COMPRESS_BUF_SIZE_RATIO 1.3
+#define COMPRESS_BUF_SIZE_RATIO_OVERFLOW 0.2
 #define NUM_LARGE_MBUFS 16
 #define SMALL_SEG_SIZE 256
 #define MAX_SEGS 16
@@ -72,6 +73,11 @@ enum varied_buff {
 	LB_TO_SGL	/* output buffer is chained */
 };
 
+enum overflow_test {
+	OVERFLOW_DISABLED,
+	OVERFLOW_ENABLED
+};
+
 struct priv_op_data {
 	uint16_t orig_idx;
 };
@@ -110,6 +116,8 @@ struct test_data_params {
 	const struct rte_memzone *inbuf_memzone;
 	const struct rte_memzone *compbuf_memzone;
 	const struct rte_memzone *uncompbuf_memzone;
+	/* overflow test activation */
+	enum overflow_test overflow;
 };
 
 static struct comp_testsuite_params testsuite_params = { 0 };
@@ -725,6 +733,49 @@ extbuf_free_callback(void *addr __rte_unused, void *opaque __rte_unused)
 {
 }
 
+static int
+test_run_enqueue_dequeue(struct rte_comp_op **ops, unsigned int num_bufs,
+		  struct rte_comp_op **ops_processed)
+{
+	uint16_t num_enqd, num_deqd, num_total_deqd;
+	unsigned int deqd_retries = 0;
+
+	/* Enqueue and dequeue all operations */
+	num_enqd = rte_compressdev_enqueue_burst(0, 0, ops, num_bufs);
+	if (num_enqd < num_bufs) {
+		RTE_LOG(ERR, USER1,
+			"Some operations could not be enqueued\n");
+		return -1;
+	}
+
+	num_total_deqd = 0;
+	do {
+		/*
+		 * If retrying a dequeue call, wait for 10 ms to allow
+		 * enough time to the driver to process the operations
+		 */
+		if (deqd_retries != 0) {
+			/*
+			 * Avoid infinite loop if not all the
+			 * operations get out of the device
+			 */
+			if (deqd_retries == MAX_DEQD_RETRIES) {
+				RTE_LOG(ERR, USER1,
+					"Not all operations could be dequeued\n");
+				return -1;
+			}
+			usleep(DEQUEUE_WAIT_TIME);
+		}
+		num_deqd = rte_compressdev_dequeue_burst(0, 0,
+				&ops_processed[num_total_deqd], num_bufs);
+		num_total_deqd += num_deqd;
+		deqd_retries++;
+
+	} while (num_total_deqd < num_enqd);
+
+	return 0;
+}
+
 /*
  * Compresses and decompresses buffer with compressdev API and Zlib API
  */
@@ -745,6 +796,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 	unsigned int out_of_space = test_data->out_of_space;
 	unsigned int big_data = test_data->big_data;
 	enum zlib_direction zlib_dir = test_data->zlib_dir;
+	enum overflow_test overflow_tst = test_data->overflow;
 	int ret_status = TEST_FAILED;
 	struct rte_mbuf_ext_shared_info inbuf_info;
 	struct rte_mbuf_ext_shared_info compbuf_info;
@@ -773,6 +825,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 		rte_compressdev_capability_get(0, RTE_COMP_ALGO_DEFLATE);
 	char *contig_buf = NULL;
 	uint64_t compress_checksum[num_bufs];
+	uint32_t compressed_data_size[num_bufs];
 	void *stream = NULL;
 	char *all_decomp_data = NULL;
 	unsigned int decomp_produced_data_size = 0;
@@ -793,6 +846,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 	memset(ops, 0, sizeof(struct rte_comp_op *) * num_bufs);
 	memset(ops_processed, 0, sizeof(struct rte_comp_op *) * num_bufs);
 	memset(priv_xforms, 0, sizeof(void *) * num_bufs);
+	memset(compressed_data_size, 0, sizeof(uint32_t) * num_bufs);
 
 	if (decompress_state == RTE_COMP_OP_STATEFUL) {
 		data_size = strlen(test_bufs[0]) + 1;
@@ -845,6 +899,11 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 		for (i = 0; i < num_bufs; i++) {
 			data_size = strlen(test_bufs[i]) + 1;
 			buf_ptr = rte_pktmbuf_append(uncomp_bufs[i], data_size);
+			if (buf_ptr == NULL) {
+				RTE_LOG(ERR, USER1,
+					"Append extra bytes to the source mbuf failed\n");
+				goto exit;
+			}
 			strlcpy(buf_ptr, test_bufs[i], data_size);
 		}
 	}
@@ -893,11 +952,22 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 		for (i = 0; i < num_bufs; i++) {
 			if (out_of_space == 1 && oos_zlib_decompress)
 				data_size = OUT_OF_SPACE_BUF;
-			else
-				(data_size = strlen(test_bufs[i]) *
-					COMPRESS_BUF_SIZE_RATIO);
-
-			rte_pktmbuf_append(comp_bufs[i], data_size);
+			else {
+				float ratio =
+				((test_data->zlib_dir == ZLIB_DECOMPRESS ||
+				   test_data->zlib_dir == ZLIB_NONE) &&
+				  overflow_tst == OVERFLOW_ENABLED) ?
+					 COMPRESS_BUF_SIZE_RATIO_OVERFLOW :
+					 COMPRESS_BUF_SIZE_RATIO;
+
+				data_size = strlen(test_bufs[i]) * ratio;
+			}
+			buf_ptr = rte_pktmbuf_append(comp_bufs[i], data_size);
+			if (buf_ptr == NULL) {
+				RTE_LOG(ERR, USER1,
+					"Append extra bytes to the destination mbuf failed\n");
+				goto exit;
+			}
 		}
 	}
 
@@ -963,7 +1033,8 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 			num_priv_xforms++;
 		}
 
-		if (capa->comp_feature_flags & RTE_COMP_FF_SHAREABLE_PRIV_XFORM) {
+		if (capa->comp_feature_flags &
+				RTE_COMP_FF_SHAREABLE_PRIV_XFORM) {
 			/* Attach shareable private xform data to ops */
 			for (i = 0; i < num_bufs; i++)
 				ops[i]->private_xform = priv_xforms[i % num_xforms];
@@ -987,40 +1058,41 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 				ops[i]->private_xform = priv_xforms[i];
 		}
 
-		/* Enqueue and dequeue all operations */
-		num_enqd = rte_compressdev_enqueue_burst(0, 0, ops, num_bufs);
-		if (num_enqd < num_bufs) {
+recovery_lb:
+		ret = test_run_enqueue_dequeue(ops, num_bufs, ops_processed);
+		if (ret < 0) {
 			RTE_LOG(ERR, USER1,
-				"The operations could not be enqueued\n");
+				"Enqueue/dequeue operation failed\n");
 			goto exit;
 		}
 
-		num_total_deqd = 0;
-		do {
-			/*
-			 * If retrying a dequeue call, wait for 10 ms to allow
-			 * enough time to the driver to process the operations
-			 */
-			if (deqd_retries != 0) {
-				/*
-				 * Avoid infinite loop if not all the
-				 * operations get out of the device
-				 */
-				if (deqd_retries == MAX_DEQD_RETRIES) {
+		for (i = 0; i < num_bufs; i++) {
+			compressed_data_size[i] += ops_processed[i]->produced;
+
+			if (ops_processed[i]->status ==
+				RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE) {
+
+				ops[i]->status =
+					RTE_COMP_OP_STATUS_NOT_PROCESSED;
+				ops[i]->src.offset +=
+					ops_processed[i]->consumed;
+				ops[i]->src.length -=
+					ops_processed[i]->consumed;
+				ops[i]->dst.offset +=
+					ops_processed[i]->produced;
+
+				buf_ptr = rte_pktmbuf_append(
+					ops[i]->m_dst,
+					ops_processed[i]->produced);
+
+				if (buf_ptr == NULL) {
 					RTE_LOG(ERR, USER1,
-						"Not all operations could be "
-						"dequeued\n");
+						"Data recovery: append extra bytes to the current mbuf failed\n");
 					goto exit;
 				}
-				usleep(DEQUEUE_WAIT_TIME);
+				goto recovery_lb;
 			}
-			num_deqd = rte_compressdev_dequeue_burst(0, 0,
-					&ops_processed[num_total_deqd], num_bufs);
-			num_total_deqd += num_deqd;
-			deqd_retries++;
-
-		} while (num_total_deqd < num_enqd);
-
+		}
 		deqd_retries = 0;
 
 		/* Free compress private xforms */
@@ -1064,7 +1136,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 	for (i = 0; i < num_bufs; i++) {
 		if (out_of_space && oos_zlib_decompress) {
 			if (ops_processed[i]->status !=
-					RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED) {
+				RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED) {
 				ret_status = TEST_FAILED;
 				RTE_LOG(ERR, USER1,
 					"Operation without expected out of "
@@ -1075,6 +1147,16 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 		}
 
 		if (ops_processed[i]->status != RTE_COMP_OP_STATUS_SUCCESS) {
+			if (overflow_tst == OVERFLOW_ENABLED) {
+				if (ops_processed[i]->status ==
+				RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED) {
+					ret_status = 1;
+					RTE_LOG(INFO, USER1,
+					"Out-of-space-recoverable functionality"
+					" is not supported on this device\n");
+					goto exit;
+				}
+			}
 			RTE_LOG(ERR, USER1,
 				"Some operations were not successful\n");
 			goto exit;
@@ -1147,7 +1229,12 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 				data_size =
 				strlen(test_bufs[priv_data->orig_idx]) + 1;
 
-			rte_pktmbuf_append(uncomp_bufs[i], data_size);
+			buf_ptr = rte_pktmbuf_append(uncomp_bufs[i], data_size);
+			if (buf_ptr == NULL) {
+				RTE_LOG(ERR, USER1,
+					"Append extra bytes to the decompressed mbuf failed\n");
+				goto exit;
+			}
 		}
 	}
 
@@ -1169,7 +1256,10 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 		 * Set the length of the compressed data to the
 		 * number of bytes that were produced in the previous stage
 		 */
-		ops[i]->src.length = ops_processed[i]->produced;
+		if (compressed_data_size[i])
+			ops[i]->src.length = compressed_data_size[i];
+		else
+			ops[i]->src.length = ops_processed[i]->produced;
 
 		ops[i]->dst.offset = 0;
 		if (decompress_state == RTE_COMP_OP_STATELESS) {
@@ -1541,7 +1631,8 @@ test_compressdev_deflate_stateless_fixed(void)
 		.buff_type = LB_BOTH,
 		.zlib_dir = ZLIB_DECOMPRESS,
 		.out_of_space = 0,
-		.big_data = 0
+		.big_data = 0,
+		.overflow = OVERFLOW_DISABLED
 	};
 
 	for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
@@ -1611,7 +1702,8 @@ test_compressdev_deflate_stateless_dynamic(void)
 		.buff_type = LB_BOTH,
 		.zlib_dir = ZLIB_DECOMPRESS,
 		.out_of_space = 0,
-		.big_data = 0
+		.big_data = 0,
+		.overflow = OVERFLOW_DISABLED
 	};
 
 	for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
@@ -1665,7 +1757,8 @@ test_compressdev_deflate_stateless_multi_op(void)
 		.buff_type = LB_BOTH,
 		.zlib_dir = ZLIB_DECOMPRESS,
 		.out_of_space = 0,
-		.big_data = 0
+		.big_data = 0,
+		.overflow = OVERFLOW_DISABLED
 	};
 
 	/* Compress with compressdev, decompress with Zlib */
@@ -1718,7 +1811,8 @@ test_compressdev_deflate_stateless_multi_level(void)
 		.buff_type = LB_BOTH,
 		.zlib_dir = ZLIB_DECOMPRESS,
 		.out_of_space = 0,
-		.big_data = 0
+		.big_data = 0,
+		.overflow = OVERFLOW_DISABLED
 	};
 
 	for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
@@ -1807,7 +1901,8 @@ test_compressdev_deflate_stateless_multi_xform(void)
 		.buff_type = LB_BOTH,
 		.zlib_dir = ZLIB_DECOMPRESS,
 		.out_of_space = 0,
-		.big_data = 0
+		.big_data = 0,
+		.overflow = OVERFLOW_DISABLED
 	};
 
 	/* Compress with compressdev, decompress with Zlib */
@@ -1855,7 +1950,8 @@ test_compressdev_deflate_stateless_sgl(void)
 		.buff_type = SGL_BOTH,
 		.zlib_dir = ZLIB_DECOMPRESS,
 		.out_of_space = 0,
-		.big_data = 0
+		.big_data = 0,
+		.overflow = OVERFLOW_DISABLED
 	};
 
 	for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
@@ -1965,7 +2061,8 @@ test_compressdev_deflate_stateless_checksum(void)
 		.buff_type = LB_BOTH,
 		.zlib_dir = ZLIB_DECOMPRESS,
 		.out_of_space = 0,
-		.big_data = 0
+		.big_data = 0,
+		.overflow = OVERFLOW_DISABLED
 	};
 
 	/* Check if driver supports crc32 checksum and test */
@@ -2082,7 +2179,8 @@ test_compressdev_out_of_space_buffer(void)
 		.buff_type = LB_BOTH,
 		.zlib_dir = ZLIB_DECOMPRESS,
 		.out_of_space = 1,  /* run out-of-space test */
-		.big_data = 0
+		.big_data = 0,
+		.overflow = OVERFLOW_DISABLED
 	};
 	/* Compress with compressdev, decompress with Zlib */
 	test_data.zlib_dir = ZLIB_DECOMPRESS;
@@ -2159,7 +2257,8 @@ test_compressdev_deflate_stateless_dynamic_big(void)
 		.buff_type = SGL_BOTH,
 		.zlib_dir = ZLIB_DECOMPRESS,
 		.out_of_space = 0,
-		.big_data = 1
+		.big_data = 1,
+		.overflow = OVERFLOW_DISABLED
 	};
 
 	ts_params->def_comp_xform->compress.deflate.huffman =
@@ -2223,7 +2322,8 @@ test_compressdev_deflate_stateful_decomp(void)
 		.out_of_space = 0,
 		.big_data = 0,
 		.decompress_output_block_size = 2000,
-		.decompress_steps_max = 4
+		.decompress_steps_max = 4,
+		.overflow = OVERFLOW_DISABLED
 	};
 
 	/* Compress with Zlib, decompress with compressdev */
@@ -2305,7 +2405,8 @@ test_compressdev_deflate_stateful_decomp_checksum(void)
 		.out_of_space = 0,
 		.big_data = 0,
 		.decompress_output_block_size = 2000,
-		.decompress_steps_max = 4
+		.decompress_steps_max = 4,
+		.overflow = OVERFLOW_DISABLED
 	};
 
 	/* Check if driver supports crc32 checksum and test */
@@ -2441,7 +2542,8 @@ test_compressdev_external_mbufs(void)
 		.inbuf_memzone = make_memzone("inbuf", data_len),
 		.compbuf_memzone = make_memzone("compbuf", data_len *
 						COMPRESS_BUF_SIZE_RATIO),
-		.uncompbuf_memzone = make_memzone("decompbuf", data_len)
+		.uncompbuf_memzone = make_memzone("decompbuf", data_len),
+		.overflow = OVERFLOW_DISABLED
 	};
 
 	for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
@@ -2472,6 +2574,88 @@ test_compressdev_external_mbufs(void)
 	return ret;
 }
 
+static int
+test_compressdev_deflate_stateless_fixed_oos_recoverable(void)
+{
+	struct comp_testsuite_params *ts_params = &testsuite_params;
+	uint16_t i;
+	int ret;
+	int comp_result;
+	const struct rte_compressdev_capabilities *capab;
+
+	capab = rte_compressdev_capability_get(0, RTE_COMP_ALGO_DEFLATE);
+	TEST_ASSERT(capab != NULL, "Failed to retrieve device capabilities");
+
+	if ((capab->comp_feature_flags & RTE_COMP_FF_HUFFMAN_FIXED) == 0)
+		return -ENOTSUP;
+
+	struct rte_comp_xform *compress_xform =
+			rte_malloc(NULL, sizeof(struct rte_comp_xform), 0);
+
+	if (compress_xform == NULL) {
+		RTE_LOG(ERR, USER1,
+			"Compress xform could not be created\n");
+		ret = TEST_FAILED;
+		goto exit;
+	}
+
+	memcpy(compress_xform, ts_params->def_comp_xform,
+			sizeof(struct rte_comp_xform));
+	compress_xform->compress.deflate.huffman = RTE_COMP_HUFFMAN_FIXED;
+
+	struct interim_data_params int_data = {
+		NULL,
+		1,
+		NULL,
+		&compress_xform,
+		&ts_params->def_decomp_xform,
+		1
+	};
+
+	struct test_data_params test_data = {
+		.compress_state = RTE_COMP_OP_STATELESS,
+		.decompress_state = RTE_COMP_OP_STATELESS,
+		.buff_type = LB_BOTH,
+		.zlib_dir = ZLIB_DECOMPRESS,
+		.out_of_space = 0,
+		.big_data = 0,
+		.overflow = OVERFLOW_ENABLED
+	};
+
+	for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
+		int_data.test_bufs = &compress_test_bufs[i];
+		int_data.buf_idx = &i;
+
+		/* Compress with compressdev, decompress with Zlib */
+		test_data.zlib_dir = ZLIB_DECOMPRESS;
+		comp_result = test_deflate_comp_decomp(&int_data, &test_data);
+		if (comp_result < 0) {
+			ret = TEST_FAILED;
+			goto exit;
+		} else if (comp_result > 0) {
+			ret = -ENOTSUP;
+			goto exit;
+		}
+
+		/* Compress with Zlib, decompress with compressdev */
+		test_data.zlib_dir = ZLIB_COMPRESS;
+		comp_result = test_deflate_comp_decomp(&int_data, &test_data);
+		if (comp_result < 0) {
+			ret = TEST_FAILED;
+			goto exit;
+		} else if (comp_result > 0) {
+			ret = -ENOTSUP;
+			goto exit;
+		}
+	}
+
+	ret = TEST_SUCCESS;
+
+exit:
+	rte_free(compress_xform);
+	return ret;
+}
+
 static struct unit_test_suite compressdev_testsuite  = {
 	.suite_name = "compressdev unit test suite",
 	.setup = testsuite_setup,
@@ -2503,6 +2687,8 @@ static struct unit_test_suite compressdev_testsuite  = {
 			test_compressdev_deflate_stateful_decomp_checksum),
 		TEST_CASE_ST(generic_ut_setup, generic_ut_teardown,
 			test_compressdev_external_mbufs),
+		TEST_CASE_ST(generic_ut_setup, generic_ut_teardown,
+		test_compressdev_deflate_stateless_fixed_oos_recoverable),
 		TEST_CASES_END() /**< NULL terminate unit test array */
 	}
 };
-- 
2.17.1

Re: [dpdk-dev] [PATCH v4 0/2] Add overflow recovery for stateless compression

[Trahe, Fiona]

> -----Original Message-----
> From: Trybula, ArturX
> Sent: Wednesday, October 16, 2019 10:41 AM
> To: dev@dpdk.org; Trahe, Fiona <fiona.trahe@intel.com>; shallyv@marvell.com; Dybkowski, AdamX
> <adamx.dybkowski@intel.com>; Trybula, ArturX <arturx.trybula@intel.com>; akhil.goyal@nxp.com
> Subject: [PATCH v4 0/2] Add overflow recovery for stateless compression
> 
> This patch adds out-of-space recoverable feature for stateless
> compression scenario to QuickAssist compressdev driver. Unit test
> was also created to verify the new feature.
> 
> v4:	- if a device doesn't support out-of-space recoverable feature,
> 	then instead of "failed", "unsupported" result will be
> 	printed out on the screen
> 
> v3:	- rebasing onto the recent master
> 
> v2:	- rebasing over the external mbufs
> 
> Artur Trybula (2):
>   compress/qat: overflow catch for stateless compression
>   test/compress: unit test for stateless overflow recovery
> 
>  app/test/test_compressdev.c     | 280 ++++++++++++++++++++++++++------
>  drivers/compress/qat/qat_comp.c |  28 +++-
>  2 files changed, 258 insertions(+), 50 deletions(-)
> 
> --
> 2.17.1
Series 
Acked-by: Fiona Trahe <fiona.trahe@intel.com>

Re: [dpdk-dev] [PATCH v4 0/2] Add overflow recovery for stateless compression

[Akhil Goyal]

> 
> > -----Original Message-----
> > From: Trybula, ArturX
> > Sent: Wednesday, October 16, 2019 10:41 AM
> > To: dev@dpdk.org; Trahe, Fiona <fiona.trahe@intel.com>;
> shallyv@marvell.com; Dybkowski, AdamX
> > <adamx.dybkowski@intel.com>; Trybula, ArturX <arturx.trybula@intel.com>;
> akhil.goyal@nxp.com
> > Subject: [PATCH v4 0/2] Add overflow recovery for stateless compression
> >
> > This patch adds out-of-space recoverable feature for stateless
> > compression scenario to QuickAssist compressdev driver. Unit test
> > was also created to verify the new feature.
> >
> > v4:	- if a device doesn't support out-of-space recoverable feature,
> > 	then instead of "failed", "unsupported" result will be
> > 	printed out on the screen
> >
> > v3:	- rebasing onto the recent master
> >
> > v2:	- rebasing over the external mbufs
> >
> > Artur Trybula (2):
> >   compress/qat: overflow catch for stateless compression
> >   test/compress: unit test for stateless overflow recovery
> >
> >  app/test/test_compressdev.c     | 280 ++++++++++++++++++++++++++------
> >  drivers/compress/qat/qat_comp.c |  28 +++-
> >  2 files changed, 258 insertions(+), 50 deletions(-)
> >
> > --
> > 2.17.1
> Series
> Acked-by: Fiona Trahe <fiona.trahe@intel.com>
Applied to dpdk-next-crypto

Thanks.