DPDK patches and discussions
 help / color / mirror / Atom feed
* [dpdk-dev] [PATCH] test/compress: unit tests refactoring
@ 2019-09-09 14:45 Artur Trybula
  2019-09-12 14:34 ` [dpdk-dev] [PATCH v2 0/1] compression: " Artur Trybula
  0 siblings, 1 reply; 21+ messages in thread
From: Artur Trybula @ 2019-09-09 14:45 UTC (permalink / raw)
  To: dev, fiona.trahe, shallyv, adamx.dybkowski, arturx.trybula, akhil.goyal

Core engine refactoring (test_deflate_comp_decomp function).
Smaller specialized functions created.

Signed-off-by: Artur Trybula <arturx.trybula@intel.com>
---
This patch depends on "test/compress: unit test for stateless
overflow recovery": http://patches.dpdk.org/patch/58834/

 app/test/test_compressdev.c | 833 +++++++++++++++++++++++++++---------
 1 file changed, 630 insertions(+), 203 deletions(-)

diff --git a/app/test/test_compressdev.c b/app/test/test_compressdev.c
index 3a655164f..2bb164a37 100644
--- a/app/test/test_compressdev.c
+++ b/app/test/test_compressdev.c
@@ -110,6 +110,16 @@ struct test_data_params {
 	enum overflow_test overflow;
 };
 
+struct test_private_arrays {
+	struct rte_mbuf **uncomp_bufs;
+	struct rte_mbuf **comp_bufs;
+	struct rte_comp_op **ops;
+	struct rte_comp_op **ops_processed;
+	void **priv_xforms;
+	uint64_t *compress_checksum;
+	uint32_t *compressed_data_size;
+};
+
 static struct comp_testsuite_params testsuite_params = { 0 };
 
 static void
@@ -721,8 +731,9 @@ prepare_sgl_bufs(const char *test_buf, struct rte_mbuf *head_buf,
 
 
 static int
-test_run_enqueue_dequeue(struct rte_comp_op **ops, unsigned int num_bufs,
-		  struct rte_comp_op **ops_processed)
+test_run_enqueue_dequeue(struct rte_comp_op **ops,
+			 struct rte_comp_op **ops_processed,
+			 unsigned int num_bufs)
 {
 	uint16_t num_enqd, num_deqd, num_total_deqd;
 	unsigned int deqd_retries = 0;
@@ -763,66 +774,70 @@ test_run_enqueue_dequeue(struct rte_comp_op **ops, unsigned int num_bufs,
 	return 0;
 }
 
-/*
- * Compresses and decompresses buffer with compressdev API and Zlib API
+/**
+ * Initialize all the arrays used in comp/decomp to NULL.
+ *
+ * If the length is greater than the length of the last segment, the
+ * function will fail and return -1 without modifying the mbuf.
+ *
+ * @param priv_arrays
+ *   A container used for aggregation all the arrays.
+ * @param num_bufs
+ *   The number of elements in each array.
+ */
+static void
+test_objects_init(struct test_private_arrays *prv,
+		unsigned int num_bufs)
+{
+	/* Initialize all arrays to NULL */
+	memset(prv->uncomp_bufs, 0, sizeof(struct rte_mbuf *) * num_bufs);
+	memset(prv->comp_bufs, 0, sizeof(struct rte_mbuf *) * num_bufs);
+	memset(prv->ops, 0, sizeof(struct rte_comp_op *) * num_bufs);
+	memset(prv->ops_processed, 0, sizeof(struct rte_comp_op *) * num_bufs);
+	memset(prv->priv_xforms, 0, sizeof(void *) * num_bufs);
+	memset(prv->compressed_data_size, 0, sizeof(uint32_t) * num_bufs);
+}
+
+/**
+ * Source buffers preparation (for compression).
+ *
+ * Memory allocation for each buffer from mempool
+ * -1 returned if function fail, without modifying the mbuf.
+ *
+ * @param int_data
+ *   Interim data containing session/transformation objects.
+ * @param test_data
+ *   The test parameters set by users (command line parameters).
+ * @param priv_arrays
+ *   A container used for aggregation all the private test arrays.
+ * @return
+ *   - 0: On success.
+ *   - -1: On error.
  */
 static int
-test_deflate_comp_decomp(const struct interim_data_params *int_data,
-		const struct test_data_params *test_data)
+test_mbufs_source_preparation(const struct interim_data_params *int_data,
+		const struct test_data_params *test_data,
+		const struct test_private_arrays *priv_arrays)
 {
+	/* local variables: */
+	unsigned int i;
+	uint32_t data_size;
+	char *buf_ptr;
+	int ret;
+
 	struct comp_testsuite_params *ts_params = &testsuite_params;
+
+	/* from int_data: */
 	const char * const *test_bufs = int_data->test_bufs;
 	unsigned int num_bufs = int_data->num_bufs;
-	uint16_t *buf_idx = int_data->buf_idx;
-	struct rte_comp_xform **compress_xforms = int_data->compress_xforms;
-	struct rte_comp_xform **decompress_xforms = int_data->decompress_xforms;
-	unsigned int num_xforms = int_data->num_xforms;
-	enum rte_comp_op_type state = test_data->state;
+
+	/* from test_data: */
 	unsigned int buff_type = test_data->buff_type;
-	unsigned int out_of_space = test_data->out_of_space;
 	unsigned int big_data = test_data->big_data;
-	enum zlib_direction zlib_dir = test_data->zlib_dir;
-	enum overflow_test overflow_tst = test_data->overflow;
-	int ret_status = -1;
-	int ret;
-	struct rte_mbuf *uncomp_bufs[num_bufs];
-	struct rte_mbuf *comp_bufs[num_bufs];
-	struct rte_comp_op *ops[num_bufs];
-	struct rte_comp_op *ops_processed[num_bufs];
-	void *priv_xforms[num_bufs];
-	uint16_t num_enqd, num_deqd, num_total_deqd;
-	uint16_t num_priv_xforms = 0;
-	unsigned int deqd_retries = 0;
-	struct priv_op_data *priv_data;
-	char *buf_ptr;
-	unsigned int i;
-	struct rte_mempool *buf_pool;
-	uint32_t data_size;
-	/* Compressing with CompressDev */
-	unsigned int oos_zlib_decompress =
-			(zlib_dir == ZLIB_NONE || zlib_dir == ZLIB_DECOMPRESS);
-	/* Decompressing with CompressDev */
-	unsigned int oos_zlib_compress =
-			(zlib_dir == ZLIB_NONE || zlib_dir == ZLIB_COMPRESS);
-	const struct rte_compressdev_capabilities *capa =
-		rte_compressdev_capability_get(0, RTE_COMP_ALGO_DEFLATE);
-	char *contig_buf = NULL;
-	uint64_t compress_checksum[num_bufs];
-	uint32_t compressed_data_size[num_bufs];
 
-	if (capa == NULL) {
-		RTE_LOG(ERR, USER1,
-			"Compress device does not support DEFLATE\n");
-		return -1;
-	}
-
-	/* Initialize all arrays to NULL */
-	memset(uncomp_bufs, 0, sizeof(struct rte_mbuf *) * num_bufs);
-	memset(comp_bufs, 0, sizeof(struct rte_mbuf *) * num_bufs);
-	memset(ops, 0, sizeof(struct rte_comp_op *) * num_bufs);
-	memset(ops_processed, 0, sizeof(struct rte_comp_op *) * num_bufs);
-	memset(priv_xforms, 0, sizeof(void *) * num_bufs);
-	memset(compressed_data_size, 0, sizeof(uint32_t) * num_bufs);
+	/* from priv_arrays: */
+	struct rte_mbuf **uncomp_bufs = priv_arrays->uncomp_bufs;
+	struct rte_mempool *buf_pool;
 
 	if (big_data)
 		buf_pool = ts_params->big_mbuf_pool;
@@ -831,14 +846,15 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 	else
 		buf_pool = ts_params->large_mbuf_pool;
 
-	/* Prepare the source mbufs with the data */
+	/* for compression uncomp_bufs is used as a source buffer */
+	/* allocation from buf_pool (mempool type) */
 	ret = rte_pktmbuf_alloc_bulk(buf_pool,
 				uncomp_bufs, num_bufs);
 	if (ret < 0) {
 		RTE_LOG(ERR, USER1,
 			"Source mbufs could not be allocated "
 			"from the mempool\n");
-		goto exit;
+		return -1;
 	}
 
 	if (buff_type == SGL_BOTH || buff_type == SGL_TO_LB) {
@@ -850,7 +866,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 			    big_data ? buf_pool : ts_params->large_mbuf_pool,
 			    big_data ? 0 : MAX_SEGS,
 			    big_data ? MAX_DATA_MBUF_SIZE : SMALL_SEG_SIZE) < 0)
-				goto exit;
+				return -1;
 		}
 	} else {
 		for (i = 0; i < num_bufs; i++) {
@@ -860,68 +876,194 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 			if (buf_ptr == NULL) {
 				RTE_LOG(ERR, USER1,
 					"Append extra bytes to the source mbuf failed\n");
-				goto exit;
+				return -1;
 			}
 			strlcpy(buf_ptr, test_bufs[i], data_size);
 		}
 	}
 
-	/* Prepare the destination mbufs */
-	ret = rte_pktmbuf_alloc_bulk(buf_pool, comp_bufs, num_bufs);
+	return 0;
+}
+
+/**
+ * Memory buffers preparation (for both compression and decompression).
+ *
+ * Memory allocation for comp/decomp buffers from mempool, depending on
+ * ops_processed value. Developer is requested to provide input params
+ * according to the following rule:
+ *      if ops_processed == NULL -> current_bufs = comp_bufs[]
+ *      if ops_processed != NULL -> current_bufs = decomp_bufs[]
+ * -1 returned if function fail, without modifying the mbuf.
+ *
+ * @param ops_processed
+ *   Operations created as a result of compression phase. Should be
+ *   set to NULL for compression
+ * @param current_bufs
+ *   mbufs being prepared in the function.
+ * @param out_of_space_and_zlib
+ *   Boolean value to switch into "out of space" buffer if set.
+ * @param int_data
+ *   Interim data containing session/transformation objects.
+ * @param test_data
+ *   The test parameters set by users (command line parameters).
+ * @return
+ *   - 0: On success.
+ *   - -1: On error.
+ */
+static int
+test_mbufs_destination_preparation(
+		struct rte_comp_op *ops_processed[], /*can be equal to NULL*/
+		struct rte_mbuf *current_bufs[],
+		unsigned int out_of_space_and_zlib,
+		const struct interim_data_params *int_data,
+		const struct test_data_params *test_data)
+{
+	/* local variables: */
+	unsigned int i;
+	uint32_t data_size;
+	struct priv_op_data *priv_data;
+	int ret;
+	char *buf_ptr;
+
+	struct comp_testsuite_params *ts_params = &testsuite_params;
+
+	/* from int_data: */
+	const char * const *test_bufs = int_data->test_bufs;
+	unsigned int num_bufs = int_data->num_bufs;
+
+	/* from test_data: */
+	unsigned int buff_type = test_data->buff_type;
+	unsigned int big_data = test_data->big_data;
+
+	/* from priv_arrays: */
+	struct rte_mempool *buf_pool;
+
+	if (big_data)
+		buf_pool = ts_params->big_mbuf_pool;
+	else if (buff_type == SGL_BOTH)
+		buf_pool = ts_params->small_mbuf_pool;
+	else
+		buf_pool = ts_params->large_mbuf_pool;
+
+	/* the mbufs allocation*/
+	ret = rte_pktmbuf_alloc_bulk(buf_pool, current_bufs, num_bufs);
 	if (ret < 0) {
 		RTE_LOG(ERR, USER1,
 			"Destination mbufs could not be allocated "
 			"from the mempool\n");
-		goto exit;
+		return -1;
 	}
 
-	if (buff_type == SGL_BOTH || buff_type == LB_TO_SGL) {
-		for (i = 0; i < num_bufs; i++) {
-			if (out_of_space == 1 && oos_zlib_decompress)
-				data_size = OUT_OF_SPACE_BUF;
-			else
-				(data_size = strlen(test_bufs[i]) *
-					COMPRESS_BUF_SIZE_RATIO);
-
-			if (prepare_sgl_bufs(NULL, comp_bufs[i],
-			      data_size,
-			      big_data ? buf_pool : ts_params->small_mbuf_pool,
-			      big_data ? buf_pool : ts_params->large_mbuf_pool,
-			      big_data ? 0 : MAX_SEGS,
-			      big_data ? MAX_DATA_MBUF_SIZE : SMALL_SEG_SIZE)
-					< 0)
-				goto exit;
-		}
+	for (i = 0; i < num_bufs; i++) {
 
-	} else {
-		for (i = 0; i < num_bufs; i++) {
-			if (out_of_space == 1 && oos_zlib_decompress)
-				data_size = OUT_OF_SPACE_BUF;
-			else {
+		/* data size calculation */
+		if (out_of_space_and_zlib)
+			data_size = OUT_OF_SPACE_BUF;
+		else {
+			if (ops_processed == NULL) {
 				float ratio =
 				((test_data->zlib_dir == ZLIB_DECOMPRESS ||
-				   test_data->zlib_dir == ZLIB_NONE) &&
-				  overflow_tst == OVERFLOW_ENABLED) ?
+				  test_data->zlib_dir == ZLIB_NONE) &&
+				  test_data->overflow == OVERFLOW_ENABLED) ?
 					 COMPRESS_BUF_SIZE_RATIO_OVERFLOW :
 					 COMPRESS_BUF_SIZE_RATIO;
 
+				printf("ratio %2.2f\n", ratio);
 				data_size = strlen(test_bufs[i]) * ratio;
+
+			} else {
+				priv_data = (struct priv_op_data *)
+					(ops_processed[i] + 1);
+				data_size =
+					strlen(test_bufs[priv_data->orig_idx])
+					+ 1;
 			}
-			buf_ptr = rte_pktmbuf_append(comp_bufs[i], data_size);
+		}
+
+		/* data allocation */
+		if (buff_type == SGL_BOTH || buff_type == LB_TO_SGL) {
+			ret = prepare_sgl_bufs(NULL, current_bufs[i],
+			      data_size,
+			      big_data ? buf_pool : ts_params->small_mbuf_pool,
+			      big_data ? buf_pool : ts_params->large_mbuf_pool,
+			      big_data ? 0 : MAX_SEGS,
+			      big_data ? MAX_DATA_MBUF_SIZE : SMALL_SEG_SIZE);
+			if (ret < 0)
+				return -1;
+		} else {
+			buf_ptr = rte_pktmbuf_append(current_bufs[i],
+					data_size);
 			if (buf_ptr == NULL) {
 				RTE_LOG(ERR, USER1,
 					"Append extra bytes to the destination mbuf failed\n");
-				goto exit;
+				return -1;
 			}
 		}
 	}
 
+	return 0;
+}
+
+/**
+ * The main compression function.
+ *
+ * Operation(s) configuration, depending on CLI parameters.
+ * Operation(s) processing.
+ * -1 returned if function fail.
+ *
+ * @param int_data
+ *   Interim data containing session/transformation objects.
+ * @param test_data
+ *   The test parameters set by users (command line parameters).
+ * @param priv_arrays
+ *   A container used for aggregation all the private test arrays.
+ * @return
+ *   - 0: On success.
+ *   - -1: On error.
+ */
+static int
+test_deflate_comp_run(const struct interim_data_params *int_data,
+		const struct test_data_params *test_data,
+		const struct test_private_arrays *priv_arrays)
+{
+	/* local variables: */
+	struct priv_op_data *priv_data;
+	unsigned int i;
+	uint16_t num_priv_xforms = 0;
+	int ret;
+	int ret_status = 0;
+	char *buf_ptr;
+
+	struct comp_testsuite_params *ts_params = &testsuite_params;
+
+	/* from test_data: */
+	enum rte_comp_op_type operation_type = test_data->state;
+	unsigned int zlib_compress =
+			(test_data->zlib_dir == ZLIB_ALL ||
+			test_data->zlib_dir == ZLIB_COMPRESS);
+
+	/* from int_data: */
+	struct rte_comp_xform **compress_xforms = int_data->compress_xforms;
+	unsigned int num_xforms = int_data->num_xforms;
+	unsigned int num_bufs = int_data->num_bufs;
+
+	/* from priv_arrays: */
+	struct rte_mbuf **comp_bufs = priv_arrays->comp_bufs;
+	struct rte_mbuf **uncomp_bufs = priv_arrays->uncomp_bufs;
+	struct rte_comp_op **ops = priv_arrays->ops;
+	struct rte_comp_op **ops_processed = priv_arrays->ops_processed;
+	void **priv_xforms = priv_arrays->priv_xforms;
+
+	const struct rte_compressdev_capabilities *capa =
+		rte_compressdev_capability_get(0, RTE_COMP_ALGO_DEFLATE);
+
 	/* Build the compression operations */
 	ret = rte_comp_op_bulk_alloc(ts_params->op_pool, ops, num_bufs);
 	if (ret < 0) {
 		RTE_LOG(ERR, USER1,
 			"Compress operations could not be allocated "
 			"from the mempool\n");
+		ret_status = -1;
 		goto exit;
 	}
 
@@ -931,12 +1073,13 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 		ops[i]->src.offset = 0;
 		ops[i]->src.length = rte_pktmbuf_pkt_len(uncomp_bufs[i]);
 		ops[i]->dst.offset = 0;
-		if (state == RTE_COMP_OP_STATELESS) {
+		if (operation_type == RTE_COMP_OP_STATELESS) {
 			ops[i]->flush_flag = RTE_COMP_FLUSH_FINAL;
 		} else {
 			RTE_LOG(ERR, USER1,
 				"Stateful operations are not supported "
 				"in these tests yet\n");
+			ret_status = -1;
 			goto exit;
 		}
 		ops[i]->input_chksum = 0;
@@ -951,14 +1094,16 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 	}
 
 	/* Compress data (either with Zlib API or compressdev API */
-	if (zlib_dir == ZLIB_COMPRESS || zlib_dir == ZLIB_ALL) {
+	if (zlib_compress) {
 		for (i = 0; i < num_bufs; i++) {
 			const struct rte_comp_xform *compress_xform =
 				compress_xforms[i % num_xforms];
 			ret = compress_zlib(ops[i], compress_xform,
 					DEFAULT_MEM_LEVEL);
-			if (ret < 0)
+			if (ret < 0) {
+				ret_status = -1;
 				goto exit;
+			}
 
 			ops_processed[i] = ops[i];
 		}
@@ -972,11 +1117,11 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 				RTE_LOG(ERR, USER1,
 					"Compression private xform "
 					"could not be created\n");
+				ret_status = -1;
 				goto exit;
 			}
 			num_priv_xforms++;
 		}
-
 		if (capa->comp_feature_flags & RTE_COMP_FF_SHAREABLE_PRIV_XFORM) {
 			/* Attach shareable private xform data to ops */
 			for (i = 0; i < num_bufs; i++)
@@ -991,25 +1136,28 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 					RTE_LOG(ERR, USER1,
 						"Compression private xform "
 						"could not be created\n");
+					ret_status = -1;
 					goto exit;
 				}
 				num_priv_xforms++;
 			}
-
 			/* Attach non shareable private xform data to ops */
 			for (i = 0; i < num_bufs; i++)
 				ops[i]->private_xform = priv_xforms[i];
 		}
+
 recovery_lb:
-		ret = test_run_enqueue_dequeue(ops, num_bufs, ops_processed);
+		ret = test_run_enqueue_dequeue(ops, ops_processed, num_bufs);
 		if (ret < 0) {
 			RTE_LOG(ERR, USER1,
-				"Enqueue/dequeue operation failed\n");
+				"Compression: enqueue/dequeue operation failed\n");
+			ret_status = -1;
 			goto exit;
 		}
 
 		for (i = 0; i < num_bufs; i++) {
-			compressed_data_size[i] += ops_processed[i]->produced;
+			priv_arrays->compressed_data_size[i] +=
+					ops_processed[i]->produced;
 
 			if (ops_processed[i]->status ==
 				RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE) {
@@ -1030,21 +1178,83 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 				if (buf_ptr == NULL) {
 					RTE_LOG(ERR, USER1,
 						"Data recovery: append extra bytes to the current mbuf failed\n");
+					ret_status = -1;
 					goto exit;
 				}
 				goto recovery_lb;
 			}
 		}
-		deqd_retries = 0;
+	}
 
-		/* Free compress private xforms */
-		for (i = 0; i < num_priv_xforms; i++) {
+exit:
+	/* Free resources */
+	if (ret_status < 0)
+		for (i = 0; i < num_bufs; i++) {
+			rte_comp_op_free(ops[i]);
+			ops_processed[i] = NULL;
+		}
+
+	/* Free compress private xforms */
+	for (i = 0; i < num_priv_xforms; i++) {
+		if (priv_xforms[i] != NULL) {
 			rte_compressdev_private_xform_free(0, priv_xforms[i]);
 			priv_xforms[i] = NULL;
 		}
-		num_priv_xforms = 0;
 	}
 
+
+
+	return ret_status;
+}
+
+/**
+ * Prints out the test report. Memory freeing.
+ *
+ * Called after successful compression.
+ * Operation(s) status validation and decompression buffers freeing.
+
+ * -1 returned if function fail.
+ *
+ * @param int_data
+ *   Interim data containing session/transformation objects.
+ * @param test_data
+ *   The test parameters set by users (command line parameters).
+ * @param priv_arrays
+ *   A container used for aggregation all the private test arrays.
+ * @return
+ *   - 0: On success.
+ *   - -1: On error.
+ */
+static int
+test_deflate_comp_finalize(const struct interim_data_params *int_data,
+		const struct test_data_params *test_data,
+		const struct test_private_arrays *priv_arrays)
+{
+	/* local variables: */
+	unsigned int i;
+	struct priv_op_data *priv_data;
+
+	/* from int_data: */
+	unsigned int num_xforms = int_data->num_xforms;
+	struct rte_comp_xform **compress_xforms = int_data->compress_xforms;
+	uint16_t *buf_idx = int_data->buf_idx;
+	unsigned int num_bufs = int_data->num_bufs;
+
+	/* from priv_arrays: */
+	struct rte_comp_op **ops_processed = priv_arrays->ops_processed;
+	uint64_t *compress_checksum = priv_arrays->compress_checksum;
+	struct rte_mbuf **uncomp_bufs = priv_arrays->uncomp_bufs;
+	struct rte_comp_op **ops = priv_arrays->ops;
+
+	/* from test_data: */
+	unsigned int out_of_space = test_data->out_of_space;
+	unsigned int zlib_compress =
+			(test_data->zlib_dir == ZLIB_ALL ||
+			test_data->zlib_dir == ZLIB_COMPRESS);
+	unsigned int zlib_decompress =
+			(test_data->zlib_dir == ZLIB_ALL ||
+			test_data->zlib_dir == ZLIB_DECOMPRESS);
+
 	for (i = 0; i < num_bufs; i++) {
 		priv_data = (struct priv_op_data *)(ops_processed[i] + 1);
 		uint16_t xform_idx = priv_data->orig_idx % num_xforms;
@@ -1053,7 +1263,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 		enum rte_comp_huffman huffman_type =
 			compress_xform->deflate.huffman;
 		char engine[] = "zlib (directly, not PMD)";
-		if (zlib_dir != ZLIB_COMPRESS && zlib_dir != ZLIB_ALL)
+		if (zlib_decompress)
 			strlcpy(engine, "PMD", sizeof(engine));
 
 		RTE_LOG(DEBUG, USER1, "Buffer %u compressed by %s from %u to"
@@ -1076,15 +1286,13 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 	 * compress operation information is needed for the decompression stage)
 	 */
 	for (i = 0; i < num_bufs; i++) {
-		if (out_of_space && oos_zlib_decompress) {
+		if (out_of_space && !zlib_compress) {
 			if (ops_processed[i]->status !=
 				RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED) {
-				ret_status = -1;
-
 				RTE_LOG(ERR, USER1,
 					"Operation without expected out of "
 					"space status error\n");
-				goto exit;
+				return -1;
 			} else
 				continue;
 		}
@@ -1092,73 +1300,78 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 		if (ops_processed[i]->status != RTE_COMP_OP_STATUS_SUCCESS) {
 			RTE_LOG(ERR, USER1,
 				"Some operations were not successful\n");
-			goto exit;
+			return -1;
 		}
 		priv_data = (struct priv_op_data *)(ops_processed[i] + 1);
 		rte_pktmbuf_free(uncomp_bufs[priv_data->orig_idx]);
 		uncomp_bufs[priv_data->orig_idx] = NULL;
 	}
 
-	if (out_of_space && oos_zlib_decompress) {
-		ret_status = 0;
-		goto exit;
-	}
+	if (out_of_space && !zlib_compress)
+		return 1;
 
-	/* Allocate buffers for decompressed data */
-	ret = rte_pktmbuf_alloc_bulk(buf_pool, uncomp_bufs, num_bufs);
-	if (ret < 0) {
-		RTE_LOG(ERR, USER1,
-			"Destination mbufs could not be allocated "
-			"from the mempool\n");
-		goto exit;
-	}
+	return 0;
+}
 
-	if (buff_type == SGL_BOTH || buff_type == LB_TO_SGL) {
-		for (i = 0; i < num_bufs; i++) {
-			priv_data = (struct priv_op_data *)
-					(ops_processed[i] + 1);
-			if (out_of_space == 1 && oos_zlib_compress)
-				data_size = OUT_OF_SPACE_BUF;
-			else
-				data_size =
-				strlen(test_bufs[priv_data->orig_idx]) + 1;
-
-			if (prepare_sgl_bufs(NULL, uncomp_bufs[i],
-			       data_size,
-			       big_data ? buf_pool : ts_params->small_mbuf_pool,
-			       big_data ? buf_pool : ts_params->large_mbuf_pool,
-			       big_data ? 0 : MAX_SEGS,
-			       big_data ? MAX_DATA_MBUF_SIZE : SMALL_SEG_SIZE)
-					< 0)
-				goto exit;
-		}
+/**
+ * The main decompression function.
+ *
+ * Operation(s) configuration, depending on CLI parameters.
+ * Operation(s) processing.
+ * -1 returned if function fail.
+ *
+ * @param int_data
+ *   Interim data containing session/transformation objects.
+ * @param test_data
+ *   The test parameters set by users (command line parameters).
+ * @param priv_arrays
+ *   A container used for aggregation all the private test arrays.
+ * @return
+ *   - 0: On success.
+ *   - -1: On error.
+ */
+static int
+test_deflate_decomp_run(const struct interim_data_params *int_data,
+		const struct test_data_params *test_data,
+		const struct test_private_arrays *priv_arrays)
+{
 
-	} else {
-		for (i = 0; i < num_bufs; i++) {
-			priv_data = (struct priv_op_data *)
-					(ops_processed[i] + 1);
-			if (out_of_space == 1 && oos_zlib_compress)
-				data_size = OUT_OF_SPACE_BUF;
-			else
-				data_size =
-				strlen(test_bufs[priv_data->orig_idx]) + 1;
+	/* local variables: */
+	struct priv_op_data *priv_data;
+	unsigned int i;
+	uint16_t num_priv_xforms = 0;
+	int ret;
+	int ret_status = 0;
 
-			buf_ptr = rte_pktmbuf_append(uncomp_bufs[i], data_size);
-			if (buf_ptr == NULL) {
-				RTE_LOG(ERR, USER1,
-					"Append extra bytes to the decompressed mbuf failed\n");
-				goto exit;
-			}
+	struct comp_testsuite_params *ts_params = &testsuite_params;
 
-		}
-	}
+	/* from test_data: */
+	enum rte_comp_op_type operation_type = test_data->state;
+	unsigned int zlib_decompress =
+			(test_data->zlib_dir == ZLIB_ALL ||
+			test_data->zlib_dir == ZLIB_DECOMPRESS);
+
+	/* from int_data: */
+	struct rte_comp_xform **decompress_xforms = int_data->decompress_xforms;
+	unsigned int num_xforms = int_data->num_xforms;
+	unsigned int num_bufs = int_data->num_bufs;
+
+	/* from priv_arrays: */
+	struct rte_mbuf **uncomp_bufs = priv_arrays->uncomp_bufs;
+	struct rte_comp_op **ops = priv_arrays->ops;
+	struct rte_comp_op **ops_processed = priv_arrays->ops_processed;
+	void **priv_xforms = priv_arrays->priv_xforms;
+	uint32_t *compressed_data_size = priv_arrays->compressed_data_size;
+
+	const struct rte_compressdev_capabilities *capa =
+		rte_compressdev_capability_get(0, RTE_COMP_ALGO_DEFLATE);
 
-	/* Build the decompression operations */
 	ret = rte_comp_op_bulk_alloc(ts_params->op_pool, ops, num_bufs);
 	if (ret < 0) {
 		RTE_LOG(ERR, USER1,
 			"Decompress operations could not be allocated "
 			"from the mempool\n");
+		ret_status = -1;
 		goto exit;
 	}
 
@@ -1178,12 +1391,14 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 			ops[i]->src.length = ops_processed[i]->produced;
 
 		ops[i]->dst.offset = 0;
-		if (state == RTE_COMP_OP_STATELESS) {
+
+		if (operation_type == RTE_COMP_OP_STATELESS) {
 			ops[i]->flush_flag = RTE_COMP_FLUSH_FINAL;
 		} else {
 			RTE_LOG(ERR, USER1,
 				"Stateful operations are not supported "
 				"in these tests yet\n");
+			ret_status = -1;
 			goto exit;
 		}
 		ops[i]->input_chksum = 0;
@@ -1202,7 +1417,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 	rte_comp_op_bulk_free(ops_processed, num_bufs);
 
 	/* Decompress data (either with Zlib API or compressdev API */
-	if (zlib_dir == ZLIB_DECOMPRESS || zlib_dir == ZLIB_ALL) {
+	if (zlib_decompress) {
 		for (i = 0; i < num_bufs; i++) {
 			priv_data = (struct priv_op_data *)(ops[i] + 1);
 			uint16_t xform_idx = priv_data->orig_idx % num_xforms;
@@ -1210,8 +1425,10 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 				decompress_xforms[xform_idx];
 
 			ret = decompress_zlib(ops[i], decompress_xform);
-			if (ret < 0)
+			if (ret < 0) {
+				ret_status = -1;
 				goto exit;
+			}
 
 			ops_processed[i] = ops[i];
 		}
@@ -1225,6 +1442,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 				RTE_LOG(ERR, USER1,
 					"Decompression private xform "
 					"could not be created\n");
+				ret_status = -1;
 				goto exit;
 			}
 			num_priv_xforms++;
@@ -1249,6 +1467,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 					RTE_LOG(ERR, USER1,
 						"Decompression private xform "
 						"could not be created\n");
+					ret_status = -1;
 					goto exit;
 				}
 				num_priv_xforms++;
@@ -1262,47 +1481,83 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 			}
 		}
 
-		/* Enqueue and dequeue all operations */
-		num_enqd = rte_compressdev_enqueue_burst(0, 0, ops, num_bufs);
-		if (num_enqd < num_bufs) {
+		ret = test_run_enqueue_dequeue(ops, ops_processed, num_bufs);
+		if (ret < 0) {
+			ret_status = -1;
 			RTE_LOG(ERR, USER1,
-				"The operations could not be enqueued\n");
-			goto exit;
+				"Decompression: enqueue/dequeue operation failed\n");
 		}
+	}
 
-		num_total_deqd = 0;
-		do {
-			/*
-			 * If retrying a dequeue call, wait for 10 ms to allow
-			 * enough time to the driver to process the operations
-			 */
-			if (deqd_retries != 0) {
-				/*
-				 * Avoid infinite loop if not all the
-				 * operations get out of the device
-				 */
-				if (deqd_retries == MAX_DEQD_RETRIES) {
-					RTE_LOG(ERR, USER1,
-						"Not all operations could be "
-						"dequeued\n");
-					goto exit;
-				}
-				usleep(DEQUEUE_WAIT_TIME);
-			}
-			num_deqd = rte_compressdev_dequeue_burst(0, 0,
-				&ops_processed[num_total_deqd], num_bufs);
-			num_total_deqd += num_deqd;
-			deqd_retries++;
-
-		} while (num_total_deqd < num_enqd);
+exit:
+	/* Free resources */
+	if (ret_status < 0) {
+		rte_comp_op_bulk_free(ops_processed, num_bufs);
+		rte_comp_op_bulk_free(ops, num_bufs);
+	}
 
-		deqd_retries = 0;
+	/* Free compress private xforms */
+	for (i = 0; i < num_priv_xforms; i++) {
+		if (priv_xforms[i] != NULL) {
+			rte_compressdev_private_xform_free(0, priv_xforms[i]);
+			priv_xforms[i] = NULL;
+		}
 	}
 
+	return ret_status;
+}
+
+/**
+ * Prints out the test report. Memory freeing.
+ *
+ * Called after successful decompression.
+ * Operation(s) status validation and compression buffers freeing.
+
+ * -1 returned if function fail.
+ *
+ * @param int_data
+ *   Interim data containing session/transformation objects.
+ * @param test_data
+ *   The test parameters set by users (command line parameters).
+ * @param priv_arrays
+ *   A container used for aggregation all the private test arrays.
+ * @return
+ *     1: On success (caller should stop and exit)
+ *   - 0: On success.
+ *   - -1: On error.
+ */
+static int
+test_deflate_decomp_finalize(const struct interim_data_params *int_data,
+		const struct test_data_params *test_data,
+		const struct test_private_arrays *priv_arrays)
+{
+	/* local variables: */
+	unsigned int i;
+	struct priv_op_data *priv_data;
+
+	/* from int_data: */
+	uint16_t *buf_idx = int_data->buf_idx;
+	unsigned int num_bufs = int_data->num_bufs;
+
+	/* from priv_arrays: */
+	struct rte_comp_op **ops_processed = priv_arrays->ops_processed;
+	struct rte_mbuf **comp_bufs = priv_arrays->comp_bufs;
+	struct rte_comp_op **ops = priv_arrays->ops;
+
+	/* from test_data: */
+	unsigned int out_of_space = test_data->out_of_space;
+
+	unsigned int zlib_compress =
+			(test_data->zlib_dir == ZLIB_ALL ||
+			test_data->zlib_dir == ZLIB_COMPRESS);
+	unsigned int zlib_decompress =
+			(test_data->zlib_dir == ZLIB_ALL ||
+			test_data->zlib_dir == ZLIB_DECOMPRESS);
+
 	for (i = 0; i < num_bufs; i++) {
 		priv_data = (struct priv_op_data *)(ops_processed[i] + 1);
 		char engine[] = "zlib, (directly, no PMD)";
-		if (zlib_dir != ZLIB_DECOMPRESS && zlib_dir != ZLIB_ALL)
+		if (zlib_compress)
 			strlcpy(engine, "pmd", sizeof(engine));
 		RTE_LOG(DEBUG, USER1,
 			"Buffer %u decompressed by %s from %u to %u bytes\n",
@@ -1316,15 +1571,14 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 	 * compress operation information is still needed)
 	 */
 	for (i = 0; i < num_bufs; i++) {
-		if (out_of_space && oos_zlib_compress) {
+		if (out_of_space && !zlib_decompress) {
 			if (ops_processed[i]->status !=
 				RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED) {
-				ret_status = -1;
 
 				RTE_LOG(ERR, USER1,
 					"Operation without expected out of "
 					"space status error\n");
-				goto exit;
+				return -1;
 			} else
 				continue;
 		}
@@ -1332,17 +1586,54 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 		if (ops_processed[i]->status != RTE_COMP_OP_STATUS_SUCCESS) {
 			RTE_LOG(ERR, USER1,
 				"Some operations were not successful\n");
-			goto exit;
+			return -1;
 		}
 		priv_data = (struct priv_op_data *)(ops_processed[i] + 1);
 		rte_pktmbuf_free(comp_bufs[priv_data->orig_idx]);
 		comp_bufs[priv_data->orig_idx] = NULL;
 	}
 
-	if (out_of_space && oos_zlib_compress) {
-		ret_status = 0;
-		goto exit;
-	}
+	if (out_of_space && !zlib_decompress)
+		return 1;
+
+	return 0;
+}
+
+/**
+ * Validation of the output (compression/decompression) data.
+ *
+ * The function compares the source stream with the output stream,
+ * after decompression, to check if compression/decompression
+ * was correct.
+ * -1 returned if function fail.
+ *
+ * @param int_data
+ *   Interim data containing session/transformation objects.
+ * @param priv_arrays
+ *   A container used for aggregation all the private test arrays.
+ * @return
+ *   - 0: On success.
+ *   - -1: On error.
+ */
+static int
+test_results_validation(const struct interim_data_params *int_data,
+		const struct test_private_arrays *priv_arrays)
+{
+	/* local variables: */
+	unsigned int i;
+	struct priv_op_data *priv_data;
+	const char *buf1;
+	const char *buf2;
+	char *contig_buf = NULL;
+
+	/* from int_data: */
+	struct rte_comp_xform **compress_xforms = int_data->compress_xforms;
+	unsigned int num_bufs = int_data->num_bufs;
+	const char * const *test_bufs = int_data->test_bufs;
+
+	/* from priv_arrays: */
+	uint64_t *compress_checksum = priv_arrays->compress_checksum;
+	struct rte_comp_op **ops_processed = priv_arrays->ops_processed;
 
 	/*
 	 * Compare the original stream with the decompressed stream
@@ -1350,8 +1641,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 	 */
 	for (i = 0; i < num_bufs; i++) {
 		priv_data = (struct priv_op_data *)(ops_processed[i] + 1);
-		const char *buf1 = test_bufs[priv_data->orig_idx];
-		const char *buf2;
+		buf1 = test_bufs[priv_data->orig_idx];
 		contig_buf = rte_malloc(NULL, ops_processed[i]->produced, 0);
 		if (contig_buf == NULL) {
 			RTE_LOG(ERR, USER1, "Contiguous buffer could not "
@@ -1381,9 +1671,150 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 		rte_free(contig_buf);
 		contig_buf = NULL;
 	}
+	return 0;
 
-	ret_status = 0;
+exit:
+	rte_free(contig_buf);
+	return -1;
+}
+
+/**
+ * Compresses and decompresses input stream with compressdev API and Zlib API
+ *
+ * Basic test function. Common for all the functional tests.
+ * -1 returned if function fail.
+ *
+ * @param int_data
+ *   Interim data containing session/transformation objects.
+ * @param test_data
+ *   The test parameters set by users (command line parameters).
+ * @return
+ *   - 0: On success.
+ *   - -1: On error.
+ */
+static int
+test_deflate_comp_decomp(const struct interim_data_params *int_data,
+		const struct test_data_params *test_data)
+{
+	unsigned int num_bufs = int_data->num_bufs;
+	unsigned int out_of_space = test_data->out_of_space;
+
+	int ret_status = -1;
+	int ret;
+	struct rte_mbuf *uncomp_bufs[num_bufs];
+	struct rte_mbuf *comp_bufs[num_bufs];
+	struct rte_comp_op *ops[num_bufs];
+	struct rte_comp_op *ops_processed[num_bufs];
+	void *priv_xforms[num_bufs];
+	unsigned int i;
+
+	uint64_t compress_checksum[num_bufs];
+	uint32_t compressed_data_size[num_bufs];
+	char *contig_buf = NULL;
+
+	const struct rte_compressdev_capabilities *capa;
+
+	/* Compressing with CompressDev */
+	unsigned int zlib_compress =
+			(test_data->zlib_dir == ZLIB_ALL ||
+			test_data->zlib_dir == ZLIB_COMPRESS);
+	unsigned int zlib_decompress =
+			(test_data->zlib_dir == ZLIB_ALL ||
+			test_data->zlib_dir == ZLIB_DECOMPRESS);
+
+	struct test_private_arrays priv_arrays;
+
+	priv_arrays.uncomp_bufs = uncomp_bufs;
+	priv_arrays.comp_bufs = comp_bufs;
+	priv_arrays.ops = ops;
+	priv_arrays.ops_processed = ops_processed;
+	priv_arrays.priv_xforms = priv_xforms;
+	priv_arrays.compress_checksum = compress_checksum;
+	priv_arrays.compressed_data_size = compressed_data_size;
+
+	capa = rte_compressdev_capability_get(0, RTE_COMP_ALGO_DEFLATE);
+	if (capa == NULL) {
+		RTE_LOG(ERR, USER1,
+			"Compress device does not support DEFLATE\n");
+		return -1;
+	}
+	test_objects_init(&priv_arrays, num_bufs);
+
+	/* Prepare the source mbufs with the data */
+	ret = test_mbufs_source_preparation(int_data, test_data, &priv_arrays);
+	if (ret < 0) {
+		ret_status = -1;
+		goto exit;
+	}
+
+/* COMPRESSION  */
+
+	/* Prepare the destination mbufs */
+	ret = test_mbufs_destination_preparation(
+			NULL, /*can be equal to NULL*/
+			comp_bufs,
+			out_of_space == 1 && !zlib_compress,
+			int_data,
+			test_data);
+	if (ret < 0) {
+		ret_status = -1;
+		goto exit;
+	}
+
+	/* Run compression */
+	ret = test_deflate_comp_run(int_data, test_data, &priv_arrays);
+	if (ret < 0) {
+		ret_status = -1;
+		goto exit;
+	}
+
+	ret = test_deflate_comp_finalize(int_data, test_data, &priv_arrays);
+	if (ret < 0) {
+		ret_status = -1;
+		goto exit;
+	} else if (ret == 1) {
+		ret_status = 0;
+		goto exit;
+	}
+
+/* DECOMPRESSION  */
 
+	/* Allocate buffers for decompressed data */
+	ret = test_mbufs_destination_preparation(
+			ops_processed, /*can be equal to NULL*/
+			uncomp_bufs,
+			out_of_space == 1 && !zlib_decompress,
+			int_data,
+			test_data);
+	if (ret < 0) {
+		ret_status = -1;
+		goto exit;
+	}
+
+	/* Run decompression */
+	ret = test_deflate_decomp_run(int_data, test_data, &priv_arrays);
+	if (ret < 0) {
+		ret_status = -1;
+		goto exit;
+	}
+
+	ret = test_deflate_decomp_finalize(int_data, test_data, &priv_arrays);
+	if (ret < 0) {
+		ret_status = -1;
+		goto exit;
+	} else if (ret == 1) {
+		ret_status = 0;
+		goto exit;
+	}
+
+/* FINAL PROCESSING  */
+
+	ret = test_results_validation(int_data, &priv_arrays);
+	if (ret < 0) {
+		ret_status = -1;
+		goto exit;
+	}
+	ret_status = 0;
 exit:
 	/* Free resources */
 	for (i = 0; i < num_bufs; i++) {
@@ -1392,10 +1823,6 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
 		rte_comp_op_free(ops[i]);
 		rte_comp_op_free(ops_processed[i]);
 	}
-	for (i = 0; i < num_priv_xforms; i++) {
-		if (priv_xforms[i] != NULL)
-			rte_compressdev_private_xform_free(0, priv_xforms[i]);
-	}
 	rte_free(contig_buf);
 
 	return ret_status;
-- 
2.17.1


^ permalink raw reply	[flat|nested] 21+ messages in thread

end of thread, other threads:[~2019-11-08 18:40 UTC | newest]

Thread overview: 21+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2019-09-09 14:45 [dpdk-dev] [PATCH] test/compress: unit tests refactoring Artur Trybula
2019-09-12 14:34 ` [dpdk-dev] [PATCH v2 0/1] compression: " Artur Trybula
2019-09-12 14:34   ` [dpdk-dev] [PATCH v2 1/1] test/compress: " Artur Trybula
2019-10-24  9:16   ` [dpdk-dev] [PATCH v3 0/1] compression: " Artur Trybula
2019-10-24  9:16     ` [dpdk-dev] [PATCH v3 1/1] test/compress: " Artur Trybula
2019-10-24  9:22       ` Dybkowski, AdamX
2019-10-24  9:27       ` Akhil Goyal
2019-10-31  8:38         ` Akhil Goyal
2019-10-31 18:23       ` [dpdk-dev] [EXT] " Shally Verma
2019-11-04 10:24         ` Trybula, ArturX
2019-11-04 15:24           ` Shally Verma
2019-11-06  7:43             ` Akhil Goyal
2019-11-06 11:33             ` Trybula, ArturX
2019-11-06 15:00               ` Shally Verma
2019-11-06 15:33                 ` Trybula, ArturX
2019-11-06 15:36                   ` Shally Verma
2019-11-07 17:26     ` [dpdk-dev] [PATCH v4 0/1] compression: " Artur Trybula
2019-11-07 17:26       ` [dpdk-dev] [PATCH v4 1/1] test/compress: " Artur Trybula
2019-11-08  6:35         ` [dpdk-dev] [EXT] " Shally Verma
2019-11-08 10:16           ` Akhil Goyal
2019-11-08 18:40         ` [dpdk-dev] " Thomas Monjalon

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).