From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 58A87A0613 for ; Thu, 26 Sep 2019 11:33:14 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 9AF771BF3D; Thu, 26 Sep 2019 11:33:00 +0200 (CEST) Received: from mga04.intel.com (mga04.intel.com [192.55.52.120]) by dpdk.org (Postfix) with ESMTP id 967331BF3F for ; Thu, 26 Sep 2019 11:32:53 +0200 (CEST) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga002.fm.intel.com ([10.253.24.26]) by fmsmga104.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 26 Sep 2019 02:32:53 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.64,551,1559545200"; d="scan'208";a="219303140" Received: from atrybulx-mobl.ger.corp.intel.com (HELO build-VirtualBox.isw.intel.com) ([10.103.104.115]) by fmsmga002.fm.intel.com with ESMTP; 26 Sep 2019 02:32:51 -0700 From: Artur Trybula To: dev@dpdk.org, fiona.trahe@intel.com, shallyv@marvell.com, adamx.dybkowski@intel.com, arturx.trybula@intel.com, akhil.goyal@nxp.com Date: Thu, 26 Sep 2019 11:32:38 +0200 Message-Id: <20190926093238.16663-3-arturx.trybula@intel.com> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20190926093238.16663-1-arturx.trybula@intel.com> References: <20190906111138.1582-1-arturx.trybula@intel.com> <20190926093238.16663-1-arturx.trybula@intel.com> Subject: [dpdk-dev] [PATCH v2 2/2] test/compress: unit test for stateless overflow recovery X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Added unit test to check out-of-space recoverable feature. Signed-off-by: Artur Trybula --- app/test/test_compressdev.c | 261 +++++++++++++++++++++++++++++------- 1 file changed, 215 insertions(+), 46 deletions(-) diff --git a/app/test/test_compressdev.c b/app/test/test_compressdev.c index d470c34b9..0a5c76721 100644 --- a/app/test/test_compressdev.c +++ b/app/test/test_compressdev.c @@ -30,6 +30,7 @@ * due to the compress block headers */ #define COMPRESS_BUF_SIZE_RATIO 1.3 +#define COMPRESS_BUF_SIZE_RATIO_OVERFLOW 0.2 #define NUM_LARGE_MBUFS 16 #define SMALL_SEG_SIZE 256 #define MAX_SEGS 16 @@ -72,6 +73,11 @@ enum varied_buff { LB_TO_SGL /* output buffer is chained */ }; +enum overflow_test { + OVERFLOW_DISABLED, + OVERFLOW_ENABLED +}; + struct priv_op_data { uint16_t orig_idx; }; @@ -110,6 +116,8 @@ struct test_data_params { const struct rte_memzone *inbuf_memzone; const struct rte_memzone *compbuf_memzone; const struct rte_memzone *uncompbuf_memzone; + /* overflow test activation */ + enum overflow_test overflow; }; static struct comp_testsuite_params testsuite_params = { 0 }; @@ -725,6 +733,49 @@ extbuf_free_callback(void *addr __rte_unused, void *opaque __rte_unused) { } +static int +test_run_enqueue_dequeue(struct rte_comp_op **ops, unsigned int num_bufs, + struct rte_comp_op **ops_processed) +{ + uint16_t num_enqd, num_deqd, num_total_deqd; + unsigned int deqd_retries = 0; + + /* Enqueue and dequeue all operations */ + num_enqd = rte_compressdev_enqueue_burst(0, 0, ops, num_bufs); + if (num_enqd < num_bufs) { + RTE_LOG(ERR, USER1, + "Some operations could not be enqueued\n"); + return -1; + } + + num_total_deqd = 0; + do { + /* + * If retrying a dequeue call, wait for 10 ms to allow + * enough time to the driver to process the operations + */ + if (deqd_retries != 0) { + /* + * Avoid infinite loop if not all the + * operations get out of the device + */ + if (deqd_retries == MAX_DEQD_RETRIES) { + RTE_LOG(ERR, USER1, + "Not all operations could be dequeued\n"); + return -1; + } + usleep(DEQUEUE_WAIT_TIME); + } + num_deqd = rte_compressdev_dequeue_burst(0, 0, + &ops_processed[num_total_deqd], num_bufs); + num_total_deqd += num_deqd; + deqd_retries++; + + } while (num_total_deqd < num_enqd); + + return 0; +} + /* * Compresses and decompresses buffer with compressdev API and Zlib API */ @@ -745,6 +796,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data, unsigned int out_of_space = test_data->out_of_space; unsigned int big_data = test_data->big_data; enum zlib_direction zlib_dir = test_data->zlib_dir; + enum overflow_test overflow_tst = test_data->overflow; int ret_status = TEST_FAILED; struct rte_mbuf_ext_shared_info inbuf_info; struct rte_mbuf_ext_shared_info compbuf_info; @@ -773,6 +825,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data, rte_compressdev_capability_get(0, RTE_COMP_ALGO_DEFLATE); char *contig_buf = NULL; uint64_t compress_checksum[num_bufs]; + uint32_t compressed_data_size[num_bufs]; void *stream = NULL; char *all_decomp_data = NULL; unsigned int decomp_produced_data_size = 0; @@ -793,6 +846,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data, memset(ops, 0, sizeof(struct rte_comp_op *) * num_bufs); memset(ops_processed, 0, sizeof(struct rte_comp_op *) * num_bufs); memset(priv_xforms, 0, sizeof(void *) * num_bufs); + memset(compressed_data_size, 0, sizeof(uint32_t) * num_bufs); if (decompress_state == RTE_COMP_OP_STATEFUL) { data_size = strlen(test_bufs[0]) + 1; @@ -845,6 +899,11 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data, for (i = 0; i < num_bufs; i++) { data_size = strlen(test_bufs[i]) + 1; buf_ptr = rte_pktmbuf_append(uncomp_bufs[i], data_size); + if (buf_ptr == NULL) { + RTE_LOG(ERR, USER1, + "Append extra bytes to the source mbuf failed\n"); + goto exit; + } strlcpy(buf_ptr, test_bufs[i], data_size); } } @@ -893,11 +952,23 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data, for (i = 0; i < num_bufs; i++) { if (out_of_space == 1 && oos_zlib_decompress) data_size = OUT_OF_SPACE_BUF; - else - (data_size = strlen(test_bufs[i]) * - COMPRESS_BUF_SIZE_RATIO); + else { + float ratio = + ((test_data->zlib_dir == ZLIB_DECOMPRESS || + test_data->zlib_dir == ZLIB_NONE) && + overflow_tst == OVERFLOW_ENABLED) ? + COMPRESS_BUF_SIZE_RATIO_OVERFLOW : + COMPRESS_BUF_SIZE_RATIO; + + data_size = strlen(test_bufs[i]) * ratio; + } + buf_ptr = rte_pktmbuf_append(comp_bufs[i], data_size); + if (buf_ptr == NULL) { + RTE_LOG(ERR, USER1, + "Append extra bytes to the destination mbuf failed\n"); + goto exit; + } - rte_pktmbuf_append(comp_bufs[i], data_size); } } @@ -963,7 +1034,8 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data, num_priv_xforms++; } - if (capa->comp_feature_flags & RTE_COMP_FF_SHAREABLE_PRIV_XFORM) { + if (capa->comp_feature_flags & + RTE_COMP_FF_SHAREABLE_PRIV_XFORM) { /* Attach shareable private xform data to ops */ for (i = 0; i < num_bufs; i++) ops[i]->private_xform = priv_xforms[i % num_xforms]; @@ -987,40 +1059,41 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data, ops[i]->private_xform = priv_xforms[i]; } - /* Enqueue and dequeue all operations */ - num_enqd = rte_compressdev_enqueue_burst(0, 0, ops, num_bufs); - if (num_enqd < num_bufs) { +recovery_lb: + ret = test_run_enqueue_dequeue(ops, num_bufs, ops_processed); + if (ret < 0) { RTE_LOG(ERR, USER1, - "The operations could not be enqueued\n"); + "Enqueue/dequeue operation failed\n"); goto exit; } - num_total_deqd = 0; - do { - /* - * If retrying a dequeue call, wait for 10 ms to allow - * enough time to the driver to process the operations - */ - if (deqd_retries != 0) { - /* - * Avoid infinite loop if not all the - * operations get out of the device - */ - if (deqd_retries == MAX_DEQD_RETRIES) { + for (i = 0; i < num_bufs; i++) { + compressed_data_size[i] += ops_processed[i]->produced; + + if (ops_processed[i]->status == + RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE) { + + ops[i]->status = + RTE_COMP_OP_STATUS_NOT_PROCESSED; + ops[i]->src.offset += + ops_processed[i]->consumed; + ops[i]->src.length -= + ops_processed[i]->consumed; + ops[i]->dst.offset += + ops_processed[i]->produced; + + buf_ptr = rte_pktmbuf_append( + ops[i]->m_dst, + ops_processed[i]->produced); + + if (buf_ptr == NULL) { RTE_LOG(ERR, USER1, - "Not all operations could be " - "dequeued\n"); + "Data recovery: append extra bytes to the current mbuf failed\n"); goto exit; } - usleep(DEQUEUE_WAIT_TIME); + goto recovery_lb; } - num_deqd = rte_compressdev_dequeue_burst(0, 0, - &ops_processed[num_total_deqd], num_bufs); - num_total_deqd += num_deqd; - deqd_retries++; - - } while (num_total_deqd < num_enqd); - + } deqd_retries = 0; /* Free compress private xforms */ @@ -1064,7 +1137,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data, for (i = 0; i < num_bufs; i++) { if (out_of_space && oos_zlib_decompress) { if (ops_processed[i]->status != - RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED) { + RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED) { ret_status = TEST_FAILED; RTE_LOG(ERR, USER1, "Operation without expected out of " @@ -1147,7 +1220,12 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data, data_size = strlen(test_bufs[priv_data->orig_idx]) + 1; - rte_pktmbuf_append(uncomp_bufs[i], data_size); + buf_ptr = rte_pktmbuf_append(uncomp_bufs[i], data_size); + if (buf_ptr == NULL) { + RTE_LOG(ERR, USER1, + "Append extra bytes to the decompressed mbuf failed\n"); + goto exit; + } } } @@ -1169,7 +1247,10 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data, * Set the length of the compressed data to the * number of bytes that were produced in the previous stage */ - ops[i]->src.length = ops_processed[i]->produced; + if (compressed_data_size[i]) + ops[i]->src.length = compressed_data_size[i]; + else + ops[i]->src.length = ops_processed[i]->produced; ops[i]->dst.offset = 0; if (decompress_state == RTE_COMP_OP_STATELESS) { @@ -1541,7 +1622,8 @@ test_compressdev_deflate_stateless_fixed(void) .buff_type = LB_BOTH, .zlib_dir = ZLIB_DECOMPRESS, .out_of_space = 0, - .big_data = 0 + .big_data = 0, + .overflow = OVERFLOW_DISABLED }; for (i = 0; i < RTE_DIM(compress_test_bufs); i++) { @@ -1611,7 +1693,8 @@ test_compressdev_deflate_stateless_dynamic(void) .buff_type = LB_BOTH, .zlib_dir = ZLIB_DECOMPRESS, .out_of_space = 0, - .big_data = 0 + .big_data = 0, + .overflow = OVERFLOW_DISABLED }; for (i = 0; i < RTE_DIM(compress_test_bufs); i++) { @@ -1665,7 +1748,8 @@ test_compressdev_deflate_stateless_multi_op(void) .buff_type = LB_BOTH, .zlib_dir = ZLIB_DECOMPRESS, .out_of_space = 0, - .big_data = 0 + .big_data = 0, + .overflow = OVERFLOW_DISABLED }; /* Compress with compressdev, decompress with Zlib */ @@ -1718,7 +1802,8 @@ test_compressdev_deflate_stateless_multi_level(void) .buff_type = LB_BOTH, .zlib_dir = ZLIB_DECOMPRESS, .out_of_space = 0, - .big_data = 0 + .big_data = 0, + .overflow = OVERFLOW_DISABLED }; for (i = 0; i < RTE_DIM(compress_test_bufs); i++) { @@ -1807,7 +1892,8 @@ test_compressdev_deflate_stateless_multi_xform(void) .buff_type = LB_BOTH, .zlib_dir = ZLIB_DECOMPRESS, .out_of_space = 0, - .big_data = 0 + .big_data = 0, + .overflow = OVERFLOW_DISABLED }; /* Compress with compressdev, decompress with Zlib */ @@ -1855,7 +1941,8 @@ test_compressdev_deflate_stateless_sgl(void) .buff_type = SGL_BOTH, .zlib_dir = ZLIB_DECOMPRESS, .out_of_space = 0, - .big_data = 0 + .big_data = 0, + .overflow = OVERFLOW_DISABLED }; for (i = 0; i < RTE_DIM(compress_test_bufs); i++) { @@ -1965,7 +2052,8 @@ test_compressdev_deflate_stateless_checksum(void) .buff_type = LB_BOTH, .zlib_dir = ZLIB_DECOMPRESS, .out_of_space = 0, - .big_data = 0 + .big_data = 0, + .overflow = OVERFLOW_DISABLED }; /* Check if driver supports crc32 checksum and test */ @@ -2082,7 +2170,8 @@ test_compressdev_out_of_space_buffer(void) .buff_type = LB_BOTH, .zlib_dir = ZLIB_DECOMPRESS, .out_of_space = 1, /* run out-of-space test */ - .big_data = 0 + .big_data = 0, + .overflow = OVERFLOW_DISABLED }; /* Compress with compressdev, decompress with Zlib */ test_data.zlib_dir = ZLIB_DECOMPRESS; @@ -2159,7 +2248,8 @@ test_compressdev_deflate_stateless_dynamic_big(void) .buff_type = SGL_BOTH, .zlib_dir = ZLIB_DECOMPRESS, .out_of_space = 0, - .big_data = 1 + .big_data = 1, + .overflow = OVERFLOW_DISABLED }; ts_params->def_comp_xform->compress.deflate.huffman = @@ -2223,7 +2313,8 @@ test_compressdev_deflate_stateful_decomp(void) .out_of_space = 0, .big_data = 0, .decompress_output_block_size = 2000, - .decompress_steps_max = 4 + .decompress_steps_max = 4, + .overflow = OVERFLOW_DISABLED }; /* Compress with Zlib, decompress with compressdev */ @@ -2305,7 +2396,8 @@ test_compressdev_deflate_stateful_decomp_checksum(void) .out_of_space = 0, .big_data = 0, .decompress_output_block_size = 2000, - .decompress_steps_max = 4 + .decompress_steps_max = 4, + .overflow = OVERFLOW_DISABLED }; /* Check if driver supports crc32 checksum and test */ @@ -2386,6 +2478,7 @@ test_compressdev_deflate_stateful_decomp_checksum(void) return ret; } + static const struct rte_memzone * make_memzone(const char *name, size_t size) { @@ -2441,7 +2534,8 @@ test_compressdev_external_mbufs(void) .inbuf_memzone = make_memzone("inbuf", data_len), .compbuf_memzone = make_memzone("compbuf", data_len * COMPRESS_BUF_SIZE_RATIO), - .uncompbuf_memzone = make_memzone("decompbuf", data_len) + .uncompbuf_memzone = make_memzone("decompbuf", data_len), + .overflow = OVERFLOW_DISABLED }; for (i = 0; i < RTE_DIM(compress_test_bufs); i++) { @@ -2472,6 +2566,79 @@ test_compressdev_external_mbufs(void) return ret; } +static int +test_compressdev_deflate_stateless_fixed_overflow(void) +{ + struct comp_testsuite_params *ts_params = &testsuite_params; + uint16_t i; + int ret; + const struct rte_compressdev_capabilities *capab; + + capab = rte_compressdev_capability_get(0, RTE_COMP_ALGO_DEFLATE); + TEST_ASSERT(capab != NULL, "Failed to retrieve device capabilities"); + + if ((capab->comp_feature_flags & RTE_COMP_FF_HUFFMAN_FIXED) == 0) + return -ENOTSUP; + + struct rte_comp_xform *compress_xform = + rte_malloc(NULL, sizeof(struct rte_comp_xform), 0); + + if (compress_xform == NULL) { + RTE_LOG(ERR, USER1, + "Compress xform could not be created\n"); + ret = TEST_FAILED; + goto exit; + } + + memcpy(compress_xform, ts_params->def_comp_xform, + sizeof(struct rte_comp_xform)); + compress_xform->compress.deflate.huffman = RTE_COMP_HUFFMAN_FIXED; + + struct interim_data_params int_data = { + NULL, + 1, + NULL, + &compress_xform, + &ts_params->def_decomp_xform, + 1 + }; + + struct test_data_params test_data = { + .compress_state = RTE_COMP_OP_STATELESS, + .decompress_state = RTE_COMP_OP_STATELESS, + .buff_type = LB_BOTH, + .zlib_dir = ZLIB_DECOMPRESS, + .out_of_space = 0, + .big_data = 0, + .overflow = OVERFLOW_ENABLED + }; + + for (i = 0; i < RTE_DIM(compress_test_bufs); i++) { + int_data.test_bufs = &compress_test_bufs[i]; + int_data.buf_idx = &i; + + /* Compress with compressdev, decompress with Zlib */ + test_data.zlib_dir = ZLIB_DECOMPRESS; + if (test_deflate_comp_decomp(&int_data, &test_data) < 0) { + ret = TEST_FAILED; + goto exit; + } + + /* Compress with Zlib, decompress with compressdev */ + test_data.zlib_dir = ZLIB_COMPRESS; + if (test_deflate_comp_decomp(&int_data, &test_data) < 0) { + ret = TEST_FAILED; + goto exit; + } + } + + ret = TEST_SUCCESS; + +exit: + rte_free(compress_xform); + return ret; +} + static struct unit_test_suite compressdev_testsuite = { .suite_name = "compressdev unit test suite", .setup = testsuite_setup, @@ -2503,6 +2670,8 @@ static struct unit_test_suite compressdev_testsuite = { test_compressdev_deflate_stateful_decomp_checksum), TEST_CASE_ST(generic_ut_setup, generic_ut_teardown, test_compressdev_external_mbufs), + TEST_CASE_ST(generic_ut_setup, generic_ut_teardown, + test_compressdev_deflate_stateless_fixed_overflow), TEST_CASES_END() /**< NULL terminate unit test array */ } }; -- 2.17.1