* [dpdk-dev] [PATCH 1/2] compress/qat: overflow catching
@ 2019-09-06 11:11 Artur Trybula
2019-09-06 11:11 ` [dpdk-dev] [PATCH 2/2] test/compress: unit test for stateless overflow recovery Artur Trybula
2019-09-26 9:32 ` [dpdk-dev] [PATCH v2 0/2] Add overflow recovery for stateless compression Artur Trybula
0 siblings, 2 replies; 16+ messages in thread
From: Artur Trybula @ 2019-09-06 11:11 UTC (permalink / raw)
To: dev, fiona.trahe, shallyv, adamx.dybkowski, arturx.trybula, akhil.goyal
Out-of-space recoverable feature added to QuickAssist compressdev driver
Signed-off-by: Artur Trybula <arturx.trybula@intel.com>
---
drivers/compress/qat/qat_comp.c | 25 ++++++++++++++++++++++---
1 file changed, 22 insertions(+), 3 deletions(-)
diff --git a/drivers/compress/qat/qat_comp.c b/drivers/compress/qat/qat_comp.c
index 835aaa838..aa43df6b1 100644
--- a/drivers/compress/qat/qat_comp.c
+++ b/drivers/compress/qat/qat_comp.c
@@ -259,10 +259,29 @@ qat_comp_process_response(void **op, uint8_t *resp, void *op_cookie,
(!cmp_err_code && xlat_err_code == ERR_CODE_OVERFLOW_ERROR)
||
(cmp_err_code == ERR_CODE_OVERFLOW_ERROR &&
- xlat_err_code == ERR_CODE_OVERFLOW_ERROR))
- rx_op->status =
+ xlat_err_code == ERR_CODE_OVERFLOW_ERROR)) {
+
+ struct icp_qat_fw_resp_comp_pars *comp_resp =
+ (struct icp_qat_fw_resp_comp_pars *)&resp_msg->comp_resp_pars;
+
+ if (comp_resp->input_byte_counter) {
+ if ((qat_xform->qat_comp_request_type
+ == QAT_COMP_REQUEST_FIXED_COMP_STATELESS) ||
+ (qat_xform->qat_comp_request_type
+ == QAT_COMP_REQUEST_DYNAMIC_COMP_STATELESS)) {
+ rx_op->status =
+ RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE;
+ rx_op->consumed =
+ comp_resp->input_byte_counter;
+ rx_op->produced =
+ comp_resp->output_byte_counter;
+ } else
+ rx_op->status =
RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
- else
+ } else
+ rx_op->status =
+ RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
+ } else
rx_op->status = RTE_COMP_OP_STATUS_ERROR;
++(*dequeue_err_count);
--
2.17.1
^ permalink raw reply [flat|nested] 16+ messages in thread
* [dpdk-dev] [PATCH 2/2] test/compress: unit test for stateless overflow recovery
2019-09-06 11:11 [dpdk-dev] [PATCH 1/2] compress/qat: overflow catching Artur Trybula
@ 2019-09-06 11:11 ` Artur Trybula
2019-09-26 9:32 ` [dpdk-dev] [PATCH v2 0/2] Add overflow recovery for stateless compression Artur Trybula
1 sibling, 0 replies; 16+ messages in thread
From: Artur Trybula @ 2019-09-06 11:11 UTC (permalink / raw)
To: dev, fiona.trahe, shallyv, adamx.dybkowski, arturx.trybula, akhil.goyal
Added unit test to check out-of-space recoverable feature.
Signed-off-by: Artur Trybula <arturx.trybula@intel.com>
---
app/test/test_compressdev.c | 264 +++++++++++++++++++++++++++++-------
1 file changed, 216 insertions(+), 48 deletions(-)
diff --git a/app/test/test_compressdev.c b/app/test/test_compressdev.c
index 992eac8e0..3a655164f 100644
--- a/app/test/test_compressdev.c
+++ b/app/test/test_compressdev.c
@@ -30,6 +30,7 @@
* due to the compress block headers
*/
#define COMPRESS_BUF_SIZE_RATIO 1.3
+#define COMPRESS_BUF_SIZE_RATIO_OVERFLOW 0.2
#define NUM_LARGE_MBUFS 16
#define SMALL_SEG_SIZE 256
#define MAX_SEGS 16
@@ -65,6 +66,12 @@ enum zlib_direction {
ZLIB_ALL
};
+enum overflow_test {
+ OVERFLOW_DISABLED,
+ OVERFLOW_ENABLED
+};
+
+
enum varied_buff {
LB_BOTH = 0, /* both input and output are linear*/
SGL_BOTH, /* both input and output are chained */
@@ -100,6 +107,7 @@ struct test_data_params {
enum zlib_direction zlib_dir;
unsigned int out_of_space;
unsigned int big_data;
+ enum overflow_test overflow;
};
static struct comp_testsuite_params testsuite_params = { 0 };
@@ -644,6 +652,7 @@ prepare_sgl_bufs(const char *test_buf, struct rte_mbuf *head_buf,
data_size = remaining_data;
else
data_size = seg_size;
+
buf_ptr = rte_pktmbuf_append(head_buf, data_size);
if (buf_ptr == NULL) {
RTE_LOG(ERR, USER1,
@@ -710,6 +719,50 @@ prepare_sgl_bufs(const char *test_buf, struct rte_mbuf *head_buf,
return 0;
}
+
+static int
+test_run_enqueue_dequeue(struct rte_comp_op **ops, unsigned int num_bufs,
+ struct rte_comp_op **ops_processed)
+{
+ uint16_t num_enqd, num_deqd, num_total_deqd;
+ unsigned int deqd_retries = 0;
+
+ /* Enqueue and dequeue all operations */
+ num_enqd = rte_compressdev_enqueue_burst(0, 0, ops, num_bufs);
+ if (num_enqd < num_bufs) {
+ RTE_LOG(ERR, USER1,
+ "Some operations could not be enqueued\n");
+ return -1;
+ }
+
+ num_total_deqd = 0;
+ do {
+ /*
+ * If retrying a dequeue call, wait for 10 ms to allow
+ * enough time to the driver to process the operations
+ */
+ if (deqd_retries != 0) {
+ /*
+ * Avoid infinite loop if not all the
+ * operations get out of the device
+ */
+ if (deqd_retries == MAX_DEQD_RETRIES) {
+ RTE_LOG(ERR, USER1,
+ "Not all operations could be dequeued\n");
+ return -1;
+ }
+ usleep(DEQUEUE_WAIT_TIME);
+ }
+ num_deqd = rte_compressdev_dequeue_burst(0, 0,
+ &ops_processed[num_total_deqd], num_bufs);
+ num_total_deqd += num_deqd;
+ deqd_retries++;
+
+ } while (num_total_deqd < num_enqd);
+
+ return 0;
+}
+
/*
* Compresses and decompresses buffer with compressdev API and Zlib API
*/
@@ -729,6 +782,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
unsigned int out_of_space = test_data->out_of_space;
unsigned int big_data = test_data->big_data;
enum zlib_direction zlib_dir = test_data->zlib_dir;
+ enum overflow_test overflow_tst = test_data->overflow;
int ret_status = -1;
int ret;
struct rte_mbuf *uncomp_bufs[num_bufs];
@@ -754,6 +808,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
rte_compressdev_capability_get(0, RTE_COMP_ALGO_DEFLATE);
char *contig_buf = NULL;
uint64_t compress_checksum[num_bufs];
+ uint32_t compressed_data_size[num_bufs];
if (capa == NULL) {
RTE_LOG(ERR, USER1,
@@ -767,6 +822,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
memset(ops, 0, sizeof(struct rte_comp_op *) * num_bufs);
memset(ops_processed, 0, sizeof(struct rte_comp_op *) * num_bufs);
memset(priv_xforms, 0, sizeof(void *) * num_bufs);
+ memset(compressed_data_size, 0, sizeof(uint32_t) * num_bufs);
if (big_data)
buf_pool = ts_params->big_mbuf_pool;
@@ -799,7 +855,13 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
} else {
for (i = 0; i < num_bufs; i++) {
data_size = strlen(test_bufs[i]) + 1;
+
buf_ptr = rte_pktmbuf_append(uncomp_bufs[i], data_size);
+ if (buf_ptr == NULL) {
+ RTE_LOG(ERR, USER1,
+ "Append extra bytes to the source mbuf failed\n");
+ goto exit;
+ }
strlcpy(buf_ptr, test_bufs[i], data_size);
}
}
@@ -835,11 +897,22 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
for (i = 0; i < num_bufs; i++) {
if (out_of_space == 1 && oos_zlib_decompress)
data_size = OUT_OF_SPACE_BUF;
- else
- (data_size = strlen(test_bufs[i]) *
- COMPRESS_BUF_SIZE_RATIO);
-
- rte_pktmbuf_append(comp_bufs[i], data_size);
+ else {
+ float ratio =
+ ((test_data->zlib_dir == ZLIB_DECOMPRESS ||
+ test_data->zlib_dir == ZLIB_NONE) &&
+ overflow_tst == OVERFLOW_ENABLED) ?
+ COMPRESS_BUF_SIZE_RATIO_OVERFLOW :
+ COMPRESS_BUF_SIZE_RATIO;
+
+ data_size = strlen(test_bufs[i]) * ratio;
+ }
+ buf_ptr = rte_pktmbuf_append(comp_bufs[i], data_size);
+ if (buf_ptr == NULL) {
+ RTE_LOG(ERR, USER1,
+ "Append extra bytes to the destination mbuf failed\n");
+ goto exit;
+ }
}
}
@@ -852,7 +925,6 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
goto exit;
}
-
for (i = 0; i < num_bufs; i++) {
ops[i]->m_src = uncomp_bufs[i];
ops[i]->m_dst = comp_bufs[i];
@@ -928,41 +1000,41 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
for (i = 0; i < num_bufs; i++)
ops[i]->private_xform = priv_xforms[i];
}
-
- /* Enqueue and dequeue all operations */
- num_enqd = rte_compressdev_enqueue_burst(0, 0, ops, num_bufs);
- if (num_enqd < num_bufs) {
+recovery_lb:
+ ret = test_run_enqueue_dequeue(ops, num_bufs, ops_processed);
+ if (ret < 0) {
RTE_LOG(ERR, USER1,
- "The operations could not be enqueued\n");
+ "Enqueue/dequeue operation failed\n");
goto exit;
}
- num_total_deqd = 0;
- do {
- /*
- * If retrying a dequeue call, wait for 10 ms to allow
- * enough time to the driver to process the operations
- */
- if (deqd_retries != 0) {
- /*
- * Avoid infinite loop if not all the
- * operations get out of the device
- */
- if (deqd_retries == MAX_DEQD_RETRIES) {
+ for (i = 0; i < num_bufs; i++) {
+ compressed_data_size[i] += ops_processed[i]->produced;
+
+ if (ops_processed[i]->status ==
+ RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE) {
+
+ ops[i]->status =
+ RTE_COMP_OP_STATUS_NOT_PROCESSED;
+ ops[i]->src.offset +=
+ ops_processed[i]->consumed;
+ ops[i]->src.length -=
+ ops_processed[i]->consumed;
+ ops[i]->dst.offset +=
+ ops_processed[i]->produced;
+
+ buf_ptr = rte_pktmbuf_append(
+ ops[i]->m_dst,
+ ops_processed[i]->produced);
+
+ if (buf_ptr == NULL) {
RTE_LOG(ERR, USER1,
- "Not all operations could be "
- "dequeued\n");
+ "Data recovery: append extra bytes to the current mbuf failed\n");
goto exit;
}
- usleep(DEQUEUE_WAIT_TIME);
+ goto recovery_lb;
}
- num_deqd = rte_compressdev_dequeue_burst(0, 0,
- &ops_processed[num_total_deqd], num_bufs);
- num_total_deqd += num_deqd;
- deqd_retries++;
-
- } while (num_total_deqd < num_enqd);
-
+ }
deqd_retries = 0;
/* Free compress private xforms */
@@ -1006,7 +1078,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
for (i = 0; i < num_bufs; i++) {
if (out_of_space && oos_zlib_decompress) {
if (ops_processed[i]->status !=
- RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED) {
+ RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED) {
ret_status = -1;
RTE_LOG(ERR, USER1,
@@ -1071,7 +1143,13 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
data_size =
strlen(test_bufs[priv_data->orig_idx]) + 1;
- rte_pktmbuf_append(uncomp_bufs[i], data_size);
+ buf_ptr = rte_pktmbuf_append(uncomp_bufs[i], data_size);
+ if (buf_ptr == NULL) {
+ RTE_LOG(ERR, USER1,
+ "Append extra bytes to the decompressed mbuf failed\n");
+ goto exit;
+ }
+
}
}
@@ -1093,7 +1171,12 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
* Set the length of the compressed data to the
* number of bytes that were produced in the previous stage
*/
- ops[i]->src.length = ops_processed[i]->produced;
+
+ if (compressed_data_size[i])
+ ops[i]->src.length = compressed_data_size[i];
+ else
+ ops[i]->src.length = ops_processed[i]->produced;
+
ops[i]->dst.offset = 0;
if (state == RTE_COMP_OP_STATELESS) {
ops[i]->flush_flag = RTE_COMP_FLUSH_FINAL;
@@ -1147,7 +1230,8 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
num_priv_xforms++;
}
- if (capa->comp_feature_flags & RTE_COMP_FF_SHAREABLE_PRIV_XFORM) {
+ if (capa->comp_feature_flags &
+ RTE_COMP_FF_SHAREABLE_PRIV_XFORM) {
/* Attach shareable private xform data to ops */
for (i = 0; i < num_bufs; i++) {
priv_data = (struct priv_op_data *)(ops[i] + 1);
@@ -1206,9 +1290,10 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
usleep(DEQUEUE_WAIT_TIME);
}
num_deqd = rte_compressdev_dequeue_burst(0, 0,
- &ops_processed[num_total_deqd], num_bufs);
+ &ops_processed[num_total_deqd], num_bufs);
num_total_deqd += num_deqd;
deqd_retries++;
+
} while (num_total_deqd < num_enqd);
deqd_retries = 0;
@@ -1233,7 +1318,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
for (i = 0; i < num_bufs; i++) {
if (out_of_space && oos_zlib_compress) {
if (ops_processed[i]->status !=
- RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED) {
+ RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED) {
ret_status = -1;
RTE_LOG(ERR, USER1,
@@ -1358,7 +1443,8 @@ test_compressdev_deflate_stateless_fixed(void)
LB_BOTH,
ZLIB_DECOMPRESS,
0,
- 0
+ 0,
+ OVERFLOW_DISABLED
};
for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
@@ -1429,7 +1515,8 @@ test_compressdev_deflate_stateless_dynamic(void)
LB_BOTH,
ZLIB_DECOMPRESS,
0,
- 0
+ 0,
+ OVERFLOW_DISABLED
};
for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
@@ -1483,7 +1570,8 @@ test_compressdev_deflate_stateless_multi_op(void)
LB_BOTH,
ZLIB_DECOMPRESS,
0,
- 0
+ 0,
+ OVERFLOW_DISABLED
};
/* Compress with compressdev, decompress with Zlib */
@@ -1533,7 +1621,8 @@ test_compressdev_deflate_stateless_multi_level(void)
LB_BOTH,
ZLIB_DECOMPRESS,
0,
- 0
+ 0,
+ OVERFLOW_DISABLED
};
for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
@@ -1623,7 +1712,8 @@ test_compressdev_deflate_stateless_multi_xform(void)
LB_BOTH,
ZLIB_DECOMPRESS,
0,
- 0
+ 0,
+ OVERFLOW_DISABLED
};
/* Compress with compressdev, decompress with Zlib */
@@ -1669,7 +1759,8 @@ test_compressdev_deflate_stateless_sgl(void)
SGL_BOTH,
ZLIB_DECOMPRESS,
0,
- 0
+ 0,
+ OVERFLOW_DISABLED
};
for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
@@ -1777,7 +1868,8 @@ test_compressdev_deflate_stateless_checksum(void)
LB_BOTH,
ZLIB_DECOMPRESS,
0,
- 0
+ 0,
+ OVERFLOW_DISABLED
};
/* Check if driver supports crc32 checksum and test */
@@ -1908,7 +2000,8 @@ test_compressdev_out_of_space_buffer(void)
LB_BOTH,
ZLIB_DECOMPRESS,
1, /* run out-of-space test */
- 0
+ 0,
+ OVERFLOW_DISABLED
};
/* Compress with compressdev, decompress with Zlib */
test_data.zlib_dir = ZLIB_DECOMPRESS;
@@ -1989,7 +2082,8 @@ test_compressdev_deflate_stateless_dynamic_big(void)
SGL_BOTH,
ZLIB_DECOMPRESS,
0,
- 1
+ 1,
+ OVERFLOW_DISABLED
};
ts_params->def_comp_xform->compress.deflate.huffman =
@@ -2023,6 +2117,78 @@ test_compressdev_deflate_stateless_dynamic_big(void)
}
+static int
+test_compressdev_deflate_stateless_fixed_overflow(void)
+{
+ struct comp_testsuite_params *ts_params = &testsuite_params;
+ uint16_t i;
+ int ret;
+ const struct rte_compressdev_capabilities *capab;
+
+ capab = rte_compressdev_capability_get(0, RTE_COMP_ALGO_DEFLATE);
+ TEST_ASSERT(capab != NULL, "Failed to retrieve device capabilities");
+
+ if ((capab->comp_feature_flags & RTE_COMP_FF_HUFFMAN_FIXED) == 0)
+ return -ENOTSUP;
+
+ struct rte_comp_xform *compress_xform =
+ rte_malloc(NULL, sizeof(struct rte_comp_xform), 0);
+
+ if (compress_xform == NULL) {
+ RTE_LOG(ERR, USER1,
+ "Compress xform could not be created\n");
+ ret = TEST_FAILED;
+ goto exit;
+ }
+
+ memcpy(compress_xform, ts_params->def_comp_xform,
+ sizeof(struct rte_comp_xform));
+ compress_xform->compress.deflate.huffman = RTE_COMP_HUFFMAN_FIXED;
+
+ struct interim_data_params int_data = {
+ NULL,
+ 1,
+ NULL,
+ &compress_xform,
+ &ts_params->def_decomp_xform,
+ 1
+ };
+
+ struct test_data_params test_data = {
+ RTE_COMP_OP_STATELESS,
+ LB_BOTH,
+ ZLIB_DECOMPRESS,
+ 0,
+ 0,
+ OVERFLOW_ENABLED
+ };
+
+ for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
+ int_data.test_bufs = &compress_test_bufs[i];
+ int_data.buf_idx = &i;
+
+ /* Compress with compressdev, decompress with Zlib */
+ test_data.zlib_dir = ZLIB_DECOMPRESS;
+ if (test_deflate_comp_decomp(&int_data, &test_data) < 0) {
+ ret = TEST_FAILED;
+ goto exit;
+ }
+
+ /* Compress with Zlib, decompress with compressdev */
+ test_data.zlib_dir = ZLIB_COMPRESS;
+ if (test_deflate_comp_decomp(&int_data, &test_data) < 0) {
+ ret = TEST_FAILED;
+ goto exit;
+ }
+ }
+
+ ret = TEST_SUCCESS;
+
+exit:
+ rte_free(compress_xform);
+ return ret;
+}
+
static struct unit_test_suite compressdev_testsuite = {
.suite_name = "compressdev unit test suite",
.setup = testsuite_setup,
@@ -2048,6 +2214,8 @@ static struct unit_test_suite compressdev_testsuite = {
test_compressdev_deflate_stateless_checksum),
TEST_CASE_ST(generic_ut_setup, generic_ut_teardown,
test_compressdev_out_of_space_buffer),
+ TEST_CASE_ST(generic_ut_setup, generic_ut_teardown,
+ test_compressdev_deflate_stateless_fixed_overflow),
TEST_CASES_END() /**< NULL terminate unit test array */
}
};
--
2.17.1
^ permalink raw reply [flat|nested] 16+ messages in thread
* [dpdk-dev] [PATCH v2 0/2] Add overflow recovery for stateless compression
2019-09-06 11:11 [dpdk-dev] [PATCH 1/2] compress/qat: overflow catching Artur Trybula
2019-09-06 11:11 ` [dpdk-dev] [PATCH 2/2] test/compress: unit test for stateless overflow recovery Artur Trybula
@ 2019-09-26 9:32 ` Artur Trybula
2019-09-26 9:32 ` [dpdk-dev] [PATCH v2 1/2] compress/qat: overflow catch " Artur Trybula
` (2 more replies)
1 sibling, 3 replies; 16+ messages in thread
From: Artur Trybula @ 2019-09-26 9:32 UTC (permalink / raw)
To: dev, fiona.trahe, shallyv, adamx.dybkowski, arturx.trybula, akhil.goyal
This patch adds out-of-space recoverable feature for stateless
compression scenario to QuickAssist compressdev driver. Unit test
was also created to verify the new feature.
v2:
- rebasing over the external mbufs
This patch depends on http://patches.dpdk.org/cover/59680/
Artur Trybula (2):
compress/qat: overflow catch for stateless compression
test/compress: unit test for stateless overflow recovery
app/test/test_compressdev.c | 261 ++++++++++++++++++++++++++------
drivers/compress/qat/qat_comp.c | 28 +++-
2 files changed, 240 insertions(+), 49 deletions(-)
--
2.17.1
^ permalink raw reply [flat|nested] 16+ messages in thread
* [dpdk-dev] [PATCH v2 1/2] compress/qat: overflow catch for stateless compression
2019-09-26 9:32 ` [dpdk-dev] [PATCH v2 0/2] Add overflow recovery for stateless compression Artur Trybula
@ 2019-09-26 9:32 ` Artur Trybula
2019-09-26 9:32 ` [dpdk-dev] [PATCH v2 2/2] test/compress: unit test for stateless overflow recovery Artur Trybula
2019-10-11 17:46 ` [dpdk-dev] [PATCH v3 0/2] Add overflow recovery for stateless compression Artur Trybula
2 siblings, 0 replies; 16+ messages in thread
From: Artur Trybula @ 2019-09-26 9:32 UTC (permalink / raw)
To: dev, fiona.trahe, shallyv, adamx.dybkowski, arturx.trybula, akhil.goyal
This patch adds out-of-space recoverable feature for stateless
compression scenario to QuickAssist compressdev driver
Signed-off-by: Artur Trybula <arturx.trybula@intel.com>
---
drivers/compress/qat/qat_comp.c | 28 +++++++++++++++++++++++++---
1 file changed, 25 insertions(+), 3 deletions(-)
diff --git a/drivers/compress/qat/qat_comp.c b/drivers/compress/qat/qat_comp.c
index 8717b7432..533e34f6b 100644
--- a/drivers/compress/qat/qat_comp.c
+++ b/drivers/compress/qat/qat_comp.c
@@ -324,10 +324,32 @@ qat_comp_process_response(void **op, uint8_t *resp, void *op_cookie,
(!cmp_err_code && xlat_err_code == ERR_CODE_OVERFLOW_ERROR)
||
(cmp_err_code == ERR_CODE_OVERFLOW_ERROR &&
- xlat_err_code == ERR_CODE_OVERFLOW_ERROR))
- rx_op->status =
+ xlat_err_code == ERR_CODE_OVERFLOW_ERROR)){
+
+ struct icp_qat_fw_resp_comp_pars *comp_resp =
+ (struct icp_qat_fw_resp_comp_pars *)&resp_msg->comp_resp_pars;
+
+ /* handle recoverable out-of-buffer condition */
+ /* in stateless compression scenario */
+ if (comp_resp->input_byte_counter) {
+ if ((qat_xform->qat_comp_request_type
+ == QAT_COMP_REQUEST_FIXED_COMP_STATELESS) ||
+ (qat_xform->qat_comp_request_type
+ == QAT_COMP_REQUEST_DYNAMIC_COMP_STATELESS)) {
+
+ rx_op->status =
+ RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE;
+ rx_op->consumed =
+ comp_resp->input_byte_counter;
+ rx_op->produced =
+ comp_resp->output_byte_counter;
+ } else
+ rx_op->status =
+ RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
+ } else
+ rx_op->status =
RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
- else
+ } else
rx_op->status = RTE_COMP_OP_STATUS_ERROR;
++(*dequeue_err_count);
--
2.17.1
^ permalink raw reply [flat|nested] 16+ messages in thread
* [dpdk-dev] [PATCH v2 2/2] test/compress: unit test for stateless overflow recovery
2019-09-26 9:32 ` [dpdk-dev] [PATCH v2 0/2] Add overflow recovery for stateless compression Artur Trybula
2019-09-26 9:32 ` [dpdk-dev] [PATCH v2 1/2] compress/qat: overflow catch " Artur Trybula
@ 2019-09-26 9:32 ` Artur Trybula
2019-10-11 17:46 ` [dpdk-dev] [PATCH v3 0/2] Add overflow recovery for stateless compression Artur Trybula
2 siblings, 0 replies; 16+ messages in thread
From: Artur Trybula @ 2019-09-26 9:32 UTC (permalink / raw)
To: dev, fiona.trahe, shallyv, adamx.dybkowski, arturx.trybula, akhil.goyal
Added unit test to check out-of-space recoverable feature.
Signed-off-by: Artur Trybula <arturx.trybula@intel.com>
---
app/test/test_compressdev.c | 261 +++++++++++++++++++++++++++++-------
1 file changed, 215 insertions(+), 46 deletions(-)
diff --git a/app/test/test_compressdev.c b/app/test/test_compressdev.c
index d470c34b9..0a5c76721 100644
--- a/app/test/test_compressdev.c
+++ b/app/test/test_compressdev.c
@@ -30,6 +30,7 @@
* due to the compress block headers
*/
#define COMPRESS_BUF_SIZE_RATIO 1.3
+#define COMPRESS_BUF_SIZE_RATIO_OVERFLOW 0.2
#define NUM_LARGE_MBUFS 16
#define SMALL_SEG_SIZE 256
#define MAX_SEGS 16
@@ -72,6 +73,11 @@ enum varied_buff {
LB_TO_SGL /* output buffer is chained */
};
+enum overflow_test {
+ OVERFLOW_DISABLED,
+ OVERFLOW_ENABLED
+};
+
struct priv_op_data {
uint16_t orig_idx;
};
@@ -110,6 +116,8 @@ struct test_data_params {
const struct rte_memzone *inbuf_memzone;
const struct rte_memzone *compbuf_memzone;
const struct rte_memzone *uncompbuf_memzone;
+ /* overflow test activation */
+ enum overflow_test overflow;
};
static struct comp_testsuite_params testsuite_params = { 0 };
@@ -725,6 +733,49 @@ extbuf_free_callback(void *addr __rte_unused, void *opaque __rte_unused)
{
}
+static int
+test_run_enqueue_dequeue(struct rte_comp_op **ops, unsigned int num_bufs,
+ struct rte_comp_op **ops_processed)
+{
+ uint16_t num_enqd, num_deqd, num_total_deqd;
+ unsigned int deqd_retries = 0;
+
+ /* Enqueue and dequeue all operations */
+ num_enqd = rte_compressdev_enqueue_burst(0, 0, ops, num_bufs);
+ if (num_enqd < num_bufs) {
+ RTE_LOG(ERR, USER1,
+ "Some operations could not be enqueued\n");
+ return -1;
+ }
+
+ num_total_deqd = 0;
+ do {
+ /*
+ * If retrying a dequeue call, wait for 10 ms to allow
+ * enough time to the driver to process the operations
+ */
+ if (deqd_retries != 0) {
+ /*
+ * Avoid infinite loop if not all the
+ * operations get out of the device
+ */
+ if (deqd_retries == MAX_DEQD_RETRIES) {
+ RTE_LOG(ERR, USER1,
+ "Not all operations could be dequeued\n");
+ return -1;
+ }
+ usleep(DEQUEUE_WAIT_TIME);
+ }
+ num_deqd = rte_compressdev_dequeue_burst(0, 0,
+ &ops_processed[num_total_deqd], num_bufs);
+ num_total_deqd += num_deqd;
+ deqd_retries++;
+
+ } while (num_total_deqd < num_enqd);
+
+ return 0;
+}
+
/*
* Compresses and decompresses buffer with compressdev API and Zlib API
*/
@@ -745,6 +796,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
unsigned int out_of_space = test_data->out_of_space;
unsigned int big_data = test_data->big_data;
enum zlib_direction zlib_dir = test_data->zlib_dir;
+ enum overflow_test overflow_tst = test_data->overflow;
int ret_status = TEST_FAILED;
struct rte_mbuf_ext_shared_info inbuf_info;
struct rte_mbuf_ext_shared_info compbuf_info;
@@ -773,6 +825,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
rte_compressdev_capability_get(0, RTE_COMP_ALGO_DEFLATE);
char *contig_buf = NULL;
uint64_t compress_checksum[num_bufs];
+ uint32_t compressed_data_size[num_bufs];
void *stream = NULL;
char *all_decomp_data = NULL;
unsigned int decomp_produced_data_size = 0;
@@ -793,6 +846,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
memset(ops, 0, sizeof(struct rte_comp_op *) * num_bufs);
memset(ops_processed, 0, sizeof(struct rte_comp_op *) * num_bufs);
memset(priv_xforms, 0, sizeof(void *) * num_bufs);
+ memset(compressed_data_size, 0, sizeof(uint32_t) * num_bufs);
if (decompress_state == RTE_COMP_OP_STATEFUL) {
data_size = strlen(test_bufs[0]) + 1;
@@ -845,6 +899,11 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
for (i = 0; i < num_bufs; i++) {
data_size = strlen(test_bufs[i]) + 1;
buf_ptr = rte_pktmbuf_append(uncomp_bufs[i], data_size);
+ if (buf_ptr == NULL) {
+ RTE_LOG(ERR, USER1,
+ "Append extra bytes to the source mbuf failed\n");
+ goto exit;
+ }
strlcpy(buf_ptr, test_bufs[i], data_size);
}
}
@@ -893,11 +952,23 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
for (i = 0; i < num_bufs; i++) {
if (out_of_space == 1 && oos_zlib_decompress)
data_size = OUT_OF_SPACE_BUF;
- else
- (data_size = strlen(test_bufs[i]) *
- COMPRESS_BUF_SIZE_RATIO);
+ else {
+ float ratio =
+ ((test_data->zlib_dir == ZLIB_DECOMPRESS ||
+ test_data->zlib_dir == ZLIB_NONE) &&
+ overflow_tst == OVERFLOW_ENABLED) ?
+ COMPRESS_BUF_SIZE_RATIO_OVERFLOW :
+ COMPRESS_BUF_SIZE_RATIO;
+
+ data_size = strlen(test_bufs[i]) * ratio;
+ }
+ buf_ptr = rte_pktmbuf_append(comp_bufs[i], data_size);
+ if (buf_ptr == NULL) {
+ RTE_LOG(ERR, USER1,
+ "Append extra bytes to the destination mbuf failed\n");
+ goto exit;
+ }
- rte_pktmbuf_append(comp_bufs[i], data_size);
}
}
@@ -963,7 +1034,8 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
num_priv_xforms++;
}
- if (capa->comp_feature_flags & RTE_COMP_FF_SHAREABLE_PRIV_XFORM) {
+ if (capa->comp_feature_flags &
+ RTE_COMP_FF_SHAREABLE_PRIV_XFORM) {
/* Attach shareable private xform data to ops */
for (i = 0; i < num_bufs; i++)
ops[i]->private_xform = priv_xforms[i % num_xforms];
@@ -987,40 +1059,41 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
ops[i]->private_xform = priv_xforms[i];
}
- /* Enqueue and dequeue all operations */
- num_enqd = rte_compressdev_enqueue_burst(0, 0, ops, num_bufs);
- if (num_enqd < num_bufs) {
+recovery_lb:
+ ret = test_run_enqueue_dequeue(ops, num_bufs, ops_processed);
+ if (ret < 0) {
RTE_LOG(ERR, USER1,
- "The operations could not be enqueued\n");
+ "Enqueue/dequeue operation failed\n");
goto exit;
}
- num_total_deqd = 0;
- do {
- /*
- * If retrying a dequeue call, wait for 10 ms to allow
- * enough time to the driver to process the operations
- */
- if (deqd_retries != 0) {
- /*
- * Avoid infinite loop if not all the
- * operations get out of the device
- */
- if (deqd_retries == MAX_DEQD_RETRIES) {
+ for (i = 0; i < num_bufs; i++) {
+ compressed_data_size[i] += ops_processed[i]->produced;
+
+ if (ops_processed[i]->status ==
+ RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE) {
+
+ ops[i]->status =
+ RTE_COMP_OP_STATUS_NOT_PROCESSED;
+ ops[i]->src.offset +=
+ ops_processed[i]->consumed;
+ ops[i]->src.length -=
+ ops_processed[i]->consumed;
+ ops[i]->dst.offset +=
+ ops_processed[i]->produced;
+
+ buf_ptr = rte_pktmbuf_append(
+ ops[i]->m_dst,
+ ops_processed[i]->produced);
+
+ if (buf_ptr == NULL) {
RTE_LOG(ERR, USER1,
- "Not all operations could be "
- "dequeued\n");
+ "Data recovery: append extra bytes to the current mbuf failed\n");
goto exit;
}
- usleep(DEQUEUE_WAIT_TIME);
+ goto recovery_lb;
}
- num_deqd = rte_compressdev_dequeue_burst(0, 0,
- &ops_processed[num_total_deqd], num_bufs);
- num_total_deqd += num_deqd;
- deqd_retries++;
-
- } while (num_total_deqd < num_enqd);
-
+ }
deqd_retries = 0;
/* Free compress private xforms */
@@ -1064,7 +1137,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
for (i = 0; i < num_bufs; i++) {
if (out_of_space && oos_zlib_decompress) {
if (ops_processed[i]->status !=
- RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED) {
+ RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED) {
ret_status = TEST_FAILED;
RTE_LOG(ERR, USER1,
"Operation without expected out of "
@@ -1147,7 +1220,12 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
data_size =
strlen(test_bufs[priv_data->orig_idx]) + 1;
- rte_pktmbuf_append(uncomp_bufs[i], data_size);
+ buf_ptr = rte_pktmbuf_append(uncomp_bufs[i], data_size);
+ if (buf_ptr == NULL) {
+ RTE_LOG(ERR, USER1,
+ "Append extra bytes to the decompressed mbuf failed\n");
+ goto exit;
+ }
}
}
@@ -1169,7 +1247,10 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
* Set the length of the compressed data to the
* number of bytes that were produced in the previous stage
*/
- ops[i]->src.length = ops_processed[i]->produced;
+ if (compressed_data_size[i])
+ ops[i]->src.length = compressed_data_size[i];
+ else
+ ops[i]->src.length = ops_processed[i]->produced;
ops[i]->dst.offset = 0;
if (decompress_state == RTE_COMP_OP_STATELESS) {
@@ -1541,7 +1622,8 @@ test_compressdev_deflate_stateless_fixed(void)
.buff_type = LB_BOTH,
.zlib_dir = ZLIB_DECOMPRESS,
.out_of_space = 0,
- .big_data = 0
+ .big_data = 0,
+ .overflow = OVERFLOW_DISABLED
};
for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
@@ -1611,7 +1693,8 @@ test_compressdev_deflate_stateless_dynamic(void)
.buff_type = LB_BOTH,
.zlib_dir = ZLIB_DECOMPRESS,
.out_of_space = 0,
- .big_data = 0
+ .big_data = 0,
+ .overflow = OVERFLOW_DISABLED
};
for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
@@ -1665,7 +1748,8 @@ test_compressdev_deflate_stateless_multi_op(void)
.buff_type = LB_BOTH,
.zlib_dir = ZLIB_DECOMPRESS,
.out_of_space = 0,
- .big_data = 0
+ .big_data = 0,
+ .overflow = OVERFLOW_DISABLED
};
/* Compress with compressdev, decompress with Zlib */
@@ -1718,7 +1802,8 @@ test_compressdev_deflate_stateless_multi_level(void)
.buff_type = LB_BOTH,
.zlib_dir = ZLIB_DECOMPRESS,
.out_of_space = 0,
- .big_data = 0
+ .big_data = 0,
+ .overflow = OVERFLOW_DISABLED
};
for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
@@ -1807,7 +1892,8 @@ test_compressdev_deflate_stateless_multi_xform(void)
.buff_type = LB_BOTH,
.zlib_dir = ZLIB_DECOMPRESS,
.out_of_space = 0,
- .big_data = 0
+ .big_data = 0,
+ .overflow = OVERFLOW_DISABLED
};
/* Compress with compressdev, decompress with Zlib */
@@ -1855,7 +1941,8 @@ test_compressdev_deflate_stateless_sgl(void)
.buff_type = SGL_BOTH,
.zlib_dir = ZLIB_DECOMPRESS,
.out_of_space = 0,
- .big_data = 0
+ .big_data = 0,
+ .overflow = OVERFLOW_DISABLED
};
for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
@@ -1965,7 +2052,8 @@ test_compressdev_deflate_stateless_checksum(void)
.buff_type = LB_BOTH,
.zlib_dir = ZLIB_DECOMPRESS,
.out_of_space = 0,
- .big_data = 0
+ .big_data = 0,
+ .overflow = OVERFLOW_DISABLED
};
/* Check if driver supports crc32 checksum and test */
@@ -2082,7 +2170,8 @@ test_compressdev_out_of_space_buffer(void)
.buff_type = LB_BOTH,
.zlib_dir = ZLIB_DECOMPRESS,
.out_of_space = 1, /* run out-of-space test */
- .big_data = 0
+ .big_data = 0,
+ .overflow = OVERFLOW_DISABLED
};
/* Compress with compressdev, decompress with Zlib */
test_data.zlib_dir = ZLIB_DECOMPRESS;
@@ -2159,7 +2248,8 @@ test_compressdev_deflate_stateless_dynamic_big(void)
.buff_type = SGL_BOTH,
.zlib_dir = ZLIB_DECOMPRESS,
.out_of_space = 0,
- .big_data = 1
+ .big_data = 1,
+ .overflow = OVERFLOW_DISABLED
};
ts_params->def_comp_xform->compress.deflate.huffman =
@@ -2223,7 +2313,8 @@ test_compressdev_deflate_stateful_decomp(void)
.out_of_space = 0,
.big_data = 0,
.decompress_output_block_size = 2000,
- .decompress_steps_max = 4
+ .decompress_steps_max = 4,
+ .overflow = OVERFLOW_DISABLED
};
/* Compress with Zlib, decompress with compressdev */
@@ -2305,7 +2396,8 @@ test_compressdev_deflate_stateful_decomp_checksum(void)
.out_of_space = 0,
.big_data = 0,
.decompress_output_block_size = 2000,
- .decompress_steps_max = 4
+ .decompress_steps_max = 4,
+ .overflow = OVERFLOW_DISABLED
};
/* Check if driver supports crc32 checksum and test */
@@ -2386,6 +2478,7 @@ test_compressdev_deflate_stateful_decomp_checksum(void)
return ret;
}
+
static const struct rte_memzone *
make_memzone(const char *name, size_t size)
{
@@ -2441,7 +2534,8 @@ test_compressdev_external_mbufs(void)
.inbuf_memzone = make_memzone("inbuf", data_len),
.compbuf_memzone = make_memzone("compbuf", data_len *
COMPRESS_BUF_SIZE_RATIO),
- .uncompbuf_memzone = make_memzone("decompbuf", data_len)
+ .uncompbuf_memzone = make_memzone("decompbuf", data_len),
+ .overflow = OVERFLOW_DISABLED
};
for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
@@ -2472,6 +2566,79 @@ test_compressdev_external_mbufs(void)
return ret;
}
+static int
+test_compressdev_deflate_stateless_fixed_overflow(void)
+{
+ struct comp_testsuite_params *ts_params = &testsuite_params;
+ uint16_t i;
+ int ret;
+ const struct rte_compressdev_capabilities *capab;
+
+ capab = rte_compressdev_capability_get(0, RTE_COMP_ALGO_DEFLATE);
+ TEST_ASSERT(capab != NULL, "Failed to retrieve device capabilities");
+
+ if ((capab->comp_feature_flags & RTE_COMP_FF_HUFFMAN_FIXED) == 0)
+ return -ENOTSUP;
+
+ struct rte_comp_xform *compress_xform =
+ rte_malloc(NULL, sizeof(struct rte_comp_xform), 0);
+
+ if (compress_xform == NULL) {
+ RTE_LOG(ERR, USER1,
+ "Compress xform could not be created\n");
+ ret = TEST_FAILED;
+ goto exit;
+ }
+
+ memcpy(compress_xform, ts_params->def_comp_xform,
+ sizeof(struct rte_comp_xform));
+ compress_xform->compress.deflate.huffman = RTE_COMP_HUFFMAN_FIXED;
+
+ struct interim_data_params int_data = {
+ NULL,
+ 1,
+ NULL,
+ &compress_xform,
+ &ts_params->def_decomp_xform,
+ 1
+ };
+
+ struct test_data_params test_data = {
+ .compress_state = RTE_COMP_OP_STATELESS,
+ .decompress_state = RTE_COMP_OP_STATELESS,
+ .buff_type = LB_BOTH,
+ .zlib_dir = ZLIB_DECOMPRESS,
+ .out_of_space = 0,
+ .big_data = 0,
+ .overflow = OVERFLOW_ENABLED
+ };
+
+ for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
+ int_data.test_bufs = &compress_test_bufs[i];
+ int_data.buf_idx = &i;
+
+ /* Compress with compressdev, decompress with Zlib */
+ test_data.zlib_dir = ZLIB_DECOMPRESS;
+ if (test_deflate_comp_decomp(&int_data, &test_data) < 0) {
+ ret = TEST_FAILED;
+ goto exit;
+ }
+
+ /* Compress with Zlib, decompress with compressdev */
+ test_data.zlib_dir = ZLIB_COMPRESS;
+ if (test_deflate_comp_decomp(&int_data, &test_data) < 0) {
+ ret = TEST_FAILED;
+ goto exit;
+ }
+ }
+
+ ret = TEST_SUCCESS;
+
+exit:
+ rte_free(compress_xform);
+ return ret;
+}
+
static struct unit_test_suite compressdev_testsuite = {
.suite_name = "compressdev unit test suite",
.setup = testsuite_setup,
@@ -2503,6 +2670,8 @@ static struct unit_test_suite compressdev_testsuite = {
test_compressdev_deflate_stateful_decomp_checksum),
TEST_CASE_ST(generic_ut_setup, generic_ut_teardown,
test_compressdev_external_mbufs),
+ TEST_CASE_ST(generic_ut_setup, generic_ut_teardown,
+ test_compressdev_deflate_stateless_fixed_overflow),
TEST_CASES_END() /**< NULL terminate unit test array */
}
};
--
2.17.1
^ permalink raw reply [flat|nested] 16+ messages in thread
* [dpdk-dev] [PATCH v3 0/2] Add overflow recovery for stateless compression
2019-09-26 9:32 ` [dpdk-dev] [PATCH v2 0/2] Add overflow recovery for stateless compression Artur Trybula
2019-09-26 9:32 ` [dpdk-dev] [PATCH v2 1/2] compress/qat: overflow catch " Artur Trybula
2019-09-26 9:32 ` [dpdk-dev] [PATCH v2 2/2] test/compress: unit test for stateless overflow recovery Artur Trybula
@ 2019-10-11 17:46 ` Artur Trybula
2019-10-11 17:46 ` [dpdk-dev] [PATCH v3 1/2] compress/qat: overflow catch " Artur Trybula
` (2 more replies)
2 siblings, 3 replies; 16+ messages in thread
From: Artur Trybula @ 2019-10-11 17:46 UTC (permalink / raw)
To: dev, fiona.trahe, shallyv, adamx.dybkowski, arturx.trybula, akhil.goyal
This patch adds out-of-space recoverable feature for stateless
compression scenario to QuickAssist compressdev driver. Unit test
was also created to verify the new feature.
v3:
- rebasing onto the recent master
v2:
- rebasing over the external mbufs
Artur Trybula (2):
compress/qat: overflow catch for stateless compression
test/compress: unit test for stateless overflow recovery
app/test/test_compressdev.c | 261 ++++++++++++++++++++++++++------
drivers/compress/qat/qat_comp.c | 28 +++-
2 files changed, 239 insertions(+), 50 deletions(-)
--
2.17.1
^ permalink raw reply [flat|nested] 16+ messages in thread
* [dpdk-dev] [PATCH v3 1/2] compress/qat: overflow catch for stateless compression
2019-10-11 17:46 ` [dpdk-dev] [PATCH v3 0/2] Add overflow recovery for stateless compression Artur Trybula
@ 2019-10-11 17:46 ` Artur Trybula
2019-10-14 13:38 ` Trahe, Fiona
2019-10-11 17:46 ` [dpdk-dev] [PATCH v3 2/2] test/compress: unit test for stateless overflow recovery Artur Trybula
2019-10-16 9:40 ` [dpdk-dev] [PATCH v4 0/2] Add overflow recovery for stateless compression Artur Trybula
2 siblings, 1 reply; 16+ messages in thread
From: Artur Trybula @ 2019-10-11 17:46 UTC (permalink / raw)
To: dev, fiona.trahe, shallyv, adamx.dybkowski, arturx.trybula, akhil.goyal
This patch adds out-of-space recoverable feature for stateless
compression scenario to QuickAssist compressdev driver
Signed-off-by: Artur Trybula <arturx.trybula@intel.com>
---
drivers/compress/qat/qat_comp.c | 28 +++++++++++++++++++++++++---
1 file changed, 25 insertions(+), 3 deletions(-)
diff --git a/drivers/compress/qat/qat_comp.c b/drivers/compress/qat/qat_comp.c
index 8717b7432..533e34f6b 100644
--- a/drivers/compress/qat/qat_comp.c
+++ b/drivers/compress/qat/qat_comp.c
@@ -324,10 +324,32 @@ qat_comp_process_response(void **op, uint8_t *resp, void *op_cookie,
(!cmp_err_code && xlat_err_code == ERR_CODE_OVERFLOW_ERROR)
||
(cmp_err_code == ERR_CODE_OVERFLOW_ERROR &&
- xlat_err_code == ERR_CODE_OVERFLOW_ERROR))
- rx_op->status =
+ xlat_err_code == ERR_CODE_OVERFLOW_ERROR)){
+
+ struct icp_qat_fw_resp_comp_pars *comp_resp =
+ (struct icp_qat_fw_resp_comp_pars *)&resp_msg->comp_resp_pars;
+
+ /* handle recoverable out-of-buffer condition */
+ /* in stateless compression scenario */
+ if (comp_resp->input_byte_counter) {
+ if ((qat_xform->qat_comp_request_type
+ == QAT_COMP_REQUEST_FIXED_COMP_STATELESS) ||
+ (qat_xform->qat_comp_request_type
+ == QAT_COMP_REQUEST_DYNAMIC_COMP_STATELESS)) {
+
+ rx_op->status =
+ RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE;
+ rx_op->consumed =
+ comp_resp->input_byte_counter;
+ rx_op->produced =
+ comp_resp->output_byte_counter;
+ } else
+ rx_op->status =
+ RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
+ } else
+ rx_op->status =
RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
- else
+ } else
rx_op->status = RTE_COMP_OP_STATUS_ERROR;
++(*dequeue_err_count);
--
2.17.1
^ permalink raw reply [flat|nested] 16+ messages in thread
* [dpdk-dev] [PATCH v3 2/2] test/compress: unit test for stateless overflow recovery
2019-10-11 17:46 ` [dpdk-dev] [PATCH v3 0/2] Add overflow recovery for stateless compression Artur Trybula
2019-10-11 17:46 ` [dpdk-dev] [PATCH v3 1/2] compress/qat: overflow catch " Artur Trybula
@ 2019-10-11 17:46 ` Artur Trybula
2019-10-14 13:48 ` Trahe, Fiona
2019-10-16 9:40 ` [dpdk-dev] [PATCH v4 0/2] Add overflow recovery for stateless compression Artur Trybula
2 siblings, 1 reply; 16+ messages in thread
From: Artur Trybula @ 2019-10-11 17:46 UTC (permalink / raw)
To: dev, fiona.trahe, shallyv, adamx.dybkowski, arturx.trybula, akhil.goyal
Added unit test to check out-of-space recoverable feature.
Signed-off-by: Artur Trybula <arturx.trybula@intel.com>
---
app/test/test_compressdev.c | 261 +++++++++++++++++++++++++++++-------
1 file changed, 214 insertions(+), 47 deletions(-)
diff --git a/app/test/test_compressdev.c b/app/test/test_compressdev.c
index d470c34b9..335deb263 100644
--- a/app/test/test_compressdev.c
+++ b/app/test/test_compressdev.c
@@ -30,6 +30,7 @@
* due to the compress block headers
*/
#define COMPRESS_BUF_SIZE_RATIO 1.3
+#define COMPRESS_BUF_SIZE_RATIO_OVERFLOW 0.2
#define NUM_LARGE_MBUFS 16
#define SMALL_SEG_SIZE 256
#define MAX_SEGS 16
@@ -72,6 +73,11 @@ enum varied_buff {
LB_TO_SGL /* output buffer is chained */
};
+enum overflow_test {
+ OVERFLOW_DISABLED,
+ OVERFLOW_ENABLED
+};
+
struct priv_op_data {
uint16_t orig_idx;
};
@@ -110,6 +116,8 @@ struct test_data_params {
const struct rte_memzone *inbuf_memzone;
const struct rte_memzone *compbuf_memzone;
const struct rte_memzone *uncompbuf_memzone;
+ /* overflow test activation */
+ enum overflow_test overflow;
};
static struct comp_testsuite_params testsuite_params = { 0 };
@@ -725,6 +733,49 @@ extbuf_free_callback(void *addr __rte_unused, void *opaque __rte_unused)
{
}
+static int
+test_run_enqueue_dequeue(struct rte_comp_op **ops, unsigned int num_bufs,
+ struct rte_comp_op **ops_processed)
+{
+ uint16_t num_enqd, num_deqd, num_total_deqd;
+ unsigned int deqd_retries = 0;
+
+ /* Enqueue and dequeue all operations */
+ num_enqd = rte_compressdev_enqueue_burst(0, 0, ops, num_bufs);
+ if (num_enqd < num_bufs) {
+ RTE_LOG(ERR, USER1,
+ "Some operations could not be enqueued\n");
+ return -1;
+ }
+
+ num_total_deqd = 0;
+ do {
+ /*
+ * If retrying a dequeue call, wait for 10 ms to allow
+ * enough time to the driver to process the operations
+ */
+ if (deqd_retries != 0) {
+ /*
+ * Avoid infinite loop if not all the
+ * operations get out of the device
+ */
+ if (deqd_retries == MAX_DEQD_RETRIES) {
+ RTE_LOG(ERR, USER1,
+ "Not all operations could be dequeued\n");
+ return -1;
+ }
+ usleep(DEQUEUE_WAIT_TIME);
+ }
+ num_deqd = rte_compressdev_dequeue_burst(0, 0,
+ &ops_processed[num_total_deqd], num_bufs);
+ num_total_deqd += num_deqd;
+ deqd_retries++;
+
+ } while (num_total_deqd < num_enqd);
+
+ return 0;
+}
+
/*
* Compresses and decompresses buffer with compressdev API and Zlib API
*/
@@ -745,6 +796,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
unsigned int out_of_space = test_data->out_of_space;
unsigned int big_data = test_data->big_data;
enum zlib_direction zlib_dir = test_data->zlib_dir;
+ enum overflow_test overflow_tst = test_data->overflow;
int ret_status = TEST_FAILED;
struct rte_mbuf_ext_shared_info inbuf_info;
struct rte_mbuf_ext_shared_info compbuf_info;
@@ -773,6 +825,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
rte_compressdev_capability_get(0, RTE_COMP_ALGO_DEFLATE);
char *contig_buf = NULL;
uint64_t compress_checksum[num_bufs];
+ uint32_t compressed_data_size[num_bufs];
void *stream = NULL;
char *all_decomp_data = NULL;
unsigned int decomp_produced_data_size = 0;
@@ -793,6 +846,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
memset(ops, 0, sizeof(struct rte_comp_op *) * num_bufs);
memset(ops_processed, 0, sizeof(struct rte_comp_op *) * num_bufs);
memset(priv_xforms, 0, sizeof(void *) * num_bufs);
+ memset(compressed_data_size, 0, sizeof(uint32_t) * num_bufs);
if (decompress_state == RTE_COMP_OP_STATEFUL) {
data_size = strlen(test_bufs[0]) + 1;
@@ -845,6 +899,11 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
for (i = 0; i < num_bufs; i++) {
data_size = strlen(test_bufs[i]) + 1;
buf_ptr = rte_pktmbuf_append(uncomp_bufs[i], data_size);
+ if (buf_ptr == NULL) {
+ RTE_LOG(ERR, USER1,
+ "Append extra bytes to the source mbuf failed\n");
+ goto exit;
+ }
strlcpy(buf_ptr, test_bufs[i], data_size);
}
}
@@ -893,11 +952,22 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
for (i = 0; i < num_bufs; i++) {
if (out_of_space == 1 && oos_zlib_decompress)
data_size = OUT_OF_SPACE_BUF;
- else
- (data_size = strlen(test_bufs[i]) *
- COMPRESS_BUF_SIZE_RATIO);
-
- rte_pktmbuf_append(comp_bufs[i], data_size);
+ else {
+ float ratio =
+ ((test_data->zlib_dir == ZLIB_DECOMPRESS ||
+ test_data->zlib_dir == ZLIB_NONE) &&
+ overflow_tst == OVERFLOW_ENABLED) ?
+ COMPRESS_BUF_SIZE_RATIO_OVERFLOW :
+ COMPRESS_BUF_SIZE_RATIO;
+
+ data_size = strlen(test_bufs[i]) * ratio;
+ }
+ buf_ptr = rte_pktmbuf_append(comp_bufs[i], data_size);
+ if (buf_ptr == NULL) {
+ RTE_LOG(ERR, USER1,
+ "Append extra bytes to the destination mbuf failed\n");
+ goto exit;
+ }
}
}
@@ -963,7 +1033,8 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
num_priv_xforms++;
}
- if (capa->comp_feature_flags & RTE_COMP_FF_SHAREABLE_PRIV_XFORM) {
+ if (capa->comp_feature_flags &
+ RTE_COMP_FF_SHAREABLE_PRIV_XFORM) {
/* Attach shareable private xform data to ops */
for (i = 0; i < num_bufs; i++)
ops[i]->private_xform = priv_xforms[i % num_xforms];
@@ -987,40 +1058,41 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
ops[i]->private_xform = priv_xforms[i];
}
- /* Enqueue and dequeue all operations */
- num_enqd = rte_compressdev_enqueue_burst(0, 0, ops, num_bufs);
- if (num_enqd < num_bufs) {
+recovery_lb:
+ ret = test_run_enqueue_dequeue(ops, num_bufs, ops_processed);
+ if (ret < 0) {
RTE_LOG(ERR, USER1,
- "The operations could not be enqueued\n");
+ "Enqueue/dequeue operation failed\n");
goto exit;
}
- num_total_deqd = 0;
- do {
- /*
- * If retrying a dequeue call, wait for 10 ms to allow
- * enough time to the driver to process the operations
- */
- if (deqd_retries != 0) {
- /*
- * Avoid infinite loop if not all the
- * operations get out of the device
- */
- if (deqd_retries == MAX_DEQD_RETRIES) {
+ for (i = 0; i < num_bufs; i++) {
+ compressed_data_size[i] += ops_processed[i]->produced;
+
+ if (ops_processed[i]->status ==
+ RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE) {
+
+ ops[i]->status =
+ RTE_COMP_OP_STATUS_NOT_PROCESSED;
+ ops[i]->src.offset +=
+ ops_processed[i]->consumed;
+ ops[i]->src.length -=
+ ops_processed[i]->consumed;
+ ops[i]->dst.offset +=
+ ops_processed[i]->produced;
+
+ buf_ptr = rte_pktmbuf_append(
+ ops[i]->m_dst,
+ ops_processed[i]->produced);
+
+ if (buf_ptr == NULL) {
RTE_LOG(ERR, USER1,
- "Not all operations could be "
- "dequeued\n");
+ "Data recovery: append extra bytes to the current mbuf failed\n");
goto exit;
}
- usleep(DEQUEUE_WAIT_TIME);
+ goto recovery_lb;
}
- num_deqd = rte_compressdev_dequeue_burst(0, 0,
- &ops_processed[num_total_deqd], num_bufs);
- num_total_deqd += num_deqd;
- deqd_retries++;
-
- } while (num_total_deqd < num_enqd);
-
+ }
deqd_retries = 0;
/* Free compress private xforms */
@@ -1064,7 +1136,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
for (i = 0; i < num_bufs; i++) {
if (out_of_space && oos_zlib_decompress) {
if (ops_processed[i]->status !=
- RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED) {
+ RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED) {
ret_status = TEST_FAILED;
RTE_LOG(ERR, USER1,
"Operation without expected out of "
@@ -1147,7 +1219,12 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
data_size =
strlen(test_bufs[priv_data->orig_idx]) + 1;
- rte_pktmbuf_append(uncomp_bufs[i], data_size);
+ buf_ptr = rte_pktmbuf_append(uncomp_bufs[i], data_size);
+ if (buf_ptr == NULL) {
+ RTE_LOG(ERR, USER1,
+ "Append extra bytes to the decompressed mbuf failed\n");
+ goto exit;
+ }
}
}
@@ -1169,7 +1246,10 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
* Set the length of the compressed data to the
* number of bytes that were produced in the previous stage
*/
- ops[i]->src.length = ops_processed[i]->produced;
+ if (compressed_data_size[i])
+ ops[i]->src.length = compressed_data_size[i];
+ else
+ ops[i]->src.length = ops_processed[i]->produced;
ops[i]->dst.offset = 0;
if (decompress_state == RTE_COMP_OP_STATELESS) {
@@ -1541,7 +1621,8 @@ test_compressdev_deflate_stateless_fixed(void)
.buff_type = LB_BOTH,
.zlib_dir = ZLIB_DECOMPRESS,
.out_of_space = 0,
- .big_data = 0
+ .big_data = 0,
+ .overflow = OVERFLOW_DISABLED
};
for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
@@ -1611,7 +1692,8 @@ test_compressdev_deflate_stateless_dynamic(void)
.buff_type = LB_BOTH,
.zlib_dir = ZLIB_DECOMPRESS,
.out_of_space = 0,
- .big_data = 0
+ .big_data = 0,
+ .overflow = OVERFLOW_DISABLED
};
for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
@@ -1665,7 +1747,8 @@ test_compressdev_deflate_stateless_multi_op(void)
.buff_type = LB_BOTH,
.zlib_dir = ZLIB_DECOMPRESS,
.out_of_space = 0,
- .big_data = 0
+ .big_data = 0,
+ .overflow = OVERFLOW_DISABLED
};
/* Compress with compressdev, decompress with Zlib */
@@ -1718,7 +1801,8 @@ test_compressdev_deflate_stateless_multi_level(void)
.buff_type = LB_BOTH,
.zlib_dir = ZLIB_DECOMPRESS,
.out_of_space = 0,
- .big_data = 0
+ .big_data = 0,
+ .overflow = OVERFLOW_DISABLED
};
for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
@@ -1807,7 +1891,8 @@ test_compressdev_deflate_stateless_multi_xform(void)
.buff_type = LB_BOTH,
.zlib_dir = ZLIB_DECOMPRESS,
.out_of_space = 0,
- .big_data = 0
+ .big_data = 0,
+ .overflow = OVERFLOW_DISABLED
};
/* Compress with compressdev, decompress with Zlib */
@@ -1855,7 +1940,8 @@ test_compressdev_deflate_stateless_sgl(void)
.buff_type = SGL_BOTH,
.zlib_dir = ZLIB_DECOMPRESS,
.out_of_space = 0,
- .big_data = 0
+ .big_data = 0,
+ .overflow = OVERFLOW_DISABLED
};
for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
@@ -1965,7 +2051,8 @@ test_compressdev_deflate_stateless_checksum(void)
.buff_type = LB_BOTH,
.zlib_dir = ZLIB_DECOMPRESS,
.out_of_space = 0,
- .big_data = 0
+ .big_data = 0,
+ .overflow = OVERFLOW_DISABLED
};
/* Check if driver supports crc32 checksum and test */
@@ -2082,7 +2169,8 @@ test_compressdev_out_of_space_buffer(void)
.buff_type = LB_BOTH,
.zlib_dir = ZLIB_DECOMPRESS,
.out_of_space = 1, /* run out-of-space test */
- .big_data = 0
+ .big_data = 0,
+ .overflow = OVERFLOW_DISABLED
};
/* Compress with compressdev, decompress with Zlib */
test_data.zlib_dir = ZLIB_DECOMPRESS;
@@ -2159,7 +2247,8 @@ test_compressdev_deflate_stateless_dynamic_big(void)
.buff_type = SGL_BOTH,
.zlib_dir = ZLIB_DECOMPRESS,
.out_of_space = 0,
- .big_data = 1
+ .big_data = 1,
+ .overflow = OVERFLOW_DISABLED
};
ts_params->def_comp_xform->compress.deflate.huffman =
@@ -2223,7 +2312,8 @@ test_compressdev_deflate_stateful_decomp(void)
.out_of_space = 0,
.big_data = 0,
.decompress_output_block_size = 2000,
- .decompress_steps_max = 4
+ .decompress_steps_max = 4,
+ .overflow = OVERFLOW_DISABLED
};
/* Compress with Zlib, decompress with compressdev */
@@ -2305,7 +2395,8 @@ test_compressdev_deflate_stateful_decomp_checksum(void)
.out_of_space = 0,
.big_data = 0,
.decompress_output_block_size = 2000,
- .decompress_steps_max = 4
+ .decompress_steps_max = 4,
+ .overflow = OVERFLOW_DISABLED
};
/* Check if driver supports crc32 checksum and test */
@@ -2441,7 +2532,8 @@ test_compressdev_external_mbufs(void)
.inbuf_memzone = make_memzone("inbuf", data_len),
.compbuf_memzone = make_memzone("compbuf", data_len *
COMPRESS_BUF_SIZE_RATIO),
- .uncompbuf_memzone = make_memzone("decompbuf", data_len)
+ .uncompbuf_memzone = make_memzone("decompbuf", data_len),
+ .overflow = OVERFLOW_DISABLED
};
for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
@@ -2472,6 +2564,79 @@ test_compressdev_external_mbufs(void)
return ret;
}
+static int
+test_compressdev_deflate_stateless_fixed_overflow(void)
+{
+ struct comp_testsuite_params *ts_params = &testsuite_params;
+ uint16_t i;
+ int ret;
+ const struct rte_compressdev_capabilities *capab;
+
+ capab = rte_compressdev_capability_get(0, RTE_COMP_ALGO_DEFLATE);
+ TEST_ASSERT(capab != NULL, "Failed to retrieve device capabilities");
+
+ if ((capab->comp_feature_flags & RTE_COMP_FF_HUFFMAN_FIXED) == 0)
+ return -ENOTSUP;
+
+ struct rte_comp_xform *compress_xform =
+ rte_malloc(NULL, sizeof(struct rte_comp_xform), 0);
+
+ if (compress_xform == NULL) {
+ RTE_LOG(ERR, USER1,
+ "Compress xform could not be created\n");
+ ret = TEST_FAILED;
+ goto exit;
+ }
+
+ memcpy(compress_xform, ts_params->def_comp_xform,
+ sizeof(struct rte_comp_xform));
+ compress_xform->compress.deflate.huffman = RTE_COMP_HUFFMAN_FIXED;
+
+ struct interim_data_params int_data = {
+ NULL,
+ 1,
+ NULL,
+ &compress_xform,
+ &ts_params->def_decomp_xform,
+ 1
+ };
+
+ struct test_data_params test_data = {
+ .compress_state = RTE_COMP_OP_STATELESS,
+ .decompress_state = RTE_COMP_OP_STATELESS,
+ .buff_type = LB_BOTH,
+ .zlib_dir = ZLIB_DECOMPRESS,
+ .out_of_space = 0,
+ .big_data = 0,
+ .overflow = OVERFLOW_ENABLED
+ };
+
+ for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
+ int_data.test_bufs = &compress_test_bufs[i];
+ int_data.buf_idx = &i;
+
+ /* Compress with compressdev, decompress with Zlib */
+ test_data.zlib_dir = ZLIB_DECOMPRESS;
+ if (test_deflate_comp_decomp(&int_data, &test_data) < 0) {
+ ret = TEST_FAILED;
+ goto exit;
+ }
+
+ /* Compress with Zlib, decompress with compressdev */
+ test_data.zlib_dir = ZLIB_COMPRESS;
+ if (test_deflate_comp_decomp(&int_data, &test_data) < 0) {
+ ret = TEST_FAILED;
+ goto exit;
+ }
+ }
+
+ ret = TEST_SUCCESS;
+
+exit:
+ rte_free(compress_xform);
+ return ret;
+}
+
static struct unit_test_suite compressdev_testsuite = {
.suite_name = "compressdev unit test suite",
.setup = testsuite_setup,
@@ -2503,6 +2668,8 @@ static struct unit_test_suite compressdev_testsuite = {
test_compressdev_deflate_stateful_decomp_checksum),
TEST_CASE_ST(generic_ut_setup, generic_ut_teardown,
test_compressdev_external_mbufs),
+ TEST_CASE_ST(generic_ut_setup, generic_ut_teardown,
+ test_compressdev_deflate_stateless_fixed_overflow),
TEST_CASES_END() /**< NULL terminate unit test array */
}
};
--
2.17.1
^ permalink raw reply [flat|nested] 16+ messages in thread
* Re: [dpdk-dev] [PATCH v3 1/2] compress/qat: overflow catch for stateless compression
2019-10-11 17:46 ` [dpdk-dev] [PATCH v3 1/2] compress/qat: overflow catch " Artur Trybula
@ 2019-10-14 13:38 ` Trahe, Fiona
0 siblings, 0 replies; 16+ messages in thread
From: Trahe, Fiona @ 2019-10-14 13:38 UTC (permalink / raw)
To: Trybula, ArturX, dev, shallyv, Dybkowski, AdamX, akhil.goyal; +Cc: Trahe, Fiona
> -----Original Message-----
> From: Trybula, ArturX
> Sent: Friday, October 11, 2019 6:46 PM
> To: dev@dpdk.org; Trahe, Fiona <fiona.trahe@intel.com>; shallyv@marvell.com; Dybkowski, AdamX
> <adamx.dybkowski@intel.com>; Trybula, ArturX <arturx.trybula@intel.com>; akhil.goyal@nxp.com
> Subject: [PATCH v3 1/2] compress/qat: overflow catch for stateless compression
>
> This patch adds out-of-space recoverable feature for stateless
> compression scenario to QuickAssist compressdev driver
>
> Signed-off-by: Artur Trybula <arturx.trybula@intel.com>
Acked-by: Fiona Trahe <fiona.trahe@intel.com>
^ permalink raw reply [flat|nested] 16+ messages in thread
* Re: [dpdk-dev] [PATCH v3 2/2] test/compress: unit test for stateless overflow recovery
2019-10-11 17:46 ` [dpdk-dev] [PATCH v3 2/2] test/compress: unit test for stateless overflow recovery Artur Trybula
@ 2019-10-14 13:48 ` Trahe, Fiona
2019-10-14 14:20 ` Trybula, ArturX
0 siblings, 1 reply; 16+ messages in thread
From: Trahe, Fiona @ 2019-10-14 13:48 UTC (permalink / raw)
To: Trybula, ArturX, dev, shallyv, Dybkowski, AdamX, akhil.goyal; +Cc: Trahe, Fiona
Hi Artur,
> -----Original Message-----
> From: Trybula, ArturX
> Sent: Friday, October 11, 2019 6:46 PM
> To: dev@dpdk.org; Trahe, Fiona <fiona.trahe@intel.com>; shallyv@marvell.com; Dybkowski, AdamX
> <adamx.dybkowski@intel.com>; Trybula, ArturX <arturx.trybula@intel.com>; akhil.goyal@nxp.com
> Subject: [PATCH v3 2/2] test/compress: unit test for stateless overflow recovery
>
> Added unit test to check out-of-space recoverable feature.
>
> Signed-off-by: Artur Trybula <arturx.trybula@intel.com>
> ---
> app/test/test_compressdev.c | 261 +++++++++++++++++++++++++++++-------
> 1 file changed, 214 insertions(+), 47 deletions(-)
//snip//
> @@ -987,40 +1058,41 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
> ops[i]->private_xform = priv_xforms[i];
> }
>
> - /* Enqueue and dequeue all operations */
> - num_enqd = rte_compressdev_enqueue_burst(0, 0, ops, num_bufs);
> - if (num_enqd < num_bufs) {
> +recovery_lb:
> + ret = test_run_enqueue_dequeue(ops, num_bufs, ops_processed);
> + if (ret < 0) {
> RTE_LOG(ERR, USER1,
> - "The operations could not be enqueued\n");
> + "Enqueue/dequeue operation failed\n");
> goto exit;
> }
>
> - num_total_deqd = 0;
> - do {
> - /*
> - * If retrying a dequeue call, wait for 10 ms to allow
> - * enough time to the driver to process the operations
> - */
> - if (deqd_retries != 0) {
> - /*
> - * Avoid infinite loop if not all the
> - * operations get out of the device
> - */
> - if (deqd_retries == MAX_DEQD_RETRIES) {
> + for (i = 0; i < num_bufs; i++) {
> + compressed_data_size[i] += ops_processed[i]->produced;
> +
> + if (ops_processed[i]->status ==
> + RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE) {
> +
> + ops[i]->status =
> + RTE_COMP_OP_STATUS_NOT_PROCESSED;
> + ops[i]->src.offset +=
> + ops_processed[i]->consumed;
> + ops[i]->src.length -=
> + ops_processed[i]->consumed;
> + ops[i]->dst.offset +=
> + ops_processed[i]->produced;
> +
> + buf_ptr = rte_pktmbuf_append(
> + ops[i]->m_dst,
> + ops_processed[i]->produced);
> +
> + if (buf_ptr == NULL) {
> RTE_LOG(ERR, USER1,
> - "Not all operations could be "
> - "dequeued\n");
> + "Data recovery: append extra bytes to the current
> mbuf failed\n");
> goto exit;
> }
> - usleep(DEQUEUE_WAIT_TIME);
> + goto recovery_lb;
> }
> - num_deqd = rte_compressdev_dequeue_burst(0, 0,
> - &ops_processed[num_total_deqd], num_bufs);
> - num_total_deqd += num_deqd;
> - deqd_retries++;
> -
> - } while (num_total_deqd < num_enqd);
> -
> + }
> deqd_retries = 0;
//snip//
Not all drivers support OUT_OF_SPACE_RECOVERABLE - some may return OUT_OF_SPACE_TERMINATED.
There's no capability feature-flag which can be checked before running the test.
How would this test behave in this case?
Ideally it should recognise that OUT_OF_SPACE_TERMINATED is a valid response, but indicates that the
driver doesn't support RECOVERABLE status and so return unsupported rather than failed.
^ permalink raw reply [flat|nested] 16+ messages in thread
* Re: [dpdk-dev] [PATCH v3 2/2] test/compress: unit test for stateless overflow recovery
2019-10-14 13:48 ` Trahe, Fiona
@ 2019-10-14 14:20 ` Trybula, ArturX
0 siblings, 0 replies; 16+ messages in thread
From: Trybula, ArturX @ 2019-10-14 14:20 UTC (permalink / raw)
To: Trahe, Fiona, dev, shallyv, Dybkowski, AdamX, akhil.goyal
Hi Fiona,
Please find my answer below,
-----Original Message-----
From: Trahe, Fiona
Sent: Monday, October 14, 2019 15:49
To: Trybula, ArturX <arturx.trybula@intel.com>; dev@dpdk.org; shallyv@marvell.com; Dybkowski, AdamX <adamx.dybkowski@intel.com>; akhil.goyal@nxp.com
Cc: Trahe, Fiona <fiona.trahe@intel.com>
Subject: RE: [PATCH v3 2/2] test/compress: unit test for stateless overflow recovery
Hi Artur,
> -----Original Message-----
> From: Trybula, ArturX
> Sent: Friday, October 11, 2019 6:46 PM
> To: dev@dpdk.org; Trahe, Fiona <fiona.trahe@intel.com>;
> shallyv@marvell.com; Dybkowski, AdamX <adamx.dybkowski@intel.com>;
> Trybula, ArturX <arturx.trybula@intel.com>; akhil.goyal@nxp.com
> Subject: [PATCH v3 2/2] test/compress: unit test for stateless
> overflow recovery
>
> Added unit test to check out-of-space recoverable feature.
>
> Signed-off-by: Artur Trybula <arturx.trybula@intel.com>
> ---
> app/test/test_compressdev.c | 261
> +++++++++++++++++++++++++++++-------
> 1 file changed, 214 insertions(+), 47 deletions(-)
//snip//
> @@ -987,40 +1058,41 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
> ops[i]->private_xform = priv_xforms[i];
> }
>
> - /* Enqueue and dequeue all operations */
> - num_enqd = rte_compressdev_enqueue_burst(0, 0, ops, num_bufs);
> - if (num_enqd < num_bufs) {
> +recovery_lb:
> + ret = test_run_enqueue_dequeue(ops, num_bufs, ops_processed);
> + if (ret < 0) {
> RTE_LOG(ERR, USER1,
> - "The operations could not be enqueued\n");
> + "Enqueue/dequeue operation failed\n");
> goto exit;
> }
>
> - num_total_deqd = 0;
> - do {
> - /*
> - * If retrying a dequeue call, wait for 10 ms to allow
> - * enough time to the driver to process the operations
> - */
> - if (deqd_retries != 0) {
> - /*
> - * Avoid infinite loop if not all the
> - * operations get out of the device
> - */
> - if (deqd_retries == MAX_DEQD_RETRIES) {
> + for (i = 0; i < num_bufs; i++) {
> + compressed_data_size[i] += ops_processed[i]->produced;
> +
> + if (ops_processed[i]->status ==
> + RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE) {
> +
> + ops[i]->status =
> + RTE_COMP_OP_STATUS_NOT_PROCESSED;
> + ops[i]->src.offset +=
> + ops_processed[i]->consumed;
> + ops[i]->src.length -=
> + ops_processed[i]->consumed;
> + ops[i]->dst.offset +=
> + ops_processed[i]->produced;
> +
> + buf_ptr = rte_pktmbuf_append(
> + ops[i]->m_dst,
> + ops_processed[i]->produced);
> +
> + if (buf_ptr == NULL) {
> RTE_LOG(ERR, USER1,
> - "Not all operations could be "
> - "dequeued\n");
> + "Data recovery: append extra bytes to the current
> mbuf failed\n");
> goto exit;
> }
> - usleep(DEQUEUE_WAIT_TIME);
> + goto recovery_lb;
> }
> - num_deqd = rte_compressdev_dequeue_burst(0, 0,
> - &ops_processed[num_total_deqd], num_bufs);
> - num_total_deqd += num_deqd;
> - deqd_retries++;
> -
> - } while (num_total_deqd < num_enqd);
> -
> + }
> deqd_retries = 0;
//snip//
Not all drivers support OUT_OF_SPACE_RECOVERABLE - some may return OUT_OF_SPACE_TERMINATED.
There's no capability feature-flag which can be checked before running the test.
How would this test behave in this case?
Ideally it should recognise that OUT_OF_SPACE_TERMINATED is a valid response, but indicates that the driver doesn't support RECOVERABLE status and so return unsupported rather than failed.
[Artur] It can fail if a device doesn't support overflow capability. Tomorrow I will run tests using ISAL.
^ permalink raw reply [flat|nested] 16+ messages in thread
* [dpdk-dev] [PATCH v4 0/2] Add overflow recovery for stateless compression
2019-10-11 17:46 ` [dpdk-dev] [PATCH v3 0/2] Add overflow recovery for stateless compression Artur Trybula
2019-10-11 17:46 ` [dpdk-dev] [PATCH v3 1/2] compress/qat: overflow catch " Artur Trybula
2019-10-11 17:46 ` [dpdk-dev] [PATCH v3 2/2] test/compress: unit test for stateless overflow recovery Artur Trybula
@ 2019-10-16 9:40 ` Artur Trybula
2019-10-16 9:40 ` [dpdk-dev] [PATCH v4 1/2] compress/qat: overflow catch " Artur Trybula
` (2 more replies)
2 siblings, 3 replies; 16+ messages in thread
From: Artur Trybula @ 2019-10-16 9:40 UTC (permalink / raw)
To: dev, fiona.trahe, shallyv, adamx.dybkowski, arturx.trybula, akhil.goyal
This patch adds out-of-space recoverable feature for stateless
compression scenario to QuickAssist compressdev driver. Unit test
was also created to verify the new feature.
v4: - if a device doesn't support out-of-space recoverable feature,
then instead of "failed", "unsupported" result will be
printed out on the screen
v3: - rebasing onto the recent master
v2: - rebasing over the external mbufs
Artur Trybula (2):
compress/qat: overflow catch for stateless compression
test/compress: unit test for stateless overflow recovery
app/test/test_compressdev.c | 280 ++++++++++++++++++++++++++------
drivers/compress/qat/qat_comp.c | 28 +++-
2 files changed, 258 insertions(+), 50 deletions(-)
--
2.17.1
^ permalink raw reply [flat|nested] 16+ messages in thread
* [dpdk-dev] [PATCH v4 1/2] compress/qat: overflow catch for stateless compression
2019-10-16 9:40 ` [dpdk-dev] [PATCH v4 0/2] Add overflow recovery for stateless compression Artur Trybula
@ 2019-10-16 9:40 ` Artur Trybula
2019-10-16 9:40 ` [dpdk-dev] [PATCH v4 2/2] test/compress: unit test for stateless overflow recovery Artur Trybula
2019-10-16 11:16 ` [dpdk-dev] [PATCH v4 0/2] Add overflow recovery for stateless compression Trahe, Fiona
2 siblings, 0 replies; 16+ messages in thread
From: Artur Trybula @ 2019-10-16 9:40 UTC (permalink / raw)
To: dev, fiona.trahe, shallyv, adamx.dybkowski, arturx.trybula, akhil.goyal
This patch adds out-of-space recoverable feature for stateless
compression scenario to QuickAssist compressdev driver
Signed-off-by: Artur Trybula <arturx.trybula@intel.com>
---
drivers/compress/qat/qat_comp.c | 28 +++++++++++++++++++++++++---
1 file changed, 25 insertions(+), 3 deletions(-)
diff --git a/drivers/compress/qat/qat_comp.c b/drivers/compress/qat/qat_comp.c
index 8717b7432..533e34f6b 100644
--- a/drivers/compress/qat/qat_comp.c
+++ b/drivers/compress/qat/qat_comp.c
@@ -324,10 +324,32 @@ qat_comp_process_response(void **op, uint8_t *resp, void *op_cookie,
(!cmp_err_code && xlat_err_code == ERR_CODE_OVERFLOW_ERROR)
||
(cmp_err_code == ERR_CODE_OVERFLOW_ERROR &&
- xlat_err_code == ERR_CODE_OVERFLOW_ERROR))
- rx_op->status =
+ xlat_err_code == ERR_CODE_OVERFLOW_ERROR)){
+
+ struct icp_qat_fw_resp_comp_pars *comp_resp =
+ (struct icp_qat_fw_resp_comp_pars *)&resp_msg->comp_resp_pars;
+
+ /* handle recoverable out-of-buffer condition */
+ /* in stateless compression scenario */
+ if (comp_resp->input_byte_counter) {
+ if ((qat_xform->qat_comp_request_type
+ == QAT_COMP_REQUEST_FIXED_COMP_STATELESS) ||
+ (qat_xform->qat_comp_request_type
+ == QAT_COMP_REQUEST_DYNAMIC_COMP_STATELESS)) {
+
+ rx_op->status =
+ RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE;
+ rx_op->consumed =
+ comp_resp->input_byte_counter;
+ rx_op->produced =
+ comp_resp->output_byte_counter;
+ } else
+ rx_op->status =
+ RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
+ } else
+ rx_op->status =
RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
- else
+ } else
rx_op->status = RTE_COMP_OP_STATUS_ERROR;
++(*dequeue_err_count);
--
2.17.1
^ permalink raw reply [flat|nested] 16+ messages in thread
* [dpdk-dev] [PATCH v4 2/2] test/compress: unit test for stateless overflow recovery
2019-10-16 9:40 ` [dpdk-dev] [PATCH v4 0/2] Add overflow recovery for stateless compression Artur Trybula
2019-10-16 9:40 ` [dpdk-dev] [PATCH v4 1/2] compress/qat: overflow catch " Artur Trybula
@ 2019-10-16 9:40 ` Artur Trybula
2019-10-16 11:16 ` [dpdk-dev] [PATCH v4 0/2] Add overflow recovery for stateless compression Trahe, Fiona
2 siblings, 0 replies; 16+ messages in thread
From: Artur Trybula @ 2019-10-16 9:40 UTC (permalink / raw)
To: dev, fiona.trahe, shallyv, adamx.dybkowski, arturx.trybula, akhil.goyal
Added unit test to check out-of-space recoverable feature.
Signed-off-by: Artur Trybula <arturx.trybula@intel.com>
---
app/test/test_compressdev.c | 280 ++++++++++++++++++++++++++++++------
1 file changed, 233 insertions(+), 47 deletions(-)
diff --git a/app/test/test_compressdev.c b/app/test/test_compressdev.c
index d470c34b9..283c64971 100644
--- a/app/test/test_compressdev.c
+++ b/app/test/test_compressdev.c
@@ -30,6 +30,7 @@
* due to the compress block headers
*/
#define COMPRESS_BUF_SIZE_RATIO 1.3
+#define COMPRESS_BUF_SIZE_RATIO_OVERFLOW 0.2
#define NUM_LARGE_MBUFS 16
#define SMALL_SEG_SIZE 256
#define MAX_SEGS 16
@@ -72,6 +73,11 @@ enum varied_buff {
LB_TO_SGL /* output buffer is chained */
};
+enum overflow_test {
+ OVERFLOW_DISABLED,
+ OVERFLOW_ENABLED
+};
+
struct priv_op_data {
uint16_t orig_idx;
};
@@ -110,6 +116,8 @@ struct test_data_params {
const struct rte_memzone *inbuf_memzone;
const struct rte_memzone *compbuf_memzone;
const struct rte_memzone *uncompbuf_memzone;
+ /* overflow test activation */
+ enum overflow_test overflow;
};
static struct comp_testsuite_params testsuite_params = { 0 };
@@ -725,6 +733,49 @@ extbuf_free_callback(void *addr __rte_unused, void *opaque __rte_unused)
{
}
+static int
+test_run_enqueue_dequeue(struct rte_comp_op **ops, unsigned int num_bufs,
+ struct rte_comp_op **ops_processed)
+{
+ uint16_t num_enqd, num_deqd, num_total_deqd;
+ unsigned int deqd_retries = 0;
+
+ /* Enqueue and dequeue all operations */
+ num_enqd = rte_compressdev_enqueue_burst(0, 0, ops, num_bufs);
+ if (num_enqd < num_bufs) {
+ RTE_LOG(ERR, USER1,
+ "Some operations could not be enqueued\n");
+ return -1;
+ }
+
+ num_total_deqd = 0;
+ do {
+ /*
+ * If retrying a dequeue call, wait for 10 ms to allow
+ * enough time to the driver to process the operations
+ */
+ if (deqd_retries != 0) {
+ /*
+ * Avoid infinite loop if not all the
+ * operations get out of the device
+ */
+ if (deqd_retries == MAX_DEQD_RETRIES) {
+ RTE_LOG(ERR, USER1,
+ "Not all operations could be dequeued\n");
+ return -1;
+ }
+ usleep(DEQUEUE_WAIT_TIME);
+ }
+ num_deqd = rte_compressdev_dequeue_burst(0, 0,
+ &ops_processed[num_total_deqd], num_bufs);
+ num_total_deqd += num_deqd;
+ deqd_retries++;
+
+ } while (num_total_deqd < num_enqd);
+
+ return 0;
+}
+
/*
* Compresses and decompresses buffer with compressdev API and Zlib API
*/
@@ -745,6 +796,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
unsigned int out_of_space = test_data->out_of_space;
unsigned int big_data = test_data->big_data;
enum zlib_direction zlib_dir = test_data->zlib_dir;
+ enum overflow_test overflow_tst = test_data->overflow;
int ret_status = TEST_FAILED;
struct rte_mbuf_ext_shared_info inbuf_info;
struct rte_mbuf_ext_shared_info compbuf_info;
@@ -773,6 +825,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
rte_compressdev_capability_get(0, RTE_COMP_ALGO_DEFLATE);
char *contig_buf = NULL;
uint64_t compress_checksum[num_bufs];
+ uint32_t compressed_data_size[num_bufs];
void *stream = NULL;
char *all_decomp_data = NULL;
unsigned int decomp_produced_data_size = 0;
@@ -793,6 +846,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
memset(ops, 0, sizeof(struct rte_comp_op *) * num_bufs);
memset(ops_processed, 0, sizeof(struct rte_comp_op *) * num_bufs);
memset(priv_xforms, 0, sizeof(void *) * num_bufs);
+ memset(compressed_data_size, 0, sizeof(uint32_t) * num_bufs);
if (decompress_state == RTE_COMP_OP_STATEFUL) {
data_size = strlen(test_bufs[0]) + 1;
@@ -845,6 +899,11 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
for (i = 0; i < num_bufs; i++) {
data_size = strlen(test_bufs[i]) + 1;
buf_ptr = rte_pktmbuf_append(uncomp_bufs[i], data_size);
+ if (buf_ptr == NULL) {
+ RTE_LOG(ERR, USER1,
+ "Append extra bytes to the source mbuf failed\n");
+ goto exit;
+ }
strlcpy(buf_ptr, test_bufs[i], data_size);
}
}
@@ -893,11 +952,22 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
for (i = 0; i < num_bufs; i++) {
if (out_of_space == 1 && oos_zlib_decompress)
data_size = OUT_OF_SPACE_BUF;
- else
- (data_size = strlen(test_bufs[i]) *
- COMPRESS_BUF_SIZE_RATIO);
-
- rte_pktmbuf_append(comp_bufs[i], data_size);
+ else {
+ float ratio =
+ ((test_data->zlib_dir == ZLIB_DECOMPRESS ||
+ test_data->zlib_dir == ZLIB_NONE) &&
+ overflow_tst == OVERFLOW_ENABLED) ?
+ COMPRESS_BUF_SIZE_RATIO_OVERFLOW :
+ COMPRESS_BUF_SIZE_RATIO;
+
+ data_size = strlen(test_bufs[i]) * ratio;
+ }
+ buf_ptr = rte_pktmbuf_append(comp_bufs[i], data_size);
+ if (buf_ptr == NULL) {
+ RTE_LOG(ERR, USER1,
+ "Append extra bytes to the destination mbuf failed\n");
+ goto exit;
+ }
}
}
@@ -963,7 +1033,8 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
num_priv_xforms++;
}
- if (capa->comp_feature_flags & RTE_COMP_FF_SHAREABLE_PRIV_XFORM) {
+ if (capa->comp_feature_flags &
+ RTE_COMP_FF_SHAREABLE_PRIV_XFORM) {
/* Attach shareable private xform data to ops */
for (i = 0; i < num_bufs; i++)
ops[i]->private_xform = priv_xforms[i % num_xforms];
@@ -987,40 +1058,41 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
ops[i]->private_xform = priv_xforms[i];
}
- /* Enqueue and dequeue all operations */
- num_enqd = rte_compressdev_enqueue_burst(0, 0, ops, num_bufs);
- if (num_enqd < num_bufs) {
+recovery_lb:
+ ret = test_run_enqueue_dequeue(ops, num_bufs, ops_processed);
+ if (ret < 0) {
RTE_LOG(ERR, USER1,
- "The operations could not be enqueued\n");
+ "Enqueue/dequeue operation failed\n");
goto exit;
}
- num_total_deqd = 0;
- do {
- /*
- * If retrying a dequeue call, wait for 10 ms to allow
- * enough time to the driver to process the operations
- */
- if (deqd_retries != 0) {
- /*
- * Avoid infinite loop if not all the
- * operations get out of the device
- */
- if (deqd_retries == MAX_DEQD_RETRIES) {
+ for (i = 0; i < num_bufs; i++) {
+ compressed_data_size[i] += ops_processed[i]->produced;
+
+ if (ops_processed[i]->status ==
+ RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE) {
+
+ ops[i]->status =
+ RTE_COMP_OP_STATUS_NOT_PROCESSED;
+ ops[i]->src.offset +=
+ ops_processed[i]->consumed;
+ ops[i]->src.length -=
+ ops_processed[i]->consumed;
+ ops[i]->dst.offset +=
+ ops_processed[i]->produced;
+
+ buf_ptr = rte_pktmbuf_append(
+ ops[i]->m_dst,
+ ops_processed[i]->produced);
+
+ if (buf_ptr == NULL) {
RTE_LOG(ERR, USER1,
- "Not all operations could be "
- "dequeued\n");
+ "Data recovery: append extra bytes to the current mbuf failed\n");
goto exit;
}
- usleep(DEQUEUE_WAIT_TIME);
+ goto recovery_lb;
}
- num_deqd = rte_compressdev_dequeue_burst(0, 0,
- &ops_processed[num_total_deqd], num_bufs);
- num_total_deqd += num_deqd;
- deqd_retries++;
-
- } while (num_total_deqd < num_enqd);
-
+ }
deqd_retries = 0;
/* Free compress private xforms */
@@ -1064,7 +1136,7 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
for (i = 0; i < num_bufs; i++) {
if (out_of_space && oos_zlib_decompress) {
if (ops_processed[i]->status !=
- RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED) {
+ RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED) {
ret_status = TEST_FAILED;
RTE_LOG(ERR, USER1,
"Operation without expected out of "
@@ -1075,6 +1147,16 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
}
if (ops_processed[i]->status != RTE_COMP_OP_STATUS_SUCCESS) {
+ if (overflow_tst == OVERFLOW_ENABLED) {
+ if (ops_processed[i]->status ==
+ RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED) {
+ ret_status = 1;
+ RTE_LOG(INFO, USER1,
+ "Out-of-space-recoverable functionality"
+ " is not supported on this device\n");
+ goto exit;
+ }
+ }
RTE_LOG(ERR, USER1,
"Some operations were not successful\n");
goto exit;
@@ -1147,7 +1229,12 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
data_size =
strlen(test_bufs[priv_data->orig_idx]) + 1;
- rte_pktmbuf_append(uncomp_bufs[i], data_size);
+ buf_ptr = rte_pktmbuf_append(uncomp_bufs[i], data_size);
+ if (buf_ptr == NULL) {
+ RTE_LOG(ERR, USER1,
+ "Append extra bytes to the decompressed mbuf failed\n");
+ goto exit;
+ }
}
}
@@ -1169,7 +1256,10 @@ test_deflate_comp_decomp(const struct interim_data_params *int_data,
* Set the length of the compressed data to the
* number of bytes that were produced in the previous stage
*/
- ops[i]->src.length = ops_processed[i]->produced;
+ if (compressed_data_size[i])
+ ops[i]->src.length = compressed_data_size[i];
+ else
+ ops[i]->src.length = ops_processed[i]->produced;
ops[i]->dst.offset = 0;
if (decompress_state == RTE_COMP_OP_STATELESS) {
@@ -1541,7 +1631,8 @@ test_compressdev_deflate_stateless_fixed(void)
.buff_type = LB_BOTH,
.zlib_dir = ZLIB_DECOMPRESS,
.out_of_space = 0,
- .big_data = 0
+ .big_data = 0,
+ .overflow = OVERFLOW_DISABLED
};
for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
@@ -1611,7 +1702,8 @@ test_compressdev_deflate_stateless_dynamic(void)
.buff_type = LB_BOTH,
.zlib_dir = ZLIB_DECOMPRESS,
.out_of_space = 0,
- .big_data = 0
+ .big_data = 0,
+ .overflow = OVERFLOW_DISABLED
};
for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
@@ -1665,7 +1757,8 @@ test_compressdev_deflate_stateless_multi_op(void)
.buff_type = LB_BOTH,
.zlib_dir = ZLIB_DECOMPRESS,
.out_of_space = 0,
- .big_data = 0
+ .big_data = 0,
+ .overflow = OVERFLOW_DISABLED
};
/* Compress with compressdev, decompress with Zlib */
@@ -1718,7 +1811,8 @@ test_compressdev_deflate_stateless_multi_level(void)
.buff_type = LB_BOTH,
.zlib_dir = ZLIB_DECOMPRESS,
.out_of_space = 0,
- .big_data = 0
+ .big_data = 0,
+ .overflow = OVERFLOW_DISABLED
};
for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
@@ -1807,7 +1901,8 @@ test_compressdev_deflate_stateless_multi_xform(void)
.buff_type = LB_BOTH,
.zlib_dir = ZLIB_DECOMPRESS,
.out_of_space = 0,
- .big_data = 0
+ .big_data = 0,
+ .overflow = OVERFLOW_DISABLED
};
/* Compress with compressdev, decompress with Zlib */
@@ -1855,7 +1950,8 @@ test_compressdev_deflate_stateless_sgl(void)
.buff_type = SGL_BOTH,
.zlib_dir = ZLIB_DECOMPRESS,
.out_of_space = 0,
- .big_data = 0
+ .big_data = 0,
+ .overflow = OVERFLOW_DISABLED
};
for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
@@ -1965,7 +2061,8 @@ test_compressdev_deflate_stateless_checksum(void)
.buff_type = LB_BOTH,
.zlib_dir = ZLIB_DECOMPRESS,
.out_of_space = 0,
- .big_data = 0
+ .big_data = 0,
+ .overflow = OVERFLOW_DISABLED
};
/* Check if driver supports crc32 checksum and test */
@@ -2082,7 +2179,8 @@ test_compressdev_out_of_space_buffer(void)
.buff_type = LB_BOTH,
.zlib_dir = ZLIB_DECOMPRESS,
.out_of_space = 1, /* run out-of-space test */
- .big_data = 0
+ .big_data = 0,
+ .overflow = OVERFLOW_DISABLED
};
/* Compress with compressdev, decompress with Zlib */
test_data.zlib_dir = ZLIB_DECOMPRESS;
@@ -2159,7 +2257,8 @@ test_compressdev_deflate_stateless_dynamic_big(void)
.buff_type = SGL_BOTH,
.zlib_dir = ZLIB_DECOMPRESS,
.out_of_space = 0,
- .big_data = 1
+ .big_data = 1,
+ .overflow = OVERFLOW_DISABLED
};
ts_params->def_comp_xform->compress.deflate.huffman =
@@ -2223,7 +2322,8 @@ test_compressdev_deflate_stateful_decomp(void)
.out_of_space = 0,
.big_data = 0,
.decompress_output_block_size = 2000,
- .decompress_steps_max = 4
+ .decompress_steps_max = 4,
+ .overflow = OVERFLOW_DISABLED
};
/* Compress with Zlib, decompress with compressdev */
@@ -2305,7 +2405,8 @@ test_compressdev_deflate_stateful_decomp_checksum(void)
.out_of_space = 0,
.big_data = 0,
.decompress_output_block_size = 2000,
- .decompress_steps_max = 4
+ .decompress_steps_max = 4,
+ .overflow = OVERFLOW_DISABLED
};
/* Check if driver supports crc32 checksum and test */
@@ -2441,7 +2542,8 @@ test_compressdev_external_mbufs(void)
.inbuf_memzone = make_memzone("inbuf", data_len),
.compbuf_memzone = make_memzone("compbuf", data_len *
COMPRESS_BUF_SIZE_RATIO),
- .uncompbuf_memzone = make_memzone("decompbuf", data_len)
+ .uncompbuf_memzone = make_memzone("decompbuf", data_len),
+ .overflow = OVERFLOW_DISABLED
};
for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
@@ -2472,6 +2574,88 @@ test_compressdev_external_mbufs(void)
return ret;
}
+static int
+test_compressdev_deflate_stateless_fixed_oos_recoverable(void)
+{
+ struct comp_testsuite_params *ts_params = &testsuite_params;
+ uint16_t i;
+ int ret;
+ int comp_result;
+ const struct rte_compressdev_capabilities *capab;
+
+ capab = rte_compressdev_capability_get(0, RTE_COMP_ALGO_DEFLATE);
+ TEST_ASSERT(capab != NULL, "Failed to retrieve device capabilities");
+
+ if ((capab->comp_feature_flags & RTE_COMP_FF_HUFFMAN_FIXED) == 0)
+ return -ENOTSUP;
+
+ struct rte_comp_xform *compress_xform =
+ rte_malloc(NULL, sizeof(struct rte_comp_xform), 0);
+
+ if (compress_xform == NULL) {
+ RTE_LOG(ERR, USER1,
+ "Compress xform could not be created\n");
+ ret = TEST_FAILED;
+ goto exit;
+ }
+
+ memcpy(compress_xform, ts_params->def_comp_xform,
+ sizeof(struct rte_comp_xform));
+ compress_xform->compress.deflate.huffman = RTE_COMP_HUFFMAN_FIXED;
+
+ struct interim_data_params int_data = {
+ NULL,
+ 1,
+ NULL,
+ &compress_xform,
+ &ts_params->def_decomp_xform,
+ 1
+ };
+
+ struct test_data_params test_data = {
+ .compress_state = RTE_COMP_OP_STATELESS,
+ .decompress_state = RTE_COMP_OP_STATELESS,
+ .buff_type = LB_BOTH,
+ .zlib_dir = ZLIB_DECOMPRESS,
+ .out_of_space = 0,
+ .big_data = 0,
+ .overflow = OVERFLOW_ENABLED
+ };
+
+ for (i = 0; i < RTE_DIM(compress_test_bufs); i++) {
+ int_data.test_bufs = &compress_test_bufs[i];
+ int_data.buf_idx = &i;
+
+ /* Compress with compressdev, decompress with Zlib */
+ test_data.zlib_dir = ZLIB_DECOMPRESS;
+ comp_result = test_deflate_comp_decomp(&int_data, &test_data);
+ if (comp_result < 0) {
+ ret = TEST_FAILED;
+ goto exit;
+ } else if (comp_result > 0) {
+ ret = -ENOTSUP;
+ goto exit;
+ }
+
+ /* Compress with Zlib, decompress with compressdev */
+ test_data.zlib_dir = ZLIB_COMPRESS;
+ comp_result = test_deflate_comp_decomp(&int_data, &test_data);
+ if (comp_result < 0) {
+ ret = TEST_FAILED;
+ goto exit;
+ } else if (comp_result > 0) {
+ ret = -ENOTSUP;
+ goto exit;
+ }
+ }
+
+ ret = TEST_SUCCESS;
+
+exit:
+ rte_free(compress_xform);
+ return ret;
+}
+
static struct unit_test_suite compressdev_testsuite = {
.suite_name = "compressdev unit test suite",
.setup = testsuite_setup,
@@ -2503,6 +2687,8 @@ static struct unit_test_suite compressdev_testsuite = {
test_compressdev_deflate_stateful_decomp_checksum),
TEST_CASE_ST(generic_ut_setup, generic_ut_teardown,
test_compressdev_external_mbufs),
+ TEST_CASE_ST(generic_ut_setup, generic_ut_teardown,
+ test_compressdev_deflate_stateless_fixed_oos_recoverable),
TEST_CASES_END() /**< NULL terminate unit test array */
}
};
--
2.17.1
^ permalink raw reply [flat|nested] 16+ messages in thread
* Re: [dpdk-dev] [PATCH v4 0/2] Add overflow recovery for stateless compression
2019-10-16 9:40 ` [dpdk-dev] [PATCH v4 0/2] Add overflow recovery for stateless compression Artur Trybula
2019-10-16 9:40 ` [dpdk-dev] [PATCH v4 1/2] compress/qat: overflow catch " Artur Trybula
2019-10-16 9:40 ` [dpdk-dev] [PATCH v4 2/2] test/compress: unit test for stateless overflow recovery Artur Trybula
@ 2019-10-16 11:16 ` Trahe, Fiona
2019-10-16 13:13 ` Akhil Goyal
2 siblings, 1 reply; 16+ messages in thread
From: Trahe, Fiona @ 2019-10-16 11:16 UTC (permalink / raw)
To: Trybula, ArturX, dev, shallyv, Dybkowski, AdamX, akhil.goyal
> -----Original Message-----
> From: Trybula, ArturX
> Sent: Wednesday, October 16, 2019 10:41 AM
> To: dev@dpdk.org; Trahe, Fiona <fiona.trahe@intel.com>; shallyv@marvell.com; Dybkowski, AdamX
> <adamx.dybkowski@intel.com>; Trybula, ArturX <arturx.trybula@intel.com>; akhil.goyal@nxp.com
> Subject: [PATCH v4 0/2] Add overflow recovery for stateless compression
>
> This patch adds out-of-space recoverable feature for stateless
> compression scenario to QuickAssist compressdev driver. Unit test
> was also created to verify the new feature.
>
> v4: - if a device doesn't support out-of-space recoverable feature,
> then instead of "failed", "unsupported" result will be
> printed out on the screen
>
> v3: - rebasing onto the recent master
>
> v2: - rebasing over the external mbufs
>
> Artur Trybula (2):
> compress/qat: overflow catch for stateless compression
> test/compress: unit test for stateless overflow recovery
>
> app/test/test_compressdev.c | 280 ++++++++++++++++++++++++++------
> drivers/compress/qat/qat_comp.c | 28 +++-
> 2 files changed, 258 insertions(+), 50 deletions(-)
>
> --
> 2.17.1
Series
Acked-by: Fiona Trahe <fiona.trahe@intel.com>
^ permalink raw reply [flat|nested] 16+ messages in thread
* Re: [dpdk-dev] [PATCH v4 0/2] Add overflow recovery for stateless compression
2019-10-16 11:16 ` [dpdk-dev] [PATCH v4 0/2] Add overflow recovery for stateless compression Trahe, Fiona
@ 2019-10-16 13:13 ` Akhil Goyal
0 siblings, 0 replies; 16+ messages in thread
From: Akhil Goyal @ 2019-10-16 13:13 UTC (permalink / raw)
To: Trahe, Fiona, Trybula, ArturX, dev, shallyv, Dybkowski, AdamX
>
> > -----Original Message-----
> > From: Trybula, ArturX
> > Sent: Wednesday, October 16, 2019 10:41 AM
> > To: dev@dpdk.org; Trahe, Fiona <fiona.trahe@intel.com>;
> shallyv@marvell.com; Dybkowski, AdamX
> > <adamx.dybkowski@intel.com>; Trybula, ArturX <arturx.trybula@intel.com>;
> akhil.goyal@nxp.com
> > Subject: [PATCH v4 0/2] Add overflow recovery for stateless compression
> >
> > This patch adds out-of-space recoverable feature for stateless
> > compression scenario to QuickAssist compressdev driver. Unit test
> > was also created to verify the new feature.
> >
> > v4: - if a device doesn't support out-of-space recoverable feature,
> > then instead of "failed", "unsupported" result will be
> > printed out on the screen
> >
> > v3: - rebasing onto the recent master
> >
> > v2: - rebasing over the external mbufs
> >
> > Artur Trybula (2):
> > compress/qat: overflow catch for stateless compression
> > test/compress: unit test for stateless overflow recovery
> >
> > app/test/test_compressdev.c | 280 ++++++++++++++++++++++++++------
> > drivers/compress/qat/qat_comp.c | 28 +++-
> > 2 files changed, 258 insertions(+), 50 deletions(-)
> >
> > --
> > 2.17.1
> Series
> Acked-by: Fiona Trahe <fiona.trahe@intel.com>
Applied to dpdk-next-crypto
Thanks.
^ permalink raw reply [flat|nested] 16+ messages in thread
end of thread, other threads:[~2019-10-16 13:13 UTC | newest]
Thread overview: 16+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2019-09-06 11:11 [dpdk-dev] [PATCH 1/2] compress/qat: overflow catching Artur Trybula
2019-09-06 11:11 ` [dpdk-dev] [PATCH 2/2] test/compress: unit test for stateless overflow recovery Artur Trybula
2019-09-26 9:32 ` [dpdk-dev] [PATCH v2 0/2] Add overflow recovery for stateless compression Artur Trybula
2019-09-26 9:32 ` [dpdk-dev] [PATCH v2 1/2] compress/qat: overflow catch " Artur Trybula
2019-09-26 9:32 ` [dpdk-dev] [PATCH v2 2/2] test/compress: unit test for stateless overflow recovery Artur Trybula
2019-10-11 17:46 ` [dpdk-dev] [PATCH v3 0/2] Add overflow recovery for stateless compression Artur Trybula
2019-10-11 17:46 ` [dpdk-dev] [PATCH v3 1/2] compress/qat: overflow catch " Artur Trybula
2019-10-14 13:38 ` Trahe, Fiona
2019-10-11 17:46 ` [dpdk-dev] [PATCH v3 2/2] test/compress: unit test for stateless overflow recovery Artur Trybula
2019-10-14 13:48 ` Trahe, Fiona
2019-10-14 14:20 ` Trybula, ArturX
2019-10-16 9:40 ` [dpdk-dev] [PATCH v4 0/2] Add overflow recovery for stateless compression Artur Trybula
2019-10-16 9:40 ` [dpdk-dev] [PATCH v4 1/2] compress/qat: overflow catch " Artur Trybula
2019-10-16 9:40 ` [dpdk-dev] [PATCH v4 2/2] test/compress: unit test for stateless overflow recovery Artur Trybula
2019-10-16 11:16 ` [dpdk-dev] [PATCH v4 0/2] Add overflow recovery for stateless compression Trahe, Fiona
2019-10-16 13:13 ` Akhil Goyal
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).