From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 9DADEA057C; Fri, 27 Mar 2020 12:47:03 +0100 (CET) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 7753C1C138; Fri, 27 Mar 2020 12:47:03 +0100 (CET) Received: from mail-wr1-f66.google.com (mail-wr1-f66.google.com [209.85.221.66]) by dpdk.org (Postfix) with ESMTP id 6DFD51C10E for ; Fri, 27 Mar 2020 12:47:02 +0100 (CET) Received: by mail-wr1-f66.google.com with SMTP id h9so11017407wrc.8 for ; Fri, 27 Mar 2020 04:47:02 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=accelercomm-com.20150623.gappssmtp.com; s=20150623; h=subject:to:references:from:message-id:date:user-agent:mime-version :in-reply-to:content-transfer-encoding:content-language; bh=06QC7pMcOq94iY7R5XGm6tbN67rYFUpFO/ao/8mn3Vo=; b=M4//Flb2+ltS0haeAZi4xPOPXsWU5A0f21zGmw5eGEJTx6ZMZnxhJn3sWROoUr+xgm TTsPiP87jRst3/9rhvcOQy9gabskgzohsMEzqVp8uVd4muyjgIw25uxF1p5STuKhgTod DqyJIpuX/DhX/C78T/ssky94pGzM48joFRyXZABjAOSWQwcQrd/jSoHuRHC9FG6MNAUR fslHqCeWD5rskxku4PxF429DXiFDMQM25e8SntkW4Al/PqCJJTpMAIdg1SBMEH/K+3UM hIDlbpW47mVdwuvPW06zQmLdJu2a0gn1QMVx7q36yF0PKTx3Gy57XDLhBqkz4QdRhiDB Jynw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:subject:to:references:from:message-id:date :user-agent:mime-version:in-reply-to:content-transfer-encoding :content-language; bh=06QC7pMcOq94iY7R5XGm6tbN67rYFUpFO/ao/8mn3Vo=; b=OmAN9XkPJzzl5XBSQeybmfTM7NMH0PYJCehPa6Mh+Xcs3ebap/dvYaf0dWMNIPIaBs y4GFSYVsb9vgffwz9AWhs7YIpq3kOMtUplPVeTUjbXo6xBGjvLEU+9vVQ49wO32bS5In h/Z1XFzw5+V5ywY7EfoiK3EX1O8Rw+Sswm1iYnbV3k8bgcXgKj/BEA2Bvo2YKDLzfDO0 r4P6NmQraM5gyr5iNJbBve3akmoX7LoTTV4Froj6Z4e2PGTsfZZlxcUrwD2upPP/2/nM pqaj0Zs6TYoS49YxXiOYJQ2tIELk4KD2jk5d1nbsys79JdBOJHB0n8j2JYS50W9PhmiQ +AOA== X-Gm-Message-State: ANhLgQ29wYt8Nx/nT/VYK0POLV26A2f4/ay2VsnrajJ+3zckJ7eJxUrn 8k+stnFv/B01jzI/Bxsp2Um3wxygYMw= X-Google-Smtp-Source: ADFU+vsi0rb4XYRHeoXSs2vpjK23ak4AouRSsm798IwT6hNODs+ad3+QSZPvNJbJ57Ee8Wi3juKgNA== X-Received: by 2002:a05:6000:a:: with SMTP id h10mr15631289wrx.226.1585309621520; Fri, 27 Mar 2020 04:47:01 -0700 (PDT) Received: from ?IPv6:2a00:23c4:cf2d:a200:6023:b810:2491:c6a9? ([2a00:23c4:cf2d:a200:6023:b810:2491:c6a9]) by smtp.gmail.com with ESMTPSA id y200sm7820220wmc.20.2020.03.27.04.47.00 for (version=TLS1_3 cipher=TLS_AES_128_GCM_SHA256 bits=128/128); Fri, 27 Mar 2020 04:47:01 -0700 (PDT) To: dev@dpdk.org References: <1582778348-113547-15-git-send-email-nicolas.chautru@intel.com> <1585193268-74468-1-git-send-email-nicolas.chautru@intel.com> <1585193268-74468-7-git-send-email-nicolas.chautru@intel.com> From: Dave Burley Message-ID: <6133dee7-c613-f930-e674-a465059ca142@accelercomm.com> Date: Fri, 27 Mar 2020 11:47:00 +0000 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:68.0) Gecko/20100101 Thunderbird/68.5.0 MIME-Version: 1.0 In-Reply-To: <1585193268-74468-7-git-send-email-nicolas.chautru@intel.com> Content-Type: text/plain; charset=utf-8; format=flowed Content-Transfer-Encoding: 7bit Content-Language: en-US Subject: Re: [dpdk-dev] [PATCH v5 06/10] test-bbdev: support HARQ validation X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Acked-by: Dave Burley On 26/03/2020 03:27, Nicolas Chautru wrote: > From: Nic Chautru > > Adding functionality to validate HARQ for different > devices implementation. > Adding capacity to fetch HARQ data when required as > part of this validation. > > Signed-off-by: Nic Chautru > --- > app/test-bbdev/test_bbdev_perf.c | 564 ++++++++++++++++++++++++++++++++----- > app/test-bbdev/test_bbdev_vector.c | 9 +- > 2 files changed, 504 insertions(+), 69 deletions(-) > > diff --git a/app/test-bbdev/test_bbdev_perf.c b/app/test-bbdev/test_bbdev_perf.c > index b17fc95..4d7dc4e 100644 > --- a/app/test-bbdev/test_bbdev_perf.c > +++ b/app/test-bbdev/test_bbdev_perf.c > @@ -44,14 +44,34 @@ > > #define SYNC_WAIT 0 > #define SYNC_START 1 > +#define INVALID_OPAQUE -1 > > #define INVALID_QUEUE_ID -1 > +/* Increment for next code block in external HARQ memory */ > +#define HARQ_INCR 32768 > +/* Headroom for filler LLRs insertion in HARQ buffer */ > +#define FILLER_HEADROOM 1024 > +/* Constants from K0 computation from 3GPP 38.212 Table 5.4.2.1-2 */ > +#define N_ZC_1 66 /* N = 66 Zc for BG 1 */ > +#define N_ZC_2 50 /* N = 50 Zc for BG 2 */ > +#define K0_1_1 17 /* K0 fraction numerator for rv 1 and BG 1 */ > +#define K0_1_2 13 /* K0 fraction numerator for rv 1 and BG 2 */ > +#define K0_2_1 33 /* K0 fraction numerator for rv 2 and BG 1 */ > +#define K0_2_2 25 /* K0 fraction numerator for rv 2 and BG 2 */ > +#define K0_3_1 56 /* K0 fraction numerator for rv 3 and BG 1 */ > +#define K0_3_2 43 /* K0 fraction numerator for rv 3 and BG 2 */ > > static struct test_bbdev_vector test_vector; > > /* Switch between PMD and Interrupt for throughput TC */ > static bool intr_enabled; > > +/* LLR arithmetic representation for numerical conversion */ > +static int ldpc_llr_decimals; > +static int ldpc_llr_size; > +/* Keep track of the LDPC decoder device capability flag */ > +static uint32_t ldpc_cap_flags; > + > /* Represents tested active devices */ > static struct active_device { > const char *driver_name; > @@ -293,7 +313,7 @@ typedef int (test_case_function)(struct active_device *ad, > return TEST_FAILED; > } > if (intr_enabled && !(cap->capability_flags & > - RTE_BBDEV_TURBO_ENC_INTERRUPTS)) { > + RTE_BBDEV_LDPC_ENC_INTERRUPTS)) { > printf( > "Dequeue interrupts are not supported!\n"); > return TEST_FAILED; > @@ -336,12 +356,19 @@ typedef int (test_case_function)(struct active_device *ad, > return TEST_FAILED; > } > if (intr_enabled && !(cap->capability_flags & > - RTE_BBDEV_TURBO_DEC_INTERRUPTS)) { > + RTE_BBDEV_LDPC_DEC_INTERRUPTS)) { > printf( > "Dequeue interrupts are not supported!\n"); > return TEST_FAILED; > } > - > + if (intr_enabled && (test_vector.ldpc_dec.op_flags & > + (RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE | > + RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE | > + RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK > + ))) { > + printf("Skip loop-back with interrupt\n"); > + return TEST_FAILED; > + } > return TEST_SUCCESS; > } > } > @@ -377,7 +404,8 @@ typedef int (test_case_function)(struct active_device *ad, > snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str, > dev_id); > return rte_pktmbuf_pool_create(pool_name, mbuf_pool_size, 0, 0, > - RTE_MAX(max_seg_sz + RTE_PKTMBUF_HEADROOM, > + RTE_MAX(max_seg_sz + RTE_PKTMBUF_HEADROOM > + + FILLER_HEADROOM, > (unsigned int)RTE_MBUF_DEFAULT_BUF_SIZE), socket_id); > } > > @@ -432,27 +460,33 @@ typedef int (test_case_function)(struct active_device *ad, > return TEST_SUCCESS; > > /* Inputs */ > - mbuf_pool_size = optimal_mempool_size(ops_pool_size * in->nb_segments); > - mp = create_mbuf_pool(in, ad->dev_id, socket_id, mbuf_pool_size, "in"); > - TEST_ASSERT_NOT_NULL(mp, > - "ERROR Failed to create %u items input pktmbuf pool for dev %u on socket %u.", > - mbuf_pool_size, > - ad->dev_id, > - socket_id); > - ad->in_mbuf_pool = mp; > + if (in->nb_segments > 0) { > + mbuf_pool_size = optimal_mempool_size(ops_pool_size * > + in->nb_segments); > + mp = create_mbuf_pool(in, ad->dev_id, socket_id, > + mbuf_pool_size, "in"); > + TEST_ASSERT_NOT_NULL(mp, > + "ERROR Failed to create %u items input pktmbuf pool for dev %u on socket %u.", > + mbuf_pool_size, > + ad->dev_id, > + socket_id); > + ad->in_mbuf_pool = mp; > + } > > /* Hard outputs */ > - mbuf_pool_size = optimal_mempool_size(ops_pool_size * > - hard_out->nb_segments); > - mp = create_mbuf_pool(hard_out, ad->dev_id, socket_id, mbuf_pool_size, > - "hard_out"); > - TEST_ASSERT_NOT_NULL(mp, > - "ERROR Failed to create %u items hard output pktmbuf pool for dev %u on socket %u.", > - mbuf_pool_size, > - ad->dev_id, > - socket_id); > - ad->hard_out_mbuf_pool = mp; > - > + if (hard_out->nb_segments > 0) { > + mbuf_pool_size = optimal_mempool_size(ops_pool_size * > + hard_out->nb_segments); > + mp = create_mbuf_pool(hard_out, ad->dev_id, socket_id, > + mbuf_pool_size, > + "hard_out"); > + TEST_ASSERT_NOT_NULL(mp, > + "ERROR Failed to create %u items hard output pktmbuf pool for dev %u on socket %u.", > + mbuf_pool_size, > + ad->dev_id, > + socket_id); > + ad->hard_out_mbuf_pool = mp; > + } > > /* Soft outputs */ > if (soft_out->nb_segments > 0) { > @@ -901,6 +935,45 @@ typedef int (test_case_function)(struct active_device *ad, > } > } > > +/* > + * We may have to insert filler bits > + * when they are required by the HARQ assumption > + */ > +static void > +ldpc_add_filler(struct rte_bbdev_op_data *input_ops, > + const uint16_t n, struct test_op_params *op_params) > +{ > + struct rte_bbdev_op_ldpc_dec dec = op_params->ref_dec_op->ldpc_dec; > + > + if (input_ops == NULL) > + return; > + /* No need to add filler if not required by device */ > + if (!(ldpc_cap_flags & > + RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS)) > + return; > + /* No need to add filler for loopback operation */ > + if (dec.op_flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK) > + return; > + > + uint16_t i, j, parity_offset; > + for (i = 0; i < n; ++i) { > + struct rte_mbuf *m = input_ops[i].data; > + int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *, > + input_ops[i].offset); > + parity_offset = (dec.basegraph == 1 ? 20 : 8) > + * dec.z_c - dec.n_filler; > + uint16_t new_hin_size = input_ops[i].length + dec.n_filler; > + m->data_len = new_hin_size; > + input_ops[i].length = new_hin_size; > + for (j = new_hin_size - 1; j >= parity_offset + dec.n_filler; > + j--) > + llr[j] = llr[j - dec.n_filler]; > + uint16_t llr_max_pre_scaling = (1 << (ldpc_llr_size - 1)) - 1; > + for (j = 0; j < dec.n_filler; j++) > + llr[parity_offset + j] = llr_max_pre_scaling; > + } > +} > + > static void > ldpc_input_llr_scaling(struct rte_bbdev_op_data *input_ops, > const uint16_t n, const int8_t llr_size, > @@ -923,7 +996,9 @@ typedef int (test_case_function)(struct active_device *ad, > ++byte_idx) { > > llr_tmp = llr[byte_idx]; > - if (llr_decimals == 2) > + if (llr_decimals == 4) > + llr_tmp *= 8; > + else if (llr_decimals == 2) > llr_tmp *= 2; > else if (llr_decimals == 0) > llr_tmp /= 2; > @@ -991,12 +1066,24 @@ typedef int (test_case_function)(struct active_device *ad, > capabilities->cap.turbo_dec.max_llr_modulus); > > if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) { > - ldpc_input_llr_scaling(*queue_ops[DATA_INPUT], n, > - capabilities->cap.ldpc_dec.llr_size, > - capabilities->cap.ldpc_dec.llr_decimals); > - ldpc_input_llr_scaling(*queue_ops[DATA_HARQ_INPUT], n, > - capabilities->cap.ldpc_dec.llr_size, > - capabilities->cap.ldpc_dec.llr_decimals); > + bool loopback = op_params->ref_dec_op->ldpc_dec.op_flags & > + RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK; > + bool llr_comp = op_params->ref_dec_op->ldpc_dec.op_flags & > + RTE_BBDEV_LDPC_LLR_COMPRESSION; > + bool harq_comp = op_params->ref_dec_op->ldpc_dec.op_flags & > + RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; > + ldpc_llr_decimals = capabilities->cap.ldpc_dec.llr_decimals; > + ldpc_llr_size = capabilities->cap.ldpc_dec.llr_size; > + ldpc_cap_flags = capabilities->cap.ldpc_dec.capability_flags; > + if (!loopback && !llr_comp) > + ldpc_input_llr_scaling(*queue_ops[DATA_INPUT], n, > + ldpc_llr_size, ldpc_llr_decimals); > + if (!loopback && !harq_comp) > + ldpc_input_llr_scaling(*queue_ops[DATA_HARQ_INPUT], n, > + ldpc_llr_size, ldpc_llr_decimals); > + if (!loopback) > + ldpc_add_filler(*queue_ops[DATA_HARQ_INPUT], n, > + op_params); > } > > return 0; > @@ -1159,17 +1246,21 @@ typedef int (test_case_function)(struct active_device *ad, > ops[i]->ldpc_dec.op_flags = ldpc_dec->op_flags; > ops[i]->ldpc_dec.code_block_mode = ldpc_dec->code_block_mode; > > - ops[i]->ldpc_dec.hard_output = hard_outputs[start_idx + i]; > - ops[i]->ldpc_dec.input = inputs[start_idx + i]; > + if (hard_outputs != NULL) > + ops[i]->ldpc_dec.hard_output = > + hard_outputs[start_idx + i]; > + if (inputs != NULL) > + ops[i]->ldpc_dec.input = > + inputs[start_idx + i]; > if (soft_outputs != NULL) > ops[i]->ldpc_dec.soft_output = > - soft_outputs[start_idx + i]; > + soft_outputs[start_idx + i]; > if (harq_inputs != NULL) > ops[i]->ldpc_dec.harq_combined_input = > harq_inputs[start_idx + i]; > if (harq_outputs != NULL) > ops[i]->ldpc_dec.harq_combined_output = > - harq_outputs[start_idx + i]; > + harq_outputs[start_idx + i]; > } > } > > @@ -1211,7 +1302,22 @@ typedef int (test_case_function)(struct active_device *ad, > check_dec_status_and_ordering(struct rte_bbdev_dec_op *op, > unsigned int order_idx, const int expected_status) > { > - TEST_ASSERT(op->status == expected_status, > + int status = op->status; > + /* ignore parity mismatch false alarms for long iterations */ > + if (get_iter_max() >= 10) { > + if (!(expected_status & (1 << RTE_BBDEV_SYNDROME_ERROR)) && > + (status & (1 << RTE_BBDEV_SYNDROME_ERROR))) { > + printf("WARNING: Ignore Syndrome Check mismatch\n"); > + status -= (1 << RTE_BBDEV_SYNDROME_ERROR); > + } > + if ((expected_status & (1 << RTE_BBDEV_SYNDROME_ERROR)) && > + !(status & (1 << RTE_BBDEV_SYNDROME_ERROR))) { > + printf("WARNING: Ignore Syndrome Check mismatch\n"); > + status += (1 << RTE_BBDEV_SYNDROME_ERROR); > + } > + } > + > + TEST_ASSERT(status == expected_status, > "op_status (%d) != expected_status (%d)", > op->status, expected_status); > > @@ -1230,9 +1336,10 @@ typedef int (test_case_function)(struct active_device *ad, > "op_status (%d) != expected_status (%d)", > op->status, expected_status); > > - TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, > - "Ordering error, expected %p, got %p", > - (void *)(uintptr_t)order_idx, op->opaque_data); > + if (op->opaque_data != (void *)(uintptr_t)INVALID_OPAQUE) > + TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, > + "Ordering error, expected %p, got %p", > + (void *)(uintptr_t)order_idx, op->opaque_data); > > return TEST_SUCCESS; > } > @@ -1276,6 +1383,173 @@ typedef int (test_case_function)(struct active_device *ad, > return TEST_SUCCESS; > } > > +/* > + * Compute K0 for a given configuration for HARQ output length computation > + * As per definition in 3GPP 38.212 Table 5.4.2.1-2 > + */ > +static inline uint16_t > +get_k0(uint16_t n_cb, uint16_t z_c, uint8_t bg, uint8_t rv_index) > +{ > + if (rv_index == 0) > + return 0; > + uint16_t n = (bg == 1 ? N_ZC_1 : N_ZC_2) * z_c; > + if (n_cb == n) { > + if (rv_index == 1) > + return (bg == 1 ? K0_1_1 : K0_1_2) * z_c; > + else if (rv_index == 2) > + return (bg == 1 ? K0_2_1 : K0_2_2) * z_c; > + else > + return (bg == 1 ? K0_3_1 : K0_3_2) * z_c; > + } > + /* LBRM case - includes a division by N */ > + if (rv_index == 1) > + return (((bg == 1 ? K0_1_1 : K0_1_2) * n_cb) > + / n) * z_c; > + else if (rv_index == 2) > + return (((bg == 1 ? K0_2_1 : K0_2_2) * n_cb) > + / n) * z_c; > + else > + return (((bg == 1 ? K0_3_1 : K0_3_2) * n_cb) > + / n) * z_c; > +} > + > +/* HARQ output length including the Filler bits */ > +static inline uint16_t > +compute_harq_len(struct rte_bbdev_op_ldpc_dec *ops_ld) > +{ > + uint16_t k0 = 0; > + uint8_t max_rv = (ops_ld->rv_index == 1) ? 3 : ops_ld->rv_index; > + k0 = get_k0(ops_ld->n_cb, ops_ld->z_c, ops_ld->basegraph, max_rv); > + /* Compute RM out size and number of rows */ > + uint16_t parity_offset = (ops_ld->basegraph == 1 ? 20 : 8) > + * ops_ld->z_c - ops_ld->n_filler; > + uint16_t deRmOutSize = RTE_MIN( > + k0 + ops_ld->cb_params.e + > + ((k0 > parity_offset) ? > + 0 : ops_ld->n_filler), > + ops_ld->n_cb); > + uint16_t numRows = ((deRmOutSize + ops_ld->z_c - 1) > + / ops_ld->z_c); > + uint16_t harq_output_len = numRows * ops_ld->z_c; > + return harq_output_len; > +} > + > +static inline int > +validate_op_harq_chain(struct rte_bbdev_op_data *op, > + struct op_data_entries *orig_op, > + struct rte_bbdev_op_ldpc_dec *ops_ld) > +{ > + uint8_t i; > + uint32_t j, jj, k; > + struct rte_mbuf *m = op->data; > + uint8_t nb_dst_segments = orig_op->nb_segments; > + uint32_t total_data_size = 0; > + int8_t *harq_orig, *harq_out, abs_harq_origin; > + uint32_t byte_error = 0, cum_error = 0, error; > + int16_t llr_max = (1 << (ldpc_llr_size - ldpc_llr_decimals)) - 1; > + int16_t llr_max_pre_scaling = (1 << (ldpc_llr_size - 1)) - 1; > + uint16_t parity_offset; > + > + TEST_ASSERT(nb_dst_segments == m->nb_segs, > + "Number of segments differ in original (%u) and filled (%u) op", > + nb_dst_segments, m->nb_segs); > + > + /* Validate each mbuf segment length */ > + for (i = 0; i < nb_dst_segments; ++i) { > + /* Apply offset to the first mbuf segment */ > + uint16_t offset = (i == 0) ? op->offset : 0; > + uint16_t data_len = rte_pktmbuf_data_len(m) - offset; > + total_data_size += orig_op->segments[i].length; > + > + TEST_ASSERT(orig_op->segments[i].length < > + (uint32_t)(data_len + 64), > + "Length of segment differ in original (%u) and filled (%u) op", > + orig_op->segments[i].length, data_len); > + harq_orig = (int8_t *) orig_op->segments[i].addr; > + harq_out = rte_pktmbuf_mtod_offset(m, int8_t *, offset); > + > + if (!(ldpc_cap_flags & > + RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS > + ) || (ops_ld->op_flags & > + RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) { > + data_len -= ops_ld->z_c; > + parity_offset = data_len; > + } else { > + /* Compute RM out size and number of rows */ > + parity_offset = (ops_ld->basegraph == 1 ? 20 : 8) > + * ops_ld->z_c - ops_ld->n_filler; > + uint16_t deRmOutSize = compute_harq_len(ops_ld) - > + ops_ld->n_filler; > + if (data_len > deRmOutSize) > + data_len = deRmOutSize; > + if (data_len > orig_op->segments[i].length) > + data_len = orig_op->segments[i].length; > + } > + /* > + * HARQ output can have minor differences > + * due to integer representation and related scaling > + */ > + for (j = 0, jj = 0; j < data_len; j++, jj++) { > + if (j == parity_offset) { > + /* Special Handling of the filler bits */ > + for (k = 0; k < ops_ld->n_filler; k++) { > + if (harq_out[jj] != > + llr_max_pre_scaling) { > + printf("HARQ Filler issue %d: %d %d\n", > + jj, harq_out[jj], > + llr_max); > + byte_error++; > + } > + jj++; > + } > + } > + if (!(ops_ld->op_flags & > + RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) { > + if (ldpc_llr_decimals > 1) > + harq_out[jj] = (harq_out[jj] + 1) > + >> (ldpc_llr_decimals - 1); > + /* Saturated to S7 */ > + if (harq_orig[j] > llr_max) > + harq_orig[j] = llr_max; > + if (harq_orig[j] < -llr_max) > + harq_orig[j] = -llr_max; > + } > + if (harq_orig[j] != harq_out[jj]) { > + error = (harq_orig[j] > harq_out[jj]) ? > + harq_orig[j] - harq_out[jj] : > + harq_out[jj] - harq_orig[j]; > + abs_harq_origin = harq_orig[j] > 0 ? > + harq_orig[j] : > + -harq_orig[j]; > + /* Residual quantization error */ > + if ((error > 8 && (abs_harq_origin < > + (llr_max - 16))) || > + (error > 16)) { > + printf("HARQ mismatch %d: exp %d act %d => %d\n", > + j, harq_orig[j], > + harq_out[jj], error); > + byte_error++; > + cum_error += error; > + } > + } > + } > + m = m->next; > + } > + > + if (byte_error) > + TEST_ASSERT(byte_error <= 1, > + "HARQ output mismatch (%d) %d", > + byte_error, cum_error); > + > + /* Validate total mbuf pkt length */ > + uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset; > + TEST_ASSERT(total_data_size < pkt_len + 64, > + "Length of data differ in original (%u) and filled (%u) op", > + total_data_size, pkt_len); > + > + return TEST_SUCCESS; > +} > + > static int > validate_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n, > struct rte_bbdev_dec_op *ref_op, const int vector_mask) > @@ -1319,7 +1593,6 @@ typedef int (test_case_function)(struct active_device *ad, > return TEST_SUCCESS; > } > > - > static int > validate_ldpc_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n, > struct rte_bbdev_dec_op *ref_op, const int vector_mask) > @@ -1351,8 +1624,15 @@ typedef int (test_case_function)(struct active_device *ad, > TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count, > "Returned iter_count (%d) > expected iter_count (%d)", > ops_td->iter_count, ref_td->iter_count); > - /* We can ignore data when the decoding failed to converge */ > - if ((ops[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR)) == 0) > + /* > + * We can ignore output data when the decoding failed to > + * converge or for loop-back cases > + */ > + if (!check_bit(ops[i]->ldpc_dec.op_flags, > + RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK > + ) && ( > + ops[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR > + )) == 0) > TEST_ASSERT_SUCCESS(validate_op_chain(hard_output, > hard_data_orig), > "Hard output buffers (CB=%u) are not equal", > @@ -1365,12 +1645,18 @@ typedef int (test_case_function)(struct active_device *ad, > i); > if (ref_op->ldpc_dec.op_flags & > RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE) { > - ldpc_input_llr_scaling(harq_output, 1, 8, 0); > - TEST_ASSERT_SUCCESS(validate_op_chain(harq_output, > - harq_data_orig), > + TEST_ASSERT_SUCCESS(validate_op_harq_chain(harq_output, > + harq_data_orig, ops_td), > "HARQ output buffers (CB=%u) are not equal", > i); > } > + if (ref_op->ldpc_dec.op_flags & > + RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK) > + TEST_ASSERT_SUCCESS(validate_op_harq_chain(harq_output, > + harq_data_orig, ops_td), > + "HARQ output buffers (CB=%u) are not equal", > + i); > + > } > > return TEST_SUCCESS; > @@ -1709,6 +1995,105 @@ typedef int (test_case_function)(struct active_device *ad, > return ret; > } > > + > +/* Push back the HARQ output from DDR to host */ > +static void > +retrieve_harq_ddr(uint16_t dev_id, uint16_t queue_id, > + struct rte_bbdev_dec_op **ops, > + const uint16_t n) > +{ > + uint16_t j; > + int save_status, ret; > + uint32_t harq_offset = (uint32_t) queue_id * HARQ_INCR * 1024; > + struct rte_bbdev_dec_op *ops_deq[MAX_BURST]; > + uint32_t flags = ops[0]->ldpc_dec.op_flags; > + bool loopback = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK; > + bool mem_out = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; > + bool hc_out = flags & RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE; > + bool h_comp = flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; > + for (j = 0; j < n; ++j) { > + if ((loopback && mem_out) || hc_out) { > + save_status = ops[j]->status; > + ops[j]->ldpc_dec.op_flags = > + RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK + > + RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE; > + if (h_comp) > + ops[j]->ldpc_dec.op_flags += > + RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; > + ops[j]->ldpc_dec.harq_combined_input.offset = > + harq_offset; > + ops[j]->ldpc_dec.harq_combined_output.offset = 0; > + harq_offset += HARQ_INCR; > + if (!loopback) > + ops[j]->ldpc_dec.harq_combined_input.length = > + ops[j]->ldpc_dec.harq_combined_output.length; > + rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, > + &ops[j], 1); > + ret = 0; > + while (ret == 0) > + ret = rte_bbdev_dequeue_ldpc_dec_ops( > + dev_id, queue_id, > + &ops_deq[j], 1); > + ops[j]->ldpc_dec.op_flags = flags; > + ops[j]->status = save_status; > + } > + } > +} > + > +/* > + * Push back the HARQ output from HW DDR to Host > + * Preload HARQ memory input and adjust HARQ offset > + */ > +static void > +preload_harq_ddr(uint16_t dev_id, uint16_t queue_id, > + struct rte_bbdev_dec_op **ops, const uint16_t n, > + bool preload) > +{ > + uint16_t j; > + int ret; > + uint32_t harq_offset = (uint32_t) queue_id * HARQ_INCR * 1024; > + struct rte_bbdev_op_data save_hc_in, save_hc_out; > + struct rte_bbdev_dec_op *ops_deq[MAX_BURST]; > + uint32_t flags = ops[0]->ldpc_dec.op_flags; > + bool mem_in = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE; > + bool hc_in = flags & RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE; > + bool mem_out = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; > + bool hc_out = flags & RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE; > + bool h_comp = flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; > + for (j = 0; j < n; ++j) { > + if ((mem_in || hc_in) && preload) { > + save_hc_in = ops[j]->ldpc_dec.harq_combined_input; > + save_hc_out = ops[j]->ldpc_dec.harq_combined_output; > + ops[j]->ldpc_dec.op_flags = > + RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK + > + RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; > + if (h_comp) > + ops[j]->ldpc_dec.op_flags += > + RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; > + ops[j]->ldpc_dec.harq_combined_output.offset = > + harq_offset; > + ops[j]->ldpc_dec.harq_combined_input.offset = 0; > + rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, > + &ops[j], 1); > + ret = 0; > + while (ret == 0) > + ret = rte_bbdev_dequeue_ldpc_dec_ops( > + dev_id, queue_id, &ops_deq[j], 1); > + ops[j]->ldpc_dec.op_flags = flags; > + ops[j]->ldpc_dec.harq_combined_input = save_hc_in; > + ops[j]->ldpc_dec.harq_combined_output = save_hc_out; > + } > + /* Adjust HARQ offset when we reach external DDR */ > + if (mem_in || hc_in) > + ops[j]->ldpc_dec.harq_combined_input.offset > + = harq_offset; > + if (mem_out || hc_out) > + ops[j]->ldpc_dec.harq_combined_output.offset > + = harq_offset; > + harq_offset += HARQ_INCR; > + } > +} > + > static void > dequeue_event_callback(uint16_t dev_id, > enum rte_bbdev_event_type event, void *cb_arg, > @@ -1744,13 +2129,22 @@ typedef int (test_case_function)(struct active_device *ad, > burst_sz = rte_atomic16_read(&tp->burst_sz); > num_ops = tp->op_params->num_to_process; > > - if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC || > - test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) > + if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) > deq = rte_bbdev_dequeue_dec_ops(dev_id, queue_id, > &tp->dec_ops[ > rte_atomic16_read(&tp->nb_dequeued)], > burst_sz); > - else > + else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) > + deq = rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, > + &tp->dec_ops[ > + rte_atomic16_read(&tp->nb_dequeued)], > + burst_sz); > + else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) > + deq = rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, > + &tp->enc_ops[ > + rte_atomic16_read(&tp->nb_dequeued)], > + burst_sz); > + else /*RTE_BBDEV_OP_TURBO_ENC*/ > deq = rte_bbdev_dequeue_enc_ops(dev_id, queue_id, > &tp->enc_ops[ > rte_atomic16_read(&tp->nb_dequeued)], > @@ -2127,6 +2521,12 @@ typedef int (test_case_function)(struct active_device *ad, > int i, j, ret; > struct rte_bbdev_info info; > uint16_t num_to_enq; > + bool extDdr = check_bit(ldpc_cap_flags, > + RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE); > + bool loopback = check_bit(ref_op->ldpc_dec.op_flags, > + RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK); > + bool hc_out = check_bit(ref_op->ldpc_dec.op_flags, > + RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE); > > TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), > "BURST_SIZE should be <= %u", MAX_BURST); > @@ -2164,13 +2564,18 @@ typedef int (test_case_function)(struct active_device *ad, > > for (i = 0; i < TEST_REPETITIONS; ++i) { > for (j = 0; j < num_ops; ++j) { > - mbuf_reset(ops_enq[j]->ldpc_dec.hard_output.data); > - if (check_bit(ref_op->ldpc_dec.op_flags, > - RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE)) > + if (!loopback) > + mbuf_reset( > + ops_enq[j]->ldpc_dec.hard_output.data); > + if (hc_out || loopback) > mbuf_reset( > ops_enq[j]->ldpc_dec.harq_combined_output.data); > } > - > + if (extDdr) { > + bool preload = i == (TEST_REPETITIONS - 1); > + preload_harq_ddr(tp->dev_id, queue_id, ops_enq, > + num_ops, preload); > + } > start_time = rte_rdtsc_precise(); > > for (enq = 0, deq = 0; enq < num_ops;) { > @@ -2201,6 +2606,10 @@ typedef int (test_case_function)(struct active_device *ad, > tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count, > tp->iter_count); > } > + if (extDdr) { > + /* Read loopback is not thread safe */ > + retrieve_harq_ddr(tp->dev_id, queue_id, ops_enq, num_ops); > + } > > if (test_vector.op_type != RTE_BBDEV_OP_NONE) { > ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op, > @@ -2690,6 +3099,8 @@ typedef int (test_case_function)(struct active_device *ad, > uint16_t i, j, dequeued; > struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; > uint64_t start_time = 0, last_time = 0; > + bool extDdr = ldpc_cap_flags & > + RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; > > for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { > uint16_t enq = 0, deq = 0; > @@ -2702,6 +3113,15 @@ typedef int (test_case_function)(struct active_device *ad, > ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); > TEST_ASSERT_SUCCESS(ret, > "rte_bbdev_dec_op_alloc_bulk() failed"); > + > + /* For latency tests we need to disable early termination */ > + if (check_bit(ref_op->ldpc_dec.op_flags, > + RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE)) > + ref_op->ldpc_dec.op_flags -= > + RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE; > + ref_op->ldpc_dec.iter_max = 6; > + ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max; > + > if (test_vector.op_type != RTE_BBDEV_OP_NONE) > copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued, > bufs->inputs, > @@ -2711,6 +3131,10 @@ typedef int (test_case_function)(struct active_device *ad, > bufs->harq_outputs, > ref_op); > > + if (extDdr) > + preload_harq_ddr(dev_id, queue_id, ops_enq, > + burst_sz, true); > + > /* Set counter to validate the ordering */ > for (j = 0; j < burst_sz; ++j) > ops_enq[j]->opaque_data = (void *)(uintptr_t)j; > @@ -2737,6 +3161,9 @@ typedef int (test_case_function)(struct active_device *ad, > *min_time = RTE_MIN(*min_time, last_time); > *total_time += last_time; > > + if (extDdr) > + retrieve_harq_ddr(dev_id, queue_id, ops_enq, burst_sz); > + > if (test_vector.op_type != RTE_BBDEV_OP_NONE) { > ret = validate_ldpc_dec_op(ops_deq, burst_sz, ref_op, > vector_mask); > @@ -2746,7 +3173,6 @@ typedef int (test_case_function)(struct active_device *ad, > rte_bbdev_dec_op_free_bulk(ops_enq, deq); > dequeued += deq; > } > - > return i; > } > > @@ -2838,7 +3264,6 @@ typedef int (test_case_function)(struct active_device *ad, > burst_sz = num_to_process - dequeued; > > ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); > - > TEST_ASSERT_SUCCESS(ret, > "rte_bbdev_enc_op_alloc_bulk() failed"); > if (test_vector.op_type != RTE_BBDEV_OP_NONE) > @@ -3075,6 +3500,8 @@ typedef int (test_case_function)(struct active_device *ad, > uint64_t enq_start_time, deq_start_time; > uint64_t enq_sw_last_time, deq_last_time; > struct rte_bbdev_stats stats; > + bool extDdr = ldpc_cap_flags & > + RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; > > for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { > uint16_t enq = 0, deq = 0; > @@ -3092,6 +3519,10 @@ typedef int (test_case_function)(struct active_device *ad, > bufs->harq_outputs, > ref_op); > > + if (extDdr) > + preload_harq_ddr(dev_id, queue_id, ops_enq, > + burst_sz, true); > + > /* Start time meas for enqueue function offload latency */ > enq_start_time = rte_rdtsc_precise(); > do { > @@ -3099,13 +3530,13 @@ typedef int (test_case_function)(struct active_device *ad, > &ops_enq[enq], burst_sz - enq); > } while (unlikely(burst_sz != enq)); > > + enq_sw_last_time = rte_rdtsc_precise() - enq_start_time; > ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); > TEST_ASSERT_SUCCESS(ret, > "Failed to get stats for queue (%u) of device (%u)", > queue_id, dev_id); > > - enq_sw_last_time = rte_rdtsc_precise() - enq_start_time - > - stats.acc_offload_cycles; > + enq_sw_last_time -= stats.acc_offload_cycles; > time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, > enq_sw_last_time); > time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, > @@ -3138,9 +3569,14 @@ typedef int (test_case_function)(struct active_device *ad, > > /* Dequeue remaining operations if needed*/ > while (burst_sz != deq) > - deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, > + deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, > &ops_deq[deq], burst_sz - deq); > > + if (extDdr) { > + /* Read loopback is not thread safe */ > + retrieve_harq_ddr(dev_id, queue_id, ops_enq, burst_sz); > + } > + > rte_bbdev_dec_op_free_bulk(ops_enq, deq); > dequeued += deq; > } > @@ -3167,7 +3603,8 @@ typedef int (test_case_function)(struct active_device *ad, > burst_sz = num_to_process - dequeued; > > ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); > - TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed"); > + TEST_ASSERT_SUCCESS(ret, > + "rte_bbdev_enc_op_alloc_bulk() failed"); > if (test_vector.op_type != RTE_BBDEV_OP_NONE) > copy_reference_enc_op(ops_enq, burst_sz, dequeued, > bufs->inputs, > @@ -3181,13 +3618,13 @@ typedef int (test_case_function)(struct active_device *ad, > &ops_enq[enq], burst_sz - enq); > } while (unlikely(burst_sz != enq)); > > + enq_sw_last_time = rte_rdtsc_precise() - enq_start_time; > + > ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); > TEST_ASSERT_SUCCESS(ret, > "Failed to get stats for queue (%u) of device (%u)", > queue_id, dev_id); > - > - enq_sw_last_time = rte_rdtsc_precise() - enq_start_time - > - stats.acc_offload_cycles; > + enq_sw_last_time -= stats.acc_offload_cycles; > time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, > enq_sw_last_time); > time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, > @@ -3249,7 +3686,8 @@ typedef int (test_case_function)(struct active_device *ad, > burst_sz = num_to_process - dequeued; > > ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); > - TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed"); > + TEST_ASSERT_SUCCESS(ret, > + "rte_bbdev_enc_op_alloc_bulk() failed"); > if (test_vector.op_type != RTE_BBDEV_OP_NONE) > copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued, > bufs->inputs, > @@ -3263,13 +3701,13 @@ typedef int (test_case_function)(struct active_device *ad, > &ops_enq[enq], burst_sz - enq); > } while (unlikely(burst_sz != enq)); > > + enq_sw_last_time = rte_rdtsc_precise() - enq_start_time; > ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); > TEST_ASSERT_SUCCESS(ret, > "Failed to get stats for queue (%u) of device (%u)", > queue_id, dev_id); > > - enq_sw_last_time = rte_rdtsc_precise() - enq_start_time - > - stats.acc_offload_cycles; > + enq_sw_last_time -= stats.acc_offload_cycles; > time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, > enq_sw_last_time); > time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, > diff --git a/app/test-bbdev/test_bbdev_vector.c b/app/test-bbdev/test_bbdev_vector.c > index 77356cb..50d1da0 100644 > --- a/app/test-bbdev/test_bbdev_vector.c > +++ b/app/test-bbdev/test_bbdev_vector.c > @@ -197,6 +197,9 @@ > else if (!strcmp(token, > "RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE")) > *op_flag_value = RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; > + else if (!strcmp(token, > + "RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK")) > + *op_flag_value = RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK; > else { > printf("The given value is not a LDPC decoder flag\n"); > return -1; > @@ -943,16 +946,10 @@ > unsigned char i; > struct rte_bbdev_op_ldpc_dec *ldpc_dec = &vector->ldpc_dec; > > - if (vector->entries[DATA_INPUT].nb_segments == 0) > - return -1; > - > for (i = 0; i < vector->entries[DATA_INPUT].nb_segments; i++) > if (vector->entries[DATA_INPUT].segments[i].addr == NULL) > return -1; > > - if (vector->entries[DATA_HARD_OUTPUT].nb_segments == 0) > - return -1; > - > for (i = 0; i < vector->entries[DATA_HARD_OUTPUT].nb_segments; i++) > if (vector->entries[DATA_HARD_OUTPUT].segments[i].addr == NULL) > return -1;