From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id B561E42D32 for ; Fri, 23 Jun 2023 16:47:56 +0200 (CEST) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id ABA1542B71; Fri, 23 Jun 2023 16:47:56 +0200 (CEST) Received: from mga01.intel.com (mga01.intel.com [192.55.52.88]) by mails.dpdk.org (Postfix) with ESMTP id BFE37406B8; Fri, 23 Jun 2023 16:47:53 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=intel.com; i=@intel.com; q=dns/txt; s=Intel; t=1687531674; x=1719067674; h=from:to:cc:subject:date:message-id:in-reply-to: references:mime-version:content-transfer-encoding; bh=KK/RUD0plzNDSfdAypI0Pth3vVJt0gZ1RTGaA1n17CA=; b=HygVdVYDYjG4fL6l+vlPO0uxJxh+lY30ElHAeGaREIDQm4EJGebm6hm1 dQTbPJKmgUd25kmYD5vRVFN38qP23r6XjlwfccOvoYtjPBEkfLBuMREym OyKWEyQwJ4yvZYv6L28Fwv6QOidD/EFvogp/nQCiP5K13LZn4pfveVzNF LYFIRCQbo9TcB+TYhpCE+TCCyr8xh20JBn3htf73O9/wfFrBPyJYV+Ci/ EM5JBjv0ycezqMMqDQLSI7GiOkNHCQmHGkKTy9rGKkFxynOsnHqF2oxvy ZvMEZkyEyPP45UxOQeQHI6n+8c1JHJ/FHgW7N2Lyn7DIF4Vn32JWgmjZ5 g==; X-IronPort-AV: E=McAfee;i="6600,9927,10750"; a="390769136" X-IronPort-AV: E=Sophos;i="6.01,152,1684825200"; d="scan'208";a="390769136" Received: from orsmga003.jf.intel.com ([10.7.209.27]) by fmsmga101.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 23 Jun 2023 07:47:52 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=McAfee;i="6600,9927,10750"; a="665503837" X-IronPort-AV: E=Sophos;i="6.01,152,1684825200"; d="scan'208";a="665503837" Received: from silpixa00400355.ir.intel.com (HELO silpixa00400355.ger.corp.intel.com.) ([10.237.222.80]) by orsmga003.jf.intel.com with ESMTP; 23 Jun 2023 07:47:49 -0700 From: Vikash Poddar To: Kai Ji , Fan Zhang , Ashish Gupta Cc: dev@dpdk.org, Vikash Poddar , bruce.richardson@intel.com, stable@dpdk.org Subject: [PATCH v2] common/qat: detach QAT crypto compress build Date: Fri, 23 Jun 2023 14:47:47 +0000 Message-Id: <20230623144747.1379182-1-vikash.chandrax.poddar@intel.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20230623134821.1377590-1-vikash.chandrax.poddar@intel.com> References: <20230623134821.1377590-1-vikash.chandrax.poddar@intel.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-BeenThere: stable@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: patches for DPDK stable branches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: stable-bounces@dpdk.org qat_qp.c is a common file for QAT crypto and compress. Moved compress function from common file to compress QAT file qat_comp.c Fixes: 2ca75c65af4c ("common/qat: build drivers from common folder") Cc: bruce.richardson@intel.com Cc: stable@dpdk.org Signed-off-by: Vikash Poddar --- v2: Fixed coding style issue --- drivers/common/qat/meson.build | 8 -- drivers/common/qat/qat_qp.c | 187 -------------------------------- drivers/common/qat/qat_qp.h | 20 +++- drivers/compress/qat/qat_comp.c | 182 +++++++++++++++++++++++++++++++ drivers/compress/qat/qat_comp.h | 3 + 5 files changed, 201 insertions(+), 199 deletions(-) diff --git a/drivers/common/qat/meson.build b/drivers/common/qat/meson.build index b84e5b3c6c..95b52b78c3 100644 --- a/drivers/common/qat/meson.build +++ b/drivers/common/qat/meson.build @@ -54,14 +54,6 @@ if libipsecmb.found() and libcrypto_3.found() endif endif -# The driver should not build if both compression and crypto are disabled -#FIXME common code depends on compression files so check only compress! -if not qat_compress # and not qat_crypto - build = false - reason = '' # rely on reason for compress/crypto above - subdir_done() -endif - deps += ['bus_pci', 'cryptodev', 'net', 'compressdev'] sources += files( 'qat_common.c', diff --git a/drivers/common/qat/qat_qp.c b/drivers/common/qat/qat_qp.c index 348a1d574d..197e8bac75 100644 --- a/drivers/common/qat/qat_qp.c +++ b/drivers/common/qat/qat_qp.c @@ -490,20 +490,6 @@ adf_configure_queues(struct qat_qp *qp, enum qat_device_gen qat_dev_gen) return 0; } -static inline void -txq_write_tail(enum qat_device_gen qat_dev_gen, - struct qat_qp *qp, struct qat_queue *q) -{ - struct qat_qp_hw_spec_funcs *ops = - qat_qp_hw_spec[qat_dev_gen]; - - /* - * Pointer check should be done during - * initialization - */ - ops->qat_qp_csr_write_tail(qp, q); -} - static inline void qat_qp_csr_write_head(enum qat_device_gen qat_dev_gen, struct qat_qp *qp, struct qat_queue *q, uint32_t new_head) @@ -672,179 +658,6 @@ qat_enqueue_op_burst(void *qp, qat_op_build_request_t op_build_request, return nb_ops_sent; } -/* Use this for compression only - but keep consistent with above common - * function as much as possible. - */ -uint16_t -qat_enqueue_comp_op_burst(void *qp, void **ops, uint16_t nb_ops) -{ - register struct qat_queue *queue; - struct qat_qp *tmp_qp = (struct qat_qp *)qp; - register uint32_t nb_ops_sent = 0; - register int nb_desc_to_build; - uint16_t nb_ops_possible = nb_ops; - register uint8_t *base_addr; - register uint32_t tail; - - int descriptors_built, total_descriptors_built = 0; - int nb_remaining_descriptors; - int overflow = 0; - - if (unlikely(nb_ops == 0)) - return 0; - - /* read params used a lot in main loop into registers */ - queue = &(tmp_qp->tx_q); - base_addr = (uint8_t *)queue->base_addr; - tail = queue->tail; - - /* Find how many can actually fit on the ring */ - { - /* dequeued can only be written by one thread, but it may not - * be this thread. As it's 4-byte aligned it will be read - * atomically here by any Intel CPU. - * enqueued can wrap before dequeued, but cannot - * lap it as var size of enq/deq (uint32_t) > var size of - * max_inflights (uint16_t). In reality inflights is never - * even as big as max uint16_t, as it's <= ADF_MAX_DESC. - * On wrapping, the calculation still returns the correct - * positive value as all three vars are unsigned. - */ - uint32_t inflights = - tmp_qp->enqueued - tmp_qp->dequeued; - - /* Find how many can actually fit on the ring */ - overflow = (inflights + nb_ops) - tmp_qp->max_inflights; - if (overflow > 0) { - nb_ops_possible = nb_ops - overflow; - if (nb_ops_possible == 0) - return 0; - } - - /* QAT has plenty of work queued already, so don't waste cycles - * enqueueing, wait til the application has gathered a bigger - * burst or some completed ops have been dequeued - */ - if (tmp_qp->min_enq_burst_threshold && inflights > - QAT_QP_MIN_INFL_THRESHOLD && nb_ops_possible < - tmp_qp->min_enq_burst_threshold) { - tmp_qp->stats.threshold_hit_count++; - return 0; - } - } - - /* At this point nb_ops_possible is assuming a 1:1 mapping - * between ops and descriptors. - * Fewer may be sent if some ops have to be split. - * nb_ops_possible is <= burst size. - * Find out how many spaces are actually available on the qp in case - * more are needed. - */ - nb_remaining_descriptors = nb_ops_possible - + ((overflow >= 0) ? 0 : overflow * (-1)); - QAT_DP_LOG(DEBUG, "Nb ops requested %d, nb descriptors remaining %d", - nb_ops, nb_remaining_descriptors); - - while (nb_ops_sent != nb_ops_possible && - nb_remaining_descriptors > 0) { - struct qat_comp_op_cookie *cookie = - tmp_qp->op_cookies[tail >> queue->trailz]; - - descriptors_built = 0; - - QAT_DP_LOG(DEBUG, "--- data length: %u", - ((struct rte_comp_op *)*ops)->src.length); - - nb_desc_to_build = qat_comp_build_request(*ops, - base_addr + tail, cookie, tmp_qp->qat_dev_gen); - QAT_DP_LOG(DEBUG, "%d descriptors built, %d remaining, " - "%d ops sent, %d descriptors needed", - total_descriptors_built, nb_remaining_descriptors, - nb_ops_sent, nb_desc_to_build); - - if (unlikely(nb_desc_to_build < 0)) { - /* this message cannot be enqueued */ - tmp_qp->stats.enqueue_err_count++; - if (nb_ops_sent == 0) - return 0; - goto kick_tail; - } else if (unlikely(nb_desc_to_build > 1)) { - /* this op is too big and must be split - get more - * descriptors and retry - */ - - QAT_DP_LOG(DEBUG, "Build %d descriptors for this op", - nb_desc_to_build); - - nb_remaining_descriptors -= nb_desc_to_build; - if (nb_remaining_descriptors >= 0) { - /* There are enough remaining descriptors - * so retry - */ - int ret2 = qat_comp_build_multiple_requests( - *ops, tmp_qp, tail, - nb_desc_to_build); - - if (unlikely(ret2 < 1)) { - QAT_DP_LOG(DEBUG, - "Failed to build (%d) descriptors, status %d", - nb_desc_to_build, ret2); - - qat_comp_free_split_op_memzones(cookie, - nb_desc_to_build - 1); - - tmp_qp->stats.enqueue_err_count++; - - /* This message cannot be enqueued */ - if (nb_ops_sent == 0) - return 0; - goto kick_tail; - } else { - descriptors_built = ret2; - total_descriptors_built += - descriptors_built; - nb_remaining_descriptors -= - descriptors_built; - QAT_DP_LOG(DEBUG, - "Multiple descriptors (%d) built ok", - descriptors_built); - } - } else { - QAT_DP_LOG(ERR, "For the current op, number of requested descriptors (%d) " - "exceeds number of available descriptors (%d)", - nb_desc_to_build, - nb_remaining_descriptors + - nb_desc_to_build); - - qat_comp_free_split_op_memzones(cookie, - nb_desc_to_build - 1); - - /* Not enough extra descriptors */ - if (nb_ops_sent == 0) - return 0; - goto kick_tail; - } - } else { - descriptors_built = 1; - total_descriptors_built++; - nb_remaining_descriptors--; - QAT_DP_LOG(DEBUG, "Single descriptor built ok"); - } - - tail = adf_modulo(tail + (queue->msg_size * descriptors_built), - queue->modulo_mask); - ops++; - nb_ops_sent++; - } - -kick_tail: - queue->tail = tail; - tmp_qp->enqueued += total_descriptors_built; - tmp_qp->stats.enqueued_count += nb_ops_sent; - txq_write_tail(tmp_qp->qat_dev_gen, tmp_qp, queue); - return nb_ops_sent; -} - uint16_t qat_dequeue_op_burst(void *qp, void **ops, qat_op_dequeue_t qat_dequeue_process_response, uint16_t nb_ops) diff --git a/drivers/common/qat/qat_qp.h b/drivers/common/qat/qat_qp.h index d19fc387e4..ae18fb942e 100644 --- a/drivers/common/qat/qat_qp.h +++ b/drivers/common/qat/qat_qp.h @@ -127,9 +127,6 @@ uint16_t qat_enqueue_op_burst(void *qp, qat_op_build_request_t op_build_request, void **ops, uint16_t nb_ops); -uint16_t -qat_enqueue_comp_op_burst(void *qp, void **ops, uint16_t nb_ops); - uint16_t qat_dequeue_op_burst(void *qp, void **ops, qat_op_dequeue_t qat_dequeue_process_response, uint16_t nb_ops); @@ -206,6 +203,21 @@ struct qat_qp_hw_spec_funcs { qat_qp_get_hw_data_t qat_qp_get_hw_data; }; -extern struct qat_qp_hw_spec_funcs *qat_qp_hw_spec[]; +extern struct qat_qp_hw_spec_funcs* + qat_qp_hw_spec[]; + +static inline void +txq_write_tail(enum qat_device_gen qat_dev_gen, + struct qat_qp *qp, struct qat_queue *q) +{ + struct qat_qp_hw_spec_funcs *ops = + qat_qp_hw_spec[qat_dev_gen]; + + /* + * Pointer check should be done during + * initialization + */ + ops->qat_qp_csr_write_tail(qp, q); +} #endif /* _QAT_QP_H_ */ diff --git a/drivers/compress/qat/qat_comp.c b/drivers/compress/qat/qat_comp.c index fe4a4999c6..559948a46a 100644 --- a/drivers/compress/qat/qat_comp.c +++ b/drivers/compress/qat/qat_comp.c @@ -1144,3 +1144,185 @@ qat_comp_stream_free(struct rte_compressdev *dev, void *stream) } return -EINVAL; } + +/** + * Enqueue packets for processing on queue pair of a device + * + * @param qp + * qat queue pair + * @param ops + * Compressdev operation + * @param nb_ops + * number of operations + * @return + * - nb_ops_sent if successful + */ +uint16_t +qat_enqueue_comp_op_burst(void *qp, void **ops, uint16_t nb_ops) +{ + register struct qat_queue *queue; + struct qat_qp *tmp_qp = (struct qat_qp *)qp; + register uint32_t nb_ops_sent = 0; + register int nb_desc_to_build; + uint16_t nb_ops_possible = nb_ops; + register uint8_t *base_addr; + register uint32_t tail; + + int descriptors_built, total_descriptors_built = 0; + int nb_remaining_descriptors; + int overflow = 0; + + if (unlikely(nb_ops == 0)) + return 0; + + /* read params used a lot in main loop into registers */ + queue = &(tmp_qp->tx_q); + base_addr = (uint8_t *)queue->base_addr; + tail = queue->tail; + + /* Find how many can actually fit on the ring */ + { + /* dequeued can only be written by one thread, but it may not + * be this thread. As it's 4-byte aligned it will be read + * atomically here by any Intel CPU. + * enqueued can wrap before dequeued, but cannot + * lap it as var size of enq/deq (uint32_t) > var size of + * max_inflights (uint16_t). In reality inflights is never + * even as big as max uint16_t, as it's <= ADF_MAX_DESC. + * On wrapping, the calculation still returns the correct + * positive value as all three vars are unsigned. + */ + uint32_t inflights = + tmp_qp->enqueued - tmp_qp->dequeued; + + /* Find how many can actually fit on the ring */ + overflow = (inflights + nb_ops) - tmp_qp->max_inflights; + if (overflow > 0) { + nb_ops_possible = nb_ops - overflow; + if (nb_ops_possible == 0) + return 0; + } + + /* QAT has plenty of work queued already, so don't waste cycles + * enqueueing, wait til the application has gathered a bigger + * burst or some completed ops have been dequeued + */ + if (tmp_qp->min_enq_burst_threshold && inflights > + QAT_QP_MIN_INFL_THRESHOLD && nb_ops_possible < + tmp_qp->min_enq_burst_threshold) { + tmp_qp->stats.threshold_hit_count++; + return 0; + } + } + + /* At this point nb_ops_possible is assuming a 1:1 mapping + * between ops and descriptors. + * Fewer may be sent if some ops have to be split. + * nb_ops_possible is <= burst size. + * Find out how many spaces are actually available on the qp in case + * more are needed. + */ + nb_remaining_descriptors = nb_ops_possible + + ((overflow >= 0) ? 0 : overflow * (-1)); + QAT_DP_LOG(DEBUG, "Nb ops requested %d, nb descriptors remaining %d", + nb_ops, nb_remaining_descriptors); + + while (nb_ops_sent != nb_ops_possible && + nb_remaining_descriptors > 0) { + struct qat_comp_op_cookie *cookie = + tmp_qp->op_cookies[tail >> queue->trailz]; + + descriptors_built = 0; + + QAT_DP_LOG(DEBUG, "--- data length: %u", + ((struct rte_comp_op *)*ops)->src.length); + + nb_desc_to_build = qat_comp_build_request(*ops, + base_addr + tail, cookie, tmp_qp->qat_dev_gen); + QAT_DP_LOG(DEBUG, "%d descriptors built, %d remaining, " + "%d ops sent, %d descriptors needed", + total_descriptors_built, nb_remaining_descriptors, + nb_ops_sent, nb_desc_to_build); + + if (unlikely(nb_desc_to_build < 0)) { + /* this message cannot be enqueued */ + tmp_qp->stats.enqueue_err_count++; + if (nb_ops_sent == 0) + return 0; + goto kick_tail; + } else if (unlikely(nb_desc_to_build > 1)) { + /* this op is too big and must be split - get more + * descriptors and retry + */ + + QAT_DP_LOG(DEBUG, "Build %d descriptors for this op", + nb_desc_to_build); + + nb_remaining_descriptors -= nb_desc_to_build; + if (nb_remaining_descriptors >= 0) { + /* There are enough remaining descriptors + * so retry + */ + int ret2 = qat_comp_build_multiple_requests( + *ops, tmp_qp, tail, + nb_desc_to_build); + + if (unlikely(ret2 < 1)) { + QAT_DP_LOG(DEBUG, + "Failed to build (%d) descriptors, status %d", + nb_desc_to_build, ret2); + + qat_comp_free_split_op_memzones(cookie, + nb_desc_to_build - 1); + + tmp_qp->stats.enqueue_err_count++; + + /* This message cannot be enqueued */ + if (nb_ops_sent == 0) + return 0; + goto kick_tail; + } else { + descriptors_built = ret2; + total_descriptors_built += + descriptors_built; + nb_remaining_descriptors -= + descriptors_built; + QAT_DP_LOG(DEBUG, + "Multiple descriptors (%d) built ok", + descriptors_built); + } + } else { + QAT_DP_LOG(ERR, "For the current op, number of requested descriptors (%d) " + "exceeds number of available descriptors (%d)", + nb_desc_to_build, + nb_remaining_descriptors + + nb_desc_to_build); + + qat_comp_free_split_op_memzones(cookie, + nb_desc_to_build - 1); + + /* Not enough extra descriptors */ + if (nb_ops_sent == 0) + return 0; + goto kick_tail; + } + } else { + descriptors_built = 1; + total_descriptors_built++; + nb_remaining_descriptors--; + QAT_DP_LOG(DEBUG, "Single descriptor built ok"); + } + + tail = adf_modulo(tail + (queue->msg_size * descriptors_built), + queue->modulo_mask); + ops++; + nb_ops_sent++; + } + +kick_tail: + queue->tail = tail; + tmp_qp->enqueued += total_descriptors_built; + tmp_qp->stats.enqueued_count += nb_ops_sent; + txq_write_tail(tmp_qp->qat_dev_gen, tmp_qp, queue); + return nb_ops_sent; +} diff --git a/drivers/compress/qat/qat_comp.h b/drivers/compress/qat/qat_comp.h index da7b9a6eec..dc220cd6e3 100644 --- a/drivers/compress/qat/qat_comp.h +++ b/drivers/compress/qat/qat_comp.h @@ -141,5 +141,8 @@ qat_comp_stream_create(struct rte_compressdev *dev, int qat_comp_stream_free(struct rte_compressdev *dev, void *stream); +uint16_t +qat_enqueue_comp_op_burst(void *qp, void **ops, uint16_t nb_ops); + #endif #endif -- 2.25.1