patches for DPDK stable branches
 help / color / mirror / Atom feed
* [dpdk-stable] [19.11 1/4] common/qat: remove tail write coalescing
@ 2020-08-07 12:56 Arek Kusztal
  2020-08-07 12:56 ` [dpdk-stable] [19.11 2/4] common/qat: move max inflights param into qp Arek Kusztal
                   ` (2 more replies)
  0 siblings, 3 replies; 7+ messages in thread
From: Arek Kusztal @ 2020-08-07 12:56 UTC (permalink / raw)
  To: fiona.trahe; +Cc: stable, Arek Kusztal

From: Fiona Trahe <fiona.trahe@intel.com>

[ upstream commit 6cde900bd59d00ea970be085dbe01af63050ddf9 ]

The feature Coalescing Tail Writes on Enqueue is removed
as it is not thread-safe and a dual-thread feature will be added shortly.

Cc: stable@dpdk.org

Signed-off-by: Fiona Trahe <fiona.trahe@intel.com>
Signed-off-by: Arek Kusztal <arkadiuszx.kusztal@intel.com>
---
 drivers/common/qat/qat_qp.c | 16 +++-------------
 drivers/common/qat/qat_qp.h |  6 ------
 2 files changed, 3 insertions(+), 19 deletions(-)

diff --git a/drivers/common/qat/qat_qp.c b/drivers/common/qat/qat_qp.c
index e0c174d..d189fc3 100644
--- a/drivers/common/qat/qat_qp.c
+++ b/drivers/common/qat/qat_qp.c
@@ -542,7 +542,6 @@ static inline void
 txq_write_tail(struct qat_qp *qp, struct qat_queue *q) {
 	WRITE_CSR_RING_TAIL(qp->mmap_bar_addr, q->hw_bundle_number,
 			q->hw_queue_number, q->tail);
-	q->nb_pending_requests = 0;
 	q->csr_tail = q->tail;
 }
 
@@ -641,25 +640,20 @@ qat_enqueue_op_burst(void *qp, void **ops, uint16_t nb_ops)
 kick_tail:
 	queue->tail = tail;
 	tmp_qp->stats.enqueued_count += nb_ops_sent;
-	queue->nb_pending_requests += nb_ops_sent;
-	if (tmp_qp->inflights16 < QAT_CSR_TAIL_FORCE_WRITE_THRESH ||
-		    queue->nb_pending_requests > QAT_CSR_TAIL_WRITE_THRESH) {
-		txq_write_tail(tmp_qp, queue);
-	}
+	txq_write_tail(tmp_qp, queue);
 	return nb_ops_sent;
 }
 
 uint16_t
 qat_dequeue_op_burst(void *qp, void **ops, uint16_t nb_ops)
 {
-	struct qat_queue *rx_queue, *tx_queue;
+	struct qat_queue *rx_queue;
 	struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 	uint32_t head;
 	uint32_t resp_counter = 0;
 	uint8_t *resp_msg;
 
 	rx_queue = &(tmp_qp->rx_q);
-	tx_queue = &(tmp_qp->tx_q);
 	head = rx_queue->head;
 	resp_msg = (uint8_t *)rx_queue->base_addr + rx_queue->head;
 
@@ -696,11 +690,7 @@ qat_dequeue_op_burst(void *qp, void **ops, uint16_t nb_ops)
 						QAT_CSR_HEAD_WRITE_THRESH)
 			rxq_free_desc(tmp_qp, rx_queue);
 	}
-	/* also check if tail needs to be advanced */
-	if (tmp_qp->inflights16 <= QAT_CSR_TAIL_FORCE_WRITE_THRESH &&
-		tx_queue->tail != tx_queue->csr_tail) {
-		txq_write_tail(tmp_qp, tx_queue);
-	}
+
 	return resp_counter;
 }
 
diff --git a/drivers/common/qat/qat_qp.h b/drivers/common/qat/qat_qp.h
index d0bf915..a9a0184 100644
--- a/drivers/common/qat/qat_qp.h
+++ b/drivers/common/qat/qat_qp.h
@@ -11,10 +11,6 @@ struct qat_pci_device;
 
 #define QAT_CSR_HEAD_WRITE_THRESH 32U
 /* number of requests to accumulate before writing head CSR */
-#define QAT_CSR_TAIL_WRITE_THRESH 32U
-/* number of requests to accumulate before writing tail CSR */
-#define QAT_CSR_TAIL_FORCE_WRITE_THRESH 256U
-/* number of inflights below which no tail write coalescing should occur */
 
 typedef int (*build_request_t)(void *op,
 		uint8_t *req, void *op_cookie,
@@ -64,8 +60,6 @@ struct qat_queue {
 	uint32_t	csr_tail;		/* last written tail value */
 	uint16_t	nb_processed_responses;
 	/* number of responses processed since last CSR head write */
-	uint16_t	nb_pending_requests;
-	/* number of requests pending since last CSR tail write */
 };
 
 struct qat_qp {
-- 
2.1.0


^ permalink raw reply	[flat|nested] 7+ messages in thread

* [dpdk-stable] [19.11 2/4] common/qat: move max inflights param into qp
  2020-08-07 12:56 [dpdk-stable] [19.11 1/4] common/qat: remove tail write coalescing Arek Kusztal
@ 2020-08-07 12:56 ` Arek Kusztal
  2020-08-07 12:57 ` [dpdk-stable] [19.11 3/4] common/qat: support dual threads for enqueue/dequeue Arek Kusztal
  2020-08-07 12:57 ` [dpdk-stable] [19.11 4/4] crypto/qat: add minimum enq threshold Arek Kusztal
  2 siblings, 0 replies; 7+ messages in thread
From: Arek Kusztal @ 2020-08-07 12:56 UTC (permalink / raw)
  To: fiona.trahe; +Cc: stable, Arek Kusztal

From: Fiona Trahe <fiona.trahe@intel.com>

[ upstream commit 8f185e7c3e651eb845bd82f7ab0bbb862cc0a2e2 ]

The max_inflights parameter is moved from qat_queue to qat_qp as it's
a more appropriate location.

Cc: stable@dpdk.org

Signed-off-by: Fiona Trahe <fiona.trahe@intel.com>
Signed-off-by: Arek Kusztal <arkadiuszx.kusztal@intel.com>
Acked-by: Fiona Trahe <fiona.trahe@intel.com>
---
 drivers/common/qat/qat_qp.c | 23 ++++++++++++-----------
 drivers/common/qat/qat_qp.h |  2 +-
 2 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/drivers/common/qat/qat_qp.c b/drivers/common/qat/qat_qp.c
index d189fc3..791c469 100644
--- a/drivers/common/qat/qat_qp.c
+++ b/drivers/common/qat/qat_qp.c
@@ -242,6 +242,15 @@ int qat_qp_setup(struct qat_pci_device *qat_dev,
 		goto create_err;
 	}
 
+	qp->max_inflights = ADF_MAX_INFLIGHTS(qp->tx_q.queue_size,
+				ADF_BYTES_TO_MSG_SIZE(qp->tx_q.msg_size));
+
+	if (qp->max_inflights < 2) {
+		QAT_LOG(ERR, "Invalid num inflights");
+		qat_queue_delete(&(qp->tx_q));
+		goto create_err;
+	}
+
 	if (qat_queue_create(qat_dev, &(qp->rx_q), qat_qp_conf,
 					ADF_RING_DIR_RX) != 0) {
 		QAT_LOG(ERR, "Rx queue create failed "
@@ -420,15 +429,7 @@ qat_queue_create(struct qat_pci_device *qat_dev, struct qat_queue *queue,
 		goto queue_create_err;
 	}
 
-	queue->max_inflights = ADF_MAX_INFLIGHTS(queue->queue_size,
-					ADF_BYTES_TO_MSG_SIZE(desc_size));
 	queue->modulo_mask = (1 << ADF_RING_SIZE_MODULO(queue->queue_size)) - 1;
-
-	if (queue->max_inflights < 2) {
-		QAT_LOG(ERR, "Invalid num inflights");
-		ret = -EINVAL;
-		goto queue_create_err;
-	}
 	queue->head = 0;
 	queue->tail = 0;
 	queue->msg_size = desc_size;
@@ -447,11 +448,11 @@ qat_queue_create(struct qat_pci_device *qat_dev, struct qat_queue *queue,
 			queue->hw_queue_number, queue_base);
 
 	QAT_LOG(DEBUG, "RING: Name:%s, size in CSR: %u, in bytes %u,"
-		" nb msgs %u, msg_size %u, max_inflights %u modulo mask %u",
+		" nb msgs %u, msg_size %u, modulo mask %u",
 			queue->memz_name,
 			queue->queue_size, queue_size_bytes,
 			qp_conf->nb_descriptors, desc_size,
-			queue->max_inflights, queue->modulo_mask);
+			queue->modulo_mask);
 
 	return 0;
 
@@ -594,7 +595,7 @@ qat_enqueue_op_burst(void *qp, void **ops, uint16_t nb_ops)
 
 	/* Find how many can actually fit on the ring */
 	tmp_qp->inflights16 += nb_ops;
-	overflow = tmp_qp->inflights16 - queue->max_inflights;
+	overflow = tmp_qp->inflights16 - tmp_qp->max_inflights;
 	if (overflow > 0) {
 		tmp_qp->inflights16 -= overflow;
 		nb_ops_possible = nb_ops - overflow;
diff --git a/drivers/common/qat/qat_qp.h b/drivers/common/qat/qat_qp.h
index a9a0184..02a613a 100644
--- a/drivers/common/qat/qat_qp.h
+++ b/drivers/common/qat/qat_qp.h
@@ -51,7 +51,6 @@ struct qat_queue {
 	uint32_t	tail;			/* Shadow copy of the tail */
 	uint32_t	modulo_mask;
 	uint32_t	msg_size;
-	uint16_t	max_inflights;
 	uint32_t	queue_size;
 	uint8_t		hw_bundle_number;
 	uint8_t		hw_queue_number;
@@ -76,6 +75,7 @@ struct qat_qp {
 	enum qat_service_type service_type;
 	struct qat_pci_device *qat_dev;
 	/**< qat device this qp is on */
+	uint16_t max_inflights;
 } __rte_cache_aligned;
 
 extern const struct qat_qp_hw_data qat_gen1_qps[][ADF_MAX_QPS_ON_ANY_SERVICE];
-- 
2.1.0


^ permalink raw reply	[flat|nested] 7+ messages in thread

* [dpdk-stable] [19.11 3/4] common/qat: support dual threads for enqueue/dequeue
  2020-08-07 12:56 [dpdk-stable] [19.11 1/4] common/qat: remove tail write coalescing Arek Kusztal
  2020-08-07 12:56 ` [dpdk-stable] [19.11 2/4] common/qat: move max inflights param into qp Arek Kusztal
@ 2020-08-07 12:57 ` Arek Kusztal
  2020-08-07 12:57 ` [dpdk-stable] [19.11 4/4] crypto/qat: add minimum enq threshold Arek Kusztal
  2 siblings, 0 replies; 7+ messages in thread
From: Arek Kusztal @ 2020-08-07 12:57 UTC (permalink / raw)
  To: fiona.trahe; +Cc: stable, Arek Kusztal

From: Fiona Trahe <fiona.trahe@intel.com>

[ upstream commit 026f21c0b95120a4e249af4480a0ddad75838ff9 ]

Remove the limitation whereby enqueue and dequeue must be
done in same thread.
The inflight calculation is reworked to be thread-safe for 2
threads - note this is not general multi-thread support, i.e
all enqueues to a qp must still be done in one thread and
all dequeues must be done in one thread, but enqueues and
dequeues may be in separate threads.
Documentation updated.

Cc: stable@dpdk.org

Signed-off-by: Fiona Trahe <fiona.trahe@intel.com>
Signed-off-by: Arek Kusztal <arkadiuszx.kusztal@intel.com>
Acked-by: Fiona Trahe <fiona.trahe@intel.com>
---
 doc/guides/compressdevs/qat_comp.rst |  5 ++++-
 doc/guides/cryptodevs/qat.rst        | 10 +++++++--
 drivers/common/qat/qat_qp.c          | 40 +++++++++++++++++++++---------------
 drivers/common/qat/qat_qp.h          |  3 ++-
 4 files changed, 38 insertions(+), 20 deletions(-)

diff --git a/doc/guides/compressdevs/qat_comp.rst b/doc/guides/compressdevs/qat_comp.rst
index 6421f76..757611a 100644
--- a/doc/guides/compressdevs/qat_comp.rst
+++ b/doc/guides/compressdevs/qat_comp.rst
@@ -37,7 +37,10 @@ Limitations
 -----------
 
 * Compressdev level 0, no compression, is not supported.
-* Queue pairs are not thread-safe (that is, within a single queue pair, RX and TX from different lcores is not supported).
+* Queue-pairs are thread-safe on Intel CPUs but Queues are not (that is, within a single
+  queue-pair all enqueues to the TX queue must be done from one thread and all dequeues
+  from the RX queue must be done from one thread, but enqueues and dequeues may be done
+  in different threads.)
 * No BSD support as BSD QAT kernel driver not available.
 * When using Deflate dynamic huffman encoding for compression, the input size (op.src.length)
   must be < CONFIG_RTE_PMD_QAT_COMP_IM_BUFFER_SIZE from the config file,
diff --git a/doc/guides/cryptodevs/qat.rst b/doc/guides/cryptodevs/qat.rst
index 8981a1e..5135703 100644
--- a/doc/guides/cryptodevs/qat.rst
+++ b/doc/guides/cryptodevs/qat.rst
@@ -109,7 +109,10 @@ Limitations
 * No BSD support as BSD QAT kernel driver not available.
 * ZUC EEA3/EIA3 is not supported by dh895xcc devices
 * Maximum additional authenticated data (AAD) for GCM is 240 bytes long and must be passed to the device in a buffer rounded up to the nearest block-size multiple (x16) and padded with zeros.
-* Queue pairs are not thread-safe (that is, within a single queue pair, RX and TX from different lcores is not supported).
+* Queue-pairs are thread-safe on Intel CPUs but Queues are not (that is, within a single
+  queue-pair all enqueues to the TX queue must be done from one thread and all dequeues
+  from the RX queue must be done from one thread, but enqueues and dequeues may be done
+  in different threads.)
 * A GCM limitation exists, but only in the case where there are multiple
   generations of QAT devices on a single platform.
   To optimise performance, the GCM crypto session should be initialised for the
@@ -163,7 +166,10 @@ Limitations
 ~~~~~~~~~~~
 
 * Big integers longer than 4096 bits are not supported.
-* Queue pairs are not thread-safe (that is, within a single queue pair, RX and TX from different lcores is not supported).
+* Queue-pairs are thread-safe on Intel CPUs but Queues are not (that is, within a single
+  queue-pair all enqueues to the TX queue must be done from one thread and all dequeues
+  from the RX queue must be done from one thread, but enqueues and dequeues may be done
+  in different threads.)
 * RSA-2560, RSA-3584 are not supported
 
 .. _building_qat:
diff --git a/drivers/common/qat/qat_qp.c b/drivers/common/qat/qat_qp.c
index 791c469..0725708 100644
--- a/drivers/common/qat/qat_qp.c
+++ b/drivers/common/qat/qat_qp.c
@@ -233,7 +233,7 @@ int qat_qp_setup(struct qat_pci_device *qat_dev,
 	}
 
 	qp->mmap_bar_addr = pci_dev->mem_resource[0].addr;
-	qp->inflights16 = 0;
+	qp->enqueued = qp->dequeued = 0;
 
 	if (qat_queue_create(qat_dev, &(qp->tx_q), qat_qp_conf,
 					ADF_RING_DIR_TX) != 0) {
@@ -324,7 +324,7 @@ int qat_qp_release(struct qat_qp **qp_addr)
 				qp->qat_dev->qat_dev_id);
 
 	/* Don't free memory if there are still responses to be processed */
-	if (qp->inflights16 == 0) {
+	if ((qp->enqueued - qp->dequeued) == 0) {
 		qat_queue_delete(&(qp->tx_q));
 		qat_queue_delete(&(qp->rx_q));
 	} else {
@@ -583,7 +583,6 @@ qat_enqueue_op_burst(void *qp, void **ops, uint16_t nb_ops)
 	uint16_t nb_ops_possible = nb_ops;
 	register uint8_t *base_addr;
 	register uint32_t tail;
-	int overflow;
 
 	if (unlikely(nb_ops == 0))
 		return 0;
@@ -594,13 +593,25 @@ qat_enqueue_op_burst(void *qp, void **ops, uint16_t nb_ops)
 	tail = queue->tail;
 
 	/* Find how many can actually fit on the ring */
-	tmp_qp->inflights16 += nb_ops;
-	overflow = tmp_qp->inflights16 - tmp_qp->max_inflights;
-	if (overflow > 0) {
-		tmp_qp->inflights16 -= overflow;
-		nb_ops_possible = nb_ops - overflow;
-		if (nb_ops_possible == 0)
-			return 0;
+	{
+		/* dequeued can only be written by one thread, but it may not
+		 * be this thread. As it's 4-byte aligned it will be read
+		 * atomically here by any Intel CPU.
+		 * enqueued can wrap before dequeued, but cannot
+		 * lap it as var size of enq/deq (uint32_t) > var size of
+		 * max_inflights (uint16_t). In reality inflights is never
+		 * even as big as max uint16_t, as it's <= ADF_MAX_DESC.
+		 * On wrapping, the calculation still returns the correct
+		 * positive value as all three vars are unsigned.
+		 */
+		uint32_t inflights =
+			tmp_qp->enqueued - tmp_qp->dequeued;
+
+		if ((inflights + nb_ops) > tmp_qp->max_inflights) {
+			nb_ops_possible = tmp_qp->max_inflights - inflights;
+			if (nb_ops_possible == 0)
+				return 0;
+		}
 	}
 
 	while (nb_ops_sent != nb_ops_possible) {
@@ -624,11 +635,7 @@ qat_enqueue_op_burst(void *qp, void **ops, uint16_t nb_ops)
 
 		if (ret != 0) {
 			tmp_qp->stats.enqueue_err_count++;
-			/*
-			 * This message cannot be enqueued,
-			 * decrease number of ops that wasn't sent
-			 */
-			tmp_qp->inflights16 -= nb_ops_possible - nb_ops_sent;
+			/* This message cannot be enqueued */
 			if (nb_ops_sent == 0)
 				return 0;
 			goto kick_tail;
@@ -640,6 +647,7 @@ qat_enqueue_op_burst(void *qp, void **ops, uint16_t nb_ops)
 	}
 kick_tail:
 	queue->tail = tail;
+	tmp_qp->enqueued += nb_ops_sent;
 	tmp_qp->stats.enqueued_count += nb_ops_sent;
 	txq_write_tail(tmp_qp, queue);
 	return nb_ops_sent;
@@ -683,9 +691,9 @@ qat_dequeue_op_burst(void *qp, void **ops, uint16_t nb_ops)
 	}
 	if (resp_counter > 0) {
 		rx_queue->head = head;
+		tmp_qp->dequeued += resp_counter;
 		tmp_qp->stats.dequeued_count += resp_counter;
 		rx_queue->nb_processed_responses += resp_counter;
-		tmp_qp->inflights16 -= resp_counter;
 
 		if (rx_queue->nb_processed_responses >
 						QAT_CSR_HEAD_WRITE_THRESH)
diff --git a/drivers/common/qat/qat_qp.h b/drivers/common/qat/qat_qp.h
index 02a613a..973e883 100644
--- a/drivers/common/qat/qat_qp.h
+++ b/drivers/common/qat/qat_qp.h
@@ -63,7 +63,6 @@ struct qat_queue {
 
 struct qat_qp {
 	void			*mmap_bar_addr;
-	uint16_t		inflights16;
 	struct qat_queue	tx_q;
 	struct qat_queue	rx_q;
 	struct qat_common_stats stats;
@@ -75,6 +74,8 @@ struct qat_qp {
 	enum qat_service_type service_type;
 	struct qat_pci_device *qat_dev;
 	/**< qat device this qp is on */
+	uint32_t enqueued;
+	uint32_t dequeued __rte_aligned(4);
 	uint16_t max_inflights;
 } __rte_cache_aligned;
 
-- 
2.1.0


^ permalink raw reply	[flat|nested] 7+ messages in thread

* [dpdk-stable] [19.11 4/4] crypto/qat: add minimum enq threshold
  2020-08-07 12:56 [dpdk-stable] [19.11 1/4] common/qat: remove tail write coalescing Arek Kusztal
  2020-08-07 12:56 ` [dpdk-stable] [19.11 2/4] common/qat: move max inflights param into qp Arek Kusztal
  2020-08-07 12:57 ` [dpdk-stable] [19.11 3/4] common/qat: support dual threads for enqueue/dequeue Arek Kusztal
@ 2020-08-07 12:57 ` Arek Kusztal
  2020-08-07 14:53   ` Luca Boccassi
  2 siblings, 1 reply; 7+ messages in thread
From: Arek Kusztal @ 2020-08-07 12:57 UTC (permalink / raw)
  To: fiona.trahe; +Cc: Arek Kusztal, stable

[ upstream commit 47c3f7a41a26c9943f9804691d03828b2ce09f40 ]

This patch adds minimum enqueue threshold to Intel
QuickAssist Technology PMD.
It is an optimisation, configured by a command line option,
which can be used to reduce MMIO write occurrences.

Cc: stable@dpdk.org

Signed-off-by: Fiona Trahe <fiona.trahe@intel.com>
Signed-off-by: Arek Kusztal <arkadiuszx.kusztal@intel.com>
---
 doc/guides/cryptodevs/qat.rst       | 18 +++++++
 drivers/common/qat/qat_common.c     |  3 ++
 drivers/common/qat/qat_common.h     |  3 ++
 drivers/common/qat/qat_device.c     | 99 ++++++++++++++++++++++++++++++++-----
 drivers/common/qat/qat_device.h     | 22 +++++++--
 drivers/common/qat/qat_qp.c         | 10 ++++
 drivers/common/qat/qat_qp.h         |  3 ++
 drivers/compress/qat/qat_comp_pmd.c | 14 +++++-
 drivers/compress/qat/qat_comp_pmd.h |  4 +-
 drivers/crypto/qat/qat_asym_pmd.c   | 14 +++++-
 drivers/crypto/qat/qat_asym_pmd.h   |  4 +-
 drivers/crypto/qat/qat_sym_pmd.c    | 14 +++++-
 drivers/crypto/qat/qat_sym_pmd.h    |  4 +-
 13 files changed, 190 insertions(+), 22 deletions(-)

diff --git a/doc/guides/cryptodevs/qat.rst b/doc/guides/cryptodevs/qat.rst
index 5135703..aa3cdcf 100644
--- a/doc/guides/cryptodevs/qat.rst
+++ b/doc/guides/cryptodevs/qat.rst
@@ -273,6 +273,24 @@ allocated while for GEN1 devices, 12 buffers are allocated, plus 1472 bytes over
 	larger than the input size).
 
 
+Running QAT PMD with minimum threshold for burst size
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+If only a small number or packets can be enqueued. Each enqueue causes an expensive MMIO write.
+These MMIO write occurrences can be optimised by setting any of the following parameters
+- qat_sym_enq_threshold
+- qat_asym_enq_threshold
+- qat_comp_enq_threshold
+When any of these parameters is set rte_cryptodev_enqueue_burst function will
+return 0 (thereby avoiding an MMIO) if the device is congested and number of packets
+possible to enqueue is smaller.
+To use this feature the user must set the parameter on process start as a device additional parameter:
+ .example: '-w 03:01.1,qat_sym_enq_threshold=32,qat_comp_enq_threshold=16'
+All parameters can be used with the same device regardless of order. Parameters are separated
+by comma. When the same parameter is used more than once first occurrence of the parameter
+is used.
+Maximum threshold that can be set is 32.
+
 Device and driver naming
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/drivers/common/qat/qat_common.c b/drivers/common/qat/qat_common.c
index 4753866..5343a14 100644
--- a/drivers/common/qat/qat_common.c
+++ b/drivers/common/qat/qat_common.c
@@ -94,6 +94,9 @@ void qat_stats_get(struct qat_pci_device *dev,
 		stats->dequeued_count += qp[i]->stats.dequeued_count;
 		stats->enqueue_err_count += qp[i]->stats.enqueue_err_count;
 		stats->dequeue_err_count += qp[i]->stats.dequeue_err_count;
+		stats->threshold_hit_count += qp[i]->stats.threshold_hit_count;
+		QAT_LOG(DEBUG, "Threshold was used for qp %d %"PRIu64" times",
+				i, stats->threshold_hit_count);
 	}
 }
 
diff --git a/drivers/common/qat/qat_common.h b/drivers/common/qat/qat_common.h
index de9a3ba..cf840fe 100644
--- a/drivers/common/qat/qat_common.h
+++ b/drivers/common/qat/qat_common.h
@@ -61,6 +61,9 @@ struct qat_common_stats {
 	/**< Total error count on operations enqueued */
 	uint64_t dequeue_err_count;
 	/**< Total error count on operations dequeued */
+	uint64_t threshold_hit_count;
+	/**< Total number of times min qp threshold condition was fulfilled */
+
 };
 
 struct qat_pci_device;
diff --git a/drivers/common/qat/qat_device.c b/drivers/common/qat/qat_device.c
index 18dbdd5..03f5fd1 100644
--- a/drivers/common/qat/qat_device.c
+++ b/drivers/common/qat/qat_device.c
@@ -3,6 +3,8 @@
  */
 
 #include <rte_string_fns.h>
+#include <rte_devargs.h>
+#include <ctype.h>
 
 #include "qat_device.h"
 #include "adf_transport_access_macros.h"
@@ -100,12 +102,72 @@ qat_get_qat_dev_from_pci_dev(struct rte_pci_device *pci_dev)
 	return qat_pci_get_named_dev(name);
 }
 
+static void qat_dev_parse_cmd(const char *str, struct qat_dev_cmd_param
+		*qat_dev_cmd_param)
+{
+	int i = 0;
+	const char *param;
+
+	while (1) {
+		char value_str[4] = { };
+
+		param = qat_dev_cmd_param[i].name;
+		if (param == NULL)
+			return;
+		long value = 0;
+		const char *arg = strstr(str, param);
+		const char *arg2 = NULL;
+
+		if (arg) {
+			arg2 = arg + strlen(param);
+			if (*arg2 != '=') {
+			QAT_LOG(DEBUG, "parsing error '=' sign"
+						" should immediately follow %s",
+						param);
+				arg2 = NULL;
+			} else
+				arg2++;
+		} else {
+			QAT_LOG(DEBUG, "%s not provided", param);
+		}
+		if (arg2) {
+			int iter = 0;
+
+			while (iter < 2) {
+				if (!isdigit(*(arg2 + iter)))
+					break;
+				iter++;
+			}
+			if (!iter) {
+				QAT_LOG(DEBUG, "parsing error %s"
+					       " no number provided",
+					       param);
+			} else {
+				memcpy(value_str, arg2, iter);
+				value = strtol(value_str, NULL, 10);
+				if (value > MAX_QP_THRESHOLD_SIZE) {
+					QAT_LOG(DEBUG, "Exceeded max size of"
+						" threshold, setting to %d",
+						MAX_QP_THRESHOLD_SIZE);
+					value = MAX_QP_THRESHOLD_SIZE;
+				}
+				QAT_LOG(DEBUG, "parsing %s = %ld",
+						param, value);
+			}
+		}
+		qat_dev_cmd_param[i].val = value;
+		i++;
+	}
+}
+
 struct qat_pci_device *
-qat_pci_device_allocate(struct rte_pci_device *pci_dev)
+qat_pci_device_allocate(struct rte_pci_device *pci_dev,
+		struct qat_dev_cmd_param *qat_dev_cmd_param)
 {
 	struct qat_pci_device *qat_dev;
 	uint8_t qat_dev_id = 0;
 	char name[QAT_DEV_NAME_MAX_LEN];
+	struct rte_devargs *devargs = pci_dev->device.devargs;
 
 	rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
 	snprintf(name+strlen(name), QAT_DEV_NAME_MAX_LEN-strlen(name), "_qat");
@@ -174,6 +236,9 @@ qat_pci_device_allocate(struct rte_pci_device *pci_dev)
 		return NULL;
 	}
 
+	if (devargs && devargs->drv_str)
+		qat_dev_parse_cmd(devargs->drv_str, qat_dev_cmd_param);
+
 	rte_spinlock_init(&qat_dev->arb_csr_lock);
 
 	qat_nb_pci_devices++;
@@ -244,37 +309,44 @@ qat_pci_dev_destroy(struct qat_pci_device *qat_pci_dev,
 static int qat_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 		struct rte_pci_device *pci_dev)
 {
-	int ret = 0;
+	int sym_ret = 0, asym_ret = 0, comp_ret = 0;
 	int num_pmds_created = 0;
 	struct qat_pci_device *qat_pci_dev;
+	struct qat_dev_cmd_param qat_dev_cmd_param[] = {
+			{ SYM_ENQ_THRESHOLD_NAME, 0 },
+			{ ASYM_ENQ_THRESHOLD_NAME, 0 },
+			{ COMP_ENQ_THRESHOLD_NAME, 0 },
+			{ NULL, 0 },
+	};
 
 	QAT_LOG(DEBUG, "Found QAT device at %02x:%02x.%x",
 			pci_dev->addr.bus,
 			pci_dev->addr.devid,
 			pci_dev->addr.function);
 
-	qat_pci_dev = qat_pci_device_allocate(pci_dev);
+	qat_pci_dev = qat_pci_device_allocate(pci_dev, qat_dev_cmd_param);
 	if (qat_pci_dev == NULL)
 		return -ENODEV;
 
-	ret = qat_sym_dev_create(qat_pci_dev);
-	if (ret == 0)
+	sym_ret = qat_sym_dev_create(qat_pci_dev, qat_dev_cmd_param);
+	if (sym_ret == 0) {
 		num_pmds_created++;
+	}
 	else
 		QAT_LOG(WARNING,
 				"Failed to create QAT SYM PMD on device %s",
 				qat_pci_dev->name);
 
-	ret = qat_comp_dev_create(qat_pci_dev);
-	if (ret == 0)
+	comp_ret = qat_comp_dev_create(qat_pci_dev, qat_dev_cmd_param);
+	if (comp_ret == 0)
 		num_pmds_created++;
 	else
 		QAT_LOG(WARNING,
 				"Failed to create QAT COMP PMD on device %s",
 				qat_pci_dev->name);
 
-	ret = qat_asym_dev_create(qat_pci_dev);
-	if (ret == 0)
+	asym_ret = qat_asym_dev_create(qat_pci_dev, qat_dev_cmd_param);
+	if (asym_ret == 0)
 		num_pmds_created++;
 	else
 		QAT_LOG(WARNING,
@@ -309,13 +381,15 @@ static struct rte_pci_driver rte_qat_pmd = {
 };
 
 __rte_weak int
-qat_sym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused)
+qat_sym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused,
+		struct qat_dev_cmd_param *qat_dev_cmd_param __rte_unused)
 {
 	return 0;
 }
 
 __rte_weak int
-qat_asym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused)
+qat_asym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused,
+		struct qat_dev_cmd_param *qat_dev_cmd_param __rte_unused)
 {
 	return 0;
 }
@@ -333,7 +407,8 @@ qat_asym_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused)
 }
 
 __rte_weak int
-qat_comp_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused)
+qat_comp_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused,
+		struct qat_dev_cmd_param *qat_dev_cmd_param __rte_unused)
 {
 	return 0;
 }
diff --git a/drivers/common/qat/qat_device.h b/drivers/common/qat/qat_device.h
index a23975f..c3f5ae8 100644
--- a/drivers/common/qat/qat_device.h
+++ b/drivers/common/qat/qat_device.h
@@ -16,6 +16,16 @@
 
 #define QAT_DEV_NAME_MAX_LEN	64
 
+#define SYM_ENQ_THRESHOLD_NAME "qat_sym_enq_threshold"
+#define ASYM_ENQ_THRESHOLD_NAME "qat_asym_enq_threshold"
+#define COMP_ENQ_THRESHOLD_NAME "qat_comp_enq_threshold"
+#define MAX_QP_THRESHOLD_SIZE	32
+
+struct qat_dev_cmd_param {
+	const char *name;
+	uint16_t val;
+};
+
 enum qat_comp_num_im_buffers {
 	QAT_NUM_INTERM_BUFS_GEN1 = 12,
 	QAT_NUM_INTERM_BUFS_GEN2 = 20,
@@ -106,7 +116,8 @@ struct qat_gen_hw_data {
 extern struct qat_gen_hw_data qat_gen_config[];
 
 struct qat_pci_device *
-qat_pci_device_allocate(struct rte_pci_device *pci_dev);
+qat_pci_device_allocate(struct rte_pci_device *pci_dev,
+		struct qat_dev_cmd_param *qat_dev_cmd_param);
 
 int
 qat_pci_device_release(struct rte_pci_device *pci_dev);
@@ -116,10 +127,12 @@ qat_get_qat_dev_from_pci_dev(struct rte_pci_device *pci_dev);
 
 /* declaration needed for weak functions */
 int
-qat_sym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused);
+qat_sym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused,
+		struct qat_dev_cmd_param *qat_dev_cmd_param);
 
 int
-qat_asym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused);
+qat_asym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused,
+		struct qat_dev_cmd_param *qat_dev_cmd_param);
 
 int
 qat_sym_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused);
@@ -128,7 +141,8 @@ int
 qat_asym_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused);
 
 int
-qat_comp_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused);
+qat_comp_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused,
+		struct qat_dev_cmd_param *qat_dev_cmd_param);
 
 int
 qat_comp_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused);
diff --git a/drivers/common/qat/qat_qp.c b/drivers/common/qat/qat_qp.c
index 0725708..3f30871 100644
--- a/drivers/common/qat/qat_qp.c
+++ b/drivers/common/qat/qat_qp.c
@@ -612,6 +612,16 @@ qat_enqueue_op_burst(void *qp, void **ops, uint16_t nb_ops)
 			if (nb_ops_possible == 0)
 				return 0;
 		}
+		/* QAT has plenty of work queued already, so don't waste cycles
+		 * enqueueing, wait til the application has gathered a bigger
+		 * burst or some completed ops have been dequeued
+		 */
+		if (tmp_qp->min_enq_burst_threshold && inflights >
+				QAT_QP_MIN_INFL_THRESHOLD && nb_ops_possible <
+				tmp_qp->min_enq_burst_threshold) {
+			tmp_qp->stats.threshold_hit_count++;
+			return 0;
+		}
 	}
 
 	while (nb_ops_sent != nb_ops_possible) {
diff --git a/drivers/common/qat/qat_qp.h b/drivers/common/qat/qat_qp.h
index 973e883..47ad5dd 100644
--- a/drivers/common/qat/qat_qp.h
+++ b/drivers/common/qat/qat_qp.h
@@ -12,6 +12,8 @@ struct qat_pci_device;
 #define QAT_CSR_HEAD_WRITE_THRESH 32U
 /* number of requests to accumulate before writing head CSR */
 
+#define QAT_QP_MIN_INFL_THRESHOLD	256
+
 typedef int (*build_request_t)(void *op,
 		uint8_t *req, void *op_cookie,
 		enum qat_device_gen qat_dev_gen);
@@ -77,6 +79,7 @@ struct qat_qp {
 	uint32_t enqueued;
 	uint32_t dequeued __rte_aligned(4);
 	uint16_t max_inflights;
+	uint16_t min_enq_burst_threshold;
 } __rte_cache_aligned;
 
 extern const struct qat_qp_hw_data qat_gen1_qps[][ADF_MAX_QPS_ON_ANY_SERVICE];
diff --git a/drivers/compress/qat/qat_comp_pmd.c b/drivers/compress/qat/qat_comp_pmd.c
index cb25916..5cdabad 100644
--- a/drivers/compress/qat/qat_comp_pmd.c
+++ b/drivers/compress/qat/qat_comp_pmd.c
@@ -139,6 +139,7 @@ qat_comp_qp_setup(struct rte_compressdev *dev, uint16_t qp_id,
 								= *qp_addr;
 
 	qp = (struct qat_qp *)*qp_addr;
+	qp->min_enq_burst_threshold = qat_private->min_enq_burst_threshold;
 
 	for (i = 0; i < qp->nb_descriptors; i++) {
 
@@ -660,8 +661,10 @@ static const struct rte_driver compdev_qat_driver = {
 	.alias = qat_comp_drv_name
 };
 int
-qat_comp_dev_create(struct qat_pci_device *qat_pci_dev)
+qat_comp_dev_create(struct qat_pci_device *qat_pci_dev,
+		struct qat_dev_cmd_param *qat_dev_cmd_param)
 {
+	int i = 0;
 	struct qat_device_info *qat_dev_instance =
 			&qat_pci_devs[qat_pci_dev->qat_dev_id];
 	if (qat_pci_dev->qat_dev_gen == QAT_GEN3) {
@@ -751,6 +754,15 @@ qat_comp_dev_create(struct qat_pci_device *qat_pci_dev)
 	memcpy(comp_dev->capa_mz->addr, capabilities, capa_size);
 	comp_dev->qat_dev_capabilities = comp_dev->capa_mz->addr;
 
+	while (1) {
+		if (qat_dev_cmd_param[i].name == NULL)
+			break;
+		if (!strcmp(qat_dev_cmd_param[i].name, COMP_ENQ_THRESHOLD_NAME))
+			comp_dev->min_enq_burst_threshold =
+					qat_dev_cmd_param[i].val;
+		i++;
+	}
+
 	qat_pci_dev->comp_dev = comp_dev;
 	QAT_LOG(DEBUG,
 		    "Created QAT COMP device %s as compressdev instance %d",
diff --git a/drivers/compress/qat/qat_comp_pmd.h b/drivers/compress/qat/qat_comp_pmd.h
index bd5a64f..ed27120 100644
--- a/drivers/compress/qat/qat_comp_pmd.h
+++ b/drivers/compress/qat/qat_comp_pmd.h
@@ -34,10 +34,12 @@ struct qat_comp_dev_private {
 	/**< The device's pool for qat_comp_streams */
 	const struct rte_memzone *capa_mz;
 	/* Shared memzone for storing capabilities */
+	uint16_t min_enq_burst_threshold;
 };
 
 int
-qat_comp_dev_create(struct qat_pci_device *qat_pci_dev);
+qat_comp_dev_create(struct qat_pci_device *qat_pci_dev,
+		struct qat_dev_cmd_param *qat_dev_cmd_param);
 
 int
 qat_comp_dev_destroy(struct qat_pci_device *qat_pci_dev);
diff --git a/drivers/crypto/qat/qat_asym_pmd.c b/drivers/crypto/qat/qat_asym_pmd.c
index 7ac41f2..2b69b8e 100644
--- a/drivers/crypto/qat/qat_asym_pmd.c
+++ b/drivers/crypto/qat/qat_asym_pmd.c
@@ -160,6 +160,7 @@ static int qat_asym_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 							= *qp_addr;
 
 	qp = (struct qat_qp *)*qp_addr;
+	qp->min_enq_burst_threshold = qat_private->min_enq_burst_threshold;
 
 	for (i = 0; i < qp->nb_descriptors; i++) {
 		int j;
@@ -235,8 +236,10 @@ static const struct rte_driver cryptodev_qat_asym_driver = {
 };
 
 int
-qat_asym_dev_create(struct qat_pci_device *qat_pci_dev)
+qat_asym_dev_create(struct qat_pci_device *qat_pci_dev,
+		struct qat_dev_cmd_param *qat_dev_cmd_param)
 {
+	int i = 0;
 	struct qat_device_info *qat_dev_instance =
 			&qat_pci_devs[qat_pci_dev->qat_dev_id];
 
@@ -325,6 +328,15 @@ qat_asym_dev_create(struct qat_pci_device *qat_pci_dev)
 			sizeof(qat_gen1_asym_capabilities));
 	internals->qat_dev_capabilities = internals->capa_mz->addr;
 
+	while (1) {
+		if (qat_dev_cmd_param[i].name == NULL)
+			break;
+		if (!strcmp(qat_dev_cmd_param[i].name, ASYM_ENQ_THRESHOLD_NAME))
+			internals->min_enq_burst_threshold =
+					qat_dev_cmd_param[i].val;
+		i++;
+	}
+
 	qat_pci_dev->asym_dev = internals;
 	QAT_LOG(DEBUG, "Created QAT ASYM device %s as cryptodev instance %d",
 			cryptodev->data->name, internals->asym_dev_id);
diff --git a/drivers/crypto/qat/qat_asym_pmd.h b/drivers/crypto/qat/qat_asym_pmd.h
index 3efb900..3b5abdd 100644
--- a/drivers/crypto/qat/qat_asym_pmd.h
+++ b/drivers/crypto/qat/qat_asym_pmd.h
@@ -28,6 +28,7 @@ struct qat_asym_dev_private {
 	/* QAT device asymmetric crypto capabilities */
 	const struct rte_memzone *capa_mz;
 	/* Shared memzone for storing capabilities */
+	uint16_t min_enq_burst_threshold;
 };
 
 uint16_t
@@ -44,7 +45,8 @@ int qat_asym_session_configure(struct rte_cryptodev *dev,
 		struct rte_mempool *mempool);
 
 int
-qat_asym_dev_create(struct qat_pci_device *qat_pci_dev);
+qat_asym_dev_create(struct qat_pci_device *qat_pci_dev,
+		struct qat_dev_cmd_param *qat_dev_cmd_param);
 
 int
 qat_asym_dev_destroy(struct qat_pci_device *qat_pci_dev);
diff --git a/drivers/crypto/qat/qat_sym_pmd.c b/drivers/crypto/qat/qat_sym_pmd.c
index 7a0d4c5..31ed6a9 100644
--- a/drivers/crypto/qat/qat_sym_pmd.c
+++ b/drivers/crypto/qat/qat_sym_pmd.c
@@ -177,6 +177,7 @@ static int qat_sym_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 							= *qp_addr;
 
 	qp = (struct qat_qp *)*qp_addr;
+	qp->min_enq_burst_threshold = qat_private->min_enq_burst_threshold;
 
 	for (i = 0; i < qp->nb_descriptors; i++) {
 
@@ -270,8 +271,10 @@ static const struct rte_driver cryptodev_qat_sym_driver = {
 };
 
 int
-qat_sym_dev_create(struct qat_pci_device *qat_pci_dev)
+qat_sym_dev_create(struct qat_pci_device *qat_pci_dev,
+		struct qat_dev_cmd_param *qat_dev_cmd_param __rte_unused)
 {
+	int i = 0;
 	struct qat_device_info *qat_dev_instance =
 			&qat_pci_devs[qat_pci_dev->qat_dev_id];
 
@@ -392,6 +395,15 @@ qat_sym_dev_create(struct qat_pci_device *qat_pci_dev)
 	memcpy(internals->capa_mz->addr, capabilities, capa_size);
 	internals->qat_dev_capabilities = internals->capa_mz->addr;
 
+	while (1) {
+		if (qat_dev_cmd_param[i].name == NULL)
+			break;
+		if (!strcmp(qat_dev_cmd_param[i].name, SYM_ENQ_THRESHOLD_NAME))
+			internals->min_enq_burst_threshold =
+					qat_dev_cmd_param[i].val;
+		i++;
+	}
+
 	qat_pci_dev->sym_dev = internals;
 	QAT_LOG(DEBUG, "Created QAT SYM device %s as cryptodev instance %d",
 			cryptodev->data->name, internals->sym_dev_id);
diff --git a/drivers/crypto/qat/qat_sym_pmd.h b/drivers/crypto/qat/qat_sym_pmd.h
index 6137c46..e2ed112 100644
--- a/drivers/crypto/qat/qat_sym_pmd.h
+++ b/drivers/crypto/qat/qat_sym_pmd.h
@@ -35,10 +35,12 @@ struct qat_sym_dev_private {
 	uint32_t internal_capabilities; /* see flags QAT_SYM_CAP_xxx */
 	const struct rte_memzone *capa_mz;
 	/* Shared memzone for storing capabilities */
+	uint16_t min_enq_burst_threshold;
 };
 
 int
-qat_sym_dev_create(struct qat_pci_device *qat_pci_dev);
+qat_sym_dev_create(struct qat_pci_device *qat_pci_dev,
+		struct qat_dev_cmd_param *qat_dev_cmd_param);
 
 int
 qat_sym_dev_destroy(struct qat_pci_device *qat_pci_dev);
-- 
2.1.0


^ permalink raw reply	[flat|nested] 7+ messages in thread

* Re: [dpdk-stable] [19.11 4/4] crypto/qat: add minimum enq threshold
  2020-08-07 12:57 ` [dpdk-stable] [19.11 4/4] crypto/qat: add minimum enq threshold Arek Kusztal
@ 2020-08-07 14:53   ` Luca Boccassi
  2020-08-07 15:24     ` Trahe, Fiona
  0 siblings, 1 reply; 7+ messages in thread
From: Luca Boccassi @ 2020-08-07 14:53 UTC (permalink / raw)
  To: Arek Kusztal, fiona.trahe; +Cc: stable

On Fri, 2020-08-07 at 14:57 +0200, Arek Kusztal wrote:
> [ upstream commit 47c3f7a41a26c9943f9804691d03828b2ce09f40 ]
> 
> This patch adds minimum enqueue threshold to Intel
> QuickAssist Technology PMD.
> It is an optimisation, configured by a command line option,
> which can be used to reduce MMIO write occurrences.
> 
> Cc: stable@dpdk.org
> 
> Signed-off-by: Fiona Trahe <fiona.trahe@intel.com>
> Signed-off-by: Arek Kusztal <arkadiuszx.kusztal@intel.com>
> ---
>  doc/guides/cryptodevs/qat.rst       | 18 +++++++
>  drivers/common/qat/qat_common.c     |  3 ++
>  drivers/common/qat/qat_common.h     |  3 ++
>  drivers/common/qat/qat_device.c     | 99 ++++++++++++++++++++++++++++++++-----
>  drivers/common/qat/qat_device.h     | 22 +++++++--
>  drivers/common/qat/qat_qp.c         | 10 ++++
>  drivers/common/qat/qat_qp.h         |  3 ++
>  drivers/compress/qat/qat_comp_pmd.c | 14 +++++-
>  drivers/compress/qat/qat_comp_pmd.h |  4 +-
>  drivers/crypto/qat/qat_asym_pmd.c   | 14 +++++-
>  drivers/crypto/qat/qat_asym_pmd.h   |  4 +-
>  drivers/crypto/qat/qat_sym_pmd.c    | 14 +++++-
>  drivers/crypto/qat/qat_sym_pmd.h    |  4 +-
>  13 files changed, 190 insertions(+), 22 deletions(-)
> 
> diff --git a/doc/guides/cryptodevs/qat.rst b/doc/guides/cryptodevs/qat.rst
> index 5135703..aa3cdcf 100644
> --- a/doc/guides/cryptodevs/qat.rst
> +++ b/doc/guides/cryptodevs/qat.rst
> @@ -273,6 +273,24 @@ allocated while for GEN1 devices, 12 buffers are allocated, plus 1472 bytes over
>  	larger than the input size).
>  
>  
> +Running QAT PMD with minimum threshold for burst size
> +~~~~~~~~~~~~~~~~~~~~~~~~
> +
> +If only a small number or packets can be enqueued. Each enqueue causes an expensive MMIO write.
> +These MMIO write occurrences can be optimised by setting any of the following parameters
> +- qat_sym_enq_threshold
> +- qat_asym_enq_threshold
> +- qat_comp_enq_threshold
> +When any of these parameters is set rte_cryptodev_enqueue_burst function will
> +return 0 (thereby avoiding an MMIO) if the device is congested and number of packets
> +possible to enqueue is smaller.
> +To use this feature the user must set the parameter on process start as a device additional parameter:
> + .example: '-w 03:01.1,qat_sym_enq_threshold=32,qat_comp_enq_threshold=16'
> +All parameters can be used with the same device regardless of order. Parameters are separated
> +by comma. When the same parameter is used more than once first occurrence of the parameter
> +is used.
> +Maximum threshold that can be set is 32.
> +
>  Device and driver naming
>  ~~~~~~~~~~~~~~~~~~~~~~~

Normally I wouldn't accept a patch that adds new command line features,
but given it's restricted to a single PMD _and_ the feature is present
in the first short term release after 19.11 (20.02) thus allowing non-
breaking upgrades, I'll merge the series.

-- 
Kind regards,
Luca Boccassi

^ permalink raw reply	[flat|nested] 7+ messages in thread

* Re: [dpdk-stable] [19.11 4/4] crypto/qat: add minimum enq threshold
  2020-08-07 14:53   ` Luca Boccassi
@ 2020-08-07 15:24     ` Trahe, Fiona
  2020-08-30 11:47       ` Ali Alnubani
  0 siblings, 1 reply; 7+ messages in thread
From: Trahe, Fiona @ 2020-08-07 15:24 UTC (permalink / raw)
  To: Luca Boccassi, Kusztal, ArkadiuszX; +Cc: stable

Thanks Luca

> -----Original Message-----
> From: Luca Boccassi <bluca@debian.org>
> Sent: Friday, August 7, 2020 3:53 PM
> To: Kusztal, ArkadiuszX <arkadiuszx.kusztal@intel.com>; Trahe, Fiona <fiona.trahe@intel.com>
> Cc: stable@dpdk.org
> Subject: Re: [dpdk-stable] [19.11 4/4] crypto/qat: add minimum enq threshold
> 
> On Fri, 2020-08-07 at 14:57 +0200, Arek Kusztal wrote:
> > [ upstream commit 47c3f7a41a26c9943f9804691d03828b2ce09f40 ]
> >
> > This patch adds minimum enqueue threshold to Intel
> > QuickAssist Technology PMD.
> > It is an optimisation, configured by a command line option,
> > which can be used to reduce MMIO write occurrences.
> >
> > Cc: stable@dpdk.org
> >
> > Signed-off-by: Fiona Trahe <fiona.trahe@intel.com>
> > Signed-off-by: Arek Kusztal <arkadiuszx.kusztal@intel.com>
> > ---
> >  doc/guides/cryptodevs/qat.rst       | 18 +++++++
> >  drivers/common/qat/qat_common.c     |  3 ++
> >  drivers/common/qat/qat_common.h     |  3 ++
> >  drivers/common/qat/qat_device.c     | 99 ++++++++++++++++++++++++++++++++-----
> >  drivers/common/qat/qat_device.h     | 22 +++++++--
> >  drivers/common/qat/qat_qp.c         | 10 ++++
> >  drivers/common/qat/qat_qp.h         |  3 ++
> >  drivers/compress/qat/qat_comp_pmd.c | 14 +++++-
> >  drivers/compress/qat/qat_comp_pmd.h |  4 +-
> >  drivers/crypto/qat/qat_asym_pmd.c   | 14 +++++-
> >  drivers/crypto/qat/qat_asym_pmd.h   |  4 +-
> >  drivers/crypto/qat/qat_sym_pmd.c    | 14 +++++-
> >  drivers/crypto/qat/qat_sym_pmd.h    |  4 +-
> >  13 files changed, 190 insertions(+), 22 deletions(-)
> >
> > diff --git a/doc/guides/cryptodevs/qat.rst b/doc/guides/cryptodevs/qat.rst
> > index 5135703..aa3cdcf 100644
> > --- a/doc/guides/cryptodevs/qat.rst
> > +++ b/doc/guides/cryptodevs/qat.rst
> > @@ -273,6 +273,24 @@ allocated while for GEN1 devices, 12 buffers are allocated, plus 1472 bytes
> over
> >  	larger than the input size).
> >
> >
> > +Running QAT PMD with minimum threshold for burst size
> > +~~~~~~~~~~~~~~~~~~~~~~~~
> > +
> > +If only a small number or packets can be enqueued. Each enqueue causes an expensive MMIO write.
> > +These MMIO write occurrences can be optimised by setting any of the following parameters
> > +- qat_sym_enq_threshold
> > +- qat_asym_enq_threshold
> > +- qat_comp_enq_threshold
> > +When any of these parameters is set rte_cryptodev_enqueue_burst function will
> > +return 0 (thereby avoiding an MMIO) if the device is congested and number of packets
> > +possible to enqueue is smaller.
> > +To use this feature the user must set the parameter on process start as a device additional parameter:
> > + .example: '-w 03:01.1,qat_sym_enq_threshold=32,qat_comp_enq_threshold=16'
> > +All parameters can be used with the same device regardless of order. Parameters are separated
> > +by comma. When the same parameter is used more than once first occurrence of the parameter
> > +is used.
> > +Maximum threshold that can be set is 32.
> > +
> >  Device and driver naming
> >  ~~~~~~~~~~~~~~~~~~~~~~~
> 
> Normally I wouldn't accept a patch that adds new command line features,
> but given it's restricted to a single PMD _and_ the feature is present
> in the first short term release after 19.11 (20.02) thus allowing non-
> breaking upgrades, I'll merge the series.
> 
> --
> Kind regards,
> Luca Boccassi

^ permalink raw reply	[flat|nested] 7+ messages in thread

* Re: [dpdk-stable] [19.11 4/4] crypto/qat: add minimum enq threshold
  2020-08-07 15:24     ` Trahe, Fiona
@ 2020-08-30 11:47       ` Ali Alnubani
  0 siblings, 0 replies; 7+ messages in thread
From: Ali Alnubani @ 2020-08-30 11:47 UTC (permalink / raw)
  To: Trahe, Fiona, Luca Boccassi, Kusztal, ArkadiuszX; +Cc: stable

Hi,

> -----Original Message-----
> From: stable <stable-bounces@dpdk.org> On Behalf Of Trahe, Fiona
> Sent: Friday, August 7, 2020 6:24 PM
> To: Luca Boccassi <bluca@debian.org>; Kusztal, ArkadiuszX
> <arkadiuszx.kusztal@intel.com>
> Cc: stable@dpdk.org
> Subject: Re: [dpdk-stable] [19.11 4/4] crypto/qat: add minimum enq
> threshold
> 
> Thanks Luca
> 
> > -----Original Message-----
> > From: Luca Boccassi <bluca@debian.org>
> > Sent: Friday, August 7, 2020 3:53 PM
> > To: Kusztal, ArkadiuszX <arkadiuszx.kusztal@intel.com>; Trahe, Fiona
> > <fiona.trahe@intel.com>
> > Cc: stable@dpdk.org
> > Subject: Re: [dpdk-stable] [19.11 4/4] crypto/qat: add minimum enq
> > threshold
> >
> > On Fri, 2020-08-07 at 14:57 +0200, Arek Kusztal wrote:
> > > [ upstream commit 47c3f7a41a26c9943f9804691d03828b2ce09f40 ]
> > >
> > > This patch adds minimum enqueue threshold to Intel QuickAssist
> > > Technology PMD.
> > > It is an optimisation, configured by a command line option, which
> > > can be used to reduce MMIO write occurrences.
> > >
> > > Cc: stable@dpdk.org
> > >
> > > Signed-off-by: Fiona Trahe <fiona.trahe@intel.com>
> > > Signed-off-by: Arek Kusztal <arkadiuszx.kusztal@intel.com>
> > > ---
> > >  doc/guides/cryptodevs/qat.rst       | 18 +++++++
> > >  drivers/common/qat/qat_common.c     |  3 ++
> > >  drivers/common/qat/qat_common.h     |  3 ++
> > >  drivers/common/qat/qat_device.c     | 99
> ++++++++++++++++++++++++++++++++-----
> > >  drivers/common/qat/qat_device.h     | 22 +++++++--
> > >  drivers/common/qat/qat_qp.c         | 10 ++++
> > >  drivers/common/qat/qat_qp.h         |  3 ++
> > >  drivers/compress/qat/qat_comp_pmd.c | 14 +++++-
> > > drivers/compress/qat/qat_comp_pmd.h |  4 +-
> > >  drivers/crypto/qat/qat_asym_pmd.c   | 14 +++++-
> > >  drivers/crypto/qat/qat_asym_pmd.h   |  4 +-
> > >  drivers/crypto/qat/qat_sym_pmd.c    | 14 +++++-
> > >  drivers/crypto/qat/qat_sym_pmd.h    |  4 +-
> > >  13 files changed, 190 insertions(+), 22 deletions(-)
> > >
> > > diff --git a/doc/guides/cryptodevs/qat.rst
> > > b/doc/guides/cryptodevs/qat.rst index 5135703..aa3cdcf 100644
> > > --- a/doc/guides/cryptodevs/qat.rst
> > > +++ b/doc/guides/cryptodevs/qat.rst
> > > @@ -273,6 +273,24 @@ allocated while for GEN1 devices, 12 buffers
> > > are allocated, plus 1472 bytes
> > over
> > >  	larger than the input size).
> > >
> > >
> > > +Running QAT PMD with minimum threshold for burst size
> > > +~~~~~~~~~~~~~~~~~~~~~~~~
> > > +
> > > +If only a small number or packets can be enqueued. Each enqueue
> causes an expensive MMIO write.
> > > +These MMIO write occurrences can be optimised by setting any of the
> > > +following parameters
> > > +- qat_sym_enq_threshold
> > > +- qat_asym_enq_threshold
> > > +- qat_comp_enq_threshold
> > > +When any of these parameters is set rte_cryptodev_enqueue_burst
> > > +function will return 0 (thereby avoiding an MMIO) if the device is
> > > +congested and number of packets possible to enqueue is smaller.
> > > +To use this feature the user must set the parameter on process start as
> a device additional parameter:
> > > + .example: '-w
> 03:01.1,qat_sym_enq_threshold=32,qat_comp_enq_threshold=16'
> > > +All parameters can be used with the same device regardless of
> > > +order. Parameters are separated by comma. When the same parameter
> > > +is used more than once first occurrence of the parameter is used.
> > > +Maximum threshold that can be set is 32.
> > > +
> > >  Device and driver naming
> > >  ~~~~~~~~~~~~~~~~~~~~~~~
> >
> > Normally I wouldn't accept a patch that adds new command line
> > features, but given it's restricted to a single PMD _and_ the feature
> > is present in the first short term release after 19.11 (20.02) thus
> > allowing non- breaking upgrades, I'll merge the series.
> >
> > --
> > Kind regards,
> > Luca Boccassi

This change is causing a warning when building the html guides:

"""
$ make doc-guides-html > /dev/null

/tmp/dpdk-stable/doc/guides/cryptodevs/qat.rst:277: WARNING: Title underline too short.

Running QAT PMD with minimum threshold for burst size
~~~~~~~~~~~~~~~~~~~~~~~~
/tmp/dpdk-stable/doc/guides/cryptodevs/qat.rst:277: WARNING: Title underline too short.

Running QAT PMD with minimum threshold for burst size
~~~~~~~~~~~~~~~~~~~~~~~~
/tmp/dpdk-stable/doc/guides/cryptodevs/qat.rst:288: WARNING: Unexpected indentation.
/tmp/dpdk-stable/doc/guides/cryptodevs/qat.rst:289: WARNING: Block quote ends without a blank line; unexpected unindent.
"""

Reproduces on both Ubuntu 18.04 LTS with sphinx-build 1.6.7, and Ubuntu 20.04 with sphinx-build 1.8.5.

Regards,
Ali

^ permalink raw reply	[flat|nested] 7+ messages in thread

end of thread, other threads:[~2020-08-30 14:36 UTC | newest]

Thread overview: 7+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2020-08-07 12:56 [dpdk-stable] [19.11 1/4] common/qat: remove tail write coalescing Arek Kusztal
2020-08-07 12:56 ` [dpdk-stable] [19.11 2/4] common/qat: move max inflights param into qp Arek Kusztal
2020-08-07 12:57 ` [dpdk-stable] [19.11 3/4] common/qat: support dual threads for enqueue/dequeue Arek Kusztal
2020-08-07 12:57 ` [dpdk-stable] [19.11 4/4] crypto/qat: add minimum enq threshold Arek Kusztal
2020-08-07 14:53   ` Luca Boccassi
2020-08-07 15:24     ` Trahe, Fiona
2020-08-30 11:47       ` Ali Alnubani

patches for DPDK stable branches

This inbox may be cloned and mirrored by anyone:

	git clone --mirror https://inbox.dpdk.org/stable/0 stable/git/0.git

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V2 stable stable/ https://inbox.dpdk.org/stable \
		stable@dpdk.org
	public-inbox-index stable

Example config snippet for mirrors.
Newsgroup available over NNTP:
	nntp://inbox.dpdk.org/inbox.dpdk.stable


AGPL code for this site: git clone https://public-inbox.org/public-inbox.git