* [PATCH 21.11] net/ionic: add watchdogs to protect each queue type
@ 2022-10-06 13:56 Andrew Boyer
2023-02-15 22:36 ` Boyer, Andrew
0 siblings, 1 reply; 2+ messages in thread
From: Andrew Boyer @ 2022-10-06 13:56 UTC (permalink / raw)
To: stable; +Cc: Andrew Boyer, R Mohamed Shah
Ring the doorbell again for the following scenarios:
* No receives posted but Rx queue not empty after deadline
* No transmits posted but Tx work still pending after deadline
* Admin queue work still pending after deadline
This will help the queues recover in the extremely rare case that
a doorbell is missed by the FW.
Signed-off-by: Andrew Boyer <andrew.boyer@amd.com>
Signed-off-by: R Mohamed Shah <mohamedshah.r@amd.com>
---
drivers/net/ionic/ionic_dev.h | 4 +++
drivers/net/ionic/ionic_lif.h | 3 ++
drivers/net/ionic/ionic_main.c | 22 +++++++++++++++
drivers/net/ionic/ionic_rxtx.c | 50 ++++++++++++++++++++++++++++++++++
4 files changed, 79 insertions(+)
diff --git a/drivers/net/ionic/ionic_dev.h b/drivers/net/ionic/ionic_dev.h
index 38c078efdf..30a8ebe1aa 100644
--- a/drivers/net/ionic/ionic_dev.h
+++ b/drivers/net/ionic/ionic_dev.h
@@ -23,6 +23,10 @@
#define IONIC_DEVCMD_TIMEOUT 5 /* devcmd_timeout */
#define IONIC_DEVCMD_CHECK_PERIOD_US 10 /* devcmd status chk period */
+#define IONIC_Q_WDOG_MS 10 /* 10ms */
+#define IONIC_Q_WDOG_MAX_MS 5000 /* 5s */
+#define IONIC_ADMINQ_WDOG_MS 500 /* 500ms */
+
#define IONIC_ALIGN 4096
struct ionic_adapter;
diff --git a/drivers/net/ionic/ionic_lif.h b/drivers/net/ionic/ionic_lif.h
index 9f00ba2973..eac6a58aaa 100644
--- a/drivers/net/ionic/ionic_lif.h
+++ b/drivers/net/ionic/ionic_lif.h
@@ -81,6 +81,8 @@ struct ionic_rx_qcq {
/* cacheline2 */
struct rte_mempool *mb_pool;
+ uint64_t last_wdog_cycles;
+ uint64_t wdog_ms;
uint16_t flags;
/* cacheline3 (inside stats) */
@@ -92,6 +94,7 @@ struct ionic_tx_qcq {
struct ionic_qcq qcq;
/* cacheline2 */
+ uint64_t last_wdog_cycles;
uint16_t num_segs_fw; /* # segs supported by current FW */
uint16_t flags;
diff --git a/drivers/net/ionic/ionic_main.c b/drivers/net/ionic/ionic_main.c
index 7301f53342..3d1915daa7 100644
--- a/drivers/net/ionic/ionic_main.c
+++ b/drivers/net/ionic/ionic_main.c
@@ -230,10 +230,16 @@ static int
ionic_adminq_wait_for_completion(struct ionic_lif *lif,
struct ionic_admin_ctx *ctx, unsigned long max_wait)
{
+ struct ionic_queue *q = &lif->adminqcq->qcq.q;
unsigned long step_usec = IONIC_DEVCMD_CHECK_PERIOD_US;
+ unsigned long step_deadline;
unsigned long max_wait_usec = max_wait * 1000000L;
unsigned long elapsed_usec = 0;
int budget = 8;
+ uint16_t idx;
+ void **info;
+
+ step_deadline = IONIC_ADMINQ_WDOG_MS * 1000 / step_usec;
while (ctx->pending_work && elapsed_usec < max_wait_usec) {
/*
@@ -245,10 +251,26 @@ ionic_adminq_wait_for_completion(struct ionic_lif *lif,
ionic_qcq_service(&lif->adminqcq->qcq, budget,
ionic_adminq_service, NULL);
+ /*
+ * Ring the doorbell again if work is pending after deadline.
+ */
+ if (ctx->pending_work && !step_deadline) {
+ step_deadline = IONIC_ADMINQ_WDOG_MS *
+ 1000 / step_usec;
+
+ rte_spinlock_lock(&lif->adminq_lock);
+ idx = Q_NEXT_TO_POST(q, -1);
+ info = IONIC_INFO_PTR(q, idx);
+ if (info[0] == ctx)
+ ionic_q_flush(q);
+ rte_spinlock_unlock(&lif->adminq_lock);
+ }
+
rte_spinlock_unlock(&lif->adminq_service_lock);
rte_delay_us_block(step_usec);
elapsed_usec += step_usec;
+ step_deadline--;
}
return (!ctx->pending_work);
diff --git a/drivers/net/ionic/ionic_rxtx.c b/drivers/net/ionic/ionic_rxtx.c
index 9f602de6a9..029b827e59 100644
--- a/drivers/net/ionic/ionic_rxtx.c
+++ b/drivers/net/ionic/ionic_rxtx.c
@@ -536,6 +536,7 @@ ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
uint32_t next_q_head_idx;
uint32_t bytes_tx = 0;
uint16_t nb_avail, nb_tx = 0;
+ uint64_t then, now, hz, delta;
int err;
/* Cleaning old buffers */
@@ -571,6 +572,24 @@ ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
if (nb_tx > 0) {
rte_wmb();
ionic_q_flush(q);
+
+ txq->last_wdog_cycles = rte_get_timer_cycles();
+ } else {
+ /*
+ * Ring the doorbell again if no work could be posted and work
+ * is still pending after the deadline.
+ */
+ if (q->head_idx != q->tail_idx) {
+ then = txq->last_wdog_cycles;
+ now = rte_get_timer_cycles();
+ hz = rte_get_timer_hz();
+ delta = (now - then) * 1000;
+
+ if (delta >= hz * IONIC_Q_WDOG_MS) {
+ ionic_q_flush(q);
+ txq->last_wdog_cycles = now;
+ }
+ }
}
stats->packets += nb_tx;
@@ -1059,6 +1078,7 @@ ionic_rxq_service(struct ionic_rx_qcq *rxq, uint32_t work_to_do,
bool more;
uint32_t curr_q_tail_idx, curr_cq_tail_idx;
uint32_t work_done = 0;
+ uint64_t then, now, hz, delta;
if (work_to_do == 0)
return;
@@ -1096,6 +1116,36 @@ ionic_rxq_service(struct ionic_rx_qcq *rxq, uint32_t work_to_do,
cq_desc = &cq_desc_base[cq->tail_idx];
}
+
+ if (work_done) {
+ rxq->last_wdog_cycles = rte_get_timer_cycles();
+ rxq->wdog_ms = IONIC_Q_WDOG_MS;
+ } else {
+ /*
+ * Ring the doorbell again if no recvs were posted and the
+ * recv queue is not empty after the deadline.
+ *
+ * Exponentially back off the deadline to avoid excessive
+ * doorbells when the recv queue is idle.
+ */
+ if (q->head_idx != q->tail_idx) {
+ then = rxq->last_wdog_cycles;
+ now = rte_get_timer_cycles();
+ hz = rte_get_timer_hz();
+ delta = (now - then) * 1000;
+
+ if (delta >= hz * rxq->wdog_ms) {
+ ionic_q_flush(q);
+ rxq->last_wdog_cycles = now;
+
+ delta = 2 * rxq->wdog_ms;
+ if (delta > IONIC_Q_WDOG_MAX_MS)
+ delta = IONIC_Q_WDOG_MAX_MS;
+
+ rxq->wdog_ms = delta;
+ }
+ }
+ }
}
/*
--
2.17.1
^ permalink raw reply [flat|nested] 2+ messages in thread
* Re: [PATCH 21.11] net/ionic: add watchdogs to protect each queue type
2022-10-06 13:56 [PATCH 21.11] net/ionic: add watchdogs to protect each queue type Andrew Boyer
@ 2023-02-15 22:36 ` Boyer, Andrew
0 siblings, 0 replies; 2+ messages in thread
From: Boyer, Andrew @ 2023-02-15 22:36 UTC (permalink / raw)
To: stable; +Cc: R, Mohamed Shah
This doesn’t appear to have gone anywhere. Do I need to resend it?
-Andrew
> On Oct 6, 2022, at 9:56 AM, Boyer, Andrew <Andrew.Boyer@amd.com> wrote:
>
> Ring the doorbell again for the following scenarios:
> * No receives posted but Rx queue not empty after deadline
> * No transmits posted but Tx work still pending after deadline
> * Admin queue work still pending after deadline
>
> This will help the queues recover in the extremely rare case that
> a doorbell is missed by the FW.
>
> Signed-off-by: Andrew Boyer <andrew.boyer@amd.com>
> Signed-off-by: R Mohamed Shah <mohamedshah.r@amd.com>
> ---
> drivers/net/ionic/ionic_dev.h | 4 +++
> drivers/net/ionic/ionic_lif.h | 3 ++
> drivers/net/ionic/ionic_main.c | 22 +++++++++++++++
> drivers/net/ionic/ionic_rxtx.c | 50 ++++++++++++++++++++++++++++++++++
> 4 files changed, 79 insertions(+)
>
> diff --git a/drivers/net/ionic/ionic_dev.h b/drivers/net/ionic/ionic_dev.h
> index 38c078efdf..30a8ebe1aa 100644
> --- a/drivers/net/ionic/ionic_dev.h
> +++ b/drivers/net/ionic/ionic_dev.h
> @@ -23,6 +23,10 @@
> #define IONIC_DEVCMD_TIMEOUT 5 /* devcmd_timeout */
> #define IONIC_DEVCMD_CHECK_PERIOD_US 10 /* devcmd status chk period */
>
> +#define IONIC_Q_WDOG_MS 10 /* 10ms */
> +#define IONIC_Q_WDOG_MAX_MS 5000 /* 5s */
> +#define IONIC_ADMINQ_WDOG_MS 500 /* 500ms */
> +
> #define IONIC_ALIGN 4096
>
> struct ionic_adapter;
> diff --git a/drivers/net/ionic/ionic_lif.h b/drivers/net/ionic/ionic_lif.h
> index 9f00ba2973..eac6a58aaa 100644
> --- a/drivers/net/ionic/ionic_lif.h
> +++ b/drivers/net/ionic/ionic_lif.h
> @@ -81,6 +81,8 @@ struct ionic_rx_qcq {
>
> /* cacheline2 */
> struct rte_mempool *mb_pool;
> + uint64_t last_wdog_cycles;
> + uint64_t wdog_ms;
> uint16_t flags;
>
> /* cacheline3 (inside stats) */
> @@ -92,6 +94,7 @@ struct ionic_tx_qcq {
> struct ionic_qcq qcq;
>
> /* cacheline2 */
> + uint64_t last_wdog_cycles;
> uint16_t num_segs_fw; /* # segs supported by current FW */
> uint16_t flags;
>
> diff --git a/drivers/net/ionic/ionic_main.c b/drivers/net/ionic/ionic_main.c
> index 7301f53342..3d1915daa7 100644
> --- a/drivers/net/ionic/ionic_main.c
> +++ b/drivers/net/ionic/ionic_main.c
> @@ -230,10 +230,16 @@ static int
> ionic_adminq_wait_for_completion(struct ionic_lif *lif,
> struct ionic_admin_ctx *ctx, unsigned long max_wait)
> {
> + struct ionic_queue *q = &lif->adminqcq->qcq.q;
> unsigned long step_usec = IONIC_DEVCMD_CHECK_PERIOD_US;
> + unsigned long step_deadline;
> unsigned long max_wait_usec = max_wait * 1000000L;
> unsigned long elapsed_usec = 0;
> int budget = 8;
> + uint16_t idx;
> + void **info;
> +
> + step_deadline = IONIC_ADMINQ_WDOG_MS * 1000 / step_usec;
>
> while (ctx->pending_work && elapsed_usec < max_wait_usec) {
> /*
> @@ -245,10 +251,26 @@ ionic_adminq_wait_for_completion(struct ionic_lif *lif,
> ionic_qcq_service(&lif->adminqcq->qcq, budget,
> ionic_adminq_service, NULL);
>
> + /*
> + * Ring the doorbell again if work is pending after deadline.
> + */
> + if (ctx->pending_work && !step_deadline) {
> + step_deadline = IONIC_ADMINQ_WDOG_MS *
> + 1000 / step_usec;
> +
> + rte_spinlock_lock(&lif->adminq_lock);
> + idx = Q_NEXT_TO_POST(q, -1);
> + info = IONIC_INFO_PTR(q, idx);
> + if (info[0] == ctx)
> + ionic_q_flush(q);
> + rte_spinlock_unlock(&lif->adminq_lock);
> + }
> +
> rte_spinlock_unlock(&lif->adminq_service_lock);
>
> rte_delay_us_block(step_usec);
> elapsed_usec += step_usec;
> + step_deadline--;
> }
>
> return (!ctx->pending_work);
> diff --git a/drivers/net/ionic/ionic_rxtx.c b/drivers/net/ionic/ionic_rxtx.c
> index 9f602de6a9..029b827e59 100644
> --- a/drivers/net/ionic/ionic_rxtx.c
> +++ b/drivers/net/ionic/ionic_rxtx.c
> @@ -536,6 +536,7 @@ ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
> uint32_t next_q_head_idx;
> uint32_t bytes_tx = 0;
> uint16_t nb_avail, nb_tx = 0;
> + uint64_t then, now, hz, delta;
> int err;
>
> /* Cleaning old buffers */
> @@ -571,6 +572,24 @@ ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
> if (nb_tx > 0) {
> rte_wmb();
> ionic_q_flush(q);
> +
> + txq->last_wdog_cycles = rte_get_timer_cycles();
> + } else {
> + /*
> + * Ring the doorbell again if no work could be posted and work
> + * is still pending after the deadline.
> + */
> + if (q->head_idx != q->tail_idx) {
> + then = txq->last_wdog_cycles;
> + now = rte_get_timer_cycles();
> + hz = rte_get_timer_hz();
> + delta = (now - then) * 1000;
> +
> + if (delta >= hz * IONIC_Q_WDOG_MS) {
> + ionic_q_flush(q);
> + txq->last_wdog_cycles = now;
> + }
> + }
> }
>
> stats->packets += nb_tx;
> @@ -1059,6 +1078,7 @@ ionic_rxq_service(struct ionic_rx_qcq *rxq, uint32_t work_to_do,
> bool more;
> uint32_t curr_q_tail_idx, curr_cq_tail_idx;
> uint32_t work_done = 0;
> + uint64_t then, now, hz, delta;
>
> if (work_to_do == 0)
> return;
> @@ -1096,6 +1116,36 @@ ionic_rxq_service(struct ionic_rx_qcq *rxq, uint32_t work_to_do,
>
> cq_desc = &cq_desc_base[cq->tail_idx];
> }
> +
> + if (work_done) {
> + rxq->last_wdog_cycles = rte_get_timer_cycles();
> + rxq->wdog_ms = IONIC_Q_WDOG_MS;
> + } else {
> + /*
> + * Ring the doorbell again if no recvs were posted and the
> + * recv queue is not empty after the deadline.
> + *
> + * Exponentially back off the deadline to avoid excessive
> + * doorbells when the recv queue is idle.
> + */
> + if (q->head_idx != q->tail_idx) {
> + then = rxq->last_wdog_cycles;
> + now = rte_get_timer_cycles();
> + hz = rte_get_timer_hz();
> + delta = (now - then) * 1000;
> +
> + if (delta >= hz * rxq->wdog_ms) {
> + ionic_q_flush(q);
> + rxq->last_wdog_cycles = now;
> +
> + delta = 2 * rxq->wdog_ms;
> + if (delta > IONIC_Q_WDOG_MAX_MS)
> + delta = IONIC_Q_WDOG_MAX_MS;
> +
> + rxq->wdog_ms = delta;
> + }
> + }
> + }
> }
>
> /*
> --
> 2.17.1
>
^ permalink raw reply [flat|nested] 2+ messages in thread
end of thread, other threads:[~2023-02-15 22:36 UTC | newest]
Thread overview: 2+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2022-10-06 13:56 [PATCH 21.11] net/ionic: add watchdogs to protect each queue type Andrew Boyer
2023-02-15 22:36 ` Boyer, Andrew
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).