Re: [PATCH 21.11] net/ionic: add watchdogs to protect each queue type

["Boyer, Andrew" <Andrew.Boyer@amd.com>]

This doesn’t appear to have gone anywhere. Do I need to resend it?

-Andrew

> On Oct 6, 2022, at 9:56 AM, Boyer, Andrew <Andrew.Boyer@amd.com> wrote:
> 
> Ring the doorbell again for the following scenarios:
> * No receives posted but Rx queue not empty after deadline
> * No transmits posted but Tx work still pending after deadline
> * Admin queue work still pending after deadline
> 
> This will help the queues recover in the extremely rare case that
> a doorbell is missed by the FW.
> 
> Signed-off-by: Andrew Boyer <andrew.boyer@amd.com>
> Signed-off-by: R Mohamed Shah <mohamedshah.r@amd.com>
> ---
> drivers/net/ionic/ionic_dev.h  |  4 +++
> drivers/net/ionic/ionic_lif.h  |  3 ++
> drivers/net/ionic/ionic_main.c | 22 +++++++++++++++
> drivers/net/ionic/ionic_rxtx.c | 50 ++++++++++++++++++++++++++++++++++
> 4 files changed, 79 insertions(+)
> 
> diff --git a/drivers/net/ionic/ionic_dev.h b/drivers/net/ionic/ionic_dev.h
> index 38c078efdf..30a8ebe1aa 100644
> --- a/drivers/net/ionic/ionic_dev.h
> +++ b/drivers/net/ionic/ionic_dev.h
> @@ -23,6 +23,10 @@
> #define IONIC_DEVCMD_TIMEOUT		5	/* devcmd_timeout */
> #define IONIC_DEVCMD_CHECK_PERIOD_US	10	/* devcmd status chk period */
> 
> +#define IONIC_Q_WDOG_MS			10	/* 10ms */
> +#define IONIC_Q_WDOG_MAX_MS		5000	/* 5s */
> +#define IONIC_ADMINQ_WDOG_MS		500	/* 500ms */
> +
> #define	IONIC_ALIGN             4096
> 
> struct ionic_adapter;
> diff --git a/drivers/net/ionic/ionic_lif.h b/drivers/net/ionic/ionic_lif.h
> index 9f00ba2973..eac6a58aaa 100644
> --- a/drivers/net/ionic/ionic_lif.h
> +++ b/drivers/net/ionic/ionic_lif.h
> @@ -81,6 +81,8 @@ struct ionic_rx_qcq {
> 
> 	/* cacheline2 */
> 	struct rte_mempool *mb_pool;
> +	uint64_t last_wdog_cycles;
> +	uint64_t wdog_ms;
> 	uint16_t flags;
> 
> 	/* cacheline3 (inside stats) */
> @@ -92,6 +94,7 @@ struct ionic_tx_qcq {
> 	struct ionic_qcq qcq;
> 
> 	/* cacheline2 */
> +	uint64_t last_wdog_cycles;
> 	uint16_t num_segs_fw;	/* # segs supported by current FW */
> 	uint16_t flags;
> 
> diff --git a/drivers/net/ionic/ionic_main.c b/drivers/net/ionic/ionic_main.c
> index 7301f53342..3d1915daa7 100644
> --- a/drivers/net/ionic/ionic_main.c
> +++ b/drivers/net/ionic/ionic_main.c
> @@ -230,10 +230,16 @@ static int
> ionic_adminq_wait_for_completion(struct ionic_lif *lif,
> 		struct ionic_admin_ctx *ctx, unsigned long max_wait)
> {
> +	struct ionic_queue *q = &lif->adminqcq->qcq.q;
> 	unsigned long step_usec = IONIC_DEVCMD_CHECK_PERIOD_US;
> +	unsigned long step_deadline;
> 	unsigned long max_wait_usec = max_wait * 1000000L;
> 	unsigned long elapsed_usec = 0;
> 	int budget = 8;
> +	uint16_t idx;
> +	void **info;
> +
> +	step_deadline = IONIC_ADMINQ_WDOG_MS * 1000 / step_usec;
> 
> 	while (ctx->pending_work && elapsed_usec < max_wait_usec) {
> 		/*
> @@ -245,10 +251,26 @@ ionic_adminq_wait_for_completion(struct ionic_lif *lif,
> 		ionic_qcq_service(&lif->adminqcq->qcq, budget,
> 				ionic_adminq_service, NULL);
> 
> +		/*
> +		 * Ring the doorbell again if work is pending after deadline.
> +		 */
> +		if (ctx->pending_work && !step_deadline) {
> +			step_deadline = IONIC_ADMINQ_WDOG_MS *
> +				1000 / step_usec;
> +
> +			rte_spinlock_lock(&lif->adminq_lock);
> +			idx = Q_NEXT_TO_POST(q, -1);
> +			info = IONIC_INFO_PTR(q, idx);
> +			if (info[0] == ctx)
> +				ionic_q_flush(q);
> +			rte_spinlock_unlock(&lif->adminq_lock);
> +		}
> +
> 		rte_spinlock_unlock(&lif->adminq_service_lock);
> 
> 		rte_delay_us_block(step_usec);
> 		elapsed_usec += step_usec;
> +		step_deadline--;
> 	}
> 
> 	return (!ctx->pending_work);
> diff --git a/drivers/net/ionic/ionic_rxtx.c b/drivers/net/ionic/ionic_rxtx.c
> index 9f602de6a9..029b827e59 100644
> --- a/drivers/net/ionic/ionic_rxtx.c
> +++ b/drivers/net/ionic/ionic_rxtx.c
> @@ -536,6 +536,7 @@ ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
> 	uint32_t next_q_head_idx;
> 	uint32_t bytes_tx = 0;
> 	uint16_t nb_avail, nb_tx = 0;
> +	uint64_t then, now, hz, delta;
> 	int err;
> 
> 	/* Cleaning old buffers */
> @@ -571,6 +572,24 @@ ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
> 	if (nb_tx > 0) {
> 		rte_wmb();
> 		ionic_q_flush(q);
> +
> +		txq->last_wdog_cycles = rte_get_timer_cycles();
> +	} else {
> +		/*
> +		 * Ring the doorbell again if no work could be posted and work
> +		 * is still pending after the deadline.
> +		 */
> +		if (q->head_idx != q->tail_idx) {
> +			then = txq->last_wdog_cycles;
> +			now = rte_get_timer_cycles();
> +			hz = rte_get_timer_hz();
> +			delta = (now - then) * 1000;
> +
> +			if (delta >= hz * IONIC_Q_WDOG_MS) {
> +				ionic_q_flush(q);
> +				txq->last_wdog_cycles = now;
> +			}
> +		}
> 	}
> 
> 	stats->packets += nb_tx;
> @@ -1059,6 +1078,7 @@ ionic_rxq_service(struct ionic_rx_qcq *rxq, uint32_t work_to_do,
> 	bool more;
> 	uint32_t curr_q_tail_idx, curr_cq_tail_idx;
> 	uint32_t work_done = 0;
> +	uint64_t then, now, hz, delta;
> 
> 	if (work_to_do == 0)
> 		return;
> @@ -1096,6 +1116,36 @@ ionic_rxq_service(struct ionic_rx_qcq *rxq, uint32_t work_to_do,
> 
> 		cq_desc = &cq_desc_base[cq->tail_idx];
> 	}
> +
> +	if (work_done) {
> +		rxq->last_wdog_cycles = rte_get_timer_cycles();
> +		rxq->wdog_ms = IONIC_Q_WDOG_MS;
> +	} else {
> +		/*
> +		 * Ring the doorbell again if no recvs were posted and the
> +		 * recv queue is not empty after the deadline.
> +		 *
> +		 * Exponentially back off the deadline to avoid excessive
> +		 * doorbells when the recv queue is idle.
> +		 */
> +		if (q->head_idx != q->tail_idx) {
> +			then = rxq->last_wdog_cycles;
> +			now = rte_get_timer_cycles();
> +			hz = rte_get_timer_hz();
> +			delta = (now - then) * 1000;
> +
> +			if (delta >= hz * rxq->wdog_ms) {
> +				ionic_q_flush(q);
> +				rxq->last_wdog_cycles = now;
> +
> +				delta = 2 * rxq->wdog_ms;
> +				if (delta > IONIC_Q_WDOG_MAX_MS)
> +					delta = IONIC_Q_WDOG_MAX_MS;
> +
> +				rxq->wdog_ms = delta;
> +                       }
> +               }
> +	}
> }
> 
> /*
> -- 
> 2.17.1
>