From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id B1041A0543 for ; Mon, 6 Jun 2022 18:06:38 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id A77134161A; Mon, 6 Jun 2022 18:06:38 +0200 (CEST) Received: from mga02.intel.com (mga02.intel.com [134.134.136.20]) by mails.dpdk.org (Postfix) with ESMTP id DA9DB4069C; Mon, 6 Jun 2022 18:06:34 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=intel.com; i=@intel.com; q=dns/txt; s=Intel; t=1654531596; x=1686067596; h=from:to:cc:subject:date:message-id:mime-version: content-transfer-encoding; bh=jxtUyJ0ihr9sNwkmBx4carGCAJe5nW5dhaNutIX3wEY=; b=cA92kls+YuOJU7+dbSKLDn/AijvTTc3fzJ1ZxsR2HLLgeObs4e39aL1Q plbtY4exFgHXjJcRaZ0Y6NniMSjjqlE3PXGZLvosCeFgixMKXF4KmTINq Do3qTMia6yiVIvyQbkFgFn6w4CHKPbk3Z+uu7I2RKJcp/GKuRYEAga+wW xJ0pZN3IkyZ39iBMCBTqpanr6torHYvulgi1l9No/xvXgnsRUY3TCIk0c qBJ8KwJCG12kmZLqjeaCVEdr/h1UsjNI2WNlcz4d77UVx3Al8/JgriXLP nIQEN/HpVGDMeVBobOum0BFPwqMJPq68X4d+QYjKggzM1MmKrTbalLRq8 g==; X-IronPort-AV: E=McAfee;i="6400,9594,10370"; a="264665020" X-IronPort-AV: E=Sophos;i="5.91,280,1647327600"; d="scan'208";a="264665020" Received: from fmsmga003.fm.intel.com ([10.253.24.29]) by orsmga101.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 06 Jun 2022 08:55:20 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.91,280,1647327600"; d="scan'208";a="669536985" Received: from txanpdk03.an.intel.com ([10.123.117.78]) by FMSMGA003.fm.intel.com with ESMTP; 06 Jun 2022 08:55:19 -0700 From: Timothy McDaniel To: jerinj@marvell.com Cc: dev@dpdk.org, stable@dpdk.org Subject: [PATCH] event/dlb2: rework queue drain handling Date: Mon, 6 Jun 2022 10:55:17 -0500 Message-Id: <20220606155517.1226558-1-timothy.mcdaniel@intel.com> X-Mailer: git-send-email 2.23.0 MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-BeenThere: stable@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: patches for DPDK stable branches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: stable-bounces@dpdk.org Previously, anything that prevented the CQs from being drained would hang the PMD for an unacceptably long time. This commit updates the drain logic to be resource and time based, thus eliminating the potential for a long hang when draining the queues in preparation for a reset. Fixes: 1857f1922ce2 ("event/dlb2: use new implementation of resource file") Cc: stable@dpdk.org Signed-off-by: Timothy McDaniel --- drivers/event/dlb2/pf/base/dlb2_hw_types.h | 2 +- drivers/event/dlb2/pf/base/dlb2_resource.c | 45 +++++++++++++++++----- 2 files changed, 37 insertions(+), 10 deletions(-) diff --git a/drivers/event/dlb2/pf/base/dlb2_hw_types.h b/drivers/event/dlb2/pf/base/dlb2_hw_types.h index 6b8fee3416..9511521e67 100644 --- a/drivers/event/dlb2/pf/base/dlb2_hw_types.h +++ b/drivers/event/dlb2/pf/base/dlb2_hw_types.h @@ -27,7 +27,7 @@ #define DLB2_MAX_NUM_SEQUENCE_NUMBER_GROUPS 2 #define DLB2_MAX_NUM_SEQUENCE_NUMBER_MODES 5 #define DLB2_MAX_CQ_COMP_CHECK_LOOPS 409600 -#define DLB2_MAX_QID_EMPTY_CHECK_LOOPS (32 * 64 * 1024 * (800 / 30)) +#define DLB2_MAX_QID_EMPTY_CHECK_LOOPS (4 * DLB2_MAX_NUM_LDB_CREDITS) #define DLB2_FUNC_BAR 0 #define DLB2_CSR_BAR 2 diff --git a/drivers/event/dlb2/pf/base/dlb2_resource.c b/drivers/event/dlb2/pf/base/dlb2_resource.c index 548bebd068..4011c24aef 100644 --- a/drivers/event/dlb2/pf/base/dlb2_resource.c +++ b/drivers/event/dlb2/pf/base/dlb2_resource.c @@ -1057,7 +1057,7 @@ static u32 dlb2_dir_cq_token_count(struct dlb2_hw *hw, port->init_tkn_cnt; } -static void dlb2_drain_dir_cq(struct dlb2_hw *hw, +static int dlb2_drain_dir_cq(struct dlb2_hw *hw, struct dlb2_dir_pq_pair *port) { unsigned int port_id = port->id.phys_id; @@ -1089,6 +1089,8 @@ static void dlb2_drain_dir_cq(struct dlb2_hw *hw, os_unmap_producer_port(hw, pp_addr); } + + return cnt; } static void dlb2_dir_port_cq_enable(struct dlb2_hw *hw, @@ -1107,6 +1109,7 @@ static int dlb2_domain_drain_dir_cqs(struct dlb2_hw *hw, { struct dlb2_list_entry *iter; struct dlb2_dir_pq_pair *port; + int drain_cnt = 0; RTE_SET_USED(iter); DLB2_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter) { @@ -1120,13 +1123,13 @@ static int dlb2_domain_drain_dir_cqs(struct dlb2_hw *hw, if (toggle_port) dlb2_dir_port_cq_disable(hw, port); - dlb2_drain_dir_cq(hw, port); + drain_cnt = dlb2_drain_dir_cq(hw, port); if (toggle_port) dlb2_dir_port_cq_enable(hw, port); } - return 0; + return drain_cnt; } static u32 dlb2_dir_queue_depth(struct dlb2_hw *hw, @@ -1170,10 +1173,20 @@ static int dlb2_domain_drain_dir_queues(struct dlb2_hw *hw, return 0; for (i = 0; i < DLB2_MAX_QID_EMPTY_CHECK_LOOPS; i++) { - dlb2_domain_drain_dir_cqs(hw, domain, true); + int drain_cnt; + + drain_cnt = dlb2_domain_drain_dir_cqs(hw, domain, false); if (dlb2_domain_dir_queues_empty(hw, domain)) break; + + /* + * Allow time for DLB to schedule QEs before draining + * the CQs again. + */ + if (!drain_cnt) + rte_delay_us(1); + } if (i == DLB2_MAX_QID_EMPTY_CHECK_LOOPS) { @@ -1249,7 +1262,7 @@ static u32 dlb2_ldb_cq_token_count(struct dlb2_hw *hw, port->init_tkn_cnt; } -static void dlb2_drain_ldb_cq(struct dlb2_hw *hw, struct dlb2_ldb_port *port) +static int dlb2_drain_ldb_cq(struct dlb2_hw *hw, struct dlb2_ldb_port *port) { u32 infl_cnt, tkn_cnt; unsigned int i; @@ -1289,32 +1302,37 @@ static void dlb2_drain_ldb_cq(struct dlb2_hw *hw, struct dlb2_ldb_port *port) os_unmap_producer_port(hw, pp_addr); } + + return tkn_cnt; } -static void dlb2_domain_drain_ldb_cqs(struct dlb2_hw *hw, +static int dlb2_domain_drain_ldb_cqs(struct dlb2_hw *hw, struct dlb2_hw_domain *domain, bool toggle_port) { struct dlb2_list_entry *iter; struct dlb2_ldb_port *port; + int drain_cnt = 0; int i; RTE_SET_USED(iter); /* If the domain hasn't been started, there's no traffic to drain */ if (!domain->started) - return; + return 0; for (i = 0; i < DLB2_NUM_COS_DOMAINS; i++) { DLB2_DOM_LIST_FOR(domain->used_ldb_ports[i], port, iter) { if (toggle_port) dlb2_ldb_port_cq_disable(hw, port); - dlb2_drain_ldb_cq(hw, port); + drain_cnt = dlb2_drain_ldb_cq(hw, port); if (toggle_port) dlb2_ldb_port_cq_enable(hw, port); } } + + return drain_cnt; } static u32 dlb2_ldb_queue_depth(struct dlb2_hw *hw, @@ -1375,10 +1393,19 @@ static int dlb2_domain_drain_mapped_queues(struct dlb2_hw *hw, } for (i = 0; i < DLB2_MAX_QID_EMPTY_CHECK_LOOPS; i++) { - dlb2_domain_drain_ldb_cqs(hw, domain, true); + int drain_cnt; + + drain_cnt = dlb2_domain_drain_ldb_cqs(hw, domain, false); if (dlb2_domain_mapped_queues_empty(hw, domain)) break; + + /* + * Allow time for DLB to schedule QEs before draining + * the CQs again. + */ + if (!drain_cnt) + rte_delay_us(1); } if (i == DLB2_MAX_QID_EMPTY_CHECK_LOOPS) { -- 2.25.1