From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id B172642B7D; Tue, 23 May 2023 09:53:15 +0200 (CEST) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 8CF6240EF0; Tue, 23 May 2023 09:53:15 +0200 (CEST) Received: from mga07.intel.com (mga07.intel.com [134.134.136.100]) by mails.dpdk.org (Postfix) with ESMTP id CB49D40A80 for ; Tue, 23 May 2023 09:53:13 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=intel.com; i=@intel.com; q=dns/txt; s=Intel; t=1684828393; x=1716364393; h=from:to:cc:subject:date:message-id:mime-version: content-transfer-encoding; bh=zi3Ei2N4BIrxPDoakAPywJpCjq4hnUB/JKrJAJ0nVeU=; b=WFEqbKEjb/2sInFb7qfHyOp34w+RFTCls8doOivQ/tsox3yWneNqaY38 a2zy2wEGD2nitlrKS5HJS94oPUyu0L3Ul0451chWAqFLu5ljG7jA8pz8X xNYMMCkvfPKK9pwpfpbo5SDyO591a+TnMJhgZnC00l3U3KqMEzyIW2oLG 9exqcD1AU5+wH++VAfxm8jp4EMDhHPQ7Gq7zCwNifY1gGIX2Tbw34duCU BO+bSPU4O7/4vo5ylECLHb9sDBOVcHW59A1LNmgV74SumoLvcGYaV5rpO L0Q15/zBrqfAGwZoZeolOl5XyRJzgdvWIsJYy368JaWOitbw6/piMzRVC A==; X-IronPort-AV: E=McAfee;i="6600,9927,10718"; a="418872921" X-IronPort-AV: E=Sophos;i="6.00,185,1681196400"; d="scan'208";a="418872921" Received: from orsmga005.jf.intel.com ([10.7.209.41]) by orsmga105.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 23 May 2023 00:53:13 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=McAfee;i="6600,9927,10718"; a="878106005" X-IronPort-AV: E=Sophos;i="6.00,185,1681196400"; d="scan'208";a="878106005" Received: from unknown (HELO root..) ([10.239.252.115]) by orsmga005-auth.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 23 May 2023 00:53:10 -0700 From: Shiyang He To: qi.z.zhang@intel.com, dev@dpdk.org Cc: Shiyang He , Jingjing Wu , Beilei Xing Subject: [RFC] net/iavf: handle iavf reset gracefully Date: Tue, 23 May 2023 15:22:36 +0000 Message-Id: <20230523152236.23746-1-shiyangx.he@intel.com> X-Mailer: git-send-email 2.37.2 MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Originally, when vf received PF-to-VF reset event, the iavf PMD did not perform special actions, resulting in vf being offline and unavailable. This commit handle the PF-to-VF reset event by performing all necessary actions to bring the vf back online and available. Signed-off-by: Shiyang He --- drivers/net/iavf/iavf.h | 29 +++++++ drivers/net/iavf/iavf_ethdev.c | 144 +++++++++++++++++++++++++++++++++ drivers/net/iavf/iavf_rxtx.c | 25 ++++++ drivers/net/iavf/iavf_rxtx.h | 1 + drivers/net/iavf/iavf_vchnl.c | 12 +++ 5 files changed, 211 insertions(+) diff --git a/drivers/net/iavf/iavf.h b/drivers/net/iavf/iavf.h index aa18650ffa..79030944f7 100644 --- a/drivers/net/iavf/iavf.h +++ b/drivers/net/iavf/iavf.h @@ -95,6 +95,24 @@ #define IAVF_L2TPV2_FLAGS_LEN 0x4000 +#ifndef DIV_ROUND_UP +#define DIV_ROUND_UP(n, d) ( \ +{ \ + const typeof(d) __d = d; \ + (((n) + (__d) - 1) / (__d));\ +} \ +) +#endif +#ifndef DELAY +#define DELAY(x) rte_delay_us(x) +#endif +#ifndef msleep +#define msleep(x) DELAY(1000 * (x)) +#endif +#ifndef usleep_range +#define usleep_range(min, max) msleep(DIV_ROUND_UP(min, 1000)) +#endif + struct iavf_adapter; struct iavf_rx_queue; struct iavf_tx_queue; @@ -279,6 +297,8 @@ struct iavf_info { uint32_t ptp_caps; rte_spinlock_t phc_time_aq_lock; + + bool auto_reset_enabled; }; #define IAVF_MAX_PKT_TYPE 1024 @@ -423,6 +443,14 @@ _atomic_set_async_response_cmd(struct iavf_info *vf, enum virtchnl_ops ops) return !ret; } + +static inline bool +iavf_is_reset(struct iavf_hw *hw) +{ + return !(IAVF_READ_REG(hw, IAVF_VF_ARQLEN1) & + IAVF_VF_ARQLEN1_ARQENABLE_MASK); +} + int iavf_check_api_version(struct iavf_adapter *adapter); int iavf_get_vf_resource(struct iavf_adapter *adapter); void iavf_dev_event_handler_fini(void); @@ -498,4 +526,5 @@ int iavf_flow_unsub(struct iavf_adapter *adapter, struct iavf_fsub_conf *filter); int iavf_flow_sub_check(struct iavf_adapter *adapter, struct iavf_fsub_conf *filter); +int iavf_handle_hw_reset(struct rte_eth_dev *dev); #endif /* _IAVF_ETHDEV_H_ */ diff --git a/drivers/net/iavf/iavf_ethdev.c b/drivers/net/iavf/iavf_ethdev.c index f6d68403ce..f421febcac 100644 --- a/drivers/net/iavf/iavf_ethdev.c +++ b/drivers/net/iavf/iavf_ethdev.c @@ -337,6 +337,12 @@ iavf_dev_watchdog_disable(struct iavf_adapter *adapter __rte_unused) #endif } +static void +iavf_dev_auto_reset_enable(struct iavf_adapter *adapter) +{ + adapter->vf.auto_reset_enabled = true; +} + static int iavf_set_mc_addr_list(struct rte_eth_dev *dev, struct rte_ether_addr *mc_addrs, @@ -2687,6 +2693,8 @@ iavf_dev_init(struct rte_eth_dev *eth_dev) iavf_dev_watchdog_enable(adapter); adapter->closed = false; + iavf_dev_auto_reset_enable(adapter); + return 0; flow_init_err: @@ -2814,6 +2822,142 @@ iavf_dev_reset(struct rte_eth_dev *dev) return iavf_dev_init(dev); } +static bool +iavf_is_reset_detected(struct iavf_adapter *adapter) +{ + struct iavf_hw *hw = &adapter->hw; + int i; + + /* poll until we see the reset actually happen */ + for (i = 0; i < IAVF_RESET_WAIT_CNT; i++) { + if (iavf_is_reset(hw)) + return true; + usleep_range(5000, 10000); + } + + return false; +} + +static int +iavf_uninit_hw(struct rte_eth_dev *dev, struct iavf_hw *hw) +{ + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = dev->intr_handle; + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); + struct iavf_adapter *adapter = dev->data->dev_private; + + iavf_reset_queues(dev); + + /* Disable the interrupt for Rx */ + rte_intr_efd_disable(intr_handle); + /* Rx interrupt vector mapping free */ + rte_intr_vec_list_free(intr_handle); + + adapter->stopped = 1; + dev->data->dev_started = 0; + + adapter->closed = true; + + /* free iAVF security device context all related resources */ + iavf_security_ctx_destroy(adapter); + + iavf_flow_flush(dev, NULL); + iavf_flow_uninit(adapter); + + if (vf->promisc_unicast_enabled || vf->promisc_multicast_enabled) + iavf_config_promisc(adapter, false, false); + + iavf_shutdown_adminq(hw); + if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_WB_ON_ITR) { + /* disable uio intr before callback unregister */ + rte_intr_disable(intr_handle); + + /* unregister callback func from eal lib */ + rte_intr_callback_unregister(intr_handle, + iavf_dev_interrupt_handler, dev); + } + iavf_disable_irq0(hw); + + if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_QOS) + iavf_tm_conf_uninit(dev); + + if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) { + if (vf->rss_lut) { + rte_free(vf->rss_lut); + vf->rss_lut = NULL; + } + if (vf->rss_key) { + rte_free(vf->rss_key); + vf->rss_key = NULL; + } + } + + rte_free(vf->vf_res); + vf->vsi_res = NULL; + vf->vf_res = NULL; + + rte_free(vf->aq_resp); + vf->aq_resp = NULL; + + if (vf->vf_reset && !rte_pci_set_bus_master(pci_dev, true)) + vf->vf_reset = false; + + /* disable watchdog */ + iavf_dev_watchdog_disable(adapter); + + return 0; +} + +static int +iavf_reset_hw(struct rte_eth_dev *dev, struct iavf_hw *hw) +{ + iavf_uninit_hw(dev, hw); + + return iavf_dev_init(dev); +} + +/* + * Handle hardware reset + */ +int +iavf_handle_hw_reset(struct rte_eth_dev *dev) +{ + struct iavf_adapter *adapter = dev->data->dev_private; + struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); + int ret; + + if (!adapter->vf.auto_reset_enabled) + return 0; + + if (!iavf_is_reset_detected(adapter)) + return 0; + + if (iavf_reset_hw(dev, hw)) + return -1; + + /* configure the device */ + ret = iavf_dev_configure(dev); + if (ret) { + PMD_DRV_LOG(ERR, "Failed to configure dev"); + return ret; + } + + iavf_dev_xstats_reset(dev); + + /* start the device */ + ret = iavf_dev_start(dev); + if (ret) { + PMD_DRV_LOG(ERR, "Failed to start dev"); + return ret; + } + + dev->data->dev_started = 1; + vf->vf_reset = false; + + return 0; +} + static int iavf_dcf_cap_check_handler(__rte_unused const char *key, const char *value, __rte_unused void *opaque) diff --git a/drivers/net/iavf/iavf_rxtx.c b/drivers/net/iavf/iavf_rxtx.c index b1d0fbceb6..cc58564461 100644 --- a/drivers/net/iavf/iavf_rxtx.c +++ b/drivers/net/iavf/iavf_rxtx.c @@ -1093,6 +1093,31 @@ iavf_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid) rte_free(q); } +void +iavf_reset_queues(struct rte_eth_dev *dev) +{ + struct iavf_rx_queue *rxq; + struct iavf_tx_queue *txq; + int i; + + for (i = 0; i < dev->data->nb_tx_queues; i++) { + txq = dev->data->tx_queues[i]; + if (!txq) + continue; + iavf_txq_release_mbufs_ops[txq->rel_mbufs_type].release_mbufs(txq); + reset_tx_queue(txq); + dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; + } + for (i = 0; i < dev->data->nb_rx_queues; i++) { + rxq = dev->data->rx_queues[i]; + if (!rxq) + continue; + iavf_rxq_release_mbufs_ops[rxq->rel_mbufs_type].release_mbufs(rxq); + reset_rx_queue(rxq); + dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED; + } +} + void iavf_stop_queues(struct rte_eth_dev *dev) { diff --git a/drivers/net/iavf/iavf_rxtx.h b/drivers/net/iavf/iavf_rxtx.h index 09e2127db0..622a9dbfef 100644 --- a/drivers/net/iavf/iavf_rxtx.h +++ b/drivers/net/iavf/iavf_rxtx.h @@ -640,6 +640,7 @@ int iavf_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id); int iavf_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id); int iavf_dev_tx_done_cleanup(void *txq, uint32_t free_cnt); void iavf_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid); +void iavf_reset_queues(struct rte_eth_dev *dev); void iavf_stop_queues(struct rte_eth_dev *dev); uint16_t iavf_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts); diff --git a/drivers/net/iavf/iavf_vchnl.c b/drivers/net/iavf/iavf_vchnl.c index ab721e082b..2a36526b10 100644 --- a/drivers/net/iavf/iavf_vchnl.c +++ b/drivers/net/iavf/iavf_vchnl.c @@ -31,6 +31,10 @@ #define MAX_EVENT_PENDING 16 +static void iavf_dev_event_post(struct rte_eth_dev *dev, + enum rte_eth_event_type event, + void *param, size_t param_alloc_size); + struct iavf_event_element { TAILQ_ENTRY(iavf_event_element) next; struct rte_eth_dev *dev; @@ -80,6 +84,14 @@ iavf_dev_event_handle(void *param __rte_unused) TAILQ_FOREACH_SAFE(pos, &pending, next, save_next) { TAILQ_REMOVE(&pending, pos, next); rte_eth_dev_callback_process(pos->dev, pos->event, pos->param); + + switch (pos->event) { + case RTE_ETH_EVENT_INTR_RESET: + iavf_handle_hw_reset(pos->dev); + break; + default: + break; + } rte_free(pos); } } -- 2.37.2