From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 41CF9429DB; Mon, 24 Apr 2023 15:09:25 +0200 (CEST) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 2CE5C42D0B; Mon, 24 Apr 2023 15:09:17 +0200 (CEST) Received: from szxga01-in.huawei.com (szxga01-in.huawei.com [45.249.212.187]) by mails.dpdk.org (Postfix) with ESMTP id A93EA410ED for ; Mon, 24 Apr 2023 15:09:13 +0200 (CEST) Received: from dggpeml500024.china.huawei.com (unknown [172.30.72.53]) by szxga01-in.huawei.com (SkyGuard) with ESMTP id 4Q4lm81PlHzsRTH; Mon, 24 Apr 2023 21:07:36 +0800 (CST) Received: from localhost.localdomain (10.50.163.32) by dggpeml500024.china.huawei.com (7.185.36.10) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256) id 15.1.2507.23; Mon, 24 Apr 2023 21:09:12 +0800 From: Chengwen Feng To: , CC: Subject: [RFC 3/3] net/hns3: refactor reset process with coroutine Date: Mon, 24 Apr 2023 13:02:08 +0000 Message-ID: <20230424130208.9517-4-fengchengwen@huawei.com> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20230424130208.9517-1-fengchengwen@huawei.com> References: <20230424130208.9517-1-fengchengwen@huawei.com> MIME-Version: 1.0 Content-Type: text/plain X-Originating-IP: [10.50.163.32] X-ClientProxiedBy: dggems706-chm.china.huawei.com (10.3.19.183) To dggpeml500024.china.huawei.com (7.185.36.10) X-CFilter-Loop: Reflected X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org This patch adds reset mode 1 which use coroutine to refactor the reset process. And this just a demo which only function at PF driver. Using the coroutine will make the reset process more intuitive. Signed-off-by: Chengwen Feng --- drivers/net/hns3/hns3_ethdev.c | 217 +++++++++++++++++++++++++++++++++ drivers/net/hns3/hns3_ethdev.h | 3 + drivers/net/hns3/hns3_intr.c | 38 ++++++ drivers/net/hns3/meson.build | 2 +- 4 files changed, 259 insertions(+), 1 deletion(-) diff --git a/drivers/net/hns3/hns3_ethdev.c b/drivers/net/hns3/hns3_ethdev.c index 36896f8989..06ff0bcae1 100644 --- a/drivers/net/hns3/hns3_ethdev.c +++ b/drivers/net/hns3/hns3_ethdev.c @@ -6487,7 +6487,224 @@ static const struct eth_dev_ops hns3_eth_dev_ops = { .eth_tx_descriptor_dump = hns3_tx_descriptor_dump, }; +#include + +static const char *reset_string[HNS3_MAX_RESET] = { + "flr", "vf_func", "vf_pf_func", "vf_full", "vf_global", + "pf_func", "global", "IMP", "none", +}; + +static void +hns3_clear_reset_level(struct hns3_hw *hw, uint64_t *levels) +{ + uint64_t merge_cnt = hw->reset.stats.merge_cnt; + uint64_t tmp; + + switch (hw->reset.level) { + case HNS3_IMP_RESET: + hns3_atomic_clear_bit(HNS3_IMP_RESET, levels); + tmp = hns3_test_and_clear_bit(HNS3_GLOBAL_RESET, levels); + merge_cnt = tmp > 0 ? merge_cnt + 1 : merge_cnt; + tmp = hns3_test_and_clear_bit(HNS3_FUNC_RESET, levels); + merge_cnt = tmp > 0 ? merge_cnt + 1 : merge_cnt; + break; + case HNS3_GLOBAL_RESET: + hns3_atomic_clear_bit(HNS3_GLOBAL_RESET, levels); + tmp = hns3_test_and_clear_bit(HNS3_FUNC_RESET, levels); + merge_cnt = tmp > 0 ? merge_cnt + 1 : merge_cnt; + break; + case HNS3_FUNC_RESET: + hns3_atomic_clear_bit(HNS3_FUNC_RESET, levels); + break; + case HNS3_VF_RESET: + hns3_atomic_clear_bit(HNS3_VF_RESET, levels); + tmp = hns3_test_and_clear_bit(HNS3_VF_PF_FUNC_RESET, levels); + merge_cnt = tmp > 0 ? merge_cnt + 1 : merge_cnt; + tmp = hns3_test_and_clear_bit(HNS3_VF_FUNC_RESET, levels); + merge_cnt = tmp > 0 ? merge_cnt + 1 : merge_cnt; + break; + case HNS3_VF_FULL_RESET: + hns3_atomic_clear_bit(HNS3_VF_FULL_RESET, levels); + tmp = hns3_test_and_clear_bit(HNS3_VF_FUNC_RESET, levels); + merge_cnt = tmp > 0 ? merge_cnt + 1 : merge_cnt; + break; + case HNS3_VF_PF_FUNC_RESET: + hns3_atomic_clear_bit(HNS3_VF_PF_FUNC_RESET, levels); + tmp = hns3_test_and_clear_bit(HNS3_VF_FUNC_RESET, levels); + merge_cnt = tmp > 0 ? merge_cnt + 1 : merge_cnt; + break; + case HNS3_VF_FUNC_RESET: + hns3_atomic_clear_bit(HNS3_VF_FUNC_RESET, levels); + break; + case HNS3_FLR_RESET: + hns3_atomic_clear_bit(HNS3_FLR_RESET, levels); + break; + case HNS3_NONE_RESET: + default: + return; + }; + + if (merge_cnt != hw->reset.stats.merge_cnt) { + hns3_warn(hw, + "No need to do low-level reset after %s reset. " + "merge cnt: %" PRIu64 " total merge cnt: %" PRIu64, + reset_string[hw->reset.level], + hw->reset.stats.merge_cnt - merge_cnt, + hw->reset.stats.merge_cnt); + hw->reset.stats.merge_cnt = merge_cnt; + } +} + +static void hns3_reset_coroutine(void *arg) +{ + struct hns3_hw *hw = (struct hns3_hw *)arg; + struct hns3_adapter *hns = HNS3_DEV_HW_TO_ADAPTER(hw); + struct timeval tv, tv_delta; + int ret, i; + + /* + * calc reset level. + */ + hns3_clock_gettime(&hw->reset.start_time); + hw->reset.level = hns3_get_reset_level(hns, &hw->reset.pending); + if (hw->reset.level == HNS3_NONE_RESET) + hw->reset.level = HNS3_IMP_RESET; + hns3_warn(hw, "start reset level: %s", reset_string[hw->reset.level]); + + + /* + * stop service. + */ + ret = hns3_stop_service(hns); + hns3_clock_gettime(&tv); + if (ret) { + hns3_warn(hw, "Reset step1 down fail=%d time=%ld.%.6ld", + ret, tv.tv_sec, tv.tv_usec); + return; + } + hns3_warn(hw, "Reset step1 down success time=%ld.%.6ld", + tv.tv_sec, tv.tv_usec); + + + /* + * yield CPU to schedule other function's reset. + */ + rte_co_yield(); + + + /* + * prepare reset. + */ + ret = hns3_prepare_reset(hns); + hns3_clock_gettime(&tv); + if (ret) { + hns3_warn(hw, + "Reset step2 prepare wait fail=%d time=%ld.%.6ld", + ret, tv.tv_sec, tv.tv_usec); + return; + } + hns3_warn(hw, "Reset step2 prepare wait success time=%ld.%.6ld", + tv.tv_sec, tv.tv_usec); + + + /* + * delay 100ms which refer the manual of hardware. + */ + rte_co_delay(100 * 1000); + + + /* + * notify IMP reset. + */ + /* inform hardware that preparatory work is done */ + hns3_notify_reset_ready(hw, true); + hns3_clock_gettime(&tv); + hns3_warn(hw, + "Reset step3 request IMP reset success time=%ld.%.6ld", + tv.tv_sec, tv.tv_usec); + + + /* + * Wait hardware reset done. + */ + for (i = 0; i < HNS3_RESET_WAIT_CNT; i++) { + rte_co_delay(HNS3_RESET_WAIT_MS * USEC_PER_MSEC); + if (is_pf_reset_done(hw)) + break; + } + if (i >= HNS3_RESET_WAIT_CNT) { + hns3_clock_gettime(&tv); + hns3_warn(hw, "Reset step4 hardware not ready after reset time=%ld.%.6ld", + tv.tv_sec, tv.tv_usec); + return; + } + hns3_clock_gettime(&tv); + hns3_warn(hw, "Reset step4 reset wait success time=%ld.%.6ld", + tv.tv_sec, tv.tv_usec); + + + /* + * yield CPU to schedule other function's reset. + */ + rte_co_yield(); + + + /* + * Devinit. + */ + rte_spinlock_lock(&hw->lock); + if (hw->reset.mbuf_deferred_free) { + hns3_dev_release_mbufs(hns); + hw->reset.mbuf_deferred_free = false; + } + ret = hns3_reinit_dev(hns); + rte_spinlock_unlock(&hw->lock); + hns3_clock_gettime(&tv); + if (ret) { + hns3_warn(hw, "Reset step5 devinit fail=%d retries=%d", + ret, hw->reset.retries); + return; + } + hns3_warn(hw, "Reset step5 devinit success time=%ld.%.6ld", + tv.tv_sec, tv.tv_usec); + + + /* + * yield CPU to schedule other function's reset. + */ + rte_co_yield(); + + + /* + * Start service. + */ + /* IMP will wait ready flag before reset */ + hns3_notify_reset_ready(hw, false); + hns3_clear_reset_level(hw, &hw->reset.pending); + rte_spinlock_lock(&hw->lock); + hns3_start_service(hns); + rte_spinlock_unlock(&hw->lock); + hns3_clock_gettime(&tv); + timersub(&tv, &hw->reset.start_time, &tv_delta); + hns3_warn(hw, "%s reset done fail_cnt:%" PRIu64 + " success_cnt:%" PRIu64 " global_cnt:%" PRIu64 + " imp_cnt:%" PRIu64 " request_cnt:%" PRIu64 + " exec_cnt:%" PRIu64 " merge_cnt:%" PRIu64, + reset_string[hw->reset.level], + hw->reset.stats.fail_cnt, hw->reset.stats.success_cnt, + hw->reset.stats.global_cnt, hw->reset.stats.imp_cnt, + hw->reset.stats.request_cnt, hw->reset.stats.exec_cnt, + hw->reset.stats.merge_cnt); + hns3_warn(hw, + "%s reset done delta %" PRIu64 " ms time=%ld.%.6ld", + reset_string[hw->reset.level], + hns3_clock_calctime_ms(&tv_delta), + tv.tv_sec, tv.tv_usec); + hw->reset.level = HNS3_NONE_RESET; +} + static const struct hns3_reset_ops hns3_reset_ops = { + .coroutine = hns3_reset_coroutine, .reset_service = hns3_reset_service, .stop_service = hns3_stop_service, .prepare_reset = hns3_prepare_reset, diff --git a/drivers/net/hns3/hns3_ethdev.h b/drivers/net/hns3/hns3_ethdev.h index 9acc5a3d7e..4029bea133 100644 --- a/drivers/net/hns3/hns3_ethdev.h +++ b/drivers/net/hns3/hns3_ethdev.h @@ -379,6 +379,7 @@ struct hns3_wait_data { }; struct hns3_reset_ops { + void (*coroutine)(void *arg); void (*reset_service)(void *arg); int (*stop_service)(struct hns3_adapter *hns); int (*prepare_reset)(struct hns3_adapter *hns); @@ -396,6 +397,8 @@ enum hns3_schedule { }; struct hns3_reset_data { + struct rte_schedule *reset_sched; + uint32_t mode; enum hns3_reset_stage stage; uint16_t schedule; /* Reset flag, covering the entire reset process */ diff --git a/drivers/net/hns3/hns3_intr.c b/drivers/net/hns3/hns3_intr.c index 44a1119415..327d548bc6 100644 --- a/drivers/net/hns3/hns3_intr.c +++ b/drivers/net/hns3/hns3_intr.c @@ -2393,10 +2393,36 @@ hns3_handle_error(struct hns3_adapter *hns) } } +#include + +static uint32_t thread_run(void *arg) +{ + struct rte_schedule *s = (struct rte_schedule *)arg; + int ret; + + rte_thread_setname(pthread_self(), "co-sched-hns3"); + + while (1) { + ret = rte_schedule_run(s); + if (ret != 0) + break; + } + + return 0; +} + int hns3_reset_init(struct hns3_hw *hw) { rte_spinlock_init(&hw->lock); + hw->reset.mode = 1; /* mode 1 means use coroutine. */ + static struct rte_schedule *s; + if (s == NULL) { + rte_thread_t thread; + s = rte_schedule_create("co-sched-hns3", 128); + rte_thread_create(&thread, NULL, thread_run, (void *)s); + } + hw->reset.reset_sched = s; hw->reset.level = HNS3_NONE_RESET; hw->reset.stage = RESET_STAGE_NONE; hw->reset.request = 0; @@ -2427,6 +2453,12 @@ hns3_schedule_reset(struct hns3_adapter *hns) if (hw->adapter_state >= HNS3_NIC_CLOSED) return; + if (hw->reset.mode == 1) { + rte_co_create(hw->reset.reset_sched, hw->reset.ops->coroutine, + (void *)hw, 128 * 1024); + return; + } + /* Schedule restart alarm if it is not scheduled yet */ if (__atomic_load_n(&hw->reset.schedule, __ATOMIC_RELAXED) == SCHEDULE_REQUESTED) @@ -2453,6 +2485,12 @@ hns3_schedule_delayed_reset(struct hns3_adapter *hns) return; } + if (hw->reset.mode == 1) { + rte_co_create(hw->reset.reset_sched, hw->reset.ops->coroutine, + (void *)hw, 128 * 1024); + return; + } + if (__atomic_load_n(&hw->reset.schedule, __ATOMIC_RELAXED) != SCHEDULE_NONE) return; diff --git a/drivers/net/hns3/meson.build b/drivers/net/hns3/meson.build index 33f61f9883..7abb3d1986 100644 --- a/drivers/net/hns3/meson.build +++ b/drivers/net/hns3/meson.build @@ -37,7 +37,7 @@ require_iova_in_mbuf = false annotate_locks = false -deps += ['hash'] +deps += ['hash', 'coroutine'] if arch_subdir == 'arm' and dpdk_conf.get('RTE_ARCH_64') sources += files('hns3_rxtx_vec.c') -- 2.17.1