From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 4B7EC43723; Wed, 20 Dec 2023 11:33:45 +0100 (CET) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id A33E84069F; Wed, 20 Dec 2023 11:33:39 +0100 (CET) Received: from mail-ed1-f51.google.com (mail-ed1-f51.google.com [209.85.208.51]) by mails.dpdk.org (Postfix) with ESMTP id 75B3140266 for ; Wed, 20 Dec 2023 11:33:37 +0100 (CET) Received: by mail-ed1-f51.google.com with SMTP id 4fb4d7f45d1cf-54c5d041c23so6742473a12.2 for ; Wed, 20 Dec 2023 02:33:37 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=pantheon.tech; s=google; t=1703068417; x=1703673217; darn=dpdk.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=Q0u/kk4T6JqRIvq8MzQsLP56aHeYd/gLkYhJLWMm+eY=; b=PDZqxFdmsc3pijUILI+q4mvFzfomspRJEU8QqslVstVVShRc2Xb9pQ/nzBlm+uyFp/ FgP9uA0hO+OhhCzo8wsSVd6RKJSaEUy3TkzJgzZ6Cmiw4KWpBeiKKf6GNmfW5IzFAnZY 0bEAjvd12YS81cU528yeT9fPJFwSf1MizKq0HVfX1YFRD5vtYhN7SkNbFEr3BjgI/AEJ 5b7LmIfRyOuXMD1L+O4EO3Eo5/QOr8mhZYujiSqdg7oBccTFZNzzFz4Kb1/kH2fE4tML 42NvNZwMCS/JPjkW+7fognO7WxN1Igyj5LFRDa+Igy26bfjKKDg18hXuIBqKSHYDVwU7 lRmw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1703068417; x=1703673217; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=Q0u/kk4T6JqRIvq8MzQsLP56aHeYd/gLkYhJLWMm+eY=; b=gc+z6ConrkkAgDYOgbBr+dsw953ypsymc9uqYoQame4+j4GnSoEAw+QTiNQWzEEBJt 6OVDOyaXY+N2gmyYXBQhLZKcACeFgXPA5XWQVVXy7uaPixMHbVBmKEdd4DR/fc9H2+zr 8xh7+MPn8cSZMxyWI57dQkij0IJgNo/FY5hbbLnrUK85vCuMuNlyg6ibeuvvexxR0E9F wC7IAhO7ghraJr1EkawGVSDAkKST73esqbyfyWAAK5sz/wiKearuH/UdvEgbjyqprI4t ovuLGDcE9llpEHOHdbcfuzqJLiF/ZolaQRS1lQuvwMYf0g5G1JgBPEiCwAkxKFLgUoh8 Vi4Q== X-Gm-Message-State: AOJu0YxBsnZYNaibDnhlcrLpO4+V88dqEpc+UmMHSxj2RpdyOm03pAnt 8UFa2fZvpAbgEQi1F0/ytOmE/g== X-Google-Smtp-Source: AGHT+IHMpsbJe952KcyJUfPWJ7N3UC6r8WSSGvIjIZntTxehIhyjFkgl32ky6nb0ugnEw0UZedtVuQ== X-Received: by 2002:a50:aad0:0:b0:553:3a99:4ca7 with SMTP id r16-20020a50aad0000000b005533a994ca7mr1599109edc.78.1703068416910; Wed, 20 Dec 2023 02:33:36 -0800 (PST) Received: from jlinkes-PT-Latitude-5530.pantheon.local (81.89.53.154.host.vnet.sk. [81.89.53.154]) by smtp.gmail.com with ESMTPSA id bd18-20020a056402207200b00542db304680sm12588981edb.63.2023.12.20.02.33.35 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 20 Dec 2023 02:33:36 -0800 (PST) From: =?UTF-8?q?Juraj=20Linke=C5=A1?= To: thomas@monjalon.net, Honnappa.Nagarahalli@arm.com, jspewock@iol.unh.edu, probb@iol.unh.edu, paul.szczepanek@arm.com, yoan.picchi@foss.arm.com, Luca.Vizzarro@arm.com Cc: dev@dpdk.org, =?UTF-8?q?Juraj=20Linke=C5=A1?= Subject: [RFC PATCH v1 1/5] dts: convert dts.py methods to class Date: Wed, 20 Dec 2023 11:33:27 +0100 Message-Id: <20231220103331.60888-2-juraj.linkes@pantheon.tech> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20231220103331.60888-1-juraj.linkes@pantheon.tech> References: <20231220103331.60888-1-juraj.linkes@pantheon.tech> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org The dts.py module deviates from the rest of the code without a clear reason. Converting it into a class and using better naming will improve organization and code readability. Signed-off-by: Juraj Linkeš --- dts/framework/config/__init__.py | 3 - dts/framework/dts.py | 228 ----------------------------- dts/framework/runner.py | 243 +++++++++++++++++++++++++++++++ dts/main.py | 6 +- 4 files changed, 247 insertions(+), 233 deletions(-) delete mode 100644 dts/framework/dts.py create mode 100644 dts/framework/runner.py diff --git a/dts/framework/config/__init__.py b/dts/framework/config/__init__.py index 9b32cf0532..497847afb9 100644 --- a/dts/framework/config/__init__.py +++ b/dts/framework/config/__init__.py @@ -314,6 +314,3 @@ def load_config() -> Configuration: config: dict[str, Any] = warlock.model_factory(schema, name="_Config")(config_data) config_obj: Configuration = Configuration.from_dict(dict(config)) return config_obj - - -CONFIGURATION = load_config() diff --git a/dts/framework/dts.py b/dts/framework/dts.py deleted file mode 100644 index 25d6942d81..0000000000 --- a/dts/framework/dts.py +++ /dev/null @@ -1,228 +0,0 @@ -# SPDX-License-Identifier: BSD-3-Clause -# Copyright(c) 2010-2019 Intel Corporation -# Copyright(c) 2022-2023 PANTHEON.tech s.r.o. -# Copyright(c) 2022-2023 University of New Hampshire - -import sys - -from .config import ( - CONFIGURATION, - BuildTargetConfiguration, - ExecutionConfiguration, - TestSuiteConfig, -) -from .exception import BlockingTestSuiteError -from .logger import DTSLOG, getLogger -from .test_result import BuildTargetResult, DTSResult, ExecutionResult, Result -from .test_suite import get_test_suites -from .testbed_model import SutNode, TGNode -from .utils import check_dts_python_version - -dts_logger: DTSLOG = getLogger("DTSRunner") -result: DTSResult = DTSResult(dts_logger) - - -def run_all() -> None: - """ - The main process of DTS. Runs all build targets in all executions from the main - config file. - """ - global dts_logger - global result - - # check the python version of the server that run dts - check_dts_python_version() - - sut_nodes: dict[str, SutNode] = {} - tg_nodes: dict[str, TGNode] = {} - try: - # for all Execution sections - for execution in CONFIGURATION.executions: - sut_node = sut_nodes.get(execution.system_under_test_node.name) - tg_node = tg_nodes.get(execution.traffic_generator_node.name) - - try: - if not sut_node: - sut_node = SutNode(execution.system_under_test_node) - sut_nodes[sut_node.name] = sut_node - if not tg_node: - tg_node = TGNode(execution.traffic_generator_node) - tg_nodes[tg_node.name] = tg_node - result.update_setup(Result.PASS) - except Exception as e: - failed_node = execution.system_under_test_node.name - if sut_node: - failed_node = execution.traffic_generator_node.name - dts_logger.exception(f"Creation of node {failed_node} failed.") - result.update_setup(Result.FAIL, e) - - else: - _run_execution(sut_node, tg_node, execution, result) - - except Exception as e: - dts_logger.exception("An unexpected error has occurred.") - result.add_error(e) - raise - - finally: - try: - for node in (sut_nodes | tg_nodes).values(): - node.close() - result.update_teardown(Result.PASS) - except Exception as e: - dts_logger.exception("Final cleanup of nodes failed.") - result.update_teardown(Result.ERROR, e) - - # we need to put the sys.exit call outside the finally clause to make sure - # that unexpected exceptions will propagate - # in that case, the error that should be reported is the uncaught exception as - # that is a severe error originating from the framework - # at that point, we'll only have partial results which could be impacted by the - # error causing the uncaught exception, making them uninterpretable - _exit_dts() - - -def _run_execution( - sut_node: SutNode, - tg_node: TGNode, - execution: ExecutionConfiguration, - result: DTSResult, -) -> None: - """ - Run the given execution. This involves running the execution setup as well as - running all build targets in the given execution. - """ - dts_logger.info(f"Running execution with SUT '{execution.system_under_test_node.name}'.") - execution_result = result.add_execution(sut_node.config) - execution_result.add_sut_info(sut_node.node_info) - - try: - sut_node.set_up_execution(execution) - execution_result.update_setup(Result.PASS) - except Exception as e: - dts_logger.exception("Execution setup failed.") - execution_result.update_setup(Result.FAIL, e) - - else: - for build_target in execution.build_targets: - _run_build_target(sut_node, tg_node, build_target, execution, execution_result) - - finally: - try: - sut_node.tear_down_execution() - execution_result.update_teardown(Result.PASS) - except Exception as e: - dts_logger.exception("Execution teardown failed.") - execution_result.update_teardown(Result.FAIL, e) - - -def _run_build_target( - sut_node: SutNode, - tg_node: TGNode, - build_target: BuildTargetConfiguration, - execution: ExecutionConfiguration, - execution_result: ExecutionResult, -) -> None: - """ - Run the given build target. - """ - dts_logger.info(f"Running build target '{build_target.name}'.") - build_target_result = execution_result.add_build_target(build_target) - - try: - sut_node.set_up_build_target(build_target) - result.dpdk_version = sut_node.dpdk_version - build_target_result.add_build_target_info(sut_node.get_build_target_info()) - build_target_result.update_setup(Result.PASS) - except Exception as e: - dts_logger.exception("Build target setup failed.") - build_target_result.update_setup(Result.FAIL, e) - - else: - _run_all_suites(sut_node, tg_node, execution, build_target_result) - - finally: - try: - sut_node.tear_down_build_target() - build_target_result.update_teardown(Result.PASS) - except Exception as e: - dts_logger.exception("Build target teardown failed.") - build_target_result.update_teardown(Result.FAIL, e) - - -def _run_all_suites( - sut_node: SutNode, - tg_node: TGNode, - execution: ExecutionConfiguration, - build_target_result: BuildTargetResult, -) -> None: - """ - Use the given build_target to run execution's test suites - with possibly only a subset of test cases. - If no subset is specified, run all test cases. - """ - end_build_target = False - if not execution.skip_smoke_tests: - execution.test_suites[:0] = [TestSuiteConfig.from_dict("smoke_tests")] - for test_suite_config in execution.test_suites: - try: - _run_single_suite(sut_node, tg_node, execution, build_target_result, test_suite_config) - except BlockingTestSuiteError as e: - dts_logger.exception( - f"An error occurred within {test_suite_config.test_suite}. Skipping build target." - ) - result.add_error(e) - end_build_target = True - # if a blocking test failed and we need to bail out of suite executions - if end_build_target: - break - - -def _run_single_suite( - sut_node: SutNode, - tg_node: TGNode, - execution: ExecutionConfiguration, - build_target_result: BuildTargetResult, - test_suite_config: TestSuiteConfig, -) -> None: - """Runs a single test suite. - - Args: - sut_node: Node to run tests on. - execution: Execution the test case belongs to. - build_target_result: Build target configuration test case is run on - test_suite_config: Test suite configuration - - Raises: - BlockingTestSuiteError: If a test suite that was marked as blocking fails. - """ - try: - full_suite_path = f"tests.TestSuite_{test_suite_config.test_suite}" - test_suite_classes = get_test_suites(full_suite_path) - suites_str = ", ".join((x.__name__ for x in test_suite_classes)) - dts_logger.debug(f"Found test suites '{suites_str}' in '{full_suite_path}'.") - except Exception as e: - dts_logger.exception("An error occurred when searching for test suites.") - result.update_setup(Result.ERROR, e) - - else: - for test_suite_class in test_suite_classes: - test_suite = test_suite_class( - sut_node, - tg_node, - test_suite_config.test_cases, - execution.func, - build_target_result, - ) - test_suite.run() - - -def _exit_dts() -> None: - """ - Process all errors and exit with the proper exit code. - """ - result.process() - - if dts_logger: - dts_logger.info("DTS execution has ended.") - sys.exit(result.get_return_code()) diff --git a/dts/framework/runner.py b/dts/framework/runner.py new file mode 100644 index 0000000000..5b077c5805 --- /dev/null +++ b/dts/framework/runner.py @@ -0,0 +1,243 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2010-2019 Intel Corporation +# Copyright(c) 2022-2023 PANTHEON.tech s.r.o. +# Copyright(c) 2022-2023 University of New Hampshire + +import logging +import sys + +from .config import ( + BuildTargetConfiguration, + Configuration, + ExecutionConfiguration, + TestSuiteConfig, +) +from .exception import BlockingTestSuiteError +from .logger import DTSLOG, getLogger +from .test_result import BuildTargetResult, DTSResult, ExecutionResult, Result +from .test_suite import get_test_suites +from .testbed_model import SutNode, TGNode +from .utils import check_dts_python_version + + +class DTSRunner: + _logger: DTSLOG + _result: DTSResult + _configuration: Configuration + + def __init__(self, configuration: Configuration): + self._logger = getLogger("DTSRunner") + self._result = DTSResult(self._logger) + self._configuration = configuration + + def run(self): + """ + The main process of DTS. Runs all build targets in all executions from the main + config file. + """ + # check the python version of the server that run dts + check_dts_python_version() + sut_nodes: dict[str, SutNode] = {} + tg_nodes: dict[str, TGNode] = {} + try: + # for all Execution sections + for execution in self._configuration.executions: + sut_node = sut_nodes.get(execution.system_under_test_node.name) + tg_node = tg_nodes.get(execution.traffic_generator_node.name) + + try: + if not sut_node: + sut_node = SutNode(execution.system_under_test_node) + sut_nodes[sut_node.name] = sut_node + if not tg_node: + tg_node = TGNode(execution.traffic_generator_node) + tg_nodes[tg_node.name] = tg_node + self._result.update_setup(Result.PASS) + except Exception as e: + failed_node = execution.system_under_test_node.name + if sut_node: + failed_node = execution.traffic_generator_node.name + self._logger.exception( + f"The Creation of node {failed_node} failed." + ) + self._result.update_setup(Result.FAIL, e) + + else: + self._run_execution(sut_node, tg_node, execution) + + except Exception as e: + self._logger.exception("An unexpected error has occurred.") + self._result.add_error(e) + raise + + finally: + try: + for node in (sut_nodes | tg_nodes).values(): + node.close() + self._result.update_teardown(Result.PASS) + except Exception as e: + self._logger.exception("The final cleanup of nodes failed.") + self._result.update_teardown(Result.ERROR, e) + + # we need to put the sys.exit call outside the finally clause to make sure + # that unexpected exceptions will propagate + # in that case, the error that should be reported is the uncaught exception as + # that is a severe error originating from the framework + # at that point, we'll only have partial results which could be impacted by the + # error causing the uncaught exception, making them uninterpretable + self._exit_dts() + + def _run_execution( + self, + sut_node: SutNode, + tg_node: TGNode, + execution: ExecutionConfiguration, + ) -> None: + """ + Run the given execution. This involves running the execution setup as well as + running all build targets in the given execution. + """ + self._logger.info( + f"Running execution with SUT '{execution.system_under_test_node.name}'." + ) + execution_result = self._result.add_execution(sut_node.config) + execution_result.add_sut_info(sut_node.node_info) + + try: + sut_node.set_up_execution(execution) + execution_result.update_setup(Result.PASS) + except Exception as e: + self._logger.exception("Execution setup failed.") + execution_result.update_setup(Result.FAIL, e) + + else: + for build_target in execution.build_targets: + self._run_build_target( + sut_node, tg_node, build_target, execution, execution_result + ) + + finally: + try: + sut_node.tear_down_execution() + execution_result.update_teardown(Result.PASS) + except Exception as e: + self._logger.exception("Execution teardown failed.") + execution_result.update_teardown(Result.FAIL, e) + + def _run_build_target( + self, + sut_node: SutNode, + tg_node: TGNode, + build_target: BuildTargetConfiguration, + execution: ExecutionConfiguration, + execution_result: ExecutionResult, + ) -> None: + """ + Run the given build target. + """ + self._logger.info(f"Running build target '{build_target.name}'.") + build_target_result = execution_result.add_build_target(build_target) + + try: + sut_node.set_up_build_target(build_target) + self._result.dpdk_version = sut_node.dpdk_version + build_target_result.add_build_target_info(sut_node.get_build_target_info()) + build_target_result.update_setup(Result.PASS) + except Exception as e: + self._logger.exception("Build target setup failed.") + build_target_result.update_setup(Result.FAIL, e) + + else: + self._run_all_suites(sut_node, tg_node, execution, build_target_result) + + finally: + try: + sut_node.tear_down_build_target() + build_target_result.update_teardown(Result.PASS) + except Exception as e: + self._logger.exception("Build target teardown failed.") + build_target_result.update_teardown(Result.FAIL, e) + + def _run_all_suites( + self, + sut_node: SutNode, + tg_node: TGNode, + execution: ExecutionConfiguration, + build_target_result: BuildTargetResult, + ) -> None: + """ + Use the given build_target to run execution's test suites + with possibly only a subset of test cases. + If no subset is specified, run all test cases. + """ + end_build_target = False + if not execution.skip_smoke_tests: + execution.test_suites[:0] = [TestSuiteConfig.from_dict("smoke_tests")] + for test_suite_config in execution.test_suites: + try: + self._run_single_suite( + sut_node, tg_node, execution, build_target_result, test_suite_config + ) + except BlockingTestSuiteError as e: + self._logger.exception( + f"An error occurred within {test_suite_config.test_suite}. " + "Skipping build target..." + ) + self._result.add_error(e) + end_build_target = True + # if a blocking test failed and we need to bail out of suite executions + if end_build_target: + break + + def _run_single_suite( + self, + sut_node: SutNode, + tg_node: TGNode, + execution: ExecutionConfiguration, + build_target_result: BuildTargetResult, + test_suite_config: TestSuiteConfig, + ) -> None: + """Runs a single test suite. + + Args: + sut_node: Node to run tests on. + execution: Execution the test case belongs to. + build_target_result: Build target configuration test case is run on + test_suite_config: Test suite configuration + + Raises: + BlockingTestSuiteError: If a test suite that was marked as blocking fails. + """ + try: + full_suite_path = f"tests.TestSuite_{test_suite_config.test_suite}" + test_suite_classes = get_test_suites(full_suite_path) + suites_str = ", ".join((x.__name__ for x in test_suite_classes)) + self._logger.debug( + f"Found test suites '{suites_str}' in '{full_suite_path}'." + ) + except Exception as e: + self._logger.exception("An error occurred when searching for test suites.") + self._result.update_setup(Result.ERROR, e) + + else: + for test_suite_class in test_suite_classes: + test_suite = test_suite_class( + sut_node, + tg_node, + test_suite_config.test_cases, + execution.func, + build_target_result, + ) + test_suite.run() + + def _exit_dts(self) -> None: + """ + Process all errors and exit with the proper exit code. + """ + self._result.process() + + if self._logger: + self._logger.info("DTS execution has ended.") + + logging.shutdown() + sys.exit(self._result.get_return_code()) diff --git a/dts/main.py b/dts/main.py index 43311fa847..879ce5cb89 100755 --- a/dts/main.py +++ b/dts/main.py @@ -10,11 +10,13 @@ import logging -from framework import dts +from framework.config import load_config +from framework.runner import DTSRunner def main() -> None: - dts.run_all() + dts = DTSRunner(configuration=load_config()) + dts.run() # Main program begins here -- 2.34.1