From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 5F8C743723; Wed, 20 Dec 2023 11:34:09 +0100 (CET) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 7E91542EAC; Wed, 20 Dec 2023 11:33:44 +0100 (CET) Received: from mail-ed1-f50.google.com (mail-ed1-f50.google.com [209.85.208.50]) by mails.dpdk.org (Postfix) with ESMTP id D495742EAC for ; Wed, 20 Dec 2023 11:33:43 +0100 (CET) Received: by mail-ed1-f50.google.com with SMTP id 4fb4d7f45d1cf-54cd2281ccbso6662749a12.2 for ; Wed, 20 Dec 2023 02:33:43 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=pantheon.tech; s=google; t=1703068423; x=1703673223; darn=dpdk.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=C8I+tzUYCstQOXtLA4opXJHZXqBE8ATIFlqtXbIH0EA=; b=ftOiS19khzB0l4vESrb87pm9ae+SMD/p/VCKi/yzQ2hETLG2krUS1rJ2O2RQiLYHjg WWlityPJ9KMUQpJjQtOHuZV4dgW9P3FRKsTJu0zDEyo4QsMLHMfGAVKVsdwVZ/PNn33b CCsfIYu7kSMyBmqxcbj8zg27wXjOFOwkuhXwBcIOLqEUr7tEuS34TMYKAY2hSQxqUxrO bCV2jlrblp/aYMPC5gNjlILKj9nski4kHDD+x/v6RGfHp00fW98C31fYSpJFTGqFfYJY 2n2nFsbpydDurfr3LCvKRuLEmEis6+dmYpB3EVrO0DnEJhCswtM8qNMpyO4/lDzxulc/ mD4Q== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1703068423; x=1703673223; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=C8I+tzUYCstQOXtLA4opXJHZXqBE8ATIFlqtXbIH0EA=; b=tL47M2cdX7hv7tixRe8a1OWhY0g1SHF5ebXwMiio5I7H44My+u1eD4gfJLiz9E4ejQ UNLCnNjAT1Ss1H48t31JvLaRLFGM2lRwUwokgKEJ9IzYU+0cgZ0rkbgoDdVOHXtknEBm EZMnNqitbbyvmoHV/7CJ30wT1/bwmW9bBqwSEYHpipk7fJOh4K8ruwazhC9jfhOek6Ya yy+yozgrJaCaMLhvq8HnVyDyInr240vJTo9WeeM520PA55GEFLuK4R0O0S6fu00yEKAH rdon2T7QVI5WXyhcvQeCOoBd9c9DxwDWLzPAzriZoWTapCx5CpX66cf1SdqfHvlxH3ey W86Q== X-Gm-Message-State: AOJu0Yy+Sa6Xcy8JHKHXSp3IiPnHzV/bVyju13Yn9Fe4eK1zrj/WYhFt N+BgMzbVL/HJT5FF0W4RmrB3NrWIMt4nMZHe+umWlw== X-Google-Smtp-Source: AGHT+IGSbhSKO2xqaSdbX5J3uTjjQ6IqD3qR+EC16UBS+IFXBtIQbTj6+Ayvlw2vXJ4Tzme2s5HN+A== X-Received: by 2002:a50:c8c2:0:b0:551:edec:a91d with SMTP id k2-20020a50c8c2000000b00551edeca91dmr6089535edh.82.1703068423430; Wed, 20 Dec 2023 02:33:43 -0800 (PST) Received: from jlinkes-PT-Latitude-5530.pantheon.local (81.89.53.154.host.vnet.sk. [81.89.53.154]) by smtp.gmail.com with ESMTPSA id bd18-20020a056402207200b00542db304680sm12588981edb.63.2023.12.20.02.33.41 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 20 Dec 2023 02:33:42 -0800 (PST) From: =?UTF-8?q?Juraj=20Linke=C5=A1?= To: thomas@monjalon.net, Honnappa.Nagarahalli@arm.com, jspewock@iol.unh.edu, probb@iol.unh.edu, paul.szczepanek@arm.com, yoan.picchi@foss.arm.com, Luca.Vizzarro@arm.com Cc: dev@dpdk.org, =?UTF-8?q?Juraj=20Linke=C5=A1?= Subject: [RFC PATCH v1 4/5] dts: block all testcases when earlier setup fails Date: Wed, 20 Dec 2023 11:33:30 +0100 Message-Id: <20231220103331.60888-5-juraj.linkes@pantheon.tech> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20231220103331.60888-1-juraj.linkes@pantheon.tech> References: <20231220103331.60888-1-juraj.linkes@pantheon.tech> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org In case of a failure during execution, build target or suite setup the test case results will be recorded as blocked. We also unify methods add_ to add_child_result to be more consistently. Now we store the corresponding config in each result with child configs and parent result. Signed-off-by: Juraj Linkeš --- dts/framework/runner.py | 12 +- dts/framework/test_result.py | 361 ++++++++++++++++++++--------------- 2 files changed, 216 insertions(+), 157 deletions(-) diff --git a/dts/framework/runner.py b/dts/framework/runner.py index acc3342f0c..28570d4a1c 100644 --- a/dts/framework/runner.py +++ b/dts/framework/runner.py @@ -74,7 +74,7 @@ class DTSRunner: def __init__(self, configuration: Configuration): self._logger = getLogger("DTSRunner") - self._result = DTSResult(self._logger) + self._result = DTSResult(configuration, self._logger) self._executions = create_executions(configuration.executions) def run(self): @@ -150,7 +150,7 @@ def _run_execution( "Running execution with SUT " f"'{execution.config.system_under_test_node.name}'." ) - execution_result = self._result.add_execution(sut_node.config) + execution_result = self._result.add_child_result(execution.processed_config()) execution_result.add_sut_info(sut_node.node_info) try: @@ -190,7 +190,7 @@ def _run_build_target( Run the given build target. """ self._logger.info(f"Running build target '{build_target.name}'.") - build_target_result = execution_result.add_build_target(build_target) + build_target_result = execution_result.add_child_result(build_target) try: sut_node.set_up_build_target(build_target) @@ -265,7 +265,9 @@ def _run_test_suite( """ test_suite = test_suite_setup.test_suite(sut_node, tg_node) test_suite_name = test_suite_setup.test_suite.__name__ - test_suite_result = build_target_result.add_test_suite(test_suite_name) + test_suite_result = build_target_result.add_child_result( + test_suite_setup.processed_config() + ) try: self._logger.info(f"Starting test suite setup: {test_suite_name}") test_suite.set_up_suite() @@ -308,7 +310,7 @@ def _execute_test_suite( """ for test_case_method in test_cases: test_case_name = test_case_method.__name__ - test_case_result = test_suite_result.add_test_case(test_case_name) + test_case_result = test_suite_result.add_child_result(test_case_name) all_attempts = SETTINGS.re_run + 1 attempt_nr = 1 self._run_test_case(test_case_method, test_suite, test_case_result) diff --git a/dts/framework/test_result.py b/dts/framework/test_result.py index 4c2e7e2418..dba2c55d36 100644 --- a/dts/framework/test_result.py +++ b/dts/framework/test_result.py @@ -9,6 +9,7 @@ import os.path from collections.abc import MutableSequence from enum import Enum, auto +from typing import Any, Union from .config import ( OS, @@ -16,9 +17,11 @@ BuildTargetConfiguration, BuildTargetInfo, Compiler, + Configuration, CPUType, - NodeConfiguration, + ExecutionConfiguration, NodeInfo, + TestSuiteConfig, ) from .exception import DTSError, ErrorSeverity from .logger import DTSLOG @@ -35,6 +38,7 @@ class Result(Enum): FAIL = auto() ERROR = auto() SKIP = auto() + BLOCK = auto() def __bool__(self) -> bool: return self is self.PASS @@ -63,42 +67,6 @@ def __bool__(self) -> bool: return bool(self.result) -class Statistics(dict): - """ - A helper class used to store the number of test cases by its result - along a few other basic information. - Using a dict provides a convenient way to format the data. - """ - - def __init__(self, dpdk_version: str | None): - super(Statistics, self).__init__() - for result in Result: - self[result.name] = 0 - self["PASS RATE"] = 0.0 - self["DPDK VERSION"] = dpdk_version - - def __iadd__(self, other: Result) -> "Statistics": - """ - Add a Result to the final count. - """ - self[other.name] += 1 - self["PASS RATE"] = ( - float(self[Result.PASS.name]) * 100 / sum(self[result.name] for result in Result) - ) - return self - - def __str__(self) -> str: - """ - Provide a string representation of the data. - """ - stats_str = "" - for key, value in self.items(): - stats_str += f"{key:<12} = {value}\n" - # according to docs, we should use \n when writing to text files - # on all platforms - return stats_str - - class BaseResult(object): """ The Base class for all results. Stores the results of @@ -109,6 +77,12 @@ class BaseResult(object): setup_result: FixtureResult teardown_result: FixtureResult _inner_results: MutableSequence["BaseResult"] + _child_configs: Union[ + list[ExecutionConfiguration], + list[BuildTargetConfiguration], + list[TestSuiteConfig], + list[str], + ] def __init__(self): self.setup_result = FixtureResult() @@ -119,6 +93,23 @@ def update_setup(self, result: Result, error: Exception | None = None) -> None: self.setup_result.result = result self.setup_result.error = error + if result in [Result.BLOCK, Result.ERROR, Result.FAIL]: + for child_config in self._child_configs: + child_result = self.add_child_result(child_config) + child_result.block() + + def add_child_result(self, config: Any) -> "BaseResult": + """ + Adding corresponding result for each classes. + """ + + def block(self): + """ + Mark the result as block on corresponding classes. + """ + self.update_setup(Result.BLOCK) + self.update_teardown(Result.BLOCK) + def update_teardown(self, result: Result, error: Exception | None = None) -> None: self.teardown_result.result = result self.teardown_result.error = error @@ -139,119 +130,11 @@ def _get_inner_errors(self) -> list[Exception]: def get_errors(self) -> list[Exception]: return self._get_setup_teardown_errors() + self._get_inner_errors() - def add_stats(self, statistics: Statistics) -> None: + def add_stats(self, statistics: "Statistics") -> None: for inner_result in self._inner_results: inner_result.add_stats(statistics) -class TestCaseResult(BaseResult, FixtureResult): - """ - The test case specific result. - Stores the result of the actual test case. - Also stores the test case name. - """ - - test_case_name: str - - def __init__(self, test_case_name: str): - super(TestCaseResult, self).__init__() - self.test_case_name = test_case_name - - def update(self, result: Result, error: Exception | None = None) -> None: - self.result = result - self.error = error - - def _get_inner_errors(self) -> list[Exception]: - if self.error: - return [self.error] - return [] - - def add_stats(self, statistics: Statistics) -> None: - statistics += self.result - - def __bool__(self) -> bool: - return bool(self.setup_result) and bool(self.teardown_result) and bool(self.result) - - -class TestSuiteResult(BaseResult): - """ - The test suite specific result. - The _inner_results list stores results of test cases in a given test suite. - Also stores the test suite name. - """ - - suite_name: str - - def __init__(self, suite_name: str): - super(TestSuiteResult, self).__init__() - self.suite_name = suite_name - - def add_test_case(self, test_case_name: str) -> TestCaseResult: - test_case_result = TestCaseResult(test_case_name) - self._inner_results.append(test_case_result) - return test_case_result - - -class BuildTargetResult(BaseResult): - """ - The build target specific result. - The _inner_results list stores results of test suites in a given build target. - Also stores build target specifics, such as compiler used to build DPDK. - """ - - arch: Architecture - os: OS - cpu: CPUType - compiler: Compiler - compiler_version: str | None - dpdk_version: str | None - - def __init__(self, build_target: BuildTargetConfiguration): - super(BuildTargetResult, self).__init__() - self.arch = build_target.arch - self.os = build_target.os - self.cpu = build_target.cpu - self.compiler = build_target.compiler - self.compiler_version = None - self.dpdk_version = None - - def add_build_target_info(self, versions: BuildTargetInfo) -> None: - self.compiler_version = versions.compiler_version - self.dpdk_version = versions.dpdk_version - - def add_test_suite(self, test_suite_name: str) -> TestSuiteResult: - test_suite_result = TestSuiteResult(test_suite_name) - self._inner_results.append(test_suite_result) - return test_suite_result - - -class ExecutionResult(BaseResult): - """ - The execution specific result. - The _inner_results list stores results of build targets in a given execution. - Also stores the SUT node configuration. - """ - - sut_node: NodeConfiguration - sut_os_name: str - sut_os_version: str - sut_kernel_version: str - - def __init__(self, sut_node: NodeConfiguration): - super(ExecutionResult, self).__init__() - self.sut_node = sut_node - - def add_build_target(self, build_target: BuildTargetConfiguration) -> BuildTargetResult: - build_target_result = BuildTargetResult(build_target) - self._inner_results.append(build_target_result) - return build_target_result - - def add_sut_info(self, sut_info: NodeInfo): - self.sut_os_name = sut_info.os_name - self.sut_os_version = sut_info.os_version - self.sut_kernel_version = sut_info.kernel_version - - class DTSResult(BaseResult): """ Stores environment information and test results from a DTS run, which are: @@ -269,25 +152,27 @@ class DTSResult(BaseResult): """ dpdk_version: str | None + _child_configs: list[ExecutionConfiguration] _logger: DTSLOG _errors: list[Exception] _return_code: ErrorSeverity - _stats_result: Statistics | None + _stats_result: Union["Statistics", None] _stats_filename: str - def __init__(self, logger: DTSLOG): + def __init__(self, configuration: Configuration, logger: DTSLOG): super(DTSResult, self).__init__() self.dpdk_version = None + self._child_configs = configuration.executions self._logger = logger self._errors = [] self._return_code = ErrorSeverity.NO_ERR self._stats_result = None self._stats_filename = os.path.join(SETTINGS.output_dir, "statistics.txt") - def add_execution(self, sut_node: NodeConfiguration) -> ExecutionResult: - execution_result = ExecutionResult(sut_node) - self._inner_results.append(execution_result) - return execution_result + def add_child_result(self, config: ExecutionConfiguration) -> "ExecutionResult": + result = ExecutionResult(config, self) + self._inner_results.append(result) + return result def add_error(self, error) -> None: self._errors.append(error) @@ -325,3 +210,175 @@ def get_return_code(self) -> int: self._return_code = error_return_code return int(self._return_code) + + +class ExecutionResult(BaseResult): + """ + The execution specific result. + The _inner_results list stores results of build targets in a given execution. + Also stores the SUT node configuration. + """ + + sut_os_name: str + sut_os_version: str + sut_kernel_version: str + _config: ExecutionConfiguration + _parent_result: DTSResult + _child_configs: list[BuildTargetConfiguration] + + def __init__(self, config: ExecutionConfiguration, parent_result: DTSResult): + super(ExecutionResult, self).__init__() + self._config = config + self._parent_result = parent_result + self._child_configs = config.build_targets + + def add_sut_info(self, sut_info: NodeInfo): + self.sut_os_name = sut_info.os_name + self.sut_os_version = sut_info.os_version + self.sut_kernel_version = sut_info.kernel_version + + def add_child_result(self, config: BuildTargetConfiguration) -> "BuildTargetResult": + result = BuildTargetResult(config, self) + self._inner_results.append(result) + return result + + +class BuildTargetResult(BaseResult): + """ + The build target specific result. + The _inner_results list stores results of test suites in a given build target. + Also stores build target specifics, such as compiler used to build DPDK. + """ + + arch: Architecture + os: OS + cpu: CPUType + compiler: Compiler + compiler_version: str | None + dpdk_version: str | None + _config: BuildTargetConfiguration + _parent_result: ExecutionResult + _child_configs: list[TestSuiteConfig] + + def __init__( + self, config: BuildTargetConfiguration, parent_result: ExecutionResult + ): + super(BuildTargetResult, self).__init__() + self.arch = config.arch + self.os = config.os + self.cpu = config.cpu + self.compiler = config.compiler + self.compiler_version = None + self.dpdk_version = None + self._config = config + self._parent_result = parent_result + self._child_configs = parent_result._config.test_suites + + def add_build_target_info(self, versions: BuildTargetInfo) -> None: + self.compiler_version = versions.compiler_version + self.dpdk_version = versions.dpdk_version + + def add_child_result( + self, + config: TestSuiteConfig, + ) -> "TestSuiteResult": + result = TestSuiteResult(config, self) + self._inner_results.append(result) + return result + + +class TestSuiteResult(BaseResult): + """ + The test suite specific result. + The _inner_results list stores results of test cases in a given test suite. + Also stores the test suite name. + """ + + _config: TestSuiteConfig + _parent_result: BuildTargetResult + _child_configs: list[str] + + def __init__(self, config: TestSuiteConfig, parent_result: BuildTargetResult): + super(TestSuiteResult, self).__init__() + self._config = config + self._parent_result = parent_result + self._child_configs = config.test_cases + + def add_child_result(self, config: str) -> "TestCaseResult": + result = TestCaseResult(config, self) + self._inner_results.append(result) + return result + + +class TestCaseResult(BaseResult, FixtureResult): + """ + The test case specific result. + Stores the result of the actual test case. + Also stores the test case name. + """ + + _config: str + _parent_result: TestSuiteResult + + def __init__(self, config: str, parent_result: TestSuiteResult): + super(TestCaseResult, self).__init__() + self._config = config + self._parent_result = parent_result + + def block(self): + self.update(Result.BLOCK) + + def update(self, result: Result, error: Exception | None = None) -> None: + self.result = result + self.error = error + + def _get_inner_errors(self) -> list[Exception]: + if self.error: + return [self.error] + return [] + + def add_stats(self, statistics: "Statistics") -> None: + statistics += self.result + + def __bool__(self) -> bool: + return ( + bool(self.setup_result) and bool(self.teardown_result) and bool(self.result) + ) + + +class Statistics(dict): + """ + A helper class used to store the number of test cases by its result + along a few other basic information. + Using a dict provides a convenient way to format the data. + """ + + def __init__(self, dpdk_version: str | None): + super(Statistics, self).__init__() + for result in Result: + self[result.name] = 0 + self["PASS RATE"] = 0.0 + self["DPDK VERSION"] = dpdk_version + + def __iadd__(self, other: Result) -> "Statistics": + """ + Add a Result to the final count. + """ + self[other.name] += 1 + self["PASS RATE"] = ( + float(self[Result.PASS.name]) + * 100 + / sum(self[result.name] for result in Result) + ) + return self + + def __str__(self) -> str: + """ + Provide a string representation of the data. + """ + stats_str = "" + for key, value in self.items(): + stats_str += f"{key:<12} = {value}\n" + # according to docs, we should use \n when writing to text files + # on all platforms + return stats_str -- 2.34.1