From mboxrd@z Thu Jan 1 00:00:00 1970
Return-Path: <dev-bounces@dpdk.org>
Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124])
by inbox.dpdk.org (Postfix) with ESMTP id 5F8C743723;
Wed, 20 Dec 2023 11:34:09 +0100 (CET)
Received: from mails.dpdk.org (localhost [127.0.0.1])
by mails.dpdk.org (Postfix) with ESMTP id 7E91542EAC;
Wed, 20 Dec 2023 11:33:44 +0100 (CET)
Received: from mail-ed1-f50.google.com (mail-ed1-f50.google.com
[209.85.208.50]) by mails.dpdk.org (Postfix) with ESMTP id D495742EAC
for <dev@dpdk.org>; Wed, 20 Dec 2023 11:33:43 +0100 (CET)
Received: by mail-ed1-f50.google.com with SMTP id
4fb4d7f45d1cf-54cd2281ccbso6662749a12.2
for <dev@dpdk.org>; Wed, 20 Dec 2023 02:33:43 -0800 (PST)
DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
d=pantheon.tech; s=google; t=1703068423; x=1703673223; darn=dpdk.org;
h=content-transfer-encoding:mime-version:references:in-reply-to
:message-id:date:subject:cc:to:from:from:to:cc:subject:date
:message-id:reply-to;
bh=C8I+tzUYCstQOXtLA4opXJHZXqBE8ATIFlqtXbIH0EA=;
b=ftOiS19khzB0l4vESrb87pm9ae+SMD/p/VCKi/yzQ2hETLG2krUS1rJ2O2RQiLYHjg
WWlityPJ9KMUQpJjQtOHuZV4dgW9P3FRKsTJu0zDEyo4QsMLHMfGAVKVsdwVZ/PNn33b
CCsfIYu7kSMyBmqxcbj8zg27wXjOFOwkuhXwBcIOLqEUr7tEuS34TMYKAY2hSQxqUxrO
bCV2jlrblp/aYMPC5gNjlILKj9nski4kHDD+x/v6RGfHp00fW98C31fYSpJFTGqFfYJY
2n2nFsbpydDurfr3LCvKRuLEmEis6+dmYpB3EVrO0DnEJhCswtM8qNMpyO4/lDzxulc/
mD4Q==
X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
d=1e100.net; s=20230601; t=1703068423; x=1703673223;
h=content-transfer-encoding:mime-version:references:in-reply-to
:message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc
:subject:date:message-id:reply-to;
bh=C8I+tzUYCstQOXtLA4opXJHZXqBE8ATIFlqtXbIH0EA=;
b=tL47M2cdX7hv7tixRe8a1OWhY0g1SHF5ebXwMiio5I7H44My+u1eD4gfJLiz9E4ejQ
UNLCnNjAT1Ss1H48t31JvLaRLFGM2lRwUwokgKEJ9IzYU+0cgZ0rkbgoDdVOHXtknEBm
EZMnNqitbbyvmoHV/7CJ30wT1/bwmW9bBqwSEYHpipk7fJOh4K8ruwazhC9jfhOek6Ya
yy+yozgrJaCaMLhvq8HnVyDyInr240vJTo9WeeM520PA55GEFLuK4R0O0S6fu00yEKAH
rdon2T7QVI5WXyhcvQeCOoBd9c9DxwDWLzPAzriZoWTapCx5CpX66cf1SdqfHvlxH3ey
W86Q==
X-Gm-Message-State: AOJu0Yy+Sa6Xcy8JHKHXSp3IiPnHzV/bVyju13Yn9Fe4eK1zrj/WYhFt
N+BgMzbVL/HJT5FF0W4RmrB3NrWIMt4nMZHe+umWlw==
X-Google-Smtp-Source: AGHT+IGSbhSKO2xqaSdbX5J3uTjjQ6IqD3qR+EC16UBS+IFXBtIQbTj6+Ayvlw2vXJ4Tzme2s5HN+A==
X-Received: by 2002:a50:c8c2:0:b0:551:edec:a91d with SMTP id
k2-20020a50c8c2000000b00551edeca91dmr6089535edh.82.1703068423430;
Wed, 20 Dec 2023 02:33:43 -0800 (PST)
Received: from jlinkes-PT-Latitude-5530.pantheon.local
(81.89.53.154.host.vnet.sk. [81.89.53.154])
by smtp.gmail.com with ESMTPSA id
bd18-20020a056402207200b00542db304680sm12588981edb.63.2023.12.20.02.33.41
(version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256);
Wed, 20 Dec 2023 02:33:42 -0800 (PST)
From: =?UTF-8?q?Juraj=20Linke=C5=A1?= <juraj.linkes@pantheon.tech>
To: thomas@monjalon.net, Honnappa.Nagarahalli@arm.com, jspewock@iol.unh.edu,
probb@iol.unh.edu, paul.szczepanek@arm.com, yoan.picchi@foss.arm.com,
Luca.Vizzarro@arm.com
Cc: dev@dpdk.org, =?UTF-8?q?Juraj=20Linke=C5=A1?= <juraj.linkes@pantheon.tech>
Subject: [RFC PATCH v1 4/5] dts: block all testcases when earlier setup fails
Date: Wed, 20 Dec 2023 11:33:30 +0100
Message-Id: <20231220103331.60888-5-juraj.linkes@pantheon.tech>
X-Mailer: git-send-email 2.34.1
In-Reply-To: <20231220103331.60888-1-juraj.linkes@pantheon.tech>
References: <20231220103331.60888-1-juraj.linkes@pantheon.tech>
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
X-BeenThere: dev@dpdk.org
X-Mailman-Version: 2.1.29
Precedence: list
List-Id: DPDK patches and discussions <dev.dpdk.org>
List-Unsubscribe: <https://mails.dpdk.org/options/dev>,
<mailto:dev-request@dpdk.org?subject=unsubscribe>
List-Archive: <http://mails.dpdk.org/archives/dev/>
List-Post: <mailto:dev@dpdk.org>
List-Help: <mailto:dev-request@dpdk.org?subject=help>
List-Subscribe: <https://mails.dpdk.org/listinfo/dev>,
<mailto:dev-request@dpdk.org?subject=subscribe>
Errors-To: dev-bounces@dpdk.org
In case of a failure during execution, build target or suite setup
the test case results will be recorded as blocked. We also unify
methods add_<result> to add_child_result to be more consistently.
Now we store the corresponding config in each result with child
configs and parent result.
Signed-off-by: Juraj Linkeš <juraj.linkes@pantheon.tech>
---
dts/framework/runner.py | 12 +-
dts/framework/test_result.py | 361 ++++++++++++++++++++---------------
2 files changed, 216 insertions(+), 157 deletions(-)
diff --git a/dts/framework/runner.py b/dts/framework/runner.py
index acc3342f0c..28570d4a1c 100644
--- a/dts/framework/runner.py
+++ b/dts/framework/runner.py
@@ -74,7 +74,7 @@ class DTSRunner:
def __init__(self, configuration: Configuration):
self._logger = getLogger("DTSRunner")
- self._result = DTSResult(self._logger)
+ self._result = DTSResult(configuration, self._logger)
self._executions = create_executions(configuration.executions)
def run(self):
@@ -150,7 +150,7 @@ def _run_execution(
"Running execution with SUT "
f"'{execution.config.system_under_test_node.name}'."
)
- execution_result = self._result.add_execution(sut_node.config)
+ execution_result = self._result.add_child_result(execution.processed_config())
execution_result.add_sut_info(sut_node.node_info)
try:
@@ -190,7 +190,7 @@ def _run_build_target(
Run the given build target.
"""
self._logger.info(f"Running build target '{build_target.name}'.")
- build_target_result = execution_result.add_build_target(build_target)
+ build_target_result = execution_result.add_child_result(build_target)
try:
sut_node.set_up_build_target(build_target)
@@ -265,7 +265,9 @@ def _run_test_suite(
"""
test_suite = test_suite_setup.test_suite(sut_node, tg_node)
test_suite_name = test_suite_setup.test_suite.__name__
- test_suite_result = build_target_result.add_test_suite(test_suite_name)
+ test_suite_result = build_target_result.add_child_result(
+ test_suite_setup.processed_config()
+ )
try:
self._logger.info(f"Starting test suite setup: {test_suite_name}")
test_suite.set_up_suite()
@@ -308,7 +310,7 @@ def _execute_test_suite(
"""
for test_case_method in test_cases:
test_case_name = test_case_method.__name__
- test_case_result = test_suite_result.add_test_case(test_case_name)
+ test_case_result = test_suite_result.add_child_result(test_case_name)
all_attempts = SETTINGS.re_run + 1
attempt_nr = 1
self._run_test_case(test_case_method, test_suite, test_case_result)
diff --git a/dts/framework/test_result.py b/dts/framework/test_result.py
index 4c2e7e2418..dba2c55d36 100644
--- a/dts/framework/test_result.py
+++ b/dts/framework/test_result.py
@@ -9,6 +9,7 @@
import os.path
from collections.abc import MutableSequence
from enum import Enum, auto
+from typing import Any, Union
from .config import (
OS,
@@ -16,9 +17,11 @@
BuildTargetConfiguration,
BuildTargetInfo,
Compiler,
+ Configuration,
CPUType,
- NodeConfiguration,
+ ExecutionConfiguration,
NodeInfo,
+ TestSuiteConfig,
)
from .exception import DTSError, ErrorSeverity
from .logger import DTSLOG
@@ -35,6 +38,7 @@ class Result(Enum):
FAIL = auto()
ERROR = auto()
SKIP = auto()
+ BLOCK = auto()
def __bool__(self) -> bool:
return self is self.PASS
@@ -63,42 +67,6 @@ def __bool__(self) -> bool:
return bool(self.result)
-class Statistics(dict):
- """
- A helper class used to store the number of test cases by its result
- along a few other basic information.
- Using a dict provides a convenient way to format the data.
- """
-
- def __init__(self, dpdk_version: str | None):
- super(Statistics, self).__init__()
- for result in Result:
- self[result.name] = 0
- self["PASS RATE"] = 0.0
- self["DPDK VERSION"] = dpdk_version
-
- def __iadd__(self, other: Result) -> "Statistics":
- """
- Add a Result to the final count.
- """
- self[other.name] += 1
- self["PASS RATE"] = (
- float(self[Result.PASS.name]) * 100 / sum(self[result.name] for result in Result)
- )
- return self
-
- def __str__(self) -> str:
- """
- Provide a string representation of the data.
- """
- stats_str = ""
- for key, value in self.items():
- stats_str += f"{key:<12} = {value}\n"
- # according to docs, we should use \n when writing to text files
- # on all platforms
- return stats_str
-
-
class BaseResult(object):
"""
The Base class for all results. Stores the results of
@@ -109,6 +77,12 @@ class BaseResult(object):
setup_result: FixtureResult
teardown_result: FixtureResult
_inner_results: MutableSequence["BaseResult"]
+ _child_configs: Union[
+ list[ExecutionConfiguration],
+ list[BuildTargetConfiguration],
+ list[TestSuiteConfig],
+ list[str],
+ ]
def __init__(self):
self.setup_result = FixtureResult()
@@ -119,6 +93,23 @@ def update_setup(self, result: Result, error: Exception | None = None) -> None:
self.setup_result.result = result
self.setup_result.error = error
+ if result in [Result.BLOCK, Result.ERROR, Result.FAIL]:
+ for child_config in self._child_configs:
+ child_result = self.add_child_result(child_config)
+ child_result.block()
+
+ def add_child_result(self, config: Any) -> "BaseResult":
+ """
+ Adding corresponding result for each classes.
+ """
+
+ def block(self):
+ """
+ Mark the result as block on corresponding classes.
+ """
+ self.update_setup(Result.BLOCK)
+ self.update_teardown(Result.BLOCK)
+
def update_teardown(self, result: Result, error: Exception | None = None) -> None:
self.teardown_result.result = result
self.teardown_result.error = error
@@ -139,119 +130,11 @@ def _get_inner_errors(self) -> list[Exception]:
def get_errors(self) -> list[Exception]:
return self._get_setup_teardown_errors() + self._get_inner_errors()
- def add_stats(self, statistics: Statistics) -> None:
+ def add_stats(self, statistics: "Statistics") -> None:
for inner_result in self._inner_results:
inner_result.add_stats(statistics)
-class TestCaseResult(BaseResult, FixtureResult):
- """
- The test case specific result.
- Stores the result of the actual test case.
- Also stores the test case name.
- """
-
- test_case_name: str
-
- def __init__(self, test_case_name: str):
- super(TestCaseResult, self).__init__()
- self.test_case_name = test_case_name
-
- def update(self, result: Result, error: Exception | None = None) -> None:
- self.result = result
- self.error = error
-
- def _get_inner_errors(self) -> list[Exception]:
- if self.error:
- return [self.error]
- return []
-
- def add_stats(self, statistics: Statistics) -> None:
- statistics += self.result
-
- def __bool__(self) -> bool:
- return bool(self.setup_result) and bool(self.teardown_result) and bool(self.result)
-
-
-class TestSuiteResult(BaseResult):
- """
- The test suite specific result.
- The _inner_results list stores results of test cases in a given test suite.
- Also stores the test suite name.
- """
-
- suite_name: str
-
- def __init__(self, suite_name: str):
- super(TestSuiteResult, self).__init__()
- self.suite_name = suite_name
-
- def add_test_case(self, test_case_name: str) -> TestCaseResult:
- test_case_result = TestCaseResult(test_case_name)
- self._inner_results.append(test_case_result)
- return test_case_result
-
-
-class BuildTargetResult(BaseResult):
- """
- The build target specific result.
- The _inner_results list stores results of test suites in a given build target.
- Also stores build target specifics, such as compiler used to build DPDK.
- """
-
- arch: Architecture
- os: OS
- cpu: CPUType
- compiler: Compiler
- compiler_version: str | None
- dpdk_version: str | None
-
- def __init__(self, build_target: BuildTargetConfiguration):
- super(BuildTargetResult, self).__init__()
- self.arch = build_target.arch
- self.os = build_target.os
- self.cpu = build_target.cpu
- self.compiler = build_target.compiler
- self.compiler_version = None
- self.dpdk_version = None
-
- def add_build_target_info(self, versions: BuildTargetInfo) -> None:
- self.compiler_version = versions.compiler_version
- self.dpdk_version = versions.dpdk_version
-
- def add_test_suite(self, test_suite_name: str) -> TestSuiteResult:
- test_suite_result = TestSuiteResult(test_suite_name)
- self._inner_results.append(test_suite_result)
- return test_suite_result
-
-
-class ExecutionResult(BaseResult):
- """
- The execution specific result.
- The _inner_results list stores results of build targets in a given execution.
- Also stores the SUT node configuration.
- """
-
- sut_node: NodeConfiguration
- sut_os_name: str
- sut_os_version: str
- sut_kernel_version: str
-
- def __init__(self, sut_node: NodeConfiguration):
- super(ExecutionResult, self).__init__()
- self.sut_node = sut_node
-
- def add_build_target(self, build_target: BuildTargetConfiguration) -> BuildTargetResult:
- build_target_result = BuildTargetResult(build_target)
- self._inner_results.append(build_target_result)
- return build_target_result
-
- def add_sut_info(self, sut_info: NodeInfo):
- self.sut_os_name = sut_info.os_name
- self.sut_os_version = sut_info.os_version
- self.sut_kernel_version = sut_info.kernel_version
-
-
class DTSResult(BaseResult):
"""
Stores environment information and test results from a DTS run, which are:
@@ -269,25 +152,27 @@ class DTSResult(BaseResult):
"""
dpdk_version: str | None
+ _child_configs: list[ExecutionConfiguration]
_logger: DTSLOG
_errors: list[Exception]
_return_code: ErrorSeverity
- _stats_result: Statistics | None
+ _stats_result: Union["Statistics", None]
_stats_filename: str
- def __init__(self, logger: DTSLOG):
+ def __init__(self, configuration: Configuration, logger: DTSLOG):
super(DTSResult, self).__init__()
self.dpdk_version = None
+ self._child_configs = configuration.executions
self._logger = logger
self._errors = []
self._return_code = ErrorSeverity.NO_ERR
self._stats_result = None
self._stats_filename = os.path.join(SETTINGS.output_dir, "statistics.txt")
- def add_execution(self, sut_node: NodeConfiguration) -> ExecutionResult:
- execution_result = ExecutionResult(sut_node)
- self._inner_results.append(execution_result)
- return execution_result
+ def add_child_result(self, config: ExecutionConfiguration) -> "ExecutionResult":
+ result = ExecutionResult(config, self)
+ self._inner_results.append(result)
+ return result
def add_error(self, error) -> None:
self._errors.append(error)
@@ -325,3 +210,175 @@ def get_return_code(self) -> int:
self._return_code = error_return_code
return int(self._return_code)
+
+
+class ExecutionResult(BaseResult):
+ """
+ The execution specific result.
+ The _inner_results list stores results of build targets in a given execution.
+ Also stores the SUT node configuration.
+ """
+
+ sut_os_name: str
+ sut_os_version: str
+ sut_kernel_version: str
+ _config: ExecutionConfiguration
+ _parent_result: DTSResult
+ _child_configs: list[BuildTargetConfiguration]
+
+ def __init__(self, config: ExecutionConfiguration, parent_result: DTSResult):
+ super(ExecutionResult, self).__init__()
+ self._config = config
+ self._parent_result = parent_result
+ self._child_configs = config.build_targets
+
+ def add_sut_info(self, sut_info: NodeInfo):
+ self.sut_os_name = sut_info.os_name
+ self.sut_os_version = sut_info.os_version
+ self.sut_kernel_version = sut_info.kernel_version
+
+ def add_child_result(self, config: BuildTargetConfiguration) -> "BuildTargetResult":
+ result = BuildTargetResult(config, self)
+ self._inner_results.append(result)
+ return result
+
+
+class BuildTargetResult(BaseResult):
+ """
+ The build target specific result.
+ The _inner_results list stores results of test suites in a given build target.
+ Also stores build target specifics, such as compiler used to build DPDK.
+ """
+
+ arch: Architecture
+ os: OS
+ cpu: CPUType
+ compiler: Compiler
+ compiler_version: str | None
+ dpdk_version: str | None
+ _config: BuildTargetConfiguration
+ _parent_result: ExecutionResult
+ _child_configs: list[TestSuiteConfig]
+
+ def __init__(
+ self, config: BuildTargetConfiguration, parent_result: ExecutionResult
+ ):
+ super(BuildTargetResult, self).__init__()
+ self.arch = config.arch
+ self.os = config.os
+ self.cpu = config.cpu
+ self.compiler = config.compiler
+ self.compiler_version = None
+ self.dpdk_version = None
+ self._config = config
+ self._parent_result = parent_result
+ self._child_configs = parent_result._config.test_suites
+
+ def add_build_target_info(self, versions: BuildTargetInfo) -> None:
+ self.compiler_version = versions.compiler_version
+ self.dpdk_version = versions.dpdk_version
+
+ def add_child_result(
+ self,
+ config: TestSuiteConfig,
+ ) -> "TestSuiteResult":
+ result = TestSuiteResult(config, self)
+ self._inner_results.append(result)
+ return result
+
+
+class TestSuiteResult(BaseResult):
+ """
+ The test suite specific result.
+ The _inner_results list stores results of test cases in a given test suite.
+ Also stores the test suite name.
+ """
+
+ _config: TestSuiteConfig
+ _parent_result: BuildTargetResult
+ _child_configs: list[str]
+
+ def __init__(self, config: TestSuiteConfig, parent_result: BuildTargetResult):
+ super(TestSuiteResult, self).__init__()
+ self._config = config
+ self._parent_result = parent_result
+ self._child_configs = config.test_cases
+
+ def add_child_result(self, config: str) -> "TestCaseResult":
+ result = TestCaseResult(config, self)
+ self._inner_results.append(result)
+ return result
+
+
+class TestCaseResult(BaseResult, FixtureResult):
+ """
+ The test case specific result.
+ Stores the result of the actual test case.
+ Also stores the test case name.
+ """
+
+ _config: str
+ _parent_result: TestSuiteResult
+
+ def __init__(self, config: str, parent_result: TestSuiteResult):
+ super(TestCaseResult, self).__init__()
+ self._config = config
+ self._parent_result = parent_result
+
+ def block(self):
+ self.update(Result.BLOCK)
+
+ def update(self, result: Result, error: Exception | None = None) -> None:
+ self.result = result
+ self.error = error
+
+ def _get_inner_errors(self) -> list[Exception]:
+ if self.error:
+ return [self.error]
+ return []
+
+ def add_stats(self, statistics: "Statistics") -> None:
+ statistics += self.result
+
+ def __bool__(self) -> bool:
+ return (
+ bool(self.setup_result) and bool(self.teardown_result) and bool(self.result)
+ )
+
+
+class Statistics(dict):
+ """
+ A helper class used to store the number of test cases by its result
+ along a few other basic information.
+ Using a dict provides a convenient way to format the data.
+ """
+
+ def __init__(self, dpdk_version: str | None):
+ super(Statistics, self).__init__()
+ for result in Result:
+ self[result.name] = 0
+ self["PASS RATE"] = 0.0
+ self["DPDK VERSION"] = dpdk_version
+
+ def __iadd__(self, other: Result) -> "Statistics":
+ """
+ Add a Result to the final count.
+ """
+ self[other.name] += 1
+ self["PASS RATE"] = (
+ float(self[Result.PASS.name])
+ * 100
+ / sum(self[result.name] for result in Result)
+ )
+ return self
+
+ def __str__(self) -> str:
+ """
+ Provide a string representation of the data.
+ """
+ stats_str = ""
+ for key, value in self.items():
+ stats_str += f"{key:<12} = {value}\n"
+ # according to docs, we should use \n when writing to text files
+ # on all platforms
+ return stats_str
--
2.34.1