[RFC PATCH v1 4/5] dts: block all testcases when earlier setup fails

["Juraj Linkeš" <juraj.linkes@pantheon.tech>]

In case of a failure during execution, build target or suite setup
the test case results will be recorded as blocked. We also unify
methods add_<result> to add_child_result to be more consistently.
Now we store the corresponding config in each result with child
configs and parent result.

Signed-off-by: Juraj Linkeš <juraj.linkes@pantheon.tech>
---
 dts/framework/runner.py      |  12 +-
 dts/framework/test_result.py | 361 ++++++++++++++++++++---------------
 2 files changed, 216 insertions(+), 157 deletions(-)

diff --git a/dts/framework/runner.py b/dts/framework/runner.py
index acc3342f0c..28570d4a1c 100644
--- a/dts/framework/runner.py
+++ b/dts/framework/runner.py
@@ -74,7 +74,7 @@ class DTSRunner:
 
     def __init__(self, configuration: Configuration):
         self._logger = getLogger("DTSRunner")
-        self._result = DTSResult(self._logger)
+        self._result = DTSResult(configuration, self._logger)
         self._executions = create_executions(configuration.executions)
 
     def run(self):
@@ -150,7 +150,7 @@ def _run_execution(
             "Running execution with SUT "
             f"'{execution.config.system_under_test_node.name}'."
         )
-        execution_result = self._result.add_execution(sut_node.config)
+        execution_result = self._result.add_child_result(execution.processed_config())
         execution_result.add_sut_info(sut_node.node_info)
 
         try:
@@ -190,7 +190,7 @@ def _run_build_target(
         Run the given build target.
         """
         self._logger.info(f"Running build target '{build_target.name}'.")
-        build_target_result = execution_result.add_build_target(build_target)
+        build_target_result = execution_result.add_child_result(build_target)
 
         try:
             sut_node.set_up_build_target(build_target)
@@ -265,7 +265,9 @@ def _run_test_suite(
         """
         test_suite = test_suite_setup.test_suite(sut_node, tg_node)
         test_suite_name = test_suite_setup.test_suite.__name__
-        test_suite_result = build_target_result.add_test_suite(test_suite_name)
+        test_suite_result = build_target_result.add_child_result(
+            test_suite_setup.processed_config()
+        )
         try:
             self._logger.info(f"Starting test suite setup: {test_suite_name}")
             test_suite.set_up_suite()
@@ -308,7 +310,7 @@ def _execute_test_suite(
         """
         for test_case_method in test_cases:
             test_case_name = test_case_method.__name__
-            test_case_result = test_suite_result.add_test_case(test_case_name)
+            test_case_result = test_suite_result.add_child_result(test_case_name)
             all_attempts = SETTINGS.re_run + 1
             attempt_nr = 1
             self._run_test_case(test_case_method, test_suite, test_case_result)
diff --git a/dts/framework/test_result.py b/dts/framework/test_result.py
index 4c2e7e2418..dba2c55d36 100644
--- a/dts/framework/test_result.py
+++ b/dts/framework/test_result.py
@@ -9,6 +9,7 @@
 import os.path
 from collections.abc import MutableSequence
 from enum import Enum, auto
+from typing import Any, Union
 
 from .config import (
     OS,
@@ -16,9 +17,11 @@
     BuildTargetConfiguration,
     BuildTargetInfo,
     Compiler,
+    Configuration,
     CPUType,
-    NodeConfiguration,
+    ExecutionConfiguration,
     NodeInfo,
+    TestSuiteConfig,
 )
 from .exception import DTSError, ErrorSeverity
 from .logger import DTSLOG
@@ -35,6 +38,7 @@ class Result(Enum):
     FAIL = auto()
     ERROR = auto()
     SKIP = auto()
+    BLOCK = auto()
 
     def __bool__(self) -> bool:
         return self is self.PASS
@@ -63,42 +67,6 @@ def __bool__(self) -> bool:
         return bool(self.result)
 
 
-class Statistics(dict):
-    """
-    A helper class used to store the number of test cases by its result
-    along a few other basic information.
-    Using a dict provides a convenient way to format the data.
-    """
-
-    def __init__(self, dpdk_version: str | None):
-        super(Statistics, self).__init__()
-        for result in Result:
-            self[result.name] = 0
-        self["PASS RATE"] = 0.0
-        self["DPDK VERSION"] = dpdk_version
-
-    def __iadd__(self, other: Result) -> "Statistics":
-        """
-        Add a Result to the final count.
-        """
-        self[other.name] += 1
-        self["PASS RATE"] = (
-            float(self[Result.PASS.name]) * 100 / sum(self[result.name] for result in Result)
-        )
-        return self
-
-    def __str__(self) -> str:
-        """
-        Provide a string representation of the data.
-        """
-        stats_str = ""
-        for key, value in self.items():
-            stats_str += f"{key:<12} = {value}\n"
-            # according to docs, we should use \n when writing to text files
-            # on all platforms
-        return stats_str
-
-
 class BaseResult(object):
     """
     The Base class for all results. Stores the results of
@@ -109,6 +77,12 @@ class BaseResult(object):
     setup_result: FixtureResult
     teardown_result: FixtureResult
     _inner_results: MutableSequence["BaseResult"]
+    _child_configs: Union[
+        list[ExecutionConfiguration],
+        list[BuildTargetConfiguration],
+        list[TestSuiteConfig],
+        list[str],
+    ]
 
     def __init__(self):
         self.setup_result = FixtureResult()
@@ -119,6 +93,23 @@ def update_setup(self, result: Result, error: Exception | None = None) -> None:
         self.setup_result.result = result
         self.setup_result.error = error
 
+        if result in [Result.BLOCK, Result.ERROR, Result.FAIL]:
+            for child_config in self._child_configs:
+                child_result = self.add_child_result(child_config)
+                child_result.block()
+
+    def add_child_result(self, config: Any) -> "BaseResult":
+        """
+        Adding corresponding result for each classes.
+        """
+
+    def block(self):
+        """
+        Mark the result as block on corresponding classes.
+        """
+        self.update_setup(Result.BLOCK)
+        self.update_teardown(Result.BLOCK)
+
     def update_teardown(self, result: Result, error: Exception | None = None) -> None:
         self.teardown_result.result = result
         self.teardown_result.error = error
@@ -139,119 +130,11 @@ def _get_inner_errors(self) -> list[Exception]:
     def get_errors(self) -> list[Exception]:
         return self._get_setup_teardown_errors() + self._get_inner_errors()
 
-    def add_stats(self, statistics: Statistics) -> None:
+    def add_stats(self, statistics: "Statistics") -> None:
         for inner_result in self._inner_results:
             inner_result.add_stats(statistics)
 
 
-class TestCaseResult(BaseResult, FixtureResult):
-    """
-    The test case specific result.
-    Stores the result of the actual test case.
-    Also stores the test case name.
-    """
-
-    test_case_name: str
-
-    def __init__(self, test_case_name: str):
-        super(TestCaseResult, self).__init__()
-        self.test_case_name = test_case_name
-
-    def update(self, result: Result, error: Exception | None = None) -> None:
-        self.result = result
-        self.error = error
-
-    def _get_inner_errors(self) -> list[Exception]:
-        if self.error:
-            return [self.error]
-        return []
-
-    def add_stats(self, statistics: Statistics) -> None:
-        statistics += self.result
-
-    def __bool__(self) -> bool:
-        return bool(self.setup_result) and bool(self.teardown_result) and bool(self.result)
-
-
-class TestSuiteResult(BaseResult):
-    """
-    The test suite specific result.
-    The _inner_results list stores results of test cases in a given test suite.
-    Also stores the test suite name.
-    """
-
-    suite_name: str
-
-    def __init__(self, suite_name: str):
-        super(TestSuiteResult, self).__init__()
-        self.suite_name = suite_name
-
-    def add_test_case(self, test_case_name: str) -> TestCaseResult:
-        test_case_result = TestCaseResult(test_case_name)
-        self._inner_results.append(test_case_result)
-        return test_case_result
-
-
-class BuildTargetResult(BaseResult):
-    """
-    The build target specific result.
-    The _inner_results list stores results of test suites in a given build target.
-    Also stores build target specifics, such as compiler used to build DPDK.
-    """
-
-    arch: Architecture
-    os: OS
-    cpu: CPUType
-    compiler: Compiler
-    compiler_version: str | None
-    dpdk_version: str | None
-
-    def __init__(self, build_target: BuildTargetConfiguration):
-        super(BuildTargetResult, self).__init__()
-        self.arch = build_target.arch
-        self.os = build_target.os
-        self.cpu = build_target.cpu
-        self.compiler = build_target.compiler
-        self.compiler_version = None
-        self.dpdk_version = None
-
-    def add_build_target_info(self, versions: BuildTargetInfo) -> None:
-        self.compiler_version = versions.compiler_version
-        self.dpdk_version = versions.dpdk_version
-
-    def add_test_suite(self, test_suite_name: str) -> TestSuiteResult:
-        test_suite_result = TestSuiteResult(test_suite_name)
-        self._inner_results.append(test_suite_result)
-        return test_suite_result
-
-
-class ExecutionResult(BaseResult):
-    """
-    The execution specific result.
-    The _inner_results list stores results of build targets in a given execution.
-    Also stores the SUT node configuration.
-    """
-
-    sut_node: NodeConfiguration
-    sut_os_name: str
-    sut_os_version: str
-    sut_kernel_version: str
-
-    def __init__(self, sut_node: NodeConfiguration):
-        super(ExecutionResult, self).__init__()
-        self.sut_node = sut_node
-
-    def add_build_target(self, build_target: BuildTargetConfiguration) -> BuildTargetResult:
-        build_target_result = BuildTargetResult(build_target)
-        self._inner_results.append(build_target_result)
-        return build_target_result
-
-    def add_sut_info(self, sut_info: NodeInfo):
-        self.sut_os_name = sut_info.os_name
-        self.sut_os_version = sut_info.os_version
-        self.sut_kernel_version = sut_info.kernel_version
-
-
 class DTSResult(BaseResult):
     """
     Stores environment information and test results from a DTS run, which are:
@@ -269,25 +152,27 @@ class DTSResult(BaseResult):
     """
 
     dpdk_version: str | None
+    _child_configs: list[ExecutionConfiguration]
     _logger: DTSLOG
     _errors: list[Exception]
     _return_code: ErrorSeverity
-    _stats_result: Statistics | None
+    _stats_result: Union["Statistics", None]
     _stats_filename: str
 
-    def __init__(self, logger: DTSLOG):
+    def __init__(self, configuration: Configuration, logger: DTSLOG):
         super(DTSResult, self).__init__()
         self.dpdk_version = None
+        self._child_configs = configuration.executions
         self._logger = logger
         self._errors = []
         self._return_code = ErrorSeverity.NO_ERR
         self._stats_result = None
         self._stats_filename = os.path.join(SETTINGS.output_dir, "statistics.txt")
 
-    def add_execution(self, sut_node: NodeConfiguration) -> ExecutionResult:
-        execution_result = ExecutionResult(sut_node)
-        self._inner_results.append(execution_result)
-        return execution_result
+    def add_child_result(self, config: ExecutionConfiguration) -> "ExecutionResult":
+        result = ExecutionResult(config, self)
+        self._inner_results.append(result)
+        return result
 
     def add_error(self, error) -> None:
         self._errors.append(error)
@@ -325,3 +210,175 @@ def get_return_code(self) -> int:
                 self._return_code = error_return_code
 
         return int(self._return_code)
+
+
+class ExecutionResult(BaseResult):
+    """
+    The execution specific result.
+    The _inner_results list stores results of build targets in a given execution.
+    Also stores the SUT node configuration.
+    """
+
+    sut_os_name: str
+    sut_os_version: str
+    sut_kernel_version: str
+    _config: ExecutionConfiguration
+    _parent_result: DTSResult
+    _child_configs: list[BuildTargetConfiguration]
+
+    def __init__(self, config: ExecutionConfiguration, parent_result: DTSResult):
+        super(ExecutionResult, self).__init__()
+        self._config = config
+        self._parent_result = parent_result
+        self._child_configs = config.build_targets
+
+    def add_sut_info(self, sut_info: NodeInfo):
+        self.sut_os_name = sut_info.os_name
+        self.sut_os_version = sut_info.os_version
+        self.sut_kernel_version = sut_info.kernel_version
+
+    def add_child_result(self, config: BuildTargetConfiguration) -> "BuildTargetResult":
+        result = BuildTargetResult(config, self)
+        self._inner_results.append(result)
+        return result
+
+
+class BuildTargetResult(BaseResult):
+    """
+    The build target specific result.
+    The _inner_results list stores results of test suites in a given build target.
+    Also stores build target specifics, such as compiler used to build DPDK.
+    """
+
+    arch: Architecture
+    os: OS
+    cpu: CPUType
+    compiler: Compiler
+    compiler_version: str | None
+    dpdk_version: str | None
+    _config: BuildTargetConfiguration
+    _parent_result: ExecutionResult
+    _child_configs: list[TestSuiteConfig]
+
+    def __init__(
+        self, config: BuildTargetConfiguration, parent_result: ExecutionResult
+    ):
+        super(BuildTargetResult, self).__init__()
+        self.arch = config.arch
+        self.os = config.os
+        self.cpu = config.cpu
+        self.compiler = config.compiler
+        self.compiler_version = None
+        self.dpdk_version = None
+        self._config = config
+        self._parent_result = parent_result
+        self._child_configs = parent_result._config.test_suites
+
+    def add_build_target_info(self, versions: BuildTargetInfo) -> None:
+        self.compiler_version = versions.compiler_version
+        self.dpdk_version = versions.dpdk_version
+
+    def add_child_result(
+        self,
+        config: TestSuiteConfig,
+    ) -> "TestSuiteResult":
+        result = TestSuiteResult(config, self)
+        self._inner_results.append(result)
+        return result
+
+
+class TestSuiteResult(BaseResult):
+    """
+    The test suite specific result.
+    The _inner_results list stores results of test cases in a given test suite.
+    Also stores the test suite name.
+    """
+
+    _config: TestSuiteConfig
+    _parent_result: BuildTargetResult
+    _child_configs: list[str]
+
+    def __init__(self, config: TestSuiteConfig, parent_result: BuildTargetResult):
+        super(TestSuiteResult, self).__init__()
+        self._config = config
+        self._parent_result = parent_result
+        self._child_configs = config.test_cases
+
+    def add_child_result(self, config: str) -> "TestCaseResult":
+        result = TestCaseResult(config, self)
+        self._inner_results.append(result)
+        return result
+
+
+class TestCaseResult(BaseResult, FixtureResult):
+    """
+    The test case specific result.
+    Stores the result of the actual test case.
+    Also stores the test case name.
+    """
+
+    _config: str
+    _parent_result: TestSuiteResult
+
+    def __init__(self, config: str, parent_result: TestSuiteResult):
+        super(TestCaseResult, self).__init__()
+        self._config = config
+        self._parent_result = parent_result
+
+    def block(self):
+        self.update(Result.BLOCK)
+
+    def update(self, result: Result, error: Exception | None = None) -> None:
+        self.result = result
+        self.error = error
+
+    def _get_inner_errors(self) -> list[Exception]:
+        if self.error:
+            return [self.error]
+        return []
+
+    def add_stats(self, statistics: "Statistics") -> None:
+        statistics += self.result
+
+    def __bool__(self) -> bool:
+        return (
+            bool(self.setup_result) and bool(self.teardown_result) and bool(self.result)
+        )
+
+
+class Statistics(dict):
+    """
+    A helper class used to store the number of test cases by its result
+    along a few other basic information.
+    Using a dict provides a convenient way to format the data.
+    """
+
+    def __init__(self, dpdk_version: str | None):
+        super(Statistics, self).__init__()
+        for result in Result:
+            self[result.name] = 0
+        self["PASS RATE"] = 0.0
+        self["DPDK VERSION"] = dpdk_version
+
+    def __iadd__(self, other: Result) -> "Statistics":
+        """
+        Add a Result to the final count.
+        """
+        self[other.name] += 1
+        self["PASS RATE"] = (
+            float(self[Result.PASS.name])
+            * 100
+            / sum(self[result.name] for result in Result)
+        )
+        return self
+
+    def __str__(self) -> str:
+        """
+        Provide a string representation of the data.
+        """
+        stats_str = ""
+        for key, value in self.items():
+            stats_str += f"{key:<12} = {value}\n"
+            # according to docs, we should use \n when writing to text files
+            # on all platforms
+        return stats_str
-- 
2.34.1