[RFC PATCH v1 02/12] dts: one dpdk build per test run

["Juraj Linkeš" <juraj.linkes@pantheon.tech>]

From: Tomáš Ďurovec <tomas.durovec@pantheon.tech>

Signed-off-by: Tomáš Ďurovec <tomas.durovec@pantheon.tech>
---
 dts/conf.yaml                              |  14 +--
 dts/framework/config/__init__.py           |   9 +-
 dts/framework/config/conf_yaml_schema.json |  10 +-
 dts/framework/config/types.py              |   2 +-
 dts/framework/logger.py                    |   4 -
 dts/framework/runner.py                    | 117 +++++---------------
 dts/framework/test_result.py               | 119 ++++++---------------
 dts/framework/test_suite.py                |   2 +-
 dts/framework/testbed_model/sut_node.py    |   6 +-
 dts/tests/TestSuite_smoke_tests.py         |   2 +-
 10 files changed, 80 insertions(+), 205 deletions(-)

diff --git a/dts/conf.yaml b/dts/conf.yaml
index d43e6fcfeb..3d5ee5aee5 100644
--- a/dts/conf.yaml
+++ b/dts/conf.yaml
@@ -4,13 +4,13 @@
 
 test_runs:
   # define one test run environment
-  - dpdk_builds:
-      - arch: x86_64
-        os: linux
-        cpu: native
-        # the combination of the following two makes CC="ccache gcc"
-        compiler: gcc
-        compiler_wrapper: ccache
+  - dpdk_build:
+      arch: x86_64
+      os: linux
+      cpu: native
+      # the combination of the following two makes CC="ccache gcc"
+      compiler: gcc
+      compiler_wrapper: ccache
     perf: false # disable performance testing
     func: true # enable functional testing
     skip_smoke_tests: false # optional
diff --git a/dts/framework/config/__init__.py b/dts/framework/config/__init__.py
index 598d7101ed..aba49143ae 100644
--- a/dts/framework/config/__init__.py
+++ b/dts/framework/config/__init__.py
@@ -437,7 +437,7 @@ class TestRunConfiguration:
     and with what DPDK build.
 
     Attributes:
-        dpdk_builds: A list of DPDK builds to test.
+        dpdk_build: A DPDK build to test.
         perf: Whether to run performance tests.
         func: Whether to run functional tests.
         skip_smoke_tests: Whether to skip smoke tests.
@@ -447,7 +447,7 @@ class TestRunConfiguration:
         vdevs: The names of virtual devices to test.
     """
 
-    dpdk_builds: list[DPDKBuildConfiguration]
+    dpdk_build: DPDKBuildConfiguration
     perf: bool
     func: bool
     skip_smoke_tests: bool
@@ -475,9 +475,6 @@ def from_dict(
         Returns:
             The test run configuration instance.
         """
-        dpdk_builds: list[DPDKBuildConfiguration] = list(
-            map(DPDKBuildConfiguration.from_dict, d["dpdk_builds"])
-        )
         test_suites: list[TestSuiteConfig] = list(map(TestSuiteConfig.from_dict, d["test_suites"]))
         sut_name = d["system_under_test_node"]["node_name"]
         skip_smoke_tests = d.get("skip_smoke_tests", False)
@@ -498,7 +495,7 @@ def from_dict(
             d["system_under_test_node"]["vdevs"] if "vdevs" in d["system_under_test_node"] else []
         )
         return cls(
-            dpdk_builds=dpdk_builds,
+            dpdk_build=DPDKBuildConfiguration.from_dict(d["dpdk_build"]),
             perf=d["perf"],
             func=d["func"],
             skip_smoke_tests=skip_smoke_tests,
diff --git a/dts/framework/config/conf_yaml_schema.json b/dts/framework/config/conf_yaml_schema.json
index 4b63e9710e..c0c347199e 100644
--- a/dts/framework/config/conf_yaml_schema.json
+++ b/dts/framework/config/conf_yaml_schema.json
@@ -327,12 +327,8 @@
       "items": {
         "type": "object",
         "properties": {
-          "dpdk_builds": {
-            "type": "array",
-            "items": {
-              "$ref": "#/definitions/dpdk_build"
-            },
-            "minimum": 1
+          "dpdk_build": {
+            "$ref": "#/definitions/dpdk_build"
           },
           "perf": {
             "type": "boolean",
@@ -383,7 +379,7 @@
         },
         "additionalProperties": false,
         "required": [
-          "dpdk_builds",
+          "dpdk_build",
           "perf",
           "func",
           "test_suites",
diff --git a/dts/framework/config/types.py b/dts/framework/config/types.py
index 703d9eb48e..9b3c997c80 100644
--- a/dts/framework/config/types.py
+++ b/dts/framework/config/types.py
@@ -108,7 +108,7 @@ class TestRunConfigDict(TypedDict):
     """Allowed keys and values."""
 
     #:
-    dpdk_builds: list[DPDKBuildConfigDict]
+    dpdk_build: DPDKBuildConfigDict
     #:
     perf: bool
     #:
diff --git a/dts/framework/logger.py b/dts/framework/logger.py
index 3fbe618219..d2b8e37da4 100644
--- a/dts/framework/logger.py
+++ b/dts/framework/logger.py
@@ -33,16 +33,12 @@ class DtsStage(StrEnum):
     #:
     test_run_setup = auto()
     #:
-    dpdk_build_setup = auto()
-    #:
     test_suite_setup = auto()
     #:
     test_suite = auto()
     #:
     test_suite_teardown = auto()
     #:
-    dpdk_build_teardown = auto()
-    #:
     test_run_teardown = auto()
     #:
     post_run = auto()
diff --git a/dts/framework/runner.py b/dts/framework/runner.py
index 2b5403e51c..a212ca2470 100644
--- a/dts/framework/runner.py
+++ b/dts/framework/runner.py
@@ -12,7 +12,7 @@
     #. Test suite stage,
     #. Test case stage.
 
-The test run and DPDK build stages set up the environment before running test suites.
+The test run stage sets up the environment before running test suites.
 The test suite stage sets up steps common to all test cases
 and the test case stage runs test cases individually.
 """
@@ -29,13 +29,7 @@
 from framework.testbed_model.sut_node import SutNode
 from framework.testbed_model.tg_node import TGNode
 
-from .config import (
-    Configuration,
-    DPDKBuildConfiguration,
-    TestRunConfiguration,
-    TestSuiteConfig,
-    load_config,
-)
+from .config import Configuration, TestRunConfiguration, TestSuiteConfig, load_config
 from .exception import (
     BlockingTestSuiteError,
     ConfigurationError,
@@ -45,7 +39,6 @@
 from .logger import DTSLogger, DtsStage, get_dts_logger
 from .settings import SETTINGS
 from .test_result import (
-    DPDKBuildResult,
     DTSResult,
     Result,
     TestCaseResult,
@@ -69,9 +62,9 @@ class DTSRunner:
     :class:`~.framework.exception.DTSError`\s.
 
     Example:
-        An error occurs in a DPDK build setup. The current DPDK build is aborted,
-        all test suites and their test cases are marked as blocked and the run continues
-        with the next DPDK build. If the errored DPDK build was the last one in the
+        An error occurs in a test suite setup. The current test suite is aborted,
+        all its test cases are marked as blocked and the run continues
+        with the next test suite. If the errored test suite was the last one in the
         given test run, the next test run begins.
     """
 
@@ -97,16 +90,16 @@ def __init__(self):
         self._perf_test_case_regex = r"test_perf_"
 
     def run(self) -> None:
-        """Run all DPDK build in all test runs from the test run configuration.
+        """Run all test runs from the test run configuration.
 
-        Before running test suites, test runs and DPDK builds are first set up.
-        The test runs and DPDK builds defined in the test run configuration are iterated over.
-        The test runs define which tests to run and where to run them and DPDK builds define
-        the DPDK build setup.
+        Before running test suites, test runs are first set up.
+        The test runs defined in the test run configuration are iterated over.
+        The test runs define which tests to run and where to run them.
 
-        The tests suites are set up for each test run/DPDK build tuple and each discovered
+        The test suites are set up for each test run and each discovered
         test case within the test suite is set up, executed and torn down. After all test cases
-        have been executed, the test suite is torn down and the next DPDK build will be tested.
+        have been executed, the test suite is torn down and the next test suite will be run. Once
+        all test suites have been run, the test test run will be tested.
 
         In order to properly mark test suites and test cases as blocked in case of a failure,
         we need to have discovered which test suites and test cases to run before any failures
@@ -116,17 +109,13 @@ def run(self) -> None:
 
             #. Test run setup
 
-                #. DPDK build setup
-
-                    #. Test suite setup
+                #. Test suite setup
 
-                        #. Test case setup
-                        #. Test case logic
-                        #. Test case teardown
+                    #. Test case setup
+                    #. Test case logic
+                    #. Test case teardown
 
-                    #. Test suite teardown
-
-                #. DPDK build teardown
+                #. Test suite teardown
 
             #. Test run teardown
 
@@ -414,7 +403,7 @@ def _run_test_run(
     ) -> None:
         """Run the given test run.
 
-        This involves running the test run setup as well as running all DPDK builds
+        This involves running the test run setup as well as running all test suites
         in the given test run. After that, the test run teardown is run.
 
         Args:
@@ -430,6 +419,7 @@ def _run_test_run(
         test_run_result.add_sut_info(sut_node.node_info)
         try:
             sut_node.set_up_test_run(test_run_config)
+            test_run_result.add_dpdk_build_info(sut_node.get_dpdk_build_info())
             tg_node.set_up_test_run(test_run_config)
             test_run_result.update_setup(Result.PASS)
         except Exception as e:
@@ -437,15 +427,7 @@ def _run_test_run(
             test_run_result.update_setup(Result.FAIL, e)
 
         else:
-            for dpdk_build_config in test_run_config.dpdk_builds:
-                dpdk_build_result = test_run_result.add_dpdk_build(dpdk_build_config)
-                self._run_dpdk_build(
-                    sut_node,
-                    tg_node,
-                    dpdk_build_config,
-                    dpdk_build_result,
-                    test_suites_with_cases,
-                )
+            self._run_test_suites(sut_node, tg_node, test_run_result, test_suites_with_cases)
 
         finally:
             try:
@@ -457,82 +439,35 @@ def _run_test_run(
                 self._logger.exception("Test run teardown failed.")
                 test_run_result.update_teardown(Result.FAIL, e)
 
-    def _run_dpdk_build(
-        self,
-        sut_node: SutNode,
-        tg_node: TGNode,
-        dpdk_build_config: DPDKBuildConfiguration,
-        dpdk_build_result: DPDKBuildResult,
-        test_suites_with_cases: Iterable[TestSuiteWithCases],
-    ) -> None:
-        """Run the given DPDK build.
-
-        This involves running the DPDK build setup as well as running all test suites
-        of the DPDK build's test run.
-        After that, DPDK build teardown is run.
-
-        Args:
-            sut_node: The test run's sut node.
-            tg_node: The test run's tg node.
-            dpdk_build_config: A DPDK build's test run configuration.
-            dpdk_build_result: The DPDK build level result object associated
-                with the current DPDK build.
-            test_suites_with_cases: The test suites with test cases to run.
-        """
-        self._logger.set_stage(DtsStage.dpdk_build_setup)
-        self._logger.info(f"Running DPDK build '{dpdk_build_config.name}'.")
-
-        try:
-            sut_node.set_up_dpdk(dpdk_build_config)
-            self._result.dpdk_version = sut_node.dpdk_version
-            dpdk_build_result.add_dpdk_build_info(sut_node.get_dpdk_build_info())
-            dpdk_build_result.update_setup(Result.PASS)
-        except Exception as e:
-            self._logger.exception("DPDK build setup failed.")
-            dpdk_build_result.update_setup(Result.FAIL, e)
-
-        else:
-            self._run_test_suites(sut_node, tg_node, dpdk_build_result, test_suites_with_cases)
-
-        finally:
-            try:
-                self._logger.set_stage(DtsStage.dpdk_build_teardown)
-                sut_node.tear_down_dpdk()
-                dpdk_build_result.update_teardown(Result.PASS)
-            except Exception as e:
-                self._logger.exception("DPDK build teardown failed.")
-                dpdk_build_result.update_teardown(Result.FAIL, e)
-
     def _run_test_suites(
         self,
         sut_node: SutNode,
         tg_node: TGNode,
-        dpdk_build_result: DPDKBuildResult,
+        test_run_result: TestRunResult,
         test_suites_with_cases: Iterable[TestSuiteWithCases],
     ) -> None:
-        """Run `test_suites_with_cases` with the current DPDK build.
+        """Run `test_suites_with_cases` with the current test run.
 
         The method assumes the DPDK we're testing has already been built on the SUT node.
 
         If a blocking test suite (such as the smoke test suite) fails, the rest of the test suites
-        in the current DPDK build won't be executed.
+        in the current test run won't be executed.
 
         Args:
             sut_node: The test run's SUT node.
             tg_node: The test run's TG node.
-            dpdk_build_result: The DPDK build level result object associated
-                with the current DPDK build.
+            test_run_result: The test run's result.
             test_suites_with_cases: The test suites with test cases to run.
         """
         end_dpdk_build = False
         for test_suite_with_cases in test_suites_with_cases:
-            test_suite_result = dpdk_build_result.add_test_suite(test_suite_with_cases)
+            test_suite_result = test_run_result.add_test_suite(test_suite_with_cases)
             try:
                 self._run_test_suite(sut_node, tg_node, test_suite_result, test_suite_with_cases)
             except BlockingTestSuiteError as e:
                 self._logger.exception(
                     f"An error occurred within {test_suite_with_cases.test_suite_class.__name__}. "
-                    "Skipping DPDK build ..."
+                    "Skipping the rest of the test suites in this test run."
                 )
                 self._result.add_error(e)
                 end_dpdk_build = True
diff --git a/dts/framework/test_result.py b/dts/framework/test_result.py
index 95788b7d2e..9e9cd94e33 100644
--- a/dts/framework/test_result.py
+++ b/dts/framework/test_result.py
@@ -8,12 +8,11 @@
 
     * :class:`DTSResult` contains
     * :class:`TestRunResult` contains
-    * :class:`DPDKBuildResult` contains
     * :class:`TestSuiteResult` contains
     * :class:`TestCaseResult`
 
 Each result may contain multiple lower level results, e.g. there are multiple
-:class:`TestSuiteResult`\s in a :class:`DPDKBuildResult`.
+:class:`TestSuiteResult`\s in an :class:`TestRunResult`.
 The results have common parts, such as setup and teardown results, captured in :class:`BaseResult`,
 which also defines some common behaviors in its methods.
 
@@ -35,7 +34,6 @@
     Architecture,
     Compiler,
     CPUType,
-    DPDKBuildConfiguration,
     DPDKBuildInfo,
     NodeInfo,
     TestRunConfiguration,
@@ -138,7 +136,7 @@ class BaseResult:
     Stores the results of the setup and teardown portions of the corresponding stage.
     The hierarchical nature of DTS results is captured recursively in an internal list.
     A stage is each level in this particular hierarchy (pre-run or the top-most level,
-    test run, DPDK build, test suite and test case.)
+    test run, test suite and test case.)
 
     Attributes:
         setup_result: The result of the setup of the particular stage.
@@ -222,8 +220,8 @@ def add_stats(self, statistics: "Statistics") -> None:
 class DTSResult(BaseResult):
     """Stores environment information and test results from a DTS run.
 
-        * Test run level information, such as testbed and the test suite list,
-        * DPDK build level information, such as compiler, target OS and cpu,
+        * Test run level information, such as testbed, compiler, target OS and cpu and
+          the test suite list,
         * Test suite and test case results,
         * All errors that are caught and recorded during DTS execution.
 
@@ -317,44 +315,61 @@ def get_return_code(self) -> int:
 class TestRunResult(BaseResult):
     """The test run specific result.
 
-    The internal list stores the results of all DPDK builds in a given test run.
+    The internal list stores the results of all test suites in a given test run.
 
     Attributes:
+        arch: The DPDK build architecture.
+        os: The DPDK build operating system.
+        cpu: The DPDK build CPU.
+        compiler: The DPDK build compiler.
+        compiler_version: The DPDK build compiler version.
+        dpdk_version: The built DPDK version.
         sut_os_name: The operating system of the SUT node.
         sut_os_version: The operating system version of the SUT node.
         sut_kernel_version: The operating system kernel version of the SUT node.
     """
 
+    arch: Architecture
+    os: OS
+    cpu: CPUType
+    compiler: Compiler
+    compiler_version: str | None
+    dpdk_version: str | None
     sut_os_name: str
     sut_os_version: str
     sut_kernel_version: str
     _config: TestRunConfiguration
-    _parent_result: DTSResult
     _test_suites_with_cases: list[TestSuiteWithCases]
 
     def __init__(self, test_run_config: TestRunConfiguration):
-        """Extend the constructor with the test run's config and DTSResult.
+        """Extend the constructor with the test run's config.
 
         Args:
             test_run_config: A test run configuration.
         """
         super().__init__()
+        self.arch = test_run_config.dpdk_build.arch
+        self.os = test_run_config.dpdk_build.os
+        self.cpu = test_run_config.dpdk_build.cpu
+        self.compiler = test_run_config.dpdk_build.compiler
+        self.compiler_version = None
+        self.dpdk_version = None
         self._config = test_run_config
         self._test_suites_with_cases = []
 
-    def add_dpdk_build(self, dpdk_build_config: DPDKBuildConfiguration) -> "DPDKBuildResult":
-        """Add and return the child result (DPDK build).
+    def add_test_suite(
+        self,
+        test_suite_with_cases: TestSuiteWithCases,
+    ) -> "TestSuiteResult":
+        """Add and return the child result (test suite).
 
         Args:
-            dpdk_build: The DPDK build's test run configuration.
+            test_suite_with_cases: The test suite with test cases.
 
         Returns:
-            The DPDK build's result.
+            The test suite's result.
         """
-        result = DPDKBuildResult(
-            self._test_suites_with_cases,
-            dpdk_build_config,
-        )
+        result = TestSuiteResult(test_suite_with_cases)
         self.child_results.append(result)
         return result
 
@@ -390,71 +405,6 @@ def add_sut_info(self, sut_info: NodeInfo) -> None:
         self.sut_os_version = sut_info.os_version
         self.sut_kernel_version = sut_info.kernel_version
 
-    def _block_result(self) -> None:
-        r"""Mark the result as :attr:`~Result.BLOCK`\ed."""
-        for dpdk_build in self._config.dpdk_builds:
-            child_result = self.add_dpdk_build(dpdk_build)
-            child_result.update_setup(Result.BLOCK)
-
-
-class DPDKBuildResult(BaseResult):
-    """The DPDK build specific result.
-
-    The internal list stores the results of all test suites in a given DPDK build.
-
-    Attributes:
-        arch: The DPDK DPDK build architecture.
-        os: The DPDK DPDK build operating system.
-        cpu: The DPDK DPDK build CPU.
-        compiler: The DPDK DPDK build compiler.
-        compiler_version: The DPDK DPDK build compiler version.
-        dpdk_version: The built DPDK version.
-    """
-
-    arch: Architecture
-    os: OS
-    cpu: CPUType
-    compiler: Compiler
-    compiler_version: str | None
-    dpdk_version: str | None
-    _test_suites_with_cases: list[TestSuiteWithCases]
-
-    def __init__(
-        self,
-        test_suites_with_cases: list[TestSuiteWithCases],
-        dpdk_build_config: DPDKBuildConfiguration,
-    ):
-        """Extend the constructor with the DPDK build's config and test suites with cases.
-
-        Args:
-            test_suites_with_cases: The test suites with test cases to be run in this DPDK build.
-            dpdk_build_config: The DPDK build's test run configuration.
-        """
-        super().__init__()
-        self.arch = dpdk_build_config.arch
-        self.os = dpdk_build_config.os
-        self.cpu = dpdk_build_config.cpu
-        self.compiler = dpdk_build_config.compiler
-        self.compiler_version = None
-        self.dpdk_version = None
-        self._test_suites_with_cases = test_suites_with_cases
-
-    def add_test_suite(
-        self,
-        test_suite_with_cases: TestSuiteWithCases,
-    ) -> "TestSuiteResult":
-        """Add and return the child result (test suite).
-
-        Args:
-            test_suite_with_cases: The test suite with test cases.
-
-        Returns:
-            The test suite's result.
-        """
-        result = TestSuiteResult(test_suite_with_cases)
-        self.child_results.append(result)
-        return result
-
     def add_dpdk_build_info(self, versions: DPDKBuildInfo) -> None:
         """Add information about the DPDK build gathered at runtime.
 
@@ -482,11 +432,10 @@ class TestSuiteResult(BaseResult):
 
     test_suite_name: str
     _test_suite_with_cases: TestSuiteWithCases
-    _parent_result: DPDKBuildResult
     _child_configs: list[str]
 
     def __init__(self, test_suite_with_cases: TestSuiteWithCases):
-        """Extend the constructor with test suite's config and DPDKBuildResult.
+        """Extend the constructor with test suite's config.
 
         Args:
             test_suite_with_cases: The test suite with test cases.
@@ -529,7 +478,7 @@ class TestCaseResult(BaseResult, FixtureResult):
     test_case_name: str
 
     def __init__(self, test_case_name: str):
-        """Extend the constructor with test case's name and TestSuiteResult.
+        """Extend the constructor with test case's name.
 
         Args:
             test_case_name: The test case's name.
diff --git a/dts/framework/test_suite.py b/dts/framework/test_suite.py
index c473b91c5a..80b9fc456d 100644
--- a/dts/framework/test_suite.py
+++ b/dts/framework/test_suite.py
@@ -65,7 +65,7 @@ class TestSuite:
     sut_node: SutNode
     tg_node: TGNode
     #: Whether the test suite is blocking. A failure of a blocking test suite
-    #: will block the execution of all subsequent test suites in the current DPDK build.
+    #: will block the execution of all subsequent test suites in the current test run.
     is_blocking: ClassVar[bool] = False
     _logger: DTSLogger
     _port_links: list[PortLink]
diff --git a/dts/framework/testbed_model/sut_node.py b/dts/framework/testbed_model/sut_node.py
index 6b6fb894ca..9bfb91816e 100644
--- a/dts/framework/testbed_model/sut_node.py
+++ b/dts/framework/testbed_model/sut_node.py
@@ -181,13 +181,15 @@ def set_up_test_run(self, test_run_config: TestRunConfiguration) -> None:
         super().set_up_test_run(test_run_config)
         for vdev in test_run_config.vdevs:
             self.virtual_devices.append(VirtualDevice(vdev))
+        self._set_up_dpdk(test_run_config.dpdk_build)
 
     def tear_down_test_run(self) -> None:
         """Extend the test run teardown with virtual device teardown."""
         super().tear_down_test_run()
         self.virtual_devices = []
+        self._tear_down_dpdk()
 
-    def set_up_dpdk(self, dpdk_build_config: DPDKBuildConfiguration) -> None:
+    def _set_up_dpdk(self, dpdk_build_config: DPDKBuildConfiguration) -> None:
         """Set up DPDK the SUT node and bind ports.
 
         DPDK setup includes setting all internals needed for the build, the copying of DPDK tarball
@@ -202,7 +204,7 @@ def set_up_dpdk(self, dpdk_build_config: DPDKBuildConfiguration) -> None:
         self._build_dpdk()
         self.bind_ports_to_driver()
 
-    def tear_down_dpdk(self) -> None:
+    def _tear_down_dpdk(self) -> None:
         """Reset DPDK variables and bind port driver to the OS driver."""
         self._env_vars = {}
         self._dpdk_build_config = None
diff --git a/dts/tests/TestSuite_smoke_tests.py b/dts/tests/TestSuite_smoke_tests.py
index ac661472b9..99fa8d19c7 100644
--- a/dts/tests/TestSuite_smoke_tests.py
+++ b/dts/tests/TestSuite_smoke_tests.py
@@ -29,7 +29,7 @@ class TestSmokeTests(TestSuite):
 
     Attributes:
         is_blocking: This test suite will block the execution of all other test suites
-            in the DPDK build after it.
+            in the test run after it.
         nics_in_node: The NICs present on the SUT node.
     """
 
-- 
2.43.0