[PATCH v4 03/11] dts: add mechanism to skip test cases or suites

["Juraj Linkeš" <juraj.linkes@pantheon.tech>]

If a test case is not relevant to the testing environment (such as when
a NIC doesn't support a tested feature), the framework should skip it.
The mechanism is a skeleton without actual logic that would set a test
case or suite to be skipped.

The mechanism uses a protocol to extend test suites and test cases with
additional attributes that track whether the test case or suite should
be skipped the reason for skipping it.

Also update the results module with the new SKIP result.

Signed-off-by: Juraj Linkeš <juraj.linkes@pantheon.tech>
Reviewed-by: Dean Marx <dmarx@iol.unh.edu>
---
 dts/framework/runner.py                   | 34 ++++++++---
 dts/framework/test_result.py              | 74 ++++++++++++++---------
 dts/framework/test_suite.py               |  7 ++-
 dts/framework/testbed_model/capability.py | 28 +++++++++
 4 files changed, 105 insertions(+), 38 deletions(-)
 create mode 100644 dts/framework/testbed_model/capability.py

diff --git a/dts/framework/runner.py b/dts/framework/runner.py
index 68482dc9af..57284e510e 100644
--- a/dts/framework/runner.py
+++ b/dts/framework/runner.py
@@ -479,7 +479,20 @@ def _run_test_suites(
         for test_suite_with_cases in test_suites_with_cases:
             test_suite_result = build_target_result.add_test_suite(test_suite_with_cases)
             try:
-                self._run_test_suite(sut_node, tg_node, test_suite_result, test_suite_with_cases)
+                if not test_suite_with_cases.skip:
+                    self._run_test_suite(
+                        sut_node,
+                        tg_node,
+                        test_suite_result,
+                        test_suite_with_cases,
+                    )
+                else:
+                    self._logger.info(
+                        f"Test suite execution SKIPPED: "
+                        f"'{test_suite_with_cases.test_suite_class.__name__}'. Reason: "
+                        f"{test_suite_with_cases.test_suite_class.skip_reason}"
+                    )
+                    test_suite_result.update_setup(Result.SKIP)
             except BlockingTestSuiteError as e:
                 self._logger.exception(
                     f"An error occurred within {test_suite_with_cases.test_suite_class.__name__}. "
@@ -578,14 +591,21 @@ def _execute_test_suite(
             test_case_result = test_suite_result.add_test_case(test_case_name)
             all_attempts = SETTINGS.re_run + 1
             attempt_nr = 1
-            self._run_test_case(test_suite, test_case, test_case_result)
-            while not test_case_result and attempt_nr < all_attempts:
-                attempt_nr += 1
+            if not test_case.skip:
+                self._run_test_case(test_suite, test_case, test_case_result)
+                while not test_case_result and attempt_nr < all_attempts:
+                    attempt_nr += 1
+                    self._logger.info(
+                        f"Re-running FAILED test case '{test_case_name}'. "
+                        f"Attempt number {attempt_nr} out of {all_attempts}."
+                    )
+                    self._run_test_case(test_suite, test_case, test_case_result)
+            else:
                 self._logger.info(
-                    f"Re-running FAILED test case '{test_case_name}'. "
-                    f"Attempt number {attempt_nr} out of {all_attempts}."
+                    f"Test case execution SKIPPED: {test_case_name}. Reason: "
+                    f"{test_case.skip_reason}"
                 )
-                self._run_test_case(test_suite, test_case, test_case_result)
+                test_case_result.update_setup(Result.SKIP)
 
     def _run_test_case(
         self,
diff --git a/dts/framework/test_result.py b/dts/framework/test_result.py
index b1ca584523..50633eee4e 100644
--- a/dts/framework/test_result.py
+++ b/dts/framework/test_result.py
@@ -75,6 +75,15 @@ def create_config(self) -> TestSuiteConfig:
             test_cases=[test_case.__name__ for test_case in self.test_cases],
         )
 
+    @property
+    def skip(self) -> bool:
+        """Skip the test suite if all test cases or the suite itself are to be skipped.
+
+        Returns:
+            :data:`True` if the test suite should be skipped, :data:`False` otherwise.
+        """
+        return all(test_case.skip for test_case in self.test_cases) or self.test_suite_class.skip
+
 
 class Result(Enum):
     """The possible states that a setup, a teardown or a test case may end up in."""
@@ -86,12 +95,12 @@ class Result(Enum):
     #:
     ERROR = auto()
     #:
-    SKIP = auto()
-    #:
     BLOCK = auto()
+    #:
+    SKIP = auto()
 
     def __bool__(self) -> bool:
-        """Only PASS is True."""
+        """Only :attr:`PASS` is True."""
         return self is self.PASS
 
 
@@ -169,14 +178,15 @@ def update_setup(self, result: Result, error: Exception | None = None) -> None:
         self.setup_result.result = result
         self.setup_result.error = error
 
-        if result in [Result.BLOCK, Result.ERROR, Result.FAIL]:
-            self.update_teardown(Result.BLOCK)
-            self._block_result()
+        if result != Result.PASS:
+            result_to_mark = Result.BLOCK if result != Result.SKIP else Result.SKIP
+            self.update_teardown(result_to_mark)
+            self._mark_results(result_to_mark)
 
-    def _block_result(self) -> None:
-        r"""Mark the result as :attr:`~Result.BLOCK`\ed.
+    def _mark_results(self, result) -> None:
+        """Mark the child results or the result of the level itself as `result`.
 
-        The blocking of child results should be done in overloaded methods.
+        The marking of results should be done in overloaded methods.
         """
 
     def update_teardown(self, result: Result, error: Exception | None = None) -> None:
@@ -391,11 +401,11 @@ def add_sut_info(self, sut_info: NodeInfo) -> None:
         self.sut_os_version = sut_info.os_version
         self.sut_kernel_version = sut_info.kernel_version
 
-    def _block_result(self) -> None:
-        r"""Mark the result as :attr:`~Result.BLOCK`\ed."""
+    def _mark_results(self, result) -> None:
+        """Mark the build target results as `result`."""
         for build_target in self._config.build_targets:
             child_result = self.add_build_target(build_target)
-            child_result.update_setup(Result.BLOCK)
+            child_result.update_setup(result)
 
 
 class BuildTargetResult(BaseResult):
@@ -465,11 +475,11 @@ def add_build_target_info(self, versions: BuildTargetInfo) -> None:
         self.compiler_version = versions.compiler_version
         self.dpdk_version = versions.dpdk_version
 
-    def _block_result(self) -> None:
-        r"""Mark the result as :attr:`~Result.BLOCK`\ed."""
+    def _mark_results(self, result) -> None:
+        """Mark the test suite results as `result`."""
         for test_suite_with_cases in self._test_suites_with_cases:
             child_result = self.add_test_suite(test_suite_with_cases)
-            child_result.update_setup(Result.BLOCK)
+            child_result.update_setup(result)
 
 
 class TestSuiteResult(BaseResult):
@@ -509,11 +519,11 @@ def add_test_case(self, test_case_name: str) -> "TestCaseResult":
         self.child_results.append(result)
         return result
 
-    def _block_result(self) -> None:
-        r"""Mark the result as :attr:`~Result.BLOCK`\ed."""
+    def _mark_results(self, result) -> None:
+        """Mark the test case results as `result`."""
         for test_case_method in self._test_suite_with_cases.test_cases:
             child_result = self.add_test_case(test_case_method.__name__)
-            child_result.update_setup(Result.BLOCK)
+            child_result.update_setup(result)
 
 
 class TestCaseResult(BaseResult, FixtureResult):
@@ -567,9 +577,9 @@ def add_stats(self, statistics: "Statistics") -> None:
         """
         statistics += self.result
 
-    def _block_result(self) -> None:
-        r"""Mark the result as :attr:`~Result.BLOCK`\ed."""
-        self.update(Result.BLOCK)
+    def _mark_results(self, result) -> None:
+        r"""Mark the result as `result`."""
+        self.update(result)
 
     def __bool__(self) -> bool:
         """The test case passed only if setup, teardown and the test case itself passed."""
@@ -583,7 +593,8 @@ class Statistics(dict):
 
     The data are stored in the following keys:
 
-    * **PASS RATE** (:class:`int`) -- The FAIL/PASS ratio of all test cases.
+    * **PASS RATE** (:class:`int`) -- The :attr:`~Result.FAIL`/:attr:`~Result.PASS` ratio
+        of all test cases.
     * **DPDK VERSION** (:class:`str`) -- The tested DPDK version.
     """
 
@@ -600,22 +611,27 @@ def __init__(self, dpdk_version: str | None):
         self["DPDK VERSION"] = dpdk_version
 
     def __iadd__(self, other: Result) -> "Statistics":
-        """Add a Result to the final count.
+        """Add a :class:`Result` to the final count.
+
+        :attr:`~Result.SKIP` is not taken into account
 
         Example:
-            stats: Statistics = Statistics()  # empty Statistics
-            stats += Result.PASS  # add a Result to `stats`
+            stats: Statistics = Statistics()  # empty :class:`Statistics`
+            stats += Result.PASS  # add a :class:`Result` to `stats`
 
         Args:
-            other: The Result to add to this statistics object.
+            other: The :class:`Result` to add to this statistics object.
 
         Returns:
             The modified statistics object.
         """
         self[other.name] += 1
-        self["PASS RATE"] = (
-            float(self[Result.PASS.name]) * 100 / sum(self[result.name] for result in Result)
-        )
+        if other != Result.SKIP:
+            self["PASS RATE"] = (
+                float(self[Result.PASS.name])
+                * 100
+                / sum([self[result.name] for result in Result if result != Result.SKIP])
+            )
         return self
 
     def __str__(self) -> str:
diff --git a/dts/framework/test_suite.py b/dts/framework/test_suite.py
index a1fe7f7ebc..5154bb0514 100644
--- a/dts/framework/test_suite.py
+++ b/dts/framework/test_suite.py
@@ -24,6 +24,7 @@
 from scapy.layers.l2 import Ether  # type: ignore[import-untyped]
 from scapy.packet import Packet, Padding, raw  # type: ignore[import-untyped]
 
+from framework.testbed_model.capability import TestProtocol
 from framework.testbed_model.port import Port, PortLink
 from framework.testbed_model.sut_node import SutNode
 from framework.testbed_model.tg_node import TGNode
@@ -36,7 +37,7 @@
 from .utils import get_packet_summaries
 
 
-class TestSuite:
+class TestSuite(TestProtocol):
     """The base class with building blocks needed by most test cases.
 
         * Test suite setup/cleanup methods to override,
@@ -505,7 +506,7 @@ class TestCaseType(Enum):
     PERFORMANCE = auto()
 
 
-class TestCase(Protocol[TestSuiteMethodType]):
+class TestCase(TestProtocol, Protocol[TestSuiteMethodType]):
     """Definition of the test case type for static type checking purposes.
 
     The type is applied to test case functions through a decorator, which casts the decorated
@@ -536,6 +537,8 @@ def make_decorator(
 
         def _decorator(func: TestSuiteMethodType) -> type[TestCase]:
             test_case = cast(type[TestCase], func)
+            test_case.skip = cls.skip
+            test_case.skip_reason = cls.skip_reason
             test_case.test_type = test_case_type
             return test_case
 
diff --git a/dts/framework/testbed_model/capability.py b/dts/framework/testbed_model/capability.py
new file mode 100644
index 0000000000..662f691a0e
--- /dev/null
+++ b/dts/framework/testbed_model/capability.py
@@ -0,0 +1,28 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2024 PANTHEON.tech s.r.o.
+
+"""Testbed capabilities.
+
+This module provides a protocol that defines the common attributes of test cases and suites.
+"""
+
+from collections.abc import Sequence
+from typing import ClassVar, Protocol
+
+
+class TestProtocol(Protocol):
+    """Common test suite and test case attributes."""
+
+    #: Whether to skip the test case or suite.
+    skip: ClassVar[bool] = False
+    #: The reason for skipping the test case or suite.
+    skip_reason: ClassVar[str] = ""
+
+    @classmethod
+    def get_test_cases(cls, test_case_sublist: Sequence[str] | None = None) -> tuple[set, set]:
+        """Get test cases. Should be implemented by subclasses containing test cases.
+
+        Raises:
+            NotImplementedError: The subclass does not implement the method.
+        """
+        raise NotImplementedError()
-- 
2.43.0