[PATCH v4 05/11] dts: add basic capability support

["Juraj Linkeš" <juraj.linkes@pantheon.tech>]

A test case or suite may require certain capabilities to be present in
the tested environment. Add the basic infrastructure for checking the
support status of capabilities:
* The Capability ABC defining the common capability API
* Extension of the TestProtocol with required capabilities (each test
  suite or case stores the capabilities it requires)
* Integration with the runner which calls the new APIs to get which
  capabilities are supported.

Signed-off-by: Juraj Linkeš <juraj.linkes@pantheon.tech>
Reviewed-by: Jeremy Spewock <jspewock@iol.unh.edu>
Reviewed-by: Dean Marx <dmarx@iol.unh.edu>
---
 dts/framework/runner.py                   |  26 +++++
 dts/framework/test_result.py              |  40 +++++++-
 dts/framework/test_suite.py               |   1 +
 dts/framework/testbed_model/capability.py | 117 +++++++++++++++++++++-
 4 files changed, 181 insertions(+), 3 deletions(-)

diff --git a/dts/framework/runner.py b/dts/framework/runner.py
index bf2bfa156f..fff7f1c0a1 100644
--- a/dts/framework/runner.py
+++ b/dts/framework/runner.py
@@ -26,6 +26,7 @@
 from types import MethodType
 from typing import Iterable
 
+from framework.testbed_model.capability import Capability, get_supported_capabilities
 from framework.testbed_model.sut_node import SutNode
 from framework.testbed_model.tg_node import TGNode
 
@@ -454,6 +455,21 @@ def _run_build_target(
                 self._logger.exception("Build target teardown failed.")
                 build_target_result.update_teardown(Result.FAIL, e)
 
+    def _get_supported_capabilities(
+        self,
+        sut_node: SutNode,
+        topology_config: Topology,
+        test_suites_with_cases: Iterable[TestSuiteWithCases],
+    ) -> set[Capability]:
+
+        capabilities_to_check = set()
+        for test_suite_with_cases in test_suites_with_cases:
+            capabilities_to_check.update(test_suite_with_cases.required_capabilities)
+
+        self._logger.debug(f"Found capabilities to check: {capabilities_to_check}")
+
+        return get_supported_capabilities(sut_node, topology_config, capabilities_to_check)
+
     def _run_test_suites(
         self,
         sut_node: SutNode,
@@ -466,6 +482,12 @@ def _run_test_suites(
         The method assumes the build target we're testing has already been built on the SUT node.
         The current build target thus corresponds to the current DPDK build present on the SUT node.
 
+        Before running any suites, the method determines whether they should be skipped
+        by inspecting any required capabilities the test suite needs and comparing those
+        to capabilities supported by the tested environment. If all capabilities are supported,
+        the suite is run. If all test cases in a test suite would be skipped, the whole test suite
+        is skipped (the setup and teardown is not run).
+
         If a blocking test suite (such as the smoke test suite) fails, the rest of the test suites
         in the current build target won't be executed.
 
@@ -478,7 +500,11 @@ def _run_test_suites(
         """
         end_build_target = False
         topology = Topology(sut_node.ports, tg_node.ports)
+        supported_capabilities = self._get_supported_capabilities(
+            sut_node, topology, test_suites_with_cases
+        )
         for test_suite_with_cases in test_suites_with_cases:
+            test_suite_with_cases.mark_skip_unsupported(supported_capabilities)
             test_suite_result = build_target_result.add_test_suite(test_suite_with_cases)
             try:
                 if not test_suite_with_cases.skip:
diff --git a/dts/framework/test_result.py b/dts/framework/test_result.py
index 50633eee4e..40f3fbb77e 100644
--- a/dts/framework/test_result.py
+++ b/dts/framework/test_result.py
@@ -25,10 +25,12 @@
 
 import os.path
 from collections.abc import MutableSequence
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from enum import Enum, auto
 from typing import Union
 
+from framework.testbed_model.capability import Capability
+
 from .config import (
     OS,
     Architecture,
@@ -59,10 +61,18 @@ class is to hold a subset of test cases (which could be all test cases) because
     Attributes:
         test_suite_class: The test suite class.
         test_cases: The test case methods.
+        required_capabilities: The combined required capabilities of both the test suite
+            and the subset of test cases.
     """
 
     test_suite_class: type[TestSuite]
     test_cases: list[type[TestCase]]
+    required_capabilities: set[Capability] = field(default_factory=set, init=False)
+
+    def __post_init__(self):
+        """Gather the required capabilities of the test suite and all test cases."""
+        for test_object in [self.test_suite_class] + self.test_cases:
+            self.required_capabilities.update(test_object.required_capabilities)
 
     def create_config(self) -> TestSuiteConfig:
         """Generate a :class:`TestSuiteConfig` from the stored test suite with test cases.
@@ -75,6 +85,34 @@ def create_config(self) -> TestSuiteConfig:
             test_cases=[test_case.__name__ for test_case in self.test_cases],
         )
 
+    def mark_skip_unsupported(self, supported_capabilities: set[Capability]) -> None:
+        """Mark the test suite and test cases to be skipped.
+
+        The mark is applied if object to be skipped requires any capabilities and at least one of
+        them is not among `supported_capabilities`.
+
+        Args:
+            supported_capabilities: The supported capabilities.
+        """
+        for test_object in [self.test_suite_class, *self.test_cases]:
+            capabilities_not_supported = test_object.required_capabilities - supported_capabilities
+            if capabilities_not_supported:
+                test_object.skip = True
+                capability_str = (
+                    "capability" if len(capabilities_not_supported) == 1 else "capabilities"
+                )
+                test_object.skip_reason = (
+                    f"Required {capability_str} '{capabilities_not_supported}' not found."
+                )
+        if not self.test_suite_class.skip:
+            if all(test_case.skip for test_case in self.test_cases):
+                self.test_suite_class.skip = True
+
+                self.test_suite_class.skip_reason = (
+                    "All test cases are marked to be skipped with reasons: "
+                    f"{' '.join(test_case.skip_reason for test_case in self.test_cases)}"
+                )
+
     @property
     def skip(self) -> bool:
         """Skip the test suite if all test cases or the suite itself are to be skipped.
diff --git a/dts/framework/test_suite.py b/dts/framework/test_suite.py
index 367203f67e..bdc57a6339 100644
--- a/dts/framework/test_suite.py
+++ b/dts/framework/test_suite.py
@@ -527,6 +527,7 @@ def _decorator(func: TestSuiteMethodType) -> type[TestCase]:
             test_case = cast(type[TestCase], func)
             test_case.skip = cls.skip
             test_case.skip_reason = cls.skip_reason
+            test_case.required_capabilities = set()
             test_case.test_type = test_case_type
             return test_case
 
diff --git a/dts/framework/testbed_model/capability.py b/dts/framework/testbed_model/capability.py
index 662f691a0e..8899f07f76 100644
--- a/dts/framework/testbed_model/capability.py
+++ b/dts/framework/testbed_model/capability.py
@@ -3,11 +3,97 @@
 
 """Testbed capabilities.
 
-This module provides a protocol that defines the common attributes of test cases and suites.
+This module provides a protocol that defines the common attributes of test cases and suites
+and support for test environment capabilities.
 """
 
+from abc import ABC, abstractmethod
 from collections.abc import Sequence
-from typing import ClassVar, Protocol
+from typing import Callable, ClassVar, Protocol
+
+from typing_extensions import Self
+
+from .sut_node import SutNode
+from .topology import Topology
+
+
+class Capability(ABC):
+    """The base class for various capabilities.
+
+    The same capability should always be represented by the same object,
+    meaning the same capability required by different test cases or suites
+    should point to the same object.
+
+    Example:
+        ``test_case1`` and ``test_case2`` each require ``capability1``
+        and in both instances, ``capability1`` should point to the same capability object.
+
+    It is up to the subclasses how they implement this.
+
+    The instances are used in sets so they must be hashable.
+    """
+
+    #: A set storing the capabilities whose support should be checked.
+    capabilities_to_check: ClassVar[set[Self]] = set()
+
+    def register_to_check(self) -> Callable[[SutNode, "Topology"], set[Self]]:
+        """Register the capability to be checked for support.
+
+        Returns:
+            The callback function that checks the support of capabilities of the particular subclass
+            which should be called after all capabilities have been registered.
+        """
+        if not type(self).capabilities_to_check:
+            type(self).capabilities_to_check = set()
+        type(self).capabilities_to_check.add(self)
+        return type(self)._get_and_reset
+
+    def add_to_required(self, test_case_or_suite: type["TestProtocol"]) -> None:
+        """Add the capability instance to the required test case or suite's capabilities.
+
+        Args:
+            test_case_or_suite: The test case or suite among whose required capabilities
+                to add this instance.
+        """
+        if not test_case_or_suite.required_capabilities:
+            test_case_or_suite.required_capabilities = set()
+        self._preprocess_required(test_case_or_suite)
+        test_case_or_suite.required_capabilities.add(self)
+
+    def _preprocess_required(self, test_case_or_suite: type["TestProtocol"]) -> None:
+        """An optional method that modifies the required capabilities."""
+
+    @classmethod
+    def _get_and_reset(cls, sut_node: SutNode, topology: "Topology") -> set[Self]:
+        """The callback method to be called after all capabilities have been registered.
+
+        Not only does this method check the support of capabilities,
+        but it also reset the internal set of registered capabilities
+        so that the "register, then get support" workflow works in subsequent test runs.
+        """
+        supported_capabilities = cls.get_supported_capabilities(sut_node, topology)
+        cls.capabilities_to_check = set()
+        return supported_capabilities
+
+    @classmethod
+    @abstractmethod
+    def get_supported_capabilities(cls, sut_node: SutNode, topology: "Topology") -> set[Self]:
+        """Get the support status of each registered capability.
+
+        Each subclass must implement this method and return the subset of supported capabilities
+        of :attr:`capabilities_to_check`.
+
+        Args:
+            sut_node: The SUT node of the current test run.
+            topology: The topology of the current test run.
+
+        Returns:
+            The supported capabilities.
+        """
+
+    @abstractmethod
+    def __hash__(self) -> int:
+        """The subclasses must be hashable so that they can be stored in sets."""
 
 
 class TestProtocol(Protocol):
@@ -17,6 +103,8 @@ class TestProtocol(Protocol):
     skip: ClassVar[bool] = False
     #: The reason for skipping the test case or suite.
     skip_reason: ClassVar[str] = ""
+    #: The capabilities the test case or suite requires in order to be executed.
+    required_capabilities: ClassVar[set[Capability]] = set()
 
     @classmethod
     def get_test_cases(cls, test_case_sublist: Sequence[str] | None = None) -> tuple[set, set]:
@@ -26,3 +114,28 @@ def get_test_cases(cls, test_case_sublist: Sequence[str] | None = None) -> tuple
             NotImplementedError: The subclass does not implement the method.
         """
         raise NotImplementedError()
+
+
+def get_supported_capabilities(
+    sut_node: SutNode,
+    topology_config: Topology,
+    capabilities_to_check: set[Capability],
+) -> set[Capability]:
+    """Probe the environment for `capabilities_to_check` and return the supported ones.
+
+    Args:
+        sut_node: The SUT node to check for capabilities.
+        topology_config: The topology config to check for capabilities.
+        capabilities_to_check: The capabilities to check.
+
+    Returns:
+        The capabilities supported by the environment.
+    """
+    callbacks = set()
+    for capability_to_check in capabilities_to_check:
+        callbacks.add(capability_to_check.register_to_check())
+    supported_capabilities = set()
+    for callback in callbacks:
+        supported_capabilities.update(callback(sut_node, topology_config))
+
+    return supported_capabilities
-- 
2.43.0