Re: [PATCH v2 3/7] dts: filter test suites in executions

[Jeremy Spewock <jspewock@iol.unh.edu>]

On Tue, Feb 6, 2024 at 9:57 AM Juraj Linkeš <juraj.linkes@pantheon.tech> wrote:
>
> We're currently filtering which test cases to run after some setup
> steps, such as DPDK build, have already been taken. This prohibits us to
> mark the test suites and cases that were supposed to be run as blocked
> when an earlier setup fails, as that information is not available at
> that time.
>
> To remedy this, move the filtering to the beginning of each execution.
> This is the first action taken in each execution and if we can't filter
> the test cases, such as due to invalid inputs, we abort the whole
> execution. No test suites nor cases will be marked as blocked as we
> don't know which were supposed to be run.
>
> On top of that, the filtering takes place in the TestSuite class, which
> should only concern itself with test suite and test case logic, not the
> processing behind the scenes. The logic has been moved to DTSRunner
> which should do all the processing needed to run test suites.
>
> The filtering itself introduces a few changes/assumptions which are more
> sensible than before:
> 1. Assumption: There is just one TestSuite child class in each test
>    suite module. This was an implicit assumption before as we couldn't
>    specify the TestSuite classes in the test run configuration, just the
>    modules. The name of the TestSuite child class starts with "Test" and
>    then corresponds to the name of the module with CamelCase naming.
> 2. Unknown test cases specified both in the test run configuration and
>    the environment variable/command line argument are no longer silently
>    ignored. This is a quality of life improvement for users, as they
>    could easily be not aware of the silent ignoration.
>
> Also, a change in the code results in pycodestyle warning and error:
> [E] E203 whitespace before ':'
> [W] W503 line break before binary operator
>
> These two are not PEP8 compliant, so they're disabled.
>
> Signed-off-by: Juraj Linkeš <juraj.linkes@pantheon.tech>
> ---
>  dts/framework/config/__init__.py           |  24 +-
>  dts/framework/config/conf_yaml_schema.json |   2 +-
>  dts/framework/runner.py                    | 426 +++++++++++++++------
>  dts/framework/settings.py                  |   3 +-
>  dts/framework/test_result.py               |  34 ++
>  dts/framework/test_suite.py                |  85 +---
>  dts/pyproject.toml                         |   3 +
>  dts/tests/TestSuite_smoke_tests.py         |   2 +-
>  8 files changed, 382 insertions(+), 197 deletions(-)
>
> diff --git a/dts/framework/config/__init__.py b/dts/framework/config/__init__.py
> index 62eded7f04..c6a93b3b89 100644
> --- a/dts/framework/config/__init__.py
> +++ b/dts/framework/config/__init__.py
> @@ -36,7 +36,7 @@
>  import json
>  import os.path
>  import pathlib
> -from dataclasses import dataclass
> +from dataclasses import dataclass, fields
>  from enum import auto, unique
>  from typing import Union
>
> @@ -506,6 +506,28 @@ def from_dict(
>              vdevs=vdevs,
>          )
>
> +    def copy_and_modify(self, **kwargs) -> "ExecutionConfiguration":
> +        """Create a shallow copy with any of the fields modified.
> +
> +        The only new data are those passed to this method.
> +        The rest are copied from the object's fields calling the method.
> +
> +        Args:
> +            **kwargs: The names and types of keyword arguments are defined
> +                by the fields of the :class:`ExecutionConfiguration` class.
> +
> +        Returns:
> +            The copied and modified execution configuration.
> +        """
> +        new_config = {}
> +        for field in fields(self):
> +            if field.name in kwargs:
> +                new_config[field.name] = kwargs[field.name]
> +            else:
> +                new_config[field.name] = getattr(self, field.name)
> +
> +        return ExecutionConfiguration(**new_config)
> +
>
>  @dataclass(slots=True, frozen=True)
>  class Configuration:
> diff --git a/dts/framework/config/conf_yaml_schema.json b/dts/framework/config/conf_yaml_schema.json
> index 84e45fe3c2..051b079fe4 100644
> --- a/dts/framework/config/conf_yaml_schema.json
> +++ b/dts/framework/config/conf_yaml_schema.json
> @@ -197,7 +197,7 @@
>          },
>          "cases": {
>            "type": "array",
> -          "description": "If specified, only this subset of test suite's test cases will be run. Unknown test cases will be silently ignored.",
> +          "description": "If specified, only this subset of test suite's test cases will be run.",
>            "items": {
>              "type": "string"
>            },
> diff --git a/dts/framework/runner.py b/dts/framework/runner.py
> index 933685d638..3e95cf9e26 100644
> --- a/dts/framework/runner.py
> +++ b/dts/framework/runner.py
> @@ -17,17 +17,27 @@
>  and the test case stage runs test cases individually.
>  """
>
> +import importlib
> +import inspect
>  import logging
> +import re
>  import sys
>  from types import MethodType
> +from typing import Iterable
>
>  from .config import (
>      BuildTargetConfiguration,
> +    Configuration,
>      ExecutionConfiguration,
>      TestSuiteConfig,
>      load_config,
>  )
> -from .exception import BlockingTestSuiteError, SSHTimeoutError, TestCaseVerifyError
> +from .exception import (
> +    BlockingTestSuiteError,
> +    ConfigurationError,
> +    SSHTimeoutError,
> +    TestCaseVerifyError,
> +)
>  from .logger import DTSLOG, getLogger
>  from .settings import SETTINGS
>  from .test_result import (
> @@ -37,8 +47,9 @@
>      Result,
>      TestCaseResult,
>      TestSuiteResult,
> +    TestSuiteWithCases,
>  )
> -from .test_suite import TestSuite, get_test_suites
> +from .test_suite import TestSuite
>  from .testbed_model import SutNode, TGNode
>
>
> @@ -59,13 +70,23 @@ class DTSRunner:
>          given execution, the next execution begins.
>      """
>
> +    _configuration: Configuration
>      _logger: DTSLOG
>      _result: DTSResult
> +    _test_suite_class_prefix: str
> +    _test_suite_module_prefix: str
> +    _func_test_case_regex: str
> +    _perf_test_case_regex: str
>
>      def __init__(self):
> -        """Initialize the instance with logger and result."""
> +        """Initialize the instance with configuration, logger, result and string constants."""
> +        self._configuration = load_config()
>          self._logger = getLogger("DTSRunner")
>          self._result = DTSResult(self._logger)
> +        self._test_suite_class_prefix = "Test"
> +        self._test_suite_module_prefix = "tests.TestSuite_"
> +        self._func_test_case_regex = r"test_(?!perf_)"
> +        self._perf_test_case_regex = r"test_perf_"
>
>      def run(self):
>          """Run all build targets in all executions from the test run configuration.
> @@ -106,29 +127,28 @@ def run(self):
>          try:
>              # check the python version of the server that runs dts
>              self._check_dts_python_version()
> +            self._result.update_setup(Result.PASS)
>
>              # for all Execution sections
> -            for execution in load_config().executions:
> -                sut_node = sut_nodes.get(execution.system_under_test_node.name)
> -                tg_node = tg_nodes.get(execution.traffic_generator_node.name)
> -
> +            for execution in self._configuration.executions:
> +                self._logger.info(
> +                    f"Running execution with SUT '{execution.system_under_test_node.name}'."
> +                )
> +                execution_result = self._result.add_execution(execution.system_under_test_node)
>                  try:
> -                    if not sut_node:
> -                        sut_node = SutNode(execution.system_under_test_node)
> -                        sut_nodes[sut_node.name] = sut_node
> -                    if not tg_node:
> -                        tg_node = TGNode(execution.traffic_generator_node)
> -                        tg_nodes[tg_node.name] = tg_node
> -                    self._result.update_setup(Result.PASS)
> +                    test_suites_with_cases = self._get_test_suites_with_cases(
> +                        execution.test_suites, execution.func, execution.perf
> +                    )
>                  except Exception as e:
> -                    failed_node = execution.system_under_test_node.name
> -                    if sut_node:
> -                        failed_node = execution.traffic_generator_node.name
> -                    self._logger.exception(f"The Creation of node {failed_node} failed.")
> -                    self._result.update_setup(Result.FAIL, e)
> +                    self._logger.exception(
> +                        f"Invalid test suite configuration found: " f"{execution.test_suites}."
> +                    )
> +                    execution_result.update_setup(Result.FAIL, e)
>
>                  else:
> -                    self._run_execution(sut_node, tg_node, execution)
> +                    self._connect_nodes_and_run_execution(
> +                        sut_nodes, tg_nodes, execution, execution_result, test_suites_with_cases
> +                    )
>
>          except Exception as e:
>              self._logger.exception("An unexpected error has occurred.")
> @@ -163,11 +183,204 @@ def _check_dts_python_version(self) -> None:
>              )
>              self._logger.warning("Please use Python >= 3.10 instead.")
>
> +    def _get_test_suites_with_cases(
> +        self,
> +        test_suite_configs: list[TestSuiteConfig],
> +        func: bool,
> +        perf: bool,
> +    ) -> list[TestSuiteWithCases]:
> +        """Test suites with test cases discovery.
> +
> +        The test suites with test cases defined in the user configuration are discovered
> +        and stored for future use so that we don't import the modules twice and so that
> +        the list of test suites with test cases is available for recording right away.
> +
> +        Args:
> +            test_suite_configs: Test suite configurations.
> +            func: Whether to include functional test cases in the final list.
> +            perf: Whether to include performance test cases in the final list.
> +
> +        Returns:
> +            The discovered test suites, each with test cases.
> +        """
> +        test_suites_with_cases = []
> +
> +        for test_suite_config in test_suite_configs:
> +            test_suite_class = self._get_test_suite_class(test_suite_config.test_suite)
> +            test_cases = []
> +            func_test_cases, perf_test_cases = self._filter_test_cases(
> +                test_suite_class, set(test_suite_config.test_cases + SETTINGS.test_cases)
> +            )
> +            if func:
> +                test_cases.extend(func_test_cases)
> +            if perf:
> +                test_cases.extend(perf_test_cases)
> +
> +            test_suites_with_cases.append(
> +                TestSuiteWithCases(test_suite_class=test_suite_class, test_cases=test_cases)
> +            )
> +
> +        return test_suites_with_cases
> +
> +    def _get_test_suite_class(self, test_suite_name: str) -> type[TestSuite]:
> +        """Find the :class:`TestSuite` class with `test_suite_name` in the corresponding module.
> +
> +        The method assumes that the :class:`TestSuite` class starts
> +        with `self._test_suite_class_prefix`,
> +        continuing with `test_suite_name` with CamelCase convention.
> +        It also assumes there's only one test suite in each module and the module name
> +        is `test_suite_name` prefixed with `self._test_suite_module_prefix`.
> +
> +        The CamelCase convention is not tested, only lowercase strings are compared.
> +
> +        Args:
> +            test_suite_name: The name of the test suite to find.
> +
> +        Returns:
> +            The found test suite.
> +
> +        Raises:
> +            ConfigurationError: If the corresponding module is not found or
> +                a valid :class:`TestSuite` is not found in the module.
> +        """
> +
> +        def is_test_suite(object) -> bool:
> +            """Check whether `object` is a :class:`TestSuite`.
> +
> +            The `object` is a subclass of :class:`TestSuite`, but not :class:`TestSuite` itself.
> +
> +            Args:
> +                object: The object to be checked.
> +
> +            Returns:
> +                :data:`True` if `object` is a subclass of `TestSuite`.
> +            """
> +            try:
> +                if issubclass(object, TestSuite) and object is not TestSuite:
> +                    return True
> +            except TypeError:
> +                return False
> +            return False
> +
> +        testsuite_module_path = f"{self._test_suite_module_prefix}{test_suite_name}"
> +        try:
> +            test_suite_module = importlib.import_module(testsuite_module_path)
> +        except ModuleNotFoundError as e:
> +            raise ConfigurationError(
> +                f"Test suite module '{testsuite_module_path}' not found."
> +            ) from e
> +
> +        lowercase_suite_name = test_suite_name.replace("_", "").lower()
> +        for class_name, class_obj in inspect.getmembers(test_suite_module, is_test_suite):
> +            if (
> +                class_name.startswith(self._test_suite_class_prefix)
> +                and lowercase_suite_name == class_name[len(self._test_suite_class_prefix) :].lower()
> +            ):

Would it be simpler to instead just make lowercase_suite_name =
f"{self._test_suite_class_prefix}{test_suite_name.replace("_",
"").lower()}" so that you can just directly compare class_name ==
lowercase_suite_name? Both ways should have the exact same result of
course so it isn't important, I was just curious.

> +                return class_obj
> +        raise ConfigurationError(
> +            f"Couldn't find any valid test suites in {test_suite_module.__name__}."
> +        )
> +
> +    def _filter_test_cases(
> +        self, test_suite_class: type[TestSuite], test_cases_to_run: set[str]
> +    ) -> tuple[list[MethodType], list[MethodType]]:
> +        """Filter `test_cases_to_run` from `test_suite_class`.
> +
> +        There are two rounds of filtering if `test_cases_to_run` is not empty.
> +        The first filters `test_cases_to_run` from all methods of `test_suite_class`.
> +        Then the methods are separated into functional and performance test cases.
> +        If a method doesn't match neither the functional nor performance name prefix, it's an error.

I think this is a double negative but could be either "if a method
doesn't match either ... or ..." or "if a method matches neither ...
nor ...". I have a small preference to the second of the two options
though because the "neither" makes the negative more clear in my mind.

> +
> +        Args:
> +            test_suite_class: The class of the test suite.
> +            test_cases_to_run: Test case names to filter from `test_suite_class`.
> +                If empty, return all matching test cases.
> +
> +        Returns:
> +            A list of test case methods that should be executed.
> +
> +        Raises:
> +            ConfigurationError: If a test case from `test_cases_to_run` is not found
> +                or it doesn't match either the functional nor performance name prefix.
> +        """
> +        func_test_cases = []
> +        perf_test_cases = []
> +        name_method_tuples = inspect.getmembers(test_suite_class, inspect.isfunction)
> +        if test_cases_to_run:
> +            name_method_tuples = [
> +                (name, method) for name, method in name_method_tuples if name in test_cases_to_run
> +            ]
> +            if len(name_method_tuples) < len(test_cases_to_run):
> +                missing_test_cases = test_cases_to_run - {name for name, _ in name_method_tuples}
> +                raise ConfigurationError(
> +                    f"Test cases {missing_test_cases} not found among methods "
> +                    f"of {test_suite_class.__name__}."
> +                )
> +
> +        for test_case_name, test_case_method in name_method_tuples:
> +            if re.match(self._func_test_case_regex, test_case_name):
> +                func_test_cases.append(test_case_method)
> +            elif re.match(self._perf_test_case_regex, test_case_name):
> +                perf_test_cases.append(test_case_method)
> +            elif test_cases_to_run:
> +                raise ConfigurationError(
> +                    f"Method '{test_case_name}' doesn't match neither "
> +                    f"a functional nor a performance test case name."

Same thing here with the double negative.



> +                )
> +
> +        return func_test_cases, perf_test_cases
> +
> +    def _connect_nodes_and_run_execution(
> +        self,
> +        sut_nodes: dict[str, SutNode],
> +        tg_nodes: dict[str, TGNode],
> +        execution: ExecutionConfiguration,
> +        execution_result: ExecutionResult,
> +        test_suites_with_cases: Iterable[TestSuiteWithCases],
> +    ) -> None:
> +        """Connect nodes, then continue to run the given execution.
> +
> +        Connect the :class:`SutNode` and the :class:`TGNode` of this `execution`.
> +        If either has already been connected, it's going to be in either `sut_nodes` or `tg_nodes`,
> +        respectively.
> +        If not, connect and add the node to the respective `sut_nodes` or `tg_nodes` :class:`dict`.
> +
> +        Args:
> +            sut_nodes: A dictionary storing connected/to be connected SUT nodes.
> +            tg_nodes: A dictionary storing connected/to be connected TG nodes.
> +            execution: An execution's test run configuration.
> +            execution_result: The execution's result.
> +            test_suites_with_cases: The test suites with test cases to run.
> +        """
> +        sut_node = sut_nodes.get(execution.system_under_test_node.name)
> +        tg_node = tg_nodes.get(execution.traffic_generator_node.name)
> +
> +        try:
> +            if not sut_node:
> +                sut_node = SutNode(execution.system_under_test_node)
> +                sut_nodes[sut_node.name] = sut_node
> +            if not tg_node:
> +                tg_node = TGNode(execution.traffic_generator_node)
> +                tg_nodes[tg_node.name] = tg_node
> +        except Exception as e:
> +            failed_node = execution.system_under_test_node.name
> +            if sut_node:
> +                failed_node = execution.traffic_generator_node.name
> +            self._logger.exception(f"The Creation of node {failed_node} failed.")
> +            execution_result.update_setup(Result.FAIL, e)
> +
> +        else:
> +            self._run_execution(
> +                sut_node, tg_node, execution, execution_result, test_suites_with_cases
> +            )
> +
>      def _run_execution(
>          self,
>          sut_node: SutNode,
>          tg_node: TGNode,
>          execution: ExecutionConfiguration,
> +        execution_result: ExecutionResult,
> +        test_suites_with_cases: Iterable[TestSuiteWithCases],
>      ) -> None:
>          """Run the given execution.
>
> @@ -178,11 +391,11 @@ def _run_execution(
>              sut_node: The execution's SUT node.
>              tg_node: The execution's TG node.
>              execution: An execution's test run configuration.
> +            execution_result: The execution's result.
> +            test_suites_with_cases: The test suites with test cases to run.
>          """
>          self._logger.info(f"Running execution with SUT '{execution.system_under_test_node.name}'.")
> -        execution_result = self._result.add_execution(sut_node.config)
>          execution_result.add_sut_info(sut_node.node_info)
> -
>          try:
>              sut_node.set_up_execution(execution)
>              execution_result.update_setup(Result.PASS)
> @@ -192,7 +405,10 @@ def _run_execution(
>
>          else:
>              for build_target in execution.build_targets:
> -                self._run_build_target(sut_node, tg_node, build_target, execution, execution_result)
> +                build_target_result = execution_result.add_build_target(build_target)
> +                self._run_build_target(
> +                    sut_node, tg_node, build_target, build_target_result, test_suites_with_cases
> +                )
>
>          finally:
>              try:
> @@ -207,8 +423,8 @@ def _run_build_target(
>          sut_node: SutNode,
>          tg_node: TGNode,
>          build_target: BuildTargetConfiguration,
> -        execution: ExecutionConfiguration,
> -        execution_result: ExecutionResult,
> +        build_target_result: BuildTargetResult,
> +        test_suites_with_cases: Iterable[TestSuiteWithCases],
>      ) -> None:
>          """Run the given build target.
>
> @@ -220,11 +436,11 @@ def _run_build_target(
>              sut_node: The execution's sut node.
>              tg_node: The execution's tg node.
>              build_target: A build target's test run configuration.
> -            execution: The build target's execution's test run configuration.
> -            execution_result: The execution level result object associated with the execution.
> +            build_target_result: The build target level result object associated
> +                with the current build target.
> +            test_suites_with_cases: The test suites with test cases to run.
>          """
>          self._logger.info(f"Running build target '{build_target.name}'.")
> -        build_target_result = execution_result.add_build_target(build_target)
>
>          try:
>              sut_node.set_up_build_target(build_target)
> @@ -236,7 +452,7 @@ def _run_build_target(
>              build_target_result.update_setup(Result.FAIL, e)
>
>          else:
> -            self._run_test_suites(sut_node, tg_node, execution, build_target_result)
> +            self._run_test_suites(sut_node, tg_node, build_target_result, test_suites_with_cases)
>
>          finally:
>              try:
> @@ -250,10 +466,10 @@ def _run_test_suites(
>          self,
>          sut_node: SutNode,
>          tg_node: TGNode,
> -        execution: ExecutionConfiguration,
>          build_target_result: BuildTargetResult,
> +        test_suites_with_cases: Iterable[TestSuiteWithCases],
>      ) -> None:
> -        """Run the execution's (possibly a subset of) test suites using the current build target.
> +        """Run `test_suites_with_cases` with the current build target.
>
>          The method assumes the build target we're testing has already been built on the SUT node.
>          The current build target thus corresponds to the current DPDK build present on the SUT node.
> @@ -264,22 +480,20 @@ def _run_test_suites(
>          Args:
>              sut_node: The execution's SUT node.
>              tg_node: The execution's TG node.
> -            execution: The execution's test run configuration associated
> -                with the current build target.
>              build_target_result: The build target level result object associated
>                  with the current build target.
> +            test_suites_with_cases: The test suites with test cases to run.
>          """
>          end_build_target = False
> -        if not execution.skip_smoke_tests:
> -            execution.test_suites[:0] = [TestSuiteConfig.from_dict("smoke_tests")]
> -        for test_suite_config in execution.test_suites:
> +        for test_suite_with_cases in test_suites_with_cases:
> +            test_suite_result = build_target_result.add_test_suite(
> +                test_suite_with_cases.test_suite_class.__name__
> +            )
>              try:
> -                self._run_test_suite_module(
> -                    sut_node, tg_node, execution, build_target_result, test_suite_config
> -                )
> +                self._run_test_suite(sut_node, tg_node, test_suite_result, test_suite_with_cases)
>              except BlockingTestSuiteError as e:
>                  self._logger.exception(
> -                    f"An error occurred within {test_suite_config.test_suite}. "
> +                    f"An error occurred within {test_suite_with_cases.test_suite_class.__name__}. "
>                      "Skipping build target..."
>                  )
>                  self._result.add_error(e)
> @@ -288,15 +502,14 @@ def _run_test_suites(
>              if end_build_target:
>                  break
>
> -    def _run_test_suite_module(
> +    def _run_test_suite(
>          self,
>          sut_node: SutNode,
>          tg_node: TGNode,
> -        execution: ExecutionConfiguration,
> -        build_target_result: BuildTargetResult,
> -        test_suite_config: TestSuiteConfig,
> +        test_suite_result: TestSuiteResult,
> +        test_suite_with_cases: TestSuiteWithCases,
>      ) -> None:
> -        """Set up, execute and tear down all test suites in a single test suite module.
> +        """Set up, execute and tear down `test_suite_with_cases`.
>
>          The method assumes the build target we're testing has already been built on the SUT node.
>          The current build target thus corresponds to the current DPDK build present on the SUT node.
> @@ -306,92 +519,79 @@ def _run_test_suite_module(
>
>          Record the setup and the teardown and handle failures.
>
> -        The test cases to execute are discovered when creating the :class:`TestSuite` object.
> -
>          Args:
>              sut_node: The execution's SUT node.
>              tg_node: The execution's TG node.
> -            execution: The execution's test run configuration associated
> -                with the current build target.
> -            build_target_result: The build target level result object associated
> -                with the current build target.
> -            test_suite_config: Test suite test run configuration specifying the test suite module
> -                and possibly a subset of test cases of test suites in that module.
> +            test_suite_result: The test suite level result object associated
> +                with the current test suite.
> +            test_suite_with_cases: The test suite with test cases to run.
>
>          Raises:
>              BlockingTestSuiteError: If a blocking test suite fails.
>          """
> +        test_suite_name = test_suite_with_cases.test_suite_class.__name__
> +        test_suite = test_suite_with_cases.test_suite_class(sut_node, tg_node)
>          try:
> -            full_suite_path = f"tests.TestSuite_{test_suite_config.test_suite}"
> -            test_suite_classes = get_test_suites(full_suite_path)
> -            suites_str = ", ".join((x.__name__ for x in test_suite_classes))
> -            self._logger.debug(f"Found test suites '{suites_str}' in '{full_suite_path}'.")
> +            self._logger.info(f"Starting test suite setup: {test_suite_name}")
> +            test_suite.set_up_suite()
> +            test_suite_result.update_setup(Result.PASS)
> +            self._logger.info(f"Test suite setup successful: {test_suite_name}")
>          except Exception as e:
> -            self._logger.exception("An error occurred when searching for test suites.")
> -            self._result.update_setup(Result.ERROR, e)
> +            self._logger.exception(f"Test suite setup ERROR: {test_suite_name}")
> +            test_suite_result.update_setup(Result.ERROR, e)
>
>          else:
> -            for test_suite_class in test_suite_classes:
> -                test_suite = test_suite_class(sut_node, tg_node, test_suite_config.test_cases)
> -
> -                test_suite_name = test_suite.__class__.__name__
> -                test_suite_result = build_target_result.add_test_suite(test_suite_name)
> -                try:
> -                    self._logger.info(f"Starting test suite setup: {test_suite_name}")
> -                    test_suite.set_up_suite()
> -                    test_suite_result.update_setup(Result.PASS)
> -                    self._logger.info(f"Test suite setup successful: {test_suite_name}")
> -                except Exception as e:
> -                    self._logger.exception(f"Test suite setup ERROR: {test_suite_name}")
> -                    test_suite_result.update_setup(Result.ERROR, e)
> -
> -                else:
> -                    self._execute_test_suite(execution.func, test_suite, test_suite_result)
> -
> -                finally:
> -                    try:
> -                        test_suite.tear_down_suite()
> -                        sut_node.kill_cleanup_dpdk_apps()
> -                        test_suite_result.update_teardown(Result.PASS)
> -                    except Exception as e:
> -                        self._logger.exception(f"Test suite teardown ERROR: {test_suite_name}")
> -                        self._logger.warning(
> -                            f"Test suite '{test_suite_name}' teardown failed, "
> -                            f"the next test suite may be affected."
> -                        )
> -                        test_suite_result.update_setup(Result.ERROR, e)
> -                    if len(test_suite_result.get_errors()) > 0 and test_suite.is_blocking:
> -                        raise BlockingTestSuiteError(test_suite_name)
> +            self._execute_test_suite(
> +                test_suite,
> +                test_suite_with_cases.test_cases,
> +                test_suite_result,
> +            )
> +        finally:
> +            try:
> +                test_suite.tear_down_suite()
> +                sut_node.kill_cleanup_dpdk_apps()
> +                test_suite_result.update_teardown(Result.PASS)
> +            except Exception as e:
> +                self._logger.exception(f"Test suite teardown ERROR: {test_suite_name}")
> +                self._logger.warning(
> +                    f"Test suite '{test_suite_name}' teardown failed, "
> +                    "the next test suite may be affected."
> +                )
> +                test_suite_result.update_setup(Result.ERROR, e)
> +            if len(test_suite_result.get_errors()) > 0 and test_suite.is_blocking:
> +                raise BlockingTestSuiteError(test_suite_name)
>
>      def _execute_test_suite(
> -        self, func: bool, test_suite: TestSuite, test_suite_result: TestSuiteResult
> +        self,
> +        test_suite: TestSuite,
> +        test_cases: Iterable[MethodType],
> +        test_suite_result: TestSuiteResult,
>      ) -> None:
> -        """Execute all discovered test cases in `test_suite`.
> +        """Execute all `test_cases` in `test_suite`.
>
>          If the :option:`--re-run` command line argument or the :envvar:`DTS_RERUN` environment
>          variable is set, in case of a test case failure, the test case will be executed again
>          until it passes or it fails that many times in addition of the first failure.
>
>          Args:
> -            func: Whether to execute functional test cases.
>              test_suite: The test suite object.
> +            test_cases: The list of test case methods.
>              test_suite_result: The test suite level result object associated
>                  with the current test suite.
>          """
> -        if func:
> -            for test_case_method in test_suite._get_functional_test_cases():
> -                test_case_name = test_case_method.__name__
> -                test_case_result = test_suite_result.add_test_case(test_case_name)
> -                all_attempts = SETTINGS.re_run + 1
> -                attempt_nr = 1
> +        for test_case_method in test_cases:
> +            test_case_name = test_case_method.__name__
> +            test_case_result = test_suite_result.add_test_case(test_case_name)
> +            all_attempts = SETTINGS.re_run + 1
> +            attempt_nr = 1
> +            self._run_test_case(test_suite, test_case_method, test_case_result)
> +            while not test_case_result and attempt_nr < all_attempts:
> +                attempt_nr += 1
> +                self._logger.info(
> +                    f"Re-running FAILED test case '{test_case_name}'. "
> +                    f"Attempt number {attempt_nr} out of {all_attempts}."
> +                )
>                  self._run_test_case(test_suite, test_case_method, test_case_result)
> -                while not test_case_result and attempt_nr < all_attempts:
> -                    attempt_nr += 1
> -                    self._logger.info(
> -                        f"Re-running FAILED test case '{test_case_name}'. "
> -                        f"Attempt number {attempt_nr} out of {all_attempts}."
> -                    )
> -                    self._run_test_case(test_suite, test_case_method, test_case_result)
>
>      def _run_test_case(
>          self,
> @@ -399,7 +599,7 @@ def _run_test_case(
>          test_case_method: MethodType,
>          test_case_result: TestCaseResult,
>      ) -> None:
> -        """Setup, execute and teardown a test case in `test_suite`.
> +        """Setup, execute and teardown `test_case_method` from `test_suite`.
>
>          Record the result of the setup and the teardown and handle failures.
>
> @@ -424,7 +624,7 @@ def _run_test_case(
>
>          else:
>              # run test case if setup was successful
> -            self._execute_test_case(test_case_method, test_case_result)
> +            self._execute_test_case(test_suite, test_case_method, test_case_result)
>
>          finally:
>              try:
> @@ -440,11 +640,15 @@ def _run_test_case(
>                  test_case_result.update(Result.ERROR)
>
>      def _execute_test_case(
> -        self, test_case_method: MethodType, test_case_result: TestCaseResult
> +        self,
> +        test_suite: TestSuite,
> +        test_case_method: MethodType,
> +        test_case_result: TestCaseResult,
>      ) -> None:
> -        """Execute one test case, record the result and handle failures.
> +        """Execute `test_case_method` from `test_suite`, record the result and handle failures.
>
>          Args:
> +            test_suite: The test suite object.
>              test_case_method: The test case method.
>              test_case_result: The test case level result object associated
>                  with the current test case.
> @@ -452,7 +656,7 @@ def _execute_test_case(
>          test_case_name = test_case_method.__name__
>          try:
>              self._logger.info(f"Starting test case execution: {test_case_name}")
> -            test_case_method()
> +            test_case_method(test_suite)
>              test_case_result.update(Result.PASS)
>              self._logger.info(f"Test case execution PASSED: {test_case_name}")
>
> diff --git a/dts/framework/settings.py b/dts/framework/settings.py
> index 609c8d0e62..2b8bfbe0ed 100644
> --- a/dts/framework/settings.py
> +++ b/dts/framework/settings.py
> @@ -253,8 +253,7 @@ def _get_parser() -> argparse.ArgumentParser:
>          "--test-cases",
>          action=_env_arg("DTS_TESTCASES"),
>          default="",
> -        help="[DTS_TESTCASES] Comma-separated list of test cases to execute. "
> -        "Unknown test cases will be silently ignored.",
> +        help="[DTS_TESTCASES] Comma-separated list of test cases to execute.",
>      )
>
>      parser.add_argument(
> diff --git a/dts/framework/test_result.py b/dts/framework/test_result.py
> index 4467749a9d..075195fd5b 100644
> --- a/dts/framework/test_result.py
> +++ b/dts/framework/test_result.py
> @@ -25,7 +25,9 @@
>
>  import os.path
>  from collections.abc import MutableSequence
> +from dataclasses import dataclass
>  from enum import Enum, auto
> +from types import MethodType
>
>  from .config import (
>      OS,
> @@ -36,10 +38,42 @@
>      CPUType,
>      NodeConfiguration,
>      NodeInfo,
> +    TestSuiteConfig,
>  )
>  from .exception import DTSError, ErrorSeverity
>  from .logger import DTSLOG
>  from .settings import SETTINGS
> +from .test_suite import TestSuite
> +
> +
> +@dataclass(slots=True, frozen=True)
> +class TestSuiteWithCases:
> +    """A test suite class with test case methods.
> +
> +    An auxiliary class holding a test case class with test case methods. The intended use of this
> +    class is to hold a subset of test cases (which could be all test cases) because we don't have
> +    all the data to instantiate the class at the point of inspection. The knowledge of this subset
> +    is needed in case an error occurs before the class is instantiated and we need to record
> +    which test cases were blocked by the error.
> +
> +    Attributes:
> +        test_suite_class: The test suite class.
> +        test_cases: The test case methods.
> +    """
> +
> +    test_suite_class: type[TestSuite]
> +    test_cases: list[MethodType]
> +
> +    def create_config(self) -> TestSuiteConfig:
> +        """Generate a :class:`TestSuiteConfig` from the stored test suite with test cases.
> +
> +        Returns:
> +            The :class:`TestSuiteConfig` representation.
> +        """
> +        return TestSuiteConfig(
> +            test_suite=self.test_suite_class.__name__,
> +            test_cases=[test_case.__name__ for test_case in self.test_cases],
> +        )
>
>
>  class Result(Enum):
> diff --git a/dts/framework/test_suite.py b/dts/framework/test_suite.py
> index b02fd36147..f9fe88093e 100644
> --- a/dts/framework/test_suite.py
> +++ b/dts/framework/test_suite.py
> @@ -11,25 +11,17 @@
>      * Testbed (SUT, TG) configuration,
>      * Packet sending and verification,
>      * Test case verification.
> -
> -The module also defines a function, :func:`get_test_suites`,
> -for gathering test suites from a Python module.
>  """
>
> -import importlib
> -import inspect
> -import re
>  from ipaddress import IPv4Interface, IPv6Interface, ip_interface
> -from types import MethodType
> -from typing import Any, ClassVar, Union
> +from typing import ClassVar, Union
>
>  from scapy.layers.inet import IP  # type: ignore[import]
>  from scapy.layers.l2 import Ether  # type: ignore[import]
>  from scapy.packet import Packet, Padding  # type: ignore[import]
>
> -from .exception import ConfigurationError, TestCaseVerifyError
> +from .exception import TestCaseVerifyError
>  from .logger import DTSLOG, getLogger
> -from .settings import SETTINGS
>  from .testbed_model import Port, PortLink, SutNode, TGNode
>  from .utils import get_packet_summaries
>
> @@ -37,7 +29,6 @@
>  class TestSuite(object):
>      """The base class with building blocks needed by most test cases.
>
> -        * Test case filtering and collection,
>          * Test suite setup/cleanup methods to override,
>          * Test case setup/cleanup methods to override,
>          * Test case verification,
> @@ -71,7 +62,6 @@ class TestSuite(object):
>      #: will block the execution of all subsequent test suites in the current build target.
>      is_blocking: ClassVar[bool] = False
>      _logger: DTSLOG
> -    _test_cases_to_run: list[str]
>      _port_links: list[PortLink]
>      _sut_port_ingress: Port
>      _sut_port_egress: Port
> @@ -86,24 +76,19 @@ def __init__(
>          self,
>          sut_node: SutNode,
>          tg_node: TGNode,
> -        test_cases: list[str],
>      ):
>          """Initialize the test suite testbed information and basic configuration.
>
> -        Process what test cases to run, find links between ports and set up
> -        default IP addresses to be used when configuring them.
> +        Find links between ports and set up default IP addresses to be used when
> +        configuring them.
>
>          Args:
>              sut_node: The SUT node where the test suite will run.
>              tg_node: The TG node where the test suite will run.
> -            test_cases: The list of test cases to execute.
> -                If empty, all test cases will be executed.
>          """
>          self.sut_node = sut_node
>          self.tg_node = tg_node
>          self._logger = getLogger(self.__class__.__name__)
> -        self._test_cases_to_run = test_cases
> -        self._test_cases_to_run.extend(SETTINGS.test_cases)
>          self._port_links = []
>          self._process_links()
>          self._sut_port_ingress, self._tg_port_egress = (
> @@ -364,65 +349,3 @@ def _verify_l3_packet(self, received_packet: IP, expected_packet: IP) -> bool:
>          if received_packet.src != expected_packet.src or received_packet.dst != expected_packet.dst:
>              return False
>          return True
> -
> -    def _get_functional_test_cases(self) -> list[MethodType]:
> -        """Get all functional test cases defined in this TestSuite.
> -
> -        Returns:
> -            The list of functional test cases of this TestSuite.
> -        """
> -        return self._get_test_cases(r"test_(?!perf_)")
> -
> -    def _get_test_cases(self, test_case_regex: str) -> list[MethodType]:
> -        """Return a list of test cases matching test_case_regex.
> -
> -        Returns:
> -            The list of test cases matching test_case_regex of this TestSuite.
> -        """
> -        self._logger.debug(f"Searching for test cases in {self.__class__.__name__}.")
> -        filtered_test_cases = []
> -        for test_case_name, test_case in inspect.getmembers(self, inspect.ismethod):
> -            if self._should_be_executed(test_case_name, test_case_regex):
> -                filtered_test_cases.append(test_case)
> -        cases_str = ", ".join((x.__name__ for x in filtered_test_cases))
> -        self._logger.debug(f"Found test cases '{cases_str}' in {self.__class__.__name__}.")
> -        return filtered_test_cases
> -
> -    def _should_be_executed(self, test_case_name: str, test_case_regex: str) -> bool:
> -        """Check whether the test case should be scheduled to be executed."""
> -        match = bool(re.match(test_case_regex, test_case_name))
> -        if self._test_cases_to_run:
> -            return match and test_case_name in self._test_cases_to_run
> -
> -        return match
> -
> -
> -def get_test_suites(testsuite_module_path: str) -> list[type[TestSuite]]:
> -    r"""Find all :class:`TestSuite`\s in a Python module.
> -
> -    Args:
> -        testsuite_module_path: The path to the Python module.
> -
> -    Returns:
> -        The list of :class:`TestSuite`\s found within the Python module.
> -
> -    Raises:
> -        ConfigurationError: The test suite module was not found.
> -    """
> -
> -    def is_test_suite(object: Any) -> bool:
> -        try:
> -            if issubclass(object, TestSuite) and object is not TestSuite:
> -                return True
> -        except TypeError:
> -            return False
> -        return False
> -
> -    try:
> -        testcase_module = importlib.import_module(testsuite_module_path)
> -    except ModuleNotFoundError as e:
> -        raise ConfigurationError(f"Test suite '{testsuite_module_path}' not found.") from e
> -    return [
> -        test_suite_class
> -        for _, test_suite_class in inspect.getmembers(testcase_module, is_test_suite)
> -    ]
> diff --git a/dts/pyproject.toml b/dts/pyproject.toml
> index 28bd970ae4..8eb92b4f11 100644
> --- a/dts/pyproject.toml
> +++ b/dts/pyproject.toml
> @@ -51,6 +51,9 @@ linters = "mccabe,pycodestyle,pydocstyle,pyflakes"
>  format = "pylint"
>  max_line_length = 100
>
> +[tool.pylama.linter.pycodestyle]
> +ignore = "E203,W503"
> +
>  [tool.pylama.linter.pydocstyle]
>  convention = "google"
>
> diff --git a/dts/tests/TestSuite_smoke_tests.py b/dts/tests/TestSuite_smoke_tests.py
> index 5e2bac14bd..7b2a0e97f8 100644
> --- a/dts/tests/TestSuite_smoke_tests.py
> +++ b/dts/tests/TestSuite_smoke_tests.py
> @@ -21,7 +21,7 @@
>  from framework.utils import REGEX_FOR_PCI_ADDRESS
>
>
> -class SmokeTests(TestSuite):
> +class TestSmokeTests(TestSuite):
>      """DPDK and infrastructure smoke test suite.
>
>      The test cases validate the most basic DPDK functionality needed for all other test suites.
> --
> 2.34.1
>