[PATCH v3 3/4] dts: add per-test-suite configuration

[Luca Vizzarro <luca.vizzarro@arm.com>]

Allow test suites to be configured individually. Moreover enable them to
implement their own custom configuration.

This solution adds some new complexity to DTS, which is generated source
code. In order to ensure strong typing, the test suites and their custom
configurations need to be linked in the main configuration class.
Unfortunately, this is not feasible during runtime as it will incur in
circular dependencies. Generating the links appear to be the most
straightforward approach.

This commit also brings a new major change to the configuration schema.
Test suites are no longer defined as a list of strings, like:

    test_suites:
    - hello_world
    - pmd_buffer_scatter

but as mapping of mappings or strings:

    test_suites:
      hello_world: {} # any custom fields or test cases can be set here
      pmd_buffer_scatter: all # "all" defines all the test cases, or
                              # they can individually be set separated
                              # by a space

Not defining the `test_cases` field in the configuration is equivalent
to `all`, therefore the definitions for either test suite above are
also equivalent.

Creating the __init__.py file under the tests folder, allows it to be
picked up as a package. This is a mypy requirement to import the tests
from within the framework.

Bugzilla ID: 1375

Signed-off-by: Luca Vizzarro <luca.vizzarro@arm.com>
Reviewed-by: Paul Szczepanek <paul.szczepanek@arm.com>
---
 devtools/dts-check-format.sh                |  28 +++-
 devtools/dts-generate-tests-mappings.py     | 117 +++++++++++++++
 doc/api/dts/framework.config.generated.rst  |   8 +
 doc/api/dts/framework.config.rst            |   7 +
 doc/api/dts/framework.config.test_suite.rst |   8 +
 doc/api/dts/tests.config.rst                |   9 ++
 doc/api/dts/tests.rst                       |   1 +
 doc/guides/tools/dts.rst                    |  23 +++
 dts/conf.yaml                               |   4 +-
 dts/framework/config/__init__.py            |  76 +---------
 dts/framework/config/generated.py           |  25 ++++
 dts/framework/config/test_suite.py          | 154 ++++++++++++++++++++
 dts/framework/runner.py                     |  69 +++++++--
 dts/framework/settings.py                   |  30 ++--
 dts/framework/test_result.py                |  15 +-
 dts/framework/test_suite.py                 |  22 ++-
 dts/tests/TestSuite_hello_world.py          |   5 +-
 dts/tests/__init__.py                       |   7 +
 dts/tests/config.py                         |  20 +++
 19 files changed, 513 insertions(+), 115 deletions(-)
 create mode 100755 devtools/dts-generate-tests-mappings.py
 create mode 100644 doc/api/dts/framework.config.generated.rst
 create mode 100644 doc/api/dts/framework.config.test_suite.rst
 create mode 100644 doc/api/dts/tests.config.rst
 create mode 100644 dts/framework/config/generated.py
 create mode 100644 dts/framework/config/test_suite.py
 create mode 100644 dts/tests/__init__.py
 create mode 100644 dts/tests/config.py

diff --git a/devtools/dts-check-format.sh b/devtools/dts-check-format.sh
index 3f43e17e88..bad771fea3 100755
--- a/devtools/dts-check-format.sh
+++ b/devtools/dts-check-format.sh
@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright(c) 2022 University of New Hampshire
 # Copyright(c) 2023 PANTHEON.tech s.r.o.
+# Copyright(c) 2024 Arm Limited
 
 usage() {
 	echo "Usage: $(basename $0) [options] [directory]"
@@ -13,15 +14,19 @@ usage() {
 format=true
 lint=true
 typecheck=true
+generate=true
 
 # Comments after args serve as documentation; must be present
-while getopts "hflt" arg; do
+while getopts "hgflt" arg; do
 	case $arg in
 	h) # Display this message
-		echo 'Run formatting and linting programs for DTS.'
+		echo 'Run generating, formatting and linting programs for DTS.'
 		usage
 		exit 0
 		;;
+	g) # Don't run code generator
+		generate=false
+		;;
 	f) # Don't run formatters
 		format=false
 		;;
@@ -48,7 +53,22 @@ heading() {
 
 errors=0
 
+if $generate; then
+	heading "Generating test suites to configuration mappings"
+	if command -v python3 > /dev/null; then
+		../devtools/dts-generate-tests-mappings.py
+		errors=$((errors + $?))
+	else
+		echo "python3 not found, unable to run generator"
+		errors=$((errors + 1))
+	fi
+fi
+
 if $format; then
+	if $generate; then
+		echo
+	fi
+
 	if command -v git > /dev/null; then
 		if git rev-parse --is-inside-work-tree >&-; then
 			heading "Formatting in $directory/"
@@ -85,7 +105,7 @@ if $format; then
 fi
 
 if $lint; then
-	if $format; then
+	if $generate || $format; then
 		echo
 	fi
 	heading "Linting in $directory/"
@@ -99,7 +119,7 @@ if $lint; then
 fi
 
 if $typecheck; then
-	if $format || $lint; then
+	if $generate || $format || $lint; then
 		echo
 	fi
 	heading "Checking types in $directory/"
diff --git a/devtools/dts-generate-tests-mappings.py b/devtools/dts-generate-tests-mappings.py
new file mode 100755
index 0000000000..26ecc1018c
--- /dev/null
+++ b/devtools/dts-generate-tests-mappings.py
@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2024 Arm Limited
+
+"""DTS Test Suites to Configuration mappings generation script."""
+
+import os
+import sys
+from collections import defaultdict
+from pathlib import Path
+from textwrap import indent
+from typing import Iterable
+
+DTS_DIR = Path(__file__).parent.joinpath("..", "dts").resolve()
+SCRIPT_FILE_NAME = Path(__file__).relative_to(Path(__file__).parent.parent)
+
+sys.path.append(str(DTS_DIR))
+
+from framework.config.test_suite import BaseTestSuitesConfigs, TestSuiteConfig
+from framework.exception import InternalError
+from framework.test_suite import AVAILABLE_TEST_SUITES, TestSuiteSpec
+
+FRAMEWORK_IMPORTS = [BaseTestSuitesConfigs, TestSuiteConfig]
+
+RELATIVE_PATH_TO_GENERATED_FILE = "framework/config/generated.py"
+SMOKE_TESTS_SUITE_NAME = "smoke_tests"
+CUSTOM_CONFIG_TYPES_VAR_NAME = "CUSTOM_CONFIG_TYPES"
+CUSTOM_CONFIG_TYPES_VAR_DOCSTRING = [
+    "#: Mapping of test suites to their corresponding custom configuration objects if any."
+]
+TEST_SUITES_CONFIG_CLASS_NAME = "TestSuitesConfigs"
+TEST_SUITES_CONFIG_CLASS_DOCSTRING = [
+    '"""Configuration mapping class to select and configure the test suites."""',
+]
+
+
+GENERATED_FILE_HEADER = f"""# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2024 The DPDK contributors
+# This file is automatically generated by {SCRIPT_FILE_NAME}.
+# Do NOT modify this file manually.
+
+\"\"\"Generated file containing the links between the test suites and the configuration.\"\"\"
+"""
+
+
+def join(lines: Iterable[str]) -> str:
+    """Join list of strings into text lines."""
+    return "\n".join(lines)
+
+
+def join_and_indent(lines: Iterable[str], indentation_level=1, indentation_spaces=4) -> str:
+    """Join list of strings into indented text lines."""
+    return "\n".join([indent(line, " " * indentation_level * indentation_spaces) for line in lines])
+
+
+def format_attributes_types(test_suite_spec: TestSuiteSpec):
+    """Format the config type into the respective configuration class field attribute type."""
+    config_type = test_suite_spec.config_obj.__name__
+    return f"Optional[{config_type}]"
+
+
+try:
+    framework_imports: dict[str, list[str]] = defaultdict(list)
+    for _import in FRAMEWORK_IMPORTS:
+        framework_imports[_import.__module__].append(_import.__name__)
+    formatted_framework_imports = sorted(
+        [
+            f"from {module} import {', '.join(sorted(imports))}"
+            for module, imports in framework_imports.items()
+        ]
+    )
+
+    test_suites = [
+        test_suite_spec
+        for test_suite_spec in AVAILABLE_TEST_SUITES
+        if test_suite_spec.name != SMOKE_TESTS_SUITE_NAME
+    ]
+
+    custom_configs = [t for t in test_suites if t.config_obj is not TestSuiteConfig]
+
+    custom_config_imports = [
+        f"from {t.config_obj.__module__} import {t.config_obj.__name__}" for t in custom_configs
+    ]
+
+    test_suites_attributes = [f"{t.name}: {format_attributes_types(t)} = None" for t in test_suites]
+
+    custom_config_mappings = [f'"{t.name}": {t.config_obj.__name__},' for t in custom_configs]
+
+    generated_file_contents = f"""{GENERATED_FILE_HEADER}
+from typing import Optional
+
+{join(formatted_framework_imports)}
+
+{join(custom_config_imports)}
+
+{join(CUSTOM_CONFIG_TYPES_VAR_DOCSTRING)}
+{CUSTOM_CONFIG_TYPES_VAR_NAME}: dict[str, type[{TestSuiteConfig.__name__}]] = {'{'}
+{join_and_indent(custom_config_mappings)}
+{'}'}
+
+
+class {TEST_SUITES_CONFIG_CLASS_NAME}({BaseTestSuitesConfigs.__name__}):
+{join_and_indent(TEST_SUITES_CONFIG_CLASS_DOCSTRING)}
+
+{join_and_indent(test_suites_attributes)}
+"""
+
+    path = os.path.join(DTS_DIR, RELATIVE_PATH_TO_GENERATED_FILE)
+
+    with open(path, "w") as generated_file:
+        generated_file.write(generated_file_contents)
+
+    print("Test suites to configuration mappings generated successfully!")
+except Exception as e:
+    raise InternalError(
+        "Failed to generate test suites to configuration mappings."
+    ) from e
diff --git a/doc/api/dts/framework.config.generated.rst b/doc/api/dts/framework.config.generated.rst
new file mode 100644
index 0000000000..5dfa9342f0
--- /dev/null
+++ b/doc/api/dts/framework.config.generated.rst
@@ -0,0 +1,8 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+
+generated - Generated Test Suite Configurations
+===============================================
+
+.. automodule:: framework.config.generated
+   :members:
+   :show-inheritance:
diff --git a/doc/api/dts/framework.config.rst b/doc/api/dts/framework.config.rst
index cc266276c1..217fe026c4 100644
--- a/doc/api/dts/framework.config.rst
+++ b/doc/api/dts/framework.config.rst
@@ -6,3 +6,10 @@ config - Configuration Package
 .. automodule:: framework.config
    :members:
    :show-inheritance:
+
+.. toctree::
+   :hidden:
+   :maxdepth: 1
+
+   framework.config.generated
+   framework.config.test_suite
diff --git a/doc/api/dts/framework.config.test_suite.rst b/doc/api/dts/framework.config.test_suite.rst
new file mode 100644
index 0000000000..d59dcf5d6e
--- /dev/null
+++ b/doc/api/dts/framework.config.test_suite.rst
@@ -0,0 +1,8 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+
+test_suite - Test Suite Configuration Definitions
+=================================================
+
+.. automodule:: framework.config.test_suite
+   :members:
+   :show-inheritance:
diff --git a/doc/api/dts/tests.config.rst b/doc/api/dts/tests.config.rst
new file mode 100644
index 0000000000..ce3d9df868
--- /dev/null
+++ b/doc/api/dts/tests.config.rst
@@ -0,0 +1,9 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+
+Test Suites Configurations
+==========================
+
+.. automodule:: tests.config
+   :members:
+   :show-inheritance:
+
diff --git a/doc/api/dts/tests.rst b/doc/api/dts/tests.rst
index 0c136b4bb0..7fc25f2123 100644
--- a/doc/api/dts/tests.rst
+++ b/doc/api/dts/tests.rst
@@ -11,6 +11,7 @@ tests - Test Suites Package
    :hidden:
    :maxdepth: 1
 
+   tests.config
    tests.TestSuite_hello_world
    tests.TestSuite_os_udp
    tests.TestSuite_pmd_buffer_scatter
diff --git a/doc/guides/tools/dts.rst b/doc/guides/tools/dts.rst
index f4e297413d..4e63601b19 100644
--- a/doc/guides/tools/dts.rst
+++ b/doc/guides/tools/dts.rst
@@ -408,6 +408,29 @@ There are four types of methods that comprise a test suite:
    should be implemented in the ``SutNode`` class (and the underlying classes that ``SutNode`` uses)
    and used by the test suite via the ``sut_node`` field.
 
+The test suites can also implement their own custom configuration fields. This can be achieved by
+creating a new test suite config file which inherits from ``TestSuiteConfig`` defined in
+``dts/framework/config/test_suite.py``. So that this new custom configuration class is used, the
+test suite class must override the ``config`` attribute annotation with your new class, for example:
+
+.. code:: python
+
+   # place this under tests/config.py to avoid circular dependencies
+   class CustomConfig(TestSuiteConfig):
+      my_custom_field: int = 10
+
+   # place this under tests/TestSuite_my_new_test_suite.py
+   class TestMyNewTestSuite(TestSuite):
+      config: CustomConfig
+
+Finally, the test suites and the custom configuration files need to linked in the global configuration.
+This can be easily achieved by running the ``dts/generate-test-mappings.py``, e.g.:
+
+.. code-block:: console
+
+   $ poetry shell
+   (dts-py3.10) $ ./generate-test-mappings.py
+
 
 .. _dts_dev_tools:
 
diff --git a/dts/conf.yaml b/dts/conf.yaml
index 2496262854..377304dddf 100644
--- a/dts/conf.yaml
+++ b/dts/conf.yaml
@@ -28,8 +28,8 @@ test_runs:
     func: true # enable functional testing
     skip_smoke_tests: false # optional
     test_suites: # the following test suites will be run in their entirety
-      - hello_world
-      - os_udp
+      hello_world: all
+      os_udp: all
     # The machine running the DPDK test executable
     system_under_test_node:
       node_name: "SUT 1"
diff --git a/dts/framework/config/__init__.py b/dts/framework/config/__init__.py
index 82113a6257..0ac7ab5c46 100644
--- a/dts/framework/config/__init__.py
+++ b/dts/framework/config/__init__.py
@@ -32,11 +32,13 @@
       and makes it thread safe should we ever want to move in that direction.
 """
 
+# pylama:ignore=W0611
+
 import tarfile
 from enum import Enum, auto, unique
 from functools import cached_property
 from pathlib import Path, PurePath
-from typing import TYPE_CHECKING, Annotated, Any, Literal, NamedTuple
+from typing import Annotated, Literal, NamedTuple
 
 import yaml
 from pydantic import (
@@ -52,8 +54,7 @@
 from framework.exception import ConfigurationError
 from framework.utils import REGEX_FOR_PCI_ADDRESS, StrEnum
 
-if TYPE_CHECKING:
-    from framework.test_suite import TestSuiteSpec
+from .generated import TestSuitesConfigs
 
 
 class FrozenModel(BaseModel):
@@ -382,69 +383,6 @@ class DPDKUncompiledBuildConfiguration(BaseDPDKBuildConfiguration):
 DPDKBuildConfiguration = DPDKPrecompiledBuildConfiguration | DPDKUncompiledBuildConfiguration
 
 
-class TestSuiteConfig(FrozenModel):
-    """Test suite configuration.
-
-    Information about a single test suite to be executed. This can also be represented as a string
-    instead of a mapping, example:
-
-    .. code:: yaml
-
-        test_runs:
-        - test_suites:
-            # As string representation:
-            - hello_world # test all of `hello_world`, or
-            - hello_world hello_world_single_core # test only `hello_world_single_core`
-            # or as model fields:
-            - test_suite: hello_world
-              test_cases: [hello_world_single_core] # without this field all test cases are run
-    """
-
-    #: The name of the test suite module without the starting ``TestSuite_``.
-    test_suite_name: str = Field(alias="test_suite")
-    #: The names of test cases from this test suite to execute. If empty, all test cases will be
-    #: executed.
-    test_cases_names: list[str] = Field(default_factory=list, alias="test_cases")
-
-    @cached_property
-    def test_suite_spec(self) -> "TestSuiteSpec":
-        """The specification of the requested test suite."""
-        from framework.test_suite import find_by_name
-
-        test_suite_spec = find_by_name(self.test_suite_name)
-        assert (
-            test_suite_spec is not None
-        ), f"{self.test_suite_name} is not a valid test suite module name."
-        return test_suite_spec
-
-    @model_validator(mode="before")
-    @classmethod
-    def convert_from_string(cls, data: Any) -> Any:
-        """Convert the string representation of the model into a valid mapping."""
-        if isinstance(data, str):
-            [test_suite, *test_cases] = data.split()
-            return dict(test_suite=test_suite, test_cases=test_cases)
-        return data
-
-    @model_validator(mode="after")
-    def validate_names(self) -> Self:
-        """Validate the supplied test suite and test cases names.
-
-        This validator relies on the cached property `test_suite_spec` to run for the first
-        time in this call, therefore triggering the assertions if needed.
-        """
-        available_test_cases = map(
-            lambda t: t.name, self.test_suite_spec.class_obj.get_test_cases()
-        )
-        for requested_test_case in self.test_cases_names:
-            assert requested_test_case in available_test_cases, (
-                f"{requested_test_case} is not a valid test case "
-                f"of test suite {self.test_suite_name}."
-            )
-
-        return self
-
-
 class TestRunSUTNodeConfiguration(FrozenModel):
     """The SUT node configuration of a test run."""
 
@@ -469,8 +407,8 @@ class TestRunConfiguration(FrozenModel):
     func: bool
     #: Whether to skip smoke tests.
     skip_smoke_tests: bool = False
-    #: The names of test suites and/or test cases to execute.
-    test_suites: list[TestSuiteConfig] = Field(min_length=1)
+    #: The test suites to be selected and/or configured.
+    test_suites: TestSuitesConfigs
     #: The SUT node configuration to use in this test run.
     system_under_test_node: TestRunSUTNodeConfiguration
     #: The TG node name to use in this test run.
@@ -602,6 +540,6 @@ def load_config(config_file_path: Path) -> Configuration:
         config_data = yaml.safe_load(f)
 
     try:
-        return Configuration.model_validate(config_data)
+        return Configuration.model_validate(config_data, context={})
     except ValidationError as e:
         raise ConfigurationError("failed to load the supplied configuration") from e
diff --git a/dts/framework/config/generated.py b/dts/framework/config/generated.py
new file mode 100644
index 0000000000..cc4a539987
--- /dev/null
+++ b/dts/framework/config/generated.py
@@ -0,0 +1,25 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2024 The DPDK contributors
+# This file is automatically generated by devtools/dts-generate-tests-mappings.py.
+# Do NOT modify this file manually.
+
+"""Generated file containing the links between the test suites and the configuration."""
+
+from typing import Optional
+
+from framework.config.test_suite import BaseTestSuitesConfigs, TestSuiteConfig
+from tests.config import HelloWorldConfig
+
+#: Mapping of test suites to their corresponding custom configuration objects if any.
+CUSTOM_CONFIG_TYPES: dict[str, type[TestSuiteConfig]] = {
+    "hello_world": HelloWorldConfig,
+}
+
+
+class TestSuitesConfigs(BaseTestSuitesConfigs):
+    """Configuration mapping class to select and configure the test suites."""
+
+    hello_world: Optional[HelloWorldConfig] = None
+    os_udp: Optional[TestSuiteConfig] = None
+    pmd_buffer_scatter: Optional[TestSuiteConfig] = None
+    vlan: Optional[TestSuiteConfig] = None
diff --git a/dts/framework/config/test_suite.py b/dts/framework/config/test_suite.py
new file mode 100644
index 0000000000..863052cbc1
--- /dev/null
+++ b/dts/framework/config/test_suite.py
@@ -0,0 +1,154 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2024 Arm Limited
+
+"""Test suites configuration module.
+
+Test suites can inherit :class:`TestSuiteConfig` to create their own custom configuration.
+By doing so, the test suite class must also override the annotation of the field
+`~framework.test_suite.TestSuite.config` to use their custom configuration type.
+"""
+
+from functools import cached_property
+from typing import TYPE_CHECKING, Any, Iterable
+
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Field,
+    ValidationInfo,
+    field_validator,
+    model_validator,
+)
+from typing_extensions import Self
+
+if TYPE_CHECKING:
+    from framework.test_suite import TestSuiteSpec
+
+
+class TestSuiteConfig(BaseModel):
+    """Test suite configuration base model.
+
+    By default the configuration of a generic test suite does not contain any attributes. Any test
+    suite should inherit this class to create their own custom configuration. Finally override the
+    type of the :attr:`~TestSuite.config` to use the newly created one.
+
+    If no custom fields require setting, this can also be represented as a string instead of
+    a mapping, example:
+
+    .. code:: yaml
+
+        test_runs:
+        - test_suites:
+            # As string representation:
+            hello_world: all # test all of `hello_world`, or
+            hello_world: hello_world_single_core # test only `hello_world_single_core`
+            # or as a mapping of the model's fields:
+            hello_world:
+              test_cases: [hello_world_single_core] # without this field all test cases are run
+
+    .. warning::
+
+        This class sets `protected_namespaces` to an empty tuple as a workaround for autodoc.
+        Due to autodoc loading this class first before any other child ones, it causes the Pydantic
+        fields in the protected namespace ``model_`` to be set on the parent. Leading any child
+        classes to inherit these protected fields as user-defined ones, finally triggering their
+        instances to complain about the presence of protected fields.
+
+        Because any class inheriting this class will therefore have protected namespaces disabled,
+        you won't be blocked to create fields starting with ``model_``. Nonetheless, you **must**
+        refrain from doing so as this is not the intended behavior.
+    """
+
+    model_config = ConfigDict(frozen=True, extra="forbid", protected_namespaces=())
+
+    #: The name of the test suite module without the starting ``TestSuite_``. This field **cannot**
+    #: be used in the configuration file. The name will be inherited from the mapping key instead.
+    test_suite_name: str
+    #: The names of test cases from this test suite to execute. If empty, all test cases will be
+    #: executed.
+    test_cases_names: list[str] = Field(default_factory=list, alias="test_cases")
+
+    @cached_property
+    def test_suite_spec(self) -> "TestSuiteSpec":
+        """The specification of the requested test suite."""
+        from framework.test_suite import find_by_name
+
+        test_suite_spec = find_by_name(self.test_suite_name)
+        assert (
+            test_suite_spec is not None
+        ), f"{self.test_suite_name} is not a valid test suite module name."
+        return test_suite_spec
+
+    @model_validator(mode="before")
+    @classmethod
+    def load_test_suite_name_from_context(cls, data: Any, info: ValidationInfo) -> dict:
+        """Load the test suite name from the validation context, if any."""
+        assert isinstance(data, dict), "The test suite configuration value is invalid."
+        name = data.get("test_suite_name")
+        # If the context is carrying the test suite name, then use it instead.
+        if info.context is not None and (test_suite_name := info.context.get("test_suite_name")):
+            assert not name, "The test suite name cannot be set manually."
+            data["test_suite_name"] = test_suite_name
+        return data
+
+    @model_validator(mode="before")
+    @classmethod
+    def convert_from_string(cls, data: Any) -> dict:
+        """Convert the string representation of the model into a valid mapping."""
+        if isinstance(data, str):
+            test_cases = [] if data == "all" else data.split()
+            return dict(test_cases=test_cases)
+        return data
+
+    @model_validator(mode="after")
+    def validate_names(self) -> Self:
+        """Validate the supplied test suite and test cases names.
+
+        This validator relies on the cached property `test_suite_spec` to run for the first
+        time in this call, therefore triggering the assertions if needed.
+        """
+        available_test_cases = map(
+            lambda t: t.name, self.test_suite_spec.class_obj.get_test_cases()
+        )
+        for requested_test_case in self.test_cases_names:
+            assert requested_test_case in available_test_cases, (
+                f"{requested_test_case} is not a valid test case "
+                f"of test suite {self.test_suite_name}."
+            )
+
+        return self
+
+
+class BaseTestSuitesConfigs(BaseModel):
+    """Base class for test suites configs."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    def __contains__(self, key) -> bool:
+        """Check if the provided test suite name has been selected and/or configured."""
+        return key in self.model_fields_set
+
+    def __getitem__(self, key) -> TestSuiteConfig:
+        """Get test suite configuration."""
+        return self.__getattribute__(key)
+
+    def get_configs(self) -> Iterable[TestSuiteConfig]:
+        """Get all the test suite configurations."""
+        return map(lambda t: self[t], self.model_fields_set)
+
+    @classmethod
+    def available_test_suites(cls) -> Iterable[str]:
+        """List all the available test suites."""
+        return cls.model_fields.keys()
+
+    @field_validator("*", mode="before")
+    @classmethod
+    def pass_test_suite_name_to_config(cls, field_value: Any, info: ValidationInfo) -> Any:
+        """Before validating any :class:`TestSuiteConfig`, pass the test suite name via context."""
+        test_suite_name = info.field_name
+        assert test_suite_name is not None
+
+        assert info.context is not None, "A context dictionary is required to load test suites."
+        info.context.update({"test_suite_name": test_suite_name})
+
+        return field_value
diff --git a/dts/framework/runner.py b/dts/framework/runner.py
index 5f5837a132..2ab8861f99 100644
--- a/dts/framework/runner.py
+++ b/dts/framework/runner.py
@@ -25,6 +25,8 @@
 from types import MethodType
 from typing import Iterable
 
+from pydantic import ValidationError
+
 from framework.testbed_model.capability import Capability, get_supported_capabilities
 from framework.testbed_model.sut_node import SutNode
 from framework.testbed_model.tg_node import TGNode
@@ -34,11 +36,17 @@
     DPDKPrecompiledBuildConfiguration,
     SutNodeConfiguration,
     TestRunConfiguration,
-    TestSuiteConfig,
     TGNodeConfiguration,
     load_config,
 )
-from .exception import BlockingTestSuiteError, SSHTimeoutError, TestCaseVerifyError
+from .config.generated import CUSTOM_CONFIG_TYPES
+from .config.test_suite import TestSuiteConfig
+from .exception import (
+    BlockingTestSuiteError,
+    ConfigurationError,
+    SSHTimeoutError,
+    TestCaseVerifyError,
+)
 from .logger import DTSLogger, DtsStage, get_dts_logger
 from .settings import SETTINGS
 from .test_result import (
@@ -141,12 +149,7 @@ def run(self) -> None:
                 self._logger.info(f"Running test run with SUT '{sut_node_config.name}'.")
                 self._init_random_seed(test_run_config)
                 test_run_result = self._result.add_test_run(test_run_config)
-                # we don't want to modify the original config, so create a copy
-                test_run_test_suites = list(
-                    SETTINGS.test_suites if SETTINGS.test_suites else test_run_config.test_suites
-                )
-                if not test_run_config.skip_smoke_tests:
-                    test_run_test_suites[:0] = [TestSuiteConfig(test_suite="smoke_tests")]
+                test_run_test_suites = self._prepare_test_suites(test_run_config)
                 try:
                     test_suites_with_cases = self._get_test_suites_with_cases(
                         test_run_test_suites, test_run_config.func, test_run_config.perf
@@ -203,6 +206,46 @@ def _check_dts_python_version(self) -> None:
             )
             self._logger.warning("Please use Python >= 3.10 instead.")
 
+    def _prepare_test_suites(self, test_run_config: TestRunConfiguration) -> list[TestSuiteConfig]:
+        if SETTINGS.test_suites:
+            test_suites_configs = []
+            for selected_test_suite, selected_test_cases in SETTINGS.test_suites:
+                if selected_test_suite in test_run_config.test_suites:
+                    config = test_run_config.test_suites[selected_test_suite].model_copy(
+                        update={"test_cases_names": selected_test_cases}
+                    )
+                else:
+                    try:
+                        config = CUSTOM_CONFIG_TYPES[selected_test_suite](
+                            test_suite_name=selected_test_suite, test_cases=selected_test_cases
+                        )
+                    except AssertionError as e:
+                        raise ConfigurationError(
+                            "Invalid test cases were selected "
+                            f"for test suite {selected_test_suite}."
+                        ) from e
+                    except ValidationError as e:
+                        raise ConfigurationError(
+                            f"Test suite {selected_test_suite} needs to be explicitly configured "
+                            "in order to be selected."
+                        ) from e
+                    except KeyError:
+                        # not a custom configuration
+                        config = TestSuiteConfig(
+                            test_suite_name=selected_test_suite, test_cases=selected_test_cases
+                        )
+                test_suites_configs.append(config)
+        else:
+            # we don't want to modify the original config, so create a copy
+            test_suites_configs = [
+                config.model_copy() for config in test_run_config.test_suites.get_configs()
+            ]
+
+        if not test_run_config.skip_smoke_tests:
+            test_suites_configs[:0] = [TestSuiteConfig(test_suite_name="smoke_tests")]
+
+        return test_suites_configs
+
     def _get_test_suites_with_cases(
         self,
         test_suite_configs: list[TestSuiteConfig],
@@ -236,7 +279,11 @@ def _get_test_suites_with_cases(
                 test_cases.extend(perf_test_cases)
 
             test_suites_with_cases.append(
-                TestSuiteWithCases(test_suite_class=test_suite_class, test_cases=test_cases)
+                TestSuiteWithCases(
+                    test_suite_class=test_suite_class,
+                    test_cases=test_cases,
+                    config=test_suite_config,
+                )
             )
         return test_suites_with_cases
 
@@ -453,7 +500,9 @@ def _run_test_suite(
         self._logger.set_stage(
             DtsStage.test_suite_setup, Path(SETTINGS.output_dir, test_suite_name)
         )
-        test_suite = test_suite_with_cases.test_suite_class(sut_node, tg_node, topology)
+        test_suite = test_suite_with_cases.test_suite_class(
+            sut_node, tg_node, topology, test_suite_with_cases.config
+        )
         try:
             self._logger.info(f"Starting test suite setup: {test_suite_name}")
             test_suite.set_up_suite()
diff --git a/dts/framework/settings.py b/dts/framework/settings.py
index 5a8e6e5aee..f8783c4b59 100644
--- a/dts/framework/settings.py
+++ b/dts/framework/settings.py
@@ -107,7 +107,7 @@
     LocalDPDKTreeLocation,
     RemoteDPDKTarballLocation,
     RemoteDPDKTreeLocation,
-    TestSuiteConfig,
+    TestSuitesConfigs,
 )
 
 
@@ -133,7 +133,7 @@ class Settings:
     #:
     compile_timeout: float = 1200
     #:
-    test_suites: list[TestSuiteConfig] = field(default_factory=list)
+    test_suites: list[tuple[str, list[str]]] = field(default_factory=list)
     #:
     re_run: int = 0
     #:
@@ -508,7 +508,7 @@ def _process_dpdk_location(
 
 def _process_test_suites(
     parser: _DTSArgumentParser, args: list[list[str]]
-) -> list[TestSuiteConfig]:
+) -> list[tuple[str, list[str]]]:
     """Process the given argument to a list of :class:`TestSuiteConfig` to execute.
 
     Args:
@@ -524,19 +524,17 @@ def _process_test_suites(
         # Environment variable in the form of "SUITE1 CASE1 CASE2, SUITE2 CASE1, SUITE3, ..."
         args = [suite_with_cases.split() for suite_with_cases in args[0][0].split(",")]
 
-    try:
-        return [
-            TestSuiteConfig(test_suite=test_suite, test_cases=test_cases)
-            for [test_suite, *test_cases] in args
-        ]
-    except ValidationError as e:
-        print(
-            "An error has occurred while validating the test suites supplied in the "
-            f"{'environment variable' if action else 'arguments'}:",
-            file=sys.stderr,
-        )
-        print(e, file=sys.stderr)
-        sys.exit(1)
+    available_test_suites = TestSuitesConfigs.available_test_suites()
+    for test_suite_name, *_ in args:
+        if test_suite_name not in available_test_suites:
+            print(
+                f"The test suite {test_suite_name} supplied in the "
+                f"{'environment variable' if action else 'arguments'} is invalid.",
+                file=sys.stderr,
+            )
+            sys.exit(1)
+
+    return [(test_suite, test_cases) for test_suite, *test_cases in args]
 
 
 def get_settings() -> Settings:
diff --git a/dts/framework/test_result.py b/dts/framework/test_result.py
index 6014d281b5..8c0c1bcfb3 100644
--- a/dts/framework/test_result.py
+++ b/dts/framework/test_result.py
@@ -30,7 +30,8 @@
 
 from framework.testbed_model.capability import Capability
 
-from .config import TestRunConfiguration, TestSuiteConfig
+from .config import TestRunConfiguration
+from .config.test_suite import TestSuiteConfig
 from .exception import DTSError, ErrorSeverity
 from .logger import DTSLogger
 from .settings import SETTINGS
@@ -59,23 +60,13 @@ class is to hold a subset of test cases (which could be all test cases) because
     test_suite_class: type[TestSuite]
     test_cases: list[type[TestCase]]
     required_capabilities: set[Capability] = field(default_factory=set, init=False)
+    config: TestSuiteConfig
 
     def __post_init__(self):
         """Gather the required capabilities of the test suite and all test cases."""
         for test_object in [self.test_suite_class] + self.test_cases:
             self.required_capabilities.update(test_object.required_capabilities)
 
-    def create_config(self) -> TestSuiteConfig:
-        """Generate a :class:`TestSuiteConfig` from the stored test suite with test cases.
-
-        Returns:
-            The :class:`TestSuiteConfig` representation.
-        """
-        return TestSuiteConfig(
-            test_suite=self.test_suite_class.__name__,
-            test_cases=[test_case.__name__ for test_case in self.test_cases],
-        )
-
     def mark_skip_unsupported(self, supported_capabilities: set[Capability]) -> None:
         """Mark the test suite and test cases to be skipped.
 
diff --git a/dts/framework/test_suite.py b/dts/framework/test_suite.py
index fb5d646ce3..24cd0d38c1 100644
--- a/dts/framework/test_suite.py
+++ b/dts/framework/test_suite.py
@@ -24,13 +24,14 @@
 from ipaddress import IPv4Interface, IPv6Interface, ip_interface
 from pkgutil import iter_modules
 from types import ModuleType
-from typing import ClassVar, Protocol, TypeVar, Union, cast
+from typing import ClassVar, Protocol, TypeVar, Union, cast, get_type_hints
 
 from scapy.layers.inet import IP  # type: ignore[import-untyped]
 from scapy.layers.l2 import Ether  # type: ignore[import-untyped]
 from scapy.packet import Packet, Padding, raw  # type: ignore[import-untyped]
 from typing_extensions import Self
 
+from framework.config.test_suite import TestSuiteConfig
 from framework.testbed_model.capability import TestProtocol
 from framework.testbed_model.port import Port
 from framework.testbed_model.sut_node import SutNode
@@ -80,6 +81,7 @@ class TestSuite(TestProtocol):
     #: Whether the test suite is blocking. A failure of a blocking test suite
     #: will block the execution of all subsequent test suites in the current test run.
     is_blocking: ClassVar[bool] = False
+    config: TestSuiteConfig
     _logger: DTSLogger
     _sut_port_ingress: Port
     _sut_port_egress: Port
@@ -95,6 +97,7 @@ def __init__(
         sut_node: SutNode,
         tg_node: TGNode,
         topology: Topology,
+        config: TestSuiteConfig,
     ):
         """Initialize the test suite testbed information and basic configuration.
 
@@ -105,9 +108,11 @@ def __init__(
             sut_node: The SUT node where the test suite will run.
             tg_node: The TG node where the test suite will run.
             topology: The topology where the test suite will run.
+            config: The test suite configuration.
         """
         self.sut_node = sut_node
         self.tg_node = tg_node
+        self.config = config
         self._logger = get_dts_logger(self.__class__.__name__)
         self._tg_port_egress = topology.tg_port_egress
         self._sut_port_ingress = topology.sut_port_ingress
@@ -663,6 +668,21 @@ def is_test_suite(obj) -> bool:
             f"Expected class {self.class_name} not found in module {self.module_name}."
         )
 
+    @cached_property
+    def config_obj(self) -> type[TestSuiteConfig]:
+        """A reference to the test suite's configuration type."""
+        fields = get_type_hints(self.class_obj)
+        config_obj = fields.get("config")
+        if config_obj is None:
+            raise InternalError(
+                "Test suite class {self.class_name} is missing the `config` attribute."
+            )
+        if not issubclass(config_obj, TestSuiteConfig):
+            raise InternalError(
+                f"Test suite class {self.class_name} has an invalid configuration type assigned."
+            )
+        return config_obj
+
     @classmethod
     def discover_all(
         cls, package_name: str | None = None, module_prefix: str | None = None
diff --git a/dts/tests/TestSuite_hello_world.py b/dts/tests/TestSuite_hello_world.py
index 734f006026..f2998b968e 100644
--- a/dts/tests/TestSuite_hello_world.py
+++ b/dts/tests/TestSuite_hello_world.py
@@ -15,12 +15,15 @@
     LogicalCoreCountFilter,
     LogicalCoreList,
 )
+from tests.config import HelloWorldConfig
 
 
 @requires(topology_type=TopologyType.no_link)
 class TestHelloWorld(TestSuite):
     """DPDK hello world app test suite."""
 
+    config: HelloWorldConfig
+
     def set_up_suite(self) -> None:
         """Set up the test suite.
 
@@ -63,7 +66,7 @@ def hello_world_all_cores(self) -> None:
         eal_para = compute_eal_params(
             self.sut_node, lcore_filter_specifier=LogicalCoreList(self.sut_node.lcores)
         )
-        result = self.sut_node.run_dpdk_app(self.app_helloworld_path, eal_para, 50)
+        result = self.sut_node.run_dpdk_app(self.app_helloworld_path, eal_para, self.config.timeout)
         for lcore in self.sut_node.lcores:
             self.verify(
                 f"hello from core {int(lcore)}" in result.stdout,
diff --git a/dts/tests/__init__.py b/dts/tests/__init__.py
new file mode 100644
index 0000000000..a300eb26fc
--- /dev/null
+++ b/dts/tests/__init__.py
@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2024 Arm Limited
+
+"""Test suites.
+
+This package contains all the available test suites in DTS.
+"""
diff --git a/dts/tests/config.py b/dts/tests/config.py
new file mode 100644
index 0000000000..300ad3ef6a
--- /dev/null
+++ b/dts/tests/config.py
@@ -0,0 +1,20 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2024 Arm Limited
+
+"""Module for test suites custom configurations.
+
+Any test suite that requires custom configuration fields should create a new config class inheriting
+:class:`~framework.config.test_suite.TestSuiteConfig`, while respecting the parents' frozen state.
+Any custom fields can be added in this class.
+
+The custom configuration classes can be stored in this module.
+"""
+
+from framework.config.test_suite import TestSuiteConfig
+
+
+class HelloWorldConfig(TestSuiteConfig):
+    """Example custom configuration for the `TestHelloWorld` test suite."""
+
+    #: Timeout for the DPDK apps.
+    timeout: int = 50
-- 
2.43.0