DPDK patches and discussions
 help / color / mirror / Atom feed
From: "Juraj Linkeš" <juraj.linkes@pantheon.tech>
To: thomas@monjalon.net, Honnappa.Nagarahalli@arm.com,
	lijuan.tu@intel.com, bruce.richardson@intel.com,
	probb@iol.unh.edu
Cc: dev@dpdk.org, "Juraj Linkeš" <juraj.linkes@pantheon.tech>
Subject: [PATCH v6 08/10] dts: add test suite config and runner
Date: Fri,  3 Mar 2023 11:25:05 +0100	[thread overview]
Message-ID: <20230303102507.527790-9-juraj.linkes@pantheon.tech> (raw)
In-Reply-To: <20230303102507.527790-1-juraj.linkes@pantheon.tech>

The config allows users to specify which test suites and test cases
within test suites to run.
Also add test suite running capabilities to dts runner.

Signed-off-by: Juraj Linkeš <juraj.linkes@pantheon.tech>
---
 dts/conf.yaml                              |  2 ++
 dts/framework/config/__init__.py           | 29 +++++++++++++++-
 dts/framework/config/conf_yaml_schema.json | 40 ++++++++++++++++++++++
 dts/framework/dts.py                       | 19 ++++++++++
 dts/framework/test_suite.py                | 24 ++++++++++++-
 5 files changed, 112 insertions(+), 2 deletions(-)

diff --git a/dts/conf.yaml b/dts/conf.yaml
index 75e33e8ccf..a9bd8a3ecf 100644
--- a/dts/conf.yaml
+++ b/dts/conf.yaml
@@ -10,6 +10,8 @@ executions:
         compiler_wrapper: ccache
     perf: false
     func: true
+    test_suites:
+      - hello_world
     system_under_test: "SUT 1"
 nodes:
   - name: "SUT 1"
diff --git a/dts/framework/config/__init__.py b/dts/framework/config/__init__.py
index 544fceca6a..ebb0823ff5 100644
--- a/dts/framework/config/__init__.py
+++ b/dts/framework/config/__init__.py
@@ -12,7 +12,7 @@
 import pathlib
 from dataclasses import dataclass
 from enum import Enum, auto, unique
-from typing import Any
+from typing import Any, TypedDict
 
 import warlock  # type: ignore
 import yaml
@@ -128,11 +128,34 @@ def from_dict(d: dict) -> "BuildTargetConfiguration":
         )
 
 
+class TestSuiteConfigDict(TypedDict):
+    suite: str
+    cases: list[str]
+
+
+@dataclass(slots=True, frozen=True)
+class TestSuiteConfig:
+    test_suite: str
+    test_cases: list[str]
+
+    @staticmethod
+    def from_dict(
+        entry: str | TestSuiteConfigDict,
+    ) -> "TestSuiteConfig":
+        if isinstance(entry, str):
+            return TestSuiteConfig(test_suite=entry, test_cases=[])
+        elif isinstance(entry, dict):
+            return TestSuiteConfig(test_suite=entry["suite"], test_cases=entry["cases"])
+        else:
+            raise TypeError(f"{type(entry)} is not valid for a test suite config.")
+
+
 @dataclass(slots=True, frozen=True)
 class ExecutionConfiguration:
     build_targets: list[BuildTargetConfiguration]
     perf: bool
     func: bool
+    test_suites: list[TestSuiteConfig]
     system_under_test: NodeConfiguration
 
     @staticmethod
@@ -140,6 +163,9 @@ def from_dict(d: dict, node_map: dict) -> "ExecutionConfiguration":
         build_targets: list[BuildTargetConfiguration] = list(
             map(BuildTargetConfiguration.from_dict, d["build_targets"])
         )
+        test_suites: list[TestSuiteConfig] = list(
+            map(TestSuiteConfig.from_dict, d["test_suites"])
+        )
         sut_name = d["system_under_test"]
         assert sut_name in node_map, f"Unknown SUT {sut_name} in execution {d}"
 
@@ -147,6 +173,7 @@ def from_dict(d: dict, node_map: dict) -> "ExecutionConfiguration":
             build_targets=build_targets,
             perf=d["perf"],
             func=d["func"],
+            test_suites=test_suites,
             system_under_test=node_map[sut_name],
         )
 
diff --git a/dts/framework/config/conf_yaml_schema.json b/dts/framework/config/conf_yaml_schema.json
index 878ca3aec2..ca2d4a1ef2 100644
--- a/dts/framework/config/conf_yaml_schema.json
+++ b/dts/framework/config/conf_yaml_schema.json
@@ -93,6 +93,32 @@
       "required": [
         "amount"
       ]
+    },
+    "test_suite": {
+      "type": "string",
+      "enum": [
+        "hello_world"
+      ]
+    },
+    "test_target": {
+      "type": "object",
+      "properties": {
+        "suite": {
+          "$ref": "#/definitions/test_suite"
+        },
+        "cases": {
+          "type": "array",
+          "description": "If specified, only this subset of test suite's test cases will be run. Unknown test cases will be silently ignored.",
+          "items": {
+            "type": "string"
+          },
+          "minimum": 1
+        }
+      },
+      "required": [
+        "suite"
+      ],
+      "additionalProperties": false
     }
   },
   "type": "object",
@@ -172,6 +198,19 @@
             "type": "boolean",
             "description": "Enable functional testing."
           },
+          "test_suites": {
+            "type": "array",
+            "items": {
+              "oneOf": [
+                {
+                  "$ref": "#/definitions/test_suite"
+                },
+                {
+                  "$ref": "#/definitions/test_target"
+                }
+              ]
+            }
+          },
           "system_under_test": {
             "$ref": "#/definitions/node_name"
           }
@@ -181,6 +220,7 @@
           "build_targets",
           "perf",
           "func",
+          "test_suites",
           "system_under_test"
         ]
       },
diff --git a/dts/framework/dts.py b/dts/framework/dts.py
index 3d4170d10f..9012a499a3 100644
--- a/dts/framework/dts.py
+++ b/dts/framework/dts.py
@@ -8,6 +8,7 @@
 from .config import CONFIGURATION, BuildTargetConfiguration, ExecutionConfiguration
 from .exception import DTSError, ErrorSeverity
 from .logger import DTSLOG, getLogger
+from .test_suite import get_test_suites
 from .testbed_model import SutNode
 from .utils import check_dts_python_version
 
@@ -132,6 +133,24 @@ def _run_suites(
     with possibly only a subset of test cases.
     If no subset is specified, run all test cases.
     """
+    for test_suite_config in execution.test_suites:
+        try:
+            full_suite_path = f"tests.TestSuite_{test_suite_config.test_suite}"
+            test_suite_classes = get_test_suites(full_suite_path)
+            suites_str = ", ".join((x.__name__ for x in test_suite_classes))
+            dts_logger.debug(
+                f"Found test suites '{suites_str}' in '{full_suite_path}'."
+            )
+        except Exception as e:
+            dts_logger.exception("An error occurred when searching for test suites.")
+            errors.append(e)
+
+        else:
+            for test_suite_class in test_suite_classes:
+                test_suite = test_suite_class(
+                    sut_node, test_suite_config.test_cases, execution.func, errors
+                )
+                test_suite.run()
 
 
 def _exit_dts() -> None:
diff --git a/dts/framework/test_suite.py b/dts/framework/test_suite.py
index 9002d43297..12bf3b6420 100644
--- a/dts/framework/test_suite.py
+++ b/dts/framework/test_suite.py
@@ -6,12 +6,13 @@
 Base class for creating DTS test cases.
 """
 
+import importlib
 import inspect
 import re
 from collections.abc import MutableSequence
 from types import MethodType
 
-from .exception import SSHTimeoutError, TestCaseVerifyError
+from .exception import ConfigurationError, SSHTimeoutError, TestCaseVerifyError
 from .logger import DTSLOG, getLogger
 from .settings import SETTINGS
 from .testbed_model import SutNode
@@ -226,3 +227,24 @@ def _execute_test_case(self, test_case_method: MethodType) -> bool:
             raise KeyboardInterrupt("Stop DTS")
 
         return result
+
+
+def get_test_suites(testsuite_module_path: str) -> list[type[TestSuite]]:
+    def is_test_suite(object) -> bool:
+        try:
+            if issubclass(object, TestSuite) and object is not TestSuite:
+                return True
+        except TypeError:
+            return False
+        return False
+
+    try:
+        testcase_module = importlib.import_module(testsuite_module_path)
+    except ModuleNotFoundError as e:
+        raise ConfigurationError(
+            f"Test suite '{testsuite_module_path}' not found."
+        ) from e
+    return [
+        test_suite_class
+        for _, test_suite_class in inspect.getmembers(testcase_module, is_test_suite)
+    ]
-- 
2.30.2


  parent reply	other threads:[~2023-03-03 10:26 UTC|newest]

Thread overview: 97+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2022-08-24 16:24 [RFC PATCH v1 00/10] dts: add hello world testcase Juraj Linkeš
2022-08-24 16:24 ` [RFC PATCH v1 01/10] dts: hello world config options Juraj Linkeš
2022-08-24 16:24 ` [RFC PATCH v1 02/10] dts: hello world cli parameters and env vars Juraj Linkeš
2022-08-24 16:24 ` [RFC PATCH v1 03/10] dts: ssh connection additions for hello world Juraj Linkeš
2022-08-24 16:24 ` [RFC PATCH v1 04/10] dts: add basic node management methods Juraj Linkeš
2022-08-24 16:24 ` [RFC PATCH v1 05/10] dts: add system under test node Juraj Linkeš
2022-08-24 16:24 ` [RFC PATCH v1 06/10] dts: add traffic generator node Juraj Linkeš
2022-08-24 16:24 ` [RFC PATCH v1 07/10] dts: add testcase and basic test results Juraj Linkeš
2022-08-24 16:24 ` [RFC PATCH v1 08/10] dts: add test runner and statistics collector Juraj Linkeš
2022-08-24 16:24 ` [RFC PATCH v1 09/10] dts: add hello world testplan Juraj Linkeš
2022-08-24 16:24 ` [RFC PATCH v1 10/10] dts: add hello world testsuite Juraj Linkeš
2022-11-14 16:54 ` [RFC PATCH v2 00/10] dts: add hello world testcase Juraj Linkeš
2022-11-14 16:54   ` [RFC PATCH v2 01/10] dts: add node and os abstractions Juraj Linkeš
2022-11-14 16:54   ` [RFC PATCH v2 02/10] dts: add ssh command verification Juraj Linkeš
2022-11-14 16:54   ` [RFC PATCH v2 03/10] dts: add dpdk build on sut Juraj Linkeš
2022-11-16 13:15     ` Owen Hilyard
     [not found]       ` <30ad4f7d087d4932845b6ca13934b1d2@pantheon.tech>
     [not found]         ` <CAHx6DYDOFMuEm4xc65OTrtUmGBtk8Z6UtSgS2grnR_RBY5HcjQ@mail.gmail.com>
2022-11-23 12:37           ` Juraj Linkeš
2022-11-14 16:54   ` [RFC PATCH v2 04/10] dts: add dpdk execution handling Juraj Linkeš
2022-11-16 13:28     ` Owen Hilyard
     [not found]       ` <df13ee41efb64e7bb37791f21ae5bac1@pantheon.tech>
     [not found]         ` <CAHx6DYCEYxZ0Osm6fKhp3Jx8n7s=r7qVh8R41c6nCan8Or-dpA@mail.gmail.com>
2022-11-23 13:03           ` Juraj Linkeš
2022-11-28 13:05             ` Owen Hilyard
2022-11-14 16:54   ` [RFC PATCH v2 05/10] dts: add node memory setup Juraj Linkeš
2022-11-16 13:47     ` Owen Hilyard
2022-11-23 13:58       ` Juraj Linkeš
2022-11-14 16:54   ` [RFC PATCH v2 06/10] dts: add test results module Juraj Linkeš
2022-11-14 16:54   ` [RFC PATCH v2 07/10] dts: add simple stats report Juraj Linkeš
2022-11-16 13:57     ` Owen Hilyard
2022-11-14 16:54   ` [RFC PATCH v2 08/10] dts: add testsuite class Juraj Linkeš
2022-11-16 15:15     ` Owen Hilyard
2022-11-14 16:54   ` [RFC PATCH v2 09/10] dts: add hello world testplan Juraj Linkeš
2022-11-14 16:54   ` [RFC PATCH v2 10/10] dts: add hello world testsuite Juraj Linkeš
2023-01-17 15:48   ` [PATCH v3 00/10] dts: add hello world testcase Juraj Linkeš
2023-01-17 15:48     ` [PATCH v3 01/10] dts: add node and os abstractions Juraj Linkeš
2023-01-17 15:48     ` [PATCH v3 02/10] dts: add ssh command verification Juraj Linkeš
2023-01-17 15:48     ` [PATCH v3 03/10] dts: add dpdk build on sut Juraj Linkeš
2023-01-17 15:49     ` [PATCH v3 04/10] dts: add dpdk execution handling Juraj Linkeš
2023-01-17 15:49     ` [PATCH v3 05/10] dts: add node memory setup Juraj Linkeš
2023-01-17 15:49     ` [PATCH v3 06/10] dts: add test suite module Juraj Linkeš
2023-01-17 15:49     ` [PATCH v3 07/10] dts: add hello world testplan Juraj Linkeš
2023-01-17 15:49     ` [PATCH v3 08/10] dts: add hello world testsuite Juraj Linkeš
2023-01-17 15:49     ` [PATCH v3 09/10] dts: add test suite config and runner Juraj Linkeš
2023-01-17 15:49     ` [PATCH v3 10/10] dts: add test results module Juraj Linkeš
2023-01-19 16:16     ` [PATCH v3 00/10] dts: add hello world testcase Owen Hilyard
2023-02-09 16:47       ` Patrick Robb
2023-02-13 15:28     ` [PATCH v4 " Juraj Linkeš
2023-02-13 15:28       ` [PATCH v4 01/10] dts: add node and os abstractions Juraj Linkeš
2023-02-17 17:44         ` Bruce Richardson
2023-02-20 13:24           ` Juraj Linkeš
2023-02-13 15:28       ` [PATCH v4 02/10] dts: add ssh command verification Juraj Linkeš
2023-02-13 15:28       ` [PATCH v4 03/10] dts: add dpdk build on sut Juraj Linkeš
2023-02-22 16:44         ` Bruce Richardson
2023-02-13 15:28       ` [PATCH v4 04/10] dts: add dpdk execution handling Juraj Linkeš
2023-02-13 15:28       ` [PATCH v4 05/10] dts: add node memory setup Juraj Linkeš
2023-02-13 15:28       ` [PATCH v4 06/10] dts: add test suite module Juraj Linkeš
2023-02-13 15:28       ` [PATCH v4 07/10] dts: add hello world testsuite Juraj Linkeš
2023-02-13 15:28       ` [PATCH v4 08/10] dts: add test suite config and runner Juraj Linkeš
2023-02-13 15:28       ` [PATCH v4 09/10] dts: add test results module Juraj Linkeš
2023-02-13 15:28       ` [PATCH v4 10/10] doc: update DTS setup and test suite cookbook Juraj Linkeš
2023-02-17 17:26       ` [PATCH v4 00/10] dts: add hello world testcase Bruce Richardson
2023-02-20 10:13         ` Juraj Linkeš
2023-02-20 11:56           ` Bruce Richardson
2023-02-22 16:39           ` Bruce Richardson
2023-02-23  8:27             ` Juraj Linkeš
2023-02-23  9:17               ` Bruce Richardson
2023-02-23 15:28       ` [PATCH v5 " Juraj Linkeš
2023-02-23 15:28         ` [PATCH v5 01/10] dts: add node and os abstractions Juraj Linkeš
2023-02-23 15:28         ` [PATCH v5 02/10] dts: add ssh command verification Juraj Linkeš
2023-02-23 15:28         ` [PATCH v5 03/10] dts: add dpdk build on sut Juraj Linkeš
2023-02-23 15:28         ` [PATCH v5 04/10] dts: add dpdk execution handling Juraj Linkeš
2023-02-23 15:28         ` [PATCH v5 05/10] dts: add node memory setup Juraj Linkeš
2023-02-23 15:28         ` [PATCH v5 06/10] dts: add test suite module Juraj Linkeš
2023-02-23 15:28         ` [PATCH v5 07/10] dts: add hello world testsuite Juraj Linkeš
2023-02-23 15:28         ` [PATCH v5 08/10] dts: add test suite config and runner Juraj Linkeš
2023-02-23 15:28         ` [PATCH v5 09/10] dts: add test results module Juraj Linkeš
2023-02-23 15:28         ` [PATCH v5 10/10] doc: update DTS setup and test suite cookbook Juraj Linkeš
2023-03-03  8:31           ` Huang, ChenyuX
2023-02-23 16:13         ` [PATCH v5 00/10] dts: add hello world testcase Bruce Richardson
2023-02-26 19:11         ` Wathsala Wathawana Vithanage
2023-02-27  8:28           ` Juraj Linkeš
2023-02-28 15:27             ` Wathsala Wathawana Vithanage
2023-03-01  8:35               ` Juraj Linkeš
2023-03-03 10:24         ` [PATCH v6 00/10] dts: add hello world test case Juraj Linkeš
2023-03-03 10:24           ` [PATCH v6 01/10] dts: add node and os abstractions Juraj Linkeš
2023-03-03 10:24           ` [PATCH v6 02/10] dts: add ssh command verification Juraj Linkeš
2023-03-03 10:25           ` [PATCH v6 03/10] dts: add dpdk build on sut Juraj Linkeš
2023-03-20  8:30             ` David Marchand
2023-03-20 13:12               ` Juraj Linkeš
2023-03-20 13:22                 ` David Marchand
2023-03-03 10:25           ` [PATCH v6 04/10] dts: add dpdk execution handling Juraj Linkeš
2023-03-03 10:25           ` [PATCH v6 05/10] dts: add node memory setup Juraj Linkeš
2023-03-03 10:25           ` [PATCH v6 06/10] dts: add test suite module Juraj Linkeš
2023-03-03 10:25           ` [PATCH v6 07/10] dts: add hello world testsuite Juraj Linkeš
2023-03-03 10:25           ` Juraj Linkeš [this message]
2023-03-03 10:25           ` [PATCH v6 09/10] dts: add test results module Juraj Linkeš
2023-03-03 10:25           ` [PATCH v6 10/10] doc: update dts setup and test suite cookbook Juraj Linkeš
2023-03-09 21:47             ` Patrick Robb
2023-03-19 15:26           ` [PATCH v6 00/10] dts: add hello world test case Thomas Monjalon

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20230303102507.527790-9-juraj.linkes@pantheon.tech \
    --to=juraj.linkes@pantheon.tech \
    --cc=Honnappa.Nagarahalli@arm.com \
    --cc=bruce.richardson@intel.com \
    --cc=dev@dpdk.org \
    --cc=lijuan.tu@intel.com \
    --cc=probb@iol.unh.edu \
    --cc=thomas@monjalon.net \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).