* [RFC PATCH 0/1] add DTS smoke tests
@ 2023-04-13 17:54 jspewock
2023-04-13 17:54 ` [RFC PATCH 1/1] dts: add " jspewock
0 siblings, 1 reply; 2+ messages in thread
From: jspewock @ 2023-04-13 17:54 UTC (permalink / raw)
To: dev; +Cc: Jeremy Spewock
From: Jeremy Spewock <jspewock@iol.unh.edu>
This patch series adds a set of smoke tests to be run at
the beginning of DTS execution. The point is to validate
the user’s setup before running “real” tests. This helps
save time by bailing out of DTS early when the setup is
not valid, and also prevents DTS displaying “false failures”
associated with an incorrect DTS setup.
More specificially, these tests will verify the following:
* DPDK fast-tests suite
* DPDK driver-test suite
* Devices are bound to the correct driver
* General information about the SUT (kernel version, compiler version,
etc.)
* DPDK testpmd starts, stops, and receives packets
Jeremy Spewock (1):
dts: add smoke tests
dts/conf.yaml | 7 ++-
dts/framework/config/__init__.py | 15 ++++++
dts/framework/config/conf_yaml_schema.json | 16 +++++-
dts/framework/dts.py | 19 ++++++-
dts/framework/exception.py | 11 ++++
dts/framework/test_result.py | 13 +++--
dts/framework/test_suite.py | 24 ++++++++-
dts/tests/TestSuite_smoke_tests.py | 63 ++++++++++++++++++++++
8 files changed, 159 insertions(+), 9 deletions(-)
create mode 100644 dts/tests/TestSuite_smoke_tests.py
--
2.40.0
^ permalink raw reply [flat|nested] 2+ messages in thread
* [RFC PATCH 1/1] dts: add smoke tests
2023-04-13 17:54 [RFC PATCH 0/1] add DTS smoke tests jspewock
@ 2023-04-13 17:54 ` jspewock
0 siblings, 0 replies; 2+ messages in thread
From: jspewock @ 2023-04-13 17:54 UTC (permalink / raw)
To: dev; +Cc: Jeremy Spewock
From: Jeremy Spewock <jspewock@iol.unh.edu>
Adds a new test suite for running smoke tests that verify general
configuration aspects of the system under test. If any of these tests
fail, the DTS execution terminates as part of a "fail-fast" model.
Signed-off-by: Jeremy Spewock <jspewock@iol.unh.edu>
---
dts/conf.yaml | 7 ++-
dts/framework/config/__init__.py | 15 ++++++
dts/framework/config/conf_yaml_schema.json | 16 +++++-
dts/framework/dts.py | 19 ++++++-
dts/framework/exception.py | 11 ++++
dts/framework/test_result.py | 13 +++--
dts/framework/test_suite.py | 24 ++++++++-
dts/tests/TestSuite_smoke_tests.py | 63 ++++++++++++++++++++++
8 files changed, 159 insertions(+), 9 deletions(-)
create mode 100644 dts/tests/TestSuite_smoke_tests.py
diff --git a/dts/conf.yaml b/dts/conf.yaml
index a9bd8a3e..8caef719 100644
--- a/dts/conf.yaml
+++ b/dts/conf.yaml
@@ -10,13 +10,18 @@ executions:
compiler_wrapper: ccache
perf: false
func: true
+ nic: #physical devices to be used for testing
+ address: "0000:00:10.1"
+ driver: "vfio-pci"
test_suites:
+ - smoke_tests
- hello_world
system_under_test: "SUT 1"
nodes:
- name: "SUT 1"
- hostname: sut1.change.me.localhost
+ hostname: host.example.name
user: root
+ password: ""
arch: x86_64
os: linux
lcores: ""
diff --git a/dts/framework/config/__init__.py b/dts/framework/config/__init__.py
index ebb0823f..78d23422 100644
--- a/dts/framework/config/__init__.py
+++ b/dts/framework/config/__init__.py
@@ -106,6 +106,18 @@ def from_dict(d: dict) -> "NodeConfiguration":
hugepages=hugepage_config,
)
+@dataclass(slots=True, frozen=True)
+class NICConfiguration:
+ address: str
+ driver: str
+
+ @staticmethod
+ def from_dict(d:dict) -> "NICConfiguration":
+ return NICConfiguration(
+ address=d.get("address"),
+ driver=d.get("driver")
+ )
+
@dataclass(slots=True, frozen=True)
class BuildTargetConfiguration:
@@ -157,6 +169,7 @@ class ExecutionConfiguration:
func: bool
test_suites: list[TestSuiteConfig]
system_under_test: NodeConfiguration
+ nic: NICConfiguration
@staticmethod
def from_dict(d: dict, node_map: dict) -> "ExecutionConfiguration":
@@ -166,6 +179,7 @@ def from_dict(d: dict, node_map: dict) -> "ExecutionConfiguration":
test_suites: list[TestSuiteConfig] = list(
map(TestSuiteConfig.from_dict, d["test_suites"])
)
+ nic_conf: NICConfiguration = NICConfiguration.from_dict(d['nic'])
sut_name = d["system_under_test"]
assert sut_name in node_map, f"Unknown SUT {sut_name} in execution {d}"
@@ -174,6 +188,7 @@ def from_dict(d: dict, node_map: dict) -> "ExecutionConfiguration":
perf=d["perf"],
func=d["func"],
test_suites=test_suites,
+ nic=nic_conf,
system_under_test=node_map[sut_name],
)
diff --git a/dts/framework/config/conf_yaml_schema.json b/dts/framework/config/conf_yaml_schema.json
index ca2d4a1e..c98a9370 100644
--- a/dts/framework/config/conf_yaml_schema.json
+++ b/dts/framework/config/conf_yaml_schema.json
@@ -97,7 +97,8 @@
"test_suite": {
"type": "string",
"enum": [
- "hello_world"
+ "hello_world",
+ "smoke_tests"
]
},
"test_target": {
@@ -211,6 +212,19 @@
]
}
},
+ "nic": {
+ "type": "object",
+ "properties": {
+ "address": {
+ "type":"string",
+ "description": "PCI address of a physical device to test"
+ },
+ "driver": {
+ "type": "string",
+ "description": "The name of the driver that the physical device should be bound to"
+ }
+ }
+ },
"system_under_test": {
"$ref": "#/definitions/node_name"
}
diff --git a/dts/framework/dts.py b/dts/framework/dts.py
index 05022845..0d03e158 100644
--- a/dts/framework/dts.py
+++ b/dts/framework/dts.py
@@ -5,6 +5,8 @@
import sys
+from .exception import BlockingTestSuiteError
+
from .config import CONFIGURATION, BuildTargetConfiguration, ExecutionConfiguration
from .logger import DTSLOG, getLogger
from .test_result import BuildTargetResult, DTSResult, ExecutionResult, Result
@@ -49,6 +51,7 @@ def run_all() -> None:
nodes[sut_node.name] = sut_node
if sut_node:
+ #SMOKE TEST EXECUTION GOES HERE!
_run_execution(sut_node, execution, result)
except Exception as e:
@@ -118,7 +121,7 @@ def _run_build_target(
try:
sut_node.set_up_build_target(build_target)
- result.dpdk_version = sut_node.dpdk_version
+ # result.dpdk_version = sut_node.dpdk_version
build_target_result.update_setup(Result.PASS)
except Exception as e:
dts_logger.exception("Build target setup failed.")
@@ -146,6 +149,7 @@ def _run_suites(
with possibly only a subset of test cases.
If no subset is specified, run all test cases.
"""
+ end_execution = False
for test_suite_config in execution.test_suites:
try:
full_suite_path = f"tests.TestSuite_{test_suite_config.test_suite}"
@@ -160,13 +164,24 @@ def _run_suites(
else:
for test_suite_class in test_suite_classes:
+ #HERE NEEDS CHANGING
test_suite = test_suite_class(
sut_node,
test_suite_config.test_cases,
execution.func,
build_target_result,
+ sut_node._build_target_config,
+ result
)
- test_suite.run()
+ try:
+ test_suite.run()
+ except BlockingTestSuiteError as e:
+ dts_logger.exception("An error occurred within a blocking TestSuite, execution will now end.")
+ result.add_error(e)
+ end_execution = True
+ #if a blocking test failed and we need to bail out of suite executions
+ if end_execution:
+ break
def _exit_dts() -> None:
diff --git a/dts/framework/exception.py b/dts/framework/exception.py
index ca353d98..4e3f63d1 100644
--- a/dts/framework/exception.py
+++ b/dts/framework/exception.py
@@ -25,6 +25,7 @@ class ErrorSeverity(IntEnum):
SSH_ERR = 4
DPDK_BUILD_ERR = 10
TESTCASE_VERIFY_ERR = 20
+ BLOCKING_TESTSUITE_ERR = 25
class DTSError(Exception):
@@ -144,3 +145,13 @@ def __init__(self, value: str):
def __str__(self) -> str:
return repr(self.value)
+
+class BlockingTestSuiteError(DTSError):
+ suite_name: str
+ severity: ClassVar[ErrorSeverity] = ErrorSeverity.BLOCKING_TESTSUITE_ERR
+
+ def __init__(self, suite_name:str) -> None:
+ self.suite_name = suite_name
+
+ def __str__(self) -> str:
+ return f"Blocking suite {self.suite_name} failed."
diff --git a/dts/framework/test_result.py b/dts/framework/test_result.py
index 74391982..77202ae2 100644
--- a/dts/framework/test_result.py
+++ b/dts/framework/test_result.py
@@ -8,6 +8,7 @@
import os.path
from collections.abc import MutableSequence
from enum import Enum, auto
+from typing import Dict
from .config import (
OS,
@@ -67,12 +68,13 @@ class Statistics(dict):
Using a dict provides a convenient way to format the data.
"""
- def __init__(self, dpdk_version):
+ def __init__(self, output_info: Dict[str, str] | None):
super(Statistics, self).__init__()
for result in Result:
self[result.name] = 0
self["PASS RATE"] = 0.0
- self["DPDK VERSION"] = dpdk_version
+ if output_info:
+ for info_key, info_val in output_info.items(): self[info_key] = info_val
def __iadd__(self, other: Result) -> "Statistics":
"""
@@ -258,6 +260,7 @@ class DTSResult(BaseResult):
"""
dpdk_version: str | None
+ output: dict | None
_logger: DTSLOG
_errors: list[Exception]
_return_code: ErrorSeverity
@@ -267,6 +270,7 @@ class DTSResult(BaseResult):
def __init__(self, logger: DTSLOG):
super(DTSResult, self).__init__()
self.dpdk_version = None
+ self.output = None
self._logger = logger
self._errors = []
self._return_code = ErrorSeverity.NO_ERR
@@ -296,7 +300,10 @@ def process(self) -> None:
for error in self._errors:
self._logger.debug(repr(error))
- self._stats_result = Statistics(self.dpdk_version)
+ self._stats_result = Statistics(self.output)
+ #add information gathered from the smoke tests to the statistics
+ # for info_key, info_val in smoke_test_info.items(): self._stats_result[info_key] = info_val
+ # print(self._stats_result)
self.add_stats(self._stats_result)
with open(self._stats_filename, "w+") as stats_file:
stats_file.write(str(self._stats_result))
diff --git a/dts/framework/test_suite.py b/dts/framework/test_suite.py
index 0705f38f..1518fb8a 100644
--- a/dts/framework/test_suite.py
+++ b/dts/framework/test_suite.py
@@ -10,11 +10,14 @@
import inspect
import re
from types import MethodType
+from typing import Dict
-from .exception import ConfigurationError, SSHTimeoutError, TestCaseVerifyError
+from .config import BuildTargetConfiguration
+
+from .exception import BlockingTestSuiteError, ConfigurationError, SSHTimeoutError, TestCaseVerifyError
from .logger import DTSLOG, getLogger
from .settings import SETTINGS
-from .test_result import BuildTargetResult, Result, TestCaseResult, TestSuiteResult
+from .test_result import BuildTargetResult, DTSResult, Result, TestCaseResult, TestSuiteResult
from .testbed_model import SutNode
@@ -37,10 +40,12 @@ class TestSuite(object):
"""
sut_node: SutNode
+ is_blocking = False
_logger: DTSLOG
_test_cases_to_run: list[str]
_func: bool
_result: TestSuiteResult
+ _dts_result: DTSResult
def __init__(
self,
@@ -48,6 +53,8 @@ def __init__(
test_cases: list[str],
func: bool,
build_target_result: BuildTargetResult,
+ build_target_conf: BuildTargetConfiguration,
+ dts_result: DTSResult
):
self.sut_node = sut_node
self._logger = getLogger(self.__class__.__name__)
@@ -55,6 +62,8 @@ def __init__(
self._test_cases_to_run.extend(SETTINGS.test_cases)
self._func = func
self._result = build_target_result.add_test_suite(self.__class__.__name__)
+ self.build_target_info = build_target_conf
+ self._dts_result = dts_result
def set_up_suite(self) -> None:
"""
@@ -118,6 +127,9 @@ def run(self) -> None:
f"the next test suite may be affected."
)
self._result.update_setup(Result.ERROR, e)
+ if len(self._result.get_errors()) > 0 and self.is_blocking:
+ raise BlockingTestSuiteError(test_suite_name)
+
def _execute_test_suite(self) -> None:
"""
@@ -137,6 +149,7 @@ def _execute_test_suite(self) -> None:
f"Attempt number {attempt_nr} out of {all_attempts}."
)
self._run_test_case(test_case_method, test_case_result)
+
def _get_functional_test_cases(self) -> list[MethodType]:
"""
@@ -232,6 +245,11 @@ def _execute_test_case(
test_case_result.update(Result.SKIP)
raise KeyboardInterrupt("Stop DTS")
+ def write_to_statistics_file(self, output: Dict[str, str]):
+ if self._dts_result.output != None:
+ self._dts_result.output.update(output)
+ else:
+ self._dts_result.output = output
def get_test_suites(testsuite_module_path: str) -> list[type[TestSuite]]:
def is_test_suite(object) -> bool:
@@ -252,3 +270,5 @@ def is_test_suite(object) -> bool:
test_suite_class
for _, test_suite_class in inspect.getmembers(testcase_module, is_test_suite)
]
+
+
diff --git a/dts/tests/TestSuite_smoke_tests.py b/dts/tests/TestSuite_smoke_tests.py
new file mode 100644
index 00000000..b661f850
--- /dev/null
+++ b/dts/tests/TestSuite_smoke_tests.py
@@ -0,0 +1,63 @@
+from framework.test_suite import TestSuite
+from framework.testbed_model.sut_node import SutNode
+
+
+def get_compiler_version(compiler_name: str, sut_node: SutNode) -> str:
+ match compiler_name:
+ case "gcc":
+ return sut_node.main_session.send_command(f"{compiler_name} --version", 60).stdout.split("\n")[0]
+ case "clang":
+ return sut_node.main_session.send_command(f"{compiler_name} --version", 60).stdout.split("\n")[0]
+ case "msvc":
+ return sut_node.main_session.send_command(f"cl", 60).stdout
+ case "icc":
+ return sut_node.main_session.send_command(f"{compiler_name} -V", 60).stdout
+
+class SmokeTests(TestSuite):
+ is_blocking = True
+
+
+ def set_up_suite(self) -> None:
+ """
+ Setup:
+ build all DPDK
+ """
+ self.dpdk_build_dir_path = self.sut_node.remote_dpdk_build_dir
+
+
+ def test_unit_tests(self) -> None:
+ """
+ Test:
+ run the fast-test unit-test suite through meson
+ """
+ self.sut_node.main_session.send_command(f"meson test -C {self.dpdk_build_dir_path} --suite fast-tests", 300, True)
+
+ def test_driver_tests(self) -> None:
+ """
+ Test:
+ run the driver-test unit-test suite through meson
+ """
+ self.sut_node.main_session.send_command(f"meson test -C {self.dpdk_build_dir_path} --suite driver-tests", 300)
+
+ def test_gather_info(self) -> None:
+ out = {}
+
+ out['OS'] = self.sut_node.main_session.send_command("awk -F= '$1==\"NAME\" {print $2}' /etc/os-release", 60).stdout
+ out["OS VERSION"] = self.sut_node.main_session.send_command("awk -F= '$1==\"VERSION\" {print $2}' /etc/os-release", 60, True).stdout
+ out["COMPILER VERSION"] = get_compiler_version(self.build_target_info.compiler.name, self.sut_node)
+ out["DPDK VERSION"] = self.sut_node.dpdk_version
+ if self.build_target_info.os.name == "linux":
+ out['KERNEL VERSION'] = self.sut_node.main_session.send_command("uname -r", 60).stdout
+ elif self.build_target_info.os.name == "windows":
+ out['KERNEL VERSION'] = self.sut_node.main_session.send_command("uname -a", 60).stdout
+ self.write_to_statistics_file(out)
+
+ def test_start_testpmd(self) -> None:
+ """
+ Still heavily in development
+ """
+ self.sut_node.main_session.send_command(f"{self.dpdk_build_dir_path}/app/dpdk-testpmd -- -i", 60)
+ out = self.sut_node.main_session.send_command("show port summary all")
+ self.sut_node.main_session.send_command("quit")
+ print(out.stdout)
+
\ No newline at end of file
--
2.40.0
^ permalink raw reply [flat|nested] 2+ messages in thread
end of thread, other threads:[~2023-04-13 17:54 UTC | newest]
Thread overview: 2+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2023-04-13 17:54 [RFC PATCH 0/1] add DTS smoke tests jspewock
2023-04-13 17:54 ` [RFC PATCH 1/1] dts: add " jspewock
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).