Re: [RFC PATCH v2 08/10] dts: add testsuite class

[Owen Hilyard <ohilyard@iol.unh.edu>]

[-- Attachment #1: Type: text/plain, Size: 20880 bytes --]

On Mon, Nov 14, 2022 at 11:54 AM Juraj Linkeš <juraj.linkes@pantheon.tech>
wrote:

> This is the base class that all test suites inherit from. The base class
> implements methods common to all test suites. The derived test suites
> implement tests and any particular setup needed for the suite or tests.
>
> Signed-off-by: Juraj Linkeš <juraj.linkes@pantheon.tech>
> ---
>  dts/conf.yaml                              |   4 +
>  dts/framework/config/__init__.py           |  33 ++-
>  dts/framework/config/conf_yaml_schema.json |  49 ++++
>  dts/framework/dts.py                       |  29 +++
>  dts/framework/exception.py                 |  65 ++++++
>  dts/framework/settings.py                  |  25 +++
>  dts/framework/test_case.py                 | 246 +++++++++++++++++++++
>  7 files changed, 450 insertions(+), 1 deletion(-)
>  create mode 100644 dts/framework/test_case.py
>
> diff --git a/dts/conf.yaml b/dts/conf.yaml
> index 976888a88e..0b0f2c59b0 100644
> --- a/dts/conf.yaml
> +++ b/dts/conf.yaml
> @@ -7,6 +7,10 @@ executions:
>          os: linux
>          cpu: native
>          compiler: gcc
> +    perf: false
> +    func: true
> +    test_suites:
> +      - hello_world
>      system_under_test: "SUT 1"
>  nodes:
>    - name: "SUT 1"
> diff --git a/dts/framework/config/__init__.py
> b/dts/framework/config/__init__.py
> index 344d697a69..8874b10030 100644
> --- a/dts/framework/config/__init__.py
> +++ b/dts/framework/config/__init__.py
> @@ -11,7 +11,7 @@
>  import pathlib
>  from dataclasses import dataclass
>  from enum import Enum, auto, unique
> -from typing import Any, Iterable
> +from typing import Any, Iterable, TypedDict
>
>  import warlock  # type: ignore
>  import yaml
> @@ -186,9 +186,34 @@ def from_dict(d: dict) -> "BuildTargetConfiguration":
>          )
>
>
> +class TestSuiteConfigDict(TypedDict):
> +    suite: str
> +    cases: list[str]
> +
> +
> +@dataclass(slots=True, frozen=True)
> +class TestSuiteConfig:
> +    test_suite: str
> +    test_cases: list[str]
> +
> +    @staticmethod
> +    def from_dict(
> +        entry: str | TestSuiteConfigDict,
> +    ) -> "TestSuiteConfig":
> +        if isinstance(entry, str):
> +            return TestSuiteConfig(test_suite=entry, test_cases=[])
> +        elif isinstance(entry, dict):
> +            return TestSuiteConfig(test_suite=entry["suite"],
> test_cases=entry["cases"])
> +        else:
> +            raise TypeError(f"{type(entry)} is not valid for a test suite
> config.")
> +
> +
>  @dataclass(slots=True, frozen=True)
>  class ExecutionConfiguration:
>      build_targets: list[BuildTargetConfiguration]
> +    perf: bool
> +    func: bool
> +    test_suites: list[TestSuiteConfig]
>      system_under_test: NodeConfiguration
>
>      @staticmethod
> @@ -196,11 +221,17 @@ def from_dict(d: dict, node_map: dict) ->
> "ExecutionConfiguration":
>          build_targets: list[BuildTargetConfiguration] = list(
>              map(BuildTargetConfiguration.from_dict, d["build_targets"])
>          )
> +        test_suites: list[TestSuiteConfig] = list(
> +            map(TestSuiteConfig.from_dict, d["test_suites"])
> +        )
>          sut_name = d["system_under_test"]
>          assert sut_name in node_map, f"Unknown SUT {sut_name} in
> execution {d}"
>
>          return ExecutionConfiguration(
>              build_targets=build_targets,
> +            perf=d["perf"],
> +            func=d["func"],
> +            test_suites=test_suites,
>              system_under_test=node_map[sut_name],
>          )
>
> diff --git a/dts/framework/config/conf_yaml_schema.json
> b/dts/framework/config/conf_yaml_schema.json
> index c59d3e30e6..e37ced65fe 100644
> --- a/dts/framework/config/conf_yaml_schema.json
> +++ b/dts/framework/config/conf_yaml_schema.json
> @@ -63,6 +63,31 @@
>          }
>        },
>        "additionalProperties": false
> +    },
> +    "test_suite": {
> +      "type": "string",
> +      "enum": [
> +        "hello_world"
> +      ]
> +    },
> +    "test_target": {
> +      "type": "object",
> +      "properties": {
> +        "suite": {
> +          "$ref": "#/definitions/test_suite"
> +        },
> +        "cases": {
> +          "type": "array",
> +          "items": {
> +            "type": "string"
> +          },
> +          "minimum": 1
> +        }
> +      },
> +      "required": [
> +        "suite"
> +      ],
> +      "additionalProperties": false
>      }
>    },
>    "type": "object",
> @@ -130,6 +155,27 @@
>              },
>              "minimum": 1
>            },
> +          "perf": {
> +            "type": "boolean",
> +            "description": "Enable performance testing"
> +          },
> +          "func": {
> +            "type": "boolean",
> +            "description": "Enable functional testing"
> +          },
> +          "test_suites": {
> +            "type": "array",
> +            "items": {
> +              "oneOf": [
> +                {
> +                  "$ref": "#/definitions/test_suite"
> +                },
> +                {
> +                  "$ref": "#/definitions/test_target"
> +                }
> +              ]
> +            }
> +          },
>            "system_under_test": {
>              "$ref": "#/definitions/node_name"
>            }
> @@ -137,6 +183,9 @@
>          "additionalProperties": false,
>          "required": [
>            "build_targets",
> +          "perf",
> +          "func",
> +          "test_suites",
>            "system_under_test"
>          ]
>        },
> diff --git a/dts/framework/dts.py b/dts/framework/dts.py
> index a7c243a5c3..ba3f4b4168 100644
> --- a/dts/framework/dts.py
> +++ b/dts/framework/dts.py
> @@ -15,6 +15,7 @@
>  from .logger import DTSLOG, getLogger
>  from .settings import SETTINGS
>  from .stats_reporter import TestStats
> +from .test_case import TestCase
>  from .test_result import Result
>  from .utils import check_dts_python_version
>
> @@ -129,6 +130,34 @@ def run_suite(
>      Use the given build_target to run the test suite with possibly only a
> subset
>      of tests. If no subset is specified, run all tests.
>      """
> +    for test_suite_config in execution.test_suites:
> +        result.test_suite = test_suite_config.test_suite
> +        full_suite_path =
> f"tests.TestSuite_{test_suite_config.test_suite}"
> +        testcase_classes = TestCase.get_testcases(full_suite_path)
> +        dts_logger.debug(
> +            f"Found testcase classes '{testcase_classes}' in
> '{full_suite_path}'"
> +        )
> +        for testcase_class in testcase_classes:
> +            testcase = testcase_class(
> +                sut_node, test_suite_config.test_suite, build_target,
> execution
> +            )
> +
> +            testcase.init_log()
> +            testcase.set_requested_cases(SETTINGS.test_cases)
> +            testcase.set_requested_cases(test_suite_config.test_cases)
> +
> +            dts_logger.info(f"Running test suite
> '{testcase_class.__name__}'")
> +            try:
> +                testcase.execute_setup_all()
> +                testcase.execute_test_cases()
> +                dts_logger.info(
> +                    f"Finished running test suite
> '{testcase_class.__name__}'"
> +                )
> +                result.copy_suite(testcase.get_result())
> +                test_stats.save(result)  # this was originally after
> teardown
> +
> +            finally:
>

You should probably move the "finished" log message down here, so that it
always runs.


> +                testcase.execute_tear_downall()
>
>
>  def quit_execution(nodes: Iterable[Node], return_code: ReturnCode) ->
> None:
> diff --git a/dts/framework/exception.py b/dts/framework/exception.py
> index 93d99432ae..a35eeff640 100644
> --- a/dts/framework/exception.py
> +++ b/dts/framework/exception.py
> @@ -29,6 +29,10 @@ class ReturnCode(IntEnum):
>      DPDK_BUILD_ERR = 10
>      NODE_SETUP_ERR = 20
>      NODE_CLEANUP_ERR = 21
> +    SUITE_SETUP_ERR = 30
> +    SUITE_EXECUTION_ERR = 31
> +    TESTCASE_VERIFY_ERR = 32
> +    SUITE_CLEANUP_ERR = 33
>
>
>  class DTSError(Exception):
> @@ -153,6 +157,67 @@ def __init__(self):
>          )
>
>
> +class TestSuiteNotFound(DTSError):
> +    """
> +    Raised when a configured test suite cannot be imported.
> +    """
> +
> +    return_code: ClassVar[ReturnCode] = ReturnCode.SUITE_SETUP_ERR
> +
> +
> +class SuiteSetupError(DTSError):
> +    """
> +    Raised when an error occurs during suite setup.
> +    """
> +
> +    return_code: ClassVar[ReturnCode] = ReturnCode.SUITE_SETUP_ERR
> +
> +    def __init__(self):
> +        super(SuiteSetupError, self).__init__("An error occurred during
> suite setup.")
> +
> +
> +class SuiteExecutionError(DTSError):
> +    """
> +    Raised when an error occurs during suite execution.
> +    """
> +
> +    return_code: ClassVar[ReturnCode] = ReturnCode.SUITE_EXECUTION_ERR
> +
> +    def __init__(self):
> +        super(SuiteExecutionError, self).__init__(
> +            "An error occurred during suite execution."
> +        )
> +
> +
> +class VerifyError(DTSError):
> +    """
> +    To be used within the test cases to verify if a command output
> +    is as it was expected.
> +    """
> +
> +    value: str
> +    return_code: ClassVar[ReturnCode] = ReturnCode.TESTCASE_VERIFY_ERR
> +
> +    def __init__(self, value: str):
> +        self.value = value
> +
> +    def __str__(self) -> str:
> +        return repr(self.value)
> +
> +
> +class SuiteCleanupError(DTSError):
> +    """
> +    Raised when an error occurs during suite cleanup.
> +    """
> +
> +    return_code: ClassVar[ReturnCode] = ReturnCode.SUITE_CLEANUP_ERR
> +
> +    def __init__(self):
> +        super(SuiteCleanupError, self).__init__(
> +            "An error occurred during suite cleanup."
> +        )
> +
> +
>  def convert_exception(exception: type[DTSError]) -> Callable[...,
> Callable[..., None]]:
>      """
>      When a non-DTS exception is raised while executing the decorated
> function,
> diff --git a/dts/framework/settings.py b/dts/framework/settings.py
> index e2bf3d2ce4..069f28ce81 100644
> --- a/dts/framework/settings.py
> +++ b/dts/framework/settings.py
> @@ -64,6 +64,8 @@ class _Settings:
>      skip_setup: bool
>      dpdk_ref: Path
>      compile_timeout: float
> +    test_cases: list
> +    re_run: int
>
>
>  def _get_parser() -> argparse.ArgumentParser:
> @@ -138,6 +140,25 @@ def _get_parser() -> argparse.ArgumentParser:
>          help="[DTS_COMPILE_TIMEOUT] The timeout for compiling DPDK.",
>      )
>
> +    parser.add_argument(
> +        "--test-cases",
> +        action=_env_arg("DTS_TESTCASES"),
> +        default="",
> +        required=False,
> +        help="[DTS_TESTCASES] Comma-separated list of testcases to
> execute",
> +    )
> +
> +    parser.add_argument(
> +        "--re-run",
> +        "--re_run",
> +        action=_env_arg("DTS_RERUN"),
> +        default=0,
> +        type=int,
> +        required=False,
> +        help="[DTS_RERUN] Re-run tests the specified amount of times if a
> test failure "
> +        "occurs",
> +    )
> +
>      return parser
>
>
> @@ -151,6 +172,10 @@ def _get_settings() -> _Settings:
>          skip_setup=(parsed_args.skip_setup == "Y"),
>          dpdk_ref=parsed_args.dpdk_ref,
>          compile_timeout=parsed_args.compile_timeout,
> +        test_cases=parsed_args.test_cases.split(",")
> +        if parsed_args.test_cases != ""
> +        else [],
> +        re_run=parsed_args.re_run,
>      )
>
>
> diff --git a/dts/framework/test_case.py b/dts/framework/test_case.py
> new file mode 100644
> index 0000000000..0479f795bb
> --- /dev/null
> +++ b/dts/framework/test_case.py
> @@ -0,0 +1,246 @@
> +# SPDX-License-Identifier: BSD-3-Clause
> +# Copyright(c) 2010-2014 Intel Corporation
> +# Copyright(c) 2022 PANTHEON.tech s.r.o.
> +
> +"""
> +A base class for creating DTS test cases.
> +"""
> +
> +import importlib
> +import inspect
> +import re
> +import time
> +import traceback
> +
> +from .exception import (
> +    SSHTimeoutError,
> +    SuiteCleanupError,
> +    SuiteExecutionError,
> +    SuiteSetupError,
> +    TestSuiteNotFound,
> +    VerifyError,
> +    convert_exception,
> +)
> +from .logger import getLogger
> +from .settings import SETTINGS
> +from .test_result import Result
> +
> +
> +class TestCase(object):
> +    def __init__(self, sut_node, suitename, target, execution):
> +        self.sut_node = sut_node
> +        self.suite_name = suitename
> +        self.target = target
> +
> +        # local variable
> +        self._requested_tests = []
> +        self._subtitle = None
> +
> +        # result object for save suite result
> +        self._suite_result = Result()
> +        self._suite_result.sut = self.sut_node.config.hostname
> +        self._suite_result.target = target
> +        self._suite_result.test_suite = self.suite_name
> +        if self._suite_result is None:
> +            raise ValueError("Result object should not None")
> +
> +        self._enable_func = execution.func
> +
> +        # command history
> +        self.setup_history = list()
> +        self.test_history = list()
> +
> +    def init_log(self):
> +        # get log handler
> +        class_name = self.__class__.__name__
> +        self.logger = getLogger(class_name)
> +
> +    def set_up_all(self):
> +        pass
> +
> +    def set_up(self):
> +        pass
> +
> +    def tear_down(self):
> +        pass
> +
> +    def tear_down_all(self):
> +        pass
> +
> +    def verify(self, passed, description):
> +        if not passed:
> +            raise VerifyError(description)
> +
> +    def _get_functional_cases(self):
> +        """
> +        Get all functional test cases.
> +        """
> +        return self._get_test_cases(r"test_(?!perf_)")
> +
> +    def _has_it_been_requested(self, test_case, test_name_regex):
> +        """
> +        Check whether test case has been requested for validation.
> +        """
> +        name_matches = re.match(test_name_regex, test_case.__name__)
> +        if self._requested_tests:
> +            return name_matches and test_case.__name__ in
> self._requested_tests
> +
> +        return name_matches
> +
> +    def set_requested_cases(self, case_list):
> +        """
> +        Pass down input cases list for check
> +        """
> +        self._requested_tests += case_list
> +
> +    def _get_test_cases(self, test_name_regex):
> +        """
> +        Return case list which name matched regex.
> +        """
> +        self.logger.debug(f"Searching for testcases in {self.__class__}")
> +        for test_case_name in dir(self):
> +            test_case = getattr(self, test_case_name)
> +            if callable(test_case) and self._has_it_been_requested(
> +                test_case, test_name_regex
> +            ):
> +                yield test_case
> +
> +    @convert_exception(SuiteSetupError)
> +    def execute_setup_all(self):
> +        """
> +        Execute suite setup_all function before cases.
> +        """
> +        try:
> +            self.set_up_all()
> +            return True
> +        except Exception as v:
> +            self.logger.error("set_up_all failed:\n" +
> traceback.format_exc())
> +            # record all cases blocked
> +            if self._enable_func:
> +                for case_obj in self._get_functional_cases():
> +                    self._suite_result.test_case = case_obj.__name__
> +                    self._suite_result.test_case_blocked(
> +                        "set_up_all failed: {}".format(str(v))
> +                    )
> +            return False
> +
> +    def _execute_test_case(self, case_obj):
> +        """
> +        Execute specified test case in specified suite. If any exception
> occurred in
> +        validation process, save the result and tear down this case.
> +        """
> +        case_name = case_obj.__name__
> +        self._suite_result.test_case = case_obj.__name__
> +
> +        case_result = True
> +        try:
> +            self.logger.info("Test Case %s Begin" % case_name)
> +
> +            self.running_case = case_name
> +            # run set_up function for each case
> +            self.set_up()
> +            # run test case
> +            case_obj()
> +
> +            self._suite_result.test_case_passed()
> +
> +            self.logger.info("Test Case %s Result PASSED:" % case_name)
> +
> +        except VerifyError as v:
> +            case_result = False
> +            self._suite_result.test_case_failed(str(v))
> +            self.logger.error("Test Case %s Result FAILED: " %
> (case_name) + str(v))
> +        except KeyboardInterrupt:
> +            self._suite_result.test_case_blocked("Skipped")
> +            self.logger.error("Test Case %s SKIPPED: " % (case_name))
> +            self.tear_down()
> +            raise KeyboardInterrupt("Stop DTS")
> +        except SSHTimeoutError as e:
> +            case_result = False
> +            self._suite_result.test_case_failed(str(e))
> +            self.logger.error("Test Case %s Result FAILED: " %
> (case_name) + str(e))
> +            self.logger.error("%s" % (e.get_output()))
> +        except Exception:
> +            case_result = False
> +            trace = traceback.format_exc()
> +            self._suite_result.test_case_failed(trace)
> +            self.logger.error("Test Case %s Result ERROR: " % (case_name)
> + trace)
> +        finally:
> +            self.execute_tear_down()
> +            return case_result
> +
> +    @convert_exception(SuiteExecutionError)
> +    def execute_test_cases(self):
> +        """
> +        Execute all test cases in one suite.
> +        """
> +        # prepare debugger rerun case environment
> +        if self._enable_func:
> +            for case_obj in self._get_functional_cases():
> +                for i in range(SETTINGS.re_run + 1):
> +                    ret = self.execute_test_case(case_obj)
> +
> +                    if ret is False and SETTINGS.re_run:
> +                        self.sut_node.get_session_output(timeout=0.5 * (i
> + 1))
> +                        time.sleep(i + 1)
> +                        self.logger.info(
> +                            " Test case %s failed and re-run %d time"
> +                            % (case_obj.__name__, i + 1)
> +                        )
> +                    else:
> +                        break
> +
> +    def execute_test_case(self, case_obj):
> +        """
> +        Execute test case or enter into debug mode.
> +        """
> +        return self._execute_test_case(case_obj)
> +
> +    def get_result(self):
> +        """
> +        Return suite test result
> +        """
> +        return self._suite_result
> +
> +    @convert_exception(SuiteCleanupError)
> +    def execute_tear_downall(self):
> +        """
> +        execute suite tear_down_all function
> +        """
> +        self.tear_down_all()
> +
> +        self.sut_node.kill_cleanup_dpdk_apps()
> +
> +    def execute_tear_down(self):
> +        """
> +        execute suite tear_down function
> +        """
> +        try:
> +            self.tear_down()
> +        except Exception:
> +            self.logger.error("tear_down failed:\n" +
> traceback.format_exc())
> +            self.logger.warning(
> +                "tear down %s failed, might iterfere next case's result!"
> +                % self.running_case
> +            )
> +
> +    @staticmethod
> +    def get_testcases(testsuite_module_path: str) ->
> list[type["TestCase"]]:
> +        def is_testcase(object) -> bool:
> +            try:
> +                if issubclass(object, TestCase) and object != TestCase:
> +                    return True
> +            except TypeError:
> +                return False
> +            return False
> +
> +        try:
> +            testcase_module =
> importlib.import_module(testsuite_module_path)
> +        except ModuleNotFoundError as e:
> +            raise TestSuiteNotFound(
> +                f"Testsuite '{testsuite_module_path}' not found."
> +            ) from e
> +        return [
> +            testcase_class
> +            for _, testcase_class in inspect.getmembers(testcase_module,
> is_testcase)
> +        ]
> --
> 2.30.2
>
>

[-- Attachment #2: Type: text/html, Size: 26431 bytes --]