From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 98D1A41DC6; Fri, 3 Mar 2023 11:26:01 +0100 (CET) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 8B14742D3B; Fri, 3 Mar 2023 11:25:21 +0100 (CET) Received: from mail-ed1-f50.google.com (mail-ed1-f50.google.com [209.85.208.50]) by mails.dpdk.org (Postfix) with ESMTP id 76FCC42BDA for ; Fri, 3 Mar 2023 11:25:17 +0100 (CET) Received: by mail-ed1-f50.google.com with SMTP id cy23so8160751edb.12 for ; Fri, 03 Mar 2023 02:25:17 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=pantheon-tech.20210112.gappssmtp.com; s=20210112; t=1677839117; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=HLymHjIpgsykTzyW2JfzbHEXQEWkVSUvTLXIWTkD32Q=; b=J6mAecU2bp+kQCsBxmK5mgzGswBryUaqzWzamGsVQapAZuhDgrSq9GYhGGtFUGmqTJ n4b16eK8Hu0daMUamCWv40aNScPOTxJcGLvGM2eAZ2iC13NYk56lvb9z9Rl1+8l/3TZk 1Z2RKoyKsjJk8AxntiVwJ13JEz0fpKGVfFPzdUBB6yGfT9At0qra+qeTFHTQ5CkCUU70 ZCBv7S8RDJu/dBYJIDjcJZJ8KD8ZMs9YKyRWePEmyBxOyrfIQBQTgzpxef3/zXyvDjYl 5zi80W4340SZAoMHyD7WRrGPtflZHVJ9vgJRONIdC5xRzxBZNFB6Cqp/R3iPTPfZlPPO vOxw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; t=1677839117; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=HLymHjIpgsykTzyW2JfzbHEXQEWkVSUvTLXIWTkD32Q=; b=p4TCCr98xXeLp28VhgccnOoe1dHL5JIdsu0MEFy9T+FXCa2ytzGVAoi+beYkYs/e7O WqUUCsW+qhobskVUunRdM2vG+VhEOfPzxfl1bSzFntHWcQrTO2jFAlVQ6cAHP/dC+oiz L9lzLEMFgCwakKDXda3IWn6FTma5RBSgtx1kspEYn4J54/oKKOuBhhoyQmybDLyKGvH0 X/AWoDpi33w+4fVZlXUz9IZ7EiUnF48TpXo+tax3vcuCIz1QpmPTlXMAc68CqvqkBmt+ nwFn9+26eO52DWWAyH1Gl9LwBhaDdRfScnUKlMD0HS8R9fwTGuTsg8sAT/jxE6X0+OKb DX5Q== X-Gm-Message-State: AO0yUKVLDS0oHjEKgufhMDGsUzpWvcsaIFP7h/yLyoFXf/WY1Cphw2LD i3+M9OWBKFyjZTNW+RlNOTLb1g== X-Google-Smtp-Source: AK7set+L9lWK94girBdy/AR88IthkBtriJglmQhYFcY8Gp3d4gBqOUEJi5Yq/Vl8DKSAdJUpJnpPTg== X-Received: by 2002:aa7:c3d0:0:b0:4be:3918:9217 with SMTP id l16-20020aa7c3d0000000b004be39189217mr4836996edr.8.1677839117178; Fri, 03 Mar 2023 02:25:17 -0800 (PST) Received: from localhost.localdomain (ip-46.34.234.35.o2inet.sk. [46.34.234.35]) by smtp.gmail.com with ESMTPSA id j19-20020a508a93000000b004c3e3a6136dsm984028edj.21.2023.03.03.02.25.16 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Fri, 03 Mar 2023 02:25:17 -0800 (PST) From: =?UTF-8?q?Juraj=20Linke=C5=A1?= To: thomas@monjalon.net, Honnappa.Nagarahalli@arm.com, lijuan.tu@intel.com, bruce.richardson@intel.com, probb@iol.unh.edu Cc: dev@dpdk.org, =?UTF-8?q?Juraj=20Linke=C5=A1?= Subject: [PATCH v6 06/10] dts: add test suite module Date: Fri, 3 Mar 2023 11:25:03 +0100 Message-Id: <20230303102507.527790-7-juraj.linkes@pantheon.tech> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20230303102507.527790-1-juraj.linkes@pantheon.tech> References: <20230223152840.634183-1-juraj.linkes@pantheon.tech> <20230303102507.527790-1-juraj.linkes@pantheon.tech> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org The module implements the base class that all test suites inherit from. It implements methods common to all test suites. The derived test suites implement test cases and any particular setup needed for the suite or tests. Signed-off-by: Juraj Linkeš --- dts/conf.yaml | 2 + dts/framework/config/__init__.py | 4 + dts/framework/config/conf_yaml_schema.json | 10 + dts/framework/exception.py | 16 ++ dts/framework/settings.py | 24 +++ dts/framework/test_suite.py | 228 +++++++++++++++++++++ 6 files changed, 284 insertions(+) create mode 100644 dts/framework/test_suite.py diff --git a/dts/conf.yaml b/dts/conf.yaml index 6540a45ef7..75e33e8ccf 100644 --- a/dts/conf.yaml +++ b/dts/conf.yaml @@ -8,6 +8,8 @@ executions: cpu: native compiler: gcc compiler_wrapper: ccache + perf: false + func: true system_under_test: "SUT 1" nodes: - name: "SUT 1" diff --git a/dts/framework/config/__init__.py b/dts/framework/config/__init__.py index 0e5f493c5d..544fceca6a 100644 --- a/dts/framework/config/__init__.py +++ b/dts/framework/config/__init__.py @@ -131,6 +131,8 @@ def from_dict(d: dict) -> "BuildTargetConfiguration": @dataclass(slots=True, frozen=True) class ExecutionConfiguration: build_targets: list[BuildTargetConfiguration] + perf: bool + func: bool system_under_test: NodeConfiguration @staticmethod @@ -143,6 +145,8 @@ def from_dict(d: dict, node_map: dict) -> "ExecutionConfiguration": return ExecutionConfiguration( build_targets=build_targets, + perf=d["perf"], + func=d["func"], system_under_test=node_map[sut_name], ) diff --git a/dts/framework/config/conf_yaml_schema.json b/dts/framework/config/conf_yaml_schema.json index 56f93def36..878ca3aec2 100644 --- a/dts/framework/config/conf_yaml_schema.json +++ b/dts/framework/config/conf_yaml_schema.json @@ -164,6 +164,14 @@ }, "minimum": 1 }, + "perf": { + "type": "boolean", + "description": "Enable performance testing." + }, + "func": { + "type": "boolean", + "description": "Enable functional testing." + }, "system_under_test": { "$ref": "#/definitions/node_name" } @@ -171,6 +179,8 @@ "additionalProperties": false, "required": [ "build_targets", + "perf", + "func", "system_under_test" ] }, diff --git a/dts/framework/exception.py b/dts/framework/exception.py index b4545a5a40..ca353d98fc 100644 --- a/dts/framework/exception.py +++ b/dts/framework/exception.py @@ -24,6 +24,7 @@ class ErrorSeverity(IntEnum): REMOTE_CMD_EXEC_ERR = 3 SSH_ERR = 4 DPDK_BUILD_ERR = 10 + TESTCASE_VERIFY_ERR = 20 class DTSError(Exception): @@ -128,3 +129,18 @@ class DPDKBuildError(DTSError): """ severity: ClassVar[ErrorSeverity] = ErrorSeverity.DPDK_BUILD_ERR + + +class TestCaseVerifyError(DTSError): + """ + Used in test cases to verify the expected behavior. + """ + + value: str + severity: ClassVar[ErrorSeverity] = ErrorSeverity.TESTCASE_VERIFY_ERR + + def __init__(self, value: str): + self.value = value + + def __str__(self) -> str: + return repr(self.value) diff --git a/dts/framework/settings.py b/dts/framework/settings.py index f787187ade..4ccc98537d 100644 --- a/dts/framework/settings.py +++ b/dts/framework/settings.py @@ -66,6 +66,8 @@ class _Settings: skip_setup: bool dpdk_tarball_path: Path compile_timeout: float + test_cases: list + re_run: int def _get_parser() -> argparse.ArgumentParser: @@ -137,6 +139,26 @@ def _get_parser() -> argparse.ArgumentParser: help="[DTS_COMPILE_TIMEOUT] The timeout for compiling DPDK.", ) + parser.add_argument( + "--test-cases", + action=_env_arg("DTS_TESTCASES"), + default="", + required=False, + help="[DTS_TESTCASES] Comma-separated list of test cases to execute. " + "Unknown test cases will be silently ignored.", + ) + + parser.add_argument( + "--re-run", + "--re_run", + action=_env_arg("DTS_RERUN"), + default=0, + type=int, + required=False, + help="[DTS_RERUN] Re-run each test case the specified amount of times " + "if a test failure occurs", + ) + return parser @@ -156,6 +178,8 @@ def _get_settings() -> _Settings: skip_setup=(parsed_args.skip_setup == "Y"), dpdk_tarball_path=parsed_args.tarball, compile_timeout=parsed_args.compile_timeout, + test_cases=parsed_args.test_cases.split(",") if parsed_args.test_cases else [], + re_run=parsed_args.re_run, ) diff --git a/dts/framework/test_suite.py b/dts/framework/test_suite.py new file mode 100644 index 0000000000..9002d43297 --- /dev/null +++ b/dts/framework/test_suite.py @@ -0,0 +1,228 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2010-2014 Intel Corporation +# Copyright(c) 2023 PANTHEON.tech s.r.o. + +""" +Base class for creating DTS test cases. +""" + +import inspect +import re +from collections.abc import MutableSequence +from types import MethodType + +from .exception import SSHTimeoutError, TestCaseVerifyError +from .logger import DTSLOG, getLogger +from .settings import SETTINGS +from .testbed_model import SutNode + + +class TestSuite(object): + """ + The base TestSuite class provides methods for handling basic flow of a test suite: + * test case filtering and collection + * test suite setup/cleanup + * test setup/cleanup + * test case execution + * error handling and results storage + Test cases are implemented by derived classes. Test cases are all methods + starting with test_, further divided into performance test cases + (starting with test_perf_) and functional test cases (all other test cases). + By default, all test cases will be executed. A list of testcase str names + may be specified in conf.yaml or on the command line + to filter which test cases to run. + The methods named [set_up|tear_down]_[suite|test_case] should be overridden + in derived classes if the appropriate suite/test case fixtures are needed. + """ + + sut_node: SutNode + _logger: DTSLOG + _test_cases_to_run: list[str] + _func: bool + _errors: MutableSequence[Exception] + + def __init__( + self, + sut_node: SutNode, + test_cases: list[str], + func: bool, + errors: MutableSequence[Exception], + ): + self.sut_node = sut_node + self._logger = getLogger(self.__class__.__name__) + self._test_cases_to_run = test_cases + self._test_cases_to_run.extend(SETTINGS.test_cases) + self._func = func + self._errors = errors + + def set_up_suite(self) -> None: + """ + Set up test fixtures common to all test cases; this is done before + any test case is run. + """ + + def tear_down_suite(self) -> None: + """ + Tear down the previously created test fixtures common to all test cases. + """ + + def set_up_test_case(self) -> None: + """ + Set up test fixtures before each test case. + """ + + def tear_down_test_case(self) -> None: + """ + Tear down the previously created test fixtures after each test case. + """ + + def verify(self, condition: bool, failure_description: str) -> None: + if not condition: + self._logger.debug( + "A test case failed, showing the last 10 commands executed on SUT:" + ) + for command_res in self.sut_node.main_session.remote_session.history[-10:]: + self._logger.debug(command_res.command) + raise TestCaseVerifyError(failure_description) + + def run(self) -> None: + """ + Setup, execute and teardown the whole suite. + Suite execution consists of running all test cases scheduled to be executed. + A test cast run consists of setup, execution and teardown of said test case. + """ + test_suite_name = self.__class__.__name__ + + try: + self._logger.info(f"Starting test suite setup: {test_suite_name}") + self.set_up_suite() + self._logger.info(f"Test suite setup successful: {test_suite_name}") + except Exception as e: + self._logger.exception(f"Test suite setup ERROR: {test_suite_name}") + self._errors.append(e) + + else: + self._execute_test_suite() + + finally: + try: + self.tear_down_suite() + self.sut_node.kill_cleanup_dpdk_apps() + except Exception as e: + self._logger.exception(f"Test suite teardown ERROR: {test_suite_name}") + self._logger.warning( + f"Test suite '{test_suite_name}' teardown failed, " + f"the next test suite may be affected." + ) + self._errors.append(e) + + def _execute_test_suite(self) -> None: + """ + Execute all test cases scheduled to be executed in this suite. + """ + if self._func: + for test_case_method in self._get_functional_test_cases(): + all_attempts = SETTINGS.re_run + 1 + attempt_nr = 1 + while ( + not self._run_test_case(test_case_method) + and attempt_nr < all_attempts + ): + attempt_nr += 1 + self._logger.info( + f"Re-running FAILED test case '{test_case_method.__name__}'. " + f"Attempt number {attempt_nr} out of {all_attempts}." + ) + + def _get_functional_test_cases(self) -> list[MethodType]: + """ + Get all functional test cases. + """ + return self._get_test_cases(r"test_(?!perf_)") + + def _get_test_cases(self, test_case_regex: str) -> list[MethodType]: + """ + Return a list of test cases matching test_case_regex. + """ + self._logger.debug(f"Searching for test cases in {self.__class__.__name__}.") + filtered_test_cases = [] + for test_case_name, test_case in inspect.getmembers(self, inspect.ismethod): + if self._should_be_executed(test_case_name, test_case_regex): + filtered_test_cases.append(test_case) + cases_str = ", ".join((x.__name__ for x in filtered_test_cases)) + self._logger.debug( + f"Found test cases '{cases_str}' in {self.__class__.__name__}." + ) + return filtered_test_cases + + def _should_be_executed(self, test_case_name: str, test_case_regex: str) -> bool: + """ + Check whether the test case should be executed. + """ + match = bool(re.match(test_case_regex, test_case_name)) + if self._test_cases_to_run: + return match and test_case_name in self._test_cases_to_run + + return match + + def _run_test_case(self, test_case_method: MethodType) -> bool: + """ + Setup, execute and teardown a test case in this suite. + Exceptions are caught and recorded in logs. + """ + test_case_name = test_case_method.__name__ + result = False + + try: + # run set_up function for each case + self.set_up_test_case() + except SSHTimeoutError as e: + self._logger.exception(f"Test case setup FAILED: {test_case_name}") + self._errors.append(e) + except Exception as e: + self._logger.exception(f"Test case setup ERROR: {test_case_name}") + self._errors.append(e) + + else: + # run test case if setup was successful + result = self._execute_test_case(test_case_method) + + finally: + try: + self.tear_down_test_case() + except Exception as e: + self._logger.exception(f"Test case teardown ERROR: {test_case_name}") + self._logger.warning( + f"Test case '{test_case_name}' teardown failed, " + f"the next test case may be affected." + ) + self._errors.append(e) + result = False + + return result + + def _execute_test_case(self, test_case_method: MethodType) -> bool: + """ + Execute one test case and handle failures. + """ + test_case_name = test_case_method.__name__ + result = False + try: + self._logger.info(f"Starting test case execution: {test_case_name}") + test_case_method() + result = True + self._logger.info(f"Test case execution PASSED: {test_case_name}") + + except TestCaseVerifyError as e: + self._logger.exception(f"Test case execution FAILED: {test_case_name}") + self._errors.append(e) + except Exception as e: + self._logger.exception(f"Test case execution ERROR: {test_case_name}") + self._errors.append(e) + except KeyboardInterrupt: + self._logger.error( + f"Test case execution INTERRUPTED by user: {test_case_name}" + ) + raise KeyboardInterrupt("Stop DTS") + + return result -- 2.30.2