From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 9CAD141C8A; Mon, 13 Feb 2023 16:29:41 +0100 (CET) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id EEDD242D41; Mon, 13 Feb 2023 16:29:06 +0100 (CET) Received: from mail-wr1-f54.google.com (mail-wr1-f54.google.com [209.85.221.54]) by mails.dpdk.org (Postfix) with ESMTP id 6947A42D2C for ; Mon, 13 Feb 2023 16:28:58 +0100 (CET) Received: by mail-wr1-f54.google.com with SMTP id bu23so12633310wrb.8 for ; Mon, 13 Feb 2023 07:28:58 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=pantheon-tech.20210112.gappssmtp.com; s=20210112; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=2qXSBiTfUo82HVYud5/on+mciiK3m+44UBW4+mfU9LQ=; b=1FYLrWOLsokinLfTjACO1n8FuaLLAYpV3/vh2atVZ/CZTYSVUrKnFT1/NQpPks9kRm 0DcUvA8X/8e4cDG6fp/i6WbYXJH9zzBa6//6ysxzMUEBO0lEqdqB6Ycf8vee5wUX5VJ8 UkIZp/DSuUHSGXVHEQrXj7hF9dTprPtfngfvtjzUw/5qwklM7FgV08JuFoEQB4O0F53U GRgWIUDl/xBl1K6mUbNceSwZJeXtrTHjasgSUVsuu7HgBlx6w8iu+5INW2A6qUajXMBq NZiLPf1NjIMyAlcLZiZ7FtP801KGMbPyRZ1ZeU4Fi8IpZwOXTV77OOsxItJ+sJ1BWEN/ RmKg== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=2qXSBiTfUo82HVYud5/on+mciiK3m+44UBW4+mfU9LQ=; b=Vvodz+H72er4cYEwfv+CaBdC37lraPCvpC0qMojMorv6uKd9mBIngEoYHFBeaSlUbz y0A+OgrZMNqUit/W70knTLaWyaXiSV69OdnRWJvZtGg4/C1FMLjcAr2jy2xxoVRjnvNo D2lw2yKau8PDd52pdRCOWaf1ivBHhcVBTEufHBPgH/LgQ1EameZWV6Yay/JDUgganJe3 e4Ylqjq0PMkP0JRu64rFaZliXhUXL1j6709MKVgYACnjG3oob08B02lkdkuXIbO5fUcx lH19HjmyS/pRvvuYSCdYq4YlXY6OB8jVHcKX7RpOMNZnqh8LWL3mNErUJF/R9aj9m/dX lveg== X-Gm-Message-State: AO0yUKW1rvYbJePDcC/2pUj+Yq3r/zV3PCr5o9vLCE3xHgIn8texzOMT urILCNWo+4CD17becygJ9LWeqA== X-Google-Smtp-Source: AK7set+TyfnuQ6c79mn1hZd1UhddwkXmk0B6m4R4lK7/3IPhcPLJKthO9j+Umdukx6eOoTwjc0QCbQ== X-Received: by 2002:a5d:4b85:0:b0:2c5:60e2:ed6d with SMTP id b5-20020a5d4b85000000b002c560e2ed6dmr734316wrt.3.1676302138072; Mon, 13 Feb 2023 07:28:58 -0800 (PST) Received: from localhost.localdomain ([84.245.121.112]) by smtp.gmail.com with ESMTPSA id d13-20020adfe88d000000b002c54f4d0f71sm5848613wrm.38.2023.02.13.07.28.57 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Mon, 13 Feb 2023 07:28:57 -0800 (PST) From: =?UTF-8?q?Juraj=20Linke=C5=A1?= To: thomas@monjalon.net, Honnappa.Nagarahalli@arm.com, ohilyard@iol.unh.edu, lijuan.tu@intel.com, bruce.richardson@intel.com, wathsala.vithanage@arm.com, probb@iol.unh.edu Cc: dev@dpdk.org, =?UTF-8?q?Juraj=20Linke=C5=A1?= Subject: [PATCH v4 06/10] dts: add test suite module Date: Mon, 13 Feb 2023 16:28:42 +0100 Message-Id: <20230213152846.284191-7-juraj.linkes@pantheon.tech> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20230213152846.284191-1-juraj.linkes@pantheon.tech> References: <20230117154906.860916-1-juraj.linkes@pantheon.tech> <20230213152846.284191-1-juraj.linkes@pantheon.tech> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org The module implements the base class that all test suites inherit from. It implements methods common to all test suites. The derived test suites implement test cases and any particular setup needed for the suite or tests. Signed-off-by: Juraj Linkeš --- dts/conf.yaml | 2 + dts/framework/config/__init__.py | 4 + dts/framework/config/conf_yaml_schema.json | 10 + dts/framework/exception.py | 16 ++ dts/framework/settings.py | 24 +++ dts/framework/test_suite.py | 228 +++++++++++++++++++++ 6 files changed, 284 insertions(+) create mode 100644 dts/framework/test_suite.py diff --git a/dts/conf.yaml b/dts/conf.yaml index 32ec6e97a5..b606aa0df1 100644 --- a/dts/conf.yaml +++ b/dts/conf.yaml @@ -8,6 +8,8 @@ executions: cpu: native compiler: gcc compiler_wrapper: ccache + perf: false + func: true system_under_test: "SUT 1" nodes: - name: "SUT 1" diff --git a/dts/framework/config/__init__.py b/dts/framework/config/__init__.py index 0e5f493c5d..544fceca6a 100644 --- a/dts/framework/config/__init__.py +++ b/dts/framework/config/__init__.py @@ -131,6 +131,8 @@ def from_dict(d: dict) -> "BuildTargetConfiguration": @dataclass(slots=True, frozen=True) class ExecutionConfiguration: build_targets: list[BuildTargetConfiguration] + perf: bool + func: bool system_under_test: NodeConfiguration @staticmethod @@ -143,6 +145,8 @@ def from_dict(d: dict, node_map: dict) -> "ExecutionConfiguration": return ExecutionConfiguration( build_targets=build_targets, + perf=d["perf"], + func=d["func"], system_under_test=node_map[sut_name], ) diff --git a/dts/framework/config/conf_yaml_schema.json b/dts/framework/config/conf_yaml_schema.json index 56f93def36..878ca3aec2 100644 --- a/dts/framework/config/conf_yaml_schema.json +++ b/dts/framework/config/conf_yaml_schema.json @@ -164,6 +164,14 @@ }, "minimum": 1 }, + "perf": { + "type": "boolean", + "description": "Enable performance testing." + }, + "func": { + "type": "boolean", + "description": "Enable functional testing." + }, "system_under_test": { "$ref": "#/definitions/node_name" } @@ -171,6 +179,8 @@ "additionalProperties": false, "required": [ "build_targets", + "perf", + "func", "system_under_test" ] }, diff --git a/dts/framework/exception.py b/dts/framework/exception.py index b4545a5a40..ca353d98fc 100644 --- a/dts/framework/exception.py +++ b/dts/framework/exception.py @@ -24,6 +24,7 @@ class ErrorSeverity(IntEnum): REMOTE_CMD_EXEC_ERR = 3 SSH_ERR = 4 DPDK_BUILD_ERR = 10 + TESTCASE_VERIFY_ERR = 20 class DTSError(Exception): @@ -128,3 +129,18 @@ class DPDKBuildError(DTSError): """ severity: ClassVar[ErrorSeverity] = ErrorSeverity.DPDK_BUILD_ERR + + +class TestCaseVerifyError(DTSError): + """ + Used in test cases to verify the expected behavior. + """ + + value: str + severity: ClassVar[ErrorSeverity] = ErrorSeverity.TESTCASE_VERIFY_ERR + + def __init__(self, value: str): + self.value = value + + def __str__(self) -> str: + return repr(self.value) diff --git a/dts/framework/settings.py b/dts/framework/settings.py index f787187ade..4ccc98537d 100644 --- a/dts/framework/settings.py +++ b/dts/framework/settings.py @@ -66,6 +66,8 @@ class _Settings: skip_setup: bool dpdk_tarball_path: Path compile_timeout: float + test_cases: list + re_run: int def _get_parser() -> argparse.ArgumentParser: @@ -137,6 +139,26 @@ def _get_parser() -> argparse.ArgumentParser: help="[DTS_COMPILE_TIMEOUT] The timeout for compiling DPDK.", ) + parser.add_argument( + "--test-cases", + action=_env_arg("DTS_TESTCASES"), + default="", + required=False, + help="[DTS_TESTCASES] Comma-separated list of test cases to execute. " + "Unknown test cases will be silently ignored.", + ) + + parser.add_argument( + "--re-run", + "--re_run", + action=_env_arg("DTS_RERUN"), + default=0, + type=int, + required=False, + help="[DTS_RERUN] Re-run each test case the specified amount of times " + "if a test failure occurs", + ) + return parser @@ -156,6 +178,8 @@ def _get_settings() -> _Settings: skip_setup=(parsed_args.skip_setup == "Y"), dpdk_tarball_path=parsed_args.tarball, compile_timeout=parsed_args.compile_timeout, + test_cases=parsed_args.test_cases.split(",") if parsed_args.test_cases else [], + re_run=parsed_args.re_run, ) diff --git a/dts/framework/test_suite.py b/dts/framework/test_suite.py new file mode 100644 index 0000000000..0972a70c14 --- /dev/null +++ b/dts/framework/test_suite.py @@ -0,0 +1,228 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2010-2014 Intel Corporation +# Copyright(c) 2023 PANTHEON.tech s.r.o. + +""" +Base class for creating DTS test cases. +""" + +import inspect +import re +from collections.abc import MutableSequence +from types import MethodType + +from .exception import SSHTimeoutError, TestCaseVerifyError +from .logger import DTSLOG, getLogger +from .settings import SETTINGS +from .testbed_model import SutNode + + +class TestSuite(object): + """ + The base TestSuite class provides methods for handling basic flow of a test suite: + * test case filtering and collection + * test suite setup/cleanup + * test setup/cleanup + * test case execution + * error handling and results storage + Test cases are implemented by derived classes. Test cases are all methods + starting with test_, further divided into performance test cases + (starting with test_perf_) and functional test cases (all other test cases). + By default, all test cases will be executed. A list of testcase str names + may be specified in conf.yaml or on the command line + to filter which test cases to run. + The methods named [set_up|tear_down]_[suite|test_case] should be overridden + in derived classes if the appropriate suite/test case fixtures are needed. + """ + + sut_node: SutNode + _logger: DTSLOG + _test_cases_to_run: list[str] + _func: bool + _errors: MutableSequence[Exception] + + def __init__( + self, + sut_node: SutNode, + test_cases: list[str], + func: bool, + errors: MutableSequence[Exception], + ): + self.sut_node = sut_node + self._logger = getLogger(self.__class__.__name__) + self._test_cases_to_run = test_cases + self._test_cases_to_run.extend(SETTINGS.test_cases) + self._func = func + self._errors = errors + + def set_up_suite(self) -> None: + """ + Set up test fixtures common to all test cases; this is done before + any test case is run. + """ + + def tear_down_suite(self) -> None: + """ + Tear down the previously created test fixtures common to all test cases. + """ + + def set_up_test_case(self) -> None: + """ + Set up test fixtures before each test case. + """ + + def tear_down_test_case(self) -> None: + """ + Tear down the previously created test fixtures after each test case. + """ + + def verify(self, condition: bool, failure_description: str) -> None: + if not condition: + self._logger.debug( + "A test case failed, showing the last 10 commands executed on SUT:" + ) + for command_res in self.sut_node.main_session.remote_session.history[-10:]: + self._logger.debug(command_res.command) + raise TestCaseVerifyError(failure_description) + + def run(self) -> None: + """ + Setup, execute and teardown the whole suite. + Suite execution consists of running all test cases scheduled to be executed. + A test cast run consists of setup, execution and teardown of said test case. + """ + test_suite_name = self.__class__.__name__ + + try: + self._logger.info(f"Starting test suite setup: {test_suite_name}") + self.set_up_suite() + self._logger.info(f"Test suite setup successful: {test_suite_name}") + except Exception as e: + self._logger.exception(f"Test suite setup ERROR: {test_suite_name}") + self._errors.append(e) + + else: + self._execute_test_suite() + + finally: + try: + self.tear_down_suite() + self.sut_node.kill_cleanup_dpdk_apps() + except Exception as e: + self._logger.exception(f"Test suite teardown ERROR: {test_suite_name}") + self._logger.warning( + f"Test suite '{test_suite_name}' teardown failed, " + f"the next test suite may be affected." + ) + self._errors.append(e) + + def _execute_test_suite(self) -> None: + """ + Execute all test cases scheduled to be executed in this suite. + """ + if self._func: + for test_case_method in self._get_functional_test_cases(): + all_attempts = SETTINGS.re_run + 1 + attempt_nr = 1 + while ( + not self._run_test_case(test_case_method) + and attempt_nr <= all_attempts + ): + attempt_nr += 1 + self._logger.info( + f"Re-running FAILED test case '{test_case_method.__name__}'. " + f"Attempt number {attempt_nr} out of {all_attempts}." + ) + + def _get_functional_test_cases(self) -> list[MethodType]: + """ + Get all functional test cases. + """ + return self._get_test_cases(r"test_(?!perf_)") + + def _get_test_cases(self, test_case_regex: str) -> list[MethodType]: + """ + Return a list of test cases matching test_case_regex. + """ + self._logger.debug(f"Searching for test cases in {self.__class__.__name__}.") + filtered_test_cases = [] + for test_case_name, test_case in inspect.getmembers(self, inspect.ismethod): + if self._should_be_executed(test_case_name, test_case_regex): + filtered_test_cases.append(test_case) + cases_str = ", ".join((x.__name__ for x in filtered_test_cases)) + self._logger.debug( + f"Found test cases '{cases_str}' in {self.__class__.__name__}." + ) + return filtered_test_cases + + def _should_be_executed(self, test_case_name: str, test_case_regex: str) -> bool: + """ + Check whether the test case should be executed. + """ + match = bool(re.match(test_case_regex, test_case_name)) + if self._test_cases_to_run: + return match and test_case_name in self._test_cases_to_run + + return match + + def _run_test_case(self, test_case_method: MethodType) -> bool: + """ + Setup, execute and teardown a test case in this suite. + Exceptions are caught and recorded in logs. + """ + test_case_name = test_case_method.__name__ + result = False + + try: + # run set_up function for each case + self.set_up_test_case() + except SSHTimeoutError as e: + self._logger.exception(f"Test case setup FAILED: {test_case_name}") + self._errors.append(e) + except Exception as e: + self._logger.exception(f"Test case setup ERROR: {test_case_name}") + self._errors.append(e) + + else: + # run test case if setup was successful + result = self._execute_test_case(test_case_method) + + finally: + try: + self.tear_down_test_case() + except Exception as e: + self._logger.exception(f"Test case teardown ERROR: {test_case_name}") + self._logger.warning( + f"Test case '{test_case_name}' teardown failed, " + f"the next test case may be affected." + ) + self._errors.append(e) + result = False + + return result + + def _execute_test_case(self, test_case_method: MethodType) -> bool: + """ + Execute one test case and handle failures. + """ + test_case_name = test_case_method.__name__ + result = False + try: + self._logger.info(f"Starting test case execution: {test_case_name}") + test_case_method() + result = True + self._logger.info(f"Test case execution PASSED: {test_case_name}") + + except TestCaseVerifyError as e: + self._logger.exception(f"Test case execution FAILED: {test_case_name}") + self._errors.append(e) + except Exception as e: + self._logger.exception(f"Test case execution ERROR: {test_case_name}") + self._errors.append(e) + except KeyboardInterrupt: + self._logger.error( + f"Test case execution INTERRUPTED by user: {test_case_name}" + ) + raise KeyboardInterrupt("Stop DTS") + + return result -- 2.30.2