From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id CE53C43723; Wed, 20 Dec 2023 11:34:01 +0100 (CET) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 4985F42EA9; Wed, 20 Dec 2023 11:33:43 +0100 (CET) Received: from mail-ed1-f50.google.com (mail-ed1-f50.google.com [209.85.208.50]) by mails.dpdk.org (Postfix) with ESMTP id 2696342DDB for ; Wed, 20 Dec 2023 11:33:42 +0100 (CET) Received: by mail-ed1-f50.google.com with SMTP id 4fb4d7f45d1cf-55114c073b8so6921072a12.1 for ; Wed, 20 Dec 2023 02:33:42 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=pantheon.tech; s=google; t=1703068422; x=1703673222; darn=dpdk.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=sEgXVa20St1fiPJh8PLIZO86zHVPpnGhA21XCRjdwn0=; b=Z1qAg9rgtdIfcTr/xvgEHR2cl64rkISv14q+hM/9zrmMOdvfB8OXcsUeUk5hhd4wwZ LklIfOQFeloRRh62qKAziiwtiMT34+Pemy71HmXcXgs9H5fcwOlcxDQ9eQ6RAG9SivPx 1MjrLw63X30GcfbO/JMQT39n1F3CH9LvzV6ODhwYMAHNeXD9c8quWs+WEge5DFV1AEkA Zl5w2Kl91+e1Md6Xa5TzGHiyUfPcKGraDlItlpJ7ZO9ddaD8SlIKaI+kToc5fA9+0UGs ndcaXgVmd0phYvJTWpcew0P9Xmvfck0k3MT/uLAH0pkU295PzMJJzSkLiyKUeVMGe7F/ 4wFA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1703068422; x=1703673222; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=sEgXVa20St1fiPJh8PLIZO86zHVPpnGhA21XCRjdwn0=; b=WM0SM/q9mhh/Ri7B1mU1BUBrcxLgYpnAnmzYsqsToQUmgVmcMR6Wjxq5Uu2OUkJpdq dqndamDl80ctLutpe05o+DpU+JlKhFuhQgVWGsEXn0dMcAl4F4Vc5tY9+bq1QWHcnQSy 0p/T6ntgTMoOjEZjHJcpHIjgV52Mr9G9Tggt4YWaGghuWTDLZcma3gsNYqP6WXztLmdy /RRYwpQ4fa00Y/P3O1ID4kC4rRZGnY3sf1U8QiDSz+5Ew/+T5RvfmLnSE0W8OmKgzB6Z 8ZG8eUW3w+8FITW8wetK7za6N5fKl2lcsTw6RKvLZF4R47Lyx2ruQHA8VDfX9OM+gVNM dYZA== X-Gm-Message-State: AOJu0YzkLpI10QEBdJN5O10KcpDRkoa0cS1pRigQsvTwjadHpBQi3rR6 EUlW1U3SmWgdaFW+EVg3UC8OAQ== X-Google-Smtp-Source: AGHT+IHLwYrLoV5+HmxKwlWknoVQdp/n+Zr633VXLp9oFyBtq9b7TXU36ZtZR2ZvWlYKMXrbrKcvMg== X-Received: by 2002:a50:9546:0:b0:552:84bb:c124 with SMTP id v6-20020a509546000000b0055284bbc124mr3538066eda.86.1703068421720; Wed, 20 Dec 2023 02:33:41 -0800 (PST) Received: from jlinkes-PT-Latitude-5530.pantheon.local (81.89.53.154.host.vnet.sk. [81.89.53.154]) by smtp.gmail.com with ESMTPSA id bd18-20020a056402207200b00542db304680sm12588981edb.63.2023.12.20.02.33.39 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 20 Dec 2023 02:33:40 -0800 (PST) From: =?UTF-8?q?Juraj=20Linke=C5=A1?= To: thomas@monjalon.net, Honnappa.Nagarahalli@arm.com, jspewock@iol.unh.edu, probb@iol.unh.edu, paul.szczepanek@arm.com, yoan.picchi@foss.arm.com, Luca.Vizzarro@arm.com Cc: dev@dpdk.org, =?UTF-8?q?Juraj=20Linke=C5=A1?= Subject: [RFC PATCH v1 3/5] dts: process test suites at the beginning of run Date: Wed, 20 Dec 2023 11:33:29 +0100 Message-Id: <20231220103331.60888-4-juraj.linkes@pantheon.tech> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20231220103331.60888-1-juraj.linkes@pantheon.tech> References: <20231220103331.60888-1-juraj.linkes@pantheon.tech> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org We initialize test suite/case objects at the start of the program and store them with custom execution config (add test case names) in Execution. This change helps identify errors with test suites earlier, and we have access to the right data when programs crash earlier. Signed-off-by: Juraj Linkeš --- dts/framework/config/__init__.py | 5 +- dts/framework/runner.py | 309 ++++++++++++++++++++--------- dts/framework/test_suite.py | 59 +----- dts/tests/TestSuite_smoke_tests.py | 2 +- 4 files changed, 217 insertions(+), 158 deletions(-) diff --git a/dts/framework/config/__init__.py b/dts/framework/config/__init__.py index 497847afb9..d65ac625f8 100644 --- a/dts/framework/config/__init__.py +++ b/dts/framework/config/__init__.py @@ -238,7 +238,6 @@ class ExecutionConfiguration: system_under_test_node: SutNodeConfiguration traffic_generator_node: TGNodeConfiguration vdevs: list[str] - skip_smoke_tests: bool @staticmethod def from_dict( @@ -247,9 +246,10 @@ def from_dict( build_targets: list[BuildTargetConfiguration] = list( map(BuildTargetConfiguration.from_dict, d["build_targets"]) ) + if not d.get("skip_smoke_tests", False): + d["test_suites"].insert(0, "smoke_tests") test_suites: list[TestSuiteConfig] = list(map(TestSuiteConfig.from_dict, d["test_suites"])) sut_name = d["system_under_test_node"]["node_name"] - skip_smoke_tests = d.get("skip_smoke_tests", False) assert sut_name in node_map, f"Unknown SUT {sut_name} in execution {d}" system_under_test_node = node_map[sut_name] assert isinstance( @@ -270,7 +270,6 @@ def from_dict( build_targets=build_targets, perf=d["perf"], func=d["func"], - skip_smoke_tests=skip_smoke_tests, test_suites=test_suites, system_under_test_node=system_under_test_node, traffic_generator_node=traffic_generator_node, diff --git a/dts/framework/runner.py b/dts/framework/runner.py index 5e145a8066..acc3342f0c 100644 --- a/dts/framework/runner.py +++ b/dts/framework/runner.py @@ -3,9 +3,14 @@ # Copyright(c) 2022-2023 PANTHEON.tech s.r.o. # Copyright(c) 2022-2023 University of New Hampshire +import importlib +import inspect import logging +import re import sys -from types import MethodType +from copy import deepcopy +from dataclasses import dataclass +from types import MethodType, ModuleType from .config import ( BuildTargetConfiguration, @@ -13,7 +18,12 @@ ExecutionConfiguration, TestSuiteConfig, ) -from .exception import BlockingTestSuiteError, SSHTimeoutError, TestCaseVerifyError +from .exception import ( + BlockingTestSuiteError, + ConfigurationError, + SSHTimeoutError, + TestCaseVerifyError, +) from .logger import DTSLOG, getLogger from .settings import SETTINGS from .test_result import ( @@ -24,25 +34,55 @@ TestCaseResult, TestSuiteResult, ) -from .test_suite import TestSuite, get_test_suites +from .test_suite import TestSuite from .testbed_model import SutNode, TGNode from .utils import check_dts_python_version +@dataclass +class TestSuiteSetup: + test_suite: type[TestSuite] + test_cases: list[MethodType] + + def processed_config(self) -> TestSuiteConfig: + return TestSuiteConfig( + test_suite=self.test_suite.__name__, + test_cases=[test_case.__name__ for test_case in self.test_cases], + ) + + +@dataclass +class Execution: + config: ExecutionConfiguration + test_suite_setups: list[TestSuiteSetup] + + def processed_config(self) -> ExecutionConfiguration: + """ + Creating copy of execution config witch add test-case names. + """ + modified_execution_config = deepcopy(self.config) + modified_execution_config.test_suites[:] = [ + test_suite.processed_config() for test_suite in self.test_suite_setups + ] + return modified_execution_config + + class DTSRunner: _logger: DTSLOG _result: DTSResult - _configuration: Configuration + _executions: list[Execution] def __init__(self, configuration: Configuration): self._logger = getLogger("DTSRunner") self._result = DTSResult(self._logger) - self._configuration = configuration + self._executions = create_executions(configuration.executions) def run(self): """ The main process of DTS. Runs all build targets in all executions from the main config file. + Suite execution consists of running all test cases scheduled to be executed. + A test case run consists of setup, execution and teardown of said test case. """ # check the python version of the server that run dts check_dts_python_version() @@ -50,22 +90,22 @@ def run(self): tg_nodes: dict[str, TGNode] = {} try: # for all Execution sections - for execution in self._configuration.executions: - sut_node = sut_nodes.get(execution.system_under_test_node.name) - tg_node = tg_nodes.get(execution.traffic_generator_node.name) + for execution in self._executions: + sut_node = sut_nodes.get(execution.config.system_under_test_node.name) + tg_node = tg_nodes.get(execution.config.traffic_generator_node.name) try: if not sut_node: - sut_node = SutNode(execution.system_under_test_node) + sut_node = SutNode(execution.config.system_under_test_node) sut_nodes[sut_node.name] = sut_node if not tg_node: - tg_node = TGNode(execution.traffic_generator_node) + tg_node = TGNode(execution.config.traffic_generator_node) tg_nodes[tg_node.name] = tg_node self._result.update_setup(Result.PASS) except Exception as e: - failed_node = execution.system_under_test_node.name + failed_node = execution.config.system_under_test_node.name if sut_node: - failed_node = execution.traffic_generator_node.name + failed_node = execution.config.traffic_generator_node.name self._logger.exception( f"The Creation of node {failed_node} failed." ) @@ -100,29 +140,34 @@ def _run_execution( self, sut_node: SutNode, tg_node: TGNode, - execution: ExecutionConfiguration, + execution: Execution, ) -> None: """ Run the given execution. This involves running the execution setup as well as running all build targets in the given execution. """ self._logger.info( - f"Running execution with SUT '{execution.system_under_test_node.name}'." + "Running execution with SUT " + f"'{execution.config.system_under_test_node.name}'." ) execution_result = self._result.add_execution(sut_node.config) execution_result.add_sut_info(sut_node.node_info) try: - sut_node.set_up_execution(execution) + sut_node.set_up_execution(execution.config) execution_result.update_setup(Result.PASS) except Exception as e: self._logger.exception("Execution setup failed.") execution_result.update_setup(Result.FAIL, e) else: - for build_target in execution.build_targets: + for build_target in execution.config.build_targets: self._run_build_target( - sut_node, tg_node, build_target, execution, execution_result + sut_node, + tg_node, + build_target, + execution, + execution_result, ) finally: @@ -138,7 +183,7 @@ def _run_build_target( sut_node: SutNode, tg_node: TGNode, build_target: BuildTargetConfiguration, - execution: ExecutionConfiguration, + execution: Execution, execution_result: ExecutionResult, ) -> None: """ @@ -171,7 +216,7 @@ def _run_test_suites( self, sut_node: SutNode, tg_node: TGNode, - execution: ExecutionConfiguration, + execution: Execution, build_target_result: BuildTargetResult, ) -> None: """ @@ -180,16 +225,18 @@ def _run_test_suites( If no subset is specified, run all test cases. """ end_build_target = False - if not execution.skip_smoke_tests: - execution.test_suites[:0] = [TestSuiteConfig.from_dict("smoke_tests")] - for test_suite_config in execution.test_suites: + for test_suite_setup in execution.test_suite_setups: try: self._run_test_suite( - sut_node, tg_node, execution, build_target_result, test_suite_config + sut_node, + tg_node, + test_suite_setup, + build_target_result, ) except BlockingTestSuiteError as e: self._logger.exception( - f"An error occurred within {test_suite_config.test_suite}. " + "An error occurred within " + f"{test_suite_setup.test_suite.__name__}. " "Skipping build target..." ) self._result.add_error(e) @@ -202,14 +249,10 @@ def _run_test_suite( self, sut_node: SutNode, tg_node: TGNode, - execution: ExecutionConfiguration, + test_suite_setup: TestSuiteSetup, build_target_result: BuildTargetResult, - test_suite_config: TestSuiteConfig, ) -> None: """Runs a single test suite. - Setup, execute and teardown the whole suite. - Suite execution consists of running all test cases scheduled to be executed. - A test cast run consists of setup, execution and teardown of said test case. Args: sut_node: Node to run tests on. @@ -220,84 +263,67 @@ def _run_test_suite( Raises: BlockingTestSuiteError: If a test suite that was marked as blocking fails. """ + test_suite = test_suite_setup.test_suite(sut_node, tg_node) + test_suite_name = test_suite_setup.test_suite.__name__ + test_suite_result = build_target_result.add_test_suite(test_suite_name) try: - full_suite_path = f"tests.TestSuite_{test_suite_config.test_suite}" - test_suite_classes = get_test_suites(full_suite_path) - suites_str = ", ".join((x.__name__ for x in test_suite_classes)) - self._logger.debug( - f"Found test suites '{suites_str}' in '{full_suite_path}'." - ) + self._logger.info(f"Starting test suite setup: {test_suite_name}") + test_suite.set_up_suite() + test_suite_result.update_setup(Result.PASS) + self._logger.info(f"Test suite setup successful: {test_suite_name}") except Exception as e: - self._logger.exception("An error occurred when searching for test suites.") - self._result.update_setup(Result.ERROR, e) + self._logger.exception(f"Test suite setup ERROR: {test_suite_name}") + test_suite_result.update_setup(Result.ERROR, e) else: - for test_suite_class in test_suite_classes: - test_suite = test_suite_class( - sut_node, tg_node, test_suite_config.test_cases - ) - - test_suite_name = test_suite.__class__.__name__ - test_suite_result = build_target_result.add_test_suite(test_suite_name) - try: - self._logger.info(f"Starting test suite setup: {test_suite_name}") - test_suite.set_up_suite() - test_suite_result.update_setup(Result.PASS) - self._logger.info(f"Test suite setup successful: {test_suite_name}") - except Exception as e: - self._logger.exception(f"Test suite setup ERROR: {test_suite_name}") - test_suite_result.update_setup(Result.ERROR, e) - - else: - self._execute_test_suite( - execution.func, test_suite, test_suite_result - ) + self._execute_test_suite( + test_suite, + test_suite_setup.test_cases, + test_suite_result, + ) - finally: - try: - test_suite.tear_down_suite() - sut_node.kill_cleanup_dpdk_apps() - test_suite_result.update_teardown(Result.PASS) - except Exception as e: - self._logger.exception( - f"Test suite teardown ERROR: {test_suite_name}" - ) - self._logger.warning( - f"Test suite '{test_suite_name}' teardown failed, " - f"the next test suite may be affected." - ) - test_suite_result.update_setup(Result.ERROR, e) - if ( - len(test_suite_result.get_errors()) > 0 - and test_suite.is_blocking - ): - raise BlockingTestSuiteError(test_suite_name) + finally: + try: + test_suite.tear_down_suite() + sut_node.kill_cleanup_dpdk_apps() + test_suite_result.update_teardown(Result.PASS) + except Exception as e: + self._logger.exception(f"Test suite teardown ERROR: {test_suite_name}") + self._logger.warning( + f"Test suite '{test_suite_name}' teardown failed, " + "the next test suite may be affected." + ) + test_suite_result.update_setup(Result.ERROR, e) + if len(test_suite_result.get_errors()) > 0 and test_suite.is_blocking: + raise BlockingTestSuiteError(test_suite_name) def _execute_test_suite( - self, func: bool, test_suite: TestSuite, test_suite_result: TestSuiteResult + self, + test_suite: TestSuite, + test_cases: list[MethodType], + test_suite_result: TestSuiteResult, ) -> None: """ Execute all test cases scheduled to be executed in this suite. """ - if func: - for test_case_method in test_suite._get_functional_test_cases(): - test_case_name = test_case_method.__name__ - test_case_result = test_suite_result.add_test_case(test_case_name) - all_attempts = SETTINGS.re_run + 1 - attempt_nr = 1 - self._run_test_case(test_suite, test_case_method, test_case_result) - while not test_case_result and attempt_nr < all_attempts: - attempt_nr += 1 - self._logger.info( - f"Re-running FAILED test case '{test_case_name}'. " - f"Attempt number {attempt_nr} out of {all_attempts}." - ) - self._run_test_case(test_suite, test_case_method, test_case_result) + for test_case_method in test_cases: + test_case_name = test_case_method.__name__ + test_case_result = test_suite_result.add_test_case(test_case_name) + all_attempts = SETTINGS.re_run + 1 + attempt_nr = 1 + self._run_test_case(test_case_method, test_suite, test_case_result) + while not test_case_result and attempt_nr < all_attempts: + attempt_nr += 1 + self._logger.info( + f"Re-running FAILED test case '{test_case_name}'. " + f"Attempt number {attempt_nr} out of {all_attempts}." + ) + self._run_test_case(test_case_method, test_suite, test_case_result) def _run_test_case( self, - test_suite: TestSuite, test_case_method: MethodType, + test_suite: TestSuite, test_case_result: TestCaseResult, ) -> None: """ @@ -305,7 +331,6 @@ def _run_test_case( Exceptions are caught and recorded in logs and results. """ test_case_name = test_case_method.__name__ - try: # run set_up function for each case test_suite.set_up_test_case() @@ -319,7 +344,7 @@ def _run_test_case( else: # run test case if setup was successful - self._execute_test_case(test_case_method, test_case_result) + self._execute_test_case(test_case_method, test_suite, test_case_result) finally: try: @@ -335,7 +360,10 @@ def _run_test_case( test_case_result.update(Result.ERROR) def _execute_test_case( - self, test_case_method: MethodType, test_case_result: TestCaseResult + self, + test_case_method: MethodType, + test_suite: TestSuite, + test_case_result: TestCaseResult, ) -> None: """ Execute one test case and handle failures. @@ -343,7 +371,7 @@ def _execute_test_case( test_case_name = test_case_method.__name__ try: self._logger.info(f"Starting test case execution: {test_case_name}") - test_case_method() + test_case_method(test_suite) test_case_result.update(Result.PASS) self._logger.info(f"Test case execution PASSED: {test_case_name}") @@ -371,3 +399,92 @@ def _exit_dts(self) -> None: logging.shutdown() sys.exit(self._result.get_return_code()) + + +def create_executions( + execution_configs: list[ExecutionConfiguration], +) -> list[Execution]: + executions: list[Execution] = [] + for execution_config in execution_configs: + test_suite_setups: list[TestSuiteSetup] = [] + + for test_suite_config in execution_config.test_suites: + testsuite_module_path = f"tests.TestSuite_{test_suite_config.test_suite}" + try: + suite_module = importlib.import_module(testsuite_module_path) + except ModuleNotFoundError as e: + raise ConfigurationError( + f"Test suite '{testsuite_module_path}' not found." + ) from e + + test_suite = _get_suite_class(suite_module, test_suite_config.test_suite) + + test_cases_to_run = test_suite_config.test_cases + test_cases_to_run.extend(SETTINGS.test_cases) + + test_cases = [] + if execution_config.func: + # add functional test cases + test_cases.extend( + _get_test_cases(test_suite, r"test_(?!perf_)", test_cases_to_run) + ) + + if execution_config.perf: + # add performance test cases + test_cases.extend( + _get_test_cases(test_suite, r"test_perf_", test_cases_to_run) + ) + + test_suite_setups.append( + TestSuiteSetup(test_suite=test_suite, test_cases=test_cases) + ) + + executions.append( + Execution( + config=execution_config, + test_suite_setups=test_suite_setups, + ) + ) + + return executions + + +def _get_suite_class(suite_module: ModuleType, suite_name: str) -> type[TestSuite]: + def is_test_suite(object) -> bool: + try: + if issubclass(object, TestSuite) and object is not TestSuite: + return True + except TypeError: + return False + return False + + suite_name_regex = suite_name.replace("_", "").lower() + for class_name, suite_class in inspect.getmembers(suite_module, is_test_suite): + if not class_name.startswith("Test"): + continue + + if suite_name_regex == class_name[4:].lower(): + return suite_class + raise ConfigurationError( + f"Cannot find valid test suite in {suite_module.__name__}." + ) + + +def _get_test_cases( + suite_class: type[TestSuite], test_case_regex: str, test_cases_to_run: list[str] +) -> list[MethodType]: + def should_be_executed(test_case_name: str) -> bool: + match = bool(re.match(test_case_regex, test_case_name)) + if test_cases_to_run: + return match and test_case_name in test_cases_to_run + + return match + + test_cases = [] + for test_case_name, test_case_method in inspect.getmembers( + suite_class, inspect.isfunction + ): + if should_be_executed(test_case_name): + test_cases.append(test_case_method) + + return test_cases diff --git a/dts/framework/test_suite.py b/dts/framework/test_suite.py index e96305deb0..e73206993d 100644 --- a/dts/framework/test_suite.py +++ b/dts/framework/test_suite.py @@ -6,20 +6,15 @@ Base class for creating DTS test cases. """ -import importlib -import inspect -import re from ipaddress import IPv4Interface, IPv6Interface, ip_interface -from types import MethodType from typing import Union from scapy.layers.inet import IP # type: ignore[import] from scapy.layers.l2 import Ether # type: ignore[import] from scapy.packet import Packet, Padding # type: ignore[import] -from .exception import ConfigurationError, TestCaseVerifyError +from .exception import TestCaseVerifyError from .logger import DTSLOG, getLogger -from .settings import SETTINGS from .testbed_model import SutNode, TGNode from .testbed_model.hw.port import Port, PortLink from .utils import get_packet_summaries @@ -47,7 +42,6 @@ class TestSuite(object): tg_node: TGNode is_blocking = False _logger: DTSLOG - _test_cases_to_run: list[str] _port_links: list[PortLink] _sut_port_ingress: Port _sut_port_egress: Port @@ -62,13 +56,10 @@ def __init__( self, sut_node: SutNode, tg_node: TGNode, - test_cases_to_run: list[str], ): self.sut_node = sut_node self.tg_node = tg_node self._logger = getLogger(self.__class__.__name__) - self._test_cases_to_run = test_cases_to_run - self._test_cases_to_run.extend(SETTINGS.test_cases) self._port_links = [] self._process_links() self._sut_port_ingress, self._tg_port_egress = ( @@ -268,51 +259,3 @@ def _verify_l3_packet(self, received_packet: IP, expected_packet: IP) -> bool: if received_packet.src != expected_packet.src or received_packet.dst != expected_packet.dst: return False return True - - def _get_functional_test_cases(self) -> list[MethodType]: - """ - Get all functional test cases. - """ - return self._get_test_cases(r"test_(?!perf_)") - - def _get_test_cases(self, test_case_regex: str) -> list[MethodType]: - """ - Return a list of test cases matching test_case_regex. - """ - self._logger.debug(f"Searching for test cases in {self.__class__.__name__}.") - filtered_test_cases = [] - for test_case_name, test_case in inspect.getmembers(self, inspect.ismethod): - if self._should_be_executed(test_case_name, test_case_regex): - filtered_test_cases.append(test_case) - cases_str = ", ".join((x.__name__ for x in filtered_test_cases)) - self._logger.debug(f"Found test cases '{cases_str}' in {self.__class__.__name__}.") - return filtered_test_cases - - def _should_be_executed(self, test_case_name: str, test_case_regex: str) -> bool: - """ - Check whether the test case should be executed. - """ - match = bool(re.match(test_case_regex, test_case_name)) - if self._test_cases_to_run: - return match and test_case_name in self._test_cases_to_run - - return match - - -def get_test_suites(testsuite_module_path: str) -> list[type[TestSuite]]: - def is_test_suite(object) -> bool: - try: - if issubclass(object, TestSuite) and object is not TestSuite: - return True - except TypeError: - return False - return False - - try: - testcase_module = importlib.import_module(testsuite_module_path) - except ModuleNotFoundError as e: - raise ConfigurationError(f"Test suite '{testsuite_module_path}' not found.") from e - return [ - test_suite_class - for _, test_suite_class in inspect.getmembers(testcase_module, is_test_suite) - ] diff --git a/dts/tests/TestSuite_smoke_tests.py b/dts/tests/TestSuite_smoke_tests.py index 8958f58dac..aa4bae5b17 100644 --- a/dts/tests/TestSuite_smoke_tests.py +++ b/dts/tests/TestSuite_smoke_tests.py @@ -10,7 +10,7 @@ from framework.utils import REGEX_FOR_PCI_ADDRESS -class SmokeTests(TestSuite): +class TestSmokeTests(TestSuite): is_blocking = True # dicts in this list are expected to have two keys: # "pci_address" and "current_driver" -- 2.34.1