From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id B9AD945848; Thu, 22 Aug 2024 18:40:37 +0200 (CEST) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 8B3DC42F15; Thu, 22 Aug 2024 18:40:10 +0200 (CEST) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mails.dpdk.org (Postfix) with ESMTP id 4AB8542F15 for ; Thu, 22 Aug 2024 18:40:07 +0200 (CEST) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id C2DA8DA7; Thu, 22 Aug 2024 09:40:32 -0700 (PDT) Received: from localhost.localdomain (unknown [10.57.86.28]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id ED95D3F66E; Thu, 22 Aug 2024 09:40:05 -0700 (PDT) From: Luca Vizzarro To: dev@dpdk.org Cc: Honnappa Nagarahalli , =?UTF-8?q?Juraj=20Linke=C5=A1?= , Luca Vizzarro , Paul Szczepanek Subject: [PATCH 4/5] dts: use TestSuiteSpec class imports Date: Thu, 22 Aug 2024 17:39:40 +0100 Message-Id: <20240822163941.1390326-5-luca.vizzarro@arm.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240822163941.1390326-1-luca.vizzarro@arm.com> References: <20240822163941.1390326-1-luca.vizzarro@arm.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org The introduction of TestSuiteSpec adds auto-discovery of test suites, which are also automatically imported. This causes double imports as the runner loads the test suites. This changes the behaviour of the runner to load the imported classes from TestSuiteSpec instead of importing them again. Signed-off-by: Luca Vizzarro Reviewed-by: Paul Szczepanek --- dts/framework/runner.py | 167 +++++++--------------------------------- 1 file changed, 27 insertions(+), 140 deletions(-) diff --git a/dts/framework/runner.py b/dts/framework/runner.py index 14e405aced..00b63cc292 100644 --- a/dts/framework/runner.py +++ b/dts/framework/runner.py @@ -2,6 +2,7 @@ # Copyright(c) 2010-2019 Intel Corporation # Copyright(c) 2022-2023 PANTHEON.tech s.r.o. # Copyright(c) 2022-2023 University of New Hampshire +# Copyright(c) 2024 Arm Limited """Test suite runner module. @@ -17,14 +18,11 @@ and the test case stage runs test cases individually. """ -import importlib -import inspect import os -import re import sys from pathlib import Path from types import FunctionType -from typing import Iterable, Sequence +from typing import Iterable from framework.testbed_model.sut_node import SutNode from framework.testbed_model.tg_node import TGNode @@ -38,12 +36,7 @@ TGNodeConfiguration, load_config, ) -from .exception import ( - BlockingTestSuiteError, - ConfigurationError, - SSHTimeoutError, - TestCaseVerifyError, -) +from .exception import BlockingTestSuiteError, SSHTimeoutError, TestCaseVerifyError from .logger import DTSLogger, DtsStage, get_dts_logger from .settings import SETTINGS from .test_result import ( @@ -55,7 +48,7 @@ TestSuiteResult, TestSuiteWithCases, ) -from .test_suite import TestSuite +from .test_suite import TestCase, TestCaseVariant, TestSuite class DTSRunner: @@ -217,11 +210,10 @@ def _get_test_suites_with_cases( func: bool, perf: bool, ) -> list[TestSuiteWithCases]: - """Test suites with test cases discovery. + """Get test suites with selected cases. - The test suites with test cases defined in the user configuration are discovered - and stored for future use so that we don't import the modules twice and so that - the list of test suites with test cases is available for recording right away. + The test suites with test cases defined in the user configuration are selected + and the corresponding functions and classes are gathered. Args: test_suite_configs: Test suite configurations. @@ -229,139 +221,34 @@ def _get_test_suites_with_cases( perf: Whether to include performance test cases in the final list. Returns: - The discovered test suites, each with test cases. + The test suites, each with test cases. """ test_suites_with_cases = [] for test_suite_config in test_suite_configs: - test_suite_class = self._get_test_suite_class(test_suite_config.test_suite_name) - test_cases = [] - func_test_cases, perf_test_cases = self._filter_test_cases( - test_suite_class, test_suite_config.test_cases_names - ) - if func: - test_cases.extend(func_test_cases) - if perf: - test_cases.extend(perf_test_cases) - - test_suites_with_cases.append( - TestSuiteWithCases(test_suite_class=test_suite_class, test_cases=test_cases) - ) - - return test_suites_with_cases - - def _get_test_suite_class(self, module_name: str) -> type[TestSuite]: - """Find the :class:`TestSuite` class in `module_name`. - - The full module name is `module_name` prefixed with `self._test_suite_module_prefix`. - The module name is a standard filename with words separated with underscores. - Search the `module_name` for a :class:`TestSuite` class which starts - with `self._test_suite_class_prefix`, continuing with CamelCase `module_name`. - The first matching class is returned. - - The CamelCase convention applies to abbreviations, acronyms, initialisms and so on:: - - OS -> Os - TCP -> Tcp - - Args: - module_name: The module name without prefix where to search for the test suite. - - Returns: - The found test suite class. - - Raises: - ConfigurationError: If the corresponding module is not found or - a valid :class:`TestSuite` is not found in the module. - """ - - def is_test_suite(object) -> bool: - """Check whether `object` is a :class:`TestSuite`. - - The `object` is a subclass of :class:`TestSuite`, but not :class:`TestSuite` itself. - - Args: - object: The object to be checked. - - Returns: - :data:`True` if `object` is a subclass of `TestSuite`. - """ - try: - if issubclass(object, TestSuite) and object is not TestSuite: - return True - except TypeError: - return False - return False - - testsuite_module_path = f"{self._test_suite_module_prefix}{module_name}" - try: - test_suite_module = importlib.import_module(testsuite_module_path) - except ModuleNotFoundError as e: - raise ConfigurationError( - f"Test suite module '{testsuite_module_path}' not found." - ) from e - - camel_case_suite_name = "".join( - [suite_word.capitalize() for suite_word in module_name.split("_")] - ) - full_suite_name_to_find = f"{self._test_suite_class_prefix}{camel_case_suite_name}" - for class_name, class_obj in inspect.getmembers(test_suite_module, is_test_suite): - if class_name == full_suite_name_to_find: - return class_obj - raise ConfigurationError( - f"Couldn't find any valid test suites in {test_suite_module.__name__}." - ) - - def _filter_test_cases( - self, test_suite_class: type[TestSuite], test_cases_to_run: Sequence[str] - ) -> tuple[list[FunctionType], list[FunctionType]]: - """Filter `test_cases_to_run` from `test_suite_class`. - - There are two rounds of filtering if `test_cases_to_run` is not empty. - The first filters `test_cases_to_run` from all methods of `test_suite_class`. - Then the methods are separated into functional and performance test cases. - If a method matches neither the functional nor performance name prefix, it's an error. - - Args: - test_suite_class: The class of the test suite. - test_cases_to_run: Test case names to filter from `test_suite_class`. - If empty, return all matching test cases. - - Returns: - A list of test case methods that should be executed. + test_suite_spec = test_suite_config.test_suite_spec + test_suite_class = test_suite_spec.class_type + + filtered_test_cases: list[TestCase] = [ + test_case + for test_case in test_suite_spec.test_cases + if not test_suite_config.test_cases_names + or test_case.name in test_suite_config.test_cases_names + ] - Raises: - ConfigurationError: If a test case from `test_cases_to_run` is not found - or it doesn't match either the functional nor performance name prefix. - """ - func_test_cases = [] - perf_test_cases = [] - name_method_tuples = inspect.getmembers(test_suite_class, inspect.isfunction) - if test_cases_to_run: - name_method_tuples = [ - (name, method) for name, method in name_method_tuples if name in test_cases_to_run + selected_test_cases: list[FunctionType] = [ + test_case.function_type # type: ignore[misc] + for test_case in filtered_test_cases + if (func and test_case.variant == TestCaseVariant.FUNCTIONAL) + or (perf and test_case.variant == TestCaseVariant.PERFORMANCE) ] - if len(name_method_tuples) < len(test_cases_to_run): - missing_test_cases = set(test_cases_to_run) - { - name for name, _ in name_method_tuples - } - raise ConfigurationError( - f"Test cases {missing_test_cases} not found among methods " - f"of {test_suite_class.__name__}." - ) - for test_case_name, test_case_method in name_method_tuples: - if re.match(self._func_test_case_regex, test_case_name): - func_test_cases.append(test_case_method) - elif re.match(self._perf_test_case_regex, test_case_name): - perf_test_cases.append(test_case_method) - elif test_cases_to_run: - raise ConfigurationError( - f"Method '{test_case_name}' matches neither " - f"a functional nor a performance test case name." + test_suites_with_cases.append( + TestSuiteWithCases( + test_suite_class=test_suite_class, test_cases=selected_test_cases ) - - return func_test_cases, perf_test_cases + ) + return test_suites_with_cases def _connect_nodes_and_run_test_run( self, -- 2.34.1