From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 1C76145848; Thu, 22 Aug 2024 18:40:11 +0200 (CEST) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id B392342F11; Thu, 22 Aug 2024 18:40:06 +0200 (CEST) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mails.dpdk.org (Postfix) with ESMTP id 5199842EE3 for ; Thu, 22 Aug 2024 18:40:04 +0200 (CEST) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id A0999FEC; Thu, 22 Aug 2024 09:40:29 -0700 (PDT) Received: from localhost.localdomain (unknown [10.57.86.28]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id CA5673F66E; Thu, 22 Aug 2024 09:40:02 -0700 (PDT) From: Luca Vizzarro To: dev@dpdk.org Cc: Honnappa Nagarahalli , =?UTF-8?q?Juraj=20Linke=C5=A1?= , Luca Vizzarro , Paul Szczepanek Subject: [PATCH 1/5] dts: add TestSuiteSpec class and discovery Date: Thu, 22 Aug 2024 17:39:37 +0100 Message-Id: <20240822163941.1390326-2-luca.vizzarro@arm.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240822163941.1390326-1-luca.vizzarro@arm.com> References: <20240822163941.1390326-1-luca.vizzarro@arm.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Currently there is a lack of a definition which identifies all the test suites available to test. This change intends to simplify the process to discover all the test suites and idenfity them. Signed-off-by: Luca Vizzarro Reviewed-by: Paul Szczepanek --- dts/framework/test_suite.py | 182 +++++++++++++++++++++++++++++++++++- 1 file changed, 181 insertions(+), 1 deletion(-) diff --git a/dts/framework/test_suite.py b/dts/framework/test_suite.py index 694b2eba65..972968b036 100644 --- a/dts/framework/test_suite.py +++ b/dts/framework/test_suite.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright(c) 2010-2014 Intel Corporation # Copyright(c) 2023 PANTHEON.tech s.r.o. +# Copyright(c) 2024 Arm Limited """Features common to all test suites. @@ -13,12 +14,22 @@ * Test case verification. """ +import inspect +import re +from dataclasses import dataclass +from enum import Enum, auto +from functools import cached_property +from importlib import import_module from ipaddress import IPv4Interface, IPv6Interface, ip_interface -from typing import ClassVar, Union +from pkgutil import iter_modules +from types import FunctionType, ModuleType +from typing import ClassVar, NamedTuple, Union +from pydantic.alias_generators import to_pascal from scapy.layers.inet import IP # type: ignore[import-untyped] from scapy.layers.l2 import Ether # type: ignore[import-untyped] from scapy.packet import Packet, Padding # type: ignore[import-untyped] +from typing_extensions import Self from framework.testbed_model.port import Port, PortLink from framework.testbed_model.sut_node import SutNode @@ -365,3 +376,172 @@ def _verify_l3_packet(self, received_packet: IP, expected_packet: IP) -> bool: if received_packet.src != expected_packet.src or received_packet.dst != expected_packet.dst: return False return True + + +class TestCaseVariant(Enum): + """Enum representing the variant of the test case.""" + + #: + FUNCTIONAL = auto() + #: + PERFORMANCE = auto() + + +class TestCase(NamedTuple): + """Tuple representing a test case.""" + + #: The name of the test case without prefix + name: str + #: The reference to the function + function_type: FunctionType + #: The test case variant + variant: TestCaseVariant + + +@dataclass +class TestSuiteSpec: + """A class defining the specification of a test suite. + + Apart from defining all the specs of a test suite, a helper function :meth:`discover_all` is + provided to automatically discover all the available test suites. + + Attributes: + module_name: The name of the test suite's module. + """ + + #: + TEST_SUITES_PACKAGE_NAME = "tests" + #: + TEST_SUITE_MODULE_PREFIX = "TestSuite_" + #: + TEST_SUITE_CLASS_PREFIX = "Test" + #: + TEST_CASE_METHOD_PREFIX = "test_" + #: + FUNC_TEST_CASE_REGEX = r"test_(?!perf_)" + #: + PERF_TEST_CASE_REGEX = r"test_perf_" + + module_name: str + + @cached_property + def name(self) -> str: + """The name of the test suite's module.""" + return self.module_name[len(self.TEST_SUITE_MODULE_PREFIX) :] + + @cached_property + def module_type(self) -> ModuleType: + """A reference to the test suite's module.""" + return import_module(f"{self.TEST_SUITES_PACKAGE_NAME}.{self.module_name}") + + @cached_property + def class_name(self) -> str: + """The name of the test suite's class.""" + return f"{self.TEST_SUITE_CLASS_PREFIX}{to_pascal(self.name)}" + + @cached_property + def class_type(self) -> type[TestSuite]: + """A reference to the test suite's class.""" + + def is_test_suite(obj) -> bool: + """Check whether `obj` is a :class:`TestSuite`. + + The `obj` is a subclass of :class:`TestSuite`, but not :class:`TestSuite` itself. + + Args: + obj: The object to be checked. + + Returns: + :data:`True` if `obj` is a subclass of `TestSuite`. + """ + try: + if issubclass(obj, TestSuite) and obj is not TestSuite: + return True + except TypeError: + return False + return False + + for class_name, class_type in inspect.getmembers(self.module_type, is_test_suite): + if class_name == self.class_name: + return class_type + + raise Exception("class not found in eligible test module") + + @cached_property + def test_cases(self) -> list[TestCase]: + """A list of all the available test cases.""" + test_cases = [] + + functions = inspect.getmembers(self.class_type, inspect.isfunction) + for fn_name, fn_type in functions: + if prefix := re.match(self.FUNC_TEST_CASE_REGEX, fn_name): + variant = TestCaseVariant.FUNCTIONAL + elif prefix := re.match(self.PERF_TEST_CASE_REGEX, fn_name): + variant = TestCaseVariant.PERFORMANCE + else: + continue + + name = fn_name[len(prefix.group(0)) :] + test_cases.append(TestCase(name, fn_type, variant)) + + return test_cases + + @classmethod + def discover_all( + cls, package_name: str | None = None, module_prefix: str | None = None + ) -> list[Self]: + """Discover all the test suites. + + The test suites are discovered in the provided `package_name`. The full module name, + expected under that package, is prefixed with `module_prefix`. + The module name is a standard filename with words separated with underscores. + For each module found, search for a :class:`TestSuite` class which starts + with `self.TEST_SUITE_CLASS_PREFIX`, continuing with the module name in PascalCase. + + The PascalCase convention applies to abbreviations, acronyms, initialisms and so on:: + + OS -> Os + TCP -> Tcp + + Args: + package_name: The name of the package where to find the test suites, if none is set the + constant :attr:`~TestSuiteSpec.TEST_SUITES_PACKAGE_NAME` is used instead. + module_prefix: The name prefix defining the test suite module, if none is set the + constant :attr:`~TestSuiteSpec.TEST_SUITE_MODULE_PREFIX` is used instead. + + Returns: + A list containing all the discovered test suites. + """ + if package_name is None: + package_name = cls.TEST_SUITES_PACKAGE_NAME + if module_prefix is None: + module_prefix = cls.TEST_SUITE_MODULE_PREFIX + + test_suites = [] + + test_suites_pkg = import_module(package_name) + for _, module_name, is_pkg in iter_modules(test_suites_pkg.__path__): + if not module_name.startswith(module_prefix) or is_pkg: + continue + + test_suite = cls(module_name) + try: + if test_suite.class_type: + test_suites.append(test_suite) + except Exception: + pass + + return test_suites + + +AVAILABLE_TEST_SUITES: list[TestSuiteSpec] = TestSuiteSpec.discover_all() +"""Constant to store all the available, discovered and imported test suites. + +The test suites should be gathered from this list to avoid importing more than once. +""" + + +def find_by_name(name: str) -> TestSuiteSpec | None: + """Find a requested test suite by name from the available ones.""" + test_suites = filter(lambda t: t.name == name, AVAILABLE_TEST_SUITES) + return next(test_suites, None) -- 2.34.1