From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 09AE945C2D; Thu, 31 Oct 2024 20:32:26 +0100 (CET) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id E95DB4028A; Thu, 31 Oct 2024 20:32:25 +0100 (CET) Received: from mail-lf1-f48.google.com (mail-lf1-f48.google.com [209.85.167.48]) by mails.dpdk.org (Postfix) with ESMTP id 15E1F40264 for ; Thu, 31 Oct 2024 20:32:25 +0100 (CET) Received: by mail-lf1-f48.google.com with SMTP id 2adb3069b0e04-539f1d96668so181422e87.3 for ; Thu, 31 Oct 2024 12:32:25 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=iol.unh.edu; s=unh-iol; t=1730403144; x=1731007944; darn=dpdk.org; h=content-transfer-encoding:cc:to:subject:message-id:date:from :in-reply-to:references:mime-version:from:to:cc:subject:date :message-id:reply-to; bh=98hpLPFdUFVM5E4zQL+VgQ/Pe2QL87CLiLocOuQ61qQ=; b=c+CJgOoUGuCezDSLNsNTViiFpGcr/3P3uikjUFCMDbDDphaHr7FXbqglGwhtb1T1Yn da+AtF7G2Sd0ScnwfW4DEsnH4tRD/oQVFrLetRonUok+4WKI/QNAQkj69rmH1enws/Tc pD5Z1yFTwRq5FLce/9aIdAF0j2tkJr8OphCzE= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1730403144; x=1731007944; h=content-transfer-encoding:cc:to:subject:message-id:date:from :in-reply-to:references:mime-version:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=98hpLPFdUFVM5E4zQL+VgQ/Pe2QL87CLiLocOuQ61qQ=; b=i/jukNgYqW4QSWrLscrNYfFVOUXWxiyw7veeETFrIa5WwJaxwJJ7F3BonPbysmNxnP qWNmnDEtYcp92Px2eAl7yaqb3RdZP+jJ7JFkuPd4IWxlEVfy+51mydEe4T0j6g5tcS2t XfxdMJZfN8bpy6mqeJKYUSJGn7e2Ivwfv0aadDIoFEBsBleIVlsHS6oHfbcbIHhVN22O Z0OntE71WfV0wg3qBJezZyuVndhV32xwuoilpL0YfXoExWg4lC9cMTNobKSUgwYNwYGL NazQz750Rl+gIjQNa3Bg48+OrS0Z5Uh9HzBdwED2qJmYOdYx5+jQLcj3ZrnIxZC24wTS y83A== X-Gm-Message-State: AOJu0YzBpQtOsyfEBXuGeCa1HmCTuA94+DGThU36QHjAxWbIQ2/jNfiE bOhJ59EXNA6Kev4rIBtq9RwWWGTXk/AZSA7rcoRS1zwZzoWXU3l/EGJZ0DUF2ZkzHBFpFat4qLu eYn3qaI0Tw8/pPNhv10mcORy35kYikQV5QnHE4w== X-Google-Smtp-Source: AGHT+IF8JrtCmty2Ztn5oiH0yI+MUtKKa887uQQoawnRKDvdF14NqCO639t38QU4M+sJ+84Oxk4I0xIb+Is7ZueuZ28= X-Received: by 2002:a2e:a9a4:0:b0:2fa:cf48:14a3 with SMTP id 38308e7fff4ca-2fcbe062327mr32132181fa.7.1730403143955; Thu, 31 Oct 2024 12:32:23 -0700 (PDT) MIME-Version: 1.0 References: <20240822163941.1390326-1-luca.vizzarro@arm.com> <20241028174949.3283701-1-luca.vizzarro@arm.com> <20241028174949.3283701-3-luca.vizzarro@arm.com> In-Reply-To: <20241028174949.3283701-3-luca.vizzarro@arm.com> From: Nicholas Pratte Date: Thu, 31 Oct 2024 15:32:12 -0400 Message-ID: Subject: Re: [PATCH v4 2/8] dts: add TestSuiteSpec class and discovery To: Luca Vizzarro Cc: dev@dpdk.org, Paul Szczepanek , Patrick Robb Content-Type: text/plain; charset="UTF-8" Content-Transfer-Encoding: quoted-printable X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Reviewed-by: Nicholas Pratte On Mon, Oct 28, 2024 at 1:51=E2=80=AFPM Luca Vizzarro wrote: > > Currently there is a lack of a definition which identifies all the test > suites available to test. This change intends to simplify the process to > discover all the test suites and idenfity them. > > Signed-off-by: Luca Vizzarro > Reviewed-by: Paul Szczepanek > --- > dts/framework/runner.py | 2 +- > dts/framework/test_suite.py | 189 +++++++++++++++++++--- > dts/framework/testbed_model/capability.py | 12 +- > 3 files changed, 177 insertions(+), 26 deletions(-) > > diff --git a/dts/framework/runner.py b/dts/framework/runner.py > index 8bbe698eaf..195622c653 100644 > --- a/dts/framework/runner.py > +++ b/dts/framework/runner.py > @@ -225,7 +225,7 @@ def _get_test_suites_with_cases( > for test_suite_config in test_suite_configs: > test_suite_class =3D self._get_test_suite_class(test_suite_c= onfig.test_suite) > test_cases: list[type[TestCase]] =3D [] > - func_test_cases, perf_test_cases =3D test_suite_class.get_te= st_cases( > + func_test_cases, perf_test_cases =3D test_suite_class.filter= _test_cases( > test_suite_config.test_cases > ) > if func: > diff --git a/dts/framework/test_suite.py b/dts/framework/test_suite.py > index cbe3b30ffc..936eb2cede 100644 > --- a/dts/framework/test_suite.py > +++ b/dts/framework/test_suite.py > @@ -1,6 +1,7 @@ > # SPDX-License-Identifier: BSD-3-Clause > # Copyright(c) 2010-2014 Intel Corporation > # Copyright(c) 2023 PANTHEON.tech s.r.o. > +# Copyright(c) 2024 Arm Limited > > """Features common to all test suites. > > @@ -16,13 +17,20 @@ > import inspect > from collections import Counter > from collections.abc import Callable, Sequence > +from dataclasses import dataclass > from enum import Enum, auto > +from functools import cached_property > +from importlib import import_module > from ipaddress import IPv4Interface, IPv6Interface, ip_interface > +from pkgutil import iter_modules > +from types import ModuleType > from typing import ClassVar, Protocol, TypeVar, Union, cast > > +from pydantic.alias_generators import to_pascal > from scapy.layers.inet import IP # type: ignore[import-untyped] > from scapy.layers.l2 import Ether # type: ignore[import-untyped] > from scapy.packet import Packet, Padding, raw # type: ignore[import-unt= yped] > +from typing_extensions import Self > > from framework.testbed_model.capability import TestProtocol > from framework.testbed_model.port import Port > @@ -33,7 +41,7 @@ > PacketFilteringConfig, > ) > > -from .exception import ConfigurationError, TestCaseVerifyError > +from .exception import ConfigurationError, InternalError, TestCaseVerify= Error > from .logger import DTSLogger, get_dts_logger > from .utils import get_packet_summaries > > @@ -112,10 +120,24 @@ def __init__( > self._tg_ip_address_ingress =3D ip_interface("192.168.101.3/24") > > @classmethod > - def get_test_cases( > + def get_test_cases(cls) -> list[type["TestCase"]]: > + """A list of all the available test cases.""" > + > + def is_test_case(function: Callable) -> bool: > + if inspect.isfunction(function): > + # TestCase is not used at runtime, so we can't use isins= tance() with `function`. > + # But function.test_type exists. > + if hasattr(function, "test_type"): > + return isinstance(function.test_type, TestCaseType) > + return False > + > + return [test_case for _, test_case in inspect.getmembers(cls, is= _test_case)] > + > + @classmethod > + def filter_test_cases( > cls, test_case_sublist: Sequence[str] | None =3D None > ) -> tuple[set[type["TestCase"]], set[type["TestCase"]]]: > - """Filter `test_case_subset` from this class. > + """Filter `test_case_sublist` from this class. > > Test cases are regular (or bound) methods decorated with :func:`= func_test` > or :func:`perf_test`. > @@ -129,17 +151,8 @@ def get_test_cases( > as methods are bound to instances and this method only has a= ccess to the class. > > Raises: > - ConfigurationError: If a test case from `test_case_subset` i= s not found. > + ConfigurationError: If a test case from `test_case_sublist` = is not found. > """ > - > - def is_test_case(function: Callable) -> bool: > - if inspect.isfunction(function): > - # TestCase is not used at runtime, so we can't use isins= tance() with `function`. > - # But function.test_type exists. > - if hasattr(function, "test_type"): > - return isinstance(function.test_type, TestCaseType) > - return False > - > if test_case_sublist is None: > test_case_sublist =3D [] > > @@ -149,22 +162,22 @@ def is_test_case(function: Callable) -> bool: > func_test_cases =3D set() > perf_test_cases =3D set() > > - for test_case_name, test_case_function in inspect.getmembers(cls= , is_test_case): > - if test_case_name in test_case_sublist_copy: > + for test_case in cls.get_test_cases(): > + if test_case.name in test_case_sublist_copy: > # if test_case_sublist_copy is non-empty, remove the fou= nd test case > # so that we can look at the remainder at the end > - test_case_sublist_copy.remove(test_case_name) > + test_case_sublist_copy.remove(test_case.name) > elif test_case_sublist: > # the original list not being empty means we're filterin= g test cases > - # since we didn't remove test_case_name in the previous = branch, > + # since we didn't remove test_case.name in the previous = branch, > # it doesn't match the filter and we don't want to remov= e it > continue > > - match test_case_function.test_type: > + match test_case.test_type: > case TestCaseType.PERFORMANCE: > - perf_test_cases.add(test_case_function) > + perf_test_cases.add(test_case) > case TestCaseType.FUNCTIONAL: > - func_test_cases.add(test_case_function) > + func_test_cases.add(test_case) > > if test_case_sublist_copy: > raise ConfigurationError( > @@ -536,6 +549,8 @@ class TestCase(TestProtocol, Protocol[TestSuiteMethod= Type]): > test case function to :class:`TestCase` and sets common variables. > """ > > + #: > + name: ClassVar[str] > #: > test_type: ClassVar[TestCaseType] > #: necessary for mypy so that it can treat this class as the functio= n it's shadowing > @@ -560,6 +575,7 @@ def make_decorator( > > def _decorator(func: TestSuiteMethodType) -> type[TestCase]: > test_case =3D cast(type[TestCase], func) > + test_case.name =3D func.__name__ > test_case.skip =3D cls.skip > test_case.skip_reason =3D cls.skip_reason > test_case.required_capabilities =3D set() > @@ -575,3 +591,136 @@ def _decorator(func: TestSuiteMethodType) -> type[T= estCase]: > func_test: Callable =3D TestCase.make_decorator(TestCaseType.FUNCTIONAL) > #: The decorator for performance test cases. > perf_test: Callable =3D TestCase.make_decorator(TestCaseType.PERFORMANCE= ) > + > + > +@dataclass > +class TestSuiteSpec: > + """A class defining the specification of a test suite. > + > + Apart from defining all the specs of a test suite, a helper function= :meth:`discover_all` is > + provided to automatically discover all the available test suites. > + > + Attributes: > + module_name: The name of the test suite's module. > + """ > + > + #: > + TEST_SUITES_PACKAGE_NAME =3D "tests" > + #: > + TEST_SUITE_MODULE_PREFIX =3D "TestSuite_" > + #: > + TEST_SUITE_CLASS_PREFIX =3D "Test" > + #: > + TEST_CASE_METHOD_PREFIX =3D "test_" > + #: > + FUNC_TEST_CASE_REGEX =3D r"test_(?!perf_)" > + #: > + PERF_TEST_CASE_REGEX =3D r"test_perf_" > + > + module_name: str > + > + @cached_property > + def name(self) -> str: > + """The name of the test suite's module.""" > + return self.module_name[len(self.TEST_SUITE_MODULE_PREFIX) :] > + > + @cached_property > + def module(self) -> ModuleType: > + """A reference to the test suite's module.""" > + return import_module(f"{self.TEST_SUITES_PACKAGE_NAME}.{self.mod= ule_name}") > + > + @cached_property > + def class_name(self) -> str: > + """The name of the test suite's class.""" > + return f"{self.TEST_SUITE_CLASS_PREFIX}{to_pascal(self.name)}" > + > + @cached_property > + def class_obj(self) -> type[TestSuite]: > + """A reference to the test suite's class.""" > + > + def is_test_suite(obj) -> bool: > + """Check whether `obj` is a :class:`TestSuite`. > + > + The `obj` is a subclass of :class:`TestSuite`, but not :clas= s:`TestSuite` itself. > + > + Args: > + obj: The object to be checked. > + > + Returns: > + :data:`True` if `obj` is a subclass of `TestSuite`. > + """ > + try: > + if issubclass(obj, TestSuite) and obj is not TestSuite: > + return True > + except TypeError: > + return False > + return False > + > + for class_name, class_obj in inspect.getmembers(self.module, is_= test_suite): > + if class_name =3D=3D self.class_name: > + return class_obj > + > + raise InternalError( > + f"Expected class {self.class_name} not found in module {self= .module_name}." > + ) > + > + @classmethod > + def discover_all( > + cls, package_name: str | None =3D None, module_prefix: str | Non= e =3D None > + ) -> list[Self]: > + """Discover all the test suites. > + > + The test suites are discovered in the provided `package_name`. T= he full module name, > + expected under that package, is prefixed with `module_prefix`. > + The module name is a standard filename with words separated with= underscores. > + For each module found, search for a :class:`TestSuite` class whi= ch starts > + with :attr:`~TestSuiteSpec.TEST_SUITE_CLASS_PREFIX`, continuing = with the module name in > + PascalCase. > + > + The PascalCase convention applies to abbreviations, acronyms, in= itialisms and so on:: > + > + OS -> Os > + TCP -> Tcp > + > + Args: > + package_name: The name of the package where to find the test= suites. If :data:`None`, > + the :attr:`~TestSuiteSpec.TEST_SUITES_PACKAGE_NAME` is u= sed. > + module_prefix: The name prefix defining the test suite modul= e. If :data:`None`, the > + :attr:`~TestSuiteSpec.TEST_SUITE_MODULE_PREFIX` constant= is used. > + > + Returns: > + A list containing all the discovered test suites. > + """ > + if package_name is None: > + package_name =3D cls.TEST_SUITES_PACKAGE_NAME > + if module_prefix is None: > + module_prefix =3D cls.TEST_SUITE_MODULE_PREFIX > + > + test_suites =3D [] > + > + test_suites_pkg =3D import_module(package_name) > + for _, module_name, is_pkg in iter_modules(test_suites_pkg.__pat= h__): > + if not module_name.startswith(module_prefix) or is_pkg: > + continue > + > + test_suite =3D cls(module_name) > + try: > + if test_suite.class_obj: > + test_suites.append(test_suite) > + except InternalError as err: > + get_dts_logger().warning(err) > + > + return test_suites > + > + > +AVAILABLE_TEST_SUITES: list[TestSuiteSpec] =3D TestSuiteSpec.discover_al= l() > +"""Constant to store all the available, discovered and imported test sui= tes. > + > +The test suites should be gathered from this list to avoid importing mor= e than once. > +""" > + > + > +def find_by_name(name: str) -> TestSuiteSpec | None: > + """Find a requested test suite by name from the available ones.""" > + test_suites =3D filter(lambda t: t.name =3D=3D name, AVAILABLE_TEST_= SUITES) > + return next(test_suites, None) > diff --git a/dts/framework/testbed_model/capability.py b/dts/framework/te= stbed_model/capability.py > index 2207957a7a..0d5f0e0b32 100644 > --- a/dts/framework/testbed_model/capability.py > +++ b/dts/framework/testbed_model/capability.py > @@ -47,9 +47,9 @@ def test_scatter_mbuf_2048(self): > > import inspect > from abc import ABC, abstractmethod > -from collections.abc import MutableSet, Sequence > +from collections.abc import MutableSet > from dataclasses import dataclass > -from typing import Callable, ClassVar, Protocol > +from typing import TYPE_CHECKING, Callable, ClassVar, Protocol > > from typing_extensions import Self > > @@ -66,6 +66,9 @@ def test_scatter_mbuf_2048(self): > from .sut_node import SutNode > from .topology import Topology, TopologyType > > +if TYPE_CHECKING: > + from framework.test_suite import TestCase > + > > class Capability(ABC): > """The base class for various capabilities. > @@ -354,8 +357,7 @@ def set_required(self, test_case_or_suite: type["Test= Protocol"]) -> None: > if inspect.isclass(test_case_or_suite): > if self.topology_type is not TopologyType.default: > self.add_to_required(test_case_or_suite) > - func_test_cases, perf_test_cases =3D test_case_or_suite.= get_test_cases() > - for test_case in func_test_cases | perf_test_cases: > + for test_case in test_case_or_suite.get_test_cases(): > if test_case.topology_type.topology_type is Topology= Type.default: > # test case topology has not been set, use the o= ne set by the test suite > self.add_to_required(test_case) > @@ -446,7 +448,7 @@ class TestProtocol(Protocol): > required_capabilities: ClassVar[set[Capability]] =3D set() > > @classmethod > - def get_test_cases(cls, test_case_sublist: Sequence[str] | None =3D = None) -> tuple[set, set]: > + def get_test_cases(cls) -> list[type["TestCase"]]: > """Get test cases. Should be implemented by subclasses containin= g test cases. > > Raises: > -- > 2.43.0 >