From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id B947F45A11; Mon, 23 Sep 2024 17:02:48 +0200 (CEST) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 8ADB6402EA; Mon, 23 Sep 2024 17:02:21 +0200 (CEST) Received: from mail-lf1-f52.google.com (mail-lf1-f52.google.com [209.85.167.52]) by mails.dpdk.org (Postfix) with ESMTP id DE3D0402CF for ; Mon, 23 Sep 2024 17:02:17 +0200 (CEST) Received: by mail-lf1-f52.google.com with SMTP id 2adb3069b0e04-5365a9574b6so6140982e87.1 for ; Mon, 23 Sep 2024 08:02:17 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=pantheon.tech; s=google; t=1727103737; x=1727708537; darn=dpdk.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=M2WlhsqbuotPBCfK7ViC1LCucD/8iUNq/6SybSOaU9s=; b=oNs/FZPULmEwaRvLi1yfuU1TBIvsGBP5M23XVM9cqvR/Z2u1nXXuWFvK141N6d1wM6 rltSPjXEh/Z2zbQbHxUEDoqSL7MT9pPt143vzmISIYykwTCcHkvAvHHirHPk2AEfnXK2 W9B09irPjzmiK89pFyOl1HwkM3IP9sTc1XAuWeIFHT09fHkt8X8ForiGnEDx0lnutATw e2d25DrEdTbGWpyLR6JVbMvtroavVzoSYRcXTjAca4iDx/j/gbq96+smo81cfb45Y5jk Y5ZtP9LZ9fCJxTvFPuZPSv7+ivTvHTeXldgS3r4UkyylwKlht/9rr/z5ScD3P0swACzF eIdw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1727103737; x=1727708537; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=M2WlhsqbuotPBCfK7ViC1LCucD/8iUNq/6SybSOaU9s=; b=ibpBftK+kpL8PwRhUd2cn2CBgfoabuqCR5+IVEP05yydggi5MFBcMx1gOmcTwkn03M dqV45H3+qq+oGH4Sp7XWXkbLsJNDhSjWY1BhsoX5idwRhIOCegdXXy0M5W6S+q4scETL YN/mC/8Ciz588/6EjhJp0g9YriYWng8IMd6dcmsRzMIUNmMzjAh45LBJH0h3+pr+xr6B Kcz8qXXKAUM/u7e1ncW+KDO/ueRgeyIB9//89MPvV1Zi2b3UAXwbVdr26bw6ksAeOLBw SiyVfUw88ZcfTLbe3cyo1G60ZMcJfFmddxzv9kbCqeaRMdDIAtLndQkNhc91SEo0r9Bd cAuQ== X-Gm-Message-State: AOJu0YzZUqc4LRB6yfuDqhKjNZtaaEFK7EH4B2mZ1FuwYVLsxBK43SK6 uePTqXIMZ1hy91HGhpR9E+1fRfzE3Wo6TXrB2PPNlB3ZgTykz0LhJqZK1gjM/Q4= X-Google-Smtp-Source: AGHT+IHoX3GkOMrYz80+DM5m2Lww7jxBP5ILkv7oOVt7jcBS/xeFp7xuot3hFxzeNx20P3h+H5ZIIw== X-Received: by 2002:a05:6512:3e1b:b0:533:483f:9562 with SMTP id 2adb3069b0e04-536ac32e3f9mr7537031e87.42.1727103736536; Mon, 23 Sep 2024 08:02:16 -0700 (PDT) Received: from jlinkes-PT-Latitude-5530.. ([84.245.121.62]) by smtp.gmail.com with ESMTPSA id a640c23a62f3a-a90612b3ba7sm1235318866b.119.2024.09.23.08.02.14 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Mon, 23 Sep 2024 08:02:14 -0700 (PDT) From: =?UTF-8?q?Juraj=20Linke=C5=A1?= To: thomas@monjalon.net, Honnappa.Nagarahalli@arm.com, paul.szczepanek@arm.com, Luca.Vizzarro@arm.com, alex.chapman@arm.com, probb@iol.unh.edu, jspewock@iol.unh.edu, npratte@iol.unh.edu, dmarx@iol.unh.edu Cc: dev@dpdk.org, =?UTF-8?q?Juraj=20Linke=C5=A1?= Subject: [PATCH v4 03/11] dts: add mechanism to skip test cases or suites Date: Mon, 23 Sep 2024 17:02:02 +0200 Message-ID: <20240923150210.57269-3-juraj.linkes@pantheon.tech> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20240923150210.57269-1-juraj.linkes@pantheon.tech> References: <20240301155416.96960-1-juraj.linkes@pantheon.tech> <20240923150210.57269-1-juraj.linkes@pantheon.tech> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org If a test case is not relevant to the testing environment (such as when a NIC doesn't support a tested feature), the framework should skip it. The mechanism is a skeleton without actual logic that would set a test case or suite to be skipped. The mechanism uses a protocol to extend test suites and test cases with additional attributes that track whether the test case or suite should be skipped the reason for skipping it. Also update the results module with the new SKIP result. Signed-off-by: Juraj Linkeš Reviewed-by: Dean Marx --- dts/framework/runner.py | 34 ++++++++--- dts/framework/test_result.py | 74 ++++++++++++++--------- dts/framework/test_suite.py | 7 ++- dts/framework/testbed_model/capability.py | 28 +++++++++ 4 files changed, 105 insertions(+), 38 deletions(-) create mode 100644 dts/framework/testbed_model/capability.py diff --git a/dts/framework/runner.py b/dts/framework/runner.py index 68482dc9af..57284e510e 100644 --- a/dts/framework/runner.py +++ b/dts/framework/runner.py @@ -479,7 +479,20 @@ def _run_test_suites( for test_suite_with_cases in test_suites_with_cases: test_suite_result = build_target_result.add_test_suite(test_suite_with_cases) try: - self._run_test_suite(sut_node, tg_node, test_suite_result, test_suite_with_cases) + if not test_suite_with_cases.skip: + self._run_test_suite( + sut_node, + tg_node, + test_suite_result, + test_suite_with_cases, + ) + else: + self._logger.info( + f"Test suite execution SKIPPED: " + f"'{test_suite_with_cases.test_suite_class.__name__}'. Reason: " + f"{test_suite_with_cases.test_suite_class.skip_reason}" + ) + test_suite_result.update_setup(Result.SKIP) except BlockingTestSuiteError as e: self._logger.exception( f"An error occurred within {test_suite_with_cases.test_suite_class.__name__}. " @@ -578,14 +591,21 @@ def _execute_test_suite( test_case_result = test_suite_result.add_test_case(test_case_name) all_attempts = SETTINGS.re_run + 1 attempt_nr = 1 - self._run_test_case(test_suite, test_case, test_case_result) - while not test_case_result and attempt_nr < all_attempts: - attempt_nr += 1 + if not test_case.skip: + self._run_test_case(test_suite, test_case, test_case_result) + while not test_case_result and attempt_nr < all_attempts: + attempt_nr += 1 + self._logger.info( + f"Re-running FAILED test case '{test_case_name}'. " + f"Attempt number {attempt_nr} out of {all_attempts}." + ) + self._run_test_case(test_suite, test_case, test_case_result) + else: self._logger.info( - f"Re-running FAILED test case '{test_case_name}'. " - f"Attempt number {attempt_nr} out of {all_attempts}." + f"Test case execution SKIPPED: {test_case_name}. Reason: " + f"{test_case.skip_reason}" ) - self._run_test_case(test_suite, test_case, test_case_result) + test_case_result.update_setup(Result.SKIP) def _run_test_case( self, diff --git a/dts/framework/test_result.py b/dts/framework/test_result.py index b1ca584523..50633eee4e 100644 --- a/dts/framework/test_result.py +++ b/dts/framework/test_result.py @@ -75,6 +75,15 @@ def create_config(self) -> TestSuiteConfig: test_cases=[test_case.__name__ for test_case in self.test_cases], ) + @property + def skip(self) -> bool: + """Skip the test suite if all test cases or the suite itself are to be skipped. + + Returns: + :data:`True` if the test suite should be skipped, :data:`False` otherwise. + """ + return all(test_case.skip for test_case in self.test_cases) or self.test_suite_class.skip + class Result(Enum): """The possible states that a setup, a teardown or a test case may end up in.""" @@ -86,12 +95,12 @@ class Result(Enum): #: ERROR = auto() #: - SKIP = auto() - #: BLOCK = auto() + #: + SKIP = auto() def __bool__(self) -> bool: - """Only PASS is True.""" + """Only :attr:`PASS` is True.""" return self is self.PASS @@ -169,14 +178,15 @@ def update_setup(self, result: Result, error: Exception | None = None) -> None: self.setup_result.result = result self.setup_result.error = error - if result in [Result.BLOCK, Result.ERROR, Result.FAIL]: - self.update_teardown(Result.BLOCK) - self._block_result() + if result != Result.PASS: + result_to_mark = Result.BLOCK if result != Result.SKIP else Result.SKIP + self.update_teardown(result_to_mark) + self._mark_results(result_to_mark) - def _block_result(self) -> None: - r"""Mark the result as :attr:`~Result.BLOCK`\ed. + def _mark_results(self, result) -> None: + """Mark the child results or the result of the level itself as `result`. - The blocking of child results should be done in overloaded methods. + The marking of results should be done in overloaded methods. """ def update_teardown(self, result: Result, error: Exception | None = None) -> None: @@ -391,11 +401,11 @@ def add_sut_info(self, sut_info: NodeInfo) -> None: self.sut_os_version = sut_info.os_version self.sut_kernel_version = sut_info.kernel_version - def _block_result(self) -> None: - r"""Mark the result as :attr:`~Result.BLOCK`\ed.""" + def _mark_results(self, result) -> None: + """Mark the build target results as `result`.""" for build_target in self._config.build_targets: child_result = self.add_build_target(build_target) - child_result.update_setup(Result.BLOCK) + child_result.update_setup(result) class BuildTargetResult(BaseResult): @@ -465,11 +475,11 @@ def add_build_target_info(self, versions: BuildTargetInfo) -> None: self.compiler_version = versions.compiler_version self.dpdk_version = versions.dpdk_version - def _block_result(self) -> None: - r"""Mark the result as :attr:`~Result.BLOCK`\ed.""" + def _mark_results(self, result) -> None: + """Mark the test suite results as `result`.""" for test_suite_with_cases in self._test_suites_with_cases: child_result = self.add_test_suite(test_suite_with_cases) - child_result.update_setup(Result.BLOCK) + child_result.update_setup(result) class TestSuiteResult(BaseResult): @@ -509,11 +519,11 @@ def add_test_case(self, test_case_name: str) -> "TestCaseResult": self.child_results.append(result) return result - def _block_result(self) -> None: - r"""Mark the result as :attr:`~Result.BLOCK`\ed.""" + def _mark_results(self, result) -> None: + """Mark the test case results as `result`.""" for test_case_method in self._test_suite_with_cases.test_cases: child_result = self.add_test_case(test_case_method.__name__) - child_result.update_setup(Result.BLOCK) + child_result.update_setup(result) class TestCaseResult(BaseResult, FixtureResult): @@ -567,9 +577,9 @@ def add_stats(self, statistics: "Statistics") -> None: """ statistics += self.result - def _block_result(self) -> None: - r"""Mark the result as :attr:`~Result.BLOCK`\ed.""" - self.update(Result.BLOCK) + def _mark_results(self, result) -> None: + r"""Mark the result as `result`.""" + self.update(result) def __bool__(self) -> bool: """The test case passed only if setup, teardown and the test case itself passed.""" @@ -583,7 +593,8 @@ class Statistics(dict): The data are stored in the following keys: - * **PASS RATE** (:class:`int`) -- The FAIL/PASS ratio of all test cases. + * **PASS RATE** (:class:`int`) -- The :attr:`~Result.FAIL`/:attr:`~Result.PASS` ratio + of all test cases. * **DPDK VERSION** (:class:`str`) -- The tested DPDK version. """ @@ -600,22 +611,27 @@ def __init__(self, dpdk_version: str | None): self["DPDK VERSION"] = dpdk_version def __iadd__(self, other: Result) -> "Statistics": - """Add a Result to the final count. + """Add a :class:`Result` to the final count. + + :attr:`~Result.SKIP` is not taken into account Example: - stats: Statistics = Statistics() # empty Statistics - stats += Result.PASS # add a Result to `stats` + stats: Statistics = Statistics() # empty :class:`Statistics` + stats += Result.PASS # add a :class:`Result` to `stats` Args: - other: The Result to add to this statistics object. + other: The :class:`Result` to add to this statistics object. Returns: The modified statistics object. """ self[other.name] += 1 - self["PASS RATE"] = ( - float(self[Result.PASS.name]) * 100 / sum(self[result.name] for result in Result) - ) + if other != Result.SKIP: + self["PASS RATE"] = ( + float(self[Result.PASS.name]) + * 100 + / sum([self[result.name] for result in Result if result != Result.SKIP]) + ) return self def __str__(self) -> str: diff --git a/dts/framework/test_suite.py b/dts/framework/test_suite.py index a1fe7f7ebc..5154bb0514 100644 --- a/dts/framework/test_suite.py +++ b/dts/framework/test_suite.py @@ -24,6 +24,7 @@ from scapy.layers.l2 import Ether # type: ignore[import-untyped] from scapy.packet import Packet, Padding, raw # type: ignore[import-untyped] +from framework.testbed_model.capability import TestProtocol from framework.testbed_model.port import Port, PortLink from framework.testbed_model.sut_node import SutNode from framework.testbed_model.tg_node import TGNode @@ -36,7 +37,7 @@ from .utils import get_packet_summaries -class TestSuite: +class TestSuite(TestProtocol): """The base class with building blocks needed by most test cases. * Test suite setup/cleanup methods to override, @@ -505,7 +506,7 @@ class TestCaseType(Enum): PERFORMANCE = auto() -class TestCase(Protocol[TestSuiteMethodType]): +class TestCase(TestProtocol, Protocol[TestSuiteMethodType]): """Definition of the test case type for static type checking purposes. The type is applied to test case functions through a decorator, which casts the decorated @@ -536,6 +537,8 @@ def make_decorator( def _decorator(func: TestSuiteMethodType) -> type[TestCase]: test_case = cast(type[TestCase], func) + test_case.skip = cls.skip + test_case.skip_reason = cls.skip_reason test_case.test_type = test_case_type return test_case diff --git a/dts/framework/testbed_model/capability.py b/dts/framework/testbed_model/capability.py new file mode 100644 index 0000000000..662f691a0e --- /dev/null +++ b/dts/framework/testbed_model/capability.py @@ -0,0 +1,28 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2024 PANTHEON.tech s.r.o. + +"""Testbed capabilities. + +This module provides a protocol that defines the common attributes of test cases and suites. +""" + +from collections.abc import Sequence +from typing import ClassVar, Protocol + + +class TestProtocol(Protocol): + """Common test suite and test case attributes.""" + + #: Whether to skip the test case or suite. + skip: ClassVar[bool] = False + #: The reason for skipping the test case or suite. + skip_reason: ClassVar[str] = "" + + @classmethod + def get_test_cases(cls, test_case_sublist: Sequence[str] | None = None) -> tuple[set, set]: + """Get test cases. Should be implemented by subclasses containing test cases. + + Raises: + NotImplementedError: The subclass does not implement the method. + """ + raise NotImplementedError() -- 2.43.0