From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 9FD7E45830; Wed, 21 Aug 2024 16:53:49 +0200 (CEST) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id CC5E041611; Wed, 21 Aug 2024 16:53:26 +0200 (CEST) Received: from mail-ed1-f53.google.com (mail-ed1-f53.google.com [209.85.208.53]) by mails.dpdk.org (Postfix) with ESMTP id DF1254111B for ; Wed, 21 Aug 2024 16:53:22 +0200 (CEST) Received: by mail-ed1-f53.google.com with SMTP id 4fb4d7f45d1cf-5bec4e00978so6110672a12.0 for ; Wed, 21 Aug 2024 07:53:22 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=pantheon.tech; s=google; t=1724252002; x=1724856802; darn=dpdk.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=xLJGrZT/Vkt94m1c7pc/cyz1oXmi9UGnpHjBfDc+9Xs=; b=u5pNtVt+lvwbYINicnRveQ29TrqPitrB2iTUpUPSJ2UPEgP2uqrSulu2DVhej8womD 82hCIIAiipBkM0fqpmZNJONfIIK7N+E5bnSc/CPgAsi/5XES6Yhhw6WBXRwfoexVAjDs bMpwSgptVfJrwziizDdPVZI6HhJchFAeIE4UvzPlkLT5W7EGB+Wa+ksA6unMAkHHEEO7 6C2BSmwbKCB9+Btoqd6cgqq3KhAjgWdNb9+SvlBQvYFTNOarYOW6lHTk8xQA2sATwK0L 4zwZzX5z3fOAyXSqZRn1PIN9oZiSOJLB09Bfk0wXceSZmttnZ4k6Krv1I580nJKIDN44 kFMw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1724252002; x=1724856802; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=xLJGrZT/Vkt94m1c7pc/cyz1oXmi9UGnpHjBfDc+9Xs=; b=TrCaWW4O4nqIV9ensRVt54FlB4DuvMUa6uXLccGjK+EQ+k6viPQxyxUR2TpLvjyumX YxJxyoQQHi8j78CEyuO4VcxQaICtfmsr/LRSfMMSJHnm/KFfGrKfwoWu8B43aMP1uIu4 SEoFDkZlMcD54/ynw3DWe5h/u8agCcfYX62nH3VJX33qKpsowREsjRLQL61aiREw2hS7 2w6aWa7h4JjvVSoAS7c/7is5U75/JkzWOmsxH4+bO5ICQ5VcaNNFpUjlVOR3elLQz/DI xY4Cqri4guZhIg9QxWKWYDS5OIEYCKAAwH4UZ7FKyTyYYptfbYAQocpUqQpb0GIrOLZo oHAA== X-Gm-Message-State: AOJu0Yz60YqPlsrFyZxlzECjOqn7lztA6okqoYkmcdF4TKwL5cjSfa9T JWHXGftB+4jw1cpnMsFXQiyP7foAwXuC6v9hv2HcD+0KUiCx6TWaOtwF0/Ltd7o= X-Google-Smtp-Source: AGHT+IED2ZRXwC1pJGrRRymkF1xa6lTdG0/Ta8uT3OTXU9+Ufnq/+lK6tCfnAidEWIyhy0uDuWzabA== X-Received: by 2002:a17:907:d3d5:b0:a86:6fb3:fda5 with SMTP id a640c23a62f3a-a866fb40780mr181190966b.32.1724252002167; Wed, 21 Aug 2024 07:53:22 -0700 (PDT) Received: from localhost.localdomain ([84.245.121.236]) by smtp.gmail.com with ESMTPSA id a640c23a62f3a-a8680c4725csm52434866b.91.2024.08.21.07.53.21 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 21 Aug 2024 07:53:21 -0700 (PDT) From: =?UTF-8?q?Juraj=20Linke=C5=A1?= To: thomas@monjalon.net, Honnappa.Nagarahalli@arm.com, jspewock@iol.unh.edu, probb@iol.unh.edu, paul.szczepanek@arm.com, Luca.Vizzarro@arm.com, npratte@iol.unh.edu, dmarx@iol.unh.edu, alex.chapman@arm.com Cc: dev@dpdk.org, =?UTF-8?q?Juraj=20Linke=C5=A1?= Subject: [PATCH v3 04/12] dts: add mechanism to skip test cases or suites Date: Wed, 21 Aug 2024 16:53:07 +0200 Message-Id: <20240821145315.97974-5-juraj.linkes@pantheon.tech> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240821145315.97974-1-juraj.linkes@pantheon.tech> References: <20240301155416.96960-1-juraj.linkes@pantheon.tech> <20240821145315.97974-1-juraj.linkes@pantheon.tech> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org If a test case is not relevant to the testing environment (such as when a NIC doesn't support a tested feature), the framework should skip it. The mechanism is a skeleton without actual logic that would set a test case or suite to be skipped. The mechanism uses a protocol to extend test suites and test cases with additional attributes that track whether the test case or suite should be skipped the reason for skipping it. Also update the results module with the new SKIP result. Signed-off-by: Juraj Linkeš --- dts/framework/runner.py | 34 +++++++--- dts/framework/test_result.py | 77 ++++++++++++++--------- dts/framework/test_suite.py | 7 ++- dts/framework/testbed_model/capability.py | 28 +++++++++ 4 files changed, 109 insertions(+), 37 deletions(-) create mode 100644 dts/framework/testbed_model/capability.py diff --git a/dts/framework/runner.py b/dts/framework/runner.py index 525f119ab6..55357ea1fe 100644 --- a/dts/framework/runner.py +++ b/dts/framework/runner.py @@ -477,7 +477,20 @@ def _run_test_suites( for test_suite_with_cases in test_suites_with_cases: test_suite_result = build_target_result.add_test_suite(test_suite_with_cases) try: - self._run_test_suite(sut_node, tg_node, test_suite_result, test_suite_with_cases) + if not test_suite_with_cases.skip: + self._run_test_suite( + sut_node, + tg_node, + test_suite_result, + test_suite_with_cases, + ) + else: + self._logger.info( + f"Test suite execution SKIPPED: " + f"'{test_suite_with_cases.test_suite_class.__name__}'. Reason: " + f"{test_suite_with_cases.test_suite_class.skip_reason}" + ) + test_suite_result.update_setup(Result.SKIP) except BlockingTestSuiteError as e: self._logger.exception( f"An error occurred within {test_suite_with_cases.test_suite_class.__name__}. " @@ -576,14 +589,21 @@ def _execute_test_suite( test_case_result = test_suite_result.add_test_case(test_case_name) all_attempts = SETTINGS.re_run + 1 attempt_nr = 1 - self._run_test_case(test_suite, test_case, test_case_result) - while not test_case_result and attempt_nr < all_attempts: - attempt_nr += 1 + if not test_case.skip: + self._run_test_case(test_suite, test_case, test_case_result) + while not test_case_result and attempt_nr < all_attempts: + attempt_nr += 1 + self._logger.info( + f"Re-running FAILED test case '{test_case_name}'. " + f"Attempt number {attempt_nr} out of {all_attempts}." + ) + self._run_test_case(test_suite, test_case, test_case_result) + else: self._logger.info( - f"Re-running FAILED test case '{test_case_name}'. " - f"Attempt number {attempt_nr} out of {all_attempts}." + f"Test case execution SKIPPED: {test_case_name}. Reason: " + f"{test_case.skip_reason}" ) - self._run_test_case(test_suite, test_case, test_case_result) + test_case_result.update_setup(Result.SKIP) def _run_test_case( self, diff --git a/dts/framework/test_result.py b/dts/framework/test_result.py index b1ca584523..306b100bc6 100644 --- a/dts/framework/test_result.py +++ b/dts/framework/test_result.py @@ -75,6 +75,20 @@ def create_config(self) -> TestSuiteConfig: test_cases=[test_case.__name__ for test_case in self.test_cases], ) + @property + def skip(self) -> bool: + """Skip the test suite if all test cases or the suite itself are to be skipped. + + Returns: + :data:`True` if the test suite should be skipped, :data:`False` otherwise. + """ + all_test_cases_skipped = True + for test_case in self.test_cases: + if not test_case.skip: + all_test_cases_skipped = False + break + return all_test_cases_skipped or self.test_suite_class.skip + class Result(Enum): """The possible states that a setup, a teardown or a test case may end up in.""" @@ -86,12 +100,12 @@ class Result(Enum): #: ERROR = auto() #: - SKIP = auto() - #: BLOCK = auto() + #: + SKIP = auto() def __bool__(self) -> bool: - """Only PASS is True.""" + """Only :attr:`PASS` is True.""" return self is self.PASS @@ -169,12 +183,13 @@ def update_setup(self, result: Result, error: Exception | None = None) -> None: self.setup_result.result = result self.setup_result.error = error - if result in [Result.BLOCK, Result.ERROR, Result.FAIL]: - self.update_teardown(Result.BLOCK) - self._block_result() + if result != Result.PASS: + result_to_mark = Result.BLOCK if result != Result.SKIP else Result.SKIP + self.update_teardown(result_to_mark) + self._mark_results(result_to_mark) - def _block_result(self) -> None: - r"""Mark the result as :attr:`~Result.BLOCK`\ed. + def _mark_results(self, result) -> None: + """Mark the result as well as the child result as `result`. The blocking of child results should be done in overloaded methods. """ @@ -391,11 +406,11 @@ def add_sut_info(self, sut_info: NodeInfo) -> None: self.sut_os_version = sut_info.os_version self.sut_kernel_version = sut_info.kernel_version - def _block_result(self) -> None: - r"""Mark the result as :attr:`~Result.BLOCK`\ed.""" + def _mark_results(self, result) -> None: + """Mark the result as well as the child result as `result`.""" for build_target in self._config.build_targets: child_result = self.add_build_target(build_target) - child_result.update_setup(Result.BLOCK) + child_result.update_setup(result) class BuildTargetResult(BaseResult): @@ -465,11 +480,11 @@ def add_build_target_info(self, versions: BuildTargetInfo) -> None: self.compiler_version = versions.compiler_version self.dpdk_version = versions.dpdk_version - def _block_result(self) -> None: - r"""Mark the result as :attr:`~Result.BLOCK`\ed.""" + def _mark_results(self, result) -> None: + """Mark the result as well as the child result as `result`.""" for test_suite_with_cases in self._test_suites_with_cases: child_result = self.add_test_suite(test_suite_with_cases) - child_result.update_setup(Result.BLOCK) + child_result.update_setup(result) class TestSuiteResult(BaseResult): @@ -509,11 +524,11 @@ def add_test_case(self, test_case_name: str) -> "TestCaseResult": self.child_results.append(result) return result - def _block_result(self) -> None: - r"""Mark the result as :attr:`~Result.BLOCK`\ed.""" + def _mark_results(self, result) -> None: + """Mark the result as well as the child result as `result`.""" for test_case_method in self._test_suite_with_cases.test_cases: child_result = self.add_test_case(test_case_method.__name__) - child_result.update_setup(Result.BLOCK) + child_result.update_setup(result) class TestCaseResult(BaseResult, FixtureResult): @@ -567,9 +582,9 @@ def add_stats(self, statistics: "Statistics") -> None: """ statistics += self.result - def _block_result(self) -> None: - r"""Mark the result as :attr:`~Result.BLOCK`\ed.""" - self.update(Result.BLOCK) + def _mark_results(self, result) -> None: + r"""Mark the result as `result`.""" + self.update(result) def __bool__(self) -> bool: """The test case passed only if setup, teardown and the test case itself passed.""" @@ -583,7 +598,8 @@ class Statistics(dict): The data are stored in the following keys: - * **PASS RATE** (:class:`int`) -- The FAIL/PASS ratio of all test cases. + * **PASS RATE** (:class:`int`) -- The :attr:`~Result.FAIL`/:attr:`~Result.PASS` ratio + of all test cases. * **DPDK VERSION** (:class:`str`) -- The tested DPDK version. """ @@ -600,22 +616,27 @@ def __init__(self, dpdk_version: str | None): self["DPDK VERSION"] = dpdk_version def __iadd__(self, other: Result) -> "Statistics": - """Add a Result to the final count. + """Add a :class:`Result` to the final count. + + :attr:`~Result.SKIP` is not taken into account Example: - stats: Statistics = Statistics() # empty Statistics - stats += Result.PASS # add a Result to `stats` + stats: Statistics = Statistics() # empty :class:`Statistics` + stats += Result.PASS # add a :class:`Result` to `stats` Args: - other: The Result to add to this statistics object. + other: The :class:`Result` to add to this statistics object. Returns: The modified statistics object. """ self[other.name] += 1 - self["PASS RATE"] = ( - float(self[Result.PASS.name]) * 100 / sum(self[result.name] for result in Result) - ) + if other != Result.SKIP: + self["PASS RATE"] = ( + float(self[Result.PASS.name]) + * 100 + / sum([self[result.name] for result in Result if result != Result.SKIP]) + ) return self def __str__(self) -> str: diff --git a/dts/framework/test_suite.py b/dts/framework/test_suite.py index b4ee0f9039..c59fc9c6e6 100644 --- a/dts/framework/test_suite.py +++ b/dts/framework/test_suite.py @@ -23,6 +23,7 @@ from scapy.layers.l2 import Ether # type: ignore[import-untyped] from scapy.packet import Packet, Padding # type: ignore[import-untyped] +from framework.testbed_model.capability import TestProtocol from framework.testbed_model.port import Port, PortLink from framework.testbed_model.sut_node import SutNode from framework.testbed_model.tg_node import TGNode @@ -35,7 +36,7 @@ from .utils import get_packet_summaries -class TestSuite: +class TestSuite(TestProtocol): """The base class with building blocks needed by most test cases. * Test suite setup/cleanup methods to override, @@ -445,7 +446,7 @@ class TestCaseType(Enum): PERFORMANCE = auto() -class TestCase(Protocol[TestSuiteMethodType]): +class TestCase(TestProtocol, Protocol[TestSuiteMethodType]): """Definition of the test case type for static type checking purposes. The type is applied to test case functions through a decorator, which casts the decorated @@ -476,6 +477,8 @@ def make_decorator( def _decorator(func: TestSuiteMethodType) -> type[TestCase]: test_case = cast(type[TestCase], func) + test_case.skip = cls.skip + test_case.skip_reason = cls.skip_reason test_case.test_type = test_case_type return test_case diff --git a/dts/framework/testbed_model/capability.py b/dts/framework/testbed_model/capability.py new file mode 100644 index 0000000000..662f691a0e --- /dev/null +++ b/dts/framework/testbed_model/capability.py @@ -0,0 +1,28 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2024 PANTHEON.tech s.r.o. + +"""Testbed capabilities. + +This module provides a protocol that defines the common attributes of test cases and suites. +""" + +from collections.abc import Sequence +from typing import ClassVar, Protocol + + +class TestProtocol(Protocol): + """Common test suite and test case attributes.""" + + #: Whether to skip the test case or suite. + skip: ClassVar[bool] = False + #: The reason for skipping the test case or suite. + skip_reason: ClassVar[str] = "" + + @classmethod + def get_test_cases(cls, test_case_sublist: Sequence[str] | None = None) -> tuple[set, set]: + """Get test cases. Should be implemented by subclasses containing test cases. + + Raises: + NotImplementedError: The subclass does not implement the method. + """ + raise NotImplementedError() -- 2.34.1