From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 9AE7F43B9B; Fri, 23 Feb 2024 08:55:39 +0100 (CET) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 59C37427D7; Fri, 23 Feb 2024 08:55:13 +0100 (CET) Received: from mail-ej1-f49.google.com (mail-ej1-f49.google.com [209.85.218.49]) by mails.dpdk.org (Postfix) with ESMTP id 4D08C402ED for ; Fri, 23 Feb 2024 08:55:10 +0100 (CET) Received: by mail-ej1-f49.google.com with SMTP id a640c23a62f3a-a3e4765c86eso11962966b.0 for ; Thu, 22 Feb 2024 23:55:10 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=pantheon.tech; s=google; t=1708674910; x=1709279710; darn=dpdk.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=Xl86siJVlDGlX5Wkzr/K/eUz6vp6W3gCEYpA70LynMk=; b=EDRE8VlDHdaC+oyuAutZfQckWkAJqpJDRJuW9RwOl1Gg9NjZcPOUm6VMzdet1aexIy hXCldlIE4OUkOovWciv0xmFz5RWZtHdacXbDKDo+Mmg0htr4Z/DnNCFYT0OzuGAGBRy+ pqZ9H/NxDJPr5EoXRTKIABKhVvn/H9y0ThMKJmkT4pcFSimUcxDZ80SvqrBtuq/U6SKc whoCGpPYa9okPFCjQOFUpZE3vu476Xt7eNFOEfVPM0StpBTeW46OKGxrVaigi0Th5iCe pFcvIw7/puq1Xq4TSOB5FCeTMwlay4wif9FJOtZ6gHKHmE4c5KWT9rFIPR+gKkixVWbY Fd1g== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1708674910; x=1709279710; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=Xl86siJVlDGlX5Wkzr/K/eUz6vp6W3gCEYpA70LynMk=; b=NYjo7XhXMwcsyv6jMKuZIXDV8h9rTIdnGUjQzSZlx5AvQWvXE9v5SvD0o4ReEePC/V 3SHP8MbB3EAdgNiC53jE1LYswFTCYiu6EH3IKwonyRVZa3coGrqSJiRhqSj5tFSyi2ir QVasY8nM8GoCjIZpfhsFQTbZ1gS7Nw++IkUHq664FD8aoUAZpMz0sGXxcQkTh/+fbQ2S n0YBzx90LsalOG+OL9Ois1+tVewGb4U7uIjGhfLQFfJe1Zy31jQSfcUQpCqjOawWM4uC LTS80OAMwOFGqfZCbrTRzQpa7888qBKcZ/OhmkkKc9eTLsWO90ocJVUjt7tQ9iJJPaNQ Rg6w== X-Gm-Message-State: AOJu0Yz1bjUh8NkiXm9vnMfu2w20qmekqHr3SK03jC5oIKcIfz3dztk7 H94MzM050c7xHFI9VvQBpLTAJkB75qUVF1Pt3z+FpFTQT4GN4hik90XkjfXazGg= X-Google-Smtp-Source: AGHT+IGisrO0vvldoyz+vLzCnUgvmgqOrTptfjavxokFBxNn76x9Ql0V1xppM/yYc98JVS3cqucnfA== X-Received: by 2002:a17:907:1008:b0:a3e:c738:884b with SMTP id ox8-20020a170907100800b00a3ec738884bmr626373ejb.69.1708674909634; Thu, 22 Feb 2024 23:55:09 -0800 (PST) Received: from localhost.localdomain ([84.245.120.62]) by smtp.gmail.com with ESMTPSA id th7-20020a1709078e0700b00a3e059c5c5fsm6660235ejc.188.2024.02.22.23.55.08 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 22 Feb 2024 23:55:09 -0800 (PST) From: =?UTF-8?q?Juraj=20Linke=C5=A1?= To: thomas@monjalon.net, Honnappa.Nagarahalli@arm.com, jspewock@iol.unh.edu, probb@iol.unh.edu, paul.szczepanek@arm.com, Luca.Vizzarro@arm.com Cc: dev@dpdk.org, =?UTF-8?q?Juraj=20Linke=C5=A1?= Subject: [PATCH v3 4/7] dts: reorganize test result Date: Fri, 23 Feb 2024 08:54:59 +0100 Message-Id: <20240223075502.60485-5-juraj.linkes@pantheon.tech> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240223075502.60485-1-juraj.linkes@pantheon.tech> References: <20231220103331.60888-1-juraj.linkes@pantheon.tech> <20240223075502.60485-1-juraj.linkes@pantheon.tech> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org The current order of Result classes in the test_suite.py module is guided by the needs of type hints, which is not as intuitively readable as ordering them by the occurrences in code. The order goes from the topmost level to lowermost: BaseResult DTSResult ExecutionResult BuildTargetResult TestSuiteResult TestCaseResult This is the same order as they're used in the runner module and they're also used in the same order between themselves in the test_result module. Signed-off-by: Juraj Linkeš --- dts/framework/test_result.py | 411 ++++++++++++++++++----------------- 1 file changed, 206 insertions(+), 205 deletions(-) diff --git a/dts/framework/test_result.py b/dts/framework/test_result.py index 075195fd5b..abdbafab10 100644 --- a/dts/framework/test_result.py +++ b/dts/framework/test_result.py @@ -28,6 +28,7 @@ from dataclasses import dataclass from enum import Enum, auto from types import MethodType +from typing import Union from .config import ( OS, @@ -129,58 +130,6 @@ def __bool__(self) -> bool: return bool(self.result) -class Statistics(dict): - """How many test cases ended in which result state along some other basic information. - - Subclassing :class:`dict` provides a convenient way to format the data. - - The data are stored in the following keys: - - * **PASS RATE** (:class:`int`) -- The FAIL/PASS ratio of all test cases. - * **DPDK VERSION** (:class:`str`) -- The tested DPDK version. - """ - - def __init__(self, dpdk_version: str | None): - """Extend the constructor with keys in which the data are stored. - - Args: - dpdk_version: The version of tested DPDK. - """ - super(Statistics, self).__init__() - for result in Result: - self[result.name] = 0 - self["PASS RATE"] = 0.0 - self["DPDK VERSION"] = dpdk_version - - def __iadd__(self, other: Result) -> "Statistics": - """Add a Result to the final count. - - Example: - stats: Statistics = Statistics() # empty Statistics - stats += Result.PASS # add a Result to `stats` - - Args: - other: The Result to add to this statistics object. - - Returns: - The modified statistics object. - """ - self[other.name] += 1 - self["PASS RATE"] = ( - float(self[Result.PASS.name]) * 100 / sum(self[result.name] for result in Result) - ) - return self - - def __str__(self) -> str: - """Each line contains the formatted key = value pair.""" - stats_str = "" - for key, value in self.items(): - stats_str += f"{key:<12} = {value}\n" - # according to docs, we should use \n when writing to text files - # on all platforms - return stats_str - - class BaseResult(object): """Common data and behavior of DTS results. @@ -245,7 +194,7 @@ def get_errors(self) -> list[Exception]: """ return self._get_setup_teardown_errors() + self._get_inner_errors() - def add_stats(self, statistics: Statistics) -> None: + def add_stats(self, statistics: "Statistics") -> None: """Collate stats from the whole result hierarchy. Args: @@ -255,91 +204,149 @@ def add_stats(self, statistics: Statistics) -> None: inner_result.add_stats(statistics) -class TestCaseResult(BaseResult, FixtureResult): - r"""The test case specific result. +class DTSResult(BaseResult): + """Stores environment information and test results from a DTS run. - Stores the result of the actual test case. This is done by adding an extra superclass - in :class:`FixtureResult`. The setup and teardown results are :class:`FixtureResult`\s and - the class is itself a record of the test case. + * Execution level information, such as testbed and the test suite list, + * Build target level information, such as compiler, target OS and cpu, + * Test suite and test case results, + * All errors that are caught and recorded during DTS execution. + + The information is stored hierarchically. This is the first level of the hierarchy + and as such is where the data form the whole hierarchy is collated or processed. + + The internal list stores the results of all executions. Attributes: - test_case_name: The test case name. + dpdk_version: The DPDK version to record. """ - test_case_name: str + dpdk_version: str | None + _logger: DTSLOG + _errors: list[Exception] + _return_code: ErrorSeverity + _stats_result: Union["Statistics", None] + _stats_filename: str - def __init__(self, test_case_name: str): - """Extend the constructor with `test_case_name`. + def __init__(self, logger: DTSLOG): + """Extend the constructor with top-level specifics. Args: - test_case_name: The test case's name. + logger: The logger instance the whole result will use. """ - super(TestCaseResult, self).__init__() - self.test_case_name = test_case_name + super(DTSResult, self).__init__() + self.dpdk_version = None + self._logger = logger + self._errors = [] + self._return_code = ErrorSeverity.NO_ERR + self._stats_result = None + self._stats_filename = os.path.join(SETTINGS.output_dir, "statistics.txt") - def update(self, result: Result, error: Exception | None = None) -> None: - """Update the test case result. + def add_execution(self, sut_node: NodeConfiguration) -> "ExecutionResult": + """Add and return the inner result (execution). - This updates the result of the test case itself and doesn't affect - the results of the setup and teardown steps in any way. + Args: + sut_node: The SUT node's test run configuration. + + Returns: + The execution's result. + """ + execution_result = ExecutionResult(sut_node) + self._inner_results.append(execution_result) + return execution_result + + def add_error(self, error: Exception) -> None: + """Record an error that occurred outside any execution. Args: - result: The result of the test case. - error: The error that occurred in case of a failure. + error: The exception to record. """ - self.result = result - self.error = error + self._errors.append(error) - def _get_inner_errors(self) -> list[Exception]: - if self.error: - return [self.error] - return [] + def process(self) -> None: + """Process the data after a whole DTS run. - def add_stats(self, statistics: Statistics) -> None: - r"""Add the test case result to statistics. + The data is added to inner objects during runtime and this object is not updated + at that time. This requires us to process the inner data after it's all been gathered. - The base method goes through the hierarchy recursively and this method is here to stop - the recursion, as the :class:`TestCaseResult`\s are the leaves of the hierarchy tree. + The processing gathers all errors and the statistics of test case results. + """ + self._errors += self.get_errors() + if self._errors and self._logger: + self._logger.debug("Summary of errors:") + for error in self._errors: + self._logger.debug(repr(error)) - Args: - statistics: The :class:`Statistics` object where the stats will be added. + self._stats_result = Statistics(self.dpdk_version) + self.add_stats(self._stats_result) + with open(self._stats_filename, "w+") as stats_file: + stats_file.write(str(self._stats_result)) + + def get_return_code(self) -> int: + """Go through all stored Exceptions and return the final DTS error code. + + Returns: + The highest error code found. """ - statistics += self.result + for error in self._errors: + error_return_code = ErrorSeverity.GENERIC_ERR + if isinstance(error, DTSError): + error_return_code = error.severity - def __bool__(self) -> bool: - """The test case passed only if setup, teardown and the test case itself passed.""" - return bool(self.setup_result) and bool(self.teardown_result) and bool(self.result) + if error_return_code > self._return_code: + self._return_code = error_return_code + return int(self._return_code) -class TestSuiteResult(BaseResult): - """The test suite specific result. - The internal list stores the results of all test cases in a given test suite. +class ExecutionResult(BaseResult): + """The execution specific result. + + The internal list stores the results of all build targets in a given execution. Attributes: - suite_name: The test suite name. + sut_node: The SUT node used in the execution. + sut_os_name: The operating system of the SUT node. + sut_os_version: The operating system version of the SUT node. + sut_kernel_version: The operating system kernel version of the SUT node. """ - suite_name: str + sut_node: NodeConfiguration + sut_os_name: str + sut_os_version: str + sut_kernel_version: str - def __init__(self, suite_name: str): - """Extend the constructor with `suite_name`. + def __init__(self, sut_node: NodeConfiguration): + """Extend the constructor with the `sut_node`'s config. Args: - suite_name: The test suite's name. + sut_node: The SUT node's test run configuration used in the execution. """ - super(TestSuiteResult, self).__init__() - self.suite_name = suite_name + super(ExecutionResult, self).__init__() + self.sut_node = sut_node - def add_test_case(self, test_case_name: str) -> TestCaseResult: - """Add and return the inner result (test case). + def add_build_target(self, build_target: BuildTargetConfiguration) -> "BuildTargetResult": + """Add and return the inner result (build target). + + Args: + build_target: The build target's test run configuration. Returns: - The test case's result. + The build target's result. """ - test_case_result = TestCaseResult(test_case_name) - self._inner_results.append(test_case_result) - return test_case_result + build_target_result = BuildTargetResult(build_target) + self._inner_results.append(build_target_result) + return build_target_result + + def add_sut_info(self, sut_info: NodeInfo) -> None: + """Add SUT information gathered at runtime. + + Args: + sut_info: The additional SUT node information. + """ + self.sut_os_name = sut_info.os_name + self.sut_os_version = sut_info.os_version + self.sut_kernel_version = sut_info.kernel_version class BuildTargetResult(BaseResult): @@ -386,7 +393,7 @@ def add_build_target_info(self, versions: BuildTargetInfo) -> None: self.compiler_version = versions.compiler_version self.dpdk_version = versions.dpdk_version - def add_test_suite(self, test_suite_name: str) -> TestSuiteResult: + def add_test_suite(self, test_suite_name: str) -> "TestSuiteResult": """Add and return the inner result (test suite). Returns: @@ -397,146 +404,140 @@ def add_test_suite(self, test_suite_name: str) -> TestSuiteResult: return test_suite_result -class ExecutionResult(BaseResult): - """The execution specific result. +class TestSuiteResult(BaseResult): + """The test suite specific result. - The internal list stores the results of all build targets in a given execution. + The internal list stores the results of all test cases in a given test suite. Attributes: - sut_node: The SUT node used in the execution. - sut_os_name: The operating system of the SUT node. - sut_os_version: The operating system version of the SUT node. - sut_kernel_version: The operating system kernel version of the SUT node. + suite_name: The test suite name. """ - sut_node: NodeConfiguration - sut_os_name: str - sut_os_version: str - sut_kernel_version: str + suite_name: str - def __init__(self, sut_node: NodeConfiguration): - """Extend the constructor with the `sut_node`'s config. + def __init__(self, suite_name: str): + """Extend the constructor with `suite_name`. Args: - sut_node: The SUT node's test run configuration used in the execution. + suite_name: The test suite's name. """ - super(ExecutionResult, self).__init__() - self.sut_node = sut_node - - def add_build_target(self, build_target: BuildTargetConfiguration) -> BuildTargetResult: - """Add and return the inner result (build target). + super(TestSuiteResult, self).__init__() + self.suite_name = suite_name - Args: - build_target: The build target's test run configuration. + def add_test_case(self, test_case_name: str) -> "TestCaseResult": + """Add and return the inner result (test case). Returns: - The build target's result. - """ - build_target_result = BuildTargetResult(build_target) - self._inner_results.append(build_target_result) - return build_target_result - - def add_sut_info(self, sut_info: NodeInfo) -> None: - """Add SUT information gathered at runtime. - - Args: - sut_info: The additional SUT node information. + The test case's result. """ - self.sut_os_name = sut_info.os_name - self.sut_os_version = sut_info.os_version - self.sut_kernel_version = sut_info.kernel_version + test_case_result = TestCaseResult(test_case_name) + self._inner_results.append(test_case_result) + return test_case_result -class DTSResult(BaseResult): - """Stores environment information and test results from a DTS run. - - * Execution level information, such as testbed and the test suite list, - * Build target level information, such as compiler, target OS and cpu, - * Test suite and test case results, - * All errors that are caught and recorded during DTS execution. - - The information is stored hierarchically. This is the first level of the hierarchy - and as such is where the data form the whole hierarchy is collated or processed. +class TestCaseResult(BaseResult, FixtureResult): + r"""The test case specific result. - The internal list stores the results of all executions. + Stores the result of the actual test case. This is done by adding an extra superclass + in :class:`FixtureResult`. The setup and teardown results are :class:`FixtureResult`\s and + the class is itself a record of the test case. Attributes: - dpdk_version: The DPDK version to record. + test_case_name: The test case name. """ - dpdk_version: str | None - _logger: DTSLOG - _errors: list[Exception] - _return_code: ErrorSeverity - _stats_result: Statistics | None - _stats_filename: str + test_case_name: str - def __init__(self, logger: DTSLOG): - """Extend the constructor with top-level specifics. + def __init__(self, test_case_name: str): + """Extend the constructor with `test_case_name`. Args: - logger: The logger instance the whole result will use. + test_case_name: The test case's name. """ - super(DTSResult, self).__init__() - self.dpdk_version = None - self._logger = logger - self._errors = [] - self._return_code = ErrorSeverity.NO_ERR - self._stats_result = None - self._stats_filename = os.path.join(SETTINGS.output_dir, "statistics.txt") + super(TestCaseResult, self).__init__() + self.test_case_name = test_case_name - def add_execution(self, sut_node: NodeConfiguration) -> ExecutionResult: - """Add and return the inner result (execution). + def update(self, result: Result, error: Exception | None = None) -> None: + """Update the test case result. - Args: - sut_node: The SUT node's test run configuration. + This updates the result of the test case itself and doesn't affect + the results of the setup and teardown steps in any way. - Returns: - The execution's result. + Args: + result: The result of the test case. + error: The error that occurred in case of a failure. """ - execution_result = ExecutionResult(sut_node) - self._inner_results.append(execution_result) - return execution_result + self.result = result + self.error = error - def add_error(self, error: Exception) -> None: - """Record an error that occurred outside any execution. + def _get_inner_errors(self) -> list[Exception]: + if self.error: + return [self.error] + return [] + + def add_stats(self, statistics: "Statistics") -> None: + r"""Add the test case result to statistics. + + The base method goes through the hierarchy recursively and this method is here to stop + the recursion, as the :class:`TestCaseResult`\s are the leaves of the hierarchy tree. Args: - error: The exception to record. + statistics: The :class:`Statistics` object where the stats will be added. """ - self._errors.append(error) + statistics += self.result - def process(self) -> None: - """Process the data after a whole DTS run. + def __bool__(self) -> bool: + """The test case passed only if setup, teardown and the test case itself passed.""" + return bool(self.setup_result) and bool(self.teardown_result) and bool(self.result) - The data is added to inner objects during runtime and this object is not updated - at that time. This requires us to process the inner data after it's all been gathered. - The processing gathers all errors and the statistics of test case results. +class Statistics(dict): + """How many test cases ended in which result state along some other basic information. + + Subclassing :class:`dict` provides a convenient way to format the data. + + The data are stored in the following keys: + + * **PASS RATE** (:class:`int`) -- The FAIL/PASS ratio of all test cases. + * **DPDK VERSION** (:class:`str`) -- The tested DPDK version. + """ + + def __init__(self, dpdk_version: str | None): + """Extend the constructor with keys in which the data are stored. + + Args: + dpdk_version: The version of tested DPDK. """ - self._errors += self.get_errors() - if self._errors and self._logger: - self._logger.debug("Summary of errors:") - for error in self._errors: - self._logger.debug(repr(error)) + super(Statistics, self).__init__() + for result in Result: + self[result.name] = 0 + self["PASS RATE"] = 0.0 + self["DPDK VERSION"] = dpdk_version - self._stats_result = Statistics(self.dpdk_version) - self.add_stats(self._stats_result) - with open(self._stats_filename, "w+") as stats_file: - stats_file.write(str(self._stats_result)) + def __iadd__(self, other: Result) -> "Statistics": + """Add a Result to the final count. - def get_return_code(self) -> int: - """Go through all stored Exceptions and return the final DTS error code. + Example: + stats: Statistics = Statistics() # empty Statistics + stats += Result.PASS # add a Result to `stats` + + Args: + other: The Result to add to this statistics object. Returns: - The highest error code found. + The modified statistics object. """ - for error in self._errors: - error_return_code = ErrorSeverity.GENERIC_ERR - if isinstance(error, DTSError): - error_return_code = error.severity - - if error_return_code > self._return_code: - self._return_code = error_return_code + self[other.name] += 1 + self["PASS RATE"] = ( + float(self[Result.PASS.name]) * 100 / sum(self[result.name] for result in Result) + ) + return self - return int(self._return_code) + def __str__(self) -> str: + """Each line contains the formatted key = value pair.""" + stats_str = "" + for key, value in self.items(): + stats_str += f"{key:<12} = {value}\n" + # according to docs, we should use \n when writing to text files + # on all platforms + return stats_str -- 2.34.1