From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id DD329A00C4; Mon, 14 Nov 2022 17:55:33 +0100 (CET) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 4A29642D34; Mon, 14 Nov 2022 17:54:53 +0100 (CET) Received: from lb.pantheon.sk (lb.pantheon.sk [46.229.239.20]) by mails.dpdk.org (Postfix) with ESMTP id 1B06F42D31 for ; Mon, 14 Nov 2022 17:54:50 +0100 (CET) Received: from localhost (localhost [127.0.0.1]) by lb.pantheon.sk (Postfix) with ESMTP id 42F4026AEFE; Mon, 14 Nov 2022 17:54:49 +0100 (CET) X-Virus-Scanned: amavisd-new at siecit.sk Received: from lb.pantheon.sk ([127.0.0.1]) by localhost (lb.pantheon.sk [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id A1D5npxuGhBJ; Mon, 14 Nov 2022 17:54:48 +0100 (CET) Received: from entguard.lab.pantheon.local (unknown [46.229.239.141]) by lb.pantheon.sk (Postfix) with ESMTP id 964ED243CCC; Mon, 14 Nov 2022 17:54:42 +0100 (CET) From: =?UTF-8?q?Juraj=20Linke=C5=A1?= To: thomas@monjalon.net, Honnappa.Nagarahalli@arm.com, ohilyard@iol.unh.edu, lijuan.tu@intel.com, bruce.richardson@intel.com Cc: dev@dpdk.org, =?UTF-8?q?Juraj=20Linke=C5=A1?= Subject: [RFC PATCH v2 06/10] dts: add test results module Date: Mon, 14 Nov 2022 16:54:34 +0000 Message-Id: <20221114165438.1133783-7-juraj.linkes@pantheon.tech> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20221114165438.1133783-1-juraj.linkes@pantheon.tech> References: <20220824162454.394285-1-juraj.linkes@pantheon.tech> <20221114165438.1133783-1-juraj.linkes@pantheon.tech> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org The module keeps track of test case results along with miscellaneous information, such as on which SUT's did a failure occur and during the testing of which build target. Signed-off-by: Juraj Linkeš --- dts/framework/dts.py | 5 + dts/framework/test_result.py | 217 +++++++++++++++++++++++++++++++++++ 2 files changed, 222 insertions(+) create mode 100644 dts/framework/test_result.py diff --git a/dts/framework/dts.py b/dts/framework/dts.py index 262c392d8e..d606f8de2e 100644 --- a/dts/framework/dts.py +++ b/dts/framework/dts.py @@ -14,9 +14,11 @@ from .exception import DTSError, ReturnCode from .logger import DTSLOG, getLogger from .settings import SETTINGS +from .test_result import Result from .utils import check_dts_python_version dts_logger: DTSLOG = getLogger("dts") +result: Result = Result() def run_all() -> None: @@ -26,6 +28,7 @@ def run_all() -> None: """ return_code = ReturnCode.NO_ERR global dts_logger + global result # check the python version of the server that run dts check_dts_python_version() @@ -45,6 +48,7 @@ def run_all() -> None: # for all Execution sections for execution in CONFIGURATION.executions: sut_node = init_nodes(execution, nodes) + result.sut = sut_node run_execution(sut_node, execution) except DTSError as e: @@ -104,6 +108,7 @@ def run_build_target( Run the given build target. """ dts_logger.info(f"Running target '{build_target.name}'.") + result.target = build_target try: sut_node.setup_build_target(build_target) run_suite(sut_node, build_target, execution) diff --git a/dts/framework/test_result.py b/dts/framework/test_result.py new file mode 100644 index 0000000000..a12517b9bc --- /dev/null +++ b/dts/framework/test_result.py @@ -0,0 +1,217 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2010-2014 Intel Corporation +# Copyright(c) 2022 PANTHEON.tech s.r.o. + +""" +Generic result container and reporters +""" + + +class Result(object): + """ + Generic result container. Useful to store/retrieve results during + a DTF execution. + + It manages and hide an internal complex structure like the one shown below. + This is presented to the user with a property based interface. + + internals = [ + 'sut1', [ + 'kdriver', + 'firmware', + 'pkg', + 'driver', + 'dpdk_version', + 'target1', 'nic1', [ + 'suite1', [ + 'case1', ['PASSED', ''], + 'case2', ['PASSED', ''], + ], + ], + 'target2', 'nic1', [ + 'suite2', [ + 'case3', ['PASSED', ''], + 'case4', ['FAILED', 'message'], + ], + 'suite3', [ + 'case5', ['BLOCKED', 'message'], + ], + ] + ] + ] + + """ + + def __init__(self): + self.__sut = 0 + self.__target = 0 + self.__test_suite = 0 + self.__test_case = 0 + self.__test_result = None + self.__message = None + self.__internals = [] + self.__failed_suts = {} + self.__failed_targets = {} + + def __set_sut(self, sut): + if sut not in self.__internals: + self.__internals.append(sut) + self.__internals.append([]) + self.__sut = self.__internals.index(sut) + + def __get_sut(self): + return self.__internals[self.__sut] + + def current_dpdk_version(self, sut): + """ + Returns the dpdk version for a given SUT + """ + try: + sut_idx = self.__internals.index(sut) + return self.__internals[sut_idx + 1][4] + except: + return "" + + def __set_dpdk_version(self, dpdk_version): + if dpdk_version not in self.internals[self.__sut + 1]: + dpdk_current = self.__get_dpdk_version() + if dpdk_current: + if dpdk_version not in dpdk_current: + self.internals[self.__sut + 1][4] = ( + dpdk_current + "/" + dpdk_version + ) + else: + self.internals[self.__sut + 1].append(dpdk_version) + + def __get_dpdk_version(self): + try: + return self.internals[self.__sut + 1][4] + except: + return "" + + def __current_targets(self): + return self.internals[self.__sut + 1] + + def __set_target(self, target): + targets = self.__current_targets() + if target not in targets: + targets.append(target) + targets.append("_nic_") + targets.append([]) + self.__target = targets.index(target) + + def __get_target(self): + return self.__current_targets()[self.__target] + + def __current_suites(self): + return self.__current_targets()[self.__target + 2] + + def __set_test_suite(self, test_suite): + suites = self.__current_suites() + if test_suite not in suites: + suites.append(test_suite) + suites.append([]) + self.__test_suite = suites.index(test_suite) + + def __get_test_suite(self): + return self.__current_suites()[self.__test_suite] + + def __current_cases(self): + return self.__current_suites()[self.__test_suite + 1] + + def __set_test_case(self, test_case): + cases = self.__current_cases() + cases.append(test_case) + cases.append([]) + self.__test_case = cases.index(test_case) + + def __get_test_case(self): + return self.__current_cases()[self.__test_case] + + def __get_internals(self): + return self.__internals + + def __current_result(self): + return self.__current_cases()[self.__test_case + 1] + + def __set_test_case_result(self, result, message): + test_case = self.__current_result() + test_case.append(result) + test_case.append(message) + self.__test_result = result + self.__message = message + + def copy_suite(self, suite_result): + self.__current_suites()[self.__test_suite + 1] = suite_result.__current_cases() + + def test_case_passed(self): + """ + Set last test case added as PASSED + """ + self.__set_test_case_result(result="PASSED", message="") + + def test_case_failed(self, message): + """ + Set last test case added as FAILED + """ + self.__set_test_case_result(result="FAILED", message=message) + + def test_case_blocked(self, message): + """ + Set last test case added as BLOCKED + """ + self.__set_test_case_result(result="BLOCKED", message=message) + + def all_suts(self): + """ + Returns all the SUTs it's aware of. + """ + return self.__internals[::2] + + def all_targets(self, sut): + """ + Returns the targets for a given SUT + """ + try: + sut_idx = self.__internals.index(sut) + except: + return None + return self.__internals[sut_idx + 1][5::3] + + def add_failed_sut(self, sut, msg): + """ + Sets the given SUT as failing due to msg + """ + self.__failed_suts[sut] = msg + + def remove_failed_sut(self, sut): + """ + Remove the given SUT from failed SUTs collection + """ + if sut in self.__failed_suts: + self.__failed_suts.pop(sut) + + def add_failed_target(self, sut, target, msg): + """ + Sets the given SUT, target as failing due to msg + """ + self.__failed_targets[sut + target] = msg + + def remove_failed_target(self, sut, target): + """ + Remove the given SUT, target from failed targets collection + """ + key_word = sut + target + if key_word in self.__failed_targets: + self.__failed_targets.pop(key_word) + + """ + Attributes defined as properties to hide the implementation from the + presented interface. + """ + sut = property(__get_sut, __set_sut) + dpdk_version = property(__get_dpdk_version, __set_dpdk_version) + target = property(__get_target, __set_target) + test_suite = property(__get_test_suite, __set_test_suite) + test_case = property(__get_test_case, __set_test_case) + internals = property(__get_internals) -- 2.30.2