From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id B0904A0543; Wed, 24 Aug 2022 18:25:47 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 758BF42B84; Wed, 24 Aug 2022 18:25:09 +0200 (CEST) Received: from lb.pantheon.sk (lb.pantheon.sk [46.229.239.20]) by mails.dpdk.org (Postfix) with ESMTP id 54B954282F for ; Wed, 24 Aug 2022 18:25:06 +0200 (CEST) Received: from localhost (localhost [127.0.0.1]) by lb.pantheon.sk (Postfix) with ESMTP id 4F8CECD26D; Wed, 24 Aug 2022 18:25:05 +0200 (CEST) X-Virus-Scanned: amavisd-new at siecit.sk Received: from lb.pantheon.sk ([127.0.0.1]) by localhost (lb.pantheon.sk [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id oPzGsw12E5YL; Wed, 24 Aug 2022 18:25:04 +0200 (CEST) Received: from entguard.lab.pantheon.local (unknown [46.229.239.141]) by lb.pantheon.sk (Postfix) with ESMTP id 813B5CD275; Wed, 24 Aug 2022 18:24:59 +0200 (CEST) From: =?UTF-8?q?Juraj=20Linke=C5=A1?= To: thomas@monjalon.net, david.marchand@redhat.com, ronan.randles@intel.com, Honnappa.Nagarahalli@arm.com, ohilyard@iol.unh.edu, lijuan.tu@intel.com Cc: dev@dpdk.org, =?UTF-8?q?Juraj=20Linke=C5=A1?= Subject: [RFC PATCH v1 08/10] dts: add test runner and statistics collector Date: Wed, 24 Aug 2022 16:24:52 +0000 Message-Id: <20220824162454.394285-9-juraj.linkes@pantheon.tech> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20220824162454.394285-1-juraj.linkes@pantheon.tech> References: <20220824162454.394285-1-juraj.linkes@pantheon.tech> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Add functions responsible for initializing testbed setup, testcase discovery and execution. The stats collector gathers the pass/fail results and provides a short report. Signed-off-by: Juraj Linkeš --- dts/framework/dts.py | 174 +++++++++++++++++++++++++++++--- dts/framework/stats_reporter.py | 70 +++++++++++++ 2 files changed, 232 insertions(+), 12 deletions(-) create mode 100644 dts/framework/stats_reporter.py diff --git a/dts/framework/dts.py b/dts/framework/dts.py index 1938ea6af8..39e07d9eec 100644 --- a/dts/framework/dts.py +++ b/dts/framework/dts.py @@ -4,38 +4,179 @@ # Copyright(c) 2022 University of New Hampshire # +import os import sys +import traceback from typing import Iterable, Optional import framework.logger as logger from .config import CONFIGURATION +from .exception import VerifyFailure from .logger import getLogger from .node import Node -from .settings import SETTINGS -from .utils import check_dts_python_version +from .settings import SETTINGS, DTSRuntimeError, DTSRuntimeErrors +from .stats_reporter import StatsReporter +from .sut_node import SutNode +from .test_case import TestCase +from .test_result import Result +from .tg_node import TrafficGeneratorNode +from .utils import check_dts_python_version, get_subclasses +requested_tests: Optional[list[str]] = None +result: Optional[Result] = None +stats_report: Optional[StatsReporter] = None log_handler: Optional[logger.DTSLOG] = None +def dts_nodes_init(): + """ + Create dts SUT/TG instance and initialize them. + """ + sut_nodes = [] + tg_node = None + for node_config in CONFIGURATION.nodes: + if hasattr(node_config, 'memory_channels'): + sut_nodes.append(SutNode(node_config)) + else: + tg_node = TrafficGeneratorNode(node_config) + tg_node.set_re_run(SETTINGS.re_run if SETTINGS.re_run > 0 else 0) + + return sut_nodes, tg_node + + +def dts_run_prerequisites(nodes): + """ + Run dts prerequisites function. + """ + # TODO nodes config contains both sut and tg nodes + try: + for node in nodes: + node.prerequisites() + except Exception as ex: + log_handler.error("NODE PREREQ EXCEPTION " + traceback.format_exc()) + result.add_failed_sut(node, str(ex)) + if isinstance(node, TrafficGeneratorNode): + DTSRuntimeError = DTSRuntimeErrors.TG_SETUP_ERR + else: + DTSRuntimeError = DTSRuntimeErrors.SUT_SETUP_ERR + return False + return True + + +def dts_run_target(sut_nodes, tg_node, targets, test_suites): + """ + Run each target in execution targets. + """ + for target in targets: + target = str(target) + log_handler.info("\nTARGET " + target) + result.target = target + + try: + for sut_node in sut_nodes: + sut_node.set_target(target) + except AssertionError as ex: + DTSRuntimeError = DTSRuntimeErrors.DPDK_BUILD_ERR + log_handler.error(" TARGET ERROR: " + str(ex)) + result.add_failed_target(result.sut, target, str(ex)) + continue + except Exception as ex: + DTSRuntimeError = DTSRuntimeErrors.GENERIC_ERR + log_handler.error(" !!! DEBUG IT: " + traceback.format_exc()) + result.add_failed_target(result.sut, target, str(ex)) + continue + + dts_run_suite(sut_nodes, tg_node, test_suites, target) + + +def dts_run_suite(sut_nodes, tg_node, test_suites, target): + """ + Run each suite in test suite list. + """ + for suite_name in test_suites: + try: + # check whether config the test cases + append_requested_case_list = None + if ":" in suite_name: + case_list = suite_name[suite_name.find(":") + 1 :] + append_requested_case_list = case_list.split("\\") + suite_name = suite_name[: suite_name.find(":")] + result.test_suite = suite_name + _suite_full_name = "TestSuite_" + suite_name + suite_module = __import__( + "tests." + _suite_full_name, fromlist=[_suite_full_name] + ) + for test_classname, test_class in get_subclasses(suite_module, TestCase): + + suite_obj = test_class(sut_nodes, tg_node, target, suite_name) + suite_obj.init_log() + suite_obj.set_requested_cases(requested_tests) + suite_obj.set_requested_cases(append_requested_case_list) + + log_handler.info("\nTEST SUITE : " + test_classname) + + if suite_obj.execute_setup_all(): + suite_obj.execute_test_cases() + + # save suite cases result + result.copy_suite(suite_obj.get_result()) + + log_handler.info("\nTEST SUITE ENDED: " + test_classname) + except VerifyFailure: + DTSRuntimeError = DTSRuntimeErrors.SUITE_EXECUTE_ERR + log_handler.error(" !!! DEBUG IT: " + traceback.format_exc()) + except KeyboardInterrupt: + # stop/save result/skip execution + log_handler.error(" !!! STOPPING DTS") + break + except Exception as e: + DTSRuntimeError = DTSRuntimeErrors.GENERIC_ERR + log_handler.error(str(e)) + finally: + try: + suite_obj.execute_tear_downall() + except Exception as e: + DTSRuntimeError = DTSRuntimeErrors.GENERIC_ERR + log_handler.error(str(e)) + try: + stats_report.save(result) + except Exception as e: + DTSRuntimeError = DTSRuntimeErrors.GENERIC_ERR + log_handler.error(str(e)) + + def run_all() -> None: """ Main process of DTS, it will run all test suites in the config file. """ global log_handler + global result + global stats_report + global requested_tests # check the python version of the server that run dts check_dts_python_version() + # prepare the output folder + if not os.path.exists(SETTINGS.output_dir): + os.mkdir(SETTINGS.output_dir) + # init log_handler handler if SETTINGS.verbose is True: logger.set_verbose() log_handler = getLogger("dts") - nodes = {} - # This try/finally block means "Run the try block, if there is an exception, + # run designated test cases + requested_tests = SETTINGS.test_cases + + # report objects + stats_report = StatsReporter(SETTINGS.output_dir + "/statistics.txt") + result = Result() + + # This try/finally block means "Run the try block and if there is an exception, # run the finally block before passing it upward. If there is not an exception, # run the finally block after the try block is finished." This helps avoid the # problem of python's interpreter exit context, which essentially prevents you @@ -45,26 +186,35 @@ def run_all() -> None: # An except block SHOULD NOT be added to this. A failure at this level should # deliver a full stack trace for debugging, since the only place that exceptions # should be caught and handled is in the testing code. + nodes = [] try: # for all Execution sections for execution in CONFIGURATION.executions: - sut_config = execution.system_under_test - if sut_config.name not in nodes: - nodes[sut_config.name] = Node(sut_config) + sut_nodes, tg_node = dts_nodes_init() + nodes.extend(sut_nodes) + nodes.append(tg_node) + + # Run SUT prerequisites + if dts_run_prerequisites(nodes) is False: + continue + result.dpdk_version = sut_nodes[0].dpdk_version + dts_run_target( + sut_nodes, tg_node, execution.target_descriptions, execution.test_suites + ) finally: - quit_execution(nodes.values()) + quit_execution(nodes) -def quit_execution(sut_nodes: Iterable[Node]) -> None: +def quit_execution(nodes: Iterable[Node]) -> None: """ Close session to SUT and TG before quit. Return exit status when failure occurred. """ - for sut_node in sut_nodes: + for node in nodes: # close all session - sut_node.node_exit() + node.node_exit() if log_handler is not None: log_handler.info("DTS ended") - sys.exit(0) + sys.exit(DTSRuntimeError) diff --git a/dts/framework/stats_reporter.py b/dts/framework/stats_reporter.py new file mode 100644 index 0000000000..a8d589bc7b --- /dev/null +++ b/dts/framework/stats_reporter.py @@ -0,0 +1,70 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2010-2014 Intel Corporation +# Copyright(c) 2022 PANTHEON.tech s.r.o. +# + +""" +Simple text file statistics generator +""" + + +class StatsReporter(object): + """ + Generates a small statistics file containing the number of passing, + failing and blocked tests. It makes use of a Result instance as input. + """ + + def __init__(self, filename): + self.filename = filename + + def __add_stat(self, test_result): + if test_result is not None: + if test_result[0] == "PASSED": + self.passed += 1 + if test_result[0] == "FAILED": + self.failed += 1 + if test_result[0] == "BLOCKED": + self.blocked += 1 + self.total += 1 + + def __count_stats(self): + for sut in self.result.all_suts(): + for target in self.result.all_targets(sut): + for suite in self.result.all_test_suites(sut, target): + for case in self.result.all_test_cases(sut, target, suite): + test_result = self.result.result_for(sut, target, suite, case) + if len(test_result): + self.__add_stat(test_result) + + def __write_stats(self): + sut_nodes = self.result.all_suts() + if len(sut_nodes) == 1: + self.stats_file.write( + "dpdk_version = {}\n".format( + self.result.current_dpdk_version(sut_nodes[0]) + ) + ) + else: + for sut in sut_nodes: + dpdk_version = self.result.current_dpdk_version(sut) + self.stats_file.write( + "{}.dpdk_version = {}\n".format(sut, dpdk_version) + ) + self.__count_stats() + self.stats_file.write("Passed = %d\n" % self.passed) + self.stats_file.write("Failed = %d\n" % self.failed) + self.stats_file.write("Blocked = %d\n" % self.blocked) + rate = 0 + if self.total > 0: + rate = self.passed * 100.0 / self.total + self.stats_file.write("Pass rate = %.1f\n" % rate) + + def save(self, result): + self.passed = 0 + self.failed = 0 + self.blocked = 0 + self.total = 0 + self.stats_file = open(self.filename, "w+") + self.result = result + self.__write_stats() + self.stats_file.close() -- 2.30.2