From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id D5C7BA0507; Wed, 6 Apr 2022 16:57:31 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id E7B3E4286C; Wed, 6 Apr 2022 16:56:30 +0200 (CEST) Received: from lb.pantheon.sk (lb.pantheon.sk [46.229.239.20]) by mails.dpdk.org (Postfix) with ESMTP id 0539342855 for ; Wed, 6 Apr 2022 16:56:27 +0200 (CEST) Received: from localhost (localhost [127.0.0.1]) by lb.pantheon.sk (Postfix) with ESMTP id 55D521B27A7; Wed, 6 Apr 2022 16:56:26 +0200 (CEST) X-Virus-Scanned: amavisd-new at siecit.sk Received: from lb.pantheon.sk ([127.0.0.1]) by localhost (lb.pantheon.sk [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id uL9i8VW39e6z; Wed, 6 Apr 2022 16:56:24 +0200 (CEST) Received: from entguard.lab.pantheon.local (unknown [46.229.239.141]) by lb.pantheon.sk (Postfix) with ESMTP id 7B1C11B1F4D; Wed, 6 Apr 2022 16:56:10 +0200 (CEST) From: =?UTF-8?q?Juraj=20Linke=C5=A1?= To: thomas@monjalon.net, david.marchand@redhat.com, Honnappa.Nagarahalli@arm.com, ohilyard@iol.unh.edu, lijuan.tu@intel.com Cc: dev@dpdk.org, =?UTF-8?q?Juraj=20Linke=C5=A1?= Subject: [RFC PATCH v1 07/15] dts: merge DTS framework/dts.py to DPDK Date: Wed, 6 Apr 2022 14:55:58 +0000 Message-Id: <20220406145606.2913834-8-juraj.linkes@pantheon.tech> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20220406145606.2913834-1-juraj.linkes@pantheon.tech> References: <20220406145606.2913834-1-juraj.linkes@pantheon.tech> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org --- dts/framework/dts.py | 721 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 721 insertions(+) create mode 100644 dts/framework/dts.py diff --git a/dts/framework/dts.py b/dts/framework/dts.py new file mode 100644 index 0000000000..c06bcedf33 --- /dev/null +++ b/dts/framework/dts.py @@ -0,0 +1,721 @@ +# BSD LICENSE +# +# Copyright(c) 2010-2019 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import atexit # register callback when exit +import configparser # config parse module +import copy # copy module for duplicate variable +import imp +import inspect # load attribute +import json # json format +import logging +import os # operation system module +import re # regular expressions module +import signal # signal module for debug mode +import sys # system module +import time # time module for unique output folder +import traceback # exception traceback + +import framework.debugger as debugger +import framework.logger as logger +import framework.rst as rst # rst file support +import framework.settings as settings # dts settings +from framework.asan_test import ASanTestProcess + +from .checkCase import CheckCase +from .config import CrbsConf +from .dut import Dut +from .excel_reporter import ExcelReporter +from .exception import ConfigParseException, TimeoutException, VerifyFailure +from .json_reporter import JSONReporter +from .logger import getLogger +from .serializer import Serializer +from .stats_reporter import StatsReporter +from .test_case import TestCase +from .test_result import Result +from .tester import Tester +from .utils import ( + check_dts_python_version, + copy_instance_attr, + create_parallel_locks, + get_subclasses, +) + +imp.reload(sys) + +requested_tests = None +result = None +excel_report = None +json_report = None +stats_report = None +log_handler = None + + +def dts_parse_param(config, section, log_handler): + """ + Parse execution file parameters. + """ + # default value + performance = False + functional = False + # Set parameters + shared_lib_parameters = "" + try: + shared_lib_parameters = config.get(section, "shared_lib_param") + except Exception as e: + shared_lib_parameters = "" + shared_lib_parameters = shared_lib_parameters.split(":") + + parameters = config.get(section, "parameters").split(":") + drivername = config.get(section, "drivername").split("=")[-1] + + driver = drivername.split(":") + if len(driver) == 2: + drivername = driver[0] + drivermode = driver[1] + settings.save_global_setting(settings.HOST_DRIVER_MODE_SETTING, drivermode) + else: + drivername = driver[0] + + settings.save_global_setting(settings.HOST_DRIVER_SETTING, drivername) + + shared_lib_paramDict = dict() + for param in shared_lib_parameters: + (key, _, value) = param.partition("=") + shared_lib_paramDict[key] = value + if ( + "use_shared_lib" in shared_lib_paramDict + and shared_lib_paramDict["use_shared_lib"].lower() == "true" + ): + settings.save_global_setting(settings.HOST_SHARED_LIB_SETTING, "true") + else: + settings.save_global_setting(settings.HOST_SHARED_LIB_SETTING, "false") + + if "shared_lib_path" in shared_lib_paramDict: + if not shared_lib_paramDict["shared_lib_path"] and settings.load_global_setting( + settings.HOST_SHARED_LIB_SETTING + ): + raise ValueError("use shared lib but shared lib path is empty") + settings.save_global_setting( + settings.HOST_SHARED_LIB_PATH, shared_lib_paramDict["shared_lib_path"] + ) + + paramDict = dict() + for param in parameters: + (key, _, value) = param.partition("=") + paramDict[key] = value + + if "perf" in paramDict and paramDict["perf"] == "true": + performance = True + if "func" in paramDict and paramDict["func"] == "true": + functional = True + + if "nic_type" not in paramDict: + paramDict["nic_type"] = "any" + + settings.save_global_setting(settings.HOST_NIC_SETTING, paramDict["nic_type"]) + + # save perf/function setting in environment + if performance: + settings.save_global_setting(settings.PERF_SETTING, "yes") + else: + settings.save_global_setting(settings.PERF_SETTING, "no") + + if functional: + settings.save_global_setting(settings.FUNC_SETTING, "yes") + else: + settings.save_global_setting(settings.FUNC_SETTING, "no") + + +def dts_parse_config(config, section): + """ + Parse execution file configuration. + """ + duts = [dut_.strip() for dut_ in config.get(section, "crbs").split(",")] + targets = [target.strip() for target in config.get(section, "targets").split(",")] + test_suites = [ + suite.strip() for suite in config.get(section, "test_suites").split(",") + ] + try: + rx_mode = config.get(section, "rx_mode").strip().lower() + except: + rx_mode = "default" + + settings.save_global_setting(settings.DPDK_RXMODE_SETTING, rx_mode) + + for suite in test_suites: + if suite == "": + test_suites.remove(suite) + + return duts, targets, test_suites + + +def dts_parse_commands(commands): + """ + Parse command information from dts arguments + """ + dts_commands = [] + + if commands is None: + return dts_commands + + args_format = {"shell": 0, "crb": 1, "stage": 2, "check": 3, "max_num": 4} + cmd_fmt = r"\[(.*)\]" + + for command in commands: + args = command.split(":") + if len(args) != args_format["max_num"]: + log_handler.error("Command [%s] is lack of arguments" % command) + raise VerifyFailure("commands input is not corrected") + continue + dts_command = {} + + m = re.match(cmd_fmt, args[0]) + if m: + cmds = m.group(1).split(",") + shell_cmd = "" + for cmd in cmds: + shell_cmd += cmd + shell_cmd += " " + dts_command["command"] = shell_cmd[:-1] + else: + dts_command["command"] = args[0] + if args[1] == "tester": + dts_command["host"] = "tester" + else: + dts_command["host"] = "dut" + if args[2] == "post-init": + dts_command["stage"] = "post-init" + else: + dts_command["stage"] = "pre-init" + if args[3] == "ignore": + dts_command["verify"] = False + else: + dts_command["verify"] = True + + dts_commands.append(dts_command) + + return dts_commands + + +def dts_run_commands(crb, dts_commands): + """ + Run dts input commands + """ + for dts_command in dts_commands: + command = dts_command["command"] + if dts_command["host"] in crb.NAME: + if crb.stage == dts_command["stage"]: + ret = crb.send_expect(command, expected="# ", verify=True) + if type(ret) is int: + log_handler.error("[%s] return failure" % command) + if dts_command["verify"] is True: + raise VerifyFailure("Command execution failed") + + +def get_project_obj(project_name, super_class, crbInst, serializer, dut_id): + """ + Load project module and return crb instance. + """ + project_obj = None + PROJECT_MODULE_PREFIX = "project_" + try: + _project_name = PROJECT_MODULE_PREFIX + project_name + project_module = __import__( + "framework." + _project_name, fromlist=[_project_name] + ) + + for project_subclassname, project_subclass in get_subclasses( + project_module, super_class + ): + project_obj = project_subclass(crbInst, serializer, dut_id) + if project_obj is None: + project_obj = super_class(crbInst, serializer, dut_id) + except Exception as e: + log_handler.info("LOAD PROJECT MODULE INFO: " + str(e)) + project_obj = super_class(crbInst, serializer, dut_id) + + return project_obj + + +def dts_log_testsuite(duts, tester, suite_obj, log_handler, test_classname): + """ + Change to SUITE self logger handler. + """ + log_handler.config_suite(test_classname, "dts") + tester.logger.config_suite(test_classname, "tester") + if hasattr(tester, "logger_alt"): + tester.logger_alt.config_suite(test_classname, "tester") + if hasattr(tester, "logger_scapy"): + tester.logger_scapy.config_suite(test_classname, "tester") + + for dutobj in duts: + dutobj.logger.config_suite(test_classname, "dut") + dutobj.test_classname = test_classname + + try: + if tester.it_uses_external_generator(): + if ( + tester.is_pktgen + and hasattr(tester, "pktgen") + and getattr(tester, "pktgen") + ): + tester.pktgen.logger.config_suite(test_classname, "pktgen") + except Exception as ex: + pass + + +def dts_log_execution(duts, tester, log_handler): + """ + Change to DTS default logger handler. + """ + log_handler.config_execution("dts") + tester.logger.config_execution("tester") + + for dutobj in duts: + dutobj.logger.config_execution( + "dut" + settings.LOG_NAME_SEP + "%s" % dutobj.crb["My IP"] + ) + + try: + if tester.it_uses_external_generator(): + if ( + tester.is_pktgen + and hasattr(tester, "pktgen") + and getattr(tester, "pktgen") + ): + tester.pktgen.logger.config_execution("pktgen") + except Exception as ex: + pass + + +def dts_crbs_init( + crbInsts, skip_setup, read_cache, project, base_dir, serializer, virttype +): + """ + Create dts dut/tester instance and initialize them. + """ + duts = [] + + serializer.set_serialized_filename( + settings.FOLDERS["Output"] + "/.%s.cache" % crbInsts[0]["IP"] + ) + serializer.load_from_file() + + testInst = copy.copy(crbInsts[0]) + testInst["My IP"] = crbInsts[0]["tester IP"] + tester = get_project_obj(project, Tester, testInst, serializer, dut_id=0) + + dut_id = 0 + for crbInst in crbInsts: + dutInst = copy.copy(crbInst) + dutInst["My IP"] = crbInst["IP"] + dutobj = get_project_obj(project, Dut, dutInst, serializer, dut_id=dut_id) + duts.append(dutobj) + dut_id += 1 + + dts_log_execution(duts, tester, log_handler) + + tester.duts = duts + show_speedup_options_messages(read_cache, skip_setup) + tester.set_speedup_options(read_cache, skip_setup) + try: + tester.init_ext_gen() + except Exception as e: + log_handler.error(str(e)) + tester.close() + for dutobj in duts: + dutobj.close() + raise e + + nic = settings.load_global_setting(settings.HOST_NIC_SETTING) + for dutobj in duts: + dutobj.tester = tester + dutobj.setup_virtenv(virttype) + dutobj.set_speedup_options(read_cache, skip_setup) + dutobj.set_directory(base_dir) + # save execution nic setting + dutobj.set_nic_type(nic) + + return duts, tester + + +def dts_crbs_exit(duts, tester): + """ + Call dut and tester exit function after execution finished + """ + for dutobj in duts: + dutobj.crb_exit() + + tester.crb_exit() + + +def dts_run_prerequisties(duts, tester, pkgName, patch, dts_commands, serializer): + """ + Run dts prerequisties function. + """ + try: + dts_run_commands(tester, dts_commands) + tester.prerequisites() + dts_run_commands(tester, dts_commands) + except Exception as ex: + log_handler.error(" PREREQ EXCEPTION " + traceback.format_exc()) + log_handler.info("CACHE: Discarding cache.") + serializer.discard_cache() + settings.report_error("TESTER_SETUP_ERR") + return False + + try: + for dutobj in duts: + dts_run_commands(dutobj, dts_commands) + dutobj.set_package(pkgName, patch) + dutobj.prerequisites() + dts_run_commands(dutobj, dts_commands) + + serializer.save_to_file() + except Exception as ex: + log_handler.error(" PREREQ EXCEPTION " + traceback.format_exc()) + result.add_failed_dut(duts[0], str(ex)) + log_handler.info("CACHE: Discarding cache.") + serializer.discard_cache() + settings.report_error("DUT_SETUP_ERR") + return False + + +def dts_run_target(duts, tester, targets, test_suites, subtitle): + """ + Run each target in execution targets. + """ + for target in targets: + log_handler.info("\nTARGET " + target) + result.target = target + + try: + drivername = settings.load_global_setting(settings.HOST_DRIVER_SETTING) + if drivername == "": + for dutobj in duts: + dutobj.set_target(target, bind_dev=False) + else: + for dutobj in duts: + dutobj.set_target(target) + except AssertionError as ex: + log_handler.error(" TARGET ERROR: " + str(ex)) + settings.report_error("DPDK_BUILD_ERR") + result.add_failed_target(result.dut, target, str(ex)) + continue + except Exception as ex: + settings.report_error("GENERIC_ERR") + log_handler.error(" !!! DEBUG IT: " + traceback.format_exc()) + result.add_failed_target(result.dut, target, str(ex)) + continue + + dts_run_suite(duts, tester, test_suites, target, subtitle) + + tester.restore_interfaces() + + for dutobj in duts: + dutobj.stop_ports() + dutobj.restore_interfaces() + dutobj.restore_modules() + + +def dts_run_suite(duts, tester, test_suites, target, subtitle): + """ + Run each suite in test suite list. + """ + for suite_name in test_suites: + try: + # check whether config the test cases + append_requested_case_list = None + if ":" in suite_name: + case_list = suite_name[suite_name.find(":") + 1 :] + append_requested_case_list = case_list.split("\\") + suite_name = suite_name[: suite_name.find(":")] + result.test_suite = suite_name + _suite_full_name = "TestSuite_" + suite_name + suite_module = __import__( + "tests." + _suite_full_name, fromlist=[_suite_full_name] + ) + for test_classname, test_class in get_subclasses(suite_module, TestCase): + + suite_obj = test_class(duts, tester, target, suite_name) + suite_obj.init_log() + suite_obj.set_requested_cases(requested_tests) + suite_obj.set_requested_cases(append_requested_case_list) + suite_obj.set_check_inst(check=check_case_inst) + suite_obj.set_subtitle(subtitle) + result.nic = suite_obj.nic + + dts_log_testsuite(duts, tester, suite_obj, log_handler, test_classname) + + log_handler.info("\nTEST SUITE : " + test_classname) + log_handler.info("NIC : " + result.nic) + + if suite_obj.execute_setup_all(): + suite_obj.execute_test_cases() + + # save suite cases result + result.copy_suite(suite_obj.get_result()) + + log_handler.info("\nTEST SUITE ENDED: " + test_classname) + dts_log_execution(duts, tester, log_handler) + except VerifyFailure: + settings.report_error("SUITE_EXECUTE_ERR") + log_handler.error(" !!! DEBUG IT: " + traceback.format_exc()) + except KeyboardInterrupt: + # stop/save result/skip execution + log_handler.error(" !!! STOPPING DTS") + break + except Exception as e: + settings.report_error("GENERIC_ERR") + log_handler.error(str(e)) + finally: + try: + suite_obj.execute_tear_downall() + except Exception as e: + settings.report_error("GENERIC_ERR") + log_handler.error(str(e)) + try: + save_all_results() + except Exception as e: + settings.report_error("GENERIC_ERR") + log_handler.error(str(e)) + + +def run_all( + config_file, + pkgName, + git, + patch, + skip_setup, + read_cache, + project, + suite_dir, + test_cases, + base_dir, + output_dir, + verbose, + virttype, + debug, + debugcase, + re_run, + commands, + subtitle, + update_expected, + asan, +): + """ + Main process of DTS, it will run all test suites in the config file. + """ + + global requested_tests + global result + global excel_report + global json_report + global stats_report + global log_handler + global check_case_inst + + # check the python version of the server that run dts + check_dts_python_version() + + # save global variable + serializer = Serializer() + + # load check/support case lists + check_case_inst = CheckCase() + + # prepare the output folder + if output_dir == "": + output_dir = settings.FOLDERS["Output"] + + # prepare ASan test + ASanTestProcess.test_prepare(asan, output_dir) + + if not os.path.exists(output_dir): + os.mkdir(output_dir) + + # enable debug mode + if debug is True: + settings.save_global_setting(settings.DEBUG_SETTING, "yes") + if debugcase is True: + settings.save_global_setting(settings.DEBUG_CASE_SETTING, "yes") + + # enable update-expected + if update_expected is True: + settings.save_global_setting(settings.UPDATE_EXPECTED, "yes") + + # init log_handler handler + if verbose is True: + logger.set_verbose() + + re_run = int(re_run) + if re_run < 0: + re_run = 0 + + logger.log_dir = output_dir + log_handler = getLogger("dts") + log_handler.config_execution("dts") + + # run designated test case + requested_tests = test_cases + + # Read config file + dts_cfg_folder = settings.load_global_setting(settings.DTS_CFG_FOLDER) + if dts_cfg_folder != "": + config_file = dts_cfg_folder + os.sep + config_file + + config = configparser.SafeConfigParser() + load_cfg = config.read(config_file) + if len(load_cfg) == 0: + raise ConfigParseException(config_file) + + # parse commands + dts_commands = dts_parse_commands(commands) + + os.environ["TERM"] = "dumb" + + # change rst output folder + rst.path2Result = output_dir + + # report objects + excel_report = ExcelReporter(output_dir + "/test_results.xls") + json_report = JSONReporter(output_dir + "/test_results.json") + stats_report = StatsReporter(output_dir + "/statistics.txt") + result = Result() + + crbs_conf = CrbsConf() + crbs = crbs_conf.load_crbs_config() + + # for all Execution sections + for section in config.sections(): + crbInsts = list() + dts_parse_param(config, section, log_handler) + + # verify if the delimiter is good if the lists are vertical + duts, targets, test_suites = dts_parse_config(config, section) + for dut in duts: + log_handler.info("\nDUT " + dut) + + # look up in crbs - to find the matching IP + for dut in duts: + for crb in crbs: + if crb["section"] == dut: + crbInsts.append(crb) + break + + # only run on the dut in known crbs + if len(crbInsts) == 0: + log_handler.error(" SKIP UNKNOWN CRB") + continue + + result.dut = duts[0] + + # init global lock + create_parallel_locks(len(duts)) + + # init dut, tester crb + duts, tester = dts_crbs_init( + crbInsts, skip_setup, read_cache, project, base_dir, serializer, virttype + ) + tester.set_re_run(re_run) + # register exit action + atexit.register(quit_execution, duts, tester) + + check_case_inst.check_dut(duts[0]) + + # Run DUT prerequisites + if ( + dts_run_prerequisties( + duts, tester, pkgName, patch, dts_commands, serializer + ) + is False + ): + dts_crbs_exit(duts, tester) + continue + result.kdriver = duts[0].nic.default_driver + "-" + duts[0].nic.driver_version + result.firmware = duts[0].nic.firmware + result.package = ( + duts[0].nic.pkg["type"] + " " + duts[0].nic.pkg["version"] + if duts[0].nic.pkg + else None + ) + result.driver = settings.load_global_setting(settings.HOST_DRIVER_SETTING) + result.dpdk_version = duts[0].dpdk_version + dts_run_target(duts, tester, targets, test_suites, subtitle) + + dts_crbs_exit(duts, tester) + + save_all_results() + + # process ASan test report + ASanTestProcess.test_process() + + +def show_speedup_options_messages(read_cache, skip_setup): + if read_cache: + log_handler.info("CACHE: All configuration will be read from cache.") + else: + log_handler.info("CACHE: Cache will not be read.") + + if skip_setup: + log_handler.info("SKIP: Skipping DPDK setup.") + else: + log_handler.info("SKIP: The DPDK setup steps will be executed.") + + +def save_all_results(): + """ + Save all result to files. + """ + excel_report.save(result) + json_report.save(result) + stats_report.save(result) + + +def quit_execution(duts, tester): + """ + Close session to DUT and tester before quit. + Return exit status when failure occurred. + """ + # close all nics + for dutobj in duts: + if getattr(dutobj, "ports_info", None) and dutobj.ports_info: + for port_info in dutobj.ports_info: + netdev = port_info["port"] + netdev.close() + # close all session + dutobj.close() + if tester is not None: + tester.close() + log_handler.info("DTS ended") + + # return value + settings.exit_error() -- 2.20.1