[RFC PATCH v1 07/15] dts: merge DTS framework/dts.py to DPDK

["Juraj Linkeš" <juraj.linkes@pantheon.tech>]

---
 dts/framework/dts.py | 721 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 721 insertions(+)
 create mode 100644 dts/framework/dts.py

diff --git a/dts/framework/dts.py b/dts/framework/dts.py
new file mode 100644
index 0000000000..c06bcedf33
--- /dev/null
+++ b/dts/framework/dts.py
@@ -0,0 +1,721 @@
+# BSD LICENSE
+#
+# Copyright(c) 2010-2019 Intel Corporation. All rights reserved.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+#   * Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#   * Redistributions in binary form must reproduce the above copyright
+#     notice, this list of conditions and the following disclaimer in
+#     the documentation and/or other materials provided with the
+#     distribution.
+#   * Neither the name of Intel Corporation nor the names of its
+#     contributors may be used to endorse or promote products derived
+#     from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import atexit  # register callback when exit
+import configparser  # config parse module
+import copy  # copy module for duplicate variable
+import imp
+import inspect  # load attribute
+import json  # json format
+import logging
+import os  # operation system module
+import re  # regular expressions module
+import signal  # signal module for debug mode
+import sys  # system module
+import time  # time module for unique output folder
+import traceback  # exception traceback
+
+import framework.debugger as debugger
+import framework.logger as logger
+import framework.rst as rst  # rst file support
+import framework.settings as settings  # dts settings
+from framework.asan_test import ASanTestProcess
+
+from .checkCase import CheckCase
+from .config import CrbsConf
+from .dut import Dut
+from .excel_reporter import ExcelReporter
+from .exception import ConfigParseException, TimeoutException, VerifyFailure
+from .json_reporter import JSONReporter
+from .logger import getLogger
+from .serializer import Serializer
+from .stats_reporter import StatsReporter
+from .test_case import TestCase
+from .test_result import Result
+from .tester import Tester
+from .utils import (
+    check_dts_python_version,
+    copy_instance_attr,
+    create_parallel_locks,
+    get_subclasses,
+)
+
+imp.reload(sys)
+
+requested_tests = None
+result = None
+excel_report = None
+json_report = None
+stats_report = None
+log_handler = None
+
+
+def dts_parse_param(config, section, log_handler):
+    """
+    Parse execution file parameters.
+    """
+    # default value
+    performance = False
+    functional = False
+    # Set parameters
+    shared_lib_parameters = ""
+    try:
+        shared_lib_parameters = config.get(section, "shared_lib_param")
+    except Exception as e:
+        shared_lib_parameters = ""
+    shared_lib_parameters = shared_lib_parameters.split(":")
+
+    parameters = config.get(section, "parameters").split(":")
+    drivername = config.get(section, "drivername").split("=")[-1]
+
+    driver = drivername.split(":")
+    if len(driver) == 2:
+        drivername = driver[0]
+        drivermode = driver[1]
+        settings.save_global_setting(settings.HOST_DRIVER_MODE_SETTING, drivermode)
+    else:
+        drivername = driver[0]
+
+    settings.save_global_setting(settings.HOST_DRIVER_SETTING, drivername)
+
+    shared_lib_paramDict = dict()
+    for param in shared_lib_parameters:
+        (key, _, value) = param.partition("=")
+        shared_lib_paramDict[key] = value
+    if (
+        "use_shared_lib" in shared_lib_paramDict
+        and shared_lib_paramDict["use_shared_lib"].lower() == "true"
+    ):
+        settings.save_global_setting(settings.HOST_SHARED_LIB_SETTING, "true")
+    else:
+        settings.save_global_setting(settings.HOST_SHARED_LIB_SETTING, "false")
+
+    if "shared_lib_path" in shared_lib_paramDict:
+        if not shared_lib_paramDict["shared_lib_path"] and settings.load_global_setting(
+            settings.HOST_SHARED_LIB_SETTING
+        ):
+            raise ValueError("use shared lib but shared lib path is empty")
+        settings.save_global_setting(
+            settings.HOST_SHARED_LIB_PATH, shared_lib_paramDict["shared_lib_path"]
+        )
+
+    paramDict = dict()
+    for param in parameters:
+        (key, _, value) = param.partition("=")
+        paramDict[key] = value
+
+    if "perf" in paramDict and paramDict["perf"] == "true":
+        performance = True
+    if "func" in paramDict and paramDict["func"] == "true":
+        functional = True
+
+    if "nic_type" not in paramDict:
+        paramDict["nic_type"] = "any"
+
+    settings.save_global_setting(settings.HOST_NIC_SETTING, paramDict["nic_type"])
+
+    # save perf/function setting in environment
+    if performance:
+        settings.save_global_setting(settings.PERF_SETTING, "yes")
+    else:
+        settings.save_global_setting(settings.PERF_SETTING, "no")
+
+    if functional:
+        settings.save_global_setting(settings.FUNC_SETTING, "yes")
+    else:
+        settings.save_global_setting(settings.FUNC_SETTING, "no")
+
+
+def dts_parse_config(config, section):
+    """
+    Parse execution file configuration.
+    """
+    duts = [dut_.strip() for dut_ in config.get(section, "crbs").split(",")]
+    targets = [target.strip() for target in config.get(section, "targets").split(",")]
+    test_suites = [
+        suite.strip() for suite in config.get(section, "test_suites").split(",")
+    ]
+    try:
+        rx_mode = config.get(section, "rx_mode").strip().lower()
+    except:
+        rx_mode = "default"
+
+    settings.save_global_setting(settings.DPDK_RXMODE_SETTING, rx_mode)
+
+    for suite in test_suites:
+        if suite == "":
+            test_suites.remove(suite)
+
+    return duts, targets, test_suites
+
+
+def dts_parse_commands(commands):
+    """
+    Parse command information from dts arguments
+    """
+    dts_commands = []
+
+    if commands is None:
+        return dts_commands
+
+    args_format = {"shell": 0, "crb": 1, "stage": 2, "check": 3, "max_num": 4}
+    cmd_fmt = r"\[(.*)\]"
+
+    for command in commands:
+        args = command.split(":")
+        if len(args) != args_format["max_num"]:
+            log_handler.error("Command [%s] is lack of arguments" % command)
+            raise VerifyFailure("commands input is not corrected")
+            continue
+        dts_command = {}
+
+        m = re.match(cmd_fmt, args[0])
+        if m:
+            cmds = m.group(1).split(",")
+            shell_cmd = ""
+            for cmd in cmds:
+                shell_cmd += cmd
+                shell_cmd += " "
+            dts_command["command"] = shell_cmd[:-1]
+        else:
+            dts_command["command"] = args[0]
+        if args[1] == "tester":
+            dts_command["host"] = "tester"
+        else:
+            dts_command["host"] = "dut"
+        if args[2] == "post-init":
+            dts_command["stage"] = "post-init"
+        else:
+            dts_command["stage"] = "pre-init"
+        if args[3] == "ignore":
+            dts_command["verify"] = False
+        else:
+            dts_command["verify"] = True
+
+        dts_commands.append(dts_command)
+
+    return dts_commands
+
+
+def dts_run_commands(crb, dts_commands):
+    """
+    Run dts input commands
+    """
+    for dts_command in dts_commands:
+        command = dts_command["command"]
+        if dts_command["host"] in crb.NAME:
+            if crb.stage == dts_command["stage"]:
+                ret = crb.send_expect(command, expected="# ", verify=True)
+                if type(ret) is int:
+                    log_handler.error("[%s] return failure" % command)
+                    if dts_command["verify"] is True:
+                        raise VerifyFailure("Command execution failed")
+
+
+def get_project_obj(project_name, super_class, crbInst, serializer, dut_id):
+    """
+    Load project module and return crb instance.
+    """
+    project_obj = None
+    PROJECT_MODULE_PREFIX = "project_"
+    try:
+        _project_name = PROJECT_MODULE_PREFIX + project_name
+        project_module = __import__(
+            "framework." + _project_name, fromlist=[_project_name]
+        )
+
+        for project_subclassname, project_subclass in get_subclasses(
+            project_module, super_class
+        ):
+            project_obj = project_subclass(crbInst, serializer, dut_id)
+        if project_obj is None:
+            project_obj = super_class(crbInst, serializer, dut_id)
+    except Exception as e:
+        log_handler.info("LOAD PROJECT MODULE INFO: " + str(e))
+        project_obj = super_class(crbInst, serializer, dut_id)
+
+    return project_obj
+
+
+def dts_log_testsuite(duts, tester, suite_obj, log_handler, test_classname):
+    """
+    Change to SUITE self logger handler.
+    """
+    log_handler.config_suite(test_classname, "dts")
+    tester.logger.config_suite(test_classname, "tester")
+    if hasattr(tester, "logger_alt"):
+        tester.logger_alt.config_suite(test_classname, "tester")
+    if hasattr(tester, "logger_scapy"):
+        tester.logger_scapy.config_suite(test_classname, "tester")
+
+    for dutobj in duts:
+        dutobj.logger.config_suite(test_classname, "dut")
+        dutobj.test_classname = test_classname
+
+    try:
+        if tester.it_uses_external_generator():
+            if (
+                tester.is_pktgen
+                and hasattr(tester, "pktgen")
+                and getattr(tester, "pktgen")
+            ):
+                tester.pktgen.logger.config_suite(test_classname, "pktgen")
+    except Exception as ex:
+        pass
+
+
+def dts_log_execution(duts, tester, log_handler):
+    """
+    Change to DTS default logger handler.
+    """
+    log_handler.config_execution("dts")
+    tester.logger.config_execution("tester")
+
+    for dutobj in duts:
+        dutobj.logger.config_execution(
+            "dut" + settings.LOG_NAME_SEP + "%s" % dutobj.crb["My IP"]
+        )
+
+    try:
+        if tester.it_uses_external_generator():
+            if (
+                tester.is_pktgen
+                and hasattr(tester, "pktgen")
+                and getattr(tester, "pktgen")
+            ):
+                tester.pktgen.logger.config_execution("pktgen")
+    except Exception as ex:
+        pass
+
+
+def dts_crbs_init(
+    crbInsts, skip_setup, read_cache, project, base_dir, serializer, virttype
+):
+    """
+    Create dts dut/tester instance and initialize them.
+    """
+    duts = []
+
+    serializer.set_serialized_filename(
+        settings.FOLDERS["Output"] + "/.%s.cache" % crbInsts[0]["IP"]
+    )
+    serializer.load_from_file()
+
+    testInst = copy.copy(crbInsts[0])
+    testInst["My IP"] = crbInsts[0]["tester IP"]
+    tester = get_project_obj(project, Tester, testInst, serializer, dut_id=0)
+
+    dut_id = 0
+    for crbInst in crbInsts:
+        dutInst = copy.copy(crbInst)
+        dutInst["My IP"] = crbInst["IP"]
+        dutobj = get_project_obj(project, Dut, dutInst, serializer, dut_id=dut_id)
+        duts.append(dutobj)
+        dut_id += 1
+
+    dts_log_execution(duts, tester, log_handler)
+
+    tester.duts = duts
+    show_speedup_options_messages(read_cache, skip_setup)
+    tester.set_speedup_options(read_cache, skip_setup)
+    try:
+        tester.init_ext_gen()
+    except Exception as e:
+        log_handler.error(str(e))
+        tester.close()
+        for dutobj in duts:
+            dutobj.close()
+        raise e
+
+    nic = settings.load_global_setting(settings.HOST_NIC_SETTING)
+    for dutobj in duts:
+        dutobj.tester = tester
+        dutobj.setup_virtenv(virttype)
+        dutobj.set_speedup_options(read_cache, skip_setup)
+        dutobj.set_directory(base_dir)
+        # save execution nic setting
+        dutobj.set_nic_type(nic)
+
+    return duts, tester
+
+
+def dts_crbs_exit(duts, tester):
+    """
+    Call dut and tester exit function after execution finished
+    """
+    for dutobj in duts:
+        dutobj.crb_exit()
+
+    tester.crb_exit()
+
+
+def dts_run_prerequisties(duts, tester, pkgName, patch, dts_commands, serializer):
+    """
+    Run dts prerequisties function.
+    """
+    try:
+        dts_run_commands(tester, dts_commands)
+        tester.prerequisites()
+        dts_run_commands(tester, dts_commands)
+    except Exception as ex:
+        log_handler.error(" PREREQ EXCEPTION " + traceback.format_exc())
+        log_handler.info("CACHE: Discarding cache.")
+        serializer.discard_cache()
+        settings.report_error("TESTER_SETUP_ERR")
+        return False
+
+    try:
+        for dutobj in duts:
+            dts_run_commands(dutobj, dts_commands)
+            dutobj.set_package(pkgName, patch)
+            dutobj.prerequisites()
+            dts_run_commands(dutobj, dts_commands)
+
+        serializer.save_to_file()
+    except Exception as ex:
+        log_handler.error(" PREREQ EXCEPTION " + traceback.format_exc())
+        result.add_failed_dut(duts[0], str(ex))
+        log_handler.info("CACHE: Discarding cache.")
+        serializer.discard_cache()
+        settings.report_error("DUT_SETUP_ERR")
+        return False
+
+
+def dts_run_target(duts, tester, targets, test_suites, subtitle):
+    """
+    Run each target in execution targets.
+    """
+    for target in targets:
+        log_handler.info("\nTARGET " + target)
+        result.target = target
+
+        try:
+            drivername = settings.load_global_setting(settings.HOST_DRIVER_SETTING)
+            if drivername == "":
+                for dutobj in duts:
+                    dutobj.set_target(target, bind_dev=False)
+            else:
+                for dutobj in duts:
+                    dutobj.set_target(target)
+        except AssertionError as ex:
+            log_handler.error(" TARGET ERROR: " + str(ex))
+            settings.report_error("DPDK_BUILD_ERR")
+            result.add_failed_target(result.dut, target, str(ex))
+            continue
+        except Exception as ex:
+            settings.report_error("GENERIC_ERR")
+            log_handler.error(" !!! DEBUG IT: " + traceback.format_exc())
+            result.add_failed_target(result.dut, target, str(ex))
+            continue
+
+        dts_run_suite(duts, tester, test_suites, target, subtitle)
+
+    tester.restore_interfaces()
+
+    for dutobj in duts:
+        dutobj.stop_ports()
+        dutobj.restore_interfaces()
+        dutobj.restore_modules()
+
+
+def dts_run_suite(duts, tester, test_suites, target, subtitle):
+    """
+    Run each suite in test suite list.
+    """
+    for suite_name in test_suites:
+        try:
+            # check whether config the test cases
+            append_requested_case_list = None
+            if ":" in suite_name:
+                case_list = suite_name[suite_name.find(":") + 1 :]
+                append_requested_case_list = case_list.split("\\")
+                suite_name = suite_name[: suite_name.find(":")]
+            result.test_suite = suite_name
+            _suite_full_name = "TestSuite_" + suite_name
+            suite_module = __import__(
+                "tests." + _suite_full_name, fromlist=[_suite_full_name]
+            )
+            for test_classname, test_class in get_subclasses(suite_module, TestCase):
+
+                suite_obj = test_class(duts, tester, target, suite_name)
+                suite_obj.init_log()
+                suite_obj.set_requested_cases(requested_tests)
+                suite_obj.set_requested_cases(append_requested_case_list)
+                suite_obj.set_check_inst(check=check_case_inst)
+                suite_obj.set_subtitle(subtitle)
+                result.nic = suite_obj.nic
+
+                dts_log_testsuite(duts, tester, suite_obj, log_handler, test_classname)
+
+                log_handler.info("\nTEST SUITE : " + test_classname)
+                log_handler.info("NIC :        " + result.nic)
+
+                if suite_obj.execute_setup_all():
+                    suite_obj.execute_test_cases()
+
+                # save suite cases result
+                result.copy_suite(suite_obj.get_result())
+
+                log_handler.info("\nTEST SUITE ENDED: " + test_classname)
+                dts_log_execution(duts, tester, log_handler)
+        except VerifyFailure:
+            settings.report_error("SUITE_EXECUTE_ERR")
+            log_handler.error(" !!! DEBUG IT: " + traceback.format_exc())
+        except KeyboardInterrupt:
+            # stop/save result/skip execution
+            log_handler.error(" !!! STOPPING DTS")
+            break
+        except Exception as e:
+            settings.report_error("GENERIC_ERR")
+            log_handler.error(str(e))
+        finally:
+            try:
+                suite_obj.execute_tear_downall()
+            except Exception as e:
+                settings.report_error("GENERIC_ERR")
+                log_handler.error(str(e))
+            try:
+                save_all_results()
+            except Exception as e:
+                settings.report_error("GENERIC_ERR")
+                log_handler.error(str(e))
+
+
+def run_all(
+    config_file,
+    pkgName,
+    git,
+    patch,
+    skip_setup,
+    read_cache,
+    project,
+    suite_dir,
+    test_cases,
+    base_dir,
+    output_dir,
+    verbose,
+    virttype,
+    debug,
+    debugcase,
+    re_run,
+    commands,
+    subtitle,
+    update_expected,
+    asan,
+):
+    """
+    Main process of DTS, it will run all test suites in the config file.
+    """
+
+    global requested_tests
+    global result
+    global excel_report
+    global json_report
+    global stats_report
+    global log_handler
+    global check_case_inst
+
+    # check the python version of the server that run dts
+    check_dts_python_version()
+
+    # save global variable
+    serializer = Serializer()
+
+    # load check/support case lists
+    check_case_inst = CheckCase()
+
+    # prepare the output folder
+    if output_dir == "":
+        output_dir = settings.FOLDERS["Output"]
+
+    # prepare ASan test
+    ASanTestProcess.test_prepare(asan, output_dir)
+
+    if not os.path.exists(output_dir):
+        os.mkdir(output_dir)
+
+    # enable debug mode
+    if debug is True:
+        settings.save_global_setting(settings.DEBUG_SETTING, "yes")
+    if debugcase is True:
+        settings.save_global_setting(settings.DEBUG_CASE_SETTING, "yes")
+
+    # enable update-expected
+    if update_expected is True:
+        settings.save_global_setting(settings.UPDATE_EXPECTED, "yes")
+
+    # init log_handler handler
+    if verbose is True:
+        logger.set_verbose()
+
+    re_run = int(re_run)
+    if re_run < 0:
+        re_run = 0
+
+    logger.log_dir = output_dir
+    log_handler = getLogger("dts")
+    log_handler.config_execution("dts")
+
+    # run designated test case
+    requested_tests = test_cases
+
+    # Read config file
+    dts_cfg_folder = settings.load_global_setting(settings.DTS_CFG_FOLDER)
+    if dts_cfg_folder != "":
+        config_file = dts_cfg_folder + os.sep + config_file
+
+    config = configparser.SafeConfigParser()
+    load_cfg = config.read(config_file)
+    if len(load_cfg) == 0:
+        raise ConfigParseException(config_file)
+
+    # parse commands
+    dts_commands = dts_parse_commands(commands)
+
+    os.environ["TERM"] = "dumb"
+
+    # change rst output folder
+    rst.path2Result = output_dir
+
+    # report objects
+    excel_report = ExcelReporter(output_dir + "/test_results.xls")
+    json_report = JSONReporter(output_dir + "/test_results.json")
+    stats_report = StatsReporter(output_dir + "/statistics.txt")
+    result = Result()
+
+    crbs_conf = CrbsConf()
+    crbs = crbs_conf.load_crbs_config()
+
+    # for all Execution sections
+    for section in config.sections():
+        crbInsts = list()
+        dts_parse_param(config, section, log_handler)
+
+        # verify if the delimiter is good if the lists are vertical
+        duts, targets, test_suites = dts_parse_config(config, section)
+        for dut in duts:
+            log_handler.info("\nDUT " + dut)
+
+        # look up in crbs - to find the matching IP
+        for dut in duts:
+            for crb in crbs:
+                if crb["section"] == dut:
+                    crbInsts.append(crb)
+                    break
+
+        # only run on the dut in known crbs
+        if len(crbInsts) == 0:
+            log_handler.error(" SKIP UNKNOWN CRB")
+            continue
+
+        result.dut = duts[0]
+
+        # init global lock
+        create_parallel_locks(len(duts))
+
+        # init dut, tester crb
+        duts, tester = dts_crbs_init(
+            crbInsts, skip_setup, read_cache, project, base_dir, serializer, virttype
+        )
+        tester.set_re_run(re_run)
+        # register exit action
+        atexit.register(quit_execution, duts, tester)
+
+        check_case_inst.check_dut(duts[0])
+
+        # Run DUT prerequisites
+        if (
+            dts_run_prerequisties(
+                duts, tester, pkgName, patch, dts_commands, serializer
+            )
+            is False
+        ):
+            dts_crbs_exit(duts, tester)
+            continue
+        result.kdriver = duts[0].nic.default_driver + "-" + duts[0].nic.driver_version
+        result.firmware = duts[0].nic.firmware
+        result.package = (
+            duts[0].nic.pkg["type"] + " " + duts[0].nic.pkg["version"]
+            if duts[0].nic.pkg
+            else None
+        )
+        result.driver = settings.load_global_setting(settings.HOST_DRIVER_SETTING)
+        result.dpdk_version = duts[0].dpdk_version
+        dts_run_target(duts, tester, targets, test_suites, subtitle)
+
+        dts_crbs_exit(duts, tester)
+
+    save_all_results()
+
+    # process ASan test report
+    ASanTestProcess.test_process()
+
+
+def show_speedup_options_messages(read_cache, skip_setup):
+    if read_cache:
+        log_handler.info("CACHE: All configuration will be read from cache.")
+    else:
+        log_handler.info("CACHE: Cache will not be read.")
+
+    if skip_setup:
+        log_handler.info("SKIP: Skipping DPDK setup.")
+    else:
+        log_handler.info("SKIP: The DPDK setup steps will be executed.")
+
+
+def save_all_results():
+    """
+    Save all result to files.
+    """
+    excel_report.save(result)
+    json_report.save(result)
+    stats_report.save(result)
+
+
+def quit_execution(duts, tester):
+    """
+    Close session to DUT and tester before quit.
+    Return exit status when failure occurred.
+    """
+    # close all nics
+    for dutobj in duts:
+        if getattr(dutobj, "ports_info", None) and dutobj.ports_info:
+            for port_info in dutobj.ports_info:
+                netdev = port_info["port"]
+                netdev.close()
+        # close all session
+        dutobj.close()
+    if tester is not None:
+        tester.close()
+    log_handler.info("DTS ended")
+
+    # return value
+    settings.exit_error()
-- 
2.20.1