[RFC PATCH v1 08/10] dts: add test runner and statistics collector

["Juraj Linkeš" <juraj.linkes@pantheon.tech>]

Add functions responsible for initializing testbed setup, testcase
discovery and execution. The stats collector gathers the pass/fail
results and provides a short report.

Signed-off-by: Juraj Linkeš <juraj.linkes@pantheon.tech>
---
 dts/framework/dts.py            | 174 +++++++++++++++++++++++++++++---
 dts/framework/stats_reporter.py |  70 +++++++++++++
 2 files changed, 232 insertions(+), 12 deletions(-)
 create mode 100644 dts/framework/stats_reporter.py

diff --git a/dts/framework/dts.py b/dts/framework/dts.py
index 1938ea6af8..39e07d9eec 100644
--- a/dts/framework/dts.py
+++ b/dts/framework/dts.py
@@ -4,38 +4,179 @@
 # Copyright(c) 2022 University of New Hampshire
 #
 
+import os
 import sys
+import traceback
 from typing import Iterable, Optional
 
 import framework.logger as logger
 
 from .config import CONFIGURATION
+from .exception import VerifyFailure
 from .logger import getLogger
 from .node import Node
-from .settings import SETTINGS
-from .utils import check_dts_python_version
+from .settings import SETTINGS, DTSRuntimeError, DTSRuntimeErrors
+from .stats_reporter import StatsReporter
+from .sut_node import SutNode
+from .test_case import TestCase
+from .test_result import Result
+from .tg_node import TrafficGeneratorNode
+from .utils import check_dts_python_version, get_subclasses
 
+requested_tests: Optional[list[str]] = None
+result: Optional[Result] = None
+stats_report: Optional[StatsReporter] = None
 log_handler: Optional[logger.DTSLOG] = None
 
 
+def dts_nodes_init():
+    """
+    Create dts SUT/TG instance and initialize them.
+    """
+    sut_nodes = []
+    tg_node = None
+    for node_config in CONFIGURATION.nodes:
+        if hasattr(node_config, 'memory_channels'):
+            sut_nodes.append(SutNode(node_config))
+        else:
+            tg_node = TrafficGeneratorNode(node_config)
+            tg_node.set_re_run(SETTINGS.re_run if SETTINGS.re_run > 0 else 0)
+
+    return sut_nodes, tg_node
+
+
+def dts_run_prerequisites(nodes):
+    """
+    Run dts prerequisites function.
+    """
+    # TODO nodes config contains both sut and tg nodes
+    try:
+        for node in nodes:
+            node.prerequisites()
+    except Exception as ex:
+        log_handler.error("NODE PREREQ EXCEPTION " + traceback.format_exc())
+        result.add_failed_sut(node, str(ex))
+        if isinstance(node, TrafficGeneratorNode):
+            DTSRuntimeError = DTSRuntimeErrors.TG_SETUP_ERR
+        else:
+            DTSRuntimeError = DTSRuntimeErrors.SUT_SETUP_ERR
+        return False
+    return True
+
+
+def dts_run_target(sut_nodes, tg_node, targets, test_suites):
+    """
+    Run each target in execution targets.
+    """
+    for target in targets:
+        target = str(target)
+        log_handler.info("\nTARGET " + target)
+        result.target = target
+
+        try:
+            for sut_node in sut_nodes:
+                sut_node.set_target(target)
+        except AssertionError as ex:
+            DTSRuntimeError = DTSRuntimeErrors.DPDK_BUILD_ERR
+            log_handler.error(" TARGET ERROR: " + str(ex))
+            result.add_failed_target(result.sut, target, str(ex))
+            continue
+        except Exception as ex:
+            DTSRuntimeError = DTSRuntimeErrors.GENERIC_ERR
+            log_handler.error(" !!! DEBUG IT: " + traceback.format_exc())
+            result.add_failed_target(result.sut, target, str(ex))
+            continue
+
+        dts_run_suite(sut_nodes, tg_node, test_suites, target)
+
+
+def dts_run_suite(sut_nodes, tg_node, test_suites, target):
+    """
+    Run each suite in test suite list.
+    """
+    for suite_name in test_suites:
+        try:
+            # check whether config the test cases
+            append_requested_case_list = None
+            if ":" in suite_name:
+                case_list = suite_name[suite_name.find(":") + 1 :]
+                append_requested_case_list = case_list.split("\\")
+                suite_name = suite_name[: suite_name.find(":")]
+            result.test_suite = suite_name
+            _suite_full_name = "TestSuite_" + suite_name
+            suite_module = __import__(
+                "tests." + _suite_full_name, fromlist=[_suite_full_name]
+            )
+            for test_classname, test_class in get_subclasses(suite_module, TestCase):
+
+                suite_obj = test_class(sut_nodes, tg_node, target, suite_name)
+                suite_obj.init_log()
+                suite_obj.set_requested_cases(requested_tests)
+                suite_obj.set_requested_cases(append_requested_case_list)
+
+                log_handler.info("\nTEST SUITE : " + test_classname)
+
+                if suite_obj.execute_setup_all():
+                    suite_obj.execute_test_cases()
+
+                # save suite cases result
+                result.copy_suite(suite_obj.get_result())
+
+                log_handler.info("\nTEST SUITE ENDED: " + test_classname)
+        except VerifyFailure:
+            DTSRuntimeError = DTSRuntimeErrors.SUITE_EXECUTE_ERR
+            log_handler.error(" !!! DEBUG IT: " + traceback.format_exc())
+        except KeyboardInterrupt:
+            # stop/save result/skip execution
+            log_handler.error(" !!! STOPPING DTS")
+            break
+        except Exception as e:
+            DTSRuntimeError = DTSRuntimeErrors.GENERIC_ERR
+            log_handler.error(str(e))
+        finally:
+            try:
+                suite_obj.execute_tear_downall()
+            except Exception as e:
+                DTSRuntimeError = DTSRuntimeErrors.GENERIC_ERR
+                log_handler.error(str(e))
+            try:
+                stats_report.save(result)
+            except Exception as e:
+                DTSRuntimeError = DTSRuntimeErrors.GENERIC_ERR
+                log_handler.error(str(e))
+
+
 def run_all() -> None:
     """
     Main process of DTS, it will run all test suites in the config file.
     """
 
     global log_handler
+    global result
+    global stats_report
+    global requested_tests
 
     # check the python version of the server that run dts
     check_dts_python_version()
 
+    # prepare the output folder
+    if not os.path.exists(SETTINGS.output_dir):
+        os.mkdir(SETTINGS.output_dir)
+
     # init log_handler handler
     if SETTINGS.verbose is True:
         logger.set_verbose()
 
     log_handler = getLogger("dts")
 
-    nodes = {}
-    # This try/finally block means "Run the try block, if there is an exception,
+    # run designated test cases
+    requested_tests = SETTINGS.test_cases
+
+    # report objects
+    stats_report = StatsReporter(SETTINGS.output_dir + "/statistics.txt")
+    result = Result()
+
+    # This try/finally block means "Run the try block and if there is an exception,
     # run the finally block before passing it upward. If there is not an exception,
     # run the finally block after the try block is finished." This helps avoid the
     # problem of python's interpreter exit context, which essentially prevents you
@@ -45,26 +186,35 @@ def run_all() -> None:
     # An except block SHOULD NOT be added to this. A failure at this level should
     # deliver a full stack trace for debugging, since the only place that exceptions
     # should be caught and handled is in the testing code.
+    nodes = []
     try:
         # for all Execution sections
         for execution in CONFIGURATION.executions:
-            sut_config = execution.system_under_test
-            if sut_config.name not in nodes:
-                nodes[sut_config.name] = Node(sut_config)
+            sut_nodes, tg_node = dts_nodes_init()
+            nodes.extend(sut_nodes)
+            nodes.append(tg_node)
+
+            # Run SUT prerequisites
+            if dts_run_prerequisites(nodes) is False:
+                continue
+            result.dpdk_version = sut_nodes[0].dpdk_version
+            dts_run_target(
+                sut_nodes, tg_node, execution.target_descriptions, execution.test_suites
+            )
 
     finally:
-        quit_execution(nodes.values())
+        quit_execution(nodes)
 
 
-def quit_execution(sut_nodes: Iterable[Node]) -> None:
+def quit_execution(nodes: Iterable[Node]) -> None:
     """
     Close session to SUT and TG before quit.
     Return exit status when failure occurred.
     """
-    for sut_node in sut_nodes:
+    for node in nodes:
         # close all session
-        sut_node.node_exit()
+        node.node_exit()
 
     if log_handler is not None:
         log_handler.info("DTS ended")
-    sys.exit(0)
+    sys.exit(DTSRuntimeError)
diff --git a/dts/framework/stats_reporter.py b/dts/framework/stats_reporter.py
new file mode 100644
index 0000000000..a8d589bc7b
--- /dev/null
+++ b/dts/framework/stats_reporter.py
@@ -0,0 +1,70 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2010-2014 Intel Corporation
+# Copyright(c) 2022 PANTHEON.tech s.r.o.
+#
+
+"""
+Simple text file statistics generator
+"""
+
+
+class StatsReporter(object):
+    """
+    Generates a small statistics file containing the number of passing,
+    failing and blocked tests. It makes use of a Result instance as input.
+    """
+
+    def __init__(self, filename):
+        self.filename = filename
+
+    def __add_stat(self, test_result):
+        if test_result is not None:
+            if test_result[0] == "PASSED":
+                self.passed += 1
+            if test_result[0] == "FAILED":
+                self.failed += 1
+            if test_result[0] == "BLOCKED":
+                self.blocked += 1
+            self.total += 1
+
+    def __count_stats(self):
+        for sut in self.result.all_suts():
+            for target in self.result.all_targets(sut):
+                for suite in self.result.all_test_suites(sut, target):
+                    for case in self.result.all_test_cases(sut, target, suite):
+                        test_result = self.result.result_for(sut, target, suite, case)
+                        if len(test_result):
+                            self.__add_stat(test_result)
+
+    def __write_stats(self):
+        sut_nodes = self.result.all_suts()
+        if len(sut_nodes) == 1:
+            self.stats_file.write(
+                "dpdk_version = {}\n".format(
+                    self.result.current_dpdk_version(sut_nodes[0])
+                )
+            )
+        else:
+            for sut in sut_nodes:
+                dpdk_version = self.result.current_dpdk_version(sut)
+                self.stats_file.write(
+                    "{}.dpdk_version = {}\n".format(sut, dpdk_version)
+                )
+        self.__count_stats()
+        self.stats_file.write("Passed     = %d\n" % self.passed)
+        self.stats_file.write("Failed     = %d\n" % self.failed)
+        self.stats_file.write("Blocked    = %d\n" % self.blocked)
+        rate = 0
+        if self.total > 0:
+            rate = self.passed * 100.0 / self.total
+        self.stats_file.write("Pass rate  = %.1f\n" % rate)
+
+    def save(self, result):
+        self.passed = 0
+        self.failed = 0
+        self.blocked = 0
+        self.total = 0
+        self.stats_file = open(self.filename, "w+")
+        self.result = result
+        self.__write_stats()
+        self.stats_file.close()
-- 
2.30.2