DPDK patches and discussions
 help / color / mirror / Atom feed
From: "Juraj Linkeš" <juraj.linkes@pantheon.tech>
To: thomas@monjalon.net, david.marchand@redhat.com,
	ronan.randles@intel.com, Honnappa.Nagarahalli@arm.com,
	ohilyard@iol.unh.edu, lijuan.tu@intel.com
Cc: dev@dpdk.org, "Juraj Linkeš" <juraj.linkes@pantheon.tech>
Subject: [RFC PATCH v1 08/10] dts: add test runner and statistics collector
Date: Wed, 24 Aug 2022 16:24:52 +0000	[thread overview]
Message-ID: <20220824162454.394285-9-juraj.linkes@pantheon.tech> (raw)
In-Reply-To: <20220824162454.394285-1-juraj.linkes@pantheon.tech>

Add functions responsible for initializing testbed setup, testcase
discovery and execution. The stats collector gathers the pass/fail
results and provides a short report.

Signed-off-by: Juraj Linkeš <juraj.linkes@pantheon.tech>
---
 dts/framework/dts.py            | 174 +++++++++++++++++++++++++++++---
 dts/framework/stats_reporter.py |  70 +++++++++++++
 2 files changed, 232 insertions(+), 12 deletions(-)
 create mode 100644 dts/framework/stats_reporter.py

diff --git a/dts/framework/dts.py b/dts/framework/dts.py
index 1938ea6af8..39e07d9eec 100644
--- a/dts/framework/dts.py
+++ b/dts/framework/dts.py
@@ -4,38 +4,179 @@
 # Copyright(c) 2022 University of New Hampshire
 #
 
+import os
 import sys
+import traceback
 from typing import Iterable, Optional
 
 import framework.logger as logger
 
 from .config import CONFIGURATION
+from .exception import VerifyFailure
 from .logger import getLogger
 from .node import Node
-from .settings import SETTINGS
-from .utils import check_dts_python_version
+from .settings import SETTINGS, DTSRuntimeError, DTSRuntimeErrors
+from .stats_reporter import StatsReporter
+from .sut_node import SutNode
+from .test_case import TestCase
+from .test_result import Result
+from .tg_node import TrafficGeneratorNode
+from .utils import check_dts_python_version, get_subclasses
 
+requested_tests: Optional[list[str]] = None
+result: Optional[Result] = None
+stats_report: Optional[StatsReporter] = None
 log_handler: Optional[logger.DTSLOG] = None
 
 
+def dts_nodes_init():
+    """
+    Create dts SUT/TG instance and initialize them.
+    """
+    sut_nodes = []
+    tg_node = None
+    for node_config in CONFIGURATION.nodes:
+        if hasattr(node_config, 'memory_channels'):
+            sut_nodes.append(SutNode(node_config))
+        else:
+            tg_node = TrafficGeneratorNode(node_config)
+            tg_node.set_re_run(SETTINGS.re_run if SETTINGS.re_run > 0 else 0)
+
+    return sut_nodes, tg_node
+
+
+def dts_run_prerequisites(nodes):
+    """
+    Run dts prerequisites function.
+    """
+    # TODO nodes config contains both sut and tg nodes
+    try:
+        for node in nodes:
+            node.prerequisites()
+    except Exception as ex:
+        log_handler.error("NODE PREREQ EXCEPTION " + traceback.format_exc())
+        result.add_failed_sut(node, str(ex))
+        if isinstance(node, TrafficGeneratorNode):
+            DTSRuntimeError = DTSRuntimeErrors.TG_SETUP_ERR
+        else:
+            DTSRuntimeError = DTSRuntimeErrors.SUT_SETUP_ERR
+        return False
+    return True
+
+
+def dts_run_target(sut_nodes, tg_node, targets, test_suites):
+    """
+    Run each target in execution targets.
+    """
+    for target in targets:
+        target = str(target)
+        log_handler.info("\nTARGET " + target)
+        result.target = target
+
+        try:
+            for sut_node in sut_nodes:
+                sut_node.set_target(target)
+        except AssertionError as ex:
+            DTSRuntimeError = DTSRuntimeErrors.DPDK_BUILD_ERR
+            log_handler.error(" TARGET ERROR: " + str(ex))
+            result.add_failed_target(result.sut, target, str(ex))
+            continue
+        except Exception as ex:
+            DTSRuntimeError = DTSRuntimeErrors.GENERIC_ERR
+            log_handler.error(" !!! DEBUG IT: " + traceback.format_exc())
+            result.add_failed_target(result.sut, target, str(ex))
+            continue
+
+        dts_run_suite(sut_nodes, tg_node, test_suites, target)
+
+
+def dts_run_suite(sut_nodes, tg_node, test_suites, target):
+    """
+    Run each suite in test suite list.
+    """
+    for suite_name in test_suites:
+        try:
+            # check whether config the test cases
+            append_requested_case_list = None
+            if ":" in suite_name:
+                case_list = suite_name[suite_name.find(":") + 1 :]
+                append_requested_case_list = case_list.split("\\")
+                suite_name = suite_name[: suite_name.find(":")]
+            result.test_suite = suite_name
+            _suite_full_name = "TestSuite_" + suite_name
+            suite_module = __import__(
+                "tests." + _suite_full_name, fromlist=[_suite_full_name]
+            )
+            for test_classname, test_class in get_subclasses(suite_module, TestCase):
+
+                suite_obj = test_class(sut_nodes, tg_node, target, suite_name)
+                suite_obj.init_log()
+                suite_obj.set_requested_cases(requested_tests)
+                suite_obj.set_requested_cases(append_requested_case_list)
+
+                log_handler.info("\nTEST SUITE : " + test_classname)
+
+                if suite_obj.execute_setup_all():
+                    suite_obj.execute_test_cases()
+
+                # save suite cases result
+                result.copy_suite(suite_obj.get_result())
+
+                log_handler.info("\nTEST SUITE ENDED: " + test_classname)
+        except VerifyFailure:
+            DTSRuntimeError = DTSRuntimeErrors.SUITE_EXECUTE_ERR
+            log_handler.error(" !!! DEBUG IT: " + traceback.format_exc())
+        except KeyboardInterrupt:
+            # stop/save result/skip execution
+            log_handler.error(" !!! STOPPING DTS")
+            break
+        except Exception as e:
+            DTSRuntimeError = DTSRuntimeErrors.GENERIC_ERR
+            log_handler.error(str(e))
+        finally:
+            try:
+                suite_obj.execute_tear_downall()
+            except Exception as e:
+                DTSRuntimeError = DTSRuntimeErrors.GENERIC_ERR
+                log_handler.error(str(e))
+            try:
+                stats_report.save(result)
+            except Exception as e:
+                DTSRuntimeError = DTSRuntimeErrors.GENERIC_ERR
+                log_handler.error(str(e))
+
+
 def run_all() -> None:
     """
     Main process of DTS, it will run all test suites in the config file.
     """
 
     global log_handler
+    global result
+    global stats_report
+    global requested_tests
 
     # check the python version of the server that run dts
     check_dts_python_version()
 
+    # prepare the output folder
+    if not os.path.exists(SETTINGS.output_dir):
+        os.mkdir(SETTINGS.output_dir)
+
     # init log_handler handler
     if SETTINGS.verbose is True:
         logger.set_verbose()
 
     log_handler = getLogger("dts")
 
-    nodes = {}
-    # This try/finally block means "Run the try block, if there is an exception,
+    # run designated test cases
+    requested_tests = SETTINGS.test_cases
+
+    # report objects
+    stats_report = StatsReporter(SETTINGS.output_dir + "/statistics.txt")
+    result = Result()
+
+    # This try/finally block means "Run the try block and if there is an exception,
     # run the finally block before passing it upward. If there is not an exception,
     # run the finally block after the try block is finished." This helps avoid the
     # problem of python's interpreter exit context, which essentially prevents you
@@ -45,26 +186,35 @@ def run_all() -> None:
     # An except block SHOULD NOT be added to this. A failure at this level should
     # deliver a full stack trace for debugging, since the only place that exceptions
     # should be caught and handled is in the testing code.
+    nodes = []
     try:
         # for all Execution sections
         for execution in CONFIGURATION.executions:
-            sut_config = execution.system_under_test
-            if sut_config.name not in nodes:
-                nodes[sut_config.name] = Node(sut_config)
+            sut_nodes, tg_node = dts_nodes_init()
+            nodes.extend(sut_nodes)
+            nodes.append(tg_node)
+
+            # Run SUT prerequisites
+            if dts_run_prerequisites(nodes) is False:
+                continue
+            result.dpdk_version = sut_nodes[0].dpdk_version
+            dts_run_target(
+                sut_nodes, tg_node, execution.target_descriptions, execution.test_suites
+            )
 
     finally:
-        quit_execution(nodes.values())
+        quit_execution(nodes)
 
 
-def quit_execution(sut_nodes: Iterable[Node]) -> None:
+def quit_execution(nodes: Iterable[Node]) -> None:
     """
     Close session to SUT and TG before quit.
     Return exit status when failure occurred.
     """
-    for sut_node in sut_nodes:
+    for node in nodes:
         # close all session
-        sut_node.node_exit()
+        node.node_exit()
 
     if log_handler is not None:
         log_handler.info("DTS ended")
-    sys.exit(0)
+    sys.exit(DTSRuntimeError)
diff --git a/dts/framework/stats_reporter.py b/dts/framework/stats_reporter.py
new file mode 100644
index 0000000000..a8d589bc7b
--- /dev/null
+++ b/dts/framework/stats_reporter.py
@@ -0,0 +1,70 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2010-2014 Intel Corporation
+# Copyright(c) 2022 PANTHEON.tech s.r.o.
+#
+
+"""
+Simple text file statistics generator
+"""
+
+
+class StatsReporter(object):
+    """
+    Generates a small statistics file containing the number of passing,
+    failing and blocked tests. It makes use of a Result instance as input.
+    """
+
+    def __init__(self, filename):
+        self.filename = filename
+
+    def __add_stat(self, test_result):
+        if test_result is not None:
+            if test_result[0] == "PASSED":
+                self.passed += 1
+            if test_result[0] == "FAILED":
+                self.failed += 1
+            if test_result[0] == "BLOCKED":
+                self.blocked += 1
+            self.total += 1
+
+    def __count_stats(self):
+        for sut in self.result.all_suts():
+            for target in self.result.all_targets(sut):
+                for suite in self.result.all_test_suites(sut, target):
+                    for case in self.result.all_test_cases(sut, target, suite):
+                        test_result = self.result.result_for(sut, target, suite, case)
+                        if len(test_result):
+                            self.__add_stat(test_result)
+
+    def __write_stats(self):
+        sut_nodes = self.result.all_suts()
+        if len(sut_nodes) == 1:
+            self.stats_file.write(
+                "dpdk_version = {}\n".format(
+                    self.result.current_dpdk_version(sut_nodes[0])
+                )
+            )
+        else:
+            for sut in sut_nodes:
+                dpdk_version = self.result.current_dpdk_version(sut)
+                self.stats_file.write(
+                    "{}.dpdk_version = {}\n".format(sut, dpdk_version)
+                )
+        self.__count_stats()
+        self.stats_file.write("Passed     = %d\n" % self.passed)
+        self.stats_file.write("Failed     = %d\n" % self.failed)
+        self.stats_file.write("Blocked    = %d\n" % self.blocked)
+        rate = 0
+        if self.total > 0:
+            rate = self.passed * 100.0 / self.total
+        self.stats_file.write("Pass rate  = %.1f\n" % rate)
+
+    def save(self, result):
+        self.passed = 0
+        self.failed = 0
+        self.blocked = 0
+        self.total = 0
+        self.stats_file = open(self.filename, "w+")
+        self.result = result
+        self.__write_stats()
+        self.stats_file.close()
-- 
2.30.2


  parent reply	other threads:[~2022-08-24 16:25 UTC|newest]

Thread overview: 97+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2022-08-24 16:24 [RFC PATCH v1 00/10] dts: add hello world testcase Juraj Linkeš
2022-08-24 16:24 ` [RFC PATCH v1 01/10] dts: hello world config options Juraj Linkeš
2022-08-24 16:24 ` [RFC PATCH v1 02/10] dts: hello world cli parameters and env vars Juraj Linkeš
2022-08-24 16:24 ` [RFC PATCH v1 03/10] dts: ssh connection additions for hello world Juraj Linkeš
2022-08-24 16:24 ` [RFC PATCH v1 04/10] dts: add basic node management methods Juraj Linkeš
2022-08-24 16:24 ` [RFC PATCH v1 05/10] dts: add system under test node Juraj Linkeš
2022-08-24 16:24 ` [RFC PATCH v1 06/10] dts: add traffic generator node Juraj Linkeš
2022-08-24 16:24 ` [RFC PATCH v1 07/10] dts: add testcase and basic test results Juraj Linkeš
2022-08-24 16:24 ` Juraj Linkeš [this message]
2022-08-24 16:24 ` [RFC PATCH v1 09/10] dts: add hello world testplan Juraj Linkeš
2022-08-24 16:24 ` [RFC PATCH v1 10/10] dts: add hello world testsuite Juraj Linkeš
2022-11-14 16:54 ` [RFC PATCH v2 00/10] dts: add hello world testcase Juraj Linkeš
2022-11-14 16:54   ` [RFC PATCH v2 01/10] dts: add node and os abstractions Juraj Linkeš
2022-11-14 16:54   ` [RFC PATCH v2 02/10] dts: add ssh command verification Juraj Linkeš
2022-11-14 16:54   ` [RFC PATCH v2 03/10] dts: add dpdk build on sut Juraj Linkeš
2022-11-16 13:15     ` Owen Hilyard
     [not found]       ` <30ad4f7d087d4932845b6ca13934b1d2@pantheon.tech>
     [not found]         ` <CAHx6DYDOFMuEm4xc65OTrtUmGBtk8Z6UtSgS2grnR_RBY5HcjQ@mail.gmail.com>
2022-11-23 12:37           ` Juraj Linkeš
2022-11-14 16:54   ` [RFC PATCH v2 04/10] dts: add dpdk execution handling Juraj Linkeš
2022-11-16 13:28     ` Owen Hilyard
     [not found]       ` <df13ee41efb64e7bb37791f21ae5bac1@pantheon.tech>
     [not found]         ` <CAHx6DYCEYxZ0Osm6fKhp3Jx8n7s=r7qVh8R41c6nCan8Or-dpA@mail.gmail.com>
2022-11-23 13:03           ` Juraj Linkeš
2022-11-28 13:05             ` Owen Hilyard
2022-11-14 16:54   ` [RFC PATCH v2 05/10] dts: add node memory setup Juraj Linkeš
2022-11-16 13:47     ` Owen Hilyard
2022-11-23 13:58       ` Juraj Linkeš
2022-11-14 16:54   ` [RFC PATCH v2 06/10] dts: add test results module Juraj Linkeš
2022-11-14 16:54   ` [RFC PATCH v2 07/10] dts: add simple stats report Juraj Linkeš
2022-11-16 13:57     ` Owen Hilyard
2022-11-14 16:54   ` [RFC PATCH v2 08/10] dts: add testsuite class Juraj Linkeš
2022-11-16 15:15     ` Owen Hilyard
2022-11-14 16:54   ` [RFC PATCH v2 09/10] dts: add hello world testplan Juraj Linkeš
2022-11-14 16:54   ` [RFC PATCH v2 10/10] dts: add hello world testsuite Juraj Linkeš
2023-01-17 15:48   ` [PATCH v3 00/10] dts: add hello world testcase Juraj Linkeš
2023-01-17 15:48     ` [PATCH v3 01/10] dts: add node and os abstractions Juraj Linkeš
2023-01-17 15:48     ` [PATCH v3 02/10] dts: add ssh command verification Juraj Linkeš
2023-01-17 15:48     ` [PATCH v3 03/10] dts: add dpdk build on sut Juraj Linkeš
2023-01-17 15:49     ` [PATCH v3 04/10] dts: add dpdk execution handling Juraj Linkeš
2023-01-17 15:49     ` [PATCH v3 05/10] dts: add node memory setup Juraj Linkeš
2023-01-17 15:49     ` [PATCH v3 06/10] dts: add test suite module Juraj Linkeš
2023-01-17 15:49     ` [PATCH v3 07/10] dts: add hello world testplan Juraj Linkeš
2023-01-17 15:49     ` [PATCH v3 08/10] dts: add hello world testsuite Juraj Linkeš
2023-01-17 15:49     ` [PATCH v3 09/10] dts: add test suite config and runner Juraj Linkeš
2023-01-17 15:49     ` [PATCH v3 10/10] dts: add test results module Juraj Linkeš
2023-01-19 16:16     ` [PATCH v3 00/10] dts: add hello world testcase Owen Hilyard
2023-02-09 16:47       ` Patrick Robb
2023-02-13 15:28     ` [PATCH v4 " Juraj Linkeš
2023-02-13 15:28       ` [PATCH v4 01/10] dts: add node and os abstractions Juraj Linkeš
2023-02-17 17:44         ` Bruce Richardson
2023-02-20 13:24           ` Juraj Linkeš
2023-02-13 15:28       ` [PATCH v4 02/10] dts: add ssh command verification Juraj Linkeš
2023-02-13 15:28       ` [PATCH v4 03/10] dts: add dpdk build on sut Juraj Linkeš
2023-02-22 16:44         ` Bruce Richardson
2023-02-13 15:28       ` [PATCH v4 04/10] dts: add dpdk execution handling Juraj Linkeš
2023-02-13 15:28       ` [PATCH v4 05/10] dts: add node memory setup Juraj Linkeš
2023-02-13 15:28       ` [PATCH v4 06/10] dts: add test suite module Juraj Linkeš
2023-02-13 15:28       ` [PATCH v4 07/10] dts: add hello world testsuite Juraj Linkeš
2023-02-13 15:28       ` [PATCH v4 08/10] dts: add test suite config and runner Juraj Linkeš
2023-02-13 15:28       ` [PATCH v4 09/10] dts: add test results module Juraj Linkeš
2023-02-13 15:28       ` [PATCH v4 10/10] doc: update DTS setup and test suite cookbook Juraj Linkeš
2023-02-17 17:26       ` [PATCH v4 00/10] dts: add hello world testcase Bruce Richardson
2023-02-20 10:13         ` Juraj Linkeš
2023-02-20 11:56           ` Bruce Richardson
2023-02-22 16:39           ` Bruce Richardson
2023-02-23  8:27             ` Juraj Linkeš
2023-02-23  9:17               ` Bruce Richardson
2023-02-23 15:28       ` [PATCH v5 " Juraj Linkeš
2023-02-23 15:28         ` [PATCH v5 01/10] dts: add node and os abstractions Juraj Linkeš
2023-02-23 15:28         ` [PATCH v5 02/10] dts: add ssh command verification Juraj Linkeš
2023-02-23 15:28         ` [PATCH v5 03/10] dts: add dpdk build on sut Juraj Linkeš
2023-02-23 15:28         ` [PATCH v5 04/10] dts: add dpdk execution handling Juraj Linkeš
2023-02-23 15:28         ` [PATCH v5 05/10] dts: add node memory setup Juraj Linkeš
2023-02-23 15:28         ` [PATCH v5 06/10] dts: add test suite module Juraj Linkeš
2023-02-23 15:28         ` [PATCH v5 07/10] dts: add hello world testsuite Juraj Linkeš
2023-02-23 15:28         ` [PATCH v5 08/10] dts: add test suite config and runner Juraj Linkeš
2023-02-23 15:28         ` [PATCH v5 09/10] dts: add test results module Juraj Linkeš
2023-02-23 15:28         ` [PATCH v5 10/10] doc: update DTS setup and test suite cookbook Juraj Linkeš
2023-03-03  8:31           ` Huang, ChenyuX
2023-02-23 16:13         ` [PATCH v5 00/10] dts: add hello world testcase Bruce Richardson
2023-02-26 19:11         ` Wathsala Wathawana Vithanage
2023-02-27  8:28           ` Juraj Linkeš
2023-02-28 15:27             ` Wathsala Wathawana Vithanage
2023-03-01  8:35               ` Juraj Linkeš
2023-03-03 10:24         ` [PATCH v6 00/10] dts: add hello world test case Juraj Linkeš
2023-03-03 10:24           ` [PATCH v6 01/10] dts: add node and os abstractions Juraj Linkeš
2023-03-03 10:24           ` [PATCH v6 02/10] dts: add ssh command verification Juraj Linkeš
2023-03-03 10:25           ` [PATCH v6 03/10] dts: add dpdk build on sut Juraj Linkeš
2023-03-20  8:30             ` David Marchand
2023-03-20 13:12               ` Juraj Linkeš
2023-03-20 13:22                 ` David Marchand
2023-03-03 10:25           ` [PATCH v6 04/10] dts: add dpdk execution handling Juraj Linkeš
2023-03-03 10:25           ` [PATCH v6 05/10] dts: add node memory setup Juraj Linkeš
2023-03-03 10:25           ` [PATCH v6 06/10] dts: add test suite module Juraj Linkeš
2023-03-03 10:25           ` [PATCH v6 07/10] dts: add hello world testsuite Juraj Linkeš
2023-03-03 10:25           ` [PATCH v6 08/10] dts: add test suite config and runner Juraj Linkeš
2023-03-03 10:25           ` [PATCH v6 09/10] dts: add test results module Juraj Linkeš
2023-03-03 10:25           ` [PATCH v6 10/10] doc: update dts setup and test suite cookbook Juraj Linkeš
2023-03-09 21:47             ` Patrick Robb
2023-03-19 15:26           ` [PATCH v6 00/10] dts: add hello world test case Thomas Monjalon

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20220824162454.394285-9-juraj.linkes@pantheon.tech \
    --to=juraj.linkes@pantheon.tech \
    --cc=Honnappa.Nagarahalli@arm.com \
    --cc=david.marchand@redhat.com \
    --cc=dev@dpdk.org \
    --cc=lijuan.tu@intel.com \
    --cc=ohilyard@iol.unh.edu \
    --cc=ronan.randles@intel.com \
    --cc=thomas@monjalon.net \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).