[RFC PATCH v2 06/10] dts: add test results module

["Juraj Linkeš" <juraj.linkes@pantheon.tech>]

The module keeps track of test case results along with miscellaneous
information, such as on which SUT's did a failure occur and during the
testing of which build target.

Signed-off-by: Juraj Linkeš <juraj.linkes@pantheon.tech>
---
 dts/framework/dts.py         |   5 +
 dts/framework/test_result.py | 217 +++++++++++++++++++++++++++++++++++
 2 files changed, 222 insertions(+)
 create mode 100644 dts/framework/test_result.py

diff --git a/dts/framework/dts.py b/dts/framework/dts.py
index 262c392d8e..d606f8de2e 100644
--- a/dts/framework/dts.py
+++ b/dts/framework/dts.py
@@ -14,9 +14,11 @@
 from .exception import DTSError, ReturnCode
 from .logger import DTSLOG, getLogger
 from .settings import SETTINGS
+from .test_result import Result
 from .utils import check_dts_python_version
 
 dts_logger: DTSLOG = getLogger("dts")
+result: Result = Result()
 
 
 def run_all() -> None:
@@ -26,6 +28,7 @@ def run_all() -> None:
     """
     return_code = ReturnCode.NO_ERR
     global dts_logger
+    global result
 
     # check the python version of the server that run dts
     check_dts_python_version()
@@ -45,6 +48,7 @@ def run_all() -> None:
         # for all Execution sections
         for execution in CONFIGURATION.executions:
             sut_node = init_nodes(execution, nodes)
+            result.sut = sut_node
             run_execution(sut_node, execution)
 
     except DTSError as e:
@@ -104,6 +108,7 @@ def run_build_target(
     Run the given build target.
     """
     dts_logger.info(f"Running target '{build_target.name}'.")
+    result.target = build_target
     try:
         sut_node.setup_build_target(build_target)
         run_suite(sut_node, build_target, execution)
diff --git a/dts/framework/test_result.py b/dts/framework/test_result.py
new file mode 100644
index 0000000000..a12517b9bc
--- /dev/null
+++ b/dts/framework/test_result.py
@@ -0,0 +1,217 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2010-2014 Intel Corporation
+# Copyright(c) 2022 PANTHEON.tech s.r.o.
+
+"""
+Generic result container and reporters
+"""
+
+
+class Result(object):
+    """
+    Generic result container. Useful to store/retrieve results during
+    a DTF execution.
+
+    It manages and hide an internal complex structure like the one shown below.
+    This is presented to the user with a property based interface.
+
+    internals = [
+        'sut1', [
+            'kdriver',
+            'firmware',
+            'pkg',
+            'driver',
+            'dpdk_version',
+            'target1', 'nic1', [
+                'suite1', [
+                    'case1', ['PASSED', ''],
+                    'case2', ['PASSED', ''],
+                ],
+            ],
+            'target2', 'nic1', [
+                'suite2', [
+                    'case3', ['PASSED', ''],
+                    'case4', ['FAILED', 'message'],
+                ],
+                'suite3', [
+                    'case5', ['BLOCKED', 'message'],
+                ],
+            ]
+        ]
+    ]
+
+    """
+
+    def __init__(self):
+        self.__sut = 0
+        self.__target = 0
+        self.__test_suite = 0
+        self.__test_case = 0
+        self.__test_result = None
+        self.__message = None
+        self.__internals = []
+        self.__failed_suts = {}
+        self.__failed_targets = {}
+
+    def __set_sut(self, sut):
+        if sut not in self.__internals:
+            self.__internals.append(sut)
+            self.__internals.append([])
+        self.__sut = self.__internals.index(sut)
+
+    def __get_sut(self):
+        return self.__internals[self.__sut]
+
+    def current_dpdk_version(self, sut):
+        """
+        Returns the dpdk version for a given SUT
+        """
+        try:
+            sut_idx = self.__internals.index(sut)
+            return self.__internals[sut_idx + 1][4]
+        except:
+            return ""
+
+    def __set_dpdk_version(self, dpdk_version):
+        if dpdk_version not in self.internals[self.__sut + 1]:
+            dpdk_current = self.__get_dpdk_version()
+            if dpdk_current:
+                if dpdk_version not in dpdk_current:
+                    self.internals[self.__sut + 1][4] = (
+                        dpdk_current + "/" + dpdk_version
+                    )
+            else:
+                self.internals[self.__sut + 1].append(dpdk_version)
+
+    def __get_dpdk_version(self):
+        try:
+            return self.internals[self.__sut + 1][4]
+        except:
+            return ""
+
+    def __current_targets(self):
+        return self.internals[self.__sut + 1]
+
+    def __set_target(self, target):
+        targets = self.__current_targets()
+        if target not in targets:
+            targets.append(target)
+            targets.append("_nic_")
+            targets.append([])
+        self.__target = targets.index(target)
+
+    def __get_target(self):
+        return self.__current_targets()[self.__target]
+
+    def __current_suites(self):
+        return self.__current_targets()[self.__target + 2]
+
+    def __set_test_suite(self, test_suite):
+        suites = self.__current_suites()
+        if test_suite not in suites:
+            suites.append(test_suite)
+            suites.append([])
+        self.__test_suite = suites.index(test_suite)
+
+    def __get_test_suite(self):
+        return self.__current_suites()[self.__test_suite]
+
+    def __current_cases(self):
+        return self.__current_suites()[self.__test_suite + 1]
+
+    def __set_test_case(self, test_case):
+        cases = self.__current_cases()
+        cases.append(test_case)
+        cases.append([])
+        self.__test_case = cases.index(test_case)
+
+    def __get_test_case(self):
+        return self.__current_cases()[self.__test_case]
+
+    def __get_internals(self):
+        return self.__internals
+
+    def __current_result(self):
+        return self.__current_cases()[self.__test_case + 1]
+
+    def __set_test_case_result(self, result, message):
+        test_case = self.__current_result()
+        test_case.append(result)
+        test_case.append(message)
+        self.__test_result = result
+        self.__message = message
+
+    def copy_suite(self, suite_result):
+        self.__current_suites()[self.__test_suite + 1] = suite_result.__current_cases()
+
+    def test_case_passed(self):
+        """
+        Set last test case added as PASSED
+        """
+        self.__set_test_case_result(result="PASSED", message="")
+
+    def test_case_failed(self, message):
+        """
+        Set last test case added as FAILED
+        """
+        self.__set_test_case_result(result="FAILED", message=message)
+
+    def test_case_blocked(self, message):
+        """
+        Set last test case added as BLOCKED
+        """
+        self.__set_test_case_result(result="BLOCKED", message=message)
+
+    def all_suts(self):
+        """
+        Returns all the SUTs it's aware of.
+        """
+        return self.__internals[::2]
+
+    def all_targets(self, sut):
+        """
+        Returns the targets for a given SUT
+        """
+        try:
+            sut_idx = self.__internals.index(sut)
+        except:
+            return None
+        return self.__internals[sut_idx + 1][5::3]
+
+    def add_failed_sut(self, sut, msg):
+        """
+        Sets the given SUT as failing due to msg
+        """
+        self.__failed_suts[sut] = msg
+
+    def remove_failed_sut(self, sut):
+        """
+        Remove the given SUT from failed SUTs collection
+        """
+        if sut in self.__failed_suts:
+            self.__failed_suts.pop(sut)
+
+    def add_failed_target(self, sut, target, msg):
+        """
+        Sets the given SUT, target as failing due to msg
+        """
+        self.__failed_targets[sut + target] = msg
+
+    def remove_failed_target(self, sut, target):
+        """
+        Remove the given SUT, target from failed targets collection
+        """
+        key_word = sut + target
+        if key_word in self.__failed_targets:
+            self.__failed_targets.pop(key_word)
+
+    """
+    Attributes defined as properties to hide the implementation from the
+    presented interface.
+    """
+    sut = property(__get_sut, __set_sut)
+    dpdk_version = property(__get_dpdk_version, __set_dpdk_version)
+    target = property(__get_target, __set_target)
+    test_suite = property(__get_test_suite, __set_test_suite)
+    test_case = property(__get_test_case, __set_test_case)
+    internals = property(__get_internals)
-- 
2.30.2