[RFC PATCH v1 0/6] merge DTS test report files to DPDK

[RFC PATCH v1 0/6] merge DTS test report files to DPDK

[Juraj Linkeš]

The libraries and the output folder related to test reporting.

Juraj Linkeš (6):
  dts: merge DTS framework/excel_reporter.py to DPDK
  dts: merge DTS framework/json_reporter.py to DPDK
  dts: merge DTS framework/rst.py to DPDK
  dts: merge DTS framework/stats_reporter.py to DPDK
  dts: merge DTS framework/test_result.py to DPDK
  dts: merge DTS output/Readme.txt to DPDK

 dts/framework/excel_reporter.py | 280 ++++++++++++++++++
 dts/framework/json_reporter.py  |  86 ++++++
 dts/framework/rst.py            | 168 +++++++++++
 dts/framework/stats_reporter.py |  95 ++++++
 dts/framework/test_result.py    | 494 ++++++++++++++++++++++++++++++++
 dts/output/Readme.txt           |   2 +
 6 files changed, 1125 insertions(+)
 create mode 100644 dts/framework/excel_reporter.py
 create mode 100644 dts/framework/json_reporter.py
 create mode 100644 dts/framework/rst.py
 create mode 100644 dts/framework/stats_reporter.py
 create mode 100644 dts/framework/test_result.py
 create mode 100644 dts/output/Readme.txt

-- 
2.20.1

[RFC PATCH v1 1/6] dts: merge DTS framework/excel_reporter.py to DPDK

[Juraj Linkeš]

---
 dts/framework/excel_reporter.py | 280 ++++++++++++++++++++++++++++++++
 1 file changed, 280 insertions(+)
 create mode 100644 dts/framework/excel_reporter.py

diff --git a/dts/framework/excel_reporter.py b/dts/framework/excel_reporter.py
new file mode 100644
index 0000000000..3a6bdb5db6
--- /dev/null
+++ b/dts/framework/excel_reporter.py
@@ -0,0 +1,280 @@
+# BSD LICENSE
+#
+# Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+#   * Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#   * Redistributions in binary form must reproduce the above copyright
+#     notice, this list of conditions and the following disclaimer in
+#     the documentation and/or other materials provided with the
+#     distribution.
+#   * Neither the name of Intel Corporation nor the names of its
+#     contributors may be used to endorse or promote products derived
+#     from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""
+Excel spreadsheet generator
+
+Example:
+    excel_report = ExcelReporter('../output/test_results.xls')
+    result = Result()
+    result.dut = dutIP
+    result.target = target
+    result.nic = nic
+        result.test_suite = test_suite
+            result.test_case = test_case.__name__
+            result.test_case_passed()
+    excel_report.save(result)
+
+Result:
+    execl will be formatted as
+    DUT             Target                      NIC      Test suite Test case Results
+    10.239.128.117  x86_64-native-linuxapp-gcc  niantic
+                                                         SUITE      CASE      PASSED
+
+
+"""
+import xlwt
+from xlwt.ExcelFormula import Formula
+
+
+class ExcelReporter(object):
+
+    """
+    Make use of a Result object generates an Excel Spreadsheet with those
+    results.
+    It supports saving the same file with incremental results.
+    """
+
+    def __init__(self, filename):
+        self.filename = filename
+        self.xsl_file = None
+        self.result = None
+        self.__styles()
+
+    def __init(self):
+        self.workbook = xlwt.Workbook()
+        self.sheet = self.workbook.add_sheet("Test Results", cell_overwrite_ok=True)
+
+    def __add_header(self):
+        self.sheet.write(0, 0, "DUT", self.header_style)
+        self.sheet.write(0, 1, "DPDK version", self.header_style)
+        self.sheet.write(0, 2, "Target", self.header_style)
+        self.sheet.write(0, 3, "NIC", self.header_style)
+        self.sheet.write(0, 4, "Test suite", self.header_style)
+        self.sheet.write(0, 5, "Test case", self.header_style)
+        self.sheet.write(0, 6, "Results", self.header_style)
+
+        self.sheet.write(0, 8, "Pass", self.header_style)
+        self.sheet.write(0, 9, "Fail", self.header_style)
+        self.sheet.write(0, 10, "Blocked", self.header_style)
+        self.sheet.write(0, 11, "N/A", self.header_style)
+        self.sheet.write(0, 12, "Not Run", self.header_style)
+        self.sheet.write(0, 13, "Total", self.header_style)
+
+        self.sheet.write(1, 8, Formula('COUNTIF(G2:G2000,"PASSED")'))
+        self.sheet.write(
+            1, 9, Formula('COUNTIF(G2:G2000,"FAILED*") + COUNTIF(G2:G2000,"IXA*")')
+        )
+        self.sheet.write(1, 10, Formula('COUNTIF(G2:G2000,"BLOCKED*")'))
+        self.sheet.write(1, 11, Formula('COUNTIF(G2:G2000,"N/A*")'))
+        self.sheet.write(1, 13, Formula("I2+J2+K2+L2+M2"))
+
+        self.sheet.col(0).width = 4000
+        self.sheet.col(1).width = 4500
+        self.sheet.col(2).width = 7500
+        self.sheet.col(3).width = 3000
+        self.sheet.col(4).width = 5000
+        self.sheet.col(5).width = 8000
+        self.sheet.col(6).width = 3000
+        self.sheet.col(7).width = 1000
+        self.sheet.col(8).width = 3000
+        self.sheet.col(9).width = 3000
+        self.sheet.col(10).width = 3000
+        self.sheet.col(11).width = 3000
+        self.sheet.col(12).width = 3000
+        self.sheet.col(13).width = 3000
+
+    def __styles(self):
+        header_pattern = xlwt.Pattern()
+        header_pattern.pattern = xlwt.Pattern.SOLID_PATTERN
+        header_pattern.pattern_fore_colour = xlwt.Style.colour_map["ocean_blue"]
+
+        passed_font = xlwt.Font()
+        passed_font.colour_index = xlwt.Style.colour_map["black"]
+        self.passed_style = xlwt.XFStyle()
+        self.passed_style.font = passed_font
+
+        failed_font = xlwt.Font()
+        failed_font.bold = True
+        failed_font.colour_index = xlwt.Style.colour_map["red"]
+        self.failed_style = xlwt.XFStyle()
+        self.failed_style.font = failed_font
+
+        header_font = xlwt.Font()
+        header_font.bold = True
+        header_font.height = 260
+        header_font.italic = True
+        header_font.colour_index = xlwt.Style.colour_map["white"]
+
+        title_font = xlwt.Font()
+        title_font.bold = True
+        title_font.height = 220
+        title_font.italic = True
+
+        self.header_style = xlwt.XFStyle()
+        self.header_style.font = header_font
+        self.header_style.pattern = header_pattern
+
+        self.title_style = xlwt.XFStyle()
+        self.title_style.font = title_font
+
+    def __get_case_result(self, dut, target, suite, case):
+        case_list = self.result.all_test_cases(dut, target, suite)
+        if case_list.count(case) > 1:
+            tmp_result = []
+            for case_name in case_list:
+                if case == case_name:
+                    test_result = self.result.result_for(dut, target, suite, case)
+                    if "PASSED" in test_result:
+                        return ["PASSED", ""]
+                    else:
+                        tmp_result.append(test_result)
+            return tmp_result[-1]
+        else:
+            return self.result.result_for(dut, target, suite, case)
+
+    def __write_result(self, dut, target, suite, case, test_result):
+        if test_result is not None and len(test_result) > 0:
+            result = test_result[0]
+            if test_result[1] != "":
+                result = "{0} '{1}'".format(result, test_result[1])
+            if test_result[0] == "PASSED":
+                self.sheet.write(self.row, self.col + 1, result)
+            else:
+                self.sheet.write(
+                    self.row,
+                    self.col + 1,
+                    result
+                    if len(result) < 5000
+                    else result[:2000] + "\r\n...\r\n...\r\n...\r\n" + result[-2000:],
+                    self.failed_style,
+                )
+
+    def __write_cases(self, dut, target, suite):
+        for case in set(self.result.all_test_cases(dut, target, suite)):
+            result = self.__get_case_result(dut, target, suite, case)
+            self.col += 1
+            if case[:5] == "test_":
+                self.sheet.write(self.row, self.col, case[5:])
+            else:
+                self.sheet.write(self.row, self.col, case)
+            self.__write_result(dut, target, suite, case, result)
+            self.row += 1
+            self.col -= 1
+
+    def __write_suites(self, dut, target):
+        for suite in self.result.all_test_suites(dut, target):
+            self.row += 1
+            self.col += 1
+            self.sheet.write(self.row, self.col, suite)
+            self.__write_cases(dut, target, suite)
+            self.col -= 1
+
+    def __write_nic(self, dut, target):
+        nic = self.result.current_nic(dut, target)
+        driver = self.result.current_driver(dut)
+        kdriver = self.result.current_kdriver(dut)
+        firmware = self.result.current_firmware_version(dut)
+        pkg = self.result.current_package_version(dut)
+        self.col += 1
+        self.sheet.col(self.col).width = 32 * 256  # 32 characters
+        self.sheet.write(self.row, self.col, nic, self.title_style)
+        self.sheet.write(self.row + 1, self.col, "driver: " + driver)
+        self.sheet.write(self.row + 2, self.col, "kdriver: " + kdriver)
+        self.sheet.write(self.row + 3, self.col, "firmware: " + firmware)
+        if pkg is not None:
+            self.sheet.write(self.row + 4, self.col, "pkg: " + pkg)
+            self.row = self.row + 1
+        self.row = self.row + 3
+        self.__write_suites(dut, target)
+        self.col -= 1
+
+    def __write_failed_target(self, dut, target):
+        msg = "TARGET ERROR '%s'" % self.result.target_failed_msg(dut, target)
+        self.sheet.write(
+            self.row,
+            self.col + 4,
+            msg
+            if len(msg) < 5000
+            else msg[:2000] + "\r\n...\r\n...\r\n...\r\n" + msg[-2000:],
+            self.failed_style,
+        )
+        self.row += 1
+
+    def __write_targets(self, dut):
+        for target in self.result.all_targets(dut):
+            self.col += 1
+            self.sheet.write(self.row, self.col, target, self.title_style)
+            if self.result.is_target_failed(dut, target):
+                self.__write_failed_target(dut, target)
+            else:
+                self.__write_nic(dut, target)
+            self.row += 1
+            self.col -= 1
+
+    def __write_failed_dut(self, dut):
+        msg = "PREREQ FAILED '%s'" % self.result.dut_failed_msg(dut)
+        self.sheet.write(
+            self.row,
+            self.col + 5,
+            msg
+            if len(msg) < 5000
+            else msg[:2000] + "\r\n...\r\n...\r\n...\r\n" + msg[-2000:],
+            self.failed_style,
+        )
+        self.row += 1
+
+    def __parse_result(self):
+        for dut in self.result.all_duts():
+            self.sheet.write(self.row, self.col, dut, self.title_style)
+            if self.result.is_dut_failed(dut):
+                self.__write_failed_dut(dut)
+            else:
+                self.col = self.col + 1
+                self.sheet.write(
+                    self.row,
+                    self.col,
+                    self.result.current_dpdk_version(dut),
+                    self.title_style,
+                )
+                self.__write_targets(dut)
+            self.row += 1
+
+    def save(self, result):
+        self.__init()
+        self.__add_header()
+        self.row = 1
+        self.col = 0
+
+        self.result = result
+        self.__parse_result()
+
+        self.workbook.save(self.filename)
-- 
2.20.1

[RFC PATCH v1 2/6] dts: merge DTS framework/json_reporter.py to DPDK

[Juraj Linkeš]

---
 dts/framework/json_reporter.py | 86 ++++++++++++++++++++++++++++++++++
 1 file changed, 86 insertions(+)
 create mode 100644 dts/framework/json_reporter.py

diff --git a/dts/framework/json_reporter.py b/dts/framework/json_reporter.py
new file mode 100644
index 0000000000..ba12efaf22
--- /dev/null
+++ b/dts/framework/json_reporter.py
@@ -0,0 +1,86 @@
+# BSD LICENSE
+#
+# Copyright(c) 2017 Linaro. All rights reserved.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+#   * Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#   * Redistributions in binary form must reproduce the above copyright
+#     notice, this list of conditions and the following disclaimer in
+#     the documentation and/or other materials provided with the
+#     distribution.
+#   * Neither the name of Intel Corporation nor the names of its
+#     contributors may be used to endorse or promote products derived
+#     from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import json
+
+
+class JSONReporter(object):
+    def __init__(self, filename):
+        self.filename = filename
+
+    def __scan_cases(self, result, dut, target, suite):
+        case_results = {}
+        for case in result.all_test_cases(dut, target, suite):
+            test_result = result.result_for(dut, target, suite, case)
+            case_name = "{}/{}".format(suite, case)
+            case_results[case_name] = test_result
+            if "PASSED" in test_result:
+                case_results[case_name] = "passed"
+            elif "N/A" in test_result:
+                case_results[case_name] = "n/a"
+            elif "FAILED" in test_result:
+                case_results[case_name] = "failed"
+            elif "BLOCKED" in test_result:
+                case_results[case_name] = "blocked"
+        return case_results
+
+    def __scan_target(self, result, dut, target):
+        if result.is_target_failed(dut, target):
+            return "fail"
+        case_results = {}
+        for suite in result.all_test_suites(dut, target):
+            case_results.update(self.__scan_cases(result, dut, target, suite))
+        return case_results
+
+    def __scan_dut(self, result, dut):
+        if result.is_dut_failed(dut):
+            return "fail"
+        target_map = {}
+        target_map["dpdk_version"] = result.current_dpdk_version(dut)
+        target_map["nic"] = {}
+        for target in result.all_targets(dut):
+            target_map["nic"]["name"] = result.current_nic(dut, target)
+            target_map[target] = self.__scan_target(result, dut, target)
+            target_map["nic"]["kdriver"] = result.current_kdriver(dut)
+            target_map["nic"]["driver"] = result.current_driver(dut)
+            target_map["nic"]["firmware"] = result.current_firmware_version(dut)
+            if result.current_package_version(dut) is not None:
+                target_map["nic"]["pkg"] = result.current_package_version(dut)
+        return target_map
+
+    def save(self, result):
+        result_map = {}
+        for dut in result.all_duts():
+            result_map[dut] = self.__scan_dut(result, dut)
+        with open(self.filename, "w") as outfile:
+            json.dump(
+                result_map, outfile, indent=4, separators=(",", ": "), sort_keys=True
+            )
-- 
2.20.1

[RFC PATCH v1 3/6] dts: merge DTS framework/rst.py to DPDK

[Juraj Linkeš]

---
 dts/framework/rst.py | 168 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 168 insertions(+)
 create mode 100644 dts/framework/rst.py

diff --git a/dts/framework/rst.py b/dts/framework/rst.py
new file mode 100644
index 0000000000..18d7ccea39
--- /dev/null
+++ b/dts/framework/rst.py
@@ -0,0 +1,168 @@
+# BSD LICENSE
+#
+# Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+#   * Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#   * Redistributions in binary form must reproduce the above copyright
+#     notice, this list of conditions and the following disclaimer in
+#     the documentation and/or other materials provided with the
+#     distribution.
+#   * Neither the name of Intel Corporation nor the names of its
+#     contributors may be used to endorse or promote products derived
+#     from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import os
+import re
+import shutil
+
+from .exception import VerifyFailure
+
+"""
+Generate Rst Test Result Report
+
+Example:
+    import framework.rst as rst
+    rst.write_title("Test Case: " + test_case.__name__)
+    out = table.draw()
+    rst.write_text('\n' + out + '\n\n')
+    rst.write_result("PASS")
+
+Result:
+    <copyright>
+    <Prerequisites>
+    Test Case: CASE
+    ---------------
+    Result: PASS
+"""
+
+path2Plan = "test_plans"
+path2Result = "output"
+
+
+class RstReport(object):
+    def __init__(self, crbName, target, nic, suite, perf=False):
+        """
+        copy desc from #Name#_test_plan.rst to TestResult_#Name#.rst
+        """
+        try:
+            path = [path2Result, crbName, target, nic]
+            # ensure the level folder exist
+            for node in range(0, len(path)):
+                if not os.path.exists("/".join(path[: node + 1])):
+                    for level in range(node, len(path)):
+                        os.mkdir("/".join(path[: level + 1]))
+                    break
+
+            self.rstName = "%s/TestResult_%s.rst" % ("/".join(path), suite)
+            rstReport = open(self.rstName, "w")
+
+            if perf is True:
+                self.rstAnnexName = "%s/TestResult_%s_Annex.rst" % (
+                    "/".join(path),
+                    suite,
+                )
+                rstAnnexReport = open(self.rstAnnexName, "w")
+
+            f = open("%s/%s_test_plan.rst" % (path2Plan, suite), "r")
+            for line in f:
+                if line[:13] == "Prerequisites":
+                    break
+                rstReport.write(line)
+                if perf is True:
+                    rstAnnexReport.write(line)
+            f.close()
+
+            rstReport.close()
+
+        except Exception as e:
+            raise VerifyFailure("RST Error: " + str(e))
+
+    def clear_all_rst(self, crbName, target):
+        path = [path2Result, crbName, target]
+        shutil.rmtree("/".join(path), True)
+
+    def write_title(self, text):
+        """
+        write case title Test Case: #Name#
+        -----------------
+        """
+        line = "\n%s\n" % text
+        with open(self.rstName, "a") as f:
+            f.write(line)
+            f.write("-" * len(line) + "\n")
+
+    def write_subtitle(self):
+        if self._subtitle is not None:
+            with open(self.rstName, "a") as f:
+                f.write("%s\n" % self._subtitle)
+
+    def write_annex_title(self, text):
+        """
+        write annex to test case title Annex to #Name#
+        -----------------
+        """
+        line = "\n%s\n" % text
+        with open(self.rstAnnexName, "a") as f:
+            f.write(line)
+            f.write("-" * len(line) + "\n")
+
+    def write_text(self, text, annex=False):
+        rstFile = self.rstAnnexName if annex else self.rstName
+
+        with open(rstFile, "a") as f:
+            f.write(text)
+
+    def write_frame(self, text, annex=False):
+        self.write_text("\n::\n\n", annex)
+        parts = re.findall(r"\S+", text)
+        text = ""
+        length = 0
+
+        for part in parts:
+            if length + len(part) > 75:
+                text = text + "\n" + " " + part
+                length = len(part)
+            else:
+                length = length + len(part)
+                text = text + " " + part
+        self.write_text(text, annex)
+        self.write_text("\n\n", annex)
+
+    def write_result(self, result):
+        with open(self.rstName, "a") as f:
+            f.write("\nResult: " + result + "\n")
+
+    def include_image(self, image, width=90):
+        """
+        Includes an image in the RST file.
+        The argument must include path, name and extension.
+        """
+        with open(self.rstName, "a") as f:
+            f.write(".. image:: %s\n   :width: %d%%\n\n" % (image, width))
+
+    def report(self, text, frame=False, annex=False):
+        """
+        Save report text into rst file.
+        """
+        if frame:
+            self.write_frame(text, annex)
+        else:
+            self.write_text(text, annex)
-- 
2.20.1

[RFC PATCH v1 4/6] dts: merge DTS framework/stats_reporter.py to DPDK

[Juraj Linkeš]

---
 dts/framework/stats_reporter.py | 95 +++++++++++++++++++++++++++++++++
 1 file changed, 95 insertions(+)
 create mode 100644 dts/framework/stats_reporter.py

diff --git a/dts/framework/stats_reporter.py b/dts/framework/stats_reporter.py
new file mode 100644
index 0000000000..80ac942c5b
--- /dev/null
+++ b/dts/framework/stats_reporter.py
@@ -0,0 +1,95 @@
+# BSD LICENSE
+#
+# Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+#   * Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#   * Redistributions in binary form must reproduce the above copyright
+#     notice, this list of conditions and the following disclaimer in
+#     the documentation and/or other materials provided with the
+#     distribution.
+#   * Neither the name of Intel Corporation nor the names of its
+#     contributors may be used to endorse or promote products derived
+#     from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""
+Simple text file statistics generator
+"""
+
+
+class StatsReporter(object):
+
+    """
+    Generates a small statistics file containing the number of passing,
+    failing and blocked tests. It makes use of a Result instance as input.
+    """
+
+    def __init__(self, filename):
+        self.filename = filename
+
+    def __add_stat(self, test_result):
+        if test_result is not None:
+            if test_result[0] == "PASSED":
+                self.passed += 1
+            if test_result[0] == "FAILED":
+                self.failed += 1
+            if test_result[0] == "BLOCKED":
+                self.blocked += 1
+            self.total += 1
+
+    def __count_stats(self):
+        for dut in self.result.all_duts():
+            for target in self.result.all_targets(dut):
+                for suite in self.result.all_test_suites(dut, target):
+                    for case in self.result.all_test_cases(dut, target, suite):
+                        test_result = self.result.result_for(dut, target, suite, case)
+                        if len(test_result):
+                            self.__add_stat(test_result)
+
+    def __write_stats(self):
+        duts = self.result.all_duts()
+        if len(duts) == 1:
+            self.stats_file.write(
+                "dpdk_version = {}\n".format(self.result.current_dpdk_version(duts[0]))
+            )
+        else:
+            for dut in duts():
+                dpdk_version = self.result.current_dpdk_version(dut)
+                self.stats_file.write(
+                    "{}.dpdk_version = {}\n".format(dut, dpdk_version)
+                )
+        self.__count_stats()
+        self.stats_file.write("Passed     = %d\n" % self.passed)
+        self.stats_file.write("Failed     = %d\n" % self.failed)
+        self.stats_file.write("Blocked    = %d\n" % self.blocked)
+        rate = 0
+        if self.total > 0:
+            rate = self.passed * 100.0 / self.total
+        self.stats_file.write("Pass rate  = %.1f\n" % rate)
+
+    def save(self, result):
+        self.passed = 0
+        self.failed = 0
+        self.blocked = 0
+        self.total = 0
+        self.stats_file = open(self.filename, "w+")
+        self.result = result
+        self.__write_stats()
+        self.stats_file.close()
-- 
2.20.1

[RFC PATCH v1 5/6] dts: merge DTS framework/test_result.py to DPDK

[Juraj Linkeš]

---
 dts/framework/test_result.py | 494 +++++++++++++++++++++++++++++++++++
 1 file changed, 494 insertions(+)
 create mode 100644 dts/framework/test_result.py

diff --git a/dts/framework/test_result.py b/dts/framework/test_result.py
new file mode 100644
index 0000000000..abf8edb99e
--- /dev/null
+++ b/dts/framework/test_result.py
@@ -0,0 +1,494 @@
+# BSD LICENSE
+#
+# Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+#   * Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#   * Redistributions in binary form must reproduce the above copyright
+#     notice, this list of conditions and the following disclaimer in
+#     the documentation and/or other materials provided with the
+#     distribution.
+#   * Neither the name of Intel Corporation nor the names of its
+#     contributors may be used to endorse or promote products derived
+#     from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""
+Generic result container and reporters
+"""
+
+
+class Result(object):
+
+    """
+    Generic result container. Useful to store/retrieve results during
+    a DTF execution.
+
+    It manages and hide an internal complex structure like the one shown below.
+    This is presented to the user with a property based interface.
+
+    internals = [
+        'dut1', [
+            'kdriver',
+            'firmware',
+            'pkg',
+            'driver',
+            'dpdk_version',
+            'target1', 'nic1', [
+                'suite1', [
+                    'case1', ['PASSED', ''],
+                    'case2', ['PASSED', ''],
+                ],
+            ],
+            'target2', 'nic1', [
+                'suite2', [
+                    'case3', ['PASSED', ''],
+                    'case4', ['FAILED', 'message'],
+                ],
+                'suite3', [
+                    'case5', ['BLOCKED', 'message'],
+                ],
+            ]
+        ]
+    ]
+
+    """
+
+    def __init__(self):
+        self.__dut = 0
+        self.__target = 0
+        self.__test_suite = 0
+        self.__test_case = 0
+        self.__test_result = None
+        self.__message = None
+        self.__internals = []
+        self.__failed_duts = {}
+        self.__failed_targets = {}
+
+    def __set_dut(self, dut):
+        if dut not in self.__internals:
+            self.__internals.append(dut)
+            self.__internals.append([])
+        self.__dut = self.__internals.index(dut)
+
+    def __get_dut(self):
+        return self.__internals[self.__dut]
+
+    def current_dpdk_version(self, dut):
+        """
+        Returns the dpdk version for a given DUT
+        """
+        try:
+            dut_idx = self.__internals.index(dut)
+            return self.__internals[dut_idx + 1][4]
+        except:
+            return ""
+
+    def __set_dpdk_version(self, dpdk_version):
+        if dpdk_version not in self.internals[self.__dut + 1]:
+            dpdk_current = self.__get_dpdk_version()
+            if dpdk_current:
+                if dpdk_version not in dpdk_current:
+                    self.internals[self.__dut + 1][4] = (
+                        dpdk_current + "/" + dpdk_version
+                    )
+            else:
+                self.internals[self.__dut + 1].append(dpdk_version)
+
+    def __get_dpdk_version(self):
+        try:
+            return self.internals[self.__dut + 1][4]
+        except:
+            return ""
+
+    def current_kdriver(self, dut):
+        """
+        Returns the driver version for a given DUT
+        """
+        try:
+            dut_idx = self.__internals.index(dut)
+            return self.__internals[dut_idx + 1][0]
+        except:
+            return ""
+
+    def __set_kdriver(self, driver):
+        if not self.internals[self.__dut + 1]:
+            kdriver_current = self.__get_kdriver()
+            if kdriver_current:
+                if driver not in kdriver_current:
+                    self.internals[self.__dut + 1][0] = kdriver_current + "/" + driver
+            else:
+                self.internals[self.__dut + 1].append(driver)
+
+    def __get_kdriver(self):
+        try:
+            return self.internals[self.__dut + 1][0]
+        except:
+            return ""
+
+    def current_firmware_version(self, dut):
+        """
+        Returns the firmware version for a given DUT
+        """
+        try:
+            dut_idx = self.__internals.index(dut)
+            return self.__internals[dut_idx + 1][1]
+        except:
+            return ""
+
+    def __set_firmware(self, firmware):
+        if firmware not in self.internals[self.__dut + 1]:
+            firmware_current = self.__get_firmware()
+            if firmware_current:
+                if firmware not in firmware_current:
+                    self.internals[self.__dut + 1][1] = (
+                        firmware_current + "/" + firmware
+                    )
+            else:
+                self.internals[self.__dut + 1].append(firmware)
+
+    def __get_firmware(self):
+        try:
+            return self.internals[self.__dut + 1][1]
+        except:
+            return ""
+
+    def current_package_version(self, dut):
+        """
+        Returns the DDP package version for a given DUT
+        """
+        try:
+            dut_idx = self.__internals.index(dut)
+            return self.__internals[dut_idx + 1][2]
+        except:
+            return ""
+
+    def __set_ddp_package(self, package):
+        if package not in self.internals[self.__dut + 1]:
+            pkg_current = self.__get_ddp_package()
+            if pkg_current != "":
+                if pkg_current and package not in pkg_current:
+                    self.internals[self.__dut + 1][2] = pkg_current + "/" + package
+            else:
+                self.internals[self.__dut + 1].append(package)
+
+    def __get_ddp_package(self):
+        try:
+            return self.internals[self.__dut + 1][2]
+        except:
+            return ""
+
+    def current_driver(self, dut):
+        """
+        Returns the DDP package version for a given DUT
+        """
+        try:
+            dut_idx = self.__internals.index(dut)
+            return self.__internals[dut_idx + 1][3]
+        except:
+            return ""
+
+    def __set_driver(self, driver):
+        if driver not in self.internals[self.__dut + 1]:
+            driver_current = self.__get_driver()
+            if driver_current:
+                if driver not in driver_current:
+                    self.internals[self.__dut + 1][3] = driver_current + "/" + driver
+            else:
+                self.internals[self.__dut + 1].append(driver)
+
+    def __get_driver(self):
+        try:
+            return self.internals[self.__dut + 1][3]
+        except:
+            return ""
+
+    def __current_targets(self):
+        return self.internals[self.__dut + 1]
+
+    def __set_target(self, target):
+        targets = self.__current_targets()
+        if target not in targets:
+            targets.append(target)
+            targets.append("_nic_")
+            targets.append([])
+        self.__target = targets.index(target)
+
+    def __get_target(self):
+        return self.__current_targets()[self.__target]
+
+    def __set_nic(self, nic):
+        targets = self.__current_targets()
+        targets[self.__target + 1] = nic
+
+    def __get_nic(self):
+        targets = self.__current_targets()
+        return targets[self.__target + 1]
+
+    def __current_suites(self):
+        return self.__current_targets()[self.__target + 2]
+
+    def __set_test_suite(self, test_suite):
+        suites = self.__current_suites()
+        if test_suite not in suites:
+            suites.append(test_suite)
+            suites.append([])
+        self.__test_suite = suites.index(test_suite)
+
+    def __get_test_suite(self):
+        return self.__current_suites()[self.__test_suite]
+
+    def __current_cases(self):
+        return self.__current_suites()[self.__test_suite + 1]
+
+    def __set_test_case(self, test_case):
+        cases = self.__current_cases()
+        cases.append(test_case)
+        cases.append([])
+        self.__test_case = cases.index(test_case)
+
+    def __get_test_case(self):
+        return self.__current_cases()[self.__test_case]
+
+    def __get_test_result(self):
+        return self.__test_result
+
+    def __get_message(self):
+        return self.__message
+
+    def __get_internals(self):
+        return self.__internals
+
+    def __current_result(self):
+        return self.__current_cases()[self.__test_case + 1]
+
+    def __set_test_case_result(self, result, message):
+        test_case = self.__current_result()
+        test_case.append(result)
+        test_case.append(message)
+        self.__test_result = result
+        self.__message = message
+
+    def copy_suite(self, suite_result):
+        self.__current_suites()[self.__test_suite + 1] = suite_result.__current_cases()
+
+    def test_case_passed(self):
+        """
+        Set last test case added as PASSED
+        """
+        self.__set_test_case_result(result="PASSED", message="")
+
+    def test_case_skip(self, message):
+        """
+        set last test case add as N/A
+        """
+        self.__set_test_case_result(result="N/A", message=message)
+
+    def test_case_failed(self, message):
+        """
+        Set last test case added as FAILED
+        """
+        self.__set_test_case_result(result="FAILED", message=message)
+
+    def test_case_blocked(self, message):
+        """
+        Set last test case added as BLOCKED
+        """
+        self.__set_test_case_result(result="BLOCKED", message=message)
+
+    def all_duts(self):
+        """
+        Returns all the DUTs it's aware of.
+        """
+        return self.__internals[::2]
+
+    def all_targets(self, dut):
+        """
+        Returns the targets for a given DUT
+        """
+        try:
+            dut_idx = self.__internals.index(dut)
+        except:
+            return None
+        return self.__internals[dut_idx + 1][5::3]
+
+    def current_nic(self, dut, target):
+        """
+        Returns the NIC for a given DUT and target
+        """
+        try:
+            dut_idx = self.__internals.index(dut)
+            target_idx = self.__internals[dut_idx + 1].index(target)
+        except:
+            return None
+        return self.__internals[dut_idx + 1][target_idx + 1]
+
+    def all_test_suites(self, dut, target):
+        """
+        Returns all the test suites for a given DUT and target.
+        """
+        try:
+            dut_idx = self.__internals.index(dut)
+            target_idx = self.__internals[dut_idx + 1].index(target)
+        except:
+            return None
+        return self.__internals[dut_idx + 1][target_idx + 2][::2]
+
+    def all_test_cases(self, dut, target, suite):
+        """
+        Returns all the test cases for a given DUT, target and test case.
+        """
+        try:
+            dut_idx = self.__internals.index(dut)
+            target_idx = self.__internals[dut_idx + 1].index(target)
+            suite_idx = self.__internals[dut_idx + 1][target_idx + 2].index(suite)
+        except:
+            return None
+        return self.__internals[dut_idx + 1][target_idx + 2][suite_idx + 1][::2]
+
+    def result_for(self, dut, target, suite, case):
+        """
+        Returns the test case result/message for a given DUT, target, test
+        suite and test case.
+        """
+        try:
+            dut_idx = self.__internals.index(dut)
+            target_idx = self.__internals[dut_idx + 1].index(target)
+            suite_idx = self.__internals[dut_idx + 1][target_idx + 2].index(suite)
+            case_idx = self.__internals[dut_idx + 1][target_idx + 2][
+                suite_idx + 1
+            ].index(case)
+        except:
+            return None
+        return self.__internals[dut_idx + 1][target_idx + 2][suite_idx + 1][
+            case_idx + 1
+        ]
+
+    def add_failed_dut(self, dut, msg):
+        """
+        Sets the given DUT as failing due to msg
+        """
+        self.__failed_duts[dut] = msg
+
+    def is_dut_failed(self, dut):
+        """
+        True if the given DUT was marked as failing
+        """
+        return dut in self.__failed_duts
+
+    def dut_failed_msg(self, dut):
+        """
+        Returns the reason of failure for a given DUT
+        """
+        return self.__failed_duts[dut]
+
+    def add_failed_target(self, dut, target, msg):
+        """
+        Sets the given DUT, target as failing due to msg
+        """
+        self.__failed_targets[dut + target] = msg
+
+    def is_target_failed(self, dut, target):
+        """
+        True if the given DUT,target were marked as failing
+        """
+        return (dut + target) in self.__failed_targets
+
+    def target_failed_msg(self, dut, target):
+        """
+        Returns the reason of failure for a given DUT,target
+        """
+        return self.__failed_targets[dut + target]
+
+    """
+    Attributes defined as properties to hide the implementation from the
+    presented interface.
+    """
+    dut = property(__get_dut, __set_dut)
+    dpdk_version = property(__get_dpdk_version, __set_dpdk_version)
+    kdriver = property(__get_kdriver, __set_kdriver)
+    driver = property(__get_driver, __set_driver)
+    firmware = property(__get_firmware, __set_firmware)
+    package = property(__get_ddp_package, __set_ddp_package)
+    target = property(__get_target, __set_target)
+    test_suite = property(__get_test_suite, __set_test_suite)
+    test_case = property(__get_test_case, __set_test_case)
+    test_result = property(__get_test_result)
+    message = property(__get_message)
+    nic = property(__get_nic, __set_nic)
+    internals = property(__get_internals)
+
+
+class ResultTable(object):
+    def __init__(self, header):
+        """
+        Add the title of result table.
+        Usage:
+        rt = ResultTable(header)
+        rt.add_row(row)
+        rt.table_print()
+        """
+        from texttable import Texttable
+
+        self.results_table_rows = []
+        self.results_table_rows.append([])
+        self.table = Texttable(max_width=150)
+        self.results_table_header = header
+        self.logger = None
+        self.rst = None
+
+    def set_rst(self, rst):
+        self.rst = rst
+
+    def set_logger(self, logger):
+        self.logger = logger
+
+    def add_row(self, row):
+        """
+        Add one row to result table.
+        """
+        self.results_table_rows.append(row)
+
+    def table_print(self):
+        """
+        Show off result table.
+        """
+        self.table.add_rows(self.results_table_rows)
+        self.table.header(self.results_table_header)
+
+        alignments = []
+        # all header align to left
+        for _ in self.results_table_header:
+            alignments.append("l")
+        self.table.set_cols_align(alignments)
+
+        out = self.table.draw()
+        if self.rst:
+            self.rst.write_text("\n" + out + "\n\n")
+        if self.logger:
+            self.logger.info("\n" + out)
+
+
+###############################################################################
+###############################################################################
+if __name__ == "__main__":
+    rt = ResultTable(header=["name", "age"])
+    rt.add_row(["Jane", "30"])
+    rt.add_row(["Mark", "32"])
+    rt.table_print()
-- 
2.20.1

[RFC PATCH v1 6/6] dts: merge DTS output/Readme.txt to DPDK

[Juraj Linkeš]

---
 dts/output/Readme.txt | 2 ++
 1 file changed, 2 insertions(+)
 create mode 100644 dts/output/Readme.txt

diff --git a/dts/output/Readme.txt b/dts/output/Readme.txt
new file mode 100644
index 0000000000..8d40ec01b5
--- /dev/null
+++ b/dts/output/Readme.txt
@@ -0,0 +1,2 @@
+Output dir for DTF results.
+
-- 
2.20.1