[dts] [PATCH V3 1/4] add re_run parameter

[dts] [PATCH V3 1/4] add re_run parameter

[xu,huilong]

add a new re_run parameter for failed case re run time. the default value is 0.
---
 framework/dts.py  | 7 +++++--
 framework/main.py | 5 ++++-
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/framework/dts.py b/framework/dts.py
index 369599d..2c42d33 100644
--- a/framework/dts.py
+++ b/framework/dts.py
@@ -424,7 +424,7 @@ def dts_run_suite(duts, tester, test_suites, target):
 def run_all(config_file, pkgName, git, patch, skip_setup,
             read_cache, project, suite_dir, test_cases,
             base_dir, output_dir, verbose, virttype, debug,
-            debugcase, commands):
+            debugcase, re_run, commands):
     """
     Main process of DTS, it will run all test suites in the config file.
     """
@@ -465,6 +465,9 @@ def run_all(config_file, pkgName, git, patch, skip_setup,
     if verbose is True:
         logger.set_verbose()
 
+    if re_run < 0:
+        re_run = 0
+
     logger.log_dir = output_dir
     log_handler = getLogger('dts')
     log_handler.config_execution('dts')
@@ -520,7 +523,7 @@ def run_all(config_file, pkgName, git, patch, skip_setup,
 
         # init dut, tester crb
         duts, tester = dts_crbs_init(crbInsts, skip_setup, read_cache, project, base_dir, serializer, virttype)
-
+        tester.set_re_run(re_run)
         # register exit action
         atexit.register(quit_execution, duts, tester)
 
diff --git a/framework/main.py b/framework/main.py
index a018af0..0aa54fd 100755
--- a/framework/main.py
+++ b/framework/main.py
@@ -134,6 +134,9 @@ parser.add_argument('--debug',
 parser.add_argument('--debugcase',
                     action='store_true',
                     help='enable debug mode in the first case, user can further debug')
+parser.add_argument('--re_run',
+                    default=0,
+                    help='when case failed will re-run times, and this value must >= 0')
 
 parser.add_argument('--commands',
                     action='append',
@@ -156,4 +159,4 @@ dts.run_all(args.config_file, args.snapshot, args.git,
             args.patch, args.skip_setup, args.read_cache,
             args.project, args.suite_dir, args.test_cases,
             args.dir, args.output, args.verbose,args.virttype,
-            args.debug, args.debugcase, args.commands)
+            args.debug, args.debugcase, args.re_run, args.commands)
-- 
1.9.3

[dts] [PATCH V3 2/4] update exec case function

[xu,huilong]

when test case run failed, failed case will re run some times. if have passed result, this case will passed.
---
 framework/test_case.py | 29 +++++++++++++++++++++--------
 1 file changed, 21 insertions(+), 8 deletions(-)

diff --git a/framework/test_case.py b/framework/test_case.py
index d97d898..0f33768 100644
--- a/framework/test_case.py
+++ b/framework/test_case.py
@@ -36,6 +36,7 @@ import re
 import debugger
 import traceback
 import signal
+import time
 
 from exception import VerifyFailure, TimeoutException
 from settings import DRIVERS, NICS, get_nic_name, load_global_setting
@@ -44,7 +45,6 @@ from rst import RstReport
 from test_result import ResultTable, Result
 from logger import getLogger
 
-
 class TestCase(object):
 
     def __init__(self, duts, tester, target, suitename):
@@ -244,20 +244,20 @@ class TestCase(object):
         self._suite_result.test_case = case_obj.__name__
 
         self._rst_obj.write_title("Test Case: " + case_name)
-
+        case_result = True
         if self._check_inst is not None:
             if self._check_inst.case_skip(case_name[len("test_"):]):
                 self.logger.info('Test Case %s Result SKIPED:' % case_name)
                 self._rst_obj.write_result("N/A")
                 self._suite_result.test_case_skip(self._check_inst.comments)
-                return
+                return case_result
 
         if self._support_inst is not None:
             if not self._support_inst.case_support(case_name[len("test_"):]):
                 self.logger.info('Test Case %s Result SKIPED:' % case_name)
                 self._rst_obj.write_result("N/A")
                 self._suite_result.test_case_skip(self._support_inst.comments)
-                return
+                return case_result
 
         if self._enable_perf:
             self._rst_obj.write_annex_title("Annex: " + case_name)
@@ -287,6 +287,7 @@ class TestCase(object):
             self.logger.info('Test Case %s Result PASSED:' % case_name)
 
         except VerifyFailure as v:
+            case_result = False
             self._suite_result.test_case_failed(str(v))
             self._rst_obj.write_result("FAIL")
             self.logger.error('Test Case %s Result FAILED: ' % (case_name) + str(v))
@@ -296,25 +297,37 @@ class TestCase(object):
             self.tear_down()
             raise KeyboardInterrupt("Stop DTS")
         except TimeoutException as e:
+            case_result = False
             self._rst_obj.write_result("FAIL")
-            msg = str(e)
-            self._suite_result.test_case_failed(msg)
-            self.logger.error('Test Case %s Result FAILED: ' % (case_name) + msg)
+            self._suite_result.test_case_failed(str(e))
+            self.logger.error('Test Case %s Result FAILED: ' % (case_name) + str(e))
             self.logger.error('%s' % (e.get_output()))
         except Exception:
+            case_result = False
             trace = traceback.format_exc()
             self._suite_result.test_case_failed(trace)
             self.logger.error('Test Case %s Result ERROR: ' % (case_name) + trace)
         finally:
             self.tear_down()
+            return case_result
 
     def execute_test_cases(self):
         """
         Execute all test cases in one suite.
         """
+
         if load_global_setting(FUNC_SETTING) == 'yes':
             for case_obj in self._get_functional_cases():
-                self._execute_test_case(case_obj)
+                for i in range(self.tester.re_run_time + 1):
+                    if self._execute_test_case(case_obj):
+                        break
+                    else:
+                        for dutobj in self.duts:
+                            dutobj.get_session_output(timeout = 0.5 * (i + 1))
+                        self.tester.get_session_output(timeout = 0.5 * (i + 1))
+                        time.sleep(i + 1)
+                        self.logger.info(" Test case %s re-run %d time" % (case_obj.__name__, i + 1))
+
         if load_global_setting(PERF_SETTING) == 'yes':
             for case_obj in self._get_performance_cases():
                 self._execute_test_case(case_obj)
-- 
1.9.3

[dts] [PATCH V3 3/4] update case result write function

[xu,huilong]

update case result write function, all same case only have one test result
if one time case test result passed, this result will passed and save in excel file.
if all times case test result failed, this resulr will failed and save in excel file.
---
 framework/excel_reporter.py | 25 ++++++++++++++++++++-----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/framework/excel_reporter.py b/framework/excel_reporter.py
index 2030d87..5b8e4cf 100644
--- a/framework/excel_reporter.py
+++ b/framework/excel_reporter.py
@@ -142,9 +142,23 @@ class ExcelReporter(object):
 
         self.title_style = xlwt.XFStyle()
         self.title_style.font = title_font
-
-    def __write_result(self, dut, target, suite, case):
-        test_result = self.result.result_for(dut, target, suite, case)
+ 
+    def __get_case_result(self, dut, target, suite, case):
+        case_list = self.result.all_test_cases(dut, target, suite)
+        if case_list.count(case) > 1:
+            tmp_result = []
+            for case_name in case_list:
+                if case == case_name:
+                    test_result = self.result.result_for(dut, target, suite, case)
+                    if 'PASSED' in test_result:
+                        return ['PASSED', '']
+                    else:
+                        tmp_result.append(test_result)
+            return tmp_result[-1]
+        else:
+            return self.result.result_for(dut, target, suite, case)
+
+    def __write_result(self, dut, target, suite, case, test_result):
         if test_result is not None and len(test_result) > 0:
             result = test_result[0]
             if test_result[1] != '':
@@ -156,13 +170,14 @@ class ExcelReporter(object):
                     self.row, self.col + 1, result, self.failed_style)
 
     def __write_cases(self, dut, target, suite):
-        for case in self.result.all_test_cases(dut, target, suite):
+        for case in set(self.result.all_test_cases(dut, target, suite)):
+            result = self.__get_case_result(dut, target, suite, case)
             self.col += 1
             if case[:5] == "test_":
                 self.sheet.write(self.row, self.col, case[5:])
             else:
                 self.sheet.write(self.row, self.col, case)
-            self.__write_result(dut, target, suite, case)
+            self.__write_result(dut, target, suite, case, result)
             self.row += 1
             self.col -= 1
 
-- 
1.9.3

[dts] [PATCH V3 4/4] set re run failed case time

[xu,huilong]

---
 framework/tester.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/framework/tester.py b/framework/tester.py
index 8f927ea..5e1afeb 100644
--- a/framework/tester.py
+++ b/framework/tester.py
@@ -69,6 +69,7 @@ class Tester(Crb):
         self.scapyCmds = []
         self.bgCmds = []
         self.bgItf = ''
+        self.re_run_time = 0
 
     def init_ext_gen(self):
         """
@@ -78,6 +79,12 @@ class Tester(Crb):
             self.ixia_packet_gen = IxiaPacketGenerator(self)
         self.packet_gen = SoftwarePacketGenerator(self)
 
+    def set_re_run(self, re_run_time):
+        """
+        set failed case re-run time
+        """
+        self.re_run_time = int(re_run_time)
+
     def get_ip_address(self):
         """
         Get ip address of tester CRB.
-- 
1.9.3

Re: [dts] [PATCH V3 1/4] add re_run parameter

[Liu, Yong]

Thanks Huilong, applied into master branch.

On 07/07/2017 01:21 PM, xu,huilong wrote:
> add a new re_run parameter for failed case re run time. the default value is 0.
> ---
>   framework/dts.py  | 7 +++++--
>   framework/main.py | 5 ++++-
>   2 files changed, 9 insertions(+), 3 deletions(-)