[dts] [PATCH V1 1/3] add re_run parameter

[dts] [PATCH V1 1/3] add re_run parameter

[xu,huilong]

add a new re_run parameter for failed case re run time. the default value is 1.

Signed-off-by: xu,huilong <huilongx.xu@intel.com>
---
 framework/dts.py  | 7 +++++--
 framework/main.py | 5 ++++-
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/framework/dts.py b/framework/dts.py
index 369599d..48be694 100644
--- a/framework/dts.py
+++ b/framework/dts.py
@@ -424,7 +424,7 @@ def dts_run_suite(duts, tester, test_suites, target):
 def run_all(config_file, pkgName, git, patch, skip_setup,
             read_cache, project, suite_dir, test_cases,
             base_dir, output_dir, verbose, virttype, debug,
-            debugcase, commands):
+            debugcase, re_run, commands):
     """
     Main process of DTS, it will run all test suites in the config file.
     """
@@ -465,6 +465,9 @@ def run_all(config_file, pkgName, git, patch, skip_setup,
     if verbose is True:
         logger.set_verbose()
 
+    if re_run < 1:
+        re_run = 1
+
     logger.log_dir = output_dir
     log_handler = getLogger('dts')
     log_handler.config_execution('dts')
@@ -520,7 +523,7 @@ def run_all(config_file, pkgName, git, patch, skip_setup,
 
         # init dut, tester crb
         duts, tester = dts_crbs_init(crbInsts, skip_setup, read_cache, project, base_dir, serializer, virttype)
-
+        tester.set_re_run(re_run)
         # register exit action
         atexit.register(quit_execution, duts, tester)
 
diff --git a/framework/main.py b/framework/main.py
index a018af0..de9d54f 100755
--- a/framework/main.py
+++ b/framework/main.py
@@ -134,6 +134,9 @@ parser.add_argument('--debug',
 parser.add_argument('--debugcase',
                     action='store_true',
                     help='enable debug mode in the first case, user can further debug')
+parser.add_argument('--re_run',
+                    default=1,
+                    help='when case failed will re-run times, and this value must >= 1')
 
 parser.add_argument('--commands',
                     action='append',
@@ -156,4 +159,4 @@ dts.run_all(args.config_file, args.snapshot, args.git,
             args.patch, args.skip_setup, args.read_cache,
             args.project, args.suite_dir, args.test_cases,
             args.dir, args.output, args.verbose,args.virttype,
-            args.debug, args.debugcase, args.commands)
+            args.debug, args.debugcase, args.re_run, args.commands)
-- 
1.9.3

[dts] [PATCH V1 2/3] update exec case function

[xu,huilong]

when test case run failed, failed case will re run some times. if have passed result, this case will passed.

Signed-off-by: xu,huilong <huilongx.xu@intel.com>
---
 framework/test_case.py | 29 +++++++++++++++++++++--------
 1 file changed, 21 insertions(+), 8 deletions(-)

diff --git a/framework/test_case.py b/framework/test_case.py
index d97d898..4f49fe4 100644
--- a/framework/test_case.py
+++ b/framework/test_case.py
@@ -36,6 +36,7 @@ import re
 import debugger
 import traceback
 import signal
+import time
 
 from exception import VerifyFailure, TimeoutException
 from settings import DRIVERS, NICS, get_nic_name, load_global_setting
@@ -44,7 +45,6 @@ from rst import RstReport
 from test_result import ResultTable, Result
 from logger import getLogger
 
-
 class TestCase(object):
 
     def __init__(self, duts, tester, target, suitename):
@@ -244,20 +244,20 @@ class TestCase(object):
         self._suite_result.test_case = case_obj.__name__
 
         self._rst_obj.write_title("Test Case: " + case_name)
-
+        case_result = True
         if self._check_inst is not None:
             if self._check_inst.case_skip(case_name[len("test_"):]):
                 self.logger.info('Test Case %s Result SKIPED:' % case_name)
                 self._rst_obj.write_result("N/A")
                 self._suite_result.test_case_skip(self._check_inst.comments)
-                return
+                return case_result
 
         if self._support_inst is not None:
             if not self._support_inst.case_support(case_name[len("test_"):]):
                 self.logger.info('Test Case %s Result SKIPED:' % case_name)
                 self._rst_obj.write_result("N/A")
                 self._suite_result.test_case_skip(self._support_inst.comments)
-                return
+                return case_result
 
         if self._enable_perf:
             self._rst_obj.write_annex_title("Annex: " + case_name)
@@ -287,6 +287,7 @@ class TestCase(object):
             self.logger.info('Test Case %s Result PASSED:' % case_name)
 
         except VerifyFailure as v:
+            case_result = False
             self._suite_result.test_case_failed(str(v))
             self._rst_obj.write_result("FAIL")
             self.logger.error('Test Case %s Result FAILED: ' % (case_name) + str(v))
@@ -296,25 +297,37 @@ class TestCase(object):
             self.tear_down()
             raise KeyboardInterrupt("Stop DTS")
         except TimeoutException as e:
+            case_result = False
             self._rst_obj.write_result("FAIL")
-            msg = str(e)
-            self._suite_result.test_case_failed(msg)
-            self.logger.error('Test Case %s Result FAILED: ' % (case_name) + msg)
+            self._suite_result.test_case_failed(str(e))
+            self.logger.error('Test Case %s Result FAILED: ' % (case_name) + str(e))
             self.logger.error('%s' % (e.get_output()))
         except Exception:
+            case_result = False
             trace = traceback.format_exc()
             self._suite_result.test_case_failed(trace)
             self.logger.error('Test Case %s Result ERROR: ' % (case_name) + trace)
         finally:
             self.tear_down()
+            return case_result
 
     def execute_test_cases(self):
         """
         Execute all test cases in one suite.
         """
+
         if load_global_setting(FUNC_SETTING) == 'yes':
             for case_obj in self._get_functional_cases():
-                self._execute_test_case(case_obj)
+                for i in range(self.tester.re_run_time):
+                    if self._execute_test_case(case_obj):
+                        break
+                    else:
+                        for dutobj in self.duts:
+                            dutobj.get_session_output(timeout = 0.5 * (i + 1))
+                        self.tester.get_session_output(timeout = 0.5 * (i + 1))
+                        time.sleep(i + 1)
+                        self.logger.info(" Test case %s re-run %d time" % (case_obj.__name__, i + 1))
+
         if load_global_setting(PERF_SETTING) == 'yes':
             for case_obj in self._get_performance_cases():
                 self._execute_test_case(case_obj)
-- 
1.9.3

[dts] [PATCH V1 3/3] update case result write function

[xu,huilong]

update case result write function, all same case only have one test result
if one time case test result passed, this result will passed and save in excel file.
if all times case test result failed, this resulr will failed and save in excel file.

Signed-off-by: xu,huilong <huilongx.xu@intel.com>
---
 framework/excel_reporter.py | 25 ++++++++++++++++++++-----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/framework/excel_reporter.py b/framework/excel_reporter.py
index 2030d87..aef50c6 100644
--- a/framework/excel_reporter.py
+++ b/framework/excel_reporter.py
@@ -142,9 +142,23 @@ class ExcelReporter(object):
 
         self.title_style = xlwt.XFStyle()
         self.title_style.font = title_font
-
-    def __write_result(self, dut, target, suite, case):
-        test_result = self.result.result_for(dut, target, suite, case)
+    
+    def __get_suite_result(self, dut, target, suite, case):
+        case_list = self.result.all_test_cases(dut, target, suite)
+        if case_list.count(case) > 1:
+            tmp_result = []
+            for case_name in case_list:
+                if case == case_name:
+                    test_result = self.result.result_for(dut, target, suite, case)
+                    if 'PASSED' in test_result:
+                        return ['PASSED', '']
+                    else:
+                        tmp_result.append(test_result)
+            return tmp_result[-1]
+        else:
+            return self.result.result_for(dut, target, suite, case)
+
+    def __write_result(self, dut, target, suite, case, test_result):
         if test_result is not None and len(test_result) > 0:
             result = test_result[0]
             if test_result[1] != '':
@@ -156,13 +170,14 @@ class ExcelReporter(object):
                     self.row, self.col + 1, result, self.failed_style)
 
     def __write_cases(self, dut, target, suite):
-        for case in self.result.all_test_cases(dut, target, suite):
+        for case in set(self.result.all_test_cases(dut, target, suite)):
+            result = self.__get_suite_result(dut, target, suite, case)
             self.col += 1
             if case[:5] == "test_":
                 self.sheet.write(self.row, self.col, case[5:])
             else:
                 self.sheet.write(self.row, self.col, case)
-            self.__write_result(dut, target, suite, case)
+            self.__write_result(dut, target, suite, case, result)
             self.row += 1
             self.col -= 1
 
-- 
1.9.3

Re: [dts] [PATCH V1 1/3] add re_run parameter

[Liu, Yong]

Huilong,
Rerun is not supposed to be the default action for failed cases. The default value for rerun times should be zero, 

> -----Original Message-----
> From: dts [mailto:dts-bounces@dpdk.org] On Behalf Of xu,huilong
> Sent: Thursday, June 22, 2017 10:57 PM
> To: dts@dpdk.org
> Cc: Xu, HuilongX <huilongx.xu@intel.com>
> Subject: [dts] [PATCH V1 1/3] add re_run parameter
> 
> add a new re_run parameter for failed case re run time. the default value
> is 1.
> 
> Signed-off-by: xu,huilong <huilongx.xu@intel.com>
> ---
>  framework/dts.py  | 7 +++++--
>  framework/main.py | 5 ++++-
>  2 files changed, 9 insertions(+), 3 deletions(-)
> 
> diff --git a/framework/dts.py b/framework/dts.py
> index 369599d..48be694 100644
> --- a/framework/dts.py
> +++ b/framework/dts.py
> @@ -424,7 +424,7 @@ def dts_run_suite(duts, tester, test_suites, target):
>  def run_all(config_file, pkgName, git, patch, skip_setup,
>              read_cache, project, suite_dir, test_cases,
>              base_dir, output_dir, verbose, virttype, debug,
> -            debugcase, commands):
> +            debugcase, re_run, commands):
>      """
>      Main process of DTS, it will run all test suites in the config file.
>      """
> @@ -465,6 +465,9 @@ def run_all(config_file, pkgName, git, patch,
> skip_setup,
>      if verbose is True:
>          logger.set_verbose()
> 
> +    if re_run < 1:
> +        re_run = 1
> +
>      logger.log_dir = output_dir
>      log_handler = getLogger('dts')
>      log_handler.config_execution('dts')
> @@ -520,7 +523,7 @@ def run_all(config_file, pkgName, git, patch,
> skip_setup,
> 
>          # init dut, tester crb
>          duts, tester = dts_crbs_init(crbInsts, skip_setup, read_cache,
> project, base_dir, serializer, virttype)
> -
> +        tester.set_re_run(re_run)
>          # register exit action
>          atexit.register(quit_execution, duts, tester)
> 
> diff --git a/framework/main.py b/framework/main.py
> index a018af0..de9d54f 100755
> --- a/framework/main.py
> +++ b/framework/main.py
> @@ -134,6 +134,9 @@ parser.add_argument('--debug',
>  parser.add_argument('--debugcase',
>                      action='store_true',
>                      help='enable debug mode in the first case, user can
> further debug')
> +parser.add_argument('--re_run',
> +                    default=1,
> +                    help='when case failed will re-run times, and this
> value must >= 1')
> 

Suggest add  "type=int" in re-run argument definition. 

>  parser.add_argument('--commands',
>                      action='append',
> @@ -156,4 +159,4 @@ dts.run_all(args.config_file, args.snapshot, args.git,
>              args.patch, args.skip_setup, args.read_cache,
>              args.project, args.suite_dir, args.test_cases,
>              args.dir, args.output, args.verbose,args.virttype,
> -            args.debug, args.debugcase, args.commands)
> +            args.debug, args.debugcase, args.re_run, args.commands)
> --
> 1.9.3

Re: [dts] [PATCH V1 3/3] update case result write function

[Liu, Yong]

Huilong, some comments below.

> -----Original Message-----
> From: dts [mailto:dts-bounces@dpdk.org] On Behalf Of xu,huilong
> Sent: Thursday, June 22, 2017 10:57 PM
> To: dts@dpdk.org
> Cc: Xu, HuilongX <huilongx.xu@intel.com>
> Subject: [dts] [PATCH V1 3/3] update case result write function
> 
> update case result write function, all same case only have one test result
> if one time case test result passed, this result will passed and save in
> excel file.
> if all times case test result failed, this resulr will failed and save in
> excel file.
> 
> Signed-off-by: xu,huilong <huilongx.xu@intel.com>
> ---
>  framework/excel_reporter.py | 25 ++++++++++++++++++++-----
>  1 file changed, 20 insertions(+), 5 deletions(-)
> 
> diff --git a/framework/excel_reporter.py b/framework/excel_reporter.py
> index 2030d87..aef50c6 100644
> --- a/framework/excel_reporter.py
> +++ b/framework/excel_reporter.py
> @@ -142,9 +142,23 @@ class ExcelReporter(object):
> 
>          self.title_style = xlwt.XFStyle()
>          self.title_style.font = title_font
> -
> -    def __write_result(self, dut, target, suite, case):
> -        test_result = self.result.result_for(dut, target, suite, case)
> +
> +    def __get_suite_result(self, dut, target, suite, case):


Look like function has been change to get result of certain case.  Suggest to change name of this function to __get_case_result.

> +        case_list = self.result.all_test_cases(dut, target, suite)
> +        if case_list.count(case) > 1:
> +            tmp_result = []
> +            for case_name in case_list:
> +                if case == case_name:
> +                    test_result = self.result.result_for(dut, target,
> suite, case)
> +                    if 'PASSED' in test_result:
> +                        return ['PASSED', '']

Why return result list with two items here and only return one is other condition?
 
> +                    else:
> +                        tmp_result.append(test_result)
> +            return tmp_result[-1]
> +        else:
> +            return self.result.result_for(dut, target, suite, case)
> +
> +    def __write_result(self, dut, target, suite, case, test_result):
>          if test_result is not None and len(test_result) > 0:
>              result = test_result[0]
>              if test_result[1] != '':
> @@ -156,13 +170,14 @@ class ExcelReporter(object):
>                      self.row, self.col + 1, result, self.failed_style)
> 
>      def __write_cases(self, dut, target, suite):
> -        for case in self.result.all_test_cases(dut, target, suite):
> +        for case in set(self.result.all_test_cases(dut, target, suite)):
> +            result = self.__get_suite_result(dut, target, suite, case)
>              self.col += 1
>              if case[:5] == "test_":
>                  self.sheet.write(self.row, self.col, case[5:])
>              else:
>                  self.sheet.write(self.row, self.col, case)
> -            self.__write_result(dut, target, suite, case)
> +            self.__write_result(dut, target, suite, case, result)
>              self.row += 1
>              self.col -= 1
> 
> --
> 1.9.3

Re: [dts] [PATCH V1 1/3] add re_run parameter

[Xu, HuilongX]

Hi yong,
Thanks  a lot for your comments, I will changed it in next version

> -----Original Message-----
> From: Liu, Yong
> Sent: Saturday, June 24, 2017 12:42 AM
> To: Xu, HuilongX; dts@dpdk.org
> Cc: Xu, HuilongX
> Subject: RE: [dts] [PATCH V1 1/3] add re_run parameter
> 
> Huilong,
> Rerun is not supposed to be the default action for failed cases. The default
> value for rerun times should be zero,
> 
> > -----Original Message-----
> > From: dts [mailto:dts-bounces@dpdk.org] On Behalf Of xu,huilong
> > Sent: Thursday, June 22, 2017 10:57 PM
> > To: dts@dpdk.org
> > Cc: Xu, HuilongX <huilongx.xu@intel.com>
> > Subject: [dts] [PATCH V1 1/3] add re_run parameter
> >
> > add a new re_run parameter for failed case re run time. the default
> > value is 1.
> >
> > Signed-off-by: xu,huilong <huilongx.xu@intel.com>
> > ---
> >  framework/dts.py  | 7 +++++--
> >  framework/main.py | 5 ++++-
> >  2 files changed, 9 insertions(+), 3 deletions(-)
> >
> > diff --git a/framework/dts.py b/framework/dts.py index
> > 369599d..48be694 100644
> > --- a/framework/dts.py
> > +++ b/framework/dts.py
> > @@ -424,7 +424,7 @@ def dts_run_suite(duts, tester, test_suites, target):
> >  def run_all(config_file, pkgName, git, patch, skip_setup,
> >              read_cache, project, suite_dir, test_cases,
> >              base_dir, output_dir, verbose, virttype, debug,
> > -            debugcase, commands):
> > +            debugcase, re_run, commands):
> >      """
> >      Main process of DTS, it will run all test suites in the config file.
> >      """
> > @@ -465,6 +465,9 @@ def run_all(config_file, pkgName, git, patch,
> > skip_setup,
> >      if verbose is True:
> >          logger.set_verbose()
> >
> > +    if re_run < 1:
> > +        re_run = 1
> > +
> >      logger.log_dir = output_dir
> >      log_handler = getLogger('dts')
> >      log_handler.config_execution('dts')
> > @@ -520,7 +523,7 @@ def run_all(config_file, pkgName, git, patch,
> > skip_setup,
> >
> >          # init dut, tester crb
> >          duts, tester = dts_crbs_init(crbInsts, skip_setup,
> > read_cache, project, base_dir, serializer, virttype)
> > -
> > +        tester.set_re_run(re_run)
> >          # register exit action
> >          atexit.register(quit_execution, duts, tester)
> >
> > diff --git a/framework/main.py b/framework/main.py index
> > a018af0..de9d54f 100755
> > --- a/framework/main.py
> > +++ b/framework/main.py
> > @@ -134,6 +134,9 @@ parser.add_argument('--debug',
> > parser.add_argument('--debugcase',
> >                      action='store_true',
> >                      help='enable debug mode in the first case, user
> > can further debug')
> > +parser.add_argument('--re_run',
> > +                    default=1,
> > +                    help='when case failed will re-run times, and
> > +this
> > value must >= 1')
> >
> 
> Suggest add  "type=int" in re-run argument definition.
> 
> >  parser.add_argument('--commands',
> >                      action='append',
> > @@ -156,4 +159,4 @@ dts.run_all(args.config_file, args.snapshot, args.git,
> >              args.patch, args.skip_setup, args.read_cache,
> >              args.project, args.suite_dir, args.test_cases,
> >              args.dir, args.output, args.verbose,args.virttype,
> > -            args.debug, args.debugcase, args.commands)
> > +            args.debug, args.debugcase, args.re_run, args.commands)
> > --
> > 1.9.3

Re: [dts] [PATCH V1 3/3] update case result write function

[Xu, HuilongX]

> -----Original Message-----
> From: Liu, Yong
> Sent: Saturday, June 24, 2017 1:06 AM
> To: Xu, HuilongX; dts@dpdk.org
> Cc: Xu, HuilongX
> Subject: RE: [dts] [PATCH V1 3/3] update case result write function
> 
> Huilong, some comments below.
> 
> > -----Original Message-----
> > From: dts [mailto:dts-bounces@dpdk.org] On Behalf Of xu,huilong
> > Sent: Thursday, June 22, 2017 10:57 PM
> > To: dts@dpdk.org
> > Cc: Xu, HuilongX <huilongx.xu@intel.com>
> > Subject: [dts] [PATCH V1 3/3] update case result write function
> >
> > update case result write function, all same case only have one test
> > result if one time case test result passed, this result will passed
> > and save in excel file.
> > if all times case test result failed, this resulr will failed and save
> > in excel file.
> >
> > Signed-off-by: xu,huilong <huilongx.xu@intel.com>
> > ---
> >  framework/excel_reporter.py | 25 ++++++++++++++++++++-----
> >  1 file changed, 20 insertions(+), 5 deletions(-)
> >
> > diff --git a/framework/excel_reporter.py b/framework/excel_reporter.py
> > index 2030d87..aef50c6 100644
> > --- a/framework/excel_reporter.py
> > +++ b/framework/excel_reporter.py
> > @@ -142,9 +142,23 @@ class ExcelReporter(object):
> >
> >          self.title_style = xlwt.XFStyle()
> >          self.title_style.font = title_font
> > -
> > -    def __write_result(self, dut, target, suite, case):
> > -        test_result = self.result.result_for(dut, target, suite, case)
> > +
> > +    def __get_suite_result(self, dut, target, suite, case):
> 
> 
> Look like function has been change to get result of certain case.  Suggest to
> change name of this function to __get_case_result.

Shure, I will update it in next version
> 
> > +        case_list = self.result.all_test_cases(dut, target, suite)
> > +        if case_list.count(case) > 1:
> > +            tmp_result = []
> > +            for case_name in case_list:
> > +                if case == case_name:
> > +                    test_result = self.result.result_for(dut, target,
> > suite, case)
> > +                    if 'PASSED' in test_result:
> > +                        return ['PASSED', '']
> 
> Why return result list with two items here and only return one is other
> condition?
Yes, because when write test result to excel file, the result must is a list. And result[0] is result, and result[1] is error info.
> 
> > +                    else:
> > +                        tmp_result.append(test_result)
> > +            return tmp_result[-1]
> > +        else:
> > +            return self.result.result_for(dut, target, suite, case)
> > +
> > +    def __write_result(self, dut, target, suite, case, test_result):
> >          if test_result is not None and len(test_result) > 0:
> >              result = test_result[0]
> >              if test_result[1] != '':
> > @@ -156,13 +170,14 @@ class ExcelReporter(object):
> >                      self.row, self.col + 1, result,
> > self.failed_style)
> >
> >      def __write_cases(self, dut, target, suite):
> > -        for case in self.result.all_test_cases(dut, target, suite):
> > +        for case in set(self.result.all_test_cases(dut, target, suite)):
> > +            result = self.__get_suite_result(dut, target, suite,
> > + case)
> >              self.col += 1
> >              if case[:5] == "test_":
> >                  self.sheet.write(self.row, self.col, case[5:])
> >              else:
> >                  self.sheet.write(self.row, self.col, case)
> > -            self.__write_result(dut, target, suite, case)
> > +            self.__write_result(dut, target, suite, case, result)
> >              self.row += 1
> >              self.col -= 1
> >
> > --
> > 1.9.3