[dts] [PATCH 0/4] enhance framework and fix some issues

[dts] [PATCH 0/4] enhance framework and fix some issues

[Marvin Liu]

This patch set based on next branch and enhanced framework stability.
It also fixed cache option not work issue and some tope issues in suite.

Marvin Liu (4):
  framework dts: fix exit when exception raised
  framework: fix cache option not work
  framework: add session check function
  tests: fix tope issue in suites

 framework/dts.py                    | 39 ++++++++++++++++++++-----------------
 framework/dut.py                    | 24 +++++++++++++++--------
 framework/ssh_connection.py         | 13 +++++++++++++
 framework/test_case.py              | 21 +++++++++++++-------
 framework/tester.py                 |  7 ++++++-
 tests/TestSuite_checksum_offload.py | 14 ++++++-------
 tests/TestSuite_ipv4_reassembly.py  |  6 +++---
 tests/TestSuite_pmd_bonded.py       |  8 ++------
 8 files changed, 81 insertions(+), 51 deletions(-)

-- 
1.9.3

[dts] [PATCH 1/4] framework dts: fix program exit when exception raised

[Marvin Liu]

After handled test suite exception, dts should continue to next suite.
Remove global configuration file object, it will be passed as parameter.
When exit from suite, should call tear_down_all to stop everything.

Signed-off-by: Marvin Liu <yong.liu@intel.com>

diff --git a/framework/dts.py b/framework/dts.py
index 6077a04..e9879e5 100644
--- a/framework/dts.py
+++ b/framework/dts.py
@@ -64,7 +64,6 @@ reload(sys)
 sys.setdefaultencoding('UTF8')
 
 
-config = None
 requested_tests = None
 result = None
 excel_report = None
@@ -72,7 +71,7 @@ stats_report = None
 log_handler = None
 
 
-def dts_parse_param(section):
+def dts_parse_param(config, section):
     """
     Parse execution file parameters.
     """
@@ -112,7 +111,7 @@ def dts_parse_param(section):
         settings.save_global_setting(settings.FUNC_SETTING, 'no')
 
 
-def dts_parse_config(section):
+def dts_parse_config(config, section):
     """
     Parse execution file configuration.
     """
@@ -369,8 +368,8 @@ def dts_run_suite(duts, tester, test_suites, target):
     """
     Run each suite in test suite list.
     """
-    try:
-        for suite_name in test_suites:
+    for suite_name in test_suites:
+        try:
             result.test_suite = suite_name
             suite_module = __import__('TestSuite_' + suite_name)
             for test_classname, test_class in get_subclasses(suite_module, TestCase):
@@ -395,14 +394,19 @@ def dts_run_suite(duts, tester, test_suites, target):
 
                 log_handler.info("\nTEST SUITE ENDED: " + test_classname)
                 dts_log_execution(duts, tester, log_handler)
-    except VerifyFailure:
-        log_handler.error(" !!! DEBUG IT: " + traceback.format_exc())
-    except KeyboardInterrupt:
-        log_handler.error(" !!! STOPPING DCTS")
-    except Exception as e:
-        log_handler.error(str(e))
-    finally:
-        suite_obj.execute_tear_downall()
+        except VerifyFailure:
+            log_handler.error(" !!! DEBUG IT: " + traceback.format_exc())
+        except KeyboardInterrupt:
+            # stop/save result/skip execution
+            log_handler.error(" !!! STOPPING DTS")
+            suite_obj.execute_tear_downall()
+            save_all_results()
+            break
+        except Exception as e:
+            log_handler.error(str(e))
+        finally:
+            suite_obj.execute_tear_downall()
+            save_all_results()
 
 
 def run_all(config_file, pkgName, git, patch, skip_setup,
@@ -413,7 +417,6 @@ def run_all(config_file, pkgName, git, patch, skip_setup,
     Main process of DTS, it will run all test suites in the config file.
     """
 
-    global config
     global requested_tests
     global result
     global excel_report
@@ -442,9 +445,9 @@ def run_all(config_file, pkgName, git, patch, skip_setup,
 
     # enable debug mode
     if debug is True:
-        setting.set_local_variable(settings.DEBUG_SETTING, 'yes')
+        settings.save_global_setting(settings.DEBUG_SETTING, 'yes')
     if debugcase is True:
-        setting.set_local_variable(settings.DEBUG_CASE_SETTING, 'yes')
+        settings.save_global_setting(settings.DEBUG_CASE_SETTING, 'yes')
 
     # init log_handler handler
     if verbose is True:
@@ -482,10 +485,10 @@ def run_all(config_file, pkgName, git, patch, skip_setup,
 
     # for all Exectuion sections
     for section in config.sections():
-        dts_parse_param(section)
+        dts_parse_param(config, section)
 
         # verify if the delimiter is good if the lists are vertical
-        dutIPs, targets, test_suites = dts_parse_config(section)
+        dutIPs, targets, test_suites = dts_parse_config(config, section)
         for dutIP in dutIPs:
             log_handler.info("\nDUT " + dutIP)
 
-- 
1.9.3

[dts] [PATCH 2/4] framework: fix cache option not work

[Marvin Liu]

When cache option enabled, port information retrieved from cache file.
Netdevice object should be initialized based on port infor.

Signed-off-by: Marvin Liu <yong.liu@intel.com>

diff --git a/framework/dut.py b/framework/dut.py
index 88c4679..f76b4f4 100644
--- a/framework/dut.py
+++ b/framework/dut.py
@@ -213,6 +213,10 @@ class Dut(Crb):
         """
         Restore all ports's interfaces.
         """
+        # no need to restore for all info has been recorded
+        if self.read_cache:
+            return
+
         restore_interfaces = getattr(self, 'restore_interfaces_%s' % self.get_os_type())
         return restore_interfaces()
 
@@ -546,7 +550,7 @@ class Dut(Crb):
         """
         Rescan ports information
         """
-        if self.read_cache and self.load_serializer_ports():
+        if self.read_cache:
             return
 
         if self.ports_info:
@@ -565,12 +569,13 @@ class Dut(Crb):
 
         for port_info in self.ports_info:
             port = port_info['port']
-            intf = port_info['intf']
+            intf = port.get_interface_name()
+            port_info['intf'] = intf
             out = self.send_expect("ip link show %s" % intf, "# ")
             if "DOWN" in out:
                 self.send_expect("ip link set %s up" % intf, "# ")
                 time.sleep(5)
-            macaddr = port_info['mac']
+            port_info['mac'] = port.get_mac_addr()
             out = self.send_expect("ip -family inet6 address show dev %s | awk '/inet6/ { print $2 }'"
                                    % intf, "# ")
             ipv6 = out.split('/')[0]
@@ -644,14 +649,17 @@ class Dut(Crb):
             return
 
         for port_info in self.ports_info:
-            port = GetNicObj(self, port_info['pci'], port_info['type'])
-            intf = port.get_interface_name()
-
-            self.logger.info("DUT cached: [%s %s] %s" % (port_info['pci'],
-                                                         port_info['type'], intf))
+            addr_array = port_info['pci'].split(':')
+            domain_id = addr_array[0]
+            bus_id = addr_array[1]
+            devfun_id = addr_array[2]
 
+            port = GetNicObj(self, domain_id, bus_id, devfun_id)
             port_info['port'] = port
 
+            self.logger.info("DUT cached: [%s %s] %s" % (port_info['pci'],
+                                port_info['type'], port_info['intf']))
+
     def scan_ports_uncached(self):
         """
         Scan ports and collect port's pci id, mac adress, ipv6 address.
diff --git a/framework/tester.py b/framework/tester.py
index cadc3ad..962aaf8 100644
--- a/framework/tester.py
+++ b/framework/tester.py
@@ -293,7 +293,12 @@ class Tester(Crb):
             if port_info['type'] == 'ixia':
                 continue
 
-            port = GetNicObj(self, port_info['pci'], port_info['type'])
+            addr_array = port_info['pci'].split(':')
+            domain_id = addr_array[0]
+            bus_id = addr_array[1]
+            devfun_id = addr_array[2]
+
+            port = GetNicObj(self, domain_id, bus_id, devfun_id)
             intf = port.get_interface_name()
 
             self.logger.info("Tester cached: [000:%s %s] %s" % (
-- 
1.9.3

[dts] [PATCH 3/4] framework: add session check function

[Marvin Liu]

Before running any case, suite module should check dut/tester sessions.
When sessions are not avaiable, raise exception to dts.

Signed-off-by: Marvin Liu <yong.liu@intel.com>

diff --git a/framework/ssh_connection.py b/framework/ssh_connection.py
index edb8170..38ab36d 100644
--- a/framework/ssh_connection.py
+++ b/framework/ssh_connection.py
@@ -84,6 +84,19 @@ class SSHConnection(object):
     def isalive(self):
         return self.session.isalive()
 
+    def check_available(self):
+        MAGIC_STR = "DTS_CHECK_SESSION"
+        out = self.session.send_command('echo %s' % MAGIC_STR, timeout=0.1)
+        # if not avaiable, try to send ^C and check again
+        if MAGIC_STR not in out:
+            self.logger.info("Try to recover session...")
+            self.session.send_command('^C', timeout=TIMEOUT)
+            out = self.session.send_command('echo %s' % MAGIC_STR, timeout=0.1)
+            if MAGIC_STR not in out:
+                return False
+
+        return True
+
     def copy_file_from(self, src, dst=".", password=''):
         self.session.copy_file_from(src, dst, password)
 
diff --git a/framework/test_case.py b/framework/test_case.py
index b603b48..7ace75c 100644
--- a/framework/test_case.py
+++ b/framework/test_case.py
@@ -54,9 +54,15 @@ class TestCase(object):
         self.tester = tester
         self.target = target
 
+        # make sure session workable
+        for dutobj in duts:
+            self.verify(dutobj.session.check_available(), "DUT session can't work")
+            self.verify(dutobj.alt_session.check_available(), "DUT alt_session can't work")
+        self.verify(tester.session.check_available(), "Tester session can't work!!!")
+        self.verify(tester.alt_session.check_available(), "Tester alt_session can't work!!!")
+
         # get log handler
         class_name = self.__class__.__name__
-        print class_name
         self.logger = getLogger(class_name)
         self.logger.config_suite(class_name)
         # local variable
@@ -138,7 +144,7 @@ class TestCase(object):
         self._rst_obj.report(*args, **kwargs)
 
     def result_table_create(self, header):
-        self._result_table = ResultTable(self.table_header)
+        self._result_table = ResultTable(header)
         self._result_table.set_rst(self._rst_obj)
         self._result_table.set_logger(self.logger)
 
@@ -155,7 +161,7 @@ class TestCase(object):
         """
         Get all functional test cases.
         """
-        return self._get_test_cases(self, r'test_(?!perf_)')
+        return self._get_test_cases(r'test_(?!perf_)')
 
     def _get_performance_cases(self):
         """
@@ -228,14 +234,14 @@ class TestCase(object):
             if self._check_inst.case_skip(case_name[len("test_"):]):
                 self.logger.info('Test Case %s Result SKIPED:' % case_name)
                 self._rst_obj.write_result("N/A")
-                self._suite_result.test_case_skip(check_case_inst.comments)
+                self._suite_result.test_case_skip(self._check_inst.comments)
                 return
 
         if self._support_inst is not None:
             if not self._support_inst.case_support(case_name[len("test_"):]):
                 self.logger.info('Test Case %s Result SKIPED:' % case_name)
                 self._rst_obj.write_result("N/A")
-                self._suite_result.test_case_skip(support_case_inst.comments)
+                self._suite_result.test_case_skip(self._support_inst.comments)
                 return
 
         if self._enable_perf:
@@ -267,12 +273,13 @@ class TestCase(object):
 
         except VerifyFailure as v:
             self._suite_result.test_case_failed(str(v))
-            self_rst_obj.write_result("FAIL")
+            self._rst_obj.write_result("FAIL")
             self.logger.error('Test Case %s Result FAILED: ' % (case_name) + str(v))
         except KeyboardInterrupt:
             self._suite_result.test_case_blocked("Skipped")
             self.logger.error('Test Case %s SKIPED: ' % (case_name))
-            raise KeyboardInterrupt("Stop DCTS")
+            self.tear_down()
+            raise KeyboardInterrupt("Stop DTS")
         except TimeoutException as e:
             self._rst_obj.write_result("FAIL")
             msg = str(e)
-- 
1.9.3

[dts] [PATCH 4/4] tests: fix tope issue in suites

[Marvin Liu]

Signed-off-by: Marvin Liu <yong.liu@intel.com>

diff --git a/tests/TestSuite_checksum_offload.py b/tests/TestSuite_checksum_offload.py
index 7a05b6d..57069f0 100644
--- a/tests/TestSuite_checksum_offload.py
+++ b/tests/TestSuite_checksum_offload.py
@@ -63,12 +63,11 @@ class TestChecksumOffload(TestCase):
         """
         Run before each test case.
         """
-        if self.dut.want_func_tests:
-            self.pmdout.start_testpmd("Default", "--portmask=%s " %
-                                      (self.portMask) + "--disable-hw-vlan --enable-rx-cksum " +
-                                      "--crc-strip --port-topology=loop", socket=self.ports_socket)
-            self.dut.send_expect("set verbose 1", "testpmd>")
-            self.dut.send_expect("set fwd csum", "testpmd>")
+        self.pmdout.start_testpmd("Default", "--portmask=%s " %
+                                  (self.portMask) + "--disable-hw-vlan --enable-rx-cksum " +
+                                  "--crc-strip --port-topology=loop", socket=self.ports_socket)
+        self.dut.send_expect("set verbose 1", "testpmd>")
+        self.dut.send_expect("set fwd csum", "testpmd>")
 
     def checksum_enablehw(self, port):
             self.dut.send_expect("csum set ip hw %d" % port, "testpmd>")
@@ -327,8 +326,7 @@ class TestChecksumOffload(TestCase):
         """
         Run after each test case.
         """
-        if self.dut.want_func_tests:
-            self.dut.send_expect("quit", "#")
+        self.dut.send_expect("quit", "#")
 
     def tear_down_all(self):
         """
diff --git a/tests/TestSuite_ipv4_reassembly.py b/tests/TestSuite_ipv4_reassembly.py
index a1e15a2..3f0e725 100644
--- a/tests/TestSuite_ipv4_reassembly.py
+++ b/tests/TestSuite_ipv4_reassembly.py
@@ -305,7 +305,7 @@ class TestIpReassembly(TestCase):
         """
 
         sent_packets = self.number_of_sent_packets(self.test_config.mac_src)
-        print 'sent packets: %d - expected: %d' % (sent_packets, expected)
+        self.logger.info('sent packets: %d - expected: %d' % (sent_packets, expected))
         self.verify(sent_packets == expected, 'Not all fragments have been sent')
 
     def verify_received_packets(self, expected):
@@ -314,7 +314,7 @@ class TestIpReassembly(TestCase):
         """
 
         received_packets = self.number_of_received_packets(self.test_config.tcp_dst_port)
-        print 'received packets: %d - expected: %d' % (received_packets, expected)
+        self.logger.info('received packets: %d - expected: %d' % (received_packets, expected))
         self.verify(received_packets == expected,
                     'Not all frames have been forwarded')
 
@@ -324,7 +324,7 @@ class TestIpReassembly(TestCase):
         """
 
         tcp_valid_checksum = self.number_of_tcp_valid_checksum(self.test_config.tcp_dst_port)
-        print 'tcp valid: %d - expected: %d' % (tcp_valid_checksum, expected)
+        self.logger.info('tcp valid: %d - expected: %d' % (tcp_valid_checksum, expected))
         self.verify(tcp_valid_checksum == expected,
                     'Not all TCP packets have valid checksum')
 
diff --git a/tests/TestSuite_pmd_bonded.py b/tests/TestSuite_pmd_bonded.py
index 8b8b8ee..1b3f781 100644
--- a/tests/TestSuite_pmd_bonded.py
+++ b/tests/TestSuite_pmd_bonded.py
@@ -810,16 +810,12 @@ UDP(sport=srcport, dport=destport)/Raw(load="\x50"*%s)], iface="%s", count=%d)'
         """
         Run before each test case.
         """
-        if self.dut.want_func_tests:
-            self.launch_app()
-        elif self.dut.want_perf_tests:
+        if self._enable_perf:
             pmd_param = "--burst=32 --rxfreet=32 --mbcache=250 --txpt=32 \
 --rxht=8 --rxwt=0 --txfreet=32 --txrst=32 --txqflags=0xf01"
             self.launch_app(pmd_param)
         else:
-            self.verify(False,
-                        "Test type not etting," +
-                        "please check framework to set test type to be function or performance.")
+            self.launch_app()
 
     def verify_bound_basic_opt(self, mode_set):
         """
-- 
1.9.3