[dts] [PATCH] framework: support reconnect session when unavailable

[Marvin Liu <yong.liu@intel.com>]

Suite will check session status before running any case.
When found session is not available for useage, will create new one for
replace.

Signed-off-by: Marvin Liu <yong.liu@intel.com>

diff --git a/framework/crb.py b/framework/crb.py
index 44ddc42..bc8b24a 100644
--- a/framework/crb.py
+++ b/framework/crb.py
@@ -57,6 +57,7 @@ class Crb(object):
         self.ports_info = None
         self.sessions = []
         self.stage = 'pre-init'
+        self.name = name
 
         self.logger = getLogger(name)
         self.session = SSHConnection(self.get_ip_address(), name,
@@ -88,7 +89,7 @@ class Crb(object):
         """
         logger = getLogger(name)
         session = SSHConnection(self.get_ip_address(), name,
-                                     self.get_password())
+                                self.get_password())
         session.init_log(logger)
         self.sessions.append(session)
         return session
@@ -104,6 +105,31 @@ class Crb(object):
                 logger.logger_exit()
             self.sessions.remove(save_session)
 
+    def reconnect_session(self, alt_session=False):
+        """
+        When session can't used anymore, recreate another one for replace
+        """
+        try:
+            if alt_session:
+                self.alt_session.close(force=True)
+            else:
+                self.session.close(force=True)
+        except Exception as e:
+            self.loggger.error("Session close failed for [%s]" % e)
+
+        if alt_session:
+            session = SSHConnection(
+                self.get_ip_address(),
+                self.name + '_alt',
+                self.get_password())
+            self.alt_session = session
+        else:
+            session = SSHConnection(self.get_ip_address(), self.name,
+                                    self.get_password())
+            self.session = session
+
+        session.init_log(self.logger)
+
     def send_command(self, cmds, timeout=TIMEOUT, alt_session=False):
         """
         Send commands to crb and return string before timeout.
@@ -166,10 +192,9 @@ class Crb(object):
         if numa == -1:
             self.send_expect('echo %d > /sys/kernel/mm/hugepages/hugepages-%skB/nr_hugepages' % (huge_pages, page_size), '# ', 5)
         else:
-            #sometimes we set hugepage on kernel cmdline, so we need clear default hugepage
+            # sometimes we set hugepage on kernel cmdline, so we need clear default hugepage
             self.send_expect('echo 0 > /sys/kernel/mm/hugepages/hugepages-%skB/nr_hugepages' % (page_size), '# ', 5)
-            
-            #some platform not support numa, example vm dut
+            # some platform not support numa, example vm dut
             try:
                 self.send_expect('echo %d > /sys/devices/system/node/node%d/hugepages/hugepages-%skB/nr_hugepages' % (huge_pages, numa, page_size), '# ', 5)
             except:
@@ -687,7 +712,7 @@ class Crb(object):
         for port_info in self.ports_info:
             if port_info['pci'] == pci:
                 return port_info
-    
-    def enable_promisc(self,intf):
-        if intf !='N/A':
-            self.send_expect("ifconfig %s promisc" %intf, "# ",alt_session=True)
+
+    def enable_promisc(self, intf):
+        if intf != 'N/A':
+            self.send_expect("ifconfig %s promisc" % intf, "# ", alt_session=True)
diff --git a/framework/test_case.py b/framework/test_case.py
index 7ace75c..85fdb70 100644
--- a/framework/test_case.py
+++ b/framework/test_case.py
@@ -54,13 +54,6 @@ class TestCase(object):
         self.tester = tester
         self.target = target
 
-        # make sure session workable
-        for dutobj in duts:
-            self.verify(dutobj.session.check_available(), "DUT session can't work")
-            self.verify(dutobj.alt_session.check_available(), "DUT alt_session can't work")
-        self.verify(tester.session.check_available(), "Tester session can't work!!!")
-        self.verify(tester.alt_session.check_available(), "Tester alt_session can't work!!!")
-
         # get log handler
         class_name = self.__class__.__name__
         self.logger = getLogger(class_name)
@@ -68,6 +61,11 @@ class TestCase(object):
         # local variable
         self._requested_tests = None
 
+        # check session and reconnect if possible
+        for dutobj in self.duts:
+            self._check_and_reconnect(crb=dutobj)
+        self._check_and_reconnect(crb=self.tester)
+
         # covert netdevice to codename
         self.nics = []
         for portid in range(len(self.dut.ports_info)):
@@ -114,6 +112,23 @@ class TestCase(object):
         # create rst format report for this suite
         self._rst_obj = RstReport('rst_report', target, self.nic, self.suite_name, self._enable_perf)
 
+    def _check_and_reconnect(self, crb=None):
+        try:
+            result = crb.session.check_available()
+        except:
+            result = False
+
+        if result is False:
+            crb.reconnect_session()
+
+        try:
+            result = crb.alt_session.check_available()
+        except:
+            result = False
+
+        if result is False:
+            crb.reconnect_session(alt_session=True)
+
     def set_up_all(self):
         pass
 
-- 
1.9.3