[dts] [PATCH V2 0/5] framework: add a proposal of recognizing pkgs

[dts] [PATCH V2 0/5] framework: add a proposal of recognizing pkgs

[Haiyang Zhao]

v2:
	fix exception when get pkg info.
v1:

  Now there are os default/comms/wireless pkgs for CVL NIC,
and they support different protocals. Some suite like l2tp_esp_coverage
are partly support these pkgs, we replace the pkg in suite to support the
case in the past, but this may occur exception and the result may not be we want,
so we provide a new proposal by adding a decorate above test case to check if 
the pkg support current case, and do not replace pkg in suite. 

This patch set is the proposal of framework modification.

Haiyang Zhao (5):
  framework/exception: add new exception VerifySkip
  framework/test_case: handle the VerifySkip exception and add some
    functions
  nics/net_device: add attribute pkg and get method
  framework/dut: get nic package in dut prerequisites
  tests: add nic and pkg check for rss_gtpu

 framework/dut.py                              |  23 +++-
 framework/exception.py                        |  13 +++
 framework/test_case.py                        | 102 +++++++++++++++---
 nics/net_device.py                            |  23 +++-
 tests/TestSuite_cvl_advanced_iavf_rss_gtpu.py |   5 +-
 tests/TestSuite_cvl_advanced_rss_gtpu.py      |   5 +-
 6 files changed, 153 insertions(+), 18 deletions(-)

-- 
2.17.1

[dts] [PATCH V2 1/5] framework/exception: add new exception VerifySkip

[Haiyang Zhao]

this new type exeption is used for framework to handle
the cases which should be skipped.

Signed-off-by: Haiyang Zhao <haiyangx.zhao@intel.com>
---
 framework/exception.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/framework/exception.py b/framework/exception.py
index cc2f724b..bdedd743 100644
--- a/framework/exception.py
+++ b/framework/exception.py
@@ -35,6 +35,19 @@ class VerifyFailure(Exception):
         return repr(self.value)
 
 
+class VerifySkip(Exception):
+
+    """
+    To be used within the test cases to verify if case should be skipped.
+    """
+
+    def __init__(self, value):
+        self.value = value
+
+    def __str__(self):
+        return repr(self.value)
+
+
 class SSHConnectionException(Exception):
 
     """
-- 
2.17.1

[dts] [PATCH V2 2/5] framework/test_case: handle the VerifySkip exception and add some functions

[Haiyang Zhao]

handle the VerfiySkip exception and mark the related case with N/A in result.
add some functions to check if the nic or pkg support current case.

Signed-off-by: Haiyang Zhao <haiyangx.zhao@intel.com>
---
 framework/test_case.py | 102 +++++++++++++++++++++++++++++++++++------
 1 file changed, 89 insertions(+), 13 deletions(-)

diff --git a/framework/test_case.py b/framework/test_case.py
index 57bea562..3347adad 100644
--- a/framework/test_case.py
+++ b/framework/test_case.py
@@ -38,7 +38,7 @@ import traceback
 import signal
 import time
 
-from exception import VerifyFailure, TimeoutException
+from exception import VerifyFailure, VerifySkip, TimeoutException
 from settings import DRIVERS, NICS, get_nic_name, load_global_setting
 from settings import PERF_SETTING, FUNC_SETTING, DEBUG_SETTING
 from settings import DEBUG_CASE_SETTING, HOST_DRIVER_SETTING
@@ -48,6 +48,7 @@ from test_result import ResultTable, Result
 from logger import getLogger
 from config import SuiteConf
 from utils import BLUE, RED
+from functools import wraps
 
 class TestCase(object):
 
@@ -68,15 +69,10 @@ class TestCase(object):
         self._check_and_reconnect(crb=self.tester)
 
         # convert netdevice to codename
-        self.nics = []
-        for portid in range(len(self.dut.ports_info)):
-            nic_type = self.dut.ports_info[portid]['type']
-            self.nics.append(get_nic_name(nic_type))
-        if len(self.nics):
-            self.nic = self.nics[0]
-        else:
-            self.nic = ''
-        self.kdriver = self._get_nic_driver(self.nic)
+        self.nic = self.dut.nic.name
+        self.nic_obj = self.dut.nic
+        self.kdriver = self.dut.nic.default_driver
+        self.pkg = self.dut.nic.pkg
 
         # result object for save suite result
         self._suite_result = Result()
@@ -168,6 +164,12 @@ class TestCase(object):
                 print(RED("History dump finished."))
             raise VerifyFailure(description)
 
+    def skip_case(self, passed, description):
+        if not passed:
+            if self._enable_debug:
+                print("skip case: \"%s\" " % RED(description))
+            raise VerifySkip(description)
+
     def _get_nic_driver(self, nic_name):
         if nic_name in list(DRIVERS.keys()):
             return DRIVERS[nic_name]
@@ -257,17 +259,28 @@ class TestCase(object):
         try:
             self.set_up_all()
             return True
-        except Exception:
+        except VerifySkip as v:
+            self.logger.info('set_up_all SKIPPED:\n' + traceback.format_exc())
+            # record all cases N/A
+            if self._enable_func:
+                for case_obj in self._get_functional_cases():
+                    self._suite_result.test_case = case_obj.__name__
+                    self._suite_result.test_case_skip(str(v))
+            if self._enable_perf:
+                for case_obj in self._get_performance_cases():
+                    self._suite_result.test_case = case_obj.__name__
+                    self._suite_result.test_case_skip(str(v))
+        except Exception as v:
             self.logger.error('set_up_all failed:\n' + traceback.format_exc())
             # record all cases blocked
             if self._enable_func:
                 for case_obj in self._get_functional_cases():
                     self._suite_result.test_case = case_obj.__name__
-                    self._suite_result.test_case_blocked('set_up_all failed')
+                    self._suite_result.test_case_blocked('set_up_all failed: {}'.format(str(v)))
             if self._enable_perf:
                 for case_obj in self._get_performance_cases():
                     self._suite_result.test_case = case_obj.__name__
-                    self._suite_result.test_case_blocked('set_up_all failed')
+                    self._suite_result.test_case_blocked('set_up_all failed: {}'.format(str(v)))
             return False
 
     def _execute_test_case(self, case_obj):
@@ -328,6 +341,10 @@ class TestCase(object):
             self._suite_result.test_case_failed(str(v))
             self._rst_obj.write_result("FAIL")
             self.logger.error('Test Case %s Result FAILED: ' % (case_name) + str(v))
+        except VerifySkip as v:
+            self._suite_result.test_case_skip(str(v))
+            self._rst_obj.write_result("N/A")
+            self.logger.info('Test Case %s N/A: ' % (case_name))
         except KeyboardInterrupt:
             self._suite_result.test_case_blocked("Skipped")
             self.logger.error('Test Case %s SKIPPED: ' % (case_name))
@@ -504,3 +521,62 @@ class TestCase(object):
             bitrate *= 100
 
         return bitrate * num_ports / 8 / (frame_size + 20)
+
+
+def skip_unsupported_pkg(pkgs):
+    """
+    Skip case which are not supported by the input pkgs
+    """
+    if isinstance(pkgs, str):
+        pkgs = [pkgs]
+
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            test_case = args[0]
+            pkg_type = test_case.pkg.get('type')
+            pkg_version = test_case.pkg.get('version')
+            if not pkg_type or not pkg_version:
+                raise VerifyFailure('Failed due to pkg is empty'.format(test_case.pkg))
+            for pkg in pkgs:
+                if pkg in pkg_type:
+                    raise VerifySkip('{} {} do not support this case'.format(pkg_type, pkg_version))
+            return func(*args, **kwargs)
+        return wrapper
+    return decorator
+
+
+def skip_unsupported_nic(nics):
+    """
+    Skip case which are not supported by the input nics
+    """
+    if isinstance(nics, str):
+        nics = [nics]
+
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            test_case = args[0]
+            if test_case.nic in nics:
+                raise VerifySkip('{} do not support this case'.format(test_case.nic))
+            return func(*args, **kwargs)
+        return wrapper
+    return decorator
+
+
+def check_supported_nic(nics):
+    """
+    check if the test case is supported by the input nics
+    """
+    if isinstance(nics, str):
+        nics = [nics]
+
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            test_case = args[0]
+            if test_case.nic not in nics:
+                raise VerifySkip('{} do not support this case'.format(test_case.nic))
+            return func(*args, **kwargs)
+        return wrapper
+    return decorator
-- 
2.17.1

[dts] [PATCH V2 3/5] nics/net_device: add attribute pkg and get method

[Haiyang Zhao]

add attribute pkg to record nic current package and add the related get method.

Signed-off-by: Haiyang Zhao <haiyangx.zhao@intel.com>
---
 nics/net_device.py | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/nics/net_device.py b/nics/net_device.py
index bdc9d37d..6a26d1eb 100644
--- a/nics/net_device.py
+++ b/nics/net_device.py
@@ -71,6 +71,7 @@ class NetDevice(object):
         self.intf2_name = None
         self.get_interface_name()
         self.socket = self.get_nic_socket()
+        self.pkg = {}
 
     def stop(self):
         pass
@@ -118,6 +119,27 @@ class NetDevice(object):
         """
         return self.crb.get_pci_dev_driver(self.domain_id, self.bus_id, self.devfun_id)
 
+    def get_nic_pkg(self):
+        """
+        Get the NIC pkg.
+        """
+        out = self.__send_expect('dmesg | grep "DDP package" | tail -1', '# ')
+        if 'could not load' in out:
+            print(RED(out))
+            print(RED('Warning: The loaded DDP package version may not as you expected'))
+            try:
+                pkg_info = out.split('. ')[1].lower()
+                self.pkg['type'] = re.findall(".*package '(.*)'", pkg_info)[0].strip()
+                self.pkg['version'] = re.findall("version(.*)", pkg_info)[0].strip()
+            except:
+                print(RED('Warning: get pkg info failed'))
+        else:
+            pkg_info = out.split(': ')[-1].lower().split('package version')
+            if len(pkg_info) > 1:
+                self.pkg['type'] = pkg_info[0].strip()
+                self.pkg['version'] = pkg_info[1].strip()
+        return self.pkg
+
     def get_nic_socket(self):
         """
         Get socket id of specified pci device.
@@ -428,7 +450,6 @@ class NetDevice(object):
         self.__send_expect("ifconfig %s down" % intf, "# ")
         self.__send_expect("ifconfig %s up" % intf, "# ")
 
-
     @nic_has_driver
     def disable_ipv6(self):
         """
-- 
2.17.1

[dts] [PATCH V2 4/5] framework/dut: get nic package in dut prerequisites

[Haiyang Zhao]

get nic package if nic kernel driver is ice, and it will retry three times
in case of get package failed.

Signed-off-by: Haiyang Zhao <haiyangx.zhao@intel.com>
---
 framework/dut.py | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/framework/dut.py b/framework/dut.py
index 113116f2..5bc84fca 100644
--- a/framework/dut.py
+++ b/framework/dut.py
@@ -404,11 +404,12 @@ class Dut(Crb):
         self.map_available_ports()
         # disable tester port ipv6
         self.disable_tester_ipv6()
+        self.get_nic_configurations()
+
         # print latest ports_info
         for port_info in self.ports_info:
             self.logger.info(port_info)
-            if self.nic is None:
-                self.nic = port_info['port']
+
         if self.ports_map is None or len(self.ports_map) == 0:
             self.logger.warning("ports_map should not be empty, please check all links")
 
@@ -419,6 +420,24 @@ class Dut(Crb):
         name_cfg = AppNameConf()
         self.apps_name_conf = name_cfg.load_app_name_conf()
 
+    def get_nic_configurations(self):
+        retry_times = 3
+        if self.ports_info:
+            self.nic = self.ports_info[0]['port']
+            # TODO: get nic driver/firmware version
+            if self.nic.default_driver == 'ice':
+                self.get_nic_pkg(retry_times)
+
+    def get_nic_pkg(self, retry_times=3):
+        self.nic.pkg = self.nic.get_nic_pkg()
+        while not self.nic.pkg and retry_times > 0:
+            self.restore_interfaces()
+            self.nic.pkg = self.nic.get_nic_pkg()
+            retry_times = retry_times - 1
+        self.logger.info('pkg: {}'.format(self.nic.pkg))
+        if not self.nic.pkg:
+            raise Exception('Get nic pkg failed')
+
     def restore_interfaces(self):
         """
         Restore all ports's interfaces.
-- 
2.17.1

[dts] [PATCH V2 5/5] tests: add nic and pkg check for rss_gtpu

[Haiyang Zhao]

add decorate in set_up_all to check if nic and pkg
support current suite, if not, all the cases will be
marked as N/A.

Signed-off-by: Haiyang Zhao <haiyangx.zhao@intel.com>
---
 tests/TestSuite_cvl_advanced_iavf_rss_gtpu.py | 5 ++++-
 tests/TestSuite_cvl_advanced_rss_gtpu.py      | 5 ++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/tests/TestSuite_cvl_advanced_iavf_rss_gtpu.py b/tests/TestSuite_cvl_advanced_iavf_rss_gtpu.py
index 93e1309b..6490dcbf 100755
--- a/tests/TestSuite_cvl_advanced_iavf_rss_gtpu.py
+++ b/tests/TestSuite_cvl_advanced_iavf_rss_gtpu.py
@@ -35,7 +35,7 @@ import random
 import time
 from packet import Packet
 from pmd_output import PmdOutput
-from test_case import TestCase
+from test_case import TestCase, skip_unsupported_pkg, check_supported_nic
 from rte_flow_common import RssProcessing
 
 mac_ipv4_gtpu_ipv4_basic = {
@@ -7807,7 +7807,10 @@ mac_ipv6_gtpc_symmetric_toeplitz = [mac_ipv6_gtpc_symmetric]
 
 
 class TestCVLAdvancedIAVFRSSGTPU(TestCase):
+    supported_nic = ['columbiaville_100g', 'columbiaville_25g', 'columbiaville_25gx2']
 
+    @check_supported_nic(supported_nic)
+    @skip_unsupported_pkg('os default')
     def set_up_all(self):
         """
         Run at the start of each test suite.
diff --git a/tests/TestSuite_cvl_advanced_rss_gtpu.py b/tests/TestSuite_cvl_advanced_rss_gtpu.py
index df514198..054f1907 100755
--- a/tests/TestSuite_cvl_advanced_rss_gtpu.py
+++ b/tests/TestSuite_cvl_advanced_rss_gtpu.py
@@ -34,7 +34,7 @@ import re
 import time
 from packet import Packet
 from pmd_output import PmdOutput
-from test_case import TestCase
+from test_case import TestCase, skip_unsupported_pkg, check_supported_nic
 from rte_flow_common import RssProcessing
 
 
@@ -5555,7 +5555,10 @@ mac_ipv4_gtpu_eh_ipv6_tcp_without_ul_dl_symmetric = eval(str(mac_ipv4_gtpu_eh_ip
 
 
 class TestCVLAdvancedRSSGTPU(TestCase):
+    supported_nic = ['columbiaville_100g', 'columbiaville_25g', 'columbiaville_25gx2']
 
+    @check_supported_nic(supported_nic)
+    @skip_unsupported_pkg('os default')
     def set_up_all(self):
         """
         Run at the start of each test suite.
-- 
2.17.1

Re: [dts] [PATCH V2 0/5] framework: add a proposal of recognizing pkgs

[Tu, Lijuan]

> v2:
> 	fix exception when get pkg info.
> v1:
> 
>   Now there are os default/comms/wireless pkgs for CVL NIC, and they support
> different protocals. Some suite like l2tp_esp_coverage are partly support these
> pkgs, we replace the pkg in suite to support the case in the past, but this may
> occur exception and the result may not be we want, so we provide a new
> proposal by adding a decorate above test case to check if the pkg support
> current case, and do not replace pkg in suite.
> 
> This patch set is the proposal of framework modification.
> 
> Haiyang Zhao (5):
>   framework/exception: add new exception VerifySkip
>   framework/test_case: handle the VerifySkip exception and add some
>     functions
>   nics/net_device: add attribute pkg and get method
>   framework/dut: get nic package in dut prerequisites
>   tests: add nic and pkg check for rss_gtpu

Applied, thanks