test suite reviews and discussions
 help / color / mirror / Atom feed
From: Haiyang Zhao <haiyangx.zhao@intel.com>
To: dts@dpdk.org
Cc: Lijuan.Tu@intel.com, Haiyang Zhao <haiyangx.zhao@intel.com>
Subject: [dts] [PATCH V1 2/5] framework/test_case: handle the VerifySkip exception and add some functions
Date: Wed, 17 Mar 2021 15:16:22 +0800
Message-ID: <20210317071625.13041-3-haiyangx.zhao@intel.com> (raw)
In-Reply-To: <20210317071625.13041-1-haiyangx.zhao@intel.com>

handle the VerfiySkip exception and mark the related case with N/A in result.
add some functions to check if the nic or pkg support current case.

Signed-off-by: Haiyang Zhao <haiyangx.zhao@intel.com>
---
 framework/test_case.py | 102 +++++++++++++++++++++++++++++++++++------
 1 file changed, 89 insertions(+), 13 deletions(-)

diff --git a/framework/test_case.py b/framework/test_case.py
index 57bea562..3347adad 100644
--- a/framework/test_case.py
+++ b/framework/test_case.py
@@ -38,7 +38,7 @@ import traceback
 import signal
 import time
 
-from exception import VerifyFailure, TimeoutException
+from exception import VerifyFailure, VerifySkip, TimeoutException
 from settings import DRIVERS, NICS, get_nic_name, load_global_setting
 from settings import PERF_SETTING, FUNC_SETTING, DEBUG_SETTING
 from settings import DEBUG_CASE_SETTING, HOST_DRIVER_SETTING
@@ -48,6 +48,7 @@ from test_result import ResultTable, Result
 from logger import getLogger
 from config import SuiteConf
 from utils import BLUE, RED
+from functools import wraps
 
 class TestCase(object):
 
@@ -68,15 +69,10 @@ class TestCase(object):
         self._check_and_reconnect(crb=self.tester)
 
         # convert netdevice to codename
-        self.nics = []
-        for portid in range(len(self.dut.ports_info)):
-            nic_type = self.dut.ports_info[portid]['type']
-            self.nics.append(get_nic_name(nic_type))
-        if len(self.nics):
-            self.nic = self.nics[0]
-        else:
-            self.nic = ''
-        self.kdriver = self._get_nic_driver(self.nic)
+        self.nic = self.dut.nic.name
+        self.nic_obj = self.dut.nic
+        self.kdriver = self.dut.nic.default_driver
+        self.pkg = self.dut.nic.pkg
 
         # result object for save suite result
         self._suite_result = Result()
@@ -168,6 +164,12 @@ class TestCase(object):
                 print(RED("History dump finished."))
             raise VerifyFailure(description)
 
+    def skip_case(self, passed, description):
+        if not passed:
+            if self._enable_debug:
+                print("skip case: \"%s\" " % RED(description))
+            raise VerifySkip(description)
+
     def _get_nic_driver(self, nic_name):
         if nic_name in list(DRIVERS.keys()):
             return DRIVERS[nic_name]
@@ -257,17 +259,28 @@ class TestCase(object):
         try:
             self.set_up_all()
             return True
-        except Exception:
+        except VerifySkip as v:
+            self.logger.info('set_up_all SKIPPED:\n' + traceback.format_exc())
+            # record all cases N/A
+            if self._enable_func:
+                for case_obj in self._get_functional_cases():
+                    self._suite_result.test_case = case_obj.__name__
+                    self._suite_result.test_case_skip(str(v))
+            if self._enable_perf:
+                for case_obj in self._get_performance_cases():
+                    self._suite_result.test_case = case_obj.__name__
+                    self._suite_result.test_case_skip(str(v))
+        except Exception as v:
             self.logger.error('set_up_all failed:\n' + traceback.format_exc())
             # record all cases blocked
             if self._enable_func:
                 for case_obj in self._get_functional_cases():
                     self._suite_result.test_case = case_obj.__name__
-                    self._suite_result.test_case_blocked('set_up_all failed')
+                    self._suite_result.test_case_blocked('set_up_all failed: {}'.format(str(v)))
             if self._enable_perf:
                 for case_obj in self._get_performance_cases():
                     self._suite_result.test_case = case_obj.__name__
-                    self._suite_result.test_case_blocked('set_up_all failed')
+                    self._suite_result.test_case_blocked('set_up_all failed: {}'.format(str(v)))
             return False
 
     def _execute_test_case(self, case_obj):
@@ -328,6 +341,10 @@ class TestCase(object):
             self._suite_result.test_case_failed(str(v))
             self._rst_obj.write_result("FAIL")
             self.logger.error('Test Case %s Result FAILED: ' % (case_name) + str(v))
+        except VerifySkip as v:
+            self._suite_result.test_case_skip(str(v))
+            self._rst_obj.write_result("N/A")
+            self.logger.info('Test Case %s N/A: ' % (case_name))
         except KeyboardInterrupt:
             self._suite_result.test_case_blocked("Skipped")
             self.logger.error('Test Case %s SKIPPED: ' % (case_name))
@@ -504,3 +521,62 @@ class TestCase(object):
             bitrate *= 100
 
         return bitrate * num_ports / 8 / (frame_size + 20)
+
+
+def skip_unsupported_pkg(pkgs):
+    """
+    Skip case which are not supported by the input pkgs
+    """
+    if isinstance(pkgs, str):
+        pkgs = [pkgs]
+
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            test_case = args[0]
+            pkg_type = test_case.pkg.get('type')
+            pkg_version = test_case.pkg.get('version')
+            if not pkg_type or not pkg_version:
+                raise VerifyFailure('Failed due to pkg is empty'.format(test_case.pkg))
+            for pkg in pkgs:
+                if pkg in pkg_type:
+                    raise VerifySkip('{} {} do not support this case'.format(pkg_type, pkg_version))
+            return func(*args, **kwargs)
+        return wrapper
+    return decorator
+
+
+def skip_unsupported_nic(nics):
+    """
+    Skip case which are not supported by the input nics
+    """
+    if isinstance(nics, str):
+        nics = [nics]
+
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            test_case = args[0]
+            if test_case.nic in nics:
+                raise VerifySkip('{} do not support this case'.format(test_case.nic))
+            return func(*args, **kwargs)
+        return wrapper
+    return decorator
+
+
+def check_supported_nic(nics):
+    """
+    check if the test case is supported by the input nics
+    """
+    if isinstance(nics, str):
+        nics = [nics]
+
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            test_case = args[0]
+            if test_case.nic not in nics:
+                raise VerifySkip('{} do not support this case'.format(test_case.nic))
+            return func(*args, **kwargs)
+        return wrapper
+    return decorator
-- 
2.17.1


  parent reply	other threads:[~2021-03-17  7:25 UTC|newest]

Thread overview: 7+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2021-03-17  7:16 [dts] [PATCH V1 0/5] framework: add a proposal of recognizing pkgs Haiyang Zhao
2021-03-17  7:16 ` [dts] [PATCH V1 1/5] framework/exception: add new exception VerifySkip Haiyang Zhao
2021-03-17  7:16 ` Haiyang Zhao [this message]
2021-03-17  7:16 ` [dts] [PATCH V1 3/5] nics/net_device: add attribute pkg and get method Haiyang Zhao
2021-03-17  7:16 ` [dts] [PATCH V1 4/5] framework/dut: get nic package in dut prerequisites Haiyang Zhao
2021-03-17  7:16 ` [dts] [PATCH V1 5/5] tests: add nic and pkg check for rss_gtpu Haiyang Zhao
2021-03-17  7:30 ` [dts] [PATCH V1 0/5] framework: add a proposal of recognizing pkgs Zhao, HaiyangX

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20210317071625.13041-3-haiyangx.zhao@intel.com \
    --to=haiyangx.zhao@intel.com \
    --cc=Lijuan.Tu@intel.com \
    --cc=dts@dpdk.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

test suite reviews and discussions

This inbox may be cloned and mirrored by anyone:

	git clone --mirror https://inbox.dpdk.org/dts/0 dts/git/0.git

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V2 dts dts/ https://inbox.dpdk.org/dts \
		dts@dpdk.org
	public-inbox-index dts

Example config snippet for mirrors.
Newsgroup available over NNTP:
	nntp://inbox.dpdk.org/inbox.dpdk.dts


AGPL code for this site: git clone https://public-inbox.org/public-inbox.git