[dts] [PATCH v1 0/4] support suite and case level configuration

[dts] [PATCH v1 0/4] support suite and case level configuration

[Marvin Liu]

his patch set will enhance the overall configration process in DTS.
User can assign separated folder for configuration file by setting
"DTS_CFG_FOLDER" environment variable.

Support suite and case level configuration which can add flexibility in
case design.

Marvin Liu (4):
  framework setting: support configuration file folder change
  framework config: support suite&case level configuration
  framework: enhance test case execution process
  conf: add suite and case level configuration sample

 conf/suite_sample.cfg  |  9 +++++++
 framework/config.py    | 71 ++++++++++++++++++++++++++++++++++++++++++++------
 framework/debugger.py  |  2 +-
 framework/dts.py       |  4 +++
 framework/settings.py  | 19 +++++++++++---
 framework/test_case.py | 40 +++++++++++++++++++---------
 6 files changed, 120 insertions(+), 25 deletions(-)
 create mode 100644 conf/suite_sample.cfg

-- 
1.9.3

[dts] [PATCH v1 1/4] framework setting: support configuration file folder change

[Marvin Liu]

By default, configuration files will be loaded from default conf folder.
It will be inconvenience for backup local configuration files.

In this patch, will add new environment variable "DTS_CFG_FOLDER" which
can change the default folder of configuration files. Execution
configuration file will also loaded from this folder.

Signed-off-by: Marvin Liu <yong.liu@intel.com>

diff --git a/framework/dts.py b/framework/dts.py
index 7835574..26042cf 100644
--- a/framework/dts.py
+++ b/framework/dts.py
@@ -474,6 +474,10 @@ def run_all(config_file, pkgName, git, patch, skip_setup,
     requested_tests = test_cases
 
     # Read config file
+    dts_cfg_folder = settings.load_global_setting(settings.DTS_CFG_FOLDER)
+    if dts_cfg_folder != '':
+        config_file = dts_cfg_folder + os.sep +  config_file
+
     config = ConfigParser.SafeConfigParser()
     load_cfg = config.read(config_file)
     if len(load_cfg) == 0:
diff --git a/framework/settings.py b/framework/settings.py
index f0f3c8f..d306de2 100644
--- a/framework/settings.py
+++ b/framework/settings.py
@@ -188,14 +188,14 @@ Global macro for dts.
 IXIA = "ixia"
 
 """
-The root path of framework configs.
+The log name seperater.
 """
-CONFIG_ROOT_PATH = "./conf/"
+LOG_NAME_SEP = '.'
 
 """
-The log name seperater.
+Section name for suite level configuration
 """
-LOG_NAME_SEP = '.'
+SUITE_SECTION_NAME = "suite"
 
 """
 DTS global environment variable
@@ -209,6 +209,8 @@ DEBUG_SETTING = "DTS_DEBUG_ENABLE"
 DEBUG_CASE_SETTING = "DTS_DEBUGCASE_ENABLE"
 DPDK_RXMODE_SETTING = "DTS_DPDK_RXMODE"
 DTS_ERROR_ENV = "DTS_RUNNING_ERROR"
+DTS_CFG_FOLDER = "DTS_CFG_FOLDER"
+
 
 """
 DTS global error table
@@ -335,3 +337,12 @@ def accepted_nic(pci_id):
             return True
 
     return False
+
+"""
+The root path of framework configs.
+"""
+dts_cfg_folder = load_global_setting(DTS_CFG_FOLDER)
+if dts_cfg_folder != '':
+    CONFIG_ROOT_PATH = dts_cfg_folder
+else:
+    CONFIG_ROOT_PATH = "./conf"
-- 
1.9.3

[dts] [PATCH v1 2/4] framework config: support suite&case level configuration

[Marvin Liu]

Suite level configuration will be shared with all cases belonged to the
suite. Case level configuration will only be used in the case. Case
level configuration will overlap suite level configuration.

Few types of data supported in suite&case level configuration.
They are "value_int", "value_hex", "list_int", "list_str" and default
string type. For more details, please reference to
conf/suite_sample.cfg.

Signed-off-by: Marvin Liu <yong.liu@intel.com>

diff --git a/framework/config.py b/framework/config.py
index 8d44bfe..9e514a7 100644
--- a/framework/config.py
+++ b/framework/config.py
@@ -32,17 +32,19 @@
 """
 Generic port and crbs configuration file load function
 """
-
+import os
 import re
 import ConfigParser  # config parse module
 import argparse      # prase arguments module
-from settings import IXIA
-from exception import ConfigParseException, VirtConfigParseException
+from settings import IXIA, CONFIG_ROOT_PATH, SUITE_SECTION_NAME
+from settings import load_global_setting, DTS_CFG_FOLDER
+from exception import ConfigParseException, VirtConfigParseException, PortConfigParseException
 
-PORTCONF = "conf/ports.cfg"
-CRBCONF = "conf/crbs.cfg"
-VIRTCONF = "conf/virt_global.cfg"
-IXIACONF = "conf/ixia.cfg"
+PORTCONF = "%s/ports.cfg" % CONFIG_ROOT_PATH
+CRBCONF = "%s/crbs.cfg" % CONFIG_ROOT_PATH
+VIRTCONF = "%s/virt_global.cfg" % CONFIG_ROOT_PATH
+IXIACONF = "%s/ixia.cfg" % CONFIG_ROOT_PATH
+SUITECONF_SAMPLE = "%s/suite_sample.cfg" % CONFIG_ROOT_PATH
 
 
 class UserConf():
@@ -51,7 +53,6 @@ class UserConf():
         self.conf = ConfigParser.SafeConfigParser()
         load_files = self.conf.read(config)
         if load_files == []:
-            print "FAILED LOADING %s!!!" % config
             self.conf = None
             raise ConfigParseException(config)
 
@@ -87,6 +88,55 @@ class UserConf():
         return paramDict
 
 
+class SuiteConf(UserConf):
+    def __init__(self, suite_name=""):
+        self.config_file = CONFIG_ROOT_PATH + os.sep + suite_name + ".cfg"
+        self.suite_cfg = {}
+        try:
+            self.suite_conf = UserConf(self.config_file)
+        except ConfigParseException:
+            self.suite_conf = None
+
+        # load default suite configuration
+        self.suite_cfg = self.load_case_config(SUITE_SECTION_NAME)
+
+    def load_case_config(self, case_name=""):
+        case_cfg = self.suite_cfg.copy()
+        if self.suite_conf is None:
+            return case_cfg
+
+        try:
+            case_confs = self.suite_conf.load_section(case_name)
+        except:
+            print "FAILED FIND CASE[%s] CONFIG!!!" % case_name
+            return case_cfg
+
+        if case_confs is None:
+            return case_cfg
+
+        conf = dict(case_confs)
+        for key, data_string in conf.items():
+            if data_string.startswith("value_int:"):
+                value = data_string[len("value_int:"):]
+                case_cfg[key] = int(value)
+            elif data_string.startswith("value_hex:"):
+                value = data_string[len("value_hex:"):]
+                case_cfg[key] = int(value, 16)
+            elif data_string.startswith("list_int:"):
+                value = data_string[len("list_int:"):]
+                datas = value.split(',')
+                int_list = map(lambda x: int(x), datas)
+                case_cfg[key] = int_list
+            elif data_string.startswith("list_str:"):
+                value = data_string[len("list_str:"):]
+                str_list = value.split(',')
+                case_cfg[key] = str_list
+            else:
+                case_cfg[key] = data_string
+
+        return case_cfg
+
+
 class VirtConf(UserConf):
 
     def __init__(self, virt_conf=VIRTCONF):
@@ -344,3 +394,8 @@ if __name__ == '__main__':
     # example for ixia configuration file
     ixiaconf = IxiaConf(IXIACONF)
     print ixiaconf.load_ixia_config()
+
+    # example for suite configure file
+    suiteconf = SuiteConf(SUITECONF_SAMPLE)
+    print suiteconf.load_case_config("case1")
+    print suiteconf.load_case_config("case2")
-- 
1.9.3

[dts] [PATCH v1 3/4] framework: enhance test case execution process

[Marvin Liu]

1. Support suite and case level configuration in test_case module.
2. Debugger rerun command will call _execute_test_case which can do more
things than just call case object.

Signed-off-by: Marvin Liu <yong.liu@intel.com>

diff --git a/framework/debugger.py b/framework/debugger.py
index 8375590..07c2bdf 100644
--- a/framework/debugger.py
+++ b/framework/debugger.py
@@ -98,7 +98,7 @@ def rerun_command():
         # re-run specified test case
         for case in suite_obj._get_test_cases(r'%s' % AliveCase):
             if callable(case):
-                case()
+                suite_obj._execute_test_case(case)
 
 
 def exit_command():
diff --git a/framework/test_case.py b/framework/test_case.py
index 31a5eaf..2d1a58d 100644
--- a/framework/test_case.py
+++ b/framework/test_case.py
@@ -45,6 +45,7 @@ from settings import report_error
 from rst import RstReport
 from test_result import ResultTable, Result
 from logger import getLogger
+from config import SuiteConf
 
 class TestCase(object):
 
@@ -254,6 +255,12 @@ class TestCase(object):
         self._suite_result.test_case = case_obj.__name__
 
         self._rst_obj.write_title("Test Case: " + case_name)
+
+        # load suite configuration file here for rerun command
+        self._suite_conf = SuiteConf(self.suite_name)
+        self.case_cfg = self._suite_conf.load_case_config(case_name)
+        del(self._suite_conf)
+
         case_result = True
         if self._check_inst is not None:
             if self._check_inst.case_skip(case_name[len("test_"):]):
@@ -281,15 +288,8 @@ class TestCase(object):
             self.tester.get_session_output(timeout=0.1)
             # run set_up function for each case
             self.set_up()
-            # prepare debugger re-run case environment
-            if self._enable_debug or self._debug_case:
-                debugger.AliveSuite = self
-                debugger.AliveModule = __import__('TestSuite_' + self.suite_name)
-                debugger.AliveCase = case_name
-            if self._debug_case:
-                debugger.keyboard_handle(signal.SIGINT, None)
-            else:
-                case_obj()
+            # run test case
+            case_obj()
 
             self._suite_result.test_case_passed()
 
@@ -328,22 +328,38 @@ class TestCase(object):
         """
         Execute all test cases in one suite.
         """
+        # prepare debugger rerun case environment
+        if self._enable_debug or self._debug_case:
+            debugger.AliveSuite = self
+            debugger.AliveModule = __import__('TestSuite_' + self.suite_name)
 
         if load_global_setting(FUNC_SETTING) == 'yes':
             for case_obj in self._get_functional_cases():
                 for i in range(self.tester.re_run_time + 1):
-                    if self._execute_test_case(case_obj):
+                    if self.execute_test_case(case_obj):
                         break
                     else:
                         for dutobj in self.duts:
                             dutobj.get_session_output(timeout = 0.5 * (i + 1))
                         self.tester.get_session_output(timeout = 0.5 * (i + 1))
                         time.sleep(i + 1)
-                        self.logger.info(" Test case %s re-run %d time" % (case_obj.__name__, i + 1))
+                        self.logger.info(" Test case %s failed and re-run %d time" % (case_obj.__name__, i + 1))
 
         if load_global_setting(PERF_SETTING) == 'yes':
             for case_obj in self._get_performance_cases():
-                self._execute_test_case(case_obj)
+                self.execute_test_case(case_obj)
+
+    def execute_test_case(self, case_obj):
+        """
+        Execute test case or enter into debug mode.
+        """
+        debugger.AliveCase = case_obj.__name__
+
+        if self._debug_case:
+            self.logger.info("Rerun Test Case %s Begin" % debugger.AliveCase)
+            debugger.keyboard_handle(signal.SIGINT, None)
+        else:
+            self._execute_test_case(case_obj)
 
     def get_result(self):
         return self._suite_result
-- 
1.9.3

[dts] [PATCH v1 4/4] conf: add suite and case level configuration sample

[Marvin Liu]

Signed-off-by: Marvin Liu <yong.liu@intel.com>

diff --git a/conf/suite_sample.cfg b/conf/suite_sample.cfg
new file mode 100644
index 0000000..434a31e
--- /dev/null
+++ b/conf/suite_sample.cfg
@@ -0,0 +1,9 @@
+# sample for suite configuration
+[suite]
+int_value=value_int:10
+hex_value=value_hex:ff
+[case1]
+string=testpmd
+command=help
+packet_sizes=list_int:64,128,256
+protocals=list_str:TCP,UDP,SCTP
-- 
1.9.3

[dts] [PATCH v2 0/4] support suite and case level configuration

[Marvin Liu]

This patch set will enhance the overall configuration process in DTS.
User can assign separated folder for configuration file by setting
"DTS_CFG_FOLDER" environment variable.

Support suite and case level settings which can add flexibility in case
design.

v2: fix failed case re-run setting not work

Marvin Liu (4):
  framework setting: support configuration file folder change
  framework config: support suite&case level configuration
  framework: enhance test case execution process
  conf: add suite and case level configuration sample

 conf/suite_sample.cfg  |  9 +++++++
 framework/config.py    | 71 ++++++++++++++++++++++++++++++++++++++++++++------
 framework/debugger.py  |  3 ++-
 framework/dts.py       |  4 +++
 framework/settings.py  | 19 +++++++++++---
 framework/test_case.py | 46 ++++++++++++++++++++++----------
 6 files changed, 125 insertions(+), 27 deletions(-)
 create mode 100644 conf/suite_sample.cfg

-- 
1.9.3

[dts] [PATCH v2 1/4] framework setting: support configuration file folder change

[Marvin Liu]

By default, configuration files will be loaded from default folder.
It will be inconvenience for backup local configuration files.

In this patch, add one new environment variable "DTS_CFG_FOLDER" which
can change the default folder of configuration files. Execution
configuration file will also be loaded from this folder.

Signed-off-by: Marvin Liu <yong.liu@intel.com>

diff --git a/framework/dts.py b/framework/dts.py
index 7835574..26042cf 100644
--- a/framework/dts.py
+++ b/framework/dts.py
@@ -474,6 +474,10 @@ def run_all(config_file, pkgName, git, patch, skip_setup,
     requested_tests = test_cases
 
     # Read config file
+    dts_cfg_folder = settings.load_global_setting(settings.DTS_CFG_FOLDER)
+    if dts_cfg_folder != '':
+        config_file = dts_cfg_folder + os.sep +  config_file
+
     config = ConfigParser.SafeConfigParser()
     load_cfg = config.read(config_file)
     if len(load_cfg) == 0:
diff --git a/framework/settings.py b/framework/settings.py
index f0f3c8f..d306de2 100644
--- a/framework/settings.py
+++ b/framework/settings.py
@@ -188,14 +188,14 @@ Global macro for dts.
 IXIA = "ixia"
 
 """
-The root path of framework configs.
+The log name seperater.
 """
-CONFIG_ROOT_PATH = "./conf/"
+LOG_NAME_SEP = '.'
 
 """
-The log name seperater.
+Section name for suite level configuration
 """
-LOG_NAME_SEP = '.'
+SUITE_SECTION_NAME = "suite"
 
 """
 DTS global environment variable
@@ -209,6 +209,8 @@ DEBUG_SETTING = "DTS_DEBUG_ENABLE"
 DEBUG_CASE_SETTING = "DTS_DEBUGCASE_ENABLE"
 DPDK_RXMODE_SETTING = "DTS_DPDK_RXMODE"
 DTS_ERROR_ENV = "DTS_RUNNING_ERROR"
+DTS_CFG_FOLDER = "DTS_CFG_FOLDER"
+
 
 """
 DTS global error table
@@ -335,3 +337,12 @@ def accepted_nic(pci_id):
             return True
 
     return False
+
+"""
+The root path of framework configs.
+"""
+dts_cfg_folder = load_global_setting(DTS_CFG_FOLDER)
+if dts_cfg_folder != '':
+    CONFIG_ROOT_PATH = dts_cfg_folder
+else:
+    CONFIG_ROOT_PATH = "./conf"
-- 
1.9.3

[dts] [PATCH v2 2/4] framework config: support suite&case level configuration

[Marvin Liu]

Suite level configuration will be shared with all cases belonged to the
suite. Case level configuration will be used just in the case. Case
level configuration will overlap those suite level configurations.

Few types of data are supported in suite&case level configuration.
They are "value_int", "value_hex", "list_int", "list_str" and default
string type. For more details, please reference to
conf/suite_sample.cfg.

Signed-off-by: Marvin Liu <yong.liu@intel.com>

diff --git a/framework/config.py b/framework/config.py
index 8d44bfe..9e514a7 100644
--- a/framework/config.py
+++ b/framework/config.py
@@ -32,17 +32,19 @@
 """
 Generic port and crbs configuration file load function
 """
-
+import os
 import re
 import ConfigParser  # config parse module
 import argparse      # prase arguments module
-from settings import IXIA
-from exception import ConfigParseException, VirtConfigParseException
+from settings import IXIA, CONFIG_ROOT_PATH, SUITE_SECTION_NAME
+from settings import load_global_setting, DTS_CFG_FOLDER
+from exception import ConfigParseException, VirtConfigParseException, PortConfigParseException
 
-PORTCONF = "conf/ports.cfg"
-CRBCONF = "conf/crbs.cfg"
-VIRTCONF = "conf/virt_global.cfg"
-IXIACONF = "conf/ixia.cfg"
+PORTCONF = "%s/ports.cfg" % CONFIG_ROOT_PATH
+CRBCONF = "%s/crbs.cfg" % CONFIG_ROOT_PATH
+VIRTCONF = "%s/virt_global.cfg" % CONFIG_ROOT_PATH
+IXIACONF = "%s/ixia.cfg" % CONFIG_ROOT_PATH
+SUITECONF_SAMPLE = "%s/suite_sample.cfg" % CONFIG_ROOT_PATH
 
 
 class UserConf():
@@ -51,7 +53,6 @@ class UserConf():
         self.conf = ConfigParser.SafeConfigParser()
         load_files = self.conf.read(config)
         if load_files == []:
-            print "FAILED LOADING %s!!!" % config
             self.conf = None
             raise ConfigParseException(config)
 
@@ -87,6 +88,55 @@ class UserConf():
         return paramDict
 
 
+class SuiteConf(UserConf):
+    def __init__(self, suite_name=""):
+        self.config_file = CONFIG_ROOT_PATH + os.sep + suite_name + ".cfg"
+        self.suite_cfg = {}
+        try:
+            self.suite_conf = UserConf(self.config_file)
+        except ConfigParseException:
+            self.suite_conf = None
+
+        # load default suite configuration
+        self.suite_cfg = self.load_case_config(SUITE_SECTION_NAME)
+
+    def load_case_config(self, case_name=""):
+        case_cfg = self.suite_cfg.copy()
+        if self.suite_conf is None:
+            return case_cfg
+
+        try:
+            case_confs = self.suite_conf.load_section(case_name)
+        except:
+            print "FAILED FIND CASE[%s] CONFIG!!!" % case_name
+            return case_cfg
+
+        if case_confs is None:
+            return case_cfg
+
+        conf = dict(case_confs)
+        for key, data_string in conf.items():
+            if data_string.startswith("value_int:"):
+                value = data_string[len("value_int:"):]
+                case_cfg[key] = int(value)
+            elif data_string.startswith("value_hex:"):
+                value = data_string[len("value_hex:"):]
+                case_cfg[key] = int(value, 16)
+            elif data_string.startswith("list_int:"):
+                value = data_string[len("list_int:"):]
+                datas = value.split(',')
+                int_list = map(lambda x: int(x), datas)
+                case_cfg[key] = int_list
+            elif data_string.startswith("list_str:"):
+                value = data_string[len("list_str:"):]
+                str_list = value.split(',')
+                case_cfg[key] = str_list
+            else:
+                case_cfg[key] = data_string
+
+        return case_cfg
+
+
 class VirtConf(UserConf):
 
     def __init__(self, virt_conf=VIRTCONF):
@@ -344,3 +394,8 @@ if __name__ == '__main__':
     # example for ixia configuration file
     ixiaconf = IxiaConf(IXIACONF)
     print ixiaconf.load_ixia_config()
+
+    # example for suite configure file
+    suiteconf = SuiteConf(SUITECONF_SAMPLE)
+    print suiteconf.load_case_config("case1")
+    print suiteconf.load_case_config("case2")
-- 
1.9.3

[dts] [PATCH v2 3/4] framework: enhance test case execution process

[Marvin Liu]

1. Support suite and case level configuration in test_case module.
2. Debugger rerun command will call _execute_test_case which can do more
things than just call case object.

Signed-off-by: Marvin Liu <yong.liu@intel.com>

diff --git a/framework/debugger.py b/framework/debugger.py
index 8375590..c3ff33d 100644
--- a/framework/debugger.py
+++ b/framework/debugger.py
@@ -98,7 +98,8 @@ def rerun_command():
         # re-run specified test case
         for case in suite_obj._get_test_cases(r'%s' % AliveCase):
             if callable(case):
-                case()
+                suite_obj.logger.info("Rerun Test Case %s Begin" % case.__name__)
+                suite_obj._execute_test_case(case)
 
 
 def exit_command():
diff --git a/framework/test_case.py b/framework/test_case.py
index 31a5eaf..8671ffa 100644
--- a/framework/test_case.py
+++ b/framework/test_case.py
@@ -45,6 +45,7 @@ from settings import report_error
 from rst import RstReport
 from test_result import ResultTable, Result
 from logger import getLogger
+from config import SuiteConf
 
 class TestCase(object):
 
@@ -254,6 +255,12 @@ class TestCase(object):
         self._suite_result.test_case = case_obj.__name__
 
         self._rst_obj.write_title("Test Case: " + case_name)
+
+        # load suite configuration file here for rerun command
+        self._suite_conf = SuiteConf(self.suite_name)
+        self.case_cfg = self._suite_conf.load_case_config(case_name)
+        del(self._suite_conf)
+
         case_result = True
         if self._check_inst is not None:
             if self._check_inst.case_skip(case_name[len("test_"):]):
@@ -281,15 +288,8 @@ class TestCase(object):
             self.tester.get_session_output(timeout=0.1)
             # run set_up function for each case
             self.set_up()
-            # prepare debugger re-run case environment
-            if self._enable_debug or self._debug_case:
-                debugger.AliveSuite = self
-                debugger.AliveModule = __import__('TestSuite_' + self.suite_name)
-                debugger.AliveCase = case_name
-            if self._debug_case:
-                debugger.keyboard_handle(signal.SIGINT, None)
-            else:
-                case_obj()
+            # run test case
+            case_obj()
 
             self._suite_result.test_case_passed()
 
@@ -328,22 +328,40 @@ class TestCase(object):
         """
         Execute all test cases in one suite.
         """
+        # prepare debugger rerun case environment
+        if self._enable_debug or self._debug_case:
+            debugger.AliveSuite = self
+            debugger.AliveModule = __import__('TestSuite_' + self.suite_name)
 
         if load_global_setting(FUNC_SETTING) == 'yes':
             for case_obj in self._get_functional_cases():
                 for i in range(self.tester.re_run_time + 1):
-                    if self._execute_test_case(case_obj):
-                        break
-                    else:
+                    ret = self.execute_test_case(case_obj):
+
+                    if ret is False:
                         for dutobj in self.duts:
                             dutobj.get_session_output(timeout = 0.5 * (i + 1))
                         self.tester.get_session_output(timeout = 0.5 * (i + 1))
                         time.sleep(i + 1)
-                        self.logger.info(" Test case %s re-run %d time" % (case_obj.__name__, i + 1))
+                        self.logger.info(" Test case %s failed and re-run %d time" % (case_obj.__name__, i + 1))
+                    else:
+                        break
 
         if load_global_setting(PERF_SETTING) == 'yes':
             for case_obj in self._get_performance_cases():
-                self._execute_test_case(case_obj)
+                self.execute_test_case(case_obj)
+
+    def execute_test_case(self, case_obj):
+        """
+        Execute test case or enter into debug mode.
+        """
+        debugger.AliveCase = case_obj.__name__
+
+        if self._debug_case:
+            self.logger.info("Rerun Test Case %s Begin" % debugger.AliveCase)
+            debugger.keyboard_handle(signal.SIGINT, None)
+        else:
+            return self._execute_test_case(case_obj)
 
     def get_result(self):
         return self._suite_result
-- 
1.9.3

[dts] [PATCH v2 4/4] conf: add suite and case level configuration sample

[Marvin Liu]

Signed-off-by: Marvin Liu <yong.liu@intel.com>

diff --git a/conf/suite_sample.cfg b/conf/suite_sample.cfg
new file mode 100644
index 0000000..434a31e
--- /dev/null
+++ b/conf/suite_sample.cfg
@@ -0,0 +1,9 @@
+# sample for suite configuration
+[suite]
+int_value=value_int:10
+hex_value=value_hex:ff
+[case1]
+string=testpmd
+command=help
+packet_sizes=list_int:64,128,256
+protocals=list_str:TCP,UDP,SCTP
-- 
1.9.3

[dts] [PATCH v3 0/4] support suite and case level configuration

[Marvin Liu]

This patch set will enhance the overall configuration process in DTS.
User can assign separated folder for configuration file by setting
"DTS_CFG_FOLDER" environment variable.

Support suite and case level settings which can add flexibility in case
design.

v3: case based configuration can be retrieved by method get_case_cfg
    fix debugger rerun command will run several cases

v2: fix failed case re-run setting not work

Marvin Liu (4):
  framework setting: support change configuration file folder
  framework config: support suite&case level configuration
  framework: enhance test case execution process
  conf: add suite and case level configuration sample

 conf/suite_sample.cfg  |  9 +++++++
 framework/config.py    | 71 ++++++++++++++++++++++++++++++++++++++++++++------
 framework/debugger.py  |  5 ++--
 framework/dts.py       |  4 +++
 framework/settings.py  | 19 +++++++++++---
 framework/test_case.py | 55 ++++++++++++++++++++++++++++----------
 6 files changed, 135 insertions(+), 28 deletions(-)
 create mode 100644 conf/suite_sample.cfg

-- 
1.9.3

[dts] [PATCH v3 1/4] framework setting: support change configuration file folder

[Marvin Liu]

By default, configuration files will be loaded from default folder.
It will be inconvenience for backup local configuration files.

In this patch, add one new environment variable "DTS_CFG_FOLDER" which
can change the default folder of configuration files. Execution
configuration file will also be loaded from this folder.

Signed-off-by: Marvin Liu <yong.liu@intel.com>

diff --git a/framework/dts.py b/framework/dts.py
index 7835574..26042cf 100644
--- a/framework/dts.py
+++ b/framework/dts.py
@@ -474,6 +474,10 @@ def run_all(config_file, pkgName, git, patch, skip_setup,
     requested_tests = test_cases
 
     # Read config file
+    dts_cfg_folder = settings.load_global_setting(settings.DTS_CFG_FOLDER)
+    if dts_cfg_folder != '':
+        config_file = dts_cfg_folder + os.sep +  config_file
+
     config = ConfigParser.SafeConfigParser()
     load_cfg = config.read(config_file)
     if len(load_cfg) == 0:
diff --git a/framework/settings.py b/framework/settings.py
index f0f3c8f..d306de2 100644
--- a/framework/settings.py
+++ b/framework/settings.py
@@ -188,14 +188,14 @@ Global macro for dts.
 IXIA = "ixia"
 
 """
-The root path of framework configs.
+The log name seperater.
 """
-CONFIG_ROOT_PATH = "./conf/"
+LOG_NAME_SEP = '.'
 
 """
-The log name seperater.
+Section name for suite level configuration
 """
-LOG_NAME_SEP = '.'
+SUITE_SECTION_NAME = "suite"
 
 """
 DTS global environment variable
@@ -209,6 +209,8 @@ DEBUG_SETTING = "DTS_DEBUG_ENABLE"
 DEBUG_CASE_SETTING = "DTS_DEBUGCASE_ENABLE"
 DPDK_RXMODE_SETTING = "DTS_DPDK_RXMODE"
 DTS_ERROR_ENV = "DTS_RUNNING_ERROR"
+DTS_CFG_FOLDER = "DTS_CFG_FOLDER"
+
 
 """
 DTS global error table
@@ -335,3 +337,12 @@ def accepted_nic(pci_id):
             return True
 
     return False
+
+"""
+The root path of framework configs.
+"""
+dts_cfg_folder = load_global_setting(DTS_CFG_FOLDER)
+if dts_cfg_folder != '':
+    CONFIG_ROOT_PATH = dts_cfg_folder
+else:
+    CONFIG_ROOT_PATH = "./conf"
-- 
1.9.3

[dts] [PATCH v3 2/4] framework config: support suite&case level configuration

[Marvin Liu]

Suite level configuration will be shared with all cases belonged to the
suite. Case level configuration will be used just in the case. Case
level configuration will overlap those suite level configurations.

Few types of data are supported in suite&case level configuration.
They are "value_int", "value_hex", "list_int", "list_str" and default
string type. For more details, please reference to
conf/suite_sample.cfg.

Signed-off-by: Marvin Liu <yong.liu@intel.com>

diff --git a/framework/config.py b/framework/config.py
index 8d44bfe..9e514a7 100644
--- a/framework/config.py
+++ b/framework/config.py
@@ -32,17 +32,19 @@
 """
 Generic port and crbs configuration file load function
 """
-
+import os
 import re
 import ConfigParser  # config parse module
 import argparse      # prase arguments module
-from settings import IXIA
-from exception import ConfigParseException, VirtConfigParseException
+from settings import IXIA, CONFIG_ROOT_PATH, SUITE_SECTION_NAME
+from settings import load_global_setting, DTS_CFG_FOLDER
+from exception import ConfigParseException, VirtConfigParseException, PortConfigParseException
 
-PORTCONF = "conf/ports.cfg"
-CRBCONF = "conf/crbs.cfg"
-VIRTCONF = "conf/virt_global.cfg"
-IXIACONF = "conf/ixia.cfg"
+PORTCONF = "%s/ports.cfg" % CONFIG_ROOT_PATH
+CRBCONF = "%s/crbs.cfg" % CONFIG_ROOT_PATH
+VIRTCONF = "%s/virt_global.cfg" % CONFIG_ROOT_PATH
+IXIACONF = "%s/ixia.cfg" % CONFIG_ROOT_PATH
+SUITECONF_SAMPLE = "%s/suite_sample.cfg" % CONFIG_ROOT_PATH
 
 
 class UserConf():
@@ -51,7 +53,6 @@ class UserConf():
         self.conf = ConfigParser.SafeConfigParser()
         load_files = self.conf.read(config)
         if load_files == []:
-            print "FAILED LOADING %s!!!" % config
             self.conf = None
             raise ConfigParseException(config)
 
@@ -87,6 +88,55 @@ class UserConf():
         return paramDict
 
 
+class SuiteConf(UserConf):
+    def __init__(self, suite_name=""):
+        self.config_file = CONFIG_ROOT_PATH + os.sep + suite_name + ".cfg"
+        self.suite_cfg = {}
+        try:
+            self.suite_conf = UserConf(self.config_file)
+        except ConfigParseException:
+            self.suite_conf = None
+
+        # load default suite configuration
+        self.suite_cfg = self.load_case_config(SUITE_SECTION_NAME)
+
+    def load_case_config(self, case_name=""):
+        case_cfg = self.suite_cfg.copy()
+        if self.suite_conf is None:
+            return case_cfg
+
+        try:
+            case_confs = self.suite_conf.load_section(case_name)
+        except:
+            print "FAILED FIND CASE[%s] CONFIG!!!" % case_name
+            return case_cfg
+
+        if case_confs is None:
+            return case_cfg
+
+        conf = dict(case_confs)
+        for key, data_string in conf.items():
+            if data_string.startswith("value_int:"):
+                value = data_string[len("value_int:"):]
+                case_cfg[key] = int(value)
+            elif data_string.startswith("value_hex:"):
+                value = data_string[len("value_hex:"):]
+                case_cfg[key] = int(value, 16)
+            elif data_string.startswith("list_int:"):
+                value = data_string[len("list_int:"):]
+                datas = value.split(',')
+                int_list = map(lambda x: int(x), datas)
+                case_cfg[key] = int_list
+            elif data_string.startswith("list_str:"):
+                value = data_string[len("list_str:"):]
+                str_list = value.split(',')
+                case_cfg[key] = str_list
+            else:
+                case_cfg[key] = data_string
+
+        return case_cfg
+
+
 class VirtConf(UserConf):
 
     def __init__(self, virt_conf=VIRTCONF):
@@ -344,3 +394,8 @@ if __name__ == '__main__':
     # example for ixia configuration file
     ixiaconf = IxiaConf(IXIACONF)
     print ixiaconf.load_ixia_config()
+
+    # example for suite configure file
+    suiteconf = SuiteConf(SUITECONF_SAMPLE)
+    print suiteconf.load_case_config("case1")
+    print suiteconf.load_case_config("case2")
-- 
1.9.3

[dts] [PATCH v3 3/4] framework: enhance test case execution process

[Marvin Liu]

1. Support suite and case level configuration in test_case module.
2. Debugger rerun command will call _execute_test_case which can do more
things than just call case object.
3. Debugger rerun command should run the case with the specified name.

Signed-off-by: Marvin Liu <yong.liu@intel.com>

diff --git a/framework/debugger.py b/framework/debugger.py
index 8375590..5c02414 100644
--- a/framework/debugger.py
+++ b/framework/debugger.py
@@ -96,9 +96,10 @@ def rerun_command():
         # copy all element from previous suite to reloaded suite
         copy_instance_attr(AliveSuite, suite_obj)
         # re-run specified test case
-        for case in suite_obj._get_test_cases(r'%s' % AliveCase):
+        for case in suite_obj._get_test_cases(r'\A%s\Z' % AliveCase):
             if callable(case):
-                case()
+                suite_obj.logger.info("Rerun Test Case %s Begin" % case.__name__)
+                suite_obj._execute_test_case(case)
 
 
 def exit_command():
diff --git a/framework/test_case.py b/framework/test_case.py
index 31a5eaf..fec17a5 100644
--- a/framework/test_case.py
+++ b/framework/test_case.py
@@ -45,6 +45,7 @@ from settings import report_error
 from rst import RstReport
 from test_result import ResultTable, Result
 from logger import getLogger
+from config import SuiteConf
 
 class TestCase(object):
 
@@ -254,6 +255,12 @@ class TestCase(object):
         self._suite_result.test_case = case_obj.__name__
 
         self._rst_obj.write_title("Test Case: " + case_name)
+
+        # load suite configuration file here for rerun command
+        self._suite_conf = SuiteConf(self.suite_name)
+        self._case_cfg = self._suite_conf.load_case_config(case_name)
+        del(self._suite_conf)
+
         case_result = True
         if self._check_inst is not None:
             if self._check_inst.case_skip(case_name[len("test_"):]):
@@ -281,15 +288,8 @@ class TestCase(object):
             self.tester.get_session_output(timeout=0.1)
             # run set_up function for each case
             self.set_up()
-            # prepare debugger re-run case environment
-            if self._enable_debug or self._debug_case:
-                debugger.AliveSuite = self
-                debugger.AliveModule = __import__('TestSuite_' + self.suite_name)
-                debugger.AliveCase = case_name
-            if self._debug_case:
-                debugger.keyboard_handle(signal.SIGINT, None)
-            else:
-                case_obj()
+            # run test case
+            case_obj()
 
             self._suite_result.test_case_passed()
 
@@ -328,26 +328,53 @@ class TestCase(object):
         """
         Execute all test cases in one suite.
         """
+        # prepare debugger rerun case environment
+        if self._enable_debug or self._debug_case:
+            debugger.AliveSuite = self
+            debugger.AliveModule = __import__('TestSuite_' + self.suite_name)
 
         if load_global_setting(FUNC_SETTING) == 'yes':
             for case_obj in self._get_functional_cases():
                 for i in range(self.tester.re_run_time + 1):
-                    if self._execute_test_case(case_obj):
-                        break
-                    else:
+                    ret = self.execute_test_case(case_obj):
+
+                    if ret is False:
                         for dutobj in self.duts:
                             dutobj.get_session_output(timeout = 0.5 * (i + 1))
                         self.tester.get_session_output(timeout = 0.5 * (i + 1))
                         time.sleep(i + 1)
-                        self.logger.info(" Test case %s re-run %d time" % (case_obj.__name__, i + 1))
+                        self.logger.info(" Test case %s failed and re-run %d time" % (case_obj.__name__, i + 1))
+                    else:
+                        break
 
         if load_global_setting(PERF_SETTING) == 'yes':
             for case_obj in self._get_performance_cases():
-                self._execute_test_case(case_obj)
+                self.execute_test_case(case_obj)
+
+    def execute_test_case(self, case_obj):
+        """
+        Execute test case or enter into debug mode.
+        """
+        debugger.AliveCase = case_obj.__name__
+
+        if self._debug_case:
+            self.logger.info("Rerun Test Case %s Begin" % debugger.AliveCase)
+            debugger.keyboard_handle(signal.SIGINT, None)
+        else:
+            return self._execute_test_case(case_obj)
 
     def get_result(self):
+        """
+        Return suite test result
+        """
         return self._suite_result
 
+    def get_case_cfg(self):
+        """
+        Return case based configuration
+        """
+        return self._case_cfg
+
     def execute_tear_downall(self):
         """
         execute suite tear_down_all function
-- 
1.9.3

[dts] [PATCH v3 4/4] conf: add suite and case level configuration sample

[Marvin Liu]

Signed-off-by: Marvin Liu <yong.liu@intel.com>

diff --git a/conf/suite_sample.cfg b/conf/suite_sample.cfg
new file mode 100644
index 0000000..434a31e
--- /dev/null
+++ b/conf/suite_sample.cfg
@@ -0,0 +1,9 @@
+# sample for suite configuration
+[suite]
+int_value=value_int:10
+hex_value=value_hex:ff
+[case1]
+string=testpmd
+command=help
+packet_sizes=list_int:64,128,256
+protocals=list_str:TCP,UDP,SCTP
-- 
1.9.3