From: Lijuan Tu <lijuan.tu@intel.com>
To: dts@dpdk.org, pmacarth@iol.unh.edu, alialnu@mellanox.com
Cc: Lijuan Tu <lijuan.tu@intel.com>
Subject: [dts] [next] [PATCH 1/3] framework: support argument: --update-expected.
Date: Fri, 20 Jul 2018 23:22:02 +0800 [thread overview]
Message-ID: <1532100124-64769-2-git-send-email-lijuan.tu@intel.com> (raw)
In-Reply-To: <1532100124-64769-1-git-send-email-lijuan.tu@intel.com>
A DPDK Performance Test Lab are established, and Specification requires
DTS to support an --update-expected argument which will cause the
script to update values based on the results of the current test run.
Signed-off-by: Lijuan Tu <lijuan.tu@intel.com>
---
framework/config.py | 11 +++++++++++
framework/dts.py | 6 +++++-
framework/main.py | 6 +++++-
framework/settings.py | 1 +
framework/test_case.py | 22 ++++++++++++++++++----
5 files changed, 40 insertions(+), 6 deletions(-)
diff --git a/framework/config.py b/framework/config.py
index 628fc6d..0b112d8 100644
--- a/framework/config.py
+++ b/framework/config.py
@@ -147,6 +147,17 @@ class SuiteConf(UserConf):
return case_cfg
+ def update_case_config(self, case_name=""):
+ """
+ update section (case_name) of the configure file
+ """
+ update_suite_cfg_obj = UserConf(self.config_file)
+ update_suite_cfg = update_suite_cfg_obj.load_section(case_name)
+ for key in update_suite_cfg_obj.conf.options(case_name):
+ update_suite_cfg_obj.conf.set(
+ case_name, key, str(self.suite_cfg[key]))
+ update_suite_cfg_obj.conf.write(open(self.config_file, 'w'))
+
class VirtConf(UserConf):
diff --git a/framework/dts.py b/framework/dts.py
index 0b2240c..4435418 100644
--- a/framework/dts.py
+++ b/framework/dts.py
@@ -439,7 +439,7 @@ def dts_run_suite(duts, tester, test_suites, target):
def run_all(config_file, pkgName, git, patch, skip_setup,
read_cache, project, suite_dir, test_cases,
base_dir, output_dir, verbose, virttype, debug,
- debugcase, re_run, commands):
+ debugcase, re_run, commands, update_expected):
"""
Main process of DTS, it will run all test suites in the config file.
"""
@@ -479,6 +479,10 @@ def run_all(config_file, pkgName, git, patch, skip_setup,
if debugcase is True:
settings.save_global_setting(settings.DEBUG_CASE_SETTING, 'yes')
+ # enable update-expected
+ if update_expected is True:
+ settings.save_global_setting(settings.UPDATE_EXPECTED, 'yes')
+
# init log_handler handler
if verbose is True:
logger.set_verbose()
diff --git a/framework/main.py b/framework/main.py
index 0aa54fd..9d7ef31 100755
--- a/framework/main.py
+++ b/framework/main.py
@@ -143,6 +143,10 @@ parser.add_argument('--commands',
help='run command on tester or dut. The command format is ' +
'[commands]:dut|tester:pre-init|post-init:check|ignore')
+parser.add_argument('--update-expected',
+ action='store_true',
+ help='update expected values based on test results')
+
args = parser.parse_args()
@@ -159,4 +163,4 @@ dts.run_all(args.config_file, args.snapshot, args.git,
args.patch, args.skip_setup, args.read_cache,
args.project, args.suite_dir, args.test_cases,
args.dir, args.output, args.verbose,args.virttype,
- args.debug, args.debugcase, args.re_run, args.commands)
+ args.debug, args.debugcase, args.re_run, args.commands, args.update_expected)
diff --git a/framework/settings.py b/framework/settings.py
index 07c3ac6..2561ddb 100644
--- a/framework/settings.py
+++ b/framework/settings.py
@@ -218,6 +218,7 @@ DPDK_RXMODE_SETTING = "DTS_DPDK_RXMODE"
DTS_ERROR_ENV = "DTS_RUNNING_ERROR"
DTS_CFG_FOLDER = "DTS_CFG_FOLDER"
DTS_PARALLEL_SETTING = "DTS_PARALLEL_ENABLE"
+UPDATE_EXPECTED = "DTS_UPDATE_EXPECTED_ENABLE"
"""
diff --git a/framework/test_case.py b/framework/test_case.py
index a84e2bb..ffa03db 100644
--- a/framework/test_case.py
+++ b/framework/test_case.py
@@ -40,7 +40,9 @@ import time
from exception import VerifyFailure, TimeoutException
from settings import DRIVERS, NICS, get_nic_name, load_global_setting
-from settings import PERF_SETTING, FUNC_SETTING, DEBUG_SETTING, DEBUG_CASE_SETTING, HOST_DRIVER_SETTING
+from settings import PERF_SETTING, FUNC_SETTING, DEBUG_SETTING
+from settings import DEBUG_CASE_SETTING, HOST_DRIVER_SETTING
+from settings import UPDATE_EXPECTED, SUITE_SECTION_NAME
from rst import RstReport
from test_result import ResultTable, Result
from logger import getLogger
@@ -254,10 +256,7 @@ class TestCase(object):
self._rst_obj.write_title("Test Case: " + case_name)
# load suite configuration file here for rerun command
- self._suite_conf = SuiteConf(self.suite_name)
- self._suite_cfg = self._suite_conf.suite_cfg
self._case_cfg = self._suite_conf.load_case_config(case_name)
- del(self._suite_conf)
case_result = True
if self._check_inst is not None:
@@ -315,6 +314,9 @@ class TestCase(object):
self._suite_result.test_case_failed(trace)
self.logger.error('Test Case %s Result ERROR: ' % (case_name) + trace)
finally:
+ # update expected
+ if load_global_setting(UPDATE_EXPECTED) == "yes":
+ self._suite_conf.update_case_config(SUITE_SECTION_NAME)
self.tear_down()
return case_result
@@ -375,6 +377,18 @@ class TestCase(object):
"""
return self._suite_cfg
+ def update_suite_cfg(self, suite_cfg):
+ """
+ Update suite based configuration
+ """
+ self._suite_cfg = suite_cfg
+
+ def update_suite_cfg_ele(self, key, value):
+ """
+ update one element of suite configuration
+ """
+ self._suite_cfg[key]=value
+
def execute_tear_downall(self):
"""
execute suite tear_down_all function
--
1.8.3.1
next prev parent reply other threads:[~2018-07-20 6:53 UTC|newest]
Thread overview: 5+ messages / expand[flat|nested] mbox.gz Atom feed top
2018-07-20 15:22 [dts] [next] [PATCH 0/3] support update-expected to meet Open Lab requirement Lijuan Tu
2018-07-20 15:22 ` Lijuan Tu [this message]
2018-07-20 7:23 ` [dts] [next] [PATCH 1/3] framework: support argument: --update-expected Liu, Yong
2018-07-20 15:22 ` [dts] [next] [PATCH 2/3] tests/nic_single_core_perf: remove hardcode Lijuan Tu
2018-07-20 15:22 ` [dts] [next] [PATCH 3/3] conf/nic_single_core_perf: redefine elements Lijuan Tu
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1532100124-64769-2-git-send-email-lijuan.tu@intel.com \
--to=lijuan.tu@intel.com \
--cc=alialnu@mellanox.com \
--cc=dts@dpdk.org \
--cc=pmacarth@iol.unh.edu \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).