From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mga14.intel.com (mga14.intel.com [192.55.52.115]) by dpdk.org (Postfix) with ESMTP id 06EF1F72 for ; Tue, 24 Jul 2018 03:51:11 +0200 (CEST) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga001.jf.intel.com ([10.7.209.18]) by fmsmga103.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 23 Jul 2018 18:51:10 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.51,394,1526367600"; d="scan'208";a="75337974" Received: from juan.sh.intel.com ([10.67.118.154]) by orsmga001.jf.intel.com with ESMTP; 23 Jul 2018 18:50:55 -0700 From: Lijuan Tu To: dts@dpdk.org Cc: Lijuan Tu Date: Tue, 24 Jul 2018 18:20:03 +0800 Message-Id: <1532427605-127449-2-git-send-email-lijuan.tu@intel.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1532427605-127449-1-git-send-email-lijuan.tu@intel.com> References: <1532427605-127449-1-git-send-email-lijuan.tu@intel.com> Subject: [dts] [next] [PATCH V3 1/3] framework: support argument: --update-expected. X-BeenThere: dts@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: test suite reviews and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 24 Jul 2018 01:51:12 -0000 A DPDK Performance Test Lab are established, and Specification requires DTS to support an --update-expected argument which will cause the script to update values based on the results of the current test run. * V2: narrow impact from global to suite When "--update-expected" added in bash command, and there is "update-expected = Ture" in suite configuration file. All objects in configuration file will be updated. Take single core performance test for example: If "./dts --update-expected" and "update-expected = Ture" in conf/nic_single_core_perf.cfg, all objects will be updated in conf/nic_single_core_perf.cfg * V3: revert to reload configuration file in rerun command. Signed-off-by: Lijuan Tu --- framework/config.py | 11 +++++++++++ framework/dts.py | 6 +++++- framework/main.py | 6 +++++- framework/settings.py | 1 + framework/test_case.py | 22 ++++++++++++++++++++-- 5 files changed, 42 insertions(+), 4 deletions(-) diff --git a/framework/config.py b/framework/config.py index 628fc6d..0b112d8 100644 --- a/framework/config.py +++ b/framework/config.py @@ -147,6 +147,17 @@ class SuiteConf(UserConf): return case_cfg + def update_case_config(self, case_name=""): + """ + update section (case_name) of the configure file + """ + update_suite_cfg_obj = UserConf(self.config_file) + update_suite_cfg = update_suite_cfg_obj.load_section(case_name) + for key in update_suite_cfg_obj.conf.options(case_name): + update_suite_cfg_obj.conf.set( + case_name, key, str(self.suite_cfg[key])) + update_suite_cfg_obj.conf.write(open(self.config_file, 'w')) + class VirtConf(UserConf): diff --git a/framework/dts.py b/framework/dts.py index 0b2240c..4435418 100644 --- a/framework/dts.py +++ b/framework/dts.py @@ -439,7 +439,7 @@ def dts_run_suite(duts, tester, test_suites, target): def run_all(config_file, pkgName, git, patch, skip_setup, read_cache, project, suite_dir, test_cases, base_dir, output_dir, verbose, virttype, debug, - debugcase, re_run, commands): + debugcase, re_run, commands, update_expected): """ Main process of DTS, it will run all test suites in the config file. """ @@ -479,6 +479,10 @@ def run_all(config_file, pkgName, git, patch, skip_setup, if debugcase is True: settings.save_global_setting(settings.DEBUG_CASE_SETTING, 'yes') + # enable update-expected + if update_expected is True: + settings.save_global_setting(settings.UPDATE_EXPECTED, 'yes') + # init log_handler handler if verbose is True: logger.set_verbose() diff --git a/framework/main.py b/framework/main.py index 0aa54fd..9d7ef31 100755 --- a/framework/main.py +++ b/framework/main.py @@ -143,6 +143,10 @@ parser.add_argument('--commands', help='run command on tester or dut. The command format is ' + '[commands]:dut|tester:pre-init|post-init:check|ignore') +parser.add_argument('--update-expected', + action='store_true', + help='update expected values based on test results') + args = parser.parse_args() @@ -159,4 +163,4 @@ dts.run_all(args.config_file, args.snapshot, args.git, args.patch, args.skip_setup, args.read_cache, args.project, args.suite_dir, args.test_cases, args.dir, args.output, args.verbose,args.virttype, - args.debug, args.debugcase, args.re_run, args.commands) + args.debug, args.debugcase, args.re_run, args.commands, args.update_expected) diff --git a/framework/settings.py b/framework/settings.py index 07c3ac6..2561ddb 100644 --- a/framework/settings.py +++ b/framework/settings.py @@ -218,6 +218,7 @@ DPDK_RXMODE_SETTING = "DTS_DPDK_RXMODE" DTS_ERROR_ENV = "DTS_RUNNING_ERROR" DTS_CFG_FOLDER = "DTS_CFG_FOLDER" DTS_PARALLEL_SETTING = "DTS_PARALLEL_ENABLE" +UPDATE_EXPECTED = "DTS_UPDATE_EXPECTED_ENABLE" """ diff --git a/framework/test_case.py b/framework/test_case.py index a84e2bb..bb7c1c8 100644 --- a/framework/test_case.py +++ b/framework/test_case.py @@ -40,7 +40,9 @@ import time from exception import VerifyFailure, TimeoutException from settings import DRIVERS, NICS, get_nic_name, load_global_setting -from settings import PERF_SETTING, FUNC_SETTING, DEBUG_SETTING, DEBUG_CASE_SETTING, HOST_DRIVER_SETTING +from settings import PERF_SETTING, FUNC_SETTING, DEBUG_SETTING +from settings import DEBUG_CASE_SETTING, HOST_DRIVER_SETTING +from settings import UPDATE_EXPECTED, SUITE_SECTION_NAME from rst import RstReport from test_result import ResultTable, Result from logger import getLogger @@ -257,7 +259,6 @@ class TestCase(object): self._suite_conf = SuiteConf(self.suite_name) self._suite_cfg = self._suite_conf.suite_cfg self._case_cfg = self._suite_conf.load_case_config(case_name) - del(self._suite_conf) case_result = True if self._check_inst is not None: @@ -315,6 +316,11 @@ class TestCase(object): self._suite_result.test_case_failed(trace) self.logger.error('Test Case %s Result ERROR: ' % (case_name) + trace) finally: + # update expected + if load_global_setting(UPDATE_EXPECTED) == "yes" and \ + self.get_suite_cfg().has_key('update_expected') and \ + self.get_suite_cfg()['update_expected'] == True: + self._suite_conf.update_case_config(SUITE_SECTION_NAME) self.tear_down() return case_result @@ -375,6 +381,18 @@ class TestCase(object): """ return self._suite_cfg + def update_suite_cfg(self, suite_cfg): + """ + Update suite based configuration + """ + self._suite_cfg = suite_cfg + + def update_suite_cfg_ele(self, key, value): + """ + update one element of suite configuration + """ + self._suite_cfg[key]=value + def execute_tear_downall(self): """ execute suite tear_down_all function -- 1.8.3.1