From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mga18.intel.com (mga18.intel.com [134.134.136.126]) by dpdk.org (Postfix) with ESMTP id 2963229AC for ; Fri, 20 Jul 2018 09:23:43 +0200 (CEST) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga003.fm.intel.com ([10.253.24.29]) by orsmga106.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 20 Jul 2018 00:23:42 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.51,377,1526367600"; d="scan'208";a="66122342" Received: from fmsmsx106.amr.corp.intel.com ([10.18.124.204]) by FMSMGA003.fm.intel.com with ESMTP; 20 Jul 2018 00:23:26 -0700 Received: from fmsmsx157.amr.corp.intel.com (10.18.116.73) by FMSMSX106.amr.corp.intel.com (10.18.124.204) with Microsoft SMTP Server (TLS) id 14.3.319.2; Fri, 20 Jul 2018 00:23:25 -0700 Received: from shsmsx104.ccr.corp.intel.com (10.239.4.70) by FMSMSX157.amr.corp.intel.com (10.18.116.73) with Microsoft SMTP Server (TLS) id 14.3.319.2; Fri, 20 Jul 2018 00:23:25 -0700 Received: from shsmsx103.ccr.corp.intel.com ([169.254.4.100]) by SHSMSX104.ccr.corp.intel.com ([169.254.5.81]) with mapi id 14.03.0319.002; Fri, 20 Jul 2018 15:23:23 +0800 From: "Liu, Yong" To: "Tu, Lijuan" , "dts@dpdk.org" , "pmacarth@iol.unh.edu" , "alialnu@mellanox.com" CC: "Tu, Lijuan" Thread-Topic: [dts] [next] [PATCH 1/3] framework: support argument: --update-expected. Thread-Index: AQHUH/ZnfHGidlTosEafQa5AsIxTlKSXsZ5Q Date: Fri, 20 Jul 2018 07:23:22 +0000 Message-ID: <86228AFD5BCD8E4EBFD2B90117B5E81E630FD4C2@SHSMSX103.ccr.corp.intel.com> References: <1532100124-64769-1-git-send-email-lijuan.tu@intel.com> <1532100124-64769-2-git-send-email-lijuan.tu@intel.com> In-Reply-To: <1532100124-64769-2-git-send-email-lijuan.tu@intel.com> Accept-Language: zh-CN, en-US Content-Language: en-US X-MS-Has-Attach: X-MS-TNEF-Correlator: x-titus-metadata-40: eyJDYXRlZ29yeUxhYmVscyI6IiIsIk1ldGFkYXRhIjp7Im5zIjoiaHR0cDpcL1wvd3d3LnRpdHVzLmNvbVwvbnNcL0ludGVsMyIsImlkIjoiM2QyMzgzNDEtMWJjMi00YzU2LWE5NDQtZTIwMTA5YWUyY2YwIiwicHJvcHMiOlt7Im4iOiJDVFBDbGFzc2lmaWNhdGlvbiIsInZhbHMiOlt7InZhbHVlIjoiQ1RQX05UIn1dfV19LCJTdWJqZWN0TGFiZWxzIjpbXSwiVE1DVmVyc2lvbiI6IjE3LjEwLjE4MDQuNDkiLCJUcnVzdGVkTGFiZWxIYXNoIjoiT3J0RHVMTUZLT1FlRllsZzJlaFJkZmRJYVZUUUFcL1ZNMXFMeG10Tis5WHZBeXZYcG5jeTBJN0lwaFlzXC9iRjN5In0= x-ctpclassification: CTP_NT dlp-product: dlpe-windows dlp-version: 11.0.200.100 dlp-reaction: no-action x-originating-ip: [10.239.127.40] Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Subject: Re: [dts] [next] [PATCH 1/3] framework: support argument: --update-expected. X-BeenThere: dts@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: test suite reviews and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 20 Jul 2018 07:23:44 -0000 Lijuan, This feature request is more like suite level requirement. In current imple= mentation, all suite configurations will be overwrite if this flag is enabl= ed. Thus may affect other suites. How about give the permission to suite? Suite can update its own configurat= ion when update option is enabled. It can limit the impact. Thanks, Marvin > -----Original Message----- > From: dts [mailto:dts-bounces@dpdk.org] On Behalf Of Lijuan Tu > Sent: Friday, July 20, 2018 11:22 PM > To: dts@dpdk.org; pmacarth@iol.unh.edu; alialnu@mellanox.com > Cc: Tu, Lijuan > Subject: [dts] [next] [PATCH 1/3] framework: support argument: --update- > expected. >=20 > A DPDK Performance Test Lab are established, and Specification requires > DTS to support an --update-expected argument which will cause the > script to update values based on the results of the current test run. >=20 > Signed-off-by: Lijuan Tu > --- > framework/config.py | 11 +++++++++++ > framework/dts.py | 6 +++++- > framework/main.py | 6 +++++- > framework/settings.py | 1 + > framework/test_case.py | 22 ++++++++++++++++++---- > 5 files changed, 40 insertions(+), 6 deletions(-) >=20 > diff --git a/framework/config.py b/framework/config.py > index 628fc6d..0b112d8 100644 > --- a/framework/config.py > +++ b/framework/config.py > @@ -147,6 +147,17 @@ class SuiteConf(UserConf): >=20 > return case_cfg >=20 > + def update_case_config(self, case_name=3D""): > + """ > + update section (case_name) of the configure file > + """ > + update_suite_cfg_obj =3D UserConf(self.config_file) > + update_suite_cfg =3D update_suite_cfg_obj.load_section(case_name= ) > + for key in update_suite_cfg_obj.conf.options(case_name): > + update_suite_cfg_obj.conf.set( > + case_name, key, str(self.suite_cfg[key])) > + update_suite_cfg_obj.conf.write(open(self.config_file, 'w')) > + >=20 > class VirtConf(UserConf): >=20 > diff --git a/framework/dts.py b/framework/dts.py > index 0b2240c..4435418 100644 > --- a/framework/dts.py > +++ b/framework/dts.py > @@ -439,7 +439,7 @@ def dts_run_suite(duts, tester, test_suites, target): > def run_all(config_file, pkgName, git, patch, skip_setup, > read_cache, project, suite_dir, test_cases, > base_dir, output_dir, verbose, virttype, debug, > - debugcase, re_run, commands): > + debugcase, re_run, commands, update_expected): > """ > Main process of DTS, it will run all test suites in the config file. > """ > @@ -479,6 +479,10 @@ def run_all(config_file, pkgName, git, patch, > skip_setup, > if debugcase is True: > settings.save_global_setting(settings.DEBUG_CASE_SETTING, 'yes') >=20 > + # enable update-expected > + if update_expected is True: > + settings.save_global_setting(settings.UPDATE_EXPECTED, 'yes') > + > # init log_handler handler > if verbose is True: > logger.set_verbose() > diff --git a/framework/main.py b/framework/main.py > index 0aa54fd..9d7ef31 100755 > --- a/framework/main.py > +++ b/framework/main.py > @@ -143,6 +143,10 @@ parser.add_argument('--commands', > help=3D'run command on tester or dut. The command > format is ' + > '[commands]:dut|tester:pre-init|post- > init:check|ignore') >=20 > +parser.add_argument('--update-expected', > + action=3D'store_true', > + help=3D'update expected values based on test results= ') > + > args =3D parser.parse_args() >=20 >=20 > @@ -159,4 +163,4 @@ dts.run_all(args.config_file, args.snapshot, args.git= , > args.patch, args.skip_setup, args.read_cache, > args.project, args.suite_dir, args.test_cases, > args.dir, args.output, args.verbose,args.virttype, > - args.debug, args.debugcase, args.re_run, args.commands) > + args.debug, args.debugcase, args.re_run, args.commands, > args.update_expected) > diff --git a/framework/settings.py b/framework/settings.py > index 07c3ac6..2561ddb 100644 > --- a/framework/settings.py > +++ b/framework/settings.py > @@ -218,6 +218,7 @@ DPDK_RXMODE_SETTING =3D "DTS_DPDK_RXMODE" > DTS_ERROR_ENV =3D "DTS_RUNNING_ERROR" > DTS_CFG_FOLDER =3D "DTS_CFG_FOLDER" > DTS_PARALLEL_SETTING =3D "DTS_PARALLEL_ENABLE" > +UPDATE_EXPECTED =3D "DTS_UPDATE_EXPECTED_ENABLE" >=20 >=20 > """ > diff --git a/framework/test_case.py b/framework/test_case.py > index a84e2bb..ffa03db 100644 > --- a/framework/test_case.py > +++ b/framework/test_case.py > @@ -40,7 +40,9 @@ import time >=20 > from exception import VerifyFailure, TimeoutException > from settings import DRIVERS, NICS, get_nic_name, load_global_setting > -from settings import PERF_SETTING, FUNC_SETTING, DEBUG_SETTING, > DEBUG_CASE_SETTING, HOST_DRIVER_SETTING > +from settings import PERF_SETTING, FUNC_SETTING, DEBUG_SETTING > +from settings import DEBUG_CASE_SETTING, HOST_DRIVER_SETTING > +from settings import UPDATE_EXPECTED, SUITE_SECTION_NAME > from rst import RstReport > from test_result import ResultTable, Result > from logger import getLogger > @@ -254,10 +256,7 @@ class TestCase(object): > self._rst_obj.write_title("Test Case: " + case_name) >=20 > # load suite configuration file here for rerun command > - self._suite_conf =3D SuiteConf(self.suite_name) > - self._suite_cfg =3D self._suite_conf.suite_cfg > self._case_cfg =3D self._suite_conf.load_case_config(case_name) > - del(self._suite_conf) >=20 Lijuan, Code here has mentioned that suite config file will reload for later usage.= I think it won't affect your updated expected function. You should keep it for functionality. > case_result =3D True > if self._check_inst is not None: > @@ -315,6 +314,9 @@ class TestCase(object): > self._suite_result.test_case_failed(trace) > self.logger.error('Test Case %s Result ERROR: ' % (case_name= ) > + trace) > finally: > + # update expected > + if load_global_setting(UPDATE_EXPECTED) =3D=3D "yes": > + self._suite_conf.update_case_config(SUITE_SECTION_NAME) > self.tear_down() > return case_result >=20 > @@ -375,6 +377,18 @@ class TestCase(object): > """ > return self._suite_cfg >=20 > + def update_suite_cfg(self, suite_cfg): > + """ > + Update suite based configuration > + """ > + self._suite_cfg =3D suite_cfg > + > + def update_suite_cfg_ele(self, key, value): > + """ > + update one element of suite configuration > + """ > + self._suite_cfg[key]=3Dvalue > + These functions were not called in test suite nic_single_core_perf. Please = make sure they are useful.=20 > def execute_tear_downall(self): > """ > execute suite tear_down_all function > -- > 1.8.3.1