From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mga07.intel.com (mga07.intel.com [134.134.136.100]) by dpdk.org (Postfix) with ESMTP id 54F163DC for ; Wed, 6 Sep 2017 05:09:46 +0200 (CEST) Received: from fmsmga001.fm.intel.com ([10.253.24.23]) by orsmga105.jf.intel.com with ESMTP; 05 Sep 2017 20:09:45 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.41,482,1498546800"; d="scan'208";a="1191954097" Received: from fmsmsx106.amr.corp.intel.com ([10.18.124.204]) by fmsmga001.fm.intel.com with ESMTP; 05 Sep 2017 20:09:45 -0700 Received: from fmsmsx101.amr.corp.intel.com (10.18.124.199) by FMSMSX106.amr.corp.intel.com (10.18.124.204) with Microsoft SMTP Server (TLS) id 14.3.319.2; Tue, 5 Sep 2017 20:09:45 -0700 Received: from shsmsx152.ccr.corp.intel.com (10.239.6.52) by fmsmsx101.amr.corp.intel.com (10.18.124.199) with Microsoft SMTP Server (TLS) id 14.3.319.2; Tue, 5 Sep 2017 20:09:45 -0700 Received: from shsmsx103.ccr.corp.intel.com ([169.254.4.219]) by SHSMSX152.ccr.corp.intel.com ([169.254.6.93]) with mapi id 14.03.0319.002; Wed, 6 Sep 2017 11:09:43 +0800 From: "Liu, Yong" To: Radoslaw Biernacki , "dts@dpdk.org" CC: "jianbo.liu@linaro.org" , "herbert.guan@arm.com" Thread-Topic: [dts] [PATCH] framework: Adding JSON reporter Thread-Index: AQHTJltrhnN7y7xDWkShqKmJSo8blKKnLIUg Date: Wed, 6 Sep 2017 03:09:42 +0000 Message-ID: <86228AFD5BCD8E4EBFD2B90117B5E81E62E9BA98@SHSMSX103.ccr.corp.intel.com> References: <1504625174-17845-1-git-send-email-radoslaw.biernacki@linaro.org> In-Reply-To: <1504625174-17845-1-git-send-email-radoslaw.biernacki@linaro.org> Accept-Language: zh-CN, en-US Content-Language: en-US X-MS-Has-Attach: X-MS-TNEF-Correlator: dlp-product: dlpe-windows dlp-version: 11.0.0.116 dlp-reaction: no-action x-originating-ip: [10.239.127.40] Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Subject: Re: [dts] [PATCH] framework: Adding JSON reporter X-BeenThere: dts@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: test suite reviews and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 06 Sep 2017 03:09:47 -0000 Thanks, Radoslaw. One comment below. > -----Original Message----- > From: dts [mailto:dts-bounces@dpdk.org] On Behalf Of Radoslaw Biernacki > Sent: Tuesday, September 05, 2017 11:26 PM > To: dts@dpdk.org > Cc: jianbo.liu@linaro.org; herbert.guan@arm.com; Radoslaw Biernacki > > Subject: [dts] [PATCH] framework: Adding JSON reporter >=20 > This patch adds the JSON reporter class which puts the results > into output/test_results.json file > Having JSON file format for results is usefull for CI integration. >=20 > Signed-off-by: Radoslaw Biernacki > --- > framework/dts.py | 5 ++++ > framework/json_reporter.py | 75 > ++++++++++++++++++++++++++++++++++++++++++++++ > 2 files changed, 80 insertions(+) > create mode 100644 framework/json_reporter.py >=20 > diff --git a/framework/dts.py b/framework/dts.py > index 931bf38..b38deb7 100644 > --- a/framework/dts.py > +++ b/framework/dts.py > @@ -51,6 +51,7 @@ from test_case import TestCase > from test_result import Result > from stats_reporter import StatsReporter > from excel_reporter import ExcelReporter > +from json_reporter import JSONReporter > from exception import TimeoutException, ConfigParseException, > VerifyFailure > from logger import getLogger > import logger > @@ -66,6 +67,7 @@ sys.setdefaultencoding('UTF8') > requested_tests =3D None > result =3D None > excel_report =3D None > +json_report =3D None > stats_report =3D None > log_handler =3D None >=20 > @@ -443,6 +445,7 @@ def run_all(config_file, pkgName, git, patch, > skip_setup, > global requested_tests > global result > global excel_report > + global json_report > global stats_report > global log_handler > global check_case_inst > @@ -506,6 +509,7 @@ def run_all(config_file, pkgName, git, patch, > skip_setup, >=20 > # report objects > excel_report =3D ExcelReporter(output_dir + '/test_results.xls') > + json_report =3D JSONReporter(output_dir + '/test_results.json') > stats_report =3D StatsReporter(output_dir + '/statistics.txt') > result =3D Result() >=20 > @@ -574,6 +578,7 @@ def save_all_results(): > Save all result to files. > """ > excel_report.save(result) > + json_report.save(result) > stats_report.save(result) >=20 >=20 > diff --git a/framework/json_reporter.py b/framework/json_reporter.py > new file mode 100644 > index 0000000..47e6869 > --- /dev/null > +++ b/framework/json_reporter.py > @@ -0,0 +1,75 @@ > +# BSD LICENSE > +# > +# Copyright(c) 2017 Linaro. All rights reserved. > +# All rights reserved. > +# > +# Redistribution and use in source and binary forms, with or without > +# modification, are permitted provided that the following conditions > +# are met: > +# > +# * Redistributions of source code must retain the above copyright > +# notice, this list of conditions and the following disclaimer. > +# * Redistributions in binary form must reproduce the above copyright > +# notice, this list of conditions and the following disclaimer in > +# the documentation and/or other materials provided with the > +# distribution. > +# * Neither the name of Intel Corporation nor the names of its > +# contributors may be used to endorse or promote products derived > +# from this software without specific prior written permission. > +# > +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > + > +import json > +import os > + > +class JSONReporter(object): > + > + def __init__(self, filename): > + self.filename =3D filename > + > + def __scan_cases(self, result, dut, target, suite): > + case_results =3D {} > + for case in result.all_test_cases(dut, target, suite): > + test_result =3D result.result_for(dut, target, suite, case) > + case_name =3D '{}/{}'.format(suite,case) > + if 'PASSED' in test_result: > + case_results[case_name] =3D 'pass' > + else: > + case_results[case_name] =3D 'fail' > + return case_results > + > + def __scan_target(self, result, dut, target): > + if result.is_target_failed(dut, target): > + return {'Target failed', 'fail'} > + case_results =3D {} > + for suite in result.all_test_suites(dut, target): > + case_results.update(self.__scan_cases(result, dut, target, > suite)) > + abspath =3D os.path.abspath(self.filename) > + filename =3D os.path.basename(abspath) > + dirname =3D os.path.dirname(abspath) > + splitname =3D os.path.splitext(filename) > + extfilename =3D '{}/{}_{}_{}{}'.format(dirname, splitname[0], du= t, > target, splitname[1]) > + with open(extfilename, 'w') as outfile: > + json.dump(case_results, outfile, indent=3D4, separators=3D('= ,', ': > '), encoding=3D"utf-8", sort_keys=3DTrue) Look like multiple json report files will be created in just one execution. This behavior is not align with excel report and may cause confusion.=20 If there's no special reason, please save all results into one json file. > + > + def __scan_dut(self, result, dut): > + if result.is_dut_failed(dut): > + return {'DUT failed', 'fail'} > + case_results =3D {} > + for target in result.all_targets(dut): > + self.__scan_target(result, dut, target) > + > + def save(self, result): > + case_results =3D {} > + for dut in result.all_duts(): > + self.__scan_dut(result, dut) > -- > 1.9.1