From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mail-wm0-f46.google.com (mail-wm0-f46.google.com [74.125.82.46]) by dpdk.org (Postfix) with ESMTP id 4ACDD108F for ; Wed, 6 Sep 2017 15:28:42 +0200 (CEST) Received: by mail-wm0-f46.google.com with SMTP id 187so31033682wmn.1 for ; Wed, 06 Sep 2017 06:28:42 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linaro.org; s=google; h=mime-version:in-reply-to:references:from:date:message-id:subject:to :cc; bh=5MOyOfJiTUogpsr3A/WjIb8wmXYRBQTO7pDQD4/bgpc=; b=KKw1V8wuibNhw+X0dd+ZvORJxmQ3eSg2apz3nQ2PSdgOVXigEiDg8O6hEXOEVuKB3Z 9hc0kp42NDjmLeU42LBW3lV3qcIzcMfO6Tpgg9xLdG5wGm+9PKDLOAYl4/B2ja0QQ1og iZzaM52ncZu9plWanBMNAP3mlVh0ZfiEdXRXo= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:mime-version:in-reply-to:references:from:date :message-id:subject:to:cc; bh=5MOyOfJiTUogpsr3A/WjIb8wmXYRBQTO7pDQD4/bgpc=; b=pnRXZBCKV5IpldU9+OlSCGgjeM+cS9vwEJa0XhQGsN6a8+g0AyVRLafzVRDv9qPAZI xW/au6FNdfGnp0rNs2HB+u3S3rSKSf8koKvn+BFJLY4YKj0bi3kN39Q0RQw0TxYLtgb5 +ubrr86Ymxn6/hjWFhrv1PjxAKQe+hmnE3+kDlblipC+57k74aYUY+p1wdle1El243x0 wDtzxJKuFQ9Uqaxk5gqgIlygncmIRmQKBwhuvRAzynwavr/4/gPeaUOac+pMqvtdR3Ql uy+BmE7x90MlYkFFkYlUzREfidIPxNo6VdT+UtGn6g/qJmPu4c/YmkKMJ7c7coJrLt36 xqUg== X-Gm-Message-State: AHPjjUhhaFD61vVB8rmzTqZkedlEDjdDA3nTQsG2QP6La3Kptb6NRYSa qSitrYMEarJlQTMpxsylYoqMmAaPqFwH X-Google-Smtp-Source: ADKCNb7z+zWQKsrK4kz07EIuJxJXpzfBh4/9fQR9Cx/KwBo0p251WnS1807OTnkS49ToElM/wyrEF4U02RhBCpmUYZI= X-Received: by 10.28.48.79 with SMTP id w76mr101633wmw.17.1504704522024; Wed, 06 Sep 2017 06:28:42 -0700 (PDT) MIME-Version: 1.0 Received: by 10.223.197.204 with HTTP; Wed, 6 Sep 2017 06:28:21 -0700 (PDT) In-Reply-To: <86228AFD5BCD8E4EBFD2B90117B5E81E62E9BA98@SHSMSX103.ccr.corp.intel.com> References: <1504625174-17845-1-git-send-email-radoslaw.biernacki@linaro.org> <86228AFD5BCD8E4EBFD2B90117B5E81E62E9BA98@SHSMSX103.ccr.corp.intel.com> From: Radoslaw Biernacki Date: Wed, 6 Sep 2017 15:28:21 +0200 Message-ID: To: "Liu, Yong" Cc: "dts@dpdk.org" , "jianbo.liu@linaro.org" , "herbert.guan@arm.com" Content-Type: multipart/alternative; boundary="001a114230fa87121205588552a0" Subject: Re: [dts] [PATCH] framework: Adding JSON reporter X-BeenThere: dts@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: test suite reviews and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 06 Sep 2017 13:28:42 -0000 --001a114230fa87121205588552a0 Content-Type: text/plain; charset="UTF-8" OK will do. On 6 September 2017 at 05:09, Liu, Yong wrote: > Thanks, Radoslaw. One comment below. > > > -----Original Message----- > > From: dts [mailto:dts-bounces@dpdk.org] On Behalf Of Radoslaw Biernacki > > Sent: Tuesday, September 05, 2017 11:26 PM > > To: dts@dpdk.org > > Cc: jianbo.liu@linaro.org; herbert.guan@arm.com; Radoslaw Biernacki > > > > Subject: [dts] [PATCH] framework: Adding JSON reporter > > > > This patch adds the JSON reporter class which puts the results > > into output/test_results.json file > > Having JSON file format for results is usefull for CI integration. > > > > Signed-off-by: Radoslaw Biernacki > > --- > > framework/dts.py | 5 ++++ > > framework/json_reporter.py | 75 > > ++++++++++++++++++++++++++++++++++++++++++++++ > > 2 files changed, 80 insertions(+) > > create mode 100644 framework/json_reporter.py > > > > diff --git a/framework/dts.py b/framework/dts.py > > index 931bf38..b38deb7 100644 > > --- a/framework/dts.py > > +++ b/framework/dts.py > > @@ -51,6 +51,7 @@ from test_case import TestCase > > from test_result import Result > > from stats_reporter import StatsReporter > > from excel_reporter import ExcelReporter > > +from json_reporter import JSONReporter > > from exception import TimeoutException, ConfigParseException, > > VerifyFailure > > from logger import getLogger > > import logger > > @@ -66,6 +67,7 @@ sys.setdefaultencoding('UTF8') > > requested_tests = None > > result = None > > excel_report = None > > +json_report = None > > stats_report = None > > log_handler = None > > > > @@ -443,6 +445,7 @@ def run_all(config_file, pkgName, git, patch, > > skip_setup, > > global requested_tests > > global result > > global excel_report > > + global json_report > > global stats_report > > global log_handler > > global check_case_inst > > @@ -506,6 +509,7 @@ def run_all(config_file, pkgName, git, patch, > > skip_setup, > > > > # report objects > > excel_report = ExcelReporter(output_dir + '/test_results.xls') > > + json_report = JSONReporter(output_dir + '/test_results.json') > > stats_report = StatsReporter(output_dir + '/statistics.txt') > > result = Result() > > > > @@ -574,6 +578,7 @@ def save_all_results(): > > Save all result to files. > > """ > > excel_report.save(result) > > + json_report.save(result) > > stats_report.save(result) > > > > > > diff --git a/framework/json_reporter.py b/framework/json_reporter.py > > new file mode 100644 > > index 0000000..47e6869 > > --- /dev/null > > +++ b/framework/json_reporter.py > > @@ -0,0 +1,75 @@ > > +# BSD LICENSE > > +# > > +# Copyright(c) 2017 Linaro. All rights reserved. > > +# All rights reserved. > > +# > > +# Redistribution and use in source and binary forms, with or without > > +# modification, are permitted provided that the following conditions > > +# are met: > > +# > > +# * Redistributions of source code must retain the above copyright > > +# notice, this list of conditions and the following disclaimer. > > +# * Redistributions in binary form must reproduce the above copyright > > +# notice, this list of conditions and the following disclaimer in > > +# the documentation and/or other materials provided with the > > +# distribution. > > +# * Neither the name of Intel Corporation nor the names of its > > +# contributors may be used to endorse or promote products derived > > +# from this software without specific prior written permission. > > +# > > +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > > +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > > +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > > +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > > +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > > +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > > +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > > +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > > +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > > +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > > +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > > + > > +import json > > +import os > > + > > +class JSONReporter(object): > > + > > + def __init__(self, filename): > > + self.filename = filename > > + > > + def __scan_cases(self, result, dut, target, suite): > > + case_results = {} > > + for case in result.all_test_cases(dut, target, suite): > > + test_result = result.result_for(dut, target, suite, case) > > + case_name = '{}/{}'.format(suite,case) > > + if 'PASSED' in test_result: > > + case_results[case_name] = 'pass' > > + else: > > + case_results[case_name] = 'fail' > > + return case_results > > + > > + def __scan_target(self, result, dut, target): > > + if result.is_target_failed(dut, target): > > + return {'Target failed', 'fail'} > > + case_results = {} > > + for suite in result.all_test_suites(dut, target): > > + case_results.update(self.__scan_cases(result, dut, target, > > suite)) > > + abspath = os.path.abspath(self.filename) > > + filename = os.path.basename(abspath) > > + dirname = os.path.dirname(abspath) > > + splitname = os.path.splitext(filename) > > + extfilename = '{}/{}_{}_{}{}'.format(dirname, splitname[0], > dut, > > target, splitname[1]) > > + with open(extfilename, 'w') as outfile: > > + json.dump(case_results, outfile, indent=4, separators=(',', > ': > > '), encoding="utf-8", sort_keys=True) > > Look like multiple json report files will be created in just one execution. > This behavior is not align with excel report and may cause confusion. > If there's no special reason, please save all results into one json file. > > > + > > + def __scan_dut(self, result, dut): > > + if result.is_dut_failed(dut): > > + return {'DUT failed', 'fail'} > > + case_results = {} > > + for target in result.all_targets(dut): > > + self.__scan_target(result, dut, target) > > + > > + def save(self, result): > > + case_results = {} > > + for dut in result.all_duts(): > > + self.__scan_dut(result, dut) > > -- > > 1.9.1 > > --001a114230fa87121205588552a0 Content-Type: text/html; charset="UTF-8" Content-Transfer-Encoding: quoted-printable
OK will do.


On 6 September 2017 at 05:09, Liu, Yong <= yong.liu@intel.com> wrote:
= Thanks, Radoslaw. One comment below.

> -----Original Message-----
> From: dts [mailto:dts-bounces@= dpdk.org] On Behalf Of Radoslaw Biernacki
> Sent: Tuesday, September 05, 2017 11:26 PM
> To: dts@dpdk.org
> Cc: jianbo.liu@linaro.org= ; herbert.guan@arm.com; Radosla= w Biernacki
> <radoslaw.biernack= i@linaro.org>
> Subject: [dts] [PATCH] framework: Adding JSON reporter
>
> This patch adds the JSON reporter class which puts the results
> into output/test_results.json file
> Having JSON file format for results is usefull for CI integration.
>
> Signed-off-by: Radoslaw Biernacki <radoslaw.biernacki@linaro.org>
> ---
>=C2=A0 framework/dts.py=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0|=C2=A0= 5 ++++
>=C2=A0 framework/json_reporter.py | 75
> ++++++++++++++++++++++++++++++++++++++++++++++
>=C2=A0 2 files changed, 80 insertions(+)
>=C2=A0 create mode 100644 framework/json_reporter.py
>
> diff --git a/framework/dts.py b/framework/dts.py
> index 931bf38..b38deb7 100644
> --- a/framework/dts.py
> +++ b/framework/dts.py
> @@ -51,6 +51,7 @@ from test_case import TestCase
>=C2=A0 from test_result import Result
>=C2=A0 from stats_reporter import StatsReporter
>=C2=A0 from excel_reporter import ExcelReporter
> +from json_reporter import JSONReporter
>=C2=A0 from exception import TimeoutException, ConfigParseException, > VerifyFailure
>=C2=A0 from logger import getLogger
>=C2=A0 import logger
> @@ -66,6 +67,7 @@ sys.setdefaultencoding('UTF8')
>=C2=A0 requested_tests =3D None
>=C2=A0 result =3D None
>=C2=A0 excel_report =3D None
> +json_report =3D None
>=C2=A0 stats_report =3D None
>=C2=A0 log_handler =3D None
>
> @@ -443,6 +445,7 @@ def run_all(config_file, pkgName, git, patch,
> skip_setup,
>=C2=A0 =C2=A0 =C2=A0 global requested_tests
>=C2=A0 =C2=A0 =C2=A0 global result
>=C2=A0 =C2=A0 =C2=A0 global excel_report
> +=C2=A0 =C2=A0 global json_report
>=C2=A0 =C2=A0 =C2=A0 global stats_report
>=C2=A0 =C2=A0 =C2=A0 global log_handler
>=C2=A0 =C2=A0 =C2=A0 global check_case_inst
> @@ -506,6 +509,7 @@ def run_all(config_file, pkgName, git, patch,
> skip_setup,
>
>=C2=A0 =C2=A0 =C2=A0 # report objects
>=C2=A0 =C2=A0 =C2=A0 excel_report =3D ExcelReporter(output_dir + '/= test_results.xls')
> +=C2=A0 =C2=A0 json_report =3D JSONReporter(output_dir + '/test_re= sults.json')
>=C2=A0 =C2=A0 =C2=A0 stats_report =3D StatsReporter(output_dir + '/= statistics.txt')
>=C2=A0 =C2=A0 =C2=A0 result =3D Result()
>
> @@ -574,6 +578,7 @@ def save_all_results():
>=C2=A0 =C2=A0 =C2=A0 Save all result to files.
>=C2=A0 =C2=A0 =C2=A0 """
>=C2=A0 =C2=A0 =C2=A0 excel_report.save(result)
> +=C2=A0 =C2=A0 json_report.save(result)
>=C2=A0 =C2=A0 =C2=A0 stats_report.save(result)
>
>
> diff --git a/framework/json_reporter.py b/framework/json_reporter.py > new file mode 100644
> index 0000000..47e6869
> --- /dev/null
> +++ b/framework/json_reporter.py
> @@ -0,0 +1,75 @@
> +# BSD LICENSE
> +#
> +# Copyright(c) 2017 Linaro. All rights reserved.
> +# All rights reserved.
> +#
> +# Redistribution and use in source and binary forms, with or without<= br> > +# modification, are permitted provided that the following conditions<= br> > +# are met:
> +#
> +#=C2=A0 =C2=A0* Redistributions of source code must retain the above = copyright
> +#=C2=A0 =C2=A0 =C2=A0notice, this list of conditions and the followin= g disclaimer.
> +#=C2=A0 =C2=A0* Redistributions in binary form must reproduce the abo= ve copyright
> +#=C2=A0 =C2=A0 =C2=A0notice, this list of conditions and the followin= g disclaimer in
> +#=C2=A0 =C2=A0 =C2=A0the documentation and/or other materials provide= d with the
> +#=C2=A0 =C2=A0 =C2=A0distribution.
> +#=C2=A0 =C2=A0* Neither the name of Intel Corporation nor the names o= f its
> +#=C2=A0 =C2=A0 =C2=A0contributors may be used to endorse or promote p= roducts derived
> +#=C2=A0 =C2=A0 =C2=A0from this software without specific prior writte= n permission.
> +#
> +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS=
> +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,= BUT NOT
> +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS F= OR
> +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGH= T
> +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTA= L,
> +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF US= E,
> +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON A= NY
> +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT=
> +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE U= SE
> +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE= .
> +
> +import json
> +import os
> +
> +class JSONReporter(object):
> +
> +=C2=A0 =C2=A0 def __init__(self, filename):
> +=C2=A0 =C2=A0 =C2=A0 =C2=A0 self.filename =3D filename
> +
> +=C2=A0 =C2=A0 def __scan_cases(self, result, dut, target, suite):
> +=C2=A0 =C2=A0 =C2=A0 =C2=A0 case_results =3D {}
> +=C2=A0 =C2=A0 =C2=A0 =C2=A0 for case in result.all_test_cases(dut, ta= rget, suite):
> +=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0test_result =3D resul= t.result_for(dut, target, suite, case)
> +=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0case_name =3D '{}= /{}'.format(suite,case)
> +=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0if 'PASSED' i= n test_result:
> +=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0case_re= sults[case_name] =3D 'pass'
> +=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0else:
> +=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0case_re= sults[case_name] =3D 'fail'
> +=C2=A0 =C2=A0 =C2=A0 =C2=A0 return case_results
> +
> +=C2=A0 =C2=A0 def __scan_target(self, result, dut, target):
> +=C2=A0 =C2=A0 =C2=A0 =C2=A0 if result.is_target_failed(dut, target):<= br> > +=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 return {'Target failed&= #39;, 'fail'}
> +=C2=A0 =C2=A0 =C2=A0 =C2=A0 case_results =3D {}
> +=C2=A0 =C2=A0 =C2=A0 =C2=A0 for suite in result.all_test_suites(dut, = target):
> +=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 case_results.update(self.__= scan_cases(result, dut, target,
> suite))
> +=C2=A0 =C2=A0 =C2=A0 =C2=A0 abspath =3D os.path.abspath(self.filename= )
> +=C2=A0 =C2=A0 =C2=A0 =C2=A0 filename =3D os.path.basename(abspath) > +=C2=A0 =C2=A0 =C2=A0 =C2=A0 dirname =3D os.path.dirname(abspath)
> +=C2=A0 =C2=A0 =C2=A0 =C2=A0 splitname =3D os.path.splitext(filename)<= br> > +=C2=A0 =C2=A0 =C2=A0 =C2=A0 extfilename =3D '{}/{}_{}_{}{}'.f= ormat(dirname, splitname[0], dut,
> target, splitname[1])
> +=C2=A0 =C2=A0 =C2=A0 =C2=A0 with open(extfilename, 'w') as ou= tfile:
> +=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 json.dump(case_results, out= file, indent=3D4, separators=3D(',', ':
> '), encoding=3D"utf-8", sort_keys=3DTrue)

Look like multiple json report files will be created in just on= e execution.
This behavior is not align with excel report and may cause confusion.
If there's no special reason, please save all results into one json fil= e.

> +
> +=C2=A0 =C2=A0 def __scan_dut(self, result, dut):
> +=C2=A0 =C2=A0 =C2=A0 =C2=A0 if result.is_dut_failed(dut):
> +=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 return {'DUT failed'= ;, 'fail'}
> +=C2=A0 =C2=A0 =C2=A0 =C2=A0 case_results =3D {}
> +=C2=A0 =C2=A0 =C2=A0 =C2=A0 for target in result.all_targets(dut): > +=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 self.__scan_target(result, = dut, target)
> +
> +=C2=A0 =C2=A0 def save(self, result):
> +=C2=A0 =C2=A0 =C2=A0 =C2=A0 case_results =3D {}
> +=C2=A0 =C2=A0 =C2=A0 =C2=A0 for dut in result.all_duts():
> +=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 self.__scan_dut(result, dut= )
> --
> 1.9.1


--001a114230fa87121205588552a0--