From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mail-yw0-f173.google.com (mail-yw0-f173.google.com [209.85.161.173]) by dpdk.org (Postfix) with ESMTP id 3324A235 for ; Wed, 6 Sep 2017 04:25:04 +0200 (CEST) Received: by mail-yw0-f173.google.com with SMTP id x144so18195153ywg.2 for ; Tue, 05 Sep 2017 19:25:04 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linaro.org; s=google; h=mime-version:in-reply-to:references:from:date:message-id:subject:to :cc; bh=sQ/FX0wWgtCFK3RNoL9ojwwqETQtALEQsEqofIbizw8=; b=JH1oCAaLn2cMKE3Jlxb8forkolypmuHOsHQBrdOfiSrCxX/mjoWCwwrgpLUu1zil1l NkfmvwY/n30ABZPunkmPmF7Ropnot74zmxGUDA0bbYT3Zi7OnLxbs3Xmd9PFwhDr5xRg qD0BRENnqyBJyvKzSiSasQyc0e96C0L37Iv6o= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:mime-version:in-reply-to:references:from:date :message-id:subject:to:cc; bh=sQ/FX0wWgtCFK3RNoL9ojwwqETQtALEQsEqofIbizw8=; b=kTKL54yVFBH3pX+vITUOpir6JgO5zdznSwGRuUOTuw17uz9PIhTVk12JM3n/yYXbQ1 /rlZvV5CN7IBFsBiYs+L0zms86mrMQMDRsmpzih3vL8AiJlS0ceNJFXK0CTSJsEFUfjF 8uPyX6eoMhr8PZ9bEGw/9ZhFc/PMQrctdjkR06MGGiZkPniK+BOdPKzNVlSXhkS+pP92 BGapoV4gNmVEYbsxgGQfw+9Em42RsczQ4yl48MXbTPHrKJlm8s2bkXU8dzHNZAelRrBy pZEkuVtl/1ZyCdVaZOuc4MobkEucVd+c1LNr3YUD/JCBSlx21EKmg0G9iSoaVzKH9Cgb pE/A== X-Gm-Message-State: AHPjjUiSC67shP0bCZrBCAaG8p4fWIa4r+ZZ+zlTvnWxmFA00Iuk0r+n 4fL02whIZXD6wpemT3t1uCFa+sQAo2A4 X-Google-Smtp-Source: ADKCNb5EJbwx0yW9295vrctGUazG/hvzAX9q5iYZpmf/azvxUPtRub5FXBTnxSUfJX3iFxoj1XaeY4InPr2s6r6ZNRM= X-Received: by 10.37.7.70 with SMTP id 67mr846200ybh.229.1504664703510; Tue, 05 Sep 2017 19:25:03 -0700 (PDT) MIME-Version: 1.0 Received: by 10.37.183.73 with HTTP; Tue, 5 Sep 2017 19:25:03 -0700 (PDT) In-Reply-To: <1504625174-17845-1-git-send-email-radoslaw.biernacki@linaro.org> References: <1504625174-17845-1-git-send-email-radoslaw.biernacki@linaro.org> From: Jianbo Liu Date: Wed, 6 Sep 2017 10:25:03 +0800 Message-ID: To: Radoslaw Biernacki Cc: dts@dpdk.org, Herbert Guan Content-Type: text/plain; charset="UTF-8" Subject: Re: [dts] [PATCH] framework: Adding JSON reporter X-BeenThere: dts@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: test suite reviews and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 06 Sep 2017 02:25:04 -0000 On 5 September 2017 at 23:26, Radoslaw Biernacki wrote: > This patch adds the JSON reporter class which puts the results > into output/test_results.json file > Having JSON file format for results is usefull for CI integration. > > Signed-off-by: Radoslaw Biernacki > --- > framework/dts.py | 5 ++++ > framework/json_reporter.py | 75 ++++++++++++++++++++++++++++++++++++++++++++++ > 2 files changed, 80 insertions(+) > create mode 100644 framework/json_reporter.py > > diff --git a/framework/dts.py b/framework/dts.py > index 931bf38..b38deb7 100644 > --- a/framework/dts.py > +++ b/framework/dts.py > @@ -51,6 +51,7 @@ from test_case import TestCase > from test_result import Result > from stats_reporter import StatsReporter > from excel_reporter import ExcelReporter > +from json_reporter import JSONReporter > from exception import TimeoutException, ConfigParseException, VerifyFailure > from logger import getLogger > import logger > @@ -66,6 +67,7 @@ sys.setdefaultencoding('UTF8') > requested_tests = None > result = None > excel_report = None > +json_report = None > stats_report = None > log_handler = None > > @@ -443,6 +445,7 @@ def run_all(config_file, pkgName, git, patch, skip_setup, > global requested_tests > global result > global excel_report > + global json_report > global stats_report > global log_handler > global check_case_inst > @@ -506,6 +509,7 @@ def run_all(config_file, pkgName, git, patch, skip_setup, > > # report objects > excel_report = ExcelReporter(output_dir + '/test_results.xls') > + json_report = JSONReporter(output_dir + '/test_results.json') > stats_report = StatsReporter(output_dir + '/statistics.txt') > result = Result() > > @@ -574,6 +578,7 @@ def save_all_results(): > Save all result to files. > """ > excel_report.save(result) > + json_report.save(result) > stats_report.save(result) > > > diff --git a/framework/json_reporter.py b/framework/json_reporter.py > new file mode 100644 > index 0000000..47e6869 > --- /dev/null > +++ b/framework/json_reporter.py > @@ -0,0 +1,75 @@ > +# BSD LICENSE > +# > +# Copyright(c) 2017 Linaro. All rights reserved. > +# All rights reserved. > +# > +# Redistribution and use in source and binary forms, with or without > +# modification, are permitted provided that the following conditions > +# are met: > +# > +# * Redistributions of source code must retain the above copyright > +# notice, this list of conditions and the following disclaimer. > +# * Redistributions in binary form must reproduce the above copyright > +# notice, this list of conditions and the following disclaimer in > +# the documentation and/or other materials provided with the > +# distribution. > +# * Neither the name of Intel Corporation nor the names of its > +# contributors may be used to endorse or promote products derived > +# from this software without specific prior written permission. > +# > +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > + > +import json > +import os > + > +class JSONReporter(object): > + > + def __init__(self, filename): > + self.filename = filename > + > + def __scan_cases(self, result, dut, target, suite): > + case_results = {} > + for case in result.all_test_cases(dut, target, suite): > + test_result = result.result_for(dut, target, suite, case) > + case_name = '{}/{}'.format(suite,case) > + if 'PASSED' in test_result: > + case_results[case_name] = 'pass' > + else: > + case_results[case_name] = 'fail' Is it reasonable to consider "N/A" or "SKIP" as fail? > + return case_results > + > + def __scan_target(self, result, dut, target): > + if result.is_target_failed(dut, target): > + return {'Target failed', 'fail'} > + case_results = {} > + for suite in result.all_test_suites(dut, target): > + case_results.update(self.__scan_cases(result, dut, target, suite)) > + abspath = os.path.abspath(self.filename) > + filename = os.path.basename(abspath) > + dirname = os.path.dirname(abspath) > + splitname = os.path.splitext(filename) > + extfilename = '{}/{}_{}_{}{}'.format(dirname, splitname[0], dut, target, splitname[1]) > + with open(extfilename, 'w') as outfile: > + json.dump(case_results, outfile, indent=4, separators=(',', ': '), encoding="utf-8", sort_keys=True) > + > + def __scan_dut(self, result, dut): > + if result.is_dut_failed(dut): > + return {'DUT failed', 'fail'} > + case_results = {} > + for target in result.all_targets(dut): > + self.__scan_target(result, dut, target) > + > + def save(self, result): > + case_results = {} > + for dut in result.all_duts(): > + self.__scan_dut(result, dut) > -- > 1.9.1 >