From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mail-lf0-f45.google.com (mail-lf0-f45.google.com [209.85.215.45]) by dpdk.org (Postfix) with ESMTP id 6B3C1237 for ; Wed, 6 Sep 2017 19:51:32 +0200 (CEST) Received: by mail-lf0-f45.google.com with SMTP id q132so19344450lfe.5 for ; Wed, 06 Sep 2017 10:51:32 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linaro.org; s=google; h=from:to:cc:subject:date:message-id:in-reply-to:references; bh=2nq9wGI/R26JfHAXMU0r75JrkfJgccPen3NIWjnwMM4=; b=B65h6SKoz4fd7dnkLnNIk26vM/sb9Wj0qTfnRG0lh1yfbe9Mis6Vru7uUZCewNND46 fompTuGTXE8lYPCbRVBTaptkZaCEsrPixQLAdgCYN5yETmryOp1k/JwJ4fUfC4gNTF56 RFFfeJMttqziHwXqBl0O2ExhxGTb2JLbzFvEA= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references; bh=2nq9wGI/R26JfHAXMU0r75JrkfJgccPen3NIWjnwMM4=; b=DndiWlPG0Dik29dnT+FyZciSbe7RQ8lsiMFAsOHxndT6BhNqclKtVxOZsBxwEEols2 zaazMttBnqWKa3SweKYe/dzC1je/CYhpmK/1+2HjfD+aaAdFBotxuQDH7dCIvpJqWl5a OXXbRxloppQ4H+RuZpS1Hs72GT33Om+KkEA0psNVT6YAyQaZZKEe1+jzzyzEcMwqtS8g eBetQazKWM5bCcnxh3bL7viFzbg2MpI0I9mtel4emvYePQJ58UrvPHSjS9OO3QC9frx6 +LUnciXMxIIw6nAZOz90xoNY/lp4LQ4RkxZohNlFpcMY6r6/221/TLd42MJ721VFHY4d lDrg== X-Gm-Message-State: AHPjjUjJxrSlJo43m/lXYIznRw59g4zJkhzhFQrNSrGsspUVtaSdpua+ 2zqH6Xe9ZKwCeLbJNAW/tQ== X-Google-Smtp-Source: ADKCNb4GbdvKUHpE3VNPbkS10tvx985wd7iHE2dtO18lX7NicjdnQkq9bgbMWTzghYBuaxYjSeVc+g== X-Received: by 10.25.78.5 with SMTP id c5mr1141540lfb.74.1504720292178; Wed, 06 Sep 2017 10:51:32 -0700 (PDT) Received: from rad-H81M-S1.semihalf.local (31-172-191-173.noc.fibertech.net.pl. [31.172.191.173]) by smtp.gmail.com with ESMTPSA id g79sm71649ljb.66.2017.09.06.10.51.30 (version=TLS1_2 cipher=ECDHE-RSA-AES128-SHA bits=128/128); Wed, 06 Sep 2017 10:51:31 -0700 (PDT) From: Radoslaw Biernacki To: dts@dpdk.org, yong.liu@intel.com Cc: jianbo.liu@linaro.org, herbert.guan@arm.com, Radoslaw Biernacki Date: Wed, 6 Sep 2017 19:50:58 +0200 Message-Id: <1504720258-19076-1-git-send-email-radoslaw.biernacki@linaro.org> X-Mailer: git-send-email 1.9.1 In-Reply-To: <1504625174-17845-1-git-send-email-radoslaw.biernacki@linaro.org> References: <1504625174-17845-1-git-send-email-radoslaw.biernacki@linaro.org> Subject: [dts] [PATCH v2] framework: Adding JSON reporter X-BeenThere: dts@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: test suite reviews and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 06 Sep 2017 17:51:33 -0000 This patch adds the JSON reporter class which puts the results into output/test_results.json file Having JSON file format for results is usefull for CI integration. v2: - results from all DUT's and targets are now stored in single JSON file - "N/A" and "BLOCKED" are also used as test result Signed-off-by: Radoslaw Biernacki --- framework/dts.py | 5 +++ framework/json_reporter.py | 76 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) create mode 100644 framework/json_reporter.py diff --git a/framework/dts.py b/framework/dts.py index 931bf38..b38deb7 100644 --- a/framework/dts.py +++ b/framework/dts.py @@ -51,6 +51,7 @@ from test_case import TestCase from test_result import Result from stats_reporter import StatsReporter from excel_reporter import ExcelReporter +from json_reporter import JSONReporter from exception import TimeoutException, ConfigParseException, VerifyFailure from logger import getLogger import logger @@ -66,6 +67,7 @@ sys.setdefaultencoding('UTF8') requested_tests = None result = None excel_report = None +json_report = None stats_report = None log_handler = None @@ -443,6 +445,7 @@ def run_all(config_file, pkgName, git, patch, skip_setup, global requested_tests global result global excel_report + global json_report global stats_report global log_handler global check_case_inst @@ -506,6 +509,7 @@ def run_all(config_file, pkgName, git, patch, skip_setup, # report objects excel_report = ExcelReporter(output_dir + '/test_results.xls') + json_report = JSONReporter(output_dir + '/test_results.json') stats_report = StatsReporter(output_dir + '/statistics.txt') result = Result() @@ -574,6 +578,7 @@ def save_all_results(): Save all result to files. """ excel_report.save(result) + json_report.save(result) stats_report.save(result) diff --git a/framework/json_reporter.py b/framework/json_reporter.py new file mode 100644 index 0000000..80b6b7e --- /dev/null +++ b/framework/json_reporter.py @@ -0,0 +1,76 @@ +# BSD LICENSE +# +# Copyright(c) 2017 Linaro. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import json + +class JSONReporter(object): + + def __init__(self, filename): + self.filename = filename + + def __scan_cases(self, result, dut, target, suite): + case_results = {} + for case in result.all_test_cases(dut, target, suite): + test_result = result.result_for(dut, target, suite, case) + case_name = '{}/{}'.format(suite,case) + case_results[case_name] = test_result + if 'PASSED' in test_result: + case_results[case_name] = 'passed' + elif 'N/A' in test_result: + case_results[case_name] = 'n/a' + elif 'FAILED' in test_result: + case_results[case_name] = 'failed' + elif 'BLOCKED' in test_result: + case_results[case_name] = 'blocked' + return case_results + + def __scan_target(self, result, dut, target): + if result.is_target_failed(dut, target): + return 'fail' + case_results = {} + for suite in result.all_test_suites(dut, target): + case_results.update(self.__scan_cases(result, dut, target, suite)) + return case_results + + def __scan_dut(self, result, dut): + if result.is_dut_failed(dut): + return 'fail' + target_map = {} + for target in result.all_targets(dut): + target_map[target] = self.__scan_target(result, dut, target) + return target_map + + def save(self, result): + result_map = {} + for dut in result.all_duts(): + result_map[dut] = self.__scan_dut(result, dut) + with open(self.filename, 'w') as outfile: + json.dump(result_map, outfile, indent=4, separators=(',', ': '), encoding="utf-8", sort_keys=True) -- 1.9.1