From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mga17.intel.com (mga17.intel.com [192.55.52.151]) by dpdk.org (Postfix) with ESMTP id 9C9934CB3 for ; Fri, 7 Sep 2018 18:48:59 +0200 (CEST) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga002.jf.intel.com ([10.7.209.21]) by fmsmga107.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 07 Sep 2018 09:48:59 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.53,343,1531810800"; d="scan'208";a="89843328" Received: from juan.sh.intel.com ([10.67.118.154]) by orsmga002.jf.intel.com with ESMTP; 07 Sep 2018 09:48:57 -0700 From: Lijuan Tu To: dts@dpdk.org Cc: Lijuan Tu Date: Sat, 8 Sep 2018 09:17:01 +0800 Message-Id: <1536369422-184743-2-git-send-email-lijuan.tu@intel.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1536369422-184743-1-git-send-email-lijuan.tu@intel.com> References: <1536369422-184743-1-git-send-email-lijuan.tu@intel.com> Subject: [dts] [next][PATCH V1 1/2] tests/nic_single_core_perf: redefine accecpt_tolerance X-BeenThere: dts@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: test suite reviews and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 07 Sep 2018 16:49:00 -0000 Current accecpt_tolerance is a delta value, change it to a percentage value, as a percentage value is more make sense for tolerance. Signed-off-by: Lijuan Tu --- tests/TestSuite_nic_single_core_perf.py | 40 ++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/tests/TestSuite_nic_single_core_perf.py b/tests/TestSuite_nic_single_core_perf.py index 847a94c..b4d682e 100644 --- a/tests/TestSuite_nic_single_core_perf.py +++ b/tests/TestSuite_nic_single_core_perf.py @@ -74,6 +74,18 @@ class TestNicSingleCorePerf(TestCase): # determine if to save test result as a separated file self.save_result_flag = True + def convert_tolenrance_to_gap(self): + # Accepted tolerance in percentage + gap = dict() + rate = self.get_suite_cfg()['accepted_tolerance'] * 0.01 + for frame_size in self.test_parameters.keys(): + gap[frame_size] = dict() + for nb_desc in self.test_parameters[frame_size]: + expected_gap = rate * self.expected_throughput[frame_size][nb_desc] + gap[frame_size][nb_desc] = round(expected_gap, 3) + self.logger.info("Accept tolerance are (Mpps)%s" % gap) + return gap + def set_up(self): """ Run before each test case. @@ -104,14 +116,27 @@ class TestNicSingleCorePerf(TestCase): # {'$framesize':{"$nb_desc": 'throughput'} self.throughput = {} - # Accepted tolerance in Mpps - self.gap = self.get_suite_cfg()['accepted_tolerance'] + # self.gap has same structure as throughput. + self.gap = self.convert_tolenrance_to_gap() # header to print test result table self.table_header = ['Frame Size', 'TXD/RXD', 'Throughput', 'Rate', 'Expected Throughput', 'Throughput Difference'] self.test_result = {} + def get_gap_border(self, frame_size, nb_desc): + return -self.gap[frame_size][nb_desc] + + def verify_tolerance(self): + for frame_size in self.test_parameters.keys(): + for nb_desc in self.test_parameters[frame_size]: + cur_gap = self.throughput[frame_size][nb_desc] - self.expected_throughput[frame_size][nb_desc] + gap_border = self.get_gap_border(frame_size, nb_desc) + show_str = "Possible regression, " +\ + "Frame size/Descriptors: {}/{}, " +\ + "Acceptable Gap/Actual Gap: {}/{}" + self.verify(cur_gap >= gap_border,show_str.format(frame_size, nb_desc, gap_border, cur_gap)) + def test_nic_single_core_perf(self): """ Run nic single core performance @@ -124,12 +149,7 @@ class TestNicSingleCorePerf(TestCase): # check the gap between expected throughput and actual throughput try: - for frame_size in self.test_parameters.keys(): - for nb_desc in self.test_parameters[frame_size]: - cur_gap = (self.expected_throughput[frame_size][nb_desc] - - self.throughput[frame_size][nb_desc]) - self.verify(cur_gap < self.gap, - "Beyond Gap, Possible regression") + self.verify_tolerance() except Exception as e: self.logger.error(e) self.handle_expected() @@ -146,7 +166,7 @@ class TestNicSingleCorePerf(TestCase): for frame_size in self.test_parameters.keys(): for nb_desc in self.test_parameters[frame_size]: self.expected_throughput[frame_size][nb_desc] = \ - round(self.throughput[frame_size][nb_desc],3) + round(self.throughput[frame_size][nb_desc], 3) def perf_test(self, port_num): """ @@ -303,7 +323,7 @@ class TestNicSingleCorePerf(TestCase): value = row_in['TXD/RXD'], unit = 'descriptors') delta = (float(row_in['Throughput'].split()[0]) - float(row_in['Expected Throughput'].split()[0])) - if delta >= -self.gap: + if delta >= self.get_gap_border(frame_size, nb_desc): result = 'PASS' else: result = 'FAIL' -- 1.8.3.1