From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 6D027A056A; Thu, 11 Mar 2021 03:48:40 +0100 (CET) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 5FF9722A65F; Thu, 11 Mar 2021 03:48:40 +0100 (CET) Received: from mga18.intel.com (mga18.intel.com [134.134.136.126]) by mails.dpdk.org (Postfix) with ESMTP id 12121406A2 for ; Thu, 11 Mar 2021 03:48:37 +0100 (CET) IronPort-SDR: pvOp/nwp2n5u6csR/zoXqUGQFnqrTJY31JxXYlSXUEJsleYQLslgPvxKJckrsfikSv7k5FF0or eu2rnjCf8LoQ== X-IronPort-AV: E=McAfee;i="6000,8403,9919"; a="176197673" X-IronPort-AV: E=Sophos;i="5.81,238,1610438400"; d="scan'208";a="176197673" Received: from fmsmga007.fm.intel.com ([10.253.24.52]) by orsmga106.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 10 Mar 2021 18:48:36 -0800 IronPort-SDR: c44XzIml4YmAp6WZhH1Ile/DYavvM89mVmzFLHLbABBwM6rHdReOtxZDcqEV0N78FegeIKMFI8 6JrT2RYM/+DA== X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.81,238,1610438400"; d="scan'208";a="377214400" Received: from dpdk-icelake-perf.sh.intel.com ([10.67.118.199]) by fmsmga007.fm.intel.com with ESMTP; 10 Mar 2021 18:48:35 -0800 From: Lijuan Tu To: dpdklab@iol.unh.edu Cc: dts@dpdk.org, Lijuan Tu Date: Thu, 11 Mar 2021 10:48:00 +0000 Message-Id: <20210311104800.243752-1-lijuan.tu@intel.com> X-Mailer: git-send-email 2.25.1 MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Subject: [dts] [PATCH] tests/nic_single_core_perf: modify accepted_tolerance to percentage X-BeenThere: dts@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: test suite reviews and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dts-bounces@dpdk.org Sender: "dts" * accepted_tolerance in conf/nic_single_core_perf.cfg are changed to percentage, e.g. accepted_tolerance = '1%%' if not set, it will be set to 2% by default. * delat in json are chagned to percentage too, e.g. 'delta': '1.5%' * expected_throughput will be forced to 1Mpps for compute delat in percentage * add new conf test_interval in conf/nic_single_core_perf.cfg, which indicates interval of getting throughput statistics in second, e.g. test_interval = 10 if not set, it will be set to 5 seconds by default. * collect better statistics , get multiple throughput statistics during the test run, and compute the average as the final results. Signed-off-by: Lijuan Tu --- conf/nic_single_core_perf.cfg | 9 ++- tests/TestSuite_nic_single_core_perf.py | 102 +++++++++++++++--------- 2 files changed, 71 insertions(+), 40 deletions(-) diff --git a/conf/nic_single_core_perf.cfg b/conf/nic_single_core_perf.cfg index 4d3279ba..b673a2fa 100644 --- a/conf/nic_single_core_perf.cfg +++ b/conf/nic_single_core_perf.cfg @@ -8,8 +8,12 @@ # - test_duration is how many seconds each combination performance will # be recorded. # +# -test_interval defines interval of get throughput statistics (second). +# If not set, it is 5 seconds by default. +# # - accepted_tolerance defines the accepted tolerance between test -# results and expected numbers. +# results and expected numbers, it is a pecentage. If not set, it is 2% +# by default. # # - expected_throughput is a dictionary defining expected throughput # numbers based on NIC, and the pattern is @@ -34,7 +38,8 @@ test_parameters = {'1C/1T': {64: [512, 2048]}, '1C/2T': {64: [512, 2048]}} rx_desc_16byte = 'y' test_duration = 60 -accepted_tolerance = 1 +test_interval = 10 +accepted_tolerance = '1%%' expected_throughput = { 'fortville_spirit': { '1C/1T': {64: {512: 0.00, 2048: 0.00}}, diff --git a/tests/TestSuite_nic_single_core_perf.py b/tests/TestSuite_nic_single_core_perf.py index 8266506b..6c4095d3 100644 --- a/tests/TestSuite_nic_single_core_perf.py +++ b/tests/TestSuite_nic_single_core_perf.py @@ -41,6 +41,7 @@ from exception import VerifyFailure from settings import HEADER_SIZE, UPDATE_EXPECTED, load_global_setting from pmd_output import PmdOutput from copy import deepcopy +from numpy import mean import rst from pktgen import PacketGeneratorHelper @@ -57,7 +58,7 @@ class TestNicSingleCorePerf(TestCase): "Not required NIC ") self.headers_size = HEADER_SIZE['eth'] + HEADER_SIZE['ip'] - self.rx_desc = self.get_suite_cfg()['rx_desc_16byte'] + self.rx_desc = self.get_suite_cfg().get('rx_desc_16byte', 'y') if 'y' == self.rx_desc: # Update DPDK config file and rebuild to get best perf on fortville if self.nic in ["fortville_25g", "fortville_spirit"]: @@ -92,10 +93,11 @@ class TestNicSingleCorePerf(TestCase): """ # test parameters include: frames size, descriptor numbers - self.test_parameters = self.get_suite_cfg()['test_parameters'] + self.test_parameters = self.get_suite_cfg().get('test_parameters', {'1C/1T': {64: [512, 2048]}}) # traffic duraion in second - self.test_duration = self.get_suite_cfg()['test_duration'] + self.test_duration = self.get_suite_cfg().get('test_duration', 60) + self.test_interval = self.get_suite_cfg().get('test_interval', 5) # load the expected throughput for required nic if self.nic in ["ConnectX4_LX_MT4117"]: @@ -114,12 +116,12 @@ class TestNicSingleCorePerf(TestCase): # {'$framesize':{"$nb_desc": 'throughput'} self.throughput = {} - # Accepted tolerance in Mpps - self.gap = self.get_suite_cfg()['accepted_tolerance'] + # Accepted tolerance in pecentage + self.gap = float(self.get_suite_cfg().get('accepted_tolerance', '2%')[:-1]) # header to print test result table - self.table_header = ['Fwd_core', 'Frame Size', 'TXD/RXD', 'Throughput', 'Rate', - 'Expected Throughput', 'Throughput Difference'] + self.table_header = ['Fwd_core', 'Frame Size', 'TXD/RXD', 'Real-Mpps', 'Rate', + 'Expected-Mpps', 'Real vs Expected', 'Status'] self.test_result = {} def flows(self): @@ -232,7 +234,7 @@ class TestNicSingleCorePerf(TestCase): # need add one more core for start testpmd core_list = [core_list[0]] + [str(int(i) + 1) for i in core_list] - self.logger.info("Executing Test Using cores: %s of config %s" % (core_list, fwd_config)) + self.logger.info("Executing Test Using cores: %s of config %s" % (core_list[1:], fwd_config)) nb_cores = thread_num @@ -242,8 +244,6 @@ class TestNicSingleCorePerf(TestCase): # columbiaville use one queue per port for best performance. elif self.nic in ["columbiaville_100g", "columbiaville_25g", "columbiaville_25gx2"]: param += " --rxq=1 --txq=1" - # workaround for that testpmd can't forward packets in io forward mode - param += " --port-topology=loop" self.throughput[fwd_config] = dict() for frame_size in list(self.test_parameters[fwd_config].keys()): @@ -265,21 +265,42 @@ class TestNicSingleCorePerf(TestCase): # run packet generator streams = self.pktgen_helper.prepare_stream_from_tginput(tgenInput, 100, vm_config, self.tester.pktgen) # set traffic option - traffic_opt = {'duration': self.test_duration} - # _, pps = self.tester.traffic_generator_throughput(tgenInput, rate_percent=100, delay=30) - _, packets_received = self.tester.pktgen.measure_throughput(stream_ids=streams, options=traffic_opt) - self.verify(packets_received > 0, "No traffic detected") - throughput = packets_received / 1000000.0 - self.throughput[fwd_config][frame_size][nb_desc] = throughput + traffic_opt = { + 'method': 'throughput', + 'rate': 100, + 'duration': self.test_duration, + 'interval': self.test_interval, + } + stats = self.tester.pktgen.measure(stream_ids=streams, traffic_opt=traffic_opt) + + ##################################################### + # Remove max and min, then get average + ##################################################### + if isinstance(stats, list): + total_pps_rxs = [] + for i in range(len(stats)): + stats_pps = stats[i][1] + if isinstance(stats_pps, tuple): + total_pps_rxs.append(stats_pps[1]) + else: + total_pps_rxs.append(stats_pps) + + total_pps_rxs.remove(max(total_pps_rxs)) + total_pps_rxs.remove(min(total_pps_rxs)) + total_pps_rx = mean(total_pps_rxs) + else: + total_pps_rx = stats + + self.verify(total_pps_rx > 0, "No traffic detected, please check your configuration") + total_mpps_rx = total_pps_rx / 1000000.0 + self.throughput[fwd_config][frame_size][nb_desc] = total_mpps_rx self.dut.send_expect("stop", "testpmd> ") self.dut.send_expect("quit", "# ", 30) - self.verify(throughput, - "No traffic detected, please check your configuration") self.logger.info("Trouthput of " + "framesize: {}, rxd/txd: {} is :{} Mpps".format( - frame_size, nb_desc, throughput)) + frame_size, nb_desc, total_mpps_rx)) return self.throughput @@ -303,16 +324,25 @@ class TestNicSingleCorePerf(TestCase): ret_data[header[0]] = fwd_config ret_data[header[1]] = frame_size ret_data[header[2]] = nb_desc - ret_data[header[3]] = "{:.3f} Mpps".format( + ret_data[header[3]] = "{:.3f}".format( self.throughput[fwd_config][frame_size][nb_desc]) ret_data[header[4]] = "{:.3f}%".format( self.throughput[fwd_config][frame_size][nb_desc] * 100 / wirespeed) - ret_data[header[5]] = "{:.3f} Mpps".format( - self.expected_throughput[fwd_config][frame_size][nb_desc]) - ret_data[header[6]] = "{:.3f} Mpps".format( - self.throughput[fwd_config][frame_size][nb_desc] - + if 0 == self.expected_throughput[fwd_config][frame_size][nb_desc]: + self.expected_throughput[fwd_config][frame_size][nb_desc] = 1 + self.logger.warning("Expected throughput can't be zero !!") + msg = "Force it to 1 Mpps for frame size: {0}, RXD/TXD: {1}, Core: {2}" + self.logger.warning(msg.format(frame_size, nb_desc, fwd_config)) + ret_data[header[5]] = "{:.3f}".format( self.expected_throughput[fwd_config][frame_size][nb_desc]) - + diff = (self.throughput[fwd_config][frame_size][nb_desc] - + self.expected_throughput[fwd_config][frame_size][nb_desc])/ \ + self.expected_throughput[fwd_config][frame_size][nb_desc] + ret_data[header[6]] = "{:.3f}%".format(diff) + if diff >= -self.gap and diff <= self.gap: + ret_data[header[7]] = 'PASS' + else: + ret_data[header[7]] = 'FAIL' ret_datas[frame_size][nb_desc] = deepcopy(ret_data) self.test_result[fwd_config] = deepcopy(ret_datas) @@ -350,15 +380,11 @@ class TestNicSingleCorePerf(TestCase): row_dict0 = dict() row_dict0['performance'] = list() row_dict0['parameters'] = list() - result_throughput = float(row_in['Throughput'].split()[0]) - expected_throughput = float(row_in['Expected Throughput'].split()[0]) - # delta value and accepted tolerance in percentage - delta = result_throughput - expected_throughput - if delta > -self.gap: - row_dict0['status'] = 'PASS' - else: - row_dict0['status'] = 'FAIL' - row_dict1 = dict(name="Throughput", value=result_throughput, unit="Mpps", delta=delta) + result_throughput = float(row_in['Real-Mpps'].split()[0]) + expected_throughput = float(row_in['Expected-Mpps'].split()[0]) + row_dict0['status'] = row_in['Status'] + row_dict1 = dict(name="Throughput", value=result_throughput, unit="Mpps", + delta=row_in['Real vs Expected'], expected=expected_throughput) row_dict2 = dict(name="Txd/Rxd", value=row_in["TXD/RXD"], unit="descriptor") row_dict3 = dict(name="frame_size", value=row_in["Frame Size"], unit="bytes") row_dict4 = dict(name="Fwd_core", value=row_in["Fwd_core"]) @@ -368,10 +394,10 @@ class TestNicSingleCorePerf(TestCase): row_dict0['parameters'].append(row_dict4) json_obj[case_name].append(row_dict0) status_result.append(row_dict0['status']) - with open(os.path.join(rst.path2Result, - '{0:s}_single_core_perf.json'.format( - self.nic)), 'w') as fp: - json.dump(json_obj, fp) + + json_file = os.path.join(rst.path2Result, '{0:s}_single_core_perf.json'.format(self.nic)) + with open(json_file, 'w') as fp: + json.dump(json_obj, fp, indent=4, separators=(',', ': '), sort_keys=True) self.verify("FAIL" not in status_result, "Excessive gap between test results and expectations") def set_fields(self): -- 2.25.1