From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id BD7FAA0563; Wed, 4 Mar 2020 03:49:40 +0100 (CET) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id B371037AF; Wed, 4 Mar 2020 03:49:40 +0100 (CET) Received: from mga18.intel.com (mga18.intel.com [134.134.136.126]) by dpdk.org (Postfix) with ESMTP id 025032C38 for ; Wed, 4 Mar 2020 03:49:38 +0100 (CET) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga005.fm.intel.com ([10.253.24.32]) by orsmga106.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 03 Mar 2020 18:49:38 -0800 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.70,511,1574150400"; d="scan'208";a="438968483" Received: from fmsmsx108.amr.corp.intel.com ([10.18.124.206]) by fmsmga005.fm.intel.com with ESMTP; 03 Mar 2020 18:49:37 -0800 Received: from shsmsx602.ccr.corp.intel.com (10.109.6.142) by FMSMSX108.amr.corp.intel.com (10.18.124.206) with Microsoft SMTP Server (TLS) id 14.3.439.0; Tue, 3 Mar 2020 18:49:37 -0800 Received: from shsmsx603.ccr.corp.intel.com (10.109.6.143) by SHSMSX602.ccr.corp.intel.com (10.109.6.142) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256) id 15.1.1713.5; Wed, 4 Mar 2020 10:49:35 +0800 Received: from shsmsx603.ccr.corp.intel.com ([10.109.6.143]) by SHSMSX603.ccr.corp.intel.com ([10.109.6.143]) with mapi id 15.01.1713.004; Wed, 4 Mar 2020 10:49:35 +0800 From: "Ma, LihongX" To: "dts@dpdk.org" Thread-Topic: [dts][PATCH V2 2/2] tests/pvp_multi_paths_virtio_single_core_performance: modify code to support per patch performance Thread-Index: AQHV8c9pjYGnOKZRvEWsiJ8nzN+i9Kg3u3Eg Date: Wed, 4 Mar 2020 02:49:35 +0000 Message-ID: <6cf2b5f09136449fb4a824bd5a836462@intel.com> References: <1583263176-25645-1-git-send-email-lihongx.ma@intel.com> <1583263176-25645-3-git-send-email-lihongx.ma@intel.com> In-Reply-To: <1583263176-25645-3-git-send-email-lihongx.ma@intel.com> Accept-Language: en-US Content-Language: en-US X-MS-Has-Attach: X-MS-TNEF-Correlator: x-originating-ip: [10.239.127.36] Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Subject: Re: [dts] [PATCH V2 2/2] tests/pvp_multi_paths_virtio_single_core_performance: modify code to support per patch performance X-BeenThere: dts@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: test suite reviews and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dts-bounces@dpdk.org Sender: "dts" Tested-by: ma,lihong Regards, Ma,lihong -----Original Message----- From: Ma, LihongX=20 Sent: Wednesday, March 4, 2020 3:20 AM To: dts@dpdk.org Cc: Ma, LihongX Subject: [dts][PATCH V2 2/2] tests/pvp_multi_paths_virtio_single_core_perfo= rmance: modify code to support per patch performance Signed-off-by: lihong --- ...p_multi_paths_virtio_single_core_performance.py | 212 +++++++++++++++++= +--- 1 file changed, 190 insertions(+), 22 deletions(-) diff --git a/tests/TestSuite_pvp_multi_paths_virtio_single_core_performance= .py b/tests/TestSuite_pvp_multi_paths_virtio_single_core_performance.py index 8c5fb43..ccf14fc 100644 --- a/tests/TestSuite_pvp_multi_paths_virtio_single_core_performance.py +++ b/tests/TestSuite_pvp_multi_paths_virtio_single_core_performance.py @@ -34,10 +34,15 @@ DPDK Test suite. Test PVP virtio single core performance using virtio_user on 8 tx/rx path. """ =20 +import json +import rst +import os import utils from test_case import TestCase from packet import Packet from pktgen import PacketGeneratorHelper +from settings import UPDATE_EXPECTED, load_global_setting from copy=20 +import deepcopy =20 =20 class TestPVPMultiPathVirtioPerformance(TestCase): @@ -63,39 +68,59 @@ class TestPVPMultiPathVirtioPerformance(TestCase): self.tester.send_expect('mkdir -p %s' % self.out_path, '# ') # create an instance to set stream field setting self.pktgen_helper =3D PacketGeneratorHelper() + self.vhost_user =3D self.dut.new_session(suite=3D"user") + self.vhost =3D self.dut.new_session(suite=3D"vhost") + self.save_result_flag =3D True + self.json_obj =3D {} =20 def set_up(self): """ Run before each test case. """ - self.vhost_user =3D self.dut.new_session(suite=3D"user") - self.vhost =3D self.dut.new_session(suite=3D"vhost") # Prepare the result table self.table_header =3D ['Frame'] - self.table_header.append("Mode") + self.table_header.append("Mode/RXD-TXD") self.table_header.append("Mpps") self.table_header.append("% linerate") self.result_table_create(self.table_header) =20 + self.test_parameters =3D self.get_suite_cfg()['test_parameters'] + # test parameters include: frames size, descriptor numbers + self.test_parameters =3D self.get_suite_cfg()['test_parameters'] + + # traffic duraion in second + self.test_duration =3D self.get_suite_cfg()['test_duration'] + + # initilize throughput attribution + # {'$framesize':{"$nb_desc": 'throughput'} + self.throughput =3D {} + + # Accepted tolerance in Mpps + self.gap =3D self.get_suite_cfg()['accepted_tolerance'] + self.test_result =3D {} + self.nb_desc =3D self.test_parameters[64][0] + def send_and_verify(self, case_info): """ Send packet with packet generator and verify """ for frame_size in self.frame_sizes: tgen_input =3D [] - for port in range(self.number_of_ports): - rx_port =3D self.tester.get_local_port( - self.dut_ports[port % self.number_of_ports]) - tx_port =3D self.tester.get_local_port( - self.dut_ports[(port) % self.number_of_ports]) - destination_mac =3D self.dut.get_mac_address( - self.dut_ports[(port) % self.number_of_ports]) + self.throughput[frame_size] =3D dict() + self.logger.info("Test running at parameters: " + + "framesize: {}, rxd/txd: {}".format(frame_size, self.nb_de= sc)) + rx_port =3D self.tester.get_local_port( + self.dut_ports[0]) + tx_port =3D self.tester.get_local_port( + self.dut_ports[0]) + destination_mac =3D self.dut.get_mac_address( + self.dut_ports[0]) =20 - pkt =3D Packet(pkt_type=3D'UDP', pkt_len=3Dframe_size) - pkt.config_layer('ether', {'dst': '%s' % destination_mac}) - pkt.save_pcapfile(self.tester, "%s/multi_path_%d.pcap" % (= self.out_path, port)) + pkt =3D Packet(pkt_type=3D'UDP', pkt_len=3Dframe_size) + pkt.config_layer('ether', {'dst': '%s' % destination_mac}) + pkt.save_pcapfile(self.tester, "%s/multi_path.pcap" %=20 + (self.out_path)) =20 - tgen_input.append((tx_port, rx_port, "%s/multi_path_%d.pca= p" % (self.out_path, port))) + tgen_input.append((tx_port, rx_port, "%s/multi_path.pcap" %=20 + (self.out_path))) =20 self.tester.pktgen.clear_streams() streams =3D self.pktgen_helper.prepare_stream_from_tginput(tge= n_input, 100, None, self.tester.pktgen) @@ -103,15 +128,16 @@ class TestPVP= MultiPathVirtioPerformance(TestCase): traffic_opt =3D {'delay': 5} _, pps =3D self.tester.pktgen.measure_throughput(stream_ids=3D= streams, options=3Dtraffic_opt) Mpps =3D pps / 1000000.0 - self.verify(Mpps > 0, "%s can not receive packets of frame siz= e %d" % (self.running_case, frame_size)) + self.verify(Mpps > 0.0, "%s can not receive packets of=20 + frame size %d" % (self.running_case, frame_size)) =20 - throughput =3D Mpps * 100 / \ + linerate =3D Mpps * 100 / \ float(self.wirespeed(self.nic, frame_size, self.n= umber_of_ports)) + self.throughput[frame_size][self.nb_desc] =3D Mpps =20 results_row =3D [frame_size] results_row.append(case_info) results_row.append(Mpps) - results_row.append(throughput) + results_row.append(linerate) self.result_table_add(results_row) =20 @property @@ -130,7 +156,8 @@ class TestPVPMultiPathVirtioPerformance(TestCase): eal_param =3D self.dut.create_eal_parameters(socket=3Dself.ports_s= ocket, cores=3Dself.core_list_host, prefix=3D'vhost', ports=3D[self.dut.ports= _info[self.dut_ports[0]]['pci']], vdevs=3D['net_vhost0,if= ace=3Dvhost-net,queues=3D1,client=3D0']) - command_line_client =3D "./%s/app/testpmd " % self.target + eal_pa= ram + ' -- -i --nb-cores=3D2 --txd=3D1024 --rxd=3D1024' + command_line_client =3D "./%s/app/testpmd " % self.target + eal_pa= ram + \ + ' -- -i --nb-cores=3D2 --txd=3D%d --rxd=3D%d' %=20 + (self.nb_desc, self.nb_desc) self.vhost.send_expect(command_line_client, "testpmd> ", 120) self.vhost.send_expect("set fwd io", "testpmd> ", 120) self.vhost.send_expect("start", "testpmd> ", 120) @@ -142,11 +169,= 108 @@ class TestPVPMultiPathVirtioPerformance(TestCase): eal_param =3D self.dut.create_eal_parameters(socket=3Dself.ports_s= ocket, cores=3Dself.core_list_user, prefix=3D'virtio', no_pci=3DTrue, vdevs= =3D['net_virtio_user0,mac=3D00:01:02:03:04:05,path=3D./vhost-net,%s' % args= ["version"]]) if self.check_2M_env: eal_param +=3D " --single-file-segments" - command_line_user =3D "./%s/app/testpmd " % self.target + eal_para= m + " -- -i %s --rss-ip --nb-cores=3D1 --txd=3D1024 --rxd=3D1024" % args["p= ath"] + command_line_user =3D "./%s/app/testpmd " % self.target + eal_para= m + \ + " -- -i %s --rss-ip --nb-cores=3D1 --txd=3D%d --rxd=3D%d" % ( + args["path"], self.nb_desc, self.nb_desc) self.vhost_user.send_expect(command_line_user, "testpmd> ", 120) self.vhost_user.send_expect("set fwd mac", "testpmd> ", 120) self.vhost_user.send_expect("start", "testpmd> ", 120) =20 + def handle_expected(self): + """ + Update expected numbers to configurate file: conf/$suite_name.cfg + """ + if load_global_setting(UPDATE_EXPECTED) =3D=3D "yes": + for frame_size in self.test_parameters.keys(): + for nb_desc in self.test_parameters[frame_size]: + self.expected_throughput[frame_size][nb_desc] =3D=20 + round(self.throughput[frame_size][nb_desc], 3) + + def handle_results(self): + """ + results handled process: + 1, save to self.test_results + 2, create test results table + 3, save to json file for Open Lab + """ + header =3D self.table_header + header.append("Expected Throughput") + header.append("Throughput Difference") + for frame_size in self.test_parameters.keys(): + wirespeed =3D self.wirespeed(self.nic, frame_size, self.number= _of_ports) + ret_datas =3D {} + for nb_desc in self.test_parameters[frame_size]: + ret_data =3D {} + ret_data[header[0]] =3D frame_size + ret_data[header[1]] =3D nb_desc + ret_data[header[2]] =3D "{:.3f} Mpps".format( + self.throughput[frame_size][nb_desc]) + ret_data[header[3]] =3D "{:.3f}%".format( + self.throughput[frame_size][nb_desc] * 100 / wires= peed) + ret_data[header[4]] =3D "{:.3f} Mpps".format( + self.expected_throughput[frame_size][nb_desc]) + ret_data[header[5]] =3D "{:.3f} Mpps".format( + self.throughput[frame_size][nb_desc] - + self.expected_throughput[frame_size][nb_desc]) + ret_datas[nb_desc] =3D deepcopy(ret_data) + self.test_result[frame_size] =3D deepcopy(ret_datas) + # Create test results table + self.result_table_create(header) + for frame_size in self.test_parameters.keys(): + for nb_desc in self.test_parameters[frame_size]: + table_row =3D list() + for i in range(len(header)): + table_row.append( + self.test_result[frame_size][nb_desc][header[i]]) + self.result_table_add(table_row) + # present test results to screen + self.result_table_print() + # save test results as a file + if self.save_result_flag: + self.save_result(self.test_result) + + def save_result(self, data): + ''' + Saves the test results as a separated file named with + self.nic+_perf_virtio_user_pvp.json in output folder + if self.save_result_flag is True + ''' + case_name =3D self.running_case + self.json_obj[case_name] =3D list() + status_result =3D [] + for frame_size in self.test_parameters.keys(): + for nb_desc in self.test_parameters[frame_size]: + row_in =3D self.test_result[frame_size][nb_desc] + row_dict0 =3D dict() + row_dict0['performance'] =3D list() + row_dict0['parameters'] =3D list() + row_dict0['parameters'] =3D list() + result_throughput =3D float(row_in['Mpps'].split()[0]) + expected_throughput =3D float(row_in['Expected Throughput'= ].split()[0]) + # delta value and accepted tolerance in percentage + delta =3D result_throughput - expected_throughput + gap =3D expected_throughput * -self.gap * 0.01 + delta =3D float(delta) + gap =3D float(gap) + self.logger.info("Accept tolerance are (Mpps) %f" % gap) + self.logger.info("Throughput Difference are (Mpps) %f" % d= elta) + if result_throughput > expected_throughput + gap: + row_dict0['status'] =3D 'PASS' + else: + row_dict0['status'] =3D 'FAIL' + row_dict1 =3D dict(name=3D"Throughput", value=3Dresult_thr= oughput, unit=3D"Mpps", delta=3Ddelta) + row_dict2 =3D dict(name=3D"Txd/Rxd", value=3Drow_in["Mode/= RXD-TXD"], unit=3D"descriptor") + row_dict3 =3D dict(name=3D"frame_size", value=3Drow_in["Fr= ame"], unit=3D"bytes") + row_dict0['performance'].append(row_dict1) + row_dict0['parameters'].append(row_dict2) + row_dict0['parameters'].append(row_dict3) + self.json_obj[case_name].append(row_dict0) + status_result.append(row_dict0['status']) + with open(os.path.join(rst.path2Result, + '{0:s}_{1}.json'.format( + self.nic, self.suite_name)), 'w') as fp: + json.dump(self.json_obj, fp) + self.verify("FAIL" not in status_result, "Exceeded Gap") + def close_all_testpmd(self): """ close all testpmd of vhost and virtio @@ -165,119 +289,163 @@ clas= s TestPVPMultiPathVirtioPerformance(TestCase): """ performance for Vhost PVP virtio 1.1 Mergeable Path. """ + self.test_target =3D self.running_case + self.expected_throughput =3D=20 + self.get_suite_cfg()['expected_throughput'][self.test_target] virtio_pmd_arg =3D {"version": "packed_vq=3D1,in_order=3D0,mrg_rxb= uf=3D1", "path": "--tx-offloads=3D0x0 --enable-hw-vlan-st= rip"} self.start_vhost_testpmd() self.start_virtio_testpmd(virtio_pmd_arg) self.send_and_verify("virtio_1.1_mergeable on") self.close_all_testpmd() + self.logger.info('result of all framesize result') self.result_table_print() + self.handle_expected() + self.handle_results() =20 def test_perf_virtio_single_core_virtio11_normal(self): """ performance for Vhost PVP virtio1.1 Normal Path. """ + self.test_target =3D self.running_case + self.expected_throughput =3D=20 + self.get_suite_cfg()['expected_throughput'][self.test_target] virtio_pmd_arg =3D {"version": "packed_vq=3D1,in_order=3D0,mrg_rxb= uf=3D0", "path": "--tx-offloads=3D0x0 --enable-hw-vlan-st= rip"} self.start_vhost_testpmd() self.start_virtio_testpmd(virtio_pmd_arg) self.send_and_verify("virtio_1.1_normal") self.close_all_testpmd() + self.logger.info('result of all framesize result') self.result_table_print() + self.handle_expected() + self.handle_results() =20 def test_perf_virtio_single_core_virtio11_inorder_mergeable(self): """ performance for Vhost PVP virtio 1.1 inorder Mergeable Path. """ + self.test_target =3D self.running_case + self.expected_throughput =3D=20 + self.get_suite_cfg()['expected_throughput'][self.test_target] virtio_pmd_arg =3D {"version": "packed_vq=3D1,in_order=3D1,mrg_rxb= uf=3D1", "path": "--tx-offloads=3D0x0 --enable-hw-vlan-st= rip"} self.start_vhost_testpmd() self.start_virtio_testpmd(virtio_pmd_arg) self.send_and_verify("virtio_1.1_inorder_mergeable on") self.close_all_testpmd() + self.logger.info('result of all framesize result') self.result_table_print() + self.handle_expected() + self.handle_results() =20 def test_perf_virtio_single_core_virtio11_inorder_normal(self): """ performance for Vhost PVP virtio1.1 inorder Normal Path. """ + self.test_target =3D self.running_case + self.expected_throughput =3D=20 + self.get_suite_cfg()['expected_throughput'][self.test_target] virtio_pmd_arg =3D {"version": "packed_vq=3D1,in_order=3D1,mrg_rxb= uf=3D0", "path": "--tx-offloads=3D0x0 --enable-hw-vlan-st= rip"} self.start_vhost_testpmd() self.start_virtio_testpmd(virtio_pmd_arg) self.send_and_verify("virtio_1.1_inorder_normal") self.close_all_testpmd() + self.logger.info('result of all framesize result') self.result_table_print() + self.handle_expected() + self.handle_results() =20 def test_perf_virtio_single_core_inorder_mergeable(self): """ performance for Vhost PVP In_order mergeable Path. """ + self.test_target =3D self.running_case + self.expected_throughput =3D=20 + self.get_suite_cfg()['expected_throughput'][self.test_target] virtio_pmd_arg =3D {"version": "packed_vq=3D0,in_order=3D1,mrg_rxb= uf=3D1", "path": "--tx-offloads=3D0x0 --enable-hw-vlan-st= rip"} self.start_vhost_testpmd() self.start_virtio_testpmd(virtio_pmd_arg) self.send_and_verify("inoder mergeable on") self.close_all_testpmd() + self.logger.info('result of all framesize result') self.result_table_print() + self.handle_expected() + self.handle_results() =20 def test_perf_virtio_single_core_inorder_no_mergeable(self): """ performance for Vhost PVP In_order no_mergeable Path. """ + self.test_target =3D self.running_case + self.expected_throughput =3D=20 + self.get_suite_cfg()['expected_throughput'][self.test_target] virtio_pmd_arg =3D {"version": "packed_vq=3D0,in_order=3D1,mrg_rxb= uf=3D0", "path": "--tx-offloads=3D0x0 --enable-hw-vlan-st= rip"} self.start_vhost_testpmd() self.start_virtio_testpmd(virtio_pmd_arg) self.send_and_verify("inoder mergeable off") self.close_all_testpmd() + self.logger.info('result of all framesize result') self.result_table_print() + self.handle_expected() + self.handle_results() =20 def test_perf_virtio_single_core_mergeable(self): """ performance for Vhost PVP Mergeable Path. """ + self.test_target =3D self.running_case + self.expected_throughput =3D=20 + self.get_suite_cfg()['expected_throughput'][self.test_target] virtio_pmd_arg =3D {"version": "packed_vq=3D0,in_order=3D0,mrg_rxb= uf=3D1", "path": "--tx-offloads=3D0x0 --enable-hw-vlan-st= rip"} self.start_vhost_testpmd() self.start_virtio_testpmd(virtio_pmd_arg) self.send_and_verify("virito mergeable") self.close_all_testpmd() + self.logger.info('result of all framesize result') self.result_table_print() + self.handle_expected() + self.handle_results() =20 def test_perf_virtio_single_core_normal(self): """ performance for Vhost PVP Normal Path. """ + self.test_target =3D self.running_case + self.expected_throughput =3D=20 + self.get_suite_cfg()['expected_throughput'][self.test_target] virtio_pmd_arg =3D {"version": "packed_vq=3D0,in_order=3D0,mrg_rxb= uf=3D0", "path": "--tx-offloads=3D0x0 --enable-hw-vlan-st= rip"} self.start_vhost_testpmd() self.start_virtio_testpmd(virtio_pmd_arg) self.send_and_verify("virito normal") self.close_all_testpmd() + self.logger.info('result of all framesize result') self.result_table_print() + self.handle_expected() + self.handle_results() =20 def test_perf_virtio_single_core_vector_rx(self): """ performance for Vhost PVP Vector Path """ + self.test_target =3D self.running_case + self.expected_throughput =3D=20 + self.get_suite_cfg()['expected_throughput'][self.test_target] virtio_pmd_arg =3D {"version": "packed_vq=3D0,in_order=3D0,mrg_rxb= uf=3D0", "path": "--tx-offloads=3D0x0"} self.start_vhost_testpmd() self.start_virtio_testpmd(virtio_pmd_arg) self.send_and_verify("virtio vector rx") self.close_all_testpmd() + self.logger.info('result of all framesize result') self.result_table_print() + self.handle_expected() + self.handle_results() =20 def tear_down(self): """ Run after each test case. """ - self.dut.send_expect("killall -s INT testpmd", "#") - self.close_all_session() + self.dut.kill_all() =20 def tear_down_all(self): """ Run after each test suite. """ - pass + self.close_all_session() -- 2.7.4