From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 28BAEA056B; Tue, 3 Mar 2020 10:01:25 +0100 (CET) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 1CAB12C02; Tue, 3 Mar 2020 10:01:25 +0100 (CET) Received: from mga17.intel.com (mga17.intel.com [192.55.52.151]) by dpdk.org (Postfix) with ESMTP id F14BA3B5 for ; Tue, 3 Mar 2020 10:01:22 +0100 (CET) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga008.jf.intel.com ([10.7.209.65]) by fmsmga107.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 03 Mar 2020 01:01:21 -0800 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.70,510,1574150400"; d="scan'208";a="233555068" Received: from fmsmsx105.amr.corp.intel.com ([10.18.124.203]) by orsmga008.jf.intel.com with ESMTP; 03 Mar 2020 01:01:16 -0800 Received: from shsmsx603.ccr.corp.intel.com (10.109.6.143) by FMSMSX105.amr.corp.intel.com (10.18.124.203) with Microsoft SMTP Server (TLS) id 14.3.439.0; Tue, 3 Mar 2020 01:01:11 -0800 Received: from shsmsx603.ccr.corp.intel.com (10.109.6.143) by SHSMSX603.ccr.corp.intel.com (10.109.6.143) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256) id 15.1.1713.5; Tue, 3 Mar 2020 17:01:05 +0800 Received: from shsmsx603.ccr.corp.intel.com ([10.109.6.143]) by SHSMSX603.ccr.corp.intel.com ([10.109.6.143]) with mapi id 15.01.1713.004; Tue, 3 Mar 2020 17:01:05 +0800 From: "Ma, LihongX" To: "dts@dpdk.org" Thread-Topic: [dts][PATCH V2 2/2] tests/pvp_multi_paths_performance: modify code to support per patch performance Thread-Index: AQHV8TohlwIaJpW//EWmWEqXO0T5RKg2khPA Date: Tue, 3 Mar 2020 09:01:05 +0000 Message-ID: References: <1583199061-12136-1-git-send-email-lihongx.ma@intel.com> <1583199061-12136-3-git-send-email-lihongx.ma@intel.com> In-Reply-To: <1583199061-12136-3-git-send-email-lihongx.ma@intel.com> Accept-Language: en-US Content-Language: en-US X-MS-Has-Attach: X-MS-TNEF-Correlator: x-originating-ip: [10.239.127.36] Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Subject: Re: [dts] [PATCH V2 2/2] tests/pvp_multi_paths_performance: modify code to support per patch performance X-BeenThere: dts@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: test suite reviews and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dts-bounces@dpdk.org Sender: "dts" Tested-by: ma,lihong Regards, Ma,lihong -----Original Message----- From: Ma, LihongX=20 Sent: Tuesday, March 3, 2020 9:31 AM To: dts@dpdk.org Cc: Ma, LihongX Subject: [dts][PATCH V2 2/2] tests/pvp_multi_paths_performance: modify code= to support per patch performance Signed-off-by: lihong --- tests/TestSuite_pvp_multi_paths_performance.py | 217 +++++++++++++++++++++= +--- 1 file changed, 192 insertions(+), 25 deletions(-) diff --git a/tests/TestSuite_pvp_multi_paths_performance.py b/tests/TestSui= te_pvp_multi_paths_performance.py index 62fef12..092d98c 100644 --- a/tests/TestSuite_pvp_multi_paths_performance.py +++ b/tests/TestSuite_pvp_multi_paths_performance.py @@ -33,11 +33,15 @@ DPDK Test suite. Test PVP performance using virtio_user on 8 tx/rx path. """ - +import json +import rst +import os import utils from test_case import TestCase from packet import Packet from pktgen import PacketGeneratorHelper +from settings import UPDATE_EXPECTED, load_global_setting from copy=20 +import deepcopy =20 =20 class TestPVPMultiPathPerformance(TestCase): @@ -63,38 +67,58 @@ class TestPVPMultiPathPerformance(TestCase): self.tester.send_expect('mkdir -p %s' % self.out_path, '# ') # create an instance to set stream field setting self.pktgen_helper =3D PacketGeneratorHelper() + self.vhost_user =3D self.dut.new_session(suite=3D"user") + self.vhost =3D self.dut.new_session(suite=3D"vhost") + self.save_result_flag =3D True + self.json_obj =3D {} =20 def set_up(self): """ Run before each test case. """ - self.vhost_user =3D self.dut.new_session(suite=3D"user") - self.vhost =3D self.dut.new_session(suite=3D"vhost") # Prepare the result table self.table_header =3D ['Frame'] - self.table_header.append("Mode") + self.table_header.append("Mode/RXD-TXD") self.table_header.append("Mpps") self.table_header.append("% linerate") self.result_table_create(self.table_header) =20 + self.test_parameters =3D self.get_suite_cfg()['test_parameters'] + # test parameters include: frames size, descriptor numbers + self.test_parameters =3D self.get_suite_cfg()['test_parameters'] + + # traffic duraion in second + self.test_duration =3D self.get_suite_cfg()['test_duration'] + + # initilize throughput attribution + # {'$framesize':{"$nb_desc": 'throughput'} + self.throughput =3D {} + + # Accepted tolerance in Mpps + self.gap =3D self.get_suite_cfg()['accepted_tolerance'] + self.test_result =3D {} + self.nb_desc =3D self.test_parameters[64][0] + def send_and_verify(self, case_info): """ Send packet with packet generator and verify """ for frame_size in self.frame_sizes: tgen_input =3D [] - for port in range(self.number_of_ports): - rx_port =3D self.tester.get_local_port( - self.dut_ports[port % self.number_of_ports]) - tx_port =3D self.tester.get_local_port( - self.dut_ports[(port) % self.number_of_ports]) - destination_mac =3D self.dut.get_mac_address( - self.dut_ports[(port) % self.number_of_ports]) - - pkt =3D Packet(pkt_type=3D'UDP', pkt_len=3Dframe_size) - pkt.config_layer('ether', {'dst': '%s' % destination_mac}) - pkt.save_pcapfile(self.tester, "%s/multi_path_%d.pcap" % (= self.out_path, port)) - tgen_input.append((tx_port, rx_port, "%s/multi_path_%d.pca= p" % (self.out_path, port))) + self.throughput[frame_size] =3D dict() + self.logger.info("Test running at parameters: " + + "framesize: {}, rxd/txd: {}".format(frame_size, self.nb_de= sc)) + rx_port =3D self.tester.get_local_port( + self.dut_ports[0]) + tx_port =3D self.tester.get_local_port( + self.dut_ports[0]) + destination_mac =3D self.dut.get_mac_address( + self.dut_ports[0]) + + pkt =3D Packet(pkt_type=3D'UDP', pkt_len=3Dframe_size) + pkt.config_layer('ether', {'dst': '%s' % destination_mac}) + pkt.save_pcapfile(self.tester, "%s/multi_path.pcap" % (self.ou= t_path)) + tgen_input.append((tx_port, rx_port, "%s/multi_path.pcap" %=20 + (self.out_path))) =20 self.tester.pktgen.clear_streams() streams =3D self.pktgen_helper.prepare_stream_from_tginput(tge= n_input, 100, None, self.tester.pktgen) @@ -102,15 +126,16 @@ class TestPVP= MultiPathPerformance(TestCase): traffic_opt =3D {'delay': 5} _, pps =3D self.tester.pktgen.measure_throughput(stream_ids=3D= streams, options=3Dtraffic_opt) Mpps =3D pps / 1000000.0 - self.verify(Mpps > 0, "%s can not receive packets of frame siz= e %d" % (self.running_case, frame_size)) + self.verify(Mpps > 0.0, "%s can not receive packets of=20 + frame size %d" % (self.running_case, frame_size)) =20 - throughput =3D Mpps * 100 / \ - float(self.wirespeed(self.nic, frame_size, self.n= umber_of_ports)) + self.throughput[frame_size][self.nb_desc] =3D Mpps + linerate =3D Mpps * 100 / \ + float(self.wirespeed(self.nic, frame_size,=20 + self.number_of_ports)) =20 results_row =3D [frame_size] results_row.append(case_info) results_row.append(Mpps) - results_row.append(throughput) + results_row.append(linerate) self.result_table_add(results_row) =20 @property @@ -129,7 +154,8 @@ class TestPVPMultiPathPerformance(TestCase): eal_param =3D self.dut.create_eal_parameters(cores=3Dself.core_lis= t_host, prefix=3D'vhost', ports=3D[self.dut.ports= _info[self.dut_ports[0]]['pci']], vdevs=3D['net_vhost0,if= ace=3Dvhost-net,queues=3D1,client=3D0']) - command_line_client =3D "./%s/app/testpmd " % self.target + eal_pa= ram + " -- -i --nb-cores=3D1 --txd=3D1024 --rxd=3D1024" + command_line_client =3D "./%s/app/testpmd " % self.target + eal_pa= ram + \ + " -- -i --nb-cores=3D1 --txd=3D%d --rxd=3D%d" %=20 + (self.nb_desc, self.nb_desc) self.vhost.send_expect(command_line_client, "testpmd> ", 120) self.vhost.send_expect("set fwd mac", "testpmd> ", 120) self.vhost.send_expect("start", "testpmd> ", 120) @@ -144,13 +170,= 109 @@ class TestPVPMultiPathPerformance(TestCase): args["version"]]= ) if self.check_2M_env: eal_param +=3D " --single-file-segments" - command_line_user =3D "./%s/app/testpmd " % self.target + eal_para= m + " -- -i %s --rss-ip --nb-cores=3D1 --txd=3D1024 --rxd=3D1024" % \ - args["path"] + command_line_user =3D "./%s/app/testpmd " % self.target + eal_para= m + \ + " -- -i %s --rss-ip --nb-cores=3D1 --txd=3D%d --rxd=3D= %d" % \ + (args["path"], self.nb_desc, self.nb_desc) self.vhost_user =3D self.dut.new_session(suite=3D"user") self.vhost_user.send_expect(command_line_user, "testpmd> ", 120) self.vhost_user.send_expect("set fwd mac", "testpmd> ", 120) self.vhost_user.send_expect("start", "testpmd> ", 120) =20 + def handle_expected(self): + """ + Update expected numbers to configurate file: conf/$suite_name.cfg + """ + if load_global_setting(UPDATE_EXPECTED) =3D=3D "yes": + for frame_size in self.test_parameters.keys(): + for nb_desc in self.test_parameters[frame_size]: + self.expected_throughput[frame_size][nb_desc] =3D=20 + round(self.throughput[frame_size][nb_desc], 3) + + def handle_results(self): + """ + results handled process: + 1, save to self.test_results + 2, create test results table + 3, save to json file for Open Lab + """ + header =3D self.table_header + header.append("Expected Throughput") + header.append("Throughput Difference") + for frame_size in self.test_parameters.keys(): + wirespeed =3D self.wirespeed(self.nic, frame_size, self.number= _of_ports) + ret_datas =3D {} + for nb_desc in self.test_parameters[frame_size]: + ret_data =3D {} + ret_data[header[0]] =3D frame_size + ret_data[header[1]] =3D nb_desc + ret_data[header[2]] =3D "{:.3f} Mpps".format( + self.throughput[frame_size][nb_desc]) + ret_data[header[3]] =3D "{:.3f}%".format( + self.throughput[frame_size][nb_desc] * 100 / wires= peed) + ret_data[header[4]] =3D "{:.3f} Mpps".format( + self.expected_throughput[frame_size][nb_desc]) + ret_data[header[5]] =3D "{:.3f} Mpps".format( + self.throughput[frame_size][nb_desc] - + self.expected_throughput[frame_size][nb_desc]) + ret_datas[nb_desc] =3D deepcopy(ret_data) + self.test_result[frame_size] =3D deepcopy(ret_datas) + # Create test results table + self.result_table_create(header) + for frame_size in self.test_parameters.keys(): + for nb_desc in self.test_parameters[frame_size]: + table_row =3D list() + for i in range(len(header)): + table_row.append( + self.test_result[frame_size][nb_desc][header[i]]) + self.result_table_add(table_row) + # present test results to screen + self.result_table_print() + # save test results as a file + if self.save_result_flag: + self.save_result(self.test_result) + + def save_result(self, data): + ''' + Saves the test results as a separated file named with + self.nic+_perf_virtio_user_pvp.json in output folder + if self.save_result_flag is True + ''' + case_name =3D self.running_case + self.json_obj[case_name] =3D list() + status_result =3D [] + for frame_size in self.test_parameters.keys(): + for nb_desc in self.test_parameters[frame_size]: + row_in =3D self.test_result[frame_size][nb_desc] + row_dict0 =3D dict() + row_dict0['performance'] =3D list() + row_dict0['parameters'] =3D list() + row_dict0['parameters'] =3D list() + result_throughput =3D float(row_in['Mpps'].split()[0]) + expected_throughput =3D float(row_in['Expected Throughput'= ].split()[0]) + # delta value and accepted tolerance in percentage + delta =3D result_throughput - expected_throughput + gap =3D expected_throughput * -self.gap * 0.01 + delta =3D float(delta) + gap =3D float(gap) + self.logger.info("Accept tolerance are (Mpps) %f" % gap) + self.logger.info("Throughput Difference are (Mpps) %f" % d= elta) + if result_throughput > expected_throughput + gap: + row_dict0['status'] =3D 'PASS' + else: + row_dict0['status'] =3D 'FAIL' + row_dict1 =3D dict(name=3D"Throughput", value=3Dresult_thr= oughput, unit=3D"Mpps", delta=3Ddelta) + row_dict2 =3D dict(name=3D"Txd/Rxd", value=3Drow_in["Mode/= RXD-TXD"], unit=3D"descriptor") + row_dict3 =3D dict(name=3D"frame_size", value=3Drow_in["Fr= ame"], unit=3D"bytes") + row_dict0['performance'].append(row_dict1) + row_dict0['parameters'].append(row_dict2) + row_dict0['parameters'].append(row_dict3) + self.json_obj[case_name].append(row_dict0) + status_result.append(row_dict0['status']) + with open(os.path.join(rst.path2Result, + '{0:s}_{1}.json'.format( + self.nic, self.suite_name)), 'w') as fp: + json.dump(self.json_obj, fp) + self.verify("FAIL" not in status_result, "Exceeded Gap") + def close_all_testpmd(self): """ close all testpmd of vhost and virtio @@ -169,118 +291,163 @@ clas= s TestPVPMultiPathPerformance(TestCase): """ performance for PVP virtio 1.1 Mergeable Path. """ + self.test_target =3D self.running_case + self.expected_throughput =3D=20 + self.get_suite_cfg()['expected_throughput'][self.test_target] virtio_pmd_arg =3D {"version": "in_order=3D0,packed_vq=3D1,mrg_rxb= uf=3D1", "path": "--tx-offloads=3D0x0 --enable-hw-vlan-= strip"} self.start_vhost_testpmd() self.start_virtio_testpmd(virtio_pmd_arg) self.send_and_verify("virtio_1.1_mergeable on") self.close_all_testpmd() + self.logger.info('result of all framesize result') self.result_table_print() + self.handle_expected() + self.handle_results() =20 def test_perf_pvp_virtio11_normal(self): """ performance for PVP virtio1.1 Normal Path. """ + self.test_target =3D self.running_case + self.expected_throughput =3D=20 + self.get_suite_cfg()['expected_throughput'][self.test_target] virtio_pmd_arg =3D {"version": "in_order=3D0,packed_vq=3D1,mrg_rxb= uf=3D0", "path": "--tx-offloads=3D0x0 --enable-hw-vlan-= strip"} self.start_vhost_testpmd() self.start_virtio_testpmd(virtio_pmd_arg) self.send_and_verify("virtio_1.1_normal") self.close_all_testpmd() + self.logger.info('result of all framesize result') self.result_table_print() + self.handle_expected() + self.handle_results() =20 def test_perf_pvp_virtio11_inorder_mergeable(self): """ performance for PVP virtio 1.1 inorder Mergeable Path. """ + self.test_target =3D self.running_case + self.expected_throughput =3D=20 + self.get_suite_cfg()['expected_throughput'][self.test_target] virtio_pmd_arg =3D {"version": "in_order=3D1,packed_vq=3D1,mrg_rxb= uf=3D1", "path": "--tx-offloads=3D0x0 --enable-hw-vlan-= strip"} self.start_vhost_testpmd() self.start_virtio_testpmd(virtio_pmd_arg) self.send_and_verify("virtio_1.1_inorder_mergeable on") self.close_all_testpmd() + self.logger.info('result of all framesize result') self.result_table_print() + self.handle_expected() + self.handle_results() =20 def test_perf_pvp_virtio11_inorder_normal(self): """ performance for PVP virtio1.1 inorder Normal Path. """ + self.test_target =3D self.running_case + self.expected_throughput =3D=20 + self.get_suite_cfg()['expected_throughput'][self.test_target] virtio_pmd_arg =3D {"version": "in_order=3D1,packed_vq=3D1,mrg_rxb= uf=3D0", "path": "--tx-offloads=3D0x0 --enable-hw-vlan-= strip"} self.start_vhost_testpmd() self.start_virtio_testpmd(virtio_pmd_arg) self.send_and_verify("virtio_1.1_inorder_normal") self.close_all_testpmd() + self.logger.info('result of all framesize result') self.result_table_print() + self.handle_expected() + self.handle_results() =20 def test_perf_pvp_inorder_mergeable(self): """ performance for PVP In_order mergeable Path. """ + self.test_target =3D self.running_case + self.expected_throughput =3D=20 + self.get_suite_cfg()['expected_throughput'][self.test_target] virtio_pmd_arg =3D {"version": "packed_vq=3D0,in_order=3D1,mrg_rxb= uf=3D1", "path": "--tx-offloads=3D0x0 --enable-hw-vlan-= strip"} self.start_vhost_testpmd() self.start_virtio_testpmd(virtio_pmd_arg) self.send_and_verify("inoder mergeable on") self.close_all_testpmd() + self.logger.info('result of all framesize result') self.result_table_print() + self.handle_expected() + self.handle_results() =20 def test_perf_pvp_inorder_no_mergeable(self): """ performance for PVP In_order no_mergeable Path. """ + self.test_target =3D self.running_case + self.expected_throughput =3D=20 + self.get_suite_cfg()['expected_throughput'][self.test_target] virtio_pmd_arg =3D {"version": "packed_vq=3D0,in_order=3D1,mrg_rxb= uf=3D0", "path": "--tx-offloads=3D0x0 --enable-hw-vlan-stri= p"} self.start_vhost_testpmd() self.start_virtio_testpmd(virtio_pmd_arg) self.send_and_verify("inoder mergeable off") self.close_all_testpmd() + self.logger.info('result of all framesize result') self.result_table_print() + self.handle_expected() + self.handle_results() =20 def test_perf_pvp_mergeable(self): """ performance for PVP Mergeable Path. """ + self.test_target =3D self.running_case + self.expected_throughput =3D=20 + self.get_suite_cfg()['expected_throughput'][self.test_target] virtio_pmd_arg =3D {"version": "packed_vq=3D0,in_order=3D0,mrg_rxb= uf=3D1", "path": "--tx-offloads=3D0x0 --enable-hw-vlan-= strip"} self.start_vhost_testpmd() self.start_virtio_testpmd(virtio_pmd_arg) self.send_and_verify("virito mergeable") self.close_all_testpmd() + self.logger.info('result of all framesize result') self.result_table_print() + self.handle_expected() + self.handle_results() =20 def test_perf_pvp_normal(self): """ performance for PVP Normal Path. """ + self.test_target =3D self.running_case + self.expected_throughput =3D=20 + self.get_suite_cfg()['expected_throughput'][self.test_target] virtio_pmd_arg =3D {"version": "packed_vq=3D0,in_order=3D0,mrg_rxb= uf=3D0", "path": "--tx-offloads=3D0x0 --enable-hw-vlan-= strip"} self.start_vhost_testpmd() self.start_virtio_testpmd(virtio_pmd_arg) self.send_and_verify("virito normal") self.close_all_testpmd() + self.logger.info('result of all framesize result') self.result_table_print() + self.handle_expected() + self.handle_results() =20 def test_perf_pvp_vector_rx(self): """ performance for PVP Vector Path """ + self.test_target =3D self.running_case + self.expected_throughput =3D=20 + self.get_suite_cfg()['expected_throughput'][self.test_target] virtio_pmd_arg =3D {"version": "packed_vq=3D0,in_order=3D0,mrg_rxb= uf=3D0", "path": "--tx-offloads=3D0x0 "} self.start_vhost_testpmd() self.start_virtio_testpmd(virtio_pmd_arg) self.send_and_verify("virito vector rx") self.close_all_testpmd() + self.logger.info('result of all framesize result') self.result_table_print() + self.handle_expected() + self.handle_results() =20 def tear_down(self): """ Run after each test case. """ - self.dut.send_expect("killall -s INT testpmd", "#") - self.close_all_session() + self.dut.kill_all() =20 def tear_down_all(self): """ Run after each test suite. """ + self.close_all_session() -- 2.7.4