* [dts] [PATCH v1] tests/virtio_user_loopback_ci: add virtio loopback performance test
@ 2020-01-08 1:31 Yinan
0 siblings, 0 replies; only message in thread
From: Yinan @ 2020-01-08 1:31 UTC (permalink / raw)
To: dts; +Cc: Wang Yinan
From: Wang Yinan <yinan.wang@intel.com>
Add virtio-user loopback test for performance track in CI
Signed-off-by: Wang Yinan <yinan.wang@intel.com>
---
conf/virtio_user_loopback_ci.cfg | 13 +
tests/TestSuite_virtio_user_loopback_ci.py | 401 +++++++++++++++++++++
2 files changed, 414 insertions(+)
create mode 100644 conf/virtio_user_loopback_ci.cfg
create mode 100644 tests/TestSuite_virtio_user_loopback_ci.py
diff --git a/conf/virtio_user_loopback_ci.cfg b/conf/virtio_user_loopback_ci.cfg
new file mode 100644
index 0000000..2021eb8
--- /dev/null
+++ b/conf/virtio_user_loopback_ci.cfg
@@ -0,0 +1,13 @@
+[suite]
+update_expected = True
+test_parameters = {64: [1024], 1518: [1024]}
+test_duration = 60
+accepted_tolerance = 1
+expected_throughput = {'inorder_mergeable_on': {64: {1024: 7.394}, 1518: {1024: 4.084}}, 'virtio_mergeable_off': {64: {1024: 26.021}, 1518: {1024: 26.021}}, 'virtio_mergeable_on': {64: {1024: 7.321}, 1518: {1024: 4.05}}, 'virtio_vector_rx': {64: {1024: 8.944}, 1518: {1024: 4.19}}, 'virtio_1.1_normal': {64: {1024: 6.847}, 1518: {1024: 3.929}}, 'virtio_1.1_mergeable_on': {64: {1024: 6.818}, 1518: {1024: 3.883}}, 'inorder_mergeable_off': {64: {1024: 7.91}, 1518: {1024: 4.19}}}
+throughput_nnt = {64: {128: 53.435, 512: 53.699, 2048: 42.798}}
+throughput_fvl25g = {64: {512: 43.777, 2048: 43.651}}
+throughput_fvl40g = {64: {512: 62.35, 2048: 47.89}}
+throughput_cx5 = {64: {128: 42.161, 256: 56.651, 512: 47.091, 2048: 40.104}}
+throughput_cx4lx25g = {64: {128: 28.178, 256: 34.581, 512: 30.528, 2048: 26.004}}
+throughput_cx4lx40g = {64: {128: 31.635, 256: 32.473, 512: 30.72, 2048: 26.94}}
+
diff --git a/tests/TestSuite_virtio_user_loopback_ci.py b/tests/TestSuite_virtio_user_loopback_ci.py
new file mode 100644
index 0000000..a44f7e8
--- /dev/null
+++ b/tests/TestSuite_virtio_user_loopback_ci.py
@@ -0,0 +1,401 @@
+# BSD LICENSE
+#
+# Copyright(c) 2019-2020 Intel Corporation. All rights reserved.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in
+# the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Intel Corporation nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""
+DPDK Test suite.
+Test vhost/virtio-user loopback multi-queues on 7 tx/rx path.
+Includes Mergeable, Normal, Vector_RX, Inorder mergeable,
+Inorder no-mergeable, Virtio 1.1 mergeable, Virtio 1.1 no-mergeable Path.
+"""
+
+import utils
+import time
+import re
+import json
+import rst
+import os
+from copy import deepcopy
+from test_case import TestCase
+from settings import UPDATE_EXPECTED, load_global_setting
+
+
+class TestLoopbackMultiQueues(TestCase):
+
+ def set_up_all(self):
+ """
+ Run at the start of each test suite.
+ """
+ self.frame_sizes = [64, 1518]
+ self.nb_ports = 1
+ self.nb_cores = 1
+ self.queue_number = 1
+ cores_num = len(set([int(core['socket']) for core in self.dut.cores]))
+ # set diff arg about mem_socket base on socket number
+ self.socket_mem = ','.join(['1024']*cores_num)
+
+ self.core_config = "1S/4C/1T"
+ self.verify(len(self.core_config) >= 4,
+ "There has not enought cores to test this suite")
+ self.core_list = self.dut.get_core_list(self.core_config)
+ self.core_list_user = self.core_list[0:2]
+ self.core_list_host = self.core_list[2:4]
+
+ self.vhost = self.dut.new_session(suite="vhost")
+ self.virtio_user = self.dut.new_session(suite="virtio-user")
+ self.save_result_flag = True
+
+ def set_up(self):
+ """
+ Run before each test case.
+ It's more convenient to load suite configuration here than
+ set_up_all in debug mode.
+ """
+ self.dut.send_expect('rm ./vhost-net*', '# ')
+ # test parameters include: frames size, descriptor numbers
+ self.test_parameters = self.get_suite_cfg()['test_parameters']
+
+ # traffic duraion in second
+ self.test_duration = self.get_suite_cfg()['test_duration']
+
+ # initilize throughput attribution
+ # {'$framesize':{"$nb_desc": 'throughput'}
+ self.throughput = {}
+
+ # Accepted tolerance in Mpps
+ self.gap = self.get_suite_cfg()['accepted_tolerance']
+
+ # header to print test result table
+ self.table_header = ['Frame Size', 'TXD/RXD', 'Throughput', 'Rate',
+ 'Expected Throughput', 'Throughput Difference']
+ self.test_result = {}
+ # get the frame_sizes from cfg file
+ if 'packet_sizes' in self.get_suite_cfg():
+ self.frame_sizes = self.get_suite_cfg()['packet_sizes']
+
+ def start_vhost_testpmd(self, nb_desc):
+ """
+ start testpmd on vhost
+ """
+ eal_params = self.dut.create_eal_parameters(cores=self.core_list_host,
+ no_pci=True, prefix='vhost')
+ command_line_client = self.dut.target + "/app/testpmd %s " + \
+ "--socket-mem %s --legacy-mem --vdev " + \
+ "'net_vhost0,iface=vhost-net,queues=%d' -- -i --nb-cores=%d " + \
+ "--rxq=%d --txq=%d --txd=%d --rxd=%d"
+ command_line_client = command_line_client % (
+ eal_params, self.socket_mem, self.queue_number,
+ self.nb_cores, self.queue_number, self.queue_number,
+ nb_desc, nb_desc)
+ self.vhost.send_expect(command_line_client, "testpmd> ", 120)
+ self.vhost.send_expect("set fwd mac", "testpmd> ", 120)
+
+ def start_virtio_testpmd(self, nb_desc, args):
+ """
+ start testpmd on virtio
+ """
+ eal_params = self.dut.create_eal_parameters(cores=self.core_list_user,
+ no_pci=True, prefix='virtio')
+ command_line_user = self.dut.target + "/app/testpmd %s " + \
+ " --socket-mem %s --legacy-mem " + \
+ "--vdev=net_virtio_user0,mac=00:01:02:03:04:05,path=./vhost-net,queues=%d,%s " + \
+ "-- -i %s --rss-ip --nb-cores=%d --rxq=%d --txq=%d --txd=%d --rxd=%d"
+ command_line_user = command_line_user % (
+ eal_params, self.socket_mem,
+ self.queue_number, args["version"], args["path"], self.nb_cores,
+ self.queue_number, self.queue_number, nb_desc, nb_desc)
+ self.virtio_user.send_expect(command_line_user, "testpmd> ", 120)
+ self.virtio_user.send_expect("set fwd mac", "testpmd> ", 120)
+ self.virtio_user.send_expect("start", "testpmd> ", 120)
+
+ def calculate_avg_throughput(self):
+ """
+ calculate the average throughput
+ """
+ results = 0.0
+ results_row = []
+ for i in range(10):
+ out = self.vhost.send_expect("show port stats all", "testpmd>", 60)
+ time.sleep(5)
+ lines = re.search("Rx-pps:\s*(\d*)", out)
+ result = lines.group(1)
+ results += float(result)
+ Mpps = results / (1000000 * 10)
+ self.verify(Mpps > 0, "port can not receive packets")
+ return Mpps
+
+ def send_and_verify(self, nb_desc, case_info):
+ """
+ start to send packets and calculate avg throughput
+ """
+ for frame_size in self.frame_sizes:
+ self.throughput[frame_size] = dict()
+ self.vhost.send_expect("set txpkts %d" % frame_size, "testpmd> ", 30)
+ self.vhost.send_expect("start tx_first 32", "testpmd> ", 30)
+ Mpps = self.calculate_avg_throughput()
+ self.verify(Mpps > 0, "%s can not receive packets of frame size %d" % (self.running_case, frame_size))
+ self.throughput[frame_size][nb_desc] = Mpps
+ self.logger.info("Trouthput of " +
+ "framesize: {}, rxd/txd: {} is :{} Mpps".format(
+ frame_size, nb_desc, Mpps))
+ self.vhost.send_expect("stop", "testpmd> ", 60)
+
+ def close_all_testpmd(self):
+ """
+ close all testpmd of vhost and virtio
+ """
+ self.vhost.send_expect("quit", "#", 60)
+ self.virtio_user.send_expect("quit", "#", 60)
+
+ def close_all_session(self):
+ """
+ close all session of vhost and vhost-user
+ """
+ self.dut.close_session(self.vhost)
+ self.dut.close_session(self.virtio_user)
+
+ def handle_expected(self):
+ """
+ Update expected numbers to configurate file: conf/$suite_name.cfg
+ """
+ if load_global_setting(UPDATE_EXPECTED) == "yes":
+ for frame_size in self.test_parameters.keys():
+ for nb_desc in self.test_parameters[frame_size]:
+ self.expected_throughput[frame_size][nb_desc] = round(self.throughput[frame_size][nb_desc],3)
+
+ def handle_results(self):
+ """
+ results handled process:
+ 1, save to self.test_results
+ 2, create test results table
+ 3, save to json file for Open Lab
+ """
+ header = self.table_header
+ for frame_size in self.test_parameters.keys():
+ wirespeed = self.wirespeed(self.nic, frame_size, self.nb_ports)
+ ret_datas = {}
+ for nb_desc in self.test_parameters[frame_size]:
+ ret_data = {}
+ ret_data[header[0]] = frame_size
+ ret_data[header[1]] = nb_desc
+ ret_data[header[2]] = "{:.3f} Mpps".format(
+ self.throughput[frame_size][nb_desc])
+ ret_data[header[3]] = "{:.3f}%".format(
+ self.throughput[frame_size][nb_desc] * 100 / wirespeed)
+ ret_data[header[4]] = "{:.3f} Mpps".format(
+ self.expected_throughput[frame_size][nb_desc])
+ ret_data[header[5]] = "{:.3f} Mpps".format(
+ self.throughput[frame_size][nb_desc] -
+ self.expected_throughput[frame_size][nb_desc])
+ ret_datas[nb_desc] = deepcopy(ret_data)
+ self.test_result[frame_size] = deepcopy(ret_datas)
+ # Create test results table
+ self.result_table_create(header)
+ for frame_size in self.test_parameters.keys():
+ for nb_desc in self.test_parameters[frame_size]:
+ table_row = list()
+ for i in range(len(header)):
+ table_row.append(
+ self.test_result[frame_size][nb_desc][header[i]])
+ self.result_table_add(table_row)
+ # present test results to screen
+ self.result_table_print()
+ # save test results as a file
+ if self.save_result_flag:
+ self.save_result(self.test_result)
+
+ def save_result(self, data):
+ '''
+ Saves the test results as a separated file named with
+ self.nic+_single_core_perf.json in output folder
+ if self.save_result_flag is True
+ '''
+ json_obj = dict()
+ case_name = self.running_case
+ json_obj[case_name] = list()
+ status_result = []
+ for frame_size in self.test_parameters.keys():
+ for nb_desc in self.test_parameters[frame_size]:
+ row_in = self.test_result[frame_size][nb_desc]
+ row_dict0 = dict()
+ row_dict0['performance'] = list()
+ row_dict0['parameters'] = list()
+ row_dict0['parameters'] = list()
+ result_throughput = float(row_in['Throughput'].split()[0])
+ expected_throughput = float(row_in['Expected Throughput'].split()[0])
+ # delta value and accepted tolerance in percentage
+ delta = result_throughput - expected_throughput
+ gap = expected_throughput * -self.gap * 0.01
+ delta=float(delta)
+ gap=float(gap)
+ self.logger.info("Accept tolerance are (Mpps) %f" % gap)
+ self.logger.info("Throughput Difference are (Mpps) %f" % delta)
+ if result_throughput > expected_throughput + gap:
+ row_dict0['status'] = 'PASS'
+ else:
+ row_dict0['status'] = 'FAIL'
+ self.verify(row_dict0['status'] == 'PASS', 'The throughput is not in correct range')
+ row_dict1 = dict(name="Throughput", value=result_throughput, unit="Mpps", delta=delta)
+ row_dict2 = dict(name="Txd/Rxd", value=row_in["TXD/RXD"], unit="descriptor")
+ row_dict3 = dict(name="frame_size", value=row_in["Frame Size"], unit="bytes")
+ row_dict0['performance'].append(row_dict1)
+ row_dict0['parameters'].append(row_dict2)
+ row_dict0['parameters'].append(row_dict3)
+ json_obj[case_name].append(row_dict0)
+ status_result.append(row_dict0['status'])
+ with open(os.path.join(rst.path2Result,
+ '{0:s}_vhost_loopback_performance_virtio_user.json'.format(
+ self.nic)), 'w') as fp:
+ json.dump(json_obj, fp)
+ self.verify("Fail" not in status_result, "Exceeded Gap")
+
+ def test_perf_loopback_multi_queue_virtio11_mergeable(self):
+ """
+ performance for Vhost PVP virtio 1.1 Mergeable Path.
+ """
+ virtio_pmd_arg = {"version": "in_order=0,packed_vq=1,mrg_rxbuf=1",
+ "path": "--tx-offloads=0x0 --enable-hw-vlan-strip"}
+ self.test_target = "virtio_1.1_mergeable_on"
+ self.expected_throughput = self.get_suite_cfg()['expected_throughput'][self.test_target]
+ nb_desc = self.test_parameters[64][0]
+ self.start_vhost_testpmd(nb_desc)
+ self.start_virtio_testpmd(nb_desc, virtio_pmd_arg)
+ self.send_and_verify(nb_desc, "virtio_1.1 mergeable on")
+ self.close_all_testpmd()
+ self.handle_expected()
+ self.handle_results()
+
+ def test_perf_loopback_multi_queue_virtio11_normal(self):
+ """
+ performance for Vhost PVP virtio1.1 Normal Path.
+ """
+ self.test_target = "virtio_1.1_normal"
+ virtio_pmd_arg = {"version": "in_order=0,packed_vq=1,mrg_rxbuf=0",
+ "path": "--tx-offloads=0x0 --enable-hw-vlan-strip"}
+ self.expected_throughput = self.get_suite_cfg()['expected_throughput'][self.test_target]
+ nb_desc = self.test_parameters[64][0]
+ self.start_vhost_testpmd(nb_desc)
+ self.start_virtio_testpmd(nb_desc, virtio_pmd_arg)
+ self.send_and_verify(nb_desc, "virtio_1.1 normal")
+ self.close_all_testpmd()
+ self.handle_expected()
+ self.handle_results()
+
+ def test_perf_loopback_multi_queue_inorder_mergeable(self):
+ """
+ performance for Vhost PVP In_order mergeable Path.
+ """
+ self.test_target = "inorder_mergeable_on"
+ virtio_pmd_arg = {"version": "packed_vq=0,in_order=1,mrg_rxbuf=1",
+ "path": "--tx-offloads=0x0 --enable-hw-vlan-strip"}
+ self.expected_throughput = self.get_suite_cfg()['expected_throughput'][self.test_target]
+ nb_desc = self.test_parameters[64][0]
+ self.start_vhost_testpmd(nb_desc)
+ self.start_virtio_testpmd(nb_desc, virtio_pmd_arg)
+ self.send_and_verify(nb_desc, "inorder mergeable on")
+ self.close_all_testpmd()
+ self.handle_expected()
+ self.handle_results()
+
+ def test_perf_loopback_multi_queue_inorder_no_mergeable(self):
+ """
+ performance for Vhost PVP In_order no_mergeable Path.
+ """
+ self.test_target = "inorder_mergeable_off"
+ virtio_pmd_arg = {"version": "packed_vq=0,in_order=1,mrg_rxbuf=0",
+ "path": "--tx-offloads=0x0 --enable-hw-vlan-strip"}
+ self.expected_throughput = self.get_suite_cfg()['expected_throughput'][self.test_target]
+ nb_desc = self.test_parameters[64][0]
+ self.start_vhost_testpmd(nb_desc)
+ self.start_virtio_testpmd(nb_desc, virtio_pmd_arg)
+ self.send_and_verify(nb_desc, "inorder mergeable off")
+ self.close_all_testpmd()
+ self.handle_expected()
+ self.handle_results()
+
+ def test_perf_loopback_mulit_queue_mergeable(self):
+ """
+ performance for Vhost PVP Mergeable Path.
+ """
+ self.test_target = "virtio_mergeable_on"
+ virtio_pmd_arg = {"version": "packed_vq=0,in_order=0,mrg_rxbuf=1",
+ "path": "--tx-offloads=0x0 --enable-hw-vlan-strip"}
+ self.expected_throughput = self.get_suite_cfg()['expected_throughput'][self.test_target]
+ nb_desc = self.test_parameters[64][0]
+ self.start_vhost_testpmd(nb_desc)
+ self.start_virtio_testpmd(nb_desc, virtio_pmd_arg)
+ self.send_and_verify(nb_desc, "virtio mergeable")
+ self.close_all_testpmd()
+ self.handle_expected()
+ self.handle_results()
+
+ def test_pef_loopback_multi_queue_normal(self):
+ """
+ performance for Vhost PVP Normal Path.
+ """
+ self.test_target = "virtio_mergeable_off"
+ virtio_pmd_arg = {"version": "packed_vq=0,in_order=0,mrg_rxbuf=0",
+ "path": "--tx-offloads=0x0 --enable-hw-vlan-strip"}
+ self.expected_throughput = self.get_suite_cfg()['expected_throughput'][self.test_target]
+ nb_desc = self.test_parameters[64][0]
+ self.start_vhost_testpmd(nb_desc)
+ self.start_virtio_testpmd(nb_desc, virtio_pmd_arg)
+ self.send_and_verify(nb_desc, "virtio mergeable off")
+ self.close_all_testpmd()
+ self.handle_expected()
+ self.handle_results()
+
+ def test_perf_loopback_multi_queue_vector_rx(self):
+ """
+ performance for Vhost PVP Vector_RX Path
+ """
+ self.test_target = "virtio_vector_rx"
+ virtio_pmd_arg = {"version": "packed_vq=0,in_order=0,mrg_rxbuf=0",
+ "path": "--tx-offloads=0x0"}
+ self.expected_throughput = self.get_suite_cfg()['expected_throughput'][self.test_target]
+ nb_desc = self.test_parameters[64][0]
+ self.start_vhost_testpmd(nb_desc)
+ self.start_virtio_testpmd(nb_desc, virtio_pmd_arg)
+ self.send_and_verify(nb_desc, "virtio vector rx")
+ self.close_all_testpmd()
+ self.handle_expected()
+ self.handle_results()
+
+ def tear_down(self):
+ """
+ Run after each test case.
+ """
+ self.dut.kill_all()
+
+ def tear_down_all(self):
+ """
+ Run after each test suite.
+ """
+ self.close_all_session()
--
2.17.1
^ permalink raw reply [flat|nested] only message in thread
only message in thread, other threads:[~2020-01-08 8:36 UTC | newest]
Thread overview: (only message) (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2020-01-08 1:31 [dts] [PATCH v1] tests/virtio_user_loopback_ci: add virtio loopback performance test Yinan
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).