[dts] [PATCH V2 00/17] update mpps in virtio cases to check performance

[dts] [PATCH V2 00/17] update mpps in virtio cases to check performance

[Xiao Qimai]

*. change mpps check value based on nic linerate

Xiao Qimai (17):
  update mpps value to check perf
  update mpps value to check perf
  update mpps value to check perf
  update mpps value to check perf
  update mpps value to check perf
  update mpps value to check perf
  update mpps value to check perf
  update mpps value to check perf
  update mpps value to check perf
  update mpps value to check perf
  update mpps value to check perf
  update mpps value to check perf
  update mpps value to check perf
  update mpps value to check perf
  update mpps value to check perf
  update mpps value to check perf
  update mpps value to check perf

 .../TestSuite_loopback_multi_paths_port_restart.py |  5 +-
 tests/TestSuite_loopback_multi_queues.py           |  7 ++-
 .../TestSuite_loopback_virtio_user_server_mode.py  |  7 +--
 tests/TestSuite_pvp_diff_qemu_version.py           | 14 +++++-
 tests/TestSuite_pvp_multi_paths_performance.py     | 56 +++++++++++++---------
 ...vp_multi_paths_vhost_single_core_performance.py | 14 +++++-
 ...p_multi_paths_virtio_single_core_performance.py | 14 +++++-
 .../TestSuite_pvp_qemu_multi_paths_port_restart.py | 14 +++++-
 ...TestSuite_pvp_vhost_user_built_in_net_driver.py | 14 +++++-
 tests/TestSuite_pvp_virtio_user_2M_hugepages.py    | 14 +++++-
 tests/TestSuite_pvp_virtio_user_4k_pages.py        | 14 +++++-
 ...te_pvp_virtio_user_multi_queues_port_restart.py | 14 +++++-
 tests/TestSuite_vhost_dequeue_zero_copy.py         | 14 +++++-
 tests/TestSuite_vhost_multi_queue_qemu.py          | 17 +++++--
 tests/TestSuite_virtio_pvp_regression.py           | 14 +++++-
 ...stSuite_virtio_user_for_container_networking.py | 14 +++++-
 tests/TestSuite_vm2vm_virtio_pmd.py                |  7 +--
 17 files changed, 201 insertions(+), 52 deletions(-)

-- 
1.8.3.1

[dts] [PATCH V2 01/17]loopback_multi_paths_port_restart: update mpps value to check perf

[Xiao Qimai]

Signed-off-by: Xiao Qimai <qimaix.xiao@intel.com>
---
 tests/TestSuite_loopback_multi_paths_port_restart.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/TestSuite_loopback_multi_paths_port_restart.py b/tests/TestSuite_loopback_multi_paths_port_restart.py
index 8d23196..3f44232 100644
--- a/tests/TestSuite_loopback_multi_paths_port_restart.py
+++ b/tests/TestSuite_loopback_multi_paths_port_restart.py
@@ -152,14 +152,15 @@ class TestLoopbackPortRestart(TestCase):
         calculate the average throughput
         """
         results = 0.0
+        self.vhost.send_expect("show port stats all", "testpmd>", 60)
         for i in range(10):
             out = self.vhost.send_expect("show port stats all", "testpmd>", 60)
-            time.sleep(5)
+            time.sleep(1)
             lines = re.search("Rx-pps:\s*(\d*)", out)
             result = lines.group(1)
             results += float(result)
         Mpps = results / (1000000 * 10)
-        self.verify(Mpps > 0, "%s can not receive packets" % self.running_case)
+        self.verify(Mpps > 0.5, "%s can not receive packets" % self.running_case)
         return Mpps
 
     def send_and_verify(self, case_info, frame_size, restart_times=1):
-- 
1.8.3.1

[dts] [PATCH V2 02/17]loopback_multi_queues: update mpps value to check perf

[Xiao Qimai]

Signed-off-by: Xiao Qimai <qimaix.xiao@intel.com>
---
 tests/TestSuite_loopback_multi_queues.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/tests/TestSuite_loopback_multi_queues.py b/tests/TestSuite_loopback_multi_queues.py
index 029c1f0..10abb72 100644
--- a/tests/TestSuite_loopback_multi_queues.py
+++ b/tests/TestSuite_loopback_multi_queues.py
@@ -108,15 +108,15 @@ class TestLoopbackMultiQueues(TestCase):
         calculate the average throughput
         """
         results = 0.0
-        results_row = []
+        self.vhost.send_expect("show port stats all", "testpmd>", 60)
         for i in range(10):
             out = self.vhost.send_expect("show port stats all", "testpmd>", 60)
-            time.sleep(5)
+            time.sleep(1)
             lines = re.search("Rx-pps:\s*(\d*)", out)
             result = lines.group(1)
             results += float(result)
         Mpps = results / (1000000 * 10)
-        self.verify(Mpps > 0, "port can not receive packets")
+        self.verify(Mpps > 5, "port can not receive packets")
         return Mpps
 
     def update_result_table(self, frame_size, case_info, Mpps):
@@ -365,7 +365,6 @@ class TestLoopbackMultiQueues(TestCase):
             self.start_virtio_testpmd(virtio_pmd_arg)
             self.send_and_verify("virtio 1.1 inorder non-mergeable")
             self.close_all_testpmd()
-
         self.result_table_print()
         self.verify_liner_for_multi_queue()
 
-- 
1.8.3.1

[dts] [PATCH V2 03/17]loopback_virtio_user_server_mode: update mpps value to check perf

[Xiao Qimai]

Signed-off-by: Xiao Qimai <qimaix.xiao@intel.com>
---
 tests/TestSuite_loopback_virtio_user_server_mode.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/tests/TestSuite_loopback_virtio_user_server_mode.py b/tests/TestSuite_loopback_virtio_user_server_mode.py
index 32ee1c3..742aa09 100644
--- a/tests/TestSuite_loopback_virtio_user_server_mode.py
+++ b/tests/TestSuite_loopback_virtio_user_server_mode.py
@@ -111,7 +111,7 @@ class TestLoopbackVirtioUserServerMode(TestCase):
         start the testpmd of vhost-user and virtio-user
         start to send packets
         """
-        session_rx.send_expect("start", "testpmd> ", 30)
+        session_rx.send_command("start", 3)
         session_tx.send_expect("start tx_first 32", "testpmd> ", 30)
 
     def check_port_throughput_after_port_stop(self):
@@ -176,14 +176,15 @@ class TestLoopbackVirtioUserServerMode(TestCase):
         """
         results = 0.0
         results_row = []
+        self.vhost.send_expect("show port stats all", "testpmd>", 60)
         for i in range(10):
             out = self.vhost.send_expect("show port stats all", "testpmd>", 60)
-            time.sleep(5)
+            time.sleep(1)
             lines = re.search("Rx-pps:\s*(\d*)", out)
             result = lines.group(1)
             results += float(result)
         Mpps = results / (1000000 * 10)
-        self.verify(Mpps > 0, "port can not receive packets")
+        self.verify(Mpps > 5, "port can not receive packets")
 
         results_row.append(case_info)
         results_row.append('64')
-- 
1.8.3.1

[dts] [PATCH V2 04/17]pvp_diff_qemu_version: update mpps value to check perf

[Xiao Qimai]

Signed-off-by: Xiao Qimai <qimaix.xiao@intel.com>
---
 tests/TestSuite_pvp_diff_qemu_version.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/tests/TestSuite_pvp_diff_qemu_version.py b/tests/TestSuite_pvp_diff_qemu_version.py
index b9ce822..91b7bd3 100644
--- a/tests/TestSuite_pvp_diff_qemu_version.py
+++ b/tests/TestSuite_pvp_diff_qemu_version.py
@@ -81,6 +81,7 @@ class TestVhostPVPDiffQemuVersion(TestCase):
         self.pktgen_helper = PacketGeneratorHelper()
         self.base_dir = self.dut.base_dir.replace('~', '/root')
         self.pci_info = self.dut.ports_info[0]['pci']
+        self.number_of_ports = 1
 
     def set_up(self):
         """
@@ -220,6 +221,15 @@ class TestVhostPVPDiffQemuVersion(TestCase):
             self.vm_dut.send_expect("set fwd mac", "testpmd> ", 20)
             self.vm_dut.send_expect("start", "testpmd> ")
 
+    @property
+    def check_value(self):
+        check_dict = dict.fromkeys(self.frame_sizes)
+        linerate = {64: 0.07, 128: 0.10, 256: 0.17, 512: 0.18, 1024: 0.35, 1280: 0.40, 1500: 0.45}
+        for size in self.frame_sizes:
+            speed = self.wirespeed(self.nic, size, self.number_of_ports)
+            check_dict[size] = round(speed * linerate[size], 2)
+        return check_dict
+
     def send_verify(self, qemu_version, vlan_id1=0, tag="Performance"):
         self.result_table_create(self.header_row)
         for frame_size in self.frame_sizes:
@@ -243,7 +253,9 @@ class TestVhostPVPDiffQemuVersion(TestCase):
             _, pps = self.tester.pktgen.measure_throughput(stream_ids=streams, options=traffic_opt)
             Mpps = pps / 1000000.0
             pct = Mpps * 100 / float(self.wirespeed(self.nic, frame_size, 1))
-            self.verify(Mpps != 0, "can not received data of frame size %d" % frame_size)
+            self.verify(Mpps > self.check_value[frame_size],
+                        "%s of frame size %d speed verify failed, expect %s, result %s" % (
+                        self.running_case, frame_size, self.check_value[frame_size], Mpps))
             # update print table info
             data_row = [qemu_version, frame_size, str(Mpps), str(pct), tag]
             self.result_table_add(data_row)
-- 
1.8.3.1

[dts] [PATCH V2 05/17]pvp_multi_paths_performance: update mpps value to check perf

[Xiao Qimai]

Signed-off-by: Xiao Qimai <qimaix.xiao@intel.com>
---
 tests/TestSuite_pvp_multi_paths_performance.py | 56 +++++++++++++++-----------
 1 file changed, 33 insertions(+), 23 deletions(-)

diff --git a/tests/TestSuite_pvp_multi_paths_performance.py b/tests/TestSuite_pvp_multi_paths_performance.py
index 14a0fd7..3e6e593 100644
--- a/tests/TestSuite_pvp_multi_paths_performance.py
+++ b/tests/TestSuite_pvp_multi_paths_performance.py
@@ -99,6 +99,15 @@ class TestPVPMultiPathPerformance(TestCase):
         self.test_result = {}
         self.nb_desc = self.test_parameters[64][0]
 
+    @property
+    def check_value(self):
+        check_dict = dict.fromkeys(self.frame_sizes)
+        linerate = {64: 0.085, 128: 0.12, 256: 0.20, 512: 0.35, 1024: 0.50, 1280: 0.55, 1518: 0.60}
+        for size in self.frame_sizes:
+            speed = self.wirespeed(self.nic, size, self.number_of_ports)
+            check_dict[size] = round(speed * linerate[size], 2)
+        return check_dict
+
     def send_and_verify(self, case_info):
         """
         Send packet with packet generator and verify
@@ -108,7 +117,7 @@ class TestPVPMultiPathPerformance(TestCase):
             self.throughput[frame_size] = dict()
 
             self.logger.info("Test running at parameters: " +
-                "framesize: {}, rxd/txd: {}".format(frame_size, self.nb_desc))
+                             "framesize: {}, rxd/txd: {}".format(frame_size, self.nb_desc))
             rx_port = self.tester.get_local_port(
                 self.dut_ports[0])
             tx_port = self.tester.get_local_port(
@@ -124,11 +133,12 @@ class TestPVPMultiPathPerformance(TestCase):
             streams = self.pktgen_helper.prepare_stream_from_tginput(tgen_input, 100, None, self.tester.pktgen)
             _, pps = self.tester.pktgen.measure_throughput(stream_ids=streams)
             Mpps = pps / 1000000.0
-            self.verify(Mpps > 0.0, "%s can not receive packets of frame size %d" % (self.running_case, frame_size))
-
+            self.verify(Mpps > self.check_value[frame_size],
+                        "%s of frame size %d speed verify failed, expect %s, result %s" % (
+                        self.running_case, frame_size, self.check_value[frame_size], Mpps))
             self.throughput[frame_size][self.nb_desc] = Mpps
             linerate = Mpps * 100 / \
-                    float(self.wirespeed(self.nic, frame_size, self.number_of_ports))
+                       float(self.wirespeed(self.nic, frame_size, self.number_of_ports))
 
             results_row = [frame_size]
             results_row.append(case_info)
@@ -153,7 +163,7 @@ class TestPVPMultiPathPerformance(TestCase):
                                                    ports=[self.dut.ports_info[self.dut_ports[0]]['pci']],
                                                    vdevs=['net_vhost0,iface=vhost-net,queues=1,client=0'])
         command_line_client = "./%s/app/testpmd " % self.target + eal_param + \
-                    " -- -i --nb-cores=1 --txd=%d --rxd=%d" % (self.nb_desc, self.nb_desc)
+                              " -- -i --nb-cores=1 --txd=%d --rxd=%d" % (self.nb_desc, self.nb_desc)
         self.vhost.send_expect(command_line_client, "testpmd> ", 120)
         self.vhost.send_expect("set fwd mac", "testpmd> ", 120)
         self.vhost.send_expect("start", "testpmd> ", 120)
@@ -169,7 +179,7 @@ class TestPVPMultiPathPerformance(TestCase):
         if self.check_2M_env:
             eal_param += " --single-file-segments"
         command_line_user = "./%s/app/testpmd " % self.target + eal_param + \
-                    " -- -i %s --rss-ip --nb-cores=1 --txd=%d --rxd=%d" % \
+                            " -- -i %s --rss-ip --nb-cores=1 --txd=%d --rxd=%d" % \
                             (args["path"], self.nb_desc, self.nb_desc)
         self.vhost_user = self.dut.new_session(suite="user")
         self.vhost_user.send_expect(command_line_user, "testpmd> ", 120)
@@ -203,14 +213,14 @@ class TestPVPMultiPathPerformance(TestCase):
                 ret_data[header[0]] = frame_size
                 ret_data[header[1]] = nb_desc
                 ret_data[header[2]] = "{:.3f} Mpps".format(
-                        self.throughput[frame_size][nb_desc])
+                    self.throughput[frame_size][nb_desc])
                 ret_data[header[3]] = "{:.3f}%".format(
-                        self.throughput[frame_size][nb_desc] * 100 / wirespeed)
+                    self.throughput[frame_size][nb_desc] * 100 / wirespeed)
                 ret_data[header[4]] = "{:.3f} Mpps".format(
-                        self.expected_throughput[frame_size][nb_desc])
+                    self.expected_throughput[frame_size][nb_desc])
                 ret_data[header[5]] = "{:.3f} Mpps".format(
-                        self.throughput[frame_size][nb_desc] -
-                        self.expected_throughput[frame_size][nb_desc])
+                    self.throughput[frame_size][nb_desc] -
+                    self.expected_throughput[frame_size][nb_desc])
                 ret_datas[nb_desc] = deepcopy(ret_data)
             self.test_result[frame_size] = deepcopy(ret_datas)
         # Create test results table
@@ -266,8 +276,8 @@ class TestPVPMultiPathPerformance(TestCase):
                 self.json_obj[case_name].append(row_dict0)
                 status_result.append(row_dict0['status'])
         with open(os.path.join(rst.path2Result,
-                        '{0:s}_{1}.json'.format(
-                            self.nic, self.suite_name)), 'w') as fp:
+                               '{0:s}_{1}.json'.format(
+                                   self.nic, self.suite_name)), 'w') as fp:
             json.dump(self.json_obj, fp)
         self.verify("FAIL" not in status_result, "Exceeded Gap")
 
@@ -292,7 +302,7 @@ class TestPVPMultiPathPerformance(TestCase):
         self.test_target = self.running_case
         self.expected_throughput = self.get_suite_cfg()['expected_throughput'][self.test_target]
         virtio_pmd_arg = {"version": "in_order=0,packed_vq=1,mrg_rxbuf=1",
-                            "path": "--tx-offloads=0x0 --enable-hw-vlan-strip"}
+                          "path": "--tx-offloads=0x0 --enable-hw-vlan-strip"}
         self.start_vhost_testpmd()
         self.start_virtio_testpmd(virtio_pmd_arg)
         self.send_and_verify("virtio_1.1_mergeable on")
@@ -309,7 +319,7 @@ class TestPVPMultiPathPerformance(TestCase):
         self.test_target = self.running_case
         self.expected_throughput = self.get_suite_cfg()['expected_throughput'][self.test_target]
         virtio_pmd_arg = {"version": "in_order=0,packed_vq=1,mrg_rxbuf=0",
-                            "path": "--tx-offloads=0x0 --enable-hw-vlan-strip"}
+                          "path": "--tx-offloads=0x0 --enable-hw-vlan-strip"}
         self.start_vhost_testpmd()
         self.start_virtio_testpmd(virtio_pmd_arg)
         self.send_and_verify("virtio_1.1_normal")
@@ -326,7 +336,7 @@ class TestPVPMultiPathPerformance(TestCase):
         self.test_target = self.running_case
         self.expected_throughput = self.get_suite_cfg()['expected_throughput'][self.test_target]
         virtio_pmd_arg = {"version": "in_order=1,packed_vq=1,mrg_rxbuf=1",
-                            "path": "--tx-offloads=0x0 --enable-hw-vlan-strip"}
+                          "path": "--tx-offloads=0x0 --enable-hw-vlan-strip"}
         self.start_vhost_testpmd()
         self.start_virtio_testpmd(virtio_pmd_arg)
         self.send_and_verify("virtio_1.1_inorder_mergeable on")
@@ -343,7 +353,7 @@ class TestPVPMultiPathPerformance(TestCase):
         self.test_target = self.running_case
         self.expected_throughput = self.get_suite_cfg()['expected_throughput'][self.test_target]
         virtio_pmd_arg = {"version": "in_order=1,packed_vq=1,mrg_rxbuf=0,vectorized=1",
-                            "path": "--rx-offloads=0x10 --enable-hw-vlan-strip"}
+                          "path": "--rx-offloads=0x10 --enable-hw-vlan-strip"}
         self.start_vhost_testpmd()
         self.start_virtio_testpmd(virtio_pmd_arg)
         self.send_and_verify("virtio_1.1_inorder_normal")
@@ -360,7 +370,7 @@ class TestPVPMultiPathPerformance(TestCase):
         self.test_target = self.running_case
         self.expected_throughput = self.get_suite_cfg()['expected_throughput'][self.test_target]
         virtio_pmd_arg = {"version": "in_order=1,packed_vq=1,mrg_rxbuf=0,vectorized=1",
-                            "path": "--tx-offloads=0x0 --enable-hw-vlan-strip"}
+                          "path": "--tx-offloads=0x0 --enable-hw-vlan-strip"}
         self.start_vhost_testpmd()
         self.start_virtio_testpmd(virtio_pmd_arg)
         self.send_and_verify("virtio_1.1_inorder_normal")
@@ -377,7 +387,7 @@ class TestPVPMultiPathPerformance(TestCase):
         self.test_target = self.running_case
         self.expected_throughput = self.get_suite_cfg()['expected_throughput'][self.test_target]
         virtio_pmd_arg = {"version": "packed_vq=0,in_order=1,mrg_rxbuf=1",
-                            "path": "--tx-offloads=0x0 --enable-hw-vlan-strip"}
+                          "path": "--tx-offloads=0x0 --enable-hw-vlan-strip"}
         self.start_vhost_testpmd()
         self.start_virtio_testpmd(virtio_pmd_arg)
         self.send_and_verify("inoder mergeable on")
@@ -394,7 +404,7 @@ class TestPVPMultiPathPerformance(TestCase):
         self.test_target = self.running_case
         self.expected_throughput = self.get_suite_cfg()['expected_throughput'][self.test_target]
         virtio_pmd_arg = {"version": "packed_vq=0,in_order=1,mrg_rxbuf=0",
-                        "path": "--tx-offloads=0x0 --enable-hw-vlan-strip"}
+                          "path": "--tx-offloads=0x0 --enable-hw-vlan-strip"}
         self.start_vhost_testpmd()
         self.start_virtio_testpmd(virtio_pmd_arg)
         self.send_and_verify("inoder mergeable off")
@@ -411,7 +421,7 @@ class TestPVPMultiPathPerformance(TestCase):
         self.test_target = self.running_case
         self.expected_throughput = self.get_suite_cfg()['expected_throughput'][self.test_target]
         virtio_pmd_arg = {"version": "packed_vq=0,in_order=0,mrg_rxbuf=1",
-                            "path": "--tx-offloads=0x0 --enable-hw-vlan-strip"}
+                          "path": "--tx-offloads=0x0 --enable-hw-vlan-strip"}
         self.start_vhost_testpmd()
         self.start_virtio_testpmd(virtio_pmd_arg)
         self.send_and_verify("virito mergeable")
@@ -428,7 +438,7 @@ class TestPVPMultiPathPerformance(TestCase):
         self.test_target = self.running_case
         self.expected_throughput = self.get_suite_cfg()['expected_throughput'][self.test_target]
         virtio_pmd_arg = {"version": "packed_vq=0,in_order=0,mrg_rxbuf=0,vectorized=1",
-                            "path": "--tx-offloads=0x0 --enable-hw-vlan-strip"}
+                          "path": "--tx-offloads=0x0 --enable-hw-vlan-strip"}
         self.start_vhost_testpmd()
         self.start_virtio_testpmd(virtio_pmd_arg)
         self.send_and_verify("virito normal")
@@ -445,7 +455,7 @@ class TestPVPMultiPathPerformance(TestCase):
         self.test_target = self.running_case
         self.expected_throughput = self.get_suite_cfg()['expected_throughput'][self.test_target]
         virtio_pmd_arg = {"version": "packed_vq=0,in_order=0,mrg_rxbuf=0,vectorized=1",
-                            "path": "--tx-offloads=0x0 "}
+                          "path": "--tx-offloads=0x0 "}
         self.start_vhost_testpmd()
         self.start_virtio_testpmd(virtio_pmd_arg)
         self.send_and_verify("virito vector rx")
-- 
1.8.3.1

[dts] [PATCH V2 06/17]pvp_multi_paths_vhost_single_core_performance: update mpps value to check perf

[Xiao Qimai]

Signed-off-by: Xiao Qimai <qimaix.xiao@intel.com>
---
 ...tSuite_pvp_multi_paths_vhost_single_core_performance.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/tests/TestSuite_pvp_multi_paths_vhost_single_core_performance.py b/tests/TestSuite_pvp_multi_paths_vhost_single_core_performance.py
index cddd831..6f0f139 100644
--- a/tests/TestSuite_pvp_multi_paths_vhost_single_core_performance.py
+++ b/tests/TestSuite_pvp_multi_paths_vhost_single_core_performance.py
@@ -99,6 +99,15 @@ class TestPVPMultiPathVhostPerformance(TestCase):
         self.test_result = {}
         self.nb_desc = self.test_parameters[64][0]
 
+    @property
+    def check_value(self):
+        check_dict = dict.fromkeys(self.frame_sizes)
+        linerate = {64: 0.085, 128: 0.12, 256: 0.20, 512: 0.35, 1024: 0.50, 1280: 0.55, 1518: 0.60}
+        for size in self.frame_sizes:
+            speed = self.wirespeed(self.nic, size, self.number_of_ports)
+            check_dict[size] = round(speed * linerate[size], 2)
+        return check_dict
+
     def send_and_verify(self, case_info):
         """
         Send packet with packet generator and verify
@@ -123,8 +132,9 @@ class TestPVPMultiPathVhostPerformance(TestCase):
             streams = self.pktgen_helper.prepare_stream_from_tginput(tgen_input, 100, None, self.tester.pktgen)
             _, pps = self.tester.pktgen.measure_throughput(stream_ids=streams)
             Mpps = pps / 1000000.0
-            self.verify(Mpps > 0.0, "%s can not receive packets of frame size %d" % (self.running_case, frame_size))
-
+            self.verify(Mpps > self.check_value[frame_size],
+                        "%s of frame size %d speed verify failed, expect %s, result %s" % (
+                        self.running_case, frame_size, self.check_value[frame_size], Mpps))
             linerate = Mpps * 100 / \
                          float(self.wirespeed(self.nic, frame_size, self.number_of_ports))
             self.throughput[frame_size][self.nb_desc] = Mpps
-- 
1.8.3.1

[dts] [PATCH V2 07/17]pvp_multi_paths_virtio_single_core_performance: update mpps value to check perf

[Xiao Qimai]

Signed-off-by: Xiao Qimai <qimaix.xiao@intel.com>
---
 ...Suite_pvp_multi_paths_virtio_single_core_performance.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/tests/TestSuite_pvp_multi_paths_virtio_single_core_performance.py b/tests/TestSuite_pvp_multi_paths_virtio_single_core_performance.py
index b98048b..13ec493 100644
--- a/tests/TestSuite_pvp_multi_paths_virtio_single_core_performance.py
+++ b/tests/TestSuite_pvp_multi_paths_virtio_single_core_performance.py
@@ -100,6 +100,15 @@ class TestPVPMultiPathVirtioPerformance(TestCase):
         self.test_result = {}
         self.nb_desc = self.test_parameters[64][0]
 
+    @property
+    def check_value(self):
+        check_dict = dict.fromkeys(self.frame_sizes)
+        linerate = {64: 0.085, 128: 0.12, 256: 0.20, 512: 0.35, 1024: 0.50, 1280: 0.55, 1518: 0.60}
+        for size in self.frame_sizes:
+            speed = self.wirespeed(self.nic, size, self.number_of_ports)
+            check_dict[size] = round(speed * linerate[size], 2)
+        return check_dict
+
     def send_and_verify(self, case_info):
         """
         Send packet with packet generator and verify
@@ -126,8 +135,9 @@ class TestPVPMultiPathVirtioPerformance(TestCase):
             traffic_opt = {'delay': 5}
             _, pps = self.tester.pktgen.measure_throughput(stream_ids=streams, options=traffic_opt)
             Mpps = pps / 1000000.0
-            self.verify(Mpps > 0.0, "%s can not receive packets of frame size %d" % (self.running_case, frame_size))
-
+            self.verify(Mpps > self.check_value[frame_size],
+                        "%s of frame size %d speed verify failed, expect %s, result %s" % (
+                        self.running_case, frame_size, self.check_value[frame_size], Mpps))
             linerate = Mpps * 100 / \
                          float(self.wirespeed(self.nic, frame_size, self.number_of_ports))
             self.throughput[frame_size][self.nb_desc] = Mpps
-- 
1.8.3.1

[dts] [PATCH V2 08/17]pvp_qemu_multi_paths_port_restart: update mpps value to check perf

[Xiao Qimai]

Signed-off-by: Xiao Qimai <qimaix.xiao@intel.com>
---
 tests/TestSuite_pvp_qemu_multi_paths_port_restart.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/tests/TestSuite_pvp_qemu_multi_paths_port_restart.py b/tests/TestSuite_pvp_qemu_multi_paths_port_restart.py
index 601000e..877d40b 100644
--- a/tests/TestSuite_pvp_qemu_multi_paths_port_restart.py
+++ b/tests/TestSuite_pvp_qemu_multi_paths_port_restart.py
@@ -72,6 +72,7 @@ class TestPVPQemuMultiPathPortRestart(TestCase):
         # create an instance to set stream field setting
         self.pktgen_helper = PacketGeneratorHelper()
         self.pci_info = self.dut.ports_info[0]['pci']
+        self.number_of_ports = 1
 
     def set_up(self):
         """
@@ -197,6 +198,15 @@ class TestPVPQemuMultiPathPortRestart(TestCase):
         results_row.append(Cycle)
         self.result_table_add(results_row)
 
+    @property
+    def check_value(self):
+        check_dict = dict.fromkeys(self.frame_sizes)
+        linerate = {64: 0.075, 128: 0.10, 256: 0.10, 512: 0.20, 1024: 0.35, 1280: 0.40, 1518: 0.45}
+        for size in self.frame_sizes:
+            speed = self.wirespeed(self.nic, size, self.number_of_ports)
+            check_dict[size] = round(speed * linerate[size], 2)
+        return check_dict
+
     def calculate_avg_throughput(self, frame_size):
         """
         start to send packet and get the throughput
@@ -214,7 +224,9 @@ class TestPVPQemuMultiPathPortRestart(TestCase):
         traffic_opt = {'delay': 5}
         _, pps = self.tester.pktgen.measure_throughput(stream_ids=streams, options=traffic_opt)
         Mpps = pps / 1000000.0
-        self.verify(Mpps > 0, "can not receive packets of frame size %d" % (frame_size))
+        self.verify(Mpps > self.check_value[frame_size],
+                    "%s of frame size %d speed verify failed, expect %s, result %s" % (
+                        self.running_case, frame_size, self.check_value[frame_size], Mpps))
         throughput = Mpps * 100 / \
                     float(self.wirespeed(self.nic, frame_size, 1))
         return Mpps, throughput
-- 
1.8.3.1

[dts] [PATCH V2 09/17]pvp_vhost_user_built_in_net_driver: update mpps value to check perf

[Xiao Qimai]

Signed-off-by: Xiao Qimai <qimaix.xiao@intel.com>
---
 tests/TestSuite_pvp_vhost_user_built_in_net_driver.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/tests/TestSuite_pvp_vhost_user_built_in_net_driver.py b/tests/TestSuite_pvp_vhost_user_built_in_net_driver.py
index cfdd704..1b7d9c5 100644
--- a/tests/TestSuite_pvp_vhost_user_built_in_net_driver.py
+++ b/tests/TestSuite_pvp_vhost_user_built_in_net_driver.py
@@ -64,6 +64,7 @@ class TestPVPVhostUserBuiltInNetDriver(TestCase):
         self.mem_channels = self.dut.get_memory_channels()
         self.headers_size = HEADER_SIZE['eth'] + HEADER_SIZE['ip']
         self.prepare_vhost_switch()
+        self.number_of_ports = 1
 
         self.logger.info("You can config packet_size in file %s.cfg," % self.suite_name + \
                         " in region 'suite' like packet_sizes=[64, 128, 256]")
@@ -129,6 +130,15 @@ class TestPVPVhostUserBuiltInNetDriver(TestCase):
         self.verify("Error" not in out, "compilation l3fwd-power error")
         self.verify("No such" not in out, "Compilation error")
 
+    @property
+    def check_value(self):
+        check_dict = dict.fromkeys(self.frame_sizes)
+        linerate = {64: 0.085, 128: 0.12, 256: 0.20, 512: 0.35, 1024: 0.50, 1280: 0.55, 1518: 0.60}
+        for size in self.frame_sizes:
+            speed = self.wirespeed(self.nic, size, self.number_of_ports)
+            check_dict[size] = round(speed * linerate[size], 2)
+        return check_dict
+
     def send_and_verify(self):
         """
         Send packet with packet generator and verify
@@ -150,7 +160,9 @@ class TestPVPVhostUserBuiltInNetDriver(TestCase):
             trans_options={'delay':5, 'duration': 20}
             _, pps = self.tester.pktgen.measure_throughput(stream_ids=streams, options=trans_options)
             Mpps = pps / 1000000.0
-            self.verify(Mpps > 0, "%s can not receive packets of frame size %d" % (self.running_case, frame_size))
+            self.verify(Mpps > self.check_value[frame_size],
+                        "%s of frame size %d speed verify failed, expect %s, result %s" % (
+                            self.running_case, frame_size, self.check_value[frame_size], Mpps))
             throughput = Mpps * 100 / \
                      float(self.wirespeed(self.nic, frame_size, 1))
 
-- 
1.8.3.1

[dts] [PATCH V2 10/17]pvp_virtio_user_2M_hugepages: update mpps value to check perf

[Xiao Qimai]

Signed-off-by: Xiao Qimai <qimaix.xiao@intel.com>
---
 tests/TestSuite_pvp_virtio_user_2M_hugepages.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/tests/TestSuite_pvp_virtio_user_2M_hugepages.py b/tests/TestSuite_pvp_virtio_user_2M_hugepages.py
index 71df7d5..0e7af54 100644
--- a/tests/TestSuite_pvp_virtio_user_2M_hugepages.py
+++ b/tests/TestSuite_pvp_virtio_user_2M_hugepages.py
@@ -75,6 +75,7 @@ class TestPVPVirtioWith2Mhuge(TestCase):
         # create an instance to set stream field setting
         self.pktgen_helper = PacketGeneratorHelper()
         self.pci_info = self.dut.ports_info[0]['pci']
+        self.number_of_ports = 1
 
     def set_up(self):
         """
@@ -92,6 +93,15 @@ class TestPVPVirtioWith2Mhuge(TestCase):
         self.table_header.append("% linerate")
         self.result_table_create(self.table_header)
 
+    @property
+    def check_value(self):
+        check_dict = dict.fromkeys(self.frame_sizes)
+        linerate = {64: 0.085, 128: 0.12, 256: 0.20, 512: 0.35, 1024: 0.50, 1280: 0.55, 1518: 0.60}
+        for size in self.frame_sizes:
+            speed = self.wirespeed(self.nic, size, self.number_of_ports)
+            check_dict[size] = round(speed * linerate[size], 2)
+        return check_dict
+
     def send_and_verify(self):
         """
         Send packet with packet generator and verify
@@ -112,7 +122,9 @@ class TestPVPVirtioWith2Mhuge(TestCase):
                             None, self.tester.pktgen)
             _, pps = self.tester.pktgen.measure_throughput(stream_ids=streams)
             Mpps = pps / 1000000.0
-            self.verify(Mpps > 0, "%s can not receive packets" % (self.running_case))
+            self.verify(Mpps > self.check_value[frame_size],
+                        "%s of frame size %d speed verify failed, expect %s, result %s" % (
+                            self.running_case, frame_size, self.check_value[frame_size], Mpps))
             throughput = Mpps * 100 / \
                      float(self.wirespeed(self.nic, 64, 1))
 
-- 
1.8.3.1

[dts] [PATCH V2 11/17]pvp_virtio_user_4k_pages: update mpps value to check perf

[Xiao Qimai]

Signed-off-by: Xiao Qimai <qimaix.xiao@intel.com>
---
 tests/TestSuite_pvp_virtio_user_4k_pages.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/tests/TestSuite_pvp_virtio_user_4k_pages.py b/tests/TestSuite_pvp_virtio_user_4k_pages.py
index 532b6c1..9bfd526 100644
--- a/tests/TestSuite_pvp_virtio_user_4k_pages.py
+++ b/tests/TestSuite_pvp_virtio_user_4k_pages.py
@@ -79,6 +79,7 @@ class TestPvpVirtioUser4kPages(TestCase):
             self.tester.send_expect('mkdir -p %s' % self.out_path, '# ')
         # create an instance to set stream field setting
         self.pktgen_helper = PacketGeneratorHelper()
+        self.number_of_ports = 1
 
     def set_up(self):
         """
@@ -96,6 +97,15 @@ class TestPvpVirtioUser4kPages(TestCase):
         self.table_header.append("% linerate")
         self.result_table_create(self.table_header)
 
+    @property
+    def check_value(self):
+        check_dict = dict.fromkeys(self.frame_sizes)
+        linerate = {64: 0.08, 128: 0.10, 256: 0.18, 512: 0.20, 1024: 0.40, 1280: 0.45, 1518: 0.50}
+        for size in self.frame_sizes:
+            speed = self.wirespeed(self.nic, size, self.number_of_ports)
+            check_dict[size] = round(speed * linerate[size], 2)
+        return check_dict
+
     def send_and_verify(self):
         """
         Send packet with packet generator and verify
@@ -114,7 +124,9 @@ class TestPvpVirtioUser4kPages(TestCase):
                             None, self.tester.pktgen)
             _, pps = self.tester.pktgen.measure_throughput(stream_ids=streams)
             Mpps = pps / 1000000.0
-            self.verify(Mpps > 0, "%s can not receive packets" % (self.running_case))
+            self.verify(Mpps > self.check_value[frame_size],
+                        "%s of frame size %d speed verify failed, expect %s, result %s" % (
+                            self.running_case, frame_size, self.check_value[frame_size], Mpps))
             throughput = Mpps * 100 / \
                      float(self.wirespeed(self.nic, 64, 1))
 
-- 
1.8.3.1

[dts] [PATCH V2 12/17]pvp_virtio_user_multi_queues_port_restart: update mpps value to check perf

[Xiao Qimai]

Signed-off-by: Xiao Qimai <qimaix.xiao@intel.com>
---
 .../TestSuite_pvp_virtio_user_multi_queues_port_restart.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/tests/TestSuite_pvp_virtio_user_multi_queues_port_restart.py b/tests/TestSuite_pvp_virtio_user_multi_queues_port_restart.py
index d948e15..0cd6fb2 100644
--- a/tests/TestSuite_pvp_virtio_user_multi_queues_port_restart.py
+++ b/tests/TestSuite_pvp_virtio_user_multi_queues_port_restart.py
@@ -70,6 +70,7 @@ class TestPVPVirtioUserMultiQueuesPortRestart(TestCase):
         self.tx_port = self.tester.get_local_port(self.dut_ports[0])
         self.queue_number = 2
         self.dut.kill_all()
+        self.number_of_ports = 1
 
     def set_up(self):
         """
@@ -166,11 +167,22 @@ class TestPVPVirtioUserMultiQueuesPortRestart(TestCase):
         traffic_opt = {'delay': 5}
         _, pps = self.tester.pktgen.measure_throughput(stream_ids=streams, options=traffic_opt)
         Mpps = pps / 1000000.0
-        self.verify(Mpps > 0, "can not receive packets of frame size %d" % (frame_size))
+        self.verify(Mpps > self.check_value[frame_size],
+                    "%s of frame size %d speed verify failed, expect %s, result %s" % (
+                        self.running_case, frame_size, self.check_value[frame_size], Mpps))
         throughput = Mpps * 100 / \
                     float(self.wirespeed(self.nic, frame_size, 1))
         return Mpps, throughput
 
+    @property
+    def check_value(self):
+        check_dict = dict.fromkeys(self.frame_sizes)
+        linerate = {64: 0.07, 128: 0.09, 256: 0.17, 512: 0.25, 1024: 0.40, 1280: 0.45, 1518: 0.50}
+        for size in self.frame_sizes:
+            speed = self.wirespeed(self.nic, size, self.number_of_ports)
+            check_dict[size] = round(speed * linerate[size], 2)
+        return check_dict
+
     def send_and_verify(self, case_info):
         """
         start to send packets and verify it
-- 
1.8.3.1

[dts] [PATCH V2 13/17]vhost_dequeue_zero_copy: update mpps value to check perf

[Xiao Qimai]

Signed-off-by: Xiao Qimai <qimaix.xiao@intel.com>
---
 tests/TestSuite_vhost_dequeue_zero_copy.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/tests/TestSuite_vhost_dequeue_zero_copy.py b/tests/TestSuite_vhost_dequeue_zero_copy.py
index 500204b..8beb1ba 100644
--- a/tests/TestSuite_vhost_dequeue_zero_copy.py
+++ b/tests/TestSuite_vhost_dequeue_zero_copy.py
@@ -82,6 +82,7 @@ class TestVhostDequeueZeroCopy(TestCase):
 
         self.base_dir = self.dut.base_dir.replace('~', '/root')
         self.vhost_user = self.dut.new_session(suite="vhost-user")
+        self.number_of_ports = 1
 
     def set_up(self):
         """
@@ -303,7 +304,9 @@ class TestVhostDequeueZeroCopy(TestCase):
             Mpps = stats[1] / 1000000.0
         # when the fwd mode is rxonly, we can not receive data, so should not verify it
         if fwd_mode != "rxonly":
-            self.verify(Mpps > 0, "can not receive packets of frame size %d" % (frame_size))
+            self.verify(Mpps > self.check_value[frame_size],
+                        "%s of frame size %d speed verify failed, expect %s, result %s" % (
+                            self.running_case, frame_size, self.check_value[frame_size], Mpps))
         throughput = Mpps * 100 / \
                     float(self.wirespeed(self.nic, frame_size, 1))
         return Mpps, throughput
@@ -335,6 +338,15 @@ class TestVhostDequeueZeroCopy(TestCase):
 
         self.vhost_user.send_expect("start", "testpmd> ", 60)
 
+    @property
+    def check_value(self):
+        check_dict = dict.fromkeys(self.frame_sizes)
+        linerate = {64: 0.07, 128: 0.10, 256: 0.20, 512: 0.18, 1024: 0.10, 1280: 0.10, 1518: 0.10}
+        for size in self.frame_sizes:
+            speed = self.wirespeed(self.nic, size, self.number_of_ports)
+            check_dict[size] = round(speed * linerate[size], 2)
+        return check_dict
+
     def send_and_verify_throughput(self, cycle="", fwd_mode=""):
         """
         start to send packets and verify it
-- 
1.8.3.1

[dts] [PATCH V2 14/17]vhost_multi_queue_qemu: update mpps value to check perf

[Xiao Qimai]

Signed-off-by: Xiao Qimai <qimaix.xiao@intel.com>
---
 tests/TestSuite_vhost_multi_queue_qemu.py | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/tests/TestSuite_vhost_multi_queue_qemu.py b/tests/TestSuite_vhost_multi_queue_qemu.py
index c111846..88524d0 100644
--- a/tests/TestSuite_vhost_multi_queue_qemu.py
+++ b/tests/TestSuite_vhost_multi_queue_qemu.py
@@ -58,7 +58,7 @@ class TestVhostMultiQueueQemu(TestCase):
         self.cores = self.dut.get_core_list("1S/3C/1T", socket=self.socket)
         self.verify(len(self.cores) >= 3, "Insufficient cores for speed testing")
         self.pci_info = self.dut.ports_info[0]['pci']
-
+        self.frame_sizes = [64, 128, 256, 512, 1024, 1500]
         self.queue_number = 2
         # Using file to save the vhost sample output since in jumboframe case,
         # there will be lots of output
@@ -87,8 +87,6 @@ class TestVhostMultiQueueQemu(TestCase):
         self.dut.send_expect("rm -rf ./vhost.out", "#")
         self.dut.send_expect("rm -rf %s/vhost-net*" % self.base_dir, "#")
         self.dut.send_expect("killall -s INT testpmd", "#")
-
-        self.frame_sizes = [64, 128, 256, 512, 1024, 1500]
         self.vm_testpmd_vector = self.target + "/app/testpmd -c %s -n 3" + \
                                  " -- -i --tx-offloads=0x0 " + \
                                  " --rxq=%d --txq=%d --rss-ip --nb-cores=2" % (self.queue_number, self.queue_number)
@@ -137,6 +135,15 @@ class TestVhostMultiQueueQemu(TestCase):
         self.verify(len(cores) >= 3, "Insufficient cores for speed testing, add the cpu number in cfg file.")
         self.vm_coremask = utils.create_mask(cores)
 
+    @property
+    def check_value(self):
+        check_dict = dict.fromkeys(self.frame_sizes)
+        linerate = {64: 0.09, 128: 0.15, 256: 0.25, 512: 0.40, 1024: 0.50, 1280: 0.55, 1500: 0.60}
+        for size in self.frame_sizes:
+            speed = self.wirespeed(self.nic, size, self.number_of_ports)
+            check_dict[size] = round(speed * linerate[size], 2)
+        return check_dict
+
     def vhost_performance(self):
         """
         Verify the testpmd can receive and forward the data
@@ -168,7 +175,9 @@ class TestVhostMultiQueueQemu(TestCase):
                                      self.number_of_ports))
             data_row = [frame_size, str(Mpps), str(pct), "Mergeable Multiqueue Performance"]
             self.result_table_add(data_row)
-            self.verify(Mpps != 0, "The receive data of frame-size: %d is 0" % frame_size)
+            self.verify(Mpps > self.check_value[frame_size],
+                        "%s of frame size %d speed verify failed, expect %s, result %s" % (
+                            self.running_case, frame_size, self.check_value[frame_size], Mpps))
         self.result_table_print()
 
     def send_and_verify(self, verify_type):
-- 
1.8.3.1

[dts] [PATCH V2 15/17]virtio_pvp_regression: update mpps value to check perf

[Xiao Qimai]

Signed-off-by: Xiao Qimai <qimaix.xiao@intel.com>
---
 tests/TestSuite_virtio_pvp_regression.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/tests/TestSuite_virtio_pvp_regression.py b/tests/TestSuite_virtio_pvp_regression.py
index 6a67fed..d9e0d30 100644
--- a/tests/TestSuite_virtio_pvp_regression.py
+++ b/tests/TestSuite_virtio_pvp_regression.py
@@ -55,6 +55,7 @@ class TestVirtioPVPRegression(TestCase):
         # Get and verify the ports
         self.dut_ports = self.dut.get_ports()
         self.pf = self.dut_ports[0]
+        self.number_of_ports = 1
         # Get the port's socket
         netdev = self.dut.ports_info[self.pf]['port']
         self.pci_info = self.dut.ports_info[self.pf]['pci']
@@ -302,6 +303,15 @@ class TestVirtioPVPRegression(TestCase):
 
         self.vhost.send_expect("start", "testpmd> ", 60)
 
+    @property
+    def check_value(self):
+        check_dict = dict.fromkeys(self.frame_sizes)
+        linerate = {64: 0.08, 128: 0.10, 256: 0.17, 512: 0.18, 1024: 0.40, 1280: 0.45, 1518: 0.50}
+        for size in self.frame_sizes:
+            speed = self.wirespeed(self.nic, size, self.number_of_ports)
+            check_dict[size] = round(speed * linerate[size], 2)
+        return check_dict
+
     def send_verify(self, case_info, qemu_version, tag):
         for frame_size in self.frame_sizes:
             info = "Running test %s, and %d frame size." % (self.running_case, frame_size)
@@ -326,7 +336,9 @@ class TestVirtioPVPRegression(TestCase):
             _, pps = self.tester.pktgen.measure_throughput(stream_ids=streams, options=traffic_opt)
             Mpps = pps / 1000000.0
             pct = Mpps * 100 / float(self.wirespeed(self.nic, frame_size, 1))
-            self.verify(Mpps != 0, "can not received data of frame size %d" % frame_size)
+            self.verify(Mpps > self.check_value[frame_size],
+                        "%s of frame size %d speed verify failed, expect %s, result %s" % (
+                            self.running_case, frame_size, self.check_value[frame_size], Mpps))
             # check each queue has data
             self.check_packets_of_each_queue(frame_size)
             # update print table info
-- 
1.8.3.1

[dts] [PATCH V2 16/17]virtio_user_for_container_networking: update mpps value to check perf

[Xiao Qimai]

Signed-off-by: Xiao Qimai <qimaix.xiao@intel.com>
---
 tests/TestSuite_virtio_user_for_container_networking.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/tests/TestSuite_virtio_user_for_container_networking.py b/tests/TestSuite_virtio_user_for_container_networking.py
index 1f9a666..76f0aea 100644
--- a/tests/TestSuite_virtio_user_for_container_networking.py
+++ b/tests/TestSuite_virtio_user_for_container_networking.py
@@ -71,6 +71,7 @@ class TestVirtioUserForContainer(TestCase):
             self.tester.send_expect('mkdir -p %s' % self.out_path, '# ')
         # create an instance to set stream field setting
         self.pktgen_helper = PacketGeneratorHelper()
+        self.number_of_ports = 1
 
     def set_up(self):
         """
@@ -101,6 +102,15 @@ class TestVirtioUserForContainer(TestCase):
         self.core_list_virtio_user = core_list[self.nb_cores+1:self.nb_cores*2+2]
         self.core_mask_virtio_user = utils.create_mask(core_list_virtio_user)
 
+    @property
+    def check_value(self):
+        check_dict = dict.fromkeys(self.frame_sizes)
+        linerate = {64: 0.085, 128: 0.12, 256: 0.20, 512: 0.35, 1024: 0.50, 1280: 0.55, 1518: 0.60}
+        for size in self.frame_sizes:
+            speed = self.wirespeed(self.nic, size, self.number_of_ports)
+            check_dict[size] = round(speed * linerate[size], 2)
+        return check_dict
+
     def send_and_verify(self):
         """
         Send packet with packet generator and verify
@@ -122,7 +132,9 @@ class TestVirtioUserForContainer(TestCase):
                         vm_config, self.tester.pktgen)
             _, pps = self.tester.pktgen.measure_throughput(stream_ids=streams)
             Mpps = pps / 1000000.0
-            self.verify(Mpps > 0, '%s can not receive packets of frame size %d' % (self.running_case, frame_size))
+            self.verify(Mpps > self.check_value[frame_size],
+                        "%s of frame size %d speed verify failed, expect %s, result %s" % (
+                            self.running_case, frame_size, self.check_value[frame_size], Mpps))
             throughput = Mpps * 100 / \
                      float(self.wirespeed(self.nic, frame_size, 1))
 
-- 
1.8.3.1

[dts] [PATCH V2 17/17]vm2vm_virtio_pmd: update mpps value to check perf

[Xiao Qimai]

Signed-off-by: Xiao Qimai <qimaix.xiao@intel.com>
---
 tests/TestSuite_vm2vm_virtio_pmd.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/tests/TestSuite_vm2vm_virtio_pmd.py b/tests/TestSuite_vm2vm_virtio_pmd.py
index d4ffdc7..7fea5bf 100644
--- a/tests/TestSuite_vm2vm_virtio_pmd.py
+++ b/tests/TestSuite_vm2vm_virtio_pmd.py
@@ -222,14 +222,15 @@ class TestVM2VMVirtioPMD(TestCase):
 
     def calculate_avg_throughput(self):
         results = 0.0
+        self.vhost_user.send_expect("show port stats 1", "testpmd> ", 60)
         for i in range(10):
             out = self.vhost_user.send_expect("show port stats 1", "testpmd> ", 60)
-            time.sleep(5)
+            time.sleep(1)
             lines = re.search("Rx-pps:\s*(\d*)", out)
             result = lines.group(1)
             results += float(result)
         Mpps = results / (1000000 * 10)
-        self.verify(Mpps > 0, "%s can not receive packets" % self.running_case)
+        self.verify(Mpps > 4.5, "%s can not receive packets" % self.running_case)
         return Mpps
 
     def update_table_info(self, case_info, frame_size, Mpps, path):
@@ -246,7 +247,7 @@ class TestVM2VMVirtioPMD(TestCase):
         """
         # start to send packets
         self.vm_dut[0].send_expect("set fwd rxonly", "testpmd> ", 10)
-        self.vm_dut[0].send_expect("start", "testpmd> ", 10)
+        self.vm_dut[0].send_command("start", 3)
         self.vm_dut[1].send_expect("set fwd txonly", "testpmd> ", 10)
         self.vm_dut[1].send_expect("set txpkts 64", "testpmd> ", 10)
         self.vm_dut[1].send_expect("start tx_first 32", "testpmd> ", 10)
-- 
1.8.3.1

Re: [dts] [PATCH V2 00/17] update mpps in virtio cases to check performance

[Xiao, QimaiX]

Tested-by: Xiao, QimaiX <qimaix.xiao@intel.com>

Regards,
Xiao Qimai

> -----Original Message-----
> From: Xiao, QimaiX <qimaix.xiao@intel.com>
> Sent: Friday, June 5, 2020 5:18 PM
> To: dts@dpdk.org
> Cc: Xiao, QimaiX <qimaix.xiao@intel.com>
> Subject: [dts][PATCH V2 00/17] update mpps in virtio cases to check
> performance
> 
> *. change mpps check value based on nic linerate
> 
> Xiao Qimai (17):
>   update mpps value to check perf
>   update mpps value to check perf
>   update mpps value to check perf
>   update mpps value to check perf
>   update mpps value to check perf
>   update mpps value to check perf
>   update mpps value to check perf
>   update mpps value to check perf
>   update mpps value to check perf
>   update mpps value to check perf
>   update mpps value to check perf
>   update mpps value to check perf
>   update mpps value to check perf
>   update mpps value to check perf
>   update mpps value to check perf
>   update mpps value to check perf
>   update mpps value to check perf
> 
>  .../TestSuite_loopback_multi_paths_port_restart.py |  5 +-
>  tests/TestSuite_loopback_multi_queues.py           |  7 ++-
>  .../TestSuite_loopback_virtio_user_server_mode.py  |  7 +--
>  tests/TestSuite_pvp_diff_qemu_version.py           | 14 +++++-
>  tests/TestSuite_pvp_multi_paths_performance.py     | 56 +++++++++++++-
> --------
>  ...vp_multi_paths_vhost_single_core_performance.py | 14 +++++-
>   ...p_multi_paths_virtio_single_core_performance.py | 14 +++++-
>   .../TestSuite_pvp_qemu_multi_paths_port_restart.py | 14 +++++-
>   ...TestSuite_pvp_vhost_user_built_in_net_driver.py | 14 +++++-
>  tests/TestSuite_pvp_virtio_user_2M_hugepages.py    | 14 +++++-
>  tests/TestSuite_pvp_virtio_user_4k_pages.py        | 14 +++++-
>  ...te_pvp_virtio_user_multi_queues_port_restart.py | 14 +++++-
>  tests/TestSuite_vhost_dequeue_zero_copy.py         | 14 +++++-
>  tests/TestSuite_vhost_multi_queue_qemu.py          | 17 +++++--
>  tests/TestSuite_virtio_pvp_regression.py           | 14 +++++-
>  ...stSuite_virtio_user_for_container_networking.py | 14 +++++-
>  tests/TestSuite_vm2vm_virtio_pmd.py                |  7 +--
>  17 files changed, 201 insertions(+), 52 deletions(-)
> 
> --
> 1.8.3.1

Re: [dts] [PATCH V2 00/17] update mpps in virtio cases to check performance

[Wang, Yinan]

Acked-by: Wang, Yinan <yinan.wang@intel.com>

> -----Original Message-----
> From: dts <dts-bounces@dpdk.org> On Behalf Of Xiao, QimaiX
> Sent: 2020年6月5日 17:31
> To: dts@dpdk.org
> Subject: Re: [dts] [PATCH V2 00/17] update mpps in virtio cases to check
> performance
> 
> Tested-by: Xiao, QimaiX <qimaix.xiao@intel.com>
> 
> Regards,
> Xiao Qimai
> 
> > -----Original Message-----
> > From: Xiao, QimaiX <qimaix.xiao@intel.com>
> > Sent: Friday, June 5, 2020 5:18 PM
> > To: dts@dpdk.org
> > Cc: Xiao, QimaiX <qimaix.xiao@intel.com>
> > Subject: [dts][PATCH V2 00/17] update mpps in virtio cases to check
> > performance
> >
> > *. change mpps check value based on nic linerate
> >
> > Xiao Qimai (17):
> >   update mpps value to check perf
> >   update mpps value to check perf
> >   update mpps value to check perf
> >   update mpps value to check perf
> >   update mpps value to check perf
> >   update mpps value to check perf
> >   update mpps value to check perf
> >   update mpps value to check perf
> >   update mpps value to check perf
> >   update mpps value to check perf
> >   update mpps value to check perf
> >   update mpps value to check perf
> >   update mpps value to check perf
> >   update mpps value to check perf
> >   update mpps value to check perf
> >   update mpps value to check perf
> >   update mpps value to check perf
> >
> >  .../TestSuite_loopback_multi_paths_port_restart.py |  5 +-
> >  tests/TestSuite_loopback_multi_queues.py           |  7 ++-
> >  .../TestSuite_loopback_virtio_user_server_mode.py  |  7 +--
> >  tests/TestSuite_pvp_diff_qemu_version.py           | 14 +++++-
> >  tests/TestSuite_pvp_multi_paths_performance.py     | 56 +++++++++++++-
> > --------
> >  ...vp_multi_paths_vhost_single_core_performance.py | 14 +++++-
> >   ...p_multi_paths_virtio_single_core_performance.py | 14 +++++-
> >   .../TestSuite_pvp_qemu_multi_paths_port_restart.py | 14 +++++-
> >   ...TestSuite_pvp_vhost_user_built_in_net_driver.py | 14 +++++-
> >  tests/TestSuite_pvp_virtio_user_2M_hugepages.py    | 14 +++++-
> >  tests/TestSuite_pvp_virtio_user_4k_pages.py        | 14 +++++-
> >  ...te_pvp_virtio_user_multi_queues_port_restart.py | 14 +++++-
> >  tests/TestSuite_vhost_dequeue_zero_copy.py         | 14 +++++-
> >  tests/TestSuite_vhost_multi_queue_qemu.py          | 17 +++++--
> >  tests/TestSuite_virtio_pvp_regression.py           | 14 +++++-
> >  ...stSuite_virtio_user_for_container_networking.py | 14 +++++-
> >  tests/TestSuite_vm2vm_virtio_pmd.py                |  7 +--
> >  17 files changed, 201 insertions(+), 52 deletions(-)
> >
> > --
> > 1.8.3.1

Re: [dts] [PATCH V2 00/17] update mpps in virtio cases to check performance

[Tu, Lijuan]

Applied the series, thanks

-----Original Message-----
From: dts <dts-bounces@dpdk.org> On Behalf Of Xiao, QimaiX
Sent: 2020年6月5日 17:31
To: dts@dpdk.org
Subject: Re: [dts] [PATCH V2 00/17] update mpps in virtio cases to check performance

Tested-by: Xiao, QimaiX <qimaix.xiao@intel.com>

Regards,
Xiao Qimai

> -----Original Message-----
> From: Xiao, QimaiX <qimaix.xiao@intel.com>
> Sent: Friday, June 5, 2020 5:18 PM
> To: dts@dpdk.org
> Cc: Xiao, QimaiX <qimaix.xiao@intel.com>
> Subject: [dts][PATCH V2 00/17] update mpps in virtio cases to check 
> performance
> 
> *. change mpps check value based on nic linerate
> 
> Xiao Qimai (17):
>   update mpps value to check perf
>   update mpps value to check perf
>   update mpps value to check perf
>   update mpps value to check perf
>   update mpps value to check perf
>   update mpps value to check perf
>   update mpps value to check perf
>   update mpps value to check perf
>   update mpps value to check perf
>   update mpps value to check perf
>   update mpps value to check perf
>   update mpps value to check perf
>   update mpps value to check perf
>   update mpps value to check perf
>   update mpps value to check perf
>   update mpps value to check perf
>   update mpps value to check perf
> 
>  .../TestSuite_loopback_multi_paths_port_restart.py |  5 +-
>  tests/TestSuite_loopback_multi_queues.py           |  7 ++-
>  .../TestSuite_loopback_virtio_user_server_mode.py  |  7 +--
>  tests/TestSuite_pvp_diff_qemu_version.py           | 14 +++++-
>  tests/TestSuite_pvp_multi_paths_performance.py     | 56 +++++++++++++-
> --------
>  ...vp_multi_paths_vhost_single_core_performance.py | 14 +++++-
>   ...p_multi_paths_virtio_single_core_performance.py | 14 +++++-
>   .../TestSuite_pvp_qemu_multi_paths_port_restart.py | 14 +++++-
>   ...TestSuite_pvp_vhost_user_built_in_net_driver.py | 14 +++++-
>  tests/TestSuite_pvp_virtio_user_2M_hugepages.py    | 14 +++++-
>  tests/TestSuite_pvp_virtio_user_4k_pages.py        | 14 +++++-
>  ...te_pvp_virtio_user_multi_queues_port_restart.py | 14 +++++-
>  tests/TestSuite_vhost_dequeue_zero_copy.py         | 14 +++++-
>  tests/TestSuite_vhost_multi_queue_qemu.py          | 17 +++++--
>  tests/TestSuite_virtio_pvp_regression.py           | 14 +++++-
>  ...stSuite_virtio_user_for_container_networking.py | 14 +++++-
>  tests/TestSuite_vm2vm_virtio_pmd.py                |  7 +--
>  17 files changed, 201 insertions(+), 52 deletions(-)
> 
> --
> 1.8.3.1

Re: [dts] [PATCH V2 00/17] update mpps in virtio cases to check performance

[Wang, Yinan]

Acked-by: Wang, Yinan <yinan.wang@intel.com>

> -----Original Message-----
> From: dts <dts-bounces@dpdk.org> On Behalf Of Xiao, QimaiX
> Sent: 2020?6?5? 17:31
> To: dts@dpdk.org
> Subject: Re: [dts] [PATCH V2 00/17] update mpps in virtio cases to check
> performance
> 
> Tested-by: Xiao, QimaiX <qimaix.xiao@intel.com>
> 
> Regards,
> Xiao Qimai
> 
> > -----Original Message-----
> > From: Xiao, QimaiX <qimaix.xiao@intel.com>
> > Sent: Friday, June 5, 2020 5:18 PM
> > To: dts@dpdk.org
> > Cc: Xiao, QimaiX <qimaix.xiao@intel.com>
> > Subject: [dts][PATCH V2 00/17] update mpps in virtio cases to check
> > performance
> >
> > *. change mpps check value based on nic linerate
> >
> > Xiao Qimai (17):
> >   update mpps value to check perf
> >   update mpps value to check perf
> >   update mpps value to check perf
> >   update mpps value to check perf
> >   update mpps value to check perf
> >   update mpps value to check perf
> >   update mpps value to check perf
> >   update mpps value to check perf
> >   update mpps value to check perf
> >   update mpps value to check perf
> >   update mpps value to check perf
> >   update mpps value to check perf
> >   update mpps value to check perf
> >   update mpps value to check perf
> >   update mpps value to check perf
> >   update mpps value to check perf
> >   update mpps value to check perf
> >
> >  .../TestSuite_loopback_multi_paths_port_restart.py |  5 +-
> >  tests/TestSuite_loopback_multi_queues.py           |  7 ++-
> >  .../TestSuite_loopback_virtio_user_server_mode.py  |  7 +--
> >  tests/TestSuite_pvp_diff_qemu_version.py           | 14 +++++-
> >  tests/TestSuite_pvp_multi_paths_performance.py     | 56 +++++++++++++-
> > --------
> >  ...vp_multi_paths_vhost_single_core_performance.py | 14 +++++-
> >   ...p_multi_paths_virtio_single_core_performance.py | 14 +++++-
> >   .../TestSuite_pvp_qemu_multi_paths_port_restart.py | 14 +++++-
> >   ...TestSuite_pvp_vhost_user_built_in_net_driver.py | 14 +++++-
> >  tests/TestSuite_pvp_virtio_user_2M_hugepages.py    | 14 +++++-
> >  tests/TestSuite_pvp_virtio_user_4k_pages.py        | 14 +++++-
> >  ...te_pvp_virtio_user_multi_queues_port_restart.py | 14 +++++-
> >  tests/TestSuite_vhost_dequeue_zero_copy.py         | 14 +++++-
> >  tests/TestSuite_vhost_multi_queue_qemu.py          | 17 +++++--
> >  tests/TestSuite_virtio_pvp_regression.py           | 14 +++++-
> >  ...stSuite_virtio_user_for_container_networking.py | 14 +++++-
> >  tests/TestSuite_vm2vm_virtio_pmd.py                |  7 +--
> >  17 files changed, 201 insertions(+), 52 deletions(-)
> >
> > --
> > 1.8.3.1

Re: [dts] [PATCH V2 00/17] update mpps in virtio cases to check performance

[Tu, Lijuan]

Replied the series, thanks

-----Original Message-----
From: dts <dts-bounces@dpdk.org> On Behalf Of Xiao Qimai
Sent: 2020年6月5日 17:18
To: dts@dpdk.org
Cc: Xiao, QimaiX <qimaix.xiao@intel.com>
Subject: [dts] [PATCH V2 00/17] update mpps in virtio cases to check performance

*. change mpps check value based on nic linerate

Xiao Qimai (17):
  update mpps value to check perf
  update mpps value to check perf
  update mpps value to check perf
  update mpps value to check perf
  update mpps value to check perf
  update mpps value to check perf
  update mpps value to check perf
  update mpps value to check perf
  update mpps value to check perf
  update mpps value to check perf
  update mpps value to check perf
  update mpps value to check perf
  update mpps value to check perf
  update mpps value to check perf
  update mpps value to check perf
  update mpps value to check perf
  update mpps value to check perf

 .../TestSuite_loopback_multi_paths_port_restart.py |  5 +-
 tests/TestSuite_loopback_multi_queues.py           |  7 ++-
 .../TestSuite_loopback_virtio_user_server_mode.py  |  7 +--
 tests/TestSuite_pvp_diff_qemu_version.py           | 14 +++++-
 tests/TestSuite_pvp_multi_paths_performance.py     | 56 +++++++++++++---------
 ...vp_multi_paths_vhost_single_core_performance.py | 14 +++++-  ...p_multi_paths_virtio_single_core_performance.py | 14 +++++-  .../TestSuite_pvp_qemu_multi_paths_port_restart.py | 14 +++++-  ...TestSuite_pvp_vhost_user_built_in_net_driver.py | 14 +++++-
 tests/TestSuite_pvp_virtio_user_2M_hugepages.py    | 14 +++++-
 tests/TestSuite_pvp_virtio_user_4k_pages.py        | 14 +++++-
 ...te_pvp_virtio_user_multi_queues_port_restart.py | 14 +++++-
 tests/TestSuite_vhost_dequeue_zero_copy.py         | 14 +++++-
 tests/TestSuite_vhost_multi_queue_qemu.py          | 17 +++++--
 tests/TestSuite_virtio_pvp_regression.py           | 14 +++++-
 ...stSuite_virtio_user_for_container_networking.py | 14 +++++-
 tests/TestSuite_vm2vm_virtio_pmd.py                |  7 +--
 17 files changed, 201 insertions(+), 52 deletions(-)

--
1.8.3.1