[dts][PATCH V1 0/2] add expected value in suite config to

[dts][PATCH V1 0/2] add expected value in suite config to

[Wei Ling]

Optimize the verify expected value from hard code to read from 
conf/pvp_virtio_user_2M_hugepages.cfg to adapt different platform.

Wei Ling (2):
  tests/pvp_virtio_user_2M_hugepages: add expected value in suite config
    to verify
  conf/pvp_virtio_user_2M_hugepages: add suite config to save expected
    value

 conf/pvp_virtio_user_2M_hugepages.cfg         |  11 ++
 .../TestSuite_pvp_virtio_user_2M_hugepages.py | 141 +++++++++++-------
 2 files changed, 101 insertions(+), 51 deletions(-)
 create mode 100644 conf/pvp_virtio_user_2M_hugepages.cfg

-- 
2.25.1

[dts][PATCH V1 1/2] tests/pvp_virtio_user_2M_hugepages: add expected value in suite config to verify

[Wei Ling]

Optimize the verify expected value from hard code to read from
conf/pvp_virtio_user_2M_hugepages.cfg to adapt different platform.

Signed-off-by: Wei Ling <weix.ling@intel.com>
---
 .../TestSuite_pvp_virtio_user_2M_hugepages.py | 141 +++++++++++-------
 1 file changed, 90 insertions(+), 51 deletions(-)

diff --git a/tests/TestSuite_pvp_virtio_user_2M_hugepages.py b/tests/TestSuite_pvp_virtio_user_2M_hugepages.py
index 7fe0bcbb..e48e772a 100644
--- a/tests/TestSuite_pvp_virtio_user_2M_hugepages.py
+++ b/tests/TestSuite_pvp_virtio_user_2M_hugepages.py
@@ -2,20 +2,14 @@
 # Copyright(c) 2019 Intel Corporation
 #
 
-"""
-DPDK Test suite.
-vhost/virtio-user pvp with 2M hugepage.
-"""
+from copy import deepcopy
 
-import time
-
-import framework.utils as utils
 from framework.pktgen import PacketGeneratorHelper
-from framework.settings import HEADER_SIZE
+from framework.settings import HEADER_SIZE, UPDATE_EXPECTED, load_global_setting
 from framework.test_case import TestCase
 
 
-class TestPVPVirtioWith2Mhuge(TestCase):
+class TestPVPVirtioWith2MHugepages(TestCase):
     def set_up_all(self):
         """
         Run at the start of each test suite.
@@ -40,6 +34,8 @@ class TestPVPVirtioWith2Mhuge(TestCase):
 
         self.core_list_virtio_user = self.core_list[0:2]
         self.core_list_vhost_user = self.core_list[2:4]
+        self.nb_ports = 1
+        self.gap = self.get_suite_cfg()["accepted_tolerance"]
         self.dst_mac = self.dut.get_mac_address(self.dut_ports[0])
         self.header_size = HEADER_SIZE["eth"] + HEADER_SIZE["ip"] + HEADER_SIZE["tcp"]
         self.frame_sizes = [64, 128, 256, 512, 1024, 1518]
@@ -59,6 +55,8 @@ class TestPVPVirtioWith2Mhuge(TestCase):
         self.number_of_ports = 1
         self.path = self.dut.apps_name["test-pmd"]
         self.testpmd_name = self.path.split("/")[-1]
+        self.vhost_user = self.dut.new_session(suite="vhost-user")
+        self.virtio_user = self.dut.new_session(suite="virtio-user")
 
     def set_up(self):
         """
@@ -66,37 +64,19 @@ class TestPVPVirtioWith2Mhuge(TestCase):
         """
         self.dut.send_expect("rm -rf ./vhost-net*", "# ")
         self.dut.send_expect("killall -s INT %s" % self.testpmd_name, "#")
-        self.vhost_user = self.dut.new_session(suite="vhost-user")
-        self.virtio_user = self.dut.new_session(suite="virtio-user")
+        self.throughput = dict()
+        self.test_result = dict()
         # Prepare the result table
         self.table_header = ["Frame"]
-        self.table_header.append("Mode")
         self.table_header.append("Mpps")
-        self.table_header.append("Queue Num")
         self.table_header.append("% linerate")
-        self.result_table_create(self.table_header)
 
-    @property
-    def check_value(self):
-        check_dict = dict.fromkeys(self.frame_sizes)
-        linerate = {
-            64: 0.085,
-            128: 0.12,
-            256: 0.20,
-            512: 0.35,
-            1024: 0.50,
-            1280: 0.55,
-            1518: 0.60,
-        }
-        for size in self.frame_sizes:
-            speed = self.wirespeed(self.nic, size, self.number_of_ports)
-            check_dict[size] = round(speed * linerate[size], 2)
-        return check_dict
-
-    def send_and_verify(self):
-        """
-        Send packet with packet generator and verify
+    def perf_test(self):
+        """
+        Send packet with packet generator
         """
+        self.result_table_create(self.table_header)
+        self.throughput = {}
         for frame_size in self.frame_sizes:
             payload_size = frame_size - self.header_size
             tgen_input = []
@@ -113,21 +93,73 @@ class TestPVPVirtioWith2Mhuge(TestCase):
             streams = self.pktgen_helper.prepare_stream_from_tginput(
                 tgen_input, 100, None, self.tester.pktgen
             )
-            _, pps = self.tester.pktgen.measure_throughput(stream_ids=streams)
+            # set traffic option
+            traffic_opt = {
+                "delay": 5,
+                "duration": self.get_suite_cfg()["test_duration"],
+            }
+            _, pps = self.tester.pktgen.measure_throughput(
+                stream_ids=streams, options=traffic_opt
+            )
             Mpps = pps / 1000000.0
-            self.verify(
-                Mpps > self.check_value[frame_size],
-                "%s of frame size %d speed verify failed, expect %s, result %s"
-                % (self.running_case, frame_size, self.check_value[frame_size], Mpps),
+            line_rate = (
+                Mpps * 100 / float(self.wirespeed(self.nic, frame_size, self.nb_ports))
             )
-            throughput = Mpps * 100 / float(self.wirespeed(self.nic, 64, 1))
-
+            self.throughput[frame_size] = Mpps
             results_row = [frame_size]
-            results_row.append("2M Hugepages")
             results_row.append(Mpps)
-            results_row.append("1")
-            results_row.append(throughput)
+            results_row.append(line_rate)
             self.result_table_add(results_row)
+        self.result_table_print()
+
+    def handle_expected(self):
+        """
+        Update expected numbers to configurate file: $DTS_CFG_FOLDER/$suite_name.cfg
+        """
+        if load_global_setting(UPDATE_EXPECTED) == "yes":
+            for frame_size in self.frame_sizes:
+                self.expected_throughput[frame_size] = round(
+                    self.throughput[frame_size], 3
+                )
+
+    def handle_results(self):
+        """
+        results handled process:
+        1, save to self.test_results
+        2, create test results table
+        """
+        # save test results to self.test_result
+        header = self.table_header
+        header.append("Expected Throughput(Mpps)")
+        header.append("Status")
+        self.result_table_create(self.table_header)
+        for frame_size in self.frame_sizes:
+            wirespeed = self.wirespeed(self.nic, frame_size, self.nb_ports)
+            ret_data = {}
+            ret_data[header[0]] = str(frame_size)
+            _real = float(self.throughput[frame_size])
+            _exp = float(self.expected_throughput[frame_size])
+            ret_data[header[1]] = "{:.3f}".format(_real)
+            ret_data[header[2]] = "{:.3f}%".format(_real * 100 / wirespeed)
+            ret_data[header[3]] = "{:.3f}".format(_exp)
+            gap = _exp * -self.gap * 0.01
+            if _real > _exp + gap:
+                ret_data[header[4]] = "PASS"
+            else:
+                ret_data[header[4]] = "FAIL"
+            self.test_result[frame_size] = deepcopy(ret_data)
+
+        for frame_size in self.test_result.keys():
+            table_row = list()
+            for i in range(len(header)):
+                table_row.append(self.test_result[frame_size][header[i]])
+            self.result_table_add(table_row)
+        # present test results to screen
+        self.result_table_print()
+        self.verify(
+            "FAIL" not in self.test_result,
+            "Excessive gap between test results and expectations",
+        )
 
     def start_testpmd_as_vhost(self):
         """
@@ -170,27 +202,33 @@ class TestPVPVirtioWith2Mhuge(TestCase):
         """
         self.virtio_user.send_expect("quit", "# ", 60)
         self.vhost_user.send_expect("quit", "# ", 60)
-        self.dut.close_session(self.vhost_user)
-        self.dut.close_session(self.virtio_user)
 
     def test_perf_pvp_virtio_user_split_ring_2M_hugepages(self):
         """
         Basic test for virtio-user 2M hugepage
         """
+        self.expected_throughput = self.get_suite_cfg()["expected_throughput"][
+            self.running_case
+        ]
         self.start_testpmd_as_vhost()
         self.start_testpmd_as_virtio()
-        self.send_and_verify()
-        self.result_table_print()
+        self.perf_test()
+        self.handle_expected()
+        self.handle_results()
         self.close_all_apps()
 
     def test_perf_pvp_virtio_user_packed_ring_2M_hugepages(self):
         """
         Basic test for virtio-user 2M hugepage
         """
+        self.expected_throughput = self.get_suite_cfg()["expected_throughput"][
+            self.running_case
+        ]
         self.start_testpmd_as_vhost()
         self.start_testpmd_as_virtio(packed=True)
-        self.send_and_verify()
-        self.result_table_print()
+        self.perf_test()
+        self.handle_expected()
+        self.handle_results()
         self.close_all_apps()
 
     def tear_down(self):
@@ -203,4 +241,5 @@ class TestPVPVirtioWith2Mhuge(TestCase):
         """
         Run after each test suite.
         """
-        pass
+        self.dut.close_session(self.vhost_user)
+        self.dut.close_session(self.virtio_user)
-- 
2.25.1

[dts][PATCH V1 2/2] conf/pvp_virtio_user_2M_hugepages: add suite config to save expected value

[Wei Ling]

Add conf/pvp_virtio_user_2M_hugepages.cfg to save the test parameter and
expected value to verify.

Signed-off-by: Wei Ling <weix.ling@intel.com>
---
 conf/pvp_virtio_user_2M_hugepages.cfg | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 conf/pvp_virtio_user_2M_hugepages.cfg

diff --git a/conf/pvp_virtio_user_2M_hugepages.cfg b/conf/pvp_virtio_user_2M_hugepages.cfg
new file mode 100644
index 00000000..bca4ceda
--- /dev/null
+++ b/conf/pvp_virtio_user_2M_hugepages.cfg
@@ -0,0 +1,11 @@
+[suite]
+update_expected = True
+packet_sizes = [64, 128, 256, 512, 1024, 1518]
+test_duration = 30
+accepted_tolerance = 5
+expected_throughput = {
+    'test_perf_pvp_virtio_user_split_ring_2M_hugepages': {
+        64: 0.00, 128: 0.00, 256: 0.00, 512: 0.00, 1024: 0.00, 1518: 0.00}, 
+    'test_perf_pvp_virtio_user_packed_ring_2M_hugepages': {
+        64: 0.00, 128: 0.00, 256: 0.00, 512: 0.00, 1024: 0.00, 1518: 0.00}}
+
-- 
2.25.1

RE: [dts][PATCH V1 2/2] conf/pvp_virtio_user_2M_hugepages: add suite config to save expected value

[He, Xingguang]

> -----Original Message-----
> From: Wei Ling <weix.ling@intel.com>
> Sent: Tuesday, April 25, 2023 11:04 AM
> To: dts@dpdk.org
> Cc: Ling, WeiX <weix.ling@intel.com>
> Subject: [dts][PATCH V1 2/2] conf/pvp_virtio_user_2M_hugepages: add suite
> config to save expected value
> 
> Add conf/pvp_virtio_user_2M_hugepages.cfg to save the test parameter and
> expected value to verify.
> 
> Signed-off-by: Wei Ling <weix.ling@intel.com>
> ---
Acked-by: Xingguang He<xingguang.he@intel.com>

[dts][PATCH V1 2/2] conf/pvp_virtio_user_2M_hugepages: add suite config to save expected value

[lijuan.tu]

On Tue, 25 Apr 2023 11:03:44 +0800, Wei Ling <weix.ling@intel.com> wrote:
> Add conf/pvp_virtio_user_2M_hugepages.cfg to save the test parameter and
> expected value to verify.
> 
> Signed-off-by: Wei Ling <weix.ling@intel.com>

Acked-by: Lijuan Tu <lijuan.tu@intel.com>
Series applied, thanks