[dts] [PATCH V3 00/13] vf_l3fwd_kernelpf: add more test scenario

[dts] [PATCH V3 00/13] vf_l3fwd_kernelpf: add more test scenario

[yufengmx]

v3: 
 -  fix checkpatch error. 

v2: 
 -  add stage 2 demand. 

v1: 
 -  add more test scenario. 

yufengmx (13):
  conf/vf_l3fwd_base: add more flows
  tests/perf_test_base: extend vf l3fwd content
  conf/vf_l3fwd_em_kernelpf: configuration
  tests/vf_l3fwd_em_kernelpf: script
  conf/vf_l3fwd_kernelpf: configuration
  tests/vf_l3fwd_kernelpf: add more test scenario
  conf/vf_l3fwd_lpm_ipv4_kernelpf: configuration
  tests/vf_l3fwd_lpm_ipv4_kernelpf: script
  conf/vf_l3fwd_lpm_ipv4_rfc2544_kernelpf: configuration
  tests/vf_l3fwd_lpm_ipv4_rfc2544_kernelpf: script
  conf/vf_l3fwd_lpm_ipv6_kernelpf: configuration
  tests/vf_l3fwd_lpm_ipv6_kernelpf: script
  tests/perf_test_base: script

 conf/vf_l3fwd_base.cfg                        |  93 +++++-
 conf/vf_l3fwd_em_kernelpf.cfg                 | 101 ++++++
 conf/vf_l3fwd_kernelpf.cfg                    | 303 +++++++++++++++++-
 conf/vf_l3fwd_lpm_ipv4_kernelpf.cfg           |  59 ++++
 conf/vf_l3fwd_lpm_ipv4_rfc2544_kernelpf.cfg   |  59 ++++
 conf/vf_l3fwd_lpm_ipv6_kernelpf.cfg           |  59 ++++
 tests/TestSuite_vf_l3fwd_em_kernelpf.py       |  89 +++++
 tests/TestSuite_vf_l3fwd_kernelpf.py          |  40 ++-
 tests/TestSuite_vf_l3fwd_lpm_ipv4_kernelpf.py |  85 +++++
 ...uite_vf_l3fwd_lpm_ipv4_rfc2544_kernelpf.py |  85 +++++
 tests/TestSuite_vf_l3fwd_lpm_ipv6_kernelpf.py |  85 +++++
 tests/perf_test_base.py                       | 122 ++++++-
 12 files changed, 1151 insertions(+), 29 deletions(-)
 create mode 100644 conf/vf_l3fwd_em_kernelpf.cfg
 create mode 100644 conf/vf_l3fwd_lpm_ipv4_kernelpf.cfg
 create mode 100644 conf/vf_l3fwd_lpm_ipv4_rfc2544_kernelpf.cfg
 create mode 100644 conf/vf_l3fwd_lpm_ipv6_kernelpf.cfg
 create mode 100644 tests/TestSuite_vf_l3fwd_em_kernelpf.py
 create mode 100644 tests/TestSuite_vf_l3fwd_lpm_ipv4_kernelpf.py
 create mode 100644 tests/TestSuite_vf_l3fwd_lpm_ipv4_rfc2544_kernelpf.py
 create mode 100644 tests/TestSuite_vf_l3fwd_lpm_ipv6_kernelpf.py

-- 
2.21.0

[dts] [PATCH V3 01/13] conf/vf_l3fwd_base: add more flows

[yufengmx]

add vf_l3fwd_base more flows configuration.

Signed-off-by: yufengmx <yufengx.mo@intel.com>
---
 conf/vf_l3fwd_base.cfg | 93 +++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 92 insertions(+), 1 deletion(-)

diff --git a/conf/vf_l3fwd_base.cfg b/conf/vf_l3fwd_base.cfg
index 9d9ede33..8d281144 100644
--- a/conf/vf_l3fwd_base.cfg
+++ b/conf/vf_l3fwd_base.cfg
@@ -1,6 +1,50 @@
+# stream_type=  "tcp"/"udp"/"raw". If not set this key, default is "raw"
+#
 [suite]
 l3fwd_flows = {
     "ipv4": {
+        "em": [
+            {
+                "ipv4": {
+                    "dst": "101.0.0.0",
+                    "src": "100.10.0.1"
+                },
+                "udp": {
+                    "dst": 11,
+                    "src": 101
+                }
+            },
+            {
+                "ipv4": {
+                    "dst": "201.0.0.0",
+                    "src": "200.20.0.1"
+                },
+                "udp": {
+                    "dst": 12,
+                    "src": 102
+                }
+            },
+            {
+                "ipv4": {
+                    "dst": "111.0.0.0",
+                    "src": "100.30.0.1"
+                },
+                "udp": {
+                    "dst": 11,
+                    "src": 101
+                }
+            },
+            {
+                "ipv4": {
+                    "dst": "211.0.0.0",
+                    "src": "200.40.0.1"
+                },
+                "udp": {
+                    "dst": 11,
+                    "src": 102
+                }
+            }
+        ],
         "lpm": [
             "198.18.0.0/24",
             "198.18.1.0/24",
@@ -11,4 +55,51 @@ l3fwd_flows = {
             "198.18.6.0/24",
             "198.18.7.0/24"
         ]
-    }, }
\ No newline at end of file
+    },
+    "ipv6": {
+        "em": [
+            {
+                "ipv6": {
+                    "dst": "fe80:0000:0000:0000:021e:67ff:fe00:0000",
+                    "src": "fe80:0000:0000:0000:021b:21ff:fe91:3805"
+                },
+                "udp": {
+                    "dst": 101,
+                    "src": 11 }
+            },
+            {
+                "ipv6": {
+                    "dst": "fe90:0000:0000:0000:021e:67ff:fe00:0000",
+                    "src": "fe90:0000:0000:0000:021b:21ff:fe91:3805" },
+                "udp": {
+                    "dst": 102,
+                    "src": 12 }
+            },
+            {
+                "ipv6": {
+                    "dst": "fea0:0000:0000:0000:021e:67ff:fe00:0000",
+                    "src": "fea0:0000:0000:0000:021b:21ff:fe91:3805" },
+                "udp": {
+                    "dst": 101,
+                    "src": 11 }
+            },
+            {
+                "ipv6": {
+                    "dst": "feb0:0000:0000:0000:021e:67ff:fe00:0000",
+                    "src": "feb0:0000:0000:0000:021b:21ff:fe91:3805" },
+                "udp": {
+                    "dst": 102,
+                    "src": 12 }
+            }
+        ],
+        "lpm": [
+            "3201:20:00:00:00:00:00:00/48",
+            "3201:20:00:00:01:00:00:00/48",
+            "3201:20:00:00:02:00:00:00/48",
+            "3201:20:00:00:03:00:00:00/48",
+            "3201:20:00:00:04:00:00:00/48",
+            "3201:20:00:00:05:00:00:00/48",
+            "3201:20:00:00:06:00:00:00/48",
+            "3201:20:00:00:07:00:00:00/48"
+        ]
+    } }
-- 
2.21.0

[dts] [PATCH V3 02/13] tests/perf_test_base: extend vf l3fwd content

[yufengmx]

#. unify vf l3fwd nic support list.
#. add suite series name method.
#. remove comiple restore limitation.
#. rename host testpmd whitelist method.

Signed-off-by: yufengmx <yufengx.mo@intel.com>
---
 tests/perf_test_base.py | 38 ++++++++++++++++++++++++++++----------
 1 file changed, 28 insertions(+), 10 deletions(-)

diff --git a/tests/perf_test_base.py b/tests/perf_test_base.py
index 20f3e152..4b76e6b1 100644
--- a/tests/perf_test_base.py
+++ b/tests/perf_test_base.py
@@ -51,6 +51,17 @@ from exception import VerifyFailure
 import utils
 
 
+VF_L3FWD_NIC_SUPPORT = frozenset((
+    "niantic",
+    "fortville_spirit",
+    "fortville_25g",
+    "fortville_eagle",
+    "columbiaville_100g",
+    "columbiaville_25g",
+    "columbiaville_25gx2",
+))
+
+
 @unique
 class BIN_TYPE(Enum):
     L3FWD = 'l3fwd'
@@ -625,14 +636,11 @@ class PerfTestBase(object):
                 "define RTE_TEST_TX_DESC_DEFAULT 2048/' "
                 "./examples/l3fwd/l3fwd.h"))
         if self.__mode is SUITE_TYPE.VF:
-            self.__l3fwd_lpm = self.__l3fwd_em = \
-                self.__l3fwd_init(MATCH_MODE.EM, rename=False)
             # init testpmd
             if self.__pf_driver is not NIC_DRV.PCI_STUB:
                 self.__init_host_testpmd()
-        else:
-            self.__l3fwd_em = self.__l3fwd_init(MATCH_MODE.EM)
-            self.__l3fwd_lpm = self.__l3fwd_init(MATCH_MODE.LPM)
+        self.__l3fwd_em = self.__l3fwd_init(MATCH_MODE.EM)
+        self.__l3fwd_lpm = self.__l3fwd_init(MATCH_MODE.LPM)
 
     def __preset_compilation(self):
         # Update compile config file and rebuild to get best perf on different nics
@@ -673,7 +681,7 @@ class PerfTestBase(object):
                 'core_mask': core_mask,
                 'mem_channel': self.dut.get_memory_channels(),
                 'memsize': mem_size,
-                'whitelist': self.__get_testpmd_whitelist(),
+                'whitelist': self.__get_host_testpmd_whitelist(),
                 'prefix': 'pf', })
         self.__host_pmd_con([cmd, "testpmd> ", 120])
         self.__is_pmd_on = True
@@ -1238,8 +1246,19 @@ class PerfTestBase(object):
             vf_driver = test_content.get('vf_driver')
         return pf_driver, vf_driver
 
+    def __set_suite_series_name(self):
+        pat = "vf_l3fwd.*_kernelpf"
+        if re.match(pat, self.suite_name):
+            self.__suite = SUITE_NAME.VF_KERNELPF
+            return
+        pat = "testpmd*_perf"
+        if re.match(pat, self.suite_name):
+            self.__suite = SUITE_NAME.TESTPMD_PERF
+            return
+        self.__suite = None
+
     def __get_vf_test_content_from_cfg(self, test_content):
-        self.__suite = get_enum_name(self.suite_name, SUITE_NAME)
+        self.__set_suite_series_name()
         # pf driver
         pf_driver, vf_driver = self.__get_suite_vf_pf_driver(test_content)
         if pf_driver and isinstance(pf_driver, str):
@@ -1356,7 +1375,7 @@ class PerfTestBase(object):
 
         return whitelist
 
-    def __get_testpmd_whitelist(self):
+    def __get_host_testpmd_whitelist(self):
         whitelist = ''.join(['-w {} '.format(info.get('pf_pci'))
                              for _, info in self.__vf_ports_info.items()])
         return whitelist
@@ -1404,8 +1423,7 @@ class PerfTestBase(object):
                     self.dut.close_session(self.__pmd_session)
                     self.__pmd_session = None
             self.__vf_destroy()
-        if self.__mode is SUITE_TYPE.PF:
-            self.__restore_compilation()
+        self.__restore_compilation()
 
     def perf_set_cur_case(self, name):
         self.__cur_case = name
-- 
2.21.0

[dts] [PATCH V3 03/13] conf/vf_l3fwd_em_kernelpf: configuration

[yufengmx]

add vf_l3fwd_em_kernelpf suite testing configuration.

Signed-off-by: yufengmx <yufengx.mo@intel.com>
---
 conf/vf_l3fwd_em_kernelpf.cfg | 101 ++++++++++++++++++++++++++++++++++
 1 file changed, 101 insertions(+)
 create mode 100644 conf/vf_l3fwd_em_kernelpf.cfg

diff --git a/conf/vf_l3fwd_em_kernelpf.cfg b/conf/vf_l3fwd_em_kernelpf.cfg
new file mode 100644
index 00000000..11490bf9
--- /dev/null
+++ b/conf/vf_l3fwd_em_kernelpf.cfg
@@ -0,0 +1,101 @@
+# Users could change these configuration on demand:
+#
+#  detail content reference to ``vf_l3fwd_kernelpf.cfg``
+#
+
+[suite]
+test_duration = 60
+accepted_tolerance = 1
+
+test_parameters = {
+    '1C/1T-1Q': ['64', '128', '256', '512', '1024', '1280', '1518',],
+    '1C/2T-2Q': ['64', '128', '256', '512', '1024', '1280', '1518',],
+    '2C/2T-2Q': ['64', '128', '256', '512', '1024', '1280', '1518',],
+    '2C/4T-4Q': ['64', '128', '256', '512', '1024', '1280', '1518',],
+    '4C/4T-4Q': ['64', '128', '256', '512', '1024', '1280', '1518',],}
+
+expected_throughput = {
+    'test_perf_vf_throughput_ipv4_em': {
+        'niantic': {
+            '1C/1T-1Q': {
+                 '64': '0.00',
+                 '128': '0.00',
+                 '256': '0.00',
+                 '512': '0.00',
+                 '1024': '0.00',
+                 '1280': '0.00',
+                 '1518': '0.00', },
+            '1C/2T-2Q': {
+                 '64': '0.00',
+                 '128': '0.00',
+                 '256': '0.00',
+                 '512': '0.00',
+                 '1024': '0.00',
+                 '1280': '0.00',
+                 '1518': '0.00', },
+            '2C/2T-2Q': {
+                 '64': '0.00',
+                 '128': '0.00',
+                 '256': '0.00',
+                 '512': '0.00',
+                 '1024': '0.00',
+                 '1280': '0.00',
+                 '1518': '0.00', },
+            '2C/4T-4Q': {
+                 '64': '0.00',
+                 '128': '0.00',
+                 '256': '0.00',
+                 '512': '0.00',
+                 '1024': '0.00',
+                 '1280': '0.00',
+                 '1518': '0.00', },
+            '4C/4T-4Q': {
+                 '64': '0.00',
+                 '128': '0.00',
+                 '256': '0.00',
+                 '512': '0.00',
+                 '1024': '0.00',
+                 '1280': '0.00',
+                 '1518': '0.00', },},},
+    'test_perf_vf_throughput_ipv6_em': {
+        'niantic': {
+            '1C/1T-1Q': {
+                 '64': '0.00',
+                 '128': '0.00',
+                 '256': '0.00',
+                 '512': '0.00',
+                 '1024': '0.00',
+                 '1280': '0.00',
+                 '1518': '0.00', },
+            '1C/2T-2Q': {
+                 '64': '0.00',
+                 '128': '0.00',
+                 '256': '0.00',
+                 '512': '0.00',
+                 '1024': '0.00',
+                 '1280': '0.00',
+                 '1518': '0.00', },
+            '2C/2T-2Q': {
+                 '64': '0.00',
+                 '128': '0.00',
+                 '256': '0.00',
+                 '512': '0.00',
+                 '1024': '0.00',
+                 '1280': '0.00',
+                 '1518': '0.00', },
+            '2C/4T-4Q': {
+                 '64': '0.00',
+                 '128': '0.00',
+                 '256': '0.00',
+                 '512': '0.00',
+                 '1024': '0.00',
+                 '1280': '0.00',
+                 '1518': '0.00', },
+            '4C/4T-4Q': {
+                 '64': '0.00',
+                 '128': '0.00',
+                 '256': '0.00',
+                 '512': '0.00',
+                 '1024': '0.00',
+                 '1280': '0.00',
+                 '1518': '0.00', },},},}
-- 
2.21.0

[dts] [PATCH V3 04/13] tests/vf_l3fwd_em_kernelpf: script

[yufengmx]

vf_l3fwd_em_kernelpf suite automation script.

Signed-off-by: yufengmx <yufengx.mo@intel.com>
---
 tests/TestSuite_vf_l3fwd_em_kernelpf.py | 89 +++++++++++++++++++++++++
 1 file changed, 89 insertions(+)
 create mode 100644 tests/TestSuite_vf_l3fwd_em_kernelpf.py

diff --git a/tests/TestSuite_vf_l3fwd_em_kernelpf.py b/tests/TestSuite_vf_l3fwd_em_kernelpf.py
new file mode 100644
index 00000000..05cda1d1
--- /dev/null
+++ b/tests/TestSuite_vf_l3fwd_em_kernelpf.py
@@ -0,0 +1,89 @@
+# BSD LICENSE
+#
+# Copyright(c) 2010-2021 Intel Corporation. All rights reserved.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+#   * Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#   * Redistributions in binary form must reproduce the above copyright
+#     notice, this list of conditions and the following disclaimer in
+#     the documentation and/or other materials provided with the
+#     distribution.
+#   * Neither the name of Intel Corporation nor the names of its
+#     contributors may be used to endorse or promote products derived
+#     from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import os
+import time
+from test_case import TestCase
+from perf_test_base import PerfTestBase, IP_TYPE, MATCH_MODE, SUITE_TYPE, VF_L3FWD_NIC_SUPPORT
+
+
+class TestVfL3fwdEmKernelPf(TestCase, PerfTestBase):
+    #
+    # Test cases.
+    #
+
+    def set_up_all(self):
+        """
+        Run at the start of each test suite.
+        """
+        self.verify(self.nic in VF_L3FWD_NIC_SUPPORT,
+                    "NIC Unsupported: " + str(self.nic))
+        self.dut_ports = self.dut.get_ports(self.nic)
+        valports = [
+            _ for _ in self.dut_ports if self.tester.get_local_port(_) != -1]
+        self.logger.debug(valports)
+        self.verify_ports_number(valports)
+        # get socket and cores
+        socket = self.dut.get_numa_id(self.dut_ports[0])
+        cores = self.dut.get_core_list("1S/6C/1T", socket=socket)
+        self.verify(cores, "Requested 6 cores failed")
+        # init l3fwd common base class parameters
+        PerfTestBase.__init__(self, valports, socket, mode=SUITE_TYPE.VF)
+        # preset testing environment
+        self.perf_preset_test_environment(self.get_suite_cfg())
+
+    def tear_down_all(self):
+        """
+        Run after each test suite.
+        """
+        self.perf_destroy_resource()
+        self.l3fwd_save_results(json_file="{}.json".format(self.suite_name))
+
+    def set_up(self):
+        """
+        Run before each test case.
+        """
+        pass
+
+    def tear_down(self):
+        """
+        Run after each test case.
+        """
+        self.dut.kill_all()
+        self.perf_reset_cur_case()
+
+    def test_perf_vf_throughput_ipv4_em(self):
+        self.perf_set_cur_case('test_perf_vf_throughput_ipv4_em')
+        self.ms_throughput(l3_proto=IP_TYPE.V4, mode=MATCH_MODE.EM)
+
+    def test_perf_vf_throughput_ipv6_em(self):
+        self.perf_set_cur_case('test_perf_vf_throughput_ipv6_em')
+        self.ms_throughput(l3_proto=IP_TYPE.V6, mode=MATCH_MODE.EM)
-- 
2.21.0

[dts] [PATCH V3 05/13] conf/vf_l3fwd_kernelpf: configuration

[yufengmx]

add vf_l3fwd_kernelpf suite testing configuration.

Signed-off-by: yufengmx <yufengx.mo@intel.com>
---
 conf/vf_l3fwd_kernelpf.cfg | 303 ++++++++++++++++++++++++++++++++++++-
 1 file changed, 300 insertions(+), 3 deletions(-)

diff --git a/conf/vf_l3fwd_kernelpf.cfg b/conf/vf_l3fwd_kernelpf.cfg
index 9c711770..c2c71c1b 100644
--- a/conf/vf_l3fwd_kernelpf.cfg
+++ b/conf/vf_l3fwd_kernelpf.cfg
@@ -50,16 +50,59 @@
 # tests as usual.
 
 [suite]
-update_expected = True
-test_duration = 12
+test_duration = 60
 accepted_tolerance = 1
+
 test_parameters = {
     '1C/1T-1Q': ['64', '128', '256', '512', '1024', '1280', '1518',],
     '1C/2T-2Q': ['64', '128', '256', '512', '1024', '1280', '1518',],
     '2C/2T-2Q': ['64', '128', '256', '512', '1024', '1280', '1518',],
     '2C/4T-4Q': ['64', '128', '256', '512', '1024', '1280', '1518',],
     '4C/4T-4Q': ['64', '128', '256', '512', '1024', '1280', '1518',],}
+
 expected_throughput = {
+    'test_perf_vf_throughput_ipv4_em': {
+        'niantic': {
+            '1C/1T-1Q': {
+                 '64': '0.00',
+                 '128': '0.00',
+                 '256': '0.00',
+                 '512': '0.00',
+                 '1024': '0.00',
+                 '1280': '0.00',
+                 '1518': '0.00', },
+            '1C/2T-2Q': {
+                 '64': '0.00',
+                 '128': '0.00',
+                 '256': '0.00',
+                 '512': '0.00',
+                 '1024': '0.00',
+                 '1280': '0.00',
+                 '1518': '0.00', },
+            '2C/2T-2Q': {
+                 '64': '0.00',
+                 '128': '0.00',
+                 '256': '0.00',
+                 '512': '0.00',
+                 '1024': '0.00',
+                 '1280': '0.00',
+                 '1518': '0.00', },
+            '2C/4T-4Q': {
+                 '64': '0.00',
+                 '128': '0.00',
+                 '256': '0.00',
+                 '512': '0.00',
+                 '1024': '0.00',
+                 '1280': '0.00',
+                 '1518': '0.00', },
+            '4C/4T-4Q': {
+                 '64': '0.00',
+                 '128': '0.00',
+                 '256': '0.00',
+                 '512': '0.00',
+                 '1024': '0.00',
+                 '1280': '0.00',
+                 '1518': '0.00', },},},
     'test_perf_vf_throughput_ipv4_lpm': {
         'niantic': {
             '1C/1T-1Q': {
@@ -101,4 +144,258 @@ expected_throughput = {
                  '512': '0.00',
                  '1024': '0.00',
                  '1280': '0.00',
-                 '1518': '0.00', },},},}
\ No newline at end of file
+                 '1518': '0.00', },},},
+    'test_perf_vf_throughput_ipv6_em': {
+        'niantic': {
+            '1C/1T-1Q': {
+                 '64': '0.00',
+                 '128': '0.00',
+                 '256': '0.00',
+                 '512': '0.00',
+                 '1024': '0.00',
+                 '1280': '0.00',
+                 '1518': '0.00', },
+            '1C/2T-2Q': {
+                 '64': '0.00',
+                 '128': '0.00',
+                 '256': '0.00',
+                 '512': '0.00',
+                 '1024': '0.00',
+                 '1280': '0.00',
+                 '1518': '0.00', },
+            '2C/2T-2Q': {
+                 '64': '0.00',
+                 '128': '0.00',
+                 '256': '0.00',
+                 '512': '0.00',
+                 '1024': '0.00',
+                 '1280': '0.00',
+                 '1518': '0.00', },
+            '2C/4T-4Q': {
+                 '64': '0.00',
+                 '128': '0.00',
+                 '256': '0.00',
+                 '512': '0.00',
+                 '1024': '0.00',
+                 '1280': '0.00',
+                 '1518': '0.00', },
+            '4C/4T-4Q': {
+                 '64': '0.00',
+                 '128': '0.00',
+                 '256': '0.00',
+                 '512': '0.00',
+                 '1024': '0.00',
+                 '1280': '0.00',
+                 '1518': '0.00', },},},
+    'test_perf_vf_throughput_ipv6_lpm': {
+        'niantic': {
+            '1C/1T-1Q': {
+                 '64': '0.00',
+                 '128': '0.00',
+                 '256': '0.00',
+                 '512': '0.00',
+                 '1024': '0.00',
+                 '1280': '0.00',
+                 '1518': '0.00', },
+            '1C/2T-2Q': {
+                 '64': '0.00',
+                 '128': '0.00',
+                 '256': '0.00',
+                 '512': '0.00',
+                 '1024': '0.00',
+                 '1280': '0.00',
+                 '1518': '0.00', },
+            '2C/2T-2Q': {
+                 '64': '0.00',
+                 '128': '0.00',
+                 '256': '0.00',
+                 '512': '0.00',
+                 '1024': '0.00',
+                 '1280': '0.00',
+                 '1518': '0.00', },
+            '2C/4T-4Q': {
+                 '64': '0.00',
+                 '128': '0.00',
+                 '256': '0.00',
+                 '512': '0.00',
+                 '1024': '0.00',
+                 '1280': '0.00',
+                 '1518': '0.00', },
+            '4C/4T-4Q': {
+                 '64': '0.00',
+                 '128': '0.00',
+                 '256': '0.00',
+                 '512': '0.00',
+                 '1024': '0.00',
+                 '1280': '0.00',
+                 '1518': '0.00', },},},}
+
+expected_rfc2544 = {
+    'test_perf_vf_rfc2544_ipv4_em': {
+        'niantic': {
+            '1C/1T-1Q': {
+                '64': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '128': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '256': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '512': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1024': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1280': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1518': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } } },
+            '1C/2T-2Q': {
+                '64': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '128': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '256': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '512': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1024': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1280': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1518': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } } },
+            '2C/2T-2Q': {
+                '64': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '128': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '256': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '512': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1024': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1280': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1518': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } } },
+            '2C/4T-4Q': {
+                '64': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '128': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '256': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '512': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1024': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1280': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1518': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } } },
+            '4C/4T-4Q': {
+                '64': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '128': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '256': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '512': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1024': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1280': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1518': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } } },},},
+    'test_perf_vf_rfc2544_ipv4_lpm': {
+        'niantic': {
+            '1C/1T-1Q': {
+                '64': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '128': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '256': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '512': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1024': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1280': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1518': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } } },
+            '1C/2T-2Q': {
+                '64': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '128': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '256': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '512': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1024': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1280': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1518': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } } },
+            '2C/2T-2Q': {
+                '64': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '128': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '256': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '512': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1024': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1280': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1518': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } } },
+            '2C/4T-4Q': {
+                '64': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '128': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '256': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '512': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1024': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1280': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1518': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } } },
+            '4C/4T-4Q': {
+                '64': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '128': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '256': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '512': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1024': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1280': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1518': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } } },},},
+    'test_perf_vf_rfc2544_ipv6_em': {
+        'niantic': {
+            '1C/1T-1Q': {
+                '64': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '128': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '256': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '512': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1024': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1280': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1518': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } } },
+            '1C/2T-2Q': {
+                '64': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '128': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '256': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '512': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1024': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1280': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1518': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } } },
+            '2C/2T-2Q': {
+                '64': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '128': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '256': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '512': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1024': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1280': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1518': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } } },
+            '2C/4T-4Q': {
+                '64': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '128': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '256': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '512': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1024': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1280': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1518': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } } },
+            '4C/4T-4Q': {
+                '64': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '128': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '256': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '512': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1024': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1280': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1518': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } } },},},
+    'test_perf_vf_rfc2544_ipv6_lpm': {
+        'niantic': {
+            '1C/1T-1Q': {
+                '64': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '128': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '256': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '512': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1024': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1280': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1518': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } } },
+            '1C/2T-2Q': {
+                '64': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '128': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '256': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '512': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1024': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1280': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1518': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } } },
+            '2C/2T-2Q': {
+                '64': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '128': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '256': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '512': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1024': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1280': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1518': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } } },
+            '2C/4T-4Q': {
+                '64': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '128': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '256': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '512': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1024': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1280': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1518': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } } },
+            '4C/4T-4Q': {
+                '64': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '128': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '256': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '512': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1024': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1280': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1518': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } } },},},}
\ No newline at end of file
-- 
2.21.0

[dts] [PATCH V3 06/13] tests/vf_l3fwd_kernelpf: add more test scenario

[yufengmx]

add more test scenario cases.

Signed-off-by: yufengmx <yufengx.mo@intel.com>
---
 tests/TestSuite_vf_l3fwd_kernelpf.py | 40 +++++++++++++++++++++-------
 1 file changed, 31 insertions(+), 9 deletions(-)

diff --git a/tests/TestSuite_vf_l3fwd_kernelpf.py b/tests/TestSuite_vf_l3fwd_kernelpf.py
index 4cd44aaa..d5ed897a 100644
--- a/tests/TestSuite_vf_l3fwd_kernelpf.py
+++ b/tests/TestSuite_vf_l3fwd_kernelpf.py
@@ -32,7 +32,7 @@
 import os
 import time
 from test_case import TestCase
-from perf_test_base import PerfTestBase, IP_TYPE, MATCH_MODE, SUITE_TYPE
+from perf_test_base import PerfTestBase, IP_TYPE, MATCH_MODE, SUITE_TYPE, VF_L3FWD_NIC_SUPPORT
 
 
 class TestVfL3fwdKernelPf(TestCase, PerfTestBase):
@@ -44,14 +44,8 @@ class TestVfL3fwdKernelPf(TestCase, PerfTestBase):
         """
         Run at the start of each test suite.
         """
-        self.verify(self.nic in [
-            "niantic",
-            "fortville_spirit",
-            "fortville_25g",
-            "fortville_eagle",
-            "columbiaville_100g",
-            "columbiaville_25g",
-            "columbiaville_25gx2"], "NIC Unsupported: " + str(self.nic))
+        self.verify(self.nic in VF_L3FWD_NIC_SUPPORT,
+                    "NIC Unsupported: " + str(self.nic))
         self.dut_ports = self.dut.get_ports(self.nic)
         valports = [
             _ for _ in self.dut_ports if self.tester.get_local_port(_) != -1]
@@ -86,6 +80,34 @@ class TestVfL3fwdKernelPf(TestCase, PerfTestBase):
         self.dut.kill_all()
         self.perf_reset_cur_case()
 
+    def test_perf_vf_rfc2544_ipv4_lpm(self):
+        self.perf_set_cur_case('test_perf_vf_rfc2544_ipv4_lpm')
+        self.qt_rfc2544(l3_proto=IP_TYPE.V4, mode=MATCH_MODE.LPM)
+
+    def test_perf_vf_rfc2544_ipv4_em(self):
+        self.perf_set_cur_case('test_perf_vf_rfc2544_ipv4_em')
+        self.qt_rfc2544(l3_proto=IP_TYPE.V4, mode=MATCH_MODE.EM)
+
     def test_perf_vf_throughput_ipv4_lpm(self):
         self.perf_set_cur_case('test_perf_vf_throughput_ipv4_lpm')
         self.ms_throughput(l3_proto=IP_TYPE.V4, mode=MATCH_MODE.LPM)
+
+    def test_perf_vf_throughput_ipv4_em(self):
+        self.perf_set_cur_case('test_perf_vf_throughput_ipv4_em')
+        self.ms_throughput(l3_proto=IP_TYPE.V4, mode=MATCH_MODE.EM)
+
+    def test_perf_vf_rfc2544_ipv6_lpm(self):
+        self.perf_set_cur_case('test_perf_vf_rfc2544_ipv6_lpm')
+        self.qt_rfc2544(l3_proto=IP_TYPE.V6, mode=MATCH_MODE.LPM)
+
+    def test_perf_vf_rfc2544_ipv6_em(self):
+        self.perf_set_cur_case('test_perf_vf_rfc2544_ipv6_em')
+        self.qt_rfc2544(l3_proto=IP_TYPE.V6, mode=MATCH_MODE.EM)
+
+    def test_perf_vf_throughput_ipv6_lpm(self):
+        self.perf_set_cur_case('test_perf_vf_throughput_ipv6_lpm')
+        self.ms_throughput(l3_proto=IP_TYPE.V6, mode=MATCH_MODE.LPM)
+
+    def test_perf_vf_throughput_ipv6_em(self):
+        self.perf_set_cur_case('test_perf_vf_throughput_ipv6_em')
+        self.ms_throughput(l3_proto=IP_TYPE.V6, mode=MATCH_MODE.EM)
-- 
2.21.0

[dts] [PATCH V3 07/13] conf/vf_l3fwd_lpm_ipv4_kernelpf: configuration

[yufengmx]

add vf_l3fwd_lpm_ipv4_kernelpf suite testing configuration.

Signed-off-by: yufengmx <yufengx.mo@intel.com>
---
 conf/vf_l3fwd_lpm_ipv4_kernelpf.cfg | 59 +++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)
 create mode 100644 conf/vf_l3fwd_lpm_ipv4_kernelpf.cfg

diff --git a/conf/vf_l3fwd_lpm_ipv4_kernelpf.cfg b/conf/vf_l3fwd_lpm_ipv4_kernelpf.cfg
new file mode 100644
index 00000000..960886a5
--- /dev/null
+++ b/conf/vf_l3fwd_lpm_ipv4_kernelpf.cfg
@@ -0,0 +1,59 @@
+# Users could change these configuration on demand:
+#
+#  detail content reference to ``vf_l3fwd_kernelpf.cfg``
+#
+
+[suite]
+test_duration = 60
+accepted_tolerance = 1
+
+test_parameters = {
+    '1C/1T-1Q': ['64', '128', '256', '512', '1024', '1280', '1518',],
+    '1C/2T-2Q': ['64', '128', '256', '512', '1024', '1280', '1518',],
+    '2C/2T-2Q': ['64', '128', '256', '512', '1024', '1280', '1518',],
+    '2C/4T-4Q': ['64', '128', '256', '512', '1024', '1280', '1518',],
+    '4C/4T-4Q': ['64', '128', '256', '512', '1024', '1280', '1518',],}
+
+expected_throughput = {
+    'test_perf_vf_throughput_ipv4_lpm': {
+        'niantic': {
+            '1C/1T-1Q': {
+                 '64': '0.00',
+                 '128': '0.00',
+                 '256': '0.00',
+                 '512': '0.00',
+                 '1024': '0.00',
+                 '1280': '0.00',
+                 '1518': '0.00', },
+            '1C/2T-2Q': {
+                 '64': '0.00',
+                 '128': '0.00',
+                 '256': '0.00',
+                 '512': '0.00',
+                 '1024': '0.00',
+                 '1280': '0.00',
+                 '1518': '0.00', },
+            '2C/2T-2Q': {
+                 '64': '0.00',
+                 '128': '0.00',
+                 '256': '0.00',
+                 '512': '0.00',
+                 '1024': '0.00',
+                 '1280': '0.00',
+                 '1518': '0.00', },
+            '2C/4T-4Q': {
+                 '64': '0.00',
+                 '128': '0.00',
+                 '256': '0.00',
+                 '512': '0.00',
+                 '1024': '0.00',
+                 '1280': '0.00',
+                 '1518': '0.00', },
+            '4C/4T-4Q': {
+                 '64': '0.00',
+                 '128': '0.00',
+                 '256': '0.00',
+                 '512': '0.00',
+                 '1024': '0.00',
+                 '1280': '0.00',
+                 '1518': '0.00', },},},}
-- 
2.21.0

[dts] [PATCH V3 08/13] tests/vf_l3fwd_lpm_ipv4_kernelpf: script

[yufengmx]

vf_l3fwd_lpm_ipv4_kernelpf suite automation script.

Signed-off-by: yufengmx <yufengx.mo@intel.com>
---
 tests/TestSuite_vf_l3fwd_lpm_ipv4_kernelpf.py | 85 +++++++++++++++++++
 1 file changed, 85 insertions(+)
 create mode 100644 tests/TestSuite_vf_l3fwd_lpm_ipv4_kernelpf.py

diff --git a/tests/TestSuite_vf_l3fwd_lpm_ipv4_kernelpf.py b/tests/TestSuite_vf_l3fwd_lpm_ipv4_kernelpf.py
new file mode 100644
index 00000000..51490047
--- /dev/null
+++ b/tests/TestSuite_vf_l3fwd_lpm_ipv4_kernelpf.py
@@ -0,0 +1,85 @@
+# BSD LICENSE
+#
+# Copyright(c) 2010-2021 Intel Corporation. All rights reserved.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+#   * Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#   * Redistributions in binary form must reproduce the above copyright
+#     notice, this list of conditions and the following disclaimer in
+#     the documentation and/or other materials provided with the
+#     distribution.
+#   * Neither the name of Intel Corporation nor the names of its
+#     contributors may be used to endorse or promote products derived
+#     from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import os
+import time
+from test_case import TestCase
+from perf_test_base import PerfTestBase, IP_TYPE, MATCH_MODE, SUITE_TYPE, VF_L3FWD_NIC_SUPPORT
+
+
+class TestVfL3fwdLpmIpv4KernelPf(TestCase, PerfTestBase):
+    #
+    # Test cases.
+    #
+
+    def set_up_all(self):
+        """
+        Run at the start of each test suite.
+        """
+        self.verify(self.nic in VF_L3FWD_NIC_SUPPORT,
+                    "NIC Unsupported: " + str(self.nic))
+        self.dut_ports = self.dut.get_ports(self.nic)
+        valports = [
+            _ for _ in self.dut_ports if self.tester.get_local_port(_) != -1]
+        self.logger.debug(valports)
+        self.verify_ports_number(valports)
+        # get socket and cores
+        socket = self.dut.get_numa_id(self.dut_ports[0])
+        cores = self.dut.get_core_list("1S/6C/1T", socket=socket)
+        self.verify(cores, "Requested 6 cores failed")
+        # init l3fwd common base class parameters
+        PerfTestBase.__init__(self, valports, socket, mode=SUITE_TYPE.VF)
+        # preset testing environment
+        self.perf_preset_test_environment(self.get_suite_cfg())
+
+    def tear_down_all(self):
+        """
+        Run after each test suite.
+        """
+        self.perf_destroy_resource()
+        self.l3fwd_save_results(json_file="{}.json".format(self.suite_name))
+
+    def set_up(self):
+        """
+        Run before each test case.
+        """
+        pass
+
+    def tear_down(self):
+        """
+        Run after each test case.
+        """
+        self.dut.kill_all()
+        self.perf_reset_cur_case()
+
+    def test_perf_vf_throughput_ipv4_lpm(self):
+        self.perf_set_cur_case('test_perf_vf_throughput_ipv4_lpm')
+        self.ms_throughput(l3_proto=IP_TYPE.V4, mode=MATCH_MODE.LPM)
-- 
2.21.0

[dts] [PATCH V3 09/13] conf/vf_l3fwd_lpm_ipv4_rfc2544_kernelpf: configuration

[yufengmx]

add vf_l3fwd_lpm_ipv4_rfc2544_kernelpf suite testing configuration.

Signed-off-by: yufengmx <yufengx.mo@intel.com>
---
 conf/vf_l3fwd_lpm_ipv4_rfc2544_kernelpf.cfg | 59 +++++++++++++++++++++
 1 file changed, 59 insertions(+)
 create mode 100644 conf/vf_l3fwd_lpm_ipv4_rfc2544_kernelpf.cfg

diff --git a/conf/vf_l3fwd_lpm_ipv4_rfc2544_kernelpf.cfg b/conf/vf_l3fwd_lpm_ipv4_rfc2544_kernelpf.cfg
new file mode 100644
index 00000000..6e526693
--- /dev/null
+++ b/conf/vf_l3fwd_lpm_ipv4_rfc2544_kernelpf.cfg
@@ -0,0 +1,59 @@
+# Users could change these configuration on demand:
+#
+#  detail content reference to ``vf_l3fwd_kernelpf.cfg``
+#
+
+[suite]
+test_duration = 60
+accepted_tolerance = 1
+
+test_parameters = {
+    '1C/1T-1Q': ['64', '128', '256', '512', '1024', '1280', '1518',],
+    '1C/2T-2Q': ['64', '128', '256', '512', '1024', '1280', '1518',],
+    '2C/2T-2Q': ['64', '128', '256', '512', '1024', '1280', '1518',],
+    '2C/4T-4Q': ['64', '128', '256', '512', '1024', '1280', '1518',],
+    '4C/4T-4Q': ['64', '128', '256', '512', '1024', '1280', '1518',],}
+
+expected_rfc2544 = {
+    'test_perf_vf_rfc2544_ipv4_lpm': {
+        'niantic': {
+            '1C/1T-1Q': {
+                '64': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '128': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '256': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '512': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1024': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1280': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1518': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } } },
+            '1C/2T-2Q': {
+                '64': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '128': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '256': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '512': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1024': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1280': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1518': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } } },
+            '2C/2T-2Q': {
+                '64': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '128': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '256': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '512': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1024': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1280': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1518': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } } },
+            '2C/4T-4Q': {
+                '64': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '128': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '256': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '512': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1024': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1280': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1518': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } } },
+            '4C/4T-4Q': {
+                '64': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '128': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '256': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '512': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1024': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1280': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } },
+                '1518': { 'rate' :'100.00', 'traffic_opt': {'min_rate': '10.0', 'max_rate': '100.0', 'pdr': '0.01', 'accuracy': '5', } } },},},}
-- 
2.21.0

[dts] [PATCH V3 10/13] tests/vf_l3fwd_lpm_ipv4_rfc2544_kernelpf: script

[yufengmx]

vf_l3fwd_lpm_ipv4_rfc2544_kernelpf suite automation script.

Signed-off-by: yufengmx <yufengx.mo@intel.com>
---
 ...uite_vf_l3fwd_lpm_ipv4_rfc2544_kernelpf.py | 85 +++++++++++++++++++
 1 file changed, 85 insertions(+)
 create mode 100644 tests/TestSuite_vf_l3fwd_lpm_ipv4_rfc2544_kernelpf.py

diff --git a/tests/TestSuite_vf_l3fwd_lpm_ipv4_rfc2544_kernelpf.py b/tests/TestSuite_vf_l3fwd_lpm_ipv4_rfc2544_kernelpf.py
new file mode 100644
index 00000000..c37d18ad
--- /dev/null
+++ b/tests/TestSuite_vf_l3fwd_lpm_ipv4_rfc2544_kernelpf.py
@@ -0,0 +1,85 @@
+# BSD LICENSE
+#
+# Copyright(c) 2010-2021 Intel Corporation. All rights reserved.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+#   * Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#   * Redistributions in binary form must reproduce the above copyright
+#     notice, this list of conditions and the following disclaimer in
+#     the documentation and/or other materials provided with the
+#     distribution.
+#   * Neither the name of Intel Corporation nor the names of its
+#     contributors may be used to endorse or promote products derived
+#     from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import os
+import time
+from test_case import TestCase
+from perf_test_base import PerfTestBase, IP_TYPE, MATCH_MODE, SUITE_TYPE, VF_L3FWD_NIC_SUPPORT
+
+
+class TestVfL3fwdLpmIpv4Rfc2544KernelPf(TestCase, PerfTestBase):
+    #
+    # Test cases.
+    #
+
+    def set_up_all(self):
+        """
+        Run at the start of each test suite.
+        """
+        self.verify(self.nic in VF_L3FWD_NIC_SUPPORT,
+                    "NIC Unsupported: " + str(self.nic))
+        self.dut_ports = self.dut.get_ports(self.nic)
+        valports = [
+            _ for _ in self.dut_ports if self.tester.get_local_port(_) != -1]
+        self.logger.debug(valports)
+        self.verify_ports_number(valports)
+        # get socket and cores
+        socket = self.dut.get_numa_id(self.dut_ports[0])
+        cores = self.dut.get_core_list("1S/6C/1T", socket=socket)
+        self.verify(cores, "Requested 6 cores failed")
+        # init l3fwd common base class parameters
+        PerfTestBase.__init__(self, valports, socket, mode=SUITE_TYPE.VF)
+        # preset testing environment
+        self.perf_preset_test_environment(self.get_suite_cfg())
+
+    def tear_down_all(self):
+        """
+        Run after each test suite.
+        """
+        self.perf_destroy_resource()
+        self.l3fwd_save_results(json_file="{}.json".format(self.suite_name))
+
+    def set_up(self):
+        """
+        Run before each test case.
+        """
+        pass
+
+    def tear_down(self):
+        """
+        Run after each test case.
+        """
+        self.dut.kill_all()
+        self.perf_reset_cur_case()
+
+    def test_perf_vf_rfc2544_ipv4_lpm(self):
+        self.perf_set_cur_case('test_perf_vf_rfc2544_ipv4_lpm')
+        self.qt_rfc2544(l3_proto=IP_TYPE.V4, mode=MATCH_MODE.LPM)
-- 
2.21.0

[dts] [PATCH V3 11/13] conf/vf_l3fwd_lpm_ipv6_kernelpf: configuration

[yufengmx]

add vf_l3fwd_lpm_ipv6_kernelpf suite testing configuration.

Signed-off-by: yufengmx <yufengx.mo@intel.com>
---
 conf/vf_l3fwd_lpm_ipv6_kernelpf.cfg | 59 +++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)
 create mode 100644 conf/vf_l3fwd_lpm_ipv6_kernelpf.cfg

diff --git a/conf/vf_l3fwd_lpm_ipv6_kernelpf.cfg b/conf/vf_l3fwd_lpm_ipv6_kernelpf.cfg
new file mode 100644
index 00000000..90440fab
--- /dev/null
+++ b/conf/vf_l3fwd_lpm_ipv6_kernelpf.cfg
@@ -0,0 +1,59 @@
+# Users could change these configuration on demand:
+#
+#  detail content reference to ``vf_l3fwd_kernelpf.cfg``
+#
+
+[suite]
+test_duration = 60
+accepted_tolerance = 1
+
+test_parameters = {
+    '1C/1T-1Q': ['64', '128', '256', '512', '1024', '1280', '1518',],
+    '1C/2T-2Q': ['64', '128', '256', '512', '1024', '1280', '1518',],
+    '2C/2T-2Q': ['64', '128', '256', '512', '1024', '1280', '1518',],
+    '2C/4T-4Q': ['64', '128', '256', '512', '1024', '1280', '1518',],
+    '4C/4T-4Q': ['64', '128', '256', '512', '1024', '1280', '1518',],}
+
+expected_throughput = {
+    'test_perf_vf_throughput_ipv6_lpm': {
+        'niantic': {
+            '1C/1T-1Q': {
+                 '64': '0.00',
+                 '128': '0.00',
+                 '256': '0.00',
+                 '512': '0.00',
+                 '1024': '0.00',
+                 '1280': '0.00',
+                 '1518': '0.00', },
+            '1C/2T-2Q': {
+                 '64': '0.00',
+                 '128': '0.00',
+                 '256': '0.00',
+                 '512': '0.00',
+                 '1024': '0.00',
+                 '1280': '0.00',
+                 '1518': '0.00', },
+            '2C/2T-2Q': {
+                 '64': '0.00',
+                 '128': '0.00',
+                 '256': '0.00',
+                 '512': '0.00',
+                 '1024': '0.00',
+                 '1280': '0.00',
+                 '1518': '0.00', },
+            '2C/4T-4Q': {
+                 '64': '0.00',
+                 '128': '0.00',
+                 '256': '0.00',
+                 '512': '0.00',
+                 '1024': '0.00',
+                 '1280': '0.00',
+                 '1518': '0.00', },
+            '4C/4T-4Q': {
+                 '64': '0.00',
+                 '128': '0.00',
+                 '256': '0.00',
+                 '512': '0.00',
+                 '1024': '0.00',
+                 '1280': '0.00',
+                 '1518': '0.00', },},},}
-- 
2.21.0

[dts] [PATCH V3 12/13] tests/vf_l3fwd_lpm_ipv6_kernelpf: script

[yufengmx]

vf_l3fwd_lpm_ipv6_kernelpf suite automation script.

Signed-off-by: yufengmx <yufengx.mo@intel.com>
---
 tests/TestSuite_vf_l3fwd_lpm_ipv6_kernelpf.py | 85 +++++++++++++++++++
 1 file changed, 85 insertions(+)
 create mode 100644 tests/TestSuite_vf_l3fwd_lpm_ipv6_kernelpf.py

diff --git a/tests/TestSuite_vf_l3fwd_lpm_ipv6_kernelpf.py b/tests/TestSuite_vf_l3fwd_lpm_ipv6_kernelpf.py
new file mode 100644
index 00000000..c7cb1b6c
--- /dev/null
+++ b/tests/TestSuite_vf_l3fwd_lpm_ipv6_kernelpf.py
@@ -0,0 +1,85 @@
+# BSD LICENSE
+#
+# Copyright(c) 2010-2021 Intel Corporation. All rights reserved.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+#   * Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#   * Redistributions in binary form must reproduce the above copyright
+#     notice, this list of conditions and the following disclaimer in
+#     the documentation and/or other materials provided with the
+#     distribution.
+#   * Neither the name of Intel Corporation nor the names of its
+#     contributors may be used to endorse or promote products derived
+#     from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import os
+import time
+from test_case import TestCase
+from perf_test_base import PerfTestBase, IP_TYPE, MATCH_MODE, SUITE_TYPE, VF_L3FWD_NIC_SUPPORT
+
+
+class TestVfL3fwdLpmIpv6KernelPf(TestCase, PerfTestBase):
+    #
+    # Test cases.
+    #
+
+    def set_up_all(self):
+        """
+        Run at the start of each test suite.
+        """
+        self.verify(self.nic in VF_L3FWD_NIC_SUPPORT,
+                    "NIC Unsupported: " + str(self.nic))
+        self.dut_ports = self.dut.get_ports(self.nic)
+        valports = [
+            _ for _ in self.dut_ports if self.tester.get_local_port(_) != -1]
+        self.logger.debug(valports)
+        self.verify_ports_number(valports)
+        # get socket and cores
+        socket = self.dut.get_numa_id(self.dut_ports[0])
+        cores = self.dut.get_core_list("1S/6C/1T", socket=socket)
+        self.verify(cores, "Requested 6 cores failed")
+        # init l3fwd common base class parameters
+        PerfTestBase.__init__(self, valports, socket, mode=SUITE_TYPE.VF)
+        # preset testing environment
+        self.perf_preset_test_environment(self.get_suite_cfg())
+
+    def tear_down_all(self):
+        """
+        Run after each test suite.
+        """
+        self.perf_destroy_resource()
+        self.l3fwd_save_results(json_file="{}.json".format(self.suite_name))
+
+    def set_up(self):
+        """
+        Run before each test case.
+        """
+        pass
+
+    def tear_down(self):
+        """
+        Run after each test case.
+        """
+        self.dut.kill_all()
+        self.perf_reset_cur_case()
+
+    def test_perf_vf_throughput_ipv6_lpm(self):
+        self.perf_set_cur_case('test_perf_vf_throughput_ipv6_lpm')
+        self.ms_throughput(l3_proto=IP_TYPE.V6, mode=MATCH_MODE.LPM)
-- 
2.21.0

[dts] [PATCH V3 13/13] tests/perf_test_base: script

[yufengmx]

stage 2 demand as below

#. accepted_tolerance in [test suit].cfg are changed to percentage, if not set,
    it will be set to 2% by default.

#.  delta in json are chagned to percentage.

#. add expected throughput in json.

#. expected_throughput will be forced to NA if it not set.

#. add new conf throughput_stat_sample_interval in conf/<test suite>.cfg, which
    indicates interval of getting throughput statistics in second, e.g.
    throughput_stat_sample_interval = 10
    if not set, it will be set to 5 seconds by default.

#. collect better statistics , get multiple throughput statistics during
    the test run, and compute the average as the final results.
    Every  <throughput_stat_sample_interval >  seconds get throughput once, so it gets multiple throughput statistics during
    the test run, remove the max and the min, then compute the average of the left as the final throughput.

Signed-off-by: yufengmx <yufengx.mo@intel.com>
---
 tests/perf_test_base.py | 84 ++++++++++++++++++++++++++++++++++++++---
 1 file changed, 78 insertions(+), 6 deletions(-)

diff --git a/tests/perf_test_base.py b/tests/perf_test_base.py
index 4b76e6b1..5fbda9fa 100644
--- a/tests/perf_test_base.py
+++ b/tests/perf_test_base.py
@@ -38,6 +38,7 @@ import time
 import traceback
 import texttable
 import json
+import numpy as np
 from enum import Enum, unique
 from pprint import pformat
 from itertools import product
@@ -109,6 +110,12 @@ class STREAM_TYPE(Enum):
     RAW = 'RAW'
 
 
+@unique
+class STAT_ALGORITHM(Enum):
+    ONE = "one"
+    MEAN = "mean"
+
+
 HEADER_SIZE = {
     'ether': 18,
     'ipv4': 20,
@@ -449,6 +456,37 @@ class PerfTestBase(object):
         time.sleep(self.__traffic_stop_wait_time)
         return result
 
+    def __throughput_set_traffic_opt(self, duration):
+        option = {
+            'method': 'throughput',
+            'duration': duration,
+            'interval': self.__throughput_stat_sample_interval,
+        }
+        return option
+
+    def __throughput_calculate_result(self, result):
+        if self.__stat_algorithm is STAT_ALGORITHM.ONE:
+            return result
+
+        if not result or len(result) <= 0:
+            msg = "failed to get throughput stat"
+            self.logger.error(msg)
+            return (0.0, 0.0)
+
+        result = sorted(result)
+        if self.__stat_algorithm is STAT_ALGORITHM.MEAN:
+            result = result[1:-1] if len(result) >= 5 else result
+            bps, pps = [], []
+            for v1, v2 in result:
+                bps.append(v1), pps.append(v2)
+            ret_stat = round(np.array(bps).mean(), 1), round(np.array(pps).mean(), 1)
+        else:
+            msg = f"current not support algorithm <{self.__stat_algorithm}>"
+            self.logger.warning(msg)
+            ret_stat = (0.0, 0.0)
+
+        return ret_stat
+
     def __throughput(self, l3_proto, mode, frame_size):
         """
         measure __throughput according to Layer-3 Protocol and Lookup Mode
@@ -464,11 +502,11 @@ class PerfTestBase(object):
         option = {
             'stream': streams,
             'rate': 100,
-            'traffic_opt': {
-                'method': 'throughput',
-                'duration': duration, }}
+            'traffic_opt': self.__throughput_set_traffic_opt(duration),
+        }
         # run traffic
         result = self.__send_packets_by_pktgen(option)
+        result = self.__throughput_calculate_result(result)
         # statistics result
         _, pps = result
         self.verify(pps > 0, "No traffic detected")
@@ -871,7 +909,8 @@ class PerfTestBase(object):
     def __json_throughput(self, value):
         return {"unit": "Mpps", "name": "Throughput",
                 "value": round(value[0], 3),
-                "delta": round(value[1], 3),
+                "delta": value[1],
+                "expected": value[2],
                 }
 
     def __json_rate_percent(self, value):
@@ -987,12 +1026,16 @@ class PerfTestBase(object):
                         expected, pps, round(gap, 2))
                     self.logger.error(msg)
             else:
-                expected = pps
+                expected = None
                 msg = ('{0} {1} expected throughput value is not set, '
                        'ignore check').format(config, frame_size)
                 self.logger.warning(msg)
                 status = 'pass'
-            js_results.append([status, [pps, pps - expected], percentage, config, frame_size])
+            js_results.append([
+                status,
+                [pps, (str(round((pps - expected)/expected * 100, 3) ) + '%') if expected is not None else "N/A",
+                 round(expected, 3) if expected is not None else "N/A"],
+                percentage, config, frame_size])
         # save data with json format
         self.__save_throughput_result(self.__cur_case, js_results)
         # display result table
@@ -1394,11 +1437,40 @@ class PerfTestBase(object):
             self.__valports = port_list
         return port_list
 
+    @property
+    def __stat_algorithm(self):
+        algorithm_type = get_enum_name(
+            self.__test_content.get('stat_algorithm') or 'one', STAT_ALGORITHM)
+        return algorithm_type
+
+    @property
+    def __throughput_stat_sample_interval(self):
+        if self.__stat_algorithm is STAT_ALGORITHM.ONE:
+            return None
+        else:
+            return self.__test_content.get('throughput_stat_sample_interval') or 5
+
+    def __parse_accepted_tolerance(self):
+        tolerance = self.__test_content.get('accepted_tolerance')
+        def get_num(value):
+            pat = "(\d+\.?\d?)"
+            result = re.findall(pat, value)
+            return result[0] if result else '2.0'
+
+        if isinstance(tolerance, int):
+            _tolerance = float(tolerance)
+        elif isinstance(tolerance, str):
+            _tolerance = float(tolerance if tolerance.isdigit() else get_num(tolerance))
+        else:
+            _tolerance = None
+        self.__test_content['accepted_tolerance'] = float(_tolerance or 2.0)
+
     def perf_preset_test_environment(self, test_content):
         # if user set port list in cfg file, use
         port_list = self.__preset_port_list(test_content)
         # get test content
         self.__test_content = self.__get_test_content_from_cfg(test_content)
+        self.__parse_accepted_tolerance()
         # prepare target source code application
         self.__preset_compilation()
         # set up testing environment
-- 
2.21.0

Re: [dts] [PATCH V3 13/13] tests/perf_test_base: script

[Tu, Lijuan]

Hi Yufeng,

> -----Original Message-----
> From: Mo, YufengX <yufengx.mo@intel.com>
> Sent: 2021年3月25日 16:55
> To: dts@dpdk.org; Tu, Lijuan <lijuan.tu@intel.com>
> Cc: Mo, YufengX <yufengx.mo@intel.com>
> Subject: [dts][PATCH V3 13/13] tests/perf_test_base: script
> 
> 
> stage 2 demand as below
> 
> #. accepted_tolerance in [test suit].cfg are changed to percentage, if not set,
>     it will be set to 2% by default.
> 
> #.  delta in json are chagned to percentage.

Could you please change accepted_tolerance and delta to float for better integration to CI.
More details see: http://patchwork.dpdk.org/project/dts/patch/20210311104800.243752-1-lijuan.tu@intel.com/

> 
> #. add expected throughput in json.
> 
> #. expected_throughput will be forced to NA if it not set.
> 
> #. add new conf throughput_stat_sample_interval in conf/<test suite>.cfg,
> which
>     indicates interval of getting throughput statistics in second, e.g.
>     throughput_stat_sample_interval = 10
>     if not set, it will be set to 5 seconds by default.
> 
> #. collect better statistics , get multiple throughput statistics during
>     the test run, and compute the average as the final results.
>     Every  <throughput_stat_sample_interval >  seconds get throughput once, so
> it gets multiple throughput statistics during
>     the test run, remove the max and the min, then compute the average of the
> left as the final throughput.
> 
> Signed-off-by: yufengmx <yufengx.mo@intel.com>
> ---
>  tests/perf_test_base.py | 84 ++++++++++++++++++++++++++++++++++++++---
>  1 file changed, 78 insertions(+), 6 deletions(-)
> 
> diff --git a/tests/perf_test_base.py b/tests/perf_test_base.py index
> 4b76e6b1..5fbda9fa 100644
> --- a/tests/perf_test_base.py
> +++ b/tests/perf_test_base.py
> @@ -38,6 +38,7 @@ import time
>  import traceback
>  import texttable
>  import json
> +import numpy as np
>  from enum import Enum, unique
>  from pprint import pformat
>  from itertools import product
> @@ -109,6 +110,12 @@ class STREAM_TYPE(Enum):
>      RAW = 'RAW'
> 
> 
> +@unique
> +class STAT_ALGORITHM(Enum):
> +    ONE = "one"
> +    MEAN = "mean"
> +
> +
>  HEADER_SIZE = {
>      'ether': 18,
>      'ipv4': 20,
> @@ -449,6 +456,37 @@ class PerfTestBase(object):
>          time.sleep(self.__traffic_stop_wait_time)
>          return result
> 
> +    def __throughput_set_traffic_opt(self, duration):
> +        option = {
> +            'method': 'throughput',
> +            'duration': duration,
> +            'interval': self.__throughput_stat_sample_interval,
> +        }
> +        return option
> +
> +    def __throughput_calculate_result(self, result):
> +        if self.__stat_algorithm is STAT_ALGORITHM.ONE:
> +            return result
> +
> +        if not result or len(result) <= 0:
> +            msg = "failed to get throughput stat"
> +            self.logger.error(msg)
> +            return (0.0, 0.0)
> +
> +        result = sorted(result)
> +        if self.__stat_algorithm is STAT_ALGORITHM.MEAN:
> +            result = result[1:-1] if len(result) >= 5 else result
> +            bps, pps = [], []
> +            for v1, v2 in result:
> +                bps.append(v1), pps.append(v2)
> +            ret_stat = round(np.array(bps).mean(), 1), round(np.array(pps).mean(), 1)
> +        else:
> +            msg = f"current not support algorithm <{self.__stat_algorithm}>"
> +            self.logger.warning(msg)
> +            ret_stat = (0.0, 0.0)
> +
> +        return ret_stat
> +
>      def __throughput(self, l3_proto, mode, frame_size):
>          """
>          measure __throughput according to Layer-3 Protocol and Lookup Mode
> @@ -464,11 +502,11 @@ class PerfTestBase(object):
>          option = {
>              'stream': streams,
>              'rate': 100,
> -            'traffic_opt': {
> -                'method': 'throughput',
> -                'duration': duration, }}
> +            'traffic_opt': self.__throughput_set_traffic_opt(duration),
> +        }
>          # run traffic
>          result = self.__send_packets_by_pktgen(option)
> +        result = self.__throughput_calculate_result(result)
>          # statistics result
>          _, pps = result
>          self.verify(pps > 0, "No traffic detected") @@ -871,7 +909,8 @@ class
> PerfTestBase(object):
>      def __json_throughput(self, value):
>          return {"unit": "Mpps", "name": "Throughput",
>                  "value": round(value[0], 3),
> -                "delta": round(value[1], 3),
> +                "delta": value[1],
> +                "expected": value[2],
>                  }
> 
>      def __json_rate_percent(self, value):
> @@ -987,12 +1026,16 @@ class PerfTestBase(object):
>                          expected, pps, round(gap, 2))
>                      self.logger.error(msg)
>              else:
> -                expected = pps
> +                expected = None
>                  msg = ('{0} {1} expected throughput value is not set, '
>                         'ignore check').format(config, frame_size)
>                  self.logger.warning(msg)
>                  status = 'pass'
> -            js_results.append([status, [pps, pps - expected], percentage, config,
> frame_size])
> +            js_results.append([
> +                status,
> +                [pps, (str(round((pps - expected)/expected * 100, 3) ) + '%') if expected
> is not None else "N/A",
> +                 round(expected, 3) if expected is not None else "N/A"],
> +                percentage, config, frame_size])
>          # save data with json format
>          self.__save_throughput_result(self.__cur_case, js_results)
>          # display result table
> @@ -1394,11 +1437,40 @@ class PerfTestBase(object):
>              self.__valports = port_list
>          return port_list
> 
> +    @property
> +    def __stat_algorithm(self):
> +        algorithm_type = get_enum_name(
> +            self.__test_content.get('stat_algorithm') or 'one', STAT_ALGORITHM)
> +        return algorithm_type
> +
> +    @property
> +    def __throughput_stat_sample_interval(self):
> +        if self.__stat_algorithm is STAT_ALGORITHM.ONE:
> +            return None
> +        else:
> +            return
> + self.__test_content.get('throughput_stat_sample_interval') or 5
> +
> +    def __parse_accepted_tolerance(self):
> +        tolerance = self.__test_content.get('accepted_tolerance')
> +        def get_num(value):
> +            pat = "(\d+\.?\d?)"
> +            result = re.findall(pat, value)
> +            return result[0] if result else '2.0'
> +
> +        if isinstance(tolerance, int):
> +            _tolerance = float(tolerance)
> +        elif isinstance(tolerance, str):
> +            _tolerance = float(tolerance if tolerance.isdigit() else
> get_num(tolerance))
> +        else:
> +            _tolerance = None
> +        self.__test_content['accepted_tolerance'] = float(_tolerance or
> + 2.0)
> +
>      def perf_preset_test_environment(self, test_content):
>          # if user set port list in cfg file, use
>          port_list = self.__preset_port_list(test_content)
>          # get test content
>          self.__test_content = self.__get_test_content_from_cfg(test_content)
> +        self.__parse_accepted_tolerance()
>          # prepare target source code application
>          self.__preset_compilation()
>          # set up testing environment
> --
> 2.21.0