[dts] [PATCH] tests/nic_single_core_perf: modify accepted_tolerance to percentage

[Lijuan Tu <lijuan.tu@intel.com>]

* accepted_tolerance in conf/nic_single_core_perf.cfg are changed to
percentage, e.g.
	accepted_tolerance = '1%%'
	if not set, it will be set to 2% by default.
* delat in json are chagned to percentage too, e.g.
	'delta': '1.5%'
* expected_throughput will be forced to 1Mpps for compute delat in
percentage
* add new conf test_interval in conf/nic_single_core_perf.cfg, which
indicates interval of getting throughput statistics in second, e.g.
	test_interval = 10
	if not set, it will be set to 5 seconds by default.
* collect better statistics , get multiple throughput statistics during
the test run, and compute the average as the final results.

Signed-off-by: Lijuan Tu <lijuan.tu@intel.com>
---
 conf/nic_single_core_perf.cfg           |   9 ++-
 tests/TestSuite_nic_single_core_perf.py | 102 +++++++++++++++---------
 2 files changed, 71 insertions(+), 40 deletions(-)

diff --git a/conf/nic_single_core_perf.cfg b/conf/nic_single_core_perf.cfg
index 4d3279ba..b673a2fa 100644
--- a/conf/nic_single_core_perf.cfg
+++ b/conf/nic_single_core_perf.cfg
@@ -8,8 +8,12 @@
 #  - test_duration is how many seconds each combination performance will
 #  be recorded.
 #
+#  -test_interval defines interval of get throughput statistics (second).
+#  If not set, it is 5 seconds by default.
+#
 #  - accepted_tolerance defines the accepted tolerance between test
-# results and expected numbers.
+# results and expected numbers, it is a pecentage. If not set, it is 2%
+# by default.
 #
 #  - expected_throughput is a dictionary defining expected throughput
 # numbers based on NIC, and the pattern is
@@ -34,7 +38,8 @@ test_parameters = {'1C/1T': {64: [512, 2048]},
                     '1C/2T': {64: [512, 2048]}}
 rx_desc_16byte = 'y'
 test_duration = 60
-accepted_tolerance = 1
+test_interval = 10
+accepted_tolerance = '1%%'
 expected_throughput = {
     'fortville_spirit': {
         '1C/1T': {64: {512: 0.00, 2048: 0.00}},
diff --git a/tests/TestSuite_nic_single_core_perf.py b/tests/TestSuite_nic_single_core_perf.py
index 8266506b..6c4095d3 100644
--- a/tests/TestSuite_nic_single_core_perf.py
+++ b/tests/TestSuite_nic_single_core_perf.py
@@ -41,6 +41,7 @@ from exception import VerifyFailure
 from settings import HEADER_SIZE, UPDATE_EXPECTED, load_global_setting
 from pmd_output import PmdOutput
 from copy import deepcopy
+from numpy import mean
 import rst
 from pktgen import PacketGeneratorHelper
 
@@ -57,7 +58,7 @@ class TestNicSingleCorePerf(TestCase):
                                  "Not required NIC ")
         self.headers_size = HEADER_SIZE['eth'] + HEADER_SIZE['ip']
 
-        self.rx_desc = self.get_suite_cfg()['rx_desc_16byte']
+        self.rx_desc = self.get_suite_cfg().get('rx_desc_16byte', 'y')
         if 'y' == self.rx_desc:
             # Update DPDK config file and rebuild to get best perf on fortville
             if self.nic in ["fortville_25g", "fortville_spirit"]:
@@ -92,10 +93,11 @@ class TestNicSingleCorePerf(TestCase):
         """
 
         # test parameters include: frames size, descriptor numbers
-        self.test_parameters = self.get_suite_cfg()['test_parameters']
+        self.test_parameters = self.get_suite_cfg().get('test_parameters', {'1C/1T': {64: [512, 2048]}})
 
         # traffic duraion in second
-        self.test_duration = self.get_suite_cfg()['test_duration']
+        self.test_duration = self.get_suite_cfg().get('test_duration', 60)
+        self.test_interval = self.get_suite_cfg().get('test_interval', 5)
 
         # load the expected throughput for required nic
         if self.nic in ["ConnectX4_LX_MT4117"]:
@@ -114,12 +116,12 @@ class TestNicSingleCorePerf(TestCase):
         # {'$framesize':{"$nb_desc": 'throughput'}
         self.throughput = {}
 
-        # Accepted tolerance in Mpps
-        self.gap = self.get_suite_cfg()['accepted_tolerance']
+        # Accepted tolerance in pecentage
+        self.gap = float(self.get_suite_cfg().get('accepted_tolerance', '2%')[:-1])
 
         # header to print test result table
-        self.table_header = ['Fwd_core', 'Frame Size', 'TXD/RXD', 'Throughput', 'Rate',
-                             'Expected Throughput', 'Throughput Difference']
+        self.table_header = ['Fwd_core', 'Frame Size', 'TXD/RXD', 'Real-Mpps', 'Rate',
+                             'Expected-Mpps', 'Real vs Expected', 'Status']
         self.test_result = {}
 
     def flows(self):
@@ -232,7 +234,7 @@ class TestNicSingleCorePerf(TestCase):
             # need add one more core for start testpmd
             core_list = [core_list[0]] + [str(int(i) + 1) for i in core_list]
 
-            self.logger.info("Executing Test Using cores: %s of config %s" % (core_list, fwd_config))
+            self.logger.info("Executing Test Using cores: %s of config %s" % (core_list[1:], fwd_config))
 
             nb_cores = thread_num
 
@@ -242,8 +244,6 @@ class TestNicSingleCorePerf(TestCase):
             # columbiaville use one queue per port for best performance.
             elif self.nic in ["columbiaville_100g", "columbiaville_25g", "columbiaville_25gx2"]:
                 param += " --rxq=1 --txq=1"
-                # workaround for that testpmd can't forward packets in io forward mode
-                param += " --port-topology=loop"
 
             self.throughput[fwd_config] = dict()
             for frame_size in list(self.test_parameters[fwd_config].keys()):
@@ -265,21 +265,42 @@ class TestNicSingleCorePerf(TestCase):
                     # run packet generator
                     streams = self.pktgen_helper.prepare_stream_from_tginput(tgenInput, 100, vm_config, self.tester.pktgen)
                     # set traffic option
-                    traffic_opt = {'duration': self.test_duration}
-                    # _, pps = self.tester.traffic_generator_throughput(tgenInput, rate_percent=100, delay=30)
-                    _, packets_received = self.tester.pktgen.measure_throughput(stream_ids=streams, options=traffic_opt)
-                    self.verify(packets_received > 0, "No traffic detected")
-                    throughput = packets_received / 1000000.0
-                    self.throughput[fwd_config][frame_size][nb_desc] = throughput
+                    traffic_opt = {
+                        'method': 'throughput',
+                        'rate': 100,
+                        'duration': self.test_duration,
+                        'interval': self.test_interval,
+                        }
+                    stats = self.tester.pktgen.measure(stream_ids=streams, traffic_opt=traffic_opt)
+
+                    #####################################################
+                    # Remove max and min, then get average
+                    #####################################################
+                    if isinstance(stats, list):
+                        total_pps_rxs = []
+                        for i in range(len(stats)):
+                            stats_pps = stats[i][1]
+                            if isinstance(stats_pps, tuple):
+                                total_pps_rxs.append(stats_pps[1])
+                            else:
+                                total_pps_rxs.append(stats_pps)
+
+                        total_pps_rxs.remove(max(total_pps_rxs))
+                        total_pps_rxs.remove(min(total_pps_rxs))
+                        total_pps_rx = mean(total_pps_rxs)
+                    else:
+                        total_pps_rx = stats
+
+                    self.verify(total_pps_rx > 0, "No traffic detected, please check your configuration")
+                    total_mpps_rx = total_pps_rx / 1000000.0
+                    self.throughput[fwd_config][frame_size][nb_desc] = total_mpps_rx
 
                     self.dut.send_expect("stop", "testpmd> ")
                     self.dut.send_expect("quit", "# ", 30)
 
-                    self.verify(throughput,
-                        "No traffic detected, please check your configuration")
                     self.logger.info("Trouthput of " +
                         "framesize: {}, rxd/txd: {} is :{} Mpps".format(
-                            frame_size, nb_desc, throughput))
+                            frame_size, nb_desc, total_mpps_rx))
 
         return self.throughput
 
@@ -303,16 +324,25 @@ class TestNicSingleCorePerf(TestCase):
                     ret_data[header[0]] = fwd_config
                     ret_data[header[1]] = frame_size
                     ret_data[header[2]] = nb_desc
-                    ret_data[header[3]] = "{:.3f} Mpps".format(
+                    ret_data[header[3]] = "{:.3f}".format(
                         self.throughput[fwd_config][frame_size][nb_desc])
                     ret_data[header[4]] = "{:.3f}%".format(
                         self.throughput[fwd_config][frame_size][nb_desc] * 100 / wirespeed)
-                    ret_data[header[5]] = "{:.3f} Mpps".format(
-                        self.expected_throughput[fwd_config][frame_size][nb_desc])
-                    ret_data[header[6]] = "{:.3f} Mpps".format(
-                        self.throughput[fwd_config][frame_size][nb_desc] -
+                    if 0 == self.expected_throughput[fwd_config][frame_size][nb_desc]:
+                        self.expected_throughput[fwd_config][frame_size][nb_desc] = 1
+                        self.logger.warning("Expected throughput can't be zero !!")
+                        msg = "Force it to 1 Mpps for frame size: {0}, RXD/TXD: {1}, Core: {2}"
+                        self.logger.warning(msg.format(frame_size, nb_desc, fwd_config))
+                    ret_data[header[5]] = "{:.3f}".format(
                             self.expected_throughput[fwd_config][frame_size][nb_desc])
-
+                    diff = (self.throughput[fwd_config][frame_size][nb_desc] -
+                            self.expected_throughput[fwd_config][frame_size][nb_desc])/ \
+                                    self.expected_throughput[fwd_config][frame_size][nb_desc]
+                    ret_data[header[6]] = "{:.3f}%".format(diff)
+                    if diff >= -self.gap and diff <= self.gap:
+                        ret_data[header[7]] = 'PASS'
+                    else:
+                        ret_data[header[7]] = 'FAIL'
                     ret_datas[frame_size][nb_desc] = deepcopy(ret_data)
                 self.test_result[fwd_config] = deepcopy(ret_datas)
 
@@ -350,15 +380,11 @@ class TestNicSingleCorePerf(TestCase):
                     row_dict0 = dict()
                     row_dict0['performance'] = list()
                     row_dict0['parameters'] = list()
-                    result_throughput = float(row_in['Throughput'].split()[0])
-                    expected_throughput = float(row_in['Expected Throughput'].split()[0])
-                    # delta value and accepted tolerance in percentage
-                    delta = result_throughput - expected_throughput
-                    if delta > -self.gap:
-                        row_dict0['status'] = 'PASS'
-                    else:
-                        row_dict0['status'] = 'FAIL'
-                    row_dict1 = dict(name="Throughput", value=result_throughput, unit="Mpps", delta=delta)
+                    result_throughput = float(row_in['Real-Mpps'].split()[0])
+                    expected_throughput = float(row_in['Expected-Mpps'].split()[0])
+                    row_dict0['status'] = row_in['Status']
+                    row_dict1 = dict(name="Throughput", value=result_throughput, unit="Mpps",
+                            delta=row_in['Real vs Expected'], expected=expected_throughput)
                     row_dict2 = dict(name="Txd/Rxd", value=row_in["TXD/RXD"], unit="descriptor")
                     row_dict3 = dict(name="frame_size", value=row_in["Frame Size"], unit="bytes")
                     row_dict4 = dict(name="Fwd_core", value=row_in["Fwd_core"])
@@ -368,10 +394,10 @@ class TestNicSingleCorePerf(TestCase):
                     row_dict0['parameters'].append(row_dict4)
                     json_obj[case_name].append(row_dict0)
                     status_result.append(row_dict0['status'])
-        with open(os.path.join(rst.path2Result,
-                               '{0:s}_single_core_perf.json'.format(
-                                   self.nic)), 'w') as fp:
-            json.dump(json_obj, fp)
+
+        json_file = os.path.join(rst.path2Result, '{0:s}_single_core_perf.json'.format(self.nic))
+        with open(json_file, 'w') as fp:
+            json.dump(json_obj, fp, indent=4, separators=(',', ': '), sort_keys=True)
         self.verify("FAIL" not in status_result, "Excessive gap between test results and expectations")
 
     def set_fields(self):
-- 
2.25.1