[dts] [v3 4/6] tests/nic_single_core_perf: revise delta to ratio

[Lijuan Tu <lijuan.tu@intel.com>]

the delta used to defines the difference between real pps and expected
pps, the case fails if it exceed the absolute number specified by
'accepted_tolerance' in conf/nic_single_core_perf.cfg, the deault is
1Mpps, which is not friendly when the expected is only a few of million,
so updated the accepted_tolerance to a percentage, considering the CI
intergation, so the type is float.

Here are main changes:
* the meaning of accepted_tolerance
    before: threshold for (real-mpps - expected-mpps)
    after: threshold for (real-mpps - expected-mpps)/expected-mpps
* Pass criteria
    before:
      real-mpps - expected-mpps > -accepted_tolerance
    after:
      (real-mpps - expected-mpps)/expected-mpps > -accepted_tolerance
* rst table
    before:
      ... +---------------+-----------------+--------+
      ... | Expected-Mpps | Real - Expected | Status |
      ... +===============+=================+========+
      ... | 9.900         | -0.040          | PASS   |
      ... +---------------+-----------------+--------+

    after:
      ... +---------------+------------+--------+
      ... | Expected-Mpps | Fluc Ratio | Status |
      ... +===============+============+========+
      ... | 9.900         | -0.004     | PASS   |
      ... +---------------+------------+--------+
* delta in JSON:
    before: real-mpps - expected-mpps
    after: (real-mpps - expected-mpps)/expected-mpps

Signed-off-by: Lijuan Tu <lijuan.tu@intel.com>
---
 conf/nic_single_core_perf.cfg           | 10 +++++---
 tests/TestSuite_nic_single_core_perf.py | 32 ++++++++++++-------------
 2 files changed, 22 insertions(+), 20 deletions(-)

diff --git a/conf/nic_single_core_perf.cfg b/conf/nic_single_core_perf.cfg
index e6b0a5af..72b8c533 100644
--- a/conf/nic_single_core_perf.cfg
+++ b/conf/nic_single_core_perf.cfg
@@ -11,8 +11,12 @@
 #  - test_duration is how many seconds each combination performance will
 #  be recorded.
 #
-#  - accepted_tolerance defines the accepted tolerance between test
-# results and expected numbers.
+#  - accepted_tolerance defines the accepted tolerance for the
+#  fluctuation ratio of performance, it is a float(e.g. 0.05).
+#  Assume:
+#    F = (Real-Mpps - Expected-Mpps)/Expected-Mpps
+#    T = accepted_tolerance
+#  If F < -T, the case is PASS, else it is FAIL
 #
 #  - expected_throughput is a dictionary defining expected throughput
 # numbers based on NIC, and the pattern is
@@ -38,7 +42,7 @@ test_parameters = {'1C/1T': {64: [512, 2048]},
 rx_desc_16byte = 'y'
 test_duration = 60
 throughput_stat_sample_interval = 2
-accepted_tolerance = 1
+accepted_tolerance = 0.05
 expected_throughput = {
     'fortville_spirit': {
         '1C/1T': {64: {512: 0.00, 2048: 0.00}},
diff --git a/tests/TestSuite_nic_single_core_perf.py b/tests/TestSuite_nic_single_core_perf.py
index 095b508a..dd35caac 100644
--- a/tests/TestSuite_nic_single_core_perf.py
+++ b/tests/TestSuite_nic_single_core_perf.py
@@ -116,12 +116,12 @@ class TestNicSingleCorePerf(TestCase):
         # {'$framesize':{"$nb_desc": 'throughput'}
         self.throughput = {}
 
-        # Accepted tolerance in Mpps
-        self.gap = self.get_suite_cfg()['accepted_tolerance']
+        # Accepted tolerance is ratio
+        self.gap = self.get_suite_cfg().get('accepted_tolerance', 0.1)
 
         # header to print test result table
         self.table_header = ['Fwd_core', 'Frame Size', 'TXD/RXD', 'Real-Mpps', 'Rate',
-                             'Expected-Mpps', 'Real - Expected', 'Status']
+                             'Expected-Mpps', 'Fluc Ratio', 'Status']
         self.test_result = {}
 
     def flows(self):
@@ -332,12 +332,16 @@ class TestNicSingleCorePerf(TestCase):
                     ret_data[header[3]] = "{:.3f}".format(_real)
                     ret_data[header[4]] = "{:.3f}%".format(_real * 100 / wirespeed)
                     ret_data[header[5]] = "{:.3f}".format(_exp)
-                    delta = _real - _exp
-                    ret_data[header[6]] = "{:.3f}".format(delta)
-                    if delta > -self.gap:
-                        ret_data[header[7]] = 'PASS'
+                    delta = (_real - _exp)/_exp
+                    if _exp != 0:
+                        ret_data[header[6]] = "{:.3f}".format(delta)
+                        if delta > -self.gap:
+                            ret_data[header[7]] = 'PASS'
+                        else:
+                            ret_data[header[7]] = 'FAIL'
                     else:
-                        ret_data[header[7]] = 'FAIL'
+                        ret_data[header[6]] = "N/A"
+                        ret_data[header[7]] = 'PASS'
 
                     ret_datas[frame_size][nb_desc] = deepcopy(ret_data)
                 self.test_result[fwd_config] = deepcopy(ret_datas)
@@ -376,15 +380,9 @@ class TestNicSingleCorePerf(TestCase):
                     row_dict0 = dict()
                     row_dict0['performance'] = list()
                     row_dict0['parameters'] = list()
-                    result_throughput = float(row_in['Real-Mpps'])
-                    expected_throughput = float(row_in['Expected-Mpps'])
-                    # delta value and accepted tolerance in percentage
-                    delta = result_throughput - expected_throughput
-                    if delta > -self.gap:
-                        row_dict0['status'] = 'PASS'
-                    else:
-                        row_dict0['status'] = 'FAIL'
-                    row_dict1 = dict(name="Throughput", value=result_throughput, unit="Mpps", delta=delta)
+                    row_dict0['status'] = row_in['Status']
+                    row_dict1 = dict(name="Throughput", value=row_in['Real-Mpps'], unit="Mpps",
+                                     delta=row_in['Fluc Ratio'])
                     row_dict2 = dict(name="Txd/Rxd", value=row_in["TXD/RXD"], unit="descriptor")
                     row_dict3 = dict(name="frame_size", value=row_in["Frame Size"], unit="bytes")
                     row_dict4 = dict(name="Fwd_core", value=row_in["Fwd_core"])
-- 
2.25.1