[dts] [next][PATCH V1 0/2]redefine tolerance for single core performance testing

[dts] [next][PATCH V1 0/2]redefine tolerance for single core performance testing

[Lijuan Tu]

Change tolerance value to percentage instead of delta.

Lijuan Tu (2):
  tests/nic_single_core_perf: redefine accecpt_tolerance
  conf/nic_single_core_perf: clarify the value of tolerence

 conf/nic_single_core_perf.cfg           |  4 ++--
 tests/TestSuite_nic_single_core_perf.py | 40 ++++++++++++++++++++++++---------
 2 files changed, 32 insertions(+), 12 deletions(-)

-- 
1.8.3.1

[dts] [next][PATCH V1 1/2] tests/nic_single_core_perf: redefine accecpt_tolerance

[Lijuan Tu]

Current accecpt_tolerance is a delta value, change it to a percentage
value, as a percentage value is more make sense for tolerance.

Signed-off-by: Lijuan Tu <lijuan.tu@intel.com>
---
 tests/TestSuite_nic_single_core_perf.py | 40 ++++++++++++++++++++++++---------
 1 file changed, 30 insertions(+), 10 deletions(-)

diff --git a/tests/TestSuite_nic_single_core_perf.py b/tests/TestSuite_nic_single_core_perf.py
index 847a94c..b4d682e 100644
--- a/tests/TestSuite_nic_single_core_perf.py
+++ b/tests/TestSuite_nic_single_core_perf.py
@@ -74,6 +74,18 @@ class TestNicSingleCorePerf(TestCase):
         # determine if to save test result as a separated file
         self.save_result_flag = True
 
+    def convert_tolenrance_to_gap(self):
+        # Accepted tolerance in percentage
+        gap = dict()
+        rate = self.get_suite_cfg()['accepted_tolerance'] * 0.01
+        for frame_size in self.test_parameters.keys():
+            gap[frame_size] = dict()
+            for nb_desc in self.test_parameters[frame_size]:
+                expected_gap = rate * self.expected_throughput[frame_size][nb_desc]
+                gap[frame_size][nb_desc] = round(expected_gap, 3)
+        self.logger.info("Accept tolerance are (Mpps)%s" % gap)
+        return gap
+
     def set_up(self):
         """
         Run before each test case.
@@ -104,14 +116,27 @@ class TestNicSingleCorePerf(TestCase):
         # {'$framesize':{"$nb_desc": 'throughput'}
         self.throughput = {}
 
-        # Accepted tolerance in Mpps
-        self.gap = self.get_suite_cfg()['accepted_tolerance']
+        # self.gap has same structure as throughput.
+        self.gap = self.convert_tolenrance_to_gap()
 
         # header to print test result table
         self.table_header = ['Frame Size', 'TXD/RXD', 'Throughput', 'Rate',
                              'Expected Throughput', 'Throughput Difference']
         self.test_result = {}
 
+    def get_gap_border(self, frame_size, nb_desc):
+        return -self.gap[frame_size][nb_desc]
+
+    def verify_tolerance(self):
+        for frame_size in self.test_parameters.keys():
+            for nb_desc in self.test_parameters[frame_size]:
+                cur_gap = self.throughput[frame_size][nb_desc] - self.expected_throughput[frame_size][nb_desc]
+                gap_border = self.get_gap_border(frame_size, nb_desc)
+                show_str = "Possible regression, " +\
+                           "Frame size/Descriptors: {}/{}, " +\
+                           "Acceptable Gap/Actual Gap: {}/{}"
+                self.verify(cur_gap >= gap_border,show_str.format(frame_size, nb_desc, gap_border, cur_gap))
+
     def test_nic_single_core_perf(self):
         """
         Run nic single core performance 
@@ -124,12 +149,7 @@ class TestNicSingleCorePerf(TestCase):
 
         # check the gap between expected throughput and actual throughput
         try:
-            for frame_size in self.test_parameters.keys():
-                for nb_desc in self.test_parameters[frame_size]:
-                    cur_gap = (self.expected_throughput[frame_size][nb_desc] -
-                                self.throughput[frame_size][nb_desc])
-                    self.verify(cur_gap < self.gap,
-                                 "Beyond Gap, Possible regression")
+            self.verify_tolerance()
         except Exception as e:
             self.logger.error(e)
             self.handle_expected()
@@ -146,7 +166,7 @@ class TestNicSingleCorePerf(TestCase):
             for frame_size in self.test_parameters.keys():
                 for nb_desc in self.test_parameters[frame_size]:
                     self.expected_throughput[frame_size][nb_desc] = \
-                        round(self.throughput[frame_size][nb_desc],3)
+                        round(self.throughput[frame_size][nb_desc], 3)
 
     def perf_test(self, port_num):
         """
@@ -303,7 +323,7 @@ class TestNicSingleCorePerf(TestCase):
                     value = row_in['TXD/RXD'], unit = 'descriptors')
                 delta = (float(row_in['Throughput'].split()[0]) -
                          float(row_in['Expected Throughput'].split()[0]))
-                if delta >= -self.gap:
+                if delta >= self.get_gap_border(frame_size, nb_desc):
                     result = 'PASS'
                 else:
                     result = 'FAIL'
-- 
1.8.3.1

[dts] [next][PATCH V1 2/2] conf/nic_single_core_perf: clarify the value of tolerence

[Lijuan Tu]

Signed-off-by: Lijuan Tu <lijuan.tu@intel.com>
---
 conf/nic_single_core_perf.cfg | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/conf/nic_single_core_perf.cfg b/conf/nic_single_core_perf.cfg
index da1a050..be4f130 100644
--- a/conf/nic_single_core_perf.cfg
+++ b/conf/nic_single_core_perf.cfg
@@ -6,7 +6,7 @@
 #  - test_duration is how many seconds each combination performance will
 #  be recorded.
 #  - accepted_tolerance defines the accepted tolerance between test
-# results and expected numbers.
+# results and expected numbers, value "2" means "2%"
 #  - expected_throughput is a dictionary defining expected throughput
 # numbers based on NIC, and the pattern is
 # {'NIC': {'frame size': {'descriptor number': 'excepted throughput'}}}
@@ -22,7 +22,7 @@
 update_expected = True
 test_parameters = {64: [512, 2048]}
 test_duration = 60
-accepted_tolerance = 1
+accepted_tolerance = 2
 expected_throughput = {'fortville_spirit': {64: {512: 62.35, 2048: 47.89}},
     'niantic': {64: {128: 53.435, 512: 53.699, 2048: 42.798}},
     'fortville_25g': {64: {512: 62.35, 2048: 47.651}},
-- 
1.8.3.1