test suite reviews and discussions
 help / color / mirror / Atom feed
* [dts] [PATCH] tests/nic_single_core_perf: modify accepted_tolerance to percentage
@ 2021-03-11 10:48 Lijuan Tu
  2021-03-22 18:13 ` Ali Alnubani
  2021-03-23  8:43 ` Ali Alnubani
  0 siblings, 2 replies; 9+ messages in thread
From: Lijuan Tu @ 2021-03-11 10:48 UTC (permalink / raw)
  To: dpdklab; +Cc: dts, Lijuan Tu

* accepted_tolerance in conf/nic_single_core_perf.cfg are changed to
percentage, e.g.
	accepted_tolerance = '1%%'
	if not set, it will be set to 2% by default.
* delat in json are chagned to percentage too, e.g.
	'delta': '1.5%'
* expected_throughput will be forced to 1Mpps for compute delat in
percentage
* add new conf test_interval in conf/nic_single_core_perf.cfg, which
indicates interval of getting throughput statistics in second, e.g.
	test_interval = 10
	if not set, it will be set to 5 seconds by default.
* collect better statistics , get multiple throughput statistics during
the test run, and compute the average as the final results.

Signed-off-by: Lijuan Tu <lijuan.tu@intel.com>
---
 conf/nic_single_core_perf.cfg           |   9 ++-
 tests/TestSuite_nic_single_core_perf.py | 102 +++++++++++++++---------
 2 files changed, 71 insertions(+), 40 deletions(-)

diff --git a/conf/nic_single_core_perf.cfg b/conf/nic_single_core_perf.cfg
index 4d3279ba..b673a2fa 100644
--- a/conf/nic_single_core_perf.cfg
+++ b/conf/nic_single_core_perf.cfg
@@ -8,8 +8,12 @@
 #  - test_duration is how many seconds each combination performance will
 #  be recorded.
 #
+#  -test_interval defines interval of get throughput statistics (second).
+#  If not set, it is 5 seconds by default.
+#
 #  - accepted_tolerance defines the accepted tolerance between test
-# results and expected numbers.
+# results and expected numbers, it is a pecentage. If not set, it is 2%
+# by default.
 #
 #  - expected_throughput is a dictionary defining expected throughput
 # numbers based on NIC, and the pattern is
@@ -34,7 +38,8 @@ test_parameters = {'1C/1T': {64: [512, 2048]},
                     '1C/2T': {64: [512, 2048]}}
 rx_desc_16byte = 'y'
 test_duration = 60
-accepted_tolerance = 1
+test_interval = 10
+accepted_tolerance = '1%%'
 expected_throughput = {
     'fortville_spirit': {
         '1C/1T': {64: {512: 0.00, 2048: 0.00}},
diff --git a/tests/TestSuite_nic_single_core_perf.py b/tests/TestSuite_nic_single_core_perf.py
index 8266506b..6c4095d3 100644
--- a/tests/TestSuite_nic_single_core_perf.py
+++ b/tests/TestSuite_nic_single_core_perf.py
@@ -41,6 +41,7 @@ from exception import VerifyFailure
 from settings import HEADER_SIZE, UPDATE_EXPECTED, load_global_setting
 from pmd_output import PmdOutput
 from copy import deepcopy
+from numpy import mean
 import rst
 from pktgen import PacketGeneratorHelper
 
@@ -57,7 +58,7 @@ class TestNicSingleCorePerf(TestCase):
                                  "Not required NIC ")
         self.headers_size = HEADER_SIZE['eth'] + HEADER_SIZE['ip']
 
-        self.rx_desc = self.get_suite_cfg()['rx_desc_16byte']
+        self.rx_desc = self.get_suite_cfg().get('rx_desc_16byte', 'y')
         if 'y' == self.rx_desc:
             # Update DPDK config file and rebuild to get best perf on fortville
             if self.nic in ["fortville_25g", "fortville_spirit"]:
@@ -92,10 +93,11 @@ class TestNicSingleCorePerf(TestCase):
         """
 
         # test parameters include: frames size, descriptor numbers
-        self.test_parameters = self.get_suite_cfg()['test_parameters']
+        self.test_parameters = self.get_suite_cfg().get('test_parameters', {'1C/1T': {64: [512, 2048]}})
 
         # traffic duraion in second
-        self.test_duration = self.get_suite_cfg()['test_duration']
+        self.test_duration = self.get_suite_cfg().get('test_duration', 60)
+        self.test_interval = self.get_suite_cfg().get('test_interval', 5)
 
         # load the expected throughput for required nic
         if self.nic in ["ConnectX4_LX_MT4117"]:
@@ -114,12 +116,12 @@ class TestNicSingleCorePerf(TestCase):
         # {'$framesize':{"$nb_desc": 'throughput'}
         self.throughput = {}
 
-        # Accepted tolerance in Mpps
-        self.gap = self.get_suite_cfg()['accepted_tolerance']
+        # Accepted tolerance in pecentage
+        self.gap = float(self.get_suite_cfg().get('accepted_tolerance', '2%')[:-1])
 
         # header to print test result table
-        self.table_header = ['Fwd_core', 'Frame Size', 'TXD/RXD', 'Throughput', 'Rate',
-                             'Expected Throughput', 'Throughput Difference']
+        self.table_header = ['Fwd_core', 'Frame Size', 'TXD/RXD', 'Real-Mpps', 'Rate',
+                             'Expected-Mpps', 'Real vs Expected', 'Status']
         self.test_result = {}
 
     def flows(self):
@@ -232,7 +234,7 @@ class TestNicSingleCorePerf(TestCase):
             # need add one more core for start testpmd
             core_list = [core_list[0]] + [str(int(i) + 1) for i in core_list]
 
-            self.logger.info("Executing Test Using cores: %s of config %s" % (core_list, fwd_config))
+            self.logger.info("Executing Test Using cores: %s of config %s" % (core_list[1:], fwd_config))
 
             nb_cores = thread_num
 
@@ -242,8 +244,6 @@ class TestNicSingleCorePerf(TestCase):
             # columbiaville use one queue per port for best performance.
             elif self.nic in ["columbiaville_100g", "columbiaville_25g", "columbiaville_25gx2"]:
                 param += " --rxq=1 --txq=1"
-                # workaround for that testpmd can't forward packets in io forward mode
-                param += " --port-topology=loop"
 
             self.throughput[fwd_config] = dict()
             for frame_size in list(self.test_parameters[fwd_config].keys()):
@@ -265,21 +265,42 @@ class TestNicSingleCorePerf(TestCase):
                     # run packet generator
                     streams = self.pktgen_helper.prepare_stream_from_tginput(tgenInput, 100, vm_config, self.tester.pktgen)
                     # set traffic option
-                    traffic_opt = {'duration': self.test_duration}
-                    # _, pps = self.tester.traffic_generator_throughput(tgenInput, rate_percent=100, delay=30)
-                    _, packets_received = self.tester.pktgen.measure_throughput(stream_ids=streams, options=traffic_opt)
-                    self.verify(packets_received > 0, "No traffic detected")
-                    throughput = packets_received / 1000000.0
-                    self.throughput[fwd_config][frame_size][nb_desc] = throughput
+                    traffic_opt = {
+                        'method': 'throughput',
+                        'rate': 100,
+                        'duration': self.test_duration,
+                        'interval': self.test_interval,
+                        }
+                    stats = self.tester.pktgen.measure(stream_ids=streams, traffic_opt=traffic_opt)
+
+                    #####################################################
+                    # Remove max and min, then get average
+                    #####################################################
+                    if isinstance(stats, list):
+                        total_pps_rxs = []
+                        for i in range(len(stats)):
+                            stats_pps = stats[i][1]
+                            if isinstance(stats_pps, tuple):
+                                total_pps_rxs.append(stats_pps[1])
+                            else:
+                                total_pps_rxs.append(stats_pps)
+
+                        total_pps_rxs.remove(max(total_pps_rxs))
+                        total_pps_rxs.remove(min(total_pps_rxs))
+                        total_pps_rx = mean(total_pps_rxs)
+                    else:
+                        total_pps_rx = stats
+
+                    self.verify(total_pps_rx > 0, "No traffic detected, please check your configuration")
+                    total_mpps_rx = total_pps_rx / 1000000.0
+                    self.throughput[fwd_config][frame_size][nb_desc] = total_mpps_rx
 
                     self.dut.send_expect("stop", "testpmd> ")
                     self.dut.send_expect("quit", "# ", 30)
 
-                    self.verify(throughput,
-                        "No traffic detected, please check your configuration")
                     self.logger.info("Trouthput of " +
                         "framesize: {}, rxd/txd: {} is :{} Mpps".format(
-                            frame_size, nb_desc, throughput))
+                            frame_size, nb_desc, total_mpps_rx))
 
         return self.throughput
 
@@ -303,16 +324,25 @@ class TestNicSingleCorePerf(TestCase):
                     ret_data[header[0]] = fwd_config
                     ret_data[header[1]] = frame_size
                     ret_data[header[2]] = nb_desc
-                    ret_data[header[3]] = "{:.3f} Mpps".format(
+                    ret_data[header[3]] = "{:.3f}".format(
                         self.throughput[fwd_config][frame_size][nb_desc])
                     ret_data[header[4]] = "{:.3f}%".format(
                         self.throughput[fwd_config][frame_size][nb_desc] * 100 / wirespeed)
-                    ret_data[header[5]] = "{:.3f} Mpps".format(
-                        self.expected_throughput[fwd_config][frame_size][nb_desc])
-                    ret_data[header[6]] = "{:.3f} Mpps".format(
-                        self.throughput[fwd_config][frame_size][nb_desc] -
+                    if 0 == self.expected_throughput[fwd_config][frame_size][nb_desc]:
+                        self.expected_throughput[fwd_config][frame_size][nb_desc] = 1
+                        self.logger.warning("Expected throughput can't be zero !!")
+                        msg = "Force it to 1 Mpps for frame size: {0}, RXD/TXD: {1}, Core: {2}"
+                        self.logger.warning(msg.format(frame_size, nb_desc, fwd_config))
+                    ret_data[header[5]] = "{:.3f}".format(
                             self.expected_throughput[fwd_config][frame_size][nb_desc])
-
+                    diff = (self.throughput[fwd_config][frame_size][nb_desc] -
+                            self.expected_throughput[fwd_config][frame_size][nb_desc])/ \
+                                    self.expected_throughput[fwd_config][frame_size][nb_desc]
+                    ret_data[header[6]] = "{:.3f}%".format(diff)
+                    if diff >= -self.gap and diff <= self.gap:
+                        ret_data[header[7]] = 'PASS'
+                    else:
+                        ret_data[header[7]] = 'FAIL'
                     ret_datas[frame_size][nb_desc] = deepcopy(ret_data)
                 self.test_result[fwd_config] = deepcopy(ret_datas)
 
@@ -350,15 +380,11 @@ class TestNicSingleCorePerf(TestCase):
                     row_dict0 = dict()
                     row_dict0['performance'] = list()
                     row_dict0['parameters'] = list()
-                    result_throughput = float(row_in['Throughput'].split()[0])
-                    expected_throughput = float(row_in['Expected Throughput'].split()[0])
-                    # delta value and accepted tolerance in percentage
-                    delta = result_throughput - expected_throughput
-                    if delta > -self.gap:
-                        row_dict0['status'] = 'PASS'
-                    else:
-                        row_dict0['status'] = 'FAIL'
-                    row_dict1 = dict(name="Throughput", value=result_throughput, unit="Mpps", delta=delta)
+                    result_throughput = float(row_in['Real-Mpps'].split()[0])
+                    expected_throughput = float(row_in['Expected-Mpps'].split()[0])
+                    row_dict0['status'] = row_in['Status']
+                    row_dict1 = dict(name="Throughput", value=result_throughput, unit="Mpps",
+                            delta=row_in['Real vs Expected'], expected=expected_throughput)
                     row_dict2 = dict(name="Txd/Rxd", value=row_in["TXD/RXD"], unit="descriptor")
                     row_dict3 = dict(name="frame_size", value=row_in["Frame Size"], unit="bytes")
                     row_dict4 = dict(name="Fwd_core", value=row_in["Fwd_core"])
@@ -368,10 +394,10 @@ class TestNicSingleCorePerf(TestCase):
                     row_dict0['parameters'].append(row_dict4)
                     json_obj[case_name].append(row_dict0)
                     status_result.append(row_dict0['status'])
-        with open(os.path.join(rst.path2Result,
-                               '{0:s}_single_core_perf.json'.format(
-                                   self.nic)), 'w') as fp:
-            json.dump(json_obj, fp)
+
+        json_file = os.path.join(rst.path2Result, '{0:s}_single_core_perf.json'.format(self.nic))
+        with open(json_file, 'w') as fp:
+            json.dump(json_obj, fp, indent=4, separators=(',', ': '), sort_keys=True)
         self.verify("FAIL" not in status_result, "Excessive gap between test results and expectations")
 
     def set_fields(self):
-- 
2.25.1


^ permalink raw reply	[flat|nested] 9+ messages in thread

* Re: [dts] [PATCH] tests/nic_single_core_perf: modify accepted_tolerance to percentage
  2021-03-11 10:48 [dts] [PATCH] tests/nic_single_core_perf: modify accepted_tolerance to percentage Lijuan Tu
@ 2021-03-22 18:13 ` Ali Alnubani
  2021-03-23  8:43 ` Ali Alnubani
  1 sibling, 0 replies; 9+ messages in thread
From: Ali Alnubani @ 2021-03-22 18:13 UTC (permalink / raw)
  To: Lijuan Tu; +Cc: dts, dpdklab

> -----Original Message-----
> From: dts <dts-bounces@dpdk.org> On Behalf Of Lijuan Tu
> Sent: Thursday, March 11, 2021 12:48 PM
> To: dpdklab@iol.unh.edu
> Cc: dts@dpdk.org; Lijuan Tu <lijuan.tu@intel.com>
> Subject: [dts] [PATCH] tests/nic_single_core_perf: modify
> accepted_tolerance to percentage
> 
> * accepted_tolerance in conf/nic_single_core_perf.cfg are changed to
> percentage, e.g.
> 	accepted_tolerance = '1%%'
> 	if not set, it will be set to 2% by default.
> * delat in json are chagned to percentage too, e.g.
> 	'delta': '1.5%'
> * expected_throughput will be forced to 1Mpps for compute delat in
> percentage
> * add new conf test_interval in conf/nic_single_core_perf.cfg, which
> indicates interval of getting throughput statistics in second, e.g.
> 	test_interval = 10
> 	if not set, it will be set to 5 seconds by default.
> * collect better statistics , get multiple throughput statistics during
> the test run, and compute the average as the final results.
> 
> Signed-off-by: Lijuan Tu <lijuan.tu@intel.com>
> ---

Thanks for the patch Lijuan.
I'm testing it locally and I'll update soon.

Thanks,
Ali

^ permalink raw reply	[flat|nested] 9+ messages in thread

* Re: [dts] [PATCH] tests/nic_single_core_perf: modify accepted_tolerance to percentage
  2021-03-11 10:48 [dts] [PATCH] tests/nic_single_core_perf: modify accepted_tolerance to percentage Lijuan Tu
  2021-03-22 18:13 ` Ali Alnubani
@ 2021-03-23  8:43 ` Ali Alnubani
  2021-03-23  8:48   ` Tu, Lijuan
  1 sibling, 1 reply; 9+ messages in thread
From: Ali Alnubani @ 2021-03-23  8:43 UTC (permalink / raw)
  To: Lijuan Tu; +Cc: dts, dpdklab

> -----Original Message-----
> From: dts <dts-bounces@dpdk.org> On Behalf Of Lijuan Tu
> Sent: Thursday, March 11, 2021 12:48 PM
> To: dpdklab@iol.unh.edu
> Cc: dts@dpdk.org; Lijuan Tu <lijuan.tu@intel.com>
> Subject: [dts] [PATCH] tests/nic_single_core_perf: modify
> accepted_tolerance to percentage
> 
> * accepted_tolerance in conf/nic_single_core_perf.cfg are changed to
> percentage, e.g.
> 	accepted_tolerance = '1%%'
> 	if not set, it will be set to 2% by default.
> * delat in json are chagned to percentage too, e.g.
> 	'delta': '1.5%'
> * expected_throughput will be forced to 1Mpps for compute delat in
> percentage
> * add new conf test_interval in conf/nic_single_core_perf.cfg, which
> indicates interval of getting throughput statistics in second, e.g.
> 	test_interval = 10
> 	if not set, it will be set to 5 seconds by default.
> * collect better statistics , get multiple throughput statistics during
> the test run, and compute the average as the final results.
> 
> Signed-off-by: Lijuan Tu <lijuan.tu@intel.com>
> ---
<snipped>
> +accepted_tolerance = '1%%'
<snipped>
> +                    diff = (self.throughput[fwd_config][frame_size][nb_desc] -
> +
> self.expected_throughput[fwd_config][frame_size][nb_desc])/ \
> +
> self.expected_throughput[fwd_config][frame_size][nb_desc]

The value of `diff` here is still a decimal, and needs to be multiplied by a 100 before it can be compared to `accepted_tolerance`.

I suggest splitting this patch into multiple ones. One for making the percentage change and one for taking the mean for multiple readings since they are unrelated changes.
I also see other unrelated changes not explained in the commit message.

Thanks,
Ali

^ permalink raw reply	[flat|nested] 9+ messages in thread

* Re: [dts] [PATCH] tests/nic_single_core_perf: modify accepted_tolerance to percentage
  2021-03-23  8:43 ` Ali Alnubani
@ 2021-03-23  8:48   ` Tu, Lijuan
  2021-03-23 14:52     ` David Liu
  0 siblings, 1 reply; 9+ messages in thread
From: Tu, Lijuan @ 2021-03-23  8:48 UTC (permalink / raw)
  To: Ali Alnubani; +Cc: dts, dpdklab



> -----Original Message-----
> From: Ali Alnubani <alialnu@nvidia.com>
> Sent: 2021年3月23日 16:44
> To: Tu, Lijuan <lijuan.tu@intel.com>
> Cc: dts@dpdk.org; dpdklab@iol.unh.edu
> Subject: RE: [dts] [PATCH] tests/nic_single_core_perf: modify
> accepted_tolerance to percentage
> 
> > -----Original Message-----
> > From: dts <dts-bounces@dpdk.org> On Behalf Of Lijuan Tu
> > Sent: Thursday, March 11, 2021 12:48 PM
> > To: dpdklab@iol.unh.edu
> > Cc: dts@dpdk.org; Lijuan Tu <lijuan.tu@intel.com>
> > Subject: [dts] [PATCH] tests/nic_single_core_perf: modify
> > accepted_tolerance to percentage
> >
> > * accepted_tolerance in conf/nic_single_core_perf.cfg are changed to
> > percentage, e.g.
> > 	accepted_tolerance = '1%%'
> > 	if not set, it will be set to 2% by default.
> > * delat in json are chagned to percentage too, e.g.
> > 	'delta': '1.5%'
> > * expected_throughput will be forced to 1Mpps for compute delat in
> > percentage
> > * add new conf test_interval in conf/nic_single_core_perf.cfg, which
> > indicates interval of getting throughput statistics in second, e.g.
> > 	test_interval = 10
> > 	if not set, it will be set to 5 seconds by default.
> > * collect better statistics , get multiple throughput statistics
> > during the test run, and compute the average as the final results.
> >
> > Signed-off-by: Lijuan Tu <lijuan.tu@intel.com>
> > ---
> <snipped>
> > +accepted_tolerance = '1%%'
> <snipped>
> > +                    diff =
> > + (self.throughput[fwd_config][frame_size][nb_desc] -
> > +
> > self.expected_throughput[fwd_config][frame_size][nb_desc])/ \
> > +
> > self.expected_throughput[fwd_config][frame_size][nb_desc]
> 
> The value of `diff` here is still a decimal, and needs to be multiplied by a 100
> before it can be compared to `accepted_tolerance`.
> 
> I suggest splitting this patch into multiple ones. One for making the percentage
> change and one for taking the mean for multiple readings since they are
> unrelated changes.
> I also see other unrelated changes not explained in the commit message.

Really appreciate for your review, I will accept your advices.

> 
> Thanks,
> Ali

^ permalink raw reply	[flat|nested] 9+ messages in thread

* Re: [dts] [PATCH] tests/nic_single_core_perf: modify accepted_tolerance to percentage
  2021-03-23  8:48   ` Tu, Lijuan
@ 2021-03-23 14:52     ` David Liu
  2021-03-23 17:23       ` [dts] [dpdklab] " Owen Hilyard
  0 siblings, 1 reply; 9+ messages in thread
From: David Liu @ 2021-03-23 14:52 UTC (permalink / raw)
  To: Tu, Lijuan; +Cc: Ali Alnubani, dts, dpdklab

[-- Attachment #1: Type: text/plain, Size: 2552 bytes --]

Thank you for Ali and Lijuan.

We had tested the patch on the Arm machines setup. Other than the value
issues, the test is running as expected.

On Tue, Mar 23, 2021 at 4:48 AM Tu, Lijuan <lijuan.tu@intel.com> wrote:

>
>
> > -----Original Message-----
> > From: Ali Alnubani <alialnu@nvidia.com>
> > Sent: 2021年3月23日 16:44
> > To: Tu, Lijuan <lijuan.tu@intel.com>
> > Cc: dts@dpdk.org; dpdklab@iol.unh.edu
> > Subject: RE: [dts] [PATCH] tests/nic_single_core_perf: modify
> > accepted_tolerance to percentage
> >
> > > -----Original Message-----
> > > From: dts <dts-bounces@dpdk.org> On Behalf Of Lijuan Tu
> > > Sent: Thursday, March 11, 2021 12:48 PM
> > > To: dpdklab@iol.unh.edu
> > > Cc: dts@dpdk.org; Lijuan Tu <lijuan.tu@intel.com>
> > > Subject: [dts] [PATCH] tests/nic_single_core_perf: modify
> > > accepted_tolerance to percentage
> > >
> > > * accepted_tolerance in conf/nic_single_core_perf.cfg are changed to
> > > percentage, e.g.
> > >     accepted_tolerance = '1%%'
> > >     if not set, it will be set to 2% by default.
> > > * delat in json are chagned to percentage too, e.g.
> > >     'delta': '1.5%'
> > > * expected_throughput will be forced to 1Mpps for compute delat in
> > > percentage
> > > * add new conf test_interval in conf/nic_single_core_perf.cfg, which
> > > indicates interval of getting throughput statistics in second, e.g.
> > >     test_interval = 10
> > >     if not set, it will be set to 5 seconds by default.
> > > * collect better statistics , get multiple throughput statistics
> > > during the test run, and compute the average as the final results.
> > >
> > > Signed-off-by: Lijuan Tu <lijuan.tu@intel.com>
> > > ---
> > <snipped>
> > > +accepted_tolerance = '1%%'
> > <snipped>
> > > +                    diff =
> > > + (self.throughput[fwd_config][frame_size][nb_desc] -
> > > +
> > > self.expected_throughput[fwd_config][frame_size][nb_desc])/ \
> > > +
> > > self.expected_throughput[fwd_config][frame_size][nb_desc]
> >
> > The value of `diff` here is still a decimal, and needs to be multiplied
> by a 100
> > before it can be compared to `accepted_tolerance`.
> >
> > I suggest splitting this patch into multiple ones. One for making the
> percentage
> > change and one for taking the mean for multiple readings since they are
> > unrelated changes.
> > I also see other unrelated changes not explained in the commit message.
>
> Really appreciate for your review, I will accept your advices.
>
> >
> > Thanks,
> > Ali
>

[-- Attachment #2: Type: text/html, Size: 3803 bytes --]

^ permalink raw reply	[flat|nested] 9+ messages in thread

* Re: [dts] [dpdklab] Re: [PATCH] tests/nic_single_core_perf: modify accepted_tolerance to percentage
  2021-03-23 14:52     ` David Liu
@ 2021-03-23 17:23       ` Owen Hilyard
  2021-03-25  7:28         ` Tu, Lijuan
  0 siblings, 1 reply; 9+ messages in thread
From: Owen Hilyard @ 2021-03-23 17:23 UTC (permalink / raw)
  To: Tu, Lijuan; +Cc: David Liu, Ali Alnubani, dts, dpdklab

[-- Attachment #1: Type: text/plain, Size: 3187 bytes --]

Hello Lijuan,

Would it be possible to have the delta be a float in the json
representation? For example, a 20% delta would be encoded as "delta": .2 in
json. Currently, most of the community lab's infrastructure expects to make
pass/fail determinations on a numeric type, so having the delta as a float
would reduce the required downtime to change everything over to percentages
since we shouldn't need to patch all of our services to expect a string.

Thanks,
Owen Hilyard

On Tue, Mar 23, 2021 at 10:53 AM David Liu <dliu@iol.unh.edu> wrote:

> Thank you for Ali and Lijuan.
>
> We had tested the patch on the Arm machines setup. Other than the value
> issues, the test is running as expected.
>
> On Tue, Mar 23, 2021 at 4:48 AM Tu, Lijuan <lijuan.tu@intel.com> wrote:
>
>>
>>
>> > -----Original Message-----
>> > From: Ali Alnubani <alialnu@nvidia.com>
>> > Sent: 2021年3月23日 16:44
>> > To: Tu, Lijuan <lijuan.tu@intel.com>
>> > Cc: dts@dpdk.org; dpdklab@iol.unh.edu
>> > Subject: RE: [dts] [PATCH] tests/nic_single_core_perf: modify
>> > accepted_tolerance to percentage
>> >
>> > > -----Original Message-----
>> > > From: dts <dts-bounces@dpdk.org> On Behalf Of Lijuan Tu
>> > > Sent: Thursday, March 11, 2021 12:48 PM
>> > > To: dpdklab@iol.unh.edu
>> > > Cc: dts@dpdk.org; Lijuan Tu <lijuan.tu@intel.com>
>> > > Subject: [dts] [PATCH] tests/nic_single_core_perf: modify
>> > > accepted_tolerance to percentage
>> > >
>> > > * accepted_tolerance in conf/nic_single_core_perf.cfg are changed to
>> > > percentage, e.g.
>> > >     accepted_tolerance = '1%%'
>> > >     if not set, it will be set to 2% by default.
>> > > * delat in json are chagned to percentage too, e.g.
>> > >     'delta': '1.5%'
>> > > * expected_throughput will be forced to 1Mpps for compute delat in
>> > > percentage
>> > > * add new conf test_interval in conf/nic_single_core_perf.cfg, which
>> > > indicates interval of getting throughput statistics in second, e.g.
>> > >     test_interval = 10
>> > >     if not set, it will be set to 5 seconds by default.
>> > > * collect better statistics , get multiple throughput statistics
>> > > during the test run, and compute the average as the final results.
>> > >
>> > > Signed-off-by: Lijuan Tu <lijuan.tu@intel.com>
>> > > ---
>> > <snipped>
>> > > +accepted_tolerance = '1%%'
>> > <snipped>
>> > > +                    diff =
>> > > + (self.throughput[fwd_config][frame_size][nb_desc] -
>> > > +
>> > > self.expected_throughput[fwd_config][frame_size][nb_desc])/ \
>> > > +
>> > > self.expected_throughput[fwd_config][frame_size][nb_desc]
>> >
>> > The value of `diff` here is still a decimal, and needs to be multiplied
>> by a 100
>> > before it can be compared to `accepted_tolerance`.
>> >
>> > I suggest splitting this patch into multiple ones. One for making the
>> percentage
>> > change and one for taking the mean for multiple readings since they are
>> > unrelated changes.
>> > I also see other unrelated changes not explained in the commit message.
>>
>> Really appreciate for your review, I will accept your advices.
>>
>> >
>> > Thanks,
>> > Ali
>>
>

[-- Attachment #2: Type: text/html, Size: 4725 bytes --]

^ permalink raw reply	[flat|nested] 9+ messages in thread

* Re: [dts] [dpdklab] Re: [PATCH] tests/nic_single_core_perf: modify accepted_tolerance to percentage
  2021-03-23 17:23       ` [dts] [dpdklab] " Owen Hilyard
@ 2021-03-25  7:28         ` Tu, Lijuan
  2021-03-26  3:34           ` Tu, Lijuan
  0 siblings, 1 reply; 9+ messages in thread
From: Tu, Lijuan @ 2021-03-25  7:28 UTC (permalink / raw)
  To: Owen Hilyard; +Cc: David Liu, Ali Alnubani, dts, dpdklab

[-- Attachment #1: Type: text/plain, Size: 3638 bytes --]

sure, I prefer to use a float too.

From: Owen Hilyard <ohilyard@iol.unh.edu>
Sent: 2021年3月24日 1:24
To: Tu, Lijuan <lijuan.tu@intel.com>
Cc: David Liu <dliu@iol.unh.edu>; Ali Alnubani <alialnu@nvidia.com>; dts@dpdk.org; dpdklab@iol.unh.edu
Subject: Re: [dpdklab] Re: [dts] [PATCH] tests/nic_single_core_perf: modify accepted_tolerance to percentage

Hello Lijuan,

Would it be possible to have the delta be a float in the json representation? For example, a 20% delta would be encoded as "delta": .2 in json. Currently, most of the community lab's infrastructure expects to make pass/fail determinations on a numeric type, so having the delta as a float would reduce the required downtime to change everything over to percentages since we shouldn't need to patch all of our services to expect a string.


Thanks,
Owen Hilyard

On Tue, Mar 23, 2021 at 10:53 AM David Liu <dliu@iol.unh.edu<mailto:dliu@iol.unh.edu>> wrote:
Thank you for Ali and Lijuan.

We had tested the patch on the Arm machines setup. Other than the value issues, the test is running as expected.

On Tue, Mar 23, 2021 at 4:48 AM Tu, Lijuan <lijuan.tu@intel.com<mailto:lijuan.tu@intel.com>> wrote:


> -----Original Message-----
> From: Ali Alnubani <alialnu@nvidia.com<mailto:alialnu@nvidia.com>>
> Sent: 2021年3月23日 16:44
> To: Tu, Lijuan <lijuan.tu@intel.com<mailto:lijuan.tu@intel.com>>
> Cc: dts@dpdk.org<mailto:dts@dpdk.org>; dpdklab@iol.unh.edu<mailto:dpdklab@iol.unh.edu>
> Subject: RE: [dts] [PATCH] tests/nic_single_core_perf: modify
> accepted_tolerance to percentage
>
> > -----Original Message-----
> > From: dts <dts-bounces@dpdk.org<mailto:dts-bounces@dpdk.org>> On Behalf Of Lijuan Tu
> > Sent: Thursday, March 11, 2021 12:48 PM
> > To: dpdklab@iol.unh.edu<mailto:dpdklab@iol.unh.edu>
> > Cc: dts@dpdk.org<mailto:dts@dpdk.org>; Lijuan Tu <lijuan.tu@intel.com<mailto:lijuan.tu@intel.com>>
> > Subject: [dts] [PATCH] tests/nic_single_core_perf: modify
> > accepted_tolerance to percentage
> >
> > * accepted_tolerance in conf/nic_single_core_perf.cfg are changed to
> > percentage, e.g.
> >     accepted_tolerance = '1%%'
> >     if not set, it will be set to 2% by default.
> > * delat in json are chagned to percentage too, e.g.
> >     'delta': '1.5%'
> > * expected_throughput will be forced to 1Mpps for compute delat in
> > percentage
> > * add new conf test_interval in conf/nic_single_core_perf.cfg, which
> > indicates interval of getting throughput statistics in second, e.g.
> >     test_interval = 10
> >     if not set, it will be set to 5 seconds by default.
> > * collect better statistics , get multiple throughput statistics
> > during the test run, and compute the average as the final results.
> >
> > Signed-off-by: Lijuan Tu <lijuan.tu@intel.com<mailto:lijuan.tu@intel.com>>
> > ---
> <snipped>
> > +accepted_tolerance = '1%%'
> <snipped>
> > +                    diff =
> > + (self.throughput[fwd_config][frame_size][nb_desc] -
> > +
> > self.expected_throughput[fwd_config][frame_size][nb_desc])/ \
> > +
> > self.expected_throughput[fwd_config][frame_size][nb_desc]
>
> The value of `diff` here is still a decimal, and needs to be multiplied by a 100
> before it can be compared to `accepted_tolerance`.
>
> I suggest splitting this patch into multiple ones. One for making the percentage
> change and one for taking the mean for multiple readings since they are
> unrelated changes.
> I also see other unrelated changes not explained in the commit message.

Really appreciate for your review, I will accept your advices.

>
> Thanks,
> Ali

[-- Attachment #2: Type: text/html, Size: 8241 bytes --]

^ permalink raw reply	[flat|nested] 9+ messages in thread

* Re: [dts] [dpdklab] Re: [PATCH] tests/nic_single_core_perf: modify accepted_tolerance to percentage
  2021-03-25  7:28         ` Tu, Lijuan
@ 2021-03-26  3:34           ` Tu, Lijuan
  2021-03-30 14:27             ` David Liu
  0 siblings, 1 reply; 9+ messages in thread
From: Tu, Lijuan @ 2021-03-26  3:34 UTC (permalink / raw)
  To: Tu, Lijuan, Owen Hilyard; +Cc: David Liu, Ali Alnubani, dts, dpdklab

[-- Attachment #1: Type: text/plain, Size: 4390 bytes --]

Hi Owen,

I ‘d like to add 2 more changes in V2:

1,  test_interval  will be renamed to throughput_stat_sample_interval for better understanding.
2,  accepted_tolerance will be a float to algin with delta in json, both format are {:.2f}


From: dts <dts-bounces@dpdk.org> On Behalf Of Tu, Lijuan
Sent: 2021年3月25日 15:29
To: Owen Hilyard <ohilyard@iol.unh.edu>
Cc: David Liu <dliu@iol.unh.edu>; Ali Alnubani <alialnu@nvidia.com>; dts@dpdk.org; dpdklab@iol.unh.edu
Subject: Re: [dts] [dpdklab] Re: [PATCH] tests/nic_single_core_perf: modify accepted_tolerance to percentage

sure, I prefer to use a float too.

From: Owen Hilyard <ohilyard@iol.unh.edu<mailto:ohilyard@iol.unh.edu>>
Sent: 2021年3月24日 1:24
To: Tu, Lijuan <lijuan.tu@intel.com<mailto:lijuan.tu@intel.com>>
Cc: David Liu <dliu@iol.unh.edu<mailto:dliu@iol.unh.edu>>; Ali Alnubani <alialnu@nvidia.com<mailto:alialnu@nvidia.com>>; dts@dpdk.org<mailto:dts@dpdk.org>; dpdklab@iol.unh.edu<mailto:dpdklab@iol.unh.edu>
Subject: Re: [dpdklab] Re: [dts] [PATCH] tests/nic_single_core_perf: modify accepted_tolerance to percentage

Hello Lijuan,

Would it be possible to have the delta be a float in the json representation? For example, a 20% delta would be encoded as "delta": .2 in json. Currently, most of the community lab's infrastructure expects to make pass/fail determinations on a numeric type, so having the delta as a float would reduce the required downtime to change everything over to percentages since we shouldn't need to patch all of our services to expect a string.


Thanks,
Owen Hilyard

On Tue, Mar 23, 2021 at 10:53 AM David Liu <dliu@iol.unh.edu<mailto:dliu@iol.unh.edu>> wrote:
Thank you for Ali and Lijuan.

We had tested the patch on the Arm machines setup. Other than the value issues, the test is running as expected.

On Tue, Mar 23, 2021 at 4:48 AM Tu, Lijuan <lijuan.tu@intel.com<mailto:lijuan.tu@intel.com>> wrote:


> -----Original Message-----
> From: Ali Alnubani <alialnu@nvidia.com<mailto:alialnu@nvidia.com>>
> Sent: 2021年3月23日 16:44
> To: Tu, Lijuan <lijuan.tu@intel.com<mailto:lijuan.tu@intel.com>>
> Cc: dts@dpdk.org<mailto:dts@dpdk.org>; dpdklab@iol.unh.edu<mailto:dpdklab@iol.unh.edu>
> Subject: RE: [dts] [PATCH] tests/nic_single_core_perf: modify
> accepted_tolerance to percentage
>
> > -----Original Message-----
> > From: dts <dts-bounces@dpdk.org<mailto:dts-bounces@dpdk.org>> On Behalf Of Lijuan Tu
> > Sent: Thursday, March 11, 2021 12:48 PM
> > To: dpdklab@iol.unh.edu<mailto:dpdklab@iol.unh.edu>
> > Cc: dts@dpdk.org<mailto:dts@dpdk.org>; Lijuan Tu <lijuan.tu@intel.com<mailto:lijuan.tu@intel.com>>
> > Subject: [dts] [PATCH] tests/nic_single_core_perf: modify
> > accepted_tolerance to percentage
> >
> > * accepted_tolerance in conf/nic_single_core_perf.cfg are changed to
> > percentage, e.g.
> >     accepted_tolerance = '1%%'
> >     if not set, it will be set to 2% by default.
> > * delat in json are chagned to percentage too, e.g.
> >     'delta': '1.5%'
> > * expected_throughput will be forced to 1Mpps for compute delat in
> > percentage
> > * add new conf test_interval in conf/nic_single_core_perf.cfg, which
> > indicates interval of getting throughput statistics in second, e.g.
> >     test_interval = 10
> >     if not set, it will be set to 5 seconds by default.
> > * collect better statistics , get multiple throughput statistics
> > during the test run, and compute the average as the final results.
> >
> > Signed-off-by: Lijuan Tu <lijuan.tu@intel.com<mailto:lijuan.tu@intel.com>>
> > ---
> <snipped>
> > +accepted_tolerance = '1%%'
> <snipped>
> > +                    diff =
> > + (self.throughput[fwd_config][frame_size][nb_desc] -
> > +
> > self.expected_throughput[fwd_config][frame_size][nb_desc])/ \
> > +
> > self.expected_throughput[fwd_config][frame_size][nb_desc]
>
> The value of `diff` here is still a decimal, and needs to be multiplied by a 100
> before it can be compared to `accepted_tolerance`.
>
> I suggest splitting this patch into multiple ones. One for making the percentage
> change and one for taking the mean for multiple readings since they are
> unrelated changes.
> I also see other unrelated changes not explained in the commit message.

Really appreciate for your review, I will accept your advices.

>
> Thanks,
> Ali

[-- Attachment #2: Type: text/html, Size: 10272 bytes --]

^ permalink raw reply	[flat|nested] 9+ messages in thread

* Re: [dts] [dpdklab] Re: [PATCH] tests/nic_single_core_perf: modify accepted_tolerance to percentage
  2021-03-26  3:34           ` Tu, Lijuan
@ 2021-03-30 14:27             ` David Liu
  0 siblings, 0 replies; 9+ messages in thread
From: David Liu @ 2021-03-30 14:27 UTC (permalink / raw)
  To: Tu, Lijuan; +Cc: Owen Hilyard, Ali Alnubani, dts, dpdklab

[-- Attachment #1: Type: text/plain, Size: 4488 bytes --]

Hi Lijuan,

Just a quick update.

We had tested the patch on all our bare-metal machines.

And they all work as expected.

Best,
David



On Thu, Mar 25, 2021 at 11:34 PM Tu, Lijuan <lijuan.tu@intel.com> wrote:

> Hi Owen,
>
>
>
> I ‘d like to add 2 more changes in V2:
>
>
>
> 1,  test_interval  will be renamed to throughput_stat_sample_interval for
> better understanding.
>
> 2,  accepted_tolerance will be a float to algin with delta in json, both
> format are {:.2f}
>
>
>
>
>
> *From:* dts <dts-bounces@dpdk.org> *On Behalf Of * Tu, Lijuan
> *Sent:* 2021年3月25日 15:29
> *To:* Owen Hilyard <ohilyard@iol.unh.edu>
> *Cc:* David Liu <dliu@iol.unh.edu>; Ali Alnubani <alialnu@nvidia.com>;
> dts@dpdk.org; dpdklab@iol.unh.edu
> *Subject:* Re: [dts] [dpdklab] Re: [PATCH] tests/nic_single_core_perf:
> modify accepted_tolerance to percentage
>
>
>
> sure, I prefer to use a float too.
>
>
>
> *From:* Owen Hilyard <ohilyard@iol.unh.edu>
> *Sent:* 2021年3月24日 1:24
> *To:* Tu, Lijuan <lijuan.tu@intel.com>
> *Cc:* David Liu <dliu@iol.unh.edu>; Ali Alnubani <alialnu@nvidia.com>;
> dts@dpdk.org; dpdklab@iol.unh.edu
> *Subject:* Re: [dpdklab] Re: [dts] [PATCH] tests/nic_single_core_perf:
> modify accepted_tolerance to percentage
>
>
>
> Hello Lijuan,
>
>
>
> Would it be possible to have the delta be a float in the json
> representation? For example, a 20% delta would be encoded as "delta": .2 in
> json. Currently, most of the community lab's infrastructure expects to make
> pass/fail determinations on a numeric type, so having the delta as a float
> would reduce the required downtime to change everything over to percentages
> since we shouldn't need to patch all of our services to expect a string.
>
>
>
>
>
> Thanks,
>
> Owen Hilyard
>
>
>
> On Tue, Mar 23, 2021 at 10:53 AM David Liu <dliu@iol.unh.edu> wrote:
>
> Thank you for Ali and Lijuan.
>
> We had tested the patch on the Arm machines setup. Other than the value
> issues, the test is running as expected.
>
>
>
> On Tue, Mar 23, 2021 at 4:48 AM Tu, Lijuan <lijuan.tu@intel.com> wrote:
>
>
>
> > -----Original Message-----
> > From: Ali Alnubani <alialnu@nvidia.com>
> > Sent: 2021年3月23日 16:44
> > To: Tu, Lijuan <lijuan.tu@intel.com>
> > Cc: dts@dpdk.org; dpdklab@iol.unh.edu
> > Subject: RE: [dts] [PATCH] tests/nic_single_core_perf: modify
> > accepted_tolerance to percentage
> >
> > > -----Original Message-----
> > > From: dts <dts-bounces@dpdk.org> On Behalf Of Lijuan Tu
> > > Sent: Thursday, March 11, 2021 12:48 PM
> > > To: dpdklab@iol.unh.edu
> > > Cc: dts@dpdk.org; Lijuan Tu <lijuan.tu@intel.com>
> > > Subject: [dts] [PATCH] tests/nic_single_core_perf: modify
> > > accepted_tolerance to percentage
> > >
> > > * accepted_tolerance in conf/nic_single_core_perf.cfg are changed to
> > > percentage, e.g.
> > >     accepted_tolerance = '1%%'
> > >     if not set, it will be set to 2% by default.
> > > * delat in json are chagned to percentage too, e.g.
> > >     'delta': '1.5%'
> > > * expected_throughput will be forced to 1Mpps for compute delat in
> > > percentage
> > > * add new conf test_interval in conf/nic_single_core_perf.cfg, which
> > > indicates interval of getting throughput statistics in second, e.g.
> > >     test_interval = 10
> > >     if not set, it will be set to 5 seconds by default.
> > > * collect better statistics , get multiple throughput statistics
> > > during the test run, and compute the average as the final results.
> > >
> > > Signed-off-by: Lijuan Tu <lijuan.tu@intel.com>
> > > ---
> > <snipped>
> > > +accepted_tolerance = '1%%'
> > <snipped>
> > > +                    diff =
> > > + (self.throughput[fwd_config][frame_size][nb_desc] -
> > > +
> > > self.expected_throughput[fwd_config][frame_size][nb_desc])/ \
> > > +
> > > self.expected_throughput[fwd_config][frame_size][nb_desc]
> >
> > The value of `diff` here is still a decimal, and needs to be multiplied
> by a 100
> > before it can be compared to `accepted_tolerance`.
> >
> > I suggest splitting this patch into multiple ones. One for making the
> percentage
> > change and one for taking the mean for multiple readings since they are
> > unrelated changes.
> > I also see other unrelated changes not explained in the commit message.
>
> Really appreciate for your review, I will accept your advices.
>
> >
> > Thanks,
> > Ali
>
>

[-- Attachment #2: Type: text/html, Size: 9641 bytes --]

^ permalink raw reply	[flat|nested] 9+ messages in thread

end of thread, other threads:[~2021-03-30 14:27 UTC | newest]

Thread overview: 9+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-03-11 10:48 [dts] [PATCH] tests/nic_single_core_perf: modify accepted_tolerance to percentage Lijuan Tu
2021-03-22 18:13 ` Ali Alnubani
2021-03-23  8:43 ` Ali Alnubani
2021-03-23  8:48   ` Tu, Lijuan
2021-03-23 14:52     ` David Liu
2021-03-23 17:23       ` [dts] [dpdklab] " Owen Hilyard
2021-03-25  7:28         ` Tu, Lijuan
2021-03-26  3:34           ` Tu, Lijuan
2021-03-30 14:27             ` David Liu

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).