From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 5EDA8A034F; Tue, 30 Mar 2021 04:12:16 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 2FE9D140F25; Tue, 30 Mar 2021 04:12:16 +0200 (CEST) Received: from mga12.intel.com (mga12.intel.com [192.55.52.136]) by mails.dpdk.org (Postfix) with ESMTP id 329F0406B4 for ; Tue, 30 Mar 2021 04:12:15 +0200 (CEST) IronPort-SDR: 5DMGoKGNcC1ESu9rm6MtTFYD2HAUYyUptPZihFDI1nAzcxYDptIowcfEADngjZm+QrrZTMYnBO XE4DFt+CdtBw== X-IronPort-AV: E=McAfee;i="6000,8403,9938"; a="171070315" X-IronPort-AV: E=Sophos;i="5.81,289,1610438400"; d="scan'208";a="171070315" Received: from fmsmga008.fm.intel.com ([10.253.24.58]) by fmsmga106.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 29 Mar 2021 19:12:14 -0700 IronPort-SDR: B98sraQOygcLcQ+fMWrWS+JTSnLfVYwENboLMNKBoiXwbuigOqYE60gEof3xEmGWbWBuQ70CEH ftm6jPz0TtDQ== X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.81,289,1610438400"; d="scan'208";a="411377937" Received: from fmsmsx605.amr.corp.intel.com ([10.18.126.85]) by fmsmga008.fm.intel.com with ESMTP; 29 Mar 2021 19:12:14 -0700 Received: from shsmsx604.ccr.corp.intel.com (10.109.6.214) by fmsmsx605.amr.corp.intel.com (10.18.126.85) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256) id 15.1.2106.2; Mon, 29 Mar 2021 19:12:13 -0700 Received: from shsmsx601.ccr.corp.intel.com (10.109.6.141) by SHSMSX604.ccr.corp.intel.com (10.109.6.214) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256) id 15.1.2106.2; Tue, 30 Mar 2021 10:12:11 +0800 Received: from shsmsx601.ccr.corp.intel.com ([10.109.6.141]) by SHSMSX601.ccr.corp.intel.com ([10.109.6.141]) with mapi id 15.01.2106.013; Tue, 30 Mar 2021 10:12:11 +0800 From: "Tu, Lijuan" To: "Mo, YufengX" , "dts@dpdk.org" Thread-Topic: [dts][PATCH V3 13/13] tests/perf_test_base: script Thread-Index: AQHXIVWMeEkLyvctJkO8pv5Vw1uw96qb0Oqw Date: Tue, 30 Mar 2021 02:12:11 +0000 Message-ID: <518d7165d63441c692145cc5e7b5fcf6@intel.com> References: <20210325085449.18195-1-yufengx.mo@intel.com> <20210325085449.18195-14-yufengx.mo@intel.com> In-Reply-To: <20210325085449.18195-14-yufengx.mo@intel.com> Accept-Language: zh-CN, en-US Content-Language: en-US X-MS-Has-Attach: X-MS-TNEF-Correlator: dlp-reaction: no-action dlp-version: 11.5.1.3 dlp-product: dlpe-windows x-originating-ip: [10.239.127.36] Content-Type: text/plain; charset="iso-2022-jp" Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Subject: Re: [dts] [PATCH V3 13/13] tests/perf_test_base: script X-BeenThere: dts@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: test suite reviews and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dts-bounces@dpdk.org Sender: "dts" Hi Yufeng, > -----Original Message----- > From: Mo, YufengX > Sent: 2021=1B$BG/=1B(B3=1B$B7n=1B(B25=1B$BF|=1B(B 16:55 > To: dts@dpdk.org; Tu, Lijuan > Cc: Mo, YufengX > Subject: [dts][PATCH V3 13/13] tests/perf_test_base: script >=20 >=20 > stage 2 demand as below >=20 > #. accepted_tolerance in [test suit].cfg are changed to percentage, if no= t set, > it will be set to 2% by default. >=20 > #. delta in json are chagned to percentage. Could you please change accepted_tolerance and delta to float for better in= tegration to CI. More details see: http://patchwork.dpdk.org/project/dts/patch/2021031110480= 0.243752-1-lijuan.tu@intel.com/ >=20 > #. add expected throughput in json. >=20 > #. expected_throughput will be forced to NA if it not set. >=20 > #. add new conf throughput_stat_sample_interval in conf/.cfg, > which > indicates interval of getting throughput statistics in second, e.g. > throughput_stat_sample_interval =3D 10 > if not set, it will be set to 5 seconds by default. >=20 > #. collect better statistics , get multiple throughput statistics during > the test run, and compute the average as the final results. > Every seconds get throughput onc= e, so > it gets multiple throughput statistics during > the test run, remove the max and the min, then compute the average of= the > left as the final throughput. >=20 > Signed-off-by: yufengmx > --- > tests/perf_test_base.py | 84 ++++++++++++++++++++++++++++++++++++++--- > 1 file changed, 78 insertions(+), 6 deletions(-) >=20 > diff --git a/tests/perf_test_base.py b/tests/perf_test_base.py index > 4b76e6b1..5fbda9fa 100644 > --- a/tests/perf_test_base.py > +++ b/tests/perf_test_base.py > @@ -38,6 +38,7 @@ import time > import traceback > import texttable > import json > +import numpy as np > from enum import Enum, unique > from pprint import pformat > from itertools import product > @@ -109,6 +110,12 @@ class STREAM_TYPE(Enum): > RAW =3D 'RAW' >=20 >=20 > +@unique > +class STAT_ALGORITHM(Enum): > + ONE =3D "one" > + MEAN =3D "mean" > + > + > HEADER_SIZE =3D { > 'ether': 18, > 'ipv4': 20, > @@ -449,6 +456,37 @@ class PerfTestBase(object): > time.sleep(self.__traffic_stop_wait_time) > return result >=20 > + def __throughput_set_traffic_opt(self, duration): > + option =3D { > + 'method': 'throughput', > + 'duration': duration, > + 'interval': self.__throughput_stat_sample_interval, > + } > + return option > + > + def __throughput_calculate_result(self, result): > + if self.__stat_algorithm is STAT_ALGORITHM.ONE: > + return result > + > + if not result or len(result) <=3D 0: > + msg =3D "failed to get throughput stat" > + self.logger.error(msg) > + return (0.0, 0.0) > + > + result =3D sorted(result) > + if self.__stat_algorithm is STAT_ALGORITHM.MEAN: > + result =3D result[1:-1] if len(result) >=3D 5 else result > + bps, pps =3D [], [] > + for v1, v2 in result: > + bps.append(v1), pps.append(v2) > + ret_stat =3D round(np.array(bps).mean(), 1), round(np.array(= pps).mean(), 1) > + else: > + msg =3D f"current not support algorithm <{self.__stat_algori= thm}>" > + self.logger.warning(msg) > + ret_stat =3D (0.0, 0.0) > + > + return ret_stat > + > def __throughput(self, l3_proto, mode, frame_size): > """ > measure __throughput according to Layer-3 Protocol and Lookup Mo= de > @@ -464,11 +502,11 @@ class PerfTestBase(object): > option =3D { > 'stream': streams, > 'rate': 100, > - 'traffic_opt': { > - 'method': 'throughput', > - 'duration': duration, }} > + 'traffic_opt': self.__throughput_set_traffic_opt(duration), > + } > # run traffic > result =3D self.__send_packets_by_pktgen(option) > + result =3D self.__throughput_calculate_result(result) > # statistics result > _, pps =3D result > self.verify(pps > 0, "No traffic detected") @@ -871,7 +909,8 @@ = class > PerfTestBase(object): > def __json_throughput(self, value): > return {"unit": "Mpps", "name": "Throughput", > "value": round(value[0], 3), > - "delta": round(value[1], 3), > + "delta": value[1], > + "expected": value[2], > } >=20 > def __json_rate_percent(self, value): > @@ -987,12 +1026,16 @@ class PerfTestBase(object): > expected, pps, round(gap, 2)) > self.logger.error(msg) > else: > - expected =3D pps > + expected =3D None > msg =3D ('{0} {1} expected throughput value is not set, = ' > 'ignore check').format(config, frame_size) > self.logger.warning(msg) > status =3D 'pass' > - js_results.append([status, [pps, pps - expected], percentage= , config, > frame_size]) > + js_results.append([ > + status, > + [pps, (str(round((pps - expected)/expected * 100, 3) ) += '%') if expected > is not None else "N/A", > + round(expected, 3) if expected is not None else "N/A"], > + percentage, config, frame_size]) > # save data with json format > self.__save_throughput_result(self.__cur_case, js_results) > # display result table > @@ -1394,11 +1437,40 @@ class PerfTestBase(object): > self.__valports =3D port_list > return port_list >=20 > + @property > + def __stat_algorithm(self): > + algorithm_type =3D get_enum_name( > + self.__test_content.get('stat_algorithm') or 'one', STAT_ALG= ORITHM) > + return algorithm_type > + > + @property > + def __throughput_stat_sample_interval(self): > + if self.__stat_algorithm is STAT_ALGORITHM.ONE: > + return None > + else: > + return > + self.__test_content.get('throughput_stat_sample_interval') or 5 > + > + def __parse_accepted_tolerance(self): > + tolerance =3D self.__test_content.get('accepted_tolerance') > + def get_num(value): > + pat =3D "(\d+\.?\d?)" > + result =3D re.findall(pat, value) > + return result[0] if result else '2.0' > + > + if isinstance(tolerance, int): > + _tolerance =3D float(tolerance) > + elif isinstance(tolerance, str): > + _tolerance =3D float(tolerance if tolerance.isdigit() else > get_num(tolerance)) > + else: > + _tolerance =3D None > + self.__test_content['accepted_tolerance'] =3D float(_tolerance o= r > + 2.0) > + > def perf_preset_test_environment(self, test_content): > # if user set port list in cfg file, use > port_list =3D self.__preset_port_list(test_content) > # get test content > self.__test_content =3D self.__get_test_content_from_cfg(test_co= ntent) > + self.__parse_accepted_tolerance() > # prepare target source code application > self.__preset_compilation() > # set up testing environment > -- > 2.21.0