From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mga17.intel.com (mga17.intel.com [192.55.52.151]) by dpdk.org (Postfix) with ESMTP id 47A251B40E for ; Wed, 6 Jun 2018 07:37:03 +0200 (CEST) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga007.fm.intel.com ([10.253.24.52]) by fmsmga107.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 05 Jun 2018 22:37:02 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.49,482,1520924400"; d="scan'208";a="44986258" Received: from shecgisg006.sh.intel.com ([10.239.39.68]) by fmsmga007.fm.intel.com with ESMTP; 05 Jun 2018 22:37:01 -0700 Received: from shecgisg006.sh.intel.com (localhost [127.0.0.1]) by shecgisg006.sh.intel.com with ESMTP id w565b1K0025516; Wed, 6 Jun 2018 13:37:01 +0800 Received: (from yufengmx@localhost) by shecgisg006.sh.intel.com with œ id w565b1fA025512; Wed, 6 Jun 2018 13:37:01 +0800 From: yufengx.mo@intel.com To: dts@dpdk.org Cc: yufengmx Date: Wed, 6 Jun 2018 13:36:58 +0800 Message-Id: <1528263420-25329-3-git-send-email-yufengx.mo@intel.com> X-Mailer: git-send-email 1.7.0.7 In-Reply-To: <1528263420-25329-1-git-send-email-yufengx.mo@intel.com> References: <1528263420-25329-1-git-send-email-yufengx.mo@intel.com> Subject: [dts] [PATCH V1 2/3] metrics: upload automation script X-BeenThere: dts@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: test suite reviews and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 06 Jun 2018 05:37:04 -0000 From: yufengmx This automation script is for metrics feature. The Metrics implements a mechanism by which *producers* can publish numeric information for later querying by *consumers*. Here dpdk-procinfo process is the *consumers*. latency stats and bit rate are the two implements based on metrics lib. Signed-off-by: yufengmx --- tests/TestSuite_metrics.py | 891 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 891 insertions(+) create mode 100644 tests/TestSuite_metrics.py diff --git a/tests/TestSuite_metrics.py b/tests/TestSuite_metrics.py new file mode 100644 index 0000000..29b4790 --- /dev/null +++ b/tests/TestSuite_metrics.py @@ -0,0 +1,891 @@ +# BSD LICENSE +# +# Copyright(c) 2010-2018 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# 'AS IS' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +''' +DPDK Test suite metrics +''' + +import re +import time +import sys, os +import utils +import shutil +from collections import defaultdict + +from test_case import TestCase +from pmd_output import PmdOutput +from exception import VerifyFailure +from etgen import IxiaPacketGenerator +from settings import HEADER_SIZE, SCAPY2IXIA +from packet import Packet +import serializer + +class TestMetrics(TestCase, IxiaPacketGenerator): + + def clear_ASLR(self): + ''' + DPDK don't support ASLR open under multiprocess status, so close it + ''' + cmd = "echo 0 > /proc/sys/kernel/randomize_va_space" + self.dut_2.send_expect(cmd, "# ", timeout=10) + time.sleep(2) + + def reset_ASLR(self): + ''' + reset ASLR to original status + ''' + cmd = "echo 2 > /proc/sys/kernel/randomize_va_space" + self.dut_2.send_expect(cmd, "# ", timeout=10) + time.sleep(4) + + def save_ref_data(self, ref_data, ref_data_path): + data_name = 'ref_data' + # save to pickle file + pkl = serializer.SerializerBase() + pkl.set_serialized_filename(ref_data_path) + pkl.save(data_name, ref_data) + pkl.save_to_file() + + def load_ref_data(self, ref_data_path): + data_name = 'ref_data' + # save to pickle file + pkl = serializer.SerializerBase() + pkl.set_serialized_filename(ref_data_path) + pkl.load_from_file() + return pkl.volatile_cache[data_name] + + def result_parse(self, stat_data, data_type, port_id='0'): + #config_streams() + check = getattr(self, "{0}_check_data".format(data_type)) + ref_data_path = self.ref_data_path.format(data_type) + # if no ref data file exist, save one + if not os.path.exists(ref_data_path): + ref_stats_ex = stat_data + self.save_ref_data(ref_stats_ex, ref_data_path) + self.logger.info("save this testing data as reference data") + ref_data = self.load_ref_data(ref_data_path) + for sub_stat in stat_data: + test_configs = sub_stat['test_configs'] + for sub_ref_stat in ref_data: + test_ref_configs = sub_ref_stat['test_configs'] + if cmp(test_configs, test_ref_configs) == 0: + break + else: + self.logger.warning("missing testing content") + continue + check(sub_stat, sub_ref_stat, port_id) + + def get_pkt(self, payload_size): + fmt = 'wrpcap("pause_rx.pcap", [Ether()/IP()/UDP()/("X"*{0})])'.format + return fmt(payload_size) + + def check_packet_transmission(self, pkt_types): + time.sleep(1) + for pkt_type in pkt_types: + pkt = Packet(pkt_type=pkt_type) + refPacket = "/root/{0}_ref.pcap".format(pkt_type) + pkt.pktgen.write_pcap(refPacket) + pkt.send_pkt(tx_port=self.src_intf) + time.sleep(2) + # check dut testpmd packet received information + out = self.dut.get_session_output(timeout=2) + time.sleep(1) + + def preset_testpmd(self, core_mask, options): + self.testpmd.start_testpmd( core_mask, + param=' '.join(['--port-topology=loop', + options])) + self.execute_testpmd_cmd(self.preset_testpmd_cmds) + self.test_pmd_status = True + self.preset_testpmd_cmds = list() + time.sleep(1) + + def execute_testpmd_cmd(self, cmds): + if len(cmds) == 0: + return + for item in cmds: + if len(item) == 2: + output = self.testpmd.execute_cmd(item[0], int(item[1])) + else: + output = self.testpmd.execute_cmd(item[0]) + self.logger.info(output) + time.sleep(2) + + def stop_testpmd(self): + time.sleep(1) + testpmd_cmds =[['stop']] + self.execute_testpmd_cmd(testpmd_cmds) + time.sleep(1) + + def close_testpmd(self): + if not self.test_pmd_status: + return + time.sleep(1) + self.testpmd.quit() + time.sleep(1) + self.test_pmd_status = False + + def execute_dpdk_proc(self, required_type=None): + self.dut_2.send_expect( self.dpdk_proc_cmd, "# ") + time.sleep(2) + msg = self.dut_2.session.get_session_before(0.5) + if not required_type: + expect_content = self.latency_content + self.bit_rate_content + elif required_type == 'latency': + expect_content = self.latency_content + elif required_type == 'bit rate': + expect_content = self.bit_rate_content + + missing_content = [] + for item in expect_content: + if item not in msg: + missing_content.append(item) + if missing_content: + self.logger.error(os.linesep.join(["metrics has no these data::"] + + missing_content)) + self.logger.info(msg) + + #raise VerifyFailure("{0} exception ! ".format(self.query_tool)) + self.logger.info("{0}".format(msg)) + return msg + + def ixia_pre(self): + pass + + def check_dpdk_proc_tool(self, msg, flag=None): + # check data statistic tool boot up status + # send a specified packet, check tool output data format + # check loop mode/chained mode statistics data format + portStatus = {} + curPortNo = None + portPat = "metrics for port (\d)+.*#" + for item2 in msg.splitlines(): + item = item2.strip(os.linesep) + if 'metrics' in item: + curPortNo = "".join(re.findall(portPat, item, re.M)) + portStatus[curPortNo] = {} + if curPortNo == None: + continue + if ":" in item: + status = item.strip().split(': ') + if len(status) == 2: + portStatus[curPortNo][status[0]] = int(status[1].strip()) + + retPortStatus = {} + for item in portStatus: + if flag == 'bit rate': + if item == '': + continue + retPortStatus[item] = {} + for item2 in self.bit_rate_content: + retPortStatus[item][item2] = portStatus[item][item2] + elif flag == 'latency': + if item != '': + continue + retPortStatus[item] = {} + for item2 in self.latency_content: + retPortStatus[item][item2] = portStatus[item][item2] + else: + retPortStatus = portStatus + + return retPortStatus + + def bit_rate_check_data(self, stats, ref_stats, port_id): + expected_rate = 1/10e2 + running_stats = stats['running'] + self.compare_with_ref(stats, ref_stats, port_id, expected_rate) + #---------------- + # check stats data of ixia stop + #---------------- + stop_stats = stats['stop'] + # check peak_bits_out/peak_bits_in + # it should be the same as running data + check_keys = ['peak_bits_out', 'peak_bits_in'] + for key in check_keys: + running_data = running_stats[-1][port_id][key] + stop_data = stop_stats[0][port_id][key] + rate = abs(stop_data - running_data)/running_data + if rate < expected_rate: + msg = "peak_bits_out/peak_bits_in are correct after stop ixia" + else: + msg = ("peak_bits_out/peak_bits_in data are more than " + "expect {0}").format(expected_rate) + self.logger.error(msg) + raise VerifyFailure(msg) + self.logger.info(msg) + # check mean_bits_in/mean_bits_out, it should be zero + even_stats = defaultdict(long) + for item in stop_stats: + for key, value in item[port_id].items(): + even_stats[key] += value + check_keys = ['mean_bits_in', 'mean_bits_out'] + for key in check_keys: + if key in even_stats: + if even_stats[key] != 0: + msg = "{0} data is not zero".format(key) + self.logger.error(msg) + raise VerifyFailure(msg) + else: + msg = "{0} data is correct after stop ixia".format(key) + self.logger.info(msg) + # check ewma_bits_out/ewma_bits_in, + # it should have a none-zero data at first group data + check_keys = ['ewma_bits_out', 'ewma_bits_in'] + for key in check_keys: + if key in stop_stats[0]: + if stop_stats[0][key] == 0: + msg = ("{0} data should not be zero " + "at once at the time of stopping ixia").format(key) + self.logger.error(msg) + raise VerifyFailure(msg) + else: + msg = "{0} data is correct after stop ixia".format(key) + self.logger.info(msg) + # it should be zero at last group data + for key in check_keys: + if key in stop_stats[-1]: + if stop_stats[-1][key] != 0: + msg = ("{0} data is not zero after " + "stop a long time").format(key) + self.logger.error(msg) + raise VerifyFailure(msg) + else: + msg = "{0} data is correct after stop ixia".format(key) + self.logger.info(msg) + + def latency_stat_check_data(self, stats, ref_stats, port_id): + expected_rate = 8/10e1 + running_stats = stats['running'] + stop_stats = stats['stop'] + check_keys = ['min_latency_ns', 'max_latency_ns'] + # compare with reference data + self.compare_with_ref(stats, ref_stats, port_id, expected_rate) + #---------------- + # check stats data of ixia stop + #---------------- + # all should be the same as running data + for key in check_keys: + running_data = running_stats[0][port_id][key] + stop_data = stop_stats[-1][port_id][key] + rate = abs(stop_data - running_data)/running_data + if rate < expected_rate: + msg = "{0} is correct after stop ixia".format(key) + else: + msg = "{0} data is {1}% more than expected".format(key, + rate*100) + self.logger.info(msg) + # ``jitter_ns`` is not a reliable value on packet > 1024, so ignore it + check_keys = ['avg_latency_ns'] + #expected_rate = float(1/10) + # all should be the same as running data + for key in check_keys: + running_data = running_stats[0][port_id][key] + stop_data = stop_stats[-1][port_id][key] + rate = abs(stop_data - running_data)/running_data + if rate < expected_rate: + msg = "{0} is correct after stop ixia".format(key) + else: + msg = "{0} data is {1}% more than expected".format(key, + rate*100) + self.logger.info(msg) + # the sample data should be the same + last_stat = stop_stats[0] + for stat in stop_stats[1:]: + if cmp(last_stat, stat) != 0: + msg = "stat data after stop should be kept".format(key) + break + else: + msg = "stat data of stop is correct" + self.logger.info(msg) + + def compare_with_ref(self, stats, ref_stats, port_id, expected_rate): + #---------------- + # check stats data of ixia on running status + #---------------- + # all stats data should be closed to reference sample data + # one type data is calculated by all group data's average value + ref_running_stats = ref_stats['running'] + ref_even_stats = defaultdict(long) + for item in ref_running_stats: + for key, value in item[port_id].items(): + ref_even_stats[key] += value + + running_stats = stats['running'] + even_stats = defaultdict(long) + for item in running_stats: + for key, value in item[port_id].items(): + even_stats[key] += value + if cmp(sorted(even_stats.keys()), sorted(ref_even_stats.keys())) != 0: + msg = "original data have different values as current data" + self.logger.error(msg) + raise VerifyFailure(msg) + return + for key in even_stats: + current_value = even_stats[key] + ref_value = ref_even_stats[key] + rate = float(abs(float(current_value - ref_value))/float(ref_value)) + if rate < expected_rate: + msg = "{0} is correct".format(key) + self.logger.info(msg) + else: + msg = ("{0} data is {1}% more than " + "reference data").format(key, rate*100) + self.logger.error(msg) + #raise VerifyFailure(msg) + + def calculate_latency_deviation(self, ixia_result, dpdk_result): + compare_group = {"max latency" : [ixia_result["max"], + dpdk_result["max_latency_ns"]], + "min latency" : [ixia_result["min"], + dpdk_result["min_latency_ns"]], + "average latency" : [ixia_result["average"], + dpdk_result["avg_latency_ns"]], + "jitter value" : ["12", dpdk_result["jitter_ns"]],} + maxWidth = max(map(lambda x: len(x), compare_group.keys())) + for item, value in compare_group.items(): + percent = round((float(value[0])/(float(value[1] or 1)*10))*100, 2) + tagStr = "{0}".format(item).ljust(maxWidth) + " deviation: " + msg = tagStr + "{0}%".format(percent) + self.logger.info(msg) + return + + def display_metrics_data(self, port_status): + textLength = max(map(lambda x: len(x), self.display_seq)) + for port in sorted(port_status.keys()): + port_value = port_status[port] + if port != None and port!= '' and int(port) == self.test_port: + self.logger.info("port {0}".format(port)) + for key in self.display_seq: + value = port_value[key] + self.logger.info("{0} = [{1}]".format( + key.ljust(textLength), value)) + else: + maxvalue = max(map(lambda x: int(x), port_value.values())) + if not maxvalue: + continue + self.logger.info("port {0}".format(port)) + for key in self.display_seq: + value = port_value[key] + if value: + self.logger.info("{0} = [{1}]".format( + key.ljust(textLength), + value)) + + def set_up_all(self): + self.dut_ports = self.dut.get_ports(self.nic) + self.verify(len(self.dut_ports) >= 1, 'Insufficient ports') + self.src_intf = self.tester.get_interface(self.tester.get_local_port(0)) + self.test_port = 0 + # secondary process command (testpmd) + self.preset_testpmd_cmds = list() + self.testpmd = PmdOutput(self.dut) + self.test_pmd_status = False + # secondary process command (dpdk-procinfo) + self.dut_2 = self.dut.new_session() + self.query_tool = "dpdk-procinfo" + option = " -- --metrics" + self.dpdk_proc_cmd = os.sep.join([self.dut.base_dir, self.target, + "app/{0}".format(self.query_tool)]) + option + # package size + self.latency_content = ['min_latency_ns', 'max_latency_ns', + 'avg_latency_ns', 'jitter_ns'] + self.bit_rate_content = ['mean_bits_in', 'peak_bits_in', + 'mean_bits_out','peak_bits_out', + 'ewma_bits_in', 'ewma_bits_out'] + self.display_seq = self.latency_content + self.bit_rate_content + # This configuration is used for daily testing + # testing packet frame size + self.test_frame_sizes = [64, 256, 1024] + #self.test_frame_sizes = [64, 72, 128, 128+8, 256, 512, 512+8, 1024] + self.max_times = [60*15] + self.query_max = 15 + self.query_times_after_stop = 5 + data_type = 'daily' + #data_type = 'full' + + self.clear_ASLR() + # set configs for saving data using pickle + ref_dir = '/root' + folder = 'metrics_{0}'.format(data_type) + base_path = os.sep.join([ref_dir, folder]) + if not os.path.exists(base_path): + os.makedirs(base_path) + self.ref_data_path = os.sep.join([base_path, '{0}_data.pkl']) + + def set_up(self): + pass + + def tear_down(self): + self.close_testpmd() + + def tear_down_all(self): + if self.dut_2: + self.reset_ASLR() + self.dut_2.close() + self.dut_2 = None + + def test_basic(self): + ''' + use to do metrics feature testing + ''' + testpmd_cmds =[['set fwd io'], + ['set verbose 1'], + ['start']] + pkt_types = {'TCP': ""} + + self.execute_testpmd_cmd(testpmd_cmds) + # get original data + msg = self.execute_dpdk_proc() + pre_status = self.check_dpdk_proc_tool(msg) + for port, port_value in pre_status.items(): + for key in self.display_seq: + value = port_value[key] + if int(value) == 0: + continue + msg = "{0} {1} value is error <{2}>".format(port, key, value) + self.logger.error(msg) + raise VerifyFailure(msg) + + # send out packet + self.check_packet_transmission(pkt_types) + # get transmission data + msg = self.execute_dpdk_proc() + post_status = self.check_dpdk_proc_tool(msg) + msg = self.execute_dpdk_proc() + second_post_status = self.check_dpdk_proc_tool(msg) + self.display_metrics_data(post_status) + # compare twice query data + if cmp(post_status, second_post_status) != 0: + msg = "data has changed when do multi query operation !!! " + self.logger.error(msg) + self.display_metrics_data(second_post_status) + + def ixia_traffic_start(self, frame_size, rate_percent=100): + pause_time = 65535 + pause_rate = 0.50 + # calculate number of packets + expect_pps = self.wirespeed(self.nic, frame_size, 1) * 1000000.0 + # get line rate + linerate = expect_pps * (frame_size + 20) * 8 + # calculate default sleep time for one pause frame + sleep = (1 / linerate) * pause_time * 512 + # calculate packets dropped in sleep time + self.n_pkts = int((sleep / (1 / expect_pps)) * (1 / pause_rate)) + # + tgen_input = [] + headers_size = sum(map(lambda x: HEADER_SIZE[x], + ['eth', 'ip', 'udp'])) + + payload_size = frame_size - headers_size + self.logger.warning(str(payload_size)) + self.tester.scapy_append(self.get_pkt(payload_size)) + self.tester.scapy_execute() + # rx and tx is the same port + #for cnt in range(len(self.dut_ports)): + # only use ixia link peer + for cnt in range(len(self.dut_ports)): + if self.tester.get_local_port_type(cnt) != 'ixia': + continue + tester_port = self.tester.get_local_port(self.dut_ports[cnt]) + tgen_input.append((tester_port, tester_port, "pause_rx.pcap")) + #---------------------------------------------- + # run traffic + self.tester.loop_traffic_generator_throughput(tgen_input, rate_percent) + return tgen_input, expect_pps + + def ixia_traffic_start2(self, frame_size, rate_percent=100): + pause_time = 65535 + pause_rate = 0.50 + # calculate number of packets + expect_pps = self.wirespeed(self.nic, frame_size, 1) * 1000000.0 + # get line rate + linerate = expect_pps * (frame_size + 20) * 8 + # calculate default sleep time for one pause frame + sleep = (1 / linerate) * pause_time * 512 + # calculate packets dropped in sleep time + self.n_pkts = int((sleep / (1 / expect_pps)) * (1 / pause_rate)) + # + tgen_input = [] + headers_size = sum(map(lambda x: HEADER_SIZE[x], + ['eth', 'ip', 'udp'])) + payload_size = frame_size - headers_size + self.logger.warning(str(payload_size)) + self.tester.scapy_append(self.get_pkt(payload_size)) + self.tester.scapy_execute() + # rx and tx is the same port + for cnt in range(len(self.dut_ports)): + if self.tester.get_local_port_type(cnt) != 'ixia': + continue + tester_port = self.tester.get_local_port(self.dut_ports[cnt]) + tgen_input.append((tester_port, tester_port, "pause_rx.pcap")) + #---------------------------------------------- + # run traffic + self.tester.loop_traffic_generator_latency(tgen_input, rate_percent) + return tgen_input, expect_pps + + def test_perf_bit_rate(self): + """ + testing metrics bit rate + """ + start_time = time.time() + lasting_time = 60*15 + self.logger.info("start time is [%d]s"%(start_time)) + # set testpmd on ready status + testpmd_cmds =[['stop'], ['set fwd io'], + ['set verbose 1'], ['start']] + options = '--bitrate-stats=2' + self.preset_testpmd('1S/2C/1T', options) + stats = [] + for testing_time in self.max_times: + for frame_size in self.test_frame_sizes: + # run testpmd + self.execute_testpmd_cmd(testpmd_cmds) + # save testing configuration + sub_stats = {} + stats.append(sub_stats) + sub_stats['test_configs'] = {} + test_configs = sub_stats['test_configs'] + test_configs['frame_size'] = frame_size + test_configs['testing time'] = testing_time + max_query_max = self.query_max + wait_interval = testing_time/max_query_max + test_configs['query interval time'] = wait_interval + tgen_input, expect_pps = self.ixia_traffic_start(frame_size) + # initialize bit rate result + bit_rate_results = [] + bit_rate_stop_results = [] + bit_rate_ixia_results = [] + # query metrics status at a interval + max_query_count = max_query_max + while max_query_count: + time.sleep(wait_interval) + msg = self.execute_dpdk_proc('bit rate') + bit_rate_results.append( + self.check_dpdk_proc_tool(msg, 'bit rate')) + max_query_count -= 1 + sub_stats['running'] = bit_rate_results + #--------------------- + msgs = ["stop continuous packet transmission sampling", + "next to sample statistics after stop packet transmission"] + self.logger.warning(os.linesep.join(msgs)) + rx_bps, rx_pps = \ + self.tester.stop_traffic_generator_throughput_loop(tgen_input) + bit_rate_ixia_results.append([rx_pps, rx_bps, expect_pps]) + sub_stats['ixia_stats'] = bit_rate_ixia_results + # sample data after stop + max_query_count = self.query_times_after_stop + while max_query_count: + time.sleep(wait_interval) + msg = self.execute_dpdk_proc('bit rate') + bit_rate_stop_results.append( + self.check_dpdk_proc_tool(msg, 'bit rate')) + max_query_count -= 1 + sub_stats['stop'] = bit_rate_stop_results + #---------------------------- + # check statistic status after stop testpmd + msg = "verify bit rate statistics stop after stop testpmd::" + self.logger.warning(msg) + self.stop_testpmd() + stop_testpmd_results = [] + max_query_count = 2 + while max_query_count: + time.sleep(wait_interval) + msg = self.execute_dpdk_proc('bit rate') + stop_testpmd_results.append( + self.check_dpdk_proc_tool(msg, 'bit rate')) + max_query_count -= 1 + # compare results + cnt = 1 + first_result = stop_testpmd_results[0] + second_result = stop_testpmd_results[1] + if cmp(first_result, second_result) == 0: + msg = ("bit rate statistics stop successful " + "after stop testpmd") + self.logger.warning(msg) + else: + msg = "bit rate statistics fail to stop after stop testpmd" + self.logger.error(msg) + #----------------------------------------------------- + end_time = time.time() + self.logger.info("current time is [%d]s"%(end_time)) + if end_time - start_time > lasting_time: + pass + else: + msg = "left time is [%d]s"%(lasting_time - + (end_time - start_time)) + self.logger.info(msg) + ##################################################### + msgs = [ + "frame_size [{0}]".format(frame_size), + "testing time [{0}]".format(testing_time), + "query interval time [{0}]".format(wait_interval)] + self.logger.info(os.linesep.join(msgs)) + self.logger.warning("dpdk bit rate statistics ::") + cnt = 1 + for result in bit_rate_results: + msg = "-"*10 + "No.{0} query result".format(cnt) + "-"*10 + self.logger.warning(msg) + for key, value in sorted(result.items()): + self.logger.info("Port {0} statistics:".format(key)) + self.logger.info(value) + cnt += 1 + #----------------------------------------------- + # used for debug + msg = "dpdk bit rate statistics after stop transmission::" + self.logger.warning(msg) + cnt = 1 + for result in bit_rate_stop_results: + msg = "-"*10 + "No.{0} query result".format(cnt) + "-"*10 + self.logger.warning(msg) + for key, value in sorted(result.items()): + self.logger.info("Port {0} statistics:".format(key)) + self.logger.info(value) + cnt += 1 + #---------------------------- + self.logger.warning("ixia bit rate statistics ::") + for rx_pps, rx_bps, expect_pps in bit_rate_ixia_results: + msgs = [ + "expect_pps {0} Mpps".format(expect_pps), + "rx_pps {0} Mpps".format(rx_pps), + "rx_bps {0} Mbps".format(rx_bps)] + self.logger.info(os.linesep.join(msgs)) + # executing testing data check + self.result_parse(stats, data_type = 'bit_rate') + + def test_perf_bit_rate_peak(self): + """ testing bit rate change with higher rate """ + for frame_size in self.test_frame_sizes: + # set testpmd on ready status + testpmd_cmds =[['set fwd io'], ['set verbose 1'], ['start']] + options = '--bitrate-stats=2' + self.preset_testpmd('1S/2C/1T', options) + self.execute_testpmd_cmd(testpmd_cmds) + max_bit_rate_in = 0 + max_bit_rate_out = 0 + testing_time = 60*3 + rate_percents = [100, 80, 40, 20] + for rate_percent in rate_percents: + max_query_max = self.query_max + wait_interval = testing_time/max_query_max + tgen_input, expect_pps = self.ixia_traffic_start(frame_size, + rate_percent) + # initialize bit rate result + bit_rate_results = [] + bit_rate_stop_results = [] + bit_rate_ixia_results = [] + + max_query_count = max_query_max + while max_query_count: + time.sleep(wait_interval) + msg = self.execute_dpdk_proc('bit rate') + bit_rate_results.append( + self.check_dpdk_proc_tool(msg, 'bit rate')) + max_query_count -= 1 + msgs = ["stop continuous packet transmission sampling", + "next to sample statistics after stop packet transmission"] + self.logger.warning(os.linesep.join(msgs)) + rx_bps, rx_pps = \ + self.tester.stop_traffic_generator_throughput_loop(tgen_input) + bit_rate_ixia_results.append([rx_pps, rx_bps, expect_pps]) + # sample data after stop + max_query_count = self.query_times_after_stop + while max_query_count: + time.sleep(wait_interval) + msg = self.execute_dpdk_proc('bit rate') + bit_rate_stop_results.append( + self.check_dpdk_proc_tool(msg, 'bit rate')) + max_query_count -= 1 + ####################################################### + msgs = [ + "frame_size [{0}]".format(frame_size), + "query interval time [{0}]".format(wait_interval)] + self.logger.info(os.linesep.join(msgs)) + self.logger.warning("dpdk bit rate statistics ::") + cnt = 1 + peak_in = 0 + peak_out = 0 + for result in bit_rate_results: + msg = "-"*10 + "No.{0} query result".format(cnt) + "-"*10 + self.logger.warning(msg) + for key, value in sorted(result.items()): + self.logger.info("Port {0} statistics:".format(key)) + self.logger.info(value) + if value['peak_bits_in'] > peak_in: + peak_in = value['peak_bits_in'] + if value['peak_bits_out'] > peak_out: + peak_out = value['peak_bits_out'] + cnt += 1 + #----------------------------------------------- + # check peak bit rate + if peak_in <= 0 or peak_out <= 0: + msg = "peak_bits(in/out) value should not be zero" + raise VerifyFailure(msg) + # save maximum bit rate value of ``peak_bits_in`` + # since ixia work with decreasing traffic rate, + # max_bit_rate should keep the first value + #------------ check in statistcis + if max_bit_rate_in == 0: + max_bit_rate_in = peak_in + elif max_bit_rate_in != peak_in: + fmt = ("peak bit rate(in) should be {0}, " + "current value is {1}").format + msg = fmt(max_bit_rate_in, peak_in) + raise VerifyFailure(msg) + #------------ check out statistcis + if max_bit_rate_out == 0: + max_bit_rate_out = peak_out + elif max_bit_rate_out != peak_out: + fmt = ("peak bit rate(out) should be {0}, " + "current value is {1}").format + msg = fmt(max_bit_rate_out, peak_out) + raise VerifyFailure(msg) + #----------------------------------------------- + # used for debug + msg = "dpdk bit rate statistics after stop transmission::" + self.logger.warning(msg) + cnt = 1 + for result in bit_rate_stop_results: + msg = "-"*10 + "No.{0} query result".format(cnt) + "-"*10 + self.logger.warning(msg) + for key, value in sorted(result.items()): + self.logger.info("Port {0} statistics:".format(key)) + self.logger.info(value) + cnt += 1 + + self.logger.warning("ixia bit rate statistics ::") + for rx_pps, rx_bps, expect_pps in bit_rate_ixia_results: + msgs = ["expect_pps {0} Mpps".format(expect_pps), + "rx_pps {0} Mpps".format(rx_pps), + "rx_bps {0} Mbps".format(rx_bps)] + self.logger.info(os.linesep.join(msgs)) + + def test_perf_latency_stat(self): + """ + Test + """ + start_time = time.time() + lasting_time = 60*15 + query_interval = 0 + self.logger.info("start time is [%d]s"%(start_time)) + max_time = self.max_times + # set packet for send + # pause frame basic configuration + # set testpmd on ready status + testpmd_cmds =[['stop'], + ['set fwd io'], + ['set verbose 1'], + ['start']] + options = '--latencystats=2' + self.preset_testpmd('1S/2C/1T', options) + stats = [] + for testing_time in self.max_times: + for frame_size in self.test_frame_sizes: + # run testpmd + self.execute_testpmd_cmd(testpmd_cmds) + # save testing configuration + sub_stats = {} + stats.append(sub_stats) + sub_stats['test_configs'] = {} + test_configs = sub_stats['test_configs'] + test_configs['frame_size'] = frame_size + test_configs['testing time'] = testing_time + # start run traffic + tgen_input, expect_pps = self.ixia_traffic_start2(frame_size) + latency_results = [] + latency_stop_results = [] + latency_ixia_results = [] + # set query interval + max_query_max = self.query_max + wait_interval = testing_time/max_query_max + max_query_count = max_query_max + while max_query_count: + time.sleep(wait_interval) + msg = self.execute_dpdk_proc('latency') + latency_results.append( + self.check_dpdk_proc_tool(msg, 'latency')) + max_query_count -= 1 + sub_stats['running'] = latency_results + msgs = ["stop continuous packet transmission sampling", + "next to sample statistics after stop packet transmission"] + self.logger.warning(os.linesep.join(msgs)) + latencys = \ + self.tester.stop_traffic_generator_latency(tgen_input) + latency_ixia_results.append([latencys]) + sub_stats['ixia_stats'] = latency_ixia_results + max_query_count = self.query_times_after_stop + while max_query_count: + time.sleep(wait_interval) + msg = self.execute_dpdk_proc('latency') + latency_stop_results.append( + self.check_dpdk_proc_tool(msg, 'latency')) + max_query_count -= 1 + sub_stats['stop'] = latency_stop_results + #--------------------------------------------------------- + end_time = time.time() + self.logger.info("current time is [%d]s"%(end_time)) + if end_time - start_time > lasting_time: + pass + else: + self.logger.info("left time is [%d]s"%(lasting_time - + (end_time - start_time))) + ############################################################ + msgs = [ + "frame_size [{0}]".format(frame_size), + "testing time [{0}]".format(testing_time), + "query interval time [{0}]".format(wait_interval)] + self.logger.info(os.linesep.join(msgs)) + self.logger.warning("dpdk latency statistics ::") + cnt = 1 + for result in latency_results: + msg = "-"*10 + "No.{0} query result".format(cnt) + "-"*10 + self.logger.warning(msg) + for key, value in sorted(result.items()): + self.logger.info("Port {0} statistics:".format(key)) + self.logger.info(value) + cnt += 1 + #----------------------------------------------- + # used for debug + msg = "dpdk latency statistics after stop transmission::" + self.logger.warning(msg) + cnt = 1 + for result in latency_stop_results: + msg = "-"*10 + "No.{0} query result".format(cnt) + "-"*10 + self.logger.warning(msg) + for key, value in sorted(result.items()): + self.logger.info("Port {0} statistics:".format(key)) + self.logger.info(value) + cnt += 1 + + self.logger.warning("ixia latency statistics ::") + for latencys in latency_ixia_results: + self.logger.info("latencys(ns) {0}".format(latencys)) + + self.result_parse(stats, data_type = 'latency_stat', port_id='') -- 1.9.3