Re: [dts] [PATCH V2 4/4] tests/metrics: upload automation script

["Wang, Yinan" <yinan.wang@intel.com>]

Acked-by: Wang, Yinan <yinan.wang@intel.com>

> -----Original Message-----
> From: Mo, YufengX <yufengx.mo@intel.com>
> Sent: 2019年12月13日 16:52
> To: dts@dpdk.org; Wang, Yinan <yinan.wang@intel.com>
> Cc: Mo, YufengX <yufengx.mo@intel.com>
> Subject: [dts][PATCH V2 4/4] tests/metrics: upload automation script
> 
> 
> The Metrics implements a mechanism by which producers can publish numeric
> information for later querying by consumers. Here dpdk-procinfo process is the
> consumers. latency stats and bit rate are the two implements based on metrics
> lib.
> 
> Signed-off-by: yufengmx <yufengx.mo@intel.com>
> ---
>  tests/TestSuite_metrics.py | 824
> +++++++++++++++++++++++++++++++++++++
>  1 file changed, 824 insertions(+)
>  create mode 100644 tests/TestSuite_metrics.py
> 
> diff --git a/tests/TestSuite_metrics.py b/tests/TestSuite_metrics.py new file
> mode 100644 index 0000000..d098ccd
> --- /dev/null
> +++ b/tests/TestSuite_metrics.py
> @@ -0,0 +1,824 @@
> +# BSD LICENSE
> +#
> +# Copyright(c) 2010-2019 Intel Corporation. All rights reserved.
> +# All rights reserved.
> +#
> +# Redistribution and use in source and binary forms, with or without #
> +modification, are permitted provided that the following conditions #
> +are met:
> +#
> +#   * Redistributions of source code must retain the above copyright
> +#     notice, this list of conditions and the following disclaimer.
> +#   * Redistributions in binary form must reproduce the above copyright
> +#     notice, this list of conditions and the following disclaimer in
> +#     the documentation and/or other materials provided with the
> +#     distribution.
> +#   * Neither the name of Intel Corporation nor the names of its
> +#     contributors may be used to endorse or promote products derived
> +#     from this software without specific prior written permission.
> +#
> +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
> CONTRIBUTORS #
> +'AS IS' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT #
> +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
> FOR #
> +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
> COPYRIGHT #
> +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
> INCIDENTAL, #
> +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT #
> +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
> USE, #
> +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
> ANY #
> +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT #
> +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
> USE #
> +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> +
> +'''
> +DPDK Test suite.
> +'''
> +
> +import os
> +import re
> +import time
> +import random
> +import traceback
> +from copy import deepcopy
> +from pprint import pformat
> +
> +from test_case import TestCase
> +from pmd_output import PmdOutput
> +from exception import VerifyFailure
> +from settings import HEADER_SIZE
> +from packet import Packet
> +from pktgen import TRANSMIT_CONT
> +from config import SuiteConf
> +
> +
> +class TestMetrics(TestCase):
> +    BIT_RATE = 'bit_rate'
> +    LATENCY = 'latency'
> +    display_seq = {
> +        # metrics bit rate
> +        BIT_RATE: [
> +            'mean_bits_in',
> +            'peak_bits_in',
> +            'ewma_bits_in',
> +            'mean_bits_out',
> +            'peak_bits_out',
> +            'ewma_bits_out'],
> +        # metrics latency
> +        LATENCY: [
> +            'min_latency_ns',
> +            'max_latency_ns',
> +            'avg_latency_ns',
> +            'jitter_ns'], }
> +
> +    def d_a_con(self, cmd):
> +        _cmd = [cmd, '# ', 10] if isinstance(cmd, (str, unicode)) else cmd
> +        output = self.dut.alt_session.send_expect(*_cmd)
> +        output2 = self.dut.alt_session.session.get_session_before(2)
> +        return output + os.linesep + output2
> +
> +    @property
> +    def target_dir(self):
> +        # get absolute directory of target source code
> +        target_dir = '/root' + self.dut.base_dir[1:] \
> +                     if self.dut.base_dir.startswith('~') else \
> +                     self.dut.base_dir
> +        return target_dir
> +
> +    def get_pkt_len(self, pkt_type, frame_size=64):
> +        headers_size = sum(
> +            map(lambda x: HEADER_SIZE[x], ['eth', 'ip', pkt_type]))
> +        pktlen = frame_size - headers_size
> +        return pktlen
> +
> +    def config_stream(self, framesize):
> +        payload_size = self.get_pkt_len('udp', framesize)
> +        # set streams for traffic
> +        pkt_config = {
> +            'type': 'UDP',
> +            'pkt_layers': {
> +                'raw': {'payload': ['58'] * payload_size}}, }
> +        # create packet for send
> +        pkt_type = pkt_config.get('type')
> +        pkt_layers = pkt_config.get('pkt_layers')
> +        pkt = Packet(pkt_type=pkt_type)
> +        for layer in pkt_layers.keys():
> +            pkt.config_layer(layer, pkt_layers[layer])
> +        self.logger.debug(pformat(pkt.pktgen.pkt.command()))
> +
> +        return pkt.pktgen.pkt
> +
> +    def add_stream_to_pktgen(self, txport, rxport, send_pkt, option):
> +        stream_ids = []
> +        for pkt in send_pkt:
> +            _option = deepcopy(option)
> +            _option['pcap'] = pkt
> +            # link peer 0
> +            stream_id = self.tester.pktgen.add_stream(txport, rxport, pkt)
> +            self.tester.pktgen.config_stream(stream_id, _option)
> +            stream_ids.append(stream_id)
> +            # link peer 1
> +            stream_id = self.tester.pktgen.add_stream(rxport, txport, pkt)
> +            self.tester.pktgen.config_stream(stream_id, _option)
> +            stream_ids.append(stream_id)
> +        return stream_ids
> +
> +    def send_packets_by_pktgen(self, option):
> +        txport = option.get('tx_intf')
> +        rxport = option.get('rx_intf')
> +        rate = option.get('rate', float(100))
> +        send_pkt = option.get('stream')
> +        # clear streams before add new streams
> +        self.tester.pktgen.clear_streams()
> +        # attach streams to pktgen
> +        stream_option = {
> +            'stream_config': {
> +                'txmode': {},
> +                'transmit_mode': TRANSMIT_CONT,
> +                'rate': rate, }
> +        }
> +        stream_ids = self.add_stream_to_pktgen(
> +            txport, rxport, send_pkt, stream_option)
> +        # run pktgen traffic
> +        traffic_opt = option.get('traffic_opt')
> +        self.logger.debug(traffic_opt)
> +        result = self.tester.pktgen.measure(stream_ids, traffic_opt)
> +
> +        return result
> +
> +    def run_traffic(self, option):
> +        tester_tx_port_id = self.tester.get_local_port(self.dut_ports[0])
> +        tester_rx_port_id = self.tester.get_local_port(self.dut_ports[1])
> +        ports_topo = {
> +            'tx_intf': tester_tx_port_id,
> +            'rx_intf': tester_rx_port_id,
> +            'stream': option.get('stream'),
> +            'rate': option.get('rate') or 100.0,
> +            'traffic_opt': option.get('traffic_opt'), }
> +        # begin traffic
> +        result = self.send_packets_by_pktgen(ports_topo)
> +
> +        return result
> +
> +    def init_testpmd(self):
> +        self.testpmd = PmdOutput(self.dut)
> +
> +    def start_testpmd(self, mode):
> +        table = {
> +            self.BIT_RATE: 'bitrate-stats',
> +            self.LATENCY: 'latencystats', }
> +        if mode not in table:
> +            return
> +        option = '--{0}={1}'.format(table.get(mode), self.monitor_cores)
> +        self.testpmd.start_testpmd(
> +            '1S/2C/1T',
> +            eal_param='-v',
> +            param=option)
> +        self.is_pmd_on = True
> +
> +    def set_testpmd(self):
> +        cmds = [
> +            'set fwd io',
> +            'start']
> +        [self.testpmd.execute_cmd(cmd) for cmd in cmds]
> +
> +    def close_testpmd(self):
> +        if not self.is_pmd_on:
> +            return
> +        self.testpmd.quit()
> +        self.is_pmd_on = False
> +
> +    def init_proc_info_tool(self):
> +        option = ' -v -- --metrics'
> +        self.dpdk_proc = os.path.join(
> +            self.target_dir, self.target, "app", "dpdk-procinfo" + option)
> +        self.metrics_stat = []
> +
> +    def proc_info_query(self, flag=None):
> +        msg = self.d_a_con(self.dpdk_proc)
> +        self.logger.debug(msg)
> +        portStatus = {}
> +        keys = self.display_seq.get(flag or self.BIT_RATE)
> +        curPortNo = None
> +        portPat = r"metrics for port (\d)+.*#"
> +        summary = 'non port'
> +        for item2 in msg.splitlines():
> +            item = item2.strip(os.linesep)
> +            if 'metrics' in item:
> +                curPortNo = summary \
> +                    if summary in item.lower() else \
> +                    int("".join(re.findall(portPat, item, re.M)))
> +                portStatus[curPortNo] = {}
> +            if curPortNo is None:
> +                continue
> +            if ":" in item:
> +                status = item.strip().split(': ')
> +                if len(status) == 2:
> +                    portStatus[curPortNo][status[0]] =
> int(status[1].strip())
> +        retPortStatus = {}
> +        for port_id in portStatus:
> +            retPortStatus[port_id] = {}
> +            for key in keys:
> +                retPortStatus[port_id][key] = portStatus[port_id][key]
> +        self.logger.debug(pformat(retPortStatus))
> +
> +        return retPortStatus
> +
> +    def proc_info_query_bit_rate(self):
> +        self.metrics_stat.append(self.proc_info_query(self.BIT_RATE))
> +
> +    def proc_info_query_latency(self):
> +        self.metrics_stat.append(self.proc_info_query(self.LATENCY))
> +
> +    def display_suite_result(self, data):
> +        values = data.get('values')
> +        title = data.get('title')
> +        self.result_table_create(title)
> +        for value in values:
> +            self.result_table_add(value)
> +        self.result_table_print()
> +
> +    def display_metrics_data(self, port_status, mode=None):
> +        mode = mode if mode else self.BIT_RATE
> +        display_seq = self.display_seq.get(mode)
> +        textLength = max(map(lambda x: len(x), display_seq))
> +        for port in sorted(port_status.keys()):
> +            port_value = port_status[port]
> +            if port != 'non port':
> +                self.logger.info("port {0}".format(port))
> +                for key in display_seq:
> +                    value = port_value[key]
> +                    self.logger.info("{0} = [{1}]".format(
> +                        key.ljust(textLength), value))
> +            else:
> +                maxvalue = max(map(lambda x: int(x),
> port_value.values()))
> +                if not maxvalue:
> +                    continue
> +                self.logger.info("port {0}".format(port))
> +                for key in display_seq:
> +                    value = port_value[key]
> +                    if value:
> +                        self.logger.info("{0} = [{1}]".format(
> +                            key.ljust(textLength), value))
> +
> +    def sample_bit_rate_after_stop_traffic(self, query_interval):
> +        self.logger.info("sample data after stop traffic")
> +        max_query_count = self.query_times_after_stop
> +        bit_rate_stop_results = []
> +        while max_query_count:
> +            time.sleep(query_interval)
> +
> bit_rate_stop_results.append(self.proc_info_query(self.BIT_RATE))
> +            max_query_count -= 1
> +        # get statistic after stop testpmd
> +        self.testpmd.execute_cmd('stop')
> +        stop_testpmd_results = []
> +        max_query_count = 2
> +        while max_query_count:
> +            time.sleep(query_interval)
> +
> stop_testpmd_results.append(self.proc_info_query(self.BIT_RATE))
> +            max_query_count -= 1
> +        # check metrics status
> +        first_result = stop_testpmd_results[0]
> +        second_result = stop_testpmd_results[1]
> +        if cmp(first_result, second_result) == 0:
> +            msg = "bit rate statistics stop successful after stop testpmd"
> +            self.logger.info(msg)
> +        else:
> +            msg = "bit rate statistics fail to stop after stop testpmd"
> +            self.logger.warning(msg)
> +        return bit_rate_stop_results
> +
> +    def sample_bit_rate(self, frame_size, content):
> +        duration = content.get('duration')
> +        sample_number = content.get('sample_number')
> +        query_interval = duration / sample_number
> +        # start testpmd
> +        self.testpmd.execute_cmd('start')
> +        # run traffic
> +        self.metrics_stat = []
> +        opt = {
> +            'stream': self.streams.get(frame_size),
> +            'traffic_opt': {
> +                'method': 'throughput',
> +                'duration': duration,
> +                'interval': query_interval,
> +                'callback': self.proc_info_query_bit_rate}, }
> +        result = self.run_traffic(opt)
> +        pktgen_results = [{'total': {'rx_bps': rx_pps, 'rx_pps': rx_bps}}
> +                          for rx_pps, rx_bps in result]
> +        # get data after traffic stop
> +        metrcis_stats_after_stop = self.sample_bit_rate_after_stop_traffic(
> +            query_interval)
> +        # save testing configuration
> +        sub_stats = {
> +            'pktgen_stats_on_traffic': pktgen_results,
> +            'metrics_stats_on_traffic': deepcopy(self.metrics_stat),
> +            'metrics_stats_after_traffic_stop': metrcis_stats_after_stop,
> +            'test_content': {
> +                'frame_size': frame_size,
> +                'traffic_duration': duration,
> +                'query_interval': query_interval}}
> +
> +        return sub_stats
> +
> +    def display_metrics_bit_rate(self, metrics_data):
> +        title = ['No', 'port']
> +        values = []
> +        for index, result in enumerate(metrics_data):
> +            for port, data in sorted(result.iteritems()):
> +                _value = [index, port]
> +                for key, value in data.iteritems():
> +                    if key not in title:
> +                        title.append(key)
> +                    _value.append(value)
> +                values.append(_value)
> +        metrics_data = {
> +            'title': title,
> +            'values': values}
> +        self.display_suite_result(metrics_data)
> +
> +    def calculate_bit_rate_deviation(self, pktgen_stats, metrics_stats):
> +        pktgen_bps = max([result.get('total').get('rx_bps')
> +                          for result in pktgen_stats])
> +        metrics_bps_in = 0
> +        metrics_bps_out = 0
> +        for index, result in enumerate(metrics_stats):
> +            for port_id in self.dut_ports:
> +                metrics_bps_in += result.get(port_id).get('mean_bits_in')
> +                metrics_bps_out +=
> result.get(port_id).get('mean_bits_out')
> +        mean_metrics_bps_in = metrics_bps_in / (index + 1)
> +        mean_metrics_bps_out = metrics_bps_out / (index + 1)
> +
> +        return (1.0 - float(mean_metrics_bps_in) / float(pktgen_bps),
> +                1.0 - float(mean_metrics_bps_out) / float(pktgen_bps))
> +
> +    def check_metrics_data_after_stop_traffic(self, data):
> +        # check mean_bits, it should be zero
> +        for port_id in self.dut_ports:
> +            for result in data:
> +                metrics_bps_in = result.get(port_id).get('mean_bits_in')
> +                metrics_bps_out =
> result.get(port_id).get('mean_bits_out')
> +                if metrics_bps_in or metrics_bps_out:
> +                    msg = 'mean_bits bps is not cleared as exepected'
> +                    raise VerifyFailure(msg)
> +        # check peak_bits, it should be the same
> +        for port_id in self.dut_ports:
> +            peak_bits_in = []
> +            peak_bits_out = []
> +            for result in data:
> +
> peak_bits_in.append(result.get(port_id).get('peak_bits_in'))
> +
> peak_bits_out.append(result.get(port_id).get('peak_bits_out'))
> +            if len(set(peak_bits_in)) > 1 or len(set(peak_bits_out)) > 1:
> +                msg = 'peak_bits bps is not keep the maximum value'
> +                raise VerifyFailure(msg)
> +        # check ewma_bits, it should decrease step by step
> +        for port_id in self.dut_ports:
> +            for key in ['ewma_bits_in', 'ewma_bits_out']:
> +                ewma_bits = []
> +                for result in data:
> +                    ewma_bits.append(result.get(port_id).get(key))
> +                status = [ewma_bits[index] > ewma_bits[port_id + 1]
> +                          for index in range(len(ewma_bits) - 1)]
> +                if all(status):
> +                    continue
> +                msg = 'ewma_bits bps not decrease'
> +                raise VerifyFailure(msg)
> +
> +    def check_one_bit_rate_deviation(self, data, bias):
> +        # display test content
> +        test_content = data.get('test_content')
> +        test_cfg = {
> +            'title': test_content.keys(),
> +            'values': [test_content.values()]}
> +        self.display_suite_result(test_cfg)
> +        # display pktgen bit rate statistics on traffic
> +        self.logger.info("pktgen bit rate statistics:")
> +        pktgen_results = data.get('pktgen_stats_on_traffic')
> +        self.display_metrics_bit_rate(pktgen_results)
> +        # display metrics bit rate statistics on traffic
> +        self.logger.info("dpdk metrics bit rate statistics on traffic:")
> +        metrics_results = data.get('metrics_stats_on_traffic')
> +        self.display_metrics_bit_rate(metrics_results)
> +        # check bit rate bias between packet generator and dpdk metircs
> +        in_bias, out_bias = \
> +            self.calculate_bit_rate_deviation(pktgen_results,
> metrics_results)
> +        msg = ('in bps bias is {0} '
> +               'out bps bias is {1} '
> +               'expected bias is {2}').format(in_bias, out_bias, bias)
> +        self.logger.info(msg)
> +        if in_bias > bias or out_bias > bias:
> +            msg = ('metrics mean_bits bps has more than {} bias'
> +                   'compared with pktgen bps').format(bias)
> +            raise VerifyFailure(msg)
> +        # display dpdk metrics bit rate statistics after stop traffic
> +        self.logger.info("dpdk metrics bit rate statistics when stop traffic:")
> +        metrics_results = data.get('metrics_stats_after_traffic_stop')
> +        self.display_metrics_bit_rate(metrics_results)
> +        # check metrics tool mean_bits/ewma_bits/peak_bits behavior after
> stop
> +        # traffic
> +        self.check_metrics_data_after_stop_traffic(metrics_results)
> +
> +    def sample_bit_rate_peak_after_stop_traffic(self, query_interval):
> +        self.logger.info("sample data after stop traffic")
> +        # sample data after stop
> +        max_query_count = self.query_times_after_stop
> +        bit_rate_stop_results = []
> +        while max_query_count:
> +            time.sleep(query_interval)
> +
> bit_rate_stop_results.append(self.proc_info_query(self.BIT_RATE))
> +            max_query_count -= 1
> +        # check metrics status after stop testpmd
> +        self.testpmd.execute_cmd('stop')
> +        self.logger.info("get metrics bit rate stop after stop testpmd:")
> +        stop_testpmd_results = []
> +        max_query_count = 2
> +        while max_query_count:
> +            time.sleep(query_interval)
> +
> stop_testpmd_results.append(self.proc_info_query(self.BIT_RATE))
> +            max_query_count -= 1
> +        # check metrics tool status
> +        first_result = stop_testpmd_results[0]
> +        second_result = stop_testpmd_results[1]
> +        if cmp(first_result, second_result) == 0:
> +            msg = "metrics bit rate stop successful after stop testpmd"
> +            self.logger.info(msg)
> +        else:
> +            msg = "metrics bit rate fail to stop after stop testpmd"
> +            self.logger.warning(msg)
> +        return bit_rate_stop_results
> +
> +    def sample_bit_rate_peak(self, frame_size, rate, content):
> +        duration = content.get('duration')
> +        sample_number = content.get('sample_number')
> +        query_interval = duration / sample_number
> +        # start testpmd
> +        self.testpmd.execute_cmd('start')
> +        # run traffic
> +        opt = {
> +            'stream': self.streams.get(frame_size),
> +            'rate': rate,
> +            'traffic_opt': {
> +                'method': 'throughput',
> +                'duration': duration, }, }
> +        result = self.run_traffic(opt)
> +        pktgen_results = [{'total': {'rx_bps': rx_pps, 'rx_pps': rx_bps}}
> +                          for rx_pps, rx_bps in [result]]
> +        # get data after traffic stop
> +        metrcis_stats_after_stop = \
> +            self.sample_bit_rate_peak_after_stop_traffic(query_interval)
> +        # save testing configuration
> +        sub_stats = {
> +            'pktgen_stats_on_traffic': pktgen_results,
> +            'metrics_stats_after_traffic_stop': metrcis_stats_after_stop,
> +            'test_content': {
> +                'rate': rate,
> +                'frame_size': frame_size,
> +                'traffic_duration': duration,
> +                'query_interval': query_interval}}
> +
> +        return sub_stats
> +
> +    def get_one_bit_rate_peak(self, data):
> +        # display test content
> +        test_content = data.get('test_content')
> +        test_cfg = {
> +            'title': test_content.keys(),
> +            'values': [test_content.values()]}
> +        self.display_suite_result(test_cfg)
> +        # display pktgen bit rate statistics on traffic
> +        self.logger.info("pktgen bit rate statistics :")
> +        pktgen_results = data.get('pktgen_stats_on_traffic')
> +        self.display_metrics_bit_rate(pktgen_results)
> +        pktgen_bps = max([result.get('total').get('rx_bps')
> +                          for result in pktgen_results])
> +        # display dpdk metrics bit rate statistics after stop traffic
> +        self.logger.info("dpdk bit rate statistics after stop traffic:")
> +        metrics_results = data.get('metrics_stats_after_traffic_stop')
> +        self.display_metrics_bit_rate(metrics_results)
> +        metrics_peak_data = {}
> +        for port_id in self.dut_ports:
> +            metrics_peak_data[port_id] = {
> +                'peak_bits_in':
> +                max([result.get(port_id).get('peak_bits_in')
> +                     for result in metrics_results]),
> +                'peak_bits_out':
> +                max([result.get(port_id).get('peak_bits_out')
> +                     for result in metrics_results]),
> +            }
> +        return pktgen_bps, metrics_peak_data
> +
> +    def check_bit_rate_peak_data(self, data):
> +        '''
> +        check ``peak_bits_in/peak_bits_out`` should keep the first max value
> +        when packet generator work with decreasing traffic rate percent.
> +        '''
> +        pktgen_stats = []
> +        metrics_stats = []
> +        for sub_data in data:
> +            pktgen_stat, metrics_stat =
> self.get_one_bit_rate_peak(sub_data)
> +            pktgen_stats.append(pktgen_stat)
> +            metrics_stats.append(metrics_stat)
> +        # check if traffic run with decreasing rate percent
> +        status = [pktgen_stats[index] > pktgen_stats[index + 1]
> +                  for index in range(len(pktgen_stats) - 1)]
> +        msg = 'traffic rate percent does not run with decreasing rate
> percent'
> +        self.verify(all(status), msg)
> +        # check ``peak_bits_in/peak_bits_out`` keep the first max value
> +        for port_id in self.dut_ports:
> +            for key in ['peak_bits_in', 'peak_bits_out']:
> +                peak_values = [metrics_stat.get(port_id).get(key)
> +                               for metrics_stat in metrics_stats]
> +                max_value = max(peak_values)
> +                if max_value != metrics_stats[0].get(port_id).get(key):
> +                    msg = 'port {0} {1} does not keep maximum
> value'.format(
> +                        port_id, key)
> +                    raise VerifyFailure(msg)
> +
> +    def sample_latency_after_stop_traffic(self, query_interval):
> +        self.logger.info("sample statistics after stop traffic")
> +        # sample data after stop
> +        max_query_count = self.query_times_after_stop
> +        latency_stop_results = []
> +        while max_query_count:
> +            time.sleep(5)
> +
> latency_stop_results.append(self.proc_info_query(self.LATENCY))
> +            max_query_count -= 1
> +        # check statistic status after stop testpmd
> +        self.testpmd.execute_cmd('stop')
> +        self.logger.info("query metrics latency after stop testpmd:")
> +        stop_testpmd_results = []
> +        max_query_count = 2
> +        while max_query_count:
> +            time.sleep(query_interval)
> +
> stop_testpmd_results.append(self.proc_info_query(self.LATENCY))
> +            max_query_count -= 1
> +        # check metrics behavior
> +        first_result = stop_testpmd_results[0]
> +        second_result = stop_testpmd_results[1]
> +        if cmp(first_result, second_result) == 0:
> +            msg = "metrics latency stop successful after stop testpmd"
> +            self.logger.info(msg)
> +        else:
> +            msg = "metrics latency fail to stop after stop testpmd"
> +            self.logger.warning(msg)
> +        return latency_stop_results
> +
> +    def display_metrics_latency(self, metrics_data):
> +        title = ['No', 'port']
> +        values = []
> +        for index, result in enumerate(metrics_data):
> +            for port_id, data in result.iteritems():
> +                _value = [index, port_id]
> +                for key, value in data.iteritems():
> +                    if key not in title:
> +                        title.append(key)
> +                    _value.append(value)
> +                values.append(_value)
> +        metrics_data = {
> +            'title': title,
> +            'values': values}
> +        self.display_suite_result(metrics_data)
> +
> +    def sample_latency(self, frame_size, content):
> +        self.metrics_stat = []
> +        duration = content.get('duration')
> +        sample_number = content.get('sample_number')
> +        query_interval = duration / sample_number
> +        # start testpmd
> +        self.testpmd.execute_cmd('start')
> +        # run traffic
> +        opt = {
> +            'stream': self.streams.get(frame_size),
> +            'traffic_opt': {
> +                'method': 'latency',
> +                'duration': duration, }, }
> +        pktgen_results = self.run_traffic(opt)
> +        # get data after traffic stop
> +        metrcis_stats_after_stop = self.sample_latency_after_stop_traffic(
> +            query_interval)
> +        # save testing configuration and results
> +        sub_stats = {
> +            'pktgen_stats_on_traffic': pktgen_results,
> +            'metrics_stats_after_traffic_stop': metrcis_stats_after_stop,
> +            'test_content': {
> +                'rate': 100.0,
> +                'frame_size': frame_size,
> +                'traffic_duration': duration,
> +                'query_interval': query_interval}}
> +
> +        return sub_stats
> +
> +    def check_one_latecny_data(self, data):
> +        '''
> +        packet generator calculates line latency between tx port and rx port,
> +        dpdk metrics calculates packet forward latency between rx and tx
> inside
> +        testpmd. These two types latency data are used for different
> purposes.
> +        '''
> +        # display test content
> +        test_content = data.get('test_content')
> +        test_cfg = {
> +            'title': test_content.keys(),
> +            'values': [test_content.values()]}
> +        self.display_suite_result(test_cfg)
> +        # display pktgen latency statistics on traffic
> +        self.logger.info("pktgen line latency statistics :")
> +        pktgen_results = data.get('pktgen_stats_on_traffic')
> +        self.display_metrics_latency([pktgen_results])
> +        # check if the value is reasonable, no reference data
> +        for port, value in pktgen_results.iteritems():
> +            max_value = value.get('max')
> +            min_value = value.get('min')
> +            average = value.get('average')
> +            if max_value == 0 and average == 0 and min_value == 0:
> +                msg = 'failed to get pktgen latency data'
> +                raise VerifyFailure(msg)
> +                continue
> +            if max_value > average and average > min_value and min_value >
> 0:
> +                continue
> +            msg = ('pktgen latency is wrong: '
> +                   'max <{0}> '
> +                   'average <{1}> '
> +                   'min <{2}>').format(
> +                max_value, average, min_value)
> +            raise VerifyFailure(msg)
> +        # display dpdk metrics latency statistics
> +        self.logger.info("dpdk forward latency statistics :")
> +        metrics_results = data.get('metrics_stats_after_traffic_stop')
> +        self.display_metrics_latency(metrics_results)
> +        # check if the value is reasonable, no reference data
> +        for index, result in enumerate(metrics_results):
> +            for port, value in result.iteritems():
> +                if port != 'non port':
> +                    continue
> +                max_value = value.get('max_latency_ns')
> +                min_value = value.get('min_latency_ns')
> +                average = value.get('avg_latency_ns')
> +                # ignore invalid data
> +                if max_value == 0 and average == 0 and min_value == 0:
> +                    msg = 'failed to get metrics latency data'
> +                    raise VerifyFailure(msg)
> +                if max_value > average and \
> +                   average > min_value and min_value > 0:
> +                    continue
> +                msg = ('metrics latency is wrong : '
> +                       'min_latency_ns <{0}> '
> +                       'avg_latency_ns <{1}> '
> +                       'min_latency_ns <{2}>').format(
> +                    max_value, average, min_value)
> +                raise VerifyFailure(msg)
> +        msg = 'frame_size {0} latency data is ok.'.format(
> +            test_content.get('frame_size'))
> +        self.logger.info(msg)
> +
> +    def verify_bit_rate(self):
> +        except_content = None
> +        try:
> +            # set testpmd on ready status
> +            self.start_testpmd(self.BIT_RATE)
> +            self.set_testpmd()
> +            stats = []
> +            for frame_size in self.test_content.get('frame_sizes'):
> +                sub_stats = self.sample_bit_rate(frame_size,
> self.test_content)
> +                stats.append(
> +                    [sub_stats,
> self.test_content.get('bias').get(frame_size)])
> +            for data, bias in stats:
> +                self.check_one_bit_rate_deviation(data, bias)
> +        except Exception as e:
> +            self.logger.error(traceback.format_exc())
> +            except_content = e
> +        finally:
> +            self.close_testpmd()
> +
> +        # re-raise verify exception result
> +        if except_content:
> +            raise Exception(except_content)
> +
> +    def verify_bit_rate_peak(self):
> +        except_content = None
> +        try:
> +            # set testpmd on ready status
> +            self.start_testpmd(self.BIT_RATE)
> +            self.set_testpmd()
> +            stats = []
> +            frame_sizes = self.test_content.get('frame_sizes')
> +            frame_size = frame_sizes[random.randint(0, len(frame_sizes) -
> 1)]
> +            for rate in self.test_content.get('rates'):
> +                sub_stats = self.sample_bit_rate_peak(
> +                    frame_size, rate, self.test_content)
> +                stats.append(sub_stats)
> +            self.check_bit_rate_peak_data(stats)
> +        except Exception as e:
> +            self.logger.error(traceback.format_exc())
> +            except_content = e
> +        finally:
> +            self.close_testpmd()
> +
> +        # re-raise verify exception result
> +        if except_content:
> +            raise Exception(except_content)
> +
> +    def verify_latency_stat(self):
> +        except_content = None
> +        try:
> +            # set testpmd on ready status
> +            self.start_testpmd(self.LATENCY)
> +            self.set_testpmd()
> +            # get test content
> +            stats = []
> +            for frame_size in self.test_content.get('frame_sizes'):
> +                sub_stats = self.sample_latency(frame_size,
> self.test_content)
> +                stats.append(sub_stats)
> +            for data in stats:
> +                self.check_one_latecny_data(data)
> +        except Exception as e:
> +            self.logger.error(traceback.format_exc())
> +            except_content = e
> +        finally:
> +            self.close_testpmd()
> +
> +        # re-raise verify exception result
> +        if except_content:
> +            raise Exception(except_content)
> +
> +    def verify_supported_nic(self):
> +        supported_drivers = ['ixgbe']
> +        result = all([self.dut.ports_info[port_id]['port'].default_driver in
> +                      supported_drivers
> +                      for port_id in self.dut_ports])
> +        msg = "current nic is not supported"
> +        self.verify(result, msg)
> +
> +    def get_test_content_from_cfg(self):
> +        conf = SuiteConf(self.suite_name)
> +        cfg_content = dict(conf.suite_conf.load_section('content'))
> +        frames_cfg = cfg_content.get('frames_cfg')
> +        info = [(int(item[0]), float(item[1]))
> +                for item in [item.split(':') for item in frames_cfg.split(',')]]
> +        frames_info = dict(info)
> +        test_content = {
> +            'frame_sizes': frames_info.keys(),
> +            'duration': int(cfg_content.get('duration') or 0),
> +            'sample_number': int(cfg_content.get('sample_number') or 0),
> +            'rates': [int(item)
> +                      for item in cfg_content.get('rates').split(',')],
> +            'bias': frames_info}
> +        self.query_times_after_stop = 5
> +
> +        return test_content
> +
> +    def preset_traffic(self):
> +        self.streams = {}
> +        # prepare streams instance
> +        for frame_size in self.test_content.get('frame_sizes'):
> +            self.streams[frame_size] = self.config_stream(frame_size)
> +
> +    def preset_test_environment(self):
> +        # get test content
> +        self.test_content = self.get_test_content_from_cfg()
> +        self.logger.debug(pformat(self.test_content))
> +        # binary status flag
> +        self.is_pmd_on = None
> +        # monitor cores
> +        self.monitor_cores = '2'
> +        # init binary
> +        self.init_testpmd()
> +        self.init_proc_info_tool()
> +        # traffic relevant
> +        self.preset_traffic()
> +    #
> +    # Test cases.
> +    #
> +
> +    def set_up_all(self):
> +        self.dut_ports = self.dut.get_ports(self.nic)
> +        self.verify(len(self.dut_ports) >= 2, "Not enough ports")
> +        # prepare testing environment
> +        self.preset_test_environment()
> +
> +    def tear_down_all(self):
> +        """ Run after each test suite. """
> +        pass
> +
> +    def set_up(self):
> +        """ Run before each test case. """
> +        pass
> +
> +    def tear_down(self):
> +        """ Run after each test case. """
> +        self.dut.kill_all()
> +
> +    def test_perf_bit_rate_peak(self):
> +        """
> +        Test bit rate peak
> +        """
> +        self.verify_bit_rate_peak()
> +
> +    def test_perf_bit_rate(self):
> +        """
> +        Test bit rate
> +        """
> +        self.verify_bit_rate()
> +
> +    def test_perf_latency_stat(self):
> +        """
> +        Test latency stat
> +        """
> +        self.verify_latency_stat()
> --
> 2.21.0