Re: [dts] [PATCH V3 3/3] ethtool_stats: add automation script

["Wang, Yinan" <yinan.wang@intel.com>]

Acked-by: Wang, Yinan <yinan.wang@intel.com>

Sorry, pls ignore below comment as it’s for another tests.

> -----Original Message-----
> From: Wang, Yinan
> Sent: 2019年11月12日 23:39
> To: Mo, YufengX <yufengx.mo@intel.com>; dts@dpdk.org
> Subject: RE: [dts][PATCH V3 3/3] ethtool_stats: add automation script
> 
> Hi Yufeng,
> 
> As discussion, may need add below points:
> #. add a callback function in pktgen_base class, execute this callback function
> and query pktgen statistics data at a custom interval.
> #. implemented automation for ethtool by using this callback function #. Add
> parameter refer test plan change
> 
> Br,
> yinan
> > -----Original Message-----
> > From: Mo, YufengX <yufengx.mo@intel.com>
> > Sent: 2019年11月1日 15:07
> > To: dts@dpdk.org; Wang, Yinan <yinan.wang@intel.com>
> > Cc: Mo, YufengX <yufengx.mo@intel.com>
> > Subject: [dts][PATCH V3 3/3] ethtool_stats: add automation script
> >
> >
> > Currently Ethtool supports a more complete list of stats for the same
> > drivers that DPDK supports. The idea behind this epic is two fold as following:
> > 1. To achieve metric equivalence with what linux's ethtool provides.
> > 2. To extend the functionality of the xstats API to enable the following
> options:
> >    ## the retrieval of aggregate stats upon request (Top level stats).
> >    ## the retrieval of the extended NIC stats.
> >    ## grouping of stats logically so they can be retrieved per logical
> grouping.
> >    ## the option to enable/disable the stats groups to retrieve similar to set
> >       private flags in ethtool.
> >
> > Signed-off-by: yufengmx <yufengx.mo@intel.com>
> > ---
> >  tests/TestSuite_ethtool_stats.py | 498
> > +++++++++++++++++++++++++++++++
> >  1 file changed, 498 insertions(+)
> >  create mode 100644 tests/TestSuite_ethtool_stats.py
> >
> > diff --git a/tests/TestSuite_ethtool_stats.py
> > b/tests/TestSuite_ethtool_stats.py
> > new file mode 100644
> > index 0000000..e8ea941
> > --- /dev/null
> > +++ b/tests/TestSuite_ethtool_stats.py
> > @@ -0,0 +1,498 @@
> > +# BSD LICENSE
> > +#
> > +# Copyright(c) 2010-2019 Intel Corporation. All rights reserved.
> > +# All rights reserved.
> > +#
> > +# Redistribution and use in source and binary forms, with or without
> > +# modification, are permitted provided that the following conditions
> > +# are met:
> > +#
> > +#   * Redistributions of source code must retain the above copyright
> > +#     notice, this list of conditions and the following disclaimer.
> > +#   * Redistributions in binary form must reproduce the above copyright
> > +#     notice, this list of conditions and the following disclaimer in
> > +#     the documentation and/or other materials provided with the
> > +#     distribution.
> > +#   * Neither the name of Intel Corporation nor the names of its
> > +#     contributors may be used to endorse or promote products derived
> > +#     from this software without specific prior written permission.
> > +#
> > +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
> > CONTRIBUTORS #
> > +'AS IS' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT #
> > +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
> > FOR #
> > +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
> > COPYRIGHT #
> > +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
> > INCIDENTAL, #
> > +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
> NOT
> > #
> > +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
> > USE, #
> > +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
> > ON ANY #
> > +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT #
> > +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
> THE
> > USE #
> > +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
> > DAMAGE.
> > +
> > +'''
> > +DPDK Test suite.
> > +Test support of dpdk-procinfo tool feature '''
> > +
> > +import re
> > +import time
> > +import os
> > +import traceback
> > +
> > +from utils import create_mask as dts_create_mask from test_case
> > +import TestCase from pmd_output import PmdOutput from exception
> > +import VerifyFailure
> > +
> > +from packet import Packet
> > +from scapy.sendrecv import sendp
> > +from settings import HEADER_SIZE
> > +
> > +
> > +class TestEthtoolStats(TestCase):
> > +
> > +    @property
> > +    def target_dir(self):
> > +        # get absolute directory of target source code
> > +        target_dir = '/root' + self.dut.base_dir[1:] \
> > +                     if self.dut.base_dir.startswith('~') else \
> > +                     self.dut.base_dir
> > +        return target_dir
> > +
> > +    def d_a_con(self, cmd):
> > +        _cmd = [cmd, '# ', 10] if isinstance(cmd, (str, unicode)) else cmd
> > +        output = self.dut.alt_session.send_expect(*_cmd)
> > +        output2 = self.dut.alt_session.session.get_session_before(1)
> > +        return output + os.linesep + output2
> > +
> > +    def send_packet(self, pkt_config, src_intf):
> > +        for pkt_type in pkt_config.keys():
> > +            pkt = Packet(pkt_type=pkt_type)
> > +            # set packet every layer's input parameters
> > +            if 'layer_configs' in pkt_config[pkt_type].keys():
> > +                pkt_configs = pkt_config[pkt_type]['layer_configs']
> > +                if pkt_configs:
> > +                    for layer in pkt_configs.keys():
> > +                        pkt.config_layer(layer, pkt_configs[layer])
> > +            pkt.send_pkt(crb=self.tester, tx_port=src_intf, count=1)
> > +            time.sleep(1)
> > +
> > +    def traffic(self):
> > +        # make sure interface in link up status
> > +        src_intf, src_mac = self.link_topo
> > +        cmd = "ifconfig {0} up".format(src_intf)
> > +        self.d_a_con(cmd)
> > +        # send out packet
> > +        for frame_size in self.frame_sizes:
> > +            headers_size = sum(
> > +                map(lambda x: HEADER_SIZE[x], ['eth', 'ip', 'udp']))
> > +            payload_size = frame_size - headers_size
> > +            config_layers = {
> > +                'ether': {'src': src_mac},
> > +                'raw':   {'payload': ['58'] * payload_size}}
> > +            pkt_config = {'UDP': {'layer_configs': config_layers}}
> > +            self.send_packet(pkt_config, src_intf)
> > +
> > +    def init_testpmd(self):
> > +        self.testpmd = PmdOutput(self.dut)
> > +        self.is_pmd_on = False
> > +
> > +    def start_testpmd(self):
> > +        self.testpmd.start_testpmd('1S/2C/1T',
> > param='--port-topology=loop')
> > +        self.is_pmd_on = True
> > +        time.sleep(2)
> > +
> > +    def set_testpmd(self):
> > +        cmds = [
> > +            'set fwd io',
> > +            'clear port xstats all',
> > +            'start']
> > +        [self.testpmd.execute_cmd(cmd) for cmd in cmds]
> > +        time.sleep(2)
> > +
> > +    def close_testpmd(self):
> > +        if not self.is_pmd_on:
> > +            return
> > +        self.testpmd.quit()
> > +        self.is_pmd_on = False
> > +
> > +    def get_pmd_xstat_data(self):
> > +        ''' get testpmd nic extended statistics '''
> > +        cmd = 'show port xstats all'
> > +        output = self.testpmd.execute_cmd(cmd)
> > +        if "statistics" not in output:
> > +            self.logger.error(output)
> > +            raise Exception("failed to get port extended statistics data")
> > +        data_str = output.splitlines()
> > +        port_xstat = {}
> > +        cur_port = None
> > +        pat = r".*extended statistics for port (\d+).*"
> > +        for line in data_str:
> > +            if not line.strip():
> > +                continue
> > +            if "statistics" in line:
> > +                result = re.findall(pat, line.strip())
> > +                if len(result):
> > +                    cur_port = result[0]
> > +            elif cur_port is not None and ": " in line:
> > +                if cur_port not in port_xstat:
> > +                    port_xstat[cur_port] = {}
> > +                result = line.strip().split(": ")
> > +                if len(result) == 2 and result[0]:
> > +                    name, value = result
> > +                    port_xstat[cur_port][name] = value
> > +                else:
> > +                    raise Exception("invalid data")
> > +
> > +        return port_xstat
> > +
> > +    def clear_pmd_ports_stat(self):
> > +        options = ["--xstats-reset ", "--stats-reset "]
> > +        for option in options:
> > +            cmd = self.dpdk_proc_info + " %s" % option
> > +            self.d_a_con(cmd)
> > +            time.sleep(1)
> > +
> > +    def init_proc_info(self):
> > +        ports_count = len(self.dut_ports)
> > +        ports_mask = reduce(lambda x, y: x | y,
> > +                            map(lambda x: 0x1 << x,
> > range(ports_count)))
> > +        self.query_tool = os.path.join(
> > +            self.target_dir, self.target, 'app', 'dpdk-procinfo')
> > +        self.dpdk_proc_info = "%s -v -- -p %s" % (self.query_tool,
> > + ports_mask)
> > +
> > +    def parse_proc_info_xstat_output(self, msg):
> > +        if "statistics" not in msg:
> > +            self.logger.error(msg)
> > +            raise VerifyFailure("get port statistics data failed")
> > +
> > +        port_xstat = {}
> > +        cur_port = None
> > +        pat = ".*for port (\d)+ .*"
> > +        data_str = msg.splitlines()
> > +        for line in data_str:
> > +            if not line.strip():
> > +                continue
> > +            if "statistics" in line:
> > +                result = re.findall(pat, line.strip())
> > +                if len(result):
> > +                    cur_port = result[0]
> > +            elif cur_port is not None and ": " in line:
> > +                if cur_port not in port_xstat:
> > +                    port_xstat[cur_port] = {}
> > +                result = line.strip().split(": ")
> > +                if len(result) == 2 and result[0]:
> > +                    name, value = result
> > +                    port_xstat[cur_port][name] = value
> > +                else:
> > +                    raise VerifyFailure("invalid data")
> > +
> > +        return port_xstat
> > +
> > +    def query_dpdk_xstat_all(self, option="xstats"):
> > +        cmd = self.dpdk_proc_info + " --%s" % (option)
> > +        output = self.d_a_con(cmd)
> > +        infos = self.parse_proc_info_xstat_output(output)
> > +        if not infos:
> > +            msg = 'get xstat data failed'
> > +            raise VerifyFailure(msg)
> > +        return infos
> > +
> > +    def get_xstat_statistic_id(self, sub_option):
> > +        option = "xstats-name"
> > +        execept_msgs = []
> > +        cmd = self.dpdk_proc_info + " --%s %s" % (option, sub_option)
> > +        msg = self.d_a_con(cmd)
> > +        sub_stat_data = self.parse_proc_info_xstat_output(msg)
> > +        if sub_option not in msg or not len(sub_stat_data):
> > +            execept_msgs.append([option, msg])
> > +        else:
> > +            for port in sub_stat_data:
> > +                if sub_option not in sub_stat_data[port]:
> > +                    msg = "{0} {1} data doesn't existed".format(
> > +                        port, sub_option)
> > +                    self.logger.error(msg)
> > +                    continue
> > +                if not port:
> > +                    msg1 = "port {0} [{1}]".format(port, sub_option)
> > +                    execept_msgs.append([msg1, msg2])
> > +                    continue
> > +        return sub_stat_data, execept_msgs
> > +
> > +    def check_single_stats_result(self, sub_stat_data, all_xstat_data):
> > +        execept_msgs = []
> > +        for port, infos in sub_stat_data.items():
> > +            for item in infos:
> > +                if not port or \
> > +                   port not in all_xstat_data or \
> > +                   item not in all_xstat_data[port] or \
> > +                   sub_stat_data[port][item] !=
> all_xstat_data[port][item]:
> > +                    msg1 = "port {0} [{1}]".format(port, item)
> > +                    msg2 = "expect {0}
> > + ".format(all_xstat_data[port][item])
> > + \
> > +                           "show
> {0}".format(sub_stat_data[port][item])
> > +                    execept_msgs.append([msg1, msg2])
> > +                    continue
> > +                msg2 = "expect {0} ".format(all_xstat_data[port][item]) +
> \
> > +                       "show {0}".format(sub_stat_data[port][item])
> > +                self.logger.info(msg2)
> > +        return execept_msgs
> > +
> > +    def get_xstat_single_statistic(self, stat, all_xstat_data):
> > +        option = "xstats-id"
> > +        execept_msgs = []
> > +        for id in stat.values():
> > +            cmd = self.dpdk_proc_info + " --%s %s" % (option, id)
> > +            msg = self.d_a_con(cmd)
> > +            sub_stat_data = self.parse_proc_info_xstat_output(msg)
> > +            if not sub_stat_data or not len(sub_stat_data):
> > +                execept_msgs.append([option, msg])
> > +            else:
> > +                execept_msgs += self.check_single_stats_result(
> > +                    sub_stat_data, all_xstat_data)
> > +            if len(execept_msgs):
> > +                for msgs in execept_msgs:
> > +                    self.logger.error(msgs[0])
> > +                    self.logger.info(msgs[1])
> > +                raise VerifyFailure("query data exception ")
> > +
> > +        self.logger.info("all port is correct")
> > +
> > +        time.sleep(1)
> > +
> > +    def check_xstat_command_list(self):
> > +        output = self.d_a_con(self.query_tool)
> > +        expected_command = [
> > +            "xstats-reset",
> > +            "xstats-name NAME",
> > +            "xstats-ids IDLIST",
> > +            "xstats-reset"]
> > +        pat = ".*--(.*):.*"
> > +        handle = re.compile(pat)
> > +        result = handle.findall(output)
> > +        if not result or len(result) == 0:
> > +            cmds = " | ".join(expected_command)
> > +            msg = "expected commands {0} have not been
> > included".format(cmds)
> > +            raise VerifyFailure(msg)
> > +        missing_cmds = []
> > +        for cmd in expected_command:
> > +            if cmd not in result:
> > +                missing_cmds.append(cmd)
> > +
> > +        if len(missing_cmds):
> > +            msg = " | ".join(missing_cmds) + " have not been included"
> > +            raise VerifyFailure(msg)
> > +
> > +        cmds = " | ".join(expected_command)
> > +        msg = "expected commands {0} have been included".format(cmds)
> > +        self.logger.info(msg)
> > +
> > +    def check_xstat_reset_status(self):
> > +        all_xstat_data = self.query_dpdk_xstat_all()
> > +        execept_msgs = []
> > +        for port in all_xstat_data:
> > +            stats_info = all_xstat_data[port]
> > +            for stat_name, value in stats_info.items():
> > +                if int(value) != 0:
> > +                    msg = "port {0} <{1}> [{2}] has not been reset"
> > +                    execept_msgs.append(msg.format(port, stat_name,
> > value))
> > +        if len(execept_msgs):
> > +            self.logger.info(os.linesep.join(execept_msgs))
> > +            raise VerifyFailure("xstat-reset failed")
> > +
> > +        self.logger.info("xstat-reset success !")
> > +
> > +    def check_xstat_id_cmd(self, all_xstat_data):
> > +        execept_msgs = []
> > +        option = "xstats-id"
> > +        sub_option = reduce(lambda x, y: str(x) + "," + str(y),
> > +                            range(len(all_xstat_data['0'].keys())))
> > +        cmd = self.dpdk_proc_info + " --%s %s" % (option, sub_option)
> > +        msg = self.d_a_con(cmd)
> > +        sub_stat_data = self.parse_proc_info_xstat_output(msg)
> > +        if not sub_stat_data or not len(sub_stat_data):
> > +            execept_msgs.append([option, msg])
> > +        else:
> > +            for port, infos in sub_stat_data.items():
> > +                for item in infos:
> > +                    if not port or \
> > +                       port not in all_xstat_data or \
> > +                       item not in all_xstat_data[port]:
> > +                        msg1 = "port {0} get [{1}] failed".format(
> > +                            port, item)
> > +                        execept_msgs.append([msg1])
> > +                        continue
> > +        if len(execept_msgs):
> > +            for msgs in execept_msgs:
> > +                self.logger.error(msgs[0])
> > +            raise VerifyFailure("query data exception ")
> > +
> > +        self.logger.info("all ports stat id can get")
> > +        time.sleep(1)
> > +
> > +    def check_xstat_name_cmd(self, all_xstat_data):
> > +        option = "xstats-name"
> > +        _sub_options = all_xstat_data['0'].keys()
> > +        execept_msgs = []
> > +        for sub_option in _sub_options:
> > +            cmd = self.dpdk_proc_info + " --%s %s" % (option, sub_option)
> > +            msg = self.d_a_con(cmd)
> > +            sub_stat_data = self.parse_proc_info_xstat_output(msg)
> > +            if sub_option not in msg or not len(sub_stat_data):
> > +                execept_msgs.append([option, msg])
> > +            else:
> > +                for port in sub_stat_data:
> > +                    if sub_option not in sub_stat_data[port]:
> > +                        msg = "{0} {1} data doesn't existed".format(
> > +                            port, sub_option)
> > +                        self.logger.error(msg)
> > +                        continue
> > +                    if not port or \
> > +                       port not in all_xstat_data or \
> > +                       sub_option not in all_xstat_data[port]:
> > +                        msg1 = "port {0} [{1}]".format(port, sub_option)
> > +                        execept_msgs.append([msg1])
> > +                        continue
> > +        if len(execept_msgs):
> > +            for msgs in execept_msgs:
> > +                self.logger.error(msgs[0])
> > +                self.logger.info(msgs[1])
> > +            raise VerifyFailure("query data exception ")
> > +
> > +        self.logger.info("all port's stat value can get")
> > +
> > +    def check_xstat_statistic_integrity(self, sub_options_ex=None):
> > +        all_xstat_data = self.query_dpdk_xstat_all()
> > +        self.check_xstat_id_cmd(all_xstat_data)
> > +        self.check_xstat_name_cmd(all_xstat_data)
> > +
> > +    def check_xstat_single_statistic(self, sub_options_ex=None):
> > +        all_xstat_data = self.get_pmd_xstat_data()
> > +        self.logger.info("total stat names [%d]" % len(all_xstat_data['0']))
> > +        for stat_name in all_xstat_data['0'].keys():
> > +            # firstly, get statistic name.
> > +            stats, execept_msgs = self.get_xstat_statistic_id(stat_name)
> > +            if len(execept_msgs):
> > +                for msgs in execept_msgs:
> > +                    self.logger.error(msgs[0])
> > +                    self.logger.info(msgs[1])
> > +                continue
> > +            self.logger.info(stat_name)
> > +            self.get_xstat_single_statistic(stats['0'],
> > + all_xstat_data)
> > +
> > +    def verify_xstat_command_options(self):
> > +        ''' test xstat command set integrity '''
> > +        except_content = None
> > +        try:
> > +            self.start_testpmd()
> > +            self.set_testpmd()
> > +            self.check_xstat_command_list()
> > +        except Exception as e:
> > +            self.logger.error(traceback.format_exc())
> > +            except_content = e
> > +        finally:
> > +            self.close_testpmd()
> > +
> > +        # re-raise verify exception result
> > +        if except_content:
> > +            raise VerifyFailure(except_content)
> > +
> > +    def verify_xstat_reset(self):
> > +        ''' test xstat-reset command '''
> > +        except_content = None
> > +        try:
> > +            self.start_testpmd()
> > +            self.set_testpmd()
> > +            self.traffic()
> > +            self.clear_pmd_ports_stat()
> > +            self.check_xstat_reset_status()
> > +        except Exception as e:
> > +            self.logger.error(traceback.format_exc())
> > +            except_content = e
> > +        finally:
> > +            self.close_testpmd()
> > +
> > +        # re-raise verify exception result
> > +        if except_content:
> > +            raise VerifyFailure(except_content)
> > +
> > +    def verify_xstat_integrity(self):
> > +        ''' test xstat command '''
> > +        except_content = None
> > +        try:
> > +            self.start_testpmd()
> > +            self.set_testpmd()
> > +            self.check_xstat_statistic_integrity()
> > +        except Exception as e:
> > +            self.logger.error(traceback.format_exc())
> > +            except_content = e
> > +        finally:
> > +            self.close_testpmd()
> > +
> > +        # re-raise verify exception result
> > +        if except_content:
> > +            raise VerifyFailure(except_content)
> > +
> > +    def verify_xstat_single_statistic(self):
> > +        except_content = None
> > +        try:
> > +            self.start_testpmd()
> > +            self.set_testpmd()
> > +            self.clear_pmd_ports_stat()
> > +            self.traffic()
> > +            self.check_xstat_single_statistic()
> > +        except Exception as e:
> > +            self.logger.error(traceback.format_exc())
> > +            except_content = e
> > +        finally:
> > +            self.close_testpmd()
> > +
> > +        # re-raise verify exception result
> > +        if except_content:
> > +            raise VerifyFailure(except_content)
> > +
> > +    def preset_test_environment(self):
> > +        self.is_pmd_on = None
> > +        # get link port pairs
> > +        port_num = 0
> > +        local_port = self.tester.get_local_port(port_num)
> > +        self.link_topo = [
> > +            self.tester.get_interface(local_port),
> > +            self.tester.get_mac(local_port)]
> > +        # set packet sizes for testing different type
> > +        self.frame_sizes = [64, 72, 128, 256, 512, 1024]
> > +        # init binary
> > +        self.init_testpmd()
> > +        self.init_proc_info()
> > +    #
> > +    # Test cases.
> > +    #
> > +
> > +    def set_up_all(self):
> > +        self.dut_ports = self.dut.get_ports(self.nic)
> > +        self.verify(len(self.dut_ports) >= 1, 'Insufficient ports')
> > +        self.preset_test_environment()
> > +
> > +    def set_up(self):
> > +        pass
> > +
> > +    def tear_down(self):
> > +        pass
> > +
> > +    def tear_down_all(self):
> > +        pass
> > +
> > +    def test_xstat(self):
> > +        ''' test xstat command set integrity '''
> > +        self.verify_xstat_command_options()
> > +
> > +    def test_xstat_integrity(self):
> > +        ''' test xstat date types '''
> > +        self.verify_xstat_integrity()
> > +
> > +    def test_xstat_reset(self):
> > +        ''' test xstat-reset command '''
> > +        self.verify_xstat_reset()
> > +
> > +    def test_xstat_single_statistic(self):
> > +        ''' test xstat single data type '''
> > +        self.verify_xstat_single_statistic()
> > --
> > 2.21.0