Re: [dts] [PATCH V3 3/3] ethtool_stats: add automation script

["Wang, Yinan" <yinan.wang@intel.com>]

Hi Yufeng,

As discussion, may need add below points:
#. add a callback function in pktgen_base class, execute this callback function and query pktgen statistics data at a custom interval.
#. implemented automation for ethtool by using this callback function
#. Add parameter refer test plan change

Br,
yinan
> -----Original Message-----
> From: Mo, YufengX <yufengx.mo@intel.com>
> Sent: 2019年11月1日 15:07
> To: dts@dpdk.org; Wang, Yinan <yinan.wang@intel.com>
> Cc: Mo, YufengX <yufengx.mo@intel.com>
> Subject: [dts][PATCH V3 3/3] ethtool_stats: add automation script
> 
> 
> Currently Ethtool supports a more complete list of stats for the same drivers
> that DPDK supports. The idea behind this epic is two fold as following:
> 1. To achieve metric equivalence with what linux's ethtool provides.
> 2. To extend the functionality of the xstats API to enable the following options:
>    ## the retrieval of aggregate stats upon request (Top level stats).
>    ## the retrieval of the extended NIC stats.
>    ## grouping of stats logically so they can be retrieved per logical grouping.
>    ## the option to enable/disable the stats groups to retrieve similar to set
>       private flags in ethtool.
> 
> Signed-off-by: yufengmx <yufengx.mo@intel.com>
> ---
>  tests/TestSuite_ethtool_stats.py | 498
> +++++++++++++++++++++++++++++++
>  1 file changed, 498 insertions(+)
>  create mode 100644 tests/TestSuite_ethtool_stats.py
> 
> diff --git a/tests/TestSuite_ethtool_stats.py b/tests/TestSuite_ethtool_stats.py
> new file mode 100644
> index 0000000..e8ea941
> --- /dev/null
> +++ b/tests/TestSuite_ethtool_stats.py
> @@ -0,0 +1,498 @@
> +# BSD LICENSE
> +#
> +# Copyright(c) 2010-2019 Intel Corporation. All rights reserved.
> +# All rights reserved.
> +#
> +# Redistribution and use in source and binary forms, with or without #
> +modification, are permitted provided that the following conditions #
> +are met:
> +#
> +#   * Redistributions of source code must retain the above copyright
> +#     notice, this list of conditions and the following disclaimer.
> +#   * Redistributions in binary form must reproduce the above copyright
> +#     notice, this list of conditions and the following disclaimer in
> +#     the documentation and/or other materials provided with the
> +#     distribution.
> +#   * Neither the name of Intel Corporation nor the names of its
> +#     contributors may be used to endorse or promote products derived
> +#     from this software without specific prior written permission.
> +#
> +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
> CONTRIBUTORS #
> +'AS IS' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT #
> +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
> FOR #
> +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
> COPYRIGHT #
> +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
> INCIDENTAL, #
> +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> #
> +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
> USE, #
> +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
> ON ANY #
> +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT #
> +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
> USE #
> +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
> DAMAGE.
> +
> +'''
> +DPDK Test suite.
> +Test support of dpdk-procinfo tool feature '''
> +
> +import re
> +import time
> +import os
> +import traceback
> +
> +from utils import create_mask as dts_create_mask from test_case import
> +TestCase from pmd_output import PmdOutput from exception import
> +VerifyFailure
> +
> +from packet import Packet
> +from scapy.sendrecv import sendp
> +from settings import HEADER_SIZE
> +
> +
> +class TestEthtoolStats(TestCase):
> +
> +    @property
> +    def target_dir(self):
> +        # get absolute directory of target source code
> +        target_dir = '/root' + self.dut.base_dir[1:] \
> +                     if self.dut.base_dir.startswith('~') else \
> +                     self.dut.base_dir
> +        return target_dir
> +
> +    def d_a_con(self, cmd):
> +        _cmd = [cmd, '# ', 10] if isinstance(cmd, (str, unicode)) else cmd
> +        output = self.dut.alt_session.send_expect(*_cmd)
> +        output2 = self.dut.alt_session.session.get_session_before(1)
> +        return output + os.linesep + output2
> +
> +    def send_packet(self, pkt_config, src_intf):
> +        for pkt_type in pkt_config.keys():
> +            pkt = Packet(pkt_type=pkt_type)
> +            # set packet every layer's input parameters
> +            if 'layer_configs' in pkt_config[pkt_type].keys():
> +                pkt_configs = pkt_config[pkt_type]['layer_configs']
> +                if pkt_configs:
> +                    for layer in pkt_configs.keys():
> +                        pkt.config_layer(layer, pkt_configs[layer])
> +            pkt.send_pkt(crb=self.tester, tx_port=src_intf, count=1)
> +            time.sleep(1)
> +
> +    def traffic(self):
> +        # make sure interface in link up status
> +        src_intf, src_mac = self.link_topo
> +        cmd = "ifconfig {0} up".format(src_intf)
> +        self.d_a_con(cmd)
> +        # send out packet
> +        for frame_size in self.frame_sizes:
> +            headers_size = sum(
> +                map(lambda x: HEADER_SIZE[x], ['eth', 'ip', 'udp']))
> +            payload_size = frame_size - headers_size
> +            config_layers = {
> +                'ether': {'src': src_mac},
> +                'raw':   {'payload': ['58'] * payload_size}}
> +            pkt_config = {'UDP': {'layer_configs': config_layers}}
> +            self.send_packet(pkt_config, src_intf)
> +
> +    def init_testpmd(self):
> +        self.testpmd = PmdOutput(self.dut)
> +        self.is_pmd_on = False
> +
> +    def start_testpmd(self):
> +        self.testpmd.start_testpmd('1S/2C/1T',
> param='--port-topology=loop')
> +        self.is_pmd_on = True
> +        time.sleep(2)
> +
> +    def set_testpmd(self):
> +        cmds = [
> +            'set fwd io',
> +            'clear port xstats all',
> +            'start']
> +        [self.testpmd.execute_cmd(cmd) for cmd in cmds]
> +        time.sleep(2)
> +
> +    def close_testpmd(self):
> +        if not self.is_pmd_on:
> +            return
> +        self.testpmd.quit()
> +        self.is_pmd_on = False
> +
> +    def get_pmd_xstat_data(self):
> +        ''' get testpmd nic extended statistics '''
> +        cmd = 'show port xstats all'
> +        output = self.testpmd.execute_cmd(cmd)
> +        if "statistics" not in output:
> +            self.logger.error(output)
> +            raise Exception("failed to get port extended statistics data")
> +        data_str = output.splitlines()
> +        port_xstat = {}
> +        cur_port = None
> +        pat = r".*extended statistics for port (\d+).*"
> +        for line in data_str:
> +            if not line.strip():
> +                continue
> +            if "statistics" in line:
> +                result = re.findall(pat, line.strip())
> +                if len(result):
> +                    cur_port = result[0]
> +            elif cur_port is not None and ": " in line:
> +                if cur_port not in port_xstat:
> +                    port_xstat[cur_port] = {}
> +                result = line.strip().split(": ")
> +                if len(result) == 2 and result[0]:
> +                    name, value = result
> +                    port_xstat[cur_port][name] = value
> +                else:
> +                    raise Exception("invalid data")
> +
> +        return port_xstat
> +
> +    def clear_pmd_ports_stat(self):
> +        options = ["--xstats-reset ", "--stats-reset "]
> +        for option in options:
> +            cmd = self.dpdk_proc_info + " %s" % option
> +            self.d_a_con(cmd)
> +            time.sleep(1)
> +
> +    def init_proc_info(self):
> +        ports_count = len(self.dut_ports)
> +        ports_mask = reduce(lambda x, y: x | y,
> +                            map(lambda x: 0x1 << x,
> range(ports_count)))
> +        self.query_tool = os.path.join(
> +            self.target_dir, self.target, 'app', 'dpdk-procinfo')
> +        self.dpdk_proc_info = "%s -v -- -p %s" % (self.query_tool,
> + ports_mask)
> +
> +    def parse_proc_info_xstat_output(self, msg):
> +        if "statistics" not in msg:
> +            self.logger.error(msg)
> +            raise VerifyFailure("get port statistics data failed")
> +
> +        port_xstat = {}
> +        cur_port = None
> +        pat = ".*for port (\d)+ .*"
> +        data_str = msg.splitlines()
> +        for line in data_str:
> +            if not line.strip():
> +                continue
> +            if "statistics" in line:
> +                result = re.findall(pat, line.strip())
> +                if len(result):
> +                    cur_port = result[0]
> +            elif cur_port is not None and ": " in line:
> +                if cur_port not in port_xstat:
> +                    port_xstat[cur_port] = {}
> +                result = line.strip().split(": ")
> +                if len(result) == 2 and result[0]:
> +                    name, value = result
> +                    port_xstat[cur_port][name] = value
> +                else:
> +                    raise VerifyFailure("invalid data")
> +
> +        return port_xstat
> +
> +    def query_dpdk_xstat_all(self, option="xstats"):
> +        cmd = self.dpdk_proc_info + " --%s" % (option)
> +        output = self.d_a_con(cmd)
> +        infos = self.parse_proc_info_xstat_output(output)
> +        if not infos:
> +            msg = 'get xstat data failed'
> +            raise VerifyFailure(msg)
> +        return infos
> +
> +    def get_xstat_statistic_id(self, sub_option):
> +        option = "xstats-name"
> +        execept_msgs = []
> +        cmd = self.dpdk_proc_info + " --%s %s" % (option, sub_option)
> +        msg = self.d_a_con(cmd)
> +        sub_stat_data = self.parse_proc_info_xstat_output(msg)
> +        if sub_option not in msg or not len(sub_stat_data):
> +            execept_msgs.append([option, msg])
> +        else:
> +            for port in sub_stat_data:
> +                if sub_option not in sub_stat_data[port]:
> +                    msg = "{0} {1} data doesn't existed".format(
> +                        port, sub_option)
> +                    self.logger.error(msg)
> +                    continue
> +                if not port:
> +                    msg1 = "port {0} [{1}]".format(port, sub_option)
> +                    execept_msgs.append([msg1, msg2])
> +                    continue
> +        return sub_stat_data, execept_msgs
> +
> +    def check_single_stats_result(self, sub_stat_data, all_xstat_data):
> +        execept_msgs = []
> +        for port, infos in sub_stat_data.items():
> +            for item in infos:
> +                if not port or \
> +                   port not in all_xstat_data or \
> +                   item not in all_xstat_data[port] or \
> +                   sub_stat_data[port][item] != all_xstat_data[port][item]:
> +                    msg1 = "port {0} [{1}]".format(port, item)
> +                    msg2 = "expect {0} ".format(all_xstat_data[port][item])
> + \
> +                           "show {0}".format(sub_stat_data[port][item])
> +                    execept_msgs.append([msg1, msg2])
> +                    continue
> +                msg2 = "expect {0} ".format(all_xstat_data[port][item]) + \
> +                       "show {0}".format(sub_stat_data[port][item])
> +                self.logger.info(msg2)
> +        return execept_msgs
> +
> +    def get_xstat_single_statistic(self, stat, all_xstat_data):
> +        option = "xstats-id"
> +        execept_msgs = []
> +        for id in stat.values():
> +            cmd = self.dpdk_proc_info + " --%s %s" % (option, id)
> +            msg = self.d_a_con(cmd)
> +            sub_stat_data = self.parse_proc_info_xstat_output(msg)
> +            if not sub_stat_data or not len(sub_stat_data):
> +                execept_msgs.append([option, msg])
> +            else:
> +                execept_msgs += self.check_single_stats_result(
> +                    sub_stat_data, all_xstat_data)
> +            if len(execept_msgs):
> +                for msgs in execept_msgs:
> +                    self.logger.error(msgs[0])
> +                    self.logger.info(msgs[1])
> +                raise VerifyFailure("query data exception ")
> +
> +        self.logger.info("all port is correct")
> +
> +        time.sleep(1)
> +
> +    def check_xstat_command_list(self):
> +        output = self.d_a_con(self.query_tool)
> +        expected_command = [
> +            "xstats-reset",
> +            "xstats-name NAME",
> +            "xstats-ids IDLIST",
> +            "xstats-reset"]
> +        pat = ".*--(.*):.*"
> +        handle = re.compile(pat)
> +        result = handle.findall(output)
> +        if not result or len(result) == 0:
> +            cmds = " | ".join(expected_command)
> +            msg = "expected commands {0} have not been
> included".format(cmds)
> +            raise VerifyFailure(msg)
> +        missing_cmds = []
> +        for cmd in expected_command:
> +            if cmd not in result:
> +                missing_cmds.append(cmd)
> +
> +        if len(missing_cmds):
> +            msg = " | ".join(missing_cmds) + " have not been included"
> +            raise VerifyFailure(msg)
> +
> +        cmds = " | ".join(expected_command)
> +        msg = "expected commands {0} have been included".format(cmds)
> +        self.logger.info(msg)
> +
> +    def check_xstat_reset_status(self):
> +        all_xstat_data = self.query_dpdk_xstat_all()
> +        execept_msgs = []
> +        for port in all_xstat_data:
> +            stats_info = all_xstat_data[port]
> +            for stat_name, value in stats_info.items():
> +                if int(value) != 0:
> +                    msg = "port {0} <{1}> [{2}] has not been reset"
> +                    execept_msgs.append(msg.format(port, stat_name,
> value))
> +        if len(execept_msgs):
> +            self.logger.info(os.linesep.join(execept_msgs))
> +            raise VerifyFailure("xstat-reset failed")
> +
> +        self.logger.info("xstat-reset success !")
> +
> +    def check_xstat_id_cmd(self, all_xstat_data):
> +        execept_msgs = []
> +        option = "xstats-id"
> +        sub_option = reduce(lambda x, y: str(x) + "," + str(y),
> +                            range(len(all_xstat_data['0'].keys())))
> +        cmd = self.dpdk_proc_info + " --%s %s" % (option, sub_option)
> +        msg = self.d_a_con(cmd)
> +        sub_stat_data = self.parse_proc_info_xstat_output(msg)
> +        if not sub_stat_data or not len(sub_stat_data):
> +            execept_msgs.append([option, msg])
> +        else:
> +            for port, infos in sub_stat_data.items():
> +                for item in infos:
> +                    if not port or \
> +                       port not in all_xstat_data or \
> +                       item not in all_xstat_data[port]:
> +                        msg1 = "port {0} get [{1}] failed".format(
> +                            port, item)
> +                        execept_msgs.append([msg1])
> +                        continue
> +        if len(execept_msgs):
> +            for msgs in execept_msgs:
> +                self.logger.error(msgs[0])
> +            raise VerifyFailure("query data exception ")
> +
> +        self.logger.info("all ports stat id can get")
> +        time.sleep(1)
> +
> +    def check_xstat_name_cmd(self, all_xstat_data):
> +        option = "xstats-name"
> +        _sub_options = all_xstat_data['0'].keys()
> +        execept_msgs = []
> +        for sub_option in _sub_options:
> +            cmd = self.dpdk_proc_info + " --%s %s" % (option, sub_option)
> +            msg = self.d_a_con(cmd)
> +            sub_stat_data = self.parse_proc_info_xstat_output(msg)
> +            if sub_option not in msg or not len(sub_stat_data):
> +                execept_msgs.append([option, msg])
> +            else:
> +                for port in sub_stat_data:
> +                    if sub_option not in sub_stat_data[port]:
> +                        msg = "{0} {1} data doesn't existed".format(
> +                            port, sub_option)
> +                        self.logger.error(msg)
> +                        continue
> +                    if not port or \
> +                       port not in all_xstat_data or \
> +                       sub_option not in all_xstat_data[port]:
> +                        msg1 = "port {0} [{1}]".format(port, sub_option)
> +                        execept_msgs.append([msg1])
> +                        continue
> +        if len(execept_msgs):
> +            for msgs in execept_msgs:
> +                self.logger.error(msgs[0])
> +                self.logger.info(msgs[1])
> +            raise VerifyFailure("query data exception ")
> +
> +        self.logger.info("all port's stat value can get")
> +
> +    def check_xstat_statistic_integrity(self, sub_options_ex=None):
> +        all_xstat_data = self.query_dpdk_xstat_all()
> +        self.check_xstat_id_cmd(all_xstat_data)
> +        self.check_xstat_name_cmd(all_xstat_data)
> +
> +    def check_xstat_single_statistic(self, sub_options_ex=None):
> +        all_xstat_data = self.get_pmd_xstat_data()
> +        self.logger.info("total stat names [%d]" % len(all_xstat_data['0']))
> +        for stat_name in all_xstat_data['0'].keys():
> +            # firstly, get statistic name.
> +            stats, execept_msgs = self.get_xstat_statistic_id(stat_name)
> +            if len(execept_msgs):
> +                for msgs in execept_msgs:
> +                    self.logger.error(msgs[0])
> +                    self.logger.info(msgs[1])
> +                continue
> +            self.logger.info(stat_name)
> +            self.get_xstat_single_statistic(stats['0'], all_xstat_data)
> +
> +    def verify_xstat_command_options(self):
> +        ''' test xstat command set integrity '''
> +        except_content = None
> +        try:
> +            self.start_testpmd()
> +            self.set_testpmd()
> +            self.check_xstat_command_list()
> +        except Exception as e:
> +            self.logger.error(traceback.format_exc())
> +            except_content = e
> +        finally:
> +            self.close_testpmd()
> +
> +        # re-raise verify exception result
> +        if except_content:
> +            raise VerifyFailure(except_content)
> +
> +    def verify_xstat_reset(self):
> +        ''' test xstat-reset command '''
> +        except_content = None
> +        try:
> +            self.start_testpmd()
> +            self.set_testpmd()
> +            self.traffic()
> +            self.clear_pmd_ports_stat()
> +            self.check_xstat_reset_status()
> +        except Exception as e:
> +            self.logger.error(traceback.format_exc())
> +            except_content = e
> +        finally:
> +            self.close_testpmd()
> +
> +        # re-raise verify exception result
> +        if except_content:
> +            raise VerifyFailure(except_content)
> +
> +    def verify_xstat_integrity(self):
> +        ''' test xstat command '''
> +        except_content = None
> +        try:
> +            self.start_testpmd()
> +            self.set_testpmd()
> +            self.check_xstat_statistic_integrity()
> +        except Exception as e:
> +            self.logger.error(traceback.format_exc())
> +            except_content = e
> +        finally:
> +            self.close_testpmd()
> +
> +        # re-raise verify exception result
> +        if except_content:
> +            raise VerifyFailure(except_content)
> +
> +    def verify_xstat_single_statistic(self):
> +        except_content = None
> +        try:
> +            self.start_testpmd()
> +            self.set_testpmd()
> +            self.clear_pmd_ports_stat()
> +            self.traffic()
> +            self.check_xstat_single_statistic()
> +        except Exception as e:
> +            self.logger.error(traceback.format_exc())
> +            except_content = e
> +        finally:
> +            self.close_testpmd()
> +
> +        # re-raise verify exception result
> +        if except_content:
> +            raise VerifyFailure(except_content)
> +
> +    def preset_test_environment(self):
> +        self.is_pmd_on = None
> +        # get link port pairs
> +        port_num = 0
> +        local_port = self.tester.get_local_port(port_num)
> +        self.link_topo = [
> +            self.tester.get_interface(local_port),
> +            self.tester.get_mac(local_port)]
> +        # set packet sizes for testing different type
> +        self.frame_sizes = [64, 72, 128, 256, 512, 1024]
> +        # init binary
> +        self.init_testpmd()
> +        self.init_proc_info()
> +    #
> +    # Test cases.
> +    #
> +
> +    def set_up_all(self):
> +        self.dut_ports = self.dut.get_ports(self.nic)
> +        self.verify(len(self.dut_ports) >= 1, 'Insufficient ports')
> +        self.preset_test_environment()
> +
> +    def set_up(self):
> +        pass
> +
> +    def tear_down(self):
> +        pass
> +
> +    def tear_down_all(self):
> +        pass
> +
> +    def test_xstat(self):
> +        ''' test xstat command set integrity '''
> +        self.verify_xstat_command_options()
> +
> +    def test_xstat_integrity(self):
> +        ''' test xstat date types '''
> +        self.verify_xstat_integrity()
> +
> +    def test_xstat_reset(self):
> +        ''' test xstat-reset command '''
> +        self.verify_xstat_reset()
> +
> +    def test_xstat_single_statistic(self):
> +        ''' test xstat single data type '''
> +        self.verify_xstat_single_statistic()
> --
> 2.21.0