From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mga09.intel.com (mga09.intel.com [134.134.136.24]) by dpdk.org (Postfix) with ESMTP id D0144236 for ; Fri, 22 Sep 2017 07:42:28 +0200 (CEST) Received: from fmsmga006.fm.intel.com ([10.253.24.20]) by orsmga102.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 21 Sep 2017 22:42:27 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.42,427,1500966000"; d="scan'208";a="154737161" Received: from fmsmsx105.amr.corp.intel.com ([10.18.124.203]) by fmsmga006.fm.intel.com with ESMTP; 21 Sep 2017 22:42:27 -0700 Received: from fmsmsx118.amr.corp.intel.com (10.18.116.18) by FMSMSX105.amr.corp.intel.com (10.18.124.203) with Microsoft SMTP Server (TLS) id 14.3.319.2; Thu, 21 Sep 2017 22:42:27 -0700 Received: from shsmsx102.ccr.corp.intel.com (10.239.4.154) by fmsmsx118.amr.corp.intel.com (10.18.116.18) with Microsoft SMTP Server (TLS) id 14.3.319.2; Thu, 21 Sep 2017 22:42:26 -0700 Received: from shsmsx103.ccr.corp.intel.com ([169.254.4.213]) by shsmsx102.ccr.corp.intel.com ([169.254.2.175]) with mapi id 14.03.0319.002; Fri, 22 Sep 2017 13:42:24 +0800 From: "Liu, Yong" To: "Wang, FeiX Y" , "dts@dpdk.org" CC: "Wang, FeiX Y" Thread-Topic: [dts] [Patch V1] ramework/pktgen.py: Add packet generator framework, and integrate Trex into the framework sent this patch on behalf of chen hongli Thread-Index: AQHTMfVH/62ZS8o/AEW7OCVDedoAVKLAQDew Date: Fri, 22 Sep 2017 05:42:24 +0000 Message-ID: <86228AFD5BCD8E4EBFD2B90117B5E81E62EC9896@SHSMSX103.ccr.corp.intel.com> References: <1505909368-49832-1-git-send-email-feix.y.wang@intel.com> In-Reply-To: <1505909368-49832-1-git-send-email-feix.y.wang@intel.com> Accept-Language: zh-CN, en-US Content-Language: en-US X-MS-Has-Attach: X-MS-TNEF-Correlator: dlp-product: dlpe-windows dlp-version: 11.0.0.116 dlp-reaction: no-action x-originating-ip: [10.239.127.40] Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Subject: Re: [dts] [Patch V1] ramework/pktgen.py: Add packet generator framework, and integrate Trex into the framework sent this patch on behalf of chen hongli X-BeenThere: dts@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: test suite reviews and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 22 Sep 2017 05:42:29 -0000 Fei, Please check with pep8 first, please add description and each function. I think this patch is one of packet generator patch set. Please send with p= atch set. And also some comments are inline. Thanks, Marvin > -----Original Message----- > From: dts [mailto:dts-bounces@dpdk.org] On Behalf Of wang fei > Sent: Wednesday, September 20, 2017 8:09 PM > To: dts@dpdk.org > Cc: Wang, FeiX Y > Subject: [dts] [Patch V1] ramework/pktgen.py: Add packet generator > framework, and integrate Trex into the framework sent this patch on behal= f > of chen hongli >=20 > Signed-off-by: wang fei > --- > framework/pktgen.py | 420 > ++++++++++++++++++++++++++++++++++++++++++++++++++++ > 1 file changed, 420 insertions(+) > create mode 100644 framework/pktgen.py >=20 > diff --git a/framework/pktgen.py b/framework/pktgen.py > new file mode 100644 > index 0000000..03550bd > --- /dev/null > +++ b/framework/pktgen.py > @@ -0,0 +1,420 @@ > +# BSD LICENSE > +# > +# Copyright(c) 2010-2017 Intel Corporation. All rights reserved. > +# All rights reserved. > +# > +# Redistribution and use in source and binary forms, with or without > +# modification, are permitted provided that the following conditions > +# are met: > +# > +# * Redistributions of source code must retain the above copyright > +# notice, this list of conditions and the following disclaimer. > +# * Redistributions in binary form must reproduce the above copyright > +# notice, this list of conditions and the following disclaimer in > +# the documentation and/or other materials provided with the > +# distribution. > +# * Neither the name of Intel Corporation nor the names of its > +# contributors may be used to endorse or promote products derived > +# from this software without specific prior written permission. > +# > +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > + > +import os > +import sys > +import re > +import string > +import time > +import json > +import argparse > +import IPy > +import logging > + > +from abc import abstractmethod > +#from trex_stl_lib.api import STLClient, STLStream, STLPktBuilder, Ether= , > IP, STLTXCont > +from config import IxiaConf > +from ssh_connection import SSHConnection > +from settings import SCAPY2IXIA > +from logger import getLogger > +from exception import VerifyFailure > +from utils import create_mask > +from uuid import uuid4 > +from pickletools import optimize > +from tester import Tester > +#from serializer import Serializer > + > +FORMAT =3D '%(message)s' > +logging.basicConfig(format=3DFORMAT) > +logger =3D logging.getLogger('TrexA') > +logger.setLevel(logging.INFO) > +# change operation directory > +cwd =3D os.getcwd() > +sys.path.append(cwd + '/nics') > +sys.path.append(cwd + '/framework') > +sys.path.append(cwd + '/tests') > +sys.path.append(cwd + '/dep') > +#sys.path.append("/opt/trex-core- > 2.26/scripts/automation/trex_control_plane/stl/trex_stl_lib") > +sys.path.insert(0, "/opt/trex-core-2.26/scripts/automation/"+\ > + "trex_control_plane/stl") Please do not hardcoded directory, this folder should be configured in conf= iguration file. > +#from api import STLClient, STLStream, STLPktBuilder, Ether, IP, > STLTXCont Please remove useless code. > +from trex_stl_lib.api import * > +from crb import Crb > +from config import PktgenConf, CrbsConf > +#from net_device import GetNicObj > + > +class PacketGenerator(object): > +#class PacketGenerator(Crb): > + """ > + Basic class for packet generator, define basic function for each > kinds of > + generators > + """ > + def __init__(self, tester): > + self.__streams =3D [] > + self.tester =3D tester > + > + @abstractmethod > + def _check_options(self, opts=3D{}): > + pass > + > + @abstractmethod > + def prepare_generator(self): > + pass > + > + @abstractmethod > + def _prepare_transmission(self, stream_ids=3D[]): > + pass > + > + @abstractmethod > + def _start_transmission(self, stream_ids, delay=3D50): > + pass > + > + @abstractmethod > + def _stop_transmission(self, stream_id): > + pass > + > + @abstractmethod > + def _retrieve_port_statistic(self, stream_id): > + pass > + > + def add_stream(self, tx_port, rx_port, pcap_file): > + stream_id =3D None > + > + stream_id =3D len(self.__streams) > + stream =3D {'tx_port': tx_port, > + 'rx_port': rx_port, > + 'pcap_file': pcap_file} > + self.__streams.append(stream) > + > + return stream_id > + > + def config_stream(self, stream_id=3D0, opts=3D{}): > + if self._check_options(opts) is not True: > + self.logger.error("Failed to configure stream[%d]" % > stream_id) > + return > + > + stream =3D self.__streams[stream_id] > + stream['options'] =3D opts > + > + def measure_throughput(self, stream_ids=3D[], delay=3D50): > + """ > + Measure throughput on each tx ports > + """ > + > + bps_rx =3D [] > + pps_rx =3D [] > + self._prepare_transmission(stream_ids=3Dstream_ids) > + self._start_transmission(stream_ids) > + > + time.sleep(delay) > + for stream_id in stream_ids: > + rxbps_rates, rxpps_rates =3D > self._retrieve_port_statistic(stream_id) > + > + bps_rx.append(rxbps_rates) > + pps_rx.append(rxpps_rates) Need to separate statistic record and summary, need another loop here.=20 > + self._stop_transmission(stream_id) > + bps_rx_total =3D self._summary_statistic(bps_rx) > + pps_rx_total =3D self._summary_statistic(pps_rx) > + > + print "throughput: pps_rx %f, bps_rx %f" % (pps_rx_total, > bps_rx_total) > + > + return bps_rx_total, pps_rx_total > + > + > + > + def _summary_statistic(self, array=3D[]): > + """ > + Summary all values in statistic array > + """ > + summary =3D 0.000 > + for value in array: > + summary +=3D value > + > + > + return summary > + > + def _get_stream(self, stream_id): > + return self.__streams[stream_id] > + > + def _get_generator_conf_instance(self, pktgen_type): > + conf_inst =3D PktgenConf(pktgen_type=3Dpktgen_type) > + return conf_inst > + > + @abstractmethod > + def quit_generator(self): > + pass > + > +class TrexPacketGenerator(PacketGenerator): > + """ > + Trex packet generator, detail usage can be seen at > + https://trex-tgn.cisco.com/trex/doc/trex_manual.html > + """ > + def __init__(self, tester): > + self._conn =3D None > + self._ports =3D [] > + self._traffic_ports =3D [] > + self._transmit_streams =3D {} > + self.trex_app =3D "scripts/t-rex-64" > + > + self.conf_inst =3D self._get_generator_conf_instance("trex") > + self.conf =3D self.conf_inst.load_pktgen_config() > + self.options_keys =3D [ 'rate', 'ip', 'vlan'] > + self.ip_keys =3D ['start', 'end','action', 'mask', 'step'] > + self.vlan_keys =3D ['start', 'end', 'action', 'step', 'count'] > + super(TrexPacketGenerator, self).__init__(tester) > + > + def connect(self): Need to check whether connection is successful, if not need raise some kind= of errors. > + self._conn =3D STLClient(server=3Dself.conf["server"]) > + time.sleep(30) > + self._conn.connect() > + for p in self._conn.get_all_ports(): > + self._ports.append(p) > + > + logger.debug(self._ports) > + > + def disconnect(self): > + self._conn.disconnect() > + > + def _check_options(self, opts=3D{}): Could you implement the check function based on some kind of dict? Like: {'ip': { {'start': ip_format, 'end': ip_format, 'action': ["inc", "dec"], '= mask': mask_format, 'step': ip_format} > + for key in opts: > + if key in self.options_keys: > + if key =3D=3D 'ip': > + ip =3D opts['ip'] > + for ip_key in ip: > + if not ip_key in self.ip_keys: > + print " %s is invalid ip option" % ip_key Please use function "self.tester.logger.error" to log error. > + return False > + if key =3D=3D 'action': > + if not ip[key] =3D=3D 'inc' or not ip[key] = =3D=3D > 'dec': > + print " %s is invalid ip action" % ip[ke= y] > + return False > + elif key =3D=3D 'vlan': > + vlan =3D opts['vlan'] > + for vlan_key in vlan: > + if not vlan_key in self.vlan_keys: > + print " %s is invalid vlan option" % vlan_ke= y > + return False > + if key =3D=3D 'action': > + if not vlan[key] =3D=3D 'inc' or not ip[key]= =3D=3D > 'dec': > + print " %s is invalid vlan action" % > vlan[key] > + return False > + else: > + print " %s is invalid option" % key > + return False > + return True > + > + def create_vm (self, ip_src_range, ip_dst_range, action=3D'inc', > step=3D1): > + if not ip_src_range and not ip_dst_range: > + return None > + > + vm =3D [] > + > + if ip_src_range: > + vm +=3D [STLVmFlowVar(name=3D"src", min_value =3D > ip_src_range['start'], max_value =3D ip_src_range['end'], size =3D 4, op = =3D > action), > + STLVmWrFlowVar(fv_name=3D"src",pkt_offset=3D "IP.src"= ) > + ] > + > + if ip_dst_range: > + vm +=3D [STLVmFlowVar(name=3D"dst", min_value =3D > ip_dst_range['start'], max_value =3D ip_dst_range['end'], size =3D 4, op = =3D > action), > + STLVmWrFlowVar(fv_name=3D"dst",pkt_offset =3D "IP.dst= ") > + ] > + > + vm +=3D [STLVmFixIpv4(offset =3D "IP") > + ] > + > + return vm > + > + def prepare_generator(self): > + app_param_temp =3D "-i" > + > + for key in self.conf: > + #key, value =3D pktgen_conf > + if key =3D=3D 'config_file': > + app_param_temp =3D app_param_temp + " --cfg " + > self.conf[key] > + elif key =3D=3D 'core_num': > + app_param_temp =3D app_param_temp + " -c " + self.conf[k= ey] > + > + app =3D self.conf['trex_root_path'] + os.sep + self.trex_app > + > + cmd =3D app + " " + app_param_temp > + > + self.tester.send_expect("cd /opt/trex-core-2.26/scripts", "#", 7= 0) > + self.tester.send_expect(cmd, "", 40) If here is to start trex, need some criteria check for trex. > + Please do not change working folder, it may cause unexpected behaviors. > + time.sleep(15) > + > + self.connect() > + > + self.tester.send_expect("cd " + cwd, "#", 70) > + > + def _prepare_transmission(self, stream_ids=3D[]): > + # Create base packet and pad it to size > + streams =3D [] > + ip_src_range =3D {} > + ip_dst_range =3D {} > + ip_src_range_temp =3D [] > + ip_dst_range_temp =3D [] > + > + # prepare stream configuration > + for stream_id in stream_ids: > + stream =3D self._get_stream(stream_id) > + tx_port =3D stream['tx_port'] > + rx_port =3D stream['rx_port'] > + rx_port_name =3D "port%d" % rx_port > + option =3D stream['options'] > + > + #set rate > + rate =3D option['rate'] > + ip =3D option['ip'] > + mask =3D ip['mask'] > + step_temp =3D ip['step'].split('.') > + > + #get the subnet range of src and dst ip > + if self.conf.has_key("ip_src"): > + ip_src =3D self.conf['ip_src'] > + ip_src_range_string =3D > IPy.IP(IPy.IP(ip_src).make_net(mask).strNormal()).strNormal(3) > + ip_src_range_temp =3D ip_src_range_string.split('-') > + ip_src_range['start'] =3D ip_src_range_temp[0] > + ip_src_range['end'] =3D ip_src_range_temp[1] > + > + if self.conf.has_key("ip_dst"): > + ip_dst =3D self.conf['ip_dst'] > + ip_dst_range_string =3D > IPy.IP(IPy.IP(ip_dst).make_net(mask).strNormal()).strNormal(3) > + ip_dst_range_temp =3D ip_dst_range_string.split('-') > + ip_dst_range['start'] =3D ip_dst_range_temp[0] > + ip_dst_range['end'] =3D ip_dst_range_temp[1] > + > + pcap_file =3D stream['pcap_file'] > + > + vm =3D self.create_vm(ip_src_range, ip_dst_range, > action=3Dip['action'], step=3Dstep_temp[3]) > + Here need to check whether vm is valid. > + stl_stream =3D STLStream(packet =3D STLPktBuilder(pkt =3D pc= ap_file, > vm=3Dvm), mode =3D STLTXCont(percentage=3D100)) > + > + self._transmit_streams[stream_id] =3D stl_stream > + > + def _start_transmission(self, stream_ids, delay=3D50): > + self._conn.reset(ports=3Dself._ports) > + self._conn.clear_stats() > + self._conn.set_port_attr(self._ports, promiscuous=3DTrue) > + duration_int =3D int(self.conf["duration"]) > + rate =3D "100%" > + warmup =3D 15 > + > + if self.conf.has_key("warmup"): > + warmup =3D int(self.conf["warmup"]) > + > + for p in self._ports: > + for stream_id in stream_ids: > + stream =3D self._get_stream(stream_id) > + if stream["tx_port"] =3D=3D p: > + > self._conn.add_streams(self._transmit_streams[stream_id], ports=3D[p]) > + rate =3D stream["options"]["rate"] > + self._traffic_ports.append(p) > + > + if self.conf.has_key("core_mask"): > + self._conn.start(ports=3Dself._traffic_ports, mult=3Drate, > duration=3Dwarmup, core_mask=3Dself.conf["core_mask"]) > + self._conn.wait_on_traffic(ports=3Dself._traffic_ports, > timeout=3Dwarmup+30) > + else: > + self._conn.start(ports=3Dself._traffic_ports, mult=3Drate, > duration=3Dwarmup) > + self._conn.wait_on_traffic(ports=3Dself._traffic_ports, > timeout=3Dwarmup+30) > + > + self._conn.clear_stats() > + > + if self.conf.has_key("core_mask"): > + self._conn.start(ports=3Dself._traffic_ports, mult=3Drate, > duration=3Dduration_int, core_mask=3Dself.conf["core_mask"]) > + else: > + self._conn.start(ports=3Dself._traffic_ports, mult=3Drate, > duration=3Dduration_int) > + > + if self._conn.get_warnings(): > + for warning in self._conn.get_warnings(): > + logger.warn(warning) > + > + def _stop_transmission(self, stream_id): > + self._conn.stop(ports=3Dself._traffic_ports, rx_delay_ms=3D5000) > + > + def _retrieve_port_statistic(self, stream_id): > + stats =3D self._conn.get_stats() > + stream =3D self._get_stream(stream_id) > + port_id =3D stream["rx_port"] > + port_stats =3D stats[port_id] > + print "Port %d stats: %s " % (port_id,port_stats) > + rate_rx_pkts =3D port_stats["rx_pps"] > + rate_rx_bits =3D port_stats["rx_bps_L1"] > + print "rx_port: %d, rate_rx_pkts: %f, rate_rx_bits:%f " % > (port_id,rate_rx_pkts,rate_rx_bits) > + return rate_rx_bits, rate_rx_pkts > + > + def quit_generator(self): > + self.disconnect() > + > +def getPacketGenerator(tester, pktgen_type=3D"trex"): > + """ > + Get packet generator object > + """ > + > + if pktgen_type =3D=3D "dpdk": > + return DpdkPacketGenerator(tester) > + elif pktgen_type =3D=3D "ixia": > + return IxiaPacketGenerator(tester) > + elif pktgen_type =3D=3D "trex": > + return TrexPacketGenerator(tester) > + > + > +if __name__ =3D=3D "__main__": > + # init pktgen stream options > + options =3D { > + 'rate' : '100%', > + 'ip': {'action': 'inc', 'mask' : '255.255.255.0', 'step':'0.0.0.1'} > + } > + crbsconf =3D CrbsConf() > + crb =3D (crbsconf.load_crbs_config())[0] > + tester =3D Tester(crb, None) > + # framework initial > + trex =3D getPacketGenerator(tester, pktgen_type=3D"trex") > + > + conf_inst =3D trex._get_generator_conf_instance("trex") > + conf =3D conf_inst.load_pktgen_config() > + # prepare running environment > + trex.prepare_generator() > + > + #config stream and convert options into pktgen commands > + stream_id1 =3D trex.add_stream(0, 1, conf['pcap_file']) > + trex.config_stream(stream_id=3Dstream_id1, opts=3Doptions) > + stream_id2 =3D trex.add_stream(1, 0, conf['pcap_file']) > + trex.config_stream(stream_id=3Dstream_id2, opts=3Doptions) > + stream_id3 =3D trex.add_stream(0, 1, conf['pcap_file']) > + trex.config_stream(stream_id=3Dstream_id3, opts=3Doptions) > + stream_id4 =3D trex.add_stream(1, 0, conf['pcap_file']) > + trex.config_stream(stream_id=3Dstream_id4, opts=3Doptions) > + #pktgen.prepare_transmission(stream_ids=3D[stream_id]) > + > trex.measure_throughput(stream_ids=3D[stream_id1,stream_id2,stream_id3,st= rea > m_id4], delay=3D5) > + #trex.measure_throughput(stream_ids=3D[stream_id1,stream_id2], delay= =3D5) > + # comeback to framework > + trex.quit_generator() > -- > 2.7.4