From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mga06.intel.com (mga06.intel.com [134.134.136.31]) by dpdk.org (Postfix) with ESMTP id 1E7C37CB6 for ; Fri, 22 Sep 2017 11:51:30 +0200 (CEST) Received: from fmsmga003.fm.intel.com ([10.253.24.29]) by orsmga104.jf.intel.com with ESMTP; 22 Sep 2017 02:51:29 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.42,427,1500966000"; d="scan'208";a="902858942" Received: from dpdk-test47.sh.intel.com ([10.67.118.152]) by FMSMGA003.fm.intel.com with ESMTP; 22 Sep 2017 02:51:29 -0700 From: wang fei To: dts@dpdk.org Cc: wang fei Date: Fri, 22 Sep 2017 20:23:55 +0800 Message-Id: <1506083035-35681-1-git-send-email-feix.y.wang@intel.com> X-Mailer: git-send-email 2.7.4 Subject: [dts] [patch v3] framework/pktgen.py: Add packet generator framework, and integrate Trex into the framework X-BeenThere: dts@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: test suite reviews and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 22 Sep 2017 09:51:31 -0000 Signed-off-by: wang fei --- framework/pktgen.py | 387 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 387 insertions(+) create mode 100644 framework/pktgen.py diff --git a/framework/pktgen.py b/framework/pktgen.py new file mode 100644 index 0000000..4ba6b25 --- /dev/null +++ b/framework/pktgen.py @@ -0,0 +1,387 @@ +# BSD LICENSE +# +# Copyright(c) 2010-2017 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import sys +import re +import string +import time +import json +import argparse +import IPy +import logging + +from abc import abstractmethod +#from trex_stl_lib.api import STLClient, STLStream, STLPktBuilder, Ether, IP, STLTXCont +from config import IxiaConf +from ssh_connection import SSHConnection +from settings import SCAPY2IXIA +from logger import getLogger +from exception import VerifyFailure +from utils import create_mask +from uuid import uuid4 +from pickletools import optimize +from tester import Tester +#from serializer import Serializer + +FORMAT = '%(message)s' +logging.basicConfig(format=FORMAT) +logger = logging.getLogger('TrexA') +logger.setLevel(logging.INFO) +# change operation directory +cwd = os.getcwd() +sys.path.append(cwd + '/nics') +sys.path.append(cwd + '/framework') +sys.path.append(cwd + '/tests') +sys.path.append(cwd + '/dep') +#sys.path.append("/opt/trex-core-2.26/scripts/automation/trex_control_plane/stl/trex_stl_lib") +sys.path.insert(0, "/opt/trex-core-2.26/scripts/automation/"+\ + "trex_control_plane/stl") +#from api import STLClient, STLStream, STLPktBuilder, Ether, IP, STLTXCont +from trex_stl_lib.api import * +from crb import Crb +from config import PktgenConf, CrbsConf +#from net_device import GetNicObj + +class PacketGenerator(object): +#class PacketGenerator(Crb): + """ + Basic class for packet generator, define basic function for each kinds of + generators + """ + def __init__(self, tester): + self.__streams = [] + self.tester = tester + + @abstractmethod + def _check_options(self, opts={}): + pass + + @abstractmethod + def prepare_generator(self): + pass + + @abstractmethod + def _prepare_transmission(self, stream_ids=[]): + pass + + @abstractmethod + def _start_transmission(self, stream_ids): + pass + + @abstractmethod + def _stop_transmission(self, stream_id): + pass + + @abstractmethod + def _retrieve_port_statistic(self, stream_id): + pass + + def add_stream(self, tx_port, rx_port, pcap_file): + stream_id = None + + stream_id = len(self.__streams) + stream = {'tx_port': tx_port, + 'rx_port': rx_port, + 'pcap_file': pcap_file} + self.__streams.append(stream) + + return stream_id + + def config_stream(self, stream_id=0, opts={}): + if self._check_options(opts) is not True: + self.logger.error("Failed to configure stream[%d]" % stream_id) + return + + stream = self.__streams[stream_id] + stream['options'] = opts + + def measure_throughput(self, stream_ids=[], delay=50): + """ + Measure throughput on each tx ports + """ + + bps_rx = [] + pps_rx = [] + self._prepare_transmission(stream_ids=stream_ids) + self._start_transmission(stream_ids) + + time.sleep(delay) + for stream_id in stream_ids: + rx_statistics = self._retrieve_port_statistic(stream_id) + + bps_rx.append(rx_statistics["rate_rx_bits"]) + pps_rx.append(rx_statistics["rate_rx_pkts"]) + self._stop_transmission(stream_id) + bps_rx_total = self._summary_statistic(bps_rx) + pps_rx_total = self._summary_statistic(pps_rx) + + print "throughput: pps_rx %f, bps_rx %f" % (pps_rx_total, bps_rx_total) + + return bps_rx_total, pps_rx_total + + def _summary_statistic(self, array=[]): + """ + Summary all values in statistic array + """ + summary = 0.000 + for value in array: + summary += value + + return summary + + def _get_stream(self, stream_id): + return self.__streams[stream_id] + + def _get_generator_conf_instance(self, pktgen_type): + conf_inst = PktgenConf(pktgen_type=pktgen_type) + return conf_inst + + @abstractmethod + def quit_generator(self): + pass + +class TrexPacketGenerator(PacketGenerator): + """ + Trex packet generator, detail usage can be seen at + https://trex-tgn.cisco.com/trex/doc/trex_manual.html + """ + def __init__(self, tester): + self._conn = None + self._ports = [] + self._traffic_ports = [] + self._transmit_streams = {} + self._trex_app = "scripts/t-rex-64" + + self._conf_inst = self._get_generator_conf_instance("trex") + self._conf = self._conf_inst.load_pktgen_config() + self._options_keys = [ 'rate', 'ip', 'vlan'] + self._ip_keys = ['start', 'end','action', 'mask', 'step'] + self._vlan_keys = ['start', 'end', 'action', 'step', 'count'] + super(TrexPacketGenerator, self).__init__(tester) + + def connect(self): + self._conn = STLClient(server=self._conf["server"]) + time.sleep(30) + self._conn.connect() + for p in self._conn.get_all_ports(): + self._ports.append(p) + + logger.debug(self._ports) + + def disconnect(self): + self._conn.disconnect() + + def _check_options(self, opts={}): + for key in opts: + if key in self._options_keys: + if key == 'ip': + ip = opts['ip'] + for ip_key in ip: + if not ip_key in self._ip_keys: + print " %s is invalid ip option" % ip_key + return False + if key == 'action': + if not ip[key] == 'inc' or not ip[key] == 'dec': + print " %s is invalid ip action" % ip[key] + return False + elif key == 'vlan': + vlan = opts['vlan'] + for vlan_key in vlan: + if not vlan_key in self._vlan_keys: + print " %s is invalid vlan option" % vlan_key + return False + if key == 'action': + if not vlan[key] == 'inc' or not ip[key] == 'dec': + print " %s is invalid vlan action" % vlan[key] + return False + else: + print " %s is invalid option" % key + return False + return True + + def _create_vm (self, ip_src_range, ip_dst_range, action='inc', step=1): + if not ip_src_range and not ip_dst_range: + return None + + vm = [] + + if ip_src_range: + vm += [STLVmFlowVar(name="src", min_value = ip_src_range['start'], max_value = ip_src_range['end'], size = 4, op = action), + STLVmWrFlowVar(fv_name="src",pkt_offset= "IP.src") + ] + + if ip_dst_range: + vm += [STLVmFlowVar(name="dst", min_value = ip_dst_range['start'], max_value = ip_dst_range['end'], size = 4, op = action), + STLVmWrFlowVar(fv_name="dst",pkt_offset = "IP.dst") + ] + + vm += [STLVmFixIpv4(offset = "IP") + ] + + return vm + + def prepare_generator(self): + app_param_temp = "-i" + + for key in self._conf: + #key, value = pktgen_conf + if key == 'config_file': + app_param_temp = app_param_temp + " --cfg " + self._conf[key] + elif key == 'core_num': + app_param_temp = app_param_temp + " -c " + self._conf[key] + + app = self._conf['trex_root_path'] + os.sep + self._trex_app + + cmd = app + " " + app_param_temp + + self.tester.send_expect("cd /opt/trex-core-2.26/scripts", "#", 70) + self.tester.send_expect(cmd, "", 40) + + time.sleep(15) + + self.connect() + + self.tester.send_expect("cd " + cwd, "#", 70) + + def _prepare_transmission(self, stream_ids=[]): + # Create base packet and pad it to size + streams = [] + ip_src_range = {} + ip_dst_range = {} + ip_src_range_temp = [] + ip_dst_range_temp = [] + + # prepare stream configuration + for stream_id in stream_ids: + stream = self._get_stream(stream_id) + tx_port = stream['tx_port'] + rx_port = stream['rx_port'] + rx_port_name = "port%d" % rx_port + option = stream['options'] + + #set rate + rate = option['rate'] + ip = option['ip'] + mask = ip['mask'] + step_temp = ip['step'].split('.') + + #get the subnet range of src and dst ip + if self._conf.has_key("ip_src"): + ip_src = self._conf['ip_src'] + ip_src_range_string = IPy.IP(IPy.IP(ip_src).make_net(mask).strNormal()).strNormal(3) + ip_src_range_temp = ip_src_range_string.split('-') + ip_src_range['start'] = ip_src_range_temp[0] + ip_src_range['end'] = ip_src_range_temp[1] + + if self._conf.has_key("ip_dst"): + ip_dst = self._conf['ip_dst'] + ip_dst_range_string = IPy.IP(IPy.IP(ip_dst).make_net(mask).strNormal()).strNormal(3) + ip_dst_range_temp = ip_dst_range_string.split('-') + ip_dst_range['start'] = ip_dst_range_temp[0] + ip_dst_range['end'] = ip_dst_range_temp[1] + + pcap_file = stream['pcap_file'] + + vm = self._create_vm(ip_src_range, ip_dst_range, action=ip['action'], step=step_temp[3]) + + stl_stream = STLStream(packet = STLPktBuilder(pkt = pcap_file, vm=vm), mode = STLTXCont(percentage=100)) + + self._transmit_streams[stream_id] = stl_stream + + def _start_transmission(self, stream_ids): + self._conn.reset(ports=self._ports) + self._conn.clear_stats() + self._conn.set_port_attr(self._ports, promiscuous=True) + duration_int = int(self._conf["duration"]) + rate = "100%" + warmup = 15 + + if self._conf.has_key("warmup"): + warmup = int(self._conf["warmup"]) + + for p in self._ports: + for stream_id in stream_ids: + stream = self._get_stream(stream_id) + if stream["tx_port"] == p: + self._conn.add_streams(self._transmit_streams[stream_id], ports=[p]) + rate = stream["options"]["rate"] + self._traffic_ports.append(p) + + if self._conf.has_key("core_mask"): + self._conn.start(ports=self._traffic_ports, mult=rate, duration=warmup, core_mask=self._conf["core_mask"]) + self._conn.wait_on_traffic(ports=self._traffic_ports, timeout=warmup+30) + else: + self._conn.start(ports=self._traffic_ports, mult=rate, duration=warmup) + self._conn.wait_on_traffic(ports=self._traffic_ports, timeout=warmup+30) + + self._conn.clear_stats() + + if self._conf.has_key("core_mask"): + self._conn.start(ports=self._traffic_ports, mult=rate, duration=duration_int, core_mask=self._conf["core_mask"]) + else: + self._conn.start(ports=self._traffic_ports, mult=rate, duration=duration_int) + + if self._conn.get_warnings(): + for warning in self._conn.get_warnings(): + logger.warn(warning) + + def _stop_transmission(self, stream_id): + self._conn.stop(ports=self._traffic_ports, rx_delay_ms=5000) + + def _retrieve_port_statistic(self, stream_id): + rx_statistics = {} + stats = self._conn.get_stats() + stream = self._get_stream(stream_id) + port_id = stream["rx_port"] + port_stats = stats[port_id] + print "Port %d stats: %s " % (port_id,port_stats) + rate_rx_pkts = port_stats["rx_pps"] + rate_rx_bits = port_stats["rx_bps_L1"] + print "rx_port: %d, rate_rx_pkts: %f, rate_rx_bits:%f " % (port_id,rate_rx_pkts,rate_rx_bits) + rx_statistics["rate_rx_bits"] = rate_rx_bits + rx_statistics["rate_rx_pkts"] = rate_rx_pkts + return rx_statistics + + def quit_generator(self): + self.disconnect() + +def getPacketGenerator(tester, pktgen_type="trex"): + """ + Get packet generator object + """ + + if pktgen_type == "dpdk": + return DpdkPacketGenerator(tester) + elif pktgen_type == "ixia": + return IxiaPacketGenerator(tester) + elif pktgen_type == "trex": + return TrexPacketGenerator(tester) -- 2.7.4