From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from dpdk.org (dpdk.org [92.243.14.124]) by dpdk.space (Postfix) with ESMTP id 6E671A0679 for ; Sun, 28 Apr 2019 04:45:43 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id D75A21B55F; Sun, 28 Apr 2019 04:45:42 +0200 (CEST) Received: from mga14.intel.com (mga14.intel.com [192.55.52.115]) by dpdk.org (Postfix) with ESMTP id A6FD81B54D for ; Sun, 28 Apr 2019 04:45:40 +0200 (CEST) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga007.fm.intel.com ([10.253.24.52]) by fmsmga103.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 27 Apr 2019 19:45:39 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.60,403,1549958400"; d="scan'208";a="146438609" Received: from itecstvdts01.sh.intel.com ([10.67.111.114]) by fmsmga007.fm.intel.com with ESMTP; 27 Apr 2019 19:45:39 -0700 From: yufengmx To: dts@dpdk.org Cc: yufengmx Date: Sun, 28 Apr 2019 10:49:08 +0800 Message-Id: <1556419751-41723-12-git-send-email-yufengx.mo@intel.com> X-Mailer: git-send-email 1.9.3 In-Reply-To: <1556419751-41723-1-git-send-email-yufengx.mo@intel.com> References: <1556419751-41723-1-git-send-email-yufengx.mo@intel.com> Subject: [dts] [next][PATCH V1 1/14] framework/pktgen: pktgen instance creation and helper X-BeenThere: dts@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: test suite reviews and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dts-bounces@dpdk.org Sender: "dts" class pktgen instance creation and helper class Initialize packet generator instance with custom generator name. Create PacketGeneratorHelper class to meet with the simple packet generator usage scenario and it is compatible with old coding style in suite source code. All suite source code should import pktgen objects from this module. Signed-off-by: yufengmx --- framework/pktgen.py | 610 +++++++++++++++------------------------------------- 1 file changed, 172 insertions(+), 438 deletions(-) diff --git a/framework/pktgen.py b/framework/pktgen.py index 5294ff8..1c8acac 100644 --- a/framework/pktgen.py +++ b/framework/pktgen.py @@ -1,6 +1,6 @@ # BSD LICENSE # -# Copyright(c) 2010-2017 Intel Corporation. All rights reserved. +# Copyright(c) 2010-2019 Intel Corporation. All rights reserved. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -29,459 +29,193 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import os -import sys -import re -import string -import time -import json -import argparse -import IPy -import logging +import os, sys +from copy import deepcopy -from abc import abstractmethod -from config import IxiaConf -from ssh_connection import SSHConnection -from settings import SCAPY2IXIA -from logger import getLogger -from exception import VerifyFailure -from utils import create_mask -from uuid import uuid4 -from pickletools import optimize -#from serializer import Serializer +from scapy.all import conf +from scapy.packet import NoPayload +from scapy.packet import Packet as scapyPacket +from scapy.fields import ConditionalField +from scapy.utils import rdpcap -FORMAT = '%(message)s' -logging.basicConfig(format=FORMAT) -logger = logging.getLogger('TrexA') -logger.setLevel(logging.INFO) -# change operation directory +# import dts libs cwd = os.getcwd() sys.path.append(cwd + '/nics') sys.path.append(cwd + '/framework') sys.path.append(cwd + '/tests') sys.path.append(cwd + '/dep') -from crb import Crb -from config import PktgenConf, CrbsConf, PortConf - - -class PacketGenerator(object): -#class PacketGenerator(Crb): - """ - Basic class for packet generator, define basic function for each kinds of - generators - """ - def __init__(self, tester): - self.__streams = [] - self._ports_map = [] - self.tester = tester - - @abstractmethod - def _check_options(self, opts={}): - pass - - def prepare_generator(self): - self._prepare_generator() - - # extened tester port map and self port map - ports = self._get_ports() - print ports - tester_portnum = len(self.tester.ports_info) - for port_idx in range(len(ports)): - port_info = {'type': '%s' % self.pktgen_type, 'pci': '%s' % ports[port_idx]} - self._ports_map.append(tester_portnum + port_idx) - self.tester.ports_info.append(port_info) - print self._ports_map - # update dut port map - portconf = PortConf() - for dut in self.tester.duts: - dut.map_available_ports() - - def _convert_pktgen_port(self, port_id): - try: - port = self._ports_map[port_id] - except: - port = -1 - - return port - - def _convert_tester_port(self, port_id): - try: - port = self._ports_map.index(port_id) - except: - port = -1 - - return port - - @abstractmethod - def _prepare_transmission(self, stream_ids=[]): - pass - - @abstractmethod - def _start_transmission(self, stream_ids, delay=50): - pass - - @abstractmethod - def _stop_transmission(self, stream_id): - pass - - @abstractmethod - def _retrieve_port_statistic(self, stream_id): - pass - - def add_stream(self, tx_port, rx_port, pcap_file): - stream_id = None - - pktgen_tx_port = self._convert_tester_port(tx_port) - pktgen_rx_port = self._convert_tester_port(rx_port) - - stream_id = len(self.__streams) - stream = {'tx_port': pktgen_tx_port, - 'rx_port': pktgen_rx_port, - 'pcap_file': pcap_file} - self.__streams.append(stream) - - return stream_id - - def config_stream(self, stream_id=0, opts={}): - if self._check_options(opts) is not True: - self.logger.error("Failed to configure stream[%d]" % stream_id) +# dts libs +from utils import (convert_int2ip, convert_ip2int, + convert_mac2long, convert_mac2str) + +from pktgen_base import (PKTGEN_DPDK, PKTGEN_TREX, PKTGEN_IXIA, + TRANSMIT_CONT, TRANSMIT_M_BURST, TRANSMIT_S_BURST) +from pktgen_base import DpdkPacketGenerator +from pktgen_ixia import IxiaPacketGenerator +from pktgen_trex import TrexPacketGenerator + + +class PacketGeneratorHelper(object): + ''' default packet generator stream option for all streams ''' + default_opt = { + 'stream_config':{ + 'txmode' : {}, + 'transmit_mode': TRANSMIT_CONT, + # for temporary usage because current pktgen design don't support + # port level configuration, here using stream configuration to pass + # rate percent + 'rate': 100,}} + + def __init__(self): + self.packetLayers = dict() + + def _parse_packet_layer(self, pkt_object): + ''' parse one packet every layers' fields and value ''' + if pkt_object == None: return - stream = self.__streams[stream_id] - stream['options'] = opts - - def measure_throughput(self, stream_ids=[], delay=50): - """ - Measure throughput on each tx ports - """ - - bps_rx = [] - pps_rx = [] - self._prepare_transmission(stream_ids=stream_ids) - self._start_transmission(stream_ids) - - time.sleep(delay) - used_rx_port = [] - for stream_id in stream_ids: - if self.__streams[stream_id]['rx_port'] not in used_rx_port: - rxbps_rates, rxpps_rates = self._retrieve_port_statistic(stream_id) - used_rx_port.append(self.__streams[stream_id]['rx_port']) - bps_rx.append(rxbps_rates) - pps_rx.append(rxpps_rates) - self._stop_transmission(stream_id) - bps_rx_total = self._summary_statistic(bps_rx) - pps_rx_total = self._summary_statistic(pps_rx) - - print "throughput: pps_rx %f, bps_rx %f" % (pps_rx_total, bps_rx_total) - - return bps_rx_total, pps_rx_total - - def _summary_statistic(self, array=[]): - """ - Summary all values in statistic array - """ - summary = 0.000 - for value in array: - summary += value - - return summary - - def _get_stream(self, stream_id): - return self.__streams[stream_id] - - def _get_generator_conf_instance(self): - conf_inst = PktgenConf(self.pktgen_type) - return conf_inst - - @abstractmethod - def quit_generator(self): - pass - -class TrexPacketGenerator(PacketGenerator): - """ - Trex packet generator, detail usage can be seen at - https://trex-tgn.cisco.com/trex/doc/trex_manual.html - """ - def __init__(self, tester): - self.pktgen_type = "trex" - self._conn = None - self.control_session = None - self._ports = [] - self._traffic_ports = [] - self._transmit_streams = {} - self.trex_app = "scripts/t-rex-64" - - self.conf_inst = self._get_generator_conf_instance() - self.conf = self.conf_inst.load_pktgen_config() - self.options_keys = [ 'rate', 'ip', 'vlan'] - self.ip_keys = ['start', 'end','action', 'mask', 'step'] - self.vlan_keys = ['start', 'end', 'action', 'step', 'count'] - super(TrexPacketGenerator, self).__init__(tester) - - def connect(self): - self._conn = self.trex_client(server=self.conf["server"]) - self._conn.connect() - for p in self._conn.get_all_ports(): - self._ports.append(p) - - logger.debug(self._ports) - - def _get_ports(self): - """ - Return self ports information - """ - ports = [] - for idx in range(len(self._ports)): - ports.append('TREX:%d' % idx) - return ports - - def disconnect(self): - self._conn.disconnect() - - def _check_options(self, opts={}): - for key in opts: - if key in self.options_keys: - if key == 'ip': - ip = opts['ip'] - for ip_key in ip: - if not ip_key in self.ip_keys: - print " %s is invalid ip option" % ip_key - return False - if key == 'action': - if not ip[key] == 'inc' or not ip[key] == 'dec': - print " %s is invalid ip action" % ip[key] - return False - elif key == 'vlan': - vlan = opts['vlan'] - for vlan_key in vlan: - if not vlan_key in self.vlan_keys: - print " %s is invalid vlan option" % vlan_key - return False - if key == 'action': - if not vlan[key] == 'inc' or not ip[key] == 'dec': - print " %s is invalid vlan action" % vlan[key] - return False - else: - print " %s is invalid option" % key - return False - return True - - def create_vm (self, ip_src_range, ip_dst_range, action='inc', step=1): - if not ip_src_range and not ip_dst_range: - return None - - vm = [] - - if ip_src_range: - vm += [self.trex_vm_flow(name="src", min_value = ip_src_range['start'], max_value = ip_src_range['end'], size = 4, op = action), - self.trex_vm_wr_flow(fv_name="src",pkt_offset= "IP.src") - ] - - if ip_dst_range: - vm += [self.trex_vm_flow(name="dst", min_value = ip_dst_range['start'], max_value = ip_dst_range['end'], size = 4, op = action), - self.trex_vm_wr_flow(fv_name="dst",pkt_offset = "IP.dst") - ] - - vm += [self.trex_vm_ipv4(offset = "IP") - ] - - return vm - - def _prepare_generator(self): - if self.conf.has_key('start_trex') and self.conf['start_trex']: - app_param_temp = "-i" - - for key in self.conf: - #key, value = pktgen_conf - if key == 'config_file': - app_param_temp = app_param_temp + " --cfg " + self.conf[key] - elif key == 'core_num': - app_param_temp = app_param_temp + " -c " + self.conf[key] - app = self.conf['trex_root_path'] + os.sep + self.trex_app - cmd = app + " " + app_param_temp - self.control_session = self.tester.create_session('trex_control_session') - self.control_session.send_expect('cd ' + self.conf['trex_root_path'] + os.sep + 'scripts', '# ') - self.control_session.send_expect(app + " " + app_param_temp, '-Per port stats table', 30) - - # Insert Trex api library - sys.path.insert(0, "{0}/scripts/automation/trex_control_plane/stl".format(self.conf['trex_root_path'])) - #from trex_stl_lib.api import * - from trex_stl_lib.api import STLStream, STLPktBuilder, STLTXCont, STLVmFlowVar, STLVmWrFlowVar,\ - STLVmFixIpv4 - - mod = __import__("trex_stl_lib.api") - client_mod = getattr(mod, "trex_stl_client", None) - self.trex_client = getattr(client_mod, "STLClient", None) - self.trex_vm_flow = getattr(client_mod, "STLVmFlowVar", None) - self.trex_vm_wr_flow = getattr(client_mod, "STLVmWrFlowVar", None) - self.trex_vm_ipv4 = getattr(client_mod, "STLVmFixIpv4", None) - self.trex_stream = getattr(client_mod, "STLStream", None) - self.trex_pkt_builder = getattr(client_mod, "STLPktBuilder", None) - self.trex_tx_count = getattr(client_mod, "STLTXCont", None) - - self.connect() - #self.control_session.send_expect("cd " + cwd, "", 70) - - def _prepare_transmission(self, stream_ids=[]): - # Create base packet and pad it to size - streams = [] - ip_src_range = {} - ip_dst_range = {} - ip_src_range_temp = [] - ip_dst_range_temp = [] - - # prepare stream configuration - for stream_id in stream_ids: - stream = self._get_stream(stream_id) - tx_port = stream['tx_port'] - rx_port = stream['rx_port'] - rx_port_name = "port%d" % rx_port - option = stream['options'] - pcap_file = stream["pcap_file"] - #set rate - rate = option['rate'] - if "ip" not in option: - stl_stream = self.trex_stream(packet=self.trex_pkt_builder(pkt=pcap_file), mode=self.trex_tx_count(percentage=100)) - self._transmit_streams[stream_id] = stl_stream + self.packetLayers[pkt_object.name] = dict() + for curfield in pkt_object.fields_desc: + if isinstance(curfield, ConditionalField) and \ + not curfield._evalcond(pkt_object): continue + field_value = pkt_object.getfieldval(curfield.name) + if isinstance(field_value, scapyPacket) or (curfield.islist and \ + curfield.holds_packets and type(field_value) is list): + continue + repr_value = curfield.i2repr(pkt_object, field_value) + if isinstance(repr_value, str): + repr_value = repr_value.replace(os.linesep, + os.linesep + " "*(len(curfield.name) +4)) + self.packetLayers[pkt_object.name][curfield.name] = repr_value - ip = option['ip'] - mask = ip['mask'] - step_temp = ip['step'].split('.') - - #get the subnet range of src and dst ip - if self.conf.has_key("ip_src"): - ip_src = self.conf['ip_src'] - ip_src_range_string = IPy.IP(IPy.IP(ip_src).make_net(mask).strNormal()).strNormal(3) - ip_src_range_temp = ip_src_range_string.split('-') - ip_src_range['start'] = ip_src_range_temp[0] - ip_src_range['end'] = ip_src_range_temp[1] - - if self.conf.has_key("ip_dst"): - ip_dst = self.conf['ip_dst'] - ip_dst_range_string = IPy.IP(IPy.IP(ip_dst).make_net(mask).strNormal()).strNormal(3) - ip_dst_range_temp = ip_dst_range_string.split('-') - ip_dst_range['start'] = ip_dst_range_temp[0] - ip_dst_range['end'] = ip_dst_range_temp[1] - - # pcap_file = stream['pcap_file'] - - vm = self.create_vm(ip_src_range, ip_dst_range, action=ip['action'], step=step_temp[3]) - - stl_stream = self.trex_stream( - packet=self.trex_pkt_builder(pkt=pcap_file, vm=vm), - mode=self.trex_tx_count(percentage=100)) - - self._transmit_streams[stream_id] = stl_stream - - def _start_transmission(self, stream_ids, delay=50): - self._conn.reset(ports=self._ports) - self._conn.clear_stats() - self._conn.set_port_attr(self._ports, promiscuous=True) - duration_int = int(self.conf["duration"]) - rate = "100%" - warmup = 15 - - if self.conf.has_key("warmup"): - warmup = int(self.conf["warmup"]) - - self._traffic_ports = [] - for stream_id in stream_ids: - stream = self._get_stream(stream_id) - # tester port to Trex port - tx_port = stream["tx_port"] - p = self._ports[tx_port] - self._conn.add_streams(self._transmit_streams[stream_id], ports=[p]) - rate = stream["options"]["rate"] - self._traffic_ports.append(p) - - print self._traffic_ports - - if self.conf.has_key("core_mask"): - self._conn.start(ports=self._traffic_ports, mult=rate, duration=warmup, core_mask=self.conf["core_mask"]) - self._conn.wait_on_traffic(ports=self._traffic_ports, timeout=warmup+30) + if isinstance(pkt_object.payload, NoPayload): + return else: - self._conn.start(ports=self._traffic_ports, mult=rate, duration=warmup) - self._conn.wait_on_traffic(ports=self._traffic_ports, timeout=warmup+30) - - self._conn.clear_stats() - - if self.conf.has_key("core_mask"): - self._conn.start(ports=self._traffic_ports, mult=rate, duration=duration_int, core_mask=self.conf["core_mask"]) + self._parse_packet_layer(pkt_object.payload) + + def _parse_pcap(self, pcapFile, number=0): + ''' parse one packet content ''' + pcap_pkts = [] + if os.path.exists(pcapFile) == False: + warning = "{0} is not exist !".format(pcapFile) + raise Exception(warning) + + pcap_pkts = rdpcap(pcapFile) + # parse packets' every layers and fields + if len(pcap_pkts) == 0: + warning = "{0} is empty".format(pcapFile) + raise Exception(warning) + elif number>= len(pcap_pkts): + warning = "{0} is missing No.{1} packet".format(pcapFile, number) + raise Exception(warning) else: - self._conn.start(ports=self._traffic_ports, mult=rate, duration=duration_int) - - if self._conn.get_warnings(): - for warning in self._conn.get_warnings(): - logger.warn(warning) - - def _stop_transmission(self, stream_id): - self._conn.stop(ports=self._traffic_ports, rx_delay_ms=5000) - - def _retrieve_port_statistic(self, stream_id): - stats = self._conn.get_stats() - stream = self._get_stream(stream_id) - port_id = stream["rx_port"] - port_stats = stats[port_id] - print "Port %d stats: %s " % (port_id,port_stats) - rate_rx_pkts = port_stats["rx_pps"] - rate_rx_bits = port_stats["rx_bps_L1"] - print "rx_port: %d, rate_rx_pkts: %f, rate_rx_bits:%f " % (port_id,rate_rx_pkts,rate_rx_bits) - return rate_rx_bits, rate_rx_pkts - - def quit_generator(self): - if self._conn is not None: - self.disconnect() - if self.control_session is not None: - self.tester.send_expect('pkill -f _t-rex-64', '# ') - time.sleep(5) - self.tester.destroy_session(self.control_session) - self.control_session = None - -def getPacketGenerator(tester, pktgen_type="trex"): + self._parse_packet_layer(pcap_pkts[number]) + + def _set_pktgen_fields_config(self, pcap, suite_config): + ''' + get default fields value from a pcap file and unify layer fields + variables for trex/ixia + ''' + self._parse_pcap(pcap) + if not self.packetLayers: + msg = "pcap content is empty" + raise Exception(msg) + # suite fields config convert to pktgen fields config + fields_config = {} + # set ethernet protocol layer fields + layer_name = 'mac' + if layer_name in suite_config.keys() and \ + 'Ethernet' in self.packetLayers: + fields_config[layer_name] = {} + suite_fields = suite_config.get(layer_name) + pcap_fields = self.packetLayers.get('Ethernet') + for name, config in suite_fields.iteritems(): + action = config.get('action') or 'default' + range = config.get('range') or 64 + step = config.get('step') or 1 + start_mac = pcap_fields.get(name) + end_mac = convert_mac2str(convert_mac2long(start_mac) + range-1) + fields_config[layer_name][name] = {} + fields_config[layer_name][name]['start'] = start_mac + fields_config[layer_name][name]['end'] = end_mac + fields_config[layer_name][name]['step'] = step + fields_config[layer_name][name]['action'] = action + # set ip protocol layer fields + layer_name = 'ip' + if layer_name in suite_config.keys() and \ + 'IP' in self.packetLayers: + fields_config[layer_name] = {} + suite_fields = suite_config.get(layer_name) + pcap_fields = self.packetLayers.get('IP') + for name, config in suite_fields.iteritems(): + action = config.get('action') or 'default' + range = config.get('range') or 64 + step = config.get('step') or 1 + start_ip = pcap_fields.get(name) + end_ip = convert_int2ip(convert_ip2int(start_ip) + range - 1) + fields_config[layer_name][name] = {} + fields_config[layer_name][name]['start'] = start_ip + fields_config[layer_name][name]['end'] = end_ip + fields_config[layer_name][name]['step'] = step + fields_config[layer_name][name]['action'] = action + # set vlan protocol layer fields, only support one layer vlan here + layer_name = 'vlan' + if layer_name in suite_config.keys() and \ + '802.1Q' in self.packetLayers: + fields_config[layer_name] = {} + suite_fields = suite_config.get(layer_name) + pcap_fields = self.packetLayers.get('802.1Q') + # only support one layer vlan here, so set name to `0` + name = 0 + if name in suite_fields.keys(): + config = suite_fields[name] + action = config.get('action') or 'default' + range = config.get('range') or 64 + # ignore 'L' suffix + start_vlan = int(pcap_fields.get(layer_name)[:-1]) + end_vlan = start_vlan + range - 1 + fields_config[layer_name][name] = {} + fields_config[layer_name][name]['start'] = start_vlan + fields_config[layer_name][name]['end'] = end_vlan + fields_config[layer_name][name]['step'] = 1 + fields_config[layer_name][name]['action'] = action + + return fields_config + + def prepare_stream_from_tginput(self, tgen_input, ratePercent, + vm_config, pktgen_inst): + ''' create streams for ports, one port one stream ''' + # set stream in pktgen + stream_ids = [] + for config in tgen_input: + stream_id = pktgen_inst.add_stream(*config) + pcap = config[2] + _options = deepcopy(self.default_opt) + _options['pcap'] = pcap + # if vm is set + if vm_config: + _options['fields_config'] = \ + self._set_pktgen_fields_config(pcap, vm_config) + pktgen_inst.config_stream(stream_id, _options) + stream_ids.append(stream_id) + return stream_ids + +def getPacketGenerator(tester, pktgen_type=PKTGEN_IXIA): """ Get packet generator object """ pktgen_type = pktgen_type.lower() - if pktgen_type == "dpdk": - return DpdkPacketGenerator(tester) - elif pktgen_type == "ixia": - return IxiaPacketGenerator(tester) - elif pktgen_type == "trex": - return TrexPacketGenerator(tester) - - -if __name__ == "__main__": - # init pktgen stream options - - from tester import Tester - options = { - 'rate' : '100%', - 'ip': {'action': 'inc', 'mask' : '255.255.255.0', 'step':'0.0.0.1'} - } - crbsconf = CrbsConf() - crb = (crbsconf.load_crbs_config())[0] - tester = Tester(crb, None) - # framework initial - trex = getPacketGenerator(tester, pktgen_type="trex") - - conf = conf_inst.load_pktgen_config() - # prepare running environment - trex.prepare_generator() - - #config stream and convert options into pktgen commands - stream_id1 = trex.add_stream(0, 1, conf['pcap_file']) - trex.config_stream(stream_id=stream_id1, opts=options) - stream_id2 = trex.add_stream(1, 0, conf['pcap_file']) - trex.config_stream(stream_id=stream_id2, opts=options) - stream_id3 = trex.add_stream(0, 1, conf['pcap_file']) - trex.config_stream(stream_id=stream_id3, opts=options) - stream_id4 = trex.add_stream(1, 0, conf['pcap_file']) - trex.config_stream(stream_id=stream_id4, opts=options) - #pktgen.prepare_transmission(stream_ids=[stream_id]) - trex.measure_throughput(stream_ids=[stream_id1,stream_id2,stream_id3,stream_id4], delay=5) - #trex.measure_throughput(stream_ids=[stream_id1,stream_id2], delay=5) - # comeback to framework - trex.quit_generator() + pktgen_cls = { + PKTGEN_DPDK: DpdkPacketGenerator, + PKTGEN_IXIA: IxiaPacketGenerator, + PKTGEN_TREX: TrexPacketGenerator,} + + if pktgen_type in pktgen_cls.keys(): + CLS = pktgen_cls.get(pktgen_type) + return CLS(tester) + else: + msg = "not support <{0}> packet generator".format(pktgen_type) + raise Exception(msg) \ No newline at end of file -- 1.9.3