From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 833DFA10D6 for ; Mon, 5 Aug 2019 07:50:00 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 67E0B1BE74; Mon, 5 Aug 2019 07:50:00 +0200 (CEST) Received: from mga06.intel.com (mga06.intel.com [134.134.136.31]) by dpdk.org (Postfix) with ESMTP id AE13B1BE63 for ; Mon, 5 Aug 2019 07:49:58 +0200 (CEST) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga005.fm.intel.com ([10.253.24.32]) by orsmga104.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 04 Aug 2019 22:49:57 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.64,348,1559545200"; d="scan'208";a="372969526" Received: from dpdk-moyufen01.sh.intel.com ([10.67.111.77]) by fmsmga005.fm.intel.com with ESMTP; 04 Aug 2019 22:49:56 -0700 From: yufengmx To: dts@dpdk.org Cc: yufengmx Date: Mon, 5 Aug 2019 13:50:42 +0800 Message-Id: <1564984244-151448-9-git-send-email-yufengx.mo@intel.com> X-Mailer: git-send-email 1.9.3 In-Reply-To: <1564984244-151448-1-git-send-email-yufengx.mo@intel.com> References: <1564984244-151448-1-git-send-email-yufengx.mo@intel.com> Subject: [dts] [PATCH V1 8/10] [next]framework/pktgen_trex: add new feature and fix X-BeenThere: dts@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: test suite reviews and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dts-bounces@dpdk.org Sender: "dts" internal bug *. remove duration option in trex module, move duration option in testing scenario methods(latency/loss/throughput) in pktgen_base module. *. remove sample_delay option in trex module. *. remove runtim_stat in trex module. *. convert core mask setting to list format. *. create _get_traffic_option to convert pktgen.cfg setting value to traffic start options. *. use tester alt session to close trex. *. set part of information logger display to debug level. *. remove logger format setting after import libs to fix dts redundant logs. *. fix pep8 issue. Signed-off-by: yufengmx --- framework/pktgen_trex.py | 211 +++++++++++++++++++++-------------------------- 1 file changed, 95 insertions(+), 116 deletions(-) diff --git a/framework/pktgen_trex.py b/framework/pktgen_trex.py index 159750e..326c7f0 100644 --- a/framework/pktgen_trex.py +++ b/framework/pktgen_trex.py @@ -35,21 +35,18 @@ import time import logging from pprint import pformat -from pktgen_base import (PacketGenerator, PKTGEN_TREX, +from pktgen_base import (PacketGenerator, PKTGEN_TREX, PKTGEN, TRANSMIT_CONT, TRANSMIT_M_BURST, TRANSMIT_S_BURST) -FORMAT = '%(message)s' -logging.basicConfig(format=FORMAT) -logger = logging.getLogger(os.path.basename(__file__)[:-3].upper()) -logger.setLevel(logging.INFO) - class TrexConfigVm(object): ''' config one stream vm format of trex ''' + def __init__(self): - from trex_stl_lib.api import (ipv4_str_to_num, mac2str, is_valid_ipv4_ret) + from trex_stl_lib.api import ( + ipv4_str_to_num, mac2str, is_valid_ipv4_ret) self.ipv4_str_to_num = ipv4_str_to_num self.is_valid_ipv4_ret = is_valid_ipv4_ret self.mac2str = mac2str @@ -74,7 +71,7 @@ class TrexConfigVm(object): 'max_value': max_value, 'size': 4, 'step': step, - 'op': mode,}, + 'op': mode, }, {'write': {'add_val': add_val, 'offset_fixup': 2}}] return var @@ -86,7 +83,8 @@ class TrexConfigVm(object): _ip_start = self.ipv4_str_to_num(self.is_valid_ipv4_ret(ip_start)) _ip_end = self.ipv4_str_to_num(self.is_valid_ipv4_ret(ip_end)) _step = self.ipv4_str_to_num(self.is_valid_ipv4_ret(step)) \ - if isinstance(step, (str, unicode)) else step + if isinstance(step, (str, unicode)) else \ + step if mode == 'inc' or mode == 'dec': min_value = _ip_start max_value = _ip_end @@ -101,7 +99,7 @@ class TrexConfigVm(object): 'max_value': max_value, 'size': 4, 'step': _step, - 'op': mode,}, + 'op': mode, }, {'write': {'add_val': add_val}, 'fix_chksum': {}}] @@ -120,12 +118,11 @@ class TrexConfigVm(object): mac_end = config.get('end') or 'FF:FF:FF:FF:FF:FF' step = config.get('step') or 1 mode = config.get('action') or 'inc' - #----------------- fv_name = 'Ethernet.{0}'.format(name) # layer/field name vm_var[fv_name] = self._mac_var(fv_name, - mac_start, mac_end, - step, mode) + mac_start, mac_end, + step, mode) ################################################################### # src ip mask inc/dec/random if 'ip' in option: @@ -134,11 +131,10 @@ class TrexConfigVm(object): ip_end = config.get('end') or '0.0.0.255' step = config.get('step') or 1 mode = config.get('action') or 'inc' - #----------------- fv_name = 'IP.{0}'.format(name) # layer/field name vm_var[fv_name] = self._ip_vm_var(fv_name, ip_start, ip_end, - step, mode) + step, mode) ################################################################### # merge var1/var2/random/cache into one method ################################################################### @@ -150,7 +146,6 @@ class TrexConfigVm(object): port_end = config.get('end') or 255 step = config.get('step') or 1 mode = config.get('action') or 'inc' - #----------------- fv_name = '{0}.{1}'.format(protocol.upper(), name) # layer/field name vm_var[fv_name] = { @@ -159,17 +154,17 @@ class TrexConfigVm(object): 'max_value': port_end, 'size': 2, 'step': step, - 'op': mode,} + 'op': mode, } ################################################################### # vlan field inc/dec/random if 'vlan' in option: for name, config in option['vlan'].iteritems(): vlan_start = config.get('start') \ - if config.get('start') != None else 0 + if config.get('start') != None else \ + 0 vlan_end = config.get('end') or 256 step = config.get('step') or 1 mode = config.get('action') or 'inc' - #----------------- fv_name = '802|1Q:{0}.vlan'.format(name) # vlan layer/field name vm_var[fv_name] = { @@ -178,7 +173,7 @@ class TrexConfigVm(object): 'max_value': vlan_end, 'size': 2, 'step': step, - 'op': mode,} + 'op': mode, } ################################################################### # payload change with custom sizes if 'pkt_size' in option: @@ -189,7 +184,6 @@ class TrexConfigVm(object): mode = 'random' min_pkt_size = option['pkt_size']['start'] max_pkt_size = option['pkt_size']['end'] - #----------------- l3_len_fix = -len(Ether()) l4_len_fix = l3_len_fix - len(IP()) @@ -200,17 +194,17 @@ class TrexConfigVm(object): 'max_value': max_pkt_size - 4, 'size': 2, 'step': step, - 'op': mode,} + 'op': mode, } vm_var = { - 'IP.len': [ - var, {'write': {'add_val': l3_len_fix}, - 'trim': {}, - 'fix_chksum': {}}], - 'UDP.len': [ - var, {'write': {'add_val': l4_len_fix}, - 'trim': {}, - 'fix_chksum': {}}]} + 'IP.len': [ + var, {'write': {'add_val': l3_len_fix}, + 'trim': {}, + 'fix_chksum': {}}], + 'UDP.len': [ + var, {'write': {'add_val': l4_len_fix}, + 'trim': {}, + 'fix_chksum': {}}]} return vm_var @@ -219,9 +213,9 @@ class TrexConfigStream(object): def __init__(self): from trex_stl_lib.api import ( - STLTXCont, STLTXSingleBurst, STLTXMultiBurst, - STLPktBuilder, STLProfile, STLVM, - STLStream, STLFlowLatencyStats) + STLTXCont, STLTXSingleBurst, STLTXMultiBurst, + STLPktBuilder, STLProfile, STLVM, + STLStream, STLFlowLatencyStats) # set trex class self.STLStream = STLStream @@ -270,8 +264,9 @@ class TrexConfigStream(object): if isinstance(layer, (tuple, list)): vm_var.tuple_var(**config) for offset in layer: - fv_name = name+'.ip' if offset.startswith('IP') else \ - name+'.port' + fv_name = name + '.ip' \ + if offset.startswith('IP') else \ + name + '.port' _vars = {'fv_name': fv_name, 'pkt_offset': offset} if op_config and 'write' in op_config: _vars.update(op_config['write']) @@ -324,16 +319,16 @@ class TrexConfigStream(object): burst_pkts = txmode_opt.get('burst_pkts') or 32 bursts_count = txmode_opt.get('bursts_count') or 2 ibg = txmode_opt.get('ibg') or 10 - mode_inst = self.STLTXMultiBurst(pkts_per_burst = burst_pkts, - count = bursts_count, - ibg = ibg) + mode_inst = self.STLTXMultiBurst(pkts_per_burst=burst_pkts, + count=bursts_count, + ibg=ibg) else: msg = 'not support format {0}'.format(mode) raise Exception(msg) pkt = self.STLPktBuilder(pkt=_pkt, vm=vm) _stream = self.STLStream(packet=pkt, mode=mode_inst, isg=isg, - flow_stats=flow_stats) + flow_stats=flow_stats) return _stream @@ -369,7 +364,7 @@ class TrexConfigStream(object): # configure trex vm vm_var = self._generate_vm(vm_conf) # create - streams.append(self._create_stream( _pkt, _stream_op, vm_var)) + streams.append(self._create_stream(_pkt, _stream_op, vm_var)) _streams = self.STLProfile(streams).get_streams() return _streams @@ -388,8 +383,8 @@ class TrexConfigStream(object): latency_opt = streams_config[0] _pkt = latency_opt.get('pcap') _stream_op = latency_opt.get('stream_config') - _stream = self._create_stream( _pkt, _stream_op, - flow_stats=flow_stats) + _stream = self._create_stream(_pkt, _stream_op, + flow_stats=flow_stats) streams.append(_stream) else: streams = _streams @@ -402,6 +397,7 @@ class TrexPacketGenerator(PacketGenerator): Trex packet generator, detail usage can be seen at https://trex-tgn.cisco.com/trex/doc/trex_manual.html """ + def __init__(self, tester): self.pktgen_type = PKTGEN_TREX self.trex_app = "t-rex-64" @@ -412,34 +408,35 @@ class TrexPacketGenerator(PacketGenerator): self._ports = [] self._traffic_ports = [] self._rx_ports = [] - self.runtime_stats = {} conf_inst = self._get_generator_conf_instance() self.conf = conf_inst.load_pktgen_config() self.options_keys = [ 'txmode', 'ip', 'vlan', 'transmit_mode', 'rate'] - self.ip_keys = ['start', 'end','action', 'mask', 'step'] + self.ip_keys = ['start', 'end', 'action', 'mask', 'step'] self.vlan_keys = ['start', 'end', 'action', 'step', 'count'] super(TrexPacketGenerator, self).__init__(tester) # check trex binary file - trex_bin = os.sep.join([self.conf.get('trex_root_path'), self.trex_app]) + trex_bin = os.sep.join( + [self.conf.get('trex_root_path'), self.trex_app]) if not os.path.exists(trex_bin): msg = "{0} is not existed, please check {1} content".format( - trex_bin, conf_inst.config_file) + trex_bin, conf_inst.config_file) raise Exception(msg) # if `trex_lib_path` is not set, use a default relative directory. trex_lib_dir = \ - self.conf.get('trex_lib_path') \ - if self.conf.get('trex_lib_path') else \ - "{0}/automation/trex_control_plane/stl".format( - self.conf.get('trex_root_path')) + self.conf.get('trex_lib_path') \ + if self.conf.get('trex_lib_path') else \ + "{0}/automation/trex_control_plane/stl".format( + self.conf.get('trex_root_path')) # check trex lib root directory if not os.path.exists(trex_lib_dir): msg = ("{0} is not existed, please check {1} content and " - "set `trex_lib_path`").format(trex_lib_dir, conf_inst.config_file) + "set `trex_lib_path`").format( + trex_lib_dir, conf_inst.config_file) raise Exception(msg) # check if trex lib is existed trex_lib = os.sep.join([trex_lib_dir, 'trex_stl_lib']) @@ -451,6 +448,23 @@ class TrexPacketGenerator(PacketGenerator): from trex_stl_lib.api import STLClient # set trex class self.STLClient = STLClient + # get configuration from pktgen config file + self._get_traffic_option() + + def _get_traffic_option(self): + ''' get configuration from pktgen config file ''' + # set trex coremask + _core_mask = self.conf.get("core_mask") + self.core_mask = \ + [int(item[2:], 16) for item in _core_mask.split(',')] \ + if _core_mask and '0x' in _core_mask else \ + None + # In case of several ports, ensure their transmitting time is + # synchronized. + _synchronized = self.conf.get("synchronized") + self.sync = True \ + if _synchronized and _synchronized.lower() == 'true' else \ + False def _connect(self): self._conn = self.STLClient(server=self.conf["server"]) @@ -488,7 +502,7 @@ class TrexPacketGenerator(PacketGenerator): ''' for name, _port_obj in self._conn.ports.iteritems(): _pci = _port_obj.info['pci_addr'] - self.logger.info((_pci, pci)) + self.logger.debug((_pci, pci)) if _pci == pci: return True else: @@ -507,7 +521,7 @@ class TrexPacketGenerator(PacketGenerator): 'intf': 'TREX:%d' % idx, 'mac': mac, 'pci': pci, - 'type': 'trex',}) + 'type': 'trex', }) return ports def _clear_streams(self): @@ -581,13 +595,11 @@ class TrexPacketGenerator(PacketGenerator): app_param_temp = app_param_temp + " --cfg " + self.conf[key] elif key == 'core_num': app_param_temp = app_param_temp + " -c " + self.conf[key] - self.control_session = \ - self.tester.create_session('trex_control_session') - + self.control_session = self.tester.create_session(PKTGEN) self.control_session.send_expect( - ';'.join(['cd ' + self.conf['trex_root_path'], - './' + self.trex_app + " " + app_param_temp]), - '-Per port stats table', 30) + ';'.join(['cd ' + self.conf['trex_root_path'], + './' + self.trex_app + " " + app_param_temp]), + '-Per port stats table', 30) try: self._connect() except Exception as e: @@ -598,7 +610,7 @@ class TrexPacketGenerator(PacketGenerator): def _vm_conf(self): return None # close it and wait for more discussion about pktgen framework conf = {} - #get the subnet range of src and dst ip + # get the subnet range of src and dst ip if self.conf.has_key("ip_src"): conf['src'] = {} ip_src = self.conf['ip_src'] @@ -624,7 +636,7 @@ class TrexPacketGenerator(PacketGenerator): if port_id not in ports: return None features = self._conn.ports[port_id].get_formatted_info() - self.logger.info(pformat(features)) + self.logger.debug(pformat(features)) return features @@ -644,18 +656,19 @@ class TrexPacketGenerator(PacketGenerator): # for trex design requirement, all ports of trex should be the same type # nic, here use first port to check flow control attribute flow_ctrl = self._traffic_opt.get('flow_control') \ - if self._is_support_flow_control(rx_ports[0]) else None + if self._is_support_flow_control(rx_ports[0]) else \ + None flow_ctrl_flag = flow_ctrl.get('flag') or 1 if flow_ctrl else None # flow control of port running trex traffic - self._conn.set_port_attr( rx_ports, - promiscuous=True, - link_up=True, - flow_ctrl = flow_ctrl_flag) + self._conn.set_port_attr(rx_ports, + promiscuous=True, + link_up=True, + flow_ctrl=flow_ctrl_flag) def _throughput_stats(self, stream, stats): # tx packet tx_port_id = stream["tx_port"] - port_stats = self.runtime_stats.get(tx_port_id) + port_stats = stats.get(tx_port_id) if not port_stats: msg = "failed to get tx_port {0} statistics".format(tx_port_id) raise Exception(msg) @@ -664,12 +677,12 @@ class TrexPacketGenerator(PacketGenerator): msg = [ "Tx Port %d stats: " % (tx_port_id), "tx_port: %d, tx_bps: %f, tx_pps: %f " % ( - tx_port_id, tx_bps, tx_pps)] - self.logger.info(pformat(port_stats)) + tx_port_id, tx_bps, tx_pps)] + self.logger.debug(pformat(port_stats)) self.logger.info(os.linesep.join(msg)) # rx bps/pps rx_port_id = stream["rx_port"] - port_stats = self.runtime_stats.get(rx_port_id) + port_stats = stats.get(rx_port_id) if not port_stats: msg = "failed to get rx_port {0} statistics".format(rx_port_id) raise Exception(msg) @@ -678,9 +691,9 @@ class TrexPacketGenerator(PacketGenerator): msg = [ "Rx Port %d stats: " % (rx_port_id), "rx_port: %d, rx_bps: %f, rx_pps: %f" % ( - rx_port_id, rx_bps, rx_pps)] + rx_port_id, rx_bps, rx_pps)] - self.logger.info(pformat(port_stats)) + self.logger.debug(pformat(port_stats)) self.logger.info(os.linesep.join(msg)) return rx_bps, rx_pps @@ -719,7 +732,7 @@ class TrexPacketGenerator(PacketGenerator): latency_stats = { 'min': port_stats.get('total_min'), 'max': port_stats.get('total_max'), - 'average':port_stats.get('average'),} + 'average': port_stats.get('average'), } return latency_stats @@ -747,7 +760,7 @@ class TrexPacketGenerator(PacketGenerator): # the same. stream_config = options.get('stream_config') or {} self._traffic_opt['rate'] = stream_config.get('rate') or 100 - if stream_config.get('pps'): # reserve feature + if stream_config.get('pps'): # reserve feature self._traffic_opt['pps'] = stream_config.get('pps') # flow control option is deployed on all ports by design self._traffic_opt['flow_control'] = options.get('flow_control') or {} @@ -771,58 +784,24 @@ class TrexPacketGenerator(PacketGenerator): self._preset_trex_port() def _start_transmission(self, stream_ids, options={}): - ''' - :param sample_delay: - After traffic start ``sample_delay`` seconds, start get runtime statistics - ''' + test_mode = options.get('method') # get rate percentage rate_percent = "{0}%".format(options.get('rate') or self._traffic_opt.get('rate') or '100') - # get duration - duration = options.get("duration") or 20 - duration = int(duration) if isinstance(duration, (str, unicode)) \ - else duration - # get sample interval - _sample_delay = options.get("sample_delay") or duration/2 - sample_delay = int(_sample_delay) \ - if isinstance(_sample_delay, (str, unicode)) \ - else _sample_delay - # get configuration from pktgen config file - warmup = int(self.conf["warmup"]) if self.conf.has_key("warmup") \ - else 25 - # set trex coremask - wait_interval, core_mask = ( - warmup+30, int(self.conf["core_mask"], 16)) \ - if self.conf.has_key("core_mask") \ - else (warmup+5, 0x3) - try: - ########################################### # clear the stats before injecting self._conn.clear_stats() # Start traffic on port(s) run_opt = { 'ports': self._traffic_ports, 'mult': rate_percent, - 'duration': duration, - 'core_mask':core_mask, - 'force': True,} + 'core_mask': self.core_mask, + 'synchronized': self.sync, + 'force': True, } self.logger.info("begin traffic ......") + self.logger.debug(run_opt) self._conn.start(**run_opt) - ########################################### - if sample_delay: - time.sleep(sample_delay) # wait - # get ports runtime statistics - self.runtime_stats = self._conn.get_stats() - self.logger.info(pformat(self.runtime_stats)) - ########################################### - # Block until traffic on specified port(s) has ended - wait_opt = {'ports': self._traffic_ports} - if duration: - time.sleep(wait_interval + 10) - wait_opt['timeout'] = wait_interval + duration - self._conn.wait_on_traffic(**wait_opt) except Exception as e: self.logger.error(e) @@ -836,8 +815,8 @@ class TrexPacketGenerator(PacketGenerator): ''' stats = self._conn.get_stats() stream = self._get_stream(stream_id) - self.logger.info(pformat(stream)) - self.logger.info(pformat(stats)) + self.logger.debug(pformat(stream)) + self.logger.debug(pformat(stats)) if mode == 'throughput': return self._throughput_stats(stream, stats) elif mode == 'loss': @@ -851,7 +830,7 @@ class TrexPacketGenerator(PacketGenerator): if self._conn is not None: self._disconnect() if self.control_session is not None: - self.tester.send_expect('pkill -f _t-rex-64', '# ') + self.tester.alt_session.send_expect('pkill -f _t-rex-64', '# ') time.sleep(5) self.tester.destroy_session(self.control_session) self.control_session = None -- 1.9.3