From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mga03.intel.com (mga03.intel.com [134.134.136.65]) by dpdk.org (Postfix) with ESMTP id 22C2ECF9 for ; Fri, 22 May 2015 11:04:52 +0200 (CEST) Received: from fmsmga002.fm.intel.com ([10.253.24.26]) by orsmga103.jf.intel.com with ESMTP; 22 May 2015 02:04:51 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.13,474,1427785200"; d="scan'208";a="730110204" Received: from shvmail01.sh.intel.com ([10.239.29.42]) by fmsmga002.fm.intel.com with ESMTP; 22 May 2015 02:04:51 -0700 Received: from shecgisg003.sh.intel.com (shecgisg003.sh.intel.com [10.239.29.90]) by shvmail01.sh.intel.com with ESMTP id t4M94nBs004080; Fri, 22 May 2015 17:04:49 +0800 Received: from shecgisg003.sh.intel.com (localhost [127.0.0.1]) by shecgisg003.sh.intel.com (8.13.6/8.13.6/SuSE Linux 0.8) with ESMTP id t4M94k7E014420; Fri, 22 May 2015 17:04:48 +0800 Received: (from yliu84x@localhost) by shecgisg003.sh.intel.com (8.13.6/8.13.6/Submit) id t4M94kNJ014416; Fri, 22 May 2015 17:04:46 +0800 From: "Jiajia, Sun" To: dts@dpdk.org Date: Fri, 22 May 2015 17:04:07 +0800 Message-Id: <1432285452-14286-15-git-send-email-sunx.jiajia@intel.com> X-Mailer: git-send-email 1.7.4.1 In-Reply-To: <1432285452-14286-1-git-send-email-sunx.jiajia@intel.com> References: <1432285452-14286-1-git-send-email-sunx.jiajia@intel.com> Subject: [dts] [PATCH v2 14/19] Change some codes to support network device instantiation and virtualization test X-BeenThere: dts@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: test suite reviews and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 22 May 2015 09:04:54 -0000 From: sjiajiax Signed-off-by: sjiajiax --- framework/dut.py | 235 +++++++++++++++++++++++++++++++++++++++------- framework/project_dpdk.py | 29 +++--- 2 files changed, 219 insertions(+), 45 deletions(-) diff --git a/framework/dut.py b/framework/dut.py index 5b7aba2..caaae62 100644 --- a/framework/dut.py +++ b/framework/dut.py @@ -33,11 +33,17 @@ import os import re import time import dts -from config import UserConf -from settings import NICS +import settings +from config import PortConf +from settings import NICS, LOG_NAME_SEP from ssh_connection import SSHConnection from crb import Crb +from net_device import NetDevice from logger import getLogger +from virt_resource import VirtResource +from utils import RED + + class Dut(Crb): @@ -60,20 +66,36 @@ class Dut(Crb): def __init__(self, crb, serializer): super(Dut, self).__init__(crb, serializer) self.NAME = 'dut' + + self.host_init_flag = 0 self.logger = getLogger(self.NAME) self.session = SSHConnection(self.get_ip_address(), self.NAME, self.get_password()) self.session.init_log(self.logger) - self.alt_session = SSHConnection(self.get_ip_address(), self.NAME + '_alt', - self.get_password()) + self.alt_session = SSHConnection( + self.get_ip_address(), + self.NAME + '_alt', + self.get_password()) self.alt_session.init_log(self.logger) self.number_of_cores = 0 self.tester = None self.cores = [] self.architecture = None self.ports_info = None - self.conf = UserConf() + self.conf = PortConf() self.ports_map = [] + self.virt_pool = None + + def init_host_session(self): + if self.host_init_flag: + pass + else: + self.host_session = SSHConnection( + self.get_ip_address(), + self.NAME + '_host', + self.get_password()) + self.host_session.init_log(self.logger) + self.host_init_flag = 1 def change_config_option(self, target, parameter, value): """ @@ -82,12 +104,12 @@ class Dut(Crb): self.send_expect("sed -i 's/%s=.*$/%s=%s/' config/defconfig_%s" % (parameter, parameter, value, target), "# ") - def set_nic_type(self, nic): + def set_nic_type(self, nic_type): """ Set CRB NICS ready to validated. """ - self.nic = nic - if 'cfg' in nic: + self.nic_type = nic_type + if 'cfg' in nic_type: self.conf.load_ports_config(self.get_ip_address()) def set_toolchain(self, target): @@ -167,11 +189,17 @@ class Dut(Crb): self.mount_procfs() # auto detect network topology self.map_available_ports() + # disable tester port ipv6 + self.disable_tester_ipv6() # print latest ports_info - self.logger.info(dts.pprint(self.ports_info)) + for port_info in self.ports_info: + self.logger.info(port_info) if self.ports_map is None or len(self.ports_map) == 0: raise ValueError("ports_map should not be empty, please check all links") + # initialize virtualization resource pool + self.virt_pool = VirtResource(self) + def restore_interfaces(self): """ Restore all ports's interfaces. @@ -195,20 +223,22 @@ class Dut(Crb): pci_bus = port['pci'] pci_id = port['type'] # get device driver - driver = dts.get_nic_driver(pci_id) + driver = settings.get_nic_driver(pci_id) if driver is not None: # unbind device driver addr_array = pci_bus.split(':') bus_id = addr_array[0] devfun_id = addr_array[1] + port = NetDevice(self, bus_id, devfun_id) + self.send_expect('echo 0000:%s > /sys/bus/pci/devices/0000\:%s\:%s/driver/unbind' % (pci_bus, bus_id, devfun_id), '# ') # bind to linux kernel driver self.send_expect('modprobe %s' % driver, '# ') self.send_expect('echo 0000:%s > /sys/bus/pci/drivers/%s/bind' % (pci_bus, driver), '# ') - itf = self.get_interface_name(addr_array[0], addr_array[1]) + itf = port.get_interface_name() self.send_expect("ifconfig %s up" % itf, "# ") else: self.logger.info("NOT FOUND DRIVER FOR PORT (%s|%s)!!!" % (pci_bus, pci_id)) @@ -265,6 +295,20 @@ class Dut(Crb): return 'taskset %s ' % core + def is_ssh_session_port(self, pci_bus): + """ + Check if the pci device is the dut SSH session port. + """ + port = None + for port_info in self.ports_info: + if pci_bus == port_info['pci']: + port = port_info['port'] + break + if port and port.get_ipv4_addr() == crbs['IP'].strip(): + return True + else: + return False + def bind_interfaces_linux(self, driver='igb_uio', nics_to_bind=None): """ Bind the interfaces to the selected driver. nics_to_bind can be None @@ -275,12 +319,17 @@ class Dut(Crb): current_nic = 0 for (pci_bus, pci_id) in self.pci_devices_info: - if dts.accepted_nic(pci_id): + if settings.accepted_nic(pci_id): + if self.is_ssh_session_port(pci_bus): + continue if nics_to_bind is None or current_nic in nics_to_bind: binding_list += '%s ' % (pci_bus) current_nic += 1 + if current_nic == 0: + self.logger.info("Not nic need bind driver: %s" % driver) + return self.send_expect('tools/dpdk_nic_bind.py %s' % binding_list, '# ') @@ -293,13 +342,19 @@ class Dut(Crb): current_nic = 0 for (pci_bus, pci_id) in self.pci_devices_info: - if dts.accepted_nic(pci_id): + if settings.accepted_nic(pci_id): + if self.is_ssh_session_port(pci_bus): + continue if nics_to_bind is None or current_nic in nics_to_bind: binding_list += '%s ' % (pci_bus) current_nic += 1 + if current_nic == 0: + self.logger.info("Not nic need unbind driver") + return + self.send_expect('tools/dpdk_nic_bind.py %s' % binding_list, '# ', 30) def get_ports(self, nic_type='any', perf=None, socket=None): @@ -321,7 +376,10 @@ class Dut(Crb): elif nic_type == 'cfg': for portid in range(len(self.ports_info)): if self.ports_info[portid]['source'] == 'cfg': - ports.append(portid) + if (socket is None or + self.ports_info[portid]['numa'] == -1 or + socket == self.ports_info[portid]['numa']): + ports.append(portid) return ports else: for portid in range(len(self.ports_info)): @@ -440,36 +498,38 @@ class Dut(Crb): Check that whether auto scanned ports ready to use """ pci_addr = "%s:%s" % (pci_bus, pci_id) - if self.nic == 'any': + if self.nic_type == 'any': return True - elif self.nic == 'cfg': + elif self.nic_type == 'cfg': if self.conf.check_port_available(pci_bus) is True: return True - elif self.nic not in NICS.keys(): - self.logger.warning("NOT SUPPORTED NIC TYPE: %s" % self.nic) + elif self.nic_type not in NICS.keys(): + self.logger.warning("NOT SUPPORTED NIC TYPE: %s" % self.nic_type) else: - codename = NICS[self.nic] + codename = NICS[self.nic_type] if pci_id == codename: return True return False def rescan_ports(self): - unknow_interface = dts.RED('Skipped: unknow_interface') + unknow_interface = RED('Skipped: unknow_interface') for port_info in self.ports_info: pci_bus = port_info['pci'] addr_array = pci_bus.split(':') bus_id = addr_array[0] devfun_id = addr_array[1] - intf = self.get_interface_name(bus_id, devfun_id) + + port = NetDevice(self, bus_id, devfun_id) + intf = port.get_interface_name() if "No such file" in intf: self.logger.info("DUT: [0000:%s] %s" % (pci_bus, unknow_interface)) out = self.send_expect("ip link show %s" % intf, "# ") if "DOWN" in out: self.send_expect("ip link set %s up" % intf, "# ") time.sleep(5) - macaddr = self.get_mac_addr(intf, bus_id, devfun_id) + macaddr = port.get_mac_addr() out = self.send_expect("ip -family inet6 address show dev %s | awk '/inet6/ { print $2 }'" % intf, "# ") ipv6 = out.split('/')[0] @@ -481,16 +541,32 @@ class Dut(Crb): port_info['intf'] = intf port_info['ipv6'] = ipv6 + def load_serializer_ports(self): + self.ports_info = [] + cached_ports_info = self.serializer.load(self.PORT_INFO_CACHE_KEY) + if cached_ports_info is None: + return + for port in cached_ports_info: + self.ports_info.append({'pci': port['pci'], 'type': port['type'], + 'numa': port['numa']}) + + def save_serializer_ports(self): + cached_ports_info = [] + for port in self.ports_info: + cached_ports_info.append({'pci': port['pci'], 'type': port['type'], + 'numa': port['numa']}) + self.serializer.save(self.PORT_INFO_CACHE_KEY, cached_ports_info) + def scan_ports(self): """ Scan ports information or just read it from cache file. """ if self.read_cache: - self.ports_info = self.serializer.load(self.PORT_INFO_CACHE_KEY) + self.load_serializer_ports() if not self.read_cache or self.ports_info is None: self.scan_ports_uncached() - self.serializer.save(self.PORT_INFO_CACHE_KEY, self.ports_info) + self.save_serializer_ports() def scan_ports_uncached(self): """ @@ -505,8 +581,8 @@ class Dut(Crb): """ self.ports_info = [] - skipped = dts.RED('Skipped: Unknown/not selected') - unknow_interface = dts.RED('Skipped: unknow_interface') + skipped = RED('Skipped: Unknown/not selected') + unknow_interface = RED('Skipped: unknow_interface') for (pci_bus, pci_id) in self.pci_devices_info: if self.check_ports_available(pci_bus, pci_id) is False: @@ -518,9 +594,11 @@ class Dut(Crb): bus_id = addr_array[0] devfun_id = addr_array[1] + port = NetDevice(self, bus_id, devfun_id) numa = self.get_device_numa(bus_id, devfun_id) # store the port info to port mapping - self.ports_info.append({'pci': pci_bus, 'type': pci_id, 'numa': numa}) + self.ports_info.append( + {'port': port, 'pci': pci_bus, 'type': pci_id, 'numa': numa}) def scan_ports_uncached_freebsd(self): """ @@ -528,16 +606,17 @@ class Dut(Crb): """ self.ports_info = [] - skipped = dts.RED('Skipped: Unknown/not selected') + skipped = RED('Skipped: Unknown/not selected') for (pci_bus, pci_id) in self.pci_devices_info: - if not dts.accepted_nic(pci_id): + if not setttings.accepted_nic(pci_id): self.logger.info("DUT: [%s %s] %s" % (pci_bus, pci_id, skipped)) continue - intf = self.get_interface_name(pci_bus) + port = NetDevice(self, pci_bus, '') + intf = port.get_interface_name(pci_bus) macaddr = self.get_mac_addr(intf) ipv6 = self.get_ipv6_addr(intf) @@ -560,9 +639,69 @@ class Dut(Crb): pci_str = "%s:%s.%s" % (pci_bus_id, pci_dev_str, pci_split[2]) # store the port info to port mapping - self.ports_info.append({'pci': pci_str, 'type': pci_id, 'intf': + self.ports_info.append({'port': port, 'pci': pci_str, 'type': pci_id, 'intf': intf, 'mac': macaddr, 'ipv6': ipv6, 'numa': -1}) + def generate_sriov_vfs_by_port(self, port_id, vf_num, driver='default'): + """ + Generate SRIOV VFs with default driver it is bound now or specifid driver. + """ + port = self.ports_info[port_id]['port'] + port_driver = port.get_nic_driver() + + if driver == 'default': + if not port_driver: + self.logger.info( + "No driver on specified port, can not generate SRIOV VF.") + return None + else: + if port_driver != driver: + port.bind_driver(driver) + port.generate_sriov_vfs(vf_num) + + # append the VF PCIs into the ports_info + sriov_vfs_pci = port.get_sriov_vfs_pci() + self.ports_info[port_id]['sriov_vfs_pci'] = sriov_vfs_pci + + # instantiate the VF with NetDevice + vfs_port = [] + for vf_pci in sriov_vfs_pci: + addr_array = vf_pci.split(':') + bus_id = addr_array[0] + devfun_id = addr_array[1] + vf_port = NetDevice(self, bus_id, devfun_id) + vfs_port.append(vf_port) + self.ports_info[port_id]['vfs_port'] = vfs_port + + pci = self.ports_info[port_id]['pci'] + self.virt_pool.add_vf_on_pf(pf_pci=pci, vflist=sriov_vfs_pci) + + def destroy_sriov_vfs_by_port(self, port_id): + port = self.ports_info[port_id]['port'] + vflist = [] + port_driver = port.get_nic_driver() + if 'sriov_vfs_pci' in self.ports_info[port_id] and \ + self.ports_info[port_id]['sriov_vfs_pci']: + vflist = self.ports_info[port_id]['sriov_vfs_pci'] + else: + if not port.get_sriov_vfs_pci(): + return + + if not port_driver: + self.logger.info( + "No driver on specified port, skip destroy SRIOV VF.") + else: + sriov_vfs_pci = port.destroy_sriov_vfs() + self.ports_info[port_id]['sriov_vfs_pci'] = [] + self.ports_info[port_id]['vfs_port'] = [] + + pci = self.ports_info[port_id]['pci'] + self.virt_pool.del_vf_on_pf(pf_pci=pci, vflist=vflist) + + def get_vm_core_list(self): + return VMCORELIST[self.crb['VM CoreList']] + + def load_portconf(self): """ Load port configurations for ports_info. If manually configured infor @@ -637,7 +776,11 @@ class Dut(Crb): if ipv6 == "Not connected": continue - out = self.tester.send_ping6(remotePort, ipv6, self.get_mac_address(dutPort)) + if getattr(self, 'send_ping6', None): + self.send_ping6( + dutPort, self.tester.ports_info[remotePort]['ipv6']) + out = self.tester.send_ping6( + remotePort, ipv6, self.get_mac_address(dutPort)) if ('64 bytes from' in out): self.logger.info("PORT MAP: [dut %d: tester %d]" % (dutPort, remotePort)) @@ -656,3 +799,31 @@ class Dut(Crb): for port in remove: self.ports_info.remove(port) + + def disable_tester_ipv6(self): + for tester_port in self.ports_map: + intf = self.tester.get_interface(tester_port) + self.tester.disable_ipv6(intf) + + def enable_tester_ipv6(self): + for tester_port in self.ports_map: + intf = self.tester.get_interface(tester_port) + self.tester.enable_ipv6(intf) + + def check_port_occupied(self, port): + out = self.alt_session.send_expect('lsof -i:%d' % port, '# ') + if out == '': + return False + else: + return True + + def get_maximal_vnc_num(self): + out = self.send_expect("ps aux | grep '\-vnc' | grep -v grep", '# ') + if out: + ports = re.findall(r'-vnc .*?:(\d+)', out) + for num in range(len(ports)): + ports[num] = int(ports[num]) + ports.sort() + else: + ports = [0, ] + return ports[-1] diff --git a/framework/project_dpdk.py b/framework/project_dpdk.py index 8963924..0b94b99 100644 --- a/framework/project_dpdk.py +++ b/framework/project_dpdk.py @@ -39,7 +39,7 @@ from crb import Crb from dut import Dut from tester import Tester from logger import getLogger -from settings import IXIA +from settings import IXIA, accepted_nic class DPDKdut(Dut): @@ -50,8 +50,8 @@ class DPDKdut(Dut): """ def __init__(self, crb, serializer): - self.NAME = 'dut' super(DPDKdut, self).__init__(crb, serializer) + self.testpmd = None def set_target(self, target): """ @@ -60,6 +60,7 @@ class DPDKdut(Dut): Set hugepage on DUT and install modules required by DPDK. Configure default ixgbe PMD function. """ + self.target = target self.set_toolchain(target) # set env variable @@ -110,7 +111,7 @@ class DPDKdut(Dut): binding_list = '' for (pci_bus, pci_id) in self.pci_devices_info: - if dts.accepted_nic(pci_id): + if accepted_nic(pci_id): binding_list += '%s,' % (pci_bus) self.send_expect("kldunload if_ixgbe.ko", "#") @@ -191,10 +192,7 @@ class DPDKdut(Dut): assert ("Error" not in out), "Compilation error..." assert ("No rule to make" not in out), "No rule to make error..." - def prerequisites(self, pkgName, patch): - """ - Copy DPDK package to DUT and apply patch files. - """ + def prepare_package(self, pkgName, patch): if not self.skip_setup: assert (os.path.isfile(pkgName) is True), "Invalid package" @@ -249,6 +247,11 @@ class DPDKdut(Dut): (self.base_dir, dst_dir + p), "# ") assert "****" not in out + def prerequisites(self, pkgName, patch): + """ + Copy DPDK package to DUT and apply patch files. + """ + self.prepare_package(pkgName, patch) self.dut_prerequisites() def bind_interfaces_linux(self, driver='igb_uio', nics_to_bind=None): @@ -354,7 +357,7 @@ class DPDKtester(Tester): total_huge_pages = self.get_total_huge_pages() if total_huge_pages == 0: self.mount_huge_pages() - self.set_huge_pages(4096) + self.set_huge_pages(1024) self.session.copy_file_to("dep/tgen.tgz") self.session.copy_file_to("dep/tclclient.tgz") @@ -386,10 +389,10 @@ class DPDKtester(Tester): """ hugepages_size = self.send_expect("awk '/Hugepagesize/ {print $2}' /proc/meminfo", "# ") - if int(hugepages_size) < (1024 * 1024): - arch_huge_pages = hugepages if hugepages > 0 else 4096 + if int(hugepages_size) < (2048 * 2048): + arch_huge_pages = hugepages if hugepages > 0 else 2048 total_huge_pages = self.get_total_huge_pages() - self.mount_huge_pages() - if total_huge_pages != arch_huge_pages: - self.set_huge_pages(arch_huge_pages) + self.mount_huge_pages() + if total_huge_pages != arch_huge_pages: + self.set_huge_pages(arch_huge_pages) -- 1.9.3