From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mga09.intel.com (mga09.intel.com [134.134.136.24]) by dpdk.org (Postfix) with ESMTP id B8E2437A4 for ; Thu, 14 Jul 2016 15:18:21 +0200 (CEST) Received: from fmsmga004.fm.intel.com ([10.253.24.48]) by orsmga102.jf.intel.com with ESMTP; 14 Jul 2016 06:17:56 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.28,362,1464678000"; d="scan'208";a="139142143" Received: from shvmail01.sh.intel.com ([10.239.29.42]) by fmsmga004.fm.intel.com with ESMTP; 14 Jul 2016 06:17:54 -0700 Received: from shecgisg003.sh.intel.com (shecgisg003.sh.intel.com [10.239.29.90]) by shvmail01.sh.intel.com with ESMTP id u6EDHrk0004485; Thu, 14 Jul 2016 21:17:53 +0800 Received: from shecgisg003.sh.intel.com (localhost [127.0.0.1]) by shecgisg003.sh.intel.com (8.13.6/8.13.6/SuSE Linux 0.8) with ESMTP id u6EDHoBw001361; Thu, 14 Jul 2016 21:17:52 +0800 Received: (from yliu84x@localhost) by shecgisg003.sh.intel.com (8.13.6/8.13.6/Submit) id u6EDHoU1001357; Thu, 14 Jul 2016 21:17:50 +0800 From: Marvin Liu To: dts@dpdk.org Cc: Marvin Liu Date: Thu, 14 Jul 2016 21:17:33 +0800 Message-Id: <1468502253-1276-8-git-send-email-yong.liu@intel.com> X-Mailer: git-send-email 1.7.4.1 In-Reply-To: <1468502253-1276-1-git-send-email-yong.liu@intel.com> References: <1468502253-1276-1-git-send-email-yong.liu@intel.com> Subject: [dts] [PATCH 7/7] tests: add vhost_user_live_migration suite X-BeenThere: dts@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: test suite reviews and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 14 Jul 2016 13:18:22 -0000 This suite will verify virtio netdevice work fine before and after migration. Virtio device is in virtual machine and drived either by kernel driver or by dpdk. Signed-off-by: Marvin Liu diff --git a/tests/TestSuite_vhost_user_live_migration.py b/tests/TestSuite_vhost_user_live_migration.py new file mode 100644 index 0000000..95e2fb3 --- /dev/null +++ b/tests/TestSuite_vhost_user_live_migration.py @@ -0,0 +1,327 @@ +# + +import re +import time + +import dts +from qemu_kvm import QEMUKvm +from test_case import TestCase +from exception import VirtDutInitException + + +class TestVhostUserLiveMigration(TestCase): + + def set_up_all(self): + # verify at least two duts + self.verify(len(self.duts) >= 2, "Insufficient duts for live migration!!!") + + # each dut required one ports + self.dut_ports = self.dut.get_ports() + # Verify that enough ports are available + self.verify(len(self.dut_ports) >= 1, "Insufficient ports for testing") + self.dut_port = self.dut_ports[0] + dut_ip = self.dut.crb['My IP'] + self.host_tport = self.tester.get_local_port_bydut(self.dut_port, dut_ip) + self.host_tintf = self.tester.get_interface(self.host_tport) + + self.backup_ports = self.duts[1].get_ports() + # Verify that enough ports are available + self.verify(len(self.backup_ports) >= 1, "Insufficient ports for testing") + self.backup_port = self.backup_ports[0] + # backup host ip will be used in migrate command + self.backup_dutip = self.duts[1].crb['My IP'] + self.backup_tport = self.tester.get_local_port_bydut(self.backup_port, self.backup_dutip) + self.backup_tintf = self.tester.get_interface(self.backup_tport) + + # build backup vhost-switch + out = self.duts[0].send_expect("make -C examples/vhost", "# ") + self.verify("Error" not in out, "compilation error 1") + self.verify("No such file" not in out, "compilation error 2") + + # build backup vhost-switch + out = self.duts[1].send_expect("make -C examples/vhost", "# ") + self.verify("Error" not in out, "compilation error 1") + self.verify("No such file" not in out, "compilation error 2") + + self.vhost = "./examples/vhost/build/app/vhost-switch" + self.vm_testpmd = "./%s/app/testpmd -c 0x3 -n 4 -- -i" % self.target + self.virio_mac = "00:00:00:00:00:01" + + # flag for environment + self.env_done = False + + def set_up(self): + self.setup_vm_env() + pass + + def bind_nic_driver(self, crb, ports, driver=""): + # modprobe vfio driver + if driver == "vfio-pci": + for port in ports: + netdev = crb.ports_info[port]['port'] + driver = netdev.get_nic_driver() + if driver != 'vfio-pci': + netdev.bind_driver(driver='vfio-pci') + + elif driver == "igb_uio": + # igb_uio should insmod as default, no need to check + for port in ports: + netdev = crb.ports_info[port]['port'] + driver = netdev.get_nic_driver() + if driver != 'igb_uio': + netdev.bind_driver(driver='igb_uio') + else: + for port in ports: + netdev = crb.ports_info[port]['port'] + driver_now = netdev.get_nic_driver() + if driver == "": + driver = netdev.default_driver + if driver != driver_now: + netdev.bind_driver(driver=driver) + + def setup_vm_env(self, driver='default'): + """ + Create testing environment on Host and Backup + """ + if self.env_done: + return + + # start vhost-switch on host and backup machines + self.logger.info("Start vhost on host and backup host") + for crb in self.duts[:2]: + self.bind_nic_driver(crb, [crb.get_ports()[0]], driver="igb_uio") + # start vhost-switch, predict hugepage on both sockets + base_dir = crb.base_dir.replace('~', '/root') + crb.send_expect("rm -f %s/vhost-net" % base_dir, "# ") + crb.send_expect("%s -c f -n 4 --socket-mem 1024 -- -p 0x1" % self.vhost, "bind to vhost-net") + + try: + # set up host virtual machine + self.host_vm = QEMUKvm(self.duts[0], 'host', 'vhost_user_live_migration') + vhost_params = {} + vhost_params['driver'] = 'vhost-user' + # qemu command can't use ~ + base_dir = self.dut.base_dir.replace('~', '/root') + vhost_params['opt_path'] = base_dir + '/vhost-net' + vhost_params['opt_mac'] = self.virio_mac + self.host_vm.set_vm_device(**vhost_params) + + self.logger.info("Start virtual machine on host") + self.vm_host = self.host_vm.start() + + if self.vm_host is None: + raise Exception("Set up host VM ENV failed!") + + self.host_serial = self.host_vm.connect_serial_port(name='vhost_user_live_migration') + if self.host_serial is None: + raise Exception("Connect host serial port failed!") + + self.logger.info("Start virtual machine on backup host") + # set up backup virtual machine + self.backup_vm = QEMUKvm(self.duts[1], 'backup', 'vhost_user_live_migration') + vhost_params = {} + vhost_params['driver'] = 'vhost-user' + # qemu command can't use ~ + base_dir = self.dut.base_dir.replace('~', '/root') + vhost_params['opt_path'] = base_dir + '/vhost-net' + vhost_params['opt_mac'] = self.virio_mac + self.backup_vm.set_vm_device(**vhost_params) + + # start qemu command + self.backup_vm.start() + + except Exception as ex: + if ex is VirtDutInitException: + self.host_vm.stop() + self.host_vm = None + # no session created yet, call internal stop function + self.backup_vm._stop_vm() + self.backup_vm = None + else: + self.destroy_vm_env() + raise Exception(ex) + + self.env_done = True + + def destroy_vm_env(self): + # if environment has been destroyed, just skip + if self.env_done is False: + return + + if getattr(self, 'host_serial', None): + if self.host_vm is not None: + self.host_vm.close_serial_port() + + if getattr(self, 'backup_serial', None): + if self.backup_serial is not None and self.backup_vm is not None: + self.backup_vm.close_serial_port() + + self.logger.info("Stop virtual machine on host") + if getattr(self, 'vm_host', None): + if self.vm_host is not None: + self.host_vm.stop() + self.host_vm = None + + self.logger.info("Stop virtual machine on backup host") + if getattr(self, 'vm_backup', None): + if self.vm_backup is not None: + self.vm_backup.kill_all() + # backup vm dut has been initialized, destroy backup vm + self.backup_vm.stop() + self.backup_vm = None + + if getattr(self, 'backup_vm', None): + # only qemu start, no session created + if self.backup_vm is not None: + self.backup_vm.stop() + self.backup_vm = None + + # after vm stopped, stop vhost-switch + for crb in self.duts[:2]: + crb.kill_all() + + for crb in self.duts[:2]: + self.bind_nic_driver(crb, [crb.get_ports()[0]], driver="igb_uio") + + self.env_done = False + + def send_pkts(self, intf, number=0): + """ + send packet from tester + """ + sendp_fmt = "sendp([Ether(dst='%(DMAC)s')/Dot1Q(vlan=1000)/IP()/UDP()/Raw('x'*18)], iface='%(INTF)s', count=%(COUNT)d)" + sendp_cmd = sendp_fmt % {'DMAC': self.virio_mac, 'INTF': intf, 'COUNT': number} + self.tester.scapy_append(sendp_cmd) + self.tester.scapy_execute() + # sleep 10 seconds for heavy load with backup host + time.sleep(10) + + def verify_dpdk(self, tester_port, serial_session): + num_pkts = 10 + + stats_pat = re.compile("RX-packets: (\d+)") + intf = self.tester.get_interface(tester_port) + serial_session.send_expect("stop", "testpmd> ") + serial_session.send_expect("set fwd rxonly", "testpmd> ") + serial_session.send_expect("clear port stats all", "testpmd> ") + serial_session.send_expect("start tx_first", "testpmd> ") + + # send packets from tester + self.send_pkts(intf, number=num_pkts) + + out = serial_session.send_expect("show port stats 0", "testpmd> ") + m = stats_pat.search(out) + if m: + num_received = int(m.group(1)) + else: + num_received = 0 + + self.verify(num_received >= num_pkts, "Not receive packets as expected!!!") + self.logger.info("Verified %s packets recevied" % num_received) + + def verify_kernel(self, tester_port, vm_dut): + """ + Function to verify packets received by virtIO + """ + intf = self.tester.get_interface(tester_port) + num_pkts = 10 + + # get host interface + vm_intf = vm_dut.ports_info[0]['port'].get_interface_name() + # start tcpdump the interface + vm_dut.send_expect("ifconfig %s up" % vm_intf, "# ") + vm_dut.send_expect("tcpdump -i %s -P in -v" % vm_intf, "listening on") + # wait for promisc on + time.sleep(3) + # send packets from tester + self.send_pkts(intf, number=num_pkts) + + # killall tcpdump and verify packet received + out = vm_dut.get_session_output(timeout=1) + vm_dut.send_expect("^C", "# ") + num = out.count('UDP') + self.verify(num == num_pkts, "Not receive packets as expected!!!") + self.logger.info("Verified %s packets recevied" % num_pkts) + + def test_migrate_with_kernel(self): + """ + Verify migrate virtIO device from host to backup host, + Verify before/in/after migration, device with kernel driver can receive packets + """ + # bind virtio-net back to virtio-pci + self.bind_nic_driver(self.vm_host, [self.vm_host.get_ports()[0]], driver="") + # verify host virtio-net work fine + self.verify_kernel(self.host_tport, self.vm_host) + + self.logger.info("Migrate host VM to backup host") + # start live migration + self.host_vm.start_migration(self.backup_dutip, self.backup_vm.migrate_port) + + # make sure still can receive packets in migration process + self.verify_kernel(self.host_tport, self.vm_host) + + self.logger.info("Waiting migration process done") + # wait live migration done + self.host_vm.wait_migration_done() + + # check vhost-switch log after migration + out = self.duts[0].get_session_output(timeout=1) + self.verify("device has been removed" in out, "Device not removed for host") + out = self.duts[1].get_session_output(timeout=1) + self.verify("virtio is now ready" in out, "Device not ready on backup host") + + self.logger.info("Migration process done, init backup VM") + # connected backup VM + self.vm_backup = self.backup_vm.migrated_start() + + # make sure still can receive packets + self.verify_kernel(self.backup_tport, self.vm_backup) + + def test_migrate_with_dpdk(self): + # bind virtio-net to igb_uio + self.bind_nic_driver(self.vm_host, [self.vm_host.get_ports()[0]], driver="igb_uio") + + # start testpmd on host vm + base_dir = self.vm_host.base_dir.replace('~', '/root') + self.host_serial.send_expect('cd %s' % base_dir, "# ") + self.host_serial.send_expect(self.vm_testpmd, "testpmd> ") + + # verify testpmd receive packets + self.verify_dpdk(self.host_tport, self.host_serial) + + self.logger.info("Migrate host VM to backup host") + # start live migration + self.host_vm.start_migration(self.backup_dutip, self.backup_vm.migrate_port) + + # make sure still can receive packets in migration process + self.verify_dpdk(self.host_tport, self.host_serial) + + self.logger.info("Waiting migration process done") + # wait live migration done + self.host_vm.wait_migration_done() + + # check vhost-switch log after migration + out = self.duts[0].get_session_output(timeout=1) + self.verify("device has been removed" in out, "Device not removed for host") + out = self.duts[1].get_session_output(timeout=1) + self.verify("virtio is now ready" in out, "Device not ready on backup host") + + self.logger.info("Migration process done, init backup VM") + time.sleep(5) + + # make sure still can receive packets + self.backup_serial = self.backup_vm.connect_serial_port(name='vhost_user_live_migration', first=False) + if self.backup_serial is None: + raise Exception("Connect backup host serial port failed!") + + self.verify_dpdk(self.backup_tport, self.backup_serial) + + # quit testpmd + self.backup_serial.send_expect("quit", "# ") + + def tear_down(self): + self.destroy_vm_env() + pass + + def tear_down_all(self): + pass -- 1.9.3