Re: [dts] [PATCH] add vf port start/stop test script, plan and config file

["Pei, Yulong" <yulong.pei@intel.com>]

Please see my comment below,


-----Original Message-----
From: dts [mailto:dts-bounces@dpdk.org] On Behalf Of Lijuan Tu
Sent: Wednesday, January 6, 2016 5:44 PM
To: dts@dpdk.org
Cc: Lijuan Tu <lijuanx.tu.a@intel.com>
Subject: [dts] [PATCH] add vf port start/stop test script, plan and config file

From: Lijuan Tu <lijuanx.tu.a@intel.com>

Send packets from tester ,at the same time, vf prot start/stop several time and check if it running right.
Three files added in the patch:
1, vf_port_start_stop.cfg: vm setting and qemu parameters.
2, vf_port_start_stop_test_plan.rst: test plan, describe 1 case 3, TestSuite_vf_port_start_stop.py: implement one case according to the tes plan

Signed-off-by: Lijuan Tu <lijuanx.tu.a@intel.com>
---
 conf/vf_port_start_stop.cfg                 | 107 ++++++++++++++
 test_plans/vf_port_start_stop_test_plan.rst | 173 +++++++++++++++++++++++
 tests/TestSuite_vf_port_start_stop.py       | 212 ++++++++++++++++++++++++++++
 3 files changed, 492 insertions(+)
 create mode 100644 conf/vf_port_start_stop.cfg  create mode 100644 test_plans/vf_port_start_stop_test_plan.rst
 create mode 100644 tests/TestSuite_vf_port_start_stop.py

diff --git a/conf/vf_port_start_stop.cfg b/conf/vf_port_start_stop.cfg new file mode 100644 index 0000000..ab1c0c7
--- /dev/null
+++ b/conf/vf_port_start_stop.cfg
@@ -0,0 +1,107 @@
+# QEMU options
+# name
+#       name: vm0
+#
+# enable_kvm
+#       enable: [yes | no]
+#
+# cpu
+#       model: [host | core2duo | ...]
+#           usage:
+#               choose model value from the command
+#                   qemu-system-x86_64 -cpu help
+#       number: '4' #number of vcpus
+#       cpupin: '3 4 5 6' # host cpu list
+#
+# mem
+#       size: 1024
+#
+# disk
+#       file: /path/to/image/test.img
+#
+# net
+#        type: [nic | user | tap | bridge | ...]
+#           nic
+#               opt_vlan: 0
+#                   note: Default is 0.
+#               opt_macaddr: 00:00:00:00:01:01
+#                   note: if creating a nic, it`s better to specify a MAC,
+#                         else it will get a random number.
+#               opt_model:["e1000" | "virtio" | "i82551" | ...]
+#                   note: Default is e1000.
+#               opt_name: 'nic1'
+#               opt_addr: ''
+#                   note: PCI cards only.
+#               opt_vectors:
+#                   note: This option currently only affects virtio cards.
+#           user
+#               opt_vlan: 0
+#                   note: default is 0.
+#               opt_hostfwd: [tcp|udp]:[hostaddr]:hostport-[guestaddr]:guestport
+#                   note: If not specified, it will be setted automatically.
+#           tap
+#               opt_vlan: 0
+#                   note: default is 0.
+#               opt_br: br0
+#                   note: if choosing tap, need to specify bridge name,
+#                         else it will be br0.
+#               opt_script: QEMU_IFUP_PATH
+#                   note: if not specified, default is self.QEMU_IFUP_PATH.
+#               opt_downscript: QEMU_IFDOWN_PATH
+#                   note: if not specified, default is self.QEMU_IFDOWN_PATH.
+#
+# device
+#       driver: [pci-assign | virtio-net-pci | ...]
+#           pci-assign
+#               prop_host: 08:00.0
+#               prop_addr: 00:00:00:00:01:02
+#           virtio-net-pci
+#               prop_netdev: mynet1
+#               prop_id: net1
+#               prop_mac: 00:00:00:00:01:03
+#               prop_bus: pci.0
+#               prop_addr: 0x3
+#
+# monitor
+#       port: 6061   
+#           note: if adding monitor to vm, need to specicy
+#                 this port, else it will get a free port
+#                 on the host machine.
+#
+# qga
+#       enable: [yes | no]
+#
+# serial_port
+#       enable: [yes | no]
+#
+# vnc
+#       displayNum: 1
+#           note: you can choose a number not used on the host.
+#
+# daemon
+#       enable: 'yes'
+#           note:
+#               By default VM will start with the daemonize status.
+#               Not support starting it on the stdin now.
+
+# vm configuration for pmd sriov case
+[vm0]
+cpu =
+    model=host,number=4,cpupin=5 6 7 8 9; disk =
+    file=/home/image/fedora23.img;
+mem =
+    size=8196
+login =
+    user=root,password=tester;
+net = 
+   type=nic,opt_vlan=0;
+   type=user,opt_vlan=0;
+monitor = 
+    port=;
+qga = 
+    enable=yes;
+vnc = 
+    displayNum=11;
+daemon =
+    enable=yes;
diff --git a/test_plans/vf_port_start_stop_test_plan.rst b/test_plans/vf_port_start_stop_test_plan.rst
new file mode 100644
index 0000000..6464b55
--- /dev/null
+++ b/test_plans/vf_port_start_stop_test_plan.rst
@@ -0,0 +1,173 @@
+.. Copyright (c) <2015>, Intel Corporation
+      All rights reserved.
+
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions
+   are met:
+
+   - Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+
+   - Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in
+     the documentation and/or other materials provided with the
+     distribution.
+
+   - Neither the name of Intel Corporation nor the names of its
+     contributors may be used to endorse or promote products derived
+     from this software without specific prior written permission.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+   FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+   COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+   INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+   (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+   SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+   HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+   STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+   OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Prerequisites for Topo1- PF kernel driver 
+=========================================
+
+Create Two VF interfaces from two kernel PF ineterfaces, and then attach them to VM. Suppose PF is 0000:04:00.0 and 0000:04:00.1. Generate 2VFs using commands below and make them in pci-stub mods.
+


Here Create Two VF interfaces from one kernel PF is enough.


+1. Get the pci device id of DUT::
+
+    ./dpdk_nic_bind.py --st
+    0000:04:00.0 '82599ES 10-Gigabit SFI/SFP+ Network Connection' if=ens261f0 drv=ixgbe unused=igb_uio
+    0000:04:00.1 '82599ES 10-Gigabit SFI/SFP+ Network Connection' 
+ if=ens261f1 drv=ixgbe unused=igb_uio
+
+2. Create 2 VFs from 2 PFs::
+
+    echo 1 > /sys/bus/pci/devices/0000\:04\:00.0/sriov_numvfs
+    echo 1 > /sys/bus/pci/devices/0000\:04\:00.1/sriov_numvfs
+
+VFs 04:10.0 & 04:10.1 have been created::
+
+    ./dpdk_nic_bind.py --st
+    0000:04:00.0 '82599ES 10-Gigabit SFI/SFP+ Network Connection' if=ens261f0 drv=ixgbe unused=
+    0000:04:00.1 '82599ES 10-Gigabit SFI/SFP+ Network Connection' if=ens261f1 drv=ixgbe unused=
+    0000:04:10.0 '82599 Ethernet Controller Virtual Function' if=enp4s16 drv=ixgbevf unused=
+    0000:04:10.1 '82599 Ethernet Controller Virtual Function' 
+ if=enp4s16f1 drv=ixgbevf unused=
+
+3. detach VFs from the host, bind them to pci-stub driver::
+
+    /sbin/modprobe pci-stub
+    echo "8086 10ed" > /sys/bus/pci/drivers/pci-stub/new_id
+    echo 0000:04:10.0 > /sys/bus/pci/devices/0000\:04\:10.0/driver/unbind
+    echo 0000:04:10.0 > /sys/bus/pci/drivers/pci-stub/bind
+    echo 0000:04:10.1 > /sys/bus/pci/devices/0000\:04\:10.1/driver/unbind
+    echo 0000:04:10.1 > /sys/bus/pci/drivers/pci-stub/bind
+
+or using the following more easy way::
+
+    ./dpdk_nic_bind.py -b pci-stub 04:10.0 04:10.1
+
+it can be seen that VFs 04:10.0 & 04:10.1 's drv is pci-stub::
+
+    ./dpdk_nic_bind.py --st
+    0000:04:00.0 '82599ES 10-Gigabit SFI/SFP+ Network Connection' if=ens261f0 drv=ixgbe unused=vfio-pci
+    0000:04:00.1 '82599ES 10-Gigabit SFI/SFP+ Network Connection' if=ens261f1 drv=ixgbe unused=vfio-pci
+    0000:04:10.0 '82599 Ethernet Controller Virtual Function' if= drv=pci-stub unused=ixgbevf,vfio-pci
+    0000:04:10.1 '82599 Ethernet Controller Virtual Function' if= 
+ drv=pci-stub unused=ixgbevf,vfio-pci
+
+4. Do not forget bring up PFs::
+
+    ifconfig ens261f0 up
+    ifconfig ens261f1 up
+
+Passthrough VFs 04:10.0 & 04:10.1 to vm0, and start vm0, you can refer to below command::
+
+    taskset -c 6-12 qemu-system-x86_64 \
+    -enable-kvm -m 8192  -smp 6 -cpu host -name dpdk15-vm1 \
+    -drive file=/home/image/fedora23.img \
+    -netdev tap,id=hostnet1,ifname=tap1,script=/etc/qemu-ifup,vhost=on \
+    -device rtl8139,netdev=hostnet1,id=net1,mac=52:54:01:6b:10:61,bus=pci.0,addr=0xa \
+    -device pci-assign,bus=pci.0,addr=0x6,host=04:10.0 \
+    -device pci-assign,bus=pci.0,addr=0x7,host=04:10.1 \
+    -vnc :11 -daemonize
+
+the /etc/qemu-ifup can be below script, need you to create first::
+
+    #!/bin/sh
+    set -x
+    switch=br0
+    if [ -n "$1" ];then
+        /usr/sbin/tunctl -u `whoami` -t $1
+        /sbin/ip link set $1 up
+        sleep 0.5s
+        /usr/sbin/brctl addif $switch $1
+        exit 0
+    else
+        echo "Error: no interface specified"
+    exit 1
+    fi
+
+Set up bridge br0 before create /etc/qemu-ifup, for example::
+
+    cd /etc/sysconfig/network-scripts
+    vim ifcfg-enp1s0f0
+
+    HWADDR=00:1e:67:fb:0f:d4
+    TYPE=Ethernet
+    NAME=enp1s0f0
+    ONBOOT=yes
+    DEVICE=enp1s0f0
+    NM_CONTROLLED=no
+    BRIDGE=br0
+
+    vim ifcfg-br0
+    TYPE=Bridge
+    DEVICE=br0
+    ONBOOT=yes
+    NM_CONTROLLED=no
+    BOOTPROTO=dhcp
+    HOSTNAME="dpdk-test58"
+
+Login vm0, got VFs pci device id in vm0, assume they are 00:06.0 & 
+00:07.0, bind them to igb_uio driver, and then start testpmd, set it in 
+mac forward mode::
+
+    ./tools/dpdk_nic_bind.py --bind=igb_uio 00:06.0 00:07.0
+    ./x86_64-native-linuxapp-gcc/app/testpmd -c 0x0f -n 4 -- -i
+    testpmd-> set fwd mac
+    testpmd-> start
+
+Test Case: port start/stop
+==========================
+Start send packets from tester , then start/stop ports several times ,verify if it running right.
+
+Commands could be used to start/stop ports refer to below::
+
+Start port::
+
+    testpmd-> port start all
+
+Stop port::
+
+    testpmd-> port stop all
+
+Send IP+UDP packet::
+
+    Ether(dst="0E:CB:F8:FF:4E:02", 
+ src="0E:CB:F8:FF:4E:02")/IP(src="127.0.0.2")/UDP()/("X"*46)
+
+Send IP+TCP packet::
+
+    Ether(dst="0E:CB:F8:FF:4E:02", 
+ src="0E:CB:F8:FF:4E:02")/IP(src="127.0.0.2")/TCP()/("X"*46)
+
+Send IP+SCTP packet::
+
+    Ether(dst="0E:CB:F8:FF:4E:02", 
+ src="0E:CB:F8:FF:4E:02")/IP(src="127.0.0.2")/SCTP()/("X"*46)
+
+Send IPv6+UDP packet::
+
+    Ether(dst="0E:CB:F8:FF:4E:02", 
+ src="0E:CB:F8:FF:4E:02")/IP(src="::2")/UDP()/("X"*46)
+
+Send IPv6+TCP packet::
+
+    Ether(dst="0E:CB:F8:FF:4E:02", 
+ src="0E:CB:F8:FF:4E:02")/IP(src="::2")/TCP()/("X"*46)
+
diff --git a/tests/TestSuite_vf_port_start_stop.py b/tests/TestSuite_vf_port_start_stop.py
new file mode 100644
index 0000000..051f988
--- /dev/null
+++ b/tests/TestSuite_vf_port_start_stop.py
@@ -0,0 +1,212 @@
+# <COPYRIGHT_TAG>
+
+import re
+import time
+
+import dts
+from qemu_kvm import QEMUKvm
+from test_case import TestCase
+from pmd_output import PmdOutput
+from utils import RED, GREEN
+from net_device import NetDevice
+from crb import Crb
+from scapy.all import *
+VM_CORES_MASK = 'all'
+
+class TestVfPortStartStop(TestCase):
+
+    def set_up_all(self):
+
+        self.dut_ports = self.dut.get_ports(self.nic)
+        self.verify(len(self.dut_ports) > 1, "Insufficient ports")
+        self.vm0 = None
+        self.filename = "/root/vf.pcap"
+
+    def set_up(self):
+
+        self.setup_2pf_2vf_1vm_env_flag = 0
+
+    def pktgen_prerequisites(self):
+        """
+        igb_uio.ko should be put in ~ before you using pktgen
+        """
+        out = self.tester.send_expect("ls", "#")
+        self.verify("igb_uio.ko" in out, "No file igb_uio.ko")
+        self.tester.send_expect("modprobe uio", "#", 70)
+        out = self.tester.send_expect("lsmod | grep igb_uio", "#")
+        if "igb_uio" in out:
+            out = self.tester.send_expect("insmod igb_uio.ko", "#")
+        self.verify("could not load module" not in out, "%s" %out)
+
+        total_huge_pages = self.tester.get_total_huge_pages()
+        if total_huge_pages == 0:
+            self.tester.mount_huge_pages()
+            self.tester.set_huge_pages(1024)
+
+    def send_and_verify(self, dst_mac, testpmd):
+        """
+        Generates packets by pktgen
+        """
+        self.pktgen_prerequisites()
+        # bind ports
+        tx_port = self.tester.get_local_port(self.dut_ports[0])
+        tx_pci = self.tester.ports_info[tx_port]['pci']
+        port = self.tester.ports_info[tx_port]['port']
+        port_driver = port.get_nic_driver()
+        self.tester.send_expect("./dpdk_nic_bind.py --bind=igb_uio %s" 
+ % tx_pci, "#")
+
+        src_mac = self.tester.get_mac(tx_port)
+
+        pkts = {'IP/UDP': Ether(dst="%s" % dst_mac, src="%s" % src_mac)/IP(src="127.0.0.2")/UDP()/("X"*46),
+                'IP/TCP': Ether(dst="%s" % dst_mac, src="%s" % src_mac)/IP(src="127.0.0.2")/TCP()/("X"*46),
+                'IP/SCTP': Ether(dst="%s" % dst_mac, src="%s" % src_mac)/IP(src="127.0.0.2")/SCTP()/("X"*48),
+                'IPv6/UDP': Ether(dst="%s" % dst_mac, src="%s" % src_mac)/IPv6(src="::2")/UDP()/("X"*46),
+                'IPv6/TCP': Ether(dst="%s" % dst_mac, src="%s" % 
+ src_mac)/IPv6(src="::2")/TCP()/("X"*46),}
+
+        self.testpmd_reset_status(testpmd)
+
+        for key in pkts.keys():
+            self.create_pcap_file(self.filename, pkts[key]) 


It is better to write pkts.keys() to one pcap file and then using pktgen send the pcap, then it avoid to start/stop pktgen in loop. 


+            self.tester.send_expect("./pktgen -c 0x1f -n 2  --proc-type auto --socket-mem 128,128 --file-prefix pg -- -P -T -m '1.0' -s 0:%s" % self.filename, "Pktgen >", 100)
+            self.tester.send_expect("start all", "Pktgen>")
+            time.sleep(1)
+            self.check_port_start_stop(testpmd)            
+            # quit pktgen
+            self.tester.send_expect("stop all", "Pktgen>")
+            self.tester.send_expect("quit", "# ")
+

It is better to check if it has 'RX-errors' & 'TX-errors' after frequently port start/stop.


+        self.tester.send_expect("./dpdk_nic_bind.py --bind=%s %s" 
+ %(port_driver, tx_pci), "#")
+ 
+    def create_pcap_file(self, filename, pkt):
+        """
+        Generates a valid PCAP file with the given configuration.
+        """
+        wrpcap(filename, pkt)
+
+    def testpmd_reset_status(self, testpmd):
+        """
+        Reset testpmd :stop forword & stop port
+        """
+        testpmd.execute_cmd('stop')
+        testpmd.execute_cmd('port stop all')
+        testpmd.execute_cmd('clear port stats all')
+
+    def check_port_start_stop(self, testpmd, times=100):


I think that 100 is not enough for stress, maybe 1000 is better.

Best Regards
Yulong Pei

+        """
+        VF port start/stop several times , check if it work well.
+        """
+        for i in range(times):
+            out = testpmd.execute_cmd('port start all')
+            self.verify("Checking link statuses" in out, "ERROR: port start all")
+            testpmd.execute_cmd('start')
+            time.sleep(.5)
+            testpmd.execute_cmd('stop')
+            out = testpmd.execute_cmd('port stop all')
+            self.verify("Checking link statuses" in out, "ERROR: port 
+ stop all")
+            
+    def setup_2pf_2vf_1vm_env(self, driver='default'):
+
+        self.used_dut_port_0 = self.dut_ports[0]
+        self.dut.generate_sriov_vfs_by_port(self.used_dut_port_0, 1, driver=driver)
+        self.sriov_vfs_port_0 = 
+ self.dut.ports_info[self.used_dut_port_0]['vfs_port']
+
+        self.used_dut_port_1 = self.dut_ports[1]
+        self.dut.generate_sriov_vfs_by_port(self.used_dut_port_1, 1, driver=driver)
+        self.sriov_vfs_port_1 = 
+ self.dut.ports_info[self.used_dut_port_1]['vfs_port']
+
+        try:
+
+            for port in self.sriov_vfs_port_0:
+                port.bind_driver('pci-stub')
+
+            for port in self.sriov_vfs_port_1:
+                port.bind_driver('pci-stub')
+
+            time.sleep(1)
+            vf0_prop = {'opt_host': self.sriov_vfs_port_0[0].pci}
+            vf1_prop = {'opt_host': self.sriov_vfs_port_1[0].pci}
+
+            if driver == 'igb_uio':
+                # start testpmd without the two VFs on the host
+                self.host_testpmd = PmdOutput(self.dut)
+                eal_param = '-b %(vf0)s -b %(vf1)s' % {'vf0': self.sriov_vfs_port_0[0].pci,
+                                                       'vf1': self.sriov_vfs_port_1[0].pci}
+                self.host_testpmd.start_testpmd("1S/2C/2T", 
+ eal_param=eal_param)
+
+            # set up VM0 ENV
+            self.vm0 = QEMUKvm(self.dut, 'vm0', 'vf_port_start_stop')
+            self.vm0.set_vm_device(driver='pci-assign', **vf0_prop)
+            self.vm0.set_vm_device(driver='pci-assign', **vf1_prop)
+            self.vm_dut_0 = self.vm0.start()
+            if self.vm_dut_0 is None:
+                raise Exception("Set up VM0 ENV failed!")
+
+            self.setup_2pf_2vf_1vm_env_flag = 1
+        except Exception as e:
+            self.destroy_2pf_2vf_1vm_env()
+            raise Exception(e)
+
+    def destroy_2pf_2vf_1vm_env(self):
+        if getattr(self, 'vm0', None):
+            #destroy testpmd in vm0
+            self.vm0_testpmd.execute_cmd('stop')
+            self.vm0_testpmd.execute_cmd('quit', '# ')
+            self.vm0_testpmd = None
+            self.vm0_dut_ports = None
+            #destroy vm0
+            self.vm0.stop()
+            self.vm0 = None
+
+        if getattr(self, 'host_testpmd', None):
+            self.host_testpmd.execute_cmd('quit', '# ')
+            self.host_testpmd = None
+
+        if getattr(self, 'used_dut_port_0', None):
+            self.dut.destroy_sriov_vfs_by_port(self.used_dut_port_0)
+            port = self.dut.ports_info[self.used_dut_port_0]['port']
+            port.bind_driver()
+            self.used_dut_port_0 = None
+
+        if getattr(self, 'used_dut_port_1', None):
+            self.dut.destroy_sriov_vfs_by_port(self.used_dut_port_1)
+            port = self.dut.ports_info[self.used_dut_port_1]['port']
+            port.bind_driver()
+            self.used_dut_port_1 = None
+
+        for port_id in self.dut_ports:
+            port = self.dut.ports_info[port_id]['port']
+            port.bind_driver()
+
+        self.setup_2pf_2vf_1vm_env_flag = 0
+
+######1. test case for kernel pf and dpdk vf 2pf_2vf_1vm scenario
+
+    def test_start_stop_with_kernel_2pf_2vf_1vm(self):
+
+        self.setup_2pf_2vf_1vm_env(driver='')
+
+        self.vm0_dut_ports = self.vm_dut_0.get_ports('any')
+
+        self.vm0_testpmd = PmdOutput(self.vm_dut_0)
+        self.vm0_testpmd.start_testpmd(VM_CORES_MASK)
+        self.vm0_testpmd.execute_cmd('set fwd mac')
+
+        time.sleep(2)
+
+        dst_mac = self.vm_dut_0.get_mac_address(self.vm0_dut_ports[0])
+        print "dst_mac:", dst_mac
+        self.send_and_verify(dst_mac, self.vm0_testpmd)
+
+    def tear_down(self):
+
+        if self.setup_2pf_2vf_1vm_env_flag == 1:
+            self.destroy_2pf_2vf_1vm_env()
+
+    def tear_down_all(self):
+
+        if getattr(self, 'vm0', None):
+            self.vm0.stop()
+
+        for port_id in self.dut_ports:
+            self.dut.destroy_sriov_vfs_by_port(port_id)
+
--
2.5.0