[dts] [PATCH V1 0/2] userspace_ethtool: update automation testing script/test plan *. add nic's firmware/bus-info information query checking

[dts] [PATCH V1 0/2] userspace_ethtool: update automation testing script/test plan *. add nic's firmware/bus-info information query checking

[yufengx.mo]

From: yufengmx <yufengx.mo@intel.com>


yufengmx (2):
  userspace_ethtool: update test plan.
  userspace_ethtool: update automation testing script.

 test_plans/userspace_ethtool_test_plan.rst |   8 +-
 tests/TestSuite_userspace_ethtool.py       | 132 +++++++++++++++++++++++++----
 2 files changed, 121 insertions(+), 19 deletions(-)

-- 
1.9.3

[dts] [PATCH V1 1/2] userspace_ethtool: update test plan.

[yufengx.mo]

From: yufengmx <yufengx.mo@intel.com>


*. add nic's firmware/bus-info information query checking

Signed-off-by: yufengmx <yufengx.mo@intel.com>
---
 test_plans/userspace_ethtool_test_plan.rst | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/test_plans/userspace_ethtool_test_plan.rst b/test_plans/userspace_ethtool_test_plan.rst
index 83e7fac..498cc38 100644
--- a/test_plans/userspace_ethtool_test_plan.rst
+++ b/test_plans/userspace_ethtool_test_plan.rst
@@ -65,8 +65,12 @@ dumped information, which are dumped separately by dpdk's ethtool and
 linux's ethtool, were exactly the same.
 
 	EthApp> drvinfo
-	Port 0 driver: rte_ixgbe_pmd (ver: RTE 2.1.0)
-	Port 1 driver: rte_ixgbe_pmd (ver: RTE 2.1.0)
+    Port 0 driver: net_ixgbe (ver: DPDK 17.02.0-rc0)
+    bus-info: 0000:84:00.0
+    firmware-version: 0x61bf0001
+    Port 1 driver: net_ixgbe (ver: DPDK 17.02.0-rc0)
+    bus-info: 0000:84:00.1
+    firmware-version: 0x61bf0001
 
 Use "link" command to dump all ports link status.
 Notice:: On FVL, link detect need a physical link disconnect.
-- 
1.9.3

[dts] [PATCH V1 2/2] userspace_ethtool: update automation testing script.

[yufengx.mo]

From: yufengmx <yufengx.mo@intel.com>


*. add nic's firmware/bus-info information query checking

Signed-off-by: yufengmx <yufengx.mo@intel.com>
---
 tests/TestSuite_userspace_ethtool.py | 132 ++++++++++++++++++++++++++++++-----
 1 file changed, 115 insertions(+), 17 deletions(-)

diff --git a/tests/TestSuite_userspace_ethtool.py b/tests/TestSuite_userspace_ethtool.py
index ced7e96..dc4c6b2 100644
--- a/tests/TestSuite_userspace_ethtool.py
+++ b/tests/TestSuite_userspace_ethtool.py
@@ -129,23 +129,26 @@ class TestUserspaceEthtool(TestCase, IxiaPacketGenerator):
             return 1518
 
     def resize_linux_eeprom_file(self, dpdk_eeprom_file, linux_eeprom_file):
-        basePath = os.sep + "root" + self.dut.base_dir[1:] + os.sep
+        if self.dut.base_dir.startswith('~'):
+            basePath = "/root" + self.dut.base_dir[1:]
+        else:
+            basePath = self.dut.base_dir
         with open( basePath + os.sep + dpdk_eeprom_file, 'rb') as fpDpdk:
             dpdk_bytes = fpDpdk.read()
             dpdk_length = len(dpdk_bytes)
 
-        with open( basePath + linux_eeprom_file, 'rb') as fplinux:
+        with open( basePath + os.sep + linux_eeprom_file, 'rb') as fplinux:
             linux_bytes = fplinux.read()
             linux_length = len(linux_bytes)
         
         self.verify(dpdk_length <= linux_length, 
                     "linux ethtool haven't dump out enough data as dpdk ethtool")
 
-        with open( basePath + linux_eeprom_file, 'wb') as fplinux:
+        with open( basePath + os.sep + linux_eeprom_file, 'wb') as fplinux:
             fplinux.write(linux_bytes[:dpdk_length])
 
     def strip_md5(self, filename):
-        md5_info = self.dut.send_expect("md5sum %s" % filename, "# ")
+        md5_info = self.dut.send_expect("md5sum %s" % ( self.dut.base_dir + os.sep + filename), "# ", 30)
         md5_pattern = r"(\w+)  (\w+)"
         m = re.match(md5_pattern, md5_info)
         if m:
@@ -153,25 +156,120 @@ class TestUserspaceEthtool(TestCase, IxiaPacketGenerator):
         else:
             return ""
 
+    def get_driver_info(self, dpdk_driver_msg):
+        # get nic driver information using dpdk's ethtool
+        info_lines = dpdk_driver_msg.strip().splitlines()
+        driver_pattern = r"Port (\d+) driver: (.*) \(ver: (.*)\)"
+        pattern = "(.*): (.*)"
+        firmwarePat = "0x([0-9a-f]+)"
+        check_content = ['firmware-version', 'driver']
+        dpdk_drv_infos = {}
+        port = None
+        cnt = 0
+        while cnt < len(info_lines):
+            if not info_lines[cnt]:
+                pass
+            else:
+                m = re.match(driver_pattern, info_lines[cnt])
+                if m:
+                    port = m.group(1)
+                    dpdk_drv_infos[port] = {}
+                    dpdk_drv_infos[port]['driver'] = m.group(2).split('_').pop()
+                    dpdk_version = m.group(3)
+                else:
+                    if port:
+                        out = re.findall(pattern, info_lines[cnt], re.M)[0]
+                        if len(out) == 2:
+                            if out[0] == 'firmware-version':
+                                dpdk_drv_infos[port][out[0]] = "0x" + re.findall(firmwarePat, out[1], re.M)[0]
+                            else:
+                                dpdk_drv_infos[port][out[0]] = out[1]
+            cnt += 1
+        # check driver content
+        msg = ''
+        if 'firmware-version' not in dpdk_driver_msg:
+            self.logger.warning('firmware-version query not supported by current dpdk version')
+            return dpdk_drv_infos, msg
+        for port_no in dpdk_drv_infos:
+            dpdk_drv_info = dpdk_drv_infos[port_no]
+            for item in check_content:
+                if item not in dpdk_drv_info.keys():
+                    msg = os.linesep.join([msg, "port {0} get {1} failed".format(port_no, item)])
+                    dpdk_drv_infos = None
+                    break
+
+        return dpdk_drv_infos, msg
+
+    def check_driver_info(self, port_name, dpdk_drv_info):
+        # get nic driver information using linux's ethtool
+        pattern = "(.*): (.*)"
+        firmwarePat = "0x([0-9a-f]+)"
+        handle = os.popen("ethtool -i %s"%port_name)
+        infos = handle.readlines()
+        handle.close()
+        sys_nic_info = {}
+        for info in infos:
+            if not info:
+                continue
+            result = re.findall(pattern, info, re.M)
+            if not result:
+                continue
+            out = result[0]
+            if len(out) == 2:
+                if out[0] == 'firmware-version':
+                    sys_nic_info[out[0]] = "0x" + re.findall(firmwarePat, out[1], re.M)[0]
+                else:
+                    sys_nic_info[out[0]] = out[1]
+        # compare two information data 
+        for item, value in dpdk_drv_info.items():
+            if item not in sys_nic_info.keys():
+                msg = "linux ethtool failed to dump driver info"
+                status = False
+                break
+            if value != sys_nic_info[item]:
+                msg = "Userspace ethtool failed to dump driver info"
+                status = False
+                break
+        else:
+            msg = "{0}: dpdk ethtool dump driver info done".format(port_name)
+            status = True
+
+        return status, msg
+
     def test_dump_driver_info(self):
         """
         Test ethtool can dump basic information
         """
         self.dut.send_expect(self.cmd, "EthApp>", 60)
-        out = self.dut.send_expect("drvinfo", "EthApp>")
-        driver_pattern = r"Port (\d+) driver: rte_(.*)_pmd \(ver: RTE (.*)\)"
-        driver_infos = out.split("\r\n")
-        self.verify(len(driver_infos) > 1, "Userspace tool failed to dump driver infor")
-
-        # check dump driver info function
-        for driver_info in driver_infos:
-            m = re.match(driver_pattern, driver_info)
-            if m:
-                port = m.group(1)
-                driver = m.group(2)
-                version = m.group(3)
-                print utils.GREEN("Detect port %s with %s driver\n" % (port, driver))
+        dpdk_driver_msg = self.dut.send_expect("drvinfo", "EthApp>")
+        self.dut.send_expect("quit", "# ")
+        dpdk_drv_infos, msg = self.get_driver_info(dpdk_driver_msg)
+        self.verify(dpdk_drv_infos, msg)
+        
+        portsinfo = {}
+        for index in range(len(self.ports)):
+            portsinfo[index] = {}
+            portinfo = portsinfo[index]
+            port = self.ports[index]
+            netdev = self.dut.ports_info[port]['port']
+            intf_name = self.dut.ports_info[port]['intf']
+            # strip orignal driver
+            portinfo['ori_driver'] = netdev.get_nic_driver()
+            portinfo['net_dev'] = netdev
+            # bind to default driver
+            netdev.bind_driver()
+            # get linux interface
+            intf = netdev.get_interface_name()
+            status, msg = self.check_driver_info(intf_name, dpdk_drv_infos[str(index)])
+            self.logger.info(msg)
+            self.verify(status, msg)
 
+        for index in range(len(self.ports)):
+            # bind to original driver
+            portinfo = portsinfo[index]
+            portinfo['net_dev'].bind_driver(portinfo['ori_driver'])
+
+        self.dut.send_expect(self.cmd, "EthApp>", 60)
         # ethtool doesn't support port disconnect by tools of linux 
         # only detect physical link disconnect status
         if self.nic.startswith("fortville") == False:  
-- 
1.9.3

Re: [dts] [PATCH V1 2/2] userspace_ethtool: update automation testing script.

[Liu, Yong]

Yufen,
Some comments below.

Thanks,
Marvin

> -----Original Message-----
> From: dts [mailto:dts-bounces@dpdk.org] On Behalf Of yufengx.mo@intel.com
> Sent: Monday, January 16, 2017 11:49 AM
> To: dts@dpdk.org
> Cc: Mo, YufengX <yufengx.mo@intel.com>
> Subject: [dts] [PATCH V1 2/2] userspace_ethtool: update automation testing
> script.
> 
> From: yufengmx <yufengx.mo@intel.com>
> 
> 
> *. add nic's firmware/bus-info information query checking, which is
> corresponding with Qiming Yang's patchset 18597-18600(accepted) patchset
> 18601(accepted)
> 
> Signed-off-by: yufengmx <yufengx.mo@intel.com>
> ---
>  tests/TestSuite_userspace_ethtool.py | 119
> +++++++++++++++++++++++++++++++----
>  1 file changed, 106 insertions(+), 13 deletions(-)
> 
> diff --git a/tests/TestSuite_userspace_ethtool.py
> b/tests/TestSuite_userspace_ethtool.py
> index ced7e96..6024b87 100644
> --- a/tests/TestSuite_userspace_ethtool.py
> +++ b/tests/TestSuite_userspace_ethtool.py
> @@ -153,25 +153,118 @@ class TestUserspaceEthtool(TestCase,
> IxiaPacketGenerator):
>          else:
>              return ""
> 
> +    def get_driver_info(self, dpdk_driver_msg):
> +        # get nic driver information using dpdk's ethtool
> +        info_lines = dpdk_driver_msg.strip().splitlines()
> +        driver_pattern = r"Port (\d+) driver: (.*) \(ver: (.*)\)"
> +        pattern = "(.*): (.*)"
> +        firmwarePat = "0x([0-9a-f]+)"
> +        check_content = ['firmware-version', 'driver']
> +        dpdk_drv_infos = {}
> +        port = None
> +        cnt = 0
> +        while cnt < len(info_lines):
> +            if not info_lines[cnt]:
> +                pass
> +            else:
> +                m = re.match(driver_pattern, info_lines[cnt])
> +                if m:
> +                    port = m.group(1)
> +                    dpdk_drv_infos[port] = {}
> +                    dpdk_drv_infos[port]['driver'] = m.group(2).split('_').pop()
> +                    dpdk_version = m.group(3)
> +                else:
> +                    if port:
> +                        out = re.findall(pattern, info_lines[cnt], re.M)[0]
> +                        if len(out) == 2:
> +                            if out[0] == 'firmware-version':
> +                                dpdk_drv_infos[port][out[0]] = "0x" + re.findall(firmwarePat,
> out[1], re.M)[0]
> +                            else:
> +                                dpdk_drv_infos[port][out[0]] = out[1]
> +            cnt += 1
> +        # check driver content
> +        msg = ''
> +        if 'firmware-version' not in dpdk_driver_msg:
> +            self.logger.warning('firmware-version query not supported by current
> dpdk version')
> +            return dpdk_drv_infos, msg


Item ['firmware-version', 'driver'] should be existed in dpdk_drv_infos. Case should not consider tested code  whether unsupported itself.
We will use excel file for tracking this.

> +        for port_no in dpdk_drv_infos:
> +            dpdk_drv_info = dpdk_drv_infos[port_no]
> +            for item in check_content:
> +                if item not in dpdk_drv_info.keys():
> +                    msg = os.linesep.join([msg, "port {0} get {1} failed".format(port_no,
> item)])
> +                    dpdk_drv_infos = None
> +                    break
> +
> +        return dpdk_drv_infos, msg
> +
> +    def check_driver_info(self, port_name, dpdk_drv_info):
> +        # get nic driver information using linux's ethtool
> +        pattern = "(.*): (.*)"
> +        firmwarePat = "0x([0-9a-f]+)"
> +        infos = self.dut.send_expect("ethtool -i %s"%port_name, "# ").splitlines()

Not satisfy with pep8 rule, please check with python-pep8 first.

> +        sys_nic_info = {}
> +        for info in infos:
> +            if not info:
> +                continue
> +            result = re.findall(pattern, info, re.M)
> +            if not result:
> +                continue
> +            out = result[0]
> +            if len(out) == 2:
> +                if out[0] == 'firmware-version':
> +                    sys_nic_info[out[0]] = "0x" + re.findall(firmwarePat, out[1], re.M)[0]
> +                else:
> +                    sys_nic_info[out[0]] = out[1]


Suggest to split into two functions. One is that strip driver information, another is compare function.

> +        # compare two information data
> +        for item, value in dpdk_drv_info.items():
> +            if item not in sys_nic_info.keys():
> +                msg = "linux ethtool failed to dump driver info"
> +                status = False
> +                break
> +            if value != sys_nic_info[item]:
> +                msg = "Userspace ethtool failed to dump driver info"
> +                status = False
> +                break
> +        else:
> +            msg = "{0}: dpdk ethtool dump driver info done".format(port_name)
> +            status = True
> +
> +        return status, msg
> +
>      def test_dump_driver_info(self):
>          """
>          Test ethtool can dump basic information
>          """
>          self.dut.send_expect(self.cmd, "EthApp>", 60)
> -        out = self.dut.send_expect("drvinfo", "EthApp>")
> -        driver_pattern = r"Port (\d+) driver: rte_(.*)_pmd \(ver: RTE (.*)\)"
> -        driver_infos = out.split("\r\n")
> -        self.verify(len(driver_infos) > 1, "Userspace tool failed to dump driver infor")
> -
> -        # check dump driver info function
> -        for driver_info in driver_infos:
> -            m = re.match(driver_pattern, driver_info)
> -            if m:
> -                port = m.group(1)
> -                driver = m.group(2)
> -                version = m.group(3)
> -                print utils.GREEN("Detect port %s with %s driver\n" % (port, driver))
> +        dpdk_driver_msg = self.dut.send_expect("drvinfo", "EthApp>")
> +        self.dut.send_expect("quit", "# ")
> +        dpdk_drv_infos, msg = self.get_driver_info(dpdk_driver_msg)
> +        self.verify(dpdk_drv_infos, msg)
> +
> +        portsinfo = {}
> +        for index in range(len(self.ports)):
> +            portsinfo[index] = {}
> +            portinfo = portsinfo[index]
> +            port = self.ports[index]
> +            netdev = self.dut.ports_info[port]['port']
> +            intf_name = self.dut.ports_info[port]['intf']
> +            # strip orignal driver
> +            portinfo['ori_driver'] = netdev.get_nic_driver()
> +            portinfo['net_dev'] = netdev
> +            # bind to default driver
> +            netdev.bind_driver()
> +            # get linux interface
> +            intf = netdev.get_interface_name()
> +            status, msg = self.check_driver_info(intf_name,
> dpdk_drv_infos[str(index)])
> +            self.logger.info(msg)
> +            self.verify(status, msg)
> 
> +        for index in range(len(self.ports)):
> +            # bind to original driver
> +            portinfo = portsinfo[index]
> +            portinfo['net_dev'].bind_driver(portinfo['ori_driver'])
> +
> +        self.dut.send_expect(self.cmd, "EthApp>", 60)
>          # ethtool doesn't support port disconnect by tools of linux
>          # only detect physical link disconnect status
>          if self.nic.startswith("fortville") == False:
> --
> 1.9.3

[dts] [PATCH V1 2/2] userspace_ethtool: update automation testing script.

[yufengx.mo]

From: yufengmx <yufengx.mo@intel.com>


*. add nic's firmware/bus-info information query checking, which is corresponding with
Qiming Yang's
patchset 18597-18600(accepted)
patchset 18601(accepted)

Signed-off-by: yufengmx <yufengx.mo@intel.com>
---
 tests/TestSuite_userspace_ethtool.py | 119 +++++++++++++++++++++++++++++++----
 1 file changed, 106 insertions(+), 13 deletions(-)

diff --git a/tests/TestSuite_userspace_ethtool.py b/tests/TestSuite_userspace_ethtool.py
index ced7e96..6024b87 100644
--- a/tests/TestSuite_userspace_ethtool.py
+++ b/tests/TestSuite_userspace_ethtool.py
@@ -153,25 +153,118 @@ class TestUserspaceEthtool(TestCase, IxiaPacketGenerator):
         else:
             return ""
 
+    def get_driver_info(self, dpdk_driver_msg):
+        # get nic driver information using dpdk's ethtool
+        info_lines = dpdk_driver_msg.strip().splitlines()
+        driver_pattern = r"Port (\d+) driver: (.*) \(ver: (.*)\)"
+        pattern = "(.*): (.*)"
+        firmwarePat = "0x([0-9a-f]+)"
+        check_content = ['firmware-version', 'driver']
+        dpdk_drv_infos = {}
+        port = None
+        cnt = 0
+        while cnt < len(info_lines):
+            if not info_lines[cnt]:
+                pass
+            else:
+                m = re.match(driver_pattern, info_lines[cnt])
+                if m:
+                    port = m.group(1)
+                    dpdk_drv_infos[port] = {}
+                    dpdk_drv_infos[port]['driver'] = m.group(2).split('_').pop()
+                    dpdk_version = m.group(3)
+                else:
+                    if port:
+                        out = re.findall(pattern, info_lines[cnt], re.M)[0]
+                        if len(out) == 2:
+                            if out[0] == 'firmware-version':
+                                dpdk_drv_infos[port][out[0]] = "0x" + re.findall(firmwarePat, out[1], re.M)[0]
+                            else:
+                                dpdk_drv_infos[port][out[0]] = out[1]
+            cnt += 1
+        # check driver content
+        msg = ''
+        if 'firmware-version' not in dpdk_driver_msg:
+            self.logger.warning('firmware-version query not supported by current dpdk version')
+            return dpdk_drv_infos, msg
+        for port_no in dpdk_drv_infos:
+            dpdk_drv_info = dpdk_drv_infos[port_no]
+            for item in check_content:
+                if item not in dpdk_drv_info.keys():
+                    msg = os.linesep.join([msg, "port {0} get {1} failed".format(port_no, item)])
+                    dpdk_drv_infos = None
+                    break
+
+        return dpdk_drv_infos, msg
+
+    def check_driver_info(self, port_name, dpdk_drv_info):
+        # get nic driver information using linux's ethtool
+        pattern = "(.*): (.*)"
+        firmwarePat = "0x([0-9a-f]+)"
+        infos = self.dut.send_expect("ethtool -i %s"%port_name, "# ").splitlines()
+        sys_nic_info = {}
+        for info in infos:
+            if not info:
+                continue
+            result = re.findall(pattern, info, re.M)
+            if not result:
+                continue
+            out = result[0]
+            if len(out) == 2:
+                if out[0] == 'firmware-version':
+                    sys_nic_info[out[0]] = "0x" + re.findall(firmwarePat, out[1], re.M)[0]
+                else:
+                    sys_nic_info[out[0]] = out[1]
+        # compare two information data 
+        for item, value in dpdk_drv_info.items():
+            if item not in sys_nic_info.keys():
+                msg = "linux ethtool failed to dump driver info"
+                status = False
+                break
+            if value != sys_nic_info[item]:
+                msg = "Userspace ethtool failed to dump driver info"
+                status = False
+                break
+        else:
+            msg = "{0}: dpdk ethtool dump driver info done".format(port_name)
+            status = True
+
+        return status, msg
+
     def test_dump_driver_info(self):
         """
         Test ethtool can dump basic information
         """
         self.dut.send_expect(self.cmd, "EthApp>", 60)
-        out = self.dut.send_expect("drvinfo", "EthApp>")
-        driver_pattern = r"Port (\d+) driver: rte_(.*)_pmd \(ver: RTE (.*)\)"
-        driver_infos = out.split("\r\n")
-        self.verify(len(driver_infos) > 1, "Userspace tool failed to dump driver infor")
-
-        # check dump driver info function
-        for driver_info in driver_infos:
-            m = re.match(driver_pattern, driver_info)
-            if m:
-                port = m.group(1)
-                driver = m.group(2)
-                version = m.group(3)
-                print utils.GREEN("Detect port %s with %s driver\n" % (port, driver))
+        dpdk_driver_msg = self.dut.send_expect("drvinfo", "EthApp>")
+        self.dut.send_expect("quit", "# ")
+        dpdk_drv_infos, msg = self.get_driver_info(dpdk_driver_msg)
+        self.verify(dpdk_drv_infos, msg)
+        
+        portsinfo = {}
+        for index in range(len(self.ports)):
+            portsinfo[index] = {}
+            portinfo = portsinfo[index]
+            port = self.ports[index]
+            netdev = self.dut.ports_info[port]['port']
+            intf_name = self.dut.ports_info[port]['intf']
+            # strip orignal driver
+            portinfo['ori_driver'] = netdev.get_nic_driver()
+            portinfo['net_dev'] = netdev
+            # bind to default driver
+            netdev.bind_driver()
+            # get linux interface
+            intf = netdev.get_interface_name()
+            status, msg = self.check_driver_info(intf_name, dpdk_drv_infos[str(index)])
+            self.logger.info(msg)
+            self.verify(status, msg)
 
+        for index in range(len(self.ports)):
+            # bind to original driver
+            portinfo = portsinfo[index]
+            portinfo['net_dev'].bind_driver(portinfo['ori_driver'])
+
+        self.dut.send_expect(self.cmd, "EthApp>", 60)
         # ethtool doesn't support port disconnect by tools of linux 
         # only detect physical link disconnect status
         if self.nic.startswith("fortville") == False:  
-- 
1.9.3