[RFC PATCH v1 11/15] dts: merge DTS framework/project_dpdk.py to DPDK

["Juraj Linkeš" <juraj.linkes@pantheon.tech>]

---
 dts/framework/project_dpdk.py | 627 ++++++++++++++++++++++++++++++++++
 1 file changed, 627 insertions(+)
 create mode 100644 dts/framework/project_dpdk.py

diff --git a/dts/framework/project_dpdk.py b/dts/framework/project_dpdk.py
new file mode 100644
index 0000000000..4104e016a3
--- /dev/null
+++ b/dts/framework/project_dpdk.py
@@ -0,0 +1,627 @@
+# BSD LICENSE
+#
+# Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+#   * Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#   * Redistributions in binary form must reproduce the above copyright
+#     notice, this list of conditions and the following disclaimer in
+#     the documentation and/or other materials provided with the
+#     distribution.
+#   * Neither the name of Intel Corporation nor the names of its
+#     contributors may be used to endorse or promote products derived
+#     from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import os
+import re
+
+from .crb import Crb
+from .dut import Dut
+from .logger import getLogger
+from .settings import (
+    CONFIG_ROOT_PATH,
+    DPDK_RXMODE_SETTING,
+    DRIVERS,
+    HOST_DRIVER_MODE_SETTING,
+    HOST_DRIVER_SETTING,
+    HOST_SHARED_LIB_PATH,
+    HOST_SHARED_LIB_SETTING,
+    NICS,
+    accepted_nic,
+    load_global_setting,
+    save_global_setting,
+)
+from .ssh_connection import SSHConnection
+from .tester import Tester
+from .utils import RED
+
+
+class DPDKdut(Dut):
+
+    """
+    DPDK project class for DUT. DTS will call set_target function to setup
+    build, memory and kernel module.
+    """
+
+    def __init__(self, crb, serializer, dut_id=0, name=None, alt_session=True):
+        super(DPDKdut, self).__init__(crb, serializer, dut_id, name, alt_session)
+        self.testpmd = None
+
+    def set_target(self, target, bind_dev=True):
+        """
+        Set env variable, these have to be setup all the time. Some tests
+        need to compile example apps by themselves and will fail otherwise.
+        Set hugepage on DUT and install modules required by DPDK.
+        Configure default ixgbe PMD function.
+        """
+        self.target = target
+
+        self.set_toolchain(target)
+
+        # set env variable
+        self.set_env_variable()
+
+        self.set_rxtx_mode()
+
+        self.apps_name = self.apps_name_conf["meson"]
+        # use the dut target directory instead of 'target' string in app name
+        for app in self.apps_name:
+            cur_app_path = self.apps_name[app].replace("target", self.target)
+            self.apps_name[app] = cur_app_path + " "
+
+        if not self.skip_setup:
+            self.build_install_dpdk(target)
+
+        self.setup_memory()
+
+        drivername = load_global_setting(HOST_DRIVER_SETTING)
+        drivermode = load_global_setting(HOST_DRIVER_MODE_SETTING)
+        self.setup_modules(target, drivername, drivermode)
+
+        if bind_dev and self.get_os_type() == "linux":
+            self.bind_interfaces_linux(drivername)
+        self.extra_nic_setup()
+
+    def set_env_variable(self):
+        # These have to be setup all the time. Some tests need to compile
+        # example apps by themselves and will fail otherwise.
+        self.send_expect("export RTE_TARGET=" + self.target, "#")
+        self.send_expect("export RTE_SDK=`pwd`", "#")
+
+    def setup_modules(self, target, drivername, drivermode):
+        """
+        Install DPDK required kernel module on DUT.
+        """
+        setup_modules = getattr(self, "setup_modules_%s" % self.get_os_type())
+        setup_modules(target, drivername, drivermode)
+
+    def setup_modules_linux(self, target, drivername, drivermode):
+        if drivername == "vfio-pci":
+            self.send_expect("rmmod vfio_pci", "#")
+            self.send_expect("rmmod vfio_iommu_type1", "#")
+            self.send_expect("rmmod vfio", "#")
+            self.send_expect("modprobe vfio", "#")
+            self.send_expect("modprobe vfio-pci", "#")
+            if drivermode == "noiommu":
+                self.send_expect(
+                    "echo 1 > /sys/module/vfio/parameters/enable_unsafe_noiommu_mode",
+                    "#",
+                )
+            out = self.send_expect("ls /sys/module|grep vfio_pci", "#")
+            assert "vfio_pci" in out, "load vfio_pci failed"
+
+        elif drivername == "uio_pci_generic":
+            self.send_expect("modprobe uio", "#", 70)
+            self.send_expect("modprobe uio_pci_generic", "#", 70)
+            out = self.send_expect("lsmod | grep uio_pci_generic", "#")
+            assert "uio_pci_generic" in out, "Failed to setup uio_pci_generic"
+
+        elif drivername == "mlx5_core":
+            pass
+
+        elif drivername == "igb_uio":
+            self.send_expect("modprobe uio", "#", 70)
+            out = self.send_expect("lsmod | grep igb_uio", "#")
+            if "igb_uio" in out:
+                self.send_expect("rmmod -f igb_uio", "#", 70)
+            self.send_expect("insmod ./" + target + "/kmod/igb_uio.ko", "#", 60)
+
+            out = self.send_expect("lsmod | grep igb_uio", "#")
+            assert "igb_uio" in out, "Failed to insmod igb_uio"
+
+        else:
+            pass
+
+    def setup_modules_freebsd(self, target, drivername, drivermode):
+        """
+        Install DPDK required Freebsd kernel module on DUT.
+        """
+        binding_list = ""
+
+        for (pci_bus, pci_id) in self.pci_devices_info:
+            if accepted_nic(pci_id):
+                binding_list += "%s," % (pci_bus)
+
+        self.send_expect("kldunload if_ixgbe.ko", "#")
+        self.send_expect('kenv hw.nic_uio.bdfs="%s"' % binding_list[:-1], "# ")
+        self.send_expect("kldload ./%s/kmod/nic_uio.ko" % target, "#", 20)
+        out = self.send_expect("kldstat", "#")
+        assert "nic_uio" in out, "Failed to insmod nic_uio"
+
+    def restore_modules(self):
+        """
+        Restore DPDK kernel module on DUT.
+        """
+        restore_modules = getattr(self, "restore_modules_%s" % self.get_os_type())
+        restore_modules()
+
+    def restore_modules_linux(self):
+        """
+        Restore DPDK Linux kernel module on DUT.
+        """
+        drivername = load_global_setting(HOST_DRIVER_SETTING)
+        if drivername == "vfio-pci":
+            drivermode = load_global_setting(HOST_DRIVER_MODE_SETTING)
+            if drivermode == "noiommu":
+                self.send_expect(
+                    "echo 0 > /sys/module/vfio/parameters/enable_unsafe_noiommu_mode",
+                    "#",
+                    70,
+                )
+
+    def restore_modules_freebsd(self):
+        """
+        Restore DPDK Freebsd kernel module on DUT.
+        """
+        pass
+
+    def set_rxtx_mode(self):
+        """
+        Set default RX/TX PMD function,
+        the rx mode scalar/full/novector are supported dynamically since DPDK 20.11,
+        Rx mode avx512 is only supported in DPDK 20.11 and later version.
+        """
+
+        mode = load_global_setting(DPDK_RXMODE_SETTING)
+        if mode == "avx512":
+            out = self.send_expect("lscpu | grep avx512", "#")
+            if "avx512f" not in out or "no-avx512f" in out:
+                self.logger.warning(
+                    RED("*********The DUT CPU do not support AVX512 test!!!********")
+                )
+                self.logger.warning(
+                    RED("*********Now set the rx_mode to default!!!**********")
+                )
+                save_global_setting(DPDK_RXMODE_SETTING, "default")
+
+    def set_package(self, pkg_name="", patch_list=[]):
+        self.package = pkg_name
+        self.patches = patch_list
+
+    def set_build_options(self, config_parms, config_file=""):
+        self.set_build_options_meson(config_parms, config_file)
+
+    def set_build_options_meson(self, config_parms, config_file=""):
+        """
+        Set dpdk build options of meson
+        """
+        if len(config_parms) == 0:
+            return
+        if config_file == "":
+            config_file = "config/rte_config.h"
+
+        for key in config_parms.keys():
+            value = config_parms[key]
+            if value == "n":
+                def_str = "#undef" + " " + key
+            else:
+                if isinstance(value, int):
+                    def_str = "#define %s %d" % (key, value)
+                elif value == "" or value == "y":
+                    def_str = "#define %s %d" % (key, 1)
+                else:
+                    value = '\\"%s\\"' % value
+                    def_str = "#define %s %s" % (key, value)
+
+            # delete the marco define in the config file
+            self.send_expect("sed -i '/%s/d' %s" % (key, config_file), "# ")
+            self.send_expect("sed -i '$a\%s' %s" % (def_str, config_file), "# ")
+
+    def build_install_dpdk(self, target, extra_options=""):
+        """
+        Build DPDK source code with specified target.
+        """
+        use_shared_lib = load_global_setting(HOST_SHARED_LIB_SETTING)
+        if use_shared_lib == "true" and "Virt" not in str(self):
+            extra_options += " -Dc_args=-DRTE_BUILD_SHARED_LIB"
+        if self.get_os_type() == "linux":
+            self.build_install_dpdk_linux_meson(target, extra_options)
+        elif self.get_os_type() == "freebsd":
+            self.build_install_dpdk_freebsd_meson(target, extra_options)
+
+    def build_install_dpdk_linux_meson(self, target, extra_options):
+        """
+        Build DPDK source code on linux use meson
+        """
+        build_time = 1800
+        target_info = target.split("-")
+        arch = target_info[0]
+        machine = target_info[1]
+        execenv = target_info[2]
+        toolchain = target_info[3]
+
+        default_library = "static"
+        use_shared_lib = load_global_setting(HOST_SHARED_LIB_SETTING)
+        if use_shared_lib == "true" and "Virt" not in str(self):
+            default_library = "shared"
+        if arch == "i686":
+            # find the pkg-config path and set the PKG_CONFIG_LIBDIR environmental variable to point it
+            out = self.send_expect("find /usr -type d -name pkgconfig", "# ")
+            pkg_path = ""
+            default_cflags = self.send_expect("echo $CFLAGS", "# ")
+            default_pkg_config = self.send_expect("echo $PKG_CONFIG_LIBDIR", "# ")
+            res_path = out.split("\r\n")
+            for cur_path in res_path:
+                if "i386" in cur_path:
+                    pkg_path = cur_path
+                    break
+            assert (
+                pkg_path != ""
+            ), "please make sure you env have the i386 pkg-config path"
+
+            self.send_expect("export CFLAGS=-m32", "# ")
+            self.send_expect("export PKG_CONFIG_LIBDIR=%s" % pkg_path, "# ")
+
+        self.send_expect("rm -rf " + target, "#")
+        out = self.send_expect(
+            "CC=%s meson -Denable_kmods=True -Dlibdir=lib %s --default-library=%s %s"
+            % (toolchain, extra_options, default_library, target),
+            "[~|~\]]# ",
+            build_time,
+        )
+        assert "FAILED" not in out, "meson setup failed ..."
+
+        out = self.send_expect("ninja -C %s" % target, "[~|~\]]# ", build_time)
+        assert "FAILED" not in out, "ninja complie failed ..."
+
+        # copy kmod file to the folder same as make
+        out = self.send_expect(
+            "find ./%s/kernel/ -name *.ko" % target, "# ", verify=True
+        )
+        self.send_expect("mkdir -p %s/kmod" % target, "# ")
+        if not isinstance(out, int) and len(out) > 0:
+            kmod = out.split("\r\n")
+            for mod in kmod:
+                self.send_expect("cp %s %s/kmod/" % (mod, target), "# ")
+
+    def build_install_dpdk_freebsd_meson(self, target, extra_options):
+        # meson build same as linux
+        self.build_install_dpdk_linux_meson(target, extra_options)
+
+        # the uio name different with linux, find the nic_uio
+        out = self.send_expect(
+            "find ./%s/kernel/ -name nic_uio" % target, "# ", verify=True
+        )
+        self.send_expect("mkdir -p %s/kmod" % target, "# ")
+        if not isinstance(out, int) and len(out) > 0:
+            self.send_expect("cp %s %s/kmod/" % (out, target), "# ")
+
+    def prepare_package(self):
+        if not self.skip_setup:
+            session_info = None
+            # if snapshot_load_side=dut, will copy the dpdk tar from dut side
+            # and will judge whether the path of tar is existed on dut
+            if self.crb["snapshot_load_side"] == "dut":
+                if not os.path.isabs(self.package):
+                    raise ValueError(
+                        "As snapshot_load_side=dut, will copy dpdk.tar "
+                        "from dut, please specify a abs path use params "
+                        "--snapshot when run dts"
+                    )
+                # if ':' in session, this is vm dut, use the dut session
+                if ":" in self.session.name:
+                    session_info = self.host_session
+                else:
+                    session_info = self.alt_session
+                out = session_info.send_expect("ls -F %s" % self.package, "# ")
+                assert out == self.package, "Invalid package"
+            else:
+                assert os.path.isfile(self.package) is True, "Invalid package"
+
+            p_dir, _ = os.path.split(self.base_dir)
+            # ToDo: make this configurable
+            dst_dir = "/tmp/"
+
+            out = self.send_expect("ls -d %s" % p_dir, "# ", verify=True)
+            if out == 2:
+                self.send_expect("mkdir -p %s" % p_dir, "# ")
+
+            out = self.send_expect(
+                "ls %s && cd %s" % (dst_dir, p_dir), "#", verify=True
+            )
+            if out == -1:
+                raise ValueError(
+                    "Directory %s or %s does not exist,"
+                    "please check params -d" % (p_dir, dst_dir)
+                )
+            self.session.copy_file_to(self.package, dst_dir, crb_session=session_info)
+
+            # put patches to p_dir/patches/
+            if self.patches is not None:
+                for p in self.patches:
+                    self.session.copy_file_to("dep/" + p, dst_dir)
+
+            # copy QMP file to dut
+            if ":" not in self.session.name:
+                out = self.send_expect("ls -d ~/QMP", "# ", verify=True)
+                if isinstance(out, int):
+                    self.send_expect("mkdir -p ~/QMP", "# ")
+                self.session.copy_file_to("dep/QMP/qemu-ga-client", "~/QMP/")
+                self.session.copy_file_to("dep/QMP/qmp.py", "~/QMP/")
+            self.kill_all()
+
+            # enable core dump
+            self.send_expect("ulimit -c unlimited", "#")
+
+            # unpack the code and change to the working folder
+            self.send_expect("rm -rf %s" % self.base_dir, "#")
+
+            # unpack dpdk
+            out = self.send_expect(
+                "tar zxfm %s%s -C %s" % (dst_dir, self.package.split("/")[-1], p_dir),
+                "# ",
+                60,
+                verify=True,
+            )
+            if out == -1:
+                raise ValueError(
+                    "Extract dpdk package to %s failure,"
+                    "please check params -d" % (p_dir)
+                )
+
+            # check dpdk dir name is expect
+            out = self.send_expect("ls %s" % self.base_dir, "# ", 20, verify=True)
+            if out == -1:
+                raise ValueError(
+                    "dpdk dir %s mismatch, please check params -d" % self.base_dir
+                )
+
+            if self.patches is not None:
+                for p in self.patches:
+                    out = self.send_expect(
+                        "patch -d %s -p1 < %s" % (self.base_dir, dst_dir + p), "# "
+                    )
+                    assert "****" not in out
+
+    def prerequisites(self):
+        """
+        Copy DPDK package to DUT and apply patch files.
+        """
+        self.prepare_package()
+        self.dut_prerequisites()
+        self.stage = "post-init"
+
+    def extra_nic_setup(self):
+        """
+        Some nic like RRC required additional setup after module installed
+        """
+        for port_info in self.ports_info:
+            netdev = port_info["port"]
+            netdev.setup()
+
+    def bind_interfaces_linux(self, driver="igb_uio", nics_to_bind=None):
+        """
+        Bind the interfaces to the selected driver. nics_to_bind can be None
+        to bind all interfaces or an array with the port indexes
+        """
+        binding_list = "--bind=%s " % driver
+
+        current_nic = 0
+        for port_info in self.ports_info:
+            if nics_to_bind is None or current_nic in nics_to_bind:
+                binding_list += "%s " % (port_info["pci"])
+            current_nic += 1
+
+        bind_script_path = self.get_dpdk_bind_script()
+        return self.send_expect(
+            "%s --force %s" % (bind_script_path, binding_list), "# "
+        )
+
+    def unbind_interfaces_linux(self, nics_to_bind=None):
+        """
+        Unbind the interfaces
+        """
+
+        binding_list = "-u "
+
+        current_nic = 0
+        for port_info in self.ports_info:
+            if nics_to_bind is None or current_nic in nics_to_bind:
+                binding_list += "%s " % (port_info["pci"])
+            current_nic += 1
+
+        bind_script_path = self.get_dpdk_bind_script()
+        self.send_expect("%s --force %s" % (bind_script_path, binding_list), "# ")
+
+    def build_dpdk_apps(self, folder, extra_options=""):
+        """
+        Build dpdk sample applications.
+        """
+        if self.get_os_type() == "linux":
+            return self.build_dpdk_apps_linux_meson(folder, extra_options)
+        elif self.get_os_type() == "freebsd":
+            return self.build_dpdk_apps_freebsd_meson(folder, extra_options)
+
+    def build_dpdk_apps_linux_meson(self, folder, extra_options):
+        """
+        Build dpdk sample applications on linux use meson
+        """
+        # icc compile need more time
+        if "icc" in self.target:
+            timeout = 300
+        else:
+            timeout = 90
+
+        target_info = self.target.split("-")
+        arch = target_info[0]
+        if arch == "i686":
+            # find the pkg-config path and set the PKG_CONFIG_LIBDIR environmental variable to point it
+            out = self.send_expect("find /usr -type d -name pkgconfig", "# ")
+            pkg_path = ""
+            default_cflags = self.send_expect("echo $CFLAGS", "# ")
+            default_pkg_config = self.send_expect("echo $PKG_CONFIG_LIBDIR", "# ")
+            res_path = out.split("\r\n")
+            for cur_path in res_path:
+                if "i386" in cur_path:
+                    pkg_path = cur_path
+                    break
+            assert (
+                pkg_path != ""
+            ), "please make sure you env have the i386 pkg-config path"
+
+            self.send_expect("export CFLAGS=-m32", "# ", alt_session=True)
+            self.send_expect(
+                "export PKG_CONFIG_LIBDIR=%s" % pkg_path, "# ", alt_session=True
+            )
+
+        folder_info = folder.split("/")
+        name = folder_info[-1]
+        if name != "examples" and name not in self.apps_name:
+            raise Exception(
+                f'Please config {name} file path on {os.path.join(CONFIG_ROOT_PATH, "app_name.cfg")}'
+            )
+
+        if name == "examples":
+            example = "all"
+        else:
+            example = "/".join(folder_info[folder_info.index("examples") + 1 :])
+        out = self.send_expect(
+            "meson configure -Dexamples=%s %s" % (example, self.target), "# "
+        )
+        assert "FAILED" not in out, "Compilation error..."
+        out = self.send_expect("ninja -C %s" % self.target, "[~|~\]]# ", timeout)
+        assert "FAILED" not in out, "Compilation error..."
+
+        # verify the app build in the config path
+        if example != "all":
+            out = self.send_expect("ls %s" % self.apps_name[name], "# ", verify=True)
+            assert isinstance(out, str), (
+                "please confirm %s app path and name in app_name.cfg" % name
+            )
+
+        return out
+
+    def build_dpdk_apps_freebsd_meson(self, folder, extra_options):
+        # meson build same as linux
+        return self.build_dpdk_apps_linux_meson(folder, extra_options)
+
+    def get_blocklist_string(self, target, nic):
+        """
+        Get block list command string.
+        """
+        get_blocklist_string = getattr(
+            self, "get_blocklist_string_%s" % self.get_os_type()
+        )
+        return get_blocklist_string(target, nic)
+
+    def get_blocklist_string_linux(self, target, nic):
+        """
+        Get block list command string on Linux.
+        """
+        blocklist = ""
+        dutPorts = self.get_ports(nic)
+        self.restore_interfaces()
+        self.send_expect("insmod ./%s/kmod/igb_uio.ko" % target, "# ")
+        self.bind_interfaces_linux()
+        for port in range(0, len(self.ports_info)):
+            if port not in dutPorts:
+                blocklist += "-b %s " % self.ports_info[port]["pci"]
+        return blocklist
+
+    def get_blocklist_string_freebsd(self, target, nic):
+        """
+        Get block list command string on Freebsd.
+        """
+        blocklist = ""
+        # No blocklist option in FreeBSD
+        return blocklist
+
+
+class DPDKtester(Tester):
+
+    """
+    DPDK project class for tester. DTS will call prerequisites function to setup
+    interface and generate port map.
+    """
+
+    def __init__(self, crb, serializer, dut_id):
+        self.NAME = "tester"
+        super(DPDKtester, self).__init__(crb, serializer)
+
+    def prerequisites(self, perf_test=False):
+        """
+        Setup hugepage on tester and copy validation required files to tester.
+        """
+        self.kill_all()
+
+        if not self.skip_setup:
+            total_huge_pages = self.get_total_huge_pages()
+            hugepages_size = self.send_expect(
+                "awk '/Hugepagesize/ {print $2}' /proc/meminfo", "# "
+            )
+            if total_huge_pages == 0:
+                self.mount_huge_pages()
+                if hugepages_size == "524288":
+                    self.set_huge_pages(8)
+                else:
+                    self.set_huge_pages(1024)
+
+            self.session.copy_file_to("dep/tgen.tgz")
+            self.session.copy_file_to("dep/tclclient.tgz")
+            # unpack tgen
+            out = self.send_expect("tar zxfm tgen.tgz", "# ")
+            assert "Error" not in out
+            # unpack tclclient
+            out = self.send_expect("tar zxfm tclclient.tgz", "# ")
+            assert "Error" not in out
+
+        self.send_expect("modprobe uio", "# ")
+
+        self.tester_prerequisites()
+
+        self.set_promisc()
+
+        self.stage = "post-init"
+
+    def setup_memory(self, hugepages=-1):
+        """
+        Setup hugepage on tester.
+        """
+        hugepages_size = self.send_expect(
+            "awk '/Hugepagesize/ {print $2}' /proc/meminfo", "# "
+        )
+
+        if int(hugepages_size) < (2048 * 2048):
+            arch_huge_pages = hugepages if hugepages > 0 else 2048
+            total_huge_pages = self.get_total_huge_pages()
+
+        self.mount_huge_pages()
+        if total_huge_pages != arch_huge_pages:
+            self.set_huge_pages(arch_huge_pages)
-- 
2.20.1