From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 4426BA0507; Wed, 6 Apr 2022 16:58:00 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 3F8DF428A8; Wed, 6 Apr 2022 16:56:34 +0200 (CEST) Received: from lb.pantheon.sk (lb.pantheon.sk [46.229.239.20]) by mails.dpdk.org (Postfix) with ESMTP id 92FE44288E for ; Wed, 6 Apr 2022 16:56:30 +0200 (CEST) Received: from localhost (localhost [127.0.0.1]) by lb.pantheon.sk (Postfix) with ESMTP id EEE9B1B1F6C; Wed, 6 Apr 2022 16:56:29 +0200 (CEST) X-Virus-Scanned: amavisd-new at siecit.sk Received: from lb.pantheon.sk ([127.0.0.1]) by localhost (lb.pantheon.sk [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id xsWheZYvLLQ6; Wed, 6 Apr 2022 16:56:28 +0200 (CEST) Received: from entguard.lab.pantheon.local (unknown [46.229.239.141]) by lb.pantheon.sk (Postfix) with ESMTP id 4BC6A1B1F79; Wed, 6 Apr 2022 16:56:12 +0200 (CEST) From: =?UTF-8?q?Juraj=20Linke=C5=A1?= To: thomas@monjalon.net, david.marchand@redhat.com, Honnappa.Nagarahalli@arm.com, ohilyard@iol.unh.edu, lijuan.tu@intel.com Cc: dev@dpdk.org, =?UTF-8?q?Juraj=20Linke=C5=A1?= Subject: [RFC PATCH v1 11/15] dts: merge DTS framework/project_dpdk.py to DPDK Date: Wed, 6 Apr 2022 14:56:02 +0000 Message-Id: <20220406145606.2913834-12-juraj.linkes@pantheon.tech> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20220406145606.2913834-1-juraj.linkes@pantheon.tech> References: <20220406145606.2913834-1-juraj.linkes@pantheon.tech> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org --- dts/framework/project_dpdk.py | 627 ++++++++++++++++++++++++++++++++++ 1 file changed, 627 insertions(+) create mode 100644 dts/framework/project_dpdk.py diff --git a/dts/framework/project_dpdk.py b/dts/framework/project_dpdk.py new file mode 100644 index 0000000000..4104e016a3 --- /dev/null +++ b/dts/framework/project_dpdk.py @@ -0,0 +1,627 @@ +# BSD LICENSE +# +# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import re + +from .crb import Crb +from .dut import Dut +from .logger import getLogger +from .settings import ( + CONFIG_ROOT_PATH, + DPDK_RXMODE_SETTING, + DRIVERS, + HOST_DRIVER_MODE_SETTING, + HOST_DRIVER_SETTING, + HOST_SHARED_LIB_PATH, + HOST_SHARED_LIB_SETTING, + NICS, + accepted_nic, + load_global_setting, + save_global_setting, +) +from .ssh_connection import SSHConnection +from .tester import Tester +from .utils import RED + + +class DPDKdut(Dut): + + """ + DPDK project class for DUT. DTS will call set_target function to setup + build, memory and kernel module. + """ + + def __init__(self, crb, serializer, dut_id=0, name=None, alt_session=True): + super(DPDKdut, self).__init__(crb, serializer, dut_id, name, alt_session) + self.testpmd = None + + def set_target(self, target, bind_dev=True): + """ + Set env variable, these have to be setup all the time. Some tests + need to compile example apps by themselves and will fail otherwise. + Set hugepage on DUT and install modules required by DPDK. + Configure default ixgbe PMD function. + """ + self.target = target + + self.set_toolchain(target) + + # set env variable + self.set_env_variable() + + self.set_rxtx_mode() + + self.apps_name = self.apps_name_conf["meson"] + # use the dut target directory instead of 'target' string in app name + for app in self.apps_name: + cur_app_path = self.apps_name[app].replace("target", self.target) + self.apps_name[app] = cur_app_path + " " + + if not self.skip_setup: + self.build_install_dpdk(target) + + self.setup_memory() + + drivername = load_global_setting(HOST_DRIVER_SETTING) + drivermode = load_global_setting(HOST_DRIVER_MODE_SETTING) + self.setup_modules(target, drivername, drivermode) + + if bind_dev and self.get_os_type() == "linux": + self.bind_interfaces_linux(drivername) + self.extra_nic_setup() + + def set_env_variable(self): + # These have to be setup all the time. Some tests need to compile + # example apps by themselves and will fail otherwise. + self.send_expect("export RTE_TARGET=" + self.target, "#") + self.send_expect("export RTE_SDK=`pwd`", "#") + + def setup_modules(self, target, drivername, drivermode): + """ + Install DPDK required kernel module on DUT. + """ + setup_modules = getattr(self, "setup_modules_%s" % self.get_os_type()) + setup_modules(target, drivername, drivermode) + + def setup_modules_linux(self, target, drivername, drivermode): + if drivername == "vfio-pci": + self.send_expect("rmmod vfio_pci", "#") + self.send_expect("rmmod vfio_iommu_type1", "#") + self.send_expect("rmmod vfio", "#") + self.send_expect("modprobe vfio", "#") + self.send_expect("modprobe vfio-pci", "#") + if drivermode == "noiommu": + self.send_expect( + "echo 1 > /sys/module/vfio/parameters/enable_unsafe_noiommu_mode", + "#", + ) + out = self.send_expect("ls /sys/module|grep vfio_pci", "#") + assert "vfio_pci" in out, "load vfio_pci failed" + + elif drivername == "uio_pci_generic": + self.send_expect("modprobe uio", "#", 70) + self.send_expect("modprobe uio_pci_generic", "#", 70) + out = self.send_expect("lsmod | grep uio_pci_generic", "#") + assert "uio_pci_generic" in out, "Failed to setup uio_pci_generic" + + elif drivername == "mlx5_core": + pass + + elif drivername == "igb_uio": + self.send_expect("modprobe uio", "#", 70) + out = self.send_expect("lsmod | grep igb_uio", "#") + if "igb_uio" in out: + self.send_expect("rmmod -f igb_uio", "#", 70) + self.send_expect("insmod ./" + target + "/kmod/igb_uio.ko", "#", 60) + + out = self.send_expect("lsmod | grep igb_uio", "#") + assert "igb_uio" in out, "Failed to insmod igb_uio" + + else: + pass + + def setup_modules_freebsd(self, target, drivername, drivermode): + """ + Install DPDK required Freebsd kernel module on DUT. + """ + binding_list = "" + + for (pci_bus, pci_id) in self.pci_devices_info: + if accepted_nic(pci_id): + binding_list += "%s," % (pci_bus) + + self.send_expect("kldunload if_ixgbe.ko", "#") + self.send_expect('kenv hw.nic_uio.bdfs="%s"' % binding_list[:-1], "# ") + self.send_expect("kldload ./%s/kmod/nic_uio.ko" % target, "#", 20) + out = self.send_expect("kldstat", "#") + assert "nic_uio" in out, "Failed to insmod nic_uio" + + def restore_modules(self): + """ + Restore DPDK kernel module on DUT. + """ + restore_modules = getattr(self, "restore_modules_%s" % self.get_os_type()) + restore_modules() + + def restore_modules_linux(self): + """ + Restore DPDK Linux kernel module on DUT. + """ + drivername = load_global_setting(HOST_DRIVER_SETTING) + if drivername == "vfio-pci": + drivermode = load_global_setting(HOST_DRIVER_MODE_SETTING) + if drivermode == "noiommu": + self.send_expect( + "echo 0 > /sys/module/vfio/parameters/enable_unsafe_noiommu_mode", + "#", + 70, + ) + + def restore_modules_freebsd(self): + """ + Restore DPDK Freebsd kernel module on DUT. + """ + pass + + def set_rxtx_mode(self): + """ + Set default RX/TX PMD function, + the rx mode scalar/full/novector are supported dynamically since DPDK 20.11, + Rx mode avx512 is only supported in DPDK 20.11 and later version. + """ + + mode = load_global_setting(DPDK_RXMODE_SETTING) + if mode == "avx512": + out = self.send_expect("lscpu | grep avx512", "#") + if "avx512f" not in out or "no-avx512f" in out: + self.logger.warning( + RED("*********The DUT CPU do not support AVX512 test!!!********") + ) + self.logger.warning( + RED("*********Now set the rx_mode to default!!!**********") + ) + save_global_setting(DPDK_RXMODE_SETTING, "default") + + def set_package(self, pkg_name="", patch_list=[]): + self.package = pkg_name + self.patches = patch_list + + def set_build_options(self, config_parms, config_file=""): + self.set_build_options_meson(config_parms, config_file) + + def set_build_options_meson(self, config_parms, config_file=""): + """ + Set dpdk build options of meson + """ + if len(config_parms) == 0: + return + if config_file == "": + config_file = "config/rte_config.h" + + for key in config_parms.keys(): + value = config_parms[key] + if value == "n": + def_str = "#undef" + " " + key + else: + if isinstance(value, int): + def_str = "#define %s %d" % (key, value) + elif value == "" or value == "y": + def_str = "#define %s %d" % (key, 1) + else: + value = '\\"%s\\"' % value + def_str = "#define %s %s" % (key, value) + + # delete the marco define in the config file + self.send_expect("sed -i '/%s/d' %s" % (key, config_file), "# ") + self.send_expect("sed -i '$a\%s' %s" % (def_str, config_file), "# ") + + def build_install_dpdk(self, target, extra_options=""): + """ + Build DPDK source code with specified target. + """ + use_shared_lib = load_global_setting(HOST_SHARED_LIB_SETTING) + if use_shared_lib == "true" and "Virt" not in str(self): + extra_options += " -Dc_args=-DRTE_BUILD_SHARED_LIB" + if self.get_os_type() == "linux": + self.build_install_dpdk_linux_meson(target, extra_options) + elif self.get_os_type() == "freebsd": + self.build_install_dpdk_freebsd_meson(target, extra_options) + + def build_install_dpdk_linux_meson(self, target, extra_options): + """ + Build DPDK source code on linux use meson + """ + build_time = 1800 + target_info = target.split("-") + arch = target_info[0] + machine = target_info[1] + execenv = target_info[2] + toolchain = target_info[3] + + default_library = "static" + use_shared_lib = load_global_setting(HOST_SHARED_LIB_SETTING) + if use_shared_lib == "true" and "Virt" not in str(self): + default_library = "shared" + if arch == "i686": + # find the pkg-config path and set the PKG_CONFIG_LIBDIR environmental variable to point it + out = self.send_expect("find /usr -type d -name pkgconfig", "# ") + pkg_path = "" + default_cflags = self.send_expect("echo $CFLAGS", "# ") + default_pkg_config = self.send_expect("echo $PKG_CONFIG_LIBDIR", "# ") + res_path = out.split("\r\n") + for cur_path in res_path: + if "i386" in cur_path: + pkg_path = cur_path + break + assert ( + pkg_path != "" + ), "please make sure you env have the i386 pkg-config path" + + self.send_expect("export CFLAGS=-m32", "# ") + self.send_expect("export PKG_CONFIG_LIBDIR=%s" % pkg_path, "# ") + + self.send_expect("rm -rf " + target, "#") + out = self.send_expect( + "CC=%s meson -Denable_kmods=True -Dlibdir=lib %s --default-library=%s %s" + % (toolchain, extra_options, default_library, target), + "[~|~\]]# ", + build_time, + ) + assert "FAILED" not in out, "meson setup failed ..." + + out = self.send_expect("ninja -C %s" % target, "[~|~\]]# ", build_time) + assert "FAILED" not in out, "ninja complie failed ..." + + # copy kmod file to the folder same as make + out = self.send_expect( + "find ./%s/kernel/ -name *.ko" % target, "# ", verify=True + ) + self.send_expect("mkdir -p %s/kmod" % target, "# ") + if not isinstance(out, int) and len(out) > 0: + kmod = out.split("\r\n") + for mod in kmod: + self.send_expect("cp %s %s/kmod/" % (mod, target), "# ") + + def build_install_dpdk_freebsd_meson(self, target, extra_options): + # meson build same as linux + self.build_install_dpdk_linux_meson(target, extra_options) + + # the uio name different with linux, find the nic_uio + out = self.send_expect( + "find ./%s/kernel/ -name nic_uio" % target, "# ", verify=True + ) + self.send_expect("mkdir -p %s/kmod" % target, "# ") + if not isinstance(out, int) and len(out) > 0: + self.send_expect("cp %s %s/kmod/" % (out, target), "# ") + + def prepare_package(self): + if not self.skip_setup: + session_info = None + # if snapshot_load_side=dut, will copy the dpdk tar from dut side + # and will judge whether the path of tar is existed on dut + if self.crb["snapshot_load_side"] == "dut": + if not os.path.isabs(self.package): + raise ValueError( + "As snapshot_load_side=dut, will copy dpdk.tar " + "from dut, please specify a abs path use params " + "--snapshot when run dts" + ) + # if ':' in session, this is vm dut, use the dut session + if ":" in self.session.name: + session_info = self.host_session + else: + session_info = self.alt_session + out = session_info.send_expect("ls -F %s" % self.package, "# ") + assert out == self.package, "Invalid package" + else: + assert os.path.isfile(self.package) is True, "Invalid package" + + p_dir, _ = os.path.split(self.base_dir) + # ToDo: make this configurable + dst_dir = "/tmp/" + + out = self.send_expect("ls -d %s" % p_dir, "# ", verify=True) + if out == 2: + self.send_expect("mkdir -p %s" % p_dir, "# ") + + out = self.send_expect( + "ls %s && cd %s" % (dst_dir, p_dir), "#", verify=True + ) + if out == -1: + raise ValueError( + "Directory %s or %s does not exist," + "please check params -d" % (p_dir, dst_dir) + ) + self.session.copy_file_to(self.package, dst_dir, crb_session=session_info) + + # put patches to p_dir/patches/ + if self.patches is not None: + for p in self.patches: + self.session.copy_file_to("dep/" + p, dst_dir) + + # copy QMP file to dut + if ":" not in self.session.name: + out = self.send_expect("ls -d ~/QMP", "# ", verify=True) + if isinstance(out, int): + self.send_expect("mkdir -p ~/QMP", "# ") + self.session.copy_file_to("dep/QMP/qemu-ga-client", "~/QMP/") + self.session.copy_file_to("dep/QMP/qmp.py", "~/QMP/") + self.kill_all() + + # enable core dump + self.send_expect("ulimit -c unlimited", "#") + + # unpack the code and change to the working folder + self.send_expect("rm -rf %s" % self.base_dir, "#") + + # unpack dpdk + out = self.send_expect( + "tar zxfm %s%s -C %s" % (dst_dir, self.package.split("/")[-1], p_dir), + "# ", + 60, + verify=True, + ) + if out == -1: + raise ValueError( + "Extract dpdk package to %s failure," + "please check params -d" % (p_dir) + ) + + # check dpdk dir name is expect + out = self.send_expect("ls %s" % self.base_dir, "# ", 20, verify=True) + if out == -1: + raise ValueError( + "dpdk dir %s mismatch, please check params -d" % self.base_dir + ) + + if self.patches is not None: + for p in self.patches: + out = self.send_expect( + "patch -d %s -p1 < %s" % (self.base_dir, dst_dir + p), "# " + ) + assert "****" not in out + + def prerequisites(self): + """ + Copy DPDK package to DUT and apply patch files. + """ + self.prepare_package() + self.dut_prerequisites() + self.stage = "post-init" + + def extra_nic_setup(self): + """ + Some nic like RRC required additional setup after module installed + """ + for port_info in self.ports_info: + netdev = port_info["port"] + netdev.setup() + + def bind_interfaces_linux(self, driver="igb_uio", nics_to_bind=None): + """ + Bind the interfaces to the selected driver. nics_to_bind can be None + to bind all interfaces or an array with the port indexes + """ + binding_list = "--bind=%s " % driver + + current_nic = 0 + for port_info in self.ports_info: + if nics_to_bind is None or current_nic in nics_to_bind: + binding_list += "%s " % (port_info["pci"]) + current_nic += 1 + + bind_script_path = self.get_dpdk_bind_script() + return self.send_expect( + "%s --force %s" % (bind_script_path, binding_list), "# " + ) + + def unbind_interfaces_linux(self, nics_to_bind=None): + """ + Unbind the interfaces + """ + + binding_list = "-u " + + current_nic = 0 + for port_info in self.ports_info: + if nics_to_bind is None or current_nic in nics_to_bind: + binding_list += "%s " % (port_info["pci"]) + current_nic += 1 + + bind_script_path = self.get_dpdk_bind_script() + self.send_expect("%s --force %s" % (bind_script_path, binding_list), "# ") + + def build_dpdk_apps(self, folder, extra_options=""): + """ + Build dpdk sample applications. + """ + if self.get_os_type() == "linux": + return self.build_dpdk_apps_linux_meson(folder, extra_options) + elif self.get_os_type() == "freebsd": + return self.build_dpdk_apps_freebsd_meson(folder, extra_options) + + def build_dpdk_apps_linux_meson(self, folder, extra_options): + """ + Build dpdk sample applications on linux use meson + """ + # icc compile need more time + if "icc" in self.target: + timeout = 300 + else: + timeout = 90 + + target_info = self.target.split("-") + arch = target_info[0] + if arch == "i686": + # find the pkg-config path and set the PKG_CONFIG_LIBDIR environmental variable to point it + out = self.send_expect("find /usr -type d -name pkgconfig", "# ") + pkg_path = "" + default_cflags = self.send_expect("echo $CFLAGS", "# ") + default_pkg_config = self.send_expect("echo $PKG_CONFIG_LIBDIR", "# ") + res_path = out.split("\r\n") + for cur_path in res_path: + if "i386" in cur_path: + pkg_path = cur_path + break + assert ( + pkg_path != "" + ), "please make sure you env have the i386 pkg-config path" + + self.send_expect("export CFLAGS=-m32", "# ", alt_session=True) + self.send_expect( + "export PKG_CONFIG_LIBDIR=%s" % pkg_path, "# ", alt_session=True + ) + + folder_info = folder.split("/") + name = folder_info[-1] + if name != "examples" and name not in self.apps_name: + raise Exception( + f'Please config {name} file path on {os.path.join(CONFIG_ROOT_PATH, "app_name.cfg")}' + ) + + if name == "examples": + example = "all" + else: + example = "/".join(folder_info[folder_info.index("examples") + 1 :]) + out = self.send_expect( + "meson configure -Dexamples=%s %s" % (example, self.target), "# " + ) + assert "FAILED" not in out, "Compilation error..." + out = self.send_expect("ninja -C %s" % self.target, "[~|~\]]# ", timeout) + assert "FAILED" not in out, "Compilation error..." + + # verify the app build in the config path + if example != "all": + out = self.send_expect("ls %s" % self.apps_name[name], "# ", verify=True) + assert isinstance(out, str), ( + "please confirm %s app path and name in app_name.cfg" % name + ) + + return out + + def build_dpdk_apps_freebsd_meson(self, folder, extra_options): + # meson build same as linux + return self.build_dpdk_apps_linux_meson(folder, extra_options) + + def get_blocklist_string(self, target, nic): + """ + Get block list command string. + """ + get_blocklist_string = getattr( + self, "get_blocklist_string_%s" % self.get_os_type() + ) + return get_blocklist_string(target, nic) + + def get_blocklist_string_linux(self, target, nic): + """ + Get block list command string on Linux. + """ + blocklist = "" + dutPorts = self.get_ports(nic) + self.restore_interfaces() + self.send_expect("insmod ./%s/kmod/igb_uio.ko" % target, "# ") + self.bind_interfaces_linux() + for port in range(0, len(self.ports_info)): + if port not in dutPorts: + blocklist += "-b %s " % self.ports_info[port]["pci"] + return blocklist + + def get_blocklist_string_freebsd(self, target, nic): + """ + Get block list command string on Freebsd. + """ + blocklist = "" + # No blocklist option in FreeBSD + return blocklist + + +class DPDKtester(Tester): + + """ + DPDK project class for tester. DTS will call prerequisites function to setup + interface and generate port map. + """ + + def __init__(self, crb, serializer, dut_id): + self.NAME = "tester" + super(DPDKtester, self).__init__(crb, serializer) + + def prerequisites(self, perf_test=False): + """ + Setup hugepage on tester and copy validation required files to tester. + """ + self.kill_all() + + if not self.skip_setup: + total_huge_pages = self.get_total_huge_pages() + hugepages_size = self.send_expect( + "awk '/Hugepagesize/ {print $2}' /proc/meminfo", "# " + ) + if total_huge_pages == 0: + self.mount_huge_pages() + if hugepages_size == "524288": + self.set_huge_pages(8) + else: + self.set_huge_pages(1024) + + self.session.copy_file_to("dep/tgen.tgz") + self.session.copy_file_to("dep/tclclient.tgz") + # unpack tgen + out = self.send_expect("tar zxfm tgen.tgz", "# ") + assert "Error" not in out + # unpack tclclient + out = self.send_expect("tar zxfm tclclient.tgz", "# ") + assert "Error" not in out + + self.send_expect("modprobe uio", "# ") + + self.tester_prerequisites() + + self.set_promisc() + + self.stage = "post-init" + + def setup_memory(self, hugepages=-1): + """ + Setup hugepage on tester. + """ + hugepages_size = self.send_expect( + "awk '/Hugepagesize/ {print $2}' /proc/meminfo", "# " + ) + + if int(hugepages_size) < (2048 * 2048): + arch_huge_pages = hugepages if hugepages > 0 else 2048 + total_huge_pages = self.get_total_huge_pages() + + self.mount_huge_pages() + if total_huge_pages != arch_huge_pages: + self.set_huge_pages(arch_huge_pages) -- 2.20.1