From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id BC2CEA0543; Wed, 24 Aug 2022 18:25:38 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 3C70E42B76; Wed, 24 Aug 2022 18:25:08 +0200 (CEST) Received: from lb.pantheon.sk (lb.pantheon.sk [46.229.239.20]) by mails.dpdk.org (Postfix) with ESMTP id 47C4B4281C for ; Wed, 24 Aug 2022 18:25:04 +0200 (CEST) Received: from localhost (localhost [127.0.0.1]) by lb.pantheon.sk (Postfix) with ESMTP id 27C05CD270; Wed, 24 Aug 2022 18:25:03 +0200 (CEST) X-Virus-Scanned: amavisd-new at siecit.sk Received: from lb.pantheon.sk ([127.0.0.1]) by localhost (lb.pantheon.sk [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id x3jAPhKq4faP; Wed, 24 Aug 2022 18:25:01 +0200 (CEST) Received: from entguard.lab.pantheon.local (unknown [46.229.239.141]) by lb.pantheon.sk (Postfix) with ESMTP id 33F32CD272; Wed, 24 Aug 2022 18:24:57 +0200 (CEST) From: =?UTF-8?q?Juraj=20Linke=C5=A1?= To: thomas@monjalon.net, david.marchand@redhat.com, ronan.randles@intel.com, Honnappa.Nagarahalli@arm.com, ohilyard@iol.unh.edu, lijuan.tu@intel.com Cc: dev@dpdk.org, =?UTF-8?q?Juraj=20Linke=C5=A1?= Subject: [RFC PATCH v1 05/10] dts: add system under test node Date: Wed, 24 Aug 2022 16:24:49 +0000 Message-Id: <20220824162454.394285-6-juraj.linkes@pantheon.tech> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20220824162454.394285-1-juraj.linkes@pantheon.tech> References: <20220824162454.394285-1-juraj.linkes@pantheon.tech> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org The SUT node contains methods to configure the node and build and configure DPDK. Signed-off-by: Juraj Linkeš --- dts/framework/sut_node.py | 603 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 603 insertions(+) create mode 100644 dts/framework/sut_node.py diff --git a/dts/framework/sut_node.py b/dts/framework/sut_node.py new file mode 100644 index 0000000000..c9f5e69d73 --- /dev/null +++ b/dts/framework/sut_node.py @@ -0,0 +1,603 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2010-2014 Intel Corporation +# Copyright(c) 2022 PANTHEON.tech s.r.o. +# + +import os +import re +import tarfile +import time +from typing import List, Optional, Union + +from framework.config import NodeConfiguration + +from .exception import ParameterInvalidException +from .node import Node +from .settings import SETTINGS + + +class SutNode(Node): + """ + A class for managing connections to the System under test, providing + methods that retrieve the necessary information about the node (such as + cpu, memory and NIC details) and configuration capabilities. + """ + + def __init__(self, node_config: NodeConfiguration): + super(SutNode, self).__init__(node_config) + self.tg_node = None + self.architecture = node_config.arch + self.prefix_subfix = ( + str(os.getpid()) + "_" + time.strftime("%Y%m%d%H%M%S", time.localtime()) + ) + self.hugepage_path = None + self.dpdk_version = "" + self.testpmd = None + + def prerequisites(self): + """ + Copy DPDK package to SUT and apply patch files. + """ + self.prepare_package() + self.sut_prerequisites() + + def prepare_package(self): + if not self.skip_setup: + assert os.path.isfile(SETTINGS.dpdk_ref) is True, "Invalid package" + + out = self.send_expect( + "ls -d %s" % SETTINGS.remote_dpdk_dir, "# ", verify=True + ) + if out == 2: + self.send_expect("mkdir -p %s" % SETTINGS.remote_dpdk_dir, "# ") + + out = self.send_expect( + "ls %s && cd %s" % (SETTINGS.remote_dpdk_dir, SETTINGS.remote_dpdk_dir), + "#", + verify=True, + ) + if out == -1: + raise IOError( + f"A failure occurred when creating {SETTINGS.remote_dpdk_dir} on " + f"{self}." + ) + self.main_session.copy_file_to(SETTINGS.dpdk_ref, SETTINGS.remote_dpdk_dir) + self.kill_all() + + # enable core dump + self.send_expect("ulimit -c unlimited", "#") + + with tarfile.open(SETTINGS.dpdk_ref) as dpdk_tar: + dpdk_top_dir = dpdk_tar.getnames()[0] + + remote_dpdk_top_dir = os.path.join(SETTINGS.remote_dpdk_dir, dpdk_top_dir) + + # unpack the code and change to the working folder + self.send_expect("rm -rf %s" % remote_dpdk_top_dir, "#") + + remote_dpdk_path = os.path.join( + SETTINGS.remote_dpdk_dir, os.path.basename(SETTINGS.dpdk_ref) + ) + + # unpack dpdk + out = self.send_expect( + f"tar xfm {remote_dpdk_path} -C {SETTINGS.remote_dpdk_dir}", + "# ", + 60, + verify=True, + ) + if out == -1: + raise IOError( + f"Extracting remote DPDK package {remote_dpdk_path} to " + f"{SETTINGS.remote_dpdk_dir} failed." + ) + + # check dpdk dir name is expect + out = self.send_expect("ls %s" % remote_dpdk_top_dir, "# ", 20, verify=True) + if out == -1: + raise FileNotFoundError( + f"Remote DPDK dir {remote_dpdk_top_dir} not found." + ) + + def set_target(self, target): + """ + Set env variable, these have to be setup all the time. Some tests + need to compile example apps by themselves and will fail otherwise. + Set hugepage on SUT and install modules required by DPDK. + Configure default ixgbe PMD function. + """ + self.target = target + + self.set_toolchain(target) + + # set env variable + self.set_env_variable() + + if not self.skip_setup: + self.build_install_dpdk(target) + + self.setup_memory() + + def set_env_variable(self): + # These have to be setup all the time. Some tests need to compile + # example apps by themselves and will fail otherwise. + self.send_expect("export RTE_TARGET=" + self.target, "#") + self.send_expect("export RTE_SDK=`pwd`", "#") + + def build_install_dpdk(self, target): + """ + Build DPDK source code with specified target. + """ + if self.get_os() == "linux": + self.build_install_dpdk_linux_meson(target) + + def build_install_dpdk_linux_meson(self, target): + """ + Build DPDK source code on linux use meson + """ + build_time = 1800 + target_info = target.split("-") + arch = target_info[0] + toolchain = target_info[3] + + default_library = "static" + if arch == "i686": + # find the pkg-config path and set the PKG_CONFIG_LIBDIR environmental variable to point it + out = self.send_expect("find /usr -type d -name pkgconfig", "# ") + pkg_path = "" + res_path = out.split("\r\n") + for cur_path in res_path: + if "i386" in cur_path: + pkg_path = cur_path + break + assert ( + pkg_path != "" + ), "please make sure you env have the i386 pkg-config path" + + self.send_expect("export CFLAGS=-m32", "# ") + self.send_expect("export PKG_CONFIG_LIBDIR=%s" % pkg_path, "# ") + + self.send_expect("rm -rf " + target, "#") + out = self.send_expect( + "CC=%s meson -Denable_kmods=True -Dlibdir=lib --default-library=%s %s" + % (toolchain, default_library, target), + "[~|~\]]# ", + build_time, + ) + assert "FAILED" not in out, "meson setup failed ..." + + out = self.send_expect("ninja -C %s" % target, "[~|~\]]# ", build_time) + assert "FAILED" not in out, "ninja complie failed ..." + + # copy kmod file to the folder same as make + out = self.send_expect( + "find ./%s/kernel/ -name *.ko" % target, "# ", verify=True + ) + self.send_expect("mkdir -p %s/kmod" % target, "# ") + if not isinstance(out, int) and len(out) > 0: + kmod = out.split("\r\n") + for mod in kmod: + self.send_expect("cp %s %s/kmod/" % (mod, target), "# ") + + def build_dpdk_apps(self, folder): + """ + Build dpdk sample applications. + """ + if self.get_os() == "linux": + return self.build_dpdk_apps_linux_meson(folder) + + def build_dpdk_apps_linux_meson(self, folder): + """ + Build dpdk sample applications on linux use meson + """ + # icc compile need more time + if "icc" in self.target: + timeout = 300 + else: + timeout = 90 + + target_info = self.target.split("-") + arch = target_info[0] + if arch == "i686": + # find the pkg-config path and set the PKG_CONFIG_LIBDIR environmental variable to point it + out = self.send_expect("find /usr -type d -name pkgconfig", "# ") + pkg_path = "" + res_path = out.split("\r\n") + for cur_path in res_path: + if "i386" in cur_path: + pkg_path = cur_path + break + assert ( + pkg_path != "" + ), "please make sure you env have the i386 pkg-config path" + + self.send_expect("export CFLAGS=-m32", "# ", alt_session=True) + self.send_expect( + "export PKG_CONFIG_LIBDIR=%s" % pkg_path, "# ", alt_session=True + ) + + folder_info = folder.split("/") + name = folder_info[-1] + + if name == "examples": + example = "all" + else: + example = "/".join(folder_info[folder_info.index("examples") + 1 :]) + out = self.send_expect( + "meson configure -Dexamples=%s %s" % (example, self.target), "# " + ) + assert "FAILED" not in out, "Compilation error..." + out = self.send_expect("ninja -C %s" % self.target, "[~|~\]]# ", timeout) + assert "FAILED" not in out, "Compilation error..." + + # verify the app build in the config path + if example != "all": + out = self.send_expect("ls %s" % self.apps_name[name], "# ", verify=True) + assert isinstance(out, str), ( + "please confirm %s app path and name in app_name.cfg" % name + ) + + return out + + def filter_cores_from_node_cfg(self) -> None: + # get core list from conf.yaml + core_list = [] + all_core_list = [str(core.core) for core in self.cores] + core_list_str = self._config.cores + if core_list_str == "": + core_list = all_core_list + split_by_comma = core_list_str.split(",") + range_cores = [] + for item in split_by_comma: + if "-" in item: + tmp = item.split("-") + range_cores.extend( + [str(i) for i in range(int(tmp[0]), int(tmp[1]) + 1)] + ) + else: + core_list.append(item) + core_list.extend(range_cores) + + abnormal_core_list = [] + for core in core_list: + if core not in all_core_list: + abnormal_core_list.append(core) + + if abnormal_core_list: + self.logger.info( + "those %s cores are out of range system, all core list of system are %s" + % (abnormal_core_list, all_core_list) + ) + raise Exception("configured cores out of range system") + + core_list = [core for core in self.cores if str(core.core) in core_list] + self.cores = core_list + self.number_of_cores = len(self.cores) + + def create_eal_parameters( + self, + fixed_prefix: bool = False, + socket: Optional[int] = None, + cores: Union[str, List[int], List[str]] = "default", + prefix: str = "", + no_pci: bool = False, + vdevs: List[str] = None, + other_eal_param: str = "", + ) -> str: + """ + generate eal parameters character string; + :param fixed_prefix: use fixed file-prefix or not, when it is true, + the file-prefix will not be added a timestamp + :param socket: the physical CPU socket index, -1 means no care cpu socket; + :param cores: set the core info, eg: + cores=[0,1,2,3], + cores=['0', '1', '2', '3'], + cores='default', + cores='1S/4C/1T', + cores='all'; + :param prefix: set file prefix string, eg: + prefix='vf'; + :param no_pci: switch of disable PCI bus eg: + no_pci=True; + :param vdevs: virtual device list, eg: + vdevs=['net_ring0', 'net_ring1']; + :param other_eal_param: user defined DPDK eal parameters, eg: + other_eal_param='--single-file-segments'; + :return: eal param string, eg: + '-c 0xf -a 0000:88:00.0 --file-prefix=dpdk_1112_20190809143420'; + if DPDK version < 20.11-rc4, eal_str eg: + '-c 0xf -w 0000:88:00.0 --file-prefix=dpdk_1112_20190809143420'; + """ + if vdevs is None: + vdevs = [] + + if socket is None: + socket = -1 + + config = { + "cores": cores, + "prefix": prefix, + "no_pci": no_pci, + "vdevs": vdevs, + "other_eal_param": other_eal_param, + } + + eal_parameter_creator = _EalParameter( + sut_node=self, fixed_prefix=fixed_prefix, socket=socket, **config + ) + eal_str = eal_parameter_creator.make_eal_param() + + return eal_str + + def set_toolchain(self, target): + """ + This looks at the current target and instantiates an attribute to + be either a NodeLinuxApp or NodeBareMetal object. These latter two + classes are private and should not be used directly by client code. + """ + self.kill_all() + self.target = target + [arch, _, _, toolchain] = target.split("-") + + if toolchain == "icc": + icc_vars = os.getenv("ICC_VARS", "/opt/intel/composer_xe_2013/bin/") + icc_vars += "compilervars.sh" + + if arch == "x86_64": + icc_arch = "intel64" + elif arch == "i686": + icc_arch = "ia32" + self.send_expect("source " + icc_vars + " " + icc_arch, "# ") + + self.architecture = arch + + def sut_prerequisites(self): + """ + Prerequest function should be called before execute any test case. + Will call function to scan all lcore's information which on SUT. + Then call pci scan function to collect nic device information. + At last setup SUT' environment for validation. + """ + out = self.send_expect(f"cd {SETTINGS.remote_dpdk_dir}", "# ") + assert "No such file or directory" not in out, "Can't switch to dpdk folder!!!" + out = self.send_expect("cat VERSION", "# ") + if "No such file or directory" in out: + self.logger.error("Can't get DPDK version due to VERSION not exist!!!") + else: + self.dpdk_version = out + self.send_expect("alias ls='ls --color=none'", "#") + + self.init_core_list() + self.filter_cores_from_node_cfg() + + def setup_memory(self, hugepages=-1): + """ + Setup hugepage on SUT. + """ + function_name = "setup_memory_%s" % self.get_os() + try: + setup_memory = getattr(self, function_name) + setup_memory(hugepages) + except AttributeError: + self.logger.error("%s is not implemented" % function_name) + + def setup_memory_linux(self, hugepages=-1): + """ + Setup Linux hugepages. + """ + hugepages_size = self.send_expect( + "awk '/Hugepagesize/ {print $2}' /proc/meminfo", "# " + ) + total_huge_pages = self.get_total_huge_pages() + numa_nodes = self.send_expect("ls /sys/devices/system/node | grep node*", "# ") + if not numa_nodes: + total_numa_nodes = -1 + else: + numa_nodes = numa_nodes.splitlines() + total_numa_nodes = len(numa_nodes) + self.logger.info(numa_nodes) + + force_socket = False + + if int(hugepages_size) < (1024 * 1024): + if hugepages <= 0: + if self.architecture == "x86_64": + arch_huge_pages = 4096 + elif self.architecture == "i686": + arch_huge_pages = 512 + force_socket = True + # set huge pagesize for x86_x32 abi target + elif self.architecture == "x86_x32": + arch_huge_pages = 256 + force_socket = True + elif self.architecture == "ppc_64": + arch_huge_pages = 512 + elif self.architecture == "arm64": + if int(hugepages_size) >= (512 * 1024): + arch_huge_pages = 8 + else: + arch_huge_pages = 2048 + else: + arch_huge_pages = 256 + else: + arch_huge_pages = hugepages + + if total_huge_pages != arch_huge_pages: + if total_numa_nodes == -1: + self.set_huge_pages(arch_huge_pages) + else: + # before all hugepage average distribution by all socket, + # but sometimes create mbuf pool on socket 0 failed when + # setup testpmd, so set all huge page on first socket + if force_socket: + self.set_huge_pages(arch_huge_pages, numa_nodes[0]) + self.logger.info("force_socket on %s" % numa_nodes[0]) + else: + # set huge pages to all numa_nodes + for numa_node in numa_nodes: + self.set_huge_pages(arch_huge_pages, numa_node) + + self.mount_huge_pages() + self.hugepage_path = self.strip_hugepage_path() + + def get_memory_channels(self): + n = self._config.memory_channels + if n is not None and n > 0: + return n + else: + return 1 + + +class _EalParameter(object): + def __init__( + self, + sut_node: SutNode, + fixed_prefix: bool, + socket: int, + cores: Union[str, List[int], List[str]], + prefix: str, + no_pci: bool, + vdevs: List[str], + other_eal_param: str, + ): + """ + generate eal parameters character string; + :param sut_node: SUT Node; + :param fixed_prefix: use fixed file-prefix or not, when it is true, + the file-prefix will not be added a timestamp + :param socket: the physical CPU socket index, -1 means no care cpu socket; + :param cores: set the core info, eg: + cores=[0,1,2,3], + cores=['0','1','2','3'], + cores='default', + cores='1S/4C/1T', + cores='all'; + param prefix: set file prefix string, eg: + prefix='vf'; + param no_pci: switch of disable PCI bus eg: + no_pci=True; + param vdevs: virtual device list, eg: + vdevs=['net_ring0', 'net_ring1']; + param other_eal_param: user defined DPDK eal parameters, eg: + other_eal_param='--single-file-segments'; + """ + self.os = sut_node.get_os() + self.fixed_prefix = fixed_prefix + self.socket = socket + self.sut_node = sut_node + self.cores = self._validate_cores(cores) + self.prefix = prefix + self.no_pci = no_pci + self.vdevs = vdevs + self.other_eal_param = other_eal_param + + _param_validate_exception_info_template = ( + "Invalid parameter of %s about value of %s, Please reference API doc." + ) + + @staticmethod + def _validate_cores(cores: Union[str, List[int], List[str]]): + core_string_match = r"default|all|\d+S/\d+C/\d+T|$" + if isinstance(cores, list) and ( + all(map(lambda _core: type(_core) == int, cores)) + or all(map(lambda _core: type(_core) == str, cores)) + ): + return cores + elif type(cores) == str and re.match(core_string_match, cores, re.I): + return cores + else: + raise ParameterInvalidException("cores", cores) + + def _make_cores_param(self) -> str: + is_use_default_cores = ( + self.cores == "" + or isinstance(self.cores, str) + and self.cores.lower() == "default" + ) + if is_use_default_cores: + default_cores = "1S/2C/1T" + core_list = self.sut_node.get_core_list(default_cores) + else: + core_list = self._get_cores() + + def _get_consecutive_cores_range(_cores: List[int]): + _formated_core_list = [] + _tmp_cores_list = list(sorted(map(int, _cores))) + _segment = _tmp_cores_list[:1] + for _core_num in _tmp_cores_list[1:]: + if _core_num - _segment[-1] == 1: + _segment.append(_core_num) + else: + _formated_core_list.append( + f"{_segment[0]}-{_segment[-1]}" + if len(_segment) > 1 + else f"{_segment[0]}" + ) + _index = _tmp_cores_list.index(_core_num) + _formated_core_list.extend( + _get_consecutive_cores_range(_tmp_cores_list[_index:]) + ) + _segment.clear() + break + if len(_segment) > 0: + _formated_core_list.append( + f"{_segment[0]}-{_segment[-1]}" + if len(_segment) > 1 + else f"{_segment[0]}" + ) + return _formated_core_list + + return f'-l {",".join(_get_consecutive_cores_range(core_list))}' + + def _make_memory_channels(self) -> str: + param_template = "-n {}" + return param_template.format(self.sut_node.get_memory_channels()) + + def _make_no_pci_param(self) -> str: + if self.no_pci is True: + return "--no-pci" + else: + return "" + + def _make_prefix_param(self) -> str: + if self.prefix == "": + fixed_file_prefix = "dpdk" + "_" + self.sut_node.prefix_subfix + else: + fixed_file_prefix = self.prefix + if not self.fixed_prefix: + fixed_file_prefix = ( + fixed_file_prefix + "_" + self.sut_node.prefix_subfix + ) + fixed_file_prefix = self._do_os_handle_with_prefix_param(fixed_file_prefix) + return fixed_file_prefix + + def _make_vdevs_param(self) -> str: + if len(self.vdevs) == 0: + return "" + else: + _vdevs = ["--vdev " + vdev for vdev in self.vdevs] + return " ".join(_vdevs) + + def _get_cores(self) -> List[int]: + if type(self.cores) == list: + return self.cores + elif isinstance(self.cores, str): + return self.sut_node.get_core_list(self.cores, socket=self.socket) + + def _do_os_handle_with_prefix_param(self, file_prefix: str) -> str: + self.sut_node.prefix_list.append(file_prefix) + return "--file-prefix=" + file_prefix + + def make_eal_param(self) -> str: + _eal_str = " ".join( + [ + self._make_cores_param(), + self._make_memory_channels(), + self._make_prefix_param(), + self._make_no_pci_param(), + self._make_vdevs_param(), + # append user defined eal parameters + self.other_eal_param, + ] + ) + return _eal_str -- 2.30.2