From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 4885C423FE; Tue, 17 Jan 2023 16:49:38 +0100 (CET) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id EBD3842D47; Tue, 17 Jan 2023 16:49:17 +0100 (CET) Received: from lb.pantheon.sk (lb.pantheon.sk [46.229.239.20]) by mails.dpdk.org (Postfix) with ESMTP id E5C3B410EE for ; Tue, 17 Jan 2023 16:49:15 +0100 (CET) Received: from localhost (localhost [127.0.0.1]) by lb.pantheon.sk (Postfix) with ESMTP id 4953C1D811F; Tue, 17 Jan 2023 16:49:15 +0100 (CET) X-Virus-Scanned: amavisd-new at siecit.sk Received: from lb.pantheon.sk ([127.0.0.1]) by localhost (lb.pantheon.sk [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id MeiEB7k-CBaK; Tue, 17 Jan 2023 16:49:13 +0100 (CET) Received: from entguard.lab.pantheon.local (unknown [46.229.239.141]) by lb.pantheon.sk (Postfix) with ESMTP id A598B1D8122; Tue, 17 Jan 2023 16:49:08 +0100 (CET) From: =?UTF-8?q?Juraj=20Linke=C5=A1?= To: thomas@monjalon.net, Honnappa.Nagarahalli@arm.com, ohilyard@iol.unh.edu, lijuan.tu@intel.com, bruce.richardson@intel.com Cc: dev@dpdk.org, =?UTF-8?q?Juraj=20Linke=C5=A1?= Subject: [PATCH v3 03/10] dts: add dpdk build on sut Date: Tue, 17 Jan 2023 15:48:59 +0000 Message-Id: <20230117154906.860916-4-juraj.linkes@pantheon.tech> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20230117154906.860916-1-juraj.linkes@pantheon.tech> References: <20221114165438.1133783-1-juraj.linkes@pantheon.tech> <20230117154906.860916-1-juraj.linkes@pantheon.tech> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Add the ability to build DPDK and apps on the SUT, using a configured target. Signed-off-by: Juraj Linkeš --- dts/framework/config/__init__.py | 2 + dts/framework/exception.py | 17 ++ dts/framework/remote_session/os_session.py | 90 +++++++++- dts/framework/remote_session/posix_session.py | 126 ++++++++++++++ .../remote_session/remote/remote_session.py | 38 ++++- .../remote_session/remote/ssh_session.py | 68 +++++++- dts/framework/settings.py | 55 +++++- dts/framework/testbed_model/__init__.py | 1 + dts/framework/testbed_model/dpdk.py | 33 ++++ dts/framework/testbed_model/sut_node.py | 158 ++++++++++++++++++ dts/framework/utils.py | 19 ++- 11 files changed, 589 insertions(+), 18 deletions(-) create mode 100644 dts/framework/testbed_model/dpdk.py diff --git a/dts/framework/config/__init__.py b/dts/framework/config/__init__.py index e3e2d74eac..ca61cb10fe 100644 --- a/dts/framework/config/__init__.py +++ b/dts/framework/config/__init__.py @@ -91,6 +91,7 @@ class BuildTargetConfiguration: os: OS cpu: CPUType compiler: Compiler + compiler_wrapper: str name: str @staticmethod @@ -100,6 +101,7 @@ def from_dict(d: dict) -> "BuildTargetConfiguration": os=OS(d["os"]), cpu=CPUType(d["cpu"]), compiler=Compiler(d["compiler"]), + compiler_wrapper=d.get("compiler_wrapper", ""), name=f"{d['arch']}-{d['os']}-{d['cpu']}-{d['compiler']}", ) diff --git a/dts/framework/exception.py b/dts/framework/exception.py index e776b42bd9..b4545a5a40 100644 --- a/dts/framework/exception.py +++ b/dts/framework/exception.py @@ -23,6 +23,7 @@ class ErrorSeverity(IntEnum): CONFIG_ERR = 2 REMOTE_CMD_EXEC_ERR = 3 SSH_ERR = 4 + DPDK_BUILD_ERR = 10 class DTSError(Exception): @@ -111,3 +112,19 @@ def __str__(self) -> str: f"Command {self.command} returned a non-zero exit code: " f"{self.command_return_code}" ) + + +class RemoteDirectoryExistsError(DTSError): + """ + Raised when a remote directory to be created already exists. + """ + + severity: ClassVar[ErrorSeverity] = ErrorSeverity.REMOTE_CMD_EXEC_ERR + + +class DPDKBuildError(DTSError): + """ + Raised when DPDK build fails for any reason. + """ + + severity: ClassVar[ErrorSeverity] = ErrorSeverity.DPDK_BUILD_ERR diff --git a/dts/framework/remote_session/os_session.py b/dts/framework/remote_session/os_session.py index 7a4cc5e669..06d1ffefdd 100644 --- a/dts/framework/remote_session/os_session.py +++ b/dts/framework/remote_session/os_session.py @@ -2,10 +2,14 @@ # Copyright(c) 2023 PANTHEON.tech s.r.o. # Copyright(c) 2023 University of New Hampshire -from abc import ABC +from abc import ABC, abstractmethod +from pathlib import PurePath -from framework.config import NodeConfiguration +from framework.config import Architecture, NodeConfiguration from framework.logger import DTSLOG +from framework.settings import SETTINGS +from framework.testbed_model import MesonArgs +from framework.utils import EnvVarsDict from .remote import RemoteSession, create_remote_session @@ -44,3 +48,85 @@ def is_alive(self) -> bool: Check whether the remote session is still responding. """ return self.remote_session.is_alive() + + @abstractmethod + def guess_dpdk_remote_dir(self, remote_dir) -> PurePath: + """ + Try to find DPDK remote dir in remote_dir. + """ + + @abstractmethod + def get_remote_tmp_dir(self) -> PurePath: + """ + Get the path of the temporary directory of the remote OS. + """ + + @abstractmethod + def get_dpdk_build_env_vars(self, arch: Architecture) -> dict: + """ + Create extra environment variables needed for the target architecture. Get + information from the node if needed. + """ + + @abstractmethod + def join_remote_path(self, *args: str | PurePath) -> PurePath: + """ + Join path parts using the path separator that fits the remote OS. + """ + + @abstractmethod + def copy_file( + self, + source_file: str | PurePath, + destination_file: str | PurePath, + source_remote: bool = False, + ) -> None: + """ + Copy source_file from local filesystem to destination_file + on the remote Node associated with the remote session. + If source_remote is True, reverse the direction - copy source_file from the + associated remote Node to destination_file on local storage. + """ + + @abstractmethod + def remove_remote_dir( + self, + remote_dir_path: str | PurePath, + recursive: bool = True, + force: bool = True, + ) -> None: + """ + Remove remote directory, by default remove recursively and forcefully. + """ + + @abstractmethod + def extract_remote_tarball( + self, + remote_tarball_path: str | PurePath, + expected_dir: str | PurePath | None = None, + ) -> None: + """ + Extract remote tarball in place. If expected_dir is a non-empty string, check + whether the dir exists after extracting the archive. + """ + + @abstractmethod + def build_dpdk( + self, + env_vars: EnvVarsDict, + meson_args: MesonArgs, + remote_dpdk_dir: str | PurePath, + remote_dpdk_build_dir: str | PurePath, + rebuild: bool = False, + timeout: float = SETTINGS.compile_timeout, + ) -> None: + """ + Build DPDK in the input dir with specified environment variables and meson + arguments. + """ + + @abstractmethod + def get_dpdk_version(self, version_path: str | PurePath) -> str: + """ + Inspect DPDK version on the remote node from version_path. + """ diff --git a/dts/framework/remote_session/posix_session.py b/dts/framework/remote_session/posix_session.py index 110b6a4804..d4da9f114e 100644 --- a/dts/framework/remote_session/posix_session.py +++ b/dts/framework/remote_session/posix_session.py @@ -2,6 +2,14 @@ # Copyright(c) 2023 PANTHEON.tech s.r.o. # Copyright(c) 2023 University of New Hampshire +from pathlib import PurePath, PurePosixPath + +from framework.config import Architecture +from framework.exception import DPDKBuildError, RemoteCommandExecutionError +from framework.settings import SETTINGS +from framework.testbed_model import MesonArgs +from framework.utils import EnvVarsDict + from .os_session import OSSession @@ -10,3 +18,121 @@ class PosixSession(OSSession): An intermediary class implementing the Posix compliant parts of Linux and other OS remote sessions. """ + + @staticmethod + def combine_short_options(**opts: bool) -> str: + ret_opts = "" + for opt, include in opts.items(): + if include: + ret_opts = f"{ret_opts}{opt}" + + if ret_opts: + ret_opts = f" -{ret_opts}" + + return ret_opts + + def guess_dpdk_remote_dir(self, remote_dir) -> PurePosixPath: + remote_guess = self.join_remote_path(remote_dir, "dpdk-*") + result = self.remote_session.send_command(f"ls -d {remote_guess} | tail -1") + return PurePosixPath(result.stdout) + + def get_remote_tmp_dir(self) -> PurePosixPath: + return PurePosixPath("/tmp") + + def get_dpdk_build_env_vars(self, arch: Architecture) -> dict: + """ + Create extra environment variables needed for i686 arch build. Get information + from the node if needed. + """ + env_vars = {} + if arch == Architecture.i686: + # find the pkg-config path and store it in PKG_CONFIG_LIBDIR + out = self.remote_session.send_command("find /usr -type d -name pkgconfig") + pkg_path = "" + res_path = out.stdout.split("\r\n") + for cur_path in res_path: + if "i386" in cur_path: + pkg_path = cur_path + break + assert pkg_path != "", "i386 pkg-config path not found" + + env_vars["CFLAGS"] = "-m32" + env_vars["PKG_CONFIG_LIBDIR"] = pkg_path + + return env_vars + + def join_remote_path(self, *args: str | PurePath) -> PurePosixPath: + return PurePosixPath(*args) + + def copy_file( + self, + source_file: str | PurePath, + destination_file: str | PurePath, + source_remote: bool = False, + ) -> None: + self.remote_session.copy_file(source_file, destination_file, source_remote) + + def remove_remote_dir( + self, + remote_dir_path: str | PurePath, + recursive: bool = True, + force: bool = True, + ) -> None: + opts = PosixSession.combine_short_options(r=recursive, f=force) + self.remote_session.send_command(f"rm{opts} {remote_dir_path}") + + def extract_remote_tarball( + self, + remote_tarball_path: str | PurePath, + expected_dir: str | PurePath | None = None, + ) -> None: + self.remote_session.send_command( + f"tar xfm {remote_tarball_path} " + f"-C {PurePosixPath(remote_tarball_path).parent}", + 60, + ) + if expected_dir: + self.remote_session.send_command(f"ls {expected_dir}", verify=True) + + def build_dpdk( + self, + env_vars: EnvVarsDict, + meson_args: MesonArgs, + remote_dpdk_dir: str | PurePath, + remote_dpdk_build_dir: str | PurePath, + rebuild: bool = False, + timeout: float = SETTINGS.compile_timeout, + ) -> None: + try: + if rebuild: + # reconfigure, then build + self._logger.info("Reconfiguring DPDK build.") + self.remote_session.send_command( + f"meson configure {meson_args} {remote_dpdk_build_dir}", + timeout, + verify=True, + env=env_vars, + ) + else: + # fresh build - remove target dir first, then build from scratch + self._logger.info("Configuring DPDK build from scratch.") + self.remove_remote_dir(remote_dpdk_build_dir) + self.remote_session.send_command( + f"meson {meson_args} {remote_dpdk_dir} {remote_dpdk_build_dir}", + timeout, + verify=True, + env=env_vars, + ) + + self._logger.info("Building DPDK.") + self.remote_session.send_command( + f"ninja -C {remote_dpdk_build_dir}", timeout, verify=True, env=env_vars + ) + except RemoteCommandExecutionError as e: + raise DPDKBuildError(f"DPDK build failed when doing '{e.command}'.") + + def get_dpdk_version(self, build_dir: str | PurePath) -> str: + out = self.remote_session.send_command( + f"cat {self.join_remote_path(build_dir, 'VERSION')}", verify=True + ) + return out.stdout diff --git a/dts/framework/remote_session/remote/remote_session.py b/dts/framework/remote_session/remote/remote_session.py index 5ac395ec79..91dee3cb4f 100644 --- a/dts/framework/remote_session/remote/remote_session.py +++ b/dts/framework/remote_session/remote/remote_session.py @@ -5,11 +5,13 @@ import dataclasses from abc import ABC, abstractmethod +from pathlib import PurePath from framework.config import NodeConfiguration from framework.exception import RemoteCommandExecutionError from framework.logger import DTSLOG from framework.settings import SETTINGS +from framework.utils import EnvVarsDict @dataclasses.dataclass(slots=True, frozen=True) @@ -83,15 +85,22 @@ def _connect(self) -> None: """ def send_command( - self, command: str, timeout: float = SETTINGS.timeout, verify: bool = False + self, + command: str, + timeout: float = SETTINGS.timeout, + verify: bool = False, + env: EnvVarsDict | None = None, ) -> CommandResult: """ - Send a command to the connected node and return CommandResult. + Send a command to the connected node using optional env vars + and return CommandResult. If verify is True, check the return code of the executed command and raise a RemoteCommandExecutionError if the command failed. """ - self._logger.info(f"Sending: '{command}'") - result = self._send_command(command, timeout) + self._logger.info( + f"Sending: '{command}'" + (f" with env vars: '{env}'" if env else "") + ) + result = self._send_command(command, timeout, env) if verify and result.return_code: self._logger.debug( f"Command '{command}' failed with return code '{result.return_code}'" @@ -104,9 +113,12 @@ def send_command( return result @abstractmethod - def _send_command(self, command: str, timeout: float) -> CommandResult: + def _send_command( + self, command: str, timeout: float, env: EnvVarsDict | None + ) -> CommandResult: """ - Use the underlying protocol to execute the command and return CommandResult. + Use the underlying protocol to execute the command using optional env vars + and return CommandResult. """ def close(self, force: bool = False) -> None: @@ -127,3 +139,17 @@ def is_alive(self) -> bool: """ Check whether the remote session is still responding. """ + + @abstractmethod + def copy_file( + self, + source_file: str | PurePath, + destination_file: str | PurePath, + source_remote: bool = False, + ) -> None: + """ + Copy source_file from local filesystem to destination_file on the remote Node + associated with the remote session. + If source_remote is True, reverse the direction - copy source_file from the + associated Node to destination_file on local filesystem. + """ diff --git a/dts/framework/remote_session/remote/ssh_session.py b/dts/framework/remote_session/remote/ssh_session.py index 6da5be9fff..d0863d8791 100644 --- a/dts/framework/remote_session/remote/ssh_session.py +++ b/dts/framework/remote_session/remote/ssh_session.py @@ -4,13 +4,15 @@ # Copyright(c) 2022-2023 University of New Hampshire import time +from pathlib import PurePath +import pexpect # type: ignore from pexpect import pxssh # type: ignore from framework.config import NodeConfiguration from framework.exception import SSHConnectionError, SSHSessionDeadError, SSHTimeoutError from framework.logger import DTSLOG -from framework.utils import GREEN, RED +from framework.utils import GREEN, RED, EnvVarsDict from .remote_session import CommandResult, RemoteSession @@ -163,16 +165,22 @@ def _flush(self) -> None: def is_alive(self) -> bool: return self.session.isalive() - def _send_command(self, command: str, timeout: float) -> CommandResult: - output = self._send_command_get_output(command, timeout) - return_code = int(self._send_command_get_output("echo $?", timeout)) + def _send_command( + self, command: str, timeout: float, env: EnvVarsDict | None + ) -> CommandResult: + output = self._send_command_get_output(command, timeout, env) + return_code = int(self._send_command_get_output("echo $?", timeout, None)) # we're capturing only stdout return CommandResult(self.name, command, output, "", return_code) - def _send_command_get_output(self, command: str, timeout: float) -> str: + def _send_command_get_output( + self, command: str, timeout: float, env: EnvVarsDict | None + ) -> str: try: self._clean_session() + if env: + command = f"{env} {command}" self._send_line(command) except Exception as e: raise e @@ -189,3 +197,53 @@ def _close(self, force: bool = False) -> None: else: if self.is_alive(): self.session.logout() + + def copy_file( + self, + source_file: str | PurePath, + destination_file: str | PurePath, + source_remote: bool = False, + ) -> None: + """ + Send a local file to a remote host. + """ + if source_remote: + source_file = f"{self.username}@{self.ip}:{source_file}" + else: + destination_file = f"{self.username}@{self.ip}:{destination_file}" + + port = "" + if self.port: + port = f" -P {self.port}" + + # this is not OS agnostic, find a Pythonic (and thus OS agnostic) way + # TODO Fabric should handle this + command = ( + f"scp -v{port} -o NoHostAuthenticationForLocalhost=yes" + f" {source_file} {destination_file}" + ) + + self._spawn_scp(command) + + def _spawn_scp(self, scp_cmd: str) -> None: + """ + Transfer a file with SCP + """ + self._logger.info(scp_cmd) + p: pexpect.spawn = pexpect.spawn(scp_cmd) + time.sleep(0.5) + ssh_newkey: str = "Are you sure you want to continue connecting" + i: int = p.expect( + [ssh_newkey, "[pP]assword", "# ", pexpect.EOF, pexpect.TIMEOUT], 120 + ) + if i == 0: # add once in trust list + p.sendline("yes") + i = p.expect([ssh_newkey, "[pP]assword", pexpect.EOF], 2) + + if i == 1: + time.sleep(0.5) + p.sendline(self.password) + p.expect("Exit status 0", 60) + if i == 4: + self._logger.error("SCP TIMEOUT error %d" % i) + p.close() diff --git a/dts/framework/settings.py b/dts/framework/settings.py index 800f2c7b7f..a298b1eaac 100644 --- a/dts/framework/settings.py +++ b/dts/framework/settings.py @@ -1,14 +1,17 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright(c) 2010-2021 Intel Corporation -# Copyright(c) 2022 PANTHEON.tech s.r.o. -# Copyright(c) 2022 University of New Hampshire +# Copyright(c) 2022-2023 PANTHEON.tech s.r.o. +# Copyright(c) 2022-2023 University of New Hampshire import argparse import os from collections.abc import Callable, Iterable, Sequence from dataclasses import dataclass +from pathlib import Path from typing import Any, TypeVar +from .exception import ConfigurationError + _T = TypeVar("_T") @@ -60,6 +63,9 @@ class _Settings: output_dir: str timeout: float verbose: bool + skip_setup: bool + dpdk_ref: Path | str + compile_timeout: float def _get_parser() -> argparse.ArgumentParser: @@ -88,6 +94,7 @@ def _get_parser() -> argparse.ArgumentParser: "--timeout", action=_env_arg("DTS_TIMEOUT"), default=15, + type=float, required=False, help="[DTS_TIMEOUT] The default timeout for all DTS operations except for " "compiling DPDK.", @@ -103,16 +110,58 @@ def _get_parser() -> argparse.ArgumentParser: "to the console.", ) + parser.add_argument( + "-s", + "--skip-setup", + action=_env_arg("DTS_SKIP_SETUP"), + required=False, + help="[DTS_SKIP_SETUP] Set to 'Y' to skip all setup steps on SUT and TG nodes.", + ) + + parser.add_argument( + "--dpdk-ref", + "--git", + "--snapshot", + action=_env_arg("DTS_DPDK_REF"), + default="dpdk.tar.xz", + required=False, + help="[DTS_DPDK_REF] Reference to DPDK source code, " + "can be either a path to a tarball or a git refspec. " + "In case of a tarball, it will be extracted in the same directory.", + ) + + parser.add_argument( + "--compile-timeout", + action=_env_arg("DTS_COMPILE_TIMEOUT"), + default=1200, + type=float, + required=False, + help="[DTS_COMPILE_TIMEOUT] The timeout for compiling DPDK.", + ) + return parser +def _check_dpdk_ref(parsed_args: argparse.Namespace) -> None: + if not os.path.exists(parsed_args.dpdk_ref): + raise ConfigurationError( + f"DPDK tarball '{parsed_args.dpdk_ref}' doesn't exist." + ) + else: + parsed_args.dpdk_ref = Path(parsed_args.dpdk_ref) + + def _get_settings() -> _Settings: parsed_args = _get_parser().parse_args() + _check_dpdk_ref(parsed_args) return _Settings( config_file_path=parsed_args.config_file, output_dir=parsed_args.output_dir, - timeout=float(parsed_args.timeout), + timeout=parsed_args.timeout, verbose=(parsed_args.verbose == "Y"), + skip_setup=(parsed_args.skip_setup == "Y"), + dpdk_ref=parsed_args.dpdk_ref, + compile_timeout=parsed_args.compile_timeout, ) diff --git a/dts/framework/testbed_model/__init__.py b/dts/framework/testbed_model/__init__.py index 8ead9db482..96e2ab7c3f 100644 --- a/dts/framework/testbed_model/__init__.py +++ b/dts/framework/testbed_model/__init__.py @@ -9,5 +9,6 @@ # pylama:ignore=W0611 +from .dpdk import MesonArgs from .node import Node from .sut_node import SutNode diff --git a/dts/framework/testbed_model/dpdk.py b/dts/framework/testbed_model/dpdk.py new file mode 100644 index 0000000000..0526974f72 --- /dev/null +++ b/dts/framework/testbed_model/dpdk.py @@ -0,0 +1,33 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2010-2014 Intel Corporation +# Copyright(c) 2023 PANTHEON.tech s.r.o. + +""" +Various utilities used for configuring, building and running DPDK. +""" + + +class MesonArgs(object): + """ + Aggregate the arguments needed to build DPDK: + default_library: Default library type, Meson allows "shared", "static" and "both". + Defaults to None, in which case the argument won't be used. + Keyword arguments: The arguments found in meson_option.txt in root DPDK directory. + Do not use -D with them, for example: enable_kmods=True. + """ + + default_library: str + + def __init__(self, default_library: str | None = None, **dpdk_args: str | bool): + self.default_library = ( + f"--default-library={default_library}" if default_library else "" + ) + self.dpdk_args = " ".join( + ( + f"-D{dpdk_arg_name}={dpdk_arg_value}" + for dpdk_arg_name, dpdk_arg_value in dpdk_args.items() + ) + ) + + def __str__(self) -> str: + return " ".join(f"{self.default_library} {self.dpdk_args}".split()) diff --git a/dts/framework/testbed_model/sut_node.py b/dts/framework/testbed_model/sut_node.py index 42acb6f9b2..c97d995b31 100644 --- a/dts/framework/testbed_model/sut_node.py +++ b/dts/framework/testbed_model/sut_node.py @@ -2,6 +2,15 @@ # Copyright(c) 2010-2014 Intel Corporation # Copyright(c) 2023 PANTHEON.tech s.r.o. +import os +import tarfile +from pathlib import PurePath + +from framework.config import BuildTargetConfiguration, NodeConfiguration +from framework.settings import SETTINGS +from framework.utils import EnvVarsDict, skip_setup + +from .dpdk import MesonArgs from .node import Node @@ -10,4 +19,153 @@ class SutNode(Node): A class for managing connections to the System under Test, providing methods that retrieve the necessary information about the node (such as CPU, memory and NIC details) and configuration capabilities. + Another key capability is building DPDK according to given build target. """ + + _build_target_config: BuildTargetConfiguration | None + _env_vars: EnvVarsDict + _remote_tmp_dir: PurePath + __remote_dpdk_dir: PurePath | None + _dpdk_version: str | None + _app_compile_timeout: float + + def __init__(self, node_config: NodeConfiguration): + super(SutNode, self).__init__(node_config) + self._build_target_config = None + self._env_vars = EnvVarsDict() + self._remote_tmp_dir = self.main_session.get_remote_tmp_dir() + self.__remote_dpdk_dir = None + self._dpdk_version = None + self._app_compile_timeout = 90 + + @property + def _remote_dpdk_dir(self) -> PurePath: + if self.__remote_dpdk_dir is None: + self.__remote_dpdk_dir = self._guess_dpdk_remote_dir() + return self.__remote_dpdk_dir + + @_remote_dpdk_dir.setter + def _remote_dpdk_dir(self, value: PurePath) -> None: + self.__remote_dpdk_dir = value + + @property + def remote_dpdk_build_dir(self) -> PurePath: + if self._build_target_config: + return self.main_session.join_remote_path( + self._remote_dpdk_dir, self._build_target_config.name + ) + else: + return self.main_session.join_remote_path(self._remote_dpdk_dir, "build") + + @property + def dpdk_version(self) -> str: + if self._dpdk_version is None: + self._dpdk_version = self.main_session.get_dpdk_version( + self._remote_dpdk_dir + ) + return self._dpdk_version + + def _guess_dpdk_remote_dir(self) -> PurePath: + return self.main_session.guess_dpdk_remote_dir(self._remote_tmp_dir) + + def _set_up_build_target( + self, build_target_config: BuildTargetConfiguration + ) -> None: + """ + Setup DPDK on the SUT node. + """ + self._configure_build_target(build_target_config) + self._copy_dpdk_tarball() + self._build_dpdk() + + def _configure_build_target( + self, build_target_config: BuildTargetConfiguration + ) -> None: + """ + Populate common environment variables and set build target config. + """ + self._env_vars = EnvVarsDict() + self._build_target_config = build_target_config + self._env_vars.update( + self.main_session.get_dpdk_build_env_vars(build_target_config.arch) + ) + self._env_vars["CC"] = build_target_config.compiler.name + if build_target_config.compiler_wrapper: + self._env_vars["CC"] = ( + f"'{build_target_config.compiler_wrapper} " + f"{build_target_config.compiler.name}'" + ) + + @skip_setup + def _copy_dpdk_tarball(self) -> None: + """ + Copy to and extract DPDK tarball on the SUT node. + """ + self._logger.info("Copying DPDK tarball to SUT.") + self.main_session.copy_file(SETTINGS.dpdk_ref, self._remote_tmp_dir) + + # construct remote tarball path + # the basename is the same on local host and on remote Node + remote_tarball_path = self.main_session.join_remote_path( + self._remote_tmp_dir, os.path.basename(SETTINGS.dpdk_ref) + ) + + # construct remote path after extracting + with tarfile.open(SETTINGS.dpdk_ref) as dpdk_tar: + dpdk_top_dir = dpdk_tar.getnames()[0] + self._remote_dpdk_dir = self.main_session.join_remote_path( + self._remote_tmp_dir, dpdk_top_dir + ) + + self._logger.info( + f"Extracting DPDK tarball on SUT: " + f"'{remote_tarball_path}' into '{self._remote_dpdk_dir}'." + ) + # clean remote path where we're extracting + self.main_session.remove_remote_dir(self._remote_dpdk_dir) + + # then extract to remote path + self.main_session.extract_remote_tarball( + remote_tarball_path, self._remote_dpdk_dir + ) + + @skip_setup + def _build_dpdk(self) -> None: + """ + Build DPDK. Uses the already configured target. Assumes that the tarball has + already been copied to and extracted on the SUT node. + """ + self.main_session.build_dpdk( + self._env_vars, + MesonArgs(default_library="static", enable_kmods=True, libdir="lib"), + self._remote_dpdk_dir, + self.remote_dpdk_build_dir, + ) + + def build_dpdk_app(self, app_name: str, **meson_dpdk_args: str | bool) -> PurePath: + """ + Build one or all DPDK apps. Requires DPDK to be already built on the SUT node. + When app_name is 'all', build all example apps. + When app_name is any other string, tries to build that example app. + Return the directory path of the built app. If building all apps, return + the path to the examples directory (where all apps reside). + The meson_dpdk_args are keyword arguments + found in meson_option.txt in root DPDK directory. Do not use -D with them, + for example: enable_kmods=True. + """ + self.main_session.build_dpdk( + self._env_vars, + MesonArgs(examples=app_name, **meson_dpdk_args), + self._remote_dpdk_dir, + self.remote_dpdk_build_dir, + rebuild=True, + timeout=self._app_compile_timeout, + ) + + if app_name == "all": + return self.main_session.join_remote_path( + self.remote_dpdk_build_dir, "examples" + ) + return self.main_session.join_remote_path( + self.remote_dpdk_build_dir, "examples", f"dpdk-{app_name}" + ) diff --git a/dts/framework/utils.py b/dts/framework/utils.py index c28c8f1082..611071604b 100644 --- a/dts/framework/utils.py +++ b/dts/framework/utils.py @@ -1,9 +1,12 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright(c) 2010-2014 Intel Corporation -# Copyright(c) 2022 PANTHEON.tech s.r.o. -# Copyright(c) 2022 University of New Hampshire +# Copyright(c) 2022-2023 PANTHEON.tech s.r.o. +# Copyright(c) 2022-2023 University of New Hampshire import sys +from typing import Callable + +from .settings import SETTINGS def check_dts_python_version() -> None: @@ -22,9 +25,21 @@ def check_dts_python_version() -> None: print(RED("Please use Python >= 3.10 instead"), file=sys.stderr) +def skip_setup(func) -> Callable[..., None]: + if SETTINGS.skip_setup: + return lambda *args: None + else: + return func + + def GREEN(text: str) -> str: return f"\u001B[32;1m{str(text)}\u001B[0m" def RED(text: str) -> str: return f"\u001B[31;1m{str(text)}\u001B[0m" + + +class EnvVarsDict(dict): + def __str__(self) -> str: + return " ".join(["=".join(item) for item in self.items()]) -- 2.30.2