From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 072F841C8A; Mon, 13 Feb 2023 16:29:19 +0100 (CET) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 1AAD742D35; Mon, 13 Feb 2023 16:28:59 +0100 (CET) Received: from mail-wr1-f50.google.com (mail-wr1-f50.google.com [209.85.221.50]) by mails.dpdk.org (Postfix) with ESMTP id AC33842D0B for ; Mon, 13 Feb 2023 16:28:54 +0100 (CET) Received: by mail-wr1-f50.google.com with SMTP id co8so8892968wrb.1 for ; Mon, 13 Feb 2023 07:28:54 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=pantheon-tech.20210112.gappssmtp.com; s=20210112; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=NMtAvBZo8gccnH+SB1Mr/QJqLAYHqpYQFZ6b8VIQFZY=; b=pYfmLhZ+NuEUllfTbLYGPnA4PhVPQdhq3rYlucebgh9Gp/ysTYnNVT61l9pDC1CT7i tGOorEB6KvdefLl258uFmSnutXgnm5aBJdCJUSBEwcjyW7dJQjN5dvvzsqPu1vmQcq0t WBxZhd+W7ml24di1htD2sRSN0iAB3QHogNOqAssAsBLy71RwRUvLi3qvY3maoevbabqm sHE0xXZpNkVldw+oWfF8AtwVfCpXR+yT4JyVfD+DYPvzLCssqRkbO1c5/OvzMRpYsLZV xYjHzWj314YM7adTJb2OZ2dWNT8jwkS0xQzJdg3lmvjN6fPFWAuBPH4I10K8l/4QX6/B mk0w== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=NMtAvBZo8gccnH+SB1Mr/QJqLAYHqpYQFZ6b8VIQFZY=; b=MtqROUHsRZPB2mlCgB8NCX2RVVV5pwx1hpLGIAQ1+JYJvmtKMOFU2AZubJHBjfFdUI nBSSJcIUEqyzvyjFNchsfoHlV6GRPYT/Gkciq2W6pgMQMrGZ39bqhWYn08pIks9z/mmQ jjv+eFPQAdnl4nxx28FzuSm+KbWzkWVbl7nFNBQ7LF09d8jPyrKxp1lQ8FSdnmcoHk9q 0+N4fxiESkIzgRBlxn52ZKmwF7nfVCFn+5cTF0QzTe5VMpfMTnqSgCauHCWqubatEZj/ UvG9UTik5Rotms/dAKE1RyVn95nq8CA5bAEqofj9U0PNmA8xWf/MuiX4WdCvEEXC7HXH yDxQ== X-Gm-Message-State: AO0yUKUcIgRLHEDWb9B+U45+0uTT6zXk/mspM99APptGVENvqnC4PUzA mT8prRShFBAhLHVt5m9QoSZRmg== X-Google-Smtp-Source: AK7set+DkIBh4Nc++hgu9sT8afdrdmm5JZiKUcC3tFP8uMcFhFWUrk2RMIngtl70cDNb6PQNjnd+/Q== X-Received: by 2002:a5d:4683:0:b0:2c5:5315:a7a7 with SMTP id u3-20020a5d4683000000b002c55315a7a7mr4157338wrq.32.1676302134305; Mon, 13 Feb 2023 07:28:54 -0800 (PST) Received: from localhost.localdomain ([84.245.121.112]) by smtp.gmail.com with ESMTPSA id d13-20020adfe88d000000b002c54f4d0f71sm5848613wrm.38.2023.02.13.07.28.53 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Mon, 13 Feb 2023 07:28:53 -0800 (PST) From: =?UTF-8?q?Juraj=20Linke=C5=A1?= To: thomas@monjalon.net, Honnappa.Nagarahalli@arm.com, ohilyard@iol.unh.edu, lijuan.tu@intel.com, bruce.richardson@intel.com, wathsala.vithanage@arm.com, probb@iol.unh.edu Cc: dev@dpdk.org, =?UTF-8?q?Juraj=20Linke=C5=A1?= Subject: [PATCH v4 03/10] dts: add dpdk build on sut Date: Mon, 13 Feb 2023 16:28:39 +0100 Message-Id: <20230213152846.284191-4-juraj.linkes@pantheon.tech> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20230213152846.284191-1-juraj.linkes@pantheon.tech> References: <20230117154906.860916-1-juraj.linkes@pantheon.tech> <20230213152846.284191-1-juraj.linkes@pantheon.tech> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Add the ability to build DPDK and apps on the SUT, using a configured target. Signed-off-by: Juraj Linkeš --- dts/framework/config/__init__.py | 2 + dts/framework/exception.py | 17 ++ dts/framework/remote_session/os_session.py | 90 +++++++++- dts/framework/remote_session/posix_session.py | 126 ++++++++++++++ .../remote_session/remote/remote_session.py | 38 ++++- .../remote_session/remote/ssh_session.py | 68 +++++++- dts/framework/settings.py | 44 ++++- dts/framework/testbed_model/__init__.py | 1 + dts/framework/testbed_model/dpdk.py | 33 ++++ dts/framework/testbed_model/sut_node.py | 158 ++++++++++++++++++ dts/framework/utils.py | 19 ++- 11 files changed, 580 insertions(+), 16 deletions(-) create mode 100644 dts/framework/testbed_model/dpdk.py diff --git a/dts/framework/config/__init__.py b/dts/framework/config/__init__.py index e3e2d74eac..ca61cb10fe 100644 --- a/dts/framework/config/__init__.py +++ b/dts/framework/config/__init__.py @@ -91,6 +91,7 @@ class BuildTargetConfiguration: os: OS cpu: CPUType compiler: Compiler + compiler_wrapper: str name: str @staticmethod @@ -100,6 +101,7 @@ def from_dict(d: dict) -> "BuildTargetConfiguration": os=OS(d["os"]), cpu=CPUType(d["cpu"]), compiler=Compiler(d["compiler"]), + compiler_wrapper=d.get("compiler_wrapper", ""), name=f"{d['arch']}-{d['os']}-{d['cpu']}-{d['compiler']}", ) diff --git a/dts/framework/exception.py b/dts/framework/exception.py index e776b42bd9..b4545a5a40 100644 --- a/dts/framework/exception.py +++ b/dts/framework/exception.py @@ -23,6 +23,7 @@ class ErrorSeverity(IntEnum): CONFIG_ERR = 2 REMOTE_CMD_EXEC_ERR = 3 SSH_ERR = 4 + DPDK_BUILD_ERR = 10 class DTSError(Exception): @@ -111,3 +112,19 @@ def __str__(self) -> str: f"Command {self.command} returned a non-zero exit code: " f"{self.command_return_code}" ) + + +class RemoteDirectoryExistsError(DTSError): + """ + Raised when a remote directory to be created already exists. + """ + + severity: ClassVar[ErrorSeverity] = ErrorSeverity.REMOTE_CMD_EXEC_ERR + + +class DPDKBuildError(DTSError): + """ + Raised when DPDK build fails for any reason. + """ + + severity: ClassVar[ErrorSeverity] = ErrorSeverity.DPDK_BUILD_ERR diff --git a/dts/framework/remote_session/os_session.py b/dts/framework/remote_session/os_session.py index 7a4cc5e669..06d1ffefdd 100644 --- a/dts/framework/remote_session/os_session.py +++ b/dts/framework/remote_session/os_session.py @@ -2,10 +2,14 @@ # Copyright(c) 2023 PANTHEON.tech s.r.o. # Copyright(c) 2023 University of New Hampshire -from abc import ABC +from abc import ABC, abstractmethod +from pathlib import PurePath -from framework.config import NodeConfiguration +from framework.config import Architecture, NodeConfiguration from framework.logger import DTSLOG +from framework.settings import SETTINGS +from framework.testbed_model import MesonArgs +from framework.utils import EnvVarsDict from .remote import RemoteSession, create_remote_session @@ -44,3 +48,85 @@ def is_alive(self) -> bool: Check whether the remote session is still responding. """ return self.remote_session.is_alive() + + @abstractmethod + def guess_dpdk_remote_dir(self, remote_dir) -> PurePath: + """ + Try to find DPDK remote dir in remote_dir. + """ + + @abstractmethod + def get_remote_tmp_dir(self) -> PurePath: + """ + Get the path of the temporary directory of the remote OS. + """ + + @abstractmethod + def get_dpdk_build_env_vars(self, arch: Architecture) -> dict: + """ + Create extra environment variables needed for the target architecture. Get + information from the node if needed. + """ + + @abstractmethod + def join_remote_path(self, *args: str | PurePath) -> PurePath: + """ + Join path parts using the path separator that fits the remote OS. + """ + + @abstractmethod + def copy_file( + self, + source_file: str | PurePath, + destination_file: str | PurePath, + source_remote: bool = False, + ) -> None: + """ + Copy source_file from local filesystem to destination_file + on the remote Node associated with the remote session. + If source_remote is True, reverse the direction - copy source_file from the + associated remote Node to destination_file on local storage. + """ + + @abstractmethod + def remove_remote_dir( + self, + remote_dir_path: str | PurePath, + recursive: bool = True, + force: bool = True, + ) -> None: + """ + Remove remote directory, by default remove recursively and forcefully. + """ + + @abstractmethod + def extract_remote_tarball( + self, + remote_tarball_path: str | PurePath, + expected_dir: str | PurePath | None = None, + ) -> None: + """ + Extract remote tarball in place. If expected_dir is a non-empty string, check + whether the dir exists after extracting the archive. + """ + + @abstractmethod + def build_dpdk( + self, + env_vars: EnvVarsDict, + meson_args: MesonArgs, + remote_dpdk_dir: str | PurePath, + remote_dpdk_build_dir: str | PurePath, + rebuild: bool = False, + timeout: float = SETTINGS.compile_timeout, + ) -> None: + """ + Build DPDK in the input dir with specified environment variables and meson + arguments. + """ + + @abstractmethod + def get_dpdk_version(self, version_path: str | PurePath) -> str: + """ + Inspect DPDK version on the remote node from version_path. + """ diff --git a/dts/framework/remote_session/posix_session.py b/dts/framework/remote_session/posix_session.py index 110b6a4804..d4da9f114e 100644 --- a/dts/framework/remote_session/posix_session.py +++ b/dts/framework/remote_session/posix_session.py @@ -2,6 +2,14 @@ # Copyright(c) 2023 PANTHEON.tech s.r.o. # Copyright(c) 2023 University of New Hampshire +from pathlib import PurePath, PurePosixPath + +from framework.config import Architecture +from framework.exception import DPDKBuildError, RemoteCommandExecutionError +from framework.settings import SETTINGS +from framework.testbed_model import MesonArgs +from framework.utils import EnvVarsDict + from .os_session import OSSession @@ -10,3 +18,121 @@ class PosixSession(OSSession): An intermediary class implementing the Posix compliant parts of Linux and other OS remote sessions. """ + + @staticmethod + def combine_short_options(**opts: bool) -> str: + ret_opts = "" + for opt, include in opts.items(): + if include: + ret_opts = f"{ret_opts}{opt}" + + if ret_opts: + ret_opts = f" -{ret_opts}" + + return ret_opts + + def guess_dpdk_remote_dir(self, remote_dir) -> PurePosixPath: + remote_guess = self.join_remote_path(remote_dir, "dpdk-*") + result = self.remote_session.send_command(f"ls -d {remote_guess} | tail -1") + return PurePosixPath(result.stdout) + + def get_remote_tmp_dir(self) -> PurePosixPath: + return PurePosixPath("/tmp") + + def get_dpdk_build_env_vars(self, arch: Architecture) -> dict: + """ + Create extra environment variables needed for i686 arch build. Get information + from the node if needed. + """ + env_vars = {} + if arch == Architecture.i686: + # find the pkg-config path and store it in PKG_CONFIG_LIBDIR + out = self.remote_session.send_command("find /usr -type d -name pkgconfig") + pkg_path = "" + res_path = out.stdout.split("\r\n") + for cur_path in res_path: + if "i386" in cur_path: + pkg_path = cur_path + break + assert pkg_path != "", "i386 pkg-config path not found" + + env_vars["CFLAGS"] = "-m32" + env_vars["PKG_CONFIG_LIBDIR"] = pkg_path + + return env_vars + + def join_remote_path(self, *args: str | PurePath) -> PurePosixPath: + return PurePosixPath(*args) + + def copy_file( + self, + source_file: str | PurePath, + destination_file: str | PurePath, + source_remote: bool = False, + ) -> None: + self.remote_session.copy_file(source_file, destination_file, source_remote) + + def remove_remote_dir( + self, + remote_dir_path: str | PurePath, + recursive: bool = True, + force: bool = True, + ) -> None: + opts = PosixSession.combine_short_options(r=recursive, f=force) + self.remote_session.send_command(f"rm{opts} {remote_dir_path}") + + def extract_remote_tarball( + self, + remote_tarball_path: str | PurePath, + expected_dir: str | PurePath | None = None, + ) -> None: + self.remote_session.send_command( + f"tar xfm {remote_tarball_path} " + f"-C {PurePosixPath(remote_tarball_path).parent}", + 60, + ) + if expected_dir: + self.remote_session.send_command(f"ls {expected_dir}", verify=True) + + def build_dpdk( + self, + env_vars: EnvVarsDict, + meson_args: MesonArgs, + remote_dpdk_dir: str | PurePath, + remote_dpdk_build_dir: str | PurePath, + rebuild: bool = False, + timeout: float = SETTINGS.compile_timeout, + ) -> None: + try: + if rebuild: + # reconfigure, then build + self._logger.info("Reconfiguring DPDK build.") + self.remote_session.send_command( + f"meson configure {meson_args} {remote_dpdk_build_dir}", + timeout, + verify=True, + env=env_vars, + ) + else: + # fresh build - remove target dir first, then build from scratch + self._logger.info("Configuring DPDK build from scratch.") + self.remove_remote_dir(remote_dpdk_build_dir) + self.remote_session.send_command( + f"meson {meson_args} {remote_dpdk_dir} {remote_dpdk_build_dir}", + timeout, + verify=True, + env=env_vars, + ) + + self._logger.info("Building DPDK.") + self.remote_session.send_command( + f"ninja -C {remote_dpdk_build_dir}", timeout, verify=True, env=env_vars + ) + except RemoteCommandExecutionError as e: + raise DPDKBuildError(f"DPDK build failed when doing '{e.command}'.") + + def get_dpdk_version(self, build_dir: str | PurePath) -> str: + out = self.remote_session.send_command( + f"cat {self.join_remote_path(build_dir, 'VERSION')}", verify=True + ) + return out.stdout diff --git a/dts/framework/remote_session/remote/remote_session.py b/dts/framework/remote_session/remote/remote_session.py index 5ac395ec79..91dee3cb4f 100644 --- a/dts/framework/remote_session/remote/remote_session.py +++ b/dts/framework/remote_session/remote/remote_session.py @@ -5,11 +5,13 @@ import dataclasses from abc import ABC, abstractmethod +from pathlib import PurePath from framework.config import NodeConfiguration from framework.exception import RemoteCommandExecutionError from framework.logger import DTSLOG from framework.settings import SETTINGS +from framework.utils import EnvVarsDict @dataclasses.dataclass(slots=True, frozen=True) @@ -83,15 +85,22 @@ def _connect(self) -> None: """ def send_command( - self, command: str, timeout: float = SETTINGS.timeout, verify: bool = False + self, + command: str, + timeout: float = SETTINGS.timeout, + verify: bool = False, + env: EnvVarsDict | None = None, ) -> CommandResult: """ - Send a command to the connected node and return CommandResult. + Send a command to the connected node using optional env vars + and return CommandResult. If verify is True, check the return code of the executed command and raise a RemoteCommandExecutionError if the command failed. """ - self._logger.info(f"Sending: '{command}'") - result = self._send_command(command, timeout) + self._logger.info( + f"Sending: '{command}'" + (f" with env vars: '{env}'" if env else "") + ) + result = self._send_command(command, timeout, env) if verify and result.return_code: self._logger.debug( f"Command '{command}' failed with return code '{result.return_code}'" @@ -104,9 +113,12 @@ def send_command( return result @abstractmethod - def _send_command(self, command: str, timeout: float) -> CommandResult: + def _send_command( + self, command: str, timeout: float, env: EnvVarsDict | None + ) -> CommandResult: """ - Use the underlying protocol to execute the command and return CommandResult. + Use the underlying protocol to execute the command using optional env vars + and return CommandResult. """ def close(self, force: bool = False) -> None: @@ -127,3 +139,17 @@ def is_alive(self) -> bool: """ Check whether the remote session is still responding. """ + + @abstractmethod + def copy_file( + self, + source_file: str | PurePath, + destination_file: str | PurePath, + source_remote: bool = False, + ) -> None: + """ + Copy source_file from local filesystem to destination_file on the remote Node + associated with the remote session. + If source_remote is True, reverse the direction - copy source_file from the + associated Node to destination_file on local filesystem. + """ diff --git a/dts/framework/remote_session/remote/ssh_session.py b/dts/framework/remote_session/remote/ssh_session.py index 6da5be9fff..d0863d8791 100644 --- a/dts/framework/remote_session/remote/ssh_session.py +++ b/dts/framework/remote_session/remote/ssh_session.py @@ -4,13 +4,15 @@ # Copyright(c) 2022-2023 University of New Hampshire import time +from pathlib import PurePath +import pexpect # type: ignore from pexpect import pxssh # type: ignore from framework.config import NodeConfiguration from framework.exception import SSHConnectionError, SSHSessionDeadError, SSHTimeoutError from framework.logger import DTSLOG -from framework.utils import GREEN, RED +from framework.utils import GREEN, RED, EnvVarsDict from .remote_session import CommandResult, RemoteSession @@ -163,16 +165,22 @@ def _flush(self) -> None: def is_alive(self) -> bool: return self.session.isalive() - def _send_command(self, command: str, timeout: float) -> CommandResult: - output = self._send_command_get_output(command, timeout) - return_code = int(self._send_command_get_output("echo $?", timeout)) + def _send_command( + self, command: str, timeout: float, env: EnvVarsDict | None + ) -> CommandResult: + output = self._send_command_get_output(command, timeout, env) + return_code = int(self._send_command_get_output("echo $?", timeout, None)) # we're capturing only stdout return CommandResult(self.name, command, output, "", return_code) - def _send_command_get_output(self, command: str, timeout: float) -> str: + def _send_command_get_output( + self, command: str, timeout: float, env: EnvVarsDict | None + ) -> str: try: self._clean_session() + if env: + command = f"{env} {command}" self._send_line(command) except Exception as e: raise e @@ -189,3 +197,53 @@ def _close(self, force: bool = False) -> None: else: if self.is_alive(): self.session.logout() + + def copy_file( + self, + source_file: str | PurePath, + destination_file: str | PurePath, + source_remote: bool = False, + ) -> None: + """ + Send a local file to a remote host. + """ + if source_remote: + source_file = f"{self.username}@{self.ip}:{source_file}" + else: + destination_file = f"{self.username}@{self.ip}:{destination_file}" + + port = "" + if self.port: + port = f" -P {self.port}" + + # this is not OS agnostic, find a Pythonic (and thus OS agnostic) way + # TODO Fabric should handle this + command = ( + f"scp -v{port} -o NoHostAuthenticationForLocalhost=yes" + f" {source_file} {destination_file}" + ) + + self._spawn_scp(command) + + def _spawn_scp(self, scp_cmd: str) -> None: + """ + Transfer a file with SCP + """ + self._logger.info(scp_cmd) + p: pexpect.spawn = pexpect.spawn(scp_cmd) + time.sleep(0.5) + ssh_newkey: str = "Are you sure you want to continue connecting" + i: int = p.expect( + [ssh_newkey, "[pP]assword", "# ", pexpect.EOF, pexpect.TIMEOUT], 120 + ) + if i == 0: # add once in trust list + p.sendline("yes") + i = p.expect([ssh_newkey, "[pP]assword", pexpect.EOF], 2) + + if i == 1: + time.sleep(0.5) + p.sendline(self.password) + p.expect("Exit status 0", 60) + if i == 4: + self._logger.error("SCP TIMEOUT error %d" % i) + p.close() diff --git a/dts/framework/settings.py b/dts/framework/settings.py index 6422b23499..f787187ade 100644 --- a/dts/framework/settings.py +++ b/dts/framework/settings.py @@ -7,8 +7,11 @@ import os from collections.abc import Callable, Iterable, Sequence from dataclasses import dataclass +from pathlib import Path from typing import Any, TypeVar +from .exception import ConfigurationError + _T = TypeVar("_T") @@ -60,6 +63,9 @@ class _Settings: output_dir: str timeout: float verbose: bool + skip_setup: bool + dpdk_tarball_path: Path + compile_timeout: float def _get_parser() -> argparse.ArgumentParser: @@ -91,6 +97,7 @@ def _get_parser() -> argparse.ArgumentParser: "--timeout", action=_env_arg("DTS_TIMEOUT"), default=15, + type=float, help="[DTS_TIMEOUT] The default timeout for all DTS operations except for " "compiling DPDK.", ) @@ -104,16 +111,51 @@ def _get_parser() -> argparse.ArgumentParser: "to the console.", ) + parser.add_argument( + "-s", + "--skip-setup", + action=_env_arg("DTS_SKIP_SETUP"), + default="N", + help="[DTS_SKIP_SETUP] Set to 'Y' to skip all setup steps on SUT and TG nodes.", + ) + + parser.add_argument( + "--tarball", + "--snapshot", + action=_env_arg("DTS_DPDK_TARBALL"), + default="dpdk.tar.xz", + type=Path, + help="[DTS_DPDK_TARBALL] Path to DPDK source code tarball " + "which will be used in testing.", + ) + + parser.add_argument( + "--compile-timeout", + action=_env_arg("DTS_COMPILE_TIMEOUT"), + default=1200, + type=float, + help="[DTS_COMPILE_TIMEOUT] The timeout for compiling DPDK.", + ) + return parser +def _check_tarball_path(parsed_args: argparse.Namespace) -> None: + if not os.path.exists(parsed_args.tarball): + raise ConfigurationError(f"DPDK tarball '{parsed_args.tarball}' doesn't exist.") + + def _get_settings() -> _Settings: parsed_args = _get_parser().parse_args() + _check_tarball_path(parsed_args) return _Settings( config_file_path=parsed_args.config_file, output_dir=parsed_args.output_dir, - timeout=float(parsed_args.timeout), + timeout=parsed_args.timeout, verbose=(parsed_args.verbose == "Y"), + skip_setup=(parsed_args.skip_setup == "Y"), + dpdk_tarball_path=parsed_args.tarball, + compile_timeout=parsed_args.compile_timeout, ) diff --git a/dts/framework/testbed_model/__init__.py b/dts/framework/testbed_model/__init__.py index 8ead9db482..96e2ab7c3f 100644 --- a/dts/framework/testbed_model/__init__.py +++ b/dts/framework/testbed_model/__init__.py @@ -9,5 +9,6 @@ # pylama:ignore=W0611 +from .dpdk import MesonArgs from .node import Node from .sut_node import SutNode diff --git a/dts/framework/testbed_model/dpdk.py b/dts/framework/testbed_model/dpdk.py new file mode 100644 index 0000000000..0526974f72 --- /dev/null +++ b/dts/framework/testbed_model/dpdk.py @@ -0,0 +1,33 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2010-2014 Intel Corporation +# Copyright(c) 2023 PANTHEON.tech s.r.o. + +""" +Various utilities used for configuring, building and running DPDK. +""" + + +class MesonArgs(object): + """ + Aggregate the arguments needed to build DPDK: + default_library: Default library type, Meson allows "shared", "static" and "both". + Defaults to None, in which case the argument won't be used. + Keyword arguments: The arguments found in meson_option.txt in root DPDK directory. + Do not use -D with them, for example: enable_kmods=True. + """ + + default_library: str + + def __init__(self, default_library: str | None = None, **dpdk_args: str | bool): + self.default_library = ( + f"--default-library={default_library}" if default_library else "" + ) + self.dpdk_args = " ".join( + ( + f"-D{dpdk_arg_name}={dpdk_arg_value}" + for dpdk_arg_name, dpdk_arg_value in dpdk_args.items() + ) + ) + + def __str__(self) -> str: + return " ".join(f"{self.default_library} {self.dpdk_args}".split()) diff --git a/dts/framework/testbed_model/sut_node.py b/dts/framework/testbed_model/sut_node.py index 42acb6f9b2..442a41bdc8 100644 --- a/dts/framework/testbed_model/sut_node.py +++ b/dts/framework/testbed_model/sut_node.py @@ -2,6 +2,15 @@ # Copyright(c) 2010-2014 Intel Corporation # Copyright(c) 2023 PANTHEON.tech s.r.o. +import os +import tarfile +from pathlib import PurePath + +from framework.config import BuildTargetConfiguration, NodeConfiguration +from framework.settings import SETTINGS +from framework.utils import EnvVarsDict, skip_setup + +from .dpdk import MesonArgs from .node import Node @@ -10,4 +19,153 @@ class SutNode(Node): A class for managing connections to the System under Test, providing methods that retrieve the necessary information about the node (such as CPU, memory and NIC details) and configuration capabilities. + Another key capability is building DPDK according to given build target. """ + + _build_target_config: BuildTargetConfiguration | None + _env_vars: EnvVarsDict + _remote_tmp_dir: PurePath + __remote_dpdk_dir: PurePath | None + _dpdk_version: str | None + _app_compile_timeout: float + + def __init__(self, node_config: NodeConfiguration): + super(SutNode, self).__init__(node_config) + self._build_target_config = None + self._env_vars = EnvVarsDict() + self._remote_tmp_dir = self.main_session.get_remote_tmp_dir() + self.__remote_dpdk_dir = None + self._dpdk_version = None + self._app_compile_timeout = 90 + + @property + def _remote_dpdk_dir(self) -> PurePath: + if self.__remote_dpdk_dir is None: + self.__remote_dpdk_dir = self._guess_dpdk_remote_dir() + return self.__remote_dpdk_dir + + @_remote_dpdk_dir.setter + def _remote_dpdk_dir(self, value: PurePath) -> None: + self.__remote_dpdk_dir = value + + @property + def remote_dpdk_build_dir(self) -> PurePath: + if self._build_target_config: + return self.main_session.join_remote_path( + self._remote_dpdk_dir, self._build_target_config.name + ) + else: + return self.main_session.join_remote_path(self._remote_dpdk_dir, "build") + + @property + def dpdk_version(self) -> str: + if self._dpdk_version is None: + self._dpdk_version = self.main_session.get_dpdk_version( + self._remote_dpdk_dir + ) + return self._dpdk_version + + def _guess_dpdk_remote_dir(self) -> PurePath: + return self.main_session.guess_dpdk_remote_dir(self._remote_tmp_dir) + + def _set_up_build_target( + self, build_target_config: BuildTargetConfiguration + ) -> None: + """ + Setup DPDK on the SUT node. + """ + self._configure_build_target(build_target_config) + self._copy_dpdk_tarball() + self._build_dpdk() + + def _configure_build_target( + self, build_target_config: BuildTargetConfiguration + ) -> None: + """ + Populate common environment variables and set build target config. + """ + self._env_vars = EnvVarsDict() + self._build_target_config = build_target_config + self._env_vars.update( + self.main_session.get_dpdk_build_env_vars(build_target_config.arch) + ) + self._env_vars["CC"] = build_target_config.compiler.name + if build_target_config.compiler_wrapper: + self._env_vars["CC"] = ( + f"'{build_target_config.compiler_wrapper} " + f"{build_target_config.compiler.name}'" + ) + + @skip_setup + def _copy_dpdk_tarball(self) -> None: + """ + Copy to and extract DPDK tarball on the SUT node. + """ + self._logger.info("Copying DPDK tarball to SUT.") + self.main_session.copy_file(SETTINGS.dpdk_tarball_path, self._remote_tmp_dir) + + # construct remote tarball path + # the basename is the same on local host and on remote Node + remote_tarball_path = self.main_session.join_remote_path( + self._remote_tmp_dir, os.path.basename(SETTINGS.dpdk_tarball_path) + ) + + # construct remote path after extracting + with tarfile.open(SETTINGS.dpdk_tarball_path) as dpdk_tar: + dpdk_top_dir = dpdk_tar.getnames()[0] + self._remote_dpdk_dir = self.main_session.join_remote_path( + self._remote_tmp_dir, dpdk_top_dir + ) + + self._logger.info( + f"Extracting DPDK tarball on SUT: " + f"'{remote_tarball_path}' into '{self._remote_dpdk_dir}'." + ) + # clean remote path where we're extracting + self.main_session.remove_remote_dir(self._remote_dpdk_dir) + + # then extract to remote path + self.main_session.extract_remote_tarball( + remote_tarball_path, self._remote_dpdk_dir + ) + + @skip_setup + def _build_dpdk(self) -> None: + """ + Build DPDK. Uses the already configured target. Assumes that the tarball has + already been copied to and extracted on the SUT node. + """ + self.main_session.build_dpdk( + self._env_vars, + MesonArgs(default_library="static", enable_kmods=True, libdir="lib"), + self._remote_dpdk_dir, + self.remote_dpdk_build_dir, + ) + + def build_dpdk_app(self, app_name: str, **meson_dpdk_args: str | bool) -> PurePath: + """ + Build one or all DPDK apps. Requires DPDK to be already built on the SUT node. + When app_name is 'all', build all example apps. + When app_name is any other string, tries to build that example app. + Return the directory path of the built app. If building all apps, return + the path to the examples directory (where all apps reside). + The meson_dpdk_args are keyword arguments + found in meson_option.txt in root DPDK directory. Do not use -D with them, + for example: enable_kmods=True. + """ + self.main_session.build_dpdk( + self._env_vars, + MesonArgs(examples=app_name, **meson_dpdk_args), + self._remote_dpdk_dir, + self.remote_dpdk_build_dir, + rebuild=True, + timeout=self._app_compile_timeout, + ) + + if app_name == "all": + return self.main_session.join_remote_path( + self.remote_dpdk_build_dir, "examples" + ) + return self.main_session.join_remote_path( + self.remote_dpdk_build_dir, "examples", f"dpdk-{app_name}" + ) diff --git a/dts/framework/utils.py b/dts/framework/utils.py index c28c8f1082..611071604b 100644 --- a/dts/framework/utils.py +++ b/dts/framework/utils.py @@ -1,9 +1,12 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright(c) 2010-2014 Intel Corporation -# Copyright(c) 2022 PANTHEON.tech s.r.o. -# Copyright(c) 2022 University of New Hampshire +# Copyright(c) 2022-2023 PANTHEON.tech s.r.o. +# Copyright(c) 2022-2023 University of New Hampshire import sys +from typing import Callable + +from .settings import SETTINGS def check_dts_python_version() -> None: @@ -22,9 +25,21 @@ def check_dts_python_version() -> None: print(RED("Please use Python >= 3.10 instead"), file=sys.stderr) +def skip_setup(func) -> Callable[..., None]: + if SETTINGS.skip_setup: + return lambda *args: None + else: + return func + + def GREEN(text: str) -> str: return f"\u001B[32;1m{str(text)}\u001B[0m" def RED(text: str) -> str: return f"\u001B[31;1m{str(text)}\u001B[0m" + + +class EnvVarsDict(dict): + def __str__(self) -> str: + return " ".join(["=".join(item) for item in self.items()]) -- 2.30.2