From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 7211EA055F; Wed, 16 Nov 2022 14:15:59 +0100 (CET) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 5D20D40E03; Wed, 16 Nov 2022 14:15:59 +0100 (CET) Received: from mail-pf1-f176.google.com (mail-pf1-f176.google.com [209.85.210.176]) by mails.dpdk.org (Postfix) with ESMTP id 40BEE40DFB for ; Wed, 16 Nov 2022 14:15:57 +0100 (CET) Received: by mail-pf1-f176.google.com with SMTP id 130so17430423pfu.8 for ; Wed, 16 Nov 2022 05:15:57 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=iol.unh.edu; s=unh-iol; h=cc:to:subject:message-id:date:from:in-reply-to:references :mime-version:from:to:cc:subject:date:message-id:reply-to; bh=xx4/ix22cQm6BRgG0fBTJ3ri2UEXX1/ug0m56sVyJBI=; b=Din7oRO+TmJ3MJ11ATGK6ik2gJ+kzBmOKL6Net4pYIAOO/ztv9CJ2IGxlajD4yGBas gG1YSp4d3FcASYNvD9/xkyc8yQASsf3mpz13c9I3dJau5jJPr04tfWnjPIOmzMVhVx1r Z6VC/1vbFc6iL8b7lW+0yO7TJ0uzXTx8egZUk= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=cc:to:subject:message-id:date:from:in-reply-to:references :mime-version:x-gm-message-state:from:to:cc:subject:date:message-id :reply-to; bh=xx4/ix22cQm6BRgG0fBTJ3ri2UEXX1/ug0m56sVyJBI=; b=HVsF14/aIWtBemW0SQ8doJZvKmTh9o8XCYSTooUy9XmogJW6X5C2Y3BveDO4ZXaR6a sy9Aaa/tGv7vSiKr19ArReiIa2M/m07TJ66SRtdqBJ1IjqSwXBw4d2GA0gqQvKeiTe5w g9bjzVLMlETEIUMxjF0q/XyfXuv95CbwDAmPNvdvkblNGjbtweasy25BrF0u+OwCMTsS XxY0j7rcj/l4in+NuMF/c2e6tq4a+MP/nwbAUsIeviWxt9yXCwXomRu1PTb4XtYGZ9vC NwtztgbWNPfn0hqrAVF4MKhkkecfKXZI2cTmLVVShWQtfiT22T8Uk7FDKik/nsZf5M5Q phgg== X-Gm-Message-State: ANoB5plTJrkpscb+wTWQqunpF1lz2r8BiMKYQZNIGpI4M8H6VNiDdTar yMzSeZCBs9DK5XXO/vnY7jrW3IRr5dDcie1D/PXGiA== X-Google-Smtp-Source: AA0mqf6CAYnRYywaJkLmrjEJgj9OSE2J4NhehOQrurZtElZIXi+93/usstHjkwogZ5pTJ5+PSNztje7VnDesq/++7YE= X-Received: by 2002:a63:d107:0:b0:46e:baec:59bd with SMTP id k7-20020a63d107000000b0046ebaec59bdmr20271437pgg.528.1668604556274; Wed, 16 Nov 2022 05:15:56 -0800 (PST) MIME-Version: 1.0 References: <20220824162454.394285-1-juraj.linkes@pantheon.tech> <20221114165438.1133783-1-juraj.linkes@pantheon.tech> <20221114165438.1133783-4-juraj.linkes@pantheon.tech> In-Reply-To: <20221114165438.1133783-4-juraj.linkes@pantheon.tech> From: Owen Hilyard Date: Wed, 16 Nov 2022 08:15:20 -0500 Message-ID: Subject: Re: [RFC PATCH v2 03/10] dts: add dpdk build on sut To: =?UTF-8?Q?Juraj_Linke=C5=A1?= Cc: thomas@monjalon.net, Honnappa.Nagarahalli@arm.com, lijuan.tu@intel.com, bruce.richardson@intel.com, dev@dpdk.org Content-Type: multipart/alternative; boundary="000000000000d911c505ed964567" X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org --000000000000d911c505ed964567 Content-Type: text/plain; charset="UTF-8" Content-Transfer-Encoding: quoted-printable On Mon, Nov 14, 2022 at 11:54 AM Juraj Linke=C5=A1 wrote: > Add the ability to build DPDK and apps, using a configured target. > > Signed-off-by: Juraj Linke=C5=A1 > --- > dts/framework/exception.py | 17 +++ > dts/framework/remote_session/os/os_session.py | 90 +++++++++++- > .../remote_session/os/posix_session.py | 128 +++++++++++++++++ > .../remote_session/remote_session.py | 34 ++++- > dts/framework/remote_session/ssh_session.py | 64 ++++++++- > dts/framework/settings.py | 40 +++++- > dts/framework/testbed_model/node/sut_node.py | 131 ++++++++++++++++++ > dts/framework/utils.py | 15 ++ > 8 files changed, 505 insertions(+), 14 deletions(-) > > diff --git a/dts/framework/exception.py b/dts/framework/exception.py > index b282e48198..93d99432ae 100644 > --- a/dts/framework/exception.py > +++ b/dts/framework/exception.py > @@ -26,6 +26,7 @@ class ReturnCode(IntEnum): > GENERIC_ERR =3D 1 > SSH_ERR =3D 2 > REMOTE_CMD_EXEC_ERR =3D 3 > + DPDK_BUILD_ERR =3D 10 > NODE_SETUP_ERR =3D 20 > NODE_CLEANUP_ERR =3D 21 > > @@ -110,6 +111,22 @@ def __str__(self) -> str: > ) > > > +class RemoteDirectoryExistsError(DTSError): > + """ > + Raised when a remote directory to be created already exists. > + """ > + > + return_code: ClassVar[ReturnCode] =3D ReturnCode.REMOTE_CMD_EXEC_ERR > + > + > +class DPDKBuildError(DTSError): > + """ > + Raised when DPDK build fails for any reason. > + """ > + > + return_code: ClassVar[ReturnCode] =3D ReturnCode.DPDK_BUILD_ERR > + > + > class NodeSetupError(DTSError): > """ > Raised when setting up a node. > diff --git a/dts/framework/remote_session/os/os_session.py > b/dts/framework/remote_session/os/os_session.py > index 2a72082628..57e2865282 100644 > --- a/dts/framework/remote_session/os/os_session.py > +++ b/dts/framework/remote_session/os/os_session.py > @@ -2,12 +2,15 @@ > # Copyright(c) 2022 PANTHEON.tech s.r.o. > # Copyright(c) 2022 University of New Hampshire > > -from abc import ABC > +from abc import ABC, abstractmethod > +from pathlib import PurePath > > -from framework.config import NodeConfiguration > +from framework.config import Architecture, NodeConfiguration > from framework.logger import DTSLOG > from framework.remote_session.factory import create_remote_session > from framework.remote_session.remote_session import RemoteSession > +from framework.settings import SETTINGS > +from framework.utils import EnvVarsDict > > > class OSSession(ABC): > @@ -44,3 +47,86 @@ def is_alive(self) -> bool: > Check whether the remote session is still responding. > """ > return self.remote_session.is_alive() > + > + @abstractmethod > + def guess_dpdk_remote_dir(self, remote_dir) -> PurePath: > + """ > + Try to find DPDK remote dir in remote_dir. > + """ > + > + @abstractmethod > + def get_remote_tmp_dir(self) -> PurePath: > + """ > + Get the path of the temporary directory of the remote OS. > + """ > + > + @abstractmethod > + def get_dpdk_build_env_vars(self, arch: Architecture) -> dict: > + """ > + Create extra environment variables needed for the target > architecture. Get > + information from the node if needed. > + """ > + > + @abstractmethod > + def join_remote_path(self, *args: str | PurePath) -> PurePath: > + """ > + Join path parts using the path separator that fits the remote OS= . > + """ > + > + @abstractmethod > + def copy_file( > + self, > + source_file: str | PurePath, > + destination_file: str | PurePath, > + source_remote: bool =3D False, > + ) -> None: > + """ > + Copy source_file from local storage to destination_file on the > remote Node > + associated with the remote session. > + If source_remote is True, reverse the direction - copy > source_file from the > + associated remote Node to destination_file on local storage. > + """ > + > + @abstractmethod > + def remove_remote_dir( > + self, > + remote_dir_path: str | PurePath, > + recursive: bool =3D True, > + force: bool =3D True, > + ) -> None: > + """ > + Remove remote directory, by default remove recursively and > forcefully. > + """ > + > + @abstractmethod > + def extract_remote_tarball( > + self, > + remote_tarball_path: str | PurePath, > + expected_dir: str | PurePath | None =3D None, > + ) -> None: > + """ > + Extract remote tarball in place. If expected_dir is a non-empty > string, check > + whether the dir exists after extracting the archive. > + """ > + > + @abstractmethod > + def build_dpdk( > + self, > + env_vars: EnvVarsDict, > + meson_args: str, > + remote_dpdk_dir: str | PurePath, > + target_name: str, > + rebuild: bool =3D False, > + timeout: float =3D SETTINGS.compile_timeout, > + ) -> PurePath: > I think that we should consider having a MesonArgs type which implements the builder pattern. That way common things like static vs dynamic linking, enabling lto, setting the optimization level, et can be handled via dedicated methods, and then we can add a method on that which is "add this string onto the end". This would also allow defining additional methods for DPDK-specific meson arguments, like only enabling certain drivers/applications/tests or forcing certain vector widths. I would also like to see an option to make use of ccache, because currently the only way I see to do that is via environment variables, which will make creating a test matrix that includes multiple compilers difficult. > + """ > + Build DPDK in the input dir with specified environment variables > and meson > + arguments. > + Return the directory path where DPDK was built. > + """ > + > + @abstractmethod > + def get_dpdk_version(self, version_path: str | PurePath) -> str: > + """ > + Inspect DPDK version on the remote node from version_path. > + """ > diff --git a/dts/framework/remote_session/os/posix_session.py > b/dts/framework/remote_session/os/posix_session.py > index 9622a4ea30..a36b8e8c1a 100644 > --- a/dts/framework/remote_session/os/posix_session.py > +++ b/dts/framework/remote_session/os/posix_session.py > @@ -2,6 +2,13 @@ > # Copyright(c) 2022 PANTHEON.tech s.r.o. > # Copyright(c) 2022 University of New Hampshire > > +from pathlib import PurePath, PurePosixPath > + > +from framework.config import Architecture > +from framework.exception import DPDKBuildError, > RemoteCommandExecutionError > +from framework.settings import SETTINGS > +from framework.utils import EnvVarsDict > + > from .os_session import OSSession > > > @@ -10,3 +17,124 @@ class PosixSession(OSSession): > An intermediary class implementing the Posix compliant parts of > Linux and other OS remote sessions. > """ > + > + @staticmethod > + def combine_short_options(**opts: [str, bool]) -> str: > + ret_opts =3D "" > + for opt, include in opts.items(): > + if include: > + ret_opts =3D f"{ret_opts}{opt}" > + > + if ret_opts: > + ret_opts =3D f" -{ret_opts}" > + > + return ret_opts > + > + def guess_dpdk_remote_dir(self, remote_dir) -> PurePosixPath: > + remote_guess =3D self.join_remote_path(remote_dir, "dpdk-*") > + result =3D self.remote_session.send_command(f"ls -d {remote_gues= s} > | tail -1") > + return PurePosixPath(result.stdout) > + > + def get_remote_tmp_dir(self) -> PurePosixPath: > + return PurePosixPath("/tmp") > + > + def get_dpdk_build_env_vars(self, arch: Architecture) -> dict: > + """ > + Create extra environment variables needed for i686 arch build. > Get information > + from the node if needed. > + """ > + env_vars =3D {} > + if arch =3D=3D Architecture.i686: > + # find the pkg-config path and store it in PKG_CONFIG_LIBDIR > + out =3D self.remote_session.send_command("find /usr -type d > -name pkgconfig") > + pkg_path =3D "" > + res_path =3D out.stdout.split("\r\n") > + for cur_path in res_path: > + if "i386" in cur_path: > + pkg_path =3D cur_path > + break > + assert pkg_path !=3D "", "i386 pkg-config path not found" > + > + env_vars["CFLAGS"] =3D "-m32" > + env_vars["PKG_CONFIG_LIBDIR"] =3D pkg_path > + > + return env_vars > + > + def join_remote_path(self, *args: str | PurePath) -> PurePosixPath: > + return PurePosixPath(*args) > + > + def copy_file( > + self, > + source_file: str | PurePath, > + destination_file: str | PurePath, > + source_remote: bool =3D False, > + ) -> None: > + self.remote_session.copy_file(source_file, destination_file, > source_remote) > + > + def remove_remote_dir( > + self, > + remote_dir_path: str | PurePath, > + recursive: bool =3D True, > + force: bool =3D True, > + ) -> None: > + opts =3D PosixSession.combine_short_options(r=3Drecursive, f=3Df= orce) > + self.remote_session.send_command(f"rm{opts} {remote_dir_path}") > + > + def extract_remote_tarball( > + self, > + remote_tarball_path: str | PurePath, > + expected_dir: str | PurePath | None =3D None, > + ) -> None: > + self.remote_session.send_command( > + f"tar xfm {remote_tarball_path} " > + f"-C {PurePosixPath(remote_tarball_path).parent}", > + 60, > + ) > + if expected_dir: > + self.remote_session.send_command(f"ls {expected_dir}", > verify=3DTrue) > + > + def build_dpdk( > + self, > + env_vars: EnvVarsDict, > + meson_args: str, > + remote_dpdk_dir: str | PurePath, > + target_name: str, > + rebuild: bool =3D False, > + timeout: float =3D SETTINGS.compile_timeout, > + ) -> PurePosixPath: > + build_dir =3D self.join_remote_path(remote_dpdk_dir, target_name= ) > + try: > + if rebuild: > + # reconfigure, then build > + self.logger.info("Reconfiguring DPDK build.") > + self.remote_session.send_command( > + f"meson configure {meson_args} {build_dir}", > + timeout, > + verify=3DTrue, > + env=3Denv_vars, > + ) > + else: > + # fresh build - remove target dir first, then build from > scratch > + self.logger.info("Configuring DPDK build from scratch.") > + self.remove_remote_dir(build_dir) > + self.remote_session.send_command( > + f"meson {meson_args} {remote_dpdk_dir} {build_dir}", > + timeout, > + verify=3DTrue, > + env=3Denv_vars, > + ) > + > + self.logger.info("Building DPDK.") > + self.remote_session.send_command( > + f"ninja -C {build_dir}", timeout, verify=3DTrue, > env=3Denv_vars > + ) > + except RemoteCommandExecutionError as e: > + raise DPDKBuildError(f"DPDK build failed when doing > '{e.command}'.") > + > + return build_dir > + > + def get_dpdk_version(self, build_dir: str | PurePath) -> str: > + out =3D self.remote_session.send_command( > + f"cat {self.join_remote_path(build_dir, 'VERSION')}", > verify=3DTrue > + ) > + return out.stdout > diff --git a/dts/framework/remote_session/remote_session.py > b/dts/framework/remote_session/remote_session.py > index fccd80a529..f10b1023f8 100644 > --- a/dts/framework/remote_session/remote_session.py > +++ b/dts/framework/remote_session/remote_session.py > @@ -10,6 +10,7 @@ > from framework.exception import RemoteCommandExecutionError > from framework.logger import DTSLOG > from framework.settings import SETTINGS > +from framework.utils import EnvVarsDict > > > @dataclasses.dataclass(slots=3DTrue, frozen=3DTrue) > @@ -83,15 +84,22 @@ def _connect(self) -> None: > """ > > def send_command( > - self, command: str, timeout: float =3D SETTINGS.timeout, verify: > bool =3D False > + self, > + command: str, > + timeout: float =3D SETTINGS.timeout, > + verify: bool =3D False, > + env: EnvVarsDict | None =3D None, ) -> CommandResult: > """ > - Send a command to the connected node and return CommandResult. > + Send a command to the connected node using optional env vars > + and return CommandResult. > If verify is True, check the return code of the executed command > and raise a RemoteCommandExecutionError if the command failed. > """ > - self.logger.info(f"Sending: '{command}'") > - result =3D self._send_command(command, timeout) > + self.logger.info( > + f"Sending: '{command}'" + (f" with env vars: '{env}'" if env > else "") > + ) > + result =3D self._send_command(command, timeout, env) > if verify and result.return_code: > self.logger.debug( > f"Command '{command}' failed with return code > '{result.return_code}'" > @@ -104,9 +112,12 @@ def send_command( > return result > > @abstractmethod > - def _send_command(self, command: str, timeout: float) -> > CommandResult: > + def _send_command( > + self, command: str, timeout: float, env: EnvVarsDict | None > + ) -> CommandResult: > """ > - Use the underlying protocol to execute the command and return > CommandResult. > + Use the underlying protocol to execute the command using optiona= l > env vars > + and return CommandResult. > """ > > def close(self, force: bool =3D False) -> None: > @@ -127,3 +138,14 @@ def is_alive(self) -> bool: > """ > Check whether the remote session is still responding. > """ > + > + @abstractmethod > + def copy_file( > + self, source_file: str, destination_file: str, source_remote: > bool =3D False > + ) -> None: > + """ > + Copy source_file from local storage to destination_file on the > remote Node > This should clarify that local storage means inside of the DTS container, not the system it is running on. > + associated with the remote session. > + If source_remote is True, reverse the direction - copy > source_file from the > + associated Node to destination_file on local storage. > + """ > diff --git a/dts/framework/remote_session/ssh_session.py > b/dts/framework/remote_session/ssh_session.py > index fb2f01dbc1..d4a6714e6b 100644 > --- a/dts/framework/remote_session/ssh_session.py > +++ b/dts/framework/remote_session/ssh_session.py > @@ -5,12 +5,13 @@ > > import time > > +import pexpect # type: ignore > from pexpect import pxssh # type: ignore > > from framework.config import NodeConfiguration > from framework.exception import SSHConnectionError, SSHSessionDeadError, > SSHTimeoutError > from framework.logger import DTSLOG > -from framework.utils import GREEN, RED > +from framework.utils import GREEN, RED, EnvVarsDict > > from .remote_session import CommandResult, RemoteSession > > @@ -163,16 +164,22 @@ def _flush(self) -> None: > def is_alive(self) -> bool: > return self.session.isalive() > > - def _send_command(self, command: str, timeout: float) -> > CommandResult: > - output =3D self._send_command_get_output(command, timeout) > - return_code =3D int(self._send_command_get_output("echo $?", > timeout)) > + def _send_command( > + self, command: str, timeout: float, env: EnvVarsDict | None > + ) -> CommandResult: > + output =3D self._send_command_get_output(command, timeout, env) > + return_code =3D int(self._send_command_get_output("echo $?", > timeout, None)) > > # we're capturing only stdout > return CommandResult(self.name, command, output, "", return_code= ) > > - def _send_command_get_output(self, command: str, timeout: float) -> > str: > + def _send_command_get_output( > + self, command: str, timeout: float, env: EnvVarsDict | None > + ) -> str: > try: > self._clean_session() > + if env: > + command =3D f"{env} {command}" > self._send_line(command) > except Exception as e: > raise e > @@ -189,3 +196,50 @@ def _close(self, force: bool =3D False) -> None: > else: > if self.is_alive(): > self.session.logout() > + > + def copy_file( > + self, source_file: str, destination_file: str, source_remote: > bool =3D False > + ) -> None: > + """ > + Send a local file to a remote host. > + """ > + if source_remote: > + source_file =3D f"{self.username}@{self.ip}:{source_file}" > + else: > + destination_file =3D f"{self.username}@ > {self.ip}:{destination_file}" > + > + port =3D "" > + if self.port: > + port =3D f" -P {self.port}" > + > + # this is not OS agnostic, find a Pythonic (and thus OS agnostic= ) > way > + # TODO Fabric should handle this > + command =3D ( > + f"scp -v{port} -o NoHostAuthenticationForLocalhost=3Dyes" > + f" {source_file} {destination_file}" > + ) > + > + self._spawn_scp(command) > + > + def _spawn_scp(self, scp_cmd: str) -> None: > + """ > + Transfer a file with SCP > + """ > + self.logger.info(scp_cmd) > + p: pexpect.spawn =3D pexpect.spawn(scp_cmd) > + time.sleep(0.5) > + ssh_newkey: str =3D "Are you sure you want to continue connectin= g" > + i: int =3D p.expect( > + [ssh_newkey, "[pP]assword", "# ", pexpect.EOF, > pexpect.TIMEOUT], 120 > + ) > + if i =3D=3D 0: # add once in trust list > + p.sendline("yes") > + i =3D p.expect([ssh_newkey, "[pP]assword", pexpect.EOF], 2) > + > + if i =3D=3D 1: > + time.sleep(0.5) > + p.sendline(self.password) > + p.expect("Exit status 0", 60) > + if i =3D=3D 4: > + self.logger.error("SCP TIMEOUT error %d" % i) > + p.close() > diff --git a/dts/framework/settings.py b/dts/framework/settings.py > index 800f2c7b7f..e2bf3d2ce4 100644 > --- a/dts/framework/settings.py > +++ b/dts/framework/settings.py > @@ -7,6 +7,7 @@ > import os > from collections.abc import Callable, Iterable, Sequence > from dataclasses import dataclass > +from pathlib import Path > from typing import Any, TypeVar > > _T =3D TypeVar("_T") > @@ -60,6 +61,9 @@ class _Settings: > output_dir: str > timeout: float > verbose: bool > + skip_setup: bool > + dpdk_ref: Path > + compile_timeout: float > > > def _get_parser() -> argparse.ArgumentParser: > @@ -88,6 +92,7 @@ def _get_parser() -> argparse.ArgumentParser: > "--timeout", > action=3D_env_arg("DTS_TIMEOUT"), > default=3D15, > + type=3Dfloat, > required=3DFalse, > help=3D"[DTS_TIMEOUT] The default timeout for all DTS operations > except for " > "compiling DPDK.", > @@ -103,6 +108,36 @@ def _get_parser() -> argparse.ArgumentParser: > "to the console.", > ) > > + parser.add_argument( > + "-s", > + "--skip-setup", > + action=3D_env_arg("DTS_SKIP_SETUP"), > + required=3DFalse, > + help=3D"[DTS_SKIP_SETUP] Set to 'Y' to skip all setup steps on S= UT > and TG nodes.", > + ) > + > + parser.add_argument( > + "--dpdk-ref", > + "--git", > + "--snapshot", > + action=3D_env_arg("DTS_DPDK_REF"), > + default=3D"dpdk.tar.xz", > + type=3DPath, > + required=3DFalse, > + help=3D"[DTS_DPDK_REF] Reference to DPDK source code, " > + "can be either a path to a tarball or a git refspec. " > + "In case of a tarball, it will be extracted in the same > directory.", > + ) > + > + parser.add_argument( > + "--compile-timeout", > + action=3D_env_arg("DTS_COMPILE_TIMEOUT"), > + default=3D1200, > + type=3Dfloat, > + required=3DFalse, > + help=3D"[DTS_COMPILE_TIMEOUT] The timeout for compiling DPDK.", > + ) > + > return parser > > > @@ -111,8 +146,11 @@ def _get_settings() -> _Settings: > return _Settings( > config_file_path=3Dparsed_args.config_file, > output_dir=3Dparsed_args.output_dir, > - timeout=3Dfloat(parsed_args.timeout), > + timeout=3Dparsed_args.timeout, > verbose=3D(parsed_args.verbose =3D=3D "Y"), > + skip_setup=3D(parsed_args.skip_setup =3D=3D "Y"), > + dpdk_ref=3Dparsed_args.dpdk_ref, > + compile_timeout=3Dparsed_args.compile_timeout, > ) > > > diff --git a/dts/framework/testbed_model/node/sut_node.py > b/dts/framework/testbed_model/node/sut_node.py > index 79d54585c9..53268a7565 100644 > --- a/dts/framework/testbed_model/node/sut_node.py > +++ b/dts/framework/testbed_model/node/sut_node.py > @@ -2,6 +2,14 @@ > # Copyright(c) 2010-2014 Intel Corporation > # Copyright(c) 2022 PANTHEON.tech s.r.o. > > +import os > +import tarfile > +from pathlib import PurePath > + > +from framework.config import BuildTargetConfiguration, NodeConfiguration > +from framework.settings import SETTINGS > +from framework.utils import EnvVarsDict, skip_setup > + > from .node import Node > > > @@ -10,4 +18,127 @@ class SutNode(Node): > A class for managing connections to the System under Test, providing > methods that retrieve the necessary information about the node (such > as > cpu, memory and NIC details) and configuration capabilities. > + Another key capability is building DPDK according to given build > target. > """ > + > + _build_target_config: BuildTargetConfiguration | None > + _env_vars: EnvVarsDict > + _remote_tmp_dir: PurePath > + __remote_dpdk_dir: PurePath | None > + _app_compile_timeout: float > + > + def __init__(self, node_config: NodeConfiguration): > + super(SutNode, self).__init__(node_config) > + self._build_target_config =3D None > + self._env_vars =3D EnvVarsDict() > + self._remote_tmp_dir =3D self.main_session.get_remote_tmp_dir() > + self.__remote_dpdk_dir =3D None > + self._app_compile_timeout =3D 90 > + > + @property > + def _remote_dpdk_dir(self) -> PurePath: > + if self.__remote_dpdk_dir is None: > + self.__remote_dpdk_dir =3D self._guess_dpdk_remote_dir() > + return self.__remote_dpdk_dir > + > + @_remote_dpdk_dir.setter > + def _remote_dpdk_dir(self, value: PurePath) -> None: > + self.__remote_dpdk_dir =3D value > + > + def _guess_dpdk_remote_dir(self) -> PurePath: > + return > self.main_session.guess_dpdk_remote_dir(self._remote_tmp_dir) > + > + def _setup_build_target( > + self, build_target_config: BuildTargetConfiguration > + ) -> None: > + """ > + Setup DPDK on the SUT node. > + """ > + self._configure_build_target(build_target_config) > + self._copy_dpdk_tarball() > + self._build_dpdk() > + > + def _configure_build_target( > + self, build_target_config: BuildTargetConfiguration > + ) -> None: > + """ > + Populate common environment variables and set build target confi= g. > + """ > + self._build_target_config =3D build_target_config > + self._env_vars.update( > + > self.main_session.get_dpdk_build_env_vars(build_target_config.arch) > + ) > + self._env_vars["CC"] =3D build_target_config.compiler.name > + > + @skip_setup > + def _copy_dpdk_tarball(self) -> None: > + """ > + Copy to and extract DPDK tarball on the SUT node. > + """ > + # check local path > + assert SETTINGS.dpdk_ref.exists(), f"Package {SETTINGS.dpdk_ref} > doesn't exist." > + > + self.logger.info("Copying DPDK tarball to SUT.") > + self.main_session.copy_file(SETTINGS.dpdk_ref, > self._remote_tmp_dir) > + > + # construct remote tarball path > + # the basename is the same on local host and on remote Node > + remote_tarball_path =3D self.main_session.join_remote_path( > + self._remote_tmp_dir, os.path.basename(SETTINGS.dpdk_ref) > + ) > + > + # construct remote path after extracting > + with tarfile.open(SETTINGS.dpdk_ref) as dpdk_tar: > + dpdk_top_dir =3D dpdk_tar.getnames()[0] > + self._remote_dpdk_dir =3D self.main_session.join_remote_path( > + self._remote_tmp_dir, dpdk_top_dir > + ) > + > + self.logger.info("Extracting DPDK tarball on SUT.") > Can we add a path to this log message? > + # clean remote path where we're extracting > + self.main_session.remove_remote_dir(self._remote_dpdk_dir) > + > + # then extract to remote path > + self.main_session.extract_remote_tarball( > + remote_tarball_path, self._remote_dpdk_dir > + ) > + > + @skip_setup > + def _build_dpdk(self) -> None: > + """ > + Build DPDK. Uses the already configured target. Assumes that the > tarball has > + already been copied to and extracted on the SUT node. > + """ > + meson_args =3D "-Denable_kmods=3DTrue -Dlibdir=3Dlib > --default-library=3Dstatic" > + self.main_session.build_dpdk( > + self._env_vars, > + meson_args, > + self._remote_dpdk_dir, > + self._build_target_config.name if self._build_target_config > else "build", > + ) > + self.logger.info( > + f"DPDK version: > {self.main_session.get_dpdk_version(self._remote_dpdk_dir)}" > + ) > + > + def build_dpdk_app(self, app_name: str) -> PurePath: > + """ > + Build one or all DPDK apps. Requires DPDK to be already built on > the SUT node. > + When app_name is 'all', build all example apps. > + When app_name is any other string, tries to build that example > app. > + Return the directory path of the built app. If building all apps= , > return > + the path to the examples directory (where all apps reside). > + """ > + meson_args =3D f"-Dexamples=3D{app_name}" > + build_dir =3D self.main_session.build_dpdk( > + self._env_vars, > + meson_args, > + self._remote_dpdk_dir, > + self._build_target_config.name if self._build_target_config > else "build", > + rebuild=3DTrue, > + timeout=3Dself._app_compile_timeout, > + ) > + if app_name =3D=3D "all": > + return self.main_session.join_remote_path(build_dir, > "examples") > + return self.main_session.join_remote_path( > + build_dir, "examples", f"dpdk-{app_name}" > + ) > diff --git a/dts/framework/utils.py b/dts/framework/utils.py > index c28c8f1082..91e58f3218 100644 > --- a/dts/framework/utils.py > +++ b/dts/framework/utils.py > @@ -4,6 +4,9 @@ > # Copyright(c) 2022 University of New Hampshire > > import sys > +from typing import Callable > + > +from framework.settings import SETTINGS > > > def check_dts_python_version() -> None: > @@ -22,9 +25,21 @@ def check_dts_python_version() -> None: > print(RED("Please use Python >=3D 3.10 instead"), file=3Dsys.std= err) > > > +def skip_setup(func) -> Callable[..., None]: > + if SETTINGS.skip_setup: > + return lambda *args: None > + else: > + return func > + > + > def GREEN(text: str) -> str: > return f"\u001B[32;1m{str(text)}\u001B[0m" > > > def RED(text: str) -> str: > return f"\u001B[31;1m{str(text)}\u001B[0m" > + > + > +class EnvVarsDict(dict): > + def __str__(self) -> str: > + return " ".join(["=3D".join(item) for item in self.items()]) > This needs to make sure it doesn't silently run over the line length limitations in posix sh/bash (4096 chars) or cmd (8191 chars). That would be a VERY frustrating bug to track down and it can easily be stopped by checking that this is a reasonable length (< 2k characters) and emitting a warning if something goes over that. > -- > 2.30.2 > > --000000000000d911c505ed964567 Content-Type: text/html; charset="UTF-8" Content-Transfer-Encoding: quoted-printable
On Mon, Nov 14, 2022 at 11:54 AM Juraj Li= nke=C5=A1 <juraj.linkes@pantheon.tech> wrote:
Add the abil= ity to build DPDK and apps, using a configured target.

Signed-off-by: Juraj Linke=C5=A1 <juraj.linkes@pantheon.tech>
---
=C2=A0dts/framework/exception.py=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 |=C2=A0 17 +++
=C2=A0dts/framework/remote_session/os/os_session.py |=C2=A0 90 +++++++++++-=
=C2=A0.../remote_session/os/posix_session.py=C2=A0 =C2=A0 =C2=A0 =C2=A0 | 1= 28 +++++++++++++++++
=C2=A0.../remote_session/remote_session.py=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 |=C2=A0 34 ++++-
=C2=A0dts/framework/remote_session/ssh_session.py=C2=A0 =C2=A0|=C2=A0 64 ++= ++++++-
=C2=A0dts/framework/settings.py=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 = =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0|=C2=A0 40 +++++-
=C2=A0dts/framework/testbed_model/node/sut_node.py=C2=A0 | 131 ++++++++++++= ++++++
=C2=A0dts/framework/utils.py=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 |=C2=A0 15 ++
=C2=A08 files changed, 505 insertions(+), 14 deletions(-)

diff --git a/dts/framework/exception.py b/dts/framework/exception.py
index b282e48198..93d99432ae 100644
--- a/dts/framework/exception.py
+++ b/dts/framework/exception.py
@@ -26,6 +26,7 @@ class ReturnCode(IntEnum):
=C2=A0 =C2=A0 =C2=A0GENERIC_ERR =3D 1
=C2=A0 =C2=A0 =C2=A0SSH_ERR =3D 2
=C2=A0 =C2=A0 =C2=A0REMOTE_CMD_EXEC_ERR =3D 3
+=C2=A0 =C2=A0 DPDK_BUILD_ERR =3D 10
=C2=A0 =C2=A0 =C2=A0NODE_SETUP_ERR =3D 20
=C2=A0 =C2=A0 =C2=A0NODE_CLEANUP_ERR =3D 21

@@ -110,6 +111,22 @@ def __str__(self) -> str:
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0)


+class RemoteDirectoryExistsError(DTSError):
+=C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 Raised when a remote directory to be created already exists.=
+=C2=A0 =C2=A0 """
+
+=C2=A0 =C2=A0 return_code: ClassVar[ReturnCode] =3D ReturnCode.REMOTE_CMD_= EXEC_ERR
+
+
+class DPDKBuildError(DTSError):
+=C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 Raised when DPDK build fails for any reason.
+=C2=A0 =C2=A0 """
+
+=C2=A0 =C2=A0 return_code: ClassVar[ReturnCode] =3D ReturnCode.DPDK_BUILD_= ERR
+
+
=C2=A0class NodeSetupError(DTSError):
=C2=A0 =C2=A0 =C2=A0"""
=C2=A0 =C2=A0 =C2=A0Raised when setting up a node.
diff --git a/dts/framework/remote_session/os/os_session.py b/dts/framework/= remote_session/os/os_session.py
index 2a72082628..57e2865282 100644
--- a/dts/framework/remote_session/os/os_session.py
+++ b/dts/framework/remote_session/os/os_session.py
@@ -2,12 +2,15 @@
=C2=A0# Copyright(c) 2022 PANTHEON.tech s.r.o.
=C2=A0# Copyright(c) 2022 University of New Hampshire

-from abc import ABC
+from abc import ABC, abstractmethod
+from pathlib import PurePath

-from framework.config import NodeConfiguration
+from framework.config import Architecture, NodeConfiguration
=C2=A0from framework.logger import DTSLOG
=C2=A0from framework.remote_session.factory import create_remote_session =C2=A0from framework.remote_session.remote_session import RemoteSession
+from framework.settings import SETTINGS
+from framework.utils import EnvVarsDict


=C2=A0class OSSession(ABC):
@@ -44,3 +47,86 @@ def is_alive(self) -> bool:
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0Check whether the remote session is still= responding.
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0"""
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0return self.remote_session.is_alive()
+
+=C2=A0 =C2=A0 @abstractmethod
+=C2=A0 =C2=A0 def guess_dpdk_remote_dir(self, remote_dir) -> PurePath:<= br> +=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 Try to find DPDK remote dir in remote_dir.
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+
+=C2=A0 =C2=A0 @abstractmethod
+=C2=A0 =C2=A0 def get_remote_tmp_dir(self) -> PurePath:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 Get the path of the temporary directory of the= remote OS.
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+
+=C2=A0 =C2=A0 @abstractmethod
+=C2=A0 =C2=A0 def get_dpdk_build_env_vars(self, arch: Architecture) -> = dict:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 Create extra environment variables needed for = the target architecture. Get
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 information from the node if needed.
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+
+=C2=A0 =C2=A0 @abstractmethod
+=C2=A0 =C2=A0 def join_remote_path(self, *args: str | PurePath) -> Pure= Path:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 Join path parts using the path separator that = fits the remote OS.
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+
+=C2=A0 =C2=A0 @abstractmethod
+=C2=A0 =C2=A0 def copy_file(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 source_file: str | PurePath,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 destination_file: str | PurePath,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 source_remote: bool =3D False,
+=C2=A0 =C2=A0 ) -> None:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 Copy source_file from local storage to destina= tion_file on the remote Node
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 associated with the remote session.
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 If source_remote is True, reverse the directio= n - copy source_file from the
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 associated remote Node to destination_file on = local storage.
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+
+=C2=A0 =C2=A0 @abstractmethod
+=C2=A0 =C2=A0 def remove_remote_dir(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 remote_dir_path: str | PurePath,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 recursive: bool =3D True,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 force: bool =3D True,
+=C2=A0 =C2=A0 ) -> None:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 Remove remote directory, by default remove rec= ursively and forcefully.
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+
+=C2=A0 =C2=A0 @abstractmethod
+=C2=A0 =C2=A0 def extract_remote_tarball(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 remote_tarball_path: str | PurePath,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 expected_dir: str | PurePath | None =3D None,<= br> +=C2=A0 =C2=A0 ) -> None:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 Extract remote tarball in place. If expected_d= ir is a non-empty string, check
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 whether the dir exists after extracting the ar= chive.
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+
+=C2=A0 =C2=A0 @abstractmethod
+=C2=A0 =C2=A0 def build_dpdk(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 env_vars: EnvVarsDict,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 meson_args: str,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 remote_dpdk_dir: str | PurePath,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 target_name: str,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 rebuild: bool =3D False,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 timeout: float =3D SETTINGS.compile_timeout, +=C2=A0 =C2=A0 ) -> PurePath:

I thin= k that we should consider having a MesonArgs type which implements the buil= der pattern. That way common things like static vs dynamic linking, enablin= g lto, setting the optimization level, et can be handled via dedicated meth= ods, and then we can add a method on that which is "add this string on= to the end". This would also allow defining additional methods for DPD= K-specific meson arguments, like only enabling certain=C2=A0drivers/applica= tions/tests or forcing certain vector widths. I would also like to see an o= ption to make use of ccache, because currently the only way I see to do tha= t is via environment variables, which will make creating a test matrix that= includes multiple compilers difficult.=C2=A0
=C2=A0
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 Build DPDK in the input dir with specified env= ironment variables and meson
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 arguments.
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 Return the directory path where DPDK was built= .
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+
+=C2=A0 =C2=A0 @abstractmethod
+=C2=A0 =C2=A0 def get_dpdk_version(self, version_path: str | PurePath) -&g= t; str:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 Inspect DPDK version on the remote node from v= ersion_path.
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
diff --git a/dts/framework/remote_session/os/posix_session.py b/dts/framewo= rk/remote_session/os/posix_session.py
index 9622a4ea30..a36b8e8c1a 100644
--- a/dts/framework/remote_session/os/posix_session.py
+++ b/dts/framework/remote_session/os/posix_session.py
@@ -2,6 +2,13 @@
=C2=A0# Copyright(c) 2022 PANTHEON.tech s.r.o.
=C2=A0# Copyright(c) 2022 University of New Hampshire

+from pathlib import PurePath, PurePosixPath
+
+from framework.config import Architecture
+from framework.exception import DPDKBuildError, RemoteCommandExecutionErro= r
+from framework.settings import SETTINGS
+from framework.utils import EnvVarsDict
+
=C2=A0from .os_session import OSSession


@@ -10,3 +17,124 @@ class PosixSession(OSSession):
=C2=A0 =C2=A0 =C2=A0An intermediary class implementing the Posix compliant = parts of
=C2=A0 =C2=A0 =C2=A0Linux and other OS remote sessions.
=C2=A0 =C2=A0 =C2=A0"""
+
+=C2=A0 =C2=A0 @staticmethod
+=C2=A0 =C2=A0 def combine_short_options(**opts: [str, bool]) -> str: +=C2=A0 =C2=A0 =C2=A0 =C2=A0 ret_opts =3D ""
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 for opt, include in opts.items():
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 if include:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 ret_opts =3D f&quo= t;{ret_opts}{opt}"
+
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 if ret_opts:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 ret_opts =3D f" -{ret_opts}= "
+
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 return ret_opts
+
+=C2=A0 =C2=A0 def guess_dpdk_remote_dir(self, remote_dir) -> PurePosixP= ath:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 remote_guess =3D self.join_remote_path(remote_= dir, "dpdk-*")
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 result =3D self.remote_session.send_command(f&= quot;ls -d {remote_guess} | tail -1")
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 return PurePosixPath(result.stdout)
+
+=C2=A0 =C2=A0 def get_remote_tmp_dir(self) -> PurePosixPath:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 return PurePosixPath("/tmp")
+
+=C2=A0 =C2=A0 def get_dpdk_build_env_vars(self, arch: Architecture) -> = dict:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 Create extra environment variables needed for = i686 arch build. Get information
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 from the node if needed.
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 env_vars =3D {}
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 if arch =3D=3D Architecture.i686:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 # find the pkg-config path and s= tore it in PKG_CONFIG_LIBDIR
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 out =3D self.remote_session.send= _command("find /usr -type d -name pkgconfig")
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 pkg_path =3D ""
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 res_path =3D out.stdout.split(&q= uot;\r\n")
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 for cur_path in res_path:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 if "i386"= ; in cur_path:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 pkg_= path =3D cur_path
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 brea= k
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 assert pkg_path !=3D ""= ;, "i386 pkg-config path not found"
+
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 env_vars["CFLAGS"] =3D= "-m32"
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 env_vars["PKG_CONFIG_LIBDIR= "] =3D pkg_path
+
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 return env_vars
+
+=C2=A0 =C2=A0 def join_remote_path(self, *args: str | PurePath) -> Pure= PosixPath:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 return PurePosixPath(*args)
+
+=C2=A0 =C2=A0 def copy_file(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 source_file: str | PurePath,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 destination_file: str | PurePath,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 source_remote: bool =3D False,
+=C2=A0 =C2=A0 ) -> None:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self.remote_session.copy_file(source_file, des= tination_file, source_remote)
+
+=C2=A0 =C2=A0 def remove_remote_dir(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 remote_dir_path: str | PurePath,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 recursive: bool =3D True,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 force: bool =3D True,
+=C2=A0 =C2=A0 ) -> None:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 opts =3D PosixSession.combine_short_options(r= =3Drecursive, f=3Dforce)
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self.remote_session.send_command(f"rm{opt= s} {remote_dir_path}")
+
+=C2=A0 =C2=A0 def extract_remote_tarball(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 remote_tarball_path: str | PurePath,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 expected_dir: str | PurePath | None =3D None,<= br> +=C2=A0 =C2=A0 ) -> None:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self.remote_session.send_command(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 f"tar xfm {remote_tarball_p= ath} "
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 f"-C {PurePosixPath(remote_= tarball_path).parent}",
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 60,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 )
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 if expected_dir:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 self.remote_session.send_command= (f"ls {expected_dir}", verify=3DTrue)
+
+=C2=A0 =C2=A0 def build_dpdk(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 env_vars: EnvVarsDict,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 meson_args: str,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 remote_dpdk_dir: str | PurePath,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 target_name: str,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 rebuild: bool =3D False,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 timeout: float =3D SETTINGS.compile_timeout, +=C2=A0 =C2=A0 ) -> PurePosixPath:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 build_dir =3D self.join_remote_path(remote_dpd= k_dir, target_name)
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 try:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 if rebuild:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 # reconfigure, the= n build
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 self.logger.info= ("Reconfiguring DPDK build.")
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 self.remote_sessio= n.send_command(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 f&qu= ot;meson configure {meson_args} {build_dir}",
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 time= out,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 veri= fy=3DTrue,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 env= =3Denv_vars,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 )
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 else:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 # fresh build - re= move target dir first, then build from scratch
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 self.logger.info= ("Configuring DPDK build from scratch.")
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 self.remove_remote= _dir(build_dir)
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 self.remote_sessio= n.send_command(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 f&qu= ot;meson {meson_args} {remote_dpdk_dir} {build_dir}",
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 time= out,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 veri= fy=3DTrue,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 env= =3Denv_vars,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 )
+
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 self.logger.info("Buildin= g DPDK.")
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 self.remote_session.send_command= (
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 f"ninja -C {b= uild_dir}", timeout, verify=3DTrue, env=3Denv_vars
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 )
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 except RemoteCommandExecutionError as e:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 raise DPDKBuildError(f"DPDK= build failed when doing '{e.command}'.")
+
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 return build_dir
+
+=C2=A0 =C2=A0 def get_dpdk_version(self, build_dir: str | PurePath) -> = str:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 out =3D self.remote_session.send_command(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 f"cat {self.join_remote_pat= h(build_dir, 'VERSION')}", verify=3DTrue
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 )
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 return out.stdout
diff --git a/dts/framework/remote_session/remote_session.py b/dts/framework= /remote_session/remote_session.py
index fccd80a529..f10b1023f8 100644
--- a/dts/framework/remote_session/remote_session.py
+++ b/dts/framework/remote_session/remote_session.py
@@ -10,6 +10,7 @@
=C2=A0from framework.exception import RemoteCommandExecutionError
=C2=A0from framework.logger import DTSLOG
=C2=A0from framework.settings import SETTINGS
+from framework.utils import EnvVarsDict


=C2=A0@dataclasses.dataclass(slots=3DTrue, frozen=3DTrue)
@@ -83,15 +84,22 @@ def _connect(self) -> None:
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0"""

=C2=A0 =C2=A0 =C2=A0def send_command(
-=C2=A0 =C2=A0 =C2=A0 =C2=A0 self, command: str, timeout: float =3D SETTING= S.timeout, verify: bool =3D False
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 command: str,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 timeout: float =3D SETTINGS.timeout,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 verify: bool =3D False,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 env: EnvVarsDict | None =3D None,=C2=A0
=C2=A0 =C2=A0 =C2=A0) -> CommandResult:
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0"""
-=C2=A0 =C2=A0 =C2=A0 =C2=A0 Send a command to the connected node and retur= n CommandResult.
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 Send a command to the connected node using opt= ional env vars
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 and return CommandResult.
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0If verify is True, check the return code = of the executed command
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0and raise a RemoteCommandExecutionError i= f the command failed.
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0"""
-=C2=A0 =C2=A0 =C2=A0 =C2=A0 self.logger.info(f"Sending: '{comma= nd}'")
-=C2=A0 =C2=A0 =C2=A0 =C2=A0 result =3D self._send_command(command, timeout= )
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self.logger.info(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 f"Sending: '{command}&#= 39;" + (f" with env vars: '{env}'" if env else "= ;")
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 )
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 result =3D self._send_command(command, timeout= , env)
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0if verify and result.return_code:
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0self.logger.debug(
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0f"Comman= d '{command}' failed with return code '{result.return_code}'= ;"
@@ -104,9 +112,12 @@ def send_command(
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0return result

=C2=A0 =C2=A0 =C2=A0@abstractmethod
-=C2=A0 =C2=A0 def _send_command(self, command: str, timeout: float) -> = CommandResult:
+=C2=A0 =C2=A0 def _send_command(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self, command: str, timeout: float, env: EnvVa= rsDict | None
+=C2=A0 =C2=A0 ) -> CommandResult:
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0"""
-=C2=A0 =C2=A0 =C2=A0 =C2=A0 Use the underlying protocol to execute the com= mand and return CommandResult.
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 Use the underlying protocol to execute the com= mand using optional env vars
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 and return CommandResult.
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0"""

=C2=A0 =C2=A0 =C2=A0def close(self, force: bool =3D False) -> None:
@@ -127,3 +138,14 @@ def is_alive(self) -> bool:
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0"""
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0Check whether the remote session is still= responding.
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0"""
+
+=C2=A0 =C2=A0 @abstractmethod
+=C2=A0 =C2=A0 def copy_file(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self, source_file: str, destination_file: str,= source_remote: bool =3D False
+=C2=A0 =C2=A0 ) -> None:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 Copy source_file from local storage to destina= tion_file on the remote Node

This shoul= d clarify that local storage means inside of the DTS container, not the sys= tem it is running on.=C2=A0
=C2=A0
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 associated with the remote session.
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 If source_remote is True, reverse the directio= n - copy source_file from the
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 associated Node to destination_file on local s= torage.
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
diff --git a/dts/framework/remote_session/ssh_session.py b/dts/framework/re= mote_session/ssh_session.py
index fb2f01dbc1..d4a6714e6b 100644
--- a/dts/framework/remote_session/ssh_session.py
+++ b/dts/framework/remote_session/ssh_session.py
@@ -5,12 +5,13 @@

=C2=A0import time

+import pexpect=C2=A0 # type: ignore
=C2=A0from pexpect import pxssh=C2=A0 # type: ignore

=C2=A0from framework.config import NodeConfiguration
=C2=A0from framework.exception import SSHConnectionError, SSHSessionDeadErr= or, SSHTimeoutError
=C2=A0from framework.logger import DTSLOG
-from framework.utils import GREEN, RED
+from framework.utils import GREEN, RED, EnvVarsDict

=C2=A0from .remote_session import CommandResult, RemoteSession

@@ -163,16 +164,22 @@ def _flush(self) -> None:
=C2=A0 =C2=A0 =C2=A0def is_alive(self) -> bool:
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0return self.session.isalive()

-=C2=A0 =C2=A0 def _send_command(self, command: str, timeout: float) -> = CommandResult:
-=C2=A0 =C2=A0 =C2=A0 =C2=A0 output =3D self._send_command_get_output(comma= nd, timeout)
-=C2=A0 =C2=A0 =C2=A0 =C2=A0 return_code =3D int(self._send_command_get_out= put("echo $?", timeout))
+=C2=A0 =C2=A0 def _send_command(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self, command: str, timeout: float, env: EnvVa= rsDict | None
+=C2=A0 =C2=A0 ) -> CommandResult:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 output =3D self._send_command_get_output(comma= nd, timeout, env)
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 return_code =3D int(self._send_command_get_out= put("echo $?", timeout, None))

=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0# we're capturing only stdout
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0return CommandResult(self.name, command, outpu= t, "", return_code)

-=C2=A0 =C2=A0 def _send_command_get_output(self, command: str, timeout: fl= oat) -> str:
+=C2=A0 =C2=A0 def _send_command_get_output(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self, command: str, timeout: float, env: EnvVa= rsDict | None
+=C2=A0 =C2=A0 ) -> str:
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0try:
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0self._clean_session()
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 if env:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 command =3D f"= ;{env} {command}"
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0self._send_line(command) =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0except Exception as e:
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0raise e
@@ -189,3 +196,50 @@ def _close(self, force: bool =3D False) -> None: =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0else:
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0if self.is_alive():
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0self.session.= logout()
+
+=C2=A0 =C2=A0 def copy_file(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self, source_file: str, destination_file: str,= source_remote: bool =3D False
+=C2=A0 =C2=A0 ) -> None:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 Send a local file to a remote host.
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 if source_remote:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 source_file =3D f"{self.use= rname}@{self.ip}:{source_file}"
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 else:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 destination_file =3D f"{sel= f.username}@{self.ip}:{destination_file}"
+
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 port =3D ""
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 if self.port:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 port =3D f" -P {self.port}&= quot;
+
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 # this is not OS agnostic, find a Pythonic (an= d thus OS agnostic) way
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 # TODO Fabric should handle this
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 command =3D (
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 f"scp -v{port} -o NoHostAut= henticationForLocalhost=3Dyes"
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 f" {source_file} {destinati= on_file}"
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 )
+
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self._spawn_scp(command)
+
+=C2=A0 =C2=A0 def _spawn_scp(self, scp_cmd: str) -> None:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 Transfer a file with SCP
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self.logger.info(scp_cmd)
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 p: pexpect.spawn =3D pexpect.spawn(scp_cmd) +=C2=A0 =C2=A0 =C2=A0 =C2=A0 time.sleep(0.5)
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 ssh_newkey: str =3D "Are you sure you wan= t to continue connecting"
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 i: int =3D p.expect(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 [ssh_newkey, "[pP]assword&q= uot;, "# ", pexpect.EOF, pexpect.TIMEOUT], 120
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 )
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 if i =3D=3D 0:=C2=A0 # add once in trust list<= br> +=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 p.sendline("yes")
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 i =3D p.expect([ssh_newkey, &quo= t;[pP]assword", pexpect.EOF], 2)
+
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 if i =3D=3D 1:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 time.sleep(0.5)
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 p.sendline(self.password)
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 p.expect("Exit status 0&quo= t;, 60)
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 if i =3D=3D 4:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 self.logger.error("SCP TIME= OUT error %d" % i)
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 p.close()
diff --git a/dts/framework/settings.py b/dts/framework/settings.py
index 800f2c7b7f..e2bf3d2ce4 100644
--- a/dts/framework/settings.py
+++ b/dts/framework/settings.py
@@ -7,6 +7,7 @@
=C2=A0import os
=C2=A0from collections.abc import Callable, Iterable, Sequence
=C2=A0from dataclasses import dataclass
+from pathlib import Path
=C2=A0from typing import Any, TypeVar

=C2=A0_T =3D TypeVar("_T")
@@ -60,6 +61,9 @@ class _Settings:
=C2=A0 =C2=A0 =C2=A0output_dir: str
=C2=A0 =C2=A0 =C2=A0timeout: float
=C2=A0 =C2=A0 =C2=A0verbose: bool
+=C2=A0 =C2=A0 skip_setup: bool
+=C2=A0 =C2=A0 dpdk_ref: Path
+=C2=A0 =C2=A0 compile_timeout: float


=C2=A0def _get_parser() -> argparse.ArgumentParser:
@@ -88,6 +92,7 @@ def _get_parser() -> argparse.ArgumentParser:
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0"--timeout",
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0action=3D_env_arg("DTS_TIMEOUT"= ),
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0default=3D15,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 type=3Dfloat,
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0required=3DFalse,
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0help=3D"[DTS_TIMEOUT] The default ti= meout for all DTS operations except for "
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0"compiling DPDK.",
@@ -103,6 +108,36 @@ def _get_parser() -> argparse.ArgumentParser:
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0"to the console.",
=C2=A0 =C2=A0 =C2=A0)

+=C2=A0 =C2=A0 parser.add_argument(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 "-s",
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 "--skip-setup",
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 action=3D_env_arg("DTS_SKIP_SETUP"),=
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 required=3DFalse,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 help=3D"[DTS_SKIP_SETUP] Set to 'Y= 9; to skip all setup steps on SUT and TG nodes.",
+=C2=A0 =C2=A0 )
+
+=C2=A0 =C2=A0 parser.add_argument(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 "--dpdk-ref",
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 "--git",
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 "--snapshot",
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 action=3D_env_arg("DTS_DPDK_REF"), +=C2=A0 =C2=A0 =C2=A0 =C2=A0 default=3D"dpdk.tar.xz",
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 type=3DPath,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 required=3DFalse,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 help=3D"[DTS_DPDK_REF] Reference to DPDK = source code, "
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 "can be either a path to a tarball or a g= it refspec. "
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 "In case of a tarball, it will be extract= ed in the same directory.",
+=C2=A0 =C2=A0 )
+
+=C2=A0 =C2=A0 parser.add_argument(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 "--compile-timeout",
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 action=3D_env_arg("DTS_COMPILE_TIMEOUT&qu= ot;),
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 default=3D1200,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 type=3Dfloat,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 required=3DFalse,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 help=3D"[DTS_COMPILE_TIMEOUT] The timeout= for compiling DPDK.",
+=C2=A0 =C2=A0 )
+
=C2=A0 =C2=A0 =C2=A0return parser


@@ -111,8 +146,11 @@ def _get_settings() -> _Settings:
=C2=A0 =C2=A0 =C2=A0return _Settings(
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0config_file_path=3Dparsed_args.config_fil= e,
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0output_dir=3Dparsed_args.output_dir,
-=C2=A0 =C2=A0 =C2=A0 =C2=A0 timeout=3Dfloat(parsed_args.timeout),
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 timeout=3Dparsed_args.timeout,
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0verbose=3D(parsed_args.verbose =3D=3D &qu= ot;Y"),
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 skip_setup=3D(parsed_args.skip_setup =3D=3D &q= uot;Y"),
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 dpdk_ref=3Dparsed_args.dpdk_ref,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 compile_timeout=3Dparsed_args.compile_timeout,=
=C2=A0 =C2=A0 =C2=A0)


diff --git a/dts/framework/testbed_model/node/sut_node.py b/dts/framework/t= estbed_model/node/sut_node.py
index 79d54585c9..53268a7565 100644
--- a/dts/framework/testbed_model/node/sut_node.py
+++ b/dts/framework/testbed_model/node/sut_node.py
@@ -2,6 +2,14 @@
=C2=A0# Copyright(c) 2010-2014 Intel Corporation
=C2=A0# Copyright(c) 2022 PANTHEON.tech s.r.o.

+import os
+import tarfile
+from pathlib import PurePath
+
+from framework.config import BuildTargetConfiguration, NodeConfiguration +from framework.settings import SETTINGS
+from framework.utils import EnvVarsDict, skip_setup
+
=C2=A0from .node import Node


@@ -10,4 +18,127 @@ class SutNode(Node):
=C2=A0 =C2=A0 =C2=A0A class for managing connections to the System under Te= st, providing
=C2=A0 =C2=A0 =C2=A0methods that retrieve the necessary information about t= he node (such as
=C2=A0 =C2=A0 =C2=A0cpu, memory and NIC details) and configuration capabili= ties.
+=C2=A0 =C2=A0 Another key capability is building DPDK according to given b= uild target.
=C2=A0 =C2=A0 =C2=A0"""
+
+=C2=A0 =C2=A0 _build_target_config: BuildTargetConfiguration | None
+=C2=A0 =C2=A0 _env_vars: EnvVarsDict
+=C2=A0 =C2=A0 _remote_tmp_dir: PurePath
+=C2=A0 =C2=A0 __remote_dpdk_dir: PurePath | None
+=C2=A0 =C2=A0 _app_compile_timeout: float
+
+=C2=A0 =C2=A0 def __init__(self, node_config: NodeConfiguration):
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 super(SutNode, self).__init__(node_config)
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self._build_target_config =3D None
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self._env_vars =3D EnvVarsDict()
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self._remote_tmp_dir =3D self.main_session.get= _remote_tmp_dir()
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self.__remote_dpdk_dir =3D None
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self._app_compile_timeout =3D 90
+
+=C2=A0 =C2=A0 @property
+=C2=A0 =C2=A0 def _remote_dpdk_dir(self) -> PurePath:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 if self.__remote_dpdk_dir is None:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 self.__remote_dpdk_dir =3D self.= _guess_dpdk_remote_dir()
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 return self.__remote_dpdk_dir
+
+=C2=A0 =C2=A0 @_remote_dpdk_dir.setter
+=C2=A0 =C2=A0 def _remote_dpdk_dir(self, value: PurePath) -> None:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self.__remote_dpdk_dir =3D value
+
+=C2=A0 =C2=A0 def _guess_dpdk_remote_dir(self) -> PurePath:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 return self.main_session.guess_dpdk_remote_dir= (self._remote_tmp_dir)
+
+=C2=A0 =C2=A0 def _setup_build_target(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self, build_target_config: BuildTargetConfigur= ation
+=C2=A0 =C2=A0 ) -> None:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 Setup DPDK on the SUT node.
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self._configure_build_target(build_target_conf= ig)
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self._copy_dpdk_tarball()
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self._build_dpdk()
+
+=C2=A0 =C2=A0 def _configure_build_target(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self, build_target_config: BuildTargetConfigur= ation
+=C2=A0 =C2=A0 ) -> None:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 Populate common environment variables and set = build target config.
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self._build_target_config =3D build_target_con= fig
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self._env_vars.update(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 self.main_session.get_dpdk_build= _env_vars(build_target_config.arch)
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 )
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self._env_vars["CC"] =3D build_target_config.compiler.name
+
+=C2=A0 =C2=A0 @skip_setup
+=C2=A0 =C2=A0 def _copy_dpdk_tarball(self) -> None:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 Copy to and extract DPDK tarball on the SUT no= de.
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 # check local path
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 assert SETTINGS.dpdk_ref.exists(), f"Pack= age {SETTINGS.dpdk_ref} doesn't exist."
+
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self.logger.info("Copying DPDK tarball = to SUT.")
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self.main_session.copy_file(SETTINGS.dpdk_ref,= self._remote_tmp_dir)
+
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 # construct remote tarball path
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 # the basename is the same on local host and o= n remote Node
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 remote_tarball_path =3D self.main_session.join= _remote_path(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 self._remote_tmp_dir, os.path.ba= sename(SETTINGS.dpdk_ref)
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 )
+
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 # construct remote path after extracting
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 with tarfile.open(SETTINGS.dpdk_ref) as dpdk_t= ar:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 dpdk_top_dir =3D dpdk_tar.getnam= es()[0]
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self._remote_dpdk_dir =3D self.main_session.jo= in_remote_path(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 self._remote_tmp_dir, dpdk_top_d= ir
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 )
+
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self.logger.info("Extracting DPDK tarba= ll on SUT.")

Can we add a path to = this log message?
=C2=A0
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 # clean remote path where we're extracting=
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self.main_session.remove_remote_dir(self._remo= te_dpdk_dir)
+
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 # then extract to remote path
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self.main_session.extract_remote_tarball(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 remote_tarball_path, self._remot= e_dpdk_dir
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 )
+
+=C2=A0 =C2=A0 @skip_setup
+=C2=A0 =C2=A0 def _build_dpdk(self) -> None:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 Build DPDK. Uses the already configured target= . Assumes that the tarball has
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 already been copied to and extracted on the SU= T node.
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 meson_args =3D "-Denable_kmods=3DTrue -Dl= ibdir=3Dlib --default-library=3Dstatic"
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self.main_session.build_dpdk(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 self._env_vars,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 meson_args,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 self._remote_dpdk_dir,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 self._build_target_config.= name if self._build_target_config else "build",
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 )
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 self.logger.info(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 f"DPDK version: {self.main_= session.get_dpdk_version(self._remote_dpdk_dir)}"
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 )
+
+=C2=A0 =C2=A0 def build_dpdk_app(self, app_name: str) -> PurePath:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 Build one or all DPDK apps. Requires DPDK to b= e already built on the SUT node.
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 When app_name is 'all', build all exam= ple apps.
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 When app_name is any other string, tries to bu= ild that example app.
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 Return the directory path of the built app. If= building all apps, return
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 the path to the examples directory (where all = apps reside).
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 """
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 meson_args =3D f"-Dexamples=3D{app_name}&= quot;
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 build_dir =3D self.main_session.build_dpdk( +=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 self._env_vars,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 meson_args,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 self._remote_dpdk_dir,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 self._build_target_config.= name if self._build_target_config else "build",
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 rebuild=3DTrue,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 timeout=3Dself._app_compile_time= out,
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 )
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 if app_name =3D=3D "all":
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 return self.main_session.join_re= mote_path(build_dir, "examples")
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 return self.main_session.join_remote_path(
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 build_dir, "examples",= f"dpdk-{app_name}"
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 )
diff --git a/dts/framework/utils.py b/dts/framework/utils.py
index c28c8f1082..91e58f3218 100644
--- a/dts/framework/utils.py
+++ b/dts/framework/utils.py
@@ -4,6 +4,9 @@
=C2=A0# Copyright(c) 2022 University of New Hampshire

=C2=A0import sys
+from typing import Callable
+
+from framework.settings import SETTINGS


=C2=A0def check_dts_python_version() -> None:
@@ -22,9 +25,21 @@ def check_dts_python_version() -> None:
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0print(RED("Please use Python >=3D= 3.10 instead"), file=3Dsys.stderr)


+def skip_setup(func) -> Callable[..., None]:
+=C2=A0 =C2=A0 if SETTINGS.skip_setup:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 return lambda *args: None
+=C2=A0 =C2=A0 else:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 return func
+
+
=C2=A0def GREEN(text: str) -> str:
=C2=A0 =C2=A0 =C2=A0return f"\u001B[32;1m{str(text)}\u001B[0m"

=C2=A0def RED(text: str) -> str:
=C2=A0 =C2=A0 =C2=A0return f"\u001B[31;1m{str(text)}\u001B[0m" +
+
+class EnvVarsDict(dict):
+=C2=A0 =C2=A0 def __str__(self) -> str:
+=C2=A0 =C2=A0 =C2=A0 =C2=A0 return " ".join(["=3D".joi= n(item) for item in self.items()])

This= needs to make sure it doesn't silently run over the line length limita= tions in posix sh/bash (4096 chars) or cmd (8191 chars). That would be a VE= RY frustrating bug to track down and it can easily be stopped by checking t= hat this is a reasonable length (< 2k characters) and emitting a warning= if something goes over that.=C2=A0
=C2=A0
--
2.30.2

--000000000000d911c505ed964567--