From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 86C3445A6E; Mon, 30 Sep 2024 18:18:52 +0200 (CEST) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 952A14065B; Mon, 30 Sep 2024 18:18:30 +0200 (CEST) Received: from mail-ed1-f50.google.com (mail-ed1-f50.google.com [209.85.208.50]) by mails.dpdk.org (Postfix) with ESMTP id 6237540430 for ; Mon, 30 Sep 2024 18:18:23 +0200 (CEST) Received: by mail-ed1-f50.google.com with SMTP id 4fb4d7f45d1cf-5c5b9d2195eso6088142a12.1 for ; Mon, 30 Sep 2024 09:18:23 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=pantheon.tech; s=google; t=1727713102; x=1728317902; darn=dpdk.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=C19dvdu4C0ej0Oj5KUkXmBbJHMEm4bzXD4uzwUFh5QA=; b=ue/SVbH4zkHtYXpKcQxhyTMd9OZgJM0XEJFuFSiz22xw2Xse224BPprKGjOR8HSRaB 2m25+cCByZIwzd6QIEcEOdw99lH1a95CFpuvx7eGmHW0s1bRBSOyD8E4tls1duX016JB cdHqD5A7VOva4C372mZxjqOZsCM1ySf9HPcidbdgcBAD+6Po/Rb/eLveo6FhS8grrNys Z75eIDhFhcACa7JbyoVEqr2Mxv8+40ZYYamfFj58OyHcq/X3lUhXiiSdmh9Za19+GoDp dOzLsC/z4am2eANajcgweL7Dn1ZMEgVQcoAM366+93Tw6QhTp781Vq/urwZo52bKoetq YkYw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1727713102; x=1728317902; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=C19dvdu4C0ej0Oj5KUkXmBbJHMEm4bzXD4uzwUFh5QA=; b=RnX+PayOz4/FQzW80RyY9d3HaKciDa8e/omthElBmzHy63peHRRtrPH4CqpqtWXWSQ WRtcjgbhTA1+e+yF/DsVlRZCpjO4G3pZSEL5d2uH/opKg0q8vafPZch4OWI+YVpeEo2Q J1a6NBAUo5IJ22XuPq2h/7Tt4DT8YoIFGPk7zQRFCamXh1urUxeM72zJkBZtqUM2Ukze IGTERiZFD1o4Y2MVaUvht1psc3PujDtvigxo3WPKpNsTZFDd9X+XO6ChRoDKRX4LDedb 8z9Py0Xvc1mf0KMrmvFV9zmm8jmXzg7o8uAGh6X+sNg+mMOsCfsWfWdCUhPHqeSdQ++D jf0g== X-Gm-Message-State: AOJu0YwWl1MAFy42wKNFCm2I1FqU6o6zjIEZyrppjSlFpGtmB8nrtNsm bUQDJYCwFm8DrUQfEJu6dpHGx6txvASpy6jFUOfxrEg7qguE8x6DJX1P9kINAiLcFS2X9F8bwqJ 8KKY= X-Google-Smtp-Source: AGHT+IG7dYR/fKPBFkyeKegqKnLlR5B0Fbo3V7m+snmvX/J1F/Zlr7fn7vcOFxFi8vCqpmJUTfe+wg== X-Received: by 2002:a17:907:7b95:b0:a86:8285:24a0 with SMTP id a640c23a62f3a-a93c4909a29mr1199785366b.23.1727713102449; Mon, 30 Sep 2024 09:18:22 -0700 (PDT) Received: from fedora.. (ip-46.34.236.94.o2inet.sk. [46.34.236.94]) by smtp.gmail.com with ESMTPSA id a640c23a62f3a-a93c2775d0bsm564923666b.11.2024.09.30.09.18.21 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Mon, 30 Sep 2024 09:18:21 -0700 (PDT) From: =?UTF-8?q?Tom=C3=A1=C5=A1=20=C4=8Eurovec?= To: dev@dpdk.org Cc: =?UTF-8?q?Tom=C3=A1=C5=A1=20=C4=8Eurovec?= Subject: [PATCH 4/7] dts: add the ability to copy directories via remote Date: Mon, 30 Sep 2024 18:18:11 +0200 Message-ID: <20240930161814.26070-5-tomas.durovec@pantheon.tech> X-Mailer: git-send-email 2.46.1 In-Reply-To: <20240930161814.26070-1-tomas.durovec@pantheon.tech> References: <20240930161814.26070-1-tomas.durovec@pantheon.tech> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Before, the remote session did't allows to copy directories, only files. This feature will be used in future commit. Signed-off-by: Tomáš Ďurovec --- dts/framework/testbed_model/os_session.py | 120 ++++++++++++++++++- dts/framework/testbed_model/posix_session.py | 88 +++++++++++++- dts/framework/utils.py | 91 +++++++++++++- 3 files changed, 287 insertions(+), 12 deletions(-) diff --git a/dts/framework/testbed_model/os_session.py b/dts/framework/testbed_model/os_session.py index 1aac3659bf..6c3f84dec1 100644 --- a/dts/framework/testbed_model/os_session.py +++ b/dts/framework/testbed_model/os_session.py @@ -25,7 +25,7 @@ from abc import ABC, abstractmethod from collections.abc import Iterable from ipaddress import IPv4Interface, IPv6Interface -from pathlib import Path, PurePath +from pathlib import Path, PurePath, PurePosixPath from typing import Union from framework.config import Architecture, NodeConfiguration, NodeInfo @@ -38,7 +38,7 @@ ) from framework.remote_session.remote_session import CommandResult from framework.settings import SETTINGS -from framework.utils import MesonArgs +from framework.utils import MesonArgs, TarCompressionFormat from .cpu import LogicalCore from .port import Port @@ -203,6 +203,95 @@ def copy_to(self, source_file: str | Path, destination_dir: str | PurePath) -> N will be saved. """ + @abstractmethod + def copy_dir_from( + self, + source_dir: str | PurePath, + destination_dir: str | Path, + compress_format: TarCompressionFormat = TarCompressionFormat.none, + exclude: str | list[str] | None = None, + ) -> None: + """Copy a directory from the remote node to the local filesystem. + + Copy `source_dir` from the remote node associated with this remote session to + `destination_dir` on the local filesystem. The new local directory will be created + at `destination_dir` path. + + Example: + source_dir = '/remote/path/to/source' + destination_dir = '/local/path/to/destination' + compress_format = TarCompressionFormat.xz + + The method will: + 1. Create a tarball from `source_dir`, resulting in: + '/remote/path/to/source.tar.xz', + 2. Copy '/remote/path/to/source.tar.xz' to + '/local/path/to/destination/source.tar.xz', + 3. Extract the contents of the tarball, resulting in: + '/local/path/to/destination/source/', + 4. Remove the tarball after extraction + ('/local/path/to/destination/source.tar.xz'). + + Final Path Structure: + '/local/path/to/destination/source/' + + Args: + source_dir: The directory on the remote node. + destination_dir: The directory path on the local filesystem. + compress_format: The compression format to use. Defaults to no compression. + exclude: Patterns for files or directories to exclude from the tarball. + These patterns are used with `tar`'s `--exclude` option. + """ + + @abstractmethod + def copy_dir_to( + self, + source_dir: str | Path, + destination_dir: str | PurePath, + compress_format: TarCompressionFormat = TarCompressionFormat.none, + exclude: str | list[str] | None = None, + ) -> None: + """Copy a directory from the local filesystem to the remote node. + + Copy `source_dir` from the local filesystem to `destination_dir` on the remote node + associated with this remote session. The new remote directory will be created at + `destination_dir` path. + + Example: + source_dir = '/local/path/to/source' + destination_dir = '/remote/path/to/destination' + compress_format = TarCompressionFormat.xz + + The method will: + 1. Create a tarball from `source_dir`, resulting in: + '/local/path/to/source.tar.xz', + 2. Copy '/local/path/to/source.tar.xz' to + '/remote/path/to/destination/source.tar.xz', + 3. Extract the contents of the tarball, resulting in: + '/remote/path/to/destination/source/', + 4. Remove the tarball after extraction + ('/remote/path/to/destination/source.tar.xz'). + + Final Path Structure: + '/remote/path/to/destination/source/' + + Args: + source_dir: The directory on the local filesystem. + destination_dir: The directory path on the remote node. + compress_format: The compression format to use. Defaults to no compression. + exclude: Patterns for files or directories to exclude from the tarball. + These patterns are used with `fnmatch.fnmatch` to filter out files. + """ + + @abstractmethod + def remove_remote_file(self, remote_file_path: str | PurePath, force: bool = True) -> None: + """Remove remote file, by default remove forcefully. + + Args: + remote_file_path: The file path to remove. + force: If :data:`True`, ignore all warnings and try to remove at all costs. + """ + @abstractmethod def remove_remote_dir( self, @@ -213,11 +302,34 @@ def remove_remote_dir( """Remove remote directory, by default remove recursively and forcefully. Args: - remote_dir_path: The path of the directory to remove. + remote_dir_path: The directory path to remove. recursive: If :data:`True`, also remove all contents inside the directory. force: If :data:`True`, ignore all warnings and try to remove at all costs. """ + @abstractmethod + def create_remote_tarball( + self, + remote_dir_path: str | PurePath, + compress_format: TarCompressionFormat = TarCompressionFormat.none, + exclude: str | list[str] | None = None, + ) -> PurePosixPath: + """Create a tarball from the contents of the specified remote directory. + + This method creates a tarball containing all files and directories + within `remote_dir_path`. The tarball will be saved in the directory of + `remote_dir_path` and will be named based on `remote_dir_path`. + + Args: + remote_dir_path: The directory path on the remote node. + compress_format: The compression format to use. Defaults to no compression. + exclude: Patterns for files or directories to exclude from the tarball. + These patterns are used with `tar`'s `--exclude` option. + + Returns: + The path to the created tarball on the remote node. + """ + @abstractmethod def extract_remote_tarball( self, @@ -227,7 +339,7 @@ def extract_remote_tarball( """Extract remote tarball in its remote directory. Args: - remote_tarball_path: The path of the tarball on the remote node. + remote_tarball_path: The tarball path on the remote node. expected_dir: If non-empty, check whether `expected_dir` exists after extracting the archive. """ diff --git a/dts/framework/testbed_model/posix_session.py b/dts/framework/testbed_model/posix_session.py index 2449c0ab35..94e721da61 100644 --- a/dts/framework/testbed_model/posix_session.py +++ b/dts/framework/testbed_model/posix_session.py @@ -18,7 +18,13 @@ from framework.config import Architecture, NodeInfo from framework.exception import DPDKBuildError, RemoteCommandExecutionError from framework.settings import SETTINGS -from framework.utils import MesonArgs +from framework.utils import ( + MesonArgs, + TarCompressionFormat, + convert_to_list_of_string, + create_tarball, + extract_tarball, +) from .os_session import OSSession @@ -93,6 +99,48 @@ def copy_to(self, source_file: str | Path, destination_dir: str | PurePath) -> N """Overrides :meth:`~.os_session.OSSession.copy_to`.""" self.remote_session.copy_to(source_file, destination_dir) + def copy_dir_from( + self, + source_dir: str | PurePath, + destination_dir: str | Path, + compress_format: TarCompressionFormat = TarCompressionFormat.none, + exclude: str | list[str] | None = None, + ) -> None: + """Overrides :meth:`~.os_session.OSSession.copy_dir_from`.""" + source_dir = PurePath(source_dir) + remote_tarball_path = self.create_remote_tarball(source_dir, compress_format, exclude) + + self.copy_from(remote_tarball_path, destination_dir) + self.remove_remote_file(remote_tarball_path) + + tarball_path = Path(destination_dir, f"{source_dir.name}.{compress_format.extension}") + extract_tarball(tarball_path) + tarball_path.unlink() + + def copy_dir_to( + self, + source_dir: str | Path, + destination_dir: str | PurePath, + compress_format: TarCompressionFormat = TarCompressionFormat.none, + exclude: str | list[str] | None = None, + ) -> None: + """Overrides :meth:`~.os_session.OSSession.copy_dir_to`.""" + source_dir = Path(source_dir) + tarball_path = create_tarball(source_dir, compress_format, exclude=exclude) + self.copy_to(tarball_path, destination_dir) + tarball_path.unlink() + + remote_tar_path = self.join_remote_path( + destination_dir, f"{source_dir.name}.{compress_format.extension}" + ) + self.extract_remote_tarball(remote_tar_path) + self.remove_remote_file(remote_tar_path) + + def remove_remote_file(self, remote_file_path: str | PurePath, force: bool = True) -> None: + """Overrides :meth:`~.os_session.OSSession.remove_remote_dir`.""" + opts = PosixSession.combine_short_options(f=force) + self.send_command(f"rm{opts} {remote_file_path}") + def remove_remote_dir( self, remote_dir_path: str | PurePath, @@ -103,10 +151,42 @@ def remove_remote_dir( opts = PosixSession.combine_short_options(r=recursive, f=force) self.send_command(f"rm{opts} {remote_dir_path}") - def extract_remote_tarball( + def create_remote_tarball( self, - remote_tarball_path: str | PurePath, - expected_dir: str | PurePath | None = None, + remote_dir_path: str | PurePath, + compress_format: TarCompressionFormat = TarCompressionFormat.none, + exclude: str | list[str] | None = None, + ) -> PurePosixPath: + """Overrides :meth:`~.os_session.OSSession.create_remote_tarball`.""" + + def generate_tar_exclude_args(exclude_patterns) -> str: + """Generate args to exclude patterns when creating a tarball. + + Args: + exclude_patterns: Patterns for files or directories to exclude from the tarball. + These patterns are used with `tar`'s `--exclude` option. + + Returns: + The generated string args to exclude the specified patterns. + """ + if exclude_patterns: + exclude_patterns = convert_to_list_of_string(exclude_patterns) + return "".join([f" --exclude={pattern}" for pattern in exclude_patterns]) + return "" + + posix_remote_dir_path = PurePosixPath(remote_dir_path) + target_tarball_path = PurePosixPath(f"{remote_dir_path}.{compress_format.extension}") + + self.send_command( + f"tar caf {target_tarball_path}{generate_tar_exclude_args(exclude)} " + f"-C {posix_remote_dir_path.parent} {posix_remote_dir_path.name}", + 60, + ) + + return target_tarball_path + + def extract_remote_tarball( + self, remote_tarball_path: str | PurePath, expected_dir: str | PurePath | None = None ) -> None: """Overrides :meth:`~.os_session.OSSession.extract_remote_tarball`.""" self.send_command( diff --git a/dts/framework/utils.py b/dts/framework/utils.py index c768dd0c99..382357ffe8 100644 --- a/dts/framework/utils.py +++ b/dts/framework/utils.py @@ -15,13 +15,16 @@ """ import atexit +import fnmatch import json import os import random import subprocess +import tarfile from enum import Enum, Flag from pathlib import Path from subprocess import SubprocessError +from typing import Any, Callable from scapy.layers.inet import IP, TCP, UDP, Ether # type: ignore[import-untyped] from scapy.packet import Packet # type: ignore[import-untyped] @@ -142,13 +145,17 @@ def __str__(self) -> str: return " ".join(f"{self._default_library} {self._dpdk_args}".split()) -class _TarCompressionFormat(StrEnum): +class TarCompressionFormat(StrEnum): """Compression formats that tar can use. Enum names are the shell compression commands and Enum values are the associated file extensions. + + The 'none' member represents no compression, only archiving with tar. + Its value is set to 'tar' to indicate that the file is an uncompressed tar archive. """ + none = "tar" gzip = "gz" compress = "Z" bzip2 = "bz2" @@ -158,6 +165,16 @@ class _TarCompressionFormat(StrEnum): xz = "xz" zstd = "zst" + @property + def extension(self): + """Return the extension associated with the compression format. + + If the compression format is 'none', the extension will be in the format 'tar'. + For other compression formats, the extension will be in the format + 'tar.{compression format}'. + """ + return f"{self.value}" if self == self.none else f"{self.none.value}.{self.value}" + class DPDKGitTarball: """Compressed tarball of DPDK from the repository. @@ -171,7 +188,7 @@ class DPDKGitTarball: """ _git_ref: str - _tar_compression_format: _TarCompressionFormat + _tar_compression_format: TarCompressionFormat _tarball_dir: Path _tarball_name: str _tarball_path: Path | None @@ -180,7 +197,7 @@ def __init__( self, git_ref: str, output_dir: str, - tar_compression_format: _TarCompressionFormat = _TarCompressionFormat.xz, + tar_compression_format: TarCompressionFormat = TarCompressionFormat.xz, ): """Create the tarball during initialization. @@ -201,7 +218,7 @@ def __init__( self._create_tarball_dir() self._tarball_name = ( - f"dpdk-tarball-{self._git_ref}.tar.{self._tar_compression_format.value}" + f"dpdk-tarball-{self._git_ref}.{self._tar_compression_format.extension}" ) self._tarball_path = self._check_tarball_path() if not self._tarball_path: @@ -248,6 +265,72 @@ def __fspath__(self) -> str: return str(self._tarball_path) +def convert_to_list_of_string(value: Any | list[Any]) -> list[str]: + """Convert the input to the list of strings.""" + return list(map(str, value) if isinstance(value, list) else str(value)) + + +def create_tarball( + dir_path: str | Path, + compress_format: TarCompressionFormat = TarCompressionFormat.none, + exclude: Any | list[Any] | None = None, +) -> Path: + """Create a tarball from the contents of the specified directory. + + This method creates a tarball containing all files and directories within `dir_path`. + The tarball will be saved in the directory of `dir_path` and will be named based on `dir_path`. + + Args: + dir_path: The directory path. + compress_format: The compression format to use. Defaults to no compression. + exclude: Patterns for files or directories to exclude from the tarball. + These patterns are used with `fnmatch.fnmatch` to filter out files. + + Returns: + The path to the created tarball. + """ + + def create_filter_function(exclude_patterns: str | list[str] | None) -> Callable | None: + """Create a filter function based on the provided exclude patterns. + + Args: + exclude_patterns: Patterns for files or directories to exclude from the tarball. + These patterns are used with `fnmatch.fnmatch` to filter out files. + + Returns: + The filter function that excludes files based on the patterns. + """ + if exclude_patterns: + exclude_patterns = convert_to_list_of_string(exclude_patterns) + + def filter_func(tarinfo: tarfile.TarInfo) -> tarfile.TarInfo | None: + file_name = os.path.basename(tarinfo.name) + if any(fnmatch.fnmatch(file_name, pattern) for pattern in exclude_patterns): + return None + return tarinfo + + return filter_func + return None + + target_tarball_path = Path(f"{dir_path}.{compress_format.extension}") + with tarfile.open(target_tarball_path, f"w:{compress_format.value}") as tar: + tar.add(dir_path, arcname=target_tarball_path.stem, filter=create_filter_function(exclude)) + + return target_tarball_path + + +def extract_tarball(tar_path: str | Path): + """Extract the contents of a tarball. + + The tarball will be extracted in the same path as `tar_path` parent path. + + Args: + tar_path: The path to the tarball file to extract. + """ + with tarfile.open(tar_path, "r") as tar: + tar.extractall(path=Path(tar_path).parent) + + class PacketProtocols(Flag): """Flag specifying which protocols to use for packet generation.""" -- 2.46.1