[RFC PATCH v1 05/12] dts: add the ability to copy directories via remote

["Juraj Linkeš" <juraj.linkes@pantheon.tech>]

From: Tomáš Ďurovec <tomas.durovec@pantheon.tech>

Signed-off-by: Tomáš Ďurovec <tomas.durovec@pantheon.tech>
---
 dts/framework/testbed_model/os_session.py    | 88 +++++++++++++++---
 dts/framework/testbed_model/posix_session.py | 93 ++++++++++++++++---
 dts/framework/utils.py                       | 97 ++++++++++++++++++--
 3 files changed, 246 insertions(+), 32 deletions(-)

diff --git a/dts/framework/testbed_model/os_session.py b/dts/framework/testbed_model/os_session.py
index d24f44df10..92b1a09d94 100644
--- a/dts/framework/testbed_model/os_session.py
+++ b/dts/framework/testbed_model/os_session.py
@@ -38,7 +38,7 @@
 )
 from framework.remote_session.remote_session import CommandResult
 from framework.settings import SETTINGS
-from framework.utils import MesonArgs
+from framework.utils import MesonArgs, TarCompressionFormat
 
 from .cpu import LogicalCore
 from .port import Port
@@ -178,11 +178,7 @@ def join_remote_path(self, *args: str | PurePath) -> PurePath:
         """
 
     @abstractmethod
-    def copy_from(
-        self,
-        source_file: str | PurePath,
-        destination_dir: str | Path,
-    ) -> None:
+    def copy_from(self, source_file: str | PurePath, destination_dir: str | Path) -> None:
         """Copy a file from the remote node to the local filesystem.
 
         Copy `source_file` from the remote node associated with this remote
@@ -195,11 +191,7 @@ def copy_from(
         """
 
     @abstractmethod
-    def copy_to(
-        self,
-        source_file: str | Path,
-        destination_dir: str | PurePath,
-    ) -> None:
+    def copy_to(self, source_file: str | Path, destination_dir: str | PurePath) -> None:
         """Copy a file from local filesystem to the remote node.
 
         Copy `source_file` from local filesystem to `destination_dir`
@@ -211,6 +203,57 @@ def copy_to(
                 will be saved.
         """
 
+    @abstractmethod
+    def copy_dir_from(
+        self,
+        source_dir: str | PurePath,
+        destination_dir: str | Path,
+        compress_format: TarCompressionFormat = TarCompressionFormat.none,
+        exclude: str | list[str] | None = None,
+    ) -> None:
+        """Copy a dir from the remote node to the local filesystem.
+
+        Copy `source_dir` from the remote node associated with this remote session to
+        `destination_dir` on the local filesystem. The new local dir will be created
+        at `destination_dir` path.
+
+        Args:
+            source_dir: The dir on the remote node.
+            destination_dir: A dir path on the local filesystem.
+            compress_format: The compression format to use. Default is no compression.
+            exclude: Files or dirs to exclude before creating the tarball.
+        """
+
+    @abstractmethod
+    def copy_dir_to(
+        self,
+        source_dir: str | Path,
+        destination_dir: str | PurePath,
+        compress_format: TarCompressionFormat = TarCompressionFormat.none,
+        exclude: str | list[str] | None = None,
+    ) -> None:
+        """Copy a dir from the local filesystem to the remote node.
+
+        Copy `source_dir` from the local filesystem to `destination_dir` on the remote node
+        associated with this remote session. The new remote dir will be created at
+        `destination_dir` path.
+
+        Args:
+            source_dir: The dir on the local filesystem.
+            destination_dir: A dir path on the remote node.
+            compress_format: The compression format to use. Default is no compression.
+            exclude: Files or dirs to exclude before creating the tarball.
+        """
+
+    @abstractmethod
+    def remove_remote_file(self, remote_file_path: str | PurePath, force: bool = True) -> None:
+        """Remove remote file, by default remove forcefully.
+
+        Args:
+            remote_file_path: The path of the file to remove.
+            force: If :data:`True`, ignore all warnings and try to remove at all costs.
+        """
+
     @abstractmethod
     def remove_remote_dir(
         self,
@@ -218,14 +261,31 @@ def remove_remote_dir(
         recursive: bool = True,
         force: bool = True,
     ) -> None:
-        """Remove remote directory, by default remove recursively and forcefully.
+        """Remove remote dir, by default remove recursively and forcefully.
 
         Args:
-            remote_dir_path: The path of the directory to remove.
-            recursive: If :data:`True`, also remove all contents inside the directory.
+            remote_dir_path: The path of the dir to remove.
+            recursive: If :data:`True`, also remove all contents inside the dir.
             force: If :data:`True`, ignore all warnings and try to remove at all costs.
         """
 
+    @abstractmethod
+    def create_remote_tarball(
+        self,
+        remote_dir_path: str | PurePath,
+        compress_format: TarCompressionFormat = TarCompressionFormat.none,
+        exclude: str | list[str] | None = None,
+    ) -> None:
+        """Create a tarball from dir on the remote node.
+
+        The remote tarball will be saved in the directory of `remote_dir_path`.
+
+        Args:
+            remote_dir_path: The path of dir on the remote node.
+            compress_format: The compression format to use. Default is no compression.
+            exclude: Files or dirs to exclude before creating the tarball.
+        """
+
     @abstractmethod
     def extract_remote_tarball(
         self,
diff --git a/dts/framework/testbed_model/posix_session.py b/dts/framework/testbed_model/posix_session.py
index 0d8c5f91a6..5a6d971d7d 100644
--- a/dts/framework/testbed_model/posix_session.py
+++ b/dts/framework/testbed_model/posix_session.py
@@ -18,7 +18,13 @@
 from framework.config import Architecture, NodeInfo
 from framework.exception import DPDKBuildError, RemoteCommandExecutionError
 from framework.settings import SETTINGS
-from framework.utils import MesonArgs
+from framework.utils import (
+    MesonArgs,
+    TarCompressionFormat,
+    create_tarball,
+    ensure_list_of_strings,
+    extract_tarball,
+)
 
 from .os_session import OSSession
 
@@ -85,21 +91,57 @@ def join_remote_path(self, *args: str | PurePath) -> PurePosixPath:
         """Overrides :meth:`~.os_session.OSSession.join_remote_path`."""
         return PurePosixPath(*args)
 
-    def copy_from(
+    def copy_from(self, source_file: str | PurePath, destination_dir: str | Path) -> None:
+        """Overrides :meth:`~.os_session.OSSession.copy_from`."""
+        self.remote_session.copy_from(source_file, destination_dir)
+
+    def copy_to(self, source_file: str | Path, destination_dir: str | PurePath) -> None:
+        """Overrides :meth:`~.os_session.OSSession.copy_to`."""
+        self.remote_session.copy_to(source_file, destination_dir)
+
+    def copy_dir_from(
         self,
-        source_file: str | PurePath,
+        source_dir: str | PurePath,
         destination_dir: str | Path,
+        compress_format: TarCompressionFormat = TarCompressionFormat.none,
+        exclude: str | list[str] | None = None,
     ) -> None:
-        """Overrides :meth:`~.os_session.OSSession.copy_from`."""
-        self.remote_session.copy_from(source_file, destination_dir)
+        """Overrides :meth:`~.os_session.OSSession.copy_dir_from`."""
+        tarball_name = f"{PurePath(source_dir).name}{compress_format.extension}"
+        remote_tarball_path = self.join_remote_path(PurePath(source_dir).parent, tarball_name)
+        self.create_remote_tarball(source_dir, compress_format, exclude)
+
+        self.copy_from(remote_tarball_path, destination_dir)
+        self.remove_remote_file(remote_tarball_path)
 
-    def copy_to(
+        tarball_path = Path(destination_dir, tarball_name)
+        extract_tarball(tarball_path)
+        tarball_path.unlink()
+
+    def copy_dir_to(
         self,
-        source_file: str | Path,
+        source_dir: str | Path,
         destination_dir: str | PurePath,
+        compress_format: TarCompressionFormat = TarCompressionFormat.none,
+        exclude: str | list[str] | None = None,
     ) -> None:
-        """Overrides :meth:`~.os_session.OSSession.copy_to`."""
-        self.remote_session.copy_to(source_file, destination_dir)
+        """Overrides :meth:`~.os_session.OSSession.copy_dir_to`."""
+        source_dir_name = Path(source_dir).name
+        tar_name = f"{source_dir_name}{compress_format.extension}"
+        tar_path = Path(Path(source_dir).parent, tar_name)
+
+        create_tarball(source_dir, compress_format, arcname=source_dir_name, exclude=exclude)
+        self.copy_to(tar_path, destination_dir)
+        tar_path.unlink()
+
+        remote_tar_path = self.join_remote_path(destination_dir, tar_name)
+        self.extract_remote_tarball(remote_tar_path)
+        self.remove_remote_file(remote_tar_path)
+
+    def remove_remote_file(self, remote_file_path: str | PurePath, force: bool = True) -> None:
+        """Overrides :meth:`~.os_session.OSSession.remove_remote_dir`."""
+        opts = PosixSession.combine_short_options(f=force)
+        self.send_command(f"rm{opts} {remote_file_path}")
 
     def remove_remote_dir(
         self,
@@ -111,10 +153,37 @@ def remove_remote_dir(
         opts = PosixSession.combine_short_options(r=recursive, f=force)
         self.send_command(f"rm{opts} {remote_dir_path}")
 
-    def extract_remote_tarball(
+    def create_remote_tarball(
         self,
-        remote_tarball_path: str | PurePath,
-        expected_dir: str | PurePath | None = None,
+        remote_dir_path: str | PurePath,
+        compress_format: TarCompressionFormat = TarCompressionFormat.none,
+        exclude: str | list[str] | None = None,
+    ) -> None:
+        """Overrides :meth:`~.os_session.OSSession.create_remote_tarball`."""
+
+        def generate_tar_exclude_args(exclude_patterns):
+            """Generate args to exclude patterns when creating a tarball.
+
+            Args:
+                exclude_patterns: The patterns to exclude from the tarball.
+
+            Returns:
+                The generated string args to exclude the specified patterns.
+            """
+            if exclude_patterns:
+                exclude_patterns = ensure_list_of_strings(exclude_patterns)
+                return "".join([f" --exclude={pattern}" for pattern in exclude_patterns])
+            return ""
+
+        target_tarball_path = f"{remote_dir_path}{compress_format.extension}"
+        self.send_command(
+            f"tar caf {target_tarball_path}{generate_tar_exclude_args(exclude)} "
+            f"-C {PurePath(remote_dir_path).parent} {PurePath(remote_dir_path).name}",
+            60,
+        )
+
+    def extract_remote_tarball(
+        self, remote_tarball_path: str | PurePath, expected_dir: str | PurePath | None = None
     ) -> None:
         """Overrides :meth:`~.os_session.OSSession.extract_remote_tarball`."""
         self.send_command(
diff --git a/dts/framework/utils.py b/dts/framework/utils.py
index 6b5d5a805f..5757872fbd 100644
--- a/dts/framework/utils.py
+++ b/dts/framework/utils.py
@@ -15,12 +15,15 @@
 """
 
 import atexit
+import fnmatch
 import json
 import os
 import subprocess
+import tarfile
 from enum import Enum
 from pathlib import Path
 from subprocess import SubprocessError
+from typing import Any
 
 from scapy.packet import Packet  # type: ignore[import-untyped]
 
@@ -140,13 +143,17 @@ def __str__(self) -> str:
         return " ".join(f"{self._default_library} {self._dpdk_args}".split())
 
 
-class _TarCompressionFormat(StrEnum):
+class TarCompressionFormat(StrEnum):
     """Compression formats that tar can use.
 
     Enum names are the shell compression commands
     and Enum values are the associated file extensions.
+
+    The 'none' member represents no compression, only archiving with tar.
+    Its value is set to 'tar' to indicate that the file is an uncompressed tar archive.
     """
 
+    none = "tar"
     gzip = "gz"
     compress = "Z"
     bzip2 = "bz2"
@@ -156,6 +163,16 @@ class _TarCompressionFormat(StrEnum):
     xz = "xz"
     zstd = "zst"
 
+    @property
+    def extension(self):
+        """Return the extension associated with the compression format.
+
+        If the compression format is 'none', the extension will be in the format '.tar'.
+        For other compression formats, the extension will be in the format
+        '.tar.{compression format}'.
+        """
+        return f".{self.value}" if self == self.none else f".{self.none.value}.{self.value}"
+
 
 class DPDKGitTarball:
     """Compressed tarball of DPDK from the repository.
@@ -169,7 +186,7 @@ class DPDKGitTarball:
     """
 
     _git_ref: str
-    _tar_compression_format: _TarCompressionFormat
+    _tar_compression_format: TarCompressionFormat
     _tarball_dir: Path
     _tarball_name: str
     _tarball_path: Path | None
@@ -178,7 +195,7 @@ def __init__(
         self,
         git_ref: str,
         output_dir: str,
-        tar_compression_format: _TarCompressionFormat = _TarCompressionFormat.xz,
+        tar_compression_format: TarCompressionFormat = TarCompressionFormat.xz,
     ):
         """Create the tarball during initialization.
 
@@ -198,9 +215,7 @@ def __init__(
 
         self._create_tarball_dir()
 
-        self._tarball_name = (
-            f"dpdk-tarball-{self._git_ref}.tar.{self._tar_compression_format.value}"
-        )
+        self._tarball_name = f"dpdk-tarball-{self._git_ref}{self._tar_compression_format.extension}"
         self._tarball_path = self._check_tarball_path()
         if not self._tarball_path:
             self._create_tarball()
@@ -244,3 +259,73 @@ def _delete_tarball(self) -> None:
     def __fspath__(self) -> str:
         """The os.PathLike protocol implementation."""
         return str(self._tarball_path)
+
+
+def ensure_list_of_strings(value: Any | list[Any]) -> list[str]:
+    """Ensure the input is a list of strings.
+
+    Converting all elements to list of strings format.
+
+    Args:
+        value: A single value or a list of values.
+
+    Returns:
+        A list of strings.
+    """
+    return list(map(str, value) if isinstance(value, list) else str(value))
+
+
+def create_tarball(
+    source_path: str | Path,
+    compress_format: TarCompressionFormat = TarCompressionFormat.none,
+    arcname: str | None = None,
+    exclude: Any | list[Any] | None = None,
+):
+    """Create a tarball archive from a source dir or file.
+
+    The tarball archive will be saved in the same path as `source_path` parent path.
+
+    Args:
+        source_path: The path to the source dir or file to be included in the tarball.
+        compress_format: The compression format to use. Defaults is no compression.
+        arcname: The name under which `source_path` will be archived.
+        exclude: Files or dirs to exclude before creating the tarball.
+    """
+
+    def create_filter_function(exclude_patterns: str | list[str] | None):
+        """Create a filter function based on the provided exclude patterns.
+
+        Args:
+            exclude_patterns: The patterns to exclude from the tarball.
+
+        Returns:
+            The filter function that excludes files based on the patterns.
+        """
+        if exclude_patterns:
+            exclude_patterns = ensure_list_of_strings(exclude_patterns)
+
+            def filter_func(tarinfo: tarfile.TarInfo) -> tarfile.TarInfo | None:
+                file_name = os.path.basename(tarinfo.name)
+                if any(fnmatch.fnmatch(file_name, pattern) for pattern in exclude_patterns):
+                    return None
+                return tarinfo
+
+            return filter_func
+        return None
+
+    with tarfile.open(
+        f"{source_path}{compress_format.extension}", f"w:{compress_format.value}"
+    ) as tar:
+        tar.add(source_path, arcname=arcname, filter=create_filter_function(exclude))
+
+
+def extract_tarball(tar_path: str | Path):
+    """Extract the contents of a tarball.
+
+    The tarball will be extracted in the same path as `tar_path` parent path.
+
+    Args:
+        tar_path: The path to the tarball file to extract.
+    """
+    with tarfile.open(tar_path, "r") as tar:
+        tar.extractall(path=Path(tar_path).parent)
-- 
2.43.0