[PATCH v1] dts: create tarball from git ref

[PATCH v1] dts: create tarball from git ref

[Juraj Linkeš]

Add additional convenience options for specifying what DPDK version to
test.

Signed-off-by: Juraj Linkeš <juraj.linkes@pantheon.tech>
---
 dts/framework/config/__init__.py |  11 +--
 dts/framework/settings.py        |  20 ++---
 dts/framework/utils.py           | 140 +++++++++++++++++++++++++++++++
 3 files changed, 152 insertions(+), 19 deletions(-)

diff --git a/dts/framework/config/__init__.py b/dts/framework/config/__init__.py
index ebb0823ff5..a4b73483e6 100644
--- a/dts/framework/config/__init__.py
+++ b/dts/framework/config/__init__.py
@@ -11,21 +11,14 @@
 import os.path
 import pathlib
 from dataclasses import dataclass
-from enum import Enum, auto, unique
+from enum import auto, unique
 from typing import Any, TypedDict
 
 import warlock  # type: ignore
 import yaml
 
 from framework.settings import SETTINGS
-
-
-class StrEnum(Enum):
-    @staticmethod
-    def _generate_next_value_(
-        name: str, start: int, count: int, last_values: object
-    ) -> str:
-        return name
+from framework.utils import StrEnum
 
 
 @unique
diff --git a/dts/framework/settings.py b/dts/framework/settings.py
index 71955f4581..cfa39d011b 100644
--- a/dts/framework/settings.py
+++ b/dts/framework/settings.py
@@ -10,7 +10,7 @@
 from pathlib import Path
 from typing import Any, TypeVar
 
-from .exception import ConfigurationError
+from .utils import DPDKGitTarball
 
 _T = TypeVar("_T")
 
@@ -124,11 +124,13 @@ def _get_parser() -> argparse.ArgumentParser:
     parser.add_argument(
         "--tarball",
         "--snapshot",
+        "--git-ref",
         action=_env_arg("DTS_DPDK_TARBALL"),
         default="dpdk.tar.xz",
         type=Path,
-        help="[DTS_DPDK_TARBALL] Path to DPDK source code tarball "
-        "which will be used in testing.",
+        help="[DTS_DPDK_TARBALL] Path to DPDK source code tarball or a git commit ID, "
+        "tag ID or tree ID to test. To test local changes, first commit them, "
+        "then use the commit ID with this option.",
     )
 
     parser.add_argument(
@@ -160,21 +162,19 @@ def _get_parser() -> argparse.ArgumentParser:
     return parser
 
 
-def _check_tarball_path(parsed_args: argparse.Namespace) -> None:
-    if not os.path.exists(parsed_args.tarball):
-        raise ConfigurationError(f"DPDK tarball '{parsed_args.tarball}' doesn't exist.")
-
-
 def _get_settings() -> _Settings:
     parsed_args = _get_parser().parse_args()
-    _check_tarball_path(parsed_args)
     return _Settings(
         config_file_path=parsed_args.config_file,
         output_dir=parsed_args.output_dir,
         timeout=parsed_args.timeout,
         verbose=(parsed_args.verbose == "Y"),
         skip_setup=(parsed_args.skip_setup == "Y"),
-        dpdk_tarball_path=parsed_args.tarball,
+        dpdk_tarball_path=Path(
+            DPDKGitTarball(parsed_args.tarball, parsed_args.output_dir)
+        )
+        if not os.path.exists(parsed_args.tarball)
+        else Path(parsed_args.tarball),
         compile_timeout=parsed_args.compile_timeout,
         test_cases=parsed_args.test_cases.split(",") if parsed_args.test_cases else [],
         re_run=parsed_args.re_run,
diff --git a/dts/framework/utils.py b/dts/framework/utils.py
index 55e0b0ef0e..0623106b78 100644
--- a/dts/framework/utils.py
+++ b/dts/framework/utils.py
@@ -3,7 +3,26 @@
 # Copyright(c) 2022-2023 PANTHEON.tech s.r.o.
 # Copyright(c) 2022-2023 University of New Hampshire
 
+import atexit
+import os
+import subprocess
 import sys
+from enum import Enum
+from pathlib import Path
+from subprocess import SubprocessError
+
+from .exception import ConfigurationError
+
+
+class StrEnum(Enum):
+    @staticmethod
+    def _generate_next_value_(
+        name: str, start: int, count: int, last_values: object
+    ) -> str:
+        return name
+
+    def __str__(self) -> str:
+        return self.name
 
 
 def check_dts_python_version() -> None:
@@ -80,3 +99,124 @@ def __init__(self, default_library: str | None = None, **dpdk_args: str | bool):
 
     def __str__(self) -> str:
         return " ".join(f"{self._default_library} {self._dpdk_args}".split())
+
+
+class _TarCompressionFormat(StrEnum):
+    """Compression formats that tar can use.
+
+    Enum names are the shell compression commands
+    and Enum values are the associated file extensions.
+    """
+
+    gzip = "gz"
+    compress = "Z"
+    bzip2 = "bz2"
+    lzip = "lz"
+    lzma = "lzma"
+    lzop = "lzo"
+    xz = "xz"
+    zstd = "zst"
+
+
+class DPDKGitTarball(object):
+    """Create a compressed tarball of DPDK from the repository.
+
+    The DPDK version is specified with git object git_ref.
+    The tarball will be compressed with _TarCompressionFormat,
+    which must be supported by the DTS execution environment.
+    The resulting tarball will be put into output_dir.
+
+    The class supports the os.PathLike protocol,
+    which is used to get the Path of the tarball::
+
+        from pathlib import Path
+        tarball = DPDKGitTarball("HEAD", "output")
+        tarball_path = Path(tarball)
+
+    Arguments:
+        git_ref: A git commit ID, tag ID or tree ID.
+        output_dir: The directory where to put the resulting tarball.
+        tar_compression_format: The compression format to use.
+    """
+
+    _git_ref: str
+    _tar_compression_format: _TarCompressionFormat
+    _tarball_dir: Path
+    _tarball_name: str
+    _tarball_path: Path | None
+
+    def __init__(
+        self,
+        git_ref: str,
+        output_dir: str,
+        tar_compression_format: _TarCompressionFormat = _TarCompressionFormat.xz,
+    ):
+        self._git_ref = git_ref
+        self._tar_compression_format = tar_compression_format
+
+        self._tarball_dir = Path(output_dir, "tarball")
+
+        self._get_commit_id()
+        self._create_tarball_dir()
+
+        self._tarball_name = (
+            f"dpdk-tarball-{self._git_ref}.tar.{self._tar_compression_format.value}"
+        )
+        self._tarball_path = self._check_tarball_path()
+        if not self._tarball_path:
+            self._create_tarball()
+
+    def _get_commit_id(self) -> None:
+        result = subprocess.run(
+            ["git", "rev-parse", "--verify", self._git_ref],
+            text=True,
+            capture_output=True,
+        )
+        if result.returncode != 0:
+            raise ConfigurationError(
+                f"{self._git_ref} is neither a path to an existing DPDK "
+                "archive nor a valid git reference.\n"
+                f"Command: {result.args}\n"
+                f"Stdout: {result.stdout}\n"
+                f"Stderr: {result.stderr}"
+            )
+        self._git_ref = result.stdout.strip()
+
+    def _create_tarball_dir(self) -> None:
+        os.makedirs(self._tarball_dir, exist_ok=True)
+
+    def _check_tarball_path(self) -> Path | None:
+        if self._tarball_name in os.listdir(self._tarball_dir):
+            return Path(self._tarball_dir, self._tarball_name)
+        return None
+
+    def _create_tarball(self) -> None:
+        self._tarball_path = Path(self._tarball_dir, self._tarball_name)
+
+        atexit.register(self._delete_tarball)
+
+        result = subprocess.run(
+            'git -C "$(git rev-parse --show-toplevel)" archive '
+            f'{self._git_ref} --prefix="dpdk-tarball-{self._git_ref + os.sep}" | '
+            f"{self._tar_compression_format} > {Path(self._tarball_path.absolute())}",
+            shell=True,
+            text=True,
+            capture_output=True,
+        )
+
+        if result.returncode != 0:
+            raise SubprocessError(
+                f"Git archive creation failed with exit code {result.returncode}.\n"
+                f"Command: {result.args}\n"
+                f"Stdout: {result.stdout}\n"
+                f"Stderr: {result.stderr}"
+            )
+
+        atexit.unregister(self._delete_tarball)
+
+    def _delete_tarball(self) -> None:
+        if self._tarball_path and os.path.exists(self._tarball_path):
+            os.remove(self._tarball_path)
+
+    def __fspath__(self):
+        return str(self._tarball_path)
-- 
2.30.2

Re: [PATCH v1] dts: create tarball from git ref

[Jeremy Spewock]

[-- Attachment #1: Type: text/plain, Size: 9004 bytes --]

Acked-by: Jeremy Spweock <jspweock@iol.unh.edu>

On Thu, Apr 20, 2023 at 10:16 AM Juraj Linkeš <juraj.linkes@pantheon.tech>
wrote:

> Add additional convenience options for specifying what DPDK version to
> test.
>
> Signed-off-by: Juraj Linkeš <juraj.linkes@pantheon.tech>
> ---
>  dts/framework/config/__init__.py |  11 +--
>  dts/framework/settings.py        |  20 ++---
>  dts/framework/utils.py           | 140 +++++++++++++++++++++++++++++++
>  3 files changed, 152 insertions(+), 19 deletions(-)
>
> diff --git a/dts/framework/config/__init__.py
> b/dts/framework/config/__init__.py
> index ebb0823ff5..a4b73483e6 100644
> --- a/dts/framework/config/__init__.py
> +++ b/dts/framework/config/__init__.py
> @@ -11,21 +11,14 @@
>  import os.path
>  import pathlib
>  from dataclasses import dataclass
> -from enum import Enum, auto, unique
> +from enum import auto, unique
>  from typing import Any, TypedDict
>
>  import warlock  # type: ignore
>  import yaml
>
>  from framework.settings import SETTINGS
> -
> -
> -class StrEnum(Enum):
> -    @staticmethod
> -    def _generate_next_value_(
> -        name: str, start: int, count: int, last_values: object
> -    ) -> str:
> -        return name
> +from framework.utils import StrEnum
>
>
>  @unique
> diff --git a/dts/framework/settings.py b/dts/framework/settings.py
> index 71955f4581..cfa39d011b 100644
> --- a/dts/framework/settings.py
> +++ b/dts/framework/settings.py
> @@ -10,7 +10,7 @@
>  from pathlib import Path
>  from typing import Any, TypeVar
>
> -from .exception import ConfigurationError
> +from .utils import DPDKGitTarball
>
>  _T = TypeVar("_T")
>
> @@ -124,11 +124,13 @@ def _get_parser() -> argparse.ArgumentParser:
>      parser.add_argument(
>          "--tarball",
>          "--snapshot",
> +        "--git-ref",
>          action=_env_arg("DTS_DPDK_TARBALL"),
>          default="dpdk.tar.xz",
>          type=Path,
> -        help="[DTS_DPDK_TARBALL] Path to DPDK source code tarball "
> -        "which will be used in testing.",
> +        help="[DTS_DPDK_TARBALL] Path to DPDK source code tarball or a
> git commit ID, "
> +        "tag ID or tree ID to test. To test local changes, first commit
> them, "
> +        "then use the commit ID with this option.",
>      )
>
>      parser.add_argument(
> @@ -160,21 +162,19 @@ def _get_parser() -> argparse.ArgumentParser:
>      return parser
>
>
> -def _check_tarball_path(parsed_args: argparse.Namespace) -> None:
> -    if not os.path.exists(parsed_args.tarball):
> -        raise ConfigurationError(f"DPDK tarball '{parsed_args.tarball}'
> doesn't exist.")
> -
> -
>  def _get_settings() -> _Settings:
>      parsed_args = _get_parser().parse_args()
> -    _check_tarball_path(parsed_args)
>      return _Settings(
>          config_file_path=parsed_args.config_file,
>          output_dir=parsed_args.output_dir,
>          timeout=parsed_args.timeout,
>          verbose=(parsed_args.verbose == "Y"),
>          skip_setup=(parsed_args.skip_setup == "Y"),
> -        dpdk_tarball_path=parsed_args.tarball,
> +        dpdk_tarball_path=Path(
> +            DPDKGitTarball(parsed_args.tarball, parsed_args.output_dir)
> +        )
> +        if not os.path.exists(parsed_args.tarball)
> +        else Path(parsed_args.tarball),
>          compile_timeout=parsed_args.compile_timeout,
>          test_cases=parsed_args.test_cases.split(",") if
> parsed_args.test_cases else [],
>          re_run=parsed_args.re_run,
> diff --git a/dts/framework/utils.py b/dts/framework/utils.py
> index 55e0b0ef0e..0623106b78 100644
> --- a/dts/framework/utils.py
> +++ b/dts/framework/utils.py
> @@ -3,7 +3,26 @@
>  # Copyright(c) 2022-2023 PANTHEON.tech s.r.o.
>  # Copyright(c) 2022-2023 University of New Hampshire
>
> +import atexit
> +import os
> +import subprocess
>  import sys
> +from enum import Enum
> +from pathlib import Path
> +from subprocess import SubprocessError
> +
> +from .exception import ConfigurationError
> +
> +
> +class StrEnum(Enum):
> +    @staticmethod
> +    def _generate_next_value_(
> +        name: str, start: int, count: int, last_values: object
> +    ) -> str:
> +        return name
> +
> +    def __str__(self) -> str:
> +        return self.name
>
>
>  def check_dts_python_version() -> None:
> @@ -80,3 +99,124 @@ def __init__(self, default_library: str | None = None,
> **dpdk_args: str | bool):
>
>      def __str__(self) -> str:
>          return " ".join(f"{self._default_library}
> {self._dpdk_args}".split())
> +
> +
> +class _TarCompressionFormat(StrEnum):
> +    """Compression formats that tar can use.
> +
> +    Enum names are the shell compression commands
> +    and Enum values are the associated file extensions.
> +    """
> +
> +    gzip = "gz"
> +    compress = "Z"
> +    bzip2 = "bz2"
> +    lzip = "lz"
> +    lzma = "lzma"
> +    lzop = "lzo"
> +    xz = "xz"
> +    zstd = "zst"
> +
> +
> +class DPDKGitTarball(object):
> +    """Create a compressed tarball of DPDK from the repository.
> +
> +    The DPDK version is specified with git object git_ref.
> +    The tarball will be compressed with _TarCompressionFormat,
> +    which must be supported by the DTS execution environment.
> +    The resulting tarball will be put into output_dir.
> +
> +    The class supports the os.PathLike protocol,
> +    which is used to get the Path of the tarball::
> +
> +        from pathlib import Path
> +        tarball = DPDKGitTarball("HEAD", "output")
> +        tarball_path = Path(tarball)
> +
> +    Arguments:
> +        git_ref: A git commit ID, tag ID or tree ID.
> +        output_dir: The directory where to put the resulting tarball.
> +        tar_compression_format: The compression format to use.
> +    """
> +
> +    _git_ref: str
> +    _tar_compression_format: _TarCompressionFormat
> +    _tarball_dir: Path
> +    _tarball_name: str
> +    _tarball_path: Path | None
> +
> +    def __init__(
> +        self,
> +        git_ref: str,
> +        output_dir: str,
> +        tar_compression_format: _TarCompressionFormat =
> _TarCompressionFormat.xz,
> +    ):
> +        self._git_ref = git_ref
> +        self._tar_compression_format = tar_compression_format
> +
> +        self._tarball_dir = Path(output_dir, "tarball")
> +
> +        self._get_commit_id()
> +        self._create_tarball_dir()
> +
> +        self._tarball_name = (
> +
> f"dpdk-tarball-{self._git_ref}.tar.{self._tar_compression_format.value}"
> +        )
> +        self._tarball_path = self._check_tarball_path()
> +        if not self._tarball_path:
> +            self._create_tarball()
> +
> +    def _get_commit_id(self) -> None:
> +        result = subprocess.run(
> +            ["git", "rev-parse", "--verify", self._git_ref],
> +            text=True,
> +            capture_output=True,
> +        )
> +        if result.returncode != 0:
> +            raise ConfigurationError(
> +                f"{self._git_ref} is neither a path to an existing DPDK "
> +                "archive nor a valid git reference.\n"
> +                f"Command: {result.args}\n"
> +                f"Stdout: {result.stdout}\n"
> +                f"Stderr: {result.stderr}"
> +            )
> +        self._git_ref = result.stdout.strip()
> +
> +    def _create_tarball_dir(self) -> None:
> +        os.makedirs(self._tarball_dir, exist_ok=True)
> +
> +    def _check_tarball_path(self) -> Path | None:
> +        if self._tarball_name in os.listdir(self._tarball_dir):
> +            return Path(self._tarball_dir, self._tarball_name)
> +        return None
> +
> +    def _create_tarball(self) -> None:
> +        self._tarball_path = Path(self._tarball_dir, self._tarball_name)
> +
> +        atexit.register(self._delete_tarball)
> +
> +        result = subprocess.run(
> +            'git -C "$(git rev-parse --show-toplevel)" archive '
> +            f'{self._git_ref} --prefix="dpdk-tarball-{self._git_ref +
> os.sep}" | '
> +            f"{self._tar_compression_format} >
> {Path(self._tarball_path.absolute())}",
> +            shell=True,
> +            text=True,
> +            capture_output=True,
> +        )
> +
> +        if result.returncode != 0:
> +            raise SubprocessError(
> +                f"Git archive creation failed with exit code
> {result.returncode}.\n"
> +                f"Command: {result.args}\n"
> +                f"Stdout: {result.stdout}\n"
> +                f"Stderr: {result.stderr}"
> +            )
> +
> +        atexit.unregister(self._delete_tarball)
> +
> +    def _delete_tarball(self) -> None:
> +        if self._tarball_path and os.path.exists(self._tarball_path):
> +            os.remove(self._tarball_path)
> +
> +    def __fspath__(self):
> +        return str(self._tarball_path)
> --
> 2.30.2
>
>

[-- Attachment #2: Type: text/html, Size: 11038 bytes --]

Re: [PATCH v1] dts: create tarball from git ref

[Thomas Monjalon]

28/04/2023 21:38, Jeremy Spewock:
> Acked-by: Jeremy Spweock <jspweock@iol.unh.edu>
> 
> On Thu, Apr 20, 2023 at 10:16 AM Juraj Linkeš <juraj.linkes@pantheon.tech>
> wrote:
> 
> > Add additional convenience options for specifying what DPDK version to
> > test.
> >
> > Signed-off-by: Juraj Linkeš <juraj.linkes@pantheon.tech>

Applied, thanks.
Sorry for the delay.