From: Jeremy Spewock <jspewock@iol.unh.edu>
To: "Juraj Linkeš" <juraj.linkes@pantheon.tech>
Cc: thomas@monjalon.net, Honnappa.Nagarahalli@arm.com,
lijuan.tu@intel.com, wathsala.vithanage@arm.com,
probb@iol.unh.edu, dev@dpdk.org
Subject: Re: [PATCH v1] dts: create tarball from git ref
Date: Fri, 28 Apr 2023 15:38:47 -0400 [thread overview]
Message-ID: <CAAA20UQiLRQi1EfCeKskk-PzwRoe8ppYzGCDuwg6C+K1S2KWGw@mail.gmail.com> (raw)
In-Reply-To: <20230420140244.701467-1-juraj.linkes@pantheon.tech>
[-- Attachment #1: Type: text/plain, Size: 9004 bytes --]
Acked-by: Jeremy Spweock <jspweock@iol.unh.edu>
On Thu, Apr 20, 2023 at 10:16 AM Juraj Linkeš <juraj.linkes@pantheon.tech>
wrote:
> Add additional convenience options for specifying what DPDK version to
> test.
>
> Signed-off-by: Juraj Linkeš <juraj.linkes@pantheon.tech>
> ---
> dts/framework/config/__init__.py | 11 +--
> dts/framework/settings.py | 20 ++---
> dts/framework/utils.py | 140 +++++++++++++++++++++++++++++++
> 3 files changed, 152 insertions(+), 19 deletions(-)
>
> diff --git a/dts/framework/config/__init__.py
> b/dts/framework/config/__init__.py
> index ebb0823ff5..a4b73483e6 100644
> --- a/dts/framework/config/__init__.py
> +++ b/dts/framework/config/__init__.py
> @@ -11,21 +11,14 @@
> import os.path
> import pathlib
> from dataclasses import dataclass
> -from enum import Enum, auto, unique
> +from enum import auto, unique
> from typing import Any, TypedDict
>
> import warlock # type: ignore
> import yaml
>
> from framework.settings import SETTINGS
> -
> -
> -class StrEnum(Enum):
> - @staticmethod
> - def _generate_next_value_(
> - name: str, start: int, count: int, last_values: object
> - ) -> str:
> - return name
> +from framework.utils import StrEnum
>
>
> @unique
> diff --git a/dts/framework/settings.py b/dts/framework/settings.py
> index 71955f4581..cfa39d011b 100644
> --- a/dts/framework/settings.py
> +++ b/dts/framework/settings.py
> @@ -10,7 +10,7 @@
> from pathlib import Path
> from typing import Any, TypeVar
>
> -from .exception import ConfigurationError
> +from .utils import DPDKGitTarball
>
> _T = TypeVar("_T")
>
> @@ -124,11 +124,13 @@ def _get_parser() -> argparse.ArgumentParser:
> parser.add_argument(
> "--tarball",
> "--snapshot",
> + "--git-ref",
> action=_env_arg("DTS_DPDK_TARBALL"),
> default="dpdk.tar.xz",
> type=Path,
> - help="[DTS_DPDK_TARBALL] Path to DPDK source code tarball "
> - "which will be used in testing.",
> + help="[DTS_DPDK_TARBALL] Path to DPDK source code tarball or a
> git commit ID, "
> + "tag ID or tree ID to test. To test local changes, first commit
> them, "
> + "then use the commit ID with this option.",
> )
>
> parser.add_argument(
> @@ -160,21 +162,19 @@ def _get_parser() -> argparse.ArgumentParser:
> return parser
>
>
> -def _check_tarball_path(parsed_args: argparse.Namespace) -> None:
> - if not os.path.exists(parsed_args.tarball):
> - raise ConfigurationError(f"DPDK tarball '{parsed_args.tarball}'
> doesn't exist.")
> -
> -
> def _get_settings() -> _Settings:
> parsed_args = _get_parser().parse_args()
> - _check_tarball_path(parsed_args)
> return _Settings(
> config_file_path=parsed_args.config_file,
> output_dir=parsed_args.output_dir,
> timeout=parsed_args.timeout,
> verbose=(parsed_args.verbose == "Y"),
> skip_setup=(parsed_args.skip_setup == "Y"),
> - dpdk_tarball_path=parsed_args.tarball,
> + dpdk_tarball_path=Path(
> + DPDKGitTarball(parsed_args.tarball, parsed_args.output_dir)
> + )
> + if not os.path.exists(parsed_args.tarball)
> + else Path(parsed_args.tarball),
> compile_timeout=parsed_args.compile_timeout,
> test_cases=parsed_args.test_cases.split(",") if
> parsed_args.test_cases else [],
> re_run=parsed_args.re_run,
> diff --git a/dts/framework/utils.py b/dts/framework/utils.py
> index 55e0b0ef0e..0623106b78 100644
> --- a/dts/framework/utils.py
> +++ b/dts/framework/utils.py
> @@ -3,7 +3,26 @@
> # Copyright(c) 2022-2023 PANTHEON.tech s.r.o.
> # Copyright(c) 2022-2023 University of New Hampshire
>
> +import atexit
> +import os
> +import subprocess
> import sys
> +from enum import Enum
> +from pathlib import Path
> +from subprocess import SubprocessError
> +
> +from .exception import ConfigurationError
> +
> +
> +class StrEnum(Enum):
> + @staticmethod
> + def _generate_next_value_(
> + name: str, start: int, count: int, last_values: object
> + ) -> str:
> + return name
> +
> + def __str__(self) -> str:
> + return self.name
>
>
> def check_dts_python_version() -> None:
> @@ -80,3 +99,124 @@ def __init__(self, default_library: str | None = None,
> **dpdk_args: str | bool):
>
> def __str__(self) -> str:
> return " ".join(f"{self._default_library}
> {self._dpdk_args}".split())
> +
> +
> +class _TarCompressionFormat(StrEnum):
> + """Compression formats that tar can use.
> +
> + Enum names are the shell compression commands
> + and Enum values are the associated file extensions.
> + """
> +
> + gzip = "gz"
> + compress = "Z"
> + bzip2 = "bz2"
> + lzip = "lz"
> + lzma = "lzma"
> + lzop = "lzo"
> + xz = "xz"
> + zstd = "zst"
> +
> +
> +class DPDKGitTarball(object):
> + """Create a compressed tarball of DPDK from the repository.
> +
> + The DPDK version is specified with git object git_ref.
> + The tarball will be compressed with _TarCompressionFormat,
> + which must be supported by the DTS execution environment.
> + The resulting tarball will be put into output_dir.
> +
> + The class supports the os.PathLike protocol,
> + which is used to get the Path of the tarball::
> +
> + from pathlib import Path
> + tarball = DPDKGitTarball("HEAD", "output")
> + tarball_path = Path(tarball)
> +
> + Arguments:
> + git_ref: A git commit ID, tag ID or tree ID.
> + output_dir: The directory where to put the resulting tarball.
> + tar_compression_format: The compression format to use.
> + """
> +
> + _git_ref: str
> + _tar_compression_format: _TarCompressionFormat
> + _tarball_dir: Path
> + _tarball_name: str
> + _tarball_path: Path | None
> +
> + def __init__(
> + self,
> + git_ref: str,
> + output_dir: str,
> + tar_compression_format: _TarCompressionFormat =
> _TarCompressionFormat.xz,
> + ):
> + self._git_ref = git_ref
> + self._tar_compression_format = tar_compression_format
> +
> + self._tarball_dir = Path(output_dir, "tarball")
> +
> + self._get_commit_id()
> + self._create_tarball_dir()
> +
> + self._tarball_name = (
> +
> f"dpdk-tarball-{self._git_ref}.tar.{self._tar_compression_format.value}"
> + )
> + self._tarball_path = self._check_tarball_path()
> + if not self._tarball_path:
> + self._create_tarball()
> +
> + def _get_commit_id(self) -> None:
> + result = subprocess.run(
> + ["git", "rev-parse", "--verify", self._git_ref],
> + text=True,
> + capture_output=True,
> + )
> + if result.returncode != 0:
> + raise ConfigurationError(
> + f"{self._git_ref} is neither a path to an existing DPDK "
> + "archive nor a valid git reference.\n"
> + f"Command: {result.args}\n"
> + f"Stdout: {result.stdout}\n"
> + f"Stderr: {result.stderr}"
> + )
> + self._git_ref = result.stdout.strip()
> +
> + def _create_tarball_dir(self) -> None:
> + os.makedirs(self._tarball_dir, exist_ok=True)
> +
> + def _check_tarball_path(self) -> Path | None:
> + if self._tarball_name in os.listdir(self._tarball_dir):
> + return Path(self._tarball_dir, self._tarball_name)
> + return None
> +
> + def _create_tarball(self) -> None:
> + self._tarball_path = Path(self._tarball_dir, self._tarball_name)
> +
> + atexit.register(self._delete_tarball)
> +
> + result = subprocess.run(
> + 'git -C "$(git rev-parse --show-toplevel)" archive '
> + f'{self._git_ref} --prefix="dpdk-tarball-{self._git_ref +
> os.sep}" | '
> + f"{self._tar_compression_format} >
> {Path(self._tarball_path.absolute())}",
> + shell=True,
> + text=True,
> + capture_output=True,
> + )
> +
> + if result.returncode != 0:
> + raise SubprocessError(
> + f"Git archive creation failed with exit code
> {result.returncode}.\n"
> + f"Command: {result.args}\n"
> + f"Stdout: {result.stdout}\n"
> + f"Stderr: {result.stderr}"
> + )
> +
> + atexit.unregister(self._delete_tarball)
> +
> + def _delete_tarball(self) -> None:
> + if self._tarball_path and os.path.exists(self._tarball_path):
> + os.remove(self._tarball_path)
> +
> + def __fspath__(self):
> + return str(self._tarball_path)
> --
> 2.30.2
>
>
[-- Attachment #2: Type: text/html, Size: 11038 bytes --]
next prev parent reply other threads:[~2023-04-28 19:39 UTC|newest]
Thread overview: 3+ messages / expand[flat|nested] mbox.gz Atom feed top
2023-04-20 14:02 Juraj Linkeš
2023-04-28 19:38 ` Jeremy Spewock [this message]
2023-07-12 16:31 ` Thomas Monjalon
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=CAAA20UQiLRQi1EfCeKskk-PzwRoe8ppYzGCDuwg6C+K1S2KWGw@mail.gmail.com \
--to=jspewock@iol.unh.edu \
--cc=Honnappa.Nagarahalli@arm.com \
--cc=dev@dpdk.org \
--cc=juraj.linkes@pantheon.tech \
--cc=lijuan.tu@intel.com \
--cc=probb@iol.unh.edu \
--cc=thomas@monjalon.net \
--cc=wathsala.vithanage@arm.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).