From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 11E144557F; Fri, 6 Sep 2024 15:27:50 +0200 (CEST) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id BF89742F31; Fri, 6 Sep 2024 15:27:09 +0200 (CEST) Received: from mail-ej1-f50.google.com (mail-ej1-f50.google.com [209.85.218.50]) by mails.dpdk.org (Postfix) with ESMTP id 86F2142F7F for ; Fri, 6 Sep 2024 15:27:07 +0200 (CEST) Received: by mail-ej1-f50.google.com with SMTP id a640c23a62f3a-a8a7cdfdd80so167948566b.0 for ; Fri, 06 Sep 2024 06:27:07 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=pantheon.tech; s=google; t=1725629227; x=1726234027; darn=dpdk.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=3eZ4rM2oRDNPdN09P4eTnnpu81oucVEpe/sKqNvWcmA=; b=k+9nZhi8WmkDoR2HilsCGQfZvz0PObzBow445G6TFJXZxlqMbRbWUczaOR8hbC14yt KjHn3+1uN4Ls+wrrM2IgKfLCVMQPiuaDlv7UFRYwv46CvRjw793Oqn6DlpJqlD1N5F+l Hw5DIjcoqz6yhU2oa0iPSxmjXEQpQ3nbIihdK6fzvmtNxXnRfhqfrRFutJhswMzznh5q RXU97IzsshZZJKQUKVIKAaxlTbR55HZl3m0aNozVDcE4dCXFho+mfOJ1zmrlibvzmsAc rCXg2FWBjSn9mUOdh1F9AKtajS0/OEPKoqp2r6ajHExIVgET6Q/UGA5n+gX3A07JcBlp 5UcQ== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1725629227; x=1726234027; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=3eZ4rM2oRDNPdN09P4eTnnpu81oucVEpe/sKqNvWcmA=; b=c39tpBrW61uYDkyK0/Q3yMGemsayYLd0OMNFvLUQzXV+9wKvU8dalS1YW7pR5tb63U bdEvC51x6ko+YF/OiYBkjgMbz7xvQzyb6FC66LjdYLSaCnkvSnB4pmlYM4sBFbizpvVf ci5SjkhBQ+QmQPD3H/VW1Hwo+AeCn9De4Pkv922/DWXzy5UmEvbyIRvUyL1Z+b8d+Dbj mOWwtRT8DaI55Wwv7TSME8fKxvM6VrEJrsq/ofB+FyizBemSpSicPc8FOYc1TOMDxX9w 7Vo3fCeay35aPj3OPcQJvUmOuatoaq72Mi63EYpzDQHP+9NtzeJ3bGHiNqjQjp6jIwDE 0Law== X-Gm-Message-State: AOJu0YyQheOYlHia86petQGUG+ZC0TnNY5Lnp9PQ78teTZD5VjImRA5C FVxktdHr3E4iKJR9JYuU885NOz9/p0whtjImZWC3pSGJi0KSaZMD1L2Ljs5CyVQ= X-Google-Smtp-Source: AGHT+IGZy8qhE6BsI5GPA3wR8zGoIYEtDGK+L949zMzKlbAbVBhN7opc+FctK26tSVRwt8//9/upxg== X-Received: by 2002:a17:906:db07:b0:a86:6a26:fec7 with SMTP id a640c23a62f3a-a8a886667ccmr235989466b.30.1725629226946; Fri, 06 Sep 2024 06:27:06 -0700 (PDT) Received: from jlinkes-PT-Latitude-5530.. ([84.245.121.62]) by smtp.gmail.com with ESMTPSA id a640c23a62f3a-a8a7c504701sm168943566b.25.2024.09.06.06.27.05 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Fri, 06 Sep 2024 06:27:06 -0700 (PDT) From: =?UTF-8?q?Juraj=20Linke=C5=A1?= To: thomas@monjalon.net, Honnappa.Nagarahalli@arm.com, paul.szczepanek@arm.com, Luca.Vizzarro@arm.com, alex.chapman@arm.com, probb@iol.unh.edu, jspewock@iol.unh.edu, npratte@iol.unh.edu, dmarx@iol.unh.edu Cc: dev@dpdk.org, =?UTF-8?q?Tom=C3=A1=C5=A1=20=C4=8Eurovec?= Subject: [RFC PATCH v1 06/12] dts: add ability to prevent overwriting files/dirs Date: Fri, 6 Sep 2024 15:26:50 +0200 Message-ID: <20240906132656.21729-7-juraj.linkes@pantheon.tech> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20240906132656.21729-1-juraj.linkes@pantheon.tech> References: <20240906132656.21729-1-juraj.linkes@pantheon.tech> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org From: Tomáš Ďurovec Signed-off-by: Tomáš Ďurovec --- dts/framework/settings.py | 17 ++++++++++ dts/framework/testbed_model/os_session.py | 31 +++++++++++++++--- dts/framework/testbed_model/posix_session.py | 33 +++++++++++++++++--- 3 files changed, 71 insertions(+), 10 deletions(-) diff --git a/dts/framework/settings.py b/dts/framework/settings.py index 2b8c583853..2f7089a26b 100644 --- a/dts/framework/settings.py +++ b/dts/framework/settings.py @@ -55,6 +55,11 @@ Git revision ID to test. Could be commit, tag, tree ID etc. To test local changes, first commit them, then use their commit ID. +.. option:: -f, --force +.. envvar:: DTS_FORCE + + Specify to remove an already existing dpdk tarball before copying/extracting a new one. + .. option:: --test-suite .. envvar:: DTS_TEST_SUITES @@ -110,6 +115,8 @@ class Settings: #: dpdk_tarball_path: Path | str = "" #: + force: bool = False + #: compile_timeout: float = 1200 #: test_suites: list[TestSuiteConfig] = field(default_factory=list) @@ -337,6 +344,16 @@ def _get_parser() -> _DTSArgumentParser: ) _add_env_var_to_action(action) + action = parser.add_argument( + "-f", + "--force", + action="store_true", + default=SETTINGS.force, + help="Specify to remove an already existing dpdk tarball before copying/extracting a " + "new one.", + ) + _add_env_var_to_action(action) + action = parser.add_argument( "--compile-timeout", default=SETTINGS.compile_timeout, diff --git a/dts/framework/testbed_model/os_session.py b/dts/framework/testbed_model/os_session.py index 92b1a09d94..afc9ffb814 100644 --- a/dts/framework/testbed_model/os_session.py +++ b/dts/framework/testbed_model/os_session.py @@ -178,7 +178,9 @@ def join_remote_path(self, *args: str | PurePath) -> PurePath: """ @abstractmethod - def copy_from(self, source_file: str | PurePath, destination_dir: str | Path) -> None: + def copy_from( + self, source_file: str | PurePath, destination_dir: str | Path, force: bool = SETTINGS.force + ) -> None: """Copy a file from the remote node to the local filesystem. Copy `source_file` from the remote node associated with this remote @@ -188,10 +190,14 @@ def copy_from(self, source_file: str | PurePath, destination_dir: str | Path) -> source_file: The file on the remote node. destination_dir: A dir path on the local filesystem, where the `source_file` will be saved. + force: If :data:`True`, remove an already existing `source_file` at the + `destination_dir` before copying to prevent overwriting data. """ @abstractmethod - def copy_to(self, source_file: str | Path, destination_dir: str | PurePath) -> None: + def copy_to( + self, source_file: str | Path, destination_dir: str | PurePath, force: bool = SETTINGS.force + ) -> None: """Copy a file from local filesystem to the remote node. Copy `source_file` from local filesystem to `destination_dir` @@ -201,6 +207,8 @@ def copy_to(self, source_file: str | Path, destination_dir: str | PurePath) -> N source_file: The file on the local filesystem. destination_dir: A dir path on the remote Node, where the `source_file` will be saved. + force: If :data:`True`, remove an already existing `source_file` at the + `destination_dir` before copying to prevent overwriting data. """ @abstractmethod @@ -210,6 +218,7 @@ def copy_dir_from( destination_dir: str | Path, compress_format: TarCompressionFormat = TarCompressionFormat.none, exclude: str | list[str] | None = None, + force: bool = SETTINGS.force, ) -> None: """Copy a dir from the remote node to the local filesystem. @@ -222,6 +231,8 @@ def copy_dir_from( destination_dir: A dir path on the local filesystem. compress_format: The compression format to use. Default is no compression. exclude: Files or dirs to exclude before creating the tarball. + force: If :data:`True`, remove an already existing `source_dir` at the `destination_dir` + before copying to prevent overwriting data. """ @abstractmethod @@ -231,18 +242,21 @@ def copy_dir_to( destination_dir: str | PurePath, compress_format: TarCompressionFormat = TarCompressionFormat.none, exclude: str | list[str] | None = None, + force: bool = SETTINGS.force, ) -> None: """Copy a dir from the local filesystem to the remote node. Copy `source_dir` from the local filesystem to `destination_dir` on the remote node - associated with this remote session. The new remote dir will be created at - `destination_dir` path. + associated with this remote session. The new remote dir will be created + at `destination_dir` path. Args: source_dir: The dir on the local filesystem. destination_dir: A dir path on the remote node. compress_format: The compression format to use. Default is no compression. exclude: Files or dirs to exclude before creating the tarball. + force: If :data:`True`, remove an already existing `source_dir` at the `destination_dir` + before copying to prevent overwriting data. """ @abstractmethod @@ -275,6 +289,7 @@ def create_remote_tarball( remote_dir_path: str | PurePath, compress_format: TarCompressionFormat = TarCompressionFormat.none, exclude: str | list[str] | None = None, + force: bool = SETTINGS.force, ) -> None: """Create a tarball from dir on the remote node. @@ -284,6 +299,8 @@ def create_remote_tarball( remote_dir_path: The path of dir on the remote node. compress_format: The compression format to use. Default is no compression. exclude: Files or dirs to exclude before creating the tarball. + force: If :data:`True`, remove an already existing tarball at the directory of + `remote_dir_path` before creating a new one to prevent overwriting data. """ @abstractmethod @@ -291,13 +308,17 @@ def extract_remote_tarball( self, remote_tarball_path: str | PurePath, expected_dir: str | PurePath | None = None, + force: bool = SETTINGS.force, ) -> None: - """Extract remote tarball in its remote directory. + """Extract remote tarball in its remote dir. Args: remote_tarball_path: The path of the tarball on the remote node. expected_dir: If non-empty, check whether `expected_dir` exists after extracting the archive. + force: If :data:`True` and `expected_dir` is defined, remove an already + existing `expected_dir` at the directory of `remote_dir_path` before + extracting to prevent overwriting data. """ @abstractmethod diff --git a/dts/framework/testbed_model/posix_session.py b/dts/framework/testbed_model/posix_session.py index 5a6d971d7d..94aac68e8d 100644 --- a/dts/framework/testbed_model/posix_session.py +++ b/dts/framework/testbed_model/posix_session.py @@ -91,12 +91,22 @@ def join_remote_path(self, *args: str | PurePath) -> PurePosixPath: """Overrides :meth:`~.os_session.OSSession.join_remote_path`.""" return PurePosixPath(*args) - def copy_from(self, source_file: str | PurePath, destination_dir: str | Path) -> None: + def copy_from( + self, source_file: str | PurePath, destination_dir: str | Path, force: bool = SETTINGS.force + ) -> None: """Overrides :meth:`~.os_session.OSSession.copy_from`.""" + if force: + Path(destination_dir, PurePath(source_file).name).unlink(missing_ok=True) + self.remote_session.copy_from(source_file, destination_dir) - def copy_to(self, source_file: str | Path, destination_dir: str | PurePath) -> None: + def copy_to( + self, source_file: str | Path, destination_dir: str | PurePath, force: bool = SETTINGS.force + ) -> None: """Overrides :meth:`~.os_session.OSSession.copy_to`.""" + if force: + self.remove_remote_file(PurePath(destination_dir, Path(source_file).name)) + self.remote_session.copy_to(source_file, destination_dir) def copy_dir_from( @@ -105,13 +115,14 @@ def copy_dir_from( destination_dir: str | Path, compress_format: TarCompressionFormat = TarCompressionFormat.none, exclude: str | list[str] | None = None, + force: bool = SETTINGS.force, ) -> None: """Overrides :meth:`~.os_session.OSSession.copy_dir_from`.""" tarball_name = f"{PurePath(source_dir).name}{compress_format.extension}" remote_tarball_path = self.join_remote_path(PurePath(source_dir).parent, tarball_name) self.create_remote_tarball(source_dir, compress_format, exclude) - self.copy_from(remote_tarball_path, destination_dir) + self.copy_from(remote_tarball_path, destination_dir, force) self.remove_remote_file(remote_tarball_path) tarball_path = Path(destination_dir, tarball_name) @@ -124,6 +135,7 @@ def copy_dir_to( destination_dir: str | PurePath, compress_format: TarCompressionFormat = TarCompressionFormat.none, exclude: str | list[str] | None = None, + force: bool = SETTINGS.force, ) -> None: """Overrides :meth:`~.os_session.OSSession.copy_dir_to`.""" source_dir_name = Path(source_dir).name @@ -131,7 +143,7 @@ def copy_dir_to( tar_path = Path(Path(source_dir).parent, tar_name) create_tarball(source_dir, compress_format, arcname=source_dir_name, exclude=exclude) - self.copy_to(tar_path, destination_dir) + self.copy_to(tar_path, destination_dir, force) tar_path.unlink() remote_tar_path = self.join_remote_path(destination_dir, tar_name) @@ -158,6 +170,7 @@ def create_remote_tarball( remote_dir_path: str | PurePath, compress_format: TarCompressionFormat = TarCompressionFormat.none, exclude: str | list[str] | None = None, + force: bool = SETTINGS.force, ) -> None: """Overrides :meth:`~.os_session.OSSession.create_remote_tarball`.""" @@ -176,6 +189,9 @@ def generate_tar_exclude_args(exclude_patterns): return "" target_tarball_path = f"{remote_dir_path}{compress_format.extension}" + if force: + self.remove_remote_file(target_tarball_path) + self.send_command( f"tar caf {target_tarball_path}{generate_tar_exclude_args(exclude)} " f"-C {PurePath(remote_dir_path).parent} {PurePath(remote_dir_path).name}", @@ -183,13 +199,20 @@ def generate_tar_exclude_args(exclude_patterns): ) def extract_remote_tarball( - self, remote_tarball_path: str | PurePath, expected_dir: str | PurePath | None = None + self, + remote_tarball_path: str | PurePath, + expected_dir: str | PurePath | None = None, + force: bool = SETTINGS.force, ) -> None: """Overrides :meth:`~.os_session.OSSession.extract_remote_tarball`.""" + if force and expected_dir: + self.remove_remote_dir(expected_dir) + self.send_command( f"tar xfm {remote_tarball_path} -C {PurePosixPath(remote_tarball_path).parent}", 60, ) + if expected_dir: self.send_command(f"ls {expected_dir}", verify=True) -- 2.43.0