From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 43D9C45B91; Mon, 21 Oct 2024 15:50:39 +0200 (CEST) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 79AD240697; Mon, 21 Oct 2024 15:50:05 +0200 (CEST) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mails.dpdk.org (Postfix) with ESMTP id 447F4402E2 for ; Mon, 21 Oct 2024 15:49:59 +0200 (CEST) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 811EE139F; Mon, 21 Oct 2024 06:50:28 -0700 (PDT) Received: from localhost.localdomain (usa-sjc-mx-foss1.foss.arm.com [172.31.20.19]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 1CD9C3F73B; Mon, 21 Oct 2024 06:49:57 -0700 (PDT) From: Luca Vizzarro To: dev@dpdk.org Cc: Paul Szczepanek , Patrick Robb , =?UTF-8?q?Tom=C3=A1=C5=A1=20=C4=8Eurovec?= , Luca Vizzarro Subject: [PATCH v2 5/7] dts: add support for externally compiled DPDK Date: Mon, 21 Oct 2024 14:49:33 +0100 Message-ID: <20241021134935.1210500-6-luca.vizzarro@arm.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20241021134935.1210500-1-luca.vizzarro@arm.com> References: <20240930160204.19582-1-tomas.durovec@pantheon.tech> <20241021134935.1210500-1-luca.vizzarro@arm.com> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org From: Tomáš Ďurovec Enable the user to use either a DPDK source tree directory or a tarball, with and without a pre-built build directory. These can be stored on either SUT node or the DTS host. The DPDK build setup or the pre-built binaries can be specified through the configuration file, the command line arguments or environment variables. Signed-off-by: Tomáš Ďurovec Signed-off-by: Luca Vizzarro --- dts/conf.yaml | 24 +- dts/framework/config/__init__.py | 123 ++++++- dts/framework/config/conf_yaml_schema.json | 62 +++- dts/framework/config/types.py | 17 +- dts/framework/exception.py | 4 +- dts/framework/remote_session/dpdk_shell.py | 2 +- dts/framework/runner.py | 8 +- dts/framework/settings.py | 200 +++++++++-- dts/framework/test_result.py | 23 +- dts/framework/testbed_model/node.py | 22 +- dts/framework/testbed_model/os_session.py | 63 +++- dts/framework/testbed_model/posix_session.py | 39 ++- dts/framework/testbed_model/sut_node.py | 351 +++++++++++++------ 13 files changed, 732 insertions(+), 206 deletions(-) diff --git a/dts/conf.yaml b/dts/conf.yaml index 814744a1fc..8a65a481d6 100644 --- a/dts/conf.yaml +++ b/dts/conf.yaml @@ -5,12 +5,24 @@ test_runs: # define one test run environment - dpdk_build: - arch: x86_64 - os: linux - cpu: native - # the combination of the following two makes CC="ccache gcc" - compiler: gcc - compiler_wrapper: ccache + # dpdk_tree: Commented out because `tarball` is defined. + tarball: dpdk-tarball.tar.xz + # Either `dpdk_tree` or `tarball` can be defined, but not both. + remote: false # Optional, defaults to false. If it's true, the `dpdk_tree` or `tarball` + # is located on the SUT node, instead of the execution host. + + # precompiled_build_dir: Commented out because `build_options` is defined. + build_options: + arch: x86_64 + os: linux + cpu: native + # the combination of the following two makes CC="ccache gcc" + compiler: gcc + compiler_wrapper: ccache # Optional. + # If `precompiled_build_dir` is defined, DPDK has been pre-built and the build directory is + # in a subdirectory of DPDK tree root directory. Otherwise, will be using the `build_options` + # to build the DPDK from source. Either `precompiled_build_dir` or `build_options` can be + # defined, but not both. perf: false # disable performance testing func: true # enable functional testing skip_smoke_tests: false # optional diff --git a/dts/framework/config/__init__.py b/dts/framework/config/__init__.py index 49b2e8d016..015c8d60db 100644 --- a/dts/framework/config/__init__.py +++ b/dts/framework/config/__init__.py @@ -35,6 +35,7 @@ import json import os.path +import tarfile from dataclasses import dataclass, fields from enum import auto, unique from pathlib import Path @@ -47,6 +48,7 @@ from framework.config.types import ( ConfigurationDict, DPDKBuildConfigDict, + DPDKConfigurationDict, NodeConfigDict, PortConfigDict, TestRunConfigDict, @@ -380,6 +382,117 @@ def from_dict(cls, d: DPDKBuildConfigDict) -> Self: ) +@dataclass(slots=True, frozen=True) +class DPDKLocation: + """DPDK location. + + The path to the DPDK sources, build dir and type of location. + + Attributes: + dpdk_tree: The path to the DPDK source tree directory. Only one of `dpdk_tree` or `tarball` + must be provided. + tarball: The path to the DPDK tarball. Only one of `dpdk_tree` or `tarball` must be + provided. + remote: Optional, defaults to :data:`False`. If :data:`True`, `dpdk_tree` or `tarball` is + located on the SUT node, instead of the execution host. + build_dir: If it's defined, DPDK has been pre-compiled and the build directory is located in + a subdirectory of `dpdk_tree` or `tarball` root directory. Otherwise, will be using + `build_options` from configuration to build the DPDK from source. + """ + + dpdk_tree: str | None + tarball: str | None + remote: bool + build_dir: str | None + + @classmethod + def from_dict(cls, d: DPDKConfigurationDict) -> Self: + """A convenience method that processes and validates the inputs before creating an instance. + + Validate existence and format of `dpdk_tree` or `tarball` on local filesystem, if + `remote` is False. + + Args: + d: The configuration dictionary. + + Returns: + The DPDK location instance. + + Raises: + ConfigurationError: If `dpdk_tree` or `tarball` not found in local filesystem or they + aren't in the right format. + """ + dpdk_tree = d.get("dpdk_tree") + tarball = d.get("tarball") + remote = d.get("remote", False) + + if not remote: + if dpdk_tree: + if not Path(dpdk_tree).exists(): + raise ConfigurationError( + f"DPDK tree '{dpdk_tree}' not found in local filesystem." + ) + + if not Path(dpdk_tree).is_dir(): + raise ConfigurationError(f"The DPDK tree '{dpdk_tree}' must be a directory.") + + dpdk_tree = os.path.realpath(dpdk_tree) + + if tarball: + if not Path(tarball).exists(): + raise ConfigurationError( + f"DPDK tarball '{tarball}' not found in local filesystem." + ) + + if not tarfile.is_tarfile(tarball): + raise ConfigurationError( + f"The DPDK tarball '{tarball}' must be a valid tar archive." + ) + + return cls( + dpdk_tree=dpdk_tree, + tarball=tarball, + remote=remote, + build_dir=d.get("precompiled_build_dir"), + ) + + +@dataclass +class DPDKConfiguration: + """The configuration of the DPDK build. + + The configuration contain the location of the DPDK and configuration used for + building it. + + Attributes: + dpdk_location: The location of the DPDK tree. + dpdk_build_config: A DPDK build configuration to test. If :data:`None`, + DTS will use pre-built DPDK from `build_dir` in a :dataclass:`DPDKLocation`. + """ + + dpdk_location: DPDKLocation + dpdk_build_config: DPDKBuildConfiguration | None + + @classmethod + def from_dict(cls, d: DPDKConfigurationDict) -> Self: + """A convenience method that processes the inputs before creating an instance. + + Args: + d: The configuration dictionary. + + Returns: + The DPDK configuration. + """ + return cls( + dpdk_location=DPDKLocation.from_dict(d), + dpdk_build_config=( + DPDKBuildConfiguration.from_dict(d["build_options"]) + if d.get("build_options") + else None + ), + ) + + @dataclass(slots=True, frozen=True) class DPDKBuildInfo: """Various versions and other information about a DPDK build. @@ -389,8 +502,8 @@ class DPDKBuildInfo: compiler_version: The version of the compiler used to build DPDK. """ - dpdk_version: str - compiler_version: str + dpdk_version: str | None + compiler_version: str | None @dataclass(slots=True, frozen=True) @@ -437,7 +550,7 @@ class TestRunConfiguration: and with what DPDK build. Attributes: - dpdk_build: A DPDK build to test. + dpdk_config: The DPDK configuration used to test. perf: Whether to run performance tests. func: Whether to run functional tests. skip_smoke_tests: Whether to skip smoke tests. @@ -448,7 +561,7 @@ class TestRunConfiguration: random_seed: The seed to use for pseudo-random generation. """ - dpdk_build: DPDKBuildConfiguration + dpdk_config: DPDKConfiguration perf: bool func: bool skip_smoke_tests: bool @@ -498,7 +611,7 @@ def from_dict( ) random_seed = d.get("random_seed", None) return cls( - dpdk_build=DPDKBuildConfiguration.from_dict(d["dpdk_build"]), + dpdk_config=DPDKConfiguration.from_dict(d["dpdk_build"]), perf=d["perf"], func=d["func"], skip_smoke_tests=skip_smoke_tests, diff --git a/dts/framework/config/conf_yaml_schema.json b/dts/framework/config/conf_yaml_schema.json index 94d7efa5f5..3e37555fc2 100644 --- a/dts/framework/config/conf_yaml_schema.json +++ b/dts/framework/config/conf_yaml_schema.json @@ -110,9 +110,8 @@ "mscv" ] }, - "dpdk_build": { + "build_options": { "type": "object", - "description": "DPDK build configuration supported by DTS.", "properties": { "arch": { "type": "string", @@ -133,7 +132,7 @@ "compiler": { "$ref": "#/definitions/compiler" }, - "compiler_wrapper": { + "compiler_wrapper": { "type": "string", "description": "This will be added before compiler to the CC variable when building DPDK. Optional." } @@ -146,6 +145,63 @@ "compiler" ] }, + "dpdk_build": { + "type": "object", + "description": "DPDK source and build configuration.", + "properties": { + "dpdk_tree": { + "type": "string", + "description": "The path to the DPDK source tree directory to test. Only one of `dpdk_tree` or `tarball` must be provided." + }, + "tarball": { + "type": "string", + "description": "The path to the DPDK source tarball to test. Only one of `dpdk_tree` or `tarball` must be provided." + }, + "remote": { + "type": "boolean", + "description": "Optional, defaults to false. If it's true, the `dpdk_tree` or `tarball` is located on the SUT node, instead of the execution host." + }, + "precompiled_build_dir": { + "type": "string", + "description": "If it's defined, DPDK has been pre-built and the build directory is located in a subdirectory of DPDK tree root directory. Otherwise, will be using a `build_options` to build the DPDK from source. Either this or `build_options` must be defined, but not both." + }, + "build_options": { + "$ref": "#/definitions/build_options", + "description": "Either this or `precompiled_build_dir` must be defined, but not both. DPDK build configuration supported by DTS." + } + }, + "allOf": [ + { + "oneOf": [ + { + "required": [ + "dpdk_tree" + ] + }, + { + "required": [ + "tarball" + ] + } + ] + }, + { + "oneOf": [ + { + "required": [ + "precompiled_build_dir" + ] + }, + { + "required": [ + "build_options" + ] + } + ] + } + ], + "additionalProperties": false + }, "hugepages_2mb": { "type": "object", "description": "Optional hugepage configuration. If not specified, hugepages won't be configured and DTS will use system configuration.", diff --git a/dts/framework/config/types.py b/dts/framework/config/types.py index a710c20d6a..02e738a61e 100644 --- a/dts/framework/config/types.py +++ b/dts/framework/config/types.py @@ -86,6 +86,21 @@ class DPDKBuildConfigDict(TypedDict): compiler_wrapper: str +class DPDKConfigurationDict(TypedDict): + """Allowed keys and values.""" + + #: + dpdk_tree: str | None + #: + tarball: str | None + #: + remote: bool + #: + precompiled_build_dir: str | None + #: + build_options: DPDKBuildConfigDict + + class TestSuiteConfigDict(TypedDict): """Allowed keys and values.""" @@ -108,7 +123,7 @@ class TestRunConfigDict(TypedDict): """Allowed keys and values.""" #: - dpdk_build: DPDKBuildConfigDict + dpdk_build: DPDKConfigurationDict #: perf: bool #: diff --git a/dts/framework/exception.py b/dts/framework/exception.py index f45f789825..d967ede09b 100644 --- a/dts/framework/exception.py +++ b/dts/framework/exception.py @@ -184,8 +184,8 @@ class InteractiveCommandExecutionError(DTSError): severity: ClassVar[ErrorSeverity] = ErrorSeverity.REMOTE_CMD_EXEC_ERR -class RemoteDirectoryExistsError(DTSError): - """A directory that exists on a remote node.""" +class RemoteFileNotFoundError(DTSError): + """A remote file or directory is requested but doesn’t exist.""" #: severity: ClassVar[ErrorSeverity] = ErrorSeverity.REMOTE_CMD_EXEC_ERR diff --git a/dts/framework/remote_session/dpdk_shell.py b/dts/framework/remote_session/dpdk_shell.py index c5f5c2d116..b39132cc42 100644 --- a/dts/framework/remote_session/dpdk_shell.py +++ b/dts/framework/remote_session/dpdk_shell.py @@ -104,4 +104,4 @@ def _update_real_path(self, path: PurePath) -> None: Adds the remote DPDK build directory to the path. """ - super()._update_real_path(self._node.remote_dpdk_build_dir.joinpath(path)) + super()._update_real_path(PurePath(self._node.remote_dpdk_build_dir).joinpath(path)) diff --git a/dts/framework/runner.py b/dts/framework/runner.py index 55cb16df73..8bbe698eaf 100644 --- a/dts/framework/runner.py +++ b/dts/framework/runner.py @@ -364,15 +364,19 @@ def _run_test_run( test_run_config: A test run configuration. test_run_result: The test run's result. test_suites_with_cases: The test suites with test cases to run. + + Raises: + ConfigurationError: If the DPDK sources or build is not set up from config or settings. """ self._logger.info( f"Running test run with SUT '{test_run_config.system_under_test_node.name}'." ) test_run_result.add_sut_info(sut_node.node_info) try: - sut_node.set_up_test_run(test_run_config) + dpdk_location = SETTINGS.dpdk_location or test_run_config.dpdk_config.dpdk_location + sut_node.set_up_test_run(test_run_config, dpdk_location) test_run_result.add_dpdk_build_info(sut_node.get_dpdk_build_info()) - tg_node.set_up_test_run(test_run_config) + tg_node.set_up_test_run(test_run_config, dpdk_location) test_run_result.update_setup(Result.PASS) except Exception as e: self._logger.exception("Test run setup failed.") diff --git a/dts/framework/settings.py b/dts/framework/settings.py index 52a1582d5c..08529805da 100644 --- a/dts/framework/settings.py +++ b/dts/framework/settings.py @@ -39,21 +39,37 @@ Set to any value to enable logging everything to the console. -.. option:: -s, --skip-setup -.. envvar:: DTS_SKIP_SETUP +.. option:: --dpdk-tree +.. envvar:: DTS_DPDK_TREE - Set to any value to skip building DPDK. + The path to the DPDK source tree directory to test. Cannot be used in conjunction with --tarball + and --revision. .. option:: --tarball, --snapshot .. envvar:: DTS_DPDK_TARBALL - Path to DPDK source code tarball to test. + The path to the DPDK source tarball to test. DPDK must be contained in a folder with the same + name as the tarball file. Cannot be used in conjunction with --dpdk-tree and + --revision. .. option:: --revision, --rev, --git-ref .. envvar:: DTS_DPDK_REVISION_ID - Git revision ID to test. Could be commit, tag, tree ID etc. - To test local changes, first commit them, then use their commit ID. + Git revision ID to test. Could be commit, tag, tree ID etc. To test local changes, first commit + them, then use their commit ID. Cannot be used in conjunction with --dpdk-tree and --tarball. + +.. option:: --remote-source +.. envvar:: DTS_REMOTE_SOURCE + + Set this option if either the DPDK source tree or tarball to be used are located on the SUT + node. Can only be used with --dpdk-tree or --tarball. + +.. option:: --precompiled-build-dir +.. envvar:: DTS_PRECOMPILED_BUILD_DIR + + Define the subdirectory under the DPDK tree root directory where the pre-compiled binaries are + located. If set, DTS will build DPDK under the `build` directory instead. Can only be used with + --dpdk-tree or --tarball. .. option:: --test-suite .. envvar:: DTS_TEST_SUITES @@ -86,12 +102,13 @@ import argparse import os import sys +import tarfile from argparse import Action, ArgumentDefaultsHelpFormatter, _get_action_name from dataclasses import dataclass, field from pathlib import Path from typing import Callable -from .config import TestSuiteConfig +from .config import DPDKLocation, TestSuiteConfig from .exception import ConfigurationError from .utils import DPDKGitTarball, get_commit_id @@ -112,9 +129,7 @@ class Settings: #: verbose: bool = False #: - skip_setup: bool = False - #: - dpdk_tarball_path: Path | str = "" + dpdk_location: DPDKLocation | None = None #: compile_timeout: float = 1200 #: @@ -242,14 +257,6 @@ def _get_help_string(self, action): return help -def _parse_tarball_path(file_path: str) -> Path: - """Validate whether `file_path` is valid and return a Path object.""" - path = Path(file_path) - if not path.exists() or not path.is_file(): - raise argparse.ArgumentTypeError("The file path provided is not a valid file") - return path - - def _parse_revision_id(rev_id: str) -> str: """Validate revision ID and retrieve corresponding commit ID.""" try: @@ -258,6 +265,47 @@ def _parse_revision_id(rev_id: str) -> str: raise argparse.ArgumentTypeError("The Git revision ID supplied is invalid or ambiguous") +def _required_with_one_of(parser: _DTSArgumentParser, action: Action, *required_dests: str) -> None: + """Verify that `action` is listed together with at least one of `required_dests`. + + Verify that when `action` is among the command-line arguments or + environment variables, at least one of `required_dests` is also among + the command-line arguments or environment variables. + + Args: + parser: The custom ArgumentParser object which contains `action`. + action: The action to be verified. + *required_dests: Destination variable names of the required arguments. + + Raises: + argparse.ArgumentTypeError: When none of the required_dest are defined. + + Example: + We have ``--option1`` and we only want it to be a passed alongside + either ``--option2`` or ``--option3`` (meaning if ``--option1`` is + passed without either ``--option2`` or ``--option3``, that's an error). + + parser = _DTSArgumentParser() + option1_arg = parser.add_argument('--option1', dest='option1', action='store_true') + option2_arg = parser.add_argument('--option2', dest='option2', action='store_true') + option2_arg = parser.add_argument('--option3', dest='option3', action='store_true') + + _required_with_one_of(parser, option1_arg, 'option2', 'option3') + """ + if _is_action_in_args(action): + for required_dest in required_dests: + required_action = parser.find_action(required_dest) + if required_action is None: + continue + + if _is_action_in_args(required_action): + return None + + raise argparse.ArgumentTypeError( + f"The '{action.dest}' is required at least with one of '{', '.join(required_dests)}'." + ) + + def _get_parser() -> _DTSArgumentParser: """Create the argument parser for DTS. @@ -312,22 +360,30 @@ def _get_parser() -> _DTSArgumentParser: ) _add_env_var_to_action(action) - action = parser.add_argument( - "-s", - "--skip-setup", - action="store_true", - default=SETTINGS.skip_setup, - help="Specify to skip all setup steps on SUT and TG nodes.", + dpdk_build = parser.add_argument_group( + "DPDK Build Options", + description="Arguments in this group (and subgroup) will be applied to a " + "DPDKLocation when the DPDK tree, tarball or revision will be provided, " + "other arguments like remote source and build dir are optional. A DPDKLocation " + "from settings are used instead of from config if construct successful.", ) - _add_env_var_to_action(action) - dpdk_source = parser.add_mutually_exclusive_group(required=True) + dpdk_source = dpdk_build.add_mutually_exclusive_group() + action = dpdk_source.add_argument( + "--dpdk-tree", + help="The path to the DPDK source tree directory to test. Cannot be used in conjunction " + "with --tarball and --revision.", + metavar="DIR_PATH", + dest="dpdk_tree_path", + ) + _add_env_var_to_action(action, "DPDK_TREE") action = dpdk_source.add_argument( "--tarball", "--snapshot", - type=_parse_tarball_path, - help="Path to DPDK source code tarball to test.", + help="The path to the DPDK source tarball to test. DPDK must be contained in a folder with " + "the same name as the tarball file. Cannot be used in conjunction with --dpdk-tree and " + "--revision.", metavar="FILE_PATH", dest="dpdk_tarball_path", ) @@ -338,13 +394,36 @@ def _get_parser() -> _DTSArgumentParser: "--rev", "--git-ref", type=_parse_revision_id, - help="Git revision ID to test. Could be commit, tag, tree ID etc. " - "To test local changes, first commit them, then use their commit ID.", + help="Git revision ID to test. Could be commit, tag, tree ID etc. To test local changes, " + "first commit them, then use their commit ID. Cannot be used in conjunction with " + "--dpdk-tree and --tarball.", metavar="ID", dest="dpdk_revision_id", ) _add_env_var_to_action(action) + action = dpdk_build.add_argument( + "--remote-source", + action="store_true", + default=False, + help="Set this option if either the DPDK source tree or tarball to be used are located on " + "the SUT node. Can only be used with --dpdk-tree or --tarball.", + ) + _add_env_var_to_action(action) + _required_with_one_of( + parser, action, "dpdk_tarball_path", "dpdk_tree_path" + ) # ignored if passed with git-ref + + action = dpdk_build.add_argument( + "--precompiled-build-dir", + help="Define the subdirectory under the DPDK tree root directory where the pre-compiled " + "binaries are located. If set, DTS will build DPDK under the `build` directory instead. " + "Can only be used with --dpdk-tree or --tarball.", + metavar="DIR_NAME", + ) + _add_env_var_to_action(action) + _required_with_one_of(parser, action, "dpdk_tarball_path", "dpdk_tree_path") + action = parser.add_argument( "--compile-timeout", default=SETTINGS.compile_timeout, @@ -395,6 +474,64 @@ def _get_parser() -> _DTSArgumentParser: return parser +def _process_dpdk_location( + dpdk_tree: str | None, + tarball: str | None, + remote: bool, + build_dir: str | None, +): + """Process and validate DPDK build arguments. + + Ensures that either `dpdk_tree` or `tarball` is provided. Validate existence and format of + `dpdk_tree` or `tarball` on local filesystem, if `remote` is False. Constructs and returns + the :class:`DPDKLocation` with the provided parameters if validation is successful. + + Args: + dpdk_tree: The path to the DPDK source tree directory. Only one of `dpdk_tree` or `tarball` + must be provided. + tarball: The path to the DPDK tarball. Only one of `dpdk_tree` or `tarball` must be + provided. + remote: If :data:`True`, `dpdk_tree` or `tarball` is located on the SUT node, instead of the + execution host. + build_dir: If it's defined, DPDK has been pre-built and the build directory is located in a + subdirectory of `dpdk_tree` or `tarball` root directory. + + Returns: + A DPDK location if construction is successful, otherwise None. + + Raises: + argparse.ArgumentTypeError: If `dpdk_tree` or `tarball` not found in local filesystem or + they aren't in the right format. + """ + if not (dpdk_tree or tarball): + return None + + if not remote: + if dpdk_tree: + if not Path(dpdk_tree).exists(): + raise argparse.ArgumentTypeError( + f"DPDK tree '{dpdk_tree}' not found in local filesystem." + ) + + if not Path(dpdk_tree).is_dir(): + raise argparse.ArgumentTypeError(f"DPDK tree '{dpdk_tree}' must be a directory.") + + dpdk_tree = os.path.realpath(dpdk_tree) + + if tarball: + if not Path(tarball).exists(): + raise argparse.ArgumentTypeError( + f"DPDK tarball '{tarball}' not found in local filesystem." + ) + + if not tarfile.is_tarfile(tarball): + raise argparse.ArgumentTypeError( + f"DPDK tarball '{tarball}' must be a valid tar archive." + ) + + return DPDKLocation(dpdk_tree=dpdk_tree, tarball=tarball, remote=remote, build_dir=build_dir) + + def _process_test_suites( parser: _DTSArgumentParser, args: list[list[str]] ) -> list[TestSuiteConfig]: @@ -434,6 +571,9 @@ def get_settings() -> Settings: if args.dpdk_revision_id: args.dpdk_tarball_path = Path(DPDKGitTarball(args.dpdk_revision_id, args.output_dir)) + args.dpdk_location = _process_dpdk_location( + args.dpdk_tree_path, args.dpdk_tarball_path, args.remote_source, args.precompiled_build_dir + ) args.test_suites = _process_test_suites(parser, args.test_suites) kwargs = {k: v for k, v in vars(args).items() if hasattr(SETTINGS, k)} diff --git a/dts/framework/test_result.py b/dts/framework/test_result.py index 7fe6136574..0cd27c1e4f 100644 --- a/dts/framework/test_result.py +++ b/dts/framework/test_result.py @@ -30,16 +30,7 @@ from framework.testbed_model.capability import Capability -from .config import ( - OS, - Architecture, - Compiler, - CPUType, - DPDKBuildInfo, - NodeInfo, - TestRunConfiguration, - TestSuiteConfig, -) +from .config import DPDKBuildInfo, NodeInfo, TestRunConfiguration, TestSuiteConfig from .exception import DTSError, ErrorSeverity from .logger import DTSLogger from .settings import SETTINGS @@ -365,10 +356,6 @@ class TestRunResult(BaseResult): The internal list stores the results of all test suites in a given test run. Attributes: - arch: The DPDK build architecture. - os: The DPDK build operating system. - cpu: The DPDK build CPU. - compiler: The DPDK build compiler. compiler_version: The DPDK build compiler version. dpdk_version: The built DPDK version. sut_os_name: The operating system of the SUT node. @@ -376,10 +363,6 @@ class TestRunResult(BaseResult): sut_kernel_version: The operating system kernel version of the SUT node. """ - arch: Architecture - os: OS - cpu: CPUType - compiler: Compiler compiler_version: str | None dpdk_version: str | None sut_os_name: str @@ -395,10 +378,6 @@ def __init__(self, test_run_config: TestRunConfiguration): test_run_config: A test run configuration. """ super().__init__() - self.arch = test_run_config.dpdk_build.arch - self.os = test_run_config.dpdk_build.os - self.cpu = test_run_config.dpdk_build.cpu - self.compiler = test_run_config.dpdk_build.compiler self.compiler_version = None self.dpdk_version = None self._config = test_run_config diff --git a/dts/framework/testbed_model/node.py b/dts/framework/testbed_model/node.py index 51443cd71f..62867fd80c 100644 --- a/dts/framework/testbed_model/node.py +++ b/dts/framework/testbed_model/node.py @@ -15,12 +15,11 @@ from abc import ABC from ipaddress import IPv4Interface, IPv6Interface -from typing import Any, Callable, Union +from typing import Union -from framework.config import OS, NodeConfiguration, TestRunConfiguration +from framework.config import OS, DPDKLocation, NodeConfiguration, TestRunConfiguration from framework.exception import ConfigurationError from framework.logger import DTSLogger, get_dts_logger -from framework.settings import SETTINGS from .cpu import ( LogicalCore, @@ -95,7 +94,9 @@ def _init_ports(self) -> None: for port in self.ports: self.configure_port_state(port) - def set_up_test_run(self, test_run_config: TestRunConfiguration) -> None: + def set_up_test_run( + self, test_run_config: TestRunConfiguration, dpdk_location: DPDKLocation + ) -> None: """Test run setup steps. Configure hugepages on all DTS node types. Additional steps can be added by @@ -104,6 +105,7 @@ def set_up_test_run(self, test_run_config: TestRunConfiguration) -> None: Args: test_run_config: A test run configuration according to which the setup steps will be taken. + dpdk_location: The target source of the DPDK tree. """ self._setup_hugepages() @@ -216,18 +218,6 @@ def close(self) -> None: for session in self._other_sessions: session.close() - @staticmethod - def skip_setup(func: Callable[..., Any]) -> Callable[..., Any]: - """Skip the decorated function. - - The :option:`--skip-setup` command line argument and the :envvar:`DTS_SKIP_SETUP` - environment variable enable the decorator. - """ - if SETTINGS.skip_setup: - return lambda *args: None - else: - return func - def create_session(node_config: NodeConfiguration, name: str, logger: DTSLogger) -> OSSession: """Factory for OS-aware sessions. diff --git a/dts/framework/testbed_model/os_session.py b/dts/framework/testbed_model/os_session.py index 6c3f84dec1..6194ddb989 100644 --- a/dts/framework/testbed_model/os_session.py +++ b/dts/framework/testbed_model/os_session.py @@ -137,17 +137,6 @@ def _get_privileged_command(command: str) -> str: The modified command that executes with administrative privileges. """ - @abstractmethod - def guess_dpdk_remote_dir(self, remote_dir: str | PurePath) -> PurePath: - """Try to find DPDK directory in `remote_dir`. - - The directory is the one which is created after the extraction of the tarball. The files - are usually extracted into a directory starting with ``dpdk-``. - - Returns: - The absolute path of the DPDK remote directory, empty path if not found. - """ - @abstractmethod def get_remote_tmp_dir(self) -> PurePath: """Get the path of the temporary directory of the remote OS. @@ -177,6 +166,17 @@ def join_remote_path(self, *args: str | PurePath) -> PurePath: The resulting joined path. """ + @abstractmethod + def remote_path_exists(self, remote_path: str | PurePath) -> bool: + """Check whether `remote_path` exists on the remote system. + + Args: + remote_path: The path to check. + + Returns: + :data:`True` if the path exists, :data:`False` otherwise. + """ + @abstractmethod def copy_from(self, source_file: str | PurePath, destination_dir: str | Path) -> None: """Copy a file from the remote node to the local filesystem. @@ -344,6 +344,47 @@ def extract_remote_tarball( the archive. """ + @abstractmethod + def is_remote_dir(self, remote_path: str) -> bool: + """Check if the `remote_path` is a directory. + + Args: + remote_tarball_path: The path to the remote tarball. + + Returns: + If :data:`True` the `remote_path` is a directory, otherwise :data:`False`. + """ + + @abstractmethod + def is_remote_tarfile(self, remote_tarball_path: str) -> bool: + """Check if the `remote_tarball_path` is a tar archive. + + Args: + remote_tarball_path: The path to the remote tarball. + + Returns: + If :data:`True` the `remote_tarball_path` is a tar archive, otherwise :data:`False`. + """ + + @abstractmethod + def get_tarball_top_dir( + self, remote_tarball_path: str | PurePath + ) -> str | PurePosixPath | None: + """Get the top directory of the remote tarball. + + Examines the contents of a tarball located at the given `remote_tarball_path` and + determines the top-level directory. If all files and directories in the tarball share + the same top-level directory, that directory name is returned. If the tarball contains + multiple top-level directories or is empty, the method return None. + + Args: + remote_tarball_path: The path to the remote tarball. + + Returns: + The top directory of the tarball. If there are multiple top directories + or the tarball is empty, returns :data:`None`. + """ + @abstractmethod def build_dpdk( self, diff --git a/dts/framework/testbed_model/posix_session.py b/dts/framework/testbed_model/posix_session.py index 94e721da61..5ab7c18fb7 100644 --- a/dts/framework/testbed_model/posix_session.py +++ b/dts/framework/testbed_model/posix_session.py @@ -91,6 +91,11 @@ def join_remote_path(self, *args: str | PurePath) -> PurePosixPath: """Overrides :meth:`~.os_session.OSSession.join_remote_path`.""" return PurePosixPath(*args) + def remote_path_exists(self, remote_path: str | PurePath) -> bool: + """Overrides :meth:`~.os_session.OSSession.remote_path_exists`.""" + result = self.send_command(f"test -e {remote_path}") + return not result.return_code + def copy_from(self, source_file: str | PurePath, destination_dir: str | Path) -> None: """Overrides :meth:`~.os_session.OSSession.copy_from`.""" self.remote_session.copy_from(source_file, destination_dir) @@ -196,6 +201,32 @@ def extract_remote_tarball( if expected_dir: self.send_command(f"ls {expected_dir}", verify=True) + def is_remote_dir(self, remote_path: str) -> bool: + """Overrides :meth:`~.os_session.OSSession.is_remote_dir`.""" + result = self.send_command(f"test -d {remote_path}") + return not result.return_code + + def is_remote_tarfile(self, remote_tarball_path: str) -> bool: + """Overrides :meth:`~.os_session.OSSession.is_remote_tarfile`.""" + result = self.send_command(f"tar -tvf {remote_tarball_path}") + return not result.return_code + + def get_tarball_top_dir( + self, remote_tarball_path: str | PurePath + ) -> str | PurePosixPath | None: + """Overrides :meth:`~.os_session.OSSession.get_tarball_top_dir`.""" + members = self.send_command(f"tar tf {remote_tarball_path}").stdout.split() + + top_dirs = [] + for member in members: + parts_of_member = PurePosixPath(member).parts + if parts_of_member: + top_dirs.append(parts_of_member[0]) + + if len(set(top_dirs)) == 1: + return top_dirs[0] + return None + def build_dpdk( self, env_vars: dict, @@ -301,7 +332,7 @@ def _get_dpdk_pids(self, dpdk_runtime_dirs: Iterable[str | PurePath]) -> list[in pid_regex = r"p(\d+)" for dpdk_runtime_dir in dpdk_runtime_dirs: dpdk_config_file = PurePosixPath(dpdk_runtime_dir, "config") - if self._remote_files_exists(dpdk_config_file): + if self.remote_path_exists(dpdk_config_file): out = self.send_command(f"lsof -Fp {dpdk_config_file}").stdout if out and "No such file or directory" not in out: for out_line in out.splitlines(): @@ -310,10 +341,6 @@ def _get_dpdk_pids(self, dpdk_runtime_dirs: Iterable[str | PurePath]) -> list[in pids.append(int(match.group(1))) return pids - def _remote_files_exists(self, remote_path: PurePath) -> bool: - result = self.send_command(f"test -e {remote_path}") - return not result.return_code - def _check_dpdk_hugepages(self, dpdk_runtime_dirs: Iterable[str | PurePath]) -> None: """Check there aren't any leftover hugepages. @@ -325,7 +352,7 @@ def _check_dpdk_hugepages(self, dpdk_runtime_dirs: Iterable[str | PurePath]) -> """ for dpdk_runtime_dir in dpdk_runtime_dirs: hugepage_info = PurePosixPath(dpdk_runtime_dir, "hugepage_info") - if self._remote_files_exists(hugepage_info): + if self.remote_path_exists(hugepage_info): out = self.send_command(f"lsof -Fp {hugepage_info}").stdout if out and "No such file or directory" not in out: self._logger.warning("Some DPDK processes did not free hugepages.") diff --git a/dts/framework/testbed_model/sut_node.py b/dts/framework/testbed_model/sut_node.py index 0eac12098f..e160386324 100644 --- a/dts/framework/testbed_model/sut_node.py +++ b/dts/framework/testbed_model/sut_node.py @@ -13,21 +13,21 @@ import os -import tarfile import time from pathlib import PurePath from framework.config import ( DPDKBuildConfiguration, DPDKBuildInfo, + DPDKLocation, NodeInfo, SutNodeConfiguration, TestRunConfiguration, ) +from framework.exception import ConfigurationError, RemoteFileNotFoundError from framework.params.eal import EalParams from framework.remote_session.remote_session import CommandResult -from framework.settings import SETTINGS -from framework.utils import MesonArgs +from framework.utils import MesonArgs, TarCompressionFormat from .node import Node from .os_session import OSSession @@ -39,14 +39,13 @@ class SutNode(Node): The SUT node extends :class:`Node` with DPDK specific features: - * DPDK build, + * Managing DPDK source tree on the remote SUT, + * Building the DPDK from source or using a pre-built version, * Gathering of DPDK build info, * The running of DPDK apps, interactively or one-time execution, * DPDK apps cleanup. - The :option:`--tarball` command line argument and the :envvar:`DTS_DPDK_TARBALL` - environment variable configure the path to the DPDK tarball - or the git commit ID, tag ID or tree ID to test. + Building DPDK from source uses `build` configuration inside `dpdk_build` of configuration. Attributes: config: The SUT node configuration. @@ -57,16 +56,17 @@ class SutNode(Node): virtual_devices: list[VirtualDevice] dpdk_prefix_list: list[str] dpdk_timestamp: str - _dpdk_build_config: DPDKBuildConfiguration | None _env_vars: dict _remote_tmp_dir: PurePath - __remote_dpdk_dir: PurePath | None + __remote_dpdk_tree_path: str | PurePath | None + _remote_dpdk_build_dir: PurePath | None _app_compile_timeout: float _dpdk_kill_session: OSSession | None _dpdk_version: str | None _node_info: NodeInfo | None _compiler_version: str | None _path_to_devbind_script: PurePath | None + _ports_bound_to_dpdk: bool def __init__(self, node_config: SutNodeConfiguration): """Extend the constructor with SUT node specifics. @@ -77,10 +77,10 @@ def __init__(self, node_config: SutNodeConfiguration): super().__init__(node_config) self.virtual_devices = [] self.dpdk_prefix_list = [] - self._dpdk_build_config = None self._env_vars = {} self._remote_tmp_dir = self.main_session.get_remote_tmp_dir() - self.__remote_dpdk_dir = None + self.__remote_dpdk_tree_path = None + self._remote_dpdk_build_dir = None self._app_compile_timeout = 90 self._dpdk_kill_session = None self.dpdk_timestamp = ( @@ -90,43 +90,38 @@ def __init__(self, node_config: SutNodeConfiguration): self._node_info = None self._compiler_version = None self._path_to_devbind_script = None + self._ports_bound_to_dpdk = False self._logger.info(f"Created node: {self.name}") @property - def _remote_dpdk_dir(self) -> PurePath: - """The remote DPDK dir. - - This internal property should be set after extracting the DPDK tarball. If it's not set, - that implies the DPDK setup step has been skipped, in which case we can guess where - a previous build was located. - """ - if self.__remote_dpdk_dir is None: - self.__remote_dpdk_dir = self._guess_dpdk_remote_dir() - return self.__remote_dpdk_dir - - @_remote_dpdk_dir.setter - def _remote_dpdk_dir(self, value: PurePath) -> None: - self.__remote_dpdk_dir = value + def _remote_dpdk_tree_path(self) -> str | PurePath: + """The remote DPDK tree path.""" + if self.__remote_dpdk_tree_path: + return self.__remote_dpdk_tree_path + + self._logger.warning( + "Failed to get remote dpdk tree path because we don't know the " + "location on the SUT node." + ) + return "" @property - def remote_dpdk_build_dir(self) -> PurePath: - """The remote DPDK build directory. - - This is the directory where DPDK was built. - We assume it was built in a subdirectory of the extracted tarball. - """ - if self._dpdk_build_config: - return self.main_session.join_remote_path( - self._remote_dpdk_dir, self._dpdk_build_config.name - ) - else: - return self.main_session.join_remote_path(self._remote_dpdk_dir, "build") + def remote_dpdk_build_dir(self) -> str | PurePath: + """The remote DPDK build dir path.""" + if self._remote_dpdk_build_dir: + return self._remote_dpdk_build_dir + + self._logger.warning( + "Failed to get remote dpdk build dir because we don't know the " + "location on the SUT node." + ) + return "" @property - def dpdk_version(self) -> str: + def dpdk_version(self) -> str | None: """Last built DPDK version.""" if self._dpdk_version is None: - self._dpdk_version = self.main_session.get_dpdk_version(self._remote_dpdk_dir) + self._dpdk_version = self.main_session.get_dpdk_version(self._remote_dpdk_tree_path) return self._dpdk_version @property @@ -137,26 +132,28 @@ def node_info(self) -> NodeInfo: return self._node_info @property - def compiler_version(self) -> str: + def compiler_version(self) -> str | None: """The node's compiler version.""" if self._compiler_version is None: - if self._dpdk_build_config is not None: - self._compiler_version = self.main_session.get_compiler_version( - self._dpdk_build_config.compiler.name - ) - else: - self._logger.warning( - "Failed to get compiler version because _dpdk_build_config is None." - ) - return "" + self._logger.warning("The `compiler_version` is None because a pre-built DPDK is used.") + return self._compiler_version + @compiler_version.setter + def compiler_version(self, value: str) -> None: + """Set the `compiler_version` used on the SUT node. + + Args: + value: The node's compiler version. + """ + self._compiler_version = value + @property - def path_to_devbind_script(self) -> PurePath: + def path_to_devbind_script(self) -> PurePath | str: """The path to the dpdk-devbind.py script on the node.""" if self._path_to_devbind_script is None: self._path_to_devbind_script = self.main_session.join_remote_path( - self._remote_dpdk_dir, "usertools", "dpdk-devbind.py" + self._remote_dpdk_tree_path, "usertools", "dpdk-devbind.py" ) return self._path_to_devbind_script @@ -168,101 +165,249 @@ def get_dpdk_build_info(self) -> DPDKBuildInfo: """ return DPDKBuildInfo(dpdk_version=self.dpdk_version, compiler_version=self.compiler_version) - def _guess_dpdk_remote_dir(self) -> PurePath: - return self.main_session.guess_dpdk_remote_dir(self._remote_tmp_dir) + def set_up_test_run( + self, test_run_config: TestRunConfiguration, dpdk_location: DPDKLocation + ) -> None: + """Extend the test run setup with vdev config and DPDK build set up. - def set_up_test_run(self, test_run_config: TestRunConfiguration) -> None: - """Extend the test run setup with vdev config. + This method extends the setup process by configuring virtual devices and preparing the DPDK + environment based on the provided configuration. Args: test_run_config: A test run configuration according to which the setup steps will be taken. + dpdk_location: The target source of the DPDK tree. """ - super().set_up_test_run(test_run_config) + super().set_up_test_run(test_run_config, dpdk_location) for vdev in test_run_config.vdevs: self.virtual_devices.append(VirtualDevice(vdev)) - self._set_up_dpdk(test_run_config.dpdk_build) + self._set_up_dpdk(dpdk_location, test_run_config.dpdk_config.dpdk_build_config) def tear_down_test_run(self) -> None: - """Extend the test run teardown with virtual device teardown.""" + """Extend the test run teardown with virtual device teardown and DPDK teardown.""" super().tear_down_test_run() self.virtual_devices = [] self._tear_down_dpdk() - def _set_up_dpdk(self, dpdk_build_config: DPDKBuildConfiguration) -> None: + def _set_up_dpdk( + self, dpdk_location: DPDKLocation, dpdk_build_config: DPDKBuildConfiguration | None + ) -> None: """Set up DPDK the SUT node and bind ports. - DPDK setup includes setting all internals needed for the build, the copying of DPDK tarball - and then building DPDK. The drivers are bound to those that DPDK needs. + DPDK setup includes setting all internals needed for the build, the copying of DPDK + sources and then building DPDK or using the exist ones from the `dpdk_location`. The drivers + are bound to those that DPDK needs. Args: - dpdk_build_config: The DPDK build test run configuration according to which - the setup steps will be taken. + dpdk_location: The location of the DPDK tree. + dpdk_build_config: A DPDK build configuration to test. If :data:`None`, + DTS will use pre-built DPDK from a :dataclass:`DPDKLocation`. """ - self._configure_dpdk_build(dpdk_build_config) - self._copy_dpdk_tarball() - self._build_dpdk() + self._set_remote_dpdk_tree_path(dpdk_location.dpdk_tree, dpdk_location.remote) + if not self._remote_dpdk_tree_path: + if dpdk_location.dpdk_tree: + self._copy_dpdk_tree(dpdk_location.dpdk_tree) + elif dpdk_location.tarball: + self._prepare_and_extract_dpdk_tarball(dpdk_location.tarball, dpdk_location.remote) + + self._set_remote_dpdk_build_dir(dpdk_location.build_dir) + if not self.remote_dpdk_build_dir and dpdk_build_config: + self._configure_dpdk_build(dpdk_build_config) + self._build_dpdk() + self.bind_ports_to_driver() def _tear_down_dpdk(self) -> None: """Reset DPDK variables and bind port driver to the OS driver.""" self._env_vars = {} - self._dpdk_build_config = None - self.__remote_dpdk_dir = None + self.__remote_dpdk_tree_path = None + self._remote_dpdk_build_dir = None self._dpdk_version = None - self._compiler_version = None + self.compiler_version = None self.bind_ports_to_driver(for_dpdk=False) + def _set_remote_dpdk_tree_path(self, dpdk_tree: str | None, remote: bool): + """Set the path to the remote DPDK source tree based on the provided DPDK location. + + If :data:`dpdk_tree` and :data:`remote` are defined, check existence of :data:`dpdk_tree` + on SUT node and sets the `_remote_dpdk_tree_path` property. Otherwise, sets nothing. + + Verify DPDK source tree existence on the SUT node, if exists sets the + `_remote_dpdk_tree_path` property, otherwise sets nothing. + + Args: + dpdk_tree: The path to the DPDK source tree directory. + remote: Indicates whether the `dpdk_tree` is already on the SUT node, instead of the + execution host. + + Raises: + RemoteFileNotFoundError: If the DPDK source tree is expected to be on the SUT node but + is not found. + """ + if remote and dpdk_tree: + if not self.main_session.remote_path_exists(dpdk_tree): + raise RemoteFileNotFoundError( + f"Remote DPDK source tree '{dpdk_tree}' not found in SUT node." + ) + if not self.main_session.is_remote_dir(dpdk_tree): + raise ConfigurationError( + f"Remote DPDK source tree '{dpdk_tree}' must be a directory." + ) + + self.__remote_dpdk_tree_path = PurePath(dpdk_tree) + + def _copy_dpdk_tree(self, dpdk_tree_path: str) -> None: + """Copy the DPDK source tree to the SUT. + + Args: + dpdk_tree_path: The path to DPDK source tree on local filesystem. + """ + self._logger.info( + f"Copying DPDK source tree to SUT: '{dpdk_tree_path}' into '{self._remote_tmp_dir}'." + ) + self.main_session.copy_dir_to( + dpdk_tree_path, + self._remote_tmp_dir, + exclude=[".git", "*.o"], + compress_format=TarCompressionFormat.gzip, + ) + + self.__remote_dpdk_tree_path = self.main_session.join_remote_path( + self._remote_tmp_dir, PurePath(dpdk_tree_path).name + ) + + def _prepare_and_extract_dpdk_tarball(self, dpdk_tarball: str, remote: bool) -> None: + """Ensure the DPDK tarball is available on the SUT node and extract it. + + This method ensures that the DPDK source tree tarball is available on the + SUT node. If the `dpdk_tarball` is local, it is copied to the SUT node. If the + `dpdk_tarball` is already on the SUT node, it verifies its existence. + The `dpdk_tarball` is then extracted on the SUT node. + + This method sets the `_remote_dpdk_tree_path` property to the path of the + extracted DPDK tree on the SUT node. + + Args: + dpdk_tarball: The path to the DPDK tarball, either locally or on the SUT node. + remote: Indicates whether the `dpdk_tarball` is already on the SUT node, instead of the + execution host. + + Raises: + RemoteFileNotFoundError: If the `dpdk_tarball` is expected to be on the SUT node but + is not found. + """ + + def remove_tarball_suffix(remote_tarball_path: PurePath) -> PurePath: + """Remove the tarball suffix from the path. + + Args: + remote_tarball_path: The path to the remote tarball. + + Returns: + The path without the tarball suffix. + """ + if len(remote_tarball_path.suffixes) > 1: + if remote_tarball_path.suffixes[-2] == ".tar": + suffixes_to_remove = "".join(remote_tarball_path.suffixes[-2:]) + return PurePath(str(remote_tarball_path).replace(suffixes_to_remove, "")) + return remote_tarball_path.with_suffix("") + + if remote: + if not self.main_session.remote_path_exists(dpdk_tarball): + raise RemoteFileNotFoundError( + f"Remote DPDK tarball '{dpdk_tarball}' not found in SUT." + ) + if not self.main_session.is_remote_tarfile(dpdk_tarball): + raise ConfigurationError( + f"Remote DPDK tarball '{dpdk_tarball}' must be a tar archive." + ) + + remote_tarball_path = PurePath(dpdk_tarball) + else: + self._logger.info( + f"Copying DPDK tarball to SUT: '{dpdk_tarball}' into '{self._remote_tmp_dir}'." + ) + self.main_session.copy_to(dpdk_tarball, self._remote_tmp_dir) + + remote_tarball_path = self.main_session.join_remote_path( + self._remote_tmp_dir, PurePath(dpdk_tarball).name + ) + + tarball_top_dir = self.main_session.get_tarball_top_dir(remote_tarball_path) + self.__remote_dpdk_tree_path = self.main_session.join_remote_path( + PurePath(remote_tarball_path).parent, + tarball_top_dir or remove_tarball_suffix(remote_tarball_path), + ) + + self._logger.info( + "Extracting DPDK tarball on SUT: " + f"'{remote_tarball_path}' into '{self._remote_dpdk_tree_path}'." + ) + self.main_session.extract_remote_tarball( + remote_tarball_path, + self._remote_dpdk_tree_path, + ) + + def _set_remote_dpdk_build_dir(self, build_dir: str | None): + """Set the `remote_dpdk_build_dir` on the SUT. + + If :data:`build_dir` is defined, check existence on the SUT node and sets the + `remote_dpdk_build_dir` property by joining the `_remote_dpdk_tree_path` and `build_dir`. + Otherwise, sets nothing. + + Args: + build_dir: If it's defined, DPDK has been pre-built and the build directory is located + in a subdirectory of `dpdk_tree` or `tarball` root directory. + + Raises: + RemoteFileNotFoundError: If the `build_dir` is expected but does not exist on the SUT + node. + """ + if build_dir: + remote_dpdk_build_dir = self.main_session.join_remote_path( + self._remote_dpdk_tree_path, build_dir + ) + if not self.main_session.remote_path_exists(remote_dpdk_build_dir): + raise RemoteFileNotFoundError( + f"Remote DPDK build dir '{remote_dpdk_build_dir}' not found in SUT node." + ) + + self._remote_dpdk_build_dir = PurePath(remote_dpdk_build_dir) + def _configure_dpdk_build(self, dpdk_build_config: DPDKBuildConfiguration) -> None: - """Populate common environment variables and set DPDK build config.""" + """Populate common environment variables and set the DPDK build related properties. + + This method sets `compiler_version` for additional information and `remote_dpdk_build_dir` + from DPDK build config name. + + Args: + dpdk_build_config: A DPDK build configuration to test. + """ self._env_vars = {} - self._dpdk_build_config = dpdk_build_config self._env_vars.update(self.main_session.get_dpdk_build_env_vars(dpdk_build_config.arch)) if compiler_wrapper := dpdk_build_config.compiler_wrapper: self._env_vars["CC"] = f"'{compiler_wrapper} {dpdk_build_config.compiler.name}'" else: self._env_vars["CC"] = dpdk_build_config.compiler.name - @Node.skip_setup - def _copy_dpdk_tarball(self) -> None: - """Copy to and extract DPDK tarball on the SUT node.""" - self._logger.info("Copying DPDK tarball to SUT.") - self.main_session.copy_to(SETTINGS.dpdk_tarball_path, self._remote_tmp_dir) - - # construct remote tarball path - # the basename is the same on local host and on remote Node - remote_tarball_path = self.main_session.join_remote_path( - self._remote_tmp_dir, os.path.basename(SETTINGS.dpdk_tarball_path) + self.compiler_version = self.main_session.get_compiler_version( + dpdk_build_config.compiler.name ) - # construct remote path after extracting - with tarfile.open(SETTINGS.dpdk_tarball_path) as dpdk_tar: - dpdk_top_dir = dpdk_tar.getnames()[0] - self._remote_dpdk_dir = self.main_session.join_remote_path( - self._remote_tmp_dir, dpdk_top_dir + self._remote_dpdk_build_dir = self.main_session.join_remote_path( + self._remote_dpdk_tree_path, dpdk_build_config.name ) - self._logger.info( - f"Extracting DPDK tarball on SUT: " - f"'{remote_tarball_path}' into '{self._remote_dpdk_dir}'." - ) - # clean remote path where we're extracting - self.main_session.remove_remote_dir(self._remote_dpdk_dir) - - # then extract to remote path - self.main_session.extract_remote_tarball(remote_tarball_path, self._remote_dpdk_dir) - - @Node.skip_setup def _build_dpdk(self) -> None: """Build DPDK. - Uses the already configured target. Assumes that the tarball has - already been copied to and extracted on the SUT node. + Uses the already configured DPDK build configuration. Assumes that the + `_remote_dpdk_tree_path` has already been set on the SUT node. """ self.main_session.build_dpdk( self._env_vars, MesonArgs(default_library="static", enable_kmods=True, libdir="lib"), - self._remote_dpdk_dir, + self._remote_dpdk_tree_path, self.remote_dpdk_build_dir, ) @@ -285,7 +430,7 @@ def build_dpdk_app(self, app_name: str, **meson_dpdk_args: str | bool) -> PurePa self._env_vars, MesonArgs(examples=app_name, **meson_dpdk_args), # type: ignore [arg-type] # ^^ https://github.com/python/mypy/issues/11583 - self._remote_dpdk_dir, + self._remote_dpdk_tree_path, self.remote_dpdk_build_dir, rebuild=True, timeout=self._app_compile_timeout, @@ -342,6 +487,9 @@ def bind_ports_to_driver(self, for_dpdk: bool = True) -> None: for_dpdk: If :data:`True`, binds ports to os_driver_for_dpdk. If :data:`False`, binds to os_driver. """ + if self._ports_bound_to_dpdk == for_dpdk: + return + for port in self.ports: driver = port.os_driver_for_dpdk if for_dpdk else port.os_driver self.main_session.send_command( @@ -349,3 +497,4 @@ def bind_ports_to_driver(self, for_dpdk: bool = True) -> None: privileged=True, verify=True, ) + self._ports_bound_to_dpdk = for_dpdk -- 2.43.0