From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 8E78642813; Thu, 23 Mar 2023 11:40:51 +0100 (CET) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id EC19A427E9; Thu, 23 Mar 2023 11:40:47 +0100 (CET) Received: from mail-ed1-f43.google.com (mail-ed1-f43.google.com [209.85.208.43]) by mails.dpdk.org (Postfix) with ESMTP id A748A4021D for ; Thu, 23 Mar 2023 11:40:45 +0100 (CET) Received: by mail-ed1-f43.google.com with SMTP id o12so84379875edb.9 for ; Thu, 23 Mar 2023 03:40:45 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=pantheon-tech.20210112.gappssmtp.com; s=20210112; t=1679568045; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=oz1ozGAfsGcPbj+0Q5e7u1+iFyJENA04XuWWAh1NbH8=; b=zZzmUCzGDZ17ULcoJy8IyCy5T7JW0OdX7jQKwu39M905ZEobv45Z/6JT7AxVGW1KVG +1TgsrCXPBki9GrVVI0wAppgHFv1JVrTEiNUuJaZ7NjEK9uRL2AUy0bWFIiVVUCGid7k +O6GPpY+wceCgdUZNGAnJFZuzXmDmv4AYdcLX0BesAObsU8f04nP7KUfvEFGwj2HbjVj rHGahll8bO3RN5XFuNgD+7HtzUsCxi0ApimhyO2eyDo4sm4lsvH7pyryZe3Q5uv+gCFm qYli4OzfONhHxmAAQlTEVl43LT6+ElpweB/CZsF8YZ2PzX4EoqBXuuYQ/Z0+XefKW+rI rZEw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; t=1679568045; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=oz1ozGAfsGcPbj+0Q5e7u1+iFyJENA04XuWWAh1NbH8=; b=4gVkGba5aNXYDFn/Psr2dobdUr+aXviJUjxRuHsRDzG7jKjlY15uvxIucW1ChFpDpC f8NJUqK3p/1Dx8dVtDmn6j2OpDvsbrB7sYrjBa/ujE77BTarthLtNolhzQJn9XD6CkXl 2ELW1twKv0XSSQ0gp7SwuMgPISqPYmQx1PrPBGhcMpZAhON5I2rJyyMMLH/9rATVxW4o RNUkrKN0zC1oJF//QQWtwzTSZcAhrHgaq6O4kdA6aBPSh27JvWC/WqLKfO5zVYgSrLNY vxX2+hp0TVLDit/G0LEGq5pqIbS76ICwTUgk0n2RZmp8aaLp2pP+ISyMh7gWOtmGur25 z2MA== X-Gm-Message-State: AO0yUKUIueCX/5Go+Rvyg0DvYbbLFx+clbg851lDh03bNk903Z8kXNuw sBo0VeBr2neTiT5UFNarTukQAw== X-Google-Smtp-Source: AK7set/RSq3b/nHuxzGtZGGrMpn1DHsIi5tTTPZL0diIq6eO4B3+YbRYkZPnL/Rz/IhIhhvcOlteNg== X-Received: by 2002:a17:907:d8c:b0:926:e917:133c with SMTP id go12-20020a1709070d8c00b00926e917133cmr12814205ejc.47.1679568045259; Thu, 23 Mar 2023 03:40:45 -0700 (PDT) Received: from localhost.localdomain (ip-46.34.245.107.o2inet.sk. [46.34.245.107]) by smtp.gmail.com with ESMTPSA id k10-20020a1709067aca00b009294524ac21sm8472773ejo.60.2023.03.23.03.40.44 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 23 Mar 2023 03:40:45 -0700 (PDT) From: =?UTF-8?q?Juraj=20Linke=C5=A1?= To: thomas@monjalon.net, Honnappa.Nagarahalli@arm.com, lijuan.tu@intel.com, bruce.richardson@intel.com, wathsala.vithanage@arm.com, jspewock@iol.unh.edu Cc: dev@dpdk.org, =?UTF-8?q?Juraj=20Linke=C5=A1?= Subject: [RFC PATCH v1 1/4] dts: code adjustments for sphinx Date: Thu, 23 Mar 2023 11:40:37 +0100 Message-Id: <20230323104040.484708-2-juraj.linkes@pantheon.tech> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20230323104040.484708-1-juraj.linkes@pantheon.tech> References: <20230323104040.484708-1-juraj.linkes@pantheon.tech> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org sphinx-build only imports the Python modules when building the documentation; it doesn't run DTS. This requires changes that make the code importable without running it. This means: * properly guarding argument parsing in the if __name__ == '__main__' block. * the logger used by DTS runner underwent the same treatment so that it doesn't create unnecessary log files. * however, DTS uses the arguments to construct an object holding global variables. The defaults for the global variables needed to be moved from argument parsing elsewhere. * importing the remote_session module from framework resulted in circular imports because of one module trying to import another module. This is fixed by more granular imports. Signed-off-by: Juraj Linkeš --- dts/framework/config/__init__.py | 11 ++++ .../{testbed_model/hw => config}/cpu.py | 13 +++++ dts/framework/dts.py | 8 ++- dts/framework/remote_session/__init__.py | 3 +- dts/framework/remote_session/linux_session.py | 2 +- dts/framework/remote_session/os_session.py | 12 +++- .../remote_session/remote/__init__.py | 16 ------ .../{remote => }/remote_session.py | 0 .../{remote => }/ssh_session.py | 0 dts/framework/settings.py | 55 ++++++++++--------- dts/framework/testbed_model/__init__.py | 10 +--- dts/framework/testbed_model/hw/__init__.py | 27 --------- dts/framework/testbed_model/node.py | 12 ++-- dts/framework/testbed_model/sut_node.py | 9 ++- .../testbed_model/{hw => }/virtual_device.py | 0 dts/main.py | 3 +- dts/tests/TestSuite_hello_world.py | 6 +- 17 files changed, 88 insertions(+), 99 deletions(-) rename dts/framework/{testbed_model/hw => config}/cpu.py (95%) delete mode 100644 dts/framework/remote_session/remote/__init__.py rename dts/framework/remote_session/{remote => }/remote_session.py (100%) rename dts/framework/remote_session/{remote => }/ssh_session.py (100%) delete mode 100644 dts/framework/testbed_model/hw/__init__.py rename dts/framework/testbed_model/{hw => }/virtual_device.py (100%) diff --git a/dts/framework/config/__init__.py b/dts/framework/config/__init__.py index ebb0823ff5..293c4cb15b 100644 --- a/dts/framework/config/__init__.py +++ b/dts/framework/config/__init__.py @@ -7,6 +7,8 @@ Yaml config parsing methods """ +# pylama:ignore=W0611 + import json import os.path import pathlib @@ -19,6 +21,15 @@ from framework.settings import SETTINGS +from .cpu import ( + LogicalCore, + LogicalCoreCount, + LogicalCoreCountFilter, + LogicalCoreList, + LogicalCoreListFilter, + lcore_filter, +) + class StrEnum(Enum): @staticmethod diff --git a/dts/framework/testbed_model/hw/cpu.py b/dts/framework/config/cpu.py similarity index 95% rename from dts/framework/testbed_model/hw/cpu.py rename to dts/framework/config/cpu.py index d1918a12dc..8fe785dfe4 100644 --- a/dts/framework/testbed_model/hw/cpu.py +++ b/dts/framework/config/cpu.py @@ -272,3 +272,16 @@ def filter(self) -> list[LogicalCore]: ) return filtered_lcores + + +def lcore_filter( + core_list: list[LogicalCore], + filter_specifier: LogicalCoreCount | LogicalCoreList, + ascending: bool, +) -> LogicalCoreFilter: + if isinstance(filter_specifier, LogicalCoreList): + return LogicalCoreListFilter(core_list, filter_specifier, ascending) + elif isinstance(filter_specifier, LogicalCoreCount): + return LogicalCoreCountFilter(core_list, filter_specifier, ascending) + else: + raise ValueError(f"Unsupported filter r{filter_specifier}") diff --git a/dts/framework/dts.py b/dts/framework/dts.py index 0502284580..22a09b7e34 100644 --- a/dts/framework/dts.py +++ b/dts/framework/dts.py @@ -3,6 +3,7 @@ # Copyright(c) 2022-2023 PANTHEON.tech s.r.o. # Copyright(c) 2022-2023 University of New Hampshire +import logging import sys from .config import CONFIGURATION, BuildTargetConfiguration, ExecutionConfiguration @@ -12,7 +13,8 @@ from .testbed_model import SutNode from .utils import check_dts_python_version -dts_logger: DTSLOG = getLogger("DTSRunner") +# dummy defaults to satisfy linters +dts_logger: DTSLOG | logging.Logger = logging.getLogger("DTSRunner") result: DTSResult = DTSResult(dts_logger) @@ -24,6 +26,10 @@ def run_all() -> None: global dts_logger global result + # create a regular DTS logger and create a new result with it + dts_logger = getLogger("DTSRunner") + result = DTSResult(dts_logger) + # check the python version of the server that run dts check_dts_python_version() diff --git a/dts/framework/remote_session/__init__.py b/dts/framework/remote_session/__init__.py index ee221503df..17ca1459f7 100644 --- a/dts/framework/remote_session/__init__.py +++ b/dts/framework/remote_session/__init__.py @@ -17,7 +17,8 @@ from .linux_session import LinuxSession from .os_session import OSSession -from .remote import CommandResult, RemoteSession, SSHSession +from .remote_session import CommandResult, RemoteSession +from .ssh_session import SSHSession def create_session( diff --git a/dts/framework/remote_session/linux_session.py b/dts/framework/remote_session/linux_session.py index a1e3bc3a92..c8ce5fe6da 100644 --- a/dts/framework/remote_session/linux_session.py +++ b/dts/framework/remote_session/linux_session.py @@ -2,8 +2,8 @@ # Copyright(c) 2023 PANTHEON.tech s.r.o. # Copyright(c) 2023 University of New Hampshire +from framework.config import LogicalCore from framework.exception import RemoteCommandExecutionError -from framework.testbed_model import LogicalCore from framework.utils import expand_range from .posix_session import PosixSession diff --git a/dts/framework/remote_session/os_session.py b/dts/framework/remote_session/os_session.py index 4c48ae2567..246f0358ea 100644 --- a/dts/framework/remote_session/os_session.py +++ b/dts/framework/remote_session/os_session.py @@ -6,13 +6,13 @@ from collections.abc import Iterable from pathlib import PurePath -from framework.config import Architecture, NodeConfiguration +from framework.config import Architecture, LogicalCore, NodeConfiguration from framework.logger import DTSLOG from framework.settings import SETTINGS -from framework.testbed_model import LogicalCore from framework.utils import EnvVarsDict, MesonArgs -from .remote import CommandResult, RemoteSession, create_remote_session +from .remote_session import CommandResult, RemoteSession +from .ssh_session import SSHSession class OSSession(ABC): @@ -173,3 +173,9 @@ def setup_hugepages(self, hugepage_amount: int, force_first_numa: bool) -> None: if needed and mount the hugepages if needed. If force_first_numa is True, configure hugepages just on the first socket. """ + + +def create_remote_session( + node_config: NodeConfiguration, name: str, logger: DTSLOG +) -> RemoteSession: + return SSHSession(node_config, name, logger) diff --git a/dts/framework/remote_session/remote/__init__.py b/dts/framework/remote_session/remote/__init__.py deleted file mode 100644 index 8a1512210a..0000000000 --- a/dts/framework/remote_session/remote/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# SPDX-License-Identifier: BSD-3-Clause -# Copyright(c) 2023 PANTHEON.tech s.r.o. - -# pylama:ignore=W0611 - -from framework.config import NodeConfiguration -from framework.logger import DTSLOG - -from .remote_session import CommandResult, RemoteSession -from .ssh_session import SSHSession - - -def create_remote_session( - node_config: NodeConfiguration, name: str, logger: DTSLOG -) -> RemoteSession: - return SSHSession(node_config, name, logger) diff --git a/dts/framework/remote_session/remote/remote_session.py b/dts/framework/remote_session/remote_session.py similarity index 100% rename from dts/framework/remote_session/remote/remote_session.py rename to dts/framework/remote_session/remote_session.py diff --git a/dts/framework/remote_session/remote/ssh_session.py b/dts/framework/remote_session/ssh_session.py similarity index 100% rename from dts/framework/remote_session/remote/ssh_session.py rename to dts/framework/remote_session/ssh_session.py diff --git a/dts/framework/settings.py b/dts/framework/settings.py index 71955f4581..144f9dea62 100644 --- a/dts/framework/settings.py +++ b/dts/framework/settings.py @@ -6,7 +6,7 @@ import argparse import os from collections.abc import Callable, Iterable, Sequence -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path from typing import Any, TypeVar @@ -59,15 +59,18 @@ def __call__( @dataclass(slots=True, frozen=True) class _Settings: - config_file_path: str - output_dir: str - timeout: float - verbose: bool - skip_setup: bool - dpdk_tarball_path: Path - compile_timeout: float - test_cases: list - re_run: int + config_file_path: Path = Path(Path(__file__).parent.parent, "conf.yaml") + output_dir: str = "output" + timeout: float = 15 + verbose: bool = False + skip_setup: bool = False + dpdk_tarball_path: Path | str = "dpdk.tar.xz" + compile_timeout: float = 1200 + test_cases: list[str] = field(default_factory=list) + re_run: int = 0 + + +SETTINGS: _Settings = _Settings() def _get_parser() -> argparse.ArgumentParser: @@ -81,7 +84,8 @@ def _get_parser() -> argparse.ArgumentParser: parser.add_argument( "--config-file", action=_env_arg("DTS_CFG_FILE"), - default="conf.yaml", + default=SETTINGS.config_file_path, + type=Path, help="[DTS_CFG_FILE] configuration file that describes the test cases, SUTs " "and targets.", ) @@ -90,7 +94,7 @@ def _get_parser() -> argparse.ArgumentParser: "--output-dir", "--output", action=_env_arg("DTS_OUTPUT_DIR"), - default="output", + default=SETTINGS.output_dir, help="[DTS_OUTPUT_DIR] Output directory where dts logs and results are saved.", ) @@ -98,7 +102,7 @@ def _get_parser() -> argparse.ArgumentParser: "-t", "--timeout", action=_env_arg("DTS_TIMEOUT"), - default=15, + default=SETTINGS.timeout, type=float, help="[DTS_TIMEOUT] The default timeout for all DTS operations except for " "compiling DPDK.", @@ -108,7 +112,7 @@ def _get_parser() -> argparse.ArgumentParser: "-v", "--verbose", action=_env_arg("DTS_VERBOSE"), - default="N", + default=SETTINGS.verbose, help="[DTS_VERBOSE] Set to 'Y' to enable verbose output, logging all messages " "to the console.", ) @@ -117,7 +121,7 @@ def _get_parser() -> argparse.ArgumentParser: "-s", "--skip-setup", action=_env_arg("DTS_SKIP_SETUP"), - default="N", + default=SETTINGS.skip_setup, help="[DTS_SKIP_SETUP] Set to 'Y' to skip all setup steps on SUT and TG nodes.", ) @@ -125,7 +129,7 @@ def _get_parser() -> argparse.ArgumentParser: "--tarball", "--snapshot", action=_env_arg("DTS_DPDK_TARBALL"), - default="dpdk.tar.xz", + default=SETTINGS.dpdk_tarball_path, type=Path, help="[DTS_DPDK_TARBALL] Path to DPDK source code tarball " "which will be used in testing.", @@ -134,7 +138,7 @@ def _get_parser() -> argparse.ArgumentParser: parser.add_argument( "--compile-timeout", action=_env_arg("DTS_COMPILE_TIMEOUT"), - default=1200, + default=SETTINGS.compile_timeout, type=float, help="[DTS_COMPILE_TIMEOUT] The timeout for compiling DPDK.", ) @@ -142,8 +146,9 @@ def _get_parser() -> argparse.ArgumentParser: parser.add_argument( "--test-cases", action=_env_arg("DTS_TESTCASES"), - default="", - help="[DTS_TESTCASES] Comma-separated list of test cases to execute. " + nargs="*", + default=SETTINGS.test_cases, + help="[DTS_TESTCASES] A list of test cases to execute. " "Unknown test cases will be silently ignored.", ) @@ -151,7 +156,7 @@ def _get_parser() -> argparse.ArgumentParser: "--re-run", "--re_run", action=_env_arg("DTS_RERUN"), - default=0, + default=SETTINGS.re_run, type=int, help="[DTS_RERUN] Re-run each test case the specified amount of times " "if a test failure occurs", @@ -165,10 +170,11 @@ def _check_tarball_path(parsed_args: argparse.Namespace) -> None: raise ConfigurationError(f"DPDK tarball '{parsed_args.tarball}' doesn't exist.") -def _get_settings() -> _Settings: +def set_settings() -> None: + global SETTINGS parsed_args = _get_parser().parse_args() _check_tarball_path(parsed_args) - return _Settings( + SETTINGS = _Settings( config_file_path=parsed_args.config_file, output_dir=parsed_args.output_dir, timeout=parsed_args.timeout, @@ -176,9 +182,6 @@ def _get_settings() -> _Settings: skip_setup=(parsed_args.skip_setup == "Y"), dpdk_tarball_path=parsed_args.tarball, compile_timeout=parsed_args.compile_timeout, - test_cases=parsed_args.test_cases.split(",") if parsed_args.test_cases else [], + test_cases=parsed_args.test_cases, re_run=parsed_args.re_run, ) - - -SETTINGS: _Settings = _get_settings() diff --git a/dts/framework/testbed_model/__init__.py b/dts/framework/testbed_model/__init__.py index f54a947051..148f81993d 100644 --- a/dts/framework/testbed_model/__init__.py +++ b/dts/framework/testbed_model/__init__.py @@ -9,14 +9,6 @@ # pylama:ignore=W0611 -from .hw import ( - LogicalCore, - LogicalCoreCount, - LogicalCoreCountFilter, - LogicalCoreList, - LogicalCoreListFilter, - VirtualDevice, - lcore_filter, -) from .node import Node from .sut_node import SutNode +from .virtual_device import VirtualDevice diff --git a/dts/framework/testbed_model/hw/__init__.py b/dts/framework/testbed_model/hw/__init__.py deleted file mode 100644 index 88ccac0b0e..0000000000 --- a/dts/framework/testbed_model/hw/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -# SPDX-License-Identifier: BSD-3-Clause -# Copyright(c) 2023 PANTHEON.tech s.r.o. - -# pylama:ignore=W0611 - -from .cpu import ( - LogicalCore, - LogicalCoreCount, - LogicalCoreCountFilter, - LogicalCoreFilter, - LogicalCoreList, - LogicalCoreListFilter, -) -from .virtual_device import VirtualDevice - - -def lcore_filter( - core_list: list[LogicalCore], - filter_specifier: LogicalCoreCount | LogicalCoreList, - ascending: bool, -) -> LogicalCoreFilter: - if isinstance(filter_specifier, LogicalCoreList): - return LogicalCoreListFilter(core_list, filter_specifier, ascending) - elif isinstance(filter_specifier, LogicalCoreCount): - return LogicalCoreCountFilter(core_list, filter_specifier, ascending) - else: - raise ValueError(f"Unsupported filter r{filter_specifier}") diff --git a/dts/framework/testbed_model/node.py b/dts/framework/testbed_model/node.py index d48fafe65d..90467981c3 100644 --- a/dts/framework/testbed_model/node.py +++ b/dts/framework/testbed_model/node.py @@ -12,19 +12,17 @@ from framework.config import ( BuildTargetConfiguration, ExecutionConfiguration, - NodeConfiguration, -) -from framework.logger import DTSLOG, getLogger -from framework.remote_session import OSSession, create_session -from framework.settings import SETTINGS - -from .hw import ( LogicalCore, LogicalCoreCount, LogicalCoreList, LogicalCoreListFilter, + NodeConfiguration, lcore_filter, ) +from framework.logger import DTSLOG, getLogger +from framework.remote_session import create_session +from framework.remote_session.os_session import OSSession +from framework.settings import SETTINGS class Node(object): diff --git a/dts/framework/testbed_model/sut_node.py b/dts/framework/testbed_model/sut_node.py index 2b2b50d982..6db4a505bb 100644 --- a/dts/framework/testbed_model/sut_node.py +++ b/dts/framework/testbed_model/sut_node.py @@ -7,13 +7,18 @@ import time from pathlib import PurePath -from framework.config import BuildTargetConfiguration, NodeConfiguration +from framework.config import ( + BuildTargetConfiguration, + LogicalCoreCount, + LogicalCoreList, + NodeConfiguration, +) from framework.remote_session import CommandResult, OSSession from framework.settings import SETTINGS from framework.utils import EnvVarsDict, MesonArgs -from .hw import LogicalCoreCount, LogicalCoreList, VirtualDevice from .node import Node +from .virtual_device import VirtualDevice class SutNode(Node): diff --git a/dts/framework/testbed_model/hw/virtual_device.py b/dts/framework/testbed_model/virtual_device.py similarity index 100% rename from dts/framework/testbed_model/hw/virtual_device.py rename to dts/framework/testbed_model/virtual_device.py diff --git a/dts/main.py b/dts/main.py index 43311fa847..060ff1b19a 100755 --- a/dts/main.py +++ b/dts/main.py @@ -10,10 +10,11 @@ import logging -from framework import dts +from framework import dts, settings def main() -> None: + settings.set_settings() dts.run_all() diff --git a/dts/tests/TestSuite_hello_world.py b/dts/tests/TestSuite_hello_world.py index 7e3d95c0cf..96c31a6c8c 100644 --- a/dts/tests/TestSuite_hello_world.py +++ b/dts/tests/TestSuite_hello_world.py @@ -6,12 +6,8 @@ No other EAL parameters apart from cores are used. """ +from framework.config import LogicalCoreCount, LogicalCoreCountFilter, LogicalCoreList from framework.test_suite import TestSuite -from framework.testbed_model import ( - LogicalCoreCount, - LogicalCoreCountFilter, - LogicalCoreList, -) class TestHelloWorld(TestSuite): -- 2.30.2