From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id A0A4242DA6; Thu, 13 Jul 2023 07:46:59 +0200 (CEST) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 612224114A; Thu, 13 Jul 2023 07:46:59 +0200 (CEST) Received: from mail-ed1-f49.google.com (mail-ed1-f49.google.com [209.85.208.49]) by mails.dpdk.org (Postfix) with ESMTP id 72BC540DDA for ; Thu, 13 Jul 2023 07:46:57 +0200 (CEST) Received: by mail-ed1-f49.google.com with SMTP id 4fb4d7f45d1cf-51e29ede885so300676a12.3 for ; Wed, 12 Jul 2023 22:46:57 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=pantheon.tech; s=google; t=1689227217; x=1691819217; h=content-transfer-encoding:cc:to:subject:message-id:date:from :in-reply-to:references:mime-version:from:to:cc:subject:date :message-id:reply-to; bh=PYhm+3MA+uxRuLfK5QndXNLsZIzVMloV7HKjbhIfpLw=; b=JP2ol5/dDZtcviwjhEI3fW7yZ4dqrB5ElRK4yYmy0s4op19hfLN4dUAqJxnkB0RBP6 qWQ8HvCuFuC8Yi2DTZLUmUjCMIE28UjBj0zpCpggQQVFS0AVm7p3A2Q8mAVUXIn5d45u mbq4hECFvBDrQNCAzBQp0g0iNICLsa65Z6SJYuwxS8FmRhqaEcSQxu9CqOC/3Yr44ds0 Ve1XqG7fUH2zQJ8pacUlAWNhuamzptVy0rEhGmCEkOn656E5WXHIKaVW9puwWEfMolnx dnBbw+Jdc37NUOKvAL4AEhoYNW4lLCfbTEer69blf3iiCkw95i6WNnJs+EEbKd/uiPN6 Kw8A== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20221208; t=1689227217; x=1691819217; h=content-transfer-encoding:cc:to:subject:message-id:date:from :in-reply-to:references:mime-version:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=PYhm+3MA+uxRuLfK5QndXNLsZIzVMloV7HKjbhIfpLw=; b=S5WU9c7ygQ68xAGhWK0p5216QMe9A+GPC6JABvNJjHLFjQ7Rn3IHeDYNfiO/R+VzRJ 2f32KrR+oG14btVmdEut86dGWNw7qRaDbfXe1fwGV4EGilwuwDCoCO3Zpr4I4bWUTICu U2ruAqruAVF9ltBx1xM/TlUUlI2w2uTqhUDgOfdXapC9rp0wFmQTDR1pC2Q5g3t3CQeY jIAxxo4rFOv5wzGNYSuxgk1gW12hX8hZZ68s/Qis+C8SNsUz63XXFVf2XWz3ZkAnGb2P wUceUxAJ9WXEDX6H/XxcURXt326jyO2NgH6pBDAq24uexFJQzKwIltnFPTmpWD5B0lyT 2YwA== X-Gm-Message-State: ABy/qLbqdeZzuV+mkj//iaxnHFditZRTlUb0B4DKgJs6yRgxbQT52sWo ea7Bt0+VfWPxZrCMbzUtnWGG0OE1KQxu+iZ3ZIh4XQ== X-Google-Smtp-Source: APBJJlH9hW17kjCoBl2HIs6yWqer73TRhXnVVlyzN9wjqcPvRF21SvD1ZZGx0rAzveMmm2zZkvfZlITllqvZaP+vVko= X-Received: by 2002:a50:fc0b:0:b0:51d:961a:b27a with SMTP id i11-20020a50fc0b000000b0051d961ab27amr758422edr.7.1689227215891; Wed, 12 Jul 2023 22:46:55 -0700 (PDT) MIME-Version: 1.0 References: <20230712192005.2045-3-jspewock@iol.unh.edu> <20230712192005.2045-5-jspewock@iol.unh.edu> In-Reply-To: From: =?UTF-8?Q?Juraj_Linke=C5=A1?= Date: Thu, 13 Jul 2023 07:46:43 +0200 Message-ID: Subject: Re: [PATCH v5 1/2] dts: add smoke tests To: Jeremy Spewock Cc: Honnappa.Nagarahalli@arm.com, thomas@monjalon.net, lijuan.tu@intel.com, wathsala.vithanage@arm.com, probb@iol.unh.edu, dev@dpdk.org Content-Type: text/plain; charset="UTF-8" Content-Transfer-Encoding: quoted-printable X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org On Wed, Jul 12, 2023 at 10:00=E2=80=AFPM Jeremy Spewock wrote: > > This fails to apply because it modifies some of the same files that the p= revious DTS patches that were applied today also change. These changes only= modify things in dts/ which isn't currently being used in CI testing so th= ere wouldn't be any breaking changes for these builds regardless. Apologies= for my lack of knowledge on how this is handled, but are these conflicts s= omething that I need to fix in my patch or are instead handled by maintaine= rs who are applying and merging the patches on these release candidates? > Send a new version if you have the bandwidth, it will take some work off maintainers, who are very busy. > Thanks, > Jeremy > > On Wed, Jul 12, 2023 at 3:21=E2=80=AFPM wrote: >> >> From: Jeremy Spewock >> >> Adds a new test suite for running smoke tests that verify general >> configuration aspects of the system under test. If any of these tests >> fail, the DTS execution terminates as part of a "fail-fast" model. >> >> Signed-off-by: Jeremy Spewock >> --- >> dts/conf.yaml | 17 +- >> dts/framework/config/__init__.py | 105 +++++++++-- >> dts/framework/config/conf_yaml_schema.json | 142 +++++++++++++- >> dts/framework/dts.py | 87 ++++++--- >> dts/framework/exception.py | 12 ++ >> dts/framework/remote_session/__init__.py | 11 +- >> dts/framework/remote_session/os_session.py | 53 +++++- >> dts/framework/remote_session/posix_session.py | 29 ++- >> .../remote_session/remote/__init__.py | 10 + >> .../remote/interactive_remote_session.py | 82 ++++++++ >> .../remote/interactive_shell.py | 75 ++++++++ >> .../remote_session/remote/testpmd_shell.py | 75 ++++++++ >> dts/framework/test_result.py | 37 +++- >> dts/framework/test_suite.py | 10 +- >> dts/framework/testbed_model/node.py | 2 + >> dts/framework/testbed_model/sut_node.py | 176 +++++++++++++----- >> dts/framework/utils.py | 2 + >> dts/tests/TestSuite_smoke_tests.py | 113 +++++++++++ >> 18 files changed, 941 insertions(+), 97 deletions(-) >> create mode 100644 dts/framework/remote_session/remote/interactive_remo= te_session.py >> create mode 100644 dts/framework/remote_session/remote/interactive_shel= l.py >> create mode 100644 dts/framework/remote_session/remote/testpmd_shell.py >> create mode 100644 dts/tests/TestSuite_smoke_tests.py >> >> diff --git a/dts/conf.yaml b/dts/conf.yaml >> index a9bd8a3e..c0be7848 100644 >> --- a/dts/conf.yaml >> +++ b/dts/conf.yaml >> @@ -10,9 +10,13 @@ executions: >> compiler_wrapper: ccache >> perf: false >> func: true >> + skip_smoke_tests: false # optional flag that allow you to skip smok= e tests >> test_suites: >> - hello_world >> - system_under_test: "SUT 1" >> + system_under_test: >> + node_name: "SUT 1" >> + vdevs: # optional; if removed, vdevs won't be used in the executi= on >> + - "crypto_openssl" >> nodes: >> - name: "SUT 1" >> hostname: sut1.change.me.localhost >> @@ -25,3 +29,14 @@ nodes: >> hugepages: # optional; if removed, will use system hugepage config= uration >> amount: 256 >> force_first_numa: false >> + ports: >> + - pci: "0000:00:08.0" >> + os_driver_for_dpdk: vfio-pci # OS driver that DPDK will use >> + os_driver: i40e >> + peer_node: "TG 1" >> + peer_pci: "0000:00:08.0" >> + - pci: "0000:00:08.1" >> + os_driver_for_dpdk: vfio-pci >> + os_driver: i40e >> + peer_node: "TG 1" >> + peer_pci: "0000:00:08.1" >> diff --git a/dts/framework/config/__init__.py b/dts/framework/config/__i= nit__.py >> index ebb0823f..9e144748 100644 >> --- a/dts/framework/config/__init__.py >> +++ b/dts/framework/config/__init__.py >> @@ -12,6 +12,7 @@ >> import pathlib >> from dataclasses import dataclass >> from enum import Enum, auto, unique >> +from pathlib import PurePath >> from typing import Any, TypedDict >> >> import warlock # type: ignore >> @@ -72,6 +73,20 @@ class HugepageConfiguration: >> force_first_numa: bool >> >> >> +@dataclass(slots=3DTrue, frozen=3DTrue) >> +class PortConfig: >> + node: str >> + pci: str >> + os_driver_for_dpdk: str >> + os_driver: str >> + peer_node: str >> + peer_pci: str >> + >> + @staticmethod >> + def from_dict(node: str, d: dict) -> "PortConfig": >> + return PortConfig(node=3Dnode, **d) >> + >> + >> @dataclass(slots=3DTrue, frozen=3DTrue) >> class NodeConfiguration: >> name: str >> @@ -84,6 +99,7 @@ class NodeConfiguration: >> use_first_core: bool >> memory_channels: int >> hugepages: HugepageConfiguration | None >> + ports: list[PortConfig] >> >> @staticmethod >> def from_dict(d: dict) -> "NodeConfiguration": >> @@ -92,19 +108,36 @@ def from_dict(d: dict) -> "NodeConfiguration": >> if "force_first_numa" not in hugepage_config: >> hugepage_config["force_first_numa"] =3D False >> hugepage_config =3D HugepageConfiguration(**hugepage_config= ) >> + common_config =3D { >> + "name": d["name"], >> + "hostname": d["hostname"], >> + "user": d["user"], >> + "password": d.get("password"), >> + "arch": Architecture(d["arch"]), >> + "os": OS(d["os"]), >> + "lcores": d.get("lcores", "1"), >> + "use_first_core": d.get("use_first_core", False), >> + "memory_channels": d.get("memory_channels", 1), >> + "hugepages": hugepage_config, >> + "ports": [PortConfig.from_dict(d["name"], port) for port in= d["ports"]], >> + } >> + >> + return NodeConfiguration(**common_config) >> >> - return NodeConfiguration( >> - name=3Dd["name"], >> - hostname=3Dd["hostname"], >> - user=3Dd["user"], >> - password=3Dd.get("password"), >> - arch=3DArchitecture(d["arch"]), >> - os=3DOS(d["os"]), >> - lcores=3Dd.get("lcores", "1"), >> - use_first_core=3Dd.get("use_first_core", False), >> - memory_channels=3Dd.get("memory_channels", 1), >> - hugepages=3Dhugepage_config, >> - ) >> + >> +@dataclass(slots=3DTrue, frozen=3DTrue) >> +class NodeInfo: >> + """Class to hold important versions within the node. >> + >> + This class, unlike the NodeConfiguration class, cannot be generated= at the start. >> + This is because we need to initialize a connection with the node be= fore we can >> + collect the information needed in this class. Therefore, it cannot = be a part of >> + the configuration class above. >> + """ >> + >> + os_name: str >> + os_version: str >> + kernel_version: str >> >> >> @dataclass(slots=3DTrue, frozen=3DTrue) >> @@ -128,6 +161,18 @@ def from_dict(d: dict) -> "BuildTargetConfiguration= ": >> ) >> >> >> +@dataclass(slots=3DTrue, frozen=3DTrue) >> +class BuildTargetInfo: >> + """Class to hold important versions within the build target. >> + >> + This is very similar to the NodeInfo class, it just instead holds i= nformation >> + for the build target. >> + """ >> + >> + dpdk_version: str >> + compiler_version: str >> + >> + >> class TestSuiteConfigDict(TypedDict): >> suite: str >> cases: list[str] >> @@ -157,6 +202,8 @@ class ExecutionConfiguration: >> func: bool >> test_suites: list[TestSuiteConfig] >> system_under_test: NodeConfiguration >> + vdevs: list[str] >> + skip_smoke_tests: bool >> >> @staticmethod >> def from_dict(d: dict, node_map: dict) -> "ExecutionConfiguration": >> @@ -166,15 +213,20 @@ def from_dict(d: dict, node_map: dict) -> "Executi= onConfiguration": >> test_suites: list[TestSuiteConfig] =3D list( >> map(TestSuiteConfig.from_dict, d["test_suites"]) >> ) >> - sut_name =3D d["system_under_test"] >> + sut_name =3D d["system_under_test"]["node_name"] >> + skip_smoke_tests =3D d.get("skip_smoke_tests", False) >> assert sut_name in node_map, f"Unknown SUT {sut_name} in execut= ion {d}" >> - >> + vdevs =3D ( >> + d["system_under_test"]["vdevs"] if "vdevs" in d["system_und= er_test"] else [] >> + ) >> return ExecutionConfiguration( >> build_targets=3Dbuild_targets, >> perf=3Dd["perf"], >> func=3Dd["func"], >> + skip_smoke_tests=3Dskip_smoke_tests, >> test_suites=3Dtest_suites, >> system_under_test=3Dnode_map[sut_name], >> + vdevs=3Dvdevs, >> ) >> >> >> @@ -221,3 +273,28 @@ def load_config() -> Configuration: >> >> >> CONFIGURATION =3D load_config() >> + >> + >> +@unique >> +class InteractiveApp(Enum): >> + """An enum that represents different supported interactive applicat= ions. >> + >> + The values in this enum must all be set to objects that have a key = called >> + "default_path" where "default_path" represents a PurePath object fo= r the path >> + to the application. This default path will be passed into the handl= er class >> + for the application so that it can start the application. >> + """ >> + >> + testpmd =3D {"default_path": PurePath("app", "dpdk-testpmd")} >> + >> + @property >> + def path(self) -> PurePath: >> + """Default path of the application. >> + >> + For DPDK apps, this will be appended to the DPDK build director= y. >> + """ >> + return self.value["default_path"] >> + >> + @path.setter >> + def path(self, path: PurePath) -> None: >> + self.value["default_path"] =3D path >> diff --git a/dts/framework/config/conf_yaml_schema.json b/dts/framework/= config/conf_yaml_schema.json >> index ca2d4a1e..61f52b43 100644 >> --- a/dts/framework/config/conf_yaml_schema.json >> +++ b/dts/framework/config/conf_yaml_schema.json >> @@ -6,6 +6,76 @@ >> "type": "string", >> "description": "A unique identifier for a node" >> }, >> + "NIC": { >> + "type": "string", >> + "enum": [ >> + "ALL", >> + "ConnectX3_MT4103", >> + "ConnectX4_LX_MT4117", >> + "ConnectX4_MT4115", >> + "ConnectX5_MT4119", >> + "ConnectX5_MT4121", >> + "I40E_10G-10G_BASE_T_BC", >> + "I40E_10G-10G_BASE_T_X722", >> + "I40E_10G-SFP_X722", >> + "I40E_10G-SFP_XL710", >> + "I40E_10G-X722_A0", >> + "I40E_1G-1G_BASE_T_X722", >> + "I40E_25G-25G_SFP28", >> + "I40E_40G-QSFP_A", >> + "I40E_40G-QSFP_B", >> + "IAVF-ADAPTIVE_VF", >> + "IAVF-VF", >> + "IAVF_10G-X722_VF", >> + "ICE_100G-E810C_QSFP", >> + "ICE_25G-E810C_SFP", >> + "ICE_25G-E810_XXV_SFP", >> + "IGB-I350_VF", >> + "IGB_1G-82540EM", >> + "IGB_1G-82545EM_COPPER", >> + "IGB_1G-82571EB_COPPER", >> + "IGB_1G-82574L", >> + "IGB_1G-82576", >> + "IGB_1G-82576_QUAD_COPPER", >> + "IGB_1G-82576_QUAD_COPPER_ET2", >> + "IGB_1G-82580_COPPER", >> + "IGB_1G-I210_COPPER", >> + "IGB_1G-I350_COPPER", >> + "IGB_1G-I354_SGMII", >> + "IGB_1G-PCH_LPTLP_I218_LM", >> + "IGB_1G-PCH_LPTLP_I218_V", >> + "IGB_1G-PCH_LPT_I217_LM", >> + "IGB_1G-PCH_LPT_I217_V", >> + "IGB_2.5G-I354_BACKPLANE_2_5GBPS", >> + "IGC-I225_LM", >> + "IGC-I226_LM", >> + "IXGBE_10G-82599_SFP", >> + "IXGBE_10G-82599_SFP_SF_QP", >> + "IXGBE_10G-82599_T3_LOM", >> + "IXGBE_10G-82599_VF", >> + "IXGBE_10G-X540T", >> + "IXGBE_10G-X540_VF", >> + "IXGBE_10G-X550EM_A_SFP", >> + "IXGBE_10G-X550EM_X_10G_T", >> + "IXGBE_10G-X550EM_X_SFP", >> + "IXGBE_10G-X550EM_X_VF", >> + "IXGBE_10G-X550T", >> + "IXGBE_10G-X550_VF", >> + "brcm_57414", >> + "brcm_P2100G", >> + "cavium_0011", >> + "cavium_a034", >> + "cavium_a063", >> + "cavium_a064", >> + "fastlinq_ql41000", >> + "fastlinq_ql41000_vf", >> + "fastlinq_ql45000", >> + "fastlinq_ql45000_vf", >> + "hi1822", >> + "virtio" >> + ] >> + }, >> + >> "ARCH": { >> "type": "string", >> "enum": [ >> @@ -94,6 +164,19 @@ >> "amount" >> ] >> }, >> + "pci_address": { >> + "type": "string", >> + "pattern": "^[\\da-fA-F]{4}:[\\da-fA-F]{2}:[\\da-fA-F]{2}.\\d:?\\= w*$" >> + }, >> + "port_peer_address": { >> + "description": "Peer is a TRex port, and IXIA port or a PCI addre= ss", >> + "oneOf": [ >> + { >> + "description": "PCI peer port", >> + "$ref": "#/definitions/pci_address" >> + } >> + ] >> + }, >> "test_suite": { >> "type": "string", >> "enum": [ >> @@ -165,6 +248,44 @@ >> }, >> "hugepages": { >> "$ref": "#/definitions/hugepages" >> + }, >> + "ports": { >> + "type": "array", >> + "items": { >> + "type": "object", >> + "description": "Each port should be described on both sid= es of the connection. This makes configuration slightly more verbose but gr= eatly simplifies implementation. If there are an inconsistencies, then DTS = will not run until that issue is fixed. An example inconsistency would be p= ort 1, node 1 says it is connected to port 1, node 2, but port 1, node 2 sa= ys it is connected to port 2, node 1.", >> + "properties": { >> + "pci": { >> + "$ref": "#/definitions/pci_address", >> + "description": "The local PCI address of the port" >> + }, >> + "os_driver_for_dpdk": { >> + "type": "string", >> + "description": "The driver that the kernel should bin= d this device to for DPDK to use it. (ex: vfio-pci)" >> + }, >> + "os_driver": { >> + "type": "string", >> + "description": "The driver normally used by this port= (ex: i40e)" >> + }, >> + "peer_node": { >> + "type": "string", >> + "description": "The name of the node the peer port is= on" >> + }, >> + "peer_pci": { >> + "$ref": "#/definitions/pci_address", >> + "description": "The PCI address of the peer port" >> + } >> + }, >> + "additionalProperties": false, >> + "required": [ >> + "pci", >> + "os_driver_for_dpdk", >> + "os_driver", >> + "peer_node", >> + "peer_pci" >> + ] >> + }, >> + "minimum": 1 >> } >> }, >> "additionalProperties": false, >> @@ -211,8 +332,27 @@ >> ] >> } >> }, >> + "skip_smoke_tests": { >> + "description": "Optional field that allows you to skip smok= e testing", >> + "type": "boolean" >> + }, >> "system_under_test": { >> - "$ref": "#/definitions/node_name" >> + "type":"object", >> + "properties": { >> + "node_name": { >> + "$ref": "#/definitions/node_name" >> + }, >> + "vdevs": { >> + "description": "Opentional list of names of vdevs to be= used in execution", >> + "type": "array", >> + "items": { >> + "type": "string" >> + } >> + } >> + }, >> + "required": [ >> + "node_name" >> + ] >> } >> }, >> "additionalProperties": false, >> diff --git a/dts/framework/dts.py b/dts/framework/dts.py >> index 05022845..7b09d8fb 100644 >> --- a/dts/framework/dts.py >> +++ b/dts/framework/dts.py >> @@ -5,7 +5,13 @@ >> >> import sys >> >> -from .config import CONFIGURATION, BuildTargetConfiguration, ExecutionC= onfiguration >> +from .config import ( >> + CONFIGURATION, >> + BuildTargetConfiguration, >> + ExecutionConfiguration, >> + TestSuiteConfig, >> +) >> +from .exception import BlockingTestSuiteError >> from .logger import DTSLOG, getLogger >> from .test_result import BuildTargetResult, DTSResult, ExecutionResult,= Result >> from .test_suite import get_test_suites >> @@ -82,7 +88,7 @@ def _run_execution( >> running all build targets in the given execution. >> """ >> dts_logger.info(f"Running execution with SUT '{execution.system_und= er_test.name}'.") >> - execution_result =3D result.add_execution(sut_node.config) >> + execution_result =3D result.add_execution(sut_node.config, sut_node= .node_info) >> >> try: >> sut_node.set_up_execution(execution) >> @@ -118,14 +124,15 @@ def _run_build_target( >> >> try: >> sut_node.set_up_build_target(build_target) >> - result.dpdk_version =3D sut_node.dpdk_version >> + # result.dpdk_version =3D sut_node.dpdk_version >> + build_target_result.add_build_target_versions(sut_node.get_buil= d_target_info()) >> build_target_result.update_setup(Result.PASS) >> except Exception as e: >> dts_logger.exception("Build target setup failed.") >> build_target_result.update_setup(Result.FAIL, e) >> >> else: >> - _run_suites(sut_node, execution, build_target_result) >> + _run_all_suites(sut_node, execution, build_target_result) >> >> finally: >> try: >> @@ -136,7 +143,7 @@ def _run_build_target( >> build_target_result.update_teardown(Result.FAIL, e) >> >> >> -def _run_suites( >> +def _run_all_suites( >> sut_node: SutNode, >> execution: ExecutionConfiguration, >> build_target_result: BuildTargetResult, >> @@ -146,27 +153,61 @@ def _run_suites( >> with possibly only a subset of test cases. >> If no subset is specified, run all test cases. >> """ >> + end_build_target =3D False >> + if not execution.skip_smoke_tests: >> + execution.test_suites[:0] =3D [TestSuiteConfig.from_dict("smoke= _tests")] >> for test_suite_config in execution.test_suites: >> try: >> - full_suite_path =3D f"tests.TestSuite_{test_suite_config.te= st_suite}" >> - test_suite_classes =3D get_test_suites(full_suite_path) >> - suites_str =3D ", ".join((x.__name__ for x in test_suite_cl= asses)) >> - dts_logger.debug( >> - f"Found test suites '{suites_str}' in '{full_suite_path= }'." >> + _run_single_suite( >> + sut_node, execution, build_target_result, test_suite_co= nfig >> ) >> - except Exception as e: >> - dts_logger.exception("An error occurred when searching for = test suites.") >> - result.update_setup(Result.ERROR, e) >> - >> - else: >> - for test_suite_class in test_suite_classes: >> - test_suite =3D test_suite_class( >> - sut_node, >> - test_suite_config.test_cases, >> - execution.func, >> - build_target_result, >> - ) >> - test_suite.run() >> + except BlockingTestSuiteError as e: >> + dts_logger.exception( >> + f"An error occurred within {test_suite_config.test_suit= e}. " >> + "Skipping build target..." >> + ) >> + result.add_error(e) >> + end_build_target =3D True >> + # if a blocking test failed and we need to bail out of suite ex= ecutions >> + if end_build_target: >> + break >> + >> + >> +def _run_single_suite( >> + sut_node: SutNode, >> + execution: ExecutionConfiguration, >> + build_target_result: BuildTargetResult, >> + test_suite_config: TestSuiteConfig, >> +) -> None: >> + """Runs a single test suite. >> + >> + Args: >> + sut_node: Node to run tests on. >> + execution: Execution the test case belongs to. >> + build_target_result: Build target configuration test case is ru= n on >> + test_suite_config: Test suite configuration >> + >> + Raises: >> + BlockingTestSuiteError: If a test suite that was marked as bloc= king fails. >> + """ >> + try: >> + full_suite_path =3D f"tests.TestSuite_{test_suite_config.test_s= uite}" >> + test_suite_classes =3D get_test_suites(full_suite_path) >> + suites_str =3D ", ".join((x.__name__ for x in test_suite_classe= s)) >> + dts_logger.debug(f"Found test suites '{suites_str}' in '{full_s= uite_path}'.") >> + except Exception as e: >> + dts_logger.exception("An error occurred when searching for test= suites.") >> + result.update_setup(Result.ERROR, e) >> + >> + else: >> + for test_suite_class in test_suite_classes: >> + test_suite =3D test_suite_class( >> + sut_node, >> + test_suite_config.test_cases, >> + execution.func, >> + build_target_result, >> + ) >> + test_suite.run() >> >> >> def _exit_dts() -> None: >> diff --git a/dts/framework/exception.py b/dts/framework/exception.py >> index ca353d98..dfb12df4 100644 >> --- a/dts/framework/exception.py >> +++ b/dts/framework/exception.py >> @@ -25,6 +25,7 @@ class ErrorSeverity(IntEnum): >> SSH_ERR =3D 4 >> DPDK_BUILD_ERR =3D 10 >> TESTCASE_VERIFY_ERR =3D 20 >> + BLOCKING_TESTSUITE_ERR =3D 25 >> >> >> class DTSError(Exception): >> @@ -144,3 +145,14 @@ def __init__(self, value: str): >> >> def __str__(self) -> str: >> return repr(self.value) >> + >> + >> +class BlockingTestSuiteError(DTSError): >> + suite_name: str >> + severity: ClassVar[ErrorSeverity] =3D ErrorSeverity.BLOCKING_TESTSU= ITE_ERR >> + >> + def __init__(self, suite_name: str) -> None: >> + self.suite_name =3D suite_name >> + >> + def __str__(self) -> str: >> + return f"Blocking suite {self.suite_name} failed." >> diff --git a/dts/framework/remote_session/__init__.py b/dts/framework/re= mote_session/__init__.py >> index ee221503..2c408c25 100644 >> --- a/dts/framework/remote_session/__init__.py >> +++ b/dts/framework/remote_session/__init__.py >> @@ -1,5 +1,6 @@ >> # SPDX-License-Identifier: BSD-3-Clause >> # Copyright(c) 2023 PANTHEON.tech s.r.o. >> +# Copyright(c) 2023 University of New Hampshire >> >> """ >> The package provides modules for managing remote connections to a remot= e host (node), >> @@ -17,7 +18,15 @@ >> >> from .linux_session import LinuxSession >> from .os_session import OSSession >> -from .remote import CommandResult, RemoteSession, SSHSession >> +from .remote import ( >> + CommandResult, >> + InteractiveRemoteSession, >> + InteractiveShell, >> + RemoteSession, >> + SSHSession, >> + TestPmdDevice, >> + TestPmdShell, >> +) >> >> >> def create_session( >> diff --git a/dts/framework/remote_session/os_session.py b/dts/framework/= remote_session/os_session.py >> index 4c48ae25..48f05c44 100644 >> --- a/dts/framework/remote_session/os_session.py >> +++ b/dts/framework/remote_session/os_session.py >> @@ -5,14 +5,22 @@ >> from abc import ABC, abstractmethod >> from collections.abc import Iterable >> from pathlib import PurePath >> +from typing import Union >> >> -from framework.config import Architecture, NodeConfiguration >> +from framework.config import Architecture, InteractiveApp, NodeConfigur= ation, NodeInfo >> from framework.logger import DTSLOG >> +from framework.remote_session.remote import InteractiveShell, TestPmdSh= ell >> from framework.settings import SETTINGS >> from framework.testbed_model import LogicalCore >> from framework.utils import EnvVarsDict, MesonArgs >> >> -from .remote import CommandResult, RemoteSession, create_remote_session >> +from .remote import ( >> + CommandResult, >> + InteractiveRemoteSession, >> + RemoteSession, >> + create_interactive_session, >> + create_remote_session, >> +) >> >> >> class OSSession(ABC): >> @@ -26,6 +34,7 @@ class OSSession(ABC): >> name: str >> _logger: DTSLOG >> remote_session: RemoteSession >> + interactive_session: InteractiveRemoteSession >> >> def __init__( >> self, >> @@ -37,6 +46,7 @@ def __init__( >> self.name =3D name >> self._logger =3D logger >> self.remote_session =3D create_remote_session(node_config, name= , logger) >> + self.interactive_session =3D create_interactive_session(node_co= nfig, name, logger) >> >> def close(self, force: bool =3D False) -> None: >> """ >> @@ -64,6 +74,33 @@ def send_command( >> """ >> return self.remote_session.send_command(command, timeout, verif= y, env) >> >> + def create_interactive_shell( >> + self, >> + shell_type: InteractiveApp, >> + path_to_app: PurePath, >> + eal_parameters: str, >> + timeout: float, >> + ) -> Union[InteractiveShell, TestPmdShell]: >> + """ >> + See "create_interactive_shell" in SutNode >> + """ >> + match (shell_type): >> + case InteractiveApp.testpmd: >> + return TestPmdShell( >> + self.interactive_session.session, >> + self._logger, >> + path_to_app, >> + timeout=3Dtimeout, >> + eal_flags=3Deal_parameters, >> + ) >> + case _: >> + self._logger.info( >> + f"Unhandled app type {shell_type.name}, defaulting = to shell." >> + ) >> + return InteractiveShell( >> + self.interactive_session.session, self._logger, pat= h_to_app, timeout >> + ) >> + >> @abstractmethod >> def guess_dpdk_remote_dir(self, remote_dir) -> PurePath: >> """ >> @@ -173,3 +210,15 @@ def setup_hugepages(self, hugepage_amount: int, for= ce_first_numa: bool) -> None: >> if needed and mount the hugepages if needed. >> If force_first_numa is True, configure hugepages just on the fi= rst socket. >> """ >> + >> + @abstractmethod >> + def get_compiler_version(self, compiler_name: str) -> str: >> + """ >> + Get installed version of compiler used for DPDK >> + """ >> + >> + @abstractmethod >> + def get_node_info(self) -> NodeInfo: >> + """ >> + Collect information about the node >> + """ >> diff --git a/dts/framework/remote_session/posix_session.py b/dts/framewo= rk/remote_session/posix_session.py >> index d38062e8..4e419877 100644 >> --- a/dts/framework/remote_session/posix_session.py >> +++ b/dts/framework/remote_session/posix_session.py >> @@ -6,7 +6,7 @@ >> from collections.abc import Iterable >> from pathlib import PurePath, PurePosixPath >> >> -from framework.config import Architecture >> +from framework.config import Architecture, NodeInfo >> from framework.exception import DPDKBuildError, RemoteCommandExecutionE= rror >> from framework.settings import SETTINGS >> from framework.utils import EnvVarsDict, MesonArgs >> @@ -219,3 +219,30 @@ def _remove_dpdk_runtime_dirs( >> >> def get_dpdk_file_prefix(self, dpdk_prefix) -> str: >> return "" >> + >> + def get_compiler_version(self, compiler_name: str) -> str: >> + match compiler_name: >> + case "gcc": >> + return self.send_command( >> + f"{compiler_name} --version", SETTINGS.timeout >> + ).stdout.split("\n")[0] >> + case "clang": >> + return self.send_command( >> + f"{compiler_name} --version", SETTINGS.timeout >> + ).stdout.split("\n")[0] >> + case "msvc": >> + return self.send_command("cl", SETTINGS.timeout).stdout >> + case "icc": >> + return self.send_command(f"{compiler_name} -V", SETTING= S.timeout).stdout >> + case _: >> + raise ValueError(f"Unknown compiler {compiler_name}") >> + >> + def get_node_info(self) -> NodeInfo: >> + os_release_info =3D self.send_command( >> + "awk -F=3D '$1 ~ /^NAME$|^VERSION$/ {print $2}' /etc/os-rel= ease", >> + SETTINGS.timeout, >> + ).stdout.split("\n") >> + kernel_version =3D self.send_command("uname -r", SETTINGS.timeo= ut).stdout >> + return NodeInfo( >> + os_release_info[0].strip(), os_release_info[1].strip(), ker= nel_version >> + ) >> diff --git a/dts/framework/remote_session/remote/__init__.py b/dts/frame= work/remote_session/remote/__init__.py >> index 8a151221..03fd309f 100644 >> --- a/dts/framework/remote_session/remote/__init__.py >> +++ b/dts/framework/remote_session/remote/__init__.py >> @@ -1,16 +1,26 @@ >> # SPDX-License-Identifier: BSD-3-Clause >> # Copyright(c) 2023 PANTHEON.tech s.r.o. >> +# Copyright(c) 2023 University of New Hampshire >> >> # pylama:ignore=3DW0611 >> >> from framework.config import NodeConfiguration >> from framework.logger import DTSLOG >> >> +from .interactive_remote_session import InteractiveRemoteSession >> +from .interactive_shell import InteractiveShell >> from .remote_session import CommandResult, RemoteSession >> from .ssh_session import SSHSession >> +from .testpmd_shell import TestPmdDevice, TestPmdShell >> >> >> def create_remote_session( >> node_config: NodeConfiguration, name: str, logger: DTSLOG >> ) -> RemoteSession: >> return SSHSession(node_config, name, logger) >> + >> + >> +def create_interactive_session( >> + node_config: NodeConfiguration, name: str, logger: DTSLOG >> +) -> InteractiveRemoteSession: >> + return InteractiveRemoteSession(node_config, logger) >> diff --git a/dts/framework/remote_session/remote/interactive_remote_sess= ion.py b/dts/framework/remote_session/remote/interactive_remote_session.py >> new file mode 100644 >> index 00000000..2d94daf2 >> --- /dev/null >> +++ b/dts/framework/remote_session/remote/interactive_remote_session.py >> @@ -0,0 +1,82 @@ >> +# SPDX-License-Identifier: BSD-3-Clause >> +# Copyright(c) 2023 University of New Hampshire >> + >> +import socket >> +import traceback >> + >> +from paramiko import AutoAddPolicy, SSHClient, Transport # type: ignor= e >> +from paramiko.ssh_exception import ( # type: ignore >> + AuthenticationException, >> + BadHostKeyException, >> + NoValidConnectionsError, >> + SSHException, >> +) >> + >> +from framework.config import NodeConfiguration >> +from framework.exception import SSHConnectionError >> +from framework.logger import DTSLOG >> + >> + >> +class InteractiveRemoteSession: >> + hostname: str >> + ip: str >> + port: int >> + username: str >> + password: str >> + _logger: DTSLOG >> + _node_config: NodeConfiguration >> + session: SSHClient >> + _transport: Transport | None >> + >> + def __init__(self, node_config: NodeConfiguration, _logger: DTSLOG)= -> None: >> + self._node_config =3D node_config >> + self._logger =3D _logger >> + self.hostname =3D node_config.hostname >> + self.username =3D node_config.user >> + self.password =3D node_config.password if node_config.password = else "" >> + port =3D "22" >> + self.ip =3D node_config.hostname >> + if ":" in node_config.hostname: >> + self.ip, port =3D node_config.hostname.split(":") >> + self.port =3D int(port) >> + self._logger.info( >> + f"Initializing interactive connection for {self.username}@{= self.hostname}" >> + ) >> + self._connect() >> + self._logger.info( >> + f"Interactive connection successful for {self.username}@{se= lf.hostname}" >> + ) >> + >> + def _connect(self) -> None: >> + client =3D SSHClient() >> + client.set_missing_host_key_policy(AutoAddPolicy) >> + self.session =3D client >> + retry_attempts =3D 10 >> + for retry_attempt in range(retry_attempts): >> + try: >> + client.connect( >> + self.ip, >> + username=3Dself.username, >> + port=3Dself.port, >> + password=3Dself.password, >> + timeout=3D20 if self.port else 10, >> + ) >> + except (TypeError, BadHostKeyException, AuthenticationExcep= tion) as e: >> + self._logger.exception(e) >> + raise SSHConnectionError(self.hostname) from e >> + except (NoValidConnectionsError, socket.error, SSHException= ) as e: >> + self._logger.debug(traceback.format_exc()) >> + self._logger.warning(e) >> + self._logger.info( >> + "Retrying interactive session connection: " >> + f"retry number {retry_attempt +1}" >> + ) >> + else: >> + break >> + else: >> + raise SSHConnectionError(self.hostname) >> + # Interactive sessions are used on an "as needed" basis so we h= ave >> + # to set a keepalive >> + self._transport =3D self.session.get_transport() >> + if self._transport is not None: >> + self._transport.set_keepalive(30) >> diff --git a/dts/framework/remote_session/remote/interactive_shell.py b/= dts/framework/remote_session/remote/interactive_shell.py >> new file mode 100644 >> index 00000000..0cb0f7b4 >> --- /dev/null >> +++ b/dts/framework/remote_session/remote/interactive_shell.py >> @@ -0,0 +1,75 @@ >> +from pathlib import PurePath >> + >> +from paramiko import Channel, SSHClient, channel # type: ignore >> + >> +from framework.logger import DTSLOG >> +from framework.settings import SETTINGS >> + >> + >> +class InteractiveShell: >> + >> + _interactive_session: SSHClient >> + _stdin: channel.ChannelStdinFile >> + _stdout: channel.ChannelFile >> + _ssh_channel: Channel >> + _logger: DTSLOG >> + _timeout: float >> + _path_to_app: PurePath >> + >> + def __init__( >> + self, >> + interactive_session: SSHClient, >> + logger: DTSLOG, >> + path_to_app: PurePath, >> + timeout: float =3D SETTINGS.timeout, >> + ) -> None: >> + self._interactive_session =3D interactive_session >> + self._ssh_channel =3D self._interactive_session.invoke_shell() >> + self._stdin =3D self._ssh_channel.makefile_stdin("w") >> + self._stdout =3D self._ssh_channel.makefile("r") >> + self._ssh_channel.settimeout(timeout) >> + self._ssh_channel.set_combine_stderr(True) # combines stdout a= nd stderr streams >> + self._logger =3D logger >> + self._timeout =3D timeout >> + self._path_to_app =3D path_to_app >> + self._start_application() >> + >> + def _start_application(self) -> None: >> + """Starts a new interactive application based on _path_to_app. >> + >> + This method is often overridden by subclasses as their process = for >> + starting may look different. >> + """ >> + self.send_command_get_output(f"{self._path_to_app}", "") >> + >> + def send_command_get_output(self, command: str, prompt: str) -> str= : >> + """Send a command and get all output before the expected ending= string. >> + >> + Lines that expect input are not included in the stdout buffer s= o they cannot be >> + used for expect. For example, if you were prompted to log into = something >> + with a username and password, you cannot expect "username:" bec= ause it won't >> + yet be in the stdout buffer. A work around for this could be co= nsuming an >> + extra newline character to force the current prompt into the st= dout buffer. >> + >> + Returns: >> + All output in the buffer before expected string >> + """ >> + self._logger.info(f"Sending command {command.strip()}...") >> + self._stdin.write(f"{command}\n") >> + self._stdin.flush() >> + out: str =3D "" >> + for line in self._stdout: >> + out +=3D line >> + if prompt in line and not line.rstrip().endswith( >> + command.rstrip() >> + ): # ignore line that sent command >> + break >> + self._logger.debug(f"Got output: {out}") >> + return out >> + >> + def close(self) -> None: >> + self._stdin.close() >> + self._ssh_channel.close() >> + >> + def __del__(self) -> None: >> + self.close() >> diff --git a/dts/framework/remote_session/remote/testpmd_shell.py b/dts/= framework/remote_session/remote/testpmd_shell.py >> new file mode 100644 >> index 00000000..31c25258 >> --- /dev/null >> +++ b/dts/framework/remote_session/remote/testpmd_shell.py >> @@ -0,0 +1,75 @@ >> +# SPDX-License-Identifier: BSD-3-Clause >> +# Copyright(c) 2023 University of New Hampshire >> + >> + >> +from pathlib import PurePath >> + >> +from paramiko import SSHClient # type: ignore >> + >> +from framework.logger import DTSLOG >> +from framework.settings import SETTINGS >> + >> +from .interactive_shell import InteractiveShell >> + >> + >> +class TestPmdDevice(object): >> + pci_address: str >> + >> + def __init__(self, pci_address: str): >> + self.pci_address =3D pci_address >> + >> + def __str__(self) -> str: >> + return self.pci_address >> + >> + >> +class TestPmdShell(InteractiveShell): >> + expected_prompt: str =3D "testpmd>" >> + _eal_flags: str >> + >> + def __init__( >> + self, >> + interactive_session: SSHClient, >> + logger: DTSLOG, >> + path_to_testpmd: PurePath, >> + eal_flags: str, >> + timeout: float =3D SETTINGS.timeout, >> + ) -> None: >> + """Initializes an interactive testpmd session using specified p= arameters.""" >> + self._eal_flags =3D eal_flags >> + >> + super(TestPmdShell, self).__init__( >> + interactive_session, >> + logger=3Dlogger, >> + path_to_app=3Dpath_to_testpmd, >> + timeout=3Dtimeout, >> + ) >> + >> + def _start_application(self) -> None: >> + """Starts a new interactive testpmd shell using _path_to_app.""= " >> + self.send_command( >> + f"{self._path_to_app} {self._eal_flags} -- -i", >> + ) >> + >> + def send_command(self, command: str, prompt: str =3D expected_promp= t) -> str: >> + """Specific way of handling the command for testpmd >> + >> + An extra newline character is consumed in order to force the cu= rrent line into >> + the stdout buffer. >> + """ >> + return self.send_command_get_output(f"{command}\n", prompt) >> + >> + def get_devices(self) -> list[TestPmdDevice]: >> + """Get a list of device names that are known to testpmd >> + >> + Uses the device info listed in testpmd and then parses the outp= ut to >> + return only the names of the devices. >> + >> + Returns: >> + A list of strings representing device names (e.g. 0000:14:0= 0.1) >> + """ >> + dev_info: str =3D self.send_command("show device info all") >> + dev_list: list[TestPmdDevice] =3D [] >> + for line in dev_info.split("\n"): >> + if "device name:" in line.lower(): >> + dev_list.append(TestPmdDevice(line.strip().split(": ")[= 1].strip())) >> + return dev_list >> diff --git a/dts/framework/test_result.py b/dts/framework/test_result.py >> index 74391982..fe199467 100644 >> --- a/dts/framework/test_result.py >> +++ b/dts/framework/test_result.py >> @@ -1,5 +1,6 @@ >> # SPDX-License-Identifier: BSD-3-Clause >> # Copyright(c) 2023 PANTHEON.tech s.r.o. >> +# Copyright(c) 2023 University of New Hampshire >> >> """ >> Generic result container and reporters >> @@ -13,9 +14,11 @@ >> OS, >> Architecture, >> BuildTargetConfiguration, >> + BuildTargetInfo, >> Compiler, >> CPUType, >> NodeConfiguration, >> + NodeInfo, >> ) >> from .exception import DTSError, ErrorSeverity >> from .logger import DTSLOG >> @@ -67,12 +70,14 @@ class Statistics(dict): >> Using a dict provides a convenient way to format the data. >> """ >> >> - def __init__(self, dpdk_version): >> + def __init__(self, output_info: dict[str, str] | None): >> super(Statistics, self).__init__() >> for result in Result: >> self[result.name] =3D 0 >> self["PASS RATE"] =3D 0.0 >> - self["DPDK VERSION"] =3D dpdk_version >> + if output_info: >> + for info_key, info_val in output_info.items(): >> + self[info_key] =3D info_val >> >> def __iadd__(self, other: Result) -> "Statistics": >> """ >> @@ -206,6 +211,8 @@ class BuildTargetResult(BaseResult): >> os: OS >> cpu: CPUType >> compiler: Compiler >> + compiler_version: str | None >> + dpdk_version: str | None >> >> def __init__(self, build_target: BuildTargetConfiguration): >> super(BuildTargetResult, self).__init__() >> @@ -213,6 +220,12 @@ def __init__(self, build_target: BuildTargetConfigu= ration): >> self.os =3D build_target.os >> self.cpu =3D build_target.cpu >> self.compiler =3D build_target.compiler >> + self.compiler_version =3D None >> + self.dpdk_version =3D None >> + >> + def add_build_target_versions(self, versions: BuildTargetInfo) -> N= one: >> + self.compiler_version =3D versions.compiler_version >> + self.dpdk_version =3D versions.dpdk_version >> >> def add_test_suite(self, test_suite_name: str) -> TestSuiteResult: >> test_suite_result =3D TestSuiteResult(test_suite_name) >> @@ -228,10 +241,17 @@ class ExecutionResult(BaseResult): >> """ >> >> sut_node: NodeConfiguration >> + sut_os_name: str >> + sut_os_version: str >> + sut_kernel_version: str >> >> - def __init__(self, sut_node: NodeConfiguration): >> + def __init__(self, sut_node: NodeConfiguration, sut_version_info: N= odeInfo): >> super(ExecutionResult, self).__init__() >> self.sut_node =3D sut_node >> + self.sut_version_info =3D sut_version_info >> + self.sut_os_name =3D sut_version_info.os_name >> + self.sut_os_version =3D sut_version_info.os_version >> + self.sut_kernel_version =3D sut_version_info.kernel_version >> >> def add_build_target( >> self, build_target: BuildTargetConfiguration >> @@ -258,6 +278,7 @@ class DTSResult(BaseResult): >> """ >> >> dpdk_version: str | None >> + output: dict | None >> _logger: DTSLOG >> _errors: list[Exception] >> _return_code: ErrorSeverity >> @@ -267,14 +288,17 @@ class DTSResult(BaseResult): >> def __init__(self, logger: DTSLOG): >> super(DTSResult, self).__init__() >> self.dpdk_version =3D None >> + self.output =3D None >> self._logger =3D logger >> self._errors =3D [] >> self._return_code =3D ErrorSeverity.NO_ERR >> self._stats_result =3D None >> self._stats_filename =3D os.path.join(SETTINGS.output_dir, "sta= tistics.txt") >> >> - def add_execution(self, sut_node: NodeConfiguration) -> ExecutionRe= sult: >> - execution_result =3D ExecutionResult(sut_node) >> + def add_execution( >> + self, sut_node: NodeConfiguration, sut_version_info: NodeInfo >> + ) -> ExecutionResult: >> + execution_result =3D ExecutionResult(sut_node, sut_version_info= ) >> self._inner_results.append(execution_result) >> return execution_result >> >> @@ -296,7 +320,8 @@ def process(self) -> None: >> for error in self._errors: >> self._logger.debug(repr(error)) >> >> - self._stats_result =3D Statistics(self.dpdk_version) >> + self._stats_result =3D Statistics(self.output) >> + # add information gathered from the smoke tests to the statisti= cs >> self.add_stats(self._stats_result) >> with open(self._stats_filename, "w+") as stats_file: >> stats_file.write(str(self._stats_result)) >> diff --git a/dts/framework/test_suite.py b/dts/framework/test_suite.py >> index 0705f38f..de94c933 100644 >> --- a/dts/framework/test_suite.py >> +++ b/dts/framework/test_suite.py >> @@ -11,7 +11,12 @@ >> import re >> from types import MethodType >> >> -from .exception import ConfigurationError, SSHTimeoutError, TestCaseVer= ifyError >> +from .exception import ( >> + BlockingTestSuiteError, >> + ConfigurationError, >> + SSHTimeoutError, >> + TestCaseVerifyError, >> +) >> from .logger import DTSLOG, getLogger >> from .settings import SETTINGS >> from .test_result import BuildTargetResult, Result, TestCaseResult, Tes= tSuiteResult >> @@ -37,6 +42,7 @@ class TestSuite(object): >> """ >> >> sut_node: SutNode >> + is_blocking =3D False >> _logger: DTSLOG >> _test_cases_to_run: list[str] >> _func: bool >> @@ -118,6 +124,8 @@ def run(self) -> None: >> f"the next test suite may be affected." >> ) >> self._result.update_setup(Result.ERROR, e) >> + if len(self._result.get_errors()) > 0 and self.is_blocking: >> + raise BlockingTestSuiteError(test_suite_name) >> >> def _execute_test_suite(self) -> None: >> """ >> diff --git a/dts/framework/testbed_model/node.py b/dts/framework/testbed= _model/node.py >> index d48fafe6..c5147e0e 100644 >> --- a/dts/framework/testbed_model/node.py >> +++ b/dts/framework/testbed_model/node.py >> @@ -40,6 +40,7 @@ class Node(object): >> lcores: list[LogicalCore] >> _logger: DTSLOG >> _other_sessions: list[OSSession] >> + _execution_config: ExecutionConfiguration >> >> def __init__(self, node_config: NodeConfiguration): >> self.config =3D node_config >> @@ -64,6 +65,7 @@ def set_up_execution(self, execution_config: Execution= Configuration) -> None: >> """ >> self._setup_hugepages() >> self._set_up_execution(execution_config) >> + self._execution_config =3D execution_config >> >> def _set_up_execution(self, execution_config: ExecutionConfiguratio= n) -> None: >> """ >> diff --git a/dts/framework/testbed_model/sut_node.py b/dts/framework/tes= tbed_model/sut_node.py >> index 2b2b50d9..6783e0b8 100644 >> --- a/dts/framework/testbed_model/sut_node.py >> +++ b/dts/framework/testbed_model/sut_node.py >> @@ -1,14 +1,27 @@ >> # SPDX-License-Identifier: BSD-3-Clause >> # Copyright(c) 2010-2014 Intel Corporation >> # Copyright(c) 2023 PANTHEON.tech s.r.o. >> +# Copyright(c) 2023 University of New Hampshire >> >> import os >> import tarfile >> import time >> from pathlib import PurePath >> - >> -from framework.config import BuildTargetConfiguration, NodeConfiguratio= n >> -from framework.remote_session import CommandResult, OSSession >> +from typing import Union >> + >> +from framework.config import ( >> + BuildTargetConfiguration, >> + BuildTargetInfo, >> + InteractiveApp, >> + NodeConfiguration, >> + NodeInfo, >> +) >> +from framework.remote_session import ( >> + CommandResult, >> + InteractiveShell, >> + OSSession, >> + TestPmdShell, >> +) >> from framework.settings import SETTINGS >> from framework.utils import EnvVarsDict, MesonArgs >> >> @@ -16,6 +29,52 @@ >> from .node import Node >> >> >> +class EalParameters(object): >> + def __init__( >> + self, >> + lcore_list: LogicalCoreList, >> + memory_channels: int, >> + prefix: str, >> + no_pci: bool, >> + vdevs: list[VirtualDevice], >> + other_eal_param: str, >> + ): >> + """ >> + Generate eal parameters character string; >> + :param lcore_list: the list of logical cores to use. >> + :param memory_channels: the number of memory channels to use. >> + :param prefix: set file prefix string, eg: >> + prefix=3D'vf' >> + :param no_pci: switch of disable PCI bus eg: >> + no_pci=3DTrue >> + :param vdevs: virtual device list, eg: >> + vdevs=3D[ >> + VirtualDevice('net_ring0'), >> + VirtualDevice('net_ring1') >> + ] >> + :param other_eal_param: user defined DPDK eal parameters, eg: >> + other_eal_param=3D'--single-file-segments' >> + """ >> + self._lcore_list =3D f"-l {lcore_list}" >> + self._memory_channels =3D f"-n {memory_channels}" >> + self._prefix =3D prefix >> + if prefix: >> + self._prefix =3D f"--file-prefix=3D{prefix}" >> + self._no_pci =3D "--no-pci" if no_pci else "" >> + self._vdevs =3D " ".join(f"--vdev {vdev}" for vdev in vdevs) >> + self._other_eal_param =3D other_eal_param >> + >> + def __str__(self) -> str: >> + return ( >> + f"{self._lcore_list} " >> + f"{self._memory_channels} " >> + f"{self._prefix} " >> + f"{self._no_pci} " >> + f"{self._vdevs} " >> + f"{self._other_eal_param}" >> + ) >> + >> + >> class SutNode(Node): >> """ >> A class for managing connections to the System under Test, providin= g >> @@ -30,9 +89,11 @@ class SutNode(Node): >> _env_vars: EnvVarsDict >> _remote_tmp_dir: PurePath >> __remote_dpdk_dir: PurePath | None >> - _dpdk_version: str | None >> _app_compile_timeout: float >> _dpdk_kill_session: OSSession | None >> + _dpdk_version: str | None >> + _node_info: NodeInfo | None >> + _compiler_version: str | None >> >> def __init__(self, node_config: NodeConfiguration): >> super(SutNode, self).__init__(node_config) >> @@ -41,12 +102,14 @@ def __init__(self, node_config: NodeConfiguration): >> self._env_vars =3D EnvVarsDict() >> self._remote_tmp_dir =3D self.main_session.get_remote_tmp_dir() >> self.__remote_dpdk_dir =3D None >> - self._dpdk_version =3D None >> self._app_compile_timeout =3D 90 >> self._dpdk_kill_session =3D None >> self._dpdk_timestamp =3D ( >> f"{str(os.getpid())}_{time.strftime('%Y%m%d%H%M%S', time.lo= caltime())}" >> ) >> + self._dpdk_version =3D None >> + self._node_info =3D None >> + self._compiler_version =3D None >> >> @property >> def _remote_dpdk_dir(self) -> PurePath: >> @@ -75,6 +138,32 @@ def dpdk_version(self) -> str: >> ) >> return self._dpdk_version >> >> + @property >> + def node_info(self) -> NodeInfo: >> + if self._node_info is None: >> + self._node_info =3D self.main_session.get_node_info() >> + return self._node_info >> + >> + @property >> + def compiler_version(self) -> str: >> + if self._compiler_version is None: >> + if self._build_target_config is not None: >> + self._compiler_version =3D self.main_session.get_compil= er_version( >> + self._build_target_config.compiler.name >> + ) >> + else: >> + self._logger.warning( >> + "Failed to get compiler version because" >> + "_build_target_config is None." >> + ) >> + return "" >> + return self._compiler_version >> + >> + def get_build_target_info(self) -> BuildTargetInfo: >> + return BuildTargetInfo( >> + dpdk_version=3Dself.dpdk_version, compiler_version=3Dself.c= ompiler_version >> + ) >> + >> def _guess_dpdk_remote_dir(self) -> PurePath: >> return self.main_session.guess_dpdk_remote_dir(self._remote_tmp= _dir) >> >> @@ -84,6 +173,10 @@ def _set_up_build_target( >> """ >> Setup DPDK on the SUT node. >> """ >> + # we want to ensure that dpdk_version and compiler_version is r= eset for new >> + # build targets >> + self._dpdk_version =3D None >> + self._compiler_version =3D None >> self._configure_build_target(build_target_config) >> self._copy_dpdk_tarball() >> self._build_dpdk() >> @@ -262,48 +355,37 @@ def run_dpdk_app( >> f"{app_path} {eal_args}", timeout, verify=3DTrue >> ) >> >> - >> -class EalParameters(object): >> - def __init__( >> + def create_interactive_shell( >> self, >> - lcore_list: LogicalCoreList, >> - memory_channels: int, >> - prefix: str, >> - no_pci: bool, >> - vdevs: list[VirtualDevice], >> - other_eal_param: str, >> - ): >> + shell_type: InteractiveApp, >> + timeout: float =3D SETTINGS.timeout, >> + eal_parameters: EalParameters | None =3D None, >> + ) -> Union[InteractiveShell, TestPmdShell]: >> + """Create a handler for an interactive session. >> + >> + This method is a factory that calls a method in OSSession to cr= eate shells for >> + different DPDK applications. >> + >> + Args: >> + shell_type: Enum value representing the desired application= . >> + timeout: Timeout for reading output from the SSH channel. I= f you are >> + reading from the buffer and don't receive any data with= in the timeout >> + it will throw an error. >> + eal_parameters: List of EAL parameters to use to launch the= app. If this >> + isn't provided, it will default to calling create_eal_p= arameters(). >> + This is ignored for base "shell" types. >> + Returns: >> + Instance of the desired interactive application. >> """ >> - Generate eal parameters character string; >> - :param lcore_list: the list of logical cores to use. >> - :param memory_channels: the number of memory channels to use. >> - :param prefix: set file prefix string, eg: >> - prefix=3D'vf' >> - :param no_pci: switch of disable PCI bus eg: >> - no_pci=3DTrue >> - :param vdevs: virtual device list, eg: >> - vdevs=3D[ >> - VirtualDevice('net_ring0'), >> - VirtualDevice('net_ring1') >> - ] >> - :param other_eal_param: user defined DPDK eal parameters, eg: >> - other_eal_param=3D'--single-file-segments' >> - """ >> - self._lcore_list =3D f"-l {lcore_list}" >> - self._memory_channels =3D f"-n {memory_channels}" >> - self._prefix =3D prefix >> - if prefix: >> - self._prefix =3D f"--file-prefix=3D{prefix}" >> - self._no_pci =3D "--no-pci" if no_pci else "" >> - self._vdevs =3D " ".join(f"--vdev {vdev}" for vdev in vdevs) >> - self._other_eal_param =3D other_eal_param >> - >> - def __str__(self) -> str: >> - return ( >> - f"{self._lcore_list} " >> - f"{self._memory_channels} " >> - f"{self._prefix} " >> - f"{self._no_pci} " >> - f"{self._vdevs} " >> - f"{self._other_eal_param}" >> + if not eal_parameters: >> + eal_parameters =3D self.create_eal_parameters() >> + >> + # We need to append the build directory for DPDK apps >> + shell_type.path =3D self.remote_dpdk_build_dir.joinpath(shell_t= ype.path) >> + default_path =3D self.main_session.join_remote_path(shell_type.= path) >> + return self.main_session.create_interactive_shell( >> + shell_type, >> + default_path, >> + str(eal_parameters), >> + timeout, >> ) >> diff --git a/dts/framework/utils.py b/dts/framework/utils.py >> index 55e0b0ef..2ee67145 100644 >> --- a/dts/framework/utils.py >> +++ b/dts/framework/utils.py >> @@ -5,6 +5,8 @@ >> >> import sys >> >> +REGEX_FOR_PCI_ADDRESS =3D "/[0-9a-fA-F]{4}:[0-9a-fA-F]{2}:[0-9a-fA-F]{2= }.[0-9]{1}/" >> + >> >> def check_dts_python_version() -> None: >> if sys.version_info.major < 3 or ( >> diff --git a/dts/tests/TestSuite_smoke_tests.py b/dts/tests/TestSuite_sm= oke_tests.py >> new file mode 100644 >> index 00000000..9cf54720 >> --- /dev/null >> +++ b/dts/tests/TestSuite_smoke_tests.py >> @@ -0,0 +1,113 @@ >> +# SPDX-License-Identifier: BSD-3-Clause >> +# Copyright(c) 2023 University of New Hampshire >> + >> +import re >> + >> +from framework.config import InteractiveApp, PortConfig >> +from framework.remote_session import TestPmdDevice, TestPmdShell >> +from framework.settings import SETTINGS >> +from framework.test_suite import TestSuite >> +from framework.utils import REGEX_FOR_PCI_ADDRESS >> + >> + >> +class SmokeTests(TestSuite): >> + is_blocking =3D True >> + # dicts in this list are expected to have two keys: >> + # "pci_address" and "current_driver" >> + nics_in_node: list[PortConfig] =3D [] >> + >> + def set_up_suite(self) -> None: >> + """ >> + Setup: >> + Set the build directory path and generate a list of NICs in= the SUT node. >> + """ >> + self.dpdk_build_dir_path =3D self.sut_node.remote_dpdk_build_di= r >> + self.nics_in_node =3D self.sut_node.config.ports >> + >> + def test_unit_tests(self) -> None: >> + """ >> + Test: >> + Run the fast-test unit-test suite through meson. >> + """ >> + self.sut_node.main_session.send_command( >> + f"meson test -C {self.dpdk_build_dir_path} --suite fast-tes= ts", >> + 300, >> + verify=3DTrue, >> + ) >> + >> + def test_driver_tests(self) -> None: >> + """ >> + Test: >> + Run the driver-test unit-test suite through meson. >> + """ >> + list_of_vdevs =3D "" >> + for dev in self.sut_node._execution_config.vdevs: >> + list_of_vdevs +=3D f"--vdev {dev} " >> + list_of_vdevs =3D list_of_vdevs[:-1] >> + if list_of_vdevs: >> + self._logger.info( >> + "Running driver tests with the following virtual " >> + f"devices: {list_of_vdevs}" >> + ) >> + self.sut_node.main_session.send_command( >> + f"meson test -C {self.dpdk_build_dir_path} --suite driv= er-tests " >> + f'--test-args "{list_of_vdevs}"', >> + 300, >> + verify=3DTrue, >> + ) >> + else: >> + self.sut_node.main_session.send_command( >> + f"meson test -C {self.dpdk_build_dir_path} --suite driv= er-tests", >> + 300, >> + verify=3DTrue, >> + ) >> + >> + def test_devices_listed_in_testpmd(self) -> None: >> + """ >> + Test: >> + Uses testpmd driver to verify that devices have been found = by testpmd. >> + """ >> + testpmd_driver =3D self.sut_node.create_interactive_shell(Inter= activeApp.testpmd) >> + # We know it should always be a TestPmdShell but mypy doesn't >> + assert isinstance(testpmd_driver, TestPmdShell) >> + dev_list: list[TestPmdDevice] =3D testpmd_driver.get_devices() >> + for nic in self.nics_in_node: >> + self.verify( >> + nic.pci in map(str, dev_list), >> + f"Device {nic.pci} was not listed in testpmd's availabl= e devices, " >> + "please check your configuration", >> + ) >> + >> + def test_device_bound_to_driver(self) -> None: >> + """ >> + Test: >> + Ensure that all drivers listed in the config are bound to t= he correct driver. >> + """ >> + path_to_devbind =3D self.sut_node.main_session.join_remote_path= ( >> + self.sut_node._remote_dpdk_dir, "usertools", "dpdk-devbind.= py" >> + ) >> + >> + all_nics_in_dpdk_devbind =3D self.sut_node.main_session.send_co= mmand( >> + f"{path_to_devbind} --status | awk '{REGEX_FOR_PCI_ADDRESS}= '", >> + SETTINGS.timeout, >> + ).stdout >> + >> + for nic in self.nics_in_node: >> + # This regular expression finds the line in the above strin= g that starts >> + # with the address for the nic we are on in the loop and th= en captures the >> + # name of the driver in a group >> + devbind_info_for_nic =3D re.search( >> + f"{nic.pci}[^\\n]*drv=3D([\\d\\w]*) [^\\n]*", >> + all_nics_in_dpdk_devbind, >> + ) >> + self.verify( >> + devbind_info_for_nic is not None, >> + f"Failed to find configured device ({nic.pci}) using dp= dk-devbind.py", >> + ) >> + # We know this isn't None, but mypy doesn't >> + assert devbind_info_for_nic is not None >> + self.verify( >> + devbind_info_for_nic.group(1) =3D=3D nic.os_driver, >> + f"Driver for device {nic.pci} does not match driver lis= ted in " >> + f"configuration (bound to {devbind_info_for_nic.group(1= )})", >> + ) >> -- >> 2.41.0 >>