From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id A550B41DC6; Fri, 3 Mar 2023 11:26:15 +0100 (CET) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 0A16242D49; Fri, 3 Mar 2023 11:25:24 +0100 (CET) Received: from mail-ed1-f46.google.com (mail-ed1-f46.google.com [209.85.208.46]) by mails.dpdk.org (Postfix) with ESMTP id 44ED142BAC for ; Fri, 3 Mar 2023 11:25:19 +0100 (CET) Received: by mail-ed1-f46.google.com with SMTP id o15so8143403edr.13 for ; Fri, 03 Mar 2023 02:25:19 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=pantheon-tech.20210112.gappssmtp.com; s=20210112; t=1677839119; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=W1+Bqo5zDfFYMcMPu9OvyDz2WcXjTIJytK8DU3eHLNU=; b=DTUyy44m1m80zw+9TTQ4oHnk9U+Kog6pOkQM05Nb4o557T4Z7dWFxKzudoXeiDwdKM 6ETDWxz+W9YUn/2wMs1OHsWvWHRfdVJJNlhEUzXc1pDf0e+UoVG207eLO0DiuCqQ2bDy +iX5WuhzBxjGOzOzqe3sVgsMhSTiYaQngPbYxsezUXNf55gBMx8wkhCM160IEwnsd0YY gkM76e3/fyahV6DNb2kxPnAb0OxPHAuil91R+b7xdamtCPnaDMYP5XUTeYMkYRlNNPdd vwzvMvTCWBo4JEWe8qHNI6dMz3hyRDFITTzv6HzNwgHWHV3jWbFtVQ6C5Ro+4gwKKmk4 NrxQ== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; t=1677839119; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=W1+Bqo5zDfFYMcMPu9OvyDz2WcXjTIJytK8DU3eHLNU=; b=OgXGNIApNuCw9VuQQVSvpcNR82lfHaM8DiwrInHVrAuuhxX5HOICiK/raD4T0wz8X5 kWVAP3ScGDFguX3pPp9cYJro1c/4ffGJxBO6ywwAmHDm93B6CUmVH+Kbz1qtJ10va+9V YRdepXfvphK4wjL3HmMXPgDUZqEiAapfS7WsomKnnOya0257EG2IS8Y9e9bmgXmDqqfs DN9qNQiIpcHUJCHYIsUVCdYIXjVRNV93DISuM1hqpjX3uUqZVUfwXaQlm87d4XFIlWzX PSa4kU7QNSmdulQ901B+3NWLf54DPPsM5zF3v0xibZHxc4TYONHtdtvnjOdAJ2yKA+6r 9b+A== X-Gm-Message-State: AO0yUKVcnC+MDY3/hk6RDPoRxpaTvaga76fjWMSwYs6iUTxgSQ12+T0t lXqmTC38QR2svVzFe0Jcl4GZhA== X-Google-Smtp-Source: AK7set/QUMdJN6e1g+XhBGyp/fQ+o21zdsSFAcpqfS7ch85A4FwKhT9qVikk2O9O0RLP7asC+4r7Wg== X-Received: by 2002:a05:6402:884:b0:4b1:2041:f8b2 with SMTP id e4-20020a056402088400b004b12041f8b2mr1579639edy.15.1677839118905; Fri, 03 Mar 2023 02:25:18 -0800 (PST) Received: from localhost.localdomain (ip-46.34.234.35.o2inet.sk. [46.34.234.35]) by smtp.gmail.com with ESMTPSA id j19-20020a508a93000000b004c3e3a6136dsm984028edj.21.2023.03.03.02.25.18 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Fri, 03 Mar 2023 02:25:18 -0800 (PST) From: =?UTF-8?q?Juraj=20Linke=C5=A1?= To: thomas@monjalon.net, Honnappa.Nagarahalli@arm.com, lijuan.tu@intel.com, bruce.richardson@intel.com, probb@iol.unh.edu Cc: dev@dpdk.org, =?UTF-8?q?Juraj=20Linke=C5=A1?= Subject: [PATCH v6 08/10] dts: add test suite config and runner Date: Fri, 3 Mar 2023 11:25:05 +0100 Message-Id: <20230303102507.527790-9-juraj.linkes@pantheon.tech> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20230303102507.527790-1-juraj.linkes@pantheon.tech> References: <20230223152840.634183-1-juraj.linkes@pantheon.tech> <20230303102507.527790-1-juraj.linkes@pantheon.tech> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org The config allows users to specify which test suites and test cases within test suites to run. Also add test suite running capabilities to dts runner. Signed-off-by: Juraj Linkeš --- dts/conf.yaml | 2 ++ dts/framework/config/__init__.py | 29 +++++++++++++++- dts/framework/config/conf_yaml_schema.json | 40 ++++++++++++++++++++++ dts/framework/dts.py | 19 ++++++++++ dts/framework/test_suite.py | 24 ++++++++++++- 5 files changed, 112 insertions(+), 2 deletions(-) diff --git a/dts/conf.yaml b/dts/conf.yaml index 75e33e8ccf..a9bd8a3ecf 100644 --- a/dts/conf.yaml +++ b/dts/conf.yaml @@ -10,6 +10,8 @@ executions: compiler_wrapper: ccache perf: false func: true + test_suites: + - hello_world system_under_test: "SUT 1" nodes: - name: "SUT 1" diff --git a/dts/framework/config/__init__.py b/dts/framework/config/__init__.py index 544fceca6a..ebb0823ff5 100644 --- a/dts/framework/config/__init__.py +++ b/dts/framework/config/__init__.py @@ -12,7 +12,7 @@ import pathlib from dataclasses import dataclass from enum import Enum, auto, unique -from typing import Any +from typing import Any, TypedDict import warlock # type: ignore import yaml @@ -128,11 +128,34 @@ def from_dict(d: dict) -> "BuildTargetConfiguration": ) +class TestSuiteConfigDict(TypedDict): + suite: str + cases: list[str] + + +@dataclass(slots=True, frozen=True) +class TestSuiteConfig: + test_suite: str + test_cases: list[str] + + @staticmethod + def from_dict( + entry: str | TestSuiteConfigDict, + ) -> "TestSuiteConfig": + if isinstance(entry, str): + return TestSuiteConfig(test_suite=entry, test_cases=[]) + elif isinstance(entry, dict): + return TestSuiteConfig(test_suite=entry["suite"], test_cases=entry["cases"]) + else: + raise TypeError(f"{type(entry)} is not valid for a test suite config.") + + @dataclass(slots=True, frozen=True) class ExecutionConfiguration: build_targets: list[BuildTargetConfiguration] perf: bool func: bool + test_suites: list[TestSuiteConfig] system_under_test: NodeConfiguration @staticmethod @@ -140,6 +163,9 @@ def from_dict(d: dict, node_map: dict) -> "ExecutionConfiguration": build_targets: list[BuildTargetConfiguration] = list( map(BuildTargetConfiguration.from_dict, d["build_targets"]) ) + test_suites: list[TestSuiteConfig] = list( + map(TestSuiteConfig.from_dict, d["test_suites"]) + ) sut_name = d["system_under_test"] assert sut_name in node_map, f"Unknown SUT {sut_name} in execution {d}" @@ -147,6 +173,7 @@ def from_dict(d: dict, node_map: dict) -> "ExecutionConfiguration": build_targets=build_targets, perf=d["perf"], func=d["func"], + test_suites=test_suites, system_under_test=node_map[sut_name], ) diff --git a/dts/framework/config/conf_yaml_schema.json b/dts/framework/config/conf_yaml_schema.json index 878ca3aec2..ca2d4a1ef2 100644 --- a/dts/framework/config/conf_yaml_schema.json +++ b/dts/framework/config/conf_yaml_schema.json @@ -93,6 +93,32 @@ "required": [ "amount" ] + }, + "test_suite": { + "type": "string", + "enum": [ + "hello_world" + ] + }, + "test_target": { + "type": "object", + "properties": { + "suite": { + "$ref": "#/definitions/test_suite" + }, + "cases": { + "type": "array", + "description": "If specified, only this subset of test suite's test cases will be run. Unknown test cases will be silently ignored.", + "items": { + "type": "string" + }, + "minimum": 1 + } + }, + "required": [ + "suite" + ], + "additionalProperties": false } }, "type": "object", @@ -172,6 +198,19 @@ "type": "boolean", "description": "Enable functional testing." }, + "test_suites": { + "type": "array", + "items": { + "oneOf": [ + { + "$ref": "#/definitions/test_suite" + }, + { + "$ref": "#/definitions/test_target" + } + ] + } + }, "system_under_test": { "$ref": "#/definitions/node_name" } @@ -181,6 +220,7 @@ "build_targets", "perf", "func", + "test_suites", "system_under_test" ] }, diff --git a/dts/framework/dts.py b/dts/framework/dts.py index 3d4170d10f..9012a499a3 100644 --- a/dts/framework/dts.py +++ b/dts/framework/dts.py @@ -8,6 +8,7 @@ from .config import CONFIGURATION, BuildTargetConfiguration, ExecutionConfiguration from .exception import DTSError, ErrorSeverity from .logger import DTSLOG, getLogger +from .test_suite import get_test_suites from .testbed_model import SutNode from .utils import check_dts_python_version @@ -132,6 +133,24 @@ def _run_suites( with possibly only a subset of test cases. If no subset is specified, run all test cases. """ + for test_suite_config in execution.test_suites: + try: + full_suite_path = f"tests.TestSuite_{test_suite_config.test_suite}" + test_suite_classes = get_test_suites(full_suite_path) + suites_str = ", ".join((x.__name__ for x in test_suite_classes)) + dts_logger.debug( + f"Found test suites '{suites_str}' in '{full_suite_path}'." + ) + except Exception as e: + dts_logger.exception("An error occurred when searching for test suites.") + errors.append(e) + + else: + for test_suite_class in test_suite_classes: + test_suite = test_suite_class( + sut_node, test_suite_config.test_cases, execution.func, errors + ) + test_suite.run() def _exit_dts() -> None: diff --git a/dts/framework/test_suite.py b/dts/framework/test_suite.py index 9002d43297..12bf3b6420 100644 --- a/dts/framework/test_suite.py +++ b/dts/framework/test_suite.py @@ -6,12 +6,13 @@ Base class for creating DTS test cases. """ +import importlib import inspect import re from collections.abc import MutableSequence from types import MethodType -from .exception import SSHTimeoutError, TestCaseVerifyError +from .exception import ConfigurationError, SSHTimeoutError, TestCaseVerifyError from .logger import DTSLOG, getLogger from .settings import SETTINGS from .testbed_model import SutNode @@ -226,3 +227,24 @@ def _execute_test_case(self, test_case_method: MethodType) -> bool: raise KeyboardInterrupt("Stop DTS") return result + + +def get_test_suites(testsuite_module_path: str) -> list[type[TestSuite]]: + def is_test_suite(object) -> bool: + try: + if issubclass(object, TestSuite) and object is not TestSuite: + return True + except TypeError: + return False + return False + + try: + testcase_module = importlib.import_module(testsuite_module_path) + except ModuleNotFoundError as e: + raise ConfigurationError( + f"Test suite '{testsuite_module_path}' not found." + ) from e + return [ + test_suite_class + for _, test_suite_class in inspect.getmembers(testcase_module, is_test_suite) + ] -- 2.30.2