From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 11197A0A02 for ; Thu, 14 Jan 2021 18:59:34 +0100 (CET) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id DB1891413FA; Thu, 14 Jan 2021 18:59:33 +0100 (CET) Received: from mail-ua1-f100.google.com (mail-ua1-f100.google.com [209.85.222.100]) by mails.dpdk.org (Postfix) with ESMTP id 661751413B0 for ; Thu, 14 Jan 2021 18:59:32 +0100 (CET) Received: by mail-ua1-f100.google.com with SMTP id n18so2039125ual.9 for ; Thu, 14 Jan 2021 09:59:32 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=iol.unh.edu; s=unh-iol; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=3J+DhoIcUGI1sFo02o7P6YXzINuhFr6imiVVXYrhCnU=; b=Clx7rs/p7/XjaYPtMlFd1AmuIyt4LD0rxuYQz5g13zqWvD4fmqT04acGXjrIMiT2ua seSQhXxksd1Eg3P8DtS+znYdnB6fSowfHED33w0T+9nqH79sfH8vbGgv5aRyrYcH3oiM YhJk0SSAMpQYJo3S1PQUQEv38gXYHIvxl3BY4= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=3J+DhoIcUGI1sFo02o7P6YXzINuhFr6imiVVXYrhCnU=; b=hFBiQOco7e3i3qyOSd8v8hiMHXPFneiHUMKH0MVgLBZlQSfnFjC4Fs0qID2jT6avQK xvGvhFTA6P7kNPiYcUIeEdj3aYbH12xQz2ap0sAEK+wT9FYUOYp2wuYh38zcKVug3l8N celhqtljlQc2O3epjKFw9GOvsyBMORPYmLeotslL/jJj7bNgabPB24qdlAMXOEcDQtyG r4pOZWgKtKabSyOwpKGfL68uIDzMQEuFrfOd90XoaotiCdg2tYzc3GVtl5NgVcysqHn9 kKKvAKnvIzlX92GwaHFcqZW03igWJW87kahY6/vI6GxSUVUxlCSyZO80X0g99pFmQgcT iWxQ== X-Gm-Message-State: AOAM53181sun8bphvKNxpPB1QvXdfg4vH4+6JWIKw3d0M0Hu99p8JhuM QqI1+fsX+xQS5hLISHb3dEmojo3+VW1jk7R82Uuj0wyhjCarUbj6byjT+ZmAcHS983XU0h+qvz0 Ei0bDiob46y+ZCqNkqqs27kxwhCH+1wJLi9At6rQYC2D4WvDk3eBJyl+mnrGB/mwZSJBsU6OS X-Google-Smtp-Source: ABdhPJzTgVIH1+57xt2ZFCrfA7mGqvwvVcIMR0m6j42d9g5Gq6UnuKemuJeOSxXbYf7Mi+/ZWLZqxUcnAPWG X-Received: by 2002:ab0:7811:: with SMTP id x17mr6968519uaq.51.1610647171727; Thu, 14 Jan 2021 09:59:31 -0800 (PST) Received: from postal.iol.unh.edu (postal.iol.unh.edu. [132.177.123.84]) by smtp-relay.gmail.com with ESMTPS id f62sm947154vkg.10.2021.01.14.09.59.31 (version=TLS1_2 cipher=ECDHE-ECDSA-AES128-GCM-SHA256 bits=128/128); Thu, 14 Jan 2021 09:59:31 -0800 (PST) X-Relaying-Domain: iol.unh.edu Received: from iol.unh.edu (unknown [IPv6:2606:4100:3880:1254::1033]) by postal.iol.unh.edu (Postfix) with ESMTP id 1B62E6052472; Thu, 14 Jan 2021 12:59:31 -0500 (EST) From: ohilyard@iol.unh.edu To: ci@dpdk.org Cc: thomas@monjalon.net, Owen Hilyard Date: Thu, 14 Jan 2021 12:59:25 -0500 Message-Id: <20210114175925.570013-1-ohilyard@iol.unh.edu> X-Mailer: git-send-email 2.27.0 In-Reply-To: References: MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Subject: [dpdk-ci] [PATCH V2] patch-tagging: Added tool to convert tags into dts execution file X-BeenThere: ci@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK CI discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: ci-bounces@dpdk.org Sender: "ci" Fixed parsing bugs with the tag parsing script Parsing the patch file directly proved to be too unreliabler, so the approach to patch parsing was changed so that it relies instead on using a directory with those patches already applied in them and then use a git diff command to get the changed files. This avoids needing to parse the patch files directly and should be more reliable. Added a tool to create an execution file from tags using another execution file as a template This tool is intended to allow a list of patch tags to be used in combination with an execution file containing information on crbs to only change which test suites are run. This tool enables using the mappings in the tests_for_tag.cfg file to run only necessary tests. At present, the mappings represent how tests are currently run at the UNH-IOL Community Lab, with the same tests being run for every patch, with the requested change from bug 511 where documentation changes do not cause any testing. Signed-off-by: Owen Hilyard --- config/tests_for_tag.cfg | 19 ++++++++++++-- tools/patch_parser.py | 54 ++++++++++++++++++++++++---------------- 2 files changed, 49 insertions(+), 24 deletions(-) diff --git a/config/tests_for_tag.cfg b/config/tests_for_tag.cfg index 77f797f..7d95c4a 100644 --- a/config/tests_for_tag.cfg +++ b/config/tests_for_tag.cfg @@ -5,11 +5,26 @@ core = dynamic_config, mtu_update, scatter, stats_checks, + unit_tests_mbuf +driver = dynamic_config, + link_status_interrupt, + mac_filter, + mtu_update, + scatter, + stats_checks, + unit_tests_mbuf +application = dynamic_config, + link_status_interrupt, + mac_filter, + mtu_update, + scatter, stats_checks, unit_tests_mbuf - ; Nothing in documentation documentation = [performance] -core = nic_single_core_perf, +core = nic_single_core_perf +driver = nic_single_core_perf +application = nic_single_core_perf +documentation = diff --git a/tools/patch_parser.py b/tools/patch_parser.py index 80b8194..73df91f 100755 --- a/tools/patch_parser.py +++ b/tools/patch_parser.py @@ -31,30 +31,39 @@ # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - +import subprocess import itertools from configparser import ConfigParser from typing import List, Dict, Set import argparse +import re def get_patch_files(patch_file: str) -> List[str]: + file_match_pattern = re.compile(r"[\w\-\/.]+ +\| +\d+ [+\-]+") with open(patch_file, 'r') as f: - lines = list(itertools.takewhile( - lambda line: line.strip().endswith('+') or line.strip().endswith('-'), - itertools.dropwhile( - lambda line: not line.strip().startswith("---"), - f.readlines() + file_lines = f.readlines() + lines = list(filter( + lambda line: file_match_pattern.match(line.strip()), + itertools.takewhile( + lambda line: not line.startswith('---'), + list( + itertools.dropwhile( + lambda line: not line.strip().startswith("---"), + file_lines + ) + )[1:] ) )) filenames = map(lambda line: line.strip().split(' ')[0], lines) - # takewhile includes the --- which starts the filenames - return list(filenames)[1:] + return list(filenames) def get_all_files_from_patches(patch_files: List[str]) -> Set[str]: - return set(itertools.chain.from_iterable(map(get_patch_files, patch_files))) + num_patch_files: int = len(patch_files) + files = subprocess.check_output(f"cd dpdk && git diff --name-only HEAD~{num_patch_files}", shell=True).decode('utf-8').splitlines() + return set(files) def parse_comma_delimited_list_from_string(mod_str: str) -> List[str]: @@ -80,21 +89,22 @@ def get_tags_for_patches(patch_files: Set[str], dir_attrs: Dict[str, Set[str]]) )) -parser = argparse.ArgumentParser( - description='Takes a patch file and a config file and creates a list of tags for that patch') -parser.add_argument('config_file_path', help='The path to patch_parser.cfg', default='config/patch_parser.cfg') -parser.add_argument('patch_file_paths', help='A list of patch files', type=str, metavar='patch file', nargs='+') +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Takes a patch file and a config file and creates a list of tags for that patch') + parser.add_argument('config_file_path', help='The path to patch_parser.cfg', default='config/patch_parser.cfg') + parser.add_argument('patch_file_paths', help='A list of patch files', type=str, metavar='patch file', nargs='+') -args = parser.parse_args() + args = parser.parse_args() -conf_obj = ConfigParser() -conf_obj.read(args.config_file_path) + conf_obj = ConfigParser() + conf_obj.read(args.config_file_path) -patch_files = get_all_files_from_patches(args.patch_file_paths) -dir_attrs = get_dictionary_attributes_from_config_file(conf_obj) -priority_list = parse_comma_delimited_list_from_string(conf_obj['Priority']['priority_list']) + patch_files = get_all_files_from_patches(args.patch_file_paths) + dir_attrs = get_dictionary_attributes_from_config_file(conf_obj) + priority_list = parse_comma_delimited_list_from_string(conf_obj['Priority']['priority_list']) -unordered_tags: Set[str] = get_tags_for_patches(patch_files, dir_attrs) -ordered_tags: List[str] = [tag for tag in priority_list if tag in unordered_tags] + unordered_tags: Set[str] = get_tags_for_patches(patch_files, dir_attrs) + ordered_tags: List[str] = [tag for tag in priority_list if tag in unordered_tags] -print("\n".join(ordered_tags)) + print("\n".join(ordered_tags)) -- 2.27.0