From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 1EEFF438AC; Fri, 12 Jan 2024 23:41:22 +0100 (CET) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id ACEE340685; Fri, 12 Jan 2024 23:41:21 +0100 (CET) Received: from mail-qk1-f173.google.com (mail-qk1-f173.google.com [209.85.222.173]) by mails.dpdk.org (Postfix) with ESMTP id 54093402A9 for ; Fri, 12 Jan 2024 23:41:20 +0100 (CET) Received: by mail-qk1-f173.google.com with SMTP id af79cd13be357-783137d7fb4so486526885a.2 for ; Fri, 12 Jan 2024 14:41:20 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=iol.unh.edu; s=unh-iol; t=1705099279; x=1705704079; darn=dpdk.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=Zrw12YXT8tQHZkvYzgz+iWdz8EoJJO1ZdOXGo3oyhb0=; b=D51AkFdq8tvBxn9IqtalmiK3tzbxt8mDzLYasWAHvqeoGVCUmhmx0Qz2J4EtnJUZgO X+Jk8WRhZcpAERWqWZTODRJgEX08OU3Jz2LKBX71B93CbAL1jCcfoAlq9gglz3WswWfy jJmeEsnttsuRxJK4Tatw451Mn9vIhw4OTHHC0= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1705099279; x=1705704079; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=Zrw12YXT8tQHZkvYzgz+iWdz8EoJJO1ZdOXGo3oyhb0=; b=BKx37DYMltcg27B6W9T//bATVmR5NwtPui4dM0Kh6K8MI4GYf4UbQemJjMhT3Z7PCX ffRA6EBr1CXiT8mMLRp+/Nnh1BDDOIj6Buqp88xWp0Qeh10MrBtbBVT8MoF5MBKvUQhB p+VYBqVQmMIwrgC+tGIxpb9eq9zN/ahNjFw6mLvmAQEqnJkmvl2lrPcOCJjdbPNgPaML ctM2AGzU/V3z86pH2kYLc/adBcGrm0fcUr4qknaVJFayW0ZshszYTEE6nn7fge/zZink gIAccULJpYG8yPmHidbmguFRckAtie/HtCsIzzjMXz0hNA8l0DLpdJRKEouKD5uzf987 D5sQ== X-Gm-Message-State: AOJu0YxyJKltWRncHcwPNv4wfi+J2q7s3Glm+OCrsw57LmIryE+u+1Hn VWXkZN5PRPo7CYRmRAHH0f8DneCxlAiuKFo1nz7N0DLy/veEyotlSNHbJxYeihqa2s3T119PXvw UhAI4AB5/gz+se2+tk/O/m+G6glDPIUrYSFerN+JU3mtKTieIHLDnd7/4eET8qDEuQNLXMw== X-Google-Smtp-Source: AGHT+IGxrnh2FVPFLV5xOddz4PeySQLAsPegCOSZgmuXcLW5GPW27D0T6cg5cXZUBOTxjraCJpfmUw== X-Received: by 2002:a05:6214:2428:b0:680:fb20:202f with SMTP id gy8-20020a056214242800b00680fb20202fmr1592223qvb.127.1705099279338; Fri, 12 Jan 2024 14:41:19 -0800 (PST) Received: from voidcell.iol.unh.edu ([2606:4100:3880:1220:7b65:70b6:505d:49f9]) by smtp.gmail.com with ESMTPSA id v9-20020ac87289000000b004199c98f87dsm1711774qto.74.2024.01.12.14.41.18 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Fri, 12 Jan 2024 14:41:19 -0800 (PST) From: Adam Hassick To: ci@dpdk.org Cc: Adam Hassick Subject: [PATCH v2 1/2] tools: Add script to create artifacts Date: Fri, 12 Jan 2024 17:40:13 -0500 Message-ID: <20240112224014.30955-2-ahassick@iol.unh.edu> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20240112224014.30955-1-ahassick@iol.unh.edu> References: <20240112224014.30955-1-ahassick@iol.unh.edu> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-BeenThere: ci@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK CI discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: ci-bounces@dpdk.org This script takes in a URL to a series on Patchwork and emits a tarball which may be used for running tests. Signed-off-by: Adam Hassick --- tools/create_series_artifact.py | 472 ++++++++++++++++++++++++++++++++ 1 file changed, 472 insertions(+) create mode 100755 tools/create_series_artifact.py diff --git a/tools/create_series_artifact.py b/tools/create_series_artifact.py new file mode 100755 index 0000000..c1fc636 --- /dev/null +++ b/tools/create_series_artifact.py @@ -0,0 +1,472 @@ +#!/usr/bin/env python3 + +import argparse +from dataclasses import dataclass + +import os +from git_pw import api as pw_api +import pathlib +import pygit2 +import requests +import shutil +import subprocess +import tarfile +from typing import Any, Dict, List, Optional +import yaml + +HELP = """This script will create an artifact given a URL to a Patchwork series. +Much of the information provided is acquired by this script through the use of a configuration file. +This configuration file can be found with the other script configs in the config directory of the CI repo. + +The configuration file is used to aggregate: +- Git credentials +- Patchwork configuration and the user token (user token is optional) +- The URL of the DPDK Git mirror +- Locations of dependency scripts and their configuration files + +More detail and examples can be found in the default configuration file. +This default file is located at "config/artifacts.yml" in the dpdk-ci repository. + +Example usage: + +./create_series_artifact.py ../configs/artifacts.yml https://patches.dpdk.org/api/1.3/series/12345/ + +""" + +# Map the outputs of pw_maintainers_cli to the names of branches on the +# GitHub mirror. This is temporary, and should be moved elsewhere in +# the future. +BRANCH_NAME_MAP = { + "next-baseband": "next-baseband-for-main", + "next-crypto": "next-crypto-for-main", + "next-eventdev": "next-eventdev-for-main", + "next-net": "next-net-for-main", + "next-net-intel": "next-net-intel-for-next-net", + "next-net-brcm": "next-net-brcm-for-next-net", + "next-net-mlx": "next-net-mlx-for-next-net", + "next-net-mrvl": "next-net-mrvl-for-main", + "next-virtio": "next-virtio-for-next-net", +} + + +@dataclass +class CreateSeriesParameters(object): + pw_server: str + pw_project: str + pw_token: str + git_user: str + git_email: str + series_url: str + patch_ids: List[int] + labels: List[str] + config: Dict + series: Dict + pw_mcli_script: pathlib.Path + patch_parser_script: pathlib.Path + patch_parser_cfg: pathlib.Path + lzma: bool + output_tarball: pathlib.Path + output_properties: pathlib.Path + no_depends: bool + docs_only: bool + + +class ProjectTree(object): + artifact_path: pathlib.Path + tree: str + commit_id: str + path: pathlib.Path + log_file_path: pathlib.Path + props_file_path: pathlib.Path + data: CreateSeriesParameters + repo: pygit2.Repository + log_buf: List[str] + properties: Dict[str, Any] + + def log(self, msg: str): + print(msg) + self.log_buf.append(msg) + + def write_log(self): + with open(self.log_file_path, "w") as log_file: + log_file.write("\n".join([msg for msg in self.log_buf])) + + def write_properties(self): + with open(self.props_file_path, "w") as prop_file: + for key, value in self.properties.items(): + prop_file.write(f"{key}={value}\n") + + def move_logs(self): + shutil.move(self.log_file_path, pathlib.Path(os.getcwd(), "log.txt")) + shutil.move( + self.props_file_path, pathlib.Path(os.getcwd(), self.data.output_properties) + ) + + def __init__(self, data: CreateSeriesParameters): + self.data = data + self.path = pathlib.Path(os.curdir, "dpdk").absolute() + self.log_buf = [] + self.log_file_path = pathlib.Path(self.path, "log.txt") + self.props_file_path = pathlib.Path(self.path, data.output_properties) + self.tree = "main" + self.properties = {} + self.artifact_path = data.output_tarball + + # Set the range of patch IDs this series (aka patchset) covers. + self.properties["patchset_range"] = f"{data.patch_ids[0]}-{data.patch_ids[-1]}" + + # Set the tags using tags obtained by the params class + self.properties["tags"] = " ".join(data.labels) + + # Record whether this patch is only documentation + self.properties["is_docs_only"] = str(data.docs_only) + + if not self.path.exists(): + # Find the URL to clone from based on the tree name. + repo_url = self.data.config["repo_url"] + + # Pull down the git repo we found. + for i in range(1, 4): + self.log(f"Cloning the DPDK mirror at: {repo_url} (Attempt {i} of 3)") + try: + repo = pygit2.clone_repository(repo_url, self.path) + break + except pygit2.GitError as e: + self.log(f"Failed! Reason: {e}") + else: + self.log("Failed to clone from the upstream repository.") + exit(1) + else: + # Fetch any new changes. + repo = pygit2.Repository(self.path) + origin = repo.remotes["origin"] + origin.fetch() + + self.log("Cleaning repository state...") + repo.state_cleanup() + + # Initially, check out to main. + self.repo = repo + self.checkout("main") + + self.log(f"Done: {self.tree} commit {self.commit_id}") + + def checkout(self, branch: str) -> Optional[str]: + """ + Check out to some branch. + Returns true if successful, false otherwise. + """ + + git_branch = self.repo.lookup_branch( + f"origin/{branch}", pygit2.GIT_BRANCH_REMOTE + ) + + if not git_branch: + self.log(f"Tried to checkout to non-existant branch: {branch}") + return None + + self.log(f"Trying to checkout branch: {git_branch.branch_name}") + reference = self.repo.resolve_refish(git_branch.branch_name) + self.commit_id = str(reference[0].id) + self.repo.reset(reference[0].id, pygit2.GIT_RESET_HARD) + self.repo.checkout(reference[1]) + self.tree = branch + + self.log(f"Checked out to {branch} ({self.commit_id})") + + return branch + + def guess_git_tree(self) -> Optional[str]: + """ + Run pw_maintainers_cli to guess the git tree of the patch series we are applying. + Returns None if the pw_maintainers_cli failed. + """ + + if "id" not in self.data.series: + raise Exception("ID was not found in the series JSON") + + result = subprocess.run( + [ + self.data.pw_mcli_script, + "--type", + "series", + "--pw-server", + self.data.pw_server, + "--pw-project", + self.data.pw_project, + "list-trees", + str(self.data.series["id"]), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=self.path, + env={ + "MAINTAINERS_FILE_PATH": "MAINTAINERS", + "PW_TOKEN": self.data.pw_token, + }, + ) + + if result.returncode == 0: + branch = result.stdout.decode().strip() + + if branch in ["main", "dpdk"]: + branch = "main" + else: + if branch.startswith("dpdk-"): + branch = branch[5:] + + branch = BRANCH_NAME_MAP.get(branch) + + return self.checkout(branch) + else: + self.log("Failed to guess git tree. Output from pw_maintainers_cli:") + self.log(result.stdout.decode()) + self.log(result.stderr.decode()) + return None + + def set_properties(self): + self.properties["tree"] = self.tree + self.properties["applied_commit_id"] = self.commit_id + + def apply_patch_series(self) -> bool: + # Run git-pw to apply the series. + + # Configure the tree to point at the given patchwork server and project + self.repo.config["pw.server"] = self.data.pw_server + self.repo.config["pw.project"] = self.data.pw_project + self.repo.config["user.email"] = self.data.git_email + self.repo.config["user.name"] = self.data.git_user + + result = subprocess.run( + ["git", "pw", "series", "apply", str(self.data.series["id"])], + cwd=self.path, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + # Write the log from the apply process to disk. + self.log("Applying patch...") + self.log(result.stdout.decode()) + self.log(result.stderr.decode()) + + # Store whether there was an error, and return the flag. + error = result.returncode != 0 + self.properties["apply_error"] = error + return not error + + def test_build(self) -> bool: + ninja_result: Optional[subprocess.CompletedProcess] = None + meson_result: subprocess.CompletedProcess = subprocess.run( + ["meson", "setup", "build"], + cwd=self.path, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + build_error = meson_result.returncode != 0 + + self.log("Running test build...") + self.log(meson_result.stdout.decode()) + + if not build_error: + ninja_result = subprocess.run( + ["ninja", "-C", "build"], + cwd=self.path, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + build_error = build_error or ninja_result.returncode != 0 + shutil.rmtree(pathlib.Path(self.path, "build")) + + self.log(ninja_result.stdout.decode()) + self.log(ninja_result.stderr.decode()) + + self.log(meson_result.stderr.decode()) + + if build_error: + self.log("Test build failed.") + + self.properties["build_error"] = build_error + return not build_error + + def create_tarball(self): + # Copy the logs into the artifact tarball. + self.write_log() + self.write_properties() + + # Create a tar archive containing the DPDK sources. + with tarfile.open( + self.artifact_path, mode="w:xz" if self.data.lzma else "w:gz" + ) as tar_file: + tar_file.add(self.path, "dpdk", recursive=True) + + # Move the log file out of the working directory. + self.move_logs() + + return True + + +def get_tags( + patch_parser_script: pathlib.Path, + patch_parser_cfg: pathlib.Path, + series: Dict, +) -> List[str]: + series_filename = f"{series['id']}.patch" + + # Pull down the patch series as a single file. + pw_api.download(series["mbox"], None, series_filename) + + # Call the patch parser script to obtain the tags + parse_result = subprocess.run( + [patch_parser_script, patch_parser_cfg, series_filename], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + # Assert that patch parser succeeded. + parse_result.check_returncode() + + # Return the output + return parse_result.stdout.decode().splitlines() + + +def parse_args() -> CreateSeriesParameters: + """ + Parses the arguments and returns an instance of a dataclass containing parameters + and some derived information. + """ + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description=HELP, + ) + parser.add_argument( + "config", + type=argparse.FileType(), + help="The config file to load. Must be a path to a YAML file.", + ) + parser.add_argument("series_url", type=str, help="The URL to a Patchwork series.") + parser.add_argument( + "-t", + "--pw-token", + dest="pw_token", + type=str, + help="The Patchwork token to use", + ) + parser.add_argument( + "-l", + "--lzma", + action="store_true", + help="When set, use LZMA compression rather than GNU zip compression.", + ) + parser.add_argument( + "-nd", + "--no-depends", + action="store_true", + help="When set, does not acknowledge the Depends-on label.", + ) + + args = parser.parse_args() + + # Read the configuration file. + with args.config as config_file: + config = yaml.safe_load(config_file) + + pw_server = config["patchwork"]["server"] + pw_project = config["patchwork"]["project"] + pw_token = args.pw_token or config["patchwork"].get("token") + + if not pw_token: + print("Failed to obtain the Patchworks token.") + exit(1) + + pw_mcli_script = pathlib.Path(config["pw_maintainers_cli"]["path"]).absolute() + + git_user = config["git"]["user"] + git_email = config["git"]["email"] + + patch_parser_script = pathlib.Path(config["patch_parser"]["path"]).absolute() + patch_parser_cfg = pathlib.Path(config["patch_parser"]["config"]).absolute() + + if args.lzma: + tarball_name = "dpdk.tar.xz" + else: + tarball_name = "dpdk.tar.gz" + + output_tarball = pathlib.Path(tarball_name) + output_properties = pathlib.Path(f"{tarball_name}.properties") + + # Pull the series JSON down. + resp = requests.get(args.series_url) + resp.raise_for_status() + series = resp.json() + + # Get the labels using the patch parser. + labels = get_tags(patch_parser_script, patch_parser_cfg, series) + + # See if this is a documentation-only patch. + docs_only = len(labels) == 1 and labels[0] == "documentation" + + # Get the patch ids in this patch series. + patch_ids = list(map(lambda x: int(x["id"]), series["patches"])) + patch_ids.sort() + + return CreateSeriesParameters( + pw_server=pw_server, + pw_project=pw_project, + pw_token=pw_token, + git_user=git_user, + git_email=git_email, + series_url=args.series_url, + patch_ids=patch_ids, + labels=labels, + config=config, + series=series, + pw_mcli_script=pw_mcli_script, + patch_parser_script=patch_parser_script, + patch_parser_cfg=patch_parser_cfg, + lzma=args.lzma, + output_tarball=output_tarball, + output_properties=output_properties, + no_depends=args.no_depends, + docs_only=docs_only, + ) + + +def try_to_apply(tree: ProjectTree) -> bool: + tree.set_properties() + return tree.apply_patch_series() and tree.test_build() and tree.create_tarball() + + +def main() -> int: + data = parse_args() + + # Pull down the DPDK mirror. + tree = ProjectTree(data) + + # Try to guess the Git tree for this patchset. + guessed_tree = tree.guess_git_tree() + + if not guessed_tree: + print("Failed to guess git tree.") + return 1 + + # Try to apply this patch. + if not ( + try_to_apply(tree) # First, try to apply on the guessed tree. + or guessed_tree != "main" # If that fails, and the guessed tree was not main + and tree.checkout("main") # Checkout to main, then + and try_to_apply(tree) # Try to apply on main + ): + tree.write_log() + tree.write_properties() + tree.move_logs() + + print("FAILURE") + + return 1 + return 0 + + +if __name__ == "__main__": + exit(main()) -- 2.43.0