From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id A938342B83 for ; Tue, 23 May 2023 19:04:44 +0200 (CEST) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id A710A40A80; Tue, 23 May 2023 19:04:44 +0200 (CEST) Received: from mail-qt1-f182.google.com (mail-qt1-f182.google.com [209.85.160.182]) by mails.dpdk.org (Postfix) with ESMTP id D52F842D40 for ; Tue, 23 May 2023 19:04:43 +0200 (CEST) Received: by mail-qt1-f182.google.com with SMTP id d75a77b69052e-3f6a5c7e034so34717261cf.2 for ; Tue, 23 May 2023 10:04:43 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=iol.unh.edu; s=unh-iol; t=1684861483; x=1687453483; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=2OF8aHWw6B8vrIAooJsIUBE7WOB0+ECdC6iktUiBLKo=; b=ah80N1+E0Khd9P0IeJlRCRCKG2kNkjfnlanzGMWMUfr/0ZelcSlPzZptC7paO4THOh Pyxs9El2NnK9TWyioMB8MSbI13ujfoDbNTcrIC0vRI96x7tWbf1Olq2iW93EPyqnNq6n 43RJEb3EEBv4POHIFAZM8rlGeVLV85YFmfiPY= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20221208; t=1684861483; x=1687453483; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=2OF8aHWw6B8vrIAooJsIUBE7WOB0+ECdC6iktUiBLKo=; b=Uu3YsZxo+qqMDylvfXNZ4F5ISKwPZeepJpaq1y5OVMxvntxZyjG9s0TbYDQDV+YHoc 1YECRXJBo7CZiqbv3kU4aXFEXjZyMA+OJuARDKmpFwU0ZtUoZTrIaSe4XyyIAQhUPAml U31T2G94wM2xdZuALFhvZkgwOyVpKH/iRGo30UV20uHaW7iPqB1mglwgn+Wykwsr8Sec iCWr6V3zQMxRn3lQ2vAEN2iNj6AdQr3PTlgaSlYdTEbV5Rd51DmUgS9FLVjlFc2t5tYc oUYPvtupPob1mUqJMoy/vU0rjfJ7DCp1p77VIfxAbDqT0be4xToFurdVGyK3RPzxsLa+ g1Pg== X-Gm-Message-State: AC+VfDw9xjL1Z0rYTZjQGiB9iKo0H8Q11KcNvddV94fsfbcNA+ow62RJ 2vJ6S46I44y4F0gpGKhrWrtyQ2+aKt6fP0oxaX6r4nyxiDHWnYgTqMb9u/SQ48QIRIgsm7tcs0M pHTVXEX4srIelI3F8eQhYgN89GXb0VixgfJbUPLQ01OKkyJthe2lsIEB5MBdjZg== X-Google-Smtp-Source: ACHHUZ7WhoNL7l1VtBtiJZL4+YJqV85llFgeLY/6YUlm5vpjYnzPv1Q/rEnEJLr/jJdZHEOiZLBFiw== X-Received: by 2002:ac8:5c86:0:b0:3f6:b12f:fae0 with SMTP id r6-20020ac85c86000000b003f6b12ffae0mr8845903qta.62.1684861482811; Tue, 23 May 2023 10:04:42 -0700 (PDT) Received: from ahassick-Desktop.iol.unh.edu ([2606:4100:3880:1220:7704:829d:adaa:3294]) by smtp.gmail.com with ESMTPSA id g14-20020ac8774e000000b003e38c9a2a22sm2996553qtu.92.2023.05.23.10.04.41 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Tue, 23 May 2023 10:04:42 -0700 (PDT) From: Adam Hassick To: ci@dpdk.org Cc: Adam Hassick , aconole@redhat.com, alialnu@nvidia.com Subject: [PATCH v5 08/11] containers/builder: Add arguments to templating script Date: Tue, 23 May 2023 13:04:10 -0400 Message-Id: <20230523170413.812922-9-ahassick@iol.unh.edu> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20230523170413.812922-1-ahassick@iol.unh.edu> References: <20230523170413.812922-1-ahassick@iol.unh.edu> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-BeenThere: ci@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK CI discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: ci-bounces@dpdk.org Adds arguments to the script to support builder mode, overriding the date tag, expecting Coverity Scan, and limiting the scope of target architectures. Signed-off-by: Adam Hassick --- containers/template_engine/make_dockerfile.py | 134 ++++++++++++++++-- 1 file changed, 126 insertions(+), 8 deletions(-) diff --git a/containers/template_engine/make_dockerfile.py b/containers/template_engine/make_dockerfile.py index 9a3c19b..60da2a8 100755 --- a/containers/template_engine/make_dockerfile.py +++ b/containers/template_engine/make_dockerfile.py @@ -5,8 +5,10 @@ import argparse import json import logging import os +import re from dataclasses import dataclass from datetime import datetime +import platform from typing import Any, Dict, List, Optional import jsonschema @@ -18,10 +20,16 @@ from jinja2 import Environment, FileSystemLoader, select_autoescape class Options: on_rhel: bool fail_on_unbuildable: bool + has_coverity: bool build_libabigail: bool build_abi: bool output_dir: str registry_hostname: str + host_arch_only: bool + omit_latest: bool + is_builder: bool + date_override: Optional[str] + ninja_workers: Optional[int] def _get_arg_parser() -> argparse.ArgumentParser: @@ -52,6 +60,39 @@ def _get_arg_parser() -> argparse.ArgumentParser: help="Whether to build libabigail from source for distros that do not \ package it. Implied by '--build-abi'", ) + parser.add_argument( + "--host-arch-only", + action="store_true", + help="Only build containers for the architecture of the host system", + ) + parser.add_argument( + "--omit-latest", + action="store_true", + help="Whether to include the \"latest\" tag in the generated makefile." + ) + parser.add_argument( + "--builder-mode", + action="store_true", + help="Specifies that the makefile is being templated for a builder. \ + This implicitly sets \"--host-arch-only\" to true and disables making the manifests.", + default=False + ) + parser.add_argument( + "--date", + type=str, + help="Overrides generation of the timestamp and uses the provided string instead." + ) + parser.add_argument( + "--ninja-workers", + type=int, + help="Specifies a number of ninja workers to limit builds to. Uses the ninja default when not given." + ) + parser.add_argument( + "--coverity", + action="store_true", + help="Whether the Coverity Scan binaries are available for building the Coverity containers.", + default=False + ) return parser @@ -74,7 +115,14 @@ def parse_args() -> Options: build_abi=args.build_abi, output_dir=args.output_dir, registry_hostname=registry_hostname, + host_arch_only=args.host_arch_only or args.builder_mode, + omit_latest=args.omit_latest, + is_builder=args.builder_mode, + date_override=args.date, + ninja_workers=args.ninja_workers, + has_coverity=args.coverity ) + logging.info(f"make_dockerfile.py options: {opts}") return opts @@ -169,14 +217,71 @@ def apply_group_config_to_target( return target +def apply_defaults_to_target(target: Dict[str, Any]) -> Dict[str, Any]: + def default_if_unset(target: Dict[str, Any], key: str, value: Any) -> Dict[str, Any]: + if key not in target: + target[key] = value -def get_processed_inventory(options: Options) -> Dict[str, Any]: + return target + + target = default_if_unset(target, "requires_coverity", False) + target = default_if_unset(target, "force_disable_abi", False) + target = default_if_unset(target, "minimum_dpdk_version", dict(major=0, minor=0, revision=0)) + target = default_if_unset(target, "extra_information", {}) + + return target + +def get_host_arch() -> str: + machine: str = platform.machine() + match machine: + case "aarch64" | "armv8b" | "armv8l": + return "linux/arm64" + case "ppc64le": + return "linux/ppc64le" + case "x86_64" | "x64" | "amd64": + return "linux/amd64" + case arch: + raise ValueError(f"Unknown arch {arch}") + +def process_target( + target: Dict[str, Any], + raw_inventory: Dict[str, Any], + has_coverity: bool, + on_rhel: bool, + fail_on_unbuildable: bool, + host_arch_only: bool, + build_timestamp: str +) -> Optional[Dict[str, Any]]: + target = apply_defaults_to_target(target) + # Copy the platforms, for building the manifest list. + + # Write the build timestamp. + target["extra_information"].update({ + "build_timestamp": build_timestamp + }) + + if (not has_coverity) and target["requires_coverity"]: + print(f"Disabling {target['name']}. Target requires Coverity, and it is not enabled.") + return None + + if host_arch_only: + host_arch = get_host_arch() + if host_arch in target["platforms"]: + target["platforms"] = [host_arch] + else: + return None + + return apply_group_config_to_target( + target, raw_inventory, on_rhel, fail_on_unbuildable + ) + +def get_processed_inventory(options: Options, build_timestamp: str) -> Dict[str, Any]: raw_inventory: Dict[str, Any] = get_raw_inventory() on_rhel = running_on_RHEL(options) targets = raw_inventory["dockerfiles"]["targets"] targets = [ - apply_group_config_to_target( - target, raw_inventory, on_rhel, options.fail_on_unbuildable + process_target( + target, raw_inventory, options.has_coverity, on_rhel, options.fail_on_unbuildable, options.host_arch_only, build_timestamp ) for target in targets ] @@ -194,9 +299,14 @@ def main(): loader=FileSystemLoader("templates"), ) - inventory = get_processed_inventory(options) + build_timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") - timestamp = datetime.now().strftime("%Y-%m-%d") + inventory = get_processed_inventory(options, build_timestamp) + + if options.date_override: + timestamp = options.date_override + else: + timestamp = datetime.now().strftime("%Y-%m-%d") for target in inventory["dockerfiles"]["targets"]: template = env.get_template(f"containers/{target['group']}.dockerfile.j2") @@ -205,9 +315,13 @@ def main(): ) tags: list[str] = target.get("extra_tags") or [] - tags.insert(0, "$R/$N:latest") - tags.insert(1, "$R/$N:$T") - + + tags.insert(0, "$R/$N:$T") + if not options.omit_latest: + tags.insert(0, "$R/$N:latest") + else: + tags = list(filter(lambda x: re.match('^.*:latest$', x) is None, tags)) + target["tags"] = tags rendered_dockerfile = template.render( @@ -215,7 +329,9 @@ def main(): target=target, build_libabigail=options.build_libabigail, build_abi=options.build_abi, + build_timestamp=build_timestamp, registry_hostname=options.registry_hostname, + ninja_workers=options.ninja_workers, **inventory, ) with open(dockerfile_location, "w") as output_file: @@ -226,7 +342,9 @@ def main(): timestamp=timestamp, build_libabigail=options.build_libabigail, build_abi=options.build_abi, + host_arch_only=options.host_arch_only, registry_hostname=options.registry_hostname, + is_builder=options.is_builder, **inventory, ) makefile_output_path = os.path.join(options.output_dir, "Makefile") -- 2.34.1