From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 18A4742FD3; Fri, 4 Aug 2023 15:59:52 +0200 (CEST) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 1018F40EE1; Fri, 4 Aug 2023 15:59:52 +0200 (CEST) Received: from us-smtp-delivery-124.mimecast.com (us-smtp-delivery-124.mimecast.com [170.10.133.124]) by mails.dpdk.org (Postfix) with ESMTP id C61B940E2D for ; Fri, 4 Aug 2023 15:59:49 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=redhat.com; s=mimecast20190719; t=1691157589; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version:content-type:content-type: in-reply-to:in-reply-to:references:references; bh=8AqKq6WZuUflOt9XQikAMNXhDBZvRXD9FG6kOPqLqnc=; b=UFDptVReFRdLInOAyjoN4OqjmEULX6WKNp8UMGolkBRm/TdJEE1XpR0BGW4PnYlSV94TCO 2C4RMvsmvje5k5tLXypn8b76EFmqPMA9opxaTh2F95Q7ULRn34ACUz3KghMtqA9gg0XwZc QDl9VhF3FB392DseUvDb6jwpL1Wogi8= Received: from mimecast-mx02.redhat.com (66.187.233.73 [66.187.233.73]) by relay.mimecast.com with ESMTP with STARTTLS (version=TLSv1.2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id us-mta-520-wWoELfVjPtyrsGliojWAuQ-1; Fri, 04 Aug 2023 09:59:48 -0400 X-MC-Unique: wWoELfVjPtyrsGliojWAuQ-1 Received: from smtp.corp.redhat.com (int-mx10.intmail.prod.int.rdu2.redhat.com [10.11.54.10]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mimecast-mx02.redhat.com (Postfix) with ESMTPS id 9B56E3C0D191; Fri, 4 Aug 2023 13:59:47 +0000 (UTC) Received: from RHTPC1VM0NT (unknown [10.22.33.16]) by smtp.corp.redhat.com (Postfix) with ESMTPS id 47E93477F63; Fri, 4 Aug 2023 13:59:47 +0000 (UTC) From: Aaron Conole To: Adam Hassick Cc: ci@dpdk.org, alialnu@nvidia.com, Owen Hilyard Subject: Re: [PATCH v8 3/6] containers/builder: Dockerfile creation script References: <20230717210815.29737-1-ahassick@iol.unh.edu> <20230717210815.29737-4-ahassick@iol.unh.edu> Date: Fri, 04 Aug 2023 09:59:46 -0400 In-Reply-To: <20230717210815.29737-4-ahassick@iol.unh.edu> (Adam Hassick's message of "Mon, 17 Jul 2023 17:08:12 -0400") Message-ID: User-Agent: Gnus/5.13 (Gnus v5.13) Emacs/28.2 (gnu/linux) MIME-Version: 1.0 X-Scanned-By: MIMEDefang 3.1 on 10.11.54.10 X-Mimecast-Spam-Score: 0 X-Mimecast-Originator: redhat.com Content-Type: text/plain X-BeenThere: ci@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK CI discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: ci-bounces@dpdk.org Adam Hassick writes: > From: Owen Hilyard > > This script will template out all of the Dockerfiles based on the > definitions provided in the inventory using the jinja2 templating > library. > > Signed-off-by: Owen Hilyard > Signed-off-by: Adam Hassick > --- Please run this through black. It has some formatting errors. > containers/template_engine/make_dockerfile.py | 358 ++++++++++++++++++ > 1 file changed, 358 insertions(+) > create mode 100755 containers/template_engine/make_dockerfile.py > > diff --git a/containers/template_engine/make_dockerfile.py b/containers/template_engine/make_dockerfile.py > new file mode 100755 > index 0000000..60269a0 > --- /dev/null > +++ b/containers/template_engine/make_dockerfile.py > @@ -0,0 +1,358 @@ > +#!/usr/bin/env python3 > +# SPDX-License-Identifier: BSD-3-Clause > +# Copyright (c) 2022 University of New Hampshire > +import argparse > +import json > +import logging > +import os > +import re > +from dataclasses import dataclass > +from datetime import datetime > +import platform > +from typing import Any, Dict, List, Optional > + > +import jsonschema > +import yaml > +from jinja2 import Environment, FileSystemLoader, select_autoescape > + > + > +@dataclass(frozen=True) > +class Options: > + on_rhel: bool > + fail_on_unbuildable: bool > + has_coverity: bool > + build_libabigail: bool > + build_abi: bool > + output_dir: str > + registry_hostname: str > + host_arch_only: bool > + omit_latest: bool > + is_builder: bool > + date_override: Optional[str] > + ninja_workers: Optional[int] > + > + > +def _get_arg_parser() -> argparse.ArgumentParser: > + parser = argparse.ArgumentParser(description="Makes the dockerfile") > + parser.add_argument("--output-dir", required=True) > + parser.add_argument( > + "--rhel", > + action="store_true", > + help="Overwrite the check for running on RHEL", > + default=False, > + ) > + parser.add_argument( > + "--fail-on-unbuildable", > + action="store_true", > + help="If any container would not be possible to build, fail and exit with a non-zero exit code.", > + default=False, > + ) > + parser.add_argument( > + "--build-abi", > + action="store_true", > + help="Whether to build the ABI references into the image. Disabled by \ > + default due to producing 10+ GB images. \ > + Implies '--build-libabigail'.", > + ) > + parser.add_argument( > + "--build-libabigail", > + action="store_true", > + help="Whether to build libabigail from source for distros that do not \ > + package it. Implied by '--build-abi'", > + ) > + parser.add_argument( > + "--host-arch-only", > + action="store_true", > + help="Only build containers for the architecture of the host system", > + ) > + parser.add_argument( > + "--omit-latest", > + action="store_true", > + help="Whether to include the \"latest\" tag in the generated makefile." > + ) > + parser.add_argument( > + "--builder-mode", > + action="store_true", > + help="Specifies that the makefile is being templated for a builder. \ > + This implicitly sets \"--host-arch-only\" to true and disables making the manifests.", > + default=False > + ) > + parser.add_argument( > + "--date", > + type=str, > + help="Overrides generation of the timestamp and uses the provided string instead." > + ) > + parser.add_argument( > + "--ninja-workers", > + type=int, > + help="Specifies a number of ninja workers to limit builds to. Uses the ninja default when not given." > + ) > + parser.add_argument( > + "--coverity", > + action="store_true", > + help="Whether the Coverity Scan binaries are available for building the Coverity containers.", > + default=False > + ) > + return parser > + > + > +def parse_args() -> Options: > + parser = _get_arg_parser() > + args = parser.parse_args() > + > + registry_hostname = ( > + os.environ.get("DPDK_CI_CONTAINERS_REGISTRY_HOSTNAME") or "localhost" > + ) > + > + # In order to to build the ABIs, libabigail must be built from source on > + # some platforms > + build_libabigail: bool = args.build_libabigail or args.build_abi > + > + opts = Options( > + on_rhel=args.rhel, > + fail_on_unbuildable=args.fail_on_unbuildable, > + build_libabigail=build_libabigail, > + build_abi=args.build_abi, > + output_dir=args.output_dir, > + registry_hostname=registry_hostname, > + host_arch_only=args.host_arch_only or args.builder_mode, > + omit_latest=args.omit_latest, > + is_builder=args.builder_mode, > + date_override=args.date, > + ninja_workers=args.ninja_workers, > + has_coverity=args.coverity > + ) > + > + logging.info(f"make_dockerfile.py options: {opts}") > + return opts > + > + > +def running_on_RHEL(options: Options) -> bool: > + """ > + RHEL containers can only be built on RHEL, so disable them and emit a > + warning if not on RHEL. > + """ > + redhat_release_path = "/etc/redhat-release" > + > + if os.path.exists(redhat_release_path): > + with open(redhat_release_path) as f: > + first_line = f.readline() > + on_rhel = "Red Hat Enterprise Linux" in first_line > + if on_rhel: > + logging.info("Running on RHEL, allowing RHEL containers") > + return True > + > + logging.warning("Not on RHEL, disabling RHEL containers") > + assert options is not None, "Internal state error, OPTIONS should not be None" > + > + if options.on_rhel: > + logging.info("Override enabled, enabling RHEL containers") > + > + return options.on_rhel > + > + > +def get_path_to_parent_directory() -> str: > + return os.path.dirname(__file__) > + > + > +def get_raw_inventory(): > + parent_dir = get_path_to_parent_directory() > + > + schema_path = os.path.join(parent_dir, "inventory_schema.json") > + inventory_path = os.path.join(parent_dir, "inventory.yaml") > + > + inventory: Dict[str, Any] > + with open(inventory_path, "r") as f: > + inventory = yaml.safe_load(f) > + > + schema: Dict[str, Any] > + with open(schema_path, "r") as f: > + schema = json.load(f) > + > + jsonschema.validate(instance=inventory, schema=schema) > + return inventory > + > + > +def apply_group_config_to_target( > + target: Dict[str, Any], > + raw_inventory: Dict[str, Any], > + on_rhel: bool, > + fail_on_unbuildable: bool, > +) -> Optional[Dict[str, Any]]: > + groups_for_target: List[Dict[str, Any]] = [] > + groups: List[Dict[str, Any]] = raw_inventory["dockerfiles"]["groups"] > + group = groups[target["group"]] > + > + target_primary_group = target["group"] > + > + assert isinstance(target_primary_group, str), "Target group name was not a string" > + > + requires_rhel = "rhel" in target_primary_group.lower() > + > + if requires_rhel and not on_rhel: > + logging.warning( > + f"Disabling target {target['name']}, because it must be built on RHEL." > + ) > + if fail_on_unbuildable: > + raise AssertionError( > + f"Not on RHEL and target {target['name']} must be built on RHEL" > + ) > + > + return None > + > + while group["parent"] != "NONE": > + groups_for_target.append(group) > + group = groups[group["parent"]] > + > + groups_for_target.append(group) # add the "all" group > + groups_for_target.reverse() # reverse it so overrides work > + > + target_packages: List[str] = target.get("packages") or [] > + > + for group in groups_for_target: > + target_packages = [*target_packages, *(group.get("packages") or [])] > + target = dict(target, **group) > + > + target["packages"] = target_packages > + > + return target > + > +def apply_defaults_to_target(target: Dict[str, Any]) -> Dict[str, Any]: > + def default_if_unset(target: Dict[str, Any], key: str, value: Any) -> Dict[str, Any]: > + if key not in target: > + target[key] = value > + > + return target > + > + target = default_if_unset(target, "requires_coverity", False) > + target = default_if_unset(target, "force_disable_abi", False) > + target = default_if_unset(target, "minimum_dpdk_version", dict(major=0, minor=0, revision=0)) > + target = default_if_unset(target, "extra_information", {}) > + > + return target > + > +def get_host_arch() -> str: > + machine: str = platform.machine() > + match machine: > + case "aarch64" | "armv8b" | "armv8l": > + return "linux/arm64" > + case "ppc64le": > + return "linux/ppc64le" > + case "x86_64" | "x64" | "amd64": > + return "linux/amd64" > + case arch: > + raise ValueError(f"Unknown arch {arch}") > + > +def process_target( > + target: Dict[str, Any], > + raw_inventory: Dict[str, Any], > + has_coverity: bool, > + on_rhel: bool, > + fail_on_unbuildable: bool, > + host_arch_only: bool, > + build_timestamp: str > +) -> Optional[Dict[str, Any]]: > + target = apply_defaults_to_target(target) > + # Copy the platforms, for building the manifest list. > + > + # Write the build timestamp. > + target["extra_information"].update({ > + "build_timestamp": build_timestamp > + }) > + > + if (not has_coverity) and target["requires_coverity"]: > + print(f"Disabling {target['name']}. Target requires Coverity, and it is not enabled.") > + return None > + > + if host_arch_only: > + host_arch = get_host_arch() > + if host_arch in target["platforms"]: > + target["platforms"] = [host_arch] > + else: > + return None > + > + return apply_group_config_to_target( > + target, raw_inventory, on_rhel, fail_on_unbuildable > + ) > + > +def get_processed_inventory(options: Options, build_timestamp: str) -> Dict[str, Any]: > + raw_inventory: Dict[str, Any] = get_raw_inventory() > + on_rhel = running_on_RHEL(options) > + targets = raw_inventory["dockerfiles"]["targets"] > + targets = [ > + process_target( > + target, raw_inventory, options.has_coverity, on_rhel, options.fail_on_unbuildable, options.host_arch_only, build_timestamp > + ) > + for target in targets > + ] > + # remove disabled options > + targets = [target for target in targets if target is not None] > + raw_inventory["dockerfiles"]["targets"] = targets > + > + return raw_inventory > + > + > +def main(): > + options: Options = parse_args() > + > + env = Environment( > + loader=FileSystemLoader("templates"), > + ) > + > + build_timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") > + > + inventory = get_processed_inventory(options, build_timestamp) > + > + if options.date_override: > + timestamp = options.date_override > + else: > + timestamp = datetime.now().strftime("%Y-%m-%d") > + > + for target in inventory["dockerfiles"]["targets"]: > + template = env.get_template(f"containers/{target['group']}.dockerfile.j2") > + dockerfile_location = os.path.join( > + options.output_dir, target["name"] + ".dockerfile" > + ) > + > + tags: list[str] = target.get("extra_tags") or [] > + > + tags.insert(0, "$R/$N:$T") > + if not options.omit_latest: > + tags.insert(0, "$R/$N:latest") > + else: > + tags = list(filter(lambda x: re.match('^.*:latest$', x) is None, tags)) > + > + target["tags"] = tags > + > + rendered_dockerfile = template.render( > + timestamp=timestamp, > + target=target, > + build_libabigail=options.build_libabigail, > + build_abi=options.build_abi, > + build_timestamp=build_timestamp, > + registry_hostname=options.registry_hostname, > + ninja_workers=options.ninja_workers, > + **inventory, > + ) > + with open(dockerfile_location, "w") as output_file: > + output_file.write(rendered_dockerfile) > + > + makefile_template = env.get_template(f"containers.makefile.j2") > + rendered_makefile = makefile_template.render( > + timestamp=timestamp, > + build_libabigail=options.build_libabigail, > + build_abi=options.build_abi, > + host_arch_only=options.host_arch_only, > + registry_hostname=options.registry_hostname, > + is_builder=options.is_builder, > + **inventory, > + ) > + makefile_output_path = os.path.join(options.output_dir, "Makefile") > + with open(makefile_output_path, "w") as f: > + f.write(rendered_makefile) > + > + > +if __name__ == "__main__": > + logging.basicConfig() > + logging.root.setLevel(0) # log everything > + main()