[PATCH v4 3/6] containers/builder: Dockerfile creation script

[ohilyard@iol.unh.edu]

From: Owen Hilyard <ohilyard@iol.unh.edu>

This script will template out all of the Dockerfiles based on the
definitions provided in the inventory using the jinja2 templating
library.

Signed-off-by: Owen Hilyard <ohilyard@iol.unh.edu>
---
 containers/template_engine/make_dockerfile.py | 240 ++++++++++++++++++
 1 file changed, 240 insertions(+)
 create mode 100755 containers/template_engine/make_dockerfile.py

diff --git a/containers/template_engine/make_dockerfile.py b/containers/template_engine/make_dockerfile.py
new file mode 100755
index 0000000..9a3c19b
--- /dev/null
+++ b/containers/template_engine/make_dockerfile.py
@@ -0,0 +1,240 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2022 University of New Hampshire
+import argparse
+import json
+import logging
+import os
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Any, Dict, List, Optional
+
+import jsonschema
+import yaml
+from jinja2 import Environment, FileSystemLoader, select_autoescape
+
+
+@dataclass(frozen=True)
+class Options:
+    on_rhel: bool
+    fail_on_unbuildable: bool
+    build_libabigail: bool
+    build_abi: bool
+    output_dir: str
+    registry_hostname: str
+
+
+def _get_arg_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description="Makes the dockerfile")
+    parser.add_argument("--output-dir", required=True)
+    parser.add_argument(
+        "--rhel",
+        action="store_true",
+        help="Overwrite the check for running on RHEL",
+        default=False,
+    )
+    parser.add_argument(
+        "--fail-on-unbuildable",
+        action="store_true",
+        help="If any container would not be possible to build, fail and exit with a non-zero exit code.",
+        default=False,
+    )
+    parser.add_argument(
+        "--build-abi",
+        action="store_true",
+        help="Whether to build the ABI references into the image. Disabled by \
+            default due to producing 10+ GB images. \
+            Implies '--build-libabigail'.",
+    )
+    parser.add_argument(
+        "--build-libabigail",
+        action="store_true",
+        help="Whether to build libabigail from source for distros that do not \
+            package it. Implied by '--build-abi'",
+    )
+    return parser
+
+
+def parse_args() -> Options:
+    parser = _get_arg_parser()
+    args = parser.parse_args()
+
+    registry_hostname = (
+        os.environ.get("DPDK_CI_CONTAINERS_REGISTRY_HOSTNAME") or "localhost"
+    )
+
+    # In order to to build the ABIs, libabigail must be built from source on
+    # some platforms
+    build_libabigail: bool = args.build_libabigail or args.build_abi
+
+    opts = Options(
+        on_rhel=args.rhel,
+        fail_on_unbuildable=args.fail_on_unbuildable,
+        build_libabigail=build_libabigail,
+        build_abi=args.build_abi,
+        output_dir=args.output_dir,
+        registry_hostname=registry_hostname,
+    )
+    logging.info(f"make_dockerfile.py options: {opts}")
+    return opts
+
+
+def running_on_RHEL(options: Options) -> bool:
+    """
+    RHEL containers can only be built on RHEL, so disable them and emit a
+    warning if not on RHEL.
+    """
+    redhat_release_path = "/etc/redhat-release"
+
+    if os.path.exists(redhat_release_path):
+        with open(redhat_release_path) as f:
+            first_line = f.readline()
+            on_rhel = "Red Hat Enterprise Linux" in first_line
+            if on_rhel:
+                logging.info("Running on RHEL, allowing RHEL containers")
+                return True
+
+    logging.warning("Not on RHEL, disabling RHEL containers")
+    assert options is not None, "Internal state error, OPTIONS should not be None"
+
+    if options.on_rhel:
+        logging.info("Override enabled, enabling RHEL containers")
+
+    return options.on_rhel
+
+
+def get_path_to_parent_directory() -> str:
+    return os.path.dirname(__file__)
+
+
+def get_raw_inventory():
+    parent_dir = get_path_to_parent_directory()
+
+    schema_path = os.path.join(parent_dir, "inventory_schema.json")
+    inventory_path = os.path.join(parent_dir, "inventory.yaml")
+
+    inventory: Dict[str, Any]
+    with open(inventory_path, "r") as f:
+        inventory = yaml.safe_load(f)
+
+    schema: Dict[str, Any]
+    with open(schema_path, "r") as f:
+        schema = json.load(f)
+
+    jsonschema.validate(instance=inventory, schema=schema)
+    return inventory
+
+
+def apply_group_config_to_target(
+    target: Dict[str, Any],
+    raw_inventory: Dict[str, Any],
+    on_rhel: bool,
+    fail_on_unbuildable: bool,
+) -> Optional[Dict[str, Any]]:
+    groups_for_target: List[Dict[str, Any]] = []
+    groups: List[Dict[str, Any]] = raw_inventory["dockerfiles"]["groups"]
+    group = groups[target["group"]]
+
+    target_primary_group = target["group"]
+
+    assert isinstance(target_primary_group, str), "Target group name was not a string"
+
+    requires_rhel = "rhel" in target_primary_group.lower()
+
+    if requires_rhel and not on_rhel:
+        logging.warning(
+            f"Disabling target {target['name']}, because it must be built on RHEL."
+        )
+        if fail_on_unbuildable:
+            raise AssertionError(
+                f"Not on RHEL and target {target['name']} must be built on RHEL"
+            )
+
+        return None
+
+    while group["parent"] != "NONE":
+        groups_for_target.append(group)
+        group = groups[group["parent"]]
+
+    groups_for_target.append(group)  # add the "all" group
+    groups_for_target.reverse()  # reverse it so overrides work
+
+    target_packages: List[str] = target.get("packages") or []
+
+    for group in groups_for_target:
+        target_packages = [*target_packages, *(group.get("packages") or [])]
+        target = dict(target, **group)
+
+    target["packages"] = target_packages
+
+    return target
+
+
+def get_processed_inventory(options: Options) -> Dict[str, Any]:
+    raw_inventory: Dict[str, Any] = get_raw_inventory()
+    on_rhel = running_on_RHEL(options)
+    targets = raw_inventory["dockerfiles"]["targets"]
+    targets = [
+        apply_group_config_to_target(
+            target, raw_inventory, on_rhel, options.fail_on_unbuildable
+        )
+        for target in targets
+    ]
+    # remove disabled options
+    targets = [target for target in targets if target is not None]
+    raw_inventory["dockerfiles"]["targets"] = targets
+
+    return raw_inventory
+
+
+def main():
+    options: Options = parse_args()
+
+    env = Environment(
+        loader=FileSystemLoader("templates"),
+    )
+
+    inventory = get_processed_inventory(options)
+
+    timestamp = datetime.now().strftime("%Y-%m-%d")
+
+    for target in inventory["dockerfiles"]["targets"]:
+        template = env.get_template(f"containers/{target['group']}.dockerfile.j2")
+        dockerfile_location = os.path.join(
+            options.output_dir, target["name"] + ".dockerfile"
+        )
+
+        tags: list[str] = target.get("extra_tags") or []
+        tags.insert(0, "$R/$N:latest")
+        tags.insert(1, "$R/$N:$T")
+
+        target["tags"] = tags
+
+        rendered_dockerfile = template.render(
+            timestamp=timestamp,
+            target=target,
+            build_libabigail=options.build_libabigail,
+            build_abi=options.build_abi,
+            registry_hostname=options.registry_hostname,
+            **inventory,
+        )
+        with open(dockerfile_location, "w") as output_file:
+            output_file.write(rendered_dockerfile)
+
+    makefile_template = env.get_template(f"containers.makefile.j2")
+    rendered_makefile = makefile_template.render(
+        timestamp=timestamp,
+        build_libabigail=options.build_libabigail,
+        build_abi=options.build_abi,
+        registry_hostname=options.registry_hostname,
+        **inventory,
+    )
+    makefile_output_path = os.path.join(options.output_dir, "Makefile")
+    with open(makefile_output_path, "w") as f:
+        f.write(rendered_makefile)
+
+
+if __name__ == "__main__":
+    logging.basicConfig()
+    logging.root.setLevel(0)  # log everything
+    main()
-- 
2.34.1