From: Adam Hassick <ahassick@iol.unh.edu>
To: ci@dpdk.org
Cc: Adam Hassick <ahassick@iol.unh.edu>,
aconole@redhat.com, alialnu@nvidia.com
Subject: [PATCH v5 08/11] containers/builder: Add arguments to templating script
Date: Tue, 23 May 2023 13:04:10 -0400 [thread overview]
Message-ID: <20230523170413.812922-9-ahassick@iol.unh.edu> (raw)
In-Reply-To: <20230523170413.812922-1-ahassick@iol.unh.edu>
Adds arguments to the script to support builder mode,
overriding the date tag, expecting Coverity Scan, and
limiting the scope of target architectures.
Signed-off-by: Adam Hassick <ahassick@iol.unh.edu>
---
containers/template_engine/make_dockerfile.py | 134 ++++++++++++++++--
1 file changed, 126 insertions(+), 8 deletions(-)
diff --git a/containers/template_engine/make_dockerfile.py b/containers/template_engine/make_dockerfile.py
index 9a3c19b..60da2a8 100755
--- a/containers/template_engine/make_dockerfile.py
+++ b/containers/template_engine/make_dockerfile.py
@@ -5,8 +5,10 @@ import argparse
import json
import logging
import os
+import re
from dataclasses import dataclass
from datetime import datetime
+import platform
from typing import Any, Dict, List, Optional
import jsonschema
@@ -18,10 +20,16 @@ from jinja2 import Environment, FileSystemLoader, select_autoescape
class Options:
on_rhel: bool
fail_on_unbuildable: bool
+ has_coverity: bool
build_libabigail: bool
build_abi: bool
output_dir: str
registry_hostname: str
+ host_arch_only: bool
+ omit_latest: bool
+ is_builder: bool
+ date_override: Optional[str]
+ ninja_workers: Optional[int]
def _get_arg_parser() -> argparse.ArgumentParser:
@@ -52,6 +60,39 @@ def _get_arg_parser() -> argparse.ArgumentParser:
help="Whether to build libabigail from source for distros that do not \
package it. Implied by '--build-abi'",
)
+ parser.add_argument(
+ "--host-arch-only",
+ action="store_true",
+ help="Only build containers for the architecture of the host system",
+ )
+ parser.add_argument(
+ "--omit-latest",
+ action="store_true",
+ help="Whether to include the \"latest\" tag in the generated makefile."
+ )
+ parser.add_argument(
+ "--builder-mode",
+ action="store_true",
+ help="Specifies that the makefile is being templated for a builder. \
+ This implicitly sets \"--host-arch-only\" to true and disables making the manifests.",
+ default=False
+ )
+ parser.add_argument(
+ "--date",
+ type=str,
+ help="Overrides generation of the timestamp and uses the provided string instead."
+ )
+ parser.add_argument(
+ "--ninja-workers",
+ type=int,
+ help="Specifies a number of ninja workers to limit builds to. Uses the ninja default when not given."
+ )
+ parser.add_argument(
+ "--coverity",
+ action="store_true",
+ help="Whether the Coverity Scan binaries are available for building the Coverity containers.",
+ default=False
+ )
return parser
@@ -74,7 +115,14 @@ def parse_args() -> Options:
build_abi=args.build_abi,
output_dir=args.output_dir,
registry_hostname=registry_hostname,
+ host_arch_only=args.host_arch_only or args.builder_mode,
+ omit_latest=args.omit_latest,
+ is_builder=args.builder_mode,
+ date_override=args.date,
+ ninja_workers=args.ninja_workers,
+ has_coverity=args.coverity
)
+
logging.info(f"make_dockerfile.py options: {opts}")
return opts
@@ -169,14 +217,71 @@ def apply_group_config_to_target(
return target
+def apply_defaults_to_target(target: Dict[str, Any]) -> Dict[str, Any]:
+ def default_if_unset(target: Dict[str, Any], key: str, value: Any) -> Dict[str, Any]:
+ if key not in target:
+ target[key] = value
-def get_processed_inventory(options: Options) -> Dict[str, Any]:
+ return target
+
+ target = default_if_unset(target, "requires_coverity", False)
+ target = default_if_unset(target, "force_disable_abi", False)
+ target = default_if_unset(target, "minimum_dpdk_version", dict(major=0, minor=0, revision=0))
+ target = default_if_unset(target, "extra_information", {})
+
+ return target
+
+def get_host_arch() -> str:
+ machine: str = platform.machine()
+ match machine:
+ case "aarch64" | "armv8b" | "armv8l":
+ return "linux/arm64"
+ case "ppc64le":
+ return "linux/ppc64le"
+ case "x86_64" | "x64" | "amd64":
+ return "linux/amd64"
+ case arch:
+ raise ValueError(f"Unknown arch {arch}")
+
+def process_target(
+ target: Dict[str, Any],
+ raw_inventory: Dict[str, Any],
+ has_coverity: bool,
+ on_rhel: bool,
+ fail_on_unbuildable: bool,
+ host_arch_only: bool,
+ build_timestamp: str
+) -> Optional[Dict[str, Any]]:
+ target = apply_defaults_to_target(target)
+ # Copy the platforms, for building the manifest list.
+
+ # Write the build timestamp.
+ target["extra_information"].update({
+ "build_timestamp": build_timestamp
+ })
+
+ if (not has_coverity) and target["requires_coverity"]:
+ print(f"Disabling {target['name']}. Target requires Coverity, and it is not enabled.")
+ return None
+
+ if host_arch_only:
+ host_arch = get_host_arch()
+ if host_arch in target["platforms"]:
+ target["platforms"] = [host_arch]
+ else:
+ return None
+
+ return apply_group_config_to_target(
+ target, raw_inventory, on_rhel, fail_on_unbuildable
+ )
+
+def get_processed_inventory(options: Options, build_timestamp: str) -> Dict[str, Any]:
raw_inventory: Dict[str, Any] = get_raw_inventory()
on_rhel = running_on_RHEL(options)
targets = raw_inventory["dockerfiles"]["targets"]
targets = [
- apply_group_config_to_target(
- target, raw_inventory, on_rhel, options.fail_on_unbuildable
+ process_target(
+ target, raw_inventory, options.has_coverity, on_rhel, options.fail_on_unbuildable, options.host_arch_only, build_timestamp
)
for target in targets
]
@@ -194,9 +299,14 @@ def main():
loader=FileSystemLoader("templates"),
)
- inventory = get_processed_inventory(options)
+ build_timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
- timestamp = datetime.now().strftime("%Y-%m-%d")
+ inventory = get_processed_inventory(options, build_timestamp)
+
+ if options.date_override:
+ timestamp = options.date_override
+ else:
+ timestamp = datetime.now().strftime("%Y-%m-%d")
for target in inventory["dockerfiles"]["targets"]:
template = env.get_template(f"containers/{target['group']}.dockerfile.j2")
@@ -205,9 +315,13 @@ def main():
)
tags: list[str] = target.get("extra_tags") or []
- tags.insert(0, "$R/$N:latest")
- tags.insert(1, "$R/$N:$T")
-
+
+ tags.insert(0, "$R/$N:$T")
+ if not options.omit_latest:
+ tags.insert(0, "$R/$N:latest")
+ else:
+ tags = list(filter(lambda x: re.match('^.*:latest$', x) is None, tags))
+
target["tags"] = tags
rendered_dockerfile = template.render(
@@ -215,7 +329,9 @@ def main():
target=target,
build_libabigail=options.build_libabigail,
build_abi=options.build_abi,
+ build_timestamp=build_timestamp,
registry_hostname=options.registry_hostname,
+ ninja_workers=options.ninja_workers,
**inventory,
)
with open(dockerfile_location, "w") as output_file:
@@ -226,7 +342,9 @@ def main():
timestamp=timestamp,
build_libabigail=options.build_libabigail,
build_abi=options.build_abi,
+ host_arch_only=options.host_arch_only,
registry_hostname=options.registry_hostname,
+ is_builder=options.is_builder,
**inventory,
)
makefile_output_path = os.path.join(options.output_dir, "Makefile")
--
2.34.1
next prev parent reply other threads:[~2023-05-23 17:04 UTC|newest]
Thread overview: 15+ messages / expand[flat|nested] mbox.gz Atom feed top
2023-05-23 17:04 [PATCH v5 00/11] Community Lab Containers and Builder Engine Adam Hassick
2023-05-23 17:04 ` [PATCH v5 01/11] containers/docs: Add container builder start Adam Hassick
2023-05-25 13:48 ` Aaron Conole
2023-05-23 17:04 ` [PATCH v5 02/11] containers/inventory: Add inventory for container builder Adam Hassick
2023-05-23 17:04 ` [PATCH v5 03/11] containers/builder: Dockerfile creation script Adam Hassick
2023-05-23 17:04 ` [PATCH v5 04/11] containers/templates: Templates for Dockerfiles Adam Hassick
2023-05-23 17:04 ` [PATCH v5 05/11] containers/container_builder: Container for python scripts Adam Hassick
2023-05-23 17:04 ` [PATCH v5 06/11] containers/Makefile: Makefile to automate builds Adam Hassick
2023-05-23 17:04 ` [PATCH v5 07/11] containers/inventory: Add ABI rev and coverity attribute Adam Hassick
2023-05-23 17:04 ` Adam Hassick [this message]
2023-05-23 17:04 ` [PATCH v5 09/11] containers/templates: Update templates Adam Hassick
2023-05-23 17:04 ` [PATCH v5 10/11] containers/Makefile: Add new variables and target changes Adam Hassick
2023-05-23 17:04 ` [PATCH v5 11/11] containers/docs: Update README Adam Hassick
2023-05-25 13:45 ` Aaron Conole
2023-06-05 20:09 ` Adam Hassick
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20230523170413.812922-9-ahassick@iol.unh.edu \
--to=ahassick@iol.unh.edu \
--cc=aconole@redhat.com \
--cc=alialnu@nvidia.com \
--cc=ci@dpdk.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).