diff --git a/contrib/create-image.py b/contrib/create-image.py index c4847d7..acb41af 100644 --- a/contrib/create-image.py +++ b/contrib/create-image.py @@ -6,6 +6,9 @@ import shutil import subprocess import sys +from enum import Enum +from functools import cache + from print_color import print from prettytable import PrettyTable from typing import Any @@ -14,6 +17,25 @@ import argparse +class ArchTypes(str, Enum): + AMD64 = 'amd64' + ARM64 = 'arm64' + BOTH = "all" + + def __str__(self): + return self.value + + +def arch_type_strategy(arg_value: str): + try: + return ArchTypes[arg_value.upper()] + except KeyError: + raise argparse.ArgumentTypeError( + f"Invalid option: '{arg_value.upper}'. Valid options are: " + f"{', '.join(c.name.lower() for c in ArchTypes)}") + + +@cache def get_current_git_branch_name() -> str: result = subprocess.run( ["git", "rev-parse", "--abbrev-ref", "HEAD"], @@ -33,21 +55,21 @@ def ignore_tags_constructor(loader, node): yaml.SafeLoader.add_constructor(None, ignore_tags_constructor) variant_data = dict() - with open(f"{run_dir}/.zuul.yaml", "r") as file: + with open(f"{RUN_DIR}/.zuul.yaml", "r") as file: zuul_config = yaml.safe_load(file) for element in zuul_config: for elem_name, elem_data in element.items(): if elem_name != "job" or not re.fullmatch( - r"node-image-build-.+", elem_data["name"] + r"node-image-build-.+", elem_data["name"] ): continue variant_data[elem_data["name"]] = elem_data["vars"] return variant_data -def build_template(context: dict[str, Any]) -> str: - results_filename = f"{build_dir}/user-data-{context['variant']}.yaml" - template_loader = FileSystemLoader(searchpath=f"{run_dir}/templates/") +def build_template(context: dict[str, Any], result_build_dir) -> str: + results_filename = f"{result_build_dir}/user-data-{context['variant']}.yaml" + template_loader = FileSystemLoader(searchpath=f"{RUN_DIR}/templates/") template_env = Environment(loader=template_loader, undefined=StrictUndefined) results_template = template_env.get_template("user-data.yml.j2") with open(results_filename, mode="w", encoding="utf-8") as results: @@ -59,36 +81,64 @@ def build_template(context: dict[str, Any]) -> str: return results_filename -def docker_run(cmd: str, working_dir: str, chown_glob="*.iso"): +@cache +def get_create_build_container(release: str): print() - os.chdir(run_dir) + os.chdir(RUN_DIR) + + branch = get_current_git_branch_name() + docker_build_image = f"osism-node-image-builder:latest-{branch}" + + print("*" * 50, color="green") + print(f"** DOCKER BUILD: {docker_build_image}", color="green") + + # TODO: reactivate + # f"docker build --network=host -t {DOCKER_BUILD_IMAGE} " + cmd = f"docker build --load -t {docker_build_image} --build-arg UBUNTU_VERSION={release} -f Dockerfile ." + subprocess.run( - f"docker build --network=host -t {DOCKER_BUILD_IMAGE} " - + f"--build-arg UBUNTU_VERSION={DISTRIBUTION} -f Dockerfile .", + cmd.split(), check=True, - shell=True, ) - print("DOCKER:", color="green") - print(f"+ {cmd}") + print("**", color="green") + print("*" * 50, color="green") + return docker_build_image - chown_command = "" - if chown_glob and chown_glob != "": - chown_command = f"&& chown -vR {os.getuid()}:{os.getgid()} {chown_glob}" + +def docker_run(cmd: str, working_dir: str, cargs: argparse.Namespace): + os.chdir(RUN_DIR) + get_create_build_container(cargs.release) + print() + os.chdir(RUN_DIR) + print("*" * 50, color="green") + print("** DOCKER:", color="green") + + if ";" in cmd: + cmd = f'bash -c "{cmd}"' + print(f"+ {cmd}") cmd_docker = ( - f"docker run --rm --net=host -v {working_dir}:/work " - + f"{DOCKER_BUILD_IMAGE} " - + f'bash -c "{cmd} {chown_command}"' + "docker run -ti --rm --net=host " + f"-v {RUN_DIR}:/work " + f"-v {working_dir}/:/work/build " + f"--workdir /work/build/ " + f"{get_create_build_container(cargs.release)} {cmd}" ) try: - subprocess.run(cmd_docker, check=True, shell=True) + res = subprocess.run(cmd_docker, check=True, shell=True) + if res.returncode != 0: + print(f"FAILED: Command >>>{cmd_docker}<<< returned non-zero exit status {res.returncode}.", color="red") + sys.exit(res.returncode) except subprocess.CalledProcessError as e: - print(f"Command {e.cmd} returned non-zero exit status {e.returncode}.") + print(f"FAILED: Command >>>{cmd_docker}<<< failed :\n\n{e}", color="red") + sys.exit(e.returncode) + print("**", color="green") + print("*" * 50, color="green") -def package_ffr_files(context: dict) -> str: - frr_dir = f"{build_dir}/media/frr" +def package_ffr_files(context: dict, arch_dir: str, cargs: argparse.Namespace) -> str: + frr_dir = f"{arch_dir}/media/frr" os.makedirs(frr_dir, exist_ok=True) packages = [ @@ -113,16 +163,14 @@ def package_ffr_files(context: dict) -> str: download_command = f"apt-get update && apt-get download {' '.join(packages)}" download_ready = f"{frr_dir}/.download_ready" if not os.path.exists(download_ready): - docker_run(download_command, frr_dir, ".") + docker_run(download_command, frr_dir, cargs) with open(f"{frr_dir}/.download_ready", "w"): pass # do nothing, just create an empty file env = Environment(loader=FileSystemLoader("/")) - for file_name in glob.glob(f"{run_dir}/templates/frr/*"): + for file_name in glob.glob(f"{RUN_DIR}/templates/frr/*"): if file_name.endswith(".j2"): - target_filename = ( - frr_dir + "/" + os.path.basename(file_name.removesuffix(".j2")) - ) + target_filename = ( frr_dir + "/" + os.path.basename(file_name.removesuffix(".j2")) ) print(f"rendering file : {target_filename}", color="magenta") template = env.get_template(file_name) with open(target_filename, mode="w", encoding="utf-8") as results: @@ -136,46 +184,44 @@ def package_ffr_files(context: dict) -> str: def build_image( - name: str, context_data: dict, template_only: bool = False + name: str, context_data: dict, cargs: argparse.Namespace, image_build_dir: str ) -> str | None: - print(f"build image >>>{name}<<<", color="magenta") - user_data_file = build_template(context_data) + print() + print(f"build image >>>{name}<<< for architecture {cargs.arch}", color="magenta") + print() + user_data_file = build_template(context_data, image_build_dir) add_dir = "" if context_data.get("layer3_underlay") == "true": - add_dir = package_ffr_files(context_data) + add_dir = package_ffr_files(context_data, image_build_dir, cargs) iso_file = None - if not template_only: - iso_file = f"ubuntu-autoinstall-{context_data['variant']}.iso" + if not cargs.template_only: + iso_file = f"ubuntu-autoinstall-{context_data['variant']}_{args.arch}.iso" + iso_file_checksum = f"{iso_file}.CHECKSUM" - print(f"image {iso_file}", color="magenta") - os.chdir(run_dir) + print() + print(f"image {iso_file}", color="yellow") + print() + os.chdir(RUN_DIR) build_command = ( - "/work/contrib/image-create.sh -r -a -k " - + f"-u /work/build/{os.path.basename(user_data_file)} -n {DISTRIBUTION} " - + f"--destination {iso_file} {add_dir}" + f"/work/contrib/image-create.sh -r -a -k " + f"--cpu-arch {cargs.arch} " + "--timeout 3 --banner 'Install OSISM Node' " + f"-u /work/build/{os.path.basename(user_data_file)} -n {cargs.release} " + f"--destination /work/build/{iso_file} {add_dir}" ) - docker_run(build_command, run_dir) - - iso_file_checksum = f"{iso_file}.CHECKSUM" - print(f"Creating checksum file {iso_file_checksum}", color="magenta") - try: - subprocess.run( - f"shasum -a 256 {iso_file} > {iso_file_checksum}", - check=True, - shell=True, - ) - except subprocess.CalledProcessError as e: - print(f"Command {e.cmd} returned non-zero exit status {e.returncode}.") + docker_run(build_command, image_build_dir, cargs) + docker_run(f"sha256sum {iso_file} > {iso_file_checksum}", image_build_dir, cargs) + docker_run(f"chown -vR {os.getuid()}:{os.getgid()} *.iso *.iso.CHECKSUM", image_build_dir, cargs) return iso_file def create_context( - image_name: str, commandline_args: argparse.Namespace + image_name: str, commandline_args: argparse.Namespace ) -> dict[str, str]: context_data_default = { "layer3_underlay": "false", @@ -217,7 +263,7 @@ def create_context( return context_data -def build_images(commandline_args: argparse.Namespace): +def build_images(commandline_args: argparse.Namespace, image_build_dir: str): os.makedirs(build_dir, exist_ok=True) images = set() for image in commandline_args.build: @@ -233,7 +279,8 @@ def build_images(commandline_args: argparse.Namespace): build_image( image, create_context(image, commandline_args), - commandline_args.template_only, + commandline_args, + image_build_dir, ) @@ -241,7 +288,7 @@ def wrap_text_by_words(text: str, words_per_line: int): words = text.split(" ") lines = [] for i in range(0, len(words), words_per_line): - line = " ".join(words[i : i + words_per_line]).strip() + line = " ".join(words[i: i + words_per_line]).strip() lines.append(line) return "\n".join(lines) @@ -265,20 +312,17 @@ def show_variants(): print(table) print() print( - f"A overview: https://github.com/osism/node-image/tree/{BRANCH}", color="green" + f"A overview: https://github.com/osism/node-image/tree/{get_current_git_branch_name()}", color="green" ) sys.exit(0) -DISTRIBUTION = "jammy" DOCKER_WORKDIR = "/work" -BRANCH = get_current_git_branch_name() -DOCKER_BUILD_IMAGE = f"osism-node-image-builder:latest-{BRANCH}" +RUN_DIR = os.path.realpath(os.path.dirname(os.path.realpath(__file__)) + "/../") if __name__ == "__main__": - run_dir = os.path.realpath(os.path.dirname(os.path.realpath(__file__)) + "/../") - parser = argparse.ArgumentParser(prog=f"{run_dir}/create-image.sh") + parser = argparse.ArgumentParser(prog=f"{RUN_DIR}/create-image.sh") exclusive_group = parser.add_mutually_exclusive_group(required=True) exclusive_group.add_argument( @@ -291,7 +335,7 @@ def show_variants(): "--env", "-e", action="store_true", help="Create build environment" ) exclusive_group.add_argument( - "--clean", "-r", action="store_true", help="Drop cached build data" + "--drop", "-d", action="store_true", help="Drop cached build data" ) parser.add_argument( @@ -303,6 +347,18 @@ def show_variants(): default=[], ) parser.add_argument("--config", "-c", type=str, help="A config as yaml file") + + parser.add_argument("--arch", "-a", + type=arch_type_strategy, + choices=list(ArchTypes), + default=ArchTypes.AMD64, + help="Architecture") + + parser.add_argument("--release", "-r", + type=str, + default="jammy", + help="Architecture") + parser.add_argument( "--build-directory", type=str, help="Overwrite the default build directory" ) @@ -312,35 +368,34 @@ def show_variants(): ) args = parser.parse_args() - build_dir = f"{run_dir}/build" + build_dir = f"{RUN_DIR}/build/{args.arch}/{args.release}" - os.chdir(run_dir) + os.chdir(RUN_DIR) os.umask(0o022) - variants = get_variants_data() if args.show: show_variants() - if args.clean: + if args.drop: print("Cleaning up cached build data", color="green") if os.path.exists(build_dir): shutil.rmtree(build_dir) os.makedirs(build_dir) if ( - len( - subprocess.check_output( - f"docker images {DOCKER_BUILD_IMAGE} -q ", shell=True + len( + subprocess.check_output( + f"docker images {get_create_build_container(args.release)} -q ", shell=True + ) ) - ) - > 0 + > 0 ): - subprocess.run(f"docker rmi {DOCKER_BUILD_IMAGE}", check=True, shell=True) + subprocess.run(f"docker rmi {get_create_build_container(args.release)}", check=True, shell=True) sys.exit(0) if args.build: - build_images(args) + build_images(args, build_dir) sys.exit(0) if args.env: - docker_run("cat /etc/lsb-release", run_dir, chown_glob="") + docker_run("cat /etc/lsb-release", RUN_DIR, args) diff --git a/contrib/image-create.sh b/contrib/image-create.sh index 709465d..9bd6535 100755 --- a/contrib/image-create.sh +++ b/contrib/image-create.sh @@ -1,28 +1,36 @@ #!/bin/bash -set -Eeuo pipefail -set -x # SPDX-License-Identifier: MIT # source: https://github.com/cloudymax/pxeless -trap cleanup SIGINT SIGTERM ERR EXIT +if [ "${DEBUG:-no}" = "yes" ];then + export PS4='+${BASH_SOURCE}:${LINENO}:${FUNCNAME[0]-*no-function*}: ' +else + set -Eeuo pipefail + trap "cleanup; exit " SIGINT SIGTERM ERR +fi + [[ ! -x "$(command -v date)" ]] && echo "💥 date command not found." && exit 1 # export initial variables export_metadata(){ - export TODAY=$(date +"%Y-%m-%d") + export TODAY="$(date +"%Y-%m-%d")" export USER_DATA_FILE='' export META_DATA_FILE='' export CODE_NAME="" export BASE_URL="" export ISO_FILE_NAME="" + export ARCH="" export ORIGINAL_ISO="ubuntu-original-$TODAY.iso" export EFI_IMAGE="ubuntu-original-$TODAY.efi" export MBR_IMAGE="ubuntu-original-$TODAY.mbr" - export SOURCE_ISO="${ORIGINAL_ISO}" + export SOURCE_ISO="" export DESTINATION_ISO="ubuntu-autoinstall.iso" export SHA_SUFFIX="${TODAY}" + export DEFAULT_TIMEOUT="30" + export TIMEOUT="${DEFAULT_TIMEOUT}" + export MENU_BANNER="" export UBUNTU_GPG_KEY_ID="843938DF228D22F7B3742BC0D94AA3F0EFE21092" export GPG_VERIFY=1 export ALL_IN_ONE=0 @@ -32,6 +40,7 @@ export_metadata(){ export EXTRA_FILES_FOLDER="" export OFFLINE_INSTALLER="" + export ORIGINAL_ISO_TMP="" export LEGACY_IMAGE=0 export CURRENT_RELEASE="" export ISO_NAME="" @@ -64,6 +73,8 @@ Available options: -e, --use-hwe-kernel Force the generated ISO to boot using the hardware enablement (HWE) kernel. Not supported by early Ubuntu 20.04 release ISOs. +-c, --cpu-arch Set architecture for the created image (arm64, amd64). Defaults to amd64 + -u, --user-data Path to user-data file. Required if using -a -m, --meta-data Path to meta-data file. Will be an empty file if not specified and using -a @@ -80,6 +91,9 @@ Available options: -r, --use-release-iso Use the current release ISO instead of the daily ISO. The file will be used if it already exists. +-t, --timeout Set the GRUB timeout. Defaults to 30. +-b, --banner Set a text for the grub menu banner + -s, --source Source ISO file path. By default the latest daily ISO for Ubuntu server will be downloaded and saved as