Skip to content

feat: [NODE-1048] Share podman cache across image builds #8164

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Bownairo wants to merge 5 commits into from
Closed

feat: [NODE-1048] Share podman cache across image builds #8164

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
78a2210
Share a cache across image builds
Bownairo Dec 19, 2025
0dc3f8c
Remove added guard
Bownairo Dec 20, 2025
9b39705
Drop tmpfs_tmpdir
Bownairo Dec 30, 2025
3020fcf
Move container cleanup to proc_wrapper
Bownairo Jan 6, 2026
6716c96
Simplify and unify PATH handling
Bownairo Jan 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 4 additions & 8 deletions ic-os/bootloader/build-bootloader-tree.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,6 @@

set -exo pipefail

cleanup() {
podman rm -f "${CONTAINER}"
rm -rf "${TMP_DIR}"
}
trap cleanup EXIT

while getopts "o:" OPT; do
case "${OPT}" in
o)
Expand All @@ -24,11 +18,12 @@ while getopts "o:" OPT; do
esac
done

# tempfile cleanup is handled by proc_wrapper.sh
TMP_DIR=$(mktemp -d -t build-image-XXXXXXXXXXXX)

BASE_IMAGE="ghcr.io/dfinity/library/ubuntu@sha256:6015f66923d7afbc53558d7ccffd325d43b4e249f41a6e93eef074c9505d2233"

podman build --no-cache --iidfile "${TMP_DIR}/iidfile" - <<<"
podman build --iidfile "${TMP_DIR}/iidfile" - <<<"
FROM $BASE_IMAGE
USER root:root
RUN apt-get -y update && apt-get -y --no-install-recommends install grub-efi faketime
Expand All @@ -47,7 +42,8 @@ podman build --no-cache --iidfile "${TMP_DIR}/iidfile" - <<<"

IMAGE_ID=$(cut -d':' -f2 <"${TMP_DIR}/iidfile")

CONTAINER=$(podman run -d "${IMAGE_ID}")
# container cleanup is handled by proc_wrapper.sh
CONTAINER=$(podman run -d --cidfile "${TMPDIR}/cidfile" "${IMAGE_ID}")

podman export "${CONTAINER}" | tar --strip-components=1 -C "${TMP_DIR}" -x build
tar cf "${OUT_FILE}" --sort=name --owner=root:0 --group=root:0 "--mtime=UTC 1970-01-01 00:00:00" -C "${TMP_DIR}" boot
21 changes: 5 additions & 16 deletions toolchains/sysimage/build_container_base_image.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,8 @@
#
from __future__ import annotations

import atexit
import os
import shutil
import signal
import tempfile
from dataclasses import dataclass
from pathlib import Path
Expand Down Expand Up @@ -48,7 +46,7 @@ def build_image(image_tag: str, dockerfile: str, context_dir: str, build_args: L
build_arg_strings_joined = " ".join(build_arg_strings)

log.info("Building image...")
cmd = f"podman build --squash-all --no-cache --tag {image_tag} {build_arg_strings_joined} --file {dockerfile} {context_dir}"
cmd = f"podman build --squash-all --tag {image_tag} {build_arg_strings_joined} --file {dockerfile} {context_dir}"
invoke.run(cmd)
log.info("Image built successfully")

Expand All @@ -57,7 +55,6 @@ def save_image(image_tag: str, output_file: str):
log.info("Saving image to tar file")
cmd = f"podman image save --output {output_file} {image_tag}"
invoke.run(cmd)
invoke.run("sync") # For determinism (?)

output_path = Path(output_file)
assert output_path.exists()
Expand All @@ -83,20 +80,12 @@ def main():

log.info(f"Using args: {args}")

# NOTE: /usr/bin/nsenter is required to be on $PATH for this version of
# podman (no longer in latest version). bazel strips this out - add it back
# manually, for now.
os.environ["PATH"] = ":".join([x for x in [os.environ.get("PATH"), "/usr/bin"] if x is not None])

def cleanup():
invoke.run(f"podman rm -f {image_tag}")
invoke.run(f"podman rm -f {image_tag}_container")

atexit.register(lambda: cleanup())
signal.signal(signal.SIGTERM, lambda: cleanup())
signal.signal(signal.SIGINT, lambda: cleanup())
# NOTE: /usr/bin/newuidmap is required to be on $PATH for podman. bazel
# strips this out - add it back manually.
os.environ["PATH"] = "/usr/bin:" + os.environ.get("PATH", "")

build_args = list(build_args or [])
# tempfile cleanup is handled by proc_wrapper.sh
context_dir = tempfile.mkdtemp()

# Add all context files directly into dir
Expand Down
25 changes: 8 additions & 17 deletions toolchains/sysimage/build_container_filesystem_tar.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,8 @@
from __future__ import annotations

import argparse
import atexit
import os
import shutil
import signal
import tempfile
import uuid
from dataclasses import dataclass
Expand Down Expand Up @@ -65,7 +63,6 @@ def build_container(
cmd += "build "
cmd += f"-t {image_tag} "
cmd += f"{build_arg_strings_joined} "
cmd += "--no-cache "

if base_image_override:
load_base_image_tar_file(base_image_override.image_file)
Expand All @@ -91,20 +88,22 @@ def export_container_filesystem(image_tag: str, destination_tar_filename: str):
Export the filesystem from an image.
Creates container - but does not start it, avoiding timestamp and other determinism issues.
"""
# tempfile cleanup is handled by proc_wrapper.sh
tempdir = tempfile.mkdtemp()
tar_file = tempdir + "/temp.tar"
fakeroot_statefile = tempdir + "/fakeroot.state"
tar_dir = tempdir + "/tar"

# container cleanup is handled by proc_wrapper.sh
container_name = image_tag + "_container"
invoke.run(f"podman create --name {container_name} {image_tag}")
sys_tempdir = tempfile.gettempdir()
invoke.run(f"podman create --name {container_name} --cidfile {sys_tempdir}/cidfile {image_tag}")
invoke.run(f"podman export -o {tar_file} {container_name}")
invoke.run(f"mkdir -p {tar_dir}")
invoke.run(f"fakeroot -s {fakeroot_statefile} tar xpf {tar_file} --same-owner --numeric-owner -C {tar_dir}")
invoke.run(
f"fakeroot -i {fakeroot_statefile} tar cf {destination_tar_filename} --numeric-owner --sort=name --exclude='run/*' -C {tar_dir} $(ls -A {tar_dir})"
)
invoke.run("sync")


def resolve_file_args(context_dir: str, file_build_args: List[str]) -> List[str]:
Expand Down Expand Up @@ -203,23 +202,15 @@ def main():
destination_tar_filename = args.output
build_args = list(args.build_args or [])

# NOTE: /usr/bin/nsenter is required to be on $PATH for this version of
# podman (no longer in latest version). bazel strips this out - add it back
# manually, for now.
os.environ["PATH"] = ":".join([x for x in [os.environ.get("PATH"), "/usr/bin"] if x is not None])
# NOTE: /usr/bin/newuidmap is required to be on $PATH for podman. bazel
# strips this out - add it back manually.
os.environ["PATH"] = "/usr/bin:" + os.environ.get("PATH", "")

image_tag = str(uuid.uuid4()).split("-")[0]
context_files = args.context_files
component_files = args.component_files

def cleanup():
invoke.run(f"podman rm -f {image_tag}_container")
invoke.run(f"podman rm -f {image_tag}")

atexit.register(lambda: cleanup())
signal.signal(signal.SIGTERM, lambda: cleanup())
signal.signal(signal.SIGINT, lambda: cleanup())

# tempfile cleanup is handled by proc_wrapper.sh
context_dir = tempfile.mkdtemp()

# Add all context files directly into dir
Expand Down
22 changes: 15 additions & 7 deletions toolchains/sysimage/proc_wrapper.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,20 @@

set -euo pipefail

# Temporary shim to create tmpfs on demand, until we have userspace overlayfs,
# or tmpfs natively available on CI.
tmpfs_tmpdir=$(mktemp -d --tmpdir "icosbuildXXXX")
sudo mount -t tmpfs none "${tmpfs_tmpdir}"
cleanup() {
if [ -f "${tmpdir}/cidfile" ]; then
CONTAINER_ID=$(cut -d':' -f2 <"${tmpdir}/cidfile")

# NOTE: /usr/bin/newuidmap is required to be on $PATH for podman. bazel
# strips this out - add it back manually.
export PATH="/usr/bin:$PATH"
podman container stop "${CONTAINER_ID}"
podman container cleanup --rm "${CONTAINER_ID}"
fi

sudo rm -rf "$tmpdir"
}

tmpdir=$(mktemp -d --tmpdir "icosbuildXXXX")
# NOTE: Ignore failure to cleanup the directory for now. This should not be a problem after NODE-1048.
trap 'sudo umount "${tmpfs_tmpdir}"; sudo rm -rf "$tmpdir" "${tmpfs_tmpdir}" || true' INT TERM EXIT
TMPDIR="$tmpdir" TMPFS_TMPDIR="${tmpfs_tmpdir}" "$@"
trap cleanup INT TERM EXIT
TMPDIR="$tmpdir" "$@"
Loading