Merge branch 'eero/cleanup-builds-5' into 'master'

Bazelize HostOS

- General cleanup in `icos_build`
- Unwrap HostOS from nested builder
- Use Bazel generic image build path for HostOS
- Add LVM tooling to Bazel python tools
- Add `lvm2` to builder containers 

See merge request dfinity-lab/public/ic!11201

.devcontainer/devcontainer.json

@@ -1,5 +1,5 @@
 {
-  "image": "dfinity/ic-build-bazel:89c6de057c399dac1e131c0aaee2d15aaee734c8ff3f3c717ce7bb88ffecc529",
+  "image": "dfinity/ic-build-bazel:20ed3537b5467a5143b325828d8f4f644261f1c48f80a4d38088ee3045baa157",
   "remoteUser": "ubuntu",
   "workspaceMount": "source=${localWorkspaceFolder},target=/ic,type=bind",
   "workspaceFolder": "/ic",

gitlab-ci/config/00--common--images.yml

@@ -4,9 +4,9 @@
   artifacts:
     expire_in: 3 days
   image:
-    name: "registry.gitlab.com/dfinity-lab/core/docker/ic-build:89c6de057c399dac1e131c0aaee2d15aaee734c8ff3f3c717ce7bb88ffecc529"
+    name: "registry.gitlab.com/dfinity-lab/core/docker/ic-build:20ed3537b5467a5143b325828d8f4f644261f1c48f80a4d38088ee3045baa157"
 .ic-build-bazel-image:
   artifacts:
     expire_in: 3 days
   image:
-    name: "registry.gitlab.com/dfinity-lab/core/docker/ic-build-bazel:89c6de057c399dac1e131c0aaee2d15aaee734c8ff3f3c717ce7bb88ffecc529"
+    name: "registry.gitlab.com/dfinity-lab/core/docker/ic-build-bazel:20ed3537b5467a5143b325828d8f4f644261f1c48f80a4d38088ee3045baa157"

gitlab-ci/container/Dockerfile

@@ -30,7 +30,7 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone &
         policycoreutils python-is-python3 python3 python3-pip python3-gflags python3-cbor pipenv shellcheck \
         sudo docker-ce-cli ripgrep patchelf cpio nodejs npm gosu pigz zsh hub less \
         ca-certificates curl git isolinux p7zip-full syslinux xorriso cryptsetup-bin autoconf automake \
-        grub-efi-amd64-bin podman buildah
+        grub-efi-amd64-bin podman buildah lvm2
 
 COPY --from=docker/buildx-bin:latest /buildx /usr/libexec/docker/cli-plugins/docker-buildx
 

gitlab-ci/container/Dockerfile.bazel

@@ -17,7 +17,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && ln -snf /usr/share/zoneinfo/$TZ /et
     # RUST \
     gcc lld pkg-config libssl-dev libunwind-dev libusb-1.0-0-dev libsqlite3-dev zlib1g-dev libclang-10-dev \
     # IC-OS \
-    python3 python-is-python3 fakeroot android-sdk-ext4-utils cryptsetup-bin zstd lsb-release grub-efi-amd64-bin libsystemd-dev faketime dosfstools libselinux-dev mtools
+    python3 python-is-python3 fakeroot android-sdk-ext4-utils cryptsetup-bin zstd lsb-release grub-efi-amd64-bin libsystemd-dev faketime dosfstools libselinux-dev mtools lvm2
 
 ARG buildevents_version=0.6.0
 RUN curl -fsSL https://github.com/honeycombio/buildevents/releases/download/v${buildevents_version}/buildevents-linux-amd64 -o /usr/bin/buildevents && \

gitlab-ci/container/TAG

@@ -1 +1 @@
-89c6de057c399dac1e131c0aaee2d15aaee734c8ff3f3c717ce7bb88ffecc529
+20ed3537b5467a5143b325828d8f4f644261f1c48f80a4d38088ee3045baa157

ic-os/defs.bzl

@@ -189,8 +189,8 @@ def icos_build(name, upload_prefix, image_deps, mode = None, malicious = False,
             k: v
             for k, v in (
                 image_deps["bootfs"].items() + [
-                    ("version.txt", "/boot/version.txt:0644"),
-                    ("extra_boot_args", "/boot/extra_boot_args:0644"),
+                    (":version.txt", "/boot/version.txt:0644"),
+                    (":extra_boot_args", "/boot/extra_boot_args:0644"),
                 ]
             )
             # Skip over special entries
@@ -214,8 +214,8 @@ def icos_build(name, upload_prefix, image_deps, mode = None, malicious = False,
                 k: v
                 for k, v in (
                     image_deps["bootfs"].items() + [
-                        ("version-test.txt", "/boot/version.txt:0644"),
-                        ("extra_boot_test_args", "/boot/extra_boot_args:0644"),
+                        (":version-test.txt", "/boot/version.txt:0644"),
+                        (":extra_boot_test_args", "/boot/extra_boot_args:0644"),
                     ]
                 )
                 # Skip over special entries
@@ -231,7 +231,11 @@ def icos_build(name, upload_prefix, image_deps, mode = None, malicious = False,
 
     # -------------------- Assemble disk image --------------------
 
-    custom_partitions = image_deps.get("custom_partitions", default = [])
+    # Build a list of custom partitions with a funciton, to allow "injecting" build steps at this point
+    if "custom_partitions" not in image_deps:
+        custom_partitions = []
+    else:
+        custom_partitions = image_deps["custom_partitions"]()
 
     disk_image(
         name = "disk-img.tar",
@@ -251,7 +255,7 @@ def icos_build(name, upload_prefix, image_deps, mode = None, malicious = False,
 
     native.genrule(
         name = "disk-img.tar_zst",
-        srcs = ["disk-img.tar"],
+        srcs = [":disk-img.tar"],
         outs = ["disk-img.tar.zst"],
         cmd = "zstd --threads=0 -10 -f -z $< -o $@",
         # The image is pretty big, therefore it is usually much faster to just rebuild it instead of fetching from the cache.
@@ -266,7 +270,7 @@ def icos_build(name, upload_prefix, image_deps, mode = None, malicious = False,
 
     gzip_compress(
         name = "disk-img.tar.gz",
-        srcs = ["disk-img.tar"],
+        srcs = [":disk-img.tar"],
         # The image is pretty big, therefore it is usually much faster to just rebuild it instead of fetching from the cache.
         # TODO(IDX-2221): remove this when CI jobs and bazel infrastructure will run in the same clusters.
         tags = ["no-remote-cache"],
@@ -296,7 +300,7 @@ def icos_build(name, upload_prefix, image_deps, mode = None, malicious = False,
 
         native.genrule(
             name = "update-img.tar_zst",
-            srcs = ["update-img.tar"],
+            srcs = [":update-img.tar"],
             outs = ["update-img.tar.zst"],
             cmd = "zstd --threads=0 -10 -f -z $< -o $@",
             # The image is pretty big, therefore it is usually much faster to just rebuild it instead of fetching from the cache.
@@ -311,7 +315,7 @@ def icos_build(name, upload_prefix, image_deps, mode = None, malicious = False,
 
         gzip_compress(
             name = "update-img.tar.gz",
-            srcs = ["update-img.tar"],
+            srcs = [":update-img.tar"],
             # The image is pretty big, therefore it is usually much faster to just rebuild it instead of fetching from the cache.
             # TODO(IDX-2221): remove this when CI jobs and bazel infrastructure will run in the same clusters.
             tags = ["no-remote-cache"],
@@ -337,7 +341,7 @@ def icos_build(name, upload_prefix, image_deps, mode = None, malicious = False,
 
         native.genrule(
             name = "update-img-test.tar_zst",
-            srcs = ["update-img-test.tar"],
+            srcs = [":update-img-test.tar"],
             outs = ["update-img-test.tar.zst"],
             cmd = "zstd --threads=0 -10 -f -z $< -o $@",
             # The image is pretty big, therefore it is usually much faster to just rebuild it instead of fetching from the cache.
@@ -352,7 +356,7 @@ def icos_build(name, upload_prefix, image_deps, mode = None, malicious = False,
 
         gzip_compress(
             name = "update-img-test.tar.gz",
-            srcs = ["update-img-test.tar"],
+            srcs = [":update-img-test.tar"],
             # The image is pretty big, therefore it is usually much faster to just rebuild it instead of fetching from the cache.
             # TODO(IDX-2221): remove this when CI jobs and bazel infrastructure will run in the same clusters.
             tags = ["no-remote-cache"],

ic-os/guestos/defs.bzl

@@ -62,7 +62,7 @@ def image_deps(mode, malicious = False):
     deps["bootfs_size"] = "1G"
 
     # Add any custom partitions to the manifest
-    deps["custom_partitions"] = [
+    deps["custom_partitions"] = lambda: [
         Label("//ic-os/guestos:partition-config.tar"),
         Label("//ic-os/bootloader:partition-esp.tar"),
         Label("//ic-os/bootloader:partition-grub.tar"),

ic-os/hostos/BUILD.bazel

@@ -0,0 +1,31 @@
+load("//toolchains/sysimage:toolchain.bzl", "ext4_image")
+
+package(default_visibility = ["__subpackages__"])
+
+exports_files([
+    "partitions.csv",
+    "volumes.csv",
+    "rootfs/docker-base.dev",
+    "rootfs/docker-base.prod",
+])
+
+filegroup(
+    name = "rootfs-files",
+    srcs = glob(["rootfs/**"]),
+)
+
+ext4_image(
+    name = "partition-config.tar",
+    partition_size = "100M",
+    target_compatible_with = [
+        "@platforms//os:linux",
+    ],
+)
+
+# All host-os targets are named the same, just stay in different submodules.
+# To build or run specific target:
+#
+# bazel run   //ic-os/hostos/envs/prod:upload_disk-img           [--s3_endpoint=<internal endpoint address>]
+# bazel build //ic-os/hostos/envs/dev/...
+#
+# check //ic-os/defs.bzl for the full list of targets.

ic-os/hostos/bootloader/BUILD.bazel

@@ -0,0 +1,44 @@
+load("//toolchains/sysimage:toolchain.bzl", "vfat_image")
+
+package(default_visibility = ["//visibility:public"])
+
+exports_files([
+    "extra_boot_args",
+])
+
+genrule(
+    name = "build-bootloader-tree",
+    srcs = [],
+    outs = [
+        "bootloader-tree.tar",
+    ],
+    cmd = "$(location build-bootloader-tree.sh) -o $@",
+    target_compatible_with = [
+        "@platforms//os:linux",
+    ],
+    tools = ["build-bootloader-tree.sh"],
+)
+
+vfat_image(
+    name = "partition-esp.tar",
+    src = ":bootloader-tree.tar",
+    partition_size = "100M",
+    subdir = "boot/efi",
+    target_compatible_with = [
+        "@platforms//os:linux",
+    ],
+)
+
+vfat_image(
+    name = "partition-grub.tar",
+    src = ":bootloader-tree.tar",
+    extra_files = {
+        ":grub.cfg": "/boot/grub/grub.cfg:0644",
+        ":grubenv": "/boot/grub/grubenv:0644",
+    },
+    partition_size = "100M",
+    subdir = "boot/grub",
+    target_compatible_with = [
+        "@platforms//os:linux",
+    ],
+)

ic-os/hostos/bootloader/extra_boot_args

@@ -0,0 +1,14 @@
+# Uncomment this to run system with SELinux in PERMISSIVE mode:
+# the system will use SELinux and keep track of operations that would
+# be prohibited, but will only log but not actually deny them. This is
+# useful for debug and policy development. The system behaves essentially the
+# same as if SELinux was not activated.
+#
+EXTRA_BOOT_ARGS="security=selinux selinux=1 enforcing=0"
+
+# Uncomment this to run system with SELinux in ENFORCING mode: All rules
+# of the policy are enforced, and forbidden actions are not just logged but
+# stopped. This causes the system to behave differently than in either
+# "no SELinux" or "permissive" mode.
+#
+# EXTRA_BOOT_ARGS="security=selinux selinux=1 enforcing=1"

ic-os/hostos/build.sh

@@ -50,13 +50,15 @@ if [ "${ROOT_PASSWORD}" != "" -a "${BUILD_TYPE}" != "dev" ]; then
 fi
 
 BASE_DIR=$(dirname "${BASH_SOURCE[0]}")
-TMPDIR=$(mktemp -d)
 SCRIPTS_DIR=$BASE_DIR/../scripts
 TOOL_DIR="${BASE_DIR}/../../toolchains/sysimage/"
 
-docker version
+TMPDIR=$(mktemp -d)
+trap "rm -rf $TMPDIR" exit
 
-trap "rm -rf esp.img.tar grub.img.tar rootfs.tar" EXIT
+source "${SCRIPTS_DIR}/partitions.sh" ${BASE_DIR}
+
+docker version
 
 BASE_IMAGE="$(cat ${BASE_DIR}/rootfs/docker-base.${BUILD_TYPE})"
 
@@ -65,27 +67,69 @@ echo "Set version"
 echo "${VERSION}" >"${BASE_DIR}/rootfs/opt/ic/share/version.txt"
 echo "${VERSION}" >"${BASE_DIR}/rootfs/boot/version.txt"
 
+# Build bootloader image
 BOOTLOADER_TAR="${TMPDIR}/bootloader.tar"
-ESP_IMG_TAR="${BASE_DIR}/esp.img.tar"
-GRUB_IMG_TAR="${BASE_DIR}/grub.img.tar"
 $BASE_DIR/bootloader/build-bootloader-tree.sh -o ${BOOTLOADER_TAR}
-"${TOOL_DIR}"/build_vfat_image.py -o "${ESP_IMG_TAR}" -s 100M -p boot/efi -i "${BOOTLOADER_TAR}"
-"${TOOL_DIR}"/build_vfat_image.py -o "${GRUB_IMG_TAR}" -s 100M -p boot/grub -i "${BOOTLOADER_TAR}" \
-    "${BASE_DIR}/bootloader/grub.cfg:/boot/grub/grub.cfg:644" \
-    "${BASE_DIR}/bootloader/grubenv:/boot/grub/grubenv:644"
 
+# Build main image
+ROOTFS_TAR=${TMPDIR}/rootfs.tar
 $SCRIPTS_DIR/build-docker-save.sh \
     --build-arg BASE_IMAGE="${BASE_IMAGE}" \
     --build-arg ROOT_PASSWORD="${ROOT_PASSWORD}" \
-    $BASE_DIR/rootfs >$BASE_DIR/rootfs.tar
+    $BASE_DIR/rootfs >${ROOTFS_TAR}
 
-docker build --iidfile $TMPDIR/iidfile -q -f $BASE_DIR/build/Dockerfile $BASE_DIR/.. 2>&1
-IMAGE_ID=$(cat $TMPDIR/iidfile | cut -d':' -f2)
+# Build bootloader partitions
+ESP_IMG_TAR="${TMPDIR}/esp.img.tar"
+GRUB_IMG_TAR="${TMPDIR}/grub.img.tar"
+"${TOOL_DIR}"/build_vfat_image.py -o "${ESP_IMG_TAR}" -s 100M -p boot/efi -i "${BOOTLOADER_TAR}"
+"${TOOL_DIR}"/build_vfat_image.py -o "${GRUB_IMG_TAR}" -s 100M -p boot/grub -i "${BOOTLOADER_TAR}" \
+    "${BASE_DIR}/bootloader/grub.cfg:/boot/grub/grub.cfg:644" \
+    "${BASE_DIR}/bootloader/grubenv:/boot/grub/grubenv:644"
 
-docker run -h builder --cidfile $TMPDIR/cid --privileged $IMAGE_ID
-CONTAINER_ID=$(cat $TMPDIR/cid)
-docker cp $CONTAINER_ID:/ic-os/disk-img.tar.gz disk-img.tar.gz
-docker cp $CONTAINER_ID:/ic-os/update-img.tar.gz update-img.tar.gz
-docker rm $CONTAINER_ID
+# Extract bootloader partitions.
+ESP_IMG="${TMPDIR}/esp.img"
+GRUB_IMG="${TMPDIR}/grub.img"
+tar -xOf ${ESP_IMG_TAR} >${ESP_IMG}
+tar -xOf ${GRUB_IMG_TAR} >${GRUB_IMG}
+
+# Prepare empty config partition.
+CONFIG_IMG="${TMPDIR}/config.img"
+truncate --size 100M "$CONFIG_IMG"
+make_ext4fs -T 0 -l 100M "$CONFIG_IMG"
+
+# Build partitions for system image A.
+BOOT_IMG="${TMPDIR}/boot.img"
+ROOT_IMG="${TMPDIR}/root.img"
+"${BASE_DIR}"/../scripts/build-ubuntu.sh -i "${ROOTFS_TAR}" -r "${ROOT_IMG}" -b "${BOOT_IMG}"
+
+# Assemble update image
+UPDATE_DIR=${TMPDIR}/update
+mkdir ${UPDATE_DIR}
+echo "${VERSION}" >"${UPDATE_DIR}/VERSION.TXT"
+cp "${BOOT_IMG}" "${UPDATE_DIR}/boot.img"
+cp "${ROOT_IMG}" "${UPDATE_DIR}/root.img"
+# Sort by name in tar file -- makes ordering deterministic and ensures
+# that VERSION.TXT is first entry, making it quick & easy to extract.
+# Override owner, group and mtime to make build independent of the user
+# building it.
+tar czf "update-img.tar.gz" --sort=name --owner=root:0 --group=root:0 --mtime='UTC 2020-01-01' --sparse -C "${UPDATE_DIR}" .
+
+# Create HostOS LVM Structure
+VOLUME_GROUP="hostlvm"
+LVM_IMG="${TMPDIR}/lvm.img"
+prepare_lvm_image "$LVM_IMG" 107374182400 "$VOLUME_GROUP" "4c7GVZ-Df82-QEcJ-xXtV-JgRL-IjLE-hK0FgA" "eu0VQE-HlTi-EyRc-GceP-xZtn-3j6t-iqEwyv" # 100G
+
+# Assemble disk image
+DISK_IMG="${TMPDIR}/disk.img"
+prepare_disk_image "$DISK_IMG" 108447924224 # 101G
+write_single_partition "$DISK_IMG" esp "$ESP_IMG"
+write_single_partition "$DISK_IMG" grub "$GRUB_IMG"
+write_single_partition "$DISK_IMG" hostlvm "$LVM_IMG"
+write_single_lvm_volume "$DISK_IMG" "$VOLUME_GROUP" A_boot "$BOOT_IMG"
+write_single_lvm_volume "$DISK_IMG" "$VOLUME_GROUP" A_root "$ROOT_IMG"
+write_single_lvm_volume "$DISK_IMG" "$VOLUME_GROUP" config "$CONFIG_IMG"
+
+# Package image in tar
+tar czf "disk-img.tar.gz" --sort=name --owner=root:0 --group=root:0 --mtime='UTC 2020-01-01' --sparse -C "${TMPDIR}" disk.img
 
 rm -rf $TMPDIR