feat(IDX): Add prefix to artifact bundles (#5098)

This adds a `prefix` option when creating artifact bundles. This will
allow us to use the bundles directly when uploading artifacts to the
CDN, instead of duplicating the logic of listing artifacts & computing
checksums.

This also makes the necessary changes in `build-ic.sh` to support nested
build artifacts as well as fixes a typo in the guestos directory name.

The implementation of `artifact_bundle` is also simplified to avoid
creating symlinks from Bazel; moreover the output of `artifact_bundle`
is now a _directory_ instead of a list of artifacts.

Prefixes are also added to all artifact bundles to reflect the file
structure on the CDN.

Author: nmattia

ci/container/build-ic.sh

@@ -172,7 +172,10 @@ for artifact in $(bazel cquery "${BAZEL_COMMON_ARGS[@]}" --output=files "$query"
     esac
 
     mkdir -p "$target_dir"
-    cp "$artifact" "$target_dir"
+
+    # We use -L so that find dereferences symlinks (artifacts are not
+    # necessarily duplicated in the build)
+    find -L "$artifact" -type f -exec cp {} "$target_dir" \;
 done
 
 if "$BUILD_BIN"; then

ic-os/guestos/envs/prod/BUILD.bazel

@@ -39,12 +39,13 @@ file_size_check(
 artifact_bundle(
     name = "bundle",
     inputs = [":prod"],
+    prefix = "guest-os/update-img",
 )
 
 # This image is used for GuestOS upgrades
 upload_artifacts(
     name = "upload_update-img",
     inputs = [icos_images.update_image],
-    remote_subdir = "guestos-os/update-img",
+    remote_subdir = "guest-os/update-img",
     visibility = ["//visibility:public"],  # testnet deploy tools use this
 )

ic-os/hostos/envs/prod/BUILD.bazel

@@ -37,6 +37,7 @@ file_size_check(
 artifact_bundle(
     name = "bundle",
     inputs = [":prod"],
+    prefix = "host-os/update-img",
 )
 
 # This image is used for HostOS upgrades

ic-os/setupos/envs/prod/BUILD.bazel

@@ -31,6 +31,7 @@ launch_bare_metal(
 artifact_bundle(
     name = "bundle",
     inputs = [":prod"],
+    prefix = "setup-os/disk-img",
 )
 
 # This image is used as the IC-OS installation image downloaded by Node Providers

publish/binaries/BUILD.bazel

@@ -122,6 +122,13 @@ filegroup(
 artifact_bundle(
     name = "bundle",
     inputs = [":binaries"],
+    prefix = "binaries/" + os_info,
+)
+
+artifact_bundle(
+    name = "bundle-legacy",
+    inputs = [":binaries"],
+    prefix = "release",
 )
 
 upload_artifacts(

publish/canisters/BUILD.bazel

@@ -135,6 +135,7 @@ filegroup(
 artifact_bundle(
     name = "bundle",
     inputs = [":canisters"],
+    prefix = "canisters",
 )
 
 # the further targets are all testonly for simplicity because if a target was

publish/defs.bzl

@@ -125,17 +125,12 @@ def _artifact_bundle_impl(ctx):
     # List of input files
     input_files = ctx.files.inputs
 
-    bundle_dir = "{}_bundle".format(ctx.attr.name)
+    bundle_root = ctx.actions.declare_directory("bundle-{}".format(ctx.attr.name))
 
-    # Declare output files (NOTE: not windows friendly)
-    out_checksums = ctx.actions.declare_file(bundle_dir + "/SHA256SUMS")
+    bundle_prefix = ctx.attr.prefix
 
-    def make_symlink(target):
-        symlink = ctx.actions.declare_file(bundle_dir + "/" + target.basename)
-        ctx.actions.symlink(output = symlink, target_file = target)
-        return symlink
-
-    symlinks = [make_symlink(file) for file in input_files]
+    if bundle_prefix == "":
+        fail("artifact bundle prefix must be set")
 
     # Compute checksums and print it to stdout & out file.
     # The filenames are stripped from anything but the basename.
@@ -144,32 +139,35 @@ def _artifact_bundle_impl(ctx):
     ctx.actions.run_shell(
         inputs = input_files,
         arguments = [file.path for file in input_files],
-        outputs = [out_checksums],
+        env = {
+            "BUNDLE_ROOT": bundle_root.path,
+            "BUNDLE_PREFIX": bundle_prefix,
+        },
+        outputs = [bundle_root],
         tools = [ctx.executable._sha256],
         command = """
         set -euo pipefail
 
-        out_checksums="{out}"
-        output=$(mktemp) # temporary file bc sha256 doesn't support writing to stdout (or /dev/stdout) directly
+        outdir="$BUNDLE_ROOT/$BUNDLE_PREFIX"
+
+        mkdir -p "$outdir"
+
+        out_checksums="$outdir/SHA256SUMS"
 
+        output=$(mktemp) # temporary file bc sha256 doesn't support writing to stdout (or /dev/stdout) directly
         for input in "$@"; do
-            if ! [[ $input =~ (\\.tar|\\.gz) ]]; then
-                echo "skipping non-archive file $input"
-                continue
-            fi
             {sha256} "$input" "$output"
             cat "$output" >> "$out_checksums"
             echo " $(basename $input)" >> "$out_checksums"
+            ln -s "$( realpath "$input" )" "$outdir/$(basename $input)"
         done
 
-        cat "$out_checksums"
-
         sort -o "$out_checksums" -k 2 "$out_checksums"
-        """.format(out = out_checksums.path, sha256 = ctx.executable._sha256.path),
+        """.format(sha256 = ctx.executable._sha256.path),
     )
 
     # Return the output file
-    return [DefaultInfo(files = depset([out_checksums] + symlinks))]
+    return [DefaultInfo(files = depset([bundle_root]))]
 
 # A rule that re-exports symlinks to all the inputs as well
 # as an extra file 'SHA256SUMS' containing the checksums of inputs.
@@ -180,6 +178,9 @@ artifact_bundle = rule(
             allow_files = True,
             mandatory = True,
         ),
+        "prefix": attr.string(
+            mandatory = True,
+        ),
         # The bazel-provided sha256 tool to avoid relying on tools from the container/env
         "_sha256": attr.label(
             default = "@bazel_tools//tools/build_defs/hash:sha256",