From 34efba83f4f92e5a1da3e87add312949d888d136 Mon Sep 17 00:00:00 2001 From: Eero Kelly Date: Wed, 10 Apr 2024 22:31:21 +0000 Subject: [PATCH] [NODE-1343] Use tzst and dflate over GNU tar --- ic-os/bootloader/BUILD.bazel | 2 +- ic-os/defs.bzl | 92 +++++++++++----------- ic-os/guestos/BUILD.bazel | 2 +- ic-os/guestos/defs.bzl | 2 +- ic-os/hostos/BUILD.bazel | 2 +- ic-os/hostos/defs.bzl | 10 +-- ic-os/setupos/defs.bzl | 8 +- rs/ic_os/dflate/src/main.rs | 2 - rs/ic_os/inject_files/src/main.rs | 32 +++++--- toolchains/sysimage/build_disk_image.py | 34 ++++---- toolchains/sysimage/build_ext4_image.py | 40 ++++++---- toolchains/sysimage/build_fat32_image.py | 37 +++++---- toolchains/sysimage/build_lvm_image.py | 51 +++++++----- toolchains/sysimage/build_upgrade_image.py | 48 +++++------ toolchains/sysimage/build_vfat_image.py | 37 +++++---- toolchains/sysimage/toolchain.bzl | 71 ++++++++++++----- toolchains/sysimage/verity_sign.py | 40 ++++++---- 17 files changed, 299 insertions(+), 211 deletions(-) diff --git a/ic-os/bootloader/BUILD.bazel b/ic-os/bootloader/BUILD.bazel index 9c26e8dbcc7..e87e399000c 100644 --- a/ic-os/bootloader/BUILD.bazel +++ b/ic-os/bootloader/BUILD.bazel @@ -22,7 +22,7 @@ genrule( ) vfat_image( - name = "partition-esp.tar", + name = "partition-esp.tzst", src = ":bootloader-tree.tar", partition_size = "100M", subdir = "boot/efi", diff --git a/ic-os/defs.bzl b/ic-os/defs.bzl index 348025fdd2c..12ef560eec3 100644 --- a/ic-os/defs.bzl +++ b/ic-os/defs.bzl @@ -65,7 +65,7 @@ def icos_build( # -------------------- Build grub partition -------------------- - build_grub_partition("partition-grub.tar", grub_config = image_deps.get("grub_config", default = None), tags = ["manual"]) + build_grub_partition("partition-grub.tzst", grub_config = image_deps.get("grub_config", default = None), tags = ["manual"]) # -------------------- Build the container image -------------------- @@ -118,7 +118,7 @@ def icos_build( # -------------------- Extract root partition -------------------- ext4_image( - name = "static-partition-root-unsigned.tar", + name = "static-partition-root-unsigned.tzst", src = ":rootfs-tree.tar", file_contexts = ":file_contexts", partition_size = image_deps["rootfs_size"], @@ -136,7 +136,7 @@ def icos_build( # -------------------- Extract boot partition -------------------- ext4_image( - name = "static-partition-boot.tar", + name = "static-partition-boot.tzst", src = ":rootfs-tree.tar", file_contexts = ":file_contexts", partition_size = image_deps["bootfs_size"], @@ -151,8 +151,8 @@ def icos_build( # -------------------- Inject extra files -------------------- inject_files( - name = "partition-root-unsigned.tar", - base = "static-partition-root-unsigned.tar", + name = "partition-root-unsigned.tzst", + base = "static-partition-root-unsigned.tzst", file_contexts = ":file_contexts", extra_files = { k: v @@ -163,8 +163,8 @@ def icos_build( if upgrades: inject_files( - name = "partition-root-test-unsigned.tar", - base = "static-partition-root-unsigned.tar", + name = "partition-root-test-unsigned.tzst", + base = "static-partition-root-unsigned.tzst", file_contexts = ":file_contexts", extra_files = { k: v @@ -175,22 +175,22 @@ def icos_build( # When boot_args are fixed, don't bother signing if "boot_args_template" not in image_deps: - native.alias(name = "partition-root.tar", actual = ":partition-root-unsigned.tar", tags = ["manual"]) + native.alias(name = "partition-root.tzst", actual = ":partition-root-unsigned.tzst", tags = ["manual"]) native.alias(name = "extra_boot_args", actual = image_deps["extra_boot_args"], tags = ["manual"]) if upgrades: - native.alias(name = "partition-root-test.tar", actual = ":partition-root-test-unsigned.tar", tags = ["manual"]) + native.alias(name = "partition-root-test.tzst", actual = ":partition-root-test-unsigned.tzst", tags = ["manual"]) native.alias(name = "extra_boot_test_args", actual = image_deps["extra_boot_args"], tags = ["manual"]) else: native.alias(name = "extra_boot_args_template", actual = image_deps["boot_args_template"], tags = ["manual"]) native.genrule( name = "partition-root-sign", - srcs = ["partition-root-unsigned.tar"], - outs = ["partition-root.tar", "partition-root-hash"], - cmd = "$(location //toolchains/sysimage:verity_sign.py) -i $< -o $(location :partition-root.tar) -r $(location partition-root-hash)", + srcs = ["partition-root-unsigned.tzst"], + outs = ["partition-root.tzst", "partition-root-hash"], + cmd = "$(location //toolchains/sysimage:verity_sign.py) -i $< -o $(location :partition-root.tzst) -r $(location partition-root-hash) -d $(location //rs/ic_os/dflate)", executable = False, - tools = ["//toolchains/sysimage:verity_sign.py"], + tools = ["//toolchains/sysimage:verity_sign.py", "//rs/ic_os/dflate"], tags = ["manual"], ) @@ -208,10 +208,10 @@ def icos_build( if upgrades: native.genrule( name = "partition-root-test-sign", - srcs = ["partition-root-test-unsigned.tar"], - outs = ["partition-root-test.tar", "partition-root-test-hash"], - cmd = "$(location //toolchains/sysimage:verity_sign.py) -i $< -o $(location :partition-root-test.tar) -r $(location partition-root-test-hash)", - tools = ["//toolchains/sysimage:verity_sign.py"], + srcs = ["partition-root-test-unsigned.tzst"], + outs = ["partition-root-test.tzst", "partition-root-test-hash"], + cmd = "$(location //toolchains/sysimage:verity_sign.py) -i $< -o $(location :partition-root-test.tzst) -r $(location partition-root-test-hash) -d $(location //rs/ic_os/dflate)", + tools = ["//toolchains/sysimage:verity_sign.py", "//rs/ic_os/dflate"], tags = ["manual"], ) @@ -227,8 +227,8 @@ def icos_build( ) inject_files( - name = "partition-boot.tar", - base = "static-partition-boot.tar", + name = "partition-boot.tzst", + base = "static-partition-boot.tzst", file_contexts = ":file_contexts", prefix = "/boot", extra_files = { @@ -245,8 +245,8 @@ def icos_build( if upgrades: inject_files( - name = "partition-boot-test.tar", - base = "static-partition-boot.tar", + name = "partition-boot-test.tzst", + base = "static-partition-boot.tzst", file_contexts = ":file_contexts", prefix = "/boot", extra_files = { @@ -273,10 +273,10 @@ def icos_build( name = "disk-img.tar", layout = image_deps["partition_table"], partitions = [ - "//ic-os/bootloader:partition-esp.tar", - ":partition-grub.tar", - ":partition-boot.tar", - ":partition-root.tar", + "//ic-os/bootloader:partition-esp.tzst", + ":partition-grub.tzst", + ":partition-boot.tzst", + ":partition-root.tzst", ] + custom_partitions, expanded_size = image_deps.get("expanded_size", default = None), tags = ["manual"], @@ -325,8 +325,8 @@ def icos_build( if upgrades: upgrade_image( name = "update-img.tar", - boot_partition = ":partition-boot.tar", - root_partition = ":partition-root.tar", + boot_partition = ":partition-boot.tzst", + root_partition = ":partition-root.tzst", tags = ["manual"], target_compatible_with = [ "@platforms//os:linux", @@ -371,8 +371,8 @@ def icos_build( upgrade_image( name = "update-img-test.tar", - boot_partition = ":partition-boot-test.tar", - root_partition = ":partition-root-test.tar", + boot_partition = ":partition-boot-test.tzst", + root_partition = ":partition-root-test.tzst", tags = ["manual"], target_compatible_with = [ "@platforms//os:linux", @@ -629,7 +629,7 @@ def boundary_node_icos_build( tags = ["manual"], ) - build_grub_partition("partition-grub.tar", tags = ["manual"]) + build_grub_partition("partition-grub.tzst", tags = ["manual"]) build_container_filesystem_config_file = Label(image_deps["build_container_filesystem_config_file"]) @@ -642,7 +642,7 @@ def boundary_node_icos_build( ) ext4_image( - name = "partition-config.tar", + name = "partition-config.tzst", partition_size = "100M", target_compatible_with = [ "@platforms//os:linux", @@ -659,7 +659,7 @@ def boundary_node_icos_build( ) ext4_image( - name = "static-partition-boot.tar", + name = "static-partition-boot.tzst", src = ":rootfs-tree.tar", partition_size = "1G", subdir = "boot/", @@ -670,8 +670,8 @@ def boundary_node_icos_build( ) inject_files( - name = "partition-boot.tar", - base = "static-partition-boot.tar", + name = "partition-boot.tzst", + base = "static-partition-boot.tzst", prefix = "/boot", extra_files = { k: v @@ -686,7 +686,7 @@ def boundary_node_icos_build( ) ext4_image( - name = "static-partition-root-unsigned.tar", + name = "static-partition-root-unsigned.tzst", src = ":rootfs-tree.tar", partition_size = "3G", strip_paths = [ @@ -700,8 +700,8 @@ def boundary_node_icos_build( ) inject_files( - name = "partition-root-unsigned.tar", - base = "static-partition-root-unsigned.tar", + name = "partition-root-unsigned.tzst", + base = "static-partition-root-unsigned.tzst", extra_files = { k: v for k, v in (image_deps["rootfs"].items() + [(":version.txt", "/opt/ic/share/version.txt:0644")]) @@ -711,11 +711,11 @@ def boundary_node_icos_build( native.genrule( name = "partition-root-sign", - srcs = ["partition-root-unsigned.tar"], - outs = ["partition-root.tar", "partition-root-hash"], - cmd = "$(location //toolchains/sysimage:verity_sign.py) -i $< -o $(location :partition-root.tar) -r $(location partition-root-hash)", + srcs = ["partition-root-unsigned.tzst"], + outs = ["partition-root.tzst", "partition-root-hash"], + cmd = "$(location //toolchains/sysimage:verity_sign.py) -i $< -o $(location :partition-root.tzst) -r $(location partition-root-hash) -d $(location //rs/ic_os/dflate)", executable = False, - tools = ["//toolchains/sysimage:verity_sign.py"], + tools = ["//toolchains/sysimage:verity_sign.py", "//rs/ic_os/dflate"], tags = ["manual"], ) @@ -734,11 +734,11 @@ def boundary_node_icos_build( name = "disk-img.tar", layout = "//ic-os/boundary-guestos:partitions.csv", partitions = [ - "//ic-os/bootloader:partition-esp.tar", - ":partition-grub.tar", - ":partition-config.tar", - ":partition-boot.tar", - ":partition-root.tar", + "//ic-os/bootloader:partition-esp.tzst", + ":partition-grub.tzst", + ":partition-config.tzst", + ":partition-boot.tzst", + ":partition-root.tzst", ], expanded_size = "50G", tags = ["manual"], diff --git a/ic-os/guestos/BUILD.bazel b/ic-os/guestos/BUILD.bazel index f3cc7709ba5..eac38af35a8 100644 --- a/ic-os/guestos/BUILD.bazel +++ b/ic-os/guestos/BUILD.bazel @@ -20,7 +20,7 @@ filegroup( ) ext4_image( - name = "partition-config.tar", + name = "partition-config.tzst", partition_size = "100M", tags = ["manual"], target_compatible_with = [ diff --git a/ic-os/guestos/defs.bzl b/ic-os/guestos/defs.bzl index 72da73a7118..8a1be6fe532 100644 --- a/ic-os/guestos/defs.bzl +++ b/ic-os/guestos/defs.bzl @@ -57,7 +57,7 @@ def image_deps(mode, malicious = False): "bootfs_size": "1G", # Add any custom partitions to the manifest - "custom_partitions": lambda: [Label("//ic-os/guestos:partition-config.tar")], + "custom_partitions": lambda: [Label("//ic-os/guestos:partition-config.tzst")], # We will install extra_boot_args onto the system, after substituting the # hash of the root filesystem into it. Track the template (before diff --git a/ic-os/hostos/BUILD.bazel b/ic-os/hostos/BUILD.bazel index e08f1f78d9a..99a3187c5e4 100644 --- a/ic-os/hostos/BUILD.bazel +++ b/ic-os/hostos/BUILD.bazel @@ -18,7 +18,7 @@ filegroup( ) ext4_image( - name = "partition-config.tar", + name = "partition-config.tzst", partition_size = "100M", tags = ["manual"], target_compatible_with = [ diff --git a/ic-os/hostos/defs.bzl b/ic-os/hostos/defs.bzl index dd7ec53c16c..21adbbab0ad 100644 --- a/ic-os/hostos/defs.bzl +++ b/ic-os/hostos/defs.bzl @@ -74,12 +74,12 @@ def image_deps(mode, _malicious = False): # earlier in the pipeline, and is depended on by the final disk image. def _custom_partitions(): lvm_image( - name = "partition-hostlvm.tar", + name = "partition-hostlvm.tzst", layout = Label("//ic-os/hostos:volumes.csv"), partitions = [ - Label("//ic-os/hostos:partition-config.tar"), - ":partition-boot.tar", - ":partition-root.tar", + Label("//ic-os/hostos:partition-config.tzst"), + ":partition-boot.tzst", + ":partition-root.tzst", ], vg_name = "hostlvm", vg_uuid = "4c7GVZ-Df82-QEcJ-xXtV-JgRL-IjLE-hK0FgA", @@ -92,4 +92,4 @@ def _custom_partitions(): ], ) - return [":partition-hostlvm.tar"] + return [":partition-hostlvm.tzst"] diff --git a/ic-os/setupos/defs.bzl b/ic-os/setupos/defs.bzl index 3791ac5c290..fa6c5bd7164 100644 --- a/ic-os/setupos/defs.bzl +++ b/ic-os/setupos/defs.bzl @@ -118,7 +118,7 @@ def _custom_partitions(mode): ) fat32_image( - name = "partition-config.tar", + name = "partition-config.tzst", src = "config_tar", label = "CONFIG", partition_size = "50M", @@ -151,7 +151,7 @@ def _custom_partitions(mode): ) ext4_image( - name = "partition-data.tar", + name = "partition-data.tzst", src = "data_tar", partition_size = "1750M", subdir = "data", @@ -162,6 +162,6 @@ def _custom_partitions(mode): ) return [ - ":partition-config.tar", - ":partition-data.tar", + ":partition-config.tzst", + ":partition-data.tzst", ] diff --git a/rs/ic_os/dflate/src/main.rs b/rs/ic_os/dflate/src/main.rs index da97f48419f..aa57f62497d 100644 --- a/rs/ic_os/dflate/src/main.rs +++ b/rs/ic_os/dflate/src/main.rs @@ -31,8 +31,6 @@ fn main() -> Result<()> { .to_string_lossy() .into_owned(); - println!("Adding {file_name}"); - let state = scan_file_for_holes(&mut source, file_name)?; add_file_to_archive(&mut source, &mut dest, state)?; } diff --git a/rs/ic_os/inject_files/src/main.rs b/rs/ic_os/inject_files/src/main.rs index fc5933ff7ed..d409d7dc36c 100644 --- a/rs/ic_os/inject_files/src/main.rs +++ b/rs/ic_os/inject_files/src/main.rs @@ -22,6 +22,8 @@ struct Cli { file_contexts: Option, #[arg(long)] prefix: Option, + #[arg(short)] + dflate: PathBuf, extra_files: Vec, } @@ -42,7 +44,7 @@ async fn main() -> Result<()> { .status() .await; - let mut target = ExtPartition::open(temp_file, cli.index).await?; + let mut target = ExtPartition::open(temp_file.clone(), cli.index).await?; let contexts = cli .file_contexts @@ -87,19 +89,25 @@ async fn main() -> Result<()> { target.close().await?; // TODO: Quick hack to unpack and repack file - let mut cmd = Command::new("tar"); + // If dflate is ever misbehaving, it can be replaced with: + // tar cf --sort=name --owner=root:0 --group=root:0 --mtime="UTC 1970-01-01 00:00:00" --sparse --hole-detection=raw -C + let temp_tar = temp_dir.path().join("partition.tar"); + let mut cmd = Command::new(cli.dflate); let _ = cmd - .arg("cf") + .arg("--input") + .arg(&temp_file) + .arg("--output") + .arg(&temp_tar) + .status() + .await; + + let mut cmd = Command::new("zstd"); + let _ = cmd + .arg("-q") + .arg("--threads=0") + .arg(&temp_tar) + .arg("-o") .arg(cli.output) - .arg("--sort=name") - .arg("--owner=root:0") - .arg("--group=root:0") - .arg("--mtime=UTC 1970-01-01 00:00:00") - .arg("--sparse") - .arg("--hole-detection=raw") - .arg("-C") - .arg(temp_dir.path()) - .arg("partition.img") .status() .await; diff --git a/toolchains/sysimage/build_disk_image.py b/toolchains/sysimage/build_disk_image.py index 1deebc1c448..db9eb843361 100755 --- a/toolchains/sysimage/build_disk_image.py +++ b/toolchains/sysimage/build_disk_image.py @@ -4,11 +4,10 @@ # table description. The actual disk image is wrapped up into a tar file # because the raw image file is sparse. # -# The input partition images are also expected to be given as tar files, -# where each tar archive must contain a single file named "partition.img". +# The input partition images are also expected to be given as tzst files, # # Call example: -# build_disk_image -p partitions.csv -o disk.img.tar part1.tar part2.tar ... +# build_disk_image -p partitions.csv -o disk.img.tar part1.tzst part2.tzst ... # import argparse import atexit @@ -98,7 +97,14 @@ def _copyfile(source, target, size): size -= len(data) -def write_partition_image_from_tar(gpt_entry, image_file, partition_tf): +def write_partition_image_from_tzst(gpt_entry, image_file, partition_tzst): + tmpdir = tempfile.mkdtemp(prefix="icosbuild") + atexit.register(lambda: subprocess.run(["rm", "-rf", tmpdir], check=True)) + + partition_tf = os.path.join(tmpdir, "partition.tar") + subprocess.run(["zstd", "-q", "--threads=0", "-f", "-d", partition_tzst, "-o", partition_tf], check=True) + + partition_tf = tarfile.open(partition_tf, mode="r:") base = gpt_entry["start"] * 512 with os.fdopen(os.open(image_file, os.O_RDWR), "wb+") as target: for member in partition_tf: @@ -139,6 +145,7 @@ def main(): nargs="*", help="Partitions to write. These must match the CSV partition table entries.", ) + parser.add_argument("-d", "--dflate", help="Path to dflate", type=str) args = parser.parse_args(sys.argv[1:]) @@ -172,7 +179,7 @@ def main(): partition_file = select_partition_file(name, partition_files) if partition_file: - write_partition_image_from_tar(entry, disk_image, tarfile.open(partition_file, mode="r:")) + write_partition_image_from_tzst(entry, disk_image, partition_file) else: print("No partition file for '%s' found, leaving empty" % name) @@ -180,20 +187,15 @@ def main(): if args.expanded_size: subprocess.run(["truncate", "--size", args.expanded_size, disk_image], check=True) + # If dflate is ever misbehaving, it can be replaced with: + # tar cf --sort=name --owner=root:0 --group=root:0 --mtime="UTC 1970-01-01 00:00:00" --sparse --hole-detection=raw -C subprocess.run( [ - "tar", - "cf", + args.dflate, + "--input", + disk_image, + "--output", out_file, - "--sort=name", - "--owner=root:0", - "--group=root:0", - "--mtime=UTC 1970-01-01 00:00:00", - "--sparse", - "--hole-detection=raw", - "-C", - tmpdir, - "disk.img", ], check=True, ) diff --git a/toolchains/sysimage/build_ext4_image.py b/toolchains/sysimage/build_ext4_image.py index a3a2d6b73e6..a4713ca88dd 100755 --- a/toolchains/sysimage/build_ext4_image.py +++ b/toolchains/sysimage/build_ext4_image.py @@ -2,10 +2,10 @@ # # Packs contents of a tar file into a ext4 image (possibly taking only a # subdirectory of the full tar file). The (sparse) ext4 image itself is then -# wrapped into a tar file itself. +# wrapped into a tzst file. # # Call example: -# build_ext4_image -s 10M -o partition.img.tar -p boot -i dockerimg.tar -S file_contexts +# build_ext4_image -s 10M -o partition.img.tzst -p boot -i dockerimg.tar -S file_contexts # import argparse import atexit @@ -171,7 +171,7 @@ def fixup_permissions(fs_rootdir, fakeroot_statefile, image_file): def make_argparser(): parser = argparse.ArgumentParser() parser.add_argument("-s", "--size", help="Size of image to build", type=str) - parser.add_argument("-o", "--output", help="Target (tar) file to write partition image to", type=str) + parser.add_argument("-o", "--output", help="Target (tzst) file to write partition image to", type=str) parser.add_argument( "-i", "--input", help="Source (tar) file to take files from", type=str, default="", required=False ) @@ -199,6 +199,7 @@ def make_argparser(): default=[], help="Directories to be cleared from the tree; expects a list of full paths", ) + parser.add_argument("-d", "--dflate", help="Path to dflate", type=str) return parser @@ -256,22 +257,29 @@ def main(): fixup_permissions(os.path.join(fs_basedir, limit_prefix), fakeroot_statefile, image_file) subprocess.run(['sync'], check=True) - # Wrap the built filesystem image up in a tar file. Use sparse to - # deflate all the zeroes left unwritten during build. + + # If dflate is ever misbehaving, it can be replaced with: + # tar cf --sort=name --owner=root:0 --group=root:0 --mtime="UTC 1970-01-01 00:00:00" --sparse --hole-detection=raw -C + temp_tar = os.path.join(tmpdir, "partition.tar") + subprocess.run( + [ + args.dflate, + "--input", + image_file, + "--output", + temp_tar, + ], + check=True, + ) + subprocess.run( [ - "tar", - "cf", + "zstd", + "-q", + "--threads=0", + temp_tar, + "-o", out_file, - "--sort=name", - "--owner=root:0", - "--group=root:0", - "--mtime=UTC 1970-01-01 00:00:00", - "--sparse", - "--hole-detection=raw", - "-C", - tmpdir, - "partition.img", ], check=True, ) diff --git a/toolchains/sysimage/build_fat32_image.py b/toolchains/sysimage/build_fat32_image.py index 0944929388d..3eca69f95d2 100755 --- a/toolchains/sysimage/build_fat32_image.py +++ b/toolchains/sysimage/build_fat32_image.py @@ -2,10 +2,10 @@ # # Packs contents of a tar file into a fat32 image (possibly taking only a # subdirectory of the full tar file). The (sparse) fat32 image itself is then -# wrapped into a tar file itself. +# wrapped into a tzst file. # # Call example: -# build_fat32_image -s 10M -o partition.img.tar -p boot/efi -i dockerimg.tar +# build_fat32_image -s 10M -o partition.img.tzst -p boot/efi -i dockerimg.tar # import argparse import atexit @@ -86,7 +86,7 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument("-l", "--label", help="Label to add to partition", type=str) parser.add_argument("-s", "--size", help="Size of image to build", type=str) - parser.add_argument("-o", "--output", help="Target (tar) file to write partition image to", type=str) + parser.add_argument("-o", "--output", help="Target (tzst) file to write partition image to", type=str) parser.add_argument( "-i", "--input", help="Source (tar) file to take files from", type=str, default="", required=False ) @@ -105,6 +105,7 @@ def main(): nargs="*", help="Extra files to install; expects list of sourcefile:targetfile:mode", ) + parser.add_argument("-d", "--dflate", help="Path to dflate", type=str) args = parser.parse_args(sys.argv[1:]) @@ -141,20 +142,28 @@ def path_transform(path, limit_prefix=limit_prefix): install_extra_files(image_file, extra_files, path_transform) + # If dflate is ever misbehaving, it can be replaced with: + # tar cf --sort=name --owner=root:0 --group=root:0 --mtime="UTC 1970-01-01 00:00:00" --sparse --hole-detection=raw -C + temp_tar = os.path.join(tmpdir, "partition.tar") subprocess.run( [ - "tar", - "cf", + args.dflate, + "--input", + image_file, + "--output", + temp_tar, + ], + check=True, + ) + + subprocess.run( + [ + "zstd", + "-q", + "--threads=0", + temp_tar, + "-o", out_file, - "--sort=name", - "--owner=root:0", - "--group=root:0", - "--mtime=UTC 1970-01-01 00:00:00", - "--sparse", - "--hole-detection=raw", - "-C", - tmpdir, - "partition.img", ], check=True, ) diff --git a/toolchains/sysimage/build_lvm_image.py b/toolchains/sysimage/build_lvm_image.py index 7931222fb90..2dc4a47c416 100755 --- a/toolchains/sysimage/build_lvm_image.py +++ b/toolchains/sysimage/build_lvm_image.py @@ -1,14 +1,13 @@ #!/usr/bin/env python3 # # Builds a lvm image from individual partition images and a volume -# table description. The actual lvm image is wrapped up into a tar file +# table description. The actual lvm image is wrapped up into a tzst file # because the raw file is sparse. # -# The input partition images are also expected to be given as tar files, -# where each tar archive must contain a single file named "partition.img". +# The input partition images are also expected to be given as tzst files, # # Call example: -# build_lvm_image -v volumes.csv -o partition-hostlvm.tar part1.tar part2.tar ... +# build_lvm_image -v volumes.csv -o partition-hostlvm.tzst part1.tzst part2.tzst ... # import argparse import atexit @@ -26,7 +25,7 @@ def main(): parser = argparse.ArgumentParser() - parser.add_argument("-o", "--out", help="Target (tar) file to write lvm image to", type=str) + parser.add_argument("-o", "--out", help="Target (tzst) file to write lvm image to", type=str) parser.add_argument("-v", "--volume_table", help="CSV file describing the volume table", type=str) parser.add_argument("-n", "--vg-name", metavar="vg_name", help="Volume Group name to use", type=str) parser.add_argument("-u", "--vg-uuid", metavar="vg_uuid", help="UUID to use for Volume Group", type=str) @@ -38,6 +37,7 @@ def main(): nargs="*", help="Partitions to write. These must match the CSV volume table entries.", ) + parser.add_argument("-d", "--dflate", help="Path to dflate", type=str) args = parser.parse_args(sys.argv[1:]) @@ -74,24 +74,32 @@ def main(): partition_file = select_partition_file(name, partition_files) if partition_file: - write_partition_image_from_tar(entry, lvm_image, tarfile.open(partition_file, mode="r:")) + write_partition_image_from_tzst(entry, lvm_image, partition_file) else: print("No partition file for '%s' found, leaving empty" % name) + # If dflate is ever misbehaving, it can be replaced with: + # tar cf --sort=name --owner=root:0 --group=root:0 --mtime="UTC 1970-01-01 00:00:00" --sparse --hole-detection=raw -C + temp_tar = os.path.join(tmpdir, "partition.tar") subprocess.run( [ - "tar", - "cf", + args.dflate, + "--input", + lvm_image, + "--output", + temp_tar, + ], + check=True, + ) + + subprocess.run( + [ + "zstd", + "-q", + "--threads=0", + temp_tar, + "-o", out_file, - "--sort=name", - "--owner=root:0", - "--group=root:0", - "--mtime=UTC 1970-01-01 00:00:00", - "--sparse", - "--hole-detection=raw", - "-C", - tmpdir, - "partition.img", ], check=True, ) @@ -165,7 +173,14 @@ def select_partition_file(name, partition_files): return None -def write_partition_image_from_tar(lvm_entry, image_file, partition_tf): +def write_partition_image_from_tzst(lvm_entry, image_file, partition_tzst): + tmpdir = tempfile.mkdtemp(prefix="icosbuild") + atexit.register(lambda: subprocess.run(["rm", "-rf", tmpdir], check=True)) + + partition_tf = os.path.join(tmpdir, "partition.tar") + subprocess.run(["zstd", "-q", "--threads=0", "-f", "-d", partition_tzst, "-o", partition_tf], check=True) + + partition_tf = tarfile.open(partition_tf, mode="r:") base = LVM_HEADER_SIZE_BYTES + (lvm_entry["start"] * EXTENT_SIZE_BYTES) with os.fdopen(os.open(image_file, os.O_RDWR), "wb+") as target: for member in partition_tf: diff --git a/toolchains/sysimage/build_upgrade_image.py b/toolchains/sysimage/build_upgrade_image.py index aa13794490e..4db64f72022 100755 --- a/toolchains/sysimage/build_upgrade_image.py +++ b/toolchains/sysimage/build_upgrade_image.py @@ -3,30 +3,24 @@ # Builds an upgrade image from individual partition images. # # Call example: -# build_upgrade_image -o upgrade.tar.gz -b boot.img.tar -r root.img.tar -v version.txt -c gzip +# build_upgrade_image -o upgrade.tar.gz -b boot.img.tzst -r root.img.tzst -v version.txt # import argparse import atexit +import os import shutil import subprocess import sys import tempfile -COMPRESSOR_PROGRAMS = { - "gz": ["--use-compress-program=gzip"], - "gzip": ["--use-compress-program=gzip"], - "zstd": ["--use-compress-program=zstd --threads=0 -10"], - "": [], -} - def main(): parser = argparse.ArgumentParser() parser.add_argument("-o", "--out", help="Target (tar) file to write upgrade package to", type=str) - parser.add_argument("-b", "--boot", help="The (tarred) boot filesystem image", type=str) - parser.add_argument("-r", "--root", help="The (tarred) root filesystem image", type=str) + parser.add_argument("-b", "--boot", help="The (tzst) boot filesystem image", type=str) + parser.add_argument("-r", "--root", help="The (tzst) root filesystem image", type=str) parser.add_argument("-v", "--versionfile", help="The version file in the upgrade image", type=str) - parser.add_argument("-c", "--compression", help="Compression format of upgrade package", type=str, default="") + parser.add_argument("-d", "--dflate", help="Path to dflate", type=str) args = parser.parse_args(sys.argv[1:]) @@ -34,32 +28,32 @@ def main(): root_image = args.root boot_image = args.boot version_file = args.versionfile - compression = args.compression tmpdir = tempfile.mkdtemp(prefix="icosbuild") atexit.register(lambda: subprocess.run(["rm", "-rf", tmpdir], check=True)) + boot_path = os.path.join(tmpdir, "boot.img") subprocess.run(["tar", "xf", boot_image, "--transform=s/partition.img/boot.img/", "-C", tmpdir], check=True) + root_path = os.path.join(tmpdir, "root.img") subprocess.run(["tar", "xf", root_image, "--transform=s/partition.img/root.img/", "-C", tmpdir], check=True) - shutil.copy(version_file, tmpdir + "/VERSION.TXT", follow_symlinks=True) + + version_path = os.path.join(tmpdir, "VERSION.TXT") + shutil.copy(version_file, version_path, follow_symlinks=True) + + # If dflate is ever misbehaving, it can be replaced with: + # tar cf --sort=name --owner=root:0 --group=root:0 --mtime="UTC 1970-01-01 00:00:00" --sparse --hole-detection=raw -C subprocess.run( [ - "tar", - "cf", + args.dflate, + "--input", + boot_path, + "--input", + root_path, + "--input", + version_path, + "--output", out_file, - "--sort=name", - "--owner=root:0", - "--group=root:0", - "--mtime=UTC 1970-01-01 00:00:00", - ] - + COMPRESSOR_PROGRAMS[compression] - + [ - "--sparse", - "--hole-detection=raw", - "-C", - tmpdir, - ".", ], check=True, ) diff --git a/toolchains/sysimage/build_vfat_image.py b/toolchains/sysimage/build_vfat_image.py index f1a0d679154..9ca70dd6609 100755 --- a/toolchains/sysimage/build_vfat_image.py +++ b/toolchains/sysimage/build_vfat_image.py @@ -2,10 +2,10 @@ # # Packs contents of a tar file into a vfat image (possibly taking only a # subdirectory of the full tar file). The (sparse) vfat image itself is then -# wrapped into a tar file itself. +# wrapped into a tzst file. # # Call example: -# build_vfat_image -s 10M -o partition.img.tar -p boot/efi -i dockerimg.tar +# build_vfat_image -s 10M -o partition.img.tzst -p boot/efi -i dockerimg.tar # import argparse import atexit @@ -85,7 +85,7 @@ def parse_size(s): def main(): parser = argparse.ArgumentParser() parser.add_argument("-s", "--size", help="Size of image to build", type=str) - parser.add_argument("-o", "--output", help="Target (tar) file to write partition image to", type=str) + parser.add_argument("-o", "--output", help="Target (tzst) file to write partition image to", type=str) parser.add_argument( "-i", "--input", help="Source (tar) file to take files from", type=str, default="", required=False ) @@ -104,6 +104,7 @@ def main(): nargs="*", help="Extra files to install; expects list of sourcefile:targetfile:mode", ) + parser.add_argument("-d", "--dflate", help="Path to dflate", type=str) args = parser.parse_args(sys.argv[1:]) @@ -137,20 +138,28 @@ def path_transform(path, limit_prefix=limit_prefix): install_extra_files(image_file, extra_files, path_transform) + # If dflate is ever misbehaving, it can be replaced with: + # tar cf --sort=name --owner=root:0 --group=root:0 --mtime="UTC 1970-01-01 00:00:00" --sparse --hole-detection=raw -C + temp_tar = os.path.join(tmpdir, "partition.tar") subprocess.run( [ - "tar", - "cf", + args.dflate, + "--input", + image_file, + "--output", + temp_tar, + ], + check=True, + ) + + subprocess.run( + [ + "zstd", + "-q", + "--threads=0", + temp_tar, + "-o", out_file, - "--sort=name", - "--owner=root:0", - "--group=root:0", - "--mtime=UTC 1970-01-01 00:00:00", - "--sparse", - "--hole-detection=raw", - "-C", - tmpdir, - "partition.img", ], check=True, ) diff --git a/toolchains/sysimage/toolchain.bzl b/toolchains/sysimage/toolchain.bzl index fd43b7b96f1..f6f1b76ef03 100644 --- a/toolchains/sysimage/toolchain.bzl +++ b/toolchains/sysimage/toolchain.bzl @@ -142,6 +142,7 @@ build_container_filesystem = rule( def _vfat_image_impl(ctx): tool = ctx.files._build_vfat_image[0] + dflate = ctx.files._dflate[0] if len(ctx.files.src) > 0: args = ["-i", ctx.files.src[0].path] @@ -158,6 +159,8 @@ def _vfat_image_impl(ctx): ctx.attr.partition_size, "-p", ctx.attr.subdir, + "-d", + dflate.path, ] for input_target, install_target in ctx.attr.extra_files.items(): @@ -169,7 +172,7 @@ def _vfat_image_impl(ctx): arguments = args, inputs = inputs, outputs = [out], - tools = [tool], + tools = [tool, dflate], ) return [DefaultInfo(files = depset([out]))] @@ -194,11 +197,16 @@ vfat_image = rule( allow_files = True, default = ":build_vfat_image.py", ), + "_dflate": attr.label( + allow_files = True, + default = "//rs/ic_os/dflate", + ), }, ) def _fat32_image_impl(ctx): tool = ctx.files._build_fat32_image[0] + dflate = ctx.files._dflate[0] if len(ctx.files.src) > 0: args = ["-i", ctx.files.src[0].path] @@ -215,6 +223,8 @@ def _fat32_image_impl(ctx): ctx.attr.partition_size, "-p", ctx.attr.subdir, + "-d", + dflate.path, ] for input_target, install_target in ctx.attr.extra_files.items(): @@ -229,7 +239,7 @@ def _fat32_image_impl(ctx): arguments = args, inputs = inputs, outputs = [out], - tools = [tool], + tools = [tool, dflate], ) return [DefaultInfo(files = depset([out]))] @@ -255,11 +265,16 @@ fat32_image = rule( allow_files = True, default = ":build_fat32_image.py", ), + "_dflate": attr.label( + allow_files = True, + default = "//rs/ic_os/dflate", + ), }, ) def _ext4_image_impl(ctx): tool = ctx.files._build_ext4_image[0] + dflate = ctx.files._dflate[0] out = ctx.actions.declare_file(ctx.label.name) @@ -276,6 +291,8 @@ def _ext4_image_impl(ctx): ctx.attr.partition_size, "-p", ctx.attr.subdir, + "-d", + dflate.path, ] if len(ctx.files.file_contexts) > 0: args += ["-S", ctx.files.file_contexts[0].path] @@ -289,7 +306,7 @@ def _ext4_image_impl(ctx): arguments = args, inputs = inputs, outputs = [out], - tools = [tool], + tools = [tool, dflate], ) return [DefaultInfo(files = depset([out]))] @@ -315,11 +332,16 @@ ext4_image = rule( allow_files = True, default = ":build_ext4_image.py", ), + "_dflate": attr.label( + allow_files = True, + default = "//rs/ic_os/dflate", + ), }, ) def _inject_files_impl(ctx): tool = ctx.files._inject_files[0] + dflate = ctx.files._dflate[0] out = ctx.actions.declare_file(ctx.label.name) @@ -330,6 +352,8 @@ def _inject_files_impl(ctx): ctx.files.base[0].path, "--output", out.path, + "-d", + dflate.path, ] if len(ctx.files.file_contexts) > 0: @@ -348,7 +372,7 @@ def _inject_files_impl(ctx): arguments = args, inputs = inputs, outputs = [out], - tools = [tool], + tools = [tool, dflate], ) return [DefaultInfo(files = depset([out]))] @@ -375,11 +399,16 @@ inject_files = rule( allow_files = True, default = "//rs/ic_os/inject_files:inject-files", ), + "_dflate": attr.label( + allow_files = True, + default = "//rs/ic_os/dflate", + ), }, ) def _disk_image_impl(ctx): tool_file = ctx.files._build_disk_image_tool[0] + dflate = ctx.files._dflate[0] in_layout = ctx.files.layout[0] partitions = ctx.files.partitions @@ -390,7 +419,7 @@ def _disk_image_impl(ctx): for p in partitions: partition_files.append(p.path) - args = ["-p", in_layout.path, "-o", out.path] + args = ["-p", in_layout.path, "-o", out.path, "-d", dflate.path] if expanded_size: args += ["-s", expanded_size] @@ -402,7 +431,7 @@ def _disk_image_impl(ctx): arguments = args, inputs = [in_layout] + partitions, outputs = [out], - tools = [tool_file], + tools = [tool_file, dflate], ) return [DefaultInfo(files = depset([out]))] @@ -422,11 +451,16 @@ disk_image = rule( allow_files = True, default = ":build_disk_image.py", ), + "_dflate": attr.label( + allow_files = True, + default = "//rs/ic_os/dflate", + ), }, ) def _lvm_image_impl(ctx): tool_file = ctx.files._build_lvm_image_tool[0] + dflate = ctx.files._dflate[0] in_layout = ctx.files.layout[0] vg_name = ctx.attr.vg_name @@ -439,7 +473,7 @@ def _lvm_image_impl(ctx): for p in partitions: partition_files.append(p.path) - args = ["-v", in_layout.path, "-n", vg_name, "-u", vg_uuid, "-p", pv_uuid, "-o", out.path] + args = ["-v", in_layout.path, "-n", vg_name, "-u", vg_uuid, "-p", pv_uuid, "-o", out.path, "-d", dflate.path] args += partition_files @@ -448,7 +482,7 @@ def _lvm_image_impl(ctx): arguments = args, inputs = [in_layout] + partitions, outputs = [out], - tools = [tool_file], + tools = [tool_file, dflate], ) return [DefaultInfo(files = depset([out]))] @@ -470,32 +504,32 @@ lvm_image = rule( allow_files = True, default = ":build_lvm_image.py", ), + "_dflate": attr.label( + allow_files = True, + default = "//rs/ic_os/dflate", + ), }, ) def _upgrade_image_impl(ctx): tool_file = ctx.files._build_upgrade_image_tool[0] + dflate = ctx.files._dflate[0] in_boot_partition = ctx.files.boot_partition[0] in_root_partition = ctx.files.root_partition[0] in_version_file = ctx.files.version_file[0] out = ctx.actions.declare_file(ctx.label.name) - if ctx.attr.compression: - compress = "-c %s" % ctx.attr.compression - else: - compress = "" - ctx.actions.run_shell( inputs = [in_boot_partition, in_root_partition, in_version_file], outputs = [out], - command = "python3 %s -b %s -r %s -v %s %s -o %s" % ( + command = "python3 %s -b %s -r %s -v %s -o %s -d %s" % ( tool_file.path, in_boot_partition.path, in_root_partition.path, in_version_file.path, - compress, out.path, + dflate.path, ), ) @@ -516,13 +550,14 @@ upgrade_image = rule( allow_files = True, mandatory = True, ), - "compression": attr.string( - default = "", - ), "_build_upgrade_image_tool": attr.label( allow_files = True, default = ":build_upgrade_image.py", ), + "_dflate": attr.label( + allow_files = True, + default = "//rs/ic_os/dflate", + ), }, ) diff --git a/toolchains/sysimage/verity_sign.py b/toolchains/sysimage/verity_sign.py index 30254dc952e..0e3edfff0ff 100755 --- a/toolchains/sysimage/verity_sign.py +++ b/toolchains/sysimage/verity_sign.py @@ -16,8 +16,8 @@ def main(): parser = argparse.ArgumentParser() - parser.add_argument("-i", "--input", help="Input (tar) file of tree to operate in", type=str) - parser.add_argument("-o", "--output", help="Target (tar) file of tree to write to", type=str) + parser.add_argument("-i", "--input", help="Input (tzst) file of tree to operate in", type=str) + parser.add_argument("-o", "--output", help="Target (tzst) file of tree to write to", type=str) parser.add_argument("-r", "--root-hash", help="Output file containing root hash", type=str) parser.add_argument( "-s", @@ -36,10 +36,12 @@ def main(): type=int, default=10 * 1024 * 1024 * 1024 - 128 * 1024 * 1024, ) + parser.add_argument("-d", "--dflate", help="Path to dflate", type=str) args = parser.parse_args(sys.argv[1:]) tmpdir = tempfile.mkdtemp(prefix="icosbuild") + partition = os.path.join(tmpdir, "partition.img") atexit.register(lambda: subprocess.run(["rm", "-rf", tmpdir], check=True)) subprocess.run( @@ -56,8 +58,8 @@ def main(): verity_cmdline = [ "/usr/sbin/veritysetup", "format", - os.path.join(tmpdir, "partition.img"), - os.path.join(tmpdir, "partition.img"), + partition, + partition, "--hash-offset", str(args.hash_offset), "--uuid", @@ -86,20 +88,28 @@ def main(): with open(args.root_hash, "w") as f: f.write(root_hash + "\n") + # If dflate is ever misbehaving, it can be replaced with: + # tar cf --sort=name --owner=root:0 --group=root:0 --mtime="UTC 1970-01-01 00:00:00" --sparse --hole-detection=raw -C + temp_tar = os.path.join(tmpdir, "partition.tar") subprocess.run( [ - "tar", - "cf", + args.dflate, + "--input", + partition, + "--output", + temp_tar, + ], + check=True, + ) + + subprocess.run( + [ + "zstd", + "-q", + "--threads=0", + temp_tar, + "-o", args.output, - "--sort=name", - "--owner=root:0", - "--group=root:0", - "--mtime=UTC 1970-01-01 00:00:00", - "--sparse", - "--hole-detection=raw", - "-C", - tmpdir, - "partition.img", ], check=True, )