Merge branch 'eero/zstd' into 'master'

[NODE-1343] Use tzst and dflate over GNU tar Using dflate and zstd compression gives a boost to performance and artifact size: Build time: `bazel build --config=local ic-os/*/envs/dev` GuestOS: 618.733s => 573.244s (7.4% faster) HostOS: 423.963s => 206.524s (51.3% faster) SetupOS: 701.844s => 608.022s (13.4% faster) Artifact size: `du -h ic-os/` ``` 21G ic-os/guestos/envs/dev 24G ic-os/hostos/envs/dev 11G ic-os/setupos/envs/dev 55G ic-os/ ``` => ``` 9.3G ic-os/guestos/envs/dev (55.7% smaller) 13G ic-os/hostos/envs/dev (45.8% smaller) 8.6G ic-os/setupos/envs/dev (21.8% smaller) 31G ic-os/ (43.6% smaller) ``` See merge request dfinity-lab/public/ic!18393
dfinity · Apr 10, 2024 · 19dbb5c · 19dbb5c
2 parents fd03bd3 + 34efba8
commit 19dbb5c
Show file tree

Hide file tree

Showing 17 changed files with 299 additions and 211 deletions.
diff --git a/ic-os/bootloader/BUILD.bazel b/ic-os/bootloader/BUILD.bazel
@@ -22,7 +22,7 @@ genrule(
 )
 
 vfat_image(
-    name = "partition-esp.tar",
+    name = "partition-esp.tzst",
     src = ":bootloader-tree.tar",
     partition_size = "100M",
     subdir = "boot/efi",

diff --git a/ic-os/defs.bzl b/ic-os/defs.bzl
@@ -65,7 +65,7 @@ def icos_build(
 
     # -------------------- Build grub partition --------------------
 
-    build_grub_partition("partition-grub.tar", grub_config = image_deps.get("grub_config", default = None), tags = ["manual"])
+    build_grub_partition("partition-grub.tzst", grub_config = image_deps.get("grub_config", default = None), tags = ["manual"])
 
     # -------------------- Build the container image --------------------
 
@@ -118,7 +118,7 @@ def icos_build(
     # -------------------- Extract root partition --------------------
 
     ext4_image(
-        name = "static-partition-root-unsigned.tar",
+        name = "static-partition-root-unsigned.tzst",
         src = ":rootfs-tree.tar",
         file_contexts = ":file_contexts",
         partition_size = image_deps["rootfs_size"],
@@ -136,7 +136,7 @@ def icos_build(
     # -------------------- Extract boot partition --------------------
 
     ext4_image(
-        name = "static-partition-boot.tar",
+        name = "static-partition-boot.tzst",
         src = ":rootfs-tree.tar",
         file_contexts = ":file_contexts",
         partition_size = image_deps["bootfs_size"],
@@ -151,8 +151,8 @@ def icos_build(
     # -------------------- Inject extra files --------------------
 
     inject_files(
-        name = "partition-root-unsigned.tar",
-        base = "static-partition-root-unsigned.tar",
+        name = "partition-root-unsigned.tzst",
+        base = "static-partition-root-unsigned.tzst",
         file_contexts = ":file_contexts",
         extra_files = {
             k: v
@@ -163,8 +163,8 @@ def icos_build(
 
     if upgrades:
         inject_files(
-            name = "partition-root-test-unsigned.tar",
-            base = "static-partition-root-unsigned.tar",
+            name = "partition-root-test-unsigned.tzst",
+            base = "static-partition-root-unsigned.tzst",
             file_contexts = ":file_contexts",
             extra_files = {
                 k: v
@@ -175,22 +175,22 @@ def icos_build(
 
     # When boot_args are fixed, don't bother signing
     if "boot_args_template" not in image_deps:
-        native.alias(name = "partition-root.tar", actual = ":partition-root-unsigned.tar", tags = ["manual"])
+        native.alias(name = "partition-root.tzst", actual = ":partition-root-unsigned.tzst", tags = ["manual"])
         native.alias(name = "extra_boot_args", actual = image_deps["extra_boot_args"], tags = ["manual"])
 
         if upgrades:
-            native.alias(name = "partition-root-test.tar", actual = ":partition-root-test-unsigned.tar", tags = ["manual"])
+            native.alias(name = "partition-root-test.tzst", actual = ":partition-root-test-unsigned.tzst", tags = ["manual"])
             native.alias(name = "extra_boot_test_args", actual = image_deps["extra_boot_args"], tags = ["manual"])
     else:
         native.alias(name = "extra_boot_args_template", actual = image_deps["boot_args_template"], tags = ["manual"])
 
         native.genrule(
             name = "partition-root-sign",
-            srcs = ["partition-root-unsigned.tar"],
-            outs = ["partition-root.tar", "partition-root-hash"],
-            cmd = "$(location //toolchains/sysimage:verity_sign.py) -i $< -o $(location :partition-root.tar) -r $(location partition-root-hash)",
+            srcs = ["partition-root-unsigned.tzst"],
+            outs = ["partition-root.tzst", "partition-root-hash"],
+            cmd = "$(location //toolchains/sysimage:verity_sign.py) -i $< -o $(location :partition-root.tzst) -r $(location partition-root-hash) -d $(location //rs/ic_os/dflate)",
             executable = False,
-            tools = ["//toolchains/sysimage:verity_sign.py"],
+            tools = ["//toolchains/sysimage:verity_sign.py", "//rs/ic_os/dflate"],
             tags = ["manual"],
         )
 
@@ -208,10 +208,10 @@ def icos_build(
         if upgrades:
             native.genrule(
                 name = "partition-root-test-sign",
-                srcs = ["partition-root-test-unsigned.tar"],
-                outs = ["partition-root-test.tar", "partition-root-test-hash"],
-                cmd = "$(location //toolchains/sysimage:verity_sign.py) -i $< -o $(location :partition-root-test.tar) -r $(location partition-root-test-hash)",
-                tools = ["//toolchains/sysimage:verity_sign.py"],
+                srcs = ["partition-root-test-unsigned.tzst"],
+                outs = ["partition-root-test.tzst", "partition-root-test-hash"],
+                cmd = "$(location //toolchains/sysimage:verity_sign.py) -i $< -o $(location :partition-root-test.tzst) -r $(location partition-root-test-hash) -d $(location //rs/ic_os/dflate)",
+                tools = ["//toolchains/sysimage:verity_sign.py", "//rs/ic_os/dflate"],
                 tags = ["manual"],
             )
 
@@ -227,8 +227,8 @@ def icos_build(
             )
 
     inject_files(
-        name = "partition-boot.tar",
-        base = "static-partition-boot.tar",
+        name = "partition-boot.tzst",
+        base = "static-partition-boot.tzst",
         file_contexts = ":file_contexts",
         prefix = "/boot",
         extra_files = {
@@ -245,8 +245,8 @@ def icos_build(
 
     if upgrades:
         inject_files(
-            name = "partition-boot-test.tar",
-            base = "static-partition-boot.tar",
+            name = "partition-boot-test.tzst",
+            base = "static-partition-boot.tzst",
             file_contexts = ":file_contexts",
             prefix = "/boot",
             extra_files = {
@@ -273,10 +273,10 @@ def icos_build(
         name = "disk-img.tar",
         layout = image_deps["partition_table"],
         partitions = [
-            "//ic-os/bootloader:partition-esp.tar",
-            ":partition-grub.tar",
-            ":partition-boot.tar",
-            ":partition-root.tar",
+            "//ic-os/bootloader:partition-esp.tzst",
+            ":partition-grub.tzst",
+            ":partition-boot.tzst",
+            ":partition-root.tzst",
         ] + custom_partitions,
         expanded_size = image_deps.get("expanded_size", default = None),
         tags = ["manual"],
@@ -325,8 +325,8 @@ def icos_build(
     if upgrades:
         upgrade_image(
             name = "update-img.tar",
-            boot_partition = ":partition-boot.tar",
-            root_partition = ":partition-root.tar",
+            boot_partition = ":partition-boot.tzst",
+            root_partition = ":partition-root.tzst",
             tags = ["manual"],
             target_compatible_with = [
                 "@platforms//os:linux",
@@ -371,8 +371,8 @@ def icos_build(
 
         upgrade_image(
             name = "update-img-test.tar",
-            boot_partition = ":partition-boot-test.tar",
-            root_partition = ":partition-root-test.tar",
+            boot_partition = ":partition-boot-test.tzst",
+            root_partition = ":partition-root-test.tzst",
             tags = ["manual"],
             target_compatible_with = [
                 "@platforms//os:linux",
@@ -629,7 +629,7 @@ def boundary_node_icos_build(
         tags = ["manual"],
     )
 
-    build_grub_partition("partition-grub.tar", tags = ["manual"])
+    build_grub_partition("partition-grub.tzst", tags = ["manual"])
 
     build_container_filesystem_config_file = Label(image_deps["build_container_filesystem_config_file"])
 
@@ -642,7 +642,7 @@ def boundary_node_icos_build(
     )
 
     ext4_image(
-        name = "partition-config.tar",
+        name = "partition-config.tzst",
         partition_size = "100M",
         target_compatible_with = [
             "@platforms//os:linux",
@@ -659,7 +659,7 @@ def boundary_node_icos_build(
     )
 
     ext4_image(
-        name = "static-partition-boot.tar",
+        name = "static-partition-boot.tzst",
         src = ":rootfs-tree.tar",
         partition_size = "1G",
         subdir = "boot/",
@@ -670,8 +670,8 @@ def boundary_node_icos_build(
     )
 
     inject_files(
-        name = "partition-boot.tar",
-        base = "static-partition-boot.tar",
+        name = "partition-boot.tzst",
+        base = "static-partition-boot.tzst",
         prefix = "/boot",
         extra_files = {
             k: v
@@ -686,7 +686,7 @@ def boundary_node_icos_build(
     )
 
     ext4_image(
-        name = "static-partition-root-unsigned.tar",
+        name = "static-partition-root-unsigned.tzst",
         src = ":rootfs-tree.tar",
         partition_size = "3G",
         strip_paths = [
@@ -700,8 +700,8 @@ def boundary_node_icos_build(
     )
 
     inject_files(
-        name = "partition-root-unsigned.tar",
-        base = "static-partition-root-unsigned.tar",
+        name = "partition-root-unsigned.tzst",
+        base = "static-partition-root-unsigned.tzst",
         extra_files = {
             k: v
             for k, v in (image_deps["rootfs"].items() + [(":version.txt", "/opt/ic/share/version.txt:0644")])
@@ -711,11 +711,11 @@ def boundary_node_icos_build(
 
     native.genrule(
         name = "partition-root-sign",
-        srcs = ["partition-root-unsigned.tar"],
-        outs = ["partition-root.tar", "partition-root-hash"],
-        cmd = "$(location //toolchains/sysimage:verity_sign.py) -i $< -o $(location :partition-root.tar) -r $(location partition-root-hash)",
+        srcs = ["partition-root-unsigned.tzst"],
+        outs = ["partition-root.tzst", "partition-root-hash"],
+        cmd = "$(location //toolchains/sysimage:verity_sign.py) -i $< -o $(location :partition-root.tzst) -r $(location partition-root-hash) -d $(location //rs/ic_os/dflate)",
         executable = False,
-        tools = ["//toolchains/sysimage:verity_sign.py"],
+        tools = ["//toolchains/sysimage:verity_sign.py", "//rs/ic_os/dflate"],
         tags = ["manual"],
     )
 
@@ -734,11 +734,11 @@ def boundary_node_icos_build(
         name = "disk-img.tar",
         layout = "//ic-os/boundary-guestos:partitions.csv",
         partitions = [
-            "//ic-os/bootloader:partition-esp.tar",
-            ":partition-grub.tar",
-            ":partition-config.tar",
-            ":partition-boot.tar",
-            ":partition-root.tar",
+            "//ic-os/bootloader:partition-esp.tzst",
+            ":partition-grub.tzst",
+            ":partition-config.tzst",
+            ":partition-boot.tzst",
+            ":partition-root.tzst",
         ],
         expanded_size = "50G",
         tags = ["manual"],

diff --git a/ic-os/guestos/BUILD.bazel b/ic-os/guestos/BUILD.bazel
@@ -7,7 +7,7 @@ exports_files([
 ])
 
 ext4_image(
-    name = "partition-config.tar",
+    name = "partition-config.tzst",
     partition_size = "100M",
     tags = ["manual"],
     target_compatible_with = [

diff --git a/ic-os/guestos/defs.bzl b/ic-os/guestos/defs.bzl
@@ -57,7 +57,7 @@ def image_deps(mode, malicious = False):
         "bootfs_size": "1G",
 
         # Add any custom partitions to the manifest
-        "custom_partitions": lambda: [Label("//ic-os/guestos:partition-config.tar")],
+        "custom_partitions": lambda: [Label("//ic-os/guestos:partition-config.tzst")],
 
         # We will install extra_boot_args onto the system, after substituting the
         # hash of the root filesystem into it. Track the template (before

diff --git a/ic-os/hostos/BUILD.bazel b/ic-os/hostos/BUILD.bazel
@@ -9,7 +9,7 @@ exports_files([
 ])
 
 ext4_image(
-    name = "partition-config.tar",
+    name = "partition-config.tzst",
     partition_size = "100M",
     tags = ["manual"],
     target_compatible_with = [

diff --git a/ic-os/hostos/defs.bzl b/ic-os/hostos/defs.bzl
@@ -74,12 +74,12 @@ def image_deps(mode, _malicious = False):
 # earlier in the pipeline, and is depended on by the final disk image.
 def _custom_partitions():
     lvm_image(
-        name = "partition-hostlvm.tar",
+        name = "partition-hostlvm.tzst",
         layout = Label("//ic-os/hostos:volumes.csv"),
         partitions = [
-            Label("//ic-os/hostos:partition-config.tar"),
-            ":partition-boot.tar",
-            ":partition-root.tar",
+            Label("//ic-os/hostos:partition-config.tzst"),
+            ":partition-boot.tzst",
+            ":partition-root.tzst",
         ],
         vg_name = "hostlvm",
         vg_uuid = "4c7GVZ-Df82-QEcJ-xXtV-JgRL-IjLE-hK0FgA",
@@ -92,4 +92,4 @@ def _custom_partitions():
         ],
     )
 
-    return [":partition-hostlvm.tar"]
+    return [":partition-hostlvm.tzst"]
diff --git a/ic-os/setupos/defs.bzl b/ic-os/setupos/defs.bzl
@@ -118,7 +118,7 @@ def _custom_partitions(mode):
     )
 
     fat32_image(
-        name = "partition-config.tar",
+        name = "partition-config.tzst",
         src = "config_tar",
         label = "CONFIG",
         partition_size = "50M",
@@ -151,7 +151,7 @@ def _custom_partitions(mode):
     )
 
     ext4_image(
-        name = "partition-data.tar",
+        name = "partition-data.tzst",
         src = "data_tar",
         partition_size = "1750M",
         subdir = "data",
@@ -162,6 +162,6 @@ def _custom_partitions(mode):
     )
 
     return [
-        ":partition-config.tar",
-        ":partition-data.tar",
+        ":partition-config.tzst",
+        ":partition-data.tzst",
     ]
diff --git a/rs/ic_os/dflate/src/main.rs b/rs/ic_os/dflate/src/main.rs
@@ -31,8 +31,6 @@ fn main() -> Result<()> {
             .to_string_lossy()
             .into_owned();
 
-        println!("Adding {file_name}");
-
         let state = scan_file_for_holes(&mut source, file_name)?;
         add_file_to_archive(&mut source, &mut dest, state)?;
     }

diff --git a/rs/ic_os/inject_files/src/main.rs b/rs/ic_os/inject_files/src/main.rs
@@ -22,6 +22,8 @@ struct Cli {
     file_contexts: Option<PathBuf>,
     #[arg(long)]
     prefix: Option<PathBuf>,
+    #[arg(short)]
+    dflate: PathBuf,
     extra_files: Vec<String>,
 }
 
@@ -42,7 +44,7 @@ async fn main() -> Result<()> {
         .status()
         .await;
 
-    let mut target = ExtPartition::open(temp_file, cli.index).await?;
+    let mut target = ExtPartition::open(temp_file.clone(), cli.index).await?;
 
     let contexts = cli
         .file_contexts
@@ -87,19 +89,25 @@ async fn main() -> Result<()> {
     target.close().await?;
 
     // TODO: Quick hack to unpack and repack file
-    let mut cmd = Command::new("tar");
+    // If dflate is ever misbehaving, it can be replaced with:
+    // tar cf <output> --sort=name --owner=root:0 --group=root:0 --mtime="UTC 1970-01-01 00:00:00" --sparse --hole-detection=raw -C <context_path> <item>
+    let temp_tar = temp_dir.path().join("partition.tar");
+    let mut cmd = Command::new(cli.dflate);
     let _ = cmd
-        .arg("cf")
+        .arg("--input")
+        .arg(&temp_file)
+        .arg("--output")
+        .arg(&temp_tar)
+        .status()
+        .await;
+
+    let mut cmd = Command::new("zstd");
+    let _ = cmd
+        .arg("-q")
+        .arg("--threads=0")
+        .arg(&temp_tar)
+        .arg("-o")
         .arg(cli.output)
-        .arg("--sort=name")
-        .arg("--owner=root:0")
-        .arg("--group=root:0")
-        .arg("--mtime=UTC 1970-01-01 00:00:00")
-        .arg("--sparse")
-        .arg("--hole-detection=raw")
-        .arg("-C")
-        .arg(temp_dir.path())
-        .arg("partition.img")
         .status()
         .await;