Merge pull request #1826 from bcressey/custom-image-size

add support for custom image sizes

Dockerfile

@@ -164,8 +164,12 @@ ARG VARIANT
 ARG PRETTY_NAME
 ARG IMAGE_NAME
 ARG IMAGE_FORMAT
+ARG OS_IMAGE_SIZE_GIB
+ARG DATA_IMAGE_SIZE_GIB
 ARG KERNEL_PARAMETERS
-ENV VARIANT=${VARIANT} VERSION_ID=${VERSION_ID} BUILD_ID=${BUILD_ID} PRETTY_NAME=${PRETTY_NAME} IMAGE_NAME=${IMAGE_NAME} IMAGE_FORMAT=${IMAGE_FORMAT} KERNEL_PARAMETERS=${KERNEL_PARAMETERS}
+ENV VARIANT=${VARIANT} VERSION_ID=${VERSION_ID} BUILD_ID=${BUILD_ID} \
+    PRETTY_NAME=${PRETTY_NAME} IMAGE_NAME=${IMAGE_NAME} \
+    KERNEL_PARAMETERS=${KERNEL_PARAMETERS}
 WORKDIR /root
 
 USER root
@@ -174,6 +178,8 @@ RUN --mount=target=/host \
       --package-dir=/local/rpms \
       --output-dir=/local/output \
       --output-fmt=${IMAGE_FORMAT} \
+      --os-image-size-gib=${OS_IMAGE_SIZE_GIB} \
+      --data-image-size-gib=${DATA_IMAGE_SIZE_GIB} \
     && echo ${NOCACHE}
 
 # =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^=

tools/buildsys/src/builder.rs

@@ -21,7 +21,7 @@ use std::path::{Path, PathBuf};
 use std::process::Output;
 use walkdir::{DirEntry, WalkDir};
 
-use crate::manifest::{ImageFormat, SupportedArch};
+use crate::manifest::{ImageFormat, ImageLayout, SupportedArch};
 
 /*
 There's a bug in BuildKit that can lead to a build failure during parallel
@@ -116,6 +116,7 @@ impl VariantBuilder {
     pub(crate) fn build(
         packages: &[String],
         image_format: Option<&ImageFormat>,
+        image_layout: Option<&ImageLayout>,
         kernel_parameters: Option<&Vec<String>>,
     ) -> Result<Self> {
         let output_dir: PathBuf = getenv("BUILDSYS_OUTPUT_DIR")?.into();
@@ -126,6 +127,12 @@ impl VariantBuilder {
             .context(error::UnsupportedArch { arch: &arch })?
             .goarch();
 
+        let image_layout = image_layout.cloned().unwrap_or_default();
+        let ImageLayout {
+            os_image_size_gib,
+            data_image_size_gib,
+        } = image_layout;
+
         let mut args = Vec::new();
         args.build_arg("PACKAGES", packages.join(" "));
         args.build_arg("ARCH", &arch);
@@ -143,6 +150,8 @@ impl VariantBuilder {
                 Some(ImageFormat::Vmdk) => "vmdk",
             },
         );
+        args.build_arg("OS_IMAGE_SIZE_GIB", format!("{}", os_image_size_gib));
+        args.build_arg("DATA_IMAGE_SIZE_GIB", format!("{}", data_image_size_gib));
         args.build_arg(
             "KERNEL_PARAMETERS",
             kernel_parameters

tools/buildsys/src/main.rs

@@ -192,8 +192,9 @@ fn build_variant() -> Result<()> {
 
     if let Some(packages) = manifest.included_packages() {
         let image_format = manifest.image_format();
+        let image_layout = manifest.image_layout();
         let kernel_parameters = manifest.kernel_parameters();
-        VariantBuilder::build(&packages, image_format, kernel_parameters)
+        VariantBuilder::build(&packages, image_format, image_layout, kernel_parameters)
             .context(error::BuildAttempt)?;
     } else {
         println!("cargo:warning=No included packages in manifest. Skipping variant build.");

tools/buildsys/src/manifest.rs

@@ -66,13 +66,32 @@ releases-url = "https://www.example.com/releases"
 included-packages = ["release"]
 ```
 
-`image-format` is the desired format of the built image.
+`image-format` is the desired format for the built images.
 This can be `raw` (the default), `vmdk`, or `qcow2`.
 ```
 [package.metadata.build-variant]
 image-format = "vmdk"
 ```
 
+`image-layout` is the desired layout for the built images.
+
+`os-image-size-gib` is the desired size of the "os" disk image in GiB.
+The specified size will be automatically divided into two banks, where each
+bank contains the set of partitions needed for in-place upgrades. Roughly 40%
+will be available for each root filesystem partition, with the rest allocated
+to other essential system partitions.
+
+`data-image-size-gib` is the desired size of the "data" disk image in GiB.
+The full size will be used for the single data partition, except for the 2 MiB
+overhead for the GPT labels and partition alignment. The data partition will be
+automatically resized to fill the disk on boot, so it is usually not necessary
+to increase this value.
+```
+[package.metadata.build-variant.image-layout]
+os-image-size-gib = 2
+data-image-size-gib = 1
+```
+
 `supported-arches` is the list of architectures the variant is able to run on.
 The values can be `x86_64` and `aarch64`.
 If not specified, the variant can run on any of those architectures.
@@ -101,6 +120,9 @@ use std::fmt;
 use std::fs;
 use std::path::{Path, PathBuf};
 
+static DEFAULT_OS_IMAGE_SIZE_GIB: u32 = 2;
+static DEFAULT_DATA_IMAGE_SIZE_GIB: u32 = 1;
+
 /// The nested structures here are somewhat complex, but they make it trivial
 /// to deserialize the structure we expect to find in the manifest.
 #[derive(Deserialize, Debug)]
@@ -148,6 +170,11 @@ impl ManifestInfo {
         self.build_variant().and_then(|b| b.image_format.as_ref())
     }
 
+    /// Convenience method to return the image layout, if specified.
+    pub(crate) fn image_layout(&self) -> Option<&ImageLayout> {
+        self.build_variant().and_then(|b| b.image_layout.as_ref())
+    }
+
     /// Convenience method to return the supported architectures for this variant.
     pub(crate) fn supported_arches(&self) -> Option<&HashSet<SupportedArch>> {
         self.build_variant()
@@ -204,6 +231,7 @@ pub(crate) struct BuildPackage {
 pub(crate) struct BuildVariant {
     pub(crate) included_packages: Option<Vec<String>>,
     pub(crate) image_format: Option<ImageFormat>,
+    pub(crate) image_layout: Option<ImageLayout>,
     pub(crate) supported_arches: Option<HashSet<SupportedArch>>,
     pub(crate) kernel_parameters: Option<Vec<String>>,
 }
@@ -216,6 +244,34 @@ pub(crate) enum ImageFormat {
     Vmdk,
 }
 
+#[derive(Deserialize, Debug, Copy, Clone)]
+#[serde(rename_all = "kebab-case")]
+pub(crate) struct ImageLayout {
+    #[serde(default = "ImageLayout::default_os_image_size_gib")]
+    pub(crate) os_image_size_gib: u32,
+    #[serde(default = "ImageLayout::default_data_image_size_gib")]
+    pub(crate) data_image_size_gib: u32,
+}
+
+impl ImageLayout {
+    fn default_os_image_size_gib() -> u32 {
+        DEFAULT_OS_IMAGE_SIZE_GIB
+    }
+
+    fn default_data_image_size_gib() -> u32 {
+        DEFAULT_DATA_IMAGE_SIZE_GIB
+    }
+}
+
+impl Default for ImageLayout {
+    fn default() -> Self {
+        Self {
+            os_image_size_gib: Self::default_os_image_size_gib(),
+            data_image_size_gib: Self::default_data_image_size_gib(),
+        }
+    }
+}
+
 #[derive(Deserialize, Debug, PartialEq, Eq, Hash)]
 #[serde(rename_all = "lowercase")]
 pub(crate) enum SupportedArch {

tools/partyplanner

@@ -0,0 +1,211 @@
+#!/bin/bash
+
+###############################################################################
+# Section 1: partition type GUIDs
+
+# Define partition type GUIDs for all OS-managed partitions. This is required
+# for the boot partition, where we set gptprio bits in the GUID-specific use
+# field, but we might as well do it for all of them.
+BOTTLEROCKET_BOOT_TYPECODE="6b636168-7420-6568-2070-6c616e657421"
+BOTTLEROCKET_ROOT_TYPECODE="5526016a-1a97-4ea4-b39a-b7c8c6ca4502"
+BOTTLEROCKET_HASH_TYPECODE="598f10af-c955-4456-6a99-7720068a6cea"
+BOTTLEROCKET_RESERVED_TYPECODE="0c5d99a5-d331-4147-baef-08e2b855bdc9"
+BOTTLEROCKET_PRIVATE_TYPECODE="440408bb-eb0b-4328-a6e5-a29038fad706"
+BOTTLEROCKET_DATA_TYPECODE="626f7474-6c65-6474-6861-726d61726b73"
+
+# Under BIOS, the firmware will transfer control to the MBR on the boot device,
+# which will pass control to the GRUB stage 2 binary written to the BIOS boot
+# partition. The BIOS does not attach any significance to this partition type,
+# but GRUB knows to install itself there when we run `grub-bios-setup`.
+BIOS_BOOT_TYPECODE="ef02"
+
+# Under EFI, the firmware will find the EFI system partition and execute the
+# program at a platform-defined path like `bootx64.efi`. The partition type
+# must match what the firmware expects.
+EFI_SYSTEM_TYPECODE="C12A7328-F81F-11D2-BA4B-00A0C93EC93B"
+
+# Whichever entry point is used for booting the system, it's important to note
+# that only one build of GRUB is involved - the one that's installed during the
+# image build.
+
+# GRUB understands the GPT priorities scheme we use to find the active boot
+# partition; EFI and BIOS firmware does not. This is why we do not update GRUB
+# during our system updates; we would have no way to revert to an earlier copy
+# of the bootloader if it failed to boot.
+#
+# We may eventually want to have an active/passive scheme for EFI partitions,
+# to allow for potential GRUB and shim updates on EFI platforms in cases where
+# we need to deliver security fixes. For now, add a placeholder partition type
+# for an alternate bank.
+EFI_BACKUP_TYPECODE="B39CE39C-0A00-B4AB-2D11-F18F8237A21C"
+
+###############################################################################
+# Section 2: fixed size partitions and reservations
+
+# The GPT header and footer each take up 32 sectors, but we reserve a full MiB
+# so that partitions can all be aligned on MiB boundaries.
+GPT_MIB="1" # two per disk
+
+# The BIOS partition is only used on x86 platforms, and only needs to be large
+# enough for the GRUB stage 2. Increasing its size will reduce the size of the
+# "private" and "reserved" partitions. This should be relatively safe since we
+# don't apply image updates to those partitions.
+BIOS_MIB="4" # one per disk
+
+# The GPT and BIOS reservations are fixed overhead that will be deducted from
+# the space nominally given to the private partition used to persist settings.
+OVERHEAD_MIB="$((GPT_MIB * 2 + BIOS_MIB))"
+
+# The 'recommended' size for the EFI partition is 100MB but our EFI images are
+# under 1MB, so this will suffice for now. It would be possible to increase the
+# EFI partition size by taking space from the "reserved" area below.
+EFI_MIB="5" # one per bank
+
+###############################################################################
+# Section 3: variable sized partitions
+
+# These partitions scale based on image size. The scaling factors are chosen so
+# that we end up with the same partition sizes for the banks on a 2 GiB image,
+# which was the only image size we historically supported.
+#
+# !!! WARNING !!!
+#
+# Increasing any of these constants is very likely break systems on update,
+# since the corresponding partitions are adjacent on disk and have no room to
+# grow.
+BOOT_SCALE_FACTOR="20"
+ROOT_SCALE_FACTOR="460"
+HASH_SCALE_FACTOR="5"
+RESERVE_SCALE_FACTOR="15"
+PRIVATE_SCALE_FACTOR="24"
+
+###############################################################################
+# Section 4: ASCII art gallery
+
+# Layout for a 1 GiB OS image. Sizes marked with (*) scale with overall image
+# size, based on the constant factors above.
+
+#          +---------------------------------+
+#  Prelude | GPT header               1 MiB  | 5 MiB
+#          | BIOS boot partition      4 MiB  | Fixed size.
+#          +---------------------------------+
+#          | EFI system partition     5 MiB  |
+#          | Boot partition A        20 MiB* | (image size - prelude - postlude) / 2
+#   Bank A | Root partition A       460 MiB* | Example: (1 GiB - 5 MiB - 19 MiB) / 2
+#          | Hash partition A         5 MiB* |          500 MiB
+#          | Reserved partition A    10 MiB* |
+#          +---------------------------------+
+#          | EFI backup partition     5 MiB  |
+#          | Boot partition B        20 MiB* | (image size - prelude - postlude) / 2
+#   Bank B | Root partition B       460 MiB* | Example: (1 GiB - 5 MiB - 19 MiB) / 2
+#          | Hash partition B         5 MiB* |          500 MiB
+#          | Reserved partition B    10 MiB* |
+#          +---------------------------------+
+#          | Private partition       18 MiB* | (image size * 24 as MiB) - prelude
+# Postlude | GPT footer               1 MiB  | GPT is fixed, private partition grows.
+#          +---------------------------------+
+
+##############################################################################
+# Section 5: library functions
+
+# Populate the caller's tables with sizes and offsets for known partitions.
+set_partition_sizes() {
+  local os_image_gib data_image_gib
+  local -n pp_size pp_offset
+  os_image_gib="${1:?}"
+  data_image_gib="${2:?}"
+
+  # Table for partition sizes, in MiB.
+  pp_size="${3:?}"
+
+  # Table for partition offsets from start of disk, in MiB.
+  pp_offset="${4:?}"
+
+  # Most of the partitions on the main image scale with the overall size.
+  local boot_mib root_mib hash_mib reserved_mib private_mib
+  boot_mib="$((os_image_gib * BOOT_SCALE_FACTOR))"
+  root_mib="$((os_image_gib * ROOT_SCALE_FACTOR))"
+  hash_mib="$((os_image_gib * HASH_SCALE_FACTOR))"
+
+  # Reserved space is everything left in the bank after the other partitions
+  # are scaled, minus the fixed 5 MiB EFI partition in that bank.
+  reserved_mib=$((os_image_gib * RESERVE_SCALE_FACTOR - EFI_MIB))
+
+  # Private space scales per GiB, minus the BIOS and GPT partition overhead.
+  private_mib=$((os_image_gib * PRIVATE_SCALE_FACTOR - OVERHEAD_MIB))
+
+  # Skip the GPT label at start of disk.
+  local offset
+  ((offset = 1))
+
+  pp_offset["BIOS"]="${offset}"
+  pp_size["BIOS"]="${BIOS_MIB}"
+  ((offset += BIOS_MIB))
+
+  for bank in A B ; do
+    pp_offset["EFI-${bank}"]="${offset}"
+    pp_size["EFI-${bank}"]="${EFI_MIB}"
+    ((offset += EFI_MIB))
+
+    pp_offset["BOOT-${bank}"]="${offset}"
+    pp_size["BOOT-${bank}"]="${boot_mib}"
+    ((offset += boot_mib))
+
+    pp_offset["ROOT-${bank}"]="${offset}"
+    pp_size["ROOT-${bank}"]="${root_mib}"
+    ((offset += root_mib))
+
+    pp_offset["HASH-${bank}"]="${offset}"
+    pp_size["HASH-${bank}"]="${hash_mib}"
+    ((offset += hash_mib))
+
+    pp_offset["RESERVED-${bank}"]="${offset}"
+    pp_size["RESERVED-${bank}"]="${reserved_mib}"
+    ((offset += reserved_mib))
+  done
+
+  pp_offset["PRIVATE"]="${offset}"
+  pp_size["PRIVATE"]="${private_mib}"
+  ((offset += private_mib))
+
+  # The data image is relatively easy to plan, at least until we add support
+  # for unified images. The first and last MiB are reserved for the GPT labels,
+  # and the remainder is for the lone "data" partition.
+  pp_size["DATA"]="$((data_image_gib * 1024 - GPT_MIB * 2))"
+  pp_offset["DATA"]="1"
+}
+
+# Populate the caller's table with labels for known partitions.
+set_partition_labels() {
+  local -n pp_label
+  pp_label="${1:?}"
+  pp_label["BIOS"]="BIOS-BOOT"
+  pp_label["DATA"]="BOTTLEROCKET-DATA"
+  pp_label["EFI-A"]="EFI-SYSTEM"
+  pp_label["EFI-B"]="EFI-BACKUP"
+  pp_label["PRIVATE"]="BOTTLEROCKET-PRIVATE"
+  for part in BOOT ROOT HASH RESERVED ; do
+    for bank in A B ; do
+      pp_label["${part}-${bank}"]="BOTTLEROCKET-${part}-${bank}"
+    done
+  done
+}
+
+# Populate the caller's table with GPT type codes for known partitions.
+set_partition_types() {
+  local -n pp_type
+  pp_type="${1:?}"
+  pp_type["BIOS"]="${BIOS_BOOT_TYPECODE}"
+  pp_type["DATA"]="${BOTTLEROCKET_DATA_TYPECODE}"
+  pp_type["EFI-A"]="${EFI_SYSTEM_TYPECODE}"
+  pp_type["EFI-B"]="${EFI_BACKUP_TYPECODE}"
+  pp_type["PRIVATE"]="${BOTTLEROCKET_PRIVATE_TYPECODE}"
+  local typecode
+  for part in BOOT ROOT HASH RESERVED ; do
+    for bank in A B ; do
+        typecode="BOTTLEROCKET_${part}_TYPECODE"
+        typecode="${!typecode}"
+        pp_type["${part}-${bank}"]="${typecode}"
+    done
+  done
+}