From 99dbc935f8965935431e93eae32316fef6d5b150 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 20 Nov 2025 13:06:31 -0500 Subject: [PATCH 1/3] fix: try to resize tmp volume and copy nix store when done plus test size chore: fix conflict --- amazon-arm64-nix.pkr.hcl | 2 +- scripts/copy-nix-store.sh | 106 ++++++++++++++++++++++++++++++++++ scripts/mount-build-volume.sh | 34 +++++++++++ stage2-nix-psql.pkr.hcl | 18 ++++++ 4 files changed, 159 insertions(+), 1 deletion(-) create mode 100755 scripts/copy-nix-store.sh create mode 100755 scripts/mount-build-volume.sh diff --git a/amazon-arm64-nix.pkr.hcl b/amazon-arm64-nix.pkr.hcl index 789a48538..a09914b59 100644 --- a/amazon-arm64-nix.pkr.hcl +++ b/amazon-arm64-nix.pkr.hcl @@ -149,7 +149,7 @@ source "amazon-ebssurrogate" "source" { launch_block_device_mappings { device_name = "/dev/${var.build-vol}" delete_on_termination = true - volume_size = 16 + volume_size = 40 volume_type = "gp2" omit_from_artifact = true } diff --git a/scripts/copy-nix-store.sh b/scripts/copy-nix-store.sh new file mode 100755 index 000000000..3a7ec4a44 --- /dev/null +++ b/scripts/copy-nix-store.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +# Copy nix store from temporary build volume back to root filesystem +# Includes size check and fails build if it won't fit + +set -o errexit +set -o pipefail +set -o xtrace + +echo "=== Migrating Nix store from temp volume to root filesystem ===" + +# Stop nix daemon before unmounting +echo "Stopping nix-daemon..." +sudo systemctl stop nix-daemon.service || true +sudo systemctl stop nix-daemon.socket || true + +# Get size of nix store on temp volume (in bytes) +echo "" +echo "Checking nix store size..." +NIX_STORE_SIZE=$(sudo du -sb /mnt/nix-temp | cut -f1) +NIX_STORE_SIZE_GB=$(echo "scale=2; $NIX_STORE_SIZE / 1024 / 1024 / 1024" | bc) +echo "Nix store size: ${NIX_STORE_SIZE_GB} GB (${NIX_STORE_SIZE} bytes)" + +# Get available space on root filesystem (in bytes) +# Check the actual root filesystem, not the bind mount +ROOT_FS_DEVICE=$(df / | tail -1 | awk '{print $1}') +ROOT_AVAILABLE=$(df -B1 / | tail -1 | awk '{print $4}') +ROOT_AVAILABLE_GB=$(echo "scale=2; $ROOT_AVAILABLE / 1024 / 1024 / 1024" | bc) +echo "Root filesystem (${ROOT_FS_DEVICE}) available space: ${ROOT_AVAILABLE_GB} GB (${ROOT_AVAILABLE} bytes)" + +# Add 10% buffer for safety +REQUIRED_SPACE=$(echo "$NIX_STORE_SIZE * 1.1" | bc | cut -d. -f1) +REQUIRED_SPACE_GB=$(echo "scale=2; $REQUIRED_SPACE / 1024 / 1024 / 1024" | bc) +echo "Required space (with 10% buffer): ${REQUIRED_SPACE_GB} GB (${REQUIRED_SPACE} bytes)" + +# Check if there's enough space +echo "" +if [ "$ROOT_AVAILABLE" -lt "$REQUIRED_SPACE" ]; then + echo "======================================" + echo "ERROR: Not enough space on root filesystem!" + echo "======================================" + echo " Nix store size: ${NIX_STORE_SIZE_GB} GB" + echo " Required (+ buffer): ${REQUIRED_SPACE_GB} GB" + echo " Available on root: ${ROOT_AVAILABLE_GB} GB" + echo " Shortfall: $(echo "scale=2; ($REQUIRED_SPACE - $ROOT_AVAILABLE) / 1024 / 1024 / 1024" | bc) GB" + echo "" + echo "Build FAILED: Nix store is too large to fit on the root volume." + echo "Consider increasing the root volume size in amazon-arm64-nix.pkr.hcl" + exit 1 +fi + +echo "✓ Space check passed. Sufficient space available." +echo "" + +# Unmount the bind mount +echo "Unmounting bind mount /nix..." +sudo umount /nix + +echo "/nix now shows original (empty) directory on root filesystem" +ls -la /nix/ || true + +echo "" +echo "Copying nix store from temp volume to root filesystem..." +echo "This may take several minutes..." +sudo rsync -aHAXS --info=progress2 /mnt/nix-temp/ /nix/ + +# Verify the copy +COPIED_SIZE=$(sudo du -sb /nix | cut -f1) +COPIED_SIZE_GB=$(echo "scale=2; $COPIED_SIZE / 1024 / 1024 / 1024" | bc) +echo "" +echo "Copy verification:" +echo " Original size: ${NIX_STORE_SIZE_GB} GB (${NIX_STORE_SIZE} bytes)" +echo " Copied size: ${COPIED_SIZE_GB} GB (${COPIED_SIZE} bytes)" + +# Allow small differences due to filesystem overhead +SIZE_DIFF=$(echo "$NIX_STORE_SIZE - $COPIED_SIZE" | bc | tr -d '-') +SIZE_DIFF_PERCENT=$(echo "scale=2; $SIZE_DIFF * 100 / $NIX_STORE_SIZE" | bc | tr -d '-') + +if (( $(echo "$SIZE_DIFF_PERCENT > 1" | bc -l) )); then + echo "ERROR: Significant size mismatch after copy (${SIZE_DIFF_PERCENT}% difference)!" + exit 1 +fi + +echo "✓ Copy verified (size difference: ${SIZE_DIFF_PERCENT}%)" + +# Unmount temp volume (will be discarded by packer) +echo "" +echo "Unmounting temp volume /mnt/nix-temp..." +sudo umount /mnt/nix-temp +sudo rmdir /mnt/nix-temp + +echo "" +echo "=== Nix Store Migration Complete ===" +echo "Final location: /nix (on root filesystem)" +df -h /nix +echo "" +echo "Nix store contents:" +sudo du -sh /nix/store + +# Restart nix daemon +echo "" +echo "Restarting nix-daemon..." +sudo systemctl start nix-daemon.socket || true +sudo systemctl start nix-daemon.service || true + +echo "" +echo "✓ All done! Nix store successfully migrated to root filesystem." diff --git a/scripts/mount-build-volume.sh b/scripts/mount-build-volume.sh new file mode 100755 index 000000000..f373dbaf4 --- /dev/null +++ b/scripts/mount-build-volume.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Mount temporary build volume for stage 2 nix operations +# Uses bind mount so nix operations transparently use the larger temp volume + +set -o errexit +set -o pipefail +set -o xtrace + +echo "=== Setting up temporary build volume for Nix ===" + +echo "Formatting build volume /dev/xvdc..." +sudo mkfs.ext4 -F -O ^has_journal /dev/xvdc + +echo "Creating mount point /mnt/nix-temp..." +sudo mkdir -p /mnt/nix-temp + +echo "Mounting /dev/xvdc to /mnt/nix-temp..." +sudo mount /dev/xvdc /mnt/nix-temp + +echo "Setting permissions on /mnt/nix-temp..." +sudo chmod 755 /mnt/nix-temp + +echo "Creating /nix directory..." +sudo mkdir -p /nix + +echo "Bind mounting /mnt/nix-temp over /nix..." +sudo mount --bind /mnt/nix-temp /nix + +echo "" +echo "✓ Build volume setup complete" +echo " Temp volume: /mnt/nix-temp (40 GB)" +echo " Bind mount: /nix -> /mnt/nix-temp" +echo " Nix will be installed to /nix (transparently using temp volume)" +df -h /nix diff --git a/stage2-nix-psql.pkr.hcl b/stage2-nix-psql.pkr.hcl index 2f25b6ada..8db8a0954 100644 --- a/stage2-nix-psql.pkr.hcl +++ b/stage2-nix-psql.pkr.hcl @@ -86,6 +86,14 @@ source "amazon-ebs" "ubuntu" { delay_seconds = 15 max_attempts = 120 # 120 * 15s = 30 minutes max wait } + + launch_block_device_mappings { + device_name = "/dev/xvdc" + delete_on_termination = true + volume_size = 40 + volume_type = "gp2" + omit_from_artifact = true + } ena_support = true @@ -136,6 +144,11 @@ build { destination = "/tmp/ansible-playbook" } + # Mount temporary build volume before nix operations + provisioner "shell" { + script = "scripts/mount-build-volume.sh" + } + provisioner "shell" { environment_vars = [ "GIT_SHA=${var.git_sha}", @@ -144,4 +157,9 @@ build { script = "scripts/nix-provision.sh" } + # Copy nix store from temp volume to root filesystem after all operations complete + provisioner "shell" { + script = "scripts/copy-nix-store.sh" + } + } From c0dcab5133ff05ef4bedad439b3ad82ebaa29734 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 20 Nov 2025 14:23:37 -0500 Subject: [PATCH 2/3] fix: omit_from_artifact not supported here --- stage2-nix-psql.pkr.hcl | 1 - 1 file changed, 1 deletion(-) diff --git a/stage2-nix-psql.pkr.hcl b/stage2-nix-psql.pkr.hcl index 8db8a0954..f30869895 100644 --- a/stage2-nix-psql.pkr.hcl +++ b/stage2-nix-psql.pkr.hcl @@ -92,7 +92,6 @@ source "amazon-ebs" "ubuntu" { delete_on_termination = true volume_size = 40 volume_type = "gp2" - omit_from_artifact = true } ena_support = true From f2e4acc76a52847796064d105f6f0b0944f51ce6 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 20 Nov 2025 15:38:16 -0500 Subject: [PATCH 3/3] fix: replaced all bc commands with awk --- scripts/copy-nix-store.sh | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/scripts/copy-nix-store.sh b/scripts/copy-nix-store.sh index 3a7ec4a44..d95d674a7 100755 --- a/scripts/copy-nix-store.sh +++ b/scripts/copy-nix-store.sh @@ -17,19 +17,19 @@ sudo systemctl stop nix-daemon.socket || true echo "" echo "Checking nix store size..." NIX_STORE_SIZE=$(sudo du -sb /mnt/nix-temp | cut -f1) -NIX_STORE_SIZE_GB=$(echo "scale=2; $NIX_STORE_SIZE / 1024 / 1024 / 1024" | bc) +NIX_STORE_SIZE_GB=$(awk "BEGIN {printf \"%.2f\", $NIX_STORE_SIZE / 1024 / 1024 / 1024}") echo "Nix store size: ${NIX_STORE_SIZE_GB} GB (${NIX_STORE_SIZE} bytes)" # Get available space on root filesystem (in bytes) # Check the actual root filesystem, not the bind mount ROOT_FS_DEVICE=$(df / | tail -1 | awk '{print $1}') ROOT_AVAILABLE=$(df -B1 / | tail -1 | awk '{print $4}') -ROOT_AVAILABLE_GB=$(echo "scale=2; $ROOT_AVAILABLE / 1024 / 1024 / 1024" | bc) +ROOT_AVAILABLE_GB=$(awk "BEGIN {printf \"%.2f\", $ROOT_AVAILABLE / 1024 / 1024 / 1024}") echo "Root filesystem (${ROOT_FS_DEVICE}) available space: ${ROOT_AVAILABLE_GB} GB (${ROOT_AVAILABLE} bytes)" # Add 10% buffer for safety -REQUIRED_SPACE=$(echo "$NIX_STORE_SIZE * 1.1" | bc | cut -d. -f1) -REQUIRED_SPACE_GB=$(echo "scale=2; $REQUIRED_SPACE / 1024 / 1024 / 1024" | bc) +REQUIRED_SPACE=$(awk "BEGIN {printf \"%.0f\", $NIX_STORE_SIZE * 1.1}") +REQUIRED_SPACE_GB=$(awk "BEGIN {printf \"%.2f\", $REQUIRED_SPACE / 1024 / 1024 / 1024}") echo "Required space (with 10% buffer): ${REQUIRED_SPACE_GB} GB (${REQUIRED_SPACE} bytes)" # Check if there's enough space @@ -41,7 +41,8 @@ if [ "$ROOT_AVAILABLE" -lt "$REQUIRED_SPACE" ]; then echo " Nix store size: ${NIX_STORE_SIZE_GB} GB" echo " Required (+ buffer): ${REQUIRED_SPACE_GB} GB" echo " Available on root: ${ROOT_AVAILABLE_GB} GB" - echo " Shortfall: $(echo "scale=2; ($REQUIRED_SPACE - $ROOT_AVAILABLE) / 1024 / 1024 / 1024" | bc) GB" + SHORTFALL=$(awk "BEGIN {printf \"%.2f\", ($REQUIRED_SPACE - $ROOT_AVAILABLE) / 1024 / 1024 / 1024}") + echo " Shortfall: ${SHORTFALL} GB" echo "" echo "Build FAILED: Nix store is too large to fit on the root volume." echo "Consider increasing the root volume size in amazon-arm64-nix.pkr.hcl" @@ -65,17 +66,18 @@ sudo rsync -aHAXS --info=progress2 /mnt/nix-temp/ /nix/ # Verify the copy COPIED_SIZE=$(sudo du -sb /nix | cut -f1) -COPIED_SIZE_GB=$(echo "scale=2; $COPIED_SIZE / 1024 / 1024 / 1024" | bc) +COPIED_SIZE_GB=$(awk "BEGIN {printf \"%.2f\", $COPIED_SIZE / 1024 / 1024 / 1024}") echo "" echo "Copy verification:" echo " Original size: ${NIX_STORE_SIZE_GB} GB (${NIX_STORE_SIZE} bytes)" echo " Copied size: ${COPIED_SIZE_GB} GB (${COPIED_SIZE} bytes)" # Allow small differences due to filesystem overhead -SIZE_DIFF=$(echo "$NIX_STORE_SIZE - $COPIED_SIZE" | bc | tr -d '-') -SIZE_DIFF_PERCENT=$(echo "scale=2; $SIZE_DIFF * 100 / $NIX_STORE_SIZE" | bc | tr -d '-') +SIZE_DIFF=$(awk "BEGIN {x = $NIX_STORE_SIZE - $COPIED_SIZE; print (x < 0 ? -x : x)}") +SIZE_DIFF_PERCENT=$(awk "BEGIN {printf \"%.2f\", $SIZE_DIFF * 100 / $NIX_STORE_SIZE}") -if (( $(echo "$SIZE_DIFF_PERCENT > 1" | bc -l) )); then +# Check if difference is greater than 1% +if awk "BEGIN {exit !($SIZE_DIFF_PERCENT > 1)}"; then echo "ERROR: Significant size mismatch after copy (${SIZE_DIFF_PERCENT}% difference)!" exit 1 fi