fix: try to resize build-time volume and copy nix store when done + test size before copy

amazon-arm64-nix.pkr.hcl

@@ -149,7 +149,7 @@ source "amazon-ebssurrogate" "source" {
   launch_block_device_mappings {
     device_name           = "/dev/${var.build-vol}"
     delete_on_termination = true
-    volume_size           = 16
+    volume_size           = 40
     volume_type           = "gp2"
     omit_from_artifact    = true
   }

scripts/copy-nix-store.sh

@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+# Copy nix store from temporary build volume back to root filesystem
+# Includes size check and fails build if it won't fit
+
+set -o errexit
+set -o pipefail
+set -o xtrace
+
+echo "=== Migrating Nix store from temp volume to root filesystem ==="
+
+# Stop nix daemon before unmounting
+echo "Stopping nix-daemon..."
+sudo systemctl stop nix-daemon.service || true
+sudo systemctl stop nix-daemon.socket || true
+
+# Get size of nix store on temp volume (in bytes)
+echo ""
+echo "Checking nix store size..."
+NIX_STORE_SIZE=$(sudo du -sb /mnt/nix-temp | cut -f1)
+NIX_STORE_SIZE_GB=$(awk "BEGIN {printf \"%.2f\", $NIX_STORE_SIZE / 1024 / 1024 / 1024}")
+echo "Nix store size: ${NIX_STORE_SIZE_GB} GB (${NIX_STORE_SIZE} bytes)"
+
+# Get available space on root filesystem (in bytes)
+# Check the actual root filesystem, not the bind mount
+ROOT_FS_DEVICE=$(df / | tail -1 | awk '{print $1}')
+ROOT_AVAILABLE=$(df -B1 / | tail -1 | awk '{print $4}')
+ROOT_AVAILABLE_GB=$(awk "BEGIN {printf \"%.2f\", $ROOT_AVAILABLE / 1024 / 1024 / 1024}")
+echo "Root filesystem (${ROOT_FS_DEVICE}) available space: ${ROOT_AVAILABLE_GB} GB (${ROOT_AVAILABLE} bytes)"
+
+# Add 10% buffer for safety
+REQUIRED_SPACE=$(awk "BEGIN {printf \"%.0f\", $NIX_STORE_SIZE * 1.1}")
+REQUIRED_SPACE_GB=$(awk "BEGIN {printf \"%.2f\", $REQUIRED_SPACE / 1024 / 1024 / 1024}")
+echo "Required space (with 10% buffer): ${REQUIRED_SPACE_GB} GB (${REQUIRED_SPACE} bytes)"
+
+# Check if there's enough space
+echo ""
+if [ "$ROOT_AVAILABLE" -lt "$REQUIRED_SPACE" ]; then
+    echo "======================================"
+    echo "ERROR: Not enough space on root filesystem!"
+    echo "======================================"
+    echo "  Nix store size:       ${NIX_STORE_SIZE_GB} GB"
+    echo "  Required (+ buffer):  ${REQUIRED_SPACE_GB} GB"
+    echo "  Available on root:    ${ROOT_AVAILABLE_GB} GB"
+    SHORTFALL=$(awk "BEGIN {printf \"%.2f\", ($REQUIRED_SPACE - $ROOT_AVAILABLE) / 1024 / 1024 / 1024}")
+    echo "  Shortfall:            ${SHORTFALL} GB"
+    echo ""
+    echo "Build FAILED: Nix store is too large to fit on the root volume."
+    echo "Consider increasing the root volume size in amazon-arm64-nix.pkr.hcl"
+    exit 1
+fi
+
+echo "✓ Space check passed. Sufficient space available."
+echo ""
+
+# Unmount the bind mount
+echo "Unmounting bind mount /nix..."
+sudo umount /nix
+
+echo "/nix now shows original (empty) directory on root filesystem"
+ls -la /nix/ || true
+
+echo ""
+echo "Copying nix store from temp volume to root filesystem..."
+echo "This may take several minutes..."
+sudo rsync -aHAXS --info=progress2 /mnt/nix-temp/ /nix/
+
+# Verify the copy
+COPIED_SIZE=$(sudo du -sb /nix | cut -f1)
+COPIED_SIZE_GB=$(awk "BEGIN {printf \"%.2f\", $COPIED_SIZE / 1024 / 1024 / 1024}")
+echo ""
+echo "Copy verification:"
+echo "  Original size: ${NIX_STORE_SIZE_GB} GB (${NIX_STORE_SIZE} bytes)"
+echo "  Copied size:   ${COPIED_SIZE_GB} GB (${COPIED_SIZE} bytes)"
+
+# Allow small differences due to filesystem overhead
+SIZE_DIFF=$(awk "BEGIN {x = $NIX_STORE_SIZE - $COPIED_SIZE; print (x < 0 ? -x : x)}")
+SIZE_DIFF_PERCENT=$(awk "BEGIN {printf \"%.2f\", $SIZE_DIFF * 100 / $NIX_STORE_SIZE}")
+
+# Check if difference is greater than 1%
+if awk "BEGIN {exit !($SIZE_DIFF_PERCENT > 1)}"; then
+    echo "ERROR: Significant size mismatch after copy (${SIZE_DIFF_PERCENT}% difference)!"
+    exit 1
+fi
+
+echo "✓ Copy verified (size difference: ${SIZE_DIFF_PERCENT}%)"
+
+# Unmount temp volume (will be discarded by packer)
+echo ""
+echo "Unmounting temp volume /mnt/nix-temp..."
+sudo umount /mnt/nix-temp
+sudo rmdir /mnt/nix-temp
+
+echo ""
+echo "=== Nix Store Migration Complete ==="
+echo "Final location: /nix (on root filesystem)"
+df -h /nix
+echo ""
+echo "Nix store contents:"
+sudo du -sh /nix/store
+
+# Restart nix daemon
+echo ""
+echo "Restarting nix-daemon..."
+sudo systemctl start nix-daemon.socket || true
+sudo systemctl start nix-daemon.service || true
+
+echo ""
+echo "✓ All done! Nix store successfully migrated to root filesystem."

scripts/mount-build-volume.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+# Mount temporary build volume for stage 2 nix operations
+# Uses bind mount so nix operations transparently use the larger temp volume
+
+set -o errexit
+set -o pipefail
+set -o xtrace
+
+echo "=== Setting up temporary build volume for Nix ==="
+
+echo "Formatting build volume /dev/xvdc..."
+sudo mkfs.ext4 -F -O ^has_journal /dev/xvdc
+
+echo "Creating mount point /mnt/nix-temp..."
+sudo mkdir -p /mnt/nix-temp
+
+echo "Mounting /dev/xvdc to /mnt/nix-temp..."
+sudo mount /dev/xvdc /mnt/nix-temp
+
+echo "Setting permissions on /mnt/nix-temp..."
+sudo chmod 755 /mnt/nix-temp
+
+echo "Creating /nix directory..."
+sudo mkdir -p /nix
+
+echo "Bind mounting /mnt/nix-temp over /nix..."
+sudo mount --bind /mnt/nix-temp /nix
+
+echo ""
+echo "✓ Build volume setup complete"
+echo "  Temp volume: /mnt/nix-temp (40 GB)"
+echo "  Bind mount: /nix -> /mnt/nix-temp"
+echo "  Nix will be installed to /nix (transparently using temp volume)"
+df -h /nix

stage2-nix-psql.pkr.hcl

@@ -86,6 +86,13 @@ source "amazon-ebs" "ubuntu" {
     delay_seconds = 15
     max_attempts  = 120  # 120 * 15s = 30 minutes max wait
   }
+
+  launch_block_device_mappings {
+    device_name           = "/dev/xvdc"
+    delete_on_termination = true
+    volume_size           = 40
+    volume_type           = "gp2"
+  }
 
   ena_support = true
 
@@ -136,6 +143,11 @@ build {
     destination = "/tmp/ansible-playbook"
   }
 
+  # Mount temporary build volume before nix operations
+  provisioner "shell" {
+    script = "scripts/mount-build-volume.sh"
+  }
+
   provisioner "shell" {
     environment_vars = [
       "GIT_SHA=${var.git_sha}",
@@ -144,4 +156,9 @@ build {
      script = "scripts/nix-provision.sh"
   }
 
+  # Copy nix store from temp volume to root filesystem after all operations complete
+  provisioner "shell" {
+    script = "scripts/copy-nix-store.sh"
+  }
+
 }