feat(compute): give every workspace its own volume, and bake a golden image

.github/workflows/canary.yml

@@ -105,6 +105,25 @@ jobs:
         env:
           CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
           BLITZ_DEPLOY_VAR_CLOUD_WORKSPACE_CREDENTIAL_POLICY: byok-required
+          # The golden Hetzner snapshot this deployment boots, per location. It
+          # removes the apt install, the box-image download and the sshd move
+          # from first boot: 130.6 s to 45.3 s, measured 2026-08-27.
+          #
+          # It is hardcoded here rather than kept in a secret because it is not
+          # one: it is a plain [vars] entry, and this is where the other
+          # per-deployment non-secret settings already live. A snapshot id is
+          # scoped to ONE Hetzner project, which is why it cannot be a default
+          # in code and why client prod needs its own bake before release.yml
+          # gets a line like this.
+          #
+          # Rebake with `npm run golden:bake -- --location hel1` and update the
+          # value. An id that no longer resolves is not an outage: the adapter
+          # warns and falls back to stock Ubuntu.
+          #
+          # Inert while the policy above is byok-required, because a snapshot
+          # cannot cross into an org's own project. It goes live for subscribed
+          # orgs under plans/SUBSCRIPTION-COMPUTE.md.
+          BLITZ_DEPLOY_VAR_HETZNER_SERVER_IMAGES: hel1=425047509
           # The billing service's origin, once it has one. Empty is skipped, so
           # this line changes nothing until the value is set, and setting it
           # then needs no commit. It is an environment secret rather than an

env.defaults

@@ -50,6 +50,7 @@ BLITZ_BROKER_STATE_DIR=/var/lib/blitz-broker
 # GOOGLE_CONNECT_CLIENT_SECRET (secret string): OAuth client secret for /connect/google-workspace; set as a Worker secret.
 # HETZNER_API_TOKEN (secret string): Hetzner API credential; set as a Worker secret when that provider is used.
 # HETZNER_MACHINE_TYPES (string): Comma-separated "type@location" machine catalog entries; unset or blank offers cpx21@hil and cpx31@hil.
+# HETZNER_SERVER_IMAGES (string): Comma-separated "location=image" golden-image entries for the deployment Hetzner project (for example "hel1=163000001,*=ubuntu-24.04"); unset or blank boots stock Ubuntu and pays the full bootstrap. A snapshot belongs to one project, so this never reaches a BYOK organization.
 # JWT_SECRET_MAIN (secret string): Session-signing key; set as a Worker secret.
 # LINEAR_CLIENT_ID (string): OAuth client id for /connect/linear; unset leaves the provider un-connectable and the picker says so.
 # LINEAR_CLIENT_SECRET (secret string): OAuth client secret for /connect/linear; set as a Worker secret.

packages/control-plane/core/bootstrap.ts

@@ -267,11 +267,50 @@ main().catch(function (error) {
  * the way you would edit a wire format, not a script. Recipe launches add
  * segments pinned by `test/recipe-invocation-fixtures.test.ts`; a create
  * without a recipe or usage capture emits byte-identical output. */
-export function buildBootstrapScript(options: BootstrapOptions): string {
-  const controlPlaneOrigin = new URL(options.phoneHomeUrl).origin;
+/**
+ * The shell helpers `boxImageSetupScript` calls. Emitting that setup without
+ * these gives `retry: command not found`, and under `set -e` the script dies
+ * where it stands. The golden-image bake hit exactly that on its first real
+ * run: the builder never powered off, and the bake waited 30 minutes for a
+ * shutdown that could not come.
+ *
+ * `buildBootstrapScript` emits these in its own preamble. Any other caller
+ * that embeds the setup has to emit them first.
+ */
+export const BOX_IMAGE_SETUP_HELPERS = `retry() {
+  local attempt=1
+  local max_attempts=10
+  until "$@"; do
+    if (( attempt >= max_attempts )); then
+      echo "command failed after $attempt attempts: $*"
+      return 1
+    fi
+    sleep $((attempt * 3))
+    attempt=$((attempt + 1))
+  done
+}
+`;
+
+/** The three variables that name one box image build. */
+export interface BoxImageRef {
+  boxImageRef: string;
+  boxImageTag: string;
+  boxImageSha256: string;
+}
+
+/**
+ * The bash that puts the box image into the host's docker store: download and
+ * `docker load` for an HTTPS tarball ref, or `docker pull` for a registry ref.
+ * Both branches guard on `docker image inspect`, so a host that already holds
+ * the image does no work at all. That guard is what makes a golden snapshot
+ * fast: the image is already there and the whole block is skipped.
+ *
+ * Exported so the golden-image bake script bakes the SAME bytes a workspace
+ * would download. Two copies of this would be two sides of one contract, and
+ * drift between them would produce snapshots holding the wrong image.
+ */
+export function boxImageSetupScript(options: BoxImageRef): string {
   const isTarball = options.boxImageRef.startsWith("https://");
-  const trimmedSshPublicKey = options.sshPublicKey?.trim();
-  const sshPublicKey = trimmedSshPublicKey === "" ? undefined : trimmedSshPublicKey;
   if (isTarball && options.boxImageTag.trim() === "") {
     throw new Error("BOX_IMAGE_TAG is required when BOX_IMAGE_REF is an HTTPS URL");
   }
@@ -280,8 +319,7 @@ export function buildBootstrapScript(options: BootstrapOptions): string {
       "BOX_IMAGE_SHA256 must be a 64-character hexadecimal digest when BOX_IMAGE_REF is an HTTPS URL",
     );
   }
-
-  const imageSetup = isTarball
+  return isTarball
     ? String.raw`download() {
   curl --fail --location --retry 10 --retry-all-errors --retry-delay 3 \
     --silent --show-error --output "$2" "$1"
@@ -370,6 +408,14 @@ box_image="$BOX_IMAGE_TAG"`
 fi
 docker image inspect "$BOX_IMAGE_REF" >/dev/null
 box_image="$BOX_IMAGE_REF"`;
+}
+
+export function buildBootstrapScript(options: BootstrapOptions): string {
+  const controlPlaneOrigin = new URL(options.phoneHomeUrl).origin;
+  const trimmedSshPublicKey = options.sshPublicKey?.trim();
+  const sshPublicKey = trimmedSshPublicKey === "" ? undefined : trimmedSshPublicKey;
+
+  const imageSetup = boxImageSetupScript(options);
 
   // The resolved provider's own lines. "" for a provider that needs none, so
   // its boxes never read another provider's setup.
@@ -544,19 +590,7 @@ touch "$BOOTSTRAP_LOG"
 chmod 0600 "$BOOTSTRAP_LOG"
 exec >>"$BOOTSTRAP_LOG" 2>&1
 
-retry() {
-  local attempt=1
-  local max_attempts=10
-  until "$@"; do
-    if (( attempt >= max_attempts )); then
-      echo "command failed after $attempt attempts: $*"
-      return 1
-    fi
-    sleep $((attempt * 3))
-    attempt=$((attempt + 1))
-  done
-}
-
+${BOX_IMAGE_SETUP_HELPERS}
 fail() {
   bootstrap_error="$*"
   echo "blitz bootstrap failed: $*"
@@ -593,10 +627,25 @@ apt_watchdog() {
   done
   fail "apt-get $1 kept failing or stalling after 3 attempts"
 }
-apt_watchdog update
-apt_watchdog install -y docker.io curl
+# A golden image already carries docker and curl, and re-running apt on it
+# changes nothing while costing about 36 seconds (measured 2026-08-27 on
+# cx23@hel1: 18.3 s for update, 17.4 s for the install). A stock Ubuntu image
+# has neither and takes the original path, so this is a skip, not a new
+# dependency: the box never relies on the tools being pre-baked.
+if command -v docker >/dev/null 2>&1 && command -v curl >/dev/null 2>&1; then
+  echo "blitz: docker and curl are already installed; skipping apt"
+else
+  apt_watchdog update
+  apt_watchdog install -y docker.io curl
+fi
 systemctl enable --now docker
 
+# Every phase marker carries seconds since the script started. Without these
+# the only way to attribute boot time was subtraction, which turned every
+# tuning decision into an estimate (tools/e2e/GAPS.md).
+blitz_phase() { echo "blitz-phase: $1 seconds=$SECONDS"; }
+blitz_phase apt-done
+
 mkdir -p /var/lib/blitz
 volume_device=""
 for candidate in /dev/disk/by-id/scsi-0HC_Volume_*; do
@@ -628,6 +677,7 @@ if [ -n "$volume_device" ]; then
   grep -Fqx "$fstab_entry" /etc/fstab || printf '%s\n' "$fstab_entry" >>/etc/fstab
 fi
 
+blitz_phase volume-mounted
 touch "$DURABLE_BOOTSTRAP_LOG"
 chmod 0600 "$DURABLE_BOOTSTRAP_LOG"
 cat "$BOOTSTRAP_LOG" >"$DURABLE_BOOTSTRAP_LOG"
@@ -668,9 +718,20 @@ ${sshPublicKeyProvisioning}
 # credentials are installed after this VM proves its host key to phone-home.
 rm -f /var/lib/blitz/box-credential.json /var/lib/blitz/origin
 
+port_22_free() {
+  ! ss -tln 2>/dev/null | grep -qE '(^|[^0-9.:])(0\.0\.0\.0|\[::\]|\*):22[[:space:]]'
+}
 # Ubuntu 24.04 activates sshd through ssh.socket on port 22. Validate the
 # replacement listener before stopping that socket so Docker can safely claim
 # host port 22 without losing the host SSH recovery path.
+#
+# A golden image has already made this move, and its sshd comes up on 2222 with
+# ssh.socket masked. Stopping and restarting sshd there costs seconds and
+# changes nothing, so the whole block is skipped when the invariant already
+# holds: a listener on 2222 and nothing on 22.
+if ss -tln 2>/dev/null | grep -qE ':2222[[:space:]]' && port_22_free; then
+  echo "blitz: host sshd is already on 2222 and port 22 is free; skipping the move"
+else
 install -d -m 0755 /etc/ssh/sshd_config.d
 # 00- sorts ahead of image drop-ins; sshd takes the first Port it sees.
 cat >/etc/ssh/sshd_config.d/00-blitz.conf <<'SSHD_CONFIG'
@@ -703,9 +764,6 @@ done
 # "failed to bind host port 0.0.0.0:22/tcp: address already in use" (exit 125)
 # on whichever boots fast enough to lose the race. Wait for the port to be
 # genuinely free, and say so plainly if it never is.
-port_22_free() {
-  ! ss -tln 2>/dev/null | grep -qE '(^|[^0-9.:])(0\.0\.0\.0|\[::\]|\*):22[[:space:]]'
-}
 sshd_release_deadline=$((SECONDS + 60))
 until port_22_free; do
   if (( SECONDS >= sshd_release_deadline )); then
@@ -714,8 +772,11 @@ until port_22_free; do
   fi
   sleep 1
 done
+fi
+blitz_phase sshd-ready
 
 ${imageSetup}
+blitz_phase box-image-ready
 install -d -m 0755 /etc/blitz
 ${invocationFiles}${usageDirectories}# The one docker run for the box container, extracted to a host script so
 # the initial start here and the host-side updater (blitz-box-update below)

packages/control-plane/core/compute/hetzner-config.ts

@@ -0,0 +1,157 @@
+/** Hetzner configuration: the Worker vars this provider reads, the warnings it
+ * raises about them, and the small pure helpers that parse a machine-type id.
+ * Split out of `hetzner.ts` so the adapter itself stays under the 700-line
+ * warn. Nothing here performs I/O. */
+
+export const HETZNER_USER_DATA_MAX_BYTES = 32 * 1024;
+// Current Hetzner server-type names (for example cx22, cpx31, and cax11)
+// are lowercase ASCII letters followed by decimal digits, with no dash.
+export interface MachineSelection {
+  type: string;
+  location: string | null;
+}
+
+/** Splits a machine-type id at its last `@`: `cx23@hel1` is the server type
+ * and the location. A type with no `@` names the account default location. */
+export function machineId(value: string): MachineSelection {
+  const separator = value.lastIndexOf("@");
+  if (separator === -1) return { type: value, location: null };
+  return { type: value.slice(0, separator), location: value.slice(separator + 1) };
+}
+
+export const SERVER_TYPE_NAME_PATTERN = /^[a-z]+\d+$/u;
+export const LOCATION_NAME_PATTERN = /^[a-z0-9-]+$/u;
+// Default catalog: two cheap EU types first, then the two US-west types.
+// Gross price each month, read from /v1/pricing on 2026-08-25: cx23@hel1
+// 6.49, cx33@hel1 9.99, cpx21@hil 37.49, cpx31@hil 73.49. That account bills
+// in USD. The figures are the same numbers this comment once called euro,
+// which is how the wrong sign reached the cards.
+// cx33@hel1 gives the same 4 cpu and 8 GB as cpx31@hil. It costs about one
+// seventh as much. That is the reason for the EU entries.
+// Hetzner does not sell cpx21 or cpx31 in any EU location. It sells the cx
+// line only in hel1. A cheaper EU box needs a different type, not the same
+// type in a different region.
+// Operators override the catalog with the HETZNER_MACHINE_TYPES Worker var.
+// The catalog constrains what the create page offers; existing workspaces on
+// other types keep working because ownership stays shape-based.
+export const DEFAULT_HETZNER_MACHINE_TYPES: readonly string[] = [
+  "cx23@hel1",
+  "cx33@hel1",
+  "cpx21@hil",
+  "cpx31@hil",
+];
+
+/** The stock image every Hetzner VM booted before golden images existed, and
+ * the fallback whenever a configured snapshot cannot be used. */
+export const HETZNER_STOCK_IMAGE = "ubuntu-24.04";
+// A Hetzner image is either a system-image name (`ubuntu-24.04`) or the
+// decimal id of a snapshot in this project.
+const SERVER_IMAGE_PATTERN = /^[a-z0-9][a-z0-9.-]{0,63}$/u;
+
+export interface HetznerMachineTypeCatalogWarning {
+  event: "hetzner_machine_type_catalog_entry_rejected";
+  entry: string;
+  reason: string;
+}
+
+/** A configured golden image was refused, so the create fell back to stock
+ * Ubuntu. The workspace still works; it just pays the full bootstrap again.
+ * Silence here would hide a whole fleet quietly running the slow path. */
+export interface HetznerServerImageWarning {
+  event: "hetzner_server_image_rejected";
+  location: string;
+  image: string;
+  reason: string;
+}
+
+/** Hetzner states the billing currency only in /v1/pricing. When that read
+ * fails, every Hetzner card loses its price. The operator must hear why. */
+export interface HetznerPriceCurrencyWarning {
+  event: "hetzner_price_currency_unavailable";
+  reason: string;
+}
+
+export type HetznerProviderWarning =
+  | HetznerMachineTypeCatalogWarning
+  | HetznerPriceCurrencyWarning
+  | HetznerServerImageWarning;
+
+export type HetznerWarningSink = (warning: HetznerProviderWarning) => void;
+
+type HetznerCatalogWarningSink = (
+  warning: HetznerMachineTypeCatalogWarning,
+) => void;
+
+/**
+ * Parses the HETZNER_MACHINE_TYPES Worker var (comma-separated
+ * "type@location" entries) into the machine-type catalog allowlist. An unset
+ * or blank var keeps the default catalog. Malformed entries are skipped with
+ * one structured warning each; they never crash the Worker.
+ */
+export function hetznerMachineTypeAllowlistFromEnv(
+  raw: string | undefined,
+  warn: HetznerCatalogWarningSink = () => {},
+): ReadonlySet<string> {
+  if (raw === undefined || raw.trim() === "") {
+    return new Set(DEFAULT_HETZNER_MACHINE_TYPES);
+  }
+  const allowlist = new Set<string>();
+  for (const segment of raw.split(",")) {
+    const entry = segment.trim();
+    if (entry === "") continue;
+    const selected = machineId(entry);
+    const valid = selected.location !== null
+      && SERVER_TYPE_NAME_PATTERN.test(selected.type)
+      && LOCATION_NAME_PATTERN.test(selected.location);
+    if (!valid) {
+      warn({
+        event: "hetzner_machine_type_catalog_entry_rejected",
+        entry,
+        reason: 'expected "<server-type>@<location>" (for example "cpx21@hil")',
+      });
+      continue;
+    }
+    allowlist.add(entry);
+  }
+  return allowlist;
+}
+
+/**
+ * Parses the HETZNER_SERVER_IMAGES Worker var into the golden-image map.
+ *
+ * Entries are comma-separated `location=image` pairs, and `*=image` sets the
+ * default for locations with no entry of their own. `image` is a snapshot id
+ * or a system-image name. An unset or blank var boots stock Ubuntu, which is
+ * what every deployment did before golden images existed.
+ *
+ * Snapshots are per-project, so this map belongs to one credential scope. A
+ * BYOK organization with its own Hetzner project has no entry here and boots
+ * stock Ubuntu. That is correct, not a bug: its project holds no snapshot.
+ */
+export function hetznerServerImagesFromEnv(
+  raw: string | undefined,
+  warn: (warning: HetznerServerImageWarning) => void = () => {},
+): ReadonlyMap<string, string> {
+  const images = new Map<string, string>();
+  if (raw === undefined || raw.trim() === "") return images;
+  for (const segment of raw.split(",")) {
+    const entry = segment.trim();
+    if (entry === "") continue;
+    const separator = entry.indexOf("=");
+    const location = separator === -1 ? "" : entry.slice(0, separator).trim();
+    const image = separator === -1 ? "" : entry.slice(separator + 1).trim();
+    const validLocation = location === "*" || LOCATION_NAME_PATTERN.test(location);
+    if (!validLocation || !SERVER_IMAGE_PATTERN.test(image)) {
+      warn({
+        event: "hetzner_server_image_rejected",
+        location,
+        image,
+        reason: 'expected "<location>=<image>" (for example "hel1=163000001" or "*=ubuntu-24.04")',
+      });
+      continue;
+    }
+    images.set(location, image);
+  }
+  return images;
+}
+