Codify Cloud Run scaling settings into deploy config (warm floor + max-instances 8)

.github/scripts/cloud_run_env.sh

@@ -13,7 +13,16 @@ cloud_run_set_defaults() {
   CLOUD_RUN_CPU="${CLOUD_RUN_CPU:-4}"
   CLOUD_RUN_MEMORY="${CLOUD_RUN_MEMORY:-16Gi}"
   CLOUD_RUN_TIMEOUT="${CLOUD_RUN_TIMEOUT:-300}"
+  # Revision-level minimum (gcloud --min-instances). MUST stay 0 everywhere: a
+  # positive value is set immutably on every new revision, so each tagged
+  # no-traffic candidate would pin its own warm instances (a per-tag cost bomb).
   CLOUD_RUN_MIN_INSTANCES="${CLOUD_RUN_MIN_INSTANCES:-0}"
+  # Service-level warm floor across all revisions (gcloud --min). This is the
+  # correct place to keep instances warm: one floor for whichever revision is
+  # serving, with no per-revision pinning. Production sets 2 so the service never
+  # cold-starts from zero under the overnight bot trough; staging leaves it 0
+  # (scale-to-zero). Rationale in docs/migration/cloud-run-operations.md.
+  CLOUD_RUN_SERVICE_MIN_INSTANCES="${CLOUD_RUN_SERVICE_MIN_INSTANCES:-0}"
   CLOUD_RUN_MAX_INSTANCES="${CLOUD_RUN_MAX_INSTANCES:-1}"
   # Stage 2-qualified runtime shape, pinned explicitly on every deploy —
   # rationale in docs/migration/cloud-run-operations.md ("Runtime shape").
@@ -50,6 +59,7 @@ cloud_run_set_defaults() {
   export CLOUD_RUN_MEMORY
   export CLOUD_RUN_TIMEOUT
   export CLOUD_RUN_MIN_INSTANCES
+  export CLOUD_RUN_SERVICE_MIN_INSTANCES
   export CLOUD_RUN_MAX_INSTANCES
   export CLOUD_RUN_CONCURRENCY
   export CLOUD_RUN_WEB_CONCURRENCY

.github/scripts/deploy_cloud_run_candidate.sh

@@ -52,6 +52,7 @@ cloud_run_run gcloud run deploy "${CLOUD_RUN_SERVICE}" \
   --memory "${CLOUD_RUN_MEMORY}" \
   --timeout "${CLOUD_RUN_TIMEOUT}" \
   --min-instances "${CLOUD_RUN_MIN_INSTANCES}" \
+  --min "${CLOUD_RUN_SERVICE_MIN_INSTANCES}" \
   --max-instances "${CLOUD_RUN_MAX_INSTANCES}" \
   --concurrency "${CLOUD_RUN_CONCURRENCY}" \
   --startup-probe "${CLOUD_RUN_STARTUP_PROBE}" \

.github/workflows/push.yml

@@ -214,8 +214,10 @@ jobs:
     env:
       CLOUD_RUN_SERVICE: policyengine-api-staging
       # Staging stays scale-to-zero, single instance: it exists for per-push
-      # validation, not capacity.
+      # validation, not capacity. Both the revision-level (--min-instances) and
+      # service-level (--min) floors are 0.
       CLOUD_RUN_MIN_INSTANCES: "0"
+      CLOUD_RUN_SERVICE_MIN_INSTANCES: "0"
       CLOUD_RUN_MAX_INSTANCES: "1"
     permissions:
       contents: read
@@ -542,10 +544,15 @@ jobs:
     environment: production
     env:
       CLOUD_RUN_SERVICE: policyengine-api
-      # Sized by the Stage 2 qualification — rationale and numbers in
-      # docs/migration/cloud-run-operations.md ("Runtime shape and scaling").
-      # Never pin a min-instances value here: warm capacity is service-level.
-      CLOUD_RUN_MAX_INSTANCES: "4"
+      # Sized by the Stage 2 qualification and the PR 4 host cutover — rationale
+      # and numbers in docs/migration/cloud-run-operations.md ("Runtime shape and
+      # scaling"). Warm capacity is expressed service-level (--min); the
+      # revision-level floor stays 0 (default in cloud_run_env.sh) to avoid
+      # per-tag pinning. Max raised to 8 for the 100% Cloud Run cutover: at 50/50
+      # the service was already pinned at the previous ceiling of 4 during the
+      # daytime peak, so full traffic needs the extra headroom.
+      CLOUD_RUN_SERVICE_MIN_INSTANCES: "2"
+      CLOUD_RUN_MAX_INSTANCES: "8"
     permissions:
       contents: read
       id-token: write

changelog.d/cloud-run-scaling-config-reconcile.changed.md

@@ -0,0 +1 @@
+Raise the production Cloud Run service to max-instances 8 and apply the service-level warm floor (`--min`) from the deploy config so it is asserted on every deploy instead of only as live service state.

docs/migration/cloud-run-operations.md

@@ -124,17 +124,27 @@ Values measured and justified in
   covering every boot. Note that lazily deferring the build does **not** help: it
   relocates the cost onto the first request, where readiness would lie and a user
   would absorb it.
-- **Scaling pins live in `push.yml` per job**: production `max-instances 4` (peak real
-  traffic ~11 RPS, mostly cached/light; ~1–2 concurrent uncached calculates per
-  instance), staging `min 0 / max 1`.
-- **Warm capacity is a service-level setting made manually, once** — CI keeps
-  revision-level `--min-instances 0`, because revision-level minimums keep a warm 16Gi
-  instance alive per accumulated `stage3-*` tag.
+- **Scaling pins live in `push.yml` per job**: production `max-instances 8`, staging
+  `min 0 / max 1`. Production was `max-instances 4` through Stage 10; it was raised to
+  8 for the 100% Cloud Run cutover because at the 50/50 split the service already sat
+  pinned at the ceiling of 4 during the daytime peak (`instance_count` flatlined at
+  4.0), so carrying all traffic needs the extra headroom.
+- **Warm capacity is applied by CI on every deploy as a service-level `--min`**, via
+  the `CLOUD_RUN_SERVICE_MIN_INSTANCES` env (production 2, staging 0). CI keeps the
+  revision-level `--min-instances 0`, because a revision-level minimum keeps a warm
+  16Gi instance alive per accumulated `stage3-*` tag.
 
   **The flag names differ by one word and mean opposite things:** service-level
   warm capacity is `--min` (or the `run.googleapis.com/minScale` annotation on the
   *service* metadata); `--min-instances` is the *revision-level* setting and is the
-  per-tag cost bomb. `--min` requires a recent gcloud (it does not exist in 461).
+  per-tag cost bomb. `--min` requires a recent gcloud (it does not exist in 461); the
+  CI runners and the deploy scripts assume a current gcloud. The deploy passes both
+  flags on every deploy — `--min-instances 0 --min ${CLOUD_RUN_SERVICE_MIN_INSTANCES}`
+  — so the floor is asserted in code, not carried only as live service state.
+
+  To change the floor, edit `CLOUD_RUN_SERVICE_MIN_INSTANCES` in `push.yml` (and the
+  default in `cloud_run_env.sh`); the next production deploy applies it. To apply it
+  out-of-band between deploys:
 
   ```bash
   gcloud run services update policyengine-api \
@@ -173,7 +183,9 @@ Values measured and justified in
 
   **Current value: 2.** Originally set to 1 on 2026-07-08 (via the YAML-replace
   fallback, since the gcloud in use lacked `--min`), raised to 2 on 2026-07-22 by
-  the same route. One warm instance meant every
+  the same route, and codified into the deploy config on 2026-07-23 (env
+  `CLOUD_RUN_SERVICE_MIN_INSTANCES`) so it is no longer live-state-only. One warm
+  instance meant every
   burst beyond a single instance's capacity landed on a cold boot; two warm
   instances carry the burst during the ~161s a scaled-out instance takes to become
   ready. Cost is ~$131/month per warm instance at 4 vCPU / 16Gi (idle CPU

tests/unit/test_cloud_run_deploy_scripts.py

@@ -373,6 +373,33 @@ def test_deploy_cloud_run_candidate_pins_runtime_shape():
     # docs/migration/cloud-run-operations.md ("Runtime shape and scaling").
     assert "--concurrency 6 " in result.stdout
     assert "WEB_CONCURRENCY=2 " in result.stdout
+    # Revision-level floor (--min-instances) stays 0; the warm floor is applied
+    # service-level (--min), defaulting to 0 unless a job overrides it.
+    assert "--min-instances 0 " in result.stdout
+    assert "--min 0 " in result.stdout
+
+
+def test_deploy_cloud_run_candidate_applies_service_level_min_floor():
+    """The warm floor is service-level (--min), never revision-level.
+
+    A positive --min-instances is immutably baked onto every new revision, so
+    each tagged no-traffic candidate would pin its own warm instances. --min
+    keeps one floor across whichever revision is serving, with no per-tag cost.
+    """
+    result = _run_script(
+        ".github/scripts/deploy_cloud_run_candidate.sh",
+        _script_env(
+            **_required_runtime_env(),
+            CLOUD_RUN_IMAGE_URI="us-central1-docker.pkg.dev/project/repo/api:sha",
+            CLOUD_RUN_TAG="stage3-test",
+            CLOUD_RUN_SERVICE_MIN_INSTANCES="2",
+        ),
+    )
+
+    assert result.returncode == 0, result.stderr
+    # Service-level floor honoured, and the revision-level floor stays 0 so the
+    # candidate does not pin per-tag warm instances.
+    assert "--min 2 " in result.stdout
     assert "--min-instances 0 " in result.stdout
 
 
@@ -435,11 +462,14 @@ def test_push_workflow_pins_cloud_run_scaling_per_job():
 
     assert 'CLOUD_RUN_MIN_INSTANCES: "0"' in staging_deploy
     assert 'CLOUD_RUN_MAX_INSTANCES: "1"' in staging_deploy
-    assert 'CLOUD_RUN_MAX_INSTANCES: "4"' in production_deploy
+    assert 'CLOUD_RUN_MAX_INSTANCES: "8"' in production_deploy
+    # Production keeps a service-level warm floor of 2; staging stays at 0.
+    assert 'CLOUD_RUN_SERVICE_MIN_INSTANCES: "2"' in production_deploy
+    assert 'CLOUD_RUN_SERVICE_MIN_INSTANCES: "0"' in staging_deploy
     # Revision-level min-instances must stay 0 EVERYWHERE in the workflow —
     # including workflow-level env, which flows into every job. Any mention
     # of the variable must be an explicit "0" pin.
-    assert workflow.count("CLOUD_RUN_MIN_INSTANCES") == workflow.count(
+    assert workflow.count("CLOUD_RUN_MIN_INSTANCES:") == workflow.count(
         'CLOUD_RUN_MIN_INSTANCES: "0"'
     )
     # The runtime-shape values live only in cloud_run_env.sh (where the