Kill hung runs early when a pinned pool model is delisted

scripts/ci/run_opencode_review_model_pool.sh

@@ -159,7 +159,9 @@ write_prompt() {
 	else
 		intro="Review PR #\${PR_NUMBER} in \${OPENCODE_SOURCE_WORKDIR} with \${model_candidate}."
 	fi
-	contract_file="$OPENCODE_REVIEW_WORKDIR/opencode-review-contract-${model_candidate//\//-}.md"
+	# Colon-safe: OpenRouter ":free" candidates would otherwise produce file
+	# names that Windows and actions/upload-artifact reject.
+	contract_file="$OPENCODE_REVIEW_WORKDIR/opencode-review-contract-${model_candidate//[\/:]/-}.md"
 	evidence_excerpt_file="$OPENCODE_REVIEW_WORKDIR/bounded-review-evidence-excerpt.md"
 	evidence_file_in_workdir="$OPENCODE_REVIEW_WORKDIR/bounded-review-evidence.md"
 	cp "$GITHUB_WORKSPACE/scripts/ci/opencode_review_prompt_template.md" "$contract_file"
@@ -243,7 +245,7 @@ is_fatal_provider_failure() {
 		return 0
 	fi
 	[ -s "$opencode_json_file" ] || return 1
-	grep -Eiq 'budget limit|insufficient_quota' "$opencode_json_file"
+	grep -Eiq 'budget limit|insufficient_quota|insufficient credits|payment required|model_not_found|model not found|ModelNotFoundError|not a valid model|no endpoints' "$opencode_json_file"
 }
 
 has_fatal_provider_error_event() {
@@ -253,8 +255,29 @@ has_fatal_provider_error_event() {
 	# Only structured "type":"error" events count while the process is still
 	# running: model prose or tool output quoting these signatures is
 	# JSON-escaped inside event strings, so a healthy streaming run is never
-	# killed for merely discussing context windows or quota errors.
-	awk 'tolower($0) ~ /"type"[[:space:]]*:[[:space:]]*"error"/ && tolower($0) ~ /contextoverflowerror|tokens_limit_reached|request body too large|context window|budget limit|insufficient_quota/ { found = 1; exit } END { exit !found }' "$opencode_json_file"
+	# killed for merely discussing context windows, quota errors, or missing
+	# models. Model-unavailable signatures (OpenRouter "No endpoints found" /
+	# "not a valid model ID", OpenAI-style model_not_found) matter because a
+	# delisted pinned free model would otherwise hang and burn the whole
+	# candidate run budget.
+	awk 'tolower($0) ~ /"type"[[:space:]]*:[[:space:]]*"error"/ && tolower($0) ~ /contextoverflowerror|tokens_limit_reached|request body too large|context window|budget limit|insufficient_quota|insufficient credits|payment required|model_not_found|model not found|modelnotfounderror|not a valid model|no endpoints/ { found = 1; exit } END { exit !found }' "$opencode_json_file"
+}
+
+is_credit_exhausted_failure() {
+	local opencode_json_file="$1"
+	local opencode_stderr_file="$2"
+
+	# Paid-provider credit exhaustion (OpenRouter HTTP 402 "Insufficient
+	# credits") can never recover within one run: every retry is a wasted
+	# paid request. Match structured "type":"error" events in the JSON
+	# stream (same trust model as has_fatal_provider_error_event) plus
+	# CLI diagnostics on stderr, which never contain model prose.
+	if [ -s "$opencode_json_file" ] &&
+		awk 'tolower($0) ~ /"type"[[:space:]]*:[[:space:]]*"error"/ && tolower($0) ~ /insufficient credits|payment required|(^|[^0-9])402([^0-9]|$)/ { found = 1; exit } END { exit !found }' "$opencode_json_file"; then
+		return 0
+	fi
+	[ -s "$opencode_stderr_file" ] || return 1
+	grep -Eiq 'insufficient credits|payment required|"code"[[:space:]]*:[[:space:]]*402' "$opencode_stderr_file"
 }
 
 emit_sanitized_opencode_failure_detail() {
@@ -274,8 +297,12 @@ emit_sanitized_opencode_failure_detail() {
 	failure_class="unclassified"
 	if grep -Eiq 'ContextOverflowError|tokens_limit_reached|Request body too large|context window' "$opencode_json_file" "$opencode_stderr_file" 2>/dev/null; then
 		failure_class="context-window"
+	elif grep -Eiq 'insufficient credits|payment required|"code"[[:space:]]*:[[:space:]]*402' "$opencode_json_file" "$opencode_stderr_file" 2>/dev/null; then
+		failure_class="credit-exhausted"
 	elif grep -Eiq 'budget limit|insufficient_quota|quota exceeded' "$opencode_json_file" "$opencode_stderr_file" 2>/dev/null; then
 		failure_class="quota-or-budget"
+	elif grep -Eiq 'model_not_found|model not found|ModelNotFoundError|not a valid model|no endpoints' "$opencode_json_file" "$opencode_stderr_file" 2>/dev/null; then
+		failure_class="model-unavailable"
 	elif grep -Eiq 'rate.?limit|too many requests|(^|[^0-9])429([^0-9]|$)' "$opencode_json_file" "$opencode_stderr_file" 2>/dev/null; then
 		failure_class="rate-limit"
 	elif grep -Eiq 'permission denied|authentication|authorization|(^|[^0-9])(401|403)([^0-9]|$)' "$opencode_json_file" "$opencode_stderr_file" 2>/dev/null; then
@@ -427,7 +454,7 @@ run_one_model_attempt() {
 			printf 'OpenCode %s attempt %s/%s timed out after %ss; falling through within the remaining retry budget instead of blocking the org queue.\n' "$model_candidate" "$attempt" "$attempts" "$run_timeout_seconds"
 		fi
 		if is_fatal_provider_failure "$opencode_json_file"; then
-			printf 'OpenCode %s attempt %s/%s hit a fatal provider error (context window, token budget, or quota); skipping remaining attempts for this model.\n' "$model_candidate" "$attempt" "$attempts"
+			printf 'OpenCode %s attempt %s/%s hit a fatal provider error (context window, token budget, quota, or model unavailable); skipping remaining attempts for this model.\n' "$model_candidate" "$attempt" "$attempts"
 			return 2
 		fi
 		return 1
@@ -438,7 +465,7 @@ run_one_model_attempt() {
 		printf 'OpenCode %s attempt %s/%s JSON output did not include a session id.\n' "$model_candidate" "$attempt" "$attempts"
 		emit_rejected_opencode_artifact_metadata "sessionless-json" "$opencode_json_file"
 		if is_fatal_provider_failure "$opencode_json_file"; then
-			printf 'OpenCode %s attempt %s/%s hit a fatal provider error (context window, token budget, or quota); skipping remaining attempts for this model.\n' "$model_candidate" "$attempt" "$attempts"
+			printf 'OpenCode %s attempt %s/%s hit a fatal provider error (context window, token budget, quota, or model unavailable); skipping remaining attempts for this model.\n' "$model_candidate" "$attempt" "$attempts"
 			return 2
 		fi
 		return 1
@@ -459,7 +486,7 @@ run_one_model_attempt() {
 	if ! normalize_opencode_output "$candidate_output_file"; then
 		printf 'OpenCode %s attempt %s/%s output did not include a valid control conclusion.\n' "$model_candidate" "$attempt" "$attempts"
 		emit_rejected_opencode_artifact_metadata "invalid-control-output" "$candidate_output_file"
-		return 1
+		return 3
 	fi
 	return 0
 }
@@ -469,8 +496,19 @@ main() {
 	local opencode_json_file opencode_export_file agent retry_sleep original_run_timeout run_status cycle_sleep cycle max_cycles
 	local uncapped_run_timeout
 	local changed_file_count small_file_threshold medium_file_threshold
+	local invalid_control_cap max_total_attempts total_attempts alive_candidates
+	local -A dead_candidate_reasons invalid_control_counts
 	local -a model_candidates
 
+	# Spend guards, not timing: a paid candidate that keeps producing
+	# control-rejected output or has exhausted provider credits must stop
+	# consuming paid requests instead of cycling until the retry budget
+	# elapses (run 30120972549 burned the org OpenRouter credit in ~102
+	# cycles of re-sent full prompts). Timeouts/deadlines are untouched.
+	invalid_control_cap="$(env_integer_or_default OPENCODE_INVALID_CONTROL_OUTPUT_CAP 3)"
+	max_total_attempts="$(env_integer_or_default OPENCODE_POOL_MAX_TOTAL_ATTEMPTS 30)"
+	total_attempts=0
+
 	attempts="${OPENCODE_MODEL_ATTEMPTS:-3}"
 	original_run_timeout="${OPENCODE_RUN_TIMEOUT_SECONDS:-600}"
 	budget_seconds="${OPENCODE_TOTAL_RETRY_BUDGET_SECONDS:-1500}"
@@ -529,11 +567,16 @@ main() {
 	while :; do
 		printf 'Starting OpenCode model pool cycle %s.\n' "$cycle"
 		for model_candidate in "${model_candidates[@]}"; do
+			if [ -n "${dead_candidate_reasons[$model_candidate]:-}" ]; then
+				printf 'Skipping OpenCode %s for the rest of this run: %s.\n' \
+					"$model_candidate" "${dead_candidate_reasons[$model_candidate]}"
+				continue
+			fi
 			if should_skip_model_candidate "$model_candidate"; then
 				continue
 			fi
 			assert_reasoning_effort_for_candidate "$model_candidate"
-			safe_model="${model_candidate//\//-}"
+			safe_model="${model_candidate//[\/:]/-}"
 			prompt_file="${RUNNER_TEMP}/opencode-review-${safe_model}-prompt.md"
 			candidate_output_file="${RUNNER_TEMP}/opencode-review-${safe_model}.md"
 			opencode_json_file="${candidate_output_file}.jsonl"
@@ -548,6 +591,14 @@ main() {
 					fi
 					exit 1
 				fi
+				if [ "$max_total_attempts" -gt 0 ] && [ "$total_attempts" -ge "$max_total_attempts" ]; then
+					printf 'OpenCode model pool reached the per-run provider attempt ceiling of %s attempts; ending the pool to bound provider spend. Set OPENCODE_POOL_MAX_TOTAL_ATTEMPTS=0 to disable.\n' "$max_total_attempts"
+					if finish_pool_without_model; then
+						exit 0
+					fi
+					exit 1
+				fi
+				total_attempts=$((total_attempts + 1))
 				remaining="$original_run_timeout"
 				if [ "$deadline" -gt 0 ]; then
 					remaining=$((deadline - now))
@@ -577,6 +628,20 @@ main() {
 				else
 					run_status=$?
 				fi
+				if [ "$run_status" -ne 3 ] && is_credit_exhausted_failure "$opencode_json_file" "${opencode_json_file}.stderr"; then
+					dead_candidate_reasons[$model_candidate]="provider credits exhausted (HTTP 402 / payment required)"
+					printf 'OpenCode %s provider credits are exhausted; marking this candidate failed for the rest of the run so retries cannot accrue further spend.\n' "$model_candidate"
+					break
+				fi
+				if [ "$run_status" -eq 3 ]; then
+					invalid_control_counts[$model_candidate]=$((${invalid_control_counts[$model_candidate]:-0} + 1))
+					if [ "$invalid_control_cap" -gt 0 ] && [ "${invalid_control_counts[$model_candidate]}" -ge "$invalid_control_cap" ]; then
+						dead_candidate_reasons[$model_candidate]="produced ${invalid_control_counts[$model_candidate]} control-rejected outputs"
+						printf 'OpenCode %s produced %s control-rejected outputs; marking this candidate failed for the rest of the run so paid retries cannot loop on rejected output. Set OPENCODE_INVALID_CONTROL_OUTPUT_CAP=0 to disable.\n' \
+							"$model_candidate" "${invalid_control_counts[$model_candidate]}"
+						break
+					fi
+				fi
 				if [ "$run_status" -eq 2 ]; then
 					break
 				fi
@@ -593,6 +658,20 @@ main() {
 			done
 		done
 
+		alive_candidates=0
+		for model_candidate in "${model_candidates[@]}"; do
+			if [ -z "${dead_candidate_reasons[$model_candidate]:-}" ]; then
+				alive_candidates=$((alive_candidates + 1))
+			fi
+		done
+		if [ "$alive_candidates" -eq 0 ]; then
+			printf 'Every OpenCode model candidate is marked failed for this run; ending the pool without further provider spend.\n'
+			if finish_pool_without_model; then
+				exit 0
+			fi
+			exit 1
+		fi
+
 		printf 'OpenCode completed a full model-candidate cycle without a valid control conclusion; continuing until a model succeeds or the retry budget/GitHub Actions job timeout is reached.\n'
 		if [ "$max_cycles" -gt 0 ] && [ "$cycle" -ge "$max_cycles" ]; then
 			printf 'OpenCode model pool reached configured max cycle count %s without a valid control conclusion.\n' "$max_cycles"

tests/test_opencode_model_pool_runner.py

@@ -576,6 +576,111 @@ def test_model_text_quoting_error_signatures_does_not_kill_run(tmp_path: Path) -
     assert "logged a fatal provider error while still running" not in result.stdout
 
 
+def test_delisted_openrouter_model_error_kills_hung_run_early(tmp_path: Path) -> None:
+    """A delisted pinned OpenRouter model dies seconds after a model-unavailable error."""
+    start = time.monotonic()
+    result = run_failed_model(
+        tmp_path,
+        json_line=(
+            '{"type":"error","error":{"name":"ProviderModelNotFoundError","data":'
+            '{"message":"No endpoints found for nvidia/nemotron-3-ultra-550b-a55b:free."}}}'
+        ),
+        model_candidates="openrouter/nvidia/nemotron-3-ultra-550b-a55b:free",
+        extra_env={
+            "OPENROUTER_API_KEY": "fake-openrouter-key",
+            "FAKE_OPENCODE_HANG_SECONDS": "120",
+            "OPENCODE_RUN_TIMEOUT_SECONDS": "120",
+            "OPENCODE_TOTAL_RETRY_BUDGET_SECONDS": "240",
+        },
+    )
+    elapsed = time.monotonic() - start
+
+    assert result.returncode == 1
+    assert "logged a fatal provider error while still running" in result.stdout
+    assert "skipping remaining attempts for this model" in result.stdout
+    assert "class=model-unavailable" in result.stdout
+    assert elapsed < 25
+
+
+def test_credit_exhausted_402_ends_pool_without_further_spend(tmp_path: Path) -> None:
+    """A paid candidate hitting HTTP 402 is dead for the run instead of cycling."""
+    start = time.monotonic()
+    result = run_failed_model(
+        tmp_path,
+        json_line=(
+            '{"type":"error","error":{"name":"AI_APICallError","data":'
+            '{"message":"Insufficient credits. Add more using '
+            'https://openrouter.ai/settings/credits","statusCode":402}}}'
+        ),
+        model_candidates="openrouter/deepseek/deepseek-v3.2",
+        extra_env={
+            "OPENROUTER_API_KEY": "fake-openrouter-key",
+            "OPENCODE_POOL_MAX_CYCLES": "0",
+        },
+    )
+    elapsed = time.monotonic() - start
+
+    assert result.returncode == 1
+    assert "provider credits are exhausted" in result.stdout
+    assert "marking this candidate failed for the rest of the run" in result.stdout
+    assert "Every OpenCode model candidate is marked failed for this run" in result.stdout
+    assert "class=credit-exhausted" in result.stdout
+    assert "Restarting OpenCode model pool" not in result.stdout
+    assert elapsed < 20
+
+
+def test_invalid_control_output_cap_marks_candidate_failed(tmp_path: Path) -> None:
+    """Repeated control-rejected output stops retrying at the cap, not the budget."""
+    result = run_failed_model(
+        tmp_path,
+        json_line='{"type":"step_start","sessionID":"session-1"}',
+        extra_env={
+            "FAKE_OPENCODE_RUN_EXIT": "0",
+            "FAKE_OPENCODE_EXPORT": json.dumps(
+                {
+                    "messages": [
+                        {
+                            "info": {"role": "assistant"},
+                            "parts": [
+                                {"type": "text", "text": "not a control conclusion"}
+                            ],
+                        }
+                    ]
+                }
+            ),
+            "OPENCODE_MODEL_ATTEMPTS": "3",
+            "OPENCODE_INVALID_CONTROL_OUTPUT_CAP": "2",
+            "OPENCODE_BACKOFF_INITIAL_SECONDS": "1",
+            "OPENCODE_POOL_MAX_CYCLES": "0",
+        },
+    )
+
+    assert result.returncode == 1
+    assert "produced 2 control-rejected outputs" in result.stdout
+    assert "marking this candidate failed for the rest of the run" in result.stdout
+    assert "Every OpenCode model candidate is marked failed for this run" in result.stdout
+    assert "attempt 3/3" not in result.stdout
+
+
+def test_attempt_ceiling_bounds_provider_spend(tmp_path: Path) -> None:
+    """The per-run attempt ceiling ends the pool before the retry budget elapses."""
+    result = run_failed_model(
+        tmp_path,
+        extra_env={
+            "OPENCODE_MODEL_ATTEMPTS": "3",
+            "OPENCODE_POOL_MAX_TOTAL_ATTEMPTS": "2",
+            "OPENCODE_BACKOFF_INITIAL_SECONDS": "1",
+            "OPENCODE_POOL_MAX_CYCLES": "0",
+        },
+    )
+
+    assert result.returncode == 1
+    assert (
+        "reached the per-run provider attempt ceiling of 2 attempts" in result.stdout
+    )
+    assert "attempt 3/3" not in result.stdout
+
+
 def test_dynamic_review_cadence_uses_small_change_timeout(tmp_path: Path) -> None:
     """Small PRs fail through hung/unavailable providers quickly with a visible budget reason."""
     result = run_failed_model(