From 64abdf12e7399c6d5462db32e1b63b9673595759 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 1 Aug 2026 19:38:02 +0000 Subject: [PATCH 1/4] Initial plan From 3517516c435bcfdf50865b09f1eba282a2cd56a9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 1 Aug 2026 20:06:35 +0000 Subject: [PATCH 2/4] Fix credit-ceiling handling: exit 0 on AI credits limit, remove setFailed from daily AIC guardrail Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .../daily-max-ai-credits-test.lock.yml | 11 ++- .../workflows/daily-max-ai-credits-test.md | 25 ++--- .../js/check_daily_aic_workflow_guardrail.cjs | 10 +- ...heck_daily_aic_workflow_guardrail.test.cjs | 5 +- actions/setup/js/copilot_harness.cjs | 9 ++ actions/setup/js/copilot_harness.test.cjs | 98 +++++++++++++++++++ 6 files changed, 136 insertions(+), 22 deletions(-) diff --git a/.github/workflows/daily-max-ai-credits-test.lock.yml b/.github/workflows/daily-max-ai-credits-test.lock.yml index f23214639dd..1aeaddd26ed 100644 --- a/.github/workflows/daily-max-ai-credits-test.lock.yml +++ b/.github/workflows/daily-max-ai-credits-test.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"6e1a9f597b80403c8d2f755995a0036cb59c8484f0b5b8583321c816f929e95c","body_hash":"d0cbd35665fdbbca4938280c94f3e789f67499488e8e9d803676324167d120a6","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.77"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"952ff465f595e8e7b3953f3ae306b99fcc352c2288fca41c0caef40162a55cfe","body_hash":"3e8c572c34b4fcd95bd5c1baf708429d340413b60d9913c489ba03978feeefd8","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.77"}} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.43","digest":"sha256:04e2d1987a565000a8f114b89d806ae7a3864dd4f944be65275b28c93d8690e6","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.43@sha256:04e2d1987a565000a8f114b89d806ae7a3864dd4f944be65275b28c93d8690e6"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.43","digest":"sha256:d85f57975af5ea23af4996e41ed73fbc8f5b4a47402472bfe82e508f352cb0c1","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.43@sha256:d85f57975af5ea23af4996e41ed73fbc8f5b4a47402472bfe82e508f352cb0c1"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.43","digest":"sha256:26be5e0b8c8f4c41c8a59126b29bb5d80b07253597472ded2a16bdd75abcbf9d","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.43@sha256:26be5e0b8c8f4c41c8a59126b29bb5d80b07253597472ded2a16bdd75abcbf9d"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.7","digest":"sha256:7545220a9aca134b71e51193ee0eaf4c50756ebf8fbd25a63ae7556e62815c00","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.7@sha256:7545220a9aca134b71e51193ee0eaf4c50756ebf8fbd25a63ae7556e62815c00"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748","pinned_image":"ghcr.io/github/gh-aw-node@sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748"},{"image":"ghcr.io/github/github-mcp-server:v1.8.0","digest":"sha256:d5a18c04b92714c309eb46a2305087e91a4dbd80420f6e462656699f95093520","pinned_image":"ghcr.io/github/github-mcp-server:v1.8.0@sha256:d5a18c04b92714c309eb46a2305087e91a4dbd80420f6e462656699f95093520"}]} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -23,7 +23,7 @@ # # For more information: https://github.github.com/gh-aw/introduction/overview/ # -# โš ๏ธ INTENTIONALLY FAILS โ€” Tests that max-ai-credits: 1 is enforced by the AWF firewall and that the per-run budget guardrail cuts off the agent. +# Tests that max-ai-credits: 1 is enforced by the AWF firewall and that the per-run budget guardrail cuts off the agent. Concludes success when the credit limit is reached. # # Secrets used: # - COPILOT_GITHUB_TOKEN @@ -1140,9 +1140,10 @@ jobs: GH_AW_LOCKDOWN_CHECK_FAILED: ${{ needs.activation.outputs.lockdown_check_failed }} GH_AW_OAUTH_TOKEN_CHECK_FAILED: ${{ needs.activation.outputs.oauth_token_check_failed }} GH_AW_STALE_LOCK_FILE_FAILED: ${{ needs.activation.outputs.stale_lock_file_failed }} - GH_AW_SAFE_OUTPUT_MESSAGES: "{\"runStarted\":\"๐Ÿงช [{workflow_name}]({run_url}) โ€” per-run AI credit limit test running (intentionally fails, limit: 1 AI credit/run).\",\"runSuccess\":\"โš ๏ธ [{workflow_name}]({run_url}) completed without hitting the per-run limit of 1 AI credit โ€” verify that max-ai-credits enforcement is working.\",\"runFailure\":\"๐Ÿšซ [{workflow_name}]({run_url}) {status} โ€” expected: the per-run AI credit limit of 1 was reached and the AWF firewall cut off the agent.\"}" + GH_AW_SAFE_OUTPUT_MESSAGES: "{\"runStarted\":\"๐Ÿงช [{workflow_name}]({run_url}) โ€” per-run AI credit limit test running (limit: 1 AI credit/run).\",\"runSuccess\":\"๐Ÿงช [{workflow_name}]({run_url}) โ€” expected: the per-run AI credit limit of 1 was reached and the AWF firewall cut off the agent.\",\"runFailure\":\"โš ๏ธ [{workflow_name}]({run_url}) {status} โ€” completed without hitting the per-run limit of 1 AI credit โ€” verify that max-ai-credits enforcement is working.\"}" GH_AW_GROUP_REPORTS: "false" GH_AW_FAILURE_REPORT_AS_ISSUE: "true" + GH_AW_FAILURE_EXCLUDED_CATEGORIES_FILTER: "[\"ai_credits_rate_limit_error\",\"max_ai_credits_exceeded\"]" GH_AW_MISSING_TOOL_REPORT_AS_FAILURE: "true" GH_AW_MISSING_DATA_REPORT_AS_FAILURE: "true" GH_AW_TIMEOUT_MINUTES: "5" @@ -1262,7 +1263,7 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: WORKFLOW_NAME: "Daily Max Ai Credits Test" - WORKFLOW_DESCRIPTION: "โš ๏ธ INTENTIONALLY FAILS โ€” Tests that max-ai-credits: 1 is enforced by the AWF firewall and that the per-run budget guardrail cuts off the agent." + WORKFLOW_DESCRIPTION: "Tests that max-ai-credits: 1 is enforced by the AWF firewall and that the per-run budget guardrail cuts off the agent. Concludes success when the credit limit is reached." HAS_PATCH: ${{ needs.agent.outputs.has_patch }} GH_AW_DETECTION_CONTINUE_ON_ERROR: "true" with: @@ -1401,7 +1402,7 @@ jobs: GH_AW_ENGINE_MODEL: ${{ needs.agent.outputs.model }} GH_AW_PROJECT_UTC: "-08:00" GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} - GH_AW_SAFE_OUTPUT_MESSAGES: "{\"runStarted\":\"๐Ÿงช [{workflow_name}]({run_url}) โ€” per-run AI credit limit test running (intentionally fails, limit: 1 AI credit/run).\",\"runSuccess\":\"โš ๏ธ [{workflow_name}]({run_url}) completed without hitting the per-run limit of 1 AI credit โ€” verify that max-ai-credits enforcement is working.\",\"runFailure\":\"๐Ÿšซ [{workflow_name}]({run_url}) {status} โ€” expected: the per-run AI credit limit of 1 was reached and the AWF firewall cut off the agent.\"}" + GH_AW_SAFE_OUTPUT_MESSAGES: "{\"runStarted\":\"๐Ÿงช [{workflow_name}]({run_url}) โ€” per-run AI credit limit test running (limit: 1 AI credit/run).\",\"runSuccess\":\"๐Ÿงช [{workflow_name}]({run_url}) โ€” expected: the per-run AI credit limit of 1 was reached and the AWF firewall cut off the agent.\",\"runFailure\":\"โš ๏ธ [{workflow_name}]({run_url}) {status} โ€” completed without hitting the per-run limit of 1 AI credit โ€” verify that max-ai-credits enforcement is working.\"}" GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }} GH_AW_WORKFLOW_EMOJI: "๐Ÿงช" GH_AW_WORKFLOW_ID: "daily-max-ai-credits-test" diff --git a/.github/workflows/daily-max-ai-credits-test.md b/.github/workflows/daily-max-ai-credits-test.md index 6d5f141765c..a11a2b3e44e 100644 --- a/.github/workflows/daily-max-ai-credits-test.md +++ b/.github/workflows/daily-max-ai-credits-test.md @@ -1,7 +1,7 @@ --- private: true emoji: "๐Ÿงช" -description: "โš ๏ธ INTENTIONALLY FAILS โ€” Tests that max-ai-credits: 1 is enforced by the AWF firewall and that the per-run budget guardrail cuts off the agent." +description: "Tests that max-ai-credits: 1 is enforced by the AWF firewall and that the per-run budget guardrail cuts off the agent. Concludes success when the credit limit is reached." features: intentional-failure: true gh-aw-detection: true @@ -30,32 +30,35 @@ safe-outputs: close-older-key: "daily-max-ai-credits-test" labels: [automation, testing] max: 1 + report-failure-as-issue: + - "!ai_credits_rate_limit_error" + - "!max_ai_credits_exceeded" messages: - run-started: "๐Ÿงช [{workflow_name}]({run_url}) โ€” per-run AI credit limit test running (intentionally fails, limit: 1 AI credit/run)." - run-success: "โš ๏ธ [{workflow_name}]({run_url}) completed without hitting the per-run limit of 1 AI credit โ€” verify that max-ai-credits enforcement is working." - run-failure: "๐Ÿšซ [{workflow_name}]({run_url}) {status} โ€” expected: the per-run AI credit limit of 1 was reached and the AWF firewall cut off the agent." + run-started: "๐Ÿงช [{workflow_name}]({run_url}) โ€” per-run AI credit limit test running (limit: 1 AI credit/run)." + run-success: "๐Ÿงช [{workflow_name}]({run_url}) โ€” expected: the per-run AI credit limit of 1 was reached and the AWF firewall cut off the agent." + run-failure: "โš ๏ธ [{workflow_name}]({run_url}) {status} โ€” completed without hitting the per-run limit of 1 AI credit โ€” verify that max-ai-credits enforcement is working." --- -### Daily Max AI Credits Test (Intentionally Fails) +### Daily Max AI Credits Test **Report Formatting**: Use h3 (###) or lower for all headers in your report to maintain proper document hierarchy. Wrap long sections in `
View Full Details` tags to improve readability. -> โš ๏ธ **This workflow is intentionally broken.** It exists solely to verify that +> ๐Ÿงช **This workflow tests the per-run AI credits guardrail.** It verifies that > `max-ai-credits: 1` is enforced by the AWF firewall and that the agent is -> cut off when the per-run budget is exhausted. -> **Do not fix this workflow.** +> cut off when the per-run budget is exhausted. The run **concludes success** +> when the credit limit is reached as expected. #### What This Tests 1. The AWF firewall enforces the `max-ai-credits` per-run budget. 2. Once the agent consumes more than 1 AI credit in a single run, the firewall cuts off the LLM API. 3. The prompt forces multiple turns and multiple large-file reads so the run reliably burns credits. -4. The run is expected to be cut off by the per-run budget before all jobs can complete. +4. The run is expected to be cut off by the per-run budget before all turns complete. -#### Task (broken by design) +#### Task Use **at least four separate assistant turns**. Do not combine all work into one response. @@ -70,7 +73,7 @@ Turn 4: Call `noop` with the message: "Completed max-ai-credits multi-turn guard After each job, briefly summarize what was read, then continue to the next turn. Since the per-run budget is `max-ai-credits: 1`, the AWF firewall should cut off the agent -before all turns complete. That failure is the expected and correct outcome. +before all turns complete. That is the expected and correct outcome โ€” the run will conclude success. If the workflow somehow completes without hitting the per-run limit, call `noop` with the message: "Per-run credit limit not exceeded โ€” verify that max-ai-credits: 1 is enforced by the AWF firewall." diff --git a/actions/setup/js/check_daily_aic_workflow_guardrail.cjs b/actions/setup/js/check_daily_aic_workflow_guardrail.cjs index 6ad7ed06430..6b3529af5a1 100644 --- a/actions/setup/js/check_daily_aic_workflow_guardrail.cjs +++ b/actions/setup/js/check_daily_aic_workflow_guardrail.cjs @@ -703,12 +703,16 @@ async function main() { } catch (summaryError) { core.warning(`Failed to write daily AIC summary: ${getErrorMessage(summaryError)}`); } - core.warning(`Daily workflow AIC guardrail exceeded for ${workflowName}: ${totalAIC}/${threshold}.`); - core.setFailed(`Daily workflow AIC guardrail exceeded for ${workflowName}: ${totalAIC}/${threshold}.`); + // Log as info so the activation job succeeds. The daily_ai_credits_exceeded output + // is already set to "true"; the agent job's condition (daily_ai_credits_exceeded != 'true') + // will skip the agent, and the conclusion job will handle reporting via the + // daily_ai_credits_exceeded flag. Failing the activation job here causes the overall + // workflow to fail even though hitting the daily limit is an expected, graceful outcome. + core.info(`Daily workflow AIC guardrail exceeded for ${workflowName}: ${totalAIC}/${threshold}.`); } catch (error) { // Treat unexpected guardrail execution errors as non-blocking skips so transient // API/runtime issues do not fail activation. The output stays at the default "false", - // allowing the agent to run. Legitimate threshold exceedance still fails via setFailed. + // allowing the agent to run. core.warning(`Daily workflow AI Credits guardrail encountered an unexpected error and will be skipped: ${getErrorMessage(error)}`); } } diff --git a/actions/setup/js/check_daily_aic_workflow_guardrail.test.cjs b/actions/setup/js/check_daily_aic_workflow_guardrail.test.cjs index a47c37aa97b..7a670d1365e 100644 --- a/actions/setup/js/check_daily_aic_workflow_guardrail.test.cjs +++ b/actions/setup/js/check_daily_aic_workflow_guardrail.test.cjs @@ -478,7 +478,7 @@ describe("check_daily_aic_workflow_guardrail", () => { } }); - it("main() marks the step failed when the daily AI Credits guardrail is exceeded", async () => { + it("main() does not mark the step failed when the daily AI Credits guardrail is exceeded", async () => { const getRunAICSpy = vi.spyOn(exports, "getRunAIC").mockResolvedValue(200); const coreOutputs = {}; @@ -558,8 +558,7 @@ describe("check_daily_aic_workflow_guardrail", () => { expect(coreOutputs["daily_ai_credits_exceeded"]).toBe("true"); expect(coreOutputs["daily_ai_credits_total_effective_tokens"]).toBe("200"); expect(coreOutputs["daily_ai_credits_threshold"]).toBe("100"); - expect(setFailed).toHaveBeenCalledTimes(1); - expect(setFailed.mock.calls[0][0]).toMatch(/guardrail exceeded/i); + expect(setFailed).not.toHaveBeenCalled(); } finally { delete global.core; delete global.github; diff --git a/actions/setup/js/copilot_harness.cjs b/actions/setup/js/copilot_harness.cjs index 100557ed9dd..b4e63a0af9f 100644 --- a/actions/setup/js/copilot_harness.cjs +++ b/actions/setup/js/copilot_harness.cjs @@ -1286,6 +1286,15 @@ async function main() { reasons.push("LLM invocation cap saturated โ€” the pooled per-run budget is fully exhausted; retries cannot make progress"); } log(`attempt ${attempt + 1}: ${reasons.join(" and ")} โ€” not retrying (non-retryable guard condition)`); + // When the per-run AI credits budget is exceeded the AWF firewall intentionally + // stopped the agent โ€” this is controlled budget enforcement, not an unexpected + // error. Exit 0 so the agent step and job succeed; the ai_credits_rate_limit_error + // output surfaced by parse-mcp-gateway will inform downstream handlers (e.g. + // handle_agent_failure) of the budget exceedance. + if (nonRetryableGuard.aiCreditsExceeded) { + log(`attempt ${attempt + 1}: AI credits budget enforced โ€” exiting 0 (budget control, not an error)`); + lastExitCode = 0; + } break; } diff --git a/actions/setup/js/copilot_harness.test.cjs b/actions/setup/js/copilot_harness.test.cjs index 80c1c4d65a0..4d5dbc78cf2 100644 --- a/actions/setup/js/copilot_harness.test.cjs +++ b/actions/setup/js/copilot_harness.test.cjs @@ -2622,6 +2622,104 @@ setInterval(() => {}, 1000);`, }); }); + describe("AI credits budget enforcement exits 0", () => { + it("exits 0 when the agent outputs max_ai_credits_exceeded and the CLI exits non-zero", () => { + const tempDir = makeHarnessTempDir("copilot-ai-credits-exceeded-"); + const safeOutputsPath = path.join(tempDir, "safe-outputs.jsonl"); + const stubPath = path.join(tempDir, "stub.cjs"); + const promptPath = path.join(tempDir, "prompt.txt"); + const callsPath = path.join(tempDir, "calls.jsonl"); + // Stub emits the AI-credits-exceeded marker on stdout (as the AWF firewall would) + // then exits non-zero. The harness must detect this, set lastExitCode=0, and exit 0. + fs.writeFileSync( + stubPath, + `const fs = require("fs"); +const callsPath = process.env.COPILOT_HARNESS_STUB_CALLS; +fs.appendFileSync(callsPath, JSON.stringify({args: process.argv.slice(2)}) + "\\n"); +process.stdout.write("error: max_ai_credits_exceeded=true\\n"); +process.exit(1);`, + "utf8" + ); + fs.writeFileSync(promptPath, "do some work", "utf8"); + + const result = spawnSync(process.execPath, ["copilot_harness.cjs", process.execPath, stubPath, "--prompt-file", promptPath], { + cwd: path.dirname(require.resolve("./copilot_harness.cjs")), + env: { ...process.env, COPILOT_HARNESS_STUB_CALLS: callsPath, GH_AW_SAFE_OUTPUTS: safeOutputsPath }, + encoding: "utf8", + timeout: 10000, + }); + const callCount = fs.readFileSync(callsPath, "utf8").trim().split("\n").filter(Boolean).length; + // Only one attempt โ€” credit limit is non-retryable + expect(callCount).toBe(1); + // Harness exits 0: budget enforcement is intentional, not a job failure + expect(result.status).toBe(0); + expect(result.stderr).toContain("AI credits budget exceeded"); + expect(result.stderr).toContain("AI credits budget enforced"); + }); + + it("exits 0 when the agent outputs ai_credits_rate_limit_error and the CLI exits non-zero", () => { + const tempDir = makeHarnessTempDir("copilot-ai-credits-rate-limit-"); + const safeOutputsPath = path.join(tempDir, "safe-outputs.jsonl"); + const stubPath = path.join(tempDir, "stub.cjs"); + const promptPath = path.join(tempDir, "prompt.txt"); + const callsPath = path.join(tempDir, "calls.jsonl"); + fs.writeFileSync( + stubPath, + `const fs = require("fs"); +const callsPath = process.env.COPILOT_HARNESS_STUB_CALLS; +fs.appendFileSync(callsPath, JSON.stringify({args: process.argv.slice(2)}) + "\\n"); +process.stdout.write("error: ai_credits_rate_limit_error=true\\n"); +process.exit(1);`, + "utf8" + ); + fs.writeFileSync(promptPath, "do some work", "utf8"); + + const result = spawnSync(process.execPath, ["copilot_harness.cjs", process.execPath, stubPath, "--prompt-file", promptPath], { + cwd: path.dirname(require.resolve("./copilot_harness.cjs")), + env: { ...process.env, COPILOT_HARNESS_STUB_CALLS: callsPath, GH_AW_SAFE_OUTPUTS: safeOutputsPath }, + encoding: "utf8", + timeout: 10000, + }); + const callCount = fs.readFileSync(callsPath, "utf8").trim().split("\n").filter(Boolean).length; + expect(callCount).toBe(1); + expect(result.status).toBe(0); + expect(result.stderr).toContain("AI credits budget enforced"); + }); + + it("still exits 1 for non-credit-limit failures (auth error)", () => { + const tempDir = makeHarnessTempDir("copilot-auth-failure-"); + const safeOutputsPath = path.join(tempDir, "safe-outputs.jsonl"); + const stubPath = path.join(tempDir, "stub.cjs"); + const promptPath = path.join(tempDir, "prompt.txt"); + const callsPath = path.join(tempDir, "calls.jsonl"); + fs.writeFileSync( + stubPath, + `const fs = require("fs"); +const callsPath = process.env.COPILOT_HARNESS_STUB_CALLS; +fs.appendFileSync(callsPath, JSON.stringify({args: process.argv.slice(2)}) + "\\n"); +process.stdout.write("CAPIError: 401 Unauthorized\\n"); +process.exit(1);`, + "utf8" + ); + fs.writeFileSync(promptPath, "do some work", "utf8"); + + const result = spawnSync(process.execPath, ["copilot_harness.cjs", process.execPath, stubPath, "--prompt-file", promptPath], { + cwd: path.dirname(require.resolve("./copilot_harness.cjs")), + env: { + ...process.env, + COPILOT_HARNESS_STUB_CALLS: callsPath, + GH_AW_SAFE_OUTPUTS: safeOutputsPath, + GH_AW_HARNESS_MAX_RETRIES: "0", + }, + encoding: "utf8", + timeout: 10000, + }); + // Harness exits 1: normal non-credit failures still fail the job + expect(result.status).toBe(1); + expect(result.stderr).not.toContain("AI credits budget enforced"); + }); + }); + describe("applyCopilotWireAPI", () => { afterEach(() => { delete process.env.COPILOT_MODEL; From e8e7d3d76a8203260be4b39a57f94a8247eebbca Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 1 Aug 2026 20:34:35 +0000 Subject: [PATCH 3/4] refactor: apply AI credits exit-0 fix to claude and codex harnesses Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- actions/setup/js/claude_harness.cjs | 9 ++ actions/setup/js/claude_harness.test.cjs | 98 ++++++++++++++++++++ actions/setup/js/codex_harness.cjs | 9 ++ actions/setup/js/codex_harness.test.cjs | 109 +++++++++++++++++++++++ 4 files changed, 225 insertions(+) diff --git a/actions/setup/js/claude_harness.cjs b/actions/setup/js/claude_harness.cjs index 33be2194615..401afefd1d0 100644 --- a/actions/setup/js/claude_harness.cjs +++ b/actions/setup/js/claude_harness.cjs @@ -485,6 +485,15 @@ async function main() { if (nonRetryableGuard.awfAPIProxyBlockingRequests) reasons.push("AWF API proxy is blocking requests"); if (nonRetryableGuard.maxRunsExceeded) reasons.push("maximum LLM invocations exceeded"); log(`attempt ${attempt + 1}: ${reasons.join(" and ")} โ€” not retrying (non-retryable guard condition)`); + // When the per-run AI credits budget is exceeded the AWF firewall intentionally + // stopped the agent โ€” this is controlled budget enforcement, not an unexpected + // error. Exit 0 so the agent step and job succeed; the ai_credits_rate_limit_error + // output surfaced by parse-mcp-gateway will inform downstream handlers (e.g. + // handle_agent_failure) of the budget exceedance. + if (nonRetryableGuard.aiCreditsExceeded) { + log(`attempt ${attempt + 1}: AI credits budget enforced โ€” exiting 0 (budget control, not an error)`); + lastExitCode = 0; + } break; } diff --git a/actions/setup/js/claude_harness.test.cjs b/actions/setup/js/claude_harness.test.cjs index 03893ba3f46..1ee34adadb5 100644 --- a/actions/setup/js/claude_harness.test.cjs +++ b/actions/setup/js/claude_harness.test.cjs @@ -706,4 +706,102 @@ process.exit(1);`, expect(result.stderr).toContain("noop message found in safe-outputs โ€” not retrying"); }); }); + + describe("AI credits budget enforcement exits 0", () => { + it("exits 0 when the agent outputs max_ai_credits_exceeded and the CLI exits non-zero", () => { + const tempDir = makeHarnessTempDir("claude-ai-credits-exceeded-"); + const safeOutputsPath = path.join(tempDir, "safe-outputs.jsonl"); + const stubPath = path.join(tempDir, "stub.cjs"); + const promptPath = path.join(tempDir, "prompt.txt"); + const callsPath = path.join(tempDir, "calls.jsonl"); + // Stub emits the AI-credits-exceeded marker on stdout (as the AWF firewall would) + // then exits non-zero. The harness must detect this, set lastExitCode=0, and exit 0. + fs.writeFileSync( + stubPath, + `const fs = require("fs"); +const callsPath = process.env.CLAUDE_HARNESS_STUB_CALLS; +fs.appendFileSync(callsPath, JSON.stringify({args: process.argv.slice(2)}) + "\\n"); +process.stdout.write("error: max_ai_credits_exceeded=true\\n"); +process.exit(1);`, + "utf8" + ); + fs.writeFileSync(promptPath, "do some work", "utf8"); + + const result = spawnSync(process.execPath, ["claude_harness.cjs", process.execPath, stubPath, "--print", "--prompt-file", promptPath], { + cwd: path.dirname(require.resolve("./claude_harness.cjs")), + env: { ...process.env, CLAUDE_HARNESS_STUB_CALLS: callsPath, GH_AW_SAFE_OUTPUTS: safeOutputsPath }, + encoding: "utf8", + timeout: 10000, + }); + const callCount = fs.readFileSync(callsPath, "utf8").trim().split("\n").filter(Boolean).length; + // Only one attempt โ€” credit limit is non-retryable + expect(callCount).toBe(1); + // Harness exits 0: budget enforcement is intentional, not a job failure + expect(result.status).toBe(0); + expect(result.stderr).toContain("AI credits budget exceeded"); + expect(result.stderr).toContain("AI credits budget enforced"); + }); + + it("exits 0 when the agent outputs ai_credits_rate_limit_error and the CLI exits non-zero", () => { + const tempDir = makeHarnessTempDir("claude-ai-credits-rate-limit-"); + const safeOutputsPath = path.join(tempDir, "safe-outputs.jsonl"); + const stubPath = path.join(tempDir, "stub.cjs"); + const promptPath = path.join(tempDir, "prompt.txt"); + const callsPath = path.join(tempDir, "calls.jsonl"); + fs.writeFileSync( + stubPath, + `const fs = require("fs"); +const callsPath = process.env.CLAUDE_HARNESS_STUB_CALLS; +fs.appendFileSync(callsPath, JSON.stringify({args: process.argv.slice(2)}) + "\\n"); +process.stdout.write("error: ai_credits_rate_limit_error=true\\n"); +process.exit(1);`, + "utf8" + ); + fs.writeFileSync(promptPath, "do some work", "utf8"); + + const result = spawnSync(process.execPath, ["claude_harness.cjs", process.execPath, stubPath, "--print", "--prompt-file", promptPath], { + cwd: path.dirname(require.resolve("./claude_harness.cjs")), + env: { ...process.env, CLAUDE_HARNESS_STUB_CALLS: callsPath, GH_AW_SAFE_OUTPUTS: safeOutputsPath }, + encoding: "utf8", + timeout: 10000, + }); + const callCount = fs.readFileSync(callsPath, "utf8").trim().split("\n").filter(Boolean).length; + expect(callCount).toBe(1); + expect(result.status).toBe(0); + expect(result.stderr).toContain("AI credits budget enforced"); + }); + + it("still exits 1 for non-credit-limit failures (auth error)", () => { + const tempDir = makeHarnessTempDir("claude-auth-failure-"); + const safeOutputsPath = path.join(tempDir, "safe-outputs.jsonl"); + const stubPath = path.join(tempDir, "stub.cjs"); + const promptPath = path.join(tempDir, "prompt.txt"); + const callsPath = path.join(tempDir, "calls.jsonl"); + fs.writeFileSync( + stubPath, + `const fs = require("fs"); +const callsPath = process.env.CLAUDE_HARNESS_STUB_CALLS; +fs.appendFileSync(callsPath, JSON.stringify({args: process.argv.slice(2)}) + "\\n"); +process.stdout.write("Authentication failed (Request ID: 123)\\n"); +process.exit(1);`, + "utf8" + ); + fs.writeFileSync(promptPath, "do some work", "utf8"); + + const result = spawnSync(process.execPath, ["claude_harness.cjs", process.execPath, stubPath, "--print", "--prompt-file", promptPath], { + cwd: path.dirname(require.resolve("./claude_harness.cjs")), + env: { + ...process.env, + CLAUDE_HARNESS_STUB_CALLS: callsPath, + GH_AW_SAFE_OUTPUTS: safeOutputsPath, + GH_AW_HARNESS_MAX_RETRIES: "0", + }, + encoding: "utf8", + timeout: 10000, + }); + // Harness exits 1: normal non-credit failures still fail the job + expect(result.status).toBe(1); + expect(result.stderr).not.toContain("AI credits budget enforced"); + }); + }); }); diff --git a/actions/setup/js/codex_harness.cjs b/actions/setup/js/codex_harness.cjs index 11aa5fb5c56..99ce9bb0cc8 100644 --- a/actions/setup/js/codex_harness.cjs +++ b/actions/setup/js/codex_harness.cjs @@ -668,6 +668,15 @@ async function main() { if (nonRetryableGuard.goalAlreadyActive) reasons.push("goal is already active for this thread (use update_goal when the current goal is complete)"); if (nonRetryableGuard.maxRunsExceeded) reasons.push("maximum LLM invocations exceeded"); log(`attempt ${attempt + 1}: ${reasons.join(" and ")} โ€” not retrying (non-retryable guard condition)`); + // When the per-run AI credits budget is exceeded the AWF firewall intentionally + // stopped the agent โ€” this is controlled budget enforcement, not an unexpected + // error. Exit 0 so the agent step and job succeed; the ai_credits_rate_limit_error + // output surfaced by parse-mcp-gateway will inform downstream handlers (e.g. + // handle_agent_failure) of the budget exceedance. + if (nonRetryableGuard.aiCreditsExceeded) { + log(`attempt ${attempt + 1}: AI credits budget enforced โ€” exiting 0 (budget control, not an error)`); + lastExitCode = 0; + } break; } diff --git a/actions/setup/js/codex_harness.test.cjs b/actions/setup/js/codex_harness.test.cjs index 3be0789cc44..02f6e781f34 100644 --- a/actions/setup/js/codex_harness.test.cjs +++ b/actions/setup/js/codex_harness.test.cjs @@ -945,4 +945,113 @@ process.exit(1);`, expect(logs.some(msg => msg.includes("clamping max delay"))).toBe(true); }); }); + + describe("AI credits budget enforcement exits 0", () => { + it("exits 0 when the agent outputs max_ai_credits_exceeded and the CLI exits non-zero", () => { + const tempDir = makeHarnessTempDir("codex-ai-credits-exceeded-"); + const safeOutputsPath = path.join(tempDir, "safe-outputs.jsonl"); + const stubPath = path.join(tempDir, "stub.cjs"); + const promptPath = path.join(tempDir, "prompt.txt"); + const callsPath = path.join(tempDir, "calls.jsonl"); + // Stub emits the AI-credits-exceeded marker on stdout (as the AWF firewall would) + // then exits non-zero. The harness must detect this, set lastExitCode=0, and exit 0. + fs.writeFileSync( + stubPath, + `const fs = require("fs"); +const callsPath = process.env.CODEX_HARNESS_STUB_CALLS; +fs.appendFileSync(callsPath, JSON.stringify({args: process.argv.slice(2)}) + "\\n"); +process.stdout.write("error: max_ai_credits_exceeded=true\\n"); +process.exit(1);`, + "utf8" + ); + fs.writeFileSync(promptPath, "do some work", "utf8"); + + const result = spawnSync(process.execPath, ["codex_harness.cjs", process.execPath, stubPath, "exec", "--prompt-file", promptPath], { + cwd: path.dirname(require.resolve("./codex_harness.cjs")), + env: { + ...process.env, + CODEX_HARNESS_STUB_CALLS: callsPath, + GH_AW_SAFE_OUTPUTS: safeOutputsPath, + CODEX_API_KEY: "fake-key-for-test", + }, + encoding: "utf8", + timeout: 10000, + }); + const callCount = fs.readFileSync(callsPath, "utf8").trim().split("\n").filter(Boolean).length; + // Only one attempt โ€” credit limit is non-retryable + expect(callCount).toBe(1); + // Harness exits 0: budget enforcement is intentional, not a job failure + expect(result.status).toBe(0); + expect(result.stderr).toContain("AI credits budget exceeded"); + expect(result.stderr).toContain("AI credits budget enforced"); + }); + + it("exits 0 when the agent outputs ai_credits_rate_limit_error and the CLI exits non-zero", () => { + const tempDir = makeHarnessTempDir("codex-ai-credits-rate-limit-"); + const safeOutputsPath = path.join(tempDir, "safe-outputs.jsonl"); + const stubPath = path.join(tempDir, "stub.cjs"); + const promptPath = path.join(tempDir, "prompt.txt"); + const callsPath = path.join(tempDir, "calls.jsonl"); + fs.writeFileSync( + stubPath, + `const fs = require("fs"); +const callsPath = process.env.CODEX_HARNESS_STUB_CALLS; +fs.appendFileSync(callsPath, JSON.stringify({args: process.argv.slice(2)}) + "\\n"); +process.stdout.write("error: ai_credits_rate_limit_error=true\\n"); +process.exit(1);`, + "utf8" + ); + fs.writeFileSync(promptPath, "do some work", "utf8"); + + const result = spawnSync(process.execPath, ["codex_harness.cjs", process.execPath, stubPath, "exec", "--prompt-file", promptPath], { + cwd: path.dirname(require.resolve("./codex_harness.cjs")), + env: { + ...process.env, + CODEX_HARNESS_STUB_CALLS: callsPath, + GH_AW_SAFE_OUTPUTS: safeOutputsPath, + CODEX_API_KEY: "fake-key-for-test", + }, + encoding: "utf8", + timeout: 10000, + }); + const callCount = fs.readFileSync(callsPath, "utf8").trim().split("\n").filter(Boolean).length; + expect(callCount).toBe(1); + expect(result.status).toBe(0); + expect(result.stderr).toContain("AI credits budget enforced"); + }); + + it("still exits 1 for non-credit-limit failures (auth error)", () => { + const tempDir = makeHarnessTempDir("codex-auth-failure-"); + const safeOutputsPath = path.join(tempDir, "safe-outputs.jsonl"); + const stubPath = path.join(tempDir, "stub.cjs"); + const promptPath = path.join(tempDir, "prompt.txt"); + const callsPath = path.join(tempDir, "calls.jsonl"); + fs.writeFileSync( + stubPath, + `const fs = require("fs"); +const callsPath = process.env.CODEX_HARNESS_STUB_CALLS; +fs.appendFileSync(callsPath, JSON.stringify({args: process.argv.slice(2)}) + "\\n"); +process.stdout.write("Authentication failed (Request ID: 123)\\n"); +process.exit(1);`, + "utf8" + ); + fs.writeFileSync(promptPath, "do some work", "utf8"); + + const result = spawnSync(process.execPath, ["codex_harness.cjs", process.execPath, stubPath, "exec", "--prompt-file", promptPath], { + cwd: path.dirname(require.resolve("./codex_harness.cjs")), + env: { + ...process.env, + CODEX_HARNESS_STUB_CALLS: callsPath, + GH_AW_SAFE_OUTPUTS: safeOutputsPath, + CODEX_API_KEY: "fake-key-for-test", + GH_AW_HARNESS_MAX_RETRIES: "0", + }, + encoding: "utf8", + timeout: 10000, + }); + // Harness exits 1: normal non-credit failures still fail the job + expect(result.status).toBe(1); + expect(result.stderr).not.toContain("AI credits budget enforced"); + }); + }); }); From 6ed95549813e22edd38606199339c50a79ed5e0e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 1 Aug 2026 23:16:01 +0000 Subject: [PATCH 4/4] fix: require trusted AI-credit signal before harness exit-0 Co-authored-by: gh-aw-bot <259018956+gh-aw-bot@users.noreply.github.com> --- .../daily-max-ai-credits-test.lock.yml | 4 +- .../workflows/daily-max-ai-credits-test.md | 1 - actions/setup/js/claude_harness.cjs | 12 +++- actions/setup/js/claude_harness.test.cjs | 54 +++++++++++++++++- actions/setup/js/codex_harness.cjs | 12 +++- actions/setup/js/codex_harness.test.cjs | 53 +++++++++++++++++- actions/setup/js/copilot_harness.cjs | 12 +++- actions/setup/js/copilot_harness.test.cjs | 56 +++++++++++++++++-- 8 files changed, 184 insertions(+), 20 deletions(-) diff --git a/.github/workflows/daily-max-ai-credits-test.lock.yml b/.github/workflows/daily-max-ai-credits-test.lock.yml index 1aeaddd26ed..3d59671ba20 100644 --- a/.github/workflows/daily-max-ai-credits-test.lock.yml +++ b/.github/workflows/daily-max-ai-credits-test.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"952ff465f595e8e7b3953f3ae306b99fcc352c2288fca41c0caef40162a55cfe","body_hash":"3e8c572c34b4fcd95bd5c1baf708429d340413b60d9913c489ba03978feeefd8","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.77"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"7e9da0fc0a98830f206b9d84cd52684303d7f1119eda2028dfd9a2ca6686afe2","body_hash":"3e8c572c34b4fcd95bd5c1baf708429d340413b60d9913c489ba03978feeefd8","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.77"}} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.43","digest":"sha256:04e2d1987a565000a8f114b89d806ae7a3864dd4f944be65275b28c93d8690e6","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.43@sha256:04e2d1987a565000a8f114b89d806ae7a3864dd4f944be65275b28c93d8690e6"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.43","digest":"sha256:d85f57975af5ea23af4996e41ed73fbc8f5b4a47402472bfe82e508f352cb0c1","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.43@sha256:d85f57975af5ea23af4996e41ed73fbc8f5b4a47402472bfe82e508f352cb0c1"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.43","digest":"sha256:26be5e0b8c8f4c41c8a59126b29bb5d80b07253597472ded2a16bdd75abcbf9d","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.43@sha256:26be5e0b8c8f4c41c8a59126b29bb5d80b07253597472ded2a16bdd75abcbf9d"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.7","digest":"sha256:7545220a9aca134b71e51193ee0eaf4c50756ebf8fbd25a63ae7556e62815c00","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.7@sha256:7545220a9aca134b71e51193ee0eaf4c50756ebf8fbd25a63ae7556e62815c00"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748","pinned_image":"ghcr.io/github/gh-aw-node@sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748"},{"image":"ghcr.io/github/github-mcp-server:v1.8.0","digest":"sha256:d5a18c04b92714c309eb46a2305087e91a4dbd80420f6e462656699f95093520","pinned_image":"ghcr.io/github/github-mcp-server:v1.8.0@sha256:d5a18c04b92714c309eb46a2305087e91a4dbd80420f6e462656699f95093520"}]} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -124,7 +124,7 @@ jobs: GH_AW_INFO_FIREWALL_TYPE: "squid" GH_AW_INFO_FRONTMATTER_EMOJI: "๐Ÿงช" GH_AW_COMPILED_STRICT: "true" - GH_AW_INFO_FEATURES: '{"gh-aw-detection":true,"intentional-failure":true}' + GH_AW_INFO_FEATURES: '{"gh-aw-detection":true}' uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | diff --git a/.github/workflows/daily-max-ai-credits-test.md b/.github/workflows/daily-max-ai-credits-test.md index a11a2b3e44e..e7008da2554 100644 --- a/.github/workflows/daily-max-ai-credits-test.md +++ b/.github/workflows/daily-max-ai-credits-test.md @@ -3,7 +3,6 @@ private: true emoji: "๐Ÿงช" description: "Tests that max-ai-credits: 1 is enforced by the AWF firewall and that the per-run budget guardrail cuts off the agent. Concludes success when the credit limit is reached." features: - intentional-failure: true gh-aw-detection: true on: schedule: daily around 10:30 diff --git a/actions/setup/js/claude_harness.cjs b/actions/setup/js/claude_harness.cjs index 401afefd1d0..66fc3786996 100644 --- a/actions/setup/js/claude_harness.cjs +++ b/actions/setup/js/claude_harness.cjs @@ -54,6 +54,7 @@ const { countPermissionDeniedIssues, hasNumerousPermissionDeniedIssues, extractD const { detectNonRetryableHarnessGuard, buildSoftTimeoutGuard, emitSoftTimeoutSignal, isAuthenticationFailedError } = require("./harness_retry_guard.cjs"); const { MODEL_NOT_SUPPORTED_PATTERN: INVALID_MODEL_ERROR_PATTERN } = require("./detect_agent_errors.cjs"); const { applyModelFallback } = require("./model_fallback.cjs"); +const { parseMaxAICreditsExceededFromAuditLog } = require("./ai_credits_context.cjs"); // Pattern to detect Anthropic API overload errors (HTTP 529). // Matches "overloaded_error" from the Anthropic error type field, and the @@ -479,9 +480,14 @@ async function main() { } const nonRetryableGuard = detectNonRetryableHarnessGuard(result.output); - if (nonRetryableGuard.aiCreditsExceeded || nonRetryableGuard.awfAPIProxyBlockingRequests || nonRetryableGuard.maxRunsExceeded) { + const trustedAICreditsExceeded = nonRetryableGuard.aiCreditsExceeded && parseMaxAICreditsExceededFromAuditLog(); + if (nonRetryableGuard.aiCreditsExceeded && !trustedAICreditsExceeded) { + log(`attempt ${attempt + 1}: AI credits marker found in CLI output without trusted firewall audit confirmation โ€” preserving normal failure handling`); + } + const shouldTreatAICreditsExceededAsSuccess = trustedAICreditsExceeded && !isAuthenticationFailed; + if (shouldTreatAICreditsExceededAsSuccess || nonRetryableGuard.awfAPIProxyBlockingRequests || nonRetryableGuard.maxRunsExceeded) { const reasons = []; - if (nonRetryableGuard.aiCreditsExceeded) reasons.push("AI credits budget exceeded"); + if (shouldTreatAICreditsExceededAsSuccess) reasons.push("AI credits budget exceeded"); if (nonRetryableGuard.awfAPIProxyBlockingRequests) reasons.push("AWF API proxy is blocking requests"); if (nonRetryableGuard.maxRunsExceeded) reasons.push("maximum LLM invocations exceeded"); log(`attempt ${attempt + 1}: ${reasons.join(" and ")} โ€” not retrying (non-retryable guard condition)`); @@ -490,7 +496,7 @@ async function main() { // error. Exit 0 so the agent step and job succeed; the ai_credits_rate_limit_error // output surfaced by parse-mcp-gateway will inform downstream handlers (e.g. // handle_agent_failure) of the budget exceedance. - if (nonRetryableGuard.aiCreditsExceeded) { + if (shouldTreatAICreditsExceededAsSuccess) { log(`attempt ${attempt + 1}: AI credits budget enforced โ€” exiting 0 (budget control, not an error)`); lastExitCode = 0; } diff --git a/actions/setup/js/claude_harness.test.cjs b/actions/setup/js/claude_harness.test.cjs index 1ee34adadb5..9b9a79d595d 100644 --- a/actions/setup/js/claude_harness.test.cjs +++ b/actions/setup/js/claude_harness.test.cjs @@ -708,12 +708,24 @@ process.exit(1);`, }); describe("AI credits budget enforcement exits 0", () => { + /** + * @param {string} tempDir + * @returns {string} + */ + function writeTrustedAICreditsExceededAudit(tempDir) { + const auditDir = path.join(tempDir, "sandbox", "firewall", "audit"); + fs.mkdirSync(auditDir, { recursive: true }); + fs.writeFileSync(path.join(auditDir, "log.jsonl"), `${JSON.stringify({ max_ai_credits_exceeded: true })}\n`, "utf8"); + return path.join(tempDir, "agent-output.json"); + } + it("exits 0 when the agent outputs max_ai_credits_exceeded and the CLI exits non-zero", () => { const tempDir = makeHarnessTempDir("claude-ai-credits-exceeded-"); const safeOutputsPath = path.join(tempDir, "safe-outputs.jsonl"); const stubPath = path.join(tempDir, "stub.cjs"); const promptPath = path.join(tempDir, "prompt.txt"); const callsPath = path.join(tempDir, "calls.jsonl"); + const agentOutputPath = writeTrustedAICreditsExceededAudit(tempDir); // Stub emits the AI-credits-exceeded marker on stdout (as the AWF firewall would) // then exits non-zero. The harness must detect this, set lastExitCode=0, and exit 0. fs.writeFileSync( @@ -729,7 +741,7 @@ process.exit(1);`, const result = spawnSync(process.execPath, ["claude_harness.cjs", process.execPath, stubPath, "--print", "--prompt-file", promptPath], { cwd: path.dirname(require.resolve("./claude_harness.cjs")), - env: { ...process.env, CLAUDE_HARNESS_STUB_CALLS: callsPath, GH_AW_SAFE_OUTPUTS: safeOutputsPath }, + env: { ...process.env, CLAUDE_HARNESS_STUB_CALLS: callsPath, GH_AW_SAFE_OUTPUTS: safeOutputsPath, GH_AW_AGENT_OUTPUT: agentOutputPath }, encoding: "utf8", timeout: 10000, }); @@ -748,6 +760,7 @@ process.exit(1);`, const stubPath = path.join(tempDir, "stub.cjs"); const promptPath = path.join(tempDir, "prompt.txt"); const callsPath = path.join(tempDir, "calls.jsonl"); + const agentOutputPath = writeTrustedAICreditsExceededAudit(tempDir); fs.writeFileSync( stubPath, `const fs = require("fs"); @@ -761,7 +774,7 @@ process.exit(1);`, const result = spawnSync(process.execPath, ["claude_harness.cjs", process.execPath, stubPath, "--print", "--prompt-file", promptPath], { cwd: path.dirname(require.resolve("./claude_harness.cjs")), - env: { ...process.env, CLAUDE_HARNESS_STUB_CALLS: callsPath, GH_AW_SAFE_OUTPUTS: safeOutputsPath }, + env: { ...process.env, CLAUDE_HARNESS_STUB_CALLS: callsPath, GH_AW_SAFE_OUTPUTS: safeOutputsPath, GH_AW_AGENT_OUTPUT: agentOutputPath }, encoding: "utf8", timeout: 10000, }); @@ -771,17 +784,19 @@ process.exit(1);`, expect(result.stderr).toContain("AI credits budget enforced"); }); - it("still exits 1 for non-credit-limit failures (auth error)", () => { + it("keeps non-zero exit for auth failure even when AI-credit markers and trusted audit are present", () => { const tempDir = makeHarnessTempDir("claude-auth-failure-"); const safeOutputsPath = path.join(tempDir, "safe-outputs.jsonl"); const stubPath = path.join(tempDir, "stub.cjs"); const promptPath = path.join(tempDir, "prompt.txt"); const callsPath = path.join(tempDir, "calls.jsonl"); + const agentOutputPath = writeTrustedAICreditsExceededAudit(tempDir); fs.writeFileSync( stubPath, `const fs = require("fs"); const callsPath = process.env.CLAUDE_HARNESS_STUB_CALLS; fs.appendFileSync(callsPath, JSON.stringify({args: process.argv.slice(2)}) + "\\n"); +process.stdout.write("error: max_ai_credits_exceeded=true\\n"); process.stdout.write("Authentication failed (Request ID: 123)\\n"); process.exit(1);`, "utf8" @@ -794,6 +809,7 @@ process.exit(1);`, ...process.env, CLAUDE_HARNESS_STUB_CALLS: callsPath, GH_AW_SAFE_OUTPUTS: safeOutputsPath, + GH_AW_AGENT_OUTPUT: agentOutputPath, GH_AW_HARNESS_MAX_RETRIES: "0", }, encoding: "utf8", @@ -803,5 +819,37 @@ process.exit(1);`, expect(result.status).toBe(1); expect(result.stderr).not.toContain("AI credits budget enforced"); }); + + it("keeps non-zero exit when AI-credit marker appears without trusted firewall audit evidence", () => { + const tempDir = makeHarnessTempDir("claude-ai-credits-untrusted-"); + const safeOutputsPath = path.join(tempDir, "safe-outputs.jsonl"); + const stubPath = path.join(tempDir, "stub.cjs"); + const promptPath = path.join(tempDir, "prompt.txt"); + const callsPath = path.join(tempDir, "calls.jsonl"); + fs.writeFileSync( + stubPath, + `const fs = require("fs"); +const callsPath = process.env.CLAUDE_HARNESS_STUB_CALLS; +fs.appendFileSync(callsPath, JSON.stringify({args: process.argv.slice(2)}) + "\\n"); +process.stdout.write("error: max_ai_credits_exceeded=true\\n"); +process.exit(1);`, + "utf8" + ); + fs.writeFileSync(promptPath, "do some work", "utf8"); + + const result = spawnSync(process.execPath, ["claude_harness.cjs", process.execPath, stubPath, "--print", "--prompt-file", promptPath], { + cwd: path.dirname(require.resolve("./claude_harness.cjs")), + env: { + ...process.env, + CLAUDE_HARNESS_STUB_CALLS: callsPath, + GH_AW_SAFE_OUTPUTS: safeOutputsPath, + GH_AW_HARNESS_MAX_RETRIES: "0", + }, + encoding: "utf8", + timeout: 10000, + }); + expect(result.status).toBe(1); + expect(result.stderr).toContain("without trusted firewall audit confirmation"); + }); }); }); diff --git a/actions/setup/js/codex_harness.cjs b/actions/setup/js/codex_harness.cjs index 99ce9bb0cc8..9187a81a6f1 100644 --- a/actions/setup/js/codex_harness.cjs +++ b/actions/setup/js/codex_harness.cjs @@ -55,6 +55,7 @@ const { detectNonRetryableHarnessGuard, buildSoftTimeoutGuard, emitSoftTimeoutSi const { MODEL_NOT_SUPPORTED_PATTERN: INVALID_MODEL_ERROR_PATTERN } = require("./detect_agent_errors.cjs"); const { resolveRetryConfig } = require("./harness_retry_config.cjs"); const { applyModelFallback, injectModelFlagAfterExec } = require("./model_fallback.cjs"); +const { parseMaxAICreditsExceededFromAuditLog } = require("./ai_credits_context.cjs"); // Pattern to detect OpenAI rate-limit errors. // Matches the JSON error type field ("rate_limit_exceeded"), the HTTP status code @@ -661,9 +662,14 @@ async function main() { } const nonRetryableGuard = detectNonRetryableHarnessGuard(result.output); - if (nonRetryableGuard.aiCreditsExceeded || nonRetryableGuard.awfAPIProxyBlockingRequests || nonRetryableGuard.goalAlreadyActive || nonRetryableGuard.maxRunsExceeded) { + const trustedAICreditsExceeded = nonRetryableGuard.aiCreditsExceeded && parseMaxAICreditsExceededFromAuditLog(); + if (nonRetryableGuard.aiCreditsExceeded && !trustedAICreditsExceeded) { + log(`attempt ${attempt + 1}: AI credits marker found in CLI output without trusted firewall audit confirmation โ€” preserving normal failure handling`); + } + const shouldTreatAICreditsExceededAsSuccess = trustedAICreditsExceeded && !isAuthenticationFailed && !isMissingApiKey; + if (shouldTreatAICreditsExceededAsSuccess || nonRetryableGuard.awfAPIProxyBlockingRequests || nonRetryableGuard.goalAlreadyActive || nonRetryableGuard.maxRunsExceeded) { const reasons = []; - if (nonRetryableGuard.aiCreditsExceeded) reasons.push("AI credits budget exceeded"); + if (shouldTreatAICreditsExceededAsSuccess) reasons.push("AI credits budget exceeded"); if (nonRetryableGuard.awfAPIProxyBlockingRequests) reasons.push("AWF API proxy is blocking requests"); if (nonRetryableGuard.goalAlreadyActive) reasons.push("goal is already active for this thread (use update_goal when the current goal is complete)"); if (nonRetryableGuard.maxRunsExceeded) reasons.push("maximum LLM invocations exceeded"); @@ -673,7 +679,7 @@ async function main() { // error. Exit 0 so the agent step and job succeed; the ai_credits_rate_limit_error // output surfaced by parse-mcp-gateway will inform downstream handlers (e.g. // handle_agent_failure) of the budget exceedance. - if (nonRetryableGuard.aiCreditsExceeded) { + if (shouldTreatAICreditsExceededAsSuccess) { log(`attempt ${attempt + 1}: AI credits budget enforced โ€” exiting 0 (budget control, not an error)`); lastExitCode = 0; } diff --git a/actions/setup/js/codex_harness.test.cjs b/actions/setup/js/codex_harness.test.cjs index 02f6e781f34..74ed3fe1b8c 100644 --- a/actions/setup/js/codex_harness.test.cjs +++ b/actions/setup/js/codex_harness.test.cjs @@ -947,12 +947,24 @@ process.exit(1);`, }); describe("AI credits budget enforcement exits 0", () => { + /** + * @param {string} tempDir + * @returns {string} + */ + function writeTrustedAICreditsExceededAudit(tempDir) { + const auditDir = path.join(tempDir, "sandbox", "firewall", "audit"); + fs.mkdirSync(auditDir, { recursive: true }); + fs.writeFileSync(path.join(auditDir, "log.jsonl"), `${JSON.stringify({ max_ai_credits_exceeded: true })}\n`, "utf8"); + return path.join(tempDir, "agent-output.json"); + } + it("exits 0 when the agent outputs max_ai_credits_exceeded and the CLI exits non-zero", () => { const tempDir = makeHarnessTempDir("codex-ai-credits-exceeded-"); const safeOutputsPath = path.join(tempDir, "safe-outputs.jsonl"); const stubPath = path.join(tempDir, "stub.cjs"); const promptPath = path.join(tempDir, "prompt.txt"); const callsPath = path.join(tempDir, "calls.jsonl"); + const agentOutputPath = writeTrustedAICreditsExceededAudit(tempDir); // Stub emits the AI-credits-exceeded marker on stdout (as the AWF firewall would) // then exits non-zero. The harness must detect this, set lastExitCode=0, and exit 0. fs.writeFileSync( @@ -972,6 +984,7 @@ process.exit(1);`, ...process.env, CODEX_HARNESS_STUB_CALLS: callsPath, GH_AW_SAFE_OUTPUTS: safeOutputsPath, + GH_AW_AGENT_OUTPUT: agentOutputPath, CODEX_API_KEY: "fake-key-for-test", }, encoding: "utf8", @@ -992,6 +1005,7 @@ process.exit(1);`, const stubPath = path.join(tempDir, "stub.cjs"); const promptPath = path.join(tempDir, "prompt.txt"); const callsPath = path.join(tempDir, "calls.jsonl"); + const agentOutputPath = writeTrustedAICreditsExceededAudit(tempDir); fs.writeFileSync( stubPath, `const fs = require("fs"); @@ -1009,6 +1023,7 @@ process.exit(1);`, ...process.env, CODEX_HARNESS_STUB_CALLS: callsPath, GH_AW_SAFE_OUTPUTS: safeOutputsPath, + GH_AW_AGENT_OUTPUT: agentOutputPath, CODEX_API_KEY: "fake-key-for-test", }, encoding: "utf8", @@ -1020,17 +1035,19 @@ process.exit(1);`, expect(result.stderr).toContain("AI credits budget enforced"); }); - it("still exits 1 for non-credit-limit failures (auth error)", () => { + it("keeps non-zero exit for auth failure even when AI-credit markers and trusted audit are present", () => { const tempDir = makeHarnessTempDir("codex-auth-failure-"); const safeOutputsPath = path.join(tempDir, "safe-outputs.jsonl"); const stubPath = path.join(tempDir, "stub.cjs"); const promptPath = path.join(tempDir, "prompt.txt"); const callsPath = path.join(tempDir, "calls.jsonl"); + const agentOutputPath = writeTrustedAICreditsExceededAudit(tempDir); fs.writeFileSync( stubPath, `const fs = require("fs"); const callsPath = process.env.CODEX_HARNESS_STUB_CALLS; fs.appendFileSync(callsPath, JSON.stringify({args: process.argv.slice(2)}) + "\\n"); +process.stdout.write("error: max_ai_credits_exceeded=true\\n"); process.stdout.write("Authentication failed (Request ID: 123)\\n"); process.exit(1);`, "utf8" @@ -1043,6 +1060,7 @@ process.exit(1);`, ...process.env, CODEX_HARNESS_STUB_CALLS: callsPath, GH_AW_SAFE_OUTPUTS: safeOutputsPath, + GH_AW_AGENT_OUTPUT: agentOutputPath, CODEX_API_KEY: "fake-key-for-test", GH_AW_HARNESS_MAX_RETRIES: "0", }, @@ -1053,5 +1071,38 @@ process.exit(1);`, expect(result.status).toBe(1); expect(result.stderr).not.toContain("AI credits budget enforced"); }); + + it("keeps non-zero exit when AI-credit marker appears without trusted firewall audit evidence", () => { + const tempDir = makeHarnessTempDir("codex-ai-credits-untrusted-"); + const safeOutputsPath = path.join(tempDir, "safe-outputs.jsonl"); + const stubPath = path.join(tempDir, "stub.cjs"); + const promptPath = path.join(tempDir, "prompt.txt"); + const callsPath = path.join(tempDir, "calls.jsonl"); + fs.writeFileSync( + stubPath, + `const fs = require("fs"); +const callsPath = process.env.CODEX_HARNESS_STUB_CALLS; +fs.appendFileSync(callsPath, JSON.stringify({args: process.argv.slice(2)}) + "\\n"); +process.stdout.write("error: max_ai_credits_exceeded=true\\n"); +process.exit(1);`, + "utf8" + ); + fs.writeFileSync(promptPath, "do some work", "utf8"); + + const result = spawnSync(process.execPath, ["codex_harness.cjs", process.execPath, stubPath, "exec", "--prompt-file", promptPath], { + cwd: path.dirname(require.resolve("./codex_harness.cjs")), + env: { + ...process.env, + CODEX_HARNESS_STUB_CALLS: callsPath, + GH_AW_SAFE_OUTPUTS: safeOutputsPath, + CODEX_API_KEY: "fake-key-for-test", + GH_AW_HARNESS_MAX_RETRIES: "0", + }, + encoding: "utf8", + timeout: 10000, + }); + expect(result.status).toBe(1); + expect(result.stderr).toContain("without trusted firewall audit confirmation"); + }); }); }); diff --git a/actions/setup/js/copilot_harness.cjs b/actions/setup/js/copilot_harness.cjs index b4e63a0af9f..4b08b41ff2c 100644 --- a/actions/setup/js/copilot_harness.cjs +++ b/actions/setup/js/copilot_harness.cjs @@ -77,6 +77,7 @@ const { isCAPIQuotaExceededError } = require("./detect_agent_errors.cjs"); const { applyModelFallback } = require("./model_fallback.cjs"); const { loadModelsJson } = require("./model_costs.cjs"); const { resolveConfiguredCopilotModel } = require("./resolve_model_alias.cjs"); +const { parseMaxAICreditsExceededFromAuditLog } = require("./ai_credits_context.cjs"); const AWF_CONFIG_PATH = process.env.GH_AW_AWF_CONFIG_PATH || "/tmp/gh-aw/awf-config.json"; @@ -1278,9 +1279,14 @@ async function main() { break; } - if (nonRetryableGuard.aiCreditsExceeded || nonRetryableGuard.awfAPIProxyBlockingRequests || isInvocationCapExceeded) { + const trustedAICreditsExceeded = nonRetryableGuard.aiCreditsExceeded && parseMaxAICreditsExceededFromAuditLog(); + if (nonRetryableGuard.aiCreditsExceeded && !trustedAICreditsExceeded) { + log(`attempt ${attempt + 1}: AI credits marker found in CLI output without trusted firewall audit confirmation โ€” preserving normal failure handling`); + } + const shouldTreatAICreditsExceededAsSuccess = trustedAICreditsExceeded && !isAuthenticationFailed; + if (shouldTreatAICreditsExceededAsSuccess || nonRetryableGuard.awfAPIProxyBlockingRequests || isInvocationCapExceeded) { const reasons = []; - if (nonRetryableGuard.aiCreditsExceeded) reasons.push("AI credits budget exceeded"); + if (shouldTreatAICreditsExceededAsSuccess) reasons.push("AI credits budget exceeded"); if (nonRetryableGuard.awfAPIProxyBlockingRequests) reasons.push("AWF API proxy is blocking requests"); if (isInvocationCapExceeded) { reasons.push("LLM invocation cap saturated โ€” the pooled per-run budget is fully exhausted; retries cannot make progress"); @@ -1291,7 +1297,7 @@ async function main() { // error. Exit 0 so the agent step and job succeed; the ai_credits_rate_limit_error // output surfaced by parse-mcp-gateway will inform downstream handlers (e.g. // handle_agent_failure) of the budget exceedance. - if (nonRetryableGuard.aiCreditsExceeded) { + if (shouldTreatAICreditsExceededAsSuccess) { log(`attempt ${attempt + 1}: AI credits budget enforced โ€” exiting 0 (budget control, not an error)`); lastExitCode = 0; } diff --git a/actions/setup/js/copilot_harness.test.cjs b/actions/setup/js/copilot_harness.test.cjs index 4d5dbc78cf2..fbe387f4b7b 100644 --- a/actions/setup/js/copilot_harness.test.cjs +++ b/actions/setup/js/copilot_harness.test.cjs @@ -2623,12 +2623,24 @@ setInterval(() => {}, 1000);`, }); describe("AI credits budget enforcement exits 0", () => { + /** + * @param {string} tempDir + * @returns {string} + */ + function writeTrustedAICreditsExceededAudit(tempDir) { + const auditDir = path.join(tempDir, "sandbox", "firewall", "audit"); + fs.mkdirSync(auditDir, { recursive: true }); + fs.writeFileSync(path.join(auditDir, "log.jsonl"), `${JSON.stringify({ max_ai_credits_exceeded: true })}\n`, "utf8"); + return path.join(tempDir, "agent-output.json"); + } + it("exits 0 when the agent outputs max_ai_credits_exceeded and the CLI exits non-zero", () => { const tempDir = makeHarnessTempDir("copilot-ai-credits-exceeded-"); const safeOutputsPath = path.join(tempDir, "safe-outputs.jsonl"); const stubPath = path.join(tempDir, "stub.cjs"); const promptPath = path.join(tempDir, "prompt.txt"); const callsPath = path.join(tempDir, "calls.jsonl"); + const agentOutputPath = writeTrustedAICreditsExceededAudit(tempDir); // Stub emits the AI-credits-exceeded marker on stdout (as the AWF firewall would) // then exits non-zero. The harness must detect this, set lastExitCode=0, and exit 0. fs.writeFileSync( @@ -2644,7 +2656,7 @@ process.exit(1);`, const result = spawnSync(process.execPath, ["copilot_harness.cjs", process.execPath, stubPath, "--prompt-file", promptPath], { cwd: path.dirname(require.resolve("./copilot_harness.cjs")), - env: { ...process.env, COPILOT_HARNESS_STUB_CALLS: callsPath, GH_AW_SAFE_OUTPUTS: safeOutputsPath }, + env: { ...process.env, COPILOT_HARNESS_STUB_CALLS: callsPath, GH_AW_SAFE_OUTPUTS: safeOutputsPath, GH_AW_AGENT_OUTPUT: agentOutputPath }, encoding: "utf8", timeout: 10000, }); @@ -2663,6 +2675,7 @@ process.exit(1);`, const stubPath = path.join(tempDir, "stub.cjs"); const promptPath = path.join(tempDir, "prompt.txt"); const callsPath = path.join(tempDir, "calls.jsonl"); + const agentOutputPath = writeTrustedAICreditsExceededAudit(tempDir); fs.writeFileSync( stubPath, `const fs = require("fs"); @@ -2676,7 +2689,7 @@ process.exit(1);`, const result = spawnSync(process.execPath, ["copilot_harness.cjs", process.execPath, stubPath, "--prompt-file", promptPath], { cwd: path.dirname(require.resolve("./copilot_harness.cjs")), - env: { ...process.env, COPILOT_HARNESS_STUB_CALLS: callsPath, GH_AW_SAFE_OUTPUTS: safeOutputsPath }, + env: { ...process.env, COPILOT_HARNESS_STUB_CALLS: callsPath, GH_AW_SAFE_OUTPUTS: safeOutputsPath, GH_AW_AGENT_OUTPUT: agentOutputPath }, encoding: "utf8", timeout: 10000, }); @@ -2686,18 +2699,20 @@ process.exit(1);`, expect(result.stderr).toContain("AI credits budget enforced"); }); - it("still exits 1 for non-credit-limit failures (auth error)", () => { + it("keeps non-zero exit for auth failure even when AI-credit markers and trusted audit are present", () => { const tempDir = makeHarnessTempDir("copilot-auth-failure-"); const safeOutputsPath = path.join(tempDir, "safe-outputs.jsonl"); const stubPath = path.join(tempDir, "stub.cjs"); const promptPath = path.join(tempDir, "prompt.txt"); const callsPath = path.join(tempDir, "calls.jsonl"); + const agentOutputPath = writeTrustedAICreditsExceededAudit(tempDir); fs.writeFileSync( stubPath, `const fs = require("fs"); const callsPath = process.env.COPILOT_HARNESS_STUB_CALLS; fs.appendFileSync(callsPath, JSON.stringify({args: process.argv.slice(2)}) + "\\n"); -process.stdout.write("CAPIError: 401 Unauthorized\\n"); +process.stdout.write("error: max_ai_credits_exceeded=true\\n"); +process.stdout.write("Authentication failed (Request ID: 123)\\n"); process.exit(1);`, "utf8" ); @@ -2709,6 +2724,7 @@ process.exit(1);`, ...process.env, COPILOT_HARNESS_STUB_CALLS: callsPath, GH_AW_SAFE_OUTPUTS: safeOutputsPath, + GH_AW_AGENT_OUTPUT: agentOutputPath, GH_AW_HARNESS_MAX_RETRIES: "0", }, encoding: "utf8", @@ -2718,6 +2734,38 @@ process.exit(1);`, expect(result.status).toBe(1); expect(result.stderr).not.toContain("AI credits budget enforced"); }); + + it("keeps non-zero exit when AI-credit marker appears without trusted firewall audit evidence", () => { + const tempDir = makeHarnessTempDir("copilot-ai-credits-untrusted-"); + const safeOutputsPath = path.join(tempDir, "safe-outputs.jsonl"); + const stubPath = path.join(tempDir, "stub.cjs"); + const promptPath = path.join(tempDir, "prompt.txt"); + const callsPath = path.join(tempDir, "calls.jsonl"); + fs.writeFileSync( + stubPath, + `const fs = require("fs"); +const callsPath = process.env.COPILOT_HARNESS_STUB_CALLS; +fs.appendFileSync(callsPath, JSON.stringify({args: process.argv.slice(2)}) + "\\n"); +process.stdout.write("error: max_ai_credits_exceeded=true\\n"); +process.exit(1);`, + "utf8" + ); + fs.writeFileSync(promptPath, "do some work", "utf8"); + + const result = spawnSync(process.execPath, ["copilot_harness.cjs", process.execPath, stubPath, "--prompt-file", promptPath], { + cwd: path.dirname(require.resolve("./copilot_harness.cjs")), + env: { + ...process.env, + COPILOT_HARNESS_STUB_CALLS: callsPath, + GH_AW_SAFE_OUTPUTS: safeOutputsPath, + GH_AW_HARNESS_MAX_RETRIES: "0", + }, + encoding: "utf8", + timeout: 10000, + }); + expect(result.status).toBe(1); + expect(result.stderr).toContain("without trusted firewall audit confirmation"); + }); }); describe("applyCopilotWireAPI", () => {