From db6de3d4a2205a6442dc07034e45fb3ace466f2f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 2 Aug 2026 14:55:00 +0000 Subject: [PATCH 1/2] fix noop footer evals AIC breakdown Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- actions/setup/js/handle_noop_message.cjs | 40 +++++++++++++------ actions/setup/js/handle_noop_message.test.cjs | 31 ++++++++++++-- 2 files changed, 56 insertions(+), 15 deletions(-) diff --git a/actions/setup/js/handle_noop_message.cjs b/actions/setup/js/handle_noop_message.cjs index 3ec17688f07..d8aaa45a2c0 100644 --- a/actions/setup/js/handle_noop_message.cjs +++ b/actions/setup/js/handle_noop_message.cjs @@ -80,22 +80,38 @@ async function ensureAgentRunsIssue() { /** * Build the AIC suffix string for use in comment footers. * Includes agent, threat-detection, and evals AIC when available. - * Returns a string like " · 0.001 AIC" or "" when not available. + * Returns a string like " · 0.001 AIC · ⌖ 0.002 AIC" or "" when not available. * @returns {string} */ -function buildAICSuffix() { - const agentRaw = process.env.GH_AW_AIC; - const detectionRaw = process.env.GH_AW_THREAT_DETECTION_AIC; - const evalsRaw = process.env.GH_AW_EVALS_AIC; - const agentAIC = agentRaw ? Number.parseFloat(agentRaw) : NaN; - const detectionAIC = detectionRaw ? Number.parseFloat(detectionRaw) : NaN; - const evalsAIC = evalsRaw ? Number.parseFloat(evalsRaw) : NaN; - const compressedModelName = reduceModelNameToIdentifier(process.env.GH_AW_PRIMARY_MODEL || process.env.GH_AW_ENGINE_MODEL); - const totalAIC = (Number.isFinite(agentAIC) && agentAIC > 0 ? agentAIC : 0) + (Number.isFinite(detectionAIC) && detectionAIC > 0 ? detectionAIC : 0) + (Number.isFinite(evalsAIC) && evalsAIC > 0 ? evalsAIC : 0); - if (totalAIC <= 0) { +function parsePositiveAIC(raw) { + const parsed = raw ? Number.parseFloat(raw) : NaN; + return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined; +} + +/** + * @param {string} label + * @param {number|undefined} value + * @param {string|undefined} [modelAlias] + * @returns {string} + */ +function buildAICEntry(label, value, modelAlias) { + const formatted = typeof value === "number" ? formatAIC(value) : ""; + if (!formatted) { return ""; } - return ` · ${compressedModelName ? `${compressedModelName} · ` : ""}${formatAIC(totalAIC)} AIC`; + const prefix = [label, modelAlias].filter(Boolean).join(" "); + return ` · ${prefix ? `${prefix}${modelAlias ? " · " : " "}` : ""}${formatted} AIC`; +} + +function buildAICSuffix() { + const agentAIC = parsePositiveAIC(process.env.GH_AW_AIC); + const detectionAIC = parsePositiveAIC(process.env.GH_AW_THREAT_DETECTION_AIC); + const evalsAIC = parsePositiveAIC(process.env.GH_AW_EVALS_AIC); + const compressedModelName = reduceModelNameToIdentifier(process.env.GH_AW_PRIMARY_MODEL || process.env.GH_AW_ENGINE_MODEL); + const agentSuffix = buildAICEntry("", agentAIC, compressedModelName); + const detectionSuffix = buildAICEntry("⌖", detectionAIC); + const evalsSuffix = buildAICEntry("◇", evalsAIC); + return `${agentSuffix}${detectionSuffix}${evalsSuffix}`; } /** diff --git a/actions/setup/js/handle_noop_message.test.cjs b/actions/setup/js/handle_noop_message.test.cjs index ac654543f15..1c30c3269b7 100644 --- a/actions/setup/js/handle_noop_message.test.cjs +++ b/actions/setup/js/handle_noop_message.test.cjs @@ -783,10 +783,10 @@ safe-outputs: await main(); const commentCall = mockGithub.rest.issues.createComment.mock.calls[0][0]; - expect(commentCall.body).toContain("sonnet46 · 0.125 AIC"); + expect(commentCall.body).toContain("sonnet46 · 0.1 AIC · ⌖ 0.025 AIC"); }); - it("should include evals AIC in the footer total when GH_AW_EVALS_AIC is set", async () => { + it("should include evals AIC in the footer breakdown when GH_AW_EVALS_AIC is set", async () => { process.env.GH_AW_WORKFLOW_NAME = "Evals AIC Workflow"; process.env.GH_AW_RUN_URL = "https://github.com/test/test/actions/runs/123"; process.env.GH_AW_AGENT_CONCLUSION = "success"; @@ -807,7 +807,32 @@ safe-outputs: await main(); const commentCall = mockGithub.rest.issues.createComment.mock.calls[0][0]; - expect(commentCall.body).toContain("sonnet46 · 0.125 AIC"); + expect(commentCall.body).toContain("sonnet46 · 0.1 AIC · ◇ 0.025 AIC"); + }); + + it("should place evals AIC after detection AIC in the footer breakdown", async () => { + process.env.GH_AW_WORKFLOW_NAME = "Ordered AIC Workflow"; + process.env.GH_AW_RUN_URL = "https://github.com/test/test/actions/runs/123"; + process.env.GH_AW_AGENT_CONCLUSION = "success"; + process.env.GH_AW_AIC = "0.100"; + process.env.GH_AW_THREAT_DETECTION_AIC = "0.025"; + process.env.GH_AW_EVALS_AIC = "0.010"; + process.env.GH_AW_ENGINE_MODEL = "claude-sonnet-4.6"; + + const outputFile = path.join(tempDir, "agent_output.json"); + fs.writeFileSync(outputFile, JSON.stringify({ items: [{ type: "noop", message: "No action needed" }] })); + process.env.GH_AW_AGENT_OUTPUT = outputFile; + + mockGithub.rest.search.issuesAndPullRequests.mockResolvedValue({ + data: { total_count: 1, items: [{ number: 1, node_id: "ID", html_url: "url" }] }, + }); + mockGithub.rest.issues.createComment.mockResolvedValue({ data: {} }); + + const { main } = await import("./handle_noop_message.cjs?t=" + Date.now()); + await main(); + + const commentCall = mockGithub.rest.issues.createComment.mock.calls[0][0]; + expect(commentCall.body).toContain("sonnet46 · 0.1 AIC · ⌖ 0.025 AIC · ◇ 0.01 AIC"); }); it("should not include AIC suffix in comment footer when GH_AW_AIC is not set", async () => { From 124685a4d69bd8ebc5fc08895c0f7f869f8420b2 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 2 Aug 2026 15:40:38 +0000 Subject: [PATCH 2/2] Fix JSDoc on parsePositiveAIC: correct description and return type Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- actions/setup/js/handle_noop_message.cjs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/actions/setup/js/handle_noop_message.cjs b/actions/setup/js/handle_noop_message.cjs index d8aaa45a2c0..4de6d9d2a95 100644 --- a/actions/setup/js/handle_noop_message.cjs +++ b/actions/setup/js/handle_noop_message.cjs @@ -78,10 +78,10 @@ async function ensureAgentRunsIssue() { } /** - * Build the AIC suffix string for use in comment footers. - * Includes agent, threat-detection, and evals AIC when available. - * Returns a string like " · 0.001 AIC · ⌖ 0.002 AIC" or "" when not available. - * @returns {string} + * Parse a raw AIC environment variable value and return it as a positive number. + * Returns undefined when the value is absent, non-numeric, or non-positive. + * @param {string|undefined} raw + * @returns {number|undefined} */ function parsePositiveAIC(raw) { const parsed = raw ? Number.parseFloat(raw) : NaN;