From a9d4015d5f41a2379fd1d7af1adeaf3e827ea644 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 20 May 2026 12:26:10 +0000 Subject: [PATCH 1/2] Initial plan From 23af3051fed39e54101fff032a6699d17a18e3cc Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 20 May 2026 12:36:13 +0000 Subject: [PATCH 2/2] fix(otlp): always emit gen_ai.response.finish_reasons and use GITHUB_SHA as service.version fallback - gen_ai.response.finish_reasons: always present on agent conclusion spans. Previously omitted when the engine didn't write a `stop_reason` to agent-stdio.log (copilot, codex). Now falls back to "unknown" so length-truncation is always queryable in Sentry/dashboards. - service.version: add GITHUB_SHA as a final fallback in both the setup and conclusion span paths. When no gh-aw release tag is available the commit SHA is used, enabling Sentry to populate `release` and surface per-commit regressions. Addresses P1 findings from the 2026-05-20 Daily Reliability Review. Co-authored-by: mnkiefer <8320933+mnkiefer@users.noreply.github.com> --- actions/setup/js/send_otlp_span.cjs | 15 ++++---- actions/setup/js/send_otlp_span.test.cjs | 49 +++++++++++++++++++++--- 2 files changed, 52 insertions(+), 12 deletions(-) diff --git a/actions/setup/js/send_otlp_span.cjs b/actions/setup/js/send_otlp_span.cjs index 7a5b3ac8c95..21ece71bd86 100644 --- a/actions/setup/js/send_otlp_span.cjs +++ b/actions/setup/js/send_otlp_span.cjs @@ -1173,7 +1173,7 @@ async function sendJobSetupSpan(options = {}) { startMs, endMs, serviceName, - scopeVersion: process.env.GH_AW_INFO_VERSION || process.env.GH_AW_INFO_CLI_VERSION || "unknown", + scopeVersion: process.env.GH_AW_INFO_VERSION || process.env.GH_AW_INFO_CLI_VERSION || process.env.GITHUB_SHA || "unknown", attributes, resourceAttributes, }); @@ -1627,7 +1627,7 @@ async function sendJobConclusionSpan(spanName, options = {}) { const effectiveTokens = rawET ? parseInt(rawET, 10) : NaN; const serviceName = process.env.OTEL_SERVICE_NAME || "gh-aw"; - const version = awInfo.agent_version || awInfo.version || process.env.GH_AW_INFO_VERSION || awInfo.cli_version || process.env.GH_AW_INFO_CLI_VERSION || "unknown"; + const version = awInfo.agent_version || awInfo.version || process.env.GH_AW_INFO_VERSION || awInfo.cli_version || process.env.GH_AW_INFO_CLI_VERSION || process.env.GITHUB_SHA || "unknown"; // Prefer GITHUB_AW_OTEL_TRACE_ID (written to GITHUB_ENV by this job's setup step) so // all spans in the same job share one trace. Fall back to aw_context.otel_trace_id @@ -1791,11 +1791,12 @@ async function sendJobConclusionSpan(spanName, options = {}) { if (workflowName) attributes.push(buildAttr("gen_ai.workflow.name", workflowName)); if (runtimeMetrics.resolvedModel) attributes.push(buildAttr("gen_ai.response.model", runtimeMetrics.resolvedModel)); // Use the engine-reported stop_reason when available; fall back to "timeout" when - // the agent was killed before it could write a result entry to agent-stdio.log. - const effectiveStopReason = runtimeMetrics.stopReason || (isAgentTimedOut ? "timeout" : undefined); - if (effectiveStopReason) { - attributes.push(buildArrayAttr("gen_ai.response.finish_reasons", [effectiveStopReason])); - } + // the agent was killed before it could write a result entry to agent-stdio.log; + // use "unknown" as a sentinel for engines (e.g. copilot, codex) that do not emit + // a result entry at all, so that gen_ai.response.finish_reasons is always present + // and length-truncation is always queryable in Sentry/dashboards. + const effectiveStopReason = runtimeMetrics.stopReason || (isAgentTimedOut ? "timeout" : "unknown"); + attributes.push(buildArrayAttr("gen_ai.response.finish_reasons", [effectiveStopReason])); } if (agentConclusion) { diff --git a/actions/setup/js/send_otlp_span.test.cjs b/actions/setup/js/send_otlp_span.test.cjs index 61cfacf43e0..15243981e5a 100644 --- a/actions/setup/js/send_otlp_span.test.cjs +++ b/actions/setup/js/send_otlp_span.test.cjs @@ -1822,6 +1822,22 @@ describe("sendJobSetupSpan", () => { expect(body.resourceSpans[0].scopeSpans[0].scope.version).toBe("v2.5.0"); }); + it("falls back to GITHUB_SHA for service.version when no gh-aw version is available", async () => { + const mockFetch = vi.fn().mockResolvedValue({ ok: true, status: 200, statusText: "OK" }); + vi.stubGlobal("fetch", mockFetch); + + process.env.GH_AW_OTLP_ENDPOINTS = JSON.stringify([{ url: "https://traces.example.com" }]); + // No GH_AW_INFO_VERSION or GH_AW_INFO_CLI_VERSION — only SHA available + process.env.GITHUB_SHA = "aabbccdd1122334455667788aabbccdd11223344"; + + await sendJobSetupSpan(); + + const body = JSON.parse(mockFetch.mock.calls[0][1].body); + const resourceAttrs = body.resourceSpans[0].resource.attributes; + expect(resourceAttrs).toContainEqual({ key: "service.version", value: { stringValue: "aabbccdd1122334455667788aabbccdd11223344" } }); + expect(body.resourceSpans[0].scopeSpans[0].scope.version).toBe("aabbccdd1122334455667788aabbccdd11223344"); + }); + it("includes gh-aw.awf.version and gh-aw.awmg.version resource attributes from aw_info.json", async () => { const mockFetch = vi.fn().mockResolvedValue({ ok: true, status: 200, statusText: "OK" }); vi.stubGlobal("fetch", mockFetch); @@ -3040,7 +3056,7 @@ describe("sendJobConclusionSpan", () => { expect(conclusionAttrs["gen_ai.response.model"]).toBe("claude-sonnet-4-20250514"); }); - it("omits gen_ai.response.finish_reasons from the agent span when stop_reason is absent in agent-stdio.log", async () => { + it("emits gen_ai.response.finish_reasons=[unknown] on the agent span when stop_reason is absent from agent-stdio.log", async () => { const mockFetch = vi.fn().mockResolvedValue({ ok: true, status: 200, statusText: "OK" }); vi.stubGlobal("fetch", mockFetch); @@ -3065,13 +3081,15 @@ describe("sendJobConclusionSpan", () => { const agentBody = JSON.parse(mockFetch.mock.calls[0][1].body); const agentSpan = agentBody.resourceSpans[0].scopeSpans[0].spans[0]; expect(agentSpan.name).toBe("gh-aw.agent.agent"); - const keys = agentSpan.attributes.map(a => a.key); - expect(keys).not.toContain("gen_ai.response.finish_reasons"); + const agentFinishAttr = agentSpan.attributes.find(a => a.key === "gen_ai.response.finish_reasons"); + expect(agentFinishAttr).toBeDefined(); + expect(agentFinishAttr.value.arrayValue.values).toEqual([{ stringValue: "unknown" }]); const conclusionBody = JSON.parse(mockFetch.mock.calls[1][1].body); const conclusionSpan = conclusionBody.resourceSpans[0].scopeSpans[0].spans[0]; - const conclusionKeys = conclusionSpan.attributes.map(a => a.key); - expect(conclusionKeys).not.toContain("gen_ai.response.finish_reasons"); + const conclusionFinishAttr = conclusionSpan.attributes.find(a => a.key === "gen_ai.response.finish_reasons"); + expect(conclusionFinishAttr).toBeDefined(); + expect(conclusionFinishAttr.value.arrayValue.values).toEqual([{ stringValue: "unknown" }]); }); it("includes gen_ai.response.finish_reasons with max_tokens on the agent span when truncated", async () => { @@ -3934,6 +3952,27 @@ describe("sendJobConclusionSpan", () => { expect(resourceAttrs).toContainEqual({ key: "service.version", value: { stringValue: "v3.0.0" } }); }); + it("falls back to GITHUB_SHA for service.version on conclusion span when no gh-aw version is available", async () => { + const mockFetch = vi.fn().mockResolvedValue({ ok: true, status: 200, statusText: "OK" }); + vi.stubGlobal("fetch", mockFetch); + + process.env.GH_AW_OTLP_ENDPOINTS = JSON.stringify([{ url: "https://traces.example.com" }]); + // No GH_AW_INFO_VERSION, GH_AW_INFO_CLI_VERSION, or aw_info.json version fields + process.env.GITHUB_SHA = "deadbeef1234567890abcdef1234567890abcdef"; + + const readFileSpy = vi.spyOn(fs, "readFileSync").mockImplementation(() => { + throw Object.assign(new Error("ENOENT"), { code: "ENOENT" }); + }); + + await sendJobConclusionSpan("gh-aw.job.conclusion"); + readFileSpy.mockRestore(); + + const body = JSON.parse(mockFetch.mock.calls[0][1].body); + const resourceAttrs = body.resourceSpans[0].resource.attributes; + expect(resourceAttrs).toContainEqual({ key: "service.version", value: { stringValue: "deadbeef1234567890abcdef1234567890abcdef" } }); + expect(body.resourceSpans[0].scopeSpans[0].scope.version).toBe("deadbeef1234567890abcdef1234567890abcdef"); + }); + describe("agent_output.json error enrichment", () => { let readFileSpy;