Skip to content

feat: report agent failure in OTEL conclusion span #24650

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pelikhan merged 2 commits into from
Apr 5, 2026
Merged

feat: report agent failure in OTEL conclusion span #24650

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
eb0a0e6
chore: start implementing agent failure reporting in OTEL conclusion …
Copilot Apr 5, 2026
54a7e86
feat: report agent failure in OTEL conclusion span
Copilot Apr 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions .github/workflows/hourly-ci-cleaner.lock.yml
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 4 additions & 0 deletions actions/setup/js/action_conclusion_otlp.cjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,10 @@
* INPUT_JOB_NAME – job name from the `job-name` action input; when set the
* span is named "gh-aw.job.<name>", otherwise
* "gh-aw.job.conclusion".
* GH_AW_AGENT_CONCLUSION – agent job result passed from the agent job
* ("success", "failure", "timed_out", etc.);
* "failure" and "timed_out" set the span
* status to STATUS_CODE_ERROR.
* GITHUB_AW_OTEL_TRACE_ID – parent trace ID (set by action_setup_otlp.cjs)
* GITHUB_AW_OTEL_PARENT_SPAN_ID – parent span ID (set by action_setup_otlp.cjs)
* OTEL_EXPORTER_OTLP_ENDPOINT – OTLP endpoint (no-op when not set)
Expand Down
102 changes: 102 additions & 0 deletions actions/setup/js/action_otlp.test.cjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,4 +192,106 @@ describe("action_conclusion_otlp run()", () => {
expect(spanName).toBe("gh-aw.job.conclusion");
fetchSpy.mockRestore();
});

it("records agent failure conclusion as STATUS_CODE_ERROR when GH_AW_AGENT_CONCLUSION is 'failure'", async () => {
process.env.OTEL_EXPORTER_OTLP_ENDPOINT = "http://localhost:14317";
process.env.GH_AW_AGENT_CONCLUSION = "failure";
let capturedBody;
const fetchSpy = vi.spyOn(global, "fetch").mockImplementation((_url, opts) => {
capturedBody = opts?.body;
return Promise.resolve(new Response(null, { status: 200 }));
});

await runConclusion();

const payload = JSON.parse(capturedBody);
const span = payload?.resourceSpans?.[0]?.scopeSpans?.[0]?.spans?.[0];
expect(span?.status?.code).toBe(2); // STATUS_CODE_ERROR
expect(span?.status?.message).toBe("agent failure");
const conclusionAttr = span?.attributes?.find(a => a.key === "gh-aw.agent.conclusion");
expect(conclusionAttr?.value?.stringValue).toBe("failure");
fetchSpy.mockRestore();
delete process.env.GH_AW_AGENT_CONCLUSION;
});

it("records timed_out conclusion as STATUS_CODE_ERROR when GH_AW_AGENT_CONCLUSION is 'timed_out'", async () => {
process.env.OTEL_EXPORTER_OTLP_ENDPOINT = "http://localhost:14317";
process.env.GH_AW_AGENT_CONCLUSION = "timed_out";
let capturedBody;
const fetchSpy = vi.spyOn(global, "fetch").mockImplementation((_url, opts) => {
capturedBody = opts?.body;
return Promise.resolve(new Response(null, { status: 200 }));
});

await runConclusion();

const payload = JSON.parse(capturedBody);
const span = payload?.resourceSpans?.[0]?.scopeSpans?.[0]?.spans?.[0];
expect(span?.status?.code).toBe(2); // STATUS_CODE_ERROR
expect(span?.status?.message).toBe("agent timed_out");
const conclusionAttr = span?.attributes?.find(a => a.key === "gh-aw.agent.conclusion");
expect(conclusionAttr?.value?.stringValue).toBe("timed_out");
fetchSpy.mockRestore();
delete process.env.GH_AW_AGENT_CONCLUSION;
});

it("records success conclusion as STATUS_CODE_OK when GH_AW_AGENT_CONCLUSION is 'success'", async () => {
process.env.OTEL_EXPORTER_OTLP_ENDPOINT = "http://localhost:14317";
process.env.GH_AW_AGENT_CONCLUSION = "success";
let capturedBody;
const fetchSpy = vi.spyOn(global, "fetch").mockImplementation((_url, opts) => {
capturedBody = opts?.body;
return Promise.resolve(new Response(null, { status: 200 }));
});

await runConclusion();

const payload = JSON.parse(capturedBody);
const span = payload?.resourceSpans?.[0]?.scopeSpans?.[0]?.spans?.[0];
expect(span?.status?.code).toBe(1); // STATUS_CODE_OK
expect(span?.status?.message).toBeUndefined();
const conclusionAttr = span?.attributes?.find(a => a.key === "gh-aw.agent.conclusion");
expect(conclusionAttr?.value?.stringValue).toBe("success");
fetchSpy.mockRestore();
delete process.env.GH_AW_AGENT_CONCLUSION;
});

it("records cancelled conclusion as STATUS_CODE_OK when GH_AW_AGENT_CONCLUSION is 'cancelled'", async () => {
process.env.OTEL_EXPORTER_OTLP_ENDPOINT = "http://localhost:14317";
process.env.GH_AW_AGENT_CONCLUSION = "cancelled";
let capturedBody;
const fetchSpy = vi.spyOn(global, "fetch").mockImplementation((_url, opts) => {
capturedBody = opts?.body;
return Promise.resolve(new Response(null, { status: 200 }));
});

await runConclusion();

const payload = JSON.parse(capturedBody);
const span = payload?.resourceSpans?.[0]?.scopeSpans?.[0]?.spans?.[0];
expect(span?.status?.code).toBe(1); // STATUS_CODE_OK (cancelled is not an error)
const conclusionAttr = span?.attributes?.find(a => a.key === "gh-aw.agent.conclusion");
expect(conclusionAttr?.value?.stringValue).toBe("cancelled");
fetchSpy.mockRestore();
delete process.env.GH_AW_AGENT_CONCLUSION;
});

it("omits gh-aw.agent.conclusion attribute when GH_AW_AGENT_CONCLUSION is not set", async () => {
process.env.OTEL_EXPORTER_OTLP_ENDPOINT = "http://localhost:14317";
delete process.env.GH_AW_AGENT_CONCLUSION;
let capturedBody;
const fetchSpy = vi.spyOn(global, "fetch").mockImplementation((_url, opts) => {
capturedBody = opts?.body;
return Promise.resolve(new Response(null, { status: 200 }));
});

await runConclusion();

const payload = JSON.parse(capturedBody);
const span = payload?.resourceSpans?.[0]?.scopeSpans?.[0]?.spans?.[0];
expect(span?.status?.code).toBe(1); // STATUS_CODE_OK
const conclusionAttr = span?.attributes?.find(a => a.key === "gh-aw.agent.conclusion");
expect(conclusionAttr).toBeUndefined();
fetchSpy.mockRestore();
});
});
30 changes: 28 additions & 2 deletions actions/setup/js/send_otlp_span.cjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,8 @@ function buildAttr(key, value) {
* @property {string} serviceName - Value for the service.name resource attribute
* @property {string} [scopeVersion] - gh-aw version string (e.g. from GH_AW_INFO_VERSION)
* @property {Array<{key: string, value: object}>} attributes - Span attributes
* @property {number} [statusCode] - OTLP status code: 0=UNSET, 1=OK, 2=ERROR (defaults to 1)
* @property {string} [statusMessage] - Human-readable status message (included when statusCode is 2)
*/

/**
Expand All @@ -91,7 +93,13 @@ function buildAttr(key, value) {
* @param {OTLPSpanOptions} opts
* @returns {object} - Ready to be serialised as JSON and POSTed to `/v1/traces`
*/
function buildOTLPPayload({ traceId, spanId, parentSpanId, spanName, startMs, endMs, serviceName, scopeVersion, attributes }) {
function buildOTLPPayload({ traceId, spanId, parentSpanId, spanName, startMs, endMs, serviceName, scopeVersion, attributes, statusCode, statusMessage }) {
const code = typeof statusCode === "number" ? statusCode : 1; // STATUS_CODE_OK
/** @type {{ code: number, message?: string }} */
const status = { code };
if (statusMessage) {
Copy link

Copilot AI Apr 5, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

buildOTLPPayload adds status.message whenever statusMessage is truthy, but the new JSDoc says the message is only included when statusCode is 2 (ERROR). Either gate status.message on code === 2 (and optionally ignore/clear message for OK/UNSET) or update the JSDoc to match the actual behavior.

Suggested change
if (statusMessage) {
if (code === 2 && statusMessage) {

Copilot uses AI. Check for mistakes.
status.message = statusMessage;
}
return {
resourceSpans: [
{
Expand All @@ -110,7 +118,7 @@ function buildOTLPPayload({ traceId, spanId, parentSpanId, spanName, startMs, en
kind: 2, // SPAN_KIND_SERVER
startTimeUnixNano: toNanoString(startMs),
endTimeUnixNano: toNanoString(endMs),
status: { code: 1 }, // STATUS_CODE_OK
status,
attributes,
},
],
Expand Down Expand Up @@ -411,6 +419,9 @@ function readJSONIfExists(filePath) {
* - `OTEL_EXPORTER_OTLP_ENDPOINT` – collector endpoint
* - `OTEL_SERVICE_NAME` – service name (defaults to "gh-aw")
* - `GH_AW_EFFECTIVE_TOKENS` – total effective token count for the run
* - `GH_AW_AGENT_CONCLUSION` – agent job result ("success", "failure", "timed_out",
* "cancelled", "skipped"); when "failure" or "timed_out"
* the span status is set to STATUS_CODE_ERROR (2)
* - `INPUT_JOB_NAME` – job name; set automatically by GitHub Actions from the
* `job-name` action input
* - `GITHUB_AW_OTEL_TRACE_ID` – trace ID written to GITHUB_ENV by the setup step;
Expand Down Expand Up @@ -472,6 +483,16 @@ async function sendJobConclusionSpan(spanName, options = {}) {
const actor = process.env.GITHUB_ACTOR || "";
const repository = process.env.GITHUB_REPOSITORY || "";

// Agent conclusion is passed to downstream jobs via GH_AW_AGENT_CONCLUSION.
// Values: "success", "failure", "timed_out", "cancelled", "skipped".
const agentConclusion = process.env.GH_AW_AGENT_CONCLUSION || "";

// Mark the span as an error when the agent job failed or timed out.
const isAgentFailure = agentConclusion === "failure" || agentConclusion === "timed_out";
// STATUS_CODE_ERROR = 2, STATUS_CODE_OK = 1
const statusCode = isAgentFailure ? 2 : 1;
const statusMessage = isAgentFailure ? `agent ${agentConclusion}` : undefined;

const attributes = [buildAttr("gh-aw.workflow.name", workflowName), buildAttr("gh-aw.run.id", runId), buildAttr("gh-aw.run.actor", actor), buildAttr("gh-aw.repository", repository)];

if (jobName) attributes.push(buildAttr("gh-aw.job.name", jobName));
Expand All @@ -480,6 +501,9 @@ async function sendJobConclusionSpan(spanName, options = {}) {
if (!isNaN(effectiveTokens) && effectiveTokens > 0) {
attributes.push(buildAttr("gh-aw.effective_tokens", effectiveTokens));
}
if (agentConclusion) {
attributes.push(buildAttr("gh-aw.agent.conclusion", agentConclusion));
}

const payload = buildOTLPPayload({
traceId,
Expand All @@ -491,6 +515,8 @@ async function sendJobConclusionSpan(spanName, options = {}) {
serviceName,
scopeVersion: version,
attributes,
statusCode,
statusMessage,
});

await sendOTLPSpan(endpoint, payload);
Expand Down
Loading