From cce0da7e703d7d8ca5b93f1aba314aa92b1161e0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 27 May 2026 03:57:40 +0000 Subject: [PATCH 1/8] Initial plan From be9b4183c7a9de66720f8f9901c9756fa8115825 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 27 May 2026 04:13:38 +0000 Subject: [PATCH 2/8] feat: add dedicated JS outcome evaluators Co-authored-by: gh-aw-bot <259018956+gh-aw-bot@users.noreply.github.com> --- .github/workflows/requirements.txt | 1 + actions/setup/js/add_labels.cjs | 18 + actions/setup/js/evaluate_outcomes.cjs | 407 +++++++++++++++++- actions/setup/js/evaluate_outcomes.test.cjs | 125 ++++++ actions/setup/js/safe_output_manifest.cjs | 12 +- .../setup/js/safe_output_manifest.test.cjs | 4 +- .../src/content/docs/agent-factory-status.mdx | 1 + .../docs/reference/frontmatter-full.md | 306 ++++++++++++- 8 files changed, 827 insertions(+), 47 deletions(-) create mode 100644 actions/setup/js/evaluate_outcomes.test.cjs diff --git a/.github/workflows/requirements.txt b/.github/workflows/requirements.txt index c8ca0373942..af28d50e02a 100644 --- a/.github/workflows/requirements.txt +++ b/.github/workflows/requirements.txt @@ -1,4 +1,5 @@ "mempalace==3.2.0" +/tmp/gh-aw/agent/token-audit/site-packages \\\n markitdown-mcp numpy diff --git a/actions/setup/js/add_labels.cjs b/actions/setup/js/add_labels.cjs index da229a83018..17b54d248df 100644 --- a/actions/setup/js/add_labels.cjs +++ b/actions/setup/js/add_labels.cjs @@ -177,6 +177,23 @@ const main = createCountGatedHandler({ }; } + /** @type {string[]} */ + let labelsBefore = []; + try { + const { data: existingItem } = await githubClient.rest.issues.get({ + owner: repoParts.owner, + repo: repoParts.repo, + issue_number: itemNumber, + }); + labelsBefore = Array.isArray(existingItem?.labels) + ? existingItem.labels + .map(label => (typeof label === "string" ? label : label?.name)) + .filter(name => typeof name === "string" && name.trim() !== "") + : []; + } catch (error) { + core.info(`Unable to capture labels-before snapshot for ${contextType} #${itemNumber}: ${getErrorMessage(error)}`); + } + try { await withRetry( () => @@ -195,6 +212,7 @@ const main = createCountGatedHandler({ success: true, number: itemNumber, labelsAdded: uniqueLabels, + labelsBefore, contextType, }; } catch (error) { diff --git a/actions/setup/js/evaluate_outcomes.cjs b/actions/setup/js/evaluate_outcomes.cjs index 3fd30d87947..932ef532a9f 100644 --- a/actions/setup/js/evaluate_outcomes.cjs +++ b/actions/setup/js/evaluate_outcomes.cjs @@ -42,6 +42,24 @@ const SUMMARY_PATH = "/tmp/gh-aw/outcome-summary.json"; // --------------------------------------------------------------------------- const NOOP_TYPES = new Set(["noop", "missing_tool", "missing_data", "report_incomplete"]); +const DEFAULT_ISSUE_IMMEDIATE_CLOSE_WINDOW_SEC = 60 * 60; +const DEFAULT_LABEL_RETENTION_WINDOW_SEC = 24 * 60 * 60; + +/** + * Read a positive integer from env with fallback. + * @param {string} key + * @param {number} fallback + * @returns {number} + */ +function getEnvPositiveInt(key, fallback) { + const raw = process.env[key]; + const parsed = Number.parseInt(raw || "", 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; +} + +const ISSUE_IMMEDIATE_CLOSE_WINDOW_SEC = getEnvPositiveInt("OUTCOME_ISSUE_IMMEDIATE_CLOSE_WINDOW_SEC", DEFAULT_ISSUE_IMMEDIATE_CLOSE_WINDOW_SEC); +const LABEL_RETENTION_WINDOW_SEC = getEnvPositiveInt("OUTCOME_LABEL_RETENTION_WINDOW_SEC", DEFAULT_LABEL_RETENTION_WINDOW_SEC); + // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- @@ -170,16 +188,343 @@ function secondsBetween(from, to) { * @property {boolean} zero_touch */ +/** + * @typedef {object} EvaluateDeps + * @property {(endpoint: string) => any} [ghAPI] + * @property {number} [nowMs] + */ + +/** + * Convert issue/PR reaction summary into aggregate counts. + * @param {any} reactions + * @returns {{total: number|null, positive: number|null, negative: number|null}} + */ +function summarizeReactions(reactions) { + if (!reactions || typeof reactions !== "object") { + return { total: null, positive: null, negative: null }; + } + const positive = (reactions["+1"] || 0) + (reactions.heart || 0) + (reactions.hooray || 0) + (reactions.rocket || 0); + const negative = (reactions["-1"] || 0) + (reactions.confused || 0); + const total = reactions.total_count != null ? reactions.total_count : positive + negative + (reactions.laugh || 0) + (reactions.eyes || 0); + return { total, positive, negative }; +} + +/** + * @param {string} url + * @returns {number|null} + */ +function parseIssueNumberFromURL(url) { + const match = String(url || "").match(/\/(?:issues|pull)\/(\d+)/); + if (!match) return null; + const num = Number.parseInt(match[1], 10); + return Number.isInteger(num) && num > 0 ? num : null; +} + +/** + * @param {string} url + * @returns {string} + */ +function parseCommentIDFromURL(url) { + const text = String(url || ""); + const issueCommentMatch = text.match(/#issuecomment-(\d+)/); + if (issueCommentMatch) return issueCommentMatch[1]; + const pathMatch = text.match(/\/comments\/(\d+)/); + return pathMatch ? pathMatch[1] : ""; +} + +/** + * @param {any} issue + * @returns {boolean} + */ +function hasIssueReactions(issue) { + const summary = summarizeReactions(issue?.reactions); + return typeof summary.total === "number" && summary.total > 0; +} + +/** + * Evaluate `create_issue`. + * @param {object} item + * @param {string} itemRepo + * @param {string} timestamp + * @param {EvalResult} out + * @param {(endpoint: string) => any} apiGet + * @param {number} nowMs + * @returns {EvalResult} + */ +function evaluateCreateIssue(item, itemRepo, timestamp, out, apiGet, nowMs) { + const num = parseIssueNumberFromURL(item.url || ""); + if (!num) { + out.result = "unknown"; + out.detail = "unknown: issue number not found"; + setPendingAge(out, timestamp, nowMs); + return out; + } + const issue = apiGet(`repos/${itemRepo}/issues/${num}`); + if (!issue || !issue.state) { + out.result = "unknown"; + out.detail = "unknown: issue api error"; + setPendingAge(out, timestamp, nowMs); + return out; + } + + out.comments = typeof issue.comments === "number" ? issue.comments : null; + const reactionSummary = summarizeReactions(issue.reactions); + out.reactions_total = reactionSummary.total; + out.reactions_positive = reactionSummary.positive; + out.reactions_negative = reactionSummary.negative; + + const authorLogin = issue.user && typeof issue.user.login === "string" ? issue.user.login : ""; + const comments = apiGet(`repos/${itemRepo}/issues/${num}/comments`); + const hasNonAuthorComment = + Array.isArray(comments) && + comments.some(c => c && c.user && typeof c.user.login === "string" && c.user.login !== authorLogin && c.user.login !== ""); + + const timeline = apiGet(`repos/${itemRepo}/issues/${num}/timeline`); + const timelineEvents = Array.isArray(timeline) ? timeline : []; + let hasMergedPRReference = false; + let hasCommitReference = false; + let hasClosingActionReference = false; + let closeActor = ""; + for (const event of timelineEvents) { + if (!event || typeof event !== "object") continue; + const eventType = typeof event.event === "string" ? event.event : ""; + if (eventType === "closed") { + hasClosingActionReference = true; + const actorLogin = event.actor && typeof event.actor.login === "string" ? event.actor.login : ""; + if (actorLogin) closeActor = actorLogin; + } + if (eventType === "referenced" && event.commit_id) { + hasCommitReference = true; + } + if (eventType !== "cross-referenced") continue; + const sourceIssue = event.source && event.source.issue; + const prNumber = sourceIssue && typeof sourceIssue.number === "number" ? sourceIssue.number : null; + if (!prNumber) continue; + const pr = apiGet(`repos/${itemRepo}/pulls/${prNumber}`); + if (pr && pr.merged === true) { + hasMergedPRReference = true; + } + } + + if (issue.state === "open" && (hasMergedPRReference || hasCommitReference || hasClosingActionReference)) { + out.result = "accepted"; + out.detail = "accepted:strong"; + return out; + } + + if (issue.state === "open" && (hasNonAuthorComment || hasIssueReactions(issue))) { + out.result = "accepted"; + out.detail = "accepted:medium"; + return out; + } + + if (issue.state === "closed" && issue.created_at && issue.closed_at) { + out.resolution_sec = secondsBetween(issue.created_at, issue.closed_at); + if (typeof out.resolution_sec === "number" && out.resolution_sec <= ISSUE_IMMEDIATE_CLOSE_WINDOW_SEC && closeActor && closeActor !== authorLogin) { + out.result = "rejected"; + out.detail = "rejected:strong"; + return out; + } + } + + if (issue.state === "closed") { + const hasActivity = (typeof issue.comments === "number" && issue.comments > 0) || hasIssueReactions(issue) || hasMergedPRReference || hasCommitReference; + if (!hasActivity) { + out.result = "rejected"; + out.detail = "rejected:medium"; + return out; + } + out.result = "unknown"; + out.detail = "unknown: closed with activity"; + return out; + } + + if (issue.state === "open") { + out.result = "pending"; + out.detail = "pending: open with no engagement"; + setPendingAge(out, timestamp, nowMs); + return out; + } + + out.result = "unknown"; + out.detail = "unknown: unsupported issue state"; + setPendingAge(out, timestamp, nowMs); + return out; +} + +/** + * Evaluate `add_comment`. + * @param {object} item + * @param {string} itemRepo + * @param {string} timestamp + * @param {EvalResult} out + * @param {(endpoint: string) => any} apiGet + * @param {number} nowMs + * @returns {EvalResult} + */ +function evaluateAddComment(item, itemRepo, timestamp, out, apiGet, nowMs) { + const commentID = parseCommentIDFromURL(item.url || ""); + const issueNum = parseIssueNumberFromURL(item.url || ""); + if (!commentID || !issueNum) { + out.result = "unknown"; + out.detail = "unknown: missing comment or issue id"; + setPendingAge(out, timestamp, nowMs); + return out; + } + + const comment = apiGet(`repos/${itemRepo}/issues/comments/${commentID}`); + if (!comment || !comment.id) { + out.result = "rejected"; + out.detail = "rejected:strong deleted"; + return out; + } + + const commentAuthor = comment.user && typeof comment.user.login === "string" ? comment.user.login : ""; + const commentCreatedAt = typeof comment.created_at === "string" ? comment.created_at : ""; + const reactionSummary = summarizeReactions(comment.reactions); + out.reactions_total = reactionSummary.total; + out.reactions_positive = reactionSummary.positive; + out.reactions_negative = reactionSummary.negative; + + const issueComments = apiGet(`repos/${itemRepo}/issues/${issueNum}/comments`); + const commentURL = String(item.url || ""); + let hasReply = false; + let hasQuote = false; + let threadActedOn = false; + if (Array.isArray(issueComments)) { + for (const c of issueComments) { + if (!c || typeof c !== "object") continue; + if (typeof c.created_at === "string" && commentCreatedAt && c.created_at > commentCreatedAt) { + threadActedOn = true; + } + const body = typeof c.body === "string" ? c.body : ""; + const cAuthor = c.user && typeof c.user.login === "string" ? c.user.login : ""; + if (cAuthor && cAuthor !== commentAuthor && typeof c.created_at === "string" && commentCreatedAt && c.created_at > commentCreatedAt) { + hasReply = true; + } + if (body.includes(`#issuecomment-${commentID}`) || (commentURL && body.includes(commentURL))) { + hasQuote = true; + } + } + } + + if ((typeof reactionSummary.total === "number" && reactionSummary.total > 0) || hasReply || hasQuote) { + out.result = "accepted"; + out.detail = "accepted:strong"; + return out; + } + + if (threadActedOn) { + out.result = "accepted"; + out.detail = "accepted:medium"; + return out; + } + + out.result = "pending"; + out.detail = "pending: no follow-up"; + setPendingAge(out, timestamp, nowMs); + return out; +} + +/** + * Evaluate `add_labels`. + * @param {object} item + * @param {string} itemRepo + * @param {string} timestamp + * @param {EvalResult} out + * @param {(endpoint: string) => any} apiGet + * @param {number} nowMs + * @returns {EvalResult} + */ +function evaluateAddLabels(item, itemRepo, timestamp, out, apiGet, nowMs) { + const num = parseIssueNumberFromURL(item.url || ""); + if (!num) { + out.result = "unknown"; + out.detail = "unknown: issue number not found"; + setPendingAge(out, timestamp, nowMs); + return out; + } + + const labelsBefore = Array.isArray(item.labelsBefore) + ? item.labelsBefore.map(l => String(l || "").trim()).filter(Boolean) + : []; + const labelsAdded = Array.isArray(item.labelsAdded) + ? item.labelsAdded.map(l => String(l || "").trim()).filter(Boolean) + : Array.isArray(item.labels) + ? item.labels.map(l => String(l || "").trim()).filter(Boolean) + : []; + + if (labelsBefore.length === 0 || labelsAdded.length === 0) { + out.result = "unknown"; + out.detail = "unknown: missing persisted label before-state"; + setPendingAge(out, timestamp, nowMs); + return out; + } + + const labels = apiGet(`repos/${itemRepo}/issues/${num}/labels`); + if (!Array.isArray(labels)) { + out.result = "unknown"; + out.detail = "unknown: labels api error"; + setPendingAge(out, timestamp, nowMs); + return out; + } + + const currentLabels = new Set(labels.map(l => (l && typeof l.name === "string" ? l.name : "")).filter(Boolean)); + const trackedAdded = labelsAdded.filter(l => !labelsBefore.includes(l)); + const removed = trackedAdded.filter(l => !currentLabels.has(l)); + + const nowEpoch = Math.floor(nowMs / 1000); + const createdEpoch = isoToEpoch(timestamp || ""); + const elapsedSec = createdEpoch === null ? null : nowEpoch - createdEpoch; + if (elapsedSec === null || elapsedSec < LABEL_RETENTION_WINDOW_SEC) { + out.result = "pending"; + out.detail = "pending: retention window not elapsed"; + setPendingAge(out, timestamp, nowMs); + return out; + } + + if (removed.length === 0) { + out.result = "accepted"; + out.detail = "accepted:strong"; + return out; + } + + const issue = apiGet(`repos/${itemRepo}/issues/${num}`); + const issueAuthor = issue && issue.user && typeof issue.user.login === "string" ? issue.user.login : ""; + const events = apiGet(`repos/${itemRepo}/issues/${num}/events`); + const eventList = Array.isArray(events) ? events : []; + const removedByNonAuthor = eventList.some(event => { + if (!event || event.event !== "unlabeled") return false; + const removedLabel = event.label && typeof event.label.name === "string" ? event.label.name : ""; + if (!removed.includes(removedLabel)) return false; + const actor = event.actor && typeof event.actor.login === "string" ? event.actor.login : ""; + return actor !== "" && actor !== issueAuthor; + }); + + if (removedByNonAuthor) { + out.result = "rejected"; + out.detail = "rejected:strong"; + return out; + } + + out.result = "unknown"; + out.detail = "unknown: labels removed with ambiguous actor"; + return out; +} + /** * Evaluate a single safe-output item against the GitHub API. * @param {object} item * @param {string} defaultRepo + * @param {EvaluateDeps} [deps] * @returns {EvalResult} */ -function evaluateItem(item, defaultRepo) { +function evaluateItem(item, defaultRepo, deps = {}) { const url = item.url || ""; const itemRepo = item.repo || defaultRepo; const timestamp = item.timestamp || ""; + const apiGet = typeof deps.ghAPI === "function" ? deps.ghAPI : ghAPI; + const nowMs = typeof deps.nowMs === "number" ? deps.nowMs : Date.now(); /** @type {EvalResult} */ const out = { @@ -200,18 +545,28 @@ function evaluateItem(item, defaultRepo) { if (!url) { out.detail = "no url"; - setPendingAge(out, timestamp); + setPendingAge(out, timestamp, nowMs); return out; } + if (item.type === "create_issue") { + return evaluateCreateIssue(item, itemRepo, timestamp, out, apiGet, nowMs); + } + if (item.type === "add_comment") { + return evaluateAddComment(item, itemRepo, timestamp, out, apiGet, nowMs); + } + if (item.type === "add_labels") { + return evaluateAddLabels(item, itemRepo, timestamp, out, apiGet, nowMs); + } + // Issues / issue-comments const issueMatch = url.match(/\/(?:issues|pull)\/(\d+)/); if (/\/issues\/\d+|\/issuecomment-/.test(url) && issueMatch) { const num = issueMatch[1]; - const data = ghAPI(`repos/${itemRepo}/issues/${num}`); + const data = apiGet(`repos/${itemRepo}/issues/${num}`); if (!data || !data.state) { out.detail = "api error"; - setPendingAge(out, timestamp); + setPendingAge(out, timestamp, nowMs); return out; } out.result = "accepted"; @@ -220,12 +575,10 @@ function evaluateItem(item, defaultRepo) { // Reactions on issues if (data.reactions && typeof data.reactions === "object") { - const r = data.reactions; - const positive = (r["+1"] || 0) + (r.heart || 0) + (r.hooray || 0) + (r.rocket || 0); - const negative = (r["-1"] || 0) + (r.confused || 0); - out.reactions_total = r.total_count != null ? r.total_count : positive + negative + (r.laugh || 0) + (r.eyes || 0); - out.reactions_positive = positive; - out.reactions_negative = negative; + const summary = summarizeReactions(data.reactions); + out.reactions_total = summary.total; + out.reactions_positive = summary.positive; + out.reactions_negative = summary.negative; } if (data.state === "closed" && data.created_at && data.closed_at) { @@ -238,10 +591,10 @@ function evaluateItem(item, defaultRepo) { const prMatch = url.match(/\/pull\/(\d+)/); if (prMatch) { const num = prMatch[1]; - const data = ghAPI(`repos/${itemRepo}/pulls/${num}`); + const data = apiGet(`repos/${itemRepo}/pulls/${num}`); if (!data || !data.state) { out.detail = "api error"; - setPendingAge(out, timestamp); + setPendingAge(out, timestamp, nowMs); return out; } @@ -254,12 +607,10 @@ function evaluateItem(item, defaultRepo) { // Reactions if (data.reactions && typeof data.reactions === "object") { - const r = data.reactions; - const positive = (r["+1"] || 0) + (r.heart || 0) + (r.hooray || 0) + (r.rocket || 0); - const negative = (r["-1"] || 0) + (r.confused || 0); - out.reactions_total = r.total_count != null ? r.total_count : positive + negative + (r.laugh || 0) + (r.eyes || 0); - out.reactions_positive = positive; - out.reactions_negative = negative; + const summary = summarizeReactions(data.reactions); + out.reactions_total = summary.total; + out.reactions_positive = summary.positive; + out.reactions_negative = summary.negative; } // Zero-touch: merged with no human review comments and no issue-level comments @@ -282,10 +633,10 @@ function evaluateItem(item, defaultRepo) { } else if (data.state === "open") { out.result = "pending"; out.detail = "open"; - setPendingAge(out, timestamp); + setPendingAge(out, timestamp, nowMs); } else { out.detail = "api error"; - setPendingAge(out, timestamp); + setPendingAge(out, timestamp, nowMs); } return out; } @@ -300,12 +651,13 @@ function evaluateItem(item, defaultRepo) { * Set pending_age_sec on the result if the item has a timestamp. * @param {EvalResult} out * @param {string} timestamp + * @param {number} [nowMs] */ -function setPendingAge(out, timestamp) { +function setPendingAge(out, timestamp, nowMs = Date.now()) { if (!timestamp) return; const itemEpoch = isoToEpoch(timestamp); if (itemEpoch === null) return; - out.pending_age_sec = Math.floor(Date.now() / 1000) - itemEpoch; + out.pending_age_sec = Math.floor(nowMs / 1000) - itemEpoch; } // --------------------------------------------------------------------------- @@ -534,4 +886,13 @@ if (require.main === module) { main(); } -module.exports = { main, evaluateItem, readJSONL, secondsBetween, isoToEpoch }; +module.exports = { + main, + evaluateItem, + evaluateCreateIssue, + evaluateAddComment, + evaluateAddLabels, + readJSONL, + secondsBetween, + isoToEpoch, +}; diff --git a/actions/setup/js/evaluate_outcomes.test.cjs b/actions/setup/js/evaluate_outcomes.test.cjs new file mode 100644 index 00000000000..c4113a17aed --- /dev/null +++ b/actions/setup/js/evaluate_outcomes.test.cjs @@ -0,0 +1,125 @@ +import { describe, it, expect } from "vitest"; + +const { evaluateItem } = require("./evaluate_outcomes.cjs"); + +function createAPIStub(fixtures) { + return endpoint => fixtures[endpoint] ?? null; +} + +describe("evaluate_outcomes type-specific evaluators", () => { + it("evaluates create_issue outcomes using engagement signals", () => { + const nowMs = Date.parse("2026-05-27T04:00:00Z"); + const item = { + type: "create_issue", + url: "https://github.com/acme/repo/issues/12", + timestamp: "2026-05-25T04:00:00Z", + }; + const ghAPI = createAPIStub({ + "repos/acme/repo/issues/12": { state: "open", comments: 0, reactions: { total_count: 0 }, user: { login: "author" } }, + "repos/acme/repo/issues/12/comments": [], + "repos/acme/repo/issues/12/timeline": [{ event: "cross-referenced", source: { issue: { number: 88 } } }], + "repos/acme/repo/pulls/88": { merged: true }, + }); + expect(evaluateItem(item, "acme/repo", { ghAPI, nowMs }).detail).toBe("accepted:strong"); + + const noEngagement = createAPIStub({ + "repos/acme/repo/issues/12": { state: "open", comments: 0, reactions: { total_count: 0 }, user: { login: "author" } }, + "repos/acme/repo/issues/12/comments": [], + "repos/acme/repo/issues/12/timeline": [], + }); + expect(evaluateItem(item, "acme/repo", { ghAPI: noEngagement, nowMs }).result).toBe("pending"); + }); + + it("classifies create_issue immediate close and close-without-activity as rejected", () => { + const item = { + type: "create_issue", + url: "https://github.com/acme/repo/issues/7", + timestamp: "2026-05-26T00:00:00Z", + }; + const immediateClose = createAPIStub({ + "repos/acme/repo/issues/7": { + state: "closed", + comments: 0, + reactions: { total_count: 0 }, + created_at: "2026-05-26T00:00:00Z", + closed_at: "2026-05-26T00:05:00Z", + user: { login: "author" }, + }, + "repos/acme/repo/issues/7/comments": [], + "repos/acme/repo/issues/7/timeline": [{ event: "closed", actor: { login: "reviewer" } }], + }); + expect(evaluateItem(item, "acme/repo", { ghAPI: immediateClose }).detail).toBe("rejected:strong"); + + const closedNoActivity = createAPIStub({ + "repos/acme/repo/issues/7": { + state: "closed", + comments: 0, + reactions: { total_count: 0 }, + created_at: "2026-05-26T00:00:00Z", + closed_at: "2026-05-27T00:00:00Z", + user: { login: "author" }, + }, + "repos/acme/repo/issues/7/comments": [], + "repos/acme/repo/issues/7/timeline": [], + }); + expect(evaluateItem(item, "acme/repo", { ghAPI: closedNoActivity }).detail).toBe("rejected:medium"); + }); + + it("evaluates add_comment deletion, engagement, pending, and unknown", () => { + const base = { + type: "add_comment", + url: "https://github.com/acme/repo/issues/21#issuecomment-1001", + timestamp: "2026-05-26T00:00:00Z", + }; + + const deleted = createAPIStub({ + "repos/acme/repo/issues/comments/1001": null, + }); + expect(evaluateItem(base, "acme/repo", { ghAPI: deleted }).detail).toContain("rejected:strong"); + + const reacted = createAPIStub({ + "repos/acme/repo/issues/comments/1001": { id: 1001, created_at: "2026-05-26T00:00:00Z", user: { login: "copilot" }, reactions: { total_count: 2 } }, + "repos/acme/repo/issues/21/comments": [], + }); + expect(evaluateItem(base, "acme/repo", { ghAPI: reacted }).detail).toBe("accepted:strong"); + + const actedOnThread = createAPIStub({ + "repos/acme/repo/issues/comments/1001": { id: 1001, created_at: "2026-05-26T00:00:00Z", user: { login: "copilot" }, reactions: { total_count: 0 } }, + "repos/acme/repo/issues/21/comments": [{ created_at: "2026-05-26T00:01:00Z", user: { login: "copilot" }, body: "follow-up" }], + }); + expect(evaluateItem(base, "acme/repo", { ghAPI: actedOnThread }).detail).toBe("accepted:medium"); + + const pending = createAPIStub({ + "repos/acme/repo/issues/comments/1001": { id: 1001, created_at: "2026-05-26T00:00:00Z", user: { login: "copilot" }, reactions: { total_count: 0 } }, + "repos/acme/repo/issues/21/comments": [], + }); + expect(evaluateItem(base, "acme/repo", { ghAPI: pending }).result).toBe("pending"); + + expect(evaluateItem({ type: "add_comment", url: "https://github.com/acme/repo/issues/21" }, "acme/repo", { ghAPI: pending }).result).toBe("unknown"); + }); + + it("evaluates add_labels retention using persisted before-state labels", () => { + const item = { + type: "add_labels", + url: "https://github.com/acme/repo/issues/42", + timestamp: "2026-05-25T00:00:00Z", + labelsBefore: ["bug"], + labelsAdded: ["bug", "triage"], + }; + + const retained = createAPIStub({ + "repos/acme/repo/issues/42/labels": [{ name: "bug" }, { name: "triage" }], + }); + expect(evaluateItem(item, "acme/repo", { ghAPI: retained, nowMs: Date.parse("2026-05-27T00:00:00Z") }).detail).toBe("accepted:strong"); + + const removedByNonAuthor = createAPIStub({ + "repos/acme/repo/issues/42/labels": [{ name: "bug" }], + "repos/acme/repo/issues/42": { user: { login: "copilot" } }, + "repos/acme/repo/issues/42/events": [{ event: "unlabeled", actor: { login: "maintainer" }, label: { name: "triage" } }], + }); + expect(evaluateItem(item, "acme/repo", { ghAPI: removedByNonAuthor, nowMs: Date.parse("2026-05-27T00:00:00Z") }).detail).toBe("rejected:strong"); + + expect(evaluateItem(item, "acme/repo", { ghAPI: retained, nowMs: Date.parse("2026-05-25T00:01:00Z") }).result).toBe("pending"); + expect(evaluateItem({ ...item, labelsBefore: [] }, "acme/repo", { ghAPI: retained, nowMs: Date.parse("2026-05-27T00:00:00Z") }).result).toBe("unknown"); + }); +}); diff --git a/actions/setup/js/safe_output_manifest.cjs b/actions/setup/js/safe_output_manifest.cjs index c00a1bce0a8..a1672ce127f 100644 --- a/actions/setup/js/safe_output_manifest.cjs +++ b/actions/setup/js/safe_output_manifest.cjs @@ -47,6 +47,8 @@ const NOT_LOGGED_TYPES = new Set(["noop", "missing_tool", "missing_data", "repor * @property {number} [number] - Issue/PR/discussion number if applicable * @property {string} [repo] - Repository slug (owner/repo) if applicable * @property {string} [temporaryId] - Temporary ID assigned to this item, if any + * @property {string[]} [labelsAdded] - Labels added by add_labels handler + * @property {string[]} [labelsBefore] - Labels present before add_labels execution * @property {string} timestamp - ISO 8601 timestamp of creation */ @@ -57,7 +59,7 @@ const NOT_LOGGED_TYPES = new Set(["noop", "missing_tool", "missing_data", "repor * It is designed to be easily testable by accepting the file path as a parameter. * * @param {string} [manifestFile] - Path to the manifest file (defaults to MANIFEST_FILE_PATH) - * @returns {(item: {type: string, url?: string, number?: number, repo?: string, temporaryId?: string}) => void} Logger function + * @returns {(item: {type: string, url?: string, number?: number, repo?: string, temporaryId?: string, labelsAdded?: string[], labelsBefore?: string[]}) => void} Logger function */ function createManifestLogger(manifestFile = MANIFEST_FILE_PATH) { // Touch the file immediately so it exists for artifact upload @@ -67,7 +69,7 @@ function createManifestLogger(manifestFile = MANIFEST_FILE_PATH) { /** * Log an executed safe output item to the manifest file. * - * @param {{type: string, url?: string, number?: number, repo?: string, temporaryId?: string}} item - Executed item details + * @param {{type: string, url?: string, number?: number, repo?: string, temporaryId?: string, labelsAdded?: string[], labelsBefore?: string[]}} item - Executed item details */ return function logCreatedItem(item) { if (!item) return; @@ -79,6 +81,8 @@ function createManifestLogger(manifestFile = MANIFEST_FILE_PATH) { ...(item.number != null ? { number: item.number } : {}), ...(item.repo ? { repo: item.repo } : {}), ...(item.temporaryId ? { temporaryId: item.temporaryId } : {}), + ...(Array.isArray(item.labelsAdded) && item.labelsAdded.length > 0 ? { labelsAdded: item.labelsAdded } : {}), + ...(Array.isArray(item.labelsBefore) ? { labelsBefore: item.labelsBefore } : {}), timestamp: new Date().toISOString(), }; @@ -120,7 +124,7 @@ function ensureManifestExists(manifestFile = MANIFEST_FILE_PATH) { * * @param {string} type - The handler type (e.g., "create_issue") * @param {any} result - The handler result object - * @returns {{type: string, url?: string, number?: number, repo?: string, temporaryId?: string}|null} + * @returns {{type: string, url?: string, number?: number, repo?: string, temporaryId?: string, labelsAdded?: string[], labelsBefore?: string[]}|null} */ function extractCreatedItemFromResult(type, result) { if (!result || NOT_LOGGED_TYPES.has(type)) return null; @@ -137,6 +141,8 @@ function extractCreatedItemFromResult(type, result) { ...(result.number != null ? { number: result.number } : {}), ...(result.repo ? { repo: result.repo } : {}), ...(result.temporaryId ? { temporaryId: result.temporaryId } : {}), + ...(Array.isArray(result.labelsAdded) ? { labelsAdded: result.labelsAdded } : {}), + ...(Array.isArray(result.labelsBefore) ? { labelsBefore: result.labelsBefore } : {}), }; } diff --git a/actions/setup/js/safe_output_manifest.test.cjs b/actions/setup/js/safe_output_manifest.test.cjs index 3e0fe707646..4156725244e 100644 --- a/actions/setup/js/safe_output_manifest.test.cjs +++ b/actions/setup/js/safe_output_manifest.test.cjs @@ -245,12 +245,14 @@ describe("safe_output_manifest", () => { }); it("should extract item from add_labels result (modification type without url)", () => { - const result = { success: true, number: 20875, labelsAdded: ["bug", "cli"], contextType: "issue" }; + const result = { success: true, number: 20875, labelsAdded: ["bug", "cli"], labelsBefore: ["bug"], contextType: "issue" }; const item = extractCreatedItemFromResult("add_labels", result); expect(item).not.toBeNull(); expect(item.type).toBe("add_labels"); expect(item.url).toBeUndefined(); expect(item.number).toBe(20875); + expect(item.labelsAdded).toEqual(["bug", "cli"]); + expect(item.labelsBefore).toEqual(["bug"]); }); it("should extract item from close_issue result (modification type with url)", () => { diff --git a/docs/src/content/docs/agent-factory-status.mdx b/docs/src/content/docs/agent-factory-status.mdx index 26165a4f5f9..de3608883ce 100644 --- a/docs/src/content/docs/agent-factory-status.mdx +++ b/docs/src/content/docs/agent-factory-status.mdx @@ -194,6 +194,7 @@ These are experimental agentic workflows used by the GitHub Next team to learn, | [Smoke Agent: public/approved](https://github.com/github/gh-aw/blob/main/.github/workflows/smoke-agent-public-approved.md) | claude | [![Smoke Agent: public/approved](https://github.com/github/gh-aw/actions/workflows/smoke-agent-public-approved.lock.yml/badge.svg)](https://github.com/github/gh-aw/actions/workflows/smoke-agent-public-approved.lock.yml) | - | - | | [Smoke Agent: public/none](https://github.com/github/gh-aw/blob/main/.github/workflows/smoke-agent-public-none.md) | claude | [![Smoke Agent: public/none](https://github.com/github/gh-aw/actions/workflows/smoke-agent-public-none.lock.yml/badge.svg)](https://github.com/github/gh-aw/actions/workflows/smoke-agent-public-none.lock.yml) | - | - | | [Smoke Agent: scoped/approved](https://github.com/github/gh-aw/blob/main/.github/workflows/smoke-agent-scoped-approved.md) | claude | [![Smoke Agent: scoped/approved](https://github.com/github/gh-aw/actions/workflows/smoke-agent-scoped-approved.lock.yml/badge.svg)](https://github.com/github/gh-aw/actions/workflows/smoke-agent-scoped-approved.lock.yml) | - | - | +| [Smoke Antigravity](https://github.com/github/gh-aw/blob/main/.github/workflows/smoke-antigravity.md) | antigravity | [![Smoke Antigravity](https://github.com/github/gh-aw/actions/workflows/smoke-antigravity.lock.yml/badge.svg)](https://github.com/github/gh-aw/actions/workflows/smoke-antigravity.lock.yml) | - | - | | [Smoke Call Workflow](https://github.com/github/gh-aw/blob/main/.github/workflows/smoke-call-workflow.md) | codex | [![Smoke Call Workflow](https://github.com/github/gh-aw/actions/workflows/smoke-call-workflow.lock.yml/badge.svg)](https://github.com/github/gh-aw/actions/workflows/smoke-call-workflow.lock.yml) | - | - | | [Smoke CI](https://github.com/github/gh-aw/blob/main/.github/workflows/smoke-ci.md) | copilot | [![Smoke CI](https://github.com/github/gh-aw/actions/workflows/smoke-ci.lock.yml/badge.svg)](https://github.com/github/gh-aw/actions/workflows/smoke-ci.lock.yml) | - | - | | [Smoke Claude](https://github.com/github/gh-aw/blob/main/.github/workflows/smoke-claude.md) | claude | [![Smoke Claude](https://github.com/github/gh-aw/actions/workflows/smoke-claude.lock.yml/badge.svg)](https://github.com/github/gh-aw/actions/workflows/smoke-claude.lock.yml) | - | - | diff --git a/docs/src/content/docs/reference/frontmatter-full.md b/docs/src/content/docs/reference/frontmatter-full.md index 6c5d247b77a..fe1c42eec28 100644 --- a/docs/src/content/docs/reference/frontmatter-full.md +++ b/docs/src/content/docs/reference/frontmatter-full.md @@ -1365,12 +1365,21 @@ on: statuses: "read" # Controls the stale lock file check in the activation job. Set to false to - # disable the check, true (default) to enable frontmatter hash checking, or - # "full" to check both frontmatter and body hashes. Use "full" when prompt-body - # edits should also trigger recompilation detection. + # disable the check, true (default) to enable frontmatter hash checking, or "full" + # to check both frontmatter and body hashes. Use "full" when prompt-body edits + # should also trigger recompilation detection. Useful when the workflow source + # files are managed outside the default GitHub repo context (e.g. cross-repo org + # rulesets) and the stale check is not needed (set false), or when comprehensive + # drift detection is required (set "full"). # (optional) + # Accepted formats: + + # Format 1: boolean stale-check: true + # Format 2: string + stale-check: "full" + # GitHub token permissions for the workflow. Controls what the GITHUB_TOKEN can # access during execution. Use the principle of least privilege - only grant the # minimum permissions needed. @@ -2046,6 +2055,11 @@ engine: # (optional) model: "example-value" + # Claude permission mode override. Defaults to acceptEdits (or auto when + # tools.edit is false). + # (optional) + permission-mode: "auto" + # Maximum number of chat iterations per run. Helps prevent runaway loops and # control costs. Has sensible defaults and can typically be omitted. Note: Only # supported by the claude engine. @@ -2930,7 +2944,10 @@ tools: # Format 1: Enable edit tool edit: null - # Format 2: Edit tool configuration object + # Format 2: Boolean to explicitly enable (true) or disable (false) the edit tool. + edit: true + + # Format 3: Edit tool configuration object edit: {} @@ -4538,22 +4555,25 @@ safe-outputs: # (optional) github-token-for-extra-empty-commit: "example-value" - # Controls protected-file protection. String form: blocked (default), allowed, or - # fallback-to-issue — or a GitHub Actions expression for reusable workflows. - # Object form: { policy, exclude } to customise the protected-file set. + # Controls protected-file protection. String form: request_review (default), + # blocked, allowed, or fallback-to-issue — or a GitHub Actions expression for + # reusable workflows. Object form: { policy, exclude } to customise the + # protected-file set. # (optional) # Accepted formats: - # Format 1: Controls protected-file protection. blocked (default): hard-block any - # patch that modifies package manifests (e.g. package.json, go.mod), engine - # instruction files (e.g. AGENTS.md, CLAUDE.md) or .github/ files. allowed: allow - # all changes. fallback-to-issue: push the branch but create a review issue - # instead of a PR, so a human can review the manifest changes before merging. + # Format 1: Controls protected-file protection. request_review (default): create + # the PR but prepend a caution block and submit a REQUEST_CHANGES review for + # manual scrutiny. blocked: hard-block any patch that modifies package manifests + # (e.g. package.json, go.mod), engine instruction files (e.g. AGENTS.md, + # CLAUDE.md) or .github/ files. allowed: allow all changes. fallback-to-issue: + # push the branch but create a review issue instead of a PR, so a human can review + # the manifest changes before merging. protected-files: "blocked" - # Format 2: GitHub Actions expression that resolves to 'blocked', 'allowed', or - # 'fallback-to-issue' at runtime. Use in reusable workflow_call workflows to - # parameterise the policy per caller. + # Format 2: GitHub Actions expression that resolves to 'blocked', 'allowed', + # 'fallback-to-issue', or 'request_review' at runtime. Use in reusable + # workflow_call workflows to parameterise the policy per caller. protected-files: "example-value" # Format 3: Object form for granular control over the protected-file set. Use the @@ -4563,13 +4583,14 @@ safe-outputs: # (optional) # Accepted formats: - # Format 1: Protection policy. blocked (default): hard-block any patch that - # modifies protected files. allowed: allow all changes. fallback-to-issue: push - # the branch but create a review issue instead of a PR. + # Format 1: Protection policy. request_review (default): create the PR but prepend + # a caution block and submit a REQUEST_CHANGES review. blocked: hard-block any + # patch that modifies protected files. allowed: allow all changes. + # fallback-to-issue: push the branch but create a review issue instead of a PR. policy: "blocked" - # Format 2: GitHub Actions expression that resolves to 'blocked', 'allowed', or - # 'fallback-to-issue' at runtime. + # Format 2: GitHub Actions expression that resolves to 'blocked', 'allowed', + # 'fallback-to-issue', or 'request_review' at runtime. policy: "example-value" # List of filenames or path prefixes to remove from the default protected-file @@ -5004,6 +5025,251 @@ safe-outputs: # 10) autofix-code-scanning-alert: null + # Enable AI agents to create GitHub Check Runs that surface analysis results in + # the PR checks UI. Requires checks: write permission. + # (optional) + # Accepted formats: + + # Format 1: Configuration for creating GitHub Check Runs to surface agent analysis + # results on commits and pull requests + create-check-run: + # Check run name shown in the GitHub Checks UI (e.g., 'Security Analysis'). If + # omitted, defaults to the workflow name. + # (optional) + name: "My Workflow" + + # Maximum number of check runs to create per workflow run (default: 1). Supports + # integer or GitHub Actions expression (e.g. '${{ inputs.max }}'). + # (optional) + # Accepted formats: + + # Format 1: integer + max: 1 + + # Format 2: GitHub Actions expression that resolves to an integer at runtime + max: "example-value" + + # GitHub token to use for this specific output type. Overrides global github-token + # if specified. + # (optional) + github-token: "${{ secrets.GITHUB_TOKEN }}" + + # If true, emit step summary messages instead of making GitHub API calls for this + # specific output type (preview mode) + # (optional) + staged: true + + # GitHub App credentials for minting an installation access token scoped to + # checks:write for this handler. When set, a short-lived token is minted before + # the handler runs and revoked afterwards. + # (optional) + github-app: + # Deprecated alias for client-id. GitHub App ID/client ID (e.g., '${{ vars.APP_ID + # }}'). + # (optional) + app-id: "example-value" + + # GitHub App client ID (e.g., '${{ vars.APP_ID }}'). Required to mint a GitHub App + # token. + # (optional) + client-id: "example-value" + + # GitHub App private key (e.g., '${{ secrets.APP_PRIVATE_KEY }}'). Required to + # mint a GitHub App token. + # (optional) + private-key: "example-value" + + # If true, skip token minting when client-id/private-key resolve to empty strings + # at runtime. Defaults to false. + # (optional) + ignore-if-missing: true + + # Optional owner of the GitHub App installation (defaults to current repository + # owner if not specified) + # (optional) + owner: "example-value" + + # Optional list of repositories to grant access to (defaults to current repository + # if not specified) + # (optional) + repositories: [] + # Array of strings + + # Optional extra GitHub App-only permissions to merge into the minted token. Takes + # effect for tools.github.github-app and safe-outputs.github-app; ignored in + # on.github-app and the top-level github-app fallback. Use to add GitHub App-only + # scopes (e.g. members, organization-administration) not expressible via standard + # handler declarations. + # (optional) + permissions: + # Permission level for repository administration (read/none; "write" is rejected + # by the compiler). GitHub App-only permission for repository administration. + # (optional) + administration: "read" + + # Permission level for Codespaces (read/none; "write" is rejected by the + # compiler). GitHub App-only permission. + # (optional) + codespaces: "read" + + # Permission level for Codespaces lifecycle administration (read/none; "write" is + # rejected by the compiler). GitHub App-only permission. + # (optional) + codespaces-lifecycle-admin: "read" + + # Permission level for Codespaces metadata (read/none; "write" is rejected by the + # compiler). GitHub App-only permission. + # (optional) + codespaces-metadata: "read" + + # Permission level for user email addresses (read/none; "write" is rejected by the + # compiler). GitHub App-only permission. + # (optional) + email-addresses: "read" + + # Permission level for repository environments (read/none; "write" is rejected by + # the compiler). GitHub App-only permission. + # (optional) + environments: "read" + + # Permission level for git signing (read/none; "write" is rejected by the + # compiler). GitHub App-only permission. + # (optional) + git-signing: "read" + + # Permission level for organization members (read/none; "write" is rejected by the + # compiler). Required for org team membership API calls. + # (optional) + members: "read" + + # Permission level for organization administration (read/none; "write" is rejected + # by the compiler). GitHub App-only permission. + # (optional) + organization-administration: "read" + + # Permission level for organization announcement banners (read/none; "write" is + # rejected by the compiler). GitHub App-only permission. + # (optional) + organization-announcement-banners: "read" + + # Permission level for organization Codespaces (read/none; "write" is rejected by + # the compiler). GitHub App-only permission. + # (optional) + organization-codespaces: "read" + + # Permission level for organization Copilot (read/none; "write" is rejected by the + # compiler). GitHub App-only permission. + # (optional) + organization-copilot: "read" + + # Permission level for organization custom org roles (read/none; "write" is + # rejected by the compiler). GitHub App-only permission. + # (optional) + organization-custom-org-roles: "read" + + # Permission level for organization custom properties (read/none; "write" is + # rejected by the compiler). GitHub App-only permission. + # (optional) + organization-custom-properties: "read" + + # Permission level for organization custom repository roles (read/none; "write" is + # rejected by the compiler). GitHub App-only permission. + # (optional) + organization-custom-repository-roles: "read" + + # Permission level for organization events (read/none; "write" is rejected by the + # compiler). GitHub App-only permission. + # (optional) + organization-events: "read" + + # Permission level for organization webhooks (read/none; "write" is rejected by + # the compiler). GitHub App-only permission. + # (optional) + organization-hooks: "read" + + # Permission level for organization members management (read/none; "write" is + # rejected by the compiler). GitHub App-only permission. + # (optional) + organization-members: "read" + + # Permission level for organization packages (read/none; "write" is rejected by + # the compiler). GitHub App-only permission. + # (optional) + organization-packages: "read" + + # Permission level for organization personal access token requests (read/none; + # "write" is rejected by the compiler). GitHub App-only permission. + # (optional) + organization-personal-access-token-requests: "read" + + # Permission level for organization personal access tokens (read/none; "write" is + # rejected by the compiler). GitHub App-only permission. + # (optional) + organization-personal-access-tokens: "read" + + # Permission level for organization plan (read/none; "write" is rejected by the + # compiler). GitHub App-only permission. + # (optional) + organization-plan: "read" + + # Permission level for organization self-hosted runners (read/none; "write" is + # rejected by the compiler). GitHub App-only permission. + # (optional) + organization-self-hosted-runners: "read" + + # Permission level for organization user blocking (read/none; "write" is rejected + # by the compiler). GitHub App-only permission. + # (optional) + organization-user-blocking: "read" + + # Permission level for repository custom properties (read/none; "write" is + # rejected by the compiler). GitHub App-only permission. + # (optional) + repository-custom-properties: "read" + + # Permission level for repository webhooks (read/none; "write" is rejected by the + # compiler). GitHub App-only permission. + # (optional) + repository-hooks: "read" + + # Permission level for single file access (read/none; "write" is rejected by the + # compiler). GitHub App-only permission. + # (optional) + single-file: "read" + + # Permission level for team discussions (read/none; "write" is rejected by the + # compiler). GitHub App-only permission. + # (optional) + team-discussions: "read" + + # Permission level for Dependabot vulnerability alerts (read/none; "write" is + # rejected by the compiler). Also available as a GITHUB_TOKEN scope. When used + # with a GitHub App, forwarded as permission-vulnerability-alerts input. + # (optional) + vulnerability-alerts: "read" + + # Permission level for GitHub Actions workflow files (read/none; "write" is + # rejected by the compiler). GitHub App-only permission. + # (optional) + workflows: "read" + + # Optional static fallback values for the check run output fields. Used when the + # agent does not provide the corresponding field. + # (optional) + output: + # Fallback title for the check run output (max 256 characters). Used when the + # agent does not supply a title. + # (optional) + title: "example-value" + + # Fallback summary for the check run output (max 65535 characters, GitHub API + # limit). Used when the agent does not supply a summary. + # (optional) + summary: "example-value" + + # Format 2: Enable check run creation with default configuration (max: 1) + create-check-run: null + # Enable AI agents to add labels to GitHub issues or pull requests based on # workflow analysis or classification. # (optional) From 49cab3ce91e73712a8f6969f95b2f97a45bf7ecc Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 27 May 2026 04:19:42 +0000 Subject: [PATCH 3/8] test: cover dedicated outcome evaluator branches Co-authored-by: gh-aw-bot <259018956+gh-aw-bot@users.noreply.github.com> --- actions/setup/js/add_labels.cjs | 6 +----- actions/setup/js/evaluate_outcomes.cjs | 14 +++----------- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/actions/setup/js/add_labels.cjs b/actions/setup/js/add_labels.cjs index 17b54d248df..c7aeb77b875 100644 --- a/actions/setup/js/add_labels.cjs +++ b/actions/setup/js/add_labels.cjs @@ -185,11 +185,7 @@ const main = createCountGatedHandler({ repo: repoParts.repo, issue_number: itemNumber, }); - labelsBefore = Array.isArray(existingItem?.labels) - ? existingItem.labels - .map(label => (typeof label === "string" ? label : label?.name)) - .filter(name => typeof name === "string" && name.trim() !== "") - : []; + labelsBefore = Array.isArray(existingItem?.labels) ? existingItem.labels.map(label => (typeof label === "string" ? label : label?.name)).filter(name => typeof name === "string" && name.trim() !== "") : []; } catch (error) { core.info(`Unable to capture labels-before snapshot for ${contextType} #${itemNumber}: ${getErrorMessage(error)}`); } diff --git a/actions/setup/js/evaluate_outcomes.cjs b/actions/setup/js/evaluate_outcomes.cjs index 932ef532a9f..a7280936854 100644 --- a/actions/setup/js/evaluate_outcomes.cjs +++ b/actions/setup/js/evaluate_outcomes.cjs @@ -275,9 +275,7 @@ function evaluateCreateIssue(item, itemRepo, timestamp, out, apiGet, nowMs) { const authorLogin = issue.user && typeof issue.user.login === "string" ? issue.user.login : ""; const comments = apiGet(`repos/${itemRepo}/issues/${num}/comments`); - const hasNonAuthorComment = - Array.isArray(comments) && - comments.some(c => c && c.user && typeof c.user.login === "string" && c.user.login !== authorLogin && c.user.login !== ""); + const hasNonAuthorComment = Array.isArray(comments) && comments.some(c => c && c.user && typeof c.user.login === "string" && c.user.login !== authorLogin && c.user.login !== ""); const timeline = apiGet(`repos/${itemRepo}/issues/${num}/timeline`); const timelineEvents = Array.isArray(timeline) ? timeline : []; @@ -445,14 +443,8 @@ function evaluateAddLabels(item, itemRepo, timestamp, out, apiGet, nowMs) { return out; } - const labelsBefore = Array.isArray(item.labelsBefore) - ? item.labelsBefore.map(l => String(l || "").trim()).filter(Boolean) - : []; - const labelsAdded = Array.isArray(item.labelsAdded) - ? item.labelsAdded.map(l => String(l || "").trim()).filter(Boolean) - : Array.isArray(item.labels) - ? item.labels.map(l => String(l || "").trim()).filter(Boolean) - : []; + const labelsBefore = Array.isArray(item.labelsBefore) ? item.labelsBefore.map(l => String(l || "").trim()).filter(Boolean) : []; + const labelsAdded = Array.isArray(item.labelsAdded) ? item.labelsAdded.map(l => String(l || "").trim()).filter(Boolean) : Array.isArray(item.labels) ? item.labels.map(l => String(l || "").trim()).filter(Boolean) : []; if (labelsBefore.length === 0 || labelsAdded.length === 0) { out.result = "unknown"; From 67692db04b4173c73bb243bbf900302977266204 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 27 May 2026 04:21:05 +0000 Subject: [PATCH 4/8] refactor: improve evaluator readability and edge-case coverage Co-authored-by: gh-aw-bot <259018956+gh-aw-bot@users.noreply.github.com> --- actions/setup/js/add_labels.cjs | 5 ++- actions/setup/js/evaluate_outcomes.cjs | 35 +++++++++++++++------ actions/setup/js/evaluate_outcomes.test.cjs | 14 +++++++++ 3 files changed, 43 insertions(+), 11 deletions(-) diff --git a/actions/setup/js/add_labels.cjs b/actions/setup/js/add_labels.cjs index c7aeb77b875..bf452a2c7e6 100644 --- a/actions/setup/js/add_labels.cjs +++ b/actions/setup/js/add_labels.cjs @@ -185,7 +185,10 @@ const main = createCountGatedHandler({ repo: repoParts.repo, issue_number: itemNumber, }); - labelsBefore = Array.isArray(existingItem?.labels) ? existingItem.labels.map(label => (typeof label === "string" ? label : label?.name)).filter(name => typeof name === "string" && name.trim() !== "") : []; + const existingLabels = Array.isArray(existingItem?.labels) ? existingItem.labels : []; + labelsBefore = existingLabels + .map(label => (typeof label === "string" ? label : label?.name)) + .filter(name => typeof name === "string" && name.trim() !== ""); } catch (error) { core.info(`Unable to capture labels-before snapshot for ${contextType} #${itemNumber}: ${getErrorMessage(error)}`); } diff --git a/actions/setup/js/evaluate_outcomes.cjs b/actions/setup/js/evaluate_outcomes.cjs index a7280936854..d5ddf844a83 100644 --- a/actions/setup/js/evaluate_outcomes.cjs +++ b/actions/setup/js/evaluate_outcomes.cjs @@ -45,20 +45,23 @@ const NOOP_TYPES = new Set(["noop", "missing_tool", "missing_data", "report_inco const DEFAULT_ISSUE_IMMEDIATE_CLOSE_WINDOW_SEC = 60 * 60; const DEFAULT_LABEL_RETENTION_WINDOW_SEC = 24 * 60 * 60; +const POSITIVE_REACTIONS = ["+1", "heart", "hooray", "rocket"]; +const NEGATIVE_REACTIONS = ["-1", "confused"]; + /** * Read a positive integer from env with fallback. * @param {string} key * @param {number} fallback * @returns {number} */ -function getEnvPositiveInt(key, fallback) { +function getEnvPositiveIntOrDefault(key, fallback) { const raw = process.env[key]; const parsed = Number.parseInt(raw || "", 10); return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; } -const ISSUE_IMMEDIATE_CLOSE_WINDOW_SEC = getEnvPositiveInt("OUTCOME_ISSUE_IMMEDIATE_CLOSE_WINDOW_SEC", DEFAULT_ISSUE_IMMEDIATE_CLOSE_WINDOW_SEC); -const LABEL_RETENTION_WINDOW_SEC = getEnvPositiveInt("OUTCOME_LABEL_RETENTION_WINDOW_SEC", DEFAULT_LABEL_RETENTION_WINDOW_SEC); +const ISSUE_IMMEDIATE_CLOSE_WINDOW_SEC = getEnvPositiveIntOrDefault("OUTCOME_ISSUE_IMMEDIATE_CLOSE_WINDOW_SEC", DEFAULT_ISSUE_IMMEDIATE_CLOSE_WINDOW_SEC); +const LABEL_RETENTION_WINDOW_SEC = getEnvPositiveIntOrDefault("OUTCOME_LABEL_RETENTION_WINDOW_SEC", DEFAULT_LABEL_RETENTION_WINDOW_SEC); // --------------------------------------------------------------------------- // Helpers @@ -203,12 +206,21 @@ function summarizeReactions(reactions) { if (!reactions || typeof reactions !== "object") { return { total: null, positive: null, negative: null }; } - const positive = (reactions["+1"] || 0) + (reactions.heart || 0) + (reactions.hooray || 0) + (reactions.rocket || 0); - const negative = (reactions["-1"] || 0) + (reactions.confused || 0); + const positive = POSITIVE_REACTIONS.reduce((sum, key) => sum + Number(reactions[key] || 0), 0); + const negative = NEGATIVE_REACTIONS.reduce((sum, key) => sum + Number(reactions[key] || 0), 0); const total = reactions.total_count != null ? reactions.total_count : positive + negative + (reactions.laugh || 0) + (reactions.eyes || 0); return { total, positive, negative }; } +/** + * @param {unknown} value + * @returns {string[]} + */ +function normalizeLabels(value) { + if (!Array.isArray(value)) return []; + return value.map(l => String(l || "").trim()).filter(Boolean); +} + /** * @param {string} url * @returns {number|null} @@ -318,7 +330,8 @@ function evaluateCreateIssue(item, itemRepo, timestamp, out, apiGet, nowMs) { if (issue.state === "closed" && issue.created_at && issue.closed_at) { out.resolution_sec = secondsBetween(issue.created_at, issue.closed_at); - if (typeof out.resolution_sec === "number" && out.resolution_sec <= ISSUE_IMMEDIATE_CLOSE_WINDOW_SEC && closeActor && closeActor !== authorLogin) { + const closedByDifferentUser = closeActor !== "" && closeActor !== authorLogin; + if (typeof out.resolution_sec === "number" && out.resolution_sec <= ISSUE_IMMEDIATE_CLOSE_WINDOW_SEC && closedByDifferentUser) { out.result = "rejected"; out.detail = "rejected:strong"; return out; @@ -443,10 +456,12 @@ function evaluateAddLabels(item, itemRepo, timestamp, out, apiGet, nowMs) { return out; } - const labelsBefore = Array.isArray(item.labelsBefore) ? item.labelsBefore.map(l => String(l || "").trim()).filter(Boolean) : []; - const labelsAdded = Array.isArray(item.labelsAdded) ? item.labelsAdded.map(l => String(l || "").trim()).filter(Boolean) : Array.isArray(item.labels) ? item.labels.map(l => String(l || "").trim()).filter(Boolean) : []; + const labelsBefore = normalizeLabels(item.labelsBefore); + const labelsAdded = normalizeLabels(item.labelsAdded); + const fallbackLabels = normalizeLabels(item.labels); + const effectiveLabelsAdded = labelsAdded.length > 0 ? labelsAdded : fallbackLabels; - if (labelsBefore.length === 0 || labelsAdded.length === 0) { + if (labelsBefore.length === 0 || effectiveLabelsAdded.length === 0) { out.result = "unknown"; out.detail = "unknown: missing persisted label before-state"; setPendingAge(out, timestamp, nowMs); @@ -462,7 +477,7 @@ function evaluateAddLabels(item, itemRepo, timestamp, out, apiGet, nowMs) { } const currentLabels = new Set(labels.map(l => (l && typeof l.name === "string" ? l.name : "")).filter(Boolean)); - const trackedAdded = labelsAdded.filter(l => !labelsBefore.includes(l)); + const trackedAdded = effectiveLabelsAdded.filter(l => !labelsBefore.includes(l)); const removed = trackedAdded.filter(l => !currentLabels.has(l)); const nowEpoch = Math.floor(nowMs / 1000); diff --git a/actions/setup/js/evaluate_outcomes.test.cjs b/actions/setup/js/evaluate_outcomes.test.cjs index c4113a17aed..4579b997503 100644 --- a/actions/setup/js/evaluate_outcomes.test.cjs +++ b/actions/setup/js/evaluate_outcomes.test.cjs @@ -30,6 +30,20 @@ describe("evaluate_outcomes type-specific evaluators", () => { expect(evaluateItem(item, "acme/repo", { ghAPI: noEngagement, nowMs }).result).toBe("pending"); }); + it("handles missing create_issue reaction/comment fields without existence-only acceptance", () => { + const item = { + type: "create_issue", + url: "https://github.com/acme/repo/issues/12", + timestamp: "2026-05-26T04:00:00Z", + }; + const ghAPI = createAPIStub({ + "repos/acme/repo/issues/12": { state: "open", user: { login: "author" }, comments: "unknown", reactions: null }, + "repos/acme/repo/issues/12/comments": [], + "repos/acme/repo/issues/12/timeline": [], + }); + expect(evaluateItem(item, "acme/repo", { ghAPI, nowMs: Date.parse("2026-05-27T04:00:00Z") }).result).toBe("pending"); + }); + it("classifies create_issue immediate close and close-without-activity as rejected", () => { const item = { type: "create_issue", From 1a1761a66838f84e9f3658d5e24ffa15ba63b31d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 27 May 2026 11:23:20 +0000 Subject: [PATCH 5/8] Remove unnecessary requirements path entry Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .github/workflows/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/requirements.txt b/.github/workflows/requirements.txt index af28d50e02a..c8ca0373942 100644 --- a/.github/workflows/requirements.txt +++ b/.github/workflows/requirements.txt @@ -1,5 +1,4 @@ "mempalace==3.2.0" -/tmp/gh-aw/agent/token-audit/site-packages \\\n markitdown-mcp numpy From 40ed161d6a15667a95608ca49436205b2026d8ca Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 27 May 2026 18:03:57 +0000 Subject: [PATCH 6/8] Address reviewer feedback on outcome evaluators and manifest fields Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .github/workflows/requirements.txt | 1 - actions/setup/js/add_labels.cjs | 4 +--- actions/setup/js/evaluate_outcomes.cjs | 12 +++++++----- actions/setup/js/evaluate_outcomes.test.cjs | 6 +++--- actions/setup/js/safe_output_manifest.cjs | 2 +- actions/setup/js/safe_output_manifest.test.cjs | 9 +++++++++ 6 files changed, 21 insertions(+), 13 deletions(-) diff --git a/.github/workflows/requirements.txt b/.github/workflows/requirements.txt index c8ca0373942..abcda1723e7 100644 --- a/.github/workflows/requirements.txt +++ b/.github/workflows/requirements.txt @@ -1,5 +1,4 @@ "mempalace==3.2.0" -\\\n markitdown-mcp numpy scikit-learn diff --git a/actions/setup/js/add_labels.cjs b/actions/setup/js/add_labels.cjs index bf452a2c7e6..b2345b0f50f 100644 --- a/actions/setup/js/add_labels.cjs +++ b/actions/setup/js/add_labels.cjs @@ -186,9 +186,7 @@ const main = createCountGatedHandler({ issue_number: itemNumber, }); const existingLabels = Array.isArray(existingItem?.labels) ? existingItem.labels : []; - labelsBefore = existingLabels - .map(label => (typeof label === "string" ? label : label?.name)) - .filter(name => typeof name === "string" && name.trim() !== ""); + labelsBefore = existingLabels.map(label => (typeof label === "string" ? label : label?.name)).filter(name => typeof name === "string" && name.trim() !== ""); } catch (error) { core.info(`Unable to capture labels-before snapshot for ${contextType} #${itemNumber}: ${getErrorMessage(error)}`); } diff --git a/actions/setup/js/evaluate_outcomes.cjs b/actions/setup/js/evaluate_outcomes.cjs index bf409a585f8..5d2f198a77d 100644 --- a/actions/setup/js/evaluate_outcomes.cjs +++ b/actions/setup/js/evaluate_outcomes.cjs @@ -321,7 +321,7 @@ function evaluateCreateIssue(item, itemRepo, timestamp, out, apiGet, nowMs) { } } - if (issue.state === "open" && (hasMergedPRReference || hasCommitReference || hasClosingActionReference)) { + if (issue.state === "open" && (hasMergedPRReference || hasCommitReference)) { out.result = "accepted"; out.detail = "accepted:strong"; return out; @@ -390,8 +390,9 @@ function evaluateAddComment(item, itemRepo, timestamp, out, apiGet, nowMs) { const comment = apiGet(`repos/${itemRepo}/issues/comments/${commentID}`); if (!comment || !comment.id) { - out.result = "rejected"; - out.detail = "rejected:strong deleted"; + out.result = "unknown"; + out.detail = "unknown: comment api error"; + setPendingAge(out, timestamp, nowMs); return out; } @@ -461,12 +462,13 @@ function evaluateAddLabels(item, itemRepo, timestamp, out, apiGet, nowMs) { return out; } - const labelsBefore = normalizeLabels(item.labelsBefore); + const hasLabelsBefore = Object.prototype.hasOwnProperty.call(item, "labelsBefore") && Array.isArray(item.labelsBefore); + const labelsBefore = hasLabelsBefore ? normalizeLabels(item.labelsBefore) : []; const labelsAdded = normalizeLabels(item.labelsAdded); const fallbackLabels = normalizeLabels(item.labels); const effectiveLabelsAdded = labelsAdded.length > 0 ? labelsAdded : fallbackLabels; - if (labelsBefore.length === 0 || effectiveLabelsAdded.length === 0) { + if (!hasLabelsBefore || effectiveLabelsAdded.length === 0) { out.result = "unknown"; out.detail = "unknown: missing persisted label before-state"; setPendingAge(out, timestamp, nowMs); diff --git a/actions/setup/js/evaluate_outcomes.test.cjs b/actions/setup/js/evaluate_outcomes.test.cjs index d11e5b938ee..f20684dbd63 100644 --- a/actions/setup/js/evaluate_outcomes.test.cjs +++ b/actions/setup/js/evaluate_outcomes.test.cjs @@ -98,10 +98,10 @@ describe("evaluate_outcomes type-specific evaluators", () => { timestamp: "2026-05-26T00:00:00Z", }; - const deleted = createAPIStub({ + const commentApiError = createAPIStub({ "repos/acme/repo/issues/comments/1001": null, }); - expect(evaluateItem(base, "acme/repo", { ghAPI: deleted }).detail).toContain("rejected:strong"); + expect(evaluateItem(base, "acme/repo", { ghAPI: commentApiError }).result).toBe("unknown"); const reacted = createAPIStub({ "repos/acme/repo/issues/comments/1001": { id: 1001, created_at: "2026-05-26T00:00:00Z", user: { login: "copilot" }, reactions: { total_count: 2 } }, @@ -146,7 +146,7 @@ describe("evaluate_outcomes type-specific evaluators", () => { expect(evaluateItem(item, "acme/repo", { ghAPI: removedByNonAuthor, nowMs: Date.parse("2026-05-27T00:00:00Z") }).detail).toBe("rejected:strong"); expect(evaluateItem(item, "acme/repo", { ghAPI: retained, nowMs: Date.parse("2026-05-25T00:01:00Z") }).result).toBe("pending"); - expect(evaluateItem({ ...item, labelsBefore: [] }, "acme/repo", { ghAPI: retained, nowMs: Date.parse("2026-05-27T00:00:00Z") }).result).toBe("unknown"); + expect(evaluateItem({ ...item, labelsBefore: [] }, "acme/repo", { ghAPI: retained, nowMs: Date.parse("2026-05-27T00:00:00Z") }).detail).toBe("accepted:strong"); }); }); diff --git a/actions/setup/js/safe_output_manifest.cjs b/actions/setup/js/safe_output_manifest.cjs index a1672ce127f..41588da016c 100644 --- a/actions/setup/js/safe_output_manifest.cjs +++ b/actions/setup/js/safe_output_manifest.cjs @@ -81,7 +81,7 @@ function createManifestLogger(manifestFile = MANIFEST_FILE_PATH) { ...(item.number != null ? { number: item.number } : {}), ...(item.repo ? { repo: item.repo } : {}), ...(item.temporaryId ? { temporaryId: item.temporaryId } : {}), - ...(Array.isArray(item.labelsAdded) && item.labelsAdded.length > 0 ? { labelsAdded: item.labelsAdded } : {}), + ...(Array.isArray(item.labelsAdded) ? { labelsAdded: item.labelsAdded } : {}), ...(Array.isArray(item.labelsBefore) ? { labelsBefore: item.labelsBefore } : {}), timestamp: new Date().toISOString(), }; diff --git a/actions/setup/js/safe_output_manifest.test.cjs b/actions/setup/js/safe_output_manifest.test.cjs index 4156725244e..1473b398ac0 100644 --- a/actions/setup/js/safe_output_manifest.test.cjs +++ b/actions/setup/js/safe_output_manifest.test.cjs @@ -110,6 +110,15 @@ describe("safe_output_manifest", () => { expect(entry.timestamp).toBeDefined(); }); + it("should persist labelsAdded even when empty", () => { + const log = createManifestLogger(testManifestFile); + log({ type: "add_labels", number: 20875, labelsAdded: [] }); + + const content = fs.readFileSync(testManifestFile, "utf8"); + const entry = JSON.parse(content.trim()); + expect(entry.labelsAdded).toEqual([]); + }); + it("should skip null/undefined items", () => { const log = createManifestLogger(testManifestFile); log(null); From 6a3e0b0a4101d500265bbd94bac44793be4d094f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 27 May 2026 18:05:19 +0000 Subject: [PATCH 7/8] Clarify unknown outcome details for comment and label evaluation Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- actions/setup/js/evaluate_outcomes.cjs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/actions/setup/js/evaluate_outcomes.cjs b/actions/setup/js/evaluate_outcomes.cjs index 5d2f198a77d..278dd601717 100644 --- a/actions/setup/js/evaluate_outcomes.cjs +++ b/actions/setup/js/evaluate_outcomes.cjs @@ -391,7 +391,7 @@ function evaluateAddComment(item, itemRepo, timestamp, out, apiGet, nowMs) { const comment = apiGet(`repos/${itemRepo}/issues/comments/${commentID}`); if (!comment || !comment.id) { out.result = "unknown"; - out.detail = "unknown: comment api error"; + out.detail = "unknown: failed to fetch comment from API (may be deleted or inaccessible)"; setPendingAge(out, timestamp, nowMs); return out; } @@ -468,13 +468,20 @@ function evaluateAddLabels(item, itemRepo, timestamp, out, apiGet, nowMs) { const fallbackLabels = normalizeLabels(item.labels); const effectiveLabelsAdded = labelsAdded.length > 0 ? labelsAdded : fallbackLabels; - if (!hasLabelsBefore || effectiveLabelsAdded.length === 0) { + if (!hasLabelsBefore) { out.result = "unknown"; out.detail = "unknown: missing persisted label before-state"; setPendingAge(out, timestamp, nowMs); return out; } + if (effectiveLabelsAdded.length === 0) { + out.result = "unknown"; + out.detail = "unknown: no labels added"; + setPendingAge(out, timestamp, nowMs); + return out; + } + const labels = apiGet(`repos/${itemRepo}/issues/${num}/labels`); if (!Array.isArray(labels)) { out.result = "unknown"; From 9f06439fa7c2df8743324cecc8cb7489a0c7b2f3 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 27 May 2026 18:06:41 +0000 Subject: [PATCH 8/8] Polish labels-before state check wording and property access Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- actions/setup/js/evaluate_outcomes.cjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/actions/setup/js/evaluate_outcomes.cjs b/actions/setup/js/evaluate_outcomes.cjs index 278dd601717..f4d3dda58b8 100644 --- a/actions/setup/js/evaluate_outcomes.cjs +++ b/actions/setup/js/evaluate_outcomes.cjs @@ -462,7 +462,7 @@ function evaluateAddLabels(item, itemRepo, timestamp, out, apiGet, nowMs) { return out; } - const hasLabelsBefore = Object.prototype.hasOwnProperty.call(item, "labelsBefore") && Array.isArray(item.labelsBefore); + const hasLabelsBefore = Object.hasOwn(item, "labelsBefore") && Array.isArray(item.labelsBefore); const labelsBefore = hasLabelsBefore ? normalizeLabels(item.labelsBefore) : []; const labelsAdded = normalizeLabels(item.labelsAdded); const fallbackLabels = normalizeLabels(item.labels); @@ -470,7 +470,7 @@ function evaluateAddLabels(item, itemRepo, timestamp, out, apiGet, nowMs) { if (!hasLabelsBefore) { out.result = "unknown"; - out.detail = "unknown: missing persisted label before-state"; + out.detail = "unknown: missing persisted label before state"; setPendingAge(out, timestamp, nowMs); return out; }