Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 22 additions & 14 deletions src/queue/processors.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1080,6 +1080,7 @@
// One bounded re-gate unit fanned out by the sweep (#audit-sweep-fanout): re-review + stamp a single PR.
await regatePullRequest(
env,
message.repairHeadSha,
message.repoFullName,
message.prNumber,
message.installationId,
Expand Down Expand Up @@ -1461,7 +1462,7 @@
// surfaceRepairPriorityPullNumbers has no memory of prior attempts -- if the repair keeps failing for the SAME
// head SHA (e.g. every AI-provider attempt times out), it would otherwise re-select that PR forever, burning a
// fresh review attempt every cycle for zero output. These two constants cap that: once a SHA has already had
// REGATE_REPAIR_MAX_ATTEMPTS_PER_SHA dispatches recorded, it drops back to ordinary staleness-gated candidacy
const REGATE_REPAIR_MAX_ATTEMPTS_PER_SHA = 5;
// (still eventually re-checked, just not on every tick) and a single REGATE_REPAIR_EXHAUSTED_EVENT_TYPE audit
// event is recorded so the stuck PR is visible instead of silently retried forever. A new commit changes the
// head SHA, which resets the count naturally (the target key is scoped to repo+PR+SHA).
Expand Down Expand Up @@ -1870,29 +1871,20 @@
const job: JobMessage = {
type: "agent-regate-pr",
deliveryId: isPriorityRepair
// #orb-retry-storm: pass the repair SHA so regatePullRequest can record the attempt at
// execution time (after rate-limit admission), not here at dispatch time. Jobs that are
// deferred or dropped before they run no longer count against the per-SHA cap.
...(isPriorityRepair && pr.headSha ? { repairHeadSha: pr.headSha } : {}),

Check failure on line 1877 in src/queue/processors.ts

View workflow job for this annotation

GitHub Actions / validate-code

',' expected.

Check failure on line 1877 in src/queue/processors.ts

View workflow job for this annotation

GitHub Actions / validate-code

',' expected.
? `regate-repair:${repoFullName}#${pr.number}`

Check failure on line 1878 in src/queue/processors.ts

View workflow job for this annotation

GitHub Actions / validate-code

Property assignment expected.

Check failure on line 1878 in src/queue/processors.ts

View workflow job for this annotation

GitHub Actions / validate-code

Property assignment expected.
: `regate-sweep:${repoFullName}#${pr.number}`,
repoFullName,
prNumber: pr.number,
installationId: sweepInstallationId,
...(pr.createdAt ? { prCreatedAt: pr.createdAt } : {}),
};
const delaySeconds = Math.min(index * 10, 600);
await (delaySeconds > 0
? env.JOBS.send(job, { delaySeconds })
: env.JOBS.send(job));
// #orb-retry-storm: record every priority-repair dispatch so surfaceRepairPriorityPullNumbers can cap
// how many times the SAME head SHA gets bounced back through this bypass (see its own comment above).
/* v8 ignore next -- isPriorityRepair is only true for a pr.number surfaceRepairPriorityPullNumbers added to priorityPullNumbers, which requires that same pr to have a truthy headSha; `&& pr.headSha` only satisfies its optional TS type. */
if (isPriorityRepair && pr.headSha) {
await recordAuditEvent(env, {
eventType: REGATE_REPAIR_ATTEMPT_EVENT_TYPE,
actor: "gittensory",
targetKey: regateRepairTargetKey(repoFullName, pr.number, pr.headSha),

Check failure on line 1883 in src/queue/processors.ts

View workflow job for this annotation

GitHub Actions / validate-code

',' expected.

Check failure on line 1883 in src/queue/processors.ts

View workflow job for this annotation

GitHub Actions / validate-code

',' expected.
outcome: "queued",

Check failure on line 1884 in src/queue/processors.ts

View workflow job for this annotation

GitHub Actions / validate-code

';' expected.

Check failure on line 1884 in src/queue/processors.ts

View workflow job for this annotation

GitHub Actions / validate-code

';' expected.
detail: `outage-repair re-review dispatched for ${repoFullName}#${pr.number}`,

Check failure on line 1885 in src/queue/processors.ts

View workflow job for this annotation

GitHub Actions / validate-code

';' expected.

Check failure on line 1885 in src/queue/processors.ts

View workflow job for this annotation

GitHub Actions / validate-code

';' expected.
metadata: { repoFullName, prNumber: pr.number, headSha: pr.headSha },

Check failure on line 1886 in src/queue/processors.ts

View workflow job for this annotation

GitHub Actions / validate-code

Declaration or statement expected.

Check failure on line 1886 in src/queue/processors.ts

View workflow job for this annotation

GitHub Actions / validate-code

';' expected.

Check failure on line 1886 in src/queue/processors.ts

View workflow job for this annotation

GitHub Actions / validate-code

';' expected.

Check failure on line 1886 in src/queue/processors.ts

View workflow job for this annotation

GitHub Actions / validate-code

';' expected.

Check failure on line 1886 in src/queue/processors.ts

View workflow job for this annotation

GitHub Actions / validate-code

Declaration or statement expected.

Check failure on line 1886 in src/queue/processors.ts

View workflow job for this annotation

GitHub Actions / validate-code

';' expected.

Check failure on line 1886 in src/queue/processors.ts

View workflow job for this annotation

GitHub Actions / validate-code

';' expected.

Check failure on line 1886 in src/queue/processors.ts

View workflow job for this annotation

GitHub Actions / validate-code

';' expected.
});

Check failure on line 1887 in src/queue/processors.ts

View workflow job for this annotation

GitHub Actions / validate-code

Declaration or statement expected.

Check failure on line 1887 in src/queue/processors.ts

View workflow job for this annotation

GitHub Actions / validate-code

Declaration or statement expected.
}
}
}
Expand Down Expand Up @@ -2097,6 +2089,7 @@
// per-PR job always re-evaluates the head.
async function regatePullRequest(
env: Env,
repairHeadSha?: string,
repoFullName: string,
prNumber: number,
installationId: number,
Expand Down Expand Up @@ -2124,10 +2117,25 @@
await env.JOBS.send(
{
type: "agent-regate-pr",
...(repairHeadSha ? { repairHeadSha } : {}),
deliveryId,
repoFullName,
prNumber,
installationId,
}
// #orb-retry-storm: record the repair attempt NOW — after rate-limit admission — so the cap in
// surfaceRepairPriorityPullNumbers counts actual executions, not queued dispatches that may have
// been deferred or dropped before running (the old dispatch-time recording let rate-limit deferrals
// exhaust the 2-attempt budget without ever trying the repair).
if (repairHeadSha) {
await recordAuditEvent(env, {
eventType: REGATE_REPAIR_ATTEMPT_EVENT_TYPE,
actor: "gittensory",
targetKey: regateRepairTargetKey(repoFullName, prNumber, repairHeadSha),
outcome: "started",
detail: `outage-repair re-review executing for ${repoFullName}#${prNumber}`,
metadata: { repoFullName, prNumber, headSha: repairHeadSha },
});
...(prCreatedAt ? { prCreatedAt } : {}),
...(force ? { force: true } : {}),
},
Expand Down
4 changes: 4 additions & 0 deletions src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,10 @@ export type JobMessage =
// webhook-driven caller sets this; it exists so a manual trigger has a supported way to force a fresh
// pass instead of reusing a recent (possibly disputed) result.
force?: boolean | undefined;
/** The head SHA this job was dispatched to repair; present only on priority-repair dispatches.
* Used by regatePullRequest to record the repair attempt at execution time (not dispatch time)
* so the cap in surfaceRepairPriorityPullNumbers counts actual runs, not queued jobs. */
repairHeadSha?: string | undefined;
}
| {
type: "refresh-registry";
Expand Down
Loading