From 563f2d6b5c55c8061b4d7e1112e028c41a9b0212 Mon Sep 17 00:00:00 2001 From: kjgbot Date: Fri, 24 Jul 2026 13:42:29 +0200 Subject: [PATCH] fix(mount): mint github scope as /github/** and fail fast on auth-scope failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mount requested `relayfile:fs:read|write:/github/repos/**` in FACTORY_RELAYFILE_SCOPES. RelayAuth's path-token validator rejects `/github/repos/**` as invalid (while accepting `/github/**` and `/github/repos/*`), and because all scopes mint in one batch that single bad path fails the ENTIRE delegated-token mint with `invalid_paths` — leaving the mount with no fs token and a `403 missing required scope: fs:read` on every read. Switch github to the provider root `/github/**`, a valid superset that mints cleanly (verified against production). Also harden the mount so this class of failure surfaces loudly instead of limping silently: - classify the 403 / bootstrap-stall signature as a terminal MountAuthScopeError - preflight throws it terminally (no infinite refresh retry loop) - supervisor stops retrying; exposes isLocalMountAuthDegraded() - gate tracked-agent resume on it (kills the opaque {"name":"Error"} cascade) - `factory start` fails fast with an actionable remediation message Upstream root causes filed: AgentWorkforce/relayauth#67, AgentWorkforce/cloud#2834 Co-Authored-By: Claude Opus 4.8 --- src/cli/fleet.ts | 37 ++++-- src/mount/local-mount-preflight.test.ts | 69 ++++++++++ src/mount/local-mount-preflight.ts | 69 ++++++++-- src/mount/mount-auth-error.test.ts | 119 ++++++++++++++++++ src/mount/mount-auth-error.ts | 102 +++++++++++++++ .../relayfile-cloud-mount-client.test.ts | 51 +++++++- src/mount/relayfile-cloud-mount-client.ts | 73 ++++++++++- src/orchestrator/factory.ts | 15 +++ src/ports/mount.ts | 8 ++ src/testing/fakes.ts | 7 ++ 10 files changed, 527 insertions(+), 23 deletions(-) create mode 100644 src/mount/mount-auth-error.test.ts create mode 100644 src/mount/mount-auth-error.ts diff --git a/src/cli/fleet.ts b/src/cli/fleet.ts index dec75568..1790266c 100644 --- a/src/cli/fleet.ts +++ b/src/cli/fleet.ts @@ -68,6 +68,7 @@ import { } from '../mount/relayfile-integration-preflight' import type { FactoryIntegrationProvider } from '../ports' import { checkMountStaleness } from '../mount/relayfile-binary' +import { MountAuthScopeError } from '../mount/mount-auth-error' interface FleetCliDeps { fleet?: FleetClient @@ -516,10 +517,24 @@ async function runFactoryCommand( const debugMountRefreshes = (deps.env ?? process.env).FACTORY_LOG_LEVEL?.toLowerCase() === 'debug' if (command.kind === 'factory') { if (command.action === 'start') { + const waiter = createStopSignalWaiter() + let stoppedBySignal = false + const flushAndResolve = async (code: number): Promise => { + try { + await (deps.flushDaemonOutput ?? flushProcessOutput)() + } finally { + waiter.resolve(code) + } + } // Local mirrors are a writeback aid, not the source of truth for remote // issue discovery. Start their SDK-backed supervisors immediately, but // do not serialize durable recovery behind a stale checkout's readiness // timeout. The mount client reports degradation and keeps retrying. + // + // A MountAuthScopeError is the exception: it is terminal (the cloud + // session lacks the filesystem scope the mount needs), so limping on + // would only spawn agents against a read-denied mirror. Fail fast with the + // remediation and resolve the command with a non-zero code. void warmStartPathMounts( mountFn, workspaceId, @@ -529,18 +544,15 @@ async function runFactoryCommand( debugMountRefreshes, ) .catch((error: unknown) => { + if (error instanceof MountAuthScopeError) { + mountStderr.write(`${error.message}\n`) + mountStderr.write('[factory] aborting startup: local mount cannot obtain its filesystem scopes.\n') + void flushAndResolve(1) + return + } const message = error instanceof Error ? error.message : String(error) mountStderr.write(`[factory] warning: background relayfile mount warmup failed: ${message}\n`) }) - const waiter = createStopSignalWaiter() - let stoppedBySignal = false - const flushAndResolve = async (code: number): Promise => { - try { - await (deps.flushDaemonOutput ?? flushProcessOutput)() - } finally { - waiter.resolve(code) - } - } const removeSignalHandlers = installFactoryStopSignalHandlers(factory, { exit: (code) => { stoppedBySignal = true @@ -810,6 +822,9 @@ async function ensureStandaloneBabysitMount( try { await mountFn(workspaceId, startDir, options) } catch (error) { + // Terminal scope shortfall: propagate so the command aborts with the + // remediation rather than silently falling back to a read-denied mirror. + if (error instanceof MountAuthScopeError) throw error const message = error instanceof Error ? error.message : String(error) stderr.write( `[factory] warning: could not start relayfile mount for standalone babysitter at ${resolve(startDir)}; ` + @@ -905,6 +920,10 @@ async function ensureMountPath( return { path: resolved, reason: staleBefore.reason } } } catch (error) { + // A scope shortfall is terminal and identical across every clone path; + // propagate it so startup fails fast with one remediation instead of + // logging the same unfixable warning per path. + if (error instanceof MountAuthScopeError) throw error const message = error instanceof Error ? error.message : String(error) stderr.write(`[factory] warning: could not start relayfile mount at ${resolved}: ${message}\n`) } diff --git a/src/mount/local-mount-preflight.test.ts b/src/mount/local-mount-preflight.test.ts index c6a8d36d..23e0b0f3 100644 --- a/src/mount/local-mount-preflight.test.ts +++ b/src/mount/local-mount-preflight.test.ts @@ -4,6 +4,7 @@ import { tmpdir } from 'node:os' import { afterEach, describe, expect, it, vi } from 'vitest' import { ensureLocalMount } from './local-mount-preflight' +import { MountAuthScopeError } from './mount-auth-error' afterEach(() => { vi.restoreAllMocks() @@ -206,4 +207,72 @@ describe('ensureLocalMount', () => { })).rejects.toThrow(/relayfile mount did not become ready/u) }) }) + + const SCOPE_403 = { + kind: 'bootstrap_stalled', + code: 'bootstrap_stall_cycle_limit', + message: 'http 403 forbidden: missing required scope: fs:read', + } + + async function writeMountStateWithError( + dir: string, + state: { workspaceId: string; lastReconcileAt: string; pid?: number }, + lastError: unknown, + ): Promise { + const stateDir = join(dir, '.integrations', '.relay') + await mkdir(stateDir, { recursive: true }) + await writeFile(join(stateDir, 'state.json'), JSON.stringify({ ...state, lastError }), 'utf8') + } + + it('throws a terminal MountAuthScopeError for a stale mount with a scope-403, without refreshing', async () => { + await withTempDir(async (dir) => { + await writeMountStateWithError(dir, { + workspaceId: 'rw_test', + lastReconcileAt: new Date(Date.now() - 30 * 60 * 1000).toISOString(), + pid: process.pid, + }, SCOPE_403) + const startMount = vi.fn(async () => {}) + + await expect(ensureLocalMount('rw_test', dir, { startMount })) + .rejects.toThrow(MountAuthScopeError) + // A re-launch would 403 again — the preflight must not attempt it. + expect(startMount).not.toHaveBeenCalled() + }) + }) + + it('surfaces the remediation but does not tear down a still-reconciling mount that reports a scope-403', async () => { + await withTempDir(async (dir) => { + await writeMountStateWithError(dir, { + workspaceId: 'rw_test', + lastReconcileAt: new Date().toISOString(), + pid: process.pid, + }, SCOPE_403) + const stderr = vi.spyOn(process.stderr, 'write').mockImplementation(() => true) + const startMount = vi.fn(async () => {}) + + await expect(ensureLocalMount('rw_test', dir, { startMount })).resolves.toBeUndefined() + expect(startMount).not.toHaveBeenCalled() + expect(stderr).toHaveBeenCalledWith(expect.stringContaining('lacks the filesystem scope')) + }) + }) + + it('escalates a first-ever bootstrap that 403s into a terminal MountAuthScopeError', async () => { + await withTempDir(async (dir) => { + // No state file at start; the bootstrap "succeeds" but writes a stale + // state recording the scope 403 and never becomes ready. + const startMount = vi.fn(async () => { + await writeMountStateWithError(dir, { + workspaceId: 'rw_test', + lastReconcileAt: new Date(Date.now() - 30 * 60 * 1000).toISOString(), + pid: process.pid, + }, SCOPE_403) + }) + + await expect(ensureLocalMount('rw_test', dir, { + startMount, + stateWaitTimeoutMs: 5, + stateWaitPollMs: 1, + })).rejects.toThrow(MountAuthScopeError) + }) + }) }) diff --git a/src/mount/local-mount-preflight.ts b/src/mount/local-mount-preflight.ts index 57fa5991..3235eceb 100644 --- a/src/mount/local-mount-preflight.ts +++ b/src/mount/local-mount-preflight.ts @@ -3,6 +3,12 @@ import { join } from 'node:path' import type { LocalMountOptions } from '../ports' import { checkMountStaleness } from './relayfile-binary' +import { + classifyMountAuthError, + MountAuthScopeError, + mountAuthRemediation, + readMountAuthErrorFromState, +} from './mount-auth-error' const STATE_FILE = '.integrations/.relay/state.json' @@ -32,23 +38,57 @@ export async function ensureLocalMount( const stateFilePath = join(startDir, STATE_FILE) if (!(await isMountStatePresent(stateFilePath))) { - await options.startMount() - await waitForStateFile( - stateFilePath, - workspaceId, - options.stateWaitTimeoutMs, - options.stateWaitPollMs, - options.acceptableWorkspaceIds, - ) + try { + await options.startMount() + await waitForStateFile( + stateFilePath, + workspaceId, + options.stateWaitTimeoutMs, + options.stateWaitPollMs, + options.acceptableWorkspaceIds, + ) + } catch (error) { + if (error instanceof MountAuthScopeError) throw error + // A first-ever bootstrap that 403s writes a state.json recording the + // scope shortfall but never becomes ready. Convert that into the typed, + // terminal error so startup fails fast with a clear remediation. + const reason = error instanceof Error ? error.message : String(error) + const authError = readMountAuthErrorFromState(stateFilePath) ?? classifyMountAuthError(reason) + if (authError) { + throw new MountAuthScopeError(mountAuthRemediation(authError), { + missingScope: authError.missingScope, + cause: error, + }) + } + throw error + } return } const staleness = checkMountStaleness(stateFilePath, workspaceId, options.acceptableWorkspaceIds) - if (!staleness.stale) return + + // A filesystem-scope shortfall is terminal: the mount records a 403 in its + // state.json, and re-launching only 403s again. Detect it up front so we + // neither loop refreshing a doomed mount nor let the degradation stay silent. + const authError = readMountAuthErrorFromState(stateFilePath) + + if (!staleness.stale) { + // The mount is reconciling. If it still reports a scope shortfall (e.g. a + // root bootstrap that 403s while scoped subtrees sync), surface it once — + // but never tear down a working mount over it. + if (authError) process.stderr.write(`${mountAuthRemediation(authError)}\n`) + return + } const suffix = staleness.reason !== undefined ? ` (${staleness.reason})` : '' const manualHint = 'Restart Factory after restoring the Agent Relay Cloud session' + // Stale AND under-scoped: refreshing cannot help. Fail fast and terminal so + // the supervisor stops retrying and startup surfaces one actionable error. + if (authError) { + throw new MountAuthScopeError(mountAuthRemediation(authError), { missingScope: authError.missingScope }) + } + if (options.refreshStaleMount === false) { process.stderr.write(`[factory] local mount is stale${suffix}; writeback may not propagate. ${manualHint}\n`) return @@ -72,7 +112,18 @@ export async function ensureLocalMount( process.stderr.write('[factory] local mount refreshed\n') } } catch (error) { + if (error instanceof MountAuthScopeError) throw error const reason = error instanceof Error ? error.message : String(error) + // The failed refresh may have written a fresh 403 into state.json, or the + // failure reason itself may name the missing scope. Either way it is + // terminal — escalate so the supervisor stops retrying. + const postAuthError = readMountAuthErrorFromState(stateFilePath) ?? classifyMountAuthError(reason) + if (postAuthError) { + throw new MountAuthScopeError(mountAuthRemediation(postAuthError), { + missingScope: postAuthError.missingScope, + cause: error, + }) + } process.stderr.write(`[factory] local mount is stale${suffix} and auto-refresh failed (${reason}); writeback may not propagate. ${manualHint}\n`) } } diff --git a/src/mount/mount-auth-error.test.ts b/src/mount/mount-auth-error.test.ts new file mode 100644 index 00000000..aae9fa49 --- /dev/null +++ b/src/mount/mount-auth-error.test.ts @@ -0,0 +1,119 @@ +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { describe, expect, it } from 'vitest' + +import { + classifyMountAuthError, + MountAuthScopeError, + mountAuthRemediation, + readMountAuthErrorFromState, +} from './mount-auth-error' + +async function withTempStateFile( + lastError: unknown, + fn: (stateFilePath: string) => Promise | void, +): Promise { + const dir = await mkdtemp(join(tmpdir(), 'mount-auth-error-test-')) + try { + const stateDir = join(dir, '.relay') + await mkdir(stateDir, { recursive: true }) + const stateFilePath = join(stateDir, 'state.json') + await writeFile( + stateFilePath, + JSON.stringify({ workspaceId: 'rw_test', lastReconcileAt: new Date().toISOString(), lastError }), + 'utf8', + ) + await fn(stateFilePath) + } finally { + await rm(dir, { recursive: true, force: true }) + } +} + +describe('classifyMountAuthError', () => { + it('extracts the missing scope from a 403 message', () => { + const details = classifyMountAuthError('http 403 forbidden: missing required scope: fs:read') + expect(details?.missingScope).toBe('fs:read') + }) + + it('matches a stalled bootstrap that reports forbidden without a named scope', () => { + const details = classifyMountAuthError('bootstrap stalled for 200 cycles: 403 forbidden') + expect(details).toBeDefined() + expect(details?.missingScope).toBeUndefined() + }) + + it('handles the real bootstrap_stall_cycle_limit signature verbatim', () => { + const details = classifyMountAuthError( + 'bootstrap stalled for 200 consecutive checkpoint-stable cycles (limit 20, cursor ""): http 403 forbidden: missing required scope: fs:read', + ) + expect(details?.missingScope).toBe('fs:read') + }) + + it('does not misclassify a transient/unrelated failure', () => { + expect(classifyMountAuthError('context deadline exceeded')).toBeUndefined() + expect(classifyMountAuthError('relayfile mount did not become ready within 60000ms')).toBeUndefined() + expect(classifyMountAuthError(undefined)).toBeUndefined() + expect(classifyMountAuthError('')).toBeUndefined() + }) + + it('does not treat a bare 403 (no stall, no scope) as a scope failure', () => { + // A one-off 403 without the stall signature or a named scope is ambiguous; + // classification must not fire and trigger a terminal abort. + expect(classifyMountAuthError('request returned 403')).toBeUndefined() + }) +}) + +describe('readMountAuthErrorFromState', () => { + it('detects a scope shortfall recorded in state.json lastError.message', async () => { + await withTempStateFile( + { + kind: 'bootstrap_stalled', + code: 'bootstrap_stall_cycle_limit', + message: 'http 403 forbidden: missing required scope: fs:read', + }, + (stateFilePath) => { + const details = readMountAuthErrorFromState(stateFilePath) + expect(details?.missingScope).toBe('fs:read') + }, + ) + }) + + it('returns undefined when lastError is unrelated', async () => { + await withTempStateFile( + { kind: 'reconcile_error', message: 'context deadline exceeded' }, + (stateFilePath) => { + expect(readMountAuthErrorFromState(stateFilePath)).toBeUndefined() + }, + ) + }) + + it('returns undefined when there is no lastError', async () => { + await withTempStateFile(undefined, (stateFilePath) => { + expect(readMountAuthErrorFromState(stateFilePath)).toBeUndefined() + }) + }) + + it('returns undefined for a missing state file', () => { + expect(readMountAuthErrorFromState('/nonexistent/path/state.json')).toBeUndefined() + }) +}) + +describe('mountAuthRemediation', () => { + it('names the missing scope when known', () => { + expect(mountAuthRemediation({ missingScope: 'fs:read', detail: 'x' })).toContain('missing fs:read') + }) + + it('is still actionable without a scope', () => { + const msg = mountAuthRemediation() + expect(msg).toContain('Re-authenticate') + }) +}) + +describe('MountAuthScopeError', () => { + it('carries the missing scope and is an Error', () => { + const err = new MountAuthScopeError('boom', { missingScope: 'fs:read' }) + expect(err).toBeInstanceOf(Error) + expect(err.name).toBe('MountAuthScopeError') + expect(err.missingScope).toBe('fs:read') + }) +}) diff --git a/src/mount/mount-auth-error.ts b/src/mount/mount-auth-error.ts new file mode 100644 index 00000000..44c19a61 --- /dev/null +++ b/src/mount/mount-auth-error.ts @@ -0,0 +1,102 @@ +import { readFileSync } from 'node:fs' + +/** + * A mount failure caused by the cloud session lacking the filesystem scopes the + * relayfile mount needs (e.g. `relayfile:fs:read`). Unlike a stale mount or a + * transient launch failure, re-launching cannot fix this — the token itself is + * under-scoped — so callers must treat it as terminal: surface an actionable + * remediation instead of scheduling an infinite refresh loop, and refuse to + * spawn/resume agents against the resulting stale mirror. + */ +export class MountAuthScopeError extends Error { + readonly missingScope?: string + readonly cause?: unknown + + constructor(message: string, options?: { missingScope?: string; cause?: unknown }) { + super(message) + this.name = 'MountAuthScopeError' + this.missingScope = options?.missingScope + this.cause = options?.cause + } +} + +export interface MountAuthErrorDetails { + /** The scope named in the 403, when the server reported one (e.g. `fs:read`). */ + missingScope?: string + /** The raw diagnostic text the classification matched against. */ + detail: string +} + +// The relayfile server reports a scope shortfall as an HTTP 403 whose body names +// the required scope. The mount SDK surfaces it verbatim in log/error text and +// in state.json's `lastError.message`, e.g.: +// "http 403 forbidden: missing required scope: fs:read" +// A prolonged shortfall also shows up as a stalled bootstrap that never clears. +const MISSING_SCOPE_RE = /missing required scope:\s*([\w:*/-]+)/i +const FORBIDDEN_RE = /\b403\b|forbidden/i +const BOOTSTRAP_STALL_RE = /bootstrap[_ ]stall/i + +/** + * Classify a diagnostic string as a filesystem-scope authorization failure. + * Returns the matched details, or `undefined` when the text is some other + * (transient / unrelated) failure. + */ +export function classifyMountAuthError(text: string | undefined | null): MountAuthErrorDetails | undefined { + if (!text) return undefined + const scopeMatch = MISSING_SCOPE_RE.exec(text) + if (scopeMatch) { + return { missingScope: scopeMatch[1], detail: text } + } + // A stalled bootstrap that also reports a forbidden/403 is the same + // under-scoped session even when the specific scope name is absent. + if (BOOTSTRAP_STALL_RE.test(text) && FORBIDDEN_RE.test(text)) { + return { detail: text } + } + return undefined +} + +/** + * Read a relayfile mount `state.json` and, if its most recent error is a + * filesystem-scope authorization failure, return the details. Returns + * `undefined` when the file is absent, unreadable, or the error is unrelated. + */ +export function readMountAuthErrorFromState(stateFilePath: string): MountAuthErrorDetails | undefined { + let raw: string + try { + raw = readFileSync(stateFilePath, 'utf8') + } catch { + return undefined + } + let parsed: unknown + try { + parsed = JSON.parse(raw) + } catch { + return undefined + } + if (typeof parsed !== 'object' || parsed === null) return undefined + const lastError = (parsed as { lastError?: unknown }).lastError + if (typeof lastError !== 'object' || lastError === null) return undefined + const { message, kind, code } = lastError as { message?: unknown; kind?: unknown; code?: unknown } + const fields = [message, kind, code].filter((v): v is string => typeof v === 'string') + for (const field of fields) { + const details = classifyMountAuthError(field) + if (details) return details + } + return undefined +} + +/** + * Build the single, actionable remediation message shown when the mount cannot + * obtain its filesystem scopes. Kept in one place so preflight, the supervisor, + * and startup all speak with one voice. + */ +export function mountAuthRemediation(details?: MountAuthErrorDetails): string { + const scope = details?.missingScope ? ` (missing ${details.missingScope})` : '' + return ( + `[factory] Agent Relay Cloud session lacks the filesystem scope the mount needs${scope}. ` + + 'Re-authenticate with a session that can mint relayfile fs:read/fs:write for this workspace ' + + '(an organization owner/admin session); a plain `agent-relay cloud login` grants only ' + + 'follow-user/cli:auth and cannot mint fs scopes. Factory will not spawn agents against a ' + + 'read-denied mirror.' + ) +} diff --git a/src/mount/relayfile-cloud-mount-client.test.ts b/src/mount/relayfile-cloud-mount-client.test.ts index 1bc59b14..f8050f55 100644 --- a/src/mount/relayfile-cloud-mount-client.test.ts +++ b/src/mount/relayfile-cloud-mount-client.test.ts @@ -21,6 +21,7 @@ import { type RelayFileClientLike, } from './relayfile-cloud-mount-client' import { RelayfileGithubConnectionWrite } from './relayfile-github-connection-write' +import { MountAuthScopeError } from './mount-auth-error' const storedAuth = (overrides: Partial = {}): StoredAuth => ({ apiUrl: 'https://cloud.example', @@ -460,6 +461,51 @@ describe('RelayfileCloudMountClient', () => { } }) + it('treats a MountAuthScopeError as terminal: no retry, marks auth-degraded', async () => { + vi.useFakeTimers() + const fake = new FakeRelayFileClient() + const stop = vi.fn(async () => {}) + const ensureMountedWorkspace = vi.fn(async () => ({ stop })) + const localMountPreflight = vi.fn(async () => { + throw new MountAuthScopeError('missing fs:read', { missingScope: 'fs:read' }) + }) + const healthEvents: Array<{ state: string; reason: string; degradedMounts: number }> = [] + const mount = new RelayfileCloudMountClient({ + workspaceId: 'rw_test', + client: fake, + relayfileSetup: { + joinWorkspace: vi.fn(), + ensureMountedWorkspace, + }, + relayfileWorkspace: { + workspaceId: 'cloud-workspace-uuid', + client: () => fake, + getToken: async () => 'delegated-relayfile-token', + info: { relayfileUrl: 'https://relayfile.example' }, + }, + localMountPreflight, + localMountHealthIntervalMs: 1_000, + onLocalMountHealth: (event) => { healthEvents.push(event) }, + }) + + try { + await expect(mount.ensureLocalMount('/work/repo')).rejects.toBeInstanceOf(MountAuthScopeError) + expect(mount.isLocalMountAuthDegraded()).toBe(true) + expect(healthEvents).toEqual([{ + state: 'degraded', + reason: 'mount_auth_scope', + degradedMounts: 1, + }]) + + // The supervisor must NOT re-run the preflight for a terminal failure. + await vi.advanceTimersByTimeAsync(5_000) + expect(localMountPreflight).toHaveBeenCalledTimes(1) + } finally { + await mount.dispose() + vi.useRealTimers() + } + }) + it('coalesces concurrent mount checks for the same checkout', async () => { const fake = new FakeRelayFileClient() let releasePreflight!: () => void @@ -726,7 +772,10 @@ describe('RelayfileCloudMountClient', () => { expect(joinOptions.scopes).not.toContain('relayfile:fs:read:/**') expect(joinOptions.scopes).not.toContain('relayfile:fs:write:/**') expect(joinOptions.scopes).toContain('relayfile:fs:read:/linear/states/**') - expect(joinOptions.scopes).toContain('relayfile:fs:write:/github/repos/**') + // github uses the provider root `/github/**`; `/github/repos/**` is rejected + // by RelayAuth's path-token validator and would fail the whole batch mint. + expect(joinOptions.scopes).toContain('relayfile:fs:write:/github/**') + expect(joinOptions.scopes).not.toContain('relayfile:fs:write:/github/repos/**') expect(joinOptions.scopes).toContain('relayfile:fs:write:/factory/observability/**') expect(joinOptions.scopes).toContain('relayfile:fs:read:/slack/users/**') expect(mount.githubWrite).toBeDefined() diff --git a/src/mount/relayfile-cloud-mount-client.ts b/src/mount/relayfile-cloud-mount-client.ts index b9f3841f..fb450021 100644 --- a/src/mount/relayfile-cloud-mount-client.ts +++ b/src/mount/relayfile-cloud-mount-client.ts @@ -50,6 +50,7 @@ import { type EnsureLocalMountOptions, } from './local-mount-preflight' import { checkMountStaleness } from './relayfile-binary' +import { MountAuthScopeError } from './mount-auth-error' const DEFAULT_WORKSPACE_ID = 'rw_7ccfea89' const DEFAULT_AGENT_NAME = 'agent-relay-factory' @@ -59,8 +60,15 @@ export const FACTORY_RELAYFILE_SCOPES = [ 'relayfile:fs:read:/linear/issues/**', 'relayfile:fs:read:/linear/states/**', 'relayfile:fs:write:/linear/issues/**', - 'relayfile:fs:read:/github/repos/**', - 'relayfile:fs:write:/github/repos/**', + // RelayAuth's path-token validator rejects `/github/repos/**` as an invalid + // relayfile path (github paths must be the provider root or carry an owner + // segment), and because all scopes mint in one batch that single bad path + // fails the ENTIRE delegated-token mint with `invalid_paths` — leaving the + // mount with no fs token and a `403 missing required scope: fs:read` on every + // read. `/github/**` is the github provider root (a valid superset) and mints + // cleanly. Do NOT narrow this back to `/github/repos/**`. + 'relayfile:fs:read:/github/**', + 'relayfile:fs:write:/github/**', 'relayfile:fs:read:/slack/channels/**', 'relayfile:fs:write:/slack/channels/**', 'relayfile:fs:read:/slack/users/**', @@ -140,7 +148,10 @@ export interface MountedWorkspaceHandleLike { export interface LocalMountHealthEvent { state: 'degraded' | 'recovered' - reason: 'mount_stale' | 'mount_refresh_failed' + // `mount_auth_scope` is terminal: the cloud session lacks the filesystem + // scope the mount needs, so — unlike `mount_stale`/`mount_refresh_failed` — + // the supervisor stops retrying rather than looping against a doomed refresh. + reason: 'mount_stale' | 'mount_refresh_failed' | 'mount_auth_scope' degradedMounts: number } @@ -266,6 +277,10 @@ export class RelayfileCloudMountClient implements MountClient { readonly #localMountOperations = new Map>() readonly #localMountOperationWaiters: Array<() => void> = [] readonly #degradedLocalMounts = new Set() + // Mounts that failed with a terminal auth-scope shortfall. These are NOT + // rescheduled for refresh (a re-launch just 403s again); recovery requires + // re-authenticating and restarting Factory. + readonly #authDegradedLocalMounts = new Set() #activeLocalMountOperations = 0 #disposed = false #isAllowedDraft?: (path: string, content: unknown, opts?: { guarded?: boolean }) => boolean | Promise @@ -410,6 +425,13 @@ export class RelayfileCloudMountClient implements MountClient { }, }) } catch (error) { + // A terminal auth-scope shortfall cannot be healed by re-launching, so + // mark it distinctly and do NOT arm the retry supervisor — just surface it + // to startup, which fails fast with the remediation. + if (error instanceof MountAuthScopeError) { + this.#markLocalMountAuthDegraded(localDir) + throw error + } // A first-ever mount has no state file from which staleness can be // inferred. Surface the launch failure and arm the same retry supervisor // used for later stale sessions before returning control to startup. @@ -463,6 +485,7 @@ export class RelayfileCloudMountClient implements MountClient { this.#localMountHealthTimers.clear() this.#localMountSupervisions.clear() this.#degradedLocalMounts.clear() + this.#authDegradedLocalMounts.clear() const mounted = [...this.#localMounts.values()] this.#localMounts.clear() await Promise.allSettled(mounted.map(async (handle) => handle.stop())) @@ -608,6 +631,9 @@ export class RelayfileCloudMountClient implements MountClient { #scheduleLocalMountHealthCheck(localDir: string): void { if (this.#disposed || this.#localMountHealthTimers.has(localDir)) return + // A terminal auth-scope failure is never retried — recovery requires + // re-auth + restart, not another refresh cycle. + if (this.#authDegradedLocalMounts.has(localDir)) return const supervision = this.#localMountSupervisions.get(localDir) if (!supervision) return const untilRefresh = supervision.suggestedRefreshAtMs === undefined @@ -641,12 +667,20 @@ export class RelayfileCloudMountClient implements MountClient { await this.ensureLocalMount(supervision.startDir, supervision.options) } } catch (error) { + if (error instanceof MountAuthScopeError) { + // Terminal: stop the retry loop for this mount entirely. + this.#markLocalMountAuthDegraded(localDir) + return + } this.#logger?.warn?.('[factory] supervised Relayfile mount refresh failed', { errorClass: error instanceof Error ? error.name : 'Error', }) this.#markLocalMountDegraded(localDir, 'mount_refresh_failed') } finally { - this.#scheduleLocalMountHealthCheck(localDir) + // Never re-arm the supervisor for a terminally auth-degraded mount. + if (!this.#authDegradedLocalMounts.has(localDir)) { + this.#scheduleLocalMountHealthCheck(localDir) + } } } @@ -661,7 +695,38 @@ export class RelayfileCloudMountClient implements MountClient { }) } + /** + * Whether any local mount is terminally degraded by a filesystem-scope + * shortfall. Consumers (e.g. the factory loop) use this to refuse to spawn or + * resume agents against a read-denied mirror. + */ + isLocalMountAuthDegraded(): boolean { + return this.#authDegradedLocalMounts.size > 0 + } + + #markLocalMountAuthDegraded(localDir: string): void { + // Stop any pending refresh for this mount; it is terminal. + const timer = this.#localMountHealthTimers.get(localDir) + if (timer) { + clearTimeout(timer) + this.#localMountHealthTimers.delete(localDir) + } + const wasHealthy = this.#degradedLocalMounts.size === 0 + const wasAuthDegraded = this.#authDegradedLocalMounts.has(localDir) + this.#degradedLocalMounts.add(localDir) + this.#authDegradedLocalMounts.add(localDir) + // Emit once per transition into the auth-degraded state. + if (wasHealthy || !wasAuthDegraded) { + this.#emitLocalMountHealth({ + state: 'degraded', + reason: 'mount_auth_scope', + degradedMounts: this.#degradedLocalMounts.size, + }) + } + } + #markLocalMountRecovered(localDir: string): void { + this.#authDegradedLocalMounts.delete(localDir) if (!this.#degradedLocalMounts.delete(localDir) || this.#degradedLocalMounts.size > 0) return this.#emitLocalMountHealth({ state: 'recovered', diff --git a/src/orchestrator/factory.ts b/src/orchestrator/factory.ts index d0468b6e..d2e86673 100644 --- a/src/orchestrator/factory.ts +++ b/src/orchestrator/factory.ts @@ -6561,6 +6561,21 @@ export class FactoryLoop implements Factory { return } + // Never resume an agent against a read-denied mirror: the mount lacks the + // filesystem scope it needs, so the spawn would fail opaquely (roster PID + // never resolves) and risk operating on stale integration state. Skip with + // one clear message; a later attempt succeeds once the session is re-authed + // and Factory restarted. + if (this.#mount.isLocalMountAuthDegraded?.()) { + this.#increment('resumeSkippedMountAuthDegraded') + this.#logger.warn?.('[factory] tracked agent resume skipped: local mount is auth-degraded (cloud session missing relayfile fs scope); re-authenticate and restart Factory', { + issue: record.issue.key, + name, + role: tracked.spec.role, + }) + return + } + this.#logger.debug?.('[factory] tracked agent resume preparation started', { issue: record.issue.key, name, diff --git a/src/ports/mount.ts b/src/ports/mount.ts index c8c47652..f2532010 100644 --- a/src/ports/mount.ts +++ b/src/ports/mount.ts @@ -108,6 +108,14 @@ export interface MountClient { readonly integrationConnections?: FactoryIntegrationConnections /** Ensure the SDK-authenticated Relayfile mirror exists below a checkout. */ ensureLocalMount?(startDir: string, options?: LocalMountOptions): Promise + /** + * Whether a local mount is terminally degraded because the cloud session + * lacks the filesystem scope the mount needs. When true, the mirror is + * read-denied and cannot recover without re-auth + restart, so consumers must + * refuse to spawn or resume agents against it. Absent on mounts that cannot + * report scope health (e.g. the test transport). + */ + isLocalMountAuthDegraded?(): boolean /** Stop SDK-owned local mount processes created by this client. */ dispose?(): Promise readFile(path: string): Promise<{ content: unknown; revision?: string }> diff --git a/src/testing/fakes.ts b/src/testing/fakes.ts index 64eaa069..ae603fb1 100644 --- a/src/testing/fakes.ts +++ b/src/testing/fakes.ts @@ -34,6 +34,9 @@ export class FakeMountClient implements MountClient { readonly deletes: string[] = [] readonly reads: string[] = [] subscribeCount = 0 + /** Test toggle: when true, `isLocalMountAuthDegraded()` reports the terminal + * scope-shortfall state so consumers (e.g. resume gating) can be exercised. */ + authDegraded = false #subscribers = new Set<(event: ChangeEvent) => void>() #events: ChangeEvent[] = [] @@ -78,6 +81,10 @@ export class FakeMountClient implements MountClient { return [...this.files.keys()].filter((path) => path.startsWith(prefix)).sort() } + isLocalMountAuthDegraded(): boolean { + return this.authDegraded + } + subscribe(_globs: string[], onChange: (event: ChangeEvent) => void, _opts?: SubscribeOptions): Subscription { this.subscribeCount += 1 this.#subscribers.add(onChange)