Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 28 additions & 9 deletions src/cli/fleet.ts
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ import {
} from '../mount/relayfile-integration-preflight'
import type { FactoryIntegrationProvider } from '../ports'
import { checkMountStaleness } from '../mount/relayfile-binary'
import { MountAuthScopeError } from '../mount/mount-auth-error'

interface FleetCliDeps {
fleet?: FleetClient
Expand Down Expand Up @@ -516,10 +517,24 @@ async function runFactoryCommand(
const debugMountRefreshes = (deps.env ?? process.env).FACTORY_LOG_LEVEL?.toLowerCase() === 'debug'
if (command.kind === 'factory') {
if (command.action === 'start') {
const waiter = createStopSignalWaiter()
let stoppedBySignal = false
const flushAndResolve = async (code: number): Promise<void> => {
try {
await (deps.flushDaemonOutput ?? flushProcessOutput)()
} finally {
waiter.resolve(code)
}
}
// Local mirrors are a writeback aid, not the source of truth for remote
// issue discovery. Start their SDK-backed supervisors immediately, but
// do not serialize durable recovery behind a stale checkout's readiness
// timeout. The mount client reports degradation and keeps retrying.
//
// A MountAuthScopeError is the exception: it is terminal (the cloud
// session lacks the filesystem scope the mount needs), so limping on
// would only spawn agents against a read-denied mirror. Fail fast with the
// remediation and resolve the command with a non-zero code.
void warmStartPathMounts(
mountFn,
workspaceId,
Expand All @@ -529,18 +544,15 @@ async function runFactoryCommand(
debugMountRefreshes,
)
.catch((error: unknown) => {
if (error instanceof MountAuthScopeError) {
mountStderr.write(`${error.message}\n`)
mountStderr.write('[factory] aborting startup: local mount cannot obtain its filesystem scopes.\n')
void flushAndResolve(1)
return
}
const message = error instanceof Error ? error.message : String(error)
mountStderr.write(`[factory] warning: background relayfile mount warmup failed: ${message}\n`)
})
const waiter = createStopSignalWaiter()
let stoppedBySignal = false
const flushAndResolve = async (code: number): Promise<void> => {
try {
await (deps.flushDaemonOutput ?? flushProcessOutput)()
} finally {
waiter.resolve(code)
}
}
const removeSignalHandlers = installFactoryStopSignalHandlers(factory, {
exit: (code) => {
stoppedBySignal = true
Expand Down Expand Up @@ -810,6 +822,9 @@ async function ensureStandaloneBabysitMount(
try {
await mountFn(workspaceId, startDir, options)
} catch (error) {
// Terminal scope shortfall: propagate so the command aborts with the
// remediation rather than silently falling back to a read-denied mirror.
if (error instanceof MountAuthScopeError) throw error
const message = error instanceof Error ? error.message : String(error)
stderr.write(
`[factory] warning: could not start relayfile mount for standalone babysitter at ${resolve(startDir)}; ` +
Expand Down Expand Up @@ -905,6 +920,10 @@ async function ensureMountPath(
return { path: resolved, reason: staleBefore.reason }
}
} catch (error) {
// A scope shortfall is terminal and identical across every clone path;
// propagate it so startup fails fast with one remediation instead of
// logging the same unfixable warning per path.
if (error instanceof MountAuthScopeError) throw error
const message = error instanceof Error ? error.message : String(error)
stderr.write(`[factory] warning: could not start relayfile mount at ${resolved}: ${message}\n`)
}
Expand Down
69 changes: 69 additions & 0 deletions src/mount/local-mount-preflight.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { tmpdir } from 'node:os'
import { afterEach, describe, expect, it, vi } from 'vitest'

import { ensureLocalMount } from './local-mount-preflight'
import { MountAuthScopeError } from './mount-auth-error'

afterEach(() => {
vi.restoreAllMocks()
Expand Down Expand Up @@ -206,4 +207,72 @@ describe('ensureLocalMount', () => {
})).rejects.toThrow(/relayfile mount did not become ready/u)
})
})

const SCOPE_403 = {
kind: 'bootstrap_stalled',
code: 'bootstrap_stall_cycle_limit',
message: 'http 403 forbidden: missing required scope: fs:read',
}

async function writeMountStateWithError(
dir: string,
state: { workspaceId: string; lastReconcileAt: string; pid?: number },
lastError: unknown,
): Promise<void> {
const stateDir = join(dir, '.integrations', '.relay')
await mkdir(stateDir, { recursive: true })
await writeFile(join(stateDir, 'state.json'), JSON.stringify({ ...state, lastError }), 'utf8')
}

it('throws a terminal MountAuthScopeError for a stale mount with a scope-403, without refreshing', async () => {
await withTempDir(async (dir) => {
await writeMountStateWithError(dir, {
workspaceId: 'rw_test',
lastReconcileAt: new Date(Date.now() - 30 * 60 * 1000).toISOString(),
pid: process.pid,
}, SCOPE_403)
const startMount = vi.fn(async () => {})

await expect(ensureLocalMount('rw_test', dir, { startMount }))
.rejects.toThrow(MountAuthScopeError)
// A re-launch would 403 again — the preflight must not attempt it.
expect(startMount).not.toHaveBeenCalled()
})
})

it('surfaces the remediation but does not tear down a still-reconciling mount that reports a scope-403', async () => {
await withTempDir(async (dir) => {
await writeMountStateWithError(dir, {
workspaceId: 'rw_test',
lastReconcileAt: new Date().toISOString(),
pid: process.pid,
}, SCOPE_403)
const stderr = vi.spyOn(process.stderr, 'write').mockImplementation(() => true)
const startMount = vi.fn(async () => {})

await expect(ensureLocalMount('rw_test', dir, { startMount })).resolves.toBeUndefined()
expect(startMount).not.toHaveBeenCalled()
expect(stderr).toHaveBeenCalledWith(expect.stringContaining('lacks the filesystem scope'))
})
})

it('escalates a first-ever bootstrap that 403s into a terminal MountAuthScopeError', async () => {
await withTempDir(async (dir) => {
// No state file at start; the bootstrap "succeeds" but writes a stale
// state recording the scope 403 and never becomes ready.
const startMount = vi.fn(async () => {
await writeMountStateWithError(dir, {
workspaceId: 'rw_test',
lastReconcileAt: new Date(Date.now() - 30 * 60 * 1000).toISOString(),
pid: process.pid,
}, SCOPE_403)
})

await expect(ensureLocalMount('rw_test', dir, {
startMount,
stateWaitTimeoutMs: 5,
stateWaitPollMs: 1,
})).rejects.toThrow(MountAuthScopeError)
})
})
})
69 changes: 60 additions & 9 deletions src/mount/local-mount-preflight.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,12 @@ import { join } from 'node:path'

import type { LocalMountOptions } from '../ports'
import { checkMountStaleness } from './relayfile-binary'
import {
classifyMountAuthError,
MountAuthScopeError,
mountAuthRemediation,
readMountAuthErrorFromState,
} from './mount-auth-error'

const STATE_FILE = '.integrations/.relay/state.json'

Expand Down Expand Up @@ -32,23 +38,57 @@ export async function ensureLocalMount(
const stateFilePath = join(startDir, STATE_FILE)

if (!(await isMountStatePresent(stateFilePath))) {
await options.startMount()
await waitForStateFile(
stateFilePath,
workspaceId,
options.stateWaitTimeoutMs,
options.stateWaitPollMs,
options.acceptableWorkspaceIds,
)
try {
await options.startMount()
await waitForStateFile(
stateFilePath,
workspaceId,
options.stateWaitTimeoutMs,
options.stateWaitPollMs,
options.acceptableWorkspaceIds,
)
} catch (error) {
if (error instanceof MountAuthScopeError) throw error
// A first-ever bootstrap that 403s writes a state.json recording the
// scope shortfall but never becomes ready. Convert that into the typed,
// terminal error so startup fails fast with a clear remediation.
const reason = error instanceof Error ? error.message : String(error)
const authError = readMountAuthErrorFromState(stateFilePath) ?? classifyMountAuthError(reason)
if (authError) {
throw new MountAuthScopeError(mountAuthRemediation(authError), {
missingScope: authError.missingScope,
cause: error,
})
}
throw error
}
return
}

const staleness = checkMountStaleness(stateFilePath, workspaceId, options.acceptableWorkspaceIds)
if (!staleness.stale) return

// A filesystem-scope shortfall is terminal: the mount records a 403 in its
// state.json, and re-launching only 403s again. Detect it up front so we
// neither loop refreshing a doomed mount nor let the degradation stay silent.
const authError = readMountAuthErrorFromState(stateFilePath)

if (!staleness.stale) {
// The mount is reconciling. If it still reports a scope shortfall (e.g. a
// root bootstrap that 403s while scoped subtrees sync), surface it once —
// but never tear down a working mount over it.
if (authError) process.stderr.write(`${mountAuthRemediation(authError)}\n`)
return
}

const suffix = staleness.reason !== undefined ? ` (${staleness.reason})` : ''
const manualHint = 'Restart Factory after restoring the Agent Relay Cloud session'

// Stale AND under-scoped: refreshing cannot help. Fail fast and terminal so
// the supervisor stops retrying and startup surfaces one actionable error.
if (authError) {
throw new MountAuthScopeError(mountAuthRemediation(authError), { missingScope: authError.missingScope })
}

if (options.refreshStaleMount === false) {
process.stderr.write(`[factory] local mount is stale${suffix}; writeback may not propagate. ${manualHint}\n`)
return
Expand All @@ -72,7 +112,18 @@ export async function ensureLocalMount(
process.stderr.write('[factory] local mount refreshed\n')
}
} catch (error) {
if (error instanceof MountAuthScopeError) throw error
const reason = error instanceof Error ? error.message : String(error)
// The failed refresh may have written a fresh 403 into state.json, or the
// failure reason itself may name the missing scope. Either way it is
// terminal — escalate so the supervisor stops retrying.
const postAuthError = readMountAuthErrorFromState(stateFilePath) ?? classifyMountAuthError(reason)
if (postAuthError) {
throw new MountAuthScopeError(mountAuthRemediation(postAuthError), {
missingScope: postAuthError.missingScope,
cause: error,
})
}
process.stderr.write(`[factory] local mount is stale${suffix} and auto-refresh failed (${reason}); writeback may not propagate. ${manualHint}\n`)
}
}
Expand Down
119 changes: 119 additions & 0 deletions src/mount/mount-auth-error.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
import { join } from 'node:path'
import { tmpdir } from 'node:os'
import { describe, expect, it } from 'vitest'

import {
classifyMountAuthError,
MountAuthScopeError,
mountAuthRemediation,
readMountAuthErrorFromState,
} from './mount-auth-error'

async function withTempStateFile(
lastError: unknown,
fn: (stateFilePath: string) => Promise<void> | void,
): Promise<void> {
const dir = await mkdtemp(join(tmpdir(), 'mount-auth-error-test-'))
try {
const stateDir = join(dir, '.relay')
await mkdir(stateDir, { recursive: true })
const stateFilePath = join(stateDir, 'state.json')
await writeFile(
stateFilePath,
JSON.stringify({ workspaceId: 'rw_test', lastReconcileAt: new Date().toISOString(), lastError }),
'utf8',
)
await fn(stateFilePath)
} finally {
await rm(dir, { recursive: true, force: true })
}
}

describe('classifyMountAuthError', () => {
it('extracts the missing scope from a 403 message', () => {
const details = classifyMountAuthError('http 403 forbidden: missing required scope: fs:read')
expect(details?.missingScope).toBe('fs:read')
})

it('matches a stalled bootstrap that reports forbidden without a named scope', () => {
const details = classifyMountAuthError('bootstrap stalled for 200 cycles: 403 forbidden')
expect(details).toBeDefined()
expect(details?.missingScope).toBeUndefined()
})

it('handles the real bootstrap_stall_cycle_limit signature verbatim', () => {
const details = classifyMountAuthError(
'bootstrap stalled for 200 consecutive checkpoint-stable cycles (limit 20, cursor ""): http 403 forbidden: missing required scope: fs:read',
)
expect(details?.missingScope).toBe('fs:read')
})

it('does not misclassify a transient/unrelated failure', () => {
expect(classifyMountAuthError('context deadline exceeded')).toBeUndefined()
expect(classifyMountAuthError('relayfile mount did not become ready within 60000ms')).toBeUndefined()
expect(classifyMountAuthError(undefined)).toBeUndefined()
expect(classifyMountAuthError('')).toBeUndefined()
})

it('does not treat a bare 403 (no stall, no scope) as a scope failure', () => {
// A one-off 403 without the stall signature or a named scope is ambiguous;
// classification must not fire and trigger a terminal abort.
expect(classifyMountAuthError('request returned 403')).toBeUndefined()
})
})

describe('readMountAuthErrorFromState', () => {
it('detects a scope shortfall recorded in state.json lastError.message', async () => {
await withTempStateFile(
{
kind: 'bootstrap_stalled',
code: 'bootstrap_stall_cycle_limit',
message: 'http 403 forbidden: missing required scope: fs:read',
},
(stateFilePath) => {
const details = readMountAuthErrorFromState(stateFilePath)
expect(details?.missingScope).toBe('fs:read')
},
)
})

it('returns undefined when lastError is unrelated', async () => {
await withTempStateFile(
{ kind: 'reconcile_error', message: 'context deadline exceeded' },
(stateFilePath) => {
expect(readMountAuthErrorFromState(stateFilePath)).toBeUndefined()
},
)
})

it('returns undefined when there is no lastError', async () => {
await withTempStateFile(undefined, (stateFilePath) => {
expect(readMountAuthErrorFromState(stateFilePath)).toBeUndefined()
})
})

it('returns undefined for a missing state file', () => {
expect(readMountAuthErrorFromState('/nonexistent/path/state.json')).toBeUndefined()
})
})

describe('mountAuthRemediation', () => {
it('names the missing scope when known', () => {
expect(mountAuthRemediation({ missingScope: 'fs:read', detail: 'x' })).toContain('missing fs:read')
})

it('is still actionable without a scope', () => {
const msg = mountAuthRemediation()
expect(msg).toContain('Re-authenticate')
})
})

describe('MountAuthScopeError', () => {
it('carries the missing scope and is an Error', () => {
const err = new MountAuthScopeError('boom', { missingScope: 'fs:read' })
expect(err).toBeInstanceOf(Error)
expect(err.name).toBe('MountAuthScopeError')
expect(err.missingScope).toBe('fs:read')
})
})
Loading