Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions src/cloud/api/workspaces.ts
Original file line number Diff line number Diff line change
Expand Up @@ -301,8 +301,9 @@ workspacesRouter.get('/:id/status', async (req: Request, res: Response) => {
const provisioner = getProvisioner();
const status = await provisioner.getStatus(id);

// Include provisioning progress info if status is 'provisioning'
const provisioningProgress = status === 'provisioning' ? getProvisioningStage(id) : null;
// Include provisioning progress info if it exists (even after status changes to 'running')
// This allows the frontend to see all stages including 'complete'
const provisioningProgress = getProvisioningStage(id);

res.json({
status,
Expand Down
49 changes: 39 additions & 10 deletions src/cloud/provisioner/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,17 @@ function clearProvisioningProgress(workspaceId: string): void {
provisioningProgress.delete(workspaceId);
}

/**
* Schedule cleanup of provisioning progress after a delay
* This gives the frontend time to poll and see the 'complete' stage
*/
function scheduleProgressCleanup(workspaceId: string, delayMs: number = 30_000): void {
setTimeout(() => {
clearProvisioningProgress(workspaceId);
console.log(`[provisioner] Cleaned up provisioning progress for ${workspaceId.substring(0, 8)}`);
}, delayMs);
}

/**
* Get a fresh GitHub App installation token from Nango.
* Looks up the user's connected repositories to find a valid Nango connection.
Expand Down Expand Up @@ -602,6 +613,9 @@ class FlyProvisioner implements ComputeProvisioner {
// Stage: Complete
updateProvisioningStage(workspace.id, 'complete');

// Schedule cleanup of provisioning progress after 30s (gives frontend time to see 'complete')
scheduleProgressCleanup(workspace.id);

return {
computeId: machine.id,
publicUrl,
Expand Down Expand Up @@ -1272,6 +1286,7 @@ export class WorkspaceProvisioner {

/**
* Provision a new workspace (one-click)
* Returns immediately with 'provisioning' status and runs actual provisioning in background
*/
async provision(config: ProvisionConfig): Promise<ProvisionResult> {
// Create workspace record
Expand All @@ -1297,6 +1312,25 @@ export class WorkspaceProvisioner {
// Auto-accept the creator's membership
await db.workspaceMembers.acceptInvite(workspace.id, config.userId);

// Initialize stage tracking immediately
updateProvisioningStage(workspace.id, 'creating');

// Run provisioning in the background
this.runProvisioningAsync(workspace, config).catch((error) => {
console.error(`[provisioner] Background provisioning failed for ${workspace.id}:`, error);
});

// Return immediately with 'provisioning' status
return {
workspaceId: workspace.id,
status: 'provisioning',
};
}

/**
* Run the actual provisioning work asynchronously
*/
private async runProvisioningAsync(workspace: Workspace, config: ProvisionConfig): Promise<void> {
// Get credentials
const credentials = new Map<string, string>();
for (const provider of config.providers) {
Expand Down Expand Up @@ -1336,23 +1370,18 @@ export class WorkspaceProvisioner {
publicUrl,
});

return {
workspaceId: workspace.id,
status: 'running',
publicUrl,
};
console.log(`[provisioner] Workspace ${workspace.id} provisioned successfully at ${publicUrl}`);
} catch (error) {
const errorMessage = error instanceof Error ? error.message : 'Unknown error';

await db.workspaces.updateStatus(workspace.id, 'error', {
errorMessage,
});

return {
workspaceId: workspace.id,
status: 'error',
error: errorMessage,
};
// Clear provisioning progress on error
clearProvisioningProgress(workspace.id);

console.error(`[provisioner] Workspace ${workspace.id} provisioning failed:`, errorMessage);
}
}

Expand Down
216 changes: 118 additions & 98 deletions src/daemon/connection.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -114,117 +114,137 @@ describe('Connection', () => {

describe('heartbeat timeout configuration', () => {
it('uses configurable heartbeatTimeoutMultiplier', async () => {
const socket = new MockSocket();
// 10ms heartbeat * 2 multiplier = 20ms timeout
const connection = new Connection(socket as unknown as Socket, {
heartbeatMs: 10,
heartbeatTimeoutMultiplier: 2,
});
const onError = vi.fn();
connection.onError = onError;

socket.emit('data', encodeFrame(makeHello('agent-a')));
expect(connection.state).toBe('ACTIVE');

// Wait less than timeout (20ms) - should still be alive
await new Promise((r) => setTimeout(r, 15));
expect(onError).not.toHaveBeenCalled();
expect(connection.state).toBe('ACTIVE');

// Wait past timeout - should be dead
await new Promise((r) => setTimeout(r, 30));
expect(onError).toHaveBeenCalledTimes(1);
expect(socket.destroyed).toBe(true);
vi.useFakeTimers();
try {
const socket = new MockSocket();
// 10ms heartbeat * 2 multiplier = 20ms timeout
const connection = new Connection(socket as unknown as Socket, {
heartbeatMs: 10,
heartbeatTimeoutMultiplier: 2,
});
const onError = vi.fn();
connection.onError = onError;

socket.emit('data', encodeFrame(makeHello('agent-a')));
expect(connection.state).toBe('ACTIVE');

// Wait less than timeout (20ms) - should still be alive
await vi.advanceTimersByTimeAsync(15);
expect(onError).not.toHaveBeenCalled();
expect(connection.state).toBe('ACTIVE');

// Wait past timeout - should be dead
await vi.advanceTimersByTimeAsync(30);
expect(onError).toHaveBeenCalledTimes(1);
expect(socket.destroyed).toBe(true);
} finally {
vi.useRealTimers();
}
});

it('survives with slow but timely pong responses', async () => {
const socket = new MockSocket();
// 20ms heartbeat * 3 multiplier = 60ms timeout
const connection = new Connection(socket as unknown as Socket, {
heartbeatMs: 20,
heartbeatTimeoutMultiplier: 3,
});
const onError = vi.fn();
const onPong = vi.fn();
connection.onError = onError;
connection.onPong = onPong;

socket.emit('data', encodeFrame(makeHello('agent-a')));
expect(connection.state).toBe('ACTIVE');

// Simulate slow but valid pong responses every 40ms (within 60ms timeout)
for (let i = 0; i < 3; i++) {
await new Promise((r) => setTimeout(r, 40));
// Send PONG before timeout expires
vi.useFakeTimers();
try {
const socket = new MockSocket();
// 20ms heartbeat * 3 multiplier = 60ms timeout
const connection = new Connection(socket as unknown as Socket, {
heartbeatMs: 20,
heartbeatTimeoutMultiplier: 3,
});
const onError = vi.fn();
const onPong = vi.fn();
connection.onError = onError;
connection.onPong = onPong;

socket.emit('data', encodeFrame(makeHello('agent-a')));
expect(connection.state).toBe('ACTIVE');

// Simulate slow but valid pong responses every 40ms (within 60ms timeout)
for (let i = 0; i < 3; i++) {
await vi.advanceTimersByTimeAsync(40);
// Send PONG before timeout expires
socket.emit('data', encodeFrame({
v: PROTOCOL_VERSION,
type: 'PONG',
id: `pong-${i}`,
ts: Date.now(),
payload: { nonce: 'test' },
}));
}

// Connection should still be alive after multiple slow pongs
expect(onError).not.toHaveBeenCalled();
expect(connection.state).toBe('ACTIVE');
expect(onPong).toHaveBeenCalledTimes(3);
} finally {
vi.useRealTimers();
}
});

it('dies when pong arrives too late', async () => {
vi.useFakeTimers();
try {
const socket = new MockSocket();
// 10ms heartbeat * 2 multiplier = 20ms timeout
const connection = new Connection(socket as unknown as Socket, {
heartbeatMs: 10,
heartbeatTimeoutMultiplier: 2,
});
const onError = vi.fn();
connection.onError = onError;

socket.emit('data', encodeFrame(makeHello('agent-a')));
expect(connection.state).toBe('ACTIVE');

// Wait past timeout before sending pong
await vi.advanceTimersByTimeAsync(50);

// Connection should already be dead
expect(onError).toHaveBeenCalledTimes(1);
expect(socket.destroyed).toBe(true);

// Late pong should have no effect (connection already dead)
socket.emit('data', encodeFrame({
v: PROTOCOL_VERSION,
type: 'PONG',
id: `pong-${i}`,
id: 'late-pong',
ts: Date.now(),
payload: { nonce: 'test' },
}));
}

// Connection should still be alive after multiple slow pongs
expect(onError).not.toHaveBeenCalled();
expect(connection.state).toBe('ACTIVE');
expect(onPong).toHaveBeenCalledTimes(3);
});

it('dies when pong arrives too late', async () => {
const socket = new MockSocket();
// 10ms heartbeat * 2 multiplier = 20ms timeout
const connection = new Connection(socket as unknown as Socket, {
heartbeatMs: 10,
heartbeatTimeoutMultiplier: 2,
});
const onError = vi.fn();
connection.onError = onError;

socket.emit('data', encodeFrame(makeHello('agent-a')));
expect(connection.state).toBe('ACTIVE');

// Wait past timeout before sending pong
await new Promise((r) => setTimeout(r, 50));

// Connection should already be dead
expect(onError).toHaveBeenCalledTimes(1);
expect(socket.destroyed).toBe(true);

// Late pong should have no effect (connection already dead)
socket.emit('data', encodeFrame({
v: PROTOCOL_VERSION,
type: 'PONG',
id: 'late-pong',
ts: Date.now(),
payload: { nonce: 'test' },
}));

// Error count should not increase
expect(onError).toHaveBeenCalledTimes(1);
// Error count should not increase
expect(onError).toHaveBeenCalledTimes(1);
} finally {
vi.useRealTimers();
}
});

it('uses default multiplier of 6 when not specified', async () => {
const socket = new MockSocket();
// 10ms heartbeat * 6 (default) = 60ms timeout
const connection = new Connection(socket as unknown as Socket, {
heartbeatMs: 10,
// heartbeatTimeoutMultiplier not specified - should default to 6
});
const onError = vi.fn();
connection.onError = onError;

socket.emit('data', encodeFrame(makeHello('agent-a')));
expect(connection.state).toBe('ACTIVE');

// Wait 40ms - should still be alive (timeout is 60ms)
await new Promise((r) => setTimeout(r, 40));
expect(onError).not.toHaveBeenCalled();
expect(connection.state).toBe('ACTIVE');

// Wait total of 80ms - should be dead
await new Promise((r) => setTimeout(r, 50));
expect(onError).toHaveBeenCalledTimes(1);
vi.useFakeTimers();
try {
const socket = new MockSocket();
// 10ms heartbeat * 6 (default) = 60ms timeout
const connection = new Connection(socket as unknown as Socket, {
heartbeatMs: 10,
// heartbeatTimeoutMultiplier not specified - should default to 6
});
const onError = vi.fn();
connection.onError = onError;

socket.emit('data', encodeFrame(makeHello('agent-a')));
expect(connection.state).toBe('ACTIVE');

// Wait 40ms - should still be alive (timeout is 60ms)
await vi.advanceTimersByTimeAsync(40);
expect(onError).not.toHaveBeenCalled();
expect(connection.state).toBe('ACTIVE');

// Wait total of 80ms - should be dead
await vi.advanceTimersByTimeAsync(50);
expect(onError).toHaveBeenCalledTimes(1);
} finally {
vi.useRealTimers();
}
});
});
});
Loading