From b37b1d377cb18b6ae69ef4323eeabd319a05e005 Mon Sep 17 00:00:00 2001 From: Ammar Date: Thu, 6 Nov 2025 16:53:16 +0000 Subject: [PATCH 1/2] =?UTF-8?q?=F0=9F=A4=96=20fix:=20increase=20stream=20t?= =?UTF-8?q?imeouts=20in=20CI=20to=20handle=20rate=20limits?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Integration tests were timing out in CI due to: - API rate limiting causing slower response times - CI environment overhead - Tests using 15s/25s timeouts insufficient for file operations Changes: - Add CI detection to increase timeouts by 2.5x in CI environments - STREAM_TIMEOUT_LOCAL_MS: 15s → 37.5s in CI - STREAM_TIMEOUT_SSH_MS: 25s → 62.5s in CI - waitForStreamSuccess default: 30s → 75s in CI Fixes flakes in: - tests/ipcMain/runtimeFileEditing.test.ts - tests/ipcMain/sendMessage.test.ts (anthropic provider tests) - tests/ipcMain/sendMessage.test.ts (image support test) --- tests/ipcMain/helpers.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/ipcMain/helpers.ts b/tests/ipcMain/helpers.ts index c1d3e69b0..e10a1bc38 100644 --- a/tests/ipcMain/helpers.ts +++ b/tests/ipcMain/helpers.ts @@ -22,8 +22,13 @@ export const SSH_INIT_WAIT_MS = 7000; // SSH init includes sync + checkout + hoo export const HAIKU_MODEL = "anthropic:claude-haiku-4-5"; // Fast model for tests export const TEST_TIMEOUT_LOCAL_MS = 25000; // Recommended timeout for local runtime tests export const TEST_TIMEOUT_SSH_MS = 60000; // Recommended timeout for SSH runtime tests -export const STREAM_TIMEOUT_LOCAL_MS = 15000; // Stream timeout for local runtime -export const STREAM_TIMEOUT_SSH_MS = 25000; // Stream timeout for SSH runtime +// Stream timeouts: CI environments need longer timeouts due to API rate limits and slower execution +// Local development typically completes faster +const CI_TIMEOUT_MULTIPLIER = 2.5; +const isCI = process.env.CI === "true" || process.env.CI === "1"; + +export const STREAM_TIMEOUT_LOCAL_MS = isCI ? 37500 : 15000; // 37.5s in CI, 15s locally +export const STREAM_TIMEOUT_SSH_MS = isCI ? 62500 : 25000; // 62.5s in CI, 25s locally /** * Generate a unique branch name @@ -613,7 +618,7 @@ export async function waitForInitEnd( export async function waitForStreamSuccess( sentEvents: Array<{ channel: string; data: unknown }>, workspaceId: string, - timeoutMs = 30000 + timeoutMs = isCI ? 75000 : 30000 // Use longer timeout in CI (75s vs 30s) ): Promise { const collector = createEventCollector(sentEvents, workspaceId); await collector.waitForEvent("stream-end", timeoutMs); From 6308357d6d5d7ebee99df3775d7d4a59c4e0d2db Mon Sep 17 00:00:00 2001 From: Ammar Date: Thu, 6 Nov 2025 16:58:41 +0000 Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=A4=96=20fix:=20apply=20CI=20timeout?= =?UTF-8?q?=20multiplier=20to=20all=20test=20call=20sites?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit added CI-aware timeout constants but tests were still passing hardcoded values (10s, 30s) directly to waitForStreamSuccess, bypassing the CI multiplier. Changes: - Export STREAM_TIMEOUT_SHORT_MS (10s → 25s in CI) - Export STREAM_TIMEOUT_MEDIUM_MS (30s → 75s in CI) - Replace all hardcoded timeouts in sendMessage.test.ts with constants This ensures all flaky tests get the CI timeout multiplier: - Anthropic provider tests (was 10s, now 25s in CI) - OpenAI provider tests (was 30s, now 75s in CI) - Image support tests (was 30s, now 75s in CI) --- tests/ipcMain/helpers.ts | 6 +++++- tests/ipcMain/sendMessage.test.ts | 24 +++++++++++++++++++----- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/tests/ipcMain/helpers.ts b/tests/ipcMain/helpers.ts index e10a1bc38..c21f0a3d5 100644 --- a/tests/ipcMain/helpers.ts +++ b/tests/ipcMain/helpers.ts @@ -30,6 +30,10 @@ const isCI = process.env.CI === "true" || process.env.CI === "1"; export const STREAM_TIMEOUT_LOCAL_MS = isCI ? 37500 : 15000; // 37.5s in CI, 15s locally export const STREAM_TIMEOUT_SSH_MS = isCI ? 62500 : 25000; // 62.5s in CI, 25s locally +// Additional stream timeout constants for tests (also CI-aware) +export const STREAM_TIMEOUT_SHORT_MS = isCI ? 25000 : 10000; // For quick operations +export const STREAM_TIMEOUT_MEDIUM_MS = isCI ? 75000 : 30000; // For standard operations + /** * Generate a unique branch name * Uses high-resolution time (nanosecond precision) to prevent collisions @@ -618,7 +622,7 @@ export async function waitForInitEnd( export async function waitForStreamSuccess( sentEvents: Array<{ channel: string; data: unknown }>, workspaceId: string, - timeoutMs = isCI ? 75000 : 30000 // Use longer timeout in CI (75s vs 30s) + timeoutMs = STREAM_TIMEOUT_MEDIUM_MS // Use CI-aware timeout (75s in CI, 30s locally) ): Promise { const collector = createEventCollector(sentEvents, workspaceId); await collector.waitForEvent("stream-end", timeoutMs); diff --git a/tests/ipcMain/sendMessage.test.ts b/tests/ipcMain/sendMessage.test.ts index 2363c4bc9..cbdddc859 100644 --- a/tests/ipcMain/sendMessage.test.ts +++ b/tests/ipcMain/sendMessage.test.ts @@ -18,6 +18,8 @@ import { readChatHistory, TEST_IMAGES, modelString, + STREAM_TIMEOUT_SHORT_MS, + STREAM_TIMEOUT_MEDIUM_MS, } from "./helpers"; import type { StreamDeltaEvent } from "../../src/types/stream"; import { IPC_CHANNELS } from "../../src/constants/ipc-constants"; @@ -611,7 +613,7 @@ describeIntegration("IpcMain sendMessage integration tests", () => { const collector = await waitForStreamSuccess( env.sentEvents, workspaceId, - provider === "openai" ? 30000 : 10000 + provider === "openai" ? STREAM_TIMEOUT_MEDIUM_MS : STREAM_TIMEOUT_SHORT_MS ); // Get the final assistant message @@ -858,7 +860,11 @@ These are general instructions that apply to all modes. ); // Collect response - const collector = await waitForStreamSuccess(env.sentEvents, workspaceId, 10000); + const collector = await waitForStreamSuccess( + env.sentEvents, + workspaceId, + STREAM_TIMEOUT_SHORT_MS + ); results[provider] = { success: result.success, @@ -1257,7 +1263,11 @@ These are general instructions that apply to all modes. expect(result.success).toBe(true); // Wait for stream to complete - const collector = await waitForStreamSuccess(env.sentEvents, workspaceId, 10000); + const collector = await waitForStreamSuccess( + env.sentEvents, + workspaceId, + STREAM_TIMEOUT_SHORT_MS + ); // Get the final assistant message const finalMessage = collector.getFinalMessage(); @@ -1531,7 +1541,11 @@ describe.each(PROVIDER_CONFIGS)("%s:%s image support", (provider, model) => { expect(result.success).toBe(true); // Wait for stream to complete - const collector = await waitForStreamSuccess(env.sentEvents, workspaceId, 30000); + const collector = await waitForStreamSuccess( + env.sentEvents, + workspaceId, + STREAM_TIMEOUT_MEDIUM_MS + ); // Verify we got a response about the image const deltas = collector.getDeltas(); @@ -1568,7 +1582,7 @@ describe.each(PROVIDER_CONFIGS)("%s:%s image support", (provider, model) => { expect(result.success).toBe(true); // Wait for stream to complete - await waitForStreamSuccess(env.sentEvents, workspaceId, 30000); + await waitForStreamSuccess(env.sentEvents, workspaceId, STREAM_TIMEOUT_MEDIUM_MS); // Read history from disk const messages = await readChatHistory(env.tempDir, workspaceId);