Skip to content

🤖 fix: de-flake file editing tests with increased timeouts #829

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ammario merged 1 commit into from
Dec 1, 2025
Merged

🤖 fix: de-flake file editing tests with increased timeouts #829

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion tests/ipcMain/helpers.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import type { ToolPolicy } from "../../src/common/utils/tools/toolPolicy";
export const INIT_HOOK_WAIT_MS = 1500; // Wait for async init hook completion (local runtime)
export const SSH_INIT_WAIT_MS = 7000; // SSH init includes sync + checkout + hook, takes longer
export const HAIKU_MODEL = "anthropic:claude-haiku-4-5"; // Fast model for tests
export const GPT_5_MINI_MODEL = "openai:gpt-5-mini"; // Fastest model for performance-critical tests
export const CODEX_MINI_MODEL = "openai:gpt-5.1-codex-mini"; // Fastest model for performance-critical tests
export const TEST_TIMEOUT_LOCAL_MS = 25000; // Recommended timeout for local runtime tests
export const TEST_TIMEOUT_SSH_MS = 60000; // Recommended timeout for SSH runtime tests
export const STREAM_TIMEOUT_LOCAL_MS = 15000; // Stream timeout for local runtime
Expand Down
35 changes: 19 additions & 16 deletions tests/ipcMain/runtimeFileEditing.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,8 @@ import {
sendMessageAndWait,
extractTextFromEvents,
writeFileViaBash,
configureTestRetries,
HAIKU_MODEL,
TEST_TIMEOUT_LOCAL_MS,
TEST_TIMEOUT_SSH_MS,
STREAM_TIMEOUT_LOCAL_MS,
STREAM_TIMEOUT_SSH_MS,
} from "./helpers";
import {
isDockerAvailable,
Expand All @@ -56,6 +53,13 @@ if (shouldRunIntegrationTests()) {
validateApiKeys(["ANTHROPIC_API_KEY"]);
}

// Increased timeouts for file editing tests - these tests require LLM tool calls
// which can be slow depending on API response times
const STREAM_TIMEOUT_MS = 30000; // Stream timeout (was 15s)
const SSH_STREAM_TIMEOUT_MS = 45000; // SSH stream timeout (was 25s)
const LOCAL_TEST_TIMEOUT_MS = 45000; // Test timeout (was 25s)
const SSH_TEST_TIMEOUT_MS = 90000; // SSH test timeout (was 60s)

// SSH server config (shared across all SSH tests)
let sshConfig: SSHServerConfig | undefined;

Expand All @@ -64,6 +68,9 @@ let sshConfig: SSHServerConfig | undefined;
// ============================================================================

describeIntegration("Runtime File Editing Tools", () => {
// Enable retries in CI for flaky API tests
configureTestRetries(3);

beforeAll(async () => {
// Check if Docker is available (required for SSH tests)
if (!(await isDockerAvailable())) {
Expand Down Expand Up @@ -136,8 +143,7 @@ describeIntegration("Runtime File Editing Tools", () => {
await writeFileViaBash(env, workspaceId, testFileName, testContent);

// Ask AI to read the file (explicitly request file_read tool)
const streamTimeout =
type === "ssh" ? STREAM_TIMEOUT_SSH_MS : STREAM_TIMEOUT_LOCAL_MS;
const streamTimeout = type === "ssh" ? SSH_STREAM_TIMEOUT_MS : STREAM_TIMEOUT_MS;
const readEvents = await sendMessageAndWait(
env,
workspaceId,
Expand Down Expand Up @@ -170,7 +176,7 @@ describeIntegration("Runtime File Editing Tools", () => {
await cleanupTempGitRepo(tempGitRepo);
}
},
type === "ssh" ? TEST_TIMEOUT_SSH_MS : TEST_TIMEOUT_LOCAL_MS
type === "ssh" ? SSH_TEST_TIMEOUT_MS : LOCAL_TEST_TIMEOUT_MS
);

test.concurrent(
Expand Down Expand Up @@ -206,8 +212,7 @@ describeIntegration("Runtime File Editing Tools", () => {
await writeFileViaBash(env, workspaceId, testFileName, testContent);

// Ask AI to replace text (explicitly request file_edit_replace_string tool)
const streamTimeout =
type === "ssh" ? STREAM_TIMEOUT_SSH_MS : STREAM_TIMEOUT_LOCAL_MS;
const streamTimeout = type === "ssh" ? SSH_STREAM_TIMEOUT_MS : STREAM_TIMEOUT_MS;
const replaceEvents = await sendMessageAndWait(
env,
workspaceId,
Expand Down Expand Up @@ -246,7 +251,7 @@ describeIntegration("Runtime File Editing Tools", () => {
await cleanupTempGitRepo(tempGitRepo);
}
},
type === "ssh" ? TEST_TIMEOUT_SSH_MS : TEST_TIMEOUT_LOCAL_MS
type === "ssh" ? SSH_TEST_TIMEOUT_MS : LOCAL_TEST_TIMEOUT_MS
);

test.concurrent(
Expand Down Expand Up @@ -282,8 +287,7 @@ describeIntegration("Runtime File Editing Tools", () => {
await writeFileViaBash(env, workspaceId, testFileName, testContent);

// Ask AI to insert text (explicitly request file_edit tool usage)
const streamTimeout =
type === "ssh" ? STREAM_TIMEOUT_SSH_MS : STREAM_TIMEOUT_LOCAL_MS;
const streamTimeout = type === "ssh" ? SSH_STREAM_TIMEOUT_MS : STREAM_TIMEOUT_MS;
const insertEvents = await sendMessageAndWait(
env,
workspaceId,
Expand Down Expand Up @@ -323,7 +327,7 @@ describeIntegration("Runtime File Editing Tools", () => {
await cleanupTempGitRepo(tempGitRepo);
}
},
type === "ssh" ? TEST_TIMEOUT_SSH_MS : TEST_TIMEOUT_LOCAL_MS
type === "ssh" ? SSH_TEST_TIMEOUT_MS : LOCAL_TEST_TIMEOUT_MS
);

test.concurrent(
Expand Down Expand Up @@ -359,8 +363,7 @@ describeIntegration("Runtime File Editing Tools", () => {
await writeFileViaBash(env, workspaceId, relativeTestFile, testContent);

// Now edit the file using a relative path
const streamTimeout =
type === "ssh" ? STREAM_TIMEOUT_SSH_MS : STREAM_TIMEOUT_LOCAL_MS;
const streamTimeout = type === "ssh" ? SSH_STREAM_TIMEOUT_MS : STREAM_TIMEOUT_MS;
const editEvents = await sendMessageAndWait(
env,
workspaceId,
Expand Down Expand Up @@ -408,7 +411,7 @@ describeIntegration("Runtime File Editing Tools", () => {
await cleanupTempGitRepo(tempGitRepo);
}
},
type === "ssh" ? TEST_TIMEOUT_SSH_MS : TEST_TIMEOUT_LOCAL_MS
type === "ssh" ? SSH_TEST_TIMEOUT_MS : LOCAL_TEST_TIMEOUT_MS
);
}
);
Expand Down