diff --git a/src/services/streamManager.ts b/src/services/streamManager.ts index 56668342d..1fe97dc93 100644 --- a/src/services/streamManager.ts +++ b/src/services/streamManager.ts @@ -899,41 +899,11 @@ export class StreamManager extends EventEmitter { let errorType = this.categorizeError(actualError); - // Detect and enhance model-not-found errors - if (APICallError.isInstance(actualError)) { - const apiError = actualError; - - // Type guard for error data structure - const hasErrorProperty = ( - data: unknown - ): data is { error: { code?: string; type?: string } } => { - return ( - typeof data === "object" && - data !== null && - "error" in data && - typeof data.error === "object" && - data.error !== null - ); - }; - - // OpenAI: 400 with error.code === 'model_not_found' - const isOpenAIModelError = - apiError.statusCode === 400 && - hasErrorProperty(apiError.data) && - apiError.data.error.code === "model_not_found"; - - // Anthropic: 404 with error.type === 'not_found_error' - const isAnthropicModelError = - apiError.statusCode === 404 && - hasErrorProperty(apiError.data) && - apiError.data.error.type === "not_found_error"; - - if (isOpenAIModelError || isAnthropicModelError) { - errorType = "model_not_found"; - // Extract model name from model string (e.g., "anthropic:sonnet-1m" -> "sonnet-1m") - const [, modelName] = streamInfo.model.split(":"); - errorMessage = `Model '${modelName || streamInfo.model}' does not exist or is not available. Please check your model selection.`; - } + // Enhance model-not-found error messages + if (errorType === "model_not_found") { + // Extract model name from model string (e.g., "anthropic:sonnet-1m" -> "sonnet-1m") + const [, modelName] = streamInfo.model.split(":"); + errorMessage = `Model '${modelName || streamInfo.model}' does not exist or is not available. Please check your model selection.`; } // If we detect API key issues in the error message, override the type @@ -1044,6 +1014,36 @@ export class StreamManager extends EventEmitter { if (error.statusCode === 429) return "rate_limit"; if (error.statusCode && error.statusCode >= 500) return "server_error"; + // Check for model_not_found errors (OpenAI and Anthropic) + // Type guard for error data structure + const hasErrorProperty = ( + data: unknown + ): data is { error: { code?: string; type?: string } } => { + return ( + typeof data === "object" && + data !== null && + "error" in data && + typeof data.error === "object" && + data.error !== null + ); + }; + + // OpenAI: 400 with error.code === 'model_not_found' + const isOpenAIModelError = + error.statusCode === 400 && + hasErrorProperty(error.data) && + error.data.error.code === "model_not_found"; + + // Anthropic: 404 with error.type === 'not_found_error' + const isAnthropicModelError = + error.statusCode === 404 && + hasErrorProperty(error.data) && + error.data.error.type === "not_found_error"; + + if (isOpenAIModelError || isAnthropicModelError) { + return "model_not_found"; + } + // Check for Anthropic context exceeded errors if (error.message.includes("prompt is too long:")) { return "context_exceeded"; diff --git a/tests/ipcMain/anthropic1MContext.test.ts b/tests/ipcMain/anthropic1MContext.test.ts index 34c60b27b..f3c0d6fcd 100644 --- a/tests/ipcMain/anthropic1MContext.test.ts +++ b/tests/ipcMain/anthropic1MContext.test.ts @@ -20,13 +20,6 @@ describeIntegration("IpcMain anthropic 1M context integration tests", () => { jest.retryTimes(3, { logErrorsBeforeRetry: true }); } - // Load tokenizer modules once before all tests (takes ~14s) - // This ensures accurate token counts for API calls without timing out individual tests - beforeAll(async () => { - const { loadTokenizerModules } = await import("../../src/utils/main/tokenizer"); - await loadTokenizerModules(); - }, 30000); // 30s timeout for tokenizer loading - test.concurrent( "should handle larger context with 1M flag enabled vs standard limits", async () => { diff --git a/tests/ipcMain/forkWorkspace.test.ts b/tests/ipcMain/forkWorkspace.test.ts index efe2280bf..c818ba482 100644 --- a/tests/ipcMain/forkWorkspace.test.ts +++ b/tests/ipcMain/forkWorkspace.test.ts @@ -32,13 +32,6 @@ describeIntegration("IpcMain fork workspace integration tests", () => { jest.retryTimes(3, { logErrorsBeforeRetry: true }); } - // Load tokenizer modules once before all tests (takes ~14s) - // This ensures accurate token counts for API calls without timing out individual tests - beforeAll(async () => { - const { loadTokenizerModules } = await import("../../src/utils/main/tokenizer"); - await loadTokenizerModules(); - }, 30000); // 30s timeout for tokenizer loading - test.concurrent( "should fail to fork workspace with invalid name", async () => { diff --git a/tests/ipcMain/modelNotFound.test.ts b/tests/ipcMain/modelNotFound.test.ts new file mode 100644 index 000000000..131a9c02a --- /dev/null +++ b/tests/ipcMain/modelNotFound.test.ts @@ -0,0 +1,98 @@ +import { setupWorkspace, shouldRunIntegrationTests, validateApiKeys } from "./setup"; +import { sendMessageWithModel, createEventCollector, waitFor } from "./helpers"; +import { IPC_CHANNELS } from "../../src/constants/ipc-constants"; +import type { Result } from "../../src/types/result"; +import type { SendMessageError } from "../../src/types/errors"; +import type { StreamErrorMessage } from "../../src/types/ipc"; + +// Skip all tests if TEST_INTEGRATION is not set +const describeIntegration = shouldRunIntegrationTests() ? describe : describe.skip; + +// Validate API keys before running tests +if (shouldRunIntegrationTests()) { + validateApiKeys(["ANTHROPIC_API_KEY", "OPENAI_API_KEY"]); +} + +describeIntegration("IpcMain model_not_found error handling", () => { + // Enable retries in CI for flaky API tests + if (process.env.CI && typeof jest !== "undefined" && jest.retryTimes) { + jest.retryTimes(3, { logErrorsBeforeRetry: true }); + } + + test.concurrent( + "should classify Anthropic 404 as model_not_found (not retryable)", + async () => { + const { env, workspaceId, cleanup } = await setupWorkspace("anthropic"); + try { + // Send a message with a non-existent model + // Anthropic returns 404 with error.type === 'not_found_error' + void sendMessageWithModel( + env.mockIpcRenderer, + workspaceId, + "Hello", + "anthropic", + "invalid-model-that-does-not-exist-xyz123" + ); + + // Collect events to verify error classification + const collector = createEventCollector(env.sentEvents, workspaceId); + await waitFor(() => { + collector.collect(); + return collector.getEvents().some((e) => "type" in e && e.type === "stream-error"); + }, 10000); + + const events = collector.getEvents(); + const errorEvent = events.find((e) => "type" in e && e.type === "stream-error") as + | StreamErrorMessage + | undefined; + + expect(errorEvent).toBeDefined(); + + // Bug: Error should be classified as 'model_not_found', not 'api' or 'unknown' + // This ensures it's marked as non-retryable in retryEligibility.ts + expect(errorEvent?.errorType).toBe("model_not_found"); + } finally { + await cleanup(); + } + }, + 30000 // 30s timeout + ); + + test.concurrent( + "should classify OpenAI 400 model_not_found as model_not_found (not retryable)", + async () => { + const { env, workspaceId, cleanup } = await setupWorkspace("openai"); + try { + // Send a message with a non-existent model + // OpenAI returns 400 with error.code === 'model_not_found' + void sendMessageWithModel( + env.mockIpcRenderer, + workspaceId, + "Hello", + "openai", + "gpt-nonexistent-model-xyz123" + ); + + // Collect events to verify error classification + const collector = createEventCollector(env.sentEvents, workspaceId); + await waitFor(() => { + collector.collect(); + return collector.getEvents().some((e) => "type" in e && e.type === "stream-error"); + }, 10000); + + const events = collector.getEvents(); + const errorEvent = events.find((e) => "type" in e && e.type === "stream-error") as + | StreamErrorMessage + | undefined; + + expect(errorEvent).toBeDefined(); + + // Bug: Error should be classified as 'model_not_found', not 'api' or 'unknown' + expect(errorEvent?.errorType).toBe("model_not_found"); + } finally { + await cleanup(); + } + }, + 30000 // 30s timeout + ); +}); diff --git a/tests/ipcMain/openai-web-search.test.ts b/tests/ipcMain/openai-web-search.test.ts index 441692766..5fd5e4174 100644 --- a/tests/ipcMain/openai-web-search.test.ts +++ b/tests/ipcMain/openai-web-search.test.ts @@ -15,13 +15,6 @@ describeIntegration("OpenAI web_search integration tests", () => { jest.retryTimes(3, { logErrorsBeforeRetry: true }); } - // Load tokenizer modules once before all tests (takes ~14s) - // This ensures accurate token counts for API calls without timing out individual tests - beforeAll(async () => { - const { loadTokenizerModules } = await import("../../src/utils/main/tokenizer"); - await loadTokenizerModules(); - }, 30000); // 30s timeout for tokenizer loading - test.concurrent( "should handle reasoning + web_search without itemId errors", async () => { diff --git a/tests/ipcMain/resumeStream.test.ts b/tests/ipcMain/resumeStream.test.ts index 9e03af9a4..dc0b79e18 100644 --- a/tests/ipcMain/resumeStream.test.ts +++ b/tests/ipcMain/resumeStream.test.ts @@ -20,13 +20,6 @@ describeIntegration("IpcMain resumeStream integration tests", () => { jest.retryTimes(3, { logErrorsBeforeRetry: true }); } - // Load tokenizer modules once before all tests (takes ~14s) - // This ensures accurate token counts for API calls without timing out individual tests - beforeAll(async () => { - const { loadTokenizerModules } = await import("../../src/utils/main/tokenizer"); - await loadTokenizerModules(); - }, 30000); // 30s timeout for tokenizer loading - test.concurrent( "should resume interrupted stream without new user message", async () => { diff --git a/tests/ipcMain/sendMessage.test.ts b/tests/ipcMain/sendMessage.test.ts index 2363c4bc9..544ec8cda 100644 --- a/tests/ipcMain/sendMessage.test.ts +++ b/tests/ipcMain/sendMessage.test.ts @@ -48,12 +48,6 @@ describeIntegration("IpcMain sendMessage integration tests", () => { jest.retryTimes(3, { logErrorsBeforeRetry: true }); } - // Load tokenizer modules once before all tests (takes ~14s) - // This ensures accurate token counts for API calls without timing out individual tests - beforeAll(async () => { - const { loadTokenizerModules } = await import("../../src/utils/main/tokenizer"); - await loadTokenizerModules(); - }, 30000); // 30s timeout for tokenizer loading // Run tests for each provider concurrently describe.each(PROVIDER_CONFIGS)("%s:%s provider tests", (provider, model) => { test.concurrent( diff --git a/tests/ipcMain/streamErrorRecovery.test.ts b/tests/ipcMain/streamErrorRecovery.test.ts index 5b4e8e3ce..658704ff5 100644 --- a/tests/ipcMain/streamErrorRecovery.test.ts +++ b/tests/ipcMain/streamErrorRecovery.test.ts @@ -220,13 +220,6 @@ describeIntegration("Stream Error Recovery (No Amnesia)", () => { jest.retryTimes(3, { logErrorsBeforeRetry: true }); } - // Load tokenizer modules once before all tests (takes ~14s) - // This ensures accurate token counts for API calls without timing out individual tests - beforeAll(async () => { - const { loadTokenizerModules } = await import("../../src/utils/main/tokenizer"); - await loadTokenizerModules(); - }, 30000); // 30s timeout for tokenizer loading - test.concurrent( "should preserve exact prefix and continue from exact point after stream error", async () => { diff --git a/tests/ipcMain/truncate.test.ts b/tests/ipcMain/truncate.test.ts index d60a837e1..312631c95 100644 --- a/tests/ipcMain/truncate.test.ts +++ b/tests/ipcMain/truncate.test.ts @@ -24,13 +24,6 @@ describeIntegration("IpcMain truncate integration tests", () => { jest.retryTimes(3, { logErrorsBeforeRetry: true }); } - // Load tokenizer modules once before all tests (takes ~14s) - // This ensures accurate token counts for API calls without timing out individual tests - beforeAll(async () => { - const { loadTokenizerModules } = await import("../../src/utils/main/tokenizer"); - await loadTokenizerModules(); - }, 30000); // 30s timeout for tokenizer loading - test.concurrent( "should truncate 50% of chat history and verify context is updated", async () => {