diff --git a/packages/opencode/src/provider/error.ts b/packages/opencode/src/provider/error.ts index 0d5a5130c460..5c458d515efd 100644 --- a/packages/opencode/src/provider/error.ts +++ b/packages/opencode/src/provider/error.ts @@ -131,15 +131,94 @@ export type ParsedStreamError = responseBody: string } -export function parseStreamError(input: unknown): ParsedStreamError | undefined { +type StreamErrorRecord = Record + +function streamEnvelopeMessage(body: StreamErrorRecord): string { + const nested = body.error + if (!nested || typeof nested !== "object") return "Provider is overloaded" + const record = nested as StreamErrorRecord + if (typeof record.message === "string" && record.message.trim() !== "") return record.message + if (typeof record.code === "string" && record.code.trim() !== "") return record.code + if (typeof record.type === "string" && record.type.trim() !== "") return record.type + return "Provider is overloaded" +} + +function streamEnvelopeFields(body: StreamErrorRecord) { + const nested = body.error + if (!nested || typeof nested !== "object") { + return { code: "", type: "" } + } + const record = nested as StreamErrorRecord + return { + code: typeof record.code === "string" ? record.code : "", + type: typeof record.type === "string" ? record.type : "", + } +} + +function extractStreamEnvelope(value: string): StreamErrorRecord | undefined { + const direct = json(value) + if (direct?.type === "error") return direct as StreamErrorRecord + + const marker = value.indexOf('"type":"error"') + const markerSpaced = value.indexOf('"type": "error"') + const start = marker === -1 ? markerSpaced : marker + if (start === -1) return undefined + + const open = value.lastIndexOf("{", start) + if (open === -1) return undefined + + const slice = value.slice(open) + const parsed = json(slice) + if (parsed?.type === "error") return parsed as StreamErrorRecord + return undefined +} + +function normalizeStreamEnvelope(input: unknown): StreamErrorRecord | undefined { + if (typeof input === "string") { + return (json(input) as StreamErrorRecord | undefined) ?? extractStreamEnvelope(input) + } + const raw = json(input) - const body = typeof raw?.message === "string" ? (json(raw.message) ?? raw) : raw + if (!raw) return undefined + if (raw.type === "error") return raw as StreamErrorRecord + if (typeof raw.message === "string") { + return (json(raw.message) as StreamErrorRecord | undefined) ?? extractStreamEnvelope(raw.message) + } + return undefined +} + +function isTransientOpenAIStreamEnvelope(body: StreamErrorRecord): boolean { + const { code, type } = streamEnvelopeFields(body) + if ( + code === "server_error" || + code === "server_is_overloaded" || + code === "stream_read_error" || + code === "rate_limit_error" + ) { + return true + } + if ( + type === "server_error" || + type === "upstream_error" || + type === "service_unavailable_error" || + type === "rate_limit_error" + ) { + return true + } + if (code.includes("rate_limit") || code.includes("overloaded") || code.includes("unavailable")) { + return true + } + return false +} + +export function parseStreamError(input: unknown): ParsedStreamError | undefined { + const body = normalizeStreamEnvelope(input) if (!body) return const responseBody = JSON.stringify(body) if (body.type !== "error") return - switch (body?.error?.code) { + switch (streamEnvelopeFields(body).code) { case "context_length_exceeded": return { type: "context_overflow", @@ -163,19 +242,30 @@ export function parseStreamError(input: unknown): ParsedStreamError | undefined case "invalid_prompt": return { type: "api_error", - message: typeof body?.error?.message === "string" ? body?.error?.message : "Invalid prompt.", + message: streamEnvelopeMessage(body) === "Provider is overloaded" ? "Invalid prompt." : streamEnvelopeMessage(body), isRetryable: false, responseBody, } case "server_is_overloaded": case "server_error": + case "stream_read_error": + case "rate_limit_error": return { type: "api_error", - message: typeof body?.error?.message === "string" ? body?.error?.message : "Server error.", + message: streamEnvelopeMessage(body), isRetryable: true, responseBody, } } + + if (!isTransientOpenAIStreamEnvelope(body)) return undefined + + return { + type: "api_error", + message: streamEnvelopeMessage(body), + isRetryable: true, + responseBody, + } } export type ParsedAPICallError = diff --git a/packages/opencode/src/session/message-v2.ts b/packages/opencode/src/session/message-v2.ts index 2884c4cf860b..ac9a8c859bc2 100644 --- a/packages/opencode/src/session/message-v2.ts +++ b/packages/opencode/src/session/message-v2.ts @@ -710,8 +710,29 @@ export function fromError( }, { cause: e }, ).toObject() - case e instanceof Error: + case e instanceof Error: { + const parsed = ProviderError.parseStreamError(e.message) + if (parsed) { + if (parsed.type === "context_overflow") { + return new ContextOverflowError( + { + message: parsed.message, + responseBody: parsed.responseBody, + }, + { cause: e }, + ).toObject() + } + return new APIError( + { + message: parsed.message, + isRetryable: parsed.isRetryable, + responseBody: parsed.responseBody, + }, + { cause: e }, + ).toObject() + } return new NamedError.Unknown({ message: errorMessage(e) }, { cause: e }).toObject() + } default: try { const parsed = ProviderError.parseStreamError(e) diff --git a/packages/opencode/src/session/retry.ts b/packages/opencode/src/session/retry.ts index bcfb54c47551..707bc8904ac2 100644 --- a/packages/opencode/src/session/retry.ts +++ b/packages/opencode/src/session/retry.ts @@ -137,17 +137,36 @@ export function retryable(error: Err, provider: string) { const json = parseJSON(msg) if (!json || typeof json !== "object") return undefined + const nestedError = + "error" in json && typeof json.error === "object" && json.error !== null + ? (json.error as Record) + : undefined const code = typeof json.code === "string" ? json.code : "" + const nestedCode = typeof nestedError?.code === "string" ? nestedError.code : "" + const nestedType = typeof nestedError?.type === "string" ? nestedError.type : "" + const nestedMessage = typeof nestedError?.message === "string" ? nestedError.message : undefined + const codes = [code, nestedCode, nestedType] - if (json.type === "error" && json.error?.type === "too_many_requests") { + if (json.type === "error" && nestedType === "too_many_requests") { return { message: "Too Many Requests" } } - if (code.includes("exhausted") || code.includes("unavailable")) { + if (codes.some((value) => value.includes("exhausted") || value.includes("unavailable") || value.includes("overloaded"))) { return { message: "Provider is overloaded" } } - if (json.type === "error" && typeof json.error?.code === "string" && json.error.code.includes("rate_limit")) { + if (json.type === "error" && codes.some((value) => value.includes("rate_limit") || value.includes("too_many_requests"))) { return { message: "Rate Limited" } } + if ( + json.type === "error" && + (nestedType === "server_error" || + nestedCode === "server_error" || + nestedType === "upstream_error" || + nestedCode === "stream_read_error" || + nestedType === "service_unavailable_error" || + nestedCode === "server_is_overloaded") + ) { + return { message: nestedMessage?.trim() ? nestedMessage : "Provider is overloaded" } + } return undefined } diff --git a/packages/opencode/test/session/message-v2.test.ts b/packages/opencode/test/session/message-v2.test.ts index 75850e59fb69..0b470469847d 100644 --- a/packages/opencode/test/session/message-v2.test.ts +++ b/packages/opencode/test/session/message-v2.test.ts @@ -1442,6 +1442,51 @@ describe("session.message-v2.fromError", () => { }) }) + test("serializes OpenAI response stream_read_error stream chunks as retryable APIError", () => { + const body = { + type: "error", + sequence_number: 0, + error: { + type: "upstream_error", + code: "stream_read_error", + message: "stream_read_error", + }, + } + const result = MessageV2.fromError({ message: JSON.stringify(body) }, { providerID }) + + expect(result).toStrictEqual({ + name: "APIError", + data: { + message: body.error.message, + isRetryable: true, + responseBody: JSON.stringify(body), + }, + }) + }) + + test("serializes OpenAI response server_is_overloaded stream chunks as retryable APIError", () => { + const body = { + type: "error", + sequence_number: 2, + error: { + type: "service_unavailable_error", + code: "server_is_overloaded", + message: "Our servers are currently overloaded. Please try again later.", + param: null, + }, + } + const result = MessageV2.fromError({ message: JSON.stringify(body) }, { providerID }) + + expect(result).toStrictEqual({ + name: "APIError", + data: { + message: body.error.message, + isRetryable: true, + responseBody: JSON.stringify(body), + }, + }) + }) + test("detects context overflow from APICallError provider messages", () => { const cases = [ "prompt is too long: 213462 tokens > 200000 maximum", diff --git a/packages/opencode/test/session/retry.test.ts b/packages/opencode/test/session/retry.test.ts index 080db82bc057..13acd30c5cc0 100644 --- a/packages/opencode/test/session/retry.test.ts +++ b/packages/opencode/test/session/retry.test.ts @@ -127,6 +127,89 @@ describe("session.retry.retryable", () => { expect(SessionRetry.retryable(error, retryProvider)).toEqual({ message: "Provider is overloaded" }) }) + test("retries streamed upstream stream_read_error json envelopes", () => { + const error = wrap( + JSON.stringify({ + type: "error", + sequence_number: 0, + error: { + type: "upstream_error", + code: "stream_read_error", + message: "stream_read_error", + }, + }), + ) + + expect(SessionRetry.retryable(error, retryProvider)).toEqual({ message: "stream_read_error" }) + }) + + test("retries streamed server_is_overloaded json envelopes", () => { + const error = wrap( + JSON.stringify({ + type: "error", + sequence_number: 2, + error: { + type: "service_unavailable_error", + code: "server_is_overloaded", + message: "Our servers are currently overloaded. Please try again later.", + param: null, + }, + }), + ) + + expect(SessionRetry.retryable(error, retryProvider)).toEqual({ message: "Provider is overloaded" }) + }) + + test("retries streamed server_error json envelopes with empty messages", () => { + const error = wrap( + JSON.stringify({ + type: "error", + sequence_number: 2, + error: { + type: "server_error", + code: "server_error", + message: "", + param: null, + }, + }), + ) + + expect(SessionRetry.retryable(error, retryProvider)).toEqual({ message: "Provider is overloaded" }) + }) + + test("retries streamed upstream_error json envelopes with other codes", () => { + const message = "Upstream temporarily unavailable" + const error = wrap( + JSON.stringify({ + type: "error", + sequence_number: 1, + error: { + type: "upstream_error", + code: "temporary_upstream_failure", + message, + }, + }), + ) + + expect(SessionRetry.retryable(error, retryProvider)).toEqual({ message }) + }) + + test("retries streamed rate_limit_error json envelopes", () => { + const error = wrap( + JSON.stringify({ + type: "error", + sequence_number: 1, + error: { + type: "rate_limit_error", + code: "rate_limit_error", + message: "concurrency limit exceeded for account, please retry later", + }, + }), + ) + + expect(SessionRetry.retryable(error, retryProvider)).toEqual({ message: "Rate Limited" }) + }) + test("does not retry unknown json messages", () => { const error = wrap(JSON.stringify({ error: { message: "no_kv_space" } })) expect(SessionRetry.retryable(error, retryProvider)).toBeUndefined() @@ -436,4 +519,78 @@ describe("session.message-v2.fromError", () => { message: "An error occurred while processing your request.", }) }) + + test("converts OpenAI stream_read_error chunks to retryable APIError", () => { + const body = { + type: "error", + sequence_number: 0, + error: { + type: "upstream_error", + code: "stream_read_error", + message: "stream_read_error", + }, + } + const result = MessageV2.fromError({ message: JSON.stringify(body) }, { providerID: ProviderV2.ID.make("openai") }) + + expect(SessionLegacy.APIError.isInstance(result)).toBe(true) + if (!SessionLegacy.APIError.isInstance(result)) throw new Error("expected APIError") + expect(result.data.isRetryable).toBe(true) + expect(SessionRetry.retryable(result, retryProvider)).toEqual({ message: "stream_read_error" }) + }) + + test("converts OpenAI server_is_overloaded stream chunks to retryable APIError", () => { + const body = { + type: "error", + sequence_number: 2, + error: { + type: "service_unavailable_error", + code: "server_is_overloaded", + message: "Our servers are currently overloaded. Please try again later.", + param: null, + }, + } + const result = MessageV2.fromError({ message: JSON.stringify(body) }, { providerID: ProviderV2.ID.make("openai") }) + + expect(SessionLegacy.APIError.isInstance(result)).toBe(true) + if (!SessionLegacy.APIError.isInstance(result)) throw new Error("expected APIError") + expect(result.data.isRetryable).toBe(true) + expect(SessionRetry.retryable(result, retryProvider)).toEqual({ message: body.error.message }) + }) + + test("converts wrapped OpenAI stream errors from Error.message to retryable APIError", () => { + const body = { + type: "error", + sequence_number: 0, + error: { + type: "upstream_error", + code: "stream_read_error", + message: "stream_read_error", + }, + } + const result = MessageV2.fromError(new Error(JSON.stringify(body)), { providerID: ProviderV2.ID.make("openai") }) + + expect(SessionLegacy.APIError.isInstance(result)).toBe(true) + if (!SessionLegacy.APIError.isInstance(result)) throw new Error("expected APIError") + expect(result.data.isRetryable).toBe(true) + expect(SessionRetry.retryable(result, retryProvider)).toEqual({ message: "stream_read_error" }) + }) + + test("extracts embedded OpenAI stream errors from validation wrapper text", () => { + const body = { + type: "error", + sequence_number: 1, + error: { + type: "rate_limit_error", + code: "rate_limit_error", + message: "concurrency limit exceeded for account, please retry later", + }, + } + const wrapped = `Type validation failed: Value: ${JSON.stringify(body)}` + const result = MessageV2.fromError(new Error(wrapped), { providerID: ProviderV2.ID.make("openai") }) + + expect(SessionLegacy.APIError.isInstance(result)).toBe(true) + if (!SessionLegacy.APIError.isInstance(result)) throw new Error("expected APIError") + expect(result.data.isRetryable).toBe(true) + expect(SessionRetry.retryable(result, retryProvider)).toEqual({ message: "Rate Limited" }) + }) })