Skip to content

fix(codex): name the usage limit and its reset instead of relaying "out of credits" #10473

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
juliusmarminge merged 1 commit into from
Sep 7, 2026
Merged

fix(codex): name the usage limit and its reset instead of relaying "out of credits" #10473

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
299 changes: 299 additions & 0 deletions apps/server/src/provider/Layers/CodexAdapter.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2694,3 +2694,302 @@ it.effect("flushes managed native logs when the adapter layer shuts down", () =>
}
}),
);

const usageLimitRuntimeFactory = makeRuntimeFactory();
const usageLimitLayer = it.layer(
Layer.effect(
CodexAdapter,
Effect.gen(function* () {
const codexConfig = decodeCodexSettings({});
return yield* makeCodexAdapter(codexConfig, {
makeRuntime: usageLimitRuntimeFactory.factory,
});
}),
).pipe(
Layer.provideMerge(ServerConfig.layerTest(process.cwd(), process.cwd())),
Layer.provideMerge(ServerSettingsService.layerTest()),
Layer.provideMerge(providerSessionDirectoryTestLayer),
Layer.provideMerge(NodeServices.layer),
),
);

const USAGE_LIMIT_NOW = "2026-01-01T00:00:00.000Z";
const USAGE_LIMIT_NOW_SECONDS = Date.parse(USAGE_LIMIT_NOW) / 1000;
const CODEX_OUT_OF_CREDITS =
"Your workspace is out of credits. Ask your workspace owner to refill in order to continue.";

function startUsageLimitRuntime() {
return Effect.gen(function* () {
const adapter = yield* CodexAdapter;
yield* adapter.startSession({
provider: ProviderDriverKind.make("codex"),
threadId: asThreadId("thread-1"),
runtimeMode: "full-access",
});
const runtime = usageLimitRuntimeFactory.lastRuntime;
NodeAssert.ok(runtime);
return { adapter, runtime };
});
}

function codexErrorNotification(input: {
readonly id: string;
readonly message: string;
readonly codexErrorInfo?: string;
}): ProviderEvent {
return {
id: asEventId(input.id),
kind: "notification",
provider: ProviderDriverKind.make("codex"),
threadId: asThreadId("thread-1"),
turnId: asTurnId("turn-limit"),
createdAt: USAGE_LIMIT_NOW,
method: "error",
payload: {
threadId: "thread-1",
turnId: "turn-limit",
willRetry: false,
error: {
message: input.message,
...(input.codexErrorInfo ? { codexErrorInfo: input.codexErrorInfo } : {}),
},
},
};
}

function codexRateLimitsNotification(input: {
readonly id: string;
readonly rateLimitReachedType?: string;
readonly primary?: { readonly usedPercent: number; readonly resetsInSeconds: number };
readonly secondary?: { readonly usedPercent: number; readonly resetsInSeconds: number };
}): ProviderEvent {
return {
id: asEventId(input.id),
kind: "notification",
provider: ProviderDriverKind.make("codex"),
threadId: asThreadId("thread-1"),
turnId: asTurnId("turn-limit"),
createdAt: USAGE_LIMIT_NOW,
method: "account/rateLimits/updated",
payload: {
rateLimits: {
limitId: "codex",
...(input.rateLimitReachedType ? { rateLimitReachedType: input.rateLimitReachedType } : {}),
...(input.primary
? {
primary: {
usedPercent: input.primary.usedPercent,
resetsAt: USAGE_LIMIT_NOW_SECONDS + input.primary.resetsInSeconds,
windowDurationMins: 300,
},
}
: {}),
...(input.secondary
? {
secondary: {
usedPercent: input.secondary.usedPercent,
resetsAt: USAGE_LIMIT_NOW_SECONDS + input.secondary.resetsInSeconds,
windowDurationMins: 10_080,
},
}
: {}),
},
},
};
}

function codexUsageLimitTurnFailed(id: string, turnId = "turn-limit"): ProviderEvent {
return {
id: asEventId(id),
kind: "notification",
provider: ProviderDriverKind.make("codex"),
threadId: asThreadId("thread-1"),
turnId: asTurnId(turnId),
createdAt: USAGE_LIMIT_NOW,
method: "turn/completed",
payload: {
threadId: "thread-1",
turn: {
id: turnId,
items: [],
status: "failed",
error: { message: CODEX_OUT_OF_CREDITS, codexErrorInfo: "usageLimitExceeded" },
},
},
};
}

usageLimitLayer("CodexAdapterLive usage limits", (it) => {
it.effect("names the exhausted window and the workspace's missing credits", () =>
Effect.gen(function* () {
const { adapter, runtime } = yield* startUsageLimitRuntime();
const eventsFiber = yield* adapter.streamEvents.pipe(
Stream.take(5),
Stream.runCollect,
Effect.forkChild,
);

yield* runtime.emit(
codexErrorNotification({
id: "evt-limit-error",
message: CODEX_OUT_OF_CREDITS,
codexErrorInfo: "usageLimitExceeded",
}),
);
yield* runtime.emit(
codexRateLimitsNotification({
id: "evt-limit-rate-limits",
rateLimitReachedType: "workspace_owner_credits_depleted",
primary: { usedPercent: 40, resetsInSeconds: 3_600 },
secondary: { usedPercent: 100, resetsInSeconds: 5 * 86_400 + 5 * 3_600 },
}),
);
yield* runtime.emit(codexUsageLimitTurnFailed("evt-limit-turn"));
// A second turn stopping on the same limit says as much as the first.
yield* runtime.emit(codexUsageLimitTurnFailed("evt-limit-turn-2", "turn-limit-2"));

const events = Array.from(yield* Fiber.join(eventsFiber));
const expected =
"Codex usage limit reached. The weekly limit resets in 5d 5h. The workspace has no credits to continue sooner: ask your workspace owner to add credits, or send the message again once the limit resets.";
NodeAssert.deepStrictEqual(
events.map((event) => event.type),
[
"account.rate-limits.updated",
"runtime.error",
"turn.completed",
"runtime.error",
"turn.completed",
],
);
for (const event of events) {
if (event.type === "runtime.error") {
NodeAssert.equal(event.payload.message, expected);
NodeAssert.equal(event.payload.detail, CODEX_OUT_OF_CREDITS);
}
if (event.type === "turn.completed") {
NodeAssert.equal(event.payload.errorMessage, expected);
}
}
}),
);

it.effect("names the session window for a plan limit", () =>
Effect.gen(function* () {
const { adapter, runtime } = yield* startUsageLimitRuntime();
const eventsFiber = yield* adapter.streamEvents.pipe(
Stream.take(3),
Stream.runCollect,
Effect.forkChild,
);

yield* runtime.emit(
codexErrorNotification({
id: "evt-plan-error",
message: "You've hit your usage limit.",
codexErrorInfo: "usageLimitExceeded",
}),
);
yield* runtime.emit(
codexRateLimitsNotification({
id: "evt-plan-rate-limits",
rateLimitReachedType: "rate_limit_reached",
primary: { usedPercent: 100, resetsInSeconds: 3 * 3_600 + 20 * 60 },
}),
);
yield* runtime.emit(codexUsageLimitTurnFailed("evt-plan-turn"));

const events = Array.from(yield* Fiber.join(eventsFiber));
const completed = events.find((event) => event.type === "turn.completed");
NodeAssert.equal(
completed?.payload.errorMessage,
"Codex usage limit reached. The session limit resets in 3h 20m. Send the message again once the limit resets.",
);
}),
);

it.effect("reads a rate-limit snapshot seen earlier in the session", () =>
Effect.gen(function* () {
const { adapter, runtime } = yield* startUsageLimitRuntime();
const eventsFiber = yield* adapter.streamEvents.pipe(
Stream.take(3),
Stream.runCollect,
Effect.forkChild,
);

// The window arrives long before the stop, and the update that reports the
// limit as reached carries no windows of its own.
yield* runtime.emit(
codexRateLimitsNotification({
id: "evt-early-rate-limits",
primary: { usedPercent: 100, resetsInSeconds: 3 * 3_600 + 20 * 60 },
}),
);
yield* runtime.emit(
codexRateLimitsNotification({
id: "evt-sparse-rate-limits",
rateLimitReachedType: "rate_limit_reached",
}),
);
yield* runtime.emit(codexUsageLimitTurnFailed("evt-early-turn"));

const events = Array.from(yield* Fiber.join(eventsFiber));
const completed = events.find((event) => event.type === "turn.completed");
NodeAssert.equal(
completed?.payload.errorMessage,
"Codex usage limit reached. The session limit resets in 3h 20m. Send the message again once the limit resets.",
);
}),
);

it.effect("falls back to the short message without a rate-limit snapshot", () =>
Effect.gen(function* () {
const { adapter, runtime } = yield* startUsageLimitRuntime();
const eventsFiber = yield* adapter.streamEvents.pipe(
Stream.take(2),
Stream.runCollect,
Effect.forkChild,
);

yield* runtime.emit(
codexErrorNotification({
id: "evt-bare-error",
message: CODEX_OUT_OF_CREDITS,
codexErrorInfo: "usageLimitExceeded",
}),
);
yield* runtime.emit(codexUsageLimitTurnFailed("evt-bare-turn"));

const events = Array.from(yield* Fiber.join(eventsFiber));
const expected = "Codex usage limit reached. Send the message again once the limit resets.";
NodeAssert.deepStrictEqual(
events.map((event) => event.type),
["runtime.error", "turn.completed"],
);
const runtimeError = events.find((event) => event.type === "runtime.error");
NodeAssert.equal(runtimeError?.payload.message, expected);
const completed = events.find((event) => event.type === "turn.completed");
NodeAssert.equal(completed?.payload.errorMessage, expected);
}),
);

it.effect("still relays other provider errors as they arrive", () =>
Effect.gen(function* () {
const { adapter, runtime } = yield* startUsageLimitRuntime();
const firstEventFiber = yield* Stream.runHead(adapter.streamEvents).pipe(Effect.forkChild);

yield* runtime.emit(
codexErrorNotification({
id: "evt-other-error",
message: "Codex is temporarily unavailable.",
codexErrorInfo: "internalServerError",
}),
);

const first = yield* Fiber.join(firstEventFiber);
NodeAssert.equal(first._tag, "Some");
if (first._tag !== "Some" || first.value.type !== "runtime.error") return;
NodeAssert.equal(first.value.payload.message, "Codex is temporarily unavailable.");
NodeAssert.equal(first.value.payload.class, "provider_error");
}),
);
});
62 changes: 60 additions & 2 deletions apps/server/src/provider/Layers/CodexAdapter.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,12 @@ import {
} from "./CodexSessionRuntime.ts";
import { type EventNdjsonLogger, makeEventNdjsonLogger } from "./EventNdjsonLogger.ts";
import { resolveCodexLaunchArgs } from "./codexLaunchArgs.ts";
import { codexRateLimitsToUpdate } from "./codexUsageLimits.ts";
import {
type CodexRateLimitSnapshot,
codexRateLimitsToUpdate,
codexUsageLimitMessage,
mergeCodexRateLimits,
} from "./codexUsageLimits.ts";
const isCodexAppServerProcessExitedError = Schema.is(CodexErrors.CodexAppServerProcessExitedError);
const isCodexAppServerTransportError = Schema.is(CodexErrors.CodexAppServerTransportError);
const isCodexSessionRuntimeThreadIdMissingError = Schema.is(
Expand Down Expand Up @@ -2282,6 +2287,12 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* (
: {}),
};
const turnTokenUsage = makeCodexTurnTokenUsageState();
// Codex reports a usage-limit stop as OpenAI's own sentence, which on a
// Business workspace blames credits for a window that ran out. The
// snapshot naming that window arrives in its own notification, before or
// after the stop and often sparse, so keep the session's merged view of
// it and read it when a turn fails on the limit.
let rateLimits: CodexRateLimitSnapshot | undefined;
const sessionScope = yield* Scope.make("sequential");
let sessionScopeTransferred = false;
yield* Effect.addFinalizer(() =>
Expand Down Expand Up @@ -2339,12 +2350,56 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* (
}
}

const runtimeEvents = mapToRuntimeEvents(event, event.threadId).map((runtimeEvent) => {
if (event.method === "account/rateLimits/updated") {
const limitsPayload = readPayload(
EffectCodexSchema.V2AccountRateLimitsUpdatedNotification,
event.payload,
);
if (limitsPayload) {
rateLimits = mergeCodexRateLimits(rateLimits, limitsPayload.rateLimits);
}
} else if (event.method === "error") {
const errorPayload = readPayload(
EffectCodexSchema.V2ErrorNotification,
event.payload,
);
// The failed `turn/completed` repeats this sentence and is answered
// below; relaying both would show the limit twice.
if (errorPayload?.error.codexErrorInfo === "usageLimitExceeded") return;
}

let usageLimitError: ProviderRuntimeEvent | undefined;
let usageLimitMessage: string | undefined;
if (event.method === "turn/completed") {
const completedPayload = readPayload(
EffectCodexSchema.V2TurnCompletedNotification,
event.payload,
);
const turnError =
completedPayload?.turn.status === "failed"
? completedPayload.turn.error
: undefined;
if (turnError?.codexErrorInfo === "usageLimitExceeded") {
usageLimitMessage = codexUsageLimitMessage(rateLimits, event.createdAt);
Comment thread
coderabbitai[bot] marked this conversation as resolved.
usageLimitError = {
...runtimeEventBase(event, event.threadId),
type: "runtime.error",
payload: {
message: usageLimitMessage,
class: "provider_error",
...(turnError.message ? { detail: turnError.message } : {}),
},
};
}
}

const mappedEvents = mapToRuntimeEvents(event, event.threadId).map((runtimeEvent) => {
if (runtimeEvent.type === "turn.completed" && runtimeEvent.turnId) {
return {
...runtimeEvent,
payload: {
...runtimeEvent.payload,
...(usageLimitMessage ? { errorMessage: usageLimitMessage } : {}),
tokenUsage: completeCodexTurnTokenUsage(
turnTokenUsage,
String(runtimeEvent.turnId),
Expand All @@ -2368,6 +2423,9 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* (
}
return runtimeEvent;
});
const runtimeEvents = usageLimitError
? [usageLimitError, ...mappedEvents]
: mappedEvents;
if (runtimeEvents.length === 0) {
yield* Effect.logDebug("ignoring unhandled Codex provider event", {
method: event.method,
Expand Down
Loading
Loading