From 5b40d03544a01bbf5252f2bdf824a2fb4e6035ed Mon Sep 17 00:00:00 2001 From: ndycode Date: Sat, 8 Aug 2026 17:46:09 +0800 Subject: [PATCH 1/2] fix(issue-656): defer exhausted quota windows until reset --- docs/development/CONFIG_FIELDS.md | 4 + lib/codex-manager/backend-settings-schema.ts | 2 +- lib/config.ts | 4 +- lib/preemptive-quota-scheduler.ts | 54 ++++++++----- test/preemptive-quota-scheduler.test.ts | 83 ++++++++++++++++++++ 5 files changed, 126 insertions(+), 21 deletions(-) diff --git a/docs/development/CONFIG_FIELDS.md b/docs/development/CONFIG_FIELDS.md index 41129c0a4..be1d08de0 100644 --- a/docs/development/CONFIG_FIELDS.md +++ b/docs/development/CONFIG_FIELDS.md @@ -158,6 +158,10 @@ Upgrade note: | `preemptiveQuotaRemainingPercent7d` | `5` | | `preemptiveQuotaMaxDeferralMs` | `7200000` | +`preemptiveQuotaMaxDeferralMs` is the fallback delay when a near-exhausted window has +missing, invalid, or stale reset data. A trusted future reset may schedule through the +reset time, subject to the scheduler's seven-day safety ceiling. + ### Notifications | Key | Default | diff --git a/lib/codex-manager/backend-settings-schema.ts b/lib/codex-manager/backend-settings-schema.ts index 1c4ef58fe..c783c6ea7 100644 --- a/lib/codex-manager/backend-settings-schema.ts +++ b/lib/codex-manager/backend-settings-schema.ts @@ -276,7 +276,7 @@ export const BACKEND_NUMBER_OPTIONS: BackendNumberSettingOption[] = [ { key: "preemptiveQuotaMaxDeferralMs", label: "Max Preemptive Deferral", - description: "Maximum time allowed for quota-based delay.", + description: "Fallback time for quota delay when reset data is unavailable or stale.", min: 1_000, max: 24 * 60 * 60_000, step: 60_000, diff --git a/lib/config.ts b/lib/config.ts index 18948f565..e421d6596 100644 --- a/lib/config.ts +++ b/lib/config.ts @@ -1818,7 +1818,9 @@ export function getPreemptiveQuotaRemainingPercent7d( } /** - * Get the configured maximum deferral time (in milliseconds) for preemptive quota checks. + * Get the configured fallback deferral time (in milliseconds) for preemptive quota checks. + * Trusted future quota resets may extend a deferral beyond this fallback, up to the + * scheduler's seven-day safety ceiling. * * Reads an environment override or the plugin configuration and enforces a minimum of 1000 ms. * diff --git a/lib/preemptive-quota-scheduler.ts b/lib/preemptive-quota-scheduler.ts index 92ad2feab..77d59ec63 100644 --- a/lib/preemptive-quota-scheduler.ts +++ b/lib/preemptive-quota-scheduler.ts @@ -1,3 +1,4 @@ +import { MAX_RATE_LIMIT_DELAY_MS } from "./constants.js"; import { quotaLeftPercentFromUsed } from "./quota-readiness.js"; export interface QuotaSchedulerWindow { @@ -28,6 +29,33 @@ export interface QuotaSchedulerOptions { const DEFAULT_REMAINING_PERCENT_THRESHOLD = 5; const DEFAULT_MAX_DEFERRAL_MS = 2 * 60 * 60_000; +const MAX_TRUSTED_RESET_AGE_MS = MAX_RATE_LIMIT_DELAY_MS; + +/** + * Return a reset wait only when the timestamp came from a recent, coherent snapshot. + * A future timestamp from an old or clock-invalid snapshot must not extend a + * preemptive cooldown beyond the configured fallback cap. + */ +function trustedResetWaitMs( + window: QuotaSchedulerWindow, + snapshot: QuotaSchedulerSnapshot, + now: number, +): number { + const resetAtMs = window.resetAtMs; + const updatedAt = snapshot.updatedAt; + if ( + typeof resetAtMs !== "number" || + !Number.isFinite(resetAtMs) || + resetAtMs <= now || + typeof updatedAt !== "number" || + !Number.isFinite(updatedAt) || + updatedAt > now || + now - updatedAt > MAX_TRUSTED_RESET_AGE_MS + ) { + return 0; + } + return Math.min(resetAtMs - now, MAX_RATE_LIMIT_DELAY_MS); +} /** * Clamp a number to the inclusive integer range [min, max] after flooring. @@ -250,19 +278,6 @@ export class PreemptiveQuotaScheduler { const snapshot = this.snapshots.get(key); if (!snapshot) return { defer: false, waitMs: 0 }; - const primaryWait = - typeof snapshot.primary.resetAtMs === "number" && - Number.isFinite(snapshot.primary.resetAtMs) && - snapshot.primary.resetAtMs > now - ? snapshot.primary.resetAtMs - now - : 0; - const secondaryWait = - typeof snapshot.secondary.resetAtMs === "number" && - Number.isFinite(snapshot.secondary.resetAtMs) && - snapshot.secondary.resetAtMs > now - ? snapshot.secondary.resetAtMs - now - : 0; - // For a 429 deferral, only count a window whose reset is genuinely a // rate-limit / exhaustion window. A window carrying a healthy usedPercent // (e.g. a weekly secondary at 30% from a prior 200 snapshot) is NOT the @@ -302,14 +317,15 @@ export class PreemptiveQuotaScheduler { Number.isFinite(snapshot.secondary.usedPercent) && snapshot.secondary.usedPercent >= 100 - this.secondaryRemainingPercentThreshold; const nearExhaustedWait = Math.max( - primaryNearExhausted ? primaryWait : 0, - secondaryNearExhausted ? secondaryWait : 0, + primaryNearExhausted && snapshot.status !== 429 + ? trustedResetWaitMs(snapshot.primary, snapshot, now) || this.maxDeferralMs + : 0, + secondaryNearExhausted && snapshot.status !== 429 + ? trustedResetWaitMs(snapshot.secondary, snapshot, now) || this.maxDeferralMs + : 0, ); if (nearExhaustedWait > 0) { - const bounded = Math.min(nearExhaustedWait, this.maxDeferralMs); - if (bounded > 0) { - return { defer: true, waitMs: bounded, reason: "quota-near-exhaustion" }; - } + return { defer: true, waitMs: nearExhaustedWait, reason: "quota-near-exhaustion" }; } return { defer: false, waitMs: 0 }; diff --git a/test/preemptive-quota-scheduler.test.ts b/test/preemptive-quota-scheduler.test.ts index db9aad114..f2dab677c 100644 --- a/test/preemptive-quota-scheduler.test.ts +++ b/test/preemptive-quota-scheduler.test.ts @@ -174,6 +174,89 @@ describe("preemptive quota scheduler", () => { expect(decision.waitMs).toBe(100_000); }); + it("uses a trusted future reset for an exhausted long quota window", () => { + const scheduler = new PreemptiveQuotaScheduler(); + const now = 1_000_000; + const resetWaitMs = 72 * 60 * 60_000; + scheduler.update("acc:model", { + status: 200, + primary: { usedPercent: 20, resetAtMs: now + 5 * 60 * 60_000 }, + secondary: { usedPercent: 100, resetAtMs: now + resetWaitMs }, + updatedAt: now, + }); + + const decision = scheduler.getDeferral("acc:model", now + 1_000); + + expect(decision).toEqual({ + defer: true, + waitMs: resetWaitMs - 1_000, + reason: "quota-near-exhaustion", + }); + }); + + it("caps trusted monthly-scale reset data at the seven-day safety ceiling", () => { + const scheduler = new PreemptiveQuotaScheduler(); + const now = 1_000_000; + const sevenDaysMs = 7 * 24 * 60 * 60_000; + scheduler.update("acc:model", { + status: 200, + primary: {}, + secondary: { + usedPercent: 100, + resetAtMs: now + 30 * 24 * 60 * 60_000, + }, + updatedAt: now, + }); + + const decision = scheduler.getDeferral("acc:model", now + 1_000); + + expect(decision.waitMs).toBe(sevenDaysMs); + }); + + it.each([ + ["missing", undefined], + ["invalid", Number.NaN], + ] as const)("falls back to the configured cap for %s reset data", (_label, resetAtMs) => { + const maxDeferralMs = 30 * 60_000; + const scheduler = new PreemptiveQuotaScheduler({ maxDeferralMs }); + const now = 1_000_000; + scheduler.update("acc:model", { + status: 200, + primary: { usedPercent: 100, resetAtMs }, + secondary: {}, + updatedAt: now, + }); + + const decision = scheduler.getDeferral("acc:model", now + 1_000); + + expect(decision).toEqual({ + defer: true, + waitMs: maxDeferralMs, + reason: "quota-near-exhaustion", + }); + }); + + it("falls back to the configured cap for stale reset data", () => { + const maxDeferralMs = 30 * 60_000; + const scheduler = new PreemptiveQuotaScheduler({ maxDeferralMs }); + const now = 1_000_000; + const sevenDaysMs = 7 * 24 * 60 * 60_000; + scheduler.update("acc:model", { + status: 200, + primary: { usedPercent: 100, resetAtMs: now + 72 * 60 * 60_000 }, + secondary: {}, + updatedAt: now - sevenDaysMs - 1, + }); + + const decision = scheduler.getDeferral("acc:model", now + 1_000); + + expect(decision).toEqual({ + defer: true, + waitMs: maxDeferralMs, + reason: "quota-near-exhaustion", + }); + }); + it("prunes expired snapshots", () => { const scheduler = new PreemptiveQuotaScheduler(); scheduler.update("a", { From 0f6038afc10e725e732713aa39b94683a6614fc3 Mon Sep 17 00:00:00 2001 From: ndycode Date: Sat, 8 Aug 2026 19:31:48 +0800 Subject: [PATCH 2/2] fix: resolve issues 652 through 656 --- README.md | 1 + docs/reference/storage-paths.md | 1 + lib/accounts.ts | 192 +++- lib/codex-manager.ts | 80 +- lib/codex-manager/account-pool-write.ts | 4 + lib/codex-manager/commands/status.ts | 7 + lib/codex-manager/commands/uninstall.ts | 24 +- lib/codex-manager/commands/why-selected.ts | 4 +- lib/codex-manager/health-check.ts | 30 +- lib/codex-manager/login-oauth.ts | 23 +- lib/forecast.ts | 15 +- lib/index.ts | 1 + lib/parallel-probe.ts | 6 +- lib/policy/runtime-policy.ts | 8 + lib/preemptive-quota-scheduler.ts | 17 +- lib/rotation.ts | 28 +- lib/runtime-constants.ts | 4 + lib/runtime-rotation-proxy.ts | 110 ++- lib/runtime/app-bind.ts | 839 +++++++++++++++++- lib/runtime/rotation-account-selection.ts | 2 + lib/runtime/rotation-proxy-state.ts | 2 + lib/runtime/rotation-token-refresh.ts | 3 + lib/schemas.ts | 3 + lib/storage.ts | 30 +- lib/storage/flagged-storage.ts | 18 + lib/storage/path-state.ts | 45 +- lib/storage/public-types.ts | 6 + scripts/codex-app-router.js | 50 +- scripts/codex.js | 186 +++- scripts/install-codex-auth-utils.js | 11 +- scripts/postinstall.js | 2 + scripts/preuninstall.js | 11 + test/accounts.test.ts | 118 ++- test/app-bind.test.ts | 814 ++++++++++++++++- test/codex-app-router.test.ts | 1 + test/codex-bin-wrapper.test.ts | 137 ++- test/codex-manager-cli.test.ts | 24 + ...odex-manager-selection-diagnostics.test.ts | 234 +++++ test/codex-manager-status-command.test.ts | 22 + test/flagged-storage.test.ts | 22 + test/login-oauth-callback-guidance.test.ts | 92 +- test/postinstall.test.ts | 24 + test/preemptive-quota-scheduler.test.ts | 34 + test/preuninstall.test.ts | 56 ++ test/rotation-proxy-state.test.ts | 2 + test/rotation-token-refresh.test.ts | 61 +- test/runtime-rotation-proxy.test.ts | 244 +++++ test/schemas.test.ts | 14 + test/storage.test.ts | 65 +- test/uninstall-command.test.ts | 55 ++ 50 files changed, 3622 insertions(+), 160 deletions(-) create mode 100644 test/codex-manager-selection-diagnostics.test.ts create mode 100644 test/postinstall.test.ts diff --git a/README.md b/README.md index 54a1d3073..fc5a23774 100644 --- a/README.md +++ b/README.md @@ -258,6 +258,7 @@ For remote or headless shells, prefer `codex-multi-auth login --device-auth`. | Budget guards | `~/.codex/multi-auth/budget-guards.json` | | Local client tokens | `~/.codex/multi-auth/local-client-tokens.json` | | Runtime app helper status | `~/.codex/multi-auth/runtime-rotation-app-helper.json` | +| Runtime app helper owner metadata | `~/.codex/multi-auth/runtime-rotation-app-helper-owner..json` | | Persistent app bind state/logs | `~/.codex/multi-auth/app-bind/` | | Logs | `~/.codex/multi-auth/logs/codex-plugin/` | | Per-project accounts | `~/.codex/multi-auth/projects//openai-codex-accounts.json` | diff --git a/docs/reference/storage-paths.md b/docs/reference/storage-paths.md index 098eb78db..11e948855 100644 --- a/docs/reference/storage-paths.md +++ b/docs/reference/storage-paths.md @@ -40,6 +40,7 @@ Override root: | Local bridge client tokens | `~/.codex/multi-auth/local-client-tokens.json` | | Cross-process refresh leases | `~/.codex/multi-auth/refresh-leases/` | | Runtime app helper status | `~/.codex/multi-auth/runtime-rotation-app-helper.json` | +| Runtime app helper owner metadata | `~/.codex/multi-auth/runtime-rotation-app-helper-owner..json` | | Persistent app bind directory | `~/.codex/multi-auth/app-bind/` | | Named pool backups | `~/.codex/multi-auth/backups/` | | Per-project account pools | `~/.codex/multi-auth/projects//openai-codex-accounts.json` | diff --git a/lib/accounts.ts b/lib/accounts.ts index ebbfcfe2e..8b7152e3f 100644 --- a/lib/accounts.ts +++ b/lib/accounts.ts @@ -1,4 +1,5 @@ import type { Auth } from "@codex-ai/sdk"; +import { createHash } from "node:crypto"; import { saveAccountsWithRetry } from "./storage/save-retry.js"; import { createLogger } from "./logger.js"; import { @@ -109,6 +110,36 @@ function getAccountCircuitKey(account: ManagedAccount): string { return account.circuitKeyId; } +function deriveAccountRecordId( + account: { + accountId?: string; + email?: string; + refreshToken: string; + addedAt: number; + }, +): string { + const seed = [ + account.addedAt, + account.accountId?.trim() ?? "", + account.email?.trim().toLowerCase() ?? "", + account.refreshToken.trim(), + ].join("\u0000"); + return `record:${createHash("sha256").update(seed).digest("hex")}`; +} + +function resolveAccountRecordId( + account: { + recordId?: string; + accountId?: string; + email?: string; + refreshToken: string; + addedAt: number; + }, +): string { + const stored = account.recordId?.trim(); + return stored || deriveAccountRecordId(account); +} + export function getRuntimeTrackerKey(account: ManagedAccount): string | number { if (account._runtimeTrackerKey !== undefined) { return account._runtimeTrackerKey; @@ -239,8 +270,12 @@ function isRetryableAuthPersistenceError(error: unknown): boolean { // re-exported here to preserve the historical import surface. export type { Workspace } from "./storage/public-types.js"; +/** Stable operator-facing marker for an explicitly invalidated OAuth token. */ +export const AUTH_INVALIDATION_MARKER = "token-invalid — re-login needed"; + export interface ManagedAccount { index: number; + recordId?: string; _runtimeTrackerKey?: string | number; circuitKeyId?: string; accountId?: string; @@ -264,6 +299,8 @@ export interface ManagedAccount { rateLimitResetTimes: RateLimitStateV3; coolingDownUntil?: number; cooldownReason?: CooldownReason; + authInvalidatedAt?: number; + authInvalidationErrorCode?: string; consecutiveAuthFailures?: number; workspaces?: Workspace[]; currentWorkspaceIndex?: number; @@ -488,6 +525,9 @@ export class AccountManager { return { index, + recordId: resolveAccountRecordId( + { ...account, refreshToken }, + ), accountId: matchesFallback ? (fallbackAccountId ?? account.accountId) : account.accountId, @@ -512,6 +552,8 @@ export class AccountManager { rateLimitResetTimes: account.rateLimitResetTimes ?? {}, coolingDownUntil: account.coolingDownUntil, cooldownReason: account.cooldownReason, + authInvalidatedAt: account.authInvalidatedAt, + authInvalidationErrorCode: account.authInvalidationErrorCode, workspaces: account.workspaces, currentWorkspaceIndex: account.currentWorkspaceIndex, }; @@ -525,6 +567,14 @@ export class AccountManager { const now = nowMs(); this.accounts.push({ index: this.accounts.length, + recordId: deriveAccountRecordId( + { + accountId: fallbackAccountId, + email: fallbackAccountEmail, + refreshToken: authFallback.refresh, + addedAt: now, + }, + ), accountId: fallbackAccountId, accountIdSource: fallbackAccountId ? "token" : undefined, email: fallbackAccountEmail, @@ -559,6 +609,14 @@ export class AccountManager { this.accounts = [ { index: 0, + recordId: deriveAccountRecordId( + { + accountId: fallbackAccountId, + email: fallbackAccountEmail, + refreshToken: authFallback.refresh, + addedAt: now, + }, + ), accountId: fallbackAccountId, accountIdSource: fallbackAccountId ? "token" : undefined, email: fallbackAccountEmail, @@ -640,6 +698,7 @@ export class AccountManager { clearExpiredRateLimits(account); return ( !isRateLimitedForFamily(account, family, model) && + !this.isAccountAuthInvalidated(account) && !this.isAccountCoolingDown(account) && this.isCircuitAvailable(account) ); @@ -652,8 +711,23 @@ export class AccountManager { ): string | null { const account = this.getAccountByIndex(index); if (!account) return "missing"; + return this.getManagedAccountRuntimeSkipReason(account, family, model); + } + + /** + * Evaluate a managed-account snapshot with the same runtime gates used by + * production selection. This is public for read-only diagnostics that build + * their own trace rows from storage and therefore cannot safely resolve an + * account by its compacted array position. + */ + getManagedAccountRuntimeSkipReason( + account: ManagedAccount, + family: ModelFamily, + model?: string | null, + ): string | null { if (account.enabled === false) return "disabled"; if (!this.hasEnabledWorkspaces(account)) return "workspace-disabled"; + if (this.isAccountAuthInvalidated(account)) return AUTH_INVALIDATION_MARKER; clearExpiredRateLimits(account); if (isRateLimitedForFamily(account, family, model)) return "rate-limited"; if (this.isAccountCoolingDown(account)) { @@ -770,6 +844,7 @@ export class AccountManager { clearExpiredRateLimits(account); if ( isRateLimitedForFamily(account, family, model) || + this.isAccountAuthInvalidated(account) || this.isAccountCoolingDown(account) || !this.isCircuitAvailable(account) ) { @@ -804,6 +879,7 @@ export class AccountManager { clearExpiredRateLimits(account); if ( isRateLimitedForFamily(account, family, model) || + this.isAccountAuthInvalidated(account) || this.isAccountCoolingDown(account) || !this.isCircuitAvailable(account) ) { @@ -861,6 +937,7 @@ export class AccountManager { clearExpiredRateLimits(account); return ( !isRateLimitedForFamily(account, family, model) && + !this.isAccountAuthInvalidated(account) && !this.isAccountCoolingDown(account) && this.isCircuitAvailable(account) ); @@ -917,6 +994,7 @@ export class AccountManager { clearExpiredRateLimits(account); const isAvailable = !isRateLimitedForFamily(account, family, model) && + !this.isAccountAuthInvalidated(account) && !this.isAccountCoolingDown(account) && this.isCircuitAvailable(account); return { @@ -1226,6 +1304,28 @@ export class AccountManager { account.consecutiveAuthFailures = 0; } + markAuthInvalidated( + account: ManagedAccount, + errorCode = "token_invalidated", + invalidatedAt = nowMs(), + ): void { + account.authInvalidatedAt = invalidatedAt; + account.authInvalidationErrorCode = errorCode; + } + + clearAuthInvalidation(account: ManagedAccount): void { + delete account.authInvalidatedAt; + delete account.authInvalidationErrorCode; + } + + isAccountAuthInvalidated(account: ManagedAccount): boolean { + return ( + typeof account.authInvalidatedAt === "number" && + Number.isFinite(account.authInvalidatedAt) && + account.authInvalidatedAt > 0 + ); + } + getAccountByIdentity( candidate: AccountIdentityCandidate, auth?: OAuthAuthDetails, @@ -1257,6 +1357,7 @@ export class AccountManager { account.refreshToken = auth.refresh; account.access = auth.access; account.expires = auth.expires; + this.clearAuthInvalidation(account); const tokenAccountId = extractAccountId(auth.access)?.trim() || undefined; if ( tokenAccountId && @@ -1315,6 +1416,24 @@ export class AccountManager { account.accessToken = disk.accessToken; account.expiresAt = disk.expiresAt; } + const diskInvalidatedAt = disk.authInvalidatedAt; + const diskHasValidInvalidation = + typeof diskInvalidatedAt === "number" && + Number.isFinite(diskInvalidatedAt) && + diskInvalidatedAt > 0; + const accountHasValidInvalidation = + typeof account.authInvalidatedAt === "number" && + Number.isFinite(account.authInvalidatedAt) && + account.authInvalidatedAt > 0; + if (diskHasValidInvalidation && !accountHasValidInvalidation) { + // Ordinary saves must not erase an invalidation written by another + // AccountManager. Successful refresh persistence explicitly deletes + // this marker in commitRefreshedAuth; this reconciliation path does not. + account.authInvalidatedAt = diskInvalidatedAt; + if (disk.authInvalidationErrorCode) { + account.authInvalidationErrorCode = disk.authInvalidationErrorCode; + } + } } return snapshot; } @@ -1364,27 +1483,38 @@ export class AccountManager { const snapshot: AccountStorageV3 = { version: 3, - accounts: this.accounts.map((account) => ({ - accountId: account.accountId, - accountIdSource: account.accountIdSource, - accountLabel: account.accountLabel, - email: account.email, - refreshToken: account.refreshToken, - accessToken: account.access, - expiresAt: account.expires, - enabled: account.enabled === false ? false : undefined, - addedAt: account.addedAt, - lastUsed: account.lastUsed, - lastSwitchReason: account.lastSwitchReason, - rateLimitResetTimes: - Object.keys(account.rateLimitResetTimes).length > 0 - ? account.rateLimitResetTimes + accounts: this.accounts.map((account) => { + const hasValidAuthInvalidation = + this.isAccountAuthInvalidated(account); + return { + recordId: account.recordId, + accountId: account.accountId, + accountIdSource: account.accountIdSource, + accountLabel: account.accountLabel, + email: account.email, + refreshToken: account.refreshToken, + accessToken: account.access, + expiresAt: account.expires, + enabled: account.enabled === false ? false : undefined, + addedAt: account.addedAt, + lastUsed: account.lastUsed, + lastSwitchReason: account.lastSwitchReason, + rateLimitResetTimes: + Object.keys(account.rateLimitResetTimes).length > 0 + ? account.rateLimitResetTimes + : undefined, + coolingDownUntil: account.coolingDownUntil, + cooldownReason: account.cooldownReason, + authInvalidatedAt: hasValidAuthInvalidation + ? account.authInvalidatedAt + : undefined, + authInvalidationErrorCode: hasValidAuthInvalidation + ? account.authInvalidationErrorCode : undefined, - coolingDownUntil: account.coolingDownUntil, - cooldownReason: account.cooldownReason, - workspaces: account.workspaces, - currentWorkspaceIndex: account.currentWorkspaceIndex, - })), + workspaces: account.workspaces, + currentWorkspaceIndex: account.currentWorkspaceIndex, + }; + }), activeIndex, activeIndexByFamily, }; @@ -1447,6 +1577,8 @@ export class AccountManager { storedAccount.email = nextEmail; } storedAccount.enabled = undefined; + delete storedAccount.authInvalidatedAt; + delete storedAccount.authInvalidationErrorCode; delete storedAccount.coolingDownUntil; delete storedAccount.cooldownReason; @@ -1463,6 +1595,9 @@ export class AccountManager { coolingDownUntil: liveAccount.coolingDownUntil, cooldownReason: liveAccount.cooldownReason, consecutiveAuthFailures: liveAccount.consecutiveAuthFailures, + authInvalidatedAt: liveAccount.authInvalidatedAt, + authInvalidationErrorCode: + liveAccount.authInvalidationErrorCode, }; this.updateFromAuth(liveAccount, auth); @@ -1483,6 +1618,20 @@ export class AccountManager { liveAccount.enabled = previousLiveAccountState.enabled; liveAccount.consecutiveAuthFailures = previousLiveAccountState.consecutiveAuthFailures; + if (previousLiveAccountState.authInvalidatedAt === undefined) { + delete liveAccount.authInvalidatedAt; + } else { + liveAccount.authInvalidatedAt = + previousLiveAccountState.authInvalidatedAt; + } + if ( + previousLiveAccountState.authInvalidationErrorCode === undefined + ) { + delete liveAccount.authInvalidationErrorCode; + } else { + liveAccount.authInvalidationErrorCode = + previousLiveAccountState.authInvalidationErrorCode; + } if (previousLiveAccountState.coolingDownUntil === undefined) { delete liveAccount.coolingDownUntil; } else { @@ -1531,7 +1680,8 @@ export class AccountManager { getMinWaitTimeForFamily(family: ModelFamily, model?: string | null): number { const now = nowMs(); const enabledAccounts = this.accounts.filter( - (account) => account.enabled !== false, + (account) => + account.enabled !== false && !this.isAccountAuthInvalidated(account), ); const available = enabledAccounts.filter((account) => { clearExpiredRateLimits(account); diff --git a/lib/codex-manager.ts b/lib/codex-manager.ts index 83fd9b153..4d1daf0fe 100644 --- a/lib/codex-manager.ts +++ b/lib/codex-manager.ts @@ -1,4 +1,6 @@ import { + AUTH_INVALIDATION_MARKER, + AccountManager, extractAccountEmail, extractAccountId, formatAccountLabel, @@ -61,6 +63,11 @@ import { getTokenTracker, selectHybridAccountTraced, } from "./rotation.js"; +import { + evaluateRuntimePolicy, + loadRuntimePolicyState, +} from "./policy/runtime-policy.js"; +import { CURRENT_CODEX_MODEL } from "./request/helpers/model-map.js"; import { runDoctor as runRepairDoctor, type RepairCommandDeps, @@ -387,40 +394,79 @@ export async function autoSyncActiveAccountToCodex(): Promise { ...(syncIdToken ? { idToken: syncIdToken } : {}), }); } -function buildSelectAccountTraced(): ( +/** @internal Exposed for diagnostics regression tests; not part of the CLI API. */ +export function buildSelectAccountTraced(): ( storage: AccountStorageV3, -) => ReturnType { - return (storage: AccountStorageV3) => { +) => Promise> { + return async (storage: AccountStorageV3) => { const now = Date.now(); const healthTracker = getHealthTracker(); const tokenTracker = getTokenTracker(); + const runtimeAccountManager = new AccountManager(undefined, storage); + const runtimeAccountsByIndex = new Map( + runtimeAccountManager + .getAccountsSnapshot() + .map((account) => [account.index, account] as const), + ); const accountsWithMetrics: AccountWithMetrics[] = storage.accounts.map( (account, index) => { - const enabled = account?.enabled !== false; - const rateLimits = account?.rateLimitResetTimes ?? {}; - let rateLimited = false; - for (const value of Object.values(rateLimits)) { - if (typeof value === "number" && value > now) { - rateLimited = true; - break; - } - } - const coolingDown = - typeof account?.coolingDownUntil === "number" && - account.coolingDownUntil > now; - const isAvailable = enabled && !rateLimited && !coolingDown; + const runtimeAccount = runtimeAccountsByIndex.get(index); + const runtimeSkipReason = runtimeAccount + ? runtimeAccountManager.getManagedAccountRuntimeSkipReason( + runtimeAccount, + "codex", + CURRENT_CODEX_MODEL, + ) + : "missing"; return { index, trackerKey: account?.accountId ?? index, - isAvailable, + isAvailable: runtimeSkipReason === null, + unavailableReason: runtimeSkipReason ?? undefined, lastUsed: account?.lastUsed ?? 0, }; }, ); + const policyState = await loadRuntimePolicyState(); + const policy = await evaluateRuntimePolicy({ + state: policyState, + accounts: storage.accounts.map((account, index) => ({ + index, + accountId: account.accountId, + email: account.email, + })), + model: CURRENT_CODEX_MODEL, + now, + }); + const blockedAccountIndexes = new Set(policy.blockedAccountIndexes); + const blockedReasonByAccount = { + ...(policy.blockedAccountReasons ?? {}), + }; + if (!policy.allowed) { + for (const { index } of accountsWithMetrics) { + blockedAccountIndexes.add(index); + blockedReasonByAccount[index] = + policy.errorCode ?? "policy-blocked"; + } + } + for (const [index, account] of storage.accounts.entries()) { + if ( + typeof account.authInvalidatedAt === "number" && + Number.isFinite(account.authInvalidatedAt) + ) { + blockedAccountIndexes.add(index); + blockedReasonByAccount[index] = AUTH_INVALIDATION_MARKER; + } + } return selectHybridAccountTraced({ accounts: accountsWithMetrics, healthTracker, tokenTracker, + options: { + blockedAccountIndexes, + blockedReasonByAccount, + scoreBoostByAccount: policy.scoreBoostByAccount, + }, }); }; } diff --git a/lib/codex-manager/account-pool-write.ts b/lib/codex-manager/account-pool-write.ts index 40d1b0c93..1ef2b7910 100644 --- a/lib/codex-manager/account-pool-write.ts +++ b/lib/codex-manager/account-pool-write.ts @@ -135,6 +135,8 @@ export function buildInsertedAccount( accessToken: write.accessToken, expiresAt: write.expiresAt, enabled: true, + authInvalidatedAt: undefined, + authInvalidationErrorCode: undefined, addedAt: write.now, lastUsed: write.now, workspaces: write.workspaces, @@ -191,6 +193,8 @@ export function buildUpdatedAccount( accessToken: write.accessToken, expiresAt: write.expiresAt, enabled: true, + authInvalidatedAt: undefined, + authInvalidationErrorCode: undefined, lastUsed: write.now, workspaces: mergedWorkspaces, currentWorkspaceIndex: nextCurrentWorkspaceIndex, diff --git a/lib/codex-manager/commands/status.ts b/lib/codex-manager/commands/status.ts index e6a7ffc9a..9c96a638b 100644 --- a/lib/codex-manager/commands/status.ts +++ b/lib/codex-manager/commands/status.ts @@ -1,4 +1,5 @@ import { + AUTH_INVALIDATION_MARKER, formatAccountLabel, formatCooldown, formatWaitTime, @@ -87,6 +88,12 @@ function buildAccountMarkers( const markers: string[] = []; markers.push(...resolveAccountCurrentMarkers(index, activeIndex, runtimeCurrent)); if (account.enabled === false) markers.push("disabled"); + if ( + typeof account.authInvalidatedAt === "number" && + Number.isFinite(account.authInvalidatedAt) + ) { + markers.push(AUTH_INVALIDATION_MARKER); + } if (formatRateLimitEntry(account, now, "codex")) markers.push("rate-limited"); const quotaEntry = findQuotaCacheEntryForAccount(quotaCache, account, allAccounts); if (quotaEntry?.status === 429 && !markers.some(isRateLimitedMarker)) { diff --git a/lib/codex-manager/commands/uninstall.ts b/lib/codex-manager/commands/uninstall.ts index 039a72abc..15d4d2f74 100644 --- a/lib/codex-manager/commands/uninstall.ts +++ b/lib/codex-manager/commands/uninstall.ts @@ -6,6 +6,8 @@ import { withFileOperationRetry } from "../../fs-retry.js"; import { unbindCodexAppRuntimeRotation } from "../../runtime/app-bind.js"; const PLUGIN_NAME = "codex-multi-auth"; +const LEGACY_PLUGIN_NAME = "@ndycode/codex-multi-auth"; +const PLUGIN_NAMES = [PLUGIN_NAME, LEGACY_PLUGIN_NAME]; export function resolveUninstallPaths( platform: NodeJS.Platform = process.platform, @@ -26,6 +28,12 @@ export function resolveUninstallPaths( return { configPath: join(configDir, "Codex.json"), cacheNodeModules: join(cacheDir, "node_modules", PLUGIN_NAME), + cacheLegacyNodeModules: join( + cacheDir, + "node_modules", + "@ndycode", + "codex-multi-auth", + ), cacheBunLock: join(cacheDir, "bun.lock"), }; } @@ -36,7 +44,9 @@ export function removePluginFromList(list: unknown[]): unknown[] { // stray null entry in Codex.json. return list.filter(Boolean).filter((entry) => { if (typeof entry !== "string") return true; - return entry !== PLUGIN_NAME && !entry.startsWith(`${PLUGIN_NAME}@`); + return !PLUGIN_NAMES.some( + (pluginName) => entry === pluginName || entry.startsWith(`${pluginName}@`), + ); }); } @@ -70,6 +80,7 @@ function printUninstallUsage(): void { "so cleanup must be initiated manually):", " 1. codex-multi-auth uninstall # remove residual artifacts", " 2. npm uninstall -g codex-multi-auth # remove the package itself", + " (@ndycode/codex-multi-auth is the legacy scoped package name.)", ].join("\n"), ); } @@ -276,6 +287,9 @@ export async function runUninstallCommand( try { if (dryRun) { log(`[dry-run] Would remove ${paths.cacheNodeModules}`); + if (paths.cacheLegacyNodeModules) { + log(`[dry-run] Would remove ${paths.cacheLegacyNodeModules}`); + } if (bunLockSafeToRemove) { log(`[dry-run] Would remove ${paths.cacheBunLock}`); } else { @@ -287,6 +301,11 @@ export async function runUninstallCommand( await withFileOperationRetry(() => rm(paths.cacheNodeModules, { recursive: true, force: true }), ); + if (paths.cacheLegacyNodeModules) { + await withFileOperationRetry(() => + rm(paths.cacheLegacyNodeModules, { recursive: true, force: true }), + ); + } if (bunLockSafeToRemove) { await withFileOperationRetry(() => rm(paths.cacheBunLock, { force: true }), @@ -333,6 +352,9 @@ export async function runUninstallCommand( if (warnings.length > 0) { log(`warnings: ${warnings.length} step(s) skipped (see above)`); } + log( + "To remove the installed package itself, run `npm uninstall -g codex-multi-auth`; `@ndycode/codex-multi-auth` is the legacy package name and does not remove the current package.", + ); } return partialFailure ? 1 : 0; diff --git a/lib/codex-manager/commands/why-selected.ts b/lib/codex-manager/commands/why-selected.ts index c13eacb96..b58959683 100644 --- a/lib/codex-manager/commands/why-selected.ts +++ b/lib/codex-manager/commands/why-selected.ts @@ -30,7 +30,7 @@ export interface WhySelectedCommandDeps { resolveActiveIndex: (storage: AccountStorageV3, family?: "codex") => number; selectAccountTraced: ( storage: AccountStorageV3, - ) => HybridSelectionTraceResult; + ) => HybridSelectionTraceResult | Promise; loadRuntimeObservabilitySnapshot?: () => Promise; sanitizeEmail?: (email: string | undefined) => string | undefined; logInfo?: (message: string) => void; @@ -138,7 +138,7 @@ export async function runWhySelectedCommand( return 1; } - const trace = deps.selectAccountTraced(storage); + const trace = await deps.selectAccountTraced(storage); const candidates = trace.candidates.map((candidate) => buildCandidateRecord(storage, candidate, sanitizeEmail), ); diff --git a/lib/codex-manager/health-check.ts b/lib/codex-manager/health-check.ts index c16488db3..143ca4389 100644 --- a/lib/codex-manager/health-check.ts +++ b/lib/codex-manager/health-check.ts @@ -1,4 +1,5 @@ import { + AUTH_INVALIDATION_MARKER, extractAccountEmail, extractAccountId, formatAccountLabel, @@ -41,6 +42,16 @@ import { updateQuotaCacheForAccount, } from "./quota-cache-helpers.js"; +function appendAuthInvalidationMarker( + account: { authInvalidatedAt?: number }, + detail: string, +): string { + return typeof account.authInvalidatedAt === "number" && + Number.isFinite(account.authInvalidatedAt) + ? `${detail} [${AUTH_INVALIDATION_MARKER}]` + : detail; +} + /** * Body of the `check` command, also reused by the login dashboard's quick * check / deep check actions. Moved verbatim out of lib/codex-manager.ts @@ -166,6 +177,7 @@ export async function runHealthCheck( if (hasLikelyInvalidRefreshToken(account.refreshToken)) { healthDetail += " (re-login suggested soon)"; } + healthDetail = appendAuthInvalidationMarker(account, healthDetail); ok += 1; if (display.showPerAccountRows) { const healthMarker = healthTone === "success" ? "✓" : "!"; @@ -208,6 +220,14 @@ export async function runHealthCheck( account.enabled = true; changed = true; } + if ( + typeof account.authInvalidatedAt === "number" || + account.authInvalidationErrorCode !== undefined + ) { + delete account.authInvalidatedAt; + delete account.authInvalidationErrorCode; + changed = true; + } if (accountIdentityChanged && liveProbe && workingQuotaCache) { quotaEmailFallbackState = buildQuotaEmailFallbackState( storage.accounts, @@ -272,6 +292,7 @@ export async function runHealthCheck( } } if (display.showPerAccountRows) { + healthyMessage = appendAuthInvalidationMarker(account, healthyMessage); const healthyMarker = healthyTone === "success" ? "✓" : "!"; console.log( ` ${stylePromptText(healthyMarker, healthyTone)} ${labelText} ${stylePromptText("|", "muted")} ${styleAccountDetailText(healthyMessage, healthyTone)}`, @@ -285,15 +306,20 @@ export async function runHealthCheck( signedInOnly += 1; } if (display.showPerAccountRows) { + const detailWithMarker = appendAuthInvalidationMarker( + account, + `refresh failed (${detail}) but this account still works right now`, + ); console.log( - ` ${stylePromptText("!", "warning")} ${labelText} ${stylePromptText("|", "muted")} ${stylePromptText(`refresh failed (${detail}) but this account still works right now`, "warning")}`, + ` ${stylePromptText("!", "warning")} ${labelText} ${stylePromptText("|", "muted")} ${stylePromptText(detailWithMarker, "warning")}`, ); } } else { failed += 1; if (display.showPerAccountRows) { + const detailWithMarker = appendAuthInvalidationMarker(account, detail); console.log( - ` ${stylePromptText("✗", "danger")} ${labelText} ${stylePromptText("|", "muted")} ${stylePromptText(detail, "danger")}`, + ` ${stylePromptText("✗", "danger")} ${labelText} ${stylePromptText("|", "muted")} ${stylePromptText(detailWithMarker, "danger")}`, ); } } diff --git a/lib/codex-manager/login-oauth.ts b/lib/codex-manager/login-oauth.ts index df7524d0f..099fd4744 100644 --- a/lib/codex-manager/login-oauth.ts +++ b/lib/codex-manager/login-oauth.ts @@ -262,6 +262,7 @@ export async function runOAuthFlow( signInMode: Extract, ): Promise { const { pkce, state, url } = await createAuthorizationFlow({ forceNewLogin }); + const displayUrl = redactOAuthUrlForLog(url); let code: string | null = null; let oauthServer: Awaited> | null = null; @@ -289,13 +290,6 @@ export async function runOAuthFlow( } } - // Display the OAuth URL with sensitive query parameters (state, - // code, code_challenge, code_verifier) redacted so they do not leak - // into shell history, screen captures, CI transcripts, or clipboard - // managers. The full URL is still handed to the browser opener and - // the clipboard so sign-in continues to work end-to-end. - const displayUrl = redactOAuthUrlForLog(url); - if (signInMode === "browser") { const opened = openBrowserUrl(url); if (opened) { @@ -312,10 +306,23 @@ export async function runOAuthFlow( copied ? "success" : "warning", ), ); + if (!copied) { + // The redacted line is safe for normal logs, but it cannot complete + // an incognito/manual handoff. If clipboard access also failed, + // provide the exact URL as the final recovery path. + console.log( + `${stylePromptText(UI_COPY.oauth.goTo, "accent")} ${url}`, + ); + } } } else { + // Manual/incognito sign-in depends on the exact authorization URL. In + // particular, replacing `state` with a redaction marker causes the + // provider to return that marker and the callback's CSRF validation must + // (correctly) reject it. State validation remains strict below; this + // output simply preserves the value minted for this login attempt. console.log( - `${stylePromptText(UI_COPY.oauth.goTo, "accent")} ${displayUrl}`, + `${stylePromptText(UI_COPY.oauth.goTo, "accent")} ${url}`, ); const copied = copyTextToClipboard(url); console.log( diff --git a/lib/forecast.ts b/lib/forecast.ts index cd499f0ab..3e55f3e5f 100644 --- a/lib/forecast.ts +++ b/lib/forecast.ts @@ -1,4 +1,8 @@ -import { formatAccountLabel, formatWaitTime } from "./accounts.js"; +import { + AUTH_INVALIDATION_MARKER, + formatAccountLabel, + formatWaitTime, +} from "./accounts.js"; import type { CodexQuotaSnapshot } from "./quota-probe.js"; import type { QuotaCacheData } from "./quota-cache.js"; import { @@ -204,6 +208,15 @@ export function evaluateForecastAccount( reasons.push("account is disabled"); } + if ( + typeof account.authInvalidatedAt === "number" && + Number.isFinite(account.authInvalidatedAt) + ) { + availability = "unavailable"; + riskScore += 95; + reasons.push(AUTH_INVALIDATION_MARKER); + } + if (input.refreshFailure) { const hard = isHardRefreshFailure(input.refreshFailure); hardFailure = hard; diff --git a/lib/index.ts b/lib/index.ts index d674e8bec..ac1183334 100644 --- a/lib/index.ts +++ b/lib/index.ts @@ -33,6 +33,7 @@ export * from "./request/failure-policy.js"; export * from "./entitlement-cache.js"; export * from "./preemptive-quota-scheduler.js"; export * from "./runtime-rotation-proxy.js"; +export * from "./runtime/app-bind.js"; export * from "./unified-settings.js"; export * from "./capability-policy.js"; export * from "./request/stream-failover.js"; diff --git a/lib/parallel-probe.ts b/lib/parallel-probe.ts index 44ba37556..187a0a865 100644 --- a/lib/parallel-probe.ts +++ b/lib/parallel-probe.ts @@ -142,7 +142,11 @@ export function getTopCandidates( clearExpiredRateLimits(account); const isRateLimited = isRateLimitedForFamily(account, normalizedModelFamily, resolvedModel); const isCoolingDown = account.coolingDownUntil !== undefined && account.coolingDownUntil > Date.now(); - const isAvailable = !isRateLimited && !isCoolingDown; + const isAuthInvalidated = + typeof account.authInvalidatedAt === "number" && + Number.isFinite(account.authInvalidatedAt); + const isAvailable = + !isRateLimited && !isCoolingDown && !isAuthInvalidated; accountsWithMetrics.push({ index: account.index, diff --git a/lib/policy/runtime-policy.ts b/lib/policy/runtime-policy.ts index ae5a64963..3a5a05ce6 100644 --- a/lib/policy/runtime-policy.ts +++ b/lib/policy/runtime-policy.ts @@ -39,6 +39,7 @@ export interface RuntimePolicyDecision { reasons: string[]; projectKey: string | null; blockedAccountIndexes: Set; + blockedAccountReasons?: Record; scoreBoostByAccount: Record; budgetEvaluations: BudgetGuardEvaluation[]; } @@ -147,6 +148,7 @@ export async function evaluateRuntimePolicy(input: { const now = input.now ?? Date.now(); const reasons: string[] = []; const blockedAccountIndexes = new Set(); + const blockedAccountReasons: Record = {}; const scoreBoostByAccount: Record = {}; const profile = input.state.project.profile; @@ -189,9 +191,13 @@ export async function evaluateRuntimePolicy(input: { let boost = 0; if (accountPolicy?.paused) { blockedAccountIndexes.add(account.index); + blockedAccountReasons[account.index] = "policy: paused"; } if (accountPolicy?.drained) { blockedAccountIndexes.add(account.index); + blockedAccountReasons[account.index] = accountPolicy.paused + ? "policy: paused, drained" + : "policy: drained"; } if (accountPolicy) { boost += (accountPolicy.weight - 1) * 2; @@ -228,6 +234,7 @@ export async function evaluateRuntimePolicy(input: { ); if (capabilitySnapshot && capabilitySnapshot.unsupported > 0) { blockedAccountIndexes.add(account.index); + blockedAccountReasons[account.index] = "policy: unsupported model"; } scoreBoostByAccount[account.index] = boost; } @@ -241,6 +248,7 @@ export async function evaluateRuntimePolicy(input: { reasons, projectKey: input.state.project.projectKey, blockedAccountIndexes, + blockedAccountReasons, scoreBoostByAccount, budgetEvaluations, }; diff --git a/lib/preemptive-quota-scheduler.ts b/lib/preemptive-quota-scheduler.ts index 77d59ec63..650f4cd7f 100644 --- a/lib/preemptive-quota-scheduler.ts +++ b/lib/preemptive-quota-scheduler.ts @@ -40,20 +40,23 @@ function trustedResetWaitMs( window: QuotaSchedulerWindow, snapshot: QuotaSchedulerSnapshot, now: number, -): number { +): number | null { const resetAtMs = window.resetAtMs; const updatedAt = snapshot.updatedAt; if ( typeof resetAtMs !== "number" || !Number.isFinite(resetAtMs) || - resetAtMs <= now || typeof updatedAt !== "number" || !Number.isFinite(updatedAt) || updatedAt > now || now - updatedAt > MAX_TRUSTED_RESET_AGE_MS ) { - return 0; + return null; } + // An explicit reset that has already passed is healthy again. Do not turn + // an expired primary window into another fallback deferral merely because a + // secondary window keeps the aggregate snapshot alive. + if (resetAtMs <= now) return 0; return Math.min(resetAtMs - now, MAX_RATE_LIMIT_DELAY_MS); } @@ -316,12 +319,16 @@ export class PreemptiveQuotaScheduler { typeof snapshot.secondary.usedPercent === "number" && Number.isFinite(snapshot.secondary.usedPercent) && snapshot.secondary.usedPercent >= 100 - this.secondaryRemainingPercentThreshold; + const getNearExhaustedWait = (window: QuotaSchedulerWindow): number => { + const trustedWait = trustedResetWaitMs(window, snapshot, now); + return trustedWait === null ? this.maxDeferralMs : trustedWait; + }; const nearExhaustedWait = Math.max( primaryNearExhausted && snapshot.status !== 429 - ? trustedResetWaitMs(snapshot.primary, snapshot, now) || this.maxDeferralMs + ? getNearExhaustedWait(snapshot.primary) : 0, secondaryNearExhausted && snapshot.status !== 429 - ? trustedResetWaitMs(snapshot.secondary, snapshot, now) || this.maxDeferralMs + ? getNearExhaustedWait(snapshot.secondary) : 0, ); if (nearExhaustedWait > 0) { diff --git a/lib/rotation.ts b/lib/rotation.ts index ceb695685..5ecbd779f 100644 --- a/lib/rotation.ts +++ b/lib/rotation.ts @@ -369,6 +369,8 @@ export interface AccountWithMetrics { index: number; trackerKey?: TrackerKey; isAvailable: boolean; + /** Runtime gate that made an unavailable account ineligible, for diagnostics. */ + unavailableReason?: string; lastUsed: number; } @@ -400,6 +402,8 @@ const DEFAULT_HYBRID_SELECTION_CONFIG: HybridSelectionConfig = { export interface HybridSelectionOptions { pidOffsetEnabled?: boolean; scoreBoostByAccount?: Record; + blockedAccountIndexes?: ReadonlySet; + blockedReasonByAccount?: Record; } /** @@ -477,7 +481,10 @@ export function selectHybridAccount( } const cfg = { ...DEFAULT_HYBRID_SELECTION_CONFIG, ...resolvedConfig }; - const available = resolvedAccounts.filter((a) => a.isAvailable); + const blockedAccountIndexes = resolvedOptions.blockedAccountIndexes; + const available = resolvedAccounts.filter( + (a) => a.isAvailable && !blockedAccountIndexes?.has(a.index), + ); // Contract: if NO account is currently available (all cooling down, rate-limited, // circuit-open, or otherwise blocked) the selector returns null rather than @@ -617,7 +624,11 @@ export function selectHybridAccountTraced( const pidBonus = options.pidOffsetEnabled ? (process.pid % 100) * 0.01 : 0; const accounts = Array.isArray(params.accounts) ? params.accounts : []; - const availableAccounts = accounts.filter((account) => account.isAvailable); + const blockedAccountIndexes = options.blockedAccountIndexes; + const availableAccounts = accounts.filter( + (account) => + account.isAvailable && !blockedAccountIndexes?.has(account.index), + ); const candidates: HybridSelectionCandidateTrace[] = accounts.map( (account) => { @@ -645,10 +656,11 @@ export function selectHybridAccountTraced( ((account.index * 0.131 + pidBonus) % 1) * cfg.freshnessWeight * 0.1; } + const blocked = blockedAccountIndexes?.has(account.index) ?? false; return { index: account.index, trackerKey, - isAvailable: account.isAvailable, + isAvailable: account.isAvailable && !blocked, lastUsed: account.lastUsed, health, tokens, @@ -656,9 +668,13 @@ export function selectHybridAccountTraced( capabilityBoost, pidBonus: options.pidOffsetEnabled ? pidBonus : 0, score, - reason: account.isAvailable - ? undefined - : "unavailable (rate-limited, cooling down, or circuit open)", + reason: blocked + ? (options.blockedReasonByAccount?.[account.index] ?? + "policy-blocked") + : account.isAvailable + ? undefined + : (account.unavailableReason ?? + "unavailable (rate-limited, cooling down, or circuit open)"), }; }, ); diff --git a/lib/runtime-constants.ts b/lib/runtime-constants.ts index 2bb1ca4cd..9bf1c850e 100644 --- a/lib/runtime-constants.ts +++ b/lib/runtime-constants.ts @@ -3,3 +3,7 @@ export const RUNTIME_ROTATION_PROXY_PROVIDER_ID = export const APP_RUNTIME_HELPER_STATUS_FILE = "runtime-rotation-app-helper.json" as const; + +/** Immutable launcher metadata used to verify ownership before stopping a helper. */ +export const APP_RUNTIME_HELPER_OWNER_FILE = + "runtime-rotation-app-helper-owner.json" as const; diff --git a/lib/runtime-rotation-proxy.ts b/lib/runtime-rotation-proxy.ts index 018c4abd2..c1df621ce 100644 --- a/lib/runtime-rotation-proxy.ts +++ b/lib/runtime-rotation-proxy.ts @@ -1,4 +1,4 @@ -import { randomUUID, timingSafeEqual } from "node:crypto"; +import { createHash, randomUUID, timingSafeEqual } from "node:crypto"; import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http"; import type { Socket } from "node:net"; import { @@ -20,6 +20,10 @@ import { getTokenInvalidationCooldownMs, getTokenRefreshSkewMs, getPidOffsetEnabled, + getPreemptiveQuotaEnabled, + getPreemptiveQuotaMaxDeferralMs, + getPreemptiveQuotaRemainingPercent5h, + getPreemptiveQuotaRemainingPercent7d, getRoutingMutexMode, getSchedulingStrategy, loadPluginConfig, @@ -45,13 +49,17 @@ import { type RuntimePolicyDecision, } from "./policy/runtime-policy.js"; import { isWorkspaceDisabledError } from "./request/fetch-helpers.js"; +import { + PreemptiveQuotaScheduler, + readQuotaSchedulerSnapshot, +} from "./preemptive-quota-scheduler.js"; import { createLogger, maskString, runWithCorrelationId } from "./logger.js"; import { CodexValidationError } from "./errors.js"; +import { normalizeEmailKey } from "./storage/identity.js"; import { buildPinnedUnavailableErrorBody, buildTokenInvalidationBody, extractErrorCodeFromBody, - getQuotaNearExhaustionWaitMs, isTokenInvalidationError, normalizeExhaustionStatus, parseRetryAfterBodyMs, @@ -159,6 +167,36 @@ function toUrlHost(host: string): string { const proxyLog = createLogger("runtime-proxy"); const DEFAULT_QUOTA_REMAINING_THRESHOLD = 10; +/** @internal Stable identity key for in-memory quota snapshots across reloads. */ +export function buildQuotaScheduleKey( + account: Pick & { + /** Stable per-record discriminator retained across token/account-id updates. */ + addedAt?: number; + recordId?: string; + }, + family: ModelFamily, + model?: string | null, +): string { + const emailKey = normalizeEmailKey(account.email); + const accountId = account.accountId?.trim(); + const refreshToken = account.refreshToken?.trim() ?? ""; + const recordId = account.recordId?.trim(); + const recordDiscriminator = + typeof account.addedAt === "number" && + Number.isFinite(account.addedAt) && + account.addedAt > 0 + ? `:added:${Math.floor(account.addedAt)}` + : ""; + const accountIdentity = recordId + ? `record:${recordId}` + : emailKey + ? `email:${emailKey}${recordDiscriminator}` + : accountId + ? `id:${accountId}${recordDiscriminator}` + : `refresh:${createHash("sha256").update(refreshToken).digest("hex")}${recordDiscriminator}`; + return `account:${accountIdentity}:${model ?? family}`; +} + const DEFAULT_MAX_RUNTIME_ACCOUNT_ATTEMPTS = 4; const MAX_REQUEST_BODY_BYTES = 64 * 1024 * 1024; @@ -701,6 +739,16 @@ export async function startRuntimeRotationProxy( options.maxRequestBodyBytes ?? MAX_REQUEST_BODY_BYTES; const quotaRemainingPercentThreshold = options.quotaRemainingPercentThreshold ?? DEFAULT_QUOTA_REMAINING_THRESHOLD; + const preemptiveQuotaScheduler = new PreemptiveQuotaScheduler({ + enabled: getPreemptiveQuotaEnabled(pluginConfig), + remainingPercentThresholdPrimary: + options.quotaRemainingPercentThreshold ?? + getPreemptiveQuotaRemainingPercent5h(pluginConfig), + remainingPercentThresholdSecondary: + options.quotaRemainingPercentThreshold ?? + getPreemptiveQuotaRemainingPercent7d(pluginConfig), + maxDeferralMs: getPreemptiveQuotaMaxDeferralMs(pluginConfig), + }); const sessionAffinityStore = getSessionAffinity(pluginConfig) ? new SessionAffinityStore({ ttlMs: getSessionAffinityTtlMs(pluginConfig), @@ -731,6 +779,7 @@ export async function startRuntimeRotationProxy( maxRuntimeAccountAttempts, maxRequestBodyBytes, quotaRemainingPercentThreshold, + preemptiveQuotaScheduler, sessionAffinityStore, lastObservedAffinityGeneration, forcedAccountIndex, @@ -1053,6 +1102,33 @@ async function handleRequestInner( break; } attemptedIndexes.add(selected.index); + const quotaScheduleKey = buildQuotaScheduleKey( + selected, + context.family, + context.model, + ); + const preemptiveDeferral = state.preemptiveQuotaScheduler.getDeferral( + quotaScheduleKey, + state.now(), + ); + if (preemptiveDeferral.defer && preemptiveDeferral.waitMs > 0) { + accountSkipReasons.set( + selected.index, + preemptiveDeferral.reason ?? "quota-near-exhaustion", + ); + exhaustionReason = "rate-limit"; + accountManager.markRateLimitedWithReason( + selected, + preemptiveDeferral.waitMs, + context.family, + "quota", + context.model, + ); + accountManager.recordRateLimit(selected, context.family, context.model); + accountManager.saveToDiskDebounced(); + state.status.rotations += 1; + continue; + } if (!accountManager.consumeToken(selected, context.family, context.model)) { accountSkipReasons.set(selected.index, "token-exhausted"); @@ -1165,6 +1241,14 @@ async function handleRequestInner( state.status.rotations += 1; continue; } + const quotaSnapshot = readQuotaSchedulerSnapshot( + upstream.headers, + upstream.status, + state.now(), + ); + if (quotaSnapshot) { + state.preemptiveQuotaScheduler.update(quotaScheduleKey, quotaSnapshot); + } if (upstream.status === HTTP_STATUS.TOO_MANY_REQUESTS) { const bodyText = await readErrorBody(upstream, state.streamStallTimeoutMs); @@ -1172,6 +1256,11 @@ async function handleRequestInner( parseRetryAfterHeaderMs(upstream.headers, state.now()) ?? parseRetryAfterBodyMs(bodyText, state.now()) ?? 60_000; + state.preemptiveQuotaScheduler.markRateLimited( + quotaScheduleKey, + retryAfterMs, + state.now(), + ); // A 429 is the upstream quota signal for the attempted account, so // keep the consumed runtime token drained. accountManager.recordRateLimit(refreshed.account, context.family, context.model); @@ -1298,13 +1387,20 @@ async function handleRequestInner( // account's token from the same IP triggers OpenAI's anti-abuse // detection and invalidates them in sequence. Return the 401 directly // rather than rotating so the client can prompt for re-login. + accountManager.markAuthInvalidated( + refreshed.account, + extractErrorCodeFromBody(bodyText) ?? "token_invalidated", + ); applyMonotonicAuthCooldown( accountManager, refreshed.account, state.tokenInvalidationCooldownMs, ); state.sessionAffinityStore?.forgetSession(context.sessionKey); - accountManager.saveToDiskDebounced(); + // The invalidation marker must be durable before returning the 401. + // A delayed save would allow a proxy restart to reload and route the + // revoked account again. + await accountManager.saveToDisk(); // Emit the same machine-readable shape as the refresh-failure path // (code: "token_invalidated") instead of forwarding the raw upstream // body, so the client contract is consistent across both vectors. @@ -1387,11 +1483,13 @@ async function handleRequestInner( // the forecast keeps reporting this working account as unavailable. // No-op when no reason is recorded, so the hot path stays write-free. recordRuntimeAccountRecovery(refreshed.account.index); - const nearExhaustionWaitMs = getQuotaNearExhaustionWaitMs( - upstream.headers, - state.quotaRemainingPercentThreshold, + const quotaDeferral = state.preemptiveQuotaScheduler.getDeferral( + quotaScheduleKey, state.now(), ); + const nearExhaustionWaitMs = quotaDeferral.defer + ? quotaDeferral.waitMs + : 0; if (nearExhaustionWaitMs > 0) { accountManager.markRateLimitedWithReason( refreshed.account, diff --git a/lib/runtime/app-bind.ts b/lib/runtime/app-bind.ts index d8d2c58da..3a78ecc3c 100644 --- a/lib/runtime/app-bind.ts +++ b/lib/runtime/app-bind.ts @@ -1,13 +1,17 @@ import { spawn } from "node:child_process"; import { createHash, randomBytes } from "node:crypto"; import { closeSync, existsSync, mkdirSync, openSync } from "node:fs"; -import { mkdir, open, readFile, rename, unlink } from "node:fs/promises"; +import { mkdir, open, readFile, rename, rm, unlink } from "node:fs/promises"; import { homedir } from "node:os"; import { basename, dirname, join } from "node:path"; import process from "node:process"; import { fileURLToPath } from "node:url"; import { withFileOperationRetry } from "../fs-retry.js"; import { getCodexMultiAuthDir } from "../runtime-paths.js"; +import { + APP_RUNTIME_HELPER_OWNER_FILE, + APP_RUNTIME_HELPER_STATUS_FILE, +} from "../runtime-constants.js"; import { configHasRuntimeRotationProvider, restoreConfigTomlFromRuntimeRotationProvider, @@ -18,6 +22,12 @@ import { const APP_BIND_DIR_NAME = "app-bind"; const APP_BIND_STATE_FILE = "runtime-rotation-app-bind.json"; const APP_BIND_BACKUP_FILE = "codex-config-backup.json"; +const RUNTIME_ROTATION_APP_HELPER_ARG = + "--codex-multi-auth-runtime-app-helper"; +const PROCESS_IDENTITY_PROBE_TIMEOUT_MS = 2_000; +const WINDOWS_PROCESS_IDENTITY_PROBE_TIMEOUT_MS = 5_000; +const PROCESS_START_TIME_TOLERANCE_MS = 5_000; +const WINDOWS_TASKKILL_TIMEOUT_MS = 5_000; const APP_BIND_STATUS_FILE = "runtime-rotation-app-bind-status.json"; const WINDOWS_STARTUP_FILE = "Codex Multi Auth Runtime Router.cmd"; const MACOS_LAUNCH_AGENT_ID = "com.ndycode.codex-multi-auth.runtime-router"; @@ -61,6 +71,8 @@ export interface AppBindState { nodePath: string; routerScriptPath: string; clientApiKey: string; + /** Per-bind nonce passed to the router command line for PID ownership checks. */ + identityToken?: string; startupPath: string | null; launchAgentPath: string | null; boundConfigHash: string; @@ -70,6 +82,13 @@ export interface AppBindState { export interface AppBindRouterStatus { state: string | null; pid: number | null; + startedAt?: number | null; + /** Internal path of the status file used to verify the owning command line. */ + statusPath?: string | null; + /** Full router script path persisted by newer router processes. */ + routerScriptPath?: string | null; + /** Per-process nonce echoed by the router command line. */ + identityToken?: string | null; baseUrl: string | null; totalRequests: number | null; lastAccountIndex: number | null; @@ -100,6 +119,13 @@ export interface AppBindResult { message: string; } +export type ProcessIdentityVerifier = ( + pid: number, + startedAt: number, + platform: NodeJS.Platform, + identityToken?: string, +) => boolean | Promise; + export interface AppBindOptions { env?: NodeJS.ProcessEnv; platform?: NodeJS.Platform; @@ -111,6 +137,36 @@ export interface AppBindOptions { spawnDetached?: boolean; routerReadyTimeoutMs?: number; log?: (message: string) => void; + /** Test/integration seam for deterministic ownership verification. */ + verifyProcessIdentity?: ProcessIdentityVerifier; +} + +export interface DetachedProcessStopOptions { + gracefulTimeoutMs?: number; + pollIntervalMs?: number; + isAlive?: (pid: number) => boolean; + kill?: (pid: number, signal: NodeJS.Signals) => void; + runWindowsTaskkill?: (pid: number) => Promise; + log?: (message: string) => void; + /** Expected per-process nonce when verifying a persisted PID. */ + identityToken?: string; + verifyProcessIdentity?: ProcessIdentityVerifier; +} + +export interface RuntimeRotationAppHelperStatus { + state: string | null; + kind: string | null; + pid: number | null; + startedAt: number | null; + /** Full wrapper script path persisted by newer helper processes. */ + scriptPath?: string | null; + /** Per-process nonce echoed by the helper command line. */ + identityToken?: string | null; +} + +interface RuntimeRotationAppHelperOwner { + kind: string; + identityToken: string; } // Per-key mutex. `tail` resolves only after `current` resolves, so each @@ -264,6 +320,7 @@ function readAppBindStateRecord(record: Record): AppBindState | const nodePath = readString(record, "nodePath"); const routerScriptPath = readString(record, "routerScriptPath"); const clientApiKey = readString(record, "clientApiKey"); + const identityToken = readString(record, "identityToken"); const boundConfigHash = readString(record, "boundConfigHash"); const updatedAt = readNumber(record, "updatedAt"); const platformValue = readString(record, "platform"); @@ -298,6 +355,7 @@ function readAppBindStateRecord(record: Record): AppBindState | nodePath, routerScriptPath, clientApiKey, + identityToken: identityToken ?? undefined, startupPath: readString(record, "startupPath"), launchAgentPath: readString(record, "launchAgentPath"), boundConfigHash, @@ -341,6 +399,10 @@ async function readRouterStatus(path: string): Promise> "${logPath}" 2>&1`, + `"${nodePath}" "${routerScriptPath}" --port ${state.port} --status "${statusPath}" --identity-token "${state.identityToken ?? ""}" --state "${statePath}" --log "${logPath}" --max-log-bytes ${APP_ROUTER_MAX_LOG_BYTES} >> "${logPath}" 2>&1`, "", ].join("\r\n"); } @@ -473,6 +535,8 @@ function createMacLaunchAgentPlist(state: AppBindState): string { String(state.port), "--status", state.statusPath, + "--identity-token", + state.identityToken ?? "", "--state", state.statePath, "--log", @@ -517,7 +581,9 @@ async function writeAppBindStartup(state: AppBindState): Promise { } } -async function removeAppBindStartup(state: AppBindState): Promise { +async function removeAppBindStartup( + state: Pick, +): Promise { const candidates = [state.startupPath, state.launchAgentPath].filter( (path): path is string => typeof path === "string" && path.length > 0, ); @@ -542,6 +608,8 @@ function spawnRouter(state: AppBindState): void { String(state.port), "--status", state.statusPath, + "--identity-token", + state.identityToken ?? "", "--state", state.statePath, "--log", @@ -596,17 +664,651 @@ async function waitForRouterStatus( throw new Error(`Codex app runtime router did not report ready${suffix}`); } -async function stopRouter(router: AppBindRouterStatus | null): Promise { - if (!router?.pid || !isProcessAlive(router.pid)) return; +export async function stopRuntimeRotationRouterProcess( + router: Pick< + AppBindRouterStatus, + | "state" + | "pid" + | "startedAt" + | "updatedAt" + | "statusPath" + | "routerScriptPath" + | "identityToken" + > | null, + platform: NodeJS.Platform, + routerScriptPath: string, + options: DetachedProcessStopOptions = {}, +): Promise { + if (router?.state !== "running" || router.pid === null) { + return false; + } + if ( + router.routerScriptPath && + normalizeProcessIdentityPath(router.routerScriptPath, platform) !== + normalizeProcessIdentityPath(routerScriptPath, platform) + ) { + return false; + } + const startedAt = + typeof router.startedAt === "number" && + Number.isFinite(router.startedAt) && + router.startedAt > 0 + ? router.startedAt + : null; + const updatedAt = + typeof router.updatedAt === "number" && + Number.isFinite(router.updatedAt) && + router.updatedAt > 0 + ? router.updatedAt + : null; + const identityTimestamp = startedAt ?? updatedAt; + if (identityTimestamp === null) return false; + const expectedIdentityToken = options.identityToken; + if (router.identityToken && !expectedIdentityToken) { + // A tokenized status record must be paired with the trusted token from + // bind state. The status file alone is mutable by any replacement process. + return false; + } + const verifyProcessIdentity = options.verifyProcessIdentity; + let verified: boolean; + if (verifyProcessIdentity) { + verified = expectedIdentityToken + ? await verifyProcessIdentity( + router.pid, + identityTimestamp, + platform, + expectedIdentityToken, + ) + : await verifyProcessIdentity(router.pid, identityTimestamp, platform); + } else if (startedAt !== null) { + verified = await verifyRuntimeProcessIdentity( + router.pid, + startedAt, + platform, + routerScriptPath, + router.statusPath, + options.log, + expectedIdentityToken, + ); + } else { + verified = await verifyLegacyRuntimeProcessIdentity( + router.pid, + updatedAt as number, + platform, + routerScriptPath, + router.statusPath, + options.log, + expectedIdentityToken, + ); + } + if (!verified) { + return false; + } + return stopDetachedProcess(router.pid, platform, options); +} + +async function stopRouter( + router: AppBindRouterStatus | null, + platform: NodeJS.Platform, + routerScriptPath: string, + options: DetachedProcessStopOptions = {}, +): Promise { + return stopRuntimeRotationRouterProcess(router, platform, routerScriptPath, options); +} + +async function runWindowsTaskkill(pid: number): Promise { + return new Promise((resolve) => { + let settled = false; + let child: ReturnType | null = null; + let timeout: ReturnType | null = null; + const finish = (succeeded: boolean) => { + if (settled) return; + settled = true; + if (timeout) clearTimeout(timeout); + resolve(succeeded); + }; + try { + child = spawn("taskkill", ["/PID", String(pid), "/T", "/F"], { + stdio: "ignore", + windowsHide: true, + }); + timeout = setTimeout(() => { + try { + child?.kill(); + } catch { + // The taskkill child may already have exited at the timeout boundary. + } + finish(false); + }, WINDOWS_TASKKILL_TIMEOUT_MS); + child.once("error", () => finish(false)); + child.once("close", (code) => finish(code === 0)); + } catch { + finish(false); + } + }); +} + +interface ProcessIdentitySnapshot { + startedAt: number | null; + commandLine: string; +} + +async function runProcessIdentityProbe( + command: string, + args: string[], + options: { timeoutMs?: number; log?: (message: string) => void } = {}, +): Promise { + return new Promise((resolve) => { + let output = ""; + let settled = false; + let child: ReturnType | null = null; + const timeoutMs = + typeof options.timeoutMs === "number" && + Number.isFinite(options.timeoutMs) && + options.timeoutMs > 0 + ? options.timeoutMs + : PROCESS_IDENTITY_PROBE_TIMEOUT_MS; + const finish = (value: string | null) => { + if (settled) return; + settled = true; + clearTimeout(timeout); + resolve(value); + }; + const timeout = setTimeout(() => { + options.log?.(`Process identity probe timed out while running ${command}`); + try { + child?.kill(); + } catch { + // The probe may have exited at the timeout boundary. + } + finish(null); + }, timeoutMs); + try { + child = spawn(command, args, { + stdio: ["ignore", "pipe", "ignore"], + windowsHide: true, + }); + child.stdout?.setEncoding("utf8"); + child.stdout?.on("data", (chunk: string) => { + output += chunk; + }); + child.once("error", (error) => { + options.log?.( + `Process identity probe ${command} was unavailable: ${error instanceof Error ? error.message : String(error)}`, + ); + finish(null); + }); + child.once("close", (code) => { + if (code !== 0) { + options.log?.( + `Process identity probe ${command} exited with status ${code ?? "unknown"}`, + ); + } + finish(code === 0 ? output : null); + }); + } catch (error) { + options.log?.( + `Process identity probe ${command} was unavailable: ${error instanceof Error ? error.message : String(error)}`, + ); + finish(null); + } + }); +} + +async function readProcessIdentity( + pid: number, + platform: NodeJS.Platform, + log?: (message: string) => void, +): Promise { + if (platform === "win32") { + const script = [ + `$process = Get-CimInstance Win32_Process -Filter 'ProcessId = ${pid}'`, + "if ($null -ne $process) {", + "[Console]::WriteLine($process.CreationDate.ToUniversalTime().ToString('o'))", + "[Console]::WriteLine($process.CommandLine)", + "}", + ].join("; "); + const output = await runProcessIdentityProbe( + "powershell.exe", + ["-NoProfile", "-NonInteractive", "-Command", script], + { timeoutMs: WINDOWS_PROCESS_IDENTITY_PROBE_TIMEOUT_MS, log }, + ); + if (!output) { + log?.(`Process identity probe returned no Windows record for PID ${pid}`); + return null; + } + const [startedAtRaw, ...commandLines] = output.trim().split(/\r?\n/); + const startedAt = startedAtRaw ? Date.parse(startedAtRaw.trim()) : NaN; + const commandLine = commandLines.join(" ").trim(); + if (!Number.isFinite(startedAt) || commandLine.length === 0) { + log?.(`Process identity probe returned an invalid Windows record for PID ${pid}`); + } + return { + startedAt: Number.isFinite(startedAt) ? startedAt : null, + commandLine, + }; + } + + const [startedAtRaw, commandLineRaw] = await Promise.all([ + runProcessIdentityProbe( + "ps", + ["-p", String(pid), "-o", "lstart="], + { log }, + ), + runProcessIdentityProbe( + "ps", + ["-p", String(pid), "-o", "command="], + { log }, + ), + ]); + if (!startedAtRaw || !commandLineRaw) { + log?.(`Process identity probe returned no POSIX record for PID ${pid}`); + return null; + } + const startedAt = parsePosixProcessStartTime(startedAtRaw); + if (startedAt === null) { + log?.(`Process identity probe returned an invalid POSIX start time for PID ${pid}`); + } + return { + startedAt, + commandLine: commandLineRaw.trim(), + }; +} + +const POSIX_LSTART_MONTHS = new Map([ + ["Jan", 0], + ["Feb", 1], + ["Mar", 2], + ["Apr", 3], + ["May", 4], + ["Jun", 5], + ["Jul", 6], + ["Aug", 7], + ["Sep", 8], + ["Oct", 9], + ["Nov", 10], + ["Dec", 11], +]); + +/** Parse the documented local-time format emitted by `ps -o lstart=`. */ +export function parsePosixProcessStartTime(value: string): number | null { + const match = + /^\s*[A-Za-z]{3}\s+([A-Za-z]{3})\s+(\d{1,2})\s+(\d{2}):(\d{2}):(\d{2})\s+(\d{4})\s*$/.exec( + value, + ); + if (!match) return null; + const month = POSIX_LSTART_MONTHS.get(match[1] ?? ""); + const day = Number(match[2]); + const hour = Number(match[3]); + const minute = Number(match[4]); + const second = Number(match[5]); + const year = Number(match[6]); + if ( + month === undefined || + !Number.isInteger(day) || + !Number.isInteger(hour) || + !Number.isInteger(minute) || + !Number.isInteger(second) || + !Number.isInteger(year) || + day < 1 || + day > 31 || + hour > 23 || + minute > 59 || + second > 59 + ) { + return null; + } + const date = new Date(year, month, day, hour, minute, second, 0); + if ( + !Number.isFinite(date.getTime()) || + date.getFullYear() !== year || + date.getMonth() !== month || + date.getDate() !== day || + date.getHours() !== hour || + date.getMinutes() !== minute || + date.getSeconds() !== second + ) { + return null; + } + return date.getTime(); +} + +function normalizeProcessIdentityPath(value: string, platform: NodeJS.Platform): string { + const normalized = value.trim().replace(/["']/g, "").replace(/\\/g, "/"); + return platform === "win32" ? normalized.toLowerCase() : normalized; +} + +function commandLineContainsProcessPath( + commandLine: string, + path: string, + platform: NodeJS.Platform, +): boolean { + const normalizedPath = normalizeProcessIdentityPath(path, platform); + if (normalizedPath.length === 0) return false; + const normalizedCommandLine = normalizeProcessIdentityPath(commandLine, platform); + let offset = normalizedCommandLine.indexOf(normalizedPath); + while (offset >= 0) { + const before = normalizedCommandLine[offset - 1] ?? ""; + const after = + normalizedCommandLine[offset + normalizedPath.length] ?? ""; + if ((before === "" || /\s/.test(before)) && (after === "" || /\s/.test(after))) { + return true; + } + offset = normalizedCommandLine.indexOf(normalizedPath, offset + 1); + } + return false; +} + +async function verifyRuntimeProcessIdentity( + pid: number, + startedAt: number, + platform: NodeJS.Platform, + expectedScriptPath: string, + expectedStatusPath?: string | null, + log?: (message: string) => void, + expectedIdentityToken?: string, +): Promise { + const identity = await readProcessIdentity(pid, platform, log); + if (!identity || identity.startedAt === null) return false; + if ( + !commandLineContainsProcessPath( + identity.commandLine, + expectedScriptPath, + platform, + ) || + (expectedStatusPath !== undefined && + expectedStatusPath !== null && + !commandLineContainsProcessPath( + identity.commandLine, + expectedStatusPath, + platform, + )) + ) { + return false; + } + if ( + expectedIdentityToken && + !commandLineContainsProcessPath( + identity.commandLine, + expectedIdentityToken, + platform, + ) + ) { + log?.("Process identity probe rejected a mismatched ownership token"); + return false; + } + return Math.abs(identity.startedAt - startedAt) <= PROCESS_START_TIME_TOLERANCE_MS; +} + +async function verifyLegacyRuntimeProcessIdentity( + pid: number, + lastObservedAt: number, + platform: NodeJS.Platform, + expectedScriptPath: string, + expectedStatusPath?: string | null, + log?: (message: string) => void, + expectedIdentityToken?: string, +): Promise { + if (lastObservedAt > Date.now() + PROCESS_START_TIME_TOLERANCE_MS) { + return false; + } + const identity = await readProcessIdentity(pid, platform, log); + if (!identity || identity.startedAt === null) return false; + if ( + !commandLineContainsProcessPath( + identity.commandLine, + expectedScriptPath, + platform, + ) || + (expectedStatusPath !== undefined && + expectedStatusPath !== null && + !commandLineContainsProcessPath( + identity.commandLine, + expectedStatusPath, + platform, + )) + ) { + return false; + } + if ( + expectedIdentityToken && + !commandLineContainsProcessPath( + identity.commandLine, + expectedIdentityToken, + platform, + ) + ) { + log?.("Process identity probe rejected a mismatched ownership token"); + return false; + } + // Legacy router status did not persist its own start time. A status update + // is still written after the router process starts, so a process with a + // later creation time indicates a stale/reused PID and must not be signalled. + return identity.startedAt <= lastObservedAt + PROCESS_START_TIME_TOLERANCE_MS; +} + +function isIgnorableProcessSignalError(error: unknown): boolean { + const code = + error && typeof error === "object" && "code" in error + ? error.code + : undefined; + return code === "ESRCH" || code === "EPERM"; +} + +function reportProcessStopError( + options: DetachedProcessStopOptions, + operation: string, + error: unknown, +): void { + options.log?.( + `Failed to ${operation}: ${error instanceof Error ? error.message : String(error)}`, + ); +} + +function resolveStopTimeout(value: number | undefined, fallback: number): number { + return typeof value === "number" && Number.isFinite(value) && value >= 0 + ? Math.floor(value) + : fallback; +} + +async function waitForDetachedProcessExit( + pid: number, + options: DetachedProcessStopOptions, +): Promise { + const isAlive = options.isAlive ?? isProcessAlive; + const timeoutMs = resolveStopTimeout(options.gracefulTimeoutMs, 2_000); + const pollIntervalMs = Math.max( + 1, + resolveStopTimeout(options.pollIntervalMs, 100), + ); + const deadline = Date.now() + timeoutMs; + while (isAlive(pid)) { + if (Date.now() >= deadline) return false; + await new Promise((resolve) => + setTimeout( + resolve, + Math.min(pollIntervalMs, Math.max(1, deadline - Date.now())), + ), + ); + } + return true; +} + +export async function stopDetachedProcess( + pid: number | null, + platform: NodeJS.Platform, + options: DetachedProcessStopOptions = {}, +): Promise { + if (!pid || !Number.isInteger(pid) || pid < 1) return false; + const isAlive = options.isAlive ?? isProcessAlive; + const kill = + options.kill ?? + ((target: number, signal: NodeJS.Signals) => { + process.kill(target, signal); + }); + const taskkill = options.runWindowsTaskkill ?? runWindowsTaskkill; + if (!isAlive(pid)) return true; + + if (platform === "win32") { + // Node's SIGTERM emulation can let a target exit before the tree-kill + // fallback runs. Kill the exact PID tree directly so detached descendants + // cannot survive a graceful wait and so no unrelated process is touched. + try { + const result = await taskkill(pid); + if (result !== false || !isAlive(pid)) return true; + reportProcessStopError( + options, + "terminate the Windows process tree", + new Error("taskkill reported failure while the process is still alive"), + ); + return false; + } catch (error) { + if (!isAlive(pid)) return true; + reportProcessStopError(options, "terminate the Windows process tree", error); + return false; + } + } + + try { + kill(pid, "SIGTERM"); + } catch (error) { + // ESRCH means the process exited between the liveness probe and signal; + // EPERM means it is not ours or cannot be signalled. Do not escalate an + // unexpected signal error or silently claim that cleanup succeeded. + if (isIgnorableProcessSignalError(error)) return !isAlive(pid); + reportProcessStopError(options, "send SIGTERM", error); + return false; + } + + if (await waitForDetachedProcessExit(pid, options)) return true; + + // A detached POSIX helper/router may ignore SIGTERM. Escalate once after the + // bounded graceful window; a second bounded wait prevents unbind from racing + // the process while it is still unwinding. try { - process.kill(router.pid, "SIGTERM"); + kill(pid, "SIGKILL"); + } catch (error) { + if (isIgnorableProcessSignalError(error)) return !isAlive(pid); + reportProcessStopError(options, "send SIGKILL", error); + return false; + } + return waitForDetachedProcessExit(pid, { + ...options, + gracefulTimeoutMs: Math.min( + resolveStopTimeout(options.gracefulTimeoutMs, 2_000), + 500, + ), + }); +} + +async function readRuntimeHelperStatus( + path: string, +): Promise< + | { kind: "missing" | "malformed" | "unreadable"; status: null } + | { kind: "valid"; status: RuntimeRotationAppHelperStatus } +> { + let raw: string; + try { + raw = await readFile(path, "utf8"); + } catch (error) { + const code = + error && typeof error === "object" && "code" in error + ? error.code + : undefined; + return { + kind: code === "ENOENT" ? "missing" : "unreadable", + status: null, + }; + } + const record = parseJsonRecord(raw); + if (!record) return { kind: "malformed", status: null }; + const pid = record ? readNumber(record, "pid") : null; + const startedAt = record ? readNumber(record, "startedAt") : null; + return { + kind: "valid", + status: { + state: readString(record, "state"), + kind: readString(record, "kind"), + pid: pid !== null && Number.isInteger(pid) && pid > 0 ? pid : null, + startedAt: + startedAt !== null && Number.isFinite(startedAt) && startedAt > 0 + ? startedAt + : null, + scriptPath: readString(record, "scriptPath"), + identityToken: readString(record, "identityToken"), + }, + }; +} + +async function readRuntimeHelperOwner( + path: string, +): Promise { + try { + const record = parseJsonRecord(await readFile(path, "utf8")); + const kind = record ? readString(record, "kind") : null; + const identityToken = record ? readString(record, "identityToken") : null; + return kind === "codex-app-runtime-rotation-helper-owner" && identityToken + ? { kind, identityToken } + : null; } catch { - return; + return null; } - for (let attempt = 0; attempt < 20; attempt += 1) { - if (!isProcessAlive(router.pid)) return; - await new Promise((resolve) => setTimeout(resolve, 100)); +} + +function resolveRuntimeHelperOwnerPath( + baseDir: string, + pid: number | null, +): string | null { + if (pid === null || !Number.isInteger(pid) || pid < 1) return null; + return join( + baseDir, + APP_RUNTIME_HELPER_OWNER_FILE.replace(/\.json$/i, `.${pid}.json`), + ); +} + +export async function stopRuntimeRotationAppHelperProcess( + helper: RuntimeRotationAppHelperStatus, + options: DetachedProcessStopOptions & { platform?: NodeJS.Platform } = {}, +): Promise { + if ( + helper.kind !== "codex-app-runtime-rotation-helper" || + helper.state !== "running" || + helper.pid === null || + helper.startedAt === null + ) { + return false; + } + const platform = options.platform ?? process.platform; + const expectedIdentityToken = options.identityToken; + if (helper.identityToken && !expectedIdentityToken) { + // Do not trust a token that was read from the same mutable status file + // whose PID is about to be signalled. + return false; } + const verifyProcessIdentity = + options.verifyProcessIdentity ?? + ((pid, startedAt, processPlatform) => + verifyRuntimeProcessIdentity( + pid, + startedAt, + processPlatform, + RUNTIME_ROTATION_APP_HELPER_ARG, + helper.scriptPath, + options.log, + expectedIdentityToken, + )); + const verified = expectedIdentityToken + ? await verifyProcessIdentity( + helper.pid, + helper.startedAt, + platform, + expectedIdentityToken, + ) + : await verifyProcessIdentity(helper.pid, helper.startedAt, platform); + if (!verified) { + return false; + } + return stopDetachedProcess(helper.pid, platform, options); } async function readConfigIfExists(configPath: string): Promise<{ existed: boolean; content: string }> { @@ -686,6 +1388,7 @@ async function bindCodexAppRuntimeRotationLocked( nodePath: options.nodePath ?? process.execPath, routerScriptPath: paths.routerScriptPath, clientApiKey, + identityToken: existingState?.identityToken ?? randomBytes(24).toString("hex"), startupPath: paths.startupPath, launchAgentPath: paths.launchAgentPath, boundConfigHash: sha256(boundConfig), @@ -731,7 +1434,13 @@ async function bindCodexAppRuntimeRotationLocked( if (startedRouter) { // Best-effort stop of the router we just spawned const orphan = await readRouterStatus(state.statusPath).catch(() => null); - await stopRouter(orphan).catch(() => undefined); + await stopRouter(orphan, platform, state.routerScriptPath, { + log: options.log, + identityToken: state.identityToken, + verifyProcessIdentity: options.verifyProcessIdentity, + }).catch( + () => undefined, + ); } throw new Error( "Codex app bind could not resolve a runtime router port; refusing to write config.toml with port=0.", @@ -773,15 +1482,94 @@ async function unbindCodexAppRuntimeRotationLocked( ): Promise { const state = await readAppBindState(paths.statePath); const router = await readRouterStatus(paths.statusPath); - if (state) { - await stopRouter(router); - if (router?.pid && isProcessAlive(router.pid)) { - options.log?.( - `Warning: runtime router (pid ${router.pid}) did not stop; continuing cleanup`, + const platform = options.platform ?? process.platform; + const routerStopped = await stopRouter( + router, + platform, + state?.routerScriptPath ?? paths.routerScriptPath, + { + log: options.log, + identityToken: state?.identityToken, + verifyProcessIdentity: options.verifyProcessIdentity, + }, + ); + if (router?.pid && (!routerStopped || isProcessAlive(router.pid))) { + options.log?.( + `Warning: runtime router (pid ${router.pid}) did not stop; continuing cleanup`, + ); + } + + const helperStatusPath = join( + dirname(paths.bindDir), + APP_RUNTIME_HELPER_STATUS_FILE, + ); + const helperRead = await readRuntimeHelperStatus(helperStatusPath); + let removeHelperStatus = false; + let removeHelperOwner = false; + let helperOwnerPath: string | null = null; + if (helperRead.kind === "valid") { + const helper = helperRead.status; + if (helper.kind === "codex-app-runtime-rotation-helper") { + helperOwnerPath = resolveRuntimeHelperOwnerPath( + dirname(paths.bindDir), + helper.pid, ); + const helperOwner = helperOwnerPath + ? await readRuntimeHelperOwner(helperOwnerPath) + : null; + const helperOwnershipMatches = + !helper.identityToken || + (helperOwner !== null && + helper.identityToken === helperOwner.identityToken); + if (helper.state === "running") { + if (helper.pid === null) { + options.log?.( + "Warning: runtime app helper status has no valid PID; preserving status", + ); + } else { + const wasAlive = isProcessAlive(helper.pid); + if (!wasAlive) { + removeHelperStatus = true; + removeHelperOwner = + helperOwnershipMatches && helperOwnerPath !== null; + } else if (!helperOwnershipMatches) { + options.log?.( + "Warning: runtime app helper ownership metadata does not match; preserving status", + ); + } else { + const stopped = await stopRuntimeRotationAppHelperProcess(helper, { + platform, + log: options.log, + identityToken: helper.identityToken + ? helperOwner?.identityToken + : undefined, + verifyProcessIdentity: options.verifyProcessIdentity, + }); + const stillAlive = isProcessAlive(helper.pid); + removeHelperStatus = stopped && !stillAlive; + removeHelperOwner = + removeHelperStatus && helperOwnerPath !== null; + if (!removeHelperStatus) { + options.log?.( + `Warning: runtime app helper (pid ${helper.pid}) did not stop; preserving status`, + ); + } + } + } + } else { + // A non-running, owned record is removable only when its PID is + // absent or no longer alive. This avoids deleting a status file + // while a helper is still serving despite a stale state value. + removeHelperStatus = + helper.pid === null || !isProcessAlive(helper.pid); + removeHelperOwner = + removeHelperStatus && + helperOwnershipMatches && + helperOwnerPath !== null; + } } - await removeAppBindStartup(state); } + await removeAppBindStartup(state ?? paths); const backup = await readAppBindBackup(paths.backupPath); let selfHealed = false; @@ -824,17 +1612,28 @@ async function unbindCodexAppRuntimeRotationLocked( } } - for (const candidate of [ + const cleanupCandidates = [ paths.statePath, paths.backupPath, paths.statusPath, - ]) { + state?.logPath ?? paths.logPath, + ...(removeHelperStatus ? [helperStatusPath] : []), + ...(removeHelperOwner && helperOwnerPath ? [helperOwnerPath] : []), + ]; + for (const candidate of cleanupCandidates) { try { await unlinkIfExists(candidate); } catch { // Best-effort cleanup. } } + try { + await withFileOperationRetry(() => + rm(paths.bindDir, { force: true, recursive: true }), + ); + } catch { + // The bind directory may still contain unrelated or locked files. + } const status = await getAppBindStatus(options); let message: string; diff --git a/lib/runtime/rotation-account-selection.ts b/lib/runtime/rotation-account-selection.ts index d1124cc7c..83ec907d0 100644 --- a/lib/runtime/rotation-account-selection.ts +++ b/lib/runtime/rotation-account-selection.ts @@ -158,6 +158,8 @@ export function chooseAccount(params: { ...(policy?.scoreBoostByAccount ?? {}), ...(stickyBoostByAccount ?? {}), }, + blockedAccountIndexes: policy?.blockedAccountIndexes, + blockedReasonByAccount: policy?.blockedAccountReasons, // accounts-05: carry the PID-offset distribution into the default-on proxy // path too (index.ts already does). Without it, parallel proxy processes can // stampede the same account instead of spreading across the pool. diff --git a/lib/runtime/rotation-proxy-state.ts b/lib/runtime/rotation-proxy-state.ts index b9897afa0..8164745b1 100644 --- a/lib/runtime/rotation-proxy-state.ts +++ b/lib/runtime/rotation-proxy-state.ts @@ -1,4 +1,5 @@ import { AccountManager } from "../accounts.js"; +import type { PreemptiveQuotaScheduler } from "../preemptive-quota-scheduler.js"; import { recordRuntimeReload, recordRuntimeReset, @@ -32,6 +33,7 @@ export interface RotationProxyStateInit { maxRuntimeAccountAttempts: number; maxRequestBodyBytes: number; quotaRemainingPercentThreshold: number; + preemptiveQuotaScheduler: PreemptiveQuotaScheduler; sessionAffinityStore: SessionAffinityStore | null; lastObservedAffinityGeneration: number; /** diff --git a/lib/runtime/rotation-token-refresh.ts b/lib/runtime/rotation-token-refresh.ts index 28ce7d2d5..7415e022b 100644 --- a/lib/runtime/rotation-token-refresh.ts +++ b/lib/runtime/rotation-token-refresh.ts @@ -102,6 +102,9 @@ export async function ensureFreshAccessToken(params: { // the long cooldown and signal to the caller to stop rotating rather than // presenting other accounts' tokens from the same IP. const invalidated = isTokenInvalidationError(refreshResult.message ?? ""); + if (invalidated) { + accountManager.markAuthInvalidated(account); + } applyMonotonicAuthCooldown( accountManager, account, diff --git a/lib/schemas.ts b/lib/schemas.ts index bbb6f4acc..98b113f83 100644 --- a/lib/schemas.ts +++ b/lib/schemas.ts @@ -161,6 +161,7 @@ const WorkspaceSchema = z.object({ * Account metadata V3 - current storage format. */ export const AccountMetadataV3Schema = z.object({ + recordId: z.string().min(1).optional(), accountId: z.string().optional(), accountIdSource: AccountIdSourceSchema.optional(), accountLabel: z.string().optional(), @@ -175,6 +176,8 @@ export const AccountMetadataV3Schema = z.object({ rateLimitResetTimes: RateLimitStateV3Schema.optional(), coolingDownUntil: z.number().optional(), cooldownReason: CooldownReasonSchema.optional(), + authInvalidatedAt: z.number().finite().positive().optional(), + authInvalidationErrorCode: z.string().min(1).optional(), // Multi-workspace support (#491): without these here, the strict z.object // strips workspace tracking on every load, so login-captured workspaces // silently vanish after one read/write round-trip. diff --git a/lib/storage.ts b/lib/storage.ts index 82ed2c7ff..fc2295e3d 100644 --- a/lib/storage.ts +++ b/lib/storage.ts @@ -108,6 +108,7 @@ import { import { getNamedBackupsEntry } from "./storage/named-backups-entry.js"; import { getStoragePathState, + setStoragePathDirectState, setStoragePathState, } from "./storage/path-state.js"; import { @@ -606,7 +607,7 @@ export function setStoragePath(projectPath: string | null): void { } export function setStoragePathDirect(path: string | null): void { - setStoragePathState({ + setStoragePathDirectState({ currentStoragePath: path, currentLegacyProjectStoragePath: null, currentLegacyWorktreeStoragePath: null, @@ -1204,12 +1205,27 @@ export function normalizeAccountStorage( // transforms — most importantly the scalar `rateLimitResetTime` -> map // `rateLimitResetTimes` conversion — so a rate-limited V1 account would be // treated as immediately available on upgrade and burst 429s (stress audit M3). - const validAccounts = baseStorage.accounts.filter( - (account): account is AccountMetadataV3 => - isRecord(account) && - typeof account.refreshToken === "string" && - !!account.refreshToken.trim(), - ); + const validAccounts = baseStorage.accounts + .filter( + (account): account is AccountMetadataV3 => + isRecord(account) && + typeof account.refreshToken === "string" && + !!account.refreshToken.trim(), + ) + .map((account) => { + const invalidationTimestamp = account.authInvalidatedAt; + if ( + typeof invalidationTimestamp === "number" && + Number.isFinite(invalidationTimestamp) && + invalidationTimestamp > 0 + ) { + return account; + } + const normalized = { ...account }; + delete normalized.authInvalidatedAt; + delete normalized.authInvalidationErrorCode; + return normalized; + }); const deduplicatedAccounts = deduplicateAccounts(validAccounts); diff --git a/lib/storage/flagged-storage.ts b/lib/storage/flagged-storage.ts index d50222ff1..89fbf014d 100644 --- a/lib/storage/flagged-storage.ts +++ b/lib/storage/flagged-storage.ts @@ -82,8 +82,24 @@ export function normalizeFlaggedStorage( const cooldownReason = isCooldownReason(rawAccount.cooldownReason) ? rawAccount.cooldownReason : undefined; + const authInvalidatedAt = + typeof rawAccount.authInvalidatedAt === "number" && + Number.isFinite(rawAccount.authInvalidatedAt) && + rawAccount.authInvalidatedAt > 0 + ? rawAccount.authInvalidatedAt + : undefined; + const authInvalidationErrorCode = + authInvalidatedAt !== undefined && + typeof rawAccount.authInvalidationErrorCode === "string" && + rawAccount.authInvalidationErrorCode.trim().length > 0 + ? rawAccount.authInvalidationErrorCode.trim() + : undefined; const normalized: FlaggedAccountMetadataV1 = { + recordId: + typeof rawAccount.recordId === "string" && rawAccount.recordId.trim() + ? rawAccount.recordId.trim() + : undefined, refreshToken, addedAt: typeof rawAccount.addedAt === "number" ? rawAccount.addedAt : flaggedAt, @@ -121,6 +137,8 @@ export function normalizeFlaggedStorage( ? rawAccount.coolingDownUntil : undefined, cooldownReason, + authInvalidatedAt, + authInvalidationErrorCode, workspaces: Array.isArray(rawAccount.workspaces) ? (rawAccount.workspaces as FlaggedAccountMetadataV1["workspaces"]) : undefined, diff --git a/lib/storage/path-state.ts b/lib/storage/path-state.ts index b46bb2db5..8a8becc71 100644 --- a/lib/storage/path-state.ts +++ b/lib/storage/path-state.ts @@ -7,7 +7,20 @@ export type StoragePathState = { currentProjectRoot: string | null; }; -const storagePathStateContext = new AsyncLocalStorage(); +type StoragePathStateContext = { + state: StoragePathState; + directGeneration: number; +}; + +const storagePathStateContext = new AsyncLocalStorage(); + +// `setStoragePathDirect` is an explicit override used by callers that need a +// path to win over an async context created before the override. Keep its +// generation separate so a stale context cannot replace the direct path when +// control returns to a previously-created async resource. Scoped contexts +// created after the override still win while they are active. +let directStorageStateOverride: StoragePathState | undefined; +let directStorageGeneration = 0; let currentStorageState: StoragePathState = { currentStoragePath: null, @@ -17,21 +30,45 @@ let currentStorageState: StoragePathState = { }; export function getStoragePathState(): StoragePathState { + const context = storagePathStateContext.getStore(); + if (directStorageStateOverride !== undefined) { + if (context?.directGeneration === directStorageGeneration) { + return context.state; + } + return directStorageStateOverride; + } // Keep the last synchronously assigned state as a fallback until enterWith() // has propagated through the current async chain. This is intentionally a // best-effort bridge for immediate reads; callers should still set state // before spawning child work and treat AsyncLocalStorage as the source of truth. - return storagePathStateContext.getStore() ?? currentStorageState; + return context?.state ?? currentStorageState; } export function setStoragePathState(state: StoragePathState): void { currentStorageState = state; - storagePathStateContext.enterWith(state); + directStorageStateOverride = undefined; + storagePathStateContext.enterWith({ + state, + directGeneration: directStorageGeneration, + }); +} + +export function setStoragePathDirectState(state: StoragePathState): void { + currentStorageState = state; + directStorageStateOverride = state; + directStorageGeneration += 1; + storagePathStateContext.enterWith({ + state, + directGeneration: directStorageGeneration, + }); } export async function runWithStoragePathState( state: StoragePathState, fn: () => T | Promise, ): Promise { - return await storagePathStateContext.run(state, fn); + return await storagePathStateContext.run( + { state, directGeneration: directStorageGeneration }, + fn, + ); } diff --git a/lib/storage/public-types.ts b/lib/storage/public-types.ts index 804797abb..f83c3ecb5 100644 --- a/lib/storage/public-types.ts +++ b/lib/storage/public-types.ts @@ -22,6 +22,8 @@ export interface RateLimitStateV3 { } export interface AccountMetadataV3 { + /** Stable per-record identity used for runtime quota state. */ + recordId?: string; accountId?: string; accountIdSource?: AccountIdSource; accountLabel?: string; @@ -44,6 +46,10 @@ export interface AccountMetadataV3 { rateLimitResetTimes?: RateLimitStateV3; coolingDownUntil?: number; cooldownReason?: CooldownReason; + /** Timestamp of the last explicit upstream OAuth-token invalidation. */ + authInvalidatedAt?: number; + /** Stable provider/client error code associated with the invalidation. */ + authInvalidationErrorCode?: string; workspaces?: Workspace[]; currentWorkspaceIndex?: number; } diff --git a/scripts/codex-app-router.js b/scripts/codex-app-router.js index d79766e6f..c1672b8d6 100644 --- a/scripts/codex-app-router.js +++ b/scripts/codex-app-router.js @@ -36,6 +36,7 @@ function parseArgs(argv) { host: "127.0.0.1", port: 0, statusPath: "", + identityToken: "", statePath: "", logPath: "", maxLogBytes: DEFAULT_MAX_LOG_BYTES, @@ -58,6 +59,11 @@ function parseArgs(argv) { index += 1; continue; } + if (arg === "--identity-token") { + result.identityToken = next; + index += 1; + continue; + } if (arg === "--state") { result.statePath = next; index += 1; @@ -133,7 +139,16 @@ function writeStatus(statusPath, payload) { } } -function createStatusPayload({ state, proxyServer, error, stateRecord }) { +function createStatusPayload({ + state, + proxyServer, + error, + stateRecord, + startedAt, + statusPath, + routerScriptPath, + identityToken, +}) { const proxyStatus = typeof proxyServer?.getStatus === "function" ? proxyServer.getStatus() : {}; const lastAccountIndex = proxyStatus.lastAccountIndex ?? null; @@ -149,6 +164,15 @@ function createStatusPayload({ state, proxyServer, error, stateRecord }) { kind: "codex-app-runtime-rotation-router", state, pid: process.pid, + startedAt, + statusPath: statusPath || readTrimmedString(stateRecord, "statusPath") || null, + identityToken: + identityToken || readTrimmedString(stateRecord, "identityToken") || null, + routerScriptPath: + routerScriptPath || + readTrimmedString(stateRecord, "routerScriptPath") || + process.argv[1] || + null, updatedAt: Date.now(), baseUrl: proxyServer?.baseUrl ?? stateRecord?.baseUrl ?? null, totalRequests: proxyStatus.totalRequests ?? 0, @@ -232,6 +256,7 @@ function installLogBounds(maxBytes, logPath) { } async function main() { + const routerStartedAt = Date.now(); const args = parseArgs(process.argv.slice(2)); installLogBounds(args.maxLogBytes, args.logPath).unref?.(); const stateRecord = readState(args.statePath); @@ -241,7 +266,16 @@ async function main() { ); writeStatus( args.statusPath, - createStatusPayload({ state: "error", proxyServer: null, error, stateRecord: null }), + createStatusPayload({ + state: "error", + proxyServer: null, + error, + stateRecord: null, + startedAt: routerStartedAt, + statusPath: args.statusPath, + routerScriptPath: process.argv[1], + identityToken: args.identityToken, + }), ); throw error; } @@ -272,7 +306,17 @@ async function main() { const writeCurrentStatus = (state, error) => { writeStatus( args.statusPath || stateRecord?.statusPath || "", - createStatusPayload({ state, proxyServer, error, stateRecord }), + createStatusPayload({ + state, + proxyServer, + error, + stateRecord, + startedAt: routerStartedAt, + statusPath: args.statusPath || stateRecord?.statusPath || "", + routerScriptPath: stateRecord?.routerScriptPath || process.argv[1], + identityToken: + args.identityToken || stateRecord?.identityToken || "", + }), ); }; diff --git a/scripts/codex.js b/scripts/codex.js index a0bd0332c..19d3e1e67 100755 --- a/scripts/codex.js +++ b/scripts/codex.js @@ -80,6 +80,8 @@ const APP_SERVER_CONFIG_ARGS_ENV = "CODEX_MULTI_AUTH_APP_SERVER_CONFIG_ARGS_JSON"; const APP_RUNTIME_HELPER_STATUS_FILE = RUNTIME_CONSTANTS.APP_RUNTIME_HELPER_STATUS_FILE; +const APP_RUNTIME_HELPER_OWNER_FILE = + RUNTIME_CONSTANTS.APP_RUNTIME_HELPER_OWNER_FILE; const DEFAULT_APP_RUNTIME_HELPER_IDLE_MS = 12 * 60 * 60 * 1000; const DEFAULT_APP_RUNTIME_HELPER_DETACH_GRACE_MS = 5_000; const APP_RUNTIME_HELPER_LAUNCH_TIMEOUT_MS = 15_000; @@ -110,6 +112,17 @@ let shadowHomeSyncLockOwnerWriteFailuresRemaining = Number.parseInt( process.env.CODEX_MULTI_AUTH_TEST_SHADOW_LOCK_OWNER_WRITE_FAILURES ?? "0", 10, ); +let appServerShimFileCleanupBusyFailuresRemaining = + Number.parseInt( + process.env.CODEX_MULTI_AUTH_TEST_APP_SERVER_SHIM_FILE_CLEANUP_BUSY_FAILURES ?? + "0", + 10, + ) || 0; +let appServerShimCopyBusyFailuresRemaining = + Number.parseInt( + process.env.CODEX_MULTI_AUTH_TEST_APP_SERVER_SHIM_COPY_BUSY_FAILURES ?? "0", + 10, + ) || 0; const shadowHomeCleanupRetryMarkerDir = (process.env.CODEX_MULTI_AUTH_TEST_SHADOW_RETRY_MARKER_DIR ?? "").trim(); let warnedInvalidRuntimeRotationProxyEnv = false; @@ -122,6 +135,7 @@ async function loadRuntimeConstants() { const fallback = { RUNTIME_ROTATION_PROXY_PROVIDER_ID: `${APP_SERVER_ACCOUNT_DISPLAY_NAME}-runtime-proxy`, APP_RUNTIME_HELPER_STATUS_FILE: "runtime-rotation-app-helper.json", + APP_RUNTIME_HELPER_OWNER_FILE: "runtime-rotation-app-helper-owner.json", }; try { const mod = await import("../dist/lib/runtime-constants.js"); @@ -134,6 +148,10 @@ async function loadRuntimeConstants() { typeof mod.APP_RUNTIME_HELPER_STATUS_FILE === "string" ? mod.APP_RUNTIME_HELPER_STATUS_FILE : fallback.APP_RUNTIME_HELPER_STATUS_FILE, + APP_RUNTIME_HELPER_OWNER_FILE: + typeof mod.APP_RUNTIME_HELPER_OWNER_FILE === "string" + ? mod.APP_RUNTIME_HELPER_OWNER_FILE + : fallback.APP_RUNTIME_HELPER_OWNER_FILE, }; } catch { // Keep wrapper startup resilient when dist has not been built yet. @@ -171,6 +189,39 @@ function removeDirectoryWithRetry(targetPath) { } } +function withSynchronousFileOperationRetry(operation) { + for ( + let attempt = 0; + attempt <= SHADOW_HOME_CLEANUP_BACKOFF_MS.length; + attempt += 1 + ) { + try { + return operation(); + } catch (error) { + if ( + !isRetryableShadowHomeCleanupError(error) || + attempt === SHADOW_HOME_CLEANUP_BACKOFF_MS.length + ) { + throw error; + } + sleepSync(SHADOW_HOME_CLEANUP_BACKOFF_MS[attempt]); + } + } +} + +function maybeThrowSimulatedAppServerShimFileError(kind) { + const remaining = + kind === "copy" + ? appServerShimCopyBusyFailuresRemaining + : appServerShimFileCleanupBusyFailuresRemaining; + if (remaining <= 0) return; + if (kind === "copy") appServerShimCopyBusyFailuresRemaining -= 1; + else appServerShimFileCleanupBusyFailuresRemaining -= 1; + const error = new Error(`simulated app-server shim ${kind} EBUSY`); + error.code = "EBUSY"; + throw error; +} + /** * Best-effort async directory removal for hot-path callbacks (e.g. the * status-refresh child's close/error handlers, which fire on the PARENT event @@ -3653,14 +3704,35 @@ function installRuntimeRotationAppServerCliShim(forwardedEnv, configArgs = []) { const preloadPath = join(shimDir, "codex-multi-auth-app-server-preload.mjs"); try { try { - rmSync(executablePath, { force: true }); + withSynchronousFileOperationRetry(() => { + maybeThrowSimulatedAppServerShimFileError("cleanup"); + rmSync(executablePath, { force: true }); + }); } catch { - // Best-effort stale shim cleanup only. + // Best-effort stale shim cleanup only; the copy below will report a + // persistent failure without leaving a partially-created helper. } - try { - linkSync(process.execPath, executablePath); - } catch { - copyFileSync(process.execPath, executablePath); + if ( + process.platform === "win32" || + (process.env.CODEX_MULTI_AUTH_TEST_FORCE_APP_SERVER_SHIM_COPY ?? "") === "1" + ) { + // A Windows hard link to the running node.exe remains locked by the + // helper process itself, which prevents the shim directory from being + // removed during graceful helper shutdown. Use an independent image so + // the helper can clean up its app-server shim before exiting. + withSynchronousFileOperationRetry(() => { + maybeThrowSimulatedAppServerShimFileError("copy"); + copyFileSync(process.execPath, executablePath); + }); + } else { + try { + linkSync(process.execPath, executablePath); + } catch { + withSynchronousFileOperationRetry(() => { + maybeThrowSimulatedAppServerShimFileError("copy"); + copyFileSync(process.execPath, executablePath); + }); + } } if (process.platform !== "win32") { chmodSync(executablePath, 0o755); @@ -3704,6 +3776,19 @@ function resolveRuntimeRotationAppHelperStatusPath(env = process.env) { return join(multiAuthDir, APP_RUNTIME_HELPER_STATUS_FILE); } +function resolveRuntimeRotationAppHelperOwnerPath(env = process.env, helperPid) { + const multiAuthDir = + resolveOriginalMultiAuthDir(env) ?? join(resolveCodexHomeDir(env), "multi-auth"); + const ownerFileName = + typeof helperPid === "number" && Number.isInteger(helperPid) && helperPid > 0 + ? APP_RUNTIME_HELPER_OWNER_FILE.replace( + /\.json$/i, + `.${helperPid}.json`, + ) + : APP_RUNTIME_HELPER_OWNER_FILE; + return join(multiAuthDir, ownerFileName); +} + function writeOwnerOnlyJsonFileAtomicSync(targetPath, payload) { const targetDir = dirname(targetPath); mkdirSync(targetDir, { recursive: true }); @@ -3816,9 +3901,38 @@ function writeRuntimeRotationAppHelperStatus(payload, env = process.env) { } } +function writeRuntimeRotationAppHelperOwner( + identityToken, + helperPid, + env = process.env, +) { + if ( + typeof helperPid !== "number" || + !Number.isInteger(helperPid) || + helperPid < 1 + ) { + return; + } + try { + writeOwnerOnlyJsonFileAtomicSync( + resolveRuntimeRotationAppHelperOwnerPath(env, helperPid), + { + version: 1, + kind: "codex-app-runtime-rotation-helper-owner", + identityToken, + launcherPid: process.pid, + createdAt: Date.now(), + }, + ); + } catch { + // Best-effort metadata; an unavailable owner file must never stop the helper. + } +} + function createRuntimeRotationAppHelperStatus({ proxyServer, startedAt, + identityToken, idleTimeoutMs, lastActivityAt, state, @@ -3838,6 +3952,8 @@ function createRuntimeRotationAppHelperStatus({ kind: "codex-app-runtime-rotation-helper", state, pid: process.pid, + scriptPath: process.argv[1] ?? null, + identityToken: identityToken || null, startedAt, updatedAt: Date.now(), baseUrl: proxyServer?.baseUrl ?? null, @@ -3855,7 +3971,7 @@ function createRuntimeRotationAppHelperStatus({ }; } -async function runRuntimeRotationAppHelper() { +async function runRuntimeRotationAppHelper(identityToken = "") { let proxyServer = null; let runtimeContext = null; let appServerShimDir = null; @@ -3872,6 +3988,7 @@ async function runRuntimeRotationAppHelper() { createRuntimeRotationAppHelperStatus({ proxyServer, startedAt, + identityToken, idleTimeoutMs, lastActivityAt, state, @@ -4066,25 +4183,32 @@ function startRuntimeRotationAppHelper(baseContext, options = {}) { let stdoutBuffer = ""; let stderrBuffer = ""; let settled = false; + const identityToken = randomBytes(24).toString("hex"); + const helperEnv = { + ...baseContext.env, + CODEX_MULTI_AUTH_DIR: resolveRuntimeRotationOriginalMultiAuthDir( + realCodexHome, + baseContext.env, + ), + [APP_RUNTIME_HELPER_OWNER_PID_ENV]: String(process.pid), + [APP_RUNTIME_HELPER_REAL_CODEX_HOME_ENV]: realCodexHome, + [APP_RUNTIME_HELPER_USE_CANONICAL_HOME_ENV]: + options.useCanonicalHome === true ? "1" : "0", + }; const helper = spawn( process.execPath, - [fileURLToPath(import.meta.url), INTERNAL_RUNTIME_ROTATION_APP_HELPER_ARG], + [ + fileURLToPath(import.meta.url), + INTERNAL_RUNTIME_ROTATION_APP_HELPER_ARG, + identityToken, + ], { - env: { - ...baseContext.env, - CODEX_MULTI_AUTH_DIR: resolveRuntimeRotationOriginalMultiAuthDir( - realCodexHome, - baseContext.env, - ), - [APP_RUNTIME_HELPER_OWNER_PID_ENV]: String(process.pid), - [APP_RUNTIME_HELPER_REAL_CODEX_HOME_ENV]: realCodexHome, - [APP_RUNTIME_HELPER_USE_CANONICAL_HOME_ENV]: - options.useCanonicalHome === true ? "1" : "0", - }, + env: helperEnv, stdio: ["ignore", "pipe", "pipe"], detached: true, }, ); + writeRuntimeRotationAppHelperOwner(identityToken, helper.pid, helperEnv); let timeout = null; const finish = (result) => { if (settled) return; @@ -4150,6 +4274,11 @@ async function createRuntimeRotationAppHelperContext( options, ); const helperEnv = message.env ?? {}; + const helperShimDir = + typeof helperEnv.CODEX_CLI_PATH === "string" && + helperEnv.CODEX_CLI_PATH.length > 0 + ? helperEnv.CODEX_CLI_PATH + : null; const helperArgs = Array.isArray(message.args) ? message.args.filter((arg) => typeof arg === "string") : []; @@ -4163,7 +4292,22 @@ async function createRuntimeRotationAppHelperContext( helper.unref(); return; } - await stopRuntimeRotationAppHelper(helper); + try { + await stopRuntimeRotationAppHelper(helper); + } finally { + // The helper normally removes its app-server shim before exiting. On + // Windows, a descendant may still hold the copied codex.exe briefly + // after the helper closes, so retry the parent-owned cleanup once the + // helper shutdown path has completed as well. + if (helperShimDir) { + try { + removeDirectoryWithRetry(helperShimDir); + } catch { + // Best-effort cleanup; stale helper directories are swept on the + // next runtime app launch. + } + } + } }; return { @@ -5166,7 +5310,7 @@ async function main() { const rawArgs = process.argv.slice(2); if (rawArgs[0] === INTERNAL_RUNTIME_ROTATION_APP_HELPER_ARG) { - return runRuntimeRotationAppHelper(); + return runRuntimeRotationAppHelper(rawArgs[1] ?? ""); } const normalizedArgs = normalizeAuthAlias(rawArgs); diff --git a/scripts/install-codex-auth-utils.js b/scripts/install-codex-auth-utils.js index ffa2b4410..fd1ea29d6 100644 --- a/scripts/install-codex-auth-utils.js +++ b/scripts/install-codex-auth-utils.js @@ -6,6 +6,8 @@ import { join } from "node:path"; import process from "node:process"; const PLUGIN_NAME = "codex-multi-auth"; +const LEGACY_PLUGIN_NAME = "@ndycode/codex-multi-auth"; +const PLUGIN_NAMES = [PLUGIN_NAME, LEGACY_PLUGIN_NAME]; const TRUE_VALUES = new Set(["1", "true", "yes"]); const FALSE_VALUES = new Set(["0", "false", "no"]); @@ -82,6 +84,7 @@ export function resolveInstallPaths( configPath, cacheDir, cacheNodeModules: join(cacheDir, "node_modules", PLUGIN_NAME), + cacheLegacyNodeModules: join(cacheDir, "node_modules", "@ndycode", "codex-multi-auth"), cacheBunLock: join(cacheDir, "bun.lock"), cachePackageJson: join(cacheDir, "package.json"), }; @@ -92,7 +95,9 @@ export function removePluginFromList(list) { const entries = Array.isArray(list) ? list.filter(Boolean) : []; return entries.filter((entry) => { if (typeof entry !== "string") return true; - return entry !== PLUGIN_NAME && !entry.startsWith(`${PLUGIN_NAME}@`); + return !PLUGIN_NAMES.some( + (pluginName) => entry === pluginName || entry.startsWith(`${pluginName}@`), + ); }); } @@ -101,7 +106,9 @@ export function normalizePluginList(list) { const entries = Array.isArray(list) ? list.filter(Boolean) : []; const filtered = entries.filter((entry) => { if (typeof entry !== "string") return true; - return entry !== PLUGIN_NAME && !entry.startsWith(`${PLUGIN_NAME}@`); + return !PLUGIN_NAMES.some( + (pluginName) => entry === pluginName || entry.startsWith(`${pluginName}@`), + ); }); const deduped = []; const seen = new Set(); diff --git a/scripts/postinstall.js b/scripts/postinstall.js index 0292a4637..25ec866f2 100644 --- a/scripts/postinstall.js +++ b/scripts/postinstall.js @@ -33,6 +33,8 @@ const CI_ENV_KEYS = [ export const INSTALL_NOTICE = [ "codex-multi-auth installed. Run `codex-multi-auth --help` to get started.", "App integration (Codex app bind + launcher shortcuts) completes automatically on first run.", + "To remove it later, run `codex-multi-auth uninstall`, then `npm uninstall -g codex-multi-auth`.", + "`@ndycode/codex-multi-auth` is the legacy scoped package name; it does not remove this package.", ].join("\n"); /** diff --git a/scripts/preuninstall.js b/scripts/preuninstall.js index fd891ebcd..f9138f88a 100644 --- a/scripts/preuninstall.js +++ b/scripts/preuninstall.js @@ -189,6 +189,9 @@ export async function runPreuninstallCleanup(deps = {}) { ); if (dryRun) { log(`[dry-run] Would remove ${paths.cacheNodeModules}`); + if (paths.cacheLegacyNodeModules) { + log(`[dry-run] Would remove ${paths.cacheLegacyNodeModules}`); + } if (bunLockSafe) { log(`[dry-run] Would remove ${paths.cacheBunLock}`); } else { @@ -201,6 +204,14 @@ export async function runPreuninstallCleanup(deps = {}) { await withFileOperationRetry(() => rm(paths.cacheNodeModules, { recursive: true, force: true }), ); + if (paths.cacheLegacyNodeModules) { + await withFileOperationRetry(() => + rm(paths.cacheLegacyNodeModules, { + recursive: true, + force: true, + }), + ); + } if (bunLockSafe) { await withFileOperationRetry(() => rm(paths.cacheBunLock, { force: true }), diff --git a/test/accounts.test.ts b/test/accounts.test.ts index 3a2ee5202..6bc99906b 100644 --- a/test/accounts.test.ts +++ b/test/accounts.test.ts @@ -32,6 +32,7 @@ import { getStoragePathState, setStoragePathState, } from "../lib/storage/path-state.js"; +import type { AccountStorageV3 } from "../lib/storage.js"; import type { OAuthAuthDetails } from "../lib/types.js"; import { MODEL_FAMILIES } from "../lib/prompts/codex.js"; @@ -1624,6 +1625,51 @@ describe("AccountManager", () => { }); describe("auth failure tracking", () => { + it.each([ + ["zero", 0], + ["negative", -1], + ["NaN", Number.NaN], + ["infinity", Number.POSITIVE_INFINITY], + ] as const)("does not treat %s as an auth invalidation", (_label, timestamp) => { + const now = Date.now(); + const manager = new AccountManager(undefined, { + version: 3 as const, + activeIndex: 0, + accounts: [ + { + refreshToken: "token-1", + addedAt: now, + lastUsed: now, + authInvalidatedAt: timestamp, + }, + ], + }); + const account = manager.getCurrentAccount(); + if (!account) throw new Error("account missing"); + + expect(manager.isAccountAuthInvalidated(account)).toBe(false); + }); + + it("treats a positive finite timestamp as an auth invalidation", () => { + const now = Date.now(); + const manager = new AccountManager(undefined, { + version: 3 as const, + activeIndex: 0, + accounts: [ + { + refreshToken: "token-1", + addedAt: now, + lastUsed: now, + authInvalidatedAt: now, + }, + ], + }); + const account = manager.getCurrentAccount(); + if (!account) throw new Error("account missing"); + + expect(manager.isAccountAuthInvalidated(account)).toBe(true); + }); + it("increments consecutive auth failures", () => { const now = Date.now(); const stored = { @@ -2423,7 +2469,7 @@ describe("AccountManager", () => { }); }); - describe("saveToDisk", () => { + describe("saveToDisk", () => { it("saves accounts with all fields", async () => { const { saveAccounts } = await import("../lib/storage.js"); const mockSaveAccounts = vi.mocked(saveAccounts); @@ -2457,6 +2503,76 @@ describe("AccountManager", () => { expect(savedData?.accounts[0]?.email).toBe("test@example.com"); expect(savedData?.accounts[0]?.rateLimitResetTimes).toBeDefined(); }); + + it("omits malformed auth invalidation markers from routine saves", async () => { + const { saveAccounts } = await import("../lib/storage.js"); + const mockSaveAccounts = vi.mocked(saveAccounts); + const now = Date.now(); + const manager = new AccountManager(undefined, { + version: 3 as const, + activeIndex: 0, + accounts: [ + { + refreshToken: "token-1", + addedAt: now, + lastUsed: now, + authInvalidatedAt: 0, + authInvalidationErrorCode: "oauth_token_revoked", + }, + ], + }); + + await manager.saveToDisk(); + + const savedAccount = mockSaveAccounts.mock.calls[0]?.[0]?.accounts[0]; + expect(savedAccount?.authInvalidatedAt).toBeUndefined(); + expect(savedAccount?.authInvalidationErrorCode).toBeUndefined(); + }); + + it("preserves an invalidation written by another manager before a stale routine save", async () => { + const { saveAccounts, withAccountStorageTransaction } = await import( + "../lib/storage.js" + ); + const mockSaveAccounts = vi.mocked(saveAccounts); + const mockWithAccountStorageTransaction = vi.mocked( + withAccountStorageTransaction, + ); + let disk: AccountStorageV3 | null = null; + mockWithAccountStorageTransaction.mockImplementation(async (handler) => { + const persist = async (storage: AccountStorageV3) => { + disk = structuredClone(storage); + await mockSaveAccounts(storage); + }; + return handler(disk ? structuredClone(disk) : null, persist); + }); + + const now = Date.now(); + const initialStorage: AccountStorageV3 = { + version: 3, + activeIndex: 0, + accounts: [ + { + refreshToken: "shared-refresh-token", + email: "shared@example.com", + addedAt: now, + lastUsed: now, + }, + ], + }; + const managerA = new AccountManager(undefined, structuredClone(initialStorage)); + const managerB = new AccountManager(undefined, structuredClone(initialStorage)); + + const accountA = managerA.getAccountByIndex(0); + if (!accountA) throw new Error("manager A account missing"); + managerA.markAuthInvalidated(accountA, "oauth_token_revoked", now + 1); + await managerA.saveToDisk(); + await managerB.saveToDisk(); + + expect(disk?.accounts[0]).toMatchObject({ + authInvalidatedAt: now + 1, + authInvalidationErrorCode: "oauth_token_revoked", + }); + }); }); describe("saveToDiskDebounced", () => { diff --git a/test/app-bind.test.ts b/test/app-bind.test.ts index 04312b283..535bbe95c 100644 --- a/test/app-bind.test.ts +++ b/test/app-bind.test.ts @@ -3,21 +3,29 @@ import { createHash } from "node:crypto"; import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; -import { spawnSync } from "node:child_process"; +import { spawn, spawnSync } from "node:child_process"; import { fileURLToPath } from "node:url"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { bindCodexAppRuntimeRotation, formatAppBindStatus, getAppBindStatus, + parsePosixProcessStartTime, resolveAppBindPaths, restoreConfigTomlFromAppBind, rewriteConfigTomlForAppBind, + stopDetachedProcess, + stopRuntimeRotationAppHelperProcess, + stopRuntimeRotationRouterProcess, unbindCodexAppRuntimeRotation, } from "../lib/runtime/app-bind.js"; import { tomlStringLiteral } from "../lib/runtime/config-toml.js"; import { withFileOperationRetry } from "../lib/fs-retry.js"; -import { RUNTIME_ROTATION_PROXY_PROVIDER_ID } from "../lib/runtime-constants.js"; +import { + APP_RUNTIME_HELPER_OWNER_FILE, + APP_RUNTIME_HELPER_STATUS_FILE, + RUNTIME_ROTATION_PROXY_PROVIDER_ID, +} from "../lib/runtime-constants.js"; const tempRoots: string[] = []; const thisDir = dirname(fileURLToPath(import.meta.url)); @@ -72,6 +80,109 @@ async function seedExistingAppBindState(params: { ); } +function resolveRuntimeHelperStatusPath(options: { + home: string; + env: NodeJS.ProcessEnv; +}): string { + const paths = resolveAppBindPaths({ + platform: process.platform, + home: options.home, + env: options.env, + }); + return join(dirname(paths.bindDir), APP_RUNTIME_HELPER_STATUS_FILE); +} + +function resolveRuntimeHelperOwnerPath(options: { + home: string; + env: NodeJS.ProcessEnv; +}, pid: number): string { + const paths = resolveAppBindPaths({ + platform: process.platform, + home: options.home, + env: options.env, + }); + return join( + dirname(paths.bindDir), + APP_RUNTIME_HELPER_OWNER_FILE.replace(/\.json$/i, `.${pid}.json`), + ); +} + +async function writeRuntimeHelperStatus( + options: { home: string; env: NodeJS.ProcessEnv }, + status: Record | string, +): Promise { + const statusPath = resolveRuntimeHelperStatusPath(options); + await mkdir(dirname(statusPath), { recursive: true }); + await writeFile( + statusPath, + typeof status === "string" ? status : `${JSON.stringify(status)}\n`, + "utf8", + ); + return statusPath; +} + +async function writeRuntimeHelperOwner( + options: { home: string; env: NodeJS.ProcessEnv }, + pid: number, + identityToken: string, +): Promise { + const ownerPath = resolveRuntimeHelperOwnerPath(options, pid); + await mkdir(dirname(ownerPath), { recursive: true }); + await writeFile( + ownerPath, + `${JSON.stringify({ + version: 1, + kind: "codex-app-runtime-rotation-helper-owner", + identityToken, + })}\n`, + "utf8", + ); + return ownerPath; +} + +async function spawnHelperFixture( + root: string, + name: string, +): Promise<{ + child: ReturnType; + pid: number; + scriptPath: string; +}> { + const scriptPath = join(root, `${name}.mjs`); + await writeFile( + scriptPath, + [ + "process.on('SIGTERM', () => process.exit(0));", + "setInterval(() => undefined, 1000);", + "", + ].join("\n"), + "utf8", + ); + const child = spawn(process.execPath, [scriptPath, "--codex-multi-auth-runtime-app-helper"], { + detached: true, + stdio: "ignore", + windowsHide: true, + }); + child.unref(); + const pid = child.pid; + if (!pid) throw new Error(`${name} fixture did not spawn`); + await new Promise((resolve) => setTimeout(resolve, 50)); + return { child, pid, scriptPath }; +} + +async function stopHelperFixture(child: ReturnType): Promise { + if (child.exitCode !== null || child.signalCode !== null) return; + const exited = new Promise((resolve) => + child.once("exit", () => resolve()), + ); + try { + child.kill("SIGTERM"); + } catch { + // The fixture may have exited between the state check and the signal. + } + await exited; +} + afterEach(async () => { await Promise.all( tempRoots.splice(0).map((root) => @@ -124,6 +235,21 @@ it("prints the resolved app-bind config path in reasoning guidance", () => { }); describe("Codex app runtime rotation bind", () => { + it("parses representative POSIX ps lstart output deterministically", () => { + const expected = new Date(2026, 7, 8, 14, 32, 10).getTime(); + + expect( + parsePosixProcessStartTime("Sat Aug 8 14:32:10 2026"), + ).toBe(expected); + expect(parsePosixProcessStartTime("Sun Jan 1 00:00:00 2023")).toBe( + new Date(2023, 0, 1).getTime(), + ); + expect(parsePosixProcessStartTime("not a process start time")).toBeNull(); + expect(parsePosixProcessStartTime("Sat Foo 8 14:32:10 2026")).toBeNull(); + expect(parsePosixProcessStartTime("Sat Feb 29 14:32:10 2025")).toBeNull(); + expect(parsePosixProcessStartTime("Sat Aug 8 24:00:00 2026")).toBeNull(); + }); + it("rewrites and restores Codex config TOML without disturbing other sections", () => { const original = [ 'model_provider = "openai"', @@ -290,6 +416,7 @@ describe("Codex app runtime rotation bind", () => { expect(result.status.state?.statePath).toBe( join(multiAuthDir, "app-bind", "runtime-rotation-app-bind.json"), ); + expect(result.status.state?.identityToken).toMatch(/^[0-9a-f]{48}$/); const config = await readFile(join(codexHome, "config.toml"), "utf8"); expect(config).toContain( `[model_providers.${RUNTIME_ROTATION_PROXY_PROVIDER_ID}]`, @@ -306,6 +433,8 @@ describe("Codex app runtime rotation bind", () => { } const startup = await readFile(result.status.paths.startupPath ?? "", "utf8"); expect(startup).toContain("--state"); + expect(startup).toContain("--identity-token"); + expect(startup).toContain(result.status.state?.identityToken ?? ""); expect(startup).toContain("--log"); expect(startup).toContain("--max-log-bytes 1048576"); expect(startup).toContain("runtime-rotation-app-bind.json"); @@ -329,6 +458,676 @@ describe("Codex app runtime rotation bind", () => { 'model_provider = "openai"\n', ); expect(existsSync(result.status.paths.startupPath ?? "")).toBe(false); + expect(existsSync(result.status.paths.bindDir)).toBe(false); + }); + + it("stops a detached POSIX helper with SIGKILL escalation", async () => { + let alive = true; + const signals: NodeJS.Signals[] = []; + const result = await stopRuntimeRotationAppHelperProcess( + { + kind: "codex-app-runtime-rotation-helper", + state: "running", + pid: 4242, + startedAt: Date.now(), + }, + { + platform: "linux", + gracefulTimeoutMs: 0, + isAlive: () => alive, + verifyProcessIdentity: async () => true, + kill: (_pid, signal) => { + signals.push(signal); + if (signal === "SIGKILL") alive = false; + }, + }, + ); + + expect(result).toBe(true); + expect(signals).toEqual(["SIGTERM", "SIGKILL"]); + }); + + it("uses Windows tree termination directly without sending SIGTERM", async () => { + let alive = true; + const kill = vi.fn(); + const taskkill = vi.fn(async () => { + alive = false; + }); + + await expect( + stopRuntimeRotationAppHelperProcess( + { + kind: "codex-app-runtime-rotation-helper", + state: "running", + pid: 4343, + startedAt: Date.now(), + }, + { + platform: "win32", + gracefulTimeoutMs: 0, + isAlive: () => alive, + verifyProcessIdentity: async () => true, + kill, + runWindowsTaskkill: taskkill, + }, + ), + ).resolves.toBe(true); + + expect(kill).not.toHaveBeenCalled(); + expect(taskkill).toHaveBeenCalledWith(4343); + }); + + it("uses Windows tree termination even when the target would exit immediately", async () => { + let alive = true; + const kill = vi.fn(() => { + alive = false; + }); + const taskkill = vi.fn(async () => { + alive = false; + }); + + await expect( + stopRuntimeRotationAppHelperProcess( + { + kind: "codex-app-runtime-rotation-helper", + state: "running", + pid: 4646, + startedAt: Date.now(), + }, + { + platform: "win32", + gracefulTimeoutMs: 0, + isAlive: () => alive, + verifyProcessIdentity: async () => true, + kill, + runWindowsTaskkill: taskkill, + }, + ), + ).resolves.toBe(true); + + expect(kill).not.toHaveBeenCalled(); + expect(taskkill).toHaveBeenCalledWith(4646); + }); + + it("does not signal malformed or stale helper status", async () => { + const kill = vi.fn(); + const options = { + platform: "linux" as const, + isAlive: () => true, + kill, + gracefulTimeoutMs: 0, + }; + + expect( + await stopRuntimeRotationAppHelperProcess( + { + kind: "other-process", + state: "running", + pid: 4444, + startedAt: Date.now(), + }, + options, + ), + ).toBe(false); + expect( + await stopRuntimeRotationAppHelperProcess( + { + kind: "codex-app-runtime-rotation-helper", + state: "stopped", + pid: 4444, + startedAt: Date.now(), + }, + options, + ), + ).toBe(false); + expect( + await stopRuntimeRotationAppHelperProcess( + { + kind: "codex-app-runtime-rotation-helper", + state: "running", + pid: null, + startedAt: Date.now(), + }, + options, + ), + ).toBe(false); + expect(kill).not.toHaveBeenCalled(); + }); + + it("does not signal a live PID when process identity does not match", async () => { + const kill = vi.fn(); + const verifyProcessIdentity = vi.fn(async () => false); + + const result = await stopRuntimeRotationAppHelperProcess( + { + kind: "codex-app-runtime-rotation-helper", + state: "running", + pid: 4545, + startedAt: Date.now() - 1_000, + }, + { + platform: "linux", + isAlive: () => true, + kill, + verifyProcessIdentity, + }, + ); + + expect(result).toBe(false); + expect(verifyProcessIdentity).toHaveBeenCalledWith( + 4545, + expect.any(Number), + "linux", + ); + expect(kill).not.toHaveBeenCalled(); + }); + + it("treats ESRCH after a signal race as an already-stopped helper", async () => { + let alive = true; + const error = Object.assign(new Error("process exited"), { code: "ESRCH" }); + const kill = vi.fn(() => { + alive = false; + throw error; + }); + + await expect( + stopDetachedProcess(4748, "linux", { + isAlive: () => alive, + kill, + verifyProcessIdentity: undefined, + }), + ).resolves.toBe(true); + expect(kill).toHaveBeenCalledWith(4748, "SIGTERM"); + }); + + it("does not signal a stale router PID when process identity does not match", async () => { + const kill = vi.fn(); + const verifyProcessIdentity = vi.fn(async () => false); + const startedAt = Date.now() - 1_000; + + const result = await stopRuntimeRotationRouterProcess( + { + state: "running", + pid: 4646, + startedAt, + updatedAt: startedAt, + }, + "linux", + "/tmp/codex-app-router.js", + { + isAlive: () => true, + kill, + verifyProcessIdentity, + }, + ); + + expect(result).toBe(false); + expect(verifyProcessIdentity).toHaveBeenCalledWith( + 4646, + expect.any(Number), + "linux", + ); + expect(kill).not.toHaveBeenCalled(); + }); + + it("passes the bind ownership token to the router identity verifier", async () => { + let alive = true; + const kill = vi.fn(() => { + alive = false; + }); + const verifyProcessIdentity = vi.fn(async () => true); + + await expect( + stopRuntimeRotationRouterProcess( + { + state: "running", + pid: 4749, + startedAt: Date.now(), + updatedAt: Date.now(), + identityToken: "status-token", + }, + "linux", + "/tmp/codex-app-router.js", + { + identityToken: "bind-token", + isAlive: () => alive, + kill, + verifyProcessIdentity, + }, + ), + ).resolves.toBe(true); + + expect(verifyProcessIdentity).toHaveBeenCalledWith( + 4749, + expect.any(Number), + "linux", + "bind-token", + ); + expect(kill).toHaveBeenCalledWith(4749, "SIGTERM"); + }); + + it("does not stop a replacement router with a different bind token", async () => { + const kill = vi.fn(); + const verifyProcessIdentity = vi.fn( + async ( + _pid: number, + _startedAt: number, + _platform: NodeJS.Platform, + identityToken?: string, + ) => identityToken === "replacement-token", + ); + + const result = await stopRuntimeRotationRouterProcess( + { + state: "running", + pid: 4750, + startedAt: Date.now(), + updatedAt: Date.now(), + identityToken: "replacement-token", + }, + "linux", + "/tmp/codex-app-router.js", + { + identityToken: "original-token", + isAlive: () => true, + kill, + verifyProcessIdentity, + }, + ); + + expect(result).toBe(false); + expect(verifyProcessIdentity).toHaveBeenCalledWith( + 4750, + expect.any(Number), + "linux", + "original-token", + ); + expect(kill).not.toHaveBeenCalled(); + }); + + it("stops a legacy router status after verifying its last observed time", async () => { + let alive = true; + const kill = vi.fn((_pid: number, signal: NodeJS.Signals) => { + if (signal === "SIGTERM") alive = false; + }); + const updatedAt = Date.now() - 1_000; + const verifyProcessIdentity = vi.fn(async () => true); + + const result = await stopRuntimeRotationRouterProcess( + { + state: "running", + pid: 4747, + startedAt: null, + updatedAt, + }, + "linux", + "/tmp/codex-app-router.js", + { + isAlive: () => alive, + kill, + verifyProcessIdentity, + }, + ); + + expect(result).toBe(true); + expect(verifyProcessIdentity).toHaveBeenCalledWith(4747, updatedAt, "linux"); + expect(kill).toHaveBeenCalledWith(4747, "SIGTERM"); + }); + + it("validates legacy router timestamps and command identity before stopping", async () => { + const root = await createTempRoot("codex-app-bind-legacy-router-"); + const routerScriptPath = join(root, "legacy-router.mjs"); + await writeFile( + routerScriptPath, + [ + "process.on('SIGTERM', () => process.exit(0));", + "setInterval(() => undefined, 1000);", + "", + ].join("\n"), + "utf8", + ); + const child = spawn(process.execPath, [routerScriptPath], { + detached: true, + stdio: "ignore", + windowsHide: true, + }); + child.unref(); + const pid = child.pid; + if (!pid) throw new Error("legacy router fixture did not spawn"); + await new Promise((resolve) => setTimeout(resolve, 250)); + const verifyProcessIdentity = vi.fn( + async (_pid: number, observedAt: number, platform: NodeJS.Platform) => + platform === process.platform && + observedAt >= Date.now() - 5_000 && + observedAt <= Date.now() + 5_000, + ); + + try { + await expect( + stopRuntimeRotationRouterProcess( + { + state: "running", + pid, + startedAt: null, + updatedAt: Date.now() + 60_000, + routerScriptPath, + }, + process.platform, + routerScriptPath, + { verifyProcessIdentity }, + ), + ).resolves.toBe(false); + await expect( + stopRuntimeRotationRouterProcess( + { + state: "running", + pid, + startedAt: null, + updatedAt: Date.now(), + routerScriptPath, + }, + process.platform, + join(root, "different-router.mjs"), + { verifyProcessIdentity }, + ), + ).resolves.toBe(false); + await expect( + stopRuntimeRotationRouterProcess( + { + state: "running", + pid, + startedAt: null, + updatedAt: Date.now() - 60_000, + routerScriptPath, + }, + process.platform, + routerScriptPath, + { verifyProcessIdentity }, + ), + ).resolves.toBe(false); + await expect( + stopRuntimeRotationRouterProcess( + { + state: "running", + pid, + startedAt: null, + updatedAt: Date.now(), + routerScriptPath, + }, + process.platform, + routerScriptPath, + { pollIntervalMs: 50, verifyProcessIdentity }, + ), + ).resolves.toBe(true); + } finally { + try { + child.kill("SIGKILL"); + } catch { + // The final valid stop may have already exited the fixture. + } + } + }); + + it("unbinds an owned running helper and removes its status after stopping", async () => { + const root = await createTempRoot("codex-app-bind-owned-helper-"); + const multiAuthDir = join(root, "multi-auth"); + const env = { + CODEX_MULTI_AUTH_DIR: multiAuthDir, + CODEX_MULTI_AUTH_APP_BIND_CODEX_HOME: join(root, "codex-home"), + }; + const helperScriptPath = join(root, "runtime-helper.mjs"); + await writeFile( + helperScriptPath, + [ + "process.on('SIGTERM', () => process.exit(0));", + "setInterval(() => undefined, 1000);", + "", + ].join("\n"), + "utf8", + ); + const child = spawn( + process.execPath, + [helperScriptPath, "--codex-multi-auth-runtime-app-helper"], + { detached: true, stdio: "ignore", windowsHide: true }, + ); + child.unref(); + const pid = child.pid; + if (!pid) throw new Error("owned helper fixture did not spawn"); + await new Promise((resolve) => setTimeout(resolve, 250)); + const helperStartedAt = Date.now(); + const helperIdentityToken = "owned-helper-token"; + const statusPath = await writeRuntimeHelperStatus( + { home: root, env }, + { + version: 1, + kind: "codex-app-runtime-rotation-helper", + state: "running", + pid, + startedAt: helperStartedAt, + scriptPath: helperScriptPath, + identityToken: helperIdentityToken, + }, + ); + await writeRuntimeHelperOwner( + { home: root, env }, + pid, + helperIdentityToken, + ); + + try { + const exited = new Promise((resolve) => + child.once("exit", () => resolve()), + ); + await unbindCodexAppRuntimeRotation({ + platform: process.platform, + home: root, + env, + verifyProcessIdentity: async (candidatePid, startedAt, platform) => + candidatePid === pid && + platform === process.platform && + Math.abs(startedAt - helperStartedAt) <= 5_000, + }); + await exited; + expect(existsSync(statusPath)).toBe(false); + } finally { + try { + process.kill(pid, 0); + if (process.platform === "win32") { + spawnSync("taskkill", ["/PID", String(pid), "/T", "/F"], { + stdio: "ignore", + windowsHide: true, + }); + } else { + child.kill("SIGKILL"); + } + } catch { + // The unbind path already stopped the fixture. + } + } + }); + + it("stops a legacy token-less helper after identity verification", async () => { + const root = await createTempRoot("codex-app-bind-legacy-helper-"); + const env = { + CODEX_MULTI_AUTH_DIR: join(root, "multi-auth"), + CODEX_MULTI_AUTH_APP_BIND_CODEX_HOME: join(root, "codex-home"), + }; + const fixture = await spawnHelperFixture(root, "legacy-helper"); + const startedAt = Date.now(); + const statusPath = await writeRuntimeHelperStatus( + { home: root, env }, + { + version: 1, + kind: "codex-app-runtime-rotation-helper", + state: "running", + pid: fixture.pid, + startedAt, + scriptPath: fixture.scriptPath, + }, + ); + const verifyProcessIdentity = vi.fn(async () => true); + + try { + const exited = new Promise((resolve) => + fixture.child.once("exit", () => resolve()), + ); + await unbindCodexAppRuntimeRotation({ + platform: process.platform, + home: root, + env, + verifyProcessIdentity, + }); + await exited; + + expect(verifyProcessIdentity).toHaveBeenCalledWith( + fixture.pid, + expect.any(Number), + process.platform, + ); + expect(existsSync(statusPath)).toBe(false); + } finally { + await stopHelperFixture(fixture.child); + } + }); + + it("does not stop a replacement helper with a different owner token", async () => { + const root = await createTempRoot("codex-app-bind-helper-replacement-"); + const env = { + CODEX_MULTI_AUTH_DIR: join(root, "multi-auth"), + CODEX_MULTI_AUTH_APP_BIND_CODEX_HOME: join(root, "codex-home"), + }; + const fixture = await spawnHelperFixture(root, "replacement-helper"); + const statusPath = await writeRuntimeHelperStatus( + { home: root, env }, + { + version: 1, + kind: "codex-app-runtime-rotation-helper", + state: "running", + pid: fixture.pid, + startedAt: Date.now(), + scriptPath: fixture.scriptPath, + identityToken: "replacement-token", + }, + ); + await writeRuntimeHelperOwner( + { home: root, env }, + fixture.pid, + "original-token", + ); + const verifyProcessIdentity = vi.fn(async () => true); + + try { + await unbindCodexAppRuntimeRotation({ + platform: process.platform, + home: root, + env, + verifyProcessIdentity, + }); + + expect(verifyProcessIdentity).not.toHaveBeenCalled(); + expect(existsSync(statusPath)).toBe(true); + } finally { + await stopHelperFixture(fixture.child); + } + }); + + it.each([ + [ + "foreign status", + { + version: 1, + kind: "different-runtime-helper", + state: "running", + pid: 2_147_483_647, + startedAt: Date.now(), + }, + ], + ["malformed status", "{not valid json"], + ] as const)("preserves %s during unbind", async (_label, status) => { + const root = await createTempRoot("codex-app-bind-helper-status-"); + const env = { + CODEX_MULTI_AUTH_DIR: join(root, "multi-auth"), + CODEX_MULTI_AUTH_APP_BIND_CODEX_HOME: join(root, "codex-home"), + }; + const statusPath = await writeRuntimeHelperStatus({ home: root, env }, status); + const before = await readFile(statusPath, "utf8"); + + await unbindCodexAppRuntimeRotation({ + platform: process.platform, + home: root, + env, + }); + + expect(existsSync(statusPath)).toBe(true); + expect(await readFile(statusPath, "utf8")).toBe(before); + }); + + it("preserves an owned helper status when identity verification or stopping fails", async () => { + const root = await createTempRoot("codex-app-bind-helper-stop-failure-"); + const env = { + CODEX_MULTI_AUTH_DIR: join(root, "multi-auth"), + CODEX_MULTI_AUTH_APP_BIND_CODEX_HOME: join(root, "codex-home"), + }; + const fixture = await spawnHelperFixture(root, "stop-failure-helper"); + const statusPath = await writeRuntimeHelperStatus( + { home: root, env }, + { + version: 1, + kind: "codex-app-runtime-rotation-helper", + state: "running", + pid: fixture.pid, + startedAt: Date.now(), + scriptPath: join(root, "not-the-test-runner.mjs"), + }, + ); + const before = await readFile(statusPath, "utf8"); + + try { + const verifyProcessIdentity = vi.fn(async () => false); + await unbindCodexAppRuntimeRotation({ + platform: process.platform, + home: root, + env, + verifyProcessIdentity, + }); + + expect(verifyProcessIdentity).toHaveBeenCalledWith( + fixture.pid, + expect.any(Number), + process.platform, + ); + expect(existsSync(statusPath)).toBe(true); + expect(await readFile(statusPath, "utf8")).toBe(before); + } finally { + await stopHelperFixture(fixture.child); + } + }); + + it("removes an owned helper status whose persisted PID has already exited", async () => { + const root = await createTempRoot("codex-app-bind-helper-esrch-"); + const env = { + CODEX_MULTI_AUTH_DIR: join(root, "multi-auth"), + CODEX_MULTI_AUTH_APP_BIND_CODEX_HOME: join(root, "codex-home"), + }; + const statusPath = await writeRuntimeHelperStatus( + { home: root, env }, + { + version: 1, + kind: "codex-app-runtime-rotation-helper", + state: "running", + pid: 2_147_483_647, + startedAt: Date.now(), + scriptPath: join(root, "runtime-helper.mjs"), + }, + ); + + await unbindCodexAppRuntimeRotation({ + platform: process.platform, + home: root, + env, + }); + + expect(existsSync(statusPath)).toBe(false); }); it("fails fast when the router script cannot be resolved", async () => { @@ -500,7 +1299,7 @@ describe("Codex app runtime rotation bind", () => { "const args = process.argv.slice(2);", "const statusPath = args[args.indexOf('--status') + 1];", "mkdirSync(dirname(statusPath), { recursive: true });", - "writeFileSync(statusPath, JSON.stringify({ version: 1, state: 'running', pid: process.pid, baseUrl: 'http://127.0.0.1:54321', updatedAt: Date.now() }) + '\\n', 'utf8');", + "writeFileSync(statusPath, JSON.stringify({ version: 1, state: 'running', pid: process.pid, startedAt: Date.now(), baseUrl: 'http://127.0.0.1:54321', updatedAt: Date.now() }) + '\\n', 'utf8');", "process.on('SIGTERM', () => process.exit(0));", "setInterval(() => undefined, 1000);", "", @@ -529,7 +1328,10 @@ describe("Codex app runtime rotation bind", () => { ); await unbindCodexAppRuntimeRotation({ - platform: "linux", + // The fixture intentionally exercises the POSIX bind path, but process + // identity probing must use the host platform when stopping its real + // detached child. + platform: process.platform, home: root, env, }); @@ -554,7 +1356,7 @@ describe("Codex app runtime rotation bind", () => { "const statusPath = args[args.indexOf('--status') + 1];", "setTimeout(() => {", " mkdirSync(dirname(statusPath), { recursive: true });", - " writeFileSync(statusPath, JSON.stringify({ version: 1, state: 'running', pid: process.pid, baseUrl: 'http://127.0.0.1:54322', updatedAt: Date.now() }) + '\\n', 'utf8');", + " writeFileSync(statusPath, JSON.stringify({ version: 1, state: 'running', pid: process.pid, startedAt: Date.now(), baseUrl: 'http://127.0.0.1:54322', updatedAt: Date.now() }) + '\\n', 'utf8');", "}, 2300);", "process.on('SIGTERM', () => process.exit(0));", "setInterval(() => undefined, 1000);", diff --git a/test/codex-app-router.test.ts b/test/codex-app-router.test.ts index d30f0b7d2..bc41f8f8c 100644 --- a/test/codex-app-router.test.ts +++ b/test/codex-app-router.test.ts @@ -132,6 +132,7 @@ describe("codex app router daemon", () => { (status) => status.state === "running", ); expect(running.kind).toBe("codex-app-runtime-rotation-router"); + expect(running.startedAt).toBeTypeOf("number"); expect(running.baseUrl).toBe("http://127.0.0.1:4567"); expect(running.lastAccountLabel).toBe("Account 2"); expect(running).not.toHaveProperty("clientApiKey"); diff --git a/test/codex-bin-wrapper.test.ts b/test/codex-bin-wrapper.test.ts index bfdb3293e..8c5e0bb68 100644 --- a/test/codex-bin-wrapper.test.ts +++ b/test/codex-bin-wrapper.test.ts @@ -25,7 +25,10 @@ import { import process from "node:process"; import { fileURLToPath, pathToFileURL } from "node:url"; import { afterEach, describe, expect, it } from "vitest"; -import { RUNTIME_ROTATION_PROXY_PROVIDER_ID } from "../lib/runtime-constants.js"; +import { + APP_RUNTIME_HELPER_OWNER_FILE, + RUNTIME_ROTATION_PROXY_PROVIDER_ID, +} from "../lib/runtime-constants.js"; import { sleep } from "../lib/utils.js"; import { resolveRealCodexBin } from "../scripts/codex-bin-resolver.js"; @@ -493,11 +496,10 @@ function createPathDiscoveredNativeCodexFixture(rootDir: string): { "utf8", ); const nativeExePath = join(binDir, "codex.exe"); - try { - linkSync(process.execPath, nativeExePath); - } catch { - copyFileSync(process.execPath, nativeExePath); - } + // A hard link to the test runner's node.exe cannot be removed on Windows + // while this process is still running. Use an independent image so the + // fixture teardown exercises the resolver without leaking a locked file. + copyFileSync(process.execPath, nativeExePath); return { binDir, args: [scriptPath, "--version"], @@ -2403,7 +2405,7 @@ describe("codex bin wrapper", () => { expect(existsSync(markerPath)).toBe(false); }); - it("starts an automatic runtime rotation helper for codex app launches", async () => { + it("starts an automatic helper and retries transient app-server shim file operations", async () => { const fixtureRoot = createWrapperFixture(); createRuntimeRotationProxyFixtureModule(fixtureRoot); const fakeBin = createCustomFakeCodexBin(fixtureRoot, [ @@ -2475,6 +2477,9 @@ describe("codex bin wrapper", () => { CODEX_MULTI_AUTH_RUNTIME_ROTATION_PROXY: "1", CODEX_MULTI_AUTH_APP_ROTATION_IDLE_MS: "1000", CODEX_MULTI_AUTH_TEST_PROXY_MARKER: markerPath, + CODEX_MULTI_AUTH_TEST_FORCE_APP_SERVER_SHIM_COPY: "1", + CODEX_MULTI_AUTH_TEST_APP_SERVER_SHIM_FILE_CLEANUP_BUSY_FAILURES: "2", + CODEX_MULTI_AUTH_TEST_APP_SERVER_SHIM_COPY_BUSY_FAILURES: "2", CODEX_MULTI_AUTH_TEST_PROXY_LAST_ACCOUNT_INDEX: "1", CODEX_MULTI_AUTH_TEST_PROXY_LAST_ACCOUNT_LABEL: "Account 2 (second@example.com, id:second)", @@ -2560,6 +2565,124 @@ describe("codex bin wrapper", () => { } }); + it("keeps concurrent app-helper owner metadata isolated by helper PID", async () => { + const fixtureRoot = createWrapperFixture(); + createRuntimeRotationProxyFixtureModule(fixtureRoot); + const fakeBin = createCustomFakeCodexBin(fixtureRoot, [ + "#!/usr/bin/env node", + "setTimeout(() => process.exit(0), 2000);", + ]); + const originalHome = join(fixtureRoot, "codex-home"); + const multiAuthDir = join(fixtureRoot, "multi-auth"); + mkdirSync(originalHome, { recursive: true }); + writeFileSync( + join(originalHome, "config.toml"), + 'model_provider = "openai"\n', + "utf8", + ); + + const children = [1, 2].map(() => + spawn( + process.execPath, + [join(fixtureRoot, "scripts", "codex.js"), "app", "."], + { + env: buildWrapperEnv({ + CODEX_MULTI_AUTH_REAL_CODEX_BIN: fakeBin, + CODEX_HOME: originalHome, + CODEX_MULTI_AUTH_DIR: multiAuthDir, + CODEX_MULTI_AUTH_RUNTIME_ROTATION_PROXY: "1", + CODEX_MULTI_AUTH_APP_ROTATION_IDLE_MS: "250", + }), + stdio: ["ignore", "pipe", "pipe"], + }, + ), + ); + for (const child of children) { + child.stdout?.resume(); + child.stderr?.resume(); + } + + const waitForClose = (child: (typeof children)[number]) => { + if (child.exitCode !== null || child.signalCode !== null) { + return Promise.resolve(); + } + return new Promise((resolve) => { + const timer = setTimeout(resolve, 2_000); + child.once("close", () => { + clearTimeout(timer); + resolve(); + }); + }); + }; + + let ownerFiles: string[] = []; + try { + const ownerFilePrefix = APP_RUNTIME_HELPER_OWNER_FILE.replace( + /\.json$/i, + "", + ); + const ownerFilePattern = new RegExp( + `^${ownerFilePrefix}\\.(\\d+)\\.json$`, + ); + const deadline = Date.now() + 5_000; + while (Date.now() < deadline) { + ownerFiles = existsSync(multiAuthDir) + ? readdirSync(multiAuthDir).filter((name) => + ownerFilePattern.test(name), + ) + : []; + if (ownerFiles.length >= 2) break; + await sleep(25); + } + + expect(ownerFiles).toHaveLength(2); + const ownerRecords = ownerFiles.map((name) => + JSON.parse(readFileSync(join(multiAuthDir, name), "utf8")), + ) as Array<{ + identityToken: string; + launcherPid: number; + }>; + expect(new Set(ownerRecords.map((owner) => owner.identityToken)).size).toBe( + 2, + ); + expect(new Set(ownerRecords.map((owner) => owner.launcherPid)).size).toBe(2); + expect( + existsSync(join(multiAuthDir, APP_RUNTIME_HELPER_OWNER_FILE)), + ).toBe(false); + } finally { + for (const child of children) { + try { + child.kill("SIGTERM"); + } catch { + // The wrapper may already have exited after a failed launch. + } + } + await Promise.all(children.map(waitForClose)); + + const helperPids = new Set(); + for (const name of ownerFiles) { + const match = /\.(\d+)\.json$/i.exec(name); + if (match?.[1]) helperPids.add(Number(match[1])); + } + for (const pid of helperPids) { + try { + process.kill(pid, "SIGTERM"); + } catch { + // The helper may have stopped with its launcher. + } + } + await sleep(500); + for (const pid of helperPids) { + if (!isProcessAlive(pid)) continue; + try { + process.kill(pid, "SIGKILL"); + } catch { + // Best-effort cleanup for the detached fixture. + } + } + } + }, 15_000); + it("sweeps stale app-server shim directories when a helper starts", async () => { const fixtureRoot = createWrapperFixture(); createRuntimeRotationProxyFixtureModule(fixtureRoot); diff --git a/test/codex-manager-cli.test.ts b/test/codex-manager-cli.test.ts index c0e6136ca..1112a025f 100644 --- a/test/codex-manager-cli.test.ts +++ b/test/codex-manager-cli.test.ts @@ -167,6 +167,30 @@ vi.mock("../lib/prompts/codex.js", () => ({ })); vi.mock("../lib/accounts.js", () => ({ + AccountManager: class MockAccountManager { + private readonly accounts: Array>; + + constructor( + _unusedStoragePath: unknown, + storage: { accounts: Array> }, + ) { + this.accounts = storage.accounts.map((account, index) => ({ + ...account, + index, + rateLimitResetTimes: account.rateLimitResetTimes ?? {}, + addedAt: account.addedAt ?? Date.now(), + lastUsed: account.lastUsed ?? Date.now(), + })); + } + + getAccountsSnapshot() { + return this.accounts; + } + + getManagedAccountRuntimeSkipReason(account: { enabled?: boolean }) { + return account.enabled === false ? "disabled" : null; + } + }, extractAccountEmail: vi.fn(() => undefined), extractAccountId: vi.fn(() => "acc_test"), formatAccountLabel: vi.fn((account: { email?: string }, index: number) => diff --git a/test/codex-manager-selection-diagnostics.test.ts b/test/codex-manager-selection-diagnostics.test.ts new file mode 100644 index 000000000..86036d60e --- /dev/null +++ b/test/codex-manager-selection-diagnostics.test.ts @@ -0,0 +1,234 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { + AUTH_INVALIDATION_MARKER, + AccountManager, +} from "../lib/accounts.js"; +import { buildSelectAccountTraced } from "../lib/codex-manager.js"; +import { clearCircuitBreakers } from "../lib/circuit-breaker.js"; +import { resetTrackers } from "../lib/rotation.js"; +import type { AccountStorageV3 } from "../lib/storage.js"; + +const { evaluateRuntimePolicyMock, loadRuntimePolicyStateMock } = vi.hoisted( + () => ({ + evaluateRuntimePolicyMock: vi.fn(), + loadRuntimePolicyStateMock: vi.fn(), + }), +); + +vi.mock("../lib/policy/runtime-policy.js", async (importOriginal) => { + const actual = await importOriginal< + typeof import("../lib/policy/runtime-policy.js") + >(); + return { + ...actual, + evaluateRuntimePolicy: evaluateRuntimePolicyMock, + loadRuntimePolicyState: loadRuntimePolicyStateMock, + }; +}); + +function createStorage(now: number): AccountStorageV3 { + return { + version: 3, + activeIndex: 0, + activeIndexByFamily: { codex: 0 }, + accounts: [ + { + accountId: "policy-paused", + email: "paused@example.com", + refreshToken: "refresh-paused", + addedAt: now, + lastUsed: now, + }, + { + accountId: "workspace-disabled", + email: "workspace@example.com", + refreshToken: "refresh-workspace", + addedAt: now, + lastUsed: now, + workspaces: [ + { id: "workspace-a", name: "Workspace A", enabled: false }, + ], + }, + { + accountId: "model-limited", + email: "model@example.com", + refreshToken: "refresh-model", + addedAt: now, + lastUsed: now, + rateLimitResetTimes: { + "codex:gpt-5.3-codex": now + 60_000, + }, + }, + { + accountId: "cooling-down", + email: "cooldown@example.com", + refreshToken: "refresh-cooldown", + addedAt: now, + lastUsed: now, + coolingDownUntil: now + 60_000, + cooldownReason: "network-error", + }, + { + accountId: "token-invalidated", + email: "invalid@example.com", + refreshToken: "refresh-invalid", + addedAt: now, + lastUsed: now, + authInvalidatedAt: now - 1, + authInvalidationErrorCode: "token_invalidated", + }, + { + accountId: "circuit-open", + email: "circuit@example.com", + refreshToken: "refresh-circuit", + addedAt: now, + lastUsed: now, + }, + { + accountId: "healthy", + email: "healthy@example.com", + refreshToken: "refresh-healthy", + addedAt: now, + lastUsed: now, + }, + ], + }; +} + +describe("codex-manager selection diagnostics", () => { + beforeEach(() => { + resetTrackers(); + clearCircuitBreakers(); + loadRuntimePolicyStateMock.mockResolvedValue({ + accountPolicies: { version: 1, accounts: {} }, + budgets: { version: 1, limits: {} }, + project: { + startDir: "/tmp", + projectRoot: null, + identityRoot: "/tmp", + projectKey: null, + profile: null, + }, + }); + evaluateRuntimePolicyMock.mockResolvedValue({ + allowed: true, + statusCode: 200, + errorCode: null, + reasons: [], + projectKey: null, + blockedAccountIndexes: new Set([0]), + blockedAccountReasons: { 0: "policy: paused" }, + scoreBoostByAccount: { 6: 25 }, + budgetEvaluations: [], + }); + }); + + it("uses the same runtime gates and policy overlays as production selection", async () => { + const now = Date.now(); + const storage = createStorage(now); + const circuitManager = new AccountManager(undefined, storage); + const circuitAccount = circuitManager.getAccountByIndex(5); + if (!circuitAccount) throw new Error("circuit fixture account missing"); + circuitManager.recordFailure(circuitAccount, "codex", "gpt-5.3-codex"); + circuitManager.recordFailure(circuitAccount, "codex", "gpt-5.3-codex"); + circuitManager.recordFailure(circuitAccount, "codex", "gpt-5.3-codex"); + + const trace = await buildSelectAccountTraced()(storage); + const candidates = new Map( + trace.candidates.map((candidate) => [candidate.index, candidate]), + ); + + expect(candidates.get(0)).toMatchObject({ + isAvailable: false, + reason: "policy: paused", + }); + expect(candidates.get(1)).toMatchObject({ + isAvailable: false, + reason: "workspace-disabled", + }); + expect(candidates.get(2)).toMatchObject({ + isAvailable: false, + reason: "rate-limited", + }); + expect(candidates.get(3)).toMatchObject({ + isAvailable: false, + reason: "cooling-down:network-error", + }); + expect(candidates.get(4)).toMatchObject({ + isAvailable: false, + reason: AUTH_INVALIDATION_MARKER, + }); + expect(candidates.get(5)).toMatchObject({ + isAvailable: false, + reason: "circuit-open", + }); + expect(candidates.get(6)).toMatchObject({ + isAvailable: true, + capabilityBoost: 25, + }); + expect(trace.availableCount).toBe(1); + expect(trace.selected?.index).toBe(6); + }); + + it("blocks every candidate when a model policy denies the request globally", async () => { + evaluateRuntimePolicyMock.mockResolvedValueOnce({ + allowed: false, + statusCode: 403, + errorCode: "model_not_allowed", + reasons: ["model denied"], + projectKey: null, + blockedAccountIndexes: new Set(), + blockedAccountReasons: {}, + scoreBoostByAccount: {}, + budgetEvaluations: [], + }); + + const storage = createStorage(Date.now()); + const trace = await buildSelectAccountTraced()(storage); + + expect(trace.selected).toBeNull(); + expect(trace.availableCount).toBe(0); + expect(trace.candidates.every((candidate) => candidate.isAvailable === false)).toBe( + true, + ); + expect( + trace.candidates + .filter((candidate) => candidate.index !== 4) + .every((candidate) => candidate.reason === "model_not_allowed"), + ).toBe(true); + expect(trace.candidates.find((candidate) => candidate.index === 4)?.reason).toBe( + AUTH_INVALIDATION_MARKER, + ); + }); + + it("blocks every candidate when a global budget guard denies the request", async () => { + evaluateRuntimePolicyMock.mockResolvedValueOnce({ + allowed: false, + statusCode: 429, + errorCode: "budget_exceeded", + reasons: ["global budget exhausted"], + projectKey: null, + blockedAccountIndexes: new Set(), + blockedAccountReasons: {}, + scoreBoostByAccount: {}, + budgetEvaluations: [], + }); + + const storage = createStorage(Date.now()); + const trace = await buildSelectAccountTraced()(storage); + + expect(trace.selected).toBeNull(); + expect(trace.availableCount).toBe(0); + expect(trace.candidates.every((candidate) => candidate.isAvailable === false)).toBe( + true, + ); + expect( + trace.candidates + .filter((candidate) => candidate.index !== 4) + .every((candidate) => candidate.reason === "budget_exceeded"), + ).toBe(true); + expect(trace.candidates.find((candidate) => candidate.index === 4)?.reason).toBe( + AUTH_INVALIDATION_MARKER, + ); + }); +}); diff --git a/test/codex-manager-status-command.test.ts b/test/codex-manager-status-command.test.ts index 848507842..f4b10a1b8 100644 --- a/test/codex-manager-status-command.test.ts +++ b/test/codex-manager-status-command.test.ts @@ -8,6 +8,7 @@ import { import { runCodexMultiAuthCli } from "../lib/codex-manager.js"; import type { AccountStorageV3, StorageHealthSummary } from "../lib/storage.js"; import type { RuntimeObservabilitySnapshot } from "../lib/runtime/runtime-observability.js"; +import { AUTH_INVALIDATION_MARKER } from "../lib/accounts.js"; function createStorage(): AccountStorageV3 { return { @@ -209,6 +210,27 @@ describe("runStatusCommand", () => { ); }); + it("surfaces the persisted token invalidation marker", async () => { + const deps = createStatusDeps({ + loadAccounts: vi.fn(async () => ({ + ...createStorage(), + accounts: [ + { + ...createStorage().accounts[0], + authInvalidatedAt: 1_000, + authInvalidationErrorCode: "token_invalidated", + }, + ], + })), + }); + + await runStatusCommand(deps); + + expect(deps.logInfo).toHaveBeenCalledWith( + expect.stringContaining(AUTH_INVALIDATION_MARKER), + ); + }); + it("prints the last rotated runtime account when observability has it", async () => { const deps = createStatusDeps({ loadRuntimeObservabilitySnapshot: vi.fn(async () => diff --git a/test/flagged-storage.test.ts b/test/flagged-storage.test.ts index 73458cf05..a4bc30c3c 100644 --- a/test/flagged-storage.test.ts +++ b/test/flagged-storage.test.ts @@ -64,4 +64,26 @@ describe("flagged storage helper", () => { expect(result.accounts[0]?.cooldownReason).toBe("server-error"); }); + + it.each([0, -1, Number.NaN, Number.POSITIVE_INFINITY])( + "removes malformed auth invalidation fields for timestamp %s", + (timestamp) => { + const result = normalizeFlaggedStorage( + { + version: 1, + accounts: [ + { + refreshToken: "token-1", + authInvalidatedAt: timestamp, + authInvalidationErrorCode: "oauth_token_revoked", + }, + ], + }, + { isRecord, now: () => 99 }, + ); + + expect(result.accounts[0]?.authInvalidatedAt).toBeUndefined(); + expect(result.accounts[0]?.authInvalidationErrorCode).toBeUndefined(); + }, + ); }); diff --git a/test/login-oauth-callback-guidance.test.ts b/test/login-oauth-callback-guidance.test.ts index d1c6b7ddc..9eae6e99d 100644 --- a/test/login-oauth-callback-guidance.test.ts +++ b/test/login-oauth-callback-guidance.test.ts @@ -5,17 +5,42 @@ * failure reason and forwards the bind error, which is what actually makes the * Windows/WSL conflict legible to the user. * - * Under vitest neither stdin nor stdout is a TTY, so the manual-paste prompt - * short-circuits to `cancelled` in browser mode and the flow returns without - * blocking on input. + * The manual/incognito regression below supplies a fake readline prompt so it + * can exercise the same callback validation and exchange seam without a TTY. */ import { beforeEach, describe, expect, it, vi } from "vitest"; const { hooks } = vi.hoisted(() => ({ - hooks: { - serverInfo: null as unknown, - guidanceLines: [] as string[], - }, + hooks: { + serverInfo: null as unknown, + guidanceLines: [] as string[], + manualInput: "", + exchangeAuthorizationCode: vi.fn(), + openBrowserUrl: true, + copyTextToClipboard: true, + }, + })); + +vi.mock("node:process", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + stdin: { + isTTY: false, + readableEnded: false, + destroyed: false, + once: vi.fn(), + off: vi.fn(), + }, + stdout: { isTTY: false }, + }; +}); + +vi.mock("node:readline/promises", () => ({ + createInterface: vi.fn(() => ({ + question: vi.fn(async () => hooks.manualInput), + close: vi.fn(), + })), })); vi.mock("../lib/auth/auth.js", async (importOriginal) => { @@ -25,8 +50,10 @@ vi.mock("../lib/auth/auth.js", async (importOriginal) => { createAuthorizationFlow: vi.fn(async () => ({ pkce: { challenge: "challenge", verifier: "verifier" }, state: "test-state", - url: "https://auth.openai.com/oauth/authorize?state=test-state", + url: + "https://auth.openai.com/oauth/authorize?state=test-state&code_challenge=challenge", })), + exchangeAuthorizationCode: hooks.exchangeAuthorizationCode, }; }); @@ -35,8 +62,8 @@ vi.mock("../lib/auth/server.js", () => ({ })); vi.mock("../lib/auth/browser.js", () => ({ - openBrowserUrl: vi.fn(() => true), - copyTextToClipboard: vi.fn(() => true), + openBrowserUrl: vi.fn(() => hooks.openBrowserUrl), + copyTextToClipboard: vi.fn(() => hooks.copyTextToClipboard), isBrowserLaunchSuppressed: vi.fn(() => false), getBrowserOpener: vi.fn(() => "xdg-open"), })); @@ -80,6 +107,10 @@ describe("runOAuthFlow callback-failure guidance", () => { vi.clearAllMocks(); logged = []; hooks.guidanceLines = ["GUIDANCE LINE ONE", "", "GUIDANCE LINE TWO"]; + hooks.manualInput = ""; + hooks.exchangeAuthorizationCode.mockReset(); + hooks.openBrowserUrl = true; + hooks.copyTextToClipboard = true; vi.spyOn(console, "log").mockImplementation((...args: unknown[]) => { logged.push(args.map(String).join(" ")); }); @@ -121,9 +152,40 @@ describe("runOAuthFlow callback-failure guidance", () => { expect(logged).toContain(""); }); - // Manual mode is not exercised here: it sets `allowNonTty`, so the prompt - // blocks reading stdin rather than short-circuiting, and faking that stream - // would test the harness more than the code. The guidance is gated on - // `signInMode === "browser"` in one place, and the three cases above pin the - // branch that actually selects the reason. + it("prints the complete authorization URL for manual/incognito login", async () => { + const authorizationUrl = + "https://auth.openai.com/oauth/authorize?state=test-state&code_challenge=challenge"; + hooks.manualInput = + "http://localhost:1455/auth/callback?code=callback-code&state=test-state"; + hooks.exchangeAuthorizationCode.mockResolvedValue({ + type: "success", + access: "access-token", + refresh: "refresh-token", + expires: Date.now() + 60_000, + }); + + await expect(runOAuthFlow(false, "manual")).resolves.toMatchObject({ + type: "success", + }); + + expect(logged.join("\n")).toContain(authorizationUrl); + expect(logged.join("\n")).not.toContain("%3Credacted%3E"); + expect(hooks.exchangeAuthorizationCode).toHaveBeenCalledWith( + "callback-code", + "verifier", + "http://localhost:1455/auth/callback", + ); + }); + + it("prints a usable URL when browser and clipboard fallbacks both fail", async () => { + hooks.serverInfo = serverThatTimesOut(); + hooks.openBrowserUrl = false; + hooks.copyTextToClipboard = false; + + await runOAuthFlow(false, "browser"); + + expect(logged.join("\n")).toContain( + "https://auth.openai.com/oauth/authorize?state=test-state&code_challenge=challenge", + ); + }); }); diff --git a/test/postinstall.test.ts b/test/postinstall.test.ts new file mode 100644 index 000000000..54bab148b --- /dev/null +++ b/test/postinstall.test.ts @@ -0,0 +1,24 @@ +import { describe, expect, it, vi } from "vitest"; +import { INSTALL_NOTICE, runPostinstall } from "../scripts/postinstall.js"; + +describe("postinstall notice", () => { + it("prints package removal guidance in an interactive install", () => { + const log = vi.fn(); + + expect( + runPostinstall({ env: {}, isTty: true, log }), + ).toBe(0); + expect(log).toHaveBeenCalledWith(INSTALL_NOTICE); + expect(INSTALL_NOTICE).toContain("npm uninstall -g codex-multi-auth"); + expect(INSTALL_NOTICE).toContain("@ndycode/codex-multi-auth"); + }); + + it("stays silent in CI", () => { + const log = vi.fn(); + + expect( + runPostinstall({ env: { CI: "1" }, isTty: true, log }), + ).toBe(0); + expect(log).not.toHaveBeenCalled(); + }); +}); diff --git a/test/preemptive-quota-scheduler.test.ts b/test/preemptive-quota-scheduler.test.ts index f2dab677c..6839fee9f 100644 --- a/test/preemptive-quota-scheduler.test.ts +++ b/test/preemptive-quota-scheduler.test.ts @@ -257,6 +257,40 @@ describe("preemptive quota scheduler", () => { }); }); + it("falls back to the configured cap for a future-dated snapshot", () => { + const maxDeferralMs = 30 * 60_000; + const scheduler = new PreemptiveQuotaScheduler({ maxDeferralMs }); + const now = 1_000_000; + scheduler.update("acc:model", { + status: 200, + primary: { usedPercent: 100, resetAtMs: now + 60 * 60_000 }, + secondary: {}, + updatedAt: now + 1, + }); + + expect(scheduler.getDeferral("acc:model", now)).toEqual({ + defer: true, + waitMs: maxDeferralMs, + reason: "quota-near-exhaustion", + }); + }); + + it("does not re-defer an exhausted window after its reset has passed", () => { + const scheduler = new PreemptiveQuotaScheduler({ maxDeferralMs: 30 * 60_000 }); + const now = 1_000_000; + scheduler.update("acc:model", { + status: 200, + primary: { usedPercent: 100, resetAtMs: now - 1_000 }, + secondary: { usedPercent: 20, resetAtMs: now + 7 * 24 * 60 * 60_000 }, + updatedAt: now - 500, + }); + + expect(scheduler.getDeferral("acc:model", now)).toEqual({ + defer: false, + waitMs: 0, + }); + }); + it("prunes expired snapshots", () => { const scheduler = new PreemptiveQuotaScheduler(); scheduler.update("a", { diff --git a/test/preuninstall.test.ts b/test/preuninstall.test.ts index 78daccab9..137ef0344 100644 --- a/test/preuninstall.test.ts +++ b/test/preuninstall.test.ts @@ -6,10 +6,32 @@ import { runPreuninstallCleanup } from "../scripts/preuninstall.js"; import { resolveInstallPaths } from "../scripts/install-codex-auth-utils.js"; import { removeWithRetry } from "./helpers/remove-with-retry.js"; +let injectPreuninstallEbusyOnNextRm = false; + +vi.mock("node:fs/promises", async () => { + const actual: typeof import("node:fs/promises") = await vi.importActual( + "node:fs/promises", + ); + return { + ...actual, + rm: vi.fn(async (...args: Parameters) => { + if (injectPreuninstallEbusyOnNextRm) { + injectPreuninstallEbusyOnNextRm = false; + const error = Object.assign(new Error("EBUSY: cache is busy"), { + code: "EBUSY", + }); + throw error; + } + return actual.rm(...args); + }), + }; +}); + const tempRoots: string[] = []; afterEach(async () => { vi.restoreAllMocks(); + injectPreuninstallEbusyOnNextRm = false; while (tempRoots.length > 0) { const root = tempRoots.pop(); if (root) await removeWithRetry(root, { recursive: true, force: true }); @@ -68,6 +90,40 @@ describe("runPreuninstallCleanup", () => { expect(calls).toEqual([]); }); + it("removes current and legacy caches with transient retryable failures", async () => { + const home = makeTempHome(); + const env = envFor(home); + const paths = resolveTempPaths(home); + mkdirSync(paths.configDir, { recursive: true }); + mkdirSync(paths.cacheNodeModules, { recursive: true }); + mkdirSync(paths.cacheLegacyNodeModules, { recursive: true }); + writeFileSync( + paths.configPath, + JSON.stringify({ plugins: ["codex-multi-auth"] }, null, "\t") + "\n", + "utf8", + ); + writeFileSync(path.join(paths.cacheNodeModules, "current.txt"), "current", "utf8"); + writeFileSync( + path.join(paths.cacheLegacyNodeModules, "legacy.txt"), + "legacy", + "utf8", + ); + + injectPreuninstallEbusyOnNextRm = true; + const code = await runPreuninstallCleanup({ + env, + home, + log: () => {}, + unbindCodexApp: async () => {}, + removeLauncher: async () => {}, + }); + + expect(code).toBe(0); + expect(injectPreuninstallEbusyOnNextRm).toBe(false); + expect(existsSync(paths.cacheNodeModules)).toBe(false); + expect(existsSync(paths.cacheLegacyNodeModules)).toBe(false); + }); + it("preserves bun.lock when other plugins remain after removal", async () => { const home = makeTempHome(); const env = envFor(home); diff --git a/test/rotation-proxy-state.test.ts b/test/rotation-proxy-state.test.ts index 4ca9a2acd..08487e8c9 100644 --- a/test/rotation-proxy-state.test.ts +++ b/test/rotation-proxy-state.test.ts @@ -1,5 +1,6 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { AccountManager } from "../lib/accounts.js"; +import { PreemptiveQuotaScheduler } from "../lib/preemptive-quota-scheduler.js"; import type { RotationProxyStateInit } from "../lib/runtime/rotation-proxy-state.js"; import type { AccountStorageV3 } from "../lib/storage.js"; @@ -63,6 +64,7 @@ function stateInit(): RotationProxyStateInit { maxRuntimeAccountAttempts: 3, maxRequestBodyBytes: 1024, quotaRemainingPercentThreshold: 0, + preemptiveQuotaScheduler: new PreemptiveQuotaScheduler(), sessionAffinityStore: null, lastObservedAffinityGeneration: 0, forcedAccountIndex: null, diff --git a/test/rotation-token-refresh.test.ts b/test/rotation-token-refresh.test.ts index 2ea5b507f..9404af3fa 100644 --- a/test/rotation-token-refresh.test.ts +++ b/test/rotation-token-refresh.test.ts @@ -38,7 +38,10 @@ const FAMILY = "gpt-5-codex" as const; const SKEW_MS = 30_000; const INVALIDATION_COOLDOWN_MS = 300_000; -function storageWith(expiresAt: number): AccountStorageV3 { +function storageWith( + expiresAt: number, + accountOverrides: Partial = {}, +): AccountStorageV3 { return { version: 3, activeIndex: 0, @@ -53,6 +56,7 @@ function storageWith(expiresAt: number): AccountStorageV3 { addedAt: NOW - 60_000, lastUsed: NOW - 60_000, enabled: true, + ...accountOverrides, }, ], }; @@ -64,8 +68,14 @@ const STALE_EXPIRES = NOW + 10_000; const openManagers: AccountManager[] = []; -function managerWith(expiresAt: number): AccountManager { - const accountManager = new AccountManager(undefined, storageWith(expiresAt)); +function managerWith( + expiresAt: number, + accountOverrides: Partial = {}, +): AccountManager { + const accountManager = new AccountManager( + undefined, + storageWith(expiresAt, accountOverrides), + ); openManagers.push(accountManager); return accountManager; } @@ -245,6 +255,51 @@ describe("ensureFreshAccessToken", () => { expect(coolingDownUntil).toBeGreaterThan( NOW + INVALIDATION_COOLDOWN_MS - 10_000, ); + expect(accountManager.getAccountByIndex(0)).toMatchObject({ + authInvalidatedAt: expect.any(Number), + authInvalidationErrorCode: "token_invalidated", + }); + expect( + accountManager.isAccountAvailableForFamily(0, FAMILY, "gpt-5-codex"), + ).toBe(false); + }); + + it("persists the invalidation marker until a successful refresh clears it", async () => { + let persisted: AccountStorageV3 | undefined; + withAccountStorageTransactionMock.mockImplementation(async (handler) => + handler(null, async (nextStorage: AccountStorageV3) => { + persisted = structuredClone(nextStorage); + }), + ); + const accountManager = managerWith(STALE_EXPIRES); + queuedRefreshMock.mockResolvedValue({ + type: "failed", + reason: "http_error", + statusCode: 401, + message: "OAuth token has been invalidated", + }); + + await ensureFreshAccessToken(refreshParams(accountManager)); + await accountManager.flushPendingSave(); + + expect(persisted?.accounts[0]).toMatchObject({ + authInvalidatedAt: expect.any(Number), + authInvalidationErrorCode: "token_invalidated", + }); + + queuedRefreshMock.mockResolvedValue({ + type: "success", + access: "access-recovered", + refresh: "refresh-recovered", + expires: NOW + 7_200_000, + }); + const refreshed = await ensureFreshAccessToken(refreshParams(accountManager)); + + expect(refreshed).toMatchObject({ ok: true, accessToken: "access-recovered" }); + expect(accountManager.getAccountByIndex(0)?.authInvalidatedAt).toBeUndefined(); + expect( + accountManager.getAccountByIndex(0)?.authInvalidationErrorCode, + ).toBeUndefined(); }); it("never lets a later generic failure truncate an invalidation cooldown", async () => { diff --git a/test/runtime-rotation-proxy.test.ts b/test/runtime-rotation-proxy.test.ts index 5abe1e330..a892bf923 100644 --- a/test/runtime-rotation-proxy.test.ts +++ b/test/runtime-rotation-proxy.test.ts @@ -6,10 +6,12 @@ import { HTTP_STATUS, OPENAI_HEADERS } from "../lib/constants.js"; import { startRuntimeRotationProxy, buildTokenInvalidationBody, + buildQuotaScheduleKey, chooseAccount, normalizeForcedAccountIndex, type RuntimeRotationProxyServer, } from "../lib/runtime-rotation-proxy.js"; +import { PreemptiveQuotaScheduler } from "../lib/preemptive-quota-scheduler.js"; import { SessionAffinityStore } from "../lib/session-affinity.js"; import { clearCircuitBreakers } from "../lib/circuit-breaker.js"; import { @@ -1412,6 +1414,193 @@ describe("runtime rotation proxy", () => { ).toBeTypeOf("number"); }); + it("uses the preemptive scheduler fallback when exhaustion has no reset header", async () => { + const now = Date.now(); + const accountManager = new AccountManager(undefined, createStorage(now)); + const { calls, fetchImpl } = createRecordingFetch((_call, attempt) => + textEventStream(`data: attempt-${attempt}\n\n`, { + "x-codex-primary-used-percent": attempt === 1 ? "100" : "10", + }), + ); + const proxy = await startProxy({ accountManager, fetchImpl }); + + await (await postResponses(proxy, { model: "gpt-5-codex", stream: true })).text(); + await (await postResponses(proxy, { model: "gpt-5-codex", stream: true })).text(); + + expect(calls.map((call) => call.headers.get(OPENAI_HEADERS.ACCOUNT_ID))).toEqual([ + "acc_1", + "acc_2", + ]); + expect( + (accountManager.getAccountByIndex(0)?.rateLimitResetTimes["gpt-5-codex"] ?? 0) - + Date.now(), + ).toBeGreaterThan(60 * 60 * 1_000); + }); + + it("keeps quota snapshots attached to account identity across index changes", () => { + const scheduler = new PreemptiveQuotaScheduler(); + const now = Date.now(); + const originalAccount = { + accountId: "stable-account", + email: "stable@example.com", + refreshToken: "refresh-stable", + }; + const reorderedAccount = { ...originalAccount }; + const replacementAccount = { + accountId: "different-account", + email: "different@example.com", + refreshToken: "refresh-different", + }; + const originalKey = buildQuotaScheduleKey( + originalAccount, + "codex", + "gpt-5-codex", + ); + scheduler.update(originalKey, { + status: 200, + primary: { usedPercent: 100, resetAtMs: now + 60 * 60_000 }, + secondary: {}, + updatedAt: now, + }); + + expect( + buildQuotaScheduleKey(reorderedAccount, "codex", "gpt-5-codex"), + ).toBe(originalKey); + expect( + scheduler.getDeferral( + buildQuotaScheduleKey(reorderedAccount, "codex", "gpt-5-codex"), + now, + ), + ).toMatchObject({ defer: true, reason: "quota-near-exhaustion" }); + expect( + scheduler.getDeferral( + buildQuotaScheduleKey(replacementAccount, "codex", "gpt-5-codex"), + now, + ), + ).toEqual({ defer: false, waitMs: 0 }); + }); + + it("keeps quota identity stable when an account gains an account id", () => { + const beforeRefresh = { + email: " User@Example.com ", + refreshToken: "refresh-before", + }; + const afterRefresh = { + email: "user@example.com", + accountId: "account-id-after-refresh", + refreshToken: "refresh-after", + }; + + expect( + buildQuotaScheduleKey(beforeRefresh, "codex", "gpt-5-codex"), + ).toBe(buildQuotaScheduleKey(afterRefresh, "codex", "gpt-5-codex")); + }); + + it("does not merge quota identity for distinct emails sharing an account id", () => { + const first = { + email: "first@example.com", + accountId: "shared-account-id", + refreshToken: "refresh-first", + }; + const second = { + email: "second@example.com", + accountId: "shared-account-id", + refreshToken: "refresh-second", + }; + + expect(buildQuotaScheduleKey(first, "codex", "gpt-5-codex")).not.toBe( + buildQuotaScheduleKey(second, "codex", "gpt-5-codex"), + ); + }); + + it("does not merge quota identity for distinct accounts sharing a normalized email", () => { + const first = { + email: "Shared@example.com", + accountId: "account-one", + refreshToken: "refresh-one", + addedAt: 1_000, + recordId: "record-one", + }; + const second = { + email: " shared@example.com ", + accountId: "account-two", + refreshToken: "refresh-two", + addedAt: 1_000, + recordId: "record-two", + }; + const firstKey = buildQuotaScheduleKey(first, "codex", "gpt-5-codex"); + const secondKey = buildQuotaScheduleKey(second, "codex", "gpt-5-codex"); + const scheduler = new PreemptiveQuotaScheduler(); + + scheduler.update(firstKey, { + status: 200, + primary: { usedPercent: 100, resetAtMs: Date.now() + 60 * 60_000 }, + secondary: {}, + updatedAt: Date.now(), + }); + + expect(firstKey).not.toBe(secondKey); + expect(scheduler.getDeferral(secondKey, Date.now())).toEqual({ + defer: false, + waitMs: 0, + }); + }); + + it("derives distinct stable quota record ids for same-email records", () => { + const now = Date.now(); + const accountManager = new AccountManager(undefined, { + version: 3, + activeIndex: 0, + activeIndexByFamily: { codex: 0 }, + accounts: [ + { + email: "shared@example.com", + accountId: "same-account-id", + refreshToken: "refresh-one", + addedAt: now, + lastUsed: now, + }, + { + email: "SHARED@example.com", + accountId: "same-account-id", + refreshToken: "refresh-two", + addedAt: now, + lastUsed: now, + }, + ], + }); + const first = accountManager.getAccountByIndex(0); + const second = accountManager.getAccountByIndex(1); + + expect(first?.recordId).toBeTypeOf("string"); + expect(first?.recordId).not.toBe(second?.recordId); + expect( + first && second + ? buildQuotaScheduleKey(first, "codex", "gpt-5-codex") + : null, + ).not.toBe( + first && second + ? buildQuotaScheduleKey(second, "codex", "gpt-5-codex") + : null, + ); + }); + + it("normalizes email casing and whitespace in quota identity keys", () => { + expect( + buildQuotaScheduleKey( + { email: " MixedCase@Example.COM ", refreshToken: "refresh-a" }, + "codex", + "gpt-5-codex", + ), + ).toBe( + buildQuotaScheduleKey( + { email: "mixedcase@example.com", refreshToken: "refresh-b" }, + "codex", + "gpt-5-codex", + ), + ); + }); + it("pins repeated session requests to the first served account", async () => { const now = Date.now(); const accountManager = new AccountManager(undefined, createStorage(now, 3)); @@ -2377,6 +2566,10 @@ describe("runtime rotation proxy", () => { expect(calls).toHaveLength(1); expect(calls[0]?.headers.get(OPENAI_HEADERS.ACCOUNT_ID)).toBe("acc_1"); expect(accountManager.getAccountByIndex(0)?.cooldownReason).toBe("auth-failure"); + expect(accountManager.getAccountByIndex(0)).toMatchObject({ + authInvalidatedAt: expect.any(Number), + authInvalidationErrorCode: "token_invalidated", + }); // token invalidation applies the long cooldown (~5min), not the generic 30s const coolingDownUntil = accountManager.getAccountByIndex(0)?.coolingDownUntil ?? 0; expect(coolingDownUntil).toBeGreaterThan(now + 250_000); @@ -2384,6 +2577,53 @@ describe("runtime rotation proxy", () => { expect(proxy.getStatus().rotations).toBe(0); }); + it("persists a distinct upstream invalidation code from a 401 body", async () => { + const now = Date.now(); + const accountManager = new AccountManager(undefined, createStorage(now, 2)); + let persistedStorage: AccountStorageV3 | null = null; + withAccountStorageTransactionMock.mockImplementationOnce( + async ( + handler: ( + current: AccountStorageV3 | null, + persist: (storage: AccountStorageV3) => Promise, + ) => Promise, + ) => { + await handler(null, async (storage) => { + persistedStorage = structuredClone(storage); + }); + }, + ); + const invalidationBody = JSON.stringify({ + error: { + code: "oauth_token_revoked", + message: "The OAuth token has been invalidated by the provider.", + }, + }); + const { calls, fetchImpl } = createRecordingFetch( + () => + new Response(invalidationBody, { + status: HTTP_STATUS.UNAUTHORIZED, + headers: { "content-type": "application/json" }, + }), + ); + const proxy = await startProxy({ accountManager, fetchImpl }); + + const response = await postResponses(proxy, { model: "gpt-5-codex" }); + + expect(response.status).toBe(HTTP_STATUS.UNAUTHORIZED); + expect(calls).toHaveLength(1); + expect(accountManager.getAccountByIndex(0)).toMatchObject({ + authInvalidatedAt: expect.any(Number), + authInvalidationErrorCode: "oauth_token_revoked", + }); + expect(persistedStorage).not.toBeNull(); + const reloadedManager = new AccountManager(undefined, persistedStorage); + expect(reloadedManager.getAccountByIndex(0)).toMatchObject({ + authInvalidatedAt: expect.any(Number), + authInvalidationErrorCode: "oauth_token_revoked", + }); + }); + it("rotates to next account on a generic 401 that is not a token invalidation", async () => { const now = Date.now(); const accountManager = new AccountManager(undefined, createStorage(now, 2)); @@ -2460,6 +2700,10 @@ describe("runtime rotation proxy", () => { expect(calls).toHaveLength(0); const coolingDownUntil = accountManager.getAccountByIndex(0)?.coolingDownUntil ?? 0; expect(coolingDownUntil).toBeGreaterThan(now + 250_000); + expect(accountManager.getAccountByIndex(0)).toMatchObject({ + authInvalidatedAt: expect.any(Number), + authInvalidationErrorCode: "token_invalidated", + }); expect(proxy.getStatus().rotations).toBe(0); // session affinity cleared — next request with same session routes to healthy account const followUp = await postResponses(proxy, bodyWithSession); diff --git a/test/schemas.test.ts b/test/schemas.test.ts index a5a31597c..76a204f8a 100644 --- a/test/schemas.test.ts +++ b/test/schemas.test.ts @@ -205,6 +205,20 @@ describe("AccountMetadataV3Schema", () => { expect(result.success).toBe(true); }); + it.each([ + ["zero", 0], + ["negative", -1], + ["NaN", Number.NaN], + ["infinity", Number.POSITIVE_INFINITY], + ] as const)("rejects %s auth invalidation timestamps", (_label, timestamp) => { + const result = AccountMetadataV3Schema.safeParse({ + ...validAccount, + authInvalidatedAt: timestamp, + authInvalidationErrorCode: "oauth_token_revoked", + }); + expect(result.success).toBe(false); + }); + it("rejects empty refreshToken", () => { const result = AccountMetadataV3Schema.safeParse({ ...validAccount, diff --git a/test/storage.test.ts b/test/storage.test.ts index 894c30400..25415bad1 100644 --- a/test/storage.test.ts +++ b/test/storage.test.ts @@ -4,7 +4,12 @@ import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { getConfigDir, getProjectStorageKey } from "../lib/storage/paths.js"; -import { setStoragePathState } from "../lib/storage/path-state.js"; +import { + getStoragePathState, + runWithStoragePathState, + setStoragePathDirectState, + setStoragePathState, +} from "../lib/storage/path-state.js"; import { getIntentionalResetMarkerPath } from "../lib/storage/backup-paths.js"; import { getRuntimeAccountIdentityKey } from "../lib/storage/identity.js"; import { removeWithRetry } from "./helpers/remove-with-retry.js"; @@ -42,6 +47,43 @@ describe("storage", () => { const _origCODEX_HOME = process.env.CODEX_HOME; const _origCODEX_MULTI_AUTH_DIR = process.env.CODEX_MULTI_AUTH_DIR; + it("keeps a direct path override ahead of a stale async context", async () => { + const staleState = { + currentStoragePath: "stale-storage.json", + currentLegacyProjectStoragePath: null, + currentLegacyWorktreeStoragePath: null, + currentProjectRoot: null, + }; + const directState = { + currentStoragePath: "direct-storage.json", + currentLegacyProjectStoragePath: null, + currentLegacyWorktreeStoragePath: null, + currentProjectRoot: null, + }; + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + + try { + const staleRead = runWithStoragePathState(staleState, async () => { + await gate; + return getStoragePathState(); + }); + await Promise.resolve(); + setStoragePathDirectState(directState); + release(); + await expect(staleRead).resolves.toEqual(directState); + } finally { + setStoragePathState({ + currentStoragePath: null, + currentLegacyProjectStoragePath: null, + currentLegacyWorktreeStoragePath: null, + currentProjectRoot: null, + }); + } + }); + beforeEach(() => { delete process.env.CODEX_HOME; delete process.env.CODEX_MULTI_AUTH_DIR; @@ -2560,6 +2602,27 @@ describe("storage", () => { expect(result?.accounts).toHaveLength(1); }); + it.each([0, -1, Number.NaN, Number.POSITIVE_INFINITY])( + "removes malformed auth invalidation fields for timestamp %s", + (timestamp) => { + const result = normalizeAccountStorage({ + version: 3, + accounts: [ + { + refreshToken: "t1", + authInvalidatedAt: timestamp, + authInvalidationErrorCode: "oauth_token_revoked", + }, + ], + }); + + expect(result?.accounts[0]).not.toHaveProperty("authInvalidatedAt"); + expect(result?.accounts[0]).not.toHaveProperty( + "authInvalidationErrorCode", + ); + }, + ); + it("remaps activeKey when deduplication changes indices", () => { const now = Date.now(); const data = { diff --git a/test/uninstall-command.test.ts b/test/uninstall-command.test.ts index 12d384f4c..a37b81e4f 100644 --- a/test/uninstall-command.test.ts +++ b/test/uninstall-command.test.ts @@ -2,6 +2,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { mkdtempSync, writeFileSync, mkdirSync, existsSync, readFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; +import { resolveInstallPaths } from "../scripts/install-codex-auth-utils.js"; import { parseUninstallArgs, removePluginFromList, @@ -58,6 +59,16 @@ describe("removePluginFromList", () => { ).toEqual(["keep-me"]); }); + it("strips the legacy scoped package name and its versioned variants", () => { + expect( + removePluginFromList([ + "@ndycode/codex-multi-auth", + "@ndycode/codex-multi-auth@2.0.0", + "keep-me", + ]), + ).toEqual(["keep-me"]); + }); + it("preserves non-string entries", () => { const obj = { name: "other-plugin" }; expect(removePluginFromList([obj, "codex-multi-auth"])).toEqual([obj]); @@ -98,6 +109,23 @@ describe("parseUninstallArgs", () => { }); describe("resolveUninstallPaths", () => { + it("matches installer cache paths on Windows", () => { + const home = "C:\\Users\\user"; + const env = { + APPDATA: path.join(home, "AppData", "Roaming"), + LOCALAPPDATA: path.join(home, "AppData", "Local"), + }; + const installed = resolveInstallPaths("win32", env, home); + const uninstalled = resolveUninstallPaths("win32", env, home); + + expect(uninstalled.configPath).toBe(installed.configPath); + expect(uninstalled.cacheNodeModules).toBe(installed.cacheNodeModules); + expect(uninstalled.cacheLegacyNodeModules).toBe( + installed.cacheLegacyNodeModules, + ); + expect(uninstalled.cacheBunLock).toBe(installed.cacheBunLock); + }); + it("uses XDG layout on linux", () => { const paths = resolveUninstallPaths( "linux", @@ -153,6 +181,7 @@ describe("runUninstallCommand", () => { paths: { configPath: paths.configPath, cacheNodeModules: paths.cacheNodeModules, + cacheLegacyNodeModules: paths.cacheLegacyNodeModules, cacheBunLock: paths.cacheBunLock, }, }); @@ -231,16 +260,40 @@ describe("runUninstallCommand", () => { ).toBe(false); }); + it("dry-run reports the legacy cache path too", async () => { + const home = makeTempHome(); + const paths = pathsForTempHome(home); + const messages: string[] = []; + + const code = await runUninstallCommand(["--dry-run"], { + log: (m) => messages.push(m), + unbind: async () => {}, + removeLauncher: async () => {}, + paths, + }); + + expect(code).toBe(0); + expect( + messages.some((m) => m.includes(paths.cacheLegacyNodeModules)), + ).toBe(true); + }); + it("removes plugin entry from Codex.json and clears node_modules cache", async () => { const home = makeTempHome(); const paths = pathsForTempHome(home); mkdirSync(paths.configDir, { recursive: true }); mkdirSync(paths.cacheNodeModules, { recursive: true }); + mkdirSync(paths.cacheLegacyNodeModules, { recursive: true }); writeFileSync( path.join(paths.cacheNodeModules, "marker.txt"), "present", "utf8", ); + writeFileSync( + path.join(paths.cacheLegacyNodeModules, "legacy-marker.txt"), + "present", + "utf8", + ); writeFileSync( paths.configPath, JSON.stringify( @@ -261,6 +314,7 @@ describe("runUninstallCommand", () => { paths: { configPath: paths.configPath, cacheNodeModules: paths.cacheNodeModules, + cacheLegacyNodeModules: paths.cacheLegacyNodeModules, cacheBunLock: paths.cacheBunLock, }, }); @@ -269,6 +323,7 @@ describe("runUninstallCommand", () => { const config = JSON.parse(readFileSync(paths.configPath, "utf8")); expect(config.plugins).toEqual(["other-plugin"]); expect(existsSync(paths.cacheNodeModules)).toBe(false); + expect(existsSync(paths.cacheLegacyNodeModules)).toBe(false); // Other plugins still installed → shared bun.lock must be preserved. expect(existsSync(paths.cacheBunLock)).toBe(true); });