diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 448a27f..daa7cad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,6 +21,7 @@ jobs: cache: npm - run: npm ci - run: npm run typecheck + - run: npm test format: runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index b9b15dc..6388cec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## Unreleased +- Add `/commandcode-refresh` and `/commandcode-status` commands for safe model-catalog refreshes and redacted diagnostics. +- Bound model discovery to a configurable 10-second timeout so a slow Provider API cannot block pi startup; timed-out discovery uses the validated cache when available. +- Normalize Command Code context overflow failures so pi can auto-compact and retry, while leaving unrelated rate-limit and capacity errors unchanged. +- Keep the legacy `/alpha/generate` integration explicitly text-only: image input and image tool results are rejected instead of being silently dropped, and models do not claim image capability until the protocol exposes documented support and limits. +- Replace blanket reasoning metadata with model-specific Command Code effort support. Known models expose a `thinkingLevelMap`, and selected supported Pi levels are forwarded as `params.reasoning_effort`; unsupported or unknown models do not receive reasoning request fields. - Add repository commands for testing the current checkout either in a logged-out, automatically cleaned-up pi environment or with existing credentials and only Command Code models enabled. - Refresh display pricing for the current Command Code model catalog, remove expired Qwen promotional rates, add current free and discounted models, and require review when temporary prices expire. - Use the host-provided `pi-ai` and `pi-coding-agent` core packages instead of installing private runtime copies, including for local and out-of-store development checkouts. diff --git a/README.md b/README.md index d1c034c..1fc9c3a 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,10 @@ Supported examples: Open `/model` and select one of the models provided by Command Code. Model availability changes over time and is refreshed from the Provider API when the extension loads. +### Reasoning support + +Reasoning metadata is enriched only for models whose Command Code effort support is known. Those models register a model-specific `thinkingLevelMap`, so pi and OMP expose only supported levels. A selected supported level is sent as the documented `params.reasoning_effort` field; `off`, unsupported levels, and newly discovered models without metadata do not add reasoning fields to the request. No prompt instructions are injected. + List Command Code models from the terminal: ```sh @@ -110,13 +114,25 @@ https://api.commandcode.ai/provider/v1/models The last successful catalog is cached at `/commandcode-models.json`. For pi this is `~/.pi/agent/commandcode-models.json` by default. Compatible hosts such as OMP use their own agent directory. -If the endpoint is temporarily unavailable, the provider uses the cached catalog. On a first offline start without a cache, pi still loads, but Command Code models remain unavailable until the connection is restored and `/reload` succeeds. +If the endpoint is temporarily unavailable, the provider uses the cached catalog. On a first offline start without a cache, pi still loads, but Command Code models remain unavailable until the connection is restored and `/commandcode-refresh` succeeds. + +While pi is running, use these provider commands without restarting: + +- `/commandcode-refresh` fetches and re-registers the current model catalog. Overlapping refreshes are coalesced, and a failed refresh keeps the last valid catalog active. +- `/commandcode-status` shows redacted discovery diagnostics, including the source, model count, timestamps, cache path, endpoint, and warning. The following environment variables are intended for tests, local mocks, and compatible API endpoints: - `COMMANDCODE_API_BASE` - `COMMANDCODE_MODELS_URL` - `COMMANDCODE_MODELS_CACHE` +- `COMMANDCODE_MODELS_TIMEOUT_MS` (defaults to 10 seconds; invalid or non-positive values use the default) + +## Image input + +This provider currently advertises and accepts **text input only**. The extension uses Command Code's legacy `/alpha/generate` protocol, while the public Provider API documentation describes image parts for its documented `/provider/v1` endpoints. The legacy request path has no documented image-part contract, and the model catalog fixture exposes model IDs and context lengths but no image capability or limit fields. + +To avoid silently dropping or changing image data, the provider rejects image content in user messages and tool results before making a network request. It does not claim image capability or define image-size/count limits. This limitation can be revisited when Command Code documents image parts and limits for the protocol used here. ## Pricing display diff --git a/package.json b/package.json index a128486..420c869 100644 --- a/package.json +++ b/package.json @@ -29,24 +29,26 @@ "LICENSE" ], "scripts": { - "test": "npm run typecheck && tsx tests/test-package-manifest.ts && tsx tests/test-pure-functions.ts && tsx tests/test-models.ts && tsx tests/test-pricing.ts && tsx tests/test-cost.ts && tsx tests/test-oauth.ts && tsx tests/test-abort.ts && tsx tests/test-overflow.ts && tsx tests/test-stream.ts && tsx tests/test-retry.ts && node tests/test-pi-isolated.mjs && node tests/test-pi-authenticated.mjs && node tests/test-pi-local.mjs && node tests/test-omp-compat.mjs", + "test": "npm run typecheck && tsx tests/test-package-manifest.ts && tsx tests/test-pure-functions.ts && tsx tests/test-models.ts && tsx tests/test-runtime.ts && tsx tests/test-pricing.ts && tsx tests/test-cost.ts && tsx tests/test-oauth.ts && tsx tests/test-abort.ts && tsx tests/test-overflow.ts && tsx tests/test-stream.ts && tsx tests/test-retry.ts && node tests/test-pi-isolated.mjs && node tests/test-pi-authenticated.mjs && node tests/test-pi-local.mjs && node tests/test-omp-compat.mjs", "typecheck": "tsc --noEmit", "format:check": "prettier --check '**/*.{ts,mjs,json,md}'", "format": "prettier --write '**/*.{ts,mjs,json,md}'", "pi:isolated": "node scripts/pi-isolated.mjs", "pi:authenticated": "node scripts/pi-authenticated.mjs", "test:unit": "tsx tests/test-pure-functions.ts", - "test:overflow": "tsx tests/test-overflow.ts", "test:models": "tsx tests/test-models.ts", + "test:runtime": "tsx tests/test-runtime.ts", "test:pricing": "tsx tests/test-pricing.ts", "test:oauth": "tsx tests/test-oauth.ts", "test:abort": "tsx tests/test-abort.ts", + "test:overflow": "tsx tests/test-overflow.ts", "test:stream": "tsx tests/test-stream.ts", "test:retry": "tsx tests/test-retry.ts", "test:pi-isolated": "node tests/test-pi-isolated.mjs", "test:pi-authenticated": "node tests/test-pi-authenticated.mjs", "test:pi-local": "node tests/test-pi-local.mjs", "test:smoke": "node tests/test-smoke.mjs", + "test:e2e:live": "node tests/test-live-e2e.mjs", "test:cost": "tsx tests/test-cost.ts" }, "pi": { diff --git a/tests/test-live-e2e.mjs b/tests/test-live-e2e.mjs new file mode 100644 index 0000000..d30d759 --- /dev/null +++ b/tests/test-live-e2e.mjs @@ -0,0 +1,319 @@ +#!/usr/bin/env node +/** + * Live end-to-end validation against Command Code with existing credentials. + * + * This test never reads or prints credential files. Pi resolves authentication + * through its normal provider flow. It is intentionally excluded from `npm test` + * because it consumes live provider capacity. + */ + +import assert from "node:assert/strict" +import { spawn } from "node:child_process" +import { + accessSync, + constants, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, +} from "node:fs" +import { homedir, tmpdir } from "node:os" +import { delimiter, dirname, join, resolve } from "node:path" +import { fileURLToPath } from "node:url" + +const projectDir = resolve(dirname(fileURLToPath(import.meta.url)), "..") +const extensionPath = join(projectDir, "index.ts") +const testModel = process.env.COMMANDCODE_E2E_MODEL ?? "deepseek/deepseek-v4-flash" +const marker = "commandcode-live-e2e-ok" + +function findPiBinary() { + if (process.env.PI_BIN) return process.env.PI_BIN + const localBin = resolve(projectDir, "node_modules", ".bin") + for (const entry of (process.env.PATH ?? "").split(delimiter)) { + const candidate = resolve(entry, "pi") + if (candidate.startsWith(localBin)) continue + try { + accessSync(candidate, constants.X_OK) + return candidate + } catch { + // Try the next PATH entry. + } + } + return undefined +} + +function hasAuthMetadata() { + return ( + Boolean(process.env.COMMANDCODE_API_KEY) || + existsSync(join(homedir(), ".commandcode", "auth.json")) || + existsSync(join(homedir(), ".pi", "agent", "auth.json")) + ) +} + +const piBin = findPiBinary() +if (!piBin || !hasAuthMetadata()) { + console.log("[live-e2e] SKIP — pi or Command Code auth metadata unavailable") + process.exit(0) +} + +function safeEnv(overrides = {}) { + const env = { ...process.env, PI_SKIP_VERSION_CHECK: "1", ...overrides } + delete env.COMMANDCODE_API_KEY + return env +} + +function run(command, args, options = {}) { + const timeoutMs = options.timeoutMs ?? 180_000 + return new Promise((resolve) => { + const child = spawn(command, args, { + cwd: options.cwd ?? projectDir, + env: options.env ?? safeEnv(), + stdio: ["ignore", "pipe", "pipe"], + }) + let stdout = "" + let stderr = "" + const timer = setTimeout(() => { + child.kill() + resolve({ code: -1, stdout, stderr: `${stderr}\nTIMEOUT after ${timeoutMs}ms` }) + }, timeoutMs) + child.stdout.on("data", (chunk) => { + stdout += chunk.toString("utf-8") + }) + child.stderr.on("data", (chunk) => { + stderr += chunk.toString("utf-8") + }) + child.on("close", (code) => { + clearTimeout(timer) + resolve({ code, stdout, stderr }) + }) + }) +} + +async function runRpc(extension, action, timeoutMs = 120_000) { + const child = spawn( + piBin, + [ + "--no-extensions", + "--mode", + "rpc", + "-e", + extension, + "--provider", + "commandcode", + "--model", + testModel, + ], + { cwd: projectDir, env: safeEnv(), stdio: ["pipe", "pipe", "pipe"] }, + ) + + let buffer = "" + let stderr = "" + const events = [] + const waiters = [] + + const publish = (event) => { + events.push(event) + for (let index = waiters.length - 1; index >= 0; index -= 1) { + const waiter = waiters[index] + if (!waiter.predicate(event)) continue + waiters.splice(index, 1) + clearTimeout(waiter.timer) + waiter.resolve(event) + } + } + + child.stdout.on("data", (chunk) => { + buffer += chunk.toString("utf-8") + const lines = buffer.split("\n") + buffer = lines.pop() ?? "" + for (const line of lines) { + if (!line.trim()) continue + try { + publish(JSON.parse(line)) + } catch { + // Ignore non-JSON output. + } + } + }) + child.stderr.on("data", (chunk) => { + stderr += chunk.toString("utf-8") + }) + + const waitFor = (predicate) => + new Promise((resolveWait, reject) => { + const existing = events.find(predicate) + if (existing) { + resolveWait(existing) + return + } + const timer = setTimeout(() => { + const index = waiters.findIndex((waiter) => waiter.timer === timer) + if (index >= 0) waiters.splice(index, 1) + reject(new Error(`RPC timeout. stderr: ${stderr.slice(-500)}`)) + }, timeoutMs) + waiters.push({ predicate, resolve: resolveWait, timer }) + }) + const send = (value) => child.stdin.write(`${JSON.stringify(value)}\n`) + + try { + return await action({ send, waitFor, events, getStderr: () => stderr }) + } finally { + child.kill() + } +} + +const tempRoot = mkdtempSync(join(tmpdir(), "pi-commandcode-live-e2e-")) +try { + console.log("[live-e2e] live reasoning request") + const reasoning = await run( + piBin, + [ + "--no-extensions", + "-e", + extensionPath, + "--no-session", + "-p", + "--provider", + "commandcode", + "--model", + testModel, + "--thinking", + "high", + `Reply exactly: ${marker}`, + ], + { timeoutMs: 180_000 }, + ) + assert.equal(reasoning.code, 0, reasoning.stderr) + assert.match(reasoning.stdout, new RegExp(marker)) + + console.log("[live-e2e] live runtime refresh/status commands") + const runtime = await runRpc(extensionPath, async ({ send, waitFor, getStderr }) => { + send({ id: "commands", type: "get_commands" }) + const commands = await waitFor( + (event) => event.type === "response" && event.id === "commands" && event.success, + ) + const names = commands.data?.commands?.map((command) => command.name) ?? [] + + send({ id: "refresh", type: "prompt", message: "/commandcode-refresh" }) + await waitFor((event) => event.type === "response" && event.id === "refresh" && event.success) + const refresh = await waitFor( + (event) => + event.type === "extension_ui_request" && + event.method === "notify" && + typeof event.message === "string" && + event.message.includes("model catalog"), + ) + + send({ id: "status", type: "prompt", message: "/commandcode-status" }) + await waitFor((event) => event.type === "response" && event.id === "status" && event.success) + const status = await waitFor( + (event) => + event.type === "extension_ui_request" && + event.method === "notify" && + typeof event.message === "string" && + event.message.includes("source:"), + ) + return { names, refresh: refresh.message, status: status.message, stderr: getStderr() } + }) + assert.ok(runtime.names.includes("commandcode-refresh")) + assert.ok(runtime.names.includes("commandcode-status")) + assert.match(runtime.refresh, /model catalog (?:refreshed|unchanged)/) + assert.match(runtime.status, /source: (?:live|cache)/) + assert.match(runtime.status, /model count: [1-9][0-9]*/) + assert.doesNotMatch(`${runtime.refresh}\n${runtime.status}\n${runtime.stderr}`, /Bearer\s+\S+/i) + + console.log("[live-e2e] live tool-call round trip") + const toolRoot = join(tempRoot, "tool-roundtrip") + const targetPath = join(toolRoot, "commandcode-e2e.txt") + const toolPrompt = [ + `Use the write tool to create ${targetPath}.`, + `The file content must be exactly ${marker}.`, + `After the tool succeeds, reply exactly: ${marker}`, + ].join(" ") + const toolResult = await run( + piBin, + [ + "--no-extensions", + "-e", + extensionPath, + "--no-session", + "-p", + "--provider", + "commandcode", + "--model", + testModel, + toolPrompt, + ], + { cwd: tempRoot, timeoutMs: 180_000 }, + ) + assert.equal(toolResult.code, 0, toolResult.stderr) + assert.match(toolResult.stdout, new RegExp(marker)) + assert.equal(readFileSync(targetPath, "utf-8"), marker) + + console.log("[live-e2e] image rejection through real RPC host") + const image = await runRpc(extensionPath, async ({ send, waitFor, events }) => { + send({ + id: "image", + type: "prompt", + message: "Describe this image", + images: [{ type: "image", data: "iVBORw0KGgo=", mimeType: "image/png" }], + }) + await waitFor((event) => event.type === "response" && event.id === "image") + await waitFor( + (event) => + event.type === "message_end" && + event.message?.role === "assistant" && + event.message?.stopReason === "error", + ) + return events + }) + assert.ok( + image.some( + (event) => + event.type === "message_end" && + /does not support image content/i.test(event.message?.errorMessage ?? ""), + ), + ) + + console.log("[live-e2e] packed artifact with existing authentication") + const packDir = join(tempRoot, "pack") + mkdirSync(packDir, { recursive: true }) + const pack = await run("npm", ["pack", "--pack-destination", packDir, "--silent"], { + timeoutMs: 120_000, + }) + assert.equal(pack.code, 0, pack.stderr) + const tarballName = pack.stdout.trim().split("\n").at(-1) + assert.ok(tarballName) + const tarball = join(packDir, tarballName) + const appDir = join(tempRoot, "packed-app") + const install = await run( + "npm", + ["install", "--prefix", appDir, "--ignore-scripts", "--no-save", tarball], + { timeoutMs: 180_000 }, + ) + assert.equal(install.code, 0, install.stderr) + const packedExtension = join(appDir, "node_modules", "pi-commandcode-provider", "index.ts") + const packedLive = await run( + piBin, + [ + "--no-extensions", + "-e", + packedExtension, + "--no-session", + "-p", + "--provider", + "commandcode", + "--model", + testModel, + `Reply exactly: ${marker}`, + ], + { timeoutMs: 180_000 }, + ) + assert.equal(packedLive.code, 0, packedLive.stderr) + assert.match(packedLive.stdout, new RegExp(marker)) + + console.log("[live-e2e] PASS") +} finally { + rmSync(tempRoot, { recursive: true, force: true }) +} diff --git a/tests/test-pi-local.mjs b/tests/test-pi-local.mjs index a8566c9..5099d08 100644 --- a/tests/test-pi-local.mjs +++ b/tests/test-pi-local.mjs @@ -6,7 +6,7 @@ import assert from "node:assert/strict" import { spawn, spawnSync } from "node:child_process" -import { accessSync, constants, mkdtempSync, rmSync } from "node:fs" +import { accessSync, constants, mkdtempSync, rmSync, mkdirSync, writeFileSync } from "node:fs" import { createServer } from "node:http" import { tmpdir } from "node:os" import { delimiter, dirname, join, resolve } from "node:path" @@ -15,7 +15,7 @@ import { fileURLToPath } from "node:url" const __dirname = dirname(fileURLToPath(import.meta.url)) const PROJECT_DIR = resolve(__dirname, "..") const EXT_PATH = resolve(PROJECT_DIR, "index.ts") -const TEST_MODEL = "cc-offline-cache-model" +const TEST_MODEL = "deepseek/deepseek-v4-flash" function findPiBinary() { if (process.env.PI_BIN) return process.env.PI_BIN @@ -51,34 +51,53 @@ let requestCount = 0 let modelListRequestCount = 0 let lastRequestBody let lastRequestHeaders = {} +let overflowMode = false +let overflowRequestCount = 0 +let modelsDelayMs = 0 +let includeRefreshedModel = false + +function modelCatalog() { + const data = [ + { + id: TEST_MODEL, + object: "model", + created: 1779824324, + owned_by: "command-code", + name: "DeepSeek V4 Flash", + context_length: 1_000_000, + }, + { + id: "cc-second-model", + object: "model", + created: 1779824324, + owned_by: "command-code", + name: "Qwen 3.7 Max", + context_length: 1_000_000, + }, + ] + if (includeRefreshedModel) { + data.push({ + id: "cc-refreshed-model", + object: "model", + created: 1779824324, + owned_by: "command-code", + name: "Refreshed Model", + context_length: 200_000, + }) + } + return { object: "list", data } +} const server = createServer((req, res) => { if (req.method === "GET" && req.url === "/provider/v1/models") { modelListRequestCount += 1 - res.writeHead(200, { "Content-Type": "application/json; charset=utf-8" }) - res.end( - JSON.stringify({ - object: "list", - data: [ - { - id: TEST_MODEL, - object: "model", - created: 1779824324, - owned_by: "command-code", - name: "DeepSeek V4 Flash", - context_length: 1_000_000, - }, - { - id: "cc-second-model", - object: "model", - created: 1779824324, - owned_by: "command-code", - name: "Qwen 3.7 Max", - context_length: 1_000_000, - }, - ], - }), - ) + const respond = () => { + if (res.destroyed) return + res.writeHead(200, { "Content-Type": "application/json; charset=utf-8" }) + res.end(JSON.stringify(modelCatalog())) + } + if (modelsDelayMs > 0) setTimeout(respond, modelsDelayMs) + else respond() return } @@ -89,6 +108,7 @@ const server = createServer((req, res) => { } requestCount += 1 + if (overflowMode) overflowRequestCount += 1 lastRequestHeaders = Object.fromEntries( Object.entries(req.headers).map(([key, value]) => [ key, @@ -107,11 +127,24 @@ const server = createServer((req, res) => { lastRequestBody = undefined } + if (overflowMode && overflowRequestCount === 2) { + res.writeHead(400, { "Content-Type": "application/json; charset=utf-8" }) + res.end(JSON.stringify({ error: { message: "Input exceeds context limit" } })) + return + } + res.writeHead(200, { "Content-Type": "text/plain; charset=utf-8", "Transfer-Encoding": "chunked", }) - res.write(`${JSON.stringify({ type: "text-delta", text: "mock-pi-ok" })}\n`) + const text = overflowMode + ? overflowRequestCount === 1 + ? "overflow-initial" + : overflowRequestCount === 3 + ? "compaction-summary" + : "overflow-recovered" + : "mock-pi-ok" + res.write(`${JSON.stringify({ type: "text-delta", text })}\n`) res.write( `${JSON.stringify({ type: "finish", finishReason: "stop", totalUsage: { inputTokens: 1, outputTokens: 1 } })}\n`, ) @@ -125,11 +158,18 @@ const port = typeof address === "object" && address ? address.port : 0 const apiBase = `http://127.0.0.1:${port}` const tempHome = mkdtempSync(join(tmpdir(), "pi-cc-home-")) +const agentDir = join(tempHome, "custom-pi-agent") +mkdirSync(agentDir, { recursive: true }) +writeFileSync( + join(agentDir, "settings.json"), + JSON.stringify({ compaction: { enabled: true, reserveTokens: 10, keepRecentTokens: 10 } }), +) const env = { ...process.env, HOME: tempHome, USERPROFILE: tempHome, - PI_CODING_AGENT_DIR: join(tempHome, "custom-pi-agent"), + PI_CODING_AGENT_DIR: agentDir, + PI_CODING_AGENT_SESSION_DIR: join(tempHome, "sessions"), COMMANDCODE_API_BASE: apiBase, COMMANDCODE_API_KEY: "mock-key", COMMANDCODE_MODELS_URL: `${apiBase}/provider/v1/models`, @@ -165,7 +205,12 @@ function runPi(args, timeoutMs = 30_000) { }) } -async function runRpcQuery(timeoutMs = 30_000) { +async function runRpcQuery( + timeoutMs = 30_000, + promptMessage = "say mock token", + extraArgs = [], + promptFields = {}, +) { const child = spawn( PI_BIN, [ @@ -178,6 +223,7 @@ async function runRpcQuery(timeoutMs = 30_000) { "commandcode", "--model", TEST_MODEL, + ...extraArgs, ], { cwd: PROJECT_DIR, @@ -212,7 +258,12 @@ async function runRpcQuery(timeoutMs = 30_000) { } child.stdin.write( - `${JSON.stringify({ id: "prompt-1", type: "prompt", message: "say mock token" })}\n`, + `${JSON.stringify({ + id: "prompt-1", + type: "prompt", + message: promptMessage, + ...promptFields, + })}\n`, ) child.stdout.on("data", (chunk) => { @@ -265,6 +316,225 @@ async function runRpcQuery(timeoutMs = 30_000) { } } +async function runRpcExtensionCommands(timeoutMs = 30_000) { + const child = spawn( + PI_BIN, + [ + "--no-extensions", + "--mode", + "rpc", + "-e", + EXT_PATH, + "--provider", + "commandcode", + "--model", + TEST_MODEL, + ], + { + cwd: PROJECT_DIR, + env, + stdio: ["pipe", "pipe", "pipe"], + }, + ) + + let buffer = "" + let stderr = "" + const events = [] + const waiters = [] + + const publish = (event) => { + events.push(event) + for (let index = waiters.length - 1; index >= 0; index -= 1) { + const waiter = waiters[index] + if (!waiter.predicate(event)) continue + waiters.splice(index, 1) + clearTimeout(waiter.timer) + waiter.resolve(event) + } + } + + child.stdout.on("data", (chunk) => { + buffer += chunk.toString("utf-8") + const lines = buffer.split("\n") + buffer = lines.pop() ?? "" + for (const line of lines) { + if (!line.trim()) continue + try { + publish(JSON.parse(line)) + } catch { + // Ignore non-JSON output. + } + } + }) + child.stderr.on("data", (chunk) => { + stderr += chunk.toString("utf-8") + }) + + const waitFor = (predicate) => + new Promise((resolve, reject) => { + const existing = events.find(predicate) + if (existing) { + resolve(existing) + return + } + const timer = setTimeout(() => { + const index = waiters.findIndex((waiter) => waiter.timer === timer) + if (index >= 0) waiters.splice(index, 1) + reject(new Error(`RPC event timeout. stderr: ${stderr.slice(-500)}`)) + }, timeoutMs) + waiters.push({ predicate, resolve, timer }) + }) + + const send = (value) => child.stdin.write(`${JSON.stringify(value)}\n`) + + try { + send({ id: "commands", type: "get_commands" }) + const commandsResponse = await waitFor( + (event) => event.type === "response" && event.id === "commands", + ) + const commandNames = commandsResponse.data?.commands?.map((command) => command.name) ?? [] + + send({ id: "status-before", type: "prompt", message: "/commandcode-status" }) + await waitFor( + (event) => event.type === "response" && event.id === "status-before" && event.success, + ) + const statusBefore = await waitFor( + (event) => + event.type === "extension_ui_request" && + event.method === "notify" && + typeof event.message === "string" && + event.message.includes("model count: 2"), + ) + + includeRefreshedModel = true + send({ id: "refresh", type: "prompt", message: "/commandcode-refresh" }) + await waitFor((event) => event.type === "response" && event.id === "refresh" && event.success) + const refreshNotification = await waitFor( + (event) => + event.type === "extension_ui_request" && + event.method === "notify" && + typeof event.message === "string" && + event.message.includes("3 models from live"), + ) + + send({ id: "status-after", type: "prompt", message: "/commandcode-status" }) + await waitFor( + (event) => event.type === "response" && event.id === "status-after" && event.success, + ) + const statusAfter = await waitFor( + (event) => + event.type === "extension_ui_request" && + event.method === "notify" && + typeof event.message === "string" && + event.message.includes("model count: 3"), + ) + + return { + commandNames, + statusBefore: statusBefore.message, + refreshNotification: refreshNotification.message, + statusAfter: statusAfter.message, + stderr, + } + } finally { + child.kill() + } +} + +async function runRpcOverflowRecovery(timeoutMs = 60_000) { + const child = spawn( + PI_BIN, + [ + "--no-extensions", + "--mode", + "rpc", + "-e", + EXT_PATH, + "--provider", + "commandcode", + "--model", + TEST_MODEL, + ], + { + cwd: PROJECT_DIR, + env, + stdio: ["pipe", "pipe", "pipe"], + }, + ) + + let buffer = "" + let stderr = "" + const events = [] + let firstSettled = false + let recovered = false + + const result = new Promise((resolve) => { + const timer = setTimeout(() => { + child.kill() + resolve({ ok: false }) + }, timeoutMs) + + const finish = (ok) => { + clearTimeout(timer) + child.kill() + resolve({ ok }) + } + + child.stdout.on("data", (chunk) => { + buffer += chunk.toString("utf-8") + const lines = buffer.split("\n") + buffer = lines.pop() ?? "" + for (const line of lines) { + if (!line.trim()) continue + let event + try { + event = JSON.parse(line) + } catch { + continue + } + events.push(event) + if (event.type === "agent_settled" && !firstSettled) { + firstSettled = true + child.stdin.write( + `${JSON.stringify({ id: "overflow-prompt", type: "prompt", message: "trigger overflow recovery" })}\n`, + ) + } + if ( + event.type === "compaction_end" && + event.reason === "overflow" && + event.willRetry === true + ) { + recovered = true + } + if (recovered && event.type === "agent_settled") finish(true) + } + }) + child.stderr.on("data", (chunk) => { + stderr += chunk.toString("utf-8") + }) + child.stdin.write( + `${JSON.stringify({ id: "initial-prompt", type: "prompt", message: "initial turn" })}\n`, + ) + }) + + const outcome = await result + return { + ...outcome, + requests: overflowRequestCount, + sawNormalizedOverflow: events.some( + (event) => + event.type === "message_end" && + event.message?.role === "assistant" && + typeof event.message.errorMessage === "string" && + event.message.errorMessage.startsWith("context_length_exceeded:"), + ), + sawCompactionRetry: events.some( + (event) => event.type === "compaction_end" && event.reason === "overflow" && event.willRetry, + ), + stderrHasSecrets: /mock-key|user_secret|api_key/i.test(stderr), + } +} + try { console.log("[pi-local] first offline start without a cache") const onlineModelsUrl = env.COMMANDCODE_MODELS_URL @@ -295,7 +565,7 @@ try { ) assert.equal(recoveryList.code, 0, recoveryList.stderr) const recoveryOutput = recoveryList.stdout || recoveryList.stderr - assert.match(recoveryOutput, /cc-offline-cache-model/) + assert.match(recoveryOutput, /deepseek\/deepseek-v4-flash/) assert.match(recoveryOutput, /cc-second-model/) assert.doesNotMatch(recoveryList.stderr, /no valid cached catalog/) assert.doesNotMatch(recoveryList.stderr, /Failed to load extension/) @@ -308,7 +578,7 @@ try { assert.equal(list.code, 0, list.stderr) const listOutput = list.stdout || list.stderr assert.match(listOutput, /commandcode/) - assert.match(listOutput, /cc-offline-cache-model/) + assert.match(listOutput, /deepseek\/deepseek-v4-flash/) assert.match(listOutput, /cc-second-model/) assert.equal(modelListRequestCount, 1) assert.doesNotThrow(() => accessSync(modelsCachePath, constants.R_OK)) @@ -321,7 +591,7 @@ try { ) assert.equal(offlineList.code, 0, offlineList.stderr) const offlineListOutput = offlineList.stdout || offlineList.stderr - assert.match(offlineListOutput, /cc-offline-cache-model/) + assert.match(offlineListOutput, /deepseek\/deepseek-v4-flash/) assert.match(offlineListOutput, /cc-second-model/) assert.match(offlineList.stderr, /Using the cached catalog/) @@ -347,7 +617,23 @@ try { assert.equal(requestCount, 1) env.COMMANDCODE_MODELS_URL = onlineModelsUrl - console.log("[pi-local] print mode through real extension and mock API") + console.log("[pi-local] discovery timeout through real extension") + rmSync(modelsCachePath, { force: true }) + modelsDelayMs = 5_000 + env.COMMANDCODE_MODELS_TIMEOUT_MS = "50" + const timeoutStartedAt = Date.now() + const timedOutList = await runPi( + ["--no-extensions", "-e", EXT_PATH, "--list-models", "commandcode"], + 5_000, + ) + const timeoutElapsedMs = Date.now() - timeoutStartedAt + assert.equal(timedOutList.code, 0, timedOutList.stderr) + assert.ok(timeoutElapsedMs < 2_000, `model discovery took ${timeoutElapsedMs}ms`) + assert.match(timedOutList.stderr, /timed out after 50ms/i) + modelsDelayMs = 0 + delete env.COMMANDCODE_MODELS_TIMEOUT_MS + + console.log("[pi-local] print mode with reasoning and tool schemas") requestCount = 0 const print = await runPi( [ @@ -360,6 +646,8 @@ try { "commandcode", "--model", TEST_MODEL, + "--thinking", + "high", ], 30_000, ) @@ -372,6 +660,30 @@ try { "should send a bearer Authorization header", ) assert.equal(lastRequestBody?.params?.model, TEST_MODEL) + assert.equal(lastRequestBody?.params?.reasoning_effort, "high") + const sentTools = lastRequestBody?.params?.tools + assert.ok(Array.isArray(sentTools) && sentTools.length > 0) + const editTool = sentTools.find((tool) => tool.name === "edit") + assert.equal(editTool?.input_schema?.type, "object") + assert.equal(editTool?.input_schema?.properties?.edits?.type, "array") + assert.equal(editTool?.input_schema?.properties?.edits?.items?.type, "object") + assert.equal( + editTool?.input_schema?.properties?.edits?.items?.properties?.oldText?.type, + "string", + ) + + console.log("[pi-local] runtime commands through real RPC extension lifecycle") + includeRefreshedModel = false + const runtimeCommands = await runRpcExtensionCommands() + assert.ok(runtimeCommands.commandNames.includes("commandcode-refresh")) + assert.ok(runtimeCommands.commandNames.includes("commandcode-status")) + assert.match(runtimeCommands.statusBefore, /source: live/) + assert.match(runtimeCommands.refreshNotification, /3 models from live/) + assert.match(runtimeCommands.statusAfter, /model count: 3/) + assert.doesNotMatch( + `${runtimeCommands.statusBefore}\n${runtimeCommands.statusAfter}\n${runtimeCommands.stderr}`, + /mock-key/, + ) console.log("[pi-local] RPC prompt through real extension and mock API") requestCount = 0 @@ -390,6 +702,38 @@ try { assert.equal(rpc.sawTextDelta, true) assert.equal(requestCount, 1) + console.log("[pi-local] reject image input through real RPC preflight/provider path") + requestCount = 0 + const imageRpc = await runRpcQuery(10_000, "describe image", [], { + images: [ + { + type: "image", + data: "iVBORw0KGgo=", + mimeType: "image/png", + }, + ], + }) + assert.equal(requestCount, 0) + assert.ok( + imageRpc.events.some( + (event) => + event.type === "message_end" && + event.message?.role === "assistant" && + event.message?.stopReason === "error", + ) || imageRpc.stderr.includes("does not support image"), + ) + + console.log("[pi-local] verify overflow normalization and compaction recovery") + overflowMode = true + overflowRequestCount = 0 + const overflowRpc = await runRpcOverflowRecovery() + assert.equal(overflowRpc.ok, true) + assert.ok(overflowRpc.requests >= 4) + assert.equal(overflowRpc.sawNormalizedOverflow, true) + assert.equal(overflowRpc.sawCompactionRetry, true) + assert.equal(overflowRpc.stderrHasSecrets, false) + overflowMode = false + console.log("[pi-local] PASS") } finally { await new Promise((resolve) => server.close(resolve))