diff --git a/.agentv/targets.yaml b/.agentv/targets.yaml index 6e9c25a20..b76bfc745 100644 --- a/.agentv/targets.yaml +++ b/.agentv/targets.yaml @@ -53,12 +53,18 @@ targets: - name: claude provider: claude-cli grader_target: grader - log_format: json + + # Claude via Z.ai provider (GLM models). Requires cc-mirror: + # npx cc-mirror quick --provider zai --api-key "$Z_AI_API_KEY" + # See https://github.com/numman-ali/cc-mirror + - name: claude-zai + provider: claude-cli + executable: claude-zai + grader_target: grader - name: claude-sdk provider: claude-sdk grader_target: grader - log_format: json - name: pi provider: pi-cli diff --git a/apps/web/src/content/docs/docs/targets/coding-agents.mdx b/apps/web/src/content/docs/docs/targets/coding-agents.mdx index ae5ac3002..5c18ac088 100644 --- a/apps/web/src/content/docs/docs/targets/coding-agents.mdx +++ b/apps/web/src/content/docs/docs/targets/coding-agents.mdx @@ -70,6 +70,7 @@ targets: | Field | Required | Description | |-------|----------|-------------| +| `executable` | No | CLI binary name or path (default: `claude`). Accepts a bare name looked up on PATH (e.g. `claude-zai`) or an absolute/relative file path. | | `workspace_template` | No | Path to workspace template directory | | `cwd` | No | Working directory (mutually exclusive with workspace_template) | | `grader_target` | Yes | LLM target for evaluation | diff --git a/packages/core/src/evaluation/providers/claude-cli.ts b/packages/core/src/evaluation/providers/claude-cli.ts index adf7ddd98..4635d0e26 100644 --- a/packages/core/src/evaluation/providers/claude-cli.ts +++ b/packages/core/src/evaluation/providers/claude-cli.ts @@ -269,7 +269,7 @@ export class ClaudeCliProvider implements Provider { spawnOptions.cwd = options.cwd; } - const child = spawn('claude', options.args, spawnOptions); + const child = spawn(this.config.executable, options.args, spawnOptions); let stdout = ''; let stderr = ''; @@ -339,7 +339,7 @@ export class ClaudeCliProvider implements Provider { if (err.code === 'ENOENT') { reject( new Error( - `Claude CLI executable 'claude' was not found on PATH. Install claude-code or ensure it is in PATH.`, + `Claude CLI executable '${this.config.executable}' was not found on PATH. Install claude-code or ensure it is in PATH.`, ), ); } else { diff --git a/packages/core/src/evaluation/providers/targets.ts b/packages/core/src/evaluation/providers/targets.ts index d7f67156b..ce34eae7d 100644 --- a/packages/core/src/evaluation/providers/targets.ts +++ b/packages/core/src/evaluation/providers/targets.ts @@ -551,6 +551,7 @@ export interface PiCliResolvedConfig { } export interface ClaudeResolvedConfig { + readonly executable: string; readonly model?: string; readonly systemPrompt?: string; readonly cwd?: string; @@ -1940,6 +1941,7 @@ function resolveClaudeConfig( env: EnvLookup, evalFilePath?: string, ): ClaudeResolvedConfig { + const executableSource = target.executable ?? target.command ?? target.binary; const modelSource = target.model; const cwdSource = target.cwd; const workspaceTemplateSource = target.workspace_template; @@ -1954,6 +1956,12 @@ function resolveClaudeConfig( process.stderr.write(`[agentv] ⚠ ${streamLogResult.deprecationWarning}\n`); } + const executable = + resolveOptionalString(executableSource, env, `${target.name} claude-cli executable`, { + allowLiteral: true, + optionalEnv: true, + }) ?? 'claude'; + const model = resolveOptionalString(modelSource, env, `${target.name} claude model`, { allowLiteral: true, optionalEnv: true, @@ -2006,6 +2014,7 @@ function resolveClaudeConfig( typeof target.max_budget_usd === 'number' ? target.max_budget_usd : undefined; return { + executable, model, systemPrompt, cwd, diff --git a/packages/core/src/evaluation/validation/targets-validator.ts b/packages/core/src/evaluation/validation/targets-validator.ts index f31ddc635..0c1b4ddf6 100644 --- a/packages/core/src/evaluation/validation/targets-validator.ts +++ b/packages/core/src/evaluation/validation/targets-validator.ts @@ -163,6 +163,9 @@ const MOCK_SETTINGS = new Set([ const CLAUDE_SETTINGS = new Set([ ...COMMON_SETTINGS, + 'executable', + 'command', + 'binary', 'model', 'cwd', 'timeout_seconds', diff --git a/packages/core/test/evaluation/providers/targets.test.ts b/packages/core/test/evaluation/providers/targets.test.ts index 030c044c0..c5fd88614 100644 --- a/packages/core/test/evaluation/providers/targets.test.ts +++ b/packages/core/test/evaluation/providers/targets.test.ts @@ -712,6 +712,41 @@ describe('resolveTargetDefinition', () => { expect(target.kind).toBe('copilot-cli'); }); + it('claude-cli defaults executable to claude', () => { + const target = resolveTargetDefinition( + { + name: 'claude-default', + provider: 'claude-cli', + }, + {}, + ); + + expect(target.kind).toBe('claude-cli'); + if (target.kind !== 'claude-cli') { + throw new Error('expected claude-cli target'); + } + + expect(target.config.executable).toBe('claude'); + }); + + it('claude-cli accepts custom executable', () => { + const target = resolveTargetDefinition( + { + name: 'claude-zai', + provider: 'claude-cli', + executable: 'claude-zai', + }, + {}, + ); + + expect(target.kind).toBe('claude-cli'); + if (target.kind !== 'claude-cli') { + throw new Error('expected claude-cli target'); + } + + expect(target.config.executable).toBe('claude-zai'); + }); + it('resolves copilot-cli as its own provider kind', () => { const target = resolveTargetDefinition( {