diff --git a/AGENTS.md b/AGENTS.md index 06797c7..b0158d3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -47,6 +47,6 @@ Root-level scripts (`pnpm build`, `pnpm lint`, etc.) delegate to the packages th - Wizard tests live under `packages/spark/test/`. - The default CLI entrypoint is `packages/spark/src/cli.ts`; Rolldown emits `packages/spark/dist/cli.mjs`. - The beau CLI entrypoint is `packages/spark/src/beau/cli.tsx`; Rolldown emits `packages/spark/dist/cli.beau.js`. -- Keep wizard text close to its calling flow; put reusable wizard helpers in utility modules. +- Put all default Clack wizard user-facing copy in `packages/spark/src/clack-copy.ts`, organized by wizard flow. `clack-wizard.ts` should reference `CLACK_WIZARD_COPY` and keep only control-flow identifiers/sentinels inline. - `packages/spark/src/query-client.ts` owns QueryClient creation and should remain the central place for query defaults. - Do not add SEA packaging yet; the current build targets are JavaScript bundles. diff --git a/packages/spark/src/braintrust-cli.ts b/packages/spark/src/braintrust-cli.ts new file mode 100644 index 0000000..78f028a --- /dev/null +++ b/packages/spark/src/braintrust-cli.ts @@ -0,0 +1,352 @@ +import { type Buffer } from "node:buffer"; +import { spawn } from "node:child_process"; +import { constants } from "node:fs"; +import { access } from "node:fs/promises"; +import { platform as osPlatform } from "node:os"; +import { join } from "node:path"; + +export type BraintrustCliDiscovery = { + readonly installed: boolean; + readonly commandPath?: string; + readonly version?: string; +}; + +export type BraintrustCliContext = { + readonly profile?: string; + readonly org?: string; + readonly project?: string; +}; + +export type BraintrustCliConfigureArgs = { + readonly apiKey: string; + readonly apiUrl: string; + readonly appUrl: string; + readonly orgName: string; + readonly projectName: string; +}; + +export type BraintrustCliRuntime = { + readonly discover: () => Promise; + readonly install: () => Promise; + readonly update: (commandPath: string) => Promise; + readonly status: (commandPath: string) => Promise; + readonly loginAndSwitch: ( + commandPath: string, + args: BraintrustCliConfigureArgs, + ) => Promise; +}; + +type CommandSpec = { + readonly command: string; + readonly args: readonly string[]; + readonly env?: NodeJS.ProcessEnv; +}; + +type CommandResult = { + readonly exitCode: number; + readonly signal: NodeJS.Signals | null; + readonly stdout: string; + readonly stderr: string; +}; + +type BraintrustCliRuntimeDeps = { + readonly env?: NodeJS.ProcessEnv; + readonly platform?: NodeJS.Platform; + readonly exec?: (spec: CommandSpec) => Promise; + readonly findCommand?: (command: string) => Promise; +}; + +const BT_INSTALL_URL = "https://bt.dev/cli/install.sh"; +const BT_WINDOWS_INSTALLER_URL = + "https://github.com/braintrustdata/bt/releases/latest/download/bt-installer.ps1"; + +export function createBraintrustCliRuntime( + deps: BraintrustCliRuntimeDeps = {}, +): BraintrustCliRuntime { + const env = deps.env ?? process.env; + const platform = deps.platform ?? osPlatform(); + const exec = deps.exec ?? execCapture; + const findCommand = + deps.findCommand ?? + ((command) => findCommandOnPath(command, { env, exec, platform })); + + return { + async discover() { + const commandPath = + (await findCommand("bt")) ?? + (await findKnownInstallPath(env, platform)); + if (!commandPath) return { installed: false }; + + const versionResult = await exec({ + command: commandPath, + args: ["--version"], + env, + }); + const version = + versionResult.exitCode === 0 + ? firstNonEmptyLine(versionResult.stdout, versionResult.stderr) + : undefined; + + return version + ? { installed: true, commandPath, version } + : { installed: true, commandPath }; + }, + + async install() { + const spec = + platform === "win32" + ? { + command: "powershell", + args: [ + "-NoProfile", + "-ExecutionPolicy", + "Bypass", + "-Command", + `$ProgressPreference='SilentlyContinue'; irm ${BT_WINDOWS_INSTALLER_URL} | iex`, + ], + env, + } + : { + command: "sh", + args: ["-c", `curl -fsSL ${BT_INSTALL_URL} | bash -s -- --quiet`], + env, + }; + await execChecked("Braintrust CLI install", spec, exec); + }, + + async update(commandPath) { + await execChecked( + "Braintrust CLI update", + { command: commandPath, args: ["self", "update"], env }, + exec, + ); + }, + + async status(commandPath) { + const result = await exec({ + command: commandPath, + args: ["status", "--json"], + env, + }); + if (result.exitCode !== 0) { + throw new Error( + `Braintrust CLI status failed with exit code ${result.exitCode}. ${summarizeCommandOutput(result)}`.trim(), + ); + } + return parseStatusJson(result.stdout); + }, + + async loginAndSwitch(commandPath, args) { + const childEnv = { + ...env, + BRAINTRUST_API_KEY: args.apiKey, + BRAINTRUST_API_URL: args.apiUrl, + BRAINTRUST_APP_URL: args.appUrl, + }; + const profileName = args.orgName; + + await execChecked( + "Braintrust CLI login", + { + command: commandPath, + args: [ + "auth", + "--profile", + profileName, + "--org", + args.orgName, + "--no-input", + "--quiet", + "login", + ], + env: childEnv, + }, + exec, + ); + await execChecked( + "Braintrust CLI project switch", + { + command: commandPath, + args: [ + "switch", + "--profile", + profileName, + "--org", + args.orgName, + "--no-input", + "--quiet", + "--global", + args.projectName, + ], + env: childEnv, + }, + exec, + ); + }, + }; +} + +export function summarizeBraintrustCliError(error: unknown): string { + const message = error instanceof Error ? error.message : String(error); + return truncate(redactSensitiveText(message), 500); +} + +function execCapture(spec: CommandSpec): Promise { + return new Promise((resolve) => { + const child = spawn(spec.command, [...spec.args], { + env: spec.env ?? process.env, + stdio: ["ignore", "pipe", "pipe"], + }); + const stdout: string[] = []; + const stderr: string[] = []; + child.stdout.on("data", (chunk: Buffer) => + stdout.push(chunk.toString("utf8")), + ); + child.stderr.on("data", (chunk: Buffer) => + stderr.push(chunk.toString("utf8")), + ); + child.on("error", (error) => + resolve({ + exitCode: 1, + signal: null, + stdout: "", + stderr: error.message, + }), + ); + child.on("close", (code, signal) => + resolve({ + exitCode: code ?? 1, + signal, + stdout: stdout.join(""), + stderr: stderr.join(""), + }), + ); + }); +} + +async function execChecked( + action: string, + spec: CommandSpec, + exec: (spec: CommandSpec) => Promise, +): Promise { + const result = await exec(spec); + if (result.exitCode !== 0) { + throw new Error( + `${action} failed with exit code ${result.exitCode}. ${summarizeCommandOutput(result)}`.trim(), + ); + } +} + +async function findCommandOnPath( + command: string, + args: { + readonly env: NodeJS.ProcessEnv; + readonly exec: (spec: CommandSpec) => Promise; + readonly platform: NodeJS.Platform; + }, +): Promise { + const result = + args.platform === "win32" + ? await args.exec({ command: "where", args: [command], env: args.env }) + : await args.exec({ + command: "sh", + args: ["-c", `command -v ${shellQuote(command)}`], + env: args.env, + }); + if (result.exitCode !== 0) return undefined; + return firstNonEmptyLine(result.stdout); +} + +async function findKnownInstallPath( + env: NodeJS.ProcessEnv, + platform: NodeJS.Platform, +): Promise { + const binary = platform === "win32" ? "bt.exe" : "bt"; + const candidates = [ + env["XDG_BIN_HOME"] ? join(env["XDG_BIN_HOME"], binary) : undefined, + env["XDG_DATA_HOME"] + ? join(env["XDG_DATA_HOME"], "..", "bin", binary) + : undefined, + env["HOME"] ? join(env["HOME"], ".local", "bin", binary) : undefined, + env["USERPROFILE"] + ? join(env["USERPROFILE"], ".local", "bin", binary) + : undefined, + ].filter((value): value is string => value !== undefined); + + for (const candidate of candidates) { + if (await executableExists(candidate)) return candidate; + } + return undefined; +} + +async function executableExists(path: string): Promise { + return access(path, constants.X_OK).then( + () => true, + () => false, + ); +} + +function parseStatusJson(stdout: string): BraintrustCliContext { + let parsed: unknown; + try { + parsed = JSON.parse(stdout); + } catch { + throw new Error("Braintrust CLI status returned invalid JSON."); + } + if (!parsed || typeof parsed !== "object") { + throw new Error("Braintrust CLI status returned invalid JSON."); + } + const obj = parsed as Record; + const profile = stringField(obj, "profile"); + const org = stringField(obj, "org"); + const project = stringField(obj, "project"); + + return { + ...(profile !== undefined ? { profile } : {}), + ...(org !== undefined ? { org } : {}), + ...(project !== undefined ? { project } : {}), + }; +} + +function stringField( + obj: Record, + key: string, +): string | undefined { + const value = obj[key]; + return typeof value === "string" && value.trim().length > 0 + ? value + : undefined; +} + +function firstNonEmptyLine(...values: readonly string[]): string | undefined { + for (const value of values) { + const line = value + .split(/\r?\n/) + .find((candidate) => candidate.trim().length > 0); + if (line) return line.trim(); + } + return undefined; +} + +function summarizeCommandOutput(result: CommandResult): string { + return truncate( + redactSensitiveText([result.stderr, result.stdout].join("\n").trim()), + 400, + ); +} + +function redactSensitiveText(text: string): string { + return text + .replace(/Bearer\s+[A-Za-z0-9._~+/=-]+/gi, "Bearer [REDACTED]") + .replace(/BRAINTRUST_API_KEY=\S+/g, "BRAINTRUST_API_KEY=[REDACTED]") + .replace(/\b(?:sk|bt)[-_][A-Za-z0-9._~+/=-]{8,}\b/g, "[REDACTED]"); +} + +function truncate(text: string, maxLength: number): string { + if (text.length <= maxLength) return text; + return `${text.slice(0, maxLength - 3)}...`; +} + +function shellQuote(value: string): string { + return `'${value.replaceAll("'", "'\\''")}'`; +} diff --git a/packages/spark/src/clack-copy.ts b/packages/spark/src/clack-copy.ts new file mode 100644 index 0000000..34fcb7b --- /dev/null +++ b/packages/spark/src/clack-copy.ts @@ -0,0 +1,204 @@ +import type { BraintrustCliContext } from "./braintrust-cli"; + +const INSTRUMENTATION_DOCS_URL = + "https://www.braintrust.dev/docs/instrument/trace-llm-calls"; +const BRAINTRUST_CLI_CONTEXT_FALLBACKS = { + profile: "no profile", + org: "no org", + project: "no project", +} as const; + +export const CLACK_WIZARD_COPY = { + shared: { + cancelMessage: "Wizard cancelled.", + instrumentationDocsUrl: INSTRUMENTATION_DOCS_URL, + }, + + welcome: { + intro: "Welcome to the Braintrust setup wizard", + setupPlanTitle: "Setup plan", + setupPlan: + "You'll sign in with Braintrust, choose an org and project, save an API key for local testing, set up the Braintrust CLI, then choose how to add instrumentation.", + }, + + gitRepository: { + outsideRepoWarning: + "Heads up: this folder is not a git repository. The wizard may edit files; consider running it inside a checked-in repo.", + continueOutsideRepoQuestion: "Continue without a git repository?", + }, + + auth: { + accountQuestion: "Do you already have a Braintrust account?", + browserLoginInfo: (args: { + readonly loginLink: string; + readonly verificationCode: string; + }) => + [ + `Sign in: ${args.loginLink}`, + "", + "If your browser didn't open automatically, open the link above to sign in.", + `Verification code: ${args.verificationCode}`, + "", + "Choose the org and project you want to use; the wizard will resume here.", + ].join("\n"), + waitingForBrowser: "Waiting for login in browser...", + browserSetupComplete: (args: { + readonly orgName: string; + readonly projectName: string; + }) => + `Browser setup complete. (org: ${args.orgName}, project: ${args.projectName})`, + browserSetupStopped: "Browser setup stopped.", + }, + + braintrustCli: { + installedVersionUnknown: "version unknown", + installQuestion: "Would you like to install the Braintrust CLI?", + updateQuestion: (installedLabel: string) => + `Braintrust CLI is already installed (${installedLabel}). Update it to the latest version?`, + updated: "Updated Braintrust CLI.", + installed: "Installed Braintrust CLI.", + configured: "Configured Braintrust CLI.", + updateFailed: (message: string) => + `Could not update Braintrust CLI: ${message}`, + installFailed: (message: string) => + `Could not install Braintrust CLI: ${message}`, + configureFailed: (message: string) => + `Could not configure Braintrust CLI: ${message}`, + installedButNotFound: + "Braintrust CLI was installed, but the wizard could not find `bt` in PATH or the default install location. Open a new shell and run `bt status` to verify it.", + statusFailed: (message: string) => + `Could not inspect Braintrust CLI status; leaving existing CLI context unchanged. ${message}`, + switchContextQuestion: (args: { + readonly currentContext: BraintrustCliContext; + readonly targetContext: BraintrustCliContext; + }) => + `Braintrust CLI is currently configured for ${formatBraintrustCliContext(args.currentContext)}. Switch it to ${formatBraintrustCliContext(args.targetContext)}?`, + leavingContextUnchanged: + "Leaving existing Braintrust CLI context unchanged.", + contextFallbacks: { + ...BRAINTRUST_CLI_CONTEXT_FALLBACKS, + }, + }, + + instrumentation: { + modeQuestion: "How do you want to add Braintrust instrumentation?", + modes: { + builtIn: { + label: "Use built-in coding agent", + hint: "This wizard will launch a coding agent for you that will add instrumentation to your application (supports Claude Code and Codex). Careful: This will run the chosen tool in yolo mode (full permissions).", + }, + ownAgent: { + label: "Use own coding agent", + hint: "You will receive a prompt to instrument your application with your own coding agent.", + }, + manual: { + label: "Set up manually", + hint: "Set up tracing for your application using instructions from the Braintrust docs.", + }, + }, + builtIn: { + usingTool: (label: string) => `Using ${label} for instrumentation.`, + toolQuestion: "Which coding agent should Braintrust Setup use?", + noUsableToolsWarning: (toolMessages: readonly string[]) => + ["No usable coding agents found.", ...toolMessages].join("\n"), + unavailableToolMessage: (args: { + readonly label: string; + readonly installed: boolean; + readonly unavailableReason?: string | undefined; + }) => { + if (!args.installed) return `${args.label} is not installed.`; + return args.unavailableReason ?? `${args.label} is not usable.`; + }, + unavailableToolLine: (toolLabel: string, unavailableMessage: string) => + `- ${toolLabel}: ${unavailableMessage}`, + noUsableToolsError: + "No usable coding agents found. Use your own coding agent or the Braintrust docs instead.", + codingAgentFailed: "Coding agent failed.", + toolExited: (toolLabel: string, exitCode: number) => + `${toolLabel} exited with code ${exitCode}.`, + codingToolExited: (exitCode: number) => + `Coding tool exited with code ${exitCode}.`, + incompleteRenderer: "Instrumentation incomplete.", + incompleteWarning: "The coding tool reported incomplete instrumentation.", + complete: "Instrumentation complete.", + toolFinished: (toolLabel: string) => `${toolLabel} finished.`, + }, + localToken: { + title: "Local application token", + notice: + "The wizard will now create a .env.braintrust file that is used to authenticate your application to Braintrust. It will be used for local testing.", + existingNotice: + "A .env.braintrust file already exists. The wizard can replace it with the API key for this Braintrust project.", + replaceQuestion: + ".env.braintrust already exists. Replace it with this project's Braintrust API key?", + outsideGitRepo: (apiKey: string) => + `BRAINTRUST_API_KEY=${apiKey}\nNot in a git repo — set this in your environment manually.`, + wroteEnvFile: (path: string) => `Wrote ${path}`, + keptEnvFile: (path: string) => `Kept existing ${path} unchanged.`, + gitignoreNote: (args: { + readonly added: boolean; + readonly alreadyCovered: boolean; + }) => { + if (args.added) return "Added .env.braintrust to .gitignore."; + if (args.alreadyCovered) { + return ".gitignore already covers .env.braintrust."; + } + return ".gitignore unchanged."; + }, + }, + manual: { + title: "Manual instrumentation", + note: (docsLink: string) => + [ + "Follow the Braintrust instrumentation docs for your project.", + "", + docsLink, + ].join("\n"), + completedQuestion: + "Have you completed the Braintrust instrumentation docs?", + }, + ownAgent: { + deliveryQuestion: + "How should Braintrust Setup deliver the instrumentation prompt?", + copyToClipboard: "Copy to clipboard", + printToTerminal: "Print to terminal", + copiedToClipboard: "Copied instrumentation prompt to clipboard.", + clipboardFailed: (message: string) => + `Could not copy the instrumentation prompt to the clipboard: ${message}`, + completedQuestion: + "Has your coding agent completed Braintrust instrumentation?", + promptHeader: "Braintrust instrumentation prompt:", + localApiKeyContext: (envFilePath: string) => + `\n## Local Braintrust API Key\n\nThe wizard wrote \`${envFilePath}\` with BRAINTRUST_API_KEY for local verification. Use it when running the application locally, but do not commit it.\n`, + }, + }, + + logs: { + projectLogsUrl: (url: string) => `Check your Braintrust logs: ${url}`, + }, + + productionToken: { + title: "Production token", + noteWithEnvFile: (envFilePath: string) => + `The local .env.braintrust file contains a BRAINTRUST_API_KEY token. Add that token to your deployment platform's environment variables so tracing works in production.\n\nFile: ${envFilePath}`, + noteWithoutEnvFile: + "Add the BRAINTRUST_API_KEY token to your deployment platform's environment variables so tracing works in production.", + question: "Have you added BRAINTRUST_API_KEY to your deployment platform?", + addedIt: "I added it", + doLater: "I will do that later", + laterWarning: + "Do not forget to add BRAINTRUST_API_KEY to production. Braintrust tracing will not work in production without it.", + }, + + outro: { + complete: (docsUrl: string) => + ["Setup complete.", "", `Docs: ${docsUrl}`].join("\n"), + }, +} as const; + +function formatBraintrustCliContext(context: BraintrustCliContext): string { + const profile = context.profile ?? BRAINTRUST_CLI_CONTEXT_FALLBACKS.profile; + const org = context.org ?? BRAINTRUST_CLI_CONTEXT_FALLBACKS.org; + const project = context.project ?? BRAINTRUST_CLI_CONTEXT_FALLBACKS.project; + return `${profile} (${org}/${project})`; +} diff --git a/packages/spark/src/clack-wizard.ts b/packages/spark/src/clack-wizard.ts index bed8b9d..c111d33 100644 --- a/packages/spark/src/clack-wizard.ts +++ b/packages/spark/src/clack-wizard.ts @@ -10,10 +10,16 @@ import { type WizardSessionLogin, } from "./auth"; import { BraintrustApiClient } from "./braintrust-api"; +import { + createBraintrustCliRuntime, + summarizeBraintrustCliError, + type BraintrustCliContext, + type BraintrustCliRuntime, +} from "./braintrust-cli"; import { openBrowser } from "./browser"; +import { CLACK_WIZARD_COPY } from "./clack-copy"; import { buildLogsPermalink } from "./cleanup"; import { - buildToolUnavailableMessage, codingToolLabel, discoverCodingTools, runCodingTool, @@ -23,33 +29,35 @@ import { type CodingToolRunResult, type CodingToolStatus, } from "./coding-tools"; -import { findGitRoot, isGitRepo, writeEnvBraintrust } from "./git"; +import { + ensureEnvBraintrustIgnored, + envBraintrustExists, + envBraintrustPath, + isGitRepo, + writeEnvBraintrust, +} from "./git"; import { allocateResultFile, readResultFile } from "./instrument"; import type { WizardOptions } from "./options"; import { renderPrompt } from "./prompt"; import { ClackToolRenderer } from "./tool-ui"; -import { gitignoreNote, terminalHyperlink } from "./wizard-utils"; +import { terminalHyperlink } from "./wizard-utils"; -const WIZARD_CANCEL_MESSAGE = "Wizard cancelled."; -const ACCOUNT_QUESTION = "Do you already have a Braintrust account?"; -const INSTRUMENTATION_DOCS_URL = - "https://www.braintrust.dev/docs/instrument/trace-llm-calls"; -const ENV_BRAINTRUST_NOTICE = - "The wizard will now create a .env.braintrust file that is used to authenticate your application to Braintrust. It will be used for local testing."; +const COPY = CLACK_WIZARD_COPY; +const WIZARD_CANCEL_MESSAGE = COPY.shared.cancelMessage; type BuiltInInstrumentationChoice = { readonly id: "built-in"; - readonly label: "Use built-in coding agent"; + readonly label: typeof COPY.instrumentation.modes.builtIn.label; }; type OwnAgentInstrumentationChoice = { readonly id: "own-agent"; - readonly label: "Use own coding agent"; + readonly label: typeof COPY.instrumentation.modes.ownAgent.label; }; type ManualInstrumentationChoice = { readonly id: "manual"; - readonly label: "Set up manually"; + readonly label: typeof COPY.instrumentation.modes.manual.label; }; type InstrumentationModeChoice = @@ -61,17 +69,17 @@ type OwnAgentPromptDelivery = "clipboard" | "terminal"; const BUILT_IN_INSTRUMENTATION_CHOICE: BuiltInInstrumentationChoice = { id: "built-in", - label: "Use built-in coding agent", + label: COPY.instrumentation.modes.builtIn.label, }; const OWN_AGENT_INSTRUMENTATION_CHOICE: OwnAgentInstrumentationChoice = { id: "own-agent", - label: "Use own coding agent", + label: COPY.instrumentation.modes.ownAgent.label, }; const MANUAL_INSTRUMENTATION_CHOICE: ManualInstrumentationChoice = { id: "manual", - label: "Set up manually", + label: COPY.instrumentation.modes.manual.label, }; type SelectOption = { @@ -127,6 +135,7 @@ export type WizardDeps = { readonly loginWithWizardSession: WizardSessionLogin; readonly openBrowser: (url: string) => Promise; readonly writeClipboard: (text: string) => Promise; + readonly braintrustCli: BraintrustCliRuntime; readonly codingTools: CodingToolRuntime; }; @@ -168,21 +177,16 @@ export type WizardResult = { export async function runClackWizard(deps: WizardDeps): Promise { const { prompts } = deps; - prompts.intro("Welcome to the Braintrust setup wizard"); - prompts.note( - "You'll sign in with Braintrust, choose an org and project, save an API key for local testing when needed, then choose how to add instrumentation.", - "Setup plan", - ); + prompts.intro(COPY.welcome.intro); + prompts.note(COPY.welcome.setupPlan, COPY.welcome.setupPlanTitle); if (!(await isGitRepo(deps.cwd))) { - prompts.log.warn( - "Heads up: this folder is not a git repository. The wizard may edit files; consider running it inside a checked-in repo.", - ); + prompts.log.warn(COPY.gitRepository.outsideRepoWarning); const continueOutsideGit = unwrap( prompts, await prompts.confirm({ initialValue: false, - message: "Continue without a git repository?", + message: COPY.gitRepository.continueOutsideRepoQuestion, }), ); if (!continueOutsideGit) { @@ -202,17 +206,21 @@ export async function runClackWizard(deps: WizardDeps): Promise { authMode: (await hasBraintrustAccount(prompts)) ? "signin" : "signup", }); + const envFilePath = await writeLocalEnvBraintrust(deps, session.apiKey); + + await handleBraintrustCliSetup(deps, session); + const instrumentationMode = await selectInstrumentationMode(prompts); - let envFilePath: string | undefined; if (instrumentationMode.id === "built-in") { const instrumentation = await selectBuiltInCodingTool(deps); - prompts.log.info(`Using ${instrumentation.label} for instrumentation.`); + prompts.log.info( + COPY.instrumentation.builtIn.usingTool(instrumentation.label), + ); await deps.codingTools.smokeTest({ id: instrumentation.id, cwd: deps.cwd, }); - envFilePath = await writeLocalEnvBraintrust(deps, session.apiKey); await runInstrumentation(deps, { org: session.orgName, project: session.projectName, @@ -220,7 +228,6 @@ export async function runClackWizard(deps: WizardDeps): Promise { toolId: instrumentation.id, }); } else if (instrumentationMode.id === "own-agent") { - envFilePath = await writeLocalEnvBraintrust(deps, session.apiKey); await handleOwnAgentInstrumentation(deps, { org: session.orgName, project: session.projectName, @@ -231,13 +238,11 @@ export async function runClackWizard(deps: WizardDeps): Promise { } const projectLogsUrl = `${deps.options.appUrl}/${encodeURIComponent(session.orgName)}/p/${encodeURIComponent(session.projectName)}/logs`; - prompts.log.info(`Check your Braintrust logs: ${projectLogsUrl}`); + prompts.log.info(COPY.logs.projectLogsUrl(projectLogsUrl)); await confirmProductionApiKey(prompts, envFilePath); - prompts.outro( - ["Setup complete.", "", `Docs: ${INSTRUMENTATION_DOCS_URL}`].join("\n"), - ); + prompts.outro(COPY.outro.complete(COPY.shared.instrumentationDocsUrl)); return { orgName: session.orgName, @@ -266,16 +271,14 @@ async function loginWithBrowser( events: { onLoginUrl: ({ loginUrl, verificationCode }) => { prompts.log.info( - [ - `Sign in: ${terminalHyperlink(loginUrl)}`, - "", - "If your browser didn't open automatically, open the link above to sign in.", - `Verification code: ${pc.reset(pc.bold(pc.whiteBright(verificationCode)))}`, - "", - "Choose the org and project you want to use; the wizard will resume here.", - ].join("\n"), + COPY.auth.browserLoginInfo({ + loginLink: terminalHyperlink(loginUrl), + verificationCode: pc.reset( + pc.bold(pc.whiteBright(verificationCode)), + ), + }), ); - spinner.start("Waiting for login in browser..."); + spinner.start(COPY.auth.waitingForBrowser); spinnerStarted = true; }, onTryOpenBrowser: (url) => deps.openBrowser(url), @@ -283,13 +286,16 @@ async function loginWithBrowser( }); if (spinnerStarted) { spinner.stop( - `Browser setup complete. (org: ${pc.greenBright(session.orgName)}, project: ${pc.greenBright(session.projectName)})`, + COPY.auth.browserSetupComplete({ + orgName: pc.greenBright(session.orgName), + projectName: pc.greenBright(session.projectName), + }), ); spinnerStarted = false; } return session; } finally { - if (spinnerStarted) spinner.stop("Browser setup stopped."); + if (spinnerStarted) spinner.stop(COPY.auth.browserSetupStopped); } } @@ -300,7 +306,7 @@ async function hasBraintrustAccount( prompts, await prompts.confirm({ initialValue: true, - message: ACCOUNT_QUESTION, + message: COPY.auth.accountQuestion, }), ); } @@ -311,28 +317,149 @@ async function selectInstrumentationMode( return unwrap( prompts, await prompts.select({ - message: "How do you want to add Braintrust instrumentation?", + message: COPY.instrumentation.modeQuestion, options: [ { label: BUILT_IN_INSTRUMENTATION_CHOICE.label, value: BUILT_IN_INSTRUMENTATION_CHOICE, - hint: "This wizard will launch a coding agent for you that will add instrumentation to your application (supports Claude Code and Codex). Careful: This will run the chosen tool in yolo mode (full permissions).", + hint: COPY.instrumentation.modes.builtIn.hint, }, { label: OWN_AGENT_INSTRUMENTATION_CHOICE.label, value: OWN_AGENT_INSTRUMENTATION_CHOICE, - hint: "You will receive a prompt to instrument your application with your own coding agent.", + hint: COPY.instrumentation.modes.ownAgent.hint, }, { label: MANUAL_INSTRUMENTATION_CHOICE.label, value: MANUAL_INSTRUMENTATION_CHOICE, - hint: "Set up tracing for your application using instructions from the Braintrust docs.", + hint: COPY.instrumentation.modes.manual.hint, }, ], }), ); } +async function handleBraintrustCliSetup( + deps: WizardDeps, + session: WizardSessionCompleteResult, +): Promise { + const { prompts } = deps; + let discovery = await deps.braintrustCli.discover(); + let commandPath = discovery.commandPath; + + if (discovery.installed) { + const installedLabel = + discovery.version ?? + commandPath ?? + COPY.braintrustCli.installedVersionUnknown; + const shouldUpdate = unwrap( + prompts, + await prompts.confirm({ + initialValue: false, + message: COPY.braintrustCli.updateQuestion(installedLabel), + }), + ); + if (shouldUpdate && commandPath) { + try { + await deps.braintrustCli.update(commandPath); + prompts.log.success(COPY.braintrustCli.updated); + discovery = await deps.braintrustCli.discover(); + commandPath = discovery.commandPath ?? commandPath; + } catch (error) { + prompts.log.warn( + COPY.braintrustCli.updateFailed(summarizeBraintrustCliError(error)), + ); + } + } + } else { + const shouldInstall = unwrap( + prompts, + await prompts.confirm({ + initialValue: true, + message: COPY.braintrustCli.installQuestion, + }), + ); + if (!shouldInstall) return; + + try { + await deps.braintrustCli.install(); + prompts.log.success(COPY.braintrustCli.installed); + } catch (error) { + prompts.log.warn( + COPY.braintrustCli.installFailed(summarizeBraintrustCliError(error)), + ); + return; + } + + discovery = await deps.braintrustCli.discover(); + commandPath = discovery.commandPath; + if (!discovery.installed || !commandPath) { + prompts.log.warn(COPY.braintrustCli.installedButNotFound); + return; + } + } + + if (!commandPath) return; + + let currentContext: BraintrustCliContext; + try { + currentContext = await deps.braintrustCli.status(commandPath); + } catch (error) { + prompts.log.warn( + COPY.braintrustCli.statusFailed(summarizeBraintrustCliError(error)), + ); + return; + } + + const targetContext = { + profile: session.orgName, + org: session.orgName, + project: session.projectName, + }; + if (braintrustCliContextConflicts(currentContext, targetContext)) { + const shouldSwitch = unwrap( + prompts, + await prompts.confirm({ + initialValue: false, + message: COPY.braintrustCli.switchContextQuestion({ + currentContext, + targetContext, + }), + }), + ); + if (!shouldSwitch) { + prompts.log.info(COPY.braintrustCli.leavingContextUnchanged); + return; + } + } + + try { + await deps.braintrustCli.loginAndSwitch(commandPath, { + apiKey: session.apiKey, + apiUrl: deps.options.apiUrl, + appUrl: deps.options.appUrl, + orgName: session.orgName, + projectName: session.projectName, + }); + prompts.log.success(COPY.braintrustCli.configured); + } catch (error) { + prompts.log.warn( + COPY.braintrustCli.configureFailed(summarizeBraintrustCliError(error)), + ); + } +} + +function braintrustCliContextConflicts( + current: BraintrustCliContext, + target: Required, +): boolean { + return ( + (current.profile !== undefined && current.profile !== target.profile) || + (current.org !== undefined && current.org !== target.org) || + (current.project !== undefined && current.project !== target.project) + ); +} + async function selectBuiltInCodingTool( deps: WizardDeps, ): Promise { @@ -342,23 +469,22 @@ async function selectBuiltInCodingTool( const usable = statuses.filter((status) => status.usable); if (usable.length === 0) { prompts.log.warn( - [ - "No usable coding agents found.", - ...statuses.map( - (status) => - `- ${status.label}: ${buildToolUnavailableMessage(status)}`, + COPY.instrumentation.builtIn.noUsableToolsWarning( + statuses.map((status) => + COPY.instrumentation.builtIn.unavailableToolLine( + status.label, + COPY.instrumentation.builtIn.unavailableToolMessage(status), + ), ), - ].join("\n"), - ); - throw new Error( - "No usable coding agents found. Use your own coding agent or the Braintrust docs instead.", + ), ); + throw new Error(COPY.instrumentation.builtIn.noUsableToolsError); } const value = unwrap( prompts, await prompts.select({ - message: "Which coding agent should Braintrust Setup use?", + message: COPY.instrumentation.builtIn.toolQuestion, options: [ ...usable.map((tool) => ({ label: tool.label, @@ -376,19 +502,46 @@ async function writeLocalEnvBraintrust( apiKey: string, ): Promise { const { prompts } = deps; - const gitRoot = await findGitRoot(deps.cwd); - if (!gitRoot) { - prompts.log.info( - `BRAINTRUST_API_KEY=${apiKey}\nNot in a git repo — set this in your environment manually.`, + const targetDirectory = deps.cwd; + const envFilePath = envBraintrustPath(targetDirectory); + if (await envBraintrustExists(targetDirectory)) { + prompts.note( + COPY.instrumentation.localToken.existingNotice, + COPY.instrumentation.localToken.title, + ); + const shouldReplace = unwrap( + prompts, + await prompts.confirm({ + initialValue: false, + message: COPY.instrumentation.localToken.replaceQuestion, + }), + ); + if (!shouldReplace) { + const gitignoreResult = await ensureEnvBraintrustIgnored(targetDirectory); + prompts.log.info( + COPY.instrumentation.localToken.keptEnvFile(envFilePath), + ); + prompts.log.info( + COPY.instrumentation.localToken.gitignoreNote({ + added: gitignoreResult.addedToGitignore, + alreadyCovered: gitignoreResult.alreadyCovered, + }), + ); + return undefined; + } + } else { + prompts.note( + COPY.instrumentation.localToken.notice, + COPY.instrumentation.localToken.title, ); - return undefined; } - prompts.note(ENV_BRAINTRUST_NOTICE, "Local application token"); - const result = await writeEnvBraintrust(gitRoot, apiKey); - prompts.log.success(`Wrote ${result.envFilePath}`); + const result = await writeEnvBraintrust(targetDirectory, apiKey); + prompts.log.success( + COPY.instrumentation.localToken.wroteEnvFile(result.envFilePath), + ); prompts.log.info( - gitignoreNote({ + COPY.instrumentation.localToken.gitignoreNote({ added: result.addedToGitignore, alreadyCovered: result.alreadyCovered, }), @@ -400,18 +553,16 @@ async function confirmManualInstrumentation( prompts: ClackWizardPrompts, ): Promise { prompts.note( - [ - "Follow the Braintrust instrumentation docs for your project.", - "", - terminalHyperlink(INSTRUMENTATION_DOCS_URL), - ].join("\n"), - "Manual instrumentation", + COPY.instrumentation.manual.note( + terminalHyperlink(COPY.shared.instrumentationDocsUrl), + ), + COPY.instrumentation.manual.title, ); const completed = unwrap( prompts, await prompts.confirm({ initialValue: false, - message: "Have you completed the Braintrust instrumentation docs?", + message: COPY.instrumentation.manual.completedQuestion, }), ); if (!completed) { @@ -437,15 +588,14 @@ async function handleOwnAgentInstrumentation( const delivery = unwrap( prompts, await prompts.select({ - message: - "How should Braintrust Setup deliver the instrumentation prompt?", + message: COPY.instrumentation.ownAgent.deliveryQuestion, options: [ { - label: "Copy to clipboard", + label: COPY.instrumentation.ownAgent.copyToClipboard, value: "clipboard", }, { - label: "Print to terminal", + label: COPY.instrumentation.ownAgent.printToTerminal, value: "terminal", }, ], @@ -455,12 +605,10 @@ async function handleOwnAgentInstrumentation( if (delivery === "clipboard") { try { await deps.writeClipboard(promptText); - prompts.log.success("Copied instrumentation prompt to clipboard."); + prompts.log.success(COPY.instrumentation.ownAgent.copiedToClipboard); } catch (error) { const message = error instanceof Error ? error.message : String(error); - prompts.log.warn( - `Could not copy the instrumentation prompt to the clipboard: ${message}`, - ); + prompts.log.warn(COPY.instrumentation.ownAgent.clipboardFailed(message)); printInstrumentationPrompt(prompts, promptText); } } else { @@ -471,7 +619,7 @@ async function handleOwnAgentInstrumentation( prompts, await prompts.confirm({ initialValue: false, - message: "Has your coding agent completed Braintrust instrumentation?", + message: COPY.instrumentation.ownAgent.completedQuestion, }), ); if (!completed) { @@ -485,13 +633,13 @@ function printInstrumentationPrompt( promptText: string, ): void { prompts.log.message( - ["Braintrust instrumentation prompt:", "", promptText].join("\n"), + [COPY.instrumentation.ownAgent.promptHeader, "", promptText].join("\n"), ); } function renderOwnAgentEnvFileContext(envFilePath: string | undefined): string { if (!envFilePath) return ""; - return `\n## Local Braintrust API Key\n\nThe wizard created \`${envFilePath}\` with BRAINTRUST_API_KEY for local verification. Use it when running the application locally, but do not commit it.\n`; + return COPY.instrumentation.ownAgent.localApiKeyContext(envFilePath); } async function confirmProductionApiKey( @@ -500,30 +648,28 @@ async function confirmProductionApiKey( ): Promise { prompts.note( envFilePath - ? `The generated .env.braintrust file contains a BRAINTRUST_API_KEY token. Add that token to your deployment platform's environment variables so tracing works in production.\n\nFile: ${envFilePath}` - : "Add the BRAINTRUST_API_KEY token to your deployment platform's environment variables so tracing works in production.", - "Production token", + ? COPY.productionToken.noteWithEnvFile(envFilePath) + : COPY.productionToken.noteWithoutEnvFile, + COPY.productionToken.title, ); const productionTokenStatus = unwrap( prompts, await prompts.select<"done" | "later">({ - message: "Have you added BRAINTRUST_API_KEY to your deployment platform?", + message: COPY.productionToken.question, options: [ { - label: "I added it", + label: COPY.productionToken.addedIt, value: "done", }, { - label: "I will do that later", + label: COPY.productionToken.doLater, value: "later", }, ], }), ); if (productionTokenStatus === "later") { - prompts.log.warn( - "Do not forget to add BRAINTRUST_API_KEY to production. Braintrust tracing will not work in production without it.", - ); + prompts.log.warn(COPY.productionToken.laterWarning); } } @@ -565,22 +711,26 @@ async function runInstrumentation( onEvent: (event) => renderer.event(event), }); } catch (error) { - await renderer.error("Coding agent failed."); + await renderer.error(COPY.instrumentation.builtIn.codingAgentFailed); throw error; } if (toolResult.exitCode !== 0) { await renderer.error( - `${toolLabel} exited with code ${toolResult.exitCode}.`, + COPY.instrumentation.builtIn.toolExited(toolLabel, toolResult.exitCode), + ); + prompts.log.warn( + COPY.instrumentation.builtIn.codingToolExited(toolResult.exitCode), ); - prompts.log.warn(`Coding tool exited with code ${toolResult.exitCode}.`); } else if (toolResult.finalText.includes("INSTRUMENTATION_INCOMPLETE")) { - await renderer.error("Instrumentation incomplete."); - prompts.log.warn("The coding tool reported incomplete instrumentation."); + await renderer.error(COPY.instrumentation.builtIn.incompleteRenderer); + prompts.log.warn(COPY.instrumentation.builtIn.incompleteWarning); } else if (toolResult.finalText.includes("INSTRUMENTATION_COMPLETE")) { - await renderer.success("Instrumentation complete."); + await renderer.success(COPY.instrumentation.builtIn.complete); } else { - await renderer.success(`${toolLabel} finished.`); + await renderer.success( + COPY.instrumentation.builtIn.toolFinished(toolLabel), + ); } return { @@ -634,6 +784,7 @@ export function buildDefaultDeps(args: DefaultDepsArgs): WizardDeps { }), openBrowser, writeClipboard: (text) => clipboard.write(text), + braintrustCli: createBraintrustCliRuntime({ env }), codingTools: { discover: discoverCodingTools, smokeTest: smokeTestCodingTool, diff --git a/packages/spark/src/coding-tools.ts b/packages/spark/src/coding-tools.ts index c269afc..662866a 100644 --- a/packages/spark/src/coding-tools.ts +++ b/packages/spark/src/coding-tools.ts @@ -1,7 +1,6 @@ export { buildClaudeCommandForTest, buildCodexCommandForTest, - buildToolUnavailableMessage, codingToolLabel, discoverCodingTools, parseClaudeEventForTest, diff --git a/packages/spark/src/coding-tools/index.ts b/packages/spark/src/coding-tools/index.ts index 32c0817..ffc73f2 100644 --- a/packages/spark/src/coding-tools/index.ts +++ b/packages/spark/src/coding-tools/index.ts @@ -91,11 +91,6 @@ export async function runCodingTool(args: { ); } -export function buildToolUnavailableMessage(status: CodingToolStatus): string { - if (!status.installed) return `${status.label} is not installed.`; - return status.unavailableReason ?? `${status.label} is not usable.`; -} - export function buildClaudeCommandForTest(args: { readonly commandPath: string; readonly cwd: string; diff --git a/packages/spark/src/git.ts b/packages/spark/src/git.ts index ab15e3e..1b44aad 100644 --- a/packages/spark/src/git.ts +++ b/packages/spark/src/git.ts @@ -44,6 +44,14 @@ export async function isGitRepo(cwd: string): Promise { const ENV_FILENAME = ".env.braintrust"; +export function envBraintrustPath(directory: string): string { + return join(directory, ENV_FILENAME); +} + +export async function envBraintrustExists(directory: string): Promise { + return pathExists(envBraintrustPath(directory)); +} + export type EnvFileWriteResult = { readonly envFilePath: string; readonly gitignorePath: string; @@ -52,15 +60,25 @@ export type EnvFileWriteResult = { }; export async function writeEnvBraintrust( - gitRoot: string, + directory: string, apiKey: string, ): Promise { - const envFilePath = join(gitRoot, ENV_FILENAME); + const envFilePath = envBraintrustPath(directory); await writeFile(envFilePath, `BRAINTRUST_API_KEY=${apiKey}\n`, { mode: 0o600, }); - const gitignorePath = join(gitRoot, ".gitignore"); + const gitignoreResult = await ensureEnvBraintrustIgnored(directory); + return { + envFilePath, + ...gitignoreResult, + }; +} + +export async function ensureEnvBraintrustIgnored( + directory: string, +): Promise> { + const gitignorePath = join(directory, ".gitignore"); const existing = (await pathExists(gitignorePath)) ? await readFile(gitignorePath, "utf8") : ""; @@ -68,7 +86,6 @@ export async function writeEnvBraintrust( const alreadyCovered = gitignoreCovers(existing, ENV_FILENAME); if (alreadyCovered) { return { - envFilePath, gitignorePath, addedToGitignore: false, alreadyCovered: true, @@ -78,7 +95,6 @@ export async function writeEnvBraintrust( const sep = existing.length === 0 || existing.endsWith("\n") ? "" : "\n"; await writeFile(gitignorePath, `${existing}${sep}${ENV_FILENAME}\n`); return { - envFilePath, gitignorePath, addedToGitignore: true, alreadyCovered: false, diff --git a/packages/spark/src/wizard-utils.ts b/packages/spark/src/wizard-utils.ts index 3821fa5..74e0fdd 100644 --- a/packages/spark/src/wizard-utils.ts +++ b/packages/spark/src/wizard-utils.ts @@ -1,16 +1,3 @@ -export function gitignoreNote(args: { - readonly added: boolean; - readonly alreadyCovered: boolean; -}): string { - if (args.added) { - return "Added .env.braintrust to .gitignore."; - } - if (args.alreadyCovered) { - return ".gitignore already covers .env.braintrust."; - } - return ".gitignore unchanged."; -} - export function terminalHyperlink(url: string, label: string = url): string { return `\x1b]8;;${url}\x07${label}\x1b]8;;\x07`; } diff --git a/packages/spark/test/braintrust-cli.test.ts b/packages/spark/test/braintrust-cli.test.ts new file mode 100644 index 0000000..8587666 --- /dev/null +++ b/packages/spark/test/braintrust-cli.test.ts @@ -0,0 +1,119 @@ +import { describe, expect, it } from "vitest"; + +import { createBraintrustCliRuntime } from "../src/braintrust-cli"; + +describe("Braintrust CLI runtime", () => { + it("builds the Unix installer command", async () => { + const calls: Array<{ + readonly command: string; + readonly args: readonly string[]; + readonly env?: NodeJS.ProcessEnv; + }> = []; + const runtime = createBraintrustCliRuntime({ + platform: "darwin", + env: { PATH: "/usr/bin" }, + exec: (spec) => { + calls.push(spec); + return Promise.resolve({ + exitCode: 0, + signal: null, + stdout: "", + stderr: "", + }); + }, + }); + + await runtime.install(); + + expect(calls).toEqual([ + { + command: "sh", + args: [ + "-c", + "curl -fsSL https://bt.dev/cli/install.sh | bash -s -- --quiet", + ], + env: { PATH: "/usr/bin" }, + }, + ]); + }); + + it("passes the API key only through env when configuring auth and context", async () => { + const calls: Array<{ + readonly command: string; + readonly args: readonly string[]; + readonly env?: NodeJS.ProcessEnv; + }> = []; + const runtime = createBraintrustCliRuntime({ + env: { PATH: "/usr/bin" }, + exec: (spec) => { + calls.push(spec); + return Promise.resolve({ + exitCode: 0, + signal: null, + stdout: "", + stderr: "", + }); + }, + }); + + await runtime.loginAndSwitch("/usr/local/bin/bt", { + apiKey: "bt-secret-key", + apiUrl: "https://api.test", + appUrl: "https://app.test", + orgName: "acme", + projectName: "demo", + }); + + expect(calls).toHaveLength(2); + expect(calls[0]?.args).toEqual([ + "auth", + "--profile", + "acme", + "--org", + "acme", + "--no-input", + "--quiet", + "login", + ]); + expect(calls[1]?.args).toEqual([ + "switch", + "--profile", + "acme", + "--org", + "acme", + "--no-input", + "--quiet", + "--global", + "demo", + ]); + expect(calls.flatMap((call) => [...call.args])).not.toContain( + "bt-secret-key", + ); + expect(calls[0]?.env?.["BRAINTRUST_API_KEY"]).toBe("bt-secret-key"); + expect(calls[0]?.env?.["BRAINTRUST_API_URL"]).toBe("https://api.test"); + expect(calls[0]?.env?.["BRAINTRUST_APP_URL"]).toBe("https://app.test"); + expect(calls[1]?.env?.["BRAINTRUST_API_KEY"]).toBe("bt-secret-key"); + }); + + it("parses bt status JSON", async () => { + const runtime = createBraintrustCliRuntime({ + exec: () => + Promise.resolve({ + exitCode: 0, + signal: null, + stdout: JSON.stringify({ + profile: "work", + org: "acme", + project: "demo", + }), + stderr: "", + }), + }); + + await expect(runtime.status("/bin/bt")).resolves.toEqual({ + profile: "work", + org: "acme", + project: "demo", + }); + }); +}); diff --git a/packages/spark/test/clack-wizard.test.ts b/packages/spark/test/clack-wizard.test.ts index bd6b9c9..bfafb4d 100644 --- a/packages/spark/test/clack-wizard.test.ts +++ b/packages/spark/test/clack-wizard.test.ts @@ -1,5 +1,11 @@ import { execFileSync } from "node:child_process"; -import { existsSync, mkdtempSync, mkdirSync, readFileSync } from "node:fs"; +import { + existsSync, + mkdtempSync, + mkdirSync, + readFileSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -10,6 +16,12 @@ import { type WizardSessionLoginArgs, type WizardSessionLogin, } from "../src/auth"; +import { + type BraintrustCliConfigureArgs, + type BraintrustCliDiscovery, + type BraintrustCliContext, + type BraintrustCliRuntime, +} from "../src/braintrust-cli"; import { type CodingToolRuntime, type ClackWizardPrompts, @@ -23,11 +35,16 @@ const WIZARD_CANCEL_MESSAGE = "Wizard cancelled."; const ACCOUNT_QUESTION = "Do you already have a Braintrust account?"; const INSTRUMENTATION_MODE_MESSAGE = "How do you want to add Braintrust instrumentation?"; +const CLI_INSTALL_MESSAGE = "Would you like to install the Braintrust CLI?"; +const CLI_UPDATE_MESSAGE = + "Braintrust CLI is already installed (bt 0.10.0). Update it to the latest version?"; const TOOL_SELECT_MESSAGE = "Which coding agent should Braintrust Setup use?"; const OWN_AGENT_DELIVERY_MESSAGE = "How should Braintrust Setup deliver the instrumentation prompt?"; const ENV_BRAINTRUST_NOTICE = "The wizard will now create a .env.braintrust file that is used to authenticate your application to Braintrust. It will be used for local testing."; +const ENV_BRAINTRUST_REPLACE_QUESTION = + ".env.braintrust already exists. Replace it with this project's Braintrust API key?"; const CANCEL = Symbol("cancel"); @@ -206,6 +223,7 @@ function buildDeps(args: { readonly loginWithWizardSession?: WizardSessionLogin; readonly cwd?: string; readonly codingTools?: CodingToolRuntime; + readonly braintrustCli?: BraintrustCliRuntime; readonly writeClipboard?: (text: string) => Promise; }): WizardDeps { const cwd = args.cwd ?? createGitTempDir(); @@ -239,6 +257,7 @@ function buildDeps(args: { loginWithWizardSession: stubLogin, openBrowser: () => Promise.resolve(true), writeClipboard: args.writeClipboard ?? (() => Promise.resolve()), + braintrustCli: args.braintrustCli ?? createBraintrustCliStub(), codingTools: args.codingTools ?? ({ @@ -283,10 +302,39 @@ function buildDeps(args: { }; } +function createBraintrustCliStub( + args: { + readonly discoveries?: readonly BraintrustCliDiscovery[]; + readonly install?: () => Promise; + readonly update?: (commandPath: string) => Promise; + readonly status?: (commandPath: string) => Promise; + readonly loginAndSwitch?: ( + commandPath: string, + args: BraintrustCliConfigureArgs, + ) => Promise; + } = {}, +): BraintrustCliRuntime { + let discoveryIndex = 0; + const discoveries = args.discoveries ?? [{ installed: false }]; + + return { + discover: () => { + const discovery = + discoveries[Math.min(discoveryIndex, discoveries.length - 1)]!; + discoveryIndex += 1; + return Promise.resolve(discovery); + }, + install: args.install ?? (() => Promise.resolve()), + update: args.update ?? (() => Promise.resolve()), + status: args.status ?? (() => Promise.resolve({})), + loginAndSwitch: args.loginAndSwitch ?? (() => Promise.resolve()), + }; +} + describe("runClackWizard", () => { it("walks through the happy path with one usable coding tool", async () => { const { prompts, events } = createPrompts({ - confirms: [true], + confirms: [true, false], selects: ["first", "first", "production-done"], }); const deps = buildDeps({ prompts }); @@ -299,6 +347,7 @@ describe("runClackWizard", () => { expect(events[0]).toBe(`intro:${WIZARD_INTRO_TITLE}`); expect(events).toContain("note:Setup plan"); expect(events).toContain(`confirm:${ACCOUNT_QUESTION}`); + expect(events).toContain(`confirm:${CLI_INSTALL_MESSAGE}`); expect(events).toContain(`select:${INSTRUMENTATION_MODE_MESSAGE}`); expect(events).toContain(`select:${TOOL_SELECT_MESSAGE}`); expect(events.some((event) => event.startsWith("info:Sign in:"))).toBe( @@ -331,6 +380,9 @@ describe("runClackWizard", () => { "BRAINTRUST_API_KEY=bt-secret-key\n", ); expectEnvNoticeBeforeWrite(events); + expect( + events.indexOf(`note.message:${ENV_BRAINTRUST_NOTICE}`), + ).toBeLessThan(events.indexOf(`confirm:${CLI_INSTALL_MESSAGE}`)); expect( events.some((event) => event.startsWith("info:Saved instrumentation prompt"), @@ -350,7 +402,7 @@ describe("runClackWizard", () => { for (const { answer, expectedAuthMode } of cases) { let authMode: string | undefined; const { prompts } = createPrompts({ - confirms: [answer, true], + confirms: [answer, false, true], selects: ["manual", "production-done"], }); const deps = buildDeps({ @@ -374,9 +426,375 @@ describe("runClackWizard", () => { } }); + it("asks before replacing an existing local env file", async () => { + const cwd = createGitTempDir(); + const envFilePath = join(cwd, ".env.braintrust"); + writeFileSync(envFilePath, "BRAINTRUST_API_KEY=old\n"); + const { prompts, events } = createPrompts({ + confirms: [true, true, false, true], + selects: ["manual", "production-done"], + }); + const deps = buildDeps({ prompts, cwd }); + + await runClackWizard(deps); + + expect(events).toContain(`confirm:${ENV_BRAINTRUST_REPLACE_QUESTION}`); + expect(readFileSync(envFilePath, "utf8")).toBe( + "BRAINTRUST_API_KEY=bt-secret-key\n", + ); + }); + + it("writes local env files in the cwd instead of the git root", async () => { + const root = createGitTempDir(); + const cwd = join(root, "app", "service"); + mkdirSync(cwd, { recursive: true }); + const { prompts } = createPrompts({ + confirms: [true, false, true], + selects: ["manual", "production-done"], + }); + const deps = buildDeps({ prompts, cwd }); + + await runClackWizard(deps); + + expect(readFileSync(join(cwd, ".env.braintrust"), "utf8")).toBe( + "BRAINTRUST_API_KEY=bt-secret-key\n", + ); + expect(readFileSync(join(cwd, ".gitignore"), "utf8")).toBe( + ".env.braintrust\n", + ); + expect(existsSync(join(root, ".env.braintrust"))).toBe(false); + expect(existsSync(join(root, ".gitignore"))).toBe(false); + }); + + it("keeps an existing local env file when replacement is declined", async () => { + const cwd = createGitTempDir(); + const envFilePath = join(cwd, ".env.braintrust"); + writeFileSync(envFilePath, "BRAINTRUST_API_KEY=old\n"); + const { prompts, events } = createPrompts({ + confirms: [true, false, false, true], + selects: ["manual", "production-done"], + }); + const deps = buildDeps({ prompts, cwd }); + + await runClackWizard(deps); + + expect(events).toContain(`confirm:${ENV_BRAINTRUST_REPLACE_QUESTION}`); + expect( + events.some( + (event) => + event.startsWith("info:Kept existing ") && + event.endsWith("/.env.braintrust unchanged."), + ), + ).toBe(true); + expect(events).toContain("info:Added .env.braintrust to .gitignore."); + expect(readFileSync(envFilePath, "utf8")).toBe("BRAINTRUST_API_KEY=old\n"); + expect(readFileSync(join(cwd, ".gitignore"), "utf8")).toBe( + ".env.braintrust\n", + ); + }); + + it("installs and configures the Braintrust CLI when missing and accepted", async () => { + const calls: string[] = []; + const { prompts, events } = createPrompts({ + confirms: [true, true, true], + selects: ["manual", "production-done"], + }); + const deps = buildDeps({ + prompts, + braintrustCli: createBraintrustCliStub({ + discoveries: [ + { installed: false }, + { + installed: true, + commandPath: "/usr/local/bin/bt", + version: "bt 0.10.0", + }, + ], + install: () => { + calls.push("install"); + return Promise.resolve(); + }, + status: (commandPath) => { + calls.push(`status:${commandPath}`); + return Promise.resolve({}); + }, + loginAndSwitch: (commandPath, args) => { + calls.push( + `login:${commandPath}:${args.apiKey}:${args.apiUrl}:${args.appUrl}:${args.orgName}:${args.projectName}`, + ); + return Promise.resolve(); + }, + }), + }); + + await runClackWizard(deps); + + expect(events).toContain(`confirm:${CLI_INSTALL_MESSAGE}`); + expect(events).toContain("success:Installed Braintrust CLI."); + expect(events).toContain("success:Configured Braintrust CLI."); + expect(calls).toEqual([ + "install", + "status:/usr/local/bin/bt", + "login:/usr/local/bin/bt:bt-secret-key:https://api.test:https://app.test:acme:demo", + ]); + }); + + it("skips Braintrust CLI install and configuration when declined", async () => { + const calls: string[] = []; + const { prompts, events } = createPrompts({ + confirms: [true, false, true], + selects: ["manual", "production-done"], + }); + const deps = buildDeps({ + prompts, + braintrustCli: createBraintrustCliStub({ + install: () => { + calls.push("install"); + return Promise.resolve(); + }, + loginAndSwitch: () => { + calls.push("login"); + return Promise.resolve(); + }, + }), + }); + + await runClackWizard(deps); + + expect(events).toContain(`confirm:${CLI_INSTALL_MESSAGE}`); + expect(events).toContain(`select:${INSTRUMENTATION_MODE_MESSAGE}`); + expect(calls).toEqual([]); + }); + + it("continues when Braintrust CLI install fails", async () => { + const calls: string[] = []; + const { prompts, events } = createPrompts({ + confirms: [true, true, true], + selects: ["manual", "production-done"], + }); + const deps = buildDeps({ + prompts, + braintrustCli: createBraintrustCliStub({ + install: () => Promise.reject(new Error("install failed")), + status: () => { + calls.push("status"); + return Promise.resolve({}); + }, + loginAndSwitch: () => { + calls.push("login"); + return Promise.resolve(); + }, + }), + }); + + await runClackWizard(deps); + + expect( + events.some((event) => + event.startsWith( + "warn:Could not install Braintrust CLI: install failed", + ), + ), + ).toBe(true); + expect(events).toContain(`select:${INSTRUMENTATION_MODE_MESSAGE}`); + expect(calls).toEqual([]); + }); + + it("updates and configures an installed Braintrust CLI when accepted", async () => { + const calls: string[] = []; + const { prompts, events } = createPrompts({ + confirms: [true, true, true], + selects: ["manual", "production-done"], + }); + const deps = buildDeps({ + prompts, + braintrustCli: createBraintrustCliStub({ + discoveries: [ + { installed: true, commandPath: "/bin/bt", version: "bt 0.10.0" }, + { installed: true, commandPath: "/bin/bt", version: "bt 0.10.1" }, + ], + update: (commandPath) => { + calls.push(`update:${commandPath}`); + return Promise.resolve(); + }, + status: (commandPath) => { + calls.push(`status:${commandPath}`); + return Promise.resolve({}); + }, + loginAndSwitch: () => { + calls.push("login"); + return Promise.resolve(); + }, + }), + }); + + await runClackWizard(deps); + + expect(events).toContain(`confirm:${CLI_UPDATE_MESSAGE}`); + expect(events).toContain("success:Updated Braintrust CLI."); + expect(events).toContain("success:Configured Braintrust CLI."); + expect(calls).toEqual(["update:/bin/bt", "status:/bin/bt", "login"]); + }); + + it("still configures an installed Braintrust CLI when update fails", async () => { + const calls: string[] = []; + const { prompts, events } = createPrompts({ + confirms: [true, true, true], + selects: ["manual", "production-done"], + }); + const deps = buildDeps({ + prompts, + braintrustCli: createBraintrustCliStub({ + discoveries: [ + { installed: true, commandPath: "/bin/bt", version: "bt 0.10.0" }, + ], + update: () => Promise.reject(new Error("update failed")), + status: () => { + calls.push("status"); + return Promise.resolve({}); + }, + loginAndSwitch: () => { + calls.push("login"); + return Promise.resolve(); + }, + }), + }); + + await runClackWizard(deps); + + expect( + events.some((event) => + event.startsWith("warn:Could not update Braintrust CLI: update failed"), + ), + ).toBe(true); + expect(calls).toEqual(["status", "login"]); + }); + + it("configures an installed Braintrust CLI with matching context without asking to switch", async () => { + const calls: string[] = []; + const { prompts, events } = createPrompts({ + confirms: [true, false, true], + selects: ["manual", "production-done"], + }); + const deps = buildDeps({ + prompts, + braintrustCli: createBraintrustCliStub({ + discoveries: [ + { installed: true, commandPath: "/bin/bt", version: "bt 0.10.0" }, + ], + status: () => + Promise.resolve({ profile: "acme", org: "acme", project: "demo" }), + loginAndSwitch: () => { + calls.push("login"); + return Promise.resolve(); + }, + }), + }); + + await runClackWizard(deps); + + expect(events).toContain(`confirm:${CLI_UPDATE_MESSAGE}`); + expect( + events.some((event) => + event.startsWith("confirm:Braintrust CLI is currently configured for"), + ), + ).toBe(false); + expect(calls).toEqual(["login"]); + }); + + it("leaves a different Braintrust CLI context untouched when switch is declined", async () => { + const calls: string[] = []; + const { prompts, events } = createPrompts({ + confirms: [true, false, false, true], + selects: ["manual", "production-done"], + }); + const deps = buildDeps({ + prompts, + braintrustCli: createBraintrustCliStub({ + discoveries: [ + { installed: true, commandPath: "/bin/bt", version: "bt 0.10.0" }, + ], + status: () => + Promise.resolve({ profile: "work", org: "other", project: "old" }), + loginAndSwitch: () => { + calls.push("login"); + return Promise.resolve(); + }, + }), + }); + + await runClackWizard(deps); + + expect(events).toContain( + "confirm:Braintrust CLI is currently configured for work (other/old). Switch it to acme (acme/demo)?", + ); + expect(events).toContain( + "info:Leaving existing Braintrust CLI context unchanged.", + ); + expect(calls).toEqual([]); + }); + + it("switches a different Braintrust CLI context when accepted", async () => { + const calls: string[] = []; + const { prompts } = createPrompts({ + confirms: [true, false, true, true], + selects: ["manual", "production-done"], + }); + const deps = buildDeps({ + prompts, + braintrustCli: createBraintrustCliStub({ + discoveries: [ + { installed: true, commandPath: "/bin/bt", version: "bt 0.10.0" }, + ], + status: () => + Promise.resolve({ profile: "work", org: "other", project: "old" }), + loginAndSwitch: () => { + calls.push("login"); + return Promise.resolve(); + }, + }), + }); + + await runClackWizard(deps); + + expect(calls).toEqual(["login"]); + }); + + it("does not configure the Braintrust CLI when status inspection fails", async () => { + const calls: string[] = []; + const { prompts, events } = createPrompts({ + confirms: [true, false, true], + selects: ["manual", "production-done"], + }); + const deps = buildDeps({ + prompts, + braintrustCli: createBraintrustCliStub({ + discoveries: [ + { installed: true, commandPath: "/bin/bt", version: "bt 0.10.0" }, + ], + status: () => Promise.reject(new Error("status failed")), + loginAndSwitch: () => { + calls.push("login"); + return Promise.resolve(); + }, + }), + }); + + await runClackWizard(deps); + + expect( + events.some((event) => + event.startsWith( + "warn:Could not inspect Braintrust CLI status; leaving existing CLI context unchanged. status failed", + ), + ), + ).toBe(true); + expect(calls).toEqual([]); + }); + it("cancels cleanly when the user aborts the tool select", async () => { const { prompts, events } = createPrompts({ - confirms: [true], + confirms: [true, false], selects: ["first", CANCEL], }); const deps = buildDeps({ @@ -412,7 +830,7 @@ describe("runClackWizard", () => { const dir = mkdtempSync(join(tmpdir(), "braintrust-setup-nogit-")); mkdirSync(join(dir, "child"), { recursive: true }); const { prompts, events } = createPrompts({ - confirms: [true, true], + confirms: [true, true, false], selects: ["first", "first", "production-done"], }); const deps = buildDeps({ prompts, cwd: join(dir, "child") }); @@ -432,10 +850,10 @@ describe("runClackWizard", () => { expect(events).toContain(`cancel:${WIZARD_CANCEL_MESSAGE}`); }); - it("supports manual instrumentation without creating local env files", async () => { + it("supports manual instrumentation after creating local env files", async () => { const { prompts, events } = createPrompts({ selects: ["manual", "production-later"], - confirms: [true, true], + confirms: [true, false, true], }); const deps = buildDeps({ prompts }); @@ -456,16 +874,18 @@ describe("runClackWizard", () => { expect(events).toContain( "warn:Do not forget to add BRAINTRUST_API_KEY to production. Braintrust tracing will not work in production without it.", ); - expect(events).not.toContain(`note.message:${ENV_BRAINTRUST_NOTICE}`); - expect(existsSync(join(deps.cwd, ".env.braintrust"))).toBe(false); - expect(existsSync(join(deps.cwd, ".gitignore"))).toBe(false); + expect(events).toContain(`note.message:${ENV_BRAINTRUST_NOTICE}`); + expect(readFileSync(join(deps.cwd, ".env.braintrust"), "utf8")).toBe( + "BRAINTRUST_API_KEY=bt-secret-key\n", + ); + expect(existsSync(join(deps.cwd, ".gitignore"))).toBe(true); }); it("copies an interactive prompt for the user's own coding agent", async () => { let clipboardText = ""; const { prompts, events } = createPrompts({ selects: ["own-agent", "clipboard", "production-done"], - confirms: [true, true], + confirms: [true, false, true], }); const deps = buildDeps({ prompts, @@ -495,7 +915,7 @@ describe("runClackWizard", () => { it("prints an interactive prompt for the user's own coding agent", async () => { const { prompts, events } = createPrompts({ selects: ["own-agent", "terminal", "production-done"], - confirms: [true, true], + confirms: [true, false, true], }); const deps = buildDeps({ prompts }); @@ -516,7 +936,7 @@ describe("runClackWizard", () => { it("prints the own-agent prompt when clipboard copy fails", async () => { const { prompts, events } = createPrompts({ selects: ["own-agent", "clipboard", "production-done"], - confirms: [true, true], + confirms: [true, false, true], }); const deps = buildDeps({ prompts, @@ -540,7 +960,7 @@ describe("runClackWizard", () => { it("fails clearly when the selected tool smoke test fails", async () => { const { prompts } = createPrompts({ - confirms: [true], + confirms: [true, false], selects: ["first", "first"], }); const deps = buildDeps({ diff --git a/packages/spark/test/coding-tools.test.ts b/packages/spark/test/coding-tools.test.ts index c1e02c8..e0ed6cd 100644 --- a/packages/spark/test/coding-tools.test.ts +++ b/packages/spark/test/coding-tools.test.ts @@ -3,7 +3,6 @@ import { describe, expect, it } from "vitest"; import { buildClaudeCommandForTest, buildCodexCommandForTest, - buildToolUnavailableMessage, parseClaudeEventForTest, parseClaudeEventsForTest, parseClaudeStatusForTest, @@ -95,18 +94,6 @@ describe("coding tool status parsing", () => { expect(status.usable).toBe(false); expect(status.unavailableReason).toBe("Not logged in"); }); - - it("formats missing tool messages", () => { - expect( - buildToolUnavailableMessage({ - id: "claude", - label: "Claude Code", - command: "claude", - installed: false, - usable: false, - }), - ).toBe("Claude Code is not installed."); - }); }); describe("coding tool command construction", () => {