Update Agent Evals to run in a custom home directory

scripts/agent-evals/src/runner/gemini-cli-runner.ts

@@ -10,11 +10,12 @@
 } from "./tool-matcher.js";
 import fs from "fs";
 import { throwFailure } from "./logging.js";
-import { getAgentEvalsRoot } from "./paths.js";
+import { getAgentEvalsRoot, RunDirectories } from "./paths.js";
 import { execSync } from "node:child_process";
 import { ToolMockName } from "../mock/tool-mocks.js";
 
 const READY_PROMPT = "Type your message";
+const INSTALL_ID = "238efa5b-efb2-44bd-9dce-9b081532681c";
 
 interface ParsedTelemetryLog {
   attributes?: {
@@ -44,15 +45,16 @@
 
   constructor(
     private readonly testName: string,
-    testDir: string,
-    runDir: string,
+    dirs: RunDirectories,
     toolMocks: ToolMockName[],
   ) {
     // Create a settings file to point the CLI to a local telemetry log
-    this.telemetryPath = path.join(testDir, "telemetry.log");
+    this.telemetryPath = path.join(dirs.testDir, "telemetry.log");
     const mockPath = path.resolve(path.join(getAgentEvalsRoot(), "lib/mock/mock-tools-main.js"));
     const firebasePath = execSync("which firebase").toString().trim();
-    const settings = {
+
+    // Write workspace Gemini Settings
+    this.writeGeminiSettings(dirs.runDir, {
       general: {
         disableAutoUpdate: true,
       },
@@ -71,15 +73,29 @@
           },
         },
       },
-    };
-    const geminiDir = path.join(runDir, ".gemini");
-    mkdirSync(geminiDir, { recursive: true });
-    writeFileSync(path.join(geminiDir, "settings.json"), JSON.stringify(settings, null, 2));
+    });
+
+    // Write user Gemini Settings
+    this.writeGeminiSettings(dirs.userDir, {
+      security: {
+        auth: {
+          selectedType: "gemini-api-key",
+        },
+      },
+      hasSeenIdeIntegrationNudge: true,
+    });
+
+    this.writeGeminiInstallId(dirs.userDir);
 
     this.cli = new InteractiveCLI("gemini", ["--yolo"], {
-      cwd: runDir,
+      cwd: dirs.runDir,
       readyPrompt: READY_PROMPT,
       showOutput: true,
+      env: {
+        // Overwrite $HOME so that we can support GCLI features that only apply
+        // on a per-user basis, like memories and extensions
+        HOME: dirs.userDir,
+      },
     });
   }
 
@@ -101,6 +117,21 @@
     await this.cli.kill();
   }
 
+  writeGeminiSettings(dir: string, settings: any) {
+    const geminiDir = path.join(dir, ".gemini");
+    mkdirSync(geminiDir, { recursive: true });
+    writeFileSync(path.join(geminiDir, "settings.json"), JSON.stringify(settings, null, 2));
+  }
+
+  /**
+   * Writes a constant, real install ID so that we don't bump Gemini metrics
+   * with fake users
+   */
+  writeGeminiInstallId(userDir: string) {
+    const geminiDir = path.join(userDir, ".gemini");
+    writeFileSync(path.join(geminiDir, "installation_id"), INSTALL_ID);
+  }
+
   /**
    * Reads the agent's telemetry file and looks for the given event. Throws if
    * the event is not found
@@ -121,7 +152,7 @@
         const matchingTool = toolLogs.find((log) => log.name === toolName);
         if (!matchingTool) {
           messages.push(
             `Did not find expected tool call: "${toolName}" in the telemetry log. Found [${foundToolNames}]`,
           );
           allSucceeded = false;
         } else {
@@ -145,7 +176,7 @@
   }
 
   // Implementation for this is borrowed from the Gemini CLI's test-helper
   private async waitForTelemetryReady() {
     // Wait for telemetry file to exist and have content
     await poll(() => {
       if (!fs.existsSync(this.telemetryPath)) return false;
@@ -207,8 +238,8 @@
 
     for (const jsonStr of jsonObjects) {
       try {
         const logData = JSON.parse(jsonStr);
         logs.push(logData);
       } catch (e) {
         // Skip objects that aren't valid JSON
       }

scripts/agent-evals/src/runner/index.ts

@@ -3,18 +3,18 @@
 import { mkdirSync } from "node:fs";
 import { AgentTestRunner } from "./agent-test-runner.js";
 import { GeminiCliRunner } from "./gemini-cli-runner.js";
-import { buildFirebaseCli, clearUserMcpServers } from "./setup.js";
+import { buildFirebaseCli } from "./setup.js";
 import { addCleanup } from "../helpers/cleanup.js";
 import { TemplateName, copyTemplate, buildTemplates } from "../template/index.js";
 import { ToolMockName } from "../mock/tool-mocks.js";
+import { RunDirectories } from "./paths.js";
 
 export * from "./agent-test-runner.js";
 
 const dateName = new Date().toISOString().replace("T", "_").replace(/:/g, "-").replace(".", "-");
 
 export async function setupEnvironment(): Promise<void> {
   await buildFirebaseCli();
-  await clearUserMcpServers();
   await buildTemplates();
 }
 
@@ -27,7 +27,7 @@
   toolMocks?: ToolMockName[];
 }
 
 export async function startAgentTest(
   mocha: Mocha.Context,
   options?: AgentTestOptions,
 ): Promise<AgentTestRunner> {
@@ -35,13 +35,13 @@
     throw new Error("startAgentTest must be called inside of an `it` block of a Mocha test.");
   }
   const testName = mocha.test.fullTitle();
-  const { testDir, runDir } = createRunDirectory(testName);
+  const dirs = createRunDirectory(testName);
 
   if (options?.templateName) {
-    copyTemplate(options.templateName, runDir);
+    copyTemplate(options.templateName, dirs.runDir);
   }
 
-  const run = new GeminiCliRunner(testName, testDir, runDir, options?.toolMocks || []);
+  const run = new GeminiCliRunner(testName, dirs, options?.toolMocks || []);
   await run.waitForReadyPrompt();
 
   addCleanup(async () => {
@@ -51,12 +51,17 @@
   return run;
 }
 
-function createRunDirectory(testName: string): { testDir: string; runDir: string } {
+function createRunDirectory(testName: string): RunDirectories {
   const sanitizedName = testName.toLowerCase().replace(/[^a-z0-9]/g, "-");
   const testDir = path.resolve(
     path.join("output", dateName, `${sanitizedName}-${randomBytes(8).toString("hex")}`),
   );
+
   const runDir = path.join(testDir, "repo");
   mkdirSync(runDir, { recursive: true });
-  return { testDir, runDir };
+
+  const userDir = path.join(testDir, "user");
+  mkdirSync(userDir, { recursive: true });
+
+  return { testDir, runDir, userDir };
 }

scripts/agent-evals/src/runner/paths.ts

@@ -1,7 +1,9 @@
 import path from "path";
 import { fileURLToPath } from "url";
 
+export type RunDirectories = { testDir: string; runDir: string; userDir: string };
+
 export function getAgentEvalsRoot(): string {
   const thisFilePath = path.dirname(fileURLToPath(import.meta.url));
   return path.resolve(path.join(thisFilePath, "..", ".."));
 }

scripts/agent-evals/src/runner/setup.ts

@@ -13,17 +13,3 @@ export async function buildFirebaseCli() {
   console.log(`Building Firebase CLI at ${firebaseCliRoot}`);
   await execPromise("./scripts/clean-install.sh", { cwd: firebaseCliRoot });
 }
-
-export async function clearUserMcpServers() {
-  console.log(`Clearing existing MCP servers...`);
-  try {
-    await execPromise("gemini extensions uninstall firebase");
-  } catch (_: any) {
-    /* This can fail if there's nothing installed, so ignore that */
-  }
-  try {
-    await execPromise("gemini mcp remove firebase");
-  } catch (_: any) {
-    /* This can fail if there's nothing installed, so ignore that */
-  }
-}