From f3f06fa4c8f3c37b498cf8ea8be77fb93ab20da1 Mon Sep 17 00:00:00 2001 From: Sean McGuire Date: Sun, 30 Nov 2025 22:37:11 -0800 Subject: [PATCH 1/8] add relevant keys to e2e local step --- .github/workflows/ci.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 142208731..94cd19923 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -187,8 +187,15 @@ jobs: needs: [run-lint, run-build] runs-on: ubuntu-latest timeout-minutes: 50 + if: > + github.event_name == 'push' || + (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + GOOGLE_GENERATIVE_AI_API_KEY: ${{ secrets.GOOGLE_GENERATIVE_AI_API_KEY }} + BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY }} + BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID }} HEADLESS: true steps: - name: Check out repository code From d5c58637f3fc366743403a4e4b21c0ef1f8c4488 Mon Sep 17 00:00:00 2001 From: Sean McGuire Date: Sun, 30 Nov 2025 22:59:12 -0800 Subject: [PATCH 2/8] pass in model keys directly in agent stream test --- .../core/lib/v3/tests/agent-streaming.spec.ts | 35 +++++++++++++++---- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/packages/core/lib/v3/tests/agent-streaming.spec.ts b/packages/core/lib/v3/tests/agent-streaming.spec.ts index 35020beb3..b205f56e1 100644 --- a/packages/core/lib/v3/tests/agent-streaming.spec.ts +++ b/packages/core/lib/v3/tests/agent-streaming.spec.ts @@ -24,7 +24,10 @@ test.describe("Stagehand agent streaming behavior", () => { const agent = v3.agent({ stream: true, - model: "anthropic/claude-haiku-4-5-20251001", + model: { + modelName: "anthropic/claude-haiku-4-5-20251001", + apiKey: process.env.ANTHROPIC_API_KEY, + }, }); // Navigate to a simple page first @@ -55,7 +58,10 @@ test.describe("Stagehand agent streaming behavior", () => { const agent = v3.agent({ stream: true, - model: "anthropic/claude-haiku-4-5-20251001", + model: { + modelName: "anthropic/claude-haiku-4-5-20251001", + apiKey: process.env.ANTHROPIC_API_KEY, + }, }); const page = v3.context.pages()[0]; @@ -84,7 +90,10 @@ test.describe("Stagehand agent streaming behavior", () => { const agent = v3.agent({ stream: true, - model: "anthropic/claude-haiku-4-5-20251001", + model: { + modelName: "anthropic/claude-haiku-4-5-20251001", + apiKey: process.env.ANTHROPIC_API_KEY, + }, }); const page = v3.context.pages()[0]; @@ -121,7 +130,10 @@ test.describe("Stagehand agent streaming behavior", () => { test.setTimeout(60000); const agent = v3.agent({ - model: "anthropic/claude-haiku-4-5-20251001", + model: { + modelName: "anthropic/claude-haiku-4-5-20251001", + apiKey: process.env.ANTHROPIC_API_KEY, + }, }); const page = v3.context.pages()[0]; @@ -149,7 +161,10 @@ test.describe("Stagehand agent streaming behavior", () => { v3.agent({ cua: true, stream: true, - model: "anthropic/claude-haiku-4-5-20251001", + model: { + modelName: "anthropic/claude-haiku-4-5-20251001", + apiKey: process.env.ANTHROPIC_API_KEY, + }, }); }).toThrow("Streaming is not supported with CUA"); }); @@ -158,7 +173,10 @@ test.describe("Stagehand agent streaming behavior", () => { // Should not throw const agent = v3.agent({ cua: true, - model: "anthropic/claude-haiku-4-5-20251001", + model: { + modelName: "anthropic/claude-haiku-4-5-20251001", + apiKey: process.env.ANTHROPIC_API_KEY, + }, }); expect(agent).toHaveProperty("execute"); @@ -168,7 +186,10 @@ test.describe("Stagehand agent streaming behavior", () => { // Should not throw const agent = v3.agent({ stream: true, - model: "anthropic/claude-haiku-4-5-20251001", + model: { + modelName: "anthropic/claude-haiku-4-5-20251001", + apiKey: process.env.ANTHROPIC_API_KEY, + }, }); expect(agent).toHaveProperty("execute"); From e5b35e33bd2180868683a87466e1cbc69ddf7a7f Mon Sep 17 00:00:00 2001 From: Sean McGuire Date: Sun, 30 Nov 2025 23:27:37 -0800 Subject: [PATCH 3/8] update pw test config to read from correct env dir --- packages/core/lib/v3/tests/v3.playwright.config.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/packages/core/lib/v3/tests/v3.playwright.config.ts b/packages/core/lib/v3/tests/v3.playwright.config.ts index f9745c077..819eca416 100644 --- a/packages/core/lib/v3/tests/v3.playwright.config.ts +++ b/packages/core/lib/v3/tests/v3.playwright.config.ts @@ -1,4 +1,15 @@ import { defineConfig } from "@playwright/test"; +import dotenv from "dotenv"; +import path from "path"; + +// Load environment variables so agent-streaming tests can reach Anthropic in CI +dotenv.config(); +const repoRootEnvPath = path.resolve(__dirname, "../../../../../.env"); +dotenv.config({ path: repoRootEnvPath, override: false }); + +if (!process.env.TEST_ENV) { + process.env.TEST_ENV = "LOCAL"; +} export default defineConfig({ testDir: ".", From ad41997317a699383df1e0c89c0b29c0f5697558 Mon Sep 17 00:00:00 2001 From: Sean McGuire Date: Sun, 30 Nov 2025 23:28:13 -0800 Subject: [PATCH 4/8] rm comment --- packages/core/lib/v3/tests/v3.playwright.config.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/core/lib/v3/tests/v3.playwright.config.ts b/packages/core/lib/v3/tests/v3.playwright.config.ts index 819eca416..5235c630d 100644 --- a/packages/core/lib/v3/tests/v3.playwright.config.ts +++ b/packages/core/lib/v3/tests/v3.playwright.config.ts @@ -2,7 +2,6 @@ import { defineConfig } from "@playwright/test"; import dotenv from "dotenv"; import path from "path"; -// Load environment variables so agent-streaming tests can reach Anthropic in CI dotenv.config(); const repoRootEnvPath = path.resolve(__dirname, "../../../../../.env"); dotenv.config({ path: repoRootEnvPath, override: false }); From 8301534c9913991ead4bd9210033c0725a388580 Mon Sep 17 00:00:00 2001 From: Sean McGuire Date: Sun, 30 Nov 2025 23:29:59 -0800 Subject: [PATCH 5/8] revert inline model config changes in agent-streaming.spec.ts --- .../core/lib/v3/tests/agent-streaming.spec.ts | 30 ++++--------------- 1 file changed, 6 insertions(+), 24 deletions(-) diff --git a/packages/core/lib/v3/tests/agent-streaming.spec.ts b/packages/core/lib/v3/tests/agent-streaming.spec.ts index b205f56e1..1dfb5abf1 100644 --- a/packages/core/lib/v3/tests/agent-streaming.spec.ts +++ b/packages/core/lib/v3/tests/agent-streaming.spec.ts @@ -24,10 +24,7 @@ test.describe("Stagehand agent streaming behavior", () => { const agent = v3.agent({ stream: true, - model: { - modelName: "anthropic/claude-haiku-4-5-20251001", - apiKey: process.env.ANTHROPIC_API_KEY, - }, + model: "anthropic/claude-haiku-4-5-20251001", }); // Navigate to a simple page first @@ -90,10 +87,7 @@ test.describe("Stagehand agent streaming behavior", () => { const agent = v3.agent({ stream: true, - model: { - modelName: "anthropic/claude-haiku-4-5-20251001", - apiKey: process.env.ANTHROPIC_API_KEY, - }, + model: "anthropic/claude-haiku-4-5-20251001", }); const page = v3.context.pages()[0]; @@ -130,10 +124,7 @@ test.describe("Stagehand agent streaming behavior", () => { test.setTimeout(60000); const agent = v3.agent({ - model: { - modelName: "anthropic/claude-haiku-4-5-20251001", - apiKey: process.env.ANTHROPIC_API_KEY, - }, + model: "anthropic/claude-haiku-4-5-20251001", }); const page = v3.context.pages()[0]; @@ -161,10 +152,7 @@ test.describe("Stagehand agent streaming behavior", () => { v3.agent({ cua: true, stream: true, - model: { - modelName: "anthropic/claude-haiku-4-5-20251001", - apiKey: process.env.ANTHROPIC_API_KEY, - }, + model: "anthropic/claude-haiku-4-5-20251001", }); }).toThrow("Streaming is not supported with CUA"); }); @@ -173,10 +161,7 @@ test.describe("Stagehand agent streaming behavior", () => { // Should not throw const agent = v3.agent({ cua: true, - model: { - modelName: "anthropic/claude-haiku-4-5-20251001", - apiKey: process.env.ANTHROPIC_API_KEY, - }, + model: "anthropic/claude-haiku-4-5-20251001", }); expect(agent).toHaveProperty("execute"); @@ -186,10 +171,7 @@ test.describe("Stagehand agent streaming behavior", () => { // Should not throw const agent = v3.agent({ stream: true, - model: { - modelName: "anthropic/claude-haiku-4-5-20251001", - apiKey: process.env.ANTHROPIC_API_KEY, - }, + model: "anthropic/claude-haiku-4-5-20251001", }); expect(agent).toHaveProperty("execute"); From a38ef5bc5cfa17d826c9c4e8a58a989506185b92 Mon Sep 17 00:00:00 2001 From: Sean McGuire Date: Sun, 30 Nov 2025 23:30:50 -0800 Subject: [PATCH 6/8] revert another model config change in agent-streaming.spec.ts --- packages/core/lib/v3/tests/agent-streaming.spec.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/core/lib/v3/tests/agent-streaming.spec.ts b/packages/core/lib/v3/tests/agent-streaming.spec.ts index 1dfb5abf1..35020beb3 100644 --- a/packages/core/lib/v3/tests/agent-streaming.spec.ts +++ b/packages/core/lib/v3/tests/agent-streaming.spec.ts @@ -55,10 +55,7 @@ test.describe("Stagehand agent streaming behavior", () => { const agent = v3.agent({ stream: true, - model: { - modelName: "anthropic/claude-haiku-4-5-20251001", - apiKey: process.env.ANTHROPIC_API_KEY, - }, + model: "anthropic/claude-haiku-4-5-20251001", }); const page = v3.context.pages()[0]; From f5d2d04ce593754b184c90a1e2799472349fe757 Mon Sep 17 00:00:00 2001 From: Sean McGuire Date: Mon, 1 Dec 2025 09:59:44 -0800 Subject: [PATCH 7/8] add anthropic key to turbo.json --- turbo.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/turbo.json b/turbo.json index 3f442ed53..7754d6dde 100644 --- a/turbo.json +++ b/turbo.json @@ -87,7 +87,10 @@ "playwright-report/**", "test-results/**" ], - "cache": false + "cache": false, + "env": [ + "ANTHROPIC_API_KEY" + ] }, "e2e:bb": { "dependsOn": ["build"], From 7eb1ed9f45f51ee59e56266071d50e3f4c1d8a82 Mon Sep 17 00:00:00 2001 From: Sean McGuire Date: Mon, 1 Dec 2025 10:31:10 -0800 Subject: [PATCH 8/8] revert changes to v3 playwright config --- packages/core/lib/v3/tests/v3.playwright.config.ts | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/packages/core/lib/v3/tests/v3.playwright.config.ts b/packages/core/lib/v3/tests/v3.playwright.config.ts index 5235c630d..f9745c077 100644 --- a/packages/core/lib/v3/tests/v3.playwright.config.ts +++ b/packages/core/lib/v3/tests/v3.playwright.config.ts @@ -1,14 +1,4 @@ import { defineConfig } from "@playwright/test"; -import dotenv from "dotenv"; -import path from "path"; - -dotenv.config(); -const repoRootEnvPath = path.resolve(__dirname, "../../../../../.env"); -dotenv.config({ path: repoRootEnvPath, override: false }); - -if (!process.env.TEST_ENV) { - process.env.TEST_ENV = "LOCAL"; -} export default defineConfig({ testDir: ".",