test: add browser tests for agent inspector

.github/workflows/e2e-tests-full.yml

@@ -79,3 +79,19 @@ jobs:
           GEMINI_API_KEY: ${{ env.E2E_GEMINI_API_KEY }}
           CDK_TARBALL: ${{ env.CDK_TARBALL }}
         run: npm run test:e2e
+      - name: Install Playwright browsers
+        run: npx playwright install chromium --with-deps
+      - name: Run browser tests
+        env:
+          AWS_ACCOUNT_ID: ${{ steps.aws.outputs.account_id }}
+          AWS_REGION: ${{ inputs.aws_region || 'us-east-1' }}
+          PLAYWRIGHT_TRACE: 'off'
+        run: npm run test:browser
+      - name: Print browser test debug info
+        if: failure()
+        run: |
+          echo "=== Dev server PTY output ==="
+          cat test-results/agentcore-dev-pty.log 2>/dev/null || echo "(no pty log)"
+          echo ""
+          echo "=== Error contexts ==="
+          find test-results -name 'error-context.md' -exec echo "--- {} ---" \; -exec cat {} \; 2>/dev/null || echo "(no error contexts)"

.gitignore

@@ -67,3 +67,8 @@ ProtocolTesting/
 
 # Auto-cloned CDK constructs (from scripts/bundle.mjs)
 .cdk-constructs-clone/
+
+# Browser tests
+browser-tests/.browser-test-env
+browser-tests/test-results/
+browser-tests/playwright-report/

browser-tests/constants.ts

@@ -0,0 +1,3 @@
+import { join } from 'node:path';
+
+export const ENV_FILE = join(__dirname, '.browser-test-env');

browser-tests/fixtures.ts

@@ -0,0 +1,56 @@
+import { ENV_FILE } from './constants';
+import { type Page, test as base, expect } from '@playwright/test';
+import { readFileSync } from 'node:fs';
+
+interface BrowserTestEnv {
+  projectPath: string;
+  port: number;
+  projectName: string;
+}
+
+function readTestEnv(): BrowserTestEnv {
+  const raw = readFileSync(ENV_FILE, 'utf-8');
+  const parsed: Record<string, string> = {};
+  for (const line of raw.split('\n')) {
+    const match = line.match(/^(\w+)=(.+)$/);
+    if (match) parsed[match[1]!] = match[2]!;
+  }
+  return {
+    projectPath: parsed.PROJECT_PATH!,
+    port: Number(parsed.PORT),
+    projectName: parsed.PROJECT_NAME!,
+  };
+}
+
+export const test = base.extend<{ testEnv: BrowserTestEnv }>({
+  testEnv: async ({}, use) => {
+    await use(readTestEnv());
+  },
+});
+
+/**
+ * Send a chat message and wait for the agent to finish responding.
+ * Returns the assistant message locator.
+ */
+export async function sendMessage(page: Page, text: string) {
+  const chatInput = page.getByTestId('chat-input');
+  await expect(chatInput).toBeEnabled({ timeout: 60_000 });
+
+  const messageList = page.getByTestId('message-list');
+  const existingCount = await messageList.getByTestId(/^chat-message-/).count();
+
+  await chatInput.fill(text);
+  await page.getByRole('button', { name: 'Send message' }).click();
+
+  const assistantMessage = messageList.getByTestId(`chat-message-${existingCount + 1}`);
+  await expect(assistantMessage).toBeVisible({ timeout: 60_000 });
+  await expect(assistantMessage).not.toContainText('ECONNREFUSED');
+
+  // Wait for streaming to complete so the agent is idle for subsequent tests.
+  await chatInput.fill('.');
+  await expect(page.getByRole('button', { name: 'Send message' })).toBeEnabled({ timeout: 30_000 });
+
+  return assistantMessage;
+}
+
+export { expect };

browser-tests/global-setup.ts

@@ -0,0 +1,144 @@
+import { ENV_FILE } from './constants';
+import * as pty from 'node-pty';
+import { type ExecSyncOptions, execSync } from 'node:child_process';
+import { randomUUID } from 'node:crypto';
+import { createWriteStream, mkdirSync, writeFileSync } from 'node:fs';
+import { createConnection } from 'node:net';
+import { tmpdir } from 'node:os';
+import { join, resolve } from 'node:path';
+
+const CLI_PATH = join(__dirname, '..', 'dist', 'cli', 'index.mjs');
+const PTY_LOG = join(__dirname, 'test-results', 'agentcore-dev-pty.log');
+
+function hasAwsCredentials(): boolean {
+  try {
+    execSync('aws sts get-caller-identity', { stdio: 'ignore' });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+function hasCommand(cmd: string): boolean {
+  try {
+    execSync(`which ${cmd}`, { stdio: 'ignore' });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+async function waitForServerReady(port: number, timeoutMs = 90000): Promise<boolean> {
+  const start = Date.now();
+  while (Date.now() - start < timeoutMs) {
+    const listening = await new Promise<boolean>(resolve => {
+      const socket = createConnection({ port, host: '127.0.0.1' }, () => {
+        socket.destroy();
+        resolve(true);
+      });
+      socket.on('error', () => {
+        socket.destroy();
+        resolve(false);
+      });
+    });
+    if (listening) return true;
+    await new Promise(resolve => setTimeout(resolve, 500));
+  }
+  return false;
+}
+
+export default async function globalSetup() {
+  const missing: string[] = [];
+  if (!hasAwsCredentials()) missing.push('AWS credentials (run `aws sts get-caller-identity`)');
+  if (!hasCommand('uv')) missing.push('`uv` on PATH');
+
+  if (missing.length > 0) {
+    if (process.env.CI) {
+      throw new Error(`Browser tests require: ${missing.join(', ')}`);
+    }
+    console.log(`\nSkipping browser tests — missing: ${missing.join(', ')}\n`);
+    process.exit(0);
+  }
+
+  const testDir = join(tmpdir(), `agentcore-browser-test-${randomUUID()}`);
+  mkdirSync(testDir, { recursive: true });
+
+  const projectName = `BrTest${String(Date.now()).slice(-8)}`;
+
+  console.log(`\nCreating test project "${projectName}" in ${testDir}`);
+
+  const cleanEnv = { ...process.env };
+  delete cleanEnv.INIT_CWD;
+
+  const execOpts: ExecSyncOptions = { cwd: testDir, stdio: 'pipe', env: cleanEnv };
+
+  let createRaw: string;
+  try {
+    createRaw = execSync(
+      `node ${CLI_PATH} create --name ${projectName} --language Python --framework Strands --model-provider Bedrock --memory none --json`,
+      execOpts
+    ).toString();
+  } catch (err: unknown) {
+    const e = err as { stderr?: Buffer; stdout?: Buffer; status?: number };
+    const stderr = e.stderr?.toString() ?? '';
+    const stdout = e.stdout?.toString() ?? '';
+    throw new Error(`agentcore create failed (exit ${e.status}):\nstdout: ${stdout}\nstderr: ${stderr}`);
+  }
+
+  // eslint-disable-next-line no-control-regex
+  const createResult = createRaw.replace(/\x1B\[\??\d*[a-zA-Z]/g, '').trim();
+  const parsed = JSON.parse(createResult.split('\n').pop()!);
+  const projectPath: string = resolve(testDir, parsed.projectPath);
+
+  console.log(`Project created at ${projectPath}`);
+  console.log(`Starting agentcore dev...`);
+
+  const env = { ...process.env };
+  delete env.INIT_CWD;
+  if (env.AGENT_INSPECTOR_PATH) {
+    env.AGENT_INSPECTOR_PATH = resolve(env.AGENT_INSPECTOR_PATH);
+  }
+
+  const ptyProcess = pty.spawn('node', [CLI_PATH, 'dev'], {
+    cwd: projectPath,
+    env,
+    cols: 80,
+    rows: 24,
+  });
+
+  mkdirSync(join(__dirname, 'test-results'), { recursive: true });
+  // eslint-disable-next-line no-control-regex
+  const stripAnsi = (s: string) => s.replace(/\x1B\[\??[\d;]*[a-zA-Z]/g, '');
+  const ptyLog = createWriteStream(PTY_LOG);
+
+  let serverOutput = '';
+  const webUIPort = await new Promise<number>((resolvePort, reject) => {
+    const timeout = setTimeout(() => {
+      ptyProcess.kill();
+      reject(new Error(`agentcore dev failed to start within timeout.\nOutput: ${serverOutput}`));
+    }, 90000);
+
+    ptyProcess.onData((data: string) => {
+      serverOutput += data;
+      ptyLog.write(stripAnsi(data));
+      const match = serverOutput.match(/Chat UI: http:\/\/localhost:(\d+)/);
+      if (match) {
+        clearTimeout(timeout);
+        resolvePort(parseInt(match[1]!, 10));
+      }
+    });
+  });
+
+  const ready = await waitForServerReady(webUIPort);
+  if (!ready) {
+    ptyProcess.kill();
+    throw new Error(`Web UI reported port ${webUIPort} but it is not responding.\nOutput: ${serverOutput}`);
+  }
+
+  console.log(`Dev server ready on port ${webUIPort}`);
+
+  writeFileSync(
+    ENV_FILE,
+    `PROJECT_PATH=${projectPath}\nPORT=${webUIPort}\nTEST_DIR=${testDir}\nSERVER_PID=${ptyProcess.pid}\nPROJECT_NAME=${projectName}\n`
+  );
+}

browser-tests/global-teardown.ts

@@ -0,0 +1,39 @@
+import { ENV_FILE } from './constants';
+import { cpSync, existsSync, mkdirSync, readFileSync, rmSync, unlinkSync } from 'node:fs';
+import { join } from 'node:path';
+
+export default async function globalTeardown() {
+  if (!existsSync(ENV_FILE)) return;
+
+  const raw = readFileSync(ENV_FILE, 'utf-8');
+
+  const serverPid = raw.match(/^SERVER_PID=(.+)$/m)?.[1];
+  if (serverPid) {
+    try {
+      process.kill(Number(serverPid), 'SIGTERM');
+      console.log(`\nStopped dev server (PID ${serverPid})`);
+    } catch {
+      // Process already exited
+    }
+    await new Promise<void>(resolve => setTimeout(resolve, 2000));
+  }
+
+  const projectPath = raw.match(/^PROJECT_PATH=(.+)$/m)?.[1];
+  const testDir = raw.match(/^TEST_DIR=(.+)$/m)?.[1];
+
+  if (projectPath) {
+    const logsDir = join(projectPath, 'agentcore', '.cli', 'logs');
+    const outputDir = join(__dirname, 'test-results', 'dev-server-logs');
+    if (existsSync(logsDir)) {
+      mkdirSync(outputDir, { recursive: true });
+      cpSync(logsDir, outputDir, { recursive: true });
+    }
+  }
+
+  if (testDir && existsSync(testDir)) {
+    console.log(`Cleaning up ${testDir}`);
+    rmSync(testDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 1000 });
+  }
+
+  unlinkSync(ENV_FILE);
+}

browser-tests/playwright.config.ts

@@ -0,0 +1,33 @@
+import { ENV_FILE } from './constants';
+import { defineConfig, devices } from '@playwright/test';
+import { readFileSync } from 'node:fs';
+
+function getPort(): number {
+  try {
+    const raw = readFileSync(ENV_FILE, 'utf-8');
+    const match = raw.match(/^PORT=(\d+)$/m);
+    if (match) return parseInt(match[1]!, 10);
+  } catch {}
+  return 8081;
+}
+
+export default defineConfig({
+  testDir: './tests',
+  fullyParallel: false,
+  workers: 1,
+  timeout: 120_000,
+  retries: process.env.CI ? 1 : 0,
+  outputDir: './test-results',
+  reporter: [['html', { open: 'never', outputFolder: './playwright-report' }]],
+
+  globalSetup: './global-setup.ts',
+  globalTeardown: './global-teardown.ts',
+
+  use: {
+    baseURL: `http://localhost:${getPort()}`,
+    trace: process.env.PLAYWRIGHT_TRACE === 'off' ? 'off' : 'retain-on-failure',
+    screenshot: 'only-on-failure',
+  },
+
+  projects: [{ name: 'chromium', use: { ...devices['Desktop Chrome'] } }],
+});

browser-tests/tests/chat-invocation.test.ts

@@ -0,0 +1,10 @@
+import { expect, sendMessage, test } from '../fixtures';
+
+test.describe('Chat invocation', () => {
+  test('send a message and receive a response', async ({ page }) => {
+    await page.goto('/');
+
+    const assistantMessage = await sendMessage(page, 'What is 2 plus 2? Reply with just the number.');
+    await expect(assistantMessage).not.toBeEmpty();
+  });
+});

browser-tests/tests/inspector-loads.test.ts

@@ -0,0 +1,13 @@
+import { expect, test } from '../fixtures';
+
+test.describe('Inspector loads', () => {
+  test('page renders and shows the agent', async ({ page, testEnv }) => {
+    await page.goto('/');
+
+    await expect(page.locator('header')).toBeVisible();
+
+    const agentStatus = page.getByTestId('agent-status');
+    await expect(agentStatus).toBeVisible({ timeout: 30_000 });
+    await expect(agentStatus).toContainText(testEnv.projectName);
+  });
+});

browser-tests/tests/resources.test.ts

@@ -0,0 +1,19 @@
+import { expect, test } from '../fixtures';
+
+test.describe('Resources', () => {
+  test('resource panel shows the agent', async ({ page, testEnv }) => {
+    await page.goto('/');
+
+    const resourcePanel = page.getByTestId('resource-panel');
+    await expect(resourcePanel).toBeVisible({ timeout: 10_000 });
+
+    const resourcesTab = resourcePanel.getByRole('tab', { name: 'Resources' });
+    await resourcesTab.click();
+
+    const agentNode = resourcePanel.getByRole('button', { name: new RegExp(`agent: ${testEnv.projectName}`, 'i') });
+    await expect(agentNode).toBeVisible({ timeout: 10_000 });
+
+    await page.getByRole('button', { name: 'Toggle resource panel' }).click();
+    await expect(resourcePanel).not.toBeVisible();
+  });
+});

browser-tests/tests/start-agent.test.ts

@@ -0,0 +1,16 @@
+import { expect, test } from '../fixtures';
+
+test.describe('Start agent', () => {
+  test('agent starts and shows running status', async ({ page }) => {
+    await page.goto('/');
+
+    const agentStatus = page.getByTestId('agent-status');
+    await expect(agentStatus).toBeVisible({ timeout: 30_000 });
+
+    const chatInput = page.getByTestId('chat-input');
+    await expect(chatInput).toBeVisible({ timeout: 60_000 });
+    await expect(chatInput).toBeEnabled({ timeout: 60_000 });
+
+    await expect(page.getByText('Error')).not.toBeVisible();
+  });
+});

browser-tests/tests/traces.test.ts

@@ -0,0 +1,22 @@
+import { expect, sendMessage, test } from '../fixtures';
+
+test.describe('Traces', () => {
+  test('traces panel shows trace after invocation', async ({ page }) => {
+    await page.goto('/');
+
+    await sendMessage(page, 'Say hello');
+
+    await page.getByRole('tab', { name: 'Traces' }).click();
+
+    const traceList = page.getByTestId('trace-list');
+    await expect(traceList).toBeVisible({ timeout: 30_000 });
+
+    const traceButton = traceList.getByRole('button').first();
+    await expect(traceButton).toBeVisible({ timeout: 30_000 });
+
+    await traceButton.click();
+
+    const spanRow = page.locator('[role="button"]').filter({ hasText: /.+/ });
+    await expect(spanRow.first()).toBeVisible({ timeout: 10_000 });
+  });
+});