test(app): windows e2e tests

.github/workflows/test.yml

@@ -8,7 +8,29 @@ on:
   workflow_dispatch:
 jobs:
   test:
-    runs-on: blacksmith-4vcpu-ubuntu-2404
+    name: test (${{ matrix.settings.name }})
+    strategy:
+      fail-fast: false
+      matrix:
+        settings:
+          - name: linux
+            host: blacksmith-4vcpu-ubuntu-2404
+            playwright: bunx playwright install --with-deps
+            workdir: .
+            command: |
+              git config --global user.email "bot@opencode.ai"
+              git config --global user.name "opencode"
+              bun turbo typecheck
+              bun turbo test
+          - name: windows
+            host: windows-latest
+            playwright: bunx playwright install
+            workdir: packages/app
+            command: bun test:e2e
+    runs-on: ${{ matrix.settings.host }}
+    defaults:
+      run:
+        shell: bash
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -20,42 +42,63 @@ jobs:
 
       - name: Install Playwright browsers
         working-directory: packages/app
-        run: bunx playwright install --with-deps
+        run: ${{ matrix.settings.playwright }}
+
+      - name: Set OS-specific paths
+        run: |
+          if [ "${{ runner.os }}" = "Windows" ]; then
+            printf '%s\n' "OPENCODE_E2E_ROOT=${{ runner.temp }}\\opencode-e2e" >> "$GITHUB_ENV"
+            printf '%s\n' "OPENCODE_TEST_HOME=${{ runner.temp }}\\opencode-e2e\\home" >> "$GITHUB_ENV"
+            printf '%s\n' "XDG_DATA_HOME=${{ runner.temp }}\\opencode-e2e\\share" >> "$GITHUB_ENV"
+            printf '%s\n' "XDG_CACHE_HOME=${{ runner.temp }}\\opencode-e2e\\cache" >> "$GITHUB_ENV"
+            printf '%s\n' "XDG_CONFIG_HOME=${{ runner.temp }}\\opencode-e2e\\config" >> "$GITHUB_ENV"
+            printf '%s\n' "XDG_STATE_HOME=${{ runner.temp }}\\opencode-e2e\\state" >> "$GITHUB_ENV"
+            printf '%s\n' "MODELS_DEV_API_JSON=${{ github.workspace }}\\packages\\opencode\\test\\tool\\fixtures\\models-api.json" >> "$GITHUB_ENV"
+          else
+            printf '%s\n' "OPENCODE_E2E_ROOT=${{ runner.temp }}/opencode-e2e" >> "$GITHUB_ENV"
+            printf '%s\n' "OPENCODE_TEST_HOME=${{ runner.temp }}/opencode-e2e/home" >> "$GITHUB_ENV"
+            printf '%s\n' "XDG_DATA_HOME=${{ runner.temp }}/opencode-e2e/share" >> "$GITHUB_ENV"
+            printf '%s\n' "XDG_CACHE_HOME=${{ runner.temp }}/opencode-e2e/cache" >> "$GITHUB_ENV"
+            printf '%s\n' "XDG_CONFIG_HOME=${{ runner.temp }}/opencode-e2e/config" >> "$GITHUB_ENV"
+            printf '%s\n' "XDG_STATE_HOME=${{ runner.temp }}/opencode-e2e/state" >> "$GITHUB_ENV"
+            printf '%s\n' "MODELS_DEV_API_JSON=${{ github.workspace }}/packages/opencode/test/tool/fixtures/models-api.json" >> "$GITHUB_ENV"
+          fi
 
       - name: Seed opencode data
         working-directory: packages/opencode
         run: bun script/seed-e2e.ts
         env:
-          MODELS_DEV_API_JSON: ${{ github.workspace }}/packages/opencode/test/tool/fixtures/models-api.json
+          MODELS_DEV_API_JSON: ${{ env.MODELS_DEV_API_JSON }}
           OPENCODE_DISABLE_MODELS_FETCH: "true"
           OPENCODE_DISABLE_SHARE: "true"
           OPENCODE_DISABLE_LSP_DOWNLOAD: "true"
           OPENCODE_DISABLE_DEFAULT_PLUGINS: "true"
           OPENCODE_EXPERIMENTAL_DISABLE_FILEWATCHER: "true"
-          OPENCODE_TEST_HOME: ${{ runner.temp }}/opencode-e2e/home
-          XDG_DATA_HOME: ${{ runner.temp }}/opencode-e2e/share
-          XDG_CACHE_HOME: ${{ runner.temp }}/opencode-e2e/cache
-          XDG_CONFIG_HOME: ${{ runner.temp }}/opencode-e2e/config
-          XDG_STATE_HOME: ${{ runner.temp }}/opencode-e2e/state
+          OPENCODE_TEST_HOME: ${{ env.OPENCODE_TEST_HOME }}
+          XDG_DATA_HOME: ${{ env.XDG_DATA_HOME }}
+          XDG_CACHE_HOME: ${{ env.XDG_CACHE_HOME }}
+          XDG_CONFIG_HOME: ${{ env.XDG_CONFIG_HOME }}
+          XDG_STATE_HOME: ${{ env.XDG_STATE_HOME }}
           OPENCODE_E2E_PROJECT_DIR: ${{ github.workspace }}
           OPENCODE_E2E_SESSION_TITLE: "E2E Session"
           OPENCODE_E2E_MESSAGE: "Seeded for UI e2e"
           OPENCODE_E2E_MODEL: "opencode/gpt-5-nano"
 
       - name: Run opencode server
-        run: bun run dev -- --print-logs --log-level WARN serve --port 4096 --hostname 0.0.0.0 &
+        working-directory: packages/opencode
+        run: bun dev -- --print-logs --log-level WARN serve --port 4096 --hostname 0.0.0.0 &
         env:
-          MODELS_DEV_API_JSON: ${{ github.workspace }}/packages/opencode/test/tool/fixtures/models-api.json
+          MODELS_DEV_API_JSON: ${{ env.MODELS_DEV_API_JSON }}
           OPENCODE_DISABLE_MODELS_FETCH: "true"
           OPENCODE_DISABLE_SHARE: "true"
           OPENCODE_DISABLE_LSP_DOWNLOAD: "true"
           OPENCODE_DISABLE_DEFAULT_PLUGINS: "true"
           OPENCODE_EXPERIMENTAL_DISABLE_FILEWATCHER: "true"
-          OPENCODE_TEST_HOME: ${{ runner.temp }}/opencode-e2e/home
-          XDG_DATA_HOME: ${{ runner.temp }}/opencode-e2e/share
-          XDG_CACHE_HOME: ${{ runner.temp }}/opencode-e2e/cache
-          XDG_CONFIG_HOME: ${{ runner.temp }}/opencode-e2e/config
-          XDG_STATE_HOME: ${{ runner.temp }}/opencode-e2e/state
+          OPENCODE_TEST_HOME: ${{ env.OPENCODE_TEST_HOME }}
+          XDG_DATA_HOME: ${{ env.XDG_DATA_HOME }}
+          XDG_CACHE_HOME: ${{ env.XDG_CACHE_HOME }}
+          XDG_CONFIG_HOME: ${{ env.XDG_CONFIG_HOME }}
+          XDG_STATE_HOME: ${{ env.XDG_STATE_HOME }}
           OPENCODE_CLIENT: "app"
 
       - name: Wait for opencode server
@@ -67,24 +110,21 @@ jobs:
           exit 1
 
       - name: run
-        run: |
-          git config --global user.email "bot@opencode.ai"
-          git config --global user.name "opencode"
-          bun turbo typecheck
-          bun turbo test
+        working-directory: ${{ matrix.settings.workdir }}
+        run: ${{ matrix.settings.command }}
         env:
           CI: true
-          MODELS_DEV_API_JSON: ${{ github.workspace }}/packages/opencode/test/tool/fixtures/models-api.json
+          MODELS_DEV_API_JSON: ${{ env.MODELS_DEV_API_JSON }}
           OPENCODE_DISABLE_MODELS_FETCH: "true"
           OPENCODE_DISABLE_SHARE: "true"
           OPENCODE_DISABLE_LSP_DOWNLOAD: "true"
           OPENCODE_DISABLE_DEFAULT_PLUGINS: "true"
           OPENCODE_EXPERIMENTAL_DISABLE_FILEWATCHER: "true"
-          OPENCODE_TEST_HOME: ${{ runner.temp }}/opencode-e2e/home
-          XDG_DATA_HOME: ${{ runner.temp }}/opencode-e2e/share
-          XDG_CACHE_HOME: ${{ runner.temp }}/opencode-e2e/cache
-          XDG_CONFIG_HOME: ${{ runner.temp }}/opencode-e2e/config
-          XDG_STATE_HOME: ${{ runner.temp }}/opencode-e2e/state
+          OPENCODE_TEST_HOME: ${{ env.OPENCODE_TEST_HOME }}
+          XDG_DATA_HOME: ${{ env.XDG_DATA_HOME }}
+          XDG_CACHE_HOME: ${{ env.XDG_CACHE_HOME }}
+          XDG_CONFIG_HOME: ${{ env.XDG_CONFIG_HOME }}
+          XDG_STATE_HOME: ${{ env.XDG_STATE_HOME }}
           PLAYWRIGHT_SERVER_HOST: "localhost"
           PLAYWRIGHT_SERVER_PORT: "4096"
           VITE_OPENCODE_SERVER_HOST: "localhost"

packages/app/README.md

@@ -31,18 +31,20 @@ Your app is ready to be deployed!
 
 ## E2E Testing
 
-The Playwright runner expects the app already running at `http://localhost:3000`.
+Playwright starts the Vite dev server automatically via `webServer`, and UI tests need an opencode backend (defaults to `localhost:4096`).
+Use the local runner to create a temp sandbox, seed data, and run the tests.
 
 ```bash
-bun add -D @playwright/test
 bunx playwright install
-bun run test:e2e
+bun run test:e2e:local
+bun run test:e2e:local -- --grep "settings"
 ```
 
 Environment options:
 
-- `PLAYWRIGHT_BASE_URL` (default: `http://localhost:3000`)
-- `PLAYWRIGHT_PORT` (default: `3000`)
+- `PLAYWRIGHT_SERVER_HOST` / `PLAYWRIGHT_SERVER_PORT` (backend address, default: `localhost:4096`)
+- `PLAYWRIGHT_PORT` (Vite dev server port, default: `3000`)
+- `PLAYWRIGHT_BASE_URL` (override base URL, default: `http://localhost:<PLAYWRIGHT_PORT>`)
 
 ## Deployment
 

packages/app/e2e/prompt.spec.ts

@@ -0,0 +1,62 @@
+import { test, expect } from "./fixtures"
+import { promptSelector } from "./utils"
+
+function sessionIDFromUrl(url: string) {
+  const match = /\/session\/([^/?#]+)/.exec(url)
+  return match?.[1]
+}
+
+test("can send a prompt and receive a reply", async ({ page, sdk, gotoSession }) => {
+  test.setTimeout(120_000)
+
+  const pageErrors: string[] = []
+  const onPageError = (err: Error) => {
+    pageErrors.push(err.message)
+  }
+  page.on("pageerror", onPageError)
+
+  await gotoSession()
+
+  const token = `E2E_OK_${Date.now()}`
+
+  const prompt = page.locator(promptSelector)
+  await prompt.click()
+  await page.keyboard.type(`Reply with exactly: ${token}`)
+  await page.keyboard.press("Enter")
+
+  await expect(page).toHaveURL(/\/session\/[^/?#]+/, { timeout: 30_000 })
+
+  const sessionID = (() => {
+    const id = sessionIDFromUrl(page.url())
+    if (!id) throw new Error(`Failed to parse session id from url: ${page.url()}`)
+    return id
+  })()
+
+  try {
+    await expect
+      .poll(
+        async () => {
+          const messages = await sdk.session.messages({ sessionID, limit: 50 }).then((r) => r.data ?? [])
+          return messages
+            .filter((m) => m.info.role === "assistant")
+            .flatMap((m) => m.parts)
+            .filter((p) => p.type === "text")
+            .map((p) => p.text)
+            .join("\n")
+        },
+        { timeout: 90_000 },
+      )
+
+      .toContain(token)
+
+    const reply = page.locator('[data-slot="session-turn-summary-section"]').filter({ hasText: token }).first()
+    await expect(reply).toBeVisible({ timeout: 90_000 })
+  } finally {
+    page.off("pageerror", onPageError)
+    await sdk.session.delete({ sessionID }).catch(() => undefined)
+  }
+
+  if (pageErrors.length > 0) {
+    throw new Error(`Page error(s):\n${pageErrors.join("\n")}`)
+  }
+})

packages/app/package.json

@@ -15,6 +15,7 @@
     "serve": "vite preview",
     "test": "playwright test",
     "test:e2e": "playwright test",
+    "test:e2e:local": "bun script/e2e-local.ts",
     "test:e2e:ui": "playwright test --ui",
     "test:e2e:report": "playwright show-report e2e/playwright-report"
   },

packages/app/script/e2e-local.ts

@@ -0,0 +1,130 @@
+import fs from "node:fs/promises"
+import net from "node:net"
+import os from "node:os"
+import path from "node:path"
+
+async function freePort() {
+  return await new Promise<number>((resolve, reject) => {
+    const server = net.createServer()
+    server.once("error", reject)
+    server.listen(0, () => {
+      const address = server.address()
+      if (!address || typeof address === "string") {
+        server.close(() => reject(new Error("Failed to acquire a free port")))
+        return
+      }
+      server.close((err) => {
+        if (err) {
+          reject(err)
+          return
+        }
+        resolve(address.port)
+      })
+    })
+  })
+}
+
+async function waitForHealth(url: string) {
+  const timeout = Date.now() + 60_000
+  while (Date.now() < timeout) {
+    const ok = await fetch(url)
+      .then((r) => r.ok)
+      .catch(() => false)
+    if (ok) return
+    await new Promise((r) => setTimeout(r, 250))
+  }
+  throw new Error(`Timed out waiting for server health: ${url}`)
+}
+
+const appDir = process.cwd()
+const repoDir = path.resolve(appDir, "../..")
+const opencodeDir = path.join(repoDir, "packages", "opencode")
+const modelsJson = path.join(opencodeDir, "test", "tool", "fixtures", "models-api.json")
+
+const extraArgs = (() => {
+  const args = process.argv.slice(2)
+  if (args[0] === "--") return args.slice(1)
+  return args
+})()
+
+const [serverPort, webPort] = await Promise.all([freePort(), freePort()])
+
+const sandbox = await fs.mkdtemp(path.join(os.tmpdir(), "opencode-e2e-"))
+
+const serverEnv = {
+  ...process.env,
+  MODELS_DEV_API_JSON: modelsJson,
+  OPENCODE_DISABLE_MODELS_FETCH: "true",
+  OPENCODE_DISABLE_SHARE: "true",
+  OPENCODE_DISABLE_LSP_DOWNLOAD: "true",
+  OPENCODE_DISABLE_DEFAULT_PLUGINS: "true",
+  OPENCODE_EXPERIMENTAL_DISABLE_FILEWATCHER: "true",
+  OPENCODE_TEST_HOME: path.join(sandbox, "home"),
+  XDG_DATA_HOME: path.join(sandbox, "share"),
+  XDG_CACHE_HOME: path.join(sandbox, "cache"),
+  XDG_CONFIG_HOME: path.join(sandbox, "config"),
+  XDG_STATE_HOME: path.join(sandbox, "state"),
+  OPENCODE_E2E_PROJECT_DIR: repoDir,
+  OPENCODE_E2E_SESSION_TITLE: "E2E Session",
+  OPENCODE_E2E_MESSAGE: "Seeded for UI e2e",
+  OPENCODE_E2E_MODEL: "opencode/gpt-5-nano",
+  OPENCODE_CLIENT: "app",
+} satisfies Record<string, string>
+
+const runnerEnv = {
+  ...process.env,
+  PLAYWRIGHT_SERVER_HOST: "localhost",
+  PLAYWRIGHT_SERVER_PORT: String(serverPort),
+  VITE_OPENCODE_SERVER_HOST: "localhost",
+  VITE_OPENCODE_SERVER_PORT: String(serverPort),
+  PLAYWRIGHT_PORT: String(webPort),
+} satisfies Record<string, string>
+
+const seed = Bun.spawn(["bun", "script/seed-e2e.ts"], {
+  cwd: opencodeDir,
+  env: serverEnv,
+  stdout: "inherit",
+  stderr: "inherit",
+})
+
+const seedExit = await seed.exited
+if (seedExit !== 0) {
+  process.exit(seedExit)
+}
+
+const server = Bun.spawn(
+  [
+    "bun",
+    "dev",
+    "--",
+    "--print-logs",
+    "--log-level",
+    "WARN",
+    "serve",
+    "--port",
+    String(serverPort),
+    "--hostname",
+    "127.0.0.1",
+  ],
+  {
+    cwd: opencodeDir,
+    env: serverEnv,
+    stdout: "inherit",
+    stderr: "inherit",
+  },
+)
+
+try {
+  await waitForHealth(`http://localhost:${serverPort}/global/health`)
+
+  const runner = Bun.spawn(["bun", "test:e2e", ...extraArgs], {
+    cwd: appDir,
+    env: runnerEnv,
+    stdout: "inherit",
+    stderr: "inherit",
+  })
+
+  process.exitCode = await runner.exited
+} finally {
+  server.kill()
+}