feat(app): combine browser and artifacts views and state

.devcontainer/start-daytona-electron.sh

@@ -48,5 +48,6 @@ export ELECTRON_EXTRA_LAUNCH_ARGS="${ELECTRON_EXTRA_LAUNCH_ARGS:-$DAYTONA_ELECTR
 export OPENWORK_REACT_DEVTOOLS="${OPENWORK_REACT_DEVTOOLS:-0}"
 export OPENWORK_DEV_MODE="${OPENWORK_DEV_MODE:-1}"
 export OPENWORK_ELECTRON_REMOTE_DEBUG_PORT="${OPENWORK_ELECTRON_REMOTE_DEBUG_PORT:-9825}"
+export OPENWORK_ELECTRON_FAKE_MEDIA="${OPENWORK_ELECTRON_FAKE_MEDIA:-0}"
 
 exec pnpm --filter @openwork/desktop dev:electron

.devcontainer/test-on-daytona.sh

@@ -101,6 +101,7 @@ DAYTONA_SECRETS_ENV="${DAYTONA_SECRETS_ENV:-${DAYTONA_SECRETS_MOUNT}/openai.env}
 DAYTONA_ARTIFACTS_VOLUME="${DAYTONA_ARTIFACTS_VOLUME:-openwork-eval-artifacts}"
 DAYTONA_ARTIFACTS_MOUNT="${DAYTONA_ARTIFACTS_MOUNT:-/daytona-artifacts}"
 DAYTONA_ELECTRON_EXTRA_LAUNCH_ARGS="${DAYTONA_ELECTRON_EXTRA_LAUNCH_ARGS:---disable-gpu --disable-dev-shm-usage --enable-unsafe-swiftshader}"
+OPENWORK_ELECTRON_FAKE_MEDIA="${OPENWORK_ELECTRON_FAKE_MEDIA:-0}"
 
 if [ -z "$RECORDING_NAME" ]; then
   RECORDING_NAME="$SANDBOX"
@@ -220,7 +221,7 @@ daytona exec "$SANDBOX" -- "bash -lc 'set -euo pipefail; cd /workspace; REF=\"$R
 
 echo ""
 echo "==> Starting OpenWork sandbox dev stack..."
-daytona exec "$SANDBOX" -- "bash -lc 'set -euo pipefail; cd /workspace; DEN_BASE_URL=\"$DEN_BASE_URL\"; DEN_API_BASE_URL=\"$DEN_API_BASE_URL\"; DEN_REQUIRE_SIGNIN=\"$DEN_REQUIRE_SIGNIN\"; if [ -n \"\$DEN_BASE_URL\" ] || [ -n \"\$DEN_API_BASE_URL\" ] || [ \"\$DEN_REQUIRE_SIGNIN\" = 1 ]; then mkdir -p /workspace/.openwork-daytona; DEN_BASE_URL=\"\$DEN_BASE_URL\" DEN_API_BASE_URL=\"\$DEN_API_BASE_URL\" DEN_REQUIRE_SIGNIN=\"\$DEN_REQUIRE_SIGNIN\" node -e '\''const fs = require(\"node:fs\"); const baseUrl = process.env.DEN_BASE_URL || \"https://app.openworklabs.com\"; const apiBaseUrl = process.env.DEN_API_BASE_URL || null; const requireSignin = process.env.DEN_REQUIRE_SIGNIN === \"1\"; fs.writeFileSync(\"/workspace/.openwork-daytona/desktop-bootstrap.json\", JSON.stringify({ baseUrl, apiBaseUrl, requireSignin }, null, 2) + \"\\n\");'\''; fi; export DAYTONA_SECRETS_ENV=\"$DAYTONA_SECRETS_ENV\" DAYTONA_ELECTRON_EXTRA_LAUNCH_ARGS=\"$DAYTONA_ELECTRON_EXTRA_LAUNCH_ARGS\" OPENWORK_ELECTRON_REMOTE_DEBUG_PORT=$CDP_PORT OPENWORK_WORKSPACE_DIR=/workspace OPENWORK_GOOGLE_WORKSPACE_ALLOW_PLAINTEXT_VAULT=1; if [ -f /workspace/.openwork-daytona/desktop-bootstrap.json ]; then export OPENWORK_DESKTOP_BOOTSTRAP_PATH=/workspace/.openwork-daytona/desktop-bootstrap.json; fi; pnpm dev:sandbox'"
+daytona exec "$SANDBOX" -- "bash -lc 'set -euo pipefail; cd /workspace; DEN_BASE_URL=\"$DEN_BASE_URL\"; DEN_API_BASE_URL=\"$DEN_API_BASE_URL\"; DEN_REQUIRE_SIGNIN=\"$DEN_REQUIRE_SIGNIN\"; if [ -n \"\$DEN_BASE_URL\" ] || [ -n \"\$DEN_API_BASE_URL\" ] || [ \"\$DEN_REQUIRE_SIGNIN\" = 1 ]; then mkdir -p /workspace/.openwork-daytona; DEN_BASE_URL=\"\$DEN_BASE_URL\" DEN_API_BASE_URL=\"\$DEN_API_BASE_URL\" DEN_REQUIRE_SIGNIN=\"\$DEN_REQUIRE_SIGNIN\" node -e '\''const fs = require(\"node:fs\"); const baseUrl = process.env.DEN_BASE_URL || \"https://app.openworklabs.com\"; const apiBaseUrl = process.env.DEN_API_BASE_URL || null; const requireSignin = process.env.DEN_REQUIRE_SIGNIN === \"1\"; fs.writeFileSync(\"/workspace/.openwork-daytona/desktop-bootstrap.json\", JSON.stringify({ baseUrl, apiBaseUrl, requireSignin }, null, 2) + \"\\n\");'\''; fi; export DAYTONA_SECRETS_ENV=\"$DAYTONA_SECRETS_ENV\" DAYTONA_ELECTRON_EXTRA_LAUNCH_ARGS=\"$DAYTONA_ELECTRON_EXTRA_LAUNCH_ARGS\" OPENWORK_ELECTRON_REMOTE_DEBUG_PORT=$CDP_PORT OPENWORK_ELECTRON_FAKE_MEDIA=\"$OPENWORK_ELECTRON_FAKE_MEDIA\" OPENWORK_WORKSPACE_DIR=/workspace OPENWORK_GOOGLE_WORKSPACE_ALLOW_PLAINTEXT_VAULT=1; if [ -f /workspace/.openwork-daytona/desktop-bootstrap.json ]; then export OPENWORK_DESKTOP_BOOTSTRAP_PATH=/workspace/.openwork-daytona/desktop-bootstrap.json; fi; pnpm dev:sandbox'"
 
 echo ""
 echo "==> Waiting for Electron CDP on port $CDP_PORT (up to ${MAX_WAIT}s)..."

STATS.md

@@ -129,3 +129,4 @@ Legacy cumulative release-asset totals. For classified v2 buckets, see `STATS_V2
 | 2026-05-24 | 927,485 (+3,846) | 927,485 (+3,846) |
 | 2026-05-25 | 932,042 (+4,557) | 932,042 (+4,557) |
 | 2026-05-26 | 936,592 (+4,550) | 936,592 (+4,550) |
+| 2026-05-27 | 940,646 (+4,054) | 940,646 (+4,054) |

STATS_V2.md

@@ -85,3 +85,4 @@ Classified GitHub release asset snapshots. `Manual installs` counts installer do
 | 2026-05-24 | 115,034 (+680) | 690,031 (+38) | 122,420 (+3,128) | 927,485 (+3,846) |
 | 2026-05-25 | 115,841 (+807) | 690,083 (+52) | 126,118 (+3,698) | 932,042 (+4,557) |
 | 2026-05-26 | 116,536 (+695) | 690,118 (+35) | 129,938 (+3,820) | 936,592 (+4,550) |
+| 2026-05-27 | 116,990 (+454) | 690,148 (+30) | 133,508 (+3,570) | 940,646 (+4,054) |

apps/app/package.json

@@ -21,6 +21,7 @@
     "test:events": "node scripts/events.mjs",
     "test:todos": "node scripts/todos.mjs",
     "test:permissions": "node scripts/permissions.mjs",
+    "test:voice-cdp": "node scripts/voice-cdp.mjs",
     "test:remote-diagnostics": "bun test scripts/remote-workspace-diagnostics.test.ts",
     "test:open-target": "bun test scripts/open-target.test.ts",
     "test:artifact-spreadsheet": "bun test scripts/artifact-spreadsheet.test.ts",

apps/app/scripts/open-target.test.ts

@@ -2,11 +2,9 @@ import { describe, expect, it } from "bun:test";
 import type { UIMessage } from "ai";
 
 import {
-  classifyOpenTarget,
   deriveOpenTargets,
   isCollectibleArtifactTarget,
   selectAutoOpenTarget,
-  shouldAutoOpenTarget,
 } from "../src/react-app/domains/session/artifacts/open-target";
 
 function message(id: string, role: "user" | "assistant", text: string): UIMessage {
@@ -28,17 +26,6 @@ function toolMessage(id: string, toolName: string, input: Record<string, unknown
   };
 }
 
-describe("open target classification", () => {
-  it("routes common artifact formats to deterministic previews", () => {
-    expect(classifyOpenTarget("report.md", "file")).toBe("markdown");
-    expect(classifyOpenTarget("customers.csv", "file")).toBe("sheet");
-    expect(classifyOpenTarget("forecast.xlsx", "file")).toBe("sheet");
-    expect(classifyOpenTarget("diagram.svg", "file")).toBe("image");
-    expect(classifyOpenTarget("dist/index.html", "file")).toBe("html");
-    expect(classifyOpenTarget("http://localhost:5173", "url")).toBe("browser");
-  });
-});
-
 describe("deriveOpenTargets", () => {
   it("extracts file and localhost URL targets from recent assistant output", () => {
     const targets = deriveOpenTargets([
@@ -198,12 +185,8 @@ describe("deriveOpenTargets", () => {
     const targets = deriveOpenTargets([
       toolMessage("msg_tool", "write", { filePath: "data/customers.csv" }, { filePath: "data/customers.csv" }),
       message("msg_1", "assistant", "Created data/customers.csv and see https://example.com for docs."),
-    ]);
-    const csv = targets.find((target) => target.value === "data/customers.csv");
-    const externalUrl = targets.find((target) => target.value === "https://example.com");
+    ]).map((target) => ({ ...target, exists: target.kind === "file" }));
 
-    expect(csv && shouldAutoOpenTarget({ ...csv, exists: true })).toBe(false);
-    expect(csv && shouldAutoOpenTarget({ ...csv, exists: false })).toBe(false);
-    expect(externalUrl && shouldAutoOpenTarget(externalUrl)).toBe(false);
+    expect(selectAutoOpenTarget(targets)).toBeNull();
   });
 });

apps/app/scripts/voice-cdp.mjs

@@ -0,0 +1,274 @@
+import { execFile } from "node:child_process";
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { promisify } from "node:util";
+
+const execFileAsync = promisify(execFile);
+
+const args = parseArgs(process.argv.slice(2));
+const cdpUrl = args.cdpUrl ?? process.env.CDP_URL ?? "http://127.0.0.1:9825";
+const mode = args.mode ?? "preflight";
+const text = args.text ?? "Open extension settings.";
+const expectRoute = args.expectRoute ?? "";
+const requireAudioPermission = args.requireAudioPermission === true;
+
+async function main() {
+  const target = await pickTarget(cdpUrl);
+  const client = await connectCdp(target.webSocketDebuggerUrl);
+
+  try {
+    await waitFor(client, "Boolean(window.__openworkControl)", 15000);
+    const preflight = await runPreflight(client);
+    if (mode === "preflight") {
+      console.log(JSON.stringify(preflight, null, 2));
+      return;
+    }
+
+    await ensureVoicePanel(client);
+
+    if (mode === "transcript" || mode === "full") {
+      const transcript = await executeControl(client, "voice.inject_transcript", { text });
+      console.log(JSON.stringify({ step: "transcript", result: transcript }, null, 2));
+    }
+
+    if (mode === "audio" || mode === "full") {
+      const pcm16Base64 = await synthesizePcm16Base64(text);
+      const audio = await executeControl(client, "voice.inject_audio", { pcm16Base64 });
+      console.log(JSON.stringify({ step: "audio", result: audio }, null, 2));
+      const proof = await collectProof(client, expectRoute);
+      console.log(JSON.stringify({ step: "proof", result: proof }, null, 2));
+    }
+  } finally {
+    client.close();
+  }
+}
+
+function parseArgs(values) {
+  const parsed = {};
+  for (let index = 0; index < values.length; index += 1) {
+    const value = values[index];
+    if (value === "--mode") parsed.mode = values[++index];
+    else if (value === "--text") parsed.text = values[++index];
+    else if (value === "--cdp-url") parsed.cdpUrl = values[++index];
+    else if (value === "--expect-route") parsed.expectRoute = values[++index];
+    else if (value === "--require-audio-permission") parsed.requireAudioPermission = true;
+  }
+  return parsed;
+}
+
+async function runPreflight(client) {
+  const userAgent = await evaluate(client, "navigator.userAgent");
+  const controlReady = await evaluate(client, "Boolean(window.__openworkControl)");
+  const actions = controlReady
+    ? await evaluate(client, "window.__openworkControl.listActions().map((action) => action.id)")
+    : [];
+  const media = await evaluate(client, `(${mediaPreflight.toString()})()`, true);
+
+  const result = {
+    ok: true,
+    electron: typeof userAgent === "string" && userAgent.includes("Electron/"),
+    userAgent,
+    controlReady,
+    voiceActions: actions.filter((id) => id.startsWith("voice.")),
+    media,
+  };
+
+  if (!result.electron) throw new Error("Target is not Electron.");
+  if (!controlReady) throw new Error("OpenWork control API is not available.");
+  if (requireAudioPermission && !media.audio.ok) {
+    throw new Error(`Audio getUserMedia failed: ${media.audio.name} ${media.audio.message}`);
+  }
+  if (media.video.ok) throw new Error("Video getUserMedia unexpectedly succeeded; audio-only permission guard may be broken.");
+  return result;
+}
+
+async function mediaPreflight() {
+  async function request(constraints) {
+    try {
+      const stream = await navigator.mediaDevices.getUserMedia(constraints);
+      stream.getTracks().forEach((track) => track.stop());
+      return { ok: true };
+    } catch (error) {
+      return { ok: false, name: error?.name ?? "Error", message: error?.message ?? String(error) };
+    }
+  }
+  return {
+    audio: await request({ audio: true }),
+    video: await request({ video: true }),
+  };
+}
+
+async function ensureVoicePanel(client) {
+  await evaluate(client, "window.__openworkControl.setEnabled(true)");
+  await evaluate(client, "window.localStorage.setItem('openwork.extension.enabled.openwork-voice', '1'); window.dispatchEvent(new CustomEvent('openwork:extension-state-changed', { detail: { id: 'openwork-voice', enabled: true } }))");
+  let actions = await evaluate(client, "window.__openworkControl.listActions().map((action) => action.id)");
+  if (actions.includes("voice.inject_audio")) return;
+  if (actions.includes("voice.panel.open")) {
+    await executeControl(client, "voice.panel.open");
+    await waitFor(client, "window.__openworkControl.listActions().some((action) => action.id === 'voice.inject_audio')", 8000);
+    return;
+  }
+  throw new Error(`Voice panel actions are not registered. Open a session and enable Voice Mode first. Voice actions: ${actions.filter((id) => id.startsWith("voice.")).join(", ")}`);
+}
+
+async function collectProof(client, expectedRoute) {
+  const started = Date.now();
+  let proof = null;
+  while (true) {
+    proof = await readProof(client);
+    const routeMatched = expectedRoute && proof.href.includes(expectedRoute);
+    const acted = proof.narration.includes("Done:") || proof.narration.includes("Running");
+    if (routeMatched || (!expectedRoute && acted)) return { ...proof, elapsedMs: Date.now() - started };
+    if (Date.now() - started >= 60000) break;
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+  }
+  return { ...proof, elapsedMs: Date.now() - started, timedOut: true };
+}
+
+async function readProof(client) {
+  return evaluate(client, `({
+    href: location.href,
+    narration: window.__openworkControl?.snapshot?.().narration ?? "",
+    route: window.__openworkControl?.snapshot?.().route ?? "",
+    bodyText: document.body.innerText.slice(-2400),
+  })`);
+}
+
+async function executeControl(client, actionId, actionArgs = undefined) {
+  const expression = `window.__openworkControl.execute(${JSON.stringify(actionId)}, ${JSON.stringify(actionArgs)})`;
+  const result = await evaluate(client, expression, true);
+  if (!result?.ok) throw new Error(`Control action failed: ${actionId}: ${result?.error ?? "unknown error"}`);
+  return result;
+}
+
+async function synthesizePcm16Base64(input) {
+  const ffmpeg = await requireCommand("ffmpeg", "ffmpeg is required for generated voice audio. On Daytona/Linux install it with: apt-get update && apt-get install -y ffmpeg espeak-ng");
+  const tts = await findTtsCommand();
+  const dir = await mkdtemp(join(tmpdir(), "openwork-voice-cdp-"));
+  const source = join(dir, "speech.wav");
+  const pcm = join(dir, "speech.pcm");
+
+  try {
+    if (tts.kind === "say") {
+      const aiff = join(dir, "speech.aiff");
+      await execFileAsync(tts.command, ["-v", "Samantha", "-o", aiff, input]);
+      await execFileAsync(ffmpeg, ["-y", "-i", aiff, "-ac", "1", "-ar", "24000", "-f", "s16le", pcm]);
+    } else {
+      await execFileAsync(tts.command, ["-w", source, input]);
+      await execFileAsync(ffmpeg, ["-y", "-i", source, "-ac", "1", "-ar", "24000", "-f", "s16le", pcm]);
+    }
+    return (await readFile(pcm)).toString("base64");
+  } finally {
+    await rm(dir, { force: true, recursive: true });
+  }
+}
+
+async function findTtsCommand() {
+  const say = await commandPath("say");
+  if (say) return { kind: "say", command: say };
+  const espeakNg = await commandPath("espeak-ng");
+  if (espeakNg) return { kind: "espeak", command: espeakNg };
+  const espeak = await commandPath("espeak");
+  if (espeak) return { kind: "espeak", command: espeak };
+  throw new Error("No TTS command found. On Daytona/Linux install one with: apt-get update && apt-get install -y espeak-ng ffmpeg");
+}
+
+async function requireCommand(command, message) {
+  const found = await commandPath(command);
+  if (!found) throw new Error(message);
+  return found;
+}
+
+async function commandPath(command) {
+  try {
+    const { stdout } = await execFileAsync("which", [command]);
+    return stdout.trim() || null;
+  } catch {
+    return null;
+  }
+}
+
+async function pickTarget(baseUrl) {
+  const response = await fetch(`${baseUrl.replace(/\/$/, "")}/json/list`);
+  if (!response.ok) throw new Error(`Could not list CDP targets: ${response.status}`);
+  const targets = await response.json();
+  const pages = targets.filter((target) => target.type === "page" && target.webSocketDebuggerUrl);
+  const target = pages.find((page) => page.title === "OpenWork") ??
+    pages.find((page) => page.url.includes("localhost") || page.url.includes("127.0.0.1") || page.url.includes("[::1]")) ??
+    pages[0];
+  if (!target) throw new Error("No CDP page target found.");
+  return target;
+}
+
+function connectCdp(webSocketDebuggerUrl) {
+  return new Promise((resolve, reject) => {
+    const socket = new WebSocket(webSocketDebuggerUrl);
+    let nextId = 1;
+    const pending = new Map();
+    let opened = false;
+
+    const rejectPending = (error) => {
+      for (const callbacks of pending.values()) callbacks.reject(error);
+      pending.clear();
+    };
+
+    socket.addEventListener("open", () => {
+      opened = true;
+      resolve({
+        close: () => socket.close(),
+        send(method, params = {}) {
+          const id = nextId++;
+          return new Promise((innerResolve, innerReject) => {
+            pending.set(id, { resolve: innerResolve, reject: innerReject });
+            try {
+              socket.send(JSON.stringify({ id, method, params }));
+            } catch (error) {
+              pending.delete(id);
+              innerReject(error);
+            }
+          });
+        },
+      });
+    });
+    socket.addEventListener("message", (event) => {
+      const message = JSON.parse(String(event.data));
+      if (!message.id) return;
+      const callbacks = pending.get(message.id);
+      if (!callbacks) return;
+      pending.delete(message.id);
+      if (message.error) callbacks.reject(new Error(message.error.message));
+      else callbacks.resolve(message.result);
+    });
+    socket.addEventListener("error", () => {
+      const error = new Error("CDP websocket failed.");
+      rejectPending(error);
+      if (!opened) reject(error);
+    });
+    socket.addEventListener("close", () => {
+      const error = new Error("CDP websocket closed.");
+      rejectPending(error);
+      if (!opened) reject(error);
+    });
+  });
+}
+
+async function evaluate(client, expression, awaitPromise = false) {
+  const result = await client.send("Runtime.evaluate", { expression, awaitPromise, returnByValue: true });
+  if (result.exceptionDetails) throw new Error(result.exceptionDetails.exception?.description ?? result.exceptionDetails.text ?? "Evaluation failed.");
+  return result.result?.value;
+}
+
+async function waitFor(client, expression, timeoutMs) {
+  const started = Date.now();
+  while (Date.now() - started < timeoutMs) {
+    if (await evaluate(client, expression)) return;
+    await new Promise((resolve) => setTimeout(resolve, 150));
+  }
+  throw new Error(`Timed out waiting for ${expression}`);
+}
+
+main().catch((error) => {
+  console.error(error);
+  process.exit(1);
+});