feat(panel): LLM feedback panel that drives the app and reports UX gaps

CLAUDE.md

@@ -126,6 +126,31 @@ Features live under `src/features/<feature>/`; tests are colocated. File convent
 - **Safe-area insets** are hardened in `variables.css` (`viewport-fit=cover` in index.html) so the
   toolbar clears the notch and the tab bar clears the home bar.
 
+### LLM feedback panel (`npm run panel`)
+
+A panel of **different vendors' vision LLMs** (Anthropic Claude, Amazon Nova, Meta Llama 4, Mistral
+Pixtral — all via **Bedrock Converse**, `AWS_PROFILE=personal`, us-west-2) each drive a **real
+Playwright browser** (screenshot → choose one action → execute, agentic loop) to organize a project
+for a scenario, then answer a structured feedback questionnaire. The output is a synthesized
+`report.md` (delight/clarity scores + improvement themes ranked by how many panelists raised them) —
+a cheap, reproducible way to get outside-eyes UX feedback to act on. Lives in `scripts/llm-panel/`
+(exempt from the line/CRAP gate — it's a `scripts/` harness, not `src`/`amplify`).
+
+```bash
+npm run dev -- --port 5173                 # (or point at a deployed URL)
+AWS_PROFILE=personal npm run panel         # default: wedding scenario, all 4 panelists
+PANEL_SCENARIO=trip PANEL_ONLY=claude,nova npm run panel   # subset + other scenario
+PANEL_BASE=https://taskflow.example npm run panel          # target a live deployment
+```
+
+Env: `TEST_USERNAME`/`TEST_PASSWORD` (from `.env.local`) to sign in; `AWS_PROFILE=personal` for
+Bedrock. Scenarios (`wedding` default, `launch`, `trip`) + the panel roster live in
+`scripts/llm-panel/scenarios.mjs`. Output (transcripts, per-step screenshots, `report.md`) →
+`/tmp/tf-panel/<PANEL_RUN_ID>/`. Models that need an inference profile use the region/`global.`-prefixed
+id (e.g. `us.meta.llama4-…`, `global.anthropic.claude-haiku-…`) — on-demand model ids throw
+`ValidationException`. **Treat low-delight findings as a backlog: the panel is the "measure vs Asana"
+loop automated.**
+
 ## Quality gates (non-negotiable — CI + husky pre-commit enforce them)
 
 Run `npm run quality` for the full set. **Enforce them yourself; when one fails, fix the code, never

package.json

@@ -11,6 +11,7 @@
     "prod-config": "node scripts/prod-config.mjs",
     "e2e-config": "node scripts/e2e-config.mjs",
     "gen:icons": "node scripts/gen-app-icons.mjs",
+    "panel": "node scripts/llm-panel/run.mjs",
     "seed": "tsx amplify/seed/seed.ts",
     "test": "vitest run",
     "test:watch": "vitest",

scripts/llm-panel/agent.mjs

@@ -0,0 +1,119 @@
+// The agentic loop: give a Bedrock vision model a screenshot + a compact list of
+// the interactive elements on the page, let it choose ONE action per turn, run
+// it with Playwright, and repeat until it says done or hits the step cap.
+//
+// Actions are DOM-target based (by visible text / label / testid), not pixel
+// coordinates — far more reliable across models than coordinate clicking.
+import { converse } from './bedrock.mjs';
+
+const ACTIONS = `You control a real web browser to accomplish a goal on a task-management web app.
+Each turn you receive a screenshot and a numbered list of interactive elements.
+Respond with ONE action as a single line of JSON (no prose, no code fences):
+  {"action":"click","target":<number>}            - click element by its number
+  {"action":"type","target":<number>,"text":"..."} - focus element, type text
+  {"action":"enter","target":<number>}             - focus element, press Enter (submit)
+  {"action":"goto","path":"/my-tasks"}             - navigate to an in-app path
+  {"action":"note","text":"..."}                   - record an observation, no browser change
+  {"action":"done","text":"..."}                   - finished; text summarizes what you did
+Prefer clicking visible buttons/links. Work toward the goal step by step.`;
+
+/** Snapshot the page's interactive elements into a numbered, model-readable list
+ * and a parallel array of Playwright handles. Kept small so it fits the prompt. */
+async function inventory(page) {
+  const handles = await page.$$(
+    'button, a, input, textarea, select, [role="button"], [data-testid]',
+  );
+  const items = [];
+  const kept = [];
+  for (const h of handles) {
+    if (kept.length >= 60) break;
+    const visible = await h.isVisible().catch(() => false);
+    if (!visible) continue;
+    const info = await h
+      .evaluate((el) => {
+        const t = (
+          el.getAttribute('aria-label') ||
+          el.getAttribute('placeholder') ||
+          el.value ||
+          el.innerText ||
+          el.getAttribute('data-testid') ||
+          ''
+        )
+          .trim()
+          .slice(0, 60);
+        return { tag: el.tagName.toLowerCase(), t };
+      })
+      .catch(() => null);
+    if (!info || !info.t) continue;
+    kept.push(h);
+    items.push(`${kept.length - 1}: <${info.tag}> ${info.t}`);
+  }
+  return { list: items.join('\n'), handles: kept };
+}
+
+async function runAction(page, act, handles) {
+  const el = typeof act.target === 'number' ? handles[act.target] : null;
+  if (act.action === 'goto') return page.goto(act.path).then(() => page.waitForTimeout(1500));
+  if (!el && ['click', 'type', 'enter'].includes(act.action)) return;
+  if (act.action === 'click')
+    return el.click({ timeout: 5000 }).then(() => page.waitForTimeout(1200));
+  if (act.action === 'type') {
+    await el.click({ timeout: 5000 }).catch(() => {});
+    await el.fill('').catch(() => {});
+    await el.type(act.text ?? '', { delay: 20 });
+    return page.waitForTimeout(600);
+  }
+  if (act.action === 'enter') {
+    await el.click({ timeout: 5000 }).catch(() => {});
+    await el.type(act.text ?? '', { delay: 20 }).catch(() => {});
+    await page.keyboard.press('Enter');
+    return page.waitForTimeout(1200);
+  }
+}
+
+function parseAction(text) {
+  const m = text.match(/\{[\s\S]*\}/);
+  if (!m) return { action: 'note', text: text.slice(0, 120) };
+  try {
+    return JSON.parse(m[0]);
+  } catch {
+    return { action: 'note', text: text.slice(0, 120) };
+  }
+}
+
+/**
+ * Drive `page` toward `goal` with `modelId` for up to `maxSteps`. Calls
+ * onStep({ n, action, shotPath }) after each turn (for transcript/screenshots).
+ * Returns the running transcript of actions.
+ */
+export async function runAgent({ page, modelId, goal, maxSteps = 22, onStep }) {
+  const messages = [];
+  const transcript = [];
+  for (let n = 1; n <= maxSteps; n++) {
+    const shot = await page.screenshot();
+    const { list, handles } = await inventory(page);
+    const user =
+      n === 1
+        ? `GOAL: ${goal}\n\nInteractive elements:\n${list}\n\nChoose your first action.`
+        : `Interactive elements now:\n${list}\n\nChoose the next action toward the goal.`;
+    messages.push({ role: 'user', content: user });
+    let reply;
+    try {
+      reply = await converse({ modelId, system: ACTIONS, messages, image: shot });
+    } catch (e) {
+      transcript.push({ n, error: e.message });
+      break;
+    }
+    messages.push({ role: 'assistant', content: reply });
+    const act = parseAction(reply);
+    transcript.push({ n, action: act, raw: reply.slice(0, 200) });
+    if (onStep) await onStep({ n, action: act });
+    if (act.action === 'done') break;
+    try {
+      await runAction(page, act, handles);
+    } catch (e) {
+      messages.push({ role: 'user', content: `That action failed: ${e.message}. Try another.` });
+    }
+  }
+  return transcript;
+}

scripts/llm-panel/bedrock.mjs

@@ -0,0 +1,72 @@
+// Thin wrapper over the Bedrock Converse API (no SDK dep — signs requests with
+// SigV4 by shelling out to `aws bedrock-runtime converse`). Every panelist model
+// is reached through this one call, so the panel is genuinely multi-vendor
+// (Anthropic / Amazon / Meta / Mistral) with identical plumbing.
+//
+// AWS_PROFILE=personal + region us-west-2 are the project defaults; never inline
+// keys.
+import { spawn } from 'node:child_process';
+import { writeFileSync, mkdtempSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+const REGION = process.env.AWS_REGION || 'us-west-2';
+const PROFILE = process.env.AWS_PROFILE || 'personal';
+
+/** Run the aws CLI and resolve parsed JSON stdout (rejects on non-zero exit). */
+function awsJson(args, { input } = {}) {
+  return new Promise((resolve, reject) => {
+    const proc = spawn('aws', args, { env: { ...process.env, AWS_PROFILE: PROFILE } });
+    let out = '';
+    let err = '';
+    proc.stdout.on('data', (d) => (out += d));
+    proc.stderr.on('data', (d) => (err += d));
+    proc.on('close', (code) => {
+      if (code !== 0) return reject(new Error(`aws ${args[0]} ${args[1]} failed: ${err.trim()}`));
+      try {
+        resolve(JSON.parse(out));
+      } catch (e) {
+        reject(new Error(`bad JSON from aws: ${e.message}\n${out.slice(0, 400)}`));
+      }
+    });
+  });
+}
+
+/**
+ * One Converse turn. `messages` is the running Bedrock message array; `image` is
+ * an optional PNG buffer appended to the latest user turn so the model can see
+ * the current screen. Returns the assistant's text.
+ */
+export async function converse({ modelId, system, messages, image, maxTokens = 1200 }) {
+  const msgs = messages.map((m) => ({ role: m.role, content: [{ text: m.content }] }));
+  if (image && msgs.length) {
+    msgs[msgs.length - 1].content.push({
+      image: { format: 'png', source: { bytes: image.toString('base64') } },
+    });
+  }
+  const body = {
+    messages: msgs,
+    inferenceConfig: { maxTokens, temperature: 0.4 },
+    ...(system ? { system: [{ text: system }] } : {}),
+  };
+  // The payload (with a base64 screenshot) is far past ARG_MAX, so hand it to the
+  // CLI via a temp file rather than an inline argument.
+  const dir = mkdtempSync(join(tmpdir(), 'panel-'));
+  const bodyPath = join(dir, 'req.json');
+  writeFileSync(bodyPath, JSON.stringify(body));
+  const res = await awsJson([
+    'bedrock-runtime',
+    'converse',
+    '--region',
+    REGION,
+    '--model-id',
+    modelId,
+    '--cli-input-json',
+    `file://${bodyPath}`,
+  ]);
+  const parts = res.output?.message?.content ?? [];
+  return parts
+    .map((p) => p.text ?? '')
+    .join('')
+    .trim();
+}

scripts/llm-panel/report.mjs

@@ -0,0 +1,77 @@
+// Synthesize the panel's per-model feedback into one markdown report: a score
+// table, then each panelist's notes, then an aggregated "what to fix" list
+// ranked by how many panelists raised each theme.
+
+function avg(nums) {
+  const v = nums.filter((n) => typeof n === 'number');
+  return v.length ? (v.reduce((a, b) => a + b, 0) / v.length).toFixed(1) : '—';
+}
+
+export function renderReport(scenario, base, results) {
+  const ok = results.filter((r) => r.feedback && !r.feedback.parseError);
+  const lines = [];
+  lines.push(`# Taskflow LLM feedback panel — ${scenario.title}`);
+  lines.push('');
+  lines.push(`Target: \`${base}\` · Panelists: ${results.length}`);
+  lines.push('');
+
+  // Score table.
+  lines.push('## Scores');
+  lines.push('');
+  lines.push('| Panelist | Delight | Clarity | Steps | First impression |');
+  lines.push('|---|---|---|---|---|');
+  for (const r of results) {
+    const f = r.feedback ?? {};
+    lines.push(
+      `| ${r.label} | ${f.delight ?? '—'} | ${f.clarity ?? '—'} | ${r.transcript?.length ?? '—'} | ${(f.firstImpression ?? r.error ?? '—').replace(/\|/g, '/')} |`,
+    );
+  }
+  lines.push('');
+  lines.push(
+    `**Average delight ${avg(ok.map((r) => r.feedback.delight))}/10 · clarity ${avg(ok.map((r) => r.feedback.clarity))}/10**`,
+  );
+  lines.push('');
+
+  // Aggregate improvement themes (naive fuzzy grouping by first 4 words).
+  const themes = new Map();
+  for (const r of ok) {
+    for (const imp of r.feedback.top_improvements ?? []) {
+      const key = imp.toLowerCase().split(/\s+/).slice(0, 4).join(' ');
+      const e = themes.get(key) ?? { text: imp, votes: 0 };
+      e.votes += 1;
+      themes.set(key, e);
+    }
+  }
+  const ranked = [...themes.values()].sort((a, b) => b.votes - a.votes);
+  if (ranked.length) {
+    lines.push('## Top improvements (ranked by how many panelists raised it)');
+    lines.push('');
+    for (const t of ranked) lines.push(`- **(${t.votes}×)** ${t.text}`);
+    lines.push('');
+  }
+
+  // Per-panelist detail.
+  lines.push('## Per-panelist feedback');
+  lines.push('');
+  for (const r of results) {
+    lines.push(`### ${r.label}`);
+    if (r.error) {
+      lines.push(`> Run error: ${r.error}`);
+      lines.push('');
+      continue;
+    }
+    const f = r.feedback ?? {};
+    if (f.parseError) {
+      lines.push(`> Could not parse feedback JSON. Raw: ${f.raw}`);
+      lines.push('');
+      continue;
+    }
+    lines.push(`- **Delight ${f.delight}/10 · Clarity ${f.clarity}/10**`);
+    lines.push(`- vs Asana: ${f.vs_asana ?? '—'}`);
+    if (f.worked_well?.length) lines.push(`- 👍 ${f.worked_well.join('; ')}`);
+    if (f.confusing_or_bad?.length) lines.push(`- 👎 ${f.confusing_or_bad.join('; ')}`);
+    if (f.top_improvements?.length) lines.push(`- 🛠 ${f.top_improvements.join('; ')}`);
+    lines.push('');
+  }
+  return lines.join('\n');
+}