diff --git a/CLAUDE.md b/CLAUDE.md index ae1b787..64c4c4a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -126,6 +126,31 @@ Features live under `src/features//`; tests are colocated. File convent - **Safe-area insets** are hardened in `variables.css` (`viewport-fit=cover` in index.html) so the toolbar clears the notch and the tab bar clears the home bar. +### LLM feedback panel (`npm run panel`) + +A panel of **different vendors' vision LLMs** (Anthropic Claude, Amazon Nova, Meta Llama 4, Mistral +Pixtral — all via **Bedrock Converse**, `AWS_PROFILE=personal`, us-west-2) each drive a **real +Playwright browser** (screenshot → choose one action → execute, agentic loop) to organize a project +for a scenario, then answer a structured feedback questionnaire. The output is a synthesized +`report.md` (delight/clarity scores + improvement themes ranked by how many panelists raised them) — +a cheap, reproducible way to get outside-eyes UX feedback to act on. Lives in `scripts/llm-panel/` +(exempt from the line/CRAP gate — it's a `scripts/` harness, not `src`/`amplify`). + +```bash +npm run dev -- --port 5173 # (or point at a deployed URL) +AWS_PROFILE=personal npm run panel # default: wedding scenario, all 4 panelists +PANEL_SCENARIO=trip PANEL_ONLY=claude,nova npm run panel # subset + other scenario +PANEL_BASE=https://taskflow.example npm run panel # target a live deployment +``` + +Env: `TEST_USERNAME`/`TEST_PASSWORD` (from `.env.local`) to sign in; `AWS_PROFILE=personal` for +Bedrock. Scenarios (`wedding` default, `launch`, `trip`) + the panel roster live in +`scripts/llm-panel/scenarios.mjs`. Output (transcripts, per-step screenshots, `report.md`) → +`/tmp/tf-panel//`. Models that need an inference profile use the region/`global.`-prefixed +id (e.g. `us.meta.llama4-…`, `global.anthropic.claude-haiku-…`) — on-demand model ids throw +`ValidationException`. **Treat low-delight findings as a backlog: the panel is the "measure vs Asana" +loop automated.** + ## Quality gates (non-negotiable — CI + husky pre-commit enforce them) Run `npm run quality` for the full set. **Enforce them yourself; when one fails, fix the code, never diff --git a/package.json b/package.json index aa44c89..06ba576 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "prod-config": "node scripts/prod-config.mjs", "e2e-config": "node scripts/e2e-config.mjs", "gen:icons": "node scripts/gen-app-icons.mjs", + "panel": "node scripts/llm-panel/run.mjs", "seed": "tsx amplify/seed/seed.ts", "test": "vitest run", "test:watch": "vitest", diff --git a/scripts/llm-panel/agent.mjs b/scripts/llm-panel/agent.mjs new file mode 100644 index 0000000..1bad2b3 --- /dev/null +++ b/scripts/llm-panel/agent.mjs @@ -0,0 +1,119 @@ +// The agentic loop: give a Bedrock vision model a screenshot + a compact list of +// the interactive elements on the page, let it choose ONE action per turn, run +// it with Playwright, and repeat until it says done or hits the step cap. +// +// Actions are DOM-target based (by visible text / label / testid), not pixel +// coordinates — far more reliable across models than coordinate clicking. +import { converse } from './bedrock.mjs'; + +const ACTIONS = `You control a real web browser to accomplish a goal on a task-management web app. +Each turn you receive a screenshot and a numbered list of interactive elements. +Respond with ONE action as a single line of JSON (no prose, no code fences): + {"action":"click","target":} - click element by its number + {"action":"type","target":,"text":"..."} - focus element, type text + {"action":"enter","target":} - focus element, press Enter (submit) + {"action":"goto","path":"/my-tasks"} - navigate to an in-app path + {"action":"note","text":"..."} - record an observation, no browser change + {"action":"done","text":"..."} - finished; text summarizes what you did +Prefer clicking visible buttons/links. Work toward the goal step by step.`; + +/** Snapshot the page's interactive elements into a numbered, model-readable list + * and a parallel array of Playwright handles. Kept small so it fits the prompt. */ +async function inventory(page) { + const handles = await page.$$( + 'button, a, input, textarea, select, [role="button"], [data-testid]', + ); + const items = []; + const kept = []; + for (const h of handles) { + if (kept.length >= 60) break; + const visible = await h.isVisible().catch(() => false); + if (!visible) continue; + const info = await h + .evaluate((el) => { + const t = ( + el.getAttribute('aria-label') || + el.getAttribute('placeholder') || + el.value || + el.innerText || + el.getAttribute('data-testid') || + '' + ) + .trim() + .slice(0, 60); + return { tag: el.tagName.toLowerCase(), t }; + }) + .catch(() => null); + if (!info || !info.t) continue; + kept.push(h); + items.push(`${kept.length - 1}: <${info.tag}> ${info.t}`); + } + return { list: items.join('\n'), handles: kept }; +} + +async function runAction(page, act, handles) { + const el = typeof act.target === 'number' ? handles[act.target] : null; + if (act.action === 'goto') return page.goto(act.path).then(() => page.waitForTimeout(1500)); + if (!el && ['click', 'type', 'enter'].includes(act.action)) return; + if (act.action === 'click') + return el.click({ timeout: 5000 }).then(() => page.waitForTimeout(1200)); + if (act.action === 'type') { + await el.click({ timeout: 5000 }).catch(() => {}); + await el.fill('').catch(() => {}); + await el.type(act.text ?? '', { delay: 20 }); + return page.waitForTimeout(600); + } + if (act.action === 'enter') { + await el.click({ timeout: 5000 }).catch(() => {}); + await el.type(act.text ?? '', { delay: 20 }).catch(() => {}); + await page.keyboard.press('Enter'); + return page.waitForTimeout(1200); + } +} + +function parseAction(text) { + const m = text.match(/\{[\s\S]*\}/); + if (!m) return { action: 'note', text: text.slice(0, 120) }; + try { + return JSON.parse(m[0]); + } catch { + return { action: 'note', text: text.slice(0, 120) }; + } +} + +/** + * Drive `page` toward `goal` with `modelId` for up to `maxSteps`. Calls + * onStep({ n, action, shotPath }) after each turn (for transcript/screenshots). + * Returns the running transcript of actions. + */ +export async function runAgent({ page, modelId, goal, maxSteps = 22, onStep }) { + const messages = []; + const transcript = []; + for (let n = 1; n <= maxSteps; n++) { + const shot = await page.screenshot(); + const { list, handles } = await inventory(page); + const user = + n === 1 + ? `GOAL: ${goal}\n\nInteractive elements:\n${list}\n\nChoose your first action.` + : `Interactive elements now:\n${list}\n\nChoose the next action toward the goal.`; + messages.push({ role: 'user', content: user }); + let reply; + try { + reply = await converse({ modelId, system: ACTIONS, messages, image: shot }); + } catch (e) { + transcript.push({ n, error: e.message }); + break; + } + messages.push({ role: 'assistant', content: reply }); + const act = parseAction(reply); + transcript.push({ n, action: act, raw: reply.slice(0, 200) }); + if (onStep) await onStep({ n, action: act }); + if (act.action === 'done') break; + try { + await runAction(page, act, handles); + } catch (e) { + messages.push({ role: 'user', content: `That action failed: ${e.message}. Try another.` }); + } + } + return transcript; +} diff --git a/scripts/llm-panel/bedrock.mjs b/scripts/llm-panel/bedrock.mjs new file mode 100644 index 0000000..e758d1a --- /dev/null +++ b/scripts/llm-panel/bedrock.mjs @@ -0,0 +1,72 @@ +// Thin wrapper over the Bedrock Converse API (no SDK dep — signs requests with +// SigV4 by shelling out to `aws bedrock-runtime converse`). Every panelist model +// is reached through this one call, so the panel is genuinely multi-vendor +// (Anthropic / Amazon / Meta / Mistral) with identical plumbing. +// +// AWS_PROFILE=personal + region us-west-2 are the project defaults; never inline +// keys. +import { spawn } from 'node:child_process'; +import { writeFileSync, mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +const REGION = process.env.AWS_REGION || 'us-west-2'; +const PROFILE = process.env.AWS_PROFILE || 'personal'; + +/** Run the aws CLI and resolve parsed JSON stdout (rejects on non-zero exit). */ +function awsJson(args, { input } = {}) { + return new Promise((resolve, reject) => { + const proc = spawn('aws', args, { env: { ...process.env, AWS_PROFILE: PROFILE } }); + let out = ''; + let err = ''; + proc.stdout.on('data', (d) => (out += d)); + proc.stderr.on('data', (d) => (err += d)); + proc.on('close', (code) => { + if (code !== 0) return reject(new Error(`aws ${args[0]} ${args[1]} failed: ${err.trim()}`)); + try { + resolve(JSON.parse(out)); + } catch (e) { + reject(new Error(`bad JSON from aws: ${e.message}\n${out.slice(0, 400)}`)); + } + }); + }); +} + +/** + * One Converse turn. `messages` is the running Bedrock message array; `image` is + * an optional PNG buffer appended to the latest user turn so the model can see + * the current screen. Returns the assistant's text. + */ +export async function converse({ modelId, system, messages, image, maxTokens = 1200 }) { + const msgs = messages.map((m) => ({ role: m.role, content: [{ text: m.content }] })); + if (image && msgs.length) { + msgs[msgs.length - 1].content.push({ + image: { format: 'png', source: { bytes: image.toString('base64') } }, + }); + } + const body = { + messages: msgs, + inferenceConfig: { maxTokens, temperature: 0.4 }, + ...(system ? { system: [{ text: system }] } : {}), + }; + // The payload (with a base64 screenshot) is far past ARG_MAX, so hand it to the + // CLI via a temp file rather than an inline argument. + const dir = mkdtempSync(join(tmpdir(), 'panel-')); + const bodyPath = join(dir, 'req.json'); + writeFileSync(bodyPath, JSON.stringify(body)); + const res = await awsJson([ + 'bedrock-runtime', + 'converse', + '--region', + REGION, + '--model-id', + modelId, + '--cli-input-json', + `file://${bodyPath}`, + ]); + const parts = res.output?.message?.content ?? []; + return parts + .map((p) => p.text ?? '') + .join('') + .trim(); +} diff --git a/scripts/llm-panel/report.mjs b/scripts/llm-panel/report.mjs new file mode 100644 index 0000000..df9f59b --- /dev/null +++ b/scripts/llm-panel/report.mjs @@ -0,0 +1,77 @@ +// Synthesize the panel's per-model feedback into one markdown report: a score +// table, then each panelist's notes, then an aggregated "what to fix" list +// ranked by how many panelists raised each theme. + +function avg(nums) { + const v = nums.filter((n) => typeof n === 'number'); + return v.length ? (v.reduce((a, b) => a + b, 0) / v.length).toFixed(1) : '—'; +} + +export function renderReport(scenario, base, results) { + const ok = results.filter((r) => r.feedback && !r.feedback.parseError); + const lines = []; + lines.push(`# Taskflow LLM feedback panel — ${scenario.title}`); + lines.push(''); + lines.push(`Target: \`${base}\` · Panelists: ${results.length}`); + lines.push(''); + + // Score table. + lines.push('## Scores'); + lines.push(''); + lines.push('| Panelist | Delight | Clarity | Steps | First impression |'); + lines.push('|---|---|---|---|---|'); + for (const r of results) { + const f = r.feedback ?? {}; + lines.push( + `| ${r.label} | ${f.delight ?? '—'} | ${f.clarity ?? '—'} | ${r.transcript?.length ?? '—'} | ${(f.firstImpression ?? r.error ?? '—').replace(/\|/g, '/')} |`, + ); + } + lines.push(''); + lines.push( + `**Average delight ${avg(ok.map((r) => r.feedback.delight))}/10 · clarity ${avg(ok.map((r) => r.feedback.clarity))}/10**`, + ); + lines.push(''); + + // Aggregate improvement themes (naive fuzzy grouping by first 4 words). + const themes = new Map(); + for (const r of ok) { + for (const imp of r.feedback.top_improvements ?? []) { + const key = imp.toLowerCase().split(/\s+/).slice(0, 4).join(' '); + const e = themes.get(key) ?? { text: imp, votes: 0 }; + e.votes += 1; + themes.set(key, e); + } + } + const ranked = [...themes.values()].sort((a, b) => b.votes - a.votes); + if (ranked.length) { + lines.push('## Top improvements (ranked by how many panelists raised it)'); + lines.push(''); + for (const t of ranked) lines.push(`- **(${t.votes}×)** ${t.text}`); + lines.push(''); + } + + // Per-panelist detail. + lines.push('## Per-panelist feedback'); + lines.push(''); + for (const r of results) { + lines.push(`### ${r.label}`); + if (r.error) { + lines.push(`> Run error: ${r.error}`); + lines.push(''); + continue; + } + const f = r.feedback ?? {}; + if (f.parseError) { + lines.push(`> Could not parse feedback JSON. Raw: ${f.raw}`); + lines.push(''); + continue; + } + lines.push(`- **Delight ${f.delight}/10 · Clarity ${f.clarity}/10**`); + lines.push(`- vs Asana: ${f.vs_asana ?? '—'}`); + if (f.worked_well?.length) lines.push(`- 👍 ${f.worked_well.join('; ')}`); + if (f.confusing_or_bad?.length) lines.push(`- 👎 ${f.confusing_or_bad.join('; ')}`); + if (f.top_improvements?.length) lines.push(`- 🛠 ${f.top_improvements.join('; ')}`); + lines.push(''); + } + return lines.join('\n'); +} diff --git a/scripts/llm-panel/run.mjs b/scripts/llm-panel/run.mjs new file mode 100644 index 0000000..f8fa924 --- /dev/null +++ b/scripts/llm-panel/run.mjs @@ -0,0 +1,119 @@ +// LLM feedback panel: point a panel of different vendors' vision models at a +// running Taskflow instance, let each drive a real browser (Playwright) to +// organize a project, then collect structured product feedback and synthesize a +// report. See CLAUDE.md > "LLM feedback panel". +// +// npm run panel # default: wedding scenario vs the dev server +// PANEL_SCENARIO=trip npm run panel +// PANEL_BASE=https://taskflow.example npm run panel # target a deployed URL +// +// Env: TEST_USERNAME / TEST_PASSWORD (from .env.local) to sign in; AWS_PROFILE +//=personal for Bedrock. Output: /tmp/tf-panel// (transcripts, shots, +// report.md). +import { chromium } from '@playwright/test'; +import { existsSync, readFileSync, mkdirSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { runAgent } from './agent.mjs'; +import { converse } from './bedrock.mjs'; +import { PANEL, SCENARIOS, FEEDBACK_PROMPT } from './scenarios.mjs'; +import { renderReport } from './report.mjs'; + +if (existsSync('.env.local')) { + for (const line of readFileSync('.env.local', 'utf8').split('\n')) { + const m = line.match(/^([A-Z0-9_]+)=(.*)$/); + if (m && !process.env[m[1]]) process.env[m[1]] = m[2]; + } +} + +const BASE = process.env.PANEL_BASE || 'http://localhost:5173'; +const SCENARIO = SCENARIOS[process.env.PANEL_SCENARIO || 'wedding']; +const OUT = join('/tmp/tf-panel', String(process.env.PANEL_RUN_ID || 'run')); + +async function signIn(page) { + await page.goto(`${BASE}/signin`); + await page.getByLabel('Email').waitFor({ timeout: 20_000 }); + await page.getByLabel('Email').fill(process.env.TEST_USERNAME); + await page.getByLabel('Password').fill(process.env.TEST_PASSWORD); + await page.getByTestId('signin-submit').click(); + await page.getByTestId('home-greeting').waitFor({ timeout: 25_000 }); +} + +async function askFeedback(modelId, transcript) { + const summary = transcript + .map((t) => (t.action ? `${t.n}. ${JSON.stringify(t.action)}` : `${t.n}. error: ${t.error}`)) + .join('\n'); + const reply = await converse({ + modelId, + system: FEEDBACK_PROMPT, + messages: [ + { + role: 'user', + content: `Here is what you did:\n${summary}\n\nNow give your product feedback as JSON.`, + }, + ], + maxTokens: 900, + }); + const m = reply.match(/\{[\s\S]*\}/); + try { + return JSON.parse(m[0]); + } catch { + return { parseError: true, raw: reply.slice(0, 500) }; + } +} + +async function runPanelist(browser, panelist) { + const dir = join(OUT, panelist.key); + mkdirSync(dir, { recursive: true }); + const ctx = await browser.newContext({ viewport: { width: 1280, height: 800 } }); + const page = await ctx.newPage(); + const result = { ...panelist, steps: [] }; + try { + await signIn(page); + const goal = `${SCENARIO.persona}\n\n${SCENARIO.goal}`; + result.transcript = await runAgent({ + page, + modelId: panelist.modelId, + goal, + onStep: async ({ n }) => { + await page.screenshot({ path: join(dir, `step-${String(n).padStart(2, '0')}.png`) }); + }, + }); + await page.screenshot({ path: join(dir, 'final.png') }); + result.feedback = await askFeedback(panelist.modelId, result.transcript); + } catch (e) { + result.error = e.message; + } finally { + await ctx.close(); + } + writeFileSync(join(dir, 'result.json'), JSON.stringify(result, null, 2)); + return result; +} + +async function main() { + mkdirSync(OUT, { recursive: true }); + // PANEL_ONLY=claude,nova runs a subset (cheaper smoke runs); default = all. + const only = (process.env.PANEL_ONLY || '').split(',').filter(Boolean); + const panel = only.length ? PANEL.filter((p) => only.includes(p.key)) : PANEL; + console.log(`LLM panel — scenario "${SCENARIO.title}" vs ${BASE}`); + console.log(`Panel: ${panel.map((p) => p.label).join(', ')}\nOutput: ${OUT}\n`); + const browser = await chromium.launch(); + const results = []; + for (const panelist of panel) { + console.log(`▶ ${panelist.label} …`); + const r = await runPanelist(browser, panelist); + const d = r.feedback?.delight ?? '—'; + console.log( + ` ${r.error ? 'ERROR: ' + r.error : `done (delight ${d}/10, ${r.transcript?.length} steps)`}`, + ); + results.push(r); + } + await browser.close(); + const report = renderReport(SCENARIO, BASE, results); + writeFileSync(join(OUT, 'report.md'), report); + console.log(`\n✅ Report: ${join(OUT, 'report.md')}`); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/scripts/llm-panel/scenarios.mjs b/scripts/llm-panel/scenarios.mjs new file mode 100644 index 0000000..ab1306c --- /dev/null +++ b/scripts/llm-panel/scenarios.mjs @@ -0,0 +1,57 @@ +// Panel definition + scenarios. Each panelist is a different vendor's vision +// model (reached via Bedrock) so the feedback reflects genuinely different +// "users". Swap PANEL / pick a scenario via env (see run.mjs). + +export const PANEL = [ + { + key: 'claude', + label: 'Claude Haiku 4.5', + modelId: 'global.anthropic.claude-haiku-4-5-20251001-v1:0', + }, + { key: 'nova', label: 'Amazon Nova Pro', modelId: 'amazon.nova-pro-v1:0' }, + { + key: 'llama', + label: 'Meta Llama 4 Maverick', + modelId: 'us.meta.llama4-maverick-17b-instruct-v1:0', + }, + { key: 'pixtral', label: 'Mistral Pixtral Large', modelId: 'us.mistral.pixtral-large-2502-v1:0' }, +]; + +export const SCENARIOS = { + wedding: { + title: 'Plan a wedding', + persona: + 'You are one half of a couple planning your wedding. You are NOT technical — you just want to get organized.', + goal: + 'Create a project for your wedding, add a few sections (e.g. Venue, Guests, Catering), ' + + 'and add several real tasks with due dates and priorities (book venue, send invitations, ' + + 'choose caterer, buy rings). Try the board and list views. Organize it the way a real couple would.', + }, + launch: { + title: 'Plan a product launch', + persona: 'You are a product manager organizing a software launch. You want a clear plan.', + goal: + 'Create a project for a product launch, add sections and tasks (write announcement, QA, ' + + 'marketing, ship), set priorities and due dates, and try both board and list views.', + }, + trip: { + title: 'Plan a group trip', + persona: 'You are organizing a trip for a group of friends. You want everyone aligned.', + goal: + 'Create a project for a group trip, add sections (Flights, Lodging, Activities), add tasks ' + + 'with due dates, assign priorities, and explore the views.', + }, +}; + +export const FEEDBACK_PROMPT = `You just spent time using this task-management web app to accomplish your goal. +Give honest, specific product feedback as the non-technical user you were role-playing. +Respond as JSON with EXACTLY these keys (no code fences): +{ + "delight": , + "clarity": , + "firstImpression": "", + "worked_well": ["", "..."], + "confusing_or_bad": ["", "..."], + "top_improvements": ["", "", ""], + "vs_asana": "" +}`;