From 0b1ad537c85059d2fad7cf174cf4e51dcf0a8501 Mon Sep 17 00:00:00 2001 From: "[._.]/ Adam Eivy" Date: Fri, 14 Aug 2026 18:57:52 -0700 Subject: [PATCH 1/3] feat([issue-4188]): per-item AI prompt analysis on mood boards (Phase 3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Board items backed by gallery media gain an Analyze with AI action that runs the video page's Prompt-from-media flow and persists the result on the item as an optional analysis field (prompt, negative, rationale, provider/model, analyzedAt) via the item PATCH — additive on the wire, so no moodBoards gate bump (whole-record LWW preserves unknown keys, same precedent as the pinterest sub-object). The prompt-from-media endpoint now also resolves a gallery video by on-disk filename, matching the video: refs board items carry (the gallery flow's history-id path is unchanged). --- .changelog/next/added-issue-4188.md | 1 + .../src/components/media/PromptFromMedia.jsx | 13 +- .../components/media/PromptFromMedia.test.jsx | 32 ++++ client/src/lib/README.md | 2 +- client/src/lib/moodBoardItemSrc.js | 29 ++++ client/src/lib/moodBoardItemSrc.test.js | 23 ++- client/src/pages/MoodBoardDetail.jsx | 144 ++++++++++++++++-- client/src/pages/MoodBoardDetail.test.jsx | 110 ++++++++++++- server/lib/moodBoardValidation.js | 20 ++- server/lib/moodBoardValidation.test.js | 23 +++ server/routes/mediaJobs.js | 6 +- server/routes/mediaJobs.test.js | 9 ++ server/services/mediaPromptFromMedia.js | 21 ++- server/services/mediaPromptFromMedia.test.js | 32 ++++ server/services/moodBoard/logic.js | 27 +++- server/services/moodBoard/logic.test.js | 29 ++++ 16 files changed, 500 insertions(+), 21 deletions(-) diff --git a/.changelog/next/added-issue-4188.md b/.changelog/next/added-issue-4188.md index b5e7a04752..a94a436c57 100644 --- a/.changelog/next/added-issue-4188.md +++ b/.changelog/next/added-issue-4188.md @@ -1,2 +1,3 @@ - Universes can now link a mood board directly on the record: pick or create a board from the Universe Bible tab and the link survives reload, stays per-universe, and syncs to your other machines (previously the reference strip only remembered one board per browser). - Mood boards can now pull items straight from your galleries: pick images from the image gallery, pick or upload videos, and video items play right on the board with a poster thumbnail. Uploaded videos land in the shared video gallery, so board items sync to your other machines like any other media. +- Mood board items can now be analyzed with AI: an "Analyze with AI" action on gallery-backed items runs the same Prompt-from-media flow as the video page (your choice of vision provider) and saves the resulting prompt, negative prompt, and rationale onto the item — shown with a highlighted badge, viewable/copyable/removable from the item, and synced to your other machines with the board. diff --git a/client/src/components/media/PromptFromMedia.jsx b/client/src/components/media/PromptFromMedia.jsx index 5a2c4386f8..58624a20e8 100644 --- a/client/src/components/media/PromptFromMedia.jsx +++ b/client/src/components/media/PromptFromMedia.jsx @@ -51,6 +51,8 @@ function SourceThumb({ source }) { * `kindDefault` seeds the target checkboxes (`image` / `video` / `both`). * When `setPrompt` is passed (Image Gen / Video Gen), Apply fills the host * form. Otherwise the result offers "Open in Image Gen / Video Gen". + * `onResult` (optional) fires with the full analysis payload after each + * successful run, so a host (e.g. a mood-board item — #4188) can persist it. */ export default function PromptFromMedia({ kindDefault = 'both', @@ -60,6 +62,7 @@ export default function PromptFromMedia({ initialSource = null, disabled = false, alwaysOpen = false, + onResult, }) { const navigate = useNavigate(); const idPrefix = useId(); @@ -174,6 +177,7 @@ export default function PromptFromMedia({ if (!data) return; setResult(data); toast.success('Prompts ready'); + if (onResult) onResult(data); }; const apply = (kind) => { @@ -389,7 +393,10 @@ function PromptResultField({ label, value, negative, onCopy, onApply, applyLabel ); } -export function PromptFromMediaModal({ item, open, onClose }) { +// `kindDefault` / `onResult` pass through to PromptFromMedia; `children` +// render above the analyzer in the scroll area — a host can slot in the +// item's stored analysis (mood boards — #4188). +export function PromptFromMediaModal({ item, open, onClose, kindDefault = 'both', onResult, children }) { if (!open || !item) return null; return (
+ {children}
diff --git a/client/src/components/media/PromptFromMedia.test.jsx b/client/src/components/media/PromptFromMedia.test.jsx index 1a5165d01e..dc7d510e17 100644 --- a/client/src/components/media/PromptFromMedia.test.jsx +++ b/client/src/components/media/PromptFromMedia.test.jsx @@ -110,6 +110,38 @@ describe('PromptFromMedia', () => { expect(screen.getByText('the camera dollies past the subject')).toBeInTheDocument(); }); + it('notifies the host via onResult and sends a filename-only clip without a videoId (#4188)', async () => { + const onResult = vi.fn(); + const payload = { + videoPrompt: 'a slow dolly through fog', + videoNegativePrompt: 'jitter', + rationale: 'Foggy push-in.', + providerId: 'openai', + model: 'gpt-4o', + }; + vi.mocked(api.promptFromMedia).mockResolvedValue(payload); + + renderPanel({ + kindDefault: 'video', + applyKind: undefined, + onResult, + // A mood-board video item resolves by on-disk filename — no history id. + initialSource: { kind: 'video', filename: 'clip.mp4', previewUrl: '/data/video-thumbnails/clip.jpg' }, + }); + + fireEvent.click(screen.getByRole('button', { name: /create prompt/i })); + + await waitFor(() => { + expect(api.promptFromMedia).toHaveBeenCalledWith(expect.objectContaining({ + sourceKind: 'video', + videoId: undefined, + filename: 'clip.mp4', + targets: ['video'], + })); + }); + expect(onResult).toHaveBeenCalledWith(payload); + }); + it('skips the disclosure toggle when hosted as an always-open card', () => { renderPanel({ alwaysOpen: true, initialSource: null }); expect(screen.queryByRole('button', { name: /toggle prompt from media/i })).toBeNull(); diff --git a/client/src/lib/README.md b/client/src/lib/README.md index 9c4f24d2fb..39198ae30b 100644 --- a/client/src/lib/README.md +++ b/client/src/lib/README.md @@ -114,7 +114,7 @@ grep -i "what you want to do" client/src/lib/README.md | `mediaNavigation.js` | `getAdjacentMedia(items, item)` — prev/next computation for lightboxes. | | `mediaCollectionList.js` | Search / sort ordering for the Media Collections grid (#3283) and the `CollectionPickerShell` dropdown (#3312): `applyCollectionView(collections, { query, sort })` filters via `mediaSearch.js`'s AND-token matcher over name+description and orders synthetic "Unsorted" → non-empty-or-user-created → auto-generated empties, then by `COLLECTION_SORTS` (`updated` / `name` / `count`, `normalizeCollectionSort` coerces an unknown/URL value); keys are derived once per record, not per comparison. "Hide empty" is deliberately NOT a parameter — the page owns that one predicate (`collectionItemCount(c) > 0`) because it also needs the pre-filter count. `isAutoCollection(c)` recognizes a machine-created collection from ANY of four independent markers (`AUTO_NAME_PREFIXES`, `Auto-created…`/`Auto-generated…` description, `uc-`/`sc-` id, universe/series link) so a record predating a given marker still classifies; `splitCollectionName(name)` lifts the shared `Creative Director: ` / `Writers Room: ` / `Universe: ` / `Series: ` prefix into a badge label so the distinguishing tail survives truncation, and name sorting uses that stripped title. **Adding a server-side auto-creator means adding its prefix here** or its collections silently rank as user-created. Pure — no React. | | `mediaSearch.js` | `buildMediaHaystack`, `tokenizeQuery`, `matchHaystack`, `filterByQuery` — client-side AND-token search over normalized media items (prompt/model/seed/LoRA/universe tags). Shared by MediaHistory + the Image Gen gallery picker. | -| `moodBoardItemSrc.js` | `moodBoardItemSrc(item)` resolves a mood-board item to a display image/poster src (`imageUrl` → served `image:` bytes → derived video thumbnail → null); `moodBoardItemVideoSrc(item)` resolves a `type:'video'` item's playback URL. Shared by MoodBoardDetail + MoodBoardReferenceStrip. | +| `moodBoardItemSrc.js` | `moodBoardItemSrc(item)` resolves a mood-board item to a display image/poster src (`imageUrl` → served `image:` bytes → derived video thumbnail → null); `moodBoardItemVideoSrc(item)` resolves a `type:'video'` item's playback URL; `moodBoardItemAnalysisSource(item)` resolves an item to a prompt-from-media source (null when not a local gallery asset). Shared by MoodBoardDetail + MoodBoardReferenceStrip. | | `registerServiceWorker.js` | `registerServiceWorker()` / `unregisterServiceWorkers()` — wires up the offline app-shell + low-bandwidth asset-caching service worker (`public/sw.js`). Registers only in a production secure context (HTTPS or localhost); no-ops over plain-HTTP Tailnet and tears down any stale SW in dev. Called once from `main.jsx`. | | `safeStorage.js` | `safeReadStorage` / `safeReadJsonStorage` / `safeWriteStorage` / `safeWriteJsonStorage` / `safeRemoveStorage` — guarded `localStorage` access that swallows throws (Safari private mode, blocked storage), with fallback-safe JSON parsing for structured entries. Use instead of touching `localStorage` inline so a storage failure never crashes init or a write path (#2387). Consumed by `useTheme`, `useCitySettings`, `useNavWorkingSet`, and the command palette. Also `safeReadJsonSession` / `safeWriteJsonSession` / `safeRemoveSession` — the same guarantees over `sessionStorage`, for tab-scoped crash-recovery buffers of edits the server has not accepted yet (QuotaBurn's unsaved-patch stash). | | `sameJsonShape.js` | `sameJsonShape(prev, next)` — JSON.stringify-based equality for `useAutoRefetch`'s `compare` option on small, deterministically-shaped poll payloads. | diff --git a/client/src/lib/moodBoardItemSrc.js b/client/src/lib/moodBoardItemSrc.js index cd66221e5a..47341212c8 100644 --- a/client/src/lib/moodBoardItemSrc.js +++ b/client/src/lib/moodBoardItemSrc.js @@ -30,6 +30,35 @@ export function moodBoardItemSrc(item) { return null; } +// Resolve a board item to a prompt-from-media source (#4188 Phase 3) — the +// gallery-item shape PromptFromMedia's `initialSource` expects. Returns null +// when the item's media isn't a local gallery asset the analyzer can read: +// text items, external-URL pins, and legacy `video:` pins on image items. +// A video item resolves by FILENAME (`kind:'video'` with no id — the server +// accepts filename in place of the history id); an image item resolves by its +// `image:` media-key or a `/data/images/` app-path imageUrl. +export function moodBoardItemAnalysisSource(item) { + if (item?.type === 'video') { + if (typeof item?.mediaKey === 'string' && item.mediaKey.startsWith(VIDEO_PREFIX)) { + const filename = item.mediaKey.slice(VIDEO_PREFIX.length); + if (filename) return { kind: 'video', filename, previewUrl: moodBoardItemSrc(item) }; + } + return null; + } + if (item?.type !== 'image') return null; + if (typeof item?.mediaKey === 'string' && item.mediaKey.startsWith(IMAGE_PREFIX)) { + const filename = item.mediaKey.slice(IMAGE_PREFIX.length); + if (filename) return { filename, previewUrl: moodBoardItemSrc(item) }; + } + const GALLERY_PREFIX = '/data/images/'; + if (typeof item?.imageUrl === 'string' && item.imageUrl.startsWith(GALLERY_PREFIX)) { + let filename = item.imageUrl.slice(GALLERY_PREFIX.length); + try { filename = decodeURIComponent(filename); } catch { /* keep raw */ } + if (filename && !filename.includes('/')) return { filename, previewUrl: item.imageUrl }; + } + return null; +} + // Playback URL for a `type: 'video'` item; null for anything else (including // legacy `video:` pins on image items, whose ref is not a filename). export function moodBoardItemVideoSrc(item) { diff --git a/client/src/lib/moodBoardItemSrc.test.js b/client/src/lib/moodBoardItemSrc.test.js index c2e8f5a222..efb073a6d6 100644 --- a/client/src/lib/moodBoardItemSrc.test.js +++ b/client/src/lib/moodBoardItemSrc.test.js @@ -1,5 +1,5 @@ import { describe, it, expect } from 'vitest'; -import { moodBoardItemSrc, moodBoardItemVideoSrc } from './moodBoardItemSrc'; +import { moodBoardItemSrc, moodBoardItemVideoSrc, moodBoardItemAnalysisSource } from './moodBoardItemSrc'; describe('moodBoardItemSrc', () => { it('prefers an explicit imageUrl', () => { @@ -51,3 +51,24 @@ describe('moodBoardItemVideoSrc', () => { expect(moodBoardItemVideoSrc(null)).toBeNull(); }); }); + +describe('moodBoardItemAnalysisSource (#4188 Phase 3)', () => { + it('resolves a video item to a filename video source with its poster', () => { + expect(moodBoardItemAnalysisSource({ + type: 'video', mediaKey: 'video:clip.mp4', imageUrl: '/data/video-thumbnails/clip.jpg', + })).toEqual({ kind: 'video', filename: 'clip.mp4', previewUrl: '/data/video-thumbnails/clip.jpg' }); + }); + it('resolves an image item by media-key or a /data/images app path (decoded)', () => { + expect(moodBoardItemAnalysisSource({ type: 'image', mediaKey: 'image:ref.png' })) + .toEqual({ filename: 'ref.png', previewUrl: '/data/images/ref.png' }); + expect(moodBoardItemAnalysisSource({ type: 'image', imageUrl: '/data/images/my%20render.png' })) + .toEqual({ filename: 'my render.png', previewUrl: '/data/images/my%20render.png' }); + }); + it('returns null for text items, external pins, and legacy video: pins on image items', () => { + expect(moodBoardItemAnalysisSource({ type: 'text', text: 'n' })).toBeNull(); + expect(moodBoardItemAnalysisSource({ type: 'image', imageUrl: 'https://x/y.png' })).toBeNull(); + expect(moodBoardItemAnalysisSource({ type: 'image', mediaKey: 'video:abc', imageUrl: 'https://x/t.jpg' })).toBeNull(); + expect(moodBoardItemAnalysisSource({ type: 'video', mediaKey: null })).toBeNull(); + expect(moodBoardItemAnalysisSource(null)).toBeNull(); + }); +}); diff --git a/client/src/pages/MoodBoardDetail.jsx b/client/src/pages/MoodBoardDetail.jsx index e8a10e489f..002d2c35ad 100644 --- a/client/src/pages/MoodBoardDetail.jsx +++ b/client/src/pages/MoodBoardDetail.jsx @@ -11,12 +11,14 @@ import { useEffect, useState, useCallback, useRef } from 'react'; import { useParams, useNavigate, Link } from 'react-router'; -import { ArrowLeft, ImageIcon, FileText, Trash2, Plus, Save, Link2, Unlink, RefreshCw, Images, Film, Play } from 'lucide-react'; +import { ArrowLeft, ImageIcon, FileText, Trash2, Plus, Save, Link2, Unlink, RefreshCw, Images, Film, Play, ScanEye, Copy } from 'lucide-react'; import PageSkeleton from '../components/ui/PageSkeleton'; import toast from '../components/ui/Toast'; import InlineConfirmRow from '../components/ui/InlineConfirmRow'; import GalleryImagePicker from '../components/imageGen/GalleryImagePicker'; import GalleryVideoPicker from '../components/videoGen/GalleryVideoPicker'; +import { PromptFromMediaModal } from '../components/media/PromptFromMedia'; +import { copyToClipboard } from '../lib/clipboard'; import { getMoodBoard, updateMoodBoard, @@ -27,7 +29,7 @@ import { unlinkMoodBoardPinterest, syncMoodBoardPinterest, } from '../services/api'; -import { moodBoardItemSrc, moodBoardItemVideoSrc } from '../lib/moodBoardItemSrc'; +import { moodBoardItemSrc, moodBoardItemVideoSrc, moodBoardItemAnalysisSource } from '../lib/moodBoardItemSrc'; import { timeAgo } from '../utils/formatters'; import useMounted from '../hooks/useMounted'; @@ -54,6 +56,11 @@ export default function MoodBoardDetail() { const [videoPickerOpen, setVideoPickerOpen] = useState(false); const [playingItemId, setPlayingItemId] = useState(null); + // Per-item prompt-from-media analysis (#4188 Phase 3). Track the item by id + // (not a snapshot) so the modal's stored-analysis view stays fresh after the + // persist PATCH updates the board state. + const [analyzeItemId, setAnalyzeItemId] = useState(null); + // Pinterest link/sync. const [pinUrl, setPinUrl] = useState(''); const [linking, setLinking] = useState(false); @@ -159,6 +166,42 @@ export default function MoodBoardDetail() { : prev)); }; + // Persist a prompt-from-media run onto the item (#4188 Phase 3). The + // analyzer can return an image and/or a video prompt; store the one that + // matches the item's own type, falling back to whichever was generated. + const persistAnalysis = async (item, result) => { + const preferVideo = item.type === 'video'; + const primary = preferVideo ? result.videoPrompt : result.imagePrompt; + const fallback = preferVideo ? result.imagePrompt : result.videoPrompt; + const usedPrimary = primary != null && primary !== ''; + const prompt = usedPrimary ? primary : fallback; + if (!prompt) return; + const negative = usedPrimary + ? (preferVideo ? result.videoNegativePrompt : result.imageNegativePrompt) + : (preferVideo ? result.imageNegativePrompt : result.videoNegativePrompt); + const analysis = { + prompt, + negativePrompt: negative || null, + rationale: result.rationale || null, + providerId: result.providerId || null, + model: result.model || null, + }; + const updated = await updateMoodBoardItem(id, item.id, { analysis }, { silent: true }).catch(() => null); + if (!updated) { toast.error('Analysis ran but could not be saved to the item'); return; } + setBoard((prev) => (prev + ? { ...prev, items: (prev.items || []).map((it) => (it.id === item.id ? updated : it)) } + : prev)); + toast.success('Analysis saved to item'); + }; + + const handleClearAnalysis = async (itemId) => { + const updated = await updateMoodBoardItem(id, itemId, { analysis: null }, { silent: true }).catch(() => null); + if (!updated) { toast.error('Failed to remove analysis'); return; } + setBoard((prev) => (prev + ? { ...prev, items: (prev.items || []).map((it) => (it.id === itemId ? updated : it)) } + : prev)); + }; + const handleRemoveItem = async (itemId) => { setConfirmingItemId(null); const updated = await removeMoodBoardItem(id, itemId, { silent: true }).catch(() => null); @@ -219,6 +262,7 @@ export default function MoodBoardDetail() { } const items = Array.isArray(board.items) ? board.items : []; + const analyzeItem = analyzeItemId ? (items.find((it) => it.id === analyzeItemId) || null) : null; const linkedFeedUrl = board.pinterest?.feedUrl || ''; const linkedBoardUrl = board.pinterest?.boardUrl || ''; const lastSyncedAt = board.pinterest?.lastSyncedAt || null; @@ -485,6 +529,7 @@ export default function MoodBoardDetail() { {items.map((item) => { const src = moodBoardItemSrc(item); const videoSrc = moodBoardItemVideoSrc(item); + const analysisSource = moodBoardItemAnalysisSource(item); return (
{item.type === 'video' && videoSrc ? ( @@ -563,15 +608,28 @@ export default function MoodBoardDetail() { {item.source ? ( {item.source} ) : } - +
+ {analysisSource ? ( + + ) : null} + +
{confirmingItemId === item.id ? ( + + {/* Per-item prompt-from-media analysis (#4188 Phase 3). A successful run + auto-persists onto the item; the stored analysis renders above the + analyzer with copy/remove. */} + {analyzeItem ? ( + setAnalyzeItemId(null)} + kindDefault={analyzeItem.type === 'video' ? 'video' : 'image'} + onResult={(result) => persistAnalysis(analyzeItem, result)} + > + {analyzeItem.analysis ? ( +
+
+ + Saved analysis{analyzeItem.analysis.analyzedAt ? ` · ${timeAgo(analyzeItem.analysis.analyzedAt)}` : ''} + +
+ + +
+
+ {analyzeItem.analysis.rationale ? ( +

{analyzeItem.analysis.rationale}

+ ) : null} +