From fdae9f3be2ecec9c02a89d523d1733c4d9898798 Mon Sep 17 00:00:00 2001 From: "[._.]/ Adam Eivy" Date: Mon, 27 Jul 2026 02:00:38 -0700 Subject: [PATCH 1/2] feat: add IC-LoRA Ingredients remix mode with a single-file weight fetch (#3112) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ingredients recomposes a scene from 2-8 reference stills, so it needs both a new input surface and a weight-provisioning path the existing modes didn't. The weight is the hard part. Its first-party repo (Lightricks) is gated, and the un-gated mirror that carries it is DeepBeepMeep/LTX-2 — a ~708 GB aggregate of every LTX weight. The existing download helper only enumerated and pulled whole repos, and resolveIcLoraWeight's bare-repo-id fallback hands the id to ICLoraPipeline._resolve_lora_path, which snapshot_downloads it. So: - hf_download_repo.py gains `--only ` (repeatable), which skips repo enumeration entirely rather than filtering an enumerated list — the absence of a list means nothing downstream can widen it into a snapshot. - startHfDownloadStream gains `fallbacks` (first-success-wins, official then mirror) plus a `cachedFile` predicate, because the repo-wide cache verdict is meaningless for an aggregate mirror: it reports cached off any unrelated resident weight, which would skip the file the user actually asked for. - A `requiresPreDownload` spec suppresses the repo-id fallback outright, so an un-fetched Ingredients render 400s with "download the weight first" instead of starting an unbounded pull mid-render. Its status/repair paths are scoped to the one file rather than walking the mirror's whole snapshot. referenceKind: 'image' selects the second input surface. References resolve from the gallery like `keyframes` do, on their own `icReferenceImageFiles` field — the video fields are left alone, and cross-kind requests are rejected rather than dropped (an image-kind weight fed a clip returns plausible garbage, not an error). The pipeline's reference channel is a video encoder end to end (ffprobe + a VAE needing 1+8k frames), so each still is materialized into a 9-frame clip at render resolution first. The reference count is a weight contract enforced at all three layers from one registry bound, and the metadata behind the entry was measured, not assumed: reference_downscale_factor=1 and the 1,308,778,338-byte size come from a Range read of the weight's safetensors header and HF's x-linked-size. --- .changelog/NEXT.md | 7 + .../src/components/videoGen/IcLoraPanel.jsx | 111 +++++++-- .../components/videoGen/IcLoraPanel.test.jsx | 81 +++++++ client/src/lib/videoGenParams.js | 14 ++ client/src/lib/videoGenParams.test.js | 18 +- client/src/pages/VideoGen.jsx | 60 ++++- scripts/generate_ltx2.py | 6 + scripts/generate_ltx2_ic_lora_test.py | 215 ++++++++++++++++++ scripts/hf_download_repo.py | 81 ++++--- server/lib/README.md | 4 +- server/lib/hfCache.js | 21 ++ server/lib/hfDownload.js | 18 +- server/lib/icLoraWeights.js | 121 ++++++++-- server/lib/icLoraWeights.test.js | 94 +++++++- server/lib/sseDownload.js | 145 +++++++++--- server/lib/sseDownload.test.js | 122 ++++++++++ server/routes/videoGen.js | 160 +++++++++++-- server/routes/videoGen.test.js | 96 ++++++++ server/services/videoGen/local.js | 82 ++++++- server/services/videoGen/local.test.js | 150 +++++++++++- 20 files changed, 1491 insertions(+), 115 deletions(-) create mode 100644 scripts/generate_ltx2_ic_lora_test.py diff --git a/.changelog/NEXT.md b/.changelog/NEXT.md index f078fb8ee4..c6812aa8c7 100644 --- a/.changelog/NEXT.md +++ b/.changelog/NEXT.md @@ -124,3 +124,10 @@ - **[issue-3133] Every reviewer chain in PortOS now reads as one per-reviewer table — Provider, Model, Optional, Max Iterations — and you can pick a reviewer's model from all four places that show a reviewer chain, not just the settings panel.** Code Review Defaults used to render reviewers as a chip row with a stack of per-backend model dropdowns floating below it; that model-picking logic lived in the panel, so the Add Task form, a task type's Schedule config, and the Agent Operations run drawer — all of which reuse the same picker — had no way to say which model a reviewer should run. Each reviewer is now a row carrying all four of its controls, so "run codex on this tier, cap ollama at one round, and don't let it block the merge" is one readable line instead of three controls in three places. On a phone the row stacks into labelled fields rather than scrolling sideways. - A local backend (LM Studio / Ollama) offers the models you actually have installed; Codex and Claude accept a typed id as well as a pick, because an Ollama-backed `claude` or a Bedrock-form model id is whatever your machine provides and can't be enumerated. A pinned model that's since been uninstalled is still shown as pinned rather than rendering blank and reading as unset, and a backend that isn't running says so instead of just looking empty. - The model pin now belongs to the task, not only to the install: a per-task or per-task-type choice overrides your Code Review Defaults the same way its `~opt` and round-cap siblings already do, and an explicitly empty choice means "each reviewer's own default" rather than silently inheriting. Clearing the field removes the pin instead of storing an empty model id. Your saved Code Review Defaults are read and written unchanged, so nothing needs migrating. + +## Video Gen — Ingredients remix + +- **[issue-3112] Local LTX-2 video gen can now remix a scene from 2-8 reference stills.** A new **Ingredients** mode joins Control and Colorize: instead of one reference clip it takes a 2-8 row list of gallery images — characters, props, settings — and recomposes them into a new shot from your prompt. The picker is the same multi-row gallery list the keyframe anchors use (add/remove rows, inline previews, a live 2/8 counter), and it won't let you submit an unfilled row or a count the weight can't accept. The existing single-reference clip modes are untouched. +- **The weight is fetched one file at a time, never as a repo snapshot.** Ingredients' first-party weight repo is gated, and the un-gated mirror that carries it is a ~708 GB aggregate of every LTX weight — so pulling it the ordinary way would either 401 or fill your disk to get 1.3 GB. The download helper gained a real single-file mode, and the weight download tries the official repo first (for anyone with an HF token) and falls back to the mirror automatically, pulling exactly the one file either way. No token is required and no extra button to press. +- Because neither of those repos can safely be auto-downloaded mid-render, an Ingredients render on a machine that hasn't fetched the weight fails immediately with "download it first" instead of quietly starting an unbounded pull inside the render. Its cache, repair and integrity checks are likewise scoped to that one file rather than walking the whole mirror. +- Ingredients conditions on stills, but the pipeline's reference channel is a video encoder end to end, so each still is prepared into a tiny clip at the render resolution before conditioning — that plumbing is internal and the throwaway encodes are cleaned up with the render. User LoRAs still stack on top of the weight rather than replacing it, so an Ingredients × Character render composes. The 2-8 count is enforced in the form, at the route and in the render helper, all reading one shared bound. diff --git a/client/src/components/videoGen/IcLoraPanel.jsx b/client/src/components/videoGen/IcLoraPanel.jsx index fd3d9c1664..29124805d0 100644 --- a/client/src/components/videoGen/IcLoraPanel.jsx +++ b/client/src/components/videoGen/IcLoraPanel.jsx @@ -1,30 +1,45 @@ /** * IC-LoRA remix panel (issue #3100) — dgrauet/ltx2 runtime only. The user - * supplies a reference clip (upload or a prior render) that the fused IC-LoRA - * reads structure/motion out of, then dials how strongly it conditions. + * supplies the reference(s) the fused IC-LoRA conditions on, then dials how + * strongly they apply. * - * Presentational — the reference upload File, the picked history ids, the - * strength dials, the weight's download status, and the "no compatible model - * installed" condition are all owned by the VideoGen page. `onPickHistory('')` - * clears the history selection. + * Two input surfaces, chosen by the registry's `spec.referenceKind`: + * - `video` (Control, Colorize) — ONE reference clip: a fresh upload or a + * prior render picked by history id, mutually exclusive server-side. + * - `image` (Ingredients, #3112) — a 2-8 ROW gallery list, modeled on + * KeyframePanel. Gallery-only, matching the route (references resolve under + * PATHS.images, so there's no upload shape to reconcile). + * + * Presentational — the reference upload File, the picked history id, the gallery + * reference rows, the strength dials, the weight's download status, and the "no + * compatible model installed" condition are all owned by the VideoGen page. + * `onPickHistory('')` clears the history selection. * * The IC-LoRA weight is a separate several-hundred-MB HF pull, so the panel * hosts its own ModelDownloadBadge: without it a first render silently stalls * on an un-progressed download inside the Python child. */ -import { Upload, Film } from 'lucide-react'; +import { Upload, Film, X, ListPlus } from 'lucide-react'; import { formatBytes } from '../../utils/formatters'; import { icResolutionIssue } from '../../lib/videoGenParams'; import FilePickerButton from '../ui/FilePickerButton'; import ModelDownloadBadge from '../media/ModelDownloadBadge'; +import ImagePreview from './ImagePreview'; import Ltx2RuntimeMissingNotice from './Ltx2RuntimeMissingNotice'; export default function IcLoraPanel({ - spec, // { mode, label, desc, referenceDownscaleFactor, min/maxReferences } - referenceFile, // File | null — fresh upload + spec, // { mode, label, desc, referenceDownscaleFactor, min/maxReferences, referenceKind } + referenceFile, // File | null — fresh upload (video-kind only) referenceVideoId, // string — picked prior render (mutually exclusive with the upload) inFlightReferenceNames = [], // display-only basenames of a resumed in-flight render visibleHistory, + // Image-kind (Ingredients) reference rows: gallery basenames, min/max enforced + // by the registry spec. The page owns the array and the three mutators. + referenceImageFiles = [], + visibleGallery = [], + onAddReferenceImage, + onUpdateReferenceImage, + onRemoveReferenceImage, icStrength, icSkipStage2, width, @@ -44,12 +59,24 @@ export default function IcLoraPanel({ const resolutionIssue = icResolutionIssue(spec, width, height); const strengthId = `ic-strength-${spec.mode}`; const skipId = `ic-skip-stage2-${spec.mode}`; + const imageKind = spec.referenceKind === 'image'; + // Row count is a WEIGHT CONTRACT, not a UI preference — read the bounds off the + // spec so a future weight with different limits needs no component change (and + // can't disagree with what the route enforces). + const { minReferences: minRefs, maxReferences: maxRefs } = spec; return (
- {spec.label} reference - {(referenceFile || referenceVideoId) && ( + + {spec.label} reference{imageKind ? 's' : ''} + {imageKind && ( + + Pick {minRefs}–{maxRefs} gallery stills to recompose from + + )} + + {!imageKind && (referenceFile || referenceVideoId) && (
- {referenceFile ? ( + {imageKind ? ( +
+ {referenceImageFiles.map((file, i) => ( +
+
+ + + {file && ( + + )} +
+ +
+ ))} +
+ + {referenceImageFiles.length}/{maxRefs} +
+

+ References pull from your gallery only. Each still is prepared at the render resolution before conditioning. +

+
+ ) : referenceFile ? (
{referenceFile.name} @@ -92,7 +166,7 @@ export default function IcLoraPanel({ )} - {!referenceFile && !referenceVideoId && inFlightReferenceNames.length > 0 && ( + {!imageKind && !referenceFile && !referenceVideoId && inFlightReferenceNames.length > 0 && (

In-flight render is conditioned on {inFlightReferenceNames.join(', ')} — re-pick a reference to run a new one.

@@ -127,9 +201,18 @@ export default function IcLoraPanel({

- {spec.description}. The reference is trimmed to fit the frame count. + {spec.description}. + {imageKind ? '' : ' The reference is trimmed to fit the frame count.'}

+ {weightStatus?.gated && !weightStatus.cached && ( +

+ The official weight repo is gated. Download tries it first (accept its license and add an + HF token in Image Gen settings for that path) and falls back to the un-gated mirror + automatically, so no token is required. +

+ )} + {resolutionIssue && (

{resolutionIssue} Adjust the resolution first. diff --git a/client/src/components/videoGen/IcLoraPanel.test.jsx b/client/src/components/videoGen/IcLoraPanel.test.jsx index 2c8eabd58a..5f5790d667 100644 --- a/client/src/components/videoGen/IcLoraPanel.test.jsx +++ b/client/src/components/videoGen/IcLoraPanel.test.jsx @@ -94,3 +94,84 @@ describe('IcLoraPanel', () => { expect(screen.getByText(/In-flight render is conditioned on depth\.mp4/i)).toBeTruthy(); }); }); + +// ── Image-kind (Ingredients) surface (#3112) ─────────────────────────────── +// referenceKind: 'image' swaps the single upload/history pair for a 2-8 gallery +// ROW list, modeled on KeyframePanel. Bounds are read off the spec, never +// hardcoded, so they can't disagree with what the route enforces. +describe('IcLoraPanel — image-kind references (#3112)', () => { + const ING = icLoraSpecForMode('ic-ingredients'); + const imageProps = { + ...baseProps, + spec: ING, + referenceImageFiles: ['owl.png', ''], + visibleGallery: [ + { filename: 'owl.png' }, { filename: 'store.png' }, { filename: 'tote.png' }, + ], + weightStatus: { id: 'ic-ingredients', repo: 'org/ingredients', cached: true, sizeBytes: 1_308_778_338 }, + onAddReferenceImage: vi.fn(), + onUpdateReferenceImage: vi.fn(), + onRemoveReferenceImage: vi.fn(), + }; + + it('renders one gallery row per reference, not the clip upload/history pair', () => { + render(); + expect(screen.getByLabelText(/Ingredients reference 1 gallery image/i)).toBeTruthy(); + expect(screen.getByLabelText(/Ingredients reference 2 gallery image/i)).toBeTruthy(); + // The video-only inputs must be absent — offering them would let a clip ride + // into an image-kind weight, which the route rejects. + expect(screen.queryByText(/Upload a reference still/i)).toBeNull(); + expect(screen.queryByLabelText(/Pick a previous render/i)).toBeNull(); + }); + + it('reports the picked gallery file through onUpdateReferenceImage', () => { + const onUpdateReferenceImage = vi.fn(); + render(); + fireEvent.change(screen.getByLabelText(/Ingredients reference 2 gallery image/i), { + target: { value: 'store.png' }, + }); + expect(onUpdateReferenceImage).toHaveBeenCalledWith(1, 'store.png'); + }); + + it('shows the row count against the spec maximum', () => { + render(); + expect(screen.getByText('2/8')).toBeTruthy(); + }); + + it('disables Add at the spec maximum', () => { + const eight = Array.from({ length: 8 }, (_, i) => `ref-${i}.png`); + render(); + expect(screen.getByText(/Add reference/i).closest('button').disabled).toBe(true); + expect(screen.getByText('8/8')).toBeTruthy(); + }); + + it('disables Remove at the spec minimum so the list cannot go under 2', () => { + render(); + for (const i of [1, 2]) { + expect(screen.getByLabelText(`Remove reference ${i}`).disabled).toBe(true); + } + }); + + it('enables Remove above the minimum', () => { + render(); + expect(screen.getByLabelText('Remove reference 1').disabled).toBe(false); + }); + + it('never warns about resolution (factor 1 imposes no rule)', () => { + // Ingredients' safetensors metadata reports reference_downscale_factor=1, so + // an odd resolution is legal — a copied factor-2 rule would block it for + // nothing. + render(); + expect(screen.queryByText(/divisible by/i)).toBeNull(); + }); + + it('explains the gated repo + automatic mirror fallback when un-cached', () => { + render(); + expect(screen.getByText(/falls back to the un-gated mirror/i)).toBeTruthy(); + }); + + it('omits the gated notice once the weight is cached', () => { + render(); + expect(screen.queryByText(/falls back to the un-gated mirror/i)).toBeNull(); + }); +}); diff --git a/client/src/lib/videoGenParams.js b/client/src/lib/videoGenParams.js index 481a46aa4c..f648e2ed89 100644 --- a/client/src/lib/videoGenParams.js +++ b/client/src/lib/videoGenParams.js @@ -87,6 +87,20 @@ export const IC_LORA_MODES = [ maxReferences: 1, referenceKind: 'video', }, + { + mode: 'ic-ingredients', + label: 'Ingredients', + description: 'A scene recomposed from 2-8 reference stills (characters, props, settings)', + uploadLabel: 'Upload a reference still (character / prop / setting)', + // 1 — read off the weight's safetensors metadata server-side; conditions on + // full-resolution references, so no divisibility rule. + referenceDownscaleFactor: 1, + minReferences: 2, + maxReferences: 8, + // Images, not clips: `referenceKind` drives the panel's picker (a 2-8 row + // gallery list rather than the single upload/history pair). + referenceKind: 'image', + }, ]; export const IC_LORA_MODE_VALUES = IC_LORA_MODES.map((m) => m.mode); diff --git a/client/src/lib/videoGenParams.test.js b/client/src/lib/videoGenParams.test.js index c98a7b4f78..5867480903 100644 --- a/client/src/lib/videoGenParams.test.js +++ b/client/src/lib/videoGenParams.test.js @@ -3,6 +3,7 @@ import { videoModelMemoryGb, computeFflfSafeFrames, isModelAllowedForMode, VIDEO_EDGE_BOUNDS, FRAME_OPTIONS, FPS_OPTIONS, IC_LORA_MODES, IC_LORA_MODE_VALUES, isIcLoraMode, icLoraSpecForMode, + icResolutionIssue, } from './videoGenParams.js'; describe('videoModelMemoryGb', () => { @@ -69,7 +70,7 @@ describe('constants', () => { describe('IC-LoRA remix modes (#3100)', () => { it('mirrors the server registry shape', () => { - expect(IC_LORA_MODE_VALUES).toEqual(['ic-control', 'ic-colorize']); + expect(IC_LORA_MODE_VALUES).toEqual(['ic-control', 'ic-colorize', 'ic-ingredients']); for (const spec of IC_LORA_MODES) { // The `ic-` prefix drives the download-id router in useModelDownloadStatus. expect(spec.mode.startsWith('ic-')).toBe(true); @@ -78,8 +79,23 @@ describe('IC-LoRA remix modes (#3100)', () => { // The panel renders these two directly — an empty one ships blank copy. expect(spec.uploadLabel).toBeTruthy(); expect(spec.description).toBeTruthy(); + // Drives which input surface the panel renders (single clip vs the 2-8 + // gallery row list), so an unrecognized value would render nothing. + expect(['video', 'image']).toContain(spec.referenceKind); } }); + + it('mirrors the Ingredients bounds + image kind (#3112)', () => { + // The 2-8 count is the weight's contract, mirrored here so the form blocks + // before a POST the route would reject; the parity test in + // server/lib/icLoraWeights.parity.test.js is what keeps the two in step. + const ing = icLoraSpecForMode('ic-ingredients'); + expect(ing.minReferences).toBe(2); + expect(ing.maxReferences).toBe(8); + expect(ing.referenceKind).toBe('image'); + // Factor 1 → no divisibility rule at all, so an odd resolution is legal. + expect(icResolutionIssue(ing, 705, 449)).toBeNull(); + }); it('identifies IC modes', () => { expect(isIcLoraMode('ic-control')).toBe(true); expect(isIcLoraMode('ic-colorize')).toBe(true); diff --git a/client/src/pages/VideoGen.jsx b/client/src/pages/VideoGen.jsx index f90e6cd73e..df9cab13d5 100644 --- a/client/src/pages/VideoGen.jsx +++ b/client/src/pages/VideoGen.jsx @@ -196,6 +196,11 @@ export default function VideoGen() { // both. `icStrength` weights the reference conditioning channel. const [icReferenceFile, setIcReferenceFile] = useState(null); const [icReferenceVideoId, setIcReferenceVideoId] = useState(''); + // Image-kind IC references (Ingredients, #3112) — 2-8 gallery basenames rather + // than the single clip above. Gallery-only, mirroring the route: the reference + // list is a separate submit field (icReferenceImageFiles) so a clip can never + // ride into an image-kind weight (which would silently produce garbage). + const [icReferenceImageFiles, setIcReferenceImageFiles] = useState([]); const [icStrength, setIcStrength] = useState(1.0); const [icSkipStage2, setIcSkipStage2] = useState(false); // Display-only: the reference clip name(s) of an IN-FLIGHT render restored via @@ -516,6 +521,7 @@ export default function VideoGen() { // would submit unknowingly. The dials DO round-trip. setIcReferenceFile(null); setIcReferenceVideoId(''); + setIcReferenceImageFiles([]); if (typeof item.icStrength === 'number') setIcStrength(item.icStrength); setIcSkipStage2(item.icSkipStage2 === true); // Restore the LoRA picker from the render record. `item` here is the RAW @@ -674,6 +680,12 @@ export default function VideoGen() { if (Array.isArray(p.icReferenceNames) && p.icReferenceNames.length) { setIcReferenceNames(p.icReferenceNames); } + // Unlike a clip, an image-kind reference IS re-derivable: it's a gallery + // basename, which is exactly the submit shape. So the resumed form + // repopulates its picker and the submit gate unblocks without a re-pick. + if (Array.isArray(p.icReferenceImageFiles) && p.icReferenceImageFiles.length) { + setIcReferenceImageFiles(p.icReferenceImageFiles); + } // Restore the LoRA picker — params carry { filename, scale } basenames; // resolve the display name from the loaded library (falls back to the // filename if the library hasn't loaded yet or the LoRA was deleted). @@ -826,6 +838,23 @@ export default function VideoGen() { // family, so every consumer gates on `icModeActive` first. const icSpec = icLoraSpecForMode(mode); const icModeActive = !!icSpec; + // Which input surface this weight wants — `image` swaps the single clip + // upload/history pair for the 2-8 gallery row list. + const icImageKind = icSpec?.referenceKind === 'image'; + // Pad the row list up to the weight's MINIMUM whenever an image-kind mode is + // active. Without rows the panel renders an empty list with nothing to fill, + // and the panel's remove button floors at min so it can't get back down. + // Derived from the registry (never a hardcoded 2) and driven from an effect so + // every entry path is covered — the mode bar, a ?mode= deep link, and an + // /active resume — not just handleModeChange. + useEffect(() => { + if (!icImageKind) return; + setIcReferenceImageFiles((prev) => ( + prev.length >= icSpec.minReferences + ? prev + : [...prev, ...Array.from({ length: icSpec.minReferences - prev.length }, () => '')] + )); + }, [icImageKind, icSpec?.minReferences]); // The worker clamps FFLF/ltx2 numFrames down to fit a pixel-frame budget that // depends on resolution, so at default 768×512 the real frame ceiling is far // below numFrames. Compute the same cap the server enforces so the picker can @@ -1027,6 +1056,10 @@ export default function VideoGen() { setIcReferenceFile(null); setIcReferenceVideoId(''); setIcReferenceNames([]); + // Just clear — the pad-to-minimum effect below re-seeds empty rows. Doing + // it there rather than here covers every path that lands on an IC mode + // (mode bar, ?mode= deep link, an /active resume), not only this handler. + setIcReferenceImageFiles([]); } return; } @@ -1208,8 +1241,12 @@ export default function VideoGen() { // IC-LoRA remix: the reference clip rides as either the 'icReference' // multipart upload OR an icReferenceVideoIds history id — never both (the // route rejects that with IC_LORA_REFERENCE_CONFLICT). - icReference: icModeActive ? (icReferenceFile || '') : '', - icReferenceVideoIds: (icModeActive && !icReferenceFile) ? (icReferenceVideoId || '') : '', + // Image-kind weights take gallery stills on their own field; the clip + // fields stay empty for them (the route rejects mixing the two kinds with + // IC_LORA_REFERENCE_KIND_MISMATCH). + icReference: (icModeActive && !icImageKind) ? (icReferenceFile || '') : '', + icReferenceVideoIds: (icModeActive && !icImageKind && !icReferenceFile) ? (icReferenceVideoId || '') : '', + icReferenceImageFiles: icImageKind ? icReferenceImageFiles.filter(Boolean) : undefined, icStrength: icModeActive ? icStrength : '', icSkipStage2: icModeActive && icSkipStage2 ? 'true' : '', // Keyframes and IC references each anchor a single clip — the route @@ -1305,8 +1342,14 @@ export default function VideoGen() { // resolution must divide by the weight's reference-downscale factor (the // server rejects otherwise). Block submit for all three so the request fails // the form rather than the worker. + // An image-kind weight is satisfied by min..max FILLED gallery rows (blank rows + // are dropped from the payload, so an unfilled row must block rather than + // silently submit a short list the route would 400). + const icFilledImageRefs = icReferenceImageFiles.filter(Boolean).length; const icLoraModeBlocked = icModeActive && ( - (!icReferenceFile && !icReferenceVideoId) + (icImageKind + ? (icFilledImageRefs < icSpec.minReferences || icFilledImageRefs > icSpec.maxReferences) + : (!icReferenceFile && !icReferenceVideoId)) || currentModel?.runtime !== 'ltx2' || !!icResolutionIssue(icSpec, width, height) ); @@ -1673,6 +1716,17 @@ export default function VideoGen() { referenceVideoId={icReferenceVideoId} inFlightReferenceNames={icReferenceNames} visibleHistory={visibleHistory} + referenceImageFiles={icReferenceImageFiles} + visibleGallery={visibleGallery} + onAddReferenceImage={() => setIcReferenceImageFiles((prev) => ( + prev.length >= icSpec.maxReferences ? prev : [...prev, ''] + ))} + onUpdateReferenceImage={(i, file) => setIcReferenceImageFiles((prev) => ( + prev.map((f, idx) => (idx === i ? file : f)) + ))} + onRemoveReferenceImage={(i) => setIcReferenceImageFiles((prev) => ( + prev.length <= icSpec.minReferences ? prev : prev.filter((_, idx) => idx !== i) + ))} icStrength={icStrength} icSkipStage2={icSkipStage2} width={width} diff --git a/scripts/generate_ltx2.py b/scripts/generate_ltx2.py index 62c6d6ded3..ab95ab883a 100644 --- a/scripts/generate_ltx2.py +++ b/scripts/generate_ltx2.py @@ -838,6 +838,12 @@ def run_ic_lora(args: argparse.Namespace) -> str: IC-LoRA rides `lora_paths` (fused by _fuse_loras before Stage 1) while user LoRAs ride the separate `_pending_loras` hook, so an Ingredients x Character stack composes. + - Ingredients is MULTI-reference (2-8) and conditions on stills, but the + reference channel is a video encoder end-to-end: iclora_utils probes each + reference with ffprobe and feeds it to the video VAE, whose reshape needs + a (1 + 8k)-frame input. PortOS therefore materializes each still into a + tiny 9-frame constant clip before invoking this helper, so `--ic-reference` + is uniformly a video path regardless of the weight's reference kind. """ ICLoraPipeline = _resolve_pipeline("ICLoraPipeline") ic_mode = args.ic_mode or "control" diff --git a/scripts/generate_ltx2_ic_lora_test.py b/scripts/generate_ltx2_ic_lora_test.py new file mode 100644 index 0000000000..57764f28df --- /dev/null +++ b/scripts/generate_ltx2_ic_lora_test.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +"""Unit tests for generate_ltx2's IC-LoRA reference-count contract (#3112). + +The reference count is a WEIGHT contract, not a preference: a weight fed the +wrong number of references produces plausible-looking garbage rather than an +error inside the pipeline. PortOS enforces it at three layers — the route, the +`icLoraArgs` arg builder, and here — and the bounds themselves come from ONE +place (`server/lib/icLoraWeights.js`), passed in as +`--ic-min-references` / `--ic-max-references` rather than duplicated as a second +table in Python. These tests cover the Python layer: that it honors whatever +bounds it is handed (so a direct/script caller is still guarded), and that it +rejects nonsense bounds instead of trusting them. + +Run: python3 scripts/generate_ltx2_ic_lora_test.py +""" +from __future__ import annotations + +import importlib.util +import sys +import tempfile +import types +import unittest +from pathlib import Path +from types import SimpleNamespace + + +HELPER_PATH = Path(__file__).with_name("generate_ltx2.py") + + +class RunIcLoraBoundsTest(unittest.TestCase): + """`run_ic_lora` argument validation, with the pipeline itself faked out.""" + + def setUp(self): + self.module_name = "generate_ltx2_ic_under_test" + self.original_modules = { + name: sys.modules.get(name) + for name in [ + self.module_name, + "ltx_pipelines_mlx", + "ltx_pipelines_mlx.ic_lora", + "ltx_pipelines_mlx.extend", + "ltx_pipelines_mlx.a2vid_two_stage", + "ltx_pipelines_mlx.ti2vid_two_stages", + ] + } + for name in self.original_modules: + sys.modules.pop(name, None) + + # Minimal fake of the runtime the helper imports at module load. The + # TeaCache patches walk these, so they must exist even though these tests + # never reach a real generate call. + root = types.ModuleType("ltx_pipelines_mlx") + root.__path__ = [] + extend = types.ModuleType("ltx_pipelines_mlx.extend") + a2v = types.ModuleType("ltx_pipelines_mlx.a2vid_two_stage") + two_stages = types.ModuleType("ltx_pipelines_mlx.ti2vid_two_stages") + extend.guided_denoise_loop = lambda *a, **k: None + a2v.guided_denoise_loop = lambda *a, **k: None + two_stages._build_teacache_controller = lambda num_steps, thresh: None + + # Records the constructor + generate kwargs so the tests can assert the + # LoRA-channel split without a GPU. + recorded = self.recorded = {} + + class FakeICLoraPipeline: + def __init__(self, **kwargs): + recorded["init"] = kwargs + + # `bind_output_fps` rebinds this to inject the output frame rate, so + # it has to exist even though nothing decodes here. + def _decode_and_save_video(self, video_latent, audio_latent, output_path, **kwargs): + return output_path + + def generate_and_save(self, **kwargs): + recorded["generate"] = kwargs + # Captured at generate time (not construction): the real pipeline + # reads _pending_loras when it lazily loads the DiT inside this + # call, so that is when both channels must be populated. + recorded["pending_loras"] = getattr(self, "_pending_loras", None) + return kwargs.get("output_path") + + ic_lora = types.ModuleType("ltx_pipelines_mlx.ic_lora") + ic_lora.ICLoraPipeline = FakeICLoraPipeline + # `_resolve_pipeline` reads the class off the PACKAGE root (upstream + # re-exports it there), so the attribute must live on `root` — a + # submodule alone isn't what the helper looks at. + root.ICLoraPipeline = FakeICLoraPipeline + + sys.modules["ltx_pipelines_mlx"] = root + sys.modules["ltx_pipelines_mlx.ic_lora"] = ic_lora + sys.modules["ltx_pipelines_mlx.extend"] = extend + sys.modules["ltx_pipelines_mlx.a2vid_two_stage"] = a2v + sys.modules["ltx_pipelines_mlx.ti2vid_two_stages"] = two_stages + + spec = importlib.util.spec_from_file_location(self.module_name, HELPER_PATH) + self.helper = importlib.util.module_from_spec(spec) + sys.modules[self.module_name] = self.helper + spec.loader.exec_module(self.helper) + + # References must exist on disk (the helper checks), so back them with + # real temp files. + self.tmp = tempfile.TemporaryDirectory() + self.refs = [] + for i in range(9): + p = Path(self.tmp.name) / f"ref-{i}.mp4" + p.write_bytes(b"\x00") + self.refs.append(str(p)) + self.weight = str(Path(self.tmp.name) / "ingredients.safetensors") + Path(self.weight).write_bytes(b"\x00") + + def tearDown(self): + self.tmp.cleanup() + for name, module in self.original_modules.items(): + if module is None: + sys.modules.pop(name, None) + else: + sys.modules[name] = module + + def _args(self, *, references, lo=2, hi=8, ic_mode="ingredients", user_loras=None): + return SimpleNamespace( + ic_mode=ic_mode, + ic_lora_path=self.weight, + ic_reference=list(references), + ic_min_references=lo, + ic_max_references=hi, + ic_strength=1.0, + ic_attention_strength=None, + ic_skip_stage_2=False, + user_lora_specs=user_loras, + model="/fake/model", + gemma="google/gemma-3-1b-it", + prompt="the owl greets the camera", + output=str(Path(self.tmp.name) / "out.mp4"), + height=448, + width=704, + num_frames=25, + seed=1, + steps=30, + stage2_steps=None, + fps=24, + ) + + def test_accepts_every_count_inside_the_2_8_range(self): + for n in (2, 5, 8): + with self.subTest(n=n): + self.helper.run_ic_lora(self._args(references=self.refs[:n])) + self.assertEqual( + len(self.recorded["generate"]["video_conditioning"]), n + ) + + def test_rejects_a_count_below_the_minimum(self): + for n in (0, 1): + with self.subTest(n=n): + with self.assertRaises(SystemExit) as ctx: + self.helper.run_ic_lora(self._args(references=self.refs[:n])) + self.assertIn("needs 2-8", str(ctx.exception)) + + def test_rejects_a_count_above_the_maximum(self): + with self.assertRaises(SystemExit) as ctx: + self.helper.run_ic_lora(self._args(references=self.refs[:9])) + self.assertIn("needs 2-8", str(ctx.exception)) + + def test_phrases_a_single_reference_weight_as_exactly_one(self): + # Control/Colorize pass 1/1. The message must read "exactly 1", not + # "1-1" — it's the same phrasing the JS side produces. + with self.assertRaises(SystemExit) as ctx: + self.helper.run_ic_lora( + self._args(references=self.refs[:2], lo=1, hi=1, ic_mode="control") + ) + self.assertIn("exactly 1", str(ctx.exception)) + + def test_rejects_nonsense_bounds_rather_than_trusting_them(self): + # A caller passing min > max (or min < 1) is a bug in the caller; silently + # honoring it would let ANY count through or reject every count. + with self.assertRaises(SystemExit) as ctx: + self.helper.run_ic_lora(self._args(references=self.refs[:2], lo=8, hi=2)) + self.assertIn("1 <= min <= max", str(ctx.exception)) + with self.assertRaises(SystemExit) as ctx: + self.helper.run_ic_lora(self._args(references=self.refs[:2], lo=0, hi=8)) + self.assertIn("1 <= min <= max", str(ctx.exception)) + + def test_rejects_a_reference_missing_from_disk(self): + with self.assertRaises(SystemExit) as ctx: + self.helper.run_ic_lora( + self._args(references=[self.refs[0], "/nope/missing.mp4"]) + ) + self.assertIn("does not exist", str(ctx.exception)) + + def test_ic_weight_rides_lora_paths_so_user_loras_stack(self): + # The payoff of the Phase 1 channel split: the IC weight is fused via the + # constructor's `lora_paths` (pre-Stage-1 `_fuse_loras`) while user LoRAs + # go through `_pending_loras`, so an Ingredients x Character stack + # COMPOSES rather than one displacing the other. + user = [("/fake/character.safetensors", 0.9)] + args = self._args(references=self.refs[:2], user_loras=user) + self.helper.run_ic_lora(args) + # IC weight → constructor `lora_paths` (fused pre-Stage-1 at strength 1.0: + # the weight IS the mode, not a stylistic dial). + self.assertEqual(self.recorded["init"]["lora_paths"], [(self.weight, 1.0)]) + # User LoRAs → the SEPARATE `_pending_loras` hook. Both channels populated + # at once is the assertion that matters; one list would mean replacement. + self.assertEqual(self.recorded["pending_loras"], user) + + def test_every_reference_carries_the_strength_dial(self): + args = self._args(references=self.refs[:3]) + args.ic_strength = 0.6 + self.helper.run_ic_lora(args) + self.assertEqual( + self.recorded["generate"]["video_conditioning"], + [(r, 0.6) for r in self.refs[:3]], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/hf_download_repo.py b/scripts/hf_download_repo.py index 7e0858e81b..71ed5bdfbb 100755 --- a/scripts/hf_download_repo.py +++ b/scripts/hf_download_repo.py @@ -10,6 +10,12 @@ the id to an HF repo before invoking this helper), plus `GET /api/video-gen/text-encoder/download` for the Gemma encoder. +`--only ` (repeatable) switches to SINGLE-FILE mode: the repo is +never enumerated and exactly the named files are fetched. That is the only +safe way to pull one weight out of a multi-hundred-GB aggregate repo (e.g. the +`DeepBeepMeep/LTX-2` mirror, ~708 GB) — a snapshot of that would fill the +user's disk. Prefer it whenever the caller already knows the exact filename. + Wire protocol (matches the STAGE:/DOWNLOAD: convention the rest of the image-gen runners use, so the existing SSE bridge picks it up unchanged): @@ -60,44 +66,63 @@ def main() -> int: parser.add_argument("--token-env", default=None, help="Env var name to read the HF token from (e.g. HF_TOKEN).") parser.add_argument("--local-dir", default=None, help="If set, materialize the repo as a flat copy at this dir instead of relying on the standard HF cache symlinks (used by BYOV installers like HiDream-O1 that need a real on-disk repo).") parser.add_argument("--ignore", action="append", default=[], help="Glob pattern (fnmatch) to skip from the file list. Repeat for multiple patterns. e.g. --ignore 'scripts/**' --ignore 'docs/**' to skip non-weight subdirs.") + parser.add_argument("--only", action="append", default=[], metavar="FILENAME", + help="Fetch ONLY this exact repo-relative filename, skipping repo enumeration " + "entirely. Repeat for several files. Required for aggregate repos where a " + "snapshot would be catastrophic (e.g. the ~708 GB DeepBeepMeep/LTX-2 mirror). " + "Mutually exclusive with --ignore, which only filters an enumerated list.") args = parser.parse_args() + if args.only and args.ignore: + print("USER_ERROR:bad_args:only_with_ignore", file=sys.stderr, flush=True) + print("❌ --only and --ignore are mutually exclusive (--only never enumerates the repo).", file=sys.stderr, flush=True) + return 2 + token = None if args.token_env: token = os.environ.get(args.token_env) or None # huggingface_hub also reads HF_TOKEN itself, but being explicit lets # the caller scope which env var the child trusts. - api = HfApi() - print("STAGE:list", file=sys.stderr, flush=True) - try: - files = api.list_repo_files(args.repo, revision=args.revision, token=token) - except GatedRepoError: - print(f"USER_ERROR:gated_repo:{args.repo}", file=sys.stderr, flush=True) - print(f"❌ Access to {args.repo} is gated. Accept the license at https://huggingface.co/{args.repo} and paste your HuggingFace token into Image Gen settings, then retry.", file=sys.stderr, flush=True) - return 2 - except RepositoryNotFoundError: - print(f"USER_ERROR:repo_not_found:{args.repo}", file=sys.stderr, flush=True) - print(f"❌ Repository {args.repo} not found on HuggingFace.", file=sys.stderr, flush=True) - return 2 - except Exception as err: # noqa: BLE001 - # Anything that smells like 401 from list_repo_files — surface it - # as token-rejected so the UI can prompt for a new HF_TOKEN. - if "401" in str(err) or "Unauthorized" in str(err): - print(f"USER_ERROR:hf_unauthorized:{args.repo}", file=sys.stderr, flush=True) - print(f"❌ HuggingFace rejected the token. Update HF_TOKEN in Image Gen settings.", file=sys.stderr, flush=True) + if args.only: + # SINGLE-FILE mode. Deliberately skips `list_repo_files` — enumerating a + # 125-file/708 GB aggregate repo is wasted work, and more importantly the + # absence of a list means no code path downstream can widen this into a + # whole-repo pull. A typo'd filename surfaces as download_failed (HF 404) + # rather than being silently dropped from a filtered list. + files = list(dict.fromkeys(args.only)) + else: + api = HfApi() + print("STAGE:list", file=sys.stderr, flush=True) + try: + files = api.list_repo_files(args.repo, revision=args.revision, token=token) + except GatedRepoError: + print(f"USER_ERROR:gated_repo:{args.repo}", file=sys.stderr, flush=True) + print(f"❌ Access to {args.repo} is gated. Accept the license at https://huggingface.co/{args.repo} and paste your HuggingFace token into Image Gen settings, then retry.", file=sys.stderr, flush=True) return 2 - print(f"USER_ERROR:list_failed:{args.repo}", file=sys.stderr, flush=True) - print(f"❌ Failed to list {args.repo}: {err}", file=sys.stderr, flush=True) - return 2 + except RepositoryNotFoundError: + print(f"USER_ERROR:repo_not_found:{args.repo}", file=sys.stderr, flush=True) + print(f"❌ Repository {args.repo} not found on HuggingFace.", file=sys.stderr, flush=True) + return 2 + except Exception as err: # noqa: BLE001 + # Anything that smells like 401 from list_repo_files — surface it + # as token-rejected so the UI can prompt for a new HF_TOKEN. + if "401" in str(err) or "Unauthorized" in str(err): + print(f"USER_ERROR:hf_unauthorized:{args.repo}", file=sys.stderr, flush=True) + print(f"❌ HuggingFace rejected the token. Update HF_TOKEN in Image Gen settings.", file=sys.stderr, flush=True) + return 2 + print(f"USER_ERROR:list_failed:{args.repo}", file=sys.stderr, flush=True) + print(f"❌ Failed to list {args.repo}: {err}", file=sys.stderr, flush=True) + return 2 + + # Skip the few HF housekeeping files that are not actually downloadable + # as part of a snapshot (`.gitattributes` is, but `LICENSE` and similar + # are — we keep them; the only true skip is the `.huggingface` folder). + files = [f for f in files if not f.startswith(".huggingface/")] + if args.ignore: + import fnmatch + files = [f for f in files if not any(fnmatch.fnmatch(f, pat) for pat in args.ignore)] - # Skip the few HF housekeeping files that are not actually downloadable - # as part of a snapshot (`.gitattributes` is, but `LICENSE` and similar - # are — we keep them; the only true skip is the `.huggingface` folder). - files = [f for f in files if not f.startswith(".huggingface/")] - if args.ignore: - import fnmatch - files = [f for f in files if not any(fnmatch.fnmatch(f, pat) for pat in args.ignore)] total = len(files) if total == 0: print(f"USER_ERROR:repo_empty:{args.repo}", file=sys.stderr, flush=True) diff --git a/server/lib/README.md b/server/lib/README.md index 0059e28b87..7dc4533355 100644 --- a/server/lib/README.md +++ b/server/lib/README.md @@ -115,9 +115,9 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub | `ansiStrip.js` | Streaming ANSI / control-byte stripper. | | `hfToken.js` | HuggingFace token resolution (settings > env > CLI): `getHfToken` / `getHfTokenInfo` / `hfTokenEnv`, plus `hfChildEnv(extra?)` for a complete Malloc-stripped child environment that preserves inherited variables while injecting the resolved token. | | `hfErrors.js` | Parse huggingface_hub gated-access errors: `isGatedRepoError(text)` classifies a failure as gated (403/restricted), `extractGatedRepo(text)` → `owner/name` (or null) for the UI's license deep-link. Shared by the image runner and MIDI transcription (LoRA trainer keeps its own regex to also match bare 401). Pure. | -| `hfCache.js` | HuggingFace Hub cache inspection (`inspectModelCache(repoId)` → `{cached,sizeBytes,snapshotPath}`, `isModelCached`, `getHfCacheRoot`). Drives the inline "Available / Download" badge on the image + video gen forms. Also `verifyModelCache(repoId,{deep})` (structural safetensors-header + optional sha256 integrity check) and `repairModelCache(repoId,{deep})` (delete corrupt weight files so the download path re-fetches them) — power the "Repair model" banner. `verifySafetensorsStructure(path,size)` is the reusable header/size structural check (reads only the header region) — also used by the Civitai/HF LoRA install path to reject truncated downloads. | +| `hfCache.js` | HuggingFace Hub cache inspection (`inspectModelCache(repoId)` → `{cached,sizeBytes,snapshotPath}`, `isModelCached`, `getHfCacheRoot`). Drives the inline "Available / Download" badge on the image + video gen forms. Also `verifyModelCache(repoId,{deep})` (structural safetensors-header + optional sha256 integrity check) and `repairModelCache(repoId,{deep})` (delete corrupt weight files so the download path re-fetches them) — power the "Repair model" banner. `repairCachedFile(path)` is the single-file counterpart (unlinks one snapshot entry AND the blob behind its symlink) for a weight inside an aggregate repo, where a whole-snapshot walk would be prohibitive. `verifySafetensorsStructure(path,size)` is the reusable header/size structural check (reads only the header region) — also used by the Civitai/HF LoRA install path to reject truncated downloads. | | `hfDownload.js` | `downloadHfRepo({repo,onEvent})` returning `{promise,kill}` — spawns `scripts/hf_download_repo.py` in the FLUX.2 venv (fallback: mflux pythonPath) and emits SSE-friendly stage/progress/complete events. Powers the inline "Download" button next to the model picker. | -| `icLoraWeights.js` | IC-LoRA weight registry for the local LTX-2 remix modes (`IC_LORA_MODES`, `IC_LORA_MODE_VALUES`, `listIcLoraWeights()`, `isIcLoraMode(mode)`, `icLoraSpecForMode(mode)`, `icLoraRepos()`, `resolveIcLoraWeight(mode)` → `{path,cached,spec}`). Single source of truth mapping each remix mode (`ic-control`, `ic-colorize`) → its HF repo/filename, reference-count rule, per-weight `referenceDownscaleFactor` (read from the weight's safetensors metadata, never assumed), and input surface. Also owns the two rules derived from that data: `assertIcReferenceCount(spec,count,fail)` / `describeIcReferenceRange(spec)` and `icResolutionIssue(spec,w,h)` (the output-resolution divisibility its reference encoder imposes). Mirrored to `client/src/lib/videoGenParams.js`; `icLoraWeights.parity.test.js` diffs the two. | +| `icLoraWeights.js` | IC-LoRA weight registry for the local LTX-2 remix modes (`IC_LORA_MODES`, `IC_LORA_MODE_VALUES`, `listIcLoraWeights()`, `isIcLoraMode(mode)`, `icLoraSpecForMode(mode)`, `icLoraRepos()`, `icLoraWeightCandidates(spec)`, `findCachedIcLoraWeight(spec)`, `resolveIcLoraWeight(mode)` → `{path,cached,spec}`). Single source of truth mapping each remix mode (`ic-control`, `ic-colorize`, `ic-ingredients`) → its HF repo/filename, reference-count rule, per-weight `referenceDownscaleFactor` (read from the weight's safetensors metadata, never assumed), and input surface (`referenceKind: 'video' | 'image'`). A `requiresPreDownload` spec (Ingredients) suppresses `resolveIcLoraWeight`'s bare-repo-id fallback — the pipeline would `snapshot_download` it, and its candidates are a gated first-party repo plus a ~708 GB aggregate mirror, so the weight must be pre-fetched single-file instead. Also owns the two rules derived from that data: `assertIcReferenceCount(spec,count,fail)` / `describeIcReferenceRange(spec)` and `icResolutionIssue(spec,w,h)` (the output-resolution divisibility its reference encoder imposes). Mirrored to `client/src/lib/videoGenParams.js`; `icLoraWeights.parity.test.js` diffs the two. | | `sseHeaders.js` | `SSE_HEADERS` — the canonical SSE response headers (incl. `X-Accel-Buffering:no`) in a dependency-free module so any producer (`sseDownload.js`, `sseUtils.js`) can share them without pulling in a heavier module's transitive imports. | | `sseDownload.js` | `startHfDownloadStream({req,res,repo,alreadyDownloadedMessage})`, `openSseStream(res)` (`{send,safeEnd}` SSE boilerplate; uses `SSE_HEADERS` from `sseHeaders.js`) — shared SSE driver used by both image and video gen `/models/:id/download` routes. Owns the cross-route in-flight Map so a double-click (or both pages running) can't spawn two python children against the same repo. | | `installLogger.js` | `createInstallLogger({installer,target})` → `{start,onEvent,success,failure,cancel}` — server-console chokepoint for install/venv-setup SSE flows (the install-side analogue of `sseDownload.js`). Logs a START line, throttled heartbeats + `stage` milestones (not every raw pip/bash line), and the OUTCOME (success/fail/cancel) with elapsed time. `onEvent(ev)` auto-detects terminal `complete`/`error` SSE events. Used by the music/video runtime installers and the FLUX.2 venv bootstrap. | diff --git a/server/lib/hfCache.js b/server/lib/hfCache.js index efb5e3df2d..aba16b0755 100644 --- a/server/lib/hfCache.js +++ b/server/lib/hfCache.js @@ -327,6 +327,27 @@ export async function repairModelCache(repoId, { deep = false } = {}) { return { repoId, status: 'bad', deleted }; } +// Delete ONE cached file (plus the blob behind its symlink) so the resumable HF +// fetch re-downloads it. The single-file counterpart to repairModelCache, for a +// weight that lives inside an aggregate repo: repairModelCache walks the entire +// snapshot, which against a ~700 GB mirror means stat-ing (and under `deep`, +// hashing) every unrelated weight the user happens to have. Unlinking the blob as +// well as the snapshot link is essential — `hf_hub_download` keys on the cached +// etag, not the content, so a stale blob with the right name is trusted and never +// re-fetched (same reasoning as repairModelCache). Returns true when something +// was removed. +export async function repairCachedFile(path) { + if (!path || typeof path !== 'string') return false; + const lst = await fs.lstat(path).catch(() => null); + if (!lst) return false; + if (lst.isSymbolicLink()) { + const target = await fs.readlink(path).catch(() => null); + if (target) await fs.unlink(resolvePath(dirname(path), target)).catch(() => {}); + } + await fs.unlink(path).catch(() => {}); + return true; +} + // Condense a verifyModelCache() result to the UI-facing shape `{ status, // checkedDeep, badFiles: [{ name, reason }] }`. Drops the internal file paths // and per-tensor details — the banner only needs which files are bad and why. diff --git a/server/lib/hfDownload.js b/server/lib/hfDownload.js index a91220a062..c8fd8a3dd2 100644 --- a/server/lib/hfDownload.js +++ b/server/lib/hfDownload.js @@ -56,7 +56,13 @@ export async function resolveHfDownloadPython() { // Returns `{ promise, kill }`. The promise resolves with `{ ok, sizeBytes, // errorKind, errorMessage }`. `kill()` SIGTERMs the python child so the // SSE handler can stop the download when the EventSource client closes. -export function downloadHfRepo({ repo, revision = null, onEvent }) { +// +// `only` is an array of exact repo-relative filenames. When set, the helper +// runs in SINGLE-FILE mode: it never enumerates the repo and fetches only those +// files. This is MANDATORY for aggregate repos — `DeepBeepMeep/LTX-2` mirrors +// every LTX weight in one ~708 GB repo, so a snapshot would fill the user's +// disk to pull one 1.3 GB IC-LoRA (see server/lib/icLoraWeights.js). +export function downloadHfRepo({ repo, revision = null, only = null, onEvent }) { let proc = null; let killed = false; let errorKind = null; @@ -88,8 +94,16 @@ export function downloadHfRepo({ repo, revision = null, onEvent }) { const args = [HELPER_SCRIPT, '--repo', repo, '--token-env', 'HF_TOKEN']; if (revision) args.push('--revision', revision); + const onlyFiles = Array.isArray(only) ? only.filter((f) => typeof f === 'string' && f.length > 0) : []; + for (const f of onlyFiles) args.push('--only', f); - onEvent({ type: 'stage', stage: 'starting', message: `Downloading ${repo}…` }); + onEvent({ + type: 'stage', + stage: 'starting', + message: onlyFiles.length + ? `Downloading ${onlyFiles.join(', ')} from ${repo}…` + : `Downloading ${repo}…`, + }); return new Promise((resolve) => { proc = spawn(pythonPath, args, { env, stdio: ['ignore', 'pipe', 'pipe'] }); diff --git a/server/lib/icLoraWeights.js b/server/lib/icLoraWeights.js index 49b8e77b44..6199669b26 100644 --- a/server/lib/icLoraWeights.js +++ b/server/lib/icLoraWeights.js @@ -16,7 +16,17 @@ // still works if the user skipped the explicit pre-download (it just stalls // silently on the pull instead of showing progress). // -// All registered weights are un-gated (no HF token required). +// The repo-id fallback is NOT universally safe: `_resolve_lora_path` implements +// it as `snapshot_download(repo_id)`, which pulls the ENTIRE repo. That's fine +// for a single-weight repo (Control, Colorize) and catastrophic for an aggregate +// mirror — `DeepBeepMeep/LTX-2` carries every LTX weight in one ~708 GB repo. A +// spec whose chain includes such a repo sets `requiresPreDownload`, and +// resolveIcLoraWeight then refuses to emit a bare repo id at all (see below). +// +// Gating: Control and Colorize are un-gated. Ingredients' official Lightricks +// repo is gated (`gated: "auto"` — an anonymous resolve returns 401 GatedRepo), +// so `gated: true` marks it and the mirror provides an un-gated path for users +// without an HF token. import { join } from 'node:path'; import { existsSync } from 'node:fs'; @@ -79,6 +89,44 @@ export const IC_LORA_MODES = Object.freeze({ maxReferences: 1, referenceKind: 'video', }), + ingredients: Object.freeze({ + id: 'ingredients', + mode: 'ic-ingredients', + label: 'Ingredients', + description: 'A scene recomposed from 2-8 reference stills (characters, props, settings)', + uploadLabel: 'Upload a reference still (character / prop / setting)', + repo: 'Lightricks/LTX-2.3-22b-IC-LoRA-Ingredients', + filename: 'ltx-2.3-22b-ic-lora-ingredients-0.9.safetensors', + // 1_308_778_338 bytes, read from the HF `x-linked-size` header — not a guess. + sizeBytes: 1_308_778_338, + // Read from the weight's safetensors `__metadata__.reference_downscale_factor` + // (fetched with an HTTP Range request over the header region, so no 1.3 GB + // download was needed to confirm it): "1". Like the Colorizer this weight + // conditions on FULL-resolution references and imposes no divisibility rule. + referenceDownscaleFactor: 1, + // The weight's contract: 2-8 reference stills. A wrong count yields + // plausible-looking garbage rather than an error, so it's enforced at every + // layer (route, icLoraArgs, and the Python helper via --ic-min/max-references). + minReferences: 2, + maxReferences: 8, + // Images, not clips — Ingredients recomposes a scene from stills. Drives the + // panel's `accept` filter and the route's gallery-image resolution. + referenceKind: 'image', + // The official Lightricks repo is gated (accept the license + supply an HF + // token), so surface that in the UI instead of letting the download fail + // with a bare 401. The mirror below is the un-gated path. + gated: true, + // Un-gated fallback for users without an HF token. This repo is the ~708 GB + // `DeepBeepMeep/LTX-2` aggregate mirror, so it may ONLY ever be fetched + // single-file (`--only `) — see requiresPreDownload. + mirrorRepo: 'DeepBeepMeep/LTX-2', + mirrorFilename: 'ltx-2.3-22b-ic-lora-ingredients-0.9.safetensors', + // Suppresses resolveIcLoraWeight's bare-repo-id fallback. Handing either + // repo id to ICLoraPipeline._resolve_lora_path would `snapshot_download` it: + // the official one is gated (401 mid-render) and the mirror is 708 GB. The + // user must pre-download the weight through PortOS' single-file path first. + requiresPreDownload: true, + }), }); // Every registered spec, in declaration order. Consumers use this instead of @@ -102,9 +150,30 @@ export const icLoraSpecForMode = (mode) => { return IC_LORA_MODES[id] || null; }; -// Every IC-LoRA HF repo, for the integrity-scan / status surface. +// Every IC-LoRA HF repo, for the integrity-scan / status surface. The mirror +// repos are deliberately EXCLUDED: an unscoped integrity scan walks each repo's +// whole snapshot, and for the 708 GB aggregate mirror that means stat-ing (and +// under `deep`, hashing) every unrelated weight the user happens to have. The +// weight we care about there is verified via icLoraWeightCandidates instead. export const icLoraRepos = () => listIcLoraWeights().map((m) => m.repo); +// Every (repo, filename) pair a spec's weight can legitimately come from, in +// preference order: the official repo first, then the un-gated mirror. Shared by +// the cache probe and the download endpoint so "where does this weight live?" has +// exactly one answer. +export const icLoraWeightCandidates = (spec) => { + if (!spec) return []; + const candidates = [{ repo: spec.repo, filename: spec.filename, mirror: false }]; + if (spec.mirrorRepo) { + candidates.push({ + repo: spec.mirrorRepo, + filename: spec.mirrorFilename || spec.filename, + mirror: true, + }); + } + return candidates; +}; + // "exactly 1" / "2-8" — the human phrasing of a spec's reference-count rule. // Lives here so the route and the arg builder can't word (or bound) it // differently, and so the Python helper's flags come from one place. @@ -136,22 +205,44 @@ export const icResolutionIssue = (spec, width, height) => { return `${spec.label} mode needs a resolution divisible by ${scale} (its reference encoder downscales by ${scale}); got ${width}×${height}.`; }; -// Resolve the weight to hand the Python helper. Prefers the exact filename -// inside the newest HF cache snapshot (so we pin the file rather than letting -// the pipeline glob-pick among several `.safetensors` in a multi-weight repo), -// then any cached snapshot path for the repo, and finally the bare repo id — -// which ICLoraPipeline downloads itself. +// Locate a spec's weight in the local HF cache. Walks every candidate (official +// repo, then the un-gated mirror) and pins the EXACT filename inside the newest +// snapshot rather than letting the pipeline glob-pick among several +// `.safetensors` in a multi-weight repo — which for the aggregate mirror would +// pick an arbitrary unrelated LTX weight. Returns the resolved candidate +// (`{ path, repo, filename, mirror }`) or null when nothing is cached. +export const findCachedIcLoraWeight = async (spec) => { + for (const candidate of icLoraWeightCandidates(spec)) { + const { snapshotPath } = await inspectModelCache(candidate.repo); + if (!snapshotPath) continue; + const exact = join(snapshotPath, candidate.filename); + // `existsSync` FOLLOWS symlinks, so a dangling snapshot link left by an + // interrupted download reports false here — the same "is it really resident?" + // question inspectModelCache asks of a whole snapshot. + if (existsSync(exact)) return { ...candidate, path: exact }; + } + return null; +}; + +// Resolve the weight to hand the Python helper. Prefers a real cached file (via +// findCachedIcLoraWeight), then falls back to the bare repo id — which +// ICLoraPipeline._resolve_lora_path downloads itself via `snapshot_download`. // -// Returns `{ path, cached }`: `cached` is true only when a real local file was -// found, so callers can warn the user that an un-cached weight means a silent -// multi-hundred-MB pull at render time. +// That fallback is SUPPRESSED for a `requiresPreDownload` spec. `snapshot_download` +// pulls the whole repo, and for Ingredients both candidates make that unacceptable: +// the official repo is gated (a 401 deep inside the render) and the mirror is the +// ~708 GB `DeepBeepMeep/LTX-2` aggregate, which would fill the user's disk. Those +// specs return `path: null` so the caller fails fast with a "download the weight +// first" error instead. +// +// Returns `{ path, cached, spec, repo? }`: `cached` is true only when a real local +// file was found, so callers can warn the user that an un-cached weight means a +// silent multi-hundred-MB pull at render time. export const resolveIcLoraWeight = async (mode) => { const spec = icLoraSpecForMode(mode); if (!spec) return null; - const { snapshotPath } = await inspectModelCache(spec.repo); - if (snapshotPath) { - const exact = join(snapshotPath, spec.filename); - if (existsSync(exact)) return { path: exact, cached: true, spec }; - } + const cached = await findCachedIcLoraWeight(spec); + if (cached) return { path: cached.path, cached: true, spec, repo: cached.repo }; + if (spec.requiresPreDownload) return { path: null, cached: false, spec }; return { path: spec.repo, cached: false, spec }; }; diff --git a/server/lib/icLoraWeights.test.js b/server/lib/icLoraWeights.test.js index fb76a8825c..89947575f6 100644 --- a/server/lib/icLoraWeights.test.js +++ b/server/lib/icLoraWeights.test.js @@ -14,7 +14,8 @@ vi.mock('node:fs', () => ({ existsSync: mockExistsSync })); const { IC_LORA_MODES, IC_LORA_MODE_VALUES, isIcLoraMode, icLoraSpecForMode, - icLoraRepos, resolveIcLoraWeight, icResolutionIssue, + icLoraRepos, listIcLoraWeights, icLoraWeightCandidates, findCachedIcLoraWeight, + resolveIcLoraWeight, icResolutionIssue, } = await import('./icLoraWeights.js'); beforeEach(() => { @@ -24,7 +25,7 @@ beforeEach(() => { describe('IC-LoRA registry', () => { it('exposes ic-prefixed mode values derived from the registry', () => { - expect(IC_LORA_MODE_VALUES).toEqual(['ic-control', 'ic-colorize']); + expect(IC_LORA_MODE_VALUES).toEqual(['ic-control', 'ic-colorize', 'ic-ingredients']); // The `ic-` prefix is load-bearing: the client's download-id router and the // route's mode enum both key off it. for (const v of IC_LORA_MODE_VALUES) expect(v.startsWith('ic-')).toBe(true); @@ -52,9 +53,19 @@ describe('IC-LoRA registry', () => { expect(icLoraRepos()).toEqual([ 'Lightricks/LTX-2.3-22b-IC-LoRA-Union-Control', 'DoctorDiffusion/LTX-2.3-IC-LoRA-Colorizer', + 'Lightricks/LTX-2.3-22b-IC-LoRA-Ingredients', ]); }); + it('excludes mirror repos from the integrity-scan surface (#3112)', () => { + // An unscoped integrity scan walks each repo's WHOLE snapshot. The Ingredients + // mirror is the ~708 GB `DeepBeepMeep/LTX-2` aggregate, so including it would + // stat (and under `deep`, hash) every unrelated LTX weight the user has. + const mirrors = listIcLoraWeights().map((s) => s.mirrorRepo).filter(Boolean); + expect(mirrors.length).toBeGreaterThan(0); + for (const mirror of mirrors) expect(icLoraRepos()).not.toContain(mirror); + }); + it('keeps every entry internally consistent', () => { for (const spec of Object.values(IC_LORA_MODES)) { expect(spec.mode).toBe(`ic-${spec.id}`); @@ -133,4 +144,83 @@ describe('resolveIcLoraWeight', () => { expect(await resolveIcLoraWeight('ic-nope')).toBeNull(); expect(mockInspectModelCache).not.toHaveBeenCalled(); }); + + it('SUPPRESSES the repo-id fallback for a requiresPreDownload weight (#3112)', async () => { + // This is the whole point of the flag. `_resolve_lora_path` implements a bare + // repo id as `snapshot_download(id)`: for Ingredients the official repo is + // gated (401 deep inside the render) and the mirror is the ~708 GB + // DeepBeepMeep/LTX-2 aggregate, which would fill the user's disk. Neither id + // may ever reach the pipeline — path must be null so icLoraArgs 400s with + // "download the weight first". + mockInspectModelCache.mockResolvedValue({ cached: false, sizeBytes: 0, snapshotPath: null }); + + const resolved = await resolveIcLoraWeight('ic-ingredients'); + expect(resolved.path).toBeNull(); + expect(resolved.cached).toBe(false); + expect(resolved.spec).toBe(IC_LORA_MODES.ingredients); + expect(resolved.path).not.toBe(IC_LORA_MODES.ingredients.repo); + expect(resolved.path).not.toBe(IC_LORA_MODES.ingredients.mirrorRepo); + }); + + it('still resolves a requiresPreDownload weight from the mirror snapshot', async () => { + // Official repo has no snapshot; the mirror does. The candidate walk must find + // it there rather than giving up (that's what makes the un-gated path work for + // a user with no HF token). + mockInspectModelCache.mockImplementation(async (repo) => ( + repo === IC_LORA_MODES.ingredients.mirrorRepo + ? { cached: true, sizeBytes: 1, snapshotPath: '/hf/mirror-snap' } + : { cached: false, sizeBytes: 0, snapshotPath: null } + )); + mockExistsSync.mockReturnValue(true); + + const resolved = await resolveIcLoraWeight('ic-ingredients'); + expect(resolved.path).toBe(join('/hf/mirror-snap', IC_LORA_MODES.ingredients.mirrorFilename)); + expect(resolved.cached).toBe(true); + expect(resolved.repo).toBe(IC_LORA_MODES.ingredients.mirrorRepo); + }); + + it('prefers the official repo over the mirror when both are cached', async () => { + mockInspectModelCache.mockImplementation(async (repo) => ({ + cached: true, + sizeBytes: 1, + snapshotPath: repo === IC_LORA_MODES.ingredients.repo ? '/hf/official' : '/hf/mirror', + })); + mockExistsSync.mockReturnValue(true); + + const resolved = await resolveIcLoraWeight('ic-ingredients'); + expect(resolved.repo).toBe(IC_LORA_MODES.ingredients.repo); + expect(resolved.path).toBe(join('/hf/official', IC_LORA_MODES.ingredients.filename)); + }); +}); + +describe('icLoraWeightCandidates', () => { + it('orders official first, mirror second, and pins each filename', () => { + // Order IS the policy: a user WITH an HF token gets the first-party weight; a + // user without one falls through to the un-gated mirror. + expect(icLoraWeightCandidates(IC_LORA_MODES.ingredients)).toEqual([ + { repo: IC_LORA_MODES.ingredients.repo, filename: IC_LORA_MODES.ingredients.filename, mirror: false }, + { repo: IC_LORA_MODES.ingredients.mirrorRepo, filename: IC_LORA_MODES.ingredients.mirrorFilename, mirror: true }, + ]); + }); + + it('yields a single candidate for a mirror-less spec', () => { + expect(icLoraWeightCandidates(IC_LORA_MODES.control)).toEqual([ + { repo: IC_LORA_MODES.control.repo, filename: IC_LORA_MODES.control.filename, mirror: false }, + ]); + }); + + it('returns nothing for a null spec', () => { + expect(icLoraWeightCandidates(null)).toEqual([]); + }); +}); + +describe('findCachedIcLoraWeight', () => { + it('treats a dangling snapshot symlink as not cached', async () => { + // existsSync FOLLOWS symlinks, so an interrupted download (link present, blob + // gone) must NOT count as resident — otherwise the render hands the pipeline a + // path that fails on open. + mockInspectModelCache.mockResolvedValue({ cached: true, sizeBytes: 1, snapshotPath: '/hf/snap' }); + mockExistsSync.mockReturnValue(false); + expect(await findCachedIcLoraWeight(IC_LORA_MODES.ingredients)).toBeNull(); + }); }); diff --git a/server/lib/sseDownload.js b/server/lib/sseDownload.js index 4c01a0beb9..b734796925 100644 --- a/server/lib/sseDownload.js +++ b/server/lib/sseDownload.js @@ -29,15 +29,36 @@ export function openSseStream(res) { return { send, safeEnd }; } -export async function startHfDownloadStream({ req, res, repo, repos, alreadyDownloadedMessage, force = false }) { - // Caller passes either `repo` (single string, legacy callers) OR `repos` - // (ordered array, used when a model has auxiliary repos that must be - // present alongside the main weights — e.g. HiDream's separate Llama-3.1 - // text encoder). Multi-repo runs are sequential and short-circuit on any - // single-repo error. - const targets = Array.isArray(repos) - ? repos.filter((r) => typeof r === 'string' && r.length > 0) - : (typeof repo === 'string' && repo.length > 0 ? [repo] : []); +export async function startHfDownloadStream({ req, res, repo, repos, fallbacks, cachedFile = null, alreadyDownloadedMessage, force = false }) { + // Three input shapes, two semantics: + // - `repo` (single string) / `repos` (ordered array) — ALL must succeed. Used + // when a model has auxiliary repos that must be present alongside the main + // weights (e.g. HiDream's separate Llama-3.1 text encoder). Sequential; + // short-circuits on any single-repo error. + // - `fallbacks` (ordered array of `{ repo, only? }`) — FIRST SUCCESS wins, and + // a failure advances to the next entry. Used when the same file lives in + // several repos with different access (Ingredients: gated first-party repo, + // then an un-gated mirror), so a user with no HF token still succeeds. + // + // `only` (array of exact repo-relative filenames, per entry) forwards to the + // helper's single-file mode — MANDATORY for aggregate repos where a snapshot is + // catastrophic (the ~708 GB DeepBeepMeep/LTX-2 mirror). + // + // `cachedFile` is the "is it already here?" predicate for a single-file pull: + // `inspectModelCache` reports the whole repo cached as soon as ANY weight is + // resident, which for an aggregate mirror is true after the first unrelated + // file — so the generic already-cached short-circuit would skip a weight the + // user doesn't have. Callers pass an async `() => boolean` that checks the one + // file instead. + const normalizeOnly = (v) => (Array.isArray(v) ? v.filter((f) => typeof f === 'string' && f.length > 0) : []); + const firstSuccessWins = Array.isArray(fallbacks); + const targets = firstSuccessWins + ? fallbacks + .filter((t) => t && typeof t.repo === 'string' && t.repo.length > 0) + .map((t) => ({ repo: t.repo, only: normalizeOnly(t.only) })) + : (Array.isArray(repos) ? repos : [repo]) + .filter((r) => typeof r === 'string' && r.length > 0) + .map((r) => ({ repo: r, only: [] })); const { send, safeEnd } = openSseStream(res); if (targets.length === 0) { @@ -64,72 +85,142 @@ export async function startHfDownloadStream({ req, res, repo, repos, alreadyDown let downloadedAny = false; let totalSize = 0; + // Only meaningful in first-success-wins mode: which repo actually delivered, + // and the per-attempt failures to report if none did. + let succeededRepo = null; + const attemptErrors = []; for (let i = 0; i < targets.length; i += 1) { - const r = targets[i]; + const { repo: r, only: onlyFiles } = targets[i]; + const isLastTarget = i === targets.length - 1; if (aborted) return; const existing = await inspectModelCache(r); if (aborted) return; + // For a single-file pull the repo-wide cache verdict is the wrong question: + // an aggregate mirror reports `cached` the moment any unrelated weight is + // resident, which would skip the file the user actually asked for. Defer to + // the caller's per-file predicate when one is supplied. + const alreadyHave = onlyFiles.length && typeof cachedFile === 'function' + ? await cachedFile() + : existing.cached; + if (aborted) return; // `force` is set by a repair-initiated re-download: repairModelCache() // deleted the corrupt file(s), but for a multi-shard repo the remaining // shards still make inspectModelCache() report `cached` — so skipping here // would leave the just-deleted shard missing and never re-fetched. Forcing // re-runs the HF fetch, which is a cheap no-op for files already present // (etag match) and pulls only what's gone. - if (existing.cached && !force) { - totalSize += existing.sizeBytes || 0; - console.log(`📦 HuggingFace repo already cached: ${r} (${existing.sizeBytes} bytes)`); - send({ type: 'log', message: `${r} already cached (${existing.sizeBytes} bytes).`, repo: r, sizeBytes: existing.sizeBytes }); + if (alreadyHave && !force) { + // A single-file pull must NOT claim the repo's resident total: for the + // aggregate mirror that's every unrelated weight the user has (hundreds of + // GB), which would be a wildly wrong number on the badge. Report 0 there + // and let the badge fall back to the registry's size estimate. + if (onlyFiles.length) { + console.log(`📦 HuggingFace weight already cached: ${onlyFiles.join(', ')} (${r})`); + send({ type: 'log', message: `${onlyFiles.join(', ')} already cached.`, repo: r, sizeBytes: 0 }); + } else { + totalSize += existing.sizeBytes || 0; + console.log(`📦 HuggingFace repo already cached: ${r} (${existing.sizeBytes} bytes)`); + send({ type: 'log', message: `${r} already cached (${existing.sizeBytes} bytes).`, repo: r, sizeBytes: existing.sizeBytes }); + } + // In first-success-wins mode the file being present is the whole goal — + // don't fall through and try the mirror for something we already have. + if (firstSuccessWins) { succeededRepo = r; break; } continue; } - if (inFlight.has(r)) { - console.log(`⏭️ HuggingFace download already running for ${r} — refusing duplicate`); + // Dedupe key includes the filenames: an aggregate repo hosts many unrelated + // weights, so two single-file pulls of DIFFERENT files from the same repo are + // not duplicates and must not block each other. + const flightKey = onlyFiles.length ? `${r}::${onlyFiles.join(',')}` : r; + if (inFlight.has(flightKey)) { + console.log(`⏭️ HuggingFace download already running for ${flightKey} — refusing duplicate`); send({ type: 'error', message: `Another download for ${r} is already running.`, kind: 'already_running', repo: r }); return safeEnd(); } // Server-side visibility for a pull that used to surface only on the SSE // stream (the browser) — a headless/PM2 log had no record the multi-GB // fetch ever ran. Log the start, each file as it streams, and the outcome. - console.log(`⬇️ Downloading HuggingFace repo: ${r}${force ? ' (forced re-fetch)' : ''}`); + console.log(`⬇️ Downloading HuggingFace repo: ${r}${onlyFiles.length ? ` (${onlyFiles.length} file(s) only)` : ''}${force ? ' (forced re-fetch)' : ''}`); const handle = downloadHfRepo({ repo: r, + only: onlyFiles.length ? onlyFiles : null, onEvent: (ev) => { if (ev.type === 'progress' && ev.file) { console.log(`⬇️ ${r}: ${ev.file} (${ev.step}/${ev.total})`); } + // A failure that we're about to retry against the next candidate isn't a + // user-facing error — downgrade it to a log so the UI doesn't flash a red + // "gated repo" banner for something the mirror then delivers fine. + if (firstSuccessWins && !isLastTarget && ev.type === 'error') { + send({ type: 'log', message: `${r}: ${ev.message} — trying the next source.`, repo: r }); + return; + } send({ ...ev, repo: r }); }, }); currentHandle = handle; - inFlight.set(r, handle); + inFlight.set(flightKey, handle); + let result; try { - const result = await handle.promise; - downloadedAny = true; + result = await handle.promise; if (result?.ok) { console.log(`✅ HuggingFace download complete: ${r} (${result.sizeBytes || 0} bytes)`); } else if (result?.errorKind !== 'cancelled') { console.error(`❌ HuggingFace download failed: ${r} — ${result?.errorMessage || 'unknown error'}`); } } finally { - inFlight.delete(r); + inFlight.delete(flightKey); currentHandle = null; } + if (!firstSuccessWins) { + downloadedAny = true; + continue; + } + if (result?.ok) { + downloadedAny = true; + succeededRepo = r; + totalSize += result.sizeBytes || 0; + break; + } + // A user cancel must not silently roll onto the next candidate. + if (result?.errorKind === 'cancelled') return safeEnd(); + attemptErrors.push(`${r}: ${result?.errorMessage || 'unknown error'}`); } if (!aborted) { + if (firstSuccessWins) { + if (!succeededRepo) { + send({ + type: 'error', + kind: 'all_sources_failed', + message: `Every source failed — ${attemptErrors.join('; ')}`, + }); + return safeEnd(); + } + send({ + type: 'complete', + message: downloadedAny + ? `${succeededRepo} downloaded.` + : (alreadyDownloadedMessage || `${succeededRepo} already downloaded.`), + repos: [succeededRepo], + sizeBytes: totalSize, + }); + return safeEnd(); + } + const names = targets.map((t) => t.repo); let message; - if (targets.length === 1) { + if (names.length === 1) { // Preserve legacy single-repo `complete` message semantics: if it was // already cached on entry, surface the caller's `alreadyDownloadedMessage` // (or the default already-downloaded line). message = !downloadedAny - ? (alreadyDownloadedMessage || `${targets[0]} already downloaded.`) - : `${targets[0]} downloaded.`; + ? (alreadyDownloadedMessage || `${names[0]} already downloaded.`) + : `${names[0]} downloaded.`; } else { message = downloadedAny - ? `Downloaded ${targets.length} repos: ${targets.join(', ')}` - : `All ${targets.length} repos already cached: ${targets.join(', ')}`; + ? `Downloaded ${names.length} repos: ${names.join(', ')}` + : `All ${names.length} repos already cached: ${names.join(', ')}`; } - send({ type: 'complete', message, repos: targets, sizeBytes: totalSize }); + send({ type: 'complete', message, repos: names, sizeBytes: totalSize }); } safeEnd(); } diff --git a/server/lib/sseDownload.test.js b/server/lib/sseDownload.test.js index b99eaed4c3..0f20abb88f 100644 --- a/server/lib/sseDownload.test.js +++ b/server/lib/sseDownload.test.js @@ -181,3 +181,125 @@ describe('startHfDownloadStream server-side logging', () => { expect(logged(errorSpy).some((l) => l.includes('download failed'))).toBe(false); }); }); + +// ── Single-file + first-success-wins fallbacks (#3112) ────────────────────── +// `fallbacks` exists because the Ingredients IC weight lives in TWO repos with +// different access: a gated first-party repo and the un-gated ~708 GB +// `DeepBeepMeep/LTX-2` aggregate mirror. That makes two behaviors mandatory — +// single-file pulls (a snapshot of the mirror would fill the user's disk) and +// advancing to the next source on failure (so a user with no HF token succeeds). +describe('startHfDownloadStream single-file + fallbacks (#3112)', () => { + const CANDIDATES = [ + { repo: 'org/official', only: ['weight.safetensors'] }, + { repo: 'org/mirror-708gb', only: ['weight.safetensors'] }, + ]; + + beforeEach(() => { + vi.clearAllMocks(); + inspectModelCache.mockResolvedValue({ cached: false, sizeBytes: 0, snapshotPath: null }); + downloadHfRepo.mockImplementation(() => ({ + promise: Promise.resolve({ ok: true, sizeBytes: 1000 }), + kill: vi.fn(), + })); + }); + + it('forwards `only` so the helper never enumerates the repo', async () => { + const { req, res } = makeReqRes(); + await startHfDownloadStream({ req, res, fallbacks: [CANDIDATES[0]], cachedFile: async () => false }); + expect(downloadHfRepo).toHaveBeenCalledWith(expect.objectContaining({ + repo: 'org/official', + only: ['weight.safetensors'], + })); + }); + + it('stops at the first success without touching the mirror', async () => { + const { req, res, frames } = makeReqRes(); + await startHfDownloadStream({ req, res, fallbacks: CANDIDATES, cachedFile: async () => false }); + expect(downloadHfRepo).toHaveBeenCalledTimes(1); + expect(downloadHfRepo.mock.calls[0][0].repo).toBe('org/official'); + expect(parseFrames(frames).at(-1)).toMatchObject({ type: 'complete', repos: ['org/official'] }); + }); + + it('advances to the mirror when the gated repo fails', async () => { + downloadHfRepo.mockImplementation(({ repo }) => ({ + promise: Promise.resolve(repo === 'org/official' + ? { ok: false, errorKind: 'gated_repo', errorMessage: 'gated' } + : { ok: true, sizeBytes: 1000 }), + kill: vi.fn(), + })); + + const { req, res, frames } = makeReqRes(); + await startHfDownloadStream({ req, res, fallbacks: CANDIDATES, cachedFile: async () => false }); + expect(downloadHfRepo.mock.calls.map((c) => c[0].repo)).toEqual(['org/official', 'org/mirror-708gb']); + const events = parseFrames(frames); + // The retried failure must NOT reach the client as an error — otherwise the UI + // flashes a red "gated repo" banner for something the mirror then delivers. + expect(events.some((e) => e.type === 'error')).toBe(false); + expect(events.at(-1)).toMatchObject({ type: 'complete', repos: ['org/mirror-708gb'] }); + }); + + it('reports every attempt when all sources fail', async () => { + downloadHfRepo.mockImplementation(({ repo }) => ({ + promise: Promise.resolve({ ok: false, errorKind: 'x', errorMessage: `${repo} broke` }), + kill: vi.fn(), + })); + + const { req, res, frames } = makeReqRes(); + await startHfDownloadStream({ req, res, fallbacks: CANDIDATES, cachedFile: async () => false }); + const last = parseFrames(frames).at(-1); + expect(last).toMatchObject({ type: 'error', kind: 'all_sources_failed' }); + expect(last.message).toContain('org/official broke'); + expect(last.message).toContain('org/mirror-708gb broke'); + }); + + it('does not roll onto the next source after a user cancel', async () => { + downloadHfRepo.mockImplementation(() => ({ + promise: Promise.resolve({ ok: false, errorKind: 'cancelled', errorMessage: 'Cancelled' }), + kill: vi.fn(), + })); + + const { req, res } = makeReqRes(); + await startHfDownloadStream({ req, res, fallbacks: CANDIDATES, cachedFile: async () => false }); + // Cancel means "stop", not "try somewhere else". + expect(downloadHfRepo).toHaveBeenCalledTimes(1); + }); + + it('defers the already-cached verdict to cachedFile, not the repo-wide flag', async () => { + // The aggregate mirror reports `cached: true` off ANY resident weight. Without + // the per-file predicate the driver would skip a weight the user doesn't have. + inspectModelCache.mockResolvedValue({ cached: true, sizeBytes: 700e9, snapshotPath: '/snap' }); + + const { req, res } = makeReqRes(); + await startHfDownloadStream({ req, res, fallbacks: [CANDIDATES[1]], cachedFile: async () => false }); + expect(downloadHfRepo).toHaveBeenCalledTimes(1); + }); + + it('short-circuits when cachedFile says the weight is already resident', async () => { + inspectModelCache.mockResolvedValue({ cached: false, sizeBytes: 0, snapshotPath: null }); + + const { req, res, frames } = makeReqRes(); + await startHfDownloadStream({ req, res, fallbacks: CANDIDATES, cachedFile: async () => true }); + expect(downloadHfRepo).not.toHaveBeenCalled(); + const last = parseFrames(frames).at(-1); + expect(last).toMatchObject({ type: 'complete' }); + // Present already ⇒ don't fall through to the mirror for something we have. + expect(last.repos).toEqual(['org/official']); + }); + + it('never reports the aggregate repo size for a single-file cache hit', async () => { + // 700 GB of unrelated weights is not this weight's footprint; reporting it + // would put a wildly wrong number on the download badge. + inspectModelCache.mockResolvedValue({ cached: true, sizeBytes: 700e9, snapshotPath: '/snap' }); + + const { req, res, frames } = makeReqRes(); + await startHfDownloadStream({ req, res, fallbacks: [CANDIDATES[1]], cachedFile: async () => true }); + expect(parseFrames(frames).at(-1).sizeBytes).toBe(0); + }); + + it('rejects an empty fallbacks list rather than silently completing', async () => { + const { req, res, frames } = makeReqRes(); + await startHfDownloadStream({ req, res, fallbacks: [] }); + expect(downloadHfRepo).not.toHaveBeenCalled(); + expect(parseFrames(frames)).toEqual([{ type: 'error', message: 'No repo specified for download.' }]); + }); +}); diff --git a/server/routes/videoGen.js b/server/routes/videoGen.js index 64485683c1..ec427c85b3 100644 --- a/server/routes/videoGen.js +++ b/server/routes/videoGen.js @@ -48,9 +48,10 @@ import { repoForModel, getTextEncoderRepo, isHfRepoId } from '../lib/mediaModels import { videoLoraFamily } from '../lib/runners.js'; import { IC_LORA_MODE_VALUES, icLoraSpecForMode, icLoraRepos, listIcLoraWeights, - assertIcReferenceCount, + assertIcReferenceCount, describeIcReferenceRange, + icLoraWeightCandidates, findCachedIcLoraWeight, } from '../lib/icLoraWeights.js'; -import { inspectModelCache, verifyModelCache, repairModelCache, summarizeVerify } from '../lib/hfCache.js'; +import { inspectModelCache, verifyModelCache, repairModelCache, repairCachedFile, summarizeVerify } from '../lib/hfCache.js'; import { startHfDownloadStream, openSseStream } from '../lib/sseDownload.js'; import { createInstallLogger } from '../lib/installLogger.js'; @@ -181,6 +182,25 @@ const generateBodySchema = z.object({ }, z.array(z.string().guid()).min(1).max(8).optional(), ), + // Ingredients-style IC references: 2-8 gallery STILLS, not clips. A separate + // field from icReference / icReferenceVideoIds on purpose — those are + // `video/*` and resolve against render history, and overloading them would + // let a video ride into an image-kind weight (or vice versa) and produce + // plausible-looking garbage. Gallery-only, exactly like `keyframes`: the + // route resolves each basename under PATHS.images. Multipart sends a single + // value as a bare string and repeated keys as an array; also accept a + // JSON-encoded list from a JSON client (mirrors icReferenceVideoIds). + icReferenceImageFiles: z.preprocess( + (v) => { + if (v == null || v === '') return undefined; + if (typeof v === 'string') { + if (v.trim().startsWith('[')) { try { return JSON.parse(v); } catch { return [v]; } } + return [v]; + } + return v; + }, + z.array(z.string().min(1).max(512)).min(1).max(8).optional(), + ), // Reference-video conditioning strength for the IC-LoRA channel. Distinct // from the IC-LoRA's own fusion strength (fixed at 1.0 server-side) and from // `icAttentionStrength`, which scales the conditioning ATTENTION. @@ -523,11 +543,34 @@ router.get('/models/status', asyncHandler(async (_req, res) => { // pull the IC render path needs, so they get the same cached/size/integrity // shape as the models — that's what lets the mode panel render a Download // badge and a Repair banner with the existing components. - Promise.all(listIcLoraWeights().map(async (spec) => ({ - id: spec.mode, repo: spec.repo, label: spec.label, - estimatedBytes: spec.sizeBytes, - ...await repoCacheStatus(spec.repo), - }))), + Promise.all(listIcLoraWeights().map(async (spec) => { + // A mirrored spec (Ingredients) can't use the repo-wide verdict: its + // official repo is gated and its mirror is a 708 GB aggregate that reports + // `cached` off any unrelated weight. Probe the ONE file across both + // candidates instead, and skip the integrity walk (which would stat/hash + // every sibling weight in that mirror). + if (spec.mirrorRepo) { + const found = await findCachedIcLoraWeight(spec); + return { + id: spec.mode, repo: spec.repo, label: spec.label, + estimatedBytes: spec.sizeBytes, + gated: !!spec.gated, mirrorRepo: spec.mirrorRepo, + cached: !!found, + resolvedRepo: found?.repo || null, + // The badge falls back to `estimatedBytes` when sizeBytes is 0, and the + // real number would cost a stat on a path we already proved resident — + // so report 0 and let the estimate speak. + sizeBytes: 0, + integrity: null, + }; + } + return { + id: spec.mode, repo: spec.repo, label: spec.label, + estimatedBytes: spec.sizeBytes, + gated: !!spec.gated, + ...await repoCacheStatus(spec.repo), + }; + })), ]); res.json({ models, textEncoder, icLoras }); })); @@ -596,9 +639,31 @@ const icLoraSpecFromParam = (mode) => { return spec; }; +// Download one IC weight. A spec with a `mirrorRepo` is fetched SINGLE-FILE and +// only ever single-file: the official Ingredients repo is gated (an anonymous +// pull 401s) and its un-gated mirror is the ~708 GB `DeepBeepMeep/LTX-2` +// aggregate, so a snapshot of either would either fail or fill the user's disk. +// Candidates are tried in order (official → mirror) so a user WITH an HF token +// gets the first-party weight and a user without one still succeeds via the +// mirror — no token, no extra button. The exact filename is pinned so the mirror +// can't hand back a sibling weight. router.get('/ic-loras/:mode/download', asyncHandler(async (req, res) => { const spec = icLoraSpecFromParam(req.params.mode); - await startHfDownloadStream({ req, res, repo: spec.repo, force: req.query.force === '1' }); + const force = req.query.force === '1'; + if (!spec.mirrorRepo) { + await startHfDownloadStream({ req, res, repo: spec.repo, force }); + return; + } + await startHfDownloadStream({ + req, + res, + fallbacks: icLoraWeightCandidates(spec).map((c) => ({ repo: c.repo, only: [c.filename] })), + // The repo-wide `cached` verdict is meaningless for the aggregate mirror (it + // reports cached as soon as ANY unrelated weight is resident), so gate the + // already-have short-circuit on this exact weight instead. + cachedFile: async () => !!(await findCachedIcLoraWeight(spec)), + force, + }); })); router.post('/ic-loras/:mode/repair', asyncHandler(async (req, res) => { @@ -606,6 +671,16 @@ router.post('/ic-loras/:mode/repair', asyncHandler(async (req, res) => { const parsed = z.object({ deep: z.boolean().optional() }).safeParse(req.body || {}); if (!parsed.success) failValidation(parsed); const deep = parsed.data.deep || false; + // A mirrored spec is single-file by construction, and repairModelCache walks + // the WHOLE snapshot — against the 708 GB aggregate mirror that would stat (and + // under `deep`, hash) every unrelated LTX weight the user has. Delete just this + // weight and let the single-file download re-fetch it. + if (spec.mirrorRepo) { + const found = await findCachedIcLoraWeight(spec); + if (!found) return res.json({ deep, deleted: [], repos: [spec.repo] }); + await repairCachedFile(found.path); + return res.json({ deep, deleted: [{ repo: found.repo, name: found.filename }], repos: [found.repo] }); + } const result = await repairModelCache(spec.repo, { deep }); res.json({ deep, deleted: result.deleted.map((name) => ({ repo: result.repoId, name })), repos: [spec.repo] }); })); @@ -776,6 +851,26 @@ router.post('/', frameImageUpload, asyncHandler(async (req, res) => { { status: 400, code: 'IC_LORA_REQUIRES_LOCAL_BACKEND' }, ); } + // Each weight takes exactly ONE kind of reference. An image-kind weight fed a + // clip (or vice versa) doesn't error inside the pipeline — it produces + // plausible-looking garbage — so reject the cross-kind fields explicitly + // rather than silently dropping them. + const wantsImages = icSpec.referenceKind === 'image'; + const videoShapePresent = !!uploads.icReference || !!body.icReferenceVideoIds?.length; + if (wantsImages && videoShapePresent) { + await cleanupAllStaged(); + throw new ServerError( + `${icSpec.label} mode conditions on still images — pass gallery filenames as icReferenceImageFiles, not icReference / icReferenceVideoIds.`, + { status: 400, code: 'IC_LORA_REFERENCE_KIND_MISMATCH' }, + ); + } + if (!wantsImages && body.icReferenceImageFiles?.length) { + await cleanupAllStaged(); + throw new ServerError( + `${icSpec.label} mode conditions on a reference clip — pass icReference / icReferenceVideoIds, not icReferenceImageFiles.`, + { status: 400, code: 'IC_LORA_REFERENCE_KIND_MISMATCH' }, + ); + } // The upload wins over gallery picks, so a request carrying both would // silently drop the picks. Force one shape per request instead. Checked // BEFORE the count bounds below, which would otherwise report a misleading @@ -790,14 +885,19 @@ router.post('/', frameImageUpload, asyncHandler(async (req, res) => { // Reference count is the weight's contract, asserted once here against the // registry that owns the bounds. Checked before staging so a bad request only // unlinks the cheap OS temp file, and resolution below can't change the - // count: an upload is exactly 1 and every id resolves 1:1 or throws. - const refCount = (uploads.icReference ? 1 : 0) + (body.icReferenceVideoIds?.length || 0); + // count: an upload is exactly 1, every history id resolves 1:1 or throws, and + // every gallery filename resolves 1:1 or throws. + const refCount = wantsImages + ? (body.icReferenceImageFiles?.length || 0) + : (uploads.icReference ? 1 : 0) + (body.icReferenceVideoIds?.length || 0); if (refCount === 0) { // Special-cased ahead of the bounds assertion purely for actionability — // "needs exactly 1; got 0" doesn't tell the caller HOW to supply one. await cleanupAllStaged(); throw new ServerError( - `${icSpec.label} mode requires a reference ${icSpec.referenceKind} — upload one (multipart field: icReference) or pick a prior render (icReferenceVideoIds).`, + wantsImages + ? `${icSpec.label} mode requires ${describeIcReferenceRange(icSpec)} reference images — pick them from your gallery (icReferenceImageFiles).` + : `${icSpec.label} mode requires a reference ${icSpec.referenceKind} — upload one (multipart field: icReference) or pick a prior render (icReferenceVideoIds).`, { status: 400, code: 'IC_LORA_REFERENCE_REQUIRED' }, ); } @@ -826,7 +926,7 @@ router.post('/', frameImageUpload, asyncHandler(async (req, res) => { { status: 400, code: 'IC_LORA_CHUNKS_CONFLICT' }, ); } - } else if (uploads.icReference || body.icReferenceVideoIds?.length) { + } else if (uploads.icReference || body.icReferenceVideoIds?.length || body.icReferenceImageFiles?.length) { await cleanupAllStaged(); throw new ServerError( `IC-LoRA reference inputs are only valid with an IC remix mode (${IC_LORA_MODE_VALUES.join(', ')}); got mode='${body.mode || 'unset'}'.`, @@ -1084,13 +1184,32 @@ router.post('/', frameImageUpload, asyncHandler(async (req, res) => { }); } - // IC-LoRA reference channel: either the staged upload or the picked prior - // render(s). The route already rejected the both-present case (and asserted - // the count against the weight's bounds) above, so at most one branch - // contributes and each id resolves 1:1 or throws. + // IC-LoRA reference channel: gallery stills for an image-kind weight, else the + // staged upload or the picked prior render(s). The route already rejected the + // cross-kind and both-present cases (and asserted the count against the weight's + // bounds) above, so exactly one branch contributes and each entry resolves 1:1 + // or throws. let icReferencePaths = null; if (icSpec) { - if (icReferenceUploadPath) { + if (icSpec.referenceKind === 'image') { + // Gallery-only, exactly like `keyframes` — same path-traversal guard, same + // "reject up-front so the queue never accepts a doomed job" contract. The + // service materializes each still into a VAE-compatible clip at render + // resolution (the IC reference channel is a video encoder end-to-end). + icReferencePaths = []; + for (let i = 0; i < body.icReferenceImageFiles.length; i++) { + const file = body.icReferenceImageFiles[i]; + const path = resolveGalleryImage(file); + if (!path) { + await cleanupAllStaged(); + throw new ServerError( + `icReferenceImageFiles[${i}] not found in gallery: ${file}`, + { status: 400, code: 'IC_LORA_REFERENCE_GALLERY_MISS' }, + ); + } + icReferencePaths.push(path); + } + } else if (icReferenceUploadPath) { icReferencePaths = [icReferenceUploadPath]; } else { const history = await getHistory(); @@ -1240,6 +1359,13 @@ const pickJobParams = (params) => { .filter((p) => typeof p === 'string' && p) .map((p) => basename(p)); if (names.length) out.icReferenceNames = names; + // For an IMAGE-kind weight the basename IS the gallery filename (references + // are gallery-only, never uploads), so unlike the clip case the resuming form + // CAN repopulate its picker and re-submit. Echo it under the submit field name + // so the client needs no per-kind translation. + if (names.length && icLoraSpecForMode(params.mode)?.referenceKind === 'image') { + out.icReferenceImageFiles = names; + } } return out; }; diff --git a/server/routes/videoGen.test.js b/server/routes/videoGen.test.js index efff60fcdd..4a53a0e085 100644 --- a/server/routes/videoGen.test.js +++ b/server/routes/videoGen.test.js @@ -990,6 +990,102 @@ describe('videoGen routes', () => { expect(r.body.error).toMatch(/chunks > 1/i); expect(mediaJobQueue.enqueueJob).not.toHaveBeenCalled(); }); + + // ── Ingredients: 2-8 gallery STILLS (#3112) ────────────────────────── + // A separate input surface from the clip fields above. The counts are the + // weight's contract — a wrong one yields plausible garbage rather than an + // error inside the pipeline, so the route asserts them before enqueue. + describe('ic-ingredients image references (#3112)', () => { + const stills = (n) => Array.from({ length: n }, (_, i) => `ref-${i}.png`); + + it('resolves 2-8 gallery stills to absolute paths', async () => { + for (const n of [2, 5, 8]) { + mediaJobQueue.enqueueJob.mockClear(); + const r = await request(app).post('/api/video-gen/').send({ + prompt: 'the owl greets the camera outside the store', + mode: 'ic-ingredients', icReferenceImageFiles: stills(n), + }); + expect(r.status).toBe(200); + expect(mediaJobQueue.enqueueJob).toHaveBeenCalledWith(expect.objectContaining({ + params: expect.objectContaining({ + mode: 'ic-ingredients', + icReferencePaths: stills(n).map((f) => `/mock/images/${f}`), + chunks: 1, + }), + })); + } + }); + + it('rejects fewer than 2 references', async () => { + const r = await request(app).post('/api/video-gen/').send({ + prompt: 'recompose', mode: 'ic-ingredients', icReferenceImageFiles: stills(1), + }); + expect(r.status).toBe(400); + expect(r.body.error).toMatch(/needs 2-8 reference image/i); + expect(mediaJobQueue.enqueueJob).not.toHaveBeenCalled(); + }); + + it('rejects more than 8 references at the schema boundary', async () => { + const r = await request(app).post('/api/video-gen/').send({ + prompt: 'recompose', mode: 'ic-ingredients', icReferenceImageFiles: stills(9), + }); + expect(r.status).toBe(400); + expect(mediaJobQueue.enqueueJob).not.toHaveBeenCalled(); + }); + + it('rejects no references with an actionable message', async () => { + const r = await request(app).post('/api/video-gen/').send({ + prompt: 'recompose', mode: 'ic-ingredients', + }); + expect(r.status).toBe(400); + // Names the field to use — "needs 2-8; got 0" alone doesn't say HOW. + expect(r.body.error).toMatch(/icReferenceImageFiles/); + expect(mediaJobQueue.enqueueJob).not.toHaveBeenCalled(); + }); + + it('rejects a gallery miss before enqueue', async () => { + vi.mocked(resolveGalleryImage).mockImplementation((f) => (f === 'gone.png' ? null : `/mock/images/${f}`)); + const r = await request(app).post('/api/video-gen/').send({ + prompt: 'recompose', mode: 'ic-ingredients', + icReferenceImageFiles: ['ref-0.png', 'gone.png'], + }); + expect(r.status).toBe(400); + expect(r.body.error).toMatch(/icReferenceImageFiles\[1\] not found in gallery/); + expect(mediaJobQueue.enqueueJob).not.toHaveBeenCalled(); + vi.mocked(resolveGalleryImage).mockImplementation((name) => `/mock/images/${name}`); + }); + + it('rejects clip inputs on an image-kind weight (IC_LORA_REFERENCE_KIND_MISMATCH)', async () => { + // Feeding a clip to an image-kind weight does NOT error in the pipeline — + // it produces plausible-looking garbage. So reject rather than drop. + setPendingUpload(icUpload); + const r = await request(app).post('/api/video-gen/').send({ + prompt: 'recompose', mode: 'ic-ingredients', icReferenceImageFiles: stills(2), + }); + expect(r.status).toBe(400); + expect(r.body.error).toMatch(/conditions on still images/i); + expect(mediaJobQueue.enqueueJob).not.toHaveBeenCalled(); + }); + + it('rejects image references on a video-kind weight (IC_LORA_REFERENCE_KIND_MISMATCH)', async () => { + const r = await request(app).post('/api/video-gen/').send({ + prompt: 'follow the depth clip', mode: 'ic-control', + icReferenceImageFiles: stills(2), + }); + expect(r.status).toBe(400); + expect(r.body.error).toMatch(/conditions on a reference clip/i); + expect(mediaJobQueue.enqueueJob).not.toHaveBeenCalled(); + }); + + it('rejects image references outside an IC mode', async () => { + const r = await request(app).post('/api/video-gen/').send({ + prompt: 'plain render', mode: 'text', icReferenceImageFiles: stills(2), + }); + expect(r.status).toBe(400); + expect(r.body.error).toMatch(/only valid with an IC remix mode/i); + expect(mediaJobQueue.enqueueJob).not.toHaveBeenCalled(); + }); + }); }); }); diff --git a/server/services/videoGen/local.js b/server/services/videoGen/local.js index a1684ed16f..cff96f5449 100644 --- a/server/services/videoGen/local.js +++ b/server/services/videoGen/local.js @@ -309,8 +309,12 @@ export const icLoraArgs = ({ mode, width, height, icReferencePaths, icLoraWeight throw new ServerError(`Unknown IC-LoRA remix mode: ${mode}`, { status: 400, code: 'IC_LORA_UNKNOWN_MODE' }); } if (!icLoraWeightPath) { + // A `requiresPreDownload` weight lands here by design: resolveIcLoraWeight + // refuses to hand the pipeline a bare repo id it would `snapshot_download` + // (gated official repo / 708 GB mirror), so the ONLY way forward is the + // explicit single-file download from the panel. throw new ServerError( - `IC-LoRA weight for "${spec.mode}" could not be resolved — download ${spec.repo} from the model panel first.`, + `IC-LoRA weight for "${spec.mode}" is not downloaded — download ${spec.label} (${spec.filename}) from the model panel first.`, { status: 400, code: 'IC_LORA_WEIGHT_UNRESOLVED' }, ); } @@ -704,6 +708,14 @@ const buildArgs = ({ pythonPath, modelId, model, prompt, negativePrompt, width, // use (avoiding drift between two hardcoded constants). export const DEFAULT_NUM_FRAMES = 121; +// Frame count for the throwaway clip an `image`-kind IC reference (Ingredients) +// is materialized into. The pipeline's reference channel runs every reference +// through ffprobe + the video VAE, whose `space_to_depth` reshape needs a +// (1 + 8k)-frame input — 9 is the smallest legal value, so it's the cheapest +// encode that satisfies the encoder. Every frame is identical; the reference is +// a still regardless of how many frames carry it. +export const IC_STILL_REFERENCE_FRAMES = 9; + export async function generateVideo({ pythonPath, prompt, negativePrompt = '', modelId = defaultVideoModelId(), width = 768, height = 512, numFrames = DEFAULT_NUM_FRAMES, fps = 24, steps, guidanceScale, seed, tiling = 'auto', disableAudio = false, sourceImagePath = null, uploadedTempPath = null, uploadedTempPaths = [], lastImagePath = null, keyframes = null, extendFromVideoPath = null, audioFilePath = null, mode = null, imageStrength = null, loras = null, icReferencePaths = null, icStrength = null, icAttentionStrength = null, icSkipStage2 = false, hidden = false, jobId: providedJobId = null }) { uploadedTempPaths = Array.isArray(uploadedTempPaths) ? uploadedTempPaths : []; if (!prompt?.trim()) throw new ServerError('Prompt is required', { status: 400, code: 'VALIDATION_ERROR' }); @@ -746,9 +758,15 @@ export async function generateVideo({ pythonPath, prompt, negativePrompt = '', m throw new ServerError(`Unknown IC-LoRA remix mode: ${mode}`, { status: 400, code: 'IC_LORA_UNKNOWN_MODE' }); } icLoraWeightPath = resolved.path; - if (!resolved.cached) { + if (!resolved.cached && resolved.path) { console.log(`⬇️ IC-LoRA weight not cached — ${resolved.spec.repo} will download at render time`); } + // A null path means the registry deliberately refused the repo-id fallback + // (requiresPreDownload). icLoraArgs turns that into the user-facing 400; log + // the reason here so the server log explains WHY there's no auto-download. + if (!resolved.path) { + console.log(`⛔ IC-LoRA weight for ${mode} needs an explicit download (auto-fetch would snapshot ${resolved.spec.mirrorRepo || resolved.spec.repo})`); + } } await ensureDir(PATHS.videos); @@ -878,6 +896,58 @@ export async function generateVideo({ pythonPath, prompt, negativePrompt = '', m }); } + // An `image`-kind IC weight (Ingredients) takes STILLS, but the pipeline's + // reference channel is a video encoder end-to-end: iclora_utils probes the + // reference with ffprobe and feeds it to the VAE, which requires a (1 + 8k) + // frame count. A bare PNG has neither a probeable frame count nor 9 frames, so + // materialize each still into a tiny 9-frame constant clip at the render + // resolution first. 9 = the smallest legal (1 + 8k) count, so this is the + // cheapest possible encode and every frame is identical — the reference is a + // still either way. + // + // Done here rather than in the route because the target resolution is only + // known after the 64-flooring above, and it mirrors resizeImage's contract: + // temp paths are tracked for the same cleanup sites. + const icReferenceTempPaths = []; + let resolvedIcReferencePaths = icReferencePaths; + if (isIcLoraMode(mode) && icLoraSpecForMode(mode)?.referenceKind === 'image' + && Array.isArray(icReferencePaths) && icReferencePaths.length) { + const stillFfmpeg = ffmpeg || await findFfmpeg(); + if (!stillFfmpeg) { + throw new ServerError( + 'ffmpeg is required to prepare still references for Ingredients mode — install it (brew install ffmpeg) and retry.', + { status: 400, code: 'IC_LORA_STILL_NEEDS_FFMPEG' }, + ); + } + resolvedIcReferencePaths = await Promise.all(icReferencePaths.map(async (stillPath, i) => { + const clipPath = join(tmpdir(), `ic-still-${i}-${jobId}.mp4`); + const result = await execFileAsync(stillFfmpeg, [ + '-loop', '1', '-i', stillPath, + '-vf', `scale=${w}:${h}:force_original_aspect_ratio=increase,crop=${w}:${h}`, + '-frames:v', String(IC_STILL_REFERENCE_FRAMES), + '-r', String(parsedFps), + '-pix_fmt', 'yuv420p', '-an', + '-y', clipPath, + ], { env: safeChildProcessEnv(), timeout: 30000 }).catch((err) => ({ error: err })); + if (result.error) { + // Unlike the resizeImage fallback (which degrades to the original), there + // is no usable degradation here — a still handed straight to the pipeline + // fails deep inside the VAE reshape. Fail loudly with the ffmpeg reason. + throw new ServerError( + `Failed to prepare Ingredients reference ${basename(stillPath)}: ${result.error.message}`, + { status: 400, code: 'IC_LORA_STILL_PREP_FAILED' }, + ); + } + icReferenceTempPaths.push(clipPath); + return clipPath; + })).catch(async (err) => { + // Partial success leaks the clips already written — one still failing + // aborts the render, so drop them before rethrowing. + for (const p of icReferenceTempPaths) await unlink(p).catch(() => {}); + throw err; + }); + } + const meta = { id: jobId, prompt, @@ -941,7 +1011,7 @@ export async function generateVideo({ pythonPath, prompt, negativePrompt = '', m // logic of the spawn-error handler so failure modes converge. let bin, args; try { - ({ bin, args } = buildArgs({ pythonPath, modelId, model, prompt, negativePrompt, width: w, height: h, numFrames: parsedNumFrames, fps: parsedFps, steps: actualSteps, stage2Steps: actualStage2Steps, guidance: actualGuidance, seed: actualSeed, tiling, disableAudio, sourceImagePath: resolvedSourceImage, lastImagePath: resolvedLastImage, keyframes: resolvedKeyframes, extendFromVideoPath, audioFilePath, mode, imageStrength: actualImageStrength, textEncoderRepo: actualTextEncoderRepo, outputPath, loras: resolvedLoras, icReferencePaths, icLoraWeightPath, icStrength: actualIcStrength, icAttentionStrength: actualIcAttentionStrength, icSkipStage2 })); + ({ bin, args } = buildArgs({ pythonPath, modelId, model, prompt, negativePrompt, width: w, height: h, numFrames: parsedNumFrames, fps: parsedFps, steps: actualSteps, stage2Steps: actualStage2Steps, guidance: actualGuidance, seed: actualSeed, tiling, disableAudio, sourceImagePath: resolvedSourceImage, lastImagePath: resolvedLastImage, keyframes: resolvedKeyframes, extendFromVideoPath, audioFilePath, mode, imageStrength: actualImageStrength, textEncoderRepo: actualTextEncoderRepo, outputPath, loras: resolvedLoras, icReferencePaths: resolvedIcReferencePaths, icLoraWeightPath, icStrength: actualIcStrength, icAttentionStrength: actualIcAttentionStrength, icSkipStage2 })); } catch (err) { job.status = 'error'; const reason = err.message || 'Failed to build video gen args'; @@ -951,6 +1021,7 @@ export async function generateVideo({ pythonPath, prompt, negativePrompt = '', m if (resizedSrcTempPath) unlink(resizedSrcTempPath).catch(() => {}); if (resizedLastTempPath) unlink(resizedLastTempPath).catch(() => {}); for (const p of resizedKeyframeTempPaths) unlink(p).catch(() => {}); + for (const p of icReferenceTempPaths) unlink(p).catch(() => {}); if (uploadedTempPath) unlink(uploadedTempPath).catch(() => {}); for (const p of uploadedTempPaths) unlink(p).catch(() => {}); if (audioFilePath && !uploadedTempPaths.includes(audioFilePath)) { @@ -1108,6 +1179,7 @@ export async function generateVideo({ pythonPath, prompt, negativePrompt = '', m if (resizedSrcTempPath) unlink(resizedSrcTempPath).catch(() => {}); if (resizedLastTempPath) unlink(resizedLastTempPath).catch(() => {}); for (const p of resizedKeyframeTempPaths) unlink(p).catch(() => {}); + for (const p of icReferenceTempPaths) unlink(p).catch(() => {}); if (uploadedTempPath) unlink(uploadedTempPath).catch(() => {}); for (const p of uploadedTempPaths) unlink(p).catch(() => {}); // Defensive: a direct caller (bypassing the route) may pass audioFilePath @@ -1198,6 +1270,10 @@ export async function generateVideo({ pythonPath, prompt, negativePrompt = '', m if (resizedSrcTempPath) await unlink(resizedSrcTempPath).catch(() => {}); if (resizedLastTempPath) await unlink(resizedLastTempPath).catch(() => {}); for (const p of resizedKeyframeTempPaths) await unlink(p).catch(() => {}); + // The throwaway still→clip encodes for an image-kind IC reference. The + // ORIGINAL stills are gallery files (or route-staged uploads) and are NOT + // ours to remove — only these temp clips. + for (const p of icReferenceTempPaths) await unlink(p).catch(() => {}); // Cleanup the original multipart upload temp file too — without this, // every i2v request leaves a file in os.tmpdir() forever. if (uploadedTempPath) await unlink(uploadedTempPath).catch(() => {}); diff --git a/server/services/videoGen/local.test.js b/server/services/videoGen/local.test.js index 9f70ccf984..e54f6e7b5a 100644 --- a/server/services/videoGen/local.test.js +++ b/server/services/videoGen/local.test.js @@ -1915,6 +1915,121 @@ describe('generateVideo — IC-LoRA remix arg threading (#3100)', () => { })).rejects.toThrow(/require an ltx2-runtime model/); }); + // ── Ingredients: image-kind references (#3112) ───────────────────────── + describe('ic-ingredients still references (#3112)', () => { + const baseIngredients = { + ...baseIcRender, + mode: 'ic-ingredients', + prompt: 'the owl greets the camera outside the store', + icReferencePaths: ['/mock/images/owl.png', '/mock/images/store.png'], + }; + + it('materializes each still into a 9-frame clip at the render resolution', async () => { + const { execFile } = await import('child_process'); + const { spawnDetached } = await import('../../lib/detachedSpawn.js'); + const execFileMock = vi.mocked(execFile); + const spawnMock = vi.mocked(spawnDetached); + execFileMock.mockClear(); + spawnMock.mockClear(); + + await generateVideo({ ...baseIngredients, jobId: 'ing-stills' }); + + // One ffmpeg per still. The pipeline's reference channel probes with ffprobe + // and feeds the video VAE, whose reshape needs a (1 + 8k)-frame input — a + // bare PNG has neither, so a still can't be passed through unchanged. + const stillCalls = execFileMock.mock.calls.filter(([, args]) => args.includes('-loop')); + expect(stillCalls).toHaveLength(2); + for (const [, args] of stillCalls) { + expect(args[args.indexOf('-frames:v') + 1]).toBe('9'); + // Scaled/cropped to the FLOORED render resolution — the pipeline requires + // exact dimensions and won't pad. + expect(args[args.indexOf('-vf') + 1]).toContain('crop=704:448'); + // No audio stream on a throwaway reference clip. + expect(args).toContain('-an'); + } + + // The helper receives the materialized CLIPS, never the source stills. + const args = findIcCall(spawnMock)[1]; + const refs = args.reduce((acc, a, i) => (a === '--ic-reference' ? [...acc, args[i + 1]] : acc), []); + expect(refs).toHaveLength(2); + for (const r of refs) expect(r).toMatch(/ic-still-\d+-ing-stills\.mp4$/); + expect(refs).not.toContain('/mock/images/owl.png'); + expect(args[args.indexOf('--ic-mode') + 1]).toBe('ingredients'); + expect(args[args.indexOf('--ic-min-references') + 1]).toBe('2'); + expect(args[args.indexOf('--ic-max-references') + 1]).toBe('8'); + }); + + it('accepts the full 2-8 range', async () => { + const { spawnDetached } = await import('../../lib/detachedSpawn.js'); + const spawnMock = vi.mocked(spawnDetached); + for (const n of [2, 5, 8]) { + spawnMock.mockClear(); + await generateVideo({ + ...baseIngredients, jobId: `ing-${n}`, + icReferencePaths: Array.from({ length: n }, (_, i) => `/mock/images/ref-${i}.png`), + }); + const args = findIcCall(spawnMock)[1]; + expect(args.filter((a) => a === '--ic-reference')).toHaveLength(n); + } + }); + + it('rejects a reference count outside the 2-8 contract', async () => { + await expect(generateVideo({ + ...baseIngredients, jobId: 'ing-one', icReferencePaths: ['/mock/images/owl.png'], + })).rejects.toThrow(/needs 2-8 reference image/); + await expect(generateVideo({ + ...baseIngredients, jobId: 'ing-nine', + icReferencePaths: Array.from({ length: 9 }, (_, i) => `/mock/images/ref-${i}.png`), + })).rejects.toThrow(/needs 2-8 reference image/); + }); + + it('fails fast when the weight is not downloaded (no snapshot_download fallback)', async () => { + // The registry refuses to hand the pipeline a bare repo id for this weight: + // `_resolve_lora_path` would `snapshot_download` a gated repo / a ~708 GB + // mirror. So an uncached Ingredients render must 400 rather than silently + // starting an unbounded pull. + mockInspectModelCache.mockResolvedValue({ cached: false, sizeBytes: 0, snapshotPath: null }); + await expect(generateVideo({ ...baseIngredients, jobId: 'ing-uncached' })) + .rejects.toThrow(/is not downloaded — download Ingredients/); + }); + + it('stacks user LoRAs alongside the Ingredients weight rather than replacing it', async () => { + const { spawnDetached } = await import('../../lib/detachedSpawn.js'); + const spawnMock = vi.mocked(spawnDetached); + spawnMock.mockClear(); + + await generateVideo({ + ...baseIngredients, jobId: 'ing-plus-character-lora', + loras: [{ filename: 'character.safetensors', scale: 0.9 }], + }); + + const args = findIcCall(spawnMock)[1]; + // The payoff of the Phase 1 split: the IC weight rides --ic-lora-path (→ + // `lora_paths`, fused by _fuse_loras pre-Stage-1) while the user LoRA rides + // --user-loras (→ `_pending_loras`, fused at DiT load), so an Ingredients × + // Character stack COMPOSES instead of one displacing the other. + expect(args[args.indexOf('--ic-lora-path') + 1]) + .toBe(join('/mock/hf/snap', 'ltx-2.3-22b-ic-lora-ingredients-0.9.safetensors')); + expect(JSON.parse(args[args.indexOf('--user-loras') + 1])).toEqual([ + { path: join(MOCK_PATHS.loras, 'character.safetensors'), strength: 0.9 }, + ]); + }); + + it('stamps the gallery basenames (not the temp clip paths) onto history', async () => { + const started = []; + const onStarted = (p) => started.push(p); + videoGenEvents.on('started', onStarted); + await generateVideo({ ...baseIngredients, jobId: 'ing-history' }); + videoGenEvents.off('started', onStarted); + + const meta = started.find((s) => s.generationId === 'ing-history'); + // History is user-facing: the gallery filenames are meaningful, the + // `ic-still-N-.mp4` temp encodes are not. + expect(meta.icReferenceNames).toEqual(['owl.png', 'store.png']); + expect(meta.mode).toBe('ic-ingredients'); + }); + }); + it('stamps the IC dials + reference basename onto the history record', async () => { const { spawnDetached } = await import('../../lib/detachedSpawn.js'); vi.mocked(spawnDetached).mockClear(); @@ -1977,6 +2092,39 @@ describe('icLoraArgs — direct validation (#3100)', () => { it('rejects an unresolved weight path', () => { expect(() => icLoraArgs({ ...base, icLoraWeightPath: null })) - .toThrow(/could not be resolved/); + .toThrow(/is not downloaded — download Control/); + }); + + it('enforces the 2-8 reference bounds for Ingredients and passes them to the helper (#3112)', () => { + // The count is a WEIGHT CONTRACT: a wrong one yields plausible garbage rather + // than an error inside the pipeline, so it's enforced here (and in the route, + // and again in the Python helper via the flags below). + const ing = { + ...base, mode: 'ic-ingredients', + icLoraWeightPath: '/mock/hf/snap/ingredients.safetensors', + }; + const ref = (n) => Array.from({ length: n }, (_, i) => `/mock/data/videos/ing-${i}.mp4`); + expect(() => icLoraArgs({ ...ing, icReferencePaths: ref(1) })).toThrow(/needs 2-8 reference image/); + expect(() => icLoraArgs({ ...ing, icReferencePaths: ref(9) })).toThrow(/needs 2-8 reference image/); + expect(() => icLoraArgs({ ...ing, icReferencePaths: [] })).toThrow(/needs 2-8 reference image/); + for (const n of [2, 5, 8]) { + const args = icLoraArgs({ ...ing, icReferencePaths: ref(n) }); + expect(args[args.indexOf('--ic-mode') + 1]).toBe('ingredients'); + // Bounds are PASSED, never hardcoded in the helper — one registry entry + // drives all three layers. + expect(args[args.indexOf('--ic-min-references') + 1]).toBe('2'); + expect(args[args.indexOf('--ic-max-references') + 1]).toBe('8'); + expect(args.filter((a) => a === '--ic-reference')).toHaveLength(n); + } + }); + + it('imposes no resolution-divisibility rule on Ingredients (factor 1) (#3112)', () => { + // Its safetensors metadata reports reference_downscale_factor=1 (verified by a + // Range read of the weight's header) — conditioning is full-resolution. + expect(() => icLoraArgs({ + ...base, mode: 'ic-ingredients', width: 705, height: 449, + icLoraWeightPath: '/mock/hf/snap/ingredients.safetensors', + icReferencePaths: ['/mock/a.mp4', '/mock/b.mp4'], + })).not.toThrow(); }); }); From 7d654bbe941377d54b89c0d7b893a278175cc702 Mon Sep 17 00:00:00 2001 From: "[._.]/ Adam Eivy" Date: Mon, 27 Jul 2026 02:15:41 -0700 Subject: [PATCH 2/2] address review (codex): four ways the single-file/bounds contract could still be bypassed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - findCachedRepoFile: resolve one known file in a snapshot WITHOUT walking it. icLoraWeights and sseDownload were both calling inspectModelCache on the aggregate mirror, which recursively stats every weight in the snapshot — so status, render-time resolution, download and repair could each enumerate hundreds of GB of unrelated weights. The single-file path now never asks a repo-wide question at all: sseDownload skips the walk entirely and defers to the caller's per-file predicate (defaulting to "not cached", so a caller that forgets it re-fetches instead of inheriting a wrong `cached: true`). - Still-materialization no longer leaks temp clips on partial failure. Promise.all rejected at the first bad encode while siblings were still in flight, so their files landed after cleanup had already run. Every target path is now registered before any encode starts and all encodes settle before we decide. - generate_ltx2.py's --ic-min/max-references lose their 1/1 defaults and are required for ic mode. The default let a direct `--ic-mode ingredients --ic-reference one.mp4` pass validation and render garbage — confirmed against the real weight: that invocation now exits with the contract message instead. - The Zod ceiling for both IC reference arrays derives from the registry (MAX_IC_REFERENCES) rather than a second hardcoded 8, which would have 422'd a raised per-weight maximum before the registry assertion could speak. --- scripts/generate_ltx2.py | 21 ++++- scripts/generate_ltx2_ic_lora_test.py | 28 ++++++ server/lib/README.md | 2 +- server/lib/hfCache.js | 22 +++++ server/lib/hfCache.test.js | 105 ++++++++++++++++++++++- server/lib/icLoraWeights.js | 16 ++-- server/lib/icLoraWeights.test.js | 113 +++++++++++++------------ server/lib/sseDownload.js | 35 ++++---- server/lib/sseDownload.test.js | 20 +++++ server/routes/videoGen.js | 15 +++- server/routes/videoGen.test.js | 20 ++++- server/services/videoGen/local.js | 55 ++++++------ server/services/videoGen/local.test.js | 73 ++++++++++++++-- 13 files changed, 406 insertions(+), 119 deletions(-) diff --git a/scripts/generate_ltx2.py b/scripts/generate_ltx2.py index ab95ab883a..9b5209c481 100644 --- a/scripts/generate_ltx2.py +++ b/scripts/generate_ltx2.py @@ -387,11 +387,18 @@ def parse_args() -> argparse.Namespace: p.add_argument("--ic-reference", action="append", default=None, metavar="PATH", help="Reference clip for the IC-LoRA video-conditioning channel. Repeatable; " "how many the fused weight expects is set by --ic-min/max-references.") - p.add_argument("--ic-min-references", type=int, default=1, + # NO DEFAULTS. A 1/1 default would silently green-light a direct + # `--ic-mode ingredients --ic-reference one.mp4` invocation that omitted the + # flags: the wrong reference count for a weight yields plausible-looking + # garbage rather than an error, so "caller forgot the bounds" must fail loudly + # instead of falling back to some other weight's contract. Required together. + p.add_argument("--ic-min-references", type=int, default=None, help="Fewest --ic-reference entries the fused weight accepts (from PortOS' " - "icLoraWeights registry, which owns the per-weight contract).") - p.add_argument("--ic-max-references", type=int, default=1, - help="Most --ic-reference entries the fused weight accepts.") + "icLoraWeights registry, which owns the per-weight contract). REQUIRED " + "for --mode ic, alongside --ic-max-references.") + p.add_argument("--ic-max-references", type=int, default=None, + help="Most --ic-reference entries the fused weight accepts. REQUIRED for " + "--mode ic, alongside --ic-min-references.") p.add_argument("--ic-strength", type=float, default=1.0, help="Per-reference conditioning strength (0.0-1.0+) applied to every " "--ic-reference entry.") @@ -855,6 +862,12 @@ def run_ic_lora(args: argparse.Namespace) -> str: # a weight fed the wrong reference count produces plausible-looking garbage # rather than an error, so a direct/script caller needs the guard too. lo, hi = args.ic_min_references, args.ic_max_references + if lo is None or hi is None: + raise SystemExit( + "--ic-min-references and --ic-max-references are both required for ic mode " + "(they carry the fused weight's reference contract; guessing them would let a " + "wrong reference count render plausible-looking garbage)" + ) if lo < 1 or hi < lo: raise SystemExit( f"--ic-min-references/--ic-max-references must satisfy 1 <= min <= max; got {lo}/{hi}" diff --git a/scripts/generate_ltx2_ic_lora_test.py b/scripts/generate_ltx2_ic_lora_test.py index 57764f28df..86048917ad 100644 --- a/scripts/generate_ltx2_ic_lora_test.py +++ b/scripts/generate_ltx2_ic_lora_test.py @@ -179,6 +179,34 @@ def test_rejects_nonsense_bounds_rather_than_trusting_them(self): self.helper.run_ic_lora(self._args(references=self.refs[:2], lo=0, hi=8)) self.assertIn("1 <= min <= max", str(ctx.exception)) + def test_requires_both_bounds_rather_than_defaulting_them(self): + # A 1/1 default would silently green-light a direct + # `--ic-mode ingredients --ic-reference one.mp4` that omitted the flags — + # rendering plausible-looking garbage instead of erroring. Omitting either + # bound must fail loudly. + for lo, hi in ((None, 8), (2, None), (None, None)): + with self.subTest(lo=lo, hi=hi): + with self.assertRaises(SystemExit) as ctx: + self.helper.run_ic_lora( + self._args(references=self.refs[:1], lo=lo, hi=hi) + ) + self.assertIn("both required", str(ctx.exception)) + + def test_argparse_leaves_the_bounds_unset_by_default(self): + # The flags carry no default at all — a default is exactly what would let a + # caller bypass the registry-derived contract. + argv = sys.argv + sys.argv = [ + "generate_ltx2.py", "--mode", "ic", + "--prompt", "x", "--output", "/tmp/x.mp4", "--model", "/m", + ] + try: + args = self.helper.parse_args() + finally: + sys.argv = argv + self.assertIsNone(args.ic_min_references) + self.assertIsNone(args.ic_max_references) + def test_rejects_a_reference_missing_from_disk(self): with self.assertRaises(SystemExit) as ctx: self.helper.run_ic_lora( diff --git a/server/lib/README.md b/server/lib/README.md index 7dc4533355..d992617820 100644 --- a/server/lib/README.md +++ b/server/lib/README.md @@ -115,7 +115,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub | `ansiStrip.js` | Streaming ANSI / control-byte stripper. | | `hfToken.js` | HuggingFace token resolution (settings > env > CLI): `getHfToken` / `getHfTokenInfo` / `hfTokenEnv`, plus `hfChildEnv(extra?)` for a complete Malloc-stripped child environment that preserves inherited variables while injecting the resolved token. | | `hfErrors.js` | Parse huggingface_hub gated-access errors: `isGatedRepoError(text)` classifies a failure as gated (403/restricted), `extractGatedRepo(text)` → `owner/name` (or null) for the UI's license deep-link. Shared by the image runner and MIDI transcription (LoRA trainer keeps its own regex to also match bare 401). Pure. | -| `hfCache.js` | HuggingFace Hub cache inspection (`inspectModelCache(repoId)` → `{cached,sizeBytes,snapshotPath}`, `isModelCached`, `getHfCacheRoot`). Drives the inline "Available / Download" badge on the image + video gen forms. Also `verifyModelCache(repoId,{deep})` (structural safetensors-header + optional sha256 integrity check) and `repairModelCache(repoId,{deep})` (delete corrupt weight files so the download path re-fetches them) — power the "Repair model" banner. `repairCachedFile(path)` is the single-file counterpart (unlinks one snapshot entry AND the blob behind its symlink) for a weight inside an aggregate repo, where a whole-snapshot walk would be prohibitive. `verifySafetensorsStructure(path,size)` is the reusable header/size structural check (reads only the header region) — also used by the Civitai/HF LoRA install path to reject truncated downloads. | +| `hfCache.js` | HuggingFace Hub cache inspection (`inspectModelCache(repoId)` → `{cached,sizeBytes,snapshotPath}`, `isModelCached`, `getHfCacheRoot`). Drives the inline "Available / Download" badge on the image + video gen forms. Also `verifyModelCache(repoId,{deep})` (structural safetensors-header + optional sha256 integrity check) and `repairModelCache(repoId,{deep})` (delete corrupt weight files so the download path re-fetches them) — power the "Repair model" banner. `findCachedRepoFile(repoId,filename)` resolves ONE known file in the newest snapshot **without walking it** (the whole-snapshot stat in `inspectModelCache` is prohibitive for an aggregate repo that hosts hundreds of GB of unrelated weights), and `repairCachedFile(path)` is the matching single-file repair (unlinks one snapshot entry AND the blob behind its symlink). `verifySafetensorsStructure(path,size)` is the reusable header/size structural check (reads only the header region) — also used by the Civitai/HF LoRA install path to reject truncated downloads. | | `hfDownload.js` | `downloadHfRepo({repo,onEvent})` returning `{promise,kill}` — spawns `scripts/hf_download_repo.py` in the FLUX.2 venv (fallback: mflux pythonPath) and emits SSE-friendly stage/progress/complete events. Powers the inline "Download" button next to the model picker. | | `icLoraWeights.js` | IC-LoRA weight registry for the local LTX-2 remix modes (`IC_LORA_MODES`, `IC_LORA_MODE_VALUES`, `listIcLoraWeights()`, `isIcLoraMode(mode)`, `icLoraSpecForMode(mode)`, `icLoraRepos()`, `icLoraWeightCandidates(spec)`, `findCachedIcLoraWeight(spec)`, `resolveIcLoraWeight(mode)` → `{path,cached,spec}`). Single source of truth mapping each remix mode (`ic-control`, `ic-colorize`, `ic-ingredients`) → its HF repo/filename, reference-count rule, per-weight `referenceDownscaleFactor` (read from the weight's safetensors metadata, never assumed), and input surface (`referenceKind: 'video' | 'image'`). A `requiresPreDownload` spec (Ingredients) suppresses `resolveIcLoraWeight`'s bare-repo-id fallback — the pipeline would `snapshot_download` it, and its candidates are a gated first-party repo plus a ~708 GB aggregate mirror, so the weight must be pre-fetched single-file instead. Also owns the two rules derived from that data: `assertIcReferenceCount(spec,count,fail)` / `describeIcReferenceRange(spec)` and `icResolutionIssue(spec,w,h)` (the output-resolution divisibility its reference encoder imposes). Mirrored to `client/src/lib/videoGenParams.js`; `icLoraWeights.parity.test.js` diffs the two. | | `sseHeaders.js` | `SSE_HEADERS` — the canonical SSE response headers (incl. `X-Accel-Buffering:no`) in a dependency-free module so any producer (`sseDownload.js`, `sseUtils.js`) can share them without pulling in a heavier module's transitive imports. | diff --git a/server/lib/hfCache.js b/server/lib/hfCache.js index aba16b0755..25f982b1c1 100644 --- a/server/lib/hfCache.js +++ b/server/lib/hfCache.js @@ -129,6 +129,28 @@ export async function inspectModelCache(repoId) { export const isModelCached = async (repoId) => (await inspectModelCache(repoId)).cached; +// Resolve ONE known file inside a repo's newest snapshot, without walking the +// snapshot at all. `inspectModelCache` recursively collects and stats every +// weight in the snapshot — correct when the question is "is this whole model +// downloaded?", but wrong (and expensive) for an aggregate repo where we only +// ever care about a single file: `DeepBeepMeep/LTX-2` hosts every LTX weight in +// one ~708 GB repo, so walking a populated snapshot there stats hundreds of GB +// of files that have nothing to do with the one we asked for. +// +// Returns the absolute path, or null when the repo has no snapshot or the file +// isn't resident. The `stat` FOLLOWS the symlink, so a dangling snapshot link +// left by an interrupted download reports null rather than a plausible path +// that fails on open — the same non-zero test inspectModelCache applies. +export async function findCachedRepoFile(repoId, filename) { + if (!repoId || typeof repoId !== 'string') return null; + if (!filename || typeof filename !== 'string') return null; + const snapshotPath = await latestSnapshotDir(join(getHfCacheRoot(), repoToDirName(repoId))); + if (!snapshotPath) return null; + const candidate = join(snapshotPath, filename); + const stat = await fs.stat(candidate).catch(() => null); + return stat && stat.size > 0 ? candidate : null; +} + // --------------------------------------------------------------------------- // Weight-integrity verification (issue #1324) // diff --git a/server/lib/hfCache.test.js b/server/lib/hfCache.test.js index f6830fc931..4cb5610d8d 100644 --- a/server/lib/hfCache.test.js +++ b/server/lib/hfCache.test.js @@ -5,7 +5,7 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { inspectModelCache, getHfCacheRoot, isModelCached, - verifyModelCache, repairModelCache, + verifyModelCache, repairModelCache, findCachedRepoFile, repairCachedFile, } from './hfCache.js'; // Build a minimal *valid* .safetensors buffer: 8-byte LE header length, a JSON @@ -332,3 +332,106 @@ describe('repairModelCache', () => { expect(result.deleted).toEqual([]); }); }); + +// ── Single-file probe + repair (#3112) ───────────────────────────────────── +// A weight can live inside an AGGREGATE repo: `DeepBeepMeep/LTX-2` mirrors every +// LTX weight in one ~708 GB repo. `inspectModelCache` recursively stats every +// weight in the snapshot, which there means hundreds of GB of files that have +// nothing to do with the one we asked for — so these two helpers answer the +// single-file questions without ever walking the snapshot. +describe('findCachedRepoFile', () => { + let originalEnv; + const cleanup = []; + + beforeEach(() => { originalEnv = process.env.HF_HUB_CACHE; }); + afterEach(() => { + if (originalEnv === undefined) delete process.env.HF_HUB_CACHE; + else process.env.HF_HUB_CACHE = originalEnv; + for (const dir of cleanup.splice(0)) rmSync(dir, { recursive: true, force: true }); + }); + + const aggregateCache = ({ partial = false } = {}) => { + const { root } = buildFakeCache({ + repoId: 'org/aggregate', + snapshots: { + sha1: { + 'wanted.safetensors': 1024, + 'unrelated-a.safetensors': 2048, + 'unrelated-b.safetensors': 4096, + }, + }, + partial, + }); + cleanup.push(root); + process.env.HF_HUB_CACHE = root; + return root; + }; + + it('resolves the requested file out of an aggregate repo', async () => { + aggregateCache(); + const found = await findCachedRepoFile('org/aggregate', 'wanted.safetensors'); + expect(found).toMatch(/snapshots\/sha1\/wanted\.safetensors$/); + }); + + it('returns null for a file the repo has not fetched, even though siblings exist', async () => { + // The precise failure this exists to prevent: inspectModelCache would report + // the repo `cached` off the unrelated siblings and the caller would skip a + // download for a weight the user does NOT have. + aggregateCache(); + expect(await findCachedRepoFile('org/aggregate', 'never-fetched.safetensors')).toBeNull(); + expect((await inspectModelCache('org/aggregate')).cached).toBe(true); + }); + + it('returns null for a dangling snapshot symlink (interrupted download)', async () => { + // The stat FOLLOWS the link, so a link with no blob behind it reports null + // rather than a plausible path that fails on open inside the render. + aggregateCache({ partial: true }); + expect(await findCachedRepoFile('org/aggregate', 'wanted.safetensors')).toBeNull(); + }); + + it('returns null when the repo has no snapshot at all', async () => { + aggregateCache(); + expect(await findCachedRepoFile('org/never-downloaded', 'wanted.safetensors')).toBeNull(); + }); + + it('returns null for a missing repo id or filename', async () => { + aggregateCache(); + expect(await findCachedRepoFile('', 'wanted.safetensors')).toBeNull(); + expect(await findCachedRepoFile(null, 'wanted.safetensors')).toBeNull(); + expect(await findCachedRepoFile('org/aggregate', '')).toBeNull(); + expect(await findCachedRepoFile('org/aggregate', null)).toBeNull(); + }); +}); + +describe('repairCachedFile', () => { + const cleanup = []; + afterEach(() => { + for (const dir of cleanup.splice(0)) rmSync(dir, { recursive: true, force: true }); + }); + + it('unlinks the snapshot entry AND the blob behind it, leaving siblings alone', async () => { + // Deleting only the symlink is not enough: `hf_hub_download` keys on the + // cached etag rather than the content, so a stale blob with the right name + // would be trusted and never re-fetched. + const { root, repoDir } = buildFakeCache({ + repoId: 'org/aggregate', + snapshots: { sha1: { 'wanted.safetensors': 1024, 'keep.safetensors': 2048 } }, + }); + cleanup.push(root); + const target = join(repoDir, 'snapshots', 'sha1', 'wanted.safetensors'); + const blob = readlinkSync(target); + const keep = join(repoDir, 'snapshots', 'sha1', 'keep.safetensors'); + + expect(await repairCachedFile(target)).toBe(true); + expect(existsSync(target)).toBe(false); + expect(existsSync(blob)).toBe(false); + // The aggregate repo's other weights are untouched — that's the whole point + // of not routing through repairModelCache here. + expect(existsSync(keep)).toBe(true); + }); + + it('reports false for a path that is not there', async () => { + expect(await repairCachedFile('/nope/missing.safetensors')).toBe(false); + expect(await repairCachedFile(null)).toBe(false); + }); +}); diff --git a/server/lib/icLoraWeights.js b/server/lib/icLoraWeights.js index 6199669b26..1ebe0fdedf 100644 --- a/server/lib/icLoraWeights.js +++ b/server/lib/icLoraWeights.js @@ -28,9 +28,7 @@ // so `gated: true` marks it and the mirror provides an un-gated path for users // without an HF token. -import { join } from 'node:path'; -import { existsSync } from 'node:fs'; -import { inspectModelCache } from './hfCache.js'; +import { findCachedRepoFile } from './hfCache.js'; // One entry per shipped remix mode. `minReferences`/`maxReferences` are the // weight's contract: the Python helper receives them as flags (never a second @@ -213,13 +211,11 @@ export const icResolutionIssue = (spec, width, height) => { // (`{ path, repo, filename, mirror }`) or null when nothing is cached. export const findCachedIcLoraWeight = async (spec) => { for (const candidate of icLoraWeightCandidates(spec)) { - const { snapshotPath } = await inspectModelCache(candidate.repo); - if (!snapshotPath) continue; - const exact = join(snapshotPath, candidate.filename); - // `existsSync` FOLLOWS symlinks, so a dangling snapshot link left by an - // interrupted download reports false here — the same "is it really resident?" - // question inspectModelCache asks of a whole snapshot. - if (existsSync(exact)) return { ...candidate, path: exact }; + // findCachedRepoFile, NOT inspectModelCache: the latter recursively walks and + // stats every weight in the snapshot, which for the aggregate mirror means + // hundreds of GB of unrelated files. This resolves the one filename directly. + const path = await findCachedRepoFile(candidate.repo, candidate.filename); + if (path) return { ...candidate, path }; } return null; }; diff --git a/server/lib/icLoraWeights.test.js b/server/lib/icLoraWeights.test.js index 89947575f6..2f4f511e12 100644 --- a/server/lib/icLoraWeights.test.js +++ b/server/lib/icLoraWeights.test.js @@ -5,12 +5,8 @@ import { join } from 'node:path'; // cache and existsSync probes the pinned weight file. Mock them so these tests // cover the registry + resolution logic rather than the user's ~/.cache layout // (hfCache.test.js already covers the cache walk). -const { mockInspectModelCache, mockExistsSync } = vi.hoisted(() => ({ - mockInspectModelCache: vi.fn(), - mockExistsSync: vi.fn(), -})); -vi.mock('./hfCache.js', () => ({ inspectModelCache: mockInspectModelCache })); -vi.mock('node:fs', () => ({ existsSync: mockExistsSync })); +const { mockFindCachedRepoFile } = vi.hoisted(() => ({ mockFindCachedRepoFile: vi.fn() })); +vi.mock('./hfCache.js', () => ({ findCachedRepoFile: mockFindCachedRepoFile })); const { IC_LORA_MODES, IC_LORA_MODE_VALUES, isIcLoraMode, icLoraSpecForMode, @@ -19,8 +15,7 @@ const { } = await import('./icLoraWeights.js'); beforeEach(() => { - mockInspectModelCache.mockReset(); - mockExistsSync.mockReset(); + mockFindCachedRepoFile.mockReset(); }); describe('IC-LoRA registry', () => { @@ -98,9 +93,18 @@ describe('IC-LoRA registry', () => { }); describe('resolveIcLoraWeight', () => { + // `findCachedRepoFile(repo, filename)` returns the absolute path or null. It is + // deliberately NOT inspectModelCache: that walks + stats the whole snapshot, + // which for an aggregate mirror means hundreds of GB of unrelated weights. + const cacheHas = (...pairs) => mockFindCachedRepoFile.mockImplementation( + async (repo, filename) => { + const hit = pairs.find((p) => p.repo === repo && p.filename === filename); + return hit ? join(hit.snapshot, filename) : null; + }, + ); + it('pins the exact filename inside the cached snapshot', async () => { - mockInspectModelCache.mockResolvedValue({ cached: true, sizeBytes: 1, snapshotPath: '/hf/snap' }); - mockExistsSync.mockReturnValue(true); + cacheHas({ repo: IC_LORA_MODES.control.repo, filename: IC_LORA_MODES.control.filename, snapshot: '/hf/snap' }); const resolved = await resolveIcLoraWeight('ic-control'); // Pinning the filename (rather than returning the snapshot dir or the repo @@ -111,29 +115,31 @@ describe('resolveIcLoraWeight', () => { }); it('pins each mode to its OWN filename, not the first registry entry', async () => { - mockInspectModelCache.mockResolvedValue({ cached: true, sizeBytes: 1, snapshotPath: '/hf/snap' }); - mockExistsSync.mockReturnValue(true); + cacheHas({ repo: IC_LORA_MODES.colorize.repo, filename: IC_LORA_MODES.colorize.filename, snapshot: '/hf/snap' }); const resolved = await resolveIcLoraWeight('ic-colorize'); expect(resolved.path).toBe(join('/hf/snap', IC_LORA_MODES.colorize.filename)); expect(resolved.spec).toBe(IC_LORA_MODES.colorize); - expect(mockInspectModelCache).toHaveBeenCalledWith(IC_LORA_MODES.colorize.repo); - }); - - it('falls back to the repo id when no snapshot exists', async () => { - mockInspectModelCache.mockResolvedValue({ cached: false, sizeBytes: 0, snapshotPath: null }); - - const resolved = await resolveIcLoraWeight('ic-control'); - expect(resolved.path).toBe(IC_LORA_MODES.control.repo); - expect(resolved.cached).toBe(false); + expect(mockFindCachedRepoFile).toHaveBeenCalledWith( + IC_LORA_MODES.colorize.repo, IC_LORA_MODES.colorize.filename, + ); + }); + + it('never asks for a whole-snapshot walk, only exact filenames (#3112)', async () => { + // The single-file invariant at the resolution layer: every probe names a file. + cacheHas(); + await resolveIcLoraWeight('ic-ingredients'); + expect(mockFindCachedRepoFile).toHaveBeenCalledTimes(2); + for (const [, filename] of mockFindCachedRepoFile.mock.calls) { + expect(filename).toMatch(/\.safetensors$/); + } }); - it('falls back to the repo id when the snapshot lacks the pinned file', async () => { - // A partial download leaves the snapshot dir present but the weight absent — - // returning the dir-joined path would hand the pipeline a missing file, so - // the repo-id fallback (which re-downloads) is the correct degrade. - mockInspectModelCache.mockResolvedValue({ cached: true, sizeBytes: 1, snapshotPath: '/hf/snap' }); - mockExistsSync.mockReturnValue(false); + it('falls back to the repo id when the weight is not resident', async () => { + // Covers both "no snapshot at all" and "snapshot present but the pinned file + // missing / a dangling symlink" — findCachedRepoFile collapses them to null, + // and the repo-id fallback (which re-downloads) is the correct degrade. + cacheHas(); const resolved = await resolveIcLoraWeight('ic-control'); expect(resolved.path).toBe(IC_LORA_MODES.control.repo); @@ -142,7 +148,7 @@ describe('resolveIcLoraWeight', () => { it('returns null for an unknown mode without touching the cache', async () => { expect(await resolveIcLoraWeight('ic-nope')).toBeNull(); - expect(mockInspectModelCache).not.toHaveBeenCalled(); + expect(mockFindCachedRepoFile).not.toHaveBeenCalled(); }); it('SUPPRESSES the repo-id fallback for a requiresPreDownload weight (#3112)', async () => { @@ -152,7 +158,7 @@ describe('resolveIcLoraWeight', () => { // DeepBeepMeep/LTX-2 aggregate, which would fill the user's disk. Neither id // may ever reach the pipeline — path must be null so icLoraArgs 400s with // "download the weight first". - mockInspectModelCache.mockResolvedValue({ cached: false, sizeBytes: 0, snapshotPath: null }); + cacheHas(); const resolved = await resolveIcLoraWeight('ic-ingredients'); expect(resolved.path).toBeNull(); @@ -162,16 +168,15 @@ describe('resolveIcLoraWeight', () => { expect(resolved.path).not.toBe(IC_LORA_MODES.ingredients.mirrorRepo); }); - it('still resolves a requiresPreDownload weight from the mirror snapshot', async () => { - // Official repo has no snapshot; the mirror does. The candidate walk must find - // it there rather than giving up (that's what makes the un-gated path work for - // a user with no HF token). - mockInspectModelCache.mockImplementation(async (repo) => ( - repo === IC_LORA_MODES.ingredients.mirrorRepo - ? { cached: true, sizeBytes: 1, snapshotPath: '/hf/mirror-snap' } - : { cached: false, sizeBytes: 0, snapshotPath: null } - )); - mockExistsSync.mockReturnValue(true); + it('still resolves a requiresPreDownload weight from the mirror', async () => { + // Official repo doesn't have it; the mirror does. The candidate walk must find + // it there rather than giving up — that's what makes the un-gated path work + // for a user with no HF token. + cacheHas({ + repo: IC_LORA_MODES.ingredients.mirrorRepo, + filename: IC_LORA_MODES.ingredients.mirrorFilename, + snapshot: '/hf/mirror-snap', + }); const resolved = await resolveIcLoraWeight('ic-ingredients'); expect(resolved.path).toBe(join('/hf/mirror-snap', IC_LORA_MODES.ingredients.mirrorFilename)); @@ -179,17 +184,17 @@ describe('resolveIcLoraWeight', () => { expect(resolved.repo).toBe(IC_LORA_MODES.ingredients.mirrorRepo); }); - it('prefers the official repo over the mirror when both are cached', async () => { - mockInspectModelCache.mockImplementation(async (repo) => ({ - cached: true, - sizeBytes: 1, - snapshotPath: repo === IC_LORA_MODES.ingredients.repo ? '/hf/official' : '/hf/mirror', - })); - mockExistsSync.mockReturnValue(true); + it('prefers the official repo over the mirror when both have it', async () => { + cacheHas( + { repo: IC_LORA_MODES.ingredients.repo, filename: IC_LORA_MODES.ingredients.filename, snapshot: '/hf/official' }, + { repo: IC_LORA_MODES.ingredients.mirrorRepo, filename: IC_LORA_MODES.ingredients.mirrorFilename, snapshot: '/hf/mirror' }, + ); const resolved = await resolveIcLoraWeight('ic-ingredients'); expect(resolved.repo).toBe(IC_LORA_MODES.ingredients.repo); expect(resolved.path).toBe(join('/hf/official', IC_LORA_MODES.ingredients.filename)); + // Short-circuits: the mirror is never even probed once the official hit lands. + expect(mockFindCachedRepoFile).toHaveBeenCalledTimes(1); }); }); @@ -215,12 +220,16 @@ describe('icLoraWeightCandidates', () => { }); describe('findCachedIcLoraWeight', () => { - it('treats a dangling snapshot symlink as not cached', async () => { - // existsSync FOLLOWS symlinks, so an interrupted download (link present, blob - // gone) must NOT count as resident — otherwise the render hands the pipeline a - // path that fails on open. - mockInspectModelCache.mockResolvedValue({ cached: true, sizeBytes: 1, snapshotPath: '/hf/snap' }); - mockExistsSync.mockReturnValue(false); + it('returns null when no candidate has the file resident', async () => { + // findCachedRepoFile already collapses "no snapshot", "file absent" and + // "dangling symlink / zero bytes" into null (hfCache.test.js covers those); + // this asserts the candidate walk gives up rather than returning a bad path. + mockFindCachedRepoFile.mockResolvedValue(null); expect(await findCachedIcLoraWeight(IC_LORA_MODES.ingredients)).toBeNull(); }); + + it('returns nothing for a null spec without probing the cache', async () => { + expect(await findCachedIcLoraWeight(null)).toBeNull(); + expect(mockFindCachedRepoFile).not.toHaveBeenCalled(); + }); }); diff --git a/server/lib/sseDownload.js b/server/lib/sseDownload.js index b734796925..2baabd11aa 100644 --- a/server/lib/sseDownload.js +++ b/server/lib/sseDownload.js @@ -93,14 +93,19 @@ export async function startHfDownloadStream({ req, res, repo, repos, fallbacks, const { repo: r, only: onlyFiles } = targets[i]; const isLastTarget = i === targets.length - 1; if (aborted) return; - const existing = await inspectModelCache(r); + // For a single-file pull the repo-wide cache verdict is both the wrong + // question AND too expensive to ask: `inspectModelCache` recursively stats + // every weight in the snapshot, and an aggregate mirror reports `cached` off + // any unrelated resident file — so it would skip the file the user actually + // asked for after walking hundreds of GB to get there. Skip the walk entirely + // and defer to the caller's per-file predicate. + const singleFile = onlyFiles.length > 0; + const existing = singleFile + ? { cached: false, sizeBytes: 0 } + : await inspectModelCache(r); if (aborted) return; - // For a single-file pull the repo-wide cache verdict is the wrong question: - // an aggregate mirror reports `cached` the moment any unrelated weight is - // resident, which would skip the file the user actually asked for. Defer to - // the caller's per-file predicate when one is supplied. - const alreadyHave = onlyFiles.length && typeof cachedFile === 'function' - ? await cachedFile() + const alreadyHave = singleFile + ? (typeof cachedFile === 'function' ? await cachedFile() : false) : existing.cached; if (aborted) return; // `force` is set by a repair-initiated re-download: repairModelCache() @@ -110,11 +115,11 @@ export async function startHfDownloadStream({ req, res, repo, repos, fallbacks, // re-runs the HF fetch, which is a cheap no-op for files already present // (etag match) and pulls only what's gone. if (alreadyHave && !force) { - // A single-file pull must NOT claim the repo's resident total: for the - // aggregate mirror that's every unrelated weight the user has (hundreds of - // GB), which would be a wildly wrong number on the badge. Report 0 there - // and let the badge fall back to the registry's size estimate. - if (onlyFiles.length) { + // A single-file pull has no repo-wide total to report (we deliberately + // never computed one) — and claiming the aggregate mirror's resident bytes + // would be a wildly wrong number on the badge anyway. Report 0 and let the + // badge fall back to the registry's size estimate. + if (singleFile) { console.log(`📦 HuggingFace weight already cached: ${onlyFiles.join(', ')} (${r})`); send({ type: 'log', message: `${onlyFiles.join(', ')} already cached.`, repo: r, sizeBytes: 0 }); } else { @@ -130,7 +135,7 @@ export async function startHfDownloadStream({ req, res, repo, repos, fallbacks, // Dedupe key includes the filenames: an aggregate repo hosts many unrelated // weights, so two single-file pulls of DIFFERENT files from the same repo are // not duplicates and must not block each other. - const flightKey = onlyFiles.length ? `${r}::${onlyFiles.join(',')}` : r; + const flightKey = singleFile ? `${r}::${onlyFiles.join(',')}` : r; if (inFlight.has(flightKey)) { console.log(`⏭️ HuggingFace download already running for ${flightKey} — refusing duplicate`); send({ type: 'error', message: `Another download for ${r} is already running.`, kind: 'already_running', repo: r }); @@ -139,10 +144,10 @@ export async function startHfDownloadStream({ req, res, repo, repos, fallbacks, // Server-side visibility for a pull that used to surface only on the SSE // stream (the browser) — a headless/PM2 log had no record the multi-GB // fetch ever ran. Log the start, each file as it streams, and the outcome. - console.log(`⬇️ Downloading HuggingFace repo: ${r}${onlyFiles.length ? ` (${onlyFiles.length} file(s) only)` : ''}${force ? ' (forced re-fetch)' : ''}`); + console.log(`⬇️ Downloading HuggingFace repo: ${r}${singleFile ? ` (${onlyFiles.length} file(s) only)` : ''}${force ? ' (forced re-fetch)' : ''}`); const handle = downloadHfRepo({ repo: r, - only: onlyFiles.length ? onlyFiles : null, + only: singleFile ? onlyFiles : null, onEvent: (ev) => { if (ev.type === 'progress' && ev.file) { console.log(`⬇️ ${r}: ${ev.file} (${ev.step}/${ev.total})`); diff --git a/server/lib/sseDownload.test.js b/server/lib/sseDownload.test.js index 0f20abb88f..028b67ca42 100644 --- a/server/lib/sseDownload.test.js +++ b/server/lib/sseDownload.test.js @@ -296,6 +296,26 @@ describe('startHfDownloadStream single-file + fallbacks (#3112)', () => { expect(parseFrames(frames).at(-1).sizeBytes).toBe(0); }); + it('never inspects the whole repo for a single-file pull (#3112)', async () => { + // inspectModelCache recursively stats EVERY weight in the snapshot. Against a + // populated 708 GB mirror that's the expensive walk the single-file path + // exists to avoid — and its verdict would be wrong anyway (cached off any + // unrelated resident file). It must not be called at all. + const { req, res } = makeReqRes(); + await startHfDownloadStream({ req, res, fallbacks: CANDIDATES, cachedFile: async () => false }); + expect(inspectModelCache).not.toHaveBeenCalled(); + }); + + it('does NOT treat a single-file pull as cached when no predicate is supplied', async () => { + // Fail-safe direction: a caller that forgot `cachedFile` must re-fetch (cheap, + // etag no-op) rather than inherit the repo-wide `cached: true` and skip a + // weight the user doesn't have. + inspectModelCache.mockResolvedValue({ cached: true, sizeBytes: 700e9, snapshotPath: '/snap' }); + const { req, res } = makeReqRes(); + await startHfDownloadStream({ req, res, fallbacks: [CANDIDATES[0]] }); + expect(downloadHfRepo).toHaveBeenCalledTimes(1); + }); + it('rejects an empty fallbacks list rather than silently completing', async () => { const { req, res, frames } = makeReqRes(); await startHfDownloadStream({ req, res, fallbacks: [] }); diff --git a/server/routes/videoGen.js b/server/routes/videoGen.js index ec427c85b3..7feb3888fd 100644 --- a/server/routes/videoGen.js +++ b/server/routes/videoGen.js @@ -118,6 +118,12 @@ const optionalInt = (min, max, label) => z.preprocess( (v) => v == null || v === '' ? undefined : Number(v), z.number().int().refine((n) => n >= min && n <= max, `${label} ${min}..${max}`).optional(), ); +// Coarse upper bound for any IC reference array, derived from the registry so a +// weight raising its own maxReferences doesn't get rejected by a stale literal in +// the schema before the per-mode assertion below can speak. Per-weight bounds are +// still enforced against the mode's own spec (assertIcReferenceCount). +const MAX_IC_REFERENCES = Math.max(...listIcLoraWeights().map((s) => s.maxReferences)); + const generateBodySchema = z.object({ // Render backend: the local runtimes (default) or the Grok Build CLI's // image-first image_to_video flow (#2859 phase 2). Grok ignores the @@ -180,7 +186,7 @@ const generateBodySchema = z.object({ } return v; }, - z.array(z.string().guid()).min(1).max(8).optional(), + z.array(z.string().guid()).min(1).max(MAX_IC_REFERENCES).optional(), ), // Ingredients-style IC references: 2-8 gallery STILLS, not clips. A separate // field from icReference / icReferenceVideoIds on purpose — those are @@ -199,7 +205,12 @@ const generateBodySchema = z.object({ } return v; }, - z.array(z.string().min(1).max(512)).min(1).max(8).optional(), + // Ceiling derived from the registry (the largest maxReferences any weight + // declares), NOT a hardcoded 8 — a second literal here would silently + // pre-empt the per-mode registry check with a 422 the moment a weight raised + // its own maximum. This is only a coarse sanity bound; the real per-weight + // rule is asserted below against the mode's own spec. + z.array(z.string().min(1).max(512)).min(1).max(MAX_IC_REFERENCES).optional(), ), // Reference-video conditioning strength for the IC-LoRA channel. Distinct // from the IC-LoRA's own fusion strength (fixed at 1.0 server-side) and from diff --git a/server/routes/videoGen.test.js b/server/routes/videoGen.test.js index 4a53a0e085..89bf2e4d4d 100644 --- a/server/routes/videoGen.test.js +++ b/server/routes/videoGen.test.js @@ -122,6 +122,7 @@ vi.mock('fs/promises', () => ({ import * as videoGenService from '../services/videoGen/local.js'; import * as mediaJobQueue from '../services/mediaJobQueue/index.js'; import { resolveGalleryImage } from '../lib/fileUtils.js'; +import { listIcLoraWeights } from '../lib/icLoraWeights.js'; import videoGenRoutes, { isAudioMime } from './videoGen.js'; // isAudioMime is the gating function inside the fileFilter callback. The @@ -1025,14 +1026,29 @@ describe('videoGen routes', () => { expect(mediaJobQueue.enqueueJob).not.toHaveBeenCalled(); }); - it('rejects more than 8 references at the schema boundary', async () => { + it('rejects more references than any registered weight accepts', async () => { + // The schema ceiling is DERIVED from the registry (the largest + // maxReferences declared), not a second hardcoded 8 that would pre-empt + // the per-weight check with a 422 the moment a weight raised its limit. + const ceiling = Math.max(...listIcLoraWeights().map((w) => w.maxReferences)); const r = await request(app).post('/api/video-gen/').send({ - prompt: 'recompose', mode: 'ic-ingredients', icReferenceImageFiles: stills(9), + prompt: 'recompose', mode: 'ic-ingredients', icReferenceImageFiles: stills(ceiling + 1), }); expect(r.status).toBe(400); expect(mediaJobQueue.enqueueJob).not.toHaveBeenCalled(); }); + it('lets the registry bound speak at the ceiling rather than the schema', async () => { + // At exactly the registry maximum the request must reach the per-mode + // assertion (and pass), proving the coarse schema bound isn't the one + // enforcing the weight contract. + const ceiling = Math.max(...listIcLoraWeights().map((w) => w.maxReferences)); + const r = await request(app).post('/api/video-gen/').send({ + prompt: 'recompose', mode: 'ic-ingredients', icReferenceImageFiles: stills(ceiling), + }); + expect(r.status).toBe(200); + }); + it('rejects no references with an actionable message', async () => { const r = await request(app).post('/api/video-gen/').send({ prompt: 'recompose', mode: 'ic-ingredients', diff --git a/server/services/videoGen/local.js b/server/services/videoGen/local.js index cff96f5449..5de503253f 100644 --- a/server/services/videoGen/local.js +++ b/server/services/videoGen/local.js @@ -919,33 +919,34 @@ export async function generateVideo({ pythonPath, prompt, negativePrompt = '', m { status: 400, code: 'IC_LORA_STILL_NEEDS_FFMPEG' }, ); } - resolvedIcReferencePaths = await Promise.all(icReferencePaths.map(async (stillPath, i) => { - const clipPath = join(tmpdir(), `ic-still-${i}-${jobId}.mp4`); - const result = await execFileAsync(stillFfmpeg, [ - '-loop', '1', '-i', stillPath, - '-vf', `scale=${w}:${h}:force_original_aspect_ratio=increase,crop=${w}:${h}`, - '-frames:v', String(IC_STILL_REFERENCE_FRAMES), - '-r', String(parsedFps), - '-pix_fmt', 'yuv420p', '-an', - '-y', clipPath, - ], { env: safeChildProcessEnv(), timeout: 30000 }).catch((err) => ({ error: err })); - if (result.error) { - // Unlike the resizeImage fallback (which degrades to the original), there - // is no usable degradation here — a still handed straight to the pipeline - // fails deep inside the VAE reshape. Fail loudly with the ffmpeg reason. - throw new ServerError( - `Failed to prepare Ingredients reference ${basename(stillPath)}: ${result.error.message}`, - { status: 400, code: 'IC_LORA_STILL_PREP_FAILED' }, - ); - } - icReferenceTempPaths.push(clipPath); - return clipPath; - })).catch(async (err) => { - // Partial success leaks the clips already written — one still failing - // aborts the render, so drop them before rethrowing. - for (const p of icReferenceTempPaths) await unlink(p).catch(() => {}); - throw err; - }); + // Register EVERY target path up-front, before any encode starts, and settle + // all of them before deciding. `Promise.all` + push-on-success would reject + // at the first failure while sibling encodes were still in flight, so their + // files would land after cleanup already ran and leak. Registering the paths + // eagerly also means the outer error/close handlers can clean up regardless + // of which encodes finished. + const clipPaths = icReferencePaths.map((_, i) => join(tmpdir(), `ic-still-${i}-${jobId}.mp4`)); + icReferenceTempPaths.push(...clipPaths); + const encodes = await Promise.all(icReferencePaths.map((stillPath, i) => execFileAsync(stillFfmpeg, [ + '-loop', '1', '-i', stillPath, + '-vf', `scale=${w}:${h}:force_original_aspect_ratio=increase,crop=${w}:${h}`, + '-frames:v', String(IC_STILL_REFERENCE_FRAMES), + '-r', String(parsedFps), + '-pix_fmt', 'yuv420p', '-an', + '-y', clipPaths[i], + ], { env: safeChildProcessEnv(), timeout: 30000 }).catch((err) => ({ error: err })))); + const failedAt = encodes.findIndex((r) => r?.error); + if (failedAt !== -1) { + // Unlike the resizeImage fallback (which degrades to the original), there is + // no usable degradation here — a still handed straight to the pipeline fails + // deep inside the VAE reshape. Fail loudly with the ffmpeg reason. + for (const p of clipPaths) await unlink(p).catch(() => {}); + throw new ServerError( + `Failed to prepare Ingredients reference ${basename(icReferencePaths[failedAt])}: ${encodes[failedAt].error.message}`, + { status: 400, code: 'IC_LORA_STILL_PREP_FAILED' }, + ); + } + resolvedIcReferencePaths = clipPaths; } const meta = { diff --git a/server/services/videoGen/local.test.js b/server/services/videoGen/local.test.js index e54f6e7b5a..2d36f11a64 100644 --- a/server/services/videoGen/local.test.js +++ b/server/services/videoGen/local.test.js @@ -77,11 +77,16 @@ vi.mock('../../lib/hfToken.js', () => ({ // cache walk (which hfCache.test.js covers). `snapshotPath` non-null + the // always-true existsSync mock below means the weight resolves to its exact // pinned filename; a test that wants the un-cached fallback overrides this. -const { mockInspectModelCache } = vi.hoisted(() => ({ +const { mockInspectModelCache, mockFindCachedRepoFile } = vi.hoisted(() => ({ mockInspectModelCache: vi.fn(async () => ({ cached: true, sizeBytes: 1000, snapshotPath: '/mock/hf/snap' })), + // icLoraWeights resolves an IC weight via the exact-file probe (NOT the + // snapshot walk) so an aggregate mirror is never enumerated. Default to + // "resident", derived from the same mocked snapshot path the tests assert on. + mockFindCachedRepoFile: vi.fn(async (_repo, filename) => join('/mock/hf/snap', filename)), })); vi.mock('../../lib/hfCache.js', () => ({ inspectModelCache: mockInspectModelCache, + findCachedRepoFile: mockFindCachedRepoFile, })); vi.mock('fs', () => ({ @@ -1757,7 +1762,9 @@ describe('generateVideo — IC-LoRA remix arg threading (#3100)', () => { ); beforeEach(() => { - mockInspectModelCache.mockResolvedValue({ cached: true, sizeBytes: 1000, snapshotPath: '/mock/hf/snap' }); + // Every weight resident. The probe is per-FILE (findCachedRepoFile) rather + // than a snapshot walk, so an aggregate mirror is never enumerated. + mockFindCachedRepoFile.mockImplementation(async (_repo, filename) => join('/mock/hf/snap', filename)); }); it('routes ic-control to --mode ic with the pinned weight, reference, and strength', async () => { @@ -1812,7 +1819,7 @@ describe('generateVideo — IC-LoRA remix arg threading (#3100)', () => { const { spawnDetached } = await import('../../lib/detachedSpawn.js'); const spawnMock = vi.mocked(spawnDetached); spawnMock.mockClear(); - mockInspectModelCache.mockResolvedValue({ cached: false, sizeBytes: 0, snapshotPath: null }); + mockFindCachedRepoFile.mockResolvedValue(null); await generateVideo({ ...baseIcRender, jobId: 'ic-uncached' }); @@ -1825,7 +1832,7 @@ describe('generateVideo — IC-LoRA remix arg threading (#3100)', () => { const { spawnDetached } = await import('../../lib/detachedSpawn.js'); const spawnMock = vi.mocked(spawnDetached); spawnMock.mockClear(); - mockInspectModelCache.mockResolvedValue({ cached: false, sizeBytes: 0, snapshotPath: null }); + mockFindCachedRepoFile.mockResolvedValue(null); await generateVideo({ ...baseIcRender, jobId: 'ic-colorize-uncached', mode: 'ic-colorize' }); @@ -1988,7 +1995,7 @@ describe('generateVideo — IC-LoRA remix arg threading (#3100)', () => { // `_resolve_lora_path` would `snapshot_download` a gated repo / a ~708 GB // mirror. So an uncached Ingredients render must 400 rather than silently // starting an unbounded pull. - mockInspectModelCache.mockResolvedValue({ cached: false, sizeBytes: 0, snapshotPath: null }); + mockFindCachedRepoFile.mockResolvedValue(null); await expect(generateVideo({ ...baseIngredients, jobId: 'ing-uncached' })) .rejects.toThrow(/is not downloaded — download Ingredients/); }); @@ -2015,6 +2022,62 @@ describe('generateVideo — IC-LoRA remix arg threading (#3100)', () => { ]); }); + it('cleans up EVERY temp clip when one still fails to encode', async () => { + // Promise.all rejects at the first failure while siblings are still in + // flight, so a push-on-success registry would miss the ones that landed + // afterwards. Every target path is registered before any encode starts. + const { execFile } = await import('child_process'); + const { unlink } = await import('fs/promises'); + const execFileMock = vi.mocked(execFile); + const unlinkMock = vi.mocked(unlink); + execFileMock.mockClear(); + unlinkMock.mockClear(); + + // Fail the SECOND still; the first and third still succeed. + let stillIndex = 0; + execFileMock.mockImplementation((_bin, args, _opts, cb) => { + if (args.includes('-loop')) { + const mine = stillIndex++; + if (mine === 1) return cb?.(new Error('ffmpeg exploded')); + } + return cb?.(null, '', ''); + }); + + await expect(generateVideo({ + ...baseIngredients, jobId: 'ing-partial-fail', + icReferencePaths: ['/mock/images/a.png', '/mock/images/b.png', '/mock/images/c.png'], + })).rejects.toThrow(/Failed to prepare Ingredients reference b\.png/); + + // All three temp paths unlinked, not just the ones that had resolved when + // the rejection fired. + const unlinked = unlinkMock.mock.calls.map(([p]) => String(p)); + for (const i of [0, 1, 2]) { + expect(unlinked.some((p) => p.includes(`ic-still-${i}-ing-partial-fail.mp4`))).toBe(true); + } + // The ORIGINAL gallery stills are the user's files and must survive. + for (const f of ['/mock/images/a.png', '/mock/images/b.png', '/mock/images/c.png']) { + expect(unlinked).not.toContain(f); + } + execFileMock.mockImplementation((_bin, _args, _opts, cb) => cb?.(null, '', '')); + }); + + it('cleans up the temp clips after a successful render', async () => { + const { unlink } = await import('fs/promises'); + const unlinkMock = vi.mocked(unlink); + unlinkMock.mockClear(); + + await generateVideo({ ...baseIngredients, jobId: 'ing-cleanup' }); + // Success-path cleanup runs in the child's async close handler, which fires + // after generateVideo resolves. + await vi.waitFor(() => expect(unlinkMock).toHaveBeenCalled()); + + const unlinked = unlinkMock.mock.calls.map(([p]) => String(p)); + for (const i of [0, 1]) { + expect(unlinked.some((p) => p.includes(`ic-still-${i}-ing-cleanup.mp4`))).toBe(true); + } + expect(unlinked).not.toContain('/mock/images/owl.png'); + }); + it('stamps the gallery basenames (not the temp clip paths) onto history', async () => { const started = []; const onStarted = (p) => started.push(p);