From 28744f012cd26bef08cfc7ab1b2ec00338696c3f Mon Sep 17 00:00:00 2001 From: ashish-aesthisia Date: Thu, 6 Aug 2026 17:59:15 +0000 Subject: [PATCH 1/3] clip lora training minimax --- README.md | 2 +- TRAINING.md | 54 ++++++++++++--- core/pyproject.toml | 2 + core/src/inline_core/training/arch.py | 16 +++++ core/src/inline_core/training/cache.py | 3 +- core/src/inline_core/training/caption.py | 21 +++++- core/src/inline_core/training/dataset.py | 20 ++++-- core/src/inline_core/training/h3.py | 68 ++++++++++++++----- core/src/inline_core/training/trainer.py | 17 +++-- core/tests/test_minimaxh3_training.py | 58 ++++++++++++++++ .../views/Trainer/DatasetItemsGrid.tsx | 52 +++++++++----- .../views/Trainer/TrainerSettingsPanel.tsx | 23 ++++++- src/shared/types.ts | 5 ++ 13 files changed, 284 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index b6978ae..af26c96 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Inline Studio is a free, open-source app for **AI filmmaking on a node canvas**, - **Non-destructive by default** - every render is kept as a versioned take; generating again adds one, nothing is overwritten. - **Local diffusion generation engine** - the built-in Inline Core engine runs popular diffusion models locally, on your own GPU, from a single model file, no external server. Currently supported: **Z-Image Turbo**, **Krea 2** (RAW + Turbo), **FLUX.2**, and **MiniMax H3** for video with sound. -- **Train LoRAs locally** - the Trainer canvas fine-tunes Z-Image, Krea 2, FLUX.2 or MiniMax H3 on your own images, on your own GPU. With a 4-bit base, Krea 2 trains at 512px inside about 12GB, so a 16GB card can train a LoRA for a 26GB model. See [LoRA training](#lora-training). +- **Train LoRAs locally** - the Trainer canvas fine-tunes Z-Image, Krea 2, FLUX.2 or MiniMax H3 on your own images, on your own GPU. H3 also trains on short video clips, so a LoRA can learn motion and not just look. With a 4-bit base, Krea 2 trains at 512px inside about 12GB, so a 16GB card can train a LoRA for a 26GB model. See [LoRA training](#lora-training). - **Hosted models via API Nodes** - reach for closed models with no GPU and no setup for instant creative range; see [API Nodes](#api-nodes). - **Mix both in the same film** - Inline Studio handles everything around the render: exploring options, keeping what works, and shaping a repeatable process you can iterate on and share. diff --git a/TRAINING.md b/TRAINING.md index 9564096..b487d98 100644 --- a/TRAINING.md +++ b/TRAINING.md @@ -12,7 +12,7 @@ resolution. **Contents:** [The graph](#the-graph) · [Datasets and outputs](#datasets-and-outputs) · [Stop and resume](#stop-and-resume) · [Trigger words](#trigger-words) · [Architecture and base model modes](#architecture-and-base-model-modes) · [Install](#install) · -[**Benchmark results**](#benchmark-results) · +[Training on clips](#training-on-clips) · [**Benchmark results**](#benchmark-results) · [Dataset and adapter options](#dataset-and-adapter-options) · [Base precision](#base-precision) ## The graph @@ -57,7 +57,7 @@ The Trainer's Adjust panel picks the **architecture** first (Z-Image, Krea 2, FL **MiniMax H3** is the video model, and it trains on **still images**: - **FL2VA** is the only base, and it is undistilled, so there is no adapter and nothing to drift. Put `minimax_h3_fl2va_bf16.safetensors` in `models/diffusion_models/`, train on stills, then wire the LoRA into any of the four H3 nodes. It loads on the Reference to Video node too, which uses a different checkpoint file: the two partitions are the same architecture. -- **What it learns** is appearance - look, style, character, lighting. It does not learn motion or sound, because it never sees any. This is how image LoRAs for video models are normally trained, and it is the same thing every other H3 trainer does today. +- **Stills or short clips.** Drop images and it learns appearance: look, style, character, lighting. Drop video and it learns motion too. Sound is never learned either way, because the audio rows are empty. See [Training on clips](#training-on-clips). - **The base is 4-bit, always.** H3 is 40GB after the AdaLN factorisation and 11.7GB after quantisation, so full precision is refused rather than offered and then failing. There is no base-precision control for H3 for the same reason. - **A 24GB card is comfortable and a 16GB card works, slowly.** The run encodes latents and captions in two passes that never overlap, because H3's fp32 video VAE and its 32B conditioner cannot be resident together. On a card that holds the conditioner it peaks at 20.6GB; on one that does not, the conditioner runs on the CPU and the peak drops to 12.7GB while a step goes from 0.6s to 16s. Either way there is about seven minutes of startup, and 64GB of system RAM for the smaller card. See [Benchmark results](#benchmark-results) for the split. The download is about 124GB before any of that. @@ -66,6 +66,42 @@ The Trainer's Adjust panel picks the **architecture** first (Z-Image, Krea 2, FL - **Turbo + training adapter** fuses a de-distillation adapter into the base for the duration of training and drops it when the LoRA is saved, which preserves the 8-step speed. Put [ostris/zimage_turbo_training_adapter](https://huggingface.co/ostris/zimage_turbo_training_adapter) in `models/loras/`; any filename containing `adapter` is detected automatically, or point `INLINE_ZIMAGE_TRAIN_ADAPTER` at a specific file. Keep runs short, since the adapter slows the breakdown rather than preventing it. - **De-Turbo** trains without an adapter and needs no extra download. +## Training on clips + +The H3 trainer takes video as well as stills. Drop clips into a dataset the same way, set **Clip +length** in the Adjust panel, and each clip trains as a short piece of motion rather than a frame. +Mixed datasets are fine: a still is simply a one-frame clip. + +**It costs no extra VRAM.** Measured on an L4, every clip length peaks at the same 20.4GB as a +still, because the high-water mark is the caption pass rather than the training: + +| Clip length | Frames | Latent frames | Packed rows at 512px | Peak VRAM | +| ----------- | ------ | ------------- | -------------------- | --------- | +| still | 1 | 1 | 293 | 20.55GB | +| 0.92s | 22 | 7 | 1,832 | 20.4GB | +| 1.6s | 39 | 12 | 3,112 | 20.4GB | +| 4.5s | 107 | 32 | 8,232 | 20.4GB | + +Rows are what a longer clip actually buys you, and they cost time rather than memory. That only +holds while the conditioner is resident; on a card too small for it the peak is the training phase +instead, and a long clip will push that up. + +**Lengths snap to H3's frame grid.** The VAE encodes `17n + 5` frames at 24fps, so a request lands +on the nearest grid point at or below it. The floor is a whole chunk plus the five-frame head: 22 +frames, **0.92 seconds**. Asking for less rounds up rather than being refused, because the VAE has +no way to encode a shorter clip. + +**Each clip is trimmed from its start, once.** The window is fixed at precache time so every clip is +encoded exactly once. Sampling a different window each step would mean re-encoding through the VAE +every step, which is the thing the precache exists to avoid. A clip shorter than the grid floor is +refused by name rather than silently padded. + +**Captions work the same.** A clip is auto-captioned from its middle frame, which describes the shot +better than the first frame usually does. Write them by hand if you would rather. + +Audio is not trained. H3 generates video and its soundtrack jointly, but the trainer packs zero +audio rows, so an adapter changes what a clip looks like and never what it sounds like. + ## Install If you installed with `--extra all` from [Get Started](README.md#get-started), the trainer is already set up - nothing more to do. To add it to a leaner install, its dependencies (PEFT, 8-bit Adam, the captioner) sit behind the `training` extra: @@ -109,6 +145,7 @@ The LoRA a run produces lands in `models/loras/` and shows up in the LoRA loader | MiniMax H3 | FL2VA | 512 | **4-bit** | **20.6GB** | **12.7GB** | | MiniMax H3 | FL2VA | 768 | **4-bit** | **20.6GB** | not measured | | MiniMax H3 | FL2VA | 1024 | **4-bit** | **20.6GB** | not measured | +| MiniMax H3 | FL2VA, clips | 512 | **4-bit** | **20.4GB** | not measured | **MiniMax H3 costs less on a smaller card, which is not a typo.** The run has three phases that never overlap, and the tallest is not the one doing the learning: @@ -124,13 +161,14 @@ On a card too small for the conditioner it never goes there at all, so the peak **The bill arrives as time instead.** The conditioner runs on the CPU, and bitsandbytes only quantises on the move to CUDA, so it runs unquantised: -| | L40S (46GB) | T4 (16GB, 64GB RAM) | -| ----------------------- | ----------- | ------------------- | -| Peak VRAM | 20.6GB | 12.7GB | -| Seconds per step | 0.63 | 16.2 | -| Caption pass, 26 images | 1 min | 19 min | +| | L40S (46GB) | L4 (24GB) | T4 (16GB, 64GB RAM) | +| ----------------------- | ----------- | --------- | ------------------- | +| Peak VRAM, 512px | 20.6GB | 20.55GB | 12.7GB | +| Seconds per step, 512px | 0.63 | 1.81 | 16.2 | +| Seconds per step, 768px | 0.77 | 2.73 | not measured | +| Caption pass, 26 images | 1 min | 1 min | 19 min | -A 1500-step run is about 16 minutes on the L40S and closer to seven hours on the T4. Some of that is the T4 being a T4, and some is the caption pass being on the wrong processor. +A 1500-step run at 512px is about 16 minutes on the L40S, 45 on the L4, and closer to seven hours on the T4. The L4 holds the conditioner, so it looks like a slower L40S rather than a faster T4: the 9x gap to the T4 is mostly the caption pass being on the wrong processor, not the cards themselves. **It also wants a lot of system RAM.** The unquantised conditioner pages roughly 63GB through the page cache, and on a 64GB machine that sits at 59GB resident, close enough to the edge that the caption pass is the riskiest part of the run. A T4 with only 16GB of RAM has room in neither VRAM nor RAM and is refused before anything loads, because a host-RAM overrun is killed by the kernel rather than raising. diff --git a/core/pyproject.toml b/core/pyproject.toml index 09739dd..608cdd4 100644 --- a/core/pyproject.toml +++ b/core/pyproject.toml @@ -82,6 +82,8 @@ all = [ "accelerate>=0.30", "safetensors>=0.4", "torchao>=0.14", + # Clip decode for MiniMax H3 LoRA training, and H3's reference node. + "av>=12", "scipy>=1.11", "huggingface_hub>=0.23", "controlnet-aux>=0.0.7", diff --git a/core/src/inline_core/training/arch.py b/core/src/inline_core/training/arch.py index 28b056f..b38b06f 100644 --- a/core/src/inline_core/training/arch.py +++ b/core/src/inline_core/training/arch.py @@ -333,6 +333,22 @@ def _h3_forward(transformer: Any, noisy: Any, timestep: Any, item: dict[str, Any } +def clip_frames(arch: TrainingArch, seconds: Any) -> int: + """How many frames of a clip to train on, snapped to the arch's frame grid. + + 1 for an arch with no clip support, which is what a still costs. For H3 the floor is a whole + 17-frame chunk plus the 5-frame head, so a shorter request rounds up to 0.92s rather than being + refused; the VAE has no way to encode less. + """ + if arch.key != MINIMAX_H3: + return 1 + from ..models.minimaxh3.vendor.packing import MINIMAX_H3_FPS + from ..models.minimaxh3.vendor.packing_ref2va import trim_reference_num_frames + + wanted = round(float(seconds) * MINIMAX_H3_FPS) if seconds else 1 + return trim_reference_num_frames(max(1, wanted)) + + def get(key: str | None) -> TrainingArch: """The arch to train. Defaults to Z-Image so a run predating Krea 2 still resumes.""" arch = ARCHS.get(key or Z_IMAGE) diff --git a/core/src/inline_core/training/cache.py b/core/src/inline_core/training/cache.py index 230c95b..73be51b 100644 --- a/core/src/inline_core/training/cache.py +++ b/core/src/inline_core/training/cache.py @@ -28,13 +28,14 @@ def build( *, flip: bool = False, dropout: float = 0.0, + clip_frames: int = 1, ) -> tuple[list[dict[str, Any]], dict[str, Any] | None, float]: """Return ``(items, unconditional, shift)``, all as CPU tensors, with the encoders freed.""" if arch == archs.MINIMAX_H3: from . import h3 items, unconditional = h3.precache( - dataset_dir, models_dir, device, dtype, resolution, flip, dropout > 0 + dataset_dir, models_dir, device, dtype, resolution, flip, dropout > 0, clip_frames ) return items, unconditional, _H3_SHIFT diff --git a/core/src/inline_core/training/caption.py b/core/src/inline_core/training/caption.py index 419dfdb..7827997 100644 --- a/core/src/inline_core/training/caption.py +++ b/core/src/inline_core/training/caption.py @@ -105,13 +105,30 @@ def _load_with_fallback(model_id: str) -> tuple[Any, Any, Any]: raise first from None +def _open(path: str) -> Any: + """The frame to caption. A clip is captioned from its middle frame, which is more + representative than the first and stops PIL raising on a container it cannot read.""" + from pathlib import Path + + from PIL import Image + + from . import dataset as ds + + if not ds.is_video(Path(path)): + return Image.open(path).convert("RGB") + + from ..models.minimaxh3.vendor.packing_ref2va import decode_reference_video + + frames, _fps, _audio = decode_reference_video(path) + return Image.fromarray(frames[len(frames) // 2]).convert("RGB") + + def _caption_one(model: Any, processor: Any, device: str, path: str) -> str: """One caption. Handles both shapes: task-token models (Florence-2, which post-processes a tagged string) and plain image-captioning models (BLIP), which just decode the output.""" import torch - from PIL import Image - image = Image.open(path).convert("RGB") + image = _open(path) task_style = hasattr(processor, "post_process_generation") inputs = ( processor(text=_TASK, images=image, return_tensors="pt") diff --git a/core/src/inline_core/training/dataset.py b/core/src/inline_core/training/dataset.py index 4e74615..0c7c5a4 100644 --- a/core/src/inline_core/training/dataset.py +++ b/core/src/inline_core/training/dataset.py @@ -17,15 +17,25 @@ _IMAGE_SUFFIXES = (".png", ".jpg", ".jpeg", ".webp", ".bmp") +#: Only the video archs pass these to ``_pairs``. An image arch handed a clip would reach PIL and +#: raise, so the default stays images and each caller opts in. +_VIDEO_SUFFIXES = (".mp4", ".mov", ".webm", ".mkv", ".avi") -def _pairs(dataset_dir: Path) -> list[tuple[Path, str]]: + +def is_video(path: Path) -> bool: + return path.suffix.lower() in _VIDEO_SUFFIXES + + +def _pairs( + dataset_dir: Path, suffixes: tuple[str, ...] = _IMAGE_SUFFIXES +) -> list[tuple[Path, str]]: out: list[tuple[Path, str]] = [] - for img in sorted(dataset_dir.iterdir()): - if img.suffix.lower() not in _IMAGE_SUFFIXES: + for media in sorted(dataset_dir.iterdir()): + if media.suffix.lower() not in suffixes: continue - caption_file = img.with_suffix(".txt") + caption_file = media.with_suffix(".txt") caption = caption_file.read_text(encoding="utf-8").strip() if caption_file.exists() else "" - out.append((img, caption)) + out.append((media, caption)) return out diff --git a/core/src/inline_core/training/h3.py b/core/src/inline_core/training/h3.py index 3a53b39..f239994 100644 --- a/core/src/inline_core/training/h3.py +++ b/core/src/inline_core/training/h3.py @@ -38,16 +38,17 @@ def precache( resolution: int, flip: bool, want_unconditional: bool, + clip_frames: int = 1, ) -> tuple[list[dict[str, Any]], dict[str, Any] | None]: """Every image as a latent and every caption as conditioning, as CPU tensors.""" from . import dataset as ds - pairs = ds._pairs(Path(dataset_dir)) + pairs = ds._pairs(Path(dataset_dir), ds._IMAGE_SUFFIXES + ds._VIDEO_SUFFIXES) if not pairs: raise RuntimeError("The exported dataset is empty.") root = Path(models_dir) - latents = _encode_pixels(root, pairs, device, resolution, flip) + latents = _encode_pixels(root, pairs, device, resolution, flip, clip_frames) captions = [caption for _img, caption in pairs for _ in ((False, True) if flip else (False,))] if want_unconditional: captions.append("") @@ -57,19 +58,24 @@ def precache( {"latent": latent, **_conditioning(embed, tags, latent)} for latent, (embed, tags) in zip(latents, embeds, strict=False) ] - unconditional = None if want_unconditional: + # Dropout swaps a different text length in, which moves every row after it, so the whole + # layout travels with the embedding. It also depends on the latent grid, and a dataset + # mixing stills with clips has more than one, so each item carries its own rather than + # sharing the first item's and mis-sizing every clip. embed, tags = embeds[-1] - # Caption dropout swaps in a different text length, which moves every row after it, so the - # whole layout travels with the embedding rather than just the embedding. - unconditional = _conditioning(embed, tags, latents[0]) - return items, unconditional + for item in items: + item["uncond"] = _conditioning(embed, tags, item["latent"]) + # The per-item copies are what dropout uses; the loop keeps the global slot for the image archs. + return items, None def _encode_pixels( - root: Path, pairs: list[tuple[Path, str]], device: str, resolution: int, flip: bool + root: Path, pairs: list[tuple[Path, str]], device: str, resolution: int, flip: bool, + clip_frames: int = 1, ) -> list[Any]: """Pass one: the video VAE, then dropped.""" + import numpy import torch from PIL import Image @@ -85,18 +91,21 @@ def _encode_pixels( out: list[Any] = [] try: - for img_path, _caption in pairs: + for path, _caption in pairs: + clip = ds.is_video(path) + frames = _clip_frames(path, clip_frames) if clip else [Image.open(path)] for mirrored in (False, True) if flip else (False,): - square = ds._square(Image.open(img_path), resolution, mirrored) - # H3 normalises with ImageNet statistics, not to [-1, 1] like the image archs, and - # a still is one frame: (1, 3, 1, H, W). - raw = torch.from_numpy(_as_array(square)).to(device) - pixels = raw.permute(2, 0, 1)[None, :, None] + stack = [_as_array(ds._square(f, resolution, mirrored)) for f in frames] + # ImageNet statistics, not the [-1, 1] the image archs use, and always 5D: + # (1, 3, F, H, W). + raw = torch.from_numpy(numpy.stack(stack)).to(device) + pixels = raw.permute(3, 0, 1, 2)[None] pixels = (pixels.to(torch.float32).div(255.0) - pixel_mean) / pixel_std with torch.no_grad(): - # The spatial encoder alone, the path inference uses for a single frame; the - # temporal chunking is for 17n+5 clips. - latent = _sample(vae._encode_clip(pixels)) + # A single frame takes the spatial encoder; a 17n+5 clip takes the temporal + # chunking. Mirrors the split the vendored reference encoder makes. + moments = vae._encode(pixels) if clip else vae._encode_clip(pixels) + latent = _sample(moments) out.append(((latent.cpu() - mean) / std)[0]) finally: del vae @@ -105,6 +114,31 @@ def _encode_pixels( return out +def _clip_frames(path: Path, clip_frames: int) -> list[Any]: + """A clip as PIL frames on H3's 24fps, 17n+5 grid, taken from the start. + + Trimmed rather than sampled: a fixed window keeps the precache to one encode per clip, and + re-encoding a different window every step would defeat caching the latents at all. + """ + from PIL import Image + + from ..models.minimaxh3.vendor.packing_ref2va import ( + decode_reference_video, + resample_reference_frames, + trim_reference_num_frames, + ) + + decoded, fps, _audio = decode_reference_video(str(path)) + frames = resample_reference_frames(decoded, fps) + keep = trim_reference_num_frames(min(frames.shape[0], clip_frames)) + if keep > frames.shape[0]: + raise RuntimeError( + f"{path.name} is {frames.shape[0]} frames once resampled to 24fps, and H3's shortest " + f"encodable clip is {keep}. Use a longer clip, or drop this one from the dataset." + ) + return [Image.fromarray(frame) for frame in frames[:keep]] + + def _encode_captions( root: Path, captions: list[str], device: str, dtype: Any ) -> list[tuple[Any, Any]]: diff --git a/core/src/inline_core/training/trainer.py b/core/src/inline_core/training/trainer.py index 848802e..deeebc9 100644 --- a/core/src/inline_core/training/trainer.py +++ b/core/src/inline_core/training/trainer.py @@ -139,10 +139,14 @@ def _activation_offload(enabled: bool) -> Any: def _to_device(item: dict[str, Any], device: Any, dtype: Any) -> dict[str, Any]: - """A cached item on the training device, casting only its activations.""" + """A cached item on the training device, casting only its activations. + + Anything that is not a tensor is dropped: an arch may stash its own bookkeeping on the item + (H3 keeps a per-item unconditional layout there) and the model never sees it.""" return { key: value.to(device, dtype) if key in _ACTIVATION_KEYS else value.to(device) for key, value in item.items() + if hasattr(value, "to") } @@ -172,6 +176,7 @@ def train(manifest: dict[str, Any]) -> str | None: data, unconditional, shift = cache.build( manifest["datasetDir"], manifest["modelsDir"], arch.key, str(device), dtype, resolution, flip=bool(hp.get("flipAugment")), dropout=dropout, + clip_frames=archs.clip_frames(arch, hp.get("clipSeconds")), ) quant = models.resolve_quant( @@ -227,10 +232,14 @@ def train(manifest: dict[str, Any]) -> str | None: if stop.flagged: break source = data[step % len(data)] - if unconditional is not None and random.random() < dropout: - source = {**source, **unconditional} + if dropout and random.random() < dropout: + # An arch whose layout depends on the item carries its own unconditional; the rest + # share one. H3 needs the per-item form because a clip and a still pack differently. + swap = source.get("uncond") or unconditional + if swap is not None: + source = {**source, **swap} item = _to_device(source, device, dtype) - clean = item["latent"] # (C, H, W) + clean = item["latent"] # (C, H, W) for the image archs, (C, F, H, W) for H3 noise = torch.randn_like(clean) sigma = arch.sigma(device, shift) # scalar noise fraction in (0, 1) noisy = (1 - sigma) * clean + sigma * noise diff --git a/core/tests/test_minimaxh3_training.py b/core/tests/test_minimaxh3_training.py index 22dc36e..f3a1947 100644 --- a/core/tests/test_minimaxh3_training.py +++ b/core/tests/test_minimaxh3_training.py @@ -359,3 +359,61 @@ def test_h3_forward_packs_and_unpacks_back_to_the_latent_grid() -> None: assert tuple(transformer.seen["hidden_states"].shape) == (1, 16, 96) assert tuple(transformer.seen["audio_hidden_states"].shape) == (1, 0, 32) assert tuple(transformer.seen["timestep"].shape) == (1,) + + +def test_clip_length_snaps_to_the_frame_grid() -> None: + """H3's VAE encodes 17n+5 frames, so a request lands on the grid or not at all.""" + h3 = archs.get(archs.MINIMAX_H3) + + # Its floor is a whole 17-frame chunk plus the 5-frame head: 22 frames, 0.92s. Anything + # shorter rounds up rather than being refused, because the VAE cannot encode less. + assert archs.clip_frames(h3, 0.1) == 22 + assert archs.clip_frames(h3, 1.0) == 22 + assert archs.clip_frames(h3, 2.0) == 39 + assert archs.clip_frames(h3, 5.0) == 107 + for frames in (22, 39, 107): + assert (frames - 5) % 17 == 0 + + +def test_an_arch_without_clips_always_reports_one_frame() -> None: + for key in (archs.Z_IMAGE, archs.KREA2, archs.FLUX2): + assert archs.clip_frames(archs.get(key), 5.0) == 1 + + +def test_unset_clip_length_still_gives_an_encodable_clip() -> None: + """A dataset can hold a clip with no clip length set, and 1 frame is not encodable.""" + assert archs.clip_frames(archs.get(archs.MINIMAX_H3), None) == 22 + + +def test_a_clip_packs_more_rows_than_a_still_at_the_same_resolution() -> None: + """The reason clip training costs what it does: rows scale with latent frames.""" + still = _layout(text_tokens=5, latent=8) + clip = packing.build_packed_sequence( + text_token_tags=torch.full((5,), packing.MINIMAX_H3_TEXT_TAG, dtype=torch.long), + num_latent_frames=packing.video_latent_num_frames(22), + latent_height=8, + latent_width=8, + num_audio_latents=0, + patch_size=PATCH, + keyframe_anchors=(), + ) + + assert packing.video_latent_num_frames(22) == 7 + assert clip.video_indices.numel() == 7 * still.video_indices.numel() + assert clip.audio_indices.numel() == 0 + + +def test_only_the_video_archs_see_clips_in_a_dataset(tmp_path: object) -> None: + """An image arch handed an mp4 would reach PIL and raise, so the filter is opt-in.""" + from pathlib import Path + + from inline_core.training import dataset as ds + + root = Path(str(tmp_path)) + (root / "0000.jpg").write_bytes(b"") + (root / "0001.mp4").write_bytes(b"") + + assert [p.name for p, _c in ds._pairs(root)] == ["0000.jpg"] + both = ds._pairs(root, ds._IMAGE_SUFFIXES + ds._VIDEO_SUFFIXES) + assert [p.name for p, _c in both] == ["0000.jpg", "0001.mp4"] + assert ds.is_video(root / "0001.mp4") and not ds.is_video(root / "0000.jpg") diff --git a/src/renderer/views/Trainer/DatasetItemsGrid.tsx b/src/renderer/views/Trainer/DatasetItemsGrid.tsx index 9d680e3..760bc9a 100644 --- a/src/renderer/views/Trainer/DatasetItemsGrid.tsx +++ b/src/renderer/views/Trainer/DatasetItemsGrid.tsx @@ -4,6 +4,7 @@ import type { Asset, TrainingDatasetItem } from '@shared/types' import { resolveMedia } from '@/lib/media' import { uploadFiles } from '@/lib/importFiles' import { Modal } from '../../components/Modal' +import { VideoPreview } from '../../components/VideoPreview' import { useAssetStore } from '../../store/assetStore' import { useTrainingStore } from '../../store/trainingStore' import { ipcErrorMessage } from '../../lib/ipcError' @@ -52,8 +53,21 @@ async function readCaptionFiles(files: File[]): Promise> { return captions } -const isImage = (f: File): boolean => - f.type.startsWith('image/') || /\.(png|jpe?g|webp|bmp)$/i.test(f.name) +// Clips are accepted for the archs that can train on them (MiniMax H3). An arch that cannot is +// handed images only by the precache, so a stray clip is skipped rather than breaking a run. +const isMedia = (f: File): boolean => + f.type.startsWith('image/') || + f.type.startsWith('video/') || + /\.(png|jpe?g|webp|bmp|mp4|mov|webm|mkv|avi)$/i.test(f.name) + +/** A dataset tile. Video needs a real