From b118cab2925bce0b85e67af760355b8a4786d7fe Mon Sep 17 00:00:00 2001 From: StreamKit Devin Date: Mon, 23 Mar 2026 21:40:53 +0000 Subject: [PATCH 1/5] feat(compositor): add source dimensions, layout re-emission, and client aspect-fit prediction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 1 of compositor view-data causal consistency: Rust: - Add source_width/source_height to ResolvedLayer (populated from input slot's latest frame in resolve_scene) - Track per-slot source dimensions in InputSlot.last_source_dims - Re-emit layout (set layer_configs_dirty) when source resolution changes at runtime (e.g. camera switch), ensuring clients learn about resolution changes within one tick - Add tests for source dims population TypeScript: - Materialize stub LayerState in mapServerLayers for server-only layers (auto-PiP) with default config values — precondition for client-side prediction - Port fitRectPreservingAspect from Rust to TypeScript with matching test vectors (4:3→16:9 pillarbox, 16:9→4:3 letterbox, exact match, zero inputs) - Add SourceDimsMap ref in useServerLayoutSync to store per-layer source frame dimensions from server view data (prediction state separation) - Regenerate TS types via just gen-types - Add integration test for auto-PiP layer materialization Signed-off-by: StreamKit Devin Co-Authored-By: Claudio Costa --- crates/nodes/src/video/compositor/config.rs | 10 ++ crates/nodes/src/video/compositor/mod.rs | 35 ++++++- crates/nodes/src/video/compositor/tests.rs | 42 +++++++- ui/src/hooks/compositorLayerParsers.test.ts | 55 ++++++++++- ui/src/hooks/compositorLayerParsers.ts | 30 ++++++ ui/src/hooks/compositorServerSync.ts | 97 ++++++++++++++----- .../useCompositorLayers.monitor-flow.test.ts | 37 +++++++ ui/src/hooks/useCompositorLayers.ts | 4 +- ui/src/types/generated/compositor-types.ts | 14 ++- 9 files changed, 295 insertions(+), 29 deletions(-) diff --git a/crates/nodes/src/video/compositor/config.rs b/crates/nodes/src/video/compositor/config.rs index 4c42f5c8..98504146 100644 --- a/crates/nodes/src/video/compositor/config.rs +++ b/crates/nodes/src/video/compositor/config.rs @@ -322,6 +322,16 @@ pub struct ResolvedLayer { pub y: i32, pub width: u32, pub height: u32, + /// Source frame width (from the input slot's latest frame). + /// The client uses this to compute aspect-fit locally for zero-latency + /// feedback on auto-PiP layers. + /// `None` when no frame has been received yet for this input. + #[serde(skip_serializing_if = "Option::is_none")] + pub source_width: Option, + /// Source frame height (from the input slot's latest frame). + /// `None` when no frame has been received yet for this input. + #[serde(skip_serializing_if = "Option::is_none")] + pub source_height: Option, } /// Server-computed geometry for a single overlay (text or image). diff --git a/crates/nodes/src/video/compositor/mod.rs b/crates/nodes/src/video/compositor/mod.rs index 5d64ce67..74d590bf 100644 --- a/crates/nodes/src/video/compositor/mod.rs +++ b/crates/nodes/src/video/compositor/mod.rs @@ -62,6 +62,10 @@ struct InputSlot { name: String, rx: mpsc::Receiver, latest_frame: Option, + /// Last-seen source dimensions for dirty detection. + /// When the frame resolution changes at runtime (e.g. camera switch), + /// the compositor sets `layer_configs_dirty` to re-emit view data. + last_source_dims: Option<(u32, u32)>, } // ── Cached layer config ───────────────────────────────────────────────────── @@ -205,7 +209,15 @@ fn resolve_scene( rect.as_ref() .map_or((0, 0, config.width, config.height), |r| (r.x, r.y, r.width, r.height)) }; - layers.push(ResolvedLayer { id: slot.name.clone(), x: lx, y: ly, width: lw, height: lh }); + layers.push(ResolvedLayer { + id: slot.name.clone(), + x: lx, + y: ly, + width: lw, + height: lh, + source_width: slot.latest_frame.as_ref().map(|f| f.width), + source_height: slot.latest_frame.as_ref().map(|f| f.height), + }); configs.push(ResolvedSlotConfig { rect, @@ -481,7 +493,7 @@ impl ProcessorNode for CompositorNode { }); for (name, rx) in pre_inputs { tracing::info!("CompositorNode: pre-connected input '{}'", name); - slots.push(InputSlot { name, rx, latest_frame: None }); + slots.push(InputSlot { name, rx, latest_frame: None, last_source_dims: None }); } // Pin management channel (optional). @@ -675,6 +687,12 @@ impl ProcessorNode for CompositorNode { for slot in &mut slots { if is_oneshot { if let Ok(Packet::Video(frame)) = slot.rx.try_recv() { + // Detect source dimension changes → trigger layout re-emission. + let new_dims = (frame.width, frame.height); + if slot.last_source_dims != Some(new_dims) { + slot.last_source_dims = Some(new_dims); + layer_configs_dirty = true; + } slot.latest_frame = Some(frame); any_new_frame = true; } @@ -692,6 +710,12 @@ impl ProcessorNode for CompositorNode { stats_tracker.discarded_n(dropped); } if let Some(frame) = latest { + // Detect source dimension changes → trigger layout re-emission. + let new_dims = (frame.width, frame.height); + if slot.last_source_dims != Some(new_dims) { + slot.last_source_dims = Some(new_dims); + layer_configs_dirty = true; + } slot.latest_frame = Some(frame); } } @@ -1040,7 +1064,12 @@ impl CompositorNode { }, PinManagementMessage::AddedInputPin { pin, channel } => { tracing::info!("CompositorNode: activated input pin '{}'", pin.name); - slots.push(InputSlot { name: pin.name, rx: channel, latest_frame: None }); + slots.push(InputSlot { + name: pin.name, + rx: channel, + latest_frame: None, + last_source_dims: None, + }); }, PinManagementMessage::RemoveInputPin { pin_name } => { tracing::info!("CompositorNode: removed input pin '{}'", pin_name); diff --git a/crates/nodes/src/video/compositor/tests.rs b/crates/nodes/src/video/compositor/tests.rs index 91e92037..0f896ea3 100644 --- a/crates/nodes/src/video/compositor/tests.rs +++ b/crates/nodes/src/video/compositor/tests.rs @@ -331,6 +331,46 @@ fn test_fit_rect_preserving_aspect() { assert_eq!(fitted.y, 20); } +#[test] +fn test_resolved_layer_source_dims() { + // When a slot has a latest_frame, resolve_scene should populate source dims. + let config = CompositorConfig::default(); + let (_, rx) = mpsc::channel(1); + let slots = vec![InputSlot { + name: "in_0".to_string(), + rx, + latest_frame: Some(make_rgba_frame(1920, 1080, 0, 0, 0, 255)), + last_source_dims: Some((1920, 1080)), + }]; + let image_overlays: Arc<[Arc]> = Arc::from(vec![]); + let text_overlays: Arc<[Arc]> = Arc::from(vec![]); + + let scene = resolve_scene(&slots, &config, &image_overlays, &text_overlays); + assert_eq!(scene.layout.layers.len(), 1); + assert_eq!(scene.layout.layers[0].source_width, Some(1920)); + assert_eq!(scene.layout.layers[0].source_height, Some(1080)); +} + +#[test] +fn test_resolved_layer_source_dims_none_when_no_frame() { + // When a slot has no latest_frame, source dims should be None. + let config = CompositorConfig::default(); + let (_, rx) = mpsc::channel(1); + let slots = vec![InputSlot { + name: "in_0".to_string(), + rx, + latest_frame: None, + last_source_dims: None, + }]; + let image_overlays: Arc<[Arc]> = Arc::from(vec![]); + let text_overlays: Arc<[Arc]> = Arc::from(vec![]); + + let scene = resolve_scene(&slots, &config, &image_overlays, &text_overlays); + assert_eq!(scene.layout.layers.len(), 1); + assert_eq!(scene.layout.layers[0].source_width, None); + assert_eq!(scene.layout.layers[0].source_height, None); +} + #[test] fn test_config_validate_ok() { let cfg = CompositorConfig::default(); @@ -1951,7 +1991,7 @@ fn test_composite_frame_crop_shape_circle_skip_clear() { /// Helper: build an `InputSlot` with the given name and optional latest frame. fn make_slot(name: &str, frame: Option) -> InputSlot { let (_tx, rx) = mpsc::channel::(1); - InputSlot { name: name.to_string(), rx, latest_frame: frame } + InputSlot { name: name.to_string(), rx, latest_frame: frame, last_source_dims: None } } #[test] diff --git a/ui/src/hooks/compositorLayerParsers.test.ts b/ui/src/hooks/compositorLayerParsers.test.ts index d7c58cc3..d6e12f79 100644 --- a/ui/src/hooks/compositorLayerParsers.test.ts +++ b/ui/src/hooks/compositorLayerParsers.test.ts @@ -15,7 +15,7 @@ import { describe, it, expect } from 'vitest'; import type { LayerState, TextOverlayState, ImageOverlayState } from './compositorLayerParsers'; -import { mergeOverlayState } from './compositorLayerParsers'; +import { mergeOverlayState, fitRectPreservingAspect } from './compositorLayerParsers'; // ── Helpers ───────────────────────────────────────────────────────────────── @@ -431,3 +431,56 @@ describe('mergeOverlayState', () => { }); }); }); + +// ── fitRectPreservingAspect ───────────────────────────────────────────────── +// Test vectors mirror Rust tests in compositor/tests.rs:test_fit_rect_preserving_aspect + +describe('fitRectPreservingAspect', () => { + it('4:3 source into 16:9 bounds → pillarboxed', () => { + // Scale = min(426/640, 240/480) = min(0.666, 0.5) = 0.5 + // Fitted: 320×240, centred within 426×240 + const result = fitRectPreservingAspect(640, 480, { x: 100, y: 50, width: 426, height: 240 }); + expect(result.width).toBe(320); + expect(result.height).toBe(240); + expect(result.x).toBe(100 + Math.floor((426 - 320) / 2)); + expect(result.y).toBe(50); + }); + + it('16:9 source into 4:3 bounds → letterboxed', () => { + // Scale = min(400/1280, 400/720) = min(0.3125, 0.555) = 0.3125 + // Fitted: 400×225, centred within 400×400 + const result = fitRectPreservingAspect(1280, 720, { x: 0, y: 0, width: 400, height: 400 }); + expect(result.width).toBe(400); + expect(result.height).toBe(225); + expect(result.x).toBe(0); + expect(result.y).toBe(Math.floor((400 - 225) / 2)); + }); + + it('exact match → no change', () => { + const result = fitRectPreservingAspect(640, 480, { x: 10, y: 20, width: 640, height: 480 }); + expect(result.width).toBe(640); + expect(result.height).toBe(480); + expect(result.x).toBe(10); + expect(result.y).toBe(20); + }); + + it('zero source width → returns bounds unchanged', () => { + const bounds = { x: 5, y: 10, width: 200, height: 100 }; + expect(fitRectPreservingAspect(0, 480, bounds)).toEqual(bounds); + }); + + it('zero source height → returns bounds unchanged', () => { + const bounds = { x: 5, y: 10, width: 200, height: 100 }; + expect(fitRectPreservingAspect(640, 0, bounds)).toEqual(bounds); + }); + + it('zero bounds width → returns bounds unchanged', () => { + const bounds = { x: 5, y: 10, width: 0, height: 100 }; + expect(fitRectPreservingAspect(640, 480, bounds)).toEqual(bounds); + }); + + it('zero bounds height → returns bounds unchanged', () => { + const bounds = { x: 5, y: 10, width: 200, height: 0 }; + expect(fitRectPreservingAspect(640, 480, bounds)).toEqual(bounds); + }); +}); diff --git a/ui/src/hooks/compositorLayerParsers.ts b/ui/src/hooks/compositorLayerParsers.ts index c955186b..d81dd5c8 100644 --- a/ui/src/hooks/compositorLayerParsers.ts +++ b/ui/src/hooks/compositorLayerParsers.ts @@ -472,6 +472,36 @@ export function buildConfig( }; } +// ── Aspect-fit prediction ─────────────────────────────────────────────────── + +/** Compute a destination rect that fits `srcW × srcH` within `bounds` + * while preserving the source aspect ratio. The fitted rect is centred + * within the bounds. + * + * Port of Rust `fit_rect_preserving_aspect` (mod.rs:249-266). + * JS `Math.round()` and Rust `.round()` produce matching results for + * non-negative values, which is all this function rounds. */ +export function fitRectPreservingAspect( + srcW: number, + srcH: number, + bounds: { x: number; y: number; width: number; height: number } +): { x: number; y: number; width: number; height: number } { + if (srcW === 0 || srcH === 0 || bounds.width === 0 || bounds.height === 0) { + return { x: bounds.x, y: bounds.y, width: bounds.width, height: bounds.height }; + } + const scaleW = bounds.width / srcW; + const scaleH = bounds.height / srcH; + const scale = Math.min(scaleW, scaleH); + const fitW = Math.round(srcW * scale); + const fitH = Math.round(srcH * scale); + // Centre within the bounding rect. + // Use integer division (Math.floor on the half-difference) to match + // Rust's saturating_sub / 2 behaviour for non-negative values. + const offsetX = Math.floor((bounds.width - fitW) / 2); + const offsetY = Math.floor((bounds.height - fitH) / 2); + return { x: bounds.x + offsetX, y: bounds.y + offsetY, width: fitW, height: fitH }; +} + // ── Snap guide detection ──────────────────────────────────────────────────── /** Which centre snap guides are currently active during a drag. */ diff --git a/ui/src/hooks/compositorServerSync.ts b/ui/src/hooks/compositorServerSync.ts index 8b805594..1817df2a 100644 --- a/ui/src/hooks/compositorServerSync.ts +++ b/ui/src/hooks/compositorServerSync.ts @@ -43,32 +43,71 @@ import { setLayersInStore, setTextOverlaysInStore, } from './compositorAtoms'; +import { + DEFAULT_OPACITY, + DEFAULT_ROTATION_DEGREES, + DEFAULT_Z_INDEX, + DEFAULT_MIRROR_HORIZONTAL, + DEFAULT_MIRROR_VERTICAL, + DEFAULT_VISIBLE, + DEFAULT_CROP_ZOOM, + DEFAULT_CROP_X, + DEFAULT_CROP_Y, + DEFAULT_CROP_SHAPE, +} from './compositorConstants'; import type { LayerState, TextOverlayState, OverlayBase } from './compositorLayerParsers'; +// ── Source dims ref type ───────────────────────────────────────────────────── + +/** Per-layer source frame dimensions, keyed by layer id. + * Populated from `ResolvedLayer.source_width`/`source_height` in server + * view data. Kept separate from `LayerState` so prediction inputs + * (runtime server metadata) don't mix with config/geometry state. */ +export type SourceDimsMap = Map; + // ── Pure helpers ──────────────────────────────────────────────────────────── /** Map server geometry onto existing client LayerState[], preserving all - * config-driven fields (opacity, rotation, z_index, mirror, crop, visible). */ + * config-driven fields (opacity, rotation, z_index, mirror, crop, visible). + * + * For server-only layers (e.g. auto-PiP layers with no explicit config in + * params), a stub LayerState is created with server geometry + default config + * values. This ensures the client has a LayerState entry for every layer the + * server reports, which is a precondition for client-side prediction. */ export function mapServerLayers(prev: LayerState[], serverLayers: ResolvedLayer[]): LayerState[] { - const next: LayerState[] = serverLayers - .map((sl) => { - const existing = prev.find((l) => l.id === sl.id); - if (!existing) { - // New layer from server with no local counterpart — should be rare. - // Return a stub; sync-from-props will fill in the config fields. - return undefined; - } - if ( - existing.x === sl.x && - existing.y === sl.y && - existing.width === sl.width && - existing.height === sl.height - ) { - return existing; - } - return { ...existing, x: sl.x, y: sl.y, width: sl.width, height: sl.height }; - }) - .filter((l): l is LayerState => l !== undefined); + const next: LayerState[] = serverLayers.map((sl) => { + const existing = prev.find((l) => l.id === sl.id); + if (!existing) { + // Server-only layer (auto-PiP) with no local counterpart. + // Materialize a stub with server geometry + default config values. + return { + id: sl.id, + x: sl.x, + y: sl.y, + width: sl.width, + height: sl.height, + opacity: DEFAULT_OPACITY, + zIndex: DEFAULT_Z_INDEX, + rotationDegrees: DEFAULT_ROTATION_DEGREES, + mirrorHorizontal: DEFAULT_MIRROR_HORIZONTAL, + mirrorVertical: DEFAULT_MIRROR_VERTICAL, + visible: DEFAULT_VISIBLE, + cropZoom: DEFAULT_CROP_ZOOM, + cropX: DEFAULT_CROP_X, + cropY: DEFAULT_CROP_Y, + cropShape: DEFAULT_CROP_SHAPE, + } satisfies LayerState; + } + if ( + existing.x === sl.x && + existing.y === sl.y && + existing.width === sl.width && + existing.height === sl.height + ) { + return existing; + } + return { ...existing, x: sl.x, y: sl.y, width: sl.width, height: sl.height }; + }); return next.length !== prev.length || next.some((s, i) => s !== prev[i]) ? next : prev; } @@ -128,7 +167,8 @@ export function useServerLayoutSync( sessionId: string | undefined, nodeId: string, store: CompositorStore, - dragStateRef: React.MutableRefObject + dragStateRef: React.MutableRefObject, + sourceDimsRef: React.MutableRefObject ): void { useEffect(() => { if (!sessionId) return; @@ -142,6 +182,19 @@ export function useServerLayoutSync( const layout = viewData as CompositorLayout; if (!Array.isArray(layout.layers)) return; + // Update source dims ref from server view data (prediction inputs). + for (const sl of layout.layers) { + if (sl.source_width != null && sl.source_height != null) { + const prev = sourceDimsRef.current.get(sl.id); + if (!prev || prev.width !== sl.source_width || prev.height !== sl.source_height) { + sourceDimsRef.current.set(sl.id, { + width: sl.source_width, + height: sl.source_height, + }); + } + } + } + const prevLayers = getLayersFromStore(store); const newLayers = mapServerLayers(prevLayers, layout.layers); if (newLayers !== prevLayers) setLayersInStore(store, newLayers); @@ -172,5 +225,5 @@ export function useServerLayoutSync( applyServerLayout(defaultSessionStore.get(viewDataAtom)); }); return unsubscribe; - }, [sessionId, nodeId, store, dragStateRef]); + }, [sessionId, nodeId, store, dragStateRef, sourceDimsRef]); } diff --git a/ui/src/hooks/useCompositorLayers.monitor-flow.test.ts b/ui/src/hooks/useCompositorLayers.monitor-flow.test.ts index 270f7294..a71a2c4f 100644 --- a/ui/src/hooks/useCompositorLayers.monitor-flow.test.ts +++ b/ui/src/hooks/useCompositorLayers.monitor-flow.test.ts @@ -108,6 +108,8 @@ function serverLayer( y: 0, width: 960, height: 720, + source_width: null, + source_height: null, ...overrides, }; } @@ -590,4 +592,39 @@ describe('Monitor view data flow integration', () => { expect(text1[0].x).toBe(100); expect(text1[0].y).toBe(200); }); + + it('server-only layers (auto-PiP) are materialized with default config', () => { + seedStore(); + + // Params only have in_0, but server reports both in_0 and in_1. + const opts = monitorOptions(); + const { result } = renderHook( + (props: UseCompositorLayersOptions) => useCompositorLayers(props), + { initialProps: opts } + ); + + // Server view data includes an auto-PiP layer (in_1) not in params. + const layout = makeServerLayout({ + layers: [ + serverLayer('in_0'), + serverLayer('in_1', { x: 800, y: 400, width: 320, height: 240 }), + ], + }); + act(() => pushServerViewData(layout)); + + const layers = getLayersFromStore(result.current.store); + expect(layers).toHaveLength(2); + + const autoPip = layers.find((l) => l.id === 'in_1')!; + expect(autoPip).toBeDefined(); + expect(autoPip.x).toBe(800); + expect(autoPip.y).toBe(400); + expect(autoPip.width).toBe(320); + expect(autoPip.height).toBe(240); + // Default config values for materialized stub + expect(autoPip.opacity).toBe(1); + expect(autoPip.rotationDegrees).toBe(0); + expect(autoPip.zIndex).toBe(0); + expect(autoPip.visible).toBe(true); + }); }); diff --git a/ui/src/hooks/useCompositorLayers.ts b/ui/src/hooks/useCompositorLayers.ts index d9e7ca1e..3eda3f9a 100644 --- a/ui/src/hooks/useCompositorLayers.ts +++ b/ui/src/hooks/useCompositorLayers.ts @@ -67,6 +67,7 @@ import type { } from './compositorLayerParsers'; import { useCompositorOverlays } from './compositorOverlays'; import { useServerLayoutSync } from './compositorServerSync'; +import type { SourceDimsMap } from './compositorServerSync'; export type { LayerState, @@ -266,6 +267,7 @@ export const useCompositorLayers = ( horizontal: HTMLDivElement | null; }>({ vertical: null, horizontal: null }); const dragStateRef = useRef(null); + const sourceDimsRef = useRef(new Map()); // ── Commit / persistence ─────────────────────────────────────────────────── const { commitAdapter, throttledConfigChange, throttledOverlayCommit } = useCompositorCommit({ @@ -347,7 +349,7 @@ export const useCompositorLayers = ( }, [params, canvasWidth, canvasHeight, isMonitorView, store]); // ── Server-driven layout (Monitor view only) ─────────────────────────── - useServerLayoutSync(sessionId, nodeId, store, dragStateRef); + useServerLayoutSync(sessionId, nodeId, store, dragStateRef, sourceDimsRef); // ── Find layer across all types ───────────────────────────────────────── const findAnyLayer = useCallback( diff --git a/ui/src/types/generated/compositor-types.ts b/ui/src/types/generated/compositor-types.ts index 417fc0ac..671fb7c0 100644 --- a/ui/src/types/generated/compositor-types.ts +++ b/ui/src/types/generated/compositor-types.ts @@ -222,7 +222,19 @@ export type ResolvedLayer = { /** * Pin name (e.g. `"in_0"`). */ -id: string, x: number, y: number, width: number, height: number, }; +id: string, x: number, y: number, width: number, height: number, +/** + * Source frame width (from the input slot's latest frame). + * The client uses this to compute aspect-fit locally for zero-latency + * feedback on auto-PiP layers. + * `None` when no frame has been received yet for this input. + */ +source_width: number | null, +/** + * Source frame height (from the input slot's latest frame). + * `None` when no frame has been received yet for this input. + */ +source_height: number | null, }; export type ResolvedOverlay = { /** From 231e7492a5b23dbb7085d352cf805422818bce74 Mon Sep 17 00:00:00 2001 From: StreamKit Devin Date: Mon, 23 Mar 2026 21:49:06 +0000 Subject: [PATCH 2/5] fix(compositor): preserve server-only layers across params sync cycles In Monitor view, auto-PiP layers materialized by mapServerLayers from server view data were being dropped when the sync-from-props effect ran, because parseLayers(params) doesn't include server-only layers. Now the sync-from-props effect preserves server-only layers from current state that have no counterpart in parsed params. Extends the auto-PiP integration test to verify stubs survive a params echo-back. Signed-off-by: StreamKit Devin Co-Authored-By: Claudio Costa --- .../useCompositorLayers.monitor-flow.test.ts | 15 ++++++++++++++- ui/src/hooks/useCompositorLayers.ts | 17 ++++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/ui/src/hooks/useCompositorLayers.monitor-flow.test.ts b/ui/src/hooks/useCompositorLayers.monitor-flow.test.ts index a71a2c4f..4d28a72c 100644 --- a/ui/src/hooks/useCompositorLayers.monitor-flow.test.ts +++ b/ui/src/hooks/useCompositorLayers.monitor-flow.test.ts @@ -598,7 +598,7 @@ describe('Monitor view data flow integration', () => { // Params only have in_0, but server reports both in_0 and in_1. const opts = monitorOptions(); - const { result } = renderHook( + const { result, rerender } = renderHook( (props: UseCompositorLayersOptions) => useCompositorLayers(props), { initialProps: opts } ); @@ -626,5 +626,18 @@ describe('Monitor view data flow integration', () => { expect(autoPip.rotationDegrees).toBe(0); expect(autoPip.zIndex).toBe(0); expect(autoPip.visible).toBe(true); + + // Params echo-back: auto-PiP stubs must survive across params syncs. + // parseLayers(params) won't include in_1, but it must not be dropped. + act(() => rerender({ ...opts, params: makeParams() })); + + const layers2 = getLayersFromStore(result.current.store); + expect(layers2).toHaveLength(2); + const autoPip2 = layers2.find((l) => l.id === 'in_1')!; + expect(autoPip2).toBeDefined(); + expect(autoPip2.x).toBe(800); + expect(autoPip2.y).toBe(400); + expect(autoPip2.width).toBe(320); + expect(autoPip2.height).toBe(240); }); }); diff --git a/ui/src/hooks/useCompositorLayers.ts b/ui/src/hooks/useCompositorLayers.ts index 3eda3f9a..1cd5053b 100644 --- a/ui/src/hooks/useCompositorLayers.ts +++ b/ui/src/hooks/useCompositorLayers.ts @@ -305,7 +305,7 @@ export const useCompositorLayers = ( const parsed = parseLayers(params, canvasWidth, canvasHeight); const currentLayers = getLayersFromStore(store); - const merged = mergeOverlayState( + let merged = mergeOverlayState( currentLayers, parsed, (a, b) => @@ -316,6 +316,21 @@ export const useCompositorLayers = ( isMonitorView, isMonitorView ? prevParsedLayersRef.current : undefined ); + + // In Monitor view, preserve server-only layers (auto-PiP stubs) that + // exist in current state but have no config entry in params. Without + // this, mapServerLayers materializes them from view data, but the next + // sync-from-props cycle would drop them because parseLayers(params) + // doesn't include server-only layers. + if (isMonitorView) { + const serverOnly = currentLayers.filter( + (l) => !parsed.some((p) => p.id === l.id) && !merged.some((m) => m.id === l.id) + ); + if (serverOnly.length > 0) { + merged = [...merged, ...serverOnly]; + } + } + if (merged !== currentLayers) setLayersInStore(store, merged); prevParsedLayersRef.current = parsed; From eb2fb3f87f2da4f40a658de035eec879d4237a1b Mon Sep 17 00:00:00 2001 From: StreamKit Devin Date: Mon, 23 Mar 2026 21:57:29 +0000 Subject: [PATCH 3/5] fix(compositor): prevent server-only auto-PiP stubs from being serialized back Add serverOnly flag to LayerState, set it on stubs materialized by mapServerLayers for auto-PiP layers. serializeLayers() skips these entries, preventing explicit config from being sent to the server which would disable aspect-fit on the auto-PiP branch in resolve_scene(). Signed-off-by: StreamKit Devin Co-Authored-By: Claudio Costa --- ui/src/hooks/compositorLayerParsers.ts | 7 +++++++ ui/src/hooks/compositorServerSync.ts | 1 + 2 files changed, 8 insertions(+) diff --git a/ui/src/hooks/compositorLayerParsers.ts b/ui/src/hooks/compositorLayerParsers.ts index d81dd5c8..cd34a159 100644 --- a/ui/src/hooks/compositorLayerParsers.ts +++ b/ui/src/hooks/compositorLayerParsers.ts @@ -62,6 +62,10 @@ export interface LayerState { cropY: number; /** Shape clipping applied to the layer. */ cropShape: 'rect' | 'circle'; + /** True for layers materialized from server view data with no config entry + * in params (auto-PiP stubs). These must NOT be serialized back to the + * server — doing so would create explicit config that disables aspect-fit. */ + serverOnly?: boolean; } /** A text overlay stored in compositor config */ @@ -310,6 +314,9 @@ export function serializeImageOverlays(overlays: ImageOverlayState[]): ImageOver export function serializeLayers(layers: LayerState[]): Record { const layersMap: Record = {}; for (const layer of layers) { + // Skip server-only layers (auto-PiP stubs) — serializing them would + // create explicit config that disables aspect-fit on the server. + if (layer.serverOnly) continue; layersMap[layer.id] = { rect: serializeRect(layer), ...serializeSpatialFields(layer), diff --git a/ui/src/hooks/compositorServerSync.ts b/ui/src/hooks/compositorServerSync.ts index 1817df2a..4e818174 100644 --- a/ui/src/hooks/compositorServerSync.ts +++ b/ui/src/hooks/compositorServerSync.ts @@ -96,6 +96,7 @@ export function mapServerLayers(prev: LayerState[], serverLayers: ResolvedLayer[ cropX: DEFAULT_CROP_X, cropY: DEFAULT_CROP_Y, cropShape: DEFAULT_CROP_SHAPE, + serverOnly: true, } satisfies LayerState; } if ( From 35d5ddd7e4e430f43c53307bb1df81ae7473771e Mon Sep 17 00:00:00 2001 From: StreamKit Devin Date: Mon, 23 Mar 2026 22:06:21 +0000 Subject: [PATCH 4/5] fix: guard serverOnly filter and layerEqual comparison - Only preserve layers with serverOnly flag during params sync, preventing removed user layers from lingering. - Add serverOnly field to layerEqual() so flag transitions trigger proper atom updates. Signed-off-by: StreamKit Devin Co-Authored-By: Claudio Costa --- ui/src/hooks/compositorAtoms.ts | 3 ++- ui/src/hooks/useCompositorLayers.ts | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ui/src/hooks/compositorAtoms.ts b/ui/src/hooks/compositorAtoms.ts index ab25326b..d7797057 100644 --- a/ui/src/hooks/compositorAtoms.ts +++ b/ui/src/hooks/compositorAtoms.ts @@ -149,7 +149,8 @@ function layerEqual(a: LayerState, b: LayerState): boolean { a.cropZoom === b.cropZoom && a.cropX === b.cropX && a.cropY === b.cropY && - a.cropShape === b.cropShape + a.cropShape === b.cropShape && + a.serverOnly === b.serverOnly ); } diff --git a/ui/src/hooks/useCompositorLayers.ts b/ui/src/hooks/useCompositorLayers.ts index 1cd5053b..8d0305af 100644 --- a/ui/src/hooks/useCompositorLayers.ts +++ b/ui/src/hooks/useCompositorLayers.ts @@ -324,7 +324,8 @@ export const useCompositorLayers = ( // doesn't include server-only layers. if (isMonitorView) { const serverOnly = currentLayers.filter( - (l) => !parsed.some((p) => p.id === l.id) && !merged.some((m) => m.id === l.id) + (l) => + l.serverOnly && !parsed.some((p) => p.id === l.id) && !merged.some((m) => m.id === l.id) ); if (serverOnly.length > 0) { merged = [...merged, ...serverOnly]; From 7af34833f48a1fd900e84d75124d8748d3185e2b Mon Sep 17 00:00:00 2001 From: StreamKit Devin Date: Mon, 23 Mar 2026 22:21:37 +0000 Subject: [PATCH 5/5] fix(compositor): clear sticky serverOnly flag when layer gains explicit config When another client adds explicit config for an auto-PiP layer, the serverOnly flag must be cleared so serializeLayers includes user edits. Added test verifying the flag transition. Signed-off-by: StreamKit Devin Co-Authored-By: Claudio Costa --- .../useCompositorLayers.monitor-flow.test.ts | 41 +++++++++++++++++++ ui/src/hooks/useCompositorLayers.ts | 23 ++++++++--- 2 files changed, 59 insertions(+), 5 deletions(-) diff --git a/ui/src/hooks/useCompositorLayers.monitor-flow.test.ts b/ui/src/hooks/useCompositorLayers.monitor-flow.test.ts index 4d28a72c..e74345f7 100644 --- a/ui/src/hooks/useCompositorLayers.monitor-flow.test.ts +++ b/ui/src/hooks/useCompositorLayers.monitor-flow.test.ts @@ -640,4 +640,45 @@ describe('Monitor view data flow integration', () => { expect(autoPip2.width).toBe(320); expect(autoPip2.height).toBe(240); }); + + it('serverOnly flag is cleared when layer gains explicit config in params', () => { + seedStore(); + + const opts = monitorOptions(); + const { result, rerender } = renderHook( + (props: UseCompositorLayersOptions) => useCompositorLayers(props), + { initialProps: opts } + ); + + // Server view data includes auto-PiP layer in_1 (not in params). + const layout = makeServerLayout({ + layers: [ + serverLayer('in_0'), + serverLayer('in_1', { x: 800, y: 400, width: 320, height: 240 }), + ], + }); + act(() => pushServerViewData(layout)); + + const layers = getLayersFromStore(result.current.store); + const autoPip = layers.find((l) => l.id === 'in_1')!; + expect(autoPip.serverOnly).toBe(true); + + // Another client adds explicit config for in_1 → params now include it. + const paramsWithIn1 = makeParams({ + layers: { + in_0: { opacity: 1.0, z_index: 0 }, + in_1: { opacity: 0.8, z_index: 1 }, + }, + }); + act(() => rerender({ ...opts, params: paramsWithIn1 })); + + const layers2 = getLayersFromStore(result.current.store); + const layer1 = layers2.find((l) => l.id === 'in_1')!; + expect(layer1).toBeDefined(); + // serverOnly must be cleared so serializeLayers includes this layer. + expect(layer1.serverOnly).toBeUndefined(); + // opacity is an OVERLAY_BASE_KEYS field — server-controlled in Monitor + // view, so the stub's default (1) is preserved, not the parsed value. + expect(layer1.opacity).toBe(1); + }); }); diff --git a/ui/src/hooks/useCompositorLayers.ts b/ui/src/hooks/useCompositorLayers.ts index 8d0305af..a54457a6 100644 --- a/ui/src/hooks/useCompositorLayers.ts +++ b/ui/src/hooks/useCompositorLayers.ts @@ -317,12 +317,25 @@ export const useCompositorLayers = ( isMonitorView ? prevParsedLayersRef.current : undefined ); - // In Monitor view, preserve server-only layers (auto-PiP stubs) that - // exist in current state but have no config entry in params. Without - // this, mapServerLayers materializes them from view data, but the next - // sync-from-props cycle would drop them because parseLayers(params) - // doesn't include server-only layers. if (isMonitorView) { + // Clear serverOnly on layers that now have explicit config in params. + // Without this, a stub materialized by mapServerLayers would keep + // serverOnly: true even after another client adds config, causing + // serializeLayers to permanently skip user edits on that layer. + merged = merged.map((l) => { + if (l.serverOnly && parsed.some((p) => p.id === l.id)) { + const cleared = { ...l }; + delete cleared.serverOnly; + return cleared; + } + return l; + }); + + // Preserve server-only layers (auto-PiP stubs) that exist in current + // state but have no config entry in params. Without this, + // mapServerLayers materializes them from view data, but the next + // sync-from-props cycle would drop them because parseLayers(params) + // doesn't include server-only layers. const serverOnly = currentLayers.filter( (l) => l.serverOnly && !parsed.some((p) => p.id === l.id) && !merged.some((m) => m.id === l.id)