diff --git a/Cargo.toml b/Cargo.toml index 0b35f66d45..661b3b85fd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -70,7 +70,6 @@ dbg_macro = "deny" let_underscore_future = "deny" unchecked_duration_subtraction = "deny" collapsible_if = "deny" -manual_is_multiple_of = "deny" clone_on_copy = "deny" redundant_closure = "deny" ptr_arg = "deny" diff --git a/apps/desktop/core b/apps/desktop/core new file mode 100644 index 0000000000..5a29217500 Binary files /dev/null and b/apps/desktop/core differ diff --git a/apps/desktop/src/routes/editor/Editor.tsx b/apps/desktop/src/routes/editor/Editor.tsx index 38020839e3..5ef0f84d5b 100644 --- a/apps/desktop/src/routes/editor/Editor.tsx +++ b/apps/desktop/src/routes/editor/Editor.tsx @@ -1,7 +1,7 @@ import { Button } from "@cap/ui-solid"; import { NumberField } from "@kobalte/core/number-field"; import { trackDeep } from "@solid-primitives/deep"; -import { throttle } from "@solid-primitives/scheduled"; +import { debounce, throttle } from "@solid-primitives/scheduled"; import { makePersisted } from "@solid-primitives/storage"; import { createMutation } from "@tanstack/solid-query"; import { convertFileSrc } from "@tauri-apps/api/core"; @@ -13,6 +13,7 @@ import { createSignal, Match, on, + onCleanup, Show, Switch, } from "solid-js"; @@ -85,19 +86,49 @@ function Inner() { const { project, editorState, setEditorState } = useEditorContext(); createTauriEventListener(events.editorStateChanged, (payload) => { - renderFrame.clear(); + renderFrameThrottled.clear(); setEditorState("playbackTime", payload.playhead_position / FPS); }); - const renderFrame = throttle((time: number) => { - if (!editorState.playing) { - events.renderFrameEvent.emit({ - frame_number: Math.max(Math.floor(time * FPS), 0), - fps: FPS, - resolution_base: OUTPUT_SIZE, - }); + let rafId: number | null = null; + let pendingFrameTime: number | null = null; + + const emitFrame = (time: number) => { + events.renderFrameEvent.emit({ + frame_number: Math.max(Math.floor(time * FPS), 0), + fps: FPS, + resolution_base: OUTPUT_SIZE, + }); + }; + + const renderFrameThrottled = throttle((time: number) => { + if (editorState.playing) return; + + if (rafId !== null) { + pendingFrameTime = time; + return; + } + + rafId = requestAnimationFrame(() => { + rafId = null; + const frameTime = pendingFrameTime ?? time; + pendingFrameTime = null; + emitFrame(frameTime); + }); + }, 1000 / 30); + + const renderFrameDebounced = debounce((time: number) => { + if (editorState.playing) return; + emitFrame(time); + }, 50); + + onCleanup(() => { + if (rafId !== null) { + cancelAnimationFrame(rafId); } - }, 1000 / FPS); + renderFrameThrottled.clear(); + renderFrameDebounced.clear(); + }); const frameNumberToRender = createMemo(() => { const preview = editorState.previewTime; @@ -108,14 +139,23 @@ function Inner() { createEffect( on(frameNumberToRender, (number) => { if (editorState.playing) return; - renderFrame(number); + renderFrameThrottled(number); }), ); + let lastProjectUpdateTime = 0; createEffect( on( () => trackDeep(project), - () => renderFrame(editorState.playbackTime), + () => { + const now = performance.now(); + if (now - lastProjectUpdateTime < 100) { + renderFrameDebounced(editorState.playbackTime); + } else { + renderFrameThrottled(editorState.playbackTime); + } + lastProjectUpdateTime = now; + }, ), ); diff --git a/apps/desktop/src/routes/editor/Timeline/ClipTrack.tsx b/apps/desktop/src/routes/editor/Timeline/ClipTrack.tsx index 0d1fcc13fd..4ffbc601f5 100644 --- a/apps/desktop/src/routes/editor/Timeline/ClipTrack.tsx +++ b/apps/desktop/src/routes/editor/Timeline/ClipTrack.tsx @@ -10,6 +10,7 @@ import { createRoot, createSignal, For, + Index, Match, onCleanup, Show, @@ -188,7 +189,7 @@ export function ClipTrack( systemAudioWaveforms, } = useEditorContext(); - const { secsPerPixel, duration } = useTimelineContext(); + const { secsPerPixel, duration, isSegmentVisible } = useTimelineContext(); const segments = (): Array => project.timeline?.segments ?? [{ start: 0, end: duration(), timescale: 1 }]; @@ -204,6 +205,21 @@ export function ClipTrack( return offsets; }); + const visibleSegmentIndices = createMemo(() => { + const segs = segments(); + const offsets = segmentOffsets(); + const visible: number[] = []; + for (let i = 0; i < segs.length; i++) { + const seg = segs[i]; + const segStart = offsets[i]; + const segEnd = segStart + (seg.end - seg.start) / seg.timescale; + if (isSegmentVisible(segStart, segEnd)) { + visible.push(i); + } + } + return visible; + }); + function onHandleReleased() { const { transform } = editorState.timeline; @@ -223,8 +239,10 @@ export function ClipTrack( onMouseEnter={() => setEditorState("timeline", "hoveredTrack", "clip")} onMouseLeave={() => setEditorState("timeline", "hoveredTrack", null)} > - - {(segment, i) => { + + {(segmentIndex) => { + const i = segmentIndex; + const segment = () => segments()[i()]; const [startHandleDrag, setStartHandleDrag] = createSignal { const ds = startHandleDrag(); const offset = ds?.offset ?? 0; + const seg = segment(); return { start: Math.max(prevDuration() + offset, 0), end: prevDuration() + - (offset + (segment.end - segment.start)) / segment.timescale, - timescale: segment.timescale, - recordingSegment: segment.recordingSegment, + (offset + (seg.end - seg.start)) / seg.timescale, + timescale: seg.timescale, + recordingSegment: seg.recordingSegment, }; }); @@ -269,9 +288,10 @@ export function ClipTrack( const isSelected = createMemo(() => { const selection = editorState.timeline.selection; if (!selection || selection.type !== "clip") return false; + const seg = segment(); const segmentIndex = project.timeline?.segments?.findIndex( - (s) => s.start === segment.start && s.end === segment.end, + (s) => s.start === seg.start && s.end === seg.end, ); if (segmentIndex === undefined || segmentIndex === -1) return false; @@ -283,7 +303,7 @@ export function ClipTrack( if (project.audio.micVolumeDb && project.audio.micVolumeDb < -30) return; - const idx = segment.recordingSegment ?? i(); + const idx = segment().recordingSegment ?? i(); return micWaveforms()?.[idx] ?? []; }; @@ -294,7 +314,7 @@ export function ClipTrack( ) return; - const idx = segment.recordingSegment ?? i(); + const idx = segment().recordingSegment ?? i(); return systemAudioWaveforms()?.[idx] ?? []; }; @@ -401,8 +421,9 @@ export function ClipTrack( if (editorState.timeline.interactMode === "split") { const rect = e.currentTarget.getBoundingClientRect(); const fraction = (e.clientX - rect.left) / rect.width; + const seg = segment(); - const splitTime = fraction * (segment.end - segment.start); + const splitTime = fraction * (seg.end - seg.start); projectActions.splitClipSegment(prevDuration() + splitTime); } else { @@ -486,23 +507,24 @@ export function ClipTrack( } }} > - {segment.timescale === 1 && ( + {segment().timescale === 1 && ( )} - + { if (split()) return; + const seg = segment(); - const initialStart = segment.start; + const initialStart = seg.start; setStartHandleDrag({ offset: 0, initialStart, @@ -510,17 +532,16 @@ export function ClipTrack( const maxSegmentDuration = editorInstance.recordings.segments[ - segment.recordingSegment ?? 0 + seg.recordingSegment ?? 0 ].display.duration; const availableTimelineDuration = editorInstance.recordingDuration - segments().reduce( - (acc, segment, segmentI) => + (acc, s, segmentI) => segmentI === i() ? acc - : acc + - (segment.end - segment.start) / segment.timescale, + : acc + (s.end - s.start) / s.timescale, 0, ); @@ -532,8 +553,7 @@ export function ClipTrack( const prevSegment = segments()[i() - 1]; const prevSegmentIsSameClip = prevSegment?.recordingSegment !== undefined - ? prevSegment.recordingSegment === - segment.recordingSegment + ? prevSegment.recordingSegment === seg.recordingSegment : false; function update(event: MouseEvent) { @@ -541,15 +561,15 @@ export function ClipTrack( initialStart + (event.clientX - downEvent.clientX) * secsPerPixel() * - segment.timescale; + seg.timescale; const clampedStart = Math.min( Math.max( newStart, prevSegmentIsSameClip ? prevSegment.end : 0, - segment.end - maxDuration, + seg.end - maxDuration, ), - segment.end - 1, + seg.end - 1, ); setStartHandleDrag({ @@ -590,22 +610,23 @@ export function ClipTrack( {(() => { const ctx = useSegmentContext(); + const seg = segment(); return ( 100}>
{hasMultipleRecordingSegments() - ? `Clip ${segment.recordingSegment}` + ? `Clip ${seg.recordingSegment}` : "Clip"}
{" "} - {formatTime(segment.end - segment.start)} - + {formatTime(seg.end - seg.start)} +
- {segment.timescale}x + {seg.timescale}x
@@ -617,37 +638,36 @@ export function ClipTrack( position="end" class="opacity-0 group-hover:opacity-100" onMouseDown={(downEvent) => { - const end = segment.end; + const seg = segment(); + const end = seg.end; if (split()) return; const maxSegmentDuration = editorInstance.recordings.segments[ - segment.recordingSegment ?? 0 + seg.recordingSegment ?? 0 ].display.duration; const availableTimelineDuration = editorInstance.recordingDuration - segments().reduce( - (acc, segment, segmentI) => + (acc, s, segmentI) => segmentI === i() ? acc - : acc + - (segment.end - segment.start) / segment.timescale, + : acc + (s.end - s.start) / s.timescale, 0, ); const nextSegment = segments()[i() + 1]; const nextSegmentIsSameClip = nextSegment?.recordingSegment !== undefined - ? nextSegment.recordingSegment === - segment.recordingSegment + ? nextSegment.recordingSegment === seg.recordingSegment : false; function update(event: MouseEvent) { const deltaRecorded = (event.clientX - downEvent.clientX) * secsPerPixel() * - segment.timescale; + seg.timescale; const newEnd = end + deltaRecorded; setProject( @@ -658,13 +678,12 @@ export function ClipTrack( Math.max( Math.min( newEnd, - // availableTimelineDuration is in timeline seconds; convert to recorded seconds - end + availableTimelineDuration * segment.timescale, + end + availableTimelineDuration * seg.timescale, nextSegmentIsSameClip ? nextSegment.start : maxSegmentDuration, ), - segment.start + 1, + seg.start + 1, ), ); } @@ -737,7 +756,7 @@ export function ClipTrack( ); }} - + ); } diff --git a/apps/desktop/src/routes/editor/Timeline/ZoomTrack.tsx b/apps/desktop/src/routes/editor/Timeline/ZoomTrack.tsx index 1d476bdf8b..d3238d6469 100644 --- a/apps/desktop/src/routes/editor/Timeline/ZoomTrack.tsx +++ b/apps/desktop/src/routes/editor/Timeline/ZoomTrack.tsx @@ -8,6 +8,7 @@ import { createRoot, createSignal, For, + Index, Match, Show, Switch, @@ -44,7 +45,19 @@ export function ZoomTrack(props: { projectActions, } = useEditorContext(); - const { duration, secsPerPixel } = useTimelineContext(); + const { duration, secsPerPixel, isSegmentVisible } = useTimelineContext(); + + const visibleZoomIndices = createMemo(() => { + const zoomSegments = project.timeline?.zoomSegments ?? []; + const visible: number[] = []; + for (let i = 0; i < zoomSegments.length; i++) { + const seg = zoomSegments[i]; + if (isSegmentVisible(seg.start, seg.end)) { + visible.push(i); + } + } + return visible; + }); const [creatingSegmentViaDrag, setCreatingSegmentViaDrag] = createSignal(false); @@ -266,8 +279,8 @@ export function ZoomTrack(props: { }); }} > - 0} fallback={
Click to add zoom segment
@@ -277,331 +290,352 @@ export function ZoomTrack(props: {
} > - {(segment, i) => { - const { setTrackState } = useTrackContext(); - - const zoomPercentage = () => { - const amount = segment.amount; - return `${amount.toFixed(1)}x`; - }; - - const zoomSegments = () => project.timeline?.zoomSegments ?? []; - - function createMouseDownDrag( - setup: () => T, - _update: (e: MouseEvent, v: T, initialMouseX: number) => void, - ) { - return (downEvent: MouseEvent) => { - if (editorState.timeline.interactMode !== "seek") return; + + {(segmentIndex) => { + const i = segmentIndex; + const segment = () => (project.timeline?.zoomSegments ?? [])[i()]; + const { setTrackState } = useTrackContext(); + + const zoomPercentage = () => { + const seg = segment(); + if (!seg) return "1.0x"; + const amount = seg.amount; + return `${amount.toFixed(1)}x`; + }; - downEvent.stopPropagation(); + const zoomSegments = () => project.timeline?.zoomSegments ?? []; - const initial = setup(); + function createMouseDownDrag( + setup: () => T, + _update: (e: MouseEvent, v: T, initialMouseX: number) => void, + ) { + return (downEvent: MouseEvent) => { + if (editorState.timeline.interactMode !== "seek") return; - let moved = false; - let initialMouseX: null | number = null; + downEvent.stopPropagation(); - setTrackState("draggingSegment", true); + const initial = setup(); - const resumeHistory = projectHistory.pause(); + let moved = false; + let initialMouseX: null | number = null; - props.onDragStateChanged({ type: "movePending" }); + setTrackState("draggingSegment", true); - function finish(e: MouseEvent) { - resumeHistory(); - if (!moved) { - e.stopPropagation(); + const resumeHistory = projectHistory.pause(); - const currentSelection = editorState.timeline.selection; - const segmentIndex = i(); - const isMultiSelect = e.ctrlKey || e.metaKey; - const isRangeSelect = e.shiftKey; + props.onDragStateChanged({ type: "movePending" }); - if (isRangeSelect && currentSelection?.type === "zoom") { - // Range selection: select from last selected to current - const existingIndices = currentSelection.indices; - const lastIndex = - existingIndices[existingIndices.length - 1]; - const start = Math.min(lastIndex, segmentIndex); - const end = Math.max(lastIndex, segmentIndex); - const rangeIndices: number[] = []; - for (let idx = start; idx <= end; idx++) { - rangeIndices.push(idx); - } + function finish(e: MouseEvent) { + resumeHistory(); + if (!moved) { + e.stopPropagation(); + + const currentSelection = editorState.timeline.selection; + const segmentIndex = i(); + const isMultiSelect = e.ctrlKey || e.metaKey; + const isRangeSelect = e.shiftKey; + + if (isRangeSelect && currentSelection?.type === "zoom") { + // Range selection: select from last selected to current + const existingIndices = currentSelection.indices; + const lastIndex = + existingIndices[existingIndices.length - 1]; + const start = Math.min(lastIndex, segmentIndex); + const end = Math.max(lastIndex, segmentIndex); + const rangeIndices: number[] = []; + for (let idx = start; idx <= end; idx++) { + rangeIndices.push(idx); + } - setEditorState("timeline", "selection", { - type: "zoom", - indices: rangeIndices, - }); - } else if (isMultiSelect) { - // Handle multi-selection with Ctrl/Cmd+click - if (currentSelection?.type === "zoom") { - const baseIndices = currentSelection.indices; - const exists = baseIndices.includes(segmentIndex); - const newIndices = exists - ? baseIndices.filter((idx) => idx !== segmentIndex) - : [...baseIndices, segmentIndex]; - - if (newIndices.length > 0) { + setEditorState("timeline", "selection", { + type: "zoom", + indices: rangeIndices, + }); + } else if (isMultiSelect) { + // Handle multi-selection with Ctrl/Cmd+click + if (currentSelection?.type === "zoom") { + const baseIndices = currentSelection.indices; + const exists = baseIndices.includes(segmentIndex); + const newIndices = exists + ? baseIndices.filter((idx) => idx !== segmentIndex) + : [...baseIndices, segmentIndex]; + + if (newIndices.length > 0) { + setEditorState("timeline", "selection", { + type: "zoom", + indices: newIndices, + }); + } else { + setEditorState("timeline", "selection", null); + } + } else { + // Start new multi-selection setEditorState("timeline", "selection", { type: "zoom", - indices: newIndices, + indices: [segmentIndex], }); - } else { - setEditorState("timeline", "selection", null); } } else { - // Start new multi-selection + // Normal single selection setEditorState("timeline", "selection", { type: "zoom", indices: [segmentIndex], }); } - } else { - // Normal single selection - setEditorState("timeline", "selection", { - type: "zoom", - indices: [segmentIndex], - }); + props.handleUpdatePlayhead(e); } - props.handleUpdatePlayhead(e); + props.onDragStateChanged({ type: "idle" }); + setTrackState("draggingSegment", false); } - props.onDragStateChanged({ type: "idle" }); - setTrackState("draggingSegment", false); - } - function update(event: MouseEvent) { - if (Math.abs(event.clientX - downEvent.clientX) > 2) { - if (!moved) { - moved = true; - initialMouseX = event.clientX; - props.onDragStateChanged({ - type: "moving", - }); + function update(event: MouseEvent) { + if (Math.abs(event.clientX - downEvent.clientX) > 2) { + if (!moved) { + moved = true; + initialMouseX = event.clientX; + props.onDragStateChanged({ + type: "moving", + }); + } } + + if (initialMouseX === null) return; + + _update(event, initial, initialMouseX); } - if (initialMouseX === null) return; - - _update(event, initial, initialMouseX); - } - - createRoot((dispose) => { - createEventListenerMap(window, { - mousemove: (e) => { - update(e); - }, - mouseup: (e) => { - update(e); - finish(e); - dispose(); - }, + createRoot((dispose) => { + createEventListenerMap(window, { + mousemove: (e) => { + update(e); + }, + mouseup: (e) => { + update(e); + finish(e); + dispose(); + }, + }); }); - }); - }; - } - - const isSelected = createMemo(() => { - const selection = editorState.timeline.selection; - if (!selection || selection.type !== "zoom") return false; + }; + } - const segmentIndex = project.timeline?.zoomSegments?.findIndex( - (s) => s.start === segment.start && s.end === segment.end, - ); + const isSelected = createMemo(() => { + const selection = editorState.timeline.selection; + if (!selection || selection.type !== "zoom") return false; + const seg = segment(); + if (!seg) return false; - // Support both single selection (index) and multi-selection (indices) - if (segmentIndex === undefined || segmentIndex === -1) return false; + const segmentIndex = project.timeline?.zoomSegments?.findIndex( + (s) => s.start === seg.start && s.end === seg.end, + ); - return selection.indices.includes(segmentIndex); - }); + if (segmentIndex === undefined || segmentIndex === -1) + return false; - return ( - { - e.stopPropagation(); - - if (editorState.timeline.interactMode === "split") { - const rect = e.currentTarget.getBoundingClientRect(); - const fraction = (e.clientX - rect.left) / rect.width; - - const splitTime = fraction * (segment.end - segment.start); - - projectActions.splitZoomSegment(i(), splitTime); - } - }} - > - { - const start = segment.start; - - let minValue = 0; - - const maxValue = segment.end - 1; - - for (let i = zoomSegments().length - 1; i >= 0; i--) { - const segment = zoomSegments()[i]!; - if (segment.end <= start) { - minValue = segment.end; - break; - } - } + return selection.indices.includes(segmentIndex); + }); - return { start, minValue, maxValue }; - }, - (e, value, initialMouseX) => { - const newStart = - value.start + - (e.clientX - initialMouseX) * secsPerPixel(); - - setProject( - "timeline", - "zoomSegments", - i(), - "start", - Math.min( - value.maxValue, - Math.max(value.minValue, newStart), - ), - ); - - setProject( - "timeline", - "zoomSegments", - produce((s) => { - s.sort((a, b) => a.start - b.start); - }), - ); - }, - )} - /> - { - const original = { ...segment }; - - const prevSegment = zoomSegments()[i() - 1]; - const nextSegment = zoomSegments()[i() + 1]; - - const minStart = prevSegment?.end ?? 0; - const maxEnd = nextSegment?.start ?? duration(); - - return { - original, - minStart, - maxEnd, - }; - }, - (e, value, initialMouseX) => { - const rawDelta = - (e.clientX - initialMouseX) * secsPerPixel(); - - const newStart = value.original.start + rawDelta; - const newEnd = value.original.end + rawDelta; - - let delta = rawDelta; - - if (newStart < value.minStart) - delta = value.minStart - value.original.start; - else if (newEnd > value.maxEnd) - delta = value.maxEnd - value.original.end; - - setProject("timeline", "zoomSegments", i(), { - start: value.original.start + delta, - end: value.original.end + delta, - }); - }, - )} - > - {(() => { - const ctx = useSegmentContext(); - - return ( - - -
- -
- - -
- - {zoomPercentage()} -
- - -
- Zoom -
- - {zoomPercentage()} -
-
- - - ); - })()} - - { - const end = segment.end; - - const minValue = segment.start + 1; - - let maxValue = duration(); - - for (let i = 0; i < zoomSegments().length; i++) { - const segment = zoomSegments()[i]!; - if (segment.start > end) { - maxValue = segment.start; - break; + return ( + + {(seg) => ( + { + e.stopPropagation(); + + if (editorState.timeline.interactMode === "split") { + const rect = e.currentTarget.getBoundingClientRect(); + const fraction = (e.clientX - rect.left) / rect.width; + + const splitTime = fraction * (seg().end - seg().start); + + projectActions.splitZoomSegment(i(), splitTime); } - } - - return { end, minValue, maxValue }; - }, - (e, value, initialMouseX) => { - const newEnd = - value.end + (e.clientX - initialMouseX) * secsPerPixel(); - - setProject( - "timeline", - "zoomSegments", - i(), - "end", - Math.min( - value.maxValue, - Math.max(value.minValue, newEnd), - ), - ); - - setProject( - "timeline", - "zoomSegments", - produce((s) => { - s.sort((a, b) => a.start - b.start); - }), - ); - }, + }} + > + { + const start = seg().start; + + let minValue = 0; + + const maxValue = seg().end - 1; + + for ( + let idx = zoomSegments().length - 1; + idx >= 0; + idx-- + ) { + const zs = zoomSegments()[idx]!; + if (zs.end <= start) { + minValue = zs.end; + break; + } + } + + return { start, minValue, maxValue }; + }, + (e, value, initialMouseX) => { + const newStart = + value.start + + (e.clientX - initialMouseX) * secsPerPixel(); + + setProject( + "timeline", + "zoomSegments", + i(), + "start", + Math.min( + value.maxValue, + Math.max(value.minValue, newStart), + ), + ); + + setProject( + "timeline", + "zoomSegments", + produce((s) => { + s.sort((a, b) => a.start - b.start); + }), + ); + }, + )} + /> + { + const original = { ...seg() }; + + const prevSegment = zoomSegments()[i() - 1]; + const nextSegment = zoomSegments()[i() + 1]; + + const minStart = prevSegment?.end ?? 0; + const maxEnd = nextSegment?.start ?? duration(); + + return { + original, + minStart, + maxEnd, + }; + }, + (e, value, initialMouseX) => { + const rawDelta = + (e.clientX - initialMouseX) * secsPerPixel(); + + const newStart = value.original.start + rawDelta; + const newEnd = value.original.end + rawDelta; + + let delta = rawDelta; + + if (newStart < value.minStart) + delta = value.minStart - value.original.start; + else if (newEnd > value.maxEnd) + delta = value.maxEnd - value.original.end; + + setProject("timeline", "zoomSegments", i(), { + start: value.original.start + delta, + end: value.original.end + delta, + }); + }, + )} + > + {(() => { + const ctx = useSegmentContext(); + + return ( + + +
+ +
+ + +
+ + {zoomPercentage()} +
+ + +
+ Zoom +
+ + {zoomPercentage()} +
+
+ + + ); + })()} + + { + const end = seg().end; + + const minValue = seg().start + 1; + + let maxValue = duration(); + + for ( + let idx = 0; + idx < zoomSegments().length; + idx++ + ) { + const zs = zoomSegments()[idx]!; + if (zs.start > end) { + maxValue = zs.start; + break; + } + } + + return { end, minValue, maxValue }; + }, + (e, value, initialMouseX) => { + const newEnd = + value.end + + (e.clientX - initialMouseX) * secsPerPixel(); + + setProject( + "timeline", + "zoomSegments", + i(), + "end", + Math.min( + value.maxValue, + Math.max(value.minValue, newEnd), + ), + ); + + setProject( + "timeline", + "zoomSegments", + produce((s) => { + s.sort((a, b) => a.start - b.start); + }), + ); + }, + )} + /> + )} - /> - - ); - }} - + + ); + }} + + { const { editorState: state } = useEditorContext(); - const markingResolution = () => - TIMELINE_MARKING_RESOLUTIONS.find( - (r) => state.timeline.transform.zoom / r <= MAX_TIMELINE_MARKINGS, - ) ?? 30; + const markingResolution = createMemo( + () => + TIMELINE_MARKING_RESOLUTIONS.find( + (r) => state.timeline.transform.zoom / r <= MAX_TIMELINE_MARKINGS, + ) ?? 30, + ); + + const visibleTimeRange = createMemo(() => { + const { transform } = state.timeline; + const start = transform.position - SEGMENT_RENDER_PADDING; + const end = + transform.position + transform.zoom + SEGMENT_RENDER_PADDING; + return { start: Math.max(0, start), end }; + }); + + const isSegmentVisible = (segmentStart: number, segmentEnd: number) => { + const range = visibleTimeRange(); + return segmentEnd >= range.start && segmentStart <= range.end; + }; return { duration: () => props.duration, secsPerPixel: () => props.secsPerPixel, timelineBounds: props.timelineBounds, markingResolution, + visibleTimeRange, + isSegmentVisible, }; }, null!, diff --git a/core b/core new file mode 100644 index 0000000000..9e3a488172 Binary files /dev/null and b/core differ diff --git a/crates/editor/src/playback.rs b/crates/editor/src/playback.rs index 272a5312cd..a94afec572 100644 --- a/crates/editor/src/playback.rs +++ b/crates/editor/src/playback.rs @@ -4,14 +4,22 @@ use cap_audio::{ use cap_media::MediaError; use cap_media_info::AudioInfo; use cap_project::{ProjectConfiguration, XY}; -use cap_rendering::{ProjectUniforms, RenderVideoConstants}; +use cap_rendering::{DecodedSegmentFrames, ProjectUniforms, RenderVideoConstants}; use cpal::{ BufferSize, SampleFormat, SupportedBufferSize, traits::{DeviceTrait, HostTrait, StreamTrait}, }; -use std::{sync::Arc, time::Duration}; -use tokio::{sync::watch, time::Instant}; -use tracing::{error, info, warn}; +use futures::stream::{FuturesUnordered, StreamExt}; +use std::{ + collections::{HashSet, VecDeque}, + sync::Arc, + time::Duration, +}; +use tokio::{ + sync::{mpsc as tokio_mpsc, watch}, + time::Instant, +}; +use tracing::{error, info, trace, warn}; use crate::{ audio::{AudioPlaybackBuffer, AudioSegment}, @@ -20,6 +28,9 @@ use crate::{ segments::get_audio_segments, }; +const PREFETCH_BUFFER_SIZE: usize = 16; +const PARALLEL_DECODE_TASKS: usize = 4; + #[derive(Debug)] pub enum PlaybackStartError { InvalidFps, @@ -46,6 +57,12 @@ pub struct PlaybackHandle { event_rx: watch::Receiver, } +struct PrefetchedFrame { + frame_number: u32, + segment_frames: DecodedSegmentFrames, + segment_index: u32, +} + impl Playback { pub async fn start( self, @@ -70,6 +87,99 @@ impl Playback { event_rx, }; + let (prefetch_tx, mut prefetch_rx) = + tokio_mpsc::channel::(PREFETCH_BUFFER_SIZE * 2); + let (frame_request_tx, mut frame_request_rx) = watch::channel(self.start_frame_number); + + let prefetch_stop_rx = stop_rx.clone(); + let prefetch_project = self.project.clone(); + let prefetch_segment_medias = self.segment_medias.clone(); + let prefetch_duration = if let Some(timeline) = &self.project.borrow().timeline { + timeline.duration() + } else { + f64::MAX + }; + + tokio::spawn(async move { + let mut next_prefetch_frame = *frame_request_rx.borrow(); + let mut in_flight: FuturesUnordered<_> = FuturesUnordered::new(); + let mut in_flight_frames: HashSet = HashSet::new(); + + loop { + if *prefetch_stop_rx.borrow() { + break; + } + + if let Ok(true) = frame_request_rx.has_changed() { + let requested = *frame_request_rx.borrow_and_update(); + if requested > next_prefetch_frame { + next_prefetch_frame = requested; + in_flight_frames.retain(|&f| f >= requested); + } + } + + while in_flight.len() < PARALLEL_DECODE_TASKS { + let frame_num = next_prefetch_frame; + let prefetch_time = frame_num as f64 / fps_f64; + + if prefetch_time >= prefetch_duration { + break; + } + + if in_flight_frames.contains(&frame_num) { + next_prefetch_frame += 1; + continue; + } + + let project = prefetch_project.borrow().clone(); + + if let Some((segment_time, segment)) = project.get_segment_time(prefetch_time) + && let Some(segment_media) = + prefetch_segment_medias.get(segment.recording_clip as usize) + { + let clip_offsets = project + .clips + .iter() + .find(|v| v.index == segment.recording_clip) + .map(|v| v.offsets) + .unwrap_or_default(); + + let decoders = segment_media.decoders.clone(); + let hide_camera = project.camera.hide; + let segment_index = segment.recording_clip; + + in_flight_frames.insert(frame_num); + + in_flight.push(async move { + let result = decoders + .get_frames(segment_time as f32, !hide_camera, clip_offsets) + .await; + (frame_num, segment_index, result) + }); + } + + next_prefetch_frame += 1; + } + + tokio::select! { + biased; + + Some((frame_num, segment_index, result)) = in_flight.next() => { + in_flight_frames.remove(&frame_num); + if let Some(segment_frames) = result { + let _ = prefetch_tx.send(PrefetchedFrame { + frame_number: frame_num, + segment_frames, + segment_index, + }).await; + } + } + + _ = tokio::time::sleep(Duration::from_millis(1)), if in_flight.is_empty() => {} + } + } + }); + tokio::spawn(async move { let start = Instant::now(); @@ -90,8 +200,20 @@ impl Playback { let frame_duration = Duration::from_secs_f64(1.0 / fps_f64); let mut frame_number = self.start_frame_number; + let mut prefetch_buffer: VecDeque = + VecDeque::with_capacity(PREFETCH_BUFFER_SIZE); + let max_frame_skip = 3u32; 'playback: loop { + while let Ok(prefetched) = prefetch_rx.try_recv() { + if prefetched.frame_number >= frame_number { + prefetch_buffer.push_back(prefetched); + if prefetch_buffer.len() > PREFETCH_BUFFER_SIZE { + prefetch_buffer.pop_front(); + } + } + } + let frame_offset = frame_number.saturating_sub(self.start_frame_number) as f64; let next_deadline = start + frame_duration.mul_f64(frame_offset); @@ -111,30 +233,50 @@ impl Playback { let project = self.project.borrow().clone(); - let Some((segment_time, segment)) = project.get_segment_time(playback_time) else { - break; - }; + let prefetched_idx = prefetch_buffer + .iter() + .position(|p| p.frame_number == frame_number); + + let segment_frames_opt = if let Some(idx) = prefetched_idx { + let prefetched = prefetch_buffer.remove(idx).unwrap(); + Some((prefetched.segment_frames, prefetched.segment_index)) + } else { + let Some((segment_time, segment)) = project.get_segment_time(playback_time) + else { + break; + }; - let Some(segment_media) = self.segment_medias.get(segment.recording_clip as usize) - else { - continue; - }; + let Some(segment_media) = + self.segment_medias.get(segment.recording_clip as usize) + else { + frame_number = frame_number.saturating_add(1); + continue; + }; - let clip_offsets = project - .clips - .iter() - .find(|v| v.index == segment.recording_clip) - .map(|v| v.offsets) - .unwrap_or_default(); + let clip_offsets = project + .clips + .iter() + .find(|v| v.index == segment.recording_clip) + .map(|v| v.offsets) + .unwrap_or_default(); + + let data = tokio::select! { + _ = stop_rx.changed() => break 'playback, + data = segment_media + .decoders + .get_frames(segment_time as f32, !project.camera.hide, clip_offsets) => data, + }; - let data = tokio::select! { - _ = stop_rx.changed() => break 'playback, - data = segment_media - .decoders - .get_frames(segment_time as f32, !project.camera.hide, clip_offsets) => data, + data.map(|frames| (frames, segment.recording_clip)) }; - if let Some(segment_frames) = data { + if let Some((segment_frames, segment_index)) = segment_frames_opt { + let Some(segment_media) = self.segment_medias.get(segment_index as usize) + else { + frame_number = frame_number.saturating_add(1); + continue; + }; + let uniforms = ProjectUniforms::new( &self.render_constants, &project, @@ -158,7 +300,20 @@ impl Playback { + (start.elapsed().as_secs_f64() * fps_f64).floor() as u32; if frame_number < expected_frame { - frame_number = expected_frame; + let frames_behind = expected_frame - frame_number; + if frames_behind <= max_frame_skip { + frame_number = expected_frame; + trace!("Skipping {} frames to catch up", frames_behind); + } else { + frame_number += max_frame_skip; + trace!( + "Limiting frame skip to {} (was {} behind)", + max_frame_skip, frames_behind + ); + } + + prefetch_buffer.retain(|p| p.frame_number >= frame_number); + let _ = frame_request_tx.send(frame_number); } } diff --git a/crates/rendering/src/decoder/mod.rs b/crates/rendering/src/decoder/mod.rs index 8875f63246..47eb1358fb 100644 --- a/crates/rendering/src/decoder/mod.rs +++ b/crates/rendering/src/decoder/mod.rs @@ -47,7 +47,7 @@ pub fn pts_to_frame(pts: i64, time_base: Rational, fps: u32) -> u32 { .round() as u32 } -pub const FRAME_CACHE_SIZE: usize = 100; +pub const FRAME_CACHE_SIZE: usize = 500; #[derive(Clone)] pub struct AsyncVideoDecoderHandle {