From 38f3e55481cd3dee931be6405fc6254f9d4b0b39 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 3 Sep 2026 16:48:00 +0100 Subject: [PATCH 01/20] improve: index cursor data and prepare export zoom incrementally --- crates/project/src/cursor.rs | 86 ++- crates/rendering/src/cursor_interpolation.rs | 592 +++++++++++++++++- crates/rendering/src/lib.rs | 8 - .../tests/export_zoom_equivalence.rs | 153 +++++ 4 files changed, 820 insertions(+), 19 deletions(-) create mode 100644 crates/rendering/tests/export_zoom_equivalence.rs diff --git a/crates/project/src/cursor.rs b/crates/project/src/cursor.rs index a5a4e542bda..8dad727e58c 100644 --- a/crates/project/src/cursor.rs +++ b/crates/project/src/cursor.rs @@ -5,6 +5,7 @@ use std::ops::Range; pub const SHORT_CURSOR_SHAPE_DEBOUNCE_MS: f64 = 1000.0; use std::fs::File; +use std::io::BufReader; use std::path::{Path, PathBuf}; use crate::XY; @@ -59,7 +60,8 @@ pub struct CursorData { impl CursorData { pub fn load_from_file(path: &Path) -> Result { let file = File::open(path).map_err(|e| format!("Failed to open cursor file: {e}"))?; - serde_json::from_reader(file).map_err(|e| format!("Failed to parse cursor data: {e}")) + serde_json::from_reader(BufReader::new(file)) + .map_err(|e| format!("Failed to parse cursor data: {e}")) } } @@ -72,7 +74,8 @@ pub struct CursorEvents { impl CursorEvents { pub fn load_from_file(path: &Path) -> Result { let file = File::open(path).map_err(|e| format!("Failed to open cursor file: {e}"))?; - serde_json::from_reader(file).map_err(|e| format!("Failed to parse cursor data: {e}")) + serde_json::from_reader(BufReader::new(file)) + .map_err(|e| format!("Failed to parse cursor data: {e}")) } pub fn stabilize_short_lived_cursor_shapes( @@ -279,6 +282,85 @@ struct CursorSegment { mod tests { use super::*; + #[test] + fn buffered_load_preserves_events_and_legacy_images_across_read_boundaries() { + let data = CursorData { + clicks: vec![click_event(0.0, "pointer"), click_event(9876.5, "文字🖱")], + moves: (0..2000) + .map(|index| move_event(f64::from(index) * 16.67, "文字🖱")) + .collect(), + cursor_images: CursorImages(HashMap::from([( + "文字🖱".to_string(), + CursorImage { + path: "cursors/文字.png".into(), + hotspot: XY::new(0.125, 0.75), + }, + )])), + }; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("cursor.json"); + let json = serde_json::to_vec(&data).unwrap(); + std::fs::write(&path, &json).unwrap(); + + let expected: CursorData = serde_json::from_reader(File::open(&path).unwrap()).unwrap(); + let events = CursorEvents::load_from_file(&path).unwrap(); + assert_eq!(events.moves, expected.moves); + assert_eq!(events.clicks, expected.clicks); + let legacy = CursorData::load_from_file(&path).unwrap(); + assert_eq!( + serde_json::to_value(legacy).unwrap(), + serde_json::to_value(expected).unwrap() + ); + + let events_json = serde_json::to_vec(&events).unwrap(); + std::fs::write(&path, events_json).unwrap(); + let expected: CursorEvents = serde_json::from_reader(File::open(&path).unwrap()).unwrap(); + let loaded_events = CursorEvents::load_from_file(&path).unwrap(); + assert_eq!(loaded_events.moves, expected.moves); + assert_eq!(loaded_events.clicks, expected.clicks); + } + + #[test] + fn buffered_load_preserves_parse_and_open_errors() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("cursor.json"); + for json in [ + "", + "{", + r#"{"moves":[],"clicks":[]} trailing"#, + r#"{"moves":[{"time_ms":1e999}],"clicks":[]}"#, + r#"{"moves":[],"clicks":[],"cursor_images":null}"#, + ] { + for padding in [0, 8191, 8192] { + std::fs::write(&path, format!("{}{json}", " ".repeat(padding))).unwrap(); + let expected_events = + serde_json::from_reader::<_, CursorEvents>(File::open(&path).unwrap()) + .map_err(|error| format!("Failed to parse cursor data: {error}")); + assert_eq!( + CursorEvents::load_from_file(&path).err(), + expected_events.err() + ); + let expected_legacy = + serde_json::from_reader::<_, CursorData>(File::open(&path).unwrap()) + .map_err(|error| format!("Failed to parse cursor data: {error}")); + assert_eq!( + CursorData::load_from_file(&path).err(), + expected_legacy.err() + ); + } + } + let missing = dir.path().join("missing.json"); + let expected = format!( + "Failed to open cursor file: {}", + File::open(&missing).unwrap_err() + ); + assert_eq!( + CursorEvents::load_from_file(&missing).unwrap_err(), + expected + ); + assert_eq!(CursorData::load_from_file(&missing).unwrap_err(), expected); + } + fn move_event(time_ms: f64, cursor_id: &str) -> CursorMoveEvent { CursorMoveEvent { active_modifiers: vec![], diff --git a/crates/rendering/src/cursor_interpolation.rs b/crates/rendering/src/cursor_interpolation.rs index 281bace64d4..2c4f3fc0c35 100644 --- a/crates/rendering/src/cursor_interpolation.rs +++ b/crates/rendering/src/cursor_interpolation.rs @@ -142,7 +142,11 @@ fn next_click_within( clicks.get(idx).filter(|c| c.time_ms - time_ms <= window_ms) } -fn position_at_time(moves: &[CursorMoveEvent], time_ms: f64) -> (f64, f64) { +fn position_at_time( + moves: &[CursorMoveEvent], + time_ms: f64, + moves_are_ordered: bool, +) -> (f64, f64) { if moves.is_empty() { return (0.0, 0.0); } @@ -154,9 +158,16 @@ fn position_at_time(moves: &[CursorMoveEvent], time_ms: f64) -> (f64, f64) { { return (last.x, last.y); } - moves - .windows(2) - .find_map(|w| { + let window = if moves_are_ordered && !time_ms.is_nan() { + let end = moves.partition_point(|event| event.time_ms <= time_ms); + end.checked_sub(1).and_then(|start| moves.get(start..=end)) + } else { + moves + .windows(2) + .find(|w| time_ms >= w[0].time_ms && time_ms < w[1].time_ms) + }; + window + .and_then(|w| { if time_ms >= w[0].time_ms && time_ms < w[1].time_ms { let dt = w[1].time_ms - w[0].time_ms; if dt > IDLE_GAP_THRESHOLD_MS { @@ -345,7 +356,7 @@ impl PrecomputedCursorTimeline { Self { timeline, - raw_cursor: cursor.clone(), + raw_cursor: CursorEvents::default(), has_smoothing: true, } } @@ -370,6 +381,8 @@ fn build_smoothed_timeline( return vec![]; } + let moves_are_ordered = + !cursor.clicks.is_empty() && moves.is_sorted_by(|a, b| a.time_ms <= b.time_ms); let presets = CursorSpringPresets::new(smoothing_config, click_spring); let mut context = CursorSpringContext::new(&cursor.clicks); let mut sim = SpringMassDamperSimulation::new(smoothing_config); @@ -416,7 +429,8 @@ fn build_smoothed_timeline( let target = if let Some(click) = next_click_within(&cursor.clicks, t_ms, CLICK_LOOKAHEAD_TARGET_MS) { - let (tx, ty) = position_at_time(moves, click.time_ms.min(end_time_ms)); + let (tx, ty) = + position_at_time(moves, click.time_ms.min(end_time_ms), moves_are_ordered); XY::new(tx as f32, ty as f32) } else { XY::new(cx as f32, cy as f32) @@ -481,9 +495,11 @@ fn interpolate_timeline( if events[idx].time <= query && idx + 1 < events.len() && query < events[idx + 1].time { (&events[idx], &events[idx + 1]) } else { - match events - .windows(2) - .find(|w| w[0].time <= query && query < w[1].time) + let end = events.partition_point(|event| event.time <= query); + match end + .checked_sub(1) + .and_then(|start| events.get(start..=end)) + .filter(|w| w[0].time <= query && query < w[1].time) { Some(w) => (&w[0], &w[1]), None => { @@ -830,4 +846,562 @@ mod tests { prev = pos; } } + fn linear_position_at_time(moves: &[CursorMoveEvent], time_ms: f64) -> (f64, f64) { + if moves.is_empty() { + return (0.0, 0.0); + } + if time_ms <= moves[0].time_ms { + return (moves[0].x, moves[0].y); + } + if let Some(last) = moves.last() + && time_ms >= last.time_ms + { + return (last.x, last.y); + } + moves + .windows(2) + .find_map(|w| { + if time_ms >= w[0].time_ms && time_ms < w[1].time_ms { + let dt = w[1].time_ms - w[0].time_ms; + if dt > IDLE_GAP_THRESHOLD_MS { + return Some((w[0].x, w[0].y)); + } + let u = if dt.abs() < 1e-9 { + 0.0 + } else { + (time_ms - w[0].time_ms) / dt + }; + Some(( + w[0].x + (w[1].x - w[0].x) * u, + w[0].y + (w[1].y - w[0].y) * u, + )) + } else { + None + } + }) + .unwrap_or_else(|| { + let l = moves.last().unwrap(); + (l.x, l.y) + }) + } + + fn linear_build_smoothed_timeline( + cursor: &CursorEvents, + moves: &[CursorMoveEvent], + smoothing_config: SpringMassDamperSimulationConfig, + click_spring: Option, + ) -> Vec { + if moves.is_empty() { + return vec![]; + } + + let presets = CursorSpringPresets::new(smoothing_config, click_spring); + let mut context = CursorSpringContext::new(&cursor.clicks); + let mut sim = SpringMassDamperSimulation::new(smoothing_config); + + let start_pos = XY::new(moves[0].x as f32, moves[0].y as f32); + sim.set_position(start_pos); + sim.set_velocity(XY::new(0.0, 0.0)); + sim.set_target_position(start_pos); + + let end_time_ms = moves.last().unwrap().time_ms; + let settle_end = end_time_ms + SPRING_SETTLE_EXTRA_MS; + + let capacity = ((settle_end / SIMULATION_STEP_MS).ceil() as usize) + 2; + let mut events = Vec::with_capacity(capacity); + let mut target_hint: usize = 0; + let mut cid_hint: usize = 0; + + events.push(SmoothedCursorEvent { + time: 0.0, + position: start_pos, + velocity: XY::new(0.0, 0.0), + cursor_id: moves[0].cursor_id.clone(), + }); + + let mut t_ms = SIMULATION_STEP_MS; + let mut lead_ms = spring_lag_ms(&smoothing_config); + + while t_ms <= settle_end { + let clamped_t = t_ms.min(end_time_ms); + + context.advance_to(t_ms); + let config = presets.config(context.profile(t_ms)); + sim.set_config(config); + lead_ms += (spring_lag_ms(&config) - lead_ms) * LEAD_SMOOTHING; + + let lead_t = (clamped_t + lead_ms).min(end_time_ms); + let (cx, cy) = position_at_time_hinted(moves, lead_t, &mut target_hint); + let _ = position_at_time_hinted(moves, clamped_t, &mut cid_hint); + let cid = cursor_id_at_time(moves, clamped_t, cid_hint).to_string(); + + let target = if let Some(click) = + next_click_within(&cursor.clicks, t_ms, CLICK_LOOKAHEAD_TARGET_MS) + { + let (tx, ty) = linear_position_at_time(moves, click.time_ms.min(end_time_ms)); + XY::new(tx as f32, ty as f32) + } else { + XY::new(cx as f32, cy as f32) + }; + + sim.set_target_position(target); + + sim.run(SIMULATION_STEP_MS as f32); + + events.push(SmoothedCursorEvent { + time: t_ms as f32, + position: sim.position, + velocity: sim.velocity, + cursor_id: cid, + }); + + t_ms += SIMULATION_STEP_MS; + } + + events + } + + fn linear_interpolate_timeline( + events: &[SmoothedCursorEvent], + query_ms: f64, + ) -> Option { + if events.is_empty() { + return None; + } + + let query = query_ms as f32; + + if query <= events[0].time { + let e = &events[0]; + return Some(InterpolatedCursorPosition { + position: Coord::new(XY::new(e.position.x as f64, e.position.y as f64)), + velocity: e.velocity, + cursor_id: e.cursor_id.clone(), + }); + } + + if query >= events.last().unwrap().time { + let e = events.last().unwrap(); + return Some(InterpolatedCursorPosition { + position: Coord::new(XY::new(e.position.x as f64, e.position.y as f64)), + velocity: e.velocity, + cursor_id: e.cursor_id.clone(), + }); + } + + let first_time = events[0].time; + let step = if events.len() > 1 { + events[1].time - events[0].time + } else { + SIMULATION_STEP_MS as f32 + }; + + let raw_idx = ((query - first_time) / step) as usize; + let idx = raw_idx.min(events.len().saturating_sub(2)); + + let (a, b) = if events[idx].time <= query + && idx + 1 < events.len() + && query < events[idx + 1].time + { + (&events[idx], &events[idx + 1]) + } else { + match events + .windows(2) + .find(|w| w[0].time <= query && query < w[1].time) + { + Some(w) => (&w[0], &w[1]), + None => { + let e = events.last().unwrap(); + return Some(InterpolatedCursorPosition { + position: Coord::new(XY::new(e.position.x as f64, e.position.y as f64)), + velocity: e.velocity, + cursor_id: e.cursor_id.clone(), + }); + } + } + }; + + let dt = b.time - a.time; + let t = if dt.abs() < 1e-6 { + 0.0 + } else { + ((query - a.time) / dt).clamp(0.0, 1.0) + }; + let inv = 1.0 - t; + + Some(InterpolatedCursorPosition { + position: Coord::new(XY::new( + (a.position.x * inv + b.position.x * t) as f64, + (a.position.y * inv + b.position.y * t) as f64, + )), + velocity: XY::new( + a.velocity.x * inv + b.velocity.x * t, + a.velocity.y * inv + b.velocity.y * t, + ), + cursor_id: a.cursor_id.clone(), + }) + } + + fn assert_position_bits( + expected: Option, + actual: Option, + ) { + match (expected, actual) { + (None, None) => {} + (Some(expected), Some(actual)) => { + assert_eq!( + expected.position.coord.x.to_bits(), + actual.position.coord.x.to_bits() + ); + assert_eq!( + expected.position.coord.y.to_bits(), + actual.position.coord.y.to_bits() + ); + assert_eq!(expected.velocity.x.to_bits(), actual.velocity.x.to_bits()); + assert_eq!(expected.velocity.y.to_bits(), actual.velocity.y.to_bits()); + assert_eq!(expected.cursor_id, actual.cursor_id); + } + (expected, actual) => panic!("cursor position mismatch: {expected:?} != {actual:?}"), + } + } + + fn assert_timeline_bits(expected: &[SmoothedCursorEvent], actual: &[SmoothedCursorEvent]) { + assert_eq!(expected.len(), actual.len()); + for (index, (expected, actual)) in expected.iter().zip(actual).enumerate() { + assert_eq!( + expected.time.to_bits(), + actual.time.to_bits(), + "time at {index}" + ); + assert_eq!( + expected.position.x.to_bits(), + actual.position.x.to_bits(), + "position.x at {index}" + ); + assert_eq!( + expected.position.y.to_bits(), + actual.position.y.to_bits(), + "position.y at {index}" + ); + assert_eq!( + expected.velocity.x.to_bits(), + actual.velocity.x.to_bits(), + "velocity.x at {index}" + ); + assert_eq!( + expected.velocity.y.to_bits(), + actual.velocity.y.to_bits(), + "velocity.y at {index}" + ); + assert_eq!(expected.cursor_id, actual.cursor_id, "cursor_id at {index}"); + } + } + + fn sampled_event(time: f32, index: usize) -> SmoothedCursorEvent { + SmoothedCursorEvent { + time, + position: XY::new(index as f32 * 0.013, index as f32 * -0.007), + velocity: XY::new(index as f32 * -0.002, index as f32 * 0.003), + cursor_id: (index % 3).to_string(), + } + } + + fn dense_cursor(duration_secs: usize) -> CursorEvents { + CursorEvents { + moves: (0..=duration_secs * 120) + .map(|index| { + let mut event = cursor_move( + index as f64 * 1000.0 / 120.0, + (index % 479) as f64 / 479.0, + (index % 283) as f64 / 283.0, + ); + event.cursor_id = (index / 31 % 3).to_string(); + event + }) + .collect(), + clicks: (1..duration_secs * 2) + .map(|index| click_event(index as f64 * 500.0, index % 2 != 0)) + .collect(), + } + } + + #[test] + fn smoothed_timeline_releases_unused_raw_events_without_changing_output() { + let cursor = dense_cursor(8); + let filtered = filter_cursor_shake(&cursor.moves); + let moves = decimate_cursor_moves(filtered.as_ref()); + let expected = linear_build_smoothed_timeline( + &cursor, + &moves, + DEFAULT_CLICK_SPRING, + Some(ClickSpringConfig::default()), + ); + let timeline = PrecomputedCursorTimeline::new( + &cursor, + Some(DEFAULT_CLICK_SPRING), + Some(ClickSpringConfig::default()), + ); + + assert!(timeline.has_smoothing); + assert!(timeline.raw_cursor.moves.is_empty()); + assert!(timeline.raw_cursor.clicks.is_empty()); + assert_timeline_bits(&expected, &timeline.timeline); + drop(moves); + drop(filtered); + drop(cursor); + + for index in 0..=500 { + let time_secs = index as f32 * 0.017_137; + assert_position_bits( + linear_interpolate_timeline(&expected, (time_secs * 1000.0) as f64), + timeline.interpolate(time_secs), + ); + } + } + + #[test] + fn raw_and_empty_timelines_preserve_source_events() { + let cursor = dense_cursor(2); + let timeline = PrecomputedCursorTimeline::new(&cursor, None, None); + assert!(!timeline.has_smoothing); + assert_eq!(timeline.raw_cursor.moves.len(), cursor.moves.len()); + assert_eq!(timeline.raw_cursor.clicks.len(), cursor.clicks.len()); + for time_secs in [-1.0, 0.0, 0.017, 0.5, 1.0, 2.0, 3.0] { + assert_position_bits( + interpolate_cursor(&cursor, time_secs, None), + timeline.interpolate(time_secs), + ); + } + + let empty = CursorEvents { + moves: Vec::new(), + clicks: vec![click_event(100.0, true)], + }; + let timeline = PrecomputedCursorTimeline::new(&empty, Some(DEFAULT_CLICK_SPRING), None); + assert!(!timeline.has_smoothing); + assert!(timeline.raw_cursor.moves.is_empty()); + assert_eq!(timeline.raw_cursor.clicks.len(), 1); + assert!(timeline.interpolate(0.1).is_none()); + } + + #[test] + fn cursor_position_lookup_matches_linear_reference() { + let cases = [ + vec![], + vec![0.0], + vec![f64::NAN], + vec![0.0, 0.0, 10.0, 10.0, 20.0, 100.0, 100.0], + vec![-0.0, 0.0, 0.0, 10.0, 100.0], + vec![0.0, 20.0, 10.0, 50.0, 100.0], + vec![0.0, 10.0, f64::NAN, 30.0, 100.0], + vec![f64::NAN, 10.0, 30.0, 100.0], + vec![0.0, 10.0, 30.0, f64::NAN], + vec![f64::NEG_INFINITY, 0.0, 10.0, f64::INFINITY], + vec![0.0, 0.000_000_000_1, 20.0, 100.0], + ]; + let queries = [ + f64::NEG_INFINITY, + -1.0, + -0.0, + 0.0, + 0.000_000_000_05, + 5.0, + 10.0, + 19.999, + 20.0, + 21.0, + 30.0, + 49.0, + 50.0, + 99.0, + 100.0, + 101.0, + f64::INFINITY, + f64::NAN, + ]; + for times in cases { + let moves: Vec<_> = times + .iter() + .enumerate() + .map(|(index, &time)| cursor_move(time, index as f64 * 0.17, index as f64 * -0.11)) + .collect(); + let ordered = moves.is_sorted_by(|a, b| a.time_ms <= b.time_ms); + for query in queries { + let expected = linear_position_at_time(&moves, query); + let actual = position_at_time(&moves, query, ordered); + assert_eq!( + expected.0.to_bits(), + actual.0.to_bits(), + "x for {times:?} at {query}" + ); + assert_eq!( + expected.1.to_bits(), + actual.1.to_bits(), + "y for {times:?} at {query}" + ); + } + } + } + + #[test] + fn complete_smoothed_timeline_matches_linear_reference() { + for variant in 0..7 { + let mut cursor = dense_cursor(8); + match variant { + 1 => cursor.moves[50].time_ms = cursor.moves[49].time_ms, + 2 => cursor.moves.swap(50, 100), + 3 => cursor.moves[100].time_ms = f64::NAN, + 4 => cursor.clicks.reverse(), + 5 => cursor.clicks.clear(), + 6 => cursor.moves[100].time_ms = f64::NEG_INFINITY, + _ => {} + } + let filtered = filter_cursor_shake(&cursor.moves); + let moves = decimate_cursor_moves(filtered.as_ref()); + for config in [ + DEFAULT_CLICK_SPRING, + DRAG_SPRING, + SpringMassDamperSimulationConfig { + tension: 470.0, + mass: 3.0, + friction: 70.0, + }, + ] { + for click_spring in [None, Some(ClickSpringConfig::default())] { + let expected = + linear_build_smoothed_timeline(&cursor, &moves, config, click_spring); + let actual = build_smoothed_timeline(&cursor, &moves, config, click_spring); + assert_timeline_bits(&expected, &actual); + for index in 0..=500 { + let query = index as f64 * 17.137; + assert_position_bits( + linear_interpolate_timeline(&expected, query), + interpolate_timeline(&actual, query), + ); + } + } + } + } + } + + #[test] + fn timeline_lookup_matches_linear_reference_at_long_times_and_duplicate_timestamps() { + assert_position_bits( + linear_interpolate_timeline(&[], 0.0), + interpolate_timeline(&[], 0.0), + ); + for duration_hours in [0.0, 1.0, 20.0] { + for interval_ms in [1.0, SIMULATION_STEP_MS] { + let origin_ms = duration_hours * 3_600_000.0; + let events: Vec<_> = [0.0, SIMULATION_STEP_MS as f32] + .into_iter() + .chain( + (0..512) + .map(|index| (origin_ms + 100.0 + index as f64 * interval_ms) as f32), + ) + .enumerate() + .map(|(index, time)| sampled_event(time, index)) + .collect(); + for query in [ + f64::NAN, + f64::NEG_INFINITY, + f64::INFINITY, + -1.0, + 0.0, + origin_ms, + origin_ms + 10_000.0, + ] { + assert_position_bits( + linear_interpolate_timeline(&events, query), + interpolate_timeline(&events, query), + ); + } + for event in &events { + for query in [ + event.time as f64 - 0.001, + event.time as f64, + event.time as f64 + 0.001, + event.time as f64 + interval_ms * 0.5, + ] { + assert_position_bits( + linear_interpolate_timeline(&events, query), + interpolate_timeline(&events, query), + ); + } + } + } + } + let duplicate_origin: Vec<_> = [0.0, 0.0, 0.0, 16.0, 16.0, 32.0] + .into_iter() + .enumerate() + .map(|(index, time)| sampled_event(time, index)) + .collect(); + for query in [-0.0, 0.0, 0.01, 8.0, 16.0, 16.01, 24.0, 32.0] { + assert_position_bits( + linear_interpolate_timeline(&duplicate_origin, query), + interpolate_timeline(&duplicate_origin, query), + ); + } + } + + #[test] + #[ignore] + fn benchmark_dense_cursor_lookup_against_linear_reference() { + use std::{hint::black_box, time::Instant}; + + for duration_secs in [60, 600] { + let cursor = dense_cursor(duration_secs); + let filtered = filter_cursor_shake(&cursor.moves); + let moves = decimate_cursor_moves(filtered.as_ref()); + let started = Instant::now(); + let expected = + linear_build_smoothed_timeline(&cursor, &moves, DEFAULT_CLICK_SPRING, None); + let baseline_time = started.elapsed(); + let started = Instant::now(); + let actual = build_smoothed_timeline(&cursor, &moves, DEFAULT_CLICK_SPRING, None); + let optimized_time = started.elapsed(); + assert_timeline_bits(black_box(&expected), black_box(&actual)); + println!( + "cursor_precompute duration_secs={duration_secs} moves={} clicks={} samples={} baseline_ms={:.3} optimized_ms={:.3}", + moves.len(), + cursor.clicks.len(), + actual.len(), + baseline_time.as_secs_f64() * 1000.0, + optimized_time.as_secs_f64() * 1000.0 + ); + } + + let events: Vec<_> = (0..=20 * 60 * 60 * 60) + .map(|index| { + let mut event = sampled_event((index as f64 * SIMULATION_STEP_MS) as f32, index); + event.cursor_id = String::new(); + event + }) + .collect(); + let queries: Vec<_> = (0..128) + .map(|index| 72_000_000.0 - index as f64 * 171.13 - 1.0) + .collect(); + let started = Instant::now(); + let expected: Vec<_> = queries + .iter() + .map(|&query| linear_interpolate_timeline(black_box(&events), black_box(query))) + .collect(); + let baseline_time = started.elapsed(); + let started = Instant::now(); + let actual: Vec<_> = queries + .iter() + .map(|&query| interpolate_timeline(black_box(&events), black_box(query))) + .collect(); + let optimized_time = started.elapsed(); + for (expected, actual) in expected.into_iter().zip(actual) { + assert_position_bits(expected, actual); + } + println!( + "cursor_seek duration_hours=20 samples={} queries={} baseline_ms={:.3} optimized_ms={:.3}", + events.len(), + queries.len(), + baseline_time.as_secs_f64() * 1000.0, + optimized_time.as_secs_f64() * 1000.0 + ); + } } diff --git a/crates/rendering/src/lib.rs b/crates/rendering/src/lib.rs index da28e5b9b6d..c9ced9ac994 100644 --- a/crates/rendering/src/lib.rs +++ b/crates/rendering/src/lib.rs @@ -1147,14 +1147,6 @@ pub async fn render_video_to_channel_nv12( }) .collect::>() }); - for timeline in &mut zoom_timelines { - timeline.precompute(); - } - if let Some(timelines) = &mut outgoing_zoom_timelines { - for timeline in timelines { - timeline.precompute(); - } - } let zoom_focus_interpolators_construct_ms = zoom_build_start.elapsed().as_millis() as u64; let mut frame_number = 0; diff --git a/crates/rendering/tests/export_zoom_equivalence.rs b/crates/rendering/tests/export_zoom_equivalence.rs new file mode 100644 index 00000000000..f282303d704 --- /dev/null +++ b/crates/rendering/tests/export_zoom_equivalence.rs @@ -0,0 +1,153 @@ +use cap_project::{CursorClickEvent, CursorEvents, CursorMoveEvent, ProjectConfiguration, XY}; +use cap_rendering::ZoomTransformTimeline; + +fn project() -> ProjectConfiguration { + ProjectConfiguration { + timeline: Some( + serde_json::from_value(serde_json::json!({ + "segments": [ + {"recordingClip": 0, "start": 2.0, "end": 14.0, "timescale": 1.5}, + {"recordingClip": 1, "start": 1.0, "end": 10.0, "timescale": 0.5}, + {"recordingClip": 0, "start": 0.0, "end": 4.0, "timescale": 1.0} + ], + "transitions": [ + {"segmentIndex": 0, "type": "cross-fade", "duration": 0.6}, + {"segmentIndex": 1, "type": "fade-through-black", "duration": 0.4} + ], + "zoomSegments": [ + {"start": 0.0, "end": 6.0, "amount": 2.0, "mode": "auto"}, + {"start": 6.0, "end": 9.0, "amount": 3.0, "mode": {"manual": {"x": 0.2, "y": 0.8}}}, + {"start": 11.0, "end": 14.0, "amount": 1.6, "mode": "auto", "instantAnimation": true}, + {"start": 18.0, "end": 25.0, "amount": 2.2, "mode": {"manual": {"x": 0.9, "y": 0.1}}} + ] + })) + .unwrap(), + ), + ..ProjectConfiguration::default() + } +} + +fn cursor(clip: u32) -> CursorEvents { + CursorEvents { + moves: (0..250) + .map(|i| CursorMoveEvent { + time_ms: f64::from(i) * 60.0, + x: f64::from((i * 7 + clip * 17) % 100) / 100.0, + y: f64::from((i * 3 + clip * 23) % 100) / 100.0, + cursor_id: "default".into(), + active_modifiers: Vec::new(), + }) + .collect(), + clicks: (0..20) + .map(|i| CursorClickEvent { + time_ms: f64::from(i) * 630.0, + cursor_num: 0, + cursor_id: "default".into(), + down: true, + active_modifiers: Vec::new(), + }) + .collect(), + } +} + +#[test] +fn incremental_export_zoom_matches_eager_across_clips_transitions_and_seeks() { + let project = project(); + for clip in 0..2 { + let cursor = cursor(clip); + for outgoing in [false, true] { + let make = || { + if outgoing { + ZoomTransformTimeline::from_project_for_outgoing_clip( + &project, + &cursor, + 30.0, + XY::new(1920, 1080), + clip, + ) + } else { + ZoomTransformTimeline::from_project_for_clip( + &project, + &cursor, + 30.0, + XY::new(1920, 1080), + clip, + ) + } + }; + let mut eager = make(); + eager.precompute(); + for fps in [24, 30, 60] { + let mut incremental = make(); + for frame in 0..30 * fps { + let time = frame as f32 / fps as f32; + incremental.ensure_precomputed_until((frame + 1) as f32 / fps as f32); + let previous = (time - 1.0 / fps as f32).max(0.0); + assert_eq!( + eager.snapped_within(previous, time), + incremental.snapped_within(previous, time), + ); + for query in [time, previous] { + let before = eager.sample(query); + let after = incremental.sample(query); + assert_eq!(before.t.to_bits(), after.t.to_bits()); + for (a, b) in [ + (before.bounds.top_left.x, after.bounds.top_left.x), + (before.bounds.top_left.y, after.bounds.top_left.y), + (before.bounds.bottom_right.x, after.bounds.bottom_right.x), + (before.bounds.bottom_right.y, after.bounds.bottom_right.y), + ] { + assert_eq!( + a.to_bits(), + b.to_bits(), + "clip={clip} outgoing={outgoing} fps={fps} frame={frame}" + ); + } + } + } + for time in [0.0, 24.5, 6.0, 11.01, 0.0, 29.9] { + let mut seeked = make(); + seeked.ensure_precomputed_until(time + 1.0 / fps as f32); + assert_eq!(eager.sample(time).bounds, seeked.sample(time).bounds); + assert_eq!(eager.sample(time).bounds, incremental.sample(time).bounds); + } + } + } + } +} + +#[test] +#[ignore] +fn benchmark_long_recording_zoom_startup() { + use std::{hint::black_box, time::Instant}; + let project = project(); + let cursor = cursor(0); + for duration in [60.0, 8640.0, 72000.0] { + let start = Instant::now(); + let mut eager = ZoomTransformTimeline::from_project_for_clip( + &project, + &cursor, + duration, + XY::new(1920, 1080), + 0, + ); + eager.precompute(); + let eager_ms = start.elapsed().as_secs_f64() * 1000.0; + let start = Instant::now(); + let mut incremental = ZoomTransformTimeline::from_project_for_clip( + &project, + &cursor, + duration, + XY::new(1920, 1080), + 0, + ); + incremental.ensure_precomputed_until(1.0 / 60.0); + let incremental_ms = start.elapsed().as_secs_f64() * 1000.0; + assert_eq!(eager.sample(0.0).bounds, incremental.sample(0.0).bounds); + black_box((eager, incremental)); + println!( + "{}", + serde_json::json!({"duration_seconds":duration,"eager_ms":eager_ms,"incremental_ms":incremental_ms,"scope":"zoom construction and first-frame simulation"}) + ); + } +} From 320fd4e5f496389ae4ae6c604e1bf64d6fade05b Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 3 Sep 2026 16:48:00 +0100 Subject: [PATCH 02/20] improve: reuse cursor interpolation across editor previews --- crates/editor/src/editor_instance.rs | 427 +++++++++++++++++++++++++-- 1 file changed, 404 insertions(+), 23 deletions(-) diff --git a/crates/editor/src/editor_instance.rs b/crates/editor/src/editor_instance.rs index 77121a1072b..6552e83c5b7 100644 --- a/crates/editor/src/editor_instance.rs +++ b/crates/editor/src/editor_instance.rs @@ -7,8 +7,9 @@ use cap_project::{ TimelineFrameMapping, TimelineSegment, XY, }; use cap_rendering::{ - ProjectRecordingsMeta, ProjectUniforms, RecordingSegmentDecoders, RenderVideoConstants, - SegmentVideoPaths, SharedWgpuDevice, Video, ZoomTransformTimeline, get_duration, + PrecomputedCursorTimeline, ProjectRecordingsMeta, ProjectUniforms, RecordingSegmentDecoders, + RenderVideoConstants, SegmentVideoPaths, SharedWgpuDevice, Video, ZoomTransformTimeline, + get_duration, spring_mass_damper::SpringMassDamperSimulationConfig, }; use std::{ path::{Path, PathBuf}, @@ -23,6 +24,83 @@ use tracing::warn; const PREVIEW_RENDER_MAX_ATTEMPTS: u32 = 3; const PREVIEW_RENDER_RETRY_DELAY_MS: u64 = 120; +const PREVIEW_CURSOR_CACHE_CAPACITY: usize = 2; + +#[derive(Default)] +struct PreviewCursorCache { + entries: Vec, +} + +struct PreviewCursorCacheEntry { + recording_clip: u32, + cursor: Arc, + settings: [u32; 6], + timeline: Arc, +} + +impl PreviewCursorCache { + fn get( + &mut self, + recording_clip: u32, + cursor: &Arc, + project: &ProjectConfiguration, + ) -> Option> { + if project.cursor.raw { + self.entries.clear(); + return None; + } + if cursor.moves.is_empty() { + self.entries + .retain(|entry| entry.recording_clip != recording_clip); + return None; + } + + let smoothing = SpringMassDamperSimulationConfig { + tension: project.cursor.tension, + mass: project.cursor.mass, + friction: project.cursor.friction, + }; + let click_spring = project.cursor.click_spring_config(); + let settings = [ + smoothing.tension.to_bits(), + smoothing.mass.to_bits(), + smoothing.friction.to_bits(), + click_spring.tension.to_bits(), + click_spring.mass.to_bits(), + click_spring.friction.to_bits(), + ]; + + if let Some(index) = self.entries.iter().position(|entry| { + entry.recording_clip == recording_clip + && Arc::ptr_eq(&entry.cursor, cursor) + && entry.settings == settings + }) { + let entry = self.entries.remove(index); + let timeline = Arc::clone(&entry.timeline); + self.entries.push(entry); + return Some(timeline); + } + + self.entries + .retain(|entry| entry.recording_clip != recording_clip); + if self.entries.len() == PREVIEW_CURSOR_CACHE_CAPACITY { + self.entries.remove(0); + } + + let timeline = Arc::new(PrecomputedCursorTimeline::new( + cursor, + Some(smoothing), + Some(click_spring), + )); + self.entries.push(PreviewCursorCacheEntry { + recording_clip, + cursor: Arc::clone(cursor), + settings, + timeline: Arc::clone(&timeline), + }); + Some(timeline) + } +} fn get_video_duration_fallback(path: &Path) -> Option { tracing::debug!("get_video_duration_fallback called for: {:?}", path); @@ -630,6 +708,7 @@ impl EditorInstance { ) -> tokio::task::JoinHandle<()> { tokio::spawn(async move { let mut prefetch_cancel_token: Option = None; + let mut cursor_cache = PreviewCursorCache::default(); loop { preview_rx.changed().await.unwrap(); @@ -767,17 +846,42 @@ impl EditorInstance { outgoing_zoom.ensure_precomputed_until( (frame_number as f32 + 1.0) / fps as f32, ); - let outgoing_uniforms = ProjectUniforms::new( - &self.render_constants, - &project, - frame_number, - fps, - resolution_base, + let outgoing_cursor_timeline = cursor_cache.get( + outgoing.segment.recording_clip, &outgoing_media.cursor, - &outgoing_frames, - total_duration, - &outgoing_zoom, + &project, ); + if preview_rx.has_changed().unwrap_or(false) { + continue; + } + let outgoing_uniforms = if let Some(cursor_timeline) = + &outgoing_cursor_timeline + { + ProjectUniforms::new_with_precomputed_cursor( + &self.render_constants, + &project, + frame_number, + fps, + resolution_base, + &outgoing_media.cursor, + &outgoing_frames, + total_duration, + &outgoing_zoom, + cursor_timeline, + ) + } else { + ProjectUniforms::new( + &self.render_constants, + &project, + frame_number, + fps, + resolution_base, + &outgoing_media.cursor, + &outgoing_frames, + total_duration, + &outgoing_zoom, + ) + }; Some(( outgoing_frames, outgoing_uniforms, @@ -796,6 +900,15 @@ impl EditorInstance { continue; } + let cursor_timeline = cursor_cache.get( + segment.recording_clip, + &segment_medias.cursor, + &project, + ); + if preview_rx.has_changed().unwrap_or(false) { + continue; + } + let mut next_segment_frames = segment_frames_opt; let mut rendered = false; @@ -804,17 +917,32 @@ impl EditorInstance { break; }; - let uniforms = ProjectUniforms::new( - &self.render_constants, - &project, - frame_number, - fps, - resolution_base, - &segment_medias.cursor, - &segment_frames, - total_duration, - &zoom_timeline, - ); + let uniforms = if let Some(cursor_timeline) = &cursor_timeline { + ProjectUniforms::new_with_precomputed_cursor( + &self.render_constants, + &project, + frame_number, + fps, + resolution_base, + &segment_medias.cursor, + &segment_frames, + total_duration, + &zoom_timeline, + cursor_timeline, + ) + } else { + ProjectUniforms::new( + &self.render_constants, + &project, + frame_number, + fps, + resolution_base, + &segment_medias.cursor, + &segment_frames, + total_duration, + &zoom_timeline, + ) + }; let render_confirmed = if let Some(( outgoing_frames, @@ -1381,7 +1509,260 @@ fn get_calibration_offset( #[cfg(test)] mod tests { use super::*; - use cap_project::AudioGapSummary; + use cap_project::{AudioGapSummary, CursorClickEvent, CursorConfiguration, CursorMoveEvent}; + + fn preview_cursor_events() -> Arc { + Arc::new(CursorEvents { + moves: [ + (0.0, 0.1, 0.2, "arrow"), + (100.0, 0.3, 0.4, "arrow"), + (100.0, 0.4, 0.3, "hand"), + (300.0, 0.8, 0.6, "hand"), + (1500.0, 0.6, 0.2, "arrow"), + (1600.0, 0.2, 0.8, "arrow"), + ] + .into_iter() + .map(|(time_ms, x, y, cursor_id)| CursorMoveEvent { + active_modifiers: Vec::new(), + cursor_id: cursor_id.to_string(), + time_ms, + x, + y, + }) + .collect(), + clicks: [ + (80.0, true), + (240.0, false), + (1400.0, true), + (1550.0, false), + ] + .into_iter() + .map(|(time_ms, down)| CursorClickEvent { + active_modifiers: Vec::new(), + cursor_id: "arrow".to_string(), + cursor_num: 0, + time_ms, + down, + }) + .collect(), + }) + } + + #[test] + fn preview_cursor_cache_reuses_effective_settings() { + let cursor = preview_cursor_events(); + let mut project = ProjectConfiguration::default(); + let mut cache = PreviewCursorCache::default(); + let first = cache.get(0, &cursor, &project).unwrap(); + + project.cursor.hide = !project.cursor.hide; + project.cursor.size += 1; + project.cursor.rotation_amount += 0.1; + project.cursor.stop_movement_in_last_seconds = Some(0.5); + project.cursor.click_spring = Some(project.cursor.click_spring_config()); + let repeated = cache.get(0, &Arc::clone(&cursor), &project).unwrap(); + + assert!(Arc::ptr_eq(&first, &repeated)); + assert_eq!(cache.entries.len(), 1); + } + + #[test] + fn preview_cursor_cache_invalidates_each_spring_parameter() { + let cursor = preview_cursor_events(); + let project = ProjectConfiguration::default(); + let changes: [fn(&mut CursorConfiguration); 6] = [ + |cursor| cursor.tension += 1.0, + |cursor| cursor.mass += 1.0, + |cursor| cursor.friction += 1.0, + |cursor| { + cursor + .click_spring + .get_or_insert_with(Default::default) + .tension += 1.0; + }, + |cursor| { + cursor + .click_spring + .get_or_insert_with(Default::default) + .mass += 1.0; + }, + |cursor| { + cursor + .click_spring + .get_or_insert_with(Default::default) + .friction += 1.0; + }, + ]; + + for change in changes { + let mut cache = PreviewCursorCache::default(); + let first = cache.get(0, &cursor, &project).unwrap(); + let mut changed = project.clone(); + change(&mut changed.cursor); + let updated = cache.get(0, &cursor, &changed).unwrap(); + + assert!(!Arc::ptr_eq(&first, &updated)); + assert!(Arc::ptr_eq( + &updated, + &cache.get(0, &cursor, &changed).unwrap() + )); + assert_eq!(cache.entries.len(), 1); + } + } + + #[test] + fn preview_cursor_cache_bypasses_raw_mode_without_retaining_cursor_data() { + let cursor = preview_cursor_events(); + let mut project = ProjectConfiguration::default(); + let mut cache = PreviewCursorCache::default(); + let smoothed = cache.get(0, &cursor, &project).unwrap(); + let smoothed_weak = Arc::downgrade(&smoothed); + drop(smoothed); + drop(cache.get(1, &cursor, &project).unwrap()); + project.cursor.raw = true; + + assert!(cache.get(0, &cursor, &project).is_none()); + assert!(cache.entries.is_empty()); + assert!(smoothed_weak.upgrade().is_none()); + assert_eq!(Arc::strong_count(&cursor), 1); + + project.cursor.tension += 1.0; + project.cursor.click_spring = Some(cap_project::ClickSpringConfig { + tension: 900.0, + mass: 2.0, + friction: 60.0, + }); + assert!(cache.get(0, &cursor, &project).is_none()); + assert!(cache.entries.is_empty()); + assert_eq!(Arc::strong_count(&cursor), 1); + + project.cursor.raw = false; + assert!(cache.get(0, &cursor, &project).is_some()); + assert_eq!(cache.entries.len(), 1); + } + + #[test] + fn preview_cursor_cache_bypasses_empty_moves_without_retaining_cursor_data() { + let cursor = preview_cursor_events(); + let empty = Arc::new(CursorEvents { + moves: Vec::new(), + clicks: cursor.clicks.clone(), + }); + let project = ProjectConfiguration::default(); + let mut cache = PreviewCursorCache::default(); + drop(cache.get(0, &cursor, &project).unwrap()); + let other_segment = cache.get(1, &cursor, &project).unwrap(); + + assert!(cache.get(0, &empty, &project).is_none()); + assert_eq!(cache.entries.len(), 1); + assert_eq!(Arc::strong_count(&empty), 1); + assert!(Arc::ptr_eq( + &other_segment, + &cache.get(1, &cursor, &project).unwrap() + )); + } + + #[test] + fn preview_cursor_cache_distinguishes_cursor_identity_and_segments() { + let cursor = preview_cursor_events(); + let project = ProjectConfiguration::default(); + let mut cache = PreviewCursorCache::default(); + let first = cache.get(0, &cursor, &project).unwrap(); + let replacement = Arc::new((*cursor).clone()); + let replaced = cache.get(0, &replacement, &project).unwrap(); + assert!(!Arc::ptr_eq(&first, &replaced)); + assert_eq!(cache.entries.len(), 1); + + let second_segment = cache.get(1, &replacement, &project).unwrap(); + assert!(!Arc::ptr_eq(&replaced, &second_segment)); + assert!(Arc::ptr_eq( + &replaced, + &cache.get(0, &replacement, &project).unwrap() + )); + assert_eq!(cache.entries.len(), 2); + } + + #[test] + fn preview_cursor_cache_evicts_the_least_recent_segment() { + let cursor = preview_cursor_events(); + let project = ProjectConfiguration::default(); + let mut cache = PreviewCursorCache::default(); + let first = cache.get(0, &cursor, &project).unwrap(); + let second = cache.get(1, &cursor, &project).unwrap(); + let second_weak = Arc::downgrade(&second); + drop(second); + + assert!(Arc::ptr_eq( + &first, + &cache.get(0, &cursor, &project).unwrap() + )); + drop(cache.get(2, &cursor, &project).unwrap()); + + assert!(second_weak.upgrade().is_none()); + assert_eq!(cache.entries.len(), PREVIEW_CURSOR_CACHE_CAPACITY); + assert!(Arc::ptr_eq( + &first, + &cache.get(0, &cursor, &project).unwrap() + )); + } + + #[test] + fn preview_cursor_cache_matches_fresh_interpolation_across_seeks_and_modes() { + let mut project = ProjectConfiguration::default(); + let mut cache = PreviewCursorCache::default(); + for cursor in [preview_cursor_events(), Arc::new(CursorEvents::default())] { + for (raw, click_spring) in [ + (false, None), + ( + false, + Some(cap_project::ClickSpringConfig { + tension: 720.0, + mass: 2.0, + friction: 55.0, + }), + ), + (true, None), + ] { + project.cursor.raw = raw; + project.cursor.click_spring = click_spring; + let cached = cache.get(0, &cursor, &project); + if raw || cursor.moves.is_empty() { + assert!(cached.is_none()); + continue; + } + let cached = cached.unwrap(); + let fresh = PrecomputedCursorTimeline::new( + &cursor, + (!raw).then_some(SpringMassDamperSimulationConfig { + tension: project.cursor.tension, + mass: project.cursor.mass, + friction: project.cursor.friction, + }), + Some(project.cursor.click_spring_config()), + ); + + for time in [1.5, 0.0, 0.08, 0.1, 0.23, 0.24, 0.3, 1.0, 1.6, 2.5, -1.0] { + match (cached.interpolate(time), fresh.interpolate(time)) { + (Some(actual), Some(expected)) => { + assert_eq!( + actual.position.coord.x.to_bits(), + expected.position.coord.x.to_bits() + ); + assert_eq!( + actual.position.coord.y.to_bits(), + expected.position.coord.y.to_bits() + ); + assert_eq!(actual.velocity.x.to_bits(), expected.velocity.x.to_bits()); + assert_eq!(actual.velocity.y.to_bits(), expected.velocity.y.to_bits()); + assert_eq!(actual.cursor_id, expected.cursor_id); + } + (None, None) => {} + _ => panic!("cached cursor presence differed at {time}"), + } + } + } + } + } #[test] fn audio_timing_repair_uses_startup_trimmed_overlap() { From 30df10a39209ecd9143d25535195f5c08f9ace8c Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 3 Sep 2026 16:48:00 +0100 Subject: [PATCH 03/20] improve: avoid redundant screenshot compositing work --- apps/desktop-gpui/src/screenshot_export.rs | 222 ++++++++++++++++++++- 1 file changed, 213 insertions(+), 9 deletions(-) diff --git a/apps/desktop-gpui/src/screenshot_export.rs b/apps/desktop-gpui/src/screenshot_export.rs index 0352004d949..8d30ab6b4a8 100644 --- a/apps/desktop-gpui/src/screenshot_export.rs +++ b/apps/desktop-gpui/src/screenshot_export.rs @@ -131,6 +131,16 @@ pub fn export_bounds(scaled: &[Annotation], canvas: (u32, u32)) -> ExportBounds pub fn composite(raw: &RawFrame, config: &ProjectConfiguration) -> Composited { let width = raw.width.max(1); let height = raw.height.max(1); + if config.annotations.is_empty() + && raw.rgba.len() == width as usize * height as usize * 4 + && is_opaque(&raw.rgba) + { + return Composited { + rgba: raw.rgba.clone(), + width, + height, + }; + } let scale_x = f64::from(width) / f64::from(raw.base_width.max(1)); let scale_y = f64::from(height) / f64::from(raw.base_height.max(1)); let scaled = scale_annotations(&config.annotations, scale_x, scale_y); @@ -201,16 +211,30 @@ pub fn needs_transparency(out: &Composited, config: &ProjectConfiguration) -> bo .any(|&alpha| alpha != 255) } +fn is_opaque(rgba: &[u8]) -> bool { + const ALPHA_MASK: u128 = 0xff000000ff000000ff000000ff000000; + let mut blocks = rgba.chunks_exact(16); + blocks + .by_ref() + .all(|block| u128::from_le_bytes(block.try_into().unwrap()) & ALPHA_MASK == ALPHA_MASK) + && blocks + .remainder() + .iter() + .skip(3) + .step_by(4) + .all(|&alpha| alpha == 255) +} + /// `withWhiteBackground` (`useScreenshotExport.ts:18-28`). pub fn flatten_onto_white(out: &Composited) -> Composited { - let mut rgba = vec![255u8; out.rgba.len()]; - blit_over_offset( - &mut rgba, - (out.width, out.height), - &out.rgba, - (out.width, out.height), - (0, 0), - ); + let mut rgba = out.rgba.clone(); + for pixel in rgba.chunks_exact_mut(4) { + let alpha = u32::from(pixel[3]); + for channel in &mut pixel[..3] { + *channel = ((u32::from(*channel) * alpha + 255 * (255 - alpha) + 127) / 255) as u8; + } + pixel[3] = 255; + } Composited { rgba, width: out.width, @@ -293,7 +317,7 @@ pub fn encode_for_share( /// Copy: always PNG, composited over white when transparency is not needed /// (`:183-198` -- `withWhiteBackground` only on the clipboard path). pub fn encode_for_copy(out: &Composited, config: &ProjectConfiguration) -> Result, String> { - if needs_transparency(out, config) { + if has_no_visible_background(&config.background.source) || is_opaque(&out.rgba) { encode_png(out) } else { encode_png(&flatten_onto_white(out)) @@ -969,6 +993,186 @@ mod tests { assert!(pixel[1] >= 127 && pixel[1] <= 128, "{pixel:?}"); } + fn reference_flatten(out: &Composited) -> Composited { + let mut rgba = vec![255; out.rgba.len()]; + blit_over_offset( + &mut rgba, + (out.width, out.height), + &out.rgba, + (out.width, out.height), + (0, 0), + ); + Composited { + rgba, + width: out.width, + height: out.height, + } + } + + fn reference_unannotated_composite( + raw: &RawFrame, + config: &ProjectConfiguration, + ) -> Composited { + let width = raw.width.max(1); + let height = raw.height.max(1); + let mut canvas = raw.rgba.clone(); + draw_annotations_onto(&mut canvas, width, height, &[]); + let mut rgba = vec![0; width as usize * height as usize * 4]; + if !has_no_visible_background(&config.background.source) { + rgba.fill(255); + } + blit_over_offset(&mut rgba, (width, height), &canvas, (width, height), (0, 0)); + Composited { + rgba, + width, + height, + } + } + + #[test] + fn white_flatten_matches_source_over_for_every_channel_and_alpha() { + let rgba = (0..=255u8) + .flat_map(|alpha| { + (0..=255u8).flat_map(move |channel| { + [channel, 255 - channel, channel.wrapping_mul(17), alpha] + }) + }) + .collect(); + let out = Composited { + rgba, + width: 256, + height: 256, + }; + assert_eq!(flatten_onto_white(&out).rgba, reference_flatten(&out).rgba); + } + + #[test] + fn opacity_check_matches_individual_alpha_bytes_at_block_boundaries() { + for length in 0..130 { + let opaque = vec![255; length]; + assert!(is_opaque(&opaque)); + for changed_byte in 0..length { + let mut rgba = opaque.clone(); + rgba[changed_byte] = 127; + assert_eq!( + is_opaque(&rgba), + rgba.iter().skip(3).step_by(4).all(|&alpha| alpha == 255), + ); + } + } + } + + #[test] + fn unannotated_composite_and_copy_preserve_all_alpha_values() { + for opaque in [false, true] { + let raw = RawFrame { + rgba: (0..=255u8) + .flat_map(|value| { + [ + value, + 255 - value, + value.wrapping_mul(17), + if opaque { 255 } else { value }, + ] + }) + .collect(), + width: 16, + height: 16, + base_width: 16, + base_height: 16, + }; + for invisible in [false, true] { + let mut config = ProjectConfiguration::default(); + if invisible { + invisible_background(&mut config); + } + let out = composite(&raw, &config); + let reference = reference_unannotated_composite(&raw, &config); + assert_eq!(out.rgba, reference.rgba); + let expected = if needs_transparency(&reference, &config) { + reference + } else { + reference_flatten(&reference) + }; + let png = encode_for_copy(&out, &config).unwrap(); + let decoded = image::load_from_memory(&png).unwrap().to_rgba8(); + assert_eq!(decoded.dimensions(), (16, 16)); + assert_eq!(decoded.as_raw(), &expected.rgba); + } + } + } + + #[test] + fn unannotated_composite_preserves_extra_pixel_handling() { + let raw = RawFrame { + rgba: vec![10, 20, 30, 255, 40, 50, 60, 255], + width: 1, + height: 1, + base_width: 1, + base_height: 1, + }; + let config = ProjectConfiguration::default(); + assert_eq!( + composite(&raw, &config).rgba, + reference_unannotated_composite(&raw, &config).rgba, + ); + } + + #[test] + #[ignore] + fn benchmark_screenshot_copy_preparation() { + use std::{hint::black_box, time::Instant}; + for (width, height) in [(1920, 1080), (3840, 2160)] { + for opaque in [false, true] { + let raw = RawFrame { + rgba: (0..width * height) + .flat_map(|i| { + [ + (i % 251) as u8, + ((i / width) % 253) as u8, + (i % 247) as u8, + if opaque { 255 } else { (i % 256) as u8 }, + ] + }) + .collect(), + width, + height, + base_width: width, + base_height: height, + }; + let config = ProjectConfiguration::default(); + let mut before = Vec::new(); + let mut after = Vec::new(); + for iteration in 0..8 { + let start = Instant::now(); + let reference = reference_unannotated_composite(black_box(&raw), &config); + let reference = reference_flatten(&reference); + let baseline_ms = start.elapsed().as_secs_f64() * 1000.0; + let start = Instant::now(); + let result = composite(black_box(&raw), &config); + let result = if is_opaque(&result.rgba) { + result + } else { + flatten_onto_white(&result) + }; + let candidate_ms = start.elapsed().as_secs_f64() * 1000.0; + assert_eq!(result.rgba, reference.rgba); + black_box(result); + if iteration > 0 { + before.push(baseline_ms); + after.push(candidate_ms); + } + } + before.sort_by(f64::total_cmp); + after.sort_by(f64::total_cmp); + println!( + "{}", + serde_json::json!({"width":width,"height":height,"opaque":opaque,"baseline_ms":before[3],"candidate_ms":after[3],"pixels_equal":true,"scope":"CPU composite and clipboard flatten; excludes GPU, PNG and native pasteboard"}) + ); + } + } + } + /// `withWhiteBackground` leaves an opaque canvas untouched. #[test] fn flattening_an_opaque_canvas_is_the_identity() { From f67d49c8a48db37a504ce550fcaa99a2ffe193e2 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 3 Sep 2026 16:48:00 +0100 Subject: [PATCH 04/20] improve: borrow decoded audio when preparing waveforms --- apps/desktop-gpui/src/app_windows.rs | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/apps/desktop-gpui/src/app_windows.rs b/apps/desktop-gpui/src/app_windows.rs index b5979bc0f1a..8d1212dd83a 100644 --- a/apps/desktop-gpui/src/app_windows.rs +++ b/apps/desktop-gpui/src/app_windows.rs @@ -4805,14 +4805,7 @@ fn load_editor_waveforms( (&segment.audio, &mut mic), (&segment.system_audio, &mut system), ] { - match loader.get().await { - Ok(Some(audio)) => { - out.push((audio.samples().to_vec(), audio.channels())) - } - // A failed track is an empty waveform; playback and - // export surface the actual error. - _ => out.push((Vec::new(), 1)), - } + out.push(loader.get().await.ok().flatten()); } } (mic, system) @@ -4824,15 +4817,21 @@ fn load_editor_waveforms( let peaks = cx .background_executor() .spawn(async move { - let extract = |tracks: Vec<(Vec, u16)>| { + let [mic, system] = [mic, system].map(|tracks| { tracks .into_iter() - .map(|(samples, channels)| { - Arc::new(editor_timeline::waveform_peaks(&samples, channels)) + .map(|audio| { + Arc::new(match audio { + Some(audio) => editor_timeline::waveform_peaks( + audio.samples(), + audio.channels(), + ), + None => Vec::new(), + }) }) .collect::>() - }; - (extract(mic), extract(system)) + }); + (mic, system) }) .await; let _ = handle.update(cx, |view, window, cx| { From 07401a376545d212cd2ee7e80114919ba9ad1682 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 3 Sep 2026 16:48:00 +0100 Subject: [PATCH 05/20] improve: stream MP4 export audio with bounded buffers and safe cancellation --- apps/desktop-gpui/src/editor_export.rs | 70 +- crates/audio/src/lib.rs | 2 + crates/audio/src/renderer.rs | 36 +- crates/audio/src/streaming.rs | 693 ++++++++ crates/editor/src/audio.rs | 214 ++- crates/editor/src/editor_instance.rs | 20 + crates/editor/src/export_audio.rs | 1513 ++++++++++++++++++ crates/editor/src/lib.rs | 5 + crates/enc-ffmpeg/src/audio/aac.rs | 81 + crates/enc-ffmpeg/src/audio/audio_encoder.rs | 5 + crates/enc-ffmpeg/src/audio/opus.rs | 8 + crates/enc-ffmpeg/src/mux/mp4.rs | 12 + crates/export/src/lib.rs | 304 +++- crates/export/src/mp4.rs | 516 +++++- 14 files changed, 3393 insertions(+), 86 deletions(-) create mode 100644 crates/audio/src/streaming.rs create mode 100644 crates/editor/src/export_audio.rs diff --git a/apps/desktop-gpui/src/editor_export.rs b/apps/desktop-gpui/src/editor_export.rs index 05d0623237d..908904dd8a9 100644 --- a/apps/desktop-gpui/src/editor_export.rs +++ b/apps/desktop-gpui/src/editor_export.rs @@ -2171,8 +2171,22 @@ async fn run_export( builder = builder.with_output_path(path); } - let base = builder.build().await.map_err(|error| error.to_string())?; - let total = base.total_frames(fps); + enum PreparedBase { + Mp4(cap_export::Mp4ExporterBase), + Other(ExporterBase), + } + let (base, total) = if !cursor_only && format != ExportFormatKind::Gif { + let base = builder + .build_for_mp4(cancel.clone()) + .await + .map_err(|error| error.to_string())?; + let total = base.total_frames(fps); + (PreparedBase::Mp4(base), total) + } else { + let base = builder.build().await.map_err(|error| error.to_string())?; + let total = base.total_frames(fps); + (PreparedBase::Other(base), total) + }; let _ = progress_tx.send((0, total)); let progress = { @@ -2189,33 +2203,37 @@ async fn run_export( }; let resolution = XY::new(width, height); - if cursor_only { - MovExportSettings { - fps, - resolution_base: resolution, - cursor_only: true, + match base { + PreparedBase::Other(base) if cursor_only => { + MovExportSettings { + fps, + resolution_base: resolution, + cursor_only: true, + } + .export(base, progress) + .await } - .export(base, progress) - .await - } else if format == ExportFormatKind::Gif { - GifExportSettings { - fps, - resolution_base: resolution, - quality: None, + PreparedBase::Other(base) => { + GifExportSettings { + fps, + resolution_base: resolution, + quality: None, + } + .export(base, progress) + .await } - .export(base, progress) - .await - } else { - Mp4ExportSettings { - fps, - resolution_base: resolution, - compression, - custom_bpp, - force_ffmpeg_decoder: force, - optimize_filesize: optimize, + PreparedBase::Mp4(base) => { + Mp4ExportSettings { + fps, + resolution_base: resolution, + compression, + custom_bpp, + force_ffmpeg_decoder: force, + optimize_filesize: optimize, + } + .export_prepared(base, progress) + .await } - .export(base, progress) - .await } } diff --git a/crates/audio/src/lib.rs b/crates/audio/src/lib.rs index 2a9c60e8b51..f774c7252cc 100644 --- a/crates/audio/src/lib.rs +++ b/crates/audio/src/lib.rs @@ -2,12 +2,14 @@ mod audio_data; mod calibration_store; mod latency; mod renderer; +mod streaming; mod sync_analysis; pub use audio_data::*; pub use calibration_store::*; pub use latency::*; pub use renderer::*; +pub use streaming::*; pub use sync_analysis::*; pub trait FromSampleBytes: cpal::SizedSample + std::fmt::Debug + Send + 'static { diff --git a/crates/audio/src/renderer.rs b/crates/audio/src/renderer.rs index 89b935103a5..e6afb823097 100644 --- a/crates/audio/src/renderer.rs +++ b/crates/audio/src/renderer.rs @@ -6,15 +6,35 @@ pub enum StereoMode { MonoR, } -pub struct AudioRendererTrack<'a> { - pub data: &'a AudioData, +pub trait AudioSampleSource { + fn channels(&self) -> u16; + fn sample_count(&self) -> usize; + fn sample(&self, index: usize) -> Option<&f32>; +} + +impl AudioSampleSource for AudioData { + fn channels(&self) -> u16 { + self.channels() + } + + fn sample_count(&self) -> usize { + self.samples().len() / self.channels() as usize + } + + fn sample(&self, index: usize) -> Option<&f32> { + self.samples().get(index) + } +} + +pub struct AudioRendererTrack<'a, T: AudioSampleSource = AudioData> { + pub data: &'a T, pub gain: f32, pub stereo_mode: StereoMode, pub offset: isize, } -pub fn render_audio( - tracks: &[AudioRendererTrack], +pub fn render_audio( + tracks: &[AudioRendererTrack<'_, T>], offset: usize, samples: usize, out_offset: usize, @@ -24,7 +44,7 @@ pub fn render_audio( tracks .iter() .filter_map(|t| { - let track_samples = t.data.samples().len() / t.data.channels() as usize; + let track_samples = t.data.sample_count(); let available = track_samples as i128 - offset as i128 - t.offset as i128; if available > 0 { usize::try_from(available).ok() @@ -57,16 +77,16 @@ pub fn render_audio( } if data.channels() == 1 { - if let Some(sample) = data.samples().get(source_index) { + if let Some(sample) = data.sample(source_index) { left += sample * 0.707 * gain; right += sample * 0.707 * gain; } } else if data.channels() == 2 { let base_idx = source_index.saturating_mul(2); - let Some(l_sample) = data.samples().get(base_idx) else { + let Some(l_sample) = data.sample(base_idx) else { continue; }; - let Some(r_sample) = data.samples().get(base_idx + 1) else { + let Some(r_sample) = data.sample(base_idx + 1) else { continue; }; diff --git a/crates/audio/src/streaming.rs b/crates/audio/src/streaming.rs new file mode 100644 index 00000000000..39b0a73ea99 --- /dev/null +++ b/crates/audio/src/streaming.rs @@ -0,0 +1,693 @@ +use ffmpeg::{ChannelLayout, Error, codec, format, frame::Audio, software::resampling}; +use std::{ + ffi::{CString, c_int, c_void}, + fmt, + path::Path, + ptr, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, +}; + +const MAX_CHUNK_FRAMES: usize = 48_000; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AudioStreamError { + pub stage: &'static str, + pub detail: String, + pub next_sample: u64, +} + +impl AudioStreamError { + pub fn is_cancelled(&self) -> bool { + self.stage == "cancelled" + } +} + +impl fmt::Display for AudioStreamError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + formatter, + "{} / {} / next={}", + self.stage, self.detail, self.next_sample + ) + } +} + +impl std::error::Error for AudioStreamError {} + +#[derive(Clone, Debug)] +struct Failure { + stage: &'static str, + detail: String, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Phase { + NeedPacket, + Receiving, + Draining, + Flushing, + Complete, +} + +#[derive(Debug)] +pub struct AudioChunk { + pub source_start_sample: u64, + pub channels: u16, + pub samples: Vec, +} + +#[derive(Debug)] +pub enum ChunkRead { + Chunk(AudioChunk), + Eof { next_sample: u64 }, +} + +pub struct AudioStream { + input: format::context::Input, + decoder: codec::decoder::Audio, + resampler: resampling::Context, + decoded_frame: Audio, + stream_index: usize, + channels: u16, + phase: Phase, + failure: Option, + pending: Vec, + pending_offset: usize, + position: u64, + flush_iterations: usize, + cancellation: Arc, +} + +struct StreamCancellation { + user: Arc, + abort: Option>, +} + +impl StreamCancellation { + fn is_cancelled(&self) -> bool { + self.user.load(Ordering::Relaxed) + || self + .abort + .as_ref() + .is_some_and(|abort| abort.load(Ordering::Relaxed)) + } +} + +impl AudioStream { + pub fn open(path: &Path, cancellation: Arc) -> Result { + Self::open_controlled( + path, + StreamCancellation { + user: cancellation, + abort: None, + }, + ) + } + + pub fn open_with_abort( + path: &Path, + user: Arc, + abort: Arc, + ) -> Result { + Self::open_controlled( + path, + StreamCancellation { + user, + abort: Some(abort), + }, + ) + } + + fn open_controlled( + path: &Path, + cancellation: StreamCancellation, + ) -> Result { + let cancellation = Arc::new(cancellation); + let at_open = |stage, detail: String| { + if cancellation.is_cancelled() { + cancelled_error(0) + } else { + AudioStreamError { + stage, + detail, + next_sample: 0, + } + } + }; + if cancellation.is_cancelled() { + return Err(cancelled_error(0)); + } + let input = + open_input(path, &cancellation).map_err(|detail| at_open("input-open", detail))?; + let stream = input + .streams() + .best(ffmpeg::media::Type::Audio) + .ok_or_else(|| at_open("stream", "No Stream".to_string()))?; + let stream_index = stream.index(); + let mut decoder = codec::Context::from_parameters(stream.parameters()) + .map_err(|e| at_open("decoder-parameters", e.to_string()))? + .decoder() + .audio() + .map_err(|e| at_open("decoder-open", e.to_string()))?; + let source_channels = decoder.channels().max(1); + if decoder.channel_layout().is_empty() { + decoder.set_channel_layout(ChannelLayout::default(source_channels as i32)); + } + decoder.set_packet_time_base(stream.time_base()); + let channels = if source_channels <= 1 { 1 } else { 2 }; + let mut options = ffmpeg::Dictionary::new(); + options.set("filter_size", "128"); + options.set("cutoff", "0.97"); + let resampler = resampling::Context::get_with( + decoder.format(), + decoder.channel_layout(), + decoder.rate(), + crate::AudioData::SAMPLE_FORMAT, + ChannelLayout::default(channels as i32), + crate::AudioData::SAMPLE_RATE, + options, + ) + .map_err(|e| at_open("resampler-open", e.to_string()))?; + if cancellation.is_cancelled() { + return Err(cancelled_error(0)); + } + Ok(Self { + input, + decoder, + resampler, + decoded_frame: Audio::empty(), + stream_index, + channels, + phase: Phase::NeedPacket, + failure: None, + pending: Vec::new(), + pending_offset: 0, + position: 0, + flush_iterations: 0, + cancellation, + }) + } + + pub fn channels(&self) -> u16 { + self.channels + } + pub fn position(&self) -> u64 { + self.position + } + + pub fn read_chunk(&mut self, max_frames: usize) -> Result { + if !(1..=MAX_CHUNK_FRAMES).contains(&max_frames) { + return Err(AudioStreamError { + stage: "request", + detail: "Chunk size must be 1..=48000 frames".to_string(), + next_sample: self.position, + }); + } + let max_samples = max_frames * self.channels as usize; + let mut output = Vec::with_capacity(max_samples); + while output.len() < max_samples { + if self.failure.is_none() && self.cancellation.is_cancelled() { + self.failure = Some(Failure { + stage: "cancelled", + detail: "Audio decoding cancelled".to_string(), + }); + } + if let Some(error) = self.failure.as_ref().filter(|e| e.stage == "cancelled") { + return Err(AudioStreamError { + stage: error.stage, + detail: error.detail.clone(), + next_sample: self.position, + }); + } + if self.pending_offset < self.pending.len() { + let count = + (max_samples - output.len()).min(self.pending.len() - self.pending_offset); + output.extend_from_slice( + &self.pending[self.pending_offset..self.pending_offset + count], + ); + self.pending_offset += count; + continue; + } + self.pending.clear(); + self.pending_offset = 0; + if self.failure.is_some() || self.phase == Phase::Complete { + break; + } + if let Err(error) = self.produce_pcm() { + self.failure = Some(error); + } + } + if !output.is_empty() { + let source_start_sample = self.position; + self.position += (output.len() / self.channels as usize) as u64; + return Ok(ChunkRead::Chunk(AudioChunk { + source_start_sample, + channels: self.channels, + samples: output, + })); + } + if let Some(error) = &self.failure { + return Err(AudioStreamError { + stage: error.stage, + detail: error.detail.clone(), + next_sample: self.position, + }); + } + Ok(ChunkRead::Eof { + next_sample: self.position, + }) + } + + pub fn validate_to_end(&mut self) -> Result { + loop { + match self.read_chunk(MAX_CHUNK_FRAMES)? { + ChunkRead::Chunk(_) => {} + ChunkRead::Eof { next_sample } => return Ok(next_sample), + } + } + } + + fn produce_pcm(&mut self) -> Result<(), Failure> { + loop { + if self.cancellation.is_cancelled() { + return Err(Failure { + stage: "cancelled", + detail: "Audio decoding cancelled".to_string(), + }); + } + match self.phase { + Phase::NeedPacket => { + let mut packet = ffmpeg::Packet::empty(); + match packet.read(&mut self.input) { + Ok(()) => { + if packet.stream() != self.stream_index { + continue; + } + self.decoder.send_packet(&packet).map_err(|e| Failure { + stage: "send-packet", + detail: e.to_string(), + })?; + self.phase = Phase::Receiving; + } + Err(Error::Eof) => { + self.decoder.send_eof().map_err(|e| Failure { + stage: "send-eof", + detail: e.to_string(), + })?; + self.phase = Phase::Draining; + } + Err(_) => continue, + } + } + Phase::Receiving | Phase::Draining => { + match self.decoder.receive_frame(&mut self.decoded_frame) { + Ok(()) => { + run_resampler( + &mut self.resampler, + &self.decoded_frame, + &mut self.pending, + ) + .map_err(|e| Failure { + stage: "resample", + detail: e, + })?; + if !self.pending.is_empty() { + return Ok(()); + } + } + Err(_) => { + self.phase = if self.phase == Phase::Draining { + Phase::Flushing + } else { + Phase::NeedPacket + }; + } + } + } + Phase::Flushing => { + if self.flush_iterations == 64 { + self.phase = Phase::Complete; + return Ok(()); + } + let Some(delay) = self.resampler.delay() else { + self.phase = Phase::Complete; + return Ok(()); + }; + let target = *self.resampler.output(); + let capacity = delay + .output + .max(1) + .saturating_add(16) + .min(i64::from(i32::MAX)) as usize; + let mut frame = Audio::new(target.format, capacity, target.channel_layout); + let remaining = self.resampler.flush(&mut frame).map_err(|error| Failure { + stage: "flush", + detail: format!("Flush Resampler / {error}"), + })?; + let output_samples = frame.samples(); + if output_samples > 0 { + let byte_len = output_samples + .saturating_mul(frame.channels() as usize) + .saturating_mul(std::mem::size_of::()); + let bytes = frame.data(0).get(..byte_len).ok_or_else(|| Failure { + stage: "flush", + detail: "Resampled frame data shorter than expected".to_string(), + })?; + self.pending + .extend(unsafe { crate::cast_bytes_to_f32_slice(bytes) }); + } + self.flush_iterations += 1; + if remaining.is_none() || output_samples == 0 { + self.phase = Phase::Complete; + } + if !self.pending.is_empty() || self.phase == Phase::Complete { + return Ok(()); + } + } + Phase::Complete => return Ok(()), + } + } + } +} + +fn cancelled_error(next_sample: u64) -> AudioStreamError { + AudioStreamError { + stage: "cancelled", + detail: "Audio decoding cancelled".to_string(), + next_sample, + } +} + +extern "C" fn interrupt_callback(opaque: *mut c_void) -> c_int { + let cancellation = unsafe { &*opaque.cast::() }; + c_int::from(cancellation.is_cancelled()) +} + +fn open_input( + path: &Path, + cancellation: &Arc, +) -> Result { + let path = path + .to_str() + .ok_or_else(|| "Input path is not UTF-8".to_string())?; + let path = CString::new(path).map_err(|error| error.to_string())?; + unsafe { + let mut context = ffmpeg::ffi::avformat_alloc_context(); + if context.is_null() { + return Err("Failed to allocate input context".to_string()); + } + // The pinned input_with_interrupt leaks its boxed closure. This callback borrows + // the stable Arc allocation, retained until after the input context is dropped. + (*context).interrupt_callback = ffmpeg::ffi::AVIOInterruptCB { + callback: Some(interrupt_callback), + opaque: Arc::as_ptr(cancellation).cast_mut().cast(), + }; + let opened = ffmpeg::ffi::avformat_open_input( + &mut context, + path.as_ptr(), + ptr::null_mut(), + ptr::null_mut(), + ); + if opened < 0 { + if !context.is_null() { + ffmpeg::ffi::avformat_close_input(&mut context); + } + return Err(Error::from(opened).to_string()); + } + let probed = ffmpeg::ffi::avformat_find_stream_info(context, ptr::null_mut()); + if probed < 0 { + ffmpeg::ffi::avformat_close_input(&mut context); + return Err(Error::from(probed).to_string()); + } + Ok(format::context::Input::wrap(context)) + } +} + +fn run_resampler( + resampler: &mut resampling::Context, + decoded_frame: &Audio, + samples: &mut Vec, +) -> Result<(), String> { + let target = *resampler.output(); + let capacity = resample_capacity(resampler, decoded_frame.samples()); + let mut frame = Audio::new(target.format, capacity, target.channel_layout); + resampler + .run(decoded_frame, &mut frame) + .map_err(|error| format!("Run Resampler / {error}"))?; + if frame.samples() == 0 { + return Ok(()); + } + let byte_len = frame + .samples() + .saturating_mul(frame.channels() as usize) + .saturating_mul(std::mem::size_of::()); + let data = frame + .data(0) + .get(..byte_len) + .ok_or_else(|| "Resampled frame data shorter than expected".to_string())?; + samples.extend(unsafe { crate::cast_bytes_to_f32_slice(data) }); + Ok(()) +} + +fn resample_capacity(resampler: &resampling::Context, input_samples: usize) -> usize { + let src_rate = resampler.input().rate.max(1) as u64; + let dst_rate = resampler.output().rate.max(1) as u64; + let pending_output_samples = resampler + .delay() + .map(|delay| delay.output.max(0) as u64) + .unwrap_or(0); + let resampled_from_input = (input_samples as u64) + .saturating_mul(dst_rate) + .div_ceil(src_rate); + pending_output_samples + .saturating_add(resampled_from_input) + .saturating_add(16) + .min(i32::MAX as u64) as usize +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + + fn pcm_wav(rate: u32, channels: u16, frames: usize) -> tempfile::NamedTempFile { + let data_size = (frames * usize::from(channels) * 2) as u32; + let mut bytes = Vec::new(); + bytes.extend_from_slice(b"RIFF"); + bytes.extend_from_slice(&(36 + data_size).to_le_bytes()); + bytes.extend_from_slice(b"WAVEfmt "); + bytes.extend_from_slice(&16u32.to_le_bytes()); + bytes.extend_from_slice(&1u16.to_le_bytes()); + bytes.extend_from_slice(&channels.to_le_bytes()); + bytes.extend_from_slice(&rate.to_le_bytes()); + bytes.extend_from_slice(&(rate * u32::from(channels) * 2).to_le_bytes()); + bytes.extend_from_slice(&(channels * 2).to_le_bytes()); + bytes.extend_from_slice(&16u16.to_le_bytes()); + bytes.extend_from_slice(b"data"); + bytes.extend_from_slice(&data_size.to_le_bytes()); + for index in 0..frames * usize::from(channels) { + bytes.extend_from_slice(&((index % 20_001) as i16 - 10_000).to_le_bytes()); + } + let mut file = tempfile::NamedTempFile::new().unwrap(); + file.write_all(&bytes).unwrap(); + file + } + + #[test] + fn chunks_preserve_full_decode_samples_and_repeated_eof() { + fn assert_send() {} + assert_send::(); + for (rate, channels, frames) in [ + (8_000, 1, 1_201), + (44_100, 2, 1_201), + (96_000, 6, 1_201), + (192_000, 2, 64), + (48_000, 1, 0), + ] { + let file = pcm_wav(rate, channels, frames); + let reference = crate::AudioData::from_file(file.path()).unwrap(); + for pattern in [ + &[1][..], + &[7][..], + &[997][..], + &[12_000][..], + &[48_000][..], + &[1, 509, 12_000, 3, 47, 48_000][..], + ] { + let mut stream = + AudioStream::open(file.path(), Arc::new(AtomicBool::new(false))).unwrap(); + let mut samples = Vec::new(); + let mut read_index = 0; + loop { + let max_frames = pattern[read_index % pattern.len()]; + read_index += 1; + match stream.read_chunk(max_frames).unwrap() { + ChunkRead::Chunk(chunk) => { + assert_eq!(chunk.channels, reference.channels()); + assert_eq!( + chunk.source_start_sample, + (samples.len() / usize::from(chunk.channels)) as u64 + ); + assert!( + chunk.samples.len() <= max_frames * usize::from(chunk.channels) + ); + samples.extend(chunk.samples); + assert_eq!( + stream.position(), + (samples.len() / usize::from(chunk.channels)) as u64 + ); + } + ChunkRead::Eof { next_sample } => { + assert_eq!(next_sample, reference.sample_count() as u64); + assert_eq!(stream.validate_to_end().unwrap(), next_sample); + assert!( + matches!(stream.read_chunk(7).unwrap(), ChunkRead::Eof { next_sample: next } if next == next_sample) + ); + break; + } + } + } + assert_eq!(samples.len(), reference.samples().len()); + assert!( + samples + .iter() + .zip(reference.samples()) + .all(|(a, b)| a.to_bits() == b.to_bits()) + ); + } + } + } + + #[test] + fn validation_drains_unread_tail() { + let file = pcm_wav(44_100, 2, 10_001); + let reference = crate::AudioData::from_file(file.path()).unwrap(); + let mut stream = AudioStream::open(file.path(), Arc::new(AtomicBool::new(false))).unwrap(); + assert!(matches!(stream.read_chunk(7).unwrap(), ChunkRead::Chunk(_))); + assert_eq!(stream.position(), 7); + assert_eq!( + stream.validate_to_end().unwrap(), + reference.sample_count() as u64 + ); + assert_eq!(stream.position(), reference.sample_count() as u64); + } + + #[test] + fn cancellation_is_sticky_without_publishing_pending_samples() { + let file = pcm_wav(48_000, 2, 10_001); + let cancellation = Arc::new(AtomicBool::new(false)); + let mut stream = AudioStream::open(file.path(), cancellation.clone()).unwrap(); + assert!(matches!(stream.read_chunk(7).unwrap(), ChunkRead::Chunk(_))); + cancellation.store(true, Ordering::Relaxed); + let error = stream.read_chunk(48_000).unwrap_err(); + assert!(error.is_cancelled()); + assert_eq!(error.next_sample, 7); + assert_eq!(stream.position(), 7); + cancellation.store(false, Ordering::Relaxed); + assert_eq!(stream.read_chunk(1).unwrap_err(), error); + assert_eq!(stream.validate_to_end().unwrap_err(), error); + } + + #[test] + fn terminal_failure_preserves_preceding_pcm() { + let file = pcm_wav(48_000, 2, 10_001); + let mut stream = AudioStream::open(file.path(), Arc::new(AtomicBool::new(false))).unwrap(); + stream.pending = vec![0.25, -0.25, 0.5, -0.5]; + stream.failure = Some(Failure { + stage: "send-packet", + detail: "injected failure".to_string(), + }); + let ChunkRead::Chunk(chunk) = stream.read_chunk(48_000).unwrap() else { + panic!("preceding PCM was discarded"); + }; + assert_eq!(chunk.source_start_sample, 0); + assert_eq!(chunk.samples, [0.25, -0.25, 0.5, -0.5]); + let error = stream.read_chunk(1).unwrap_err(); + assert_eq!(error.next_sample, 2); + assert_eq!(stream.read_chunk(48_000).unwrap_err(), error); + assert_eq!(stream.validate_to_end().unwrap_err(), error); + } + + #[test] + fn interrupt_ownership_releases_after_success_and_failure() { + let file = pcm_wav(48_000, 1, 1_201); + let cancellation = Arc::new(AtomicBool::new(false)); + for _ in 0..32 { + let mut stream = AudioStream::open(file.path(), cancellation.clone()).unwrap(); + assert_eq!(Arc::strong_count(&cancellation), 2); + let callback = unsafe { (*stream.input.as_mut_ptr()).interrupt_callback }; + assert_eq!( + callback.opaque, + Arc::as_ptr(&stream.cancellation).cast_mut().cast() + ); + assert_eq!(unsafe { callback.callback.unwrap()(callback.opaque) }, 0); + cancellation.store(true, Ordering::Relaxed); + assert_eq!(unsafe { callback.callback.unwrap()(callback.opaque) }, 1); + drop(stream); + assert_eq!(Arc::strong_count(&cancellation), 1); + cancellation.store(false, Ordering::Relaxed); + let missing = file.path().with_extension("missing"); + assert!(AudioStream::open(&missing, cancellation.clone()).is_err()); + assert_eq!(Arc::strong_count(&cancellation), 1); + } + cancellation.store(true, Ordering::Relaxed); + let error = AudioStream::open(file.path(), cancellation.clone()) + .err() + .unwrap(); + assert!(error.is_cancelled()); + assert_eq!(Arc::strong_count(&cancellation), 1); + } + + #[test] + fn private_abort_needs_no_async_relay_and_preserves_user_flag() { + let file = pcm_wav(48_000, 2, 10_001); + let user = Arc::new(AtomicBool::new(false)); + let abort = Arc::new(AtomicBool::new(false)); + let runtime = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + runtime.block_on(async { + let mut stream = + AudioStream::open_with_abort(file.path(), user.clone(), abort.clone()).unwrap(); + assert!(matches!(stream.read_chunk(7).unwrap(), ChunkRead::Chunk(_))); + let callback = unsafe { (*stream.input.as_mut_ptr()).interrupt_callback }; + assert_eq!(unsafe { callback.callback.unwrap()(callback.opaque) }, 0); + let worker_abort = abort.clone(); + std::thread::spawn(move || worker_abort.store(true, Ordering::Relaxed)) + .join() + .unwrap(); + assert_eq!(unsafe { callback.callback.unwrap()(callback.opaque) }, 1); + let error = stream.read_chunk(48_000).unwrap_err(); + assert!(error.is_cancelled()); + assert_eq!(error.next_sample, 7); + assert!(!user.load(Ordering::Relaxed)); + abort.store(false, Ordering::Relaxed); + assert_eq!(stream.read_chunk(1).unwrap_err(), error); + drop(stream); + assert_eq!(Arc::strong_count(&user), 1); + assert_eq!(Arc::strong_count(&abort), 1); + }); + abort.store(true, Ordering::Relaxed); + let error = AudioStream::open_with_abort(file.path(), user.clone(), abort.clone()) + .err() + .unwrap(); + assert!(error.is_cancelled()); + assert!(!user.load(Ordering::Relaxed)); + assert_eq!(Arc::strong_count(&user), 1); + assert_eq!(Arc::strong_count(&abort), 1); + } + + #[test] + fn invalid_chunk_size_does_not_consume_audio() { + let file = pcm_wav(48_000, 1, 1_201); + let mut stream = AudioStream::open(file.path(), Arc::new(AtomicBool::new(false))).unwrap(); + for size in [0, 48_001, usize::MAX] { + assert_eq!(stream.read_chunk(size).unwrap_err().stage, "request"); + assert_eq!(stream.position(), 0); + } + assert_eq!(stream.validate_to_end().unwrap(), 1_201); + } +} diff --git a/crates/editor/src/audio.rs b/crates/editor/src/audio.rs index 0a8ecdca8e4..8afa199271b 100644 --- a/crates/editor/src/audio.rs +++ b/crates/editor/src/audio.rs @@ -1,3 +1,4 @@ +use crate::export_audio::{EXPORT_AUDIO_BLOCK_SAMPLES, ExportAudioError, ExportAudioSources}; use cap_audio::{ AudioData, AudioRendererTrack, FromSampleBytes, StereoMode, cast_bytes_to_f32_slice, cast_f32_slice_to_bytes, @@ -245,6 +246,182 @@ impl AudioRenderer { self.render_linear_frame_raw(samples, project) } + pub(crate) fn render_export_chunks( + &mut self, + sources: &mut ExportAudioSources, + samples: usize, + project: &ProjectConfiguration, + mut emit: impl FnMut(usize, &[f32]) -> Result<(), ExportAudioError>, + ) -> Result, ExportAudioError> { + if samples == 0 { + return Ok(None); + } + let mut output = [0.0; EXPORT_AUDIO_BLOCK_SAMPLES * 2]; + let mut outgoing_buffer = [0.0; EXPORT_AUDIO_BLOCK_SAMPLES * 2]; + let mut incoming_buffer = [0.0; EXPORT_AUDIO_BLOCK_SAMPLES * 2]; + let mut written = 0; + let Some(timeline) = &project.timeline else { + while written < samples { + let count = (samples - written).min(EXPORT_AUDIO_BLOCK_SAMPLES); + output[..count * 2].fill(0.0); + let rendered = sources.render( + project, + self.cursor.clip_index, + self.cursor.samples + written, + count, + &mut output[..count * 2], + )?; + if rendered == 0 { + break; + } + emit(written, &output[..rendered * 2])?; + written += rendered; + if rendered < count { + break; + } + } + self.elapsed_samples += if written == 0 { samples } else { written }; + self.cursor.samples += written; + return Ok((written != 0).then_some(written)); + }; + while written < samples { + sources.check_cancelled()?; + let (mapping, span) = if !timeline.transitions.is_empty() + || !timeline.hold_windows().is_empty() + { + let Some((mapping, output_end_samples)) = self.next_transition_mapping(timeline) + else { + break; + }; + ( + mapping, + output_end_samples + .saturating_sub(self.elapsed_samples) + .min(samples - written), + ) + } else { + let Some(cursor) = self.timeline_cursor(timeline) else { + break; + }; + ( + TimelineFrameMapping::Single { + source: TimelineSource { + source_time: cursor.segment_time, + segment_index: cursor.segment_index, + segment: cursor.segment, + }, + output_end: 0.0, + }, + (cursor.segment_end_samples - self.elapsed_samples).min(samples - written), + ) + }; + if span == 0 { + break; + } + let mut span_offset = 0; + while span_offset < span { + sources.check_cancelled()?; + let count = (span - span_offset).min(EXPORT_AUDIO_BLOCK_SAMPLES); + let output = &mut output[..count * 2]; + output.fill(0.0); + match mapping { + TimelineFrameMapping::Single { source, .. } => { + if source.segment.speed_audio_mode != Some(ClipSpeedAudioMode::Mute) { + sources.render( + project, + source.segment.recording_clip, + self.playhead_to_samples(source.source_time) + span_offset, + count, + output, + )?; + } + } + TimelineFrameMapping::Hold { .. } => {} + TimelineFrameMapping::Transition { + outgoing, + incoming, + kind, + progress, + duration, + .. + } => { + let outgoing_buffer = &mut outgoing_buffer[..count * 2]; + let incoming_buffer = &mut incoming_buffer[..count * 2]; + outgoing_buffer.fill(0.0); + incoming_buffer.fill(0.0); + if outgoing.segment.speed_audio_mode != Some(ClipSpeedAudioMode::Mute) { + sources.render( + project, + outgoing.segment.recording_clip, + self.playhead_to_samples(outgoing.source_time) + span_offset, + count, + outgoing_buffer, + )?; + } + if incoming.segment.speed_audio_mode != Some(ClipSpeedAudioMode::Mute) { + sources.render( + project, + incoming.segment.recording_clip, + self.playhead_to_samples(incoming.source_time) + span_offset, + count, + incoming_buffer, + )?; + } + mix_transition_audio_at( + outgoing_buffer, + incoming_buffer, + output, + kind, + progress, + duration, + span_offset, + ); + } + } + emit(written + span_offset, output)?; + span_offset += count; + } + self.cursor = match mapping { + TimelineFrameMapping::Single { source, .. } => { + source_cursor(source, self.playhead_to_samples(source.source_time) + span) + } + TimelineFrameMapping::Hold { source, .. } => { + source_cursor(source, self.playhead_to_samples(source.source_time)) + } + TimelineFrameMapping::Transition { incoming, .. } => source_cursor( + incoming, + self.playhead_to_samples(incoming.source_time) + span, + ), + }; + self.elapsed_samples += span; + written += span; + } + Ok((written != 0).then_some(written)) + } + + fn next_transition_mapping<'a>( + &self, + timeline: &'a TimelineConfiguration, + ) -> Option<(TimelineFrameMapping<'a>, usize)> { + let mut mapping_time = self.elapsed_samples_to_playhead(); + loop { + let mapping = timeline.get_frame_mapping(mapping_time)?; + let output_end = match mapping { + TimelineFrameMapping::Single { output_end, .. } + | TimelineFrameMapping::Transition { output_end, .. } + | TimelineFrameMapping::Hold { output_end, .. } => output_end, + }; + let output_end_samples = self.playhead_to_samples(output_end); + if output_end_samples > self.elapsed_samples { + return Some((mapping, output_end_samples)); + } + if output_end <= mapping_time { + return None; + } + mapping_time = output_end; + } + } + fn render_timeline_frame_raw( &mut self, samples: usize, @@ -320,25 +497,9 @@ impl AudioRenderer { let mut output = vec![0.0; samples * 2]; let mut written = 0usize; - 'render: while written < samples { - let mut mapping_time = self.elapsed_samples_to_playhead(); - let (mapping, output_end_samples) = loop { - let Some(mapping) = timeline.get_frame_mapping(mapping_time) else { - break 'render; - }; - let output_end = match mapping { - TimelineFrameMapping::Single { output_end, .. } - | TimelineFrameMapping::Transition { output_end, .. } - | TimelineFrameMapping::Hold { output_end, .. } => output_end, - }; - let output_end_samples = self.playhead_to_samples(output_end); - if output_end_samples > self.elapsed_samples { - break (mapping, output_end_samples); - } - if output_end <= mapping_time { - break 'render; - } - mapping_time = output_end; + while written < samples { + let Some((mapping, output_end_samples)) = self.next_transition_mapping(timeline) else { + break; }; let chunk_samples = output_end_samples .saturating_sub(self.elapsed_samples) @@ -914,6 +1075,18 @@ fn mix_transition_audio( kind: ClipTransitionType, progress: f64, duration: f64, +) { + mix_transition_audio_at(outgoing, incoming, output, kind, progress, duration, 0); +} + +fn mix_transition_audio_at( + outgoing: &[f32], + incoming: &[f32], + output: &mut [f32], + kind: ClipTransitionType, + progress: f64, + duration: f64, + sample_offset: usize, ) { let progress_per_sample = 1.0 / (duration * AudioData::SAMPLE_RATE as f64); for (sample_index, ((outgoing_frame, incoming_frame), output_frame)) in outgoing @@ -922,7 +1095,8 @@ fn mix_transition_audio( .zip(output.chunks_exact_mut(2)) .enumerate() { - let progress = (progress + sample_index as f64 * progress_per_sample).clamp(0.0, 1.0); + let progress = (progress + (sample_offset + sample_index) as f64 * progress_per_sample) + .clamp(0.0, 1.0); let (outgoing_gain, incoming_gain) = match kind { ClipTransitionType::CrossFade => { let angle = progress * std::f64::consts::FRAC_PI_2; diff --git a/crates/editor/src/editor_instance.rs b/crates/editor/src/editor_instance.rs index 6552e83c5b7..f2fa632a0a3 100644 --- a/crates/editor/src/editor_instance.rs +++ b/crates/editor/src/editor_instance.rs @@ -1349,6 +1349,23 @@ pub async fn create_segments( recording_meta: &RecordingMeta, meta: &StudioRecordingMeta, force_ffmpeg: bool, +) -> Result, String> { + create_segments_with_audio(recording_meta, meta, force_ffmpeg, true).await +} + +pub async fn create_segments_without_audio( + recording_meta: &RecordingMeta, + meta: &StudioRecordingMeta, + force_ffmpeg: bool, +) -> Result, String> { + create_segments_with_audio(recording_meta, meta, force_ffmpeg, false).await +} + +async fn create_segments_with_audio( + recording_meta: &RecordingMeta, + meta: &StudioRecordingMeta, + force_ffmpeg: bool, + load_audio: bool, ) -> Result, String> { let legacy_timing_repair = LegacyAudioTimingRepair::load(&recording_meta.project_path); let legacy_timing_repair = &legacy_timing_repair; @@ -1358,6 +1375,7 @@ pub async fn create_segments( let audio = s .audio .as_ref() + .filter(|_| load_audio) .map(|audio_meta| { AudioLoader::spawn( recording_meta.path(&audio_meta.path), @@ -1423,6 +1441,7 @@ pub async fn create_segments( let audio = s .mic .as_ref() + .filter(|_| load_audio) .map(|audio| { AudioLoader::spawn( recording_meta.path(&audio.path), @@ -1434,6 +1453,7 @@ pub async fn create_segments( let system_audio = s .system_audio .as_ref() + .filter(|_| load_audio) .map(|audio| { AudioLoader::spawn( recording_meta.path(&audio.path), diff --git a/crates/editor/src/export_audio.rs b/crates/editor/src/export_audio.rs new file mode 100644 index 00000000000..98051b7abf4 --- /dev/null +++ b/crates/editor/src/export_audio.rs @@ -0,0 +1,1513 @@ +use crate::{AudioRenderer, SegmentMedia}; +use cap_audio::{ + AudioRendererTrack, AudioSampleSource, AudioStream, AudioStreamError, ChunkRead, StereoMode, +}; +use cap_project::{ClipOffsets, ProjectConfiguration, RecordingMeta, StudioRecordingMeta}; +use std::{ + collections::HashMap, + fmt, + path::{Path, PathBuf}, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, +}; + +pub(crate) const EXPORT_AUDIO_BLOCK_SAMPLES: usize = 4_096; +const MAX_PARALLEL_AUDIO_SOURCES: usize = 4; + +#[derive(Clone, Debug)] +pub enum ExportAudioError { + Cancelled, + Source { + source_index: usize, + path: PathBuf, + source: AudioStreamError, + }, + InvalidWindow, + Sink(String), + Worker { + source_index: usize, + message: String, + }, +} + +impl fmt::Display for ExportAudioError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Cancelled => f.write_str("Export cancelled"), + Self::Source { path, source, .. } => write!(f, "Audio at {}: {source}", path.display()), + Self::InvalidWindow => f.write_str("Invalid sequential export audio window"), + Self::Sink(error) => f.write_str(error), + Self::Worker { message, .. } => write!(f, "Audio source worker failed: {message}"), + } + } +} + +impl std::error::Error for ExportAudioError {} + +impl ExportAudioError { + pub fn source_index(&self) -> Option { + match self { + Self::Source { source_index, .. } | Self::Worker { source_index, .. } => { + Some(*source_index) + } + _ => None, + } + } + + fn is_cancelled(&self) -> bool { + matches!(self, Self::Cancelled) + || matches!(self, Self::Source { source, .. } if source.is_cancelled()) + } +} + +fn source_worker_panic( + source_index: usize, + panic: Box, +) -> ExportAudioError { + ExportAudioError::Worker { + source_index, + message: panic + .downcast_ref::() + .cloned() + .or_else(|| { + panic + .downcast_ref::<&str>() + .map(|message| (*message).into()) + }) + .unwrap_or_else(|| "worker panicked".into()), + } +} + +fn run_source_job( + source_index: usize, + input: &mut T, + operation: &impl Fn(usize, &mut T) -> Result, +) -> Result { + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + operation(source_index, input) + })) + .unwrap_or_else(|panic| Err(source_worker_panic(source_index, panic))) +} + +fn run_source_jobs( + inputs: &mut [(usize, T)], + abort: &AtomicBool, + operation: impl Fn(usize, &mut T) -> Result + Sync, +) -> Result, ExportAudioError> { + if inputs.len() <= 1 { + let result = inputs + .iter_mut() + .map(|(index, input)| run_source_job(*index, input, &operation)) + .collect::>(); + if result.as_ref().is_err_and(|error| !error.is_cancelled()) { + abort.store(true, Ordering::Relaxed); + } + return result; + } + + let mut results = Vec::with_capacity(inputs.len()); + for batch in inputs.chunks_mut(MAX_PARALLEL_AUDIO_SOURCES) { + let joined = std::thread::scope(|scope| { + let operation = &operation; + let handles = batch + .iter_mut() + .map(|(index, input)| { + let index = *index; + std::thread::Builder::new() + .spawn_scoped(scope, move || run_source_job(index, input, operation)) + .ok() + }) + .collect::>(); + let mut earlier_source_not_started = false; + handles + .into_iter() + .map(|handle| { + let Some(handle) = handle else { + earlier_source_not_started = true; + return None; + }; + let result = handle.join(); + if !earlier_source_not_started + && match &result { + Ok(Err(error)) => !error.is_cancelled(), + Err(_) => true, + _ => false, + } + { + abort.store(true, Ordering::Relaxed); + } + Some(result) + }) + .collect::>() + }); + for ((index, input), result) in batch.iter_mut().zip(joined) { + let result = match result { + Some(Ok(result)) => result, + Some(Err(panic)) => Err(source_worker_panic(*index, panic)), + None => run_source_job(*index, input, &operation), + }; + if result.as_ref().is_err_and(|error| !error.is_cancelled()) { + abort.store(true, Ordering::Relaxed); + } + results.push(result); + } + } + + let mut values = Vec::with_capacity(results.len()); + let mut first_error: Option = None; + for result in results { + match result { + Ok(value) => values.push(value), + Err(error) => { + if first_error + .as_ref() + .is_none_or(|previous| previous.is_cancelled() && !error.is_cancelled()) + { + first_error = Some(error); + } + } + } + } + match first_error { + Some(error) => Err(error), + None => Ok(values), + } +} + +pub struct ExportAudioRenderer { + renderer: AudioRenderer, + sources: ExportAudioSources, + failure: Option, +} + +pub struct ExportAudioPreparation { + sources: ExportAudioSources, +} + +impl ExportAudioPreparation { + fn segment_count(meta: &StudioRecordingMeta) -> usize { + match meta { + StudioRecordingMeta::SingleSegment { .. } => 1, + StudioRecordingMeta::MultipleSegments { inner, .. } => inner.segments.len(), + } + } + + pub fn open( + recording: &RecordingMeta, + meta: &StudioRecordingMeta, + cancellation: Arc, + abort: Arc, + ) -> Result { + let mut inputs = Vec::new(); + match meta { + StudioRecordingMeta::SingleSegment { segment } => { + if let Some(audio) = &segment.audio { + inputs.push((0, recording.path(&audio.path), true)); + } + } + StudioRecordingMeta::MultipleSegments { inner, .. } => { + for (index, segment) in inner.segments.iter().enumerate() { + if let Some(audio) = &segment.mic { + inputs.push((index, recording.path(&audio.path), true)); + } + if let Some(audio) = &segment.system_audio { + inputs.push((index, recording.path(&audio.path), false)); + } + } + } + } + let mut inputs = inputs.into_iter().enumerate().collect::>(); + let opened = run_source_jobs(&mut inputs, &abort, |source_index, (index, path, mic)| { + ExportAudioTrack::open(path, source_index, *mic, 0.0, &cancellation, &abort) + .map(|track| (*index, track)) + })?; + let mut tracks = (0..Self::segment_count(meta)) + .map(|_| Vec::new()) + .collect::>(); + for (index, track) in opened { + tracks[index].push(track); + } + Ok(Self { + sources: ExportAudioSources { + tracks, + cancellation, + abort, + }, + }) + } + + pub fn finish( + self, + segments: &[SegmentMedia], + ) -> Result { + self.finish_with_timing_repair(segments.iter().map(|segment| segment.audio_timing_repair)) + } + + fn finish_with_timing_repair( + mut self, + timing_repair: impl ExactSizeIterator, + ) -> Result { + if timing_repair.len() != self.sources.tracks.len() { + return Err(ExportAudioError::InvalidWindow); + } + for (tracks, repair) in self.sources.tracks.iter_mut().zip(timing_repair) { + for track in tracks { + track.timing_offset_secs = if track.mic { + repair.mic_offset_secs + } else { + repair.system_audio_offset_secs + }; + } + } + Ok(ExportAudioRenderer { + renderer: AudioRenderer::new(Vec::new()), + sources: self.sources, + failure: None, + }) + } +} + +pub struct ExportAudioValidation { + sources: ExportAudioSources, +} + +impl ExportAudioValidation { + pub fn validate_to_end(mut self) -> Result<(), ExportAudioError> { + self.sources.validate_to_end() + } +} + +impl ExportAudioRenderer { + pub fn eligible(project: &ProjectConfiguration, meta: &StudioRecordingMeta) -> bool { + let source_count = match meta { + StudioRecordingMeta::SingleSegment { segment } => usize::from(segment.audio.is_some()), + StudioRecordingMeta::MultipleSegments { inner, .. } => inner + .segments + .iter() + .map(|segment| { + usize::from(segment.mic.is_some()) + usize::from(segment.system_audio.is_some()) + }) + .sum(), + }; + Self::eligible_sources(project, source_count) + } + + fn eligible_sources(project: &ProjectConfiguration, source_count: usize) -> bool { + if !(1..=MAX_PARALLEL_AUDIO_SOURCES).contains(&source_count) + || project.clips.iter().any(|clip| { + [clip.offsets.mic, clip.offsets.system_audio] + .iter() + .any(|offset| { + !offset.is_finite() || (offset * 48_000.0).abs() >= isize::MAX as f32 / 2.0 + }) + }) + { + return false; + } + let Some(timeline) = &project.timeline else { + return true; + }; + if !timeline.audio_segments.is_empty() || !timeline.hold_windows().is_empty() { + return false; + } + let mut ends = HashMap::new(); + let mut duration = 0.0; + for segment in &timeline.segments { + if segment.timescale != 1.0 + || !segment.start.is_finite() + || !segment.end.is_finite() + || segment.start < 0.0 + || segment.end <= segment.start + || segment.end * 48_000.0 >= isize::MAX as f64 / 2.0 + { + return false; + } + duration += segment.duration(); + if !duration.is_finite() || duration * 48_000.0 >= isize::MAX as f64 / 2.0 { + return false; + } + if let Some(previous_end) = ends.insert(segment.recording_clip, segment.end) + && (!timeline.transitions.is_empty() || segment.start < previous_end) + { + return false; + } + } + true + } + + pub fn open( + recording: &RecordingMeta, + meta: &StudioRecordingMeta, + segments: &[SegmentMedia], + cancellation: Arc, + abort: Arc, + ) -> Result { + if segments.len() != ExportAudioPreparation::segment_count(meta) { + return Err(ExportAudioError::InvalidWindow); + } + ExportAudioPreparation::open(recording, meta, cancellation, abort)?.finish(segments) + } + + pub fn take_unused_sources( + &mut self, + project: &ProjectConfiguration, + ) -> Option { + let timeline = project.timeline.as_ref()?; + if self.sources.tracks.iter().flatten().count() > MAX_PARALLEL_AUDIO_SOURCES { + return None; + } + let mut unused = Vec::new(); + for (clip_index, tracks) in self.sources.tracks.iter_mut().enumerate() { + if !tracks.is_empty() + && !timeline + .segments + .iter() + .any(|segment| segment.recording_clip as usize == clip_index) + { + unused.push(std::mem::take(tracks)); + } + } + (!unused.is_empty()).then(|| ExportAudioValidation { + sources: ExportAudioSources { + tracks: unused, + cancellation: self.sources.cancellation.clone(), + abort: self.sources.abort.clone(), + }, + }) + } + + pub fn render_chunks( + &mut self, + samples: usize, + project: &ProjectConfiguration, + emit: impl FnMut(usize, &[f32]) -> Result<(), ExportAudioError>, + ) -> Result, ExportAudioError> { + if let Some(error) = &self.failure { + return Err(error.clone()); + } + let result = self + .renderer + .render_export_chunks(&mut self.sources, samples, project, emit); + if let Err(error) = &result { + self.failure = Some(error.clone()); + } + result + } + + pub fn validate_to_end(&mut self) -> Result<(), ExportAudioError> { + if let Some(error) = &self.failure { + return Err(error.clone()); + } + let result = self.sources.validate_to_end(); + if let Err(error) = &result { + self.failure = Some(error.clone()); + } + result + } +} + +pub(crate) struct ExportAudioSources { + tracks: Vec>, + cancellation: Arc, + abort: Arc, +} + +impl ExportAudioSources { + pub(crate) fn check_cancelled(&self) -> Result<(), ExportAudioError> { + if self.cancellation.load(Ordering::Relaxed) || self.abort.load(Ordering::Relaxed) { + Err(ExportAudioError::Cancelled) + } else { + Ok(()) + } + } + + fn validate_to_end(&mut self) -> Result<(), ExportAudioError> { + self.check_cancelled()?; + let mut tracks = self + .tracks + .iter_mut() + .flatten() + .map(|track| (track.source_index, track)) + .collect::>(); + run_source_jobs(&mut tracks, &self.abort, |_, track| { + track + .source + .validate_to_end() + .map_err(|source| ExportAudioError::Source { + source_index: track.source_index, + path: track.path.clone(), + source, + })?; + track.samples.clear(); + Ok(()) + })?; + self.check_cancelled() + } + + pub(crate) fn render( + &mut self, + project: &ProjectConfiguration, + clip_index: u32, + cursor: usize, + samples: usize, + output: &mut [f32], + ) -> Result { + self.check_cancelled()?; + let Some(tracks) = self.tracks.get_mut(clip_index as usize) else { + return Ok(0); + }; + let offsets = project + .clips + .iter() + .find(|clip| clip.index == clip_index) + .map(|clip| clip.offsets) + .unwrap_or_default(); + for track in tracks.iter_mut() { + let start = cursor as i128 + track.offset(&offsets) as i128; + track.prepare(start, start + samples as i128)?; + } + let views = tracks.iter().map(|track| track.view()).collect::>(); + let max_samples = tracks + .iter() + .map(|track| (track.available_end as isize - track.offset(&offsets)).max(0) as usize) + .max() + .unwrap_or(0); + if cursor >= max_samples { + return Ok(0); + } + let tracks = tracks + .iter() + .zip(&views) + .map(|(track, view)| { + let gain = if track.mic { + project.audio.mic_volume_db + } else { + project.audio.system_volume_db + }; + AudioRendererTrack { + data: view, + gain: if project.audio.mute || gain < -30.0 { + f32::NEG_INFINITY + } else { + gain + }, + stereo_mode: if !track.mic { + StereoMode::Stereo + } else { + match project.audio.mic_stereo_mode { + cap_project::StereoMode::Stereo => StereoMode::Stereo, + cap_project::StereoMode::MonoL => StereoMode::MonoL, + cap_project::StereoMode::MonoR => StereoMode::MonoR, + } + }, + offset: track.offset(&offsets), + } + }) + .collect::>(); + Ok(cap_audio::render_audio( + &tracks, + cursor, + samples.min(max_samples - cursor), + 0, + output, + )) + } +} + +struct ExportAudioTrack { + source: AudioStream, + source_index: usize, + path: PathBuf, + mic: bool, + timing_offset_secs: f32, + samples: Vec, + source_start: usize, + available_end: usize, + eof: Option, +} + +impl ExportAudioTrack { + fn open( + path: &Path, + source_index: usize, + mic: bool, + timing_offset_secs: f32, + cancellation: &Arc, + abort: &Arc, + ) -> Result { + let source = AudioStream::open_with_abort(path, cancellation.clone(), abort.clone()) + .map_err(|source| ExportAudioError::Source { + source_index, + path: path.to_path_buf(), + source, + })?; + Ok(Self { + source, + source_index, + path: path.to_path_buf(), + mic, + timing_offset_secs, + samples: Vec::new(), + source_start: 0, + available_end: 0, + eof: None, + }) + } + + fn offset(&self, offsets: &ClipOffsets) -> isize { + let offset = if self.mic { + offsets.mic + } else { + offsets.system_audio + }; + ((offset + self.timing_offset_secs) * AudioRenderer::SAMPLE_RATE as f32).round() as isize + } + + fn prepare(&mut self, start: i128, end: i128) -> Result<(), ExportAudioError> { + let start = usize::try_from(start.max(0)).map_err(|_| ExportAudioError::InvalidWindow)?; + let end = usize::try_from(end.max(0)).map_err(|_| ExportAudioError::InvalidWindow)?; + if end < start || end - start > EXPORT_AUDIO_BLOCK_SAMPLES || start < self.source_start { + return Err(ExportAudioError::InvalidWindow); + } + let channels = self.source.channels() as usize; + let discard = start + .saturating_sub(self.source_start) + .min(self.samples.len() / channels) + * channels; + self.samples.copy_within(discard.., 0); + self.samples.truncate(self.samples.len() - discard); + self.source_start = start; + while self.eof.is_none() && self.source.position() < end as u64 { + let wanted = (end as u64 - self.source.position()).min(12_000) as usize; + match self + .source + .read_chunk(wanted) + .map_err(|source| ExportAudioError::Source { + source_index: self.source_index, + path: self.path.clone(), + source, + })? { + ChunkRead::Chunk(chunk) => { + let chunk_start = usize::try_from(chunk.source_start_sample) + .map_err(|_| ExportAudioError::InvalidWindow)?; + let discard = start + .saturating_sub(chunk_start) + .min(chunk.samples.len() / channels) + * channels; + self.samples.extend_from_slice(&chunk.samples[discard..]); + } + ChunkRead::Eof { next_sample } => { + self.eof = Some( + usize::try_from(next_sample) + .map_err(|_| ExportAudioError::InvalidWindow)?, + ); + } + } + } + self.available_end = self.eof.unwrap_or(end.max(self.source.position() as usize)); + Ok(()) + } + + fn view(&self) -> ExportAudioView<'_> { + ExportAudioView { + samples: &self.samples, + channels: self.source.channels(), + source_start: self.source_start, + available_end: self.available_end, + } + } +} + +struct ExportAudioView<'a> { + samples: &'a [f32], + channels: u16, + source_start: usize, + available_end: usize, +} + +impl AudioSampleSource for ExportAudioView<'_> { + fn channels(&self) -> u16 { + self.channels + } + fn sample_count(&self) -> usize { + self.available_end + } + fn sample(&self, index: usize) -> Option<&f32> { + index + .checked_sub(self.source_start * self.channels as usize) + .and_then(|index| self.samples.get(index)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::audio::{AudioSegment, AudioSegmentTrack}; + use cap_audio::AudioData; + use cap_project::{ + ClipConfiguration, ClipSpeedAudioMode, ClipTransition, ClipTransitionType, + TimelineConfiguration, TimelineSegment, + }; + + #[test] + fn source_jobs_are_bounded_joined_and_ordered() { + use std::{ + sync::{Condvar, Mutex, atomic::AtomicUsize}, + time::Duration, + }; + + let active = AtomicUsize::new(0); + let peak = AtomicUsize::new(0); + let completed = AtomicUsize::new(0); + let arrived = Mutex::new(0); + let changed = Condvar::new(); + let abort = AtomicBool::new(false); + let mut inputs = (0..9).map(|value| (value, value)).collect::>(); + let results = run_source_jobs(&mut inputs, &abort, |_, input| { + let count = active.fetch_add(1, Ordering::SeqCst) + 1; + peak.fetch_max(count, Ordering::SeqCst); + if *input < 8 { + let mut count = arrived.lock().unwrap(); + *count += 1; + changed.notify_all(); + let (_guard, timed_out) = changed + .wait_timeout_while(count, Duration::from_secs(2), |count| { + *count + < (*input / MAX_PARALLEL_AUDIO_SOURCES + 1) * MAX_PARALLEL_AUDIO_SOURCES + }) + .unwrap(); + assert!(!timed_out.timed_out()); + } + active.fetch_sub(1, Ordering::SeqCst); + completed.fetch_add(1, Ordering::SeqCst); + Ok(*input) + }) + .unwrap(); + + assert_eq!(results, (0..9).collect::>()); + assert_eq!(peak.load(Ordering::SeqCst), MAX_PARALLEL_AUDIO_SOURCES); + assert_eq!(active.load(Ordering::SeqCst), 0); + assert_eq!(completed.load(Ordering::SeqCst), inputs.len()); + assert!(!abort.load(Ordering::Relaxed)); + } + + #[test] + fn source_jobs_preserve_first_real_error_and_join_later_sources() { + use std::sync::atomic::AtomicUsize; + + let completed = AtomicUsize::new(0); + let abort = AtomicBool::new(false); + let mut inputs = [(0, 0), (1, 1), (2, 2), (3, 3)]; + let error = run_source_jobs(&mut inputs, &abort, |_, input| { + completed.fetch_add(1, Ordering::SeqCst); + match input { + 0 => Err(ExportAudioError::Cancelled), + 1 => Err(ExportAudioError::Sink("first source error".into())), + 2 => Err(ExportAudioError::Sink("later source error".into())), + _ => Ok(()), + } + }) + .unwrap_err(); + + assert_eq!(error.to_string(), "first source error"); + assert_eq!(completed.load(Ordering::SeqCst), inputs.len()); + assert!(abort.load(Ordering::Relaxed)); + } + + #[test] + fn source_worker_panic_becomes_error_after_other_sources_join() { + use std::sync::atomic::AtomicUsize; + + let completed = AtomicUsize::new(0); + let abort = AtomicBool::new(false); + let mut inputs = [(0, 0), (1, 1), (2, 2), (3, 3)]; + let error = run_source_jobs(&mut inputs, &abort, |_, input| { + if *input == 1 { + panic!("source worker test"); + } + completed.fetch_add(1, Ordering::SeqCst); + Ok(()) + }) + .unwrap_err(); + + assert_eq!( + error.to_string(), + "Audio source worker failed: source worker test" + ); + assert_eq!(completed.load(Ordering::SeqCst), inputs.len() - 1); + assert_eq!(error.source_index(), Some(1)); + assert!(abort.load(Ordering::Relaxed)); + } + + #[test] + fn failed_source_aborts_and_joins_waiting_peer_without_user_cancellation() { + use std::time::{Duration, Instant}; + + let user_cancellation = AtomicBool::new(false); + let abort = AtomicBool::new(false); + let peer_started = AtomicBool::new(false); + let peer_observed_abort = AtomicBool::new(false); + let peer_finished = AtomicBool::new(false); + let error = run_source_jobs(&mut [(0, 0), (1, 1)], &abort, |_, input| { + let started = Instant::now(); + if *input == 0 { + while !peer_started.load(Ordering::Acquire) + && started.elapsed() < Duration::from_secs(2) + { + std::thread::yield_now(); + } + assert!(peer_started.load(Ordering::Acquire)); + return Err(ExportAudioError::Sink("first source failed".into())); + } + peer_started.store(true, Ordering::Release); + while !abort.load(Ordering::Relaxed) + && !user_cancellation.load(Ordering::Relaxed) + && started.elapsed() < Duration::from_secs(2) + { + std::thread::yield_now(); + } + peer_observed_abort.store(abort.load(Ordering::Relaxed), Ordering::Relaxed); + peer_finished.store(true, Ordering::Release); + Err::<(), _>(ExportAudioError::Cancelled) + }) + .unwrap_err(); + + assert_eq!(error.to_string(), "first source failed"); + assert!(peer_observed_abort.load(Ordering::Relaxed)); + assert!(peer_finished.load(Ordering::Acquire)); + assert!(!user_cancellation.load(Ordering::Relaxed)); + } + + #[test] + fn single_source_job_stays_on_current_thread() { + let current = std::thread::current().id(); + let results = run_source_jobs(&mut [(7, 0)], &AtomicBool::new(false), |index, value| { + assert_eq!(std::thread::current().id(), current); + assert_eq!(index, 7); + *value += 1; + Ok(*value) + }) + .unwrap(); + assert_eq!(results, [1]); + } + + #[test] + fn inline_source_panic_preserves_original_source_index() { + let abort = AtomicBool::new(false); + let error = run_source_jobs::<_, ()>(&mut [(7, ())], &abort, |_, _| { + panic!("inline source panic"); + }) + .unwrap_err(); + assert_eq!(error.source_index(), Some(7)); + assert_eq!( + error.to_string(), + "Audio source worker failed: inline source panic" + ); + assert!(abort.load(Ordering::Relaxed)); + } + + fn write_wav(path: &Path, frames: usize, channels: u16) { + let data_size = frames as u32 * u32::from(channels) * 2; + let mut bytes = Vec::new(); + bytes.extend_from_slice(b"RIFF"); + bytes.extend_from_slice(&(36 + data_size).to_le_bytes()); + bytes.extend_from_slice(b"WAVEfmt "); + bytes.extend_from_slice(&16u32.to_le_bytes()); + bytes.extend_from_slice(&1u16.to_le_bytes()); + bytes.extend_from_slice(&channels.to_le_bytes()); + bytes.extend_from_slice(&48_000u32.to_le_bytes()); + bytes.extend_from_slice(&(48_000u32 * u32::from(channels) * 2).to_le_bytes()); + bytes.extend_from_slice(&(channels * 2).to_le_bytes()); + bytes.extend_from_slice(&16u16.to_le_bytes()); + bytes.extend_from_slice(b"data"); + bytes.extend_from_slice(&data_size.to_le_bytes()); + for index in 0..frames * channels as usize { + bytes.extend_from_slice(&(((index * 73) % 60_001) as i32 - 30_000).to_le_bytes()[..2]); + } + std::fs::write(path, bytes).unwrap(); + } + + fn preparation_recording(directory: &Path) -> RecordingMeta { + std::fs::write( + directory.join("recording-meta.json"), + r#"{ + "pretty_name": "Audio preparation test", + "segments": [ + { + "display": { "path": "unused.mp4" }, + "mic": { "path": "mic.wav" }, + "system_audio": { "path": "system.wav" } + }, + { "display": { "path": "unused.mp4" } }, + { + "display": { "path": "unused.mp4" }, + "mic": { "path": "incoming.wav" } + } + ] + }"#, + ) + .unwrap(); + RecordingMeta::load_for_project(directory).unwrap() + } + + #[test] + fn preparation_preserves_readers_positions_and_timing_repair_bits() { + use crate::editor_instance::SegmentAudioTimingRepair; + + let directory = tempfile::tempdir().unwrap(); + let recording = preparation_recording(directory.path()); + for (name, channels) in [("mic.wav", 1), ("system.wav", 2), ("incoming.wav", 2)] { + write_wav(&directory.path().join(name), 17_003, channels); + } + let user = Arc::new(AtomicBool::new(false)); + let abort = Arc::new(AtomicBool::new(false)); + let prepared = ExportAudioPreparation::open( + &recording, + recording.studio_meta().unwrap(), + user.clone(), + abort.clone(), + ) + .unwrap(); + let identity = |track: &ExportAudioTrack| { + ( + std::ptr::from_ref(&track.source) as usize, + track.source_index, + track.path.clone(), + track.mic, + track.source.position(), + ) + }; + let before = prepared + .sources + .tracks + .iter() + .flatten() + .map(identity) + .collect::>(); + assert!(before.iter().all(|track| track.4 == 0)); + assert_eq!( + prepared + .sources + .tracks + .iter() + .map(Vec::len) + .collect::>(), + [2, 0, 1] + ); + let repairs = [ + SegmentAudioTimingRepair { + mic_offset_secs: -0.0, + system_audio_offset_secs: f32::from_bits(0x7fc0_1234), + }, + SegmentAudioTimingRepair::default(), + SegmentAudioTimingRepair { + mic_offset_secs: -0.137_125, + system_audio_offset_secs: 0.0, + }, + ]; + let mut renderer = prepared + .finish_with_timing_repair(repairs.into_iter()) + .unwrap(); + let after = renderer + .sources + .tracks + .iter() + .flatten() + .map(identity) + .collect::>(); + assert_eq!(before, after); + assert_eq!( + renderer + .sources + .tracks + .iter() + .flatten() + .map(|track| track.timing_offset_secs.to_bits()) + .collect::>(), + [ + repairs[0].mic_offset_secs.to_bits(), + repairs[0].system_audio_offset_secs.to_bits(), + repairs[2].mic_offset_secs.to_bits() + ], + ); + for track in renderer.sources.tracks.iter_mut().flatten() { + let reference = AudioData::from_file(&track.path).unwrap(); + let ChunkRead::Chunk(chunk) = track.source.read_chunk(257).unwrap() else { + panic!("prepared source unexpectedly empty"); + }; + assert_eq!(chunk.source_start_sample, 0); + assert_eq!( + chunk + .samples + .iter() + .map(|value| value.to_bits()) + .collect::>(), + reference.samples()[..chunk.samples.len()] + .iter() + .map(|value| value.to_bits()) + .collect::>(), + ); + } + assert!(!user.load(Ordering::Relaxed)); + assert!(!abort.load(Ordering::Relaxed)); + } + + #[test] + fn invalid_finish_drops_all_prepared_readers() { + let directory = tempfile::tempdir().unwrap(); + let recording = preparation_recording(directory.path()); + for name in ["mic.wav", "system.wav", "incoming.wav"] { + write_wav(&directory.path().join(name), 17_003, 1); + } + let user = Arc::new(AtomicBool::new(false)); + let abort = Arc::new(AtomicBool::new(false)); + let prepared = ExportAudioPreparation::open( + &recording, + recording.studio_meta().unwrap(), + user.clone(), + abort.clone(), + ) + .unwrap(); + assert!(Arc::strong_count(&user) > 1); + assert!(matches!( + prepared.finish(&[]), + Err(ExportAudioError::InvalidWindow) + )); + assert_eq!(Arc::strong_count(&user), 1); + assert_eq!(Arc::strong_count(&abort), 1); + assert!(!user.load(Ordering::Relaxed)); + assert!(!abort.load(Ordering::Relaxed)); + } + + #[test] + fn open_rejects_segment_mismatch_before_missing_source_io() { + let directory = tempfile::tempdir().unwrap(); + let recording = preparation_recording(directory.path()); + let user = Arc::new(AtomicBool::new(false)); + let abort = Arc::new(AtomicBool::new(false)); + assert!(matches!( + ExportAudioRenderer::open( + &recording, + recording.studio_meta().unwrap(), + &[], + user.clone(), + abort.clone(), + ), + Err(ExportAudioError::InvalidWindow), + )); + assert_eq!(Arc::strong_count(&user), 1); + assert_eq!(Arc::strong_count(&abort), 1); + assert!(!abort.load(Ordering::Relaxed)); + assert!(!directory.path().join("mic.wav").exists()); + } + + fn segment(index: u32, start: f64, end: f64) -> TimelineSegment { + TimelineSegment { + recording_clip: index, + start, + end, + timescale: 1.0, + name: None, + speed_audio_mode: None, + } + } + + fn project() -> ProjectConfiguration { + ProjectConfiguration { + timeline: Some(TimelineConfiguration { + segments: vec![segment(0, 0.001_01, 0.33), segment(1, 0.002_01, 0.29)], + transitions: Vec::new(), + zoom_segments: Vec::new(), + scene_segments: Vec::new(), + mask_segments: Vec::new(), + text_segments: Vec::new(), + caption_segments: Vec::new(), + keyboard_segments: Vec::new(), + audio_segments: Vec::new(), + camera3d_segments: Vec::new(), + }), + clips: vec![ + ClipConfiguration { + index: 0, + offsets: ClipOffsets { + mic: -0.003_13, + system_audio: 0.001_33, + ..Default::default() + }, + ..Default::default() + }, + ClipConfiguration { + index: 1, + ..Default::default() + }, + ], + ..Default::default() + } + } + + fn fixture(paths: &[PathBuf], data: &[Arc]) -> (AudioRenderer, ExportAudioRenderer) { + let cancellation = Arc::new(AtomicBool::new(false)); + let full = vec![ + AudioSegment { + tracks: vec![ + AudioSegmentTrack::new( + data[0].clone(), + |config| config.mic_volume_db, + |config| match config.mic_stereo_mode { + cap_project::StereoMode::Stereo => StereoMode::Stereo, + cap_project::StereoMode::MonoL => StereoMode::MonoL, + cap_project::StereoMode::MonoR => StereoMode::MonoR, + }, + |offset| offset.mic, + ), + AudioSegmentTrack::new( + data[1].clone(), + |config| config.system_volume_db, + |_| StereoMode::Stereo, + |offset| offset.system_audio, + ), + ], + }, + AudioSegment { + tracks: vec![AudioSegmentTrack::new( + data[2].clone(), + |config| config.mic_volume_db, + |config| match config.mic_stereo_mode { + cap_project::StereoMode::Stereo => StereoMode::Stereo, + cap_project::StereoMode::MonoL => StereoMode::MonoL, + cap_project::StereoMode::MonoR => StereoMode::MonoR, + }, + |offset| offset.mic, + )], + }, + ]; + let tracks = vec![ + vec![ + ExportAudioTrack::open(&paths[0], 0, true, 0.0, &cancellation, &cancellation) + .unwrap(), + ExportAudioTrack::open(&paths[1], 1, false, 0.0, &cancellation, &cancellation) + .unwrap(), + ], + vec![ + ExportAudioTrack::open(&paths[2], 2, true, 0.0, &cancellation, &cancellation) + .unwrap(), + ], + ]; + ( + AudioRenderer::new(full), + ExportAudioRenderer { + renderer: AudioRenderer::new(Vec::new()), + sources: ExportAudioSources { + tracks, + abort: cancellation.clone(), + cancellation, + }, + failure: None, + }, + ) + } + + #[test] + fn unused_source_detachment_preserves_slots_identity_and_project() { + let directory = tempfile::tempdir().unwrap(); + let paths = + ["mic.wav", "system.wav", "incoming.wav"].map(|name| directory.path().join(name)); + for (path, channels) in paths.iter().zip([1, 2, 2]) { + write_wav(path, 17_003, channels); + } + let data = paths + .iter() + .map(|path| Arc::new(AudioData::from_file(path).unwrap())) + .collect::>(); + for variant in 0..9 { + let (_, mut candidate) = fixture(&paths, &data); + candidate.sources.tracks[1].push( + ExportAudioTrack::open( + &paths[2], + 3, + false, + 0.0, + &candidate.sources.cancellation, + &candidate.sources.abort, + ) + .unwrap(), + ); + for track in candidate.sources.tracks.iter_mut().flatten() { + track.prepare(10, 100).unwrap(); + } + let identity = |track: &ExportAudioTrack| { + ( + track.source_index, + track.path.clone(), + track.source.position(), + track.samples.as_ptr() as usize, + track.samples.len(), + ) + }; + let before = candidate + .sources + .tracks + .iter() + .flatten() + .map(identity) + .collect::>(); + let mut project = project(); + let expected_unused = match variant { + 0 => { + project.timeline = None; + vec![] + } + 1 => vec![], + 2 => { + project.audio.mute = true; + for segment in &mut project.timeline.as_mut().unwrap().segments { + segment.speed_audio_mode = Some(ClipSpeedAudioMode::Mute); + } + vec![] + } + 3 => { + project + .timeline + .as_mut() + .unwrap() + .transitions + .push(ClipTransition { + segment_index: 1, + kind: ClipTransitionType::CrossFade, + duration: 0.13, + }); + vec![] + } + 4 => { + project.timeline.as_mut().unwrap().segments = vec![segment(0, 0.0, 0.1)]; + vec![2, 3] + } + 5 => { + project.timeline.as_mut().unwrap().segments = vec![segment(1, 0.0, 0.1)]; + vec![0, 1] + } + 6 => { + project.timeline.as_mut().unwrap().segments = vec![segment(9, 0.0, 0.1)]; + vec![0, 1, 2, 3] + } + 7 => { + project.timeline.as_mut().unwrap().segments.clear(); + vec![0, 1, 2, 3] + } + 8 => { + project.timeline.as_mut().unwrap().segments = + vec![segment(0, 0.0, 0.1), segment(0, 0.2, 0.3)]; + vec![2, 3] + } + _ => unreachable!(), + }; + let project_before = format!("{project:?}"); + let detached = candidate.take_unused_sources(&project); + let unused_indices = detached + .iter() + .flat_map(|validation| validation.sources.tracks.iter().flatten()) + .map(|track| track.source_index) + .collect::>(); + assert_eq!(unused_indices, expected_unused, "variant {variant}"); + assert_eq!(candidate.sources.tracks.len(), 2); + for (clip_index, tracks) in candidate.sources.tracks.iter().enumerate() { + for track in tracks { + assert_eq!(track.source_index / 2, clip_index); + } + } + let mut after = candidate + .sources + .tracks + .iter() + .flatten() + .chain( + detached + .iter() + .flat_map(|validation| validation.sources.tracks.iter().flatten()), + ) + .map(identity) + .collect::>(); + after.sort_by_key(|track| track.0); + assert_eq!(after, before, "variant {variant}"); + assert_eq!(format!("{project:?}"), project_before); + assert!(candidate.take_unused_sources(&project).is_none()); + if let Some(detached) = detached { + detached.validate_to_end().unwrap(); + } + candidate.validate_to_end().unwrap(); + } + } + + #[test] + fn detachment_does_not_split_more_than_four_readers() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("source.wav"); + write_wav(&path, 17_003, 2); + let paths = vec![path.clone(); 3]; + let data = paths + .iter() + .map(|path| Arc::new(AudioData::from_file(path).unwrap())) + .collect::>(); + let (_, mut candidate) = fixture(&paths, &data); + for index in 3..5 { + candidate.sources.tracks[1].push( + ExportAudioTrack::open( + &path, + index, + false, + 0.0, + &candidate.sources.cancellation, + &candidate.sources.abort, + ) + .unwrap(), + ); + } + let mut project = project(); + project.timeline.as_mut().unwrap().segments.truncate(1); + assert!(candidate.take_unused_sources(&project).is_none()); + assert_eq!(candidate.sources.tracks.iter().flatten().count(), 5); + } + + #[test] + fn bounded_sink_matches_full_renderer_at_original_request_boundaries() { + ffmpeg::init().unwrap(); + let directory = tempfile::tempdir().unwrap(); + let paths = + ["mic.wav", "system.wav", "incoming.wav"].map(|name| directory.path().join(name)); + for ((path, frames), channels) in paths.iter().zip([17_003, 9_007, 23_013]).zip([1, 2, 2]) { + write_wav(path, frames, channels); + } + let data = paths + .iter() + .map(|path| Arc::new(AudioData::from_file(path).unwrap())) + .collect::>(); + for variant in 0..16 { + let mut project = project(); + match variant { + 0 => project.timeline = None, + 1 => {} + 2 | 3 => project + .timeline + .as_mut() + .unwrap() + .transitions + .push(ClipTransition { + segment_index: 1, + kind: if variant == 2 { + ClipTransitionType::CrossFade + } else { + ClipTransitionType::FadeThroughBlack + }, + duration: 0.131_234_567, + }), + 4 => { + project.timeline.as_mut().unwrap().segments[0].speed_audio_mode = + Some(ClipSpeedAudioMode::Mute) + } + 5 => project.audio.mute = true, + 6 => project.audio.mic_stereo_mode = cap_project::StereoMode::MonoL, + 7 => project.audio.mic_stereo_mode = cap_project::StereoMode::MonoR, + 8 => project.audio.mic_volume_db = f32::NAN, + 9 => project.audio.mic_volume_db = f32::INFINITY, + 10 => project.audio.system_volume_db = f32::NEG_INFINITY, + 11 => project.clips[0].offsets.mic = 0.9, + 12 => project.clips[0].offsets.system_audio = -0.9, + 13 => project + .timeline + .as_mut() + .unwrap() + .segments + .push(segment(0, 0.34, 0.36)), + 14 => project.timeline.as_mut().unwrap().segments[1].recording_clip = 9, + 15 => { + project.audio.mic_volume_db = 4.0; + project.audio.system_volume_db = -29.9; + } + _ => unreachable!(), + } + for request in [1, 7, 997, 4_096, 4_800, 48_001, 96_000, 384_000] { + let (mut reference, mut candidate) = fixture(&paths, &data); + reference.set_playhead(0.0, &project); + loop { + let expected = reference.render_frame_raw(request, &project); + let mut actual = Vec::new(); + let written = candidate + .render_chunks(request, &project, |offset, samples| { + assert!(samples.len() <= EXPORT_AUDIO_BLOCK_SAMPLES * 2); + assert_eq!(offset * 2, actual.len()); + actual.extend_from_slice(samples); + Ok(()) + }) + .unwrap_or_else(|error| { + panic!("variant {variant}, request {request}: {error}") + }); + assert_eq!( + written, + expected.as_ref().map(|(count, _)| *count), + "variant {variant}, request {request}" + ); + assert_eq!( + actual + .iter() + .map(|value| value.to_bits()) + .collect::>(), + expected + .as_ref() + .map(|(_, values)| values + .iter() + .map(|value| value.to_bits()) + .collect::>()) + .unwrap_or_default(), + "variant {variant}, request {request}" + ); + assert_eq!( + reference.elapsed_samples_to_playhead().to_bits(), + candidate.renderer.elapsed_samples_to_playhead().to_bits() + ); + for track in candidate.sources.tracks.iter().flatten() { + assert!( + track.samples.len() + <= EXPORT_AUDIO_BLOCK_SAMPLES * track.source.channels() as usize + ); + } + if expected.is_none() { + break; + } + } + candidate.validate_to_end().unwrap(); + assert_eq!( + candidate + .sources + .tracks + .iter() + .flatten() + .map(|track| track.source.position()) + .collect::>(), + data.iter() + .map(|data| data.sample_count() as u64) + .collect::>() + ); + } + } + } + + #[test] + fn eligibility_rejects_unbounded_or_nonsequential_shapes() { + let original = project(); + assert!(ExportAudioRenderer::eligible_sources(&original, 3)); + assert!(!ExportAudioRenderer::eligible_sources(&original, 0)); + assert!(!ExportAudioRenderer::eligible_sources(&original, 5)); + for offset in [f32::NAN, f32::INFINITY, f32::NEG_INFINITY, f32::MAX] { + let mut project = original.clone(); + project.clips[0].offsets.mic = offset; + assert!(!ExportAudioRenderer::eligible_sources(&project, 3)); + } + for (start, end, speed) in [ + (f64::NAN, 1.0, 1.0), + (-0.1, 1.0, 1.0), + (0.0, f64::INFINITY, 1.0), + (0.0, 0.0, 1.0), + (0.0, 1.0, 2.0), + ] { + let mut project = original.clone(); + let segment = &mut project.timeline.as_mut().unwrap().segments[0]; + segment.start = start; + segment.end = end; + segment.timescale = speed; + assert!(!ExportAudioRenderer::eligible_sources(&project, 3)); + } + let mut project = original.clone(); + project + .timeline + .as_mut() + .unwrap() + .segments + .push(segment(0, 0.34, 0.4)); + assert!(ExportAudioRenderer::eligible_sources(&project, 3)); + project + .timeline + .as_mut() + .unwrap() + .transitions + .push(ClipTransition { + segment_index: 1, + kind: ClipTransitionType::CrossFade, + duration: 0.01, + }); + assert!(!ExportAudioRenderer::eligible_sources(&project, 3)); + let mut project = original; + project + .timeline + .as_mut() + .unwrap() + .segments + .push(segment(0, 0.01, 0.1)); + assert!(!ExportAudioRenderer::eligible_sources(&project, 3)); + } + + #[test] + fn source_windows_reject_backward_reads_and_keep_failures_sticky() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("source.wav"); + write_wav(&path, 17_003, 2); + let cancellation = Arc::new(AtomicBool::new(false)); + let mut track = + ExportAudioTrack::open(&path, 0, true, 0.0, &cancellation, &cancellation).unwrap(); + track.prepare(8_000, 9_000).unwrap(); + track.prepare(8_500, 9_501).unwrap(); + let full = AudioData::from_file(&path).unwrap(); + assert_eq!(track.samples, full.samples()[8_500 * 2..9_501 * 2]); + assert!(matches!( + track.prepare(0, 1), + Err(ExportAudioError::InvalidWindow) + )); + let mut candidate = ExportAudioRenderer { + renderer: AudioRenderer::new(Vec::new()), + sources: ExportAudioSources { + tracks: vec![vec![track]], + cancellation: cancellation.clone(), + abort: cancellation.clone(), + }, + failure: None, + }; + let error = candidate + .render_chunks(1, &ProjectConfiguration::default(), |_, _| Ok(())) + .unwrap_err() + .to_string(); + assert_eq!(candidate.validate_to_end().unwrap_err().to_string(), error); + cancellation.store(true, Ordering::Relaxed); + assert_eq!( + candidate + .render_chunks(1, &ProjectConfiguration::default(), |_, _| Ok(())) + .unwrap_err() + .to_string(), + error + ); + } + + #[test] + fn sink_failure_stops_before_later_source_reads() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("source.wav"); + write_wav(&path, 17_003, 2); + let paths = vec![path; 3]; + let data = paths + .iter() + .map(|path| Arc::new(AudioData::from_file(path).unwrap())) + .collect::>(); + let (_, mut candidate) = fixture(&paths, &data); + let error = candidate + .render_chunks(96_000, &project(), |_, _| { + Err(ExportAudioError::Sink("sink stopped".into())) + }) + .unwrap_err(); + assert_eq!(error.to_string(), "sink stopped"); + assert_eq!(candidate.sources.tracks[1][0].source.position(), 0); + assert_eq!( + candidate.validate_to_end().unwrap_err().to_string(), + "sink stopped" + ); + } +} diff --git a/crates/editor/src/lib.rs b/crates/editor/src/lib.rs index 228b230c644..745ae781895 100644 --- a/crates/editor/src/lib.rs +++ b/crates/editor/src/lib.rs @@ -2,6 +2,7 @@ mod audio; mod audio_output; mod editor; mod editor_instance; +mod export_audio; mod playback; mod segments; mod telemetry; @@ -17,6 +18,10 @@ pub use editor::{ }; pub use editor_instance::{ AudioLoader, EditorInstance, EditorState, SegmentMedia, create_segments, + create_segments_without_audio, +}; +pub use export_audio::{ + ExportAudioError, ExportAudioPreparation, ExportAudioRenderer, ExportAudioValidation, }; pub use playback::{Playback, PlaybackEvent, PlaybackHandle, PlaybackStartError}; pub use segments::{get_audio_segments, load_music_tracks, load_music_tracks_uncached}; diff --git a/crates/enc-ffmpeg/src/audio/aac.rs b/crates/enc-ffmpeg/src/audio/aac.rs index 85bcfd92cd0..fe80c381057 100644 --- a/crates/enc-ffmpeg/src/audio/aac.rs +++ b/crates/enc-ffmpeg/src/audio/aac.rs @@ -114,7 +114,88 @@ impl AudioEncoder for AACEncoder { let _ = self.send_frame(frame, Duration::MAX, output); } + fn try_send_frame( + &mut self, + frame: frame::Audio, + output: &mut format::context::Output, + ) -> Result<(), ffmpeg::Error> { + self.send_frame(frame, Duration::MAX, output) + } + fn flush(&mut self, output: &mut format::context::Output) -> Result<(), ffmpeg::Error> { self.flush(output) } } + +#[cfg(test)] +mod tests { + use super::*; + use ffmpeg::ChannelLayout; + + fn input_frame(start: i64, samples: usize) -> frame::Audio { + let mut audio = + frame::Audio::new(Sample::F32(Type::Packed), samples, ChannelLayout::STEREO); + audio.set_rate(48_000); + audio.set_pts(Some(start)); + for (index, sample) in audio.data_mut(0)[..samples * 2 * size_of::()] + .chunks_exact_mut(size_of::()) + .enumerate() + { + let value = ((index % 17) as f32 - 8.0) / 32.0; + sample.copy_from_slice(&value.to_le_bytes()); + } + audio + } + + fn encode_audio(checked: bool) -> Vec { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("audio.mp4"); + let mut output = format::output(&path).unwrap(); + let mut encoder = AACEncoder::init( + AudioInfo::new_raw(Sample::F32(Type::Packed), 48_000, 2), + &mut output, + ) + .unwrap(); + output.write_header().unwrap(); + + let mut position = 0; + for samples in [1, 997, 4_096, 1_024, 37] { + let audio = input_frame(position, samples); + if checked { + AudioEncoder::try_send_frame(&mut encoder, audio, &mut output).unwrap(); + } else { + AudioEncoder::send_frame(&mut encoder, audio, &mut output); + } + position += samples as i64; + } + + encoder.flush(&mut output).unwrap(); + output.write_trailer().unwrap(); + drop(encoder); + drop(output); + std::fs::read(path).unwrap() + } + + #[test] + fn checked_audio_submission_preserves_encoded_bytes() { + assert_eq!(encode_audio(false), encode_audio(true)); + } + + #[test] + fn checked_audio_submission_reports_a_closed_encoder() { + let directory = tempfile::tempdir().unwrap(); + let mut output = format::output(&directory.path().join("closed.mp4")).unwrap(); + let mut encoder = AACEncoder::init( + AudioInfo::new_raw(Sample::F32(Type::Packed), 48_000, 2), + &mut output, + ) + .unwrap(); + output.write_header().unwrap(); + encoder.flush(&mut output).unwrap(); + + assert_eq!( + AudioEncoder::try_send_frame(&mut encoder, input_frame(0, 1_024), &mut output), + Err(ffmpeg::Error::Eof) + ); + } +} diff --git a/crates/enc-ffmpeg/src/audio/audio_encoder.rs b/crates/enc-ffmpeg/src/audio/audio_encoder.rs index 118b8b51277..a6ecf7c5a87 100644 --- a/crates/enc-ffmpeg/src/audio/audio_encoder.rs +++ b/crates/enc-ffmpeg/src/audio/audio_encoder.rs @@ -9,5 +9,10 @@ pub trait AudioEncoder { } fn send_frame(&mut self, frame: frame::Audio, output: &mut format::context::Output); + fn try_send_frame( + &mut self, + frame: frame::Audio, + output: &mut format::context::Output, + ) -> Result<(), ffmpeg::Error>; fn flush(&mut self, output: &mut format::context::Output) -> Result<(), ffmpeg::Error>; } diff --git a/crates/enc-ffmpeg/src/audio/opus.rs b/crates/enc-ffmpeg/src/audio/opus.rs index 34e1902dae9..3bcfb974cff 100644 --- a/crates/enc-ffmpeg/src/audio/opus.rs +++ b/crates/enc-ffmpeg/src/audio/opus.rs @@ -121,6 +121,14 @@ impl AudioEncoder for OpusEncoder { let _ = self.queue_frame(frame, Duration::MAX, output); } + fn try_send_frame( + &mut self, + frame: frame::Audio, + output: &mut format::context::Output, + ) -> Result<(), ffmpeg::Error> { + self.queue_frame(frame, Duration::MAX, output) + } + fn flush(&mut self, output: &mut format::context::Output) -> Result<(), ffmpeg::Error> { self.flush(output) } diff --git a/crates/enc-ffmpeg/src/mux/mp4.rs b/crates/enc-ffmpeg/src/mux/mp4.rs index ff6eee0ac00..7338e423b36 100644 --- a/crates/enc-ffmpeg/src/mux/mp4.rs +++ b/crates/enc-ffmpeg/src/mux/mp4.rs @@ -135,6 +135,18 @@ impl MP4File { audio.send_frame(frame, &mut self.output); } + pub fn try_queue_audio_frame(&mut self, frame: frame::Audio) -> Result<(), ffmpeg::Error> { + if self.is_finished { + return Err(ffmpeg::Error::Eof); + } + + let Some(audio) = &mut self.audio else { + return Err(ffmpeg::Error::StreamNotFound); + }; + + audio.try_send_frame(frame, &mut self.output) + } + pub fn finish(&mut self) -> Result { if self.is_finished { return Err(FinishError::AlreadyFinished); diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs index e6ec17d386e..929e30a3683 100644 --- a/crates/export/src/lib.rs +++ b/crates/export/src/lib.rs @@ -4,13 +4,19 @@ pub mod mp4; pub mod preview; pub mod settings; -use cap_editor::SegmentMedia; +use cap_editor::{ExportAudioPreparation, ExportAudioRenderer, SegmentMedia}; use cap_project::{ BackgroundSource, ProjectConfiguration, RecordingMeta, StudioRecordingMeta, TimelineConfiguration, TimelineSegment, }; use cap_rendering::{ProjectRecordingsMeta, RenderVideoConstants}; -use std::{path::PathBuf, sync::Arc}; +use std::{ + path::PathBuf, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, +}; #[derive(thiserror::Error, Debug)] pub enum ExportError { @@ -78,6 +84,22 @@ impl ExporterBuilder { } pub async fn build(self) -> Result { + self.build_inner(None).await + } + + pub async fn build_for_mp4( + self, + cancellation: Arc, + ) -> Result { + self.build_inner(Some(cancellation)) + .await + .map(Mp4ExporterBase) + } + + async fn build_inner( + self, + cancellation: Option>, + ) -> Result { type Error = ExporterBuildError; let mut project_config = if let Some(config) = self.config { @@ -137,6 +159,15 @@ impl ExporterBuilder { } } + let output_path = self + .output_path + .unwrap_or_else(|| recording_meta.output_path()); + let streaming_output = prepare_streaming_output( + &output_path, + cancellation.is_some() && ExportAudioRenderer::eligible(&project_config, studio_meta), + ); + let stream_audio = streaming_output.is_some(); + let render_constants = Arc::new( RenderVideoConstants::new( &recordings.segments, @@ -147,21 +178,42 @@ impl ExporterBuilder { .map_err(Error::RendererSetup)?, ); - let segments = - cap_editor::create_segments(&recording_meta, studio_meta, self.force_ffmpeg_decoder) - .await - .map_err(Error::MediaLoad)?; - - // Audio decodes in the background after create_segments; exports must - // not silently drop a track, so fail loudly if any decode failed. - for segment in &segments { - segment.audio.get().await.map_err(Error::MediaLoad)?; - segment.system_audio.get().await.map_err(Error::MediaLoad)?; - } - - let output_path = self - .output_path - .unwrap_or_else(|| recording_meta.output_path()); + let audio_cancellation = if stream_audio { + cancellation.map(ExportAudioCancellation::new) + } else { + None + }; + let (segments, streaming_audio) = if let Some(control) = &audio_cancellation { + let recording = recording_meta.clone(); + let studio = studio_meta.clone(); + let cancellation = control.user.clone(); + let abort = control.stop.clone(); + let preparation = tokio::task::spawn_blocking(move || { + ExportAudioPreparation::open(&recording, &studio, cancellation, abort) + }); + let segments = cap_editor::create_segments_without_audio( + &recording_meta, + studio_meta, + self.force_ffmpeg_decoder, + ) + .await; + let (segments, audio) = + finish_audio_preparation(segments, preparation, &control.stop).await?; + (segments, Some(audio)) + } else { + let segments = cap_editor::create_segments( + &recording_meta, + studio_meta, + self.force_ffmpeg_decoder, + ) + .await + .map_err(Error::MediaLoad)?; + for segment in &segments { + segment.audio.get().await.map_err(Error::MediaLoad)?; + segment.system_audio.get().await.map_err(Error::MediaLoad)?; + } + (segments, None) + }; if let Some(parent) = output_path.parent() { std::fs::create_dir_all(parent) @@ -177,10 +229,33 @@ impl ExporterBuilder { recording_meta, project_config, project_path: self.project_path, + streaming_audio, + streaming_output, + audio_cancellation, }) } } +async fn finish_audio_preparation( + segments: Result, String>, + preparation: tokio::task::JoinHandle< + Result, + >, + abort: &AtomicBool, +) -> Result<(Vec, ExportAudioRenderer), ExporterBuildError> { + if segments.is_err() { + abort.store(true, Ordering::Relaxed); + } + let preparation = preparation.await; + let segments = segments.map_err(ExporterBuildError::MediaLoad)?; + let audio = preparation + .map_err(|error| ExporterBuildError::MediaLoad(error.to_string()))? + .map_err(|error| ExporterBuildError::MediaLoad(error.to_string()))? + .finish(&segments) + .map_err(|error| ExporterBuildError::MediaLoad(error.to_string()))?; + Ok((segments, audio)) +} + pub fn make_cursor_only_project(mut project_config: ProjectConfiguration) -> ProjectConfiguration { project_config.background.source = BackgroundSource::Color { value: [0, 0, 0], @@ -211,6 +286,51 @@ pub fn make_cursor_only_project(mut project_config: ProjectConfiguration) -> Pro project_config } +fn prepare_streaming_output( + output: &std::path::Path, + eligible: bool, +) -> Option { + if !eligible + || output.extension().and_then(|extension| extension.to_str()) != Some("mp4") + || !matches!(std::fs::symlink_metadata(output), Err(error) if error.kind() == std::io::ErrorKind::NotFound) + { + return None; + } + mp4::temporary_mp4_output(output).ok() +} + +struct ExportAudioCancellation { + user: Arc, + stop: Arc, +} + +impl ExportAudioCancellation { + fn new(user: Arc) -> Self { + Self { + user, + stop: Arc::new(AtomicBool::new(false)), + } + } +} + +impl Drop for ExportAudioCancellation { + fn drop(&mut self) { + self.stop.store(true, Ordering::Relaxed); + } +} + +pub struct Mp4ExporterBase(ExporterBase); + +impl Mp4ExporterBase { + pub fn total_frames(&self, fps: u32) -> u32 { + self.0.total_frames(fps) + } + + pub fn uses_streaming_audio(&self) -> bool { + self.0.streaming_audio.is_some() + } +} + pub struct ExporterBase { project_path: PathBuf, recording_meta: RecordingMeta, @@ -220,6 +340,9 @@ pub struct ExporterBase { render_constants: Arc, segments: Vec, output_path: PathBuf, + streaming_audio: Option, + streaming_output: Option, + audio_cancellation: Option, } impl ExporterBase { @@ -298,3 +421,150 @@ mod cursor_only_tests { } } } + +#[cfg(test)] +mod cancellation_tests { + use super::*; + + #[tokio::test(flavor = "current_thread")] + async fn segment_failure_aborts_and_joins_preparation_before_returning() { + use std::time::{Duration, Instant}; + + let user = Arc::new(AtomicBool::new(false)); + let control = ExportAudioCancellation::new(user.clone()); + let abort = control.stop.clone(); + let completed = Arc::new(AtomicBool::new(false)); + let worker_completed = completed.clone(); + let (started, entered) = tokio::sync::oneshot::channel(); + let preparation = tokio::task::spawn_blocking(move || { + started.send(()).unwrap(); + let start = Instant::now(); + while !abort.load(Ordering::Relaxed) && start.elapsed() < Duration::from_secs(2) { + std::thread::yield_now(); + } + assert!(abort.load(Ordering::Relaxed)); + worker_completed.store(true, Ordering::Release); + Err(cap_editor::ExportAudioError::Sink( + "audio preparation error".into(), + )) + }); + entered.await.unwrap(); + let result = finish_audio_preparation( + Err("segment setup error".into()), + preparation, + &control.stop, + ) + .await; + assert!( + matches!(result, Err(ExporterBuildError::MediaLoad(error)) if error == "segment setup error") + ); + assert!(completed.load(Ordering::Acquire)); + assert!(!user.load(Ordering::Relaxed)); + } + + #[tokio::test(flavor = "current_thread")] + async fn preparation_error_or_panic_is_returned_after_successful_segment_setup() { + let abort = AtomicBool::new(false); + for panic in [false, true] { + let preparation = tokio::task::spawn_blocking(move || { + assert!(!panic, "preparation panic"); + Err(cap_editor::ExportAudioError::Sink( + "preparation error".into(), + )) + }); + let result = finish_audio_preparation(Ok(Vec::new()), preparation, &abort).await; + let Err(ExporterBuildError::MediaLoad(error)) = result else { + panic!("preparation failure was lost"); + }; + assert!(error.contains(if panic { + "preparation panic" + } else { + "preparation error" + })); + } + } + + #[tokio::test(flavor = "current_thread")] + async fn dropped_builder_aborts_preparation_without_user_cancellation() { + use std::time::{Duration, Instant}; + + let user = Arc::new(AtomicBool::new(false)); + let worker_user = user.clone(); + let (started, entered) = tokio::sync::oneshot::channel(); + let (finished, observed) = std::sync::mpsc::channel(); + let builder = tokio::spawn(async move { + let control = ExportAudioCancellation::new(worker_user); + let abort = control.stop.clone(); + let preparation = tokio::task::spawn_blocking(move || { + started.send(()).unwrap(); + let start = Instant::now(); + while !abort.load(Ordering::Relaxed) && start.elapsed() < Duration::from_secs(2) { + std::thread::yield_now(); + } + finished.send(abort.load(Ordering::Relaxed)).unwrap(); + Err(cap_editor::ExportAudioError::Cancelled) + }); + finish_audio_preparation(Ok(Vec::new()), preparation, &control.stop) + .await + .map(|_| ()) + .map_err(|error| error.to_string()) + }); + entered.await.unwrap(); + builder.abort(); + assert!(builder.await.is_err_and(|error| error.is_cancelled())); + let stopped = + tokio::task::spawn_blocking(move || observed.recv_timeout(Duration::from_secs(2))) + .await + .unwrap() + .unwrap(); + assert!(stopped); + assert!(!user.load(Ordering::Relaxed)); + assert_eq!(Arc::strong_count(&user), 1); + } + + #[test] + fn unavailable_streaming_destination_falls_back_without_creating_directories() { + let directory = tempfile::tempdir().unwrap(); + let missing_parent = directory.path().join("missing"); + assert!(prepare_streaming_output(&missing_parent.join("export.mp4"), true).is_none()); + assert!(!missing_parent.exists()); + let existing = directory.path().join("existing.mp4"); + std::fs::write(&existing, b"existing").unwrap(); + assert!(prepare_streaming_output(&existing, true).is_none()); + assert_eq!(std::fs::read(existing).unwrap(), b"existing"); + } + + #[test] + fn prepared_destination_is_removed_when_preparation_is_dropped() { + let directory = tempfile::tempdir().unwrap(); + let output = directory.path().join("export.mp4"); + assert!(prepare_streaming_output(&output, false).is_none()); + assert_eq!(std::fs::read_dir(directory.path()).unwrap().count(), 0); + let prepared = prepare_streaming_output(&output, true).unwrap(); + let temporary_path = prepared.to_path_buf(); + assert!(temporary_path.exists()); + assert!(!output.exists()); + drop(prepared); + assert!(!temporary_path.exists()); + assert!(!output.exists()); + } + + #[test] + fn pipeline_stop_does_not_change_user_cancellation() { + let user = Arc::new(AtomicBool::new(false)); + let cancellation = ExportAudioCancellation::new(user.clone()); + let stop = cancellation.stop.clone(); + drop(cancellation); + assert!(stop.load(Ordering::Relaxed)); + assert!(!user.load(Ordering::Relaxed)); + } + + #[test] + fn user_cancellation_does_not_change_pipeline_abort() { + let user = Arc::new(AtomicBool::new(false)); + let cancellation = ExportAudioCancellation::new(user.clone()); + user.store(true, Ordering::Relaxed); + assert!(cancellation.user.load(Ordering::Relaxed)); + assert!(!cancellation.stop.load(Ordering::Relaxed)); + } +} diff --git a/crates/export/src/mp4.rs b/crates/export/src/mp4.rs index c972ed7384b..78a1b669646 100644 --- a/crates/export/src/mp4.rs +++ b/crates/export/src/mp4.rs @@ -1,5 +1,5 @@ -use crate::ExporterBase; -use cap_editor::{AudioRenderer, get_audio_segments, load_music_tracks_uncached}; +use crate::{ExporterBase, Mp4ExporterBase}; +use cap_editor::{AudioRenderer, ExportAudioError, get_audio_segments, load_music_tracks_uncached}; use cap_enc_ffmpeg::{AudioEncoder, aac::AACEncoder, h264::H264Encoder, mp4::*}; use cap_media_info::{RawVideoFormat, VideoInfo}; use cap_project::XY; @@ -14,12 +14,127 @@ use std::{ path::PathBuf, sync::{ Arc, Mutex, - atomic::{AtomicU64, Ordering}, + atomic::{AtomicBool, AtomicU64, Ordering}, }, time::Duration, }; +use tokio::sync::Notify; use tracing::{info, trace, warn}; +enum Mp4PipelineError { + Interrupted, + Failure(String), + AudioSourceFailure { + source_index: usize, + message: String, + }, +} + +impl From for Mp4PipelineError { + fn from(error: String) -> Self { + Self::Failure(error) + } +} + +impl From for Mp4PipelineError { + fn from(error: ExportAudioError) -> Self { + match (&error, error.source_index()) { + (ExportAudioError::Cancelled, _) => Self::Interrupted, + (ExportAudioError::Source { source, .. }, _) if source.is_cancelled() => { + Self::Interrupted + } + (_, Some(source_index)) => Self::AudioSourceFailure { + source_index, + message: error.to_string(), + }, + _ => Self::Failure(error.to_string()), + } + } +} + +impl Mp4PipelineError { + fn message(self) -> String { + match self { + Self::Interrupted => "Export cancelled".to_string(), + Self::Failure(error) | Self::AudioSourceFailure { message: error, .. } => error, + } + } +} + +fn resolve_pipeline_results( + encoder: Result, + renderer: Result<(), Mp4PipelineError>, + validation: Result<(), Mp4PipelineError>, +) -> Result<(), String> { + let mut first_source_failure: Option<(usize, String)> = None; + let mut first_other_failure = None; + let mut interrupted = false; + for error in [encoder.err(), renderer.err(), validation.err()] + .into_iter() + .flatten() + { + match error { + Mp4PipelineError::AudioSourceFailure { + source_index, + message, + } => { + if first_source_failure + .as_ref() + .is_none_or(|(previous, _)| source_index < *previous) + { + first_source_failure = Some((source_index, message)); + } + } + Mp4PipelineError::Failure(error) => { + if first_other_failure.is_none() { + first_other_failure = Some(error); + } + } + Mp4PipelineError::Interrupted => interrupted = true, + } + } + if let Some((_, error)) = first_source_failure { + Err(error) + } else if let Some(error) = first_other_failure { + Err(error) + } else if interrupted { + Err("Export cancelled".to_string()) + } else { + Ok(()) + } +} + +async fn complete_audio_validation( + task: tokio::task::JoinHandle>, + abort: Arc, + failure: Arc, +) -> Result<(), Mp4PipelineError> { + match task.await { + Ok(result) => result, + Err(error) => { + abort.store(true, Ordering::Relaxed); + failure.notify_one(); + Err(Mp4PipelineError::Failure(format!( + "Audio validation worker failed: {error}" + ))) + } + } +} + +async fn render_until_audio_failure( + render: impl std::future::Future>, + failure: Option>, +) -> Result<(), Mp4PipelineError> { + let Some(failure) = failure else { + return render.await; + }; + tokio::select! { + biased; + _ = failure.notified() => Err(Mp4PipelineError::Interrupted), + result = render => result, + } +} + #[derive(Serialize, Deserialize, Type, Clone, Copy, Debug)] pub enum ExportCompression { Maximum, @@ -82,6 +197,14 @@ impl Mp4ExportSettings { } impl Mp4ExportSettings { + pub fn export_prepared( + self, + base: Mp4ExporterBase, + on_progress: impl FnMut(u32) -> bool + Send + 'static, + ) -> impl std::future::Future> { + self.export(base.0, on_progress) + } + pub async fn export( self, base: ExporterBase, @@ -162,7 +285,7 @@ impl Mp4ExportSettings { async fn export_nv12( self, - base: ExporterBase, + mut base: ExporterBase, output_size: (u32, u32), fps: u32, on_progress: impl FnMut(u32) -> bool + Send + 'static, @@ -170,6 +293,36 @@ impl Mp4ExportSettings { ) -> Result { let pipeline_start = std::time::Instant::now(); let output_path = base.output_path.clone(); + let mut streaming_audio = base.streaming_audio.take(); + let audio_control = base.audio_cancellation.take(); + let audio_cancellation = audio_control.as_ref().map(|control| control.stop.clone()); + let user_cancellation = audio_control.as_ref().map(|control| control.user.clone()); + let temporary_output = base.streaming_output.take().map(Arc::new); + let unused_audio = streaming_audio + .as_mut() + .and_then(|audio| audio.take_unused_sources(&base.project_config)); + let validation_failure = unused_audio.as_ref().map(|_| Arc::new(Notify::new())); + let audio_validation = unused_audio.map(|validation| { + let abort = audio_cancellation.as_ref().unwrap().clone(); + let failure = validation_failure.as_ref().unwrap().clone(); + let worker_abort = abort.clone(); + let worker_failure = failure.clone(); + let temporary_output_guard = temporary_output.clone(); + let task = tokio::task::spawn_blocking(move || { + let _temporary_output_guard = temporary_output_guard; + let result = validation.validate_to_end().map_err(Mp4PipelineError::from); + if result.is_err() { + worker_abort.store(true, Ordering::Relaxed); + worker_failure.notify_one(); + } + result + }); + (task, abort, failure) + }); + let encoder_output_path = temporary_output + .as_ref() + .map(|path| path.to_path_buf()) + .unwrap_or_else(|| output_path.clone()); let meta = &base.studio_meta; let (frame_tx, frame_rx) = std::sync::mpsc::sync_channel::(4); @@ -192,7 +345,12 @@ impl Mp4ExportSettings { let project_for_audio = base.project_config.clone(); let pipeline_start_for_encoder = pipeline_start; + let encoder_temporary_output = temporary_output.clone(); + let encoder_cancellation = audio_cancellation.clone(); + let encoder_user_cancellation = user_cancellation.clone(); + let encoder_result_cancellation = audio_cancellation.clone(); let encoder_thread = tokio::task::spawn_blocking(move || { + let _temporary_output_guard = encoder_temporary_output; trace!("Creating MP4File encoder (NV12 path)"); // The encoder's input mode has to match what the renderer actually @@ -200,6 +358,9 @@ impl Mp4ExportSettings { // VideoToolbox for zero-copy hardware input (falling back to // software frames if that fails), CPU frames keep today's path. let first_frame = frame_rx.recv().ok(); + if encoder_cancellation.as_ref().is_some_and(|cancel| cancel.load(Ordering::Relaxed)) || encoder_user_cancellation.as_ref().is_some_and(|cancel| cancel.load(Ordering::Relaxed)) { + return Err(Mp4PipelineError::Interrupted); + } #[cfg(target_os = "macos")] let want_hw_input = !self.optimize_filesize && matches!( @@ -214,7 +375,7 @@ impl Mp4ExportSettings { let mut encoder = MP4File::init( "output", - base.output_path.clone(), + encoder_output_path.clone(), self.optimize_filesize, |o| { #[cfg(target_os = "macos")] @@ -262,7 +423,7 @@ impl Mp4ExportSettings { "Created MP4File encoder (NV12, export settings)" ); - let mut audio_renderer = if has_audio { + let mut audio_renderer = if has_audio && streaming_audio.is_none() { Some(AudioRenderer::new(audio_segments).with_music(music)) } else { None @@ -284,6 +445,9 @@ impl Mp4ExportSettings { .into_iter() .chain(std::iter::from_fn(|| frame_rx.recv().ok())); for input in frames { + if encoder_cancellation.as_ref().is_some_and(|cancel| cancel.load(Ordering::Relaxed)) || encoder_user_cancellation.as_ref().is_some_and(|cancel| cancel.load(Ordering::Relaxed)) { + return Err(Mp4PipelineError::Interrupted); + } if encoded_frames == 0 && let Some(audio) = &mut audio_renderer { @@ -361,6 +525,30 @@ impl Mp4ExportSettings { if let Some(audio) = audio_frame { encoder.queue_audio_frame(audio); } + if has_audio && let Some(audio) = &mut streaming_audio { + let n = u64::from(input.frame_number); + if let Some((pts, samples)) = audio_frame_budget(n, sample_rate, fps_u64, audio_sample_cursor) { + audio_sample_cursor = pts as u64 + samples as u64; + let rendered = audio.render_chunks(samples, &project_for_audio, |offset, data| { + let mut frame = packed_audio_frame(data); + frame.set_pts(Some(pts + offset as i64)); + encoder.try_queue_audio_frame(frame).map_err(|error| ExportAudioError::Sink(error.to_string())) + }).map_err(Mp4PipelineError::from)?; + if rendered.is_none() { + let mut offset = 0; + while offset < samples { + if encoder_cancellation.as_ref().is_some_and(|cancel| cancel.load(Ordering::Relaxed)) || encoder_user_cancellation.as_ref().is_some_and(|cancel| cancel.load(Ordering::Relaxed)) { + return Err(Mp4PipelineError::Interrupted); + } + let count = (samples - offset).min(4_096); + let mut frame = silent_audio_frame(count); + frame.set_pts(Some(pts + offset as i64)); + encoder.try_queue_audio_frame(frame).map_err(|error| error.to_string())?; + offset += count; + } + } + } + } encoded_frames += 1; if encoded_frames == 1 && let Some(atom) = record_first_queued_ms.as_ref() @@ -382,22 +570,52 @@ impl Mp4ExportSettings { ); } + if let Some(audio) = &mut streaming_audio { + audio.validate_to_end().map_err(Mp4PipelineError::from)?; + } + if encoder_cancellation.as_ref().is_some_and(|cancel| cancel.load(Ordering::Relaxed)) || encoder_user_cancellation.as_ref().is_some_and(|cancel| cancel.load(Ordering::Relaxed)) { + return Err(Mp4PipelineError::Interrupted); + } + let res = encoder .finish() .map_err(|e| format!("Failed to finish encoding: {e}"))?; if let Err(e) = res.video_finish { - return Err(format!("Video encoding failed: {e}")); + return Err(Mp4PipelineError::Failure(format!("Video encoding failed: {e}"))); } if let Err(e) = res.audio_finish { - return Err(format!("Audio encoding failed: {e}")); + return Err(Mp4PipelineError::Failure(format!("Audio encoding failed: {e}"))); } - Ok::<_, String>(base.output_path) + Ok::<_, Mp4PipelineError>(encoder_output_path) }) - .then(|r| async { r.map_err(|e| e.to_string()).and_then(|v| v) }); + .then(move |r| async move { + let result = r.map_err(|e| Mp4PipelineError::Failure(e.to_string())).and_then(|v| v); + if result.is_err() && let Some(cancel) = &encoder_result_cancellation { + cancel.store(true, Ordering::Relaxed); + } + result + }); let stop_after_frames_sent = mode.stop_after_frames_sent; + let render_result_cancellation = audio_cancellation.clone(); + let progress_cancellation = audio_cancellation.clone(); + let progress_user_cancellation = user_cancellation; + let mut on_progress = on_progress; + let on_progress = move |frame| { + let keep_going = !progress_user_cancellation + .as_ref() + .is_some_and(|cancel| cancel.load(Ordering::Relaxed)) + && !progress_cancellation + .as_ref() + .is_some_and(|cancel| cancel.load(Ordering::Relaxed)) + && on_progress(frame); + if !keep_going && let Some(cancel) = &progress_cancellation { + cancel.store(true, Ordering::Relaxed); + } + keep_going + }; let render_video_task = export_render_to_channel( &base.render_constants, &base.project_config, @@ -418,12 +636,64 @@ impl Mp4ExportSettings { &base.recordings, stop_after_frames_sent, nv12_render_startup_breakdown_ms, + audio_cancellation.is_some(), on_progress, base.project_path.clone(), ) - .then(|v| async { v.map_err(|e| e.to_string()) }); - - tokio::try_join!(encoder_thread, render_video_task)?; + .then(move |v| async move { + let result = v.map_err(|error| match &error { + cap_rendering::RenderingError::ImageLoadError(message) + if render_result_cancellation.is_some() && message == "Export cancelled" => + { + Mp4PipelineError::Interrupted + } + _ => Mp4PipelineError::Failure(error.to_string()), + }); + if result.is_err() + && let Some(cancel) = &render_result_cancellation + { + cancel.store(true, Ordering::Relaxed); + } + result + }); + + if audio_cancellation.is_some() { + let validation = async move { + match audio_validation { + Some((task, abort, failure)) => { + complete_audio_validation(task, abort, failure).await + } + None => Ok(()), + } + }; + let (encoder_result, render_result, validation_result) = tokio::join!( + encoder_thread, + render_until_audio_failure(render_video_task, validation_failure), + validation, + ); + resolve_pipeline_results(encoder_result, render_result, validation_result)?; + if audio_cancellation + .as_ref() + .is_some_and(|cancel| cancel.load(Ordering::Relaxed)) + { + return Err("Export cancelled".to_string()); + } + if audio_control + .as_ref() + .is_some_and(|control| control.user.load(Ordering::Relaxed)) + { + return Err("Export cancelled".to_string()); + } + if let Some(temporary_output) = temporary_output { + Arc::try_unwrap(temporary_output) + .map_err(|_| "Export output is still in use".to_string())? + .persist_noclobber(&output_path) + .map_err(|error| error.to_string())?; + } + } else { + tokio::try_join!(encoder_thread, render_video_task) + .map_err(Mp4PipelineError::message)?; + } Ok(output_path) } @@ -603,6 +873,37 @@ fn audio_frame_budget( Some((cursor as i64, (end - cursor) as usize)) } +pub(crate) fn temporary_mp4_output(output: &std::path::Path) -> Result { + let mut builder = tempfile::Builder::new(); + builder.prefix(".cap-export-").suffix(".mp4"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + builder.permissions(std::fs::Permissions::from_mode(0o666)); + } + builder + .tempfile_in(output.parent().unwrap_or_else(|| std::path::Path::new("."))) + .map(tempfile::NamedTempFile::into_temp_path) + .map_err(|error| error.to_string()) +} + +fn packed_audio_frame(data: &[f32]) -> ffmpeg::frame::Audio { + let mut frame = ffmpeg::frame::Audio::new( + AudioRenderer::SAMPLE_FORMAT, + data.len() / 2, + ffmpeg::ChannelLayout::STEREO, + ); + frame.set_rate(AudioRenderer::SAMPLE_RATE); + for (sample, bytes) in data.iter().zip( + frame + .data_mut(0) + .chunks_exact_mut(std::mem::size_of::()), + ) { + bytes.copy_from_slice(&sample.to_ne_bytes()); + } + frame +} + fn silent_audio_frame(samples: usize) -> ffmpeg::frame::Audio { let mut frame = ffmpeg::frame::Audio::new( AudioRenderer::SAMPLE_FORMAT, @@ -789,6 +1090,7 @@ async fn export_render_to_channel( recordings: &ProjectRecordingsMeta, stop_after_frames_sent: Option, startup_breakdown_ms: Option>>>, + stop_on_encoder_drop: bool, mut on_progress: impl FnMut(u32) -> bool + Send + 'static, project_path: PathBuf, ) -> Result<(), cap_rendering::RenderingError> { @@ -797,7 +1099,7 @@ async fn export_render_to_channel( let screenshot_project_path = project_path; let render_result = { - let render_future = cap_rendering::render_video_to_channel_nv12( + let render_future = Box::pin(cap_rendering::render_video_to_channel_nv12( constants, project, tx_image_data, @@ -809,7 +1111,7 @@ async fn export_render_to_channel( recordings, stop_after_frames_sent, startup_breakdown_ms, - ); + )); let forward_future = async { let mut first_frame_data: Option = None; @@ -875,6 +1177,11 @@ async fn export_render_to_channel( if sender.send(export_frame).is_err() { warn!("Encoder dropped, stopping render forwarding"); + if stop_on_encoder_drop { + return Err(cap_rendering::RenderingError::ImageLoadError( + "Export cancelled".to_string(), + )); + } break; } @@ -956,6 +1263,185 @@ mod tests { } } + #[test] + fn pipeline_interruption_preserves_the_peer_failure() { + assert_eq!( + resolve_pipeline_results( + Err(Mp4PipelineError::Interrupted), + Err(Mp4PipelineError::Failure("renderer failed".into())), + Ok(()), + ), + Err("renderer failed".into()) + ); + assert_eq!( + resolve_pipeline_results( + Err(Mp4PipelineError::Failure("audio failed".into())), + Err(Mp4PipelineError::Interrupted), + Ok(()), + ), + Err("audio failed".into()) + ); + assert_eq!( + resolve_pipeline_results(Err(Mp4PipelineError::Interrupted), Ok(()), Ok(())), + Err("Export cancelled".into()) + ); + } + + #[test] + fn detached_and_active_source_errors_keep_original_order() { + for (active, detached) in [(1, 7), (7, 1)] { + let error = resolve_pipeline_results( + Err(Mp4PipelineError::AudioSourceFailure { + source_index: active, + message: format!("source {active}"), + }), + Err(Mp4PipelineError::Interrupted), + Err(Mp4PipelineError::AudioSourceFailure { + source_index: detached, + message: format!("source {detached}"), + }), + ); + assert_eq!(error, Err("source 1".into())); + } + assert_eq!( + resolve_pipeline_results( + Err(Mp4PipelineError::Interrupted), + Err(Mp4PipelineError::Interrupted), + Err(Mp4PipelineError::from(ExportAudioError::Worker { + source_index: 3, + message: "decoder panicked".into(), + })), + ), + Err("Audio source worker failed: decoder panicked".into()), + ); + } + + #[tokio::test] + async fn validation_failure_before_render_poll_skips_producer() { + let failure = Arc::new(Notify::new()); + failure.notify_one(); + let polled = AtomicBool::new(false); + let result = render_until_audio_failure( + async { + polled.store(true, Ordering::Relaxed); + Ok(()) + }, + Some(failure), + ) + .await; + assert!(matches!(result, Err(Mp4PipelineError::Interrupted))); + assert!(!polled.load(Ordering::Relaxed)); + } + + #[tokio::test] + async fn validation_failure_releases_pending_producer_receiver_and_temp_owners() { + let directory = tempfile::tempdir().unwrap(); + let output = directory.path().join("export.mp4"); + let temporary = Arc::new(temporary_mp4_output(&output).unwrap()); + let temporary_path = temporary.to_path_buf(); + let abort = Arc::new(AtomicBool::new(false)); + let user = AtomicBool::new(false); + let failure = Arc::new(Notify::new()); + let (frame_tx, frame_rx) = std::sync::mpsc::sync_channel::<()>(1); + let (receiver_started_tx, receiver_started_rx) = tokio::sync::oneshot::channel(); + let (producer_started_tx, producer_started_rx) = tokio::sync::oneshot::channel(); + let receiver_guard = temporary.clone(); + let receiver = tokio::task::spawn_blocking(move || { + let _guard = receiver_guard; + receiver_started_tx.send(()).unwrap(); + assert!(frame_rx.recv().is_err()); + Err::(Mp4PipelineError::Interrupted) + }); + receiver_started_rx.await.unwrap(); + let producer_guard = temporary.clone(); + let producer = async move { + let _sender = frame_tx; + let _guard = producer_guard; + producer_started_tx.send(()).unwrap(); + std::future::pending::>().await + }; + let validation_guard = temporary.clone(); + let worker_abort = abort.clone(); + let worker_failure = failure.clone(); + let validator = tokio::spawn(async move { + let _guard = validation_guard; + producer_started_rx.await.unwrap(); + worker_abort.store(true, Ordering::Relaxed); + worker_failure.notify_one(); + Err(Mp4PipelineError::AudioSourceFailure { + source_index: 0, + message: "unused source failed".into(), + }) + }); + let result = tokio::time::timeout(Duration::from_secs(2), async { + let (encoder, render, validation) = tokio::join!( + async { receiver.await.unwrap() }, + render_until_audio_failure(producer, Some(failure.clone())), + complete_audio_validation(validator, abort.clone(), failure), + ); + resolve_pipeline_results(encoder, render, validation) + }) + .await + .unwrap(); + assert_eq!(result, Err("unused source failed".into())); + assert!(abort.load(Ordering::Relaxed)); + assert!(!user.load(Ordering::Relaxed)); + assert_eq!(Arc::strong_count(&temporary), 1); + drop(Arc::try_unwrap(temporary).unwrap()); + assert!(!temporary_path.exists()); + assert!(!output.exists()); + } + + #[tokio::test] + async fn validator_join_error_notifies_and_aborts() { + let abort = Arc::new(AtomicBool::new(false)); + let failure = Arc::new(Notify::new()); + let task = tokio::task::spawn_blocking(|| -> Result<(), Mp4PipelineError> { + panic!("validator wrapper panic"); + }); + let result = complete_audio_validation(task, abort.clone(), failure.clone()).await; + assert!(matches!(result, Err(Mp4PipelineError::Failure(_)))); + assert!(abort.load(Ordering::Relaxed)); + tokio::time::timeout(Duration::from_secs(2), failure.notified()) + .await + .unwrap(); + } + + #[test] + fn temporary_output_is_removed_on_failure_and_never_overwrites() { + let directory = tempfile::tempdir().unwrap(); + let output = directory.path().join("export.mp4"); + let temporary = temporary_mp4_output(&output).unwrap(); + let path = temporary.to_path_buf(); + std::fs::write(&path, b"partial").unwrap(); + drop(temporary); + assert!(!path.exists()); + assert!(!output.exists()); + let temporary = temporary_mp4_output(&output).unwrap(); + std::fs::write(&temporary, b"completed").unwrap(); + std::fs::write(&output, b"existing").unwrap(); + drop(temporary.persist_noclobber(&output).unwrap_err()); + assert_eq!(std::fs::read(&output).unwrap(), b"existing"); + assert_eq!(std::fs::read_dir(directory.path()).unwrap().count(), 1); + } + + #[cfg(unix)] + #[test] + fn temporary_output_preserves_default_creation_permissions() { + use std::os::unix::fs::PermissionsExt; + let directory = tempfile::tempdir().unwrap(); + let output = directory.path().join("export.mp4"); + let reference = directory.path().join("reference.mp4"); + std::fs::write(&reference, b"reference").unwrap(); + let temporary = temporary_mp4_output(&output).unwrap(); + std::fs::write(&temporary, b"completed").unwrap(); + temporary.persist_noclobber(&output).unwrap(); + assert_eq!( + std::fs::metadata(output).unwrap().permissions().mode() & 0o777, + std::fs::metadata(reference).unwrap().permissions().mode() & 0o777 + ); + } + #[test] fn silent_audio_frame_matches_renderer_format() { ffmpeg::init().unwrap(); From bc9a4bd44e5c17974cbf8ef952fbb2db23f0d62d Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 3 Sep 2026 16:48:01 +0100 Subject: [PATCH 06/20] fix: prevent editor saves before the project finishes loading --- apps/desktop-gpui/src/app_windows.rs | 72 +++++++++------------ apps/desktop-gpui/src/editor_clips.rs | 15 ++++- apps/desktop-gpui/src/editor_sidebar.rs | 6 ++ apps/desktop-gpui/src/editor_window.rs | 86 ++++++++++++++++++------- 4 files changed, 111 insertions(+), 68 deletions(-) diff --git a/apps/desktop-gpui/src/app_windows.rs b/apps/desktop-gpui/src/app_windows.rs index 8d1212dd83a..65b0204fce4 100644 --- a/apps/desktop-gpui/src/app_windows.rs +++ b/apps/desktop-gpui/src/app_windows.rs @@ -4463,10 +4463,7 @@ fn load_editor_project(path: PathBuf, handle: WindowHandle, cx: &m ); log_timeline_model(&summary.timeline); let recordings = summary.recordings.clone(); - if handle - .update(cx, |view, window, cx| view.set_summary(summary, window, cx)) - .is_err() - { + if handle.update(cx, |_, _, _| ()).is_err() { return; } @@ -4548,8 +4545,35 @@ fn load_editor_project(path: PathBuf, handle: WindowHandle, cx: &m }; tracing::info!(path = %path.display(), "editor instance ready"); + let (total, config) = { + let config = instance.project_config.1.borrow().clone(); + let total = config + .timeline + .as_ref() + .map_or(0.0, |timeline| timeline.duration()); + (total, config) + }; + let has_camera = instance + .recordings + .segments + .iter() + .any(|segment| segment.camera.is_some()); + let multiple_clips = instance.recordings.segments.len() > 1; + log_timeline_model(&editor_timeline::TimelineModel::build( + &config, + has_camera, + multiple_clips, + )); if handle - .update(cx, |view, _window, _cx| view.set_instance(instance.clone())) + .update(cx, |view, window, cx| { + // Loading controls can queue a save before the engine is ready. + // Publish the loaded config and instance together so those edits + // cannot replace the saved project with the initial defaults. + view.pending_save().borrow_mut().discard(); + view.set_summary(summary, window, cx); + view.set_project(config, window, cx); + view.set_instance(instance.clone()); + }) .is_err() { instance.dispose().await; @@ -4659,44 +4683,6 @@ fn load_editor_project(path: PathBuf, handle: WindowHandle, cx: &m }) .detach(); - // `totalDuration()` (`context.ts:1374-1380`). Read off the instance - // rather than the pre-flight, because `EditorInstance::new` - // synthesises a timeline for a raw bundle -- and `timeline.duration()` - // is exactly what the playback engine stops at - // (`playback.rs:560-570`). - // - // The whole track model comes from the same read: the config the - // instance actually loaded is the one being rendered, holds, clip - // offsets and all. E4 hands the window the config itself rather than - // the derived model, because it is what every edit mutates and what - // the debounced save writes back. - let (total, config) = { - let config = instance.project_config.1.borrow().clone(); - let total = config - .timeline - .as_ref() - .map_or(0.0, |timeline| timeline.duration()); - (total, config) - }; - { - let has_camera = instance - .recordings - .segments - .iter() - .any(|segment| segment.camera.is_some()); - let multiple_clips = instance.recordings.segments.len() > 1; - log_timeline_model(&editor_timeline::TimelineModel::build( - &config, - has_camera, - multiple_clips, - )); - } - if handle - .update(cx, |view, window, cx| view.set_project(config, window, cx)) - .is_err() - { - return; - } load_editor_waveforms(instance.clone(), handle, cx); if handle diff --git a/apps/desktop-gpui/src/editor_clips.rs b/apps/desktop-gpui/src/editor_clips.rs index 0ce3cf575a4..26e739bb10e 100644 --- a/apps/desktop-gpui/src/editor_clips.rs +++ b/apps/desktop-gpui/src/editor_clips.rs @@ -408,12 +408,16 @@ impl EditorWindow { ui::Button::plain(&self.theme, "clips-pill", variant, ui::ButtonSize::Md) .icon("icons/clapperboard.svg") .label("Clips") + .disabled(!self.project_ready()) .height(px(40.)) .font_weight(FontWeight::MEDIUM) .on_click(cx.listener(|this, _, window, cx| this.toggle_clips(window, cx))) } pub(crate) fn toggle_clips(&mut self, window: &mut Window, cx: &mut Context) { + if !self.project_ready() { + return; + } self.set_selection(None, cx); if self.clips.open { self.close_clips(window, cx); @@ -1200,7 +1204,7 @@ impl EditorWindow { } fn begin_editor_recording(&mut self, cx: &mut Context) -> bool { - if self.clips.importing { + if !self.project_ready() || self.clips.importing { return false; } let session = RecordingSession::global(cx); @@ -1252,6 +1256,13 @@ impl EditorWindow { _window: &mut Window, cx: &mut Context, ) { + if !self.project_ready() { + tracing::warn!( + recording = %recording_dir.display(), + "the editor is not ready; leaving the recording in the library" + ); + return; + } if self.clips.importing { // A concurrent import owns the bundle merge; the capture stays in // the library and can be pulled in through "Existing recording". @@ -1506,7 +1517,7 @@ impl EditorWindow { window: &mut Window, cx: &mut Context, ) { - if self.clips.importing { + if !self.project_ready() || self.clips.importing { return; } if self.playing { diff --git a/apps/desktop-gpui/src/editor_sidebar.rs b/apps/desktop-gpui/src/editor_sidebar.rs index 9bb5c454dc0..45a1ffbe72b 100644 --- a/apps/desktop-gpui/src/editor_sidebar.rs +++ b/apps/desktop-gpui/src/editor_sidebar.rs @@ -1043,6 +1043,9 @@ impl EditorWindow { window: &mut Window, cx: &mut Context, ) { + if !self.project_ready() { + return; + } // An edit that is not the open colour panel's closes its bracket // first: the panel is a system window and stays up while the user // does other things, and an unrelated change must not be swallowed @@ -1070,6 +1073,9 @@ impl EditorWindow { cx: &mut Context, change: impl FnOnce(&mut ProjectConfiguration) -> bool, ) { + if !self.project_ready() { + return; + } self.end_color_history(); if !change(&mut self.project) { return; diff --git a/apps/desktop-gpui/src/editor_window.rs b/apps/desktop-gpui/src/editor_window.rs index b5579e57be5..f05119b6985 100644 --- a/apps/desktop-gpui/src/editor_window.rs +++ b/apps/desktop-gpui/src/editor_window.rs @@ -652,25 +652,30 @@ impl Render for EditorSectionView { let Some(editor) = self.editor.upgrade() else { return div().into_any_element(); }; - editor.update(cx, |editor, cx| match self.section { - EditorSection::Header => editor.render_header(window, cx).into_any_element(), - EditorSection::Toolbar => editor.render_player_toolbar(cx).into_any_element(), - EditorSection::Transport => editor.render_transport(cx).into_any_element(), - // The Clips layout mode swaps the config sidebar's column for the - // clips sidebar; the config sidebar is hidden, not destroyed - // (`Editor.tsx:728-747`). - EditorSection::Sidebar => { - if editor.clips.open { - editor.render_clips_sidebar(cx).into_any_element() - } else { - editor.render_sidebar(cx).into_any_element() - } + editor.update(cx, |editor, cx| { + if !editor.project_ready() && !matches!(self.section, EditorSection::Header) { + return div().size_full().into_any_element(); } - EditorSection::Timeline => { - let viewport_width: f32 = window.viewport_size().width.into(); - editor - .render_timeline(viewport_width, cx) - .into_any_element() + match self.section { + EditorSection::Header => editor.render_header(window, cx).into_any_element(), + EditorSection::Toolbar => editor.render_player_toolbar(cx).into_any_element(), + EditorSection::Transport => editor.render_transport(cx).into_any_element(), + // The Clips layout mode swaps the config sidebar's column for the + // clips sidebar; the config sidebar is hidden, not destroyed + // (`Editor.tsx:728-747`). + EditorSection::Sidebar => { + if editor.clips.open { + editor.render_clips_sidebar(cx).into_any_element() + } else { + editor.render_sidebar(cx).into_any_element() + } + } + EditorSection::Timeline => { + let viewport_width: f32 = window.viewport_size().width.into(); + editor + .render_timeline(viewport_width, cx) + .into_any_element() + } } }) } @@ -1570,7 +1575,11 @@ impl EditorWindow { }) .detach(); - let name_input = cx.new(|cx| ui::TextInputState::single_line(window, cx)); + let name_input = cx.new(|cx| { + let mut input = ui::TextInputState::single_line(window, cx); + input.set_disabled(true, cx); + input + }); let hex_targets = [ crate::editor_sidebar::ColorTarget::BackgroundColor, crate::editor_sidebar::ColorTarget::GradientFrom, @@ -1822,7 +1831,8 @@ impl EditorWindow { // the timeline's width. self.view.transform = Transform::initial(summary.duration); self.name_input.update(cx, |input, cx| { - input.set_text(summary.pretty_name.clone(), cx) + input.set_text(summary.pretty_name.clone(), cx); + input.set_disabled(false, cx); }); self.state = LoadState::Ready(Box::new(summary)); cx.notify(); @@ -1966,6 +1976,9 @@ impl EditorWindow { } pub(crate) fn project_changed(&mut self, window: &mut Window, cx: &mut Context) { + if !self.project_ready() { + return; + } // Before `history.record`, so the re-projected caption track is part // of the same undo entry as the edit that moved it. self.rederive_caption_track(); @@ -1978,6 +1991,9 @@ impl EditorWindow { } pub(crate) fn project_changed_live(&mut self, cx: &mut Context) { + if !self.project_ready() { + return; + } self.publish_project(); cx.notify(); } @@ -2058,6 +2074,9 @@ impl EditorWindow { /// the re-render is skipped while playing exactly as `emitRenderFrame`'s /// `if (!editorState.playing)` gate does (`:493`). pub(crate) fn publish_project(&self) { + if !self.project_ready() { + return; + } let Some(instance) = &self.instance else { return; }; @@ -2076,6 +2095,9 @@ impl EditorWindow { /// executor. A later edit drops this task, which is `clearTimeout` plus a /// fresh `setTimeout`. pub(crate) fn schedule_save(&mut self, window: &mut Window, cx: &mut Context) { + if !self.project_ready() { + return; + } self.pending_save.borrow_mut().config = Some(self.project.clone()); let pending = self.pending_save.clone(); self.save_task = Some(cx.spawn_in(window, async move |_, cx| { @@ -2164,6 +2186,10 @@ impl EditorWindow { self.selection.as_ref() } + pub(crate) fn project_ready(&self) -> bool { + self.instance.is_some() && matches!(&self.state, LoadState::Ready(_)) + } + #[allow(dead_code)] /// The live project config, for the units that render from it (the config /// sidebar's controls) or serialise it (export). @@ -2464,6 +2490,9 @@ impl EditorWindow { /// (`useEditorShortcuts.ts:10`) and `e.repeat` is ignored there /// (`:42`) as `is_held` is here. fn on_key(&mut self, event: &gpui::KeyDownEvent, window: &mut Window, cx: &mut Context) { + if !self.project_ready() { + return; + } if self.frame_controls.is_open() && event.keystroke.key == "escape" { self.close_frame_controls(window, cx); cx.stop_propagation(); @@ -5330,7 +5359,7 @@ impl EditorWindow { } else { self.history.can_redo() }; - let enabled = can || self.selection.is_some(); + let enabled = self.project_ready() && (can || self.selection.is_some()); ui::EditorButton::plain(&theme, id) .left_icon(icon) .disabled(!enabled) @@ -5615,6 +5644,9 @@ impl EditorWindow { window: &mut Window, cx: &mut Context, ) { + if !self.project_ready() { + return; + } if self.presets_menu.is_some() { self.presets_menu = None; cx.notify(); @@ -7279,6 +7311,7 @@ impl EditorWindow { ui::EditorButton::plain(&theme, "presets") .left_icon("icons/presets.svg") .label("Presets") + .disabled(!self.project_ready()) .right_icon("icons/chevron-down.svg") .pressed(self.presets_menu.is_some()) .on_click(cx.listener(|this, event: &gpui::ClickEvent, window, cx| { @@ -7399,7 +7432,8 @@ impl EditorWindow { .h(px(40.)) .flex_none() .rounded(px(12.)) - .cursor_pointer() + .when(self.project_ready(), |button| button.cursor_pointer()) + .when(!self.project_ready(), |button| button.opacity(0.5)) .bg(gpui::linear_gradient( 180., gpui::linear_color_stop(gpui::rgb(0x3b82f6), 0.), @@ -7430,7 +7464,13 @@ impl EditorWindow { .text_color(gpui::white()), ) .child("Export") - .on_click(cx.listener(|this, _, window, cx| this.open_export(window, cx))) + .when(self.project_ready(), |button| { + button.on_click(cx.listener(|this, _, window, cx| { + if this.project_ready() { + this.open_export(window, cx); + } + })) + }) } // -- Player -------------------------------------------------------------- From 1055737996a3df0b87fcb1a404758a3c1f5ab4ec Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 3 Sep 2026 16:48:01 +0100 Subject: [PATCH 07/20] test: run streaming export regressions in the sync matrix --- .github/workflows/sync-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/sync-tests.yml b/.github/workflows/sync-tests.yml index 7568140562a..6af4a961f78 100644 --- a/.github/workflows/sync-tests.yml +++ b/.github/workflows/sync-tests.yml @@ -24,6 +24,7 @@ on: - "crates/timestamp/**" - "crates/rendering/**" - "crates/editor/**" + - "crates/export/**" - "crates/audio/**" - "crates/media-info/**" - "crates/project/**" @@ -144,6 +145,8 @@ jobs: - name: Editor audio playback and export regressions shell: bash run: | + cargo test --locked -p cap-audio --lib + cargo test --locked -p cap-export --lib cargo test --locked -p cap-editor --lib audio::tests:: cargo test --locked -p cap-editor --lib audio_output::tests:: cargo test --locked -p cap-editor --lib playback::tests:: From f613212d6bc92b2761d9ccf352a709944ea40fa2 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 3 Sep 2026 17:08:11 +0100 Subject: [PATCH 08/20] improve: avoid caption word copies and empty zoom allocations --- crates/rendering/src/layers/captions.rs | 71 +++++++++++++++++-------- crates/rendering/src/zoom_spring.rs | 43 +++++++++++---- 2 files changed, 82 insertions(+), 32 deletions(-) diff --git a/crates/rendering/src/layers/captions.rs b/crates/rendering/src/layers/captions.rs index a20a65ab078..6f690619428 100644 --- a/crates/rendering/src/layers/captions.rs +++ b/crates/rendering/src/layers/captions.rs @@ -1,5 +1,5 @@ use bytemuck::{Pod, Zeroable}; -use cap_project::XY; +use cap_project::{CaptionWord, XY}; use glyphon::cosmic_text::LayoutRunIter; use glyphon::{ Attrs, Buffer, Cache, Color, Family, FontSystem, Metrics, Resolution, Shaping, SwashCache, @@ -10,13 +10,6 @@ use wgpu::{Device, Queue, include_wgsl, util::DeviceExt}; use crate::{DecodedSegmentFrames, ProjectUniforms, RenderVideoConstants, parse_color_component}; -#[derive(Debug, Clone)] -pub struct CaptionWord { - pub text: String, - pub start: f32, - pub end: f32, -} - #[repr(C)] #[derive(Copy, Clone, Pod, Zeroable, Debug)] pub struct CaptionSettings { @@ -518,16 +511,7 @@ impl CaptionsLayer { } else { joined_caption_text }; - let caption_words: Vec = active - .segment - .words - .iter() - .map(|w| CaptionWord { - text: w.text.clone(), - start: w.start, - end: w.end, - }) - .collect(); + let caption_words = &active.segment.words; let fade_opacity = calculate_caption_fade( current_time, @@ -572,8 +556,8 @@ impl CaptionsLayer { active_word_highlight_enabled && !caption_words.is_empty() && !use_pill_highlight; let active_word_byte_range = if use_pill_highlight { - find_active_word_index(current_time as f32, &caption_words) - .and_then(|idx| word_byte_range(&caption_text, &caption_words, idx, uppercase)) + find_active_word_index(current_time as f32, caption_words) + .and_then(|idx| word_byte_range(&caption_text, caption_words, idx, uppercase)) } else { None }; @@ -677,7 +661,7 @@ impl CaptionsLayer { current_time as f32, word, idx, - &caption_words, + caption_words, word_transition_duration, ); @@ -1180,7 +1164,10 @@ fn calculate_caption_bounce(current_time: f64, start: f64, end: f64, fade_durati #[cfg(test)] mod tests { - use super::{caption_segment_effective_end, find_active_caption_segment}; + use super::{ + caption_segment_effective_end, find_active_caption_segment, find_active_word_index, + word_byte_range, + }; use cap_project::{CaptionTrackSegment, CaptionWord}; fn segment(start: f64, end: f64, words: Vec) -> CaptionTrackSegment { @@ -1230,4 +1217,44 @@ mod tests { // Still active while the (capped) word is on screen. assert!(find_active_caption_segment(37.0, &segments, 0.2).is_some()); } + + #[test] + fn active_word_selection_preserves_boundaries_and_gaps() { + let words = [word(0.1, 0.4), word(0.5, 0.9)]; + + for (time, expected) in [ + (-0.1, 0), + (0.1, 0), + (0.4, 0), + (0.49, 0), + (0.5, 1), + (0.9, 1), + (1.2, 1), + ] { + assert_eq!(find_active_word_index(time, &words), Some(expected)); + } + assert_eq!(find_active_word_index(0.5, &[]), None); + } + + #[test] + fn word_ranges_preserve_repeated_words_and_unicode_uppercase() { + let words = ["ʼn", "Straße", "ʼn"].map(|text| CaptionWord { + text: text.to_string(), + start: 0.0, + end: 1.0, + }); + let text = "ʼn Straße ʼn"; + + for (index, expected) in [(0, (0, 2)), (1, (3, 10)), (2, (11, 13))] { + assert_eq!(word_byte_range(text, &words, index, false), Some(expected)); + } + for (index, expected) in [(0, (0, 3)), (1, (4, 11)), (2, (12, 15))] { + assert_eq!( + word_byte_range(&text.to_uppercase(), &words, index, true), + Some(expected) + ); + } + assert_eq!(word_byte_range(text, &words, 3, false), None); + assert_eq!(word_byte_range("missing", &words, 0, false), None); + } } diff --git a/crates/rendering/src/zoom_spring.rs b/crates/rendering/src/zoom_spring.rs index 76aa4763ebe..5904a0d7b11 100644 --- a/crates/rendering/src/zoom_spring.rs +++ b/crates/rendering/src/zoom_spring.rs @@ -478,7 +478,11 @@ impl ZoomTransformTimeline { let mut zoom_segments = zoom_segments.to_vec(); zoom_segments.sort_by(|a, b| a.start.total_cmp(&b.start).then(a.end.total_cmp(&b.end))); - let time_map = build_time_map(timeline); + let time_map = if zoom_segments.is_empty() { + Vec::new() + } else { + build_time_map(timeline) + }; let clusters = zoom_segments .iter() .map(|segment| match segment.mode { @@ -518,12 +522,7 @@ impl ZoomTransformTimeline { let total_samples = (duration_secs * 1000.0 / STEP_MS).ceil() as usize + 2; if zoom_segments.is_empty() { return Self { - samples: vec![TimelineSample { - amount: 1.0, - center: XY::new(0.5, 0.5), - activity: 0.0, - snapped: false, - }], + samples: Vec::new(), state: None, zoom_segments, clusters, @@ -946,11 +945,35 @@ mod tests { #[test] fn empty_zoom_timeline_stays_constant_without_precompute_work() { let mut timeline = timeline_for(&[], &CursorEvents::default(), 60.0 * 60.0); - timeline.ensure_precomputed_until(60.0 * 60.0); + for seconds in [ + f32::NEG_INFINITY, + -1.0, + -0.0, + 0.0, + 0.5, + 60.0 * 60.0, + f32::MAX, + f32::INFINITY, + f32::NAN, + ] { + timeline.ensure_precomputed_until(seconds); + timeline.precompute(); + let sample = timeline.sample(seconds); + assert_eq!( + [ + sample.t, + sample.bounds.top_left.x, + sample.bounds.top_left.y, + sample.bounds.bottom_right.x, + sample.bounds.bottom_right.y, + ] + .map(f64::to_bits), + [0.0, 0.0, 0.0, 1.0, 1.0].map(f64::to_bits) + ); + assert!(!timeline.snapped_within(seconds, 0.0)); + } assert!(timeline.state.is_none()); - assert_eq!(timeline.samples.len(), 1); - assert_eq!(timeline.sample(60.0 * 60.0).display_amount(), 1.0); } /// Max |value delta| and |slope delta| between adjacent 8ms sample From 6cc96d3652aab7cec2113d4a251d17f294f12a21 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 3 Sep 2026 17:08:11 +0100 Subject: [PATCH 09/20] fix: preserve cursor texture sampling across sprite edges --- crates/rendering/src/shaders/cursor.wgsl | 9 +- .../rendering/tests/cursor_texture_edges.rs | 355 ++++++++++++++++++ 2 files changed, 359 insertions(+), 5 deletions(-) create mode 100644 crates/rendering/tests/cursor_texture_edges.rs diff --git a/crates/rendering/src/shaders/cursor.wgsl b/crates/rendering/src/shaders/cursor.wgsl index 00cd67d171a..9f60d75a35c 100644 --- a/crates/rendering/src/shaders/cursor.wgsl +++ b/crates/rendering/src/shaders/cursor.wgsl @@ -103,11 +103,10 @@ fn vs_main(@builtin(vertex_index) vertex_index: u32) -> VertexOutput { } fn sample_cursor(uv: vec2) -> vec4 { - if (uv.x < 0.0 || uv.x > 1.0 || uv.y < 0.0 || uv.y > 1.0) { - return vec4(0.0, 0.0, 0.0, 0.0); - } - - return textureSample(t_cursor, s_cursor, uv); + // Keep implicit mip derivatives uniform across the sprite boundary. + let color = textureSample(t_cursor, s_cursor, uv); + let inside = all(uv >= vec2(0.0)) && all(uv <= vec2(1.0)); + return select(vec4(0.0), color, inside); } // Confine the sprite to the display card (screen_bounds is the card's diff --git a/crates/rendering/tests/cursor_texture_edges.rs b/crates/rendering/tests/cursor_texture_edges.rs new file mode 100644 index 00000000000..219499807a1 --- /dev/null +++ b/crates/rendering/tests/cursor_texture_edges.rs @@ -0,0 +1,355 @@ +use std::borrow::Cow; + +use wgpu::util::DeviceExt; + +const SHADER: &str = include_str!("../src/shaders/cursor.wgsl"); +const OUTPUT_WIDTH: u32 = 192; +const OUTPUT_HEIGHT: u32 = 128; +const TEXTURE_WIDTH: u32 = 64; +const TEXTURE_HEIGHT: u32 = 128; + +#[derive(Clone, Copy, Debug)] +struct Case { + height: f32, + rotation: f32, + offset: [f32; 2], + motion: [f32; 2], +} + +impl Case { + fn uniforms(self) -> [[f32; 4]; 8] { + [ + [ + 96.0 + self.offset[0], + 38.0 + self.offset[1], + self.height * TEXTURE_WIDTH as f32 / TEXTURE_HEIGHT as f32, + self.height, + ], + [OUTPUT_WIDTH as f32, OUTPUT_HEIGHT as f32, 0.0, 0.0], + [0.0, 0.0, OUTPUT_WIDTH as f32, OUTPUT_HEIGHT as f32], + [self.motion[0], self.motion[1], 1.0, 1.0], + [0.0, self.rotation, 0.0, 0.0], + [0.0; 4], + [0.0; 4], + [0.0; 4], + ] + } +} + +fn reference_shader() -> String { + // Capture gradients before any sprite-boundary branch, including every blur tap. + let replacements = [ + ( + "fn sample_cursor(uv: vec2)", + "fn sample_cursor(uv: vec2, gradient_x: vec2, gradient_y: vec2)", + ), + ( + "textureSample(t_cursor, s_cursor, uv)", + "textureSampleGrad(t_cursor, s_cursor, uv, gradient_x, gradient_y)", + ), + ( + "sample_cursor(input.uv)", + "sample_cursor(input.uv, dpdx(input.uv), dpdy(input.uv))", + ), + ( + "sample_cursor(sample_uv)", + "sample_cursor(sample_uv, dpdx(sample_uv), dpdy(sample_uv))", + ), + ]; + let mut source = SHADER.to_string(); + for (from, to) in replacements { + assert_eq!(source.matches(from).count(), 1); + source = source.replace(from, to); + } + source +} + +fn pipeline(device: &wgpu::Device, source: &str) -> wgpu::RenderPipeline { + let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("Cursor edge regression"), + source: wgpu::ShaderSource::Wgsl(Cow::Borrowed(source)), + }); + device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("Cursor edge regression"), + layout: None, + vertex: wgpu::VertexState { + module: &shader, + entry_point: Some("vs_main"), + buffers: &[], + compilation_options: Default::default(), + }, + fragment: Some(wgpu::FragmentState { + module: &shader, + entry_point: Some("fs_main"), + targets: &[Some(wgpu::ColorTargetState { + format: wgpu::TextureFormat::Rgba8Unorm, + blend: Some(wgpu::BlendState::PREMULTIPLIED_ALPHA_BLENDING), + write_mask: wgpu::ColorWrites::ALL, + })], + compilation_options: Default::default(), + }), + primitive: wgpu::PrimitiveState { + topology: wgpu::PrimitiveTopology::TriangleStrip, + ..Default::default() + }, + depth_stencil: None, + multisample: Default::default(), + multiview: None, + cache: None, + }) +} + +fn mip_texture(device: &wgpu::Device, queue: &wgpu::Queue) -> wgpu::Texture { + let colors = [ + [255, 0, 0], + [0, 255, 0], + [0, 0, 255], + [255, 255, 0], + [255, 0, 255], + [0, 255, 255], + [128, 128, 128], + [255, 255, 255], + ]; + let texture = device.create_texture(&wgpu::TextureDescriptor { + label: Some("Distinct cursor mip levels"), + size: wgpu::Extent3d { + width: TEXTURE_WIDTH, + height: TEXTURE_HEIGHT, + depth_or_array_layers: 1, + }, + mip_level_count: colors.len() as u32, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba8Unorm, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }); + for (level, color) in colors.into_iter().enumerate() { + let width = (TEXTURE_WIDTH >> level).max(1); + let height = (TEXTURE_HEIGHT >> level).max(1); + let mut bytes = Vec::with_capacity((width * height * 4) as usize); + for y in 0..height { + for x in 0..width { + let u = (x as f32 + 0.5) / width as f32; + let v = (y as f32 + 0.5) / height as f32; + let pixel = if (0.15..0.85).contains(&u) && (0.15..0.85).contains(&v) { + [color[0], color[1], color[2], 255] + } else { + [0; 4] + }; + bytes.extend_from_slice(&pixel); + } + } + queue.write_texture( + wgpu::TexelCopyTextureInfo { + texture: &texture, + mip_level: level as u32, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + &bytes, + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(width * 4), + rows_per_image: Some(height), + }, + wgpu::Extent3d { + width, + height, + depth_or_array_layers: 1, + }, + ); + } + texture +} + +fn render( + device: &wgpu::Device, + queue: &wgpu::Queue, + pipeline: &wgpu::RenderPipeline, + cursor: &wgpu::TextureView, + sampler: &wgpu::Sampler, + case: Case, +) -> Vec { + let uniform_buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("Cursor edge uniforms"), + contents: bytemuck::cast_slice(&case.uniforms()), + usage: wgpu::BufferUsages::UNIFORM, + }); + let group = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: None, + layout: &pipeline.get_bind_group_layout(0), + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: uniform_buffer.as_entire_binding(), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::TextureView(cursor), + }, + wgpu::BindGroupEntry { + binding: 2, + resource: wgpu::BindingResource::Sampler(sampler), + }, + ], + }); + let output = device.create_texture(&wgpu::TextureDescriptor { + label: Some("Cursor edge output"), + size: wgpu::Extent3d { + width: OUTPUT_WIDTH, + height: OUTPUT_HEIGHT, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba8Unorm, + usage: wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::COPY_SRC, + view_formats: &[], + }); + let buffer = device.create_buffer(&wgpu::BufferDescriptor { + label: Some("Cursor edge readback"), + size: u64::from(OUTPUT_WIDTH * OUTPUT_HEIGHT * 4), + usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, + mapped_at_creation: false, + }); + let view = output.create_view(&Default::default()); + let mut encoder = device.create_command_encoder(&Default::default()); + { + let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: None, + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &view, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + pass.set_pipeline(pipeline); + pass.set_bind_group(0, &group, &[]); + pass.draw(0..4, 0..1); + } + encoder.copy_texture_to_buffer( + output.as_image_copy(), + wgpu::TexelCopyBufferInfo { + buffer: &buffer, + layout: wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(OUTPUT_WIDTH * 4), + rows_per_image: Some(OUTPUT_HEIGHT), + }, + }, + output.size(), + ); + queue.submit([encoder.finish()]); + let (sender, receiver) = std::sync::mpsc::channel(); + buffer + .slice(..) + .map_async(wgpu::MapMode::Read, move |result| { + sender + .send(result) + .expect("readback receiver remains alive"); + }); + device.poll(wgpu::PollType::Wait).expect("poll GPU"); + receiver + .recv() + .expect("readback callback") + .expect("map GPU"); + let pixels = buffer.slice(..).get_mapped_range().to_vec(); + buffer.unmap(); + pixels +} + +fn assert_transparent_exterior(pixels: &[u8], case: Case) { + let [x, y, width, height] = case.uniforms()[0]; + let (s, c) = case.rotation.sin_cos(); + let mut velocity = [case.motion[0] / width, case.motion[1] / height]; + let length = velocity[0].hypot(velocity[1]); + if length > 4.0 { + velocity = velocity.map(|component| component * 4.0 / length); + } + let mut transparent_pixels = 0; + for (index, pixel) in pixels.chunks_exact(4).enumerate() { + let dx = (index as u32 % OUTPUT_WIDTH) as f32 + 0.5 - x; + let dy = (index as u32 / OUTPUT_WIDTH) as f32 + 0.5 - y; + let uv = [ + c * dx / width - s * dy / height, + s * dx / width + c * dy / height, + ]; + let outside = (0..21).all(|tap| { + let sample = [ + uv[0] + velocity[0] * tap as f32 / 20.0, + uv[1] + velocity[1] * tap as f32 / 20.0, + ]; + sample.iter().any(|value| *value < -0.001 || *value > 1.001) + }); + if outside { + assert_eq!(pixel, [0; 4], "exterior pixel {index}, {case:?}"); + transparent_pixels += 1; + } + } + assert!(transparent_pixels > 1_000); +} + +#[test] +fn minified_cursor_edges_match_explicit_gradients() { + let instance = cap_rendering::create_wgpu_instance_sync(); + let Ok(adapter) = + pollster::block_on(instance.request_adapter(&wgpu::RequestAdapterOptions::default())) + else { + eprintln!("no GPU adapter available, skipping cursor edge regression"); + return; + }; + eprintln!("Cursor edge adapter: {:?}", adapter.get_info()); + let (device, queue) = + pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor::default())) + .expect("create cursor edge test device"); + let actual_pipeline = pipeline(&device, SHADER); + let reference_pipeline = pipeline(&device, &reference_shader()); + let cursor = mip_texture(&device, &queue).create_view(&Default::default()); + let sampler = device.create_sampler(&wgpu::SamplerDescriptor { + mag_filter: wgpu::FilterMode::Linear, + min_filter: wgpu::FilterMode::Linear, + mipmap_filter: wgpu::FilterMode::Linear, + anisotropy_clamp: 4, + ..Default::default() + }); + for height in [6.4, 12.0, 25.0, 50.0] { + for degrees in [-20.0_f32, 0.0, 20.0] { + for offset in [[0.0, 0.0], [0.37, 0.63]] { + for motion in [[0.0, 0.0], [36.0, 0.0], [-18.0, 13.0]] { + let case = Case { + height, + rotation: degrees.to_radians(), + offset, + motion, + }; + let actual = render(&device, &queue, &actual_pipeline, &cursor, &sampler, case); + let expected = render( + &device, + &queue, + &reference_pipeline, + &cursor, + &sampler, + case, + ); + assert!(expected.chunks_exact(4).any(|pixel| pixel[3] > 0)); + let max_error = actual + .iter() + .zip(&expected) + .map(|(actual, expected)| actual.abs_diff(*expected)) + .max() + .unwrap(); + assert!(max_error <= 2, "mip edge error {max_error}/255, {case:?}"); + assert_transparent_exterior(&actual, case); + } + } + } + } +} From 124885a3b22cb1d37daf0bdb35d098ebea18c032 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 3 Sep 2026 17:08:11 +0100 Subject: [PATCH 10/20] improve: anchor editor menus and preserve selection behavior --- apps/desktop-gpui/src/assets.rs | 1 + apps/desktop-gpui/src/editor_clips.rs | 10 +- apps/desktop-gpui/src/editor_panels.rs | 34 +- apps/desktop-gpui/src/editor_sidebar.rs | 29 +- apps/desktop-gpui/src/editor_tabs.rs | 84 ++--- apps/desktop-gpui/src/editor_window.rs | 295 ++++++++++++++++-- .../src/screenshot_annotations.rs | 2 + apps/desktop-gpui/src/screenshot_crop.rs | 1 + apps/desktop-gpui/src/screenshot_editor.rs | 4 + apps/desktop-gpui/src/ui/button.rs | 13 + apps/desktop-gpui/src/ui/editor_button.rs | 22 +- apps/desktop-gpui/src/ui/menu.rs | 118 ++++++- apps/desktop-gpui/src/ui/radio_cards.rs | 14 +- apps/desktop-gpui/src/ui/select.rs | 19 +- 14 files changed, 524 insertions(+), 122 deletions(-) diff --git a/apps/desktop-gpui/src/assets.rs b/apps/desktop-gpui/src/assets.rs index 7ed9d7be489..935e2c5919d 100644 --- a/apps/desktop-gpui/src/assets.rs +++ b/apps/desktop-gpui/src/assets.rs @@ -360,6 +360,7 @@ mod tests { include_str!("screenshot_annotations.rs"), // `ui::SelectionHeader` names the check and the trash itself. include_str!("ui/selection_header.rs"), + include_str!("ui/radio_cards.rs"), // The onboarding window's welcome cards and permissions surface; the // per-permission row glyphs are named on `OSPermission::icon`. include_str!("onboarding_window.rs"), diff --git a/apps/desktop-gpui/src/editor_clips.rs b/apps/desktop-gpui/src/editor_clips.rs index 26e739bb10e..d8cc2448c62 100644 --- a/apps/desktop-gpui/src/editor_clips.rs +++ b/apps/desktop-gpui/src/editor_clips.rs @@ -410,6 +410,7 @@ impl EditorWindow { .label("Clips") .disabled(!self.project_ready()) .height(px(40.)) + .radius(px(12.)) .font_weight(FontWeight::MEDIUM) .on_click(cx.listener(|this, _, window, cx| this.toggle_clips(window, cx))) } @@ -758,6 +759,7 @@ impl EditorWindow { .px(px(16.)) .w_full() .h(px(64.)) + .rounded_t(px(11.)) .border_b_1() .border_color(Hsla::from(theme.gray_3)) .text_size(px(14.)) @@ -842,12 +844,13 @@ impl EditorWindow { .gap(px(8.)) .font_weight(FontWeight::MEDIUM) .disabled(self.clips.importing) - .on_click(cx.listener( - |this, event: &gpui::ClickEvent, _window, cx| { + .on_open(cx.listener( + |this, bounds: &Bounds, _window, cx| { if this.clips.importing { return; } - this.clips.import_menu = Some(event.position()); + this.clips.import_menu = + Some(bounds.bottom_left() + gpui::point(px(0.), px(4.))); cx.notify(); }, )), @@ -1364,6 +1367,7 @@ impl EditorWindow { .child( div() .id("clips-import-backdrop") + .occlude() .absolute() .top_0() .left_0() diff --git a/apps/desktop-gpui/src/editor_panels.rs b/apps/desktop-gpui/src/editor_panels.rs index b50c9765e38..abafeb5b1c4 100644 --- a/apps/desktop-gpui/src/editor_panels.rs +++ b/apps/desktop-gpui/src/editor_panels.rs @@ -2694,6 +2694,17 @@ impl EditorWindow { .map(|(mode, label)| ui::MenuItem::new(*label, *mode == current)) .collect() } + SidebarMenu::Camera3DEasing(_) => { + let current = timeline + .camera3d_segments + .get(index) + .map_or(0, motion_easing); + MOTION_EASINGS + .iter() + .enumerate() + .map(|(index, (_, label, _, _))| ui::MenuItem::new(*label, index == current)) + .collect() + } _ => Vec::new(), } } @@ -2759,6 +2770,9 @@ impl EditorWindow { true }); } + SidebarMenu::Camera3DEasing(_) => { + self.set_camera3d_easing(segment, index, window, cx); + } _ => {} } } @@ -5598,6 +5612,7 @@ impl EditorWindow { .child( ui::EditorButton::plain(&theme, "camera3d-swap") .left_icon("icons/arrow-left-right.svg") + .tooltip(&theme, "Swap start and end") .on_click(cx.listener(move |this, _, window, cx| { this.swap_camera3d_poses(index, window, cx); })), @@ -5613,6 +5628,7 @@ impl EditorWindow { .child( ui::EditorButton::plain(&theme, "camera3d-flip-h") .left_icon("icons/flip-horizontal-2.svg") + .tooltip(&theme, "Flip horizontal") .on_click(cx.listener(move |this, _, window, cx| { this.flip_camera3d(index, true, window, cx); })), @@ -5620,6 +5636,7 @@ impl EditorWindow { .child( ui::EditorButton::plain(&theme, "camera3d-flip-v") .left_icon("icons/flip-vertical-2.svg") + .tooltip(&theme, "Flip vertical") .on_click(cx.listener(move |this, _, window, cx| { this.flip_camera3d(index, false, window, cx); })), @@ -5958,17 +5975,12 @@ impl EditorWindow { } fn easing_select(&self, index: usize, current: usize, cx: &mut Context) -> AnyElement { - let theme = self.theme; - // Four options, and `ui::Menu` draws at the pointer without flipping; - // the corner-style select already established the two-option toggle, - // and this one cycles for the same reason. - ui::Select::plain(&theme, "camera3d-easing", MOTION_EASINGS[current].1) - .stretch_label() - .on_click(cx.listener(move |this, _, window, cx| { - let next = (current + 1) % MOTION_EASINGS.len(); - this.set_camera3d_easing(index, next, window, cx); - })) - .into_any_element() + self.menu_select( + SidebarMenu::Camera3DEasing(index), + "camera3d-easing", + MOTION_EASINGS[current].1, + cx, + ) } /// `selectPose` (`:4933-4937`): flip the card **and** park the playhead on diff --git a/apps/desktop-gpui/src/editor_sidebar.rs b/apps/desktop-gpui/src/editor_sidebar.rs index 45a1ffbe72b..3d2ac8a3b33 100644 --- a/apps/desktop-gpui/src/editor_sidebar.rs +++ b/apps/desktop-gpui/src/editor_sidebar.rs @@ -3518,29 +3518,12 @@ impl EditorWindow { .text_color(Hsla::from(theme.gray_11)) .child("CORNER STYLE"), ) - .child( - ui::Select::plain(&theme, "corner-style", label) - .stretch_label() - .on_click(cx.listener(move |this, _, window, cx| { - // Two options: the trigger toggles between them rather - // than opening a two-row menu. `ui::Menu` draws at the - // pointer and this select is the only one in the tab; - // a real menu arrives with the tabs that have several. - let next = match this.project.background.rounding_type { - CornerStyle::Squircle => CornerStyle::Rounded, - CornerStyle::Rounded => CornerStyle::Squircle, - }; - this.edit_background( - "rounding-type", - |project| { - project.background.rounding_type = next; - true - }, - window, - cx, - ); - })), - ) + .child(self.menu_select( + crate::editor_tabs::SidebarMenu::BackgroundCornerStyle, + "corner-style", + label, + cx, + )) } fn render_border_field(&self, cx: &mut Context) -> impl IntoElement { diff --git a/apps/desktop-gpui/src/editor_tabs.rs b/apps/desktop-gpui/src/editor_tabs.rs index eb7c660a0a2..a61daa1c11c 100644 --- a/apps/desktop-gpui/src/editor_tabs.rs +++ b/apps/desktop-gpui/src/editor_tabs.rs @@ -13,15 +13,6 @@ //! [`EditorWindow::edit_project`], which is the same fan-out a timeline edit or //! a background slider takes. //! -//! Two things in this file are not the project's: the **menus** (`KSelect` has -//! no gpui equivalent, so every select opens `ui::Menu` at the pointer, and the -//! open menu's identity lives in the sidebar state) and the **transcription -//! flow** on the Captions tab, which drives [`crate::transcription`] -- the -//! in-process port of the Tauri binary's caption commands -- rather than -//! invoking them over IPC. The chosen model/language persist in the shared -//! store's `gpui` section, this app's stand-in for the webview's -//! `localStorage` keys. - use std::{ collections::HashSet, sync::{LazyLock, Mutex}, @@ -36,8 +27,8 @@ use cap_project::{ KeyboardData, KeyboardSettings, ProjectConfiguration, ShadowConfiguration, StereoMode, }; use gpui::{ - AnyElement, Context, EntityId, FontWeight, Hsla, InteractiveElement, IntoElement, - ParentElement, SharedString, StatefulInteractiveElement, Styled, Window, div, + AnyElement, Bounds, Context, EntityId, FontWeight, Hsla, InteractiveElement, IntoElement, + ParentElement, Pixels, SharedString, StatefulInteractiveElement, Styled, Window, div, prelude::FluentBuilder, px, relative, svg, }; use serde_json::Value; @@ -566,11 +557,9 @@ fn with_keyboard_settings( // Menus // --------------------------------------------------------------------------- -/// Every `KSelect` in the sidebar. `ui::Menu` draws at the pointer, so one -/// open-menu slot on the sidebar state serves all of them -- the settings -/// window's `Menu.popup()` stand-in, transcribed. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum SidebarMenu { + BackgroundCornerStyle, CameraBlur, CameraShape, CameraCornerStyle, @@ -596,6 +585,7 @@ pub enum SidebarMenu { TextAnimationIn(usize), TextAnimationOut(usize), Camera3DBlurMode(usize), + Camera3DEasing(usize), } pub struct OpenMenu { @@ -610,6 +600,12 @@ impl EditorWindow { let captions = caption_settings(project); let keyboard = keyboard_settings(project); match kind { + SidebarMenu::BackgroundCornerStyle => CORNER_STYLES + .iter() + .map(|(style, label)| { + ui::MenuItem::new(*label, *style == project.background.rounding_type) + }) + .collect(), SidebarMenu::CameraBlur => CAMERA_BLUR_MODES .iter() .map(|(mode, label)| { @@ -695,14 +691,15 @@ impl EditorWindow { | SidebarMenu::TextWeight(index) | SidebarMenu::TextAnimationIn(index) | SidebarMenu::TextAnimationOut(index) - | SidebarMenu::Camera3DBlurMode(index) => self.panel_menu_items(kind, index), + | SidebarMenu::Camera3DBlurMode(index) + | SidebarMenu::Camera3DEasing(index) => self.panel_menu_items(kind, index), } } pub(crate) fn open_sidebar_menu( &mut self, kind: SidebarMenu, - origin: gpui::Point, + trigger_bounds: Bounds, window: &mut Window, cx: &mut Context, ) { @@ -715,7 +712,7 @@ impl EditorWindow { let items = self.sidebar_menu_items(kind); self.sidebar.menu = Some(OpenMenu { kind, - state: ui::MenuState::new(origin, &items), + state: ui::MenuState::anchored(trigger_bounds, &items), }); cx.notify(); } @@ -777,6 +774,19 @@ impl EditorWindow { ) { self.sidebar.menu = None; match kind { + SidebarMenu::BackgroundCornerStyle => { + let Some((style, _)) = CORNER_STYLES.get(index) else { + return; + }; + let style = *style; + self.edit_project("rounding-type", window, cx, move |project| { + if project.background.rounding_type == style { + return false; + } + project.background.rounding_type = style; + true + }); + } SidebarMenu::CameraBlur => { let Some((mode, _)) = CAMERA_BLUR_MODES.get(index) else { return; @@ -932,7 +942,8 @@ impl EditorWindow { | SidebarMenu::TextWeight(segment) | SidebarMenu::TextAnimationIn(segment) | SidebarMenu::TextAnimationOut(segment) - | SidebarMenu::Camera3DBlurMode(segment) => { + | SidebarMenu::Camera3DBlurMode(segment) + | SidebarMenu::Camera3DEasing(segment) => { self.choose_panel_menu(kind, segment, index, window, cx) } } @@ -1154,7 +1165,6 @@ impl EditorWindow { .into_any_element() } - /// A `KSelect.Trigger` -- `ui::Select` opening `ui::Menu` at the pointer. pub(crate) fn menu_select( &self, kind: SidebarMenu, @@ -1164,10 +1174,9 @@ impl EditorWindow { ) -> AnyElement { ui::Select::plain(&self.theme, id, label) .stretch_label() - .on_click( - cx.listener(move |this, event: &gpui::ClickEvent, window, cx| { - let origin = event.position(); - this.open_sidebar_menu(kind, origin, window, cx); + .on_open( + cx.listener(move |this, bounds: &Bounds, window, cx| { + this.open_sidebar_menu(kind, *bounds, window, cx); }), ) .into_any_element() @@ -1184,10 +1193,9 @@ impl EditorWindow { ) -> AnyElement { ui::Select::plain(&self.theme, id, label) .stretch_label() - .on_click( - cx.listener(move |this, event: &gpui::ClickEvent, window, cx| { - let origin = event.position(); - this.open_sidebar_menu(kind, origin, window, cx); + .on_open( + cx.listener(move |this, bounds: &Bounds, window, cx| { + this.open_sidebar_menu(kind, *bounds, window, cx); }), ) .into_any_element() @@ -2262,10 +2270,13 @@ impl EditorWindow { .tooltip({ let model_name = SharedString::new_static(model.model_name); move |_window, cx| ui::Tooltip::new(&theme, model_name.clone()).view(cx) - }) - .on_click(cx.listener(|this, event: &gpui::ClickEvent, window, cx| { - this.open_sidebar_menu(SidebarMenu::CaptionModel, event.position(), window, cx); - })); + }); + let model_trigger = ui::Menu::trigger( + model_trigger, + cx.listener(|this, bounds: &Bounds, window, cx| { + this.open_sidebar_menu(SidebarMenu::CaptionModel, *bounds, window, cx); + }), + ); // The download / generate column (`CaptionsTab.tsx:936-1032`). let action = if model_downloaded { @@ -2667,17 +2678,12 @@ impl EditorWindow { ), ); + // An extra flex ancestor here repeats intrinsic layout while scrolling. ui::Field::plain(&theme, "Captions") .icon("icons/message-bubble.svg") .badge("Beta") - .child( - div() - .flex() - .flex_col() - .gap(px(24.)) - .child(transcription) - .child(style), - ) + .child(transcription) + .child(style.mt(px(8.))) .into_any_element() } diff --git a/apps/desktop-gpui/src/editor_window.rs b/apps/desktop-gpui/src/editor_window.rs index f05119b6985..465f718f77b 100644 --- a/apps/desktop-gpui/src/editor_window.rs +++ b/apps/desktop-gpui/src/editor_window.rs @@ -59,7 +59,7 @@ use core_foundation::base::TCFType; #[cfg(target_os = "macos")] use core_video::pixel_buffer::{CVPixelBuffer, CVPixelBufferRef}; use gpui::{ - AppContext as _, Context, Entity, FocusHandle, FontWeight, Hsla, InteractiveElement, + AppContext as _, Bounds, Context, Entity, FocusHandle, FontWeight, Hsla, InteractiveElement, IntoElement, MouseButton, MouseDownEvent, MouseMoveEvent, MouseUpEvent, ParentElement, Pixels, Point, Render, RenderImage, SharedString, StatefulInteractiveElement as _, StyleRefinement, Styled, StyledImage as _, Subscription, WeakEntity, Window, div, point, prelude::FluentBuilder, @@ -1868,6 +1868,7 @@ impl EditorWindow { /// the waveforms arrive separately and later, so whatever has landed is /// carried across. fn rebuild_timeline(&mut self) { + dismiss_indexed_sidebar_menu(&mut self.sidebar.menu); let mic = std::mem::take(&mut self.timeline.mic_waveforms); let system = std::mem::take(&mut self.timeline.system_waveforms); self.timeline = TimelineModel::build_with_lanes( @@ -2472,7 +2473,11 @@ impl EditorWindow { window: &mut Window, cx: &mut Context, ) { - if !is_playback_shortcut(&event.keystroke, ui::text_input_has_focus(window, cx)) { + if !is_playback_shortcut( + &event.keystroke, + ui::text_input_has_focus(window, cx), + self.sidebar.menu.is_some() || self.toolbar_menu.is_some(), + ) { return; } // Focused GPUI buttons arm a second click on key-up unless Space is @@ -3062,6 +3067,7 @@ impl EditorWindow { /// `setEditorState("timeline", "selection", ...)`. pub(crate) fn set_selection(&mut self, selection: Option, cx: &mut Context) { if self.selection != selection { + dismiss_indexed_sidebar_menu(&mut self.sidebar.menu); self.selection = selection; cx.notify(); } @@ -5363,6 +5369,7 @@ impl EditorWindow { ui::EditorButton::plain(&theme, id) .left_icon(icon) .disabled(!enabled) + .tooltip(&theme, if undo { "Undo" } else { "Redo" }) .on_click(cx.listener(move |this, _, window, cx| { if !(this.history.can_undo() || this.history.can_redo() || this.selection.is_some()) { @@ -5424,7 +5431,7 @@ impl EditorWindow { fn open_toolbar_menu( &mut self, kind: ToolbarMenu, - origin: gpui::Point, + trigger_bounds: Bounds, window: &mut Window, cx: &mut Context, ) { @@ -5434,7 +5441,7 @@ impl EditorWindow { let items = self.toolbar_menu_items(kind); self.toolbar_menu = Some(OpenToolbarMenu { kind, - state: ui::MenuState::new(origin, &items), + state: ui::MenuState::anchored(trigger_bounds, &items), }); cx.notify(); } @@ -7314,8 +7321,8 @@ impl EditorWindow { .disabled(!self.project_ready()) .right_icon("icons/chevron-down.svg") .pressed(self.presets_menu.is_some()) - .on_click(cx.listener(|this, event: &gpui::ClickEvent, window, cx| { - this.open_presets_menu(event.position(), window, cx); + .on_open(cx.listener(|this, bounds: &Bounds, window, cx| { + this.open_presets_menu(bounds.bottom_left(), window, cx); })), ), ) @@ -7522,10 +7529,10 @@ impl EditorWindow { .as_ref() .is_some_and(|menu| menu.kind == ToolbarMenu::AspectRatio), ) - .on_click(cx.listener(|this, event: &gpui::ClickEvent, window, cx| { + .on_open(cx.listener(|this, bounds: &Bounds, window, cx| { this.open_toolbar_menu( ToolbarMenu::AspectRatio, - event.position(), + *bounds, window, cx, ); @@ -7537,6 +7544,7 @@ impl EditorWindow { ui::EditorButton::plain(&theme, "crop") .left_icon("icons/crop.svg") .label("Crop") + .tooltip(&theme, "Crop Video") .pressed(self.crop.is_some()) .on_click(cx.listener(|this, _, window, cx| { this.open_crop(window, cx); @@ -7561,10 +7569,10 @@ impl EditorWindow { .child( ui::Select::plain(&theme, "preview-quality", self.preview_quality.label()) .stretch_label() - .on_click(cx.listener(|this, event: &gpui::ClickEvent, window, cx| { + .on_open(cx.listener(|this, bounds: &Bounds, window, cx| { this.open_toolbar_menu( ToolbarMenu::PreviewQuality, - event.position(), + *bounds, window, cx, ); @@ -7691,6 +7699,16 @@ impl EditorWindow { cx: &mut Context, ) -> impl IntoElement { let theme = self.theme; + let (label, key) = if factor > 1. { + ("Zoom out", "-") + } else { + ("Zoom in", "+") + }; + let modifier = if cfg!(target_os = "macos") { + "meta" + } else { + "ctrl" + }; div() .id(id) .flex() @@ -7698,6 +7716,12 @@ impl EditorWindow { .justify_center() .cursor_pointer() .hover(|this| this.opacity(0.7)) + .tooltip_show_delay(ui::TOOLTIP_SHOW_DELAY) + .tooltip(move |_window, cx| { + ui::Tooltip::new(&theme, label) + .keys([modifier, key]) + .view(cx) + }) .child( svg() .path(icon) @@ -7815,18 +7839,29 @@ impl EditorWindow { // `rounded-full border border-gray-300 bg-gray-3 size-9` // with `hover:bg-gray-4` -- [`ui::IconButton`]. .child( - ui::IconButton::new("transport-play", icon) - .size(px(36.)) - .icon_size(px(12.)) - .color(Hsla::from(theme.gray_12)) - .filled( - Hsla::from(theme.gray_3), - Some(Hsla::from(theme.gray_5)), - ) - .hover_bg(Hsla::from(theme.gray_4)) - .on_click(cx.listener(|this, _, window, cx| { - this.toggle_play(window, cx); - })), + div() + .id("transport-play-tooltip") + .flex() + .tooltip_show_delay(ui::TOOLTIP_SHOW_DELAY) + .tooltip(move |_window, cx| { + ui::Tooltip::new(&theme, "Play/Pause video") + .keys(["Space"]) + .view(cx) + }) + .child( + ui::IconButton::new("transport-play", icon) + .size(px(36.)) + .icon_size(px(12.)) + .color(Hsla::from(theme.gray_12)) + .filled( + Hsla::from(theme.gray_3), + Some(Hsla::from(theme.gray_5)), + ) + .hover_bg(Hsla::from(theme.gray_4)) + .on_click(cx.listener(|this, _, window, cx| { + this.toggle_play(window, cx); + })), + ), ) .child( div() @@ -7866,6 +7901,12 @@ impl EditorWindow { div() .id("transport-split") .tab_index(0) + .tooltip_show_delay(ui::TOOLTIP_SHOW_DELAY) + .tooltip(move |_window, cx| { + ui::Tooltip::new(&theme, "Toggle Split") + .keys(["S"]) + .view(cx) + }) .flex() .flex_row() .items_center() @@ -8583,8 +8624,43 @@ fn playhead_extrapolation(playing: bool, epoch_has_sample: bool, since_last_samp since_last_sample.clamp(0.0, MAX_PLAYHEAD_EXTRAPOLATION) } -fn is_playback_shortcut(keystroke: &gpui::Keystroke, text_input_focused: bool) -> bool { - keystroke.key == "space" && !keystroke.modifiers.modified() && !text_input_focused +fn dismiss_indexed_sidebar_menu(menu: &mut Option) { + use crate::editor_tabs::SidebarMenu; + + let indexed = menu.as_ref().is_some_and(|menu| match menu.kind { + SidebarMenu::TextFontFamily(_) + | SidebarMenu::TextWeight(_) + | SidebarMenu::TextAnimationIn(_) + | SidebarMenu::TextAnimationOut(_) + | SidebarMenu::Camera3DBlurMode(_) + | SidebarMenu::Camera3DEasing(_) => true, + SidebarMenu::BackgroundCornerStyle + | SidebarMenu::CameraBlur + | SidebarMenu::CameraShape + | SidebarMenu::CameraCornerStyle + | SidebarMenu::AudioStereo + | SidebarMenu::CaptionModel + | SidebarMenu::CaptionLanguage + | SidebarMenu::CaptionFont + | SidebarMenu::CaptionHighlightStyle + | SidebarMenu::CaptionPosition + | SidebarMenu::CaptionAnimation + | SidebarMenu::CaptionWeight + | SidebarMenu::KeyboardFont + | SidebarMenu::KeyboardPosition + | SidebarMenu::KeyboardWeight => false, + }); + if indexed { + *menu = None; + } +} + +fn is_playback_shortcut( + keystroke: &gpui::Keystroke, + text_input_focused: bool, + menu_open: bool, +) -> bool { + keystroke.key == "space" && !keystroke.modifiers.modified() && !text_input_focused && !menu_open } impl Render for EditorWindow { @@ -8631,6 +8707,18 @@ impl Render for EditorWindow { .child(self.render_export_page(window, cx)); } + let timeline_drag_cursor = self + .drag + .map(|drag| match drag.kind { + DragKind::Move { .. } => gpui::CursorStyle::ClosedHand, + DragKind::TrimStart { .. } + | DragKind::TrimEnd { .. } + | DragKind::ClipTrimStart { .. } + | DragKind::ClipTrimEnd { .. } => gpui::CursorStyle::ResizeLeftRight, + DragKind::CreateZoom { .. } => gpui::CursorStyle::Arrow, + }) + .or((self.scrub == Some(Scrub::Ruler)).then_some(gpui::CursorStyle::ResizeLeftRight)); + div() .size_full() .flex() @@ -8858,6 +8946,13 @@ impl Render for EditorWindow { // over everything -- the same shape the settings window's sliders // use, because gpui has no pointer capture and a 96px row would // otherwise lose the drag the moment the pointer left it. + .children(timeline_drag_cursor.map(|cursor| { + div() + .id("timeline-active-drag-cursor") + .absolute() + .inset_0() + .cursor(cursor) + })) .children(self.timeline_resize.is_some().then(|| { ui::Slider::drag_layer( "timeline-height-drag", @@ -8876,6 +8971,7 @@ impl Render for EditorWindow { cx.notify(); }), ) + .cursor(gpui::CursorStyle::ResizeRow) })) .children(self.zoom_slider_drag.then(|| { ui::Slider::drag_layer( @@ -8930,12 +9026,20 @@ impl Render for EditorWindow { this.pad_mouse_up(cx); }), ) + .cursor(gpui::CursorStyle::Crosshair) })) // The canvas display drag: the source installs `mousemove` / // `mouseup` on `window` for the duration (`CEO.tsx:611-618`), so // a drag that leaves the letterboxed rect keeps tracking and the // release closes the undo bracket wherever it happens. - .children(self.canvas_drag.is_some().then(|| { + .children(self.canvas_drag.as_ref().map(|drag| { + let cursor = drag.resize.as_ref().map_or(gpui::CursorStyle::ClosedHand, |resize| { + if resize.dir_x == resize.dir_y { + gpui::CursorStyle::ResizeUpLeftDownRight + } else { + gpui::CursorStyle::ResizeUpRightDownLeft + } + }); ui::Slider::drag_layer( "canvas-display-drag", cx.listener(|this, event: &MouseMoveEvent, window, cx| { @@ -8945,6 +9049,7 @@ impl Render for EditorWindow { this.canvas_mouse_up(window, cx); }), ) + .cursor(cursor) })) // The open `KSelect` menu, painted last of all so it is over the // sidebar and the drag layers alike. @@ -8968,8 +9073,8 @@ impl Render for EditorWindow { .children( self.crop .as_ref() - .is_some_and(|state| state.drag.is_some()) - .then(|| { + .and_then(|state| state.drag.as_ref()) + .map(|drag| { ui::Slider::drag_layer( "crop-drag", cx.listener(|this, event: &MouseMoveEvent, window, cx| { @@ -8979,6 +9084,7 @@ impl Render for EditorWindow { this.crop_mouse_up(window, cx); }), ) + .cursor(drag.cursor()) }), ) } @@ -9142,6 +9248,132 @@ fn hex_to_color(rgba: [u8; 4]) -> cap_project::Color { mod tests { use super::*; + fn open_sidebar_menu_for_test( + kind: crate::editor_tabs::SidebarMenu, + ) -> Option { + Some(crate::editor_tabs::OpenMenu { + kind, + state: ui::MenuState::new( + point(px(12.), px(24.)), + &[ + ui::MenuItem::new("First", true), + ui::MenuItem::new("Second", false), + ], + ), + }) + } + + #[test] + fn indexed_sidebar_menus_are_dismissed_when_their_target_can_change() { + use crate::editor_tabs::SidebarMenu; + + for kind in [ + SidebarMenu::TextFontFamily(0), + SidebarMenu::TextWeight(1), + SidebarMenu::TextAnimationIn(2), + SidebarMenu::TextAnimationOut(3), + SidebarMenu::Camera3DBlurMode(4), + SidebarMenu::Camera3DEasing(5), + ] { + let mut menu = open_sidebar_menu_for_test(kind); + dismiss_indexed_sidebar_menu(&mut menu); + assert!(menu.is_none(), "{kind:?}"); + } + } + + #[test] + fn indexed_sidebar_menu_invalidation_preserves_global_menu_navigation() { + use crate::editor_tabs::SidebarMenu; + + for kind in [ + SidebarMenu::BackgroundCornerStyle, + SidebarMenu::CameraBlur, + SidebarMenu::CameraShape, + SidebarMenu::CameraCornerStyle, + SidebarMenu::AudioStereo, + SidebarMenu::CaptionModel, + SidebarMenu::CaptionLanguage, + SidebarMenu::CaptionFont, + SidebarMenu::CaptionHighlightStyle, + SidebarMenu::CaptionPosition, + SidebarMenu::CaptionAnimation, + SidebarMenu::CaptionWeight, + SidebarMenu::KeyboardFont, + SidebarMenu::KeyboardPosition, + SidebarMenu::KeyboardWeight, + ] { + let mut menu = open_sidebar_menu_for_test(kind); + let state = &mut menu.as_mut().unwrap().state; + assert_eq!(state.on_key("down"), ui::MenuKey::Moved); + let expected = state.clone(); + dismiss_indexed_sidebar_menu(&mut menu); + let remaining = menu.as_mut().unwrap(); + assert_eq!(remaining.kind, kind); + assert_eq!(remaining.state, expected); + assert_eq!(remaining.state.on_key("enter"), ui::MenuKey::Commit(1)); + } + } + + #[test] + fn indexed_sidebar_menu_cannot_retarget_after_delete_or_history_change() { + use crate::editor_tabs::SidebarMenu; + + let mut project = ProjectConfiguration { + timeline: Some(TimelineConfiguration { + segments: Vec::new(), + transitions: Vec::new(), + zoom_segments: Vec::new(), + scene_segments: Vec::new(), + mask_segments: Vec::new(), + text_segments: Vec::new(), + caption_segments: Vec::new(), + keyboard_segments: Vec::new(), + audio_segments: Vec::new(), + camera3d_segments: vec![ + edits::default_camera3d_segment(0.0, 2.0), + edits::default_camera3d_segment(2.0, 4.0), + ], + }), + ..Default::default() + }; + let mut history = ProjectHistory::new(project.clone()); + let mut menu = open_sidebar_menu_for_test(SidebarMenu::Camera3DEasing(0)); + assert_eq!( + menu.as_mut().unwrap().state.on_key("backspace"), + ui::MenuKey::Ignored + ); + assert!(edits::delete_segments( + project.timeline.as_mut().unwrap(), + TrackKind::ThreeD, + &[0], + )); + assert_eq!( + project.timeline.as_ref().unwrap().camera3d_segments[0].start, + 2.0 + ); + dismiss_indexed_sidebar_menu(&mut menu); + assert!(menu.is_none()); + history.record(&project); + + menu = open_sidebar_menu_for_test(SidebarMenu::Camera3DEasing(0)); + project = history.undo().unwrap().clone(); + assert_eq!( + project.timeline.as_ref().unwrap().camera3d_segments[0].start, + 0.0 + ); + dismiss_indexed_sidebar_menu(&mut menu); + assert!(menu.is_none()); + + menu = open_sidebar_menu_for_test(SidebarMenu::Camera3DBlurMode(1)); + project = history.redo().unwrap().clone(); + assert_eq!( + project.timeline.as_ref().unwrap().camera3d_segments.len(), + 1 + ); + dismiss_indexed_sidebar_menu(&mut menu); + assert!(menu.is_none()); + } + #[test] fn failed_predelete_save_keeps_the_pending_edit_for_retry() { let root = std::env::temp_dir().join(format!( @@ -9176,10 +9408,11 @@ mod tests { } #[test] - fn playback_shortcut_is_reserved_for_bare_space_outside_text_fields() { + fn playback_shortcut_is_reserved_for_bare_space_outside_text_fields_and_menus() { let space = gpui::Keystroke::parse("space").unwrap(); - assert!(is_playback_shortcut(&space, false)); - assert!(!is_playback_shortcut(&space, true)); + assert!(is_playback_shortcut(&space, false, false)); + assert!(!is_playback_shortcut(&space, true, false)); + assert!(!is_playback_shortcut(&space, false, true)); for key in [ "enter", "s", @@ -9189,7 +9422,7 @@ mod tests { "alt-space", ] { let keystroke = gpui::Keystroke::parse(key).unwrap(); - assert!(!is_playback_shortcut(&keystroke, false), "{key}"); + assert!(!is_playback_shortcut(&keystroke, false, false), "{key}"); } } diff --git a/apps/desktop-gpui/src/screenshot_annotations.rs b/apps/desktop-gpui/src/screenshot_annotations.rs index 79a81909734..22865294400 100644 --- a/apps/desktop-gpui/src/screenshot_annotations.rs +++ b/apps/desktop-gpui/src/screenshot_annotations.rs @@ -2964,6 +2964,7 @@ impl ScreenshotEditorWindow { .child( div() .id("screenshot-annotation-color-backdrop") + .occlude() .absolute() .top_0() .left_0() @@ -2975,6 +2976,7 @@ impl ScreenshotEditorWindow { ) .child( div() + .occlude() .absolute() .left(px(left)) .top(px(top)) diff --git a/apps/desktop-gpui/src/screenshot_crop.rs b/apps/desktop-gpui/src/screenshot_crop.rs index 21e6f1c7125..a789563dcf2 100644 --- a/apps/desktop-gpui/src/screenshot_crop.rs +++ b/apps/desktop-gpui/src/screenshot_crop.rs @@ -664,6 +664,7 @@ impl ScreenshotEditorWindow { .child( div() .id("screenshot-crop-backdrop") + .occlude() .absolute() .inset_0() .bg(gpui::hsla(0., 0., 0., 0.8)) diff --git a/apps/desktop-gpui/src/screenshot_editor.rs b/apps/desktop-gpui/src/screenshot_editor.rs index 0b8f6db0811..f059bf6e745 100644 --- a/apps/desktop-gpui/src/screenshot_editor.rs +++ b/apps/desktop-gpui/src/screenshot_editor.rs @@ -3909,6 +3909,7 @@ impl ScreenshotEditorWindow { .child( div() .id("screenshot-popover-backdrop") + .occlude() .absolute() .top_0() .left_0() @@ -3920,6 +3921,7 @@ impl ScreenshotEditorWindow { ) .child( div() + .occlude() .absolute() .left(px(left)) .top(px(top)) @@ -4505,6 +4507,7 @@ fn kbd_tooltip( .id(gpui::SharedString::from(format!("{label}-tooltip"))) .flex_shrink_0() .child(child) + .tooltip_show_delay(ui::TOOLTIP_SHOW_DELAY) .tooltip(move |_window, cx| ui::Tooltip::new(&theme, label).keys(keys).view(cx)) } @@ -4538,6 +4541,7 @@ fn tool_button( } else { theme.gray_11 })) + .tooltip_show_delay(ui::TOOLTIP_SHOW_DELAY) .tooltip(move |_window, cx| { ui::Tooltip::new(&theme, label.clone()) .keys([shortcut.clone()]) diff --git a/apps/desktop-gpui/src/ui/button.rs b/apps/desktop-gpui/src/ui/button.rs index ec8f750a53f..048cadefad7 100644 --- a/apps/desktop-gpui/src/ui/button.rs +++ b/apps/desktop-gpui/src/ui/button.rs @@ -8,6 +8,7 @@ use gpui::{ prelude::FluentBuilder, px, svg, }; +use super::menu::OpenHandler; use crate::theme::Theme; /// The click handler every component takes. `cx.listener(..)` produces exactly @@ -130,6 +131,7 @@ pub struct Button { /// than the settings surface's repaint. dim_disabled: bool, on_click: Option, + on_open: Option, } impl Button { @@ -157,6 +159,7 @@ impl Button { height: None, dim_disabled: false, on_click: None, + on_open: None, } } @@ -318,6 +321,14 @@ impl Button { self.on_click = Some(Box::new(handler)); self } + + pub fn on_open( + mut self, + handler: impl Fn(&gpui::Bounds, &mut Window, &mut App) + 'static, + ) -> Self { + self.on_open = Some(Box::new(handler)); + self + } } /// The Radix fills for one variant, before any material remap. @@ -414,6 +425,7 @@ impl RenderOnce for Button { full_width, height, on_click, + on_open, } = self; let icon_color = paint.text; @@ -478,6 +490,7 @@ impl RenderOnce for Button { .when_some(on_click.filter(|_| !disabled), |this, handler| { this.on_click(move |event, window, cx| handler(event, window, cx)) }) + .when_some(on_open.filter(|_| !disabled), crate::ui::Menu::trigger) } } diff --git a/apps/desktop-gpui/src/ui/editor_button.rs b/apps/desktop-gpui/src/ui/editor_button.rs index ff9224554a9..8881f737c20 100644 --- a/apps/desktop-gpui/src/ui/editor_button.rs +++ b/apps/desktop-gpui/src/ui/editor_button.rs @@ -14,11 +14,6 @@ //! //! Disabled is `opacity-50 text-gray-11` on both. //! -//! The polymorphic `as={KSelect.Trigger}` half has no gpui equivalent -- there -//! is no element to become -- so a call site that needs this button to open a -//! menu opens one from its own `on_click`, which is what `ui::Select` already -//! does. - use gpui::{ App, ClickEvent, ElementId, Hsla, InteractiveElement, IntoElement, ParentElement, Pixels, RenderOnce, SharedString, StatefulInteractiveElement, Styled, Window, div, @@ -27,7 +22,7 @@ use gpui::{ use crate::theme::Theme; -use super::{ClickHandler, Tooltip}; +use super::{ClickHandler, Tooltip, menu::OpenHandler}; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum EditorButtonVariant { @@ -59,6 +54,7 @@ pub struct EditorButton { pressed_text: Hsla, tooltip: Option<(Theme, SharedString)>, on_click: Option, + on_open: Option, } impl EditorButton { @@ -82,6 +78,7 @@ impl EditorButton { pressed_text: Hsla::from(theme.gray_12), tooltip: None, on_click: None, + on_open: None, } } @@ -146,6 +143,14 @@ impl EditorButton { self.on_click = Some(Box::new(handler)); self } + + pub fn on_open( + mut self, + handler: impl Fn(&gpui::Bounds, &mut Window, &mut App) + 'static, + ) -> Self { + self.on_open = Some(Box::new(handler)); + self + } } impl RenderOnce for EditorButton { @@ -169,6 +174,7 @@ impl RenderOnce for EditorButton { pressed_text, tooltip, on_click, + on_open, } = self; let foreground = if disabled { @@ -216,10 +222,12 @@ impl RenderOnce for EditorButton { .text_color(foreground) })) .when_some(tooltip, |this, (theme, label)| { - this.tooltip(move |_window, cx| Tooltip::new(&theme, label.clone()).view(cx)) + this.tooltip_show_delay(crate::ui::TOOLTIP_SHOW_DELAY) + .tooltip(move |_window, cx| Tooltip::new(&theme, label.clone()).view(cx)) }) .when_some(on_click.filter(|_| !disabled), |this, handler| { this.on_click(move |event, window, cx| handler(event, window, cx)) }) + .when_some(on_open.filter(|_| !disabled), crate::ui::Menu::trigger) } } diff --git a/apps/desktop-gpui/src/ui/menu.rs b/apps/desktop-gpui/src/ui/menu.rs index c18c7552d5f..f19ed5b21e3 100644 --- a/apps/desktop-gpui/src/ui/menu.rs +++ b/apps/desktop-gpui/src/ui/menu.rs @@ -10,14 +10,18 @@ //! The state machine is a plain struct so it can be tested without a window; //! [`Menu`] is the element that draws it. +use std::{cell::Cell, rc::Rc}; + use gpui::{ - App, ClickEvent, ElementId, Hsla, InteractiveElement, IntoElement, ParentElement, Pixels, - Point, RenderOnce, SharedString, StatefulInteractiveElement, Styled, Window, div, - prelude::FluentBuilder, px, svg, + Anchor, App, Bounds, ClickEvent, ElementId, Hsla, InteractiveElement, IntoElement, + ParentElement, Pixels, Point, RenderOnce, SharedString, Size, StatefulInteractiveElement, + Styled, Window, div, point, prelude::FluentBuilder, px, svg, }; use crate::theme::Theme; +pub(crate) type OpenHandler = Box, &mut Window, &mut App) + 'static>; + /// One row: a label and whether it is the value currently in force. #[derive(Debug, Clone, PartialEq, Eq)] pub struct MenuItem { @@ -52,6 +56,7 @@ pub enum MenuKey { #[derive(Debug, Clone, PartialEq)] pub struct MenuState { pub origin: Point, + pub trigger_bounds: Option>, pub len: usize, pub highlighted: Option, /// Whether the highlight is *drawn*. A menu opened by pointer shows the @@ -69,12 +74,20 @@ impl MenuState { pub fn new(origin: Point, items: &[MenuItem]) -> Self { Self { origin, + trigger_bounds: None, len: items.len(), highlighted: items.iter().position(|item| item.checked), highlight_visible: false, } } + pub fn anchored(trigger_bounds: Bounds, items: &[MenuItem]) -> Self { + Self { + trigger_bounds: Some(trigger_bounds), + ..Self::new(trigger_bounds.bottom_left(), items) + } + } + /// The index to paint highlighted, if any. pub fn visible_highlight(&self) -> Option { self.highlighted.filter(|_| self.highlight_visible) @@ -139,6 +152,7 @@ pub struct Menu { id: ElementId, items: Vec, origin: Point, + trigger_bounds: Option>, highlighted: Option, min_width: Pixels, max_height: Pixels, @@ -166,6 +180,7 @@ impl Menu { id: id.into(), items, origin: state.origin, + trigger_bounds: state.trigger_bounds, highlighted: state.visible_highlight(), min_width: px(180.), max_height: px(320.), @@ -200,6 +215,30 @@ impl Menu { self } + pub fn trigger( + element: gpui::Stateful, + handler: impl Fn(&Bounds, &mut Window, &mut App) + 'static, + ) -> gpui::Stateful { + let bounds = Rc::new(Cell::new(None)); + let measured_bounds = bounds.clone(); + element + .tab_index(0) + .relative() + .on_click(move |_, window, cx| { + if let Some(bounds) = bounds.get() { + handler(&bounds, window, cx); + } + }) + .child( + gpui::canvas( + move |bounds, _, _| measured_bounds.set(Some(bounds)), + |_, _, _, _| {}, + ) + .absolute() + .inset_0(), + ) + } + pub fn on_select(mut self, handler: impl Fn(&usize, &mut Window, &mut App) + 'static) -> Self { self.on_select = Some(Box::new(handler)); self @@ -214,12 +253,39 @@ impl Menu { } } +fn anchored_placement( + trigger: Bounds, + item_count: usize, + max_height: Pixels, + viewport: Size, +) -> (Point, Anchor, Pixels) { + let gap = px(4.); + let margin = px(12.); + let below = (viewport.height - margin - trigger.bottom() - gap).max(px(0.)); + let above = (trigger.top() - gap - margin).max(px(0.)); + let height = (px(item_count as f32 * 24. + 10.)).min(max_height); + if height <= below || below >= above { + ( + trigger.bottom_left() + point(px(0.), gap), + Anchor::TopLeft, + max_height.min(below), + ) + } else { + ( + trigger.origin - point(px(0.), gap), + Anchor::BottomLeft, + max_height.min(above), + ) + } +} + impl RenderOnce for Menu { fn render(self, window: &mut Window, _cx: &mut App) -> impl IntoElement { let Menu { id, items, origin, + trigger_bounds, highlighted, min_width, max_height, @@ -239,6 +305,11 @@ impl RenderOnce for Menu { let viewport = window.viewport_size(); let max_width = (viewport.width - px(24.)).max(px(0.)); let max_height = max_height.min((viewport.height - px(24.)).max(px(0.))); + let min_width = trigger_bounds.map_or(min_width, |bounds| min_width.max(bounds.size.width)); + let (origin, anchor, max_height) = trigger_bounds + .map_or((origin, Anchor::TopLeft, max_height), |bounds| { + anchored_placement(bounds, items.len(), max_height, viewport) + }); div() .absolute() @@ -249,6 +320,7 @@ impl RenderOnce for Menu { // Click-away dismiss, the way a native menu closes. div() .id(SharedString::from(format!("{prefix}-backdrop"))) + .occlude() .absolute() .top_0() .left_0() @@ -260,6 +332,7 @@ impl RenderOnce for Menu { .child( gpui::anchored() .position(origin) + .anchor(anchor) .snap_to_window_with_margin(px(12.)) .child( div() @@ -327,6 +400,45 @@ mod tests { MenuState::new(point(px(0.), px(0.)), &items(checked)) } + #[test] + fn a_select_menu_uses_the_trigger_bounds_and_current_value() { + let bounds = Bounds::new(point(px(100.), px(80.)), gpui::size(px(160.), px(36.))); + let menu = MenuState::anchored(bounds, &items(Some(2))); + assert_eq!(menu.trigger_bounds, Some(bounds)); + assert_eq!(menu.origin, bounds.bottom_left()); + assert_eq!(menu.highlighted, Some(2)); + } + + #[test] + fn a_select_menu_opens_below_the_button_when_it_fits() { + let bounds = Bounds::new(point(px(100.), px(80.)), gpui::size(px(160.), px(36.))); + let (origin, anchor, height) = + anchored_placement(bounds, 4, px(320.), gpui::size(px(800.), px(600.))); + assert_eq!(origin, point(px(100.), px(120.))); + assert_eq!(anchor, Anchor::TopLeft); + assert_eq!(height, px(320.)); + } + + #[test] + fn a_select_menu_flips_above_a_button_near_the_bottom() { + let bounds = Bounds::new(point(px(100.), px(540.)), gpui::size(px(160.), px(36.))); + let (origin, anchor, height) = + anchored_placement(bounds, 4, px(320.), gpui::size(px(800.), px(600.))); + assert_eq!(origin, point(px(100.), px(536.))); + assert_eq!(anchor, Anchor::BottomLeft); + assert_eq!(height, px(320.)); + } + + #[test] + fn a_long_select_menu_is_limited_to_the_larger_side_of_the_button() { + let bounds = Bounds::new(point(px(100.), px(220.)), gpui::size(px(160.), px(36.))); + let (origin, anchor, height) = + anchored_placement(bounds, 40, px(320.), gpui::size(px(800.), px(420.))); + assert_eq!(origin, point(px(100.), px(216.))); + assert_eq!(anchor, Anchor::BottomLeft); + assert_eq!(height, px(204.)); + } + #[test] fn a_menu_opens_on_the_value_it_currently_holds() { assert_eq!(state(Some(2)).highlighted, Some(2)); diff --git a/apps/desktop-gpui/src/ui/radio_cards.rs b/apps/desktop-gpui/src/ui/radio_cards.rs index d6c3d31cbb4..f97047eacb6 100644 --- a/apps/desktop-gpui/src/ui/radio_cards.rs +++ b/apps/desktop-gpui/src/ui/radio_cards.rs @@ -16,7 +16,7 @@ use gpui::{ App, ElementId, FontWeight, Hsla, InteractiveElement, IntoElement, ParentElement, RenderOnce, - SharedString, StatefulInteractiveElement, Styled, Window, div, prelude::FluentBuilder, px, + SharedString, StatefulInteractiveElement, Styled, Window, div, prelude::FluentBuilder, px, svg, }; use crate::theme::Theme; @@ -145,10 +145,20 @@ impl RenderOnce for RadioCards { .mt(px(4.)) .size(px(16.)) .flex_none() + .flex() + .items_center() + .justify_center() .rounded_full() .border_1() .border_color(if checked { dot_fill } else { dot_border }) - .when(checked, |this| this.bg(dot_fill)), + .when(checked, |this| { + this.bg(dot_fill).child( + svg() + .path("icons/check.svg") + .size(px(12.)) + .text_color(Hsla::from(gpui::rgb(0xffffff))), + ) + }), ) .child( div() diff --git a/apps/desktop-gpui/src/ui/select.rs b/apps/desktop-gpui/src/ui/select.rs index c42f0fdb3cb..07db73057f9 100644 --- a/apps/desktop-gpui/src/ui/select.rs +++ b/apps/desktop-gpui/src/ui/select.rs @@ -7,11 +7,12 @@ //! that leaves the camera bubble's mirror button disabled). use gpui::{ - App, ClickEvent, ElementId, FontWeight, Hsla, InteractiveElement, IntoElement, ParentElement, - Pixels, RenderOnce, SharedString, StatefulInteractiveElement, Styled, Window, div, - prelude::FluentBuilder, px, svg, + App, Bounds, ClickEvent, ElementId, FontWeight, Hsla, InteractiveElement, IntoElement, + ParentElement, Pixels, RenderOnce, SharedString, StatefulInteractiveElement, Styled, Window, + div, prelude::FluentBuilder, px, svg, }; +use super::menu::OpenHandler; use crate::theme::Theme; #[derive(IntoElement)] @@ -38,6 +39,7 @@ pub struct Select { stretch: bool, disabled: bool, on_click: Option, + on_open: Option, } impl Select { @@ -68,6 +70,7 @@ impl Select { stretch: false, disabled: false, on_click: None, + on_open: None, } } @@ -117,6 +120,14 @@ impl Select { self.on_click = Some(Box::new(handler)); self } + + pub fn on_open( + mut self, + handler: impl Fn(&Bounds, &mut Window, &mut App) + 'static, + ) -> Self { + self.on_open = Some(Box::new(handler)); + self + } } impl RenderOnce for Select { @@ -139,6 +150,7 @@ impl RenderOnce for Select { stretch, disabled, on_click, + on_open, } = self; div() @@ -175,5 +187,6 @@ impl RenderOnce for Select { .when_some(on_click.filter(|_| !disabled), |this, handler| { this.on_click(move |event, window, cx| handler(event, window, cx)) }) + .when_some(on_open.filter(|_| !disabled), crate::ui::Menu::trigger) } } From dfc8c89e1e864faeabb896b06be8fb482c9f7033 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 3 Sep 2026 17:08:11 +0100 Subject: [PATCH 11/20] fix: keep Windows capture targets and window controls usable --- apps/desktop-gpui/src/devices.rs | 5 +++++ apps/desktop-gpui/src/main_window.rs | 6 ++++++ apps/desktop-gpui/src/onboarding_window.rs | 2 +- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/apps/desktop-gpui/src/devices.rs b/apps/desktop-gpui/src/devices.rs index 3d967566f02..21976bf7bff 100644 --- a/apps/desktop-gpui/src/devices.rs +++ b/apps/desktop-gpui/src/devices.rs @@ -380,6 +380,11 @@ pub fn list_window_targets() -> Vec<(WindowOption, Window)> { Window::list() .into_iter() .filter_map(|window| { + #[cfg(target_os = "windows")] + if !window.raw_handle().is_valid() || !window.raw_handle().is_on_screen() { + return None; + } + let label = window.name().filter(|name| !name.trim().is_empty())?; let app = window.owner_name()?; diff --git a/apps/desktop-gpui/src/main_window.rs b/apps/desktop-gpui/src/main_window.rs index 11bd2e252ae..80ec9cd418b 100644 --- a/apps/desktop-gpui/src/main_window.rs +++ b/apps/desktop-gpui/src/main_window.rs @@ -2665,6 +2665,9 @@ impl MainWindow { .id("microphone-warning") .absolute() .inset_0() + .when(cfg!(target_os = "windows"), |overlay| { + overlay.top(px(HEADER_HEIGHT)) + }) .rounded(px(16.)) .occlude() .flex() @@ -3383,6 +3386,9 @@ impl MainWindow { div() .absolute() .inset_0() + .when(cfg!(target_os = "windows"), |overlay| { + overlay.top(px(HEADER_HEIGHT)) + }) .rounded(px(16.)) .flex() .flex_col() diff --git a/apps/desktop-gpui/src/onboarding_window.rs b/apps/desktop-gpui/src/onboarding_window.rs index 9154b926757..36265a54172 100644 --- a/apps/desktop-gpui/src/onboarding_window.rs +++ b/apps/desktop-gpui/src/onboarding_window.rs @@ -2520,7 +2520,7 @@ impl Render for OnboardingWindow { theme, window.is_window_active(), window.is_maximized(), - false, + true, false, )); #[cfg(not(target_os = "windows"))] From 980024a773d4fa704a8beb80b7a26af8b7adc996 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 3 Sep 2026 17:08:11 +0100 Subject: [PATCH 12/20] fix: complete app handoff after the settings window closes --- apps/desktop-gpui/src/settings_pages.rs | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/apps/desktop-gpui/src/settings_pages.rs b/apps/desktop-gpui/src/settings_pages.rs index 3cbd3d9e65d..47fe8483391 100644 --- a/apps/desktop-gpui/src/settings_pages.rs +++ b/apps/desktop-gpui/src/settings_pages.rs @@ -2262,9 +2262,8 @@ impl SettingsWindow { let white = gpui::white(); let mut column = div() .absolute() - .top_0() - .left_0() - .size_full() + .inset_0() + .when(cfg!(target_os = "windows"), |overlay| overlay.top(px(36.))) // Nothing behind the takeover is clickable while it runs. .occlude() .flex() @@ -2468,26 +2467,15 @@ impl SettingsWindow { (Ok(()), ClassicTarget::DevSupervisor) => { tracing::info!("handing back to the classic app; waiting for the dev build"); self.pages.switch_back = Some(SwitchBack::WaitingForClassic); + self.pages.switch_back_ticker = None; cx.notify(); - // Occupies the ticker slot so starting or cancelling another - // sequence drops this waiter with it. - self.pages.switch_back_ticker = Some(cx.spawn(async move |this, cx| { + // A committed handoff must finish even if its settings window closes. + cx.spawn(async move |this, cx| { let started = std::time::Instant::now(); loop { cx.background_executor() .timer(Duration::from_millis(500)) .await; - let waiting = this - .update(cx, |this, _| { - matches!( - this.pages.switch_back, - Some(SwitchBack::WaitingForClassic) - ) - }) - .unwrap_or(false); - if !waiting { - break; - } if !store::classic_pending_path().exists() { tracing::info!("classic app is up; quitting"); cx.update(quit_after_flushing_editors); @@ -2506,7 +2494,8 @@ impl SettingsWindow { break; } } - })); + }) + .detach(); } (Err(message), _) => { tracing::error!("{message}"); From 6a555698a09fd06bc37f85183c86d8f49fd083ab Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Thu, 3 Sep 2026 17:08:11 +0100 Subject: [PATCH 13/20] fix: retain Windows caption controls during loading and errors --- .../src/components/CapErrorBoundary.tsx | 97 +++++++------ .../controls/CaptionControlsWindows11.tsx | 35 ++++- apps/desktop/src/routes/(window-chrome).tsx | 11 -- apps/desktop/src/routes/editor/Editor.tsx | 13 +- apps/desktop/src/routes/editor/Header.tsx | 9 -- .../src/routes/editor/ImportProgress.tsx | 127 +++++++++--------- apps/desktop/src/routes/mode-select.tsx | 17 +-- .../screenshot-editor-skeleton.tsx | 4 +- apps/desktop/src/routes/teleprompter.tsx | 10 +- 9 files changed, 160 insertions(+), 163 deletions(-) diff --git a/apps/desktop/src/components/CapErrorBoundary.tsx b/apps/desktop/src/components/CapErrorBoundary.tsx index 2dca68d1d47..b7f3801a90f 100644 --- a/apps/desktop/src/components/CapErrorBoundary.tsx +++ b/apps/desktop/src/components/CapErrorBoundary.tsx @@ -1,55 +1,72 @@ import { Button } from "@cap/ui-solid"; import { getCurrentWebviewWindow } from "@tauri-apps/api/webviewWindow"; import { writeText } from "@tauri-apps/plugin-clipboard-manager"; +import { type as ostype } from "@tauri-apps/plugin-os"; import { ErrorBoundary, type ParentProps } from "solid-js"; +import Titlebar from "./titlebar/Titlebar"; export function CapErrorBoundary(props: ParentProps) { return ( { console.error(e); + const windowLabel = getCurrentWebviewWindow().label; + const showTitlebar = + ostype() === "windows" && + ([ + "main", + "settings", + "upgrade", + "mode-select", + "onboarding", + "teleprompter", + ].includes(windowLabel) || + /^(editor|screenshot-editor)-\d+$/.test(windowLabel)); return ( -
- -

- An Error Occured -

-

- We're very sorry, but something has gone wrong. -

-
- - - -
- - {import.meta.env.DEV && ( -