diff --git a/apps/desktop/src-tauri/src/recording.rs b/apps/desktop/src-tauri/src/recording.rs index 217f14e084..4de8c1c234 100644 --- a/apps/desktop/src-tauri/src/recording.rs +++ b/apps/desktop/src-tauri/src/recording.rs @@ -1,8 +1,9 @@ use cap_fail::fail; +use cap_project::cursor::SHORT_CURSOR_SHAPE_DEBOUNCE_MS; use cap_project::{ - CursorClickEvent, Platform, ProjectConfiguration, RecordingMeta, RecordingMetaInner, - SharingMeta, StudioRecordingMeta, TimelineConfiguration, TimelineSegment, ZoomMode, - ZoomSegment, cursor::CursorEvents, + CursorClickEvent, CursorMoveEvent, Platform, ProjectConfiguration, RecordingMeta, + RecordingMetaInner, SharingMeta, StudioRecordingMeta, TimelineConfiguration, TimelineSegment, + ZoomMode, ZoomSegment, cursor::CursorEvents, }; use cap_recording::{ RecordingError, RecordingMode, @@ -15,7 +16,13 @@ use cap_rendering::ProjectRecordingsMeta; use cap_utils::{ensure_dir, spawn_actor}; use serde::Deserialize; use specta::Type; -use std::{path::PathBuf, str::FromStr, sync::Arc, time::Duration}; +use std::{ + collections::{HashMap, VecDeque}, + path::PathBuf, + str::FromStr, + sync::Arc, + time::Duration, +}; use tauri::{AppHandle, Manager}; use tauri_plugin_dialog::{DialogExt, MessageDialogBuilder}; use tauri_specta::Event; @@ -899,56 +906,161 @@ async fn handle_recording_finish( /// around user interactions to highlight important moments. fn generate_zoom_segments_from_clicks_impl( mut clicks: Vec, - recordings: &ProjectRecordingsMeta, + mut moves: Vec, + max_duration: f64, ) -> Vec { - const ZOOM_SEGMENT_AFTER_CLICK_PADDING: f64 = 1.5; - const ZOOM_SEGMENT_BEFORE_CLICK_PADDING: f64 = 0.8; - const ZOOM_DURATION: f64 = 1.0; - const CLICK_GROUP_THRESHOLD: f64 = 0.6; // seconds - const MIN_SEGMENT_PADDING: f64 = 2.0; // minimum gap between segments + const STOP_PADDING_SECONDS: f64 = 0.8; + const CLICK_PRE_PADDING: f64 = 0.6; + const CLICK_POST_PADDING: f64 = 1.6; + const MOVEMENT_PRE_PADDING: f64 = 0.4; + const MOVEMENT_POST_PADDING: f64 = 1.2; + const MERGE_GAP_THRESHOLD: f64 = 0.6; + const MIN_SEGMENT_DURATION: f64 = 1.3; + const MOVEMENT_WINDOW_SECONDS: f64 = 1.2; + const MOVEMENT_EVENT_DISTANCE_THRESHOLD: f64 = 0.025; + const MOVEMENT_WINDOW_DISTANCE_THRESHOLD: f64 = 0.1; + + if max_duration <= 0.0 { + return Vec::new(); + } + + // We trim the tail of the recording to avoid using the final + // "stop recording" click as a zoom target. + let activity_end_limit = if max_duration > STOP_PADDING_SECONDS { + max_duration - STOP_PADDING_SECONDS + } else { + max_duration + }; - let max_duration = recordings.duration(); + if activity_end_limit <= f64::EPSILON { + return Vec::new(); + } clicks.sort_by(|a, b| { a.time_ms .partial_cmp(&b.time_ms) .unwrap_or(std::cmp::Ordering::Equal) }); + moves.sort_by(|a, b| { + a.time_ms + .partial_cmp(&b.time_ms) + .unwrap_or(std::cmp::Ordering::Equal) + }); + + // Remove trailing click-down events that are too close to the end. + while let Some(index) = clicks.iter().rposition(|c| c.down) { + let time_secs = clicks[index].time_ms / 1000.0; + if time_secs > activity_end_limit { + clicks.remove(index); + } else { + break; + } + } - let mut segments = Vec::::new(); + let mut intervals: Vec<(f64, f64)> = Vec::new(); - // Generate segments around mouse clicks - for click in &clicks { - if !click.down { + for click in clicks.into_iter().filter(|c| c.down) { + let time = click.time_ms / 1000.0; + if time >= activity_end_limit { continue; } - let time = click.time_ms / 1000.0; + let start = (time - CLICK_PRE_PADDING).max(0.0); + let end = (time + CLICK_POST_PADDING).min(activity_end_limit); + + if end > start { + intervals.push((start, end)); + } + } - let proposed_start = (time - ZOOM_SEGMENT_BEFORE_CLICK_PADDING).max(0.0); - let proposed_end = (time + ZOOM_SEGMENT_AFTER_CLICK_PADDING).min(max_duration); + let mut last_move_by_cursor: HashMap = HashMap::new(); + let mut distance_window: VecDeque<(f64, f64)> = VecDeque::new(); + let mut window_distance = 0.0_f64; - if let Some(last) = segments.last_mut() { - // Merge if within group threshold OR if segments would be too close together - if time <= last.end + CLICK_GROUP_THRESHOLD - || proposed_start <= last.end + MIN_SEGMENT_PADDING - { - last.end = proposed_end; - continue; + for mv in moves.iter() { + let time = mv.time_ms / 1000.0; + if time >= activity_end_limit { + break; + } + + let distance = if let Some((_, last_x, last_y)) = last_move_by_cursor.get(&mv.cursor_id) { + let dx = mv.x - last_x; + let dy = mv.y - last_y; + (dx * dx + dy * dy).sqrt() + } else { + 0.0 + }; + + last_move_by_cursor.insert(mv.cursor_id.clone(), (time, mv.x, mv.y)); + + if distance <= f64::EPSILON { + continue; + } + + distance_window.push_back((time, distance)); + window_distance += distance; + + while let Some(&(old_time, old_distance)) = distance_window.front() { + if time - old_time > MOVEMENT_WINDOW_SECONDS { + distance_window.pop_front(); + window_distance -= old_distance; + } else { + break; } } - if time < max_duration - ZOOM_DURATION { - segments.push(ZoomSegment { - start: proposed_start, - end: proposed_end, - amount: 2.0, - mode: ZoomMode::Auto, - }); + if window_distance < 0.0 { + window_distance = 0.0; + } + + let significant_movement = distance >= MOVEMENT_EVENT_DISTANCE_THRESHOLD + || window_distance >= MOVEMENT_WINDOW_DISTANCE_THRESHOLD; + + if !significant_movement { + continue; + } + + let start = (time - MOVEMENT_PRE_PADDING).max(0.0); + let end = (time + MOVEMENT_POST_PADDING).min(activity_end_limit); + + if end > start { + intervals.push((start, end)); } } - segments + if intervals.is_empty() { + return Vec::new(); + } + + intervals.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap_or(std::cmp::Ordering::Equal)); + + let mut merged: Vec<(f64, f64)> = Vec::new(); + for interval in intervals { + if let Some(last) = merged.last_mut() { + if interval.0 <= last.1 + MERGE_GAP_THRESHOLD { + last.1 = last.1.max(interval.1); + continue; + } + } + merged.push(interval); + } + + merged + .into_iter() + .filter_map(|(start, end)| { + let duration = end - start; + if duration < MIN_SEGMENT_DURATION { + return None; + } + + Some(ZoomSegment { + start, + end, + amount: 2.0, + mode: ZoomMode::Auto, + }) + }) + .collect() } /// Generates zoom segments based on mouse click events during recording. @@ -979,24 +1091,34 @@ pub fn generate_zoom_segments_for_project( return Vec::new(); }; - let all_events = match studio_meta { + let mut all_clicks = Vec::new(); + let mut all_moves = Vec::new(); + + match studio_meta { StudioRecordingMeta::SingleSegment { segment } => { if let Some(cursor_path) = &segment.cursor { - CursorEvents::load_from_file(&recording_meta.path(cursor_path)) - .unwrap_or_default() - .clicks - } else { - vec![] + let mut events = CursorEvents::load_from_file(&recording_meta.path(cursor_path)) + .unwrap_or_default(); + let pointer_ids = studio_meta.pointer_cursor_ids(); + let pointer_ids_ref = (!pointer_ids.is_empty()).then_some(&pointer_ids); + events.stabilize_short_lived_cursor_shapes( + pointer_ids_ref, + SHORT_CURSOR_SHAPE_DEBOUNCE_MS, + ); + all_clicks = events.clicks; + all_moves = events.moves; } } - StudioRecordingMeta::MultipleSegments { inner, .. } => inner - .segments - .iter() - .flat_map(|s| s.cursor_events(recording_meta).clicks) - .collect(), - }; + StudioRecordingMeta::MultipleSegments { inner, .. } => { + for segment in inner.segments.iter() { + let events = segment.cursor_events(recording_meta); + all_clicks.extend(events.clicks); + all_moves.extend(events.moves); + } + } + } - generate_zoom_segments_from_clicks_impl(all_events, recordings) + generate_zoom_segments_from_clicks_impl(all_clicks, all_moves, recordings.duration()) } fn project_config_from_recording( @@ -1009,26 +1131,110 @@ fn project_config_from_recording( .unwrap_or(None) .unwrap_or_default(); - ProjectConfiguration { - timeline: Some(TimelineConfiguration { - segments: recordings - .segments - .iter() - .enumerate() - .map(|(i, segment)| TimelineSegment { - recording_segment: i as u32, - start: 0.0, - end: segment.duration(), - timescale: 1.0, - }) - .collect(), - zoom_segments: if settings.auto_zoom_on_clicks { - generate_zoom_segments_from_clicks(completed_recording, recordings) - } else { - Vec::new() - }, - scene_segments: Vec::new(), - }), - ..default_config.unwrap_or_default() + let mut config = default_config.unwrap_or_default(); + + let timeline_segments = recordings + .segments + .iter() + .enumerate() + .map(|(i, segment)| TimelineSegment { + recording_segment: i as u32, + start: 0.0, + end: segment.duration(), + timescale: 1.0, + }) + .collect::>(); + + let zoom_segments = if settings.auto_zoom_on_clicks { + generate_zoom_segments_from_clicks(completed_recording, recordings) + } else { + Vec::new() + }; + + if !zoom_segments.is_empty() { + config.cursor.size = 200; + } + + config.timeline = Some(TimelineConfiguration { + segments: timeline_segments, + zoom_segments, + scene_segments: Vec::new(), + }); + + config +} + +#[cfg(test)] +mod tests { + use super::*; + + fn click_event(time_ms: f64) -> CursorClickEvent { + CursorClickEvent { + active_modifiers: vec![], + cursor_num: 0, + cursor_id: "default".to_string(), + time_ms, + down: true, + } + } + + fn move_event(time_ms: f64, x: f64, y: f64) -> CursorMoveEvent { + CursorMoveEvent { + active_modifiers: vec![], + cursor_id: "default".to_string(), + time_ms, + x, + y, + } + } + + #[test] + fn skips_trailing_stop_click() { + let segments = + generate_zoom_segments_from_clicks_impl(vec![click_event(11_900.0)], vec![], 12.0); + + assert!( + segments.is_empty(), + "expected trailing stop click to be ignored" + ); + } + + #[test] + fn generates_segment_for_sustained_activity() { + let clicks = vec![click_event(1_200.0), click_event(4_200.0)]; + let moves = vec![ + move_event(1_500.0, 0.10, 0.12), + move_event(1_720.0, 0.42, 0.45), + move_event(1_940.0, 0.74, 0.78), + ]; + + let segments = generate_zoom_segments_from_clicks_impl(clicks, moves, 20.0); + + assert!( + !segments.is_empty(), + "expected activity to produce zoom segments" + ); + let first = &segments[0]; + assert!(first.start < first.end); + assert!(first.end - first.start >= 1.3); + assert!(first.end <= 19.5); + } + + #[test] + fn ignores_cursor_jitter() { + let jitter_moves = (0..30) + .map(|i| { + let t = 1_000.0 + (i as f64) * 30.0; + let delta = (i as f64) * 0.0004; + move_event(t, 0.5 + delta, 0.5) + }) + .collect::>(); + + let segments = generate_zoom_segments_from_clicks_impl(Vec::new(), jitter_moves, 15.0); + + assert!( + segments.is_empty(), + "small jitter should not generate segments" + ); } } diff --git a/apps/desktop/src/routes/editor/Timeline/ZoomTrack.tsx b/apps/desktop/src/routes/editor/Timeline/ZoomTrack.tsx index d38b5f6996..533cd56dee 100644 --- a/apps/desktop/src/routes/editor/Timeline/ZoomTrack.tsx +++ b/apps/desktop/src/routes/editor/Timeline/ZoomTrack.tsx @@ -57,6 +57,12 @@ export function ZoomTrack(props: { try { const zoomSegments = await commands.generateZoomSegmentsFromClicks(); setProject("timeline", "zoomSegments", zoomSegments); + if (zoomSegments.length > 0) { + const currentSize = project.cursor?.size ?? 0; + if (currentSize < 200) { + setProject("cursor", "size", 200); + } + } } catch (error) { console.error("Failed to generate zoom segments:", error); } diff --git a/crates/project/src/cursor.rs b/crates/project/src/cursor.rs index db636b3be1..a5a4e542bd 100644 --- a/crates/project/src/cursor.rs +++ b/crates/project/src/cursor.rs @@ -1,6 +1,9 @@ use serde::{Deserialize, Serialize}; use specta::Type; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; +use std::ops::Range; + +pub const SHORT_CURSOR_SHAPE_DEBOUNCE_MS: f64 = 1000.0; use std::fs::File; use std::path::{Path, PathBuf}; @@ -72,6 +75,125 @@ impl CursorEvents { serde_json::from_reader(file).map_err(|e| format!("Failed to parse cursor data: {e}")) } + pub fn stabilize_short_lived_cursor_shapes( + &mut self, + pointer_ids: Option<&HashSet>, + threshold_ms: f64, + ) { + if self.moves.len() < 2 { + return; + } + + let mut segments: Vec = Vec::new(); + let mut idx = 0; + + while idx < self.moves.len() { + let start_index = idx; + let start_time = self.moves[idx].time_ms; + let id = self.moves[idx].cursor_id.clone(); + + idx += 1; + while idx < self.moves.len() && self.moves[idx].cursor_id == id { + idx += 1; + } + + segments.push(CursorSegment { + range: start_index..idx, + start_time, + end_time: 0.0, + duration: 0.0, + id, + }); + } + + if segments.len() < 2 { + return; + } + + let last_move_time = self.moves.last().map(|event| event.time_ms).unwrap_or(0.0); + + for i in 0..segments.len() { + let end_time = if i + 1 < segments.len() { + segments[i + 1].start_time + } else { + last_move_time + }; + + let duration = (end_time - segments[i].start_time).max(0.0); + segments[i].duration = duration; + segments[i].end_time = if i + 1 < segments.len() { + end_time + } else { + f64::MAX + }; + } + + let mut duration_by_id = HashMap::::new(); + for segment in &segments { + *duration_by_id.entry(segment.id.clone()).or_default() += segment.duration; + } + + let preferred_pointer = pointer_ids.and_then(|set| { + segments + .iter() + .find(|segment| set.contains(&segment.id)) + .map(|segment| segment.id.clone()) + }); + + let global_fallback = duration_by_id + .iter() + .max_by(|a, b| a.1.partial_cmp(b.1).unwrap_or(std::cmp::Ordering::Equal)) + .map(|(id, _)| id.clone()); + + for i in 0..segments.len() { + let segment_id = segments[i].id.clone(); + let is_pointer_segment = pointer_ids + .map(|set| set.contains(&segment_id)) + .unwrap_or(false); + + if segments[i].duration >= threshold_ms || is_pointer_segment { + continue; + } + + let replacement = preferred_pointer + .clone() + .or_else(|| global_fallback.clone()) + .or_else(|| { + if i > 0 { + Some(segments[i - 1].id.clone()) + } else { + None + } + }) + .or_else(|| segments.get(i + 1).map(|segment| segment.id.clone())) + .unwrap_or_else(|| segment_id.clone()); + + if replacement == segment_id { + continue; + } + + for event in &mut self.moves[segments[i].range.clone()] { + event.cursor_id = replacement.clone(); + } + segments[i].id = replacement; + } + + if self.clicks.is_empty() { + return; + } + + let mut segment_index = 0; + for click in &mut self.clicks { + while segment_index + 1 < segments.len() + && click.time_ms >= segments[segment_index].end_time + { + segment_index += 1; + } + + click.cursor_id = segments[segment_index].id.clone(); + } + } + pub fn cursor_position_at(&self, time: f64) -> Option> { // Debug print to understand what we're looking for println!("Looking for cursor position at time: {time}"); @@ -143,3 +265,122 @@ impl From for CursorEvents { } } } + +#[derive(Clone)] +struct CursorSegment { + range: Range, + start_time: f64, + end_time: f64, + duration: f64, + id: String, +} + +#[cfg(test)] +mod tests { + use super::*; + + fn move_event(time_ms: f64, cursor_id: &str) -> CursorMoveEvent { + CursorMoveEvent { + active_modifiers: vec![], + cursor_id: cursor_id.to_string(), + time_ms, + x: 0.0, + y: 0.0, + } + } + + fn click_event(time_ms: f64, cursor_id: &str) -> CursorClickEvent { + CursorClickEvent { + active_modifiers: vec![], + cursor_id: cursor_id.to_string(), + cursor_num: 0, + down: true, + time_ms, + } + } + + #[test] + fn short_lived_segments_are_replaced_with_pointer() { + let mut pointer_ids = HashSet::new(); + pointer_ids.insert("pointer".to_string()); + + let mut events = CursorEvents { + moves: vec![ + move_event(0.0, "pointer"), + move_event(200.0, "ibeam"), + move_event(400.0, "pointer"), + move_event(900.0, "pointer"), + ], + clicks: vec![click_event(250.0, "ibeam")], + }; + + events.stabilize_short_lived_cursor_shapes( + Some(&pointer_ids), + SHORT_CURSOR_SHAPE_DEBOUNCE_MS, + ); + + assert!( + events + .moves + .iter() + .all(|event| event.cursor_id == "pointer") + ); + assert!( + events + .clicks + .iter() + .all(|event| event.cursor_id == "pointer") + ); + } + + #[test] + fn longer_segments_are_preserved() { + let mut pointer_ids = HashSet::new(); + pointer_ids.insert("pointer".to_string()); + + let mut events = CursorEvents { + moves: vec![ + move_event(0.0, "pointer"), + move_event(200.0, "ibeam"), + move_event(1500.0, "pointer"), + ], + clicks: vec![click_event(400.0, "ibeam")], + }; + + events.stabilize_short_lived_cursor_shapes( + Some(&pointer_ids), + SHORT_CURSOR_SHAPE_DEBOUNCE_MS, + ); + + assert_eq!(events.moves[1].cursor_id, "ibeam"); + assert_eq!(events.clicks[0].cursor_id, "ibeam"); + } + + #[test] + fn falls_back_to_dominant_cursor_without_pointer_metadata() { + let mut events = CursorEvents { + moves: vec![ + move_event(0.0, "pointer"), + move_event(200.0, "ibeam"), + move_event(400.0, "pointer"), + move_event(1200.0, "pointer"), + ], + clicks: vec![click_event(250.0, "ibeam")], + }; + + events.stabilize_short_lived_cursor_shapes(None, SHORT_CURSOR_SHAPE_DEBOUNCE_MS); + + assert!( + events + .moves + .iter() + .all(|event| event.cursor_id == "pointer") + ); + assert!( + events + .clicks + .iter() + .all(|event| event.cursor_id == "pointer") + ); + } +} diff --git a/crates/project/src/meta.rs b/crates/project/src/meta.rs index 15c08e095f..d2d0457afe 100644 --- a/crates/project/src/meta.rs +++ b/crates/project/src/meta.rs @@ -3,14 +3,17 @@ use relative_path::RelativePathBuf; use serde::{Deserialize, Serialize}; use specta::Type; use std::{ - collections::HashMap, + collections::{HashMap, HashSet}, error::Error, path::{Path, PathBuf}, }; use tracing::{debug, info, warn}; // use tracing::{debug, warn}; -use crate::{CaptionsData, CursorEvents, CursorImage, ProjectConfiguration, XY}; +use crate::{ + CaptionsData, CursorEvents, CursorImage, ProjectConfiguration, XY, + cursor::SHORT_CURSOR_SHAPE_DEBOUNCE_MS, +}; #[derive(Debug, Clone, Serialize, Deserialize, Type)] pub struct VideoMeta { @@ -174,6 +177,13 @@ impl StudioRecordingMeta { } } + pub fn pointer_cursor_ids(&self) -> HashSet { + match self { + StudioRecordingMeta::MultipleSegments { inner, .. } => inner.pointer_cursor_ids(), + _ => HashSet::new(), + } + } + pub fn min_fps(&self) -> u32 { match self { StudioRecordingMeta::SingleSegment { segment } => segment.display.fps, @@ -252,6 +262,24 @@ impl MultipleSegments { meta.project_path.join(path) } + pub fn pointer_cursor_ids(&self) -> HashSet { + match &self.cursors { + Cursors::Correct(map) => map + .iter() + .filter_map(|(id, cursor)| match cursor.shape.as_ref() { + Some(cap_cursor_info::CursorShape::MacOS( + cap_cursor_info::CursorShapeMacOS::Arrow, + )) + | Some(cap_cursor_info::CursorShape::Windows( + cap_cursor_info::CursorShapeWindows::Arrow, + )) => Some(id.clone()), + _ => None, + }) + .collect(), + Cursors::Old(_) => HashSet::new(), + } + } + pub fn get_cursor_image(&self, meta: &RecordingMeta, id: &str) -> Option { match &self.cursors { Cursors::Old(_) => None, @@ -293,13 +321,24 @@ impl MultipleSegment { let full_path = meta.path(cursor_path); // Try to load the cursor data - match CursorEvents::load_from_file(&full_path) { + let mut data = match CursorEvents::load_from_file(&full_path) { Ok(data) => data, Err(e) => { eprintln!("Failed to load cursor data: {e}"); - CursorEvents::default() + return CursorEvents::default(); } - } + }; + + let pointer_ids = if let RecordingMetaInner::Studio(studio_meta) = &meta.inner { + studio_meta.pointer_cursor_ids() + } else { + HashSet::new() + }; + + let pointer_ids_ref = (!pointer_ids.is_empty()).then_some(&pointer_ids); + data.stabilize_short_lived_cursor_shapes(pointer_ids_ref, SHORT_CURSOR_SHAPE_DEBOUNCE_MS); + + data } pub fn latest_start_time(&self) -> Option {