Skip to content

Refine auto-zoom segment generation and cursor stability #1106

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Oct 1, 2025
Merged

Refine auto-zoom segment generation and cursor stability #1106

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
bb90e52
feat: refine auto-zoom segments and cursor stability
richiemcilroy Oct 1, 2025
0bfc158
fix: Coderabbit suggestion
richiemcilroy Oct 1, 2025
c241cb8
fix: Cargo fmt
richiemcilroy Oct 1, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
340 changes: 273 additions & 67 deletions apps/desktop/src-tauri/src/recording.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
use cap_fail::fail;
use cap_project::cursor::SHORT_CURSOR_SHAPE_DEBOUNCE_MS;
use cap_project::{
CursorClickEvent, Platform, ProjectConfiguration, RecordingMeta, RecordingMetaInner,
SharingMeta, StudioRecordingMeta, TimelineConfiguration, TimelineSegment, ZoomMode,
ZoomSegment, cursor::CursorEvents,
CursorClickEvent, CursorMoveEvent, Platform, ProjectConfiguration, RecordingMeta,
RecordingMetaInner, SharingMeta, StudioRecordingMeta, TimelineConfiguration, TimelineSegment,
ZoomMode, ZoomSegment, cursor::CursorEvents,
};
use cap_recording::{
RecordingError, RecordingMode,
Expand All @@ -15,7 +16,13 @@ use cap_rendering::ProjectRecordingsMeta;
use cap_utils::{ensure_dir, spawn_actor};
use serde::Deserialize;
use specta::Type;
use std::{path::PathBuf, str::FromStr, sync::Arc, time::Duration};
use std::{
collections::{HashMap, VecDeque},
path::PathBuf,
str::FromStr,
sync::Arc,
time::Duration,
};
use tauri::{AppHandle, Manager};
use tauri_plugin_dialog::{DialogExt, MessageDialogBuilder};
use tauri_specta::Event;
Expand Down Expand Up @@ -899,56 +906,161 @@ async fn handle_recording_finish(
/// around user interactions to highlight important moments.
fn generate_zoom_segments_from_clicks_impl(
mut clicks: Vec<CursorClickEvent>,
recordings: &ProjectRecordingsMeta,
mut moves: Vec<CursorMoveEvent>,
max_duration: f64,
) -> Vec<ZoomSegment> {
const ZOOM_SEGMENT_AFTER_CLICK_PADDING: f64 = 1.5;
const ZOOM_SEGMENT_BEFORE_CLICK_PADDING: f64 = 0.8;
const ZOOM_DURATION: f64 = 1.0;
const CLICK_GROUP_THRESHOLD: f64 = 0.6; // seconds
const MIN_SEGMENT_PADDING: f64 = 2.0; // minimum gap between segments
const STOP_PADDING_SECONDS: f64 = 0.8;
const CLICK_PRE_PADDING: f64 = 0.6;
const CLICK_POST_PADDING: f64 = 1.6;
const MOVEMENT_PRE_PADDING: f64 = 0.4;
const MOVEMENT_POST_PADDING: f64 = 1.2;
const MERGE_GAP_THRESHOLD: f64 = 0.6;
const MIN_SEGMENT_DURATION: f64 = 1.3;
const MOVEMENT_WINDOW_SECONDS: f64 = 1.2;
const MOVEMENT_EVENT_DISTANCE_THRESHOLD: f64 = 0.025;
const MOVEMENT_WINDOW_DISTANCE_THRESHOLD: f64 = 0.1;

if max_duration <= 0.0 {
return Vec::new();
}

// We trim the tail of the recording to avoid using the final
// "stop recording" click as a zoom target.
let activity_end_limit = if max_duration > STOP_PADDING_SECONDS {
max_duration - STOP_PADDING_SECONDS
} else {
max_duration
};

let max_duration = recordings.duration();
if activity_end_limit <= f64::EPSILON {
return Vec::new();
}

clicks.sort_by(|a, b| {
a.time_ms
.partial_cmp(&b.time_ms)
.unwrap_or(std::cmp::Ordering::Equal)
});
moves.sort_by(|a, b| {
a.time_ms
.partial_cmp(&b.time_ms)
.unwrap_or(std::cmp::Ordering::Equal)
});

// Remove trailing click-down events that are too close to the end.
while let Some(index) = clicks.iter().rposition(|c| c.down) {
let time_secs = clicks[index].time_ms / 1000.0;
if time_secs > activity_end_limit {
clicks.remove(index);
} else {
break;
}
}

let mut segments = Vec::<ZoomSegment>::new();
let mut intervals: Vec<(f64, f64)> = Vec::new();

// Generate segments around mouse clicks
for click in &clicks {
if !click.down {
for click in clicks.into_iter().filter(|c| c.down) {
let time = click.time_ms / 1000.0;
if time >= activity_end_limit {
continue;
}

let time = click.time_ms / 1000.0;
let start = (time - CLICK_PRE_PADDING).max(0.0);
let end = (time + CLICK_POST_PADDING).min(activity_end_limit);

if end > start {
intervals.push((start, end));
}
}

let proposed_start = (time - ZOOM_SEGMENT_BEFORE_CLICK_PADDING).max(0.0);
let proposed_end = (time + ZOOM_SEGMENT_AFTER_CLICK_PADDING).min(max_duration);
let mut last_move_by_cursor: HashMap<String, (f64, f64, f64)> = HashMap::new();
let mut distance_window: VecDeque<(f64, f64)> = VecDeque::new();
let mut window_distance = 0.0_f64;

if let Some(last) = segments.last_mut() {
// Merge if within group threshold OR if segments would be too close together
if time <= last.end + CLICK_GROUP_THRESHOLD
|| proposed_start <= last.end + MIN_SEGMENT_PADDING
{
last.end = proposed_end;
continue;
for mv in moves.iter() {
let time = mv.time_ms / 1000.0;
if time >= activity_end_limit {
break;
}

let distance = if let Some((_, last_x, last_y)) = last_move_by_cursor.get(&mv.cursor_id) {
let dx = mv.x - last_x;
let dy = mv.y - last_y;
(dx * dx + dy * dy).sqrt()
} else {
0.0
};

last_move_by_cursor.insert(mv.cursor_id.clone(), (time, mv.x, mv.y));

if distance <= f64::EPSILON {
continue;
}

distance_window.push_back((time, distance));
window_distance += distance;

while let Some(&(old_time, old_distance)) = distance_window.front() {
if time - old_time > MOVEMENT_WINDOW_SECONDS {
distance_window.pop_front();
window_distance -= old_distance;
} else {
break;
}
}

if time < max_duration - ZOOM_DURATION {
segments.push(ZoomSegment {
start: proposed_start,
end: proposed_end,
amount: 2.0,
mode: ZoomMode::Auto,
});
if window_distance < 0.0 {
window_distance = 0.0;
}

let significant_movement = distance >= MOVEMENT_EVENT_DISTANCE_THRESHOLD
|| window_distance >= MOVEMENT_WINDOW_DISTANCE_THRESHOLD;

if !significant_movement {
continue;
}

let start = (time - MOVEMENT_PRE_PADDING).max(0.0);
let end = (time + MOVEMENT_POST_PADDING).min(activity_end_limit);

if end > start {
intervals.push((start, end));
}
}

segments
if intervals.is_empty() {
return Vec::new();
}

intervals.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap_or(std::cmp::Ordering::Equal));

let mut merged: Vec<(f64, f64)> = Vec::new();
for interval in intervals {
if let Some(last) = merged.last_mut() {
if interval.0 <= last.1 + MERGE_GAP_THRESHOLD {
last.1 = last.1.max(interval.1);
continue;
}
}
merged.push(interval);
}

merged
.into_iter()
.filter_map(|(start, end)| {
let duration = end - start;
if duration < MIN_SEGMENT_DURATION {
return None;
}

Some(ZoomSegment {
start,
end,
amount: 2.0,
mode: ZoomMode::Auto,
})
})
.collect()
}

/// Generates zoom segments based on mouse click events during recording.
Expand Down Expand Up @@ -979,24 +1091,34 @@ pub fn generate_zoom_segments_for_project(
return Vec::new();
};

let all_events = match studio_meta {
let mut all_clicks = Vec::new();
let mut all_moves = Vec::new();

match studio_meta {
StudioRecordingMeta::SingleSegment { segment } => {
if let Some(cursor_path) = &segment.cursor {
CursorEvents::load_from_file(&recording_meta.path(cursor_path))
.unwrap_or_default()
.clicks
} else {
vec![]
let mut events = CursorEvents::load_from_file(&recording_meta.path(cursor_path))
.unwrap_or_default();
let pointer_ids = studio_meta.pointer_cursor_ids();
let pointer_ids_ref = (!pointer_ids.is_empty()).then_some(&pointer_ids);
events.stabilize_short_lived_cursor_shapes(
pointer_ids_ref,
SHORT_CURSOR_SHAPE_DEBOUNCE_MS,
);
all_clicks = events.clicks;
all_moves = events.moves;
}
}
StudioRecordingMeta::MultipleSegments { inner, .. } => inner
.segments
.iter()
.flat_map(|s| s.cursor_events(recording_meta).clicks)
.collect(),
};
StudioRecordingMeta::MultipleSegments { inner, .. } => {
for segment in inner.segments.iter() {
let events = segment.cursor_events(recording_meta);
all_clicks.extend(events.clicks);
all_moves.extend(events.moves);
}
}
}

generate_zoom_segments_from_clicks_impl(all_events, recordings)
generate_zoom_segments_from_clicks_impl(all_clicks, all_moves, recordings.duration())
}

fn project_config_from_recording(
Expand All @@ -1009,26 +1131,110 @@ fn project_config_from_recording(
.unwrap_or(None)
.unwrap_or_default();

ProjectConfiguration {
timeline: Some(TimelineConfiguration {
segments: recordings
.segments
.iter()
.enumerate()
.map(|(i, segment)| TimelineSegment {
recording_segment: i as u32,
start: 0.0,
end: segment.duration(),
timescale: 1.0,
})
.collect(),
zoom_segments: if settings.auto_zoom_on_clicks {
generate_zoom_segments_from_clicks(completed_recording, recordings)
} else {
Vec::new()
},
scene_segments: Vec::new(),
}),
..default_config.unwrap_or_default()
let mut config = default_config.unwrap_or_default();

let timeline_segments = recordings
.segments
.iter()
.enumerate()
.map(|(i, segment)| TimelineSegment {
recording_segment: i as u32,
start: 0.0,
end: segment.duration(),
timescale: 1.0,
})
.collect::<Vec<_>>();

let zoom_segments = if settings.auto_zoom_on_clicks {
generate_zoom_segments_from_clicks(completed_recording, recordings)
} else {
Vec::new()
};

if !zoom_segments.is_empty() {
config.cursor.size = 200;
}

config.timeline = Some(TimelineConfiguration {
segments: timeline_segments,
zoom_segments,
scene_segments: Vec::new(),
});

config
}

#[cfg(test)]
mod tests {
use super::*;

fn click_event(time_ms: f64) -> CursorClickEvent {
CursorClickEvent {
active_modifiers: vec![],
cursor_num: 0,
cursor_id: "default".to_string(),
time_ms,
down: true,
}
}

fn move_event(time_ms: f64, x: f64, y: f64) -> CursorMoveEvent {
CursorMoveEvent {
active_modifiers: vec![],
cursor_id: "default".to_string(),
time_ms,
x,
y,
}
}

#[test]
fn skips_trailing_stop_click() {
let segments =
generate_zoom_segments_from_clicks_impl(vec![click_event(11_900.0)], vec![], 12.0);

assert!(
segments.is_empty(),
"expected trailing stop click to be ignored"
);
}

#[test]
fn generates_segment_for_sustained_activity() {
let clicks = vec![click_event(1_200.0), click_event(4_200.0)];
let moves = vec![
move_event(1_500.0, 0.10, 0.12),
move_event(1_720.0, 0.42, 0.45),
move_event(1_940.0, 0.74, 0.78),
];

let segments = generate_zoom_segments_from_clicks_impl(clicks, moves, 20.0);

assert!(
!segments.is_empty(),
"expected activity to produce zoom segments"
);
let first = &segments[0];
assert!(first.start < first.end);
assert!(first.end - first.start >= 1.3);
assert!(first.end <= 19.5);
}

#[test]
fn ignores_cursor_jitter() {
let jitter_moves = (0..30)
.map(|i| {
let t = 1_000.0 + (i as f64) * 30.0;
let delta = (i as f64) * 0.0004;
move_event(t, 0.5 + delta, 0.5)
})
.collect::<Vec<_>>();

let segments = generate_zoom_segments_from_clicks_impl(Vec::new(), jitter_moves, 15.0);

assert!(
segments.is_empty(),
"small jitter should not generate segments"
);
}
}
Loading
Loading