From 2431d4de99a28d15a13e36ee4210b6bc4d215881 Mon Sep 17 00:00:00 2001 From: Brendan Allan Date: Tue, 21 Oct 2025 14:07:44 +0800 Subject: [PATCH 1/6] recording: move display+crop conversion out of capture sources --- crates/recording/src/capture_pipeline.rs | 124 ++++++++++++++---- crates/recording/src/instant_recording.rs | 13 +- .../src/sources/screen_capture/mod.rs | 98 ++------------ crates/recording/src/studio_recording.rs | 21 +-- 4 files changed, 127 insertions(+), 129 deletions(-) diff --git a/crates/recording/src/capture_pipeline.rs b/crates/recording/src/capture_pipeline.rs index de757f89c5..3f2e88140b 100644 --- a/crates/recording/src/capture_pipeline.rs +++ b/crates/recording/src/capture_pipeline.rs @@ -2,13 +2,12 @@ use crate::{ feeds::microphone::MicrophoneFeedLock, output_pipeline::*, sources, - sources::screen_capture::{ - self, ScreenCaptureConfig, ScreenCaptureFormat, ScreenCaptureTarget, - }, + sources::screen_capture::{self, CropBounds, ScreenCaptureFormat, ScreenCaptureTarget}, }; +use anyhow::anyhow; use cap_timestamp::Timestamps; -use scap_targets::WindowId; -use std::{path::PathBuf, sync::Arc, time::SystemTime}; +use scap_targets::bounds::LogicalBounds; +use std::{path::PathBuf, sync::Arc}; pub trait MakeCapturePipeline: ScreenCaptureFormat + std::fmt::Debug + 'static { async fn make_studio_mode_pipeline( @@ -127,30 +126,97 @@ pub type ScreenCaptureMethod = screen_capture::CMSampleBufferCapture; #[cfg(windows)] pub type ScreenCaptureMethod = screen_capture::Direct3DCapture; -pub async fn create_screen_capture( - capture_target: &ScreenCaptureTarget, - force_show_cursor: bool, - max_fps: u32, - start_time: SystemTime, - system_audio: bool, - #[cfg(windows)] d3d_device: ::windows::Win32::Graphics::Direct3D11::ID3D11Device, - #[cfg(target_os = "macos")] shareable_content: cidre::arc::R, - #[cfg(target_os = "macos")] excluded_windows: Vec, -) -> anyhow::Result> { - Ok(ScreenCaptureConfig::::init( - capture_target, - force_show_cursor, - max_fps, - start_time, - system_audio, - #[cfg(windows)] - d3d_device, - #[cfg(target_os = "macos")] - shareable_content, - #[cfg(target_os = "macos")] - excluded_windows, - ) - .await?) +pub fn target_to_display_and_crop( + target: &ScreenCaptureTarget, +) -> anyhow::Result<(scap_targets::Display, Option)> { + use scap_targets::{bounds::*, *}; + + let display = target + .display() + .ok_or_else(|| anyhow!("Display not found"))?; + + let crop_bounds = match target { + ScreenCaptureTarget::Display { .. } => None, + ScreenCaptureTarget::Window { id } => { + let window = Window::from_id(id).ok_or_else(|| anyhow!("Window not found"))?; + + #[cfg(target_os = "macos")] + { + let raw_display_bounds = display + .raw_handle() + .logical_bounds() + .ok_or_else(|| anyhow!("No display bounds"))?; + let raw_window_bounds = window + .raw_handle() + .logical_bounds() + .ok_or_else(|| anyhow!("No window bounds"))?; + + Some(LogicalBounds::new( + LogicalPosition::new( + raw_window_bounds.position().x() - raw_display_bounds.position().x(), + raw_window_bounds.position().y() - raw_display_bounds.position().y(), + ), + raw_window_bounds.size(), + )) + } + + #[cfg(windows)] + { + let raw_display_position = display + .raw_handle() + .physical_position() + .ok_or_else(|| anyhow!("No display bounds"))?; + let raw_window_bounds = window + .raw_handle() + .physical_bounds() + .ok_or_else(|| anyhow!("No window bounds"))?; + + Some(PhysicalBounds::new( + PhysicalPosition::new( + raw_window_bounds.position().x() - raw_display_position.x(), + raw_window_bounds.position().y() - raw_display_position.y(), + ), + raw_window_bounds.size(), + )) + } + } + ScreenCaptureTarget::Area { + bounds: relative_bounds, + .. + } => { + #[cfg(target_os = "macos")] + { + Some(*relative_bounds) + } + + #[cfg(windows)] + { + let raw_display_size = display + .physical_size() + .ok_or(ScreenCaptureInitError::NoBounds)?; + let logical_display_size = display + .logical_size() + .ok_or(ScreenCaptureInitError::NoBounds)?; + + Some(PhysicalBounds::new( + PhysicalPosition::new( + (relative_bounds.position().x() / logical_display_size.width()) + * raw_display_size.width(), + (relative_bounds.position().y() / logical_display_size.height()) + * raw_display_size.height(), + ), + PhysicalSize::new( + (relative_bounds.size().width() / logical_display_size.width()) + * raw_display_size.width(), + (relative_bounds.size().height() / logical_display_size.height()) + * raw_display_size.height(), + ), + )) + } + } + }; + + Ok((display, crop_bounds)) } #[cfg(windows)] diff --git a/crates/recording/src/instant_recording.rs b/crates/recording/src/instant_recording.rs index 0b8f4290cb..e1eae7a938 100644 --- a/crates/recording/src/instant_recording.rs +++ b/crates/recording/src/instant_recording.rs @@ -1,10 +1,13 @@ use crate::{ RecordingBaseInputs, - capture_pipeline::{MakeCapturePipeline, ScreenCaptureMethod, Stop, create_screen_capture}, + capture_pipeline::{ + MakeCapturePipeline, ScreenCaptureMethod, Stop, target_to_display_and_crop, + }, feeds::microphone::MicrophoneFeedLock, output_pipeline::{self, OutputPipeline}, sources::screen_capture::{ScreenCaptureConfig, ScreenCaptureTarget}, }; +use anyhow::Context as _; use cap_media_info::{AudioInfo, VideoInfo}; use cap_project::InstantRecordingMeta; use cap_utils::ensure_dir; @@ -293,8 +296,12 @@ pub async fn spawn_instant_recording_actor( #[cfg(windows)] let d3d_device = crate::capture_pipeline::create_d3d_device()?; - let screen_source = create_screen_capture( - &inputs.capture_target, + let (display, crop) = + target_to_display_and_crop(&inputs.capture_target).context("target_display_crop")?; + + let screen_source = ScreenCaptureConfig::::init( + display, + crop, true, 30, start_time, diff --git a/crates/recording/src/sources/screen_capture/mod.rs b/crates/recording/src/sources/screen_capture/mod.rs index b4d5b5c700..78c7a2c22c 100644 --- a/crates/recording/src/sources/screen_capture/mod.rs +++ b/crates/recording/src/sources/screen_capture/mod.rs @@ -253,14 +253,17 @@ impl Clone for ScreenCaptureConfig, - #[cfg(target_os = "macos")] - crop_bounds: Option, + crop_bounds: Option, fps: u32, show_cursor: bool, } +#[cfg(target_os = "macos")] +pub type CropBounds = LogicalBounds; + +#[cfg(windows)] +pub type CropBounds = PhysicalBounds; + impl Config { pub fn fps(&self) -> u32 { self.fps @@ -280,7 +283,8 @@ pub enum ScreenCaptureInitError { impl ScreenCaptureConfig { #[allow(clippy::too_many_arguments)] pub async fn init( - target: &ScreenCaptureTarget, + display: scap_targets::Display, + crop_bounds: Option, show_cursor: bool, max_fps: u32, start_time: SystemTime, @@ -291,92 +295,10 @@ impl ScreenCaptureConfig { ) -> Result { cap_fail::fail!("ScreenCaptureSource::init"); - let display = target.display().ok_or(ScreenCaptureInitError::NoDisplay)?; - let fps = max_fps.min(display.refresh_rate() as u32); - let crop_bounds = match target { - ScreenCaptureTarget::Display { .. } => None, - ScreenCaptureTarget::Window { id } => { - let window = Window::from_id(id).ok_or(ScreenCaptureInitError::NoWindow)?; - - #[cfg(target_os = "macos")] - { - let raw_display_bounds = display - .raw_handle() - .logical_bounds() - .ok_or(ScreenCaptureInitError::NoBounds)?; - let raw_window_bounds = window - .raw_handle() - .logical_bounds() - .ok_or(ScreenCaptureInitError::NoBounds)?; - - Some(LogicalBounds::new( - LogicalPosition::new( - raw_window_bounds.position().x() - raw_display_bounds.position().x(), - raw_window_bounds.position().y() - raw_display_bounds.position().y(), - ), - raw_window_bounds.size(), - )) - } - - #[cfg(windows)] - { - let raw_display_position = display - .raw_handle() - .physical_position() - .ok_or(ScreenCaptureInitError::NoBounds)?; - let raw_window_bounds = window - .raw_handle() - .physical_bounds() - .ok_or(ScreenCaptureInitError::NoBounds)?; - - Some(PhysicalBounds::new( - PhysicalPosition::new( - raw_window_bounds.position().x() - raw_display_position.x(), - raw_window_bounds.position().y() - raw_display_position.y(), - ), - raw_window_bounds.size(), - )) - } - } - ScreenCaptureTarget::Area { - bounds: relative_bounds, - .. - } => { - #[cfg(target_os = "macos")] - { - Some(*relative_bounds) - } - - #[cfg(windows)] - { - let raw_display_size = display - .physical_size() - .ok_or(ScreenCaptureInitError::NoBounds)?; - let logical_display_size = display - .logical_size() - .ok_or(ScreenCaptureInitError::NoBounds)?; - - Some(PhysicalBounds::new( - PhysicalPosition::new( - (relative_bounds.position().x() / logical_display_size.width()) - * raw_display_size.width(), - (relative_bounds.position().y() / logical_display_size.height()) - * raw_display_size.height(), - ), - PhysicalSize::new( - (relative_bounds.size().width() / logical_display_size.width()) - * raw_display_size.width(), - (relative_bounds.size().height() / logical_display_size.height()) - * raw_display_size.height(), - ), - )) - } - } - }; - let output_size = crop_bounds + .clone() .and_then(|b| { #[cfg(target_os = "macos")] { diff --git a/crates/recording/src/studio_recording.rs b/crates/recording/src/studio_recording.rs index 75319ff2bb..58319880ef 100644 --- a/crates/recording/src/studio_recording.rs +++ b/crates/recording/src/studio_recording.rs @@ -1,10 +1,13 @@ use crate::{ ActorError, MediaError, RecordingBaseInputs, RecordingError, - capture_pipeline::{MakeCapturePipeline, ScreenCaptureMethod, Stop, create_screen_capture}, + capture_pipeline::{ + MakeCapturePipeline, ScreenCaptureMethod, Stop, target_to_display_and_crop, + }, cursor::{CursorActor, Cursors, spawn_cursor_recorder}, feeds::{camera::CameraFeedLock, microphone::MicrophoneFeedLock}, ffmpeg::{Mp4Muxer, OggMuxer}, output_pipeline::{DoneFut, FinishedOutputPipeline, OutputPipeline, PipelineDoneError}, + screen_capture::ScreenCaptureConfig, sources::{self, screen_capture}, }; use anyhow::{Context as _, anyhow}; @@ -680,11 +683,7 @@ async fn create_segment_pipeline( custom_cursor_capture: bool, start_time: Timestamps, ) -> anyhow::Result { - let display = base_inputs - .capture_target - .display() - .ok_or(CreateSegmentPipelineError::NoDisplay)?; - let crop_bounds = base_inputs + let cursor_crop_bounds = base_inputs .capture_target .cursor_crop() .ok_or(CreateSegmentPipelineError::NoBounds)?; @@ -692,8 +691,12 @@ async fn create_segment_pipeline( #[cfg(windows)] let d3d_device = crate::capture_pipeline::create_d3d_device().unwrap(); - let screen_config = create_screen_capture( - &base_inputs.capture_target, + let (display, crop) = + target_to_display_and_crop(&base_inputs.capture_target).context("target_display_crop")?; + + let screen_config = ScreenCaptureConfig::::init( + display, + crop, !custom_cursor_capture, 120, start_time.system_time(), @@ -760,7 +763,7 @@ async fn create_segment_pipeline( let cursor = custom_cursor_capture.then(move || { let cursor = spawn_cursor_recorder( - crop_bounds, + cursor_crop_bounds, display, cursors_dir.to_path_buf(), prev_cursors, From cce54acab933de78c90b68b847d926b2bdc9f8eb Mon Sep 17 00:00:00 2001 From: Brendan Allan Date: Tue, 21 Oct 2025 14:24:17 +0800 Subject: [PATCH 2/6] more anyhow errors --- crates/recording/src/capture_pipeline.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/recording/src/capture_pipeline.rs b/crates/recording/src/capture_pipeline.rs index 3f2e88140b..c99624f47e 100644 --- a/crates/recording/src/capture_pipeline.rs +++ b/crates/recording/src/capture_pipeline.rs @@ -193,10 +193,10 @@ pub fn target_to_display_and_crop( { let raw_display_size = display .physical_size() - .ok_or(ScreenCaptureInitError::NoBounds)?; + .ok_or_else(|| anyhow!("No display bounds"))?; let logical_display_size = display .logical_size() - .ok_or(ScreenCaptureInitError::NoBounds)?; + .ok_or_else(|| anyhow!("No window bounds"))?; Some(PhysicalBounds::new( PhysicalPosition::new( From bfd69f15ad8bd6ddf34da9d31e8fe93bb3d77e9f Mon Sep 17 00:00:00 2001 From: Brendan Allan Date: Tue, 21 Oct 2025 14:26:55 +0800 Subject: [PATCH 3/6] more error handling fixes --- crates/recording/src/instant_recording.rs | 3 ++- crates/recording/src/sources/screen_capture/macos.rs | 2 +- crates/recording/src/studio_recording.rs | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/recording/src/instant_recording.rs b/crates/recording/src/instant_recording.rs index e1eae7a938..3ac2bd3d4f 100644 --- a/crates/recording/src/instant_recording.rs +++ b/crates/recording/src/instant_recording.rs @@ -313,7 +313,8 @@ pub async fn spawn_instant_recording_actor( #[cfg(target_os = "macos")] inputs.excluded_windows, ) - .await?; + .await + .context("screen capture init")?; debug!("screen capture: {screen_source:#?}"); diff --git a/crates/recording/src/sources/screen_capture/macos.rs b/crates/recording/src/sources/screen_capture/macos.rs index abfbb7416f..7130444139 100644 --- a/crates/recording/src/sources/screen_capture/macos.rs +++ b/crates/recording/src/sources/screen_capture/macos.rs @@ -68,7 +68,7 @@ impl output_pipeline::VideoFrame for VideoFrame { impl ScreenCaptureConfig { pub async fn to_sources( - &self, + &ScreenCaptureConfig::::initself, ) -> anyhow::Result<(VideoSourceConfig, Option)> { let (error_tx, error_rx) = broadcast::channel(1); let (video_tx, video_rx) = flume::bounded(4); diff --git a/crates/recording/src/studio_recording.rs b/crates/recording/src/studio_recording.rs index 58319880ef..03ceec6795 100644 --- a/crates/recording/src/studio_recording.rs +++ b/crates/recording/src/studio_recording.rs @@ -709,7 +709,7 @@ async fn create_segment_pipeline( base_inputs.excluded_windows, ) .await - .unwrap(); + .context("screen capture init")?; let (capture_source, system_audio) = screen_config.to_sources().await?; From 1d223223d98f8aa54eb272552723c8559e2fdc4a Mon Sep 17 00:00:00 2001 From: Brendan Allan Date: Tue, 21 Oct 2025 14:28:59 +0800 Subject: [PATCH 4/6] fix --- crates/recording/src/sources/screen_capture/macos.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/recording/src/sources/screen_capture/macos.rs b/crates/recording/src/sources/screen_capture/macos.rs index 7130444139..abfbb7416f 100644 --- a/crates/recording/src/sources/screen_capture/macos.rs +++ b/crates/recording/src/sources/screen_capture/macos.rs @@ -68,7 +68,7 @@ impl output_pipeline::VideoFrame for VideoFrame { impl ScreenCaptureConfig { pub async fn to_sources( - &ScreenCaptureConfig::::initself, + &self, ) -> anyhow::Result<(VideoSourceConfig, Option)> { let (error_tx, error_rx) = broadcast::channel(1); let (video_tx, video_rx) = flume::bounded(4); From e231e14b9f2170b33a9780fedf93cd98aed0c85d Mon Sep 17 00:00:00 2001 From: Brendan Allan Date: Tue, 21 Oct 2025 14:33:14 +0800 Subject: [PATCH 5/6] Update crates/recording/src/capture_pipeline.rs Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- crates/recording/src/capture_pipeline.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/recording/src/capture_pipeline.rs b/crates/recording/src/capture_pipeline.rs index c99624f47e..b448f7b00d 100644 --- a/crates/recording/src/capture_pipeline.rs +++ b/crates/recording/src/capture_pipeline.rs @@ -196,8 +196,7 @@ pub fn target_to_display_and_crop( .ok_or_else(|| anyhow!("No display bounds"))?; let logical_display_size = display .logical_size() - .ok_or_else(|| anyhow!("No window bounds"))?; - + .ok_or_else(|| anyhow!("No display logical size"))?; Some(PhysicalBounds::new( PhysicalPosition::new( (relative_bounds.position().x() / logical_display_size.width()) From ba1101f780a7bbbb6d262098fa6683c2bf616489 Mon Sep 17 00:00:00 2001 From: Brendan Allan Date: Tue, 21 Oct 2025 14:42:02 +0800 Subject: [PATCH 6/6] delay cursor crop calculation --- crates/recording/src/studio_recording.rs | 42 +++++++++++++----------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/crates/recording/src/studio_recording.rs b/crates/recording/src/studio_recording.rs index 03ceec6795..146c235ac0 100644 --- a/crates/recording/src/studio_recording.rs +++ b/crates/recording/src/studio_recording.rs @@ -683,11 +683,6 @@ async fn create_segment_pipeline( custom_cursor_capture: bool, start_time: Timestamps, ) -> anyhow::Result { - let cursor_crop_bounds = base_inputs - .capture_target - .cursor_crop() - .ok_or(CreateSegmentPipelineError::NoBounds)?; - #[cfg(windows)] let d3d_device = crate::capture_pipeline::create_d3d_device().unwrap(); @@ -761,21 +756,28 @@ async fn create_segment_pipeline( .transpose() .context("microphone pipeline setup")?; - let cursor = custom_cursor_capture.then(move || { - let cursor = spawn_cursor_recorder( - cursor_crop_bounds, - display, - cursors_dir.to_path_buf(), - prev_cursors, - next_cursors_id, - start_time, - ); - - CursorPipeline { - output_path: dir.join("cursor.json"), - actor: cursor, - } - }); + let cursor = custom_cursor_capture + .then(move || { + let cursor_crop_bounds = base_inputs + .capture_target + .cursor_crop() + .ok_or(CreateSegmentPipelineError::NoBounds)?; + + let cursor = spawn_cursor_recorder( + cursor_crop_bounds, + display, + cursors_dir.to_path_buf(), + prev_cursors, + next_cursors_id, + start_time, + ); + + Ok::<_, CreateSegmentPipelineError>(CursorPipeline { + output_path: dir.join("cursor.json"), + actor: cursor, + }) + }) + .transpose()?; info!("pipeline playing");