diff --git a/Cargo.lock b/Cargo.lock index 8892a0170..e256f95cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1417,6 +1417,7 @@ dependencies = [ name = "cap-recording" version = "0.1.0" dependencies = [ + "anyhow", "cap-audio", "cap-camera", "cap-camera-ffmpeg", diff --git a/apps/cli/src/record.rs b/apps/cli/src/record.rs index 25f03e3e9..0fe468cfd 100644 --- a/apps/cli/src/record.rs +++ b/apps/cli/src/record.rs @@ -75,7 +75,7 @@ impl RecordStart { .await .unwrap(); - actor.0.stop().await.unwrap(); + actor.stop().await.unwrap(); Ok(()) } diff --git a/apps/desktop/src-tauri/src/camera.rs b/apps/desktop/src-tauri/src/camera.rs index 03ea27ef9..4160fa2f5 100644 --- a/apps/desktop/src-tauri/src/camera.rs +++ b/apps/desktop/src-tauri/src/camera.rs @@ -1,7 +1,7 @@ use anyhow::{Context, anyhow}; -use cap_recording::feeds::{ - self, - camera::{CameraFeed, RawCameraFrame}, +use cap_recording::{ + FFmpegVideoFrame, + feeds::{self, camera::CameraFeed}, }; use ffmpeg::{ format::{self, Pixel}, @@ -465,7 +465,7 @@ impl Renderer { window: WebviewWindow, default_state: CameraPreviewState, mut reconfigure: broadcast::Receiver, - camera_rx: flume::Receiver, + camera_rx: flume::Receiver, ) { let mut resampler_frame = Cached::default(); let Ok(mut scaler) = scaling::Context::get( @@ -496,7 +496,7 @@ impl Renderer { } { match event { Ok(frame) => { - let aspect_ratio = frame.frame.width() as f32 / frame.frame.height() as f32; + let aspect_ratio = frame.inner.width() as f32 / frame.inner.height() as f32; self.sync_ratio_uniform_and_resize_window_to_it(&window, &state, aspect_ratio); if let Ok(surface) = self.surface.get_current_texture().map_err(|err| { @@ -509,16 +509,16 @@ impl Renderer { .get_or_init((output_width, output_height), frame::Video::empty); scaler.cached( - frame.frame.format(), - frame.frame.width(), - frame.frame.height(), + frame.inner.format(), + frame.inner.width(), + frame.inner.height(), format::Pixel::RGBA, output_width, output_height, ffmpeg::software::scaling::flag::Flags::FAST_BILINEAR, ); - if let Err(err) = scaler.run(&frame.frame, resampler_frame) { + if let Err(err) = scaler.run(&frame.inner, resampler_frame) { error!("Error rescaling frame with ffmpeg: {err:?}"); continue; } diff --git a/apps/desktop/src-tauri/src/camera_legacy.rs b/apps/desktop/src-tauri/src/camera_legacy.rs index 2aeaeaf3b..683514e65 100644 --- a/apps/desktop/src-tauri/src/camera_legacy.rs +++ b/apps/desktop/src-tauri/src/camera_legacy.rs @@ -1,11 +1,11 @@ -use cap_recording::feeds::camera::RawCameraFrame; +use cap_recording::FFmpegVideoFrame; use flume::Sender; use tokio_util::sync::CancellationToken; use crate::frame_ws::{WSFrame, create_frame_ws}; -pub async fn create_camera_preview_ws() -> (Sender, u16, CancellationToken) { - let (camera_tx, mut _camera_rx) = flume::bounded::(4); +pub async fn create_camera_preview_ws() -> (Sender, u16, CancellationToken) { + let (camera_tx, mut _camera_rx) = flume::bounded::(4); let (_camera_tx, camera_rx) = flume::bounded::(4); std::thread::spawn(move || { use ffmpeg::format::Pixel; @@ -13,7 +13,7 @@ pub async fn create_camera_preview_ws() -> (Sender, u16, Cancell let mut converter: Option<(Pixel, ffmpeg::software::scaling::Context)> = None; while let Ok(raw_frame) = _camera_rx.recv() { - let mut frame = raw_frame.frame; + let mut frame = raw_frame.inner; if frame.format() != Pixel::RGBA || frame.width() > 1280 || frame.height() > 720 { let converter = match &mut converter { diff --git a/apps/desktop/src-tauri/src/deeplink_actions.rs b/apps/desktop/src-tauri/src/deeplink_actions.rs index ff4f05a99..aaf6a62b8 100644 --- a/apps/desktop/src-tauri/src/deeplink_actions.rs +++ b/apps/desktop/src-tauri/src/deeplink_actions.rs @@ -1,4 +1,6 @@ -use cap_recording::{RecordingMode, feeds::camera::DeviceOrModelID, sources::ScreenCaptureTarget}; +use cap_recording::{ + RecordingMode, feeds::camera::DeviceOrModelID, sources::screen_capture::ScreenCaptureTarget, +}; use serde::{Deserialize, Serialize}; use std::path::{Path, PathBuf}; use tauri::{AppHandle, Manager, Url}; diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index 4e997833c..74ca14e70 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -40,7 +40,7 @@ use cap_recording::{ camera::{CameraFeed, DeviceOrModelID}, microphone::{self, MicrophoneFeed}, }, - sources::ScreenCaptureTarget, + sources::screen_capture::ScreenCaptureTarget, }; use cap_rendering::{ProjectRecordingsMeta, RenderedFrame}; use clipboard_rs::common::RustImage; diff --git a/apps/desktop/src-tauri/src/recording.rs b/apps/desktop/src-tauri/src/recording.rs index f19e8c873..fe53716dd 100644 --- a/apps/desktop/src-tauri/src/recording.rs +++ b/apps/desktop/src-tauri/src/recording.rs @@ -9,7 +9,10 @@ use cap_recording::{ RecordingError, RecordingMode, feeds::{camera, microphone}, instant_recording, - sources::{CaptureDisplay, CaptureWindow, ScreenCaptureTarget, screen_capture}, + sources::{ + screen_capture, + screen_capture::{CaptureDisplay, CaptureWindow, ScreenCaptureTarget}, + }, studio_recording, }; use cap_rendering::ProjectRecordingsMeta; @@ -79,17 +82,19 @@ impl InProgressRecording { } pub async fn pause(&self) -> Result<(), RecordingError> { - match self { - Self::Instant { handle, .. } => handle.pause().await, - Self::Studio { handle, .. } => handle.pause().await, - } + todo!() + // match self { + // Self::Instant { handle, .. } => handle.pause().await, + // Self::Studio { handle, .. } => handle.pause().await, + // } } pub async fn resume(&self) -> Result<(), String> { - match self { - Self::Instant { handle, .. } => handle.resume().await.map_err(|e| e.to_string()), - Self::Studio { handle, .. } => handle.resume().await.map_err(|e| e.to_string()), - } + todo!() + // match self { + // Self::Instant { handle, .. } => handle.resume().await.map_err(|e| e.to_string()), + // Self::Studio { handle, .. } => handle.resume().await.map_err(|e| e.to_string()), + // } } pub fn recording_dir(&self) -> &PathBuf { @@ -99,7 +104,7 @@ impl InProgressRecording { } } - pub async fn stop(self) -> Result { + pub async fn stop(self) -> anyhow::Result { Ok(match self { Self::Instant { handle, @@ -124,13 +129,21 @@ impl InProgressRecording { }) } - pub async fn cancel(self) -> Result<(), RecordingError> { + pub fn done_fut(&self) -> cap_recording::DoneFut { match self { - Self::Instant { handle, .. } => handle.cancel().await, - Self::Studio { handle, .. } => handle.cancel().await, + Self::Instant { handle, .. } => handle.done_fut(), + Self::Studio { handle, .. } => handle.done_fut(), } } + pub async fn cancel(self) -> Result<(), RecordingError> { + todo!() + // match self { + // Self::Instant { handle, .. } => handle.cancel().await, + // Self::Studio { handle, .. } => handle.cancel().await, + // } + } + pub fn mode(&self) -> RecordingMode { match self { Self::Instant { .. } => RecordingMode::Instant, @@ -147,7 +160,7 @@ pub enum CompletedRecording { video_upload_info: VideoUploadInfo, }, Studio { - recording: studio_recording::CompletedStudioRecording, + recording: studio_recording::CompletedRecording, target_name: String, }, } @@ -406,14 +419,13 @@ pub async fn start_recording( Err(SendError::HandlerError(camera::LockFeedError::NoInput)) => None, Err(e) => return Err(e.to_string()), }; - #[cfg(target_os = "macos")] let shareable_content = crate::platform::get_shareable_content() .await .map_err(|e| format!("GetShareableContent: {e}"))? .ok_or_else(|| format!("GetShareableContent/NotAvailable"))?; - let (actor, actor_done_rx) = match inputs.mode { + let actor = match inputs.mode { RecordingMode::Studio => { let mut builder = studio_recording::Actor::builder( recording_dir.clone(), @@ -434,7 +446,7 @@ pub async fn start_recording( builder = builder.with_mic_feed(mic_feed); } - let (handle, actor_done_rx) = builder + let handle = builder .build( #[cfg(target_os = "macos")] shareable_content, @@ -445,15 +457,12 @@ pub async fn start_recording( e.to_string() })?; - ( - InProgressRecording::Studio { - handle, - target_name, - inputs, - recording_dir: recording_dir.clone(), - }, - actor_done_rx, - ) + InProgressRecording::Studio { + handle, + target_name, + inputs, + recording_dir: recording_dir.clone(), + } } RecordingMode::Instant => { let Some(video_upload_info) = video_upload_info.clone() else { @@ -470,7 +479,7 @@ pub async fn start_recording( builder = builder.with_mic_feed(mic_feed); } - let (handle, actor_done_rx) = builder + let handle = builder .build( #[cfg(target_os = "macos")] shareable_content, @@ -481,23 +490,22 @@ pub async fn start_recording( e.to_string() })?; - ( - InProgressRecording::Instant { - handle, - progressive_upload, - video_upload_info, - target_name, - inputs, - recording_dir: recording_dir.clone(), - }, - actor_done_rx, - ) + InProgressRecording::Instant { + handle, + progressive_upload, + video_upload_info, + target_name, + inputs, + recording_dir: recording_dir.clone(), + } } }; + let done_fut = actor.done_fut(); + state.set_current_recording(actor); - Ok::<_, String>(actor_done_rx) + Ok::<_, String>(done_fut) } }) .await @@ -505,8 +513,8 @@ pub async fn start_recording( } .await; - let actor_done_rx = match spawn_actor_res { - Ok(rx) => rx, + let actor_done_fut = match spawn_actor_res { + Ok(fut) => fut, Err(e) => { let _ = RecordingEvent::Failed { error: e.clone() }.emit(&app); @@ -537,22 +545,25 @@ pub async fn start_recording( let state_mtx = Arc::clone(&state_mtx); async move { fail!("recording::wait_actor_done"); - let res = actor_done_rx.await; + let res = actor_done_fut.await; info!("recording wait actor done: {:?}", &res); match res { - Ok(Ok(_)) => { + Ok(()) => { let _ = finish_upload_tx.send(()); let _ = RecordingEvent::Stopped.emit(&app); } - Ok(Err(e)) => { + Err(e) => { let mut state = state_mtx.write().await; - let _ = RecordingEvent::Failed { error: e.clone() }.emit(&app); + let _ = RecordingEvent::Failed { + error: e.to_string(), + } + .emit(&app); let mut dialog = MessageDialogBuilder::new( app.dialog().clone(), "An error occurred".to_string(), - e, + e.to_string(), ) .kind(tauri_plugin_dialog::MessageDialogKind::Error); @@ -565,10 +576,6 @@ pub async fn start_recording( // this clears the current recording for us handle_recording_end(app, None, &mut state).await.ok(); } - // Actor hasn't errored, it's just finished - v => { - info!("recording actor ended: {v:?}"); - } } } }); @@ -628,7 +635,7 @@ pub async fn restart_recording(app: AppHandle, state: MutableState<'_, App>) -> let inputs = recording.inputs().clone(); - let _ = recording.cancel().await; + // let _ = recording.cancel().await; tokio::time::sleep(Duration::from_millis(1000)).await; @@ -658,7 +665,7 @@ pub async fn delete_recording(app: AppHandle, state: MutableState<'_, App>) -> R CurrentRecordingChanged.emit(&app).ok(); RecordingStopped {}.emit(&app).ok(); - let _ = recording.cancel().await; + // let _ = recording.cancel().await; std::fs::remove_dir_all(&recording_dir).ok(); @@ -1105,7 +1112,7 @@ fn generate_zoom_segments_from_clicks_impl( /// Generates zoom segments based on mouse click events during recording. /// Used during the recording completion process. pub fn generate_zoom_segments_from_clicks( - recording: &studio_recording::CompletedStudioRecording, + recording: &studio_recording::CompletedRecording, recordings: &ProjectRecordingsMeta, ) -> Vec { // Build a temporary RecordingMeta so we can use the common implementation @@ -1162,7 +1169,7 @@ pub fn generate_zoom_segments_for_project( fn project_config_from_recording( app: &AppHandle, - completed_recording: &studio_recording::CompletedStudioRecording, + completed_recording: &studio_recording::CompletedRecording, recordings: &ProjectRecordingsMeta, default_config: Option, ) -> ProjectConfiguration { diff --git a/apps/desktop/src-tauri/src/recording_settings.rs b/apps/desktop/src-tauri/src/recording_settings.rs index 285385c9c..a611f2569 100644 --- a/apps/desktop/src-tauri/src/recording_settings.rs +++ b/apps/desktop/src-tauri/src/recording_settings.rs @@ -1,4 +1,6 @@ -use cap_recording::{RecordingMode, feeds::camera::DeviceOrModelID, sources::ScreenCaptureTarget}; +use cap_recording::{ + RecordingMode, feeds::camera::DeviceOrModelID, sources::screen_capture::ScreenCaptureTarget, +}; use serde_json::json; use tauri::{AppHandle, Wry}; use tauri_plugin_store::StoreExt; diff --git a/apps/desktop/src-tauri/src/thumbnails/mod.rs b/apps/desktop/src-tauri/src/thumbnails/mod.rs index 8ae933573..c4dbeb9da 100644 --- a/apps/desktop/src-tauri/src/thumbnails/mod.rs +++ b/apps/desktop/src-tauri/src/thumbnails/mod.rs @@ -1,4 +1,4 @@ -use cap_recording::sources::{list_displays, list_windows}; +use cap_recording::sources::screen_capture::{list_displays, list_windows}; use serde::{Deserialize, Serialize}; use specta::Type; use tracing::*; diff --git a/crates/enc-avfoundation/src/mp4.rs b/crates/enc-avfoundation/src/mp4.rs index cee416d45..eb2f3f178 100644 --- a/crates/enc-avfoundation/src/mp4.rs +++ b/crates/enc-avfoundation/src/mp4.rs @@ -1,12 +1,10 @@ use cap_media_info::{AudioInfo, VideoInfo}; use cidre::{cm::SampleTimingInfo, objc::Obj, *}; use ffmpeg::frame; -use std::path::PathBuf; +use std::{path::PathBuf, time::Duration}; use tracing::{debug, info}; pub struct MP4Encoder { - #[allow(unused)] - tag: &'static str, #[allow(unused)] config: VideoInfo, asset_writer: arc::R, @@ -44,7 +42,7 @@ pub enum InitError { #[derive(thiserror::Error, Debug)] pub enum QueueVideoFrameError { #[error("AppendError/{0}")] - AppendError(&'static cidre::ns::Exception), + AppendError(arc::R), #[error("Failed")] Failed, } @@ -65,10 +63,9 @@ pub enum QueueAudioFrameError { impl MP4Encoder { pub fn init( - tag: &'static str, + output: PathBuf, video_config: VideoInfo, audio_config: Option, - output: PathBuf, output_height: Option, ) -> Result { debug!("{video_config:#?}"); @@ -174,7 +171,6 @@ impl MP4Encoder { asset_writer.start_writing(); Ok(Self { - tag, config: video_config, audio_input, asset_writer, @@ -196,12 +192,13 @@ impl MP4Encoder { pub fn queue_video_frame( &mut self, frame: &cidre::cm::SampleBuf, + timestamp: Duration, ) -> Result<(), QueueVideoFrameError> { if self.is_paused || !self.video_input.is_ready_for_more_media_data() { return Ok(()); - } + }; - let time = frame.pts(); + let time = cm::Time::new(timestamp.as_millis() as i64, 1_000); let new_pts = self .elapsed_duration @@ -219,7 +216,7 @@ impl MP4Encoder { self.video_input .append_sample_buf(&frame) - .map_err(QueueVideoFrameError::AppendError) + .map_err(|e| QueueVideoFrameError::AppendError(e.retained())) .and_then(|v| v.then_some(()).ok_or(QueueVideoFrameError::Failed))?; self.first_timestamp.get_or_insert(time); @@ -233,7 +230,11 @@ impl MP4Encoder { /// Expects frames with pts values relative to the first frame's pts /// in the timebase of 1 / sample rate - pub fn queue_audio_frame(&mut self, frame: frame::Audio) -> Result<(), QueueAudioFrameError> { + pub fn queue_audio_frame( + &mut self, + frame: frame::Audio, + timestamp: Duration, + ) -> Result<(), QueueAudioFrameError> { if self.is_paused || !self.is_writing { return Ok(()); } @@ -246,6 +247,11 @@ impl MP4Encoder { return Ok(()); } + let time = cm::Time::new( + (timestamp.as_secs_f64() * frame.rate() as f64) as i64, + frame.rate() as i32, + ); + let audio_desc = cat::audio::StreamBasicDesc::common_f32( frame.rate() as f64, frame.channels() as u32, @@ -276,8 +282,6 @@ impl MP4Encoder { let format_desc = cm::AudioFormatDesc::with_asbd(&audio_desc).map_err(QueueAudioFrameError::Setup)?; - let time = cm::Time::new(frame.pts().unwrap_or(0), frame.rate() as i32); - let pts = self .start_time .add(self.elapsed_duration) @@ -357,6 +361,12 @@ impl MP4Encoder { } } +impl Drop for MP4Encoder { + fn drop(&mut self) { + self.finish(); + } +} + #[link(name = "AVFoundation", kind = "framework")] unsafe extern "C" { static AVVideoAverageBitRateKey: &'static cidre::ns::String; diff --git a/crates/enc-ffmpeg/src/audio/aac.rs b/crates/enc-ffmpeg/src/audio/aac.rs index 2f9765790..6418ab879 100644 --- a/crates/enc-ffmpeg/src/audio/aac.rs +++ b/crates/enc-ffmpeg/src/audio/aac.rs @@ -1,3 +1,5 @@ +use std::time::Duration; + use cap_media_info::{AudioInfo, FFRational}; use ffmpeg::{ codec::{context, encoder}, @@ -6,7 +8,10 @@ use ffmpeg::{ threading::Config, }; -use crate::{AudioEncoder, audio::buffered_resampler::BufferedResampler}; +use crate::{ + AudioEncoder, + audio::{base::AudioEncoderBase, buffered_resampler::BufferedResampler}, +}; #[derive(thiserror::Error, Debug)] pub enum AACEncoderError { @@ -16,15 +21,12 @@ pub enum AACEncoderError { CodecNotFound, #[error("Sample rate not supported: {0}")] RateNotSupported(i32), + #[error("Resampler: {0}")] + Resampler(ffmpeg::Error), } pub struct AACEncoder { - #[allow(unused)] - tag: &'static str, - encoder: encoder::Audio, - packet: ffmpeg::Packet, - resampler: BufferedResampler, - stream_index: usize, + base: AudioEncoderBase, } impl AACEncoder { @@ -32,14 +34,12 @@ impl AACEncoder { const SAMPLE_FORMAT: Sample = Sample::F32(Type::Planar); pub fn factory( - tag: &'static str, input_config: AudioInfo, ) -> impl FnOnce(&mut format::context::Output) -> Result { - move |o| Self::init(tag, input_config, o) + move |o| Self::init(input_config, o) } pub fn init( - tag: &'static str, input_config: AudioInfo, output: &mut format::context::Output, ) -> Result { @@ -72,20 +72,8 @@ impl AACEncoder { output_config.sample_format = Self::SAMPLE_FORMAT; output_config.sample_rate = rate as u32; - let resampler = ffmpeg::software::resampler( - ( - input_config.sample_format, - input_config.channel_layout(), - input_config.sample_rate, - ), - ( - output_config.sample_format, - output_config.channel_layout(), - output_config.sample_rate, - ), - ) - .unwrap(); - let resampler = BufferedResampler::new(resampler); + let resampler = BufferedResampler::new(input_config, output_config) + .map_err(AACEncoderError::Resampler)?; encoder.set_bit_rate(Self::OUTPUT_BITRATE); encoder.set_rate(rate); @@ -96,61 +84,34 @@ impl AACEncoder { let encoder = encoder.open()?; let mut output_stream = output.add_stream(codec)?; - let stream_index = output_stream.index(); output_stream.set_time_base(FFRational(1, output_config.rate())); output_stream.set_parameters(&encoder); Ok(Self { - tag, - encoder, - stream_index, - packet: ffmpeg::Packet::empty(), - resampler, + base: AudioEncoderBase::new(encoder, resampler, output_stream.index()), }) } - pub fn queue_frame(&mut self, frame: frame::Audio, output: &mut format::context::Output) { - self.resampler.add_frame(frame); - - let frame_size = self.encoder.frame_size() as usize; - - while let Some(frame) = self.resampler.get_frame(frame_size) { - self.encoder.send_frame(&frame).unwrap(); - - self.process_packets(output); - } - } - - fn process_packets(&mut self, output: &mut format::context::Output) { - while self.encoder.receive_packet(&mut self.packet).is_ok() { - self.packet.set_stream(self.stream_index); - self.packet.rescale_ts( - self.encoder.time_base(), - output.stream(self.stream_index).unwrap().time_base(), - ); - self.packet.write_interleaved(output).unwrap(); - } + pub fn send_frame( + &mut self, + frame: frame::Audio, + timestamp: Duration, + output: &mut format::context::Output, + ) -> Result<(), ffmpeg::Error> { + self.base.send_frame(frame, timestamp, output) } - pub fn finish(&mut self, output: &mut format::context::Output) { - while let Some(frame) = self.resampler.flush(self.encoder.frame_size() as usize) { - self.encoder.send_frame(&frame).unwrap(); - - self.process_packets(output); - } - - self.encoder.send_eof().unwrap(); - - self.process_packets(output); + pub fn finish(&mut self, output: &mut format::context::Output) -> Result<(), ffmpeg::Error> { + self.base.finish(output) } } impl AudioEncoder for AACEncoder { - fn queue_frame(&mut self, frame: frame::Audio, output: &mut format::context::Output) { - self.queue_frame(frame, output); + fn send_frame(&mut self, frame: frame::Audio, output: &mut format::context::Output) { + let _ = self.send_frame(frame, Duration::MAX, output); } fn finish(&mut self, output: &mut format::context::Output) { - self.finish(output); + let _ = self.finish(output); } } diff --git a/crates/enc-ffmpeg/src/audio/audio_encoder.rs b/crates/enc-ffmpeg/src/audio/audio_encoder.rs index 83fc925fb..81d1f0008 100644 --- a/crates/enc-ffmpeg/src/audio/audio_encoder.rs +++ b/crates/enc-ffmpeg/src/audio/audio_encoder.rs @@ -8,6 +8,6 @@ pub trait AudioEncoder { Box::new(self) } - fn queue_frame(&mut self, frame: frame::Audio, output: &mut format::context::Output); + fn send_frame(&mut self, frame: frame::Audio, output: &mut format::context::Output); fn finish(&mut self, output: &mut format::context::Output); } diff --git a/crates/enc-ffmpeg/src/audio/base.rs b/crates/enc-ffmpeg/src/audio/base.rs new file mode 100644 index 000000000..b15954a3d --- /dev/null +++ b/crates/enc-ffmpeg/src/audio/base.rs @@ -0,0 +1,46 @@ +use super::buffered_resampler::BufferedResampler; +use crate::base::EncoderBase; +use ffmpeg::{codec::encoder, format, frame}; +use std::time::Duration; + +pub struct AudioEncoderBase { + inner: EncoderBase, + encoder: encoder::Audio, + resampler: BufferedResampler, +} + +impl AudioEncoderBase { + pub fn new(encoder: encoder::Audio, resampler: BufferedResampler, stream_index: usize) -> Self { + Self { + inner: EncoderBase::new(stream_index), + encoder, + resampler, + } + } + + pub fn send_frame( + &mut self, + mut frame: frame::Audio, + timestamp: Duration, + output: &mut format::context::Output, + ) -> Result<(), ffmpeg::Error> { + self.inner + .update_pts(&mut frame, timestamp, &mut self.encoder); + + self.resampler.add_frame(frame); + + while let Some(frame) = self.resampler.get_frame(self.encoder.frame_size() as usize) { + self.inner.send_frame(&frame, output, &mut self.encoder)?; + } + + Ok(()) + } + + pub fn finish(&mut self, output: &mut format::context::Output) -> Result<(), ffmpeg::Error> { + while let Some(frame) = self.resampler.flush(self.encoder.frame_size() as usize) { + self.inner.send_frame(&frame, output, &mut self.encoder)?; + } + + self.inner.process_eof(output, &mut self.encoder) + } +} diff --git a/crates/enc-ffmpeg/src/audio/buffered_resampler.rs b/crates/enc-ffmpeg/src/audio/buffered_resampler.rs index c7706011d..da386b955 100644 --- a/crates/enc-ffmpeg/src/audio/buffered_resampler.rs +++ b/crates/enc-ffmpeg/src/audio/buffered_resampler.rs @@ -1,5 +1,6 @@ use std::collections::VecDeque; +use cap_media_info::AudioInfo; use ffmpeg::software::resampling; /// Consumes audio frames, resmaples them, buffers the results, @@ -16,13 +17,18 @@ pub struct BufferedResampler { } impl BufferedResampler { - pub fn new(resampler: ffmpeg::software::resampling::Context) -> Self { - Self { + pub fn new(from: AudioInfo, to: AudioInfo) -> Result { + let resampler = ffmpeg::software::resampler( + (from.sample_format, from.channel_layout(), from.sample_rate), + (to.sample_format, to.channel_layout(), to.sample_rate), + )?; + + Ok(Self { resampler, buffer: VecDeque::new(), sample_index: 0, min_next_pts: None, - } + }) } fn remaining_samples(&self) -> usize { diff --git a/crates/enc-ffmpeg/src/audio/mod.rs b/crates/enc-ffmpeg/src/audio/mod.rs index 184b460cd..377eb21df 100644 --- a/crates/enc-ffmpeg/src/audio/mod.rs +++ b/crates/enc-ffmpeg/src/audio/mod.rs @@ -1,4 +1,5 @@ mod audio_encoder; +mod base; mod buffered_resampler; pub use audio_encoder::*; diff --git a/crates/enc-ffmpeg/src/audio/opus.rs b/crates/enc-ffmpeg/src/audio/opus.rs index 70dde56cc..bed9ffbb3 100644 --- a/crates/enc-ffmpeg/src/audio/opus.rs +++ b/crates/enc-ffmpeg/src/audio/opus.rs @@ -1,3 +1,5 @@ +use std::time::Duration; + use cap_media_info::{AudioInfo, FFRational}; use ffmpeg::{ codec::{context, encoder}, @@ -6,17 +8,11 @@ use ffmpeg::{ threading::Config, }; -use crate::audio::buffered_resampler::BufferedResampler; - use super::AudioEncoder; +use crate::audio::{base::AudioEncoderBase, buffered_resampler::BufferedResampler}; pub struct OpusEncoder { - #[allow(unused)] - tag: &'static str, - encoder: encoder::Audio, - packet: ffmpeg::Packet, - resampler: BufferedResampler, - stream_index: usize, + base: AudioEncoderBase, } #[derive(thiserror::Error, Debug)] @@ -27,6 +23,8 @@ pub enum OpusEncoderError { CodecNotFound, #[error("Sample rate not supported: {0}")] RateNotSupported(i32), + #[error("Resampler: {0}")] + Resampler(ffmpeg::Error), } impl OpusEncoder { @@ -34,14 +32,12 @@ impl OpusEncoder { const SAMPLE_FORMAT: Sample = Sample::F32(Type::Packed); pub fn factory( - tag: &'static str, input_config: AudioInfo, ) -> impl FnOnce(&mut format::context::Output) -> Result { - move |o| Self::init(tag, input_config, o) + move |o| Self::init(input_config, o) } pub fn init( - tag: &'static str, input_config: AudioInfo, output: &mut format::context::Output, ) -> Result { @@ -74,20 +70,8 @@ impl OpusEncoder { output_config.sample_format = Self::SAMPLE_FORMAT; output_config.sample_rate = rate as u32; - let resampler = ffmpeg::software::resampler( - ( - input_config.sample_format, - input_config.channel_layout(), - input_config.sample_rate, - ), - ( - output_config.sample_format, - output_config.channel_layout(), - output_config.sample_rate, - ), - ) - .unwrap(); - let resampler = BufferedResampler::new(resampler); + let resampler = BufferedResampler::new(input_config, output_config) + .map_err(OpusEncoderError::Resampler)?; encoder.set_bit_rate(Self::OUTPUT_BITRATE); encoder.set_rate(rate); @@ -103,60 +87,30 @@ impl OpusEncoder { output_stream.set_parameters(&encoder); Ok(Self { - tag, - encoder, - stream_index, - packet: ffmpeg::Packet::empty(), - resampler, + base: AudioEncoderBase::new(encoder, resampler, stream_index), }) } - pub fn input_time_base(&self) -> FFRational { - self.encoder.time_base() - } - - pub fn queue_frame(&mut self, frame: frame::Audio, output: &mut format::context::Output) { - self.resampler.add_frame(frame); - - let frame_size = self.encoder.frame_size() as usize; - - while let Some(frame) = self.resampler.get_frame(frame_size) { - self.encoder.send_frame(&frame).unwrap(); - - self.process_packets(output); - } - } - - fn process_packets(&mut self, output: &mut format::context::Output) { - while self.encoder.receive_packet(&mut self.packet).is_ok() { - self.packet.set_stream(self.stream_index); - self.packet.rescale_ts( - self.encoder.time_base(), - output.stream(self.stream_index).unwrap().time_base(), - ); - self.packet.write_interleaved(output).unwrap(); - } + pub fn queue_frame( + &mut self, + frame: frame::Audio, + timestamp: Duration, + output: &mut format::context::Output, + ) -> Result<(), ffmpeg::Error> { + self.base.send_frame(frame, timestamp, output) } - pub fn finish(&mut self, output: &mut format::context::Output) { - while let Some(frame) = self.resampler.flush(self.encoder.frame_size() as usize) { - self.encoder.send_frame(&frame).unwrap(); - - self.process_packets(output); - } - - self.encoder.send_eof().unwrap(); - - self.process_packets(output); + pub fn finish(&mut self, output: &mut format::context::Output) -> Result<(), ffmpeg::Error> { + self.base.finish(output) } } impl AudioEncoder for OpusEncoder { - fn queue_frame(&mut self, frame: frame::Audio, output: &mut format::context::Output) { - self.queue_frame(frame, output); + fn send_frame(&mut self, frame: frame::Audio, output: &mut format::context::Output) { + let _ = self.queue_frame(frame, Duration::MAX, output); } fn finish(&mut self, output: &mut format::context::Output) { - self.finish(output); + let _ = self.finish(output); } } diff --git a/crates/enc-ffmpeg/src/base.rs b/crates/enc-ffmpeg/src/base.rs new file mode 100644 index 000000000..22c92384f --- /dev/null +++ b/crates/enc-ffmpeg/src/base.rs @@ -0,0 +1,88 @@ +use std::time::Duration; + +use ffmpeg::{ + Packet, + codec::encoder, + format::{self}, + frame, +}; + +pub struct EncoderBase { + packet: ffmpeg::Packet, + stream_index: usize, + first_pts: Option, +} + +impl EncoderBase { + pub(crate) fn new(stream_index: usize) -> Self { + Self { + packet: Packet::empty(), + first_pts: None, + stream_index, + } + } + + pub fn update_pts( + &mut self, + frame: &mut frame::Frame, + timestamp: Duration, + encoder: &mut encoder::encoder::Encoder, + ) { + if timestamp != Duration::MAX { + let time_base = encoder.time_base(); + let rate = time_base.denominator() as f64 / time_base.numerator() as f64; + + let pts = (timestamp.as_secs_f64() * rate).round() as i64; + let first_pts = self.first_pts.get_or_insert(pts); + + frame.set_pts(Some(pts - *first_pts)); + } else { + let Some(pts) = frame.pts() else { + tracing::error!("Frame has no pts"); + return; + }; + + let first_pts = self.first_pts.get_or_insert(pts); + + frame.set_pts(Some(pts - *first_pts)); + } + } + + pub fn send_frame( + &mut self, + frame: &frame::Frame, + output: &mut format::context::Output, + encoder: &mut encoder::encoder::Encoder, + ) -> Result<(), ffmpeg::Error> { + encoder.send_frame(frame)?; + + self.process_packets(output, encoder) + } + + fn process_packets( + &mut self, + output: &mut format::context::Output, + encoder: &mut encoder::encoder::Encoder, + ) -> Result<(), ffmpeg::Error> { + while encoder.receive_packet(&mut self.packet).is_ok() { + self.packet.set_stream(self.stream_index); + self.packet.rescale_ts( + encoder.time_base(), + output.stream(self.stream_index).unwrap().time_base(), + ); + self.packet.write_interleaved(output)?; + } + + Ok(()) + } + + pub fn process_eof( + &mut self, + output: &mut format::context::Output, + encoder: &mut encoder::encoder::Encoder, + ) -> Result<(), ffmpeg::Error> { + encoder.send_eof()?; + + self.process_packets(output, encoder) + } +} diff --git a/crates/enc-ffmpeg/src/lib.rs b/crates/enc-ffmpeg/src/lib.rs index d8d72d26d..08812ebc6 100644 --- a/crates/enc-ffmpeg/src/lib.rs +++ b/crates/enc-ffmpeg/src/lib.rs @@ -1,4 +1,5 @@ mod audio; +mod base; pub use audio::*; mod video; diff --git a/crates/enc-ffmpeg/src/mux/mp4.rs b/crates/enc-ffmpeg/src/mux/mp4.rs index 577de6129..e129afb51 100644 --- a/crates/enc-ffmpeg/src/mux/mp4.rs +++ b/crates/enc-ffmpeg/src/mux/mp4.rs @@ -1,6 +1,6 @@ use cap_media_info::RawVideoFormat; use ffmpeg::{format, frame}; -use std::path::PathBuf; +use std::{path::PathBuf, time::Duration}; use tracing::{info, trace}; use crate::{ @@ -70,12 +70,12 @@ impl MP4File { RawVideoFormat::YUYV420 } - pub fn queue_video_frame(&mut self, frame: frame::Video) { + pub fn queue_video_frame(&mut self, frame: frame::Video, timestamp: Duration) { if self.is_finished { return; } - self.video.queue_frame(frame, &mut self.output); + self.video.queue_frame(frame, timestamp, &mut self.output); } pub fn queue_audio_frame(&mut self, frame: frame::Audio) { @@ -87,7 +87,7 @@ impl MP4File { return; }; - audio.queue_frame(frame, &mut self.output); + audio.send_frame(frame, &mut self.output); } pub fn finish(&mut self) { @@ -121,6 +121,12 @@ impl MP4File { } } +impl Drop for MP4File { + fn drop(&mut self) { + self.finish(); + } +} + pub struct MP4Input { pub video: frame::Video, pub audio: Option, diff --git a/crates/enc-ffmpeg/src/mux/ogg.rs b/crates/enc-ffmpeg/src/mux/ogg.rs index 1f63a92e5..03073aa30 100644 --- a/crates/enc-ffmpeg/src/mux/ogg.rs +++ b/crates/enc-ffmpeg/src/mux/ogg.rs @@ -1,11 +1,12 @@ use ffmpeg::{format, frame}; -use std::path::PathBuf; +use std::{path::PathBuf, time::Duration}; use crate::audio::{OpusEncoder, OpusEncoderError}; pub struct OggFile { encoder: OpusEncoder, output: format::context::Output, + finished: bool, } impl OggFile { @@ -21,19 +22,32 @@ impl OggFile { // make sure this happens after adding all encoders! output.write_header()?; - Ok(Self { encoder, output }) + Ok(Self { + encoder, + output, + finished: false, + }) } pub fn encoder(&self) -> &OpusEncoder { &self.encoder } - pub fn queue_frame(&mut self, frame: frame::Audio) { - self.encoder.queue_frame(frame, &mut self.output); + pub fn queue_frame(&mut self, frame: frame::Audio, timestamp: Duration) { + let _ = self.encoder.queue_frame(frame, timestamp, &mut self.output); } pub fn finish(&mut self) { - self.encoder.finish(&mut self.output); - self.output.write_trailer().unwrap(); + if !self.finished { + let _ = self.encoder.finish(&mut self.output); + self.output.write_trailer().unwrap(); + self.finished = true; + } + } +} + +impl Drop for OggFile { + fn drop(&mut self) { + self.finish(); } } diff --git a/crates/enc-ffmpeg/src/video/h264.rs b/crates/enc-ffmpeg/src/video/h264.rs index 7e2f8ad8b..a0eb78e6a 100644 --- a/crates/enc-ffmpeg/src/video/h264.rs +++ b/crates/enc-ffmpeg/src/video/h264.rs @@ -10,8 +10,9 @@ use ffmpeg::{ }; use tracing::{debug, error}; +use crate::base::EncoderBase; + pub struct H264EncoderBuilder { - name: &'static str, bpp: f32, input_config: VideoInfo, preset: H264Preset, @@ -37,9 +38,8 @@ pub enum H264EncoderError { impl H264EncoderBuilder { pub const QUALITY_BPP: f32 = 0.3; - pub fn new(name: &'static str, input_config: VideoInfo) -> Self { + pub fn new(input_config: VideoInfo) -> Self { Self { - name, input_config, bpp: Self::QUALITY_BPP, preset: H264Preset::Ultrafast, @@ -119,43 +119,46 @@ impl H264EncoderBuilder { encoder.set_bit_rate(bitrate); encoder.set_max_bit_rate(bitrate); - let video_encoder = encoder.open_with(encoder_options)?; + let encoder = encoder.open_with(encoder_options)?; let mut output_stream = output.add_stream(codec)?; let stream_index = output_stream.index(); output_stream.set_time_base((1, H264Encoder::TIME_BASE)); output_stream.set_rate(input_config.frame_rate); - output_stream.set_parameters(&video_encoder); + output_stream.set_parameters(&encoder); Ok(H264Encoder { - tag: self.name, - encoder: video_encoder, - stream_index, + base: EncoderBase::new(stream_index), + encoder, config: self.input_config, converter, - packet: ffmpeg::Packet::empty(), }) } } pub struct H264Encoder { - #[allow(unused)] - tag: &'static str, + base: EncoderBase, encoder: encoder::Video, config: VideoInfo, converter: Option, - stream_index: usize, - packet: ffmpeg::Packet, } impl H264Encoder { const TIME_BASE: i32 = 90000; - pub fn builder(name: &'static str, input_config: VideoInfo) -> H264EncoderBuilder { - H264EncoderBuilder::new(name, input_config) + pub fn builder(input_config: VideoInfo) -> H264EncoderBuilder { + H264EncoderBuilder::new(input_config) } - pub fn queue_frame(&mut self, frame: frame::Video, output: &mut format::context::Output) { + pub fn queue_frame( + &mut self, + mut frame: frame::Video, + timestamp: Duration, + output: &mut format::context::Output, + ) { + self.base + .update_pts(&mut frame, timestamp, &mut self.encoder); + let frame = if let Some(converter) = &mut self.converter { let mut new_frame = frame::Video::empty(); match converter.run(&frame, &mut new_frame) { @@ -178,39 +181,17 @@ impl H264Encoder { frame }; - if let Err(e) = self.encoder.send_frame(&frame) { + if let Err(e) = self.base.send_frame(&frame, output, &mut self.encoder) { tracing::error!("Failed to send frame to encoder: {:?}", e); return; } - - self.process_frame(output); - } - - fn process_frame(&mut self, output: &mut format::context::Output) { - while self.encoder.receive_packet(&mut self.packet).is_ok() { - self.packet.set_stream(self.stream_index); - self.packet.rescale_ts( - self.config.time_base, - output.stream(self.stream_index).unwrap().time_base(), - ); - if let Err(e) = self.packet.write_interleaved(output) { - tracing::error!("Failed to write packet: {:?}", e); - break; - } - } } pub fn finish(&mut self, output: &mut format::context::Output) { - if let Err(e) = self.encoder.send_eof() { + if let Err(e) = self.base.process_eof(output, &mut self.encoder) { tracing::error!("Failed to send EOF to encoder: {:?}", e); return; } - self.process_frame(output); - } - - pub fn get_pts(&self, duration: Duration) -> i64 { - (duration.as_secs_f32() * self.config.time_base.denominator() as f32 - / self.config.time_base.numerator() as f32) as i64 } } diff --git a/crates/enc-mediafoundation/examples/cli.rs b/crates/enc-mediafoundation/examples/cli.rs index 6e5f97f1c..72f20574e 100644 --- a/crates/enc-mediafoundation/examples/cli.rs +++ b/crates/enc-mediafoundation/examples/cli.rs @@ -6,14 +6,14 @@ fn main() { #[cfg(windows)] mod win { use args::Args; - use cap_enc_mediafoundation::{ - d3d::create_d3d_device, - media::MF_VERSION, - video::{H264Encoder, InputSample, SampleWriter}, - }; + use cap_enc_mediafoundation::{d3d::create_d3d_device, media::MF_VERSION, video::H264Encoder}; use clap::Parser; use scap_targets::Display; - use std::{path::Path, sync::Arc, time::Duration}; + use std::{ + path::Path, + sync::{Arc, atomic::AtomicBool}, + time::Duration, + }; use windows::{ Foundation::{Metadata::ApiInformation, TimeSpan}, Graphics::Capture::GraphicsCaptureSession, @@ -34,7 +34,6 @@ mod win { output_path: &str, bit_rate: u32, frame_rate: u32, - resolution: Resolution, verbose: bool, wait_for_debugger: bool, ) -> Result<()> { @@ -77,11 +76,7 @@ mod win { .unwrap(); // Resolve encoding settings - let resolution = if let Some(resolution) = resolution.get_size() { - resolution - } else { - item.Size()? - }; + let resolution = item.Size()?; let bit_rate = bit_rate * 1000000; // Start the recording @@ -107,7 +102,7 @@ mod win { Duration: frame_time.Duration - first_time.Duration, }; - let _ = frame_tx.send(Some(frame)); + let _ = frame_tx.send(Some((frame.texture().clone(), timestamp))); Ok(()) } @@ -132,56 +127,35 @@ mod win { let output_path = std::env::current_dir().unwrap().join(output_path); - let sample_writer = Arc::new(SampleWriter::new(output_path.as_path())?); - - let stream_index = sample_writer.add_stream(&video_encoder.output_type())?; + // let sample_writer = Arc::new(SampleWriter::new(output_path.as_path())?); capturer.start()?; - sample_writer.start()?; + + let should_stop_encoder = Arc::new(AtomicBool::new(false)); std::thread::spawn({ - let sample_writer = sample_writer.clone(); + // let sample_writer = sample_writer.clone(); + let should_stop_encoder = should_stop_encoder.clone(); move || { unsafe { MFStartup(MF_VERSION, MFSTARTUP_FULL) }.unwrap(); - video_encoder.start().unwrap(); - - while let Ok(e) = video_encoder.get_event() { - match e { - MediaFoundation::METransformNeedInput => { - let Some(frame) = frame_rx.recv().unwrap() else { - break; - }; - - if video_encoder - .handle_needs_input( - frame.texture(), - frame.inner().SystemRelativeTime().unwrap(), - ) - .is_err() - { - break; - } - } - MediaFoundation::METransformHaveOutput => { - if let Some(output_sample) = - video_encoder.handle_has_output().unwrap() - { - sample_writer.write(stream_index, &output_sample).unwrap(); - } - } - _ => {} - } - } - - video_encoder.finish().unwrap(); + video_encoder + .run( + should_stop_encoder, + || Ok(frame_rx.recv().ok().flatten()), + |sample| { + dbg!(sample); + Ok(()) + // sample_writer.write(stream_index, &output_sample).unwrap() + }, + ) + .unwrap(); } }); pause(); capturer.stop().unwrap(); - sample_writer.stop()?; } Ok(()) @@ -197,20 +171,12 @@ mod win { let args = Args::parse(); - if let Some(command) = args.command { - match command { - args::Commands::EnumEncoders => enum_encoders().unwrap(), - } - return; - } - let monitor_index: usize = args.display; let output_path = args.output_file.as_str(); let verbose = args.verbose; let wait_for_debugger = args.wait_for_debugger; let bit_rate: u32 = args.bit_rate; let frame_rate: u32 = args.frame_rate; - let resolution: Resolution = args.resolution; // Validate some of the params if !validate_path(output_path) { @@ -222,7 +188,6 @@ mod win { output_path, bit_rate, frame_rate, - resolution, verbose | wait_for_debugger, wait_for_debugger, ); @@ -238,18 +203,6 @@ mod win { std::io::Read::read(&mut std::io::stdin(), &mut [0]).unwrap(); } - fn enum_encoders() -> Result<()> { - let encoder_devices = VideoEncoderDevice::enumerate()?; - if encoder_devices.is_empty() { - exit_with_error("No hardware H264 encoders found!"); - } - println!("Encoders ({}):", encoder_devices.len()); - for (i, encoder_device) in encoder_devices.iter().enumerate() { - println!(" {} - {}", i, encoder_device.display_name()); - } - Ok(()) - } - fn validate_path>(path: P) -> bool { let path = path.as_ref(); let mut valid = true; @@ -285,8 +238,6 @@ mod win { mod args { use clap::{Parser, Subcommand}; - use cap_enc_mediafoundation::resolution::Resolution; - #[derive(Parser, Debug)] #[clap(author, version, about, long_about = None)] pub struct Args { @@ -302,10 +253,6 @@ mod win { #[clap(short, long, default_value_t = 60)] pub frame_rate: u32, - /// The resolution you would like to encode at: native, 720p, 1080p, 2160p, or 4320p. - #[clap(short, long, default_value_t = Resolution::Native)] - pub resolution: Resolution, - /// The index of the encoder you'd like to use to record (use enum-encoders command for a list of encoders and their indices). #[clap(short, long, default_value_t = 0)] pub encoder: usize, diff --git a/crates/enc-mediafoundation/src/video/h264.rs b/crates/enc-mediafoundation/src/video/h264.rs index 493634853..30bf47f94 100644 --- a/crates/enc-mediafoundation/src/video/h264.rs +++ b/crates/enc-mediafoundation/src/video/h264.rs @@ -1,3 +1,12 @@ +use crate::{ + media::{MFSetAttributeRatio, MFSetAttributeSize}, + mft::EncoderDevice, + video::{NewVideoProcessorError, VideoProcessor}, +}; +use std::sync::{ + Arc, + atomic::{AtomicBool, Ordering}, +}; use windows::{ Foundation::TimeSpan, Graphics::SizeInt32, @@ -8,9 +17,9 @@ use windows::{ Dxgi::Common::{DXGI_FORMAT, DXGI_FORMAT_NV12}, }, Media::MediaFoundation::{ - IMFAttributes, IMFDXGIDeviceManager, IMFMediaEventGenerator, IMFMediaType, IMFSample, - IMFTransform, MEDIA_EVENT_GENERATOR_GET_EVENT_FLAGS, MF_E_INVALIDMEDIATYPE, - MF_E_NO_MORE_TYPES, MF_E_TRANSFORM_TYPE_NOT_SET, MF_EVENT_TYPE, + self, IMFAttributes, IMFDXGIDeviceManager, IMFMediaEventGenerator, IMFMediaType, + IMFSample, IMFTransform, MF_E_INVALIDMEDIATYPE, MF_E_NO_MORE_TYPES, + MF_E_TRANSFORM_TYPE_NOT_SET, MF_EVENT_FLAG_NONE, MF_EVENT_TYPE, MF_MT_ALL_SAMPLES_INDEPENDENT, MF_MT_AVG_BITRATE, MF_MT_FRAME_RATE, MF_MT_FRAME_SIZE, MF_MT_INTERLACE_MODE, MF_MT_MAJOR_TYPE, MF_MT_PIXEL_ASPECT_RATIO, MF_MT_SUBTYPE, MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, MF_TRANSFORM_ASYNC_UNLOCK, @@ -26,12 +35,6 @@ use windows::{ core::{Error, Interface}, }; -use crate::{ - media::{MFSetAttributeRatio, MFSetAttributeSize}, - mft::EncoderDevice, - video::{NewVideoProcessorError, VideoProcessor}, -}; - pub struct VideoEncoderOutputSample { sample: IMFSample, } @@ -379,21 +382,12 @@ impl H264Encoder { &self.output_type } - pub fn finish(&self) -> windows::core::Result<()> { - unsafe { - self.transform - .ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, 0)?; - self.transform - .ProcessMessage(MFT_MESSAGE_NOTIFY_END_OF_STREAM, 0)?; - self.transform - .ProcessMessage(MFT_MESSAGE_NOTIFY_END_STREAMING, 0)?; - self.transform - .ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, 0)?; - } - Ok(()) - } - - pub fn start(&self) -> windows::core::Result<()> { + pub fn run( + &mut self, + should_stop: Arc, + mut get_frame: impl FnMut() -> windows::core::Result>, + mut on_sample: impl FnMut(IMFSample) -> windows::core::Result<()>, + ) -> windows::core::Result<()> { unsafe { self.transform .ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, 0)?; @@ -401,70 +395,66 @@ impl H264Encoder { .ProcessMessage(MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, 0)?; self.transform .ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0)?; - } - Ok(()) - } - - pub fn get_event(&self) -> windows::core::Result { - let event = unsafe { - self.event_generator - .GetEvent(MEDIA_EVENT_GENERATOR_GET_EVENT_FLAGS(0))? - }; - - Ok(MF_EVENT_TYPE(unsafe { event.GetType()? } as i32)) - } + let mut should_exit = false; + while !should_exit { + let event = self.event_generator.GetEvent(MF_EVENT_FLAG_NONE)?; + + let event_type = MF_EVENT_TYPE(event.GetType()? as i32); + match event_type { + MediaFoundation::METransformNeedInput => { + should_exit = true; + if !should_stop.load(Ordering::SeqCst) { + if let Some((texture, timestamp)) = get_frame()? { + self.video_processor.process_texture(&texture)?; + let input_buffer = { + MFCreateDXGISurfaceBuffer( + &ID3D11Texture2D::IID, + self.video_processor.output_texture(), + 0, + false, + )? + }; + let mf_sample = MFCreateSample()?; + mf_sample.AddBuffer(&input_buffer)?; + mf_sample.SetSampleTime(timestamp.Duration)?; + self.transform + .ProcessInput(self.input_stream_id, &mf_sample, 0)?; + should_exit = false; + } + } + } + MediaFoundation::METransformHaveOutput => { + let mut status = 0; + let output_buffer = MFT_OUTPUT_DATA_BUFFER { + dwStreamID: self.output_stream_id, + ..Default::default() + }; + + let sample = { + let mut output_buffers = [output_buffer]; + self.transform + .ProcessOutput(0, &mut output_buffers, &mut status)?; + output_buffers[0].pSample.as_ref().unwrap().clone() + }; + + on_sample(sample)?; + } + _ => { + panic!("Unknown media event type: {}", event_type.0); + } + } + } - pub fn handle_needs_input( - &mut self, - texture: &ID3D11Texture2D, - timestamp: TimeSpan, - ) -> Result<(), HandleNeedsInputError> { - self.video_processor - .process_texture(texture) - .map_err(HandleNeedsInputError::ProcessTexture)?; - - let first_time = self.first_time.get_or_insert(timestamp); - - let input_buffer = unsafe { - MFCreateDXGISurfaceBuffer( - &ID3D11Texture2D::IID, - self.video_processor.output_texture(), - 0, - false, - ) - .map_err(HandleNeedsInputError::CreateSurfaceBuffer)? - }; - let mf_sample = unsafe { MFCreateSample().map_err(HandleNeedsInputError::CreateSample)? }; - unsafe { - mf_sample - .AddBuffer(&input_buffer) - .map_err(HandleNeedsInputError::AddBuffer)?; - mf_sample - .SetSampleTime(timestamp.Duration - first_time.Duration) - .map_err(HandleNeedsInputError::SetSampleTime)?; self.transform - .ProcessInput(self.input_stream_id, &mf_sample, 0) - .map_err(HandleNeedsInputError::ProcessInput)?; - }; - Ok(()) - } - - pub fn handle_has_output(&mut self) -> windows::core::Result> { - let mut status = 0; - let output_buffer = MFT_OUTPUT_DATA_BUFFER { - dwStreamID: self.output_stream_id, - ..Default::default() - }; - - let sample = unsafe { - let mut output_buffers = [output_buffer]; + .ProcessMessage(MFT_MESSAGE_NOTIFY_END_OF_STREAM, 0)?; self.transform - .ProcessOutput(0, &mut output_buffers, &mut status)?; - output_buffers[0].pSample.as_ref().cloned() - }; + .ProcessMessage(MFT_MESSAGE_NOTIFY_END_STREAMING, 0)?; + self.transform + .ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, 0)?; + } - Ok(sample) + Ok(()) } } diff --git a/crates/export/src/mp4.rs b/crates/export/src/mp4.rs index 3b744dfd3..47a389a4b 100644 --- a/crates/export/src/mp4.rs +++ b/crates/export/src/mp4.rs @@ -79,13 +79,13 @@ impl Mp4ExportSettings { "output", base.output_path.clone(), |o| { - H264Encoder::builder("output_video", video_info) + H264Encoder::builder(video_info) .with_bpp(self.compression.bits_per_pixel()) .build(o) }, |o| { has_audio.then(|| { - AACEncoder::init("output_audio", AudioRenderer::info(), o) + AACEncoder::init(AudioRenderer::info(), o) .map(|v| v.boxed()) .map_err(Into::into) }) @@ -97,7 +97,10 @@ impl Mp4ExportSettings { let mut encoded_frames = 0; while let Ok(frame) = frame_rx.recv() { - encoder.queue_video_frame(frame.video); + encoder.queue_video_frame( + frame.video, + Duration::from_secs_f32(encoded_frames as f32 / fps as f32), + ); encoded_frames += 1; if let Some(audio) = frame.audio { encoder.queue_audio_frame(audio); diff --git a/crates/mediafoundation-ffmpeg/src/h264.rs b/crates/mediafoundation-ffmpeg/src/h264.rs index 29968dc12..4cab09a6c 100644 --- a/crates/mediafoundation-ffmpeg/src/h264.rs +++ b/crates/mediafoundation-ffmpeg/src/h264.rs @@ -1,6 +1,6 @@ use cap_mediafoundation_utils::*; use ffmpeg::{Rational, ffi::av_rescale_q, packet}; -use tracing::info; +use tracing::{info, trace}; use windows::Win32::Media::MediaFoundation::{IMFSample, MFSampleExtension_CleanPoint}; /// Configuration for H264 muxing diff --git a/crates/recording/Cargo.toml b/crates/recording/Cargo.toml index d46cbfe88..83558ee34 100644 --- a/crates/recording/Cargo.toml +++ b/crates/recording/Cargo.toml @@ -22,13 +22,15 @@ cap-camera-ffmpeg = { path = "../camera-ffmpeg" } cap-enc-ffmpeg = { path = "../enc-ffmpeg" } cap-timestamp = { path = "../timestamp" } -specta.workspace = true -tokio.workspace = true -flume.workspace = true -thiserror.workspace = true +specta = { workspace = true } +tokio = { workspace = true } +flume = { workspace = true } +thiserror = { workspace = true } ffmpeg = { workspace = true } - serde = { workspace = true } +futures = { workspace = true } +anyhow = { workspace = true } + serde_json = "1" chrono = "0.4.38" tracing.workspace = true @@ -37,7 +39,6 @@ image = "0.25.2" either = "1.13.0" tracing-subscriber = { version = "0.3.19" } relative-path = "1.9.3" -futures = { workspace = true } tokio-util = "0.7.15" sha2 = "0.10.9" hex = "0.4.3" @@ -49,25 +50,27 @@ inquire = "0.7.5" replace_with = "0.1.8" [target.'cfg(target_os = "macos")'.dependencies] +cidre = { workspace = true } screencapturekit = "0.3.5" cocoa = "0.26.0" objc = "0.2.7" -cidre = { workspace = true } objc2-app-kit = "0.3.1" + scap-screencapturekit = { path = "../scap-screencapturekit" } cap-enc-avfoundation = { path = "../enc-avfoundation" } [target.'cfg(target_os = "windows")'.dependencies] -cap-enc-mediafoundation = { path = "../enc-mediafoundation" } -cap-mediafoundation-ffmpeg = { path = "../mediafoundation-ffmpeg" } -cap-mediafoundation-utils = { path = "../mediafoundation-utils" } -cap-camera-windows = { path = "../camera-windows" } windows = { workspace = true, features = [ "Win32_Foundation", "Win32_Graphics_Gdi", "Win32_UI_WindowsAndMessaging", "Win32_System_Performance", ] } + +cap-enc-mediafoundation = { path = "../enc-mediafoundation" } +cap-mediafoundation-ffmpeg = { path = "../mediafoundation-ffmpeg" } +cap-mediafoundation-utils = { path = "../mediafoundation-utils" } +cap-camera-windows = { path = "../camera-windows" } scap-direct3d = { path = "../scap-direct3d" } scap-ffmpeg = { path = "../scap-ffmpeg" } scap-cpal = { path = "../scap-cpal" } diff --git a/crates/recording/examples/recording-cli.rs b/crates/recording/examples/recording-cli.rs index 54c6c0e36..5564530db 100644 --- a/crates/recording/examples/recording-cli.rs +++ b/crates/recording/examples/recording-cli.rs @@ -1,8 +1,7 @@ -use cap_recording::{feeds::microphone, screen_capture::ScreenCaptureTarget, *}; -use kameo::Actor; +use cap_recording::{screen_capture::ScreenCaptureTarget, *}; use scap_targets::Display; use std::time::Duration; -use tracing::info; +use tracing::*; #[tokio::main] pub async fn main() { @@ -25,9 +24,7 @@ pub async fn main() { info!("Recording to directory '{}'", dir.path().display()); - // let camera_info = cap_camera::list_cameras() - // .find(|c| c.display_name().contains("NVIDIA")) - // .unwrap(); + // let camera_info = cap_camera::list_cameras().next().unwrap(); // let camera_feed = CameraFeed::spawn(CameraFeed::default()); @@ -60,7 +57,7 @@ pub async fn main() { tokio::time::sleep(Duration::from_millis(10)).await; - let (handle, _ready_rx) = instant_recording::Actor::builder( + let handle = instant_recording::Actor::builder( dir.path().into(), ScreenCaptureTarget::Display { id: Display::primary().id(), @@ -70,13 +67,24 @@ pub async fn main() { // .with_camera_feed(std::sync::Arc::new( // camera_feed.ask(feeds::camera::Lock).await.unwrap(), // )) - .build() + .build( + #[cfg(target_os = "macos")] + cidre::sc::ShareableContent::current().await.unwrap(), + ) .await .unwrap(); - tokio::time::sleep(Duration::from_secs(10)).await; - - let _ = handle.stop().await; + let _ = tokio::select!( + _ = tokio::time::sleep(Duration::from_secs(5)) => { + trace!("Sleep done"); + let _ = handle.stop().await; + } + res = handle.done_fut() => { + debug!("{res:?}"); + } + ); + + info!("Recording finished"); std::mem::forget(dir); } diff --git a/crates/recording/examples/screen_capture.rs b/crates/recording/examples/screen_capture.rs index b8f0ecf9d..6f8b71e65 100644 --- a/crates/recording/examples/screen_capture.rs +++ b/crates/recording/examples/screen_capture.rs @@ -1,6 +1,6 @@ use cap_recording::{ - pipeline::{control::PipelineControlSignal, task::PipelineSourceTask}, - sources::{CMSampleBufferCapture, ScreenCaptureSource, ScreenCaptureTarget}, + pipeline::control::PipelineControlSignal, + sources::{CMSampleBufferCapture, ScreenCaptureConfig, ScreenCaptureTarget}, }; use scap_targets::Window; use std::time::SystemTime; @@ -13,7 +13,7 @@ async fn main() { let (ready_tx, _ready_rx) = flume::unbounded(); let (_ctrl_tx, ctrl_rx) = flume::unbounded(); - let mut source = ScreenCaptureSource::::init( + let mut source = ScreenCaptureConfig::::init( &ScreenCaptureTarget::Window { id: Window::list() .into_iter() diff --git a/crates/recording/src/pipeline/audio_buffer.rs b/crates/recording/src/audio_buffer.rs similarity index 100% rename from crates/recording/src/pipeline/audio_buffer.rs rename to crates/recording/src/audio_buffer.rs diff --git a/crates/recording/src/capture_pipeline.rs b/crates/recording/src/capture_pipeline.rs index 2bf54f9b7..de19cf458 100644 --- a/crates/recording/src/capture_pipeline.rs +++ b/crates/recording/src/capture_pipeline.rs @@ -1,787 +1,125 @@ use crate::{ - RecordingError, feeds::microphone::MicrophoneFeedLock, - pipeline::builder::PipelineBuilder, - sources::{ - AudioInputSource, ScreenCaptureFormat, ScreenCaptureSource, ScreenCaptureTarget, - audio_mixer::AudioMixer, screen_capture, + output_pipeline::*, + sources, + sources::screen_capture::{ + self, ScreenCaptureConfig, ScreenCaptureFormat, ScreenCaptureTarget, }, }; -use cap_media::MediaError; -use cap_media_info::AudioInfo; -use cap_timestamp::{Timestamp, Timestamps}; -use flume::{Receiver, Sender}; -use std::{ - future::Future, - path::PathBuf, - sync::{Arc, atomic::AtomicBool}, - time::SystemTime, -}; +use cap_timestamp::Timestamps; +use std::{path::PathBuf, sync::Arc, time::SystemTime}; pub trait MakeCapturePipeline: ScreenCaptureFormat + std::fmt::Debug + 'static { - fn make_studio_mode_pipeline( - builder: PipelineBuilder, - source: ( - ScreenCaptureSource, - flume::Receiver<(Self::VideoFormat, Timestamp)>, - ), + async fn make_studio_mode_pipeline( + screen_capture: screen_capture::VideoSourceConfig, output_path: PathBuf, start_time: Timestamps, - ) -> Result<(PipelineBuilder, flume::Receiver), MediaError> + ) -> anyhow::Result where Self: Sized; - fn make_instant_mode_pipeline( - builder: PipelineBuilder, - source: ( - ScreenCaptureSource, - flume::Receiver<(Self::VideoFormat, Timestamp)>, - ), - audio: Option>, - system_audio: Option<(Receiver<(ffmpeg::frame::Audio, Timestamp)>, AudioInfo)>, + async fn make_instant_mode_pipeline( + screen_capture: screen_capture::VideoSourceConfig, + system_audio: Option, + mic_feed: Option>, output_path: PathBuf, - pause_flag: Arc, - ) -> impl Future> + Send + ) -> anyhow::Result where Self: Sized; } +pub struct Stop; + #[cfg(target_os = "macos")] impl MakeCapturePipeline for screen_capture::CMSampleBufferCapture { - fn make_studio_mode_pipeline( - mut builder: PipelineBuilder, - source: ( - ScreenCaptureSource, - flume::Receiver<(Self::VideoFormat, Timestamp)>, - ), + async fn make_studio_mode_pipeline( + screen_capture: screen_capture::VideoSourceConfig, output_path: PathBuf, - _start_time: Timestamps, - ) -> Result<(PipelineBuilder, flume::Receiver), MediaError> { - let screen_config = source.0.info(); - tracing::info!("screen config: {:?}", screen_config); - - let mut screen_encoder = cap_enc_avfoundation::MP4Encoder::init( - "screen", - screen_config, - None, - output_path, - None, - ) - .map_err(|e| MediaError::Any(e.to_string().into()))?; - - let (timestamp_tx, timestamp_rx) = flume::bounded(1); - - builder.spawn_source("screen_capture", source.0); - - builder.spawn_task("screen_capture_encoder", move |ready| { - let mut timestamp_tx = Some(timestamp_tx); - let _ = ready.send(Ok(())); - - let Ok((frame, timestamp)) = source.1.recv() else { - return Ok(()); - }; - - if let Some(timestamp_tx) = timestamp_tx.take() { - let _ = timestamp_tx.send(timestamp); - let _ = screen_encoder.queue_video_frame(frame.as_ref()); - } - - let result = loop { - match source.1.recv() { - Ok((frame, _)) => { - let _ = screen_encoder.queue_video_frame(frame.as_ref()); - } - // Err(RecvTimeoutError::Timeout) => { - // break Err("Frame receive timeout".to_string()); - // } - Err(_) => { - break Ok(()); - } - } - }; - - screen_encoder.finish(); - - result - }); - - Ok((builder, timestamp_rx)) + start_time: Timestamps, + ) -> anyhow::Result { + OutputPipeline::builder(output_path.clone()) + .with_video::(screen_capture) + .with_timestamps(start_time) + .build::(Default::default()) + .await } async fn make_instant_mode_pipeline( - mut builder: PipelineBuilder, - source: ( - ScreenCaptureSource, - flume::Receiver<(Self::VideoFormat, Timestamp)>, - ), - audio: Option>, - system_audio: Option<(Receiver<(ffmpeg::frame::Audio, Timestamp)>, AudioInfo)>, + screen_capture: screen_capture::VideoSourceConfig, + system_audio: Option, + mic_feed: Option>, output_path: PathBuf, - pause_flag: Arc, - ) -> Result { - let start_time = Timestamps::now(); - - let (audio_tx, audio_rx) = flume::bounded(64); - let mut audio_mixer = AudioMixer::builder(audio_tx); + ) -> anyhow::Result { + let mut output = OutputPipeline::builder(output_path.clone()) + .with_video::(screen_capture); if let Some(system_audio) = system_audio { - audio_mixer.add_source(system_audio.1, system_audio.0); + output = output.with_audio_source::(system_audio); } - if let Some(audio) = audio { - let (tx, rx) = flume::bounded(32); - audio_mixer.add_source(*audio.audio_info(), rx); - let source = AudioInputSource::init(audio, tx); - - builder.spawn_source("microphone_capture", source); + if let Some(mic_feed) = mic_feed { + output = output.with_audio_source::(mic_feed); } - let has_audio_sources = audio_mixer.has_sources(); - - let mp4 = Arc::new(std::sync::Mutex::new( - cap_enc_avfoundation::MP4Encoder::init( - "mp4", - source.0.info(), - has_audio_sources.then_some(AudioMixer::INFO), - output_path, - Some(1080), - ) - .map_err(|e| MediaError::Any(e.to_string().into()))?, - )); - - use cidre::cm; - use tracing::error; - - let (first_frame_tx, mut first_frame_rx) = - tokio::sync::oneshot::channel::<(cm::Time, Timestamp)>(); - - if has_audio_sources { - builder.spawn_source("audio_mixer", audio_mixer); - - let mp4 = mp4.clone(); - builder.spawn_task("audio_encoding", move |ready| { - let _ = ready.send(Ok(())); - let mut time = None; - - while let Ok((mut frame, timestamp)) = audio_rx.recv() { - if let Ok(first_time) = first_frame_rx.try_recv() { - time = Some(first_time); - }; - - let Some(time) = time else { - continue; - }; - - let ts_offset = timestamp.duration_since(start_time); - let screen_first_offset = time.1.duration_since(start_time); - - let Some(ts_offset) = ts_offset.checked_sub(screen_first_offset) else { - continue; - }; - - let pts = (ts_offset.as_secs_f64() * frame.rate() as f64) as i64; - frame.set_pts(Some(pts)); - - if let Ok(mut mp4) = mp4.lock() - && let Err(e) = mp4.queue_audio_frame(frame) - { - error!("{e}"); - return Ok(()); - } - } - - Ok(()) - }); - } - - let mut first_frame_tx = Some(first_frame_tx); - builder.spawn_task("screen_capture_encoder", move |ready| { - let _ = ready.send(Ok(())); - while let Ok((frame, timestamp)) = source.1.recv() { - if let Ok(mut mp4) = mp4.lock() { - if pause_flag.load(std::sync::atomic::Ordering::Relaxed) { - mp4.pause(); - } else { - mp4.resume(); - } - - if let Some(first_frame_tx) = first_frame_tx.take() { - let _ = first_frame_tx.send((frame.pts(), timestamp)); - } - - mp4.queue_video_frame(frame.as_ref()) - .map_err(|err| error!("Error queueing video frame: {err}")) - .ok(); - } - } - if let Ok(mut mp4) = mp4.lock() { - mp4.finish(); - } - - Ok(()) - }); - - builder.spawn_source("screen_capture", source.0); - - Ok(builder) + output + .build::(AVFoundationMp4MuxerConfig { + output_height: Some(1080), + }) + .await } } #[cfg(windows)] impl MakeCapturePipeline for screen_capture::Direct3DCapture { - fn make_studio_mode_pipeline( - mut builder: PipelineBuilder, - source: ( - ScreenCaptureSource, - flume::Receiver<(Self::VideoFormat, Timestamp)>, - ), + async fn make_studio_mode_pipeline( + screen_capture: screen_capture::VideoSourceConfig, output_path: PathBuf, start_time: Timestamps, - ) -> Result<(PipelineBuilder, flume::Receiver), MediaError> - where - Self: Sized, - { - use windows::Graphics::SizeInt32; - - cap_mediafoundation_utils::thread_init(); - - let screen_config = source.0.info(); - let frame_rate = source.0.config().fps(); - let bitrate_multiplier = 0.1f32; - let d3d_device = source.0.d3d_device().clone(); - let pixel_format = screen_capture::Direct3DCapture::PIXEL_FORMAT.as_dxgi(); - let capture_resolution = SizeInt32 { - Width: screen_config.width as i32, - Height: screen_config.height as i32, - }; - - let mut output = ffmpeg::format::output(&output_path) - .map_err(|e| MediaError::Any(format!("CreateOutput: {e}").into()))?; - - let screen_encoder = { - let native_encoder = cap_enc_mediafoundation::H264Encoder::new( - &d3d_device, - pixel_format, - capture_resolution, - frame_rate, - bitrate_multiplier, - ); - - match native_encoder { - Ok(encoder) => { - let muxer = cap_mediafoundation_ffmpeg::H264StreamMuxer::new( - &mut output, - cap_mediafoundation_ffmpeg::MuxerConfig { - width: screen_config.width, - height: screen_config.height, - fps: screen_config.fps(), - bitrate: encoder.bitrate(), - }, - ) - .map_err(|e| MediaError::Any(format!("NativeH264/{e}").into()))?; - - encoder - .start() - .map_err(|e| MediaError::Any(format!("ScreenEncoderStart: {e}").into()))?; - - either::Left((encoder, muxer)) - } - Err(e) => { - tracing::error!("Failed to create native encoder: {e}"); - tracing::info!("Falling back to software H264 encoder"); - - either::Right( - cap_enc_ffmpeg::H264Encoder::builder("screen", screen_config) - .build(&mut output) - .map_err(|e| MediaError::Any(format!("H264Encoder/{e}").into()))?, - ) - } - } - }; - - output - .write_header() - .map_err(|e| MediaError::Any(format!("OutputHeader/{e}").into()))?; - - builder.spawn_source("screen_capture", source.0); - - let (timestamp_tx, timestamp_rx) = flume::bounded(1); - - builder.spawn_task("screen_capture_encoder", move |ready| { - match screen_encoder { - either::Left((mut encoder, mut muxer)) => { - use windows::Win32::Media::MediaFoundation; - - cap_mediafoundation_utils::thread_init(); - - let _ = ready.send(Ok(())); - - let mut timestamp_tx = Some(timestamp_tx); - let mut pending_frame: Option<( - Self::VideoFormat, - Timestamp, - windows::Foundation::TimeSpan, - )> = None; - let mut using_software_encoder = false; - - 'event_loop: while let Ok(e) = encoder.get_event() { - match e { - MediaFoundation::METransformNeedInput => { - let (mut frame, timestamp, frame_time) = if let Some(pending) = - pending_frame.take() - { - pending - } else { - let Ok((frame, timestamp)) = source.1.recv() else { - break; - }; - - if let Some(timestamp_tx) = timestamp_tx.take() { - let _ = timestamp_tx.send(timestamp); - } - - let frame_time = frame - .inner() - .SystemRelativeTime() - .map_err(|e| format!("FrameTime: {e}"))?; - - (frame, timestamp, frame_time) - }; - - loop { - match encoder.handle_needs_input( - frame.texture(), - frame_time, - ) { - Ok(()) => break, - Err( - cap_enc_mediafoundation::video::HandleNeedsInputError::ProcessInput( - error, - ), - ) => { - use tracing::warn; - use windows::Win32::Foundation::E_FAIL; - - if !using_software_encoder && error.code() == E_FAIL { - warn!( - "Native H264 ProcessInput failed with {:?}; falling back to software encoder", - error.code() - ); - pending_frame = - Some((frame, timestamp, frame_time)); - - let mut software_encoder = - cap_enc_mediafoundation::H264Encoder::new_software( - &d3d_device, - pixel_format, - capture_resolution, - frame_rate, - bitrate_multiplier, - ) - .map_err(|e| format!( - "SoftwareEncoderInit: {e}" - ))?; - software_encoder - .start() - .map_err(|e| { - format!( - "ScreenEncoderStart: {e}" - ) - })?; - - encoder = software_encoder; - using_software_encoder = true; - continue 'event_loop; - } - - return Err(format!( - "NeedsInput: ProcessInput: {error}" - )); - } - Err(err) => { - return Err(format!("NeedsInput: {err}")); - } - } - } - } - MediaFoundation::METransformHaveOutput => { - if let Some(output_sample) = encoder - .handle_has_output() - .map_err(|e| format!("HasOutput: {e}"))? - { - muxer - .write_sample(&output_sample, &mut output) - .map_err(|e| format!("WriteSample: {e}"))?; - } - } - _ => {} - } - } - - encoder - .finish() - .map_err(|e| format!("EncoderFinish: {e}"))?; - } - either::Right(mut encoder) => { - let mut first_timestamp = None; - let mut timestamp_tx = Some(timestamp_tx); - let _ = ready.send(Ok(())); - - while let Ok((frame, timestamp)) = source.1.recv() { - use scap_ffmpeg::AsFFmpeg; - - if let Some(timestamp_tx) = timestamp_tx.take() { - let _ = timestamp_tx.send(timestamp); - } - - let first_timestamp = first_timestamp.get_or_insert(timestamp); - - let mut ff_frame = frame - .as_ffmpeg() - .map_err(|e| format!("FrameAsFfmpeg: {e}"))?; - - let elapsed = timestamp.duration_since(start_time) - - first_timestamp.duration_since(start_time); - ff_frame.set_pts(Some(encoder.get_pts(elapsed))); - - encoder.queue_frame(ff_frame, &mut output); - } - encoder.finish(&mut output); - } - } - - output - .write_trailer() - .map_err(|e| format!("WriteTrailer: {e}"))?; - - Ok(()) - }); - - Ok((builder, timestamp_rx)) + ) -> anyhow::Result { + let d3d_device = screen_capture.d3d_device.clone(); + + OutputPipeline::builder(output_path.clone()) + .with_video::(screen_capture) + .with_timestamps(start_time) + .build::(WindowsMuxerConfig { + pixel_format: screen_capture::Direct3DCapture::PIXEL_FORMAT.as_dxgi(), + d3d_device, + bitrate_multiplier: 0.1f32, + frame_rate: 30u32, + }) + .await } async fn make_instant_mode_pipeline( - mut builder: PipelineBuilder, - source: ( - ScreenCaptureSource, - flume::Receiver<(Self::VideoFormat, Timestamp)>, - ), - audio: Option>, - system_audio: Option<(Receiver<(ffmpeg::frame::Audio, Timestamp)>, AudioInfo)>, + screen_capture: screen_capture::VideoSourceConfig, + system_audio: Option, + mic_feed: Option>, output_path: PathBuf, - _pause_flag: Arc, - ) -> Result - where - Self: Sized, - { - use cap_enc_ffmpeg::{AACEncoder, AudioEncoder}; - use windows::Graphics::SizeInt32; + ) -> anyhow::Result { + let d3d_device = screen_capture.d3d_device.clone(); + let mut output_builder = OutputPipeline::builder(output_path.clone()) + .with_video::(screen_capture); - cap_mediafoundation_utils::thread_init(); - - let start_time = Timestamps::now(); - - let (audio_tx, audio_rx) = flume::bounded(64); - let mut audio_mixer = AudioMixer::builder(audio_tx); - - if let Some(system_audio) = system_audio { - audio_mixer.add_source(system_audio.1, system_audio.0); + if let Some(mic_feed) = mic_feed { + output_builder = output_builder.with_audio_source::(mic_feed); } - if let Some(audio) = audio { - let (tx, rx) = flume::bounded(32); - audio_mixer.add_source(*audio.audio_info(), rx); - let source = AudioInputSource::init(audio, tx); - - builder.spawn_source("microphone_capture", source); + if let Some(system_audio) = system_audio { + output_builder = + output_builder.with_audio_source::(system_audio); } - let has_audio_sources = audio_mixer.has_sources(); - let screen_config = source.0.info(); - let frame_rate = 30u32; - let bitrate_multiplier = 0.15f32; - let d3d_device = source.0.d3d_device().clone(); - let pixel_format = screen_capture::Direct3DCapture::PIXEL_FORMAT.as_dxgi(); - let input_resolution = SizeInt32 { - Width: screen_config.width as i32, - Height: screen_config.height as i32, - }; - let output_resolution = SizeInt32 { - Width: screen_config.width as i32, - Height: screen_config.height as i32, - }; - - let mut output = ffmpeg::format::output(&output_path) - .map_err(|e| MediaError::Any(format!("CreateOutput: {e}").into()))?; - - let screen_encoder = { - let native_encoder = cap_enc_mediafoundation::H264Encoder::new_with_scaled_output( - &d3d_device, - pixel_format, - input_resolution, - output_resolution, - frame_rate, - bitrate_multiplier, - ); - - match native_encoder { - Ok(screen_encoder) => { - let screen_muxer = cap_mediafoundation_ffmpeg::H264StreamMuxer::new( - &mut output, - cap_mediafoundation_ffmpeg::MuxerConfig { - width: screen_config.width, - height: screen_config.height, - fps: frame_rate, - bitrate: screen_encoder.bitrate(), - }, - ) - .map_err(|e| MediaError::Any(format!("NativeH264Muxer/{e}").into()))?; - - screen_encoder - .start() - .map_err(|e| MediaError::Any(format!("StartScreenEncoder/{e}").into()))?; - - either::Left((screen_encoder, screen_muxer)) - } - Err(e) => { - use tracing::{error, info}; - - error!("Failed to create native encoder: {e}"); - info!("Falling back to software H264 encoder"); - - either::Right( - cap_enc_ffmpeg::H264Encoder::builder("screen", screen_config) - .build(&mut output) - .map_err(|e| MediaError::Any(format!("H264Encoder/{e}").into()))?, - ) - } - } - }; - - let audio_encoder = has_audio_sources - .then(|| { - AACEncoder::init("mic_audio", AudioMixer::INFO, &mut output) - .map(|v| v.boxed()) - .map_err(|e| MediaError::Any(e.to_string().into())) + output_builder + .build::(WindowsMuxerConfig { + pixel_format: screen_capture::Direct3DCapture::PIXEL_FORMAT.as_dxgi(), + bitrate_multiplier: 0.15f32, + frame_rate: 30u32, + d3d_device, }) - .transpose() - .map_err(|e| MediaError::Any(format!("AACEncoder/{e}").into()))?; - - output - .write_header() - .map_err(|e| MediaError::Any(format!("OutputHeader/{e}").into()))?; - - let output = Arc::new(std::sync::Mutex::new(output)); - - let (first_frame_tx, first_frame_rx) = tokio::sync::oneshot::channel::(); - - if let Some(mut audio_encoder) = audio_encoder { - builder.spawn_source("audio_mixer", audio_mixer); - - let output = output.clone(); - builder.spawn_task("audio_encoding", move |ready| { - let _ = ready.send(Ok(())); - - let time = first_frame_rx.blocking_recv().unwrap(); - let screen_first_offset = time.duration_since(start_time); - - while let Ok((mut frame, timestamp)) = audio_rx.recv() { - let ts_offset = timestamp.duration_since(start_time); - - let Some(ts_offset) = ts_offset.checked_sub(screen_first_offset) else { - continue; - }; - - let pts = (ts_offset.as_secs_f64() * frame.rate() as f64) as i64; - frame.set_pts(Some(pts)); - - if let Ok(mut output) = output.lock() { - audio_encoder.queue_frame(frame, &mut output) - } - } - - Ok(()) - }); - } - - builder.spawn_source("screen_capture", source.0); - - builder.spawn_task("screen_encoder", move |ready| { - match screen_encoder { - either::Left((mut encoder, mut muxer)) => { - use windows::Win32::Media::MediaFoundation; - - cap_mediafoundation_utils::thread_init(); - - let _ = ready.send(Ok(())); - - let mut first_frame_tx = Some(first_frame_tx); - let mut pending_frame: Option<( - Self::VideoFormat, - windows::Foundation::TimeSpan, - )> = None; - let mut using_software_encoder = false; - - 'event_loop: while let Ok(e) = encoder.get_event() { - match e { - MediaFoundation::METransformNeedInput => { - use cap_timestamp::PerformanceCounterTimestamp; - use tracing::warn; - use windows::Win32::Foundation::E_FAIL; - - let (mut frame, frame_time) = if let Some(pending) = - pending_frame.take() - { - pending - } else { - let Ok((frame, _)) = source.1.recv() else { - break; - }; - - let frame_time = frame - .inner() - .SystemRelativeTime() - .map_err(|e| format!("Frame Time: {e}"))?; - - (frame, frame_time) - }; - - let timestamp = Timestamp::PerformanceCounter( - PerformanceCounterTimestamp::new(frame_time.Duration), - ); - - if let Some(first_frame_tx) = first_frame_tx.take() { - let _ = first_frame_tx.send(timestamp); - } - - loop { - match encoder.handle_needs_input( - frame.texture(), - frame_time, - ) { - Ok(()) => break, - Err( - cap_enc_mediafoundation::video::HandleNeedsInputError::ProcessInput( - error, - ), - ) => { - if !using_software_encoder && error.code() == E_FAIL { - warn!( - "Native H264 ProcessInput failed with {:?}; falling back to software encoder", - error.code() - ); - pending_frame = Some((frame, frame_time)); - - let mut software_encoder = cap_enc_mediafoundation::H264Encoder::new_with_scaled_output_software( - &d3d_device, - pixel_format, - input_resolution, - output_resolution, - frame_rate, - bitrate_multiplier, - ) - .map_err(|e| { - format!("SoftwareEncoderInit: {e}") - })?; - - software_encoder - .start() - .map_err(|e| format!( - "StartScreenEncoder: {e}" - ))?; - - encoder = software_encoder; - using_software_encoder = true; - continue 'event_loop; - } - - return Err(format!( - "NeedsInput: ProcessInput: {error}" - )); - } - Err(err) => { - return Err(format!("NeedsInput: {err}")); - } - } - } - } - MediaFoundation::METransformHaveOutput => { - if let Some(output_sample) = encoder - .handle_has_output() - .map_err(|e| format!("HasOutput: {e}"))? - { - let mut output = output.lock().unwrap(); - - muxer - .write_sample(&output_sample, &mut *output) - .map_err(|e| format!("WriteSample: {e}"))?; - } - } - _ => {} - } - } - - encoder - .finish() - .map_err(|e| format!("EncoderFinish: {e}"))?; - } - either::Right(mut encoder) => { - let mut first_timestamp = None; - let mut first_frame_tx = Some(first_frame_tx); - let output = output.clone(); - - let _ = ready.send(Ok(())); - - while let Ok((frame, timestamp)) = source.1.recv() { - let Ok(mut output) = output.lock() else { - continue; - }; - - // if pause_flag.load(std::sync::atomic::Ordering::Relaxed) { - // mp4.pause(); - // } else { - // mp4.resume(); - // } - - use scap_ffmpeg::AsFFmpeg; - - let first_timestamp = first_timestamp.get_or_insert(timestamp); - - if let Some(first_frame_tx) = first_frame_tx.take() { - let _ = first_frame_tx.send(timestamp); - } - - let mut ff_frame = frame - .as_ffmpeg() - .map_err(|e| format!("FrameAsFfmpeg: {e}"))?; - - let elapsed = timestamp.duration_since(start_time) - - first_timestamp.duration_since(start_time); - ff_frame.set_pts(Some(encoder.get_pts(elapsed))); - - encoder.queue_frame(ff_frame, &mut output); - } - } - } - - output - .lock() - .map_err(|e| format!("OutputLock: {e}"))? - .write_trailer() - .map_err(|e| format!("WriteTrailer: {e}"))?; - - Ok(()) - }); - - Ok(builder) + .await } } -type ScreenCaptureReturn = ( - ScreenCaptureSource, - Receiver<(::VideoFormat, Timestamp)>, -); - #[cfg(target_os = "macos")] pub type ScreenCaptureMethod = screen_capture::CMSampleBufferCapture; @@ -792,29 +130,23 @@ pub async fn create_screen_capture( capture_target: &ScreenCaptureTarget, force_show_cursor: bool, max_fps: u32, - audio_tx: Option>, start_time: SystemTime, + system_audio: bool, #[cfg(windows)] d3d_device: ::windows::Win32::Graphics::Direct3D11::ID3D11Device, #[cfg(target_os = "macos")] shareable_content: cidre::arc::R, -) -> Result, RecordingError> { - let (video_tx, video_rx) = flume::bounded(16); - - ScreenCaptureSource::::init( +) -> anyhow::Result> { + Ok(ScreenCaptureConfig::::init( capture_target, force_show_cursor, max_fps, - video_tx, - audio_tx, start_time, - tokio::runtime::Handle::current(), + system_audio, #[cfg(windows)] d3d_device, #[cfg(target_os = "macos")] shareable_content, ) - .await - .map(|v| (v, video_rx)) - .map_err(|e| RecordingError::Media(MediaError::TaskLaunch(e.to_string()))) + .await?) } #[cfg(windows)] diff --git a/crates/recording/src/cursor.rs b/crates/recording/src/cursor.rs index fb8a83de3..0f618804e 100644 --- a/crates/recording/src/cursor.rs +++ b/crates/recording/src/cursor.rs @@ -2,10 +2,12 @@ use cap_cursor_capture::CursorCropBounds; use cap_cursor_info::CursorShape; use cap_project::{CursorClickEvent, CursorMoveEvent, XY}; use cap_timestamp::Timestamps; +use futures::{FutureExt, future::Shared}; use std::{collections::HashMap, path::PathBuf}; use tokio::sync::oneshot; use tokio_util::sync::{CancellationToken, DropGuard}; +#[derive(Clone)] pub struct Cursor { pub file_name: String, pub id: u32, @@ -15,6 +17,7 @@ pub struct Cursor { pub type Cursors = HashMap; +#[derive(Clone)] pub struct CursorActorResponse { // pub cursor_images: HashMap>, pub cursors: Cursors, @@ -24,14 +27,13 @@ pub struct CursorActorResponse { } pub struct CursorActor { - stop: DropGuard, - rx: oneshot::Receiver, + stop: Option, + pub rx: Shared>, } impl CursorActor { - pub async fn stop(self) -> CursorActorResponse { - drop(self.stop); - self.rx.await.unwrap() + pub fn stop(&mut self) { + drop(self.stop.take()); } } @@ -180,8 +182,8 @@ pub fn spawn_cursor_recorder( }); CursorActor { - stop: stop_token.drop_guard(), - rx, + stop: Some(stop_token.drop_guard()), + rx: rx.shared(), } } diff --git a/crates/recording/src/feeds/camera.rs b/crates/recording/src/feeds/camera.rs index 87b71b719..9a5b562c4 100644 --- a/crates/recording/src/feeds/camera.rs +++ b/crates/recording/src/feeds/camera.rs @@ -2,10 +2,7 @@ use cap_camera::CameraInfo; use cap_camera_ffmpeg::*; use cap_fail::fail_err; use cap_media_info::VideoInfo; -#[cfg(windows)] -use cap_timestamp::PerformanceCounterTimestamp; use cap_timestamp::Timestamp; -use ffmpeg::frame; use futures::{FutureExt, future::BoxFuture}; use kameo::prelude::*; use replace_with::replace_with_or_abort; @@ -18,18 +15,14 @@ use std::{ use tokio::{runtime::Runtime, sync::oneshot, task::LocalSet}; use tracing::{debug, error, info, trace, warn}; -const CAMERA_INIT_TIMEOUT: Duration = Duration::from_secs(4); +use crate::ffmpeg::FFmpegVideoFrame; -#[derive(Clone)] -pub struct RawCameraFrame { - pub frame: frame::Video, - pub timestamp: Timestamp, -} +const CAMERA_INIT_TIMEOUT: Duration = Duration::from_secs(4); #[derive(Actor)] pub struct CameraFeed { state: State, - senders: Vec>, + senders: Vec>, on_ready: Vec>, on_disconnect: Vec>, } @@ -159,7 +152,7 @@ pub struct SetInput { pub struct RemoveInput; -pub struct AddSender(pub flume::Sender); +pub struct AddSender(pub flume::Sender); pub struct ListenForReady(pub oneshot::Sender<()>); @@ -180,7 +173,7 @@ struct InputConnectFailed { id: DeviceOrModelID, } -struct NewFrame(RawCameraFrame); +struct NewFrame(FFmpegVideoFrame); struct Unlock; @@ -219,7 +212,6 @@ struct SetupCameraResult { handle: cap_camera::CaptureHandle, camera_info: cap_camera::CameraInfo, video_info: VideoInfo, - // frame_rx: mpsc::Receiver, } async fn setup_camera( @@ -288,12 +280,14 @@ async fn setup_camera( } let _ = recipient - .tell(NewFrame(RawCameraFrame { - frame: ff_frame, + .tell(NewFrame(FFmpegVideoFrame { + inner: ff_frame, #[cfg(windows)] - timestamp: Timestamp::PerformanceCounter(PerformanceCounterTimestamp::new( - frame.native().perf_counter, - )), + timestamp: Timestamp::PerformanceCounter( + cap_timestamp::PerformanceCounterTimestamp::new( + frame.native().perf_counter, + ), + ), #[cfg(target_os = "macos")] timestamp: Timestamp::MachAbsoluteTime( cap_timestamp::MachAbsoluteTimestamp::new( diff --git a/crates/recording/src/feeds/microphone.rs b/crates/recording/src/feeds/microphone.rs index 7701a5eca..c6891d566 100644 --- a/crates/recording/src/feeds/microphone.rs +++ b/crates/recording/src/feeds/microphone.rs @@ -170,8 +170,8 @@ impl MicrophoneFeedLock { &self.config } - pub fn audio_info(&self) -> &AudioInfo { - &self.audio_info + pub fn audio_info(&self) -> AudioInfo { + self.audio_info } } diff --git a/crates/recording/src/instant_recording.rs b/crates/recording/src/instant_recording.rs index dab814e62..ecf9d1f22 100644 --- a/crates/recording/src/instant_recording.rs +++ b/crates/recording/src/instant_recording.rs @@ -1,120 +1,194 @@ use crate::{ - ActorError, RecordingBaseInputs, RecordingError, - capture_pipeline::{MakeCapturePipeline, create_screen_capture}, + RecordingBaseInputs, + capture_pipeline::{MakeCapturePipeline, ScreenCaptureMethod, Stop, create_screen_capture}, feeds::microphone::MicrophoneFeedLock, - pipeline::RecordingPipeline, - sources::{ScreenCaptureSource, ScreenCaptureTarget}, + output_pipeline::{self, OutputPipeline}, + sources::screen_capture::{ScreenCaptureConfig, ScreenCaptureTarget}, }; -use cap_media::MediaError; use cap_media_info::{AudioInfo, VideoInfo}; use cap_project::InstantRecordingMeta; -use cap_timestamp::Timestamp; -use cap_utils::{ensure_dir, spawn_actor}; -use flume::Receiver; +use cap_utils::ensure_dir; +use kameo::{Actor as _, prelude::*}; use std::{ path::PathBuf, - sync::{Arc, atomic::AtomicBool}, + sync::Arc, time::{SystemTime, UNIX_EPOCH}, }; -use tokio::sync::oneshot; -use tracing::{Instrument, debug, error, info, trace}; +use tracing::*; struct Pipeline { - pub inner: RecordingPipeline, - #[allow(unused)] - pub output_path: PathBuf, - pub pause_flag: Arc, + output: OutputPipeline, } enum ActorState { Recording { pipeline: Pipeline, - pipeline_done_rx: oneshot::Receiver>, + // pipeline_done_rx: oneshot::Receiver>, segment_start_time: f64, }, Paused { pipeline: Pipeline, - pipeline_done_rx: oneshot::Receiver>, + // pipeline_done_rx: oneshot::Receiver>, segment_start_time: f64, }, + Stopped, } -#[derive(Clone)] pub struct ActorHandle { - ctrl_tx: flume::Sender, + actor_ref: kameo::actor::ActorRef, pub capture_target: ScreenCaptureTarget, - // pub bounds: Bounds, -} - -macro_rules! send_message { - ($ctrl_tx:expr, $variant:path) => {{ - let (tx, rx) = oneshot::channel(); - $ctrl_tx - .send($variant(tx)) - .map_err(|_| flume::SendError(())) - .map_err(ActorError::from)?; - rx.await.map_err(|_| ActorError::ActorStopped)? - }}; + done_fut: output_pipeline::DoneFut, } impl ActorHandle { - pub async fn stop(&self) -> Result { - send_message!(self.ctrl_tx, ActorControlMessage::Stop) + pub async fn stop(&self) -> anyhow::Result { + Ok(self.actor_ref.ask(Stop).await?) } - pub async fn pause(&self) -> Result<(), RecordingError> { - send_message!(self.ctrl_tx, ActorControlMessage::Pause) + pub fn done_fut(&self) -> output_pipeline::DoneFut { + self.done_fut.clone() } - pub async fn resume(&self) -> Result<(), RecordingError> { - send_message!(self.ctrl_tx, ActorControlMessage::Resume) + pub async fn pause(&self) -> anyhow::Result<()> { + Ok(self.actor_ref.ask(Pause).await?) } - pub async fn cancel(&self) -> Result<(), RecordingError> { - send_message!(self.ctrl_tx, ActorControlMessage::Cancel) + pub async fn resume(&self) -> anyhow::Result<()> { + Ok(self.actor_ref.ask(Resume).await?) } -} -pub enum ActorControlMessage { - Pause(oneshot::Sender>), - Resume(oneshot::Sender>), - Stop(oneshot::Sender>), - Cancel(oneshot::Sender>), + pub async fn cancel(&self) -> anyhow::Result<()> { + Ok(self.actor_ref.ask(Cancel).await?) + } } -impl std::fmt::Debug for ActorControlMessage { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Pause(_) => write!(f, "Pause"), - Self::Resume(_) => write!(f, "Resume"), - Self::Stop(_) => write!(f, "Stop"), - Self::Cancel(_) => write!(f, "Cancel"), - } +impl Drop for ActorHandle { + fn drop(&mut self) { + let actor_ref = self.actor_ref.clone(); + tokio::spawn(async move { + let _ = actor_ref.tell(Stop).await; + }); } } +#[derive(kameo::Actor)] pub struct Actor { recording_dir: PathBuf, capture_target: ScreenCaptureTarget, video_info: VideoInfo, + state: ActorState, +} + +impl Actor { + async fn stop(&mut self) -> anyhow::Result<()> { + let pipeline = replace_with::replace_with_or_abort_and_return(&mut self.state, |state| { + ( + match state { + ActorState::Recording { pipeline, .. } => Some(pipeline), + ActorState::Paused { pipeline, .. } => Some(pipeline), + _ => None, + }, + ActorState::Stopped, + ) + }); + + if let Some(pipeline) = pipeline { + pipeline.output.stop().await?; + } + + Ok(()) + } +} + +impl Message for Actor { + type Reply = anyhow::Result; + + async fn handle(&mut self, _: Stop, _: &mut Context) -> Self::Reply { + self.stop().await?; + + Ok(CompletedRecording { + project_path: self.recording_dir.clone(), + meta: InstantRecordingMeta { + fps: self.video_info.fps(), + sample_rate: None, + }, + display_source: self.capture_target.clone(), + }) + } +} + +pub struct Pause; + +impl Message for Actor { + type Reply = (); + + async fn handle(&mut self, _: Pause, _: &mut Context) -> Self::Reply { + replace_with::replace_with_or_abort(&mut self.state, |state| { + if let ActorState::Recording { + pipeline, + segment_start_time, + } = state + { + pipeline.output.pause(); + return ActorState::Paused { + pipeline, + segment_start_time, + }; + } + + state + }); + } } +pub struct Resume; + +impl Message for Actor { + type Reply = (); + + async fn handle(&mut self, _: Resume, _: &mut Context) -> Self::Reply { + replace_with::replace_with_or_abort(&mut self.state, |state| { + if let ActorState::Paused { + pipeline, + segment_start_time, + } = state + { + pipeline.output.resume(); + return ActorState::Recording { + pipeline, + segment_start_time, + }; + } + + state + }); + } +} + +pub struct Cancel; + +impl Message for Actor { + type Reply = anyhow::Result<()>; + + async fn handle(&mut self, _: Cancel, _: &mut Context) -> Self::Reply { + let _ = self.stop().await; + + Ok(()) + } +} + +#[derive(Debug)] pub struct CompletedRecording { pub project_path: PathBuf, pub display_source: ScreenCaptureTarget, pub meta: InstantRecordingMeta, } -#[tracing::instrument(skip_all, name = "instant")] -async fn create_pipeline( +async fn create_pipeline( output_path: PathBuf, - screen_source: ( - ScreenCaptureSource, - flume::Receiver<(TCaptureFormat::VideoFormat, Timestamp)>, - ), + screen_source: ScreenCaptureConfig, mic_feed: Option>, - system_audio: Option>, -) -> Result<(Pipeline, oneshot::Receiver>), MediaError> { +) -> anyhow::Result { if let Some(mic_feed) = &mic_feed { debug!( "mic audio info: {:#?}", @@ -122,32 +196,17 @@ async fn create_pipeline( ); }; - let pipeline_builder = RecordingPipeline::builder(); + let (screen_capture, system_audio) = screen_source.to_sources().await?; - let pause_flag = Arc::new(AtomicBool::new(false)); - let system_audio = system_audio.map(|v| (v, screen_source.0.audio_info())); - let pipeline_builder = TCaptureFormat::make_instant_mode_pipeline( - pipeline_builder, - screen_source, - mic_feed, + let output = ScreenCaptureMethod::make_instant_mode_pipeline( + screen_capture, system_audio, + mic_feed, output_path.clone(), - pause_flag.clone(), ) .await?; - let (mut pipeline, pipeline_done_rx) = pipeline_builder.build().await?; - - pipeline.play().await?; - - Ok(( - Pipeline { - inner: pipeline, - output_path, - pause_flag, - }, - pipeline_done_rx, - )) + Ok(Pipeline { output }) } impl Actor { @@ -186,7 +245,7 @@ impl ActorBuilder { pub async fn build( self, #[cfg(target_os = "macos")] shareable_content: cidre::arc::R, - ) -> Result<(ActorHandle, oneshot::Receiver>), RecordingError> { + ) -> anyhow::Result { spawn_instant_recording_actor( self.output_path, RecordingBaseInputs { @@ -202,43 +261,31 @@ impl ActorBuilder { } } +#[tracing::instrument("instant_recording", skip_all)] pub async fn spawn_instant_recording_actor( recording_dir: PathBuf, inputs: RecordingBaseInputs, -) -> Result< - ( - ActorHandle, - tokio::sync::oneshot::Receiver>, - ), - RecordingError, -> { +) -> anyhow::Result { ensure_dir(&recording_dir)?; let start_time = SystemTime::now(); - let (done_tx, done_rx) = oneshot::channel(); - trace!("creating recording actor"); let content_dir = ensure_dir(&recording_dir.join("content"))?; - let system_audio = if inputs.capture_system_audio { - let (tx, rx) = flume::bounded(64); - (Some(tx), Some(rx)) - } else { - (None, None) - }; + #[cfg(windows)] + cap_mediafoundation_utils::thread_init(); #[cfg(windows)] - let d3d_device = crate::capture_pipeline::create_d3d_device() - .map_err(|e| MediaError::Any(format!("CreateD3DDevice: {e}").into()))?; + let d3d_device = crate::capture_pipeline::create_d3d_device()?; - let (screen_source, screen_rx) = create_screen_capture( + let screen_source = create_screen_capture( &inputs.capture_target, true, 30, - system_audio.0, start_time, + inputs.capture_system_audio, #[cfg(windows)] d3d_device, #[cfg(target_os = "macos")] @@ -248,243 +295,41 @@ pub async fn spawn_instant_recording_actor( debug!("screen capture: {screen_source:#?}"); - let (pipeline, pipeline_done_rx) = create_pipeline( + let pipeline = create_pipeline( content_dir.join("output.mp4"), - (screen_source.clone(), screen_rx.clone()), + screen_source.clone(), inputs.mic_feed.clone(), - system_audio.1, ) .await?; let segment_start_time = current_time_f64(); - let (ctrl_tx, ctrl_rx) = flume::bounded(1); - trace!("spawning recording actor"); - spawn_actor({ - let inputs = inputs.clone(); - let video_info = screen_source.info(); - async move { - let mut actor = Actor { - recording_dir, - capture_target: inputs.capture_target, - video_info, - }; - - let mut state = ActorState::Recording { - pipeline, - pipeline_done_rx, - segment_start_time, - }; - - let result = loop { - match run_actor_iteration(state, &ctrl_rx, actor).await { - Ok(None) => break Ok(()), - Ok(Some((new_state, new_actor))) => { - state = new_state; - actor = new_actor; - } - Err(err) => break Err(err), - } - }; - - info!("recording actor finished"); - - let _ = done_tx.send(result.map_err(|v| v.to_string())); - } - .in_current_span() - }); - - Ok(( - ActorHandle { - ctrl_tx, - capture_target: inputs.capture_target, - // bounds: *screen_source.get_bounds(), - }, - done_rx, - )) -} - -#[derive(thiserror::Error, Debug)] -enum InstantRecordingActorError { - #[error("Pipeline receiver dropped")] - PipelineReceiverDropped, - #[error("Control receiver dropped")] - ControlReceiverDropped, - #[error("{0}")] - Other(String), -} - -// Helper macro for sending responses -macro_rules! send_response { - ($tx:expr, $res:expr) => { - let _ = $tx.send($res); - }; -} - -async fn run_actor_iteration( - state: ActorState, - ctrl_rx: &Receiver, - actor: Actor, -) -> Result, InstantRecordingActorError> { - use ActorControlMessage as Msg; - use ActorState as State; - - // Helper function to shutdown pipeline - async fn shutdown(mut pipeline: Pipeline) -> Result<(), RecordingError> { - pipeline.inner.shutdown().await?; - Ok(()) - } - - // Log current state - info!( - "recording actor state: {:?}", - match &state { - State::Recording { .. } => "recording", - State::Paused { .. } => "paused", - } - ); - - // Receive event based on current state - let event = match state { - State::Recording { - mut pipeline_done_rx, + let done_fut = pipeline.output.done_fut(); + let actor_ref = Actor::spawn(Actor { + recording_dir, + capture_target: inputs.capture_target.clone(), + video_info: screen_source.info(), + state: ActorState::Recording { pipeline, + // pipeline_done_rx, segment_start_time, - } => { - tokio::select! { - result = &mut pipeline_done_rx => { - return match result { - Ok(Ok(())) => Ok(None), - Ok(Err(e)) => Err(InstantRecordingActorError::Other(e)), - Err(_) => Err(InstantRecordingActorError::PipelineReceiverDropped), - } - }, - msg = ctrl_rx.recv_async() => { - match msg { - Ok(msg) => { - info!("received control message: {msg:?}"); - (msg, State::Recording { pipeline, pipeline_done_rx, segment_start_time }) - }, - Err(_) => return Err(InstantRecordingActorError::ControlReceiverDropped), - } - } - } - } - paused_state @ State::Paused { .. } => match ctrl_rx.recv_async().await { - Ok(msg) => { - info!("received control message: {msg:?}"); - (msg, paused_state) - } - Err(_) => return Err(InstantRecordingActorError::ControlReceiverDropped), }, - }; - - let (event, state) = event; - - // Handle state transitions based on event and current state - Ok(match (event, state) { - // Pause from Recording - ( - Msg::Pause(tx), - State::Recording { - pipeline, - pipeline_done_rx, - segment_start_time, - }, - ) => { - pipeline - .pause_flag - .store(true, std::sync::atomic::Ordering::SeqCst); - send_response!(tx, Ok(())); - Some(( - State::Paused { - pipeline, - pipeline_done_rx, - segment_start_time, - }, - actor, - )) - } - - // Stop from any state - (Msg::Stop(tx), state) => { - let pipeline = match state { - State::Recording { pipeline, .. } => pipeline, - State::Paused { pipeline, .. } => pipeline, - }; - - let res = shutdown(pipeline).await; - let res = match res { - Ok(_) => Ok(stop_recording(actor).await), - Err(e) => Err(e), - }; - - send_response!(tx, res); - None - } - - // Resume from Paused - ( - Msg::Resume(tx), - State::Paused { - pipeline, - pipeline_done_rx, - segment_start_time, - }, - ) => { - pipeline - .pause_flag - .store(false, std::sync::atomic::Ordering::SeqCst); - - send_response!(tx, Ok(())); - - Some(( - State::Recording { - pipeline, - pipeline_done_rx, - segment_start_time, - }, - actor, - )) - } - - // Cancel from any state - (Msg::Cancel(tx), state) => { - let pipeline = match state { - State::Recording { pipeline, .. } => pipeline, - State::Paused { pipeline, .. } => pipeline, - }; - - let res = shutdown(pipeline).await; - send_response!(tx, res); - None - } + }); - // Invalid combinations - continue iteration - (Msg::Pause(_), state @ State::Paused { .. }) => { - // Already paused, ignore - Some((state, actor)) - } - (Msg::Resume(_), state @ State::Recording { .. }) => { - // Already recording, ignore - Some((state, actor)) - } - }) -} + let actor_handle = ActorHandle { + actor_ref: actor_ref.clone(), + capture_target: inputs.capture_target, + done_fut: done_fut.clone(), + }; -async fn stop_recording(actor: Actor) -> CompletedRecording { - use cap_project::*; + tokio::spawn(async move { + let _ = done_fut.await; + let _ = actor_ref.ask(Stop).await; + }); - CompletedRecording { - project_path: actor.recording_dir.clone(), - meta: InstantRecordingMeta { - fps: actor.video_info.fps(), - sample_rate: None, - }, - display_source: actor.capture_target, - } + Ok(actor_handle) } fn current_time_f64() -> f64 { diff --git a/crates/recording/src/lib.rs b/crates/recording/src/lib.rs index 671ac21ac..a4315c1b0 100644 --- a/crates/recording/src/lib.rs +++ b/crates/recording/src/lib.rs @@ -1,23 +1,24 @@ +mod audio_buffer; mod capture_pipeline; pub mod cursor; pub mod feeds; pub mod instant_recording; -pub mod pipeline; +mod output_pipeline; pub mod sources; pub mod studio_recording; pub use feeds::{camera::CameraFeed, microphone::MicrophoneFeed}; -pub use sources::{camera, screen_capture}; +pub use output_pipeline::*; +pub use sources::screen_capture; use cap_media::MediaError; use feeds::microphone::MicrophoneFeedLock; use scap_targets::bounds::LogicalBounds; use serde::{Deserialize, Serialize}; -use sources::*; use std::sync::Arc; use thiserror::Error; -use crate::feeds::camera::CameraFeedLock; +use crate::{feeds::camera::CameraFeedLock, sources::screen_capture::ScreenCaptureTarget}; #[derive(specta::Type, Serialize, Deserialize, Clone, Debug, Copy, Default)] #[serde(rename_all = "camelCase")] diff --git a/crates/recording/src/output_pipeline/core.rs b/crates/recording/src/output_pipeline/core.rs new file mode 100644 index 000000000..028f1ece4 --- /dev/null +++ b/crates/recording/src/output_pipeline/core.rs @@ -0,0 +1,784 @@ +use crate::sources::audio_mixer::AudioMixer; +use anyhow::{Context, anyhow}; +use cap_media_info::{AudioInfo, VideoInfo}; +use cap_timestamp::{Timestamp, Timestamps}; +use futures::{ + FutureExt, SinkExt, StreamExt, TryFutureExt, + channel::{mpsc, oneshot}, + future::{BoxFuture, Shared}, + lock::Mutex, + stream::FuturesUnordered, +}; +use std::{ + any::Any, + future, + marker::PhantomData, + path::PathBuf, + sync::{ + Arc, + atomic::{self, AtomicBool}, + }, + time::Duration, +}; +use tokio::task::JoinHandle; +use tokio_util::sync::{CancellationToken, DropGuard}; +use tracing::*; + +pub struct OnceSender(Option>); + +impl OnceSender { + pub fn send(&mut self, v: T) { + if let Some(tx) = self.0.take() { + let _ = tx.send(v); + } + } +} + +impl OutputPipeline { + pub fn builder(path: PathBuf) -> OutputPipelineBuilder { + OutputPipelineBuilder:: { + path, + video: NoVideo, + audio_sources: vec![], + timestamps: Timestamps::now(), + } + } +} + +pub struct SetupCtx { + tasks: TaskPool, +} + +impl SetupCtx { + pub fn tasks(&mut self) -> &mut TaskPool { + &mut self.tasks + } +} + +type AudioSourceSetupFn = Box< + dyn FnOnce( + mpsc::Sender, + &mut SetupCtx, + ) -> BoxFuture<'static, anyhow::Result> + + Send, +>; + +pub struct OutputPipelineBuilder { + path: PathBuf, + video: TVideo, + audio_sources: Vec, + timestamps: Timestamps, +} + +pub struct NoVideo; +pub struct HasVideo { + config: TVideo::Config, +} + +impl OutputPipelineBuilder { + pub fn with_audio_source( + mut self, + config: TAudio::Config, + ) -> OutputPipelineBuilder { + self.audio_sources.push(Box::new(move |tx, ctx| { + TAudio::setup(config, tx, ctx) + .map(|v| v.map(ErasedAudioSource::new)) + .boxed() + })); + + self + } + + pub fn set_timestamps(&mut self, timestamps: Timestamps) { + self.timestamps = timestamps; + } + + pub fn with_timestamps(mut self, timestamps: Timestamps) -> Self { + self.timestamps = timestamps; + self + } +} + +impl OutputPipelineBuilder { + pub fn with_video( + self, + config: TVideo::Config, + ) -> OutputPipelineBuilder> { + OutputPipelineBuilder::> { + video: HasVideo { config }, + path: self.path, + audio_sources: self.audio_sources, + timestamps: self.timestamps, + } + } +} + +pub struct TaskPool(Vec<(&'static str, JoinHandle>)>); + +impl TaskPool { + pub fn spawn(&mut self, name: &'static str, future: F) + where + F: Future> + Send + 'static, + { + self.0.push(( + name, + tokio::spawn( + async { + trace!("Task started"); + let res = future.await; + match &res { + Ok(_) => info!("Task finished successfully"), + Err(err) => error!("Task failed: {}", err), + } + res + } + .instrument(error_span!("", task = name)) + .in_current_span(), + ), + )); + } + + pub fn spawn_thread(&mut self, name: &'static str, cb: impl FnOnce() + Send + 'static) { + let span = error_span!("", task = name); + let (done_tx, done_rx) = oneshot::channel(); + std::thread::spawn(move || { + let _guard = span.enter(); + trace!("Task started"); + cb(); + let _ = done_tx.send(()); + info!("Task finished"); + }); + self.0.push(( + name, + tokio::spawn(done_rx.map_err(|_| anyhow!("Cancelled"))), + )); + } +} + +impl OutputPipelineBuilder> { + pub async fn build + AudioMuxer>( + self, + muxer_config: TMuxer::Config, + ) -> anyhow::Result { + let Self { + video, + audio_sources, + timestamps, + path, + .. + } = self; + + let (mut setup_ctx, stop_token, done_tx, done_rx, pause_flag) = setup_build(); + + let (video_source, video_rx) = + setup_video_source::(video.config, &mut setup_ctx).await?; + + let video_info = video_source.video_info(); + let (first_tx, first_rx) = oneshot::channel(); + + let muxer = setup_muxer::( + muxer_config, + &path, + Some(video_info), + Some(AudioMixer::INFO), + &pause_flag, + &mut setup_ctx, + ) + .await?; + + spawn_video_encoder( + &mut setup_ctx, + video_source, + video_rx, + first_tx, + stop_token.clone(), + muxer.clone(), + timestamps, + ); + + finish_build( + setup_ctx, + audio_sources, + stop_token.clone(), + muxer, + timestamps, + done_tx, + None, + &path, + ) + .await?; + + Ok(OutputPipeline { + path, + first_timestamp_rx: first_rx, + stop_token: Some(stop_token.drop_guard()), + video_info: Some(video_info), + done_fut: done_rx, + pause_flag, + }) + } +} + +impl OutputPipelineBuilder { + pub async fn build( + self, + muxer_config: TMuxer::Config, + ) -> anyhow::Result { + let Self { + audio_sources, + timestamps, + path, + .. + } = self; + + if audio_sources.is_empty() { + return Err(anyhow!("Invariant: No audio sources")); + } + + let (mut setup_ctx, stop_token, done_tx, done_rx, pause_flag) = setup_build(); + + let (first_tx, first_rx) = oneshot::channel(); + + let muxer = setup_muxer::( + muxer_config, + &path, + None, + Some(AudioMixer::INFO), + &pause_flag, + &mut setup_ctx, + ) + .await?; + + finish_build( + setup_ctx, + audio_sources, + stop_token.clone(), + muxer, + timestamps, + done_tx, + Some(first_tx), + &path, + ) + .await?; + + Ok(OutputPipeline { + path, + first_timestamp_rx: first_rx, + stop_token: Some(stop_token.drop_guard()), + video_info: None, + done_fut: done_rx, + pause_flag, + }) + } +} + +fn setup_build() -> ( + SetupCtx, + CancellationToken, + oneshot::Sender>, + DoneFut, + Arc, +) { + let stop_token = CancellationToken::new(); + + let (done_tx, done_rx) = oneshot::channel(); + + ( + SetupCtx { + tasks: TaskPool(vec![]), + }, + stop_token, + done_tx, + done_rx + .map(|v| { + v.map_err(|s| anyhow::Error::from(s)) + .and_then(|v| v) + .map_err(|e| PipelineDoneError(Arc::new(e))) + }) + .boxed() + .shared(), + Arc::new(AtomicBool::new(false)), + ) +} + +async fn finish_build( + mut setup_ctx: SetupCtx, + audio_sources: Vec, + stop_token: CancellationToken, + muxer: Arc>, + timestamps: Timestamps, + done_tx: oneshot::Sender>, + first_tx: Option>, + path: &PathBuf, +) -> anyhow::Result<()> { + configure_audio( + &mut setup_ctx, + audio_sources, + stop_token.clone(), + muxer.clone(), + timestamps, + first_tx, + ) + .await + .context("audio mixer setup")?; + + tokio::spawn( + async move { + let (task_names, task_handles): (Vec<_>, Vec<_>) = + setup_ctx.tasks.0.into_iter().unzip(); + + let mut futures = FuturesUnordered::from_iter( + task_handles + .into_iter() + .zip(task_names) + .map(|(f, n)| f.map(move |r| (r, n))), + ); + + while let Some((result, name)) = futures.next().await { + match result { + Err(_) => { + return Err(anyhow::anyhow!("Task {name} failed unexpectedly")); + } + Ok(Err(e)) => { + return Err(anyhow::anyhow!("Task {name} failed: {e}")); + } + _ => {} + } + } + + Ok(()) + } + .then(async move |res| { + let muxer_res = muxer.lock().await.finish(); + + let _ = done_tx.send(match (res, muxer_res) { + (Err(e), _) | (_, Err(e)) => Err(e), + _ => Ok(()), + }); + }), + ); + + info!("Built pipeline for output {}", path.display()); + + Ok(()) +} + +async fn setup_video_source( + video_config: TVideo::Config, + setup_ctx: &mut SetupCtx, +) -> anyhow::Result<(TVideo, mpsc::Receiver)> { + let (video_tx, video_rx) = mpsc::channel(8); + let video_source = TVideo::setup(video_config, video_tx, setup_ctx).await?; + + Ok((video_source, video_rx)) +} + +async fn setup_muxer( + muxer_config: TMuxer::Config, + path: &PathBuf, + video_info: Option, + audio_info: Option, + pause_flag: &Arc, + setup_ctx: &mut SetupCtx, +) -> Result>, anyhow::Error> { + let muxer = Arc::new(Mutex::new( + TMuxer::setup( + muxer_config, + path.clone(), + video_info, + audio_info, + pause_flag.clone(), + &mut setup_ctx.tasks, + ) + .await?, + )); + + Ok(muxer) +} + +fn spawn_video_encoder, TVideo: VideoSource>( + setup_ctx: &mut SetupCtx, + mut video_source: TVideo, + mut video_rx: mpsc::Receiver, + first_tx: oneshot::Sender, + stop_token: CancellationToken, + muxer: Arc>, + timestamps: Timestamps, +) { + setup_ctx.tasks().spawn("mux-video", async move { + use futures::StreamExt; + + let mut first_tx = Some(first_tx); + + video_source.start().await?; + + stop_token + .run_until_cancelled(async { + while let Some(frame) = video_rx.next().await { + let timestamp = frame.timestamp(); + + if let Some(first_tx) = first_tx.take() { + let _ = first_tx.send(timestamp); + } + + muxer + .lock() + .await + .send_video_frame(frame, timestamp.duration_since(timestamps)) + .map_err(|e| anyhow!("Error queueing video frame: {e}"))?; + } + + Ok::<(), anyhow::Error>(()) + }) + .await; + + video_source.stop().await?; + + muxer.lock().await.stop(); + + Ok(()) + }); +} + +async fn configure_audio( + setup_ctx: &mut SetupCtx, + audio_sources: Vec, + stop_token: CancellationToken, + muxer: Arc>, + timestamps: Timestamps, + mut first_tx: Option>, +) -> anyhow::Result<()> { + if audio_sources.len() < 1 { + return Ok(()); + } + + let mut audio_mixer = AudioMixer::builder(); + + let mut erased_audio_sources = vec![]; + + for audio_source_setup in audio_sources { + let (tx, rx) = mpsc::channel(64); + let source = (audio_source_setup)(tx, setup_ctx).await?; + + audio_mixer.add_source(source.audio_info, rx); + erased_audio_sources.push(source); + } + + let (audio_tx, mut audio_rx) = mpsc::channel(64); + let (ready_tx, ready_rx) = oneshot::channel::>(); + let stop_flag = Arc::new(AtomicBool::new(false)); + + setup_ctx.tasks().spawn_thread("audio-mixer", { + let stop_flag = stop_flag.clone(); + move || audio_mixer.run(audio_tx, ready_tx, stop_flag) + }); + let _ = ready_rx + .await + .map_err(|_| anyhow::format_err!("Audio mixer crashed"))??; + + setup_ctx.tasks().spawn( + "audio-mixer-stop", + stop_token.child_token().cancelled_owned().map(move |_| { + stop_flag.store(true, atomic::Ordering::Relaxed); + Ok(()) + }), + ); + + for source in &mut erased_audio_sources { + (source.start_fn)(source.inner.as_mut()).await?; + } + + setup_ctx.tasks().spawn("mux-audio", { + let stop_token = stop_token.child_token(); + let muxer = muxer.clone(); + async move { + stop_token + .run_until_cancelled(async { + while let Some(frame) = audio_rx.next().await { + if let Some(first_tx) = first_tx.take() { + let _ = first_tx.send(frame.timestamp); + } + + let timestamp = frame.timestamp.duration_since(timestamps); + if let Err(e) = muxer.lock().await.send_audio_frame(frame, timestamp) { + error!("Audio encoder: {e}"); + } + } + }) + .await; + + for source in &mut erased_audio_sources { + let _ = (source.stop_fn)(source.inner.as_mut()).await; + } + + muxer.lock().await.stop(); + + Ok(()) + } + }); + + Ok(()) +} + +pub type DoneFut = Shared>>; + +pub struct OutputPipeline { + path: PathBuf, + pub first_timestamp_rx: oneshot::Receiver, + stop_token: Option, + video_info: Option, + done_fut: DoneFut, + pause_flag: Arc, +} + +pub struct FinishedOutputPipeline { + pub path: PathBuf, + pub first_timestamp: Timestamp, + pub video_info: Option, +} + +#[derive(Clone, Debug)] +pub struct PipelineDoneError(Arc); + +impl std::fmt::Display for PipelineDoneError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +impl std::error::Error for PipelineDoneError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + self.0.as_ref().source() + } +} + +impl OutputPipeline { + pub fn path(&self) -> &PathBuf { + &self.path + } + + pub async fn stop(mut self) -> anyhow::Result { + drop(self.stop_token.take()); + + self.done_fut.await?; + + Ok(FinishedOutputPipeline { + path: self.path, + first_timestamp: self.first_timestamp_rx.await?, + video_info: self.video_info, + }) + } + + pub fn pause(&self) { + self.pause_flag.store(true, atomic::Ordering::Relaxed); + } + + pub fn resume(&self) { + self.pause_flag.store(false, atomic::Ordering::Relaxed); + } + + pub fn video_info(&self) -> Option { + self.video_info + } + + pub fn done_fut(&self) -> DoneFut { + self.done_fut.clone() + } +} + +pub struct ChannelVideoSourceConfig { + info: VideoInfo, + rx: flume::Receiver, +} + +impl ChannelVideoSourceConfig { + pub fn new(info: VideoInfo, rx: flume::Receiver) -> Self { + Self { info, rx } + } +} + +pub struct ChannelVideoSource(VideoInfo, PhantomData); + +impl VideoSource for ChannelVideoSource { + type Config = ChannelVideoSourceConfig; + type Frame = TVideoFrame; + + async fn setup( + config: Self::Config, + mut video_tx: mpsc::Sender, + _: &mut SetupCtx, + ) -> anyhow::Result + where + Self: Sized, + { + tokio::spawn(async move { + while let Ok(frame) = config.rx.recv_async().await { + let _ = video_tx.send(frame).await; + } + }); + + Ok(Self(config.info, PhantomData)) + } + + fn video_info(&self) -> VideoInfo { + self.0 + } +} + +pub struct ChannelAudioSource { + info: AudioInfo, +} + +pub struct ChannelAudioSourceConfig { + info: AudioInfo, + rx: mpsc::Receiver, +} + +impl ChannelAudioSourceConfig { + pub fn new(info: AudioInfo, rx: mpsc::Receiver) -> Self { + Self { info, rx } + } +} + +impl AudioSource for ChannelAudioSource { + type Config = ChannelAudioSourceConfig; + + fn setup( + mut config: Self::Config, + mut tx: mpsc::Sender, + _: &mut SetupCtx, + ) -> impl Future> + 'static { + tokio::spawn(async move { + while let Some(frame) = config.rx.next().await { + let _ = tx.send(frame).await; + } + }); + + async move { Ok(ChannelAudioSource { info: config.info }) } + } + + fn audio_info(&self) -> AudioInfo { + self.info + } +} + +pub struct AudioFrame { + pub inner: ::ffmpeg::frame::Audio, + pub timestamp: Timestamp, +} + +impl AudioFrame { + pub fn new(inner: ::ffmpeg::frame::Audio, timestamp: Timestamp) -> Self { + Self { inner, timestamp } + } +} + +pub trait VideoSource: Send + 'static { + type Config; + type Frame: VideoFrame; + + fn setup( + config: Self::Config, + video_tx: mpsc::Sender, + ctx: &mut SetupCtx, + ) -> impl std::future::Future> + Send + where + Self: Sized; + + fn video_info(&self) -> VideoInfo; + + fn start(&mut self) -> BoxFuture<'_, anyhow::Result<()>> { + future::ready(Ok(())).boxed() + } + + fn stop(&mut self) -> BoxFuture<'_, anyhow::Result<()>> { + future::ready(Ok(())).boxed() + } +} + +struct ErasedAudioSource { + inner: Box, + audio_info: AudioInfo, + start_fn: fn(&mut dyn Any) -> BoxFuture<'_, anyhow::Result<()>>, + stop_fn: fn(&mut dyn Any) -> BoxFuture<'_, anyhow::Result<()>>, +} + +impl ErasedAudioSource { + pub fn new(source: TAudio) -> Self { + Self { + audio_info: source.audio_info(), + start_fn: |raw| { + raw.downcast_mut::() + .expect("Wrong type") + .start() + .boxed() + }, + stop_fn: |raw| { + raw.downcast_mut::() + .expect("Wrong type") + .stop() + .boxed() + }, + inner: Box::new(source), + } + } +} + +pub trait AudioSource: Send + 'static { + type Config: Send; + + fn setup( + config: Self::Config, + tx: mpsc::Sender, + ctx: &mut SetupCtx, + ) -> impl Future> + Send + 'static + where + Self: Sized; + + fn audio_info(&self) -> AudioInfo; + + fn start(&mut self) -> impl Future> + Send { + async { Ok(()) } + } + + fn stop(&mut self) -> impl Future> + Send { + async { Ok(()) } + } +} + +pub trait VideoFrame: Send + 'static { + fn timestamp(&self) -> Timestamp; +} + +pub trait Muxer: Send + 'static { + type Config; + + async fn setup( + config: Self::Config, + output_path: PathBuf, + video_config: Option, + audio_config: Option, + pause_flag: Arc, + tasks: &mut TaskPool, + ) -> anyhow::Result + where + Self: Sized; + + fn stop(&mut self) {} + + fn finish(&mut self) -> anyhow::Result<()>; +} + +pub trait AudioMuxer: Muxer { + fn send_audio_frame(&mut self, frame: AudioFrame, timestamp: Duration) -> anyhow::Result<()>; +} + +pub trait VideoMuxer: Muxer { + type VideoFrame; + + fn send_video_frame( + &mut self, + frame: Self::VideoFrame, + timestamp: Duration, + ) -> anyhow::Result<()>; +} diff --git a/crates/recording/src/output_pipeline/ffmpeg.rs b/crates/recording/src/output_pipeline/ffmpeg.rs new file mode 100644 index 000000000..268c18345 --- /dev/null +++ b/crates/recording/src/output_pipeline/ffmpeg.rs @@ -0,0 +1,145 @@ +use crate::{ + TaskPool, + output_pipeline::{AudioFrame, AudioMuxer, Muxer, VideoFrame, VideoMuxer}, +}; +use anyhow::{Context, anyhow}; +use cap_enc_ffmpeg::*; +use cap_media_info::{AudioInfo, VideoInfo}; +use cap_timestamp::Timestamp; +use std::{ + path::PathBuf, + sync::{Arc, atomic::AtomicBool}, + time::Duration, +}; + +#[derive(Clone)] +pub struct FFmpegVideoFrame { + pub inner: ffmpeg::frame::Video, + pub timestamp: Timestamp, +} + +impl VideoFrame for FFmpegVideoFrame { + fn timestamp(&self) -> Timestamp { + self.timestamp + } +} + +pub struct Mp4Muxer { + output: ffmpeg::format::context::Output, + video_encoder: Option, + audio_encoder: Option, +} + +impl Muxer for Mp4Muxer { + type Config = (); + + async fn setup( + _: Self::Config, + output_path: std::path::PathBuf, + video_config: Option, + audio_config: Option, + _: Arc, + tasks: &mut TaskPool, + ) -> anyhow::Result + where + Self: Sized, + { + let mut output = ffmpeg::format::output(&output_path)?; + + let video_encoder = video_config + .map(|video_config| H264Encoder::builder(video_config).build(&mut output)) + .transpose() + .context("video encoder")?; + + let audio_encoder = audio_config + .map(|config| AACEncoder::init(config, &mut output)) + .transpose() + .context("audio encoder")?; + + output.write_header()?; + + Ok(Self { + output, + video_encoder, + audio_encoder, + }) + } + + fn finish(&mut self) -> anyhow::Result<()> { + if let Some(video_encoder) = self.video_encoder.as_mut() { + video_encoder.finish(&mut self.output); + } + + if let Some(audio_encoder) = self.audio_encoder.as_mut() { + audio_encoder.finish(&mut self.output); + } + + self.output.write_trailer()?; + + Ok(()) + } +} + +impl VideoMuxer for Mp4Muxer { + type VideoFrame = FFmpegVideoFrame; + + fn send_video_frame( + &mut self, + frame: Self::VideoFrame, + timestamp: Duration, + ) -> anyhow::Result<()> { + if let Some(video_encoder) = self.video_encoder.as_mut() { + video_encoder.queue_frame(frame.inner, timestamp, &mut self.output); + } + + Ok(()) + } +} + +impl AudioMuxer for Mp4Muxer { + fn send_audio_frame(&mut self, frame: AudioFrame, timestamp: Duration) -> anyhow::Result<()> { + if let Some(audio_encoder) = self.audio_encoder.as_mut() { + audio_encoder.send_frame(frame.inner, timestamp, &mut self.output); + } + + Ok(()) + } +} + +pub struct OggMuxer(OggFile); + +impl Muxer for OggMuxer { + type Config = (); + + async fn setup( + _: Self::Config, + output_path: PathBuf, + _: Option, + audio_config: Option, + _: Arc, + _: &mut TaskPool, + ) -> anyhow::Result + where + Self: Sized, + { + let audio_config = + audio_config.ok_or_else(|| anyhow!("No audio configuration provided"))?; + + Ok(Self( + OggFile::init(output_path, |o| OpusEncoder::init(audio_config, o)) + .map_err(|e| anyhow!("Failed to initialize Opus encoder: {e}"))?, + )) + } + + fn finish(&mut self) -> anyhow::Result<()> { + self.0.finish(); + Ok(()) + } +} + +impl AudioMuxer for OggMuxer { + fn send_audio_frame(&mut self, frame: AudioFrame, timestamp: Duration) -> anyhow::Result<()> { + self.0.queue_frame(frame.inner, timestamp); + Ok(()) + } +} diff --git a/crates/recording/src/output_pipeline/macos.rs b/crates/recording/src/output_pipeline/macos.rs new file mode 100644 index 000000000..fd76b4fa0 --- /dev/null +++ b/crates/recording/src/output_pipeline/macos.rs @@ -0,0 +1,87 @@ +use crate::{ + output_pipeline::{AudioFrame, AudioMuxer, Muxer, TaskPool, VideoMuxer}, + sources::screen_capture, +}; +use anyhow::anyhow; +use cap_media_info::{AudioInfo, VideoInfo}; +use std::{ + path::PathBuf, + sync::{Arc, Mutex, atomic::AtomicBool}, + time::Duration, +}; + +#[derive(Clone)] +pub struct AVFoundationMp4Muxer( + Arc>, + Arc, +); + +#[derive(Default)] +pub struct AVFoundationMp4MuxerConfig { + pub output_height: Option, +} + +impl Muxer for AVFoundationMp4Muxer { + type Config = AVFoundationMp4MuxerConfig; + + async fn setup( + config: Self::Config, + output_path: PathBuf, + video_config: Option, + audio_config: Option, + pause_flag: Arc, + _tasks: &mut TaskPool, + ) -> anyhow::Result { + let video_config = + video_config.ok_or_else(|| anyhow!("Invariant: No video source provided"))?; + + Ok(Self( + Arc::new(Mutex::new( + cap_enc_avfoundation::MP4Encoder::init( + output_path, + video_config, + audio_config, + config.output_height, + ) + .map_err(|e| anyhow!("{e}"))?, + )), + pause_flag, + )) + } + + fn finish(&mut self) -> anyhow::Result<()> { + self.0.lock().map_err(|e| anyhow!("{e}"))?.finish(); + Ok(()) + } +} + +impl VideoMuxer for AVFoundationMp4Muxer { + type VideoFrame = screen_capture::VideoFrame; + + fn send_video_frame( + &mut self, + frame: Self::VideoFrame, + timestamp: Duration, + ) -> anyhow::Result<()> { + let mut mp4 = self.0.lock().map_err(|e| anyhow!("MuxerLock/{e}"))?; + + if self.1.load(std::sync::atomic::Ordering::Relaxed) { + mp4.pause(); + } else { + mp4.resume(); + } + + mp4.queue_video_frame(&frame.sample_buf, timestamp) + .map_err(|e| anyhow!("QueueVideoFrame/{e}")) + } +} + +impl AudioMuxer for AVFoundationMp4Muxer { + fn send_audio_frame(&mut self, frame: AudioFrame, timestamp: Duration) -> anyhow::Result<()> { + self.0 + .lock() + .map_err(|e| anyhow!("{e}"))? + .queue_audio_frame(frame.inner, timestamp) + .map_err(|e| anyhow!("{e}")) + } +} diff --git a/crates/recording/src/output_pipeline/mod.rs b/crates/recording/src/output_pipeline/mod.rs new file mode 100644 index 000000000..21687aca1 --- /dev/null +++ b/crates/recording/src/output_pipeline/mod.rs @@ -0,0 +1,15 @@ +mod core; +pub mod ffmpeg; + +pub use core::*; +pub use ffmpeg::*; + +#[cfg(target_os = "macos")] +mod macos; +#[cfg(target_os = "macos")] +pub use macos::*; + +#[cfg(windows)] +mod win; +#[cfg(windows)] +pub use win::*; diff --git a/crates/recording/src/output_pipeline/win.rs b/crates/recording/src/output_pipeline/win.rs new file mode 100644 index 000000000..01ec7f44f --- /dev/null +++ b/crates/recording/src/output_pipeline/win.rs @@ -0,0 +1,234 @@ +use crate::{AudioFrame, AudioMuxer, Muxer, TaskPool, VideoMuxer, screen_capture}; +use anyhow::anyhow; +use cap_enc_ffmpeg::AACEncoder; +use cap_media_info::{AudioInfo, VideoInfo}; +use futures::channel::oneshot; +use std::{ + path::PathBuf, + sync::{ + Arc, Mutex, + atomic::AtomicBool, + mpsc::{SyncSender, sync_channel}, + }, + time::Duration, +}; +use tracing::*; +use windows::{ + Foundation::TimeSpan, + Graphics::SizeInt32, + Win32::Graphics::{Direct3D11::ID3D11Device, Dxgi::Common::DXGI_FORMAT}, +}; + +/// Muxes to MP4 using a combination of FFmpeg and Media Foundation +pub struct WindowsMuxer { + video_tx: SyncSender>, + output: Arc>, + audio_encoder: Option, +} + +pub struct WindowsMuxerConfig { + pub pixel_format: DXGI_FORMAT, + pub d3d_device: ID3D11Device, + pub frame_rate: u32, + pub bitrate_multiplier: f32, +} + +impl Muxer for WindowsMuxer { + type Config = WindowsMuxerConfig; + + async fn setup( + config: Self::Config, + output_path: PathBuf, + video_config: Option, + audio_config: Option, + _: Arc, + tasks: &mut TaskPool, + ) -> anyhow::Result + where + Self: Sized, + { + let video_config = + video_config.ok_or_else(|| anyhow!("invariant: video config expected"))?; + let (video_tx, video_rx) = sync_channel::>(8); + + let mut output = ffmpeg::format::output(&output_path)?; + let audio_encoder = audio_config + .map(|config| AACEncoder::init(config, &mut output)) + .transpose()?; + + let output = Arc::new(Mutex::new(output)); + let (ready_tx, ready_rx) = oneshot::channel(); + + { + let output = output.clone(); + + tasks.spawn_thread("windows-encoder", move || { + cap_mediafoundation_utils::thread_init(); + + let encoder = (|| { + let mut output = output.lock().unwrap(); + + let native_encoder = + cap_enc_mediafoundation::H264Encoder::new_with_scaled_output( + &config.d3d_device, + config.pixel_format, + SizeInt32 { + Width: video_config.width as i32, + Height: video_config.height as i32, + }, + SizeInt32 { + Width: video_config.width as i32, + Height: video_config.height as i32, + }, + config.frame_rate, + config.bitrate_multiplier, + ); + + match native_encoder { + Ok(encoder) => cap_mediafoundation_ffmpeg::H264StreamMuxer::new( + &mut output, + cap_mediafoundation_ffmpeg::MuxerConfig { + width: video_config.width, + height: video_config.height, + fps: config.frame_rate, + bitrate: encoder.bitrate(), + }, + ) + .map(|muxer| either::Left((encoder, muxer))) + .map_err(|e| anyhow!("{e}")), + Err(e) => { + use tracing::{error, info}; + + error!("Failed to create native encoder: {e}"); + info!("Falling back to software H264 encoder"); + + cap_enc_ffmpeg::H264Encoder::builder(video_config) + .build(&mut output) + .map(either::Right) + .map_err(|e| anyhow!("ScreenSoftwareEncoder/{e}")) + } + } + })(); + + let encoder = match encoder { + Ok(encoder) => { + if ready_tx.send(Ok(())).is_err() { + info!("Failed to send ready signal"); + return; + } + encoder + } + Err(e) => { + let _ = ready_tx.send(Err(e)); + return; + } + }; + + match encoder { + either::Left((mut encoder, mut muxer)) => { + trace!("Running native encoder"); + let mut first_timestamp = None; + encoder + .run( + Arc::new(AtomicBool::default()), + || { + let Ok(Some((frame, _))) = video_rx.recv() else { + trace!("No more frames available"); + return Ok(None); + }; + + let frame_time = frame.inner().SystemRelativeTime()?; + let first_timestamp = first_timestamp.get_or_insert(frame_time); + let frame_time = TimeSpan { + Duration: frame_time.Duration - first_timestamp.Duration, + }; + + Ok(Some((frame.texture().clone(), frame_time))) + }, + |output_sample| { + let mut output = output.lock().unwrap(); + + let _ = muxer + .write_sample(&output_sample, &mut *output) + .map_err(|e| format!("WriteSample: {e}")); + + Ok(()) + }, + ) + .unwrap(); + } + either::Right(mut encoder) => { + while let Ok(Some((frame, time))) = video_rx.recv() { + let Ok(mut output) = output.lock() else { + continue; + }; + + // if pause_flag.load(std::sync::atomic::Ordering::Relaxed) { + // mp4.pause(); + // } else { + // mp4.resume(); + // } + + use scap_ffmpeg::AsFFmpeg; + + encoder.queue_frame( + frame + .as_ffmpeg() + .map_err(|e| format!("FrameAsFFmpeg: {e}")) + .unwrap(), + time, + &mut output, + ); + } + } + } + }); + } + + let _ = ready_rx.await; + + output.lock().unwrap().write_header()?; + + Ok(Self { + video_tx, + output, + audio_encoder, + }) + } + + fn stop(&mut self) { + let _ = self.video_tx.send(None); + } + + fn finish(&mut self) -> anyhow::Result<()> { + let mut output = self.output.lock().unwrap(); + if let Some(audio_encoder) = self.audio_encoder.as_mut() { + let _ = audio_encoder.finish(&mut output); + } + Ok(output.write_trailer()?) + } +} + +impl VideoMuxer for WindowsMuxer { + type VideoFrame = screen_capture::VideoFrame; + + fn send_video_frame( + &mut self, + frame: Self::VideoFrame, + timestamp: Duration, + ) -> anyhow::Result<()> { + Ok(self.video_tx.send(Some((frame.frame, timestamp)))?) + } +} + +impl AudioMuxer for WindowsMuxer { + fn send_audio_frame(&mut self, frame: AudioFrame, timestamp: Duration) -> anyhow::Result<()> { + if let Some(encoder) = self.audio_encoder.as_mut() + && let Ok(mut output) = self.output.lock() + { + encoder.send_frame(frame.inner, timestamp, &mut output)?; + } + + Ok(()) + } +} diff --git a/crates/recording/src/pipeline/builder.rs b/crates/recording/src/pipeline/builder.rs deleted file mode 100644 index 01fee9b08..000000000 --- a/crates/recording/src/pipeline/builder.rs +++ /dev/null @@ -1,174 +0,0 @@ -use flume::Receiver; -use indexmap::IndexMap; -use std::{ - thread::{self, JoinHandle}, - time::Duration, -}; -use tokio::sync::oneshot; -use tracing::{error, info}; - -use crate::pipeline::{ - MediaError, RecordingPipeline, - control::ControlBroadcast, - task::{PipelineReadySignal, PipelineSourceTask}, -}; - -struct Task { - ready_signal: Receiver>, - join_handle: JoinHandle<()>, - done_rx: tokio::sync::oneshot::Receiver>, -} - -#[derive(Default)] -pub struct PipelineBuilder { - control: ControlBroadcast, - tasks: IndexMap, -} - -impl PipelineBuilder { - pub fn spawn_source( - &mut self, - name: impl Into, - mut task: impl PipelineSourceTask + 'static, - ) { - let name = name.into(); - let control_signal = self.control.add_listener(name.clone()); - - self.spawn_task(name.clone(), move |ready_signal| { - let res = task.run(ready_signal.clone(), control_signal); - - if let Err(e) = &res - && !ready_signal.is_disconnected() - { - let _ = ready_signal.send(Err(MediaError::Any(format!("Task/{name}/{e}").into()))); - } - - res - }); - } - - pub fn spawn_task( - &mut self, - name: impl Into, - launch: impl FnOnce(PipelineReadySignal) -> Result<(), String> + Send + 'static, - ) { - let name = name.into(); - - if self.tasks.contains_key(&name) { - panic!("A task with the name {name} has already been added to the pipeline"); - } - - let (ready_sender, ready_signal) = flume::bounded(1); - - let dispatcher = tracing::dispatcher::get_default(|d| d.clone()); - let span = tracing::error_span!("pipeline", task = &name); - - let (done_tx, done_rx) = tokio::sync::oneshot::channel::>(); - - let join_handle = thread::spawn({ - let name = name.clone(); - move || { - tracing::dispatcher::with_default(&dispatcher, || { - let result = span - .in_scope(|| { - std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - info!("launching task '{name}'"); - let res = launch(ready_sender); - info!("task '{name}' done"); - res - })) - .map_err(|e| { - if let Some(s) = e.downcast_ref::<&'static str>() { - format!("Panicked: {s}") - } else if let Some(s) = e.downcast_ref::() { - format!("Panicked: {s}") - } else { - "Panicked: Unknown error".to_string() - } - }) - }) - .and_then(|v| v); - let _ = done_tx.send(result); - }); - } - }); - - self.tasks.insert( - name, - Task { - ready_signal, - join_handle, - done_rx, - }, - ); - } -} - -impl PipelineBuilder { - pub async fn build( - self, - ) -> Result<(RecordingPipeline, oneshot::Receiver>), MediaError> { - let Self { control, tasks } = self; - - if tasks.is_empty() { - return Err(MediaError::EmptyPipeline); - } - - let mut task_handles = IndexMap::new(); - - let mut stop_rx = vec![]; - let mut task_names = vec![]; - - // TODO: Shut down tasks if launch failed. - for (name, task) in tasks.into_iter() { - // TODO: Wait for these in parallel? - let launch_timeout = if cfg!(target_os = "macos") { 15 } else { 5 }; - tokio::time::timeout( - Duration::from_secs(launch_timeout), - task.ready_signal.recv_async(), - ) - .await - .map_err(|_| MediaError::TaskLaunch(format!("task timed out: '{name}'")))? - .map_err(|e| MediaError::TaskLaunch(format!("'{name}' build / {e}")))??; - - task_handles.insert(name.clone(), task.join_handle); - stop_rx.push(task.done_rx); - task_names.push(name); - } - - tokio::time::sleep(Duration::from_millis(10)).await; - - let (done_tx, done_rx) = oneshot::channel(); - - tokio::spawn(async move { - let (result, index, _) = futures::future::select_all(stop_rx).await; - let task_name = &task_names[index]; - - let result = match result { - Ok(Err(error)) => Err(format!("Task/{task_name}/{error}")), - Err(_) => Err(format!("Task/{task_name}/Unknown")), - _ => Ok(()), - }; - - if let Err(e) = &result { - error!("{e}"); - } - - let _ = done_tx.send(result); - }); - - Ok(( - RecordingPipeline { - control, - task_handles, - is_shutdown: false, - }, - done_rx, - )) - } -} - -// pub struct PipelinePathBuilder { -// pipeline: PipelineBuilder, -// next_input: Receiver, -// } diff --git a/crates/recording/src/pipeline/control.rs b/crates/recording/src/pipeline/control.rs deleted file mode 100644 index 5df77bafa..000000000 --- a/crates/recording/src/pipeline/control.rs +++ /dev/null @@ -1,82 +0,0 @@ -use flume::{Receiver, Sender, TryRecvError}; -use indexmap::IndexMap; -use tracing::debug; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum Control { - Play, - Shutdown, -} - -#[derive(Clone)] -pub struct PipelineControlSignal { - pub last_value: Option, - pub receiver: Receiver, -} - -impl PipelineControlSignal { - pub fn receiver(&self) -> &Receiver { - &self.receiver - } - - pub fn last_cached(&self) -> Option { - self.last_value - } - - pub fn last(&mut self) -> Option { - self.blocking_last_if(false) - } - - pub fn blocking_last(&mut self) -> Option { - self.blocking_last_if(true) - } - - pub fn blocking_last_if(&mut self, should_block: bool) -> Option { - match self.last_value { - Some(Control::Play) if !should_block => { - // Only peek for a new signal, else relinquish control to the caller - match self.receiver.try_recv() { - Ok(control) => { - debug!("Received new signal: {control:?}"); - self.last_value = Some(control) - } - Err(TryRecvError::Empty) => {} - Err(TryRecvError::Disconnected) => self.last_value = None, - }; - - self.last_value - } - _ => { - // For all else, block until a signal is sent. - // TODO: Maybe also spin down until the signal is different from the last value we have? - self.last_value = self.receiver.recv().ok(); - - self.last_value - } - } - } -} - -/// An extremely naive broadcast channel. Sends values synchronously to all receivers, -/// might block if one receiver takes too long to receive value. -#[derive(Debug, Default, Clone)] -pub(super) struct ControlBroadcast { - listeners: IndexMap>, -} - -impl ControlBroadcast { - pub fn add_listener(&mut self, name: String) -> PipelineControlSignal { - let (sender, receiver) = flume::bounded(1); - self.listeners.insert(name, sender); - PipelineControlSignal { - last_value: None, - receiver, - } - } - - pub async fn broadcast(&mut self, value: Control) { - for (_, listener) in self.listeners.iter() { - let _ = listener.send_async(value).await; - } - } -} diff --git a/crates/recording/src/pipeline/mod.rs b/crates/recording/src/pipeline/mod.rs deleted file mode 100644 index f0050b794..000000000 --- a/crates/recording/src/pipeline/mod.rs +++ /dev/null @@ -1,50 +0,0 @@ -use indexmap::IndexMap; -use std::thread::JoinHandle; -use tracing::{info, trace}; - -pub mod audio_buffer; -pub mod builder; -pub mod control; -pub mod task; - -use crate::MediaError; - -use builder::PipelineBuilder; -use control::{Control, ControlBroadcast, PipelineControlSignal}; - -pub struct RecordingPipeline { - control: ControlBroadcast, - task_handles: IndexMap>, - is_shutdown: bool, -} - -impl RecordingPipeline { - pub fn builder() -> PipelineBuilder { - PipelineBuilder::default() - } - - pub async fn play(&mut self) -> Result<(), MediaError> { - if self.is_shutdown { - return Err(MediaError::ShutdownPipeline); - }; - - self.control.broadcast(Control::Play).await; - - Ok(()) - } - - pub async fn shutdown(&mut self) -> Result<(), MediaError> { - if self.is_shutdown { - return Err(MediaError::ShutdownPipeline); - }; - - trace!("Shutting down pipeline"); - self.control.broadcast(Control::Shutdown).await; - for (_name, task) in self.task_handles.drain(..) { - let _ = task.join(); - } - info!("Pipeline stopped"); - // TODO: Collect shutdown errors? - Ok(()) - } -} diff --git a/crates/recording/src/pipeline/task.rs b/crates/recording/src/pipeline/task.rs deleted file mode 100644 index 0776143eb..000000000 --- a/crates/recording/src/pipeline/task.rs +++ /dev/null @@ -1,19 +0,0 @@ -use flume::Sender; - -use crate::pipeline::{MediaError, PipelineControlSignal}; - -const DEFAULT_QUEUE_SIZE: usize = 2048; - -pub type PipelineReadySignal = Sender>; - -pub trait PipelineSourceTask: Send { - fn run( - &mut self, - ready_signal: PipelineReadySignal, - control_signal: PipelineControlSignal, - ) -> Result<(), String>; - - fn queue_size(&self) -> usize { - DEFAULT_QUEUE_SIZE - } -} diff --git a/crates/recording/src/sources/audio_input.rs b/crates/recording/src/sources/audio_input.rs deleted file mode 100644 index af10294e8..000000000 --- a/crates/recording/src/sources/audio_input.rs +++ /dev/null @@ -1,109 +0,0 @@ -use crate::{ - feeds::microphone::{self, MicrophoneFeedLock, MicrophoneSamples}, - pipeline::{control::Control, task::PipelineSourceTask}, -}; -use cap_fail::fail; -use cap_media::MediaError; -use cap_media_info::AudioInfo; -use cap_timestamp::Timestamp; -use cpal::{Device, SupportedStreamConfig}; -use ffmpeg::frame::Audio as FFAudio; -use flume::{Receiver, Sender}; -use indexmap::IndexMap; -use std::sync::Arc; -use tracing::{error, info}; - -pub type AudioInputDeviceMap = IndexMap; - -pub struct AudioInputSource { - feed: Arc, - audio_info: AudioInfo, - tx: Sender<(FFAudio, Timestamp)>, -} - -impl AudioInputSource { - pub fn init(feed: Arc, tx: Sender<(FFAudio, Timestamp)>) -> Self { - Self { - audio_info: *feed.audio_info(), - feed, - tx, - } - } - - pub fn info(&self) -> AudioInfo { - self.audio_info - } - - fn process_frame(&mut self, samples: MicrophoneSamples) -> Result<(), MediaError> { - let timestamp = Timestamp::from_cpal(samples.info.timestamp().capture); - - let frame = self.audio_info.wrap_frame(&samples.data); - if self.tx.send((frame, timestamp)).is_err() { - return Err(MediaError::Any( - "Pipeline is unreachable! Stopping capture".into(), - )); - } - - Ok(()) - } - - fn pause_and_drain_frames(&mut self, frames_rx: Receiver) { - let frames: Vec = frames_rx.drain().collect(); - - for frame in frames { - if let Err(error) = self.process_frame(frame) { - eprintln!("{error}"); - break; - } - } - } -} - -impl PipelineSourceTask for AudioInputSource { - fn run( - &mut self, - ready_signal: crate::pipeline::task::PipelineReadySignal, - mut control_signal: crate::pipeline::control::PipelineControlSignal, - ) -> Result<(), String> { - info!("Preparing audio input source thread..."); - - let mut samples_rx: Option> = None; - ready_signal.send(Ok(())).unwrap(); - - fail!("media::sources::audio_input::run"); - - let res = loop { - match control_signal.last() { - Some(Control::Play) => { - let samples = samples_rx.get_or_insert_with(|| { - let (tx, rx) = flume::bounded(5); - let _ = self.feed.ask(microphone::AddSender(tx)).blocking_send(); - rx - }); - - match samples.recv() { - Ok(samples) => { - if let Err(error) = self.process_frame(samples) { - error!("{error}"); - break Err(error.to_string()); - } - } - Err(_) => { - error!("Lost connection with the camera feed"); - break Err("Lost connection with the camera feed".to_string()); - } - } - } - Some(Control::Shutdown) | None => { - if let Some(rx) = samples_rx.take() { - self.pause_and_drain_frames(rx); - } - break Ok(()); - } - } - }; - - info!("Shut down audio input source thread."); - res - } -} diff --git a/crates/recording/src/sources/audio_mixer.rs b/crates/recording/src/sources/audio_mixer.rs index 8e28e8855..9d27e7233 100644 --- a/crates/recording/src/sources/audio_mixer.rs +++ b/crates/recording/src/sources/audio_mixer.rs @@ -1,12 +1,20 @@ -use crate::pipeline::task::PipelineSourceTask; use cap_media_info::AudioInfo; use cap_timestamp::{Timestamp, Timestamps}; -use flume::{Receiver, Sender}; +use futures::{ + SinkExt, + channel::{mpsc, oneshot}, +}; use std::{ collections::VecDeque, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, time::{Duration, Instant}, }; -use tracing::debug; +use tracing::{debug, info}; + +use crate::output_pipeline::AudioFrame; // Wait TICK_MS for frames to arrive // Assume all sources' frames for that tick have arrived after TICK_MS @@ -15,22 +23,26 @@ use tracing::debug; // Current problem is generating an output timestamp that lines up with the input's timestamp struct MixerSource { - rx: Receiver<(ffmpeg::frame::Audio, Timestamp)>, + rx: mpsc::Receiver, info: AudioInfo, - buffer: VecDeque<(ffmpeg::frame::Audio, Timestamp)>, + buffer: VecDeque, buffer_last: Option<(Timestamp, Duration)>, } pub struct AudioMixerBuilder { sources: Vec, - output: Sender<(ffmpeg::frame::Audio, Timestamp)>, +} + +impl Default for AudioMixerBuilder { + fn default() -> Self { + Self::new() + } } impl AudioMixerBuilder { - pub fn new(output: Sender<(ffmpeg::frame::Audio, Timestamp)>) -> Self { + pub fn new() -> Self { Self { sources: Vec::new(), - output, } } @@ -38,7 +50,7 @@ impl AudioMixerBuilder { !self.sources.is_empty() } - pub fn add_source(&mut self, info: AudioInfo, rx: Receiver<(ffmpeg::frame::Audio, Timestamp)>) { + pub fn add_source(&mut self, info: AudioInfo, rx: mpsc::Receiver) { self.sources.push(MixerSource { info, rx, @@ -47,7 +59,7 @@ impl AudioMixerBuilder { }); } - pub fn build(self) -> Result { + pub fn build(self, output: mpsc::Sender) -> Result { let mut filter_graph = ffmpeg::filter::Graph::new(); let mut abuffers = self @@ -109,10 +121,10 @@ impl AudioMixerBuilder { Ok(AudioMixer { sources: self.sources, samples_out: 0, - output: self.output, last_tick: None, abuffers, abuffersink, + output, _filter_graph: filter_graph, _amix: amix, _aformat: aformat, @@ -120,12 +132,67 @@ impl AudioMixerBuilder { timestamps: Timestamps::now(), }) } + + async fn spawn(self, output: mpsc::Sender) -> anyhow::Result { + let (ready_tx, ready_rx) = oneshot::channel::>(); + let stop_flag = Arc::new(AtomicBool::new(false)); + + let thread_handle = std::thread::spawn({ + let stop_flag = stop_flag.clone(); + move || self.run(output, ready_tx, stop_flag) + }); + + ready_rx + .await + .map_err(|_| anyhow::format_err!("Audio mixer crashed"))??; + + info!("Audio mixer ready"); + + Ok(AudioMixerHandle { + thread_handle, + stop_flag, + }) + } + + pub fn run( + self, + output: mpsc::Sender, + ready_tx: oneshot::Sender>, + stop_flag: Arc, + ) { + let start = Timestamps::now(); + + let mut mixer = match self.build(output) { + Ok(mixer) => mixer, + Err(e) => { + tracing::error!("Failed to build audio mixer: {}", e); + let _ = ready_tx.send(Err(e.into())); + return; + } + }; + + let _ = ready_tx.send(Ok(())); + + loop { + if stop_flag.load(Ordering::Relaxed) { + info!("Mixer stop flag triggered"); + break; + } + + if let Err(()) = mixer.tick(start, Timestamp::Instant(Instant::now())) { + info!("Mixer tick errored"); + break; + } + + std::thread::sleep(Duration::from_millis(5)); + } + } } pub struct AudioMixer { sources: Vec, samples_out: usize, - output: Sender<(ffmpeg::frame::Audio, Timestamp)>, + output: mpsc::Sender, last_tick: Option, // sample_timestamps: VecDeque<(usize, Timestamp)>, abuffers: Vec, @@ -174,14 +241,18 @@ impl AudioMixer { let timestamp = last_end + (elapsed_since_last - remaining); source.buffer_last = Some((timestamp, Self::BUFFER_TIMEOUT)); - source.buffer.push_back((frame, timestamp)); + source.buffer.push_back(AudioFrame::new(frame, timestamp)); remaining -= Self::BUFFER_TIMEOUT; } } } - while let Ok((frame, timestamp)) = source.rx.try_recv() { + while let Ok(Some(AudioFrame { + inner: frame, + timestamp, + })) = source.rx.try_next() + { // if gap between incoming and last, insert silence if let Some((buffer_last_timestamp, buffer_last_duration)) = source.buffer_last { let timestamp_elapsed = timestamp.duration_since(self.timestamps); @@ -218,7 +289,7 @@ impl AudioMixer { timestamp, Duration::from_secs_f64(silence_samples_count as f64 / rate as f64), )); - source.buffer.push_back((frame, timestamp)); + source.buffer.push_back(AudioFrame::new(frame, timestamp)); } } } @@ -227,7 +298,7 @@ impl AudioMixer { timestamp, Duration::from_secs_f64(frame.samples() as f64 / frame.rate() as f64), )); - source.buffer.push_back((frame, timestamp)); + source.buffer.push_back(AudioFrame::new(frame, timestamp)); } } @@ -235,12 +306,13 @@ impl AudioMixer { self.start_timestamp = self .sources .iter() - .filter_map(|s| s.buffer.get(0)) + .filter_map(|s| s.buffer.front()) .min_by(|a, b| { - a.1.duration_since(self.timestamps) - .cmp(&b.1.duration_since(self.timestamps)) + a.timestamp + .duration_since(self.timestamps) + .cmp(&b.timestamp.duration_since(self.timestamps)) }) - .map(|v| v.1); + .map(|v| v.timestamp); } if let Some(start_timestamp) = self.start_timestamp { @@ -275,7 +347,7 @@ impl AudioMixer { timestamp, Duration::from_secs_f64(chunk_samples as f64 / rate as f64), )); - source.buffer.push_front((frame, timestamp)); + source.buffer.push_front(AudioFrame::new(frame, timestamp)); remaining -= Self::BUFFER_TIMEOUT; } @@ -294,7 +366,7 @@ impl AudioMixer { for (i, source) in self.sources.iter_mut().enumerate() { for buffer in source.buffer.drain(..) { - let _ = self.abuffers[i].source().add(&buffer.0); + let _ = self.abuffers[i].source().add(&buffer.inner); } } @@ -307,7 +379,7 @@ impl AudioMixer { if self .output - .send((filtered, Timestamp::Instant(timestamp))) + .try_send(AudioFrame::new(filtered, Timestamp::Instant(timestamp))) .is_err() { return Err(()); @@ -321,42 +393,32 @@ impl AudioMixer { Ok(()) } - pub fn builder(output: Sender<(ffmpeg::frame::Audio, Timestamp)>) -> AudioMixerBuilder { - AudioMixerBuilder::new(output) + pub fn builder() -> AudioMixerBuilder { + AudioMixerBuilder::new() } } -impl PipelineSourceTask for AudioMixerBuilder { - fn run( - &mut self, - ready_signal: crate::pipeline::task::PipelineReadySignal, - mut control_signal: crate::pipeline::control::PipelineControlSignal, - ) -> Result<(), String> { - let start = Timestamps::now(); - - let this = std::mem::replace(self, AudioMixerBuilder::new(self.output.clone())); - - let mut mixer = this.build().map_err(|e| format!("BuildMixer: {e}"))?; - - let _ = ready_signal.send(Ok(())); - - loop { - if control_signal - .last() - .map(|v| matches!(v, crate::pipeline::control::Control::Shutdown)) - .unwrap_or(false) - { - break; - } - - mixer - .tick(start, Timestamp::Instant(Instant::now())) - .map_err(|()| format!("Audio mixer tick failed"))?; +pub struct AudioMixerHandle { + thread_handle: std::thread::JoinHandle<()>, + stop_flag: Arc, +} - std::thread::sleep(Duration::from_millis(5)); +impl AudioMixerHandle { + pub fn new(thread_handle: std::thread::JoinHandle<()>, stop_flag: Arc) -> Self { + Self { + thread_handle, + stop_flag, } + } - Ok(()) + pub fn stop(&self) { + self.stop_flag.store(true, Ordering::Relaxed); + } +} + +impl Drop for AudioMixerHandle { + fn drop(&mut self) { + self.stop_flag.store(true, Ordering::Relaxed); } } diff --git a/crates/recording/src/sources/camera.rs b/crates/recording/src/sources/camera.rs index f9da4beb3..3ef911f63 100644 --- a/crates/recording/src/sources/camera.rs +++ b/crates/recording/src/sources/camera.rs @@ -1,131 +1,44 @@ use crate::{ - MediaError, - feeds::camera::{self, CameraFeedLock, RawCameraFrame}, - pipeline::{control::Control, task::PipelineSourceTask}, + feeds::camera::{self, CameraFeedLock}, + ffmpeg::FFmpegVideoFrame, + output_pipeline::{SetupCtx, VideoSource}, }; +use anyhow::anyhow; use cap_media_info::VideoInfo; -use cap_timestamp::Timestamp; -use ffmpeg::frame; -use flume::{Receiver, Sender}; +use futures::{SinkExt, channel::mpsc}; use std::sync::Arc; -use tracing::{error, info}; -pub struct CameraSource { - feed: Arc, - video_info: VideoInfo, - output: Sender<(frame::Video, Timestamp)>, -} - -impl CameraSource { - pub fn init(feed: Arc, output: Sender<(frame::Video, Timestamp)>) -> Self { - Self { - video_info: *feed.video_info(), - feed, - output, - } - } - - pub fn info(&self) -> VideoInfo { - self.video_info - } - - fn process_frame( - &self, - camera_frame: RawCameraFrame, - // first_frame_instant: Instant, - // first_frame_timestamp: Duration, - ) -> Result<(), MediaError> { - let check_skip_send = || { - cap_fail::fail_err!("media::sources::camera::skip_send", ()); - - Ok::<(), ()>(()) - }; - - if check_skip_send().is_err() { - return Ok(()); - } - - // let relative_timestamp = camera_frame.timestamp - first_frame_timestamp; - - if self - .output - .send(( - camera_frame.frame, - camera_frame.timestamp, // (first_frame_instant + relative_timestamp - self.start_instant).as_secs_f64(), - )) - .is_err() - { - return Err(MediaError::Any( - "Pipeline is unreachable! Stopping capture".into(), - )); - } - - Ok(()) - } - - fn pause_and_drain_frames(&mut self, frames_rx: Receiver) { - let frames: Vec = frames_rx.drain().collect(); - drop(frames_rx); - - for frame in frames { - // let first_frame_instant = *self.first_frame_instant.get_or_insert(frame.reference_time); - // let first_frame_timestamp = *self.first_frame_timestamp.get_or_insert(frame.timestamp); - - if let Err(error) = self.process_frame(frame) { - eprintln!("{error}"); - break; +pub struct Camera(Arc); + +impl VideoSource for Camera { + type Config = Arc; + type Frame = FFmpegVideoFrame; + + async fn setup( + config: Self::Config, + mut video_tx: mpsc::Sender, + _: &mut SetupCtx, + ) -> anyhow::Result + where + Self: Sized, + { + let (tx, rx) = flume::bounded(8); + + config + .ask(camera::AddSender(tx)) + .await + .map_err(|e| anyhow!("Failed to add camera sender: {e}"))?; + + tokio::spawn(async move { + while let Ok(frame) = rx.recv_async().await { + let _ = video_tx.send(frame).await; } - } - } -} - -impl PipelineSourceTask for CameraSource { - fn run( - &mut self, - ready_signal: crate::pipeline::task::PipelineReadySignal, - mut control_signal: crate::pipeline::control::PipelineControlSignal, - ) -> Result<(), String> { - let mut frames_rx: Option> = None; - - info!("Camera source ready"); - - let frames = frames_rx.get_or_insert_with(|| { - let (tx, rx) = flume::bounded(5); - let _ = self.feed.ask(camera::AddSender(tx)).blocking_send(); - rx }); - ready_signal.send(Ok(())).unwrap(); - - loop { - match control_signal.last() { - Some(Control::Play) => match frames.drain().last().or_else(|| frames.recv().ok()) { - Some(frame) => { - // let first_frame_instant = - // *self.first_frame_instant.get_or_insert(frame.reference_time); - // let first_frame_timestamp = - // *self.first_frame_timestamp.get_or_insert(frame.timestamp); - - if let Err(error) = self.process_frame(frame) { - eprintln!("{error}"); - break; - } - } - None => { - error!("Lost connection with the camera feed"); - break; - } - }, - Some(Control::Shutdown) | None => { - if let Some(rx) = frames_rx.take() { - self.pause_and_drain_frames(rx); - } - info!("Camera source stopped"); - break; - } - } - } + Ok(Self(config)) + } - Ok(()) + fn video_info(&self) -> VideoInfo { + *self.0.video_info() } } diff --git a/crates/recording/src/sources/microphone.rs b/crates/recording/src/sources/microphone.rs new file mode 100644 index 000000000..36a6fad63 --- /dev/null +++ b/crates/recording/src/sources/microphone.rs @@ -0,0 +1,50 @@ +use crate::{ + feeds::microphone::{self, MicrophoneFeedLock}, + output_pipeline::{AudioFrame, AudioSource}, +}; +use anyhow::anyhow; +use cap_media_info::AudioInfo; +use futures::{SinkExt, channel::mpsc}; +use std::sync::Arc; + +pub struct Microphone(AudioInfo); + +impl AudioSource for Microphone { + type Config = Arc; + + fn setup( + config: Self::Config, + mut audio_tx: mpsc::Sender, + _: &mut crate::SetupCtx, + ) -> impl Future> + 'static + where + Self: Sized, + { + async move { + let audio_info = config.audio_info(); + let (tx, rx) = flume::bounded(8); + + config + .ask(microphone::AddSender(tx)) + .await + .map_err(|e| anyhow!("Failed to add camera sender: {e}"))?; + + tokio::spawn(async move { + while let Ok(frame) = rx.recv_async().await { + let _ = audio_tx + .send(AudioFrame::new( + audio_info.wrap_frame(&frame.data), + frame.timestamp, + )) + .await; + } + }); + + Ok(Self(audio_info)) + } + } + + fn audio_info(&self) -> AudioInfo { + self.0 + } +} diff --git a/crates/recording/src/sources/mod.rs b/crates/recording/src/sources/mod.rs index 7dcc47b78..1c5cff932 100644 --- a/crates/recording/src/sources/mod.rs +++ b/crates/recording/src/sources/mod.rs @@ -1,8 +1,8 @@ -pub mod audio_input; pub mod audio_mixer; pub mod camera; +pub mod microphone; pub mod screen_capture; -pub use audio_input::*; pub use camera::*; -pub use screen_capture::*; +pub use microphone::*; +// pub use screen_capture::*; diff --git a/crates/recording/src/sources/screen_capture/macos.rs b/crates/recording/src/sources/screen_capture/macos.rs index 53f976d80..71b4e07e1 100644 --- a/crates/recording/src/sources/screen_capture/macos.rs +++ b/crates/recording/src/sources/screen_capture/macos.rs @@ -1,7 +1,20 @@ use super::*; +use crate::{ + ChannelAudioSourceConfig, + output_pipeline::{ + self, AudioFrame, ChannelAudioSource, ChannelVideoSource, ChannelVideoSourceConfig, + SetupCtx, + }, +}; +use anyhow::anyhow; use cidre::*; -use kameo::prelude::*; -use tracing::{debug, info, trace}; +use futures::{FutureExt, channel::mpsc, future::BoxFuture}; +use std::sync::{ + Arc, + atomic::{self, AtomicBool}, +}; +use tokio::sync::broadcast; +use tracing::debug; #[derive(Debug)] pub struct CMSampleBufferCapture; @@ -23,86 +36,6 @@ impl ScreenCaptureFormat for CMSampleBufferCapture { } } -#[derive(Actor)] -struct FrameHandler { - video_tx: Sender<(arc::R, Timestamp)>, - audio_tx: Option>, -} - -impl Message for FrameHandler { - type Reply = (); - - async fn handle( - &mut self, - msg: NewFrame, - _: &mut kameo::prelude::Context, - ) -> Self::Reply { - let frame = msg.0; - let sample_buffer = frame.sample_buf(); - - let mach_timestamp = cm::Clock::convert_host_time_to_sys_units(sample_buffer.pts()); - let timestamp = - Timestamp::MachAbsoluteTime(cap_timestamp::MachAbsoluteTimestamp::new(mach_timestamp)); - - match &frame { - scap_screencapturekit::Frame::Screen(frame) => { - if frame.image_buf().height() == 0 || frame.image_buf().width() == 0 { - return; - } - - let check_skip_send = || { - cap_fail::fail_err!("media::sources::screen_capture::skip_send", ()); - - Ok::<(), ()>(()) - }; - - if check_skip_send().is_ok() - && self - .video_tx - .send((sample_buffer.retained(), timestamp)) - .is_err() - { - warn!("Pipeline is unreachable"); - } - } - scap_screencapturekit::Frame::Audio(_) => { - use ffmpeg::ChannelLayout; - - let res = || { - cap_fail::fail_err!("screen_capture audio skip", ()); - Ok::<(), ()>(()) - }; - if res().is_err() { - return; - } - - let Some(audio_tx) = &self.audio_tx else { - return; - }; - - let buf_list = sample_buffer.audio_buf_list::<2>().unwrap(); - let slice = buf_list.block().as_slice().unwrap(); - - let mut frame = ffmpeg::frame::Audio::new( - ffmpeg::format::Sample::F32(ffmpeg::format::sample::Type::Planar), - sample_buffer.num_samples() as usize, - ChannelLayout::STEREO, - ); - frame.set_rate(48_000); - let data_bytes_size = buf_list.list().buffers[0].data_bytes_size; - for i in 0..frame.planes() { - frame.data_mut(i).copy_from_slice( - &slice[i * data_bytes_size as usize..(i + 1) * data_bytes_size as usize], - ); - } - - let _ = audio_tx.send((frame, timestamp)); - } - _ => {} - } - } -} - #[derive(Debug, thiserror::Error)] enum SourceError { #[error("NoDisplay: Id '{0}'")] @@ -111,183 +44,174 @@ enum SourceError { AsContentFilter, #[error("CreateActor: {0}")] CreateActor(arc::R), - #[error("StartCapturing/{0}")] - StartCapturing(SendError), #[error("DidStopWithError: {0}")] DidStopWithError(arc::R), } -impl PipelineSourceTask for ScreenCaptureSource { - fn run( - &mut self, - ready_signal: crate::pipeline::task::PipelineReadySignal, - control_signal: crate::pipeline::control::PipelineControlSignal, - ) -> Result<(), String> { - trace!("PipelineSourceTask::run"); - - let video_tx = self.video_tx.clone(); - let audio_tx = self.audio_tx.clone(); - let config = self.config.clone(); - let shareable_content = self.shareable_content.clone(); - - self.tokio_handle - .block_on(async move { - let captures_audio = audio_tx.is_some(); - let frame_handler = FrameHandler::spawn(FrameHandler { video_tx, audio_tx }); - - let display = Display::from_id(&config.display) - .ok_or_else(|| SourceError::NoDisplay(config.display))?; - - let content_filter = display - .raw_handle() - .as_content_filter(shareable_content) - .await - .ok_or_else(|| SourceError::AsContentFilter)?; - - debug!("SCK content filter: {:?}", content_filter); - - let size = { - let logical_size = config - .crop_bounds - .map(|bounds| bounds.size()) - .or_else(|| display.logical_size()) - .unwrap(); - - let scale = display.physical_size().unwrap().width() - / display.logical_size().unwrap().width(); - - PhysicalSize::new(logical_size.width() * scale, logical_size.height() * scale) - }; - - debug!("size: {:?}", size); - - let mut settings = scap_screencapturekit::StreamCfgBuilder::default() - .with_width(size.width() as usize) - .with_height(size.height() as usize) - .with_fps(config.fps as f32) - .with_shows_cursor(config.show_cursor) - .with_captures_audio(captures_audio) - .build(); - - settings.set_pixel_format(cv::PixelFormat::_32_BGRA); - settings.set_color_space_name(cg::color_space::names::srgb()); - - if let Some(crop_bounds) = config.crop_bounds { - debug!("crop bounds: {:?}", crop_bounds); - settings.set_src_rect(cg::Rect::new( - crop_bounds.position().x(), - crop_bounds.position().y(), - crop_bounds.size().width(), - crop_bounds.size().height(), - )); - } - - let (error_tx, error_rx) = flume::bounded(1); - - trace!("Spawning ScreenCaptureActor"); - - let capturer = ScreenCaptureActor::spawn( - ScreenCaptureActor::new( - content_filter, - settings, - frame_handler.recipient(), - error_tx.clone(), - ) - .map_err(SourceError::CreateActor)?, - ); - - info!("Spawned ScreenCaptureActor"); - - capturer - .ask(StartCapturing) - .await - .map_err(SourceError::StartCapturing)?; - - info!("Started capturing"); - - let _ = ready_signal.send(Ok(())); - - let stop = async move { - let _ = capturer.ask(StopCapturing).await; - let _ = capturer.stop_gracefully().await; - }; - - loop { - use futures::future::Either; - - match futures::future::select( - error_rx.recv_async(), - control_signal.receiver().recv_async(), - ) - .await - { - Either::Left((Ok(error), _)) => { - error!("Error capturing screen: {}", error); - stop.await; - return Err(SourceError::DidStopWithError(error)); - } - Either::Right((Ok(ctrl), _)) => { - if let Control::Shutdown = ctrl { - stop.await; - return Ok(()); - } - } - _ => { - warn!("Screen capture recv channels shutdown, exiting."); - - stop.await; +pub struct VideoFrame { + pub sample_buf: arc::R, + pub timestamp: Timestamp, +} - return Ok(()); - } - } - } - }) - .map_err(|e: SourceError| e.to_string()) +impl output_pipeline::VideoFrame for VideoFrame { + fn timestamp(&self) -> Timestamp { + self.timestamp } } -#[derive(Actor)] -pub struct ScreenCaptureActor { - capturer: scap_screencapturekit::Capturer, - capturing: bool, -} +impl ScreenCaptureConfig { + pub async fn to_sources( + &self, + ) -> anyhow::Result<(VideoSourceConfig, Option)> { + let (error_tx, error_rx) = broadcast::channel(1); + let (mut video_tx, video_rx) = flume::bounded(4); + let (mut audio_tx, audio_rx) = if self.system_audio { + let (tx, rx) = mpsc::channel(32); + (Some(tx), Some(rx)) + } else { + (None, None) + }; + + let display = Display::from_id(&self.config.display) + .ok_or_else(|| SourceError::NoDisplay(self.config.display.clone()))?; + + let content_filter = display + .raw_handle() + .as_content_filter(self.shareable_content.clone()) + .await + .ok_or_else(|| SourceError::AsContentFilter)?; + + debug!("SCK content filter: {:?}", content_filter); + + let size = { + let logical_size = self + .config + .crop_bounds + .map(|bounds| bounds.size()) + .or_else(|| display.logical_size()) + .unwrap(); + + let scale = + display.physical_size().unwrap().width() / display.logical_size().unwrap().width(); + + PhysicalSize::new(logical_size.width() * scale, logical_size.height() * scale) + }; -impl ScreenCaptureActor { - pub fn new( - target: arc::R, - settings: arc::R, - frame_handler: Recipient, - error_tx: Sender>, - ) -> Result> { + debug!("size: {:?}", size); + + let mut settings = scap_screencapturekit::StreamCfgBuilder::default() + .with_width(size.width() as usize) + .with_height(size.height() as usize) + .with_fps(self.config.fps as f32) + .with_shows_cursor(self.config.show_cursor) + .with_captures_audio(self.system_audio) + .build(); + + settings.set_pixel_format(cv::PixelFormat::_32_BGRA); + settings.set_color_space_name(cg::color_space::names::srgb()); + + if let Some(crop_bounds) = self.config.crop_bounds { + debug!("crop bounds: {:?}", crop_bounds); + settings.set_src_rect(cg::Rect::new( + crop_bounds.position().x(), + crop_bounds.position().y(), + crop_bounds.size().width(), + crop_bounds.size().height(), + )); + } cap_fail::fail_err!( "macos::ScreenCaptureActor::new", ns::Error::with_domain(ns::ErrorDomain::os_status(), 69420, None) ); - let _error_tx = error_tx.clone(); - let capturer_builder = scap_screencapturekit::Capturer::builder(target, settings) - .with_output_sample_buf_cb(move |frame| { - let check_err = || { - cap_fail::fail_err!( - "macos::ScreenCaptureActor output_sample_buf", - ns::Error::with_domain(ns::ErrorDomain::os_status(), 69420, None) + let builder = scap_screencapturekit::Capturer::builder(content_filter, settings) + .with_output_sample_buf_cb({ + move |frame| { + let sample_buffer = frame.sample_buf(); + + let mach_timestamp = + cm::Clock::convert_host_time_to_sys_units(sample_buffer.pts()); + let timestamp = Timestamp::MachAbsoluteTime( + cap_timestamp::MachAbsoluteTimestamp::new(mach_timestamp), ); - Result::<_, arc::R>::Ok(()) - }; - if let Err(e) = check_err() { - let _ = _error_tx.send(e); - } - let _ = frame_handler.tell(NewFrame(frame)).try_send(); + match &frame { + scap_screencapturekit::Frame::Screen(frame) => { + if frame.image_buf().height() == 0 || frame.image_buf().width() == 0 { + return; + } + + let check_skip_send = || { + cap_fail::fail_err!( + "media::sources::screen_capture::skip_send", + () + ); + + Ok::<(), ()>(()) + }; + + let _ = video_tx.try_send(VideoFrame { + sample_buf: sample_buffer.retained(), + timestamp, + }); + } + scap_screencapturekit::Frame::Audio(_) => { + use ffmpeg::ChannelLayout; + + let res = || { + cap_fail::fail_err!("screen_capture audio skip", ()); + Ok::<(), ()>(()) + }; + if res().is_err() { + return; + } + + let Some(audio_tx) = &mut audio_tx else { + return; + }; + + let buf_list = sample_buffer.audio_buf_list::<2>().unwrap(); + let slice = buf_list.block().as_slice().unwrap(); + + let mut frame = ffmpeg::frame::Audio::new( + ffmpeg::format::Sample::F32(ffmpeg::format::sample::Type::Planar), + sample_buffer.num_samples() as usize, + ChannelLayout::STEREO, + ); + frame.set_rate(48_000); + let data_bytes_size = buf_list.list().buffers[0].data_bytes_size; + for i in 0..frame.planes() { + frame.data_mut(i).copy_from_slice( + &slice[i * data_bytes_size as usize + ..(i + 1) * data_bytes_size as usize], + ); + } + + let _ = audio_tx.try_send(AudioFrame::new(frame, timestamp)); + } + _ => {} + } + } }) .with_stop_with_err_cb(move |_, err| { let _ = error_tx.send(err.retained()); }); - Ok(ScreenCaptureActor { - capturer: capturer_builder.build()?, - capturing: false, - }) + let capturer = Capturer::new(Arc::new(builder.build()?)); + Ok(( + VideoSourceConfig( + ChannelVideoSourceConfig::new(self.video_info, video_rx), + capturer.clone(), + error_rx.resubscribe(), + ), + audio_rx.map(|rx| { + SystemAudioSourceConfig( + ChannelAudioSourceConfig::new(self.audio_info(), rx), + capturer, + error_rx, + ) + }), + )) } } @@ -303,61 +227,149 @@ pub struct NewFrame(pub scap_screencapturekit::Frame); pub struct CaptureError(pub arc::R); -#[derive(Debug, Clone, thiserror::Error)] -pub enum StartCapturingError { - #[error("AlreadyCapturing")] - AlreadyCapturing, - #[error("Start: {0}")] - Start(arc::R), +struct Capturer { + started: Arc, + capturer: Arc, + // error_rx: broadcast::Receiver>, +} + +impl Clone for Capturer { + fn clone(&self) -> Self { + Self { + started: self.started.clone(), + capturer: self.capturer.clone(), + // error_rx: self.error_rx.resubscribe(), + } + } } -impl Message for ScreenCaptureActor { - type Reply = Result<(), StartCapturingError>; +impl Capturer { + fn new( + capturer: Arc, + // error_rx: broadcast::Receiver>, + ) -> Self { + Self { + started: Arc::new(AtomicBool::new(false)), + capturer, + // error_rx, + } + } - async fn handle( - &mut self, - _: StartCapturing, - _: &mut Context, - ) -> Self::Reply { - trace!("ScreenCaptureActor.StartCapturing"); + async fn start(&mut self) -> anyhow::Result<()> { + if !self.started.fetch_xor(true, atomic::Ordering::Relaxed) { + self.capturer.start().await?; + } - if self.capturing { - return Err(StartCapturingError::AlreadyCapturing); + Ok(()) + } + + async fn stop(&mut self) -> anyhow::Result<()> { + if self.started.fetch_xor(true, atomic::Ordering::Relaxed) { + self.capturer.stop().await?; } - trace!("Starting SCK capturer"); + Ok(()) + } +} + +pub struct VideoSourceConfig( + ChannelVideoSourceConfig, + Capturer, + broadcast::Receiver>, +); +pub struct VideoSource(ChannelVideoSource, Capturer); + +impl output_pipeline::VideoSource for VideoSource { + type Config = VideoSourceConfig; + type Frame = VideoFrame; + + async fn setup( + mut config: Self::Config, + video_tx: mpsc::Sender, + ctx: &mut SetupCtx, + ) -> anyhow::Result + where + Self: Sized, + { + ctx.tasks().spawn("screen-capture", async move { + if let Ok(err) = config.2.recv().await { + return Err(anyhow!("{err}")); + } + + Ok(()) + }); - self.capturer - .start() + ChannelVideoSource::setup(config.0, video_tx, ctx) .await - .map_err(StartCapturingError::Start)?; + .map(|source| Self(source, config.1)) + } - info!("Started SCK capturer"); + fn start(&mut self) -> BoxFuture<'_, anyhow::Result<()>> { + async move { + self.1.start().await?; - self.capturing = true; + Ok(()) + } + .boxed() + } - Ok(()) + fn stop(&mut self) -> BoxFuture<'_, anyhow::Result<()>> { + async move { + self.1.stop().await?; + + Ok(()) + } + .boxed() + } + + fn video_info(&self) -> VideoInfo { + self.0.video_info() } } -impl Message for ScreenCaptureActor { - type Reply = Result<(), StopCapturingError>; +pub struct SystemAudioSourceConfig( + ChannelAudioSourceConfig, + Capturer, + broadcast::Receiver>, +); + +pub struct SystemAudioSource(ChannelAudioSource, Capturer); + +impl output_pipeline::AudioSource for SystemAudioSource { + type Config = SystemAudioSourceConfig; + + fn setup( + mut config: Self::Config, + tx: mpsc::Sender, + ctx: &mut SetupCtx, + ) -> impl Future> + 'static + where + Self: Sized, + { + ctx.tasks().spawn("system-audio", async move { + if let Ok(err) = config.2.recv().await { + return Err(anyhow!("{err}")); + } - async fn handle( - &mut self, - _: StopCapturing, - _: &mut Context, - ) -> Self::Reply { - trace!("ScreenCaptureActor.StopCapturing"); + Ok(()) + }); - if !self.capturing { - return Err(StopCapturingError::NotCapturing); - }; + ChannelAudioSource::setup(config.0, tx, ctx).map(|v| v.map(|source| Self(source, config.1))) + } - if let Err(e) = self.capturer.stop().await { - error!("Silently failed to stop macOS capturer: {}", e); - } + async fn start(&mut self) -> anyhow::Result<()> { + self.1.start().await?; Ok(()) } + + async fn stop(&mut self) -> anyhow::Result<()> { + self.1.stop().await?; + + Ok(()) + } + + fn audio_info(&self) -> AudioInfo { + self.0.audio_info() + } } diff --git a/crates/recording/src/sources/screen_capture/mod.rs b/crates/recording/src/sources/screen_capture/mod.rs index 7e438ca57..ba4297980 100644 --- a/crates/recording/src/sources/screen_capture/mod.rs +++ b/crates/recording/src/sources/screen_capture/mod.rs @@ -1,17 +1,15 @@ -use crate::pipeline::{control::Control, task::PipelineSourceTask}; use cap_cursor_capture::CursorCropBounds; use cap_media_info::{AudioInfo, VideoInfo}; use cap_timestamp::Timestamp; -use flume::Sender; use scap_targets::{Display, DisplayId, Window, WindowId, bounds::*}; use serde::{Deserialize, Serialize}; use specta::Type; use std::time::SystemTime; -use tracing::{error, warn}; +use tracing::*; -#[cfg(windows)] +#[cfg(target_os = "windows")] mod windows; -#[cfg(windows)] +#[cfg(target_os = "windows")] pub use windows::*; #[cfg(target_os = "macos")] @@ -21,8 +19,9 @@ pub use macos::*; pub struct StopCapturing; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, thiserror::Error)] pub enum StopCapturingError { + #[error("NotCapturing")] NotCapturing, } @@ -189,13 +188,11 @@ impl ScreenCaptureTarget { } } -pub struct ScreenCaptureSource { +pub struct ScreenCaptureConfig { config: Config, video_info: VideoInfo, - tokio_handle: tokio::runtime::Handle, - video_tx: Sender<(TCaptureFormat::VideoFormat, Timestamp)>, - audio_tx: Option>, start_time: SystemTime, + pub system_audio: bool, _phantom: std::marker::PhantomData, #[cfg(windows)] d3d_device: ::windows::Win32::Graphics::Direct3D11::ID3D11Device, @@ -203,23 +200,19 @@ pub struct ScreenCaptureSource { shareable_content: cidre::arc::R, } -impl std::fmt::Debug for ScreenCaptureSource { +impl std::fmt::Debug for ScreenCaptureConfig { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("ScreenCaptureSource") // .field("bounds", &self.bounds) // .field("output_resolution", &self.output_resolution) .field("fps", &self.config.fps) .field("video_info", &self.video_info) - .field( - "audio_info", - &self.audio_tx.as_ref().map(|_| self.audio_info()), - ) .finish() } } -unsafe impl Send for ScreenCaptureSource {} -unsafe impl Sync for ScreenCaptureSource {} +unsafe impl Send for ScreenCaptureConfig {} +unsafe impl Sync for ScreenCaptureConfig {} pub trait ScreenCaptureFormat { type VideoFormat; @@ -229,15 +222,13 @@ pub trait ScreenCaptureFormat { fn audio_info() -> AudioInfo; } -impl Clone for ScreenCaptureSource { +impl Clone for ScreenCaptureConfig { fn clone(&self) -> Self { Self { config: self.config.clone(), video_info: self.video_info, - video_tx: self.video_tx.clone(), - audio_tx: self.audio_tx.clone(), - tokio_handle: self.tokio_handle.clone(), start_time: self.start_time, + system_audio: self.system_audio, _phantom: std::marker::PhantomData, #[cfg(windows)] d3d_device: self.d3d_device.clone(), @@ -274,16 +265,14 @@ pub enum ScreenCaptureInitError { NoBounds, } -impl ScreenCaptureSource { +impl ScreenCaptureConfig { #[allow(clippy::too_many_arguments)] pub async fn init( target: &ScreenCaptureTarget, show_cursor: bool, max_fps: u32, - video_tx: Sender<(TCaptureFormat::VideoFormat, Timestamp)>, - audio_tx: Option>, start_time: SystemTime, - tokio_handle: tokio::runtime::Handle, + system_audio: bool, #[cfg(windows)] d3d_device: ::windows::Win32::Graphics::Direct3D11::ID3D11Device, #[cfg(target_os = "macos")] shareable_content: cidre::arc::R, ) -> Result { @@ -405,10 +394,8 @@ impl ScreenCaptureSource { output_size.height() as u32, fps, ), - video_tx, - audio_tx, - tokio_handle, start_time, + system_audio, _phantom: std::marker::PhantomData, #[cfg(windows)] d3d_device, diff --git a/crates/recording/src/sources/screen_capture/windows.rs b/crates/recording/src/sources/screen_capture/windows.rs index 819cec845..66bcb5761 100644 --- a/crates/recording/src/sources/screen_capture/windows.rs +++ b/crates/recording/src/sources/screen_capture/windows.rs @@ -1,15 +1,25 @@ -use super::*; +use crate::{ + AudioFrame, ChannelAudioSource, ChannelVideoSource, ChannelVideoSourceConfig, SetupCtx, + output_pipeline, + screen_capture::{ScreenCaptureConfig, ScreenCaptureFormat}, +}; use ::windows::{ Graphics::Capture::GraphicsCaptureItem, Win32::Graphics::Direct3D11::{D3D11_BOX, ID3D11Device}, }; +use anyhow::anyhow; use cap_fail::fail_err; -use cap_timestamp::{PerformanceCounterTimestamp, Timestamps}; +use cap_media_info::{AudioInfo, VideoInfo}; +use cap_timestamp::{PerformanceCounterTimestamp, Timestamp, Timestamps}; use cpal::traits::{DeviceTrait, HostTrait}; -use futures::channel::oneshot; +use futures::{ + FutureExt, SinkExt, StreamExt, + channel::{mpsc, oneshot}, +}; use kameo::prelude::*; use scap_direct3d::StopCapturerError; use scap_ffmpeg::*; +use scap_targets::{Display, DisplayId}; use std::{ collections::VecDeque, time::{Duration, Instant}, @@ -47,174 +57,12 @@ impl ScreenCaptureFormat for Direct3DCapture { } } -struct FrameHandler { - capturer: WeakActorRef, - frames_dropped: u32, - last_cleanup: Instant, - last_log: Instant, - frame_events: VecDeque<(Instant, bool)>, - video_tx: Sender<(scap_direct3d::Frame, Timestamp)>, - target_fps: u32, - last_timestamp: Option, - timestamps: Timestamps, -} - -impl Actor for FrameHandler { - type Args = Self; - type Error = (); - - async fn on_start(args: Self::Args, self_actor: ActorRef) -> Result { - if let Some(capturer) = args.capturer.upgrade() { - self_actor.link(&capturer).await; - } - - Ok(args) - } - - async fn on_link_died( - &mut self, - actor_ref: WeakActorRef, - id: ActorID, - _: ActorStopReason, - ) -> Result, Self::Error> { - if self.capturer.id() == id - && let Some(self_actor) = actor_ref.upgrade() - { - let _ = self_actor.stop_gracefully().await; - - return Ok(std::ops::ControlFlow::Break(ActorStopReason::Normal)); - } - - Ok(std::ops::ControlFlow::Continue(())) - } -} - -impl FrameHandler { - // Helper function to clean up old frame events - fn cleanup_old_events(&mut self, now: Instant) { - let cutoff = now - WINDOW_DURATION; - while let Some(&(timestamp, _)) = self.frame_events.front() { - if timestamp < cutoff { - self.frame_events.pop_front(); - } else { - break; - } - } - } - - // Helper function to calculate current drop rate - fn calculate_drop_rate(&mut self) -> (f64, usize, usize) { - let now = Instant::now(); - self.cleanup_old_events(now); - - if self.frame_events.is_empty() { - return (0.0, 0, 0); - } - - let total_frames = self.frame_events.len(); - let dropped_frames = self - .frame_events - .iter() - .filter(|(_, dropped)| *dropped) - .count(); - let drop_rate = dropped_frames as f64 / total_frames as f64; - - (drop_rate, dropped_frames, total_frames) - } -} - -impl Message for FrameHandler { - type Reply = (); - - async fn handle( - &mut self, - msg: NewFrame, - ctx: &mut kameo::prelude::Context, - ) -> Self::Reply { - let Ok(timestamp) = msg.0.inner().SystemRelativeTime() else { - return; - }; - - let timestamp = - Timestamp::PerformanceCounter(PerformanceCounterTimestamp::new(timestamp.Duration)); - - // manual FPS limiter - if let Some(last_timestamp) = self.last_timestamp - && let Some(time_since_last) = timestamp - .duration_since(self.timestamps) - .checked_sub(last_timestamp.duration_since(self.timestamps)) - { - let target_interval = 1.0 / self.target_fps as f32; - let tolerance = target_interval * 0.8; // Allow 20% early arrival - - if time_since_last.as_secs_f32() < tolerance { - return; - } - } - - self.last_timestamp = Some(timestamp); - - let frame_dropped = match self.video_tx.try_send((msg.0, timestamp)) { - Err(flume::TrySendError::Disconnected(_)) => { - warn!("Pipeline disconnected"); - let _ = ctx.actor_ref().stop_gracefully().await; - return; - } - Err(flume::TrySendError::Full(_)) => { - warn!("Screen capture sender is full, dropping frame"); - self.frames_dropped += 1; - true - } - _ => false, - }; - - let now = Instant::now(); - - self.frame_events.push_back((now, frame_dropped)); - - if now.duration_since(self.last_cleanup) > Duration::from_millis(100) { - self.cleanup_old_events(now); - self.last_cleanup = now; - } - - // Check drop rate and potentially exit - let (drop_rate, dropped_count, total_count) = self.calculate_drop_rate(); - - if drop_rate > MAX_DROP_RATE_THRESHOLD && total_count >= 10 { - error!( - "High frame drop rate detected: {:.1}% ({}/{} frames in last {}s). Exiting capture.", - drop_rate * 100.0, - dropped_count, - total_count, - WINDOW_DURATION.as_secs() - ); - let _ = ctx.actor_ref().stop_gracefully().await; - return; - // return ControlFlow::Break(Err("Recording can't keep up with screen capture. Try reducing your display's resolution or refresh rate.".to_string())); - } - - // Periodic logging of drop rate - if now.duration_since(self.last_log) > LOG_INTERVAL && total_count > 0 { - info!( - "Frame drop rate: {:.1}% ({}/{} frames, total dropped: {})", - drop_rate * 100.0, - dropped_count, - total_count, - self.frames_dropped - ); - self.last_log = now; - } - } -} - #[derive(Clone, Debug, thiserror::Error)] enum SourceError { #[error("NoDisplay: Id '{0}'")] NoDisplay(DisplayId), #[error("AsCaptureItem: {0}")] AsCaptureItem(::windows::core::Error), - #[error("StartCapturingVideo/{0}")] - StartCapturingVideo(SendError), #[error("CreateAudioCapture/{0}")] CreateAudioCapture(scap_cpal::CapturerError), #[error("StartCapturingAudio/{0}")] @@ -225,334 +73,260 @@ enum SourceError { Closed, } -impl PipelineSourceTask for ScreenCaptureSource { - // #[instrument(skip_all)] - fn run( - &mut self, - ready_signal: crate::pipeline::task::PipelineReadySignal, - control_signal: crate::pipeline::control::PipelineControlSignal, - ) -> Result<(), String> { - let video_tx = self.video_tx.clone(); - let audio_tx = self.audio_tx.clone(); - - let d3d_device = self.d3d_device.clone(); - - // Frame drop rate tracking state - let config = self.config.clone(); - - self.tokio_handle - .block_on(async move { - let (error_tx, error_rx) = flume::bounded(1); - let capturer = - ScreenCaptureActor::spawn(ScreenCaptureActor::new(error_tx, d3d_device)); - - let frame_handler = FrameHandler::spawn(FrameHandler { - capturer: capturer.downgrade(), - video_tx, - frame_events: Default::default(), - frames_dropped: Default::default(), - last_cleanup: Instant::now(), - last_log: Instant::now(), - target_fps: config.fps, - last_timestamp: None, - timestamps: Timestamps::now(), - }); - - let mut settings = scap_direct3d::Settings { - pixel_format: Direct3DCapture::PIXEL_FORMAT, - crop: config.crop_bounds.map(|b| { - let position = b.position(); - let size = b.size().map(|v| (v / 2.0).floor() * 2.0); - - D3D11_BOX { - left: position.x() as u32, - top: position.y() as u32, - right: (position.x() + size.width()) as u32, - bottom: (position.y() + size.height()) as u32, - front: 0, - back: 1, - } - }), - ..Default::default() - }; - - if let Ok(true) = scap_direct3d::Settings::can_is_border_required() { - settings.is_border_required = Some(false); - } - - if let Ok(true) = scap_direct3d::Settings::can_is_cursor_capture_enabled() { - settings.is_cursor_capture_enabled = Some(config.show_cursor); - } +struct CapturerHandle {} - if let Ok(true) = scap_direct3d::Settings::can_min_update_interval() { - settings.min_update_interval = - Some(Duration::from_secs_f64(1.0 / config.fps as f64)); - } +pub struct VideoFrame { + pub frame: scap_direct3d::Frame, + pub timestamp: Timestamp, +} - let display = Display::from_id(&config.display) - .ok_or_else(|| SourceError::NoDisplay(config.display))?; - - let capture_item = display - .raw_handle() - .try_as_capture_item() - .map_err(SourceError::AsCaptureItem)?; - - capturer - .ask(StartCapturing { - target: capture_item, - settings, - frame_handler: frame_handler.clone().recipient(), - }) - .await - .map_err(SourceError::StartCapturingVideo)?; - - let audio_capture = if let Some(audio_tx) = audio_tx { - let audio_capture = WindowsAudioCapture::spawn( - WindowsAudioCapture::new(audio_tx) - .map_err(SourceError::CreateAudioCapture)?, - ); +impl output_pipeline::VideoFrame for VideoFrame { + fn timestamp(&self) -> Timestamp { + self.timestamp + } +} - audio_capture - .ask(audio::StartCapturing) - .await - .map_err(|v| SourceError::StartCapturingAudio(v.to_string()))?; +impl ScreenCaptureConfig { + pub async fn to_sources( + &self, + ) -> anyhow::Result<(VideoSourceConfig, Option)> { + let mut settings = scap_direct3d::Settings { + pixel_format: Direct3DCapture::PIXEL_FORMAT, + crop: self.config.crop_bounds.map(|b| { + let position = b.position(); + let size = b.size().map(|v| (v / 2.0).floor() * 2.0); + + D3D11_BOX { + left: position.x() as u32, + top: position.y() as u32, + right: (position.x() + size.width()) as u32, + bottom: (position.y() + size.height()) as u32, + front: 0, + back: 1, + } + }), + ..Default::default() + }; - Some(audio_capture) - } else { - None - }; + if let Ok(true) = scap_direct3d::Settings::can_is_border_required() { + settings.is_border_required = Some(false); + } - let _ = ready_signal.send(Ok(())); + if let Ok(true) = scap_direct3d::Settings::can_is_cursor_capture_enabled() { + settings.is_cursor_capture_enabled = Some(self.config.show_cursor); + } - let stop = async move { - let _ = capturer.ask(StopCapturing).await; - let _ = capturer.stop_gracefully().await; + if let Ok(true) = scap_direct3d::Settings::can_min_update_interval() { + settings.min_update_interval = + Some(Duration::from_secs_f64(1.0 / self.config.fps as f64)); + } - if let Some(audio_capture) = audio_capture { - let _ = audio_capture.ask(StopCapturing).await; - let _ = audio_capture.stop_gracefully().await; - } - }; - - loop { - use futures::future::Either; - - match futures::future::select( - error_rx.recv_async(), - control_signal.receiver().recv_async(), - ) - .await - { - Either::Left((Ok(_), _)) => { - error!("Screen capture closed"); - stop.await; - return Err(SourceError::Closed); - } - Either::Right((Ok(ctrl), _)) => { - if let Control::Shutdown = ctrl { - stop.await; - return Ok(()); - } - } - _ => { - warn!("Screen capture recv channels shutdown, exiting."); - - stop.await; - - return Ok(()); - } - } - } - }) - .map_err(|e| e.to_string()) + let display = Display::from_id(&self.config.display) + .ok_or_else(|| SourceError::NoDisplay(self.config.display.clone()))?; + + let capture_item = display + .raw_handle() + .try_as_capture_item() + .map_err(SourceError::AsCaptureItem)?; + + Ok(( + VideoSourceConfig { + video_info: self.video_info, + capture_item, + settings, + d3d_device: self.d3d_device.clone(), + }, + self.system_audio.then(|| SystemAudioSourceConfig), + )) } } -#[derive(Actor)] -struct ScreenCaptureActor { - stop_tx: Option>>>, - error_tx: Sender<()>, - d3d_device: ID3D11Device, +#[derive(thiserror::Error, Clone, Copy, Debug)] +pub enum VideoSourceError { + #[error("Screen capture closed")] + Closed, } -impl ScreenCaptureActor { - pub fn new(error_tx: Sender<()>, d3d_device: ID3D11Device) -> Self { - Self { - stop_tx: None, - error_tx, - d3d_device, - } - } +pub struct VideoSourceConfig { + video_info: VideoInfo, + capture_item: GraphicsCaptureItem, + settings: scap_direct3d::Settings, + pub d3d_device: ID3D11Device, } - -#[derive(Clone)] -pub struct StartCapturing { - pub target: GraphicsCaptureItem, - pub settings: scap_direct3d::Settings, - pub frame_handler: Recipient, - // error_handler: Option>, +pub struct VideoSource { + video_info: VideoInfo, + ctrl_tx: std::sync::mpsc::SyncSender, } -#[derive(Debug, Clone, thiserror::Error)] -pub enum StartCapturingError { - #[error("AlreadyCapturing")] - AlreadyCapturing, - #[error("CreateCapturer/{0}")] - CreateCapturer(scap_direct3d::NewCapturerError), - #[error("StartCapturer/{0}")] - StartCapturer(::windows::core::Error), +enum VideoControl { + Start(oneshot::Sender>), + Stop(oneshot::Sender>), } -pub struct NewFrame(pub scap_direct3d::Frame); +impl output_pipeline::VideoSource for VideoSource { + type Config = VideoSourceConfig; + type Frame = VideoFrame; -impl Message for ScreenCaptureActor { - type Reply = Result<(), StartCapturingError>; - - async fn handle( - &mut self, - msg: StartCapturing, - _: &mut Context, - ) -> Self::Reply { - if self.stop_tx.is_some() { - return Err(StartCapturingError::AlreadyCapturing); - } - - fail_err!( - "WindowsScreenCapture.StartCapturing", - StartCapturingError::CreateCapturer(scap_direct3d::NewCapturerError::NotSupported) - ); - - trace!("Starting capturer with settings: {:?}", &msg.settings); - - let error_tx = self.error_tx.clone(); - - let (ready_tx, ready_rx) = oneshot::channel(); - - let (stop_tx, stop_rx) = - std::sync::mpsc::sync_channel::>>(1); - - let d3d_device = self.d3d_device.clone(); - std::thread::spawn(move || { + async fn setup( + VideoSourceConfig { + video_info, + capture_item, + settings, + d3d_device, + }: Self::Config, + mut video_tx: mpsc::Sender, + ctx: &mut output_pipeline::SetupCtx, + ) -> anyhow::Result + where + Self: Sized, + { + let (mut error_tx, mut error_rx) = mpsc::channel(1); + let (ctrl_tx, ctrl_rx) = std::sync::mpsc::sync_channel::(1); + + ctx.tasks().spawn_thread("d3d-capture-thread", move || { cap_mediafoundation_utils::thread_init(); - let res = (|| { - let mut capture_handle = scap_direct3d::Capturer::new( - msg.target, - msg.settings, - move |frame| { - let _ = msg.frame_handler.tell(NewFrame(frame)).try_send(); + let res = scap_direct3d::Capturer::new( + capture_item, + settings, + move |frame| { + let timestamp = frame.inner().SystemRelativeTime()?; + let timestamp = Timestamp::PerformanceCounter( + PerformanceCounterTimestamp::new(timestamp.Duration), + ); + let _ = video_tx.try_send(VideoFrame { frame, timestamp }); - Ok(()) - }, + Ok(()) + }, + { + let mut error_tx = error_tx.clone(); move || { - let _ = error_tx.send(()); + let _ = error_tx.send(anyhow!("closed")); Ok(()) - }, - Some(d3d_device), - ) - .map_err(StartCapturingError::CreateCapturer)?; - - capture_handle - .start() - .map_err(StartCapturingError::StartCapturer)?; - - Ok::<_, StartCapturingError>(capture_handle) - })(); + } + }, + Some(d3d_device), + ); let mut capturer = match res { - Ok(capturer) => { - let _ = ready_tx.send(Ok(())); - capturer - } + Ok(capturer) => capturer, Err(e) => { - let _ = ready_tx.send(Err(e)); + let _ = error_tx.send(e.into()); return; } }; - let stop_channel = stop_rx.recv(); + let Ok(VideoControl::Start(reply)) = ctrl_rx.recv() else { + return; + }; - let res = capturer.stop(); + if reply.send(capturer.start().map_err(Into::into)).is_err() { + return; + } - if let Ok(stop_channel) = stop_channel { - let _ = stop_channel.send(res); + let Ok(VideoControl::Stop(reply)) = ctrl_rx.recv() else { + return; + }; + + if reply.send(capturer.stop().map_err(Into::into)).is_err() { + return; } }); - match ready_rx.await { - Ok(res) => res?, - Err(_) => { - return Err(StartCapturingError::StartCapturer( - ::windows::core::Error::new( - ::windows::core::HRESULT(0x80004005u32 as i32), - "Capturer thread dropped ready channel", - ), - )); + ctx.tasks().spawn("d3d-capture", async move { + if let Some(err) = error_rx.next().await { + return Err(anyhow!("{err}")); } - } - info!("Capturer started"); - self.stop_tx = Some(stop_tx); + Ok(()) + }); - Ok(()) + Ok(Self { + video_info, + ctrl_tx, + }) } -} -impl Message for ScreenCaptureActor { - type Reply = Result<(), String>; + fn video_info(&self) -> VideoInfo { + self.video_info + } - async fn handle( - &mut self, - _: StopCapturing, - _: &mut Context, - ) -> Self::Reply { - let Some(stop_tx) = self.stop_tx.take() else { - return Err("Not Capturing".to_string()); - }; + fn start(&mut self) -> futures::future::BoxFuture<'_, anyhow::Result<()>> { + let (tx, rx) = oneshot::channel(); + let _ = self.ctrl_tx.send(VideoControl::Start(tx)); - let (done_tx, done_rx) = oneshot::channel(); - if let Err(e) = stop_tx.send(done_tx) { - error!("Silently failed to stop Windows capturer: {}", e); + async { + rx.await??; + Ok(()) } + .boxed() + } - match done_rx.await { - Ok(res) => res.map_err(|e| e.to_string())?, - Err(_) => return Err("Capturer thread dropped stop channel".to_string()), + fn stop(&mut self) -> futures::future::BoxFuture<'_, anyhow::Result<()>> { + let (tx, rx) = oneshot::channel(); + let _ = self.ctrl_tx.send(VideoControl::Stop(tx)); + + async { + rx.await??; + Ok(()) } + .boxed() + } +} + +#[derive(Debug, Clone, thiserror::Error)] +pub enum StartCapturingError { + #[error("AlreadyCapturing")] + AlreadyCapturing, + #[error("CreateCapturer/{0}")] + CreateCapturer(scap_direct3d::NewCapturerError), + #[error("StartCapturer/{0}")] + StartCapturer(::windows::core::Error), +} - info!("stopped windows capturer"); +pub struct SystemAudioSourceConfig; - Ok(()) - } +pub struct SystemAudioSource { + capturer: scap_cpal::Capturer, } -use audio::WindowsAudioCapture; -pub mod audio { - use super::*; - use cpal::{PauseStreamError, PlayStreamError}; +impl output_pipeline::AudioSource for SystemAudioSource { + type Config = SystemAudioSourceConfig; + + fn setup( + _: Self::Config, + mut tx: mpsc::Sender, + ctx: &mut SetupCtx, + ) -> impl Future> + 'static + where + Self: Sized, + { + let (error_tx, error_rx) = oneshot::channel(); + + ctx.tasks().spawn("system-audio", async move { + if let Ok(err) = error_rx.await { + return Err(anyhow!("{err}")); + } - #[derive(Actor)] - pub struct WindowsAudioCapture { - capturer: scap_cpal::Capturer, - } + Ok(()) + }); - unsafe impl Send for WindowsAudioCapture {} + async { + let mut error_tx = Some(error_tx); - impl WindowsAudioCapture { - pub fn new( - audio_tx: Sender<(ffmpeg::frame::Audio, Timestamp)>, - ) -> Result { let capturer = scap_cpal::create_capturer( move |data, info, config| { use scap_ffmpeg::*; let timestamp = Timestamp::from_cpal(info.timestamp().capture); - let _ = audio_tx.send((data.as_ffmpeg(config), timestamp)); + let _ = tx.try_send(AudioFrame::new(data.as_ffmpeg(config), timestamp)); }, move |e| { - dbg!(e); + if let Some(error_tx) = error_tx.take() { + let _ = error_tx.send(e); + } }, )?; @@ -560,34 +334,17 @@ pub mod audio { } } - #[derive(Clone)] - pub struct StartCapturing; - - impl Message for WindowsAudioCapture { - type Reply = Result<(), PlayStreamError>; - - async fn handle( - &mut self, - _: StartCapturing, - _: &mut Context, - ) -> Self::Reply { - self.capturer.play()?; - - Ok(()) - } + fn audio_info(&self) -> cap_media_info::AudioInfo { + Direct3DCapture::audio_info() } - impl Message for WindowsAudioCapture { - type Reply = Result<(), PauseStreamError>; - - async fn handle( - &mut self, - _: StopCapturing, - _: &mut Context, - ) -> Self::Reply { - self.capturer.pause()?; + fn start(&mut self) -> impl Future> { + let res = self.capturer.play().map_err(Into::into); + async { res } + } - Ok(()) - } + fn stop(&mut self) -> impl Future> { + let res = self.capturer.pause().map_err(Into::into); + async { res } } } diff --git a/crates/recording/src/studio_recording.rs b/crates/recording/src/studio_recording.rs index f50bf4bff..eb4784e4b 100644 --- a/crates/recording/src/studio_recording.rs +++ b/crates/recording/src/studio_recording.rs @@ -1,32 +1,36 @@ use crate::{ ActorError, MediaError, RecordingBaseInputs, RecordingError, - capture_pipeline::{MakeCapturePipeline, ScreenCaptureMethod, create_screen_capture}, + capture_pipeline::{MakeCapturePipeline, ScreenCaptureMethod, Stop, create_screen_capture}, cursor::{CursorActor, Cursors, spawn_cursor_recorder}, feeds::{camera::CameraFeedLock, microphone::MicrophoneFeedLock}, - pipeline::RecordingPipeline, - sources::{AudioInputSource, CameraSource, ScreenCaptureFormat, ScreenCaptureTarget}, + ffmpeg::{Mp4Muxer, OggMuxer}, + output_pipeline::{ + AudioFrame, DoneFut, FinishedOutputPipeline, OutputPipeline, PipelineDoneError, + }, + sources::{self, screen_capture}, }; -use cap_enc_ffmpeg::{H264Encoder, MP4File, OggFile, OpusEncoder}; +use anyhow::{Context as _, anyhow}; use cap_media_info::VideoInfo; use cap_project::{CursorEvents, StudioRecordingMeta}; use cap_timestamp::{Timestamp, Timestamps}; -use cap_utils::spawn_actor; -use flume::Receiver; +use futures::{ + FutureExt, StreamExt, channel::mpsc, future::OptionFuture, stream::FuturesUnordered, +}; +use kameo::{Actor as _, prelude::*}; use relative_path::RelativePathBuf; use std::{ - collections::HashMap, path::{Path, PathBuf}, sync::Arc, time::{Duration, Instant, SystemTime, UNIX_EPOCH}, }; -use tokio::sync::oneshot; -use tracing::{debug, info, trace}; +use tokio::sync::watch; +use tracing::{Instrument, debug, error_span, info, trace}; #[allow(clippy::large_enum_variant)] enum ActorState { Recording { pipeline: Pipeline, - pipeline_done_rx: oneshot::Receiver>, + // pipeline_done_rx: oneshot::Receiver>, index: u32, segment_start_time: f64, segment_start_instant: Instant, @@ -38,102 +42,308 @@ enum ActorState { }, } -pub enum ActorControlMessage { - Pause(oneshot::Sender>), - Resume(oneshot::Sender>), - Stop(oneshot::Sender>), - Cancel(oneshot::Sender>), +#[derive(Clone)] +pub struct ActorHandle { + actor_ref: kameo::actor::ActorRef, + pub capture_target: screen_capture::ScreenCaptureTarget, + done_fut: DoneFut, + // pub bounds: Bounds, } +#[derive(kameo::Actor)] pub struct Actor { recording_dir: PathBuf, + capture_target: screen_capture::ScreenCaptureTarget, + video_info: VideoInfo, + state: Option, fps: u32, + segment_factory: SegmentPipelineFactory, segments: Vec, + completion_tx: watch::Sender>>, +} + +impl Actor { + async fn stop_pipeline( + &mut self, + pipeline: Pipeline, + segment_start_time: f64, + ) -> anyhow::Result<(Cursors, u32)> { + tracing::info!("pipeline shuting down"); + + let mut pipeline = pipeline.stop().await?; + + tracing::info!("pipeline shutdown"); + + let segment_stop_time = current_time_f64(); + + let cursors = if let Some(cursor) = pipeline.cursor.as_mut() + && let Ok(res) = cursor.actor.rx.clone().await + { + std::fs::write( + &cursor.output_path, + serde_json::to_string_pretty(&CursorEvents { + clicks: res.clicks, + moves: res.moves, + })?, + )?; + + (res.cursors, res.next_cursor_id) + } else { + (Default::default(), 0) + }; + + self.segments.push(RecordingSegment { + start: segment_start_time, + end: segment_stop_time, + pipeline, + }); + + Ok(cursors) + } + + fn notify_completion_ok(&self) { + if self.completion_tx.borrow().is_none() { + let _ = self.completion_tx.send(Some(Ok(()))); + } + } +} + +impl Message for Actor { + type Reply = anyhow::Result; + + async fn handle(&mut self, _: Stop, ctx: &mut Context) -> Self::Reply { + let cursors = match self.state.take() { + Some(ActorState::Recording { + pipeline, + segment_start_time, + segment_start_instant, + .. + }) => { + // Wait for minimum segment duration + tokio::time::sleep_until((segment_start_instant + Duration::from_secs(1)).into()) + .await; + + let (cursors, _) = self.stop_pipeline(pipeline, segment_start_time).await?; + + cursors + } + Some(ActorState::Paused { cursors, .. }) => cursors, + _ => return Err(anyhow!("Not recording")), + }; + + ctx.actor_ref().stop_gracefully().await?; + + let recording = stop_recording( + self.recording_dir.clone(), + std::mem::take(&mut self.segments), + cursors, + ) + .await?; + + self.notify_completion_ok(); + + Ok(recording) + } +} + +struct Pause; + +impl Message for Actor { + type Reply = anyhow::Result<()>; + + async fn handle(&mut self, _: Pause, _: &mut Context) -> Self::Reply { + self.state = match self.state.take() { + Some(ActorState::Recording { + pipeline, + segment_start_time, + index, + .. + }) => { + let (cursors, next_cursor_id) = + self.stop_pipeline(pipeline, segment_start_time).await?; + + Some(ActorState::Paused { + next_index: index + 1, + cursors, + next_cursor_id, + }) + } + state => state, + }; + + Ok(()) + } +} + +struct Resume; + +impl Message for Actor { + type Reply = anyhow::Result<()>; + + async fn handle(&mut self, _: Resume, _: &mut Context) -> Self::Reply { + self.state = match self.state.take() { + Some(ActorState::Paused { + next_index, + cursors, + next_cursor_id, + }) => { + let pipeline = self + .segment_factory + .create_next(cursors, next_cursor_id) + .await?; + + Some(ActorState::Recording { + pipeline, + // pipeline_done_rx, + index: next_index, + segment_start_time: current_time_f64(), + segment_start_instant: Instant::now(), + }) + } + state => state, + }; + + Ok(()) + } +} + +struct Cancel; + +impl Message for Actor { + type Reply = anyhow::Result<()>; + + async fn handle(&mut self, _: Cancel, _: &mut Context) -> Self::Reply { + if let Some(ActorState::Recording { pipeline, .. }) = self.state.take() { + let _ = pipeline.stop().await; + + self.notify_completion_ok(); + } + + Ok(()) + } } pub struct RecordingSegment { pub start: f64, pub end: f64, - pipeline: Pipeline, -} - -pub struct PipelineOutput { - pub path: PathBuf, - pub first_timestamp_rx: flume::Receiver, + pipeline: FinishedPipeline, } pub struct ScreenPipelineOutput { - pub inner: PipelineOutput, + pub inner: OutputPipeline, pub video_info: VideoInfo, } struct Pipeline { pub start_time: Timestamps, - pub inner: RecordingPipeline, - pub screen: ScreenPipelineOutput, - pub microphone: Option, - pub camera: Option, + // sources + pub screen: OutputPipeline, + pub microphone: Option, + pub camera: Option, + pub system_audio: Option, pub cursor: Option, - pub system_audio: Option, } -struct CursorPipeline { - output_path: PathBuf, - actor: Option, +struct FinishedPipeline { + pub start_time: Timestamps, + // sources + pub screen: FinishedOutputPipeline, + pub microphone: Option, + pub camera: Option, + pub system_audio: Option, + pub cursor: Option, } -#[derive(Clone)] -pub struct ActorHandle { - ctrl_tx: flume::Sender, - pub capture_target: ScreenCaptureTarget, +impl Pipeline { + pub async fn stop(mut self) -> anyhow::Result { + let (screen, microphone, camera, system_audio) = futures::join!( + self.screen.stop(), + OptionFuture::from(self.microphone.map(|s| s.stop())), + OptionFuture::from(self.camera.map(|s| s.stop())), + OptionFuture::from(self.system_audio.map(|s| s.stop())) + ); + + if let Some(cursor) = self.cursor.as_mut() { + cursor.actor.stop(); + } + + Ok(FinishedPipeline { + start_time: self.start_time, + screen: screen?, + microphone: microphone.transpose()?, + camera: camera.transpose()?, + system_audio: system_audio.transpose()?, + cursor: self.cursor, + }) + } + + fn spawn_watcher(&self, completion_tx: watch::Sender>>) { + let mut futures = FuturesUnordered::new(); + futures.push(self.screen.done_fut()); + + if let Some(ref microphone) = self.microphone { + futures.push(microphone.done_fut()); + } + + if let Some(ref camera) = self.camera { + futures.push(camera.done_fut()); + } + + if let Some(ref system_audio) = self.system_audio { + futures.push(system_audio.done_fut()); + } + + tokio::spawn(async move { + while let Some(res) = futures.next().await { + if let Err(err) = res { + if completion_tx.borrow().is_none() { + let _ = completion_tx.send(Some(Err(err))); + } + } + } + }); + } } -macro_rules! send_message { - ($ctrl_tx:expr, $variant:path) => {{ - let (tx, rx) = oneshot::channel(); - $ctrl_tx - .send($variant(tx)) - .map_err(|_| flume::SendError(())) - .map_err(ActorError::from)?; - rx.await.map_err(|_| ActorError::ActorStopped)? - }}; +struct CursorPipeline { + output_path: PathBuf, + actor: CursorActor, } impl ActorHandle { - pub async fn stop(&self) -> Result { - send_message!(self.ctrl_tx, ActorControlMessage::Stop) + pub async fn stop(&self) -> anyhow::Result { + Ok(self.actor_ref.ask(Stop).await?) } - pub async fn pause(&self) -> Result<(), RecordingError> { - send_message!(self.ctrl_tx, ActorControlMessage::Pause) + pub fn done_fut(&self) -> DoneFut { + self.done_fut.clone() } - pub async fn resume(&self) -> Result<(), CreateSegmentPipelineError> { - send_message!(self.ctrl_tx, ActorControlMessage::Resume) + pub async fn pause(&self) -> anyhow::Result<()> { + Ok(self.actor_ref.ask(Pause).await?) } - pub async fn cancel(&self) -> Result<(), RecordingError> { - send_message!(self.ctrl_tx, ActorControlMessage::Cancel) + pub async fn resume(&self) -> anyhow::Result<()> { + Ok(self.actor_ref.ask(Resume).await?) } -} -#[derive(Debug, thiserror::Error)] -pub enum SpawnError { - #[error("{0}")] - Media(#[from] MediaError), - #[error("{0}")] - PipelineCreationError(#[from] CreateSegmentPipelineError), + pub async fn cancel(&self) -> anyhow::Result<()> { + Ok(self.actor_ref.ask(Cancel).await?) + } } impl Actor { - pub fn builder(output: PathBuf, capture_target: ScreenCaptureTarget) -> ActorBuilder { + pub fn builder( + output: PathBuf, + capture_target: screen_capture::ScreenCaptureTarget, + ) -> ActorBuilder { ActorBuilder::new(output, capture_target) } } pub struct ActorBuilder { output_path: PathBuf, - capture_target: ScreenCaptureTarget, + capture_target: screen_capture::ScreenCaptureTarget, system_audio: bool, mic_feed: Option>, camera_feed: Option>, @@ -141,7 +351,7 @@ pub struct ActorBuilder { } impl ActorBuilder { - pub fn new(output: PathBuf, capture_target: ScreenCaptureTarget) -> Self { + pub fn new(output: PathBuf, capture_target: screen_capture::ScreenCaptureTarget) -> Self { Self { output_path: output, capture_target, @@ -175,7 +385,7 @@ impl ActorBuilder { pub async fn build( self, #[cfg(target_os = "macos")] shareable_content: cidre::arc::R, - ) -> Result<(ActorHandle, oneshot::Receiver>), SpawnError> { + ) -> anyhow::Result { spawn_studio_recording_actor( self.output_path, RecordingBaseInputs { @@ -192,15 +402,14 @@ impl ActorBuilder { } } +#[tracing::instrument("studio_recording", skip_all)] async fn spawn_studio_recording_actor( recording_dir: PathBuf, base_inputs: RecordingBaseInputs, custom_cursor_capture: bool, -) -> Result<(ActorHandle, oneshot::Receiver>), SpawnError> { +) -> anyhow::Result { ensure_dir(&recording_dir)?; - let (done_tx, done_rx) = oneshot::channel(); - trace!("creating recording actor"); let content_dir = ensure_dir(&recording_dir.join("content"))?; @@ -210,6 +419,9 @@ async fn spawn_studio_recording_actor( let start_time = Timestamps::now(); + let (completion_tx, completion_rx) = + watch::channel::>>(None); + if let Some(camera_feed) = &base_inputs.camera_feed { debug!("camera device info: {:#?}", camera_feed.camera_info()); debug!("camera video info: {:#?}", camera_feed.video_info()); @@ -225,371 +437,102 @@ async fn spawn_studio_recording_actor( base_inputs.clone(), custom_cursor_capture, start_time, + completion_tx.clone(), ); let index = 0; - let (pipeline, pipeline_done_rx) = segment_pipeline_factory + let pipeline = segment_pipeline_factory .create_next(Default::default(), 0) .await?; - let segment_start_time = current_time_f64(); + let done_fut = completion_rx_to_done_fut(completion_rx); - let (ctrl_tx, ctrl_rx) = flume::bounded(1); + let segment_start_time = current_time_f64(); trace!("spawning recording actor"); let base_inputs = base_inputs.clone(); - let fps = pipeline.screen.video_info.fps(); - - spawn_actor(async move { - let mut actor = Actor { - recording_dir, - fps, - segments: Vec::new(), - }; - - let mut state = ActorState::Recording { + let fps = pipeline.screen.video_info().unwrap().fps(); + + let actor_ref = Actor::spawn(Actor { + recording_dir, + fps, + capture_target: base_inputs.capture_target.clone(), + video_info: pipeline.screen.video_info().unwrap(), + state: Some(ActorState::Recording { pipeline, - pipeline_done_rx, + /*pipeline_done_rx,*/ index, segment_start_time, segment_start_instant: Instant::now(), - }; - - let result = loop { - match run_actor_iteration(state, &ctrl_rx, actor, &mut segment_pipeline_factory).await { - Ok(None) => break Ok(()), - Ok(Some((new_state, new_actor))) => { - state = new_state; - actor = new_actor; - } - Err(err) => break Err(err), - } - }; - - info!("recording actor finished: {:?}", &result); - - let _ = done_tx.send(result.map_err(|v| v.to_string())); + }), + segment_factory: segment_pipeline_factory, + segments: Vec::new(), + completion_tx: completion_tx.clone(), }); - Ok(( - ActorHandle { - ctrl_tx, - capture_target: base_inputs.capture_target, - }, - done_rx, - )) -} - -#[derive(thiserror::Error, Debug)] -enum StudioRecordingActorError { - #[error("Pipeline receiver dropped")] - PipelineReceiverDropped, - #[error("Control receiver dropped")] - ControlReceiverDropped, - #[error("{0}")] - Other(String), -} - -// Helper macro for sending responses -macro_rules! send_response { - ($tx:expr, $res:expr) => { - let _ = $tx.send($res); - }; -} - -async fn run_actor_iteration( - state: ActorState, - ctrl_rx: &Receiver, - mut actor: Actor, - segment_pipeline_factory: &mut SegmentPipelineFactory, -) -> Result, StudioRecordingActorError> { - use ActorControlMessage as Msg; - use ActorState as State; - - // Helper function to shutdown pipeline and save cursor data - async fn shutdown( - mut pipeline: Pipeline, - actor: &mut Actor, - segment_start_time: f64, - ) -> Result<(Cursors, u32), RecordingError> { - tracing::info!("pipeline shuting down"); - - pipeline.inner.shutdown().await?; - - tracing::info!("pipeline shutdown"); - - let segment_stop_time = current_time_f64(); - - let cursors = if let Some(cursor) = &mut pipeline.cursor { - if let Some(actor) = cursor.actor.take() { - let res = actor.stop().await; - - std::fs::write( - &cursor.output_path, - serde_json::to_string_pretty(&CursorEvents { - clicks: res.clicks, - moves: res.moves, - })?, - )?; - - (res.cursors, res.next_cursor_id) - } else { - (Default::default(), 0) - } - } else { - (Default::default(), 0) - }; - - actor.segments.push(RecordingSegment { - start: segment_start_time, - end: segment_stop_time, - pipeline, - }); - - Ok(cursors) - } - - // Log current state - info!( - "recording actor state: {:?}", - match &state { - State::Recording { .. } => "recording", - State::Paused { .. } => "paused", - } - ); - - // Receive event based on current state - let event = match state { - State::Recording { - mut pipeline_done_rx, - mut pipeline, - index, - segment_start_time, - segment_start_instant, - } => { - tokio::select! { - result = &mut pipeline_done_rx => { - let res = match result { - Ok(Ok(())) => Ok(None), - Ok(Err(e)) => Err(StudioRecordingActorError::Other(e)), - Err(_) => Err(StudioRecordingActorError::PipelineReceiverDropped), - }; - - if let Some(cursor) = &mut pipeline.cursor - && let Some(actor) = cursor.actor.take() { - actor.stop().await; - } - - return res; - }, - msg = ctrl_rx.recv_async() => { - match msg { - Ok(msg) => ( - msg, - State::Recording { - pipeline, - pipeline_done_rx, - index, - segment_start_time, - segment_start_instant, - }, - ), - Err(_) => { - if let Some(cursor) = &mut pipeline.cursor - && let Some(actor) = cursor.actor.take() { - actor.stop().await; - } - - return Err(StudioRecordingActorError::ControlReceiverDropped) - }, - } - } - } - } - paused_state @ State::Paused { .. } => match ctrl_rx.recv_async().await { - Ok(msg) => (msg, paused_state), - Err(_) => return Err(StudioRecordingActorError::ControlReceiverDropped), - }, - }; - - let (event, state) = event; - - // Handle state transitions based on event and current state - Ok(match (event, state) { - // Pause from Recording - ( - Msg::Pause(tx), - State::Recording { - pipeline, - index, - segment_start_time, - .. - }, - ) => { - let (res, cursors, next_cursor_id) = - match shutdown(pipeline, &mut actor, segment_start_time).await { - Ok((cursors, next_cursor_id)) => (Ok(()), cursors, next_cursor_id), - Err(e) => (Err(e), HashMap::new(), 0), - }; - - send_response!(tx, res); - - Some(( - State::Paused { - next_index: index + 1, - cursors, - next_cursor_id, - }, - actor, - )) - } - - // Stop from any state - (Msg::Stop(tx), state) => { - let result = match state { - State::Recording { - pipeline, - segment_start_time, - segment_start_instant, - .. - } => { - // Wait for minimum segment duration - tokio::time::sleep_until( - (segment_start_instant + Duration::from_secs(1)).into(), - ) - .await; - - match shutdown(pipeline, &mut actor, segment_start_time).await { - Ok((cursors, _)) => stop_recording(actor, cursors).await, - Err(e) => Err(e), - } - } - State::Paused { cursors, .. } => stop_recording(actor, cursors).await, - }; - - println!("recording successfully stopped"); - - send_response!(tx, result); - None - } - - // Resume from Paused - ( - Msg::Resume(tx), - State::Paused { - next_index, - cursors, - next_cursor_id, - }, - ) => { - match segment_pipeline_factory - .create_next(cursors, next_cursor_id) - .await - { - Ok((pipeline, pipeline_done_rx)) => { - send_response!(tx, Ok(())); - Some(( - State::Recording { - pipeline, - pipeline_done_rx, - index: next_index, - segment_start_time: current_time_f64(), - segment_start_instant: Instant::now(), - }, - actor, - )) - } - Err(e) => { - send_response!(tx, Err(e)); - None - } - } - } - - // Cancel from any state - (Msg::Cancel(tx), state) => { - let result = match state { - State::Recording { mut pipeline, .. } => { - if let Some(cursor) = &mut pipeline.cursor - && let Some(actor) = cursor.actor.take() - { - actor.stop().await; - } - - pipeline.inner.shutdown().await - } - State::Paused { .. } => Ok(()), - }; - - send_response!(tx, result.map_err(Into::into)); - None - } - - (_, state) => Some((state, actor)), + Ok(ActorHandle { + actor_ref, + capture_target: base_inputs.capture_target, + done_fut, }) } -pub struct CompletedStudioRecording { +pub struct CompletedRecording { pub project_path: PathBuf, pub meta: StudioRecordingMeta, pub cursor_data: cap_project::CursorImages, - pub segments: Vec, } async fn stop_recording( - actor: Actor, + recording_dir: PathBuf, + segments: Vec, cursors: Cursors, -) -> Result { +) -> Result { use cap_project::*; let make_relative = |path: &PathBuf| { - RelativePathBuf::from_path(path.strip_prefix(&actor.recording_dir).unwrap()).unwrap() + RelativePathBuf::from_path(path.strip_prefix(&recording_dir).unwrap()).unwrap() }; let meta = StudioRecordingMeta::MultipleSegments { inner: MultipleSegments { - segments: { - actor - .segments - .iter() - .map(|s| { - let recv_timestamp = |pipeline: &PipelineOutput| { - pipeline - .first_timestamp_rx - .try_recv() - .ok() - .map(|v| v.duration_since(s.pipeline.start_time).as_secs_f64()) - }; - - MultipleSegment { - display: VideoMeta { - path: make_relative(&s.pipeline.screen.inner.path), - fps: actor.fps, - start_time: recv_timestamp(&s.pipeline.screen.inner), - }, - camera: s.pipeline.camera.as_ref().map(|camera| VideoMeta { - path: make_relative(&camera.inner.path), - fps: camera.fps, - start_time: recv_timestamp(&camera.inner), - }), - mic: s.pipeline.microphone.as_ref().map(|mic| AudioMeta { - path: make_relative(&mic.path), - start_time: recv_timestamp(mic), - }), - cursor: s - .pipeline - .cursor - .as_ref() - .map(|cursor| make_relative(&cursor.output_path)), - system_audio: s.pipeline.system_audio.as_ref().map(|audio| AudioMeta { - path: make_relative(&audio.path), - start_time: recv_timestamp(audio), - }), - } - }) - .collect() - }, + segments: futures::stream::iter(segments) + .then(async |s| { + let to_start_time = |timestamp: Timestamp| { + timestamp + .duration_since(s.pipeline.start_time) + .as_secs_f64() + }; + + MultipleSegment { + display: VideoMeta { + path: make_relative(&s.pipeline.screen.path), + fps: s.pipeline.screen.video_info.unwrap().fps(), + start_time: Some(to_start_time(s.pipeline.screen.first_timestamp)), + }, + camera: s.pipeline.camera.map(|camera| VideoMeta { + path: make_relative(&camera.path), + fps: camera.video_info.unwrap().fps(), + start_time: Some(to_start_time(camera.first_timestamp)), + }), + mic: s.pipeline.microphone.map(|mic| AudioMeta { + path: make_relative(&mic.path), + start_time: Some(to_start_time(mic.first_timestamp)), + }), + system_audio: s.pipeline.system_audio.map(|audio| AudioMeta { + path: make_relative(&audio.path), + start_time: Some(to_start_time(audio.first_timestamp)), + }), + cursor: s + .pipeline + .cursor + .as_ref() + .map(|cursor| make_relative(&cursor.output_path)), + } + }) + .collect::>() + .await, cursors: cap_project::Cursors::Correct( cursors .into_values() @@ -611,15 +554,15 @@ async fn stop_recording( let project_config = cap_project::ProjectConfiguration::default(); project_config - .write(&actor.recording_dir) + .write(&recording_dir) .map_err(RecordingError::from)?; - Ok(CompletedStudioRecording { - project_path: actor.recording_dir.clone(), + Ok(CompletedRecording { + project_path: recording_dir, meta, cursor_data: Default::default(), // display_source: actor.options.capture_target, - segments: actor.segments, + // segments: actor.segments, }) } @@ -630,6 +573,7 @@ struct SegmentPipelineFactory { custom_cursor_capture: bool, start_time: Timestamps, index: u32, + completion_tx: watch::Sender>>, } impl SegmentPipelineFactory { @@ -640,6 +584,7 @@ impl SegmentPipelineFactory { base_inputs: RecordingBaseInputs, custom_cursor_capture: bool, start_time: Timestamps, + completion_tx: watch::Sender>>, ) -> Self { Self { segments_dir, @@ -648,6 +593,7 @@ impl SegmentPipelineFactory { custom_cursor_capture, start_time, index: 0, + completion_tx, } } @@ -655,8 +601,8 @@ impl SegmentPipelineFactory { &mut self, cursors: Cursors, next_cursors_id: u32, - ) -> Result<(Pipeline, oneshot::Receiver>), CreateSegmentPipelineError> { - let result = create_segment_pipeline( + ) -> anyhow::Result { + let pipeline = create_segment_pipeline( &self.segments_dir, &self.cursors_dir, self.index, @@ -670,10 +616,30 @@ impl SegmentPipelineFactory { self.index += 1; - Ok(result) + pipeline.spawn_watcher(self.completion_tx.clone()); + + Ok(pipeline) } } +fn completion_rx_to_done_fut( + mut rx: watch::Receiver>>, +) -> DoneFut { + async move { + loop { + if let Some(result) = rx.borrow().clone() { + return result; + } + + if rx.changed().await.is_err() { + return Ok(()); + } + } + } + .boxed() + .shared() +} + #[derive(Debug, thiserror::Error)] pub enum CreateSegmentPipelineError { #[error("NoDisplay")] @@ -703,14 +669,7 @@ async fn create_segment_pipeline( next_cursors_id: u32, custom_cursor_capture: bool, start_time: Timestamps, -) -> Result<(Pipeline, oneshot::Receiver>), CreateSegmentPipelineError> { - let system_audio = if base_inputs.capture_system_audio { - let (tx, rx) = flume::bounded(64); - (Some(tx), Some(rx)) - } else { - (None, None) - }; - +) -> anyhow::Result { let display = base_inputs .capture_target .display() @@ -723,12 +682,12 @@ async fn create_segment_pipeline( #[cfg(windows)] let d3d_device = crate::capture_pipeline::create_d3d_device().unwrap(); - let (screen_source, screen_rx) = create_screen_capture( + let screen_config = create_screen_capture( &base_inputs.capture_target, !custom_cursor_capture, 120, - system_audio.0, start_time.system_time(), + base_inputs.capture_system_audio, #[cfg(windows)] d3d_device, #[cfg(target_os = "macos")] @@ -737,218 +696,55 @@ async fn create_segment_pipeline( .await .unwrap(); - let dir = ensure_dir(&segments_dir.join(format!("segment-{index}")))?; + let (capture_source, system_audio) = screen_config.to_sources().await?; - let mut pipeline_builder = RecordingPipeline::builder(); + let dir = ensure_dir(&segments_dir.join(format!("segment-{index}")))?; let screen_output_path = dir.join("display.mp4"); trace!("preparing segment pipeline {index}"); - let screen = { - let video_info = screen_source.info(); - - let (pipeline_builder_, screen_timestamp_rx) = - ScreenCaptureMethod::make_studio_mode_pipeline( - pipeline_builder, - (screen_source, screen_rx), - screen_output_path.clone(), - start_time, - ) - .unwrap(); - pipeline_builder = pipeline_builder_; - - info!( - r#"screen pipeline prepared, will output to "{}""#, - screen_output_path - .strip_prefix(segments_dir) - .unwrap() - .display() - ); - - ScreenPipelineOutput { - inner: PipelineOutput { - path: screen_output_path, - first_timestamp_rx: screen_timestamp_rx, - }, - video_info, - } - }; - - let microphone = if let Some(mic_feed) = base_inputs.mic_feed { - let (tx, channel) = flume::bounded(8); - - let mic_source = AudioInputSource::init(mic_feed, tx); - - let mic_config = mic_source.info(); - let output_path = dir.join("audio-input.ogg"); - - let mut output = OggFile::init( - output_path.clone(), - OpusEncoder::factory("microphone", mic_config), - ) - .map_err(|e| MediaError::Any(e.to_string().into()))?; - let time_base = output.encoder().input_time_base(); - - pipeline_builder.spawn_source("microphone_capture", mic_source); - - let (timestamp_tx, first_timestamp_rx) = flume::bounded(1); - - pipeline_builder.spawn_task("microphone_encoder", move |ready| { - let mut first_timestamp = None; - let mut timestamp_tx = Some(timestamp_tx); - let _ = ready.send(Ok(())); - - let rate = time_base.denominator() as f64 / time_base.numerator() as f64; - - while let Ok((mut frame, timestamp)) = channel.recv() { - if let Some(timestamp_tx) = timestamp_tx.take() { - let _ = timestamp_tx.send(timestamp); - } - - let first_timestamp = first_timestamp.get_or_insert(timestamp); - - let elapsed = timestamp.duration_since(start_time) - - first_timestamp.duration_since(start_time); - frame.set_pts(Some((elapsed.as_secs_f64() * rate) as i64)); - - output.queue_frame(frame); - } - - output.finish(); - Ok(()) - }); - - info!( - "mic pipeline prepared, will output to {}", - output_path.strip_prefix(segments_dir).unwrap().display() - ); - - Some(PipelineOutput { - path: output_path, - first_timestamp_rx, - }) - } else { - None - }; - - let system_audio = if let Some((config, channel)) = - Some(ScreenCaptureMethod::audio_info()).zip(system_audio.1.clone()) - { - let output_path = dir.join("system_audio.ogg"); - - let mut output = OggFile::init( - output_path.clone(), - OpusEncoder::factory("system_audio", config), - ) - .map_err(|e| MediaError::Any(e.to_string().into()))?; - - let time_base = output.encoder().input_time_base(); - - let (timestamp_tx, timestamp_rx) = flume::bounded(1); - - pipeline_builder.spawn_task("system_audio_encoder", move |ready| { - let mut first_timestamp = None; - let mut timestamp_tx = Some(timestamp_tx); - let _ = ready.send(Ok(())); - - let rate = time_base.denominator() as f64 / time_base.numerator() as f64; - - while let Ok((mut frame, timestamp)) = channel.recv() { - if let Some(timestamp_tx) = timestamp_tx.take() { - let _ = timestamp_tx.send(timestamp); - } - - let first_timestamp = first_timestamp.get_or_insert(timestamp); - - let elapsed = timestamp.duration_since(start_time) - - first_timestamp.duration_since(start_time); - frame.set_pts(Some((elapsed.as_secs_f64() * rate).round() as i64)); - - output.queue_frame(frame); - } - - output.finish(); - Ok(()) - }); - - Some(PipelineOutput { - path: output_path, - first_timestamp_rx: timestamp_rx, - }) - } else { - None - }; - - let camera = if let Some(camera_feed) = base_inputs.camera_feed { - let (tx, channel) = flume::bounded(8); - - let camera_source = CameraSource::init(camera_feed, tx); - let camera_config = camera_source.info(); - let time_base = camera_config.time_base; - let output_path = dir.join("camera.mp4"); - - let mut camera_encoder = MP4File::init( - "camera", - output_path.clone(), - |o| H264Encoder::builder("camera", camera_config).build(o), - |_| None, - ) - .map_err(|e| MediaError::Any(e.to_string().into()))?; - - pipeline_builder.spawn_source("camera_capture", camera_source); - - let (timestamp_tx, timestamp_rx) = flume::bounded(1); - - pipeline_builder.spawn_task("camera_encoder", move |ready| { - let mut first_timestamp = None; - let mut timestamp_tx = Some(timestamp_tx); - let _ = ready.send(Ok(())); - - let rate = time_base.denominator() as f64 / time_base.numerator() as f64; - - while let Ok((mut frame, timestamp)) = channel.recv() { - if let Some(timestamp_tx) = timestamp_tx.take() { - let _ = timestamp_tx.send(timestamp); - } - - let first_timestamp = first_timestamp.get_or_insert(timestamp); - - let elapsed = timestamp.duration_since(start_time) - - first_timestamp.duration_since(start_time); - frame.set_pts(Some((elapsed.as_secs_f64() * rate) as i64)); - - camera_encoder.queue_video_frame(frame); - } - camera_encoder.finish(); - Ok(()) - }); - - info!( - "camera pipeline prepared, will output to {}", - output_path.strip_prefix(segments_dir).unwrap().display() - ); - - Some(CameraPipelineInfo { - inner: PipelineOutput { - path: output_path, - first_timestamp_rx: timestamp_rx, - }, - fps: (camera_config.frame_rate.0 / camera_config.frame_rate.1) as u32, - }) - } else { - None - }; - - let (mut pipeline, pipeline_done_rx) = pipeline_builder - .build() - .await - .map_err(CreateSegmentPipelineError::PipelineBuild)?; - - pipeline - .play() - .await - .map_err(CreateSegmentPipelineError::PipelinePlay)?; + let screen = ScreenCaptureMethod::make_studio_mode_pipeline( + capture_source, + screen_output_path.clone(), + start_time, + ) + .instrument(error_span!("screen-out")) + .await + .context("screen pipeline setup")?; + + let camera = OptionFuture::from(base_inputs.camera_feed.map(|camera_feed| { + OutputPipeline::builder(dir.join("camera.mp4")) + .with_video::(camera_feed) + .with_timestamps(start_time) + .build::(()) + .instrument(error_span!("camera-out")) + })) + .await + .transpose() + .context("camera pipeline setup")?; + + let microphone = OptionFuture::from(base_inputs.mic_feed.map(|mic_feed| { + OutputPipeline::builder(dir.join("audio-input.ogg")) + .with_audio_source::(mic_feed) + .with_timestamps(start_time) + .build::(()) + .instrument(error_span!("mic-out")) + })) + .await + .transpose() + .context("microphone pipeline setup")?; + + let system_audio = OptionFuture::from(system_audio.map(|system_audio| { + OutputPipeline::builder(dir.join("system_audio.ogg")) + .with_audio_source::(system_audio) + .with_timestamps(start_time) + .build::(()) + .instrument(error_span!("system-audio-out")) + })) + .await + .transpose() + .context("microphone pipeline setup")?; let cursor = custom_cursor_capture.then(move || { let cursor = spawn_cursor_recorder( @@ -962,28 +758,24 @@ async fn create_segment_pipeline( CursorPipeline { output_path: dir.join("cursor.json"), - actor: Some(cursor), + actor: cursor, } }); info!("pipeline playing"); - Ok(( - Pipeline { - inner: pipeline, - start_time, - screen, - microphone, - camera, - cursor, - system_audio, - }, - pipeline_done_rx, - )) + Ok(Pipeline { + start_time, + screen, + microphone, + camera, + cursor, + system_audio, + }) } struct CameraPipelineInfo { - inner: PipelineOutput, + inner: OutputPipeline, fps: u32, } diff --git a/crates/scap-cpal/src/lib.rs b/crates/scap-cpal/src/lib.rs index 1c8507991..9eee0832e 100644 --- a/crates/scap-cpal/src/lib.rs +++ b/crates/scap-cpal/src/lib.rs @@ -51,6 +51,8 @@ pub fn create_capturer( }) } +unsafe impl Send for Capturer {} + pub struct Capturer { stream: Stream, config: StreamConfig, diff --git a/crates/timestamp/src/win.rs b/crates/timestamp/src/win.rs index 119616e0e..ee2edc6dd 100644 --- a/crates/timestamp/src/win.rs +++ b/crates/timestamp/src/win.rs @@ -28,8 +28,22 @@ impl PerformanceCounterTimestamp { } pub fn duration_since(&self, other: Self) -> Duration { - let freq = perf_freq(); - Duration::from_secs_f64((self.0 - other.0) as f64 / freq as f64) + let freq = perf_freq() as i128; + debug_assert!(freq > 0); + + let diff = self.0 as i128 - other.0 as i128; + + if diff <= 0 { + Duration::ZERO + } else { + let diff = diff as u128; + let freq = freq as u128; + + let secs = diff / freq; + let nanos = ((diff % freq) * 1_000_000_000u128) / freq; + + Duration::new(secs as u64, nanos as u32) + } } pub fn now() -> Self { @@ -62,3 +76,26 @@ impl Sub for PerformanceCounterTimestamp { Self(self.0 - (rhs.as_secs_f64() * freq as f64) as i64) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn duration_since_returns_zero_when_earlier() { + let freq = perf_freq(); + let base = PerformanceCounterTimestamp::new(10 * freq); + let earlier = PerformanceCounterTimestamp::new(9 * freq); + + assert_eq!(earlier.duration_since(base), Duration::ZERO); + } + + #[test] + fn duration_since_handles_positive_diff() { + let freq = perf_freq(); + let base = PerformanceCounterTimestamp::new(10 * freq); + let later = PerformanceCounterTimestamp::new(11 * freq); + + assert_eq!(later.duration_since(base), Duration::from_secs(1)); + } +}