Skip to content

Commit

Permalink
feat: add safari camera encode support
Browse files Browse the repository at this point in the history
  • Loading branch information
@alcolmenar
alcolmenar committed Jul 28, 2023
1 parent 8a5b37b commit c09fba3
Show file tree
Hide file tree
Showing 3 changed files with 67 additions and 46 deletions.
1 change: 1 addition & 0 deletions yew-ui/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ features = [
"VideoEncoderConfig",
"VideoEncoderEncodeOptions",
"VideoFrame",
"VideoFrameInit",
"VideoTrack",
"VideoDecoder",
"VideoDecoderConfig",
Expand Down
1 change: 1 addition & 0 deletions yew-ui/src/constants.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ pub const AUDIO_BITRATE: f64 = 50000f64;

pub const VIDEO_HEIGHT: i32 = 720i32;
pub const VIDEO_WIDTH: i32 = 1280i32;
pub const VIDEO_FPS: f64 = 30.0;
pub const ACTIX_WEBSOCKET: &str = concat!(std::env!("ACTIX_UI_BACKEND_URL"), "/lobby");
pub const WEBTRANSPORT_HOST: &str = concat!(std::env!("WEBTRANSPORT_HOST"), "/lobby");

Expand Down
111 changes: 65 additions & 46 deletions yew-ui/src/model/encode/camera_encoder.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,33 +2,32 @@ use gloo_console::log;
use gloo_utils::window;
use js_sys::Array;
use js_sys::Boolean;
use js_sys::JsString;
use js_sys::Reflect;
use std::sync::atomic::Ordering;
use wasm_bindgen::prelude::Closure;
use wasm_bindgen::JsCast;
use wasm_bindgen::JsValue;
use wasm_bindgen_futures::JsFuture;
use web_sys::CanvasRenderingContext2d;
use web_sys::HtmlCanvasElement;
use web_sys::HtmlVideoElement;
use web_sys::LatencyMode;
use web_sys::MediaStream;
use web_sys::MediaStreamConstraints;
use web_sys::MediaStreamTrack;
use web_sys::MediaStreamTrackProcessor;
use web_sys::MediaStreamTrackProcessorInit;
use web_sys::ReadableStreamDefaultReader;
use web_sys::VideoEncoder;
use web_sys::VideoEncoderConfig;
use web_sys::VideoEncoderEncodeOptions;
use web_sys::VideoEncoderInit;
use web_sys::VideoFrame;
use web_sys::VideoFrameInit;
use web_sys::VideoTrack;

use super::encoder_state::EncoderState;
use super::transform::transform_video_chunk;
use types::protos::media_packet::MediaPacket;

use crate::constants::VIDEO_CODEC;
use crate::constants::VIDEO_FPS;
use crate::constants::VIDEO_HEIGHT;
use crate::constants::VIDEO_WIDTH;

Expand Down Expand Up @@ -99,6 +98,18 @@ impl CameraEncoder {
.unwrap()
.unchecked_into::<HtmlVideoElement>();

let canvas_element = window()
.document()
.unwrap()
.create_element(&"canvas")
.unwrap()
.unchecked_into::<HtmlCanvasElement>();
let ctx = canvas_element
.get_context("2d")
.unwrap()
.unwrap()
.unchecked_into::<CanvasRenderingContext2d>();

let media_devices = navigator.media_devices().unwrap();
let mut constraints = MediaStreamConstraints::new();
let mut media_info = web_sys::MediaTrackConstraints::new();
Expand All @@ -116,6 +127,7 @@ impl CameraEncoder {
.unchecked_into::<MediaStream>();
video_element.set_src_object(Some(&device));
video_element.set_muted(true);
video_element.play();

let video_track = Box::new(
device
Expand All @@ -138,6 +150,8 @@ impl CameraEncoder {
video_output_handler.as_ref().unchecked_ref(),
);

video_output_handler.forget();
video_error_handler.forget();
let video_encoder = Box::new(VideoEncoder::new(&video_encoder_init).unwrap());

let video_settings = &mut video_track
Expand All @@ -154,50 +168,55 @@ impl CameraEncoder {
video_encoder_config.latency_mode(LatencyMode::Realtime);
video_encoder.configure(&video_encoder_config);

let video_processor =
MediaStreamTrackProcessor::new(&MediaStreamTrackProcessorInit::new(
&video_track.clone().unchecked_into::<MediaStreamTrack>(),
))
.unwrap();
let video_reader = video_processor
.readable()
.get_reader()
.unchecked_into::<ReadableStreamDefaultReader>();

// Start encoding video and audio.
let mut video_frame_counter = 0;
let poll_video = async {
loop {
if !enabled.load(Ordering::Acquire)
|| destroy.load(Ordering::Acquire)
|| switching.load(Ordering::Acquire)
{
video_track
.clone()
.unchecked_into::<MediaStreamTrack>()
.stop();
video_encoder.close();
switching.store(false, Ordering::Release);
return;
}
match JsFuture::from(video_reader.read()).await {
Ok(js_frame) => {
let video_frame = Reflect::get(&js_frame, &JsString::from("value"))
.unwrap()
.unchecked_into::<VideoFrame>();
let mut opts = VideoEncoderEncodeOptions::new();
video_frame_counter = (video_frame_counter + 1) % 50;
opts.key_frame(video_frame_counter == 0);
video_encoder.encode_with_options(&video_frame, &opts);
video_frame.close();
}
Err(e) => {
log!("error", e);
}
}
let mut timestamp = 0.0;
let process_frame = Closure::wrap(Box::new(move || {
if !enabled.load(Ordering::Acquire)
|| destroy.load(Ordering::Acquire)
|| switching.load(Ordering::Acquire)
{
video_track
.clone()
.unchecked_into::<MediaStreamTrack>()
.stop();
video_encoder.close();
switching.store(false, Ordering::Release);
return;
}
};
poll_video.await;
if let Err(e) = ctx.draw_image_with_html_video_element_and_dw_and_dh(
&video_element,
0.0,
0.0,
VIDEO_WIDTH.into(),
VIDEO_HEIGHT.into(),
) {
log!("Error", e);
} else {
let mut video_frame_init = VideoFrameInit::new();
video_frame_init.timestamp(timestamp);
video_frame_init.duration(1.0 / VIDEO_FPS);

let video_frame =
VideoFrame::new_with_html_canvas_element_and_video_frame_init(
&canvas_element,
&video_frame_init,
)
.unwrap();

let mut opts: VideoEncoderEncodeOptions = VideoEncoderEncodeOptions::new();
video_frame_counter = (video_frame_counter + 1) % 50;
opts.key_frame(video_frame_counter == 0);
video_encoder.encode_with_options(&video_frame, &opts);
video_frame.close();
timestamp += 1.0 / VIDEO_FPS;
}
}) as Box<dyn FnMut()>);
window().set_interval_with_callback_and_timeout_and_arguments_0(
process_frame.as_ref().unchecked_ref(),
1000 / VIDEO_FPS as i32,
);
process_frame.forget();
log!("Killing video streamer");
});
}
Expand Down

0 comments on commit c09fba3

Please sign in to comment.