Skip to content

Commit

Permalink
MediaStream playback through audio and video elements
Browse files Browse the repository at this point in the history
  • Loading branch information
@ferjm
ferjm committed Apr 10, 2019
1 parent f142b1d commit af242a0
Show file tree
Hide file tree
Showing 5 changed files with 115 additions and 70 deletions.
23 changes: 14 additions & 9 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions components/script/dom/bindings/trace.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ use servo_media::audio::graph::NodeId;
use servo_media::audio::panner_node::{DistanceModel, PanningModel};
use servo_media::audio::param::ParamType;
use servo_media::player::Player;
use servo_media::streams::MediaStream as BackendMediaStream;
use servo_media::streams::registry::MediaStreamId;
use servo_media::webrtc::WebRtcController;
use servo_url::{ImmutableOrigin, MutableOrigin, ServoUrl};
use smallvec::SmallVec;
Expand Down Expand Up @@ -488,7 +488,7 @@ unsafe_no_jsmanaged_fields!(NodeId);
unsafe_no_jsmanaged_fields!(AnalysisEngine, DistanceModel, PanningModel, ParamType);
unsafe_no_jsmanaged_fields!(dyn Player);
unsafe_no_jsmanaged_fields!(WebRtcController);
unsafe_no_jsmanaged_fields!(dyn BackendMediaStream);
unsafe_no_jsmanaged_fields!(MediaStreamId);
unsafe_no_jsmanaged_fields!(Mutex<MediaFrameRenderer>);
unsafe_no_jsmanaged_fields!(RenderApiSender);
unsafe_no_jsmanaged_fields!(ResourceFetchTiming);
Expand Down
135 changes: 90 additions & 45 deletions components/script/dom/htmlmediaelement.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,7 @@ pub struct HTMLMediaElement {
#[ignore_malloc_size_of = "promises are hard"]
in_flight_play_promises_queue: DomRefCell<VecDeque<(Box<[Rc<Promise>]>, ErrorResult)>>,
#[ignore_malloc_size_of = "servo_media"]
player: Box<Player>,
player: DomRefCell<Option<Box<Player>>>,
#[ignore_malloc_size_of = "Arc"]
frame_renderer: Arc<Mutex<MediaFrameRenderer>>,
/// https://html.spec.whatwg.org/multipage/#show-poster-flag
Expand Down Expand Up @@ -295,7 +295,7 @@ impl HTMLMediaElement {
delaying_the_load_event_flag: Default::default(),
pending_play_promises: Default::default(),
in_flight_play_promises_queue: Default::default(),
player: ServoMedia::get().unwrap().create_player(),
player: Default::default(),
frame_renderer: Arc::new(Mutex::new(MediaFrameRenderer::new(
document.window().get_webrender_api_sender(),
))),
Expand Down Expand Up @@ -330,11 +330,13 @@ impl HTMLMediaElement {
}

fn play_media(&self) {
if let Err(e) = self.player.set_rate(self.playbackRate.get()) {
warn!("Could not set the playback rate {:?}", e);
}
if let Err(e) = self.player.play() {
warn!("Could not play media {:?}", e);
if let Some(ref player) = *self.player.borrow() {
if let Err(e) = player.set_rate(self.playbackRate.get()) {
warn!("Could not set the playback rate {:?}", e);
}
if let Err(e) = player.play() {
warn!("Could not play media {:?}", e);
}
}
}

Expand Down Expand Up @@ -398,8 +400,10 @@ impl HTMLMediaElement {
// Step 2.3.2.
this.upcast::<EventTarget>().fire_event(atom!("pause"));

if let Err(e) = this.player.pause() {
eprintln!("Could not pause player {:?}", e);
if let Some(ref player) = *this.player.borrow() {
if let Err(e) = player.pause() {
eprintln!("Could not pause player {:?}", e);
}
}

// Step 2.3.3.
Expand Down Expand Up @@ -762,7 +766,7 @@ impl HTMLMediaElement {

// https://html.spec.whatwg.org/multipage/#concept-media-load-resource
fn resource_fetch_algorithm(&self, resource: Resource) {
if let Err(e) = self.setup_media_player() {
if let Err(e) = self.setup_media_player(&resource) {
eprintln!("Setup media player error {:?}", e);
self.queue_dedicated_media_source_failure_steps();
return;
Expand Down Expand Up @@ -827,8 +831,14 @@ impl HTMLMediaElement {
Some(ServoUrl::parse(&blob_url).expect("infallible"));
self.fetch_request(None);
},
SrcObject::MediaStream(_) => {
self.queue_dedicated_media_source_failure_steps();
SrcObject::MediaStream(ref stream) => {
for stream in stream.get_tracks() {
if let Err(_) =
self.player.borrow().as_ref().unwrap().set_stream(&stream)
{
self.queue_dedicated_media_source_failure_steps();
}
}
},
}
}
Expand Down Expand Up @@ -872,8 +882,10 @@ impl HTMLMediaElement {
// Step 5.
this.upcast::<EventTarget>().fire_event(atom!("error"));

if let Err(e) = this.player.stop() {
eprintln!("Could not stop player {:?}", e);
if let Some(ref player) = *this.player.borrow() {
if let Err(e) = player.stop() {
eprintln!("Could not stop player {:?}", e);
}
}

// Step 6.
Expand Down Expand Up @@ -1126,8 +1138,10 @@ impl HTMLMediaElement {
task_source.queue_simple_event(self.upcast(), atom!("seeking"), &window);

// Step 11.
if let Err(e) = self.player.seek(time) {
eprintln!("Seek error {:?}", e);
if let Some(ref player) = *self.player.borrow() {
if let Err(e) = player.seek(time) {
eprintln!("Seek error {:?}", e);
}
}

// The rest of the steps are handled when the media engine signals a
Expand Down Expand Up @@ -1175,12 +1189,28 @@ impl HTMLMediaElement {
}
}

fn setup_media_player(&self) -> Result<(), PlayerError> {
fn setup_media_player(&self, resource: &Resource) -> Result<(), ()> {
let stream_type = match *resource {
Resource::Object => {
if let Some(ref src_object) = *self.src_object.borrow() {
match src_object {
SrcObject::MediaStream(_) => StreamType::Stream,
_ => StreamType::Seekable,
}
} else {
return Err(());
}
},
_ => StreamType::Seekable,
};

let player = ServoMedia::get().unwrap().create_player(stream_type);

let (action_sender, action_receiver) = ipc::channel().unwrap();
player.register_event_handler(action_sender);
player.register_frame_renderer(self.frame_renderer.clone());

self.player.register_event_handler(action_sender);
self.player
.register_frame_renderer(self.frame_renderer.clone());
*self.player.borrow_mut() = Some(player);

let trusted_node = Trusted::new(self);
let window = window_from_node(self);
Expand Down Expand Up @@ -1536,8 +1566,10 @@ impl HTMLMediaElement {

impl Drop for HTMLMediaElement {
fn drop(&mut self) {
if let Err(err) = self.player.shutdown() {
warn!("Error shutting down player {:?}", err);
if let Some(ref player) = *self.player.borrow() {
if let Err(err) = player.shutdown() {
warn!("Error shutting down player {:?}", err);
}
}
}
}
Expand Down Expand Up @@ -1579,15 +1611,17 @@ impl HTMLMediaElementMethods for HTMLMediaElement {
if self.muted.get() == value {
return;
}
self.muted.set(value);
let _ = self.player.set_mute(value);
let window = window_from_node(self);
window
.task_manager()
.media_element_task_source()
.queue_simple_event(self.upcast(), atom!("volumechange"), &window);
if !self.is_allowed_to_play() {
self.internal_pause_steps();
if let Some(ref player) = *self.player.borrow() {
self.muted.set(value);
let _ = player.set_mute(value);
let window = window_from_node(self);
window
.task_manager()
.media_element_task_source()
.queue_simple_event(self.upcast(), atom!("volumechange"), &window);
if !self.is_allowed_to_play() {
self.internal_pause_steps();
}
}
}

Expand Down Expand Up @@ -1790,8 +1824,10 @@ impl HTMLMediaElementMethods for HTMLMediaElement {
self.playbackRate.set(*value);
self.queue_ratechange_event();
if self.is_potentially_playing() {
if let Err(e) = self.player.set_rate(*value) {
warn!("Could not set the playback rate {:?}", e);
if let Some(ref player) = *self.player.borrow() {
if let Err(e) = player.set_rate(*value) {
warn!("Could not set the playback rate {:?}", e);
}
}
}
}
Expand Down Expand Up @@ -1855,9 +1891,11 @@ impl HTMLMediaElementMethods for HTMLMediaElement {
// https://html.spec.whatwg.org/multipage/#dom-media-buffered
fn Buffered(&self) -> DomRoot<TimeRanges> {
let mut buffered = TimeRangesContainer::new();
if let Ok(ranges) = self.player.buffered() {
for range in ranges {
let _ = buffered.add(range.start as f64, range.end as f64);
if let Some(ref player) = *self.player.borrow() {
if let Ok(ranges) = player.buffered() {
for range in ranges {
let _ = buffered.add(range.start as f64, range.end as f64);
}
}
}
TimeRanges::new(self.global().as_window(), buffered)
Expand Down Expand Up @@ -2129,7 +2167,7 @@ impl FetchResponseListener for HTMLMediaElementFetchListener {
fn process_response(&mut self, metadata: Result<FetchMetadata, NetworkError>) {
let elem = self.elem.root();

if elem.generation_id.get() != self.generation_id {
if elem.generation_id.get() != self.generation_id || elem.player.borrow().is_none() {
// A new fetch request was triggered, so we ignore this response.
return;
}
Expand All @@ -2155,7 +2193,13 @@ impl FetchResponseListener for HTMLMediaElementFetchListener {
// We only set the expected input size if it changes.
if content_length != self.expected_content_length {
if let Some(content_length) = content_length {
if let Err(e) = elem.player.set_input_size(content_length) {
if let Err(e) = elem
.player
.borrow()
.as_ref()
.unwrap()
.set_input_size(content_length)
{
warn!("Could not set player input size {:?}", e);
} else {
self.expected_content_length = Some(content_length);
Expand All @@ -2178,10 +2222,6 @@ impl FetchResponseListener for HTMLMediaElementFetchListener {
if let Some(ref mut current_fetch_context) = *elem.current_fetch_context.borrow_mut() {
current_fetch_context.set_seekable(true);
}
// and we can safely set the type of stream to Seekable.
if let Err(e) = elem.player.set_stream_type(StreamType::Seekable) {
warn!("Could not set stream type to Seekable. {:?}", e);
}
}

// => "If the media data cannot be fetched at all..."
Expand All @@ -2199,7 +2239,7 @@ impl FetchResponseListener for HTMLMediaElementFetchListener {
let elem = self.elem.root();
// If an error was received previously or if we triggered a new fetch request,
// we skip processing the payload.
if elem.generation_id.get() != self.generation_id {
if elem.generation_id.get() != self.generation_id || elem.player.borrow().is_none() {
return;
}
if let Some(ref current_fetch_context) = *elem.current_fetch_context.borrow() {
Expand All @@ -2211,7 +2251,7 @@ impl FetchResponseListener for HTMLMediaElementFetchListener {
let payload_len = payload.len() as u64;

// Push input data into the player.
if let Err(e) = elem.player.push_data(payload) {
if let Err(e) = elem.player.borrow().as_ref().unwrap().push_data(payload) {
// If we are pushing too much data and we know that we can
// restart the download later from where we left, we cancel
// the current request. Otherwise, we continue the request
Expand Down Expand Up @@ -2247,13 +2287,18 @@ impl FetchResponseListener for HTMLMediaElementFetchListener {
// https://html.spec.whatwg.org/multipage/#media-data-processing-steps-list
fn process_response_eof(&mut self, status: Result<ResourceFetchTiming, NetworkError>) {
let elem = self.elem.root();

if elem.player.borrow().is_none() {
return;
}

// If an error was previously received and no new fetch request was triggered,
// we skip processing the payload and notify the media backend that we are done
// pushing data.
if elem.generation_id.get() == self.generation_id {
if let Some(ref current_fetch_context) = *elem.current_fetch_context.borrow() {
if let Some(CancelReason::Error) = current_fetch_context.cancel_reason() {
if let Err(e) = elem.player.end_of_stream() {
if let Err(e) = elem.player.borrow().as_ref().unwrap().end_of_stream() {
warn!("Could not signal EOS to player {:?}", e);
}
return;
Expand Down
Loading

0 comments on commit af242a0

Please sign in to comment.