Skip to content
Permalink
Browse files
Switch to alternate <source> element for AirPlay when necessary
https://bugs.webkit.org/show_bug.cgi?id=246466
<rdar://101136233>

Reviewed by Jer Noble.

* LayoutTests/media/media-source/remoteplayback-from-source-element-expected.txt: Added.
* LayoutTests/media/media-source/remoteplayback-from-source-element.html: Added.
* LayoutTests/TestExpectations: New test skipped everywhere.
* LayoutTests/platform/mac/TestExpectations: New test enabled.

* Source/WebCore/Modules/remoteplayback/RemotePlayback.cpp:
(WebCore::RemotePlayback::watchAvailability): Add runtime logging.
(WebCore::RemotePlayback::cancelWatchAvailability): Ditto.
(WebCore::RemotePlayback::prompt): Ditto.
(WebCore::RemotePlayback::shouldPlayToRemoteTargetChanged): Ditto.
(WebCore::RemotePlayback::setState): Ditto.
(WebCore::RemotePlayback::disconnect): Ditto.
(WebCore::RemotePlayback::availabilityChanged): Ditto.
(WebCore::RemotePlayback::setLogger):  Ditto.
(WebCore::RemotePlayback::logChannel const): Ditto.
* Source/WebCore/Modules/remoteplayback/RemotePlayback.h:

* Source/WebCore/html/HTMLMediaElement.cpp:
(WebCore::HTMLMediaElement::HTMLMediaElement): Set remote logger.
(WebCore::HTMLMediaElement::checkPlaybackTargetCompatibility): If loaded from a <source>
element and is another <source> uses a media engine that supports remote playback, try
loading that.
(WebCore::HTMLMediaElement::loadResource): Don't bother trying MSE, MediaStream, or blob
if the load requires remote playback.
(WebCore::HTMLMediaElement::applyConfiguration): Apply the stored configuration.
(WebCore::HTMLMediaElement::setReadyState): Apply the remote configuration once
HAVE_FUTURE_DATA is reached.
(WebCore::HTMLMediaElement::selectNextSourceChild): Set parameter `requiresRemotePlayback`
field.
(WebCore::HTMLMediaElement::clearMediaPlayer): Force a target availability event.
(WebCore::HTMLMediaElement::wirelessRoutesAvailableDidChange): Only post availability event
when availability actually changes.
(WebCore::HTMLMediaElement::setIsPlayingToWirelessTarget):
(WebCore::HTMLMediaElement::enqueuePlaybackTargetAvailabilityChangedEvent): Add parameter
so we don't necessarily post events when availability doesn't change.
(WebCore::HTMLMediaElement::addEventListener): Force a target availability event.
* Source/WebCore/html/HTMLMediaElement.h:

* Source/WebCore/html/MediaElementSession.cpp:
(WebCore::MediaElementSession::showPlaybackTargetPicker): Always log.

* Source/WebCore/platform/graphics/MediaPlayer.cpp:
(WebCore::MediaPlayer::load): Add `requiresRemotePlayback` parameter.
* Source/WebCore/platform/graphics/MediaPlayer.h:
(WebCore::MediaEngineSupportParameters::encode const):
(WebCore::MediaEngineSupportParameters::decode):

* Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm:
(WebCore::MediaPlayerPrivateMediaStreamAVFObjC::supportsType): Consider new support field.

* Source/WebCore/platform/graphics/cocoa/MediaPlayerPrivateWebM.mm:
(WebCore::MediaPlayerPrivateWebM::supportsType): Ditto.

* Source/WebCore/testing/Internals.cpp:
(WebCore::Internals::setMockMediaPlaybackTargetPickerEnabled): NULL-check frame and page.
(WebCore::Internals::setMockMediaPlaybackTargetPickerState): Ditto.
(WebCore::Internals::mockMediaPlaybackTargetPickerDismissPopup): Ditto.

* Source/WebKit/GPUProcess/media/RemoteMediaPlayerProxy.cpp:
(WebKit::RemoteMediaPlayerProxy::load): Add "requires remote playback" parameter.
* Source/WebKit/GPUProcess/media/RemoteMediaPlayerProxy.h:
* Source/WebKit/GPUProcess/media/RemoteMediaPlayerProxy.messages.in: Ditto.

* Source/WebKit/WebProcess/GPU/media/MediaPlayerPrivateRemote.cpp:
(WebKit::MediaPlayerPrivateRemote::load): Ditto.

* Source/WebKit/WebProcess/GPU/media/RemoteMediaPlayerMIMETypeCache.cpp:
(WebKit::RemoteMediaPlayerMIMETypeCache::supportsTypeAndCodecs): Ditto.
* Source/WebKit/WebProcess/GPU/media/RemoteMediaPlayerMIMETypeCache.h:

Canonical link: https://commits.webkit.org/255624@main
  • Loading branch information
eric-carlson committed Oct 17, 2022
1 parent 114ef69 commit 4ec10f6ab8204352d7e35c0f16f8219b6688348d
Show file tree
Hide file tree
Showing 20 changed files with 333 additions and 64 deletions.
@@ -4872,6 +4872,7 @@ imported/w3c/web-platform-tests/css/css-scroll-snap/scroll-target-snap-002.html
webkit.org/b/218325 imported/w3c/web-platform-tests/css/css-scroll-snap/scroll-target-margin-001.html [ Pass ImageOnlyFailure ]

# Cocoa-only
media/media-source/remoteplayback-from-source-element.html [ Skip ]
http/tests/media/hls/hls-hdr-switch.html [ Skip ]
http/tests/media/video-canplaythrough-webm.html [ Skip ]
media/media-session/mock-coordinator.html [ Skip ]
@@ -0,0 +1,28 @@


** Setup MSE and URL <source> elements
EXPECTED (source.readyState == 'closed') OK
EVENT(sourceopen)
EVENT(update)
EVENT(update)
EVENT(update)
EVENT(update)
EVENT(update)
EVENT(update)
EVENT(update)
EVENT(update)
EVENT(update)
EVENT(update)
EVENT(update)
EXPECTED (video.currentSrc.indexOf("blob:") === '0') OK
EXPECTED (video.readyState >= '1') OK

** Simulate a device becoming available

** Simulate selecting a device
EVENT(connect)
EXPECTED (video.currentSrc.indexOf("blob:") < '0') OK
EXPECTED (video.readyState >= '1') OK

END OF TEST

@@ -0,0 +1,132 @@
<!DOCTYPE html>
<html>
<head>
<title>Remote playback from source element</title>
<script src="media-source-loader.js"></script>
<script src="../video-test.js"></script>
<script src="../media-file.js"></script>
<script>

let source;

if (window.internals) {
internals.setMockMediaPlaybackTargetPickerEnabled(true);
internals.settings.setAllowsAirPlayForMediaPlayback(true);
}

async function setupSources()
{
consoleWrite('<br>** Setup MSE and URL &lt;source&gt; elements')

let canPlayListener = new Promise(resolve => {
video.addEventListener('canplay', event => {
resolve(event);
}, { once: true });
});

let loader = new MediaSourceLoader('content/test-fragmented-manifest.json');
await new Promise((resolve, reject) => {
loader.onload = resolve;
loader.onerror = reject;
});

source = new MediaSource();
testExpected('source.readyState', 'closed');

let sourceElement = document.createElement('source');
sourceElement.id = 'MSE';
sourceElement.type = loader.type();
sourceElement.src = URL.createObjectURL(source);
video.appendChild(sourceElement);
await waitFor(source, 'sourceopen');

let sourceBuffer = source.addSourceBuffer(loader.type());
sourceBuffer.appendBuffer(loader.initSegment());
await waitFor(sourceBuffer, 'update');

for (i = 0; i < loader.mediaSegmentsLength(); i++) {
sourceBuffer.appendBuffer(loader.mediaSegment(i));
await waitForEventWithTimeout(sourceBuffer, 'update', 5000, '"update" for sample never fired')
}

sourceElement = document.createElement('source');
let sourceFile = findMediaFile('video', '../content/test');
sourceElement.id = 'MP4';
sourceElement.src = sourceFile
sourceElement.type = mimeTypeForFile(sourceFile);
video.appendChild(sourceElement);

await canPlayListener;
testExpected('video.currentSrc.indexOf("blob:")', 0, '===');
testExpected('video.readyState', HTMLMediaElement.HAVE_METADATA, '>=');
}

async function setupRemote()
{
consoleWrite('<br>** Simulate a device becoming available')

let pendingTimeout = setTimeout(_ => {
failTest(`<br>Remote device not available after 8 seconds!`);
}, 8000);

let resolveCallback;
try {
video.remote.watchAvailability((available) => {
if (!available)
return;

video.remote.cancelWatchAvailability();
clearTimeout(pendingTimeout);
resolveCallback();
});
} catch (error) {
failTest(`<br>'watchAvailability' threw error ${error}`);
}

if (window.internals)
internals.setMockMediaPlaybackTargetPickerState('Sleepy', 'DeviceAvailable');

return new Promise(resolve => { resolveCallback = resolve; });
}

async function connectToRemote()
{
consoleWrite('<br>** Simulate selecting a device');

await video.play();
video.pause();

runWithKeyDown(async () => {
video.remote.prompt().catch(error => {
consoleWrite(`remote.prompt() failed with error '${ error }'`);
})
});

await waitForEventWithTimeout(video.remote, 'connect', 8000, "'connect' not fired after 8 seconds");

await testExpectedEventually('video.currentSrc.indexOf("blob:")', 0, '<', 8000);
testExpected('video.readyState', HTMLMediaElement.HAVE_METADATA, '>=');
consoleWrite('');
}

async function runTest()
{
findMediaElement();

waitForEventAndFail('error');

await setupSources();

await setupRemote();

await connectToRemote();

endTest();
}

</script>
</head>
<body onload="runTest()">
<video muted controls></video>
</body>
</html>
@@ -94,6 +94,7 @@ http/tests/media/fairplay [ Pass ]
media/track/track-description-cue.html [ Pass ]
media/track/track-extended-descriptions.html [ Pass ]
svg/filters/feDisplacementMap-filterUnits.svg [ Pass ]
media/media-source/remoteplayback-from-source-element.html [ Pass ]

#//////////////////////////////////////////////////////////////////////////////////////////
# End platform-specific directories.
@@ -53,6 +53,10 @@ Ref<RemotePlayback> RemotePlayback::create(HTMLMediaElement& element)

RemotePlayback::RemotePlayback(HTMLMediaElement& element)
: WebCore::ActiveDOMObject(element.scriptExecutionContext())
#if !RELEASE_LOG_DISABLED
, m_logger(element.logger())
, m_logIdentifier(element.logIdentifier())
#endif
, m_mediaElement(element)
{
}
@@ -79,8 +83,11 @@ void RemotePlayback::watchAvailability(Ref<RemotePlaybackAvailabilityCallback>&&

// 1. Let promise be a new promise->
// 2. Return promise, and run the following steps below:

queueTaskKeepingObjectAlive(*this, TaskSource::MediaElement, [this, callback = WTFMove(callback), promise = WTFMove(promise)] () mutable {

auto identifier = LOGIDENTIFIER;
ALWAYS_LOG(identifier);

queueTaskKeepingObjectAlive(*this, TaskSource::MediaElement, [this, callback = WTFMove(callback), promise = WTFMove(promise), identifier = identifier] () mutable {
if (isContextStopped())
return;

@@ -89,7 +96,7 @@ void RemotePlayback::watchAvailability(Ref<RemotePlaybackAvailabilityCallback>&&
if (!m_mediaElement
|| m_mediaElement->hasAttributeWithoutSynchronization(HTMLNames::webkitwirelessvideoplaybackdisabledAttr)
|| m_mediaElement->hasAttributeWithoutSynchronization(HTMLNames::disableremoteplaybackAttr)) {
WTFLogAlways("RemotePlayback::watchAvailability()::task - promise rejected");
ERROR_LOG(identifier, "promise rejected, remote playback disabled");
promise->reject(InvalidStateError);
return;
}
@@ -140,14 +147,18 @@ void RemotePlayback::cancelWatchAvailability(std::optional<int32_t> id, Ref<Defe
// 1. Let promise be a new promise->
// 2. Return promise, and run the following steps below:

queueTaskKeepingObjectAlive(*this, TaskSource::MediaElement, [this, id = WTFMove(id), promise = WTFMove(promise)] {
auto identifier = LOGIDENTIFIER;
ALWAYS_LOG(identifier);

queueTaskKeepingObjectAlive(*this, TaskSource::MediaElement, [this, id = WTFMove(id), promise = WTFMove(promise), identifier = identifier] {
if (isContextStopped())
return;
// 3. If the disableRemotePlayback attribute is present for the media element, reject promise with
// InvalidStateError and abort all the remaining steps.
if (!m_mediaElement
|| m_mediaElement->hasAttributeWithoutSynchronization(HTMLNames::webkitwirelessvideoplaybackdisabledAttr)
|| m_mediaElement->hasAttributeWithoutSynchronization(HTMLNames::disableremoteplaybackAttr)) {
ERROR_LOG(identifier, "promise rejected, remote playback disabled");
promise->reject(InvalidStateError);
return;
}
@@ -162,6 +173,7 @@ void RemotePlayback::cancelWatchAvailability(std::optional<int32_t> id, Ref<Defe
m_callbackMap.remove(it);
// 6. Otherwise, reject promise with NotFoundError and abort all the remaining steps.
else {
ERROR_LOG(identifier, "promise rejected, no matching callback");
promise->reject(NotFoundError);
return;
}
@@ -185,7 +197,10 @@ void RemotePlayback::prompt(Ref<DeferredPromise>&& promise)
// 1. Let promise be a new promise->
// 2. Return promise, and run the following steps below:

queueTaskKeepingObjectAlive(*this, TaskSource::MediaElement, [this, promise = WTFMove(promise), processingUserGesture = UserGestureIndicator::processingUserGesture()] () mutable {
auto identifier = LOGIDENTIFIER;
ALWAYS_LOG(identifier);

queueTaskKeepingObjectAlive(*this, TaskSource::MediaElement, [this, promise = WTFMove(promise), processingUserGesture = UserGestureIndicator::processingUserGesture(), identifier = identifier] () mutable {
if (isContextStopped())
return;

@@ -194,6 +209,7 @@ void RemotePlayback::prompt(Ref<DeferredPromise>&& promise)
if (!m_mediaElement
|| m_mediaElement->hasAttributeWithoutSynchronization(HTMLNames::webkitwirelessvideoplaybackdisabledAttr)
|| m_mediaElement->hasAttributeWithoutSynchronization(HTMLNames::disableremoteplaybackAttr)) {
ERROR_LOG(identifier, "promise rejected, remote playback disabled");
promise->reject(InvalidStateError);
return;
}
@@ -207,6 +223,7 @@ void RemotePlayback::prompt(Ref<DeferredPromise>&& promise)
// is not feasible, reject promise with a NotSupportedError and abort all remaining steps.
#if !PLATFORM(IOS)
if (m_mediaElement->readyState() < HTMLMediaElementEnums::HAVE_METADATA) {
ERROR_LOG(identifier, "promise rejected, readyState = ", m_mediaElement->readyState());
promise->reject(NotSupportedError);
return;
}
@@ -215,6 +232,7 @@ void RemotePlayback::prompt(Ref<DeferredPromise>&& promise)
// 6. If the algorithm isn't allowed to show a popup, reject promise with an InvalidAccessError exception
// and abort these steps.
if (!processingUserGesture) {
ERROR_LOG(identifier, "promise rejected, user gesture required");
promise->reject(InvalidAccessError);
return;
}
@@ -231,6 +249,7 @@ void RemotePlayback::prompt(Ref<DeferredPromise>&& promise)
// 9. If the state is disconnected and availability for the media element is false, reject promise with a
// NotSupportedError exception and abort all remaining steps.
if (m_state == State::Disconnected && !m_available) {
ERROR_LOG(identifier, "promise rejected, state = ", m_state, ", available = ", m_available);
promise->reject(NotSupportedError);
return;
}
@@ -250,7 +269,7 @@ void RemotePlayback::shouldPlayToRemoteTargetChanged(bool shouldPlayToRemoteTarg
// https://w3c.github.io/remote-playback/#prompt-user-for-changing-remote-playback-statee
// W3C Editor's Draft 15 July 2016

LOG(Media, "RemotePlayback::shouldPlayToRemoteTargetChanged(%p), shouldPlay(%d), promise count(%lu)", this, shouldPlayToRemoteTarget, m_promptPromises.size());
ALWAYS_LOG(LOGIDENTIFIER, "shouldPlay = ", shouldPlayToRemoteTarget, ", promise count = ", m_promptPromises.size());

// 10. If the user picked a remote playback device device to initiate remote playback with, the user agent
// must run the following steps:
@@ -289,6 +308,7 @@ void RemotePlayback::setState(State state)
if (m_state == state)
return;

ALWAYS_LOG(LOGIDENTIFIER, state);
m_state = state;

auto eventName = [](State state) {
@@ -331,6 +351,8 @@ void RemotePlayback::disconnect()
if (m_state == State::Disconnected)
return;

ALWAYS_LOG(LOGIDENTIFIER);

// 2. Queue a task to run the following steps:
queueTaskKeepingObjectAlive(*this, TaskSource::MediaElement, [this] {
if (isContextStopped())
@@ -403,6 +425,8 @@ void RemotePlayback::availabilityChanged(bool available)
return;
m_available = available;

ALWAYS_LOG(LOGIDENTIFIER);

queueTaskKeepingObjectAlive(*this, TaskSource::MediaElement, [this, available] {
if (isContextStopped())
return;
@@ -423,6 +447,13 @@ const char* RemotePlayback::activeDOMObjectName() const
return "RemotePlayback";
}

#if !RELEASE_LOG_DISABLED
WTFLogChannel& RemotePlayback::logChannel() const
{
return LogMedia;
}
#endif

}

#endif // ENABLE(WIRELESS_PLAYBACK_TARGET)
@@ -31,6 +31,7 @@
#include "EventTarget.h"
#include "WebCoreOpaqueRoot.h"
#include <wtf/HashMap.h>
#include <wtf/LoggerHelper.h>
#include <wtf/Ref.h>
#include <wtf/RefCounted.h>

@@ -42,7 +43,11 @@ class MediaPlaybackTarget;
class Node;
class RemotePlaybackAvailabilityCallback;

class RemotePlayback final : public RefCounted<RemotePlayback>, public ActiveDOMObject, public EventTarget {
class RemotePlayback final
: public RefCounted<RemotePlayback>
, public ActiveDOMObject
, public EventTarget
{
WTF_MAKE_ISO_ALLOCATED(RemotePlayback);
public:
static Ref<RemotePlayback> create(HTMLMediaElement&);
@@ -90,6 +95,16 @@ class RemotePlayback final : public RefCounted<RemotePlayback>, public ActiveDOM
EventTargetInterface eventTargetInterface() const final { return RemotePlaybackEventTargetInterfaceType; }
ScriptExecutionContext* scriptExecutionContext() const final { return ActiveDOMObject::scriptExecutionContext(); }

#if !RELEASE_LOG_DISABLED
const Logger& logger() const { return m_logger.get(); }
const void* logIdentifier() const { return m_logIdentifier; }
WTFLogChannel& logChannel() const;
const char* logClassName() const { return "RemotePlayback"; }

Ref<const Logger> m_logger;
const void* m_logIdentifier { nullptr };
#endif

WeakPtr<HTMLMediaElement, WeakPtrImplWithEventTargetData> m_mediaElement;
uint32_t m_nextId { 0 };

0 comments on commit 4ec10f6

Please sign in to comment.