Skip to content
Permalink
Browse files
Enforce Low Power Mode / Optimize Video Streaming setting by tone map…
…ping HDR video to SDR

https://bugs.webkit.org/show_bug.cgi?id=245863
<rdar://100597710>

Reviewed by Eric Carlson.

On macOS and iOS, WebKit will stop advertising support for HDR through the `dynamic-range:` media
query when the device is off-battery or in low-power mode. However, sites can chose to ignore those
media query changes and display HDR content anyway, causing increased power use primarily due to
higher display backlight levels.

Enforce these modes by, when they are enabled, tone mapping HDR video content to SDR, as if the
display did not support HDR at all.

Add a new MediaPlayer `shouldDisableHDR()/setShouldDisableHDR()` calls, piped through
MediaPlayerRemote/RemoteMediaPlayerProxy, and implement these methods in
MediaPlayerPrivateAVFoundationObjC and MediaPlayerPrivateMediaSourceAVFObjC.

* Source/WebCore/PAL/pal/spi/cocoa/QuartzCoreSPI.h:
* Source/WebCore/html/HTMLMediaElement.cpp:
(WebCore::HTMLMediaElement::setPreferredDynamicRangeMode):
(WebCore::HTMLMediaElement::setOverridePreferredDynamicRangeMode):
(WebCore::HTMLMediaElement::shouldDisableHDR const):
* Source/WebCore/html/HTMLMediaElement.h:
* Source/WebCore/platform/graphics/MediaPlayer.cpp:
(WebCore::MediaPlayer::setShouldDisableHDR):
* Source/WebCore/platform/graphics/MediaPlayer.h:
(WebCore::MediaPlayerClient::mediaPlayerShouldDisableHDR const):
* Source/WebCore/platform/graphics/MediaPlayerPrivate.h:
(WebCore::MediaPlayerPrivateInterface::setShouldDisableHDR):
* Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateAVFoundationObjC.h:
* Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateAVFoundationObjC.mm:
(WebCore::MediaPlayerPrivateAVFoundationObjC::createAVPlayer):
(WebCore::MediaPlayerPrivateAVFoundationObjC::setShouldDisableHDR):
* Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.h:
* Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.mm:
(WebCore::MediaPlayerPrivateMediaSourceAVFObjC::ensureLayer):
(WebCore::MediaPlayerPrivateMediaSourceAVFObjC::setShouldDisableHDR):
* Source/WebKit/GPUProcess/media/RemoteMediaPlayerProxy.cpp:
(WebKit::RemoteMediaPlayerProxy::setShouldDisableHDR):
* Source/WebKit/GPUProcess/media/RemoteMediaPlayerProxy.h:
* Source/WebKit/GPUProcess/media/RemoteMediaPlayerProxy.messages.in:
* Source/WebKit/GPUProcess/media/RemoteMediaPlayerProxyConfiguration.h:
(WebKit::RemoteMediaPlayerProxyConfiguration::encode const):
(WebKit::RemoteMediaPlayerProxyConfiguration::decode):
* Source/WebKit/UIProcess/Cocoa/WebProcessPoolCocoa.mm:
(WebKit::WebProcessPool::registerHighDynamicRangeChangeCallback):
* Source/WebKit/UIProcess/WebProcessPool.cpp:
* Source/WebKit/WebProcess/GPU/media/MediaPlayerPrivateRemote.cpp:
(WebKit::MediaPlayerPrivateRemote::setShouldDisableHDR):
* Source/WebKit/WebProcess/GPU/media/MediaPlayerPrivateRemote.h:

Canonical link: https://commits.webkit.org/255127@main
  • Loading branch information
jernoble committed Oct 4, 2022
1 parent d060871 commit d11f383c16edb72e770a3ea66471be542de54734
Show file tree
Hide file tree
Showing 18 changed files with 105 additions and 6 deletions.
@@ -136,6 +136,7 @@ typedef struct _CARenderContext CARenderContext;
#if HAVE(CORE_ANIMATION_SEPARATED_LAYERS)
@property (getter=isSeparated) BOOL separated;
#endif
@property BOOL toneMapToStandardDynamicRange;
@end

#if ENABLE(FILTERS_LEVEL_2)
@@ -7233,6 +7233,7 @@ void HTMLMediaElement::createMediaPlayer() WTF_IGNORES_THREAD_SAFETY_ANALYSIS
m_player = MediaPlayer::create(*this);
m_player->setBufferingPolicy(m_bufferingPolicy);
m_player->setPreferredDynamicRangeMode(m_overrideDynamicRangeMode.value_or(preferredDynamicRangeMode(document().view())));
m_player->setShouldDisableHDR(shouldDisableHDR());
m_player->setMuted(effectiveMuted());
m_player->setPageIsVisible(!m_elementIsHidden);
m_player->setVisibleInViewport(isVisibleInViewport());
@@ -7636,15 +7637,21 @@ String HTMLMediaElement::sourceApplicationIdentifier() const

void HTMLMediaElement::setPreferredDynamicRangeMode(DynamicRangeMode mode)
{
if (m_player && !m_overrideDynamicRangeMode)
m_player->setPreferredDynamicRangeMode(mode);
if (!m_player || m_overrideDynamicRangeMode)
return;

m_player->setPreferredDynamicRangeMode(mode);
m_player->setShouldDisableHDR(shouldDisableHDR());
}

void HTMLMediaElement::setOverridePreferredDynamicRangeMode(DynamicRangeMode mode)
{
m_overrideDynamicRangeMode = mode;
if (m_player)
m_player->setPreferredDynamicRangeMode(mode);
if (!m_player)
return;

m_player->setPreferredDynamicRangeMode(mode);
m_player->setShouldDisableHDR(shouldDisableHDR());
}

Vector<String> HTMLMediaElement::mediaPlayerPreferredAudioCharacteristics() const
@@ -8838,6 +8845,11 @@ void HTMLMediaElement::setShowingStats(bool shouldShowStats)
#endif
}

bool HTMLMediaElement::shouldDisableHDR() const
{
return !screenSupportsHighDynamicRange(document().view());
}

}

#endif
@@ -799,6 +799,8 @@ class HTMLMediaElement
void mediaPlayerBufferedTimeRangesChanged() final;
bool mediaPlayerPrefersSandboxedParsing() const final;

bool mediaPlayerShouldDisableHDR() const final { return shouldDisableHDR(); }

#if USE(GSTREAMER)
void requestInstallMissingPlugins(const String& details, const String& description, MediaPlayerRequestInstallMissingPluginsCallback&) final;
#endif
@@ -1005,6 +1007,8 @@ class HTMLMediaElement
const Logger& mediaPlayerLogger() final { return logger(); }
#endif

bool shouldDisableHDR() const;

Timer m_progressEventTimer;
Timer m_playbackProgressTimer;
Timer m_scanTimer;
@@ -1822,6 +1822,11 @@ void MediaPlayer::renderVideoWillBeDestroyed()
m_private->renderVideoWillBeDestroyed();
}

void MediaPlayer::setShouldDisableHDR(bool shouldDisable)
{
m_private->setShouldDisableHDR(shouldDisable);
}

void MediaPlayer::playerContentBoxRectChanged(const LayoutRect& rect)
{
m_private->playerContentBoxRectChanged(rect);
@@ -293,6 +293,8 @@ class MediaPlayerClient {

virtual bool mediaPlayerPrefersSandboxedParsing() const { return false; }

virtual bool mediaPlayerShouldDisableHDR() const { return false; }

#if !RELEASE_LOG_DISABLED
virtual const void* mediaPlayerLogIdentifier() { return nullptr; }
virtual const Logger& mediaPlayerLogger() = 0;
@@ -712,6 +714,9 @@ class WEBCORE_EXPORT MediaPlayer : public MediaPlayerEnums, public ThreadSafeRef

void renderVideoWillBeDestroyed();

void setShouldDisableHDR(bool);
bool shouldDisableHDR() const { return client().mediaPlayerShouldDisableHDR(); }

private:
MediaPlayer(MediaPlayerClient&);
MediaPlayer(MediaPlayerClient&, MediaPlayerEnums::MediaEngineIdentifier);
@@ -191,6 +191,8 @@ class MediaPlayerPrivateInterface {
virtual DestinationColorSpace colorSpace() = 0;
virtual bool shouldGetNativeImageForCanvasDrawing() const { return true; }

virtual void setShouldDisableHDR(bool) { }

virtual void setPreload(MediaPlayer::Preload) { }

virtual bool hasAvailableVideoFrame() const { return readyState() >= MediaPlayer::ReadyState::HaveCurrentData; }
@@ -356,6 +356,8 @@ class MediaPlayerPrivateAVFoundationObjC final : public MediaPlayerPrivateAVFoun

void checkNewVideoFrameMetadata();

void setShouldDisableHDR(bool) final;

std::optional<bool> allTracksArePlayable() const;
bool containsDisabledTracks() const;
bool trackIsPlayable(AVAssetTrack*) const;
@@ -1080,6 +1080,9 @@ static URL conformFragmentIdentifierForURL(const URL& url)
m_avPlayer.get().videoRangeOverride = convertDynamicRangeModeEnumToAVVideoRange(player()->preferredDynamicRangeMode());
#endif

if ([m_videoLayer respondsToSelector:@selector(setToneMapToStandardDynamicRange:)])
[m_videoLayer setToneMapToStandardDynamicRange:player()->shouldDisableHDR()];

#if ENABLE(WIRELESS_PLAYBACK_TARGET)
updateDisableExternalPlayback();
[m_avPlayer setAllowsExternalPlayback:m_allowsWirelessVideoPlayback];
@@ -3837,6 +3840,15 @@ void determineChangedTracksFromNewTracksAndOldItems(MediaSelectionGroupAVFObjC*
#endif
}

void MediaPlayerPrivateAVFoundationObjC::setShouldDisableHDR(bool shouldDisable)
{
if (![m_videoLayer respondsToSelector:@selector(setToneMapToStandardDynamicRange:)])
return;

ALWAYS_LOG(LOGIDENTIFIER, shouldDisable);
[m_videoLayer setToneMapToStandardDynamicRange:shouldDisable];
}

void MediaPlayerPrivateAVFoundationObjC::audioOutputDeviceChanged()
{
#if HAVE(AUDIO_OUTPUT_DEVICE_UNIQUE_ID)
@@ -297,6 +297,8 @@ class MediaPlayerPrivateMediaSourceAVFObjC

bool shouldEnsureLayer() const;

void setShouldDisableHDR(bool) final;

friend class MediaSourcePrivateAVFObjC;

struct PendingSeek {
@@ -40,6 +40,7 @@
#import "MediaSourcePrivateAVFObjC.h"
#import "MediaSourcePrivateClient.h"
#import "PixelBufferConformerCV.h"
#import "PlatformScreen.h"
#import "TextTrackRepresentation.h"
#import "VideoFrameCV.h"
#import "VideoLayerManagerObjC.h"
@@ -51,6 +52,7 @@
#import <objc_runtime.h>
#import <pal/avfoundation/MediaTimeAVFoundation.h>
#import <pal/spi/cocoa/AVFoundationSPI.h>
#import <pal/spi/cocoa/QuartzCoreSPI.h>
#import <wtf/Deque.h>
#import <wtf/FileSystem.h>
#import <wtf/MainThread.h>
@@ -911,6 +913,9 @@ void getSupportedTypes(HashSet<String, ASCIICaseInsensitiveHash>& types) const f
return;
}

if ([m_sampleBufferDisplayLayer respondsToSelector:@selector(setToneMapToStandardDynamicRange:)])
[m_sampleBufferDisplayLayer setToneMapToStandardDynamicRange:m_player->shouldDisableHDR()];

if (m_mediaSourcePrivate)
m_mediaSourcePrivate->setVideoLayer(m_sampleBufferDisplayLayer.get());
m_videoLayerManager->setVideoLayer(m_sampleBufferDisplayLayer.get(), snappedIntRect(m_player->playerContentBoxRect()).size());
@@ -1489,6 +1494,15 @@ void getSupportedTypes(HashSet<String, ASCIICaseInsensitiveHash>& types) const f
m_videoFrameMetadataGatheringObserver = nil;
}

void MediaPlayerPrivateMediaSourceAVFObjC::setShouldDisableHDR(bool shouldDisable)
{
if (![m_sampleBufferDisplayLayer respondsToSelector:@selector(setToneMapToStandardDynamicRange:)])
return;

ALWAYS_LOG(LOGIDENTIFIER, shouldDisable);
[m_sampleBufferDisplayLayer setToneMapToStandardDynamicRange:shouldDisable];
}

WTFLogChannel& MediaPlayerPrivateMediaSourceAVFObjC::logChannel() const
{
return LogMediaSource;
@@ -916,6 +916,17 @@ void RemoteMediaPlayerProxy::videoFrameForCurrentTimeIfChanged(CompletionHandler
completionHandler(WTFMove(result), changed);
}

void RemoteMediaPlayerProxy::setShouldDisableHDR(bool shouldDisable)
{
if (m_configuration.shouldDisableHDR == shouldDisable)
return;

m_configuration.shouldDisableHDR = shouldDisable;
if (m_player)
m_player->setShouldDisableHDR(shouldDisable);
}


void RemoteMediaPlayerProxy::updateCachedState(bool forceCurrentTimeUpdate)
{
if (!m_observingTimeChanges || forceCurrentTimeUpdate)
@@ -311,6 +311,7 @@ class RemoteMediaPlayerProxy final
const std::optional<Vector<WebCore::FourCC>>& allowedMediaCaptionFormatTypes() const final { return m_configuration.allowedMediaCaptionFormatTypes; };

bool mediaPlayerPrefersSandboxedParsing() const final { return m_configuration.prefersSandboxedParsing; }
bool mediaPlayerShouldDisableHDR() const final { return m_configuration.shouldDisableHDR; }

void startUpdateCachedStateMessageTimer();
void updateCachedState(bool = false);
@@ -347,6 +348,8 @@ class RemoteMediaPlayerProxy final
#endif
void videoFrameForCurrentTimeIfChanged(CompletionHandler<void(std::optional<RemoteVideoFrameProxy::Properties>&&, bool)>&&);

void setShouldDisableHDR(bool);

#if !RELEASE_LOG_DISABLED
const Logger& mediaPlayerLogger() final { return m_logger; }
const void* mediaPlayerLogIdentifier() { return reinterpret_cast<const void*>(m_configuration.logIdentifier); }
@@ -138,6 +138,8 @@ messages -> RemoteMediaPlayerProxy NotRefCounted {
StopVideoFrameMetadataGathering()

PlayerContentBoxRectChanged(WebCore::LayoutRect contentRect)

SetShouldDisableHDR(bool shouldDisable)
}

#endif
@@ -58,6 +58,7 @@ struct RemoteMediaPlayerProxyConfiguration {
bool isVideo { false };
bool renderingCanBeAccelerated { false };
bool prefersSandboxedParsing { false };
bool shouldDisableHDR { false };

template<class Encoder>
void encode(Encoder& encoder) const
@@ -83,6 +84,7 @@ struct RemoteMediaPlayerProxyConfiguration {
encoder << isVideo;
encoder << renderingCanBeAccelerated;
encoder << prefersSandboxedParsing;
encoder << shouldDisableHDR;
}

template <class Decoder>
@@ -108,7 +110,8 @@ struct RemoteMediaPlayerProxyConfiguration {
&& decoder.decode(configuration.shouldUsePersistentCache)
&& decoder.decode(configuration.isVideo)
&& decoder.decode(configuration.renderingCanBeAccelerated)
&& decoder.decode(configuration.prefersSandboxedParsing);
&& decoder.decode(configuration.prefersSandboxedParsing)
&& decoder.decode(configuration.shouldDisableHDR);
}
};

@@ -56,6 +56,7 @@
#import <WebCore/Color.h>
#import <WebCore/FontCacheCoreText.h>
#import <WebCore/LocalizedDeviceModel.h>
#import <WebCore/LowPowerModeNotifier.h>
#import <WebCore/NetworkStorageSession.h>
#import <WebCore/NotImplemented.h>
#import <WebCore/PictureInPictureSupport.h>
@@ -1168,4 +1169,15 @@ static void webProcessPoolHighDynamicRangeDidChangeCallback(CMNotificationCenter
}
#endif // PLATFORM(MAC)

#if PLATFORM(IOS)
void WebProcessPool::registerHighDynamicRangeChangeCallback()
{
static NeverDestroyed<LowPowerModeNotifier> notifier { [](bool) {
auto properties = WebCore::collectScreenProperties();
for (auto& pool : WebProcessPool::allProcessPools())
pool->sendToAllProcesses(Messages::WebProcess::SetScreenProperties(properties));
} };
}
#endif // PLATFORM(MAC) || PLATFORM(IOS)

} // namespace WebKit
@@ -670,11 +670,13 @@ RefPtr<WebProcessProxy> WebProcessPool::tryTakePrewarmedProcess(WebsiteDataStore
void WebProcessPool::registerDisplayConfigurationCallback()
{
}
#endif // !PLATFORM(MAC)

#if !PLATFORM(MAC) && !PLATFORM(IOS)
void WebProcessPool::registerHighDynamicRangeChangeCallback()
{
}
#endif // !PLATFORM(MAC)
#endif

WebProcessDataStoreParameters WebProcessPool::webProcessDataStoreParameters(WebProcessProxy& process, WebsiteDataStore& websiteDataStore)
{
@@ -1448,6 +1448,11 @@ void MediaPlayerPrivateRemote::playerContentBoxRectChanged(const LayoutRect& con
connection().send(Messages::RemoteMediaPlayerProxy::PlayerContentBoxRectChanged(contentRect), m_id);
}

void MediaPlayerPrivateRemote::setShouldDisableHDR(bool shouldDisable)
{
connection().send(Messages::RemoteMediaPlayerProxy::SetShouldDisableHDR(shouldDisable), m_id);
}

void MediaPlayerPrivateRemote::requestResource(RemoteMediaResourceIdentifier remoteMediaResourceIdentifier, WebCore::ResourceRequest&& request, WebCore::PlatformMediaResourceLoader::LoadOptions options)
{
ASSERT(!m_mediaResources.contains(remoteMediaResourceIdentifier));
@@ -417,6 +417,8 @@ class MediaPlayerPrivateRemote final

void playerContentBoxRectChanged(const WebCore::LayoutRect&) final;

void setShouldDisableHDR(bool) final;

#if PLATFORM(COCOA)
void pushVideoFrameMetadata(WebCore::VideoFrameMetadata&&, RemoteVideoFrameProxy::Properties&&);
#endif

0 comments on commit d11f383

Please sign in to comment.