REGRESSION: Netflix incorrect layout (size & location) when starting playback

Source/WebCore/html/HTMLMediaElement.cpp

@@ -8819,10 +8819,10 @@ MediaElementSession& HTMLMediaElement::mediaSession() const
     return *m_mediaSession;
 }
 
-void HTMLMediaElement::updateMediaPlayer(IntSize elementSize, bool shouldMaintainAspectRatio)
+void HTMLMediaElement::updateMediaPlayer(IntSize presentationSize, bool shouldMaintainAspectRatio)
 {
     ALWAYS_LOG(LOGIDENTIFIER);
-    m_player->setSize(elementSize);
+    m_player->setPresentationSize(presentationSize);
     visibilityStateChanged();
     m_player->setVisibleInViewport(isVisibleInViewport());
 

Source/WebCore/platform/graphics/MediaPlayer.cpp

@@ -159,7 +159,7 @@ class NullMediaPlayerPrivate final : public MediaPlayerPrivateInterface {
     unsigned long long totalBytes() const final { return 0; }
     bool didLoadingProgress() const final { return false; }
 
-    void setSize(const IntSize&) final { }
+    void setPresentationSize(const IntSize&) final { }
 
     void paint(GraphicsContext&, const FloatRect&) final { }
     DestinationColorSpace colorSpace() final { return DestinationColorSpace::SRGB(); }
@@ -1060,10 +1060,13 @@ void MediaPlayer::didLoadingProgress(DidLoadingProgressCompletionHandler&& callb
     m_private->didLoadingProgressAsync(WTFMove(callback));
 }
 
-void MediaPlayer::setSize(const IntSize& size)
-{ 
-    m_size = size;
-    m_private->setSize(size);
+void MediaPlayer::setPresentationSize(const IntSize& size)
+{
+    if (m_presentationSize == size)
+        return;
+
+    m_presentationSize = size;
+    m_private->setPresentationSize(size);
 }
 
 void MediaPlayer::setPageIsVisible(bool visible)

Source/WebCore/platform/graphics/MediaPlayer.h

@@ -358,8 +358,8 @@ class WEBCORE_EXPORT MediaPlayer : public MediaPlayerEnums, public ThreadSafeRef
     bool hasVideo() const;
     bool hasAudio() const;
 
-    IntSize size() const { return m_size; }
-    void setSize(const IntSize& size);
+    IntSize presentationSize() const { return m_presentationSize; }
+    void setPresentationSize(const IntSize& size);
 
     bool load(const URL&, const ContentType&, const String&, bool);
 #if ENABLE(MEDIA_SOURCE)
@@ -743,7 +743,7 @@ class WEBCORE_EXPORT MediaPlayer : public MediaPlayerEnums, public ThreadSafeRef
     String m_keySystem;
     std::optional<MediaPlayerEnums::MediaEngineIdentifier> m_activeEngineIdentifier;
     std::optional<MediaTime> m_pendingSeekRequest;
-    IntSize m_size;
+    IntSize m_presentationSize;
     Preload m_preload { Preload::Auto };
     double m_volume { 1 };
     bool m_pageIsVisible { false };

Source/WebCore/platform/graphics/MediaPlayerPrivate.h

@@ -174,7 +174,7 @@ class MediaPlayerPrivateInterface {
     // override didLoadingProgressAsync to create a more proper async implementation.
     virtual void didLoadingProgressAsync(MediaPlayer::DidLoadingProgressCompletionHandler&& callback) const { callback(didLoadingProgress()); }
 
-    virtual void setSize(const IntSize&) { }
+    virtual void setPresentationSize(const IntSize&) { }
 
     virtual void paint(GraphicsContext&, const FloatRect&) = 0;
 

Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateAVFoundationObjC.mm

@@ -640,7 +640,7 @@ static WallTime toSystemClockTime(NSDate *date)
     [m_videoLayer addObserver:m_objcObserver.get() forKeyPath:@"readyForDisplay" options:NSKeyValueObservingOptionNew context:(void *)MediaPlayerAVFoundationObservationContextAVPlayerLayer];
     updateVideoLayerGravity();
     [m_videoLayer setContentsScale:player()->playerContentsScale()];
-    m_videoLayerManager->setVideoLayer(m_videoLayer.get(), snappedIntRect(player()->playerContentBoxRect()).size());
+    m_videoLayerManager->setVideoLayer(m_videoLayer.get(), player()->presentationSize());
 
 #if PLATFORM(IOS_FAMILY) && !PLATFORM(WATCHOS) && !PLATFORM(APPLETV)
     if ([m_videoLayer respondsToSelector:@selector(setPIPModeEnabled:)])

Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.h

@@ -239,7 +239,7 @@ class MediaPlayerPrivateMediaSourceAVFObjC
     void acceleratedRenderingStateChanged() override;
     void notifyActiveSourceBuffersChanged() override;
 
-    void playerContentBoxRectChanged(const LayoutRect&) final;
+    void setPresentationSize(const IntSize&) final;
 
     void updateDisplayLayerAndDecompressionSession();
 

Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.mm

@@ -794,9 +794,9 @@ void getSupportedTypes(HashSet<String, ASCIICaseInsensitiveHash>& types) const f
 #endif
 }
 
-void MediaPlayerPrivateMediaSourceAVFObjC::playerContentBoxRectChanged(const LayoutRect& newRect)
+void MediaPlayerPrivateMediaSourceAVFObjC::setPresentationSize(const IntSize& newSize)
 {
-    if (!m_sampleBufferDisplayLayer && !newRect.isEmpty())
+    if (!m_sampleBufferDisplayLayer && !newSize.isEmpty())
         updateDisplayLayerAndDecompressionSession();
 }
 
@@ -918,7 +918,7 @@ void getSupportedTypes(HashSet<String, ASCIICaseInsensitiveHash>& types) const f
 
     if (m_mediaSourcePrivate)
         m_mediaSourcePrivate->setVideoLayer(m_sampleBufferDisplayLayer.get());
-    m_videoLayerManager->setVideoLayer(m_sampleBufferDisplayLayer.get(), snappedIntRect(m_player->playerContentBoxRect()).size());
+    m_videoLayerManager->setVideoLayer(m_sampleBufferDisplayLayer.get(), m_player->presentationSize());
     m_player->renderingModeChanged();
 }
 

Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm

@@ -398,7 +398,7 @@ static inline CGAffineTransform videoTransformationMatrix(VideoFrame& videoFrame
     if (activeVideoTrack->source().isCaptureSource())
         m_sampleBufferDisplayLayer->setRenderPolicy(SampleBufferDisplayLayer::RenderPolicy::Immediately);
 
-    auto size = snappedIntRect(m_player->playerContentBoxRect()).size();
+    auto size = m_player->presentationSize();
     m_sampleBufferDisplayLayer->initialize(hideRootLayer(), size, [weakThis = WeakPtr { *this }, size](auto didSucceed) {
         if (weakThis)
             weakThis->layersAreInitialized(size, didSucceed);

Source/WebCore/platform/graphics/cocoa/MediaPlayerPrivateWebM.h

@@ -164,7 +164,7 @@ class MediaPlayerPrivateWebM
     void characteristicsChanged();
 
     bool shouldEnsureLayer() const;
-    void playerContentBoxRectChanged(const LayoutRect&) final;
+    void setPresentationSize(const IntSize&) final;
     bool supportsAcceleratedRendering() const final { return true; }
     void acceleratedRenderingStateChanged() final;
     void updateDisplayLayerAndDecompressionSession();

Source/WebCore/platform/graphics/cocoa/MediaPlayerPrivateWebM.mm

@@ -615,15 +615,15 @@ static bool isCopyDisplayedPixelBufferAvailable()
 #if HAVE(AVSAMPLEBUFFERDISPLAYLAYER_COPYDISPLAYEDPIXELBUFFER)
     return isCopyDisplayedPixelBufferAvailable()
         && ((m_displayLayer && !CGRectIsEmpty([m_displayLayer bounds]))
-            || !m_player->playerContentBoxRect().isEmpty());
+            || !m_player->presentationSize().isEmpty());
 #else
     return !m_hasBeenAskedToPaintGL && !m_isGatheringVideoFrameMetadata;
 #endif
 }
 
-void MediaPlayerPrivateWebM::playerContentBoxRectChanged(const LayoutRect& newRect)
+void MediaPlayerPrivateWebM::setPresentationSize(const IntSize& newSize)
 {
-    if (m_hasVideo && !m_displayLayer && !newRect.isEmpty())
+    if (m_hasVideo && !m_displayLayer && !newSize.isEmpty())
         updateDisplayLayerAndDecompressionSession();
 }
 
@@ -1290,7 +1290,7 @@ static bool isCopyDisplayedPixelBufferAvailable()
     if (m_enabledVideoTrackID != notFound)
         reenqueSamples(m_enabledVideoTrackID);
 
-    m_videoLayerManager->setVideoLayer(m_displayLayer.get(), snappedIntRect(m_player->playerContentBoxRect()).size());
+    m_videoLayerManager->setVideoLayer(m_displayLayer.get(), m_player->presentationSize());
     m_player->renderingModeChanged();
 }
 

Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamer.cpp

@@ -3611,7 +3611,7 @@ void MediaPlayerPrivateGStreamer::flushCurrentBuffer()
 }
 #endif
 
-void MediaPlayerPrivateGStreamer::setSize(const IntSize& size)
+void MediaPlayerPrivateGStreamer::setPresentationSize(const IntSize& size)
 {
     m_size = size;
 }

Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamer.h

@@ -162,7 +162,7 @@ class MediaPlayerPrivateGStreamer : public MediaPlayerPrivateInterface
     MediaPlayer::NetworkState networkState() const final;
     MediaPlayer::ReadyState readyState() const final;
     void setPageIsVisible(bool visible) final { m_visible = visible; }
-    void setSize(const IntSize&) final;
+    void setPresentationSize(const IntSize&) final;
     // Prefer MediaTime based methods over float based.
     float duration() const final { return durationMediaTime().toFloat(); }
     double durationDouble() const final { return durationMediaTime().toDouble(); }

Source/WebCore/platform/graphics/win/MediaPlayerPrivateMediaFoundation.cpp

@@ -392,7 +392,7 @@ bool MediaPlayerPrivateMediaFoundation::didLoadingProgress() const
     return m_loadingProgress;
 }
 
-void MediaPlayerPrivateMediaFoundation::setSize(const IntSize& size)
+void MediaPlayerPrivateMediaFoundation::setPresentationSize(const IntSize& size)
 {
     m_size = size;
 }

Source/WebCore/platform/graphics/win/MediaPlayerPrivateMediaFoundation.h

@@ -96,7 +96,7 @@ class MediaPlayerPrivateMediaFoundation final : public MediaPlayerPrivateInterfa
 
     bool didLoadingProgress() const final;
 
-    void setSize(const IntSize&) final;
+    void setPresentationSize(const IntSize&) final;
 
     void paint(GraphicsContext&, const FloatRect&) final;
 

Source/WebCore/platform/mock/mediasource/MockMediaPlayerMediaSource.cpp

@@ -192,7 +192,7 @@ bool MockMediaPlayerMediaSource::didLoadingProgress() const
     return false;
 }
 
-void MockMediaPlayerMediaSource::setSize(const IntSize&)
+void MockMediaPlayerMediaSource::setPresentationSize(const IntSize&)
 {
 }
 

Source/WebCore/platform/mock/mediasource/MockMediaPlayerMediaSource.h

@@ -84,7 +84,7 @@ class MockMediaPlayerMediaSource : public MediaPlayerPrivateInterface, public Ca
     MediaTime maxMediaTimeSeekable() const override;
     std::unique_ptr<PlatformTimeRanges> buffered() const override;
     bool didLoadingProgress() const override;
-    void setSize(const IntSize&) override;
+    void setPresentationSize(const IntSize&) override;
     void paint(GraphicsContext&, const FloatRect&) override;
     MediaTime durationMediaTime() const override;
     void seekWithTolerance(const MediaTime&, const MediaTime&, const MediaTime&) override;

Source/WebKit/GPUProcess/media/RemoteMediaPlayerProxy.cpp

@@ -95,6 +95,7 @@ RemoteMediaPlayerProxy::RemoteMediaPlayerProxy(RemoteMediaPlayerManagerProxy& ma
     m_player = MediaPlayer::create(*this, m_engineIdentifier);
     if (auto* playerPrivate = m_player->playerPrivate())
         playerPrivate->setResourceOwner(resourceOwner);
+    m_player->setPresentationSize(m_configuration.presentationSize);
 }
 
 RemoteMediaPlayerProxy::~RemoteMediaPlayerProxy()
@@ -291,6 +292,15 @@ void RemoteMediaPlayerProxy::didLoadingProgress(CompletionHandler<void(bool)>&&
     m_player->didLoadingProgress(WTFMove(completionHandler));
 }
 
+void RemoteMediaPlayerProxy::setPresentationSize(const WebCore::IntSize& size)
+{
+    if (size == m_configuration.presentationSize)
+        return;
+
+    m_configuration.presentationSize = size;
+    m_player->setPresentationSize(size);
+}
+
 RefPtr<PlatformMediaResource> RemoteMediaPlayerProxy::requestResource(ResourceRequest&& request, PlatformMediaResourceLoader::LoadOptions options)
 {
     ASSERT(m_manager && m_manager->gpuConnectionToWebProcess());

Source/WebKit/GPUProcess/media/RemoteMediaPlayerProxy.h

@@ -166,6 +166,8 @@ class RemoteMediaPlayerProxy final
     void setRate(double);
     void didLoadingProgress(CompletionHandler<void(bool)>&&);
 
+    void setPresentationSize(const WebCore::IntSize&);
+
 #if PLATFORM(COCOA)
     void setVideoInlineSizeFenced(const WebCore::FloatSize&, const WTF::MachSendRight&);
 #endif

Source/WebKit/GPUProcess/media/RemoteMediaPlayerProxy.messages.in

@@ -64,6 +64,7 @@ messages -> RemoteMediaPlayerProxy NotRefCounted {
 #endif
 
     SetBufferingPolicy(WebCore::MediaPlayer::BufferingPolicy policy)
+    SetPresentationSize(WebCore::IntSize size)
 
 #if PLATFORM(COCOA)
     SetVideoInlineSizeFenced(WebCore::FloatSize size, MachSendRight machSendRight)

Source/WebKit/GPUProcess/media/RemoteMediaPlayerProxyConfiguration.h

@@ -53,6 +53,7 @@ struct RemoteMediaPlayerProxyConfiguration {
     Vector<WebCore::PlatformTextTrackData> outOfBandTrackData;
 #endif
     WebCore::SecurityOriginData documentSecurityOrigin;
+    WebCore::IntSize presentationSize { };
     uint64_t logIdentifier { 0 };
     bool shouldUsePersistentCache { false };
     bool isVideo { false };
@@ -79,6 +80,7 @@ struct RemoteMediaPlayerProxyConfiguration {
         encoder << outOfBandTrackData;
 #endif
         encoder << documentSecurityOrigin;
+        encoder << presentationSize;
         encoder << logIdentifier;
         encoder << shouldUsePersistentCache;
         encoder << isVideo;
@@ -106,6 +108,7 @@ struct RemoteMediaPlayerProxyConfiguration {
             && decoder.decode(configuration.outOfBandTrackData)
 #endif
             && decoder.decode(configuration.documentSecurityOrigin)
+            && decoder.decode(configuration.presentationSize)
             && decoder.decode(configuration.logIdentifier)
             && decoder.decode(configuration.shouldUsePersistentCache)
             && decoder.decode(configuration.isVideo)

Source/WebKit/GPUProcess/media/cocoa/RemoteMediaPlayerProxyCocoa.mm

@@ -78,6 +78,9 @@
 {
     ALWAYS_LOG(LOGIDENTIFIER);
     m_inlineLayerHostingContext->setRootLayer(m_player->platformLayer());
+    m_videoInlineSize = m_player->presentationSize();
+    setVideoInlineSizeIfPossible(m_videoInlineSize);
+    m_webProcessConnection->send(Messages::MediaPlayerPrivateRemote::VideoInlineSizeChanged(m_videoInlineSize), m_id);
     m_webProcessConnection->send(Messages::MediaPlayerPrivateRemote::RenderingModeChanged(), m_id);
 }
 

Source/WebKit/WebProcess/GPU/media/MediaPlayerPrivateRemote.cpp

@@ -161,10 +161,10 @@ void MediaPlayerPrivateRemote::prepareForPlayback(bool privateMode, MediaPlayer:
         if (!player)
             return;
 
-        auto contentBox = snappedIntRect(player->playerContentBoxRect()).size();
-        m_videoLayer = createVideoLayerRemote(this, inlineLayerHostingContextId.value(), m_videoFullscreenGravity, contentBox);
+        auto presentationSize = player->presentationSize();
+        m_videoLayer = createVideoLayerRemote(this, inlineLayerHostingContextId.value(), m_videoFullscreenGravity, presentationSize);
 #if PLATFORM(COCOA)
-        m_videoLayerManager->setVideoLayer(m_videoLayer.get(), contentBox);
+        m_videoLayerManager->setVideoLayer(m_videoLayer.get(), presentationSize);
 #endif
     }, m_id);
 }
@@ -987,6 +987,11 @@ unsigned long long MediaPlayerPrivateRemote::totalBytes() const
     return 0;
 }
 
+void MediaPlayerPrivateRemote::setPresentationSize(const IntSize& size)
+{
+    connection().send(Messages::RemoteMediaPlayerProxy::SetPresentationSize(size), m_id);
+}
+
 #if PLATFORM(COCOA)
 void MediaPlayerPrivateRemote::setVideoInlineSizeFenced(const FloatSize& size, const WTF::MachSendRight& machSendRight)
 {

Source/WebKit/WebProcess/GPU/media/MediaPlayerPrivateRemote.h

@@ -115,6 +115,7 @@ class MediaPlayerPrivateRemote final
     void firstVideoFrameAvailable();
     void renderingModeChanged();
 #if PLATFORM(COCOA)
+    void videoInlineSizeChanged(const WebCore::FloatSize&);
     void setVideoInlineSizeFenced(const WebCore::FloatSize&, const WTF::MachSendRight&);
 #endif
 
@@ -290,6 +291,8 @@ class MediaPlayerPrivateRemote final
     bool didLoadingProgress() const final;
     void didLoadingProgressAsync(WebCore::MediaPlayer::DidLoadingProgressCompletionHandler&&) const final;
 
+    void setPresentationSize(const WebCore::IntSize&) final;
+
     void paint(WebCore::GraphicsContext&, const WebCore::FloatRect&) final;
     void paintCurrentFrameInContext(WebCore::GraphicsContext&, const WebCore::FloatRect&) final;
 #if !USE(AVFOUNDATION)

Source/WebKit/WebProcess/GPU/media/MediaPlayerPrivateRemote.messages.in

@@ -38,6 +38,9 @@ messages -> MediaPlayerPrivateRemote NotRefCounted {
     CharacteristicChanged(struct WebKit::RemoteMediaPlayerState state)
     SizeChanged(WebCore::FloatSize naturalSize)
     RenderingModeChanged()
+#if PLATFORM(COCOA)
+    VideoInlineSizeChanged(WebCore::FloatSize videoSize)
+#endif
 
     CurrentTimeChanged(MediaTime mediaTime, MonotonicTime wallTime, bool timeIsProgressing)
 

Source/WebKit/WebProcess/GPU/media/RemoteMediaPlayerManager.cpp

@@ -173,6 +173,8 @@ std::unique_ptr<MediaPlayerPrivateInterface> RemoteMediaPlayerManager::createRem
     auto documentSecurityOrigin = player->documentSecurityOrigin();
     proxyConfiguration.documentSecurityOrigin = documentSecurityOrigin;
 
+    proxyConfiguration.presentationSize = player->presentationSize();
+
     proxyConfiguration.allowedMediaContainerTypes = player->allowedMediaContainerTypes();
     proxyConfiguration.allowedMediaCodecTypes = player->allowedMediaCodecTypes();
     proxyConfiguration.allowedMediaVideoCodecIDs = player->allowedMediaVideoCodecIDs();

Source/WebKit/WebProcess/GPU/media/cocoa/MediaPlayerPrivateRemoteCocoa.mm

@@ -30,6 +30,7 @@
 
 #import "RemoteAudioSourceProvider.h"
 #import "RemoteMediaPlayerProxyMessages.h"
+#import "VideoLayerRemoteCocoa.h"
 #import "WebCoreArgumentCoders.h"
 #import <WebCore/ColorSpaceCG.h>
 #import <pal/spi/cocoa/QuartzCoreSPI.h>
@@ -78,6 +79,12 @@
     return colorSpace;
 }
 
+void MediaPlayerPrivateRemote::videoInlineSizeChanged(const FloatSize& videoSize)
+{
+    if ([m_videoLayer isKindOfClass:[WKVideoLayerRemote class]])
+        [(WKVideoLayerRemote*)m_videoLayer.get() setVideoLayerFrame:CGRectMake(0, 0, videoSize.width(), videoSize.height())];
+}
+
 } // namespace WebKit
 
 #endif // ENABLE(GPU_PROCESS) && PLATFORM(COCOA)

Source/WebKit/WebProcess/GPU/media/cocoa/VideoLayerRemoteCocoa.mm

@@ -111,15 +111,25 @@ - (void)layoutSublayers
 
     WebCore::FloatRect sourceVideoFrame = self.videoLayerFrame;
     WebCore::FloatRect targetVideoFrame = self.bounds;
-    CGAffineTransform transform = CGAffineTransformIdentity;
-    if (!sourceVideoFrame.isEmpty()) {
-        if ([self resizePreservingGravity]) {
-            auto scale = std::fmax(targetVideoFrame.width() / sourceVideoFrame.width(), targetVideoFrame.height() / sourceVideoFrame.height());
-            transform = CGAffineTransformMakeScale(scale, scale);
-        } else
-            transform = CGAffineTransformMakeScale(targetVideoFrame.width() / sourceVideoFrame.width(), targetVideoFrame.height() / sourceVideoFrame.height());
+
+    if (sourceVideoFrame == targetVideoFrame && CGAffineTransformIsIdentity(self.affineTransform))
+        return;
+
+    if (sourceVideoFrame.isEmpty()) {
+        // The initial resize will have an empty videoLayerFrame, which makes
+        // the subsequent calculations incorrect. When this happens, just do
+        // the synchronous resize step instead.
+        [self resolveBounds];
+        return;
     }
 
+    CGAffineTransform transform = CGAffineTransformIdentity;
+    if ([self resizePreservingGravity]) {
+        auto scale = std::fmax(targetVideoFrame.width() / sourceVideoFrame.width(), targetVideoFrame.height() / sourceVideoFrame.height());
+        transform = CGAffineTransformMakeScale(scale, scale);
+    } else
+        transform = CGAffineTransformMakeScale(targetVideoFrame.width() / sourceVideoFrame.width(), targetVideoFrame.height() / sourceVideoFrame.height());
+
     auto* videoSublayer = [sublayers objectAtIndex:0];
     [CATransaction begin];
     [CATransaction setDisableActions:YES];