getDisplayMedia captured windows get black frames when window gets re…

…sized

https://bugs.webkit.org/show_bug.cgi?id=270568
rdar://124131045

Reviewed by Eric Carlson.

When a window size changes, we may need to update the capture size so as to respect aspect ratio and not introduce black frames.
For that purpose, we store the content size in ScreenCaptureKitCaptureSource::m_contentSize and we initialize it at start up using SCContentFilter.

Then, when capture has started, we get the content size from the sample buffer attachment, using SCStreamFrameInfoScaleFactor, SCStreamFrameInfoContentScale and SCStreamFrameInfoContentRect.
If m_contentSize changes, we update the capture size.
We ensure that the capture size keeps the aspect ratio even in case where width and height are set via media constraints by either computing width from height or the reverse but keeping aspect ratio computed from m_contentSize.

* Source/WebCore/PAL/pal/mac/ScreenCaptureKitSoftLink.h:
* Source/WebCore/PAL/pal/mac/ScreenCaptureKitSoftLink.mm:
* Source/WebCore/platform/mediastream/mac/ScreenCaptureKitCaptureSource.h:
* Source/WebCore/platform/mediastream/mac/ScreenCaptureKitCaptureSource.mm:
(WebCore::ScreenCaptureKitCaptureSource::streamConfiguration):
(WebCore::ScreenCaptureKitCaptureSource::startContentStream):
(WebCore::ScreenCaptureKitCaptureSource::streamDidOutputVideoSampleBuffer):

Canonical link: https://commits.webkit.org/275794@main

Source/WebCore/PAL/pal/mac/ScreenCaptureKitSoftLink.h

@@ -45,5 +45,11 @@ SOFT_LINK_CLASS_FOR_HEADER(PAL, SCContentSharingPickerConfiguration)
 
 SOFT_LINK_CONSTANT_FOR_HEADER(PAL, ScreenCaptureKit, SCStreamFrameInfoStatus, NSString *)
 #define SCStreamFrameInfoStatus PAL::get_ScreenCaptureKit_SCStreamFrameInfoStatus()
+SOFT_LINK_CONSTANT_FOR_HEADER(PAL, ScreenCaptureKit, SCStreamFrameInfoScaleFactor, NSString *)
+#define SCStreamFrameInfoScaleFactor PAL::get_ScreenCaptureKit_SCStreamFrameInfoScaleFactor()
+SOFT_LINK_CONSTANT_FOR_HEADER(PAL, ScreenCaptureKit, SCStreamFrameInfoContentScale, NSString *)
+#define SCStreamFrameInfoContentScale PAL::get_ScreenCaptureKit_SCStreamFrameInfoContentScale()
+SOFT_LINK_CONSTANT_FOR_HEADER(PAL, ScreenCaptureKit, SCStreamFrameInfoContentRect, NSString *)
+#define SCStreamFrameInfoContentRect PAL::get_ScreenCaptureKit_SCStreamFrameInfoContentRect()
 
 #endif // HAVE(SCREEN_CAPTURE_KIT)

Source/WebCore/PAL/pal/mac/ScreenCaptureKitSoftLink.mm

@@ -44,5 +44,8 @@
 #endif
 
 SOFT_LINK_CONSTANT_FOR_SOURCE_WITH_EXPORT(PAL, ScreenCaptureKit, SCStreamFrameInfoStatus, NSString *, PAL_EXPORT)
+SOFT_LINK_CONSTANT_FOR_SOURCE_WITH_EXPORT(PAL, ScreenCaptureKit, SCStreamFrameInfoScaleFactor, NSString *, PAL_EXPORT)
+SOFT_LINK_CONSTANT_FOR_SOURCE_WITH_EXPORT(PAL, ScreenCaptureKit, SCStreamFrameInfoContentScale, NSString *, PAL_EXPORT)
+SOFT_LINK_CONSTANT_FOR_SOURCE_WITH_EXPORT(PAL, ScreenCaptureKit, SCStreamFrameInfoContentRect, NSString *, PAL_EXPORT)
 
 #endif // PLATFORM(MAC)

Source/WebCore/platform/mediastream/mac/ScreenCaptureKitCaptureSource.h

@@ -111,6 +111,7 @@ class ScreenCaptureKitCaptureSource final
     uint32_t m_deviceID { 0 };
     mutable std::optional<IntSize> m_intrinsicSize;
 
+    FloatSize m_contentSize;
     uint32_t m_width { 0 };
     uint32_t m_height { 0 };
     float m_frameRate { 0 };

Source/WebCore/platform/mediastream/mac/ScreenCaptureKitCaptureSource.mm

@@ -323,21 +323,15 @@ - (void)stream:(SCStream *)stream didOutputSampleBuffer:(CMSampleBufferRef)sampl
     auto width = m_width;
     auto height = m_height;
 
-#if HAVE(SC_CONTENT_SHARING_PICKER)
-    if (m_contentFilter) {
-        FloatSize contentSize = FloatSize { m_contentFilter.get().contentRect.size };
-        contentSize.scale(m_contentFilter.get().pointPixelScale);
-        if (!width && !height) {
-            width = contentSize.width();
-            height = contentSize.height();
-        } else if (contentSize.width() && contentSize.height()) {
-            if (!width)
-                width = height * contentSize.width() / contentSize.height();
-            else if (!height)
-                height = width * contentSize.height() / contentSize.width();
-        }
+    if (!width && !height) {
+        width = m_contentSize.width();
+        height = m_contentSize.height();
+    } else if (!m_contentSize.isEmpty()) {
+        if (!width)
+            width = height * m_contentSize.aspectRatio();
+        else
+            height = width / m_contentSize.aspectRatio();
     }
-#endif
 
     if (width && height) {
         [m_streamConfiguration setWidth:width];
@@ -361,6 +355,11 @@ - (void)stream:(SCStream *)stream didOutputSampleBuffer:(CMSampleBufferRef)sampl
         auto filterAndSession = ScreenCaptureKitSharingSessionManager::singleton().contentFilterAndSharingSessionFromCaptureDevice(m_captureDevice);
         m_contentFilter = WTFMove(filterAndSession.first);
         m_sharingSession = WTFMove(filterAndSession.second);
+
+#if HAVE(SC_CONTENT_SHARING_PICKER)
+        m_contentSize = FloatSize { m_contentFilter.get().contentRect.size };
+        m_contentSize.scale(m_contentFilter.get().pointPixelScale);
+#endif
     }
 
 #if HAVE(SC_CONTENT_SHARING_PICKER)
@@ -491,7 +490,23 @@ - (void)stream:(SCStream *)stream didOutputSampleBuffer:(CMSampleBufferRef)sampl
 
     auto attachments = (__bridge NSArray *)PAL::CMSampleBufferGetSampleAttachmentsArray(sampleBuffer.get(), false);
     SCFrameStatus status = SCFrameStatusStopped;
+
+    double contentScale = 1;
+    double scaleFactor = 1;
+    FloatSize contentSize;
     [attachments enumerateObjectsUsingBlock:makeBlockPtr([&] (NSDictionary *attachment, NSUInteger, BOOL *stop) {
+        if (auto scaleFactorNumber = (NSNumber *)attachment[SCStreamFrameInfoScaleFactor])
+            scaleFactor = [scaleFactorNumber floatValue];
+
+        if (auto contentScaleNumber = (NSNumber *)attachment[SCStreamFrameInfoContentScale])
+            contentScale = [contentScaleNumber floatValue];
+
+        if (auto contentRectDictionary = (CFDictionaryRef)attachment[SCStreamFrameInfoContentRect]) {
+            CGRect contentRect;
+            if (CGRectMakeWithDictionaryRepresentation(contentRectDictionary, &contentRect))
+                contentSize = FloatSize { contentRect.size };
+        }
+
         auto statusNumber = (NSNumber *)attachment[SCStreamFrameInfoStatus];
         if (!statusNumber)
             return;
@@ -513,6 +528,17 @@ - (void)stream:(SCStream *)stream didOutputSampleBuffer:(CMSampleBufferRef)sampl
 
     m_currentFrame = WTFMove(sampleBuffer);
 
+    if (scaleFactor != 1)
+        contentSize.scale(scaleFactor);
+    if (contentScale && contentScale != 1)
+        contentSize.scale(1 / contentScale);
+
+    if (m_contentSize != contentSize) {
+        m_contentSize = contentSize;
+        m_streamConfiguration = nullptr;
+        updateStreamConfiguration();
+    }
+
     auto intrinsicSize = IntSize(PAL::CMVideoFormatDescriptionGetPresentationDimensions(PAL::CMSampleBufferGetFormatDescription(m_currentFrame.get()), true, true));
     if (!m_intrinsicSize || *m_intrinsicSize != intrinsicSize) {
         m_intrinsicSize = intrinsicSize;