Unreviewed, reverting r280488.

https://bugs.webkit.org/show_bug.cgi?id=228677

media/modern-media-controls/ tests are failing or timed-out

Reverted changeset:

"[Cocoa] Remove support for AVAssetImageGenerator"
https://bugs.webkit.org/show_bug.cgi?id=228560
https://commits.webkit.org/r280488

Canonical link: https://commits.webkit.org/240143@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@280511 268f45cc-cd09-0410-ab3c-d52691b4dbfc

LayoutTests/imported/w3c/ChangeLog

@@ -1,3 +1,16 @@
+2021-07-31  Commit Queue  <commit-queue@webkit.org>
+
+        Unreviewed, reverting r280488.
+        https://bugs.webkit.org/show_bug.cgi?id=228677
+
+        media/modern-media-controls/ tests are failing or timed-out
+
+        Reverted changeset:
+
+        "[Cocoa] Remove support for AVAssetImageGenerator"
+        https://bugs.webkit.org/show_bug.cgi?id=228560
+        https://commits.webkit.org/r280488
+
 2021-07-30  Rob Buis  <rbuis@igalia.com>
 
         [css-overflow] Implement clip value for overflow

LayoutTests/imported/w3c/web-platform-tests/html/rendering/replaced-elements/attributes-for-embedded-content-and-images/video-aspect-ratio.html

@@ -26,7 +26,7 @@
 
 t.step(function() {
   var video = document.getElementById("contained");
-  video.src = getVideoURI('/media/16x16-green');
+  video.src = getVideoURI('/media/2x2-green');
   assert_ratio(video, 2.5);
 }, "contain:size aspect ratio");
 
@@ -37,7 +37,7 @@
   video = document.createElement("video");
   video.setAttribute("width", "250");
   video.setAttribute("height", "100");
-  video.src = getVideoURI('/media/16x16-green');
+  video.src = getVideoURI('/media/2x2-green');
   document.body.appendChild(video);
   // Videos default to a size of 300x150px and calculate their aspect ratio
   // based on that before the video is loaded. So this should be 2, ignoring

LayoutTests/imported/w3c/web-platform-tests/html/semantics/embedded-content/media-elements/video_loop_base.html

@@ -33,7 +33,7 @@
 
         media.addEventListener("seeking", startTest, false);
         media.loop = true;
-        media.src = getVideoURI("/media/16x16-green") + "?" + new Date() + Math.random();
+        media.src = getVideoURI("/media/2x2-green") + "?" + new Date() + Math.random();
         media.play();
     </script>
   </body>

Source/WebCore/ChangeLog

@@ -1,3 +1,16 @@
+2021-07-31  Commit Queue  <commit-queue@webkit.org>
+
+        Unreviewed, reverting r280488.
+        https://bugs.webkit.org/show_bug.cgi?id=228677
+
+        media/modern-media-controls/ tests are failing or timed-out
+
+        Reverted changeset:
+
+        "[Cocoa] Remove support for AVAssetImageGenerator"
+        https://bugs.webkit.org/show_bug.cgi?id=228560
+        https://commits.webkit.org/r280488
+
 2021-07-30  Rob Buis  <rbuis@igalia.com>
 
         [css-overflow] Implement clip value for overflow

Source/WebCore/platform/graphics/avfoundation/MediaPlayerPrivateAVFoundation.cpp

@@ -543,16 +543,11 @@ void MediaPlayerPrivateAVFoundation::updateStates()
         }
     }
 
-    if (newReadyState >= MediaPlayer::ReadyState::HaveCurrentData && m_cachedHasVideo && !hasAvailableVideoFrame()) {
-        newReadyState = MediaPlayer::ReadyState::HaveMetadata;
-        ALWAYS_LOG(LOGIDENTIFIER, "!hasAvailableVideoFrame(), lowering readyState to ", newReadyState);
-    }
-
     if (isReadyForVideoSetup() && currentRenderingMode() != preferredRenderingMode())
         setUpVideoRendering();
 
     if (!m_haveReportedFirstVideoFrame && m_cachedHasVideo && hasAvailableVideoFrame()) {
-        if (newReadyState < MediaPlayer::ReadyState::HaveCurrentData)
+        if (m_readyState < MediaPlayer::ReadyState::HaveCurrentData)
             newReadyState = MediaPlayer::ReadyState::HaveCurrentData;
         m_haveReportedFirstVideoFrame = true;
         m_player->firstVideoFrameAvailable();

Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateAVFoundationObjC.h

@@ -118,8 +118,6 @@ class MediaPlayerPrivateAVFoundationObjC final : public MediaPlayerPrivateAVFoun
 
     MediaTime currentMediaTime() const final;
 
-    void outputMediaDataWillChange();
-
 private:
 #if ENABLE(ENCRYPTED_MEDIA)
     void cdmInstanceAttached(CDMInstance&) final;

Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateAVFoundationObjC.mm

@@ -67,6 +67,7 @@
 #import "WebCoreCALayerExtras.h"
 #import "WebCoreNSURLExtras.h"
 #import "WebCoreNSURLSession.h"
+#import <AVFoundation/AVAssetImageGenerator.h>
 #import <AVFoundation/AVAssetTrack.h>
 #import <AVFoundation/AVMediaSelectionGroup.h>
 #import <AVFoundation/AVMetadataItem.h>
@@ -195,7 +196,6 @@ - (BOOL)resourceLoader:(AVAssetResourceLoader *)resourceLoader shouldWaitForLoad
 @interface WebCoreAVFPullDelegate : NSObject<AVPlayerItemOutputPullDelegate> {
     BinarySemaphore m_semaphore;
 }
-- (void)setParent:(WebCore::MediaPlayerPrivateAVFoundationObjC&)parent;
 - (void)outputMediaDataWillChange:(AVPlayerItemOutput *)sender;
 - (void)outputSequenceWasFlushed:(AVPlayerItemOutput *)output;
 
@@ -561,17 +561,43 @@ static WallTime toSystemClockTime(NSDate *date)
 
 bool MediaPlayerPrivateAVFoundationObjC::hasContextRenderer() const
 {
-    return m_videoOutput;
+    return m_videoOutput || m_imageGenerator;
 }
 
 void MediaPlayerPrivateAVFoundationObjC::createContextVideoRenderer()
 {
     createVideoOutput();
 }
 
+void MediaPlayerPrivateAVFoundationObjC::createImageGenerator()
+{
+    INFO_LOG(LOGIDENTIFIER);
+
+    if (!m_avAsset || m_imageGenerator)
+        return;
+
+    m_imageGenerator = [PAL::getAVAssetImageGeneratorClass() assetImageGeneratorWithAsset:m_avAsset.get()];
+
+    [m_imageGenerator.get() setApertureMode:AVAssetImageGeneratorApertureModeCleanAperture];
+    [m_imageGenerator.get() setAppliesPreferredTrackTransform:YES];
+    [m_imageGenerator.get() setRequestedTimeToleranceBefore:PAL::kCMTimeZero];
+    [m_imageGenerator.get() setRequestedTimeToleranceAfter:PAL::kCMTimeZero];
+}
+
 void MediaPlayerPrivateAVFoundationObjC::destroyContextVideoRenderer()
 {
     destroyVideoOutput();
+    destroyImageGenerator();
+}
+
+void MediaPlayerPrivateAVFoundationObjC::destroyImageGenerator()
+{
+    if (!m_imageGenerator)
+        return;
+
+    INFO_LOG(LOGIDENTIFIER);
+
+    m_imageGenerator = 0;
 }
 
 void MediaPlayerPrivateAVFoundationObjC::createVideoLayer()
@@ -1787,6 +1813,8 @@ static URL conformFragmentIdentifierForURL(const URL& url)
     // the video output, too.
     if (videoOutputHasAvailableFrame() || (m_videoOutput && m_lastPixelBuffer))
         paintWithVideoOutput(context, rect);
+    else
+        paintWithImageGenerator(context, rect);
 
     END_BLOCK_OBJC_EXCEPTIONS
     setDelayCallbacks(false);
@@ -1810,6 +1838,38 @@ static URL conformFragmentIdentifierForURL(const URL& url)
     paintCurrentFrameInContext(context, rect);
 }
 
+void MediaPlayerPrivateAVFoundationObjC::paintWithImageGenerator(GraphicsContext& context, const FloatRect& rect)
+{
+    INFO_LOG(LOGIDENTIFIER);
+
+    RetainPtr<CGImageRef> image = createImageForTimeInRect(currentTime(), rect);
+    if (image) {
+        GraphicsContextStateSaver stateSaver(context);
+        context.translate(rect.x(), rect.y() + rect.height());
+        context.scale(FloatSize(1.0f, -1.0f));
+        context.setImageInterpolationQuality(InterpolationQuality::Low);
+        IntRect paintRect(IntPoint(0, 0), IntSize(rect.width(), rect.height()));
+        CGContextDrawImage(context.platformContext(), CGRectMake(0, 0, paintRect.width(), paintRect.height()), image.get());
+    }
+}
+
+RetainPtr<CGImageRef> MediaPlayerPrivateAVFoundationObjC::createImageForTimeInRect(float time, const FloatRect& rect)
+{
+    if (!m_imageGenerator)
+        createImageGenerator();
+    ASSERT(m_imageGenerator);
+
+    MonotonicTime start = MonotonicTime::now();
+
+    [m_imageGenerator.get() setMaximumSize:CGSize(rect.size())];
+    RetainPtr<CGImageRef> rawImage = adoptCF([m_imageGenerator.get() copyCGImageAtTime:PAL::CMTimeMakeWithSeconds(time, 600) actualTime:nil error:nil]);
+    RetainPtr<CGImageRef> image = adoptCF(CGImageCreateCopyWithColorSpace(rawImage.get(), sRGBColorSpaceRef()));
+
+    INFO_LOG(LOGIDENTIFIER, "creating image took ", (MonotonicTime::now() - start).seconds());
+
+    return image;
+}
+
 void MediaPlayerPrivateAVFoundationObjC::getSupportedTypes(HashSet<String, ASCIICaseInsensitiveHash>& supportedTypes)
 {
     supportedTypes = AVAssetMIMETypeCache::singleton().supportedTypes();
@@ -2380,18 +2440,11 @@ void determineChangedTracksFromNewTracksAndOldItems(MediaSelectionGroupAVFObjC*
     }
 
     m_videoOutputDelegate = adoptNS([[WebCoreAVFPullDelegate alloc] init]);
-    [m_videoOutputDelegate setParent:*this];
     [m_videoOutput setDelegate:m_videoOutputDelegate.get() queue:globalPullDelegateQueue()];
-    [m_videoOutput requestNotificationOfMediaDataChangeWithAdvanceInterval:0];
 
     [m_avPlayerItem.get() addOutput:m_videoOutput.get()];
 }
 
-void MediaPlayerPrivateAVFoundationObjC::outputMediaDataWillChange()
-{
-    updateStates();
-}
-
 void MediaPlayerPrivateAVFoundationObjC::destroyVideoOutput()
 {
     if (!m_videoOutput)
@@ -2486,6 +2539,12 @@ void determineChangedTracksFromNewTracksAndOldItems(MediaSelectionGroupAVFObjC*
 
     FloatRect imageRect { FloatPoint::zero(), m_lastImage->size() };
     context.drawNativeImage(*m_lastImage, imageRect.size(), outputRect, imageRect);
+
+    // If we have created an AVAssetImageGenerator in the past due to m_videoOutput not having an available
+    // video frame, destroy it now that it is no longer needed.
+    if (m_imageGenerator)
+        destroyImageGenerator();
+
 }
 
 RetainPtr<CVPixelBufferRef> MediaPlayerPrivateAVFoundationObjC::pixelBufferForCurrentTime()
@@ -3840,25 +3899,14 @@ - (void)resourceLoader:(AVAssetResourceLoader *)resourceLoader didCancelLoadingR
 
 @end
 
-@implementation WebCoreAVFPullDelegate {
-    WeakPtr<WebCore::MediaPlayerPrivateAVFoundationObjC> _parent;
-}
+@implementation WebCoreAVFPullDelegate
 
 @synthesize semaphore = m_semaphore;
 
-- (void)setParent:(WebCore::MediaPlayerPrivateAVFoundationObjC&)parent
-{
-    _parent = makeWeakPtr(parent);
-}
-
 - (void)outputMediaDataWillChange:(AVPlayerItemVideoOutput *)output
 {
     UNUSED_PARAM(output);
     m_semaphore.signal();
-    RunLoop::main().dispatch([parent = _parent] {
-        if (parent)
-            parent->outputMediaDataWillChange();
-    });
 }
 
 - (void)outputSequenceWasFlushed:(AVPlayerItemVideoOutput *)output