Implement quality-of-service tiers in WebCoreDecompressionSession

https://bugs.webkit.org/show_bug.cgi?id=177769

Reviewed by Dean Jackson.

VTDecompressionSession will suggest quality-of-service tiers to be used when decompression
can't keep up with playback speed. Use a simple exponential-moving-average heuristic to
determine when to move up and down the tiers.

Drive-by fix: When frames are so late that they miss the display deadline, mark them as
dropped rather than just delayed.

* platform/graphics/cocoa/WebCoreDecompressionSession.h:
* platform/graphics/cocoa/WebCoreDecompressionSession.mm:
(WebCore::WebCoreDecompressionSession::ensureDecompressionSessionForSample):
(WebCore::WebCoreDecompressionSession::decodeSample):
(WebCore::WebCoreDecompressionSession::handleDecompressionOutput):
(WebCore::WebCoreDecompressionSession::automaticDequeue):
(WebCore::WebCoreDecompressionSession::enqueueDecodedSample):
(WebCore::WebCoreDecompressionSession::resetQosTier):
(WebCore::WebCoreDecompressionSession::increaseQosTier):
(WebCore::WebCoreDecompressionSession::decreaseQosTier):
(WebCore::WebCoreDecompressionSession::updateQosWithDecodeTimeStatistics):
* platform/cocoa/VideoToolboxSoftLink.cpp:
* platform/cocoa/VideoToolboxSoftLink.h:


Canonical link: https://commits.webkit.org/194082@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@222803 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Source/WebCore/ChangeLog

@@ -1,3 +1,31 @@
+2017-10-03  Jer Noble  <jer.noble@apple.com>
+
+        Implement quality-of-service tiers in WebCoreDecompressionSession
+        https://bugs.webkit.org/show_bug.cgi?id=177769
+
+        Reviewed by Dean Jackson.
+
+        VTDecompressionSession will suggest quality-of-service tiers to be used when decompression
+        can't keep up with playback speed. Use a simple exponential-moving-average heuristic to
+        determine when to move up and down the tiers.
+
+        Drive-by fix: When frames are so late that they miss the display deadline, mark them as
+        dropped rather than just delayed.
+
+        * platform/graphics/cocoa/WebCoreDecompressionSession.h:
+        * platform/graphics/cocoa/WebCoreDecompressionSession.mm:
+        (WebCore::WebCoreDecompressionSession::ensureDecompressionSessionForSample):
+        (WebCore::WebCoreDecompressionSession::decodeSample):
+        (WebCore::WebCoreDecompressionSession::handleDecompressionOutput):
+        (WebCore::WebCoreDecompressionSession::automaticDequeue):
+        (WebCore::WebCoreDecompressionSession::enqueueDecodedSample):
+        (WebCore::WebCoreDecompressionSession::resetQosTier):
+        (WebCore::WebCoreDecompressionSession::increaseQosTier):
+        (WebCore::WebCoreDecompressionSession::decreaseQosTier):
+        (WebCore::WebCoreDecompressionSession::updateQosWithDecodeTimeStatistics):
+        * platform/cocoa/VideoToolboxSoftLink.cpp:
+        * platform/cocoa/VideoToolboxSoftLink.h:
+
 2017-10-03  Adrian Perez de Castro  <aperez@igalia.com>
 
         [GTK] Support the "system" CSS font family

Source/WebCore/platform/cocoa/VideoToolboxSoftLink.cpp

@@ -33,6 +33,7 @@ typedef struct OpaqueVTImageRotationSession* VTImageRotationSessionRef;
 SOFT_LINK_FRAMEWORK_FOR_SOURCE(WebCore, VideoToolbox)
 
 SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, VideoToolbox, VTSessionCopyProperty, OSStatus, (VTSessionRef session, CFStringRef propertyKey, CFAllocatorRef allocator, void* propertyValueOut), (session, propertyKey, allocator, propertyValueOut))
+SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, VideoToolbox, VTSessionSetProperties, OSStatus, (VTSessionRef session, CFDictionaryRef propertyDictionary), (session, propertyDictionary))
 SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, VideoToolbox, VTDecompressionSessionCreate, OSStatus, (CFAllocatorRef allocator, CMVideoFormatDescriptionRef videoFormatDescription, CFDictionaryRef videoDecoderSpecification, CFDictionaryRef destinationImageBufferAttributes, const VTDecompressionOutputCallbackRecord* outputCallback, VTDecompressionSessionRef* decompressionSessionOut), (allocator, videoFormatDescription, videoDecoderSpecification, destinationImageBufferAttributes, outputCallback, decompressionSessionOut))
 SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, VideoToolbox, VTDecompressionSessionCanAcceptFormatDescription, Boolean, (VTDecompressionSessionRef session, CMFormatDescriptionRef newFormatDesc), (session, newFormatDesc))
 SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, VideoToolbox, VTDecompressionSessionWaitForAsynchronousFrames, OSStatus, (VTDecompressionSessionRef session), (session))
@@ -45,6 +46,7 @@ SOFT_LINK_FUNCTION_MAY_FAIL_FOR_SOURCE(WebCore, VideoToolbox, VTGetGVADecoderAva
 SOFT_LINK_FUNCTION_MAY_FAIL_FOR_SOURCE(WebCore, VideoToolbox, VTCreateCGImageFromCVPixelBuffer, OSStatus, (CVPixelBufferRef pixelBuffer, CFDictionaryRef options, CGImageRef* imageOut), (pixelBuffer, options, imageOut))
 SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, VideoToolbox, kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder, CFStringRef)
 SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, VideoToolbox, kVTDecompressionPropertyKey_PixelBufferPool, CFStringRef)
+SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, VideoToolbox, kVTDecompressionPropertyKey_SuggestedQualityOfServiceTiers, CFStringRef)
 SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, VideoToolbox, kVTImageRotationPropertyKey_EnableHighSpeedTransfer, CFStringRef)
 SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, VideoToolbox, kVTImageRotationPropertyKey_FlipHorizontalOrientation, CFStringRef)
 SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, VideoToolbox, kVTImageRotationPropertyKey_FlipVerticalOrientation, CFStringRef)

Source/WebCore/platform/cocoa/VideoToolboxSoftLink.h

@@ -34,6 +34,8 @@ SOFT_LINK_FRAMEWORK_FOR_HEADER(WebCore, VideoToolbox)
 
 SOFT_LINK_FUNCTION_FOR_HEADER(WebCore, VideoToolbox, VTSessionCopyProperty, OSStatus, (VTSessionRef session, CFStringRef propertyKey, CFAllocatorRef allocator, void* propertyValueOut), (session, propertyKey, allocator, propertyValueOut))
 #define VTSessionCopyProperty softLink_VideoToolbox_VTSessionCopyProperty
+SOFT_LINK_FUNCTION_FOR_HEADER(WebCore, VideoToolbox, VTSessionSetProperties, OSStatus, (VTSessionRef session, CFDictionaryRef propertyDictionary), (session, propertyDictionary))
+#define VTSessionSetProperties softLink_VideoToolbox_VTSessionSetProperties
 SOFT_LINK_FUNCTION_FOR_HEADER(WebCore, VideoToolbox, VTDecompressionSessionCreate, OSStatus, (CFAllocatorRef allocator, CMVideoFormatDescriptionRef videoFormatDescription, CFDictionaryRef videoDecoderSpecification, CFDictionaryRef destinationImageBufferAttributes, const VTDecompressionOutputCallbackRecord* outputCallback, VTDecompressionSessionRef* decompressionSessionOut), (allocator, videoFormatDescription, videoDecoderSpecification, destinationImageBufferAttributes, outputCallback, decompressionSessionOut))
 #define VTDecompressionSessionCreate softLink_VideoToolbox_VTDecompressionSessionCreate
 SOFT_LINK_FUNCTION_FOR_HEADER(WebCore, VideoToolbox, VTDecompressionSessionCanAcceptFormatDescription, Boolean, (VTDecompressionSessionRef session, CMFormatDescriptionRef newFormatDesc), (session, newFormatDesc))
@@ -58,6 +60,8 @@ SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, VideoToolbox, kVTVideoDecoderSpecificatio
 #define kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder get_VideoToolbox_kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder()
 SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, VideoToolbox, kVTDecompressionPropertyKey_PixelBufferPool, CFStringRef)
 #define kVTDecompressionPropertyKey_PixelBufferPool get_VideoToolbox_kVTDecompressionPropertyKey_PixelBufferPool()
+SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, VideoToolbox, kVTDecompressionPropertyKey_SuggestedQualityOfServiceTiers, CFStringRef)
+#define kVTDecompressionPropertyKey_SuggestedQualityOfServiceTiers get_VideoToolbox_kVTDecompressionPropertyKey_SuggestedQualityOfServiceTiers()
 SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, VideoToolbox, kVTImageRotationPropertyKey_EnableHighSpeedTransfer, CFStringRef)
 #define kVTImageRotationPropertyKey_EnableHighSpeedTransfer get_VideoToolbox_kVTImageRotationPropertyKey_EnableHighSpeedTransfer()
 SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, VideoToolbox, kVTImageRotationPropertyKey_FlipHorizontalOrientation, CFStringRef)

Source/WebCore/platform/graphics/cocoa/WebCoreDecompressionSession.h

@@ -37,6 +37,7 @@
 #include <wtf/ThreadSafeRefCounted.h>
 
 typedef CFTypeRef CMBufferRef;
+typedef const struct __CFArray * CFArrayRef;
 typedef struct opaqueCMBufferQueue *CMBufferQueueRef;
 typedef struct opaqueCMSampleBuffer *CMSampleBufferRef;
 typedef struct OpaqueCMTimebase* CMTimebaseRef;
@@ -100,6 +101,11 @@ class WebCoreDecompressionSession : public ThreadSafeRefCounted<WebCoreDecompres
     static CFComparisonResult compareBuffers(CMBufferRef buf1, CMBufferRef buf2, void* refcon);
     void maybeBecomeReadyForMoreMediaData();
 
+    void resetQosTier();
+    void increaseQosTier();
+    void decreaseQosTier();
+    void updateQosWithDecodeTimeStatistics(double ratio);
+
     static const CMItemCount kMaximumCapacity = 120;
     static const CMItemCount kHighWaterMark = 60;
     static const CMItemCount kLowWaterMark = 15;
@@ -115,6 +121,10 @@ class WebCoreDecompressionSession : public ThreadSafeRefCounted<WebCoreDecompres
     OSObjectPtr<dispatch_source_t> m_timerSource;
     std::function<void()> m_notificationCallback;
     std::function<void()> m_hasAvailableFrameCallback;
+    RetainPtr<CFArrayRef> m_qosTiers;
+    long m_currentQosTier { 0 };
+    unsigned long m_framesSinceLastQosCheck { 0 };
+    double m_decodeRatioMovingAverage { 0 };
 
     bool m_invalidated { false };
     int m_framesBeingDecoded { 0 };

Source/WebCore/platform/graphics/cocoa/WebCoreDecompressionSession.mm

@@ -33,6 +33,7 @@
 #import <CoreMedia/CMBufferQueue.h>
 #import <CoreMedia/CMFormatDescription.h>
 #import <pal/avfoundation/MediaTimeAVFoundation.h>
+#import <wtf/CurrentTime.h>
 #import <wtf/MainThread.h>
 #import <wtf/MediaTime.h>
 #import <wtf/StringPrintStream.h>
@@ -228,8 +229,14 @@
             attributes = @{(NSString *)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_32BGRA)};
         }
         VTDecompressionSessionRef decompressionSessionOut = nullptr;
-        if (noErr == VTDecompressionSessionCreate(kCFAllocatorDefault, videoFormatDescription, (CFDictionaryRef)videoDecoderSpecification, (CFDictionaryRef)attributes, nullptr, &decompressionSessionOut))
+        if (noErr == VTDecompressionSessionCreate(kCFAllocatorDefault, videoFormatDescription, (CFDictionaryRef)videoDecoderSpecification, (CFDictionaryRef)attributes, nullptr, &decompressionSessionOut)) {
             m_decompressionSession = adoptCF(decompressionSessionOut);
+            CFArrayRef rawSuggestedQualityOfServiceTiers = nullptr;
+            VTSessionCopyProperty(decompressionSessionOut, kVTDecompressionPropertyKey_SuggestedQualityOfServiceTiers, kCFAllocatorDefault, &rawSuggestedQualityOfServiceTiers);
+            m_qosTiers = adoptCF(rawSuggestedQualityOfServiceTiers);
+            m_currentQosTier = 0;
+            resetQosTier();
+        }
     }
 }
 
@@ -252,7 +259,11 @@
         return;
     }
 
-    VTDecompressionSessionDecodeFrameWithOutputHandler(m_decompressionSession.get(), sample, flags, nullptr, [this, displaying] (OSStatus status, VTDecodeInfoFlags infoFlags, CVImageBufferRef imageBuffer, CMTime presentationTimeStamp, CMTime presentationDuration) {
+    double startTime = monotonicallyIncreasingTime();
+    VTDecompressionSessionDecodeFrameWithOutputHandler(m_decompressionSession.get(), sample, flags, nullptr, [this, displaying, startTime](OSStatus status, VTDecodeInfoFlags infoFlags, CVImageBufferRef imageBuffer, CMTime presentationTimeStamp, CMTime presentationDuration) {
+        double deltaRatio = (monotonicallyIncreasingTime() - startTime) / CMTimeGetSeconds(presentationDuration);
+
+        updateQosWithDecodeTimeStatistics(deltaRatio);
         handleDecompressionOutput(displaying, status, infoFlags, imageBuffer, presentationTimeStamp, presentationDuration);
     });
 }
@@ -302,13 +313,6 @@
         return;
     }
 
-    if (displaying && m_timebase) {
-        auto currentTime = CMTimebaseGetTime(m_timebase.get());
-        auto currentRate = CMTimebaseGetRate(m_timebase.get());
-        if (currentRate > 0 && CMTimeCompare(presentationTimeStamp, currentTime) < 0)
-            m_totalFrameDelay += PAL::toMediaTime(CMTimeSubtract(currentTime, presentationTimeStamp));
-    }
-
     dispatch_async(m_enqueingQueue.get(), [protectedThis = makeRefPtr(this), status, imageSampleBuffer = adoptCF(rawImageSampleBuffer), infoFlags, displaying] {
         UNUSED_PARAM(infoFlags);
         protectedThis->enqueueDecodedSample(imageSampleBuffer.get(), displaying);
@@ -363,7 +367,7 @@
     if (releasedImageBuffers)
         maybeBecomeReadyForMoreMediaData();
 
-    LOG(Media, "WebCoreDecompressionSession::automaticDequeue(%p) - queue empty", this, toString(time).utf8().data());
+    LOG(Media, "WebCoreDecompressionSession::automaticDequeue(%p) - queue empty", this);
     CMTimebaseSetTimerDispatchSourceNextFireTime(m_timebase.get(), m_timerSource.get(), PAL::toCMTime(nextFireTime), 0);
 }
 
@@ -379,6 +383,27 @@
         return;
     }
 
+    bool shouldNotify = true;
+
+    if (displaying && m_timebase) {
+        auto currentRate = CMTimebaseGetRate(m_timebase.get());
+        auto currentTime = PAL::toMediaTime(CMTimebaseGetTime(m_timebase.get()));
+        auto presentationStartTime = PAL::toMediaTime(CMSampleBufferGetPresentationTimeStamp(sample));
+        auto presentationEndTime = presentationStartTime + PAL::toMediaTime(CMSampleBufferGetDuration(sample));
+        if (currentTime < presentationStartTime || currentTime >= presentationEndTime)
+            shouldNotify = false;
+
+        if (currentRate > 0 && presentationEndTime < currentTime) {
+#if !LOG_DISABLED
+            auto begin = PAL::toMediaTime(CMBufferQueueGetFirstPresentationTimeStamp(m_producerQueue.get()));
+            auto end = PAL::toMediaTime(CMBufferQueueGetEndPresentationTimeStamp(m_producerQueue.get()));
+            LOG(Media, "WebCoreDecompressionSession::enqueueDecodedSample(%p) - dropping frame late by %s, framesBeingDecoded(%d), producerQueue(%s -> %s)", this, toString(presentationEndTime - currentTime).utf8().data(), m_framesBeingDecoded, toString(begin).utf8().data(), toString(end).utf8().data());
+#endif
+            ++m_droppedVideoFrames;
+            return;
+        }
+    }
+
     CMBufferQueueEnqueue(m_producerQueue.get(), sample);
 
 #if !LOG_DISABLED
@@ -394,13 +419,8 @@
     if (!m_hasAvailableFrameCallback)
         return;
 
-    if (m_timebase) {
-        auto currentTime = PAL::toMediaTime(CMTimebaseGetTime(m_timebase.get()));
-        auto presentationStartTime = PAL::toMediaTime(CMSampleBufferGetPresentationTimeStamp(sample));
-        auto presentationEndTime = presentationStartTime + PAL::toMediaTime(CMSampleBufferGetDuration(sample));
-        if (currentTime < presentationStartTime || currentTime >= presentationEndTime)
-            return;
-    }
+    if (!shouldNotify)
+        return;
 
     dispatch_async(dispatch_get_main_queue(), [protectedThis = makeRefPtr(this), callback = WTFMove(m_hasAvailableFrameCallback)] {
         callback();
@@ -504,6 +524,9 @@
         CMBufferQueueReset(protectedThis->m_producerQueue.get());
         dispatch_sync(protectedThis->m_enqueingQueue.get(), [protectedThis] {
             CMBufferQueueReset(protectedThis->m_consumerQueue.get());
+            protectedThis->m_framesSinceLastQosCheck = 0;
+            protectedThis->m_currentQosTier = 0;
+            protectedThis->resetQosTier();
         });
     });
 }
@@ -534,6 +557,71 @@
     return (CFComparisonResult)CMTimeCompare(getPresentationTime(buf1, refcon), getPresentationTime(buf2, refcon));
 }
 
+void WebCoreDecompressionSession::resetQosTier()
+{
+    if (!m_qosTiers || !m_decompressionSession)
+        return;
+
+    if (m_currentQosTier < 0 || m_currentQosTier >= CFArrayGetCount(m_qosTiers.get()))
+        return;
+
+    auto tier = (CFDictionaryRef)CFArrayGetValueAtIndex(m_qosTiers.get(), m_currentQosTier);
+    LOG(Media, "WebCoreDecompressionSession::resetQosTier(%p) - currentQosTier(%ld), tier(%@)", this, m_currentQosTier, [(NSDictionary *)tier description]);
+
+    VTSessionSetProperties(m_decompressionSession.get(), tier);
+    m_framesSinceLastQosCheck = 0;
+}
+
+void WebCoreDecompressionSession::increaseQosTier()
+{
+    if (!m_qosTiers)
+        return;
+
+    if (m_currentQosTier + 1 >= CFArrayGetCount(m_qosTiers.get()))
+        return;
+
+    ++m_currentQosTier;
+    resetQosTier();
+}
+
+void WebCoreDecompressionSession::decreaseQosTier()
+{
+    if (!m_qosTiers)
+        return;
+
+    if (m_currentQosTier <= 0)
+        return;
+
+    --m_currentQosTier;
+    resetQosTier();
+}
+
+void WebCoreDecompressionSession::updateQosWithDecodeTimeStatistics(double ratio)
+{
+    static const double kMovingAverageAlphaValue = 0.1;
+    static const unsigned kNumberOfFramesBeforeSwitchingTiers = 60;
+    static const double kHighWaterDecodeRatio = 1.;
+    static const double kLowWaterDecodeRatio = 0.5;
+
+    if (!m_timebase)
+        return;
+
+    double rate = CMTimebaseGetRate(m_timebase.get());
+    if (!rate)
+        rate = 1;
+
+    m_decodeRatioMovingAverage += kMovingAverageAlphaValue * (ratio - m_decodeRatioMovingAverage) * rate;
+    if (++m_framesSinceLastQosCheck < kNumberOfFramesBeforeSwitchingTiers)
+        return;
+
+    LOG(Media, "WebCoreDecompressionSession::updateQosWithDecodeTimeStatistics(%p) - framesSinceLastQosCheck(%ld), decodeRatioMovingAverage(%g)", this, m_framesSinceLastQosCheck, m_decodeRatioMovingAverage);
+    if (m_decodeRatioMovingAverage > kHighWaterDecodeRatio)
+        increaseQosTier();
+    else if (m_decodeRatioMovingAverage < kLowWaterDecodeRatio)
+        decreaseQosTier();
+    m_framesSinceLastQosCheck = 0;
+}
+
 }
 
 #endif