Skip to content
Permalink
Browse files
Implement quality-of-service tiers in WebCoreDecompressionSession
https://bugs.webkit.org/show_bug.cgi?id=177769

Reviewed by Dean Jackson.

VTDecompressionSession will suggest quality-of-service tiers to be used when decompression
can't keep up with playback speed. Use a simple exponential-moving-average heuristic to
determine when to move up and down the tiers.

Drive-by fix: When frames are so late that they miss the display deadline, mark them as
dropped rather than just delayed.

* platform/graphics/cocoa/WebCoreDecompressionSession.h:
* platform/graphics/cocoa/WebCoreDecompressionSession.mm:
(WebCore::WebCoreDecompressionSession::ensureDecompressionSessionForSample):
(WebCore::WebCoreDecompressionSession::decodeSample):
(WebCore::WebCoreDecompressionSession::handleDecompressionOutput):
(WebCore::WebCoreDecompressionSession::automaticDequeue):
(WebCore::WebCoreDecompressionSession::enqueueDecodedSample):
(WebCore::WebCoreDecompressionSession::resetQosTier):
(WebCore::WebCoreDecompressionSession::increaseQosTier):
(WebCore::WebCoreDecompressionSession::decreaseQosTier):
(WebCore::WebCoreDecompressionSession::updateQosWithDecodeTimeStatistics):
* platform/cocoa/VideoToolboxSoftLink.cpp:
* platform/cocoa/VideoToolboxSoftLink.h:


Canonical link: https://commits.webkit.org/194082@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@222803 268f45cc-cd09-0410-ab3c-d52691b4dbfc
  • Loading branch information
jernoble committed Oct 3, 2017
1 parent a2d968c commit f2741e1f0836d0dae8bce4d84445bdc2f363fa7d
Showing 5 changed files with 149 additions and 17 deletions.
@@ -1,3 +1,31 @@
2017-10-03 Jer Noble <jer.noble@apple.com>

Implement quality-of-service tiers in WebCoreDecompressionSession
https://bugs.webkit.org/show_bug.cgi?id=177769

Reviewed by Dean Jackson.

VTDecompressionSession will suggest quality-of-service tiers to be used when decompression
can't keep up with playback speed. Use a simple exponential-moving-average heuristic to
determine when to move up and down the tiers.

Drive-by fix: When frames are so late that they miss the display deadline, mark them as
dropped rather than just delayed.

* platform/graphics/cocoa/WebCoreDecompressionSession.h:
* platform/graphics/cocoa/WebCoreDecompressionSession.mm:
(WebCore::WebCoreDecompressionSession::ensureDecompressionSessionForSample):
(WebCore::WebCoreDecompressionSession::decodeSample):
(WebCore::WebCoreDecompressionSession::handleDecompressionOutput):
(WebCore::WebCoreDecompressionSession::automaticDequeue):
(WebCore::WebCoreDecompressionSession::enqueueDecodedSample):
(WebCore::WebCoreDecompressionSession::resetQosTier):
(WebCore::WebCoreDecompressionSession::increaseQosTier):
(WebCore::WebCoreDecompressionSession::decreaseQosTier):
(WebCore::WebCoreDecompressionSession::updateQosWithDecodeTimeStatistics):
* platform/cocoa/VideoToolboxSoftLink.cpp:
* platform/cocoa/VideoToolboxSoftLink.h:

2017-10-03 Adrian Perez de Castro <aperez@igalia.com>

[GTK] Support the "system" CSS font family
@@ -33,6 +33,7 @@ typedef struct OpaqueVTImageRotationSession* VTImageRotationSessionRef;
SOFT_LINK_FRAMEWORK_FOR_SOURCE(WebCore, VideoToolbox)

SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, VideoToolbox, VTSessionCopyProperty, OSStatus, (VTSessionRef session, CFStringRef propertyKey, CFAllocatorRef allocator, void* propertyValueOut), (session, propertyKey, allocator, propertyValueOut))
SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, VideoToolbox, VTSessionSetProperties, OSStatus, (VTSessionRef session, CFDictionaryRef propertyDictionary), (session, propertyDictionary))
SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, VideoToolbox, VTDecompressionSessionCreate, OSStatus, (CFAllocatorRef allocator, CMVideoFormatDescriptionRef videoFormatDescription, CFDictionaryRef videoDecoderSpecification, CFDictionaryRef destinationImageBufferAttributes, const VTDecompressionOutputCallbackRecord* outputCallback, VTDecompressionSessionRef* decompressionSessionOut), (allocator, videoFormatDescription, videoDecoderSpecification, destinationImageBufferAttributes, outputCallback, decompressionSessionOut))
SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, VideoToolbox, VTDecompressionSessionCanAcceptFormatDescription, Boolean, (VTDecompressionSessionRef session, CMFormatDescriptionRef newFormatDesc), (session, newFormatDesc))
SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, VideoToolbox, VTDecompressionSessionWaitForAsynchronousFrames, OSStatus, (VTDecompressionSessionRef session), (session))
@@ -45,6 +46,7 @@ SOFT_LINK_FUNCTION_MAY_FAIL_FOR_SOURCE(WebCore, VideoToolbox, VTGetGVADecoderAva
SOFT_LINK_FUNCTION_MAY_FAIL_FOR_SOURCE(WebCore, VideoToolbox, VTCreateCGImageFromCVPixelBuffer, OSStatus, (CVPixelBufferRef pixelBuffer, CFDictionaryRef options, CGImageRef* imageOut), (pixelBuffer, options, imageOut))
SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, VideoToolbox, kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder, CFStringRef)
SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, VideoToolbox, kVTDecompressionPropertyKey_PixelBufferPool, CFStringRef)
SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, VideoToolbox, kVTDecompressionPropertyKey_SuggestedQualityOfServiceTiers, CFStringRef)
SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, VideoToolbox, kVTImageRotationPropertyKey_EnableHighSpeedTransfer, CFStringRef)
SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, VideoToolbox, kVTImageRotationPropertyKey_FlipHorizontalOrientation, CFStringRef)
SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, VideoToolbox, kVTImageRotationPropertyKey_FlipVerticalOrientation, CFStringRef)
@@ -34,6 +34,8 @@ SOFT_LINK_FRAMEWORK_FOR_HEADER(WebCore, VideoToolbox)

SOFT_LINK_FUNCTION_FOR_HEADER(WebCore, VideoToolbox, VTSessionCopyProperty, OSStatus, (VTSessionRef session, CFStringRef propertyKey, CFAllocatorRef allocator, void* propertyValueOut), (session, propertyKey, allocator, propertyValueOut))
#define VTSessionCopyProperty softLink_VideoToolbox_VTSessionCopyProperty
SOFT_LINK_FUNCTION_FOR_HEADER(WebCore, VideoToolbox, VTSessionSetProperties, OSStatus, (VTSessionRef session, CFDictionaryRef propertyDictionary), (session, propertyDictionary))
#define VTSessionSetProperties softLink_VideoToolbox_VTSessionSetProperties
SOFT_LINK_FUNCTION_FOR_HEADER(WebCore, VideoToolbox, VTDecompressionSessionCreate, OSStatus, (CFAllocatorRef allocator, CMVideoFormatDescriptionRef videoFormatDescription, CFDictionaryRef videoDecoderSpecification, CFDictionaryRef destinationImageBufferAttributes, const VTDecompressionOutputCallbackRecord* outputCallback, VTDecompressionSessionRef* decompressionSessionOut), (allocator, videoFormatDescription, videoDecoderSpecification, destinationImageBufferAttributes, outputCallback, decompressionSessionOut))
#define VTDecompressionSessionCreate softLink_VideoToolbox_VTDecompressionSessionCreate
SOFT_LINK_FUNCTION_FOR_HEADER(WebCore, VideoToolbox, VTDecompressionSessionCanAcceptFormatDescription, Boolean, (VTDecompressionSessionRef session, CMFormatDescriptionRef newFormatDesc), (session, newFormatDesc))
@@ -58,6 +60,8 @@ SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, VideoToolbox, kVTVideoDecoderSpecificatio
#define kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder get_VideoToolbox_kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder()
SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, VideoToolbox, kVTDecompressionPropertyKey_PixelBufferPool, CFStringRef)
#define kVTDecompressionPropertyKey_PixelBufferPool get_VideoToolbox_kVTDecompressionPropertyKey_PixelBufferPool()
SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, VideoToolbox, kVTDecompressionPropertyKey_SuggestedQualityOfServiceTiers, CFStringRef)
#define kVTDecompressionPropertyKey_SuggestedQualityOfServiceTiers get_VideoToolbox_kVTDecompressionPropertyKey_SuggestedQualityOfServiceTiers()
SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, VideoToolbox, kVTImageRotationPropertyKey_EnableHighSpeedTransfer, CFStringRef)
#define kVTImageRotationPropertyKey_EnableHighSpeedTransfer get_VideoToolbox_kVTImageRotationPropertyKey_EnableHighSpeedTransfer()
SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, VideoToolbox, kVTImageRotationPropertyKey_FlipHorizontalOrientation, CFStringRef)
@@ -37,6 +37,7 @@
#include <wtf/ThreadSafeRefCounted.h>

typedef CFTypeRef CMBufferRef;
typedef const struct __CFArray * CFArrayRef;
typedef struct opaqueCMBufferQueue *CMBufferQueueRef;
typedef struct opaqueCMSampleBuffer *CMSampleBufferRef;
typedef struct OpaqueCMTimebase* CMTimebaseRef;
@@ -100,6 +101,11 @@ class WebCoreDecompressionSession : public ThreadSafeRefCounted<WebCoreDecompres
static CFComparisonResult compareBuffers(CMBufferRef buf1, CMBufferRef buf2, void* refcon);
void maybeBecomeReadyForMoreMediaData();

void resetQosTier();
void increaseQosTier();
void decreaseQosTier();
void updateQosWithDecodeTimeStatistics(double ratio);

static const CMItemCount kMaximumCapacity = 120;
static const CMItemCount kHighWaterMark = 60;
static const CMItemCount kLowWaterMark = 15;
@@ -115,6 +121,10 @@ class WebCoreDecompressionSession : public ThreadSafeRefCounted<WebCoreDecompres
OSObjectPtr<dispatch_source_t> m_timerSource;
std::function<void()> m_notificationCallback;
std::function<void()> m_hasAvailableFrameCallback;
RetainPtr<CFArrayRef> m_qosTiers;
long m_currentQosTier { 0 };
unsigned long m_framesSinceLastQosCheck { 0 };
double m_decodeRatioMovingAverage { 0 };

bool m_invalidated { false };
int m_framesBeingDecoded { 0 };
@@ -33,6 +33,7 @@
#import <CoreMedia/CMBufferQueue.h>
#import <CoreMedia/CMFormatDescription.h>
#import <pal/avfoundation/MediaTimeAVFoundation.h>
#import <wtf/CurrentTime.h>
#import <wtf/MainThread.h>
#import <wtf/MediaTime.h>
#import <wtf/StringPrintStream.h>
@@ -228,8 +229,14 @@
attributes = @{(NSString *)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_32BGRA)};
}
VTDecompressionSessionRef decompressionSessionOut = nullptr;
if (noErr == VTDecompressionSessionCreate(kCFAllocatorDefault, videoFormatDescription, (CFDictionaryRef)videoDecoderSpecification, (CFDictionaryRef)attributes, nullptr, &decompressionSessionOut))
if (noErr == VTDecompressionSessionCreate(kCFAllocatorDefault, videoFormatDescription, (CFDictionaryRef)videoDecoderSpecification, (CFDictionaryRef)attributes, nullptr, &decompressionSessionOut)) {
m_decompressionSession = adoptCF(decompressionSessionOut);
CFArrayRef rawSuggestedQualityOfServiceTiers = nullptr;
VTSessionCopyProperty(decompressionSessionOut, kVTDecompressionPropertyKey_SuggestedQualityOfServiceTiers, kCFAllocatorDefault, &rawSuggestedQualityOfServiceTiers);
m_qosTiers = adoptCF(rawSuggestedQualityOfServiceTiers);
m_currentQosTier = 0;
resetQosTier();
}
}
}

@@ -252,7 +259,11 @@
return;
}

VTDecompressionSessionDecodeFrameWithOutputHandler(m_decompressionSession.get(), sample, flags, nullptr, [this, displaying] (OSStatus status, VTDecodeInfoFlags infoFlags, CVImageBufferRef imageBuffer, CMTime presentationTimeStamp, CMTime presentationDuration) {
double startTime = monotonicallyIncreasingTime();
VTDecompressionSessionDecodeFrameWithOutputHandler(m_decompressionSession.get(), sample, flags, nullptr, [this, displaying, startTime](OSStatus status, VTDecodeInfoFlags infoFlags, CVImageBufferRef imageBuffer, CMTime presentationTimeStamp, CMTime presentationDuration) {
double deltaRatio = (monotonicallyIncreasingTime() - startTime) / CMTimeGetSeconds(presentationDuration);

updateQosWithDecodeTimeStatistics(deltaRatio);
handleDecompressionOutput(displaying, status, infoFlags, imageBuffer, presentationTimeStamp, presentationDuration);
});
}
@@ -302,13 +313,6 @@
return;
}

if (displaying && m_timebase) {
auto currentTime = CMTimebaseGetTime(m_timebase.get());
auto currentRate = CMTimebaseGetRate(m_timebase.get());
if (currentRate > 0 && CMTimeCompare(presentationTimeStamp, currentTime) < 0)
m_totalFrameDelay += PAL::toMediaTime(CMTimeSubtract(currentTime, presentationTimeStamp));
}

dispatch_async(m_enqueingQueue.get(), [protectedThis = makeRefPtr(this), status, imageSampleBuffer = adoptCF(rawImageSampleBuffer), infoFlags, displaying] {
UNUSED_PARAM(infoFlags);
protectedThis->enqueueDecodedSample(imageSampleBuffer.get(), displaying);
@@ -363,7 +367,7 @@
if (releasedImageBuffers)
maybeBecomeReadyForMoreMediaData();

LOG(Media, "WebCoreDecompressionSession::automaticDequeue(%p) - queue empty", this, toString(time).utf8().data());
LOG(Media, "WebCoreDecompressionSession::automaticDequeue(%p) - queue empty", this);
CMTimebaseSetTimerDispatchSourceNextFireTime(m_timebase.get(), m_timerSource.get(), PAL::toCMTime(nextFireTime), 0);
}

@@ -379,6 +383,27 @@
return;
}

bool shouldNotify = true;

if (displaying && m_timebase) {
auto currentRate = CMTimebaseGetRate(m_timebase.get());
auto currentTime = PAL::toMediaTime(CMTimebaseGetTime(m_timebase.get()));
auto presentationStartTime = PAL::toMediaTime(CMSampleBufferGetPresentationTimeStamp(sample));
auto presentationEndTime = presentationStartTime + PAL::toMediaTime(CMSampleBufferGetDuration(sample));
if (currentTime < presentationStartTime || currentTime >= presentationEndTime)
shouldNotify = false;

if (currentRate > 0 && presentationEndTime < currentTime) {
#if !LOG_DISABLED
auto begin = PAL::toMediaTime(CMBufferQueueGetFirstPresentationTimeStamp(m_producerQueue.get()));
auto end = PAL::toMediaTime(CMBufferQueueGetEndPresentationTimeStamp(m_producerQueue.get()));
LOG(Media, "WebCoreDecompressionSession::enqueueDecodedSample(%p) - dropping frame late by %s, framesBeingDecoded(%d), producerQueue(%s -> %s)", this, toString(presentationEndTime - currentTime).utf8().data(), m_framesBeingDecoded, toString(begin).utf8().data(), toString(end).utf8().data());
#endif
++m_droppedVideoFrames;
return;
}
}

CMBufferQueueEnqueue(m_producerQueue.get(), sample);

#if !LOG_DISABLED
@@ -394,13 +419,8 @@
if (!m_hasAvailableFrameCallback)
return;

if (m_timebase) {
auto currentTime = PAL::toMediaTime(CMTimebaseGetTime(m_timebase.get()));
auto presentationStartTime = PAL::toMediaTime(CMSampleBufferGetPresentationTimeStamp(sample));
auto presentationEndTime = presentationStartTime + PAL::toMediaTime(CMSampleBufferGetDuration(sample));
if (currentTime < presentationStartTime || currentTime >= presentationEndTime)
return;
}
if (!shouldNotify)
return;

dispatch_async(dispatch_get_main_queue(), [protectedThis = makeRefPtr(this), callback = WTFMove(m_hasAvailableFrameCallback)] {
callback();
@@ -504,6 +524,9 @@
CMBufferQueueReset(protectedThis->m_producerQueue.get());
dispatch_sync(protectedThis->m_enqueingQueue.get(), [protectedThis] {
CMBufferQueueReset(protectedThis->m_consumerQueue.get());
protectedThis->m_framesSinceLastQosCheck = 0;
protectedThis->m_currentQosTier = 0;
protectedThis->resetQosTier();
});
});
}
@@ -534,6 +557,71 @@
return (CFComparisonResult)CMTimeCompare(getPresentationTime(buf1, refcon), getPresentationTime(buf2, refcon));
}

void WebCoreDecompressionSession::resetQosTier()
{
if (!m_qosTiers || !m_decompressionSession)
return;

if (m_currentQosTier < 0 || m_currentQosTier >= CFArrayGetCount(m_qosTiers.get()))
return;

auto tier = (CFDictionaryRef)CFArrayGetValueAtIndex(m_qosTiers.get(), m_currentQosTier);
LOG(Media, "WebCoreDecompressionSession::resetQosTier(%p) - currentQosTier(%ld), tier(%@)", this, m_currentQosTier, [(NSDictionary *)tier description]);

VTSessionSetProperties(m_decompressionSession.get(), tier);
m_framesSinceLastQosCheck = 0;
}

void WebCoreDecompressionSession::increaseQosTier()
{
if (!m_qosTiers)
return;

if (m_currentQosTier + 1 >= CFArrayGetCount(m_qosTiers.get()))
return;

++m_currentQosTier;
resetQosTier();
}

void WebCoreDecompressionSession::decreaseQosTier()
{
if (!m_qosTiers)
return;

if (m_currentQosTier <= 0)
return;

--m_currentQosTier;
resetQosTier();
}

void WebCoreDecompressionSession::updateQosWithDecodeTimeStatistics(double ratio)
{
static const double kMovingAverageAlphaValue = 0.1;
static const unsigned kNumberOfFramesBeforeSwitchingTiers = 60;
static const double kHighWaterDecodeRatio = 1.;
static const double kLowWaterDecodeRatio = 0.5;

if (!m_timebase)
return;

double rate = CMTimebaseGetRate(m_timebase.get());
if (!rate)
rate = 1;

m_decodeRatioMovingAverage += kMovingAverageAlphaValue * (ratio - m_decodeRatioMovingAverage) * rate;
if (++m_framesSinceLastQosCheck < kNumberOfFramesBeforeSwitchingTiers)
return;

LOG(Media, "WebCoreDecompressionSession::updateQosWithDecodeTimeStatistics(%p) - framesSinceLastQosCheck(%ld), decodeRatioMovingAverage(%g)", this, m_framesSinceLastQosCheck, m_decodeRatioMovingAverage);
if (m_decodeRatioMovingAverage > kHighWaterDecodeRatio)
increaseQosTier();
else if (m_decodeRatioMovingAverage < kLowWaterDecodeRatio)
decreaseQosTier();
m_framesSinceLastQosCheck = 0;
}

}

#endif

0 comments on commit f2741e1

Please sign in to comment.