Skip to content
Permalink
Browse files
Unreviewed, reverting r280488.
https://bugs.webkit.org/show_bug.cgi?id=228677

media/modern-media-controls/ tests are failing or timed-out

Reverted changeset:

"[Cocoa] Remove support for AVAssetImageGenerator"
https://bugs.webkit.org/show_bug.cgi?id=228560
https://commits.webkit.org/r280488

Canonical link: https://commits.webkit.org/240143@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@280511 268f45cc-cd09-0410-ab3c-d52691b4dbfc
  • Loading branch information
webkit-commit-queue committed Jul 31, 2021
1 parent bb1e5be commit 17a0ef7ef01b6befcbce080cdb1202d6e406e2f9
Showing 8 changed files with 99 additions and 32 deletions.
@@ -1,3 +1,16 @@
2021-07-31 Commit Queue <commit-queue@webkit.org>

Unreviewed, reverting r280488.
https://bugs.webkit.org/show_bug.cgi?id=228677

media/modern-media-controls/ tests are failing or timed-out

Reverted changeset:

"[Cocoa] Remove support for AVAssetImageGenerator"
https://bugs.webkit.org/show_bug.cgi?id=228560
https://commits.webkit.org/r280488

2021-07-30 Rob Buis <rbuis@igalia.com>

[css-overflow] Implement clip value for overflow
@@ -26,7 +26,7 @@

t.step(function() {
var video = document.getElementById("contained");
video.src = getVideoURI('/media/16x16-green');
video.src = getVideoURI('/media/2x2-green');
assert_ratio(video, 2.5);
}, "contain:size aspect ratio");

@@ -37,7 +37,7 @@
video = document.createElement("video");
video.setAttribute("width", "250");
video.setAttribute("height", "100");
video.src = getVideoURI('/media/16x16-green');
video.src = getVideoURI('/media/2x2-green');
document.body.appendChild(video);
// Videos default to a size of 300x150px and calculate their aspect ratio
// based on that before the video is loaded. So this should be 2, ignoring
@@ -33,7 +33,7 @@

media.addEventListener("seeking", startTest, false);
media.loop = true;
media.src = getVideoURI("/media/16x16-green") + "?" + new Date() + Math.random();
media.src = getVideoURI("/media/2x2-green") + "?" + new Date() + Math.random();
media.play();
</script>
</body>
Binary file not shown.
@@ -1,3 +1,16 @@
2021-07-31 Commit Queue <commit-queue@webkit.org>

Unreviewed, reverting r280488.
https://bugs.webkit.org/show_bug.cgi?id=228677

media/modern-media-controls/ tests are failing or timed-out

Reverted changeset:

"[Cocoa] Remove support for AVAssetImageGenerator"
https://bugs.webkit.org/show_bug.cgi?id=228560
https://commits.webkit.org/r280488

2021-07-30 Rob Buis <rbuis@igalia.com>

[css-overflow] Implement clip value for overflow
@@ -543,16 +543,11 @@ void MediaPlayerPrivateAVFoundation::updateStates()
}
}

if (newReadyState >= MediaPlayer::ReadyState::HaveCurrentData && m_cachedHasVideo && !hasAvailableVideoFrame()) {
newReadyState = MediaPlayer::ReadyState::HaveMetadata;
ALWAYS_LOG(LOGIDENTIFIER, "!hasAvailableVideoFrame(), lowering readyState to ", newReadyState);
}

if (isReadyForVideoSetup() && currentRenderingMode() != preferredRenderingMode())
setUpVideoRendering();

if (!m_haveReportedFirstVideoFrame && m_cachedHasVideo && hasAvailableVideoFrame()) {
if (newReadyState < MediaPlayer::ReadyState::HaveCurrentData)
if (m_readyState < MediaPlayer::ReadyState::HaveCurrentData)
newReadyState = MediaPlayer::ReadyState::HaveCurrentData;
m_haveReportedFirstVideoFrame = true;
m_player->firstVideoFrameAvailable();
@@ -118,8 +118,6 @@ class MediaPlayerPrivateAVFoundationObjC final : public MediaPlayerPrivateAVFoun

MediaTime currentMediaTime() const final;

void outputMediaDataWillChange();

private:
#if ENABLE(ENCRYPTED_MEDIA)
void cdmInstanceAttached(CDMInstance&) final;
@@ -67,6 +67,7 @@
#import "WebCoreCALayerExtras.h"
#import "WebCoreNSURLExtras.h"
#import "WebCoreNSURLSession.h"
#import <AVFoundation/AVAssetImageGenerator.h>
#import <AVFoundation/AVAssetTrack.h>
#import <AVFoundation/AVMediaSelectionGroup.h>
#import <AVFoundation/AVMetadataItem.h>
@@ -195,7 +196,6 @@ - (BOOL)resourceLoader:(AVAssetResourceLoader *)resourceLoader shouldWaitForLoad
@interface WebCoreAVFPullDelegate : NSObject<AVPlayerItemOutputPullDelegate> {
BinarySemaphore m_semaphore;
}
- (void)setParent:(WebCore::MediaPlayerPrivateAVFoundationObjC&)parent;
- (void)outputMediaDataWillChange:(AVPlayerItemOutput *)sender;
- (void)outputSequenceWasFlushed:(AVPlayerItemOutput *)output;

@@ -561,17 +561,43 @@ static WallTime toSystemClockTime(NSDate *date)

bool MediaPlayerPrivateAVFoundationObjC::hasContextRenderer() const
{
return m_videoOutput;
return m_videoOutput || m_imageGenerator;
}

void MediaPlayerPrivateAVFoundationObjC::createContextVideoRenderer()
{
createVideoOutput();
}

void MediaPlayerPrivateAVFoundationObjC::createImageGenerator()
{
INFO_LOG(LOGIDENTIFIER);

if (!m_avAsset || m_imageGenerator)
return;

m_imageGenerator = [PAL::getAVAssetImageGeneratorClass() assetImageGeneratorWithAsset:m_avAsset.get()];

[m_imageGenerator.get() setApertureMode:AVAssetImageGeneratorApertureModeCleanAperture];
[m_imageGenerator.get() setAppliesPreferredTrackTransform:YES];
[m_imageGenerator.get() setRequestedTimeToleranceBefore:PAL::kCMTimeZero];
[m_imageGenerator.get() setRequestedTimeToleranceAfter:PAL::kCMTimeZero];
}

void MediaPlayerPrivateAVFoundationObjC::destroyContextVideoRenderer()
{
destroyVideoOutput();
destroyImageGenerator();
}

void MediaPlayerPrivateAVFoundationObjC::destroyImageGenerator()
{
if (!m_imageGenerator)
return;

INFO_LOG(LOGIDENTIFIER);

m_imageGenerator = 0;
}

void MediaPlayerPrivateAVFoundationObjC::createVideoLayer()
@@ -1787,6 +1813,8 @@ static URL conformFragmentIdentifierForURL(const URL& url)
// the video output, too.
if (videoOutputHasAvailableFrame() || (m_videoOutput && m_lastPixelBuffer))
paintWithVideoOutput(context, rect);
else
paintWithImageGenerator(context, rect);

END_BLOCK_OBJC_EXCEPTIONS
setDelayCallbacks(false);
@@ -1810,6 +1838,38 @@ static URL conformFragmentIdentifierForURL(const URL& url)
paintCurrentFrameInContext(context, rect);
}

void MediaPlayerPrivateAVFoundationObjC::paintWithImageGenerator(GraphicsContext& context, const FloatRect& rect)
{
INFO_LOG(LOGIDENTIFIER);

RetainPtr<CGImageRef> image = createImageForTimeInRect(currentTime(), rect);
if (image) {
GraphicsContextStateSaver stateSaver(context);
context.translate(rect.x(), rect.y() + rect.height());
context.scale(FloatSize(1.0f, -1.0f));
context.setImageInterpolationQuality(InterpolationQuality::Low);
IntRect paintRect(IntPoint(0, 0), IntSize(rect.width(), rect.height()));
CGContextDrawImage(context.platformContext(), CGRectMake(0, 0, paintRect.width(), paintRect.height()), image.get());
}
}

RetainPtr<CGImageRef> MediaPlayerPrivateAVFoundationObjC::createImageForTimeInRect(float time, const FloatRect& rect)
{
if (!m_imageGenerator)
createImageGenerator();
ASSERT(m_imageGenerator);

MonotonicTime start = MonotonicTime::now();

[m_imageGenerator.get() setMaximumSize:CGSize(rect.size())];
RetainPtr<CGImageRef> rawImage = adoptCF([m_imageGenerator.get() copyCGImageAtTime:PAL::CMTimeMakeWithSeconds(time, 600) actualTime:nil error:nil]);
RetainPtr<CGImageRef> image = adoptCF(CGImageCreateCopyWithColorSpace(rawImage.get(), sRGBColorSpaceRef()));

INFO_LOG(LOGIDENTIFIER, "creating image took ", (MonotonicTime::now() - start).seconds());

return image;
}

void MediaPlayerPrivateAVFoundationObjC::getSupportedTypes(HashSet<String, ASCIICaseInsensitiveHash>& supportedTypes)
{
supportedTypes = AVAssetMIMETypeCache::singleton().supportedTypes();
@@ -2380,18 +2440,11 @@ void determineChangedTracksFromNewTracksAndOldItems(MediaSelectionGroupAVFObjC*
}

m_videoOutputDelegate = adoptNS([[WebCoreAVFPullDelegate alloc] init]);
[m_videoOutputDelegate setParent:*this];
[m_videoOutput setDelegate:m_videoOutputDelegate.get() queue:globalPullDelegateQueue()];
[m_videoOutput requestNotificationOfMediaDataChangeWithAdvanceInterval:0];

[m_avPlayerItem.get() addOutput:m_videoOutput.get()];
}

void MediaPlayerPrivateAVFoundationObjC::outputMediaDataWillChange()
{
updateStates();
}

void MediaPlayerPrivateAVFoundationObjC::destroyVideoOutput()
{
if (!m_videoOutput)
@@ -2486,6 +2539,12 @@ void determineChangedTracksFromNewTracksAndOldItems(MediaSelectionGroupAVFObjC*

FloatRect imageRect { FloatPoint::zero(), m_lastImage->size() };
context.drawNativeImage(*m_lastImage, imageRect.size(), outputRect, imageRect);

// If we have created an AVAssetImageGenerator in the past due to m_videoOutput not having an available
// video frame, destroy it now that it is no longer needed.
if (m_imageGenerator)
destroyImageGenerator();

}

RetainPtr<CVPixelBufferRef> MediaPlayerPrivateAVFoundationObjC::pixelBufferForCurrentTime()
@@ -3840,25 +3899,14 @@ - (void)resourceLoader:(AVAssetResourceLoader *)resourceLoader didCancelLoadingR

@end

@implementation WebCoreAVFPullDelegate {
WeakPtr<WebCore::MediaPlayerPrivateAVFoundationObjC> _parent;
}
@implementation WebCoreAVFPullDelegate

@synthesize semaphore = m_semaphore;

- (void)setParent:(WebCore::MediaPlayerPrivateAVFoundationObjC&)parent
{
_parent = makeWeakPtr(parent);
}

- (void)outputMediaDataWillChange:(AVPlayerItemVideoOutput *)output
{
UNUSED_PARAM(output);
m_semaphore.signal();
RunLoop::main().dispatch([parent = _parent] {
if (parent)
parent->outputMediaDataWillChange();
});
}

- (void)outputSequenceWasFlushed:(AVPlayerItemVideoOutput *)output

0 comments on commit 17a0ef7

Please sign in to comment.