Performance: do pixel conformance and texturing in a single step.

https://bugs.webkit.org/show_bug.cgi?id=178219
<rdar://problem/34937237>

Reviewed by Dean Jackson.

No new tests; performance improvements should have no behavior change.

Rather than asking the VTDecompressionSession to conform the output CVPixelBuffer into a
pixel format compatible with OpenGL (& ES), don't constrain the output at all, and only do a
conformance step if the output is not already compatible with OpenGL. This eliminates one
copy (in hardware) operation.

Move the TextureCacheCV object into VideoTextureCopierCV; it will be conditionally used to
create the texture if the pixel buffer is compatible.

Refactor copyVideoTextureToPlatformTexture(CVOpenGLTextureRef) in VideoTextureCopierCV. The
new entry point, copyImageToPlatformTexture(), will attempt to use the texture cache first,
and call a new common copyVideoTextureToPlatformTexture(Platform3DObject) with the result.

The new copyImageToPlatformTexture() will pull planar YUV frames into two textures, and combine
the two with a color transfer function when drawing to the output texture.

* platform/graphics/GraphicsContext3D.h:
* platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.mm:
(WebCore::MediaPlayerPrivateMediaSourceAVFObjC::copyVideoTextureToPlatformTexture):
* platform/graphics/cocoa/GraphicsContext3DCocoa.mm:
(WebCore::GraphicsContext3D::texImageIOSurface2D):
* platform/graphics/cocoa/WebCoreDecompressionSession.mm:
(WebCore::WebCoreDecompressionSession::ensureDecompressionSessionForSample):
* platform/graphics/cv/TextureCacheCV.h:
* platform/graphics/cv/TextureCacheCV.mm:
(WebCore::TextureCacheCV::textureFromImage):
* platform/graphics/cv/VideoTextureCopierCV.cpp:
(WebCore::pixelRangeFromPixelFormat):
(WebCore::transferFunctionFromString):
(WebCore::YCbCrToRGBMatrixForRangeAndTransferFunction):
(WebCore::VideoTextureCopierCV::~VideoTextureCopierCV):
(WebCore::VideoTextureCopierCV::initializeUVContextObjects):
(WebCore::VideoTextureCopierCV::copyImageToPlatformTexture):
(WebCore::VideoTextureCopierCV::copyVideoTextureToPlatformTexture):
* platform/graphics/cv/VideoTextureCopierCV.h:


Canonical link: https://commits.webkit.org/194494@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@223280 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Source/WebCore/ChangeLog

@@ -1,3 +1,48 @@
+2017-10-13  Jer Noble  <jer.noble@apple.com>
+
+        Performance: do pixel conformance and texturing in a single step.
+        https://bugs.webkit.org/show_bug.cgi?id=178219
+        <rdar://problem/34937237>
+
+        Reviewed by Dean Jackson.
+
+        No new tests; performance improvements should have no behavior change.
+
+        Rather than asking the VTDecompressionSession to conform the output CVPixelBuffer into a
+        pixel format compatible with OpenGL (& ES), don't constrain the output at all, and only do a
+        conformance step if the output is not already compatible with OpenGL. This eliminates one
+        copy (in hardware) operation.
+
+        Move the TextureCacheCV object into VideoTextureCopierCV; it will be conditionally used to
+        create the texture if the pixel buffer is compatible.
+
+        Refactor copyVideoTextureToPlatformTexture(CVOpenGLTextureRef) in VideoTextureCopierCV. The
+        new entry point, copyImageToPlatformTexture(), will attempt to use the texture cache first,
+        and call a new common copyVideoTextureToPlatformTexture(Platform3DObject) with the result.
+
+        The new copyImageToPlatformTexture() will pull planar YUV frames into two textures, and combine
+        the two with a color transfer function when drawing to the output texture.
+
+        * platform/graphics/GraphicsContext3D.h:
+        * platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.mm:
+        (WebCore::MediaPlayerPrivateMediaSourceAVFObjC::copyVideoTextureToPlatformTexture):
+        * platform/graphics/cocoa/GraphicsContext3DCocoa.mm:
+        (WebCore::GraphicsContext3D::texImageIOSurface2D):
+        * platform/graphics/cocoa/WebCoreDecompressionSession.mm:
+        (WebCore::WebCoreDecompressionSession::ensureDecompressionSessionForSample):
+        * platform/graphics/cv/TextureCacheCV.h:
+        * platform/graphics/cv/TextureCacheCV.mm:
+        (WebCore::TextureCacheCV::textureFromImage):
+        * platform/graphics/cv/VideoTextureCopierCV.cpp:
+        (WebCore::pixelRangeFromPixelFormat):
+        (WebCore::transferFunctionFromString):
+        (WebCore::YCbCrToRGBMatrixForRangeAndTransferFunction):
+        (WebCore::VideoTextureCopierCV::~VideoTextureCopierCV):
+        (WebCore::VideoTextureCopierCV::initializeUVContextObjects):
+        (WebCore::VideoTextureCopierCV::copyImageToPlatformTexture):
+        (WebCore::VideoTextureCopierCV::copyVideoTextureToPlatformTexture):
+        * platform/graphics/cv/VideoTextureCopierCV.h:
+
 2017-10-13  Romain Bellessort  <romain.bellessort@crf.canon.fr>
 
         [Readable Streams API] Align queue with spec for ReadableStreamDefaultController

Source/WebCore/platform/cocoa/CoreVideoSoftLink.cpp

@@ -52,6 +52,13 @@ SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, CoreVideo, kCVPixelFormatOpenGLCompatibil
 SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, CoreVideo, kCVPixelFormatOpenGLESCompatibility, CFStringRef)
 SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, CoreVideo, kCVPixelBufferIOSurfacePropertiesKey, CFStringRef)
 SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, CoreVideo, kCVPixelBufferPoolMinimumBufferCountKey, CFStringRef)
+SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, CoreVideo, kCVImageBufferYCbCrMatrixKey, CFStringRef)
+SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, CoreVideo, kCVImageBufferYCbCrMatrix_ITU_R_709_2, CFStringRef)
+SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, CoreVideo, kCVImageBufferYCbCrMatrix_ITU_R_601_4, CFStringRef)
+SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, CoreVideo, kCVImageBufferYCbCrMatrix_SMPTE_240M_1995, CFStringRef)
+SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, CoreVideo, kCVImageBufferYCbCrMatrix_DCI_P3, CFStringRef)
+SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, CoreVideo, kCVImageBufferYCbCrMatrix_P3_D65, CFStringRef)
+SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, CoreVideo, kCVImageBufferYCbCrMatrix_ITU_R_2020, CFStringRef)
 
 #if PLATFORM(IOS)
 SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreVideo, CVOpenGLESTextureCacheCreate, CVReturn, (CFAllocatorRef allocator, CFDictionaryRef cacheAttributes, CVEAGLContext eaglContext, CFDictionaryRef textureAttributes, CVOpenGLESTextureCacheRef* cacheOut), (allocator, cacheAttributes, eaglContext, textureAttributes, cacheOut))

Source/WebCore/platform/cocoa/CoreVideoSoftLink.h

@@ -70,6 +70,20 @@ SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, CoreVideo, kCVPixelBufferIOSurfacePropert
 #define kCVPixelBufferIOSurfacePropertiesKey get_CoreVideo_kCVPixelBufferIOSurfacePropertiesKey()
 SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, CoreVideo, kCVPixelBufferPoolMinimumBufferCountKey, CFStringRef)
 #define kCVPixelBufferPoolMinimumBufferCountKey get_CoreVideo_kCVPixelBufferPoolMinimumBufferCountKey()
+SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, CoreVideo, kCVImageBufferYCbCrMatrixKey, CFStringRef)
+#define kCVImageBufferYCbCrMatrixKey get_CoreVideo_kCVImageBufferYCbCrMatrixKey()
+SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, CoreVideo, kCVImageBufferYCbCrMatrix_ITU_R_709_2, CFStringRef)
+#define kCVImageBufferYCbCrMatrix_ITU_R_709_2 get_CoreVideo_kCVImageBufferYCbCrMatrix_ITU_R_709_2()
+SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, CoreVideo, kCVImageBufferYCbCrMatrix_ITU_R_601_4, CFStringRef)
+#define kCVImageBufferYCbCrMatrix_ITU_R_601_4 get_CoreVideo_kCVImageBufferYCbCrMatrix_ITU_R_601_4()
+SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, CoreVideo, kCVImageBufferYCbCrMatrix_SMPTE_240M_1995, CFStringRef)
+#define kCVImageBufferYCbCrMatrix_SMPTE_240M_1995 get_CoreVideo_kCVImageBufferYCbCrMatrix_SMPTE_240M_1995()
+SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, CoreVideo, kCVImageBufferYCbCrMatrix_DCI_P3, CFStringRef)
+#define kCVImageBufferYCbCrMatrix_DCI_P3 get_CoreVideo_kCVImageBufferYCbCrMatrix_DCI_P3()
+SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, CoreVideo, kCVImageBufferYCbCrMatrix_P3_D65, CFStringRef)
+#define kCVImageBufferYCbCrMatrix_P3_D65 get_CoreVideo_kCVImageBufferYCbCrMatrix_P3_D65()
+SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, CoreVideo, kCVImageBufferYCbCrMatrix_ITU_R_2020, CFStringRef)
+#define kCVImageBufferYCbCrMatrix_ITU_R_2020 get_CoreVideo_kCVImageBufferYCbCrMatrix_ITU_R_2020()
 
 #if PLATFORM(IOS)
 SOFT_LINK_FUNCTION_FOR_HEADER(WebCore, CoreVideo, CVOpenGLESTextureCacheCreate, CVReturn, (CFAllocatorRef allocator, CFDictionaryRef cacheAttributes, CVEAGLContext eaglContext, CFDictionaryRef textureAttributes, CVOpenGLESTextureCacheRef* cacheOut), (allocator, cacheAttributes, eaglContext, textureAttributes, cacheOut))

Source/WebCore/platform/graphics/GraphicsContext3D.h

@@ -59,6 +59,7 @@
 #include <wtf/RetainPtr.h>
 OBJC_CLASS CALayer;
 OBJC_CLASS WebGLLayer;
+typedef struct __IOSurface* IOSurfaceRef;
 #elif PLATFORM(GTK) || PLATFORM(WIN_CAIRO) || PLATFORM(WPE)
 typedef unsigned int GLuint;
 #endif
@@ -1149,6 +1150,7 @@ class GraphicsContext3D : public RefCounted<GraphicsContext3D> {
 
 #if PLATFORM(COCOA)
     void endPaint();
+    bool texImageIOSurface2D(GC3Denum target, GC3Denum internalFormat, GC3Dsizei width, GC3Dsizei height, GC3Denum format, GC3Denum type, IOSurfaceRef, GC3Duint plane);
 #endif
 
 #if PLATFORM(MAC)

Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.mm

@@ -608,14 +608,6 @@ static void CMTimebaseEffectiveRateChangedCallback(CMNotificationCenterRef, cons
     if (updateLastPixelBuffer()) {
         if (!m_lastPixelBuffer)
             return false;
-
-        if (!m_textureCache) {
-            m_textureCache = TextureCacheCV::create(*context);
-            if (!m_textureCache)
-                return false;
-        }
-
-        m_lastTexture = m_textureCache->textureFromImage(m_lastPixelBuffer.get(), outputTarget, level, internalFormat, format, type);
     }
 
     size_t width = CVPixelBufferGetWidth(m_lastPixelBuffer.get());
@@ -624,7 +616,7 @@ static void CMTimebaseEffectiveRateChangedCallback(CMNotificationCenterRef, cons
     if (!m_videoTextureCopier)
         m_videoTextureCopier = std::make_unique<VideoTextureCopierCV>(*context);
 
-    return m_videoTextureCopier->copyVideoTextureToPlatformTexture(m_lastTexture.get(), width, height, outputTexture, outputTarget, level, internalFormat, format, type, premultiplyAlpha, flipY);
+    return m_videoTextureCopier->copyImageToPlatformTexture(m_lastPixelBuffer.get(), width, height, outputTexture, outputTarget, level, internalFormat, format, type, premultiplyAlpha, flipY);
 }
 
 bool MediaPlayerPrivateMediaSourceAVFObjC::hasAvailableVideoFrame() const

Source/WebCore/platform/graphics/cocoa/GraphicsContext3DCocoa.mm

@@ -50,6 +50,7 @@
 #if PLATFORM(IOS)
 #import <OpenGLES/EAGL.h>
 #import <OpenGLES/EAGLDrawable.h>
+#import <OpenGLES/EAGLIOSurface.h>
 #import <OpenGLES/ES2/glext.h>
 #import <QuartzCore/QuartzCore.h>
 #import <pal/spi/ios/OpenGLESSPI.h>
@@ -672,6 +673,17 @@ static void setPixelFormat(Vector<CGLPixelFormatAttribute>& attribs, int colorBi
 #endif
 }
 
+bool GraphicsContext3D::texImageIOSurface2D(GC3Denum target, GC3Denum internalFormat, GC3Dsizei width, GC3Dsizei height, GC3Denum format, GC3Denum type, IOSurfaceRef surface, GC3Duint plane)
+{
+#if PLATFORM(MAC)
+    return kCGLNoError == CGLTexImageIOSurface2D(platformGraphicsContext3D(), target, internalFormat, width, height, format, type, surface, plane);
+#elif PLATFORM(IOS) && !PLATFORM(IOS_SIMULATOR)
+    return [platformGraphicsContext3D() texImageIOSurface:surface target:target internalFormat:internalFormat width:width height:height format:format type:type plane:plane];
+#else
+    return false;
+#endif
+}
+
 #if PLATFORM(MAC)
 void GraphicsContext3D::allocateIOSurfaceBackingStore(IntSize size)
 {

Source/WebCore/platform/graphics/cocoa/WebCoreDecompressionSession.mm

@@ -219,11 +219,7 @@
 
         NSDictionary *attributes;
         if (m_mode == OpenGL) {
-#if PLATFORM(IOS)
-            attributes = @{(NSString *)kCVPixelBufferIOSurfaceOpenGLESFBOCompatibilityKey: @YES};
-#else
-            attributes = @{(NSString *)kCVPixelBufferIOSurfaceOpenGLFBOCompatibilityKey: @YES};
-#endif
+            attributes = nil;
         } else {
             ASSERT(m_mode == RGB);
             attributes = @{(NSString *)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_32BGRA)};

Source/WebCore/platform/graphics/cv/TextureCacheCV.h

@@ -31,7 +31,8 @@
 #include <wtf/RetainPtr.h>
 #include <wtf/WeakPtr.h>
 
-typedef struct  __CVBuffer* CVImageBufferRef;
+typedef struct __CVBuffer* CVImageBufferRef;
+typedef CVImageBufferRef CVPixelBufferRef;
 typedef CVImageBufferRef CVOpenGLTextureRef;
 typedef CVImageBufferRef CVOpenGLESTextureRef;
 typedef struct __CVOpenGLTextureCache *CVOpenGLTextureCacheRef;
@@ -55,7 +56,7 @@ class TextureCacheCV {
 
     TextureCacheCV(GraphicsContext3D&, RetainPtr<TextureCacheType>&&);
 
-    RetainPtr<TextureType> textureFromImage(CVImageBufferRef, GC3Denum outputTarget, GC3Dint level, GC3Denum internalFormat, GC3Denum format, GC3Denum type);
+    RetainPtr<TextureType> textureFromImage(CVPixelBufferRef, GC3Denum outputTarget, GC3Dint level, GC3Denum internalFormat, GC3Denum format, GC3Denum type);
     GraphicsContext3D& context() { return m_context.get(); }
 
 private:

Source/WebCore/platform/graphics/cv/TextureCacheCV.mm

@@ -53,7 +53,7 @@
 {
 }
 
-RetainPtr<TextureCacheCV::TextureType> TextureCacheCV::textureFromImage(CVImageBufferRef image, GC3Denum outputTarget, GC3Dint level, GC3Denum internalFormat, GC3Denum format, GC3Denum type)
+RetainPtr<TextureCacheCV::TextureType> TextureCacheCV::textureFromImage(CVPixelBufferRef image, GC3Denum outputTarget, GC3Dint level, GC3Denum internalFormat, GC3Denum format, GC3Denum type)
 {
     TextureType bareVideoTexture = nullptr;
 #if PLATFORM(IOS)