From 30bb5f1cbd5ebff46353f0dc60e4a281b6b204a5 Mon Sep 17 00:00:00 2001 From: Zachary Anderson Date: Fri, 5 Feb 2021 06:38:04 -0800 Subject: [PATCH] Revert "[metal] Darwin unified external metal textures (#24157)" (#24223) This reverts commit 6a2df7f1c9f6e33fa1d9c9aa5869e0eef232cc3a. --- ci/licenses_golden/licenses_flutter | 2 - common/config.gni | 2 - common/graphics/texture.h | 4 +- shell/platform/darwin/graphics/BUILD.gn | 9 +- .../graphics/FlutterDarwinContextMetal.h | 15 -- .../graphics/FlutterDarwinContextMetal.mm | 38 +-- .../FlutterDarwinExternalTextureMetal.h | 34 --- .../FlutterDarwinExternalTextureMetal.mm | 245 ------------------ .../platform/darwin/ios/ios_context_metal.mm | 5 +- .../darwin/ios/ios_external_texture_metal.h | 29 ++- .../darwin/ios/ios_external_texture_metal.mm | 240 ++++++++++++++++- shell/platform/darwin/macos/BUILD.gn | 4 +- 12 files changed, 265 insertions(+), 362 deletions(-) delete mode 100644 shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.h delete mode 100644 shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.mm diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index 9292b5aa78057..fefcb85679d56 100644 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -969,8 +969,6 @@ FILE: ../../../flutter/shell/platform/darwin/common/framework/Source/flutter_cod FILE: ../../../flutter/shell/platform/darwin/common/framework/Source/flutter_standard_codec_unittest.mm FILE: ../../../flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetal.h FILE: ../../../flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetal.mm -FILE: ../../../flutter/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.h -FILE: ../../../flutter/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.mm FILE: ../../../flutter/shell/platform/darwin/ios/framework/Flutter.podspec FILE: ../../../flutter/shell/platform/darwin/ios/framework/Headers/Flutter.h FILE: ../../../flutter/shell/platform/darwin/ios/framework/Headers/FlutterAppDelegate.h diff --git a/common/config.gni b/common/config.gni index 6d1cecb76fb6c..983e6bb49a899 100644 --- a/common/config.gni +++ b/common/config.gni @@ -62,6 +62,4 @@ if (is_ios || is_mac) { "-Werror=undeclared-selector", ] flutter_cflags_objcc = flutter_cflags_objc - flutter_cflags_objc_arc = flutter_cflags_objc + [ "-fobjc-arc" ] - flutter_cflags_objcc_arc = flutter_cflags_objc_arc } diff --git a/common/graphics/texture.h b/common/graphics/texture.h index 5ed8dafa668f4..5b5d6bd6d6b24 100644 --- a/common/graphics/texture.h +++ b/common/graphics/texture.h @@ -18,8 +18,8 @@ namespace flutter { class Texture { public: - explicit Texture(int64_t id); // Called from UI or raster thread. - virtual ~Texture(); // Called from raster thread. + Texture(int64_t id); // Called from UI or raster thread. + virtual ~Texture(); // Called from raster thread. // Called from raster thread. virtual void Paint(SkCanvas& canvas, diff --git a/shell/platform/darwin/graphics/BUILD.gn b/shell/platform/darwin/graphics/BUILD.gn index 7dd998a48f6a9..5d1cd0a0e9229 100644 --- a/shell/platform/darwin/graphics/BUILD.gn +++ b/shell/platform/darwin/graphics/BUILD.gn @@ -7,24 +7,19 @@ assert(is_ios || is_mac) import("//flutter/common/config.gni") source_set("graphics") { - cflags_objc = flutter_cflags_objc_arc - cflags_objcc = flutter_cflags_objcc_arc + cflags_objc = flutter_cflags_objc + cflags_objcc = flutter_cflags_objcc sources = [ "FlutterDarwinContextMetal.h", "FlutterDarwinContextMetal.mm", - "FlutterDarwinExternalTextureMetal.h", - "FlutterDarwinExternalTextureMetal.mm", ] deps = [ "//flutter/common/graphics", "//flutter/fml", - "//flutter/shell/platform/darwin/common:framework_shared", ] - libs = [ "CoreVideo.framework" ] - public_deps = [ "//third_party/skia" ] public_configs = [ "//flutter:config" ] diff --git a/shell/platform/darwin/graphics/FlutterDarwinContextMetal.h b/shell/platform/darwin/graphics/FlutterDarwinContextMetal.h index 830f857e90c5c..4c23673cfb513 100644 --- a/shell/platform/darwin/graphics/FlutterDarwinContextMetal.h +++ b/shell/platform/darwin/graphics/FlutterDarwinContextMetal.h @@ -5,12 +5,9 @@ #ifndef SHELL_PLATFORM_DARWIN_GRAPHICS_DARWIN_CONTEXT_METAL_H_ #define SHELL_PLATFORM_DARWIN_GRAPHICS_DARWIN_CONTEXT_METAL_H_ -#import #import #import -#import "flutter/shell/platform/darwin/common/framework/Headers/FlutterTexture.h" -#import "flutter/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.h" #include "third_party/skia/include/gpu/GrDirectContext.h" NS_ASSUME_NONNULL_BEGIN @@ -32,13 +29,6 @@ NS_ASSUME_NONNULL_BEGIN - (instancetype)initWithMTLDevice:(id)device commandQueue:(id)commandQueue; -/** - * Creates an external texture with the specified ID and contents. - */ -- (FlutterDarwinExternalTextureMetal*) - createExternalTextureWithIdentifier:(int64_t)textureID - texture:(NSObject*)texture; - /** * MTLDevice that is backing this context.s */ @@ -60,11 +50,6 @@ NS_ASSUME_NONNULL_BEGIN */ @property(nonatomic, readonly) sk_sp resourceContext; -/* - * Texture cache for external textures. - */ -@property(nonatomic, readonly) CVMetalTextureCacheRef textureCache; - @end NS_ASSUME_NONNULL_END diff --git a/shell/platform/darwin/graphics/FlutterDarwinContextMetal.mm b/shell/platform/darwin/graphics/FlutterDarwinContextMetal.mm index 75ca69845c1a8..3c6e884955369 100644 --- a/shell/platform/darwin/graphics/FlutterDarwinContextMetal.mm +++ b/shell/platform/darwin/graphics/FlutterDarwinContextMetal.mm @@ -6,11 +6,8 @@ #include "flutter/common/graphics/persistent_cache.h" #include "flutter/fml/logging.h" -#import "flutter/shell/platform/darwin/common/framework/Headers/FlutterMacros.h" #include "third_party/skia/include/gpu/GrContextOptions.h" -FLUTTER_ASSERT_ARC - static GrContextOptions CreateMetalGrContextOptions() { GrContextOptions options = {}; if (flutter::PersistentCache::cache_sksl()) { @@ -36,6 +33,7 @@ - (instancetype)initWithMTLDevice:(id)device if (!_device) { FML_DLOG(ERROR) << "Could not acquire Metal device."; + [self release]; return nil; } @@ -43,32 +41,24 @@ - (instancetype)initWithMTLDevice:(id)device if (!_commandQueue) { FML_DLOG(ERROR) << "Could not create Metal command queue."; + [self release]; return nil; } [_commandQueue setLabel:@"Flutter Main Queue"]; - CVReturn cvReturn = CVMetalTextureCacheCreate(kCFAllocatorDefault, // allocator - nil, // cache attributes (nil default) - _device, // metal device - nil, // texture attributes (nil default) - &_textureCache // [out] cache - ); - if (cvReturn != kCVReturnSuccess) { - FML_DLOG(ERROR) << "Could not create Metal texture cache."; - return nil; - } - auto contextOptions = CreateMetalGrContextOptions(); // Skia expect arguments to `MakeMetal` transfer ownership of the reference in for release later // when the GrDirectContext is collected. - _mainContext = GrDirectContext::MakeMetal( - (__bridge_retained void*)_device, (__bridge_retained void*)_commandQueue, contextOptions); - _resourceContext = _mainContext; + _mainContext = + GrDirectContext::MakeMetal([_device retain], [_commandQueue retain], contextOptions); + _resourceContext = + GrDirectContext::MakeMetal([_device retain], [_commandQueue retain], contextOptions); if (!_mainContext || !_resourceContext) { FML_DLOG(ERROR) << "Could not create Skia Metal contexts."; + [self release]; return nil; } @@ -77,18 +67,4 @@ - (instancetype)initWithMTLDevice:(id)device return self; } -- (void)dealloc { - if (_textureCache) { - CFRelease(_textureCache); - } -} - -- (FlutterDarwinExternalTextureMetal*) - createExternalTextureWithIdentifier:(int64_t)textureID - texture:(NSObject*)texture { - return [[FlutterDarwinExternalTextureMetal alloc] initWithTextureCache:_textureCache - textureID:textureID - texture:texture]; -} - @end diff --git a/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.h b/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.h deleted file mode 100644 index 3962512e87f58..0000000000000 --- a/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.h +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2013 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#import -#import - -#import "flutter/shell/platform/darwin/common/framework/Headers/FlutterTexture.h" -#include "third_party/skia/include/core/SkCanvas.h" -#include "third_party/skia/include/gpu/GrDirectContext.h" - -@interface FlutterDarwinExternalTextureMetal : NSObject - -- (nullable instancetype)initWithTextureCache:(nonnull CVMetalTextureCacheRef)textureCache - textureID:(int64_t)textureID - texture:(nonnull NSObject*)texture; - -- (void)paint:(SkCanvas&)canvas - bounds:(const SkRect&)bounds - freeze:(BOOL)freeze - grContext:(nonnull GrDirectContext*)grContext - sampling:(const SkSamplingOptions&)sampling; - -- (void)onGrContextCreated; - -- (void)onGrContextDestroyed; - -- (void)markNewFrameAvailable; - -- (void)onTextureUnregistered; - -@property(nonatomic, readonly) int64_t textureID; - -@end diff --git a/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.mm b/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.mm deleted file mode 100644 index 0cc314a180d97..0000000000000 --- a/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.mm +++ /dev/null @@ -1,245 +0,0 @@ -// Copyright 2013 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#import "flutter/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.h" - -#include "flutter/fml/logging.h" -#import "flutter/shell/platform/darwin/common/framework/Headers/FlutterMacros.h" -#include "third_party/skia/include/core/SkYUVAInfo.h" -#include "third_party/skia/include/gpu/GrBackendSurface.h" -#include "third_party/skia/include/gpu/GrDirectContext.h" -#include "third_party/skia/include/gpu/GrYUVABackendTextures.h" -#include "third_party/skia/include/gpu/mtl/GrMtlTypes.h" - -FLUTTER_ASSERT_ARC - -@implementation FlutterDarwinExternalTextureMetal { - CVMetalTextureCacheRef _textureCache; - NSObject* _externalTexture; - BOOL _textureFrameAvailable; - sk_sp _externalImage; - CVPixelBufferRef _lastPixelBuffer; - OSType _pixelFormat; -} - -- (instancetype)initWithTextureCache:(nonnull CVMetalTextureCacheRef)textureCache - textureID:(int64_t)textureID - texture:(NSObject*)texture { - if (self = [super init]) { - _textureCache = textureCache; - CFRetain(_textureCache); - _textureID = textureID; - _externalTexture = texture; - return self; - } - return nil; -} - -- (void)dealloc { - CVPixelBufferRelease(_lastPixelBuffer); - if (_textureCache) { - CFRelease(_textureCache); - } -} - -- (void)paint:(SkCanvas&)canvas - bounds:(const SkRect&)bounds - freeze:(BOOL)freeze - grContext:(nonnull GrDirectContext*)grContext - sampling:(const SkSamplingOptions&)sampling { - const bool needsUpdatedTexture = (!freeze && _textureFrameAvailable) || !_externalImage; - - if (needsUpdatedTexture) { - CVPixelBufferRef pixelBuffer = [_externalTexture copyPixelBuffer]; - if (!pixelBuffer) { - pixelBuffer = _lastPixelBuffer; - } else { - CVPixelBufferRetain(pixelBuffer); - _pixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer); - } - - // If the application told us there was a texture frame available but did not provide one when - // asked for it, reuse the previous texture but make sure to ask again the next time around. - sk_sp image = [self wrapExternalPixelBuffer:pixelBuffer grContext:grContext]; - if (image) { - _externalImage = image; - _textureFrameAvailable = false; - CVPixelBufferRelease(_lastPixelBuffer); - _lastPixelBuffer = pixelBuffer; - } - } - - if (_externalImage) { - canvas.drawImageRect(_externalImage, // image - SkRect::Make(_externalImage->bounds()), // source rect - bounds, // destination rect - sampling, // sampling - nullptr, // paint - SkCanvas::SrcRectConstraint::kFast_SrcRectConstraint // constraint - ); - } -} - -- (void)onGrContextCreated { - // External images in this backend have no thread affinity and are not tied to the context in any - // way. Instead, they are tied to the Metal device which is associated with the cache already and - // is consistent throughout the shell run. -} - -- (void)onGrContextDestroyed { - // The image must be reset because it is tied to the onscreen context. But the pixel buffer that - // created the image is still around. In case of context reacquisition, that last pixel - // buffer will be used to materialize the image in case the application fails to provide a new - // one. - _externalImage.reset(); - CVMetalTextureCacheFlush(_textureCache, // cache - 0 // options (must be zero) - ); -} - -- (void)markNewFrameAvailable { - _textureFrameAvailable = YES; -} - -- (void)onTextureUnregistered { - if ([_externalTexture respondsToSelector:@selector(onTextureUnregistered:)]) { - [_externalTexture onTextureUnregistered:_externalTexture]; - } -} - -#pragma mark - External texture skia wrapper methods. - -- (sk_sp)wrapExternalPixelBuffer:(CVPixelBufferRef)pixelBuffer - grContext:(GrDirectContext*)grContext { - if (!pixelBuffer) { - return nullptr; - } - - sk_sp image = nullptr; - if (_pixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange || - _pixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) { - image = [self wrapNV12ExternalPixelBuffer:pixelBuffer grContext:grContext]; - } else { - image = [self wrapRGBAExternalPixelBuffer:pixelBuffer grContext:grContext]; - } - - if (!image) { - FML_DLOG(ERROR) << "Could not wrap Metal texture as a Skia image."; - } - - return image; -} - -- (sk_sp)wrapNV12ExternalPixelBuffer:(CVPixelBufferRef)pixelBuffer - grContext:(GrDirectContext*)grContext { - SkISize textureSize = - SkISize::Make(CVPixelBufferGetWidth(pixelBuffer), CVPixelBufferGetHeight(pixelBuffer)); - CVMetalTextureRef yMetalTexture = nullptr; - { - CVReturn cvReturn = - CVMetalTextureCacheCreateTextureFromImage(/*allocator=*/kCFAllocatorDefault, - /*textureCache=*/_textureCache, - /*sourceImage=*/pixelBuffer, - /*textureAttributes=*/nullptr, - /*pixelFormat=*/MTLPixelFormatR8Unorm, - /*width=*/textureSize.width(), - /*height=*/textureSize.height(), - /*planeIndex=*/0u, - /*texture=*/&yMetalTexture); - - if (cvReturn != kCVReturnSuccess) { - FML_DLOG(ERROR) << "Could not create Metal texture from pixel buffer: CVReturn " << cvReturn; - return nullptr; - } - } - - CVMetalTextureRef uvMetalTexture = nullptr; - { - CVReturn cvReturn = - CVMetalTextureCacheCreateTextureFromImage(/*allocator=*/kCFAllocatorDefault, - /*textureCache=*/_textureCache, - /*sourceImage=*/pixelBuffer, - /*textureAttributes=*/nullptr, - /*pixelFormat=*/MTLPixelFormatRG8Unorm, - /*width=*/textureSize.width() / 2, - /*height=*/textureSize.height() / 2, - /*planeIndex=*/1u, - /*texture=*/&uvMetalTexture); - - if (cvReturn != kCVReturnSuccess) { - FML_DLOG(ERROR) << "Could not create Metal texture from pixel buffer: CVReturn " << cvReturn; - return nullptr; - } - } - - GrMtlTextureInfo ySkiaTextureInfo; - ySkiaTextureInfo.fTexture = sk_cf_obj{ - (__bridge_retained const void*)CVMetalTextureGetTexture(yMetalTexture)}; - - GrBackendTexture skiaBackendTextures[2]; - skiaBackendTextures[0] = GrBackendTexture(/*width=*/textureSize.width(), - /*height=*/textureSize.height(), - /*mipMapped=*/GrMipMapped ::kNo, - /*textureInfo=*/ySkiaTextureInfo); - - GrMtlTextureInfo uvSkiaTextureInfo; - uvSkiaTextureInfo.fTexture = sk_cf_obj{ - (__bridge_retained const void*)CVMetalTextureGetTexture(uvMetalTexture)}; - - skiaBackendTextures[1] = GrBackendTexture(/*width=*/textureSize.width(), - /*height=*/textureSize.height(), - /*mipMapped=*/GrMipMapped ::kNo, - /*textureInfo=*/uvSkiaTextureInfo); - SkYUVAInfo yuvaInfo(skiaBackendTextures[0].dimensions(), SkYUVAInfo::PlaneConfig::kY_UV, - SkYUVAInfo::Subsampling::k444, kRec601_SkYUVColorSpace); - GrYUVABackendTextures yuvaBackendTextures(yuvaInfo, skiaBackendTextures, - kTopLeft_GrSurfaceOrigin); - - sk_sp image = - SkImage::MakeFromYUVATextures(grContext, yuvaBackendTextures, /*imageColorSpace=*/nullptr, - /*releaseProc*/ nullptr, /*releaseContext*/ nullptr); - return image; -} - -- (sk_sp)wrapRGBAExternalPixelBuffer:(CVPixelBufferRef)pixelBuffer - grContext:(GrDirectContext*)grContext { - SkISize textureSize = - SkISize::Make(CVPixelBufferGetWidth(pixelBuffer), CVPixelBufferGetHeight(pixelBuffer)); - CVMetalTextureRef metalTexture = nullptr; - CVReturn cvReturn = - CVMetalTextureCacheCreateTextureFromImage(/*allocator=*/kCFAllocatorDefault, - /*textureCache=*/_textureCache, - /*sourceImage=*/pixelBuffer, - /*textureAttributes=*/nullptr, - /*pixelFormat=*/MTLPixelFormatBGRA8Unorm, - /*width=*/textureSize.width(), - /*height=*/textureSize.height(), - /*planeIndex=*/0u, - /*texture=*/&metalTexture); - - if (cvReturn != kCVReturnSuccess) { - FML_DLOG(ERROR) << "Could not create Metal texture from pixel buffer: CVReturn " << cvReturn; - return nullptr; - } - - GrMtlTextureInfo skiaTextureInfo; - skiaTextureInfo.fTexture = - sk_cf_obj{(__bridge_retained const void*)CVMetalTextureGetTexture(metalTexture)}; - - GrBackendTexture skiaBackendTexture(/*width=*/textureSize.width(), - /*height=*/textureSize.height(), - /*mipMapped=*/GrMipMapped ::kNo, - /*textureInfo=*/skiaTextureInfo); - - sk_sp image = - SkImage::MakeFromTexture(grContext, skiaBackendTexture, kTopLeft_GrSurfaceOrigin, - kBGRA_8888_SkColorType, kPremul_SkAlphaType, - /*imageColorSpace=*/nullptr, /*releaseProc*/ nullptr, - /*releaseContext*/ nullptr - - ); - return image; -} - -@end diff --git a/shell/platform/darwin/ios/ios_context_metal.mm b/shell/platform/darwin/ios/ios_context_metal.mm index 1c9a1a1f09f61..c8d28d774eaa0 100644 --- a/shell/platform/darwin/ios/ios_context_metal.mm +++ b/shell/platform/darwin/ios/ios_context_metal.mm @@ -65,10 +65,7 @@ std::unique_ptr IOSContextMetal::CreateExternalTexture( int64_t texture_id, fml::scoped_nsobject> texture) { - return std::make_unique( - fml::scoped_nsobject{ - [[darwin_context_metal_ createExternalTextureWithIdentifier:texture_id - texture:texture] retain]}); + return std::make_unique(texture_id, texture_cache_, std::move(texture)); } } // namespace flutter diff --git a/shell/platform/darwin/ios/ios_external_texture_metal.h b/shell/platform/darwin/ios/ios_external_texture_metal.h index 8aab36c676d2a..dba9ffcaf4ee1 100644 --- a/shell/platform/darwin/ios/ios_external_texture_metal.h +++ b/shell/platform/darwin/ios/ios_external_texture_metal.h @@ -5,25 +5,35 @@ #ifndef FLUTTER_SHELL_PLATFORM_DARWIN_IOS_IOS_EXTERNAL_TEXTURE_METAL_H_ #define FLUTTER_SHELL_PLATFORM_DARWIN_IOS_IOS_EXTERNAL_TEXTURE_METAL_H_ +#include + +#import + #include "flutter/common/graphics/texture.h" #include "flutter/fml/macros.h" +#include "flutter/fml/platform/darwin/cf_utils.h" #include "flutter/fml/platform/darwin/scoped_nsobject.h" -#import "flutter/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.h" +#import "flutter/shell/platform/darwin/common/framework/Headers/FlutterTexture.h" +#include "third_party/skia/include/core/SkImage.h" namespace flutter { class IOSExternalTextureMetal final : public Texture { public: - explicit IOSExternalTextureMetal( - fml::scoped_nsobject - darwin_external_texture_metal); + IOSExternalTextureMetal(int64_t texture_id, + fml::CFRef texture_cache, + fml::scoped_nsobject> external_texture); // |Texture| ~IOSExternalTextureMetal(); private: - fml::scoped_nsobject - darwin_external_texture_metal_; + fml::CFRef texture_cache_; + fml::scoped_nsobject> external_texture_; + std::atomic_bool texture_frame_available_; + fml::CFRef last_pixel_buffer_; + sk_sp external_image_; + OSType pixel_format_ = 0; // |Texture| void Paint(SkCanvas& canvas, @@ -44,6 +54,13 @@ class IOSExternalTextureMetal final : public Texture { // |Texture| void OnTextureUnregistered() override; + sk_sp WrapExternalPixelBuffer(fml::CFRef pixel_buffer, + GrDirectContext* context) const; + sk_sp WrapRGBAExternalPixelBuffer(fml::CFRef pixel_buffer, + GrDirectContext* context) const; + sk_sp WrapNV12ExternalPixelBuffer(fml::CFRef pixel_buffer, + GrDirectContext* context) const; + FML_DISALLOW_COPY_AND_ASSIGN(IOSExternalTextureMetal); }; diff --git a/shell/platform/darwin/ios/ios_external_texture_metal.mm b/shell/platform/darwin/ios/ios_external_texture_metal.mm index 52dac7b3a4723..a6380807da8b3 100644 --- a/shell/platform/darwin/ios/ios_external_texture_metal.mm +++ b/shell/platform/darwin/ios/ios_external_texture_metal.mm @@ -4,12 +4,25 @@ #import "flutter/shell/platform/darwin/ios/ios_external_texture_metal.h" +#include "flutter/fml/logging.h" +#include "third_party/skia/include/core/SkYUVAInfo.h" +#include "third_party/skia/include/gpu/GrBackendSurface.h" +#include "third_party/skia/include/gpu/GrDirectContext.h" +#include "third_party/skia/include/gpu/GrYUVABackendTextures.h" +#include "third_party/skia/include/gpu/mtl/GrMtlTypes.h" + namespace flutter { IOSExternalTextureMetal::IOSExternalTextureMetal( - fml::scoped_nsobject darwin_external_texture_metal) - : Texture([darwin_external_texture_metal textureID]), - darwin_external_texture_metal_(darwin_external_texture_metal) {} + int64_t texture_id, + fml::CFRef texture_cache, + fml::scoped_nsobject> external_texture) + : Texture(texture_id), + texture_cache_(std::move(texture_cache)), + external_texture_(std::move(external_texture)) { + FML_DCHECK(texture_cache_); + FML_DCHECK(external_texture_); +} IOSExternalTextureMetal::~IOSExternalTextureMetal() = default; @@ -18,27 +31,230 @@ bool freeze, GrDirectContext* context, const SkSamplingOptions& sampling) { - [darwin_external_texture_metal_ paint:canvas - bounds:bounds - freeze:freeze - grContext:context - sampling:sampling]; + const bool needs_updated_texture = (!freeze && texture_frame_available_) || !external_image_; + + if (needs_updated_texture) { + auto pixel_buffer = fml::CFRef([external_texture_ copyPixelBuffer]); + if (!pixel_buffer) { + pixel_buffer = std::move(last_pixel_buffer_); + } else { + pixel_format_ = CVPixelBufferGetPixelFormatType(pixel_buffer); + } + + // If the application told us there was a texture frame available but did not provide one when + // asked for it, reuse the previous texture but make sure to ask again the next time around. + if (auto wrapped_texture = WrapExternalPixelBuffer(pixel_buffer, context)) { + external_image_ = wrapped_texture; + texture_frame_available_ = false; + last_pixel_buffer_ = std::move(pixel_buffer); + } + } + + if (external_image_) { + canvas.drawImageRect(external_image_, // image + SkRect::Make(external_image_->bounds()), // source rect + bounds, // destination rect + sampling, + nullptr, // paint + SkCanvas::SrcRectConstraint::kFast_SrcRectConstraint // constraint + ); + } +} + +sk_sp IOSExternalTextureMetal::WrapExternalPixelBuffer( + fml::CFRef pixel_buffer, + GrDirectContext* context) const { + if (!pixel_buffer) { + return nullptr; + } + + sk_sp image = nullptr; + if (pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange || + pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) { + image = WrapNV12ExternalPixelBuffer(pixel_buffer, context); + } else { + image = WrapRGBAExternalPixelBuffer(pixel_buffer, context); + } + + if (!image) { + FML_DLOG(ERROR) << "Could not wrap Metal texture as a Skia image."; + } + + return image; +} + +sk_sp IOSExternalTextureMetal::WrapNV12ExternalPixelBuffer( + fml::CFRef pixel_buffer, + GrDirectContext* context) const { + auto texture_size = + SkISize::Make(CVPixelBufferGetWidth(pixel_buffer), CVPixelBufferGetHeight(pixel_buffer)); + CVMetalTextureRef y_metal_texture_raw = nullptr; + { + auto cv_return = + CVMetalTextureCacheCreateTextureFromImage(/*allocator=*/kCFAllocatorDefault, + /*textureCache=*/texture_cache_, + /*sourceImage=*/pixel_buffer, + /*textureAttributes=*/nullptr, + /*pixelFormat=*/MTLPixelFormatR8Unorm, + /*width=*/texture_size.width(), + /*height=*/texture_size.height(), + /*planeIndex=*/0u, + /*texture=*/&y_metal_texture_raw); + + if (cv_return != kCVReturnSuccess) { + FML_DLOG(ERROR) << "Could not create Metal texture from pixel buffer: CVReturn " << cv_return; + return nullptr; + } + } + + CVMetalTextureRef uv_metal_texture_raw = nullptr; + { + auto cv_return = + CVMetalTextureCacheCreateTextureFromImage(/*allocator=*/kCFAllocatorDefault, + /*textureCache=*/texture_cache_, + /*sourceImage=*/pixel_buffer, + /*textureAttributes=*/nullptr, + /*pixelFormat=*/MTLPixelFormatRG8Unorm, + /*width=*/texture_size.width() / 2, + /*height=*/texture_size.height() / 2, + /*planeIndex=*/1u, + /*texture=*/&uv_metal_texture_raw); + + if (cv_return != kCVReturnSuccess) { + FML_DLOG(ERROR) << "Could not create Metal texture from pixel buffer: CVReturn " << cv_return; + return nullptr; + } + } + + fml::CFRef y_metal_texture(y_metal_texture_raw); + + GrMtlTextureInfo y_skia_texture_info; + y_skia_texture_info.fTexture = sk_cf_obj{ + [reinterpret_cast(CVMetalTextureGetTexture(y_metal_texture)) retain]}; + + GrBackendTexture skia_backend_textures[2]; + skia_backend_textures[0] = GrBackendTexture(/*width=*/texture_size.width(), + /*height=*/texture_size.height(), + /*mipMapped=*/GrMipMapped ::kNo, + /*textureInfo=*/y_skia_texture_info); + + fml::CFRef uv_metal_texture(uv_metal_texture_raw); + + GrMtlTextureInfo uv_skia_texture_info; + uv_skia_texture_info.fTexture = sk_cf_obj{ + [reinterpret_cast(CVMetalTextureGetTexture(uv_metal_texture)) retain]}; + + skia_backend_textures[1] = GrBackendTexture(/*width=*/texture_size.width(), + /*height=*/texture_size.height(), + /*mipMapped=*/GrMipMapped ::kNo, + /*textureInfo=*/uv_skia_texture_info); + SkYUVAInfo yuva_info(skia_backend_textures[0].dimensions(), SkYUVAInfo::PlaneConfig::kY_UV, + SkYUVAInfo::Subsampling::k444, kRec601_SkYUVColorSpace); + GrYUVABackendTextures yuva_backend_textures(yuva_info, skia_backend_textures, + kTopLeft_GrSurfaceOrigin); + + struct ImageCaptures { + fml::CFRef buffer; + fml::CFRef y_texture; + fml::CFRef uv_texture; + }; + + auto captures = std::make_unique(); + captures->buffer = std::move(pixel_buffer); + captures->y_texture = std::move(y_metal_texture); + captures->uv_texture = std::move(uv_metal_texture); + + SkImage::TextureReleaseProc release_proc = [](SkImage::ReleaseContext release_context) { + auto captures = reinterpret_cast(release_context); + delete captures; + }; + sk_sp image = + SkImage::MakeFromYUVATextures(context, yuva_backend_textures, /*imageColorSpace=*/nullptr, + release_proc, captures.release()); + return image; +} + +sk_sp IOSExternalTextureMetal::WrapRGBAExternalPixelBuffer( + fml::CFRef pixel_buffer, + GrDirectContext* context) const { + auto texture_size = + SkISize::Make(CVPixelBufferGetWidth(pixel_buffer), CVPixelBufferGetHeight(pixel_buffer)); + CVMetalTextureRef metal_texture_raw = nullptr; + auto cv_return = + CVMetalTextureCacheCreateTextureFromImage(/*allocator=*/kCFAllocatorDefault, + /*textureCache=*/texture_cache_, + /*sourceImage=*/pixel_buffer, + /*textureAttributes=*/nullptr, + /*pixelFormat=*/MTLPixelFormatBGRA8Unorm, + /*width=*/texture_size.width(), + /*height=*/texture_size.height(), + /*planeIndex=*/0u, + /*texture=*/&metal_texture_raw); + + if (cv_return != kCVReturnSuccess) { + FML_DLOG(ERROR) << "Could not create Metal texture from pixel buffer: CVReturn " << cv_return; + return nullptr; + } + + fml::CFRef metal_texture(metal_texture_raw); + + GrMtlTextureInfo skia_texture_info; + skia_texture_info.fTexture = sk_cf_obj{ + [reinterpret_cast(CVMetalTextureGetTexture(metal_texture)) retain]}; + + GrBackendTexture skia_backend_texture(/*width=*/texture_size.width(), + /*height=*/texture_size.height(), + /*mipMapped=*/GrMipMapped ::kNo, + /*textureInfo=*/skia_texture_info); + + struct ImageCaptures { + fml::CFRef buffer; + fml::CFRef texture; + }; + + auto captures = std::make_unique(); + captures->buffer = std::move(pixel_buffer); + captures->texture = std::move(metal_texture); + + SkImage::TextureReleaseProc release_proc = [](SkImage::ReleaseContext release_context) { + auto captures = reinterpret_cast(release_context); + delete captures; + }; + + auto image = + SkImage::MakeFromTexture(context, skia_backend_texture, kTopLeft_GrSurfaceOrigin, + kBGRA_8888_SkColorType, kPremul_SkAlphaType, + /*imageColorSpace=*/nullptr, release_proc, captures.release() + + ); + return image; } void IOSExternalTextureMetal::OnGrContextCreated() { - [darwin_external_texture_metal_ onGrContextCreated]; + // External images in this backend have no thread affinity and are not tied to the context in any + // way. Instead, they are tied to the Metal device which is associated with the cache already and + // is consistent throughout the shell run. } void IOSExternalTextureMetal::OnGrContextDestroyed() { - [darwin_external_texture_metal_ onGrContextDestroyed]; + // The image must be reset because it is tied to the onscreen context. But the pixel buffer that + // created the image is still around. In case of context reacquisition, that last pixel + // buffer will be used to materialize the image in case the application fails to provide a new + // one. + external_image_.reset(); + CVMetalTextureCacheFlush(texture_cache_, // cache + 0 // options (must be zero) + ); } void IOSExternalTextureMetal::MarkNewFrameAvailable() { - [darwin_external_texture_metal_ markNewFrameAvailable]; + texture_frame_available_ = true; } void IOSExternalTextureMetal::OnTextureUnregistered() { - [darwin_external_texture_metal_ onTextureUnregistered]; + if ([external_texture_ respondsToSelector:@selector(onTextureUnregistered:)]) { + [external_texture_ onTextureUnregistered:external_texture_]; + } } } // namespace flutter diff --git a/shell/platform/darwin/macos/BUILD.gn b/shell/platform/darwin/macos/BUILD.gn index b68a1431a8e73..72e029e2aa0fc 100644 --- a/shell/platform/darwin/macos/BUILD.gn +++ b/shell/platform/darwin/macos/BUILD.gn @@ -115,7 +115,7 @@ source_set("flutter_framework_source") { "FLUTTER_ENGINE_NO_PROTOTYPES", ] - cflags_objcc = flutter_cflags_objcc_arc + cflags_objcc = [ "-fobjc-arc" ] libs = [ "Cocoa.framework", @@ -161,7 +161,7 @@ executable("flutter_desktop_darwin_unittests") { sources += [ "framework/Source/FlutterOpenGLRendererTest.mm" ] } - cflags_objcc = flutter_cflags_objcc_arc + cflags_objcc = [ "-fobjc-arc" ] ldflags = [ "-ObjC" ]