fix(🍏): Use Metal for creating SkImage from CVSampleBuffer (#2356)

package/ios/RNSkia-iOS/RNSkiOSPlatformContext.mm

@@ -1,11 +1,12 @@
-#include "RNSkiOSPlatformContext.h"
+#import "RNSkiOSPlatformContext.h"
 
 #import <CoreMedia/CMSampleBuffer.h>
 #import <React/RCTUtils.h>
 #include <thread>
 #include <utility>
 
-#include "SkiaMetalSurfaceFactory.h"
+#import "SkiaCVPixelBufferUtils.h"
+#import "SkiaMetalSurfaceFactory.h"
 
 #pragma clang diagnostic push
 #pragma clang diagnostic ignored "-Wdocumentation"
@@ -69,74 +70,106 @@
 }
 
 uint64_t RNSkiOSPlatformContext::makePlatformBuffer(sk_sp<SkImage> image) {
+  // 0. If Image is not in BGRA, convert to BGRA as only BGRA is supported.
+  if (image->colorType() != kBGRA_8888_SkColorType) {
+    // on iOS, 32_BGRA is the only supported RGB format for CVPixelBuffers.
+    image = image->makeColorTypeAndColorSpace(
+        ThreadContextHolder::ThreadSkiaMetalContext.skContext.get(),
+        kBGRA_8888_SkColorType, SkColorSpace::MakeSRGB());
+    if (image == nullptr) {
+      throw std::runtime_error(
+          "Failed to convert image to BGRA_8888 colortype! Only BGRA_8888 "
+          "PlatformBuffers are supported.");
+    }
+  }
+
+  // 1. Get image info
   auto bytesPerPixel = image->imageInfo().bytesPerPixel();
   int bytesPerRow = image->width() * bytesPerPixel;
   auto buf = SkData::MakeUninitialized(image->width() * image->height() *
                                        bytesPerPixel);
   SkImageInfo info = SkImageInfo::Make(image->width(), image->height(),
                                        image->colorType(), image->alphaType());
+  // 2. Copy pixels into our buffer
   image->readPixels(nullptr, info, const_cast<void *>(buf->data()), bytesPerRow,
                     0, 0);
-  auto pixelData = const_cast<void *>(buf->data());
 
-  // Create a CVPixelBuffer from the raw pixel data
-  CVPixelBufferRef pixelBuffer = nullptr;
-  // OSType pixelFormatType = MapSkColorTypeToOSType(image->colorType());
-
-  // You will need to fill in the details for creating the pixel buffer
-  // CVPixelBufferCreateWithBytes or CVPixelBufferCreateWithPlanarBytes
-  // Create the CVPixelBuffer with the image data
-  void *context = static_cast<void *>(
-      new sk_sp<SkData>(buf)); // Create a copy for the context
-  CVReturn r = CVPixelBufferCreateWithBytes(
-      nullptr, // allocator
-      image->width(), image->height(), kCVPixelFormatType_32BGRA,
-      pixelData,                                         // pixel data
-      bytesPerRow,                                       // bytes per row
-      [](void *releaseRefCon, const void *baseAddress) { // release callback
-        auto buf = static_cast<sk_sp<SkData> *>(releaseRefCon);
-        buf->reset(); // This effectively calls unref on the SkData object
-        delete buf;   // Cleanup the dynamically allocated context
-      },
-      context,     // release callback context
-      nullptr,     // pixel buffer attributes
-      &pixelBuffer // the newly created pixel buffer
-  );
-
-  if (r != kCVReturnSuccess) {
-    return 0; // or handle error appropriately
+  // 3. Create an IOSurface (GPU + CPU memory)
+  CFMutableDictionaryRef dict = CFDictionaryCreateMutable(
+      kCFAllocatorDefault, 0, &kCFTypeDictionaryKeyCallBacks,
+      &kCFTypeDictionaryValueCallBacks);
+  int width = image->width();
+  int height = image->height();
+  int pitch = width * bytesPerPixel;
+  int size = width * height * bytesPerPixel;
+  OSType pixelFormat = kCVPixelFormatType_32BGRA;
+  CFDictionarySetValue(
+      dict, kIOSurfaceBytesPerRow,
+      CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &pitch));
+  CFDictionarySetValue(
+      dict, kIOSurfaceBytesPerElement,
+      CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &bytesPerPixel));
+  CFDictionarySetValue(
+      dict, kIOSurfaceWidth,
+      CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &width));
+  CFDictionarySetValue(
+      dict, kIOSurfaceHeight,
+      CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &height));
+  CFDictionarySetValue(
+      dict, kIOSurfacePixelFormat,
+      CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &pixelFormat));
+  CFDictionarySetValue(
+      dict, kIOSurfaceAllocSize,
+      CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &size));
+  IOSurfaceRef surface = IOSurfaceCreate(dict);
+  if (surface == nil) {
+    throw std::runtime_error("Failed to create " + std::to_string(width) + "x" +
+                             std::to_string(height) + " IOSurface!");
   }
 
-  // Wrap the CVPixelBuffer in a CMSampleBuffer
-  CMSampleBufferRef sampleBuffer = nullptr;
+  // 4. Copy over the memory from the pixels into the IOSurface
+  IOSurfaceLock(surface, 0, nil);
+  void *base = IOSurfaceGetBaseAddress(surface);
+  memcpy(base, buf->data(), buf->size());
+  IOSurfaceUnlock(surface, 0, nil);
 
+  // 5. Create a CVPixelBuffer from the IOSurface
+  CVPixelBufferRef pixelBuffer = nullptr;
+  CVReturn result =
+      CVPixelBufferCreateWithIOSurface(nil, surface, nil, &pixelBuffer);
+  if (result != kCVReturnSuccess) {
+    throw std::runtime_error(
+        "Failed to create CVPixelBuffer from SkImage! Return value: " +
+        std::to_string(result));
+  }
+
+  // 6. Create CMSampleBuffer base information
   CMFormatDescriptionRef formatDescription = nullptr;
   CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, pixelBuffer,
                                                &formatDescription);
-
-  // Assuming no specific timing is required, we initialize the timing info to
-  // zero.
   CMSampleTimingInfo timingInfo = {0};
-  timingInfo.duration = kCMTimeInvalid; // Indicate an unknown duration.
-  timingInfo.presentationTimeStamp = kCMTimeZero; // Start at time zero.
-  timingInfo.decodeTimeStamp = kCMTimeInvalid;    // No specific decode time.
+  timingInfo.duration = kCMTimeInvalid;
+  timingInfo.presentationTimeStamp = kCMTimeZero;
+  timingInfo.decodeTimeStamp = kCMTimeInvalid;
 
-  // Create the sample buffer.
+  // 7. Wrap the CVPixelBuffer in a CMSampleBuffer
+  CMSampleBufferRef sampleBuffer = nullptr;
   OSStatus status = CMSampleBufferCreateReadyWithImageBuffer(
       kCFAllocatorDefault, pixelBuffer, formatDescription, &timingInfo,
       &sampleBuffer);
-
   if (status != noErr) {
     if (formatDescription) {
       CFRelease(formatDescription);
     }
     if (pixelBuffer) {
       CFRelease(pixelBuffer);
     }
-    return 0;
+    throw std::runtime_error(
+        "Failed to wrap CVPixelBuffer in CMSampleBuffer! Return value: " +
+        std::to_string(status));
   }
 
-  // Return sampleBuffer casted to uint64_t
+  // 8. Return CMsampleBuffer casted to uint64_t
   return reinterpret_cast<uint64_t>(sampleBuffer);
 }
 
@@ -152,32 +185,7 @@
 sk_sp<SkImage>
 RNSkiOSPlatformContext::makeImageFromPlatformBuffer(void *buffer) {
   CMSampleBufferRef sampleBuffer = (CMSampleBufferRef)buffer;
-  // DO the CPU transfer (debugging only)
-  //  Step 1: Extract the CVPixelBufferRef from the CMSampleBufferRef
-  CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
-
-  // Step 2: Lock the pixel buffer to access the raw pixel data
-  CVPixelBufferLockBaseAddress(pixelBuffer, 0);
-
-  // Step 3: Get information about the image
-  void *baseAddress = CVPixelBufferGetBaseAddress(pixelBuffer);
-  size_t width = CVPixelBufferGetWidth(pixelBuffer);
-  size_t height = CVPixelBufferGetHeight(pixelBuffer);
-  size_t bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer);
-
-  // Assuming the pixel format is 32BGRA, which is common for iOS video frames.
-  // You might need to adjust this based on the actual pixel format.
-  SkImageInfo info = SkImageInfo::Make(width, height, kRGBA_8888_SkColorType,
-                                       kUnpremul_SkAlphaType);
-
-  // Step 4: Create an SkImage from the pixel buffer
-  sk_sp<SkData> data =
-      SkData::MakeWithoutCopy(baseAddress, height * bytesPerRow);
-  sk_sp<SkImage> image = SkImages::RasterFromData(info, data, bytesPerRow);
-  auto texture = SkiaMetalSurfaceFactory::makeTextureFromImage(image);
-  // Step 5: Unlock the pixel buffer
-  CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);
-  return texture;
+  return SkiaMetalSurfaceFactory::makeTextureFromCMSampleBuffer(sampleBuffer);
 }
 
 sk_sp<SkFontMgr> RNSkiOSPlatformContext::createFontMgr() {

package/ios/RNSkia-iOS/SkiaCVPixelBufferUtils.h

@@ -0,0 +1,52 @@
+//
+//  SkiaCVPixelBufferUtils.h
+//  react-native-skia
+//
+//  Created by Marc Rousavy on 10.04.24.
+//
+
+#pragma once
+#import <CoreMedia/CMSampleBuffer.h>
+#import <CoreVideo/CVMetalTextureCache.h>
+#import <MetalKit/MetalKit.h>
+
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wdocumentation"
+#import "include/core/SkColorSpace.h"
+#import <include/gpu/GrBackendSurface.h>
+#pragma clang diagnostic pop
+
+class SkiaCVPixelBufferUtils {
+public:
+  enum class CVPixelBufferBaseFormat { rgb };
+
+  /**
+   Get the base format (currently only RGB) of the PixelBuffer.
+   Depending on the base-format, different methods have to be used to create
+   Skia buffers.
+   */
+  static CVPixelBufferBaseFormat
+  getCVPixelBufferBaseFormat(CVPixelBufferRef pixelBuffer);
+
+  class RGB {
+  public:
+    /**
+     Gets the Skia Color Type of the RGB pixel-buffer.
+     */
+    static SkColorType getCVPixelBufferColorType(CVPixelBufferRef pixelBuffer);
+    /**
+     Gets a GPU-backed Skia Texture for the given RGB CVPixelBuffer.
+     */
+    static GrBackendTexture
+    getSkiaTextureForCVPixelBuffer(CVPixelBufferRef pixelBuffer);
+  };
+
+private:
+  static CVMetalTextureCacheRef getTextureCache();
+  static GrBackendTexture
+  getSkiaTextureForCVPixelBufferPlane(CVPixelBufferRef pixelBuffer,
+                                      size_t planeIndex);
+  static MTLPixelFormat
+  getMTLPixelFormatForCVPixelBufferPlane(CVPixelBufferRef pixelBuffer,
+                                         size_t planeIndex);
+};

package/ios/RNSkia-iOS/SkiaCVPixelBufferUtils.mm

@@ -0,0 +1,149 @@
+//
+//  SkiaCVPixelBufferUtils.mm
+//  react-native-skia
+//
+//  Created by Marc Rousavy on 10.04.24.
+//
+
+#import "SkiaCVPixelBufferUtils.h"
+#import "SkiaMetalSurfaceFactory.h"
+
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wdocumentation"
+#import "include/core/SkColorSpace.h"
+#import <include/gpu/GrBackendSurface.h>
+#pragma clang diagnostic pop
+
+#include <TargetConditionals.h>
+#if TARGET_RT_BIG_ENDIAN
+#define FourCC2Str(fourcc)                                                     \
+  (const char[]) {                                                             \
+    *((char *)&fourcc), *(((char *)&fourcc) + 1), *(((char *)&fourcc) + 2),    \
+        *(((char *)&fourcc) + 3), 0                                            \
+  }
+#else
+#define FourCC2Str(fourcc)                                                     \
+  (const char[]) {                                                             \
+    *(((char *)&fourcc) + 3), *(((char *)&fourcc) + 2),                        \
+        *(((char *)&fourcc) + 1), *(((char *)&fourcc) + 0), 0                  \
+  }
+#endif
+
+// pragma MARK: Base
+
+SkiaCVPixelBufferUtils::CVPixelBufferBaseFormat
+SkiaCVPixelBufferUtils::getCVPixelBufferBaseFormat(
+    CVPixelBufferRef pixelBuffer) {
+  OSType format = CVPixelBufferGetPixelFormatType(pixelBuffer);
+
+  switch (format) {
+  case kCVPixelFormatType_32BGRA:
+  case kCVPixelFormatType_32RGBA:
+    return CVPixelBufferBaseFormat::rgb;
+  default:
+    [[unlikely]] throw std::runtime_error(
+        "CVPixelBuffer has unsupported pixel-format! " +
+        std::string(FourCC2Str(format)));
+  }
+}
+
+// pragma MARK: RGB
+
+SkColorType SkiaCVPixelBufferUtils::RGB::getCVPixelBufferColorType(
+    CVPixelBufferRef pixelBuffer) {
+  OSType format = CVPixelBufferGetPixelFormatType(pixelBuffer);
+
+  switch (format) {
+  case kCVPixelFormatType_32BGRA:
+    [[likely]] return kBGRA_8888_SkColorType;
+  case kCVPixelFormatType_32RGBA:
+    return kRGBA_8888_SkColorType;
+  // This can be extended with branches for specific RGB formats if new Apple
+  // uses new formats.
+  default:
+    [[unlikely]] throw std::runtime_error(
+        "CVPixelBuffer has unknown RGB format! " +
+        std::string(FourCC2Str(format)));
+  }
+}
+
+GrBackendTexture SkiaCVPixelBufferUtils::RGB::getSkiaTextureForCVPixelBuffer(
+    CVPixelBufferRef pixelBuffer) {
+  return getSkiaTextureForCVPixelBufferPlane(pixelBuffer, /* planeIndex */ 0);
+}
+
+// pragma MARK: CVPixelBuffer -> Skia Texture
+
+GrBackendTexture SkiaCVPixelBufferUtils::getSkiaTextureForCVPixelBufferPlane(
+    CVPixelBufferRef pixelBuffer, size_t planeIndex) {
+  // 1. Get cache
+  CVMetalTextureCacheRef textureCache = getTextureCache();
+
+  // 2. Get MetalTexture from CMSampleBuffer
+  CVMetalTextureRef textureHolder;
+  size_t width = CVPixelBufferGetWidthOfPlane(pixelBuffer, planeIndex);
+  size_t height = CVPixelBufferGetHeightOfPlane(pixelBuffer, planeIndex);
+  MTLPixelFormat pixelFormat =
+      getMTLPixelFormatForCVPixelBufferPlane(pixelBuffer, planeIndex);
+  CVReturn result = CVMetalTextureCacheCreateTextureFromImage(
+      kCFAllocatorDefault, textureCache, pixelBuffer, nil, pixelFormat, width,
+      height, planeIndex, &textureHolder);
+  if (result != kCVReturnSuccess) [[unlikely]] {
+    throw std::runtime_error(
+        "Failed to create Metal Texture from CMSampleBuffer! Result: " +
+        std::to_string(result));
+  }
+
+  // 2. Unwrap the underlying MTLTexture
+  id<MTLTexture> mtlTexture = CVMetalTextureGetTexture(textureHolder);
+  if (mtlTexture == nil) [[unlikely]] {
+    throw std::runtime_error(
+        "Failed to get MTLTexture from CVMetalTextureRef!");
+  }
+
+  // 3. Wrap MTLTexture in Skia's GrBackendTexture
+  GrMtlTextureInfo textureInfo;
+  textureInfo.fTexture.retain((__bridge void *)mtlTexture);
+  GrBackendTexture texture =
+      GrBackendTexture((int)mtlTexture.width, (int)mtlTexture.height,
+                       skgpu::Mipmapped::kNo, textureInfo);
+  CFRelease(textureHolder);
+  return texture;
+}
+
+// pragma MARK: getTextureCache()
+
+CVMetalTextureCacheRef SkiaCVPixelBufferUtils::getTextureCache() {
+  static thread_local CVMetalTextureCacheRef textureCache = nil;
+  if (textureCache == nil) {
+    // Create a new Texture Cache
+    auto result = CVMetalTextureCacheCreate(kCFAllocatorDefault, nil,
+                                            MTLCreateSystemDefaultDevice(), nil,
+                                            &textureCache);
+    if (result != kCVReturnSuccess || textureCache == nil) {
+      throw std::runtime_error("Failed to create Metal Texture Cache!");
+    }
+  }
+  return textureCache;
+}
+
+// pragma MARK: Get CVPixelBuffer MTLPixelFormat
+
+MTLPixelFormat SkiaCVPixelBufferUtils::getMTLPixelFormatForCVPixelBufferPlane(
+    CVPixelBufferRef pixelBuffer, size_t planeIndex) {
+  size_t width = CVPixelBufferGetWidthOfPlane(pixelBuffer, planeIndex);
+  size_t bytesPerRow =
+      CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, planeIndex);
+  double bytesPerPixel = round(static_cast<double>(bytesPerRow) / width);
+  if (bytesPerPixel == 1) {
+    return MTLPixelFormatR8Unorm;
+  } else if (bytesPerPixel == 2) {
+    return MTLPixelFormatRG8Unorm;
+  } else if (bytesPerPixel == 4) {
+    return MTLPixelFormatBGRA8Unorm;
+  } else [[unlikely]] {
+    throw std::runtime_error("Invalid bytes per row! Expected 1 (R), 2 (RG) or "
+                             "4 (RGBA), but received " +
+                             std::to_string(bytesPerPixel));
+  }
+}