Simplify video buffer capturing

Sources/LiveKit/Extensions/RTCVideoCapturerDelegate+Buffer.swift

@@ -68,91 +68,6 @@ extension CGImagePropertyOrientation {
     }
 }
 
-extension LKRTCVideoCapturerDelegate {
-    typealias OnResolveSourceDimensions = (Dimensions) -> Void
-    typealias OnDidCreateFrame = (LKRTCVideoFrame) -> Void
-
-    /// capture a `CVPixelBuffer`, all other capture methods call this method internally.
-    func capturer(_ capturer: LKRTCVideoCapturer,
-                  didCapture pixelBuffer: CVPixelBuffer,
-                  timeStampNs: Int64 = VideoCapturer.createTimeStampNs(),
-                  rotation: RTCVideoRotation = ._0,
-
-                  onDidCreateFrame: OnDidCreateFrame? = nil)
-    {
-        // check if pixel format is supported by WebRTC
-        let pixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer)
-        guard VideoCapturer.supportedPixelFormats.contains(where: { $0.uint32Value == pixelFormat }) else {
-            // kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
-            // kCVPixelFormatType_420YpCbCr8BiPlanarFullRange
-            // kCVPixelFormatType_32BGRA
-            // kCVPixelFormatType_32ARGB
-            logger.log("Skipping capture for unsupported pixel format: \(pixelFormat.toString())", .warning,
-                       type: type(of: self))
-            return
-        }
-
-        let sourceDimensions = Dimensions(width: Int32(CVPixelBufferGetWidth(pixelBuffer)),
-                                          height: Int32(CVPixelBufferGetHeight(pixelBuffer)))
-
-        guard sourceDimensions.isEncodeSafe else {
-            logger.log("Skipping capture for dimensions: \(sourceDimensions)", .warning,
-                       type: type(of: self))
-            return
-        }
-
-        let rtcBuffer = LKRTCCVPixelBuffer(pixelBuffer: pixelBuffer)
-        let rtcFrame = LKRTCVideoFrame(buffer: rtcBuffer,
-                                       rotation: rotation,
-                                       timeStampNs: timeStampNs)
-
-        self.capturer(capturer, didCapture: rtcFrame)
-        onDidCreateFrame?(rtcFrame)
-    }
-
-    /// capture a `CMSampleBuffer`
-    func capturer(_ capturer: LKRTCVideoCapturer,
-                  didCapture sampleBuffer: CMSampleBuffer,
-
-                  onDidCreateFrame: OnDidCreateFrame? = nil)
-    {
-        // check if buffer is ready
-        guard CMSampleBufferGetNumSamples(sampleBuffer) == 1,
-              CMSampleBufferIsValid(sampleBuffer),
-              CMSampleBufferDataIsReady(sampleBuffer)
-        else {
-            logger.log("Failed to capture, buffer is not ready", .warning, type: type(of: self))
-            return
-        }
-
-        // attempt to determine rotation information if buffer is coming from ReplayKit
-        var rotation: RTCVideoRotation?
-        if #available(macOS 11.0, *) {
-            // Check rotation tags. Extensions see these tags, but `RPScreenRecorder` does not appear to set them.
-            // On iOS 12.0 and 13.0 rotation tags (other than up) are set by extensions.
-            if let sampleOrientation = CMGetAttachment(sampleBuffer, key: RPVideoSampleOrientationKey as CFString, attachmentModeOut: nil),
-               let coreSampleOrientation = sampleOrientation.uint32Value
-            {
-                rotation = CGImagePropertyOrientation(rawValue: coreSampleOrientation)?.toRTCRotation()
-            }
-        }
-
-        guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else {
-            logger.log("Failed to capture, pixel buffer not found", .warning, type: type(of: self))
-            return
-        }
-
-        let timeStamp = CMSampleBufferGetPresentationTimeStamp(sampleBuffer)
-        let timeStampNs = Int64(CMTimeGetSeconds(timeStamp) * Double(NSEC_PER_SEC))
-
-        self.capturer(capturer,
-                      didCapture: pixelBuffer,
-                      timeStampNs: timeStampNs,
-                      rotation: rotation ?? ._0,
-                      onDidCreateFrame: onDidCreateFrame)
-    }
-}
-
 extension CVPixelBuffer {
     func toDimensions() -> Dimensions {
         Dimensions(width: Int32(CVPixelBufferGetWidth(self)),

Sources/LiveKit/Track/Capturers/BufferCapturer.swift

@@ -45,12 +45,18 @@ public class BufferCapturer: VideoCapturer {
 
     /// Capture a ``CMSampleBuffer``.
     public func capture(_ sampleBuffer: CMSampleBuffer) {
-        capture(sampleBuffer: sampleBuffer, capturer: capturer, options: options)
+        capture(sampleBuffer: sampleBuffer,
+                capturer: capturer,
+                options: options)
     }
 
     /// Capture a ``CVPixelBuffer``.
     public func capture(_ pixelBuffer: CVPixelBuffer, timeStampNs: Int64 = VideoCapturer.createTimeStampNs(), rotation: VideoRotation = ._0) {
-        capture(pixelBuffer: pixelBuffer, capturer: capturer, timeStampNs: timeStampNs, rotation: rotation, options: options)
+        capture(pixelBuffer: pixelBuffer,
+                capturer: capturer,
+                timeStampNs: timeStampNs,
+                rotation: rotation,
+                options: options)
     }
 }
 

Sources/LiveKit/Track/Capturers/VideoCapturer.swift

@@ -22,6 +22,10 @@ internal import LiveKitWebRTC
 @_implementationOnly import LiveKitWebRTC
 #endif
 
+#if canImport(ReplayKit)
+import ReplayKit
+#endif
+
 protocol VideoCapturerProtocol {
     var capturer: LKRTCVideoCapturer { get }
 }
@@ -202,36 +206,96 @@ extension VideoCapturer {
                  device: AVCaptureDevice? = nil,
                  options: VideoCaptureOptions)
     {
-        _processFrame(frame, capturer: capturer, device: device, options: options)
+        _process(frame: frame,
+                 capturer: capturer,
+                 device: device,
+                 options: options)
     }
 
-    // Capture a CMSampleBuffer
-    func capture(sampleBuffer: CMSampleBuffer,
+    // Capture a CVPixelBuffer
+    func capture(pixelBuffer: CVPixelBuffer,
                  capturer: LKRTCVideoCapturer,
+                 timeStampNs: Int64 = VideoCapturer.createTimeStampNs(),
+                 rotation: VideoRotation = ._0,
                  options: VideoCaptureOptions)
     {
-        delegate?.capturer(capturer, didCapture: sampleBuffer) { [weak self] frame in
-            self?._processFrame(frame, capturer: capturer, device: nil, options: options)
+        // check if pixel format is supported by WebRTC
+        let pixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer)
+        guard VideoCapturer.supportedPixelFormats.contains(where: { $0.uint32Value == pixelFormat }) else {
+            // kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
+            // kCVPixelFormatType_420YpCbCr8BiPlanarFullRange
+            // kCVPixelFormatType_32BGRA
+            // kCVPixelFormatType_32ARGB
+            logger.log("Skipping capture for unsupported pixel format: \(pixelFormat.toString())", .warning,
+                       type: type(of: self))
+            return
         }
+
+        let sourceDimensions = Dimensions(width: Int32(CVPixelBufferGetWidth(pixelBuffer)),
+                                          height: Int32(CVPixelBufferGetHeight(pixelBuffer)))
+
+        guard sourceDimensions.isEncodeSafe else {
+            logger.log("Skipping capture for dimensions: \(sourceDimensions)", .warning,
+                       type: type(of: self))
+            return
+        }
+
+        let rtcBuffer = LKRTCCVPixelBuffer(pixelBuffer: pixelBuffer)
+        let rtcFrame = LKRTCVideoFrame(buffer: rtcBuffer,
+                                       rotation: rotation.toRTCType(),
+                                       timeStampNs: timeStampNs)
+
+        capture(frame: rtcFrame,
+                capturer: capturer,
+                options: options)
     }
 
-    // Capture a CVPixelBuffer
-    func capture(pixelBuffer: CVPixelBuffer,
+    // Capture a CMSampleBuffer
+    func capture(sampleBuffer: CMSampleBuffer,
                  capturer: LKRTCVideoCapturer,
-                 timeStampNs: Int64 = VideoCapturer.createTimeStampNs(),
-                 rotation: VideoRotation = ._0,
                  options: VideoCaptureOptions)
     {
-        delegate?.capturer(capturer, didCapture: pixelBuffer, timeStampNs: timeStampNs, rotation: rotation.toRTCType()) { [weak self] frame in
-            self?._processFrame(frame, capturer: capturer, device: nil, options: options)
+        // Check if buffer is ready
+        guard CMSampleBufferGetNumSamples(sampleBuffer) == 1,
+              CMSampleBufferIsValid(sampleBuffer),
+              CMSampleBufferDataIsReady(sampleBuffer)
+        else {
+            logger.log("Failed to capture, buffer is not ready", .warning, type: type(of: self))
+            return
         }
+
+        // attempt to determine rotation information if buffer is coming from ReplayKit
+        var rotation: RTCVideoRotation?
+        if #available(macOS 11.0, *) {
+            // Check rotation tags. Extensions see these tags, but `RPScreenRecorder` does not appear to set them.
+            // On iOS 12.0 and 13.0 rotation tags (other than up) are set by extensions.
+            if let sampleOrientation = CMGetAttachment(sampleBuffer, key: RPVideoSampleOrientationKey as CFString, attachmentModeOut: nil),
+               let coreSampleOrientation = sampleOrientation.uint32Value
+            {
+                rotation = CGImagePropertyOrientation(rawValue: coreSampleOrientation)?.toRTCRotation()
+            }
+        }
+
+        guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else {
+            logger.log("Failed to capture, pixel buffer not found", .warning, type: type(of: self))
+            return
+        }
+
+        let timeStamp = CMSampleBufferGetPresentationTimeStamp(sampleBuffer)
+        let timeStampNs = Int64(CMTimeGetSeconds(timeStamp) * Double(NSEC_PER_SEC))
+
+        capture(pixelBuffer: pixelBuffer,
+                capturer: capturer,
+                timeStampNs: timeStampNs,
+                rotation: rotation?.toLKType() ?? ._0,
+                options: options)
     }
 
     // Process the captured frame
-    private func _processFrame(_ frame: LKRTCVideoFrame,
-                               capturer: LKRTCVideoCapturer,
-                               device: AVCaptureDevice?,
-                               options: VideoCaptureOptions)
+    private func _process(frame: LKRTCVideoFrame,
+                          capturer: LKRTCVideoCapturer,
+                          device: AVCaptureDevice?,
+                          options: VideoCaptureOptions)
     {
         if _state.isFrameProcessingBusy {
             log("Frame processing hasn't completed yet, skipping frame...", .warning)

Tests/LiveKitTests/PublishBufferCapturerTests.swift

@@ -20,6 +20,22 @@ import XCTest
 
 class PublishBufferCapturerTests: LKTestCase {
     func testPublishBufferTrack() async throws {
+        let testCodecs: [VideoCodec] = [.vp8]
+        for codec in testCodecs {
+            print("Testing with codec: \(codec)")
+            let publishOptions = VideoPublishOptions(
+                simulcast: false,
+                preferredCodec: codec,
+                preferredBackupCodec: .none,
+                degradationPreference: .maintainResolution
+            )
+            try await testWith(publishOptions: publishOptions)
+        }
+    }
+}
+
+extension PublishBufferCapturerTests {
+    func testWith(publishOptions: VideoPublishOptions) async throws {
         try await withRooms([RoomTestingOptions(canPublish: true), RoomTestingOptions(canSubscribe: true)]) { rooms in
             // Alias to Rooms
             let room1 = rooms[0]
@@ -29,13 +45,6 @@ class PublishBufferCapturerTests: LKTestCase {
 
             let captureOptions = BufferCaptureOptions(dimensions: targetDimensions)
 
-            let publishOptions = VideoPublishOptions(
-                simulcast: false,
-                preferredCodec: .vp8,
-                preferredBackupCodec: .none,
-                degradationPreference: .maintainResolution
-            )
-
             let bufferTrack = LocalVideoTrack.createBufferTrack(
                 options: captureOptions
             )
@@ -91,9 +100,18 @@ class PublishBufferCapturerTests: LKTestCase {
 
             print("Waiting for target dimensions: \(targetDimensions)")
             let expectTargetDimensions = videoTrackWatcher.expect(dimensions: targetDimensions)
-            await self.fulfillment(of: [expectTargetDimensions], timeout: 60)
+            await self.fulfillment(of: [expectTargetDimensions], timeout: 120)
             print("Did render target dimensions: \(targetDimensions)")
 
+            // Verify codec information
+            print("Waiting for codec information...")
+            if let codec = publishOptions.preferredCodec {
+                let expectCodec = videoTrackWatcher.expect(codec: codec)
+                await self.fulfillment(of: [expectCodec], timeout: 60)
+                print("Detected codecs: \(videoTrackWatcher.detectedCodecs.joined(separator: ", "))")
+                XCTAssertTrue(videoTrackWatcher.isCodecDetected(codec: codec), "Expected codec \(codec) was not detected")
+            }
+
             // Wait for video to complete...
             try await captureTask.value
             // Clean up