[camera_avfoundation] ignore audio samples until first video sample (#…

…4587)

Samples from ios video and audio input should be regarded as arriving from independent streams because in fact they are going from independent sources. Video is going from instance `AVCaptureVideoDataOutput` while audio from `AVCaptureAudioDataOutput`. Therefore it is possible that one of them will be initialised soon or later than the other and the first few samples will be only audio samples so the video file will start without image data and will be black until the first video sample arrives. This patch addresses this by ignoring audio samples until the first video sample arrives (alternatively it could wait until both video and audio samples are actively arriving).

Fix flutter/flutter#57831

packages/camera/camera_avfoundation/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.9.13+5
+
+* Ignores audio samples until the first video sample arrives.
+
 ## 0.9.13+4
 
 * Adds pub topics to package metadata.

packages/camera/camera_avfoundation/example/ios/RunnerTests/CameraTestUtils.h

@@ -15,4 +15,8 @@ extern FLTCam *FLTCreateCamWithCaptureSessionQueue(dispatch_queue_t captureSessi
 /// @return a test sample buffer.
 extern CMSampleBufferRef FLTCreateTestSampleBuffer(void);
 
+/// Creates a test audio sample buffer.
+/// @return a test audio sample buffer.
+extern CMSampleBufferRef FLTCreateTestAudioSampleBuffer(void);
+
 NS_ASSUME_NONNULL_END

packages/camera/camera_avfoundation/example/ios/RunnerTests/CameraTestUtils.m

@@ -47,3 +47,22 @@ CMSampleBufferRef FLTCreateTestSampleBuffer(void) {
   CFRelease(formatDescription);
   return sampleBuffer;
 }
+
+CMSampleBufferRef FLTCreateTestAudioSampleBuffer(void) {
+  CMBlockBufferRef blockBuffer;
+  CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault, NULL, 100, kCFAllocatorDefault, NULL, 0,
+                                     100, kCMBlockBufferAssureMemoryNowFlag, &blockBuffer);
+
+  CMFormatDescriptionRef formatDescription;
+  AudioStreamBasicDescription basicDescription = {44100, kAudioFormatLinearPCM, 0, 1, 1, 1, 1, 8};
+  CMAudioFormatDescriptionCreate(kCFAllocatorDefault, &basicDescription, 0, NULL, 0, NULL, NULL,
+                                 &formatDescription);
+
+  CMSampleBufferRef sampleBuffer;
+  CMAudioSampleBufferCreateReadyWithPacketDescriptions(
+      kCFAllocatorDefault, blockBuffer, formatDescription, 1, kCMTimeZero, NULL, &sampleBuffer);
+
+  CFRelease(blockBuffer);
+  CFRelease(formatDescription);
+  return sampleBuffer;
+}

packages/camera/camera_avfoundation/example/ios/RunnerTests/FLTCamSampleBufferTests.m

@@ -38,4 +38,63 @@ - (void)testCopyPixelBuffer {
   CFRelease(deliveriedPixelBuffer);
 }
 
+- (void)testDidOutputSampleBufferIgnoreAudioSamplesBeforeVideoSamples {
+  FLTCam *cam = FLTCreateCamWithCaptureSessionQueue(dispatch_queue_create("testing", NULL));
+  CMSampleBufferRef videoSample = FLTCreateTestSampleBuffer();
+  CMSampleBufferRef audioSample = FLTCreateTestAudioSampleBuffer();
+
+  id connectionMock = OCMClassMock([AVCaptureConnection class]);
+
+  id writerMock = OCMClassMock([AVAssetWriter class]);
+  OCMStub([writerMock alloc]).andReturn(writerMock);
+  OCMStub([writerMock initWithURL:OCMOCK_ANY fileType:OCMOCK_ANY error:[OCMArg setTo:nil]])
+      .andReturn(writerMock);
+  __block AVAssetWriterStatus status = AVAssetWriterStatusUnknown;
+  OCMStub([writerMock startWriting]).andDo(^(NSInvocation *invocation) {
+    status = AVAssetWriterStatusWriting;
+  });
+  OCMStub([writerMock status]).andDo(^(NSInvocation *invocation) {
+    [invocation setReturnValue:&status];
+  });
+
+  __block NSArray *writtenSamples = @[];
+
+  id videoMock = OCMClassMock([AVAssetWriterInputPixelBufferAdaptor class]);
+  OCMStub([videoMock assetWriterInputPixelBufferAdaptorWithAssetWriterInput:OCMOCK_ANY
+                                                sourcePixelBufferAttributes:OCMOCK_ANY])
+      .andReturn(videoMock);
+  OCMStub([videoMock appendPixelBuffer:[OCMArg anyPointer] withPresentationTime:kCMTimeZero])
+      .ignoringNonObjectArgs()
+      .andDo(^(NSInvocation *invocation) {
+        writtenSamples = [writtenSamples arrayByAddingObject:@"video"];
+      });
+
+  id audioMock = OCMClassMock([AVAssetWriterInput class]);
+  OCMStub([audioMock assetWriterInputWithMediaType:[OCMArg isEqual:AVMediaTypeAudio]
+                                    outputSettings:OCMOCK_ANY])
+      .andReturn(audioMock);
+  OCMStub([audioMock isReadyForMoreMediaData]).andReturn(YES);
+  OCMStub([audioMock appendSampleBuffer:[OCMArg anyPointer]]).andDo(^(NSInvocation *invocation) {
+    writtenSamples = [writtenSamples arrayByAddingObject:@"audio"];
+  });
+
+  FLTThreadSafeFlutterResult *result =
+      [[FLTThreadSafeFlutterResult alloc] initWithResult:^(id result){
+      }];
+  [cam startVideoRecordingWithResult:result];
+
+  [cam captureOutput:nil didOutputSampleBuffer:audioSample fromConnection:connectionMock];
+  [cam captureOutput:nil didOutputSampleBuffer:audioSample fromConnection:connectionMock];
+  [cam captureOutput:cam.captureVideoOutput
+      didOutputSampleBuffer:videoSample
+             fromConnection:connectionMock];
+  [cam captureOutput:nil didOutputSampleBuffer:audioSample fromConnection:connectionMock];
+
+  NSArray *expectedSamples = @[ @"video", @"audio" ];
+  XCTAssertEqualObjects(writtenSamples, expectedSamples, @"First appended sample must be video.");
+
+  CFRelease(videoSample);
+  CFRelease(audioSample);
+}
+
 @end

packages/camera/camera_avfoundation/ios/Classes/FLTCam.m

@@ -500,6 +500,12 @@ - (void)captureOutput:(AVCaptureOutput *)output
       return;
     }
 
+    // ignore audio samples until the first video sample arrives to avoid black frames
+    // https://github.com/flutter/flutter/issues/57831
+    if (_videoWriter.status != AVAssetWriterStatusWriting && output != _captureVideoOutput) {
+      return;
+    }
+
     CFRetain(sampleBuffer);
     CMTime currentSampleTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer);
 

packages/camera/camera_avfoundation/pubspec.yaml

@@ -2,7 +2,7 @@ name: camera_avfoundation
 description: iOS implementation of the camera plugin.
 repository: https://github.com/flutter/packages/tree/main/packages/camera/camera_avfoundation
 issue_tracker: https://github.com/flutter/flutter/issues?q=is%3Aissue+is%3Aopen+label%3A%22p%3A+camera%22
-version: 0.9.13+4
+version: 0.9.13+5
 
 environment:
   sdk: ">=2.19.0 <4.0.0"