Implement AVFoundation Mode support and hot-plug detection

andrewfb · andrewfb · commit e770673fb1cb · 2025-10-06T16:04:51.000-04:00
Implements Device::getModes() to enumerate available capture formats
with resolution, frame rate, and pixel format information. Adds
mode-based initialization through initWithDevice:mode: for explicit
format control.

Adds session error notifications and device disconnection detection.
The isCapturing() method now checks session state to detect unplugged
devices, preventing crashes from attempting to access disconnected
hardware.
diff --git a/include/cinder/CaptureImplAvFoundation.h b/include/cinder/CaptureImplAvFoundation.h
@@ -26,6 +26,8 @@
 #include "cinder/Capture.h"
 #import <AVFoundation/AVFoundation.h>
 #include <vector>
+#include <memory>
+#include <set>
 
 namespace cinder {
 
@@ -34,11 +36,12 @@ class CaptureImplAvFoundationDevice : public Capture::Device {
 	CaptureImplAvFoundationDevice( AVCaptureDevice *device );
 	~CaptureImplAvFoundationDevice();
 	
-	bool						checkAvailable() const;
-	bool						isConnected() const;
-	Capture::DeviceIdentifier	getUniqueId() const { return mUniqueId; }
+	bool						checkAvailable() const override;
+	bool						isConnected() const override;
+	Capture::DeviceIdentifier	getUniqueId() const override { return mUniqueId; }
 	bool						isFrontFacing() const { return mFrontFacing; }
-	void*						getNative() const { return mNativeDevice; }
+	void*						getNative() const override { return mNativeDevice; }
+	std::vector<Capture::Mode>	getModes() const override;
   private:
 	Capture::DeviceIdentifier	mUniqueId;
 	AVCaptureDevice				*mNativeDevice;
@@ -61,11 +64,13 @@ class CaptureImplAvFoundationDevice : public Capture::Device {
 	int32_t							mExposedFrameBytesPerRow;
 	int32_t							mExposedFrameHeight;
 	int32_t							mExposedFrameWidth;
+	std::unique_ptr<cinder::Capture::Mode> mSelectedMode;
 }
 
 + (const std::vector<cinder::Capture::DeviceRef>&)getDevices:(BOOL)forceRefresh;
 
 - (id)initWithDevice:(const cinder::Capture::DeviceRef)device width:(int)width height:(int)height;
+- (id)initWithDevice:(const cinder::Capture::DeviceRef)device mode:(const cinder::Capture::Mode&)mode;
 - (bool)prepareStartCapture;
 - (void)startCapture;
 - (void)stopCapture;
@@ -79,4 +84,4 @@ class CaptureImplAvFoundationDevice : public Capture::Device {
 - (int32_t)getCurrentFrameWidth;
 - (int32_t)getCurrentFrameHeight;
 
-@end
+@end
diff --git a/src/cinder/CaptureImplAvFoundation.mm b/src/cinder/CaptureImplAvFoundation.mm
@@ -24,7 +24,11 @@
 #import "cinder/CaptureImplAvFoundation.h"
 #include "cinder/cocoa/CinderCocoa.h"
 #include "cinder/Vector.h"
+#include "cinder/Utilities.h"
+#include "cinder/Log.h"
 #import <AVFoundation/AVFoundation.h>
+#include <set>
+#include <sstream>
 
 namespace cinder {
 
@@ -52,6 +56,86 @@
 	return mNativeDevice.connected;
 }
 
+std::vector<Capture::Mode> CaptureImplAvFoundationDevice::getModes() const
+{
+	std::vector<Capture::Mode> modes;
+
+	if( !mNativeDevice )
+		return modes;
+
+	// Define practical frame rates we want to expose
+	std::vector<int> practicalFrameRates = { 120, 60, 30, 24, 15, 10, 5 };
+
+	std::set<std::string> seenModes; // To avoid duplicates
+
+	for( AVCaptureDeviceFormat *format in [mNativeDevice formats] ) {
+		CMVideoDimensions dimensions = CMVideoFormatDescriptionGetDimensions( format.formatDescription );
+		FourCharCode mediaSubType = CMFormatDescriptionGetMediaSubType( format.formatDescription );
+
+		// Map FourCharCode to our PixelFormat enum
+		Capture::Mode::PixelFormat pixelFormat = Capture::Mode::PixelFormat::Unknown;
+		switch( mediaSubType ) {
+			case 'BGRA': pixelFormat = Capture::Mode::PixelFormat::BGRA32; break;
+			case '420v': pixelFormat = Capture::Mode::PixelFormat::NV12; break;
+			case '420f': pixelFormat = Capture::Mode::PixelFormat::I420; break;
+			case 'yuvs': pixelFormat = Capture::Mode::PixelFormat::YUY2; break;
+			case '2vuy': pixelFormat = Capture::Mode::PixelFormat::UYVY; break;
+			case 'y420': pixelFormat = Capture::Mode::PixelFormat::YUV420P; break;
+			default:
+				// Skip unknown formats
+				continue;
+		}
+
+		// Get the supported frame rate range for this format
+		AVFrameRateRange *frameRateRange = format.videoSupportedFrameRateRanges.firstObject;
+		if( !frameRateRange ) continue;
+
+		int minFps = (int)round( frameRateRange.minFrameRate );
+		int maxFps = (int)round( frameRateRange.maxFrameRate );
+
+		std::vector<int> framesToCreate;
+		if( minFps == maxFps ) { // Device has fixed frame rate - just use that
+			framesToCreate.push_back( minFps );
+		} else { // Device supports variable frame rates - try practical rates first
+			for( int targetFps : practicalFrameRates )
+				if( targetFps >= minFps && targetFps <= maxFps )
+					framesToCreate.push_back( targetFps );
+			
+			// If no practical rates match, fall back to max frame rate
+			if( framesToCreate.empty() )
+				framesToCreate.push_back( maxFps );
+		}
+
+		// Create modes for each frame rate
+		for( int targetFps : framesToCreate ) {
+			MediaTime frameRate( 1, targetFps ); // Frame duration: 1/fps seconds
+
+			std::string modeKey = std::to_string( dimensions.width ) + "x" + std::to_string( dimensions.height ) +
+								 "@" + std::to_string( targetFps ) + "fps_" + std::to_string( (int)pixelFormat );
+
+			// Skip if we've already seen this exact mode
+			if( seenModes.find( modeKey ) != seenModes.end() )
+				continue;
+			seenModes.insert( modeKey );
+
+			// Create description - only show range if min != max
+			std::string description = std::to_string( dimensions.width ) + "x" + std::to_string( dimensions.height ) +
+									 " @ " + std::to_string( targetFps ) + "fps";
+
+			if( minFps != maxFps ) {
+				// Device supports variable frame rates - store the range
+				description += " (range: " + std::to_string( minFps ) + "-" + std::to_string( maxFps ) + "fps)";
+				modes.emplace_back( Capture::Mode( dimensions.width, dimensions.height, frameRate, Capture::Mode::Codec::Uncompressed, pixelFormat, description ) );
+			} else {
+				// Device has fixed frame rate - use simple constructor without range
+				modes.emplace_back( Capture::Mode( dimensions.width, dimensions.height, frameRate, Capture::Mode::Codec::Uncompressed, pixelFormat, description ) );
+			}
+		}
+	}
+
+	return modes;
+}
+
 } //namespace
 
 void frameDeallocator( void *refcon )
@@ -108,6 +192,33 @@ - (id)initWithDevice:(const cinder::Capture::DeviceRef)device width:(int)width h
 	return self;
 }
 
+- (id)initWithDevice:(const cinder::Capture::DeviceRef)device mode:(const cinder::Capture::Mode&)mode
+{
+	if( ( self = [super init] ) ) {
+		mDevice = device;
+		if( ! mDevice ) {
+			if( [CaptureImplAvFoundation getDevices:NO].empty() )
+				throw cinder::CaptureExcInitFail();
+			mDevice = [CaptureImplAvFoundation getDevices:NO][0];
+		}
+
+		mDeviceUniqueId = [NSString stringWithUTF8String:mDevice->getUniqueId().c_str()];
+		[mDeviceUniqueId retain];
+
+		mIsCapturing = false;
+		mWidth = mode.getWidth();
+		mHeight = mode.getHeight();
+		mHasNewFrame = false;
+		mExposedFrameBytesPerRow = 0;
+		mExposedFrameWidth = 0;
+		mExposedFrameHeight = 0;
+
+		// Store the mode for use during capture setup
+		mSelectedMode = std::make_unique<cinder::Capture::Mode>(mode);
+	}
+	return self;
+}
+
 - (void)dealloc
 {
 	if( mIsCapturing ) {
@@ -119,7 +230,7 @@ - (void)dealloc
 	[super dealloc];
 }
 
-- (bool)prepareStartCapture 
+- (bool)prepareStartCapture
 {
 	NSError *error = nil;
 
@@ -133,64 +244,115 @@ - (bool)prepareStartCapture
 	else {
 		device = [AVCaptureDevice deviceWithUniqueID:mDeviceUniqueId];
 	}
-	
+
 	if( ! device ) {
 		throw cinder::CaptureExcInitFail();
 	}
 
+	[mSession beginConfiguration];
+
+	// Configure device format and frame rate if we have a specific mode
+	if( mSelectedMode ) {
+		// Find the best matching format for the selected mode
+		AVCaptureDeviceFormat *bestFormat = nil;
+		double targetFrameRate = 1.0 / mSelectedMode->getFrameRate().getSeconds(); // Convert duration to rate
+
+		// Look for exact or best matching format
+		for( AVCaptureDeviceFormat *format in [device formats] ) {
+			CMFormatDescriptionRef formatDesc = [format formatDescription];
+			CMVideoDimensions dimensions = CMVideoFormatDescriptionGetDimensions( formatDesc );
+
+			// Check if resolution matches
+			if( dimensions.width == mSelectedMode->getWidth() &&
+				dimensions.height == mSelectedMode->getHeight() ) {
+
+				// Check if frame rate is supported
+				for( AVFrameRateRange *range in [format videoSupportedFrameRateRanges] ) {
+					if( targetFrameRate >= range.minFrameRate && targetFrameRate <= range.maxFrameRate ) {
+						bestFormat = format;
+						break;
+					}
+				}
+				if( bestFormat )
+					break;
+			}
+		}
+
+		// Configure device with the selected format
+		if( bestFormat && [device lockForConfiguration:&error] ) {
+			device.activeFormat = bestFormat;
+
+			// Set frame rate
+			CMTime frameDuration = CMTimeMake( 1, (int32_t)targetFrameRate );
+			device.activeVideoMinFrameDuration = frameDuration;
+			device.activeVideoMaxFrameDuration = frameDuration;
+
+			[device unlockForConfiguration];
+		}
+		else {
+			// Use session preset as fallback
+			if( cinder::ivec2( mWidth, mHeight ) == cinder::ivec2( 640, 480 ) )
+				mSession.sessionPreset = AVCaptureSessionPreset640x480;
+			else if( cinder::ivec2( mWidth, mHeight ) == cinder::ivec2( 1280, 720 ) )
+				mSession.sessionPreset = AVCaptureSessionPreset1280x720;
+			else if( cinder::ivec2( mWidth, mHeight ) == cinder::ivec2( 1920, 1080 ) )
+				mSession.sessionPreset = AVCaptureSessionPreset1920x1080;
+			else
+				mSession.sessionPreset = AVCaptureSessionPresetMedium;
+		}
+	}
+	else {
+		// Legacy mode - use session presets
+		if( cinder::ivec2( mWidth, mHeight ) == cinder::ivec2( 640, 480 ) )
+			mSession.sessionPreset = AVCaptureSessionPreset640x480;
+		else if( cinder::ivec2( mWidth, mHeight ) == cinder::ivec2( 1280, 720 ) )
+			mSession.sessionPreset = AVCaptureSessionPreset1280x720;
+		else if( cinder::ivec2( mWidth, mHeight ) == cinder::ivec2( 1920, 1080 ) )
+			mSession.sessionPreset = AVCaptureSessionPreset1920x1080;
+		else
+			mSession.sessionPreset = AVCaptureSessionPresetMedium;
+	}
+
 	// Create a device input with the device and add it to the session.
 	AVCaptureDeviceInput *input = [AVCaptureDeviceInput deviceInputWithDevice:device error:&error];
 	if( ! input ) {
+		[mSession commitConfiguration];
 		throw cinder::CaptureExcInitFail();
 	}
 	[mSession addInput:input];
 
 	// Create a VideoDataOutput and add it to the session
 	AVCaptureVideoDataOutput *output = [[[AVCaptureVideoDataOutput alloc] init] autorelease];
-
 	[mSession addOutput:output];
 
-	[mSession beginConfiguration];
-	if( cinder::ivec2( mWidth, mHeight ) == cinder::ivec2( 640, 480 ) )
-		mSession.sessionPreset = AVCaptureSessionPreset640x480;
-	else if( cinder::ivec2( mWidth, mHeight ) == cinder::ivec2( 1280, 720 ) )
-		mSession.sessionPreset = AVCaptureSessionPreset1280x720;
-	else
-		mSession.sessionPreset = AVCaptureSessionPresetMedium;
 	[mSession commitConfiguration];
-	
-	//adjust connection settings
-	/*
-	//Testing indicates that at least the 3GS doesn't support video orientation changes
-	NSArray * connections = output.connections;
-	for( AVCaptureConnection *connection in connections ) {
-		AVCaptureConnection * connection = [connections objectAtIndex:i];
-		
-		if( connection.supportsVideoOrientation ) {
-			connection.videoOrientation = AVCaptureVideoOrientationPortrait;
-		}
-	}*/
 
 	// Configure your output.
 	dispatch_queue_t queue = dispatch_queue_create("myQueue", NULL);
 	[output setSampleBufferDelegate:self queue:queue];
 	dispatch_release(queue);
 
+	// Always request BGRA format to match legacy behavior and avoid YUV conversion issues
+	// TODO: Add proper YUV support in the future
+	OSType pixelFormatType = kCVPixelFormatType_32BGRA;
+
 	// Specify the pixel format
 	NSDictionary *options = [NSDictionary dictionaryWithObjectsAndKeys:
 #if ! defined( CINDER_COCOA_TOUCH )
 								[NSNumber numberWithDouble:mWidth], (id)kCVPixelBufferWidthKey,
 								[NSNumber numberWithDouble:mHeight], (id)kCVPixelBufferHeightKey,
 #endif
-								[NSNumber numberWithUnsignedInt:kCVPixelFormatType_32BGRA], (id)kCVPixelBufferPixelFormatTypeKey,
+								[NSNumber numberWithUnsignedInt:pixelFormatType], (id)kCVPixelBufferPixelFormatTypeKey,
 								nil];
 	output.videoSettings = options;
 
 	[[NSNotificationCenter defaultCenter] addObserver:self selector:@selector(avCaptureInputPortFormatDescriptionDidChange:) name:AVCaptureInputPortFormatDescriptionDidChangeNotification object:nil];
 
-	// If you wish to cap the frame rate to a known value, such as 15 fps, set 
-	// minFrameDuration.
-	// output.minFrameDuration = CMTimeMake(1, 15);
+	// Add session runtime error notifications to detect device disconnections
+	[[NSNotificationCenter defaultCenter] addObserver:self selector:@selector(sessionRuntimeError:) name:AVCaptureSessionRuntimeErrorNotification object:mSession];
+	[[NSNotificationCenter defaultCenter] addObserver:self selector:@selector(sessionWasInterrupted:) name:AVCaptureSessionWasInterruptedNotification object:mSession];
+	[[NSNotificationCenter defaultCenter] addObserver:self selector:@selector(sessionInterruptionEnded:) name:AVCaptureSessionInterruptionEndedNotification object:mSession];
+
 	return true;
 }
 
@@ -235,7 +397,21 @@ - (void)stopCapture
 
 - (bool)isCapturing
 {
-	return mIsCapturing;
+	// Check not only our flag, but also the actual session state
+	// This will detect device disconnections that don't trigger stopCapture
+	bool sessionRunning = mSession && mSession.isRunning;
+	bool hasInputs = mSession && mSession.inputs.count > 0;
+	bool result = mIsCapturing && sessionRunning && hasInputs;
+
+	// Log when state doesn't match expectations
+	if( mIsCapturing && (!sessionRunning || !hasInputs) ) {
+		CI_LOG_W( "Camera disconnection detected: mIsCapturing=" << mIsCapturing
+			<< " sessionRunning=" << sessionRunning
+			<< " hasInputs=" << hasInputs
+			<< " inputCount=" << (mSession ? mSession.inputs.count : 0) );
+	}
+
+	return result;
 }
 
 // Called initially when the camera is instantiated and then again (hypothetically) if the resolution ever changes
@@ -253,6 +429,35 @@ - (void)avCaptureInputPortFormatDescriptionDidChange:(NSNotification *)notificat
 	}
 }
 
+// Session notification handlers for device disconnection detection
+- (void)sessionRuntimeError:(NSNotification *)notification
+{
+	NSError *error = notification.userInfo[AVCaptureSessionErrorKey];
+	CI_LOG_E( "AVCapture session runtime error: " << [error.localizedDescription UTF8String]
+		<< " (code: " << error.code << ")" );
+
+	// Common error codes that indicate device disconnection:
+	// -11814: Device disconnected
+	// -11819: Media services were reset
+	// -11808: Recording stopped (can be disconnection)
+	// Check if the device is still connected
+	if( error.code == -11814 || error.code == -11819 || error.code == -11808 ) {
+		// Mark session as not capturing on these critical errors
+		mIsCapturing = false;
+		CI_LOG_W( "Device likely disconnected - marking capture as stopped" );
+	}
+}
+
+- (void)sessionWasInterrupted:(NSNotification *)notification
+{
+	CI_LOG_W( "AVCapture session was interrupted (possible device disconnection)" );
+}
+
+- (void)sessionInterruptionEnded:(NSNotification *)notification
+{
+	CI_LOG_I( "AVCapture session interruption ended" );
+}
+
 // Delegate routine that is called when a sample buffer was written
 - (void)captureOutput:(AVCaptureOutput *)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection
 { 
@@ -338,4 +543,4 @@ - (int32_t)getCurrentFrameHeight
 	return mExposedFrameHeight;
 }
 
-@end
+@end
