feat: Add support for LiDAR, TrueDepth, External (USB) and Continuity Camera Devices (iOS 17)

.github/ISSUE_TEMPLATE/BUG_REPORT.yml

@@ -85,7 +85,6 @@ body:
           "maxZoom": 123.75,
           "name": "Back Dual Wide Camera",
           "position": "back",
-          "supportsDepthCapture": false,
           "supportsFocus": true,
           "supportsLowLightBoost": false,
           "supportsParallelVideoProcessing": true,

package/android/src/main/java/com/mrousavy/camera/CameraDevicesManager.kt

@@ -73,7 +73,15 @@ class CameraDevicesManager(private val reactContext: ReactApplicationContext) :
 
   override fun hasConstants(): Boolean = true
 
-  override fun getConstants(): MutableMap<String, Any> = mutableMapOf("availableCameraDevices" to getDevicesJson())
+  override fun getConstants(): MutableMap<String, Any?> {
+    val devices = getDevicesJson()
+    val preferredDevice = if (devices.size() > 0) devices.getMap(0) else null
+
+    return mutableMapOf(
+      "availableCameraDevices" to devices,
+      "userPreferredCameraDevice" to preferredDevice
+    )
+  }
 
   // Required for NativeEventEmitter, this is just a dummy implementation:
   @ReactMethod

package/android/src/main/java/com/mrousavy/camera/core/CameraDeviceDetails.kt

@@ -183,6 +183,7 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, private val
     map.putDouble("fieldOfView", getFieldOfView())
     map.putBoolean("supportsVideoHDR", supportsVideoHdr)
     map.putBoolean("supportsPhotoHDR", supportsPhotoHdr)
+    map.putBoolean("supportsDepthCapture", supportsDepthCapture)
     map.putString("autoFocusSystem", "contrast-detection") // TODO: Is this wrong?
     map.putArray("videoStabilizationModes", createStabilizationModes())
     map.putArray("pixelFormats", createPixelFormats(videoSize))
@@ -192,14 +193,13 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, private val
   fun toMap(): ReadableMap {
     val map = Arguments.createMap()
     map.putString("id", cameraId)
-    map.putArray("devices", getDeviceTypes())
+    map.putArray("physicalDevices", getDeviceTypes())
     map.putString("position", lensFacing.unionValue)
     map.putString("name", name)
     map.putBoolean("hasFlash", hasFlash)
     map.putBoolean("hasTorch", hasFlash)
     map.putBoolean("isMultiCam", isMultiCam)
     map.putBoolean("supportsRawCapture", supportsRawCapture)
-    map.putBoolean("supportsDepthCapture", supportsDepthCapture)
     map.putBoolean("supportsLowLightBoost", supportsLowLightBoost)
     map.putBoolean("supportsFocus", true) // I believe every device here supports focussing
     map.putDouble("minZoom", minZoom)

package/example/src/CameraPage.tsx

@@ -216,7 +216,7 @@ export function CameraPage({ navigation }: Props): React.ReactElement {
         )}
         {supports60Fps && (
           <PressableOpacity style={styles.button} onPress={() => setTargetFps((t) => (t === 30 ? 60 : 30))}>
-            <Text style={styles.text}>{`${targetFps} FPS`}</Text>
+            <Text style={styles.text}>{`${targetFps}\nFPS`}</Text>
           </PressableOpacity>
         )}
         {supportsHdr && (

package/ios/CameraDevicesManager.swift

@@ -37,47 +37,55 @@ class CameraDevicesManager: RCTEventEmitter {
   }
 
   override func constantsToExport() -> [AnyHashable: Any]! {
+    let devices = getDevicesJson()
+    let preferredDevice: [String: Any]
+    // TODO: Remove this #if once Xcode 15 is rolled out
+    #if swift(>=5.9)
+      if #available(iOS 17.0, *),
+         let userPreferred = AVCaptureDevice.userPreferredCamera {
+        preferredDevice = userPreferred.toDictionary()
+      } else {
+        preferredDevice = devices[0]
+      }
+    #else
+      preferredDevice = devices[0]
+    #endif
+
     return [
-      "availableCameraDevices": getDevicesJson(),
+      "availableCameraDevices": devices,
+      "userPreferredCameraDevice": preferredDevice,
     ]
   }
 
   private func getDevicesJson() -> [[String: Any]] {
     return discoverySession.devices.map {
-      return [
-        "id": $0.uniqueID,
-        "devices": $0.physicalDevices.map(\.deviceType.descriptor),
-        "position": $0.position.descriptor,
-        "name": $0.localizedName,
-        "hasFlash": $0.hasFlash,
-        "hasTorch": $0.hasTorch,
-        "minZoom": $0.minAvailableVideoZoomFactor,
-        "neutralZoom": $0.neutralZoomFactor,
-        "maxZoom": $0.maxAvailableVideoZoomFactor,
-        "isMultiCam": $0.isMultiCam,
-        "supportsDepthCapture": false, // TODO: supportsDepthCapture
-        "supportsRawCapture": false, // TODO: supportsRawCapture
-        "supportsLowLightBoost": $0.isLowLightBoostSupported,
-        "supportsFocus": $0.isFocusPointOfInterestSupported,
-        "hardwareLevel": "full",
-        "sensorOrientation": "portrait", // TODO: Sensor Orientation?
-        "formats": $0.formats.map { format -> [String: Any] in
-          format.toDictionary()
-        },
-      ]
+      return $0.toDictionary()
     }
   }
 
   private static func getAllDeviceTypes() -> [AVCaptureDevice.DeviceType] {
     var deviceTypes: [AVCaptureDevice.DeviceType] = []
+    deviceTypes.append(.builtInDualCamera)
+    deviceTypes.append(.builtInWideAngleCamera)
+    deviceTypes.append(.builtInTelephotoCamera)
+    deviceTypes.append(.builtInTrueDepthCamera)
     if #available(iOS 13.0, *) {
       deviceTypes.append(.builtInTripleCamera)
       deviceTypes.append(.builtInDualWideCamera)
       deviceTypes.append(.builtInUltraWideCamera)
     }
-    deviceTypes.append(.builtInDualCamera)
-    deviceTypes.append(.builtInWideAngleCamera)
-    deviceTypes.append(.builtInTelephotoCamera)
+    if #available(iOS 15.4, *) {
+      deviceTypes.append(.builtInLiDARDepthCamera)
+    }
+
+    // iOS 17 specifics:
+    //  This is only reported if `NSCameraUseExternalDeviceType` is set to true in Info.plist,
+    //  otherwise external devices are just reported as wide-angle-cameras
+    // deviceTypes.append(.external)
+    //  This is only reported if `NSCameraUseContinuityCameraDeviceType` is set to true in Info.plist,
+    //  otherwise continuity camera devices are just reported as wide-angle-cameras
+    // deviceTypes.append(.continuityCamera)
+
     return deviceTypes
   }
 }

package/ios/CameraView+AVCaptureSession.swift

@@ -84,6 +84,8 @@ extension CameraView {
           photoOutput!.isDualCameraDualPhotoDeliveryEnabled = photoOutput!.isDualCameraDualPhotoDeliverySupported
         }
       }
+      // TODO: Enable isResponsiveCaptureEnabled? (iOS 17+)
+      // TODO: Enable isFastCapturePrioritizationEnabled? (iOS 17+)
       if enableDepthData {
         photoOutput!.isDepthDataDeliveryEnabled = photoOutput!.isDepthDataDeliverySupported
       }

package/ios/Extensions/AVCaptureDevice+toDictionary.swift

@@ -0,0 +1,34 @@
+//
+//  AVCaptureDevice+toDictionary.swift
+//  VisionCamera
+//
+//  Created by Marc Rousavy on 21.09.23.
+//  Copyright © 2023 mrousavy. All rights reserved.
+//
+
+import AVFoundation
+
+extension AVCaptureDevice {
+  func toDictionary() -> [String: Any] {
+    return [
+      "id": uniqueID,
+      "physicalDevices": physicalDevices.map(\.deviceType.physicalDeviceDescriptor),
+      "position": position.descriptor,
+      "name": localizedName,
+      "hasFlash": hasFlash,
+      "hasTorch": hasTorch,
+      "minZoom": minAvailableVideoZoomFactor,
+      "neutralZoom": neutralZoomFactor,
+      "maxZoom": maxAvailableVideoZoomFactor,
+      "isMultiCam": isMultiCam,
+      "supportsRawCapture": false, // TODO: supportsRawCapture
+      "supportsLowLightBoost": isLowLightBoostSupported,
+      "supportsFocus": isFocusPointOfInterestSupported,
+      "hardwareLevel": "full",
+      "sensorOrientation": "portrait", // TODO: Sensor Orientation?
+      "formats": formats.map { format -> [String: Any] in
+        format.toDictionary()
+      },
+    ]
+  }
+}

package/ios/Extensions/AVCaptureDevice.Format+toDictionary.swift

@@ -55,6 +55,7 @@ extension AVCaptureDevice.Format {
       "minFps": minFrameRate,
       "maxFps": maxFrameRate,
       "pixelFormats": pixelFormats.map(\.unionValue),
+      "supportsDepthCapture": !supportedDepthDataFormats.isEmpty,
     ]
   }
 }

package/ios/Parsers/AVCaptureDevice.DeviceType+physicalDeviceDescriptor.swift

@@ -0,0 +1,33 @@
+//
+//  AVCaptureDevice.DeviceType+physicalDeviceDescriptor.swift
+//  mrousavy
+//
+//  Created by Marc Rousavy on 15.12.20.
+//  Copyright © 2020 mrousavy. All rights reserved.
+//
+
+import AVFoundation
+import Foundation
+
+extension AVCaptureDevice.DeviceType {
+  /**
+   Gets a descriptor if this is a physical device (wide, ultra-wide and telephoto), or "unknown-camera" otherwise (TrueDepth, LiDAR, InfraRed, USB, ..)
+   */
+  var physicalDeviceDescriptor: String {
+    if #available(iOS 13.0, *) {
+      if self == .builtInUltraWideCamera {
+        return "ultra-wide-angle-camera"
+      }
+    }
+    switch self {
+    case .builtInTelephotoCamera:
+      return "telephoto-camera"
+    case .builtInWideAngleCamera:
+      return "wide-angle-camera"
+    default:
+      // e.g. Infra-Red, LiDAR, Depth Data, USB or Continuity Camera Devices
+      ReactLogger.log(level: .error, message: "Unknown AVCaptureDevice.DeviceType (\(rawValue))! Falling back to wide-angle-camera..")
+      return "wide-angle-camera"
+    }
+  }
+}

package/ios/VisionCamera.xcodeproj/project.pbxproj

@@ -22,6 +22,7 @@
 		B86DC974260E310600FB17B2 /* CameraView+AVAudioSession.swift in Sources */ = {isa = PBXBuildFile; fileRef = B86DC973260E310600FB17B2 /* CameraView+AVAudioSession.swift */; };
 		B86DC977260E315100FB17B2 /* CameraView+AVCaptureSession.swift in Sources */ = {isa = PBXBuildFile; fileRef = B86DC976260E315100FB17B2 /* CameraView+AVCaptureSession.swift */; };
 		B87B11BF2A8E63B700732EBF /* PixelFormat.swift in Sources */ = {isa = PBXBuildFile; fileRef = B87B11BE2A8E63B700732EBF /* PixelFormat.swift */; };
+		B881D35E2ABC775E009B21C8 /* AVCaptureDevice+toDictionary.swift in Sources */ = {isa = PBXBuildFile; fileRef = B881D35D2ABC775E009B21C8 /* AVCaptureDevice+toDictionary.swift */; };
 		B882721026AEB1A100B14107 /* AVCaptureConnection+setInterfaceOrientation.swift in Sources */ = {isa = PBXBuildFile; fileRef = B882720F26AEB1A100B14107 /* AVCaptureConnection+setInterfaceOrientation.swift */; };
 		B887518525E0102000DB86D6 /* PhotoCaptureDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = B887515C25E0102000DB86D6 /* PhotoCaptureDelegate.swift */; };
 		B887518625E0102000DB86D6 /* CameraView+RecordVideo.swift in Sources */ = {isa = PBXBuildFile; fileRef = B887515D25E0102000DB86D6 /* CameraView+RecordVideo.swift */; };
@@ -44,7 +45,7 @@
 		B887519A25E0102000DB86D6 /* AVVideoCodecType+descriptor.swift in Sources */ = {isa = PBXBuildFile; fileRef = B887517525E0102000DB86D6 /* AVVideoCodecType+descriptor.swift */; };
 		B887519C25E0102000DB86D6 /* AVCaptureDevice.TorchMode+descriptor.swift in Sources */ = {isa = PBXBuildFile; fileRef = B887517725E0102000DB86D6 /* AVCaptureDevice.TorchMode+descriptor.swift */; };
 		B887519E25E0102000DB86D6 /* AVCapturePhotoOutput.QualityPrioritization+descriptor.swift in Sources */ = {isa = PBXBuildFile; fileRef = B887517925E0102000DB86D6 /* AVCapturePhotoOutput.QualityPrioritization+descriptor.swift */; };
-		B887519F25E0102000DB86D6 /* AVCaptureDevice.DeviceType+descriptor.swift in Sources */ = {isa = PBXBuildFile; fileRef = B887517A25E0102000DB86D6 /* AVCaptureDevice.DeviceType+descriptor.swift */; };
+		B887519F25E0102000DB86D6 /* AVCaptureDevice.DeviceType+physicalDeviceDescriptor.swift in Sources */ = {isa = PBXBuildFile; fileRef = B887517A25E0102000DB86D6 /* AVCaptureDevice.DeviceType+physicalDeviceDescriptor.swift */; };
 		B88751A025E0102000DB86D6 /* AVAuthorizationStatus+descriptor.swift in Sources */ = {isa = PBXBuildFile; fileRef = B887517B25E0102000DB86D6 /* AVAuthorizationStatus+descriptor.swift */; };
 		B88751A125E0102000DB86D6 /* AVCaptureDevice.Position+descriptor.swift in Sources */ = {isa = PBXBuildFile; fileRef = B887517C25E0102000DB86D6 /* AVCaptureDevice.Position+descriptor.swift */; };
 		B88751A325E0102000DB86D6 /* AVCaptureDevice.FlashMode+descriptor.swift in Sources */ = {isa = PBXBuildFile; fileRef = B887517E25E0102000DB86D6 /* AVCaptureDevice.FlashMode+descriptor.swift */; };
@@ -99,6 +100,7 @@
 		B86DC973260E310600FB17B2 /* CameraView+AVAudioSession.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CameraView+AVAudioSession.swift"; sourceTree = "<group>"; };
 		B86DC976260E315100FB17B2 /* CameraView+AVCaptureSession.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CameraView+AVCaptureSession.swift"; sourceTree = "<group>"; };
 		B87B11BE2A8E63B700732EBF /* PixelFormat.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PixelFormat.swift; sourceTree = "<group>"; };
+		B881D35D2ABC775E009B21C8 /* AVCaptureDevice+toDictionary.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVCaptureDevice+toDictionary.swift"; sourceTree = "<group>"; };
 		B882720F26AEB1A100B14107 /* AVCaptureConnection+setInterfaceOrientation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVCaptureConnection+setInterfaceOrientation.swift"; sourceTree = "<group>"; };
 		B887515C25E0102000DB86D6 /* PhotoCaptureDelegate.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PhotoCaptureDelegate.swift; sourceTree = "<group>"; };
 		B887515D25E0102000DB86D6 /* CameraView+RecordVideo.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "CameraView+RecordVideo.swift"; sourceTree = "<group>"; };
@@ -122,7 +124,7 @@
 		B887517525E0102000DB86D6 /* AVVideoCodecType+descriptor.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "AVVideoCodecType+descriptor.swift"; sourceTree = "<group>"; };
 		B887517725E0102000DB86D6 /* AVCaptureDevice.TorchMode+descriptor.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "AVCaptureDevice.TorchMode+descriptor.swift"; sourceTree = "<group>"; };
 		B887517925E0102000DB86D6 /* AVCapturePhotoOutput.QualityPrioritization+descriptor.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "AVCapturePhotoOutput.QualityPrioritization+descriptor.swift"; sourceTree = "<group>"; };
-		B887517A25E0102000DB86D6 /* AVCaptureDevice.DeviceType+descriptor.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "AVCaptureDevice.DeviceType+descriptor.swift"; sourceTree = "<group>"; };
+		B887517A25E0102000DB86D6 /* AVCaptureDevice.DeviceType+physicalDeviceDescriptor.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "AVCaptureDevice.DeviceType+physicalDeviceDescriptor.swift"; sourceTree = "<group>"; };
 		B887517B25E0102000DB86D6 /* AVAuthorizationStatus+descriptor.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "AVAuthorizationStatus+descriptor.swift"; sourceTree = "<group>"; };
 		B887517C25E0102000DB86D6 /* AVCaptureDevice.Position+descriptor.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "AVCaptureDevice.Position+descriptor.swift"; sourceTree = "<group>"; };
 		B887517E25E0102000DB86D6 /* AVCaptureDevice.FlashMode+descriptor.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "AVCaptureDevice.FlashMode+descriptor.swift"; sourceTree = "<group>"; };
@@ -211,6 +213,7 @@
 				B887516625E0102000DB86D6 /* AVCaptureDevice+physicalDevices.swift */,
 				B887516725E0102000DB86D6 /* AVFrameRateRange+includes.swift */,
 				B887516825E0102000DB86D6 /* AVCapturePhotoOutput+mirror.swift */,
+				B881D35D2ABC775E009B21C8 /* AVCaptureDevice+toDictionary.swift */,
 				B887516925E0102000DB86D6 /* AVCaptureDevice.Format+matchesFilter.swift */,
 				B887516A25E0102000DB86D6 /* AVCaptureDevice.Format+toDictionary.swift */,
 				B88B47462667C8E00091F538 /* AVCaptureSession+setVideoStabilizationMode.swift */,
@@ -239,7 +242,7 @@
 				B887517525E0102000DB86D6 /* AVVideoCodecType+descriptor.swift */,
 				B887517725E0102000DB86D6 /* AVCaptureDevice.TorchMode+descriptor.swift */,
 				B887517925E0102000DB86D6 /* AVCapturePhotoOutput.QualityPrioritization+descriptor.swift */,
-				B887517A25E0102000DB86D6 /* AVCaptureDevice.DeviceType+descriptor.swift */,
+				B887517A25E0102000DB86D6 /* AVCaptureDevice.DeviceType+physicalDeviceDescriptor.swift */,
 				B887517B25E0102000DB86D6 /* AVAuthorizationStatus+descriptor.swift */,
 				B887517C25E0102000DB86D6 /* AVCaptureDevice.Position+descriptor.swift */,
 				B887517E25E0102000DB86D6 /* AVCaptureDevice.FlashMode+descriptor.swift */,
@@ -408,9 +411,10 @@
 				B887519A25E0102000DB86D6 /* AVVideoCodecType+descriptor.swift in Sources */,
 				B88751A825E0102000DB86D6 /* CameraError.swift in Sources */,
 				B85F7AE92A77BB680089C539 /* FrameProcessorPlugin.m in Sources */,
+				B881D35E2ABC775E009B21C8 /* AVCaptureDevice+toDictionary.swift in Sources */,
 				B87B11BF2A8E63B700732EBF /* PixelFormat.swift in Sources */,
 				B88751A625E0102000DB86D6 /* CameraViewManager.swift in Sources */,
-				B887519F25E0102000DB86D6 /* AVCaptureDevice.DeviceType+descriptor.swift in Sources */,
+				B887519F25E0102000DB86D6 /* AVCaptureDevice.DeviceType+physicalDeviceDescriptor.swift in Sources */,
 				B8D22CDC2642DB4D00234472 /* AVAssetWriterInputPixelBufferAdaptor+initWithVideoSettings.swift in Sources */,
 				B84760DF2608F57D004C3180 /* CameraQueues.swift in Sources */,
 				B8446E502ABA14C900E56077 /* CameraDevicesManager.m in Sources */,