Describe simulator scene with gpt-4-vision

[elkraneo]

• Consider using ScreenCaptureKit
• CGWindowListCopyWindowInfo(::) can help to restrict captures to simulator

// Define a function to capture the screen of the Xcode simulator app window
func captureSimulatorWindow() {
    let targetAppName = "Simulator"  // The name of the Xcode simulator app
    
    if let windowList = CGWindowListCopyWindowInfo(.optionOnScreenOnly, kCGNullWindowID) as? [[String: AnyObject]] {
        for window in windowList {
            if let windowOwnerName = window[kCGWindowOwnerName] as? String, windowOwnerName.contains(targetAppName) {
                if let windowID = window[kCGWindowNumber] as? CGWindowID {
                    let screenCaptureSession = ScreenCaptureSession(windowID: windowID)
                    
                    // Start the screen capture session for the Xcode simulator window
                    screenCaptureSession.startCapture { image, error in
                        if let image = image {
                            // Process the captured image
                            // For example, you can save the image to a file
                            saveImageToFile(image: image)
                        } else if let error = error {
                            // Handle the error
                            print("Error capturing the simulator window: \(error.localizedDescription)")
                        }
                    }
                }
            }
        }
    }
}

[elkraneo]

Early experimentation here: https://github.com/elkraneo/VisionDescription