Correctly handle device selection in Image and ImageEditor (#7754)

* move on:change to <select>

* add changeset

* move device logic to separate file + fix safari incompatability

* improve selected logic

* add stream_utils unit tests

* refactor default device logic

---------

Co-authored-by: gradio-pr-bot <gradio-pr-bot@users.noreply.github.com>

.changeset/brave-ducks-yawn.md

@@ -0,0 +1,6 @@
+---
+"@gradio/image": patch
+"gradio": patch
+---
+
+fix:Correctly handle device selection in `Image` and `ImageEditor`

js/image/shared/Webcam.svelte

@@ -1,19 +1,21 @@
 <script lang="ts">
 	import { createEventDispatcher, onMount } from "svelte";
-	import {
-		Camera,
-		Circle,
-		Square,
-		DropdownArrow,
-		Webcam as WebcamIcon
-	} from "@gradio/icons";
+	import { Camera, Circle, Square, DropdownArrow } from "@gradio/icons";
 	import type { I18nFormatter } from "@gradio/utils";
 	import type { FileData } from "@gradio/client";
 	import { prepare_files, upload } from "@gradio/client";
 	import WebcamPermissions from "./WebcamPermissions.svelte";
 	import { fade } from "svelte/transition";
+	import {
+		get_devices,
+		get_video_stream,
+		set_available_devices
+	} from "./stream_utils";
 
 	let video_source: HTMLVideoElement;
+	let available_video_devices: MediaDeviceInfo[] = [];
+	let selected_device: MediaDeviceInfo | null = null;
+
 	let canvas: HTMLCanvasElement;
 	export let streaming = false;
 	export let pending = false;
@@ -33,24 +35,48 @@
 	}>();
 
 	onMount(() => (canvas = document.createElement("canvas")));
-	const size = {
-		width: { ideal: 1920 },
-		height: { ideal: 1440 }
+
+	const handle_device_change = async (event: InputEvent): Promise<void> => {
+		const target = event.target as HTMLInputElement;
+		const device_id = target.value;
+
+		await get_video_stream(include_audio, video_source, device_id).then(
+			async (local_stream) => {
+				stream = local_stream;
+				selected_device =
+					available_video_devices.find(
+						(device) => device.deviceId === device_id
+					) || null;
+				options_open = false;
+			}
+		);
 	};
-	async function access_webcam(device_id?: string): Promise<void> {
-		if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
-			dispatch("error", i18n("image.no_webcam_support"));
-			return;
-		}
+
+	async function access_webcam(): Promise<void> {
 		try {
-			stream = await navigator.mediaDevices.getUserMedia({
-				video: device_id ? { deviceId: { exact: device_id }, ...size } : size,
-				audio: include_audio
-			});
-			video_source.srcObject = stream;
-			video_source.muted = true;
-			video_source.play();
-			webcam_accessed = true;
+			get_video_stream(include_audio, video_source)
+				.then(async (local_stream) => {
+					webcam_accessed = true;
+					available_video_devices = await get_devices();
+					stream = local_stream;
+				})
+				.then(() => set_available_devices(available_video_devices))
+				.then((devices) => {
+					available_video_devices = devices;
+
+					const used_devices = stream
+						.getTracks()
+						.map((track) => track.getSettings()?.deviceId)[0];
+
+					selected_device = used_devices
+						? devices.find((device) => device.deviceId === used_devices) ||
+						  available_video_devices[0]
+						: available_video_devices[0];
+				});
+
+			if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
+				dispatch("error", i18n("image.no_webcam_support"));
+			}
 		} catch (err) {
 			if (err instanceof DOMException && err.name == "NotAllowedError") {
 				dispatch("error", i18n("image.allow_webcam_access"));
@@ -169,18 +195,6 @@
 		}, 500);
 	}
 
-	async function select_source(): Promise<void> {
-		const devices = await navigator.mediaDevices.enumerateDevices();
-		video_sources = devices.filter((device) => device.kind === "videoinput");
-		options_open = true;
-	}
-
-	let video_sources: MediaDeviceInfo[] = [];
-	async function selectVideoSource(device_id: string): Promise<void> {
-		await access_webcam(device_id);
-		options_open = false;
-	}
-
 	let options_open = false;
 
 	export function click_outside(node: Node, cb: any): any {
@@ -247,31 +261,35 @@
 			{#if !recording}
 				<button
 					class="icon"
-					on:click={select_source}
+					on:click={() => (options_open = true)}
 					aria-label="select input source"
 				>
 					<DropdownArrow />
 				</button>
 			{/if}
 		</div>
-		{#if options_open}
+		{#if options_open && selected_device}
 			<select
 				class="select-wrap"
 				aria-label="select source"
 				use:click_outside={handle_click_outside}
+				on:change={handle_device_change}
 			>
 				<button
 					class="inset-icon"
 					on:click|stopPropagation={() => (options_open = false)}
 				>
 					<DropdownArrow />
 				</button>
-				{#if video_sources.length === 0}
+				{#if available_video_devices.length === 0}
 					<option value="">{i18n("common.no_devices")}</option>
 				{:else}
-					{#each video_sources as source}
-						<option on:click={() => selectVideoSource(source.deviceId)}>
-							{source.label}
+					{#each available_video_devices as device}
+						<option
+							value={device.deviceId}
+							selected={selected_device.deviceId === device.deviceId}
+						>
+							{device.label}
 						</option>
 					{/each}
 				{/if}

js/image/shared/stream_utils.test.ts

@@ -0,0 +1,134 @@
+import { describe, expect, vi } from "vitest";
+import {
+	get_devices,
+	get_video_stream,
+	set_available_devices,
+	set_local_stream
+} from "./stream_utils";
+import * as stream_utils from "./stream_utils";
+
+let test_device: MediaDeviceInfo = {
+	deviceId: "test-device",
+	kind: "videoinput",
+	label: "Test Device",
+	groupId: "camera",
+	toJSON: () => ({
+		deviceId: "test-device",
+		kind: "videoinput",
+		label: "Test Device",
+		groupId: "camera"
+	})
+};
+
+const mock_enumerateDevices = vi.fn(async () => {
+	return new Promise<MediaDeviceInfo[]>((resolve) => {
+		resolve([test_device]);
+	});
+});
+const mock_getUserMedia = vi.fn(async () => {
+	return new Promise<MediaStream>((resolve) => {
+		resolve(new MediaStream());
+	});
+});
+
+window.MediaStream = vi.fn().mockImplementation(() => ({}));
+
+Object.defineProperty(global.navigator, "mediaDevices", {
+	value: {
+		getUserMedia: mock_getUserMedia,
+		enumerateDevices: mock_enumerateDevices
+	}
+});
+
+describe("stream_utils", () => {
+	test("get_devices should enumerate media devices", async () => {
+		const devices = await get_devices();
+		expect(devices).toEqual([test_device]);
+	});
+
+	test("set_local_stream should set the local stream to the video source", () => {
+		const mock_stream = {}; // mocked MediaStream obj as it's not available in a node env
+
+		const mock_video_source = {
+			srcObject: null,
+			muted: false,
+			play: vi.fn()
+		};
+
+		// @ts-ignore
+		set_local_stream(mock_stream, mock_video_source);
+
+		expect(mock_video_source.srcObject).toEqual(mock_stream);
+		expect(mock_video_source.muted).toBeTruthy();
+		expect(mock_video_source.play).toHaveBeenCalled();
+	});
+
+	test("get_video_stream requests user media with the correct constraints and sets the local stream", async () => {
+		const mock_video_source = document.createElement("video");
+		const mock_stream = new MediaStream();
+
+		global.navigator.mediaDevices.getUserMedia = vi
+			.fn()
+			.mockResolvedValue(mock_stream);
+
+		await get_video_stream(true, mock_video_source);
+
+		expect(navigator.mediaDevices.getUserMedia).toHaveBeenCalledWith({
+			video: { width: { ideal: 1920 }, height: { ideal: 1440 } },
+			audio: true
+		});
+
+		const spy_set_local_stream = vi.spyOn(stream_utils, "set_local_stream");
+		stream_utils.set_local_stream(mock_stream, mock_video_source);
+
+		expect(spy_set_local_stream).toHaveBeenCalledWith(
+			mock_stream,
+			mock_video_source
+		);
+		spy_set_local_stream.mockRestore();
+	});
+
+	test("set_available_devices should return only video input devices", () => {
+		const mockDevices: MediaDeviceInfo[] = [
+			{
+				deviceId: "camera1",
+				kind: "videoinput",
+				label: "Camera 1",
+				groupId: "camera",
+				toJSON: () => ({
+					deviceId: "camera1",
+					kind: "videoinput",
+					label: "Camera 1",
+					groupId: "camera"
+				})
+			},
+			{
+				deviceId: "camera2",
+				kind: "videoinput",
+				label: "Camera 2",
+				groupId: "camera",
+				toJSON: () => ({
+					deviceId: "camera2",
+					kind: "videoinput",
+					label: "Camera 2",
+					groupId: "camera"
+				})
+			},
+			{
+				deviceId: "audio1",
+				kind: "audioinput",
+				label: "Audio 2",
+				groupId: "audio",
+				toJSON: () => ({
+					deviceId: "audio1",
+					kind: "audioinput",
+					label: "Audio 2",
+					groupId: "audio"
+				})
+			}
+		];
+
+		const videoDevices = set_available_devices(mockDevices);
+		expect(videoDevices).toEqual(mockDevices.splice(0, 2));
+	});
+});

js/image/shared/stream_utils.ts

@@ -0,0 +1,49 @@
+export function get_devices(): Promise<MediaDeviceInfo[]> {
+	return navigator.mediaDevices.enumerateDevices();
+}
+
+export function handle_error(error: string): void {
+	throw new Error(error);
+}
+
+export function set_local_stream(
+	local_stream: MediaStream | null,
+	video_source: HTMLVideoElement
+): void {
+	video_source.srcObject = local_stream;
+	video_source.muted = true;
+	video_source.play();
+}
+
+export async function get_video_stream(
+	include_audio: boolean,
+	video_source: HTMLVideoElement,
+	device_id?: string
+): Promise<MediaStream> {
+	const size = {
+		width: { ideal: 1920 },
+		height: { ideal: 1440 }
+	};
+
+	const constraints = {
+		video: device_id ? { deviceId: { exact: device_id }, ...size } : size,
+		audio: include_audio
+	};
+
+	return navigator.mediaDevices
+		.getUserMedia(constraints)
+		.then((local_stream: MediaStream) => {
+			set_local_stream(local_stream, video_source);
+			return local_stream;
+		});
+}
+
+export function set_available_devices(
+	devices: MediaDeviceInfo[]
+): MediaDeviceInfo[] {
+	const cameras = devices.filter(
+		(device: MediaDeviceInfo) => device.kind === "videoinput"
+	);
+
+	return cameras;
+}