Improve source selection UX (#6766)

* Add new source option styling for pasting from clipboard Use SourceSelect in Image component * prevent device selection cut off tweak source selection ux * Check for dupe sources in source selection Set sources[0] to active_source in Image * tweaks * tweak * add image interaction test * more tests * improve light/dark mode color contrast * add changeset * remove unused prop * add no device found placeholder change T<sources> -> T<source_types> * style tweak * allow pasting on click + add e2e test * fix e2e tests * formatting * add timeout to e2e test * tweak * tweak test * change `getByLabel` to `getByText` * value tweak * logic tweak * test * formatting --------- Co-authored-by: gradio-pr-bot <gradio-pr-bot@users.noreply.github.com>
gradio-app · Dec 20, 2023 · 5663f3c · 5663f3c
1 parent 0a19254
commit 5663f3c
Show file tree

Hide file tree

Showing 15 changed files with 336 additions and 253 deletions.
diff --git a/.changeset/strong-files-swim.md b/.changeset/strong-files-swim.md
@@ -0,0 +1,11 @@
+---
+"@gradio/app": patch
+"@gradio/atoms": patch
+"@gradio/audio": patch
+"@gradio/image": patch
+"@gradio/upload": patch
+"@gradio/video": patch
+"gradio": patch
+---
+
+fix:Improve source selection UX
diff --git a/js/app/src/lang/en.json b/js/app/src/lang/en.json
@@ -48,7 +48,8 @@
 		"remove": "Remove",
 		"share": "Share",
 		"submit": "Submit",
-		"undo": "Undo"
+		"undo": "Undo",
+		"no_devices": "No devices found"
 	},
 	"dataframe": {
 		"incorrect_format": "Incorrect format, only CSV and TSV files are supported",
@@ -110,6 +111,7 @@
 		"drop_csv": "Drop CSV Here",
 		"drop_file": "Drop File Here",
 		"drop_image": "Drop Image Here",
-		"drop_video": "Drop Video Here"
+		"drop_video": "Drop Video Here",
+		"paste_clipboard": "Paste from Clipboard"
 	}
 }
diff --git a/js/app/test/image_component_events.spec.ts b/js/app/test/image_component_events.spec.ts
@@ -65,7 +65,33 @@ test("Image copy from clipboard dispatches upload event.", async ({ page }) => {
 		navigator.clipboard.write([new ClipboardItem({ [blob.type]: blob })]);
 	});
 
-	await page.getByLabel("clipboard-image-toolbar-btn").click();
+	await page.getByLabel("Paste from clipboard").click();
+	await Promise.all([
+		page.waitForResponse(
+			(resp) => resp.url().includes("/clipboard.png") && resp.status() === 200
+		)
+	]);
 	await expect(page.getByLabel("# Change Events").first()).toHaveValue("1");
 	await expect(page.getByLabel("# Upload Events")).toHaveValue("1");
 });
+
+test("Image paste to clipboard via the Upload component works", async ({
+	page
+}) => {
+	await page.evaluate(async () => {
+		navigator.clipboard.writeText("123");
+	});
+
+	await page.getByLabel("Paste from clipboard").click();
+	await page.evaluate(async () => {
+		const blob = await (
+			await fetch(
+				`https://gradio-builds.s3.amazonaws.com/assets/PDFDisplay.png`
+			)
+		).blob();
+		navigator.clipboard.write([new ClipboardItem({ [blob.type]: blob })]);
+	});
+
+	await page.getByText("Paste from clipboard").click();
+	await expect(page.getByLabel("# Upload Events")).toHaveValue("1");
+});
diff --git a/js/app/test/video_component_events.spec.ts b/js/app/test/video_component_events.spec.ts
@@ -3,9 +3,7 @@ import { test, expect, drag_and_drop_file } from "@gradio/tootils";
 test("Video click-to-upload uploads video successfuly. Clear, play, and pause buttons dispatch events correctly. Downloading the file works and has the correct name.", async ({
 	page
 }) => {
-	await page
-		.getByRole("button", { name: "Drop Video Here - or - Click to Upload" })
-		.click();
+	await page.getByRole("button", { name: "Upload file" }).click();
 	const uploader = await page.locator("input[type=file]");
 	await uploader.setInputFiles(["./test/files/file_test.ogg"]);
 
@@ -14,9 +12,7 @@ test("Video click-to-upload uploads video successfuly. Clear, play, and pause bu
 
 	await page.getByLabel("Clear").click();
 	await expect(page.getByLabel("# Change Events")).toHaveValue("2");
-	await page
-		.getByRole("button", { name: "Drop Video Here - or - Click to Upload" })
-		.click();
+	await page.getByRole("button", { name: "Upload file" }).click();
 
 	await uploader.setInputFiles(["./test/files/file_test.ogg"]);
 
@@ -30,9 +26,7 @@ test("Video click-to-upload uploads video successfuly. Clear, play, and pause bu
 });
 
 test("Video play, pause events work correctly.", async ({ page }) => {
-	await page
-		.getByRole("button", { name: "Drop Video Here - or - Click to Upload" })
-		.click();
+	await page.getByLabel("Upload file").click();
 	const uploader = await page.locator("input[type=file]");
 	await uploader.setInputFiles(["./test/files/file_test.ogg"]);
 
@@ -48,6 +42,7 @@ test("Video play, pause events work correctly.", async ({ page }) => {
 test("Video drag-and-drop uploads a file to the server correctly.", async ({
 	page
 }) => {
+	await page.getByLabel("Upload file").click();
 	await drag_and_drop_file(
 		page,
 		"input[type=file]",
@@ -62,6 +57,7 @@ test("Video drag-and-drop uploads a file to the server correctly.", async ({
 test("Video drag-and-drop displays a warning when the file is of the wrong mime type.", async ({
 	page
 }) => {
+	await page.getByLabel("Upload file").click();
 	await drag_and_drop_file(
 		page,
 		"input[type=file]",

diff --git a/js/atoms/src/SelectSource.svelte b/js/atoms/src/SelectSource.svelte
@@ -1,22 +1,34 @@
 <script lang="ts">
-	import { Microphone, Upload, Video } from "@gradio/icons";
+	import { Microphone, Upload, Webcam, ImagePaste } from "@gradio/icons";
 
-	export let sources: string[];
-	export let active_source: string;
+	type source_types = "upload" | "microphone" | "webcam" | "clipboard" | null;
+
+	export let sources: Partial<source_types>[];
+	export let active_source: Partial<source_types>;
 	export let handle_clear: () => void = () => {};
+	export let handle_select: (
+		source_type: Partial<source_types>
+	) => void = () => {};
+
+	$: unique_sources = [...new Set(sources)];
+
+	async function handle_select_source(
+		source: Partial<source_types>
+	): Promise<void> {
+		handle_clear();
+		active_source = source;
+		handle_select(source);
+	}
 </script>
 
-{#if sources.length > 1}
+{#if unique_sources.length > 1}
 	<span class="source-selection" data-testid="source-select">
 		{#if sources.includes("upload")}
 			<button
 				class="icon"
-				class:selected={active_source === "upload"}
+				class:selected={active_source === "upload" || !active_source}
 				aria-label="Upload file"
-				on:click={() => {
-					handle_clear();
-					active_source = "upload";
-				}}><Upload /></button
+				on:click={() => handle_select_source("upload")}><Upload /></button
 			>
 		{/if}
 
@@ -25,22 +37,26 @@
 				class="icon"
 				class:selected={active_source === "microphone"}
 				aria-label="Record audio"
-				on:click={() => {
-					handle_clear();
-					active_source = "microphone";
-				}}><Microphone /></button
+				on:click={() => handle_select_source("microphone")}
+				><Microphone /></button
 			>
 		{/if}
 
 		{#if sources.includes("webcam")}
 			<button
 				class="icon"
 				class:selected={active_source === "webcam"}
-				aria-label="Record video"
-				on:click={() => {
-					handle_clear();
-					active_source = "webcam";
-				}}><Video /></button
+				aria-label="Capture from camera"
+				on:click={() => handle_select_source("webcam")}><Webcam /></button
+			>
+		{/if}
+		{#if sources.includes("clipboard")}
+			<button
+				class="icon"
+				class:selected={active_source === "clipboard"}
+				aria-label="Paste from clipboard"
+				on:click={() => handle_select_source("clipboard")}
+				><ImagePaste /></button
 			>
 		{/if}
 	</span>
@@ -58,7 +74,6 @@
 		right: 0;
 		margin-left: auto;
 		margin-right: auto;
-		align-self: flex-end;
 	}
 
 	.icon {

diff --git a/js/atoms/src/UploadText.svelte b/js/atoms/src/UploadText.svelte
@@ -1,7 +1,8 @@
 <script lang="ts">
 	import type { I18nFormatter } from "@gradio/utils";
-	import { Upload as UploadIcon } from "@gradio/icons";
-	export let type: "video" | "image" | "audio" | "file" | "csv" = "file";
+	import { Upload as UploadIcon, ImagePaste } from "@gradio/icons";
+	export let type: "video" | "image" | "audio" | "file" | "csv" | "clipboard" =
+		"file";
 	export let i18n: I18nFormatter;
 	export let message: string | undefined = undefined;
 	export let mode: "full" | "short" = "full";
@@ -12,12 +13,19 @@
 		video: "upload_text.drop_video",
 		audio: "upload_text.drop_audio",
 		file: "upload_text.drop_file",
-		csv: "upload_text.drop_csv"
+		csv: "upload_text.drop_csv",
+		clipboard: "upload_text.paste_clipboard"
 	};
 </script>
 
 <div class="wrap">
-	<span class="icon-wrap" class:hovered><UploadIcon /> </span>
+	<span class="icon-wrap" class:hovered>
+		{#if type === "clipboard"}
+			<ImagePaste />
+		{:else}
+			<UploadIcon />
+		{/if}
+	</span>
 
 	{i18n(defs[type] || defs.file)}
 

diff --git a/js/audio/Index.svelte b/js/audio/Index.svelte
@@ -85,7 +85,7 @@
 
 	let dragging: boolean;
 
-	$: if (sources) {
+	$: if (!active_source && sources) {
 		active_source = sources[0];
 	}
 

diff --git a/js/audio/interactive/InteractiveAudio.svelte b/js/audio/interactive/InteractiveAudio.svelte
@@ -249,7 +249,6 @@
 			bind:dragging
 			on:error={({ detail }) => dispatch("error", detail)}
 			{root}
-			include_sources={sources.length > 1}
 		>
 			<slot />
 		</Upload>

diff --git a/js/image/Image.stories.svelte b/js/image/Image.stories.svelte
@@ -1,25 +1,10 @@
 <script lang="ts">
 	import { Meta, Template, Story } from "@storybook/addon-svelte-csf";
 	import StaticImage from "./Index.svelte";
+	import { userEvent, within } from "@storybook/testing-library";
 </script>
 
-<Meta
-	title="Components/Image"
-	component={Image}
-	argTypes={{
-		value: {
-			control: "object",
-			description: "The image URL or file to display",
-			name: "value"
-		},
-		show_download_button: {
-			options: [true, false],
-			description: "If false, the download button will not be visible",
-			control: { type: "boolean" },
-			defaultValue: true
-		}
-	}}
-/>
+<Meta title="Components/Image" component={Image} />
 
 <Template let:args>
 	<div
@@ -31,7 +16,7 @@
 </Template>
 
 <Story
-	name="Static Image with label and download button"
+	name="static with label and download button"
 	args={{
 		value: {
 			path: "https://gradio-builds.s3.amazonaws.com/demo-files/ghepardo-primo-piano.jpg",
@@ -44,7 +29,7 @@
 />
 
 <Story
-	name="Static Image with no label or download button"
+	name="static with no label or download button"
 	args={{
 		value: {
 			path: "https://gradio-builds.s3.amazonaws.com/demo-files/ghepardo-primo-piano.jpg",
@@ -55,3 +40,47 @@
 		show_download_button: false
 	}}
 />
+
+<Story
+	name="interactive with upload, clipboard, and webcam"
+	args={{
+		sources: ["upload", "clipboard", "webcam"],
+		value: {
+			path: "https://gradio-builds.s3.amazonaws.com/demo-files/ghepardo-primo-piano.jpg",
+			url: "https://gradio-builds.s3.amazonaws.com/demo-files/ghepardo-primo-piano.jpg",
+			orig_name: "cheetah.jpg"
+		},
+		show_label: false,
+		show_download_button: false,
+		interactive: true
+	}}
+	play={async ({ canvasElement }) => {
+		const canvas = within(canvasElement);
+
+		const webcamButton = await canvas.findByLabelText("Capture from camera");
+		userEvent.click(webcamButton);
+
+		userEvent.click(await canvas.findByTitle("select video source"));
+		userEvent.click(await canvas.findByLabelText("select source"));
+		userEvent.click(await canvas.findByLabelText("Upload file"));
+		userEvent.click(await canvas.findByLabelText("Paste from clipboard"));
+	}}
+/>
+
+<Story
+	name="interactive with webcam"
+	args={{
+		sources: ["webcam"],
+		show_download_button: true,
+		interactive: true
+	}}
+/>
+
+<Story
+	name="interactive with clipboard"
+	args={{
+		sources: ["clipboard"],
+		show_download_button: true,
+		interactive: true
+	}}
+/>
diff --git a/js/image/Index.svelte b/js/image/Index.svelte
@@ -20,6 +20,8 @@
 	import type { LoadingStatus } from "@gradio/statustracker";
 	import { normalise_file } from "@gradio/client";
 
+	type sources = "upload" | "webcam" | "clipboard" | null;
+
 	export let elem_id = "";
 	export let elem_classes: string[] = [];
 	export let visible = true;
@@ -66,7 +68,7 @@
 	$: url && gradio.dispatch("change");
 
 	let dragging: boolean;
-	let active_tool: null | "webcam" = null;
+	let active_source: sources = null;
 </script>
 
 {#if !interactive}
@@ -124,7 +126,7 @@
 		/>
 
 		<ImageUploader
-			bind:active_tool
+			bind:active_source
 			bind:value
 			selectable={_selectable}
 			{root}
@@ -144,17 +146,17 @@
 				loading_status.status = "error";
 				gradio.dispatch("error", detail);
 			}}
-			on:click={() => gradio.dispatch("error", "bad thing happened")}
-			on:error
 			{label}
 			{show_label}
 			{pending}
 			{streaming}
 			{mirror_webcam}
 			i18n={gradio.i18n}
 		>
-			{#if sources.includes("upload")}
-				<UploadText i18n={gradio.i18n} type="image" mode="short" />
+			{#if active_source === "upload" || !active_source}
+				<UploadText i18n={gradio.i18n} type="image" />
+			{:else if active_source === "clipboard"}
+				<UploadText i18n={gradio.i18n} type="clipboard" mode="short" />
 			{:else}
 				<Empty unpadded_box={true} size="large"><Image /></Empty>
 			{/if}