Lite: Make the Examples component display media files using pseudo HTTP requests to the Wasm server

.changeset/early-planes-vanish.md

@@ -0,0 +1,10 @@
+---
+"@gradio/app": minor
+"@gradio/audio": minor
+"@gradio/image": minor
+"@gradio/video": minor
+"@gradio/wasm": minor
+"gradio": minor
+---
+
+feat:Lite: Make the Examples component display media files using pseudo HTTP requests to the Wasm server

js/app/src/Index.svelte

@@ -70,6 +70,8 @@
 	import { StatusTracker } from "@gradio/statustracker";
 	import { _ } from "svelte-i18n";
 	import { setupi18n } from "./i18n";
+	import type { WorkerProxy } from "@gradio/wasm";
+	import { setWorkerProxyContext } from "@gradio/wasm/svelte";
 
 	setupi18n();
 
@@ -89,6 +91,10 @@
 	export let mount_css: typeof default_mount_css = default_mount_css;
 	export let client: ReturnType<typeof api_factory>["client"];
 	export let upload_files: ReturnType<typeof api_factory>["upload_files"];
+	export let worker_proxy: WorkerProxy | undefined = undefined;
+	if (worker_proxy) {
+		setWorkerProxyContext(worker_proxy);
+	}
 
 	export let space: string | null;
 	export let host: string | null;

js/app/src/lite/index.ts

@@ -132,6 +132,7 @@ export function create(options: Options): GradioAppController {
 				// TODO: Remove -- i think this is just for autoscroll behavhiour, app vs embeds
 				app_mode: options.appMode,
 				// For Wasm mode
+				worker_proxy,
 				client,
 				upload_files,
 				mount_css: overridden_mount_css

js/audio/interactive/Audio.svelte

@@ -13,9 +13,9 @@
 	import { Upload, ModifyUpload } from "@gradio/upload";
 	import { BlockLabel } from "@gradio/atoms";
 	import { Music } from "@gradio/icons";
+	import Audio from "../shared/Audio.svelte";
 	// @ts-ignore
 	import Range from "svelte-range-slider-pips";
-	import { loaded } from "../shared/utils";
 	import { _ } from "svelte-i18n";
 
 	import type { IBlobEvent, IMediaRecorder } from "extendable-media-recorder";
@@ -295,17 +295,20 @@
 		absolute={true}
 	/>
 
-	<audio
-		use:loaded={{ autoplay, crop_values }}
-		controls
-		bind:this={player}
-		preload="metadata"
-		src={value?.data}
-		on:play
-		on:pause
-		on:ended={handle_ended}
-		data-testid={`${label}-audio`}
-	/>
+	<div class="container">
+		<Audio
+			controls
+			{autoplay}
+			{crop_values}
+			bind:node={player}
+			preload="metadata"
+			src={value?.data}
+			on:play
+			on:pause
+			on:ended={handle_ended}
+			data-testid={`${label}-audio`}
+		/>
+	</div>
 
 	{#if mode === "edit" && player?.duration}
 		<Range
@@ -360,7 +363,7 @@
 		}
 	}
 
-	audio {
+	.container {
 		padding: var(--size-2);
 		width: var(--size-full);
 		height: var(--size-14);

js/audio/package.json

@@ -14,6 +14,7 @@
 		"@gradio/statustracker": "workspace:^",
 		"@gradio/upload": "workspace:^",
 		"@gradio/utils": "workspace:^",
+		"@gradio/wasm": "workspace:^",
 		"extendable-media-recorder": "^9.0.0",
 		"extendable-media-recorder-wav-encoder": "^7.0.76",
 		"svelte-range-slider-pips": "^2.0.1"

js/audio/shared/Audio.svelte

@@ -0,0 +1,33 @@
+<script lang="ts">
+	import type { HTMLAudioAttributes } from "svelte/elements";
+	import { createEventDispatcher } from "svelte";
+	import { loaded, type LoadedParams } from "../shared/utils";
+	import { resolve_wasm_src } from "@gradio/wasm/svelte";
+
+	export let src: HTMLAudioAttributes["src"] = undefined;
+
+	export let autoplay: LoadedParams["autoplay"] = undefined;
+	export let crop_values: LoadedParams["crop_values"] = undefined;
+	export let controls: HTMLAudioAttributes["controls"] = undefined;
+	export let preload: HTMLAudioAttributes["preload"] = undefined;
+
+	export let node: HTMLAudioElement | undefined = undefined;
+
+	const dispatch = createEventDispatcher();
+</script>
+
+{#await resolve_wasm_src(src) then resolved_src}
+	<audio
+		src={resolved_src}
+		{controls}
+		{preload}
+		on:play={dispatch.bind(null, "play")}
+		on:pause={dispatch.bind(null, "pause")}
+		on:ended={dispatch.bind(null, "ended")}
+		bind:this={node}
+		use:loaded={{ autoplay, crop_values }}
+		data-testid={$$props["data-testid"]}
+	/>
+{:catch error}
+	<p style="color: red;">{error.message}</p>
+{/await}

js/audio/shared/utils.ts

@@ -1,6 +1,6 @@
 import type { ActionReturn } from "svelte/action";
 
-interface LoadedParams {
+export interface LoadedParams {
 	crop_values?: [number, number];
 	autoplay?: boolean;
 }

js/image/example/Image.svelte

@@ -1,29 +1,32 @@
 <script lang="ts">
+	import Image from "../shared/Image.svelte";
+
 	export let value: string;
 	export let samples_dir: string;
 	export let type: "gallery" | "table";
 	export let selected = false;
 </script>
 
-<img
-	src={samples_dir + value}
+<div
+	class="container"
 	class:table={type === "table"}
 	class:gallery={type === "gallery"}
 	class:selected
-	alt=""
-/>
+>
+	<Image src={samples_dir + value} alt="" />
+</div>
 
 <style>
-	img {
+	.container {
 		border-radius: var(--radius-lg);
 		max-width: none;
 	}
 
-	img.selected {
+	.container.selected {
 		border-color: var(--border-color-accent);
 	}
 
-	.table {
+	.container.table {
 		margin: 0 auto;
 		border: 2px solid var(--border-color-primary);
 		border-radius: var(--radius-lg);
@@ -32,8 +35,9 @@
 		object-fit: cover;
 	}
 
-	.gallery {
+	.container.gallery {
 		border: 2px solid var(--border-color-primary);
+		height: var(--size-20);
 		max-height: var(--size-20);
 		object-fit: cover;
 	}

js/image/package.json

@@ -13,6 +13,7 @@
 		"@gradio/statustracker": "workspace:^",
 		"@gradio/upload": "workspace:^",
 		"@gradio/utils": "workspace:^",
+		"@gradio/wasm": "workspace:^",
 		"cropperjs": "^1.5.12",
 		"lazy-brush": "^1.0.1",
 		"resize-observer-polyfill": "^1.5.1"

js/image/shared/Image.svelte

@@ -0,0 +1,22 @@
+<script lang="ts">
+	import type { HTMLImgAttributes } from "svelte/elements";
+	type $$Props = HTMLImgAttributes;
+
+	import { resolve_wasm_src } from "@gradio/wasm/svelte";
+
+	export let src: HTMLImgAttributes["src"] = undefined;
+</script>
+
+{#await resolve_wasm_src(src) then resolved_src}
+	<!-- svelte-ignore a11y-missing-attribute -->
+	<img src={resolved_src} {...$$restProps} />
+{:catch error}
+	<p style="color: red;">{error.message}</p>
+{/await}
+
+<style>
+	img {
+		max-width: 100%;
+		max-height: 100%;
+	}
+</style>

js/video/example/Video.svelte

@@ -1,6 +1,5 @@
 <script lang="ts">
-	import { playable } from "../shared";
-	import { onMount } from "svelte";
+	import { playable, Video } from "../shared";
 
 	export let type: "gallery" | "table";
 	export let selected = false;
@@ -17,48 +16,50 @@
 		await video.play();
 		video.pause();
 	}
-
-	onMount(() => {
-		init();
-	});
 </script>
 
 {#if playable()}
-	<video
-		muted
-		playsinline
-		bind:this={video}
+	<div
+		class="container"
 		class:table={type === "table"}
 		class:gallery={type === "gallery"}
 		class:selected
-		on:mouseover={video.play}
-		on:mouseout={video.pause}
-		src={samples_dir + value}
-	/>
+	>
+		<Video
+			muted
+			playsinline
+			bind:node={video}
+			on:loadeddata={init}
+			on:mouseover={video.play.bind(video)}
+			on:mouseout={video.pause.bind(video)}
+			src={samples_dir + value}
+		/>
+	</div>
 {:else}
 	<div>{value}</div>
 {/if}
 
 <style>
-	video {
+	.container {
 		flex: none;
 		border: 2px solid var(--border-color-primary);
 		border-radius: var(--radius-lg);
 		max-width: none;
 	}
 
-	video:hover,
-	video.selected {
+	.container:hover,
+	.container.selected {
 		border-color: var(--border-color-accent);
 	}
-	.table {
+	.container.table {
 		margin: 0 auto;
 		width: var(--size-20);
 		height: var(--size-20);
 		object-fit: cover;
 	}
 
-	.gallery {
+	.container.gallery {
+		height: var(--size-20);
 		max-height: var(--size-20);
 		object-fit: cover;
 	}

js/video/package.json

@@ -13,7 +13,8 @@
 		"@gradio/image": "workspace:^",
 		"@gradio/statustracker": "workspace:^",
 		"@gradio/upload": "workspace:^",
-		"@gradio/utils": "workspace:^"
+		"@gradio/utils": "workspace:^",
+		"@gradio/wasm": "workspace:^"
 	},
 	"main_changeset": true,
 	"exports": {

js/video/shared/Player.svelte

@@ -1,7 +1,7 @@
 <script lang="ts">
 	import { createEventDispatcher } from "svelte";
 	import { Play, Pause, Maximise, Undo } from "@gradio/icons";
-	import { loaded } from "./utils";
+	import Video from "./Video.svelte";
 
 	export let src: string;
 	export let subtitle: string | null = null;
@@ -83,23 +83,24 @@
 </script>
 
 <div class="wrap">
-	<video
-		{src}
-		preload="auto"
-		on:click={play_pause}
-		on:play
-		on:pause
-		on:ended={handle_end}
-		bind:currentTime={time}
-		bind:duration
-		bind:paused
-		bind:this={video}
-		class:mirror
-		use:loaded={{ autoplay }}
-		data-testid={`${label}-player`}
-	>
-		<track kind="captions" src={subtitle} default />
-	</video>
+	<div class:mirror>
+		<Video
+			{src}
+			preload="auto"
+			{autoplay}
+			on:click={play_pause}
+			on:play
+			on:pause
+			on:ended={handle_end}
+			bind:currentTime={time}
+			bind:duration
+			bind:paused
+			bind:node={video}
+			data-testid={`${label}-player`}
+		>
+			<track kind="captions" src={subtitle} default />
+		</Video>
+	</div>
 
 	<div class="controls">
 		<div class="inner">