WIP: @remotion/whisper-webgpu:

package.json

@@ -14,6 +14,7 @@
 		"testlambda": "turbo run testlambda --concurrency=1 --no-update-notifier",
 		"ci": "turbo run make test --concurrency=1 --no-update-notifier",
 		"watch": "turbo watch make --concurrency=2 --experimental-write-cache --ui=tui",
+		"watchwhisperwebpu": "turbo watch make --experimental-write-cache --filter='@remotion/whisper-webgpu'",
 		"watchwebcodecs": "turbo watch make --experimental-write-cache --filter='@remotion/media-parser' --filter='@remotion/webcodecs'",
 		"watchcore": "turbo watch make --experimental-write-cache --filter='remotion'",
 		"watchstudio": "turbo watch make --experimental-write-cache --filter='@remotion/studio' --filter='@remotion/studio-server'",

packages/.monorepo/builder.ts

@@ -50,6 +50,9 @@ const validateExternal = (external: string[]) => {
 		if (dep === 'stream' || dep === 'fs' || dep === 'path') {
 			continue;
 		}
+		if (dep.startsWith('.')) {
+			continue;
+		}
 		if (!packageJson.includes(stripEntryPoints(dep))) {
 			throw new Error(
 				`External dependency ${stripEntryPoints(dep)} not found in package.json`,

packages/STATS.md

@@ -1,4 +1,5 @@
 # Download statistics
+
 Monthly downloads of Remotion packages  
 [![NPM Downloads](https://img.shields.io/npm/dm/@remotion/animated-emoji.svg?style=flat&color=black&label=@remotion/animated-emoji)](https://npmcharts.com/compare/@remotion/animated-emoji?minimal=true)  
 [![NPM Downloads](https://img.shields.io/npm/dm/@remotion/animation-utils.svg?style=flat&color=black&label=@remotion/animation-utils)](https://npmcharts.com/compare/@remotion/animation-utils?minimal=true)  
@@ -54,4 +55,5 @@ Monthly downloads of Remotion packages
 [![NPM Downloads](https://img.shields.io/npm/dm/@remotion/three.svg?style=flat&color=black&label=@remotion/three)](https://npmcharts.com/compare/@remotion/three?minimal=true)  
 [![NPM Downloads](https://img.shields.io/npm/dm/@remotion/transitions.svg?style=flat&color=black&label=@remotion/transitions)](https://npmcharts.com/compare/@remotion/transitions?minimal=true)  
 [![NPM Downloads](https://img.shields.io/npm/dm/@remotion/webcodecs.svg?style=flat&color=black&label=@remotion/webcodecs)](https://npmcharts.com/compare/@remotion/webcodecs?minimal=true)  
-[![NPM Downloads](https://img.shields.io/npm/dm/@remotion/zod-types.svg?style=flat&color=black&label=@remotion/zod-types)](https://npmcharts.com/compare/@remotion/zod-types?minimal=true)  
+[![NPM Downloads](https://img.shields.io/npm/dm/@remotion/whisper-webgpu.svg?style=flat&color=black&label=@remotion/whisper-webgpu)](https://npmcharts.com/compare/@remotion/whisper-webgpu?minimal=true)  
+[![NPM Downloads](https://img.shields.io/npm/dm/@remotion/zod-types.svg?style=flat&color=black&label=@remotion/zod-types)](https://npmcharts.com/compare/@remotion/zod-types?minimal=true)

packages/cli/src/list-of-remotion-packages.ts

@@ -76,4 +76,5 @@ export const listOfRemotionPackages = [
 	'@remotion/openai-whisper',
 	'@remotion/compositor',
 	'@remotion/example-videos',
+	'@remotion/whisper-webgpu',
 ];

packages/create-video/src/list-of-remotion-packages.ts

@@ -76,4 +76,5 @@ export const listOfRemotionPackages = [
 	'@remotion/openai-whisper',
 	'@remotion/compositor',
 	'@remotion/example-videos',
+	'@remotion/whisper-webgpu',
 ];

packages/docs/docs/whisper-webgpu/can-use-whisper-webgpu.mdx

@@ -0,0 +1,101 @@
+---
+image: /generated/articles-docs-whisper-webgpu-can-use-whisper-webgpu.png
+title: canUseWhisperWebGpu()
+crumb: '@remotion/whisper-webgpu'
+---
+
+:::warning
+**Unstable API**: This package is experimental for the moment. As we test it, we might make a few changes to the API.  
+:::
+
+# canUseWhisperWebGpu()
+
+Checks if the current browser environment supports running `@remotion/whisper-webgpu` with a specified model. This function verifies various browser capabilities like `crossOriginIsolated`, `IndexedDB`, `navigator.storage.estimate()`, and available storage space.
+
+## Example usage
+
+```tsx twoslash
+import {canUseWhisperWebGpu, type WhisperWebGpuModel} from '@remotion/whisper-webgpu';
+import {useState, useEffect} from 'react';
+
+export default function MyComponent() {
+  const [supported, setSupported] = useState<boolean | null>(null);
+  const [reason, setReason] = useState<string | undefined>(undefined);
+
+  useEffect(() => {
+    const checkSupport = async () => {
+      const modelToUse: WhisperWebGpuModel = 'tiny.en'; // Or any other model
+      const result = await canUseWhisperWebGpu(modelToUse);
+      setSupported(result.supported);
+      if (!result.supported) {
+        setReason(result.detailedReason ?? result.reason);
+      }
+    };
+
+    checkSupport();
+  }, []);
+
+  if (supported === null) {
+    return <p>Checking Whisper WASM support...</p>;
+  }
+
+  if (supported) {
+    return <p>Whisper WASM is supported!</p>;
+  }
+
+  return <p>Whisper WASM is not supported: {reason}</p>;
+}
+```
+
+## Arguments
+
+### `model`
+
+The Whisper model intended to be used. This is used to check if there's enough storage space for the model.
+Possible values: `tiny`, `tiny.en`, `base`, `base.en`, `small`, `small.en`.
+
+Refer to the `WhisperWebGpuModel` type exported by the package for a comprehensive list.
+
+## Return value
+
+A `Promise` that resolves to a `CanUseWhisperWebGpuResult` object with the following properties:
+
+### `supported`
+
+A boolean indicating whether Whisper WASM can be used. `true` if supported, `false` otherwise.
+
+### `reason?`
+
+If `supported` is `false`, this field provides a brief, categorized reason for the lack of support.
+
+Possible values values include:
+
+- `window-undefined`: `window` object is not available.
+- `not-cross-origin-isolated`: The page is not cross-origin isolated.
+- `indexed-db-unavailable`: IndexedDB is not available.
+- `navigator-storage-unavailable`: `navigator.storage.estimate()` API is not available.
+- `quota-undefined`: Storage quota could not be determined.
+- `usage-undefined`: Storage usage could not be determined.
+- `not-enough-space`: Insufficient storage space for the specified model.
+- `error-estimating-storage`: An error occurred while trying to estimate storage.
+
+### `detailedReason?`
+
+If `supported` is `false`, this field may contain a more detailed, human-readable explanation of why Whisper WASM is not supported.
+
+## Important considerations
+
+- **Cross-Origin Isolation:** For `SharedArrayBuffer` to work, which is required by `@remotion/whisper-webgpu`, the page must be served with specific HTTP headers:
+
+  - `Cross-Origin-Opener-Policy: same-origin`
+  - `Cross-Origin-Embedder-Policy: require-corp`
+    Ensure your server is configured to send these headers. See [MDN documentation on `crossOriginIsolated`](https://developer.mozilla.org/en-US/docs/Web/API/Window/crossOriginIsolated) for more details.
+
+- **Browser Compatibility:** While this function checks for necessary APIs, always test on your target browsers as support for WebAssembly, IndexedDB, and storage estimation can vary.
+
+## See also
+
+- [Source code for `canUseWhisperWebGpu()`](https://github.com/remotion-dev/remotion/blob/main/packages/whisper-webgpu/src/can-use-whisper-webgpu.ts)
+- [`@remotion/whisper-webgpu`](/docs/whisper-webgpu)
+- [`transcribe()`](/docs/whisper-webgpu/transcribe)
+- [`downloadWhisperModel()`](/docs/whisper-webgpu/download-whisper-model)

packages/docs/docs/whisper-webgpu/download-whisper-model.mdx

@@ -0,0 +1,54 @@
+---
+image: /generated/articles-docs-whisper-webgpu-download-whisper-model.png
+title: downloadWhisperModel()
+crumb: '@remotion/whisper-webgpu'
+---
+
+# downloadWhisperModel()
+
+:::warning
+**Unstable API**: This package is experimental for the moment. As we test it, we might make a few changes to the API.  
+:::
+
+Downloads a Whisper model into IndexedDB.
+
+```tsx twoslash title="app.ts"
+import {downloadWhisperModel} from '@remotion/whisper-webgpu';
+
+const {alreadyDownloaded} = await downloadWhisperModel({
+  model: 'tiny.en',
+  onProgress: (progress) => {
+    console.log(progress);
+  },
+});
+```
+
+## Options
+
+### `model`
+
+The model to download. Possible values: `tiny`, `tiny.en`, `base`, `base.en`, `small`, `small.en`.
+
+### `onProgress?`
+
+Act upon download progress. This is the function signature:
+
+```tsx twoslash
+import {DownloadWhisperModelOnProgress, DownloadWhisperModelProgress} from '@remotion/whisper-webgpu';
+
+const onProgress: DownloadWhisperModelOnProgress = ({progress, totalBytes, downloadedBytes}: DownloadWhisperModelProgress) => {
+  console.log({progress, totalBytes, downloadedBytes});
+};
+```
+
+## Return Value
+
+Returns an object with the following properties:
+
+- `alreadyDownloaded`: Whether the model has already been downloaded.
+
+## See also
+
+- [Source code for this function](https://github.com/remotion-dev/remotion/blob/main/packages/whisper-webgpu/src/download-whisper-model.ts)
+- [`@remotion/whisper-webgpu`](/docs/whisper-webgpu)
+- [`transcribe()`](/docs/whisper-webgpu/transcribe)

packages/docs/docs/whisper-webgpu/get-loaded-models.mdx

@@ -0,0 +1,15 @@
+---
+image: /generated/articles-docs-whisper-webgpu-get-loaded-models.png
+title: getLoadedModels()
+crumb: '@remotion/whisper-webgpu'
+---
+
+:::warning
+**Unstable API**: This package is experimental for the moment. As we test it, we might make a few changes to the API.  
+:::
+
+Returns an array of Whisper models that have already been downloaded and stored in the browser's IndexedDB storage.
+
+## Return value
+
+Returns a Promise that resolves to an array of `WhisperWebGpuModel` strings, representing the models that are already downloaded and available locally.

packages/docs/docs/whisper-webgpu/index.mdx

@@ -0,0 +1,77 @@
+---
+image: /generated/articles-docs-whisper-webgpu-index.png
+title: '@remotion/whisper-webgpu'
+crumb: 'Transcribe audio in browser'
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+:::warning
+**Unstable API**: This package is experimental for the moment. As we test it, we might make a few changes to the API.  
+:::
+
+Similar to [@remotion/install-whisper-cpp](/docs/install-whisper-cpp) but for the browser. Allows you to transcribe audio locally in the browser, with the help of WASM.
+
+import {TableOfContents} from './whisper-webgpu';
+
+<Installation pkg="@remotion/whisper-webgpu" />
+
+## Example usage
+
+:::info
+This library is UI-agnostic and can be integrated with any frontend framework.
+:::
+
+```tsx twoslash
+import {transcribe, canUseWhisperWebGpu, resampleTo16Khz, downloadWhisperModel} from '@remotion/whisper-webgpu';
+
+// HTML:
+// <input type="file" accept="audio/*" id="audio-input" />
+// <p id="status"></p>
+
+const input = document.getElementById('audio-input') as HTMLInputElement;
+
+input.addEventListener('change', async (e) => {
+  const file = input.files?.[0];
+  if (!file) return;
+
+  console.log('Processing...');
+
+  const modelToUse = 'tiny.en';
+
+  const {supported, detailedReason} = await canUseWhisperWebGpu(modelToUse);
+  if (!supported) {
+    throw new Error(`Whisper WASM is not supported in this environment: ${detailedReason}`);
+  }
+
+  console.log('Downloading model...');
+  await downloadWhisperModel({
+    model: modelToUse,
+    onProgress: (p) => console.log(`Downloading model (${Math.round(p * 100)}%)...`),
+  });
+
+  console.log('Resampling audio...');
+  const channelWaveform = await resampleTo16Khz({
+    file,
+    onProgress: (p) => console.log(`Resampling audio (${Math.round(p * 100)}%)...`),
+  });
+
+  console.log('Transcribing...');
+  const {transcription} = await transcribe({
+    channelWaveform,
+    model: modelToUse,
+    onProgress: (p) => console.log(`Transcribing (${Math.round(p * 100)}%)...`),
+  });
+
+  console.log(transcription.map((t) => t.text).join(' '));
+});
+```
+
+## Functions
+
+<TableOfContents />
+
+## License
+
+MIT

packages/docs/docs/whisper-webgpu/resample-to-16khz.mdx

@@ -0,0 +1,50 @@
+---
+image: /generated/articles-docs-whisper-webgpu-resample-to-16khz.png
+title: resampleTo16Khz()
+crumb: '@remotion/whisper-webgpu'
+---
+
+:::warning
+**Unstable API**: This package is experimental for the moment. As we test it, we might make a few changes to the API.  
+:::
+
+# resampleTo16Khz()
+
+Processes an audio `File` or `Blob` by decoding it, converting it to mono, and resampling it to a 16kHz `Float32Array`. This prepares the audio data for use with the [`transcribe()`](/docs/whisper-webgpu/transcribe) function.
+
+This function operates in a browser environment as it relies on the Web Audio API (`AudioContext`, `OfflineAudioContext`) and `FileReader`.
+
+## Arguments
+
+### `file`
+
+The audio `File` or `Blob` object that you want to process. The function will attempt to decode the audio from common formats (e.g., WAV, MP3, Ogg) supported by the browser's Web Audio API.
+
+### `onProgress?`
+
+A callback function that receives progress updates during the resampling process. The `progress` value is a number between 0 and 1, where 0 indicates the start and 1 indicates completion.
+
+### `logLevel?`
+
+Default: `info`
+
+**Type:** `'trace' | 'verbose' | 'info' | 'warn' | 'error'`
+
+Optional. Determines the level of detail for logs printed to the console during the resampling process. Useful for debugging.
+
+## Return value
+
+`Promise<Float32Array>`
+
+This array contains the raw audio waveform data for a single channel (mono), sampled at 16kHz. This output is ready to be passed to the `channelWaveform` argument of the [`transcribe()`](/docs/whisper-webgpu/transcribe) function.
+
+## Behavior notes
+
+- **Browser environment:** This function is intended for use in a browser environment due to its reliance on Web Audio APIs (`AudioContext`, `OfflineAudioContext`) and `FileReader`.
+- **Audio decoding:** It uses the browser's built-in audio decoding capabilities. The range of supported audio formats may vary slightly between browsers.
+- **Output format:** The output is always a mono `Float32Array` at 16kHz, regardless of the input file's original channel count or sample rate.
+
+## See also
+
+- [`transcribe()`](/docs/whisper-webgpu/transcribe)
+- [`@remotion/whisper-webgpu`](/docs/whisper-webgpu)