feat: basic wasm support for turbopack (#5618)

Cargo.toml

@@ -45,6 +45,7 @@ default-members = [
   "crates/turbopack-swc-utils",
   "crates/turbopack-test-utils",
   "crates/turbopack-tests",
+  "crates/turbopack-wasm",
   "xtask",
 ]
 
@@ -126,6 +127,7 @@ turbopack-static = { path = "crates/turbopack-static" }
 turbopack-swc-utils = { path = "crates/turbopack-swc-utils" }
 turbopack-test-utils = { path = "crates/turbopack-test-utils" }
 turbopack-tests = { path = "crates/turbopack-tests" }
+turbopack-wasm = { path = "crates/turbopack-wasm" }
 turbopath = { path = "crates/turborepo-paths" }
 turborepo = { path = "crates/turborepo" }
 turborepo-api-client = { path = "crates/turborepo-api-client" }

crates/turbopack-ecmascript-runtime/js/src/build/runtime.ts

@@ -1,5 +1,5 @@
 /// <reference path="../shared/runtime-utils.ts" />
-/// <reference path="../shared-node/require.ts" />
+/// <reference path="../shared-node/node-utils.ts" />
 
 declare var RUNTIME_PUBLIC_PATH: string;
 
@@ -52,7 +52,7 @@ function loadChunk(chunkPath: ChunkPath) {
     return;
   }
 
-  const resolved = require.resolve(path.resolve(RUNTIME_ROOT, chunkPath));
+  const resolved = path.resolve(RUNTIME_ROOT, chunkPath);
   const chunkModules: ModuleFactories = require(resolved);
 
   for (const [moduleId, moduleFactory] of Object.entries(chunkModules)) {
@@ -74,6 +74,12 @@ function loadChunkAsync(source: SourceInfo, chunkPath: string): Promise<void> {
   });
 }
 
+function loadWebAssembly(chunkPath: ChunkPath, imports: WebAssembly.Imports) {
+  const resolved = path.resolve(RUNTIME_ROOT, chunkPath);
+
+  return loadWebAssemblyFromPath(resolved, imports);
+}
+
 function instantiateModule(id: ModuleId, source: SourceInfo): Module {
   const moduleFactory = moduleFactories[id];
   if (typeof moduleFactory !== "function") {
@@ -134,6 +140,7 @@ function instantiateModule(id: ModuleId, source: SourceInfo): Module {
       m: module,
       c: moduleCache,
       l: loadChunkAsync.bind(null, { type: SourceType.Parent, parentId: id }),
+      w: loadWebAssembly,
       g: globalThis,
       __dirname: module.id.replace(/(^|\/)[\/]+$/, ""),
     });

crates/turbopack-ecmascript-runtime/js/src/build/tsconfig.json

@@ -1,9 +1,9 @@
 {
   "extends": "../tsconfig.base.json",
   "compilerOptions": {
-    // environment
-    "lib": ["ESNext"],
+    // environment, we need WebWorker for WebAssembly types (not part of @types/node yet)
+    "lib": ["ESNext", "WebWorker"],
     "types": ["node"]
   },
   "include": ["*.ts"]
-}
+}

crates/turbopack-ecmascript-runtime/js/src/dev/runtime/base/dummy.ts

@@ -14,3 +14,4 @@ declare var augmentContext: (
   context: TurbopackDevBaseContext
 ) => TurbopackDevContext;
 declare var commonJsRequireContext: CommonJsRequireContext;
+declare var loadWebAssembly: (source: SourceInfo, wasmChunkPath: ChunkPath, imports: WebAssembly.Imports) => Exports;

crates/turbopack-ecmascript-runtime/js/src/dev/runtime/base/runtime-base.ts

@@ -316,6 +316,8 @@ function instantiateModule(id: ModuleId, source: SourceInfo): Module {
 
   // NOTE(alexkirsz) This can fail when the module encounters a runtime error.
   try {
+    const sourceInfo: SourceInfo = { type: SourceType.Parent, parentId: id };
+
     runModuleExecutionHooks(module, (refresh) => {
       moduleFactory.call(
         module.exports,
@@ -331,7 +333,8 @@ function instantiateModule(id: ModuleId, source: SourceInfo): Module {
           n: exportNamespace.bind(null, module),
           m: module,
           c: moduleCache,
-          l: loadChunk.bind(null, { type: SourceType.Parent, parentId: id }),
+          l: loadChunk.bind(null, sourceInfo),
+          w: loadWebAssembly.bind(null, sourceInfo),
           g: globalThis,
           k: refresh,
           __dirname: module.id.replace(/(^|\/)\/+$/, ""),

crates/turbopack-ecmascript-runtime/js/src/dev/runtime/base/tsconfig.json

@@ -1,9 +1,8 @@
 {
   "extends": "../../../tsconfig.base.json",
   "compilerOptions": {
-    // environment
-    "lib": ["ESNext"],
-    "target": "ESNext"
+    // environment, we need WebWorker for WebAssembly types
+    "lib": ["ESNext", "WebWorker"]
   },
   "include": ["runtime-base.ts", "dummy.ts"]
 }

crates/turbopack-ecmascript-runtime/js/src/dev/runtime/dom/runtime-backend-dom.ts

@@ -27,6 +27,19 @@ function commonJsRequireContext(
   return commonJsRequire(sourceModule, entry.id());
 }
 
+async function loadWebAssembly(
+  _source: SourceInfo,
+  wasmChunkPath: ChunkPath,
+  importsObj: WebAssembly.Imports
+): Promise<Exports> {
+  const chunkUrl = `/${getChunkRelativeUrl(wasmChunkPath)}`;
+
+  const req = fetch(chunkUrl);
+  const { instance } = await WebAssembly.instantiateStreaming(req, importsObj);
+
+  return instance.exports;
+}
+
 (() => {
   BACKEND = {
     async registerChunk(chunkPath, params) {

crates/turbopack-ecmascript-runtime/js/src/dev/runtime/nodejs/runtime-backend-nodejs.ts

@@ -6,7 +6,7 @@
  */
 
 /// <reference path="../base/runtime-base.ts" />
-/// <reference path="../../../shared-node/require.ts" />
+/// <reference path="../../../shared-node/node-utils.ts" />
 
 interface RequireContextEntry {
   // Only the Node.js backend has this flag.
@@ -28,6 +28,33 @@ function augmentContext(context: TurbopackDevBaseContext): TurbopackDevContext {
   return nodejsContext;
 }
 
+function resolveChunkPath(chunkPath: ChunkPath, source: SourceInfo) {
+  let fromChunkPath = undefined;
+  switch (source.type) {
+    case SourceType.Runtime:
+      fromChunkPath = source.chunkPath;
+      break;
+    case SourceType.Parent:
+      fromChunkPath = getFirstModuleChunk(source.parentId);
+      break;
+    case SourceType.Update:
+      break;
+  }
+
+  const path = require("node:path");
+  return path.resolve(__dirname, path.posix.relative(path.dirname(fromChunkPath), chunkPath));
+}
+
+function loadWebAssembly(
+  source: SourceInfo,
+  chunkPath: ChunkPath,
+  imports: WebAssembly.Imports
+) {
+  const resolved = resolveChunkPath(chunkPath, source);
+
+  return loadWebAssemblyFromPath(resolved, imports);
+}
+
 let BACKEND: RuntimeBackend;
 
 (() => {
@@ -67,28 +94,14 @@ let BACKEND: RuntimeBackend;
       return;
     }
 
-    let fromChunkPath = undefined;
-    switch (source.type) {
-      case SourceType.Runtime:
-        fromChunkPath = source.chunkPath;
-        break;
-      case SourceType.Parent:
-        fromChunkPath = getFirstModuleChunk(source.parentId);
-        break;
-      case SourceType.Update:
-        break;
-    }
-
     // We'll only mark the chunk as loaded once the script has been executed,
     // which happens in `registerChunk`. Hence the absence of `resolve()`.
-    const path = require("path");
-    const resolved = require.resolve(
-      "./" + path.relative(path.dirname(fromChunkPath), chunkPath)
-    );
+    const resolved = resolveChunkPath(chunkPath, source);
+
     require(resolved);
   }
 })();
 
-function _eval({ code, url, map }: EcmascriptModuleEntry): ModuleFactory {
+function _eval(_: EcmascriptModuleEntry): ModuleFactory {
   throw new Error("HMR evaluation is not implemented on this backend");
 }

crates/turbopack-ecmascript-runtime/js/src/dev/runtime/nodejs/tsconfig.json

@@ -1,8 +1,8 @@
 {
   "extends": "../../../tsconfig.base.json",
   "compilerOptions": {
-    // environment
-    "lib": ["ESNext"],
+    // environment, we need WebWorker for WebAssembly types (not part of @types/node yet)
+    "lib": ["ESNext", "WebWorker"],
     "types": ["node"]
   },
   "include": ["*.ts"]

crates/turbopack-ecmascript-runtime/js/src/dev/runtime/none/runtime-backend-none.ts

@@ -26,6 +26,14 @@ function commonJsRequireContext(
   return commonJsRequire(sourceModule, entry.id());
 }
 
+async function loadWebAssembly(
+  _source: SourceInfo,
+  _id: ModuleId,
+  _importsObj: any
+): Promise<Exports> {
+  throw new Error("loading WebAssemly is not supported");
+}
+
 (() => {
   BACKEND = {
     // The "none" runtime expects all chunks within the same chunk group to be

crates/turbopack-ecmascript-runtime/js/src/shared-node/require.ts → crates/turbopack-ecmascript-runtime/js/src/shared-node/node-utils.ts

@@ -46,3 +46,15 @@ externalRequire.resolve = (
 ) => {
   return require.resolve(id, options);
 };
+
+async function loadWebAssemblyFromPath(
+  path: string,
+  importsObj: WebAssembly.Imports
+): Promise<Exports> {
+  const { readFile } = require("fs/promises") as typeof import("fs/promises");
+
+  const buffer = await readFile(path);
+  const { instance } = await WebAssembly.instantiate(buffer, importsObj);
+
+  return instance.exports;
+}

crates/turbopack-ecmascript-runtime/js/src/shared-node/tsconfig.json

@@ -1,8 +1,9 @@
 {
   "extends": "../tsconfig.base.json",
   "compilerOptions": {
-    // environment
-    "lib": ["ESNext"]
+    // environment, we need WebWorker for WebAssembly types (not part of @types/node yet)
+    "lib": ["ESNext", "WebWorker"],
+    "types": ["node"]
   },
   "include": ["*.ts"]
 }

crates/turbopack-ecmascript-runtime/js/src/shared/runtime-types.d.ts

@@ -30,6 +30,7 @@ type ExportNamespace = (namespace: any) => void;
 type DynamicExport = (object: Record<string, any>) => void;
 
 type LoadChunk = (chunkPath: ChunkPath) => Promise<any> | undefined;
+type LoadWebAssembly = (wasmChunkPath: ChunkPath, imports: WebAssembly.Imports) => Exports;
 
 type ModuleCache = Record<ModuleId, Module>;
 type ModuleFactories = Record<ModuleId, ModuleFactory>;
@@ -57,6 +58,7 @@ interface TurbopackBaseContext {
   m: Module;
   c: ModuleCache;
   l: LoadChunk;
+  w: LoadWebAssembly,
   g: typeof globalThis;
   __dirname: string;
 }

crates/turbopack-ecmascript-runtime/js/src/shared/tsconfig.json

@@ -1,8 +1,8 @@
 {
   "extends": "../tsconfig.base.json",
   "compilerOptions": {
-    // environment
-    "lib": ["ESNext"]
+    // environment, we need WebWorker for WebAssembly types
+    "lib": ["ESNext", "WebWorker"]
   },
   "include": ["*.ts"]
 }

crates/turbopack-ecmascript-runtime/src/build_runtime.rs

@@ -15,7 +15,7 @@ pub async fn get_build_runtime_code(environment: Vc<Environment>) -> Result<Vc<C
     let shared_runtime_utils_code =
         embed_static_code(asset_context, "shared/runtime-utils.ts".to_string());
     let shared_node_utils_code =
-        embed_static_code(asset_context, "shared-node/require.ts".to_string());
+        embed_static_code(asset_context, "shared-node/node-utils.ts".to_string());
     let runtime_code = embed_static_code(asset_context, "build/runtime.ts".to_string());
 
     let mut code = CodeBuilder::default();

crates/turbopack-ecmascript-runtime/src/dev_runtime.rs

@@ -66,7 +66,7 @@ pub async fn get_dev_runtime_code(
 
     if matches!(chunk_loading, ChunkLoading::NodeJs) {
         code.push_code(
-            &*embed_static_code(asset_context, "shared-node/require.ts".to_string()).await?,
+            &*embed_static_code(asset_context, "shared-node/node-utils.ts".to_string()).await?,
         );
     }
 

crates/turbopack-ecmascript/src/chunk/esm_scope.rs

@@ -22,6 +22,9 @@ pub(crate) struct EsmScope {
     scc_graph: DiGraphMap<Vc<EsmScopeScc>, ()>,
 }
 
+/// Represents a strongly connected component in the EsmScope graph.
+///
+/// See https://en.wikipedia.org/wiki/Strongly_connected_component
 #[turbo_tasks::value(transparent)]
 pub(crate) struct EsmScopeScc(Vec<Vc<Box<dyn EcmascriptChunkPlaceable>>>);
 
@@ -33,6 +36,7 @@ pub(crate) struct EsmScopeSccs(Vec<Vc<EsmScopeScc>>);
 
 #[turbo_tasks::value_impl]
 impl EsmScope {
+    /// Create a new [EsmScope] from the availability root given.
     #[turbo_tasks::function]
     pub(crate) async fn new(availability_info: Value<AvailabilityInfo>) -> Result<Vc<Self>> {
         let assets = if let Some(root) = availability_info.current_availability_root() {
@@ -78,6 +82,8 @@ impl EsmScope {
         Ok(Self::cell(EsmScope { scc_map, scc_graph }))
     }
 
+    /// Gets the [EsmScopeScc] for a given [EcmascriptChunkPlaceable] if it's
+    /// part of this graph.
     #[turbo_tasks::function]
     pub(crate) async fn get_scc(
         self: Vc<Self>,
@@ -88,6 +94,7 @@ impl EsmScope {
         Ok(Vc::cell(this.scc_map.get(&placeable).copied()))
     }
 
+    /// Returns all direct children of an [EsmScopeScc].
     #[turbo_tasks::function]
     pub(crate) async fn get_scc_children(
         self: Vc<Self>,

crates/turbopack-ecmascript/src/chunk/item.rs

@@ -105,6 +105,9 @@ impl EcmascriptChunkItemContent {
         if this.options.exports {
             args.push("e: exports");
         }
+        if this.options.wasm {
+            args.push("w: __turbopack_wasm__");
+        }
         let mut code = CodeBuilder::default();
         let args = FormatIter(|| args.iter().copied().intersperse(", "));
         if this.options.this {
@@ -157,6 +160,9 @@ pub struct EcmascriptChunkItemOptions {
     /// or is importing async modules).
     pub async_module: Option<AsyncModuleOptions>,
     pub this: bool,
+    /// Whether this chunk item's module factory should include
+    /// `__turbopack_wasm__` to load WebAssembly.
+    pub wasm: bool,
     pub placeholder_for_future_extensions: (),
 }
 

crates/turbopack-ecmascript/src/lib.rs

@@ -17,7 +17,7 @@ pub mod magic_identifier;
 pub(crate) mod manifest;
 pub mod parse;
 mod path_visitor;
-pub(crate) mod references;
+pub mod references;
 pub mod resolve;
 pub(crate) mod special_cases;
 pub(crate) mod static_code;