feat: async modules / top level await (#5450)

### Description
For now this is just for top level await, in the future we want to be
able to support external esm modules

### Testing Instructions

Closes WEB-434
Closes WEB-987

crates/turbo-tasks-bytes/src/stream.rs

@@ -99,6 +99,16 @@ pub enum SingleValue<T> {
     Single(T),
 }
 
+impl<T: fmt::Debug> fmt::Debug for SingleValue<T> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            SingleValue::None => f.debug_struct("SingleValue::None").finish(),
+            SingleValue::Multiple => f.debug_struct("SingleValue::Multiple").finish(),
+            SingleValue::Single(v) => f.debug_tuple("SingleValue::Single").field(v).finish(),
+        }
+    }
+}
+
 impl<T: Clone + Send, S: StreamTrait<Item = T> + Send + Unpin + 'static> From<S> for Stream<T> {
     fn from(source: S) -> Self {
         Self::new_open(vec![], Box::new(source))

crates/turbo-tasks-macros/src/lib.rs

@@ -44,19 +44,74 @@ pub fn derive_task_input(input: TokenStream) -> TokenStream {
 /// Creates a Vc<Value> struct for a `struct` or `enum` that represent
 /// that type placed into a cell in a Task.
 ///
-/// That Vc<Value> object can be `.await?`ed to get a readonly reference
-/// to the original value.
+/// That Vc<Value> object can be `await`ed to get a readonly reference
+/// to the value contained in the cell.
 ///
-/// `into` argument (`#[turbo_tasks::value(into: xxx)]`)
+/// ## Arguments
+///
+/// Example: `#[turbo_tasks::value(into = "new", eq = "manual")]`
+///
+/// ### `cell`
+///
+/// Possible values:
+///
+/// - "new": Always overrides the value in the cell. Invalidating all
+/// dependent tasks.
+/// - "shared" (default): Compares with the existing value in the cell, before
+/// overriding it. Requires Value to implement [Eq].
+///
+/// ### `eq`
+///
+/// Possible values:
+///
+/// - "manual": Prevents deriving [Eq] so you can do it manually.
+///
+/// ### `into`
 ///
 /// When provided the Vc<Value> implement `From<Value>` to allow to convert
 /// a Value to a Vc<Value> by placing it into a cell in a Task.
 ///
-/// `into: new`: Always overrides the value in the cell. Invalidating all
-/// dependent tasks.
+/// Possible values:
 ///
-/// `into: shared`: Compares with the existing value in the cell, before
+/// - "new": Always overrides the value in the cell. Invalidating all
+/// dependent tasks.
+/// - "shared": Compares with the existing value in the cell, before
 /// overriding it. Requires Value to implement [Eq].
+/// - "none" (default): Prevents implementing `From<Value>`.
+///
+/// ### `serialization`
+///
+/// Affects serialization via [serde::Serialize] and [serde::Deserialize].
+///
+/// Possible values:
+///
+/// - "auto" (default): Derives the serialization traits and enabled
+///   serialization.
+/// - "auto_for_input": Same as "auto", but also adds the marker trait
+///   [turbo_tasks::TypedForInput].
+/// - "custom": Prevents deriving the serialization traits, but still enables
+///   serialization (you need to manually implement [serde::Serialize] and
+///   [serde::Deserialize]).
+/// - "custom_for_input":Same as "auto", but also adds the marker trait
+///   [turbo_tasks::TypedForInput].
+/// - "none": Disables serialization and prevents deriving the traits.
+///
+/// ### `shared`
+///
+/// Sets both `cell = "shared"` and `into = "shared"`
+///
+/// No value.
+///
+/// Example: `#[turbo_tasks::value(shared)]`
+///
+/// ### `transparent`
+///
+/// If applied to a unit struct (e.g. `struct Wrapper(Value)`) the outer struct
+/// is skipped for all operations (cell, into, reading).
+///
+/// No value.
+///
+/// Example: `#[turbo_tasks::value(transparent)]`
 ///
 /// TODO: add more documentation: presets, traits
 #[allow_internal_unstable(min_specialization, into_future, trivial_bounds)]

crates/turbo-tasks/src/primitives.rs

@@ -1,11 +1,13 @@
-use std::ops::Deref;
+use std::{future::IntoFuture, ops::Deref};
 
+use anyhow::Result;
 use auto_hash_map::AutoSet;
+use futures::TryFutureExt;
 // This specific macro identifier is detected by turbo-tasks-build.
 use turbo_tasks_macros::primitive as __turbo_tasks_internal_primitive;
 
 use crate::{
-    RawVc, Vc, {self as turbo_tasks},
+    RawVc, TryJoinIterExt, Vc, {self as turbo_tasks},
 };
 
 __turbo_tasks_internal_primitive!(());
@@ -74,6 +76,46 @@ __turbo_tasks_internal_primitive!(AutoSet<RawVc>);
 __turbo_tasks_internal_primitive!(serde_json::Value);
 __turbo_tasks_internal_primitive!(Vec<u8>);
 
+__turbo_tasks_internal_primitive!(Vec<bool>);
+
+#[turbo_tasks::value(transparent)]
+pub struct Bools(Vec<Vc<bool>>);
+
+#[turbo_tasks::value_impl]
+impl Bools {
+    #[turbo_tasks::function]
+    pub fn empty() -> Vc<Bools> {
+        Vc::cell(Vec::new())
+    }
+
+    #[turbo_tasks::function]
+    async fn into_bools(self: Vc<Bools>) -> Result<Vc<Vec<bool>>> {
+        let this = self.await?;
+
+        let bools = this
+            .iter()
+            .map(|b| b.into_future().map_ok(|b| *b))
+            .try_join()
+            .await?;
+
+        Ok(Vc::cell(bools))
+    }
+
+    #[turbo_tasks::function]
+    pub async fn all(self: Vc<Bools>) -> Result<Vc<bool>> {
+        let bools = self.into_bools().await?;
+
+        Ok(Vc::cell(bools.iter().all(|b| *b)))
+    }
+
+    #[turbo_tasks::function]
+    pub async fn any(self: Vc<Bools>) -> Result<Vc<bool>> {
+        let bools = self.into_bools().await?;
+
+        Ok(Vc::cell(bools.iter().any(|b| *b)))
+    }
+}
+
 #[turbo_tasks::value(transparent, eq = "manual")]
 #[derive(Debug, Clone)]
 pub struct Regex(

crates/turbopack-core/src/issue/mod.rs

@@ -328,8 +328,8 @@ pub struct Issues(Vec<Vc<Box<dyn Issue>>>);
 
 /// A list of issues captured with [`Issue::peek_issues_with_path`] and
 /// [`Issue::take_issues_with_path`].
-#[derive(Debug)]
 #[turbo_tasks::value]
+#[derive(Debug)]
 pub struct CapturedIssues {
     issues: AutoSet<Vc<Box<dyn Issue>>>,
     #[cfg(feature = "issue_path")]

crates/turbopack-core/src/resolve/mod.rs

@@ -1438,7 +1438,7 @@ pub async fn handle_resolve_error(
     })
 }
 
-/// ModulePart represnts a part of a module.
+/// ModulePart represents a part of a module.
 ///
 /// Currently this is used only for ESMs.
 #[turbo_tasks::value]

crates/turbopack-ecmascript-runtime/js/src/build/runtime.ts

@@ -1,4 +1,5 @@
 /// <reference path="../shared/runtime-utils.ts" />
+/// <reference path="../shared-node/require.ts" />
 
 declare var RUNTIME_PUBLIC_PATH: string;
 
@@ -24,27 +25,12 @@ type SourceInfo =
       parentId: ModuleId;
     };
 
-interface RequireContextEntry {
-  external: boolean;
-}
-
 type ExternalRequire = (id: ModuleId) => Exports | EsmNamespaceObject;
+type ExternalImport = (id: ModuleId) => Promise<Exports | EsmNamespaceObject>;
 
-interface TurbopackNodeBuildContext {
-  e: Module["exports"];
-  r: CommonJsRequire;
+interface TurbopackNodeBuildContext extends TurbopackBaseContext {
   x: ExternalRequire;
-  f: RequireContextFactory;
-  i: EsmImport;
-  s: EsmExport;
-  j: typeof dynamicExport;
-  v: ExportValue;
-  n: typeof exportNamespace;
-  m: Module;
-  c: ModuleCache;
-  l: LoadChunk;
-  g: typeof globalThis;
-  __dirname: string;
+  y: ExternalImport;
 }
 
 type ModuleFactory = (
@@ -59,47 +45,6 @@ const RUNTIME_ROOT = path.resolve(__filename, relativePathToRuntimeRoot);
 const moduleFactories: ModuleFactories = Object.create(null);
 const moduleCache: ModuleCache = Object.create(null);
 
-function commonJsRequireContext(
-  entry: RequireContextEntry,
-  sourceModule: Module
-): Exports {
-  return entry.external
-    ? externalRequire(entry.id(), false)
-    : commonJsRequire(sourceModule, entry.id());
-}
-
-function externalRequire(
-  id: ModuleId,
-  esm: boolean = false
-): Exports | EsmNamespaceObject {
-  let raw;
-  try {
-    raw = require(id);
-  } catch (err) {
-    // TODO(alexkirsz) This can happen when a client-side module tries to load
-    // an external module we don't provide a shim for (e.g. querystring, url).
-    // For now, we fail semi-silently, but in the future this should be a
-    // compilation error.
-    throw new Error(`Failed to load external module ${id}: ${err}`);
-  }
-  if (!esm || raw.__esModule) {
-    return raw;
-  }
-  const ns = {};
-  interopEsm(raw, ns, true);
-  return ns;
-}
-externalRequire.resolve = (
-  id: string,
-  options?:
-    | {
-        paths?: string[] | undefined;
-      }
-    | undefined
-) => {
-  return require.resolve(id, options);
-};
-
 function loadChunk(chunkPath: ChunkPath) {
   if (!chunkPath.endsWith(".js")) {
     // We only support loading JS chunks in Node.js.
@@ -176,13 +121,15 @@ function instantiateModule(id: ModuleId, source: SourceInfo): Module {
   // NOTE(alexkirsz) This can fail when the module encounters a runtime error.
   try {
     moduleFactory.call(module.exports, {
+      a: asyncModule.bind(null, module),
       e: module.exports,
       r: commonJsRequire.bind(null, module),
       x: externalRequire,
+      y: externalImport,
       f: requireContext.bind(null, module),
       i: esmImport.bind(null, module),
       s: esm.bind(null, module.exports),
-      j: dynamicExport.bind(null, module),
+      j: dynamicExport.bind(null, module, module.exports),
       v: exportValue.bind(null, module),
       n: exportNamespace.bind(null, module),
       m: module,

crates/turbopack-ecmascript-runtime/js/src/dev/runtime/base/runtime-base.ts

@@ -33,21 +33,8 @@ type RefreshContext = {
 
 type RefreshHelpers = RefreshRuntimeGlobals["$RefreshHelpers$"];
 
-interface TurbopackDevBaseContext {
-  e: Module["exports"];
-  r: CommonJsRequire;
-  f: RequireContextFactory;
-  i: EsmImport;
-  s: EsmExport;
-  j: typeof dynamicExport;
-  v: ExportValue;
-  n: typeof exportNamespace;
-  m: Module;
-  c: ModuleCache;
-  l: LoadChunk;
-  g: typeof globalThis;
+interface TurbopackDevBaseContext extends TurbopackBaseContext {
   k: RefreshContext;
-  __dirname: string;
 }
 
 interface TurbopackDevContext extends TurbopackDevBaseContext {}
@@ -333,12 +320,13 @@ function instantiateModule(id: ModuleId, source: SourceInfo): Module {
       moduleFactory.call(
         module.exports,
         augmentContext({
+          a: asyncModule.bind(null, module),
           e: module.exports,
           r: commonJsRequire.bind(null, module),
           f: requireContext.bind(null, module),
           i: esmImport.bind(null, module),
-          s: esmExport.bind(null, module),
-          j: dynamicExport.bind(null, module),
+          s: esmExport.bind(null, module, module.exports),
+          j: dynamicExport.bind(null, module, module.exports),
           v: exportValue.bind(null, module),
           n: exportNamespace.bind(null, module),
           m: module,

crates/turbopack-ecmascript-runtime/js/src/dev/runtime/nodejs/runtime-backend-nodejs.ts

@@ -6,62 +6,25 @@
  */
 
 /// <reference path="../base/runtime-base.ts" />
+/// <reference path="../../../shared-node/require.ts" />
 
 interface RequireContextEntry {
   // Only the Node.js backend has this flag.
   external: boolean;
 }
 
 type ExternalRequire = (id: ModuleId) => Exports | EsmNamespaceObject;
+type ExternalImport = (id: ModuleId) => Promise<Exports | EsmNamespaceObject>;
 
-interface TurbopackDevContext {
+interface TurbopackDevContext extends TurbopackDevBaseContext {
   x: ExternalRequire;
+  y: ExternalImport;
 }
 
-function commonJsRequireContext(
-  entry: RequireContextEntry,
-  sourceModule: Module
-): Exports {
-  return entry.external
-    ? externalRequire(entry.id(), false)
-    : commonJsRequire(sourceModule, entry.id());
-}
-
-function externalRequire(
-  id: ModuleId,
-  esm: boolean = false
-): Exports | EsmNamespaceObject {
-  let raw;
-  try {
-    raw = require(id);
-  } catch (err) {
-    // TODO(alexkirsz) This can happen when a client-side module tries to load
-    // an external module we don't provide a shim for (e.g. querystring, url).
-    // For now, we fail semi-silently, but in the future this should be a
-    // compilation error.
-    throw new Error(`Failed to load external module ${id}: ${err}`);
-  }
-  if (!esm) {
-    return raw;
-  }
-  const ns = {};
-  interopEsm(raw, ns, raw.__esModule);
-  return ns;
-}
-externalRequire.resolve = (
-  id: string,
-  options?:
-    | {
-        paths?: string[] | undefined;
-      }
-    | undefined
-) => {
-  return require.resolve(id, options);
-};
-
 function augmentContext(context: TurbopackDevBaseContext): TurbopackDevContext {
   const nodejsContext = context as TurbopackDevContext;
   nodejsContext.x = externalRequire;
+  nodejsContext.y = externalImport;
   return nodejsContext;
 }