Refactor ES chunk evaluate logic into a Runtime trait (#4141)

### Description

This refactors the EcmaScript chunk evaluate segment logic into a
separate `EcmascriptChunkRuntime` trait.

`ChunkingContext` implementors can now optionally implement the
`EcmascriptChunkContext` trait (which already existed before as a
struct). This trait has two additional methods:
* `EcmascriptChunkContext::ecmascript_runtime()` -> Creates a ES runtime
that only registers chunks' contents
* `EcmascriptChunkContext::evaluated_ecmascript_runtime()` -> Creates an
ES runtime that registers chunks' contents and evaluates its entries

This is important because we are going to start having more than one
runtime evaluation logic with `next build` (and `ncc` and any future
bundler-specific work). In the future, this could even be used to have
non-Ecmascript runtimes.

Also fixes WEB-688

PR for the Next.js side: vercel/next.js#47116

### TODO

- [x] ~~I need to ensure that `waitForChunksToLoad` works as expected on
first load and does not re-create requests to JS and CSS chunks that are
already being loaded.~~

- [x] ~~Looks like there's a hashing problem (module ident perhaps?)
that creates additional snapshots.~~

### Testing Instructions

Snapshots + Integrations tests

Cargo.toml

@@ -22,6 +22,7 @@ members = [
   "crates/turbopack-core",
   "crates/turbopack-create-test-app",
   "crates/turbopack-css",
+  "crates/turbopack-dev",
   "crates/turbopack-dev-server",
   "crates/turbopack-ecmascript",
   "crates/turbopack-env",
@@ -60,6 +61,7 @@ default-members = [
   "crates/turbopack-core",
   "crates/turbopack-create-test-app",
   "crates/turbopack-css",
+  "crates/turbopack-dev",
   "crates/turbopack-dev-server",
   "crates/turbopack-ecmascript",
   "crates/turbopack-env",
@@ -113,6 +115,7 @@ turbopack-cli-utils = { path = "crates/turbopack-cli-utils" }
 turbopack-core = { path = "crates/turbopack-core" }
 turbopack-create-test-app = { path = "crates/turbopack-create-test-app" }
 turbopack-css = { path = "crates/turbopack-css" }
+turbopack-dev = { path = "crates/turbopack-dev" }
 turbopack-dev-server = { path = "crates/turbopack-dev-server" }
 turbopack-ecmascript = { path = "crates/turbopack-ecmascript" }
 turbopack-env = { path = "crates/turbopack-env" }

crates/turbopack-core/src/chunk/list/update.rs

@@ -136,8 +136,9 @@ pub(super) async fn update_chunk_list(
             let chunk_update = content.update(TraitRef::cell(from_version.clone())).await?;
 
             match &*chunk_update {
-                // Getting a total or not found update from a merger is unexpected. If it happens,
-                // we have no better option than to short-circuit the update.
+                // Getting a total or not found update from a merger is unexpected. If it
+                // happens, we have no better option than to short-circuit
+                // the update.
                 Update::Total(_) => {
                     return Ok(Update::Total(TotalUpdate {
                         to: to_version.into(),

crates/turbopack-core/src/chunk/mod.rs

@@ -1,7 +1,6 @@
 pub mod availability_info;
 pub mod available_assets;
 pub mod chunk_in_group;
-pub mod dev;
 pub(crate) mod list;
 pub mod optimize;
 

crates/turbopack-core/src/code_builder.rs

@@ -6,7 +6,9 @@ use std::{
 
 use anyhow::Result;
 use sourcemap::SourceMapBuilder;
+use turbo_tasks::primitives::U64Vc;
 use turbo_tasks_fs::rope::{Rope, RopeBuilder};
+use turbo_tasks_hash::hash_xxh3_hash64;
 
 use crate::{
     source_map::{GenerateSourceMap, GenerateSourceMapVc, SourceMapSection, SourceMapVc},
@@ -176,6 +178,17 @@ impl GenerateSourceMap for Code {
     }
 }
 
+#[turbo_tasks::value_impl]
+impl CodeVc {
+    /// Returns the hash of the source code of this Code.
+    #[turbo_tasks::function]
+    pub async fn source_code_hash(self) -> Result<U64Vc> {
+        let code = self.await?;
+        let hash = hash_xxh3_hash64(code.source_code());
+        Ok(U64Vc::cell(hash))
+    }
+}
+
 /// A source map that contains no actual source location information (no
 /// `sources`, no mappings that point into a source). This is used to tell
 /// Chrome that the generated code starting at a particular offset is no longer

crates/turbopack-css/src/module_asset.rs

@@ -30,9 +30,9 @@ use turbopack_ecmascript::{
     chunk::{
         EcmascriptChunkItem, EcmascriptChunkItemContent, EcmascriptChunkItemContentVc,
         EcmascriptChunkItemVc, EcmascriptChunkPlaceable, EcmascriptChunkPlaceableVc,
-        EcmascriptChunkVc, EcmascriptExports, EcmascriptExportsVc,
+        EcmascriptChunkVc, EcmascriptChunkingContextVc, EcmascriptExports, EcmascriptExportsVc,
     },
-    utils::stringify_js,
+    utils::StringifyJs,
     ParseResultSourceMap, ParseResultSourceMapVc,
 };
 
@@ -213,7 +213,7 @@ impl EcmascriptChunkPlaceable for ModuleCssModuleAsset {
     #[turbo_tasks::function]
     fn as_chunk_item(
         self_vc: ModuleCssModuleAssetVc,
-        context: ChunkingContextVc,
+        context: EcmascriptChunkingContextVc,
     ) -> EcmascriptChunkItemVc {
         ModuleChunkItem {
             context,
@@ -245,7 +245,7 @@ impl ResolveOrigin for ModuleCssModuleAsset {
 #[turbo_tasks::value]
 struct ModuleChunkItem {
     module: ModuleCssModuleAssetVc,
-    context: ChunkingContextVc,
+    context: EcmascriptChunkingContextVc,
 }
 
 #[turbo_tasks::value_impl]
@@ -277,7 +277,7 @@ impl ChunkItem for ModuleChunkItem {
 #[turbo_tasks::value_impl]
 impl EcmascriptChunkItem for ModuleChunkItem {
     #[turbo_tasks::function]
-    fn chunking_context(&self) -> ChunkingContextVc {
+    fn chunking_context(&self) -> EcmascriptChunkingContextVc {
         self.context
     }
 
@@ -332,24 +332,24 @@ impl EcmascriptChunkItem for ModuleChunkItem {
                             unreachable!("ModuleCssModuleAsset implements EcmascriptChunkPlaceableVc");
                         };
 
-                        let module_id =
-                            stringify_js(&*placeable.as_chunk_item(self.context).id().await?);
-                        let original_name = stringify_js(original_name);
+                        let module_id = placeable.as_chunk_item(self.context).id().await?;
+                        let module_id = StringifyJs(&*module_id);
+                        let original_name = StringifyJs(original_name);
                         exported_class_names.push(format! {
                             "__turbopack_import__({module_id})[{original_name}]"
                         });
                     }
                     ModuleCssClass::Local { name: class_name }
                     | ModuleCssClass::Global { name: class_name } => {
-                        exported_class_names.push(stringify_js(class_name));
+                        exported_class_names.push(StringifyJs(class_name).to_string());
                     }
                 }
             }
 
             writeln!(
                 code,
                 "  {}: {},",
-                stringify_js(export_name),
+                StringifyJs(export_name),
                 exported_class_names.join(" + \" \" + ")
             )?;
         }

crates/turbopack-dev/Cargo.toml

@@ -0,0 +1,27 @@
+[package]
+name = "turbopack-dev"
+version = "0.1.0"
+description = "TBD"
+license = "MPL-2.0"
+edition = "2021"
+autobenches = false
+
+[lib]
+bench = false
+
+[dependencies]
+anyhow = { workspace = true }
+indexmap = { workspace = true }
+indoc = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+serde_qs = { workspace = true }
+
+turbo-tasks = { workspace = true }
+turbo-tasks-fs = { workspace = true }
+turbo-tasks-hash = { workspace = true }
+turbopack-core = { workspace = true }
+turbopack-ecmascript = { workspace = true }
+
+[build-dependencies]
+turbo-tasks-build = { workspace = true }

crates/turbopack-dev/build.rs

@@ -0,0 +1,5 @@
+use turbo_tasks_build::generate_register;
+
+fn main() {
+    generate_register();
+}

crates/turbopack-ecmascript/js/package.json → crates/turbopack-dev/js/package.json

@@ -1,7 +1,7 @@
 {
-  "name": "@vercel/turbopack-runtime",
+  "name": "@vercel/turbopack-dev-runtime",
   "version": "0.0.0",
-  "description": "turbopack runtime",
+  "description": "Turbopack development runtime",
   "license": "UNLICENSED",
   "private": true,
   "scripts": {