Node.js production runtime POC (#4998)

### Description

This implements a Node.js production runtime for Turbopack, to first be
used for `next build --turbo`, but that can easily be extended to
support more production use cases (e.g. ncc).

The runtime works differently from the dev runtime we're currently using
for `next dev`: instead of having each chunk register itself against a
global registry when evaluated, chunks export their module factories as
a CommonJS default export. The runtime itself lives in a separate chunk,
and then an "exported chunk" is generated that instantiates runtime
entries and exports a given entry module.

Corresponding Next.js PR:
~~vercel/next.js#49942
vercel/next.js#50375

---------

Co-authored-by: Tobias Koppers <tobias.koppers@googlemail.com>

.github/workflows/test.yml

@@ -107,6 +107,13 @@ jobs:
             crates/turbopack-bench/**
             !*.md
 
+      - name: Turbopack TypeScript related changes
+        id: turbopack_typescript
+        uses: technote-space/get-diff-action@v6
+        with:
+          PATTERNS: |
+            crates/turbopack*/js/**
+
       - name: Turborepo Rust related changes
         id: turborepo_rust
         uses: technote-space/get-diff-action@v6
@@ -172,6 +179,7 @@ jobs:
       # We only test workspace dependency changes on main, not on PRs to speed up CI
       cargo_on_main: ${{ steps.ci.outputs.diff != '' || (steps.cargo.outputs.diff != '' && github.event_name == 'push' && github.ref == 'refs/heads/main') }}
       turbopack: ${{ steps.ci.outputs.diff != '' || steps.turbopack.outputs.diff != '' }}
+      turbopack_typescript: ${{ steps.ci.outputs.diff != '' || steps.turbopack_typescript.outputs.diff != '' }}
       turborepo_rust: ${{ steps.ci.outputs.diff != '' || steps.turborepo_rust.outputs.diff != '' }}
       turbopack_bench: ${{ steps.ci.outputs.diff != '' || steps.turbopack_bench.outputs.diff != '' }}
       go: ${{ steps.ci.outputs.diff != '' || steps.turborepo_go.outputs.diff != '' }}
@@ -446,6 +454,33 @@ jobs:
         run: |
           turbo run check-types test --filter=...[${{ github.event.pull_request.base.sha || 'HEAD^1' }}] --filter="./packages/*" --color
 
+  turbopack_typescript:
+    name: Turbopack TypeScript files
+    runs-on: ubuntu-latest
+    needs: determine_jobs
+    if: needs.determine_jobs.outputs.turbopack_typescript == 'true'
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Setup Node.js
+        uses: ./.github/actions/setup-node
+
+      - name: Install dependencies
+        run: pnpm install -r --side-effects-cache false
+
+      - name: Check turbopack-node types
+        working-directory: crates/turbopack-node/js
+        run: pnpm run check
+
+      - name: Check turbopack-cli types
+        working-directory: crates/turbopack-cli/js
+        run: pnpm run check
+
+      - name: Check turbopack-ecmascript-runtime types
+        working-directory: crates/turbopack-ecmascript-runtime/js
+        run: pnpm run check
+
   rust_prepare:
     name: Check rust crates
     runs-on: ubuntu-latest-16-core-oss
@@ -1183,6 +1218,7 @@ jobs:
       - rust_lint
       - rust_check
       - turbopack_rust_test1
+      - turbopack_typescript
       - turborepo_rust_test
       - turbopack_rust_test_bench1
       - turbopack_build_release
@@ -1220,6 +1256,7 @@ jobs:
           subjob ${{needs.rust_lint.result}} "Rust lints"
           subjob ${{needs.rust_check.result}} "Rust checks"
           subjob ${{needs.turbopack_rust_test1.result}} "Turbopack Rust tests (linux)"
+          subjob ${{needs.turbopack_typescript.result}} "Turbopack Typescript checks"
           subjob ${{needs.turborepo_rust_test.result}} "TurboRepo Rust tests"
           subjob ${{needs.turbopack_rust_test_bench1.result}} "Turbopack Rust benchmark tests (linux)"
           subjob ${{needs.turbopack_build_release.result}} "Turbopack Release Build"
@@ -1298,6 +1335,7 @@ jobs:
       - rust_check
       - turbopack_rust_test1
       - turbopack_rust_test2
+      - turbopack_typescript
       - turborepo_rust_test
       - turbopack_rust_test_bench1
       - turbopack_rust_test_bench2
@@ -1338,6 +1376,7 @@ jobs:
           subjob ${{needs.rust_check.result}} "Rust checks"
           subjob ${{needs.turbopack_rust_test1.result}} "Turbopack Rust tests (linux)"
           subjob ${{needs.turbopack_rust_test2.result}} "Turbopack Rust tests (mac/win, non-blocking)"
+          subjob ${{needs.turbopack_typescript.result}} "Turbopack Typescript checks"
           subjob ${{needs.turborepo_rust_test.result}} "TurboRepo Rust tests"
           subjob ${{needs.turbopack_rust_test_bench1.result}} "Turbopack Rust benchmark tests (linux)"
           subjob ${{needs.turbopack_rust_test_bench2.result}} "Turbopack Rust benchmark tests (mac/win, non-blocking)"

Cargo.toml

@@ -92,6 +92,7 @@ turbo-tasks-testing = { path = "crates/turbo-tasks-testing" }
 turbo-updater = { path = "crates/turborepo-updater" }
 turbopack = { path = "crates/turbopack" }
 turbopack-bench = { path = "crates/turbopack-bench" }
+turbopack-build = { path = "crates/turbopack-build" }
 turbopack-cli = { path = "crates/turbopack-cli" }
 turbopack-cli-utils = { path = "crates/turbopack-cli-utils" }
 turbopack-core = { path = "crates/turbopack-core" }
@@ -101,6 +102,7 @@ turbopack-dev = { path = "crates/turbopack-dev" }
 turbopack-dev-server = { path = "crates/turbopack-dev-server" }
 turbopack-ecmascript = { path = "crates/turbopack-ecmascript" }
 turbopack-ecmascript-plugins = { path = "crates/turbopack-ecmascript-plugins" }
+turbopack-ecmascript-runtime = { path = "crates/turbopack-ecmascript-runtime" }
 turbopack-env = { path = "crates/turbopack-env" }
 turbopack-image = { path = "crates/turbopack-image" }
 turbopack-json = { path = "crates/turbopack-json" }

crates/turbopack-binding/Cargo.toml

@@ -97,6 +97,10 @@ __turbo_updater = ["__turbo", "turbo-updater"]
 
 __turbopack = ["turbopack"]
 __turbopack_bench = ["__turbopack", "turbopack-bench"]
+__turbopack_build = ["__turbopack", "turbopack-build"]
+__turbopack_build_dynamic_embed_contents = [
+  "turbopack-build/dynamic_embed_contents",
+]
 __turbopack_cli_utils = ["__turbopack", "turbopack-cli-utils"]
 __turbopack_core = ["__turbopack", "turbopack-core"]
 __turbopack_core_issue_path = ["turbopack-core/issue_path"]
@@ -114,6 +118,8 @@ __turbopack_ecmascript_plugin = [
   "turbopack-ecmascript-plugins",
   "turbopack-ecmascript-plugins/transform_emotion",
 ]
+__turbopack_ecmascript_runtime = ["__turbopack", "turbopack-ecmascript-runtime"]
+
 __turbopack_env = ["__turbopack", "turbopack-env"]
 __turbopack_image = ["__turbopack", "turbopack-image"]
 __turbopack_image_avif = ["turbopack-image/avif"]
@@ -186,6 +192,7 @@ turbo-tasks-testing = { optional = true, workspace = true }
 turbo-updater = { optional = true, workspace = true }
 turbopack = { optional = true, workspace = true }
 turbopack-bench = { optional = true, workspace = true }
+turbopack-build = { optional = true, workspace = true }
 turbopack-cli-utils = { optional = true, workspace = true }
 turbopack-core = { optional = true, workspace = true }
 turbopack-create-test-app = { optional = true, workspace = true }
@@ -194,6 +201,7 @@ turbopack-dev = { optional = true, workspace = true }
 turbopack-dev-server = { optional = true, workspace = true }
 turbopack-ecmascript = { optional = true, workspace = true }
 turbopack-ecmascript-plugins = { optional = true, workspace = true }
+turbopack-ecmascript-runtime = { optional = true, workspace = true }
 turbopack-env = { optional = true, workspace = true }
 turbopack-image = { optional = true, workspace = true }
 turbopack-json = { optional = true, workspace = true }

crates/turbopack-binding/src/lib.rs

@@ -56,6 +56,8 @@ pub mod turbopack {
     pub use turbopack;
     #[cfg(feature = "__turbopack_bench")]
     pub use turbopack_bench as bench;
+    #[cfg(feature = "__turbopack_build")]
+    pub use turbopack_build as build;
     #[cfg(feature = "__turbopack_cli_utils")]
     pub use turbopack_cli_utils as cli_utils;
     #[cfg(feature = "__turbopack_core")]
@@ -72,6 +74,8 @@ pub mod turbopack {
     pub use turbopack_ecmascript as ecmascript;
     #[cfg(feature = "__turbopack_ecmascript_plugin")]
     pub use turbopack_ecmascript_plugins as ecmascript_plugin;
+    #[cfg(feature = "__turbopack_ecmascript_runtime")]
+    pub use turbopack_ecmascript_runtime as ecmascript_runtime;
     #[cfg(feature = "__turbopack_env")]
     pub use turbopack_env as env;
     #[cfg(feature = "__turbopack_image")]

crates/turbopack-build/Cargo.toml

@@ -0,0 +1,34 @@
+[package]
+name = "turbopack-build"
+version = "0.1.0"
+description = "TBD"
+license = "MPL-2.0"
+edition = "2021"
+autobenches = false
+
+[lib]
+bench = false
+
+[features]
+# enable "HMR" for embedded assets
+dynamic_embed_contents = ["turbo-tasks-fs/dynamic_embed_contents"]
+# enable test utilities such as `RuntimeType::Dummy`
+test = ["turbopack-ecmascript-runtime/test"]
+
+[dependencies]
+anyhow = { workspace = true }
+indexmap = { workspace = true }
+indoc = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+serde_qs = { workspace = true }
+
+turbo-tasks = { workspace = true }
+turbo-tasks-fs = { workspace = true }
+turbopack-core = { workspace = true }
+turbopack-css = { workspace = true }
+turbopack-ecmascript = { workspace = true }
+turbopack-ecmascript-runtime = { workspace = true }
+
+[build-dependencies]
+turbo-tasks-build = { workspace = true }

crates/turbopack-build/build.rs

@@ -0,0 +1,5 @@
+use turbo_tasks_build::generate_register;
+
+fn main() {
+    generate_register();
+}