Turbopack: scope hoist tree shaking modules as well

test/e2e/app-dir/dynamic-io-errors/utils.ts

@@ -35,13 +35,10 @@ export function getPrerenderOutput(
       lines.push(
         isMinified
           ? line
-              .replace(/at [\w.]+ \(.next[^)]+\)/, replaceNextDistStackFrame)
-              .replace(
-                /at ([\w.]+) \(<anonymous>\)/,
-                replaceAnonymousStackFrame
-              )
+              .replace(/at \S+ \(.next[^)]+\)/, replaceNextDistStackFrame)
+              .replace(/at (\S+) \(<anonymous>\)/, replaceAnonymousStackFrame)
           : line.replace(
-              /at ([\w.]+) \((webpack:\/\/)\/src[^)]+\)/,
+              /at (\S+) \((webpack:\/\/)\/src[^)]+\)/,
               `at $1 ($2<next-src>)`
             )
       )

turbopack/crates/turbopack-ecmascript/src/lib.rs

@@ -746,28 +746,14 @@ impl MergeableModule for EcmascriptModuleAsset {
         modules: Vc<MergeableModulesExposed>,
         entry_points: Vc<MergeableModules>,
     ) -> Result<Vc<Box<dyn ChunkableModule>>> {
-        Ok(Vc::upcast(*MergedEcmascriptModule::new(
-            modules
-                .await?
-                .iter()
-                .map(|(m, exposed)| {
-                    Ok((
-                        ResolvedVc::try_sidecast::<Box<dyn EcmascriptAnalyzable>>(*m)
-                            .context("expected EcmascriptAnalyzable")?,
-                        *exposed,
-                    ))
-                })
-                .collect::<Result<Vec<_>>>()?,
-            entry_points
-                .await?
-                .iter()
-                .map(|m| {
-                    ResolvedVc::try_sidecast::<Box<dyn EcmascriptAnalyzable>>(*m)
-                        .context("expected EcmascriptAnalyzable")
-                })
-                .collect::<Result<Vec<_>>>()?,
-            self.options().to_resolved().await?,
-        )))
+        Ok(Vc::upcast(
+            *MergedEcmascriptModule::new(
+                modules,
+                entry_points,
+                self.options().to_resolved().await?,
+            )
+            .await?,
+        ))
     }
 }
 
@@ -966,7 +952,7 @@ impl EcmascriptModuleContentOptions {
 
             let part_code_gens = part_references
                 .iter()
-                .map(|r| r.code_generation(**chunking_context))
+                .map(|r| r.code_generation(**chunking_context, scope_hoisting_context))
                 .try_join()
                 .await?;
 

turbopack/crates/turbopack-ecmascript/src/merged_module.rs

@@ -1,10 +1,10 @@
-use anyhow::Result;
+use anyhow::{Context, Result};
 use turbo_tasks::{ResolvedVc, Vc};
 use turbopack_core::{
     asset::{Asset, AssetContent},
     chunk::{
         AsyncModuleInfo, ChunkItem, ChunkType, ChunkableModule, ChunkingContext,
-        MergeableModuleExposure,
+        MergeableModuleExposure, MergeableModules, MergeableModulesExposed,
     },
     ident::AssetIdent,
     module::Module,
@@ -28,20 +28,34 @@ pub(crate) struct MergedEcmascriptModule {
 }
 
 impl MergedEcmascriptModule {
-    pub fn new(
-        modules: Vec<(
-            ResolvedVc<Box<dyn EcmascriptAnalyzable>>,
-            MergeableModuleExposure,
-        )>,
-        entry_points: Vec<ResolvedVc<Box<dyn EcmascriptAnalyzable>>>,
+    pub async fn new(
+        modules: Vc<MergeableModulesExposed>,
+        entry_points: Vc<MergeableModules>,
         options: ResolvedVc<EcmascriptOptions>,
-    ) -> ResolvedVc<Self> {
-        MergedEcmascriptModule {
-            modules,
-            entry_points,
+    ) -> Result<ResolvedVc<Self>> {
+        Ok(MergedEcmascriptModule {
+            modules: modules
+                .await?
+                .iter()
+                .map(|(m, exposed)| {
+                    Ok((
+                        ResolvedVc::try_sidecast::<Box<dyn EcmascriptAnalyzable>>(*m)
+                            .context("expected EcmascriptAnalyzable")?,
+                        *exposed,
+                    ))
+                })
+                .collect::<Result<Vec<_>>>()?,
+            entry_points: entry_points
+                .await?
+                .iter()
+                .map(|m| {
+                    ResolvedVc::try_sidecast::<Box<dyn EcmascriptAnalyzable>>(*m)
+                        .context("expected EcmascriptAnalyzable")
+                })
+                .collect::<Result<Vec<_>>>()?,
             options,
         }
-        .resolved_cell()
+        .resolved_cell())
     }
 }
 

turbopack/crates/turbopack-ecmascript/src/side_effect_optimization/facade/module.rs

@@ -6,7 +6,10 @@ use turbo_tasks::{ResolvedVc, Vc};
 use turbo_tasks_fs::{File, FileContent, glob::Glob};
 use turbopack_core::{
     asset::{Asset, AssetContent},
-    chunk::{AsyncModuleInfo, ChunkableModule, ChunkingContext, EvaluatableAsset},
+    chunk::{
+        AsyncModuleInfo, ChunkableModule, ChunkingContext, EvaluatableAsset, MergeableModule,
+        MergeableModules, MergeableModulesExposed,
+    },
     ident::AssetIdent,
     module::Module,
     module_graph::ModuleGraph,
@@ -17,7 +20,7 @@ use turbopack_core::{
 use super::chunk_item::EcmascriptModuleFacadeChunkItem;
 use crate::{
     AnalyzeEcmascriptModuleResult, EcmascriptAnalyzable, EcmascriptModuleContent,
-    EcmascriptModuleContentOptions, SpecifiedModuleType,
+    EcmascriptModuleContentOptions, EcmascriptOptions, MergedEcmascriptModule, SpecifiedModuleType,
     chunk::{EcmascriptChunkPlaceable, EcmascriptExports},
     code_gen::CodeGens,
     parse::ParseResult,
@@ -258,6 +261,9 @@ impl EcmascriptAnalyzable for EcmascriptModuleFacadeModule {
             part_references,
             code_generation: CodeGens::empty().to_resolved().await?,
             async_module: ResolvedVc::cell(Some(self.async_module().to_resolved().await?)),
+            // The facade module cannot generate source maps, because the inserted references
+            // contain spans from the original module, but the facade module itself doesn't have the
+            // original module's swc_common::SourceMap in `parsed`.
             generate_source_map: false,
             original_source_map: None,
             exports: self.get_exports().to_resolved().await?,
@@ -450,3 +456,22 @@ impl ChunkableModule for EcmascriptModuleFacadeModule {
 
 #[turbo_tasks::value_impl]
 impl EvaluatableAsset for EcmascriptModuleFacadeModule {}
+
+#[turbo_tasks::value_impl]
+impl MergeableModule for EcmascriptModuleFacadeModule {
+    #[turbo_tasks::function]
+    async fn merge(
+        self: Vc<Self>,
+        modules: Vc<MergeableModulesExposed>,
+        entry_points: Vc<MergeableModules>,
+    ) -> Result<Vc<Box<dyn ChunkableModule>>> {
+        Ok(Vc::upcast(
+            *MergedEcmascriptModule::new(
+                modules,
+                entry_points,
+                EcmascriptOptions::default().resolved_cell(),
+            )
+            .await?,
+        ))
+    }
+}

turbopack/crates/turbopack-ecmascript/src/side_effect_optimization/locals/module.rs

@@ -5,7 +5,10 @@ use turbo_tasks::{ResolvedVc, Vc};
 use turbo_tasks_fs::glob::Glob;
 use turbopack_core::{
     asset::{Asset, AssetContent},
-    chunk::{AsyncModuleInfo, ChunkableModule, ChunkingContext},
+    chunk::{
+        AsyncModuleInfo, ChunkableModule, ChunkingContext, MergeableModule, MergeableModules,
+        MergeableModulesExposed,
+    },
     ident::AssetIdent,
     module::Module,
     module_graph::ModuleGraph,
@@ -16,7 +19,7 @@ use turbopack_core::{
 use super::chunk_item::EcmascriptModuleLocalsChunkItem;
 use crate::{
     AnalyzeEcmascriptModuleResult, EcmascriptAnalyzable, EcmascriptModuleAsset,
-    EcmascriptModuleContent, EcmascriptModuleContentOptions,
+    EcmascriptModuleContent, EcmascriptModuleContentOptions, MergedEcmascriptModule,
     chunk::{EcmascriptChunkPlaceable, EcmascriptExports},
     references::{
         async_module::OptionAsyncModule,
@@ -205,3 +208,17 @@ impl ChunkableModule for EcmascriptModuleLocalsModule {
         )
     }
 }
+
+#[turbo_tasks::value_impl]
+impl MergeableModule for EcmascriptModuleLocalsModule {
+    #[turbo_tasks::function]
+    async fn merge(
+        &self,
+        modules: Vc<MergeableModulesExposed>,
+        entry_points: Vc<MergeableModules>,
+    ) -> Result<Vc<Box<dyn ChunkableModule>>> {
+        Ok(Vc::upcast(
+            *MergedEcmascriptModule::new(modules, entry_points, self.module.await?.options).await?,
+        ))
+    }
+}

turbopack/crates/turbopack-ecmascript/src/side_effect_optimization/reference.rs

@@ -1,6 +1,10 @@
 use anyhow::{Context, Result, bail};
 use serde::{Deserialize, Serialize};
-use swc_core::{common::DUMMY_SP, quote};
+use swc_core::{
+    common::DUMMY_SP,
+    ecma::ast::{Ident, Lit},
+    quote,
+};
 use turbo_rcstr::RcStr;
 use turbo_tasks::{NonLocalValue, ResolvedVc, ValueToString, Vc, trace::TraceRawVcs};
 use turbopack_core::{
@@ -17,8 +21,11 @@ use super::{
     facade::module::EcmascriptModuleFacadeModule, locals::module::EcmascriptModuleLocalsModule,
 };
 use crate::{
-    ScopeHoistingContext, chunk::EcmascriptChunkPlaceable, code_gen::CodeGeneration,
-    references::esm::base::ReferencedAsset, runtime_functions::TURBOPACK_IMPORT,
+    ScopeHoistingContext,
+    chunk::EcmascriptChunkPlaceable,
+    code_gen::{CodeGeneration, CodeGenerationHoistedStmt},
+    references::esm::base::{ReferencedAsset, ReferencedAssetIdent},
+    runtime_functions::TURBOPACK_IMPORT,
     utils::module_id_to_lit,
 };
 
@@ -152,29 +159,80 @@ impl EcmascriptModulePartReference {
     pub async fn code_generation(
         self: Vc<Self>,
         chunking_context: Vc<Box<dyn ChunkingContext>>,
+        scope_hoisting_context: ScopeHoistingContext<'_>,
     ) -> Result<CodeGeneration> {
         let referenced_asset = ReferencedAsset::from_resolve_result(self.resolve_reference());
         let referenced_asset = referenced_asset.await?;
-        let ident = referenced_asset
-            .get_ident(chunking_context, None, ScopeHoistingContext::None)
-            .await?
-            .context("part module reference should have an ident")?
-            .as_expr_individual(DUMMY_SP)
-            .unwrap_left();
+        let part = &self.await?.part;
 
         let ReferencedAsset::Some(module) = *referenced_asset else {
             bail!("part module reference should have an module reference");
         };
-        let id = module.chunk_item_id(Vc::upcast(chunking_context)).await?;
-
-        Ok(CodeGeneration::hoisted_stmt(
-            ident.sym.as_str().into(),
-            quote!(
-                "var $name = $turbopack_import($id);" as Stmt,
-                name = ident,
-                turbopack_import: Expr = TURBOPACK_IMPORT.into(),
-                id: Expr = module_id_to_lit(&id),
-            ),
-        ))
+
+        let mut result = vec![];
+
+        let merged_index = scope_hoisting_context.get_module_index(module);
+        if let Some(merged_index) = merged_index {
+            // Insert a placeholder to inline the merged module at the right place
+            // relative to the other references (so to keep reference order).
+            result.push(CodeGenerationHoistedStmt::new(
+                format!("hoisted {merged_index}").into(),
+                quote!(
+                    "__turbopack_merged_esm__($id);" as Stmt,
+                    id: Expr = Lit::Num(merged_index.into()).into(),
+                ),
+            ));
+        }
+
+        if merged_index.is_some() && part == &ModulePart::Evaluation {
+            // No need to import, the module was already executed and is available in the same scope
+            // hoisting group (unless it's a namespace import)
+        } else {
+            let ident = referenced_asset
+                .get_ident(
+                    chunking_context,
+                    match part {
+                        ModulePart::Export(export)
+                        | ModulePart::RenamedExport {
+                            original_export: export,
+                            ..
+                        }
+                        | ModulePart::RenamedNamespace { export } => Some(export.clone()),
+                        ModulePart::Internal(_)
+                        | ModulePart::Locals
+                        | ModulePart::Exports
+                        | ModulePart::Facade
+                        | ModulePart::Evaluation => None,
+                    },
+                    scope_hoisting_context,
+                )
+                .await?
+                .context("part module reference should have an ident")?;
+
+            match ident {
+                ReferencedAssetIdent::LocalBinding { .. } => {
+                    // no need to import
+                }
+                ReferencedAssetIdent::Module { .. } => {
+                    let (sym, ctxt) = ident.into_module_namespace_ident().unwrap();
+                    let key = sym.as_str().into();
+                    let name = Ident::new(sym.into(), DUMMY_SP, ctxt.unwrap_or_default());
+
+                    let id = module.chunk_item_id(Vc::upcast(chunking_context)).await?;
+
+                    result.push(CodeGenerationHoistedStmt::new(
+                        key,
+                        quote!(
+                            "var $name = $turbopack_import($id);" as Stmt,
+                            name = name,
+                            turbopack_import: Expr = TURBOPACK_IMPORT.into(),
+                            id: Expr = module_id_to_lit(&id),
+                        ),
+                    ));
+                }
+            }
+        }
+
+        Ok(CodeGeneration::hoisted_stmts(result))
     }
 }

turbopack/crates/turbopack-tests/tests/execution/turbopack/exports/invalid-export/input/index.js

@@ -1,3 +1,10 @@
 it('should error when importing an invalid export', () => {
-  expect(require('./invalid-export').default).toBe(undefined)
+  // Either requiring should throw, or return undefined
+  let ns
+  try {
+    ns = require('./invalid-export')
+  } catch {
+    return
+  }
+  expect(ns.default).toBe(undefined)
 })