external node_modules for SSR

crates/next-core/src/next_server/context.rs

@@ -12,7 +12,9 @@ use turbopack_core::environment::{
 use turbopack_ecmascript::EcmascriptInputTransform;
 use turbopack_node::execution_context::ExecutionContextVc;
 
-use super::transforms::get_next_server_transforms_rules;
+use super::{
+    resolve::ExternalCjsModulesResolvePluginVc, transforms::get_next_server_transforms_rules,
+};
 use crate::{
     next_build::get_postcss_package_mapping,
     next_config::NextConfigVc,
@@ -36,25 +38,50 @@ pub async fn get_server_resolve_options_context(
     next_config: NextConfigVc,
 ) -> Result<ResolveOptionsContextVc> {
     let next_server_import_map = get_next_server_import_map(project_path, ty, next_config);
+    let foreign_code_context_condition = foreign_code_context_condition(next_config).await?;
 
     Ok(match ty.into_value() {
-        ServerContextType::Pages { .. }
-        | ServerContextType::PagesData { .. }
-        | ServerContextType::AppSSR { .. } => {
+        ServerContextType::Pages { .. } | ServerContextType::PagesData { .. } => {
+            let external_cjs_modules_plugin = ExternalCjsModulesResolvePluginVc::new(
+                project_path,
+                next_config.transpile_packages(),
+            );
+
             let resolve_options_context = ResolveOptionsContext {
                 enable_node_modules: true,
                 enable_node_externals: true,
                 enable_node_native_modules: true,
+                module: true,
                 custom_conditions: vec!["development".to_string()],
                 import_map: Some(next_server_import_map),
+                plugins: vec![external_cjs_modules_plugin.into()],
+                ..Default::default()
+            };
+            ResolveOptionsContext {
+                enable_typescript: true,
+                enable_react: true,
+                rules: vec![(
+                    foreign_code_context_condition,
+                    resolve_options_context.clone().cell(),
+                )],
+                ..resolve_options_context
+            }
+        }
+        ServerContextType::AppSSR { .. } => {
+            let resolve_options_context = ResolveOptionsContext {
+                enable_node_modules: true,
+                enable_node_externals: true,
+                enable_node_native_modules: true,
                 module: true,
+                custom_conditions: vec!["development".to_string()],
+                import_map: Some(next_server_import_map),
                 ..Default::default()
             };
             ResolveOptionsContext {
                 enable_typescript: true,
                 enable_react: true,
                 rules: vec![(
-                    foreign_code_context_condition(next_config).await?,
+                    foreign_code_context_condition,
                     resolve_options_context.clone().cell(),
                 )],
                 ..resolve_options_context
@@ -65,16 +92,16 @@ pub async fn get_server_resolve_options_context(
                 enable_node_modules: true,
                 enable_node_externals: true,
                 enable_node_native_modules: true,
+                module: true,
                 custom_conditions: vec!["development".to_string(), "react-server".to_string()],
                 import_map: Some(next_server_import_map),
-                module: true,
                 ..Default::default()
             };
             ResolveOptionsContext {
                 enable_typescript: true,
                 enable_react: true,
                 rules: vec![(
-                    foreign_code_context_condition(next_config).await?,
+                    foreign_code_context_condition,
                     resolve_options_context.clone().cell(),
                 )],
                 ..resolve_options_context
@@ -112,6 +139,12 @@ pub async fn get_server_module_options_context(
     next_config: NextConfigVc,
 ) -> Result<ModuleOptionsContextVc> {
     let custom_rules = get_next_server_transforms_rules(ty.into_value()).await?;
+    let foreign_code_context_condition = foreign_code_context_condition(next_config).await?;
+    let enable_postcss_transform = Some(PostCssTransformOptions {
+        postcss_package: Some(get_postcss_package_mapping(project_path)),
+        ..Default::default()
+    });
+    let enable_webpack_loaders = next_config.webpack_loaders_options().await?.clone_if();
 
     let module_options_context = match ty.into_value() {
         ServerContextType::Pages { .. } | ServerContextType::PagesData { .. } => {
@@ -122,14 +155,11 @@ pub async fn get_server_module_options_context(
             ModuleOptionsContext {
                 enable_jsx: true,
                 enable_styled_jsx: true,
-                enable_postcss_transform: Some(PostCssTransformOptions {
-                    postcss_package: Some(get_postcss_package_mapping(project_path)),
-                    ..Default::default()
-                }),
-                enable_webpack_loaders: next_config.webpack_loaders_options().await?.clone_if(),
+                enable_postcss_transform,
+                enable_webpack_loaders,
                 enable_typescript_transform: true,
                 rules: vec![(
-                    foreign_code_context_condition(next_config).await?,
+                    foreign_code_context_condition,
                     module_options_context.clone().cell(),
                 )],
                 custom_rules,
@@ -144,14 +174,11 @@ pub async fn get_server_module_options_context(
             ModuleOptionsContext {
                 enable_jsx: true,
                 enable_styled_jsx: true,
-                enable_postcss_transform: Some(PostCssTransformOptions {
-                    postcss_package: Some(get_postcss_package_mapping(project_path)),
-                    ..Default::default()
-                }),
-                enable_webpack_loaders: next_config.webpack_loaders_options().await?.clone_if(),
+                enable_postcss_transform,
+                enable_webpack_loaders,
                 enable_typescript_transform: true,
                 rules: vec![(
-                    foreign_code_context_condition(next_config).await?,
+                    foreign_code_context_condition,
                     module_options_context.clone().cell(),
                 )],
                 custom_rules,
@@ -168,14 +195,11 @@ pub async fn get_server_module_options_context(
             };
             ModuleOptionsContext {
                 enable_jsx: true,
-                enable_postcss_transform: Some(PostCssTransformOptions {
-                    postcss_package: Some(get_postcss_package_mapping(project_path)),
-                    ..Default::default()
-                }),
-                enable_webpack_loaders: next_config.webpack_loaders_options().await?.clone_if(),
+                enable_postcss_transform,
+                enable_webpack_loaders,
                 enable_typescript_transform: true,
                 rules: vec![(
-                    foreign_code_context_condition(next_config).await?,
+                    foreign_code_context_condition,
                     module_options_context.clone().cell(),
                 )],
                 custom_rules,

crates/next-core/src/next_server/mod.rs

@@ -1,2 +1,3 @@
 pub(crate) mod context;
+pub(crate) mod resolve;
 pub(crate) mod transforms;

crates/next-core/src/next_server/resolve.rs

@@ -0,0 +1,105 @@
+use anyhow::Result;
+use turbo_tasks::primitives::StringsVc;
+use turbo_tasks_fs::{glob::GlobVc, FileJsonContent, FileSystemPathVc};
+use turbopack_core::resolve::{
+    find_context_file, package_json,
+    parse::RequestVc,
+    plugin::{ResolvePlugin, ResolvePluginConditionVc, ResolvePluginVc},
+    FindContextFileResult, ResolveResult, ResolveResultOptionVc, SpecialType,
+};
+
+#[turbo_tasks::value]
+pub(crate) struct ExternalCjsModulesResolvePlugin {
+    root: FileSystemPathVc,
+    transpiled_packages: StringsVc,
+}
+
+#[turbo_tasks::value_impl]
+impl ExternalCjsModulesResolvePluginVc {
+    #[turbo_tasks::function]
+    pub fn new(root: FileSystemPathVc, transpiled_packages: StringsVc) -> Self {
+        ExternalCjsModulesResolvePlugin {
+            root,
+            transpiled_packages,
+        }
+        .cell()
+    }
+}
+
+#[turbo_tasks::value_impl]
+impl ResolvePlugin for ExternalCjsModulesResolvePlugin {
+    #[turbo_tasks::function]
+    fn condition(&self) -> ResolvePluginConditionVc {
+        ResolvePluginConditionVc::new(self.root, GlobVc::new("**/node_modules"))
+    }
+
+    #[turbo_tasks::function]
+    async fn after_resolve(
+        &self,
+        fs_path: FileSystemPathVc,
+        _request: RequestVc,
+    ) -> Result<ResolveResultOptionVc> {
+        let raw_fs_path = &*fs_path.await?;
+
+        // always bundle transpiled modules
+        let transpiled_glob = packages_glob(self.transpiled_packages).await?;
+        if transpiled_glob.execute(&raw_fs_path.path) {
+            return Ok(ResolveResultOptionVc::none());
+        }
+
+        // mjs -> esm module
+        if Some("mjs") == raw_fs_path.extension() {
+            return Ok(ResolveResultOptionVc::none());
+        }
+
+        let FindContextFileResult::Found(package_json, _) =
+            *find_context_file(fs_path.parent(), package_json()).await?
+        else {
+            // can't find package.json
+            return Ok(ResolveResultOptionVc::none());
+        };
+        let FileJsonContent::Content(package) = &*package_json.read_json().await? else {
+            // can't parse package.json
+            return Ok(ResolveResultOptionVc::none());
+        };
+
+        // always bundle esm modules
+        if let Some("module") = package["type"].as_str() {
+            return Ok(ResolveResultOptionVc::none());
+        }
+
+        // make sure we have a full package
+        let Some(package_name) = package["name"].as_str() else {
+            return Ok(ResolveResultOptionVc::none());
+        };
+
+        // check if we can resolve the package from the root dir (might be hidden by
+        // pnpm)
+        let FileJsonContent::Content(resolved_package) = &*self
+            .root
+            .join(&format!("node_modules/{}/package.json", package_name))
+            .read_json()
+            .await?
+        else {
+            return Ok(ResolveResultOptionVc::none());
+        };
+
+        // only mark external if the package.json files are the same
+        if resolved_package != package {
+            return Ok(ResolveResultOptionVc::none());
+        }
+
+        // mark as external
+        Ok(ResolveResultOptionVc::some(
+            ResolveResult::Special(SpecialType::OriginalReferenceExternal, Vec::new()).cell(),
+        ))
+    }
+}
+
+#[turbo_tasks::function]
+async fn packages_glob(packages: StringsVc) -> Result<GlobVc> {
+    Ok(GlobVc::new(&format!(
+        "**/node_modules/{{{}}}/**",
+        packages.await?.join(",")
+    )))
+}

crates/turbo-tasks/src/debug/internal.rs

@@ -60,7 +60,7 @@ impl<'a> std::fmt::Debug for FormattingStruct<'a> {
 }
 
 /// Debug implementation that prints an unquoted, unescaped string.
-pub(super) struct PassthroughDebug<'a>(Cow<'a, str>);
+pub struct PassthroughDebug<'a>(Cow<'a, str>);
 
 impl<'a> PassthroughDebug<'a> {
     pub fn new_str(s: &'a str) -> Self {

crates/turbopack-core/src/environment.rs

@@ -16,8 +16,8 @@ use crate::target::CompileTargetVc;
 
 static DEFAULT_NODEJS_VERSION: &str = "16.0.0";
 
-#[derive(Default)]
 #[turbo_tasks::value(shared)]
+#[derive(Default)]
 pub struct ServerAddr(#[turbo_tasks(trace_ignore)] Option<SocketAddr>);
 
 impl ServerAddr {
@@ -48,8 +48,8 @@ impl ServerAddrVc {
     }
 }
 
-#[derive(Default)]
 #[turbo_tasks::value]
+#[derive(Default)]
 pub enum Rendering {
     #[default]
     None,
@@ -63,8 +63,8 @@ impl Rendering {
     }
 }
 
-#[derive(Default)]
 #[turbo_tasks::value]
+#[derive(Default)]
 pub enum ChunkLoading {
     #[default]
     None,

crates/turbopack-core/src/resolve/alias_map.rs

@@ -11,7 +11,10 @@ use serde::{
     ser::SerializeMap,
     Deserialize, Deserializer, Serialize, Serializer,
 };
-use turbo_tasks::trace::{TraceRawVcs, TraceRawVcsContext};
+use turbo_tasks::{
+    debug::{internal::PassthroughDebug, ValueDebugFormat, ValueDebugFormatString},
+    trace::{TraceRawVcs, TraceRawVcsContext},
+};
 
 /// A map of [`AliasPattern`]s to the [`Template`]s they resolve to.
 ///
@@ -116,6 +119,50 @@ where
     }
 }
 
+impl<T> ValueDebugFormat for AliasMap<T>
+where
+    T: ValueDebugFormat,
+{
+    fn value_debug_format(&self, depth: usize) -> ValueDebugFormatString {
+        if depth == 0 {
+            return ValueDebugFormatString::Sync(std::any::type_name::<Self>().to_string());
+        }
+
+        let values = self
+            .map
+            .iter()
+            .flat_map(|(key, map)| {
+                let key = String::from_utf8(key).expect("invalid UTF-8 key in AliasMap");
+                map.iter().map(move |(alias_key, value)| match alias_key {
+                    AliasKey::Exact => (
+                        key.clone(),
+                        value.value_debug_format(depth.saturating_sub(1)),
+                    ),
+                    AliasKey::Wildcard { suffix } => (
+                        format!("{}*{}", key, suffix),
+                        value.value_debug_format(depth.saturating_sub(1)),
+                    ),
+                })
+            })
+            .collect::<Vec<_>>();
+
+        ValueDebugFormatString::Async(Box::pin(async move {
+            let mut values_string = std::collections::HashMap::new();
+            for (key, value) in values {
+                match value {
+                    ValueDebugFormatString::Sync(string) => {
+                        values_string.insert(key, PassthroughDebug::new_string(string));
+                    }
+                    ValueDebugFormatString::Async(future) => {
+                        values_string.insert(key, PassthroughDebug::new_string(future.await?));
+                    }
+                }
+            }
+            Ok(format!("{:#?}", values_string))
+        }))
+    }
+}
+
 impl<T> Debug for AliasMap<T>
 where
     T: Debug,