Refactor ContentSources to RouteTree for more efficient routing (#5360)

### Description

Routing is inefficient with many pages

next.js PR: vercel/next.js#51660

crates/turbopack-cli-utils/Cargo.toml

@@ -16,12 +16,12 @@ anyhow = { workspace = true }
 clap = { workspace = true, features = ["derive"] }
 crossbeam-channel = { workspace = true }
 crossterm = "0.26.0"
-ctrlc = "3.2.5"
 once_cell = { workspace = true }
 owo-colors = { workspace = true }
 postcard = { workspace = true, features = ["alloc", "use-std"] }
 serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true }
+tokio = { workspace = true, features = ["signal"] }
 tracing = { workspace = true }
 tracing-subscriber = { workspace = true }
 turbo-tasks = { workspace = true }

crates/turbopack-cli-utils/src/exit.rs

@@ -1,6 +1,6 @@
 use std::sync::{Arc, Mutex};
 
-use anyhow::{Context, Result};
+use anyhow::Result;
 
 /// A guard for the exit handler. When dropped, the exit guard will be dropped.
 /// It might also be dropped on Ctrl-C.
@@ -18,11 +18,11 @@ impl<T: Send + 'static> ExitGuard<T> {
         let guard = Arc::new(Mutex::new(Some(guard)));
         {
             let guard = guard.clone();
-            ctrlc::set_handler(move || {
+            tokio::spawn(async move {
+                tokio::signal::ctrl_c().await.unwrap();
                 drop(guard.lock().unwrap().take());
                 std::process::exit(0);
-            })
-            .context("Unable to set ctrl-c handler")?;
+            });
         }
         Ok(ExitGuard(guard))
     }

crates/turbopack-cli/src/dev/mod.rs

@@ -13,6 +13,7 @@ use anyhow::{Context, Result};
 use dunce::canonicalize;
 use owo_colors::OwoColorize;
 use turbo_tasks::{
+    primitives::StringVc,
     util::{FormatBytes, FormatDuration},
     StatsType, TransientInstance, TurboTasks, TurboTasksBackendApi, UpdateInfo, Value,
 };
@@ -31,7 +32,7 @@ use turbopack_dev::DevChunkingContextVc;
 use turbopack_dev_server::{
     introspect::IntrospectionSource,
     source::{
-        combined::CombinedContentSourceVc, router::RouterContentSource,
+        combined::CombinedContentSourceVc, router::PrefixedRouterContentSourceVc,
         source_maps::SourceMapContentSourceVc, static_assets::StaticAssetsContentSourceVc,
         ContentSourceVc,
     },
@@ -312,15 +313,15 @@ async fn source(
     .into();
     let main_source = main_source.into();
     let source_maps = SourceMapContentSourceVc::new(main_source).into();
-    let source = RouterContentSource {
-        routes: vec![
-            ("__turbopack__/".to_string(), introspect),
-            ("__turbo_tasks__/".to_string(), viz),
-            ("__turbopack_sourcemap__/".to_string(), source_maps),
+    let source = PrefixedRouterContentSourceVc::new(
+        StringVc::empty(),
+        vec![
+            ("__turbopack__".to_string(), introspect),
+            ("__turbo_tasks__".to_string(), viz),
+            ("__turbopack_sourcemap__".to_string(), source_maps),
         ],
-        fallback: main_source,
-    }
-    .cell()
+        main_source,
+    )
     .into();
 
     Ok(source)

crates/turbopack-cli/src/dev/turbo_tasks_viz.rs

@@ -1,6 +1,6 @@
 use std::{sync::Arc, time::Duration};
 
-use anyhow::Result;
+use anyhow::{bail, Result};
 use mime::TEXT_HTML_UTF_8;
 use turbo_tasks::{get_invalidator, TurboTasks, TurboTasksBackendApi, Value};
 use turbo_tasks_fs::File;
@@ -10,8 +10,10 @@ use turbo_tasks_memory::{
 };
 use turbopack_core::asset::AssetContentVc;
 use turbopack_dev_server::source::{
+    route_tree::{BaseSegment, RouteTreeVc, RouteTreesVc, RouteType},
     ContentSource, ContentSourceContentVc, ContentSourceData, ContentSourceDataFilter,
-    ContentSourceDataVary, ContentSourceResultVc, ContentSourceVc, NeededData,
+    ContentSourceDataVary, ContentSourceDataVaryVc, ContentSourceVc, GetContentSourceContent,
+    GetContentSourceContentVc,
 };
 
 #[turbo_tasks::value(serialization = "none", eq = "manual", cell = "new", into = "new")]
@@ -30,12 +32,51 @@ const INVALIDATION_INTERVAL: Duration = Duration::from_secs(3);
 
 #[turbo_tasks::value_impl]
 impl ContentSource for TurboTasksSource {
+    #[turbo_tasks::function]
+    fn get_routes(self_vc: TurboTasksSourceVc) -> RouteTreeVc {
+        RouteTreesVc::cell(vec![
+            RouteTreeVc::new_route(
+                vec![BaseSegment::Static("graph".to_string())],
+                RouteType::Exact,
+                self_vc.into(),
+            ),
+            RouteTreeVc::new_route(
+                vec![BaseSegment::Static("call-graph".to_string())],
+                RouteType::Exact,
+                self_vc.into(),
+            ),
+            RouteTreeVc::new_route(
+                vec![BaseSegment::Static("table".to_string())],
+                RouteType::Exact,
+                self_vc.into(),
+            ),
+            RouteTreeVc::new_route(
+                vec![BaseSegment::Static("reset".to_string())],
+                RouteType::Exact,
+                self_vc.into(),
+            ),
+        ])
+        .merge()
+    }
+}
+
+#[turbo_tasks::value_impl]
+impl GetContentSourceContent for TurboTasksSource {
+    #[turbo_tasks::function]
+    fn vary(&self) -> ContentSourceDataVaryVc {
+        ContentSourceDataVary {
+            query: Some(ContentSourceDataFilter::All),
+            ..Default::default()
+        }
+        .cell()
+    }
+
     #[turbo_tasks::function]
     async fn get(
         self_vc: TurboTasksSourceVc,
         path: &str,
         data: Value<ContentSourceData>,
-    ) -> Result<ContentSourceResultVc> {
+    ) -> Result<ContentSourceContentVc> {
         let this = self_vc.await?;
         let tt = &this.turbo_tasks;
         let invalidator = get_invalidator();
@@ -72,29 +113,21 @@ impl ContentSource for TurboTasksSource {
                 viz::graph::wrap_html(&graph)
             }
             "table" => {
-                if let Some(query) = &data.query {
-                    let mut stats = Stats::new();
-                    let b = tt.backend();
-                    let active_only = query.contains_key("active");
-                    let include_unloaded = query.contains_key("unloaded");
-                    b.with_all_cached_tasks(|task| {
-                        stats.add_id_conditional(b, task, |_, info| {
-                            (include_unloaded || !info.unloaded) && (!active_only || info.active)
-                        });
+                let Some(query) = &data.query else {
+                    bail!("Missing query");
+                };
+                let mut stats = Stats::new();
+                let b = tt.backend();
+                let active_only = query.contains_key("active");
+                let include_unloaded = query.contains_key("unloaded");
+                b.with_all_cached_tasks(|task| {
+                    stats.add_id_conditional(b, task, |_, info| {
+                        (include_unloaded || !info.unloaded) && (!active_only || info.active)
                     });
-                    let tree = stats.treeify(ReferenceType::Dependency);
-                    let table = viz::table::create_table(tree, tt.stats_type());
-                    viz::table::wrap_html(&table)
-                } else {
-                    return Ok(ContentSourceResultVc::need_data(Value::new(NeededData {
-                        source: self_vc.into(),
-                        path: path.to_string(),
-                        vary: ContentSourceDataVary {
-                            query: Some(ContentSourceDataFilter::All),
-                            ..Default::default()
-                        },
-                    })));
-                }
+                });
+                let tree = stats.treeify(ReferenceType::Dependency);
+                let table = viz::table::create_table(tree, tt.stats_type());
+                viz::table::wrap_html(&table)
             }
             "reset" => {
                 let b = tt.backend();
@@ -103,13 +136,10 @@ impl ContentSource for TurboTasksSource {
                 });
                 "Done".to_string()
             }
-            _ => return Ok(ContentSourceResultVc::not_found()),
+            _ => bail!("Unknown path: {}", path),
         };
-        Ok(ContentSourceResultVc::exact(
-            ContentSourceContentVc::static_content(
-                AssetContentVc::from(File::from(html).with_content_type(TEXT_HTML_UTF_8)).into(),
-            )
-            .into(),
+        Ok(ContentSourceContentVc::static_content(
+            AssetContentVc::from(File::from(html).with_content_type(TEXT_HTML_UTF_8)).into(),
         ))
     }
 }

crates/turbopack-convert-trace/Cargo.toml

@@ -17,6 +17,7 @@ clap = { workspace = true, features = ["derive", "env"] }
 futures = { workspace = true }
 indexmap = { workspace = true }
 intervaltree = "0.2.7"
+itertools = { workspace = true }
 owo-colors = { workspace = true }
 postcard = { workspace = true }
 serde = { workspace = true }