Auto merge of #118854 - estebank:compiler-metrics, r=<try>

[DO NOT MERGE] Introduce mechanism to write compiler metrics to disk

I want to verify the performance impact of enabling this new behavior against perf.

Cargo.lock

@@ -3853,8 +3853,10 @@ dependencies = [
 name = "rustc_feature"
 version = "0.0.0"
 dependencies = [
+ "directories",
  "rustc_data_structures",
  "rustc_span",
+ "time",
 ]
 
 [[package]]

compiler/rustc_feature/Cargo.toml

@@ -5,6 +5,8 @@ edition = "2021"
 
 [dependencies]
 # tidy-alphabetical-start
+directories = "5.0"
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_span = { path = "../rustc_span" }
+time = { version = "0.3", default-features = false, features = ["alloc", "formatting"] }
 # tidy-alphabetical-end

compiler/rustc_feature/src/lib.rs

@@ -26,8 +26,61 @@ mod unstable;
 #[cfg(test)]
 mod tests;
 
+use directories::ProjectDirs;
 use rustc_span::symbol::Symbol;
+use std::env::args_os;
+use std::fs::File;
+use std::io::{BufWriter, Write};
 use std::num::NonZeroU32;
+use std::path::PathBuf;
+use std::sync::OnceLock;
+use std::time::SystemTime;
+use time::OffsetDateTime;
+
+static PROJECT_DIRS: OnceLock<Option<ProjectDirs>> = OnceLock::new();
+static METRICS_PATH: OnceLock<Option<PathBuf>> = OnceLock::new();
+
+pub fn project_dirs() -> &'static Option<ProjectDirs> {
+    PROJECT_DIRS.get_or_init(|| ProjectDirs::from("org", "rust-lang", "rustc"))
+}
+
+pub fn metrics_path() -> &'static Option<PathBuf> {
+    METRICS_PATH.get_or_init(|| {
+        if !UnstableFeatures::from_environment(None).is_nightly_build() {
+            // Do not enable in stable.
+            return None;
+        }
+        // Use XDG_CACHE_DIR as it's the closest to correct for log files.
+        let _path = project_dirs().as_ref()?.cache_dir().to_path_buf();
+        // TEST: testing writing directly to the project's directory to avoid accidentally sneaking
+        // files without a clean-up strategy
+        let mut path: PathBuf = "/dev/null/".into();
+        let _ = std::fs::create_dir_all(&path).ok()?;
+        path.push("stats");
+
+        // Add some information about the built project: timestamp, rustc PID and crate name.
+        let now: OffsetDateTime = SystemTime::now().into();
+        let file_now = now
+            .format(
+                &time::format_description::parse("[year]-[month]-[day]T[hour]:[minute]:[second]")
+                    .unwrap(),
+            )
+            .unwrap_or_default();
+        let pid = std::process::id();
+        let file = File::options().create(true).append(true).open(&path).ok()?;
+        let mut file = BufWriter::new(file);
+        let mut args = args_os().skip(1);
+        args.find(|arg| arg == "--crate-name");
+        let crate_name = if let Some(name) = args.next() {
+            format!(" - {}", name.to_string_lossy())
+        } else {
+            String::new()
+        };
+        writeln!(file, "{file_now} - {pid}{crate_name}").ok()?;
+
+        Some(path)
+    })
+}
 
 #[derive(Debug, Clone)]
 pub struct Feature {

compiler/rustc_interface/src/passes.rs

@@ -59,9 +59,12 @@ pub fn parse<'a>(sess: &'a Session) -> PResult<'a, ast::Crate> {
         rustc_ast_passes::show_span::run(sess.diagnostic(), s, &krate);
     }
 
-    if sess.opts.unstable_opts.hir_stats {
-        hir_stats::print_ast_stats(&krate, "PRE EXPANSION AST STATS", "ast-stats-1");
-    }
+    hir_stats::ast_stats(
+        &krate,
+        "PRE EXPANSION AST STATS",
+        "ast-stats-1",
+        sess.opts.unstable_opts.hir_stats,
+    );
 
     Ok(krate)
 }
@@ -287,9 +290,12 @@ fn early_lint_checks(tcx: TyCtxt<'_>, (): ()) {
         eprintln!("Post-expansion node count: {}", count_nodes(krate));
     }
 
-    if sess.opts.unstable_opts.hir_stats {
-        hir_stats::print_ast_stats(krate, "POST EXPANSION AST STATS", "ast-stats-2");
-    }
+    hir_stats::ast_stats(
+        krate,
+        "POST EXPANSION AST STATS",
+        "ast-stats-2",
+        sess.opts.unstable_opts.hir_stats,
+    );
 
     // Needs to go *after* expansion to be able to check the results of macro expansion.
     sess.time("complete_gated_feature_checking", || {

compiler/rustc_passes/src/hir_id_validator.rs

@@ -8,9 +8,7 @@ use rustc_middle::hir::nested_filter;
 use rustc_middle::ty::TyCtxt;
 
 pub fn check_crate(tcx: TyCtxt<'_>) {
-    if tcx.sess.opts.unstable_opts.hir_stats {
-        crate::hir_stats::print_hir_stats(tcx);
-    }
+    crate::hir_stats::hir_stats(tcx, tcx.sess.opts.unstable_opts.hir_stats);
 
     #[cfg(debug_assertions)]
     {

compiler/rustc_passes/src/hir_stats.rs

@@ -6,6 +6,7 @@ use rustc_ast::visit as ast_visit;
 use rustc_ast::visit::BoundKind;
 use rustc_ast::{self as ast, AttrId, NodeId};
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
+use rustc_feature::metrics_path;
 use rustc_hir as hir;
 use rustc_hir::intravisit as hir_visit;
 use rustc_hir::HirId;
@@ -14,6 +15,7 @@ use rustc_middle::ty::TyCtxt;
 use rustc_middle::util::common::to_readable_str;
 use rustc_span::def_id::LocalDefId;
 use rustc_span::Span;
+use std::io::{BufWriter, Error as IOError, Write};
 
 #[derive(Copy, Clone, PartialEq, Eq, Hash)]
 enum Id {
@@ -67,24 +69,39 @@ struct StatCollector<'k> {
     seen: FxHashSet<Id>,
 }
 
-pub fn print_hir_stats(tcx: TyCtxt<'_>) {
+pub fn hir_stats(tcx: TyCtxt<'_>, write_to_stderr: bool) {
     let mut collector = StatCollector {
         krate: Some(tcx.hir()),
         nodes: FxHashMap::default(),
         seen: FxHashSet::default(),
     };
     tcx.hir().walk_toplevel_module(&mut collector);
     tcx.hir().walk_attributes(&mut collector);
-    collector.print("HIR STATS", "hir-stats");
+
+    if let Some(path) = metrics_path()
+        && false
+    {
+        collector.store(path, "hir-stats");
+    }
+    if write_to_stderr {
+        collector.print("HIR STATS", "hir-stats");
+    }
 }
 
-pub fn print_ast_stats(krate: &ast::Crate, title: &str, prefix: &str) {
+pub fn ast_stats(krate: &ast::Crate, title: &str, prefix: &str, write_to_stderr: bool) {
     use rustc_ast::visit::Visitor;
 
     let mut collector =
         StatCollector { krate: None, nodes: FxHashMap::default(), seen: FxHashSet::default() };
     collector.visit_crate(krate);
-    collector.print(title, prefix);
+    if let Some(path) = metrics_path()
+        && false
+    {
+        collector.store(path, prefix);
+    }
+    if write_to_stderr {
+        collector.print(title, prefix);
+    }
 }
 
 impl<'k> StatCollector<'k> {
@@ -120,52 +137,115 @@ impl<'k> StatCollector<'k> {
         }
     }
 
+    fn store(&self, path: &std::path::Path, title: &str) {
+        let Ok(file) = std::fs::File::options().create(true).append(true).open(&path) else {
+            // Keep quiet about not being able to write to the file, there's nothing we can do
+            // about it, and the automatic collection of stats is merely "best effort".
+            return;
+        };
+        let file = BufWriter::new(file);
+        let _ = self.write_json_to_output(file, title);
+    }
+
+    fn write_json_to_output(&self, mut output: impl Write, title: &str) -> Result<(), IOError> {
+        // We don't reuse the ASCII output for files because the size baloons quite quickly, and
+        // having a minimally parseable file from the start seems more reasonable.
+        writeln!(output, "{{ \"{title}\": {{")?;
+        let count = self.nodes.len();
+        for (i, (label, node)) in self.nodes.iter().enumerate() {
+            write!(
+                output,
+                "\"{}\": {{ \"count\": {}, \"size\": {}",
+                label,
+                to_readable_str(node.stats.count),
+                to_readable_str(node.stats.size)
+            )?;
+            if !node.subnodes.is_empty() {
+                writeln!(output, ", \"subnodes\": {{")?;
+                let count = node.subnodes.len();
+                for (i, (label, subnode)) in node.subnodes.iter().enumerate() {
+                    writeln!(
+                        output,
+                        "  \"{}\": {{ \"count\": {}, \"size\": {} }}{}",
+                        label,
+                        to_readable_str(subnode.count),
+                        to_readable_str(subnode.size),
+                        if i == count - 1 { "" } else { "," },
+                    )?;
+                }
+                writeln!(output, "}}")?;
+            }
+            writeln!(output, "}}{}", if i == count - 1 { "" } else { "," })?;
+        }
+        writeln!(output, "}} }}")
+    }
+
     fn print(&self, title: &str, prefix: &str) {
+        if let Err(err) = self.write_to_output(std::io::stderr(), title, prefix) {
+            eprintln!("couldn't print {title}: {err}")
+        }
+    }
+
+    fn write_to_output(
+        &self,
+        mut output: impl Write,
+        title: &str,
+        prefix: &str,
+    ) -> Result<(), IOError> {
         let mut nodes: Vec<_> = self.nodes.iter().collect();
         nodes.sort_by_key(|(_, node)| node.stats.count * node.stats.size);
 
         let total_size = nodes.iter().map(|(_, node)| node.stats.count * node.stats.size).sum();
 
-        eprintln!("{prefix} {title}");
-        eprintln!(
+        writeln!(output, "{prefix} {title}")?;
+        writeln!(
+            output,
             "{} {:<18}{:>18}{:>14}{:>14}",
             prefix, "Name", "Accumulated Size", "Count", "Item Size"
-        );
-        eprintln!("{prefix} ----------------------------------------------------------------");
+        )?;
+        writeln!(
+            output,
+            "{prefix} ----------------------------------------------------------------"
+        )?;
 
         let percent = |m, n| (m * 100) as f64 / n as f64;
 
         for (label, node) in nodes {
             let size = node.stats.count * node.stats.size;
-            eprintln!(
+            writeln!(
+                output,
                 "{} {:<18}{:>10} ({:4.1}%){:>14}{:>14}",
                 prefix,
                 label,
                 to_readable_str(size),
                 percent(size, total_size),
                 to_readable_str(node.stats.count),
                 to_readable_str(node.stats.size)
-            );
+            )?;
             if !node.subnodes.is_empty() {
                 let mut subnodes: Vec<_> = node.subnodes.iter().collect();
                 subnodes.sort_by_key(|(_, subnode)| subnode.count * subnode.size);
 
                 for (label, subnode) in subnodes {
                     let size = subnode.count * subnode.size;
-                    eprintln!(
+                    writeln!(
+                        output,
                         "{} - {:<18}{:>10} ({:4.1}%){:>14}",
                         prefix,
                         label,
                         to_readable_str(size),
                         percent(size, total_size),
                         to_readable_str(subnode.count),
-                    );
+                    )?;
                 }
             }
         }
-        eprintln!("{prefix} ----------------------------------------------------------------");
-        eprintln!("{} {:<18}{:>10}", prefix, "Total", to_readable_str(total_size));
-        eprintln!("{prefix}");
+        writeln!(
+            output,
+            "{prefix} ----------------------------------------------------------------"
+        )?;
+        writeln!(output, "{} {:<18}{:>10}", prefix, "Total", to_readable_str(total_size))?;
+        writeln!(output, "{prefix}")
     }
 }
 

src/tools/tidy/src/deps.rs

@@ -220,6 +220,8 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[
     "derive_more",
     "derive_setters",
     "digest",
+    "directories", // Already used by Miri
+    "dirs-sys",    // Dependency of `directories`
     "displaydoc",
     "dissimilar",
     "dlmalloc",
@@ -282,6 +284,7 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[
     "object",
     "odht",
     "once_cell",
+    "option-ext", // Dependency of `directories`
     "overload",
     "parking_lot",
     "parking_lot_core",
@@ -305,6 +308,7 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[
     "rand_xorshift",
     "rand_xoshiro",
     "redox_syscall",
+    "redox_users", // Dependency of `directories`
     "regex",
     "regex-automata",
     "regex-syntax",