Auto merge of rust-lang#16356 - Veykril:sysroot-metadata, r=Veykril

internal: Add unstable config for loading the sysroot sources via `cargo metadata`

cc rust-lang/rust-analyzer#7637

This takes the approach of having `cargo metadata` generate a lock file that we then delete again, hence why this is behind a flag. If people need this for their workflow they can just enable it, if not, they are probably better off keeping it disabled. [example](https://dreampuf.github.io/GraphvizOnline/#digraph%20rust_analyzer_crate_graph%20%7B%0A%20%20%20%20_0%5Blabel%3D%22core%22%5D%5Bshape%3D%22box%22%5D%3B%0A%20%20%20%20_17%5Blabel%3D%22ra_playground%22%5D%5Bshape%3D%22box%22%5D%3B%0A%20%20%20%20_14%5Blabel%3D%22getopts%22%5D%5Bshape%3D%22box%22%5D%3B%0A%20%20%20%20_11%5Blabel%3D%22std_detect%22%5D%5Bshape%3D%22box%22%5D%3B%0A%20%20%20%20_8%5Blabel%3D%22unwind%22%5D%5Bshape%3D%22box%22%5D%3B%0A%20%20%20%20_5%5Blabel%3D%22hashbrown%22%5D%5Bshape%3D%22box%22%5D%3B%0A%20%20%20%20_2%5Blabel%3D%22alloc%22%5D%5Bshape%3D%22box%22%5D%3B%0A%20%20%20%20_16%5Blabel%3D%22test%22%5D%5Bshape%3D%22box%22%5D%3B%0A%20%20%20%20_13%5Blabel%3D%22unicode_width%22%5D%5Bshape%3D%22box%22%5D%3B%0A%20%20%20%20_10%5Blabel%3D%22rustc_demangle%22%5D%5Bshape%3D%22box%22%5D%3B%0A%20%20%20%20_7%5Blabel%3D%22panic_abort%22%5D%5Bshape%3D%22box%22%5D%3B%0A%20%20%20%20_4%5Blabel%3D%22cfg_if%22%5D%5Bshape%3D%22box%22%5D%3B%0A%20%20%20%20_1%5Blabel%3D%22compiler_builtins%22%5D%5Bshape%3D%22box%22%5D%3B%0A%20%20%20%20_18%5Blabel%3D%22build_script_build%22%5D%5Bshape%3D%22box%22%5D%3B%0A%20%20%20%20_15%5Blabel%3D%22proc_macro%22%5D%5Bshape%3D%22box%22%5D%3B%0A%20%20%20%20_12%5Blabel%3D%22std%22%5D%5Bshape%3D%22box%22%5D%3B%0A%20%20%20%20_9%5Blabel%3D%22panic_unwind%22%5D%5Bshape%3D%22box%22%5D%3B%0A%20%20%20%20_6%5Blabel%3D%22libc%22%5D%5Bshape%3D%22box%22%5D%3B%0A%20%20%20%20_3%5Blabel%3D%22allocator_api2%22%5D%5Bshape%3D%22box%22%5D%3B%0A%20%20%20%20_17%20-%3E%20_0%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_17%20-%3E%20_2%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_17%20-%3E%20_12%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_17%20-%3E%20_15%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_17%20-%3E%20_16%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_14%20-%3E%20_0%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_14%20-%3E%20_12%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_14%20-%3E%20_13%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_11%20-%3E%20_0%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_11%20-%3E%20_1%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_11%20-%3E%20_2%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_11%20-%3E%20_4%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_11%20-%3E%20_6%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_8%20-%3E%20_0%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_8%20-%3E%20_1%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_8%20-%3E%20_4%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_8%20-%3E%20_6%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_5%20-%3E%20_0%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_5%20-%3E%20_1%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_5%20-%3E%20_2%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_5%20-%3E%20_3%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_2%20-%3E%20_0%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_2%20-%3E%20_1%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_16%20-%3E%20_0%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_16%20-%3E%20_7%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_16%20-%3E%20_9%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_16%20-%3E%20_12%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_16%20-%3E%20_14%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_13%20-%3E%20_0%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_13%20-%3E%20_1%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_13%20-%3E%20_12%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_10%20-%3E%20_0%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_10%20-%3E%20_1%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_7%20-%3E%20_0%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_7%20-%3E%20_1%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_7%20-%3E%20_2%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_7%20-%3E%20_4%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_7%20-%3E%20_6%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_1%20-%3E%20_0%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_18%20-%3E%20_0%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_18%20-%3E%20_2%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_18%20-%3E%20_12%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_18%20-%3E%20_15%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_18%20-%3E%20_16%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_15%20-%3E%20_0%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_15%20-%3E%20_12%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_12%20-%3E%20_0%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_12%20-%3E%20_1%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_12%20-%3E%20_2%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_12%20-%3E%20_4%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_12%20-%3E%20_4%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_12%20-%3E%20_5%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_12%20-%3E%20_6%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_12%20-%3E%20_7%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_12%20-%3E%20_8%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_12%20-%3E%20_9%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_12%20-%3E%20_10%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_12%20-%3E%20_11%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_9%20-%3E%20_0%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_9%20-%3E%20_1%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_9%20-%3E%20_2%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_9%20-%3E%20_4%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_9%20-%3E%20_6%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_9%20-%3E%20_8%5Blabel%3D%22%22%5D%3B%0A%20%20%20%20_6%20-%3E%20_0%5Blabel%3D%22%22%5D%3B%0A%7D%0A)

![image](https://github.com/rust-lang/rust-analyzer/assets/3757771/7709bb38-d948-4106-82c2-9b76677620bd)
hashbrown resolves as a dependency now

crates/base-db/src/input.rs

@@ -9,7 +9,7 @@
 use std::{fmt, mem, ops, str::FromStr};
 
 use cfg::CfgOptions;
-use la_arena::{Arena, Idx};
+use la_arena::{Arena, Idx, RawIdx};
 use rustc_hash::{FxHashMap, FxHashSet};
 use semver::Version;
 use syntax::SmolStr;
@@ -157,6 +157,10 @@ impl CrateOrigin {
     pub fn is_lib(&self) -> bool {
         matches!(self, CrateOrigin::Library { .. })
     }
+
+    pub fn is_lang(&self) -> bool {
+        matches!(self, CrateOrigin::Lang { .. })
+    }
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@@ -174,7 +178,7 @@ impl From<&str> for LangCrateOrigin {
         match s {
             "alloc" => LangCrateOrigin::Alloc,
             "core" => LangCrateOrigin::Core,
-            "proc-macro" => LangCrateOrigin::ProcMacro,
+            "proc-macro" | "proc_macro" => LangCrateOrigin::ProcMacro,
             "std" => LangCrateOrigin::Std,
             "test" => LangCrateOrigin::Test,
             _ => LangCrateOrigin::Other,
@@ -522,7 +526,7 @@ impl CrateGraph {
         self.arena.iter().map(|(idx, _)| idx)
     }
 
-    // FIXME: used for `handle_hack_cargo_workspace`, should be removed later
+    // FIXME: used for fixing up the toolchain sysroot, should be removed and done differently
     #[doc(hidden)]
     pub fn iter_mut(&mut self) -> impl Iterator<Item = (CrateId, &mut CrateData)> + '_ {
         self.arena.iter_mut()
@@ -619,7 +623,12 @@ impl CrateGraph {
     /// This will deduplicate the crates of the graph where possible.
     /// Note that for deduplication to fully work, `self`'s crate dependencies must be sorted by crate id.
     /// If the crate dependencies were sorted, the resulting graph from this `extend` call will also have the crate dependencies sorted.
-    pub fn extend(&mut self, mut other: CrateGraph, proc_macros: &mut ProcMacroPaths) {
+    pub fn extend(
+        &mut self,
+        mut other: CrateGraph,
+        proc_macros: &mut ProcMacroPaths,
+        on_finished: impl FnOnce(&FxHashMap<CrateId, CrateId>),
+    ) {
         let topo = other.crates_in_topological_order();
         let mut id_map: FxHashMap<CrateId, CrateId> = FxHashMap::default();
         for topo in topo {
@@ -670,6 +679,8 @@ impl CrateGraph {
 
         *proc_macros =
             mem::take(proc_macros).into_iter().map(|(id, macros)| (id_map[&id], macros)).collect();
+
+        on_finished(&id_map);
     }
 
     fn find_path(
@@ -721,6 +732,29 @@ impl CrateGraph {
     fn hacky_find_crate<'a>(&'a self, display_name: &'a str) -> impl Iterator<Item = CrateId> + 'a {
         self.iter().filter(move |it| self[*it].display_name.as_deref() == Some(display_name))
     }
+
+    /// Removes all crates from this crate graph except for the ones in `to_keep` and fixes up the dependencies.
+    /// Returns a mapping from old crate ids to new crate ids.
+    pub fn remove_crates_except(&mut self, to_keep: &[CrateId]) -> Vec<Option<CrateId>> {
+        let mut id_map = vec![None; self.arena.len()];
+        self.arena = std::mem::take(&mut self.arena)
+            .into_iter()
+            .filter_map(|(id, data)| if to_keep.contains(&id) { Some((id, data)) } else { None })
+            .enumerate()
+            .map(|(new_id, (id, data))| {
+                id_map[id.into_raw().into_u32() as usize] =
+                    Some(CrateId::from_raw(RawIdx::from_u32(new_id as u32)));
+                data
+            })
+            .collect();
+        for (_, data) in self.arena.iter_mut() {
+            data.dependencies.iter_mut().for_each(|dep| {
+                dep.crate_id =
+                    id_map[dep.crate_id.into_raw().into_u32() as usize].expect("crate was filtered")
+            });
+        }
+        id_map
+    }
 }
 
 impl ops::Index<CrateId> for CrateGraph {

crates/project-model/src/cargo_workspace.rs

@@ -82,6 +82,8 @@ pub struct CargoConfig {
     pub target: Option<String>,
     /// Sysroot loading behavior
     pub sysroot: Option<RustLibSource>,
+    /// Whether to invoke `cargo metadata` on the sysroot crate.
+    pub sysroot_query_metadata: bool,
     pub sysroot_src: Option<AbsPathBuf>,
     /// rustc private crate source
     pub rustc_source: Option<RustLibSource>,