diff --git a/Cargo.lock b/Cargo.lock index 6b5499ec6f351..a1a6840ce28e1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3021,6 +3021,7 @@ name = "rustc_data_structures" version = "0.0.0" dependencies = [ "cfg-if 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", "ena 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", "graphviz 0.0.0", "indexmap 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/src/librustc/arena.rs b/src/librustc/arena.rs index e8c3914e695ad..d5fd3bbba1a5a 100644 --- a/src/librustc/arena.rs +++ b/src/librustc/arena.rs @@ -31,6 +31,7 @@ macro_rules! arena_types { rustc::hir::def_id::DefId, rustc::ty::subst::SubstsRef<$tcx> )>, + [few] hir_map: rustc::hir::map::Map<$tcx>, [few, decode] mir_keys: rustc::util::nodemap::DefIdSet, [decode] specialization_graph: rustc::traits::specialization_graph::Graph, [] region_scope_tree: rustc::middle::region::ScopeTree, diff --git a/src/librustc/dep_graph/graph.rs b/src/librustc/dep_graph/graph.rs index b8c6c1e372382..236af2f578924 100644 --- a/src/librustc/dep_graph/graph.rs +++ b/src/librustc/dep_graph/graph.rs @@ -674,8 +674,6 @@ impl DepGraph { } } else { match dep_dep_node.kind { - DepKind::Hir | - DepKind::HirBody | DepKind::CrateMetadata => { if dep_dep_node.extract_def_id(tcx).is_none() { // If the node does not exist anymore, we @@ -719,7 +717,7 @@ impl DepGraph { None => { if !tcx.sess.has_errors() { bug!("try_mark_previous_green() - Forcing the DepNode \ - should have set its color") + should have set its color - dep node {:?}", dep_dep_node) } else { // If the query we just forced has resulted // in some kind of compilation error, we diff --git a/src/librustc/hir/map/mod.rs b/src/librustc/hir/map/mod.rs index 63f60d0ab9528..4673c5ba3a24e 100644 --- a/src/librustc/hir/map/mod.rs +++ b/src/librustc/hir/map/mod.rs @@ -8,8 +8,6 @@ use crate::dep_graph::{DepGraph, DepNode, DepKind, DepNodeIndex}; use crate::hir::def_id::{CRATE_DEF_INDEX, DefId, LocalDefId}; -use crate::middle::cstore::CrateStoreDyn; - use rustc_target::spec::abi::Abi; use rustc_data_structures::svh::Svh; use rustc_data_structures::indexed_vec::IndexVec; @@ -24,9 +22,11 @@ use crate::hir::itemlikevisit::ItemLikeVisitor; use crate::hir::print::Nested; use crate::util::nodemap::FxHashMap; use crate::util::common::time; +use crate::ich::StableHashingContext; use std::result::Result::Err; use crate::ty::query::Providers; +use crate::ty::TyCtxt; pub mod blocks; mod collector; @@ -1135,45 +1135,48 @@ impl Named for StructField { fn name(&self) -> Name { self.ident.name } } impl Named for TraitItem { fn name(&self) -> Name { self.ident.name } } impl Named for ImplItem { fn name(&self) -> Name { self.ident.name } } -pub fn map_crate<'hir>(sess: &crate::session::Session, - cstore: &CrateStoreDyn, - forest: &'hir Forest, - definitions: &'hir Definitions) - -> Map<'hir> { +pub fn map_crate(tcx: TyCtxt<'_>) -> Map<'_> { // Build the reverse mapping of `node_to_hir_id`. - let hir_to_node_id = definitions.node_to_hir_id.iter_enumerated() + let hir_to_node_id = tcx.hir_defs.node_to_hir_id.iter_enumerated() .map(|(node_id, &hir_id)| (hir_id, node_id)).collect(); let (map, crate_hash) = { - let hcx = crate::ich::StableHashingContext::new(sess, &forest.krate, definitions, cstore); - - let mut collector = NodeCollector::root(sess, - &forest.krate, - &forest.dep_graph, - &definitions, - &hir_to_node_id, - hcx); - intravisit::walk_crate(&mut collector, &forest.krate); - - let crate_disambiguator = sess.local_crate_disambiguator(); - let cmdline_args = sess.opts.dep_tracking_hash(); + let krate = tcx.hir_forest.untracked_krate(); + let hcx = StableHashingContext::new( + tcx.sess, + krate, + &tcx.hir_defs, + tcx.cstore + ); + let mut collector = NodeCollector::root( + tcx.sess, + krate, + &tcx.dep_graph, + &tcx.hir_defs, + &hir_to_node_id, + hcx + ); + intravisit::walk_crate(&mut collector, krate); + + let crate_disambiguator = tcx.sess.local_crate_disambiguator(); + let cmdline_args = tcx.sess.opts.dep_tracking_hash(); collector.finalize_and_compute_crate_hash( crate_disambiguator, - cstore, + tcx.cstore, cmdline_args ) }; let map = Map { - forest, - dep_graph: forest.dep_graph.clone(), + forest: &tcx.hir_forest, + dep_graph: tcx.dep_graph.clone(), crate_hash, map, hir_to_node_id, - definitions, + definitions: &tcx.hir_defs, }; - time(sess, "validate hir map", || { + time(tcx.sess, "validate hir map", || { hir_id_validator::check_crate(&map); }); diff --git a/src/librustc/query/mod.rs b/src/librustc/query/mod.rs index 442a90ab3f845..2a50cd6f98de9 100644 --- a/src/librustc/query/mod.rs +++ b/src/librustc/query/mod.rs @@ -31,6 +31,12 @@ use syntax_pos::symbol::InternedString; // as they will raise an fatal error on query cycles instead. rustc_queries! { Other { + query hir_map(_: CrateNum) -> &'tcx hir::map::Map<'tcx> { + no_hash + eval_always + desc { "indexing HIR" } + } + /// Records the type of every item. query type_of(key: DefId) -> Ty<'tcx> { cache_on_disk_if { key.is_local() } diff --git a/src/librustc/ty/context.rs b/src/librustc/ty/context.rs index 16fc46b66d9f4..b2259ac27ffd9 100644 --- a/src/librustc/ty/context.rs +++ b/src/librustc/ty/context.rs @@ -53,7 +53,7 @@ use rustc_data_structures::stable_hasher::{HashStable, hash_stable_hashmap, StableVec}; use arena::SyncDroplessArena; use rustc_data_structures::indexed_vec::{Idx, IndexVec}; -use rustc_data_structures::sync::{Lrc, Lock, WorkerLocal}; +use rustc_data_structures::sync::{Lrc, Lock, WorkerLocal, AtomicOnce}; use std::any::Any; use std::borrow::Borrow; use std::cmp::Ordering; @@ -990,7 +990,7 @@ pub struct GlobalCtxt<'tcx> { interners: CtxtInterners<'tcx>, - cstore: &'tcx CrateStoreDyn, + pub(crate) cstore: &'tcx CrateStoreDyn, pub sess: &'tcx Session, @@ -1017,7 +1017,11 @@ pub struct GlobalCtxt<'tcx> { /// Export map produced by name resolution. export_map: FxHashMap>>, - hir_map: hir_map::Map<'tcx>, + pub hir_forest: hir::map::Forest, + + pub hir_defs: hir::map::Definitions, + + hir_map: AtomicOnce<&'tcx hir_map::Map<'tcx>>, /// A map from DefPathHash -> DefId. Includes DefIds from the local crate /// as well as all upstream crates. Only populated in incremental mode. @@ -1084,7 +1088,10 @@ impl<'tcx> TyCtxt<'tcx> { #[inline(always)] pub fn hir(self) -> &'tcx hir_map::Map<'tcx> { - &self.hir_map + self.hir_map.get_or_init(|| { + // We can use `with_ignore` here because the hir map does its own tracking + self.dep_graph.with_ignore(|| self.hir_map(LOCAL_CRATE)) + }) } pub fn alloc_steal_mir(self, mir: Body<'tcx>) -> &'tcx Steal> { @@ -1169,7 +1176,8 @@ impl<'tcx> TyCtxt<'tcx> { extern_providers: ty::query::Providers<'tcx>, arenas: &'tcx AllArenas, resolutions: ty::Resolutions, - hir: hir_map::Map<'tcx>, + hir_forest: hir::map::Forest, + hir_defs: hir::map::Definitions, on_disk_query_result_cache: query::OnDiskCache<'tcx>, crate_name: &str, tx: mpsc::Sender>, @@ -1188,7 +1196,7 @@ impl<'tcx> TyCtxt<'tcx> { let common_types = CommonTypes::new(&interners); let common_lifetimes = CommonLifetimes::new(&interners); let common_consts = CommonConsts::new(&interners, &common_types); - let dep_graph = hir.dep_graph.clone(); + let dep_graph = hir_forest.dep_graph.clone(); let max_cnum = cstore.crates_untracked().iter().map(|c| c.as_usize()).max().unwrap_or(0); let mut providers = IndexVec::from_elem_n(extern_providers, max_cnum + 1); providers[LOCAL_CRATE] = local_providers; @@ -1204,7 +1212,7 @@ impl<'tcx> TyCtxt<'tcx> { upstream_def_path_tables .iter() .map(|&(cnum, ref rc)| (cnum, &**rc)) - .chain(iter::once((LOCAL_CRATE, hir.definitions().def_path_table()))) + .chain(iter::once((LOCAL_CRATE, hir_defs.def_path_table()))) }; // Precompute the capacity of the hashmap so we don't have to @@ -1227,7 +1235,7 @@ impl<'tcx> TyCtxt<'tcx> { let mut trait_map: FxHashMap<_, FxHashMap<_, _>> = FxHashMap::default(); for (k, v) in resolutions.trait_map { - let hir_id = hir.node_to_hir_id(k); + let hir_id = hir_defs.node_to_hir_id(k); let map = trait_map.entry(hir_id.owner).or_default(); map.insert(hir_id.local_id, StableVec::new(v)); } @@ -1245,25 +1253,27 @@ impl<'tcx> TyCtxt<'tcx> { trait_map, export_map: resolutions.export_map.into_iter().map(|(k, v)| { let exports: Vec<_> = v.into_iter().map(|e| { - e.map_id(|id| hir.node_to_hir_id(id)) + e.map_id(|id| hir_defs.node_to_hir_id(id)) }).collect(); (k, exports) }).collect(), maybe_unused_trait_imports: resolutions.maybe_unused_trait_imports .into_iter() - .map(|id| hir.local_def_id_from_node_id(id)) + .map(|id| hir_defs.local_def_id(id)) .collect(), maybe_unused_extern_crates: resolutions.maybe_unused_extern_crates .into_iter() - .map(|(id, sp)| (hir.local_def_id_from_node_id(id), sp)) + .map(|(id, sp)| (hir_defs.local_def_id(id), sp)) .collect(), glob_map: resolutions.glob_map.into_iter().map(|(id, names)| { - (hir.local_def_id_from_node_id(id), names) + (hir_defs.local_def_id(id), names) }).collect(), extern_prelude: resolutions.extern_prelude, - hir_map: hir, + hir_forest, + hir_defs, + hir_map: AtomicOnce::new(), def_path_hash_to_def_id, queries: query::Queries::new( providers, @@ -1377,7 +1387,9 @@ impl<'tcx> TyCtxt<'tcx> { #[inline] pub fn def_path_hash(self, def_id: DefId) -> hir_map::DefPathHash { if def_id.is_local() { - self.hir().definitions().def_path_hash(def_id.index) + // This is used when creating dep nodes, which happens when executing queries, + // so we can't use hir() here + self.hir_defs.def_path_hash(def_id.index) } else { self.cstore.def_path_hash(def_id) } @@ -1416,12 +1428,13 @@ impl<'tcx> TyCtxt<'tcx> { #[inline(always)] pub fn create_stable_hashing_context(self) -> StableHashingContext<'tcx> { - let krate = self.gcx.hir_map.forest.untracked_krate(); - - StableHashingContext::new(self.sess, - krate, - self.hir().definitions(), - self.cstore) + // This is used when executing queries. Also used when dealing with query cycles + StableHashingContext::new( + self.sess, + self.hir_forest.untracked_krate(), + &self.hir_defs, + self.cstore + ) } // This method makes sure that we have a DepNode and a Fingerprint for diff --git a/src/librustc/ty/query/plumbing.rs b/src/librustc/ty/query/plumbing.rs index 0c9e31e1ff28e..f2b29e382508f 100644 --- a/src/librustc/ty/query/plumbing.rs +++ b/src/librustc/ty/query/plumbing.rs @@ -2,12 +2,13 @@ //! generate the actual methods on tcx which find and execute the provider, //! manage the caches, and so forth. -use crate::dep_graph::{DepNodeIndex, DepNode, DepKind, SerializedDepNodeIndex}; +use crate::dep_graph::{DepNodeIndex, DepNode, DepConstructor, DepKind, SerializedDepNodeIndex}; use crate::ty::tls; use crate::ty::{self, TyCtxt}; use crate::ty::query::Query; use crate::ty::query::config::{QueryConfig, QueryDescription}; use crate::ty::query::job::{QueryJob, QueryResult, QueryInfo}; +use crate::hir::def_id::LOCAL_CRATE; use crate::util::common::{profq_msg, ProfileQueriesMsg, QueryMsg}; @@ -1220,14 +1221,28 @@ pub fn force_from_dep_node(tcx: TyCtxt<'_>, dep_node: &DepNode) -> bool { ($query:ident, $key:expr) => { force_ex!(tcx, $query, $key) } }; + let force_hir_map = || { + tcx.force_query::>( + LOCAL_CRATE, + DUMMY_SP, + DepNode::new(tcx, DepConstructor::hir_map(LOCAL_CRATE)), + ); + }; + rustc_dep_node_force!([dep_node, tcx] + // Created by the Hir map query + DepKind::AllLocalTraitImpls | + DepKind::Krate => force_hir_map(), + DepKind::HirBody | + DepKind::Hir => { + // Ensure the def_id exists + def_id!(); + force_hir_map(); + } + // These are inputs that are expected to be pre-allocated and that // should therefore always be red or green already - DepKind::AllLocalTraitImpls | - DepKind::Krate | DepKind::CrateMetadata | - DepKind::HirBody | - DepKind::Hir | // This are anonymous nodes DepKind::TraitSelect | diff --git a/src/librustc_data_structures/Cargo.toml b/src/librustc_data_structures/Cargo.toml index acddb3448ca60..79cbe26e73e83 100644 --- a/src/librustc_data_structures/Cargo.toml +++ b/src/librustc_data_structures/Cargo.toml @@ -18,6 +18,7 @@ lazy_static = "1" serialize = { path = "../libserialize" } graphviz = { path = "../libgraphviz" } cfg-if = "0.1.2" +crossbeam-utils = { version = "0.6.5", features = ["nightly"] } stable_deref_trait = "1.0.0" rayon = { version = "0.2.0", package = "rustc-rayon" } rayon-core = { version = "0.2.0", package = "rustc-rayon-core" } diff --git a/src/librustc_data_structures/sync.rs b/src/librustc_data_structures/sync.rs index 73247c1469efd..469f778551239 100644 --- a/src/librustc_data_structures/sync.rs +++ b/src/librustc_data_structures/sync.rs @@ -21,6 +21,7 @@ use std::collections::HashMap; use std::hash::{Hash, BuildHasher}; use std::marker::PhantomData; use std::ops::{Deref, DerefMut}; +use crate::cold_path; use crate::owning_ref::{Erased, OwningRef}; pub fn serial_join(oper_a: A, oper_b: B) -> (RA, RB) @@ -67,6 +68,59 @@ cfg_if! { use std::ops::Add; use std::panic::{resume_unwind, catch_unwind, AssertUnwindSafe}; + #[derive(Debug)] + pub struct AtomicCell(Cell); + + impl AtomicCell { + #[inline] + pub fn new(v: T) -> Self { + AtomicCell(Cell::new(v)) + } + } + + impl AtomicCell { + pub fn into_inner(self) -> T { + self.0.into_inner() + } + + #[inline] + pub fn load(&self) -> T { + self.0.get() + } + + #[inline] + pub fn store(&self, val: T) { + self.0.set(val) + } + + pub fn swap(&self, val: T) -> T { + self.0.replace(val) + } + } + + impl AtomicCell { + pub fn compare_exchange(&self, + current: T, + new: T) + -> Result { + let read = self.0.get(); + if read == current { + self.0.set(new); + Ok(read) + } else { + Err(read) + } + } + } + + impl + Copy> AtomicCell { + pub fn fetch_add(&self, val: T) -> T { + let old = self.0.get(); + self.0.set(old + val); + old + } + } + #[derive(Debug)] pub struct Atomic(Cell); @@ -77,7 +131,7 @@ cfg_if! { } } - impl Atomic { + impl Atomic { pub fn into_inner(self) -> T { self.0.into_inner() } @@ -95,7 +149,9 @@ cfg_if! { pub fn swap(&self, val: T, _: Ordering) -> T { self.0.replace(val) } + } + impl Atomic { pub fn compare_exchange(&self, current: T, new: T, @@ -271,6 +327,8 @@ cfg_if! { pub use std::sync::atomic::{AtomicBool, AtomicUsize, AtomicU32, AtomicU64}; + pub use crossbeam_utils::atomic::AtomicCell; + pub use std::sync::Arc as Lrc; pub use std::sync::Weak as Weak; @@ -516,6 +574,34 @@ impl Once { } } +/// A type whose inner value can be written once and then will stay read-only +pub struct AtomicOnce(AtomicCell>); + +impl AtomicOnce { + /// Creates an AtomicOnce value which is uninitialized + #[inline] + pub fn new() -> Self { + AtomicOnce(AtomicCell::new(None)) + } + + /// Gets the inner value. If the value is not already initalized. + /// It will be initalized by the closure. The closure may be called + /// concurrently by many threads. + #[inline(always)] + pub fn get_or_init(&self, f: impl FnOnce() -> T) -> T { + let value = self.0.load(); + if unlikely!(value.is_none()) { + cold_path(|| { + let value = f(); + self.0.store(Some(value)); + value + }) + } else { + value.unwrap() + } + } +} + #[derive(Debug)] pub struct Lock(InnerLock); diff --git a/src/librustc_interface/passes.rs b/src/librustc_interface/passes.rs index 9a5eb2b93d574..fa30fc5e3a0da 100644 --- a/src/librustc_interface/passes.rs +++ b/src/librustc_interface/passes.rs @@ -761,6 +761,7 @@ pub fn prepare_outputs( pub fn default_provide(providers: &mut ty::query::Providers<'_>) { providers.analysis = analysis; + providers.hir_map = hir_map; proc_macro_decls::provide(providers); plugin::build::provide(providers); hir::provide(providers); @@ -806,7 +807,7 @@ impl BoxedGlobalCtxt { pub fn create_global_ctxt( compiler: &Compiler, - mut hir_forest: hir::map::Forest, + hir_forest: hir::map::Forest, defs: hir::map::Definitions, resolutions: Resolutions, outputs: OutputFilenames, @@ -825,11 +826,6 @@ pub fn create_global_ctxt( let global_ctxt: Option>; let arenas = AllArenas::new(); - // Construct the HIR map - let hir_map = time(sess, "indexing hir", || { - hir::map::map_crate(sess, cstore, &mut hir_forest, &defs) - }); - let query_result_on_disk_cache = time(sess, "load query result cache", || { rustc_incremental::load_query_result_cache(sess) }); @@ -849,7 +845,8 @@ pub fn create_global_ctxt( extern_providers, &arenas, resolutions, - hir_map, + hir_forest, + defs, query_result_on_disk_cache, &crate_name, tx, @@ -876,6 +873,20 @@ pub fn create_global_ctxt( result } +fn hir_map<'tcx>( + tcx: TyCtxt<'tcx>, + cnum: CrateNum, +) -> &'tcx hir::map::Map<'tcx> { + assert_eq!(cnum, LOCAL_CRATE); + + // Construct the HIR map + let hir_map = time(tcx.sess, "indexing hir", || { + hir::map::map_crate(tcx) + }); + + tcx.arena.alloc(hir_map) +} + /// Runs the resolution, type-checking, region checking and other /// miscellaneous analysis passes on the crate. fn analysis(tcx: TyCtxt<'_>, cnum: CrateNum) -> Result<()> {