diff --git a/src/librustc/dep_graph/dep_node.rs b/src/librustc/dep_graph/dep_node.rs index f0c6196412adb..f8478a433a5b9 100644 --- a/src/librustc/dep_graph/dep_node.rs +++ b/src/librustc/dep_graph/dep_node.rs @@ -162,7 +162,9 @@ macro_rules! define_dep_nodes { } } - #[inline] + // FIXME: Make `is_anon`, `is_input`, `is_eval_always` and `has_params` properties + // of queries + #[inline(always)] pub fn is_anon(&self) -> bool { match *self { $( @@ -171,8 +173,8 @@ macro_rules! define_dep_nodes { } } - #[inline] - pub fn is_input(&self) -> bool { + #[inline(always)] + pub fn is_input_inlined(&self) -> bool { match *self { $( DepKind :: $variant => { contains_input_attr!($($attr),*) } @@ -180,7 +182,11 @@ macro_rules! define_dep_nodes { } } - #[inline] + pub fn is_input(&self) -> bool { + self.is_input_inlined() + } + + #[inline(always)] pub fn is_eval_always(&self) -> bool { match *self { $( @@ -190,8 +196,8 @@ macro_rules! define_dep_nodes { } #[allow(unreachable_code)] - #[inline] - pub fn has_params(&self) -> bool { + #[inline(always)] + pub fn has_params_inlined(&self) -> bool { match *self { $( DepKind :: $variant => { @@ -212,6 +218,10 @@ macro_rules! define_dep_nodes { )* } } + + pub fn has_params(&self) -> bool { + self.has_params_inlined() + } } pub enum DepConstructor<$tcx> { @@ -230,6 +240,7 @@ macro_rules! define_dep_nodes { impl DepNode { #[allow(unreachable_code, non_snake_case)] + #[inline(always)] pub fn new<'a, 'gcx, 'tcx>(tcx: TyCtxt<'a, 'gcx, 'tcx>, dep: DepConstructor<'gcx>) -> DepNode @@ -299,7 +310,7 @@ macro_rules! define_dep_nodes { /// Construct a DepNode from the given DepKind and DefPathHash. This /// method will assert that the given DepKind actually requires a /// single DefId/DefPathHash parameter. - #[inline] + #[inline(always)] pub fn from_def_path_hash(kind: DepKind, def_path_hash: DefPathHash) -> DepNode { @@ -313,9 +324,9 @@ macro_rules! define_dep_nodes { /// Create a new, parameterless DepNode. This method will assert /// that the DepNode corresponding to the given DepKind actually /// does not require any parameters. - #[inline] + #[inline(always)] pub fn new_no_params(kind: DepKind) -> DepNode { - assert!(!kind.has_params()); + assert!(!kind.has_params_inlined()); DepNode { kind, hash: Fingerprint::ZERO, @@ -418,14 +429,14 @@ impl fmt::Debug for DepNode { impl DefPathHash { - #[inline] + #[inline(always)] pub fn to_dep_node(self, kind: DepKind) -> DepNode { DepNode::from_def_path_hash(kind, self) } } impl DefId { - #[inline] + #[inline(always)] pub fn to_dep_node(self, tcx: TyCtxt<'_, '_, '_>, kind: DepKind) -> DepNode { DepNode::from_def_path_hash(kind, tcx.def_path_hash(self)) } diff --git a/src/librustc/hir/map/mod.rs b/src/librustc/hir/map/mod.rs index b98e279aef4b9..9b84e0570f244 100644 --- a/src/librustc/hir/map/mod.rs +++ b/src/librustc/hir/map/mod.rs @@ -159,6 +159,10 @@ impl Forest { self.dep_graph.read(DepNode::new_no_params(DepKind::Krate)); &self.krate } + + pub fn untracked_krate<'hir>(&'hir self) -> &'hir Crate { + &self.krate + } } /// Represents a mapping from Node IDs to AST elements and their parent diff --git a/src/librustc/ich/hcx.rs b/src/librustc/ich/hcx.rs index 2e56308daf7c2..799c2df8a53c5 100644 --- a/src/librustc/ich/hcx.rs +++ b/src/librustc/ich/hcx.rs @@ -86,6 +86,7 @@ impl<'a> StableHashingContext<'a> { // The `krate` here is only used for mapping BodyIds to Bodies. // Don't use it for anything else or you'll run the risk of // leaking data out of the tracking system. + #[inline] pub fn new(sess: &'a Session, krate: &'a hir::Crate, definitions: &'a Definitions, diff --git a/src/librustc/lib.rs b/src/librustc/lib.rs index 4324cfc7b5f10..b76fb0ed08c99 100644 --- a/src/librustc/lib.rs +++ b/src/librustc/lib.rs @@ -60,10 +60,12 @@ #![feature(slice_sort_by_cached_key)] #![feature(specialization)] #![feature(unboxed_closures)] +#![feature(thread_local)] #![feature(trace_macros)] #![feature(trusted_len)] #![feature(vec_remove_item)] #![feature(step_trait)] +#![feature(stmt_expr_attributes)] #![feature(integer_atomics)] #![feature(test)] #![feature(in_band_lifetimes)] diff --git a/src/librustc/session/mod.rs b/src/librustc/session/mod.rs index d1dd745add9a4..7d188b3601675 100644 --- a/src/librustc/session/mod.rs +++ b/src/librustc/session/mod.rs @@ -128,6 +128,9 @@ pub struct Session { /// Used by -Z profile-queries in util::common pub profile_channel: Lock>>, + /// Used by -Z self-profile + pub self_profiling_active: bool, + /// Used by -Z self-profile pub self_profiling: Lock, @@ -825,10 +828,17 @@ impl Session { } } + #[inline(never)] + #[cold] + fn profiler_active ()>(&self, f: F) { + let mut profiler = self.self_profiling.borrow_mut(); + f(&mut profiler); + } + + #[inline(always)] pub fn profiler ()>(&self, f: F) { - if self.opts.debugging_opts.self_profile || self.opts.debugging_opts.profile_json { - let mut profiler = self.self_profiling.borrow_mut(); - f(&mut profiler); + if unlikely!(self.self_profiling_active) { + self.profiler_active(f) } } @@ -1138,6 +1148,9 @@ pub fn build_session_( CguReuseTracker::new_disabled() }; + let self_profiling_active = sopts.debugging_opts.self_profile || + sopts.debugging_opts.profile_json; + let sess = Session { target: target_cfg, host, @@ -1168,6 +1181,7 @@ pub fn build_session_( imported_macro_spans: OneThread::new(RefCell::new(FxHashMap::default())), incr_comp_session: OneThread::new(RefCell::new(IncrCompSession::NotInitialized)), cgu_reuse_tracker, + self_profiling_active, self_profiling: Lock::new(SelfProfiler::new()), profile_channel: Lock::new(None), perf_stats: PerfStats { diff --git a/src/librustc/ty/context.rs b/src/librustc/ty/context.rs index a3db3a02aad92..e69c0efeb2c5c 100644 --- a/src/librustc/ty/context.rs +++ b/src/librustc/ty/context.rs @@ -1336,8 +1336,9 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> { self.cstore.crate_data_as_rc_any(cnum) } + #[inline(always)] pub fn create_stable_hashing_context(self) -> StableHashingContext<'a> { - let krate = self.dep_graph.with_ignore(|| self.hir().krate()); + let krate = self.gcx.hir_map.forest.untracked_krate(); StableHashingContext::new(self.sess, krate, @@ -1925,7 +1926,17 @@ pub mod tls { /// A thread local variable which stores a pointer to the current ImplicitCtxt #[cfg(not(parallel_queries))] - thread_local!(static TLV: Cell = Cell::new(0)); + // Accessing `thread_local` in another crate is bugged, so we have + // two accessors `set_raw_tlv` and `get_tlv` which do not have an + // inline attribute to prevent that + #[thread_local] + static TLV: Cell = Cell::new(0); + + /// This is used to set the pointer to the current ImplicitCtxt. + #[cfg(not(parallel_queries))] + fn set_raw_tlv(value: usize) { + TLV.set(value) + } /// Sets TLV to `value` during the call to `f`. /// It is restored to its previous value after. @@ -1933,15 +1944,15 @@ pub mod tls { #[cfg(not(parallel_queries))] fn set_tlv R, R>(value: usize, f: F) -> R { let old = get_tlv(); - let _reset = OnDrop(move || TLV.with(|tlv| tlv.set(old))); - TLV.with(|tlv| tlv.set(value)); + let _reset = OnDrop(move || set_raw_tlv(old)); + set_raw_tlv(value); f() } /// This is used to get the pointer to the current ImplicitCtxt. #[cfg(not(parallel_queries))] fn get_tlv() -> usize { - TLV.with(|tlv| tlv.get()) + TLV.get() } /// This is a callback from libsyntax as it cannot access the implicit state diff --git a/src/librustc/ty/query/job.rs b/src/librustc/ty/query/job.rs index 1439e41bb31fd..994a80fe4cd9e 100644 --- a/src/librustc/ty/query/job.rs +++ b/src/librustc/ty/query/job.rs @@ -18,6 +18,11 @@ use syntax_pos::Span; use ty::tls; use ty::query::Query; use ty::query::plumbing::CycleError; +#[cfg(not(parallel_queries))] +use ty::query::{ + plumbing::TryGetJob, + config::QueryDescription, +}; use ty::context::TyCtxt; use errors::Diagnostic; use std::process; @@ -83,36 +88,44 @@ impl<'tcx> QueryJob<'tcx> { /// /// For single threaded rustc there's no concurrent jobs running, so if we are waiting for any /// query that means that there is a query cycle, thus this always running a cycle error. - pub(super) fn await<'lcx>( + #[cfg(not(parallel_queries))] + #[inline(never)] + #[cold] + pub(super) fn await<'lcx, 'a, D: QueryDescription<'tcx>>( &self, tcx: TyCtxt<'_, 'tcx, 'lcx>, span: Span, - ) -> Result<(), CycleError<'tcx>> { - #[cfg(not(parallel_queries))] - { - self.find_cycle_in_stack(tcx, span) - } + ) -> TryGetJob<'a, 'tcx, D> { + TryGetJob::JobCompleted(Err(Box::new(self.find_cycle_in_stack(tcx, span)))) + } - #[cfg(parallel_queries)] - { - tls::with_related_context(tcx, move |icx| { - let mut waiter = Lrc::new(QueryWaiter { - query: icx.query.clone(), - span, - cycle: Lock::new(None), - condvar: Condvar::new(), - }); - self.latch.await(&waiter); - // FIXME: Get rid of this lock. We have ownership of the QueryWaiter - // although another thread may still have a Lrc reference so we cannot - // use Lrc::get_mut - let mut cycle = waiter.cycle.lock(); - match cycle.take() { - None => Ok(()), - Some(cycle) => Err(cycle) - } - }) - } + /// Awaits for the query job to complete. + /// + /// For single threaded rustc there's no concurrent jobs running, so if we are waiting for any + /// query that means that there is a query cycle, thus this always running a cycle error. + #[cfg(parallel_queries)] + pub(super) fn await<'lcx>( + &self, + tcx: TyCtxt<'_, 'tcx, 'lcx>, + span: Span, + ) -> Result<(), Box>> { + tls::with_related_context(tcx, move |icx| { + let mut waiter = Lrc::new(QueryWaiter { + query: icx.query.clone(), + span, + cycle: Lock::new(None), + condvar: Condvar::new(), + }); + self.latch.await(&waiter); + // FIXME: Get rid of this lock. We have ownership of the QueryWaiter + // although another thread may still have a Lrc reference so we cannot + // use Lrc::get_mut + let mut cycle = waiter.cycle.lock(); + match cycle.take() { + None => Ok(()), + Some(cycle) => Err(Box::new(cycle)) + } + }) } #[cfg(not(parallel_queries))] @@ -120,7 +133,7 @@ impl<'tcx> QueryJob<'tcx> { &self, tcx: TyCtxt<'_, 'tcx, 'lcx>, span: Span, - ) -> Result<(), CycleError<'tcx>> { + ) -> CycleError<'tcx> { // Get the current executing query (waiter) and find the waitee amongst its parents let mut current_job = tls::with_related_context(tcx, |icx| icx.query.clone()); let mut cycle = Vec::new(); @@ -140,7 +153,7 @@ impl<'tcx> QueryJob<'tcx> { let usage = job.parent.as_ref().map(|parent| { (job.info.span, parent.info.query.clone()) }); - return Err(CycleError { usage, cycle }); + return CycleError { usage, cycle }; } current_job = job.parent.clone(); diff --git a/src/librustc/ty/query/mod.rs b/src/librustc/ty/query/mod.rs index 5cd06fb8a52c0..f760ebbd759ec 100644 --- a/src/librustc/ty/query/mod.rs +++ b/src/librustc/ty/query/mod.rs @@ -705,21 +705,21 @@ impl<'a, 'tcx, 'lcx> TyCtxt<'a, 'tcx, 'lcx> { self, span: Span, key: DefId, - ) -> Result<&'tcx [Ty<'tcx>], DiagnosticBuilder<'a>> { + ) -> Result<&'tcx [Ty<'tcx>], Box>> { self.try_get_query::>(span, key) } pub fn try_needs_drop_raw( self, span: Span, key: ty::ParamEnvAnd<'tcx, Ty<'tcx>>, - ) -> Result> { + ) -> Result>> { self.try_get_query::>(span, key) } pub fn try_optimized_mir( self, span: Span, key: DefId, - ) -> Result<&'tcx mir::Mir<'tcx>, DiagnosticBuilder<'a>> { + ) -> Result<&'tcx mir::Mir<'tcx>, Box>> { self.try_get_query::>(span, key) } } diff --git a/src/librustc/ty/query/plumbing.rs b/src/librustc/ty/query/plumbing.rs index 5f33d466c4a19..157e0fba9deda 100644 --- a/src/librustc/ty/query/plumbing.rs +++ b/src/librustc/ty/query/plumbing.rs @@ -153,8 +153,14 @@ impl<'a, 'tcx, Q: QueryDescription<'tcx>> JobOwner<'a, 'tcx, Q> { }; mem::drop(lock); - if let Err(cycle) = job.await(tcx, span) { - return TryGetJob::JobCompleted(Err(cycle)); + #[cfg(not(parallel_queries))] + return job.await(tcx, span); + + #[cfg(parallel_queries)] + { + if let Err(cycle) = job.await(tcx, span) { + return TryGetJob::JobCompleted(Err(cycle)); + } } } } @@ -241,12 +247,16 @@ pub(super) enum TryGetJob<'a, 'tcx: 'a, D: QueryDescription<'tcx> + 'a> { /// The query was already completed. /// Returns the result of the query and its dep node index /// if it succeeded or a cycle error if it failed - JobCompleted(Result<(D::Value, DepNodeIndex), CycleError<'tcx>>), + JobCompleted(Result<(D::Value, DepNodeIndex), Box>>), } impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> { - pub(super) fn report_cycle(self, CycleError { usage, cycle: stack }: CycleError<'gcx>) - -> DiagnosticBuilder<'a> + #[inline(never)] + #[cold] + pub(super) fn report_cycle( + self, + box CycleError { usage, cycle: stack }: Box> + ) -> Box> { assert!(!stack.is_empty()); @@ -280,7 +290,7 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> { &format!("cycle used when {}", query.describe(self))); } - return err + return Box::new(err) }) } @@ -345,11 +355,12 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> { } } + #[inline(never)] fn try_get_with>( self, span: Span, key: Q::Key) - -> Result> + -> Result>> { debug!("ty::queries::{}::try_get_with(key={:?}, span={:?})", Q::NAME, @@ -409,7 +420,7 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> { return Ok(result); } - if !dep_node.kind.is_input() { + if !dep_node.kind.is_input_inlined() { if let Some(dep_node_index) = self.try_mark_green_and_read(&dep_node) { profq_msg!(self, ProfileQueriesMsg::CacheHit); self.sess.profiler(|p| p.record_query_hit(Q::CATEGORY)); @@ -436,7 +447,7 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> { job: JobOwner<'a, 'gcx, Q>, dep_node_index: DepNodeIndex, dep_node: &DepNode - ) -> Result> + ) -> Result>> { // Note this function can be called concurrently from the same query // We must ensure that this is handled correctly @@ -522,7 +533,7 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> { key: Q::Key, job: JobOwner<'_, 'gcx, Q>, dep_node: DepNode) - -> Result<(Q::Value, DepNodeIndex), CycleError<'gcx>> { + -> Result<(Q::Value, DepNodeIndex), Box>> { // If the following assertion triggers, it can have two reasons: // 1. Something is wrong with DepNode creation, either here or // in DepGraph::try_mark_green() @@ -585,7 +596,7 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> { // Ensuring an "input" or anonymous query makes no sense assert!(!dep_node.kind.is_anon()); - assert!(!dep_node.kind.is_input()); + assert!(!dep_node.kind.is_input_inlined()); if self.try_mark_green_and_read(&dep_node).is_none() { // A None return from `try_mark_green_and_read` means that this is either // a new dep node or that the dep node has already been marked red. @@ -611,37 +622,55 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> { key: Q::Key, span: Span, dep_node: DepNode - ) -> Result<(Q::Value, DepNodeIndex), CycleError<'gcx>> { + ) { + profq_msg!( + self, + ProfileQueriesMsg::QueryBegin(span.data(), profq_query_msg!(Q::NAME, self, key)) + ); + // We may be concurrently trying both execute and force a query // Ensure that only one of them runs the query let job = match JobOwner::try_get(self, span, &key) { TryGetJob::NotYetStarted(job) => job, - TryGetJob::JobCompleted(result) => return result, + TryGetJob::JobCompleted(_) => return, }; - self.force_query_with_job::(key, job, dep_node) + if let Err(e) = self.force_query_with_job::(key, job, dep_node) { + self.report_cycle(e).emit(); + } } pub(super) fn try_get_query>( self, span: Span, key: Q::Key, - ) -> Result> { + ) -> Result>> { match self.try_get_with::(span, key) { Ok(e) => Ok(e), Err(e) => Err(self.report_cycle(e)), } } + // FIXME: Try uninlining this + #[inline(always)] pub(super) fn get_query>( self, span: Span, key: Q::Key, ) -> Q::Value { - self.try_get_query::(span, key).unwrap_or_else(|mut e| { - e.emit(); - Q::handle_cycle_error(self) + self.try_get_with::(span, key).unwrap_or_else(|e| { + self.emit_error::(e) }) } + + #[inline(never)] + #[cold] + fn emit_error>( + self, + e: Box>, + ) -> Q::Value { + self.report_cycle(e).emit(); + Q::handle_cycle_error(self) + } } macro_rules! handle_cycle_error { @@ -806,15 +835,18 @@ macro_rules! define_queries_inner { } impl<$tcx> QueryAccessors<$tcx> for queries::$name<$tcx> { + #[inline(always)] fn query(key: Self::Key) -> Query<'tcx> { Query::$name(key) } + #[inline(always)] fn query_cache<'a>(tcx: TyCtxt<'a, $tcx, '_>) -> &'a Lock> { &tcx.queries.$name } #[allow(unused)] + #[inline(always)] fn to_dep_node(tcx: TyCtxt<'_, $tcx, '_>, key: &Self::Key) -> DepNode { use dep_graph::DepConstructor::*; @@ -861,6 +893,7 @@ macro_rules! define_queries_inner { impl<'a, 'gcx, 'tcx> Deref for TyCtxtAt<'a, 'gcx, 'tcx> { type Target = TyCtxt<'a, 'gcx, 'tcx>; + #[inline(always)] fn deref(&self) -> &Self::Target { &self.tcx } @@ -869,6 +902,7 @@ macro_rules! define_queries_inner { impl<'a, $tcx, 'lcx> TyCtxt<'a, $tcx, 'lcx> { /// Return a transparent wrapper for `TyCtxt` which uses /// `span` as the location of queries performed through it. + #[inline(always)] pub fn at(self, span: Span) -> TyCtxtAt<'a, $tcx, 'lcx> { TyCtxtAt { tcx: self, @@ -877,6 +911,7 @@ macro_rules! define_queries_inner { } $($(#[$attr])* + #[inline(always)] pub fn $name(self, key: $K) -> $V { self.at(DUMMY_SP).$name(key) })* @@ -884,6 +919,7 @@ macro_rules! define_queries_inner { impl<'a, $tcx, 'lcx> TyCtxtAt<'a, $tcx, 'lcx> { $($(#[$attr])* + #[inline(always)] pub fn $name(self, key: $K) -> $V { self.tcx.get_query::>(self.span, key) })* @@ -1023,20 +1059,7 @@ pub fn force_from_dep_node<'a, 'gcx, 'lcx>(tcx: TyCtxt<'a, 'gcx, 'lcx>, macro_rules! force { ($query:ident, $key:expr) => { { - use $crate::util::common::{ProfileQueriesMsg, profq_msg}; - - profq_msg!(tcx, - ProfileQueriesMsg::QueryBegin( - DUMMY_SP.data(), - profq_query_msg!(::ty::query::queries::$query::NAME, tcx, $key), - ) - ); - - if let Err(e) = tcx.force_query::<::ty::query::queries::$query<'_>>( - $key, DUMMY_SP, *dep_node - ) { - tcx.report_cycle(e).emit(); - } + tcx.force_query::<::ty::query::queries::$query<'_>>($key, DUMMY_SP, *dep_node); } } }; diff --git a/src/librustc_data_structures/fingerprint.rs b/src/librustc_data_structures/fingerprint.rs index aa9ddda2b9364..f8638213b3a2f 100644 --- a/src/librustc_data_structures/fingerprint.rs +++ b/src/librustc_data_structures/fingerprint.rs @@ -86,6 +86,7 @@ impl ::std::fmt::Display for Fingerprint { } impl stable_hasher::StableHasherResult for Fingerprint { + #[inline] fn finish(hasher: stable_hasher::StableHasher) -> Self { let (_0, _1) = hasher.finalize(); Fingerprint(_0, _1) diff --git a/src/librustc_data_structures/lib.rs b/src/librustc_data_structures/lib.rs index 8e0ecb70c6896..bc2b8f1d6523e 100644 --- a/src/librustc_data_structures/lib.rs +++ b/src/librustc_data_structures/lib.rs @@ -30,6 +30,8 @@ #![feature(allow_internal_unstable)] #![feature(vec_resize_with)] #![feature(hash_raw_entry)] +#![feature(stmt_expr_attributes)] +#![feature(core_intrinsics)] #![cfg_attr(unix, feature(libc))] #![cfg_attr(test, feature(test))] @@ -58,6 +60,26 @@ extern crate rustc_cratesio_shim; pub use rustc_serialize::hex::ToHex; +#[macro_export] +macro_rules! likely { + ($e:expr) => { + #[allow(unused_unsafe)] + { + unsafe { std::intrinsics::likely($e) } + } + } +} + +#[macro_export] +macro_rules! unlikely { + ($e:expr) => { + #[allow(unused_unsafe)] + { + unsafe { std::intrinsics::unlikely($e) } + } + } +} + pub mod macros; pub mod svh; pub mod base_n; diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index eb71003d3d0cf..1c783579d2cbf 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -81,6 +81,7 @@ impl ParseSess { } } + #[inline] pub fn source_map(&self) -> &SourceMap { &self.source_map }