Skip to content

Commit

Permalink
Inline mark_neighbours_as_waiting_from.
Browse files Browse the repository at this point in the history
This function is very hot, doesn't get inlined because it's recursive,
and the function calls are significant.

This commit splits it into inlined and uninlined variants, and uses the
inlined variant for the hot call site. This wins several percent on a
few benchmarks.
  • Loading branch information
nnethercote committed Sep 13, 2019
1 parent 2b8116d commit a2261ad
Showing 1 changed file with 13 additions and 4 deletions.
17 changes: 13 additions & 4 deletions src/librustc_data_structures/obligation_forest/mod.rs
Expand Up @@ -559,13 +559,20 @@ impl<O: ForestObligation> ObligationForest<O> {
trace
}

#[inline]
fn mark_neighbors_as_waiting_from(&self, node: &Node<O>) {
// This always-inlined function is for the hot call site.
#[inline(always)]
fn inlined_mark_neighbors_as_waiting_from(&self, node: &Node<O>) {
for dependent in node.parent.iter().chain(node.dependents.iter()) {
self.mark_as_waiting_from(&self.nodes[dependent.get()]);
}
}

// This never-inlined function is for the cold call site.
#[inline(never)]
fn uninlined_mark_neighbors_as_waiting_from(&self, node: &Node<O>) {
self.inlined_mark_neighbors_as_waiting_from(node)
}

/// Marks all nodes that depend on a pending node as `NodeState::Waiting`.
fn mark_as_waiting(&self) {
for node in &self.nodes {
Expand All @@ -576,7 +583,8 @@ impl<O: ForestObligation> ObligationForest<O> {

for node in &self.nodes {
if node.state.get() == NodeState::Pending {
self.mark_neighbors_as_waiting_from(node);
// This call site is hot.
self.inlined_mark_neighbors_as_waiting_from(node);
}
}
}
Expand All @@ -588,7 +596,8 @@ impl<O: ForestObligation> ObligationForest<O> {
NodeState::Pending | NodeState::Done => {},
}

self.mark_neighbors_as_waiting_from(node);
// This call site is cold.
self.uninlined_mark_neighbors_as_waiting_from(node);
}

/// Compresses the vector, removing all popped nodes. This adjusts
Expand Down

0 comments on commit a2261ad

Please sign in to comment.