Teach builder to load dyndep files when they are ready

After finishing an edge that produces a dyndep file, load the file and
update the build graph structure.  Recompute the dirty state of all its
dependents and of newly reachable portions of the graph.  Add edges to
the build plan that are discovered to be wanted.  Finally, schedule
edges that are wanted and now ready to build.

src/build.cc

@@ -174,6 +174,20 @@ void BuildStatus::BuildEdgeFinished(Edge* edge,
   }
 }
 
+void BuildStatus::BuildLoadDyndeps() {
+  // The DependencyScan calls EXPLAIN() to print lines explaining why
+  // it considers a portion of the graph to be out of date.  Normally
+  // this is done before the build starts, but our caller is about to
+  // load a dyndep file during the build.  Doing so may generate more
+  // exlanation lines (via fprintf directly to stderr), but in an
+  // interactive console the cursor is currently at the end of a status
+  // line.  Start a new line so that the first explanation does not
+  // append to the status line.  After the explanations are done a
+  // new build status line will appear.
+  if (g_explaining)
+    printer_.PrintOnNewLine("");
+}
+
 void BuildStatus::BuildStarted() {
   overall_rate_.Restart();
   current_rate_.Restart();
@@ -302,10 +316,11 @@ void Plan::Reset() {
 }
 
 bool Plan::AddTarget(Node* node, string* err) {
-  return AddSubTarget(node, NULL, err);
+  return AddSubTarget(node, NULL, err, NULL);
 }
 
-bool Plan::AddSubTarget(Node* node, Node* dependent, string* err) {
+bool Plan::AddSubTarget(Node* node, Node* dependent, string* err,
+                        set<Edge*>* dyndep_walk) {
   Edge* edge = node->in_edge();
   if (!edge) {  // Leaf node.
     if (node->dirty()) {
@@ -327,21 +342,27 @@ bool Plan::AddSubTarget(Node* node, Node* dependent, string* err) {
     want_.insert(make_pair(edge, kWantNothing));
   Want& want = want_ins.first->second;
 
+  if (dyndep_walk && want == kWantToFinish)
+    return false;  // Don't need to do anything with already-scheduled edge.
+
   // If we do need to build edge and we haven't already marked it as wanted,
   // mark it now.
   if (node->dirty() && want == kWantNothing) {
     want = kWantToStart;
     EdgeWanted(edge);
-    if (edge->AllInputsReady())
+    if (!dyndep_walk && edge->AllInputsReady())
       ScheduleWork(want_ins.first);
   }
 
+  if (dyndep_walk)
+    dyndep_walk->insert(edge);
+
   if (!want_ins.second)
     return true;  // We've already processed the inputs.
 
   for (vector<Node*>::iterator i = edge->inputs_.begin();
        i != edge->inputs_.end(); ++i) {
-    if (!AddSubTarget(*i, node, err) && !err->empty())
+    if (!AddSubTarget(*i, node, err, dyndep_walk) && !err->empty())
       return false;
   }
 
@@ -414,6 +435,14 @@ bool Plan::EdgeFinished(Edge* edge, EdgeResult result, string* err) {
 }
 
 bool Plan::NodeFinished(Node* node, string* err) {
+  // If this node provides dyndep info, load it now.
+  if (node->dyndep_pending()) {
+    assert(builder_ && "dyndep requires Plan to have a Builder");
+    // Load the now-clean dyndep file.  This will also update the
+    // build plan and schedule any new work that is ready.
+    return builder_->LoadDyndeps(node, err);
+  }
+
   // See if we we want any edges from this node.
   for (vector<Edge*>::const_iterator oe = node->out_edges().begin();
        oe != node->out_edges().end(); ++oe) {
@@ -500,6 +529,128 @@ bool Plan::CleanNode(DependencyScan* scan, Node* node, string* err) {
   return true;
 }
 
+bool Plan::DyndepsLoaded(DependencyScan* scan, Node* node,
+                         const DyndepFile& ddf, string* err) {
+  // Recompute the dirty state of all our direct and indirect dependents now
+  // that our dyndep information has been loaded.
+  if (!RefreshDyndepDependents(scan, node, err))
+    return false;
+
+  // We loaded dyndep information for those out_edges of the dyndep node that
+  // specify the node in a dyndep binding, but they may not be in the plan.
+  // Starting with those already in the plan, walk newly-reachable portion
+  // of the graph through the dyndep-discovered dependencies.
+
+  // Find edges in the the build plan for which we have new dyndep info.
+  std::vector<DyndepFile::const_iterator> dyndep_roots;
+  for (DyndepFile::const_iterator oe = ddf.begin(); oe != ddf.end(); ++oe) {
+    Edge* edge = oe->first;
+
+    // If the edge outputs are ready we do not need to consider it here.
+    if (edge->outputs_ready())
+      continue;
+
+    map<Edge*, Want>::iterator want_e = want_.find(edge);
+
+    // If the edge has not been encountered before then nothing already in the
+    // plan depends on it so we do not need to consider the edge yet either.
+    if (want_e == want_.end())
+      continue;
+
+    // This edge is already in the plan so queue it for the walk.
+    dyndep_roots.push_back(oe);
+  }
+
+  // Walk dyndep-discovered portion of the graph to add it to the build plan.
+  std::set<Edge*> dyndep_walk;
+  for (std::vector<DyndepFile::const_iterator>::iterator
+       oei = dyndep_roots.begin(); oei != dyndep_roots.end(); ++oei) {
+    DyndepFile::const_iterator oe = *oei;
+    for (vector<Node*>::const_iterator i = oe->second.implicit_inputs_.begin();
+         i != oe->second.implicit_inputs_.end(); ++i) {
+      if (!AddSubTarget(*i, oe->first->outputs_[0], err, &dyndep_walk) &&
+          !err->empty())
+        return false;
+    }
+  }
+
+  // Add out edges from this node that are in the plan (just as
+  // Plan::NodeFinished would have without taking the dyndep code path).
+  for (vector<Edge*>::const_iterator oe = node->out_edges().begin();
+       oe != node->out_edges().end(); ++oe) {
+    map<Edge*, Want>::iterator want_e = want_.find(*oe);
+    if (want_e == want_.end())
+      continue;
+    dyndep_walk.insert(want_e->first);
+  }
+
+  // See if any encountered edges are now ready.
+  for (set<Edge*>::iterator wi = dyndep_walk.begin();
+       wi != dyndep_walk.end(); ++wi) {
+    map<Edge*, Want>::iterator want_e = want_.find(*wi);
+    if (want_e == want_.end())
+      continue;
+    if (!EdgeMaybeReady(want_e, err))
+      return false;
+  }
+
+  return true;
+}
+
+bool Plan::RefreshDyndepDependents(DependencyScan* scan, Node* node,
+                                   string* err) {
+  // Collect the transitive closure of dependents and mark their edges
+  // as not yet visited by RecomputeDirty.
+  set<Node*> dependents;
+  UnmarkDependents(node, &dependents);
+
+  // Update the dirty state of all dependents and check if their edges
+  // have become wanted.
+  for (set<Node*>::iterator i = dependents.begin();
+       i != dependents.end(); ++i) {
+    Node* n = *i;
+
+    // Check if this dependent node is now dirty.  Also checks for new cycles.
+    if (!scan->RecomputeDirty(n, err))
+      return false;
+    if (!n->dirty())
+      continue;
+
+    // This edge was encountered before.  However, we may not have wanted to
+    // build it if the outputs were not known to be dirty.  With dyndep
+    // information an output is now known to be dirty, so we want the edge.
+    Edge* edge = n->in_edge();
+    assert(edge && !edge->outputs_ready());
+    map<Edge*, Want>::iterator want_e = want_.find(edge);
+    assert(want_e != want_.end());
+    if (want_e->second == kWantNothing) {
+      want_e->second = kWantToStart;
+      EdgeWanted(edge);
+    }
+  }
+  return true;
+}
+
+void Plan::UnmarkDependents(Node* node, set<Node*>* dependents) {
+  for (vector<Edge*>::const_iterator oe = node->out_edges().begin();
+       oe != node->out_edges().end(); ++oe) {
+    Edge* edge = *oe;
+
+    map<Edge*, Want>::iterator want_e = want_.find(edge);
+    if (want_e == want_.end())
+      continue;
+
+    if (edge->mark_ != Edge::VisitNone) {
+      edge->mark_ = Edge::VisitNone;
+      for (vector<Node*>::iterator o = edge->outputs_.begin();
+           o != edge->outputs_.end(); ++o) {
+        if (dependents->insert(*o).second)
+          UnmarkDependents(*o, dependents);
+      }
+    }
+  }
+}
+
 void Plan::Dump() {
   printf("pending: %d\n", (int)want_.size());
   for (map<Edge*, Want>::iterator e = want_.begin(); e != want_.end(); ++e) {
@@ -959,3 +1110,21 @@ bool Builder::ExtractDeps(CommandRunner::Result* result,
 
   return true;
 }
+
+bool Builder::LoadDyndeps(Node* node, string* err) {
+  status_->BuildLoadDyndeps();
+
+  // Load the dyndep information provided by this node.
+  DyndepFile ddf;
+  if (!scan_.LoadDyndeps(node, &ddf, err))
+    return false;
+
+  // Update the build plan to account for dyndep modifications to the graph.
+  if (!plan_.DyndepsLoaded(&scan_, node, ddf, err))
+    return false;
+
+  // New command edges may have been added to the plan.
+  status_->PlanHasTotalEdges(plan_.command_edge_count());
+
+  return true;
+}

src/build.h

@@ -64,7 +64,9 @@ struct Plan {
   };
 
   /// Mark an edge as done building (whether it succeeded or failed).
-  /// Returns 'true'.
+  /// If any of the edge's outputs are dyndep bindings of their dependents,
+  /// this loads dynamic dependencies from the nodes' paths.
+  /// Returns 'false' if loading dyndep info fails and 'true' otherwise.
   bool EdgeFinished(Edge* edge, EdgeResult result, string* err);
 
   /// Clean the given node during the build.
@@ -77,11 +79,20 @@ struct Plan {
   /// Reset state.  Clears want and ready sets.
   void Reset();
 
+  /// Update the build plan to account for modifications made to the graph
+  /// by information loaded from a dyndep file.
+  bool DyndepsLoaded(DependencyScan* scan, Node* node,
+                     const DyndepFile& ddf, string* err);
 private:
-  bool AddSubTarget(Node* node, Node* dependent, string* err);
+  bool RefreshDyndepDependents(DependencyScan* scan, Node* node, string* err);
+  void UnmarkDependents(Node* node, set<Node*>* dependents);
+  bool AddSubTarget(Node* node, Node* dependent, string* err,
+                    set<Edge*>* dyndep_walk);
 
   /// Update plan with knowledge that the given node is up to date.
-  /// Returns 'true'.
+  /// If the node is a dyndep binding on any of its dependents, this
+  /// loads dynamic dependencies from the node's path.
+  /// Returns 'false' if loading dyndep info fails and 'true' otherwise.
   bool NodeFinished(Node* node, string* err);
 
   /// Enumerate possible steps we want for an edge.
@@ -199,6 +210,9 @@ struct Builder {
     scan_.set_build_log(log);
   }
 
+  /// Load the dyndep information provided by the given node.
+  bool LoadDyndeps(Node* node, string* err);
+
   State* state_;
   const BuildConfig& config_;
   Plan plan_;
@@ -229,6 +243,7 @@ struct BuildStatus {
   void BuildEdgeStarted(Edge* edge);
   void BuildEdgeFinished(Edge* edge, bool success, const string& output,
                          int* start_time, int* end_time);
+  void BuildLoadDyndeps();
   void BuildStarted();
   void BuildFinished();