Provide capability for clients to build a node and walk the rule resu…

…lt info for the node and its dependencies

This is implemented by having the client optionally pass an implementation of a `RuleResultsWalker` which receives the related information for the nodes.
I preferred this method of allowing the client to "hook" in and get the info before the `build` call returns and does its clean up and any related "teardown".
That way we ensure the data is readily available for the client to inspect.
This method also allows to restrict the rule result queries only to the nodes related to the just built node, which simplifies things compared to exposing general APIs for querying any node, even if it wasn't involved in the build.

Related to rdar://67816715

include/llbuild/BuildSystem/BuildSystem.h

@@ -30,6 +30,9 @@ namespace basic {
   class ExecutionQueue;
   class FileSystem;
 }
+namespace core {
+  class RuleResultsWalker;
+}
 
 namespace buildsystem {
 
@@ -262,8 +265,9 @@ class BuildSystem {
   ///
   /// A build description *must* have been loaded before calling this method.
   ///
+  /// \param resultsWalker Optional walker for receiving the rule results of the node and its dependencies.
   /// \returns The result of computing the value, or nil if the build failed.
-  llvm::Optional<BuildValue> build(BuildKey target);
+  llvm::Optional<BuildValue> build(BuildKey target, core::RuleResultsWalker* resultsWalker = nullptr);
 
   /// Reset mutable build state before a new build operation.
   void resetForBuild();

include/llbuild/BuildSystem/BuildSystemFrontend.h

@@ -84,8 +84,9 @@ class BuildSystemFrontend {
 
   /// Build a single node using the specified invocation parameters.
   ///
+  /// \param resultsWalker Optional walker for receiving the rule results of the node and its dependencies.
   /// \returns True on success, or false if there were errors.
-  bool buildNode(StringRef nodeToBuild);
+  bool buildNode(StringRef nodeToBuild, core::RuleResultsWalker* resultsWalker = nullptr);
 
   /// @}
 };

include/llbuild/Core/BuildEngine.h

@@ -389,6 +389,29 @@ class BuildEngineDelegate {
 
 };
 
+/// Abstract class for visiting the rule results of a node and its dependencies.
+class RuleResultsWalker {
+public:
+  /// Specifies how node visitation should proceed.
+  enum class ActionKind {
+    /// Continue visiting the rule results of the current node dependencies.
+    VisitDependencies = 0,
+
+    /// Continue visiting but skip the current node dependencies.
+    SkipDependencies = 1,
+
+    /// Stop visitation.
+    Stop = 2
+  };
+
+  /// Accepts the rule result for a node.
+  ///
+  /// \param key The key for the currenly visited node.
+  /// \param key The rule result for the currenly visited node.
+  /// \returns An action kind to indicate how visitation should proceed.
+  virtual ActionKind visitResult(const KeyType& key, const core::Result& result) = 0;
+};
+
 /// A build engine supports fast, incremental, persistent, and parallel
 /// execution of computational graphs.
 ///
@@ -444,10 +467,11 @@ class BuildEngine {
 
   /// Build the result for a particular key.
   ///
+  /// \param resultsWalker Optional walker for receiving the rule results of the node and its dependencies.
   /// \returns The result of computing the key, or the empty value if the key
   /// could not be computed; the latter case only happens if a cycle was
   /// discovered currently.
-  const ValueType& build(const KeyType& key);
+  const ValueType& build(const KeyType& key, RuleResultsWalker* resultsWalker = nullptr);
 
   /// Cancel the currently running build.
   ///

lib/BuildSystem/BuildSystem.cpp

@@ -326,7 +326,7 @@ class BuildSystemImpl {
   }
 
   /// Build the given key, and return the result and an indication of success.
-  llvm::Optional<BuildValue> build(BuildKey key);
+  llvm::Optional<BuildValue> build(BuildKey key, RuleResultsWalker* resultsWalker);
 
   bool build(StringRef target);
 
@@ -1823,7 +1823,7 @@ BuildSystemImpl::lookupNode(StringRef name, bool isImplicit) {
   return BuildNode::makePlain(name);
 }
 
-llvm::Optional<BuildValue> BuildSystemImpl::build(BuildKey key) {
+llvm::Optional<BuildValue> BuildSystemImpl::build(BuildKey key, RuleResultsWalker* resultsWalker) {
 
   if (basic::sys::raiseOpenFileLimit() != 0) {
     error(getMainFilename(), "failed to raise open file limit");
@@ -1832,7 +1832,7 @@ llvm::Optional<BuildValue> BuildSystemImpl::build(BuildKey key) {
 
   // Build the target.
   buildWasAborted = false;
-  auto result = buildEngine.build(key.toData());
+  auto result = buildEngine.build(key.toData(), resultsWalker);
 
   // Clear out the shell handlers, as we do not want to hold on to them across
   // multiple builds.
@@ -1863,7 +1863,7 @@ bool BuildSystemImpl::build(StringRef target) {
     return false;
   }
 
-  return build(BuildKey::makeTarget(target)).hasValue();
+  return build(BuildKey::makeTarget(target), nullptr).hasValue();
 }
 
 #pragma mark - PhonyTool implementation
@@ -3864,8 +3864,8 @@ bool BuildSystem::enableTracing(StringRef path,
   return static_cast<BuildSystemImpl*>(impl)->enableTracing(path, error_out);
 }
 
-llvm::Optional<BuildValue> BuildSystem::build(BuildKey key) {
-  return static_cast<BuildSystemImpl*>(impl)->build(key);
+llvm::Optional<BuildValue> BuildSystem::build(BuildKey key, RuleResultsWalker* resultsWalker) {
+  return static_cast<BuildSystemImpl*>(impl)->build(key, resultsWalker);
 }
 
 bool BuildSystem::build(StringRef name) {

lib/BuildSystem/BuildSystemFrontend.cpp

@@ -435,7 +435,7 @@ struct BuildSystemFrontendImpl {
   }
 
 
-  bool buildNode(StringRef nodeToBuild) {
+  bool buildNode(StringRef nodeToBuild, core::RuleResultsWalker* resultsWalker) {
     llbuild_defer {
       resetAfterBuild();
     };
@@ -444,7 +444,7 @@ struct BuildSystemFrontendImpl {
       return false;
     }
 
-    auto buildValue = system->build(BuildKey::makeNode(nodeToBuild));
+    auto buildValue = system->build(BuildKey::makeNode(nodeToBuild), resultsWalker);
     if (!buildValue.hasValue()) {
       return false;
     }
@@ -808,6 +808,6 @@ bool BuildSystemFrontend::build(StringRef targetToBuild) {
   return static_cast<BuildSystemFrontendImpl*>(impl)->build(targetToBuild);
 }
 
-bool BuildSystemFrontend::buildNode(StringRef nodeToBuild) {
-  return static_cast<BuildSystemFrontendImpl*>(impl)->buildNode(nodeToBuild);
+bool BuildSystemFrontend::buildNode(StringRef nodeToBuild, core::RuleResultsWalker* resultsWalker) {
+  return static_cast<BuildSystemFrontendImpl*>(impl)->buildNode(nodeToBuild, resultsWalker);
 }

lib/Core/BuildEngine.cpp

@@ -18,6 +18,7 @@
 #include "llbuild/Core/BuildDB.h"
 #include "llbuild/Core/KeyID.h"
 
+#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringMap.h"
 
@@ -1476,7 +1477,7 @@ class BuildEngineImpl : public BuildDBDelegate {
   /// @name Client API
   /// @{
 
-  const ValueType& build(const KeyType& key) {
+  const ValueType& build(const KeyType& key, RuleResultsWalker* resultsWalker) {
     // Protect the engine against invalid concurrent use.
     if (buildRunning.exchange(true)) {
       delegate.error("build engine busy");
@@ -1574,6 +1575,35 @@ class BuildEngineImpl : public BuildDBDelegate {
       return emptyValue;
     }
 
+    if (resultsWalker) {
+      // Pass the rule results of the node and its dependencies.
+      SmallVector<KeyID, 8> queue;
+      queue.push_back(getKeyID(key));
+
+      llvm::SmallSet<KeyID, 16> visitedKeys;
+      while (!queue.empty()) {
+        KeyID key = queue.pop_back_val();
+        bool inserted = visitedKeys.insert(key).second;
+        if (!inserted) {
+          continue; // already visited.
+        }
+
+        auto& ruleInfo = getRuleInfoForKey(key);
+        switch (resultsWalker->visitResult(getKeyForID(key), ruleInfo.result)) {
+          case RuleResultsWalker::ActionKind::VisitDependencies:
+            for (AttributedKeyIDs::KeyIDAndFlag keyIDAndFlag : ruleInfo.result.dependencies) {
+              queue.push_back(keyIDAndFlag.keyID);
+            }
+            break;
+          case RuleResultsWalker::ActionKind::SkipDependencies:
+            break;
+          case RuleResultsWalker::ActionKind::Stop:
+            queue.clear();
+            break;
+        }
+      }
+    }
+
     // The task queue should be empty and the rule complete.
     auto& ruleInfo = getRuleInfoForKey(key);
     assert(taskInfos.empty() && ruleInfo.isComplete(this));
@@ -1854,8 +1884,8 @@ void BuildEngine::addRule(std::unique_ptr<Rule>&& rule) {
   static_cast<BuildEngineImpl*>(impl)->addRule(std::move(rule));
 }
 
-const ValueType& BuildEngine::build(const KeyType& key) {
-  return static_cast<BuildEngineImpl*>(impl)->build(key);
+const ValueType& BuildEngine::build(const KeyType& key, RuleResultsWalker* resultsWalker) {
+  return static_cast<BuildEngineImpl*>(impl)->build(key, resultsWalker);
 }
 
 void BuildEngine::resetForBuild() {

products/libllbuild/BuildSystem-C-API.cpp

@@ -584,6 +584,46 @@ class CAPIBuildSystem {
     return getFrontend().buildNode(key);
   }
 
+  bool buildNodeAndWalkResults(llvm::StringRef key, llb_buildsystem_rule_result_walker_t cWalker) {
+    class CAPIRuleResultsWalker : public core::RuleResultsWalker {
+      llb_buildsystem_rule_result_walker_t cAPIWalker;
+
+    public:
+      explicit CAPIRuleResultsWalker(llb_buildsystem_rule_result_walker_t walker) : cAPIWalker(walker) {}
+
+      ActionKind visitResult(const core::KeyType& key, const core::Result& result) override {
+        auto llbKey = llb_data_t {
+          key.size(),
+          (const uint8_t *)key.data()
+        };
+        auto llbValue = llb_data_t {
+          result.value.size(),
+          result.value.data()
+        };
+        auto llbResult = llb_buildsystem_rule_result_t {
+          llbValue,
+          result.signature.value,
+          result.computedAt,
+          result.builtAt,
+          result.start,
+          result.end,
+        };
+        auto llbAction = cAPIWalker.visit_result(cAPIWalker.context, llbKey, llbResult);
+        switch (llbAction) {
+          case llb_buildsystem_rule_result_walk_action_kind_visit_dependencies:
+            return ActionKind::VisitDependencies;
+          case llb_buildsystem_rule_result_walk_action_kind_skip_dependencies:
+            return ActionKind::SkipDependencies;
+          case llb_buildsystem_rule_result_walk_action_kind_stop:
+            return ActionKind::Stop;
+        }
+      }
+    };
+
+    CAPIRuleResultsWalker walker(cWalker);
+    return getFrontend().buildNode(key, &walker);
+  }
+
   void cancel() {
     frontendDelegate->cancel();
   }
@@ -900,6 +940,11 @@ bool llb_buildsystem_build_node(llb_buildsystem_t* system_p, const llb_data_t* k
   return system->buildNode(llvm::StringRef((const char*)key->data, key->length));
 }
 
+bool llb_buildsystem_build_node_and_walk_results(llb_buildsystem_t* system_p, const llb_data_t* key, llb_buildsystem_rule_result_walker_t walker) {
+  CAPIBuildSystem* system = (CAPIBuildSystem*) system_p;
+  return system->buildNodeAndWalkResults(llvm::StringRef((const char*)key->data, key->length), walker);
+}
+
 void llb_buildsystem_cancel(llb_buildsystem_t* system_p) {
   CAPIBuildSystem* system = (CAPIBuildSystem*) system_p;
   system->cancel();

products/libllbuild/include/llbuild/buildsystem.h

@@ -544,6 +544,60 @@ llb_buildsystem_build(llb_buildsystem_t* system, const llb_data_t* key);
 LLBUILD_EXPORT bool
 llb_buildsystem_build_node(llb_buildsystem_t* system, const llb_data_t* key);
 
+/// Defines the result of a task.
+typedef struct llb_buildsystem_rule_result_t_ {
+
+   /// The value that resulted from executing the task
+  llb_data_t value;
+
+   /// Signature of the node that generated the result
+  uint64_t signature;
+
+   /// The build iteration this result was computed at
+  uint64_t computed_at;
+
+   /// The build iteration this result was built at
+  uint64_t built_at;
+
+  /// The start of the command as a duration since a reference time
+  double start;
+
+  /// The duration since a reference time of when the command finished computing
+  double end;
+} llb_buildsystem_rule_result_t;
+
+/// Specifies how node visitation should proceed.
+typedef enum LLBUILD_ENUM_ATTRIBUTES {
+  llb_buildsystem_rule_result_walk_action_kind_visit_dependencies LLBUILD_SWIFT_NAME(visitDependencies) = 0,
+  llb_buildsystem_rule_result_walk_action_kind_skip_dependencies LLBUILD_SWIFT_NAME(skipDependencies) = 1,
+  llb_buildsystem_rule_result_walk_action_kind_stop = 2,
+} llb_buildsystem_rule_result_walk_action_kind_t LLBUILD_SWIFT_NAME(RuleResultWalkActionKind);
+
+/// Visitor for the rule results of a node and its dependencies.
+typedef struct llb_buildsystem_rule_result_walker_t_ {
+  /// User context pointer.
+  void* context;
+
+  /// Receives the key and rule result for the currently visited node. The `llb_data_t` objects of `key` and `result.value` point to transient memory that is only valid to access during the duration of this function call.
+  /// Copy the `llb_data_t` data of these objects if you want to refer to the data after this function returns.
+  llb_buildsystem_rule_result_walk_action_kind_t (*visit_result)(void* context, llb_data_t key,
+                                                      llb_buildsystem_rule_result_t result);
+} llb_buildsystem_rule_result_walker_t;
+
+/// Build a single node and walk the rule results of the node and its dependencies if successful. If it returns false, no visitation occurred.
+///
+/// It is an unchecked error for the client to request multiple builds
+/// concurrently.
+///
+/// This will automatically initialize the build system if it has not already
+/// been initialized.
+///
+/// \param walker Receiver for the rule results of the node and its dependencies.
+/// \returns True on success, or false if the build was aborted (for example, if
+/// a cycle was discovered).
+LLBUILD_EXPORT bool
+llb_buildsystem_build_node_and_walk_results(llb_buildsystem_t* system, const llb_data_t* key, llb_buildsystem_rule_result_walker_t walker);
+
 /// Cancel any ongoing build.
 ///
 /// This method may be called from any thread.