RFC: Expose systemlib decls in full repo builds

Summary:
The previous diffs in this stack exposed systemlib decls to:
* Incremental file compilation
* Compiling systemlib itself at HHVM startup time

This diff indexes all of the decls in `s_builtin_symbols` akin to how files in the repo itself are indexed. They're indexed with an empty file name, which I did specifically to avoid attempts to parse / emit bytecode for said files as part of `Package::resolveOnDemand`: we'll unconditionally parse / emit the bytecode for systemlib.

Reviewed By: edwinsmith

Differential Revision: D39754321

fbshipit-source-id: 3a4fa9d30823293ae9b5570040bc03b0ffec6ecf

Author: Hunter Goldstein

hphp/compiler/compiler.cpp

@@ -19,6 +19,8 @@
 #include "hphp/compiler/option.h"
 #include "hphp/compiler/package.h"
 
+#include "hphp/hack/src/hackc/ffi_bridge/compiler_ffi.rs.h"
+
 #include "hphp/hhbbc/hhbbc.h"
 #include "hphp/hhbbc/misc.h"
 #include "hphp/hhbbc/options.h"
@@ -31,6 +33,7 @@
 #include "hphp/runtime/base/variable-serializer.h"
 #include "hphp/runtime/version.h"
 
+#include "hphp/runtime/vm/builtin-symbol-map.h"
 #include "hphp/runtime/vm/disas.h"
 #include "hphp/runtime/vm/preclass-emitter.h"
 #include "hphp/runtime/vm/repo-autoload-map-builder.h"
@@ -816,6 +819,38 @@ void logPhaseStats(const std::string& phase, const Package& package,
   sample.setStr(phase + "_fellback", client.fellback() ? "true" : "false");
 }
 
+namespace {
+  // Upload all builtin decls, and pass their IndexMeta summary and
+  // Ref<UnitDecls> to callback() to include in the overall UnitIndex. This
+  // makes systemlib decls visible to files being compiled as part of the
+  // full repo build, but does not make repo decls available to systemlib.
+  coro::Task<bool> indexBuiltinSymbolDecls(
+    const Package::IndexCallback& callback,
+    coro::TicketExecutor& executor,
+    extern_worker::Client& client
+  ) {
+    std::vector<coro::TaskWithExecutor<void>> tasks;
+    auto const declCallback = [&](auto const* d) -> coro::Task<void> {
+      auto const facts = hackc::decls_to_facts_cpp_ffi(*d, "");
+      auto summary = summary_of_facts(facts);
+      callback(
+        "",
+        summary,
+        HPHP_CORO_AWAIT(client.store(Package::UnitDecls{
+          summary,
+          std::string{d->serialized.begin(), d->serialized.end()}
+        }))
+      );
+      HPHP_CORO_RETURN_VOID;
+    };
+    for (auto const& d: Native::getAllBuiltinDecls()) {
+      tasks.emplace_back(declCallback(d).scheduleOn(executor.sticky()));
+    }
+    HPHP_CORO_AWAIT(coro::collectRange(std::move(tasks)));
+    HPHP_CORO_RETURN(true);
+  }
+}
+
 // Compute a UnitIndex by parsing decls for all autoload-eligible files.
 // If no Autoload.Query is specified by RepoOptions, this just indexes
 // the input files.
@@ -862,8 +897,25 @@ std::unique_ptr<UnitIndex> computeIndex(
     // index just the input files
     addInputsToPackage(indexPackage, po);
   }
+  // Here, we are doing the following in parallel:
+  // * Indexing the build package
+  // * Indexing builtin decls to be used by decl driven bytecode compilation
+  // If DDB is not enabled, we will return early from the second task.
+  auto const [indexingRepoOK, indexingSystemlibDeclsOK] = coro::wait(
+    coro::collect(
+      indexPackage.index(indexUnit),
+      coro::invoke([&]() -> coro::Task<bool> {
+        if (RO::EvalEnableDecl) {
+          HPHP_CORO_RETURN(HPHP_CORO_AWAIT(
+            indexBuiltinSymbolDecls(indexUnit, executor, client)
+          ));
+        }
+        HPHP_CORO_RETURN(true);
+      })
+    )
+  );
 
-  if (!coro::wait(indexPackage.index(indexUnit))) return nullptr;
+  if (!indexingRepoOK || !indexingSystemlibDeclsOK) return nullptr;
 
   logPhaseStats("index", indexPackage, client, sample,
                 indexTimer.getMicroSeconds());

hphp/compiler/package.cpp

@@ -1115,19 +1115,7 @@ Package::UnitDecls IndexJob::run(
 
   // Get Facts from Decls, then populate IndexMeta.
   auto facts = hackc::decls_to_facts_cpp_ffi(decls, "");
-  Package::IndexMeta summary;
-  for (auto& e : facts.facts.types) {
-    summary.types.emplace_back(makeStaticString(std::string(e.name)));
-  }
-  for (auto& e : facts.facts.functions) {
-    summary.funcs.emplace_back(makeStaticString(std::string(e)));
-  }
-  for (auto& e : facts.facts.constants) {
-    summary.constants.emplace_back(makeStaticString(std::string(e)));
-  }
-  for (auto& e : facts.facts.modules) {
-    summary.modules.emplace_back(makeStaticString(std::string(e.name)));
-  }
+  auto summary = summary_of_facts(facts);
   s_indexMetas.emplace_back(summary);
   if (!RO::EvalEnableDecl) {
     // If decl-directed bytecode is disabled, parseRun() will not need
@@ -1518,3 +1506,20 @@ coro::Task<Package::FileAndSizeVec> Package::emitGroup(
 
   HPHP_CORO_RETURN(FileAndSizeVec{});
 }
+
+Package::IndexMeta HPHP::summary_of_facts(const hackc::FactsResult& facts) {
+  Package::IndexMeta summary;
+  for (auto& e : facts.facts.types) {
+    summary.types.emplace_back(makeStaticString(std::string(e.name)));
+  }
+  for (auto& e : facts.facts.functions) {
+    summary.funcs.emplace_back(makeStaticString(std::string(e)));
+  }
+  for (auto& e : facts.facts.constants) {
+    summary.constants.emplace_back(makeStaticString(std::string(e)));
+  }
+  for (auto& e : facts.facts.modules) {
+    summary.modules.emplace_back(makeStaticString(std::string(e.name)));
+  }
+  return summary;
+}

hphp/compiler/package.h

@@ -38,6 +38,10 @@
 namespace HPHP {
 ///////////////////////////////////////////////////////////////////////////////
 
+namespace hackc {
+  struct FactsResult;
+}
+
 struct UnitIndex;
 
 struct Package {
@@ -371,5 +375,9 @@ struct UnitIndex final {
   Map modules;
 };
 
+// Given the result of running `hackc::decls_to_facts_cpp_ffi`, create a
+// `Package::IndexMeta` containing the names of all decls in `facts`.
+Package::IndexMeta summary_of_facts(const hackc::FactsResult& facts);
+
 ///////////////////////////////////////////////////////////////////////////////
 }

hphp/runtime/vm/builtin-symbol-map.cpp

@@ -123,4 +123,22 @@ Optional<hackc::ExternalDeclProviderResult> getBuiltinDecls(
   not_reached();
 }
 
+hphp_fast_set<const hackc::DeclResult*> getAllBuiltinDecls() {
+  assertx(RuntimeOption::EvalEnableDecl);
+  hphp_fast_set<const hackc::DeclResult*> res;
+  for (const auto& it: s_types) {
+    res.emplace(it.second.get());
+  }
+  for (const auto& it: s_functions) {
+    res.emplace(it.second.get());
+  }
+  for (const auto& it: s_constants) {
+    res.emplace(it.second.get());
+  }
+  for (const auto& it: s_modules) {
+    res.emplace(it.second.get());
+  }
+  return res;
+}
+
 } // namespace HPHP::Native

hphp/runtime/vm/builtin-symbol-map.h

@@ -16,14 +16,17 @@
 
 #pragma once
 
-#include "hphp/hack/src/hackc/ffi_bridge/decl_provider.h"
-
 #include "hphp/runtime/base/autoload-map.h"
 
 #include "hphp/util/optional.h"
 
 #include <string>
 
+namespace HPHP::hackc {
+  struct ExternalDeclProviderResult;
+  struct DeclResult;
+}
+
 namespace HPHP::Native {
 
 /**
@@ -48,4 +51,6 @@ Optional<hackc::ExternalDeclProviderResult> getBuiltinDecls(
   AutoloadMap::KindOf kind
 );
 
+hphp_fast_set<const hackc::DeclResult*> getAllBuiltinDecls();
+
 } // namespace HPHP::Native