[cxxmodules] Implement global module indexing to improve performance.

The global module index represents an efficient on-disk hash table which stores
identifier->module mapping. Every time clang finds a unknown identifier we
are informed and we can load the corresponding module on demand.

This way we can provide minimal set of loaded modules. Currently, we see that
for hsimple.C only the half of the modules are loaded. This can be further
improved because we currently load all modules which have an identifier, that is
when looking for (for example TPad) we will load all modules which have the
identifier TPad, including modules which contain only a forward declaration of
it.

Kudos Arpitha Raghunandan (arpi-r)!

Author: vgvassilev

core/metacling/src/TCling.cxx

@@ -95,9 +95,11 @@ clang/LLVM technology.
 #include "clang/Lex/HeaderSearch.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Lex/PreprocessorOptions.h"
+#include "clang/Parse/Parser.h"
 #include "clang/Sema/Lookup.h"
 #include "clang/Sema/Sema.h"
-#include "clang/Parse/Parser.h"
+#include "clang/Serialization/ASTReader.h"
+#include "clang/Serialization/GlobalModuleIndex.h"
 
 #include "cling/Interpreter/ClangInternalState.h"
 #include "cling/Interpreter/DynamicLibraryManager.h"
@@ -1194,6 +1196,51 @@ static void RegisterPreIncludedHeaders(cling::Interpreter &clingInterp)
    clingInterp.declare(PreIncludes);
 }
 
+static bool HaveFullGlobalModuleIndex = false;
+static GlobalModuleIndex *loadGlobalModuleIndex(cling::Interpreter &interp, SourceLocation TriggerLoc)
+{
+   CompilerInstance &CI = *interp.getCI();
+   Preprocessor &PP = CI.getPreprocessor();
+   auto ModuleManager = CI.getModuleManager();
+   assert(ModuleManager);
+   // StringRef ModuleIndexPath = HSI.getModuleCachePath();
+   // HeaderSearch& HSI = PP.getHeaderSearchInfo();
+   // HSI.setModuleCachePath(TROOT::GetLibDir().Data());
+   std::string ModuleIndexPath = TROOT::GetLibDir().Data();
+   if (ModuleIndexPath.empty())
+      return nullptr;
+   // Get an existing global index. This loads it if not already loaded.
+   ModuleManager->resetForReload();
+   ModuleManager->loadGlobalIndex();
+   GlobalModuleIndex *GlobalIndex = ModuleManager->getGlobalIndex();
+   if (!GlobalIndex && CI.hasFileManager()) {
+   }
+
+   // For finding modules needing to be imported for fixit messages,
+   // we need to make the global index cover all modules, so we do that here.
+   if (!GlobalIndex && !HaveFullGlobalModuleIndex) {
+      ModuleMap &MMap = PP.getHeaderSearchInfo().getModuleMap();
+      bool RecreateIndex = false;
+      for (ModuleMap::module_iterator I = MMap.module_begin(), E = MMap.module_end(); I != E; ++I) {
+         Module *TheModule = I->second;
+         // We do want the index only of the prebuilt modules
+         std::string ModuleName = GetModuleNameAsString(TheModule, PP);
+         if (ModuleName.empty())
+            continue;
+         LoadModule(ModuleName, interp);
+         RecreateIndex = true;
+      }
+      if (RecreateIndex) {
+         GlobalModuleIndex::writeIndex(CI.getFileManager(), CI.getPCHContainerReader(), ModuleIndexPath);
+         ModuleManager->resetForReload();
+         ModuleManager->loadGlobalIndex();
+         GlobalIndex = ModuleManager->getGlobalIndex();
+      }
+      HaveFullGlobalModuleIndex = true;
+   }
+   return GlobalIndex;
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 /// Initialize the cling interpreter interface.
 /// \param argv - array of arguments passed to the cling::Interpreter constructor
@@ -1325,6 +1372,7 @@ TCling::TCling(const char *name, const char *title, const char* const argv[])
 
          clingArgsStorage.push_back("-fmodule-map-file=" + ModuleMapLoc);
       }
+      clingArgsStorage.push_back("-fmodules-cache-path=" + std::string(TROOT::GetLibDir()));
    }
 
    std::vector<const char*> interpArgs;
@@ -1401,7 +1449,34 @@ TCling::TCling(const char *name, const char *title, const char* const argv[])
    static llvm::raw_fd_ostream fMPOuts (STDOUT_FILENO, /*ShouldClose*/false);
    fMetaProcessor = llvm::make_unique<cling::MetaProcessor>(*fInterpreter, fMPOuts);
 
-   RegisterCxxModules(*fInterpreter);
+   if (fInterpreter->getCI()->getLangOpts().Modules) {
+      // Setup core C++ modules if we have any to setup.
+
+      // Load libc and stl first.
+#ifdef R__MACOSX
+      LoadModules({"Darwin", "std"}, *fInterpreter);
+#else
+      LoadModules({"libc", "stl"}, *fInterpreter);
+#endif
+
+      if (!fromRootCling)
+         loadGlobalModuleIndex(*fInterpreter, SourceLocation());
+
+      // C99 decided that it's a very good idea to name a macro `I` (the letter I).
+      // This seems to screw up nearly all the template code out there as `I` is
+      // common template parameter name and iterator variable name.
+      // Let's follow the GCC recommendation and undefine `I` in case any of the
+      // core modules have defined it:
+      // https://www.gnu.org/software/libc/manual/html_node/Complex-Numbers.html
+      fInterpreter->declare("#ifdef I\n #undef I\n #endif\n");
+
+      // These macros are from loading R related modules, which conflict with
+      // user's code.
+      fInterpreter->declare("#ifdef PI\n #undef PI\n #endif\n");
+      fInterpreter->declare("#ifdef ERROR\n #undef ERROR\n #endif\n");
+   }
+
+   // RegisterCxxModules(*fInterpreter);
    RegisterPreIncludedHeaders(*fInterpreter);
 
    // We are now ready (enough is loaded) to init the list of opaque typedefs.

core/metacling/src/TClingCallbacks.cxx

@@ -29,6 +29,8 @@
 #include "clang/Parse/Parser.h"
 #include "clang/Sema/Lookup.h"
 #include "clang/Sema/Scope.h"
+#include "clang/Serialization/ASTReader.h"
+#include "clang/Serialization/GlobalModuleIndex.h"
 
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
@@ -285,6 +287,28 @@ bool TClingCallbacks::LookupObject(LookupResult &R, Scope *S) {
    return tryResolveAtRuntimeInternal(R, S);
 }
 
+static bool findInGlobalIndex(cling::Interpreter &Interp, DeclarationName Name, bool loadFirstMatchOnly = true)
+{
+   GlobalModuleIndex *Index = Interp.getCI()->getModuleManager()->getGlobalIndex();
+   if (!Index)
+      return false;
+
+   GlobalModuleIndex::FileNameHitSet FoundModules;
+
+   // Find the modules that reference the identifier.
+   // Note that this only finds top-level modules.
+   if (Index->lookupIdentifier(Name.getAsString(), FoundModules)) {
+      for (auto FileName : FoundModules) {
+         StringRef ModuleName = llvm::sys::path::stem(*FileName);
+         Interp.loadModule(ModuleName);
+         if (loadFirstMatchOnly)
+            break;
+      }
+      return true;
+   }
+   return false;
+}
+
 bool TClingCallbacks::LookupObject(const DeclContext* DC, DeclarationName Name) {
    if (!fROOTSpecialNamespace) {
       // init error or rootcling
@@ -293,9 +317,21 @@ bool TClingCallbacks::LookupObject(const DeclContext* DC, DeclarationName Name)
 
    if (!IsAutoLoadingEnabled() || fIsAutoLoadingRecursively) return false;
 
-   if (Name.getNameKind() != DeclarationName::Identifier) return false;
+   // We are currently building a module, we should not autoload.
+   Sema &SemaR = m_Interpreter->getSema();
+   const LangOptions &LangOpts = SemaR.getPreprocessor().getLangOpts();
+   if (LangOpts.Modules) {
+      if (LangOpts.isCompilingModule())
+         return false;
 
+      // FIXME: We should load only the first available and rely on other callbacks
+      // such as RequireCompleteType and LookupUnqualified to load all.
+      if (findInGlobalIndex(*m_Interpreter, Name, /*loadFirstMatchOnly*/ false))
+         return true;
+   }
 
+   if (Name.getNameKind() != DeclarationName::Identifier)
+      return false;
 
    // Get the 'lookup' decl context.
    // We need to cast away the constness because we will lookup items of this
@@ -311,7 +347,6 @@ bool TClingCallbacks::LookupObject(const DeclContext* DC, DeclarationName Name)
    if (primaryDC != DC)
       return false;
 
-   Sema &SemaR = m_Interpreter->getSema();
    LookupResult R(SemaR, Name, SourceLocation(), Sema::LookupOrdinaryName);
    R.suppressDiagnostics();
    // We need the qualified name for TCling to find the right library.

interpreter/cling/lib/Interpreter/CIFactory.cpp

@@ -547,17 +547,15 @@ namespace {
     for (StringRef ModulePath : Paths) {
       // FIXME: If we have a prebuilt module path that is equal to our module
       // cache we fail to compile the clang builtin modules for some reason.
-      // This can't be reproduced in clang, so I assume we have some strange
-      // error in our interpreter setup where this is causing errors (or maybe
-      // clang is doing the same check in some hidden place).
-      // The error looks like this:
-      //   .../include/stddef.h error: unknown type name '__PTRDIFF_TYPE__'
-      //   typedef __PTRDIFF_TYPE__ ptrdiff_t;
-      //   <similar follow up errors>
+      // This makes clang to think it failed to build a dependency module, i.e.
+      // if we are building module C, clang goes off and builds B and A first.
+      // If the module cache points to the same location as the prebuilt module
+      // path, clang errors out on building module A, however, it builds it.
+      // Next time we run, it will build module B and issue diagnostics.
+      // If we run third time, it'd build successfully C and continue.
       // For now it is fixed by just checking those two paths are not identical.
-      if (normalizePath(ModulePath) != normalizePath(Opts.ModuleCachePath)) {
+      if (normalizePath(ModulePath) != normalizePath(Opts.ModuleCachePath))
         Opts.AddPrebuiltModulePath(ModulePath);
-      }
     }
   }
 

interpreter/cling/lib/Interpreter/Interpreter.cpp

@@ -873,10 +873,32 @@ namespace cling {
     if (getSema().isModuleVisible(M))
       return true;
 
+    // We cannot use #pragma clang module import because the on-demand modules
+    // may load a module in the middle of a function body for example. In this
+    // case this triggers an error:
+    // fatal error: import of module '...' appears within function '...'
+    //
+    // if (declare("#pragma clang module import \"" + M->Name + "\"") ==
+    // kSuccess)
+    //   return true;
+
     // FIXME: What about importing submodules such as std.blah. This disables
     // this functionality.
-    if (declare("#pragma clang module import \"" + M->Name + "\"") == kSuccess)
-      return true;
+    Preprocessor& PP = getCI()->getPreprocessor();
+    IdentifierInfo* II = PP.getIdentifierInfo(M->Name);
+    SourceLocation ValidLoc = M->DefinitionLoc;
+    Interpreter::PushTransactionRAII RAII(this);
+    bool success = !getCI()
+                        ->getSema()
+                        .ActOnModuleImport(ValidLoc, ValidLoc,
+                                           std::make_pair(II, ValidLoc))
+                        .isInvalid();
+
+    if (success) {
+      // Also make the module visible in the preprocessor to export its macros.
+      PP.makeModuleVisible(M, ValidLoc);
+      return success;
+    }
 
     if (complain) {
       if (M->IsSystem)

interpreter/llvm/src/tools/clang/include/clang/Serialization/GlobalModuleIndex.h

@@ -179,6 +179,9 @@ class GlobalModuleIndex {
   /// \returns true if the identifier is known to the index, false otherwise.
   bool lookupIdentifier(StringRef Name, HitSet &Hits);
 
+  typedef llvm::SmallPtrSet<std::string *, 4> FileNameHitSet;
+  bool lookupIdentifier(StringRef Name, FileNameHitSet &Hits);
+
   /// \brief Note that the given module file has been loaded.
   ///
   /// \returns false if the global module index has information about this

interpreter/llvm/src/tools/clang/lib/Sema/SemaDecl.cpp

@@ -16196,7 +16196,7 @@ DeclResult Sema::ActOnModuleImport(SourceLocation StartLoc,
 
   VisibleModules.setVisible(Mod, ImportLoc);
 
-  checkModuleImportContext(*this, Mod, ImportLoc, CurContext);
+  // checkModuleImportContext(*this, Mod, ImportLoc, CurContext);
 
   // FIXME: we should support importing a submodule within a different submodule
   // of the same top-level module. Until we do, make it an error rather than

interpreter/llvm/src/tools/clang/lib/Serialization/GlobalModuleIndex.cpp

@@ -313,6 +313,32 @@ bool GlobalModuleIndex::lookupIdentifier(StringRef Name, HitSet &Hits) {
   return true;
 }
 
+bool GlobalModuleIndex::lookupIdentifier(StringRef Name, FileNameHitSet &Hits) {
+  Hits.clear();
+
+  // If there's no identifier index, there is nothing we can do.
+  if (!IdentifierIndex)
+    return false;
+
+  // Look into the identifier index.
+  ++NumIdentifierLookups;
+  IdentifierIndexTable &Table =
+      *static_cast<IdentifierIndexTable *>(IdentifierIndex);
+  IdentifierIndexTable::iterator Known = Table.find(Name);
+  if (Known == Table.end()) {
+    return true;
+  }
+
+  SmallVector<unsigned, 2> ModuleIDs = *Known;
+  for (unsigned I = 0, N = ModuleIDs.size(); I != N; ++I) {
+    assert(!Modules[ModuleIDs[I]].FileName.empty());
+    Hits.insert(&Modules[ModuleIDs[I]].FileName);
+  }
+
+  ++NumIdentifierLookupHits;
+  return true;
+}
+
 bool GlobalModuleIndex::loadedModuleFile(ModuleFile *File) {
   // Look for the module in the global module index based on the module name.
   StringRef Name = File->ModuleName;
@@ -660,10 +686,7 @@ bool GlobalModuleIndexBuilder::loadModuleFile(const FileEntry *File) {
                                                      DEnd = Table->data_end();
            D != DEnd; ++D) {
         std::pair<StringRef, bool> Ident = *D;
-        if (Ident.second)
-          InterestingIdentifiers[Ident.first].push_back(ID);
-        else
-          (void)InterestingIdentifiers[Ident.first];
+        InterestingIdentifiers[Ident.first].push_back(ID);
       }
     }
 
@@ -725,14 +748,14 @@ bool GlobalModuleIndexBuilder::writeIndex(llvm::BitstreamWriter &Stream) {
   for (auto MapEntry : ImportedModuleFiles) {
     auto *File = MapEntry.first;
     ImportedModuleFileInfo &Info = MapEntry.second;
-    if (getModuleFileInfo(File).Signature) {
-      if (getModuleFileInfo(File).Signature != Info.StoredSignature)
-        // Verify Signature.
-        return true;
-    } else if (Info.StoredSize != File->getSize() ||
-               Info.StoredModTime != File->getModificationTime())
-      // Verify Size and ModTime.
-      return true;
+    // if (getModuleFileInfo(File).Signature) {
+    //   if (getModuleFileInfo(File).Signature != Info.StoredSignature)
+    //     // Verify Signature.
+    //     return true;
+    // } else if (Info.StoredSize != File->getSize() ||
+    //            Info.StoredModTime != File->getModificationTime())
+    //   // Verify Size and ModTime.
+    //   return true;
   }
 
   using namespace llvm;