[cxxmodules] Fix failing runtime_cxxmodules tests by preloading modules

core/clingutils/src/TClingUtils.cxx

@@ -3383,6 +3383,11 @@ void ROOT::TMetaUtils::GetFullyQualifiedTypeName(std::string &typenamestr,
                                                  const clang::QualType &qtype,
                                                  const cling::Interpreter &interpreter)
 {
+   // We need this barrior because GetFullyQualifiedTypeName is triggering deserialization
+   // This calling the same name function GetFullyQualifiedTypeName, but this should stay here because
+   // callee doesn't have an interpreter pointer
+   cling::Interpreter::PushTransactionRAII RAII(const_cast<cling::Interpreter*>(&interpreter));
+
    GetFullyQualifiedTypeName(typenamestr,
                              qtype,
                              interpreter.getCI()->getASTContext());

core/metacling/src/TCling.cxx

@@ -429,6 +429,10 @@ static void TCling__UpdateClassInfo(const NamedDecl* TD)
 void TCling::UpdateEnumConstants(TEnum* enumObj, TClass* cl) const {
    const clang::Decl* D = static_cast<const clang::Decl*>(enumObj->GetDeclId());
    if(const clang::EnumDecl* ED = dyn_cast<clang::EnumDecl>(D)) {
+
+      // clang::EnumDecl::enumerator_begin can triggering deserialization
+      cling::Interpreter::PushTransactionRAII deserRAII(fInterpreter);
+
       // Add the constants to the enum type.
       for (EnumDecl::enumerator_iterator EDI = ED->enumerator_begin(),
                 EDE = ED->enumerator_end(); EDI != EDE; ++EDI) {
@@ -626,6 +630,26 @@ extern "C" int TCling__AutoLoadCallback(const char* className)
    return ((TCling*)gCling)->AutoLoad(className);
 }
 
+extern "C" int TCling__AutoLoadLibraryForModules(const char* StemName)
+{
+   // FIXME: We're excluding some libraries from autoloading. Adding annotations to
+   // pcms from LinkDef files would fix this workaround.
+   static constexpr std::array<const char*, 4> excludelibs = { {"RooStats",
+      "RooFitCore", "RooFit", "HistFactory"} };
+   if (std::find(excludelibs.begin(), excludelibs.end(), StemName)  != excludelibs.end())
+      return -1;
+
+   // Add lib prefix
+   TString LibName("lib" + std::string(StemName));
+   // Construct the actual library name from the stem name.
+   // Eg. FindDynamicLibrary("libCore") returns "/path/to/libCore.so"
+   const char *Name = gSystem->FindDynamicLibrary(LibName, kTRUE);
+
+   if (!Name || ((TCling*)gCling)->IsLoaded(Name)) return 1;
+
+   return ((TCling*)gCling)->Load(Name);
+}
+
 extern "C" int TCling__AutoParseCallback(const char* className)
 {
    return ((TCling*)gCling)->AutoParse(className);
@@ -1114,6 +1138,21 @@ static bool IsFromRootCling() {
   return foundSymbol;
 }
 
+static std::string GetModuleNameAsString(clang::Module *M, const clang::Preprocessor &PP)
+{
+   const HeaderSearchOptions &HSOpts = PP.getHeaderSearchInfo().getHeaderSearchOpts();
+
+   std::string ModuleFileName;
+   if (!HSOpts.PrebuiltModulePaths.empty())
+      // Load the module from the prebuilt module path.
+      ModuleFileName = PP.getHeaderSearchInfo().getModuleFileName(M->Name, "", /*UsePrebuiltPath*/ true);
+   if (ModuleFileName.empty()) return "";
+
+   std::string ModuleName = llvm::sys::path::filename(ModuleFileName);
+   // Return stem of the filename
+   return std::string(llvm::sys::path::stem(ModuleName));
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 /// Initialize the cling interpreter interface.
 
@@ -1258,9 +1297,41 @@ TCling::TCling(const char *name, const char *title)
       // Load libc and stl first.
       LoadModules({"libc", "stl"}, *fInterpreter);
 
-      LoadModules({"ROOT_Foundation_C", "ROOT_Config", "ROOT_Foundation_Stage1_NoRTTI", "Core", "RIO"}, *fInterpreter);
-      if (!fromRootCling)
-         LoadModules({"GenVector", "MultiProc", "TreePlayer", "Hist", "TreePlayer", "ROOTDataFrame", "ROOTVecOps"}, *fInterpreter);
+      // Load core modules
+      // This should be vector in order to be able to pass it to LoadModules
+      std::vector<std::string> CoreModules = {"ROOT_Foundation_C","ROOT_Config",
+         "ROOT_Foundation_Stage1_NoRTTI", "Core", "RIO"};
+      // These modules contain global variables which conflict with users' code such as "PI".
+      // FIXME: Reducing those will let us be less dependent on rootmap files
+      static constexpr std::array<const char*, 4> ExcludeModules =
+         { { "Rtools", "RSQLite", "RInterface", "RMVA"} };
+
+      LoadModules(CoreModules, *fInterpreter);
+
+      // Take this branch only from ROOT because we don't need to preload modules in rootcling
+      if (!fromRootCling) {
+         // Dynamically get all the modules and load them if they are not in core modules
+         clang::CompilerInstance &CI = *fInterpreter->getCI();
+         clang::ModuleMap &moduleMap = CI.getPreprocessor().getHeaderSearchInfo().getModuleMap();
+         clang::Preprocessor &PP = CI.getPreprocessor();
+         std::vector<std::string> ModulesPreloaded;
+
+         for (auto I = moduleMap.module_begin(), E = moduleMap.module_end(); I != E; ++I) {
+            clang::Module *M = I->second;
+            assert(M);
+
+            std::string ModuleName = GetModuleNameAsString(M, PP);
+            if (!ModuleName.empty() &&
+                  std::find(CoreModules.begin(), CoreModules.end(), ModuleName) == CoreModules.end()
+                  && std::find(ExcludeModules.begin(), ExcludeModules.end(), ModuleName) == ExcludeModules.end()) {
+               if (M->IsSystem && !M->IsMissingRequirement)
+                  LoadModule(ModuleName, *fInterpreter);
+               else if (!M->IsSystem && !M->IsMissingRequirement)
+                  ModulesPreloaded.push_back(ModuleName);
+            }
+         }
+         LoadModules(ModulesPreloaded, *fInterpreter);
+      }
 
       // Check that the gROOT macro was exported by any core module.
       assert(fInterpreter->getMacro("gROOT") && "Couldn't load gROOT macro?");
@@ -1891,7 +1962,10 @@ void TCling::RegisterModule(const char* modulename,
       }
    }
 
-   if (gIgnoredPCMNames.find(modulename) == gIgnoredPCMNames.end()) {
+   // Don't do "PCM" optimization with runtime modules because we are loading libraries
+   // at decl deserialization time and it triggers infinite deserialization chain.
+   // In short, this optimization leads to infinite loop.
+   if (!hasCxxModule && gIgnoredPCMNames.find(modulename) == gIgnoredPCMNames.end()) {
       if (!LoadPCM(pcmFileName, headers, triggerFunc)) {
          ::Error("TCling::RegisterModule", "cannot find dictionary module %s",
                  ROOT::TMetaUtils::GetModuleFileName(modulename).c_str());

core/metacling/src/TClingBaseClassInfo.cxx

@@ -110,6 +110,8 @@ TClingBaseClassInfo::TClingBaseClassInfo(cling::Interpreter* interp,
    //CRD->isDerivedFrom(BaseCRD, Paths);
    // Check that base derives from derived.
    clang::CXXBasePaths Paths;
+   // isDerivedFrom can trigger deserialization
+   cling::Interpreter::PushTransactionRAII RAII(fInterp);
    if (!CRD->isDerivedFrom(BaseCRD, Paths)) {
       //Not valid fBaseInfo = 0.
       return;
@@ -258,6 +260,11 @@ int TClingBaseClassInfo::InternalNext(int onlyDirect)
          (fIter == llvm::dyn_cast<clang::CXXRecordDecl>(fDecl)->bases_end())) {
       return 0;
    }
+
+   // getASTRecordLayout() can trigger deserialization, and this should stay here
+   // instead of inside the while loop
+   cling::Interpreter::PushTransactionRAII RAII(fInterp);
+
    // Advance to the next valid base.
    while (1) {
       // Advance the iterator.
@@ -269,8 +276,6 @@ int TClingBaseClassInfo::InternalNext(int onlyDirect)
          // We previously processed a base class which itself has bases,
          // now we process the bases of that base class.
 
-         // At least getASTRecordLayout() might deserialize.
-         cling::Interpreter::PushTransactionRAII RAII(fInterp);
          fDescend = false;
          const clang::RecordType *Ty = fIter->getType()->
                                        getAs<clang::RecordType>();
@@ -504,6 +509,10 @@ long TClingBaseClassInfo::Property() const
 
    clang::CXXBasePaths Paths(/*FindAmbiguities=*/false, /*RecordPaths=*/true,
                              /*DetectVirtual=*/true);
+
+   // isDerivedFrom can trigger deserialization
+   cling::Interpreter::PushTransactionRAII RAII(fInterp);
+
    if (!CRD->isDerivedFrom(BaseCRD, Paths)) {
       // Error really unexpected here, because construction / iteration guarantees
       //inheritance;

core/metacling/src/TClingCallbacks.cxx

@@ -50,6 +50,7 @@ extern "C" {
    Decl* TCling__GetObjectDecl(TObject *obj);
    int TCling__AutoLoadCallback(const char* className);
    int TCling__AutoParseCallback(const char* className);
+   void TCling__AutoLoadLibraryForModules(const char* StemName);
    const char* TCling__GetClassSharedLibs(const char* className);
 //    int TCling__IsAutoLoadNamespaceCandidate(const char* name);
    int TCling__IsAutoLoadNamespaceCandidate(const clang::NamespaceDecl* name);
@@ -749,6 +750,29 @@ void TClingCallbacks::TransactionCommitted(const Transaction &T) {
    TCling__UpdateListsOnCommitted(T, m_Interpreter);
 }
 
+// Collect modules and put them into fPendingCxxModules at first run. Interpreter is not yet initialized at first run
+// but we need to use interpreter services when loading libraries.
+void TClingCallbacks::beforeExecuteTransaction(const Transaction &T) {
+
+  const std::vector<clang::Module*> &modules = T.getClangModules();
+
+  if (fFirstRun) {
+    for (auto M : modules)
+      fPendingCxxModules.push_back(M);
+    return;
+  }
+
+  if (!fPendingCxxModules.empty()) {
+    for (auto M : fPendingCxxModules)
+      TCling__AutoLoadLibraryForModules(M->Name.c_str());
+    fPendingCxxModules.clear();
+    return;
+  }
+
+  for (auto M : modules)
+    TCling__AutoLoadLibraryForModules(M->Name.c_str());
+}
+
 // The callback is used to update the list of globals in ROOT.
 //
 void TClingCallbacks::TransactionUnloaded(const Transaction &T) {

core/metacling/src/TClingCallbacks.h

@@ -12,6 +12,7 @@
 #include "cling/Interpreter/InterpreterCallbacks.h"
 
 #include <stack>
+#include <vector>
 
 namespace clang {
    class Decl;
@@ -27,6 +28,7 @@ namespace clang {
 namespace cling {
    class Interpreter;
    class Transaction;
+   class Module;
 }
 
 namespace llvm {
@@ -45,6 +47,9 @@ class TClingCallbacks : public cling::InterpreterCallbacks {
    bool fIsAutoParsingSuspended;
    bool fPPOldFlag;
    bool fPPChanged;
+   // This vector holds a clang cxxmodules where corresponding library should be loaded in Transaction
+   // afterwards.
+   std::vector<clang::Module*> fPendingCxxModules;
 public:
    TClingCallbacks(cling::Interpreter* interp);
 
@@ -83,6 +88,8 @@ class TClingCallbacks : public cling::InterpreterCallbacks {
    //
    virtual void TransactionCommitted(const cling::Transaction &T);
 
+   virtual void beforeExecuteTransaction(const cling::Transaction &T);
+
    // The callback is used to update the list of globals in ROOT.
    //
    virtual void TransactionUnloaded(const cling::Transaction &T);

core/metacling/src/TClingClassInfo.cxx

@@ -1267,6 +1267,10 @@ const char *TClingClassInfo::Name() const
    if (const NamedDecl* ND = llvm::dyn_cast<NamedDecl>(fDecl)) {
       PrintingPolicy Policy(fDecl->getASTContext().getPrintingPolicy());
       llvm::raw_string_ostream stream(buf);
+
+      // getNameForDiagnostic can trigger deserialization
+      cling::Interpreter::PushTransactionRAII RAII(fInterp);
+
       ND->getNameForDiagnostic(stream, Policy, /*Qualified=*/false);
    }
    return buf.c_str();

core/metacling/src/TClingDataMemberInfo.cxx

@@ -308,6 +308,9 @@ long TClingDataMemberInfo::Offset()
    const Decl *D = GetDecl();
    ASTContext& C = D->getASTContext();
    if (const FieldDecl *FldD = dyn_cast<FieldDecl>(D)) {
+      // getASTRecordLayout can trigger deserialization
+      cling::Interpreter::PushTransactionRAII RAII(fInterp);
+
       // The current member is a non-static data member.
       const clang::RecordDecl *RD = FldD->getParent();
       const clang::ASTRecordLayout &Layout = C.getASTRecordLayout(RD);

core/metacling/src/TClingTypeInfo.cxx

@@ -168,6 +168,10 @@ long TClingTypeInfo::Property() const
          // Note: Now we have class, struct, union only.
          const clang::CXXRecordDecl *CRD =
             llvm::dyn_cast<clang::CXXRecordDecl>(TD);
+
+         // isAbstract can trigger deserialization
+         cling::Interpreter::PushTransactionRAII RAII(fInterp);
+
          if (CRD->isClass()) {
             property |= kIsClass;
          }

interpreter/cling/include/cling/Interpreter/InterpreterCallbacks.h

@@ -132,6 +132,13 @@ namespace cling {
     ///
     virtual void TransactionCommitted(const Transaction&) {}
 
+    ///\brief This callback is invoked before a transaction is executed.
+    /// This event happens after a transaction was committed and LLVM IR was produced.
+    ///
+    ///\param[in] - The transaction to be executed.
+    ///
+    virtual void beforeExecuteTransaction(const Transaction&) {}
+
     ///\brief This callback is invoked whenever interpreter has reverted a
     /// transaction that has been fully committed.
     ///

interpreter/cling/include/cling/Interpreter/Transaction.h

@@ -31,6 +31,7 @@ namespace clang {
   class Preprocessor;
   struct PrintingPolicy;
   class Sema;
+  class Module;
 }
 
 namespace llvm {
@@ -115,6 +116,11 @@ namespace cling {
     typedef llvm::SmallVector<DelayCallInfo, 64> DeclQueue;
     typedef llvm::SmallVector<Transaction*, 2> NestedTransactions;
 
+    ///\brief The list of cxxmodules, which is collected by DeserializationListener
+    /// and will be used to load corresponding libraries.
+    ///
+    std::vector<clang::Module*> m_CxxModules;
+
     ///\brief All seen declarations, except the deserialized ones.
     /// If we collect the declarations by walking the clang::DeclContext we
     /// will miss the injected onces (eg. template instantiations).
@@ -268,6 +274,17 @@ namespace cling {
       return const_reverse_nested_iterator(0);
     }
 
+    void addClangModule(clang::Module* M) {
+      assert(M && "addClangModules: passed clang::Module pointer is null");
+
+      if (std::find(m_CxxModules.rbegin(), m_CxxModules.rend(), M) == m_CxxModules.rend())
+        m_CxxModules.push_back(M);
+    }
+
+    const std::vector<clang::Module*> &getClangModules() const {
+      return m_CxxModules;
+    }
+
     /// Macro iteration
     typedef MacroDirectiveInfoQueue::iterator macros_iterator;
     typedef MacroDirectiveInfoQueue::const_iterator const_macros_iterator;

interpreter/cling/lib/Interpreter/DeclCollector.cpp

@@ -321,4 +321,17 @@ namespace cling {
     m_Consumer->HandleCXXStaticMemberVarInstantiation(D);
   }
 
+  void DeclCollector::DeclRead(clang::serialization::DeclID, const clang::Decl *D) {
+    assertHasTransaction(m_CurTransaction);
+
+    assert(D && "Decl doesn't exist!");
+    if (!D->hasOwningModule()) return;
+
+    clang::Module *M = D->getOwningModule();
+    M = M->getTopLevelModule();
+
+    // Add interesting module to Transaction's m_cxxmodules; Corresponding library will be loaded.
+    m_CurTransaction->addClangModule(M);
+  }
+
 } // namespace cling

interpreter/cling/lib/Interpreter/DeclCollector.h

@@ -11,6 +11,7 @@
 #define CLING_DECL_COLLECTOR_H
 
 #include "clang/AST/ASTConsumer.h"
+#include "clang/Serialization/ASTDeserializationListener.h"
 
 #include "ASTTransformer.h"
 
@@ -24,6 +25,7 @@ namespace clang {
   class DeclGroupRef;
   class Preprocessor;
   class Token;
+  class Module;
 }
 
 namespace cling {
@@ -40,7 +42,7 @@ namespace cling {
   /// cling::DeclCollector is responsible for appending all the declarations
   /// seen by clang.
   ///
-  class DeclCollector : public clang::ASTConsumer {
+  class DeclCollector : public clang::ASTConsumer , public clang::ASTDeserializationListener {
     /// \brief PPCallbacks overrides/ Macro support
     class PPAdapter;
 
@@ -116,6 +118,11 @@ namespace cling {
 
     // dyn_cast/isa support
     static bool classof(const clang::ASTConsumer*) { return true; }
+    static bool classof(const clang::ASTDeserializationListener*) { return true; }
+
+    ///\brief ASTDeserializationListener function which gets callback when a decl is deserialized
+    void DeclRead(clang::serialization::DeclID, const clang::Decl *D) final;
+
   };
 } // namespace cling