diff --git a/core/clingutils/src/TClingUtils.cxx b/core/clingutils/src/TClingUtils.cxx index 9201753dcdecb..f486fc545fd22 100644 --- a/core/clingutils/src/TClingUtils.cxx +++ b/core/clingutils/src/TClingUtils.cxx @@ -3383,6 +3383,11 @@ void ROOT::TMetaUtils::GetFullyQualifiedTypeName(std::string &typenamestr, const clang::QualType &qtype, const cling::Interpreter &interpreter) { + // We need this barrior because GetFullyQualifiedTypeName is triggering deserialization + // This calling the same name function GetFullyQualifiedTypeName, but this should stay here because + // callee doesn't have an interpreter pointer + cling::Interpreter::PushTransactionRAII RAII(const_cast(&interpreter)); + GetFullyQualifiedTypeName(typenamestr, qtype, interpreter.getCI()->getASTContext()); diff --git a/core/metacling/src/TCling.cxx b/core/metacling/src/TCling.cxx index c4ee30d9ebba6..a1e8d837e14af 100644 --- a/core/metacling/src/TCling.cxx +++ b/core/metacling/src/TCling.cxx @@ -429,6 +429,10 @@ static void TCling__UpdateClassInfo(const NamedDecl* TD) void TCling::UpdateEnumConstants(TEnum* enumObj, TClass* cl) const { const clang::Decl* D = static_cast(enumObj->GetDeclId()); if(const clang::EnumDecl* ED = dyn_cast(D)) { + + // clang::EnumDecl::enumerator_begin can triggering deserialization + cling::Interpreter::PushTransactionRAII deserRAII(fInterpreter); + // Add the constants to the enum type. for (EnumDecl::enumerator_iterator EDI = ED->enumerator_begin(), EDE = ED->enumerator_end(); EDI != EDE; ++EDI) { @@ -626,6 +630,26 @@ extern "C" int TCling__AutoLoadCallback(const char* className) return ((TCling*)gCling)->AutoLoad(className); } +extern "C" int TCling__AutoLoadLibraryForModules(const char* StemName) +{ + // FIXME: We're excluding some libraries from autoloading. Adding annotations to + // pcms from LinkDef files would fix this workaround. + static constexpr std::array excludelibs = { {"RooStats", + "RooFitCore", "RooFit", "HistFactory"} }; + if (std::find(excludelibs.begin(), excludelibs.end(), StemName) != excludelibs.end()) + return -1; + + // Add lib prefix + TString LibName("lib" + std::string(StemName)); + // Construct the actual library name from the stem name. + // Eg. FindDynamicLibrary("libCore") returns "/path/to/libCore.so" + const char *Name = gSystem->FindDynamicLibrary(LibName, kTRUE); + + if (!Name || ((TCling*)gCling)->IsLoaded(Name)) return 1; + + return ((TCling*)gCling)->Load(Name); +} + extern "C" int TCling__AutoParseCallback(const char* className) { return ((TCling*)gCling)->AutoParse(className); @@ -1114,6 +1138,21 @@ static bool IsFromRootCling() { return foundSymbol; } +static std::string GetModuleNameAsString(clang::Module *M, const clang::Preprocessor &PP) +{ + const HeaderSearchOptions &HSOpts = PP.getHeaderSearchInfo().getHeaderSearchOpts(); + + std::string ModuleFileName; + if (!HSOpts.PrebuiltModulePaths.empty()) + // Load the module from the prebuilt module path. + ModuleFileName = PP.getHeaderSearchInfo().getModuleFileName(M->Name, "", /*UsePrebuiltPath*/ true); + if (ModuleFileName.empty()) return ""; + + std::string ModuleName = llvm::sys::path::filename(ModuleFileName); + // Return stem of the filename + return std::string(llvm::sys::path::stem(ModuleName)); +} + //////////////////////////////////////////////////////////////////////////////// /// Initialize the cling interpreter interface. @@ -1258,9 +1297,41 @@ TCling::TCling(const char *name, const char *title) // Load libc and stl first. LoadModules({"libc", "stl"}, *fInterpreter); - LoadModules({"ROOT_Foundation_C", "ROOT_Config", "ROOT_Foundation_Stage1_NoRTTI", "Core", "RIO"}, *fInterpreter); - if (!fromRootCling) - LoadModules({"GenVector", "MultiProc", "TreePlayer", "Hist", "TreePlayer", "ROOTDataFrame", "ROOTVecOps"}, *fInterpreter); + // Load core modules + // This should be vector in order to be able to pass it to LoadModules + std::vector CoreModules = {"ROOT_Foundation_C","ROOT_Config", + "ROOT_Foundation_Stage1_NoRTTI", "Core", "RIO"}; + // These modules contain global variables which conflict with users' code such as "PI". + // FIXME: Reducing those will let us be less dependent on rootmap files + static constexpr std::array ExcludeModules = + { { "Rtools", "RSQLite", "RInterface", "RMVA"} }; + + LoadModules(CoreModules, *fInterpreter); + + // Take this branch only from ROOT because we don't need to preload modules in rootcling + if (!fromRootCling) { + // Dynamically get all the modules and load them if they are not in core modules + clang::CompilerInstance &CI = *fInterpreter->getCI(); + clang::ModuleMap &moduleMap = CI.getPreprocessor().getHeaderSearchInfo().getModuleMap(); + clang::Preprocessor &PP = CI.getPreprocessor(); + std::vector ModulesPreloaded; + + for (auto I = moduleMap.module_begin(), E = moduleMap.module_end(); I != E; ++I) { + clang::Module *M = I->second; + assert(M); + + std::string ModuleName = GetModuleNameAsString(M, PP); + if (!ModuleName.empty() && + std::find(CoreModules.begin(), CoreModules.end(), ModuleName) == CoreModules.end() + && std::find(ExcludeModules.begin(), ExcludeModules.end(), ModuleName) == ExcludeModules.end()) { + if (M->IsSystem && !M->IsMissingRequirement) + LoadModule(ModuleName, *fInterpreter); + else if (!M->IsSystem && !M->IsMissingRequirement) + ModulesPreloaded.push_back(ModuleName); + } + } + LoadModules(ModulesPreloaded, *fInterpreter); + } // Check that the gROOT macro was exported by any core module. assert(fInterpreter->getMacro("gROOT") && "Couldn't load gROOT macro?"); @@ -1891,7 +1962,10 @@ void TCling::RegisterModule(const char* modulename, } } - if (gIgnoredPCMNames.find(modulename) == gIgnoredPCMNames.end()) { + // Don't do "PCM" optimization with runtime modules because we are loading libraries + // at decl deserialization time and it triggers infinite deserialization chain. + // In short, this optimization leads to infinite loop. + if (!hasCxxModule && gIgnoredPCMNames.find(modulename) == gIgnoredPCMNames.end()) { if (!LoadPCM(pcmFileName, headers, triggerFunc)) { ::Error("TCling::RegisterModule", "cannot find dictionary module %s", ROOT::TMetaUtils::GetModuleFileName(modulename).c_str()); diff --git a/core/metacling/src/TClingBaseClassInfo.cxx b/core/metacling/src/TClingBaseClassInfo.cxx index 1bba75265638c..9991a9682e626 100644 --- a/core/metacling/src/TClingBaseClassInfo.cxx +++ b/core/metacling/src/TClingBaseClassInfo.cxx @@ -110,6 +110,8 @@ TClingBaseClassInfo::TClingBaseClassInfo(cling::Interpreter* interp, //CRD->isDerivedFrom(BaseCRD, Paths); // Check that base derives from derived. clang::CXXBasePaths Paths; + // isDerivedFrom can trigger deserialization + cling::Interpreter::PushTransactionRAII RAII(fInterp); if (!CRD->isDerivedFrom(BaseCRD, Paths)) { //Not valid fBaseInfo = 0. return; @@ -258,6 +260,11 @@ int TClingBaseClassInfo::InternalNext(int onlyDirect) (fIter == llvm::dyn_cast(fDecl)->bases_end())) { return 0; } + + // getASTRecordLayout() can trigger deserialization, and this should stay here + // instead of inside the while loop + cling::Interpreter::PushTransactionRAII RAII(fInterp); + // Advance to the next valid base. while (1) { // Advance the iterator. @@ -269,8 +276,6 @@ int TClingBaseClassInfo::InternalNext(int onlyDirect) // We previously processed a base class which itself has bases, // now we process the bases of that base class. - // At least getASTRecordLayout() might deserialize. - cling::Interpreter::PushTransactionRAII RAII(fInterp); fDescend = false; const clang::RecordType *Ty = fIter->getType()-> getAs(); @@ -504,6 +509,10 @@ long TClingBaseClassInfo::Property() const clang::CXXBasePaths Paths(/*FindAmbiguities=*/false, /*RecordPaths=*/true, /*DetectVirtual=*/true); + + // isDerivedFrom can trigger deserialization + cling::Interpreter::PushTransactionRAII RAII(fInterp); + if (!CRD->isDerivedFrom(BaseCRD, Paths)) { // Error really unexpected here, because construction / iteration guarantees //inheritance; diff --git a/core/metacling/src/TClingCallbacks.cxx b/core/metacling/src/TClingCallbacks.cxx index 5a034cc5bdcac..9b18677c242ee 100644 --- a/core/metacling/src/TClingCallbacks.cxx +++ b/core/metacling/src/TClingCallbacks.cxx @@ -50,6 +50,7 @@ extern "C" { Decl* TCling__GetObjectDecl(TObject *obj); int TCling__AutoLoadCallback(const char* className); int TCling__AutoParseCallback(const char* className); + void TCling__AutoLoadLibraryForModules(const char* StemName); const char* TCling__GetClassSharedLibs(const char* className); // int TCling__IsAutoLoadNamespaceCandidate(const char* name); int TCling__IsAutoLoadNamespaceCandidate(const clang::NamespaceDecl* name); @@ -749,6 +750,29 @@ void TClingCallbacks::TransactionCommitted(const Transaction &T) { TCling__UpdateListsOnCommitted(T, m_Interpreter); } +// Collect modules and put them into fPendingCxxModules at first run. Interpreter is not yet initialized at first run +// but we need to use interpreter services when loading libraries. +void TClingCallbacks::beforeExecuteTransaction(const Transaction &T) { + + const std::vector &modules = T.getClangModules(); + + if (fFirstRun) { + for (auto M : modules) + fPendingCxxModules.push_back(M); + return; + } + + if (!fPendingCxxModules.empty()) { + for (auto M : fPendingCxxModules) + TCling__AutoLoadLibraryForModules(M->Name.c_str()); + fPendingCxxModules.clear(); + return; + } + + for (auto M : modules) + TCling__AutoLoadLibraryForModules(M->Name.c_str()); +} + // The callback is used to update the list of globals in ROOT. // void TClingCallbacks::TransactionUnloaded(const Transaction &T) { diff --git a/core/metacling/src/TClingCallbacks.h b/core/metacling/src/TClingCallbacks.h index a4524e4fdd29d..55caae110e87f 100644 --- a/core/metacling/src/TClingCallbacks.h +++ b/core/metacling/src/TClingCallbacks.h @@ -12,6 +12,7 @@ #include "cling/Interpreter/InterpreterCallbacks.h" #include +#include namespace clang { class Decl; @@ -27,6 +28,7 @@ namespace clang { namespace cling { class Interpreter; class Transaction; + class Module; } namespace llvm { @@ -45,6 +47,9 @@ class TClingCallbacks : public cling::InterpreterCallbacks { bool fIsAutoParsingSuspended; bool fPPOldFlag; bool fPPChanged; + // This vector holds a clang cxxmodules where corresponding library should be loaded in Transaction + // afterwards. + std::vector fPendingCxxModules; public: TClingCallbacks(cling::Interpreter* interp); @@ -83,6 +88,8 @@ class TClingCallbacks : public cling::InterpreterCallbacks { // virtual void TransactionCommitted(const cling::Transaction &T); + virtual void beforeExecuteTransaction(const cling::Transaction &T); + // The callback is used to update the list of globals in ROOT. // virtual void TransactionUnloaded(const cling::Transaction &T); diff --git a/core/metacling/src/TClingClassInfo.cxx b/core/metacling/src/TClingClassInfo.cxx index 35845964922e1..8d8e3cbac6d22 100644 --- a/core/metacling/src/TClingClassInfo.cxx +++ b/core/metacling/src/TClingClassInfo.cxx @@ -1267,6 +1267,10 @@ const char *TClingClassInfo::Name() const if (const NamedDecl* ND = llvm::dyn_cast(fDecl)) { PrintingPolicy Policy(fDecl->getASTContext().getPrintingPolicy()); llvm::raw_string_ostream stream(buf); + + // getNameForDiagnostic can trigger deserialization + cling::Interpreter::PushTransactionRAII RAII(fInterp); + ND->getNameForDiagnostic(stream, Policy, /*Qualified=*/false); } return buf.c_str(); diff --git a/core/metacling/src/TClingDataMemberInfo.cxx b/core/metacling/src/TClingDataMemberInfo.cxx index 9c8fa9980f42f..21c6a82f670b2 100644 --- a/core/metacling/src/TClingDataMemberInfo.cxx +++ b/core/metacling/src/TClingDataMemberInfo.cxx @@ -308,6 +308,9 @@ long TClingDataMemberInfo::Offset() const Decl *D = GetDecl(); ASTContext& C = D->getASTContext(); if (const FieldDecl *FldD = dyn_cast(D)) { + // getASTRecordLayout can trigger deserialization + cling::Interpreter::PushTransactionRAII RAII(fInterp); + // The current member is a non-static data member. const clang::RecordDecl *RD = FldD->getParent(); const clang::ASTRecordLayout &Layout = C.getASTRecordLayout(RD); diff --git a/core/metacling/src/TClingTypeInfo.cxx b/core/metacling/src/TClingTypeInfo.cxx index 10e5cc58465dd..f718dbd54b2e3 100644 --- a/core/metacling/src/TClingTypeInfo.cxx +++ b/core/metacling/src/TClingTypeInfo.cxx @@ -168,6 +168,10 @@ long TClingTypeInfo::Property() const // Note: Now we have class, struct, union only. const clang::CXXRecordDecl *CRD = llvm::dyn_cast(TD); + + // isAbstract can trigger deserialization + cling::Interpreter::PushTransactionRAII RAII(fInterp); + if (CRD->isClass()) { property |= kIsClass; } diff --git a/interpreter/cling/include/cling/Interpreter/InterpreterCallbacks.h b/interpreter/cling/include/cling/Interpreter/InterpreterCallbacks.h index 602bd016d657d..2103d5a335b88 100644 --- a/interpreter/cling/include/cling/Interpreter/InterpreterCallbacks.h +++ b/interpreter/cling/include/cling/Interpreter/InterpreterCallbacks.h @@ -132,6 +132,13 @@ namespace cling { /// virtual void TransactionCommitted(const Transaction&) {} + ///\brief This callback is invoked before a transaction is executed. + /// This event happens after a transaction was committed and LLVM IR was produced. + /// + ///\param[in] - The transaction to be executed. + /// + virtual void beforeExecuteTransaction(const Transaction&) {} + ///\brief This callback is invoked whenever interpreter has reverted a /// transaction that has been fully committed. /// diff --git a/interpreter/cling/include/cling/Interpreter/Transaction.h b/interpreter/cling/include/cling/Interpreter/Transaction.h index 454ec7986e6e8..680543d021991 100644 --- a/interpreter/cling/include/cling/Interpreter/Transaction.h +++ b/interpreter/cling/include/cling/Interpreter/Transaction.h @@ -31,6 +31,7 @@ namespace clang { class Preprocessor; struct PrintingPolicy; class Sema; + class Module; } namespace llvm { @@ -115,6 +116,11 @@ namespace cling { typedef llvm::SmallVector DeclQueue; typedef llvm::SmallVector NestedTransactions; + ///\brief The list of cxxmodules, which is collected by DeserializationListener + /// and will be used to load corresponding libraries. + /// + std::vector m_CxxModules; + ///\brief All seen declarations, except the deserialized ones. /// If we collect the declarations by walking the clang::DeclContext we /// will miss the injected onces (eg. template instantiations). @@ -268,6 +274,17 @@ namespace cling { return const_reverse_nested_iterator(0); } + void addClangModule(clang::Module* M) { + assert(M && "addClangModules: passed clang::Module pointer is null"); + + if (std::find(m_CxxModules.rbegin(), m_CxxModules.rend(), M) == m_CxxModules.rend()) + m_CxxModules.push_back(M); + } + + const std::vector &getClangModules() const { + return m_CxxModules; + } + /// Macro iteration typedef MacroDirectiveInfoQueue::iterator macros_iterator; typedef MacroDirectiveInfoQueue::const_iterator const_macros_iterator; diff --git a/interpreter/cling/lib/Interpreter/DeclCollector.cpp b/interpreter/cling/lib/Interpreter/DeclCollector.cpp index 24f43aa372c42..5275feb5ce8ab 100644 --- a/interpreter/cling/lib/Interpreter/DeclCollector.cpp +++ b/interpreter/cling/lib/Interpreter/DeclCollector.cpp @@ -321,4 +321,17 @@ namespace cling { m_Consumer->HandleCXXStaticMemberVarInstantiation(D); } + void DeclCollector::DeclRead(clang::serialization::DeclID, const clang::Decl *D) { + assertHasTransaction(m_CurTransaction); + + assert(D && "Decl doesn't exist!"); + if (!D->hasOwningModule()) return; + + clang::Module *M = D->getOwningModule(); + M = M->getTopLevelModule(); + + // Add interesting module to Transaction's m_cxxmodules; Corresponding library will be loaded. + m_CurTransaction->addClangModule(M); + } + } // namespace cling diff --git a/interpreter/cling/lib/Interpreter/DeclCollector.h b/interpreter/cling/lib/Interpreter/DeclCollector.h index 476f49d5fea54..c60c11d039b1e 100644 --- a/interpreter/cling/lib/Interpreter/DeclCollector.h +++ b/interpreter/cling/lib/Interpreter/DeclCollector.h @@ -11,6 +11,7 @@ #define CLING_DECL_COLLECTOR_H #include "clang/AST/ASTConsumer.h" +#include "clang/Serialization/ASTDeserializationListener.h" #include "ASTTransformer.h" @@ -24,6 +25,7 @@ namespace clang { class DeclGroupRef; class Preprocessor; class Token; + class Module; } namespace cling { @@ -40,7 +42,7 @@ namespace cling { /// cling::DeclCollector is responsible for appending all the declarations /// seen by clang. /// - class DeclCollector : public clang::ASTConsumer { + class DeclCollector : public clang::ASTConsumer , public clang::ASTDeserializationListener { /// \brief PPCallbacks overrides/ Macro support class PPAdapter; @@ -116,6 +118,11 @@ namespace cling { // dyn_cast/isa support static bool classof(const clang::ASTConsumer*) { return true; } + static bool classof(const clang::ASTDeserializationListener*) { return true; } + + ///\brief ASTDeserializationListener function which gets callback when a decl is deserialized + void DeclRead(clang::serialization::DeclID, const clang::Decl *D) final; + }; } // namespace cling diff --git a/interpreter/cling/lib/Interpreter/IncrementalParser.cpp b/interpreter/cling/lib/Interpreter/IncrementalParser.cpp index 6353f7b558cb6..ce322cc2de22f 100644 --- a/interpreter/cling/lib/Interpreter/IncrementalParser.cpp +++ b/interpreter/cling/lib/Interpreter/IncrementalParser.cpp @@ -44,6 +44,7 @@ #include "clang/Sema/Sema.h" #include "clang/Sema/SemaDiagnostic.h" #include "clang/Serialization/ASTWriter.h" +#include "clang/Serialization/ASTReader.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" @@ -226,6 +227,13 @@ namespace cling { Preprocessor& PP = m_CI->getPreprocessor(); DiagnosticsEngine& Diags = m_CI->getSema().getDiagnostics(); + ASTReader* Reader = m_CI->getModuleManager().get(); + assert(isa(m_Consumer)); + ASTDeserializationListener* Listener = cast(m_Consumer); + // FIXME: We should create a multiplexing deserialization listener if there is one already attached. + if (Reader && Listener && !Reader->getDeserializationListener()) + Reader->setDeserializationListener(Listener); + // Pull in PCH. const std::string& PCHFileName = m_CI->getInvocation().getPreprocessorOpts().ImplicitPCHInclude; diff --git a/interpreter/cling/lib/Interpreter/Interpreter.cpp b/interpreter/cling/lib/Interpreter/Interpreter.cpp index 22016dc626067..56c5c3ecf2d1d 100644 --- a/interpreter/cling/lib/Interpreter/Interpreter.cpp +++ b/interpreter/cling/lib/Interpreter/Interpreter.cpp @@ -1529,6 +1529,9 @@ namespace cling { assert(!isInSyntaxOnlyMode() && "Running on what?"); assert(T.getState() == Transaction::kCommitted && "Must be committed"); + if (InterpreterCallbacks* callbacks = getCallbacks()) + callbacks->beforeExecuteTransaction(T); + const std::shared_ptr& M = T.getModule(); if (!M) return Interpreter::kExeNoModule; diff --git a/interpreter/cling/lib/Interpreter/LookupHelper.cpp b/interpreter/cling/lib/Interpreter/LookupHelper.cpp index 4df32ba572a0f..b8fc0cd3eb3ca 100644 --- a/interpreter/cling/lib/Interpreter/LookupHelper.cpp +++ b/interpreter/cling/lib/Interpreter/LookupHelper.cpp @@ -1452,9 +1452,9 @@ namespace cling { DigestArgsInput inputEval; llvm::SmallVector GivenArgs; + Interpreter::PushTransactionRAII pushedT(Interp); if (!inputEval(GivenArgs,funcArgs,diagOnOff,P,Interp)) return 0; - Interpreter::PushTransactionRAII pushedT(Interp); return findFunction(foundDC, funcName, GivenArgs, objectIsConst, Context, Interp, functionSelector, diff --git a/interpreter/cling/lib/Interpreter/MultiplexInterpreterCallbacks.h b/interpreter/cling/lib/Interpreter/MultiplexInterpreterCallbacks.h index 40f6754e60235..3d98ae0418985 100644 --- a/interpreter/cling/lib/Interpreter/MultiplexInterpreterCallbacks.h +++ b/interpreter/cling/lib/Interpreter/MultiplexInterpreterCallbacks.h @@ -77,6 +77,12 @@ namespace cling { } } + void beforeExecuteTransaction(const Transaction& T) override { + for (auto&& cb : m_Callbacks) { + cb->beforeExecuteTransaction(T); + } + } + void TransactionUnloaded(const Transaction& T) override { for (auto&& cb : m_Callbacks) { cb->TransactionUnloaded(T); diff --git a/interpreter/llvm/src/tools/clang/include/clang/Serialization/ASTReader.h b/interpreter/llvm/src/tools/clang/include/clang/Serialization/ASTReader.h index ab6f4f00436c2..f515756bf2901 100644 --- a/interpreter/llvm/src/tools/clang/include/clang/Serialization/ASTReader.h +++ b/interpreter/llvm/src/tools/clang/include/clang/Serialization/ASTReader.h @@ -1555,6 +1555,10 @@ class ASTReader void setDeserializationListener(ASTDeserializationListener *Listener, bool TakeOwnership = false); + ASTDeserializationListener *getDeserializationListener() { + return DeserializationListener; + }; + /// \brief Determine whether this AST reader has a global index. bool hasGlobalIndex() const { return (bool)GlobalIndex; }