diff --git a/compiler-rt/lib/orc/macho_platform.cpp b/compiler-rt/lib/orc/macho_platform.cpp index f2cca8eb829a2..f3242a4b517c3 100644 --- a/compiler-rt/lib/orc/macho_platform.cpp +++ b/compiler-rt/lib/orc/macho_platform.cpp @@ -12,21 +12,25 @@ #include "macho_platform.h" #include "common.h" +#include "debug.h" #include "error.h" #include "wrapper_function_utils.h" +#include #include #include #include #include +#include #include +#define DEBUG_TYPE "macho_platform" + using namespace __orc_rt; using namespace __orc_rt::macho; // Declare function tags for functions in the JIT process. -ORC_RT_JIT_DISPATCH_TAG(__orc_rt_macho_get_initializers_tag) -ORC_RT_JIT_DISPATCH_TAG(__orc_rt_macho_get_deinitializers_tag) +ORC_RT_JIT_DISPATCH_TAG(__orc_rt_macho_push_initializers_tag) ORC_RT_JIT_DISPATCH_TAG(__orc_rt_macho_symbol_lookup_tag) // Objective-C types. @@ -66,147 +70,47 @@ extern "C" void swift_registerTypeMetadataRecords( namespace { -Error validatePointerSectionExtent(const char *SectionName, - const ExecutorAddrRange &SE) { - if (SE.size().getValue() % sizeof(uintptr_t)) { - std::ostringstream ErrMsg; - ErrMsg << std::hex << "Size of " << SectionName << " 0x" - << SE.Start.getValue() << " -- 0x" << SE.End.getValue() - << " is not a pointer multiple"; - return make_error(ErrMsg.str()); - } - return Error::success(); -} - -Error registerObjCSelectors( - const std::vector &ObjCSelRefsSections, - const MachOJITDylibInitializers &MOJDIs) { - - if (ORC_RT_UNLIKELY(!sel_registerName)) - return make_error("sel_registerName is not available"); - - for (const auto &ObjCSelRefs : ObjCSelRefsSections) { - - if (auto Err = validatePointerSectionExtent("__objc_selrefs", ObjCSelRefs)) - return Err; - - for (uintptr_t &SelEntry : ObjCSelRefs.toSpan()) { - const char *SelName = reinterpret_cast(SelEntry); - auto Sel = sel_registerName(SelName); - *reinterpret_cast(&SelEntry) = Sel; - } - } - - return Error::success(); -} - -Error registerObjCClasses( - const std::vector &ObjCClassListSections, - const MachOJITDylibInitializers &MOJDIs) { +struct MachOJITDylibDepInfo { + bool Sealed = false; + std::vector DepHeaders; +}; - if (ObjCClassListSections.empty()) - return Error::success(); +using MachOJITDylibDepInfoMap = + std::unordered_map; - if (ORC_RT_UNLIKELY(!objc_msgSend)) - return make_error("objc_msgSend is not available"); - if (ORC_RT_UNLIKELY(!objc_readClassPair)) - return make_error("objc_readClassPair is not available"); - - struct ObjCClassCompiled { - void *Metaclass; - void *Parent; - void *Cache1; - void *Cache2; - void *Data; - }; +} // anonymous namespace - auto *ImageInfo = - MOJDIs.ObjCImageInfoAddress.toPtr(); - auto ClassSelector = sel_registerName("class"); +namespace __orc_rt { - for (const auto &ObjCClassList : ObjCClassListSections) { +using SPSMachOObjectPlatformSectionsMap = + SPSSequence>; - if (auto Err = - validatePointerSectionExtent("__objc_classlist", ObjCClassList)) - return Err; +using SPSMachOJITDylibDepInfo = SPSTuple>; - for (uintptr_t ClassPtr : ObjCClassList.toSpan()) { - auto *Cls = reinterpret_cast(ClassPtr); - auto *ClassCompiled = reinterpret_cast(ClassPtr); - objc_msgSend(reinterpret_cast(ClassCompiled->Parent), ClassSelector); - auto Registered = objc_readClassPair(Cls, ImageInfo); +using SPSMachOJITDylibDepInfoMap = + SPSSequence>; - // FIXME: Improve diagnostic by reporting the failed class's name. - if (Registered != Cls) - return make_error("Unable to register Objective-C class"); - } +template <> +class SPSSerializationTraits { +public: + static size_t size(const MachOJITDylibDepInfo &JDI) { + return SPSMachOJITDylibDepInfo::AsArgList::size(JDI.Sealed, JDI.DepHeaders); } - return Error::success(); -} - -Error registerSwift5Protocols( - const std::vector &Swift5ProtocolSections, - const MachOJITDylibInitializers &MOJDIs) { - - if (ORC_RT_UNLIKELY(!Swift5ProtocolSections.empty() && - !swift_registerProtocols)) - return make_error("swift_registerProtocols is not available"); - - for (const auto &Swift5Protocols : Swift5ProtocolSections) - swift_registerProtocols( - Swift5Protocols.Start.toPtr(), - Swift5Protocols.End.toPtr()); - - return Error::success(); -} - -Error registerSwift5ProtocolConformances( - const std::vector &Swift5ProtocolConformanceSections, - const MachOJITDylibInitializers &MOJDIs) { - - if (ORC_RT_UNLIKELY(!Swift5ProtocolConformanceSections.empty() && - !swift_registerProtocolConformances)) - return make_error( - "swift_registerProtocolConformances is not available"); - - for (const auto &ProtoConfSec : Swift5ProtocolConformanceSections) - swift_registerProtocolConformances( - ProtoConfSec.Start.toPtr(), - ProtoConfSec.End.toPtr()); - - return Error::success(); -} - -Error registerSwift5Types(const std::vector &Sections, - const MachOJITDylibInitializers &MOJDIs) { - - if (ORC_RT_UNLIKELY(!Sections.empty() && !swift_registerTypeMetadataRecords)) - return make_error( - "swift_registerTypeMetadataRecords is not available"); - - for (const auto &Section : Sections) - swift_registerTypeMetadataRecords( - Section.Start.toPtr(), - Section.End.toPtr()); - return Error::success(); -} - -Error runModInits(const std::vector &ModInitsSections, - const MachOJITDylibInitializers &MOJDIs) { - - for (const auto &ModInits : ModInitsSections) { - if (auto Err = validatePointerSectionExtent("__mod_inits", ModInits)) - return Err; + static bool serialize(SPSOutputBuffer &OB, const MachOJITDylibDepInfo &JDI) { + return SPSMachOJITDylibDepInfo::AsArgList::serialize(OB, JDI.Sealed, + JDI.DepHeaders); + } - using InitFunc = void (*)(); - for (auto *Init : ModInits.toSpan()) - (*Init)(); + static bool deserialize(SPSInputBuffer &IB, MachOJITDylibDepInfo &JDI) { + return SPSMachOJITDylibDepInfo::AsArgList::deserialize(IB, JDI.Sealed, + JDI.DepHeaders); } +}; - return Error::success(); -} +} // namespace __orc_rt +namespace { struct TLVDescriptor { void *(*Thunk)(TLVDescriptor *) = nullptr; unsigned long Key = 0; @@ -222,11 +126,31 @@ class MachOPlatformRuntimeState { using AtExitsVector = std::vector; - struct PerJITDylibState { + struct JITDylibState { + std::string Name; void *Header = nullptr; - size_t RefCount = 0; - bool AllowReinitialization = false; + bool Sealed = false; + size_t LinkedAgainstRefCount = 0; + size_t DlRefCount = 0; + std::vector Deps; AtExitsVector AtExits; + const objc_image_info *ObjCImageInfo = nullptr; + std::vector> ModInitsSections; + std::vector> ModInitsSectionsNew; + std::vector> ObjCClassListSections; + std::vector> ObjCClassListSectionsNew; + std::vector> ObjCSelRefsSections; + std::vector> ObjCSelRefsSectionsNew; + std::vector> Swift5ProtoSections; + std::vector> Swift5ProtoSectionsNew; + std::vector> Swift5ProtosSections; + std::vector> Swift5ProtosSectionsNew; + std::vector> Swift5TypesSections; + std::vector> Swift5TypesSectionsNew; + + bool referenced() const { + return LinkedAgainstRefCount != 0 || DlRefCount != 0; + } }; public: @@ -243,8 +167,16 @@ class MachOPlatformRuntimeState { MachOPlatformRuntimeState(MachOPlatformRuntimeState &&) = delete; MachOPlatformRuntimeState &operator=(MachOPlatformRuntimeState &&) = delete; - Error registerThreadDataSection(span ThreadDataSec); - Error deregisterThreadDataSection(span ThreadDataSec); + Error registerJITDylib(std::string Name, void *Header); + Error deregisterJITDylib(void *Header); + Error registerThreadDataSection(span ThreadDataSection); + Error deregisterThreadDataSection(span ThreadDataSection); + Error registerObjectPlatformSections( + ExecutorAddr HeaderAddr, + std::vector> Secs); + Error deregisterObjectPlatformSections( + ExecutorAddr HeaderAddr, + std::vector> Secs); const char *dlerror(); void *dlopen(string_view Name, int Mode); @@ -252,6 +184,7 @@ class MachOPlatformRuntimeState { void *dlsym(void *DSOHandle, string_view Symbol); int registerAtExit(void (*F)(void *), void *Arg, void *DSOHandle); + void runAtExits(JITDylibState &JDS); void runAtExits(void *DSOHandle); /// Returns the base address of the section containing ThreadData. @@ -259,37 +192,34 @@ class MachOPlatformRuntimeState { getThreadDataSectionFor(const char *ThreadData); private: - PerJITDylibState *getJITDylibStateByHeaderAddr(void *DSOHandle); - PerJITDylibState *getJITDylibStateByName(string_view Path); - PerJITDylibState &getOrCreateJITDylibState(MachOJITDylibInitializers &MOJDIs); + JITDylibState *getJITDylibStateByHeader(void *DSOHandle); + JITDylibState *getJITDylibStateByName(string_view Path); Expected lookupSymbolInJITDylib(void *DSOHandle, string_view Symbol); - Expected - getJITDylibInitializersByName(string_view Path); - Expected dlopenInitialize(string_view Path, int Mode); - Error initializeJITDylib(MachOJITDylibInitializers &MOJDIs); + static Error registerObjCSelectors(JITDylibState &JDS); + static Error registerObjCClasses(JITDylibState &JDS); + static Error registerSwift5Protocols(JITDylibState &JDS); + static Error registerSwift5ProtocolConformances(JITDylibState &JDS); + static Error registerSwift5Types(JITDylibState &JDS); + static Error runModInits(JITDylibState &JDS); - static MachOPlatformRuntimeState *MOPS; + Expected dlopenImpl(string_view Path, int Mode); + Error dlopenFull(JITDylibState &JDS); + Error dlopenInitialize(JITDylibState &JDS, MachOJITDylibDepInfoMap &DepInfo); - using InitSectionHandler = - Error (*)(const std::vector &Sections, - const MachOJITDylibInitializers &MOJDIs); - const std::vector> InitSections = - {{"__DATA,__objc_selrefs", registerObjCSelectors}, - {"__DATA,__objc_classlist", registerObjCClasses}, - {"__TEXT,__swift5_protos", registerSwift5Protocols}, - {"__TEXT,__swift5_proto", registerSwift5ProtocolConformances}, - {"__TEXT,__swift5_types", registerSwift5Types}, - {"__DATA,__mod_init_func", runModInits}}; + Error dlcloseImpl(void *DSOHandle); + Error dlcloseDeinitialize(JITDylibState &JDS); + + static MachOPlatformRuntimeState *MOPS; // FIXME: Move to thread-state. std::string DLFcnError; std::recursive_mutex JDStatesMutex; - std::unordered_map JDStates; - std::unordered_map JDNameToHeader; + std::unordered_map JDStates; + std::unordered_map JDNameToHeader; std::mutex ThreadDataSectionsMutex; std::map ThreadDataSections; @@ -312,6 +242,57 @@ void MachOPlatformRuntimeState::destroy() { delete MOPS; } +Error MachOPlatformRuntimeState::registerJITDylib(std::string Name, + void *Header) { + ORC_RT_DEBUG({ + printdbg("Registering JITDylib %s: Header = %p\n", Name.c_str(), Header); + }); + std::lock_guard Lock(JDStatesMutex); + if (JDStates.count(Header)) { + std::ostringstream ErrStream; + ErrStream << "Duplicate JITDylib registration for header " << Header + << " (name = " << Name << ")"; + return make_error(ErrStream.str()); + } + if (JDNameToHeader.count(Name)) { + std::ostringstream ErrStream; + ErrStream << "Duplicate JITDylib registration for header " << Header + << " (header = " << Header << ")"; + return make_error(ErrStream.str()); + } + + auto &JDS = JDStates[Header]; + JDS.Name = std::move(Name); + JDS.Header = Header; + JDNameToHeader[JDS.Name] = Header; + return Error::success(); +} + +Error MachOPlatformRuntimeState::deregisterJITDylib(void *Header) { + std::lock_guard Lock(JDStatesMutex); + auto I = JDStates.find(Header); + if (I == JDStates.end()) { + std::ostringstream ErrStream; + ErrStream << "Attempted to deregister unrecognized header " << Header; + return make_error(ErrStream.str()); + } + + // Remove std::string construction once we can use C++20. + auto J = JDNameToHeader.find( + std::string(I->second.Name.data(), I->second.Name.size())); + assert(J != JDNameToHeader.end() && + "Missing JDNameToHeader entry for JITDylib"); + + ORC_RT_DEBUG({ + printdbg("Deregistering JITDylib %s: Header = %p\n", I->second.Name.c_str(), + Header); + }); + + JDNameToHeader.erase(J); + JDStates.erase(I); + return Error::success(); +} + Error MachOPlatformRuntimeState::registerThreadDataSection( span ThreadDataSection) { std::lock_guard Lock(ThreadDataSectionsMutex); @@ -337,31 +318,160 @@ Error MachOPlatformRuntimeState::deregisterThreadDataSection( return Error::success(); } -const char *MachOPlatformRuntimeState::dlerror() { return DLFcnError.c_str(); } +Error MachOPlatformRuntimeState::registerObjectPlatformSections( + ExecutorAddr HeaderAddr, + std::vector> Secs) { + ORC_RT_DEBUG({ + printdbg("MachOPlatform: Registering object sections for %p.\n", + HeaderAddr.toPtr()); + }); + + std::lock_guard Lock(JDStatesMutex); + auto *JDS = getJITDylibStateByHeader(HeaderAddr.toPtr()); + if (!JDS) { + std::ostringstream ErrStream; + ErrStream << "Could not register object platform sections for " + "unrecognized header " + << HeaderAddr.toPtr(); + return make_error(ErrStream.str()); + } + + for (auto &KV : Secs) { + // FIXME: Validate section ranges? + if (KV.first == "__DATA,__thread_data") { + if (auto Err = registerThreadDataSection(KV.second.toSpan())) + return Err; + } else if (KV.first == "__DATA,__objc_selrefs") + JDS->ObjCSelRefsSectionsNew.push_back(KV.second.toSpan()); + else if (KV.first == "__DATA,__objc_classlist") + JDS->ObjCClassListSectionsNew.push_back(KV.second.toSpan()); + else if (KV.first == "__TEXT,__swift5_protos") + JDS->Swift5ProtosSectionsNew.push_back(KV.second.toSpan()); + else if (KV.first == "__TEXT,__swift5_proto") + JDS->Swift5ProtoSectionsNew.push_back(KV.second.toSpan()); + else if (KV.first == "__TEXT,__swift5_types") + JDS->Swift5TypesSectionsNew.push_back(KV.second.toSpan()); + else if (KV.first == "__DATA,__mod_init_func") + JDS->ModInitsSectionsNew.push_back(KV.second.toSpan()); + else { + // Should this be a warning instead? + return make_error( + "Encountered unexpected section " + + std::string(KV.first.data(), KV.first.size()) + + " while registering object platform sections"); + } + } + + return Error::success(); +} + +// Remove the given range from the given vector if present. +// Returns true if the range was removed, false otherwise. +template +bool removeIfPresent(std::vector> &V, ExecutorAddrRange R) { + auto RI = std::find_if( + V.rbegin(), V.rend(), + [RS = R.toSpan()](const span &E) { return E.data() == RS.data(); }); + if (RI != V.rend()) { + V.erase(std::next(RI).base()); + return true; + } + return false; +} + +Error MachOPlatformRuntimeState::deregisterObjectPlatformSections( + ExecutorAddr HeaderAddr, + std::vector> Secs) { + // TODO: Make this more efficient? (maybe unnecessary if removal is rare?) + // TODO: Add a JITDylib prepare-for-teardown operation that clears all + // registered sections, causing this function to take the fast-path. + ORC_RT_DEBUG({ + printdbg("MachOPlatform: Registering object sections for %p.\n", + HeaderAddr.toPtr()); + }); -void *MachOPlatformRuntimeState::dlopen(string_view Path, int Mode) { std::lock_guard Lock(JDStatesMutex); + auto *JDS = getJITDylibStateByHeader(HeaderAddr.toPtr()); + if (!JDS) { + std::ostringstream ErrStream; + ErrStream << "Could not register object platform sections for unrecognized " + "header " + << HeaderAddr.toPtr(); + return make_error(ErrStream.str()); + } + + // FIXME: Implement faster-path by returning immediately if JDS is being + // torn down entirely? - // Use fast path if all JITDylibs are already loaded and don't require - // re-running initializers. - if (auto *JDS = getJITDylibStateByName(Path)) { - if (!JDS->AllowReinitialization) { - ++JDS->RefCount; - return JDS->Header; + for (auto &KV : Secs) { + // FIXME: Validate section ranges? + if (KV.first == "__DATA,__thread_data") { + if (auto Err = + deregisterThreadDataSection(KV.second.toSpan())) + return Err; + } else if (KV.first == "__DATA,__objc_selrefs") { + if (!removeIfPresent(JDS->ObjCSelRefsSections, KV.second)) + removeIfPresent(JDS->ObjCSelRefsSectionsNew, KV.second); + } else if (KV.first == "__DATA,__objc_classlist") { + if (!removeIfPresent(JDS->ObjCClassListSections, KV.second)) + removeIfPresent(JDS->ObjCClassListSectionsNew, KV.second); + } else if (KV.first == "__TEXT,__swift5_protos") { + if (!removeIfPresent(JDS->Swift5ProtosSections, KV.second)) + removeIfPresent(JDS->Swift5ProtosSectionsNew, KV.second); + } else if (KV.first == "__TEXT,__swift5_proto") { + if (!removeIfPresent(JDS->Swift5ProtoSections, KV.second)) + removeIfPresent(JDS->Swift5ProtoSectionsNew, KV.second); + } else if (KV.first == "__TEXT,__swift5_types") { + if (!removeIfPresent(JDS->Swift5TypesSections, KV.second)) + removeIfPresent(JDS->Swift5TypesSectionsNew, KV.second); + } else if (KV.first == "__DATA,__mod_init_func") { + if (!removeIfPresent(JDS->ModInitsSections, KV.second)) + removeIfPresent(JDS->ModInitsSectionsNew, KV.second); + } else { + // Should this be a warning instead? + return make_error( + "Encountered unexpected section " + + std::string(KV.first.data(), KV.first.size()) + + " while deregistering object platform sections"); } } + return Error::success(); +} - auto H = dlopenInitialize(Path, Mode); - if (!H) { +const char *MachOPlatformRuntimeState::dlerror() { return DLFcnError.c_str(); } + +void *MachOPlatformRuntimeState::dlopen(string_view Path, int Mode) { + ORC_RT_DEBUG({ + std::string S(Path.data(), Path.size()); + printdbg("MachOPlatform::dlopen(\"%s\")\n", S.c_str()); + }); + std::lock_guard Lock(JDStatesMutex); + if (auto H = dlopenImpl(Path, Mode)) + return *H; + else { + // FIXME: Make dlerror thread safe. DLFcnError = toString(H.takeError()); return nullptr; } - - return *H; } int MachOPlatformRuntimeState::dlclose(void *DSOHandle) { - runAtExits(DSOHandle); + ORC_RT_DEBUG({ + auto *JDS = getJITDylibStateByHeader(DSOHandle); + std::string DylibName; + if (JDS) { + std::string S; + printdbg("MachOPlatform::dlclose(%p) (%s)\n", DSOHandle, S.c_str()); + } else + printdbg("MachOPlatform::dlclose(%p) (%s)\n", DSOHandle, + "invalid handle"); + }); + std::lock_guard Lock(JDStatesMutex); + if (auto Err = dlcloseImpl(DSOHandle)) { + // FIXME: Make dlerror thread safe. + DLFcnError = toString(std::move(Err)); + return -1; + } return 0; } @@ -379,30 +489,39 @@ int MachOPlatformRuntimeState::registerAtExit(void (*F)(void *), void *Arg, void *DSOHandle) { // FIXME: Handle out-of-memory errors, returning -1 if OOM. std::lock_guard Lock(JDStatesMutex); - auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle); - assert(JDS && "JITDylib state not initialized"); + auto *JDS = getJITDylibStateByHeader(DSOHandle); + if (!JDS) { + ORC_RT_DEBUG({ + printdbg("MachOPlatformRuntimeState::registerAtExit called with " + "unrecognized dso handle %p\n", + DSOHandle); + }); + return -1; + } JDS->AtExits.push_back({F, Arg}); return 0; } -void MachOPlatformRuntimeState::runAtExits(void *DSOHandle) { - // FIXME: Should atexits be allowed to run concurrently with access to - // JDState? - AtExitsVector V; - { - std::lock_guard Lock(JDStatesMutex); - auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle); - assert(JDS && "JITDlybi state not initialized"); - std::swap(V, JDS->AtExits); - } - - while (!V.empty()) { - auto &AE = V.back(); +void MachOPlatformRuntimeState::runAtExits(JITDylibState &JDS) { + while (!JDS.AtExits.empty()) { + auto &AE = JDS.AtExits.back(); AE.Func(AE.Arg); - V.pop_back(); + JDS.AtExits.pop_back(); } } +void MachOPlatformRuntimeState::runAtExits(void *DSOHandle) { + std::lock_guard Lock(JDStatesMutex); + auto *JDS = getJITDylibStateByHeader(DSOHandle); + ORC_RT_DEBUG({ + printdbg("MachOPlatformRuntimeState::runAtExits called on unrecognized " + "dso_handle %p\n", + DSOHandle); + }); + if (JDS) + runAtExits(*JDS); +} + Expected> MachOPlatformRuntimeState::getThreadDataSectionFor(const char *ThreadData) { std::lock_guard Lock(ThreadDataSectionsMutex); @@ -416,43 +535,23 @@ MachOPlatformRuntimeState::getThreadDataSectionFor(const char *ThreadData) { return *I; } -MachOPlatformRuntimeState::PerJITDylibState * -MachOPlatformRuntimeState::getJITDylibStateByHeaderAddr(void *DSOHandle) { +MachOPlatformRuntimeState::JITDylibState * +MachOPlatformRuntimeState::getJITDylibStateByHeader(void *DSOHandle) { auto I = JDStates.find(DSOHandle); - if (I == JDStates.end()) - return nullptr; + if (I == JDStates.end()) { + I = JDStates.insert(std::make_pair(DSOHandle, JITDylibState())).first; + I->second.Header = DSOHandle; + } return &I->second; } -MachOPlatformRuntimeState::PerJITDylibState * +MachOPlatformRuntimeState::JITDylibState * MachOPlatformRuntimeState::getJITDylibStateByName(string_view Name) { - // FIXME: Avoid creating string copy here. + // FIXME: Avoid creating string once we have C++20. auto I = JDNameToHeader.find(std::string(Name.data(), Name.size())); - if (I == JDNameToHeader.end()) - return nullptr; - void *H = I->second; - auto J = JDStates.find(H); - assert(J != JDStates.end() && - "JITDylib has name map entry but no header map entry"); - return &J->second; -} - -MachOPlatformRuntimeState::PerJITDylibState & -MachOPlatformRuntimeState::getOrCreateJITDylibState( - MachOJITDylibInitializers &MOJDIs) { - void *Header = MOJDIs.MachOHeaderAddress.toPtr(); - - auto &JDS = JDStates[Header]; - - // If this entry hasn't been created yet. - if (!JDS.Header) { - assert(!JDNameToHeader.count(MOJDIs.Name) && - "JITDylib has header map entry but no name map entry"); - JDNameToHeader[MOJDIs.Name] = Header; - JDS.Header = Header; - } - - return JDS; + if (I != JDNameToHeader.end()) + return getJITDylibStateByHeader(I->second); + return nullptr; } Expected @@ -468,60 +567,313 @@ MachOPlatformRuntimeState::lookupSymbolInJITDylib(void *DSOHandle, return Result; } -Expected -MachOPlatformRuntimeState::getJITDylibInitializersByName(string_view Path) { - Expected Result( - (MachOJITDylibInitializerSequence())); - std::string PathStr(Path.data(), Path.size()); - if (auto Err = - WrapperFunction( - SPSString)>::call(&__orc_rt_macho_get_initializers_tag, Result, - Path)) - return std::move(Err); - return Result; +template +static void moveAppendSections(std::vector> &Dst, + std::vector> &Src) { + if (Dst.empty()) { + Dst = std::move(Src); + return; + } + + Dst.reserve(Dst.size() + Src.size()); + std::copy(Src.begin(), Src.end(), std::back_inserter(Dst)); + Src.clear(); +} + +Error MachOPlatformRuntimeState::registerObjCSelectors(JITDylibState &JDS) { + + if (JDS.ObjCSelRefsSectionsNew.empty()) + return Error::success(); + + if (ORC_RT_UNLIKELY(!sel_registerName)) + return make_error("sel_registerName is not available"); + + for (const auto &ObjCSelRefs : JDS.ObjCSelRefsSectionsNew) { + for (uintptr_t &SelEntry : ObjCSelRefs) { + const char *SelName = reinterpret_cast(SelEntry); + auto Sel = sel_registerName(SelName); + *reinterpret_cast(&SelEntry) = Sel; + } + } + + moveAppendSections(JDS.ObjCSelRefsSections, JDS.ObjCSelRefsSectionsNew); + return Error::success(); +} + +Error MachOPlatformRuntimeState::registerObjCClasses(JITDylibState &JDS) { + + if (JDS.ObjCClassListSectionsNew.empty()) + return Error::success(); + + if (ORC_RT_UNLIKELY(!objc_msgSend)) + return make_error("objc_msgSend is not available"); + if (ORC_RT_UNLIKELY(!objc_readClassPair)) + return make_error("objc_readClassPair is not available"); + + struct ObjCClassCompiled { + void *Metaclass; + void *Parent; + void *Cache1; + void *Cache2; + void *Data; + }; + + auto ClassSelector = sel_registerName("class"); + + for (const auto &ObjCClassList : JDS.ObjCClassListSectionsNew) { + for (uintptr_t ClassPtr : ObjCClassList) { + auto *Cls = reinterpret_cast(ClassPtr); + auto *ClassCompiled = reinterpret_cast(ClassPtr); + objc_msgSend(reinterpret_cast(ClassCompiled->Parent), ClassSelector); + auto Registered = objc_readClassPair(Cls, JDS.ObjCImageInfo); + + // FIXME: Improve diagnostic by reporting the failed class's name. + if (Registered != Cls) + return make_error("Unable to register Objective-C class"); + } + } + + moveAppendSections(JDS.ObjCClassListSections, JDS.ObjCClassListSectionsNew); + return Error::success(); } -Expected MachOPlatformRuntimeState::dlopenInitialize(string_view Path, - int Mode) { - // Either our JITDylib wasn't loaded, or it or one of its dependencies allows - // reinitialization. We need to call in to the JIT to see if there's any new - // work pending. - auto InitSeq = getJITDylibInitializersByName(Path); - if (!InitSeq) - return InitSeq.takeError(); +Error MachOPlatformRuntimeState::registerSwift5Protocols(JITDylibState &JDS) { + + if (JDS.Swift5ProtosSectionsNew.empty()) + return Error::success(); + + if (ORC_RT_UNLIKELY(!swift_registerProtocols)) + return make_error("swift_registerProtocols is not available"); + + for (const auto &Swift5Protocols : JDS.Swift5ProtoSectionsNew) + swift_registerProtocols( + reinterpret_cast(Swift5Protocols.data()), + reinterpret_cast(Swift5Protocols.data() + + Swift5Protocols.size())); + + moveAppendSections(JDS.Swift5ProtoSections, JDS.Swift5ProtoSectionsNew); + return Error::success(); +} + +Error MachOPlatformRuntimeState::registerSwift5ProtocolConformances( + JITDylibState &JDS) { + + if (JDS.Swift5ProtosSectionsNew.empty()) + return Error::success(); - // Init sequences should be non-empty. - if (InitSeq->empty()) + if (ORC_RT_UNLIKELY(!swift_registerProtocolConformances)) return make_error( - "__orc_rt_macho_get_initializers returned an " - "empty init sequence"); + "swift_registerProtocolConformances is not available"); + + for (const auto &ProtoConfSec : JDS.Swift5ProtosSectionsNew) + swift_registerProtocolConformances( + reinterpret_cast( + ProtoConfSec.data()), + reinterpret_cast( + ProtoConfSec.data() + ProtoConfSec.size())); + + moveAppendSections(JDS.Swift5ProtosSections, JDS.Swift5ProtosSectionsNew); + return Error::success(); +} + +Error MachOPlatformRuntimeState::registerSwift5Types(JITDylibState &JDS) { + + if (JDS.Swift5TypesSectionsNew.empty()) + return Error::success(); + + if (ORC_RT_UNLIKELY(!swift_registerTypeMetadataRecords)) + return make_error( + "swift_registerTypeMetadataRecords is not available"); + + for (const auto &TypeSec : JDS.Swift5TypesSectionsNew) + swift_registerTypeMetadataRecords( + reinterpret_cast(TypeSec.data()), + reinterpret_cast(TypeSec.data() + + TypeSec.size())); + + moveAppendSections(JDS.Swift5TypesSections, JDS.Swift5TypesSectionsNew); + return Error::success(); +} + +Error MachOPlatformRuntimeState::runModInits(JITDylibState &JDS) { - // Otherwise register and run initializers for each JITDylib. - for (auto &MOJDIs : *InitSeq) - if (auto Err = initializeJITDylib(MOJDIs)) + for (const auto &ModInits : JDS.ModInitsSectionsNew) { + for (void (*Init)() : ModInits) + (*Init)(); + } + + moveAppendSections(JDS.ModInitsSections, JDS.ModInitsSectionsNew); + return Error::success(); +} + +Expected MachOPlatformRuntimeState::dlopenImpl(string_view Path, + int Mode) { + // Try to find JITDylib state by name. + auto *JDS = getJITDylibStateByName(Path); + + if (!JDS) + return make_error("No registered JTIDylib for path " + + std::string(Path.data(), Path.size())); + + // If this JITDylib is unsealed, or this is the first dlopen then run + // full dlopen path (update deps, push and run initializers, update ref + // counts on all JITDylibs in the dep tree). + if (!JDS->referenced() || !JDS->Sealed) { + if (auto Err = dlopenFull(*JDS)) return std::move(Err); + } - // Return the header for the last item in the list. - auto *JDS = getJITDylibStateByHeaderAddr( - InitSeq->back().MachOHeaderAddress.toPtr()); - assert(JDS && "Missing state entry for JD"); + // Bump the ref-count on this dylib. + ++JDS->DlRefCount; + + // Return the header address. return JDS->Header; } -Error MachOPlatformRuntimeState::initializeJITDylib( - MachOJITDylibInitializers &MOJDIs) { +Error MachOPlatformRuntimeState::dlopenFull(JITDylibState &JDS) { + // Call back to the JIT to push the initializers. + Expected DepInfo((MachOJITDylibDepInfoMap())); + if (auto Err = WrapperFunction( + SPSExecutorAddr)>::call(&__orc_rt_macho_push_initializers_tag, + DepInfo, ExecutorAddr::fromPtr(JDS.Header))) + return Err; + if (!DepInfo) + return DepInfo.takeError(); + + if (auto Err = dlopenInitialize(JDS, *DepInfo)) + return Err; + + if (!DepInfo->empty()) { + ORC_RT_DEBUG({ + printdbg("Unrecognized dep-info key headers in dlopen of %s\n", + JDS.Name.c_str()); + }); + std::ostringstream ErrStream; + ErrStream << "Encountered unrecognized dep-info key headers " + "while processing dlopen of " + << JDS.Name; + return make_error(ErrStream.str()); + } + + return Error::success(); +} - auto &JDS = getOrCreateJITDylibState(MOJDIs); - ++JDS.RefCount; +Error MachOPlatformRuntimeState::dlopenInitialize( + JITDylibState &JDS, MachOJITDylibDepInfoMap &DepInfo) { + ORC_RT_DEBUG({ + printdbg("MachOPlatformRuntimeState::dlopenInitialize(\"%s\")\n", + JDS.Name.c_str()); + }); + + // If the header is not present in the dep map then assume that we + // already processed it earlier in the dlopenInitialize traversal and + // return. + // TODO: Keep a visited set instead so that we can error out on missing + // entries? + auto I = DepInfo.find(ExecutorAddr::fromPtr(JDS.Header)); + if (I == DepInfo.end()) + return Error::success(); - for (auto &KV : InitSections) { - const auto &Name = KV.first; - const auto &Handler = KV.second; - auto I = MOJDIs.InitSections.find(Name); - if (I != MOJDIs.InitSections.end()) { - if (auto Err = Handler(I->second, MOJDIs)) - return Err; + auto DI = std::move(I->second); + DepInfo.erase(I); + + // We don't need to re-initialize sealed JITDylibs that have already been + // initialized. Just check that their dep-map entry is empty as expected. + if (JDS.Sealed) { + if (!DI.DepHeaders.empty()) { + std::ostringstream ErrStream; + ErrStream << "Sealed JITDylib " << JDS.Header + << " already has registered dependencies"; + return make_error(ErrStream.str()); + } + if (JDS.referenced()) + return Error::success(); + } else + JDS.Sealed = DI.Sealed; + + // This is an unsealed or newly sealed JITDylib. Run initializers. + std::vector OldDeps; + std::swap(JDS.Deps, OldDeps); + JDS.Deps.reserve(DI.DepHeaders.size()); + for (auto DepHeaderAddr : DI.DepHeaders) { + auto *DepJDS = getJITDylibStateByHeader(DepHeaderAddr.toPtr()); + if (!DepJDS) { + std::ostringstream ErrStream; + ErrStream << "Encountered unrecognized dep header " + << DepHeaderAddr.toPtr() << " while initializing " + << JDS.Name; + return make_error(ErrStream.str()); } + ++DepJDS->LinkedAgainstRefCount; + if (auto Err = dlopenInitialize(*DepJDS, DepInfo)) + return Err; + } + + // Initialize this JITDylib. + if (auto Err = registerObjCSelectors(JDS)) + return Err; + if (auto Err = registerObjCClasses(JDS)) + return Err; + if (auto Err = registerSwift5Protocols(JDS)) + return Err; + if (auto Err = registerSwift5ProtocolConformances(JDS)) + return Err; + if (auto Err = registerSwift5Types(JDS)) + return Err; + if (auto Err = runModInits(JDS)) + return Err; + + // Decrement old deps. + // FIXME: We should probably continue and just report deinitialize errors + // here. + for (auto *DepJDS : OldDeps) { + --DepJDS->LinkedAgainstRefCount; + if (!DepJDS->referenced()) + if (auto Err = dlcloseDeinitialize(*DepJDS)) + return Err; + } + + return Error::success(); +} + +Error MachOPlatformRuntimeState::dlcloseImpl(void *DSOHandle) { + // Try to find JITDylib state by header. + auto *JDS = getJITDylibStateByHeader(DSOHandle); + + if (!JDS) { + std::ostringstream ErrStream; + ErrStream << "No registered JITDylib for " << DSOHandle; + return make_error(ErrStream.str()); + } + + // Bump the ref-count. + --JDS->DlRefCount; + + if (!JDS->referenced()) + return dlcloseDeinitialize(*JDS); + + return Error::success(); +} + +Error MachOPlatformRuntimeState::dlcloseDeinitialize(JITDylibState &JDS) { + + ORC_RT_DEBUG({ + printdbg("MachOPlatformRuntimeState::dlcloseDeinitialize(\"%s\")\n", + JDS.Name.c_str()); + }); + + runAtExits(JDS); + + // Reset mod-inits + moveAppendSections(JDS.ModInitsSections, JDS.ModInitsSectionsNew); + JDS.ModInitsSectionsNew = std::move(JDS.ModInitsSections); + + // Deinitialize any dependencies. + for (auto *DepJDS : JDS.Deps) { + --DepJDS->LinkedAgainstRefCount; + if (!DepJDS->referenced()) + if (auto Err = dlcloseDeinitialize(*DepJDS)) + return Err; } return Error::success(); @@ -592,27 +944,56 @@ __orc_rt_macho_platform_shutdown(char *ArgData, size_t ArgSize) { } ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult -__orc_rt_macho_register_thread_data_section(char *ArgData, size_t ArgSize) { - return WrapperFunction::handle( +__orc_rt_macho_register_jitdylib(char *ArgData, size_t ArgSize) { + return WrapperFunction::handle( ArgData, ArgSize, - [](ExecutorAddrRange R) { - return MachOPlatformRuntimeState::get() - .registerThreadDataSection(R.toSpan()); + [](std::string &Name, ExecutorAddr HeaderAddr) { + return MachOPlatformRuntimeState::get().registerJITDylib( + std::move(Name), HeaderAddr.toPtr()); }) .release(); } ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult -__orc_rt_macho_deregister_thread_data_section(char *ArgData, size_t ArgSize) { - return WrapperFunction::handle( +__orc_rt_macho_deregister_jitdylib(char *ArgData, size_t ArgSize) { + return WrapperFunction::handle( ArgData, ArgSize, - [](ExecutorAddrRange R) { - return MachOPlatformRuntimeState::get() - .deregisterThreadDataSection(R.toSpan()); + [](ExecutorAddr HeaderAddr) { + return MachOPlatformRuntimeState::get().deregisterJITDylib( + HeaderAddr.toPtr()); }) .release(); } +ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult +__orc_rt_macho_register_object_platform_sections(char *ArgData, + size_t ArgSize) { + return WrapperFunction:: + handle(ArgData, ArgSize, + [](ExecutorAddr HeaderAddr, + std::vector> &Secs) { + return MachOPlatformRuntimeState::get() + .registerObjectPlatformSections(HeaderAddr, std::move(Secs)); + }) + .release(); +} + +ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult +__orc_rt_macho_deregister_object_platform_sections(char *ArgData, + size_t ArgSize) { + return WrapperFunction:: + handle(ArgData, ArgSize, + [](ExecutorAddr HeaderAddr, + std::vector> &Secs) { + return MachOPlatformRuntimeState::get() + .deregisterObjectPlatformSections(HeaderAddr, + std::move(Secs)); + }) + .release(); +} + ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult __orc_rt_macho_run_wrapper_function_calls(char *ArgData, size_t ArgSize) { return WrapperFunction)>::handle( diff --git a/compiler-rt/lib/orc/macho_platform.h b/compiler-rt/lib/orc/macho_platform.h index 5b9820a0d1f9f..3b2242ab27ce8 100644 --- a/compiler-rt/lib/orc/macho_platform.h +++ b/compiler-rt/lib/orc/macho_platform.h @@ -31,28 +31,6 @@ ORC_RT_INTERFACE void *__orc_rt_macho_jit_dlsym(void *dso_handle, namespace __orc_rt { namespace macho { -struct MachOJITDylibInitializers { - using SectionList = std::vector; - - MachOJITDylibInitializers() = default; - MachOJITDylibInitializers(std::string Name, ExecutorAddr MachOHeaderAddress) - : Name(std::move(Name)), - MachOHeaderAddress(std::move(MachOHeaderAddress)) {} - - std::string Name; - ExecutorAddr MachOHeaderAddress; - ExecutorAddr ObjCImageInfoAddress; - - std::unordered_map InitSections; -}; - -class MachOJITDylibDeinitializers {}; - -using MachOJITDylibInitializerSequence = std::vector; - -using MachOJITDylibDeinitializerSequence = - std::vector; - enum dlopen_mode : int { ORC_RT_RTLD_LAZY = 0x1, ORC_RT_RTLD_NOW = 0x2, @@ -61,43 +39,6 @@ enum dlopen_mode : int { }; } // end namespace macho - -using SPSNamedExecutorAddrRangeSequenceMap = - SPSSequence>; - -using SPSMachOJITDylibInitializers = - SPSTuple; - -using SPSMachOJITDylibInitializerSequence = - SPSSequence; - -/// Serialization traits for MachOJITDylibInitializers. -template <> -class SPSSerializationTraits { -public: - static size_t size(const macho::MachOJITDylibInitializers &MOJDIs) { - return SPSMachOJITDylibInitializers::AsArgList::size( - MOJDIs.Name, MOJDIs.MachOHeaderAddress, MOJDIs.ObjCImageInfoAddress, - MOJDIs.InitSections); - } - - static bool serialize(SPSOutputBuffer &OB, - const macho::MachOJITDylibInitializers &MOJDIs) { - return SPSMachOJITDylibInitializers::AsArgList::serialize( - OB, MOJDIs.Name, MOJDIs.MachOHeaderAddress, MOJDIs.ObjCImageInfoAddress, - MOJDIs.InitSections); - } - - static bool deserialize(SPSInputBuffer &IB, - macho::MachOJITDylibInitializers &MOJDIs) { - return SPSMachOJITDylibInitializers::AsArgList::deserialize( - IB, MOJDIs.Name, MOJDIs.MachOHeaderAddress, MOJDIs.ObjCImageInfoAddress, - MOJDIs.InitSections); - } -}; - } // end namespace __orc_rt #endif // ORC_RT_MACHO_PLATFORM_H diff --git a/compiler-rt/test/orc/TestCases/Darwin/x86-64/Inputs/standalone-ctor-and-cxa-atexit-dtor.S b/compiler-rt/test/orc/TestCases/Darwin/x86-64/Inputs/standalone-ctor-and-cxa-atexit-dtor.S new file mode 100644 index 0000000000000..52df2be743f4a --- /dev/null +++ b/compiler-rt/test/orc/TestCases/Darwin/x86-64/Inputs/standalone-ctor-and-cxa-atexit-dtor.S @@ -0,0 +1,41 @@ +// Contains a static initializer and deinitializer registered with +// ___cxa_atexit. dlopen-ing/dlclose-ing will print "constructor" and +// "destructor" respectively. + + .section __TEXT,__text,regular,pure_instructions + .build_version macos, 12, 0 sdk_version 12, 0, 1 + .globl _deinitializer + .p2align 4, 0x90 +_deinitializer: + pushq %rbp + movq %rsp, %rbp + leaq L_str.2(%rip), %rdi + popq %rbp + jmp _puts + + .section __TEXT,__StaticInit,regular,pure_instructions + .p2align 4, 0x90 +_initializer: + pushq %rbp + movq %rsp, %rbp + leaq L_str(%rip), %rdi + callq _puts + movq _deinitializer@GOTPCREL(%rip), %rdi + leaq _I(%rip), %rsi + leaq ___dso_handle(%rip), %rdx + popq %rbp + jmp ___cxa_atexit + + .globl _I +.zerofill __DATA,__common,_I,1,0 + .section __DATA,__mod_init_func,mod_init_funcs + .p2align 3 + .quad _initializer + .section __TEXT,__cstring,cstring_literals +L_str: + .asciz "constructor" + +L_str.2: + .asciz "destructor" + +.subsections_via_symbols diff --git a/compiler-rt/test/orc/TestCases/Darwin/x86-64/trivial-jit-dlopen.S b/compiler-rt/test/orc/TestCases/Darwin/x86-64/trivial-jit-dlopen.S new file mode 100644 index 0000000000000..5076f72486003 --- /dev/null +++ b/compiler-rt/test/orc/TestCases/Darwin/x86-64/trivial-jit-dlopen.S @@ -0,0 +1,119 @@ +// Test that __orc_rt_macho_jit_dlopen and __orc_rt_macho_jit_dlclose run +// constructors and destructors as expected. +// +// This test calls dlopen and dlclose twice. We expect the inner calls to be +// no-ops. +// +// RUN: %clang -c -o %t.inits.o %p/Inputs/standalone-ctor-and-cxa-atexit-dtor.S +// RUN: %clang -c -o %t.test.o %s +// RUN: %llvm_jitlink \ +// RUN: -alias _dlopen=___orc_rt_macho_jit_dlopen \ +// RUN: -alias _dlclose=___orc_rt_macho_jit_dlclose \ +// RUN: %t.test.o -jd inits %t.inits.o -lmain | FileCheck %s +// +// CHECK: entering main +// CHECK-NEXT: first dlopen +// CHECK-NEXT: constructor +// CHECK-NEXT: second dlopen +// CHECK-NEXT: first dlclose +// CHECK-NEXT: second dlclose +// CHECK-NEXT: destructor +// CHECK-NEXT: leaving main + + .section __TEXT,__text,regular,pure_instructions + .build_version macos, 12, 0 sdk_version 13, 0 + .globl _main + .p2align 4, 0x90 +_main: + + pushq %rbp + movq %rsp, %rbp + pushq %r15 + pushq %r14 + pushq %rbx + pushq %rax + leaq L_str(%rip), %rdi + callq _puts + leaq L_str.9(%rip), %rdi + callq _puts + leaq L_.str.2(%rip), %rdi + movl $1, %esi + callq _dlopen + movl $-1, %r14d + leaq L_str.17(%rip), %r15 + testq %rax, %rax + je LBB0_6 + + movq %rax, %rbx + leaq L_str.11(%rip), %rdi + callq _puts + leaq L_.str.2(%rip), %rdi + movl $1, %esi + callq _dlopen + testq %rax, %rax + je LBB0_6 + + cmpq %rbx, %rax + je LBB0_4 + + leaq L_str.18(%rip), %r15 + jmp LBB0_6 +LBB0_4: + leaq L_str.13(%rip), %rdi + callq _puts + movq %rbx, %rdi + callq _dlclose + cmpl $-1, %eax + je LBB0_6 + + leaq L_str.14(%rip), %rdi + callq _puts + movq %rbx, %rdi + callq _dlclose + xorl %r14d, %r14d + cmpl $-1, %eax + sete %r14b + leaq L_str.17(%rip), %rax + leaq L_str.15(%rip), %r15 + cmoveq %rax, %r15 + negl %r14d +LBB0_6: + movq %r15, %rdi + callq _puts + movl %r14d, %eax + addq $8, %rsp + popq %rbx + popq %r14 + popq %r15 + popq %rbp + retq + + .section __TEXT,__cstring,cstring_literals +L_.str.2: + .asciz "inits" + +L_str: + .asciz "entering main" + +L_str.9: + .asciz "first dlopen" + +L_str.11: + .asciz "second dlopen" + +L_str.13: + .asciz "first dlclose" + +L_str.14: + .asciz "second dlclose" + +L_str.15: + .asciz "leaving main" + +L_str.17: + .asciz "failed" + +L_str.18: + .asciz "handles do not match" + +.subsections_via_symbols diff --git a/compiler-rt/test/orc/lit.cfg.py b/compiler-rt/test/orc/lit.cfg.py index dc4bac4aed1e1..5ce6c8b1fa7f4 100644 --- a/compiler-rt/test/orc/lit.cfg.py +++ b/compiler-rt/test/orc/lit.cfg.py @@ -29,5 +29,8 @@ def build_invocation(compile_flags): # Default test suffixes. config.suffixes = ['.c', '.cpp', '.S'] +# Exclude Inputs directories. +config.excludes = ['Inputs'] + if config.host_os not in ['Darwin', 'FreeBSD', 'Linux']: config.unsupported = True diff --git a/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h b/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h index 01f3f1b2ab639..141dd73548c85 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h @@ -26,30 +26,19 @@ namespace llvm { namespace orc { -struct MachOJITDylibInitializers { - using SectionList = std::vector; - - MachOJITDylibInitializers(std::string Name, ExecutorAddr MachOHeaderAddress) - : Name(std::move(Name)), - MachOHeaderAddress(std::move(MachOHeaderAddress)) {} - - std::string Name; - ExecutorAddr MachOHeaderAddress; - ExecutorAddr ObjCImageInfoAddress; - - StringMap InitSections; -}; - -class MachOJITDylibDeinitializers {}; - -using MachOJITDylibInitializerSequence = std::vector; - -using MachOJITDylibDeinitializerSequence = - std::vector; - /// Mediates between MachO initialization and ExecutionSession state. class MachOPlatform : public Platform { public: + // Used internally by MachOPlatform, but made public to enable serialization. + struct MachOJITDylibDepInfo { + bool Sealed = false; + std::vector DepHeaders; + }; + + // Used internally by MachOPlatform, but made public to enable serialization. + using MachOJITDylibDepInfoMap = + std::vector>; + /// Try to create a MachOPlatform instance, adding the ORC runtime to the /// given JITDylib. /// @@ -161,26 +150,28 @@ class MachOPlatform : public Platform { Error processObjCImageInfo(jitlink::LinkGraph &G, MaterializationResponsibility &MR); - Error registerInitSections(jitlink::LinkGraph &G, JITDylib &JD); - Error fixTLVSectionsAndEdges(jitlink::LinkGraph &G, JITDylib &JD); - Error registerEHAndTLVSections(jitlink::LinkGraph &G); + Error registerObjectPlatformSections(jitlink::LinkGraph &G, JITDylib &JD); Error registerEHSectionsPhase1(jitlink::LinkGraph &G); std::mutex PluginMutex; MachOPlatform &MP; + + // FIXME: ObjCImageInfos and HeaderAddrs need to be cleared when + // JITDylibs are removed. DenseMap> ObjCImageInfos; + DenseMap HeaderAddrs; InitSymbolDepMap InitSymbolDeps; }; - using SendInitializerSequenceFn = - unique_function)>; - - using SendDeinitializerSequenceFn = - unique_function)>; - + using GetJITDylibHeaderSendResultFn = + unique_function)>; + using GetJITDylibNameSendResultFn = + unique_function)>; + using PushInitializersSendResultFn = + unique_function)>; using SendSymbolAddressFn = unique_function)>; static bool supportedTarget(const Triple &TT); @@ -193,28 +184,24 @@ class MachOPlatform : public Platform { // Associate MachOPlatform JIT-side runtime support functions with handlers. Error associateRuntimeSupportFunctions(JITDylib &PlatformJD); - void getInitializersBuildSequencePhase(SendInitializerSequenceFn SendResult, - JITDylib &JD, - std::vector DFSLinkOrder); + // Implements rt_pushInitializers by making repeat async lookups for + // initializer symbols (each lookup may spawn more initializer symbols if + // it pulls in new materializers, e.g. from objects in a static library). + void pushInitializersLoop(PushInitializersSendResultFn SendResult, + JITDylibSP JD); - void getInitializersLookupPhase(SendInitializerSequenceFn SendResult, - JITDylib &JD); - - void rt_getInitializers(SendInitializerSequenceFn SendResult, - StringRef JDName); - - void rt_getDeinitializers(SendDeinitializerSequenceFn SendResult, - ExecutorAddr Handle); + // Handle requests from the ORC runtime to push MachO initializer info. + void rt_pushInitializers(PushInitializersSendResultFn SendResult, + ExecutorAddr JDHeaderAddr); + // Handle requests for symbol addresses from the ORC runtime. void rt_lookupSymbol(SendSymbolAddressFn SendResult, ExecutorAddr Handle, StringRef SymbolName); // Records the addresses of runtime symbols used by the platform. Error bootstrapMachORuntime(JITDylib &PlatformJD); - Error registerInitInfo(JITDylib &JD, ExecutorAddr ObjCImageInfoAddr, - ArrayRef InitSections); - + // Call the ORC runtime to create a pthread key. Expected createPThreadKey(); enum PlatformState { BootstrapPhase1, BootstrapPhase2, Initialized }; @@ -229,81 +216,24 @@ class MachOPlatform : public Platform { ExecutorAddr orc_rt_macho_platform_shutdown; ExecutorAddr orc_rt_macho_register_ehframe_section; ExecutorAddr orc_rt_macho_deregister_ehframe_section; - ExecutorAddr orc_rt_macho_register_thread_data_section; - ExecutorAddr orc_rt_macho_deregister_thread_data_section; + ExecutorAddr orc_rt_macho_register_jitdylib; + ExecutorAddr orc_rt_macho_deregister_jitdylib; + ExecutorAddr orc_rt_macho_register_object_platform_sections; + ExecutorAddr orc_rt_macho_deregister_object_platform_sections; ExecutorAddr orc_rt_macho_create_pthread_key; DenseMap RegisteredInitSymbols; - // InitSeqs gets its own mutex to avoid locking the whole session when - // aggregating data from the jitlink. std::mutex PlatformMutex; - DenseMap InitSeqs; - + DenseMap JITDylibToHeaderAddr; DenseMap HeaderAddrToJITDylib; DenseMap JITDylibToPThreadKey; }; namespace shared { -using SPSNamedExecutorAddrRangeSequenceMap = - SPSSequence>; - -using SPSMachOJITDylibInitializers = - SPSTuple; - -using SPSMachOJITDylibInitializerSequence = - SPSSequence; - -/// Serialization traits for MachOJITDylibInitializers. -template <> -class SPSSerializationTraits { -public: - static size_t size(const MachOJITDylibInitializers &MOJDIs) { - return SPSMachOJITDylibInitializers::AsArgList::size( - MOJDIs.Name, MOJDIs.MachOHeaderAddress, MOJDIs.ObjCImageInfoAddress, - MOJDIs.InitSections); - } - - static bool serialize(SPSOutputBuffer &OB, - const MachOJITDylibInitializers &MOJDIs) { - return SPSMachOJITDylibInitializers::AsArgList::serialize( - OB, MOJDIs.Name, MOJDIs.MachOHeaderAddress, MOJDIs.ObjCImageInfoAddress, - MOJDIs.InitSections); - } - - static bool deserialize(SPSInputBuffer &IB, - MachOJITDylibInitializers &MOJDIs) { - return SPSMachOJITDylibInitializers::AsArgList::deserialize( - IB, MOJDIs.Name, MOJDIs.MachOHeaderAddress, MOJDIs.ObjCImageInfoAddress, - MOJDIs.InitSections); - } -}; - -using SPSMachOJITDylibDeinitializers = SPSEmpty; - -using SPSMachOJITDylibDeinitializerSequence = - SPSSequence; - -template <> -class SPSSerializationTraits { -public: - static size_t size(const MachOJITDylibDeinitializers &MOJDDs) { return 0; } - - static bool serialize(SPSOutputBuffer &OB, - const MachOJITDylibDeinitializers &MOJDDs) { - return true; - } - - static bool deserialize(SPSInputBuffer &IB, - MachOJITDylibDeinitializers &MOJDDs) { - MOJDDs = MachOJITDylibDeinitializers(); - return true; - } -}; +using SPSNamedExecutorAddrRangeSequence = + SPSSequence>; } // end namespace shared } // end namespace orc diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h b/llvm/include/llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h index 302b60b80fd06..9be58e9f0fa9d 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h @@ -586,7 +586,7 @@ SPSSerializableExpected toSPSSerializable(Expected E) { if (E) return {true, std::move(*E), {}}; else - return {false, {}, toString(E.takeError())}; + return {false, T(), toString(E.takeError())}; } template diff --git a/llvm/lib/ExecutionEngine/Orc/Core.cpp b/llvm/lib/ExecutionEngine/Orc/Core.cpp index 8a0d450422699..7514b242d0c00 100644 --- a/llvm/lib/ExecutionEngine/Orc/Core.cpp +++ b/llvm/lib/ExecutionEngine/Orc/Core.cpp @@ -1868,7 +1868,7 @@ Error ExecutionSession::endSession() { // TODO: notifiy platform? run static deinits? Error Err = Error::success(); - for (auto &JD : JITDylibsToClose) + for (auto &JD : reverse(JITDylibsToClose)) Err = joinErrors(std::move(Err), JD->clear()); Err = joinErrors(std::move(Err), EPC->disconnect()); diff --git a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp index a364719855b40..b2de83acc2e90 100644 --- a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp +++ b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp @@ -22,6 +22,39 @@ using namespace llvm; using namespace llvm::orc; using namespace llvm::orc::shared; +namespace llvm { +namespace orc { +namespace shared { + +using SPSMachOJITDylibDepInfo = SPSTuple>; +using SPSMachOJITDylibDepInfoMap = + SPSSequence>; + +template <> +class SPSSerializationTraits { +public: + static size_t size(const MachOPlatform::MachOJITDylibDepInfo &DDI) { + return SPSMachOJITDylibDepInfo::AsArgList::size(DDI.Sealed, DDI.DepHeaders); + } + + static bool serialize(SPSOutputBuffer &OB, + const MachOPlatform::MachOJITDylibDepInfo &DDI) { + return SPSMachOJITDylibDepInfo::AsArgList::serialize(OB, DDI.Sealed, + DDI.DepHeaders); + } + + static bool deserialize(SPSInputBuffer &IB, + MachOPlatform::MachOJITDylibDepInfo &DDI) { + return SPSMachOJITDylibDepInfo::AsArgList::deserialize(IB, DDI.Sealed, + DDI.DepHeaders); + } +}; + +} // namespace shared +} // namespace orc +} // namespace llvm + namespace { class MachOHeaderMaterializationUnit : public MaterializationUnit { @@ -199,11 +232,25 @@ MachOPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, } Error MachOPlatform::setupJITDylib(JITDylib &JD) { - return JD.define(std::make_unique( - *this, MachOHeaderStartSymbol)); + if (auto Err = JD.define(std::make_unique( + *this, MachOHeaderStartSymbol))) + return Err; + + return ES.lookup({&JD}, MachOHeaderStartSymbol).takeError(); } -Error MachOPlatform::teardownJITDylib(JITDylib &JD) { return Error::success(); } +Error MachOPlatform::teardownJITDylib(JITDylib &JD) { + std::lock_guard Lock(PlatformMutex); + auto I = JITDylibToHeaderAddr.find(&JD); + if (I != JITDylibToHeaderAddr.end()) { + assert(HeaderAddrToJITDylib.count(I->second) && + "HeaderAddrToJITDylib missing entry"); + HeaderAddrToJITDylib.erase(I->second); + JITDylibToHeaderAddr.erase(I); + } + JITDylibToPThreadKey.erase(&JD); + return Error::success(); +} Error MachOPlatform::notifyAdding(ResourceTracker &RT, const MaterializationUnit &MU) { @@ -305,16 +352,6 @@ MachOPlatform::MachOPlatform( State = BootstrapPhase2; - // PlatformJD hasn't been 'set-up' by the platform yet (since we're creating - // the platform now), so set it up. - if (auto E2 = setupJITDylib(PlatformJD)) { - Err = std::move(E2); - return; - } - - RegisteredInitSymbols[&PlatformJD].add( - MachOHeaderStartSymbol, SymbolLookupFlags::WeaklyReferencedSymbol); - // Associate wrapper function tags with JIT-side function implementations. if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) { Err = std::move(E2); @@ -329,23 +366,24 @@ MachOPlatform::MachOPlatform( return; } + // PlatformJD hasn't been set up by the platform yet (since we're creating + // the platform now), so set it up. + if (auto E2 = setupJITDylib(PlatformJD)) { + Err = std::move(E2); + return; + } + State = Initialized; } Error MachOPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) { ExecutionSession::JITDispatchHandlerAssociationMap WFs; - using GetInitializersSPSSig = - SPSExpected(SPSString); - WFs[ES.intern("___orc_rt_macho_get_initializers_tag")] = - ES.wrapAsyncWithSPS( - this, &MachOPlatform::rt_getInitializers); - - using GetDeinitializersSPSSig = - SPSExpected(SPSExecutorAddr); - WFs[ES.intern("___orc_rt_macho_get_deinitializers_tag")] = - ES.wrapAsyncWithSPS( - this, &MachOPlatform::rt_getDeinitializers); + using PushInitializersSPSSig = + SPSExpected(SPSExecutorAddr); + WFs[ES.intern("___orc_rt_macho_push_initializers_tag")] = + ES.wrapAsyncWithSPS( + this, &MachOPlatform::rt_pushInitializers); using LookupSymbolSPSSig = SPSExpected(SPSExecutorAddr, SPSString); @@ -356,53 +394,83 @@ Error MachOPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) { return ES.registerJITDispatchHandlers(PlatformJD, std::move(WFs)); } -void MachOPlatform::getInitializersBuildSequencePhase( - SendInitializerSequenceFn SendResult, JITDylib &JD, - std::vector DFSLinkOrder) { - MachOJITDylibInitializerSequence FullInitSeq; - { - std::lock_guard Lock(PlatformMutex); - for (auto &InitJD : reverse(DFSLinkOrder)) { - LLVM_DEBUG({ - dbgs() << "MachOPlatform: Appending inits for \"" << InitJD->getName() - << "\" to sequence\n"; - }); - auto ISItr = InitSeqs.find(InitJD.get()); - if (ISItr != InitSeqs.end()) { - FullInitSeq.emplace_back(std::move(ISItr->second)); - InitSeqs.erase(ISItr); - } - } - } - - SendResult(std::move(FullInitSeq)); -} - -void MachOPlatform::getInitializersLookupPhase( - SendInitializerSequenceFn SendResult, JITDylib &JD) { - - auto DFSLinkOrder = JD.getDFSLinkOrder(); - if (!DFSLinkOrder) { - SendResult(DFSLinkOrder.takeError()); - return; - } - +void MachOPlatform::pushInitializersLoop( + PushInitializersSendResultFn SendResult, JITDylibSP JD) { DenseMap NewInitSymbols; + DenseMap> JDDepMap; + SmallVector Worklist({JD.get()}); + ES.runSessionLocked([&]() { - for (auto &InitJD : *DFSLinkOrder) { - auto RISItr = RegisteredInitSymbols.find(InitJD.get()); + while (!Worklist.empty()) { + // FIXME: Check for defunct dylibs. + + auto DepJD = Worklist.back(); + Worklist.pop_back(); + + // If we've already visited this JITDylib on this iteration then continue. + if (JDDepMap.count(DepJD)) + continue; + + // Add dep info. + auto &DM = JDDepMap[DepJD]; + DepJD->withLinkOrderDo([&](const JITDylibSearchOrder &O) { + for (auto &KV : O) { + if (KV.first == DepJD) + continue; + DM.push_back(KV.first); + Worklist.push_back(KV.first); + } + }); + + // Add any registered init symbols. + auto RISItr = RegisteredInitSymbols.find(DepJD); if (RISItr != RegisteredInitSymbols.end()) { - NewInitSymbols[InitJD.get()] = std::move(RISItr->second); + NewInitSymbols[DepJD] = std::move(RISItr->second); RegisteredInitSymbols.erase(RISItr); } } }); - // If there are no further init symbols to look up then move on to the next - // phase. + // If there are no further init symbols to look up then send the link order + // (as a list of header addresses) to the caller. if (NewInitSymbols.empty()) { - getInitializersBuildSequencePhase(std::move(SendResult), JD, - std::move(*DFSLinkOrder)); + + // To make the list intelligible to the runtime we need to convert all + // JITDylib pointers to their header addresses. + DenseMap HeaderAddrs; + HeaderAddrs.reserve(JDDepMap.size()); + { + std::lock_guard Lock(PlatformMutex); + for (auto &KV : JDDepMap) { + auto I = JITDylibToHeaderAddr.find(KV.first); + if (I == JITDylibToHeaderAddr.end()) { + // The header address should have been materialized by the previous + // round, but we need to handle the pathalogical case where someone + // removes the symbol on another thread while we're running. + SendResult( + make_error("JITDylib " + KV.first->getName() + + " has no registered header address", + inconvertibleErrorCode())); + return; + } + HeaderAddrs[KV.first] = I->second; + } + } + + // Build the dep info map to return. + MachOJITDylibDepInfoMap DIM; + DIM.reserve(JDDepMap.size()); + for (auto &KV : JDDepMap) { + assert(HeaderAddrs.count(KV.first) && "Missing header addr"); + auto H = HeaderAddrs[KV.first]; + MachOJITDylibDepInfo DepInfo; + for (auto &Dep : KV.second) { + assert(HeaderAddrs.count(Dep) && "Missing header addr"); + DepInfo.DepHeaders.push_back(HeaderAddrs[Dep]); + } + DIM.push_back(std::make_pair(H, std::move(DepInfo))); + } + SendResult(DIM); return; } @@ -412,58 +480,38 @@ void MachOPlatform::getInitializersLookupPhase( if (Err) SendResult(std::move(Err)); else - getInitializersLookupPhase(std::move(SendResult), JD); + pushInitializersLoop(std::move(SendResult), JD); }, ES, std::move(NewInitSymbols)); } -void MachOPlatform::rt_getInitializers(SendInitializerSequenceFn SendResult, - StringRef JDName) { - LLVM_DEBUG({ - dbgs() << "MachOPlatform::rt_getInitializers(\"" << JDName << "\")\n"; - }); - - JITDylib *JD = ES.getJITDylibByName(JDName); - if (!JD) { - LLVM_DEBUG({ - dbgs() << " No such JITDylib \"" << JDName << "\". Sending error.\n"; - }); - SendResult(make_error("No JITDylib named " + JDName, - inconvertibleErrorCode())); - return; - } - - getInitializersLookupPhase(std::move(SendResult), *JD); -} - -void MachOPlatform::rt_getDeinitializers(SendDeinitializerSequenceFn SendResult, - ExecutorAddr Handle) { - LLVM_DEBUG({ - dbgs() << "MachOPlatform::rt_getDeinitializers(\"" - << formatv("{0:x}", Handle.getValue()) << "\")\n"; - }); - - JITDylib *JD = nullptr; - +void MachOPlatform::rt_pushInitializers(PushInitializersSendResultFn SendResult, + ExecutorAddr JDHeaderAddr) { + JITDylibSP JD; { std::lock_guard Lock(PlatformMutex); - auto I = HeaderAddrToJITDylib.find(Handle); + auto I = HeaderAddrToJITDylib.find(JDHeaderAddr); if (I != HeaderAddrToJITDylib.end()) JD = I->second; } + LLVM_DEBUG({ + dbgs() << "MachOPlatform::rt_pushInitializers(" << JDHeaderAddr << ") "; + if (JD) + dbgs() << "pushing initializers for " << JD->getName() << "\n"; + else + dbgs() << "No JITDylib for header address.\n"; + }); + if (!JD) { - LLVM_DEBUG({ - dbgs() << " No JITDylib for handle " - << formatv("{0:x}", Handle.getValue()) << "\n"; - }); - SendResult(make_error("No JITDylib associated with handle " + - formatv("{0:x}", Handle.getValue()), - inconvertibleErrorCode())); + SendResult( + make_error("No JITDylib with header addr " + + formatv("{0:x}", JDHeaderAddr.getValue()), + inconvertibleErrorCode())); return; } - SendResult(MachOJITDylibDeinitializerSequence()); + pushInitializersLoop(std::move(SendResult), JD); } void MachOPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult, @@ -526,10 +574,14 @@ Error MachOPlatform::bootstrapMachORuntime(JITDylib &PlatformJD) { &orc_rt_macho_platform_bootstrap}, {ES.intern("___orc_rt_macho_platform_shutdown"), &orc_rt_macho_platform_shutdown}, - {ES.intern("___orc_rt_macho_register_thread_data_section"), - &orc_rt_macho_register_thread_data_section}, - {ES.intern("___orc_rt_macho_deregister_thread_data_section"), - &orc_rt_macho_deregister_thread_data_section}, + {ES.intern("___orc_rt_macho_register_jitdylib"), + &orc_rt_macho_register_jitdylib}, + {ES.intern("___orc_rt_macho_deregister_jitdylib"), + &orc_rt_macho_deregister_jitdylib}, + {ES.intern("___orc_rt_macho_register_object_platform_sections"), + &orc_rt_macho_register_object_platform_sections}, + {ES.intern("___orc_rt_macho_deregister_object_platform_sections"), + &orc_rt_macho_deregister_object_platform_sections}, {ES.intern("___orc_rt_macho_create_pthread_key"), &orc_rt_macho_create_pthread_key}})) return Err; @@ -537,45 +589,6 @@ Error MachOPlatform::bootstrapMachORuntime(JITDylib &PlatformJD) { return ES.callSPSWrapper(orc_rt_macho_platform_bootstrap); } -Error MachOPlatform::registerInitInfo( - JITDylib &JD, ExecutorAddr ObjCImageInfoAddr, - ArrayRef InitSections) { - - std::unique_lock Lock(PlatformMutex); - - MachOJITDylibInitializers *InitSeq = nullptr; - { - auto I = InitSeqs.find(&JD); - if (I == InitSeqs.end()) { - // If there's no init sequence entry yet then we need to look up the - // header symbol to force creation of one. - Lock.unlock(); - - auto SearchOrder = - JD.withLinkOrderDo([](const JITDylibSearchOrder &SO) { return SO; }); - if (auto Err = ES.lookup(SearchOrder, MachOHeaderStartSymbol).takeError()) - return Err; - - Lock.lock(); - I = InitSeqs.find(&JD); - assert(I != InitSeqs.end() && - "Entry missing after header symbol lookup?"); - } - InitSeq = &I->second; - } - - InitSeq->ObjCImageInfoAddress = ObjCImageInfoAddr; - - for (auto *Sec : InitSections) { - // FIXME: Avoid copy here. - jitlink::SectionRange R(*Sec); - InitSeq->InitSections[Sec->getName()].push_back( - {ExecutorAddr(R.getStart()), ExecutorAddr(R.getEnd())}); - } - - return Error::success(); -} - Expected MachOPlatform::createPThreadKey() { if (!orc_rt_macho_create_pthread_key) return make_error( @@ -617,11 +630,6 @@ void MachOPlatform::MachOPlatformPlugin::modifyPassConfig( return Err; return processObjCImageInfo(G, MR); }); - - Config.PostFixupPasses.push_back( - [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) { - return registerInitSections(G, JD); - }); } // --- Add passes for eh-frame and TLV support --- @@ -639,10 +647,12 @@ void MachOPlatform::MachOPlatformPlugin::modifyPassConfig( return fixTLVSectionsAndEdges(G, JD); }); - // Add a pass to register the final addresses of the eh-frame and TLV sections - // with the runtime. - Config.PostFixupPasses.push_back( - [this](jitlink::LinkGraph &G) { return registerEHAndTLVSections(G); }); + // Add a pass to register the final addresses of any special sections in the + // object with the runtime. + Config.PostAllocationPasses.push_back( + [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) { + return registerObjectPlatformSections(G, JD); + }); } ObjectLinkingLayer::Plugin::SyntheticSymbolDependenciesMap @@ -661,7 +671,6 @@ MachOPlatform::MachOPlatformPlugin::getSyntheticSymbolDependencies( Error MachOPlatform::MachOPlatformPlugin::associateJITDylibHeaderSymbol( jitlink::LinkGraph &G, MaterializationResponsibility &MR) { - auto I = llvm::find_if(G.defined_symbols(), [this](jitlink::Symbol *Sym) { return Sym->getName() == *MP.MachOHeaderStartSymbol; }); @@ -670,10 +679,14 @@ Error MachOPlatform::MachOPlatformPlugin::associateJITDylibHeaderSymbol( auto &JD = MR.getTargetJITDylib(); std::lock_guard Lock(MP.PlatformMutex); auto HeaderAddr = (*I)->getAddress(); + MP.JITDylibToHeaderAddr[&JD] = HeaderAddr; MP.HeaderAddrToJITDylib[HeaderAddr] = &JD; - assert(!MP.InitSeqs.count(&JD) && "InitSeq entry for JD already exists"); - MP.InitSeqs.insert( - std::make_pair(&JD, MachOJITDylibInitializers(JD.getName(), HeaderAddr))); + G.allocActions().push_back( + {cantFail( + WrapperFunctionCall::Create>( + MP.orc_rt_macho_register_jitdylib, JD.getName(), HeaderAddr)), + cantFail(WrapperFunctionCall::Create>( + MP.orc_rt_macho_deregister_jitdylib, HeaderAddr))}); return Error::success(); } @@ -792,37 +805,6 @@ Error MachOPlatform::MachOPlatformPlugin::processObjCImageInfo( return Error::success(); } -Error MachOPlatform::MachOPlatformPlugin::registerInitSections( - jitlink::LinkGraph &G, JITDylib &JD) { - - ExecutorAddr ObjCImageInfoAddr; - SmallVector InitSections; - - if (auto *ObjCImageInfoSec = G.findSectionByName(ObjCImageInfoSectionName)) { - if (auto Addr = jitlink::SectionRange(*ObjCImageInfoSec).getStart()) - ObjCImageInfoAddr = Addr; - } - - for (auto InitSectionName : InitSectionNames) - if (auto *Sec = G.findSectionByName(InitSectionName)) - InitSections.push_back(Sec); - - // Dump the scraped inits. - LLVM_DEBUG({ - dbgs() << "MachOPlatform: Scraped " << G.getName() << " init sections:\n"; - if (ObjCImageInfoAddr) - dbgs() << " " << ObjCImageInfoSectionName << ": " - << formatv("{0:x}", ObjCImageInfoAddr.getValue()) << "\n"; - for (auto *Sec : InitSections) { - jitlink::SectionRange R(*Sec); - dbgs() << " " << Sec->getName() << ": " - << formatv("[ {0:x} -- {1:x} ]", R.getStart(), R.getEnd()) << "\n"; - } - }); - - return MP.registerInitInfo(JD, ObjCImageInfoAddr, InitSections); -} - Error MachOPlatform::MachOPlatformPlugin::fixTLVSectionsAndEdges( jitlink::LinkGraph &G, JITDylib &JD) { @@ -879,11 +861,10 @@ Error MachOPlatform::MachOPlatformPlugin::fixTLVSectionsAndEdges( return Error::success(); } -Error MachOPlatform::MachOPlatformPlugin::registerEHAndTLVSections( - jitlink::LinkGraph &G) { +Error MachOPlatform::MachOPlatformPlugin::registerObjectPlatformSections( + jitlink::LinkGraph &G, JITDylib &JD) { - // Add a pass to register the final addresses of the eh-frame and TLV sections - // with the runtime. + // Add an action to register the eh-frame. if (auto *EHFrameSection = G.findSectionByName(EHFrameSectionName)) { jitlink::SectionRange R(*EHFrameSection); if (!R.empty()) @@ -912,6 +893,8 @@ Error MachOPlatform::MachOPlatformPlugin::registerEHAndTLVSections( ThreadDataSection = ThreadBSSSection; } + SmallVector, 8> MachOPlatformSecs; + // Having merged thread BSS (if present) and thread data (if present), // record the resulting section range. if (ThreadDataSection) { @@ -922,16 +905,64 @@ Error MachOPlatform::MachOPlatformPlugin::registerEHAndTLVSections( "MachOPlatform has not finished booting", inconvertibleErrorCode()); - G.allocActions().push_back( - {cantFail( - WrapperFunctionCall::Create>( - MP.orc_rt_macho_register_thread_data_section, R.getRange())), - cantFail( - WrapperFunctionCall::Create>( - MP.orc_rt_macho_deregister_thread_data_section, - R.getRange()))}); + MachOPlatformSecs.push_back({ThreadDataSectionName, R.getRange()}); + } + } + + // If any platform sections were found then add an allocation action to call + // the registration function. + StringRef PlatformSections[] = { + ModInitFuncSectionName, ObjCClassListSectionName, + ObjCImageInfoSectionName, ObjCSelRefsSectionName, + Swift5ProtoSectionName, Swift5ProtosSectionName, + Swift5TypesSectionName, + }; + + for (auto &SecName : PlatformSections) { + auto *Sec = G.findSectionByName(SecName); + if (!Sec) + continue; + jitlink::SectionRange R(*Sec); + if (R.empty()) + continue; + + MachOPlatformSecs.push_back({SecName, R.getRange()}); + } + + if (!MachOPlatformSecs.empty()) { + Optional HeaderAddr; + { + std::lock_guard Lock(MP.PlatformMutex); + auto I = MP.JITDylibToHeaderAddr.find(&JD); + if (I != MP.JITDylibToHeaderAddr.end()) + HeaderAddr = I->second; } + + if (!HeaderAddr) + return make_error("Missing header for " + JD.getName(), + inconvertibleErrorCode()); + + // Dump the scraped inits. + LLVM_DEBUG({ + dbgs() << "MachOPlatform: Scraped " << G.getName() << " init sections:\n"; + for (auto &KV : MachOPlatformSecs) + dbgs() << " " << KV.first << ": " << KV.second << "\n"; + }); + + using SPSRegisterObjectPlatformSectionsArgs = + SPSArgList>>; + G.allocActions().push_back( + {cantFail( + WrapperFunctionCall::Create( + MP.orc_rt_macho_register_object_platform_sections, *HeaderAddr, + MachOPlatformSecs)), + cantFail( + WrapperFunctionCall::Create( + MP.orc_rt_macho_deregister_object_platform_sections, + *HeaderAddr, MachOPlatformSecs))}); } + return Error::success(); }