diff --git a/compiler-rt/lib/orc/CMakeLists.txt b/compiler-rt/lib/orc/CMakeLists.txt index c49f653dc992f6..ca8e6bb267506e 100644 --- a/compiler-rt/lib/orc/CMakeLists.txt +++ b/compiler-rt/lib/orc/CMakeLists.txt @@ -4,6 +4,7 @@ set(ORC_SOURCES extensible_rtti.cpp log_error_to_stderr.cpp + macho_ehframe_registration.cpp macho_platform.cpp elfnix_platform.cpp run_program_wrapper.cpp diff --git a/compiler-rt/lib/orc/macho_platform.cpp b/compiler-rt/lib/orc/macho_platform.cpp index e25b2a9568180c..b36f9df1c3c246 100644 --- a/compiler-rt/lib/orc/macho_platform.cpp +++ b/compiler-rt/lib/orc/macho_platform.cpp @@ -29,11 +29,6 @@ ORC_RT_JIT_DISPATCH_TAG(__orc_rt_macho_get_initializers_tag) ORC_RT_JIT_DISPATCH_TAG(__orc_rt_macho_get_deinitializers_tag) ORC_RT_JIT_DISPATCH_TAG(__orc_rt_macho_symbol_lookup_tag) -// eh-frame registration functions. -// We expect these to be available for all processes. -extern "C" void __register_frame(const void *); -extern "C" void __deregister_frame(const void *); - // Objective-C types. struct objc_class; struct objc_image_info; @@ -66,28 +61,6 @@ extern "C" void swift_registerProtocolConformances( namespace { -template -void walkEHFrameSection(span EHFrameSection, - HandleFDEFn HandleFDE) { - const char *CurCFIRecord = EHFrameSection.data(); - uint64_t Size = *reinterpret_cast(CurCFIRecord); - - while (CurCFIRecord != EHFrameSection.end() && Size != 0) { - const char *OffsetField = CurCFIRecord + (Size == 0xffffffff ? 12 : 4); - if (Size == 0xffffffff) - Size = *reinterpret_cast(CurCFIRecord + 4) + 12; - else - Size += 4; - uint32_t Offset = *reinterpret_cast(OffsetField); - - if (Offset != 0) - HandleFDE(CurCFIRecord); - - CurCFIRecord += Size; - Size = *reinterpret_cast(CurCFIRecord); - } -} - Error validatePointerSectionExtent(const char *SectionName, const ExecutorAddrRange &SE) { if (SE.size().getValue() % sizeof(uintptr_t)) { @@ -250,8 +223,8 @@ class MachOPlatformRuntimeState { MachOPlatformRuntimeState(MachOPlatformRuntimeState &&) = delete; MachOPlatformRuntimeState &operator=(MachOPlatformRuntimeState &&) = delete; - Error registerObjectSections(MachOPerObjectSectionsToRegister POSR); - Error deregisterObjectSections(MachOPerObjectSectionsToRegister POSR); + Error registerThreadDataSection(span ThreadDataSec); + Error deregisterThreadDataSection(span ThreadDataSec); const char *dlerror(); void *dlopen(string_view Name, int Mode); @@ -270,8 +243,6 @@ class MachOPlatformRuntimeState { PerJITDylibState *getJITDylibStateByName(string_view Path); PerJITDylibState &getOrCreateJITDylibState(MachOJITDylibInitializers &MOJDIs); - Error registerThreadDataSection(span ThreadDataSec); - Expected lookupSymbolInJITDylib(void *DSOHandle, string_view Symbol); @@ -320,27 +291,28 @@ void MachOPlatformRuntimeState::destroy() { delete MOPS; } -Error MachOPlatformRuntimeState::registerObjectSections( - MachOPerObjectSectionsToRegister POSR) { - if (POSR.EHFrameSection.Start) - walkEHFrameSection(POSR.EHFrameSection.toSpan(), - __register_frame); - - if (POSR.ThreadDataSection.Start) { - if (auto Err = registerThreadDataSection( - POSR.ThreadDataSection.toSpan())) - return Err; +Error MachOPlatformRuntimeState::registerThreadDataSection( + span ThreadDataSection) { + std::lock_guard Lock(ThreadDataSectionsMutex); + auto I = ThreadDataSections.upper_bound(ThreadDataSection.data()); + if (I != ThreadDataSections.begin()) { + auto J = std::prev(I); + if (J->first + J->second > ThreadDataSection.data()) + return make_error("Overlapping __thread_data sections"); } - + ThreadDataSections.insert( + I, std::make_pair(ThreadDataSection.data(), ThreadDataSection.size())); return Error::success(); } -Error MachOPlatformRuntimeState::deregisterObjectSections( - MachOPerObjectSectionsToRegister POSR) { - if (POSR.EHFrameSection.Start) - walkEHFrameSection(POSR.EHFrameSection.toSpan(), - __deregister_frame); - +Error MachOPlatformRuntimeState::deregisterThreadDataSection( + span ThreadDataSection) { + std::lock_guard Lock(ThreadDataSectionsMutex); + auto I = ThreadDataSections.find(ThreadDataSection.end()); + if (I == ThreadDataSections.end()) + return make_error("Attempt to deregister unknown thread data " + "section"); + ThreadDataSections.erase(I); return Error::success(); } @@ -462,20 +434,6 @@ MachOPlatformRuntimeState::getOrCreateJITDylibState( return JDS; } -Error MachOPlatformRuntimeState::registerThreadDataSection( - span ThreadDataSection) { - std::lock_guard Lock(ThreadDataSectionsMutex); - auto I = ThreadDataSections.upper_bound(ThreadDataSection.data()); - if (I != ThreadDataSections.begin()) { - auto J = std::prev(I); - if (J->first + J->second > ThreadDataSection.data()) - return make_error("Overlapping __thread_data sections"); - } - ThreadDataSections.insert( - I, std::make_pair(ThreadDataSection.data(), ThreadDataSection.size())); - return Error::success(); -} - Expected MachOPlatformRuntimeState::lookupSymbolInJITDylib(void *DSOHandle, string_view Sym) { @@ -587,6 +545,13 @@ void destroyMachOTLVMgr(void *MachOTLVMgr) { delete static_cast(MachOTLVMgr); } +Error runWrapperFunctionCalls(std::vector WFCs) { + for (auto &WFC : WFCs) + if (auto Err = WFC.runWithSPSRet()) + return Err; + return Error::success(); +} + } // end anonymous namespace //------------------------------------------------------------------------------ @@ -605,30 +570,41 @@ __orc_rt_macho_platform_shutdown(char *ArgData, size_t ArgSize) { return WrapperFunctionResult().release(); } -/// Wrapper function for registering metadata on a per-object basis. ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult -__orc_rt_macho_register_object_sections(char *ArgData, size_t ArgSize) { - return WrapperFunction::handle( - ArgData, ArgSize, - [](MachOPerObjectSectionsToRegister &POSR) { - return MachOPlatformRuntimeState::get().registerObjectSections( - std::move(POSR)); +__orc_rt_macho_register_thread_data_section(char *ArgData, size_t ArgSize) { + // NOTE: Does not use SPS to deserialize arg buffer, instead the arg buffer + // is taken to be the range of the thread data section. + return WrapperFunction::handle( + nullptr, 0, + [&]() { + return MachOPlatformRuntimeState::get() + .registerThreadDataSection( + span(ArgData, ArgSize)); }) .release(); } -/// Wrapper for releasing per-object metadat. ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult -__orc_rt_macho_deregister_object_sections(char *ArgData, size_t ArgSize) { - return WrapperFunction::handle( - ArgData, ArgSize, - [](MachOPerObjectSectionsToRegister &POSR) { - return MachOPlatformRuntimeState::get().deregisterObjectSections( - std::move(POSR)); +__orc_rt_macho_deregister_thread_data_section(char *ArgData, size_t ArgSize) { + // NOTE: Does not use SPS to deserialize arg buffer, instead the arg buffer + // is taken to be the range of the thread data section. + return WrapperFunction::handle( + nullptr, 0, + [&]() { + return MachOPlatformRuntimeState::get() + .deregisterThreadDataSection( + span(ArgData, ArgSize)); }) .release(); } +ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult +__orc_rt_macho_run_wrapper_function_calls(char *ArgData, size_t ArgSize) { + return WrapperFunction)>::handle( + ArgData, ArgSize, runWrapperFunctionCalls) + .release(); +} + //------------------------------------------------------------------------------ // TLV support //------------------------------------------------------------------------------ diff --git a/compiler-rt/lib/orc/macho_platform.h b/compiler-rt/lib/orc/macho_platform.h index 7f0edc651f349d..5b9820a0d1f9fa 100644 --- a/compiler-rt/lib/orc/macho_platform.h +++ b/compiler-rt/lib/orc/macho_platform.h @@ -31,11 +31,6 @@ ORC_RT_INTERFACE void *__orc_rt_macho_jit_dlsym(void *dso_handle, namespace __orc_rt { namespace macho { -struct MachOPerObjectSectionsToRegister { - ExecutorAddrRange EHFrameSection; - ExecutorAddrRange ThreadDataSection; -}; - struct MachOJITDylibInitializers { using SectionList = std::vector; @@ -67,32 +62,6 @@ enum dlopen_mode : int { } // end namespace macho -using SPSMachOPerObjectSectionsToRegister = - SPSTuple; - -template <> -class SPSSerializationTraits { - -public: - static size_t size(const macho::MachOPerObjectSectionsToRegister &MOPOSR) { - return SPSMachOPerObjectSectionsToRegister::AsArgList::size( - MOPOSR.EHFrameSection, MOPOSR.ThreadDataSection); - } - - static bool serialize(SPSOutputBuffer &OB, - const macho::MachOPerObjectSectionsToRegister &MOPOSR) { - return SPSMachOPerObjectSectionsToRegister::AsArgList::serialize( - OB, MOPOSR.EHFrameSection, MOPOSR.ThreadDataSection); - } - - static bool deserialize(SPSInputBuffer &IB, - macho::MachOPerObjectSectionsToRegister &MOPOSR) { - return SPSMachOPerObjectSectionsToRegister::AsArgList::deserialize( - IB, MOPOSR.EHFrameSection, MOPOSR.ThreadDataSection); - } -}; - using SPSNamedExecutorAddrRangeSequenceMap = SPSSequence>; diff --git a/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h b/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h index 5e062601464985..d7b5e2eda6ee8b 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h @@ -26,11 +26,6 @@ namespace llvm { namespace orc { -struct MachOPerObjectSectionsToRegister { - ExecutorAddrRange EHFrameSection; - ExecutorAddrRange ThreadDataSection; -}; - struct MachOJITDylibInitializers { using SectionList = std::vector; @@ -171,6 +166,8 @@ class MachOPlatform : public Platform { Error registerEHAndTLVSections(jitlink::LinkGraph &G); + Error registerEHSectionsPhase1(jitlink::LinkGraph &G); + std::mutex PluginMutex; MachOPlatform &MP; DenseMap> ObjCImageInfos; @@ -217,19 +214,22 @@ class MachOPlatform : public Platform { Error registerInitInfo(JITDylib &JD, ExecutorAddr ObjCImageInfoAddr, ArrayRef InitSections); - Error registerPerObjectSections(const MachOPerObjectSectionsToRegister &POSR); - Expected createPThreadKey(); + enum PlatformState { BootstrapPhase1, BootstrapPhase2, Initialized }; + ExecutionSession &ES; ObjectLinkingLayer &ObjLinkingLayer; SymbolStringPtr MachOHeaderStartSymbol; - std::atomic RuntimeBootstrapped{false}; + std::atomic State{BootstrapPhase1}; ExecutorAddr orc_rt_macho_platform_bootstrap; ExecutorAddr orc_rt_macho_platform_shutdown; - ExecutorAddr orc_rt_macho_register_object_sections; + ExecutorAddr orc_rt_macho_register_ehframe_section; + ExecutorAddr orc_rt_macho_deregister_ehframe_section; + ExecutorAddr orc_rt_macho_register_thread_data_section; + ExecutorAddr orc_rt_macho_deregister_thread_data_section; ExecutorAddr orc_rt_macho_create_pthread_key; DenseMap RegisteredInitSymbols; @@ -238,7 +238,6 @@ class MachOPlatform : public Platform { // aggregating data from the jitlink. std::mutex PlatformMutex; DenseMap InitSeqs; - std::vector BootstrapPOSRs; DenseMap HeaderAddrToJITDylib; DenseMap JITDylibToPThreadKey; @@ -246,32 +245,6 @@ class MachOPlatform : public Platform { namespace shared { -using SPSMachOPerObjectSectionsToRegister = - SPSTuple; - -template <> -class SPSSerializationTraits { - -public: - static size_t size(const MachOPerObjectSectionsToRegister &MOPOSR) { - return SPSMachOPerObjectSectionsToRegister::AsArgList::size( - MOPOSR.EHFrameSection, MOPOSR.ThreadDataSection); - } - - static bool serialize(SPSOutputBuffer &OB, - const MachOPerObjectSectionsToRegister &MOPOSR) { - return SPSMachOPerObjectSectionsToRegister::AsArgList::serialize( - OB, MOPOSR.EHFrameSection, MOPOSR.ThreadDataSection); - } - - static bool deserialize(SPSInputBuffer &IB, - MachOPerObjectSectionsToRegister &MOPOSR) { - return SPSMachOPerObjectSectionsToRegister::AsArgList::deserialize( - IB, MOPOSR.EHFrameSection, MOPOSR.ThreadDataSection); - } -}; - using SPSNamedExecutorAddrRangeSequenceMap = SPSSequence>; diff --git a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp index 9f8cfd81430e3d..ad92d82e029c69 100644 --- a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp +++ b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp @@ -288,6 +288,19 @@ MachOPlatform::MachOPlatform( PlatformJD.addGenerator(std::move(OrcRuntimeGenerator)); + // Force linking of eh-frame registration functions. + if (auto Err2 = lookupAndRecordAddrs( + ES, LookupKind::Static, makeJITDylibSearchOrder(&PlatformJD), + {{ES.intern("___orc_rt_macho_register_ehframe_section"), + &orc_rt_macho_register_ehframe_section}, + {ES.intern("___orc_rt_macho_deregister_ehframe_section"), + &orc_rt_macho_deregister_ehframe_section}})) { + Err = std::move(Err2); + return; + } + + State = BootstrapPhase2; + // PlatformJD hasn't been 'set-up' by the platform yet (since we're creating // the platform now), so set it up. if (auto E2 = setupJITDylib(PlatformJD)) { @@ -311,6 +324,8 @@ MachOPlatform::MachOPlatform( Err = std::move(E2); return; } + + State = Initialized; } Error MachOPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) { @@ -496,37 +511,21 @@ void MachOPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult, } Error MachOPlatform::bootstrapMachORuntime(JITDylib &PlatformJD) { - if (auto Err = lookupAndRecordAddrs( ES, LookupKind::Static, makeJITDylibSearchOrder(&PlatformJD), {{ES.intern("___orc_rt_macho_platform_bootstrap"), &orc_rt_macho_platform_bootstrap}, {ES.intern("___orc_rt_macho_platform_shutdown"), &orc_rt_macho_platform_shutdown}, - {ES.intern("___orc_rt_macho_register_object_sections"), - &orc_rt_macho_register_object_sections}, + {ES.intern("___orc_rt_macho_register_thread_data_section"), + &orc_rt_macho_register_thread_data_section}, + {ES.intern("___orc_rt_macho_deregister_thread_data_section"), + &orc_rt_macho_deregister_thread_data_section}, {ES.intern("___orc_rt_macho_create_pthread_key"), &orc_rt_macho_create_pthread_key}})) return Err; - if (auto Err = ES.callSPSWrapper(orc_rt_macho_platform_bootstrap)) - return Err; - - // FIXME: Ordering is fuzzy here. We're probably best off saying - // "behavior is undefined if code that uses the runtime is added before - // the platform constructor returns", then move all this to the constructor. - RuntimeBootstrapped = true; - std::vector DeferredPOSRs; - { - std::lock_guard Lock(PlatformMutex); - DeferredPOSRs = std::move(BootstrapPOSRs); - } - - for (auto &D : DeferredPOSRs) - if (auto Err = registerPerObjectSections(D)) - return Err; - - return Error::success(); + return ES.callSPSWrapper(orc_rt_macho_platform_bootstrap); } Error MachOPlatform::registerInitInfo( @@ -568,23 +567,6 @@ Error MachOPlatform::registerInitInfo( return Error::success(); } -Error MachOPlatform::registerPerObjectSections( - const MachOPerObjectSectionsToRegister &POSR) { - - if (!orc_rt_macho_register_object_sections) - return make_error("Attempting to register per-object " - "sections, but runtime support has not " - "been loaded yet", - inconvertibleErrorCode()); - - Error ErrResult = Error::success(); - if (auto Err = ES.callSPSWrapper( - orc_rt_macho_register_object_sections, ErrResult, POSR)) - return Err; - return ErrResult; -} - Expected MachOPlatform::createPThreadKey() { if (!orc_rt_macho_create_pthread_key) return make_error( @@ -603,6 +585,8 @@ void MachOPlatform::MachOPlatformPlugin::modifyPassConfig( MaterializationResponsibility &MR, jitlink::LinkGraph &LG, jitlink::PassConfiguration &Config) { + auto PS = MP.State.load(); + // --- Handle Initializers --- if (auto InitSymbol = MR.getInitializerSymbol()) { @@ -632,6 +616,11 @@ void MachOPlatform::MachOPlatformPlugin::modifyPassConfig( } // --- Add passes for eh-frame and TLV support --- + if (PS == MachOPlatform::BootstrapPhase1) { + Config.PostFixupPasses.push_back( + [this](jitlink::LinkGraph &G) { return registerEHSectionsPhase1(G); }); + return; + } // Insert TLV lowering at the start of the PostPrunePasses, since we want // it to run before GOT/PLT lowering. @@ -883,13 +872,17 @@ Error MachOPlatform::MachOPlatformPlugin::fixTLVSectionsAndEdges( Error MachOPlatform::MachOPlatformPlugin::registerEHAndTLVSections( jitlink::LinkGraph &G) { - MachOPerObjectSectionsToRegister POSR; + // Add a pass to register the final addresses of the eh-frame and TLV sections + // with the runtime. if (auto *EHFrameSection = G.findSectionByName(EHFrameSectionName)) { jitlink::SectionRange R(*EHFrameSection); if (!R.empty()) - POSR.EHFrameSection = {ExecutorAddr(R.getStart()), - ExecutorAddr(R.getEnd())}; + G.allocActions().push_back( + {{MP.orc_rt_macho_register_ehframe_section.getValue(), R.getStart(), + R.getSize()}, + {MP.orc_rt_macho_deregister_ehframe_section.getValue(), R.getStart(), + R.getSize()}}); } // Get a pointer to the thread data section if there is one. It will be used @@ -912,26 +905,68 @@ Error MachOPlatform::MachOPlatformPlugin::registerEHAndTLVSections( // record the resulting section range. if (ThreadDataSection) { jitlink::SectionRange R(*ThreadDataSection); - if (!R.empty()) - POSR.ThreadDataSection = {ExecutorAddr(R.getStart()), - ExecutorAddr(R.getEnd())}; + if (!R.empty()) { + if (MP.State != MachOPlatform::Initialized) + return make_error("__thread_data section encountered, but " + "MachOPlatform has not finished booting", + inconvertibleErrorCode()); + + G.allocActions().push_back( + {{MP.orc_rt_macho_register_thread_data_section.getValue(), + R.getStart(), R.getSize()}, + {MP.orc_rt_macho_deregister_thread_data_section.getValue(), + R.getStart(), R.getSize()}}); + } } + return Error::success(); +} + +Error MachOPlatform::MachOPlatformPlugin::registerEHSectionsPhase1( + jitlink::LinkGraph &G) { - if (POSR.EHFrameSection.Start || POSR.ThreadDataSection.Start) { + // If there's no eh-frame there's nothing to do. + auto *EHFrameSection = G.findSectionByName(EHFrameSectionName); + if (!EHFrameSection) + return Error::success(); - // If we're still bootstrapping the runtime then just record this - // frame for now. - if (!MP.RuntimeBootstrapped) { - std::lock_guard Lock(MP.PlatformMutex); - MP.BootstrapPOSRs.push_back(POSR); - return Error::success(); - } + // If the eh-frame section is empty there's nothing to do. + jitlink::SectionRange R(*EHFrameSection); + if (R.empty()) + return Error::success(); - // Otherwise register it immediately. - if (auto Err = MP.registerPerObjectSections(POSR)) - return Err; + // Since we're linking the object containing the registration code now the + // addresses won't be ready in the platform. We'll have to find them in this + // graph instead. + ExecutorAddr orc_rt_macho_register_ehframe_section; + ExecutorAddr orc_rt_macho_deregister_ehframe_section; + for (auto *Sym : G.defined_symbols()) { + if (!Sym->hasName()) + continue; + if (Sym->getName() == "___orc_rt_macho_register_ehframe_section") + orc_rt_macho_register_ehframe_section = ExecutorAddr(Sym->getAddress()); + else if (Sym->getName() == "___orc_rt_macho_deregister_ehframe_section") + orc_rt_macho_deregister_ehframe_section = ExecutorAddr(Sym->getAddress()); + + if (orc_rt_macho_register_ehframe_section && + orc_rt_macho_deregister_ehframe_section) + break; } + // If we failed to find the required functions then bail out. + if (!orc_rt_macho_register_ehframe_section || + !orc_rt_macho_deregister_ehframe_section) + return make_error("Could not find eh-frame registration " + "functions during platform bootstrap", + inconvertibleErrorCode()); + + // Otherwise, add allocation actions to the graph to register eh-frames for + // this object. + G.allocActions().push_back( + {{orc_rt_macho_register_ehframe_section.getValue(), R.getStart(), + R.getSize()}, + {orc_rt_macho_deregister_ehframe_section.getValue(), R.getStart(), + R.getSize()}}); + return Error::success(); }