diff --git a/compiler-rt/lib/orc/macho_platform.cpp b/compiler-rt/lib/orc/macho_platform.cpp index 5a5c8e49e0b102..9b5b954921c1f9 100644 --- a/compiler-rt/lib/orc/macho_platform.cpp +++ b/compiler-rt/lib/orc/macho_platform.cpp @@ -71,6 +71,26 @@ extern "C" void swift_registerTypeMetadataRecords( const TypeMetadataRecord *begin, const TypeMetadataRecord *end) ORC_RT_WEAK_IMPORT; +// Libunwind prototypes. +struct unw_dynamic_unwind_sections { + uintptr_t dso_base; + uintptr_t dwarf_section; + size_t dwarf_section_length; + uintptr_t compact_unwind_section; + size_t compact_unwind_section_length; +}; + +typedef int (*unw_find_dynamic_unwind_sections)( + uintptr_t addr, struct unw_dynamic_unwind_sections *info); + +extern "C" int __unw_add_find_dynamic_unwind_sections( + unw_find_dynamic_unwind_sections find_dynamic_unwind_sections) + ORC_RT_WEAK_IMPORT; + +extern "C" int __unw_remove_find_dynamic_unwind_sections( + unw_find_dynamic_unwind_sections find_dynamic_unwind_sections) + ORC_RT_WEAK_IMPORT; + namespace { struct MachOJITDylibDepInfo { @@ -111,6 +131,35 @@ class SPSSerializationTraits { } }; +struct UnwindSectionInfo { + std::vector CodeRanges; + ExecutorAddrRange DwarfSection; + ExecutorAddrRange CompactUnwindSection; +}; + +using SPSUnwindSectionInfo = + SPSTuple, SPSExecutorAddrRange, + SPSExecutorAddrRange>; + +template <> +class SPSSerializationTraits { +public: + static size_t size(const UnwindSectionInfo &USI) { + return SPSUnwindSectionInfo::AsArgList::size( + USI.CodeRanges, USI.DwarfSection, USI.CompactUnwindSection); + } + + static bool serialize(SPSOutputBuffer &OB, const UnwindSectionInfo &USI) { + return SPSUnwindSectionInfo::AsArgList::serialize( + OB, USI.CodeRanges, USI.DwarfSection, USI.CompactUnwindSection); + } + + static bool deserialize(SPSInputBuffer &IB, UnwindSectionInfo &USI) { + return SPSUnwindSectionInfo::AsArgList::deserialize( + IB, USI.CodeRanges, USI.DwarfSection, USI.CompactUnwindSection); + } +}; + } // namespace __orc_rt namespace { @@ -216,6 +265,18 @@ class MachOPlatformRuntimeState { std::vector> New; }; + struct UnwindSections { + UnwindSections(const UnwindSectionInfo &USI) + : DwarfSection(USI.DwarfSection.toSpan()), + CompactUnwindSection(USI.CompactUnwindSection.toSpan()) {} + + span DwarfSection; + span CompactUnwindSection; + }; + + using UnwindSectionsMap = + IntervalMap; + struct JITDylibState { std::string Name; void *Header = nullptr; @@ -227,6 +288,7 @@ class MachOPlatformRuntimeState { const objc_image_info *ObjCImageInfo = nullptr; std::unordered_map> DataSectionContent; std::unordered_map ZeroInitRanges; + UnwindSectionsMap UnwindSections; RecordSectionsTracker ModInitsSections; RecordSectionsTracker ObjCClassListSections; RecordSectionsTracker ObjCSelRefsSections; @@ -240,9 +302,9 @@ class MachOPlatformRuntimeState { }; public: - static void initialize(); + static Error create(); static MachOPlatformRuntimeState &get(); - static void destroy(); + static Error destroy(); MachOPlatformRuntimeState() = default; @@ -253,15 +315,18 @@ class MachOPlatformRuntimeState { MachOPlatformRuntimeState(MachOPlatformRuntimeState &&) = delete; MachOPlatformRuntimeState &operator=(MachOPlatformRuntimeState &&) = delete; + Error initialize(); + Error shutdown(); + Error registerJITDylib(std::string Name, void *Header); Error deregisterJITDylib(void *Header); Error registerThreadDataSection(span ThreadDataSection); Error deregisterThreadDataSection(span ThreadDataSection); Error registerObjectPlatformSections( - ExecutorAddr HeaderAddr, + ExecutorAddr HeaderAddr, std::optional UnwindSections, std::vector> Secs); Error deregisterObjectPlatformSections( - ExecutorAddr HeaderAddr, + ExecutorAddr HeaderAddr, std::optional UnwindSections, std::vector> Secs); const char *dlerror(); @@ -285,6 +350,10 @@ class MachOPlatformRuntimeState { Expected lookupSymbolInJITDylib(void *DSOHandle, std::string_view Symbol); + bool lookupUnwindSections(void *Addr, unw_dynamic_unwind_sections &Info); + + static int findDynamicUnwindSections(uintptr_t addr, + unw_dynamic_unwind_sections *info); static Error registerEHFrames(span EHFrameSection); static Error deregisterEHFrames(span EHFrameSection); @@ -308,6 +377,8 @@ class MachOPlatformRuntimeState { static MachOPlatformRuntimeState *MOPS; + bool UseCallbackStyleUnwindInfo = false; + // FIXME: Move to thread-state. std::string DLFcnError; @@ -327,9 +398,10 @@ class MachOPlatformRuntimeState { MachOPlatformRuntimeState *MachOPlatformRuntimeState::MOPS = nullptr; -void MachOPlatformRuntimeState::initialize() { +Error MachOPlatformRuntimeState::create() { assert(!MOPS && "MachOPlatformRuntimeState should be null"); MOPS = new MachOPlatformRuntimeState(); + return MOPS->initialize(); } MachOPlatformRuntimeState &MachOPlatformRuntimeState::get() { @@ -337,9 +409,42 @@ MachOPlatformRuntimeState &MachOPlatformRuntimeState::get() { return *MOPS; } -void MachOPlatformRuntimeState::destroy() { +Error MachOPlatformRuntimeState::destroy() { assert(MOPS && "MachOPlatformRuntimeState not initialized"); + auto Err = MOPS->shutdown(); delete MOPS; + return Err; +} + +Error MachOPlatformRuntimeState::initialize() { + UseCallbackStyleUnwindInfo = __unw_add_find_dynamic_unwind_sections && + __unw_remove_find_dynamic_unwind_sections; + if (UseCallbackStyleUnwindInfo) { + ORC_RT_DEBUG({ + printdbg("__unw_add/remove_find_dynamic_unwind_sections available." + " Using callback-based frame info lookup.\n"); + }); + if (__unw_add_find_dynamic_unwind_sections(&findDynamicUnwindSections)) + return make_error( + "Could not register findDynamicUnwindSections"); + } else { + ORC_RT_DEBUG({ + printdbg("__unw_add/remove_find_dynamic_unwind_sections not available." + " Using classic frame info registration.\n"); + }); + } + return Error::success(); +} + +Error MachOPlatformRuntimeState::shutdown() { + if (__unw_add_find_dynamic_unwind_sections && + __unw_remove_find_dynamic_unwind_sections) { + if (__unw_remove_find_dynamic_unwind_sections(&findDynamicUnwindSections)) { + ORC_RT_DEBUG( + { printdbg("__unw_remove_find_dynamic_unwind_sections failed.\n"); }); + } + } + return Error::success(); } Error MachOPlatformRuntimeState::registerJITDylib(std::string Name, @@ -419,7 +524,7 @@ Error MachOPlatformRuntimeState::deregisterThreadDataSection( } Error MachOPlatformRuntimeState::registerObjectPlatformSections( - ExecutorAddr HeaderAddr, + ExecutorAddr HeaderAddr, std::optional UnwindInfo, std::vector> Secs) { // FIXME: Reject platform section registration after the JITDylib is @@ -440,11 +545,35 @@ Error MachOPlatformRuntimeState::registerObjectPlatformSections( return make_error(ErrStream.str()); } + if (UnwindInfo && UseCallbackStyleUnwindInfo) { + ORC_RT_DEBUG({ + printdbg(" Registering new-style unwind info for:\n" + " DWARF: %p -- %p\n" + " Compact-unwind: %p -- %p\n" + " for:\n", + UnwindInfo->DwarfSection.Start.toPtr(), + UnwindInfo->DwarfSection.End.toPtr(), + UnwindInfo->CompactUnwindSection.Start.toPtr(), + UnwindInfo->CompactUnwindSection.End.toPtr()); + }); + for (auto &CodeRange : UnwindInfo->CodeRanges) { + JDS->UnwindSections.insert(CodeRange.Start.toPtr(), + CodeRange.End.toPtr(), *UnwindInfo); + ORC_RT_DEBUG({ + printdbg(" [ %p -- %p ]\n", CodeRange.Start.toPtr(), + CodeRange.End.toPtr()); + }); + } + } + for (auto &KV : Secs) { // FIXME: Validate section ranges? if (KV.first == "__TEXT,__eh_frame") { - if (auto Err = registerEHFrames(KV.second.toSpan())) - return Err; + if (!UseCallbackStyleUnwindInfo) { + // Use classic libunwind registration. + if (auto Err = registerEHFrames(KV.second.toSpan())) + return Err; + } } else if (KV.first == "__DATA,__data") { assert(!JDS->DataSectionContent.count(KV.second.Start.toPtr()) && "Address already registered."); @@ -483,7 +612,7 @@ Error MachOPlatformRuntimeState::registerObjectPlatformSections( } Error MachOPlatformRuntimeState::deregisterObjectPlatformSections( - ExecutorAddr HeaderAddr, + ExecutorAddr HeaderAddr, std::optional UnwindInfo, std::vector> Secs) { // TODO: Make this more efficient? (maybe unnecessary if removal is rare?) // TODO: Add a JITDylib prepare-for-teardown operation that clears all @@ -510,11 +639,35 @@ Error MachOPlatformRuntimeState::deregisterObjectPlatformSections( // any Swift or ObjC. Once this happens we can clear (and no longer record) // data section content, as the library could never be re-initialized. + if (UnwindInfo && UseCallbackStyleUnwindInfo) { + ORC_RT_DEBUG({ + printdbg(" Deregistering new-style unwind info for:\n" + " DWARF: %p -- %p\n" + " Compact-unwind: %p -- %p\n" + " for:\n", + UnwindInfo->DwarfSection.Start.toPtr(), + UnwindInfo->DwarfSection.End.toPtr(), + UnwindInfo->CompactUnwindSection.Start.toPtr(), + UnwindInfo->CompactUnwindSection.End.toPtr()); + }); + for (auto &CodeRange : UnwindInfo->CodeRanges) { + JDS->UnwindSections.erase(CodeRange.Start.toPtr(), + CodeRange.End.toPtr()); + ORC_RT_DEBUG({ + printdbg(" [ %p -- %p ]\n", CodeRange.Start.toPtr(), + CodeRange.End.toPtr()); + }); + } + } + for (auto &KV : Secs) { // FIXME: Validate section ranges? if (KV.first == "__TEXT,__eh_frame") { - if (auto Err = deregisterEHFrames(KV.second.toSpan())) - return Err; + if (!UseCallbackStyleUnwindInfo) { + // Use classic libunwind registration. + if (auto Err = deregisterEHFrames(KV.second.toSpan())) + return Err; + } } else if (KV.first == "__DATA,__data") { JDS->DataSectionContent.erase(KV.second.Start.toPtr()); } else if (KV.first == "__DATA,__common") { @@ -710,6 +863,37 @@ void walkEHFrameSection(span EHFrameSection, } } +bool MachOPlatformRuntimeState::lookupUnwindSections( + void *Addr, unw_dynamic_unwind_sections &Info) { + ORC_RT_DEBUG( + { printdbg("Tried to lookup unwind-info via new lookup call.\n"); }); + std::lock_guard Lock(JDStatesMutex); + for (auto &KV : JDStates) { + auto &JD = KV.second; + auto I = JD.UnwindSections.find(reinterpret_cast(Addr)); + if (I != JD.UnwindSections.end()) { + Info.dso_base = reinterpret_cast(JD.Header); + Info.dwarf_section = + reinterpret_cast(I->second.DwarfSection.data()); + Info.dwarf_section_length = I->second.DwarfSection.size(); + Info.compact_unwind_section = + reinterpret_cast(I->second.CompactUnwindSection.data()); + Info.compact_unwind_section_length = + I->second.CompactUnwindSection.size(); + return true; + } + } + return false; +} + +int MachOPlatformRuntimeState::findDynamicUnwindSections( + uintptr_t addr, unw_dynamic_unwind_sections *info) { + if (!info) + return 0; + return MachOPlatformRuntimeState::get().lookupUnwindSections((void *)addr, + *info); +} + Error MachOPlatformRuntimeState::registerEHFrames( span EHFrameSection) { walkEHFrameSection(EHFrameSection, __register_frame); @@ -1101,10 +1285,7 @@ ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult __orc_rt_macho_platform_bootstrap(char *ArgData, size_t ArgSize) { return WrapperFunction::handle( ArgData, ArgSize, - []() -> Error { - MachOPlatformRuntimeState::initialize(); - return Error::success(); - }) + []() { return MachOPlatformRuntimeState::create(); }) .release(); } @@ -1112,10 +1293,7 @@ ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult __orc_rt_macho_platform_shutdown(char *ArgData, size_t ArgSize) { return WrapperFunction::handle( ArgData, ArgSize, - []() -> Error { - MachOPlatformRuntimeState::destroy(); - return Error::success(); - }) + []() { return MachOPlatformRuntimeState::destroy(); }) .release(); } @@ -1145,13 +1323,15 @@ ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult __orc_rt_macho_register_object_platform_sections(char *ArgData, size_t ArgSize) { return WrapperFunction, SPSMachOObjectPlatformSectionsMap)>:: handle(ArgData, ArgSize, - [](ExecutorAddr HeaderAddr, + [](ExecutorAddr HeaderAddr, std::optional USI, std::vector> &Secs) { return MachOPlatformRuntimeState::get() - .registerObjectPlatformSections(HeaderAddr, std::move(Secs)); + .registerObjectPlatformSections(HeaderAddr, std::move(USI), + std::move(Secs)); }) .release(); } @@ -1160,13 +1340,14 @@ ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult __orc_rt_macho_deregister_object_platform_sections(char *ArgData, size_t ArgSize) { return WrapperFunction, SPSMachOObjectPlatformSectionsMap)>:: handle(ArgData, ArgSize, - [](ExecutorAddr HeaderAddr, + [](ExecutorAddr HeaderAddr, std::optional USI, std::vector> &Secs) { return MachOPlatformRuntimeState::get() - .deregisterObjectPlatformSections(HeaderAddr, + .deregisterObjectPlatformSections(HeaderAddr, std::move(USI), std::move(Secs)); }) .release(); diff --git a/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h b/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h index 051ab3a85935fb..c51608899e0415 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h @@ -147,6 +147,12 @@ class MachOPlatform : public Platform { using InitSymbolDepMap = DenseMap; + struct UnwindSections { + SmallVector CodeRanges; + ExecutorAddrRange DwarfSection; + ExecutorAddrRange CompactUnwindSection; + }; + Error bootstrapPipelineStart(jitlink::LinkGraph &G); Error bootstrapPipelineRecordRuntimeFunctions(jitlink::LinkGraph &G); Error bootstrapPipelineEnd(jitlink::LinkGraph &G); @@ -164,6 +170,8 @@ class MachOPlatform : public Platform { Error fixTLVSectionsAndEdges(jitlink::LinkGraph &G, JITDylib &JD); + std::optional findUnwindSectionInfo(jitlink::LinkGraph &G); + Error registerObjectPlatformSections(jitlink::LinkGraph &G, JITDylib &JD, bool InBootstrapPhase); diff --git a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp index a926222dc458fc..914a1b5afc7123 100644 --- a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp +++ b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp @@ -249,6 +249,7 @@ class MachOPlatformCompleteBootstrapMaterializationUnit StringRef DataCommonSectionName = "__DATA,__common"; StringRef DataDataSectionName = "__DATA,__data"; StringRef EHFrameSectionName = "__TEXT,__eh_frame"; +StringRef CompactUnwindInfoSectionName = "__TEXT,__unwind_info"; StringRef ModInitFuncSectionName = "__DATA,__mod_init_func"; StringRef ObjCClassListSectionName = "__DATA,__objc_classlist"; StringRef ObjCImageInfoSectionName = "__DATA,__objc_image_info"; @@ -1068,6 +1069,81 @@ Error MachOPlatform::MachOPlatformPlugin::fixTLVSectionsAndEdges( return Error::success(); } +std::optional +MachOPlatform::MachOPlatformPlugin::findUnwindSectionInfo( + jitlink::LinkGraph &G) { + using namespace jitlink; + + UnwindSections US; + + // ScanSection records a section range and adds any executable blocks that + // that section points to to the CodeBlocks vector. + SmallVector CodeBlocks; + auto ScanUnwindInfoSection = [&](Section &Sec, ExecutorAddrRange &SecRange) { + if (Sec.blocks().empty()) + return; + SecRange = (*Sec.blocks().begin())->getRange(); + for (auto *B : Sec.blocks()) { + auto R = B->getRange(); + SecRange.Start = std::min(SecRange.Start, R.Start); + SecRange.End = std::max(SecRange.End, R.End); + for (auto &E : B->edges()) { + if (!E.getTarget().isDefined()) + continue; + auto &TargetBlock = E.getTarget().getBlock(); + auto &TargetSection = TargetBlock.getSection(); + if ((TargetSection.getMemProt() & MemProt::Exec) == MemProt::Exec) + CodeBlocks.push_back(&TargetBlock); + } + } + }; + + if (Section *EHFrameSec = G.findSectionByName(EHFrameSectionName)) + ScanUnwindInfoSection(*EHFrameSec, US.DwarfSection); + + if (Section *CUInfoSec = G.findSectionByName(CompactUnwindInfoSectionName)) + ScanUnwindInfoSection(*CUInfoSec, US.CompactUnwindSection); + + // If we didn't find any pointed-to code-blocks then there's no need to + // register any info. + if (CodeBlocks.empty()) + return std::nullopt; + + // We have info to register. Sort the code blocks into address order and + // build a list of contiguous address ranges covering them all. + llvm::sort(CodeBlocks, [](const Block *LHS, const Block *RHS) { + return LHS->getAddress() < RHS->getAddress(); + }); + for (auto *B : CodeBlocks) { + if (US.CodeRanges.empty() || US.CodeRanges.back().End != B->getAddress()) + US.CodeRanges.push_back(B->getRange()); + else + US.CodeRanges.back().End = B->getRange().End; + } + + LLVM_DEBUG({ + dbgs() << "MachOPlatform identified unwind info in " << G.getName() << ":\n" + << " DWARF: "; + if (US.DwarfSection.Start) + dbgs() << US.DwarfSection << "\n"; + else + dbgs() << "none\n"; + dbgs() << " Compact-unwind: "; + if (US.CompactUnwindSection.Start) + dbgs() << US.CompactUnwindSection << "\n"; + else + dbgs() << "none\n" + << "for code ranges:\n"; + for (auto &CR : US.CodeRanges) + dbgs() << " " << CR << "\n"; + if (US.CodeRanges.size() >= G.sections_size()) + dbgs() << "WARNING: High number of discontiguous code ranges! " + "Padding may be interfering with coalescing.\n"; + }); + + return US; +} + Error MachOPlatform::MachOPlatformPlugin::registerObjectPlatformSections( jitlink::LinkGraph &G, JITDylib &JD, bool InBootstrapPhase) { @@ -1128,7 +1204,14 @@ Error MachOPlatform::MachOPlatformPlugin::registerObjectPlatformSections( MachOPlatformSecs.push_back({SecName, R.getRange()}); } - if (!MachOPlatformSecs.empty()) { + std::optional, ExecutorAddrRange, + ExecutorAddrRange>> + UnwindInfo; + if (auto UI = findUnwindSectionInfo(G)) + UnwindInfo = std::make_tuple(std::move(UI->CodeRanges), UI->DwarfSection, + UI->CompactUnwindSection); + + if (!MachOPlatformSecs.empty() || UnwindInfo) { ExecutorAddr HeaderAddr; { std::lock_guard Lock(MP.PlatformMutex); @@ -1145,9 +1228,11 @@ Error MachOPlatform::MachOPlatformPlugin::registerObjectPlatformSections( dbgs() << " " << KV.first << ": " << KV.second << "\n"; }); - using SPSRegisterObjectPlatformSectionsArgs = - SPSArgList>>; + using SPSRegisterObjectPlatformSectionsArgs = SPSArgList< + SPSExecutorAddr, + SPSOptional, + SPSExecutorAddrRange, SPSExecutorAddrRange>>, + SPSSequence>>; shared::AllocActions &allocActions = LLVM_LIKELY(!InBootstrapPhase) ? G.allocActions() @@ -1156,12 +1241,12 @@ Error MachOPlatform::MachOPlatformPlugin::registerObjectPlatformSections( allocActions.push_back( {cantFail( WrapperFunctionCall::Create( - MP.RegisterObjectPlatformSections.Addr, HeaderAddr, + MP.RegisterObjectPlatformSections.Addr, HeaderAddr, UnwindInfo, MachOPlatformSecs)), cantFail( WrapperFunctionCall::Create( MP.DeregisterObjectPlatformSections.Addr, HeaderAddr, - MachOPlatformSecs))}); + UnwindInfo, MachOPlatformSecs))}); } return Error::success();