From 7a028495089bb3e919ffada0b4fb678471c97576 Mon Sep 17 00:00:00 2001 From: Lang Hames Date: Wed, 15 Oct 2025 15:01:05 +1100 Subject: [PATCH] [ORC] Align ExecutorSimpleMemoryManager w/ orc_rt::SimpleNativeMemoryMap Teach ExecutorSimpleMemoryManager to handle slab reserve/release operations, plus separate initialize/deinitialize for regions within the slab. The release operation automatically deinitializes any regions within each slab that have not already been released. EPCGenericJITLinkMemoryManager is updated to use the reserve (allocate), initialize (finalize), and relesae (deallocate) operations. This brings ExecutorSimpleMemoryManager into alignment with the orc_rt::SimpleNativeMemoryMap class, allowing SimpleNativeMemoryMap to be used as a backend for EPCGenericJITLinkMemoryManager. A future commit will introduce a new MemoryMapper class that will make SimpleNativeMemoryMap usable as a backend for MapperJITLinkMemoryManager. This work will make it easier to re-use in-tree APIs and tools with the new ORC runtime. --- .../Orc/EPCGenericJITLinkMemoryManager.h | 5 +- .../Orc/EPCGenericRTDyldMemoryManager.h | 4 +- .../ExecutionEngine/Orc/Shared/OrcRTBridge.h | 26 +- .../SimpleExecutorMemoryManager.h | 51 ++- .../Orc/EPCGenericJITLinkMemoryManager.cpp | 22 +- .../Orc/EPCGenericRTDyldMemoryManager.cpp | 18 +- .../Orc/Shared/OrcRTBridge.cpp | 10 +- .../ExecutionEngine/Orc/SimpleRemoteEPC.cpp | 6 +- .../SimpleExecutorMemoryManager.cpp | 367 +++++++++++------- .../EPCGenericJITLinkMemoryManagerTest.cpp | 27 +- .../Orc/SimpleExecutorMemoryManagerTest.cpp | 20 +- 11 files changed, 351 insertions(+), 205 deletions(-) diff --git a/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.h b/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.h index f9070afc1a327..eb71e9a162197 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.h @@ -32,8 +32,9 @@ class LLVM_ABI EPCGenericJITLinkMemoryManager struct SymbolAddrs { ExecutorAddr Allocator; ExecutorAddr Reserve; - ExecutorAddr Finalize; - ExecutorAddr Deallocate; + ExecutorAddr Initialize; + ExecutorAddr Deinitialize; + ExecutorAddr Release; }; /// Create an EPCGenericJITLinkMemoryManager instance from a given set of diff --git a/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.h b/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.h index faec25d81fac8..fa48480f265a9 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.h @@ -31,8 +31,8 @@ class LLVM_ABI EPCGenericRTDyldMemoryManager struct SymbolAddrs { ExecutorAddr Instance; ExecutorAddr Reserve; - ExecutorAddr Finalize; - ExecutorAddr Deallocate; + ExecutorAddr Initialize; + ExecutorAddr Release; ExecutorAddr RegisterEHFrame; ExecutorAddr DeregisterEHFrame; }; diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h b/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h index 99ba456e66c85..d68a68992a638 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h @@ -29,8 +29,9 @@ LLVM_ABI extern const char *SimpleExecutorDylibManagerResolveWrapperName; LLVM_ABI extern const char *SimpleExecutorMemoryManagerInstanceName; LLVM_ABI extern const char *SimpleExecutorMemoryManagerReserveWrapperName; -LLVM_ABI extern const char *SimpleExecutorMemoryManagerFinalizeWrapperName; -LLVM_ABI extern const char *SimpleExecutorMemoryManagerDeallocateWrapperName; +LLVM_ABI extern const char *SimpleExecutorMemoryManagerInitializeWrapperName; +LLVM_ABI extern const char *SimpleExecutorMemoryManagerDeinitializeWrapperName; +LLVM_ABI extern const char *SimpleExecutorMemoryManagerReleaseWrapperName; LLVM_ABI extern const char *ExecutorSharedMemoryMapperServiceInstanceName; LLVM_ABI extern const char *ExecutorSharedMemoryMapperServiceReserveWrapperName; @@ -73,9 +74,12 @@ using SPSSimpleExecutorDylibManagerResolveSignature = shared::SPSExpected< using SPSSimpleExecutorMemoryManagerReserveSignature = shared::SPSExpected(shared::SPSExecutorAddr, uint64_t); -using SPSSimpleExecutorMemoryManagerFinalizeSignature = - shared::SPSError(shared::SPSExecutorAddr, shared::SPSFinalizeRequest); -using SPSSimpleExecutorMemoryManagerDeallocateSignature = shared::SPSError( +using SPSSimpleExecutorMemoryManagerInitializeSignature = + shared::SPSExpected(shared::SPSExecutorAddr, + shared::SPSFinalizeRequest); +using SPSSimpleExecutorMemoryManagerDeinitializeSignature = shared::SPSError( + shared::SPSExecutorAddr, shared::SPSSequence); +using SPSSimpleExecutorMemoryManagerReleaseSignature = shared::SPSError( shared::SPSExecutorAddr, shared::SPSSequence); // ExecutorSharedMemoryMapperService @@ -93,6 +97,18 @@ using SPSExecutorSharedMemoryMapperServiceDeinitializeSignature = using SPSExecutorSharedMemoryMapperServiceReleaseSignature = shared::SPSError( shared::SPSExecutorAddr, shared::SPSSequence); +// SimpleNativeMemoryMap APIs. +using SPSSimpleRemoteMemoryMapReserveSignature = + shared::SPSExpected(shared::SPSExecutorAddr, + uint64_t); +using SPSSimpleRemoteMemoryMapInitializeSignature = + shared::SPSExpected(shared::SPSExecutorAddr, + shared::SPSFinalizeRequest); +using SPSSimpleRemoteMemoryMapDeinitializeSignature = shared::SPSError( + shared::SPSExecutorAddr, shared::SPSSequence); +using SPSSimpleRemoteMemoryMapReleaseSignature = shared::SPSError( + shared::SPSExecutorAddr, shared::SPSSequence); + using SPSRunAsMainSignature = int64_t(shared::SPSExecutorAddr, shared::SPSSequence); using SPSRunAsVoidFunctionSignature = int32_t(shared::SPSExecutorAddr); diff --git a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.h b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.h index 741f203396fac..6224e92887147 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.h @@ -34,34 +34,65 @@ class LLVM_ABI SimpleExecutorMemoryManager : public ExecutorBootstrapService { public: virtual ~SimpleExecutorMemoryManager(); - Expected allocate(uint64_t Size); - Error finalize(tpctypes::FinalizeRequest &FR); - Error deallocate(const std::vector &Bases); + Expected reserve(uint64_t Size); + Expected initialize(tpctypes::FinalizeRequest &FR); + Error deinitialize(const std::vector &InitKeys); + Error release(const std::vector &Bases); Error shutdown() override; void addBootstrapSymbols(StringMap &M) override; private: - struct Allocation { + struct RegionInfo { size_t Size = 0; - std::vector DeallocationActions; + std::vector DeallocActions; }; - using AllocationsMap = DenseMap; + struct SlabInfo { + using RegionMap = std::map; + size_t Size = 0; + RegionMap Regions; + }; + + using SlabMap = std::map; + + /// Get a reference to the slab information for the slab containing the given + /// address. + Expected getSlabInfo(ExecutorAddr A, StringRef Context); + + /// Get a reference to the slab information for the slab *covering* the given + /// range. The given range must be a subrange of e(possibly equal to) the + /// range of the slab itself. + Expected getSlabInfo(ExecutorAddrRange R, StringRef Context); - Error deallocateImpl(void *Base, Allocation &A); + /// Create a RegionInfo for the given range, which must not overlap any + /// existing region. + Expected createRegionInfo(ExecutorAddrRange R, + StringRef Context); + + /// Get a reference to the region information for the given address. This + /// address must represent the start of an existing initialized region. + Expected getRegionInfo(SlabInfo &Slab, ExecutorAddr A, + StringRef Context); + + /// Get a reference to the region information for the given address. This + /// address must represent the start of an existing initialized region. + Expected getRegionInfo(ExecutorAddr A, StringRef Context); static llvm::orc::shared::CWrapperFunctionResult reserveWrapper(const char *ArgData, size_t ArgSize); static llvm::orc::shared::CWrapperFunctionResult - finalizeWrapper(const char *ArgData, size_t ArgSize); + initializeWrapper(const char *ArgData, size_t ArgSize); + + static llvm::orc::shared::CWrapperFunctionResult + deinitializeWrapper(const char *ArgData, size_t ArgSize); static llvm::orc::shared::CWrapperFunctionResult - deallocateWrapper(const char *ArgData, size_t ArgSize); + releaseWrapper(const char *ArgData, size_t ArgSize); std::mutex M; - AllocationsMap Allocations; + SlabMap Slabs; }; } // end namespace rt_bootstrap diff --git a/llvm/lib/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.cpp b/llvm/lib/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.cpp index 50e6b254595b8..0833af7f020fc 100644 --- a/llvm/lib/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.cpp +++ b/llvm/lib/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.cpp @@ -57,16 +57,17 @@ class EPCGenericJITLinkMemoryManager::InFlightAlloc std::swap(FR.Actions, G.allocActions()); Parent.EPC.callSPSWrapperAsync< - rt::SPSSimpleExecutorMemoryManagerFinalizeSignature>( - Parent.SAs.Finalize, + rt::SPSSimpleExecutorMemoryManagerInitializeSignature>( + Parent.SAs.Initialize, [OnFinalize = std::move(OnFinalize), AllocAddr = this->AllocAddr]( - Error SerializationErr, Error FinalizeErr) mutable { + Error SerializationErr, + Expected InitializeKey) mutable { // FIXME: Release abandoned alloc. if (SerializationErr) { - cantFail(std::move(FinalizeErr)); + cantFail(InitializeKey.takeError()); OnFinalize(std::move(SerializationErr)); - } else if (FinalizeErr) - OnFinalize(std::move(FinalizeErr)); + } else if (!InitializeKey) + OnFinalize(InitializeKey.takeError()); else OnFinalize(FinalizedAlloc(AllocAddr)); }, @@ -76,8 +77,8 @@ class EPCGenericJITLinkMemoryManager::InFlightAlloc void abandon(OnAbandonedFunction OnAbandoned) override { // FIXME: Return memory to pool instead. Parent.EPC.callSPSWrapperAsync< - rt::SPSSimpleExecutorMemoryManagerDeallocateSignature>( - Parent.SAs.Deallocate, + rt::SPSSimpleExecutorMemoryManagerReleaseSignature>( + Parent.SAs.Release, [OnAbandoned = std::move(OnAbandoned)](Error SerializationErr, Error DeallocateErr) mutable { if (SerializationErr) { @@ -123,9 +124,8 @@ void EPCGenericJITLinkMemoryManager::allocate(const JITLinkDylib *JD, void EPCGenericJITLinkMemoryManager::deallocate( std::vector Allocs, OnDeallocatedFunction OnDeallocated) { - EPC.callSPSWrapperAsync< - rt::SPSSimpleExecutorMemoryManagerDeallocateSignature>( - SAs.Deallocate, + EPC.callSPSWrapperAsync( + SAs.Release, [OnDeallocated = std::move(OnDeallocated)](Error SerErr, Error DeallocErr) mutable { if (SerErr) { diff --git a/llvm/lib/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.cpp b/llvm/lib/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.cpp index fec7062ff79a6..cc72488455ec1 100644 --- a/llvm/lib/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.cpp +++ b/llvm/lib/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.cpp @@ -25,9 +25,9 @@ EPCGenericRTDyldMemoryManager::CreateWithDefaultBootstrapSymbols( if (auto Err = EPC.getBootstrapSymbols( {{SAs.Instance, rt::SimpleExecutorMemoryManagerInstanceName}, {SAs.Reserve, rt::SimpleExecutorMemoryManagerReserveWrapperName}, - {SAs.Finalize, rt::SimpleExecutorMemoryManagerFinalizeWrapperName}, - {SAs.Deallocate, - rt::SimpleExecutorMemoryManagerDeallocateWrapperName}, + {SAs.Initialize, + rt::SimpleExecutorMemoryManagerInitializeWrapperName}, + {SAs.Release, rt::SimpleExecutorMemoryManagerReleaseWrapperName}, {SAs.RegisterEHFrame, rt::RegisterEHFrameSectionAllocActionName}, {SAs.DeregisterEHFrame, rt::DeregisterEHFrameSectionAllocActionName}})) @@ -48,7 +48,7 @@ EPCGenericRTDyldMemoryManager::~EPCGenericRTDyldMemoryManager() { Error Err = Error::success(); if (auto Err2 = EPC.callSPSWrapper< - rt::SPSSimpleExecutorMemoryManagerDeallocateSignature>( + rt::SPSSimpleExecutorMemoryManagerReleaseSignature>( SAs.Reserve, Err, SAs.Instance, FinalizedAllocs)) { // FIXME: Report errors through EPC once that functionality is available. logAllUnhandledErrors(std::move(Err2), errs(), ""); @@ -267,10 +267,10 @@ bool EPCGenericRTDyldMemoryManager::finalizeMemory(std::string *ErrMsg) { // We'll also need to make an extra allocation for the eh-frame wrapper call // arguments. - Error FinalizeErr = Error::success(); + Expected InitializeKey((ExecutorAddr())); if (auto Err = EPC.callSPSWrapper< - rt::SPSSimpleExecutorMemoryManagerFinalizeSignature>( - SAs.Finalize, FinalizeErr, SAs.Instance, std::move(FR))) { + rt::SPSSimpleExecutorMemoryManagerInitializeSignature>( + SAs.Initialize, InitializeKey, SAs.Instance, std::move(FR))) { std::lock_guard Lock(M); this->ErrMsg = toString(std::move(Err)); dbgs() << "Serialization error: " << this->ErrMsg << "\n"; @@ -278,9 +278,9 @@ bool EPCGenericRTDyldMemoryManager::finalizeMemory(std::string *ErrMsg) { *ErrMsg = this->ErrMsg; return true; } - if (FinalizeErr) { + if (!InitializeKey) { std::lock_guard Lock(M); - this->ErrMsg = toString(std::move(FinalizeErr)); + this->ErrMsg = toString(InitializeKey.takeError()); dbgs() << "Finalization error: " << this->ErrMsg << "\n"; if (ErrMsg) *ErrMsg = this->ErrMsg; diff --git a/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp b/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp index 26e8f5382da0e..cc99d3c768772 100644 --- a/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp +++ b/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp @@ -23,10 +23,12 @@ const char *SimpleExecutorMemoryManagerInstanceName = "__llvm_orc_SimpleExecutorMemoryManager_Instance"; const char *SimpleExecutorMemoryManagerReserveWrapperName = "__llvm_orc_SimpleExecutorMemoryManager_reserve_wrapper"; -const char *SimpleExecutorMemoryManagerFinalizeWrapperName = - "__llvm_orc_SimpleExecutorMemoryManager_finalize_wrapper"; -const char *SimpleExecutorMemoryManagerDeallocateWrapperName = - "__llvm_orc_SimpleExecutorMemoryManager_deallocate_wrapper"; +const char *SimpleExecutorMemoryManagerInitializeWrapperName = + "__llvm_orc_SimpleExecutorMemoryManager_initialize_wrapper"; +const char *SimpleExecutorMemoryManagerDeinitializeWrapperName = + "__llvm_orc_SimpleExecutorMemoryManager_deinitialize_wrapper"; +const char *SimpleExecutorMemoryManagerReleaseWrapperName = + "__llvm_orc_SimpleExecutorMemoryManager_release_wrapper"; const char *ExecutorSharedMemoryMapperServiceInstanceName = "__llvm_orc_ExecutorSharedMemoryMapperService_Instance"; diff --git a/llvm/lib/ExecutionEngine/Orc/SimpleRemoteEPC.cpp b/llvm/lib/ExecutionEngine/Orc/SimpleRemoteEPC.cpp index 87d757805a64c..dec1df7da2f4a 100644 --- a/llvm/lib/ExecutionEngine/Orc/SimpleRemoteEPC.cpp +++ b/llvm/lib/ExecutionEngine/Orc/SimpleRemoteEPC.cpp @@ -216,9 +216,9 @@ SimpleRemoteEPC::createDefaultMemoryManager(SimpleRemoteEPC &SREPC) { if (auto Err = SREPC.getBootstrapSymbols( {{SAs.Allocator, rt::SimpleExecutorMemoryManagerInstanceName}, {SAs.Reserve, rt::SimpleExecutorMemoryManagerReserveWrapperName}, - {SAs.Finalize, rt::SimpleExecutorMemoryManagerFinalizeWrapperName}, - {SAs.Deallocate, - rt::SimpleExecutorMemoryManagerDeallocateWrapperName}})) + {SAs.Initialize, + rt::SimpleExecutorMemoryManagerInitializeWrapperName}, + {SAs.Release, rt::SimpleExecutorMemoryManagerReleaseWrapperName}})) return std::move(Err); return std::make_unique(SREPC, SAs); diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.cpp b/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.cpp index 3cdffb8cd0615..fe881a1eef773 100644 --- a/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.cpp +++ b/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.cpp @@ -8,6 +8,7 @@ #include "llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.h" +#include "llvm/ADT/ScopeExit.h" #include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h" #include "llvm/Support/FormatVariadic.h" @@ -18,166 +19,167 @@ namespace orc { namespace rt_bootstrap { SimpleExecutorMemoryManager::~SimpleExecutorMemoryManager() { - assert(Allocations.empty() && "shutdown not called?"); + assert(Slabs.empty() && "shutdown not called?"); } -Expected SimpleExecutorMemoryManager::allocate(uint64_t Size) { +Expected SimpleExecutorMemoryManager::reserve(uint64_t Size) { std::error_code EC; auto MB = sys::Memory::allocateMappedMemory( Size, nullptr, sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC); if (EC) return errorCodeToError(EC); std::lock_guard Lock(M); - assert(!Allocations.count(MB.base()) && "Duplicate allocation addr"); - Allocations[MB.base()].Size = Size; + assert(!Slabs.count(MB.base()) && "Duplicate allocation addr"); + Slabs[MB.base()].Size = Size; return ExecutorAddr::fromPtr(MB.base()); } -Error SimpleExecutorMemoryManager::finalize(tpctypes::FinalizeRequest &FR) { - ExecutorAddr Base(~0ULL); +Expected +SimpleExecutorMemoryManager::initialize(tpctypes::FinalizeRequest &FR) { std::vector DeallocationActions; - size_t SuccessfulFinalizationActions = 0; if (FR.Segments.empty()) { - // NOTE: Finalizing nothing is currently a no-op. Should it be an error? if (FR.Actions.empty()) - return Error::success(); + return make_error("Finalization request is empty", + inconvertibleErrorCode()); else return make_error("Finalization actions attached to empty " "finalization request", inconvertibleErrorCode()); } - for (auto &Seg : FR.Segments) - Base = std::min(Base, Seg.Addr); - - for (auto &ActPair : FR.Actions) - if (ActPair.Dealloc) - DeallocationActions.push_back(ActPair.Dealloc); - - // Get the Allocation for this finalization. - size_t AllocSize = 0; - { - std::lock_guard Lock(M); - auto I = Allocations.find(Base.toPtr()); - if (I == Allocations.end()) - return make_error("Attempt to finalize unrecognized " - "allocation " + - formatv("{0:x}", Base.getValue()), - inconvertibleErrorCode()); - AllocSize = I->second.Size; - I->second.DeallocationActions = std::move(DeallocationActions); - } - ExecutorAddr AllocEnd = Base + ExecutorAddrDiff(AllocSize); - - // Bail-out function: this will run deallocation actions corresponding to any - // completed finalization actions, then deallocate memory. - auto BailOut = [&](Error Err) { - std::pair AllocToDestroy; - - // Get allocation to destroy. - { - std::lock_guard Lock(M); - auto I = Allocations.find(Base.toPtr()); - - // Check for missing allocation (effective a double free). - if (I == Allocations.end()) - return joinErrors( - std::move(Err), - make_error("No allocation entry found " - "for " + - formatv("{0:x}", Base.getValue()), - inconvertibleErrorCode())); - AllocToDestroy = std::move(*I); - Allocations.erase(I); - } + ExecutorAddrRange RR(FR.Segments.front().Addr, FR.Segments.front().Addr); - // Run deallocation actions for all completed finalization actions. - while (SuccessfulFinalizationActions) - Err = - joinErrors(std::move(Err), FR.Actions[--SuccessfulFinalizationActions] - .Dealloc.runWithSPSRetErrorMerged()); - - // Deallocate memory. - sys::MemoryBlock MB(AllocToDestroy.first, AllocToDestroy.second.Size); - if (auto EC = sys::Memory::releaseMappedMemory(MB)) - Err = joinErrors(std::move(Err), errorCodeToError(EC)); - - return Err; - }; + std::vector MBsToReset; + auto ResetMBs = make_scope_exit([&]() { + for (auto &MB : MBsToReset) + sys::Memory::protectMappedMemory(MB, sys::Memory::MF_READ | + sys::Memory::MF_WRITE); + sys::Memory::InvalidateInstructionCache(RR.Start.toPtr(), + RR.size()); + }); // Copy content and apply permissions. for (auto &Seg : FR.Segments) { + RR.Start = std::min(RR.Start, Seg.Addr); + RR.End = std::max(RR.End, Seg.Addr + Seg.Size); // Check segment ranges. if (LLVM_UNLIKELY(Seg.Size < Seg.Content.size())) - return BailOut(make_error( + return make_error( formatv("Segment {0:x} content size ({1:x} bytes) " "exceeds segment size ({2:x} bytes)", Seg.Addr.getValue(), Seg.Content.size(), Seg.Size), - inconvertibleErrorCode())); + inconvertibleErrorCode()); ExecutorAddr SegEnd = Seg.Addr + ExecutorAddrDiff(Seg.Size); - if (LLVM_UNLIKELY(Seg.Addr < Base || SegEnd > AllocEnd)) - return BailOut(make_error( + if (LLVM_UNLIKELY(Seg.Addr < RR.Start || SegEnd > RR.End)) + return make_error( formatv("Segment {0:x} -- {1:x} crosses boundary of " "allocation {2:x} -- {3:x}", - Seg.Addr.getValue(), SegEnd.getValue(), Base.getValue(), - AllocEnd.getValue()), - inconvertibleErrorCode())); + Seg.Addr, SegEnd, RR.Start, RR.End), + inconvertibleErrorCode()); char *Mem = Seg.Addr.toPtr(); if (!Seg.Content.empty()) memcpy(Mem, Seg.Content.data(), Seg.Content.size()); memset(Mem + Seg.Content.size(), 0, Seg.Size - Seg.Content.size()); assert(Seg.Size <= std::numeric_limits::max()); + + sys::MemoryBlock MB(Mem, Seg.Size); if (auto EC = sys::Memory::protectMappedMemory( - {Mem, static_cast(Seg.Size)}, - toSysMemoryProtectionFlags(Seg.RAG.Prot))) - return BailOut(errorCodeToError(EC)); + MB, toSysMemoryProtectionFlags(Seg.RAG.Prot))) + return errorCodeToError(EC); + + MBsToReset.push_back(MB); + if ((Seg.RAG.Prot & MemProt::Exec) == MemProt::Exec) sys::Memory::InvalidateInstructionCache(Mem, Seg.Size); } - // Run finalization actions. - for (auto &ActPair : FR.Actions) { - if (auto Err = ActPair.Finalize.runWithSPSRetErrorMerged()) - return BailOut(std::move(Err)); - ++SuccessfulFinalizationActions; + auto DeallocActions = runFinalizeActions(FR.Actions); + if (!DeallocActions) + return DeallocActions.takeError(); + + { + std::lock_guard Lock(M); + auto Region = createRegionInfo(RR, "In initialize"); + if (!Region) + return Region.takeError(); + Region->DeallocActions = std::move(*DeallocActions); } - return Error::success(); + // Successful initialization. + ResetMBs.release(); + + return RR.Start; } -Error SimpleExecutorMemoryManager::deallocate( - const std::vector &Bases) { - std::vector> AllocPairs; - AllocPairs.reserve(Bases.size()); +Error SimpleExecutorMemoryManager::deinitialize( + const std::vector &InitKeys) { + Error Err = Error::success(); - // Get allocation to destroy. + for (auto &KeyAddr : llvm::reverse(InitKeys)) { + std::vector DeallocActions; + { + std::scoped_lock Lock(M); + auto Slab = getSlabInfo(KeyAddr, "In deinitialize"); + if (!Slab) { + Err = joinErrors(std::move(Err), Slab.takeError()); + continue; + } + + auto RI = getRegionInfo(*Slab, KeyAddr, "In deinitialize"); + if (!RI) { + Err = joinErrors(std::move(Err), RI.takeError()); + continue; + } + + DeallocActions = std::move(RI->DeallocActions); + } + + Err = joinErrors(std::move(Err), + runDeallocActions(std::move(DeallocActions))); + } + + return Err; +} + +Error SimpleExecutorMemoryManager::release( + const std::vector &Bases) { Error Err = Error::success(); - { - std::lock_guard Lock(M); - for (auto &Base : Bases) { - auto I = Allocations.find(Base.toPtr()); - - // Check for missing allocation (effective a double free). - if (I != Allocations.end()) { - AllocPairs.push_back(std::move(*I)); - Allocations.erase(I); - } else + + // TODO: Prohibit new initializations within the slabs being removed? + for (auto &Base : llvm::reverse(Bases)) { + std::vector DeallocActions; + sys::MemoryBlock MB; + + { + std::scoped_lock Lock(M); + + auto SlabI = Slabs.find(Base.toPtr()); + if (SlabI == Slabs.end()) { Err = joinErrors( std::move(Err), - make_error("No allocation entry found " - "for " + - formatv("{0:x}", Base.getValue()), + make_error("In release, " + formatv("{0:x}", Base) + + " is not part of any reserved " + "address range", inconvertibleErrorCode())); + continue; + } + + auto &Slab = SlabI->second; + + for (auto &[Addr, Region] : Slab.Regions) + llvm::copy(Region.DeallocActions, back_inserter(DeallocActions)); + + MB = {Base.toPtr(), Slab.Size}; + + Slabs.erase(SlabI); } - } - while (!AllocPairs.empty()) { - auto &P = AllocPairs.back(); - Err = joinErrors(std::move(Err), deallocateImpl(P.first, P.second)); - AllocPairs.pop_back(); + Err = joinErrors(std::move(Err), runDeallocActions(DeallocActions)); + if (auto EC = sys::Memory::releaseMappedMemory(MB)) + Err = joinErrors(std::move(Err), errorCodeToError(EC)); } return Err; @@ -185,16 +187,15 @@ Error SimpleExecutorMemoryManager::deallocate( Error SimpleExecutorMemoryManager::shutdown() { - AllocationsMap AM; + // TODO: Prevent new allocations during shutdown. + std::vector Bases; { - std::lock_guard Lock(M); - AM = std::move(Allocations); + std::scoped_lock Lock(M); + for (auto &[Base, Slab] : Slabs) + Bases.push_back(ExecutorAddr::fromPtr(Base)); } - Error Err = Error::success(); - for (auto &KV : AM) - Err = joinErrors(std::move(Err), deallocateImpl(KV.first, KV.second)); - return Err; + return release(Bases); } void SimpleExecutorMemoryManager::addBootstrapSymbols( @@ -202,58 +203,150 @@ void SimpleExecutorMemoryManager::addBootstrapSymbols( M[rt::SimpleExecutorMemoryManagerInstanceName] = ExecutorAddr::fromPtr(this); M[rt::SimpleExecutorMemoryManagerReserveWrapperName] = ExecutorAddr::fromPtr(&reserveWrapper); - M[rt::SimpleExecutorMemoryManagerFinalizeWrapperName] = - ExecutorAddr::fromPtr(&finalizeWrapper); - M[rt::SimpleExecutorMemoryManagerDeallocateWrapperName] = - ExecutorAddr::fromPtr(&deallocateWrapper); + M[rt::SimpleExecutorMemoryManagerInitializeWrapperName] = + ExecutorAddr::fromPtr(&initializeWrapper); + M[rt::SimpleExecutorMemoryManagerDeinitializeWrapperName] = + ExecutorAddr::fromPtr(&deinitializeWrapper); + M[rt::SimpleExecutorMemoryManagerReleaseWrapperName] = + ExecutorAddr::fromPtr(&releaseWrapper); } -Error SimpleExecutorMemoryManager::deallocateImpl(void *Base, Allocation &A) { - Error Err = Error::success(); +Expected +SimpleExecutorMemoryManager::getSlabInfo(ExecutorAddr A, StringRef Context) { + auto MakeBadSlabError = [&]() { + return make_error( + Context + ", address " + formatv("{0:x}", A) + + " is not part of any reserved address range", + inconvertibleErrorCode()); + }; - while (!A.DeallocationActions.empty()) { - Err = joinErrors(std::move(Err), - A.DeallocationActions.back().runWithSPSRetErrorMerged()); - A.DeallocationActions.pop_back(); + auto I = Slabs.upper_bound(A.toPtr()); + if (I == Slabs.begin()) + return MakeBadSlabError(); + --I; + if (!ExecutorAddrRange(ExecutorAddr::fromPtr(I->first), I->second.Size) + .contains(A)) + return MakeBadSlabError(); + + return I->second; +} + +Expected +SimpleExecutorMemoryManager::getSlabInfo(ExecutorAddrRange R, + StringRef Context) { + auto MakeBadSlabError = [&]() { + return make_error( + Context + ", range " + formatv("{0:x}", R) + + " is not part of any reserved address range", + inconvertibleErrorCode()); + }; + + auto I = Slabs.upper_bound(R.Start.toPtr()); + if (I == Slabs.begin()) + return MakeBadSlabError(); + --I; + if (!ExecutorAddrRange(ExecutorAddr::fromPtr(I->first), I->second.Size) + .contains(R)) + return MakeBadSlabError(); + + return I->second; +} + +Expected +SimpleExecutorMemoryManager::createRegionInfo(ExecutorAddrRange R, + StringRef Context) { + + auto Slab = getSlabInfo(R, Context); + if (!Slab) + return Slab.takeError(); + + auto MakeBadRegionError = [&](ExecutorAddrRange Other, bool Prev) { + return make_error(Context + ", region " + formatv("{0:x}", R) + + " overlaps " + + (Prev ? "previous" : "following") + + " region " + formatv("{0:x}", Other), + inconvertibleErrorCode()); + }; + + auto I = Slab->Regions.upper_bound(R.Start); + if (I != Slab->Regions.begin()) { + auto J = std::prev(I); + ExecutorAddrRange PrevRange(J->first, J->second.Size); + if (PrevRange.overlaps(R)) + return MakeBadRegionError(PrevRange, true); + } + if (I != Slab->Regions.end()) { + ExecutorAddrRange NextRange(I->first, I->second.Size); + if (NextRange.overlaps(R)) + return MakeBadRegionError(NextRange, false); } - sys::MemoryBlock MB(Base, A.Size); - if (auto EC = sys::Memory::releaseMappedMemory(MB)) - Err = joinErrors(std::move(Err), errorCodeToError(EC)); + auto &RInfo = Slab->Regions[R.Start]; + RInfo.Size = R.size(); + return RInfo; +} - return Err; +Expected +SimpleExecutorMemoryManager::getRegionInfo(SlabInfo &Slab, ExecutorAddr A, + StringRef Context) { + auto I = Slab.Regions.find(A); + if (I == Slab.Regions.end()) + return make_error( + Context + ", address " + formatv("{0:x}", A) + + " does not correspond to the start of any initialized region", + inconvertibleErrorCode()); + + return I->second; +} + +Expected +SimpleExecutorMemoryManager::getRegionInfo(ExecutorAddr A, StringRef Context) { + auto Slab = getSlabInfo(A, Context); + if (!Slab) + return Slab.takeError(); + + return getRegionInfo(*Slab, A, Context); } llvm::orc::shared::CWrapperFunctionResult SimpleExecutorMemoryManager::reserveWrapper(const char *ArgData, size_t ArgSize) { - return shared::WrapperFunction< - rt::SPSSimpleExecutorMemoryManagerReserveSignature>:: + return shared::WrapperFunction:: handle(ArgData, ArgSize, shared::makeMethodWrapperHandler( - &SimpleExecutorMemoryManager::allocate)) + &SimpleExecutorMemoryManager::reserve)) + .release(); +} + +llvm::orc::shared::CWrapperFunctionResult +SimpleExecutorMemoryManager::initializeWrapper(const char *ArgData, + size_t ArgSize) { + return shared:: + WrapperFunction::handle( + ArgData, ArgSize, + shared::makeMethodWrapperHandler( + &SimpleExecutorMemoryManager::initialize)) .release(); } llvm::orc::shared::CWrapperFunctionResult -SimpleExecutorMemoryManager::finalizeWrapper(const char *ArgData, - size_t ArgSize) { +SimpleExecutorMemoryManager::deinitializeWrapper(const char *ArgData, + size_t ArgSize) { return shared::WrapperFunction< - rt::SPSSimpleExecutorMemoryManagerFinalizeSignature>:: + rt::SPSSimpleRemoteMemoryMapDeinitializeSignature>:: handle(ArgData, ArgSize, shared::makeMethodWrapperHandler( - &SimpleExecutorMemoryManager::finalize)) + &SimpleExecutorMemoryManager::deinitialize)) .release(); } llvm::orc::shared::CWrapperFunctionResult -SimpleExecutorMemoryManager::deallocateWrapper(const char *ArgData, - size_t ArgSize) { - return shared::WrapperFunction< - rt::SPSSimpleExecutorMemoryManagerDeallocateSignature>:: +SimpleExecutorMemoryManager::releaseWrapper(const char *ArgData, + size_t ArgSize) { + return shared::WrapperFunction:: handle(ArgData, ArgSize, shared::makeMethodWrapperHandler( - &SimpleExecutorMemoryManager::deallocate)) + &SimpleExecutorMemoryManager::release)) .release(); } diff --git a/llvm/unittests/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManagerTest.cpp b/llvm/unittests/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManagerTest.cpp index d4b45ea0dca0b..2c6650d25a0ec 100644 --- a/llvm/unittests/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManagerTest.cpp +++ b/llvm/unittests/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManagerTest.cpp @@ -39,8 +39,11 @@ class SimpleAllocator { return ExecutorAddr::fromPtr(MB.base()); } - Error finalize(tpctypes::FinalizeRequest FR) { + Expected initialize(tpctypes::FinalizeRequest FR) { + assert(!FR.Segments.empty()); + ExecutorAddr Base = FR.Segments[0].Addr; for (auto &Seg : FR.Segments) { + Base = std::min(Base, Seg.Addr); char *Mem = Seg.Addr.toPtr(); memcpy(Mem, Seg.Content.data(), Seg.Content.size()); memset(Mem + Seg.Content.size(), 0, Seg.Size - Seg.Content.size()); @@ -52,10 +55,10 @@ class SimpleAllocator { if ((Seg.RAG.Prot & MemProt::Exec) != MemProt::Exec) sys::Memory::InvalidateInstructionCache(Mem, Seg.Size); } - return Error::success(); + return Base; } - Error deallocate(std::vector &Bases) { + Error release(std::vector &Bases) { Error Err = Error::success(); for (auto &Base : Bases) { auto I = Blocks.find(Base.toPtr()); @@ -86,18 +89,18 @@ CWrapperFunctionResult testReserve(const char *ArgData, size_t ArgSize) { .release(); } -CWrapperFunctionResult testFinalize(const char *ArgData, size_t ArgSize) { - return WrapperFunction:: +CWrapperFunctionResult testInitialize(const char *ArgData, size_t ArgSize) { + return WrapperFunction< + rt::SPSSimpleExecutorMemoryManagerInitializeSignature>:: handle(ArgData, ArgSize, - makeMethodWrapperHandler(&SimpleAllocator::finalize)) + makeMethodWrapperHandler(&SimpleAllocator::initialize)) .release(); } -CWrapperFunctionResult testDeallocate(const char *ArgData, size_t ArgSize) { - return WrapperFunction< - rt::SPSSimpleExecutorMemoryManagerDeallocateSignature>:: +CWrapperFunctionResult testRelease(const char *ArgData, size_t ArgSize) { + return WrapperFunction:: handle(ArgData, ArgSize, - makeMethodWrapperHandler(&SimpleAllocator::deallocate)) + makeMethodWrapperHandler(&SimpleAllocator::release)) .release(); } @@ -108,8 +111,8 @@ TEST(EPCGenericJITLinkMemoryManagerTest, AllocFinalizeFree) { EPCGenericJITLinkMemoryManager::SymbolAddrs SAs; SAs.Allocator = ExecutorAddr::fromPtr(&SA); SAs.Reserve = ExecutorAddr::fromPtr(&testReserve); - SAs.Finalize = ExecutorAddr::fromPtr(&testFinalize); - SAs.Deallocate = ExecutorAddr::fromPtr(&testDeallocate); + SAs.Initialize = ExecutorAddr::fromPtr(&testInitialize); + SAs.Release = ExecutorAddr::fromPtr(&testRelease); auto MemMgr = std::make_unique(*SelfEPC, SAs); StringRef Hello = "hello"; diff --git a/llvm/unittests/ExecutionEngine/Orc/SimpleExecutorMemoryManagerTest.cpp b/llvm/unittests/ExecutionEngine/Orc/SimpleExecutorMemoryManagerTest.cpp index 6e9b0b2d8c73f..9c6f19c82c998 100644 --- a/llvm/unittests/ExecutionEngine/Orc/SimpleExecutorMemoryManagerTest.cpp +++ b/llvm/unittests/ExecutionEngine/Orc/SimpleExecutorMemoryManagerTest.cpp @@ -34,12 +34,12 @@ TEST(SimpleExecutorMemoryManagerTest, AllocFinalizeFree) { SimpleExecutorMemoryManager MemMgr; constexpr unsigned AllocSize = 16384; - auto Mem = MemMgr.allocate(AllocSize); + auto Mem = MemMgr.reserve(AllocSize); EXPECT_THAT_ERROR(Mem.takeError(), Succeeded()); std::string HW = "Hello, world!"; - int FinalizeCounter = 0; + int InitializeCounter = 0; int DeallocateCounter = 0; tpctypes::FinalizeRequest FR; @@ -52,27 +52,27 @@ TEST(SimpleExecutorMemoryManagerTest, AllocFinalizeFree) { {/* Finalize: */ cantFail(WrapperFunctionCall::Create>( ExecutorAddr::fromPtr(incrementWrapper), - ExecutorAddr::fromPtr(&FinalizeCounter))), + ExecutorAddr::fromPtr(&InitializeCounter))), /* Deallocate: */ cantFail(WrapperFunctionCall::Create>( ExecutorAddr::fromPtr(incrementWrapper), ExecutorAddr::fromPtr(&DeallocateCounter)))}); - EXPECT_EQ(FinalizeCounter, 0); + EXPECT_EQ(InitializeCounter, 0); EXPECT_EQ(DeallocateCounter, 0); - auto FinalizeErr = MemMgr.finalize(FR); - EXPECT_THAT_ERROR(std::move(FinalizeErr), Succeeded()); + auto InitializeErr = MemMgr.initialize(FR); + EXPECT_THAT_EXPECTED(std::move(InitializeErr), Succeeded()); - EXPECT_EQ(FinalizeCounter, 1); + EXPECT_EQ(InitializeCounter, 1); EXPECT_EQ(DeallocateCounter, 0); EXPECT_EQ(HW, std::string(Mem->toPtr())); - auto DeallocateErr = MemMgr.deallocate({*Mem}); - EXPECT_THAT_ERROR(std::move(DeallocateErr), Succeeded()); + auto ReleaseErr = MemMgr.release({*Mem}); + EXPECT_THAT_ERROR(std::move(ReleaseErr), Succeeded()); - EXPECT_EQ(FinalizeCounter, 1); + EXPECT_EQ(InitializeCounter, 1); EXPECT_EQ(DeallocateCounter, 1); }