diff --git a/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.h b/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.h index f9070afc1a327..eb71e9a162197 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.h @@ -32,8 +32,9 @@ class LLVM_ABI EPCGenericJITLinkMemoryManager struct SymbolAddrs { ExecutorAddr Allocator; ExecutorAddr Reserve; - ExecutorAddr Finalize; - ExecutorAddr Deallocate; + ExecutorAddr Initialize; + ExecutorAddr Deinitialize; + ExecutorAddr Release; }; /// Create an EPCGenericJITLinkMemoryManager instance from a given set of diff --git a/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.h b/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.h index faec25d81fac8..fa48480f265a9 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.h @@ -31,8 +31,8 @@ class LLVM_ABI EPCGenericRTDyldMemoryManager struct SymbolAddrs { ExecutorAddr Instance; ExecutorAddr Reserve; - ExecutorAddr Finalize; - ExecutorAddr Deallocate; + ExecutorAddr Initialize; + ExecutorAddr Release; ExecutorAddr RegisterEHFrame; ExecutorAddr DeregisterEHFrame; }; diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h b/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h index 99ba456e66c85..d68a68992a638 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h @@ -29,8 +29,9 @@ LLVM_ABI extern const char *SimpleExecutorDylibManagerResolveWrapperName; LLVM_ABI extern const char *SimpleExecutorMemoryManagerInstanceName; LLVM_ABI extern const char *SimpleExecutorMemoryManagerReserveWrapperName; -LLVM_ABI extern const char *SimpleExecutorMemoryManagerFinalizeWrapperName; -LLVM_ABI extern const char *SimpleExecutorMemoryManagerDeallocateWrapperName; +LLVM_ABI extern const char *SimpleExecutorMemoryManagerInitializeWrapperName; +LLVM_ABI extern const char *SimpleExecutorMemoryManagerDeinitializeWrapperName; +LLVM_ABI extern const char *SimpleExecutorMemoryManagerReleaseWrapperName; LLVM_ABI extern const char *ExecutorSharedMemoryMapperServiceInstanceName; LLVM_ABI extern const char *ExecutorSharedMemoryMapperServiceReserveWrapperName; @@ -73,9 +74,12 @@ using SPSSimpleExecutorDylibManagerResolveSignature = shared::SPSExpected< using SPSSimpleExecutorMemoryManagerReserveSignature = shared::SPSExpected(shared::SPSExecutorAddr, uint64_t); -using SPSSimpleExecutorMemoryManagerFinalizeSignature = - shared::SPSError(shared::SPSExecutorAddr, shared::SPSFinalizeRequest); -using SPSSimpleExecutorMemoryManagerDeallocateSignature = shared::SPSError( +using SPSSimpleExecutorMemoryManagerInitializeSignature = + shared::SPSExpected(shared::SPSExecutorAddr, + shared::SPSFinalizeRequest); +using SPSSimpleExecutorMemoryManagerDeinitializeSignature = shared::SPSError( + shared::SPSExecutorAddr, shared::SPSSequence); +using SPSSimpleExecutorMemoryManagerReleaseSignature = shared::SPSError( shared::SPSExecutorAddr, shared::SPSSequence); // ExecutorSharedMemoryMapperService @@ -93,6 +97,18 @@ using SPSExecutorSharedMemoryMapperServiceDeinitializeSignature = using SPSExecutorSharedMemoryMapperServiceReleaseSignature = shared::SPSError( shared::SPSExecutorAddr, shared::SPSSequence); +// SimpleNativeMemoryMap APIs. +using SPSSimpleRemoteMemoryMapReserveSignature = + shared::SPSExpected(shared::SPSExecutorAddr, + uint64_t); +using SPSSimpleRemoteMemoryMapInitializeSignature = + shared::SPSExpected(shared::SPSExecutorAddr, + shared::SPSFinalizeRequest); +using SPSSimpleRemoteMemoryMapDeinitializeSignature = shared::SPSError( + shared::SPSExecutorAddr, shared::SPSSequence); +using SPSSimpleRemoteMemoryMapReleaseSignature = shared::SPSError( + shared::SPSExecutorAddr, shared::SPSSequence); + using SPSRunAsMainSignature = int64_t(shared::SPSExecutorAddr, shared::SPSSequence); using SPSRunAsVoidFunctionSignature = int32_t(shared::SPSExecutorAddr); diff --git a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.h b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.h index 741f203396fac..6224e92887147 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.h @@ -34,34 +34,65 @@ class LLVM_ABI SimpleExecutorMemoryManager : public ExecutorBootstrapService { public: virtual ~SimpleExecutorMemoryManager(); - Expected allocate(uint64_t Size); - Error finalize(tpctypes::FinalizeRequest &FR); - Error deallocate(const std::vector &Bases); + Expected reserve(uint64_t Size); + Expected initialize(tpctypes::FinalizeRequest &FR); + Error deinitialize(const std::vector &InitKeys); + Error release(const std::vector &Bases); Error shutdown() override; void addBootstrapSymbols(StringMap &M) override; private: - struct Allocation { + struct RegionInfo { size_t Size = 0; - std::vector DeallocationActions; + std::vector DeallocActions; }; - using AllocationsMap = DenseMap; + struct SlabInfo { + using RegionMap = std::map; + size_t Size = 0; + RegionMap Regions; + }; + + using SlabMap = std::map; + + /// Get a reference to the slab information for the slab containing the given + /// address. + Expected getSlabInfo(ExecutorAddr A, StringRef Context); + + /// Get a reference to the slab information for the slab *covering* the given + /// range. The given range must be a subrange of e(possibly equal to) the + /// range of the slab itself. + Expected getSlabInfo(ExecutorAddrRange R, StringRef Context); - Error deallocateImpl(void *Base, Allocation &A); + /// Create a RegionInfo for the given range, which must not overlap any + /// existing region. + Expected createRegionInfo(ExecutorAddrRange R, + StringRef Context); + + /// Get a reference to the region information for the given address. This + /// address must represent the start of an existing initialized region. + Expected getRegionInfo(SlabInfo &Slab, ExecutorAddr A, + StringRef Context); + + /// Get a reference to the region information for the given address. This + /// address must represent the start of an existing initialized region. + Expected getRegionInfo(ExecutorAddr A, StringRef Context); static llvm::orc::shared::CWrapperFunctionResult reserveWrapper(const char *ArgData, size_t ArgSize); static llvm::orc::shared::CWrapperFunctionResult - finalizeWrapper(const char *ArgData, size_t ArgSize); + initializeWrapper(const char *ArgData, size_t ArgSize); + + static llvm::orc::shared::CWrapperFunctionResult + deinitializeWrapper(const char *ArgData, size_t ArgSize); static llvm::orc::shared::CWrapperFunctionResult - deallocateWrapper(const char *ArgData, size_t ArgSize); + releaseWrapper(const char *ArgData, size_t ArgSize); std::mutex M; - AllocationsMap Allocations; + SlabMap Slabs; }; } // end namespace rt_bootstrap diff --git a/llvm/lib/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.cpp b/llvm/lib/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.cpp index 50e6b254595b8..0833af7f020fc 100644 --- a/llvm/lib/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.cpp +++ b/llvm/lib/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.cpp @@ -57,16 +57,17 @@ class EPCGenericJITLinkMemoryManager::InFlightAlloc std::swap(FR.Actions, G.allocActions()); Parent.EPC.callSPSWrapperAsync< - rt::SPSSimpleExecutorMemoryManagerFinalizeSignature>( - Parent.SAs.Finalize, + rt::SPSSimpleExecutorMemoryManagerInitializeSignature>( + Parent.SAs.Initialize, [OnFinalize = std::move(OnFinalize), AllocAddr = this->AllocAddr]( - Error SerializationErr, Error FinalizeErr) mutable { + Error SerializationErr, + Expected InitializeKey) mutable { // FIXME: Release abandoned alloc. if (SerializationErr) { - cantFail(std::move(FinalizeErr)); + cantFail(InitializeKey.takeError()); OnFinalize(std::move(SerializationErr)); - } else if (FinalizeErr) - OnFinalize(std::move(FinalizeErr)); + } else if (!InitializeKey) + OnFinalize(InitializeKey.takeError()); else OnFinalize(FinalizedAlloc(AllocAddr)); }, @@ -76,8 +77,8 @@ class EPCGenericJITLinkMemoryManager::InFlightAlloc void abandon(OnAbandonedFunction OnAbandoned) override { // FIXME: Return memory to pool instead. Parent.EPC.callSPSWrapperAsync< - rt::SPSSimpleExecutorMemoryManagerDeallocateSignature>( - Parent.SAs.Deallocate, + rt::SPSSimpleExecutorMemoryManagerReleaseSignature>( + Parent.SAs.Release, [OnAbandoned = std::move(OnAbandoned)](Error SerializationErr, Error DeallocateErr) mutable { if (SerializationErr) { @@ -123,9 +124,8 @@ void EPCGenericJITLinkMemoryManager::allocate(const JITLinkDylib *JD, void EPCGenericJITLinkMemoryManager::deallocate( std::vector Allocs, OnDeallocatedFunction OnDeallocated) { - EPC.callSPSWrapperAsync< - rt::SPSSimpleExecutorMemoryManagerDeallocateSignature>( - SAs.Deallocate, + EPC.callSPSWrapperAsync( + SAs.Release, [OnDeallocated = std::move(OnDeallocated)](Error SerErr, Error DeallocErr) mutable { if (SerErr) { diff --git a/llvm/lib/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.cpp b/llvm/lib/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.cpp index fec7062ff79a6..cc72488455ec1 100644 --- a/llvm/lib/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.cpp +++ b/llvm/lib/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.cpp @@ -25,9 +25,9 @@ EPCGenericRTDyldMemoryManager::CreateWithDefaultBootstrapSymbols( if (auto Err = EPC.getBootstrapSymbols( {{SAs.Instance, rt::SimpleExecutorMemoryManagerInstanceName}, {SAs.Reserve, rt::SimpleExecutorMemoryManagerReserveWrapperName}, - {SAs.Finalize, rt::SimpleExecutorMemoryManagerFinalizeWrapperName}, - {SAs.Deallocate, - rt::SimpleExecutorMemoryManagerDeallocateWrapperName}, + {SAs.Initialize, + rt::SimpleExecutorMemoryManagerInitializeWrapperName}, + {SAs.Release, rt::SimpleExecutorMemoryManagerReleaseWrapperName}, {SAs.RegisterEHFrame, rt::RegisterEHFrameSectionAllocActionName}, {SAs.DeregisterEHFrame, rt::DeregisterEHFrameSectionAllocActionName}})) @@ -48,7 +48,7 @@ EPCGenericRTDyldMemoryManager::~EPCGenericRTDyldMemoryManager() { Error Err = Error::success(); if (auto Err2 = EPC.callSPSWrapper< - rt::SPSSimpleExecutorMemoryManagerDeallocateSignature>( + rt::SPSSimpleExecutorMemoryManagerReleaseSignature>( SAs.Reserve, Err, SAs.Instance, FinalizedAllocs)) { // FIXME: Report errors through EPC once that functionality is available. logAllUnhandledErrors(std::move(Err2), errs(), ""); @@ -267,10 +267,10 @@ bool EPCGenericRTDyldMemoryManager::finalizeMemory(std::string *ErrMsg) { // We'll also need to make an extra allocation for the eh-frame wrapper call // arguments. - Error FinalizeErr = Error::success(); + Expected InitializeKey((ExecutorAddr())); if (auto Err = EPC.callSPSWrapper< - rt::SPSSimpleExecutorMemoryManagerFinalizeSignature>( - SAs.Finalize, FinalizeErr, SAs.Instance, std::move(FR))) { + rt::SPSSimpleExecutorMemoryManagerInitializeSignature>( + SAs.Initialize, InitializeKey, SAs.Instance, std::move(FR))) { std::lock_guard Lock(M); this->ErrMsg = toString(std::move(Err)); dbgs() << "Serialization error: " << this->ErrMsg << "\n"; @@ -278,9 +278,9 @@ bool EPCGenericRTDyldMemoryManager::finalizeMemory(std::string *ErrMsg) { *ErrMsg = this->ErrMsg; return true; } - if (FinalizeErr) { + if (!InitializeKey) { std::lock_guard Lock(M); - this->ErrMsg = toString(std::move(FinalizeErr)); + this->ErrMsg = toString(InitializeKey.takeError()); dbgs() << "Finalization error: " << this->ErrMsg << "\n"; if (ErrMsg) *ErrMsg = this->ErrMsg; diff --git a/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp b/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp index 26e8f5382da0e..cc99d3c768772 100644 --- a/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp +++ b/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp @@ -23,10 +23,12 @@ const char *SimpleExecutorMemoryManagerInstanceName = "__llvm_orc_SimpleExecutorMemoryManager_Instance"; const char *SimpleExecutorMemoryManagerReserveWrapperName = "__llvm_orc_SimpleExecutorMemoryManager_reserve_wrapper"; -const char *SimpleExecutorMemoryManagerFinalizeWrapperName = - "__llvm_orc_SimpleExecutorMemoryManager_finalize_wrapper"; -const char *SimpleExecutorMemoryManagerDeallocateWrapperName = - "__llvm_orc_SimpleExecutorMemoryManager_deallocate_wrapper"; +const char *SimpleExecutorMemoryManagerInitializeWrapperName = + "__llvm_orc_SimpleExecutorMemoryManager_initialize_wrapper"; +const char *SimpleExecutorMemoryManagerDeinitializeWrapperName = + "__llvm_orc_SimpleExecutorMemoryManager_deinitialize_wrapper"; +const char *SimpleExecutorMemoryManagerReleaseWrapperName = + "__llvm_orc_SimpleExecutorMemoryManager_release_wrapper"; const char *ExecutorSharedMemoryMapperServiceInstanceName = "__llvm_orc_ExecutorSharedMemoryMapperService_Instance"; diff --git a/llvm/lib/ExecutionEngine/Orc/SimpleRemoteEPC.cpp b/llvm/lib/ExecutionEngine/Orc/SimpleRemoteEPC.cpp index 87d757805a64c..dec1df7da2f4a 100644 --- a/llvm/lib/ExecutionEngine/Orc/SimpleRemoteEPC.cpp +++ b/llvm/lib/ExecutionEngine/Orc/SimpleRemoteEPC.cpp @@ -216,9 +216,9 @@ SimpleRemoteEPC::createDefaultMemoryManager(SimpleRemoteEPC &SREPC) { if (auto Err = SREPC.getBootstrapSymbols( {{SAs.Allocator, rt::SimpleExecutorMemoryManagerInstanceName}, {SAs.Reserve, rt::SimpleExecutorMemoryManagerReserveWrapperName}, - {SAs.Finalize, rt::SimpleExecutorMemoryManagerFinalizeWrapperName}, - {SAs.Deallocate, - rt::SimpleExecutorMemoryManagerDeallocateWrapperName}})) + {SAs.Initialize, + rt::SimpleExecutorMemoryManagerInitializeWrapperName}, + {SAs.Release, rt::SimpleExecutorMemoryManagerReleaseWrapperName}})) return std::move(Err); return std::make_unique(SREPC, SAs); diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.cpp b/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.cpp index 3cdffb8cd0615..fe881a1eef773 100644 --- a/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.cpp +++ b/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.cpp @@ -8,6 +8,7 @@ #include "llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.h" +#include "llvm/ADT/ScopeExit.h" #include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h" #include "llvm/Support/FormatVariadic.h" @@ -18,166 +19,167 @@ namespace orc { namespace rt_bootstrap { SimpleExecutorMemoryManager::~SimpleExecutorMemoryManager() { - assert(Allocations.empty() && "shutdown not called?"); + assert(Slabs.empty() && "shutdown not called?"); } -Expected SimpleExecutorMemoryManager::allocate(uint64_t Size) { +Expected SimpleExecutorMemoryManager::reserve(uint64_t Size) { std::error_code EC; auto MB = sys::Memory::allocateMappedMemory( Size, nullptr, sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC); if (EC) return errorCodeToError(EC); std::lock_guard Lock(M); - assert(!Allocations.count(MB.base()) && "Duplicate allocation addr"); - Allocations[MB.base()].Size = Size; + assert(!Slabs.count(MB.base()) && "Duplicate allocation addr"); + Slabs[MB.base()].Size = Size; return ExecutorAddr::fromPtr(MB.base()); } -Error SimpleExecutorMemoryManager::finalize(tpctypes::FinalizeRequest &FR) { - ExecutorAddr Base(~0ULL); +Expected +SimpleExecutorMemoryManager::initialize(tpctypes::FinalizeRequest &FR) { std::vector DeallocationActions; - size_t SuccessfulFinalizationActions = 0; if (FR.Segments.empty()) { - // NOTE: Finalizing nothing is currently a no-op. Should it be an error? if (FR.Actions.empty()) - return Error::success(); + return make_error("Finalization request is empty", + inconvertibleErrorCode()); else return make_error("Finalization actions attached to empty " "finalization request", inconvertibleErrorCode()); } - for (auto &Seg : FR.Segments) - Base = std::min(Base, Seg.Addr); - - for (auto &ActPair : FR.Actions) - if (ActPair.Dealloc) - DeallocationActions.push_back(ActPair.Dealloc); - - // Get the Allocation for this finalization. - size_t AllocSize = 0; - { - std::lock_guard Lock(M); - auto I = Allocations.find(Base.toPtr()); - if (I == Allocations.end()) - return make_error("Attempt to finalize unrecognized " - "allocation " + - formatv("{0:x}", Base.getValue()), - inconvertibleErrorCode()); - AllocSize = I->second.Size; - I->second.DeallocationActions = std::move(DeallocationActions); - } - ExecutorAddr AllocEnd = Base + ExecutorAddrDiff(AllocSize); - - // Bail-out function: this will run deallocation actions corresponding to any - // completed finalization actions, then deallocate memory. - auto BailOut = [&](Error Err) { - std::pair AllocToDestroy; - - // Get allocation to destroy. - { - std::lock_guard Lock(M); - auto I = Allocations.find(Base.toPtr()); - - // Check for missing allocation (effective a double free). - if (I == Allocations.end()) - return joinErrors( - std::move(Err), - make_error("No allocation entry found " - "for " + - formatv("{0:x}", Base.getValue()), - inconvertibleErrorCode())); - AllocToDestroy = std::move(*I); - Allocations.erase(I); - } + ExecutorAddrRange RR(FR.Segments.front().Addr, FR.Segments.front().Addr); - // Run deallocation actions for all completed finalization actions. - while (SuccessfulFinalizationActions) - Err = - joinErrors(std::move(Err), FR.Actions[--SuccessfulFinalizationActions] - .Dealloc.runWithSPSRetErrorMerged()); - - // Deallocate memory. - sys::MemoryBlock MB(AllocToDestroy.first, AllocToDestroy.second.Size); - if (auto EC = sys::Memory::releaseMappedMemory(MB)) - Err = joinErrors(std::move(Err), errorCodeToError(EC)); - - return Err; - }; + std::vector MBsToReset; + auto ResetMBs = make_scope_exit([&]() { + for (auto &MB : MBsToReset) + sys::Memory::protectMappedMemory(MB, sys::Memory::MF_READ | + sys::Memory::MF_WRITE); + sys::Memory::InvalidateInstructionCache(RR.Start.toPtr(), + RR.size()); + }); // Copy content and apply permissions. for (auto &Seg : FR.Segments) { + RR.Start = std::min(RR.Start, Seg.Addr); + RR.End = std::max(RR.End, Seg.Addr + Seg.Size); // Check segment ranges. if (LLVM_UNLIKELY(Seg.Size < Seg.Content.size())) - return BailOut(make_error( + return make_error( formatv("Segment {0:x} content size ({1:x} bytes) " "exceeds segment size ({2:x} bytes)", Seg.Addr.getValue(), Seg.Content.size(), Seg.Size), - inconvertibleErrorCode())); + inconvertibleErrorCode()); ExecutorAddr SegEnd = Seg.Addr + ExecutorAddrDiff(Seg.Size); - if (LLVM_UNLIKELY(Seg.Addr < Base || SegEnd > AllocEnd)) - return BailOut(make_error( + if (LLVM_UNLIKELY(Seg.Addr < RR.Start || SegEnd > RR.End)) + return make_error( formatv("Segment {0:x} -- {1:x} crosses boundary of " "allocation {2:x} -- {3:x}", - Seg.Addr.getValue(), SegEnd.getValue(), Base.getValue(), - AllocEnd.getValue()), - inconvertibleErrorCode())); + Seg.Addr, SegEnd, RR.Start, RR.End), + inconvertibleErrorCode()); char *Mem = Seg.Addr.toPtr(); if (!Seg.Content.empty()) memcpy(Mem, Seg.Content.data(), Seg.Content.size()); memset(Mem + Seg.Content.size(), 0, Seg.Size - Seg.Content.size()); assert(Seg.Size <= std::numeric_limits::max()); + + sys::MemoryBlock MB(Mem, Seg.Size); if (auto EC = sys::Memory::protectMappedMemory( - {Mem, static_cast(Seg.Size)}, - toSysMemoryProtectionFlags(Seg.RAG.Prot))) - return BailOut(errorCodeToError(EC)); + MB, toSysMemoryProtectionFlags(Seg.RAG.Prot))) + return errorCodeToError(EC); + + MBsToReset.push_back(MB); + if ((Seg.RAG.Prot & MemProt::Exec) == MemProt::Exec) sys::Memory::InvalidateInstructionCache(Mem, Seg.Size); } - // Run finalization actions. - for (auto &ActPair : FR.Actions) { - if (auto Err = ActPair.Finalize.runWithSPSRetErrorMerged()) - return BailOut(std::move(Err)); - ++SuccessfulFinalizationActions; + auto DeallocActions = runFinalizeActions(FR.Actions); + if (!DeallocActions) + return DeallocActions.takeError(); + + { + std::lock_guard Lock(M); + auto Region = createRegionInfo(RR, "In initialize"); + if (!Region) + return Region.takeError(); + Region->DeallocActions = std::move(*DeallocActions); } - return Error::success(); + // Successful initialization. + ResetMBs.release(); + + return RR.Start; } -Error SimpleExecutorMemoryManager::deallocate( - const std::vector &Bases) { - std::vector> AllocPairs; - AllocPairs.reserve(Bases.size()); +Error SimpleExecutorMemoryManager::deinitialize( + const std::vector &InitKeys) { + Error Err = Error::success(); - // Get allocation to destroy. + for (auto &KeyAddr : llvm::reverse(InitKeys)) { + std::vector DeallocActions; + { + std::scoped_lock Lock(M); + auto Slab = getSlabInfo(KeyAddr, "In deinitialize"); + if (!Slab) { + Err = joinErrors(std::move(Err), Slab.takeError()); + continue; + } + + auto RI = getRegionInfo(*Slab, KeyAddr, "In deinitialize"); + if (!RI) { + Err = joinErrors(std::move(Err), RI.takeError()); + continue; + } + + DeallocActions = std::move(RI->DeallocActions); + } + + Err = joinErrors(std::move(Err), + runDeallocActions(std::move(DeallocActions))); + } + + return Err; +} + +Error SimpleExecutorMemoryManager::release( + const std::vector &Bases) { Error Err = Error::success(); - { - std::lock_guard Lock(M); - for (auto &Base : Bases) { - auto I = Allocations.find(Base.toPtr()); - - // Check for missing allocation (effective a double free). - if (I != Allocations.end()) { - AllocPairs.push_back(std::move(*I)); - Allocations.erase(I); - } else + + // TODO: Prohibit new initializations within the slabs being removed? + for (auto &Base : llvm::reverse(Bases)) { + std::vector DeallocActions; + sys::MemoryBlock MB; + + { + std::scoped_lock Lock(M); + + auto SlabI = Slabs.find(Base.toPtr()); + if (SlabI == Slabs.end()) { Err = joinErrors( std::move(Err), - make_error("No allocation entry found " - "for " + - formatv("{0:x}", Base.getValue()), + make_error("In release, " + formatv("{0:x}", Base) + + " is not part of any reserved " + "address range", inconvertibleErrorCode())); + continue; + } + + auto &Slab = SlabI->second; + + for (auto &[Addr, Region] : Slab.Regions) + llvm::copy(Region.DeallocActions, back_inserter(DeallocActions)); + + MB = {Base.toPtr(), Slab.Size}; + + Slabs.erase(SlabI); } - } - while (!AllocPairs.empty()) { - auto &P = AllocPairs.back(); - Err = joinErrors(std::move(Err), deallocateImpl(P.first, P.second)); - AllocPairs.pop_back(); + Err = joinErrors(std::move(Err), runDeallocActions(DeallocActions)); + if (auto EC = sys::Memory::releaseMappedMemory(MB)) + Err = joinErrors(std::move(Err), errorCodeToError(EC)); } return Err; @@ -185,16 +187,15 @@ Error SimpleExecutorMemoryManager::deallocate( Error SimpleExecutorMemoryManager::shutdown() { - AllocationsMap AM; + // TODO: Prevent new allocations during shutdown. + std::vector Bases; { - std::lock_guard Lock(M); - AM = std::move(Allocations); + std::scoped_lock Lock(M); + for (auto &[Base, Slab] : Slabs) + Bases.push_back(ExecutorAddr::fromPtr(Base)); } - Error Err = Error::success(); - for (auto &KV : AM) - Err = joinErrors(std::move(Err), deallocateImpl(KV.first, KV.second)); - return Err; + return release(Bases); } void SimpleExecutorMemoryManager::addBootstrapSymbols( @@ -202,58 +203,150 @@ void SimpleExecutorMemoryManager::addBootstrapSymbols( M[rt::SimpleExecutorMemoryManagerInstanceName] = ExecutorAddr::fromPtr(this); M[rt::SimpleExecutorMemoryManagerReserveWrapperName] = ExecutorAddr::fromPtr(&reserveWrapper); - M[rt::SimpleExecutorMemoryManagerFinalizeWrapperName] = - ExecutorAddr::fromPtr(&finalizeWrapper); - M[rt::SimpleExecutorMemoryManagerDeallocateWrapperName] = - ExecutorAddr::fromPtr(&deallocateWrapper); + M[rt::SimpleExecutorMemoryManagerInitializeWrapperName] = + ExecutorAddr::fromPtr(&initializeWrapper); + M[rt::SimpleExecutorMemoryManagerDeinitializeWrapperName] = + ExecutorAddr::fromPtr(&deinitializeWrapper); + M[rt::SimpleExecutorMemoryManagerReleaseWrapperName] = + ExecutorAddr::fromPtr(&releaseWrapper); } -Error SimpleExecutorMemoryManager::deallocateImpl(void *Base, Allocation &A) { - Error Err = Error::success(); +Expected +SimpleExecutorMemoryManager::getSlabInfo(ExecutorAddr A, StringRef Context) { + auto MakeBadSlabError = [&]() { + return make_error( + Context + ", address " + formatv("{0:x}", A) + + " is not part of any reserved address range", + inconvertibleErrorCode()); + }; - while (!A.DeallocationActions.empty()) { - Err = joinErrors(std::move(Err), - A.DeallocationActions.back().runWithSPSRetErrorMerged()); - A.DeallocationActions.pop_back(); + auto I = Slabs.upper_bound(A.toPtr()); + if (I == Slabs.begin()) + return MakeBadSlabError(); + --I; + if (!ExecutorAddrRange(ExecutorAddr::fromPtr(I->first), I->second.Size) + .contains(A)) + return MakeBadSlabError(); + + return I->second; +} + +Expected +SimpleExecutorMemoryManager::getSlabInfo(ExecutorAddrRange R, + StringRef Context) { + auto MakeBadSlabError = [&]() { + return make_error( + Context + ", range " + formatv("{0:x}", R) + + " is not part of any reserved address range", + inconvertibleErrorCode()); + }; + + auto I = Slabs.upper_bound(R.Start.toPtr()); + if (I == Slabs.begin()) + return MakeBadSlabError(); + --I; + if (!ExecutorAddrRange(ExecutorAddr::fromPtr(I->first), I->second.Size) + .contains(R)) + return MakeBadSlabError(); + + return I->second; +} + +Expected +SimpleExecutorMemoryManager::createRegionInfo(ExecutorAddrRange R, + StringRef Context) { + + auto Slab = getSlabInfo(R, Context); + if (!Slab) + return Slab.takeError(); + + auto MakeBadRegionError = [&](ExecutorAddrRange Other, bool Prev) { + return make_error(Context + ", region " + formatv("{0:x}", R) + + " overlaps " + + (Prev ? "previous" : "following") + + " region " + formatv("{0:x}", Other), + inconvertibleErrorCode()); + }; + + auto I = Slab->Regions.upper_bound(R.Start); + if (I != Slab->Regions.begin()) { + auto J = std::prev(I); + ExecutorAddrRange PrevRange(J->first, J->second.Size); + if (PrevRange.overlaps(R)) + return MakeBadRegionError(PrevRange, true); + } + if (I != Slab->Regions.end()) { + ExecutorAddrRange NextRange(I->first, I->second.Size); + if (NextRange.overlaps(R)) + return MakeBadRegionError(NextRange, false); } - sys::MemoryBlock MB(Base, A.Size); - if (auto EC = sys::Memory::releaseMappedMemory(MB)) - Err = joinErrors(std::move(Err), errorCodeToError(EC)); + auto &RInfo = Slab->Regions[R.Start]; + RInfo.Size = R.size(); + return RInfo; +} - return Err; +Expected +SimpleExecutorMemoryManager::getRegionInfo(SlabInfo &Slab, ExecutorAddr A, + StringRef Context) { + auto I = Slab.Regions.find(A); + if (I == Slab.Regions.end()) + return make_error( + Context + ", address " + formatv("{0:x}", A) + + " does not correspond to the start of any initialized region", + inconvertibleErrorCode()); + + return I->second; +} + +Expected +SimpleExecutorMemoryManager::getRegionInfo(ExecutorAddr A, StringRef Context) { + auto Slab = getSlabInfo(A, Context); + if (!Slab) + return Slab.takeError(); + + return getRegionInfo(*Slab, A, Context); } llvm::orc::shared::CWrapperFunctionResult SimpleExecutorMemoryManager::reserveWrapper(const char *ArgData, size_t ArgSize) { - return shared::WrapperFunction< - rt::SPSSimpleExecutorMemoryManagerReserveSignature>:: + return shared::WrapperFunction:: handle(ArgData, ArgSize, shared::makeMethodWrapperHandler( - &SimpleExecutorMemoryManager::allocate)) + &SimpleExecutorMemoryManager::reserve)) + .release(); +} + +llvm::orc::shared::CWrapperFunctionResult +SimpleExecutorMemoryManager::initializeWrapper(const char *ArgData, + size_t ArgSize) { + return shared:: + WrapperFunction::handle( + ArgData, ArgSize, + shared::makeMethodWrapperHandler( + &SimpleExecutorMemoryManager::initialize)) .release(); } llvm::orc::shared::CWrapperFunctionResult -SimpleExecutorMemoryManager::finalizeWrapper(const char *ArgData, - size_t ArgSize) { +SimpleExecutorMemoryManager::deinitializeWrapper(const char *ArgData, + size_t ArgSize) { return shared::WrapperFunction< - rt::SPSSimpleExecutorMemoryManagerFinalizeSignature>:: + rt::SPSSimpleRemoteMemoryMapDeinitializeSignature>:: handle(ArgData, ArgSize, shared::makeMethodWrapperHandler( - &SimpleExecutorMemoryManager::finalize)) + &SimpleExecutorMemoryManager::deinitialize)) .release(); } llvm::orc::shared::CWrapperFunctionResult -SimpleExecutorMemoryManager::deallocateWrapper(const char *ArgData, - size_t ArgSize) { - return shared::WrapperFunction< - rt::SPSSimpleExecutorMemoryManagerDeallocateSignature>:: +SimpleExecutorMemoryManager::releaseWrapper(const char *ArgData, + size_t ArgSize) { + return shared::WrapperFunction:: handle(ArgData, ArgSize, shared::makeMethodWrapperHandler( - &SimpleExecutorMemoryManager::deallocate)) + &SimpleExecutorMemoryManager::release)) .release(); } diff --git a/llvm/unittests/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManagerTest.cpp b/llvm/unittests/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManagerTest.cpp index d4b45ea0dca0b..2c6650d25a0ec 100644 --- a/llvm/unittests/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManagerTest.cpp +++ b/llvm/unittests/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManagerTest.cpp @@ -39,8 +39,11 @@ class SimpleAllocator { return ExecutorAddr::fromPtr(MB.base()); } - Error finalize(tpctypes::FinalizeRequest FR) { + Expected initialize(tpctypes::FinalizeRequest FR) { + assert(!FR.Segments.empty()); + ExecutorAddr Base = FR.Segments[0].Addr; for (auto &Seg : FR.Segments) { + Base = std::min(Base, Seg.Addr); char *Mem = Seg.Addr.toPtr(); memcpy(Mem, Seg.Content.data(), Seg.Content.size()); memset(Mem + Seg.Content.size(), 0, Seg.Size - Seg.Content.size()); @@ -52,10 +55,10 @@ class SimpleAllocator { if ((Seg.RAG.Prot & MemProt::Exec) != MemProt::Exec) sys::Memory::InvalidateInstructionCache(Mem, Seg.Size); } - return Error::success(); + return Base; } - Error deallocate(std::vector &Bases) { + Error release(std::vector &Bases) { Error Err = Error::success(); for (auto &Base : Bases) { auto I = Blocks.find(Base.toPtr()); @@ -86,18 +89,18 @@ CWrapperFunctionResult testReserve(const char *ArgData, size_t ArgSize) { .release(); } -CWrapperFunctionResult testFinalize(const char *ArgData, size_t ArgSize) { - return WrapperFunction:: +CWrapperFunctionResult testInitialize(const char *ArgData, size_t ArgSize) { + return WrapperFunction< + rt::SPSSimpleExecutorMemoryManagerInitializeSignature>:: handle(ArgData, ArgSize, - makeMethodWrapperHandler(&SimpleAllocator::finalize)) + makeMethodWrapperHandler(&SimpleAllocator::initialize)) .release(); } -CWrapperFunctionResult testDeallocate(const char *ArgData, size_t ArgSize) { - return WrapperFunction< - rt::SPSSimpleExecutorMemoryManagerDeallocateSignature>:: +CWrapperFunctionResult testRelease(const char *ArgData, size_t ArgSize) { + return WrapperFunction:: handle(ArgData, ArgSize, - makeMethodWrapperHandler(&SimpleAllocator::deallocate)) + makeMethodWrapperHandler(&SimpleAllocator::release)) .release(); } @@ -108,8 +111,8 @@ TEST(EPCGenericJITLinkMemoryManagerTest, AllocFinalizeFree) { EPCGenericJITLinkMemoryManager::SymbolAddrs SAs; SAs.Allocator = ExecutorAddr::fromPtr(&SA); SAs.Reserve = ExecutorAddr::fromPtr(&testReserve); - SAs.Finalize = ExecutorAddr::fromPtr(&testFinalize); - SAs.Deallocate = ExecutorAddr::fromPtr(&testDeallocate); + SAs.Initialize = ExecutorAddr::fromPtr(&testInitialize); + SAs.Release = ExecutorAddr::fromPtr(&testRelease); auto MemMgr = std::make_unique(*SelfEPC, SAs); StringRef Hello = "hello"; diff --git a/llvm/unittests/ExecutionEngine/Orc/SimpleExecutorMemoryManagerTest.cpp b/llvm/unittests/ExecutionEngine/Orc/SimpleExecutorMemoryManagerTest.cpp index 6e9b0b2d8c73f..9c6f19c82c998 100644 --- a/llvm/unittests/ExecutionEngine/Orc/SimpleExecutorMemoryManagerTest.cpp +++ b/llvm/unittests/ExecutionEngine/Orc/SimpleExecutorMemoryManagerTest.cpp @@ -34,12 +34,12 @@ TEST(SimpleExecutorMemoryManagerTest, AllocFinalizeFree) { SimpleExecutorMemoryManager MemMgr; constexpr unsigned AllocSize = 16384; - auto Mem = MemMgr.allocate(AllocSize); + auto Mem = MemMgr.reserve(AllocSize); EXPECT_THAT_ERROR(Mem.takeError(), Succeeded()); std::string HW = "Hello, world!"; - int FinalizeCounter = 0; + int InitializeCounter = 0; int DeallocateCounter = 0; tpctypes::FinalizeRequest FR; @@ -52,27 +52,27 @@ TEST(SimpleExecutorMemoryManagerTest, AllocFinalizeFree) { {/* Finalize: */ cantFail(WrapperFunctionCall::Create>( ExecutorAddr::fromPtr(incrementWrapper), - ExecutorAddr::fromPtr(&FinalizeCounter))), + ExecutorAddr::fromPtr(&InitializeCounter))), /* Deallocate: */ cantFail(WrapperFunctionCall::Create>( ExecutorAddr::fromPtr(incrementWrapper), ExecutorAddr::fromPtr(&DeallocateCounter)))}); - EXPECT_EQ(FinalizeCounter, 0); + EXPECT_EQ(InitializeCounter, 0); EXPECT_EQ(DeallocateCounter, 0); - auto FinalizeErr = MemMgr.finalize(FR); - EXPECT_THAT_ERROR(std::move(FinalizeErr), Succeeded()); + auto InitializeErr = MemMgr.initialize(FR); + EXPECT_THAT_EXPECTED(std::move(InitializeErr), Succeeded()); - EXPECT_EQ(FinalizeCounter, 1); + EXPECT_EQ(InitializeCounter, 1); EXPECT_EQ(DeallocateCounter, 0); EXPECT_EQ(HW, std::string(Mem->toPtr())); - auto DeallocateErr = MemMgr.deallocate({*Mem}); - EXPECT_THAT_ERROR(std::move(DeallocateErr), Succeeded()); + auto ReleaseErr = MemMgr.release({*Mem}); + EXPECT_THAT_ERROR(std::move(ReleaseErr), Succeeded()); - EXPECT_EQ(FinalizeCounter, 1); + EXPECT_EQ(InitializeCounter, 1); EXPECT_EQ(DeallocateCounter, 1); }