diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Core.h b/llvm/include/llvm/ExecutionEngine/Orc/Core.h index f407b56817fc3..8613ddd8e3b11 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/Core.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Core.h @@ -26,6 +26,7 @@ #include "llvm/ExecutionEngine/Orc/Shared/ExecutorSymbolDef.h" #include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h" #include "llvm/ExecutionEngine/Orc/TaskDispatch.h" +#include "llvm/ExecutionEngine/Orc/WaitingOnGraph.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ExtensibleRTTI.h" @@ -49,6 +50,9 @@ class InProgressLookupState; enum class SymbolState : uint8_t; +using WaitingOnGraph = + detail::WaitingOnGraph; + using ResourceTrackerSP = IntrusiveRefCntPtr; using JITDylibSP = IntrusiveRefCntPtr; @@ -1131,20 +1135,6 @@ class JITDylib : public ThreadSafeRefCountedBase, using UnmaterializedInfosList = std::vector>; - struct EmissionDepUnit { - EmissionDepUnit(JITDylib &JD) : JD(&JD) {} - - JITDylib *JD = nullptr; - DenseMap Symbols; - DenseMap> Dependencies; - }; - - struct EmissionDepUnitInfo { - std::shared_ptr EDU; - DenseSet IntraEmitUsers; - DenseMap> NewDeps; - }; - // Information about not-yet-ready symbol. // * DefiningEDU will point to the EmissionDepUnit that defines the symbol. // * DependantEDUs will hold pointers to any EmissionDepUnits currently @@ -1154,9 +1144,6 @@ class JITDylib : public ThreadSafeRefCountedBase, struct MaterializingInfo { friend class ExecutionSession; - std::shared_ptr DefiningEDU; - DenseSet DependantEDUs; - LLVM_ABI void addQuery(std::shared_ptr Q); LLVM_ABI void removeQuery(const AsynchronousSymbolQuery &Q); LLVM_ABI AsynchronousSymbolQueryList @@ -1778,30 +1765,26 @@ class ExecutionSession { LLVM_ABI Error OL_notifyResolved(MaterializationResponsibility &MR, const SymbolMap &Symbols); - using EDUInfosMap = - DenseMap; - - template - void propagateExtraEmitDeps(std::deque Worklist, - EDUInfosMap &EDUInfos, - HandleNewDepFn HandleNewDep); - EDUInfosMap simplifyDepGroups(MaterializationResponsibility &MR, - ArrayRef EmittedDeps); - void IL_makeEDUReady(std::shared_ptr EDU, - JITDylib::AsynchronousSymbolQuerySet &Queries); - void IL_makeEDUEmitted(std::shared_ptr EDU, - JITDylib::AsynchronousSymbolQuerySet &Queries); - bool IL_removeEDUDependence(JITDylib::EmissionDepUnit &EDU, JITDylib &DepJD, - NonOwningSymbolStringPtr DepSym, - EDUInfosMap &EDUInfos); - - static Error makeJDClosedError(JITDylib::EmissionDepUnit &EDU, - JITDylib &ClosedJD); - static Error makeUnsatisfiedDepsError(JITDylib::EmissionDepUnit &EDU, - JITDylib &BadJD, SymbolNameSet BadDeps); - - Expected - IL_emit(MaterializationResponsibility &MR, EDUInfosMap EDUInfos); + // FIXME: We should be able to derive FailedSymsForQuery from each query once + // we fix how the detach operation works. + struct EmitQueries { + JITDylib::AsynchronousSymbolQuerySet Updated; + JITDylib::AsynchronousSymbolQuerySet Failed; + DenseMap> + FailedSymsForQuery; + }; + + WaitingOnGraph::ExternalState + IL_getSymbolState(JITDylib *JD, NonOwningSymbolStringPtr Name); + + template + void IL_collectQueries(JITDylib::AsynchronousSymbolQuerySet &Qs, + WaitingOnGraph::ContainerElementsMap &QualifiedSymbols, + UpdateSymbolFn &&UpdateSymbol, + UpdateQueryFn &&UpdateQuery); + + Expected IL_emit(MaterializationResponsibility &MR, + WaitingOnGraph::SimplifyResult SR); LLVM_ABI Error OL_notifyEmitted(MaterializationResponsibility &MR, ArrayRef EmittedDeps); @@ -1830,6 +1813,7 @@ class ExecutionSession { std::vector ResourceManagers; std::vector JDs; + WaitingOnGraph G; // FIXME: Remove this (and runOutstandingMUs) once the linking layer works // with callbacks from asynchronous queries. diff --git a/llvm/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h b/llvm/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h index ed6ea961bccfe..5b65a175527d7 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h @@ -14,6 +14,7 @@ #define LLVM_EXECUTIONENGINE_ORC_SYMBOLSTRINGPOOL_H #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/Hashing.h" #include "llvm/ADT/StringMap.h" #include "llvm/Support/Compiler.h" #include @@ -71,6 +72,7 @@ class SymbolStringPool { /// from nullptr to enable comparison with these values. class SymbolStringPtrBase { friend class SymbolStringPool; + friend class SymbolStringPoolEntryUnsafe; friend struct DenseMapInfo; friend struct DenseMapInfo; @@ -204,7 +206,7 @@ class SymbolStringPoolEntryUnsafe { SymbolStringPoolEntryUnsafe(PoolEntry *E) : E(E) {} /// Create an unsafe pool entry ref without changing the ref-count. - static SymbolStringPoolEntryUnsafe from(const SymbolStringPtr &S) { + static SymbolStringPoolEntryUnsafe from(const SymbolStringPtrBase &S) { return S.S; } @@ -318,6 +320,10 @@ SymbolStringPool::getRefCount(const SymbolStringPtrBase &S) const { LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const SymbolStringPtrBase &Sym); +inline hash_code hash_value(const orc::NonOwningSymbolStringPtr &S) { + return hash_value(orc::SymbolStringPoolEntryUnsafe::from(S).rawPtr()); +} + } // end namespace orc template <> diff --git a/llvm/include/llvm/ExecutionEngine/Orc/WaitingOnGraph.h b/llvm/include/llvm/ExecutionEngine/Orc/WaitingOnGraph.h new file mode 100644 index 0000000000000..a5b533351d4d0 --- /dev/null +++ b/llvm/include/llvm/ExecutionEngine/Orc/WaitingOnGraph.h @@ -0,0 +1,622 @@ +//===------ WaitingOnGraph.h - ORC symbol dependence graph ------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Defines WaitingOnGraph and related utilities. +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_EXECUTIONENGINE_ORC_WAITINGONGRAPH_H +#define LLVM_EXECUTIONENGINE_ORC_WAITINGONGRAPH_H + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/raw_ostream.h" + +#include + +namespace llvm::orc::detail { + +class WaitingOnGraphTest; + +/// WaitingOnGraph class template. +/// +/// This type is intended to provide efficient dependence tracking for Symbols +/// in an ORC program. +/// +/// WaitingOnGraph models a directed graph with four partitions: +/// 1. Not-yet-emitted nodes: Nodes identified as waited-on in an emit +/// operation. +/// 2. Emitted nodes: Nodes emitted and waiting on some non-empty set of +/// other nodes. +/// 3. Ready nodes: Nodes emitted and not waiting on any other nodes +/// (either because they weren't waiting on any nodes when they were +/// emitted, or because all transitively waited-on nodes have since +/// been emitted). +/// 4. Failed nodes: Nodes that have been marked as failed-to-emit, and +/// nodes that were found to transitively wait-on some failed node. +/// +/// Nodes are added to the graph by *emit* and *fail* operations. +/// +/// The *emit* operation takes a bipartite *local dependence graph* as an +/// argument and returns... +/// a. the set of nodes (both existing and newly added from the local +/// dependence graph) whose waiting-on set is the empty set, and... +/// b. the set of newly added nodes that are found to depend on failed +/// nodes. +/// +/// The *fail* operation takes a set of failed nodes and returns the set of +/// Emitted nodes that were waiting on the failed nodes. +/// +/// The concrete representation adopts several approaches for efficiency: +/// +/// 1. Only *Emitted* and *Not-yet-emitted* nodes are represented explicitly. +/// *Ready* and *Failed* nodes are represented by the values returned by the +/// GetExternalStateFn argument to *emit*. +/// +/// 2. Labels are (*Container*, *Element*) pairs that are intended to represent +/// ORC symbols (ORC uses types Container = JITDylib, +/// Element = NonOwningSymbolStringPtr). The internal representation of the +/// graph is optimized on the assumption that there are many more Elements +/// (symbol names) than Containers (JITDylibs) used to construct the labels. +/// (Consider for example the common case where most JIT'd code is placed in +/// a single "main" JITDylib). +/// +/// 3. The data structure stores *SuperNodes* which have multiple labels. This +/// reduces the number of nodes and edges in the graph in the common case +/// where many JIT symbols have the same set of dependencies. SuperNodes are +/// coalesced when their dependence sets become equal. +/// +/// 4. The *simplify* method can be applied to an initial *local dependence +/// graph* (as a list of SuperNodes) to eliminate any internal dependence +/// relationships that would have to be propagated internally by *emit*. +/// Access to the WaitingOnGraph is assumed to be guarded by a mutex (ORC +/// will access it from multiple threads) so this allows some pre-processing +/// to be performed outside the mutex. +template class WaitingOnGraph { + friend class WaitingOnGraphTest; + +public: + using ContainerId = ContainerIdT; + using ElementId = ElementIdT; + using ElementSet = DenseSet; + using ContainerElementsMap = DenseMap; + + class SuperNode { + friend class WaitingOnGraph; + friend class WaitingOnGraphTest; + + public: + SuperNode(ContainerElementsMap Defs, ContainerElementsMap Deps) + : Defs(std::move(Defs)), Deps(std::move(Deps)) {} + ContainerElementsMap &defs() { return Defs; } + const ContainerElementsMap &defs() const { return Defs; } + ContainerElementsMap &deps() { return Deps; } + const ContainerElementsMap &deps() const { return Deps; } + + private: + ContainerElementsMap Defs; + ContainerElementsMap Deps; + }; + +private: + using ElemToSuperNodeMap = + DenseMap>; + + using SuperNodeDepsMap = DenseMap>; + + class Coalescer { + public: + std::unique_ptr addOrCreateSuperNode(ContainerElementsMap Defs, + ContainerElementsMap Deps) { + auto H = getHash(Deps); + if (auto *ExistingSN = findCanonicalSuperNode(H, Deps)) { + for (auto &[Container, Elems] : Defs) { + auto &DstCElems = ExistingSN->Defs[Container]; + [[maybe_unused]] size_t ExpectedSize = + DstCElems.size() + Elems.size(); + DstCElems.insert(Elems.begin(), Elems.end()); + assert(DstCElems.size() == ExpectedSize); + } + return nullptr; + } + + auto NewSN = + std::make_unique(std::move(Defs), std::move(Deps)); + CanonicalSNs[H].push_back(NewSN.get()); + return NewSN; + } + + void coalesce(std::vector> &SNs, + ElemToSuperNodeMap &ElemToSN) { + for (size_t I = 0; I != SNs.size();) { + auto &SN = SNs[I]; + auto H = getHash(SN->Deps); + if (auto *CanonicalSN = findCanonicalSuperNode(H, SN->Deps)) { + for (auto &[Container, Elems] : SN->Defs) { + CanonicalSN->Defs[Container].insert(Elems.begin(), Elems.end()); + auto &ContainerElemToSN = ElemToSN[Container]; + for (auto &Elem : Elems) + ContainerElemToSN[Elem] = CanonicalSN; + } + std::swap(SN, SNs.back()); + SNs.pop_back(); + } else { + CanonicalSNs[H].push_back(SN.get()); + ++I; + } + } + } + + template void remove(Pred &&Remove) { + for (auto &[Hash, SNs] : CanonicalSNs) { + bool Found = false; + for (size_t I = 0; I != SNs.size(); ++I) { + if (Remove(SNs[I])) { + std::swap(SNs[I], SNs.back()); + SNs.pop_back(); + Found = true; + break; + } + } + if (Found) { + if (SNs.empty()) + CanonicalSNs.erase(Hash); + break; + } + } + } + + private: + hash_code getHash(const ContainerElementsMap &M) { + SmallVector SortedContainers; + SortedContainers.reserve(M.size()); + for (auto &[Container, Elems] : M) + SortedContainers.push_back(Container); + llvm::sort(SortedContainers); + hash_code Hash(0); + for (auto &Container : SortedContainers) { + auto &ContainerElems = M.at(Container); + SmallVector SortedElems(ContainerElems.begin(), + ContainerElems.end()); + llvm::sort(SortedElems); + Hash = hash_combine( + Hash, Container, + hash_combine_range(SortedElems.begin(), SortedElems.end())); + } + return Hash; + } + + SuperNode *findCanonicalSuperNode(hash_code H, + const ContainerElementsMap &M) { + for (auto *SN : CanonicalSNs[H]) + if (SN->Deps == M) + return SN; + return nullptr; + } + + DenseMap> CanonicalSNs; + }; + +public: + /// Build SuperNodes from (definition-set, dependence-set) pairs. + /// + /// Coalesces definition-sets with identical dependence-sets. + class SuperNodeBuilder { + public: + void add(ContainerElementsMap Defs, ContainerElementsMap Deps) { + if (Defs.empty()) + return; + // Remove any self-reference. + SmallVector ToRemove; + for (auto &[Container, Elems] : Defs) { + assert(!Elems.empty() && "Defs for container must not be empty"); + auto I = Deps.find(Container); + if (I == Deps.end()) + continue; + auto &DepsForContainer = I->second; + for (auto &Elem : Elems) + DepsForContainer.erase(Elem); + if (DepsForContainer.empty()) + ToRemove.push_back(Container); + } + for (auto &Container : ToRemove) + Deps.erase(Container); + if (auto SN = C.addOrCreateSuperNode(std::move(Defs), std::move(Deps))) + SNs.push_back(std::move(SN)); + } + std::vector> takeSuperNodes() { + return std::move(SNs); + } + + private: + Coalescer C; + std::vector> SNs; + }; + + class SimplifyResult { + friend class WaitingOnGraph; + friend class WaitingOnGraphTest; + + public: + const std::vector> &superNodes() const { + return SNs; + } + + private: + SimplifyResult(std::vector> SNs, + ElemToSuperNodeMap ElemToSN) + : SNs(std::move(SNs)), ElemToSN(std::move(ElemToSN)) {} + std::vector> SNs; + ElemToSuperNodeMap ElemToSN; + }; + + /// Preprocess a list of SuperNodes to remove all intra-SN dependencies. + static SimplifyResult simplify(std::vector> SNs) { + // Build ElemToSN map. + ElemToSuperNodeMap ElemToSN; + for (auto &SN : SNs) { + for (auto &[Container, Elements] : SN->Defs) { + auto &ContainerElemToSN = ElemToSN[Container]; + for (auto &E : Elements) + ContainerElemToSN[E] = SN.get(); + } + } + + SuperNodeDepsMap SuperNodeDeps; + hoistDeps(SuperNodeDeps, SNs, ElemToSN); + propagateSuperNodeDeps(SuperNodeDeps); + sinkDeps(SNs, SuperNodeDeps); + + // Pre-coalesce nodes. + Coalescer().coalesce(SNs, ElemToSN); + + return {std::move(SNs), std::move(ElemToSN)}; + } + + struct EmitResult { + std::vector> Ready; + std::vector> Failed; + }; + + enum class ExternalState { None, Ready, Failed }; + + /// Add the given SuperNodes to the graph, returning any SuperNodes that + /// move to the Ready or Failed states as a result. + /// The GetExternalState function is used to represent SuperNodes that have + /// already become Ready or Failed (since such nodes are not explicitly + /// represented in the graph). + template + EmitResult emit(SimplifyResult SR, GetExternalStateFn &&GetExternalState) { + auto NewSNs = std::move(SR.SNs); + auto ElemToNewSN = std::move(SR.ElemToSN); + + // First process any dependencies on nodes with external state. + auto FailedSNs = processExternalDeps(NewSNs, GetExternalState); + + // Collect the PendingSNs whose dep sets are about to be modified. + std::vector> ModifiedPendingSNs; + for (size_t I = 0; I != PendingSNs.size();) { + auto &SN = PendingSNs[I]; + bool Remove = false; + for (auto &[Container, Elems] : SN->Deps) { + auto I = ElemToNewSN.find(Container); + if (I == ElemToNewSN.end()) + continue; + for (auto Elem : Elems) { + if (I->second.contains(Elem)) { + Remove = true; + break; + } + } + if (Remove) + break; + } + if (Remove) { + ModifiedPendingSNs.push_back(std::move(SN)); + std::swap(SN, PendingSNs.back()); + PendingSNs.pop_back(); + } else + ++I; + } + + // Remove cycles from the graphs. + SuperNodeDepsMap SuperNodeDeps; + hoistDeps(SuperNodeDeps, ModifiedPendingSNs, ElemToNewSN); + + CoalesceToPendingSNs.remove( + [&](SuperNode *SN) { return SuperNodeDeps.count(SN); }); + + hoistDeps(SuperNodeDeps, NewSNs, ElemToPendingSN); + propagateSuperNodeDeps(SuperNodeDeps); + sinkDeps(NewSNs, SuperNodeDeps); + sinkDeps(ModifiedPendingSNs, SuperNodeDeps); + + // Process supernodes. Pending first, since we'll update PendingSNs when we + // incorporate NewSNs. + std::vector> ReadyNodes, FailedNodes; + processReadyOrFailed(ModifiedPendingSNs, ReadyNodes, FailedNodes, + SuperNodeDeps, ElemToPendingSN, FailedSNs); + processReadyOrFailed(NewSNs, ReadyNodes, FailedNodes, SuperNodeDeps, + ElemToNewSN, FailedSNs); + + CoalesceToPendingSNs.coalesce(ModifiedPendingSNs, ElemToPendingSN); + CoalesceToPendingSNs.coalesce(NewSNs, ElemToPendingSN); + + // Integrate remaining ModifiedPendingSNs and NewSNs into PendingSNs. + for (auto &SN : ModifiedPendingSNs) + PendingSNs.push_back(std::move(SN)); + + // Update ElemToPendingSN for the remaining elements. + for (auto &SN : NewSNs) { + for (auto &[Container, Elems] : SN->Defs) { + auto &Row = ElemToPendingSN[Container]; + for (auto &Elem : Elems) + Row[Elem] = SN.get(); + } + PendingSNs.push_back(std::move(SN)); + } + + return {std::move(ReadyNodes), std::move(FailedNodes)}; + } + + /// Identify the given symbols as Failed. + /// The elements of the Failed map will not be included in the returned + /// result, so clients should take whatever actions are needed to mark + /// this as failed in their external representation. + std::vector> + fail(const ContainerElementsMap &Failed) { + std::vector> FailedSNs; + + for (size_t I = 0; I != PendingSNs.size();) { + auto &PendingSN = PendingSNs[I]; + bool FailPendingSN = false; + for (auto &[Container, Elems] : PendingSN->Deps) { + if (FailPendingSN) + break; + auto I = Failed.find(Container); + if (I == Failed.end()) + continue; + for (auto &Elem : Elems) { + if (I->second.count(Elem)) { + FailPendingSN = true; + break; + } + } + } + if (FailPendingSN) { + FailedSNs.push_back(std::move(PendingSN)); + PendingSN = std::move(PendingSNs.back()); + PendingSNs.pop_back(); + } else + ++I; + } + + for (auto &SN : FailedSNs) { + CoalesceToPendingSNs.remove( + [&](SuperNode *SNC) { return SNC == SN.get(); }); + for (auto &[Container, Elems] : SN->Defs) { + assert(ElemToPendingSN.count(Container)); + auto &CElems = ElemToPendingSN[Container]; + for (auto &Elem : Elems) + CElems.erase(Elem); + if (CElems.empty()) + ElemToPendingSN.erase(Container); + } + } + + return FailedSNs; + } + + bool validate(raw_ostream &Log) { + bool AllGood = true; + auto ErrLog = [&]() -> raw_ostream & { + AllGood = false; + return Log; + }; + + size_t DefCount = 0; + for (auto &PendingSN : PendingSNs) { + if (PendingSN->Deps.empty()) + ErrLog() << "Pending SN " << PendingSN.get() << " has empty dep set.\n"; + else { + bool BadElem = false; + for (auto &[Container, Elems] : PendingSN->Deps) { + auto I = ElemToPendingSN.find(Container); + if (I == ElemToPendingSN.end()) + continue; + if (Elems.empty()) + ErrLog() << "Pending SN " << PendingSN.get() + << " has dependence map entry for " << Container + << " with empty element set.\n"; + for (auto &Elem : Elems) { + if (I->second.count(Elem)) { + ErrLog() << "Pending SN " << PendingSN.get() + << " has dependence on emitted element ( " << Container + << ", " << Elem << ")\n"; + BadElem = true; + break; + } + } + if (BadElem) + break; + } + } + + for (auto &[Container, Elems] : PendingSN->Defs) { + if (Elems.empty()) + ErrLog() << "Pending SN " << PendingSN.get() + << " has def map entry for " << Container + << " with empty element set.\n"; + DefCount += Elems.size(); + auto I = ElemToPendingSN.find(Container); + if (I == ElemToPendingSN.end()) + ErrLog() << "Pending SN " << PendingSN.get() << " has " + << Elems.size() << " defs in container " << Container + << " not covered by ElemsToPendingSN.\n"; + else { + for (auto &Elem : Elems) { + auto J = I->second.find(Elem); + if (J == I->second.end()) + ErrLog() << "Pending SN " << PendingSN.get() << " has element (" + << Container << ", " << Elem + << ") not covered by ElemsToPendingSN.\n"; + else if (J->second != PendingSN.get()) + ErrLog() << "ElemToPendingSN value invalid for (" << Container + << ", " << Elem << ")\n"; + } + } + } + } + + size_t DefCount2 = 0; + for (auto &[Container, Elems] : ElemToPendingSN) + DefCount2 += Elems.size(); + + assert(DefCount2 >= DefCount); + if (DefCount2 != DefCount) + ErrLog() << "ElemToPendingSN contains extra elements.\n"; + + return AllGood; + } + +private: + // Replace individual dependencies with supernode dependencies. + // + // For all dependencies in SNs, if the corresponding node is defined in + // ElemToSN then remove the individual dependency and add the record the + // dependency on the corresponding supernode in SuperNodeDeps. + static void hoistDeps(SuperNodeDepsMap &SuperNodeDeps, + std::vector> &SNs, + ElemToSuperNodeMap &ElemToSN) { + for (auto &SN : SNs) { + auto &SNDeps = SuperNodeDeps[SN.get()]; + for (auto &[DefContainer, DefElems] : ElemToSN) { + auto I = SN->Deps.find(DefContainer); + if (I == SN->Deps.end()) + continue; + for (auto &[DefElem, DefSN] : DefElems) + if (I->second.erase(DefElem)) + SNDeps.insert(DefSN); + if (I->second.empty()) + SN->Deps.erase(I); + } + } + } + + // Compute transitive closure of deps for each node. + static void propagateSuperNodeDeps(SuperNodeDepsMap &SuperNodeDeps) { + for (auto &[SN, Deps] : SuperNodeDeps) { + DenseSet Reachable({SN}); + SmallVector Worklist(Deps.begin(), Deps.end()); + + while (!Worklist.empty()) { + auto *DepSN = Worklist.pop_back_val(); + if (!Reachable.insert(DepSN).second) + continue; + auto I = SuperNodeDeps.find(DepSN); + if (I == SuperNodeDeps.end()) + continue; + for (auto *DepSNDep : I->second) + Worklist.push_back(DepSNDep); + } + + Deps = std::move(Reachable); + } + } + + // Sink SuperNode dependencies back to dependencies on individual nodes. + static void sinkDeps(std::vector> &SNs, + SuperNodeDepsMap &SuperNodeDeps) { + for (auto &SN : SNs) { + auto I = SuperNodeDeps.find(SN.get()); + if (I == SuperNodeDeps.end()) + continue; + + for (auto *DepSN : I->second) + for (auto &[Container, Elems] : DepSN->Deps) + SN->Deps[Container].insert(Elems.begin(), Elems.end()); + } + } + + template + static std::vector + processExternalDeps(std::vector> &SNs, + GetExternalStateFn &GetExternalState) { + std::vector FailedSNs; + for (auto &SN : SNs) { + bool SNHasError = false; + SmallVector ContainersToRemove; + for (auto &[Container, Elems] : SN->Deps) { + SmallVector ElemToRemove; + for (auto &Elem : Elems) { + switch (GetExternalState(Container, Elem)) { + case ExternalState::None: + break; + case ExternalState::Ready: + ElemToRemove.push_back(Elem); + break; + case ExternalState::Failed: + ElemToRemove.push_back(Elem); + SNHasError = true; + break; + } + } + for (auto &Elem : ElemToRemove) + Elems.erase(Elem); + if (Elems.empty()) + ContainersToRemove.push_back(Container); + } + for (auto &Container : ContainersToRemove) + SN->Deps.erase(Container); + if (SNHasError) + FailedSNs.push_back(SN.get()); + } + + return FailedSNs; + } + + void processReadyOrFailed(std::vector> &SNs, + std::vector> &Ready, + std::vector> &Failed, + SuperNodeDepsMap &SuperNodeDeps, + ElemToSuperNodeMap &ElemToSNs, + std::vector FailedSNs) { + for (size_t I = 0; I != SNs.size();) { + auto &SN = SNs[I]; + + bool SNFailed = false; + assert(SuperNodeDeps.count(SN.get())); + auto &SNSuperNodeDeps = SuperNodeDeps[SN.get()]; + for (auto *FailedSN : FailedSNs) { + if (FailedSN == SN.get() || SNSuperNodeDeps.count(FailedSN)) { + SNFailed = true; + break; + } + } + + bool SNReady = SN->Deps.empty(); + + if (SNReady || SNFailed) { + auto &NodeList = SNFailed ? Failed : Ready; + NodeList.push_back(std::move(SN)); + std::swap(SN, SNs.back()); + SNs.pop_back(); + } else + ++I; + } + } + + std::vector> PendingSNs; + ElemToSuperNodeMap ElemToPendingSN; + Coalescer CoalesceToPendingSNs; +}; + +} // namespace llvm::orc::detail + +#endif // LLVM_EXECUTIONENGINE_ORC_WAITINGONGRAPH_H diff --git a/llvm/lib/ExecutionEngine/Orc/Core.cpp b/llvm/lib/ExecutionEngine/Orc/Core.cpp index f47b7ecdcc7bb..0354d9ab57880 100644 --- a/llvm/lib/ExecutionEngine/Orc/Core.cpp +++ b/llvm/lib/ExecutionEngine/Orc/Core.cpp @@ -1173,39 +1173,7 @@ void JITDylib::dump(raw_ostream &OS) { << " pending queries: { "; for (const auto &Q : KV.second.pendingQueries()) OS << Q.get() << " (" << Q->getRequiredState() << ") "; - OS << "}\n Defining EDU: "; - if (KV.second.DefiningEDU) { - OS << KV.second.DefiningEDU.get() << " { "; - for (auto &[Name, Flags] : KV.second.DefiningEDU->Symbols) - OS << Name << " "; - OS << "}\n"; - OS << " Dependencies:\n"; - if (!KV.second.DefiningEDU->Dependencies.empty()) { - for (auto &[DepJD, Deps] : KV.second.DefiningEDU->Dependencies) { - OS << " " << DepJD->getName() << ": [ "; - for (auto &Dep : Deps) - OS << Dep << " "; - OS << "]\n"; - } - } else - OS << " none\n"; - } else - OS << "none\n"; - OS << " Dependant EDUs:\n"; - if (!KV.second.DependantEDUs.empty()) { - for (auto &DependantEDU : KV.second.DependantEDUs) { - OS << " " << DependantEDU << ": " - << DependantEDU->JD->getName() << " { "; - for (auto &[Name, Flags] : DependantEDU->Symbols) - OS << Name << " "; - OS << "}\n"; - } - } else - OS << " none\n"; - assert((Symbols[KV.first].getState() != SymbolState::Ready || - (KV.second.pendingQueries().empty() && !KV.second.DefiningEDU && - !KV.second.DependantEDUs.empty())) && - "Stale materializing info entry"); + OS << "}\n"; } }); } @@ -2917,359 +2885,64 @@ Error ExecutionSession::OL_notifyResolved(MaterializationResponsibility &MR, return MR.JD.resolve(MR, Symbols); } -template -void ExecutionSession::propagateExtraEmitDeps( - std::deque Worklist, EDUInfosMap &EDUInfos, - HandleNewDepFn HandleNewDep) { - - // Iterate to a fixed-point to propagate extra-emit dependencies through the - // EDU graph. - while (!Worklist.empty()) { - auto &EDU = *Worklist.front(); - Worklist.pop_front(); - - assert(EDUInfos.count(&EDU) && "No info entry for EDU"); - auto &EDUInfo = EDUInfos[&EDU]; - - // Propagate new dependencies to users. - for (auto *UserEDU : EDUInfo.IntraEmitUsers) { - - // UserEDUInfo only present if UserEDU has its own users. - JITDylib::EmissionDepUnitInfo *UserEDUInfo = nullptr; - { - auto UserEDUInfoItr = EDUInfos.find(UserEDU); - if (UserEDUInfoItr != EDUInfos.end()) - UserEDUInfo = &UserEDUInfoItr->second; - } - - for (auto &[DepJD, Deps] : EDUInfo.NewDeps) { - auto &UserEDUDepsForJD = UserEDU->Dependencies[DepJD]; - DenseSet *UserEDUNewDepsForJD = nullptr; - for (auto Dep : Deps) { - if (UserEDUDepsForJD.insert(Dep).second) { - HandleNewDep(*UserEDU, *DepJD, Dep); - if (UserEDUInfo) { - if (!UserEDUNewDepsForJD) { - // If UserEDU has no new deps then it's not in the worklist - // yet, so add it. - if (UserEDUInfo->NewDeps.empty()) - Worklist.push_back(UserEDU); - UserEDUNewDepsForJD = &UserEDUInfo->NewDeps[DepJD]; - } - // Add (DepJD, Dep) to NewDeps. - UserEDUNewDepsForJD->insert(Dep); - } - } - } - } - } - - EDUInfo.NewDeps.clear(); - } -} - -// Note: This method modifies the emitted set. -ExecutionSession::EDUInfosMap ExecutionSession::simplifyDepGroups( - MaterializationResponsibility &MR, - ArrayRef EmittedDeps) { - - auto &TargetJD = MR.getTargetJITDylib(); - - // 1. Build initial EmissionDepUnit -> EmissionDepUnitInfo and - // Symbol -> EmissionDepUnit mappings. - DenseMap EDUInfos; - EDUInfos.reserve(EmittedDeps.size()); - DenseMap EDUForSymbol; - for (auto &DG : EmittedDeps) { - assert(!DG.Symbols.empty() && "DepGroup does not cover any symbols"); - - // Skip empty EDUs. - if (DG.Dependencies.empty()) - continue; - - auto TmpEDU = std::make_shared(TargetJD); - auto &EDUInfo = EDUInfos[TmpEDU.get()]; - EDUInfo.EDU = std::move(TmpEDU); - for (const auto &Symbol : DG.Symbols) { - NonOwningSymbolStringPtr NonOwningSymbol(Symbol); - assert(!EDUForSymbol.count(NonOwningSymbol) && - "Symbol should not appear in more than one SymbolDependenceGroup"); - assert(MR.getSymbols().count(Symbol) && - "Symbol in DepGroups not in the emitted set"); - auto NewlyEmittedItr = MR.getSymbols().find(Symbol); - EDUInfo.EDU->Symbols[NonOwningSymbol] = NewlyEmittedItr->second; - EDUForSymbol[NonOwningSymbol] = EDUInfo.EDU.get(); - } - } - - // 2. Build a "residual" EDU to cover all symbols that have no dependencies. - { - DenseMap ResidualSymbolFlags; - for (auto &[Sym, Flags] : MR.getSymbols()) { - if (!EDUForSymbol.count(NonOwningSymbolStringPtr(Sym))) - ResidualSymbolFlags[NonOwningSymbolStringPtr(Sym)] = Flags; - } - if (!ResidualSymbolFlags.empty()) { - auto ResidualEDU = std::make_shared(TargetJD); - ResidualEDU->Symbols = std::move(ResidualSymbolFlags); - auto &ResidualEDUInfo = EDUInfos[ResidualEDU.get()]; - ResidualEDUInfo.EDU = std::move(ResidualEDU); - - // If the residual EDU is the only one then bail out early. - if (EDUInfos.size() == 1) - return EDUInfos; - - // Otherwise add the residual EDU to the EDUForSymbol map. - for (auto &[Sym, Flags] : ResidualEDUInfo.EDU->Symbols) - EDUForSymbol[Sym] = ResidualEDUInfo.EDU.get(); - } - } - -#ifndef NDEBUG - assert(EDUForSymbol.size() == MR.getSymbols().size() && - "MR symbols not fully covered by EDUs?"); - for (auto &[Sym, Flags] : MR.getSymbols()) { - assert(EDUForSymbol.count(NonOwningSymbolStringPtr(Sym)) && - "Sym in MR not covered by EDU"); - } -#endif // NDEBUG - - // 3. Use the DepGroups array to build a graph of dependencies between - // EmissionDepUnits in this finalization. We want to remove these - // intra-finalization uses, propagating dependencies on symbols outside - // this finalization. Add EDUs to the worklist. - for (auto &DG : EmittedDeps) { - - // Skip SymbolDependenceGroups with no dependencies. - if (DG.Dependencies.empty()) - continue; - - assert(EDUForSymbol.count(NonOwningSymbolStringPtr(*DG.Symbols.begin())) && - "No EDU for DG"); - auto &EDU = - *EDUForSymbol.find(NonOwningSymbolStringPtr(*DG.Symbols.begin())) - ->second; - - for (auto &[DepJD, Deps] : DG.Dependencies) { - DenseSet NewDepsForJD; - - assert(!Deps.empty() && "Dependence set for DepJD is empty"); - - if (DepJD != &TargetJD) { - // DepJD is some other JITDylib.There can't be any intra-finalization - // edges here, so just skip. - for (auto &Dep : Deps) - NewDepsForJD.insert(NonOwningSymbolStringPtr(Dep)); - } else { - // DepJD is the Target JITDylib. Check for intra-finaliztaion edges, - // skipping any and recording the intra-finalization use instead. - for (auto &Dep : Deps) { - NonOwningSymbolStringPtr NonOwningDep(Dep); - auto I = EDUForSymbol.find(NonOwningDep); - if (I == EDUForSymbol.end()) { - if (!MR.getSymbols().count(Dep)) - NewDepsForJD.insert(NonOwningDep); - continue; - } - - if (I->second != &EDU) - EDUInfos[I->second].IntraEmitUsers.insert(&EDU); +WaitingOnGraph::ExternalState +ExecutionSession::IL_getSymbolState(JITDylib *JD, + NonOwningSymbolStringPtr Name) { + if (JD->State != JITDylib::Open) + return WaitingOnGraph::ExternalState::Failed; + + auto I = JD->Symbols.find_as(Name); + + // FIXME: Can we eliminate this possibility if we support query binding? + if (I == JD->Symbols.end()) + return WaitingOnGraph::ExternalState::Failed; + + if (I->second.getFlags().hasError()) + return WaitingOnGraph::ExternalState::Failed; + + if (I->second.getState() == SymbolState::Ready) + return WaitingOnGraph::ExternalState::Ready; + + return WaitingOnGraph::ExternalState::None; +} + +template +void ExecutionSession::IL_collectQueries( + JITDylib::AsynchronousSymbolQuerySet &Qs, + WaitingOnGraph::ContainerElementsMap &QualifiedSymbols, + UpdateSymbolFn &&UpdateSymbol, UpdateQueryFn &&UpdateQuery) { + + for (auto &[JD, Symbols] : QualifiedSymbols) { + // IL_emit and JITDylib removal are synchronized by the session lock. + // Since JITDylib removal removes any contained nodes from the + // WaitingOnGraph, we should be able to assert that all nodes in the + // WaitingOnGraph have not been removed. + assert(JD->State == JITDylib::Open && + "WaitingOnGraph includes definition in defunct JITDylib"); + for (auto &Symbol : Symbols) { + // Update symbol table. + auto I = JD->Symbols.find_as(Symbol); + assert(I != JD->Symbols.end() && + "Failed Symbol missing from JD symbol table"); + auto &Entry = I->second; + UpdateSymbol(Entry); + + // Collect queries. + auto J = JD->MaterializingInfos.find_as(Symbol); + if (J != JD->MaterializingInfos.end()) { + for (auto &Q : J->second.takeAllPendingQueries()) { + UpdateQuery(*Q, *JD, Symbol, Entry); + Qs.insert(std::move(Q)); } + JD->MaterializingInfos.erase(J); } - - if (!NewDepsForJD.empty()) - EDU.Dependencies[DepJD] = std::move(NewDepsForJD); } } - - // 4. Build the worklist. - std::deque Worklist; - for (auto &[EDU, EDUInfo] : EDUInfos) { - // If this EDU has extra-finalization dependencies and intra-finalization - // users then add it to the worklist. - if (!EDU->Dependencies.empty()) { - auto I = EDUInfos.find(EDU); - if (I != EDUInfos.end()) { - auto &EDUInfo = I->second; - if (!EDUInfo.IntraEmitUsers.empty()) { - EDUInfo.NewDeps = EDU->Dependencies; - Worklist.push_back(EDU); - } - } - } - } - - // 4. Propagate dependencies through the EDU graph. - propagateExtraEmitDeps( - Worklist, EDUInfos, - [](JITDylib::EmissionDepUnit &, JITDylib &, NonOwningSymbolStringPtr) {}); - - return EDUInfos; } -void ExecutionSession::IL_makeEDUReady( - std::shared_ptr EDU, - JITDylib::AsynchronousSymbolQuerySet &Queries) { - - // The symbols for this EDU are ready. - auto &JD = *EDU->JD; - - for (auto &[Sym, Flags] : EDU->Symbols) { - assert(JD.Symbols.count(SymbolStringPtr(Sym)) && - "JD does not have an entry for Sym"); - auto &Entry = JD.Symbols[SymbolStringPtr(Sym)]; - - assert(((Entry.getFlags().hasMaterializationSideEffectsOnly() && - Entry.getState() == SymbolState::Materializing) || - Entry.getState() == SymbolState::Resolved || - Entry.getState() == SymbolState::Emitted) && - "Emitting from state other than Resolved"); - - Entry.setState(SymbolState::Ready); - - auto MII = JD.MaterializingInfos.find(SymbolStringPtr(Sym)); - - // Check for pending queries. - if (MII == JD.MaterializingInfos.end()) - continue; - auto &MI = MII->second; - - for (auto &Q : MI.takeQueriesMeeting(SymbolState::Ready)) { - Q->notifySymbolMetRequiredState(SymbolStringPtr(Sym), Entry.getSymbol()); - if (Q->isComplete()) - Queries.insert(Q); - Q->removeQueryDependence(JD, SymbolStringPtr(Sym)); - } - - JD.MaterializingInfos.erase(MII); - } - - JD.shrinkMaterializationInfoMemory(); -} - -void ExecutionSession::IL_makeEDUEmitted( - std::shared_ptr EDU, - JITDylib::AsynchronousSymbolQuerySet &Queries) { - - // The symbols for this EDU are emitted, but not ready. - auto &JD = *EDU->JD; - - for (auto &[Sym, Flags] : EDU->Symbols) { - assert(JD.Symbols.count(SymbolStringPtr(Sym)) && - "JD does not have an entry for Sym"); - auto &Entry = JD.Symbols[SymbolStringPtr(Sym)]; - - assert(((Entry.getFlags().hasMaterializationSideEffectsOnly() && - Entry.getState() == SymbolState::Materializing) || - Entry.getState() == SymbolState::Resolved || - Entry.getState() == SymbolState::Emitted) && - "Emitting from state other than Resolved"); - - if (Entry.getState() == SymbolState::Emitted) { - // This was already emitted, so we can skip the rest of this loop. -#ifndef NDEBUG - for (auto &[Sym, Flags] : EDU->Symbols) { - assert(JD.Symbols.count(SymbolStringPtr(Sym)) && - "JD does not have an entry for Sym"); - auto &Entry = JD.Symbols[SymbolStringPtr(Sym)]; - assert(Entry.getState() == SymbolState::Emitted && - "Symbols for EDU in inconsistent state"); - assert(JD.MaterializingInfos.count(SymbolStringPtr(Sym)) && - "Emitted symbol has no MI"); - auto MI = JD.MaterializingInfos[SymbolStringPtr(Sym)]; - assert(MI.takeQueriesMeeting(SymbolState::Emitted).empty() && - "Already-emitted symbol has waiting-on-emitted queries"); - } -#endif // NDEBUG - break; - } - - Entry.setState(SymbolState::Emitted); - auto &MI = JD.MaterializingInfos[SymbolStringPtr(Sym)]; - MI.DefiningEDU = EDU; - - for (auto &Q : MI.takeQueriesMeeting(SymbolState::Emitted)) { - Q->notifySymbolMetRequiredState(SymbolStringPtr(Sym), Entry.getSymbol()); - if (Q->isComplete()) - Queries.insert(Q); - } - } - - for (auto &[DepJD, Deps] : EDU->Dependencies) { - for (auto &Dep : Deps) - DepJD->MaterializingInfos[SymbolStringPtr(Dep)].DependantEDUs.insert( - EDU.get()); - } -} - -/// Removes the given dependence from EDU. If EDU's dependence set becomes -/// empty then this function adds an entry for it to the EDUInfos map. -/// Returns true if a new EDUInfosMap entry is added. -bool ExecutionSession::IL_removeEDUDependence(JITDylib::EmissionDepUnit &EDU, - JITDylib &DepJD, - NonOwningSymbolStringPtr DepSym, - EDUInfosMap &EDUInfos) { - assert(EDU.Dependencies.count(&DepJD) && - "JD does not appear in Dependencies of DependantEDU"); - assert(EDU.Dependencies[&DepJD].count(DepSym) && - "Symbol does not appear in Dependencies of DependantEDU"); - auto &JDDeps = EDU.Dependencies[&DepJD]; - JDDeps.erase(DepSym); - if (JDDeps.empty()) { - EDU.Dependencies.erase(&DepJD); - if (EDU.Dependencies.empty()) { - // If the dependencies set has become empty then EDU _may_ be ready - // (we won't know for sure until we've propagated the extra-emit deps). - // Create an EDUInfo for it (if it doesn't have one already) so that - // it'll be visited after propagation. - auto &DepEDUInfo = EDUInfos[&EDU]; - if (!DepEDUInfo.EDU) { - assert(EDU.JD->Symbols.count( - SymbolStringPtr(EDU.Symbols.begin()->first)) && - "Missing symbol entry for first symbol in EDU"); - auto DepEDUFirstMI = EDU.JD->MaterializingInfos.find( - SymbolStringPtr(EDU.Symbols.begin()->first)); - assert(DepEDUFirstMI != EDU.JD->MaterializingInfos.end() && - "Missing MI for first symbol in DependantEDU"); - DepEDUInfo.EDU = DepEDUFirstMI->second.DefiningEDU; - return true; - } - } - } - return false; -} - -Error ExecutionSession::makeJDClosedError(JITDylib::EmissionDepUnit &EDU, - JITDylib &ClosedJD) { - SymbolNameSet FailedSymbols; - for (auto &[Sym, Flags] : EDU.Symbols) - FailedSymbols.insert(SymbolStringPtr(Sym)); - SymbolDependenceMap BadDeps; - for (auto &Dep : EDU.Dependencies[&ClosedJD]) - BadDeps[&ClosedJD].insert(SymbolStringPtr(Dep)); - return make_error( - ClosedJD.getExecutionSession().getSymbolStringPool(), EDU.JD, - std::move(FailedSymbols), std::move(BadDeps), - ClosedJD.getName() + " is closed"); -} - -Error ExecutionSession::makeUnsatisfiedDepsError(JITDylib::EmissionDepUnit &EDU, - JITDylib &BadJD, - SymbolNameSet BadDeps) { - SymbolNameSet FailedSymbols; - for (auto &[Sym, Flags] : EDU.Symbols) - FailedSymbols.insert(SymbolStringPtr(Sym)); - SymbolDependenceMap BadDepsMap; - BadDepsMap[&BadJD] = std::move(BadDeps); - return make_error( - BadJD.getExecutionSession().getSymbolStringPool(), &BadJD, - std::move(FailedSymbols), std::move(BadDepsMap), - "dependencies removed or in error state"); -} - -Expected +Expected ExecutionSession::IL_emit(MaterializationResponsibility &MR, - EDUInfosMap EDUInfos) { + WaitingOnGraph::SimplifyResult SR) { if (MR.RT->isDefunct()) return make_error(MR.RT); @@ -3279,169 +2952,50 @@ ExecutionSession::IL_emit(MaterializationResponsibility &MR, return make_error("JITDylib " + TargetJD.getName() + " is defunct", inconvertibleErrorCode()); + #ifdef EXPENSIVE_CHECKS verifySessionState("entering ExecutionSession::IL_emit"); #endif - // Walk all EDUs: - // 1. Verifying that dependencies are available (not removed or in the error - // state. - // 2. Removing any dependencies that are already Ready. - // 3. Lifting any EDUs for Emitted symbols into the EDUInfos map. - // 4. Finding any dependant EDUs and lifting them into the EDUInfos map. - std::deque Worklist; - for (auto &[EDU, _] : EDUInfos) - Worklist.push_back(EDU); - - for (auto *EDU : Worklist) { - auto *EDUInfo = &EDUInfos[EDU]; - - SmallVector DepJDsToRemove; - for (auto &[DepJD, Deps] : EDU->Dependencies) { - if (DepJD->State != JITDylib::Open) - return makeJDClosedError(*EDU, *DepJD); - - SymbolNameSet BadDeps; - SmallVector DepsToRemove; - for (auto &Dep : Deps) { - auto DepEntryItr = DepJD->Symbols.find(SymbolStringPtr(Dep)); - - // If this dep has been removed or moved to the error state then add it - // to the bad deps set. We aggregate these bad deps for more - // comprehensive error messages. - if (DepEntryItr == DepJD->Symbols.end() || - DepEntryItr->second.getFlags().hasError()) { - BadDeps.insert(SymbolStringPtr(Dep)); - continue; - } - - // If this dep isn't emitted yet then just add it to the NewDeps set to - // be propagated. - auto &DepEntry = DepEntryItr->second; - if (DepEntry.getState() < SymbolState::Emitted) { - EDUInfo->NewDeps[DepJD].insert(Dep); - continue; - } - - // This dep has been emitted, so add it to the list to be removed from - // EDU. - DepsToRemove.push_back(Dep); - - // If Dep is Ready then there's nothing further to do. - if (DepEntry.getState() == SymbolState::Ready) { - assert(!DepJD->MaterializingInfos.count(SymbolStringPtr(Dep)) && - "Unexpected MaterializationInfo attached to ready symbol"); - continue; - } + auto ER = G.emit(std::move(SR), + [this](JITDylib *JD, NonOwningSymbolStringPtr Name) { + return IL_getSymbolState(JD, Name); + }); - // If we get here then Dep is Emitted. We need to look up its defining - // EDU and add this EDU to the defining EDU's list of users (this means - // creating an EDUInfos entry if the defining EDU doesn't have one - // already). - assert(DepJD->MaterializingInfos.count(SymbolStringPtr(Dep)) && - "Expected MaterializationInfo for emitted dependency"); - auto &DepMI = DepJD->MaterializingInfos[SymbolStringPtr(Dep)]; - assert(DepMI.DefiningEDU && - "Emitted symbol does not have a defining EDU"); - assert(DepMI.DependantEDUs.empty() && - "Already-emitted symbol has dependant EDUs?"); - auto &DepEDUInfo = EDUInfos[DepMI.DefiningEDU.get()]; - if (!DepEDUInfo.EDU) { - // No EDUInfo yet -- build initial entry, and reset the EDUInfo - // pointer, which we will have invalidated. - EDUInfo = &EDUInfos[EDU]; - DepEDUInfo.EDU = DepMI.DefiningEDU; - for (auto &[DepDepJD, DepDeps] : DepEDUInfo.EDU->Dependencies) { - if (DepDepJD == &TargetJD) { - for (auto &DepDep : DepDeps) - if (!MR.getSymbols().count(SymbolStringPtr(DepDep))) - DepEDUInfo.NewDeps[DepDepJD].insert(DepDep); - } else - DepEDUInfo.NewDeps[DepDepJD] = DepDeps; - } - } - DepEDUInfo.IntraEmitUsers.insert(EDU); - } + EmitQueries EQ; - // Some dependencies were removed or in an error state -- error out. - if (!BadDeps.empty()) - return makeUnsatisfiedDepsError(*EDU, *DepJD, std::move(BadDeps)); - - // Remove the emitted / ready deps from DepJD. - for (auto &Dep : DepsToRemove) - Deps.erase(Dep); - - // If there are no further deps in DepJD then flag it for removal too. - if (Deps.empty()) - DepJDsToRemove.push_back(DepJD); - } - - // Remove any JDs whose dependence sets have become empty. - for (auto &DepJD : DepJDsToRemove) { - assert(EDU->Dependencies.count(DepJD) && - "Trying to remove non-existent dep entries"); - EDU->Dependencies.erase(DepJD); - } - - // Now look for users of this EDU. - for (auto &[Sym, Flags] : EDU->Symbols) { - assert(TargetJD.Symbols.count(SymbolStringPtr(Sym)) && - "Sym not present in symbol table"); - assert((TargetJD.Symbols[SymbolStringPtr(Sym)].getState() == - SymbolState::Resolved || - TargetJD.Symbols[SymbolStringPtr(Sym)] - .getFlags() - .hasMaterializationSideEffectsOnly()) && - "Emitting symbol not in the resolved state"); - assert(!TargetJD.Symbols[SymbolStringPtr(Sym)].getFlags().hasError() && - "Symbol is already in an error state"); - - auto MII = TargetJD.MaterializingInfos.find(SymbolStringPtr(Sym)); - if (MII == TargetJD.MaterializingInfos.end() || - MII->second.DependantEDUs.empty()) - continue; - - for (auto &DependantEDU : MII->second.DependantEDUs) { - if (IL_removeEDUDependence(*DependantEDU, TargetJD, Sym, EDUInfos)) - EDUInfo = &EDUInfos[EDU]; - EDUInfo->IntraEmitUsers.insert(DependantEDU); - } - MII->second.DependantEDUs.clear(); - } - } - - Worklist.clear(); - for (auto &[EDU, EDUInfo] : EDUInfos) { - if (!EDUInfo.IntraEmitUsers.empty() && !EDU->Dependencies.empty()) { - if (EDUInfo.NewDeps.empty()) - EDUInfo.NewDeps = EDU->Dependencies; - Worklist.push_back(EDU); - } - } - - propagateExtraEmitDeps( - Worklist, EDUInfos, - [](JITDylib::EmissionDepUnit &EDU, JITDylib &JD, - NonOwningSymbolStringPtr Sym) { - JD.MaterializingInfos[SymbolStringPtr(Sym)].DependantEDUs.insert(&EDU); - }); + // Handle failed queries. + for (auto &SN : ER.Failed) + IL_collectQueries( + EQ.Failed, SN->defs(), + [](JITDylib::SymbolTableEntry &E) { + E.setFlags(E.getFlags() = JITSymbolFlags::HasError); + }, + [&](AsynchronousSymbolQuery &Q, JITDylib &JD, + NonOwningSymbolStringPtr Name, JITDylib::SymbolTableEntry &E) { + auto &FS = EQ.FailedSymsForQuery[&Q]; + if (!FS) + FS = std::make_shared(); + (*FS)[&JD].insert(SymbolStringPtr(Name)); + }); - JITDylib::AsynchronousSymbolQuerySet CompletedQueries; + for (auto &FQ : EQ.Failed) + FQ->detach(); - // Extract completed queries and lodge not-yet-ready EDUs in the - // session. - for (auto &[EDU, EDUInfo] : EDUInfos) { - if (EDU->Dependencies.empty()) - IL_makeEDUReady(std::move(EDUInfo.EDU), CompletedQueries); - else - IL_makeEDUEmitted(std::move(EDUInfo.EDU), CompletedQueries); - } + for (auto &SN : ER.Ready) + IL_collectQueries( + EQ.Updated, SN->defs(), + [](JITDylib::SymbolTableEntry &E) { E.setState(SymbolState::Ready); }, + [](AsynchronousSymbolQuery &Q, JITDylib &JD, + NonOwningSymbolStringPtr Name, JITDylib::SymbolTableEntry &E) { + Q.notifySymbolMetRequiredState(SymbolStringPtr(Name), E.getSymbol()); + }); #ifdef EXPENSIVE_CHECKS verifySessionState("exiting ExecutionSession::IL_emit"); #endif - return std::move(CompletedQueries); + return std::move(EQ); } Error ExecutionSession::OL_notifyEmitted( @@ -3471,40 +3025,107 @@ Error ExecutionSession::OL_notifyEmitted( } #endif // NDEBUG - auto EDUInfos = simplifyDepGroups(MR, DepGroups); + std::vector> SNs; + WaitingOnGraph::ContainerElementsMap Residual; + { + auto &JDResidual = Residual[&MR.getTargetJITDylib()]; + for (auto &[Name, Flags] : MR.getSymbols()) + JDResidual.insert(NonOwningSymbolStringPtr(Name)); + + for (auto &SDG : DepGroups) { + WaitingOnGraph::ContainerElementsMap Defs; + assert(!SDG.Symbols.empty()); + auto &JDDefs = Defs[&MR.getTargetJITDylib()]; + for (auto &Def : SDG.Symbols) { + JDDefs.insert(NonOwningSymbolStringPtr(Def)); + JDResidual.erase(NonOwningSymbolStringPtr(Def)); + } + WaitingOnGraph::ContainerElementsMap Deps; + if (!SDG.Dependencies.empty()) { + for (auto &[JD, Syms] : SDG.Dependencies) { + auto &JDDeps = Deps[JD]; + for (auto &Dep : Syms) + JDDeps.insert(NonOwningSymbolStringPtr(Dep)); + } + } + SNs.push_back(std::make_unique( + std::move(Defs), std::move(Deps))); + } + if (!JDResidual.empty()) + SNs.push_back(std::make_unique( + std::move(Residual), WaitingOnGraph::ContainerElementsMap())); + } + + auto SR = WaitingOnGraph::simplify(std::move(SNs)); LLVM_DEBUG({ dbgs() << " Simplified dependencies:\n"; - for (auto &[EDU, EDUInfo] : EDUInfos) { - dbgs() << " Symbols: { "; - for (auto &[Sym, Flags] : EDU->Symbols) - dbgs() << Sym << " "; - dbgs() << "}, Dependencies: { "; - for (auto &[DepJD, Deps] : EDU->Dependencies) { - dbgs() << "(" << DepJD->getName() << ", { "; - for (auto &Dep : Deps) - dbgs() << Dep << " "; - dbgs() << "}) "; + for (auto &SN : SR.superNodes()) { + dbgs() << " Defs: {"; + for (auto &[JD, Syms] : SN->defs()) { + dbgs() << " (" << JD->getName() << ", ["; + for (auto &Sym : Syms) + dbgs() << " " << Sym; + dbgs() << " ])"; + } + dbgs() << " }, Deps: {"; + for (auto &[JD, Syms] : SN->deps()) { + dbgs() << " (" << JD->getName() << ", ["; + for (auto &Sym : Syms) + dbgs() << " " << Sym; + dbgs() << " ])"; } - dbgs() << "}\n"; + dbgs() << " }\n"; } }); - - auto CompletedQueries = - runSessionLocked([&]() { return IL_emit(MR, EDUInfos); }); + auto EmitQueries = + runSessionLocked([&]() { return IL_emit(MR, std::move(SR)); }); // On error bail out. - if (!CompletedQueries) - return CompletedQueries.takeError(); + if (!EmitQueries) + return EmitQueries.takeError(); - MR.SymbolFlags.clear(); + // Otherwise notify failed queries, and any updated queries that have been + // completed. - // Otherwise notify all the completed queries. - for (auto &Q : *CompletedQueries) { - assert(Q->isComplete() && "Q is not complete"); - Q->handleComplete(*this); + // FIXME: Get rid of error return from notifyEmitted. + SymbolDependenceMap BadDeps; + { + for (auto &FQ : EmitQueries->Failed) { + FQ->detach(); + assert(EmitQueries->FailedSymsForQuery.count(FQ.get()) && + "Missing failed symbols for query"); + auto FailedSyms = std::move(EmitQueries->FailedSymsForQuery[FQ.get()]); + for (auto &[JD, Syms] : *FailedSyms) { + auto &BadDepsForJD = BadDeps[JD]; + for (auto &Sym : Syms) + BadDepsForJD.insert(Sym); + } + FQ->handleFailed(make_error(getSymbolStringPool(), + std::move(FailedSyms))); + } + } + + for (auto &UQ : EmitQueries->Updated) + if (UQ->isComplete()) + UQ->handleComplete(*this); + + // If there are any bad dependencies then return an error. + if (!BadDeps.empty()) { + SymbolNameSet BadNames; + // Note: The name set calculated here is bogus: it includes all symbols in + // the MR, not just the ones that failed. We want to remove the error + // return path from notifyEmitted anyway, so this is just a brief + // placeholder to maintain (roughly) the current error behavior. + for (auto &[Name, Flags] : MR.getSymbols()) + BadNames.insert(Name); + MR.SymbolFlags.clear(); + return make_error( + getSymbolStringPool(), &MR.getTargetJITDylib(), std::move(BadNames), + std::move(BadDeps), "dependencies removed or in error state"); } + MR.SymbolFlags.clear(); return Error::success(); } @@ -3535,158 +3156,48 @@ ExecutionSession::IL_failSymbols(JITDylib &JD, #endif JITDylib::AsynchronousSymbolQuerySet FailedQueries; - auto FailedSymbolsMap = std::make_shared(); - auto ExtractFailedQueries = [&](JITDylib::MaterializingInfo &MI) { - JITDylib::AsynchronousSymbolQueryList ToDetach; - for (auto &Q : MI.pendingQueries()) { - // Add the query to the list to be failed and detach it. - FailedQueries.insert(Q); - ToDetach.push_back(Q); + auto Fail = [&](JITDylib *FailJD, NonOwningSymbolStringPtr FailSym) { + auto I = FailJD->Symbols.find_as(FailSym); + assert(I != FailJD->Symbols.end()); + I->second.setFlags(I->second.getFlags() | JITSymbolFlags::HasError); + auto J = FailJD->MaterializingInfos.find_as(FailSym); + if (J != FailJD->MaterializingInfos.end()) { + for (auto &Q : J->second.takeAllPendingQueries()) + FailedQueries.insert(std::move(Q)); + FailJD->MaterializingInfos.erase(J); } - for (auto &Q : ToDetach) - Q->detach(); - assert(!MI.hasQueriesPending() && "Queries still pending after detach"); }; - for (auto &Name : SymbolsToFail) { - (*FailedSymbolsMap)[&JD].insert(Name); - - // Look up the symbol to fail. - auto SymI = JD.Symbols.find(Name); - - // FIXME: Revisit this. We should be able to assert sequencing between - // ResourceTracker removal and symbol failure. - // - // It's possible that this symbol has already been removed, e.g. if a - // materialization failure happens concurrently with a ResourceTracker or - // JITDylib removal. In that case we can safely skip this symbol and - // continue. - if (SymI == JD.Symbols.end()) - continue; - auto &Sym = SymI->second; - - // If the symbol is already in the error state then we must have visited - // it earlier. - if (Sym.getFlags().hasError()) { - assert(!JD.MaterializingInfos.count(Name) && - "Symbol in error state still has MaterializingInfo"); - continue; - } + auto FailedSymbolsMap = std::make_shared(); - // Move the symbol into the error state. - Sym.setFlags(Sym.getFlags() | JITSymbolFlags::HasError); - - // FIXME: Come up with a sane mapping of state to - // presence-of-MaterializingInfo so that we can assert presence / absence - // here, rather than testing it. - auto MII = JD.MaterializingInfos.find(Name); - if (MII == JD.MaterializingInfos.end()) - continue; - - auto &MI = MII->second; - - // Collect queries to be failed for this MII. - ExtractFailedQueries(MI); - - if (MI.DefiningEDU) { - // If there is a DefiningEDU for this symbol then remove this - // symbol from it. - assert(MI.DependantEDUs.empty() && - "Symbol with DefiningEDU should not have DependantEDUs"); - assert(Sym.getState() >= SymbolState::Emitted && - "Symbol has EDU, should have been emitted"); - assert(MI.DefiningEDU->Symbols.count(NonOwningSymbolStringPtr(Name)) && - "Symbol does not appear in its DefiningEDU"); - MI.DefiningEDU->Symbols.erase(NonOwningSymbolStringPtr(Name)); - - // Remove this EDU from the dependants lists of its dependencies. - for (auto &[DepJD, DepSyms] : MI.DefiningEDU->Dependencies) { - for (auto DepSym : DepSyms) { - assert(DepJD->Symbols.count(SymbolStringPtr(DepSym)) && - "DepSym not in DepJD"); - assert(DepJD->MaterializingInfos.count(SymbolStringPtr(DepSym)) && - "DepSym has not MaterializingInfo"); - auto &SymMI = DepJD->MaterializingInfos[SymbolStringPtr(DepSym)]; - assert(SymMI.DependantEDUs.count(MI.DefiningEDU.get()) && - "DefiningEDU missing from DependantEDUs list of dependency"); - SymMI.DependantEDUs.erase(MI.DefiningEDU.get()); - } - } + { + auto &FailedSymsForJD = (*FailedSymbolsMap)[&JD]; + for (auto &Sym : SymbolsToFail) { + FailedSymsForJD.insert(Sym); + Fail(&JD, NonOwningSymbolStringPtr(Sym)); + } + } - MI.DefiningEDU = nullptr; - } else { - // Otherwise if there are any EDUs waiting on this symbol then move - // those symbols to the error state too, and deregister them from the - // symbols that they depend on. - // Note: We use a copy of DependantEDUs here since we'll be removing - // from the original set as we go. - for (auto &DependantEDU : MI.DependantEDUs) { - - // Remove DependantEDU from all of its users DependantEDUs lists. - for (auto &[DepJD, DepSyms] : DependantEDU->Dependencies) { - for (auto DepSym : DepSyms) { - // Skip self-reference to avoid invalidating the MI.DependantEDUs - // map. We'll clear this later. - if (DepJD == &JD && DepSym == Name) - continue; - assert(DepJD->Symbols.count(SymbolStringPtr(DepSym)) && - "DepSym not in DepJD?"); - assert(DepJD->MaterializingInfos.count(SymbolStringPtr(DepSym)) && - "DependantEDU not registered with symbol it depends on"); - auto &SymMI = DepJD->MaterializingInfos[SymbolStringPtr(DepSym)]; - assert(SymMI.DependantEDUs.count(DependantEDU) && - "DependantEDU missing from DependantEDUs list"); - SymMI.DependantEDUs.erase(DependantEDU); - } - } + WaitingOnGraph::ContainerElementsMap ToFail; + auto &JDToFail = ToFail[&JD]; + for (auto &Sym : SymbolsToFail) + JDToFail.insert(NonOwningSymbolStringPtr(Sym)); - // Move any symbols defined by DependantEDU into the error state and - // fail any queries waiting on them. - auto &DepJD = *DependantEDU->JD; - auto DepEDUSymbols = std::move(DependantEDU->Symbols); - for (auto &[DepName, Flags] : DepEDUSymbols) { - auto DepSymItr = DepJD.Symbols.find(SymbolStringPtr(DepName)); - assert(DepSymItr != DepJD.Symbols.end() && - "Symbol not present in table"); - auto &DepSym = DepSymItr->second; - - assert(DepSym.getState() >= SymbolState::Emitted && - "Symbol has EDU, should have been emitted"); - assert(!DepSym.getFlags().hasError() && - "Symbol is already in the error state?"); - DepSym.setFlags(DepSym.getFlags() | JITSymbolFlags::HasError); - (*FailedSymbolsMap)[&DepJD].insert(SymbolStringPtr(DepName)); - - // This symbol has a defining EDU so its MaterializingInfo object must - // exist. - auto DepMIItr = - DepJD.MaterializingInfos.find(SymbolStringPtr(DepName)); - assert(DepMIItr != DepJD.MaterializingInfos.end() && - "Symbol has defining EDU but not MaterializingInfo"); - auto &DepMI = DepMIItr->second; - assert(DepMI.DefiningEDU.get() == DependantEDU && - "Bad EDU dependence edge"); - assert(DepMI.DependantEDUs.empty() && - "Symbol was emitted, should not have any DependantEDUs"); - ExtractFailedQueries(DepMI); - DepJD.MaterializingInfos.erase(SymbolStringPtr(DepName)); - } + auto FailedSNs = G.fail(ToFail); - DepJD.shrinkMaterializationInfoMemory(); + for (auto &SN : FailedSNs) { + for (auto &[FailJD, Defs] : SN->defs()) { + auto &FailedSymsForFailJD = (*FailedSymbolsMap)[FailJD]; + for (auto &Def : Defs) { + FailedSymsForFailJD.insert(SymbolStringPtr(Def)); + Fail(FailJD, Def); } - - MI.DependantEDUs.clear(); } - - assert(!MI.DefiningEDU && "DefiningEDU should have been reset"); - assert(MI.DependantEDUs.empty() && - "DependantEDUs should have been removed above"); - assert(!MI.hasQueriesPending() && - "Can not delete MaterializingInfo with queries pending"); - JD.MaterializingInfos.erase(Name); } - JD.shrinkMaterializationInfoMemory(); + // Detach all failed queries. + for (auto &Q : FailedQueries) + Q->detach(); #ifdef EXPENSIVE_CHECKS verifySessionState("exiting ExecutionSession::IL_failSymbols"); @@ -3721,9 +3232,11 @@ void ExecutionSession::OL_notifyFailed(MaterializationResponsibility &MR) { return IL_failSymbols(MR.getTargetJITDylib(), SymbolsToFail); }); - for (auto &Q : FailedQueries) + for (auto &Q : FailedQueries) { + Q->detach(); Q->handleFailed( make_error(getSymbolStringPool(), FailedSymbols)); + } } Error ExecutionSession::OL_replace(MaterializationResponsibility &MR, diff --git a/llvm/lib/ExecutionEngine/Orc/SimpleRemoteEPC.cpp b/llvm/lib/ExecutionEngine/Orc/SimpleRemoteEPC.cpp index 87d757805a64c..03a06792b4a30 100644 --- a/llvm/lib/ExecutionEngine/Orc/SimpleRemoteEPC.cpp +++ b/llvm/lib/ExecutionEngine/Orc/SimpleRemoteEPC.cpp @@ -448,7 +448,7 @@ Error SimpleRemoteEPC::handleHangup(SimpleRemoteEPCArgBytesVector ArgBytes) { if (const char *ErrMsg = WFR.getOutOfBandError()) return make_error(ErrMsg, inconvertibleErrorCode()); - detail::SPSSerializableError Info; + orc::shared::detail::SPSSerializableError Info; SPSInputBuffer IB(WFR.data(), WFR.size()); if (!SPSArgList::deserialize(IB, Info)) return make_error("Could not deserialize hangup info", diff --git a/llvm/test/ExecutionEngine/JITLink/AArch64/MachO_ptrauth_dependencies.s b/llvm/test/ExecutionEngine/JITLink/AArch64/MachO_ptrauth_dependencies.s index 454079923622e..e52e4367f97f7 100644 --- a/llvm/test/ExecutionEngine/JITLink/AArch64/MachO_ptrauth_dependencies.s +++ b/llvm/test/ExecutionEngine/JITLink/AArch64/MachO_ptrauth_dependencies.s @@ -1,5 +1,5 @@ # RUN: llvm-mc -triple=arm64e-apple-macosx -filetype=obj -o %t.o %s -# RUN: llvm-jitlink -num-threads=0 -debug-only=orc -noexec \ +# RUN: llvm-jitlink -debug-only=orc -noexec \ # RUN: -abs _foo=0x1 %t.o 2>&1 \ # RUN: | FileCheck %s # diff --git a/llvm/test/ExecutionEngine/JITLink/x86-64/LocalDependencyPropagation.s b/llvm/test/ExecutionEngine/JITLink/x86-64/LocalDependencyPropagation.s index 83d71cdf6fc83..529395822f5f7 100644 --- a/llvm/test/ExecutionEngine/JITLink/x86-64/LocalDependencyPropagation.s +++ b/llvm/test/ExecutionEngine/JITLink/x86-64/LocalDependencyPropagation.s @@ -16,8 +16,7 @@ # CHECK-DAG: Symbols: { _foo }, Dependencies: { (main, { _external_func }) } # CHECK-DAG: Symbols: { _baz }, Dependencies: { (main, { _foo }) } # CHECK: Simplified dependencies: -# CHECK-DAG: Symbols: { _foo }, Dependencies: { (main, { _external_func }) } -# CHECK-DAG: Symbols: { _baz }, Dependencies: { (main, { _external_func }) } +# CHECK-DAG: Defs: { (main, [ _baz _foo ]) }, Deps: { (main, [ _external_func ]) } .section __TEXT,__text,regular,pure_instructions diff --git a/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt b/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt index a2bbb10039c9a..7e3ebc88cea63 100644 --- a/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt +++ b/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt @@ -42,6 +42,7 @@ add_llvm_unittest(OrcJITTests SymbolStringPoolTest.cpp TaskDispatchTest.cpp ThreadSafeModuleTest.cpp + WaitingOnGraphTest.cpp WrapperFunctionUtilsTest.cpp JITLinkRedirectionManagerTest.cpp ReOptimizeLayerTest.cpp diff --git a/llvm/unittests/ExecutionEngine/Orc/WaitingOnGraphTest.cpp b/llvm/unittests/ExecutionEngine/Orc/WaitingOnGraphTest.cpp new file mode 100644 index 0000000000000..b988a78a3783a --- /dev/null +++ b/llvm/unittests/ExecutionEngine/Orc/WaitingOnGraphTest.cpp @@ -0,0 +1,553 @@ +//===--------- WaitingOnGraphTest.cpp - Test WaitingOnGraph APIs ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/WaitingOnGraph.h" +#include "gtest/gtest.h" + +namespace llvm::orc::detail { + +class WaitingOnGraphTest : public testing::Test { +public: + using TestGraph = WaitingOnGraph; + +protected: + using SuperNode = TestGraph::SuperNode; + using SuperNodeBuilder = TestGraph::SuperNodeBuilder; + using ContainerElementsMap = TestGraph::ContainerElementsMap; + using ElemToSuperNodeMap = TestGraph::ElemToSuperNodeMap; + using SimplifyResult = TestGraph::SimplifyResult; + using EmitResult = TestGraph::EmitResult; + + static const ContainerElementsMap &getDefs(SuperNode &SN) { return SN.Defs; } + + static const ContainerElementsMap &getDeps(SuperNode &SN) { return SN.Deps; } + + static std::vector> &getSNs(SimplifyResult &SR) { + return SR.SNs; + } + + static ElemToSuperNodeMap &getElemToSN(SimplifyResult &SR) { + return SR.ElemToSN; + } + + static std::vector> &getPendingSNs(TestGraph &G) { + return G.PendingSNs; + } + + static ContainerElementsMap merge(ContainerElementsMap M1, + const ContainerElementsMap &M2) { + ContainerElementsMap Result = std::move(M1); + for (auto &[Container, Elems] : M2) + Result[Container].insert(Elems.begin(), Elems.end()); + return Result; + } + + ContainerElementsMap + collapseDefs(std::vector> &SNs, + bool DepsMustMatch = true) { + if (SNs.empty()) + return ContainerElementsMap(); + + ContainerElementsMap Result = SNs[0]->defs(); + const ContainerElementsMap &Deps = SNs[0]->deps(); + + for (size_t I = 1; I != SNs.size(); ++I) { + assert(!DepsMustMatch || SNs[I]->deps() == Deps); + Result = merge(std::move(Result), SNs[I]->defs()); + } + + return Result; + } + + EmitResult integrate(EmitResult ER) { + for (auto &SN : ER.Ready) + for (auto &[Container, Elems] : SN->defs()) + Ready[Container].insert(Elems.begin(), Elems.end()); + for (auto &SN : ER.Failed) + for (auto &[Container, Elems] : SN->defs()) + Failed[Container].insert(Elems.begin(), Elems.end()); + return ER; + } + + EmitResult emit(SimplifyResult SR) { + return integrate(G.emit(std::move(SR), GetExternalState)); + } + + TestGraph G; + ContainerElementsMap Ready; + ContainerElementsMap Failed; + + class ExternalStateGetter { + public: + ExternalStateGetter(WaitingOnGraphTest &T) : T(T) {} + TestGraph::ExternalState operator()(TestGraph::ContainerId C, + TestGraph::ElementId E) { + { + auto I = T.Failed.find(C); + if (I != T.Failed.end()) + if (I->second.count(E)) + return TestGraph::ExternalState::Failed; + } + + { + auto I = T.Ready.find(C); + if (I != T.Ready.end()) + if (I->second.count(E)) + return TestGraph::ExternalState::Ready; + } + + return TestGraph::ExternalState::None; + } + + private: + WaitingOnGraphTest &T; + }; + + ExternalStateGetter GetExternalState{*this}; +}; + +} // namespace llvm::orc::detail + +using namespace llvm; +using namespace llvm::orc; +using namespace llvm::orc::detail; + +TEST_F(WaitingOnGraphTest, ConstructAndDestroyEmpty) { + // Nothing to do here -- we're just testing construction and destruction + // of the WaitingOnGraphTest::G member. +} + +TEST_F(WaitingOnGraphTest, Build_TrivialSingleSuperNode) { + // Add one set of trivial defs and empty deps to the builder, make sure that + // they're passed through to the resulting super-node. + SuperNodeBuilder B; + ContainerElementsMap Defs({{0, {0}}}); + ContainerElementsMap Deps; + B.add(Defs, Deps); + auto SNs = B.takeSuperNodes(); + EXPECT_EQ(SNs.size(), 1U); + EXPECT_EQ(getDefs(*SNs[0]), Defs); + EXPECT_EQ(getDeps(*SNs[0]), Deps); +} + +TEST_F(WaitingOnGraphTest, Build_EmptyDefs) { + // Adding empty def sets is ok, but should not result in creation of a + // SuperNode. + SuperNodeBuilder B; + ContainerElementsMap Empty; + B.add(Empty, Empty); + auto SNs = B.takeSuperNodes(); + EXPECT_TRUE(SNs.empty()); +} + +TEST_F(WaitingOnGraphTest, Build_NonTrivialSingleSuperNode) { + // Add one non-trivwial set of defs and deps. Make sure that they're passed + // through to the resulting super-node. + SuperNodeBuilder B; + ContainerElementsMap Defs({{0, {0, 1, 2}}}); + ContainerElementsMap Deps({{1, {3, 4, 5}}}); + B.add(Defs, Deps); + auto SNs = B.takeSuperNodes(); + EXPECT_EQ(SNs.size(), 1U); + EXPECT_EQ(getDefs(*SNs[0]), Defs); + EXPECT_EQ(getDeps(*SNs[0]), Deps); +} + +TEST_F(WaitingOnGraphTest, Build_CoalesceEmptyDeps) { + // Add two trivial defs both with empty deps to the builder. Check that + // they're coalesced into a single super-node. + SuperNodeBuilder B; + ContainerElementsMap Defs1({{0, {0}}}); + ContainerElementsMap Defs2({{0, {1}}}); + ContainerElementsMap Deps; + B.add(Defs1, Deps); + B.add(Defs2, Deps); + auto SNs = B.takeSuperNodes(); + EXPECT_EQ(SNs.size(), 1U); + EXPECT_EQ(getDefs(*SNs[0]), merge(Defs1, Defs2)); + EXPECT_EQ(getDeps(*SNs[0]), Deps); +} + +TEST_F(WaitingOnGraphTest, Build_CoalesceNonEmptyDeps) { + // Add two sets trivial of trivial defs with empty deps to the builder. Check + // that the two coalesce into a single super node. + SuperNodeBuilder B; + ContainerElementsMap Defs1({{0, {0}}}); + ContainerElementsMap Defs2({{0, {1}}}); + ContainerElementsMap Deps({{1, {1}}}); + B.add(Defs1, Deps); + B.add(Defs2, Deps); + auto SNs = B.takeSuperNodes(); + EXPECT_EQ(SNs.size(), 1U); + EXPECT_EQ(getDefs(*SNs[0]), merge(Defs1, Defs2)); + EXPECT_EQ(getDeps(*SNs[0]), Deps); +} + +TEST_F(WaitingOnGraphTest, Build_CoalesceInterleaved) { + // Add multiple sets of defs, some with the same dep sets. Check that nodes + // are still coalesced as expected. + SuperNodeBuilder B; + + ContainerElementsMap DefsA1({{0, {0}}}); + ContainerElementsMap DefsA2({{0, {1}}}); + ContainerElementsMap DefsB1({{1, {0}}}); + ContainerElementsMap DefsB2({{1, {1}}}); + ContainerElementsMap DepsA({{2, {0}}, {3, {0}}}); + ContainerElementsMap DepsB({{4, {0}}, {5, {0}}}); + B.add(DefsA1, DepsA); + B.add(DefsB1, DepsB); + B.add(DefsA2, DepsA); + B.add(DefsB2, DepsB); + auto SNs = B.takeSuperNodes(); + EXPECT_EQ(SNs.size(), 2U); + EXPECT_EQ(getDefs(*SNs[0]), merge(DefsA1, DefsA2)); + EXPECT_EQ(getDeps(*SNs[0]), DepsA); + EXPECT_EQ(getDefs(*SNs[1]), merge(DefsB1, DefsB2)); + EXPECT_EQ(getDeps(*SNs[1]), DepsB); +} + +TEST_F(WaitingOnGraphTest, Build_SelfDepRemoval) { + // Add multiple sets of defs, some with the same dep sets. Check that nodes + // are still coalesced as expected. + SuperNodeBuilder B; + ContainerElementsMap Defs({{0, {0, 1}}}); + ContainerElementsMap Deps({{0, {1}}}); + ContainerElementsMap Empty; + B.add(Defs, Deps); + auto SNs = B.takeSuperNodes(); + EXPECT_EQ(SNs.size(), 1U); + EXPECT_EQ(getDefs(*SNs[0]), Defs); + EXPECT_EQ(getDeps(*SNs[0]), Empty); +} + +TEST_F(WaitingOnGraphTest, Simplification_EmptySimplification) { + auto SR = TestGraph::simplify({}); + auto &SNs = getSNs(SR); + EXPECT_EQ(SNs.size(), 0U); + EXPECT_EQ(getElemToSN(SR), ElemToSuperNodeMap()); +} + +TEST_F(WaitingOnGraphTest, Simplification_TrivialSingleSuperNode) { + // Test trivial call to simplify. + SuperNodeBuilder B; + ContainerElementsMap Defs({{0, {0}}}); + ContainerElementsMap Deps({{0, {0}}}); + B.add(Defs, Deps); + auto SR = TestGraph::simplify(B.takeSuperNodes()); + ContainerElementsMap Empty; + + // Check SNs. + auto &SNs = getSNs(SR); + EXPECT_EQ(SNs.size(), 1U); + EXPECT_EQ(getDefs(*SNs.at(0)), Defs); + EXPECT_EQ(getDeps(*SNs.at(0)), Empty); + + // Check ElemToSNs. + ElemToSuperNodeMap ExpectedElemToSNs; + ExpectedElemToSNs[0][0] = SNs[0].get(); + EXPECT_EQ(getElemToSN(SR), ExpectedElemToSNs); +} + +TEST_F(WaitingOnGraphTest, Simplification_SimplifySingleContainerSimpleCycle) { + // Test trivial simplification call with two nodes and one internal + // dependence cycle within a single container: + // N0: (0, 0) -> (0, 1) + // N1: (0, 1) -> (0, 0) + // We expect intra-simplify cycle elimination to clear both dependence sets, + // and coalescing to join them into one supernode covering both defs. + SuperNodeBuilder B; + ContainerElementsMap Defs0({{0, {0}}}); + ContainerElementsMap Deps0({{0, {1}}}); + B.add(Defs0, Deps0); + ContainerElementsMap Defs1({{0, {1}}}); + ContainerElementsMap Deps1({{0, {0}}}); + B.add(Defs1, Deps1); + auto SR = TestGraph::simplify(B.takeSuperNodes()); + + // Check SNs. + auto &SNs = getSNs(SR); + ContainerElementsMap Empty; + EXPECT_EQ(SNs.size(), 1U); + EXPECT_EQ(getDefs(*SNs.at(0)), merge(Defs0, Defs1)); + EXPECT_EQ(getDeps(*SNs.at(0)), Empty); + + // Check ElemToSNs. + ElemToSuperNodeMap ExpectedElemToSNs; + ExpectedElemToSNs[0][0] = SNs[0].get(); + ExpectedElemToSNs[0][1] = SNs[0].get(); + + EXPECT_EQ(getElemToSN(SR), ExpectedElemToSNs); +} + +TEST_F(WaitingOnGraphTest, + Simplification_SimplifySingleContainerNElementCycle) { + // Test trivial simplification call with M nodes and one internal + // dependence cycle within a single container: + // N0: (0, 0) -> (0, 1) + // N1: (0, 1) -> (0, 2) + // ... + // NM: (0, M) -> (0, 0) + // We expect intra-simplify cycle elimination to clear all dependence sets, + // and coalescing to join them into one supernode covering all defs. + SuperNodeBuilder B; + constexpr size_t M = 10; + for (size_t I = 0; I != M; ++I) { + ContainerElementsMap Defs({{0, {I}}}); + ContainerElementsMap Deps({{0, {(I + 1) % M}}}); + B.add(Defs, Deps); + } + auto InitSNs = B.takeSuperNodes(); + EXPECT_EQ(InitSNs.size(), M); + + auto SR = TestGraph::simplify(std::move(InitSNs)); + + // Check SNs. + auto &SNs = getSNs(SR); + ContainerElementsMap ExpectedDefs; + for (size_t I = 0; I != M; ++I) + ExpectedDefs[0].insert(I); + ContainerElementsMap Empty; + EXPECT_EQ(SNs.size(), 1U); + EXPECT_EQ(getDefs(*SNs.at(0)), ExpectedDefs); + EXPECT_EQ(getDeps(*SNs.at(0)), Empty); + + // Check ElemToSNs. + ElemToSuperNodeMap ExpectedElemToSNs; + for (size_t I = 0; I != M; ++I) + ExpectedElemToSNs[0][I] = SNs[0].get(); + + EXPECT_EQ(getElemToSN(SR), ExpectedElemToSNs); +} + +TEST_F(WaitingOnGraphTest, Simplification_SimplifyIntraSimplifyPropagateDeps) { + // Test trivial simplification call with two nodes and one internal + // dependence cycle within a single container: + // N0: (0, 0) -> (0, {1, 2}) + // N1: (0, 1) -> (0, {3}) + // We expect intra-simplify cycle elimination to replace the dependence of + // (0, 0) on (0, 1) with a dependence on (0, 3) instead. + SuperNodeBuilder B; + ContainerElementsMap Defs0({{0, {0}}}); + ContainerElementsMap Deps0({{0, {1, 2}}}); + B.add(Defs0, Deps0); + ContainerElementsMap Defs1({{0, {1}}}); + ContainerElementsMap Deps1({{0, {3}}}); + B.add(Defs1, Deps1); + auto SR = TestGraph::simplify(B.takeSuperNodes()); + + // Check SNs. + auto &SNs = getSNs(SR); + EXPECT_EQ(SNs.size(), 2U); + + // ContainerElemenstMap ExpectedDefs0({{0, {0}}}); + // ContainerElemenstMap ExpectedDeps0({{0, {1, 3}}}); + EXPECT_EQ(getDefs(*SNs.at(0)), ContainerElementsMap({{0, {0}}})); + EXPECT_EQ(getDeps(*SNs.at(0)), ContainerElementsMap({{0, {2, 3}}})); + + EXPECT_EQ(getDefs(*SNs.at(1)), ContainerElementsMap({{0, {1}}})); + EXPECT_EQ(getDeps(*SNs.at(1)), ContainerElementsMap({{0, {3}}})); + + // Check ElemToSNs. + ElemToSuperNodeMap ExpectedElemToSNs; + ExpectedElemToSNs[0][0] = SNs[0].get(); + ExpectedElemToSNs[0][1] = SNs[1].get(); + + EXPECT_EQ(getElemToSN(SR), ExpectedElemToSNs); +} + +TEST_F(WaitingOnGraphTest, Emit_EmptyEmit) { + // Check that empty emits work as expected. + auto ER = G.emit(TestGraph::simplify({}), GetExternalState); + + EXPECT_EQ(ER.Ready.size(), 0U); + EXPECT_EQ(ER.Failed.size(), 0U); +} + +TEST_F(WaitingOnGraphTest, Emit_TrivialSingleNode) { + // Check that emitting a single node behaves as expected. + SuperNodeBuilder B; + ContainerElementsMap Defs({{0, {0}}}); + B.add(Defs, ContainerElementsMap()); + auto ER = emit(TestGraph::simplify(B.takeSuperNodes())); + EXPECT_EQ(collapseDefs(ER.Ready), Defs); + EXPECT_EQ(ER.Failed.size(), 0U); +} + +TEST_F(WaitingOnGraphTest, Emit_TrivialSequence) { + // Perform a sequence of two emits where the second emit depends on the + // first. Check that nodes become ready after each emit. + SuperNodeBuilder B; + ContainerElementsMap Defs0({{0, {0}}}); + ContainerElementsMap Empty; + B.add(Defs0, Empty); + auto ER0 = emit(TestGraph::simplify(B.takeSuperNodes())); + EXPECT_EQ(collapseDefs(ER0.Ready), Defs0); + EXPECT_EQ(ER0.Failed.size(), 0U); + + ContainerElementsMap Defs1({{0, {1}}}); + ContainerElementsMap Deps1({{0, {0}}}); + B.add(Defs1, Deps1); + auto ER1 = emit(TestGraph::simplify(B.takeSuperNodes())); + EXPECT_EQ(collapseDefs(ER1.Ready), Defs1); + EXPECT_EQ(ER1.Failed.size(), 0U); +} + +TEST_F(WaitingOnGraphTest, Emit_TrivialReverseSequence) { + // Perform a sequence of two emits where the first emit depends on the + // second. Check that both nodes become ready after the second emit. + SuperNodeBuilder B; + ContainerElementsMap Defs0({{0, {0}}}); + ContainerElementsMap Deps0({{0, {1}}}); + B.add(Defs0, Deps0); + auto ER0 = emit(TestGraph::simplify(B.takeSuperNodes())); + EXPECT_EQ(ER0.Ready.size(), 0U); + EXPECT_EQ(ER0.Failed.size(), 0U); + + ContainerElementsMap Defs1({{0, {1}}}); + ContainerElementsMap Empty; + B.add(Defs1, Empty); + auto ER1 = emit(TestGraph::simplify(B.takeSuperNodes())); + EXPECT_EQ(collapseDefs(ER1.Ready), merge(Defs0, Defs1)); + EXPECT_EQ(ER1.Failed.size(), 0U); +} + +TEST_F(WaitingOnGraphTest, Emit_Coalescing) { + SuperNodeBuilder B; + ContainerElementsMap Defs0({{0, {0}}}); + ContainerElementsMap Deps0({{1, {0}}}); + B.add(Defs0, Deps0); + auto ER0 = emit(TestGraph::simplify(B.takeSuperNodes())); + EXPECT_EQ(ER0.Ready.size(), 0U); + EXPECT_EQ(ER0.Failed.size(), 0U); + + ContainerElementsMap Defs1({{0, {1}}}); + ContainerElementsMap Deps1({{1, {0}}}); + B.add(Defs1, Deps1); + auto ER1 = emit(TestGraph::simplify(B.takeSuperNodes())); + EXPECT_EQ(ER1.Ready.size(), 0U); + EXPECT_EQ(ER1.Failed.size(), 0U); + + // Check that after emitting two nodes with the same dep set we have only one + // pending supernode whose defs are the union of the defs in the two emits. + auto &PendingSNs = getPendingSNs(G); + EXPECT_EQ(PendingSNs.size(), 1U); + EXPECT_EQ(getDefs(*PendingSNs.at(0)), merge(Defs0, Defs1)); + + ContainerElementsMap Defs2({{1, {0}}}); + ContainerElementsMap Empty; + B.add(Defs2, Empty); + auto ER2 = emit(TestGraph::simplify(B.takeSuperNodes())); + EXPECT_EQ(collapseDefs(ER2.Ready), merge(merge(Defs0, Defs1), Defs2)); + EXPECT_EQ(ER2.Failed.size(), 0U); +} + +TEST_F(WaitingOnGraphTest, Emit_ZigZag) { + // Perform a sequence of four emits, where the first three contain a zig-zag + // pattern: + // 1. (0, 0) -> (0, 1) + // 2. (0, 2) -> (0, 3) + // ^ -- At this point we expect two pending supernodes. + // 3. (0, 1) -> (0, 2) + // ^ -- Resolution of (0, 1) should cause all three emitted nodes to coalsce + // into one supernode defining (0, {1, 2, 3}). + // 4. (0, 3) + // ^ -- Should cause all four nodes to become ready. + + SuperNodeBuilder B; + ContainerElementsMap Defs0({{0, {0}}}); + ContainerElementsMap Deps0({{0, {1}}}); + B.add(Defs0, Deps0); + auto ER0 = emit(TestGraph::simplify(B.takeSuperNodes())); + EXPECT_EQ(ER0.Ready.size(), 0U); + EXPECT_EQ(ER0.Failed.size(), 0U); + + ContainerElementsMap Defs1({{0, {2}}}); + ContainerElementsMap Deps1({{0, {3}}}); + B.add(Defs1, Deps1); + auto ER1 = emit(TestGraph::simplify(B.takeSuperNodes())); + EXPECT_EQ(ER1.Ready.size(), 0U); + EXPECT_EQ(ER1.Failed.size(), 0U); + + // Check that after emitting two nodes with the same dep set we have only one + // pending supernode whose defs are the union of the defs in the two emits. + auto &PendingSNs = getPendingSNs(G); + EXPECT_EQ(PendingSNs.size(), 2U); + EXPECT_EQ(getDefs(*PendingSNs.at(0)), Defs0); + EXPECT_EQ(getDeps(*PendingSNs.at(0)), Deps0); + EXPECT_EQ(getDefs(*PendingSNs.at(1)), Defs1); + EXPECT_EQ(getDeps(*PendingSNs.at(1)), Deps1); + + ContainerElementsMap Defs2({{0, {1}}}); + ContainerElementsMap Deps2({{0, {2}}}); + B.add(Defs2, Deps2); + auto ER2 = emit(TestGraph::simplify(B.takeSuperNodes())); + EXPECT_EQ(ER2.Ready.size(), 0U); + EXPECT_EQ(ER2.Failed.size(), 0U); + + // Check that after emitting the third node we've coalesced all three. + EXPECT_EQ(PendingSNs.size(), 1U); + EXPECT_EQ(getDefs(*PendingSNs.at(0)), merge(merge(Defs0, Defs1), Defs2)); + EXPECT_EQ(getDeps(*PendingSNs.at(0)), Deps1); + + ContainerElementsMap Defs3({{0, {3}}}); + ContainerElementsMap Empty; + B.add(Defs3, Empty); + auto ER3 = emit(TestGraph::simplify(B.takeSuperNodes())); + + EXPECT_EQ(collapseDefs(ER3.Ready), + merge(merge(merge(Defs0, Defs1), Defs2), Defs3)); + EXPECT_EQ(ER2.Failed.size(), 0U); + EXPECT_TRUE(PendingSNs.empty()); +} + +TEST_F(WaitingOnGraphTest, Fail_Empty) { + // Check that failing an empty set is a no-op. + auto FR = G.fail(ContainerElementsMap()); + EXPECT_EQ(FR.size(), 0U); +} + +TEST_F(WaitingOnGraphTest, Fail_Single) { + // Check that failing a set with no existing dependencies works. + auto FR = G.fail({{0, {0}}}); + EXPECT_EQ(FR.size(), 0U); +} + +TEST_F(WaitingOnGraphTest, Fail_EmitDependenceOnFailure) { + // Check that emitted nodes that directly depend on failed nodes also fail. + Failed = {{0, {0}}}; + + SuperNodeBuilder B; + ContainerElementsMap Defs({{0, {1}}}); + ContainerElementsMap Deps({{0, {0}}}); + B.add(Defs, Deps); + auto ER = emit(TestGraph::simplify(B.takeSuperNodes())); + EXPECT_EQ(ER.Ready.size(), 0U); + EXPECT_EQ(collapseDefs(ER.Failed, false), Defs); +} + +TEST_F(WaitingOnGraphTest, Fail_ZigZag) { + // Check that if an emit introduces a transitive dependence of a failed + // node, then all nodes that depend on the failed node are also failed. + SuperNodeBuilder B; + + ContainerElementsMap Defs0({{0, {0}}}); + ContainerElementsMap Deps0({{0, {1}}}); + B.add(Defs0, Deps0); + auto ER0 = emit(TestGraph::simplify(B.takeSuperNodes())); + EXPECT_EQ(ER0.Ready.size(), 0U); + EXPECT_EQ(ER0.Failed.size(), 0U); + + Failed = {{0, {2}}}; + + ContainerElementsMap Defs1({{0, {1}}}); + ContainerElementsMap Deps1({{0, {2}}}); + B.add(Defs1, Deps1); + auto ER1 = emit(TestGraph::simplify(B.takeSuperNodes())); + EXPECT_EQ(ER1.Ready.size(), 0U); + EXPECT_EQ(collapseDefs(ER1.Failed, false), merge(Defs0, Defs1)); +}