diff --git a/llvm/lib/CodeGen/RDFLiveness.cpp b/llvm/lib/CodeGen/RDFLiveness.cpp index 0bcd27f8ea452c..67ebc2303f99dc 100644 --- a/llvm/lib/CodeGen/RDFLiveness.cpp +++ b/llvm/lib/CodeGen/RDFLiveness.cpp @@ -23,8 +23,10 @@ // <10.1145/2086696.2086706>. // #include "llvm/ADT/BitVector.h" +#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SetVector.h" +#include "llvm/ADT/SmallSet.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineDominanceFrontier.h" #include "llvm/CodeGen/MachineDominators.h" @@ -108,7 +110,7 @@ NodeList Liveness::getAllReachingDefs(RegisterRef RefRR, const RegisterAggr &DefRRs) { NodeList RDefs; // Return value. SetVector DefQ; - SetVector Owners; + DenseMap OrdMap; // Dead defs will be treated as if they were live, since they are actually // on the data-flow path. They cannot be ignored because even though they @@ -151,18 +153,9 @@ NodeList Liveness::getAllReachingDefs(RegisterRef RefRR, for (auto S : DFG.getRelatedRefs(TA.Addr->getOwner(DFG), TA)) if (NodeId RD = NodeAddr(S).Addr->getReachingDef()) DefQ.insert(RD); - } - - // Remove all non-phi defs that are not aliased to RefRR, and collect - // the owners of the remaining defs. - SetVector Defs; - for (NodeId N : DefQ) { - auto TA = DFG.addr(N); - bool IsPhi = TA.Addr->getFlags() & NodeAttrs::PhiRef; - if (!IsPhi && !PRI.alias(RefRR, TA.Addr->getRegRef(DFG))) - continue; - Defs.insert(TA.Id); - Owners.insert(TA.Addr->getOwner(DFG).Id); + // Don't visit sibling defs. They share the same reaching def (which + // will be visited anyway), but they define something not aliased to + // this ref. } // Return the MachineBasicBlock containing a given instruction. @@ -174,38 +167,81 @@ NodeList Liveness::getAllReachingDefs(RegisterRef RefRR, NodeAddr BA = PA.Addr->getOwner(DFG); return BA.Addr->getCode(); }; - // Less(A,B) iff instruction A is further down in the dominator tree than B. - auto Less = [&Block,this] (NodeId A, NodeId B) -> bool { + + SmallSet Defs; + + // Remove all non-phi defs that are not aliased to RefRR, and segregate + // the the remaining defs into buckets for containing blocks. + std::map> Owners; + std::map> Blocks; + for (NodeId N : DefQ) { + auto TA = DFG.addr(N); + bool IsPhi = TA.Addr->getFlags() & NodeAttrs::PhiRef; + if (!IsPhi && !PRI.alias(RefRR, TA.Addr->getRegRef(DFG))) + continue; + Defs.insert(TA.Id); + NodeAddr IA = TA.Addr->getOwner(DFG); + Owners[TA.Id] = IA; + Blocks[Block(IA)].push_back(IA.Id); + } + + auto Precedes = [this,&OrdMap] (NodeId A, NodeId B) { if (A == B) return false; - auto OA = DFG.addr(A), OB = DFG.addr(B); - MachineBasicBlock *BA = Block(OA), *BB = Block(OB); - if (BA != BB) - return MDT.dominates(BB, BA); - // They are in the same block. + NodeAddr OA = DFG.addr(A); + NodeAddr OB = DFG.addr(B); bool StmtA = OA.Addr->getKind() == NodeAttrs::Stmt; bool StmtB = OB.Addr->getKind() == NodeAttrs::Stmt; - if (StmtA) { - if (!StmtB) // OB is a phi and phis dominate statements. - return true; - MachineInstr *CA = NodeAddr(OA).Addr->getCode(); - MachineInstr *CB = NodeAddr(OB).Addr->getCode(); - // The order must be linear, so tie-break such equalities. - if (CA == CB) - return A < B; - return MDT.dominates(CB, CA); - } else { - // OA is a phi. - if (StmtB) - return false; - // Both are phis. There is no ordering between phis (in terms of - // the data-flow), so tie-break this via node id comparison. + if (StmtA && StmtB) { + const MachineInstr *InA = NodeAddr(OA).Addr->getCode(); + const MachineInstr *InB = NodeAddr(OB).Addr->getCode(); + assert(InA->getParent() == InB->getParent()); + auto FA = OrdMap.find(InA); + if (FA != OrdMap.end()) + return FA->second < OrdMap.find(InB)->second; + const MachineBasicBlock *BB = InA->getParent(); + for (auto It = BB->begin(), E = BB->end(); It != E; ++It) { + if (It == InA->getIterator()) + return true; + if (It == InB->getIterator()) + return false; + } + llvm_unreachable("InA and InB should be in the same block"); + } + // One of them is a phi node. + if (!StmtA && !StmtB) { + // Both are phis, which are unordered. Break the tie by id numbers. return A < B; } + // Only one of them is a phi. Phis always precede statements. + return !StmtA; }; - std::vector Tmp(Owners.begin(), Owners.end()); - llvm::sort(Tmp, Less); + auto GetOrder = [&OrdMap] (MachineBasicBlock &B) { + uint32_t Pos = 0; + for (MachineInstr &In : B) + OrdMap.insert({&In, ++Pos}); + }; + + // For each block, sort the nodes in it. + std::vector TmpBB; + for (auto &Bucket : Blocks) { + TmpBB.push_back(Bucket.first); + if (Bucket.second.size() > 2) + GetOrder(*Bucket.first); + std::sort(Bucket.second.begin(), Bucket.second.end(), Precedes); + } + + // Sort the blocks with respect to dominance. + std::sort(TmpBB.begin(), TmpBB.end(), [this](auto A, auto B) { + return MDT.dominates(A, B); + }); + + std::vector TmpInst; + for (auto I = TmpBB.rbegin(), E = TmpBB.rend(); I != E; ++I) { + auto &Bucket = Blocks[*I]; + TmpInst.insert(TmpInst.end(), Bucket.rbegin(), Bucket.rend()); + } // The vector is a list of instructions, so that defs coming from // the same instruction don't need to be artificially ordered. @@ -220,6 +256,9 @@ NodeList Liveness::getAllReachingDefs(RegisterRef RefRR, // *d3 If A \incl BuC, and B \incl AuC, then *d2 would be // covered if we added A first, and A would be covered // if we added B first. + // In this example we want both A and B, because we don't want to give + // either one priority over the other, since they belong to the same + // statement. RegisterAggr RRs(DefRRs); @@ -227,7 +266,8 @@ NodeList Liveness::getAllReachingDefs(RegisterRef RefRR, return TA.Addr->getKind() == NodeAttrs::Def && Defs.count(TA.Id); }; - for (NodeId T : Tmp) { + + for (NodeId T : TmpInst) { if (!FullChain && RRs.hasCoverOf(RefRR)) break; auto TA = DFG.addr(T);