diff --git a/llvm/include/llvm/CodeGen/RDFGraph.h b/llvm/include/llvm/CodeGen/RDFGraph.h index fafbf1f03a83a..97673a249237c 100644 --- a/llvm/include/llvm/CodeGen/RDFGraph.h +++ b/llvm/include/llvm/CodeGen/RDFGraph.h @@ -362,32 +362,6 @@ template struct NodeAddr { struct NodeBase; -struct RefNode; -struct DefNode; -struct UseNode; -struct PhiUseNode; - -struct CodeNode; -struct InstrNode; -struct PhiNode; -struct StmtNode; -struct BlockNode; -struct FuncNode; - -using Node = NodeAddr; - -using Ref = NodeAddr; -using Def = NodeAddr; -using Use = NodeAddr; -using PhiUse = NodeAddr; - -using Code = NodeAddr; -using Instr = NodeAddr; -using Phi = NodeAddr; -using Stmt = NodeAddr; -using Block = NodeAddr; -using Func = NodeAddr; - // Fast memory allocation and translation between node id and node address. // This is really the same idea as the one underlying the "bump pointer // allocator", the difference being in the translation. A node id is @@ -422,7 +396,7 @@ struct NodeAllocator { } NodeId id(const NodeBase *P) const; - Node New(); + NodeAddr New(); void clear(); private: @@ -495,7 +469,7 @@ struct NodeBase { void setFlags(uint16_t F) { setAttrs(NodeAttrs::set_flags(getAttrs(), F)); } // Insert node NA after "this" in the circular chain. - void append(Node NA); + void append(NodeAddr NA); // Initialize all members to 0. void init() { memset(this, 0, sizeof *this); } @@ -533,8 +507,8 @@ struct NodeBase { // The actual payload. union { - Ref_struct RefData; - Code_struct CodeData; + Ref_struct Ref; + Code_struct Code; }; }; // The allocator allocates chunks of 32 bytes for each node. The fact that @@ -543,7 +517,7 @@ struct NodeBase { static_assert(sizeof(NodeBase) <= NodeAllocator::NodeMemSize, "NodeBase must be at most NodeAllocator::NodeMemSize bytes"); -using NodeList = SmallVector; +using NodeList = SmallVector, 4>; using NodeSet = std::set; struct RefNode : public NodeBase { @@ -553,17 +527,17 @@ struct RefNode : public NodeBase { MachineOperand &getOp() { assert(!(getFlags() & NodeAttrs::PhiRef)); - return *RefData.Op; + return *Ref.Op; } void setRegRef(RegisterRef RR, DataFlowGraph &G); void setRegRef(MachineOperand *Op, DataFlowGraph &G); - NodeId getReachingDef() const { return RefData.RD; } - void setReachingDef(NodeId RD) { RefData.RD = RD; } + NodeId getReachingDef() const { return Ref.RD; } + void setReachingDef(NodeId RD) { Ref.RD = RD; } - NodeId getSibling() const { return RefData.Sib; } - void setSibling(NodeId Sib) { RefData.Sib = Sib; } + NodeId getSibling() const { return Ref.Sib; } + void setSibling(NodeId Sib) { Ref.Sib = Sib; } bool isUse() const { assert(getType() == NodeAttrs::Ref); @@ -576,46 +550,47 @@ struct RefNode : public NodeBase { } template - Ref getNextRef(RegisterRef RR, Predicate P, bool NextOnly, - const DataFlowGraph &G); - Node getOwner(const DataFlowGraph &G); + NodeAddr getNextRef(RegisterRef RR, Predicate P, bool NextOnly, + const DataFlowGraph &G); + NodeAddr getOwner(const DataFlowGraph &G); }; struct DefNode : public RefNode { - NodeId getReachedDef() const { return RefData.Def.DD; } - void setReachedDef(NodeId D) { RefData.Def.DD = D; } - NodeId getReachedUse() const { return RefData.Def.DU; } - void setReachedUse(NodeId U) { RefData.Def.DU = U; } + NodeId getReachedDef() const { return Ref.Def.DD; } + void setReachedDef(NodeId D) { Ref.Def.DD = D; } + NodeId getReachedUse() const { return Ref.Def.DU; } + void setReachedUse(NodeId U) { Ref.Def.DU = U; } - void linkToDef(NodeId Self, Def DA); + void linkToDef(NodeId Self, NodeAddr DA); }; struct UseNode : public RefNode { - void linkToDef(NodeId Self, Def DA); + void linkToDef(NodeId Self, NodeAddr DA); }; struct PhiUseNode : public UseNode { NodeId getPredecessor() const { assert(getFlags() & NodeAttrs::PhiRef); - return RefData.PhiU.PredB; + return Ref.PhiU.PredB; } void setPredecessor(NodeId B) { assert(getFlags() & NodeAttrs::PhiRef); - RefData.PhiU.PredB = B; + Ref.PhiU.PredB = B; } }; struct CodeNode : public NodeBase { template T getCode() const { // - return static_cast(CodeData.CP); + return static_cast(Code.CP); } - void setCode(void *C) { CodeData.CP = C; } + void setCode(void *C) { Code.CP = C; } - Node getFirstMember(const DataFlowGraph &G) const; - Node getLastMember(const DataFlowGraph &G) const; - void addMember(Node NA, const DataFlowGraph &G); - void addMemberAfter(Node MA, Node NA, const DataFlowGraph &G); - void removeMember(Node NA, const DataFlowGraph &G); + NodeAddr getFirstMember(const DataFlowGraph &G) const; + NodeAddr getLastMember(const DataFlowGraph &G) const; + void addMember(NodeAddr NA, const DataFlowGraph &G); + void addMemberAfter(NodeAddr MA, NodeAddr NA, + const DataFlowGraph &G); + void removeMember(NodeAddr NA, const DataFlowGraph &G); NodeList members(const DataFlowGraph &G) const; template @@ -623,7 +598,7 @@ struct CodeNode : public NodeBase { }; struct InstrNode : public CodeNode { - Node getOwner(const DataFlowGraph &G); + NodeAddr getOwner(const DataFlowGraph &G); }; struct PhiNode : public InstrNode { @@ -641,7 +616,7 @@ struct BlockNode : public CodeNode { return CodeNode::getCode(); } - void addPhi(Phi PA, const DataFlowGraph &G); + void addPhi(NodeAddr PA, const DataFlowGraph &G); }; struct FuncNode : public CodeNode { @@ -649,8 +624,9 @@ struct FuncNode : public CodeNode { return CodeNode::getCode(); } - Block findBlock(const MachineBasicBlock *BB, const DataFlowGraph &G) const; - Block getEntryBlock(const DataFlowGraph &G); + NodeAddr findBlock(const MachineBasicBlock *BB, + const DataFlowGraph &G) const; + NodeAddr getEntryBlock(const DataFlowGraph &G); }; struct DataFlowGraph { @@ -673,7 +649,7 @@ struct DataFlowGraph { return {ptr(N), N}; } - Func getFunc() const { return TheFunc; } + NodeAddr getFunc() const { return Func; } MachineFunction &getMF() const { return MF; } const TargetInstrInfo &getTII() const { return TII; } const TargetRegisterInfo &getTRI() const { return TRI; } @@ -688,7 +664,7 @@ struct DataFlowGraph { bool empty() const { return Stack.empty() || top() == bottom(); } private: - using value_type = Def; + using value_type = NodeAddr; struct Iterator { using value_type = DefStack::value_type; @@ -730,7 +706,7 @@ struct DataFlowGraph { iterator bottom() const { return Iterator(*this, false); } unsigned size() const; - void push(Def DA) { Stack.push_back(DA); } + void push(NodeAddr DA) { Stack.push_back(DA); } void pop(); void start_block(NodeId N); void clear_block(NodeId N); @@ -755,7 +731,7 @@ struct DataFlowGraph { using DefStackMap = std::unordered_map; void build(unsigned Options = BuildOptions::None); - void pushAllDefs(Instr IA, DefStackMap &DM); + void pushAllDefs(NodeAddr IA, DefStackMap &DM); void markBlock(NodeId B, DefStackMap &DefM); void releaseBlock(NodeId B, DefStackMap &DefM); @@ -772,51 +748,57 @@ struct DataFlowGraph { RegisterRef makeRegRef(unsigned Reg, unsigned Sub) const; RegisterRef makeRegRef(const MachineOperand &Op) const; - Ref getNextRelated(Instr IA, Ref RA) const; - Ref getNextShadow(Instr IA, Ref RA, bool Create); - Ref getNextShadow(Instr IA, Ref RA) const; + NodeAddr getNextRelated(NodeAddr IA, + NodeAddr RA) const; + NodeAddr getNextShadow(NodeAddr IA, + NodeAddr RA, bool Create); + NodeAddr getNextShadow(NodeAddr IA, + NodeAddr RA) const; - NodeList getRelatedRefs(Instr IA, Ref RA) const; + NodeList getRelatedRefs(NodeAddr IA, + NodeAddr RA) const; - Block findBlock(MachineBasicBlock *BB) const { return BlockNodes.at(BB); } + NodeAddr findBlock(MachineBasicBlock *BB) const { + return BlockNodes.at(BB); + } - void unlinkUse(Use UA, bool RemoveFromOwner) { + void unlinkUse(NodeAddr UA, bool RemoveFromOwner) { unlinkUseDF(UA); if (RemoveFromOwner) removeFromOwner(UA); } - void unlinkDef(Def DA, bool RemoveFromOwner) { + void unlinkDef(NodeAddr DA, bool RemoveFromOwner) { unlinkDefDF(DA); if (RemoveFromOwner) removeFromOwner(DA); } // Some useful filters. - template static bool IsRef(const Node BA) { + template static bool IsRef(const NodeAddr BA) { return BA.Addr->getType() == NodeAttrs::Ref && BA.Addr->getKind() == Kind; } - template static bool IsCode(const Node BA) { + template static bool IsCode(const NodeAddr BA) { return BA.Addr->getType() == NodeAttrs::Code && BA.Addr->getKind() == Kind; } - static bool IsDef(const Node BA) { + static bool IsDef(const NodeAddr BA) { return BA.Addr->getType() == NodeAttrs::Ref && BA.Addr->getKind() == NodeAttrs::Def; } - static bool IsUse(const Node BA) { + static bool IsUse(const NodeAddr BA) { return BA.Addr->getType() == NodeAttrs::Ref && BA.Addr->getKind() == NodeAttrs::Use; } - static bool IsPhi(const Node BA) { + static bool IsPhi(const NodeAddr BA) { return BA.Addr->getType() == NodeAttrs::Code && BA.Addr->getKind() == NodeAttrs::Phi; } - static bool IsPreservingDef(const Def DA) { + static bool IsPreservingDef(const NodeAddr DA) { uint16_t Flags = DA.Addr->getFlags(); return (Flags & NodeAttrs::Preserving) && !(Flags & NodeAttrs::Undef); } @@ -826,40 +808,49 @@ struct DataFlowGraph { RegisterAggr getLandingPadLiveIns() const; - Node newNode(uint16_t Attrs); - Node cloneNode(const Node B); - Use newUse(Instr Owner, MachineOperand &Op, uint16_t Flags = NodeAttrs::None); - PhiUse newPhiUse(Phi Owner, RegisterRef RR, Block PredB, - uint16_t Flags = NodeAttrs::PhiRef); - Def newDef(Instr Owner, MachineOperand &Op, uint16_t Flags = NodeAttrs::None); - Def newDef(Instr Owner, RegisterRef RR, uint16_t Flags = NodeAttrs::PhiRef); - Phi newPhi(Block Owner); - Stmt newStmt(Block Owner, MachineInstr *MI); - Block newBlock(Func Owner, MachineBasicBlock *BB); - Func newFunc(MachineFunction *MF); + NodeAddr newNode(uint16_t Attrs); + NodeAddr cloneNode(const NodeAddr B); + NodeAddr newUse(NodeAddr Owner, MachineOperand &Op, + uint16_t Flags = NodeAttrs::None); + NodeAddr newPhiUse(NodeAddr Owner, RegisterRef RR, + NodeAddr PredB, + uint16_t Flags = NodeAttrs::PhiRef); + NodeAddr newDef(NodeAddr Owner, MachineOperand &Op, + uint16_t Flags = NodeAttrs::None); + NodeAddr newDef(NodeAddr Owner, RegisterRef RR, + uint16_t Flags = NodeAttrs::PhiRef); + NodeAddr newPhi(NodeAddr Owner); + NodeAddr newStmt(NodeAddr Owner, MachineInstr *MI); + NodeAddr newBlock(NodeAddr Owner, + MachineBasicBlock *BB); + NodeAddr newFunc(MachineFunction *MF); template - std::pair locateNextRef(Instr IA, Ref RA, Predicate P) const; + std::pair, NodeAddr> + locateNextRef(NodeAddr IA, NodeAddr RA, + Predicate P) const; using BlockRefsMap = RegisterAggrMap; - void buildStmt(Block BA, MachineInstr &In); - void recordDefsForDF(BlockRefsMap &PhiM, Block BA); - void buildPhis(BlockRefsMap &PhiM, RegisterSet &AllRefs, Block BA); + void buildStmt(NodeAddr BA, MachineInstr &In); + void recordDefsForDF(BlockRefsMap &PhiM, NodeAddr BA); + void buildPhis(BlockRefsMap &PhiM, RegisterSet &AllRefs, + NodeAddr BA); void removeUnusedPhis(); - void pushClobbers(Instr IA, DefStackMap &DM); - void pushDefs(Instr IA, DefStackMap &DM); - template void linkRefUp(Instr IA, NodeAddr TA, DefStack &DS); + void pushClobbers(NodeAddr IA, DefStackMap &DM); + void pushDefs(NodeAddr IA, DefStackMap &DM); + template + void linkRefUp(NodeAddr IA, NodeAddr TA, DefStack &DS); template - void linkStmtRefs(DefStackMap &DefM, Stmt SA, Predicate P); - void linkBlockRefs(DefStackMap &DefM, Block BA); + void linkStmtRefs(DefStackMap &DefM, NodeAddr SA, Predicate P); + void linkBlockRefs(DefStackMap &DefM, NodeAddr BA); - void unlinkUseDF(Use UA); - void unlinkDefDF(Def DA); + void unlinkUseDF(NodeAddr UA); + void unlinkDefDF(NodeAddr DA); - void removeFromOwner(Ref RA) { - Instr IA = RA.Addr->getOwner(*this); + void removeFromOwner(NodeAddr RA) { + NodeAddr IA = RA.Addr->getOwner(*this); IA.Addr->removeMember(RA, *this); } @@ -875,25 +866,25 @@ struct DataFlowGraph { const TargetOperandInfo &TOI; RegisterAggr LiveIns; - Func TheFunc; + NodeAddr Func; NodeAllocator Memory; // Local map: MachineBasicBlock -> NodeAddr - std::map BlockNodes; + std::map> BlockNodes; // Lane mask map. LaneMaskIndex LMI; }; // struct DataFlowGraph template -Ref RefNode::getNextRef(RegisterRef RR, Predicate P, bool NextOnly, - const DataFlowGraph &G) { +NodeAddr RefNode::getNextRef(RegisterRef RR, Predicate P, + bool NextOnly, const DataFlowGraph &G) { // Get the "Next" reference in the circular list that references RR and // satisfies predicate "Pred". auto NA = G.addr(getNext()); while (NA.Addr != this) { if (NA.Addr->getType() == NodeAttrs::Ref) { - Ref RA = NA; - if (G.getPRI().equal_to(RA.Addr->getRegRef(G), RR) && P(NA)) + NodeAddr RA = NA; + if (RA.Addr->getRegRef(G) == RR && P(NA)) return NA; if (NextOnly) break; @@ -901,20 +892,12 @@ Ref RefNode::getNextRef(RegisterRef RR, Predicate P, bool NextOnly, } else { // We've hit the beginning of the chain. assert(NA.Addr->getType() == NodeAttrs::Code); - // Make sure we stop here with NextOnly. Otherwise we can return the - // wrong ref. Consider the following while creating/linking shadow uses: - // -> code -> sr1 -> sr2 -> [back to code] - // Say that shadow refs sr1, and sr2 have been linked, but we need to - // create and link another one. Starting from sr2, we'd hit the code - // node and return sr1 if the iteration didn't stop here. - if (NextOnly) - break; - Code CA = NA; + NodeAddr CA = NA; NA = CA.Addr->getFirstMember(G); } } // Return the equivalent of "nullptr" if such a node was not found. - return Ref(); + return NodeAddr(); } template @@ -948,17 +931,18 @@ template struct PrintNode : Print> { raw_ostream &operator<<(raw_ostream &OS, const Print &P); raw_ostream &operator<<(raw_ostream &OS, const Print &P); -raw_ostream &operator<<(raw_ostream &OS, const Print &P); -raw_ostream &operator<<(raw_ostream &OS, const Print &P); -raw_ostream &operator<<(raw_ostream &OS, const Print &P); -raw_ostream &operator<<(raw_ostream &OS, const Print &P); +raw_ostream &operator<<(raw_ostream &OS, const Print> &P); +raw_ostream &operator<<(raw_ostream &OS, const Print> &P); +raw_ostream &operator<<(raw_ostream &OS, + const Print> &P); +raw_ostream &operator<<(raw_ostream &OS, const Print> &P); raw_ostream &operator<<(raw_ostream &OS, const Print &P); raw_ostream &operator<<(raw_ostream &OS, const Print &P); -raw_ostream &operator<<(raw_ostream &OS, const Print &P); -raw_ostream &operator<<(raw_ostream &OS, const Print &P); -raw_ostream &operator<<(raw_ostream &OS, const Print &P); -raw_ostream &operator<<(raw_ostream &OS, const Print &P); -raw_ostream &operator<<(raw_ostream &OS, const Print &P); +raw_ostream &operator<<(raw_ostream &OS, const Print> &P); +raw_ostream &operator<<(raw_ostream &OS, const Print> &P); +raw_ostream &operator<<(raw_ostream &OS, const Print> &P); +raw_ostream &operator<<(raw_ostream &OS, const Print> &P); +raw_ostream &operator<<(raw_ostream &OS, const Print> &P); raw_ostream &operator<<(raw_ostream &OS, const Print &P); raw_ostream &operator<<(raw_ostream &OS, const Print &P); raw_ostream &operator<<(raw_ostream &OS, diff --git a/llvm/include/llvm/CodeGen/RDFRegisters.h b/llvm/include/llvm/CodeGen/RDFRegisters.h index f9126104ad70f..86c00aeb47666 100644 --- a/llvm/include/llvm/CodeGen/RDFRegisters.h +++ b/llvm/include/llvm/CodeGen/RDFRegisters.h @@ -14,7 +14,6 @@ #include "llvm/ADT/iterator_range.h" #include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/MC/LaneBitmask.h" -#include "llvm/MC/MCRegister.h" #include #include #include @@ -27,7 +26,6 @@ class MachineFunction; class raw_ostream; namespace rdf { -struct RegisterAggr; using RegisterId = uint32_t; @@ -72,51 +70,36 @@ template struct IndexedSet { struct RegisterRef { RegisterId Reg = 0; - LaneBitmask Mask = LaneBitmask::getNone(); // Only for registers. + LaneBitmask Mask = LaneBitmask::getNone(); - constexpr RegisterRef() = default; - constexpr explicit RegisterRef(RegisterId R, - LaneBitmask M = LaneBitmask::getAll()) - : Reg(R), Mask(isRegId(R) && R != 0 ? M : LaneBitmask::getNone()) {} + RegisterRef() = default; + explicit RegisterRef(RegisterId R, LaneBitmask M = LaneBitmask::getAll()) + : Reg(R), Mask(R != 0 ? M : LaneBitmask::getNone()) {} - // Classify null register as a "register". - constexpr bool isReg() const { return Reg == 0 || isRegId(Reg); } - constexpr bool isUnit() const { return isUnitId(Reg); } - constexpr bool isMask() const { return isMaskId(Reg); } + operator bool() const { return Reg != 0 && Mask.any(); } - constexpr operator bool() const { - return !isReg() || (Reg != 0 && Mask.any()); + bool operator==(const RegisterRef &RR) const { + return Reg == RR.Reg && Mask == RR.Mask; } - size_t hash() const { - return std::hash{}(Reg) ^ - std::hash{}(Mask.getAsInteger()); - } + bool operator!=(const RegisterRef &RR) const { return !operator==(RR); } - static constexpr bool isRegId(unsigned Id) { - return Register::isPhysicalRegister(Id); - } - static constexpr bool isUnitId(unsigned Id) { - return Register::isVirtualRegister(Id); - } - static constexpr bool isMaskId(unsigned Id) { - return Register::isStackSlot(Id); + bool operator<(const RegisterRef &RR) const { + return Reg < RR.Reg || (Reg == RR.Reg && Mask < RR.Mask); } - static RegisterId toUnitId(unsigned Idx) { - return Register::index2VirtReg(Idx); + size_t hash() const { + return std::hash{}(Reg) ^ + std::hash{}(Mask.getAsInteger()); } - static unsigned toRegUnit(RegisterId U) { return Register::virtReg2Index(U); } - - bool operator<(RegisterRef) const = delete; - bool operator==(RegisterRef) const = delete; - bool operator!=(RegisterRef) const = delete; }; struct PhysicalRegisterInfo { PhysicalRegisterInfo(const TargetRegisterInfo &tri, const MachineFunction &mf); + static bool isRegMaskId(RegisterId R) { return Register::isStackSlot(R); } + RegisterId getRegMaskId(const uint32_t *RM) const { return Register::index2StackSlot(RegMasks.find(RM)); } @@ -126,13 +109,11 @@ struct PhysicalRegisterInfo { } bool alias(RegisterRef RA, RegisterRef RB) const { - if (!RA.isMask()) - return !RB.isMask() ? aliasRR(RA, RB) : aliasRM(RA, RB); - return !RB.isMask() ? aliasRM(RB, RA) : aliasMM(RA, RB); + if (!isRegMaskId(RA.Reg)) + return !isRegMaskId(RB.Reg) ? aliasRR(RA, RB) : aliasRM(RA, RB); + return !isRegMaskId(RB.Reg) ? aliasRM(RB, RA) : aliasMM(RA, RB); } - // Returns the set of aliased physical registers or register masks. - // The returned set does not contain register units. std::set getAliasSet(RegisterId Reg) const; RegisterRef getRefForUnit(uint32_t U) const { @@ -150,12 +131,6 @@ struct PhysicalRegisterInfo { RegisterRef mapTo(RegisterRef RR, unsigned R) const; const TargetRegisterInfo &getTRI() const { return TRI; } - bool equal_to(RegisterRef A, RegisterRef B) const; - bool less(RegisterRef A, RegisterRef B) const; - - void print(raw_ostream &OS, RegisterRef A) const; - void print(raw_ostream &OS, const RegisterAggr &A) const; - private: struct RegInfo { const TargetRegisterClass *RegClass = nullptr; @@ -193,8 +168,6 @@ struct RegisterAggr { bool hasAliasOf(RegisterRef RR) const; bool hasCoverOf(RegisterRef RR) const; - const PhysicalRegisterInfo &getPRI() const { return PRI; } - bool operator==(const RegisterAggr &A) const { return DenseMapInfo::isEqual(Units, A.Units); } @@ -217,7 +190,9 @@ struct RegisterAggr { size_t hash() const { return DenseMapInfo::getHashValue(Units); } - struct ref_iterator { + void print(raw_ostream &OS) const; + + struct rr_iterator { using MapType = std::map; private: @@ -227,39 +202,35 @@ struct RegisterAggr { const RegisterAggr *Owner; public: - ref_iterator(const RegisterAggr &RG, bool End); + rr_iterator(const RegisterAggr &RG, bool End); RegisterRef operator*() const { return RegisterRef(Pos->first, Pos->second); } - ref_iterator &operator++() { + rr_iterator &operator++() { ++Pos; ++Index; return *this; } - bool operator==(const ref_iterator &I) const { + bool operator==(const rr_iterator &I) const { assert(Owner == I.Owner); (void)Owner; return Index == I.Index; } - bool operator!=(const ref_iterator &I) const { return !(*this == I); } + bool operator!=(const rr_iterator &I) const { return !(*this == I); } }; - ref_iterator ref_begin() const { return ref_iterator(*this, false); } - ref_iterator ref_end() const { return ref_iterator(*this, true); } + rr_iterator rr_begin() const { return rr_iterator(*this, false); } + rr_iterator rr_end() const { return rr_iterator(*this, true); } - using unit_iterator = typename BitVector::const_set_bits_iterator; - unit_iterator unit_begin() const { return Units.set_bits_begin(); } - unit_iterator unit_end() const { return Units.set_bits_end(); } - - iterator_range refs() const { - return make_range(ref_begin(), ref_end()); + iterator_range refs() { + return make_range(rr_begin(), rr_end()); } - iterator_range units() const { - return make_range(unit_begin(), unit_end()); + iterator_range refs() const { + return make_range(rr_begin(), rr_end()); } private: @@ -292,65 +263,34 @@ template struct RegisterAggrMap { using value_type = typename decltype(Map)::value_type; }; -raw_ostream &operator<<(raw_ostream &OS, const RegisterAggr &A); - -// Print the lane mask in a short form (or not at all if all bits are set). -struct PrintLaneMaskShort { - PrintLaneMaskShort(LaneBitmask M) : Mask(M) {} +// Optionally print the lane mask, if it is not ~0. +struct PrintLaneMaskOpt { + PrintLaneMaskOpt(LaneBitmask M) : Mask(M) {} LaneBitmask Mask; }; -raw_ostream &operator<<(raw_ostream &OS, const PrintLaneMaskShort &P); +raw_ostream &operator<<(raw_ostream &OS, const PrintLaneMaskOpt &P); +raw_ostream &operator<<(raw_ostream &OS, const RegisterAggr &A); } // end namespace rdf + } // end namespace llvm namespace std { - template <> struct hash { size_t operator()(llvm::rdf::RegisterRef A) const { // return A.hash(); } }; - template <> struct hash { size_t operator()(const llvm::rdf::RegisterAggr &A) const { // return A.hash(); } }; - -template <> struct equal_to { - constexpr equal_to(const llvm::rdf::PhysicalRegisterInfo &pri) : PRI(pri) {} - - bool operator()(llvm::rdf::RegisterRef A, llvm::rdf::RegisterRef B) const { - return PRI.equal_to(A, B); - } - -private: - const llvm::rdf::PhysicalRegisterInfo &PRI; -}; - template <> struct equal_to { bool operator()(const llvm::rdf::RegisterAggr &A, const llvm::rdf::RegisterAggr &B) const { return A == B; } }; - -template <> struct less { - constexpr less(const llvm::rdf::PhysicalRegisterInfo &pri) : PRI(pri) {} - - bool operator()(llvm::rdf::RegisterRef A, llvm::rdf::RegisterRef B) const { - return PRI.less(A, B); - } - -private: - const llvm::rdf::PhysicalRegisterInfo &PRI; -}; - } // namespace std - -namespace llvm::rdf { -using RegisterSet = std::set>; -} // namespace llvm::rdf - #endif // LLVM_CODEGEN_RDFREGISTERS_H diff --git a/llvm/lib/CodeGen/RDFGraph.cpp b/llvm/lib/CodeGen/RDFGraph.cpp index 382dcbe9b7495..db2589cc2d18e 100644 --- a/llvm/lib/CodeGen/RDFGraph.cpp +++ b/llvm/lib/CodeGen/RDFGraph.cpp @@ -46,8 +46,19 @@ using namespace rdf; namespace llvm { namespace rdf { +raw_ostream &operator<<(raw_ostream &OS, const PrintLaneMaskOpt &P) { + if (!P.Mask.all()) + OS << ':' << PrintLaneMask(P.Mask); + return OS; +} + raw_ostream &operator<<(raw_ostream &OS, const Print &P) { - P.G.getPRI().print(OS, P.Obj); + auto &TRI = P.G.getTRI(); + if (P.Obj.Reg > 0 && P.Obj.Reg < TRI.getNumRegs()) + OS << TRI.getName(P.Obj.Reg); + else + OS << '#' << P.Obj.Reg; + OS << PrintLaneMaskOpt(P.Obj.Mask); return OS; } @@ -110,14 +121,14 @@ raw_ostream &operator<<(raw_ostream &OS, const Print &P) { return OS; } -static void printRefHeader(raw_ostream &OS, const Ref RA, +static void printRefHeader(raw_ostream &OS, const NodeAddr RA, const DataFlowGraph &G) { OS << Print(RA.Id, G) << '<' << Print(RA.Addr->getRegRef(G), G) << '>'; if (RA.Addr->getFlags() & NodeAttrs::Fixed) OS << '!'; } -raw_ostream &operator<<(raw_ostream &OS, const Print &P) { +raw_ostream &operator<<(raw_ostream &OS, const Print> &P) { printRefHeader(OS, P.Obj, P.G); OS << '('; if (NodeId N = P.Obj.Addr->getReachingDef()) @@ -134,7 +145,7 @@ raw_ostream &operator<<(raw_ostream &OS, const Print &P) { return OS; } -raw_ostream &operator<<(raw_ostream &OS, const Print &P) { +raw_ostream &operator<<(raw_ostream &OS, const Print> &P) { printRefHeader(OS, P.Obj, P.G); OS << '('; if (NodeId N = P.Obj.Addr->getReachingDef()) @@ -145,7 +156,8 @@ raw_ostream &operator<<(raw_ostream &OS, const Print &P) { return OS; } -raw_ostream &operator<<(raw_ostream &OS, const Print &P) { +raw_ostream &operator<<(raw_ostream &OS, + const Print> &P) { printRefHeader(OS, P.Obj, P.G); OS << '('; if (NodeId N = P.Obj.Addr->getReachingDef()) @@ -159,7 +171,7 @@ raw_ostream &operator<<(raw_ostream &OS, const Print &P) { return OS; } -raw_ostream &operator<<(raw_ostream &OS, const Print &P) { +raw_ostream &operator<<(raw_ostream &OS, const Print> &P) { switch (P.Obj.Addr->getKind()) { case NodeAttrs::Def: OS << PrintNode(P.Obj, P.G); @@ -217,13 +229,13 @@ raw_ostream &operator<<(raw_ostream &OS, const PrintListV &P) { } // end anonymous namespace -raw_ostream &operator<<(raw_ostream &OS, const Print &P) { +raw_ostream &operator<<(raw_ostream &OS, const Print> &P) { OS << Print(P.Obj.Id, P.G) << ": phi [" << PrintListV(P.Obj.Addr->members(P.G), P.G) << ']'; return OS; } -raw_ostream &operator<<(raw_ostream &OS, const Print &P) { +raw_ostream &operator<<(raw_ostream &OS, const Print> &P) { const MachineInstr &MI = *P.Obj.Addr->getCode(); unsigned Opc = MI.getOpcode(); OS << Print(P.Obj.Id, P.G) << ": " << P.G.getTII().getName(Opc); @@ -247,7 +259,8 @@ raw_ostream &operator<<(raw_ostream &OS, const Print &P) { return OS; } -raw_ostream &operator<<(raw_ostream &OS, const Print &P) { +raw_ostream &operator<<(raw_ostream &OS, + const Print> &P) { switch (P.Obj.Addr->getKind()) { case NodeAttrs::Phi: OS << PrintNode(P.Obj, P.G); @@ -262,7 +275,8 @@ raw_ostream &operator<<(raw_ostream &OS, const Print &P) { return OS; } -raw_ostream &operator<<(raw_ostream &OS, const Print &P) { +raw_ostream &operator<<(raw_ostream &OS, + const Print> &P) { MachineBasicBlock *BB = P.Obj.Addr->getCode(); unsigned NP = BB->pred_size(); std::vector Ns; @@ -294,7 +308,7 @@ raw_ostream &operator<<(raw_ostream &OS, const Print &P) { return OS; } -raw_ostream &operator<<(raw_ostream &OS, const Print &P) { +raw_ostream &operator<<(raw_ostream &OS, const Print> &P) { OS << "DFG dump:[\n" << Print(P.Obj.Id, P.G) << ": Function: " << P.Obj.Addr->getCode()->getName() << '\n'; @@ -313,7 +327,7 @@ raw_ostream &operator<<(raw_ostream &OS, const Print &P) { } raw_ostream &operator<<(raw_ostream &OS, const Print &P) { - OS << P.Obj; + P.Obj.print(OS); return OS; } @@ -362,13 +376,14 @@ bool NodeAllocator::needNewBlock() { return Index >= NodesPerBlock; } -Node NodeAllocator::New() { +NodeAddr NodeAllocator::New() { if (needNewBlock()) startNewBlock(); uint32_t ActiveB = Blocks.size() - 1; uint32_t Index = (ActiveEnd - Blocks[ActiveB]) / NodeMemSize; - Node NA = {reinterpret_cast(ActiveEnd), makeId(ActiveB, Index)}; + NodeAddr NA = {reinterpret_cast(ActiveEnd), + makeId(ActiveB, Index)}; ActiveEnd += NodeMemSize; return NA; } @@ -392,7 +407,7 @@ void NodeAllocator::clear() { } // Insert node NA after "this" in the circular chain. -void NodeBase::append(Node NA) { +void NodeBase::append(NodeAddr NA) { NodeId Nx = Next; // If NA is already "next", do nothing. if (Next != NA.Id) { @@ -407,9 +422,9 @@ void NodeBase::append(Node NA) { RegisterRef RefNode::getRegRef(const DataFlowGraph &G) const { assert(NodeAttrs::type(Attrs) == NodeAttrs::Ref); if (NodeAttrs::flags(Attrs) & NodeAttrs::PhiRef) - return G.unpack(RefData.PR); - assert(RefData.Op != nullptr); - return G.makeRegRef(*RefData.Op); + return G.unpack(Ref.PR); + assert(Ref.Op != nullptr); + return G.makeRegRef(*Ref.Op); } // Set the register reference in the reference node directly (for references @@ -417,7 +432,7 @@ RegisterRef RefNode::getRegRef(const DataFlowGraph &G) const { void RefNode::setRegRef(RegisterRef RR, DataFlowGraph &G) { assert(NodeAttrs::type(Attrs) == NodeAttrs::Ref); assert(NodeAttrs::flags(Attrs) & NodeAttrs::PhiRef); - RefData.PR = G.pack(RR); + Ref.PR = G.pack(RR); } // Set the register reference in the reference node based on a machine @@ -426,12 +441,12 @@ void RefNode::setRegRef(MachineOperand *Op, DataFlowGraph &G) { assert(NodeAttrs::type(Attrs) == NodeAttrs::Ref); assert(!(NodeAttrs::flags(Attrs) & NodeAttrs::PhiRef)); (void)G; - RefData.Op = Op; + Ref.Op = Op; } // Get the owner of a given reference node. -Node RefNode::getOwner(const DataFlowGraph &G) { - Node NA = G.addr(getNext()); +NodeAddr RefNode::getOwner(const DataFlowGraph &G) { + NodeAddr NA = G.addr(getNext()); while (NA.Addr != this) { if (NA.Addr->getType() == NodeAttrs::Code) @@ -442,66 +457,67 @@ Node RefNode::getOwner(const DataFlowGraph &G) { } // Connect the def node to the reaching def node. -void DefNode::linkToDef(NodeId Self, Def DA) { - RefData.RD = DA.Id; - RefData.Sib = DA.Addr->getReachedDef(); +void DefNode::linkToDef(NodeId Self, NodeAddr DA) { + Ref.RD = DA.Id; + Ref.Sib = DA.Addr->getReachedDef(); DA.Addr->setReachedDef(Self); } // Connect the use node to the reaching def node. -void UseNode::linkToDef(NodeId Self, Def DA) { - RefData.RD = DA.Id; - RefData.Sib = DA.Addr->getReachedUse(); +void UseNode::linkToDef(NodeId Self, NodeAddr DA) { + Ref.RD = DA.Id; + Ref.Sib = DA.Addr->getReachedUse(); DA.Addr->setReachedUse(Self); } // Get the first member of the code node. -Node CodeNode::getFirstMember(const DataFlowGraph &G) const { - if (CodeData.FirstM == 0) - return Node(); - return G.addr(CodeData.FirstM); +NodeAddr CodeNode::getFirstMember(const DataFlowGraph &G) const { + if (Code.FirstM == 0) + return NodeAddr(); + return G.addr(Code.FirstM); } // Get the last member of the code node. -Node CodeNode::getLastMember(const DataFlowGraph &G) const { - if (CodeData.LastM == 0) - return Node(); - return G.addr(CodeData.LastM); +NodeAddr CodeNode::getLastMember(const DataFlowGraph &G) const { + if (Code.LastM == 0) + return NodeAddr(); + return G.addr(Code.LastM); } // Add node NA at the end of the member list of the given code node. -void CodeNode::addMember(Node NA, const DataFlowGraph &G) { - Node ML = getLastMember(G); +void CodeNode::addMember(NodeAddr NA, const DataFlowGraph &G) { + NodeAddr ML = getLastMember(G); if (ML.Id != 0) { ML.Addr->append(NA); } else { - CodeData.FirstM = NA.Id; + Code.FirstM = NA.Id; NodeId Self = G.id(this); NA.Addr->setNext(Self); } - CodeData.LastM = NA.Id; + Code.LastM = NA.Id; } // Add node NA after member node MA in the given code node. -void CodeNode::addMemberAfter(Node MA, Node NA, const DataFlowGraph &G) { +void CodeNode::addMemberAfter(NodeAddr MA, NodeAddr NA, + const DataFlowGraph &G) { MA.Addr->append(NA); - if (CodeData.LastM == MA.Id) - CodeData.LastM = NA.Id; + if (Code.LastM == MA.Id) + Code.LastM = NA.Id; } // Remove member node NA from the given code node. -void CodeNode::removeMember(Node NA, const DataFlowGraph &G) { - Node MA = getFirstMember(G); +void CodeNode::removeMember(NodeAddr NA, const DataFlowGraph &G) { + NodeAddr MA = getFirstMember(G); assert(MA.Id != 0); // Special handling if the member to remove is the first member. if (MA.Id == NA.Id) { - if (CodeData.LastM == MA.Id) { + if (Code.LastM == MA.Id) { // If it is the only member, set both first and last to 0. - CodeData.FirstM = CodeData.LastM = 0; + Code.FirstM = Code.LastM = 0; } else { // Otherwise, advance the first member. - CodeData.FirstM = MA.Addr->getNext(); + Code.FirstM = MA.Addr->getNext(); } return; } @@ -512,8 +528,8 @@ void CodeNode::removeMember(Node NA, const DataFlowGraph &G) { MA.Addr->setNext(NA.Addr->getNext()); // If the member to remove happens to be the last one, update the // LastM indicator. - if (CodeData.LastM == NA.Id) - CodeData.LastM = MA.Id; + if (Code.LastM == NA.Id) + Code.LastM = MA.Id; return; } MA = G.addr(MX); @@ -523,13 +539,13 @@ void CodeNode::removeMember(Node NA, const DataFlowGraph &G) { // Return the list of all members of the code node. NodeList CodeNode::members(const DataFlowGraph &G) const { - static auto True = [](Node) -> bool { return true; }; + static auto True = [](NodeAddr) -> bool { return true; }; return members_if(True, G); } // Return the owner of the given instr node. -Node InstrNode::getOwner(const DataFlowGraph &G) { - Node NA = G.addr(getNext()); +NodeAddr InstrNode::getOwner(const DataFlowGraph &G) { + NodeAddr NA = G.addr(getNext()); while (NA.Addr != this) { assert(NA.Addr->getType() == NodeAttrs::Code); @@ -541,8 +557,8 @@ Node InstrNode::getOwner(const DataFlowGraph &G) { } // Add the phi node PA to the given block node. -void BlockNode::addPhi(Phi PA, const DataFlowGraph &G) { - Node M = getFirstMember(G); +void BlockNode::addPhi(NodeAddr PA, const DataFlowGraph &G) { + NodeAddr M = getFirstMember(G); if (M.Id == 0) { addMember(PA, G); return; @@ -552,12 +568,12 @@ void BlockNode::addPhi(Phi PA, const DataFlowGraph &G) { if (M.Addr->getKind() == NodeAttrs::Stmt) { // If the first member of the block is a statement, insert the phi as // the first member. - CodeData.FirstM = PA.Id; + Code.FirstM = PA.Id; PA.Addr->setNext(M.Id); } else { // If the first member is a phi, find the last phi, and append PA to it. assert(M.Addr->getKind() == NodeAttrs::Phi); - Node MN = M; + NodeAddr MN = M; do { M = MN; MN = G.addr(M.Addr->getNext()); @@ -571,17 +587,19 @@ void BlockNode::addPhi(Phi PA, const DataFlowGraph &G) { // Find the block node corresponding to the machine basic block BB in the // given func node. -Block FuncNode::findBlock(const MachineBasicBlock *BB, - const DataFlowGraph &G) const { - auto EqBB = [BB](Node NA) -> bool { return Block(NA).Addr->getCode() == BB; }; +NodeAddr FuncNode::findBlock(const MachineBasicBlock *BB, + const DataFlowGraph &G) const { + auto EqBB = [BB](NodeAddr NA) -> bool { + return NodeAddr(NA).Addr->getCode() == BB; + }; NodeList Ms = members_if(EqBB, G); if (!Ms.empty()) return Ms[0]; - return Block(); + return NodeAddr(); } // Get the block node for the entry block in the given function. -Block FuncNode::getEntryBlock(const DataFlowGraph &G) { +NodeAddr FuncNode::getEntryBlock(const DataFlowGraph &G) { MachineBasicBlock *EntryB = &getCode()->front(); return findBlock(EntryB, G); } @@ -695,7 +713,7 @@ void DataFlowGraph::DefStack::pop() { // Push a delimiter for block node N on the stack. void DataFlowGraph::DefStack::start_block(NodeId N) { assert(N != 0); - Stack.push_back(Def(nullptr, N)); + Stack.push_back(NodeAddr(nullptr, N)); } // Remove all nodes from the top of the stack, until the delimited for @@ -777,8 +795,8 @@ NodeId DataFlowGraph::id(const NodeBase *P) const { } // Allocate a new node and set the attributes to Attrs. -Node DataFlowGraph::newNode(uint16_t Attrs) { - Node P = Memory.New(); +NodeAddr DataFlowGraph::newNode(uint16_t Attrs) { + NodeAddr P = Memory.New(); P.Addr->init(); P.Addr->setAttrs(Attrs); return P; @@ -786,16 +804,16 @@ Node DataFlowGraph::newNode(uint16_t Attrs) { // Make a copy of the given node B, except for the data-flow links, which // are set to 0. -Node DataFlowGraph::cloneNode(const Node B) { - Node NA = newNode(0); +NodeAddr DataFlowGraph::cloneNode(const NodeAddr B) { + NodeAddr NA = newNode(0); memcpy(NA.Addr, B.Addr, sizeof(NodeBase)); // Ref nodes need to have the data-flow links reset. if (NA.Addr->getType() == NodeAttrs::Ref) { - Ref RA = NA; + NodeAddr RA = NA; RA.Addr->setReachingDef(0); RA.Addr->setSibling(0); if (NA.Addr->getKind() == NodeAttrs::Def) { - Def DA = NA; + NodeAddr DA = NA; DA.Addr->setReachedDef(0); DA.Addr->setReachedUse(0); } @@ -805,57 +823,63 @@ Node DataFlowGraph::cloneNode(const Node B) { // Allocation routines for specific node types/kinds. -rdf::Use DataFlowGraph::newUse(Instr Owner, MachineOperand &Op, - uint16_t Flags) { - Use UA = newNode(NodeAttrs::Ref | NodeAttrs::Use | Flags); +NodeAddr DataFlowGraph::newUse(NodeAddr Owner, + MachineOperand &Op, uint16_t Flags) { + NodeAddr UA = newNode(NodeAttrs::Ref | NodeAttrs::Use | Flags); UA.Addr->setRegRef(&Op, *this); return UA; } -PhiUse DataFlowGraph::newPhiUse(Phi Owner, RegisterRef RR, Block PredB, - uint16_t Flags) { - PhiUse PUA = newNode(NodeAttrs::Ref | NodeAttrs::Use | Flags); +NodeAddr DataFlowGraph::newPhiUse(NodeAddr Owner, + RegisterRef RR, + NodeAddr PredB, + uint16_t Flags) { + NodeAddr PUA = newNode(NodeAttrs::Ref | NodeAttrs::Use | Flags); assert(Flags & NodeAttrs::PhiRef); PUA.Addr->setRegRef(RR, *this); PUA.Addr->setPredecessor(PredB.Id); return PUA; } -Def DataFlowGraph::newDef(Instr Owner, MachineOperand &Op, uint16_t Flags) { - Def DA = newNode(NodeAttrs::Ref | NodeAttrs::Def | Flags); +NodeAddr DataFlowGraph::newDef(NodeAddr Owner, + MachineOperand &Op, uint16_t Flags) { + NodeAddr DA = newNode(NodeAttrs::Ref | NodeAttrs::Def | Flags); DA.Addr->setRegRef(&Op, *this); return DA; } -Def DataFlowGraph::newDef(Instr Owner, RegisterRef RR, uint16_t Flags) { - Def DA = newNode(NodeAttrs::Ref | NodeAttrs::Def | Flags); +NodeAddr DataFlowGraph::newDef(NodeAddr Owner, + RegisterRef RR, uint16_t Flags) { + NodeAddr DA = newNode(NodeAttrs::Ref | NodeAttrs::Def | Flags); assert(Flags & NodeAttrs::PhiRef); DA.Addr->setRegRef(RR, *this); return DA; } -Phi DataFlowGraph::newPhi(Block Owner) { - Phi PA = newNode(NodeAttrs::Code | NodeAttrs::Phi); +NodeAddr DataFlowGraph::newPhi(NodeAddr Owner) { + NodeAddr PA = newNode(NodeAttrs::Code | NodeAttrs::Phi); Owner.Addr->addPhi(PA, *this); return PA; } -Stmt DataFlowGraph::newStmt(Block Owner, MachineInstr *MI) { - Stmt SA = newNode(NodeAttrs::Code | NodeAttrs::Stmt); +NodeAddr DataFlowGraph::newStmt(NodeAddr Owner, + MachineInstr *MI) { + NodeAddr SA = newNode(NodeAttrs::Code | NodeAttrs::Stmt); SA.Addr->setCode(MI); Owner.Addr->addMember(SA, *this); return SA; } -Block DataFlowGraph::newBlock(Func Owner, MachineBasicBlock *BB) { - Block BA = newNode(NodeAttrs::Code | NodeAttrs::Block); +NodeAddr DataFlowGraph::newBlock(NodeAddr Owner, + MachineBasicBlock *BB) { + NodeAddr BA = newNode(NodeAttrs::Code | NodeAttrs::Block); BA.Addr->setCode(BB); Owner.Addr->addMember(BA, *this); return BA; } -Func DataFlowGraph::newFunc(MachineFunction *MF) { - Func FA = newNode(NodeAttrs::Code | NodeAttrs::Func); +NodeAddr DataFlowGraph::newFunc(MachineFunction *MF) { + NodeAddr FA = newNode(NodeAttrs::Code | NodeAttrs::Func); FA.Addr->setCode(MF); return FA; } @@ -863,13 +887,13 @@ Func DataFlowGraph::newFunc(MachineFunction *MF) { // Build the data flow graph. void DataFlowGraph::build(unsigned Options) { reset(); - TheFunc = newFunc(&MF); + Func = newFunc(&MF); if (MF.empty()) return; for (MachineBasicBlock &B : MF) { - Block BA = newBlock(TheFunc, &B); + NodeAddr BA = newBlock(Func, &B); BlockNodes.insert(std::make_pair(&B, BA)); for (MachineInstr &I : B) { if (I.isDebugInstr()) @@ -878,14 +902,14 @@ void DataFlowGraph::build(unsigned Options) { } } - Block EA = TheFunc.Addr->getEntryBlock(*this); - NodeList Blocks = TheFunc.Addr->members(*this); + NodeAddr EA = Func.Addr->getEntryBlock(*this); + NodeList Blocks = Func.Addr->members(*this); // Collect information about block references. - RegisterSet AllRefs(getPRI()); - for (Block BA : Blocks) - for (Instr IA : BA.Addr->members(*this)) - for (Ref RA : IA.Addr->members(*this)) + RegisterSet AllRefs; + for (NodeAddr BA : Blocks) + for (NodeAddr IA : BA.Addr->members(*this)) + for (NodeAddr RA : IA.Addr->members(*this)) AllRefs.insert(RA.Addr->getRegRef(*this)); // Collect function live-ins and entry block live-ins. @@ -901,9 +925,9 @@ void DataFlowGraph::build(unsigned Options) { // Add function-entry phi nodes for the live-in registers. for (RegisterRef RR : LiveIns.refs()) { - Phi PA = newPhi(EA); + NodeAddr PA = newPhi(EA); uint16_t PhiFlags = NodeAttrs::PhiRef | NodeAttrs::Preserving; - Def DA = newDef(PA, RR, PhiFlags); + NodeAddr DA = newDef(PA, RR, PhiFlags); PA.Addr->addMember(DA, *this); } @@ -914,7 +938,7 @@ void DataFlowGraph::build(unsigned Options) { // may define certain registers as defined on entry to such a block. RegisterAggr EHRegs = getLandingPadLiveIns(); if (!EHRegs.empty()) { - for (Block BA : Blocks) { + for (NodeAddr BA : Blocks) { const MachineBasicBlock &B = *BA.Addr->getCode(); if (!B.isEHPad()) continue; @@ -926,14 +950,14 @@ void DataFlowGraph::build(unsigned Options) { // Build phi nodes for each live-in. for (RegisterRef RR : EHRegs.refs()) { - Phi PA = newPhi(BA); + NodeAddr PA = newPhi(BA); uint16_t PhiFlags = NodeAttrs::PhiRef | NodeAttrs::Preserving; // Add def: - Def DA = newDef(PA, RR, PhiFlags); + NodeAddr DA = newDef(PA, RR, PhiFlags); PA.Addr->addMember(DA, *this); // Add uses (no reaching defs for phi uses): - for (Block PBA : Preds) { - PhiUse PUA = newPhiUse(PA, RR, PBA); + for (NodeAddr PBA : Preds) { + NodeAddr PUA = newPhiUse(PA, RR, PBA); PA.Addr->addMember(PUA, *this); } } @@ -943,9 +967,9 @@ void DataFlowGraph::build(unsigned Options) { // Build a map "PhiM" which will contain, for each block, the set // of references that will require phi definitions in that block. BlockRefsMap PhiM(getPRI()); - for (Block BA : Blocks) + for (NodeAddr BA : Blocks) recordDefsForDF(PhiM, BA); - for (Block BA : Blocks) + for (NodeAddr BA : Blocks) buildPhis(PhiM, AllRefs, BA); // Link all the refs. This will recursively traverse the dominator tree. @@ -958,7 +982,8 @@ void DataFlowGraph::build(unsigned Options) { } RegisterRef DataFlowGraph::makeRegRef(unsigned Reg, unsigned Sub) const { - assert(RegisterRef::isRegId(Reg) || RegisterRef::isMaskId(Reg)); + assert(PhysicalRegisterInfo::isRegMaskId(Reg) || + Register::isPhysicalRegister(Reg)); assert(Reg != 0); if (Sub != 0) Reg = TRI.getSubReg(Reg, Sub); @@ -969,8 +994,7 @@ RegisterRef DataFlowGraph::makeRegRef(const MachineOperand &Op) const { assert(Op.isReg() || Op.isRegMask()); if (Op.isReg()) return makeRegRef(Op.getReg(), Op.getSubReg()); - return RegisterRef(getPRI().getRegMaskId(Op.getRegMask()), - LaneBitmask::getAll()); + return RegisterRef(PRI.getRegMaskId(Op.getRegMask()), LaneBitmask::getAll()); } // For each stack in the map DefM, push the delimiter for block B on it. @@ -999,14 +1023,14 @@ void DataFlowGraph::releaseBlock(NodeId B, DefStackMap &DefM) { // Push all definitions from the instruction node IA to an appropriate // stack in DefM. -void DataFlowGraph::pushAllDefs(Instr IA, DefStackMap &DefM) { +void DataFlowGraph::pushAllDefs(NodeAddr IA, DefStackMap &DefM) { pushClobbers(IA, DefM); pushDefs(IA, DefM); } // Push all definitions from the instruction node IA to an appropriate // stack in DefM. -void DataFlowGraph::pushClobbers(Instr IA, DefStackMap &DefM) { +void DataFlowGraph::pushClobbers(NodeAddr IA, DefStackMap &DefM) { NodeSet Visited; std::set Defined; @@ -1022,35 +1046,35 @@ void DataFlowGraph::pushClobbers(Instr IA, DefStackMap &DefM) { // unspecified order), but the order does not matter from the data- // -flow perspective. - for (Def DA : IA.Addr->members_if(IsDef, *this)) { + for (NodeAddr DA : IA.Addr->members_if(IsDef, *this)) { if (Visited.count(DA.Id)) continue; if (!(DA.Addr->getFlags() & NodeAttrs::Clobbering)) continue; NodeList Rel = getRelatedRefs(IA, DA); - Def PDA = Rel.front(); + NodeAddr PDA = Rel.front(); RegisterRef RR = PDA.Addr->getRegRef(*this); // Push the definition on the stack for the register and all aliases. // The def stack traversal in linkNodeUp will check the exact aliasing. DefM[RR.Reg].push(DA); Defined.insert(RR.Reg); - for (RegisterId A : getPRI().getAliasSet(RR.Reg)) { + for (RegisterId A : PRI.getAliasSet(RR.Reg)) { // Check that we don't push the same def twice. assert(A != RR.Reg); if (!Defined.count(A)) DefM[A].push(DA); } // Mark all the related defs as visited. - for (Node T : Rel) + for (NodeAddr T : Rel) Visited.insert(T.Id); } } // Push all definitions from the instruction node IA to an appropriate // stack in DefM. -void DataFlowGraph::pushDefs(Instr IA, DefStackMap &DefM) { +void DataFlowGraph::pushDefs(NodeAddr IA, DefStackMap &DefM) { NodeSet Visited; #ifndef NDEBUG std::set Defined; @@ -1068,20 +1092,20 @@ void DataFlowGraph::pushDefs(Instr IA, DefStackMap &DefM) { // unspecified order), but the order does not matter from the data- // -flow perspective. - for (Def DA : IA.Addr->members_if(IsDef, *this)) { + for (NodeAddr DA : IA.Addr->members_if(IsDef, *this)) { if (Visited.count(DA.Id)) continue; if (DA.Addr->getFlags() & NodeAttrs::Clobbering) continue; NodeList Rel = getRelatedRefs(IA, DA); - Def PDA = Rel.front(); + NodeAddr PDA = Rel.front(); RegisterRef RR = PDA.Addr->getRegRef(*this); #ifndef NDEBUG // Assert if the register is defined in two or more unrelated defs. // This could happen if there are two or more def operands defining it. if (!Defined.insert(RR.Reg).second) { - MachineInstr *MI = Stmt(IA).Addr->getCode(); + MachineInstr *MI = NodeAddr(IA).Addr->getCode(); dbgs() << "Multiple definitions of register: " << Print(RR, *this) << " in\n " << *MI << "in " << printMBBReference(*MI->getParent()) << '\n'; @@ -1091,20 +1115,21 @@ void DataFlowGraph::pushDefs(Instr IA, DefStackMap &DefM) { // Push the definition on the stack for the register and all aliases. // The def stack traversal in linkNodeUp will check the exact aliasing. DefM[RR.Reg].push(DA); - for (RegisterId A : getPRI().getAliasSet(RR.Reg)) { + for (RegisterId A : PRI.getAliasSet(RR.Reg)) { // Check that we don't push the same def twice. assert(A != RR.Reg); DefM[A].push(DA); } // Mark all the related defs as visited. - for (Node T : Rel) + for (NodeAddr T : Rel) Visited.insert(T.Id); } } // Return the list of all reference nodes related to RA, including RA itself. // See "getNextRelated" for the meaning of a "related reference". -NodeList DataFlowGraph::getRelatedRefs(Instr IA, Ref RA) const { +NodeList DataFlowGraph::getRelatedRefs(NodeAddr IA, + NodeAddr RA) const { assert(IA.Id != 0 && RA.Id != 0); NodeList Refs; @@ -1120,7 +1145,7 @@ NodeList DataFlowGraph::getRelatedRefs(Instr IA, Ref RA) const { void DataFlowGraph::reset() { Memory.clear(); BlockNodes.clear(); - TheFunc = Func(); + Func = NodeAddr(); } // Return the next reference node in the instruction node IA that is related @@ -1129,22 +1154,22 @@ void DataFlowGraph::reset() { // characteristics. Specific examples of related nodes are shadow reference // nodes. // Return the equivalent of nullptr if there are no more related references. -Ref DataFlowGraph::getNextRelated(Instr IA, Ref RA) const { +NodeAddr +DataFlowGraph::getNextRelated(NodeAddr IA, + NodeAddr RA) const { assert(IA.Id != 0 && RA.Id != 0); - auto Related = [this, RA](Ref TA) -> bool { + auto Related = [this, RA](NodeAddr TA) -> bool { if (TA.Addr->getKind() != RA.Addr->getKind()) return false; - if (!getPRI().equal_to(TA.Addr->getRegRef(*this), - RA.Addr->getRegRef(*this))) { + if (TA.Addr->getRegRef(*this) != RA.Addr->getRegRef(*this)) return false; - } return true; }; - auto RelatedStmt = [&Related, RA](Ref TA) -> bool { + auto RelatedStmt = [&Related, RA](NodeAddr TA) -> bool { return Related(TA) && &RA.Addr->getOp() == &TA.Addr->getOp(); }; - auto RelatedPhi = [&Related, RA](Ref TA) -> bool { + auto RelatedPhi = [&Related, RA](NodeAddr TA) -> bool { if (!Related(TA)) return false; if (TA.Addr->getKind() != NodeAttrs::Use) @@ -1167,11 +1192,12 @@ Ref DataFlowGraph::getNextRelated(Instr IA, Ref RA) const { // first element is the element after which such a node should be inserted, // and the second element is a null-address. template -std::pair DataFlowGraph::locateNextRef(Instr IA, Ref RA, - Predicate P) const { +std::pair, NodeAddr> +DataFlowGraph::locateNextRef(NodeAddr IA, NodeAddr RA, + Predicate P) const { assert(IA.Id != 0 && RA.Id != 0); - Ref NA; + NodeAddr NA; NodeId Start = RA.Id; while (true) { NA = getNextRelated(IA, RA); @@ -1184,16 +1210,18 @@ std::pair DataFlowGraph::locateNextRef(Instr IA, Ref RA, if (NA.Id != 0 && NA.Id != Start) return std::make_pair(RA, NA); - return std::make_pair(RA, Ref()); + return std::make_pair(RA, NodeAddr()); } // Get the next shadow node in IA corresponding to RA, and optionally create // such a node if it does not exist. -Ref DataFlowGraph::getNextShadow(Instr IA, Ref RA, bool Create) { +NodeAddr DataFlowGraph::getNextShadow(NodeAddr IA, + NodeAddr RA, + bool Create) { assert(IA.Id != 0 && RA.Id != 0); uint16_t Flags = RA.Addr->getFlags() | NodeAttrs::Shadow; - auto IsShadow = [Flags](Ref TA) -> bool { + auto IsShadow = [Flags](NodeAddr TA) -> bool { return TA.Addr->getFlags() == Flags; }; auto Loc = locateNextRef(IA, RA, IsShadow); @@ -1201,7 +1229,7 @@ Ref DataFlowGraph::getNextShadow(Instr IA, Ref RA, bool Create) { return Loc.second; // Create a copy of RA and mark is as shadow. - Ref NA = cloneNode(RA); + NodeAddr NA = cloneNode(RA); NA.Addr->setFlags(Flags | NodeAttrs::Shadow); IA.Addr->addMemberAfter(Loc.first, NA, *this); return NA; @@ -1209,10 +1237,11 @@ Ref DataFlowGraph::getNextShadow(Instr IA, Ref RA, bool Create) { // Get the next shadow node in IA corresponding to RA. Return null-address // if such a node does not exist. -Ref DataFlowGraph::getNextShadow(Instr IA, Ref RA) const { +NodeAddr DataFlowGraph::getNextShadow(NodeAddr IA, + NodeAddr RA) const { assert(IA.Id != 0 && RA.Id != 0); uint16_t Flags = RA.Addr->getFlags() | NodeAttrs::Shadow; - auto IsShadow = [Flags](Ref TA) -> bool { + auto IsShadow = [Flags](NodeAddr TA) -> bool { return TA.Addr->getFlags() == Flags; }; return locateNextRef(IA, RA, IsShadow).second; @@ -1220,8 +1249,8 @@ Ref DataFlowGraph::getNextShadow(Instr IA, Ref RA) const { // Create a new statement node in the block node BA that corresponds to // the machine instruction MI. -void DataFlowGraph::buildStmt(Block BA, MachineInstr &In) { - Stmt SA = newStmt(BA, &In); +void DataFlowGraph::buildStmt(NodeAddr BA, MachineInstr &In) { + NodeAddr SA = newStmt(BA, &In); auto isCall = [](const MachineInstr &In) -> bool { if (In.isCall()) @@ -1247,7 +1276,7 @@ void DataFlowGraph::buildStmt(Block BA, MachineInstr &In) { if (Op.getReg() == 0 || Op.isUndef()) continue; RegisterRef UR = makeRegRef(Op); - if (getPRI().alias(DR, UR)) + if (PRI.alias(DR, UR)) return false; } return true; @@ -1282,7 +1311,7 @@ void DataFlowGraph::buildStmt(Block BA, MachineInstr &In) { Flags |= NodeAttrs::Fixed; if (IsCall && Op.isDead()) Flags |= NodeAttrs::Dead; - Def DA = newDef(SA, Op, Flags); + NodeAddr DA = newDef(SA, Op, Flags); SA.Addr->addMember(DA, *this); assert(!DoneDefs.test(R)); DoneDefs.set(R); @@ -1295,7 +1324,7 @@ void DataFlowGraph::buildStmt(Block BA, MachineInstr &In) { if (!Op.isRegMask()) continue; uint16_t Flags = NodeAttrs::Clobbering | NodeAttrs::Fixed | NodeAttrs::Dead; - Def DA = newDef(SA, Op, Flags); + NodeAddr DA = newDef(SA, Op, Flags); SA.Addr->addMember(DA, *this); // Record all clobbered registers in DoneDefs. const uint32_t *RM = Op.getRegMask(); @@ -1330,7 +1359,7 @@ void DataFlowGraph::buildStmt(Block BA, MachineInstr &In) { continue; Flags |= NodeAttrs::Dead; } - Def DA = newDef(SA, Op, Flags); + NodeAddr DA = newDef(SA, Op, Flags); SA.Addr->addMember(DA, *this); DoneDefs.set(R); } @@ -1347,14 +1376,15 @@ void DataFlowGraph::buildStmt(Block BA, MachineInstr &In) { Flags |= NodeAttrs::Undef; if (TOI.isFixedReg(In, OpN)) Flags |= NodeAttrs::Fixed; - Use UA = newUse(SA, Op, Flags); + NodeAddr UA = newUse(SA, Op, Flags); SA.Addr->addMember(UA, *this); } } // Scan all defs in the block node BA and record in PhiM the locations of // phi nodes corresponding to these defs. -void DataFlowGraph::recordDefsForDF(BlockRefsMap &PhiM, Block BA) { +void DataFlowGraph::recordDefsForDF(BlockRefsMap &PhiM, + NodeAddr BA) { // Check all defs from block BA and record them in each block in BA's // iterated dominance frontier. This information will later be used to // create phi nodes. @@ -1370,8 +1400,8 @@ void DataFlowGraph::recordDefsForDF(BlockRefsMap &PhiM, Block BA) { // This is done to make sure that each defined reference gets only one // phi node, even if it is defined multiple times. RegisterAggr Defs(getPRI()); - for (Instr IA : BA.Addr->members(*this)) - for (Ref RA : IA.Addr->members_if(IsDef, *this)) + for (NodeAddr IA : BA.Addr->members(*this)) + for (NodeAddr RA : IA.Addr->members_if(IsDef, *this)) Defs.insert(RA.Addr->getRegRef(*this)); // Calculate the iterated dominance frontier of BB. @@ -1386,7 +1416,7 @@ void DataFlowGraph::recordDefsForDF(BlockRefsMap &PhiM, Block BA) { // Finally, add the set of defs to each block in the iterated dominance // frontier. for (auto *DB : IDF) { - Block DBA = findBlock(DB); + NodeAddr DBA = findBlock(DB); PhiM[DBA.Id].insert(Defs); } } @@ -1394,7 +1424,7 @@ void DataFlowGraph::recordDefsForDF(BlockRefsMap &PhiM, Block BA) { // Given the locations of phi nodes in the map PhiM, create the phi nodes // that are located in the block node BA. void DataFlowGraph::buildPhis(BlockRefsMap &PhiM, RegisterSet &AllRefs, - Block BA) { + NodeAddr BA) { // Check if this blocks has any DF defs, i.e. if there are any defs // that this block is in the iterated dominance frontier of. auto HasDF = PhiM.find(BA.Id); @@ -1411,11 +1441,11 @@ void DataFlowGraph::buildPhis(BlockRefsMap &PhiM, RegisterSet &AllRefs, uint16_t PhiFlags = NodeAttrs::PhiRef | NodeAttrs::Preserving; for (RegisterRef RR : Defs.refs()) { - Phi PA = newPhi(BA); + NodeAddr PA = newPhi(BA); PA.Addr->addMember(newDef(PA, RR, PhiFlags), *this); // Add phi uses. - for (Block PBA : Preds) { + for (NodeAddr PBA : Preds) { PA.Addr->addMember(newPhiUse(PA, RR, PBA), *this); } } @@ -1430,16 +1460,16 @@ void DataFlowGraph::removeUnusedPhis() { // that are easily determinable to be unnecessary. SetVector PhiQ; - for (Block BA : TheFunc.Addr->members(*this)) { + for (NodeAddr BA : Func.Addr->members(*this)) { for (auto P : BA.Addr->members_if(IsPhi, *this)) PhiQ.insert(P.Id); } static auto HasUsedDef = [](NodeList &Ms) -> bool { - for (Node M : Ms) { + for (NodeAddr M : Ms) { if (M.Addr->getKind() != NodeAttrs::Def) continue; - Def DA = M; + NodeAddr DA = M; if (DA.Addr->getReachedDef() != 0 || DA.Addr->getReachedUse() != 0) return true; } @@ -1455,10 +1485,10 @@ void DataFlowGraph::removeUnusedPhis() { NodeList Refs = PA.Addr->members(*this); if (HasUsedDef(Refs)) continue; - for (Ref RA : Refs) { + for (NodeAddr RA : Refs) { if (NodeId RD = RA.Addr->getReachingDef()) { auto RDA = addr(RD); - Instr OA = RDA.Addr->getOwner(*this); + NodeAddr OA = RDA.Addr->getOwner(*this); if (IsPhi(OA)) PhiQ.insert(OA.Id); } @@ -1467,7 +1497,7 @@ void DataFlowGraph::removeUnusedPhis() { else unlinkUse(RA, true); } - Block BA = PA.Addr->getOwner(*this); + NodeAddr BA = PA.Addr->getOwner(*this); BA.Addr->removeMember(PA, *this); } } @@ -1476,14 +1506,15 @@ void DataFlowGraph::removeUnusedPhis() { // reaching def of TA to the appropriate def node. Create any shadow nodes // as appropriate. template -void DataFlowGraph::linkRefUp(Instr IA, NodeAddr TA, DefStack &DS) { +void DataFlowGraph::linkRefUp(NodeAddr IA, NodeAddr TA, + DefStack &DS) { if (DS.empty()) return; RegisterRef RR = TA.Addr->getRegRef(*this); NodeAddr TAP; // References from the def stack that have been examined so far. - RegisterAggr Defs(getPRI()); + RegisterAggr Defs(PRI); for (auto I = DS.top(), E = DS.bottom(); I != E; I.down()) { RegisterRef QR = I->Addr->getRegRef(*this); @@ -1499,7 +1530,7 @@ void DataFlowGraph::linkRefUp(Instr IA, NodeAddr TA, DefStack &DS) { } // The reaching def. - Def RDA = *I; + NodeAddr RDA = *I; // Pick the reached node. if (TAP.Id == 0) { @@ -1520,13 +1551,14 @@ void DataFlowGraph::linkRefUp(Instr IA, NodeAddr TA, DefStack &DS) { // Create data-flow links for all reference nodes in the statement node SA. template -void DataFlowGraph::linkStmtRefs(DefStackMap &DefM, Stmt SA, Predicate P) { +void DataFlowGraph::linkStmtRefs(DefStackMap &DefM, NodeAddr SA, + Predicate P) { #ifndef NDEBUG - RegisterSet Defs(getPRI()); + RegisterSet Defs; #endif // Link all nodes (upwards in the data-flow) with their reaching defs. - for (Ref RA : SA.Addr->members_if(P, *this)) { + for (NodeAddr RA : SA.Addr->members_if(P, *this)) { uint16_t Kind = RA.Addr->getKind(); assert(Kind == NodeAttrs::Def || Kind == NodeAttrs::Use); RegisterRef RR = RA.Addr->getRegRef(*this); @@ -1551,14 +1583,14 @@ void DataFlowGraph::linkStmtRefs(DefStackMap &DefM, Stmt SA, Predicate P) { // Create data-flow links for all instructions in the block node BA. This // will include updating any phi nodes in BA. -void DataFlowGraph::linkBlockRefs(DefStackMap &DefM, Block BA) { +void DataFlowGraph::linkBlockRefs(DefStackMap &DefM, NodeAddr BA) { // Push block delimiters. markBlock(BA.Id, DefM); - auto IsClobber = [](Ref RA) -> bool { + auto IsClobber = [](NodeAddr RA) -> bool { return IsDef(RA) && (RA.Addr->getFlags() & NodeAttrs::Clobbering); }; - auto IsNoClobber = [](Ref RA) -> bool { + auto IsNoClobber = [](NodeAddr RA) -> bool { return IsDef(RA) && !(RA.Addr->getFlags() & NodeAttrs::Clobbering); }; @@ -1566,7 +1598,7 @@ void DataFlowGraph::linkBlockRefs(DefStackMap &DefM, Block BA) { // For each non-phi instruction in the block, link all the defs and uses // to their reaching defs. For any member of the block (including phis), // push the defs on the corresponding stacks. - for (Instr IA : BA.Addr->members(*this)) { + for (NodeAddr IA : BA.Addr->members(*this)) { // Ignore phi nodes here. They will be linked part by part from the // predecessors. if (IA.Addr->getKind() == NodeAttrs::Stmt) { @@ -1587,16 +1619,16 @@ void DataFlowGraph::linkBlockRefs(DefStackMap &DefM, Block BA) { MachineDomTreeNode *N = MDT.getNode(BA.Addr->getCode()); for (auto *I : *N) { MachineBasicBlock *SB = I->getBlock(); - Block SBA = findBlock(SB); + NodeAddr SBA = findBlock(SB); linkBlockRefs(DefM, SBA); } // Link the phi uses from the successor blocks. - auto IsUseForBA = [BA](Node NA) -> bool { + auto IsUseForBA = [BA](NodeAddr NA) -> bool { if (NA.Addr->getKind() != NodeAttrs::Use) return false; assert(NA.Addr->getFlags() & NodeAttrs::PhiRef); - PhiUse PUA = NA; + NodeAddr PUA = NA; return PUA.Addr->getPredecessor() == BA.Id; }; @@ -1605,19 +1637,19 @@ void DataFlowGraph::linkBlockRefs(DefStackMap &DefM, Block BA) { for (MachineBasicBlock *SB : MBB->successors()) { bool IsEHPad = SB->isEHPad(); - Block SBA = findBlock(SB); - for (Instr IA : SBA.Addr->members_if(IsPhi, *this)) { + NodeAddr SBA = findBlock(SB); + for (NodeAddr IA : SBA.Addr->members_if(IsPhi, *this)) { // Do not link phi uses for landing pad live-ins. if (IsEHPad) { // Find what register this phi is for. - Ref RA = IA.Addr->getFirstMember(*this); + NodeAddr RA = IA.Addr->getFirstMember(*this); assert(RA.Id != 0); if (EHLiveIns.hasCoverOf(RA.Addr->getRegRef(*this))) continue; } // Go over each phi use associated with MBB, and link it. for (auto U : IA.Addr->members_if(IsUseForBA, *this)) { - PhiUse PUA = U; + NodeAddr PUA = U; RegisterRef RR = PUA.Addr->getRegRef(*this); linkRefUp(IA, PUA, DefM[RR.Reg]); } @@ -1629,7 +1661,7 @@ void DataFlowGraph::linkBlockRefs(DefStackMap &DefM, Block BA) { } // Remove the use node UA from any data-flow and structural links. -void DataFlowGraph::unlinkUseDF(Use UA) { +void DataFlowGraph::unlinkUseDF(NodeAddr UA) { NodeId RD = UA.Addr->getReachingDef(); NodeId Sib = UA.Addr->getSibling(); @@ -1656,7 +1688,7 @@ void DataFlowGraph::unlinkUseDF(Use UA) { } // Remove the def node DA from any data-flow and structural links. -void DataFlowGraph::unlinkDefDF(Def DA) { +void DataFlowGraph::unlinkDefDF(NodeAddr DA) { // // RD // | reached @@ -1695,14 +1727,14 @@ void DataFlowGraph::unlinkDefDF(Def DA) { NodeList ReachedUses = getAllNodes(DA.Addr->getReachedUse()); if (RD == 0) { - for (Ref I : ReachedDefs) + for (NodeAddr I : ReachedDefs) I.Addr->setSibling(0); - for (Ref I : ReachedUses) + for (NodeAddr I : ReachedUses) I.Addr->setSibling(0); } - for (Def I : ReachedDefs) + for (NodeAddr I : ReachedDefs) I.Addr->setReachingDef(RD); - for (Use I : ReachedUses) + for (NodeAddr I : ReachedUses) I.Addr->setReachingDef(RD); NodeId Sib = DA.Addr->getSibling(); @@ -1733,13 +1765,13 @@ void DataFlowGraph::unlinkDefDF(Def DA) { // Splice the DA's reached defs into the RDA's reached def chain. if (!ReachedDefs.empty()) { - auto Last = Def(ReachedDefs.back()); + auto Last = NodeAddr(ReachedDefs.back()); Last.Addr->setSibling(RDA.Addr->getReachedDef()); RDA.Addr->setReachedDef(ReachedDefs.front().Id); } // Splice the DA's reached uses into the RDA's reached use chain. if (!ReachedUses.empty()) { - auto Last = Use(ReachedUses.back()); + auto Last = NodeAddr(ReachedUses.back()); Last.Addr->setSibling(RDA.Addr->getReachedUse()); RDA.Addr->setReachedUse(ReachedUses.front().Id); } diff --git a/llvm/lib/CodeGen/RDFLiveness.cpp b/llvm/lib/CodeGen/RDFLiveness.cpp index e404b02054df0..31ab7f3ed687d 100644 --- a/llvm/lib/CodeGen/RDFLiveness.cpp +++ b/llvm/lib/CodeGen/RDFLiveness.cpp @@ -65,7 +65,7 @@ raw_ostream &operator<<(raw_ostream &OS, const Print &P) { for (const auto &I : P.Obj) { OS << ' ' << printReg(I.first, &P.G.getTRI()) << '{'; for (auto J = I.second.begin(), E = I.second.end(); J != E;) { - OS << Print(J->first, P.G) << PrintLaneMaskShort(J->second); + OS << Print(J->first, P.G) << PrintLaneMaskOpt(J->second); if (++J != E) OS << ','; } @@ -659,8 +659,6 @@ void Liveness::computePhiInfo() { // The operation "clearIn" can be expensive. For a given set of intervening // defs, cache the result of subtracting these defs from a given register // ref. - using RefHash = std::hash; - using RefEqual = std::equal_to; using SubMap = std::unordered_map; std::unordered_map Subs; auto ClearIn = [](RegisterRef RR, const RegisterAggr &Mid, SubMap &SM) { @@ -692,10 +690,7 @@ void Liveness::computePhiInfo() { if (MidDefs.hasCoverOf(UR)) continue; - if (Subs.find(MidDefs) == Subs.end()) { - Subs.insert({MidDefs, SubMap(1, RefHash(), RefEqual(PRI))}); - } - SubMap &SM = Subs.at(MidDefs); + SubMap &SM = Subs[MidDefs]; // General algorithm: // for each (R,U) : U is use node of R, U is reached by PA @@ -878,7 +873,7 @@ void Liveness::computeLiveIns() { std::vector LV; for (const MachineBasicBlock::RegisterMaskPair &LI : B.liveins()) LV.push_back(RegisterRef(LI.PhysReg, LI.LaneMask)); - llvm::sort(LV, std::less(PRI)); + llvm::sort(LV); dbgs() << printMBBReference(B) << "\t rec = {"; for (auto I : LV) dbgs() << ' ' << Print(I, DFG); @@ -888,7 +883,7 @@ void Liveness::computeLiveIns() { LV.clear(); for (RegisterRef RR : LiveMap[&B].refs()) LV.push_back(RR); - llvm::sort(LV, std::less(PRI)); + llvm::sort(LV); dbgs() << "\tcomp = {"; for (auto I : LV) dbgs() << ' ' << Print(I, DFG); diff --git a/llvm/lib/CodeGen/RDFRegisters.cpp b/llvm/lib/CodeGen/RDFRegisters.cpp index 0f451b6c044ed..3e9a11d5397c9 100644 --- a/llvm/lib/CodeGen/RDFRegisters.cpp +++ b/llvm/lib/CodeGen/RDFRegisters.cpp @@ -15,7 +15,6 @@ #include "llvm/MC/LaneBitmask.h" #include "llvm/MC/MCRegisterInfo.h" #include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/Format.h" #include "llvm/Support/raw_ostream.h" #include #include @@ -105,10 +104,10 @@ PhysicalRegisterInfo::PhysicalRegisterInfo(const TargetRegisterInfo &tri, } std::set PhysicalRegisterInfo::getAliasSet(RegisterId Reg) const { - // Do not include Reg in the alias set. + // Do not include RR in the alias set. std::set AS; - assert(!RegisterRef::isUnitId(Reg) && "No units allowed"); - if (RegisterRef::isMaskId(Reg)) { + assert(isRegMaskId(Reg) || Register::isPhysicalRegister(Reg)); + if (isRegMaskId(Reg)) { // XXX SLOW const uint32_t *MB = getRegMaskBits(Reg); for (unsigned i = 1, e = TRI.getNumRegs(); i != e; ++i) { @@ -124,7 +123,6 @@ std::set PhysicalRegisterInfo::getAliasSet(RegisterId Reg) const { return AS; } - assert(RegisterRef::isRegId(Reg)); for (MCRegAliasIterator AI(Reg, &TRI, false); AI.isValid(); ++AI) AS.insert(*AI); for (const uint32_t *RM : RegMasks) { @@ -136,7 +134,8 @@ std::set PhysicalRegisterInfo::getAliasSet(RegisterId Reg) const { } bool PhysicalRegisterInfo::aliasRR(RegisterRef RA, RegisterRef RB) const { - assert(RA.isReg() && RB.isReg()); + assert(Register::isPhysicalRegister(RA.Reg)); + assert(Register::isPhysicalRegister(RB.Reg)); MCRegUnitMaskIterator UMA(RA.Reg, &TRI); MCRegUnitMaskIterator UMB(RB.Reg, &TRI); @@ -166,7 +165,7 @@ bool PhysicalRegisterInfo::aliasRR(RegisterRef RA, RegisterRef RB) const { } bool PhysicalRegisterInfo::aliasRM(RegisterRef RR, RegisterRef RM) const { - assert(RR.isReg() && RM.isMask()); + assert(Register::isPhysicalRegister(RR.Reg) && isRegMaskId(RM.Reg)); const uint32_t *MB = getRegMaskBits(RM.Reg); bool Preserved = MB[RR.Reg / 32] & (1u << (RR.Reg % 32)); // If the lane mask information is "full", e.g. when the given lane mask @@ -201,7 +200,7 @@ bool PhysicalRegisterInfo::aliasRM(RegisterRef RR, RegisterRef RM) const { } bool PhysicalRegisterInfo::aliasMM(RegisterRef RM, RegisterRef RN) const { - assert(RM.isMask() && RN.isMask()); + assert(isRegMaskId(RM.Reg) && isRegMaskId(RN.Reg)); unsigned NumRegs = TRI.getNumRegs(); const uint32_t *BM = getRegMaskBits(RM.Reg); const uint32_t *BN = getRegMaskBits(RN.Reg); @@ -243,118 +242,8 @@ RegisterRef PhysicalRegisterInfo::mapTo(RegisterRef RR, unsigned R) const { llvm_unreachable("Invalid arguments: unrelated registers?"); } -bool PhysicalRegisterInfo::equal_to(RegisterRef A, RegisterRef B) const { - if (!A.isReg() || !B.isReg()) { - // For non-regs, or comparing reg and non-reg, use only the Reg member. - return A.Reg == B.Reg; - } - - if (A.Reg == B.Reg) - return A.Mask == B.Mask; - - // Compare reg units lexicographically. - MCRegUnitMaskIterator AI(A.Reg, &getTRI()); - MCRegUnitMaskIterator BI(B.Reg, &getTRI()); - while (AI.isValid() && BI.isValid()) { - auto [AReg, AMask] = *AI; - auto [BReg, BMask] = *BI; - - // Lane masks are "none" for units that don't correspond to subregs - // e.g. a single unit in a leaf register, or aliased unit. - if (AMask.none()) - AMask = LaneBitmask::getAll(); - if (BMask.none()) - BMask = LaneBitmask::getAll(); - - // If both iterators point to a unit contained in both A and B, then - // compare the units. - if ((AMask & A.Mask).any() && (BMask & B.Mask).any()) { - if (AReg != BReg) - return false; - // Units are equal, move on to the next ones. - ++AI; - ++BI; - continue; - } - - if ((AMask & A.Mask).none()) - ++AI; - if ((BMask & B.Mask).none()) - ++BI; - } - // One or both have reached the end. - return static_cast(AI.isValid()) == static_cast(BI.isValid()); -} - -bool PhysicalRegisterInfo::less(RegisterRef A, RegisterRef B) const { - if (!A.isReg() || !B.isReg()) { - // For non-regs, or comparing reg and non-reg, use only the Reg member. - return A.Reg < B.Reg; - } - - if (A.Reg == B.Reg) - return A.Mask < B.Mask; - if (A.Mask == B.Mask) - return A.Reg < B.Reg; - - // Compare reg units lexicographically. - llvm::MCRegUnitMaskIterator AI(A.Reg, &getTRI()); - llvm::MCRegUnitMaskIterator BI(B.Reg, &getTRI()); - while (AI.isValid() && BI.isValid()) { - auto [AReg, AMask] = *AI; - auto [BReg, BMask] = *BI; - - // Lane masks are "none" for units that don't correspond to subregs - // e.g. a single unit in a leaf register, or aliased unit. - if (AMask.none()) - AMask = LaneBitmask::getAll(); - if (BMask.none()) - BMask = LaneBitmask::getAll(); - - // If both iterators point to a unit contained in both A and B, then - // compare the units. - if ((AMask & A.Mask).any() && (BMask & B.Mask).any()) { - if (AReg != BReg) - return AReg < BReg; - // Units are equal, move on to the next ones. - ++AI; - ++BI; - continue; - } - - if ((AMask & A.Mask).none()) - ++AI; - if ((BMask & B.Mask).none()) - ++BI; - } - // One or both have reached the end: assume invalid < valid. - return static_cast(AI.isValid()) < static_cast(BI.isValid()); -} - -void PhysicalRegisterInfo::print(raw_ostream &OS, RegisterRef A) const { - if (A.Reg == 0 || A.isReg()) { - if (0 < A.Reg && A.Reg < TRI.getNumRegs()) - OS << TRI.getName(A.Reg); - else - OS << printReg(A.Reg, &TRI); - OS << PrintLaneMaskShort(A.Mask); - } else if (A.isUnit()) { - OS << printRegUnit(A.toRegUnit(A.Reg), &TRI); - } else { - assert(A.isMask()); - OS << '#' << format("%08x", A.Reg); - } -} - -void PhysicalRegisterInfo::print(raw_ostream &OS, const RegisterAggr &A) const { - OS << '{'; - for (unsigned U : A.units()) - OS << ' ' << printRegUnit(U, &TRI); - OS << " }"; -} - bool RegisterAggr::hasAliasOf(RegisterRef RR) const { - if (RR.isMask()) + if (PhysicalRegisterInfo::isRegMaskId(RR.Reg)) return Units.anyCommon(PRI.getMaskUnits(RR.Reg)); for (MCRegUnitMaskIterator U(RR.Reg, &PRI.getTRI()); U.isValid(); ++U) { @@ -367,7 +256,7 @@ bool RegisterAggr::hasAliasOf(RegisterRef RR) const { } bool RegisterAggr::hasCoverOf(RegisterRef RR) const { - if (RR.isMask()) { + if (PhysicalRegisterInfo::isRegMaskId(RR.Reg)) { BitVector T(PRI.getMaskUnits(RR.Reg)); return T.reset(Units).none(); } @@ -382,7 +271,7 @@ bool RegisterAggr::hasCoverOf(RegisterRef RR) const { } RegisterAggr &RegisterAggr::insert(RegisterRef RR) { - if (RR.isMask()) { + if (PhysicalRegisterInfo::isRegMaskId(RR.Reg)) { Units |= PRI.getMaskUnits(RR.Reg); return *this; } @@ -468,7 +357,14 @@ RegisterRef RegisterAggr::makeRegRef() const { return RegisterRef(F, M); } -RegisterAggr::ref_iterator::ref_iterator(const RegisterAggr &RG, bool End) +void RegisterAggr::print(raw_ostream &OS) const { + OS << '{'; + for (int U = Units.find_first(); U >= 0; U = Units.find_next(U)) + OS << ' ' << printRegUnit(U, &PRI.getTRI()); + OS << " }"; +} + +RegisterAggr::rr_iterator::rr_iterator(const RegisterAggr &RG, bool End) : Owner(&RG) { for (int U = RG.Units.find_first(); U >= 0; U = RG.Units.find_next(U)) { RegisterRef R = RG.PRI.getRefForUnit(U); @@ -479,20 +375,6 @@ RegisterAggr::ref_iterator::ref_iterator(const RegisterAggr &RG, bool End) } raw_ostream &rdf::operator<<(raw_ostream &OS, const RegisterAggr &A) { - A.getPRI().print(OS, A); + A.print(OS); return OS; } - -raw_ostream &rdf::operator<<(raw_ostream &OS, const PrintLaneMaskShort &P) { - if (P.Mask.all()) - return OS; - if (P.Mask.none()) - return OS << ":*none*"; - - LaneBitmask::Type Val = P.Mask.getAsInteger(); - if ((Val & 0xffff) == Val) - return OS << ':' << format("%04llX", Val); - if ((Val & 0xffffffff) == Val) - return OS << ':' << format("%08llX", Val); - return OS << ':' << PrintLaneMask(P.Mask); -} diff --git a/llvm/lib/Target/Hexagon/RDFCopy.cpp b/llvm/lib/Target/Hexagon/RDFCopy.cpp index c26811e9cd05d..e24f66de653d3 100644 --- a/llvm/lib/Target/Hexagon/RDFCopy.cpp +++ b/llvm/lib/Target/Hexagon/RDFCopy.cpp @@ -76,7 +76,7 @@ void CopyPropagation::recordCopy(NodeAddr SA, EqualityMap &EM) { void CopyPropagation::updateMap(NodeAddr IA) { - RegisterSet RRs(DFG.getPRI()); + RegisterSet RRs; for (NodeAddr RA : IA.Addr->members(DFG)) RRs.insert(RA.Addr->getRegRef(DFG)); bool Common = false; @@ -107,7 +107,7 @@ bool CopyPropagation::scanBlock(MachineBasicBlock *B) { for (NodeAddr IA : BA.Addr->members(DFG)) { if (DFG.IsCode(IA)) { NodeAddr SA = IA; - EqualityMap EM(std::less(DFG.getPRI())); + EqualityMap EM; if (interpretAsCopy(SA.Addr->getCode(), EM)) recordCopy(SA, EM); } @@ -132,11 +132,9 @@ bool CopyPropagation::run() { for (NodeId I : Copies) { dbgs() << "Instr: " << *DFG.addr(I).Addr->getCode(); dbgs() << " eq: {"; - if (CopyMap.count(I)) { - for (auto J : CopyMap.at(I)) - dbgs() << ' ' << Print(J.first, DFG) << '=' - << Print(J.second, DFG); - } + for (auto J : CopyMap[I]) + dbgs() << ' ' << Print(J.first, DFG) << '=' + << Print(J.second, DFG); dbgs() << " }\n"; } dbgs() << "\nRDef map:\n"; @@ -166,8 +164,6 @@ bool CopyPropagation::run() { return 0; }; - const PhysicalRegisterInfo &PRI = DFG.getPRI(); - for (NodeId C : Copies) { #ifndef NDEBUG if (HasLimit && CpCount >= CpLimit) @@ -185,7 +181,7 @@ bool CopyPropagation::run() { if (FR == EM.end()) continue; RegisterRef SR = FR->second; - if (PRI.equal_to(DR, SR)) + if (DR == SR) continue; auto &RDefSR = RDefMap[SR]; @@ -197,7 +193,7 @@ bool CopyPropagation::run() { uint16_t F = UA.Addr->getFlags(); if ((F & NodeAttrs::PhiRef) || (F & NodeAttrs::Fixed)) continue; - if (!PRI.equal_to(UA.Addr->getRegRef(DFG), DR)) + if (UA.Addr->getRegRef(DFG) != DR) continue; NodeAddr IA = UA.Addr->getOwner(DFG); @@ -237,7 +233,7 @@ bool CopyPropagation::run() { // Update the EM map in the copy's entry. auto &M = FC->second; for (auto &J : M) { - if (!PRI.equal_to(J.second, DR)) + if (J.second != DR) continue; J.second = SR; break; diff --git a/llvm/lib/Target/Hexagon/RDFCopy.h b/llvm/lib/Target/Hexagon/RDFCopy.h index e4fb89892831d..8bca374a52887 100644 --- a/llvm/lib/Target/Hexagon/RDFCopy.h +++ b/llvm/lib/Target/Hexagon/RDFCopy.h @@ -25,8 +25,7 @@ class MachineInstr; namespace rdf { struct CopyPropagation { - CopyPropagation(DataFlowGraph &dfg) : MDT(dfg.getDT()), DFG(dfg), - RDefMap(std::less(DFG.getPRI())) {} + CopyPropagation(DataFlowGraph &dfg) : MDT(dfg.getDT()), DFG(dfg) {} virtual ~CopyPropagation() = default;