diff --git a/llvm/include/llvm/CodeGen/MachineOutliner.h b/llvm/include/llvm/CodeGen/MachineOutliner.h index 08b76295dbf2a..5767300aa74df 100644 --- a/llvm/include/llvm/CodeGen/MachineOutliner.h +++ b/llvm/include/llvm/CodeGen/MachineOutliner.h @@ -20,6 +20,7 @@ #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/TargetRegisterInfo.h" +#include namespace llvm { namespace outliner { @@ -56,6 +57,55 @@ struct Candidate { /// target. unsigned CallOverhead = 0; + /// Liveness information for this Candidate. Tracks from the end of the + /// block containing this Candidate to the beginning of its sequence. + /// + /// Optional. Can be used to fine-tune the cost model, or fine-tune legality + /// decisions. + LiveRegUnits FromEndOfBlockToStartOfSeq; + + /// Liveness information restricted to this Candidate's instruction sequence. + /// + /// Optional. Can be used to fine-tune the cost model, or fine-tune legality + /// decisions. + LiveRegUnits InSeq; + + /// True if FromEndOfBlockToStartOfSeq has been initialized. + bool FromEndOfBlockToStartOfSeqWasSet = false; + + /// True if InSeq has been initialized. + bool InSeqWasSet = false; + + /// Populate FromEndOfBlockToStartOfSeq with liveness information. + void initFromEndOfBlockToStartOfSeq(const TargetRegisterInfo &TRI) { + assert(MBB->getParent()->getRegInfo().tracksLiveness() && + "Candidate's Machine Function must track liveness"); + // Only initialize once. + if (FromEndOfBlockToStartOfSeqWasSet) + return; + FromEndOfBlockToStartOfSeqWasSet = true; + FromEndOfBlockToStartOfSeq.init(TRI); + FromEndOfBlockToStartOfSeq.addLiveOuts(*MBB); + // Compute liveness from the end of the block up to the beginning of the + // outlining candidate. + for (auto &MI : make_range(MBB->rbegin(), + (MachineBasicBlock::reverse_iterator)front())) + FromEndOfBlockToStartOfSeq.stepBackward(MI); + } + + /// Populate InSeq with liveness information. + void initInSeq(const TargetRegisterInfo &TRI) { + assert(MBB->getParent()->getRegInfo().tracksLiveness() && + "Candidate's Machine Function must track liveness"); + // Only initialize once. + if (InSeqWasSet) + return; + InSeqWasSet = true; + InSeq.init(TRI); + for (auto &MI : make_range(front(), std::next(back()))) + InSeq.accumulate(MI); + } + public: /// The index of this \p Candidate's \p OutlinedFunction in the list of /// \p OutlinedFunctions. @@ -65,26 +115,9 @@ struct Candidate { /// from this point. Defined by the target. unsigned CallConstructionID = 0; - /// Contains physical register liveness information for the MBB containing - /// this \p Candidate. - /// - /// This is optionally used by the target to calculate more fine-grained - /// cost model information. - LiveRegUnits LRU; - - /// Contains the accumulated register liveness information for the - /// instructions in this \p Candidate. - /// - /// This is optionally used by the target to determine which registers have - /// been used across the sequence. - LiveRegUnits UsedInSequence; - /// Target-specific flags for this Candidate's MBB. unsigned Flags = 0x0; - /// True if initLRU has been called on this Candidate. - bool LRUWasSet = false; - /// Return the number of instructions in this Candidate. unsigned getLength() const { return Len; } @@ -109,6 +142,50 @@ struct Candidate { MachineFunction *getMF() const { return MBB->getParent(); } MachineBasicBlock *getMBB() const { return MBB; } + /// \returns True if \p Reg is available from the end of the block to the + /// beginning of the sequence. + /// + /// This query considers the following range: + /// + /// in_seq_1 + /// in_seq_2 + /// ... + /// in_seq_n + /// not_in_seq_1 + /// ... + /// + bool isAvailableAcrossAndOutOfSeq(Register Reg, + const TargetRegisterInfo &TRI) { + if (!FromEndOfBlockToStartOfSeqWasSet) + initFromEndOfBlockToStartOfSeq(TRI); + return FromEndOfBlockToStartOfSeq.available(Reg); + } + + /// \returns True if `isAvailableAcrossAndOutOfSeq` fails for any register + /// in \p Regs. + bool isAnyUnavailableAcrossOrOutOfSeq(std::initializer_list Regs, + const TargetRegisterInfo &TRI) { + if (!FromEndOfBlockToStartOfSeqWasSet) + initFromEndOfBlockToStartOfSeq(TRI); + return any_of(Regs, [&](Register Reg) { + return !FromEndOfBlockToStartOfSeq.available(Reg); + }); + } + + /// \returns True if \p Reg is available within the sequence itself. + /// + /// This query considers the following range: + /// + /// in_seq_1 + /// in_seq_2 + /// ... + /// in_seq_n + bool isAvailableInsideSeq(Register Reg, const TargetRegisterInfo &TRI) { + if (!InSeqWasSet) + initInSeq(TRI); + return InSeq.available(Reg); + } + /// The number of instructions that would be saved by outlining every /// candidate of this type. /// @@ -132,31 +209,6 @@ struct Candidate { return getStartIdx() > RHS.getStartIdx(); } - /// Compute the registers that are live across this Candidate. - /// Used by targets that need this information for cost model calculation. - /// If a target does not need this information, then this should not be - /// called. - void initLRU(const TargetRegisterInfo &TRI) { - assert(MBB->getParent()->getRegInfo().tracksLiveness() && - "Candidate's Machine Function must track liveness"); - // Only initialize once. - if (LRUWasSet) - return; - LRUWasSet = true; - LRU.init(TRI); - LRU.addLiveOuts(*MBB); - - // Compute liveness from the end of the block up to the beginning of the - // outlining candidate. - std::for_each(MBB->rbegin(), (MachineBasicBlock::reverse_iterator)front(), - [this](MachineInstr &MI) { LRU.stepBackward(MI); }); - - // Walk over the sequence itself and figure out which registers were used - // in the sequence. - UsedInSequence.init(TRI); - std::for_each(front(), std::next(back()), - [this](MachineInstr &MI) { UsedInSequence.accumulate(MI); }); - } }; /// The information necessary to create an outlined function for some diff --git a/llvm/include/llvm/CodeGen/TargetInstrInfo.h b/llvm/include/llvm/CodeGen/TargetInstrInfo.h index dd9936971f6ce..12cd21617b0d4 100644 --- a/llvm/include/llvm/CodeGen/TargetInstrInfo.h +++ b/llvm/include/llvm/CodeGen/TargetInstrInfo.h @@ -1953,7 +1953,7 @@ class TargetInstrInfo : public MCInstrInfo { virtual MachineBasicBlock::iterator insertOutlinedCall(Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It, MachineFunction &MF, - const outliner::Candidate &C) const { + outliner::Candidate &C) const { llvm_unreachable( "Target didn't implement TargetInstrInfo::insertOutlinedCall!"); } diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp index 7900784964626..153b176ad4189 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp @@ -6534,13 +6534,12 @@ enum MachineOutlinerMBBFlags { UnsafeRegsDead = 0x8 }; -unsigned -AArch64InstrInfo::findRegisterToSaveLRTo(const outliner::Candidate &C) const { - assert(C.LRUWasSet && "LRU wasn't set?"); +Register +AArch64InstrInfo::findRegisterToSaveLRTo(outliner::Candidate &C) const { MachineFunction *MF = C.getMF(); - const AArch64RegisterInfo *ARI = static_cast( - MF->getSubtarget().getRegisterInfo()); - + const TargetRegisterInfo &TRI = *MF->getSubtarget().getRegisterInfo(); + const AArch64RegisterInfo *ARI = + static_cast(&TRI); // Check if there is an available register across the sequence that we can // use. for (unsigned Reg : AArch64::GPR64RegClass) { @@ -6548,12 +6547,11 @@ AArch64InstrInfo::findRegisterToSaveLRTo(const outliner::Candidate &C) const { Reg != AArch64::LR && // LR is not reserved, but don't use it. Reg != AArch64::X16 && // X16 is not guaranteed to be preserved. Reg != AArch64::X17 && // Ditto for X17. - C.LRU.available(Reg) && C.UsedInSequence.available(Reg)) + C.isAvailableAcrossAndOutOfSeq(Reg, TRI) && + C.isAvailableInsideSeq(Reg, TRI)) return Reg; } - - // No suitable register. Return 0. - return 0u; + return Register(); } static bool @@ -6720,10 +6718,8 @@ outliner::OutlinedFunction AArch64InstrInfo::getOutliningCandidateInfo( // to compute liveness here. if (C.Flags & UnsafeRegsDead) return false; - C.initLRU(TRI); - LiveRegUnits LRU = C.LRU; - return (!LRU.available(AArch64::W16) || !LRU.available(AArch64::W17) || - !LRU.available(AArch64::NZCV)); + return C.isAnyUnavailableAcrossOrOutOfSeq( + {AArch64::W16, AArch64::W17, AArch64::NZCV}, TRI); }; // Are there any candidates where those registers are live? @@ -6868,8 +6864,6 @@ outliner::OutlinedFunction AArch64InstrInfo::getOutliningCandidateInfo( // Check if we have to save LR. for (outliner::Candidate &C : RepeatedSequenceLocs) { - C.initLRU(TRI); - // If we have a noreturn caller, then we're going to be conservative and // say that we have to save LR. If we don't have a ret at the end of the // block, then we can't reason about liveness accurately. @@ -6880,7 +6874,7 @@ outliner::OutlinedFunction AArch64InstrInfo::getOutliningCandidateInfo( C.getMF()->getFunction().hasFnAttribute(Attribute::NoReturn); // Is LR available? If so, we don't need a save. - if (C.LRU.available(AArch64::LR) && !IsNoReturn) { + if (C.isAvailableAcrossAndOutOfSeq(AArch64::LR, TRI) && !IsNoReturn) { NumBytesNoStackCalls += 4; C.setCallInfo(MachineOutlinerNoLRSave, 4); CandidatesWithoutStackFixups.push_back(C); @@ -6896,7 +6890,7 @@ outliner::OutlinedFunction AArch64InstrInfo::getOutliningCandidateInfo( // Is SP used in the sequence at all? If not, we don't have to modify // the stack, so we are guaranteed to get the same frame. - else if (C.UsedInSequence.available(AArch64::SP)) { + else if (C.isAvailableInsideSeq(AArch64::SP, TRI)) { NumBytesNoStackCalls += 12; C.setCallInfo(MachineOutlinerDefault, 12); CandidatesWithoutStackFixups.push_back(C); @@ -6965,11 +6959,12 @@ outliner::OutlinedFunction AArch64InstrInfo::getOutliningCandidateInfo( // LR to (ie one extra stack save/restore). // if (FlagsSetInAll & MachineOutlinerMBBFlags::HasCalls) { - erase_if(RepeatedSequenceLocs, [this](outliner::Candidate &C) { + erase_if(RepeatedSequenceLocs, [this, &TRI](outliner::Candidate &C) { return (std::any_of( C.front(), std::next(C.back()), [](const MachineInstr &MI) { return MI.isCall(); })) && - (!C.LRU.available(AArch64::LR) || !findRegisterToSaveLRTo(C)); + (!C.isAvailableAcrossAndOutOfSeq(AArch64::LR, TRI) || + !findRegisterToSaveLRTo(C)); }); } } @@ -7503,7 +7498,7 @@ void AArch64InstrInfo::buildOutlinedFrame( MachineBasicBlock::iterator AArch64InstrInfo::insertOutlinedCall( Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It, - MachineFunction &MF, const outliner::Candidate &C) const { + MachineFunction &MF, outliner::Candidate &C) const { // Are we tail calling? if (C.CallConstructionID == MachineOutlinerTailCall) { @@ -7534,8 +7529,8 @@ MachineBasicBlock::iterator AArch64InstrInfo::insertOutlinedCall( if (C.CallConstructionID == MachineOutlinerRegSave) { // FIXME: This logic should be sunk into a target-specific interface so that // we don't have to recompute the register. - unsigned Reg = findRegisterToSaveLRTo(C); - assert(Reg != 0 && "No callee-saved register available?"); + Register Reg = findRegisterToSaveLRTo(C); + assert(Reg && "No callee-saved register available?"); // LR has to be a live in so that we can save it. if (!MBB.isLiveIn(AArch64::LR)) diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/llvm/lib/Target/AArch64/AArch64InstrInfo.h index 1054bea40e685..75d2eb0169ea9 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.h +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.h @@ -283,7 +283,7 @@ class AArch64InstrInfo final : public AArch64GenInstrInfo { MachineBasicBlock::iterator insertOutlinedCall(Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It, MachineFunction &MF, - const outliner::Candidate &C) const override; + outliner::Candidate &C) const override; bool shouldOutlineFromFunctionByDefault(MachineFunction &MF) const override; /// Returns the vector element size (B, H, S or D) of an SVE opcode. uint64_t getElementSizeForOpcode(unsigned Opc) const; @@ -347,7 +347,7 @@ class AArch64InstrInfo final : public AArch64GenInstrInfo { /// Returns an unused general-purpose register which can be used for /// constructing an outlined call if one exists. Returns 0 otherwise. - unsigned findRegisterToSaveLRTo(const outliner::Candidate &C) const; + Register findRegisterToSaveLRTo(outliner::Candidate &C) const; /// Remove a ptest of a predicate-generating operation that already sets, or /// can be made to set, the condition codes in an identical manner diff --git a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp index 5b0bae4d92740..bac68183b7af1 100644 --- a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp +++ b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp @@ -5766,26 +5766,25 @@ struct OutlinerCosts { SaveRestoreLROnStack(target.isThumb() ? 8 : 8) {} }; -unsigned -ARMBaseInstrInfo::findRegisterToSaveLRTo(const outliner::Candidate &C) const { - assert(C.LRUWasSet && "LRU wasn't set?"); +Register +ARMBaseInstrInfo::findRegisterToSaveLRTo(outliner::Candidate &C) const { MachineFunction *MF = C.getMF(); - const ARMBaseRegisterInfo *ARI = static_cast( - MF->getSubtarget().getRegisterInfo()); + const TargetRegisterInfo &TRI = *MF->getSubtarget().getRegisterInfo(); + const ARMBaseRegisterInfo *ARI = + static_cast(&TRI); BitVector regsReserved = ARI->getReservedRegs(*MF); // Check if there is an available register across the sequence that we can // use. - for (unsigned Reg : ARM::rGPRRegClass) { + for (Register Reg : ARM::rGPRRegClass) { if (!(Reg < regsReserved.size() && regsReserved.test(Reg)) && Reg != ARM::LR && // LR is not reserved, but don't use it. Reg != ARM::R12 && // R12 is not guaranteed to be preserved. - C.LRU.available(Reg) && C.UsedInSequence.available(Reg)) + C.isAvailableAcrossAndOutOfSeq(Reg, TRI) && + C.isAvailableInsideSeq(Reg, TRI)) return Reg; } - - // No suitable register. Return 0. - return 0u; + return Register(); } // Compute liveness of LR at the point after the interval [I, E), which @@ -5854,9 +5853,7 @@ outliner::OutlinedFunction ARMBaseInstrInfo::getOutliningCandidateInfo( // to compute liveness here. if (C.Flags & UnsafeRegsDead) return false; - C.initLRU(TRI); - LiveRegUnits LRU = C.LRU; - return (!LRU.available(ARM::R12) || !LRU.available(ARM::CPSR)); + return C.isAnyUnavailableAcrossOrOutOfSeq({ARM::R12, ARM::CPSR}, TRI); }; // Are there any candidates where those registers are live? @@ -5969,7 +5966,6 @@ outliner::OutlinedFunction ARMBaseInstrInfo::getOutliningCandidateInfo( std::vector CandidatesWithoutStackFixups; for (outliner::Candidate &C : RepeatedSequenceLocs) { - C.initLRU(TRI); // LR liveness is overestimated in return blocks, unless they end with a // tail call. const auto Last = C.getMBB()->rbegin(); @@ -5977,7 +5973,7 @@ outliner::OutlinedFunction ARMBaseInstrInfo::getOutliningCandidateInfo( C.getMBB()->isReturnBlock() && !Last->isCall() ? isLRAvailable(TRI, Last, (MachineBasicBlock::reverse_iterator)C.front()) - : C.LRU.available(ARM::LR); + : C.isAvailableAcrossAndOutOfSeq(ARM::LR, TRI); if (LRIsAvailable) { FrameID = MachineOutlinerNoLRSave; NumBytesNoStackCalls += Costs.CallNoLRSave; @@ -5996,7 +5992,7 @@ outliner::OutlinedFunction ARMBaseInstrInfo::getOutliningCandidateInfo( // Is SP used in the sequence at all? If not, we don't have to modify // the stack, so we are guaranteed to get the same frame. - else if (C.UsedInSequence.available(ARM::SP)) { + else if (C.isAvailableInsideSeq(ARM::SP, TRI)) { NumBytesNoStackCalls += Costs.CallDefault; C.setCallInfo(MachineOutlinerDefault, Costs.CallDefault); CandidatesWithoutStackFixups.push_back(C); @@ -6635,7 +6631,7 @@ void ARMBaseInstrInfo::buildOutlinedFrame( MachineBasicBlock::iterator ARMBaseInstrInfo::insertOutlinedCall( Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It, - MachineFunction &MF, const outliner::Candidate &C) const { + MachineFunction &MF, outliner::Candidate &C) const { MachineInstrBuilder MIB; MachineBasicBlock::iterator CallPt; unsigned Opc; @@ -6725,4 +6721,3 @@ unsigned llvm::getBLXpredOpcode(const MachineFunction &MF) { return (MF.getSubtarget().hardenSlsBlr()) ? ARM::BLX_pred_noip : ARM::BLX_pred; } - diff --git a/llvm/lib/Target/ARM/ARMBaseInstrInfo.h b/llvm/lib/Target/ARM/ARMBaseInstrInfo.h index defce07dd8625..ab9643592724d 100644 --- a/llvm/lib/Target/ARM/ARMBaseInstrInfo.h +++ b/llvm/lib/Target/ARM/ARMBaseInstrInfo.h @@ -360,7 +360,7 @@ class ARMBaseInstrInfo : public ARMGenInstrInfo { MachineBasicBlock::iterator insertOutlinedCall(Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It, MachineFunction &MF, - const outliner::Candidate &C) const override; + outliner::Candidate &C) const override; /// Enable outlining by default at -Oz. bool shouldOutlineFromFunctionByDefault(MachineFunction &MF) const override; @@ -375,7 +375,7 @@ class ARMBaseInstrInfo : public ARMGenInstrInfo { private: /// Returns an unused general-purpose register which can be used for /// constructing an outlined call if one exists. Returns 0 otherwise. - unsigned findRegisterToSaveLRTo(const outliner::Candidate &C) const; + Register findRegisterToSaveLRTo(outliner::Candidate &C) const; /// Adds an instruction which saves the link register on top of the stack into /// the MachineBasicBlock \p MBB at position \p It. If \p Auth is true, diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp index 4f39b9c8a9829..eb7825819e691 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp @@ -1318,7 +1318,7 @@ void RISCVInstrInfo::buildOutlinedFrame( MachineBasicBlock::iterator RISCVInstrInfo::insertOutlinedCall( Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It, - MachineFunction &MF, const outliner::Candidate &C) const { + MachineFunction &MF, outliner::Candidate &C) const { // Add in a call instruction to the outlined function at the given location. It = MBB.insert(It, diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.h b/llvm/lib/Target/RISCV/RISCVInstrInfo.h index da0877c4299ae..2697e4a04b4e4 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfo.h +++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.h @@ -153,7 +153,7 @@ class RISCVInstrInfo : public RISCVGenInstrInfo { virtual MachineBasicBlock::iterator insertOutlinedCall(Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It, MachineFunction &MF, - const outliner::Candidate &C) const override; + outliner::Candidate &C) const override; bool findCommutedOpIndices(const MachineInstr &MI, unsigned &SrcOpIdx1, unsigned &SrcOpIdx2) const override; diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp index d2b4221725116..2b3e0b122bf66 100644 --- a/llvm/lib/Target/X86/X86InstrInfo.cpp +++ b/llvm/lib/Target/X86/X86InstrInfo.cpp @@ -9439,7 +9439,7 @@ MachineBasicBlock::iterator X86InstrInfo::insertOutlinedCall(Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It, MachineFunction &MF, - const outliner::Candidate &C) const { + outliner::Candidate &C) const { // Is it a tail call? if (C.CallConstructionID == MachineOutlinerTailCall) { // Yes, just insert a JMP. diff --git a/llvm/lib/Target/X86/X86InstrInfo.h b/llvm/lib/Target/X86/X86InstrInfo.h index 33ce55bbdb2bf..9b0349668c1bb 100644 --- a/llvm/lib/Target/X86/X86InstrInfo.h +++ b/llvm/lib/Target/X86/X86InstrInfo.h @@ -552,7 +552,7 @@ class X86InstrInfo final : public X86GenInstrInfo { MachineBasicBlock::iterator insertOutlinedCall(Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It, MachineFunction &MF, - const outliner::Candidate &C) const override; + outliner::Candidate &C) const override; #define GET_INSTRINFO_HELPER_DECLS #include "X86GenInstrInfo.inc"