diff --git a/llvm/include/llvm/Analysis/BlockFrequencyInfo.h b/llvm/include/llvm/Analysis/BlockFrequencyInfo.h index 39507570a1b2c..fa141b7a0ff90 100644 --- a/llvm/include/llvm/Analysis/BlockFrequencyInfo.h +++ b/llvm/include/llvm/Analysis/BlockFrequencyInfo.h @@ -73,19 +73,19 @@ class BlockFrequencyInfo { /// Returns the estimated profile count of \p Freq. /// This uses the frequency \p Freq and multiplies it by /// the enclosing function's count (if available) and returns the value. - std::optional getProfileCountFromFreq(uint64_t Freq) const; + std::optional getProfileCountFromFreq(BlockFrequency Freq) const; /// Returns true if \p BB is an irreducible loop header /// block. Otherwise false. bool isIrrLoopHeader(const BasicBlock *BB); // Set the frequency of the given basic block. - void setBlockFreq(const BasicBlock *BB, uint64_t Freq); + void setBlockFreq(const BasicBlock *BB, BlockFrequency Freq); /// Set the frequency of \p ReferenceBB to \p Freq and scale the frequencies /// of the blocks in \p BlocksToScale such that their frequencies relative /// to \p ReferenceBB remain unchanged. - void setBlockFreqAndScale(const BasicBlock *ReferenceBB, uint64_t Freq, + void setBlockFreqAndScale(const BasicBlock *ReferenceBB, BlockFrequency Freq, SmallPtrSetImpl &BlocksToScale); /// calculate - compute block frequency info for the given function. @@ -94,13 +94,13 @@ class BlockFrequencyInfo { // Print the block frequency Freq to OS using the current functions entry // frequency to convert freq into a relative decimal form. - raw_ostream &printBlockFreq(raw_ostream &OS, const BlockFrequency Freq) const; + raw_ostream &printBlockFreq(raw_ostream &OS, BlockFrequency Freq) const; // Convenience method that attempts to look up the frequency associated with // BB and print it to OS. raw_ostream &printBlockFreq(raw_ostream &OS, const BasicBlock *BB) const; - uint64_t getEntryFreq() const; + BlockFrequency getEntryFreq() const; void releaseMemory(); void print(raw_ostream &OS) const; diff --git a/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h b/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h index 54d56f8472c2b..b9c0e27592275 100644 --- a/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h +++ b/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h @@ -527,19 +527,18 @@ class BlockFrequencyInfoImplBase { getBlockProfileCount(const Function &F, const BlockNode &Node, bool AllowSynthetic = false) const; std::optional - getProfileCountFromFreq(const Function &F, uint64_t Freq, + getProfileCountFromFreq(const Function &F, BlockFrequency Freq, bool AllowSynthetic = false) const; bool isIrrLoopHeader(const BlockNode &Node); - void setBlockFreq(const BlockNode &Node, uint64_t Freq); + void setBlockFreq(const BlockNode &Node, BlockFrequency Freq); raw_ostream &printBlockFreq(raw_ostream &OS, const BlockNode &Node) const; - raw_ostream &printBlockFreq(raw_ostream &OS, - const BlockFrequency &Freq) const; + raw_ostream &printBlockFreq(raw_ostream &OS, BlockFrequency Freq) const; - uint64_t getEntryFreq() const { + BlockFrequency getEntryFreq() const { assert(!Freqs.empty()); - return Freqs[0].Integer; + return BlockFrequency(Freqs[0].Integer); } }; @@ -1029,7 +1028,7 @@ template class BlockFrequencyInfoImpl : BlockFrequencyInfoImplBase { } std::optional - getProfileCountFromFreq(const Function &F, uint64_t Freq, + getProfileCountFromFreq(const Function &F, BlockFrequency Freq, bool AllowSynthetic = false) const { return BlockFrequencyInfoImplBase::getProfileCountFromFreq(F, Freq, AllowSynthetic); @@ -1039,7 +1038,7 @@ template class BlockFrequencyInfoImpl : BlockFrequencyInfoImplBase { return BlockFrequencyInfoImplBase::isIrrLoopHeader(getNode(BB)); } - void setBlockFreq(const BlockT *BB, uint64_t Freq); + void setBlockFreq(const BlockT *BB, BlockFrequency Freq); void forgetBlock(const BlockT *BB) { // We don't erase corresponding items from `Freqs`, `RPOT` and other to @@ -1145,12 +1144,13 @@ void BlockFrequencyInfoImpl::calculate(const FunctionT &F, // blocks and unknown blocks. for (const BlockT &BB : F) if (!Nodes.count(&BB)) - setBlockFreq(&BB, 0); + setBlockFreq(&BB, BlockFrequency()); } } template -void BlockFrequencyInfoImpl::setBlockFreq(const BlockT *BB, uint64_t Freq) { +void BlockFrequencyInfoImpl::setBlockFreq(const BlockT *BB, + BlockFrequency Freq) { if (Nodes.count(BB)) BlockFrequencyInfoImplBase::setBlockFreq(getNode(BB), Freq); else { diff --git a/llvm/include/llvm/Analysis/ProfileSummaryInfo.h b/llvm/include/llvm/Analysis/ProfileSummaryInfo.h index 38eb71ba271d0..e49538bfaf80f 100644 --- a/llvm/include/llvm/Analysis/ProfileSummaryInfo.h +++ b/llvm/include/llvm/Analysis/ProfileSummaryInfo.h @@ -209,7 +209,7 @@ class ProfileSummaryInfo { template bool isColdBlock(BlockFrequency BlockFreq, const BFIT *BFI) const { - auto Count = BFI->getProfileCountFromFreq(BlockFreq.getFrequency()); + auto Count = BFI->getProfileCountFromFreq(BlockFreq); return Count && isColdCount(*Count); } @@ -315,7 +315,7 @@ class ProfileSummaryInfo { bool isHotOrColdBlockNthPercentile(int PercentileCutoff, BlockFrequency BlockFreq, BFIT *BFI) const { - auto Count = BFI->getProfileCountFromFreq(BlockFreq.getFrequency()); + auto Count = BFI->getProfileCountFromFreq(BlockFreq); if (isHot) return Count && isHotCountNthPercentile(PercentileCutoff, *Count); else diff --git a/llvm/include/llvm/CodeGen/GlobalISel/RegBankSelect.h b/llvm/include/llvm/CodeGen/GlobalISel/RegBankSelect.h index 3f85d22be5339..e4d9ff522b5a9 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/RegBankSelect.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/RegBankSelect.h @@ -440,7 +440,7 @@ class RegBankSelect : public MachineFunctionPass { public: /// Create a MappingCost assuming that most of the instructions /// will occur in a basic block with \p LocalFreq frequency. - MappingCost(const BlockFrequency &LocalFreq); + MappingCost(BlockFrequency LocalFreq); /// Add \p Cost to the local cost. /// \return true if this cost is saturated, false otherwise. diff --git a/llvm/include/llvm/CodeGen/MBFIWrapper.h b/llvm/include/llvm/CodeGen/MBFIWrapper.h index 714ecc5d4334e..ae0ae25425f34 100644 --- a/llvm/include/llvm/CodeGen/MBFIWrapper.h +++ b/llvm/include/llvm/CodeGen/MBFIWrapper.h @@ -35,10 +35,9 @@ class MBFIWrapper { raw_ostream &printBlockFreq(raw_ostream &OS, const MachineBasicBlock *MBB) const; - raw_ostream &printBlockFreq(raw_ostream &OS, - const BlockFrequency Freq) const; + raw_ostream &printBlockFreq(raw_ostream &OS, BlockFrequency Freq) const; void view(const Twine &Name, bool isSimple = true); - uint64_t getEntryFreq() const; + BlockFrequency getEntryFreq() const; const MachineBlockFrequencyInfo &getMBFI() { return MBFI; } private: diff --git a/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h b/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h index 1152eefed6e45..40ef3f76612ae 100644 --- a/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h +++ b/llvm/include/llvm/CodeGen/MachineBlockFrequencyInfo.h @@ -66,14 +66,15 @@ class MachineBlockFrequencyInfo : public MachineFunctionPass { /// Compute the frequency of the block, relative to the entry block. /// This API assumes getEntryFreq() is non-zero. double getBlockFreqRelativeToEntryBlock(const MachineBasicBlock *MBB) const { - assert(getEntryFreq() != 0 && "getEntryFreq() should not return 0 here!"); + assert(getEntryFreq() != BlockFrequency(0) && + "getEntryFreq() should not return 0 here!"); return static_cast(getBlockFreq(MBB).getFrequency()) / - static_cast(getEntryFreq()); + static_cast(getEntryFreq().getFrequency()); } std::optional getBlockProfileCount(const MachineBasicBlock *MBB) const; - std::optional getProfileCountFromFreq(uint64_t Freq) const; + std::optional getProfileCountFromFreq(BlockFrequency Freq) const; bool isIrrLoopHeader(const MachineBasicBlock *MBB) const; @@ -101,7 +102,7 @@ class MachineBlockFrequencyInfo : public MachineFunctionPass { /// Divide a block's BlockFrequency::getFrequency() value by this value to /// obtain the entry block - relative frequency of said block. - uint64_t getEntryFreq() const; + BlockFrequency getEntryFreq() const; }; } // end namespace llvm diff --git a/llvm/include/llvm/Support/BlockFrequency.h b/llvm/include/llvm/Support/BlockFrequency.h index 12a753301b36a..8b172ee486aab 100644 --- a/llvm/include/llvm/Support/BlockFrequency.h +++ b/llvm/include/llvm/Support/BlockFrequency.h @@ -26,10 +26,11 @@ class BlockFrequency { uint64_t Frequency; public: - BlockFrequency(uint64_t Freq = 0) : Frequency(Freq) { } + BlockFrequency() : Frequency(0) {} + explicit BlockFrequency(uint64_t Freq) : Frequency(Freq) {} /// Returns the maximum possible frequency, the saturation value. - static uint64_t getMaxFrequency() { return UINT64_MAX; } + static BlockFrequency max() { return BlockFrequency(UINT64_MAX); } /// Returns the frequency as a fixpoint number scaled by the entry /// frequency. @@ -112,6 +113,10 @@ class BlockFrequency { bool operator==(BlockFrequency RHS) const { return Frequency == RHS.Frequency; } + + bool operator!=(BlockFrequency RHS) const { + return Frequency != RHS.Frequency; + } }; } // namespace llvm diff --git a/llvm/include/llvm/Transforms/Instrumentation/CFGMST.h b/llvm/include/llvm/Transforms/Instrumentation/CFGMST.h index 4d31898bb3147..269441db7a558 100644 --- a/llvm/include/llvm/Transforms/Instrumentation/CFGMST.h +++ b/llvm/include/llvm/Transforms/Instrumentation/CFGMST.h @@ -101,7 +101,8 @@ template class CFGMST { LLVM_DEBUG(dbgs() << "Build Edge on " << F.getName() << "\n"); BasicBlock *Entry = &(F.getEntryBlock()); - uint64_t EntryWeight = (BFI != nullptr ? BFI->getEntryFreq() : 2); + uint64_t EntryWeight = + (BFI != nullptr ? BFI->getEntryFreq().getFrequency() : 2); // If we want to instrument the entry count, lower the weight to 0. if (InstrumentFuncEntry) EntryWeight = 0; diff --git a/llvm/lib/Analysis/BlockFrequencyInfo.cpp b/llvm/lib/Analysis/BlockFrequencyInfo.cpp index b18d04cc73dbc..e9262bbb833c9 100644 --- a/llvm/lib/Analysis/BlockFrequencyInfo.cpp +++ b/llvm/lib/Analysis/BlockFrequencyInfo.cpp @@ -201,7 +201,7 @@ void BlockFrequencyInfo::calculate(const Function &F, } BlockFrequency BlockFrequencyInfo::getBlockFreq(const BasicBlock *BB) const { - return BFI ? BFI->getBlockFreq(BB) : 0; + return BFI ? BFI->getBlockFreq(BB) : BlockFrequency(0); } std::optional @@ -214,7 +214,7 @@ BlockFrequencyInfo::getBlockProfileCount(const BasicBlock *BB, } std::optional -BlockFrequencyInfo::getProfileCountFromFreq(uint64_t Freq) const { +BlockFrequencyInfo::getProfileCountFromFreq(BlockFrequency Freq) const { if (!BFI) return std::nullopt; return BFI->getProfileCountFromFreq(*getFunction(), Freq); @@ -225,17 +225,18 @@ bool BlockFrequencyInfo::isIrrLoopHeader(const BasicBlock *BB) { return BFI->isIrrLoopHeader(BB); } -void BlockFrequencyInfo::setBlockFreq(const BasicBlock *BB, uint64_t Freq) { +void BlockFrequencyInfo::setBlockFreq(const BasicBlock *BB, + BlockFrequency Freq) { assert(BFI && "Expected analysis to be available"); BFI->setBlockFreq(BB, Freq); } void BlockFrequencyInfo::setBlockFreqAndScale( - const BasicBlock *ReferenceBB, uint64_t Freq, + const BasicBlock *ReferenceBB, BlockFrequency Freq, SmallPtrSetImpl &BlocksToScale) { assert(BFI && "Expected analysis to be available"); // Use 128 bits APInt to avoid overflow. - APInt NewFreq(128, Freq); + APInt NewFreq(128, Freq.getFrequency()); APInt OldFreq(128, BFI->getBlockFreq(ReferenceBB).getFrequency()); APInt BBFreq(128, 0); for (auto *BB : BlocksToScale) { @@ -247,7 +248,7 @@ void BlockFrequencyInfo::setBlockFreqAndScale( // a hot spot, one of the options proposed in // https://reviews.llvm.org/D28535#650071 could be used to avoid this. BBFreq = BBFreq.udiv(OldFreq); - BFI->setBlockFreq(BB, BBFreq.getLimitedValue()); + BFI->setBlockFreq(BB, BlockFrequency(BBFreq.getLimitedValue())); } BFI->setBlockFreq(ReferenceBB, Freq); } @@ -266,8 +267,8 @@ const BranchProbabilityInfo *BlockFrequencyInfo::getBPI() const { return BFI ? &BFI->getBPI() : nullptr; } -raw_ostream &BlockFrequencyInfo:: -printBlockFreq(raw_ostream &OS, const BlockFrequency Freq) const { +raw_ostream &BlockFrequencyInfo::printBlockFreq(raw_ostream &OS, + BlockFrequency Freq) const { return BFI ? BFI->printBlockFreq(OS, Freq) : OS; } @@ -277,8 +278,8 @@ BlockFrequencyInfo::printBlockFreq(raw_ostream &OS, return BFI ? BFI->printBlockFreq(OS, BB) : OS; } -uint64_t BlockFrequencyInfo::getEntryFreq() const { - return BFI ? BFI->getEntryFreq() : 0; +BlockFrequency BlockFrequencyInfo::getEntryFreq() const { + return BFI ? BFI->getEntryFreq() : BlockFrequency(0); } void BlockFrequencyInfo::releaseMemory() { BFI.reset(); } diff --git a/llvm/lib/Analysis/BlockFrequencyInfoImpl.cpp b/llvm/lib/Analysis/BlockFrequencyInfoImpl.cpp index 82b1e3b9eede7..a4400799161f8 100644 --- a/llvm/lib/Analysis/BlockFrequencyInfoImpl.cpp +++ b/llvm/lib/Analysis/BlockFrequencyInfoImpl.cpp @@ -581,30 +581,27 @@ BlockFrequencyInfoImplBase::getBlockFreq(const BlockNode &Node) const { report_fatal_error(OS.str()); } #endif - return 0; + return BlockFrequency(0); } - return Freqs[Node.Index].Integer; + return BlockFrequency(Freqs[Node.Index].Integer); } std::optional BlockFrequencyInfoImplBase::getBlockProfileCount(const Function &F, const BlockNode &Node, bool AllowSynthetic) const { - return getProfileCountFromFreq(F, getBlockFreq(Node).getFrequency(), - AllowSynthetic); + return getProfileCountFromFreq(F, getBlockFreq(Node), AllowSynthetic); } -std::optional -BlockFrequencyInfoImplBase::getProfileCountFromFreq(const Function &F, - uint64_t Freq, - bool AllowSynthetic) const { +std::optional BlockFrequencyInfoImplBase::getProfileCountFromFreq( + const Function &F, BlockFrequency Freq, bool AllowSynthetic) const { auto EntryCount = F.getEntryCount(AllowSynthetic); if (!EntryCount) return std::nullopt; // Use 128 bit APInt to do the arithmetic to avoid overflow. APInt BlockCount(128, EntryCount->getCount()); - APInt BlockFreq(128, Freq); - APInt EntryFreq(128, getEntryFreq()); + APInt BlockFreq(128, Freq.getFrequency()); + APInt EntryFreq(128, getEntryFreq().getFrequency()); BlockCount *= BlockFreq; // Rounded division of BlockCount by EntryFreq. Since EntryFreq is unsigned // lshr by 1 gives EntryFreq/2. @@ -627,10 +624,10 @@ BlockFrequencyInfoImplBase::getFloatingBlockFreq(const BlockNode &Node) const { } void BlockFrequencyInfoImplBase::setBlockFreq(const BlockNode &Node, - uint64_t Freq) { + BlockFrequency Freq) { assert(Node.isValid() && "Expected valid node"); assert(Node.Index < Freqs.size() && "Expected legal index"); - Freqs[Node.Index].Integer = Freq; + Freqs[Node.Index].Integer = Freq.getFrequency(); } std::string @@ -651,9 +648,9 @@ BlockFrequencyInfoImplBase::printBlockFreq(raw_ostream &OS, raw_ostream & BlockFrequencyInfoImplBase::printBlockFreq(raw_ostream &OS, - const BlockFrequency &Freq) const { + BlockFrequency Freq) const { Scaled64 Block(Freq.getFrequency(), 0); - Scaled64 Entry(getEntryFreq(), 0); + Scaled64 Entry(getEntryFreq().getFrequency(), 0); return OS << Block / Entry; } diff --git a/llvm/lib/Analysis/CFGPrinter.cpp b/llvm/lib/Analysis/CFGPrinter.cpp index f05dd6852d6dc..9f55371f259b2 100644 --- a/llvm/lib/Analysis/CFGPrinter.cpp +++ b/llvm/lib/Analysis/CFGPrinter.cpp @@ -318,10 +318,11 @@ bool DOTGraphTraits::isNodeHidden(const BasicBlock *Node, const DOTFuncInfo *CFGInfo) { if (HideColdPaths.getNumOccurrences() > 0) if (auto *BFI = CFGInfo->getBFI()) { - uint64_t NodeFreq = BFI->getBlockFreq(Node).getFrequency(); - uint64_t EntryFreq = BFI->getEntryFreq(); + BlockFrequency NodeFreq = BFI->getBlockFreq(Node); + BlockFrequency EntryFreq = BFI->getEntryFreq(); // Hide blocks with relative frequency below HideColdPaths threshold. - if ((double)NodeFreq / EntryFreq < HideColdPaths) + if ((double)NodeFreq.getFrequency() / EntryFreq.getFrequency() < + HideColdPaths) return true; } if (HideUnreachablePaths || HideDeoptimizePaths) { diff --git a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp index a88622efa12db..058a107691674 100644 --- a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp +++ b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp @@ -413,7 +413,7 @@ static void computeFunctionSummary( // information. if (BFI != nullptr && Hotness == CalleeInfo::HotnessType::Unknown) { uint64_t BBFreq = BFI->getBlockFreq(&BB).getFrequency(); - uint64_t EntryFreq = BFI->getEntryFreq(); + uint64_t EntryFreq = BFI->getEntryFreq().getFrequency(); ValueInfo.updateRelBlockFreq(BBFreq, EntryFreq); } } else { diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp index e31b08df7dbbe..371f6598e6b2b 100644 --- a/llvm/lib/CodeGen/CodeGenPrepare.cpp +++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp @@ -2591,9 +2591,8 @@ bool CodeGenPrepare::dupRetToEnableTailCallOpts(BasicBlock *BB, (void)FoldReturnIntoUncondBranch(RetI, BB, TailCallBB); assert(!VerifyBFIUpdates || BFI->getBlockFreq(BB) >= BFI->getBlockFreq(TailCallBB)); - BFI->setBlockFreq( - BB, - (BFI->getBlockFreq(BB) - BFI->getBlockFreq(TailCallBB)).getFrequency()); + BFI->setBlockFreq(BB, + (BFI->getBlockFreq(BB) - BFI->getBlockFreq(TailCallBB))); ModifiedDT = ModifyDT::ModifyBBDT; Changed = true; ++NumRetsDup; @@ -7067,7 +7066,7 @@ bool CodeGenPrepare::optimizeSelectInst(SelectInst *SI) { FreshBBs.insert(EndBlock); } - BFI->setBlockFreq(EndBlock, BFI->getBlockFreq(StartBlock).getFrequency()); + BFI->setBlockFreq(EndBlock, BFI->getBlockFreq(StartBlock)); static const unsigned MD[] = { LLVMContext::MD_prof, LLVMContext::MD_unpredictable, diff --git a/llvm/lib/CodeGen/GlobalISel/RegBankSelect.cpp b/llvm/lib/CodeGen/GlobalISel/RegBankSelect.cpp index d201342cd61db..bb5363fb2527b 100644 --- a/llvm/lib/CodeGen/GlobalISel/RegBankSelect.cpp +++ b/llvm/lib/CodeGen/GlobalISel/RegBankSelect.cpp @@ -449,7 +449,8 @@ RegBankSelect::MappingCost RegBankSelect::computeMapping( return MappingCost::ImpossibleCost(); // If mapped with InstrMapping, MI will have the recorded cost. - MappingCost Cost(MBFI ? MBFI->getBlockFreq(MI.getParent()) : 1); + MappingCost Cost(MBFI ? MBFI->getBlockFreq(MI.getParent()) + : BlockFrequency(1)); bool Saturated = Cost.addLocalCost(InstrMapping.getCost()); assert(!Saturated && "Possible mapping saturated the cost"); LLVM_DEBUG(dbgs() << "Evaluating mapping cost for: " << MI); @@ -971,7 +972,7 @@ bool RegBankSelect::EdgeInsertPoint::canMaterialize() const { return Src.canSplitCriticalEdge(DstOrSplit); } -RegBankSelect::MappingCost::MappingCost(const BlockFrequency &LocalFreq) +RegBankSelect::MappingCost::MappingCost(BlockFrequency LocalFreq) : LocalFreq(LocalFreq.getFrequency()) {} bool RegBankSelect::MappingCost::addLocalCost(uint64_t Cost) { diff --git a/llvm/lib/CodeGen/MBFIWrapper.cpp b/llvm/lib/CodeGen/MBFIWrapper.cpp index 5b388be278394..4f8b921b03181 100644 --- a/llvm/lib/CodeGen/MBFIWrapper.cpp +++ b/llvm/lib/CodeGen/MBFIWrapper.cpp @@ -38,7 +38,7 @@ MBFIWrapper::getBlockProfileCount(const MachineBasicBlock *MBB) const { // Modified block frequency also impacts profile count. So we should compute // profile count from new block frequency if it has been changed. if (I != MergedBBFreq.end()) - return MBFI.getProfileCountFromFreq(I->second.getFrequency()); + return MBFI.getProfileCountFromFreq(I->second); return MBFI.getBlockProfileCount(MBB); } @@ -49,7 +49,7 @@ raw_ostream & MBFIWrapper::printBlockFreq(raw_ostream &OS, } raw_ostream & MBFIWrapper::printBlockFreq(raw_ostream &OS, - const BlockFrequency Freq) const { + BlockFrequency Freq) const { return MBFI.printBlockFreq(OS, Freq); } @@ -57,6 +57,4 @@ void MBFIWrapper::view(const Twine &Name, bool isSimple) { MBFI.view(Name, isSimple); } -uint64_t MBFIWrapper::getEntryFreq() const { - return MBFI.getEntryFreq(); -} +BlockFrequency MBFIWrapper::getEntryFreq() const { return MBFI.getEntryFreq(); } diff --git a/llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp b/llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp index b1cbe525d7e6c..76b7285f154c2 100644 --- a/llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp +++ b/llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp @@ -228,7 +228,7 @@ void MachineBlockFrequencyInfo::view(const Twine &Name, bool isSimple) const { BlockFrequency MachineBlockFrequencyInfo::getBlockFreq(const MachineBasicBlock *MBB) const { - return MBFI ? MBFI->getBlockFreq(MBB) : 0; + return MBFI ? MBFI->getBlockFreq(MBB) : BlockFrequency(0); } std::optional MachineBlockFrequencyInfo::getBlockProfileCount( @@ -241,7 +241,7 @@ std::optional MachineBlockFrequencyInfo::getBlockProfileCount( } std::optional -MachineBlockFrequencyInfo::getProfileCountFromFreq(uint64_t Freq) const { +MachineBlockFrequencyInfo::getProfileCountFromFreq(BlockFrequency Freq) const { if (!MBFI) return std::nullopt; @@ -263,7 +263,7 @@ void MachineBlockFrequencyInfo::onEdgeSplit( auto NewSuccFreq = MBFI->getBlockFreq(&NewPredecessor) * MBPI.getEdgeProbability(&NewPredecessor, &NewSuccessor); - MBFI->setBlockFreq(&NewSuccessor, NewSuccFreq.getFrequency()); + MBFI->setBlockFreq(&NewSuccessor, NewSuccFreq); } const MachineFunction *MachineBlockFrequencyInfo::getFunction() const { @@ -286,6 +286,6 @@ MachineBlockFrequencyInfo::printBlockFreq(raw_ostream &OS, return MBFI ? MBFI->printBlockFreq(OS, MBB) : OS; } -uint64_t MachineBlockFrequencyInfo::getEntryFreq() const { - return MBFI ? MBFI->getEntryFreq() : 0; +BlockFrequency MachineBlockFrequencyInfo::getEntryFreq() const { + return MBFI ? MBFI->getEntryFreq() : BlockFrequency(0); } diff --git a/llvm/lib/CodeGen/MachineBlockPlacement.cpp b/llvm/lib/CodeGen/MachineBlockPlacement.cpp index 603c0e9600afc..9d46c1477095e 100644 --- a/llvm/lib/CodeGen/MachineBlockPlacement.cpp +++ b/llvm/lib/CodeGen/MachineBlockPlacement.cpp @@ -444,9 +444,9 @@ class MachineBlockPlacement : public MachineFunctionPass { if (UseProfileCount) { auto Count = MBFI->getBlockProfileCount(BB); if (Count) - return *Count; + return BlockFrequency(*Count); else - return 0; + return BlockFrequency(0); } else return MBFI->getBlockFreq(BB); } @@ -795,10 +795,10 @@ bool MachineBlockPlacement::shouldTailDuplicate(MachineBasicBlock *BB) { /// penalty is less than 100% /// TODO(iteratee): Use 64-bit fixed point edge frequencies everywhere. static bool greaterWithBias(BlockFrequency A, BlockFrequency B, - uint64_t EntryFreq) { + BlockFrequency EntryFreq) { BranchProbability ThresholdProb(TailDupPlacementPenalty, 100); BlockFrequency Gain = A - B; - return (Gain / ThresholdProb).getFrequency() >= EntryFreq; + return (Gain / ThresholdProb) >= EntryFreq; } /// Check the edge frequencies to see if tail duplication will increase @@ -843,7 +843,7 @@ bool MachineBlockPlacement::isProfitableToTailDup( auto SuccFreq = MBFI->getBlockFreq(Succ); BlockFrequency P = BBFreq * PProb; BlockFrequency Qout = BBFreq * QProb; - uint64_t EntryFreq = MBFI->getEntryFreq(); + BlockFrequency EntryFreq = MBFI->getEntryFreq(); // If there are no more successors, it is profitable to copy, as it strictly // increases fallthrough. if (SuccSuccs.size() == 0) @@ -1927,7 +1927,7 @@ BlockFrequency MachineBlockPlacement::TopFallThroughFreq( const MachineBasicBlock *Top, const BlockFilterSet &LoopBlockSet) { - BlockFrequency MaxFreq = 0; + BlockFrequency MaxFreq = BlockFrequency(0); for (MachineBasicBlock *Pred : Top->predecessors()) { BlockChain *PredChain = BlockToChain[Pred]; if (!LoopBlockSet.count(Pred) && @@ -1986,7 +1986,7 @@ MachineBlockPlacement::FallThroughGains( const MachineBasicBlock *ExitBB, const BlockFilterSet &LoopBlockSet) { BlockFrequency FallThrough2Top = TopFallThroughFreq(OldTop, LoopBlockSet); - BlockFrequency FallThrough2Exit = 0; + BlockFrequency FallThrough2Exit = BlockFrequency(0); if (ExitBB) FallThrough2Exit = MBFI->getBlockFreq(NewTop) * MBPI->getEdgeProbability(NewTop, ExitBB); @@ -1994,58 +1994,58 @@ MachineBlockPlacement::FallThroughGains( MBPI->getEdgeProbability(NewTop, OldTop); // Find the best Pred of NewTop. - MachineBasicBlock *BestPred = nullptr; - BlockFrequency FallThroughFromPred = 0; - for (MachineBasicBlock *Pred : NewTop->predecessors()) { - if (!LoopBlockSet.count(Pred)) - continue; - BlockChain *PredChain = BlockToChain[Pred]; - if (!PredChain || Pred == *std::prev(PredChain->end())) { - BlockFrequency EdgeFreq = MBFI->getBlockFreq(Pred) * - MBPI->getEdgeProbability(Pred, NewTop); - if (EdgeFreq > FallThroughFromPred) { - FallThroughFromPred = EdgeFreq; - BestPred = Pred; - } - } - } - - // If NewTop is not placed after Pred, another successor can be placed - // after Pred. - BlockFrequency NewFreq = 0; - if (BestPred) { - for (MachineBasicBlock *Succ : BestPred->successors()) { - if ((Succ == NewTop) || (Succ == BestPred) || !LoopBlockSet.count(Succ)) - continue; - if (ComputedEdges.contains(Succ)) - continue; - BlockChain *SuccChain = BlockToChain[Succ]; - if ((SuccChain && (Succ != *SuccChain->begin())) || - (SuccChain == BlockToChain[BestPred])) - continue; - BlockFrequency EdgeFreq = MBFI->getBlockFreq(BestPred) * - MBPI->getEdgeProbability(BestPred, Succ); - if (EdgeFreq > NewFreq) - NewFreq = EdgeFreq; - } - BlockFrequency OrigEdgeFreq = MBFI->getBlockFreq(BestPred) * - MBPI->getEdgeProbability(BestPred, NewTop); - if (NewFreq > OrigEdgeFreq) { - // If NewTop is not the best successor of Pred, then Pred doesn't - // fallthrough to NewTop. So there is no FallThroughFromPred and - // NewFreq. - NewFreq = 0; - FallThroughFromPred = 0; - } - } - - BlockFrequency Result = 0; - BlockFrequency Gains = BackEdgeFreq + NewFreq; - BlockFrequency Lost = FallThrough2Top + FallThrough2Exit + - FallThroughFromPred; - if (Gains > Lost) - Result = Gains - Lost; - return Result; + MachineBasicBlock *BestPred = nullptr; + BlockFrequency FallThroughFromPred = BlockFrequency(0); + for (MachineBasicBlock *Pred : NewTop->predecessors()) { + if (!LoopBlockSet.count(Pred)) + continue; + BlockChain *PredChain = BlockToChain[Pred]; + if (!PredChain || Pred == *std::prev(PredChain->end())) { + BlockFrequency EdgeFreq = + MBFI->getBlockFreq(Pred) * MBPI->getEdgeProbability(Pred, NewTop); + if (EdgeFreq > FallThroughFromPred) { + FallThroughFromPred = EdgeFreq; + BestPred = Pred; + } + } + } + + // If NewTop is not placed after Pred, another successor can be placed + // after Pred. + BlockFrequency NewFreq = BlockFrequency(0); + if (BestPred) { + for (MachineBasicBlock *Succ : BestPred->successors()) { + if ((Succ == NewTop) || (Succ == BestPred) || !LoopBlockSet.count(Succ)) + continue; + if (ComputedEdges.contains(Succ)) + continue; + BlockChain *SuccChain = BlockToChain[Succ]; + if ((SuccChain && (Succ != *SuccChain->begin())) || + (SuccChain == BlockToChain[BestPred])) + continue; + BlockFrequency EdgeFreq = MBFI->getBlockFreq(BestPred) * + MBPI->getEdgeProbability(BestPred, Succ); + if (EdgeFreq > NewFreq) + NewFreq = EdgeFreq; + } + BlockFrequency OrigEdgeFreq = MBFI->getBlockFreq(BestPred) * + MBPI->getEdgeProbability(BestPred, NewTop); + if (NewFreq > OrigEdgeFreq) { + // If NewTop is not the best successor of Pred, then Pred doesn't + // fallthrough to NewTop. So there is no FallThroughFromPred and + // NewFreq. + NewFreq = BlockFrequency(0); + FallThroughFromPred = BlockFrequency(0); + } + } + + BlockFrequency Result = BlockFrequency(0); + BlockFrequency Gains = BackEdgeFreq + NewFreq; + BlockFrequency Lost = + FallThrough2Top + FallThrough2Exit + FallThroughFromPred; + if (Gains > Lost) + Result = Gains - Lost; + return Result; } /// Helper function of findBestLoopTop. Find the best loop top block @@ -2087,7 +2087,7 @@ MachineBlockPlacement::findBestLoopTopHelper( LLVM_DEBUG(dbgs() << "Finding best loop top for: " << getBlockName(OldTop) << "\n"); - BlockFrequency BestGains = 0; + BlockFrequency BestGains = BlockFrequency(0); MachineBasicBlock *BestPred = nullptr; for (MachineBasicBlock *Pred : OldTop->predecessors()) { if (!LoopBlockSet.count(Pred)) @@ -2112,8 +2112,9 @@ MachineBlockPlacement::findBestLoopTopHelper( BlockFrequency Gains = FallThroughGains(Pred, OldTop, OtherBB, LoopBlockSet); - if ((Gains > 0) && (Gains > BestGains || - ((Gains == BestGains) && Pred->isLayoutSuccessor(OldTop)))) { + if ((Gains > BlockFrequency(0)) && + (Gains > BestGains || + ((Gains == BestGains) && Pred->isLayoutSuccessor(OldTop)))) { BestPred = Pred; BestGains = Gains; } @@ -2425,14 +2426,14 @@ void MachineBlockPlacement::rotateLoopWithProfile( if (ChainHeaderBB->isEntryBlock()) return; - BlockFrequency SmallestRotationCost = BlockFrequency::getMaxFrequency(); + BlockFrequency SmallestRotationCost = BlockFrequency::max(); // A utility lambda that scales up a block frequency by dividing it by a // branch probability which is the reciprocal of the scale. auto ScaleBlockFrequency = [](BlockFrequency Freq, unsigned Scale) -> BlockFrequency { if (Scale == 0) - return 0; + return BlockFrequency(0); // Use operator / between BlockFrequency and BranchProbability to implement // saturating multiplication. return Freq / BranchProbability(1, Scale); @@ -2492,7 +2493,7 @@ void MachineBlockPlacement::rotateLoopWithProfile( auto TailBB = *TailIter; // Calculate the cost by putting this BB to the top. - BlockFrequency Cost = 0; + BlockFrequency Cost = BlockFrequency(0); // If the current BB is the loop header, we need to take into account the // cost of the missed fall through edge from outside of the loop to the @@ -2523,8 +2524,7 @@ void MachineBlockPlacement::rotateLoopWithProfile( if (TailBB->isSuccessor(*Iter)) { auto TailBBFreq = MBFI->getBlockFreq(TailBB); if (TailBB->succ_size() == 1) - Cost += ScaleBlockFrequency(TailBBFreq.getFrequency(), - MisfetchCost + JumpInstCost); + Cost += ScaleBlockFrequency(TailBBFreq, MisfetchCost + JumpInstCost); else if (TailBB->succ_size() == 2) { auto TailToHeadProb = MBPI->getEdgeProbability(TailBB, *Iter); auto TailToHeadFreq = TailBBFreq * TailToHeadProb; @@ -3159,7 +3159,7 @@ static uint64_t countMBBInstruction(MachineBasicBlock *MBB) { // So we should scale the threshold accordingly. But the instruction size is not // available on all targets, so we use the number of instructions instead. BlockFrequency MachineBlockPlacement::scaleThreshold(MachineBasicBlock *BB) { - return DupThreshold.getFrequency() * countMBBInstruction(BB); + return BlockFrequency(DupThreshold.getFrequency() * countMBBInstruction(BB)); } // Returns true if BB is Pred's best successor. @@ -3313,7 +3313,7 @@ void MachineBlockPlacement::findDuplicateCandidates( } void MachineBlockPlacement::initDupThreshold() { - DupThreshold = 0; + DupThreshold = BlockFrequency(0); if (!F->getFunction().hasProfileData()) return; @@ -3321,12 +3321,13 @@ void MachineBlockPlacement::initDupThreshold() { uint64_t HotThreshold = PSI->getOrCompHotCountThreshold(); if (HotThreshold != UINT64_MAX) { UseProfileCount = true; - DupThreshold = HotThreshold * TailDupProfilePercentThreshold / 100; + DupThreshold = + BlockFrequency(HotThreshold * TailDupProfilePercentThreshold / 100); return; } // Profile count is not available, we can use block frequency instead. - BlockFrequency MaxFreq = 0; + BlockFrequency MaxFreq = BlockFrequency(0); for (MachineBasicBlock &MBB : *F) { BlockFrequency Freq = MBFI->getBlockFreq(&MBB); if (Freq > MaxFreq) @@ -3334,7 +3335,7 @@ void MachineBlockPlacement::initDupThreshold() { } BranchProbability ThresholdProb(TailDupPlacementPenalty, 100); - DupThreshold = MaxFreq * ThresholdProb; + DupThreshold = BlockFrequency(MaxFreq * ThresholdProb); UseProfileCount = false; } diff --git a/llvm/lib/CodeGen/RegAllocGreedy.cpp b/llvm/lib/CodeGen/RegAllocGreedy.cpp index 248cc1ac0ee2e..b8bafc9db924c 100644 --- a/llvm/lib/CodeGen/RegAllocGreedy.cpp +++ b/llvm/lib/CodeGen/RegAllocGreedy.cpp @@ -607,7 +607,7 @@ bool RAGreedy::addSplitConstraints(InterferenceCache::Cursor Intf, // Reset interference dependent info. SplitConstraints.resize(UseBlocks.size()); - BlockFrequency StaticCost = 0; + BlockFrequency StaticCost = BlockFrequency(0); for (unsigned I = 0; I != UseBlocks.size(); ++I) { const SplitAnalysis::BlockInfo &BI = UseBlocks[I]; SpillPlacement::BlockConstraint &BC = SplitConstraints[I]; @@ -832,7 +832,7 @@ bool RAGreedy::calcCompactRegion(GlobalSplitCandidate &Cand) { /// calcSpillCost - Compute how expensive it would be to split the live range in /// SA around all use blocks instead of forming bundle regions. BlockFrequency RAGreedy::calcSpillCost() { - BlockFrequency Cost = 0; + BlockFrequency Cost = BlockFrequency(0); ArrayRef UseBlocks = SA->getUseBlocks(); for (const SplitAnalysis::BlockInfo &BI : UseBlocks) { unsigned Number = BI.MBB->getNumber(); @@ -852,7 +852,7 @@ BlockFrequency RAGreedy::calcSpillCost() { /// BlockFrequency RAGreedy::calcGlobalSplitCost(GlobalSplitCandidate &Cand, const AllocationOrder &Order) { - BlockFrequency GlobalCost = 0; + BlockFrequency GlobalCost = BlockFrequency(0); const BitVector &LiveBundles = Cand.LiveBundles; ArrayRef UseBlocks = SA->getUseBlocks(); for (unsigned I = 0; I != UseBlocks.size(); ++I) { @@ -1056,7 +1056,7 @@ MCRegister RAGreedy::tryRegionSplit(const LiveInterval &VirtReg, if (HasCompact) { // Yes, keep GlobalCand[0] as the compact region candidate. NumCands = 1; - BestCost = BlockFrequency::getMaxFrequency(); + BestCost = BlockFrequency::max(); } else { // No benefit from the compact region, our fallback will be per-block // splitting. Make sure we find a solution that is cheaper than spilling. @@ -1222,7 +1222,7 @@ bool RAGreedy::trySplitAroundHintReg(MCPhysReg Hint, if (ExtraInfo->getStage(VirtReg) >= RS_Split2) return false; - BlockFrequency Cost = 0; + BlockFrequency Cost = BlockFrequency(0); Register Reg = VirtReg.reg(); // Compute the cost of assigning a non Hint physical register to VirtReg. @@ -1249,7 +1249,7 @@ bool RAGreedy::trySplitAroundHintReg(MCPhysReg Hint, // Decrease the cost so it will be split in colder blocks. BranchProbability Threshold(SplitThresholdForRegWithHint, 100); Cost *= Threshold; - if (Cost == 0) + if (Cost == BlockFrequency(0)) return false; unsigned NumCands = 0; @@ -1630,8 +1630,8 @@ unsigned RAGreedy::tryLocalSplit(const LiveInterval &VirtReg, float BestDiff = 0; const float blockFreq = - SpillPlacer->getBlockFrequency(BI.MBB->getNumber()).getFrequency() * - (1.0f / MBFI->getEntryFreq()); + SpillPlacer->getBlockFrequency(BI.MBB->getNumber()).getFrequency() * + (1.0f / MBFI->getEntryFreq().getFrequency()); SmallVector GapWeight; for (MCPhysReg PhysReg : Order) { @@ -2199,9 +2199,9 @@ void RAGreedy::initializeCSRCost() { return; // Raw cost is relative to Entry == 2^14; scale it appropriately. - uint64_t ActualEntry = MBFI->getEntryFreq(); + uint64_t ActualEntry = MBFI->getEntryFreq().getFrequency(); if (!ActualEntry) { - CSRCost = 0; + CSRCost = BlockFrequency(0); return; } uint64_t FixedEntry = 1 << 14; @@ -2212,7 +2212,8 @@ void RAGreedy::initializeCSRCost() { CSRCost /= BranchProbability(FixedEntry, ActualEntry); else // Can't use BranchProbability in general, since it takes 32-bit numbers. - CSRCost = CSRCost.getFrequency() * (ActualEntry / FixedEntry); + CSRCost = + BlockFrequency(CSRCost.getFrequency() * (ActualEntry / FixedEntry)); } /// Collect the hint info for \p Reg. @@ -2243,7 +2244,7 @@ void RAGreedy::collectHintInfo(Register Reg, HintsInfo &Out) { /// \return The cost of \p List for \p PhysReg. BlockFrequency RAGreedy::getBrokenHintFreq(const HintsInfo &List, MCRegister PhysReg) { - BlockFrequency Cost = 0; + BlockFrequency Cost = BlockFrequency(0); for (const HintInfo &Info : List) { if (Info.PhysReg != PhysReg) Cost += Info.Freq; diff --git a/llvm/lib/CodeGen/SelectOptimize.cpp b/llvm/lib/CodeGen/SelectOptimize.cpp index 65801aa0a17d5..aaaee4dfdfac9 100644 --- a/llvm/lib/CodeGen/SelectOptimize.cpp +++ b/llvm/lib/CodeGen/SelectOptimize.cpp @@ -425,7 +425,7 @@ void SelectOptimize::convertProfitableSIGroups(SelectGroups &ProfSIGroups) { BasicBlock *StartBlock = SI->getParent(); BasicBlock::iterator SplitPt = ++(BasicBlock::iterator(LastSI)); BasicBlock *EndBlock = StartBlock->splitBasicBlock(SplitPt, "select.end"); - BFI->setBlockFreq(EndBlock, BFI->getBlockFreq(StartBlock).getFrequency()); + BFI->setBlockFreq(EndBlock, BFI->getBlockFreq(StartBlock)); // Delete the unconditional branch that was just created by the split. StartBlock->getTerminator()->eraseFromParent(); diff --git a/llvm/lib/CodeGen/ShrinkWrap.cpp b/llvm/lib/CodeGen/ShrinkWrap.cpp index 4b1d3637a7462..8628e1f2f1f63 100644 --- a/llvm/lib/CodeGen/ShrinkWrap.cpp +++ b/llvm/lib/CodeGen/ShrinkWrap.cpp @@ -139,7 +139,7 @@ class ShrinkWrap : public MachineFunctionPass { MachineOptimizationRemarkEmitter *ORE = nullptr; /// Frequency of the Entry block. - uint64_t EntryFreq = 0; + BlockFrequency EntryFreq; /// Current opcode for frame setup. unsigned FrameSetupOpcode = ~0u; @@ -640,7 +640,7 @@ bool ShrinkWrap::postShrinkWrapping(bool HasCandidate, MachineFunction &MF, FindIDom<>(**DirtyPreds.begin(), DirtyPreds, *MDT, false); while (NewSave && (hasDirtyPred(ReachableByDirty, *NewSave) || - EntryFreq < MBFI->getBlockFreq(NewSave).getFrequency() || + EntryFreq < MBFI->getBlockFreq(NewSave) || /*Entry freq has been observed more than a loop block in some cases*/ MLI->getLoopFor(NewSave))) @@ -675,8 +675,8 @@ bool ShrinkWrap::postShrinkWrapping(bool HasCandidate, MachineFunction &MF, "Incorrect save or restore point due to dominance relations"); assert((!MLI->getLoopFor(Save) && !MLI->getLoopFor(Restore)) && "Unexpected save or restore point in a loop"); - assert((EntryFreq >= MBFI->getBlockFreq(Save).getFrequency() && - EntryFreq >= MBFI->getBlockFreq(Restore).getFrequency()) && + assert((EntryFreq >= MBFI->getBlockFreq(Save) && + EntryFreq >= MBFI->getBlockFreq(Restore)) && "Incorrect save or restore point based on block frequency"); return true; } @@ -878,8 +878,8 @@ bool ShrinkWrap::performShrinkWrapping( return false; } - LLVM_DEBUG(dbgs() << "\n ** Results **\nFrequency of the Entry: " << EntryFreq - << '\n'); + LLVM_DEBUG(dbgs() << "\n ** Results **\nFrequency of the Entry: " + << EntryFreq.getFrequency() << '\n'); const TargetFrameLowering *TFI = MachineFunc->getSubtarget().getFrameLowering(); @@ -891,8 +891,8 @@ bool ShrinkWrap::performShrinkWrapping( << MBFI->getBlockFreq(Restore).getFrequency() << '\n'); bool IsSaveCheap, TargetCanUseSaveAsPrologue = false; - if (((IsSaveCheap = EntryFreq >= MBFI->getBlockFreq(Save).getFrequency()) && - EntryFreq >= MBFI->getBlockFreq(Restore).getFrequency()) && + if (((IsSaveCheap = EntryFreq >= MBFI->getBlockFreq(Save)) && + EntryFreq >= MBFI->getBlockFreq(Restore)) && ((TargetCanUseSaveAsPrologue = TFI->canUseAsPrologue(*Save)) && TFI->canUseAsEpilogue(*Restore))) break; diff --git a/llvm/lib/CodeGen/SpillPlacement.cpp b/llvm/lib/CodeGen/SpillPlacement.cpp index 91da5e49713c1..6e74e51d67828 100644 --- a/llvm/lib/CodeGen/SpillPlacement.cpp +++ b/llvm/lib/CodeGen/SpillPlacement.cpp @@ -109,8 +109,10 @@ struct SpillPlacement::Node { /// clear - Reset per-query data, but preserve frequencies that only depend on /// the CFG. - void clear(const BlockFrequency &Threshold) { - BiasN = BiasP = Value = 0; + void clear(BlockFrequency Threshold) { + BiasN = BlockFrequency(0); + BiasP = BlockFrequency(0); + Value = 0; SumLinkWeights = Threshold; Links.clear(); } @@ -142,14 +144,14 @@ struct SpillPlacement::Node { BiasN += freq; break; case MustSpill: - BiasN = BlockFrequency::getMaxFrequency(); + BiasN = BlockFrequency::max(); break; } } /// update - Recompute Value from Bias and Links. Return true when node /// preference changes. - bool update(const Node nodes[], const BlockFrequency &Threshold) { + bool update(const Node nodes[], BlockFrequency Threshold) { // Compute the weighted sum of inputs. BlockFrequency SumN = BiasN; BlockFrequency SumP = BiasP; @@ -237,8 +239,10 @@ void SpillPlacement::activate(unsigned n) { // limiting the number of blocks visited and the number of links in the // Hopfield network. if (bundles->getBlocks(n).size() > 100) { - nodes[n].BiasP = 0; - nodes[n].BiasN = (MBFI->getEntryFreq() / 16); + nodes[n].BiasP = BlockFrequency(0); + BlockFrequency BiasN = MBFI->getEntryFreq(); + BiasN >>= 4; + nodes[n].BiasN = BiasN; } } @@ -247,12 +251,12 @@ void SpillPlacement::activate(unsigned n) { /// Set the threshold relative to \c Entry. Since the threshold is used as a /// bound on the open interval (-Threshold;Threshold), 1 is the minimum /// threshold. -void SpillPlacement::setThreshold(const BlockFrequency &Entry) { +void SpillPlacement::setThreshold(BlockFrequency Entry) { // Apparently 2 is a good threshold when Entry==2^14, but we need to scale // it. Divide by 2^13, rounding as appropriate. uint64_t Freq = Entry.getFrequency(); uint64_t Scaled = (Freq >> 13) + bool(Freq & (1 << 12)); - Threshold = std::max(UINT64_C(1), Scaled); + Threshold = BlockFrequency(std::max(UINT64_C(1), Scaled)); } /// addConstraints - Compute node biases and weights from a set of constraints. diff --git a/llvm/lib/CodeGen/SpillPlacement.h b/llvm/lib/CodeGen/SpillPlacement.h index bd37d85c6c0d6..2a298c7a74aa6 100644 --- a/llvm/lib/CodeGen/SpillPlacement.h +++ b/llvm/lib/CodeGen/SpillPlacement.h @@ -162,7 +162,7 @@ class SpillPlacement : public MachineFunctionPass { void releaseMemory() override; void activate(unsigned n); - void setThreshold(const BlockFrequency &Entry); + void setThreshold(BlockFrequency Entry); bool update(unsigned n); }; diff --git a/llvm/lib/Target/PowerPC/PPCMIPeephole.cpp b/llvm/lib/Target/PowerPC/PPCMIPeephole.cpp index 76a73436d61b5..d7a51891087e4 100644 --- a/llvm/lib/Target/PowerPC/PPCMIPeephole.cpp +++ b/llvm/lib/Target/PowerPC/PPCMIPeephole.cpp @@ -113,7 +113,7 @@ struct PPCMIPeephole : public MachineFunctionPass { MachineDominatorTree *MDT; MachinePostDominatorTree *MPDT; MachineBlockFrequencyInfo *MBFI; - uint64_t EntryFreq; + BlockFrequency EntryFreq; SmallSet RegsToUpdate; // Initialize class variables. @@ -300,7 +300,7 @@ void PPCMIPeephole::UpdateTOCSaves( PPCFunctionInfo *FI = MF->getInfo(); MachineBasicBlock *Entry = &MF->front(); - uint64_t CurrBlockFreq = MBFI->getBlockFreq(MI->getParent()).getFrequency(); + BlockFrequency CurrBlockFreq = MBFI->getBlockFreq(MI->getParent()); // If the block in which the TOC save resides is in a block that // post-dominates Entry, or a block that is hotter than entry (keep in mind diff --git a/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp b/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp index aa63e2b64d9ea..429de758c7c3f 100644 --- a/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp +++ b/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp @@ -202,7 +202,7 @@ Bonus InstCostVisitor::getUserBonus(Instruction *User, Value *Use, Constant *C) CodeSize += TTI.getInstructionCost(User, TargetTransformInfo::TCK_CodeSize); uint64_t Weight = BFI.getBlockFreq(User->getParent()).getFrequency() / - BFI.getEntryFreq(); + BFI.getEntryFreq().getFrequency(); Cost Latency = Weight * TTI.getInstructionCost(User, TargetTransformInfo::TCK_Latency); diff --git a/llvm/lib/Transforms/IPO/SyntheticCountsPropagation.cpp b/llvm/lib/Transforms/IPO/SyntheticCountsPropagation.cpp index d46f9a6c67575..f6f8956760848 100644 --- a/llvm/lib/Transforms/IPO/SyntheticCountsPropagation.cpp +++ b/llvm/lib/Transforms/IPO/SyntheticCountsPropagation.cpp @@ -111,7 +111,7 @@ PreservedAnalyses SyntheticCountsPropagation::run(Module &M, // Now compute the callsite count from relative frequency and // entry count: BasicBlock *CSBB = CB.getParent(); - Scaled64 EntryFreq(BFI.getEntryFreq(), 0); + Scaled64 EntryFreq(BFI.getEntryFreq().getFrequency(), 0); Scaled64 BBCount(BFI.getBlockFreq(CSBB).getFrequency(), 0); BBCount /= EntryFreq; BBCount *= Counts[Caller]; diff --git a/llvm/lib/Transforms/Instrumentation/CGProfile.cpp b/llvm/lib/Transforms/Instrumentation/CGProfile.cpp index d53e12ad1ff5c..e2e5f21b376b0 100644 --- a/llvm/lib/Transforms/Instrumentation/CGProfile.cpp +++ b/llvm/lib/Transforms/Instrumentation/CGProfile.cpp @@ -66,7 +66,7 @@ static bool runCGProfilePass( if (F.isDeclaration() || !F.getEntryCount()) continue; auto &BFI = FAM.getResult(F); - if (BFI.getEntryFreq() == 0) + if (BFI.getEntryFreq() == BlockFrequency(0)) continue; TargetTransformInfo &TTI = FAM.getResult(F); for (auto &BB : F) { diff --git a/llvm/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp b/llvm/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp index 2906fe1909840..fd0f69eca96e6 100644 --- a/llvm/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp +++ b/llvm/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp @@ -378,7 +378,7 @@ bool MemOPSizeOpt::perform(MemOp MO) { assert(It != DefaultBB->end()); BasicBlock *MergeBB = SplitBlock(DefaultBB, &(*It), DT); MergeBB->setName("MemOP.Merge"); - BFI.setBlockFreq(MergeBB, OrigBBFreq.getFrequency()); + BFI.setBlockFreq(MergeBB, OrigBBFreq); DefaultBB->setName("MemOP.Default"); DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Eager); diff --git a/llvm/lib/Transforms/Scalar/JumpThreading.cpp b/llvm/lib/Transforms/Scalar/JumpThreading.cpp index ed69f1938ec79..8c3ff399621a8 100644 --- a/llvm/lib/Transforms/Scalar/JumpThreading.cpp +++ b/llvm/lib/Transforms/Scalar/JumpThreading.cpp @@ -2251,7 +2251,7 @@ void JumpThreadingPass::threadThroughTwoBasicBlocks(BasicBlock *PredPredBB, assert(BPI && "It's expected BPI to exist along with BFI"); auto NewBBFreq = BFI->getBlockFreq(PredPredBB) * BPI->getEdgeProbability(PredPredBB, PredBB); - BFI->setBlockFreq(NewBB, NewBBFreq.getFrequency()); + BFI->setBlockFreq(NewBB, NewBBFreq); } // We are going to have to map operands from the original BB block to the new @@ -2377,7 +2377,7 @@ void JumpThreadingPass::threadEdge(BasicBlock *BB, assert(BPI && "It's expected BPI to exist along with BFI"); auto NewBBFreq = BFI->getBlockFreq(PredBB) * BPI->getEdgeProbability(PredBB, BB); - BFI->setBlockFreq(NewBB, NewBBFreq.getFrequency()); + BFI->setBlockFreq(NewBB, NewBBFreq); } // Copy all the instructions from BB to NewBB except the terminator. @@ -2462,7 +2462,7 @@ BasicBlock *JumpThreadingPass::splitBlockPreds(BasicBlock *BB, NewBBFreq += FreqMap.lookup(Pred); } if (BFI) // Apply the summed frequency to NewBB. - BFI->setBlockFreq(NewBB, NewBBFreq.getFrequency()); + BFI->setBlockFreq(NewBB, NewBBFreq); } DTU->applyUpdatesPermissive(Updates); @@ -2502,7 +2502,7 @@ void JumpThreadingPass::updateBlockFreqAndEdgeWeight(BasicBlock *PredBB, auto NewBBFreq = BFI->getBlockFreq(NewBB); auto BB2SuccBBFreq = BBOrigFreq * BPI->getEdgeProbability(BB, SuccBB); auto BBNewFreq = BBOrigFreq - NewBBFreq; - BFI->setBlockFreq(BB, BBNewFreq.getFrequency()); + BFI->setBlockFreq(BB, BBNewFreq); // Collect updated outgoing edges' frequencies from BB and use them to update // edge probabilities. @@ -2760,7 +2760,7 @@ void JumpThreadingPass::unfoldSelectInstr(BasicBlock *Pred, BasicBlock *BB, BranchProbability PredToNewBBProb = BranchProbability::getBranchProbability( TrueWeight, TrueWeight + FalseWeight); auto NewBBFreq = BFI->getBlockFreq(Pred) * PredToNewBBProb; - BFI->setBlockFreq(NewBB, NewBBFreq.getFrequency()); + BFI->setBlockFreq(NewBB, NewBBFreq); } // The select is now dead. diff --git a/llvm/lib/Transforms/Scalar/LoopSink.cpp b/llvm/lib/Transforms/Scalar/LoopSink.cpp index 4d51f0566310c..9f99f6214748b 100644 --- a/llvm/lib/Transforms/Scalar/LoopSink.cpp +++ b/llvm/lib/Transforms/Scalar/LoopSink.cpp @@ -79,7 +79,7 @@ static cl::opt MaxNumberOfUseBBsForSinking( /// AdjustedFreq(BBs) = 99 / SinkFrequencyPercentThreshold% static BlockFrequency adjustedSumFreq(SmallPtrSetImpl &BBs, BlockFrequencyInfo &BFI) { - BlockFrequency T = 0; + BlockFrequency T(0); for (BasicBlock *B : BBs) T += BFI.getBlockFreq(B); if (BBs.size() > 1) diff --git a/llvm/lib/Transforms/Utils/BreakCriticalEdges.cpp b/llvm/lib/Transforms/Utils/BreakCriticalEdges.cpp index cd3ef42f8f29c..5fb796cc3db62 100644 --- a/llvm/lib/Transforms/Utils/BreakCriticalEdges.cpp +++ b/llvm/lib/Transforms/Utils/BreakCriticalEdges.cpp @@ -387,7 +387,7 @@ bool llvm::SplitIndirectBrCriticalEdges(Function &F, if (ShouldUpdateAnalysis) { // Copy the BFI/BPI from Target to BodyBlock. BPI->setEdgeProbability(BodyBlock, EdgeProbabilities); - BFI->setBlockFreq(BodyBlock, BFI->getBlockFreq(Target).getFrequency()); + BFI->setBlockFreq(BodyBlock, BFI->getBlockFreq(Target)); } // It's possible Target was its own successor through an indirectbr. // In this case, the indirectbr now comes from BodyBlock. @@ -411,10 +411,10 @@ bool llvm::SplitIndirectBrCriticalEdges(Function &F, BPI->getEdgeProbability(Src, DirectSucc); } if (ShouldUpdateAnalysis) { - BFI->setBlockFreq(DirectSucc, BlockFreqForDirectSucc.getFrequency()); + BFI->setBlockFreq(DirectSucc, BlockFreqForDirectSucc); BlockFrequency NewBlockFreqForTarget = BFI->getBlockFreq(Target) - BlockFreqForDirectSucc; - BFI->setBlockFreq(Target, NewBlockFreqForTarget.getFrequency()); + BFI->setBlockFreq(Target, NewBlockFreqForTarget); } // Ok, now fix up the PHIs. We know the two blocks only have PHIs, and that diff --git a/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/llvm/lib/Transforms/Utils/CodeExtractor.cpp index f7fed93a52c69..08b2b01b2ee1e 100644 --- a/llvm/lib/Transforms/Utils/CodeExtractor.cpp +++ b/llvm/lib/Transforms/Utils/CodeExtractor.cpp @@ -1779,11 +1779,11 @@ CodeExtractor::extractCodeRegion(const CodeExtractorAnalysisCache &CEAC, // Update the entry count of the function. if (BFI) { - auto Count = BFI->getProfileCountFromFreq(EntryFreq.getFrequency()); + auto Count = BFI->getProfileCountFromFreq(EntryFreq); if (Count) newFunction->setEntryCount( ProfileCount(*Count, Function::PCT_Real)); // FIXME - BFI->setBlockFreq(codeReplacer, EntryFreq.getFrequency()); + BFI->setBlockFreq(codeReplacer, EntryFreq); } CallInst *TheCall = diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp index 6d5312c5a081c..7ea51aab7dc1f 100644 --- a/llvm/lib/Transforms/Utils/InlineFunction.cpp +++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp @@ -1830,12 +1830,12 @@ static void updateCallerBFI(BasicBlock *CallSiteBlock, continue; auto *OrigBB = cast(Entry.first); auto *ClonedBB = cast(Entry.second); - uint64_t Freq = CalleeBFI->getBlockFreq(OrigBB).getFrequency(); + BlockFrequency Freq = CalleeBFI->getBlockFreq(OrigBB); if (!ClonedBBs.insert(ClonedBB).second) { // Multiple blocks in the callee might get mapped to one cloned block in // the caller since we prune the callee as we clone it. When that happens, // we want to use the maximum among the original blocks' frequencies. - uint64_t NewFreq = CallerBFI->getBlockFreq(ClonedBB).getFrequency(); + BlockFrequency NewFreq = CallerBFI->getBlockFreq(ClonedBB); if (NewFreq > Freq) Freq = NewFreq; } @@ -1843,8 +1843,7 @@ static void updateCallerBFI(BasicBlock *CallSiteBlock, } BasicBlock *EntryClone = cast(VMap.lookup(&CalleeEntryBlock)); CallerBFI->setBlockFreqAndScale( - EntryClone, CallerBFI->getBlockFreq(CallSiteBlock).getFrequency(), - ClonedBBs); + EntryClone, CallerBFI->getBlockFreq(CallSiteBlock), ClonedBBs); } /// Update the branch metadata for cloned call instructions. @@ -2811,8 +2810,8 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, if (IFI.CallerBFI) { // Copy original BB's block frequency to AfterCallBB - IFI.CallerBFI->setBlockFreq( - AfterCallBB, IFI.CallerBFI->getBlockFreq(OrigBB).getFrequency()); + IFI.CallerBFI->setBlockFreq(AfterCallBB, + IFI.CallerBFI->getBlockFreq(OrigBB)); } // Change the branch that used to go to AfterCallBB to branch to the first diff --git a/llvm/unittests/Analysis/BlockFrequencyInfoTest.cpp b/llvm/unittests/Analysis/BlockFrequencyInfoTest.cpp index 9fab0a575fd65..c1e94cd5898fb 100644 --- a/llvm/unittests/Analysis/BlockFrequencyInfoTest.cpp +++ b/llvm/unittests/Analysis/BlockFrequencyInfoTest.cpp @@ -84,7 +84,7 @@ TEST_F(BlockFrequencyInfoTest, Basic) { // Scale the frequencies of BB0, BB1 and BB2 by a factor of two. SmallPtrSet BlocksToScale({BB1, BB2}); - BFI.setBlockFreqAndScale(&BB0, BB0Freq * 2, BlocksToScale); + BFI.setBlockFreqAndScale(&BB0, BlockFrequency(BB0Freq * 2), BlocksToScale); EXPECT_EQ(BFI.getBlockFreq(&BB0).getFrequency(), 2 * BB0Freq); EXPECT_EQ(BFI.getBlockFreq(BB1).getFrequency(), 2 * BB1Freq); EXPECT_EQ(BFI.getBlockFreq(BB2).getFrequency(), 2 * BB2Freq); diff --git a/llvm/unittests/Support/BlockFrequencyTest.cpp b/llvm/unittests/Support/BlockFrequencyTest.cpp index cf5cf9213501c..d0374b0a8c519 100644 --- a/llvm/unittests/Support/BlockFrequencyTest.cpp +++ b/llvm/unittests/Support/BlockFrequencyTest.cpp @@ -11,6 +11,7 @@ #include "llvm/Support/DataTypes.h" #include "gtest/gtest.h" #include +#include using namespace llvm; @@ -106,12 +107,12 @@ TEST(BlockFrequencyTest, Saturate) { Freq /= BranchProbability(1, 2); EXPECT_EQ(Freq.getFrequency(), UINT64_MAX); - Freq = 0x1000000000000000ULL; + Freq = BlockFrequency(UINT64_C(0x1000000000000000)); Freq /= BranchProbability(10000, 170000); EXPECT_EQ(Freq.getFrequency(), UINT64_MAX); // Try to cheat the multiplication overflow check. - Freq = 0x00000001f0000001ull; + Freq = BlockFrequency(UINT64_C(0x00000001f0000001)); Freq /= BranchProbability(1000, 0xf000000f); EXPECT_EQ(33527736066704712ULL, Freq.getFrequency()); } diff --git a/llvm/unittests/Transforms/IPO/FunctionSpecializationTest.cpp b/llvm/unittests/Transforms/IPO/FunctionSpecializationTest.cpp index 643e8972cd750..d0e8977f1245d 100644 --- a/llvm/unittests/Transforms/IPO/FunctionSpecializationTest.cpp +++ b/llvm/unittests/Transforms/IPO/FunctionSpecializationTest.cpp @@ -103,9 +103,12 @@ class FunctionSpecializationTest : public testing::Test { Cost CodeSize = TTI.getInstructionCost(&I, TargetTransformInfo::TCK_CodeSize); - Cost Latency = SizeOnly ? 0 : - BFI.getBlockFreq(I.getParent()).getFrequency() / BFI.getEntryFreq() * - TTI.getInstructionCost(&I, TargetTransformInfo::TCK_Latency); + Cost Latency = + SizeOnly + ? 0 + : BFI.getBlockFreq(I.getParent()).getFrequency() / + BFI.getEntryFreq().getFrequency() * + TTI.getInstructionCost(&I, TargetTransformInfo::TCK_Latency); return {CodeSize, Latency}; }