8 changes: 3 additions & 5 deletions llvm/lib/CodeGen/MBFIWrapper.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ MBFIWrapper::getBlockProfileCount(const MachineBasicBlock *MBB) const {
// Modified block frequency also impacts profile count. So we should compute
// profile count from new block frequency if it has been changed.
if (I != MergedBBFreq.end())
return MBFI.getProfileCountFromFreq(I->second.getFrequency());
return MBFI.getProfileCountFromFreq(I->second);

return MBFI.getBlockProfileCount(MBB);
}
Expand All @@ -49,14 +49,12 @@ raw_ostream & MBFIWrapper::printBlockFreq(raw_ostream &OS,
}

raw_ostream & MBFIWrapper::printBlockFreq(raw_ostream &OS,
const BlockFrequency Freq) const {
BlockFrequency Freq) const {
return MBFI.printBlockFreq(OS, Freq);
}

void MBFIWrapper::view(const Twine &Name, bool isSimple) {
MBFI.view(Name, isSimple);
}

uint64_t MBFIWrapper::getEntryFreq() const {
return MBFI.getEntryFreq();
}
BlockFrequency MBFIWrapper::getEntryFreq() const { return MBFI.getEntryFreq(); }
10 changes: 5 additions & 5 deletions llvm/lib/CodeGen/MachineBlockFrequencyInfo.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,7 @@ void MachineBlockFrequencyInfo::view(const Twine &Name, bool isSimple) const {

BlockFrequency
MachineBlockFrequencyInfo::getBlockFreq(const MachineBasicBlock *MBB) const {
return MBFI ? MBFI->getBlockFreq(MBB) : 0;
return MBFI ? MBFI->getBlockFreq(MBB) : BlockFrequency(0);
}

std::optional<uint64_t> MachineBlockFrequencyInfo::getBlockProfileCount(
Expand All @@ -241,7 +241,7 @@ std::optional<uint64_t> MachineBlockFrequencyInfo::getBlockProfileCount(
}

std::optional<uint64_t>
MachineBlockFrequencyInfo::getProfileCountFromFreq(uint64_t Freq) const {
MachineBlockFrequencyInfo::getProfileCountFromFreq(BlockFrequency Freq) const {
if (!MBFI)
return std::nullopt;

Expand All @@ -263,7 +263,7 @@ void MachineBlockFrequencyInfo::onEdgeSplit(
auto NewSuccFreq = MBFI->getBlockFreq(&NewPredecessor) *
MBPI.getEdgeProbability(&NewPredecessor, &NewSuccessor);

MBFI->setBlockFreq(&NewSuccessor, NewSuccFreq.getFrequency());
MBFI->setBlockFreq(&NewSuccessor, NewSuccFreq);
}

const MachineFunction *MachineBlockFrequencyInfo::getFunction() const {
Expand All @@ -286,6 +286,6 @@ MachineBlockFrequencyInfo::printBlockFreq(raw_ostream &OS,
return MBFI ? MBFI->printBlockFreq(OS, MBB) : OS;
}

uint64_t MachineBlockFrequencyInfo::getEntryFreq() const {
return MBFI ? MBFI->getEntryFreq() : 0;
BlockFrequency MachineBlockFrequencyInfo::getEntryFreq() const {
return MBFI ? MBFI->getEntryFreq() : BlockFrequency(0);
}
145 changes: 73 additions & 72 deletions llvm/lib/CodeGen/MachineBlockPlacement.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -444,9 +444,9 @@ class MachineBlockPlacement : public MachineFunctionPass {
if (UseProfileCount) {
auto Count = MBFI->getBlockProfileCount(BB);
if (Count)
return *Count;
return BlockFrequency(*Count);
else
return 0;
return BlockFrequency(0);
} else
return MBFI->getBlockFreq(BB);
}
Expand Down Expand Up @@ -795,10 +795,10 @@ bool MachineBlockPlacement::shouldTailDuplicate(MachineBasicBlock *BB) {
/// penalty is less than 100%
/// TODO(iteratee): Use 64-bit fixed point edge frequencies everywhere.
static bool greaterWithBias(BlockFrequency A, BlockFrequency B,
uint64_t EntryFreq) {
BlockFrequency EntryFreq) {
BranchProbability ThresholdProb(TailDupPlacementPenalty, 100);
BlockFrequency Gain = A - B;
return (Gain / ThresholdProb).getFrequency() >= EntryFreq;
return (Gain / ThresholdProb) >= EntryFreq;
}

/// Check the edge frequencies to see if tail duplication will increase
Expand Down Expand Up @@ -843,7 +843,7 @@ bool MachineBlockPlacement::isProfitableToTailDup(
auto SuccFreq = MBFI->getBlockFreq(Succ);
BlockFrequency P = BBFreq * PProb;
BlockFrequency Qout = BBFreq * QProb;
uint64_t EntryFreq = MBFI->getEntryFreq();
BlockFrequency EntryFreq = MBFI->getEntryFreq();
// If there are no more successors, it is profitable to copy, as it strictly
// increases fallthrough.
if (SuccSuccs.size() == 0)
Expand Down Expand Up @@ -1927,7 +1927,7 @@ BlockFrequency
MachineBlockPlacement::TopFallThroughFreq(
const MachineBasicBlock *Top,
const BlockFilterSet &LoopBlockSet) {
BlockFrequency MaxFreq = 0;
BlockFrequency MaxFreq = BlockFrequency(0);
for (MachineBasicBlock *Pred : Top->predecessors()) {
BlockChain *PredChain = BlockToChain[Pred];
if (!LoopBlockSet.count(Pred) &&
Expand Down Expand Up @@ -1986,66 +1986,66 @@ MachineBlockPlacement::FallThroughGains(
const MachineBasicBlock *ExitBB,
const BlockFilterSet &LoopBlockSet) {
BlockFrequency FallThrough2Top = TopFallThroughFreq(OldTop, LoopBlockSet);
BlockFrequency FallThrough2Exit = 0;
BlockFrequency FallThrough2Exit = BlockFrequency(0);
if (ExitBB)
FallThrough2Exit = MBFI->getBlockFreq(NewTop) *
MBPI->getEdgeProbability(NewTop, ExitBB);
BlockFrequency BackEdgeFreq = MBFI->getBlockFreq(NewTop) *
MBPI->getEdgeProbability(NewTop, OldTop);

// Find the best Pred of NewTop.
MachineBasicBlock *BestPred = nullptr;
BlockFrequency FallThroughFromPred = 0;
for (MachineBasicBlock *Pred : NewTop->predecessors()) {
if (!LoopBlockSet.count(Pred))
continue;
BlockChain *PredChain = BlockToChain[Pred];
if (!PredChain || Pred == *std::prev(PredChain->end())) {
BlockFrequency EdgeFreq = MBFI->getBlockFreq(Pred) *
MBPI->getEdgeProbability(Pred, NewTop);
if (EdgeFreq > FallThroughFromPred) {
FallThroughFromPred = EdgeFreq;
BestPred = Pred;
}
}
}

// If NewTop is not placed after Pred, another successor can be placed
// after Pred.
BlockFrequency NewFreq = 0;
if (BestPred) {
for (MachineBasicBlock *Succ : BestPred->successors()) {
if ((Succ == NewTop) || (Succ == BestPred) || !LoopBlockSet.count(Succ))
continue;
if (ComputedEdges.contains(Succ))
continue;
BlockChain *SuccChain = BlockToChain[Succ];
if ((SuccChain && (Succ != *SuccChain->begin())) ||
(SuccChain == BlockToChain[BestPred]))
continue;
BlockFrequency EdgeFreq = MBFI->getBlockFreq(BestPred) *
MBPI->getEdgeProbability(BestPred, Succ);
if (EdgeFreq > NewFreq)
NewFreq = EdgeFreq;
}
BlockFrequency OrigEdgeFreq = MBFI->getBlockFreq(BestPred) *
MBPI->getEdgeProbability(BestPred, NewTop);
if (NewFreq > OrigEdgeFreq) {
// If NewTop is not the best successor of Pred, then Pred doesn't
// fallthrough to NewTop. So there is no FallThroughFromPred and
// NewFreq.
NewFreq = 0;
FallThroughFromPred = 0;
}
}

BlockFrequency Result = 0;
BlockFrequency Gains = BackEdgeFreq + NewFreq;
BlockFrequency Lost = FallThrough2Top + FallThrough2Exit +
FallThroughFromPred;
if (Gains > Lost)
Result = Gains - Lost;
return Result;
MachineBasicBlock *BestPred = nullptr;
BlockFrequency FallThroughFromPred = BlockFrequency(0);
for (MachineBasicBlock *Pred : NewTop->predecessors()) {
if (!LoopBlockSet.count(Pred))
continue;
BlockChain *PredChain = BlockToChain[Pred];
if (!PredChain || Pred == *std::prev(PredChain->end())) {
BlockFrequency EdgeFreq =
MBFI->getBlockFreq(Pred) * MBPI->getEdgeProbability(Pred, NewTop);
if (EdgeFreq > FallThroughFromPred) {
FallThroughFromPred = EdgeFreq;
BestPred = Pred;
}
}
}

// If NewTop is not placed after Pred, another successor can be placed
// after Pred.
BlockFrequency NewFreq = BlockFrequency(0);
if (BestPred) {
for (MachineBasicBlock *Succ : BestPred->successors()) {
if ((Succ == NewTop) || (Succ == BestPred) || !LoopBlockSet.count(Succ))
continue;
if (ComputedEdges.contains(Succ))
continue;
BlockChain *SuccChain = BlockToChain[Succ];
if ((SuccChain && (Succ != *SuccChain->begin())) ||
(SuccChain == BlockToChain[BestPred]))
continue;
BlockFrequency EdgeFreq = MBFI->getBlockFreq(BestPred) *
MBPI->getEdgeProbability(BestPred, Succ);
if (EdgeFreq > NewFreq)
NewFreq = EdgeFreq;
}
BlockFrequency OrigEdgeFreq = MBFI->getBlockFreq(BestPred) *
MBPI->getEdgeProbability(BestPred, NewTop);
if (NewFreq > OrigEdgeFreq) {
// If NewTop is not the best successor of Pred, then Pred doesn't
// fallthrough to NewTop. So there is no FallThroughFromPred and
// NewFreq.
NewFreq = BlockFrequency(0);
FallThroughFromPred = BlockFrequency(0);
}
}

BlockFrequency Result = BlockFrequency(0);
BlockFrequency Gains = BackEdgeFreq + NewFreq;
BlockFrequency Lost =
FallThrough2Top + FallThrough2Exit + FallThroughFromPred;
if (Gains > Lost)
Result = Gains - Lost;
return Result;
}

/// Helper function of findBestLoopTop. Find the best loop top block
Expand Down Expand Up @@ -2087,7 +2087,7 @@ MachineBlockPlacement::findBestLoopTopHelper(
LLVM_DEBUG(dbgs() << "Finding best loop top for: " << getBlockName(OldTop)
<< "\n");

BlockFrequency BestGains = 0;
BlockFrequency BestGains = BlockFrequency(0);
MachineBasicBlock *BestPred = nullptr;
for (MachineBasicBlock *Pred : OldTop->predecessors()) {
if (!LoopBlockSet.count(Pred))
Expand All @@ -2112,8 +2112,9 @@ MachineBlockPlacement::findBestLoopTopHelper(

BlockFrequency Gains = FallThroughGains(Pred, OldTop, OtherBB,
LoopBlockSet);
if ((Gains > 0) && (Gains > BestGains ||
((Gains == BestGains) && Pred->isLayoutSuccessor(OldTop)))) {
if ((Gains > BlockFrequency(0)) &&
(Gains > BestGains ||
((Gains == BestGains) && Pred->isLayoutSuccessor(OldTop)))) {
BestPred = Pred;
BestGains = Gains;
}
Expand Down Expand Up @@ -2425,14 +2426,14 @@ void MachineBlockPlacement::rotateLoopWithProfile(
if (ChainHeaderBB->isEntryBlock())
return;

BlockFrequency SmallestRotationCost = BlockFrequency::getMaxFrequency();
BlockFrequency SmallestRotationCost = BlockFrequency::max();

// A utility lambda that scales up a block frequency by dividing it by a
// branch probability which is the reciprocal of the scale.
auto ScaleBlockFrequency = [](BlockFrequency Freq,
unsigned Scale) -> BlockFrequency {
if (Scale == 0)
return 0;
return BlockFrequency(0);
// Use operator / between BlockFrequency and BranchProbability to implement
// saturating multiplication.
return Freq / BranchProbability(1, Scale);
Expand Down Expand Up @@ -2492,7 +2493,7 @@ void MachineBlockPlacement::rotateLoopWithProfile(
auto TailBB = *TailIter;

// Calculate the cost by putting this BB to the top.
BlockFrequency Cost = 0;
BlockFrequency Cost = BlockFrequency(0);

// If the current BB is the loop header, we need to take into account the
// cost of the missed fall through edge from outside of the loop to the
Expand Down Expand Up @@ -2523,8 +2524,7 @@ void MachineBlockPlacement::rotateLoopWithProfile(
if (TailBB->isSuccessor(*Iter)) {
auto TailBBFreq = MBFI->getBlockFreq(TailBB);
if (TailBB->succ_size() == 1)
Cost += ScaleBlockFrequency(TailBBFreq.getFrequency(),
MisfetchCost + JumpInstCost);
Cost += ScaleBlockFrequency(TailBBFreq, MisfetchCost + JumpInstCost);
else if (TailBB->succ_size() == 2) {
auto TailToHeadProb = MBPI->getEdgeProbability(TailBB, *Iter);
auto TailToHeadFreq = TailBBFreq * TailToHeadProb;
Expand Down Expand Up @@ -3159,7 +3159,7 @@ static uint64_t countMBBInstruction(MachineBasicBlock *MBB) {
// So we should scale the threshold accordingly. But the instruction size is not
// available on all targets, so we use the number of instructions instead.
BlockFrequency MachineBlockPlacement::scaleThreshold(MachineBasicBlock *BB) {
return DupThreshold.getFrequency() * countMBBInstruction(BB);
return BlockFrequency(DupThreshold.getFrequency() * countMBBInstruction(BB));
}

// Returns true if BB is Pred's best successor.
Expand Down Expand Up @@ -3313,28 +3313,29 @@ void MachineBlockPlacement::findDuplicateCandidates(
}

void MachineBlockPlacement::initDupThreshold() {
DupThreshold = 0;
DupThreshold = BlockFrequency(0);
if (!F->getFunction().hasProfileData())
return;

// We prefer to use prifile count.
uint64_t HotThreshold = PSI->getOrCompHotCountThreshold();
if (HotThreshold != UINT64_MAX) {
UseProfileCount = true;
DupThreshold = HotThreshold * TailDupProfilePercentThreshold / 100;
DupThreshold =
BlockFrequency(HotThreshold * TailDupProfilePercentThreshold / 100);
return;
}

// Profile count is not available, we can use block frequency instead.
BlockFrequency MaxFreq = 0;
BlockFrequency MaxFreq = BlockFrequency(0);
for (MachineBasicBlock &MBB : *F) {
BlockFrequency Freq = MBFI->getBlockFreq(&MBB);
if (Freq > MaxFreq)
MaxFreq = Freq;
}

BranchProbability ThresholdProb(TailDupPlacementPenalty, 100);
DupThreshold = MaxFreq * ThresholdProb;
DupThreshold = BlockFrequency(MaxFreq * ThresholdProb);
UseProfileCount = false;
}

Expand Down
25 changes: 13 additions & 12 deletions llvm/lib/CodeGen/RegAllocGreedy.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -607,7 +607,7 @@ bool RAGreedy::addSplitConstraints(InterferenceCache::Cursor Intf,

// Reset interference dependent info.
SplitConstraints.resize(UseBlocks.size());
BlockFrequency StaticCost = 0;
BlockFrequency StaticCost = BlockFrequency(0);
for (unsigned I = 0; I != UseBlocks.size(); ++I) {
const SplitAnalysis::BlockInfo &BI = UseBlocks[I];
SpillPlacement::BlockConstraint &BC = SplitConstraints[I];
Expand Down Expand Up @@ -832,7 +832,7 @@ bool RAGreedy::calcCompactRegion(GlobalSplitCandidate &Cand) {
/// calcSpillCost - Compute how expensive it would be to split the live range in
/// SA around all use blocks instead of forming bundle regions.
BlockFrequency RAGreedy::calcSpillCost() {
BlockFrequency Cost = 0;
BlockFrequency Cost = BlockFrequency(0);
ArrayRef<SplitAnalysis::BlockInfo> UseBlocks = SA->getUseBlocks();
for (const SplitAnalysis::BlockInfo &BI : UseBlocks) {
unsigned Number = BI.MBB->getNumber();
Expand All @@ -852,7 +852,7 @@ BlockFrequency RAGreedy::calcSpillCost() {
///
BlockFrequency RAGreedy::calcGlobalSplitCost(GlobalSplitCandidate &Cand,
const AllocationOrder &Order) {
BlockFrequency GlobalCost = 0;
BlockFrequency GlobalCost = BlockFrequency(0);
const BitVector &LiveBundles = Cand.LiveBundles;
ArrayRef<SplitAnalysis::BlockInfo> UseBlocks = SA->getUseBlocks();
for (unsigned I = 0; I != UseBlocks.size(); ++I) {
Expand Down Expand Up @@ -1056,7 +1056,7 @@ MCRegister RAGreedy::tryRegionSplit(const LiveInterval &VirtReg,
if (HasCompact) {
// Yes, keep GlobalCand[0] as the compact region candidate.
NumCands = 1;
BestCost = BlockFrequency::getMaxFrequency();
BestCost = BlockFrequency::max();
} else {
// No benefit from the compact region, our fallback will be per-block
// splitting. Make sure we find a solution that is cheaper than spilling.
Expand Down Expand Up @@ -1222,7 +1222,7 @@ bool RAGreedy::trySplitAroundHintReg(MCPhysReg Hint,
if (ExtraInfo->getStage(VirtReg) >= RS_Split2)
return false;

BlockFrequency Cost = 0;
BlockFrequency Cost = BlockFrequency(0);
Register Reg = VirtReg.reg();

// Compute the cost of assigning a non Hint physical register to VirtReg.
Expand All @@ -1249,7 +1249,7 @@ bool RAGreedy::trySplitAroundHintReg(MCPhysReg Hint,
// Decrease the cost so it will be split in colder blocks.
BranchProbability Threshold(SplitThresholdForRegWithHint, 100);
Cost *= Threshold;
if (Cost == 0)
if (Cost == BlockFrequency(0))
return false;

unsigned NumCands = 0;
Expand Down Expand Up @@ -1630,8 +1630,8 @@ unsigned RAGreedy::tryLocalSplit(const LiveInterval &VirtReg,
float BestDiff = 0;

const float blockFreq =
SpillPlacer->getBlockFrequency(BI.MBB->getNumber()).getFrequency() *
(1.0f / MBFI->getEntryFreq());
SpillPlacer->getBlockFrequency(BI.MBB->getNumber()).getFrequency() *
(1.0f / MBFI->getEntryFreq().getFrequency());
SmallVector<float, 8> GapWeight;

for (MCPhysReg PhysReg : Order) {
Expand Down Expand Up @@ -2199,9 +2199,9 @@ void RAGreedy::initializeCSRCost() {
return;

// Raw cost is relative to Entry == 2^14; scale it appropriately.
uint64_t ActualEntry = MBFI->getEntryFreq();
uint64_t ActualEntry = MBFI->getEntryFreq().getFrequency();
if (!ActualEntry) {
CSRCost = 0;
CSRCost = BlockFrequency(0);
return;
}
uint64_t FixedEntry = 1 << 14;
Expand All @@ -2212,7 +2212,8 @@ void RAGreedy::initializeCSRCost() {
CSRCost /= BranchProbability(FixedEntry, ActualEntry);
else
// Can't use BranchProbability in general, since it takes 32-bit numbers.
CSRCost = CSRCost.getFrequency() * (ActualEntry / FixedEntry);
CSRCost =
BlockFrequency(CSRCost.getFrequency() * (ActualEntry / FixedEntry));
}

/// Collect the hint info for \p Reg.
Expand Down Expand Up @@ -2243,7 +2244,7 @@ void RAGreedy::collectHintInfo(Register Reg, HintsInfo &Out) {
/// \return The cost of \p List for \p PhysReg.
BlockFrequency RAGreedy::getBrokenHintFreq(const HintsInfo &List,
MCRegister PhysReg) {
BlockFrequency Cost = 0;
BlockFrequency Cost = BlockFrequency(0);
for (const HintInfo &Info : List) {
if (Info.PhysReg != PhysReg)
Cost += Info.Freq;
Expand Down
2 changes: 1 addition & 1 deletion llvm/lib/CodeGen/SelectOptimize.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -425,7 +425,7 @@ void SelectOptimize::convertProfitableSIGroups(SelectGroups &ProfSIGroups) {
BasicBlock *StartBlock = SI->getParent();
BasicBlock::iterator SplitPt = ++(BasicBlock::iterator(LastSI));
BasicBlock *EndBlock = StartBlock->splitBasicBlock(SplitPt, "select.end");
BFI->setBlockFreq(EndBlock, BFI->getBlockFreq(StartBlock).getFrequency());
BFI->setBlockFreq(EndBlock, BFI->getBlockFreq(StartBlock));
// Delete the unconditional branch that was just created by the split.
StartBlock->getTerminator()->eraseFromParent();

Expand Down
16 changes: 8 additions & 8 deletions llvm/lib/CodeGen/ShrinkWrap.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ class ShrinkWrap : public MachineFunctionPass {
MachineOptimizationRemarkEmitter *ORE = nullptr;

/// Frequency of the Entry block.
uint64_t EntryFreq = 0;
BlockFrequency EntryFreq;

/// Current opcode for frame setup.
unsigned FrameSetupOpcode = ~0u;
Expand Down Expand Up @@ -640,7 +640,7 @@ bool ShrinkWrap::postShrinkWrapping(bool HasCandidate, MachineFunction &MF,
FindIDom<>(**DirtyPreds.begin(), DirtyPreds, *MDT, false);

while (NewSave && (hasDirtyPred(ReachableByDirty, *NewSave) ||
EntryFreq < MBFI->getBlockFreq(NewSave).getFrequency() ||
EntryFreq < MBFI->getBlockFreq(NewSave) ||
/*Entry freq has been observed more than a loop block in
some cases*/
MLI->getLoopFor(NewSave)))
Expand Down Expand Up @@ -675,8 +675,8 @@ bool ShrinkWrap::postShrinkWrapping(bool HasCandidate, MachineFunction &MF,
"Incorrect save or restore point due to dominance relations");
assert((!MLI->getLoopFor(Save) && !MLI->getLoopFor(Restore)) &&
"Unexpected save or restore point in a loop");
assert((EntryFreq >= MBFI->getBlockFreq(Save).getFrequency() &&
EntryFreq >= MBFI->getBlockFreq(Restore).getFrequency()) &&
assert((EntryFreq >= MBFI->getBlockFreq(Save) &&
EntryFreq >= MBFI->getBlockFreq(Restore)) &&
"Incorrect save or restore point based on block frequency");
return true;
}
Expand Down Expand Up @@ -878,8 +878,8 @@ bool ShrinkWrap::performShrinkWrapping(
return false;
}

LLVM_DEBUG(dbgs() << "\n ** Results **\nFrequency of the Entry: " << EntryFreq
<< '\n');
LLVM_DEBUG(dbgs() << "\n ** Results **\nFrequency of the Entry: "
<< EntryFreq.getFrequency() << '\n');

const TargetFrameLowering *TFI =
MachineFunc->getSubtarget().getFrameLowering();
Expand All @@ -891,8 +891,8 @@ bool ShrinkWrap::performShrinkWrapping(
<< MBFI->getBlockFreq(Restore).getFrequency() << '\n');

bool IsSaveCheap, TargetCanUseSaveAsPrologue = false;
if (((IsSaveCheap = EntryFreq >= MBFI->getBlockFreq(Save).getFrequency()) &&
EntryFreq >= MBFI->getBlockFreq(Restore).getFrequency()) &&
if (((IsSaveCheap = EntryFreq >= MBFI->getBlockFreq(Save)) &&
EntryFreq >= MBFI->getBlockFreq(Restore)) &&
((TargetCanUseSaveAsPrologue = TFI->canUseAsPrologue(*Save)) &&
TFI->canUseAsEpilogue(*Restore)))
break;
Expand Down
20 changes: 12 additions & 8 deletions llvm/lib/CodeGen/SpillPlacement.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -109,8 +109,10 @@ struct SpillPlacement::Node {

/// clear - Reset per-query data, but preserve frequencies that only depend on
/// the CFG.
void clear(const BlockFrequency &Threshold) {
BiasN = BiasP = Value = 0;
void clear(BlockFrequency Threshold) {
BiasN = BlockFrequency(0);
BiasP = BlockFrequency(0);
Value = 0;
SumLinkWeights = Threshold;
Links.clear();
}
Expand Down Expand Up @@ -142,14 +144,14 @@ struct SpillPlacement::Node {
BiasN += freq;
break;
case MustSpill:
BiasN = BlockFrequency::getMaxFrequency();
BiasN = BlockFrequency::max();
break;
}
}

/// update - Recompute Value from Bias and Links. Return true when node
/// preference changes.
bool update(const Node nodes[], const BlockFrequency &Threshold) {
bool update(const Node nodes[], BlockFrequency Threshold) {
// Compute the weighted sum of inputs.
BlockFrequency SumN = BiasN;
BlockFrequency SumP = BiasP;
Expand Down Expand Up @@ -237,8 +239,10 @@ void SpillPlacement::activate(unsigned n) {
// limiting the number of blocks visited and the number of links in the
// Hopfield network.
if (bundles->getBlocks(n).size() > 100) {
nodes[n].BiasP = 0;
nodes[n].BiasN = (MBFI->getEntryFreq() / 16);
nodes[n].BiasP = BlockFrequency(0);
BlockFrequency BiasN = MBFI->getEntryFreq();
BiasN >>= 4;
nodes[n].BiasN = BiasN;
}
}

Expand All @@ -247,12 +251,12 @@ void SpillPlacement::activate(unsigned n) {
/// Set the threshold relative to \c Entry. Since the threshold is used as a
/// bound on the open interval (-Threshold;Threshold), 1 is the minimum
/// threshold.
void SpillPlacement::setThreshold(const BlockFrequency &Entry) {
void SpillPlacement::setThreshold(BlockFrequency Entry) {
// Apparently 2 is a good threshold when Entry==2^14, but we need to scale
// it. Divide by 2^13, rounding as appropriate.
uint64_t Freq = Entry.getFrequency();
uint64_t Scaled = (Freq >> 13) + bool(Freq & (1 << 12));
Threshold = std::max(UINT64_C(1), Scaled);
Threshold = BlockFrequency(std::max(UINT64_C(1), Scaled));
}

/// addConstraints - Compute node biases and weights from a set of constraints.
Expand Down
2 changes: 1 addition & 1 deletion llvm/lib/CodeGen/SpillPlacement.h
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ class SpillPlacement : public MachineFunctionPass {
void releaseMemory() override;

void activate(unsigned n);
void setThreshold(const BlockFrequency &Entry);
void setThreshold(BlockFrequency Entry);

bool update(unsigned n);
};
Expand Down
4 changes: 2 additions & 2 deletions llvm/lib/Target/PowerPC/PPCMIPeephole.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ struct PPCMIPeephole : public MachineFunctionPass {
MachineDominatorTree *MDT;
MachinePostDominatorTree *MPDT;
MachineBlockFrequencyInfo *MBFI;
uint64_t EntryFreq;
BlockFrequency EntryFreq;
SmallSet<Register, 16> RegsToUpdate;

// Initialize class variables.
Expand Down Expand Up @@ -300,7 +300,7 @@ void PPCMIPeephole::UpdateTOCSaves(
PPCFunctionInfo *FI = MF->getInfo<PPCFunctionInfo>();

MachineBasicBlock *Entry = &MF->front();
uint64_t CurrBlockFreq = MBFI->getBlockFreq(MI->getParent()).getFrequency();
BlockFrequency CurrBlockFreq = MBFI->getBlockFreq(MI->getParent());

// If the block in which the TOC save resides is in a block that
// post-dominates Entry, or a block that is hotter than entry (keep in mind
Expand Down
2 changes: 1 addition & 1 deletion llvm/lib/Transforms/IPO/FunctionSpecialization.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ Bonus InstCostVisitor::getUserBonus(Instruction *User, Value *Use, Constant *C)
CodeSize += TTI.getInstructionCost(User, TargetTransformInfo::TCK_CodeSize);

uint64_t Weight = BFI.getBlockFreq(User->getParent()).getFrequency() /
BFI.getEntryFreq();
BFI.getEntryFreq().getFrequency();

Cost Latency = Weight *
TTI.getInstructionCost(User, TargetTransformInfo::TCK_Latency);
Expand Down
2 changes: 1 addition & 1 deletion llvm/lib/Transforms/IPO/SyntheticCountsPropagation.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ PreservedAnalyses SyntheticCountsPropagation::run(Module &M,
// Now compute the callsite count from relative frequency and
// entry count:
BasicBlock *CSBB = CB.getParent();
Scaled64 EntryFreq(BFI.getEntryFreq(), 0);
Scaled64 EntryFreq(BFI.getEntryFreq().getFrequency(), 0);
Scaled64 BBCount(BFI.getBlockFreq(CSBB).getFrequency(), 0);
BBCount /= EntryFreq;
BBCount *= Counts[Caller];
Expand Down
2 changes: 1 addition & 1 deletion llvm/lib/Transforms/Instrumentation/CGProfile.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ static bool runCGProfilePass(
if (F.isDeclaration() || !F.getEntryCount())
continue;
auto &BFI = FAM.getResult<BlockFrequencyAnalysis>(F);
if (BFI.getEntryFreq() == 0)
if (BFI.getEntryFreq() == BlockFrequency(0))
continue;
TargetTransformInfo &TTI = FAM.getResult<TargetIRAnalysis>(F);
for (auto &BB : F) {
Expand Down
2 changes: 1 addition & 1 deletion llvm/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -378,7 +378,7 @@ bool MemOPSizeOpt::perform(MemOp MO) {
assert(It != DefaultBB->end());
BasicBlock *MergeBB = SplitBlock(DefaultBB, &(*It), DT);
MergeBB->setName("MemOP.Merge");
BFI.setBlockFreq(MergeBB, OrigBBFreq.getFrequency());
BFI.setBlockFreq(MergeBB, OrigBBFreq);
DefaultBB->setName("MemOP.Default");

DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Eager);
Expand Down
10 changes: 5 additions & 5 deletions llvm/lib/Transforms/Scalar/JumpThreading.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2251,7 +2251,7 @@ void JumpThreadingPass::threadThroughTwoBasicBlocks(BasicBlock *PredPredBB,
assert(BPI && "It's expected BPI to exist along with BFI");
auto NewBBFreq = BFI->getBlockFreq(PredPredBB) *
BPI->getEdgeProbability(PredPredBB, PredBB);
BFI->setBlockFreq(NewBB, NewBBFreq.getFrequency());
BFI->setBlockFreq(NewBB, NewBBFreq);
}

// We are going to have to map operands from the original BB block to the new
Expand Down Expand Up @@ -2377,7 +2377,7 @@ void JumpThreadingPass::threadEdge(BasicBlock *BB,
assert(BPI && "It's expected BPI to exist along with BFI");
auto NewBBFreq =
BFI->getBlockFreq(PredBB) * BPI->getEdgeProbability(PredBB, BB);
BFI->setBlockFreq(NewBB, NewBBFreq.getFrequency());
BFI->setBlockFreq(NewBB, NewBBFreq);
}

// Copy all the instructions from BB to NewBB except the terminator.
Expand Down Expand Up @@ -2462,7 +2462,7 @@ BasicBlock *JumpThreadingPass::splitBlockPreds(BasicBlock *BB,
NewBBFreq += FreqMap.lookup(Pred);
}
if (BFI) // Apply the summed frequency to NewBB.
BFI->setBlockFreq(NewBB, NewBBFreq.getFrequency());
BFI->setBlockFreq(NewBB, NewBBFreq);
}

DTU->applyUpdatesPermissive(Updates);
Expand Down Expand Up @@ -2502,7 +2502,7 @@ void JumpThreadingPass::updateBlockFreqAndEdgeWeight(BasicBlock *PredBB,
auto NewBBFreq = BFI->getBlockFreq(NewBB);
auto BB2SuccBBFreq = BBOrigFreq * BPI->getEdgeProbability(BB, SuccBB);
auto BBNewFreq = BBOrigFreq - NewBBFreq;
BFI->setBlockFreq(BB, BBNewFreq.getFrequency());
BFI->setBlockFreq(BB, BBNewFreq);

// Collect updated outgoing edges' frequencies from BB and use them to update
// edge probabilities.
Expand Down Expand Up @@ -2760,7 +2760,7 @@ void JumpThreadingPass::unfoldSelectInstr(BasicBlock *Pred, BasicBlock *BB,
BranchProbability PredToNewBBProb = BranchProbability::getBranchProbability(
TrueWeight, TrueWeight + FalseWeight);
auto NewBBFreq = BFI->getBlockFreq(Pred) * PredToNewBBProb;
BFI->setBlockFreq(NewBB, NewBBFreq.getFrequency());
BFI->setBlockFreq(NewBB, NewBBFreq);
}

// The select is now dead.
Expand Down
2 changes: 1 addition & 1 deletion llvm/lib/Transforms/Scalar/LoopSink.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ static cl::opt<unsigned> MaxNumberOfUseBBsForSinking(
/// AdjustedFreq(BBs) = 99 / SinkFrequencyPercentThreshold%
static BlockFrequency adjustedSumFreq(SmallPtrSetImpl<BasicBlock *> &BBs,
BlockFrequencyInfo &BFI) {
BlockFrequency T = 0;
BlockFrequency T(0);
for (BasicBlock *B : BBs)
T += BFI.getBlockFreq(B);
if (BBs.size() > 1)
Expand Down
6 changes: 3 additions & 3 deletions llvm/lib/Transforms/Utils/BreakCriticalEdges.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -387,7 +387,7 @@ bool llvm::SplitIndirectBrCriticalEdges(Function &F,
if (ShouldUpdateAnalysis) {
// Copy the BFI/BPI from Target to BodyBlock.
BPI->setEdgeProbability(BodyBlock, EdgeProbabilities);
BFI->setBlockFreq(BodyBlock, BFI->getBlockFreq(Target).getFrequency());
BFI->setBlockFreq(BodyBlock, BFI->getBlockFreq(Target));
}
// It's possible Target was its own successor through an indirectbr.
// In this case, the indirectbr now comes from BodyBlock.
Expand All @@ -411,10 +411,10 @@ bool llvm::SplitIndirectBrCriticalEdges(Function &F,
BPI->getEdgeProbability(Src, DirectSucc);
}
if (ShouldUpdateAnalysis) {
BFI->setBlockFreq(DirectSucc, BlockFreqForDirectSucc.getFrequency());
BFI->setBlockFreq(DirectSucc, BlockFreqForDirectSucc);
BlockFrequency NewBlockFreqForTarget =
BFI->getBlockFreq(Target) - BlockFreqForDirectSucc;
BFI->setBlockFreq(Target, NewBlockFreqForTarget.getFrequency());
BFI->setBlockFreq(Target, NewBlockFreqForTarget);
}

// Ok, now fix up the PHIs. We know the two blocks only have PHIs, and that
Expand Down
4 changes: 2 additions & 2 deletions llvm/lib/Transforms/Utils/CodeExtractor.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1779,11 +1779,11 @@ CodeExtractor::extractCodeRegion(const CodeExtractorAnalysisCache &CEAC,

// Update the entry count of the function.
if (BFI) {
auto Count = BFI->getProfileCountFromFreq(EntryFreq.getFrequency());
auto Count = BFI->getProfileCountFromFreq(EntryFreq);
if (Count)
newFunction->setEntryCount(
ProfileCount(*Count, Function::PCT_Real)); // FIXME
BFI->setBlockFreq(codeReplacer, EntryFreq.getFrequency());
BFI->setBlockFreq(codeReplacer, EntryFreq);
}

CallInst *TheCall =
Expand Down
11 changes: 5 additions & 6 deletions llvm/lib/Transforms/Utils/InlineFunction.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1830,21 +1830,20 @@ static void updateCallerBFI(BasicBlock *CallSiteBlock,
continue;
auto *OrigBB = cast<BasicBlock>(Entry.first);
auto *ClonedBB = cast<BasicBlock>(Entry.second);
uint64_t Freq = CalleeBFI->getBlockFreq(OrigBB).getFrequency();
BlockFrequency Freq = CalleeBFI->getBlockFreq(OrigBB);
if (!ClonedBBs.insert(ClonedBB).second) {
// Multiple blocks in the callee might get mapped to one cloned block in
// the caller since we prune the callee as we clone it. When that happens,
// we want to use the maximum among the original blocks' frequencies.
uint64_t NewFreq = CallerBFI->getBlockFreq(ClonedBB).getFrequency();
BlockFrequency NewFreq = CallerBFI->getBlockFreq(ClonedBB);
if (NewFreq > Freq)
Freq = NewFreq;
}
CallerBFI->setBlockFreq(ClonedBB, Freq);
}
BasicBlock *EntryClone = cast<BasicBlock>(VMap.lookup(&CalleeEntryBlock));
CallerBFI->setBlockFreqAndScale(
EntryClone, CallerBFI->getBlockFreq(CallSiteBlock).getFrequency(),
ClonedBBs);
EntryClone, CallerBFI->getBlockFreq(CallSiteBlock), ClonedBBs);
}

/// Update the branch metadata for cloned call instructions.
Expand Down Expand Up @@ -2811,8 +2810,8 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI,

if (IFI.CallerBFI) {
// Copy original BB's block frequency to AfterCallBB
IFI.CallerBFI->setBlockFreq(
AfterCallBB, IFI.CallerBFI->getBlockFreq(OrigBB).getFrequency());
IFI.CallerBFI->setBlockFreq(AfterCallBB,
IFI.CallerBFI->getBlockFreq(OrigBB));
}

// Change the branch that used to go to AfterCallBB to branch to the first
Expand Down
2 changes: 1 addition & 1 deletion llvm/unittests/Analysis/BlockFrequencyInfoTest.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ TEST_F(BlockFrequencyInfoTest, Basic) {

// Scale the frequencies of BB0, BB1 and BB2 by a factor of two.
SmallPtrSet<BasicBlock *, 4> BlocksToScale({BB1, BB2});
BFI.setBlockFreqAndScale(&BB0, BB0Freq * 2, BlocksToScale);
BFI.setBlockFreqAndScale(&BB0, BlockFrequency(BB0Freq * 2), BlocksToScale);
EXPECT_EQ(BFI.getBlockFreq(&BB0).getFrequency(), 2 * BB0Freq);
EXPECT_EQ(BFI.getBlockFreq(BB1).getFrequency(), 2 * BB1Freq);
EXPECT_EQ(BFI.getBlockFreq(BB2).getFrequency(), 2 * BB2Freq);
Expand Down
5 changes: 3 additions & 2 deletions llvm/unittests/Support/BlockFrequencyTest.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
#include "llvm/Support/DataTypes.h"
#include "gtest/gtest.h"
#include <climits>
#include <cstdint>

using namespace llvm;

Expand Down Expand Up @@ -106,12 +107,12 @@ TEST(BlockFrequencyTest, Saturate) {
Freq /= BranchProbability(1, 2);
EXPECT_EQ(Freq.getFrequency(), UINT64_MAX);

Freq = 0x1000000000000000ULL;
Freq = BlockFrequency(UINT64_C(0x1000000000000000));
Freq /= BranchProbability(10000, 170000);
EXPECT_EQ(Freq.getFrequency(), UINT64_MAX);

// Try to cheat the multiplication overflow check.
Freq = 0x00000001f0000001ull;
Freq = BlockFrequency(UINT64_C(0x00000001f0000001));
Freq /= BranchProbability(1000, 0xf000000f);
EXPECT_EQ(33527736066704712ULL, Freq.getFrequency());
}
Expand Down
9 changes: 6 additions & 3 deletions llvm/unittests/Transforms/IPO/FunctionSpecializationTest.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -103,9 +103,12 @@ class FunctionSpecializationTest : public testing::Test {
Cost CodeSize =
TTI.getInstructionCost(&I, TargetTransformInfo::TCK_CodeSize);

Cost Latency = SizeOnly ? 0 :
BFI.getBlockFreq(I.getParent()).getFrequency() / BFI.getEntryFreq() *
TTI.getInstructionCost(&I, TargetTransformInfo::TCK_Latency);
Cost Latency =
SizeOnly
? 0
: BFI.getBlockFreq(I.getParent()).getFrequency() /
BFI.getEntryFreq().getFrequency() *
TTI.getInstructionCost(&I, TargetTransformInfo::TCK_Latency);

return {CodeSize, Latency};
}
Expand Down