287 changes: 119 additions & 168 deletions llvm/lib/Analysis/BranchProbabilityInfo.cpp
View file

Large diffs are not rendered by default.

11 changes: 6 additions & 5 deletions llvm/lib/CodeGen/StackProtector.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -453,12 +453,13 @@ bool StackProtector::InsertStackProtectors() {
LoadInst *LI1 = B.CreateLoad(StackGuardVar);
LoadInst *LI2 = B.CreateLoad(AI);
Value *Cmp = B.CreateICmpEQ(LI1, LI2);
unsigned SuccessWeight =
BranchProbabilityInfo::getBranchWeightStackProtector(true);
unsigned FailureWeight =
BranchProbabilityInfo::getBranchWeightStackProtector(false);
auto SuccessProb =
BranchProbabilityInfo::getBranchProbStackProtector(true);
auto FailureProb =
BranchProbabilityInfo::getBranchProbStackProtector(false);
MDNode *Weights = MDBuilder(F->getContext())
.createBranchWeights(SuccessWeight, FailureWeight);
.createBranchWeights(SuccessProb.getNumerator(),
FailureProb.getNumerator());
B.CreateCondBr(Cmp, NewBB, FailBB, Weights);
}
}
Expand Down
20 changes: 9 additions & 11 deletions llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -414,8 +414,8 @@ static unsigned getBranchHint(unsigned PCC, FunctionLoweringInfo *FuncInfo,
const BasicBlock *TBB = BBTerm->getSuccessor(0);
const BasicBlock *FBB = BBTerm->getSuccessor(1);

uint32_t TWeight = FuncInfo->BPI->getEdgeWeight(BB, TBB);
uint32_t FWeight = FuncInfo->BPI->getEdgeWeight(BB, FBB);
auto TProb = FuncInfo->BPI->getEdgeProbability(BB, TBB);
auto FProb = FuncInfo->BPI->getEdgeProbability(BB, FBB);

// We only want to handle cases which are easy to predict at static time, e.g.
// C++ throw statement, that is very likely not taken, or calling never
Expand All @@ -432,24 +432,22 @@ static unsigned getBranchHint(unsigned PCC, FunctionLoweringInfo *FuncInfo,
// 5. PH/ZH/FPH 20:12
const uint32_t Threshold = 10000;

// Minimal weight should be at least 1
if (std::max(TWeight, FWeight) /
std::max(1u, std::min(TWeight, FWeight)) < Threshold)
if (std::max(TProb, FProb) / Threshold < std::min(TProb, FProb))
return PPC::BR_NO_HINT;

DEBUG(dbgs() << "Use branch hint for '" << FuncInfo->Fn->getName() << "::"
<< BB->getName() << "'\n"
<< " -> " << TBB->getName() << ": " << TWeight << "\n"
<< " -> " << FBB->getName() << ": " << FWeight << "\n");
<< " -> " << TBB->getName() << ": " << TProb << "\n"
<< " -> " << FBB->getName() << ": " << FProb << "\n");

const BasicBlockSDNode *BBDN = cast<BasicBlockSDNode>(DestMBB);

// If Dest BasicBlock is False-BasicBlock (FBB), swap branch weight,
// because we want 'TWeight' stands for 'branch weight' to Dest BasicBlock
// If Dest BasicBlock is False-BasicBlock (FBB), swap branch probabilities,
// because we want 'TProb' stands for 'branch probability' to Dest BasicBlock
if (BBDN->getBasicBlock()->getBasicBlock() != TBB)
std::swap(TWeight, FWeight);
std::swap(TProb, FProb);

return (TWeight > FWeight) ? PPC::BR_TAKEN_HINT : PPC::BR_NONTAKEN_HINT;
return (TProb > FProb) ? PPC::BR_TAKEN_HINT : PPC::BR_NONTAKEN_HINT;
}

// isOpcWithIntImmediate - This method tests to see if the node is a specific
Expand Down
28 changes: 18 additions & 10 deletions llvm/lib/Transforms/Scalar/JumpThreading.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1636,7 +1636,7 @@ void JumpThreading::UpdateBlockFreqAndEdgeWeight(BasicBlock *PredBB,
BFI->setBlockFreq(BB, BBNewFreq.getFrequency());

// Collect updated outgoing edges' frequencies from BB and use them to update
// edge weights.
// edge probabilities.
SmallVector<uint64_t, 4> BBSuccFreq;
for (auto I = succ_begin(BB), E = succ_end(BB); I != E; ++I) {
auto SuccFreq = (*I == SuccBB)
Expand All @@ -1645,18 +1645,26 @@ void JumpThreading::UpdateBlockFreqAndEdgeWeight(BasicBlock *PredBB,
BBSuccFreq.push_back(SuccFreq.getFrequency());
}

// Normalize edge weights in Weights64 so that the sum of them can fit in
BranchProbability::normalizeEdgeWeights(BBSuccFreq.begin(), BBSuccFreq.end());
uint64_t MaxBBSuccFreq =
*std::max_element(BBSuccFreq.begin(), BBSuccFreq.end());
SmallVector<BranchProbability, 4> BBSuccProbs;
for (uint64_t Freq : BBSuccFreq)
BBSuccProbs.push_back(
BranchProbability::getBranchProbability(Freq, MaxBBSuccFreq));

SmallVector<uint32_t, 4> Weights;
for (auto Freq : BBSuccFreq)
Weights.push_back(static_cast<uint32_t>(Freq));
// Normalize edge probabilities so that they sum up to one.
BranchProbability::normalizeProbabilities(BBSuccProbs.begin(),
BBSuccProbs.end());

// Update edge weights in BPI.
for (int I = 0, E = Weights.size(); I < E; I++)
BPI->setEdgeWeight(BB, I, Weights[I]);
// Update edge probabilities in BPI.
for (int I = 0, E = BBSuccProbs.size(); I < E; I++)
BPI->setEdgeProbability(BB, I, BBSuccProbs[I]);

if (BBSuccProbs.size() >= 2) {
SmallVector<uint32_t, 4> Weights;
for (auto Prob : BBSuccProbs)
Weights.push_back(Prob.getNumerator());

if (Weights.size() >= 2) {
auto TI = BB->getTerminator();
TI->setMetadata(
LLVMContext::MD_prof,
Expand Down
10 changes: 5 additions & 5 deletions llvm/test/Analysis/BranchProbabilityInfo/noreturn.ll
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,11 @@ entry:
i32 2, label %case_b
i32 3, label %case_c
i32 4, label %case_d]
; CHECK: edge entry -> exit probability is 0x7fffe000 / 0x80000000 = 100.00% [HOT edge]
; CHECK: edge entry -> case_a probability is 0x00000800 / 0x80000000 = 0.00%
; CHECK: edge entry -> case_b probability is 0x00000800 / 0x80000000 = 0.00%
; CHECK: edge entry -> case_c probability is 0x00000800 / 0x80000000 = 0.00%
; CHECK: edge entry -> case_d probability is 0x00000800 / 0x80000000 = 0.00%
; CHECK: edge entry -> exit probability is 0x7ffff800 / 0x80000000 = 100.00% [HOT edge]
; CHECK: edge entry -> case_a probability is 0x00000200 / 0x80000000 = 0.00%
; CHECK: edge entry -> case_b probability is 0x00000200 / 0x80000000 = 0.00%
; CHECK: edge entry -> case_c probability is 0x00000200 / 0x80000000 = 0.00%
; CHECK: edge entry -> case_d probability is 0x00000200 / 0x80000000 = 0.00%

case_a:
br label %case_b
Expand Down
2 changes: 1 addition & 1 deletion llvm/test/Transforms/JumpThreading/update-edge-weight.ll
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

; Test if edge weights are properly updated after jump threading.

; CHECK: !2 = !{!"branch_weights", i32 22, i32 7}
; CHECK: !2 = !{!"branch_weights", i32 1629125526, i32 518358122}

define void @foo(i32 %n) !prof !0 {
entry:
Expand Down