146 changes: 80 additions & 66 deletions llvm/lib/Transforms/Utils/SimplifyCFG.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/Local.h"
#include "llvm/Transforms/Utils/ValueMapper.h"
#include <algorithm>
#include <map>
#include <set>
Expand Down Expand Up @@ -93,6 +94,7 @@ namespace {

class SimplifyCFGOpt {
const TargetTransformInfo &TTI;
unsigned BonusInstThreshold;
const DataLayout *const DL;
AssumptionTracker *AT;
Value *isValueEqualityComparison(TerminatorInst *TI);
Expand All @@ -113,9 +115,9 @@ class SimplifyCFGOpt {
bool SimplifyCondBranch(BranchInst *BI, IRBuilder <>&Builder);

public:
SimplifyCFGOpt(const TargetTransformInfo &TTI, const DataLayout *DL,
AssumptionTracker *AT)
: TTI(TTI), DL(DL), AT(AT) {}
SimplifyCFGOpt(const TargetTransformInfo &TTI, unsigned BonusInstThreshold,
const DataLayout *DL, AssumptionTracker *AT)
: TTI(TTI), BonusInstThreshold(BonusInstThreshold), DL(DL), AT(AT) {}
bool run(BasicBlock *BB);
};
}
Expand Down Expand Up @@ -1973,7 +1975,8 @@ static bool checkCSEInPredecessor(Instruction *Inst, BasicBlock *PB) {
/// FoldBranchToCommonDest - If this basic block is simple enough, and if a
/// predecessor branches to us and one of our successors, fold the block into
/// the predecessor and use logical operations to pick the right destination.
bool llvm::FoldBranchToCommonDest(BranchInst *BI, const DataLayout *DL) {
bool llvm::FoldBranchToCommonDest(BranchInst *BI, const DataLayout *DL,
unsigned BonusInstThreshold) {
BasicBlock *BB = BI->getParent();

Instruction *Cond = nullptr;
Expand Down Expand Up @@ -2010,33 +2013,6 @@ bool llvm::FoldBranchToCommonDest(BranchInst *BI, const DataLayout *DL) {
Cond->getParent() != BB || !Cond->hasOneUse())
return false;

// Only allow this if the condition is a simple instruction that can be
// executed unconditionally. It must be in the same block as the branch, and
// must be at the front of the block.
BasicBlock::iterator FrontIt = BB->front();

// Ignore dbg intrinsics.
while (isa<DbgInfoIntrinsic>(FrontIt)) ++FrontIt;

// Allow a single instruction to be hoisted in addition to the compare
// that feeds the branch. We later ensure that any values that _it_ uses
// were also live in the predecessor, so that we don't unnecessarily create
// register pressure or inhibit out-of-order execution.
Instruction *BonusInst = nullptr;
if (&*FrontIt != Cond &&
FrontIt->hasOneUse() && FrontIt->user_back() == Cond &&
isSafeToSpeculativelyExecute(FrontIt, DL)) {
BonusInst = &*FrontIt;
++FrontIt;

// Ignore dbg intrinsics.
while (isa<DbgInfoIntrinsic>(FrontIt)) ++FrontIt;
}

// Only a single bonus inst is allowed.
if (&*FrontIt != Cond)
return false;

// Make sure the instruction after the condition is the cond branch.
BasicBlock::iterator CondIt = Cond; ++CondIt;

Expand All @@ -2046,6 +2022,31 @@ bool llvm::FoldBranchToCommonDest(BranchInst *BI, const DataLayout *DL) {
if (&*CondIt != BI)
return false;

// Only allow this transformation if computing the condition doesn't involve
// too many instructions and these involved instructions can be executed
// unconditionally. We denote all involved instructions except the condition
// as "bonus instructions", and only allow this transformation when the
// number of the bonus instructions does not exceed a certain threshold.
unsigned NumBonusInsts = 0;
for (auto I = BB->begin(); Cond != I; ++I) {
// Ignore dbg intrinsics.
if (isa<DbgInfoIntrinsic>(I))
continue;
if (!I->hasOneUse() || !isSafeToSpeculativelyExecute(I, DL))
return false;
// I has only one use and can be executed unconditionally.
Instruction *User = dyn_cast<Instruction>(I->user_back());
if (User == nullptr || User->getParent() != BB)
return false;
// I is used in the same BB. Since BI uses Cond and doesn't have more slots
// to use any other instruction, User must be an instruction between next(I)
// and Cond.
++NumBonusInsts;
// Early exits once we reach the limit.
if (NumBonusInsts > BonusInstThreshold)
return false;
}

// Cond is known to be a compare or binary operator. Check to make sure that
// neither operand is a potentially-trapping constant expression.
if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Cond->getOperand(0)))
Expand Down Expand Up @@ -2115,30 +2116,41 @@ bool llvm::FoldBranchToCommonDest(BranchInst *BI, const DataLayout *DL) {
PBI->swapSuccessors();
}

// If we have a bonus inst, clone it into the predecessor block.
Instruction *NewBonus = nullptr;
if (BonusInst) {
NewBonus = BonusInst->clone();
// If we have bonus instructions, clone them into the predecessor block.
// Note that there may be mutliple predecessor blocks, so we cannot move
// bonus instructions to a predecessor block.
ValueToValueMapTy VMap; // maps original values to cloned values
// We already make sure Cond is the last instruction before BI. Therefore,
// every instructions before Cond other than DbgInfoIntrinsic are bonus
// instructions.
for (auto BonusInst = BB->begin(); Cond != BonusInst; ++BonusInst) {
if (isa<DbgInfoIntrinsic>(BonusInst))
continue;
Instruction *NewBonusInst = BonusInst->clone();
RemapInstruction(NewBonusInst, VMap,
RF_NoModuleLevelChanges | RF_IgnoreMissingEntries);
VMap[BonusInst] = NewBonusInst;

// If we moved a load, we cannot any longer claim any knowledge about
// its potential value. The previous information might have been valid
// only given the branch precondition.
// For an analogous reason, we must also drop all the metadata whose
// semantics we don't understand.
NewBonus->dropUnknownMetadata(LLVMContext::MD_dbg);
NewBonusInst->dropUnknownMetadata(LLVMContext::MD_dbg);

PredBlock->getInstList().insert(PBI, NewBonus);
NewBonus->takeName(BonusInst);
BonusInst->setName(BonusInst->getName()+".old");
PredBlock->getInstList().insert(PBI, NewBonusInst);
NewBonusInst->takeName(BonusInst);
BonusInst->setName(BonusInst->getName() + ".old");
}

// Clone Cond into the predecessor basic block, and or/and the
// two conditions together.
Instruction *New = Cond->clone();
if (BonusInst) New->replaceUsesOfWith(BonusInst, NewBonus);
RemapInstruction(New, VMap,
RF_NoModuleLevelChanges | RF_IgnoreMissingEntries);
PredBlock->getInstList().insert(PBI, New);
New->takeName(Cond);
Cond->setName(New->getName()+".old");
Cond->setName(New->getName() + ".old");

if (BI->isConditional()) {
Instruction *NewCond =
Expand Down Expand Up @@ -2616,7 +2628,7 @@ static bool SimplifyIndirectBrOnSelect(IndirectBrInst *IBI, SelectInst *SI) {
/// the PHI, merging the third icmp into the switch.
static bool TryToSimplifyUncondBranchWithICmpInIt(
ICmpInst *ICI, IRBuilder<> &Builder, const TargetTransformInfo &TTI,
const DataLayout *DL, AssumptionTracker *AT) {
unsigned BonusInstThreshold, const DataLayout *DL, AssumptionTracker *AT) {
BasicBlock *BB = ICI->getParent();

// If the block has any PHIs in it or the icmp has multiple uses, it is too
Expand Down Expand Up @@ -2649,7 +2661,7 @@ static bool TryToSimplifyUncondBranchWithICmpInIt(
ICI->eraseFromParent();
}
// BB is now empty, so it is likely to simplify away.
return SimplifyCFG(BB, TTI, DL, AT) | true;
return SimplifyCFG(BB, TTI, BonusInstThreshold, DL, AT) | true;
}

// Ok, the block is reachable from the default dest. If the constant we're
Expand All @@ -2665,7 +2677,7 @@ static bool TryToSimplifyUncondBranchWithICmpInIt(
ICI->replaceAllUsesWith(V);
ICI->eraseFromParent();
// BB is now empty, so it is likely to simplify away.
return SimplifyCFG(BB, TTI, DL, AT) | true;
return SimplifyCFG(BB, TTI, BonusInstThreshold, DL, AT) | true;
}

// The use of the icmp has to be in the 'end' block, by the only PHI node in
Expand Down Expand Up @@ -3900,12 +3912,12 @@ bool SimplifyCFGOpt::SimplifySwitch(SwitchInst *SI, IRBuilder<> &Builder) {
// see if that predecessor totally determines the outcome of this switch.
if (BasicBlock *OnlyPred = BB->getSinglePredecessor())
if (SimplifyEqualityComparisonWithOnlyPredecessor(SI, OnlyPred, Builder))
return SimplifyCFG(BB, TTI, DL, AT) | true;
return SimplifyCFG(BB, TTI, BonusInstThreshold, DL, AT) | true;

Value *Cond = SI->getCondition();
if (SelectInst *Select = dyn_cast<SelectInst>(Cond))
if (SimplifySwitchOnSelect(SI, Select))
return SimplifyCFG(BB, TTI, DL, AT) | true;
return SimplifyCFG(BB, TTI, BonusInstThreshold, DL, AT) | true;

// If the block only contains the switch, see if we can fold the block
// away into any preds.
Expand All @@ -3915,22 +3927,22 @@ bool SimplifyCFGOpt::SimplifySwitch(SwitchInst *SI, IRBuilder<> &Builder) {
++BBI;
if (SI == &*BBI)
if (FoldValueComparisonIntoPredecessors(SI, Builder))
return SimplifyCFG(BB, TTI, DL, AT) | true;
return SimplifyCFG(BB, TTI, BonusInstThreshold, DL, AT) | true;
}

// Try to transform the switch into an icmp and a branch.
if (TurnSwitchRangeIntoICmp(SI, Builder))
return SimplifyCFG(BB, TTI, DL, AT) | true;
return SimplifyCFG(BB, TTI, BonusInstThreshold, DL, AT) | true;

// Remove unreachable cases.
if (EliminateDeadSwitchCases(SI, DL, AT))
return SimplifyCFG(BB, TTI, DL, AT) | true;
return SimplifyCFG(BB, TTI, BonusInstThreshold, DL, AT) | true;

if (ForwardSwitchConditionToPHI(SI))
return SimplifyCFG(BB, TTI, DL, AT) | true;
return SimplifyCFG(BB, TTI, BonusInstThreshold, DL, AT) | true;

if (SwitchToLookupTable(SI, Builder, TTI, DL))
return SimplifyCFG(BB, TTI, DL, AT) | true;
return SimplifyCFG(BB, TTI, BonusInstThreshold, DL, AT) | true;

return false;
}
Expand Down Expand Up @@ -3967,7 +3979,7 @@ bool SimplifyCFGOpt::SimplifyIndirectBr(IndirectBrInst *IBI) {

if (SelectInst *SI = dyn_cast<SelectInst>(IBI->getAddress())) {
if (SimplifyIndirectBrOnSelect(IBI, SI))
return SimplifyCFG(BB, TTI, DL, AT) | true;
return SimplifyCFG(BB, TTI, BonusInstThreshold, DL, AT) | true;
}
return Changed;
}
Expand All @@ -3991,16 +4003,17 @@ bool SimplifyCFGOpt::SimplifyUncondBranch(BranchInst *BI, IRBuilder<> &Builder){
for (++I; isa<DbgInfoIntrinsic>(I); ++I)
;
if (I->isTerminator() &&
TryToSimplifyUncondBranchWithICmpInIt(ICI, Builder, TTI, DL, AT))
TryToSimplifyUncondBranchWithICmpInIt(ICI, Builder, TTI,
BonusInstThreshold, DL, AT))
return true;
}

// If this basic block is ONLY a compare and a branch, and if a predecessor
// branches to us and our successor, fold the comparison into the
// predecessor and use logical operations to update the incoming value
// for PHI nodes in common successor.
if (FoldBranchToCommonDest(BI, DL))
return SimplifyCFG(BB, TTI, DL, AT) | true;
if (FoldBranchToCommonDest(BI, DL, BonusInstThreshold))
return SimplifyCFG(BB, TTI, BonusInstThreshold, DL, AT) | true;
return false;
}

Expand All @@ -4015,7 +4028,7 @@ bool SimplifyCFGOpt::SimplifyCondBranch(BranchInst *BI, IRBuilder<> &Builder) {
// switch.
if (BasicBlock *OnlyPred = BB->getSinglePredecessor())
if (SimplifyEqualityComparisonWithOnlyPredecessor(BI, OnlyPred, Builder))
return SimplifyCFG(BB, TTI, DL, AT) | true;
return SimplifyCFG(BB, TTI, BonusInstThreshold, DL, AT) | true;

// This block must be empty, except for the setcond inst, if it exists.
// Ignore dbg intrinsics.
Expand All @@ -4025,14 +4038,14 @@ bool SimplifyCFGOpt::SimplifyCondBranch(BranchInst *BI, IRBuilder<> &Builder) {
++I;
if (&*I == BI) {
if (FoldValueComparisonIntoPredecessors(BI, Builder))
return SimplifyCFG(BB, TTI, DL, AT) | true;
return SimplifyCFG(BB, TTI, BonusInstThreshold, DL, AT) | true;
} else if (&*I == cast<Instruction>(BI->getCondition())){
++I;
// Ignore dbg intrinsics.
while (isa<DbgInfoIntrinsic>(I))
++I;
if (&*I == BI && FoldValueComparisonIntoPredecessors(BI, Builder))
return SimplifyCFG(BB, TTI, DL, AT) | true;
return SimplifyCFG(BB, TTI, BonusInstThreshold, DL, AT) | true;
}
}

Expand All @@ -4043,8 +4056,8 @@ bool SimplifyCFGOpt::SimplifyCondBranch(BranchInst *BI, IRBuilder<> &Builder) {
// If this basic block is ONLY a compare and a branch, and if a predecessor
// branches to us and one of our successors, fold the comparison into the
// predecessor and use logical operations to pick the right destination.
if (FoldBranchToCommonDest(BI, DL))
return SimplifyCFG(BB, TTI, DL, AT) | true;
if (FoldBranchToCommonDest(BI, DL, BonusInstThreshold))
return SimplifyCFG(BB, TTI, BonusInstThreshold, DL, AT) | true;

// We have a conditional branch to two blocks that are only reachable
// from BI. We know that the condbr dominates the two blocks, so see if
Expand All @@ -4053,15 +4066,15 @@ bool SimplifyCFGOpt::SimplifyCondBranch(BranchInst *BI, IRBuilder<> &Builder) {
if (BI->getSuccessor(0)->getSinglePredecessor()) {
if (BI->getSuccessor(1)->getSinglePredecessor()) {
if (HoistThenElseCodeToIf(BI, DL))
return SimplifyCFG(BB, TTI, DL, AT) | true;
return SimplifyCFG(BB, TTI, BonusInstThreshold, DL, AT) | true;
} else {
// If Successor #1 has multiple preds, we may be able to conditionally
// execute Successor #0 if it branches to Successor #1.
TerminatorInst *Succ0TI = BI->getSuccessor(0)->getTerminator();
if (Succ0TI->getNumSuccessors() == 1 &&
Succ0TI->getSuccessor(0) == BI->getSuccessor(1))
if (SpeculativelyExecuteBB(BI, BI->getSuccessor(0), DL))
return SimplifyCFG(BB, TTI, DL, AT) | true;
return SimplifyCFG(BB, TTI, BonusInstThreshold, DL, AT) | true;
}
} else if (BI->getSuccessor(1)->getSinglePredecessor()) {
// If Successor #0 has multiple preds, we may be able to conditionally
Expand All @@ -4070,22 +4083,22 @@ bool SimplifyCFGOpt::SimplifyCondBranch(BranchInst *BI, IRBuilder<> &Builder) {
if (Succ1TI->getNumSuccessors() == 1 &&
Succ1TI->getSuccessor(0) == BI->getSuccessor(0))
if (SpeculativelyExecuteBB(BI, BI->getSuccessor(1), DL))
return SimplifyCFG(BB, TTI, DL, AT) | true;
return SimplifyCFG(BB, TTI, BonusInstThreshold, DL, AT) | true;
}

// If this is a branch on a phi node in the current block, thread control
// through this block if any PHI node entries are constants.
if (PHINode *PN = dyn_cast<PHINode>(BI->getCondition()))
if (PN->getParent() == BI->getParent())
if (FoldCondBranchOnPHI(BI, DL))
return SimplifyCFG(BB, TTI, DL, AT) | true;
return SimplifyCFG(BB, TTI, BonusInstThreshold, DL, AT) | true;

// Scan predecessor blocks for conditional branches.
for (pred_iterator PI = pred_begin(BB), E = pred_end(BB); PI != E; ++PI)
if (BranchInst *PBI = dyn_cast<BranchInst>((*PI)->getTerminator()))
if (PBI != BI && PBI->isConditional())
if (SimplifyCondBranchToCondBranch(PBI, BI))
return SimplifyCFG(BB, TTI, DL, AT) | true;
return SimplifyCFG(BB, TTI, BonusInstThreshold, DL, AT) | true;

return false;
}
Expand Down Expand Up @@ -4229,6 +4242,7 @@ bool SimplifyCFGOpt::run(BasicBlock *BB) {
/// of the CFG. It returns true if a modification was made.
///
bool llvm::SimplifyCFG(BasicBlock *BB, const TargetTransformInfo &TTI,
unsigned BonusInstThreshold,
const DataLayout *DL, AssumptionTracker *AT) {
return SimplifyCFGOpt(TTI, DL, AT).run(BB);
return SimplifyCFGOpt(TTI, BonusInstThreshold, DL, AT).run(BB);
}
28 changes: 28 additions & 0 deletions llvm/test/Transforms/SimplifyCFG/branch-fold-threshold.ll
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
; RUN: opt %s -simplifycfg -S | FileCheck %s --check-prefix=NORMAL
; RUN: opt %s -simplifycfg -S -bonus-inst-threshold=2 | FileCheck %s --check-prefix=AGGRESSIVE

define i32 @foo(i32 %a, i32 %b, i32 %c, i32 %d, i32* %input) {
; NORMAL-LABEL: @foo(
; AGGRESSIVE-LABEL: @foo(
entry:
%cmp = icmp sgt i32 %d, 3
br i1 %cmp, label %cond.end, label %lor.lhs.false
; NORMAL: br i1
; AGGRESSIVE: br i1

lor.lhs.false:
%mul = shl i32 %c, 1
%add = add nsw i32 %mul, %a
%cmp1 = icmp slt i32 %add, %b
br i1 %cmp1, label %cond.false, label %cond.end
; NORMAL: br i1
; AGGRESSIVE-NOT: br i1

cond.false:
%0 = load i32* %input, align 4
br label %cond.end

cond.end:
%cond = phi i32 [ %0, %cond.false ], [ 0, %lor.lhs.false ], [ 0, %entry ]
ret i32 %cond
}