diff --git a/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h b/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h index fcdd2aa0e06012..fc6b0cbab056d4 100644 --- a/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h +++ b/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h @@ -575,10 +575,11 @@ bool SplitIndirectBrCriticalEdges(Function &F, bool IgnoreBlocksWithoutPHI, /// may have additional information which simplifies this fixup. For example, /// see restoreSSA() in the UnifyLoopExits pass. BasicBlock *CreateControlFlowHub(DomTreeUpdater *DTU, - SmallVectorImpl &GuardBlocks, - const SetVector &Predecessors, - const SetVector &Successors, - const StringRef Prefix); + SmallVectorImpl &GuardBlocks, + const SetVector &Predecessors, + const SetVector &Successors, + const StringRef Prefix, + Optional MaxControlFlowBooleans = None); } // end namespace llvm diff --git a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp index d31c75161e1f4b..56b2f5796681e9 100644 --- a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp +++ b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp @@ -1591,7 +1591,7 @@ static void reconnectPhis(BasicBlock *Out, BasicBlock *GuardBlock, auto Phi = cast(I); auto NewPhi = PHINode::Create(Phi->getType(), Incoming.size(), - Phi->getName() + ".moved", &FirstGuardBlock->back()); + Phi->getName() + ".moved", &FirstGuardBlock->front()); for (auto *In : Incoming) { Value *V = UndefValue::get(Phi->getType()); if (In == Out) { @@ -1612,7 +1612,7 @@ static void reconnectPhis(BasicBlock *Out, BasicBlock *GuardBlock, } } -using BBPredicates = DenseMap; +using BBPredicates = DenseMap; using BBSetVector = SetVector; // Redirects the terminator of the incoming block to the first guard @@ -1683,30 +1683,60 @@ static void setupBranchForGuard(SmallVectorImpl &GuardBlocks, GuardBlocks.pop_back(); } -// Capture the existing control flow as guard predicates, and redirect -// control flow from \p Incoming block through the \p GuardBlocks to the -// \p Outgoing blocks. -// -// There is one guard predicate for each outgoing block OutBB. The -// predicate represents whether the hub should transfer control flow -// to OutBB. These predicates are NOT ORTHOGONAL. The Hub evaluates -// them in the same order as the Outgoing set-vector, and control -// branches to the first outgoing block whose predicate evaluates to true. -static void -convertToGuardPredicates(SmallVectorImpl &GuardBlocks, - SmallVectorImpl &DeletionCandidates, - const BBSetVector &Incoming, - const BBSetVector &Outgoing, const StringRef Prefix) { - BBPredicates GuardPredicates; - auto F = Incoming.front()->getParent(); +/// We are using one integer to represent the block we are branching to. Then at +/// each guard block, the predicate was calcuated using a simple `icmp eq`. +static void calcPredicateUsingInteger( + const BBSetVector &Incoming, const BBSetVector &Outgoing, + SmallVectorImpl &GuardBlocks, BBPredicates &GuardPredicates) { auto &Context = Incoming.front()->getContext(); - auto BoolTrue = ConstantInt::getTrue(Context); - auto BoolFalse = ConstantInt::getFalse(Context); + auto FirstGuardBlock = GuardBlocks.front(); - for (int i = 0, e = Outgoing.size() - 1; i != e; ++i) - GuardBlocks.push_back( - BasicBlock::Create(F->getContext(), Prefix + ".guard", F)); + auto Phi = PHINode::Create(Type::getInt32Ty(Context), Incoming.size(), + "merged.bb.idx", FirstGuardBlock); + + for (auto In : Incoming) { + Value *Condition; + BasicBlock *Succ0; + BasicBlock *Succ1; + std::tie(Condition, Succ0, Succ1) = + redirectToHub(In, FirstGuardBlock, Outgoing); + Value *IncomingId = nullptr; + if (Succ0 && Succ1) { + // target_bb_index = Condition ? index_of_succ0 : index_of_succ1. + auto Succ0Iter = find(Outgoing, Succ0); + auto Succ1Iter = find(Outgoing, Succ1); + Value *Id0 = ConstantInt::get(Type::getInt32Ty(Context), + std::distance(Outgoing.begin(), Succ0Iter)); + Value *Id1 = ConstantInt::get(Type::getInt32Ty(Context), + std::distance(Outgoing.begin(), Succ1Iter)); + IncomingId = SelectInst::Create(Condition, Id0, Id1, "target.bb.idx", + In->getTerminator()); + } else { + // Get the index of the non-null successor. + auto SuccIter = Succ0 ? find(Outgoing, Succ0) : find(Outgoing, Succ1); + IncomingId = ConstantInt::get(Type::getInt32Ty(Context), + std::distance(Outgoing.begin(), SuccIter)); + } + Phi->addIncoming(IncomingId, In); + } + + for (int i = 0, e = Outgoing.size() - 1; i != e; ++i) { + auto Out = Outgoing[i]; + auto Cmp = ICmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, Phi, + ConstantInt::get(Type::getInt32Ty(Context), i), + Out->getName() + ".predicate", GuardBlocks[i]); + GuardPredicates[Out] = Cmp; + } +} +/// We record the predicate of each outgoing block using a phi of boolean. +static void calcPredicateUsingBooleans( + const BBSetVector &Incoming, const BBSetVector &Outgoing, + SmallVectorImpl &GuardBlocks, BBPredicates &GuardPredicates, + SmallVectorImpl &DeletionCandidates) { + auto &Context = Incoming.front()->getContext(); + auto BoolTrue = ConstantInt::getTrue(Context); + auto BoolFalse = ConstantInt::getFalse(Context); auto FirstGuardBlock = GuardBlocks.front(); // The predicate for the last outgoing is trivially true, and so we @@ -1738,7 +1768,7 @@ convertToGuardPredicates(SmallVectorImpl &GuardBlocks, bool OneSuccessorDone = false; for (int i = 0, e = Outgoing.size() - 1; i != e; ++i) { auto Out = Outgoing[i]; - auto Phi = GuardPredicates[Out]; + PHINode *Phi = cast(GuardPredicates[Out]); if (Out != Succ0 && Out != Succ1) { Phi->addIncoming(BoolFalse, In); } else if (!Succ0 || !Succ1 || OneSuccessorDone) { @@ -1758,13 +1788,48 @@ convertToGuardPredicates(SmallVectorImpl &GuardBlocks, } } } +} + +// Capture the existing control flow as guard predicates, and redirect +// control flow from \p Incoming block through the \p GuardBlocks to the +// \p Outgoing blocks. +// +// There is one guard predicate for each outgoing block OutBB. The +// predicate represents whether the hub should transfer control flow +// to OutBB. These predicates are NOT ORTHOGONAL. The Hub evaluates +// them in the same order as the Outgoing set-vector, and control +// branches to the first outgoing block whose predicate evaluates to true. +static void +convertToGuardPredicates(SmallVectorImpl &GuardBlocks, + SmallVectorImpl &DeletionCandidates, + const BBSetVector &Incoming, + const BBSetVector &Outgoing, const StringRef Prefix, + Optional MaxControlFlowBooleans) { + BBPredicates GuardPredicates; + auto F = Incoming.front()->getParent(); + + for (int i = 0, e = Outgoing.size() - 1; i != e; ++i) + GuardBlocks.push_back( + BasicBlock::Create(F->getContext(), Prefix + ".guard", F)); + + // When we are using an integer to record which target block to jump to, we + // are creating less live values, actually we are using one single integer to + // store the index of the target block. When we are using booleans to store + // the branching information, we need (N-1) boolean values, where N is the + // number of outgoing block. + if (!MaxControlFlowBooleans || Outgoing.size() <= *MaxControlFlowBooleans) + calcPredicateUsingBooleans(Incoming, Outgoing, GuardBlocks, GuardPredicates, + DeletionCandidates); + else + calcPredicateUsingInteger(Incoming, Outgoing, GuardBlocks, GuardPredicates); + setupBranchForGuard(GuardBlocks, Outgoing, GuardPredicates); } BasicBlock *llvm::CreateControlFlowHub( DomTreeUpdater *DTU, SmallVectorImpl &GuardBlocks, const BBSetVector &Incoming, const BBSetVector &Outgoing, - const StringRef Prefix) { + const StringRef Prefix, Optional MaxControlFlowBooleans) { if (Outgoing.size() < 2) return Outgoing.front(); @@ -1779,7 +1844,7 @@ BasicBlock *llvm::CreateControlFlowHub( SmallVector DeletionCandidates; convertToGuardPredicates(GuardBlocks, DeletionCandidates, Incoming, Outgoing, - Prefix); + Prefix, MaxControlFlowBooleans); auto FirstGuardBlock = GuardBlocks.front(); // Update the PHINodes in each outgoing block to match the new control flow. diff --git a/llvm/lib/Transforms/Utils/UnifyLoopExits.cpp b/llvm/lib/Transforms/Utils/UnifyLoopExits.cpp index 03140ef7feb1e2..3be96ebc93a2a7 100644 --- a/llvm/lib/Transforms/Utils/UnifyLoopExits.cpp +++ b/llvm/lib/Transforms/Utils/UnifyLoopExits.cpp @@ -23,6 +23,7 @@ #include "llvm/IR/Constants.h" #include "llvm/IR/Dominators.h" #include "llvm/InitializePasses.h" +#include "llvm/Support/CommandLine.h" #include "llvm/Transforms/Utils.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" @@ -30,6 +31,11 @@ using namespace llvm; +static cl::opt MaxBooleansInControlFlowHub( + "max-booleans-in-control-flow-hub", cl::init(32), cl::Hidden, + cl::desc("Set the maximum number of outgoing blocks for using a boolean " + "value to record the exiting block in CreateControlFlowHub.")); + namespace { struct UnifyLoopExitsLegacyPass : public FunctionPass { static char ID; @@ -114,9 +120,9 @@ static void restoreSSA(const DominatorTree &DT, const Loop *L, // didn't exist in the original CFG. auto Def = II.first; LLVM_DEBUG(dbgs() << "externally used: " << Def->getName() << "\n"); - auto NewPhi = PHINode::Create(Def->getType(), Incoming.size(), - Def->getName() + ".moved", - LoopExitBlock->getTerminator()); + auto NewPhi = + PHINode::Create(Def->getType(), Incoming.size(), + Def->getName() + ".moved", &LoopExitBlock->front()); for (auto *In : Incoming) { LLVM_DEBUG(dbgs() << "predecessor " << In->getName() << ": "); if (Def->getParent() == In || DT.dominates(Def, In)) { @@ -181,8 +187,9 @@ static bool unifyLoopExits(DominatorTree &DT, LoopInfo &LI, Loop *L) { SmallVector GuardBlocks; DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Eager); - auto LoopExitBlock = CreateControlFlowHub(&DTU, GuardBlocks, ExitingBlocks, - Exits, "loop.exit"); + auto LoopExitBlock = + CreateControlFlowHub(&DTU, GuardBlocks, ExitingBlocks, Exits, "loop.exit", + MaxBooleansInControlFlowHub.getValue()); restoreSSA(DT, L, ExitingBlocks, LoopExitBlock); diff --git a/llvm/test/CodeGen/AMDGPU/si-annotate-cf.ll b/llvm/test/CodeGen/AMDGPU/si-annotate-cf.ll index 79a733fe5a879c..695577a902e3f9 100644 --- a/llvm/test/CodeGen/AMDGPU/si-annotate-cf.ll +++ b/llvm/test/CodeGen/AMDGPU/si-annotate-cf.ll @@ -185,8 +185,8 @@ define amdgpu_kernel void @loop_land_info_assert(i32 %c0, i32 %c1, i32 %c2, i32 ; SI-NEXT: v_mov_b32_e32 v0, 3 ; SI-NEXT: s_branch .LBB3_3 ; SI-NEXT: .LBB3_1: ; in Loop: Header=BB3_3 Depth=1 -; SI-NEXT: s_mov_b64 s[10:11], -1 ; SI-NEXT: s_mov_b64 s[8:9], 0 +; SI-NEXT: s_mov_b64 s[10:11], -1 ; SI-NEXT: s_mov_b64 s[12:13], -1 ; SI-NEXT: .LBB3_2: ; %Flow ; SI-NEXT: ; in Loop: Header=BB3_3 Depth=1 @@ -194,8 +194,8 @@ define amdgpu_kernel void @loop_land_info_assert(i32 %c0, i32 %c1, i32 %c2, i32 ; SI-NEXT: s_cbranch_vccnz .LBB3_8 ; SI-NEXT: .LBB3_3: ; %while.cond ; SI-NEXT: ; =>This Inner Loop Header: Depth=1 -; SI-NEXT: s_mov_b64 s[8:9], -1 ; SI-NEXT: s_mov_b64 s[10:11], -1 +; SI-NEXT: s_mov_b64 s[8:9], -1 ; SI-NEXT: s_mov_b64 s[12:13], -1 ; SI-NEXT: s_mov_b64 vcc, s[0:1] ; SI-NEXT: s_cbranch_vccz .LBB3_2 @@ -260,8 +260,8 @@ define amdgpu_kernel void @loop_land_info_assert(i32 %c0, i32 %c1, i32 %c2, i32 ; FLAT-NEXT: v_mov_b32_e32 v0, 3 ; FLAT-NEXT: s_branch .LBB3_3 ; FLAT-NEXT: .LBB3_1: ; in Loop: Header=BB3_3 Depth=1 -; FLAT-NEXT: s_mov_b64 s[10:11], -1 ; FLAT-NEXT: s_mov_b64 s[8:9], 0 +; FLAT-NEXT: s_mov_b64 s[10:11], -1 ; FLAT-NEXT: s_mov_b64 s[12:13], -1 ; FLAT-NEXT: .LBB3_2: ; %Flow ; FLAT-NEXT: ; in Loop: Header=BB3_3 Depth=1 @@ -269,8 +269,8 @@ define amdgpu_kernel void @loop_land_info_assert(i32 %c0, i32 %c1, i32 %c2, i32 ; FLAT-NEXT: s_cbranch_vccnz .LBB3_8 ; FLAT-NEXT: .LBB3_3: ; %while.cond ; FLAT-NEXT: ; =>This Inner Loop Header: Depth=1 -; FLAT-NEXT: s_mov_b64 s[8:9], -1 ; FLAT-NEXT: s_mov_b64 s[10:11], -1 +; FLAT-NEXT: s_mov_b64 s[8:9], -1 ; FLAT-NEXT: s_mov_b64 s[12:13], -1 ; FLAT-NEXT: s_mov_b64 vcc, s[0:1] ; FLAT-NEXT: s_cbranch_vccz .LBB3_2 diff --git a/llvm/test/Transforms/FixIrreducible/basic.ll b/llvm/test/Transforms/FixIrreducible/basic.ll index fe34ac9b93bc78..7ba1360160db20 100644 --- a/llvm/test/Transforms/FixIrreducible/basic.ll +++ b/llvm/test/Transforms/FixIrreducible/basic.ll @@ -15,9 +15,9 @@ define i32 @basic(i1 %PredEntry, i1 %PredLeft, i1 %PredRight, i32 %X, i32 %Y) { ; CHECK-NEXT: [[Z:%.*]] = phi i32 [ [[L]], [[LEFT:%.*]] ], [ [[R_PHI_MOVED:%.*]], [[RIGHT:%.*]] ] ; CHECK-NEXT: ret i32 [[Z]] ; CHECK: irr.guard: -; CHECK-NEXT: [[GUARD_LEFT:%.*]] = phi i1 [ true, [[RIGHT]] ], [ [[PREDENTRY:%.*]], [[ENTRY:%.*]] ], [ false, [[LEFT]] ] +; CHECK-NEXT: [[R_PHI_MOVED]] = phi i32 [ [[R_PHI_MOVED]], [[RIGHT]] ], [ [[Y:%.*]], [[ENTRY:%.*]] ], [ [[L]], [[LEFT]] ] ; CHECK-NEXT: [[L_PHI_MOVED]] = phi i32 [ [[R_PHI_MOVED]], [[RIGHT]] ], [ [[X:%.*]], [[ENTRY]] ], [ [[L_PHI_MOVED]], [[LEFT]] ] -; CHECK-NEXT: [[R_PHI_MOVED]] = phi i32 [ [[R_PHI_MOVED]], [[RIGHT]] ], [ [[Y:%.*]], [[ENTRY]] ], [ [[L]], [[LEFT]] ] +; CHECK-NEXT: [[GUARD_LEFT:%.*]] = phi i1 [ true, [[RIGHT]] ], [ [[PREDENTRY:%.*]], [[ENTRY]] ], [ false, [[LEFT]] ] ; CHECK-NEXT: br i1 [[GUARD_LEFT]], label [[LEFT]], label [[RIGHT]] ; entry: @@ -49,9 +49,9 @@ define i32 @feedback_loop(i1 %PredEntry, i1 %PredLeft, i1 %PredRight, i32 %X, i3 ; CHECK-NEXT: [[Z:%.*]] = phi i32 [ [[L_PHI_MOVED:%.*]], [[LEFT:%.*]] ], [ [[R_PHI_MOVED:%.*]], [[RIGHT:%.*]] ] ; CHECK-NEXT: ret i32 [[Z]] ; CHECK: irr.guard: -; CHECK-NEXT: [[GUARD_LEFT:%.*]] = phi i1 [ true, [[RIGHT]] ], [ [[PREDENTRY:%.*]], [[ENTRY:%.*]] ], [ false, [[LEFT]] ] +; CHECK-NEXT: [[R_PHI_MOVED]] = phi i32 [ [[R_PHI_MOVED]], [[RIGHT]] ], [ [[Y:%.*]], [[ENTRY:%.*]] ], [ [[L_PHI_MOVED]], [[LEFT]] ] ; CHECK-NEXT: [[L_PHI_MOVED]] = phi i32 [ [[R_PHI_MOVED]], [[RIGHT]] ], [ [[X:%.*]], [[ENTRY]] ], [ [[L_PHI_MOVED]], [[LEFT]] ] -; CHECK-NEXT: [[R_PHI_MOVED]] = phi i32 [ [[R_PHI_MOVED]], [[RIGHT]] ], [ [[Y:%.*]], [[ENTRY]] ], [ [[L_PHI_MOVED]], [[LEFT]] ] +; CHECK-NEXT: [[GUARD_LEFT:%.*]] = phi i1 [ true, [[RIGHT]] ], [ [[PREDENTRY:%.*]], [[ENTRY]] ], [ false, [[LEFT]] ] ; CHECK-NEXT: br i1 [[GUARD_LEFT]], label [[LEFT]], label [[RIGHT]] ; entry: @@ -89,9 +89,9 @@ define i32 @multiple_predecessors(i1 %PredEntry, i1 %PredA, i1 %PredB, i1 %PredC ; CHECK-NEXT: [[RET:%.*]] = phi i32 [ [[C_PHI_MOVED:%.*]], [[C:%.*]] ], [ [[D_INC]], [[D:%.*]] ] ; CHECK-NEXT: ret i32 [[RET]] ; CHECK: irr.guard: +; CHECK-NEXT: [[D_PHI_MOVED]] = phi i32 [ [[D_PHI_MOVED]], [[D]] ], [ [[Y:%.*]], [[B]] ], [ [[A_INC]], [[A]] ], [ [[C_PHI_MOVED]], [[C]] ] +; CHECK-NEXT: [[C_PHI_MOVED]] = phi i32 [ [[D_INC]], [[D]] ], [ [[Y]], [[B]] ], [ [[X]], [[A]] ], [ [[C_PHI_MOVED]], [[C]] ] ; CHECK-NEXT: [[GUARD_C:%.*]] = phi i1 [ true, [[D]] ], [ [[PREDB_INV]], [[B]] ], [ [[PREDA:%.*]], [[A]] ], [ false, [[C]] ] -; CHECK-NEXT: [[C_PHI_MOVED]] = phi i32 [ [[D_INC]], [[D]] ], [ [[Y:%.*]], [[B]] ], [ [[X]], [[A]] ], [ [[C_PHI_MOVED]], [[C]] ] -; CHECK-NEXT: [[D_PHI_MOVED]] = phi i32 [ [[D_PHI_MOVED]], [[D]] ], [ [[Y]], [[B]] ], [ [[A_INC]], [[A]] ], [ [[C_PHI_MOVED]], [[C]] ] ; CHECK-NEXT: br i1 [[GUARD_C]], label [[C]], label [[D]] ; entry: @@ -136,9 +136,9 @@ define i32 @separate_predecessors(i1 %PredEntry, i1 %PredA, i1 %PredB, i1 %PredC ; CHECK-NEXT: [[RET:%.*]] = phi i32 [ [[C_PHI_MOVED:%.*]], [[C:%.*]] ], [ [[D_INC]], [[D:%.*]] ] ; CHECK-NEXT: ret i32 [[RET]] ; CHECK: irr.guard: -; CHECK-NEXT: [[GUARD_C:%.*]] = phi i1 [ true, [[D]] ], [ true, [[A]] ], [ false, [[C]] ], [ false, [[B]] ] -; CHECK-NEXT: [[C_PHI_MOVED]] = phi i32 [ [[D_INC]], [[D]] ], [ [[X]], [[A]] ], [ [[C_PHI_MOVED]], [[C]] ], [ undef, [[B]] ] ; CHECK-NEXT: [[D_PHI_MOVED]] = phi i32 [ [[D_PHI_MOVED]], [[D]] ], [ undef, [[A]] ], [ [[C_PHI_MOVED]], [[C]] ], [ [[Y:%.*]], [[B]] ] +; CHECK-NEXT: [[C_PHI_MOVED]] = phi i32 [ [[D_INC]], [[D]] ], [ [[X]], [[A]] ], [ [[C_PHI_MOVED]], [[C]] ], [ undef, [[B]] ] +; CHECK-NEXT: [[GUARD_C:%.*]] = phi i1 [ true, [[D]] ], [ true, [[A]] ], [ false, [[C]] ], [ false, [[B]] ] ; CHECK-NEXT: br i1 [[GUARD_C]], label [[C]], label [[D]] ; entry: @@ -237,9 +237,9 @@ define i32 @hidden_nodes(i1 %PredEntry, i1 %PredA, i1 %PredB, i1 %PredC, i1 %Pre ; CHECK: exit: ; CHECK-NEXT: ret i32 [[B_PHI_MOVED]] ; CHECK: irr.guard: -; CHECK-NEXT: [[GUARD_A:%.*]] = phi i1 [ true, [[E]] ], [ [[PREDENTRY:%.*]], [[ENTRY:%.*]] ], [ false, [[A:%.*]] ] +; CHECK-NEXT: [[B_PHI_MOVED]] = phi i32 [ undef, [[E]] ], [ [[Y:%.*]], [[ENTRY:%.*]] ], [ [[A_INC]], [[A:%.*]] ] ; CHECK-NEXT: [[A_PHI_MOVED]] = phi i32 [ [[C_INC]], [[E]] ], [ [[X:%.*]], [[ENTRY]] ], [ [[A_PHI_MOVED]], [[A]] ] -; CHECK-NEXT: [[B_PHI_MOVED]] = phi i32 [ undef, [[E]] ], [ [[Y:%.*]], [[ENTRY]] ], [ [[A_INC]], [[A]] ] +; CHECK-NEXT: [[GUARD_A:%.*]] = phi i1 [ true, [[E]] ], [ [[PREDENTRY:%.*]], [[ENTRY]] ], [ false, [[A]] ] ; CHECK-NEXT: br i1 [[GUARD_A]], label [[A]], label [[B:%.*]] ; entry: diff --git a/llvm/test/Transforms/StructurizeCFG/workarounds/needs-fr-ule.ll b/llvm/test/Transforms/StructurizeCFG/workarounds/needs-fr-ule.ll index b81f83a9e7094f..f588d3fe7cef3e 100644 --- a/llvm/test/Transforms/StructurizeCFG/workarounds/needs-fr-ule.ll +++ b/llvm/test/Transforms/StructurizeCFG/workarounds/needs-fr-ule.ll @@ -56,12 +56,12 @@ define void @irreducible_mountain_bug(i1 %Pred0, i1 %Pred1, i1 %Pred2, i1 %Pred3 ; CHECK-NEXT: br i1 [[PRED5_INV]], label [[WHILE_COND47:%.*]], label [[FLOW10:%.*]] ; CHECK: Flow9: ; CHECK-NEXT: [[TMP10:%.*]] = phi i1 [ [[TMP22:%.*]], [[FLOW10]] ], [ undef, [[FLOW8]] ] -; CHECK-NEXT: [[TMP11:%.*]] = phi i1 [ false, [[FLOW10]] ], [ undef, [[FLOW8]] ] -; CHECK-NEXT: [[TMP12:%.*]] = phi i1 [ true, [[FLOW10]] ], [ [[TMP7]], [[FLOW8]] ] -; CHECK-NEXT: [[TMP13:%.*]] = phi i1 [ false, [[FLOW10]] ], [ [[TMP8]], [[FLOW8]] ] +; CHECK-NEXT: [[TMP11:%.*]] = phi i1 [ true, [[FLOW10]] ], [ [[TMP7]], [[FLOW8]] ] +; CHECK-NEXT: [[TMP12:%.*]] = phi i1 [ false, [[FLOW10]] ], [ [[TMP8]], [[FLOW8]] ] +; CHECK-NEXT: [[TMP13:%.*]] = phi i1 [ false, [[FLOW10]] ], [ undef, [[FLOW8]] ] ; CHECK-NEXT: [[TMP14:%.*]] = phi i1 [ [[TMP23:%.*]], [[FLOW10]] ], [ true, [[FLOW8]] ] -; CHECK-NEXT: [[DOTINV11:%.*]] = xor i1 [[TMP12]], true -; CHECK-NEXT: [[DOTINV:%.*]] = xor i1 [[TMP13]], true +; CHECK-NEXT: [[DOTINV11:%.*]] = xor i1 [[TMP11]], true +; CHECK-NEXT: [[DOTINV:%.*]] = xor i1 [[TMP12]], true ; CHECK-NEXT: br i1 [[TMP14]], label [[LOOP_EXIT_GUARD1:%.*]], label [[IRR_GUARD]] ; CHECK: while.cond47: ; CHECK-NEXT: br label [[FLOW10]] @@ -111,7 +111,7 @@ define void @irreducible_mountain_bug(i1 %Pred0, i1 %Pred1, i1 %Pred2, i1 %Pred3 ; CHECK-NEXT: [[GUARD_COND_TRUE49_INV:%.*]] = xor i1 [[GUARD_COND_TRUE49]], true ; CHECK-NEXT: br i1 [[GUARD_COND_TRUE49_INV]], label [[COND_END61]], label [[FLOW7]] ; CHECK: Flow15: -; CHECK-NEXT: [[TMP20]] = phi i1 [ false, [[IF_THEN69:%.*]] ], [ [[TMP11]], [[LOOP_EXIT_GUARD2:%.*]] ] +; CHECK-NEXT: [[TMP20]] = phi i1 [ false, [[IF_THEN69:%.*]] ], [ [[TMP13]], [[LOOP_EXIT_GUARD2:%.*]] ] ; CHECK-NEXT: [[TMP21]] = phi i1 [ [[PRED8:%.*]], [[IF_THEN69]] ], [ [[DOTINV]], [[LOOP_EXIT_GUARD2]] ] ; CHECK-NEXT: br label [[FLOW14:%.*]] ; CHECK: loop.exit.guard: diff --git a/llvm/test/Transforms/StructurizeCFG/workarounds/needs-unified-loop-exits.ll b/llvm/test/Transforms/StructurizeCFG/workarounds/needs-unified-loop-exits.ll index 6cc2689f08fd02..4b3a43000f17a3 100644 --- a/llvm/test/Transforms/StructurizeCFG/workarounds/needs-unified-loop-exits.ll +++ b/llvm/test/Transforms/StructurizeCFG/workarounds/needs-unified-loop-exits.ll @@ -36,8 +36,8 @@ define void @exiting-block(i1 %PredH1, i1 %PredB2, i1 %PredB1, i1 %PredH2) { ; CHECK: B2: ; CHECK-NEXT: br i1 [[PREDB2_INV]], label [[L2:%.*]], label [[FLOW2:%.*]] ; CHECK: Flow: -; CHECK-NEXT: [[TMP2:%.*]] = phi i1 [ false, [[FLOW2]] ], [ undef, [[H2]] ] -; CHECK-NEXT: [[TMP3:%.*]] = phi i1 [ false, [[FLOW2]] ], [ true, [[H2]] ] +; CHECK-NEXT: [[TMP2:%.*]] = phi i1 [ false, [[FLOW2]] ], [ true, [[H2]] ] +; CHECK-NEXT: [[TMP3:%.*]] = phi i1 [ false, [[FLOW2]] ], [ undef, [[H2]] ] ; CHECK-NEXT: [[TMP4:%.*]] = phi i1 [ [[TMP7:%.*]], [[FLOW2]] ], [ true, [[H2]] ] ; CHECK-NEXT: br i1 [[TMP4]], label [[LOOP_EXIT_GUARD1:%.*]], label [[H2]] ; CHECK: L2: @@ -51,7 +51,7 @@ define void @exiting-block(i1 %PredH1, i1 %PredB2, i1 %PredB1, i1 %PredH2) { ; CHECK: exit: ; CHECK-NEXT: ret void ; CHECK: Flow5: -; CHECK-NEXT: [[TMP5:%.*]] = phi i1 [ undef, [[L1:%.*]] ], [ [[TMP2]], [[LOOP_EXIT_GUARD1]] ] +; CHECK-NEXT: [[TMP5:%.*]] = phi i1 [ undef, [[L1:%.*]] ], [ [[TMP3]], [[LOOP_EXIT_GUARD1]] ] ; CHECK-NEXT: [[TMP6:%.*]] = phi i1 [ false, [[L1]] ], [ true, [[LOOP_EXIT_GUARD1]] ] ; CHECK-NEXT: br label [[FLOW4]] ; CHECK: loop.exit.guard: @@ -64,7 +64,7 @@ define void @exiting-block(i1 %PredH1, i1 %PredB2, i1 %PredB1, i1 %PredH2) { ; CHECK-NEXT: [[TMP9:%.*]] = phi i1 [ [[TMP6]], [[FLOW5]] ], [ true, [[FLOW3]] ] ; CHECK-NEXT: br i1 [[TMP9]], label [[LOOP_EXIT_GUARD:%.*]], label [[H1]] ; CHECK: loop.exit.guard1: -; CHECK-NEXT: br i1 [[TMP3]], label [[L1]], label [[FLOW5]] +; CHECK-NEXT: br i1 [[TMP2]], label [[L1]], label [[FLOW5]] ; entry: br label %H1 @@ -116,8 +116,8 @@ define void @incorrect-backedge(i1 %PredH2, i1 %PredH3, i1 %PredL2, i1 %PredL13, ; CHECK-NEXT: br i1 [[PREDL2_INV]], label [[L13:%.*]], label [[FLOW3:%.*]] ; CHECK: Flow: ; CHECK-NEXT: [[TMP0:%.*]] = phi i1 [ [[TMP7:%.*]], [[FLOW3]] ], [ true, [[H3]] ] -; CHECK-NEXT: [[TMP1:%.*]] = phi i1 [ [[TMP7]], [[FLOW3]] ], [ false, [[H3]] ] -; CHECK-NEXT: [[TMP2:%.*]] = phi i1 [ [[TMP8:%.*]], [[FLOW3]] ], [ true, [[H3]] ] +; CHECK-NEXT: [[TMP1:%.*]] = phi i1 [ [[TMP8:%.*]], [[FLOW3]] ], [ false, [[H3]] ] +; CHECK-NEXT: [[TMP2:%.*]] = phi i1 [ [[TMP8]], [[FLOW3]] ], [ true, [[H3]] ] ; CHECK-NEXT: [[TMP3:%.*]] = phi i1 [ [[TMP9:%.*]], [[FLOW3]] ], [ true, [[H3]] ] ; CHECK-NEXT: br i1 [[TMP3]], label [[LOOP_EXIT_GUARD2:%.*]], label [[H3]] ; CHECK: L13: @@ -138,14 +138,14 @@ define void @incorrect-backedge(i1 %PredH2, i1 %PredH3, i1 %PredL2, i1 %PredL13, ; CHECK: loop.exit.guard1: ; CHECK-NEXT: br label [[FLOW5]] ; CHECK: Flow3: -; CHECK-NEXT: [[TMP7]] = phi i1 [ false, [[L13]] ], [ undef, [[L2]] ] -; CHECK-NEXT: [[TMP8]] = phi i1 [ true, [[L13]] ], [ false, [[L2]] ] +; CHECK-NEXT: [[TMP7]] = phi i1 [ true, [[L13]] ], [ false, [[L2]] ] +; CHECK-NEXT: [[TMP8]] = phi i1 [ false, [[L13]] ], [ undef, [[L2]] ] ; CHECK-NEXT: [[TMP9]] = phi i1 [ [[PREDL13_INV]], [[L13]] ], [ true, [[L2]] ] ; CHECK-NEXT: br label [[FLOW]] ; CHECK: Flow4: -; CHECK-NEXT: [[TMP10]] = phi i1 [ [[TMP0]], [[LOOP_EXIT_GUARD2]] ], [ false, [[H2]] ] +; CHECK-NEXT: [[TMP10]] = phi i1 [ [[TMP2]], [[LOOP_EXIT_GUARD2]] ], [ false, [[H2]] ] ; CHECK-NEXT: [[TMP11:%.*]] = phi i1 [ [[TMP1]], [[LOOP_EXIT_GUARD2]] ], [ true, [[H2]] ] -; CHECK-NEXT: [[TMP12:%.*]] = phi i1 [ [[TMP2]], [[LOOP_EXIT_GUARD2]] ], [ true, [[H2]] ] +; CHECK-NEXT: [[TMP12:%.*]] = phi i1 [ [[TMP0]], [[LOOP_EXIT_GUARD2]] ], [ true, [[H2]] ] ; CHECK-NEXT: [[DOTINV]] = xor i1 [[TMP11]], true ; CHECK-NEXT: br i1 [[TMP12]], label [[LOOP_EXIT_GUARD]], label [[H2]] ; CHECK: loop.exit.guard2: diff --git a/llvm/test/Transforms/UnifyLoopExits/integer_guards.ll b/llvm/test/Transforms/UnifyLoopExits/integer_guards.ll new file mode 100644 index 00000000000000..803f14b24a9767 --- /dev/null +++ b/llvm/test/Transforms/UnifyLoopExits/integer_guards.ll @@ -0,0 +1,348 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt < %s -passes=unify-loop-exits -max-booleans-in-control-flow-hub=1 -S | FileCheck %s +; RUN: opt < %s -passes=unify-loop-exits -S | FileCheck --check-prefix=BOOLEAN %s + +; A loop with multiple exit blocks. + +define void @loop_two_exits(i1 %PredEntry, i1 %PredA) { +; CHECK-LABEL: @loop_two_exits( +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[PREDENTRY:%.*]], label [[A:%.*]], label [[E:%.*]] +; CHECK: A: +; CHECK-NEXT: [[INC1:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC2:%.*]], [[C:%.*]] ] +; CHECK-NEXT: br i1 [[PREDA:%.*]], label [[LOOP_EXIT_GUARD:%.*]], label [[C]] +; CHECK: B: +; CHECK-NEXT: tail call fastcc void @check(i32 1) #[[ATTR0:[0-9]+]] +; CHECK-NEXT: br label [[D:%.*]] +; CHECK: C: +; CHECK-NEXT: [[INC2]] = add i32 [[INC1]], 1 +; CHECK-NEXT: [[CMP:%.*]] = icmp ult i32 [[INC2]], 10 +; CHECK-NEXT: br i1 [[CMP]], label [[A]], label [[LOOP_EXIT_GUARD]] +; CHECK: D: +; CHECK-NEXT: unreachable +; CHECK: E: +; CHECK-NEXT: ret void +; CHECK: loop.exit.guard: +; CHECK-NEXT: [[MERGED_BB_IDX:%.*]] = phi i32 [ 0, [[A]] ], [ 1, [[C]] ] +; CHECK-NEXT: [[B_PREDICATE:%.*]] = icmp eq i32 [[MERGED_BB_IDX]], 0 +; CHECK-NEXT: br i1 [[B_PREDICATE]], label [[B:%.*]], label [[E]] +; +; BOOLEAN-LABEL: @loop_two_exits( +; BOOLEAN-NEXT: entry: +; BOOLEAN-NEXT: br i1 [[PREDENTRY:%.*]], label [[A:%.*]], label [[E:%.*]] +; BOOLEAN: A: +; BOOLEAN-NEXT: [[INC1:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC2:%.*]], [[C:%.*]] ] +; BOOLEAN-NEXT: br i1 [[PREDA:%.*]], label [[LOOP_EXIT_GUARD:%.*]], label [[C]] +; BOOLEAN: B: +; BOOLEAN-NEXT: tail call fastcc void @check(i32 1) #[[ATTR0:[0-9]+]] +; BOOLEAN-NEXT: br label [[D:%.*]] +; BOOLEAN: C: +; BOOLEAN-NEXT: [[INC2]] = add i32 [[INC1]], 1 +; BOOLEAN-NEXT: [[CMP:%.*]] = icmp ult i32 [[INC2]], 10 +; BOOLEAN-NEXT: br i1 [[CMP]], label [[A]], label [[LOOP_EXIT_GUARD]] +; BOOLEAN: D: +; BOOLEAN-NEXT: unreachable +; BOOLEAN: E: +; BOOLEAN-NEXT: ret void +; BOOLEAN: loop.exit.guard: +; BOOLEAN-NEXT: [[GUARD_B:%.*]] = phi i1 [ true, [[A]] ], [ false, [[C]] ] +; BOOLEAN-NEXT: br i1 [[GUARD_B]], label [[B:%.*]], label [[E]] +; +entry: + br i1 %PredEntry, label %A, label %E + +A: + %inc1 = phi i32 [ 0, %entry ], [ %inc2, %C ] + br i1 %PredA, label %B, label %C + +B: + tail call fastcc void @check(i32 1) #0 + br label %D + +C: + %inc2 = add i32 %inc1, 1 + %cmp = icmp ult i32 %inc2, 10 + br i1 %cmp, label %A, label %E + +D: + unreachable + +E: + ret void +} + +; The loop exit blocks appear in an inner loop. + +define void @inner_loop(i1 %PredEntry, i1 %PredA, i1 %PredB) { +; CHECK-LABEL: @inner_loop( +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[PREDENTRY:%.*]], label [[A:%.*]], label [[I:%.*]] +; CHECK: A: +; CHECK-NEXT: [[OUTER1:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[OUTER2:%.*]], [[G:%.*]] ] +; CHECK-NEXT: br label [[B:%.*]] +; CHECK: B: +; CHECK-NEXT: [[INNER1:%.*]] = phi i32 [ 0, [[A]] ], [ [[INNER2:%.*]], [[F:%.*]] ] +; CHECK-NEXT: br i1 [[PREDA:%.*]], label [[D:%.*]], label [[LOOP_EXIT_GUARD2:%.*]] +; CHECK: C: +; CHECK-NEXT: tail call fastcc void @check(i32 1) #[[ATTR0]] +; CHECK-NEXT: br label [[H:%.*]] +; CHECK: D: +; CHECK-NEXT: br i1 [[PREDB:%.*]], label [[LOOP_EXIT_GUARD2]], label [[F]] +; CHECK: E: +; CHECK-NEXT: tail call fastcc void @check(i32 2) #[[ATTR0]] +; CHECK-NEXT: br label [[H]] +; CHECK: F: +; CHECK-NEXT: [[INNER2]] = add i32 [[INNER1]], 1 +; CHECK-NEXT: [[CMP1:%.*]] = icmp ult i32 [[INNER2]], 20 +; CHECK-NEXT: br i1 [[CMP1]], label [[B]], label [[LOOP_EXIT_GUARD2]] +; CHECK: G: +; CHECK-NEXT: [[OUTER2]] = add i32 [[OUTER1]], 1 +; CHECK-NEXT: [[CMP2:%.*]] = icmp ult i32 [[OUTER2]], 10 +; CHECK-NEXT: br i1 [[CMP2]], label [[A]], label [[LOOP_EXIT_GUARD:%.*]] +; CHECK: H: +; CHECK-NEXT: unreachable +; CHECK: I: +; CHECK-NEXT: ret void +; CHECK: loop.exit.guard: +; CHECK-NEXT: [[MERGED_BB_IDX:%.*]] = phi i32 [ 2, [[G]] ], [ [[MERGED_BB_IDX_MOVED:%.*]], [[LOOP_EXIT_GUARD2]] ] +; CHECK-NEXT: [[C_PREDICATE:%.*]] = icmp eq i32 [[MERGED_BB_IDX]], 0 +; CHECK-NEXT: br i1 [[C_PREDICATE]], label [[C:%.*]], label [[LOOP_EXIT_GUARD1:%.*]] +; CHECK: loop.exit.guard1: +; CHECK-NEXT: [[E_PREDICATE:%.*]] = icmp eq i32 [[MERGED_BB_IDX]], 1 +; CHECK-NEXT: br i1 [[E_PREDICATE]], label [[E:%.*]], label [[I]] +; CHECK: loop.exit.guard2: +; CHECK-NEXT: [[MERGED_BB_IDX_MOVED]] = phi i32 [ 0, [[B]] ], [ 1, [[D]] ], [ undef, [[F]] ] +; CHECK-NEXT: [[MERGED_BB_IDX3:%.*]] = phi i32 [ 0, [[B]] ], [ 0, [[D]] ], [ 1, [[F]] ] +; CHECK-NEXT: [[LOOP_EXIT_GUARD_PREDICATE:%.*]] = icmp eq i32 [[MERGED_BB_IDX3]], 0 +; CHECK-NEXT: br i1 [[LOOP_EXIT_GUARD_PREDICATE]], label [[LOOP_EXIT_GUARD]], label [[G]] +; +; BOOLEAN-LABEL: @inner_loop( +; BOOLEAN-NEXT: entry: +; BOOLEAN-NEXT: br i1 [[PREDENTRY:%.*]], label [[A:%.*]], label [[I:%.*]] +; BOOLEAN: A: +; BOOLEAN-NEXT: [[OUTER1:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[OUTER2:%.*]], [[G:%.*]] ] +; BOOLEAN-NEXT: br label [[B:%.*]] +; BOOLEAN: B: +; BOOLEAN-NEXT: [[INNER1:%.*]] = phi i32 [ 0, [[A]] ], [ [[INNER2:%.*]], [[F:%.*]] ] +; BOOLEAN-NEXT: br i1 [[PREDA:%.*]], label [[D:%.*]], label [[LOOP_EXIT_GUARD2:%.*]] +; BOOLEAN: C: +; BOOLEAN-NEXT: tail call fastcc void @check(i32 1) #[[ATTR0]] +; BOOLEAN-NEXT: br label [[H:%.*]] +; BOOLEAN: D: +; BOOLEAN-NEXT: br i1 [[PREDB:%.*]], label [[LOOP_EXIT_GUARD2]], label [[F]] +; BOOLEAN: E: +; BOOLEAN-NEXT: tail call fastcc void @check(i32 2) #[[ATTR0]] +; BOOLEAN-NEXT: br label [[H]] +; BOOLEAN: F: +; BOOLEAN-NEXT: [[INNER2]] = add i32 [[INNER1]], 1 +; BOOLEAN-NEXT: [[CMP1:%.*]] = icmp ult i32 [[INNER2]], 20 +; BOOLEAN-NEXT: br i1 [[CMP1]], label [[B]], label [[LOOP_EXIT_GUARD2]] +; BOOLEAN: G: +; BOOLEAN-NEXT: [[OUTER2]] = add i32 [[OUTER1]], 1 +; BOOLEAN-NEXT: [[CMP2:%.*]] = icmp ult i32 [[OUTER2]], 10 +; BOOLEAN-NEXT: br i1 [[CMP2]], label [[A]], label [[LOOP_EXIT_GUARD:%.*]] +; BOOLEAN: H: +; BOOLEAN-NEXT: unreachable +; BOOLEAN: I: +; BOOLEAN-NEXT: ret void +; BOOLEAN: loop.exit.guard: +; BOOLEAN-NEXT: [[GUARD_C:%.*]] = phi i1 [ false, [[G]] ], [ [[GUARD_C_MOVED:%.*]], [[LOOP_EXIT_GUARD2]] ] +; BOOLEAN-NEXT: [[GUARD_E:%.*]] = phi i1 [ false, [[G]] ], [ [[GUARD_E_MOVED:%.*]], [[LOOP_EXIT_GUARD2]] ] +; BOOLEAN-NEXT: br i1 [[GUARD_C]], label [[C:%.*]], label [[LOOP_EXIT_GUARD1:%.*]] +; BOOLEAN: loop.exit.guard1: +; BOOLEAN-NEXT: br i1 [[GUARD_E]], label [[E:%.*]], label [[I]] +; BOOLEAN: loop.exit.guard2: +; BOOLEAN-NEXT: [[GUARD_E_MOVED]] = phi i1 [ false, [[B]] ], [ true, [[D]] ], [ undef, [[F]] ] +; BOOLEAN-NEXT: [[GUARD_C_MOVED]] = phi i1 [ true, [[B]] ], [ false, [[D]] ], [ undef, [[F]] ] +; BOOLEAN-NEXT: [[GUARD_LOOP_EXIT_GUARD:%.*]] = phi i1 [ true, [[B]] ], [ true, [[D]] ], [ false, [[F]] ] +; BOOLEAN-NEXT: br i1 [[GUARD_LOOP_EXIT_GUARD]], label [[LOOP_EXIT_GUARD]], label [[G]] +; +entry: + br i1 %PredEntry, label %A, label %I + +A: + %outer1 = phi i32 [ 0, %entry ], [ %outer2, %G ] + br label %B + +B: + %inner1 = phi i32 [ 0, %A ], [ %inner2, %F ] + br i1 %PredA, label %D, label %C + +C: + tail call fastcc void @check(i32 1) #0 + br label %H + +D: + br i1 %PredB, label %E, label %F + +E: + tail call fastcc void @check(i32 2) #0 + br label %H + +F: + %inner2 = add i32 %inner1, 1 + %cmp1 = icmp ult i32 %inner2, 20 + br i1 %cmp1, label %B, label %G + +G: + %outer2 = add i32 %outer1, 1 + %cmp2 = icmp ult i32 %outer2, 10 + br i1 %cmp2, label %A, label %I + +H: + unreachable + +I: + ret void +} + +; A loop with more exit blocks. + +define void @loop_five_exits(i1 %PredEntry, i1 %PredA, i1 %PredB, i1 %PredC, i1 %PredD) { +; CHECK-LABEL: @loop_five_exits( +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[PREDENTRY:%.*]], label [[A:%.*]], label [[L:%.*]] +; CHECK: A: +; CHECK-NEXT: [[INC1:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC2:%.*]], [[I:%.*]] ] +; CHECK-NEXT: br i1 [[PREDA:%.*]], label [[LOOP_EXIT_GUARD:%.*]], label [[C:%.*]] +; CHECK: B: +; CHECK-NEXT: tail call fastcc void @check(i32 1) #[[ATTR0]] +; CHECK-NEXT: br label [[J:%.*]] +; CHECK: C: +; CHECK-NEXT: br i1 [[PREDB:%.*]], label [[LOOP_EXIT_GUARD]], label [[E:%.*]] +; CHECK: D: +; CHECK-NEXT: tail call fastcc void @check(i32 2) #[[ATTR0]] +; CHECK-NEXT: br label [[J]] +; CHECK: E: +; CHECK-NEXT: br i1 [[PREDC:%.*]], label [[LOOP_EXIT_GUARD]], label [[G:%.*]] +; CHECK: F: +; CHECK-NEXT: tail call fastcc void @check(i32 3) #[[ATTR0]] +; CHECK-NEXT: br label [[K:%.*]] +; CHECK: G: +; CHECK-NEXT: br i1 [[PREDD:%.*]], label [[LOOP_EXIT_GUARD]], label [[I]] +; CHECK: H: +; CHECK-NEXT: tail call fastcc void @check(i32 4) #[[ATTR0]] +; CHECK-NEXT: br label [[K]] +; CHECK: I: +; CHECK-NEXT: [[INC2]] = add i32 [[INC1]], 1 +; CHECK-NEXT: [[CMP:%.*]] = icmp ult i32 [[INC2]], 10 +; CHECK-NEXT: br i1 [[CMP]], label [[A]], label [[LOOP_EXIT_GUARD]] +; CHECK: J: +; CHECK-NEXT: br label [[L]] +; CHECK: K: +; CHECK-NEXT: br label [[L]] +; CHECK: L: +; CHECK-NEXT: ret void +; CHECK: loop.exit.guard: +; CHECK-NEXT: [[MERGED_BB_IDX:%.*]] = phi i32 [ 0, [[A]] ], [ 1, [[C]] ], [ 2, [[E]] ], [ 3, [[G]] ], [ 4, [[I]] ] +; CHECK-NEXT: [[B_PREDICATE:%.*]] = icmp eq i32 [[MERGED_BB_IDX]], 0 +; CHECK-NEXT: br i1 [[B_PREDICATE]], label [[B:%.*]], label [[LOOP_EXIT_GUARD1:%.*]] +; CHECK: loop.exit.guard1: +; CHECK-NEXT: [[D_PREDICATE:%.*]] = icmp eq i32 [[MERGED_BB_IDX]], 1 +; CHECK-NEXT: br i1 [[D_PREDICATE]], label [[D:%.*]], label [[LOOP_EXIT_GUARD2:%.*]] +; CHECK: loop.exit.guard2: +; CHECK-NEXT: [[F_PREDICATE:%.*]] = icmp eq i32 [[MERGED_BB_IDX]], 2 +; CHECK-NEXT: br i1 [[F_PREDICATE]], label [[F:%.*]], label [[LOOP_EXIT_GUARD3:%.*]] +; CHECK: loop.exit.guard3: +; CHECK-NEXT: [[H_PREDICATE:%.*]] = icmp eq i32 [[MERGED_BB_IDX]], 3 +; CHECK-NEXT: br i1 [[H_PREDICATE]], label [[H:%.*]], label [[L]] +; +; BOOLEAN-LABEL: @loop_five_exits( +; BOOLEAN-NEXT: entry: +; BOOLEAN-NEXT: br i1 [[PREDENTRY:%.*]], label [[A:%.*]], label [[L:%.*]] +; BOOLEAN: A: +; BOOLEAN-NEXT: [[INC1:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC2:%.*]], [[I:%.*]] ] +; BOOLEAN-NEXT: br i1 [[PREDA:%.*]], label [[LOOP_EXIT_GUARD:%.*]], label [[C:%.*]] +; BOOLEAN: B: +; BOOLEAN-NEXT: tail call fastcc void @check(i32 1) #[[ATTR0]] +; BOOLEAN-NEXT: br label [[J:%.*]] +; BOOLEAN: C: +; BOOLEAN-NEXT: br i1 [[PREDB:%.*]], label [[LOOP_EXIT_GUARD]], label [[E:%.*]] +; BOOLEAN: D: +; BOOLEAN-NEXT: tail call fastcc void @check(i32 2) #[[ATTR0]] +; BOOLEAN-NEXT: br label [[J]] +; BOOLEAN: E: +; BOOLEAN-NEXT: br i1 [[PREDC:%.*]], label [[LOOP_EXIT_GUARD]], label [[G:%.*]] +; BOOLEAN: F: +; BOOLEAN-NEXT: tail call fastcc void @check(i32 3) #[[ATTR0]] +; BOOLEAN-NEXT: br label [[K:%.*]] +; BOOLEAN: G: +; BOOLEAN-NEXT: br i1 [[PREDD:%.*]], label [[LOOP_EXIT_GUARD]], label [[I]] +; BOOLEAN: H: +; BOOLEAN-NEXT: tail call fastcc void @check(i32 4) #[[ATTR0]] +; BOOLEAN-NEXT: br label [[K]] +; BOOLEAN: I: +; BOOLEAN-NEXT: [[INC2]] = add i32 [[INC1]], 1 +; BOOLEAN-NEXT: [[CMP:%.*]] = icmp ult i32 [[INC2]], 10 +; BOOLEAN-NEXT: br i1 [[CMP]], label [[A]], label [[LOOP_EXIT_GUARD]] +; BOOLEAN: J: +; BOOLEAN-NEXT: br label [[L]] +; BOOLEAN: K: +; BOOLEAN-NEXT: br label [[L]] +; BOOLEAN: L: +; BOOLEAN-NEXT: ret void +; BOOLEAN: loop.exit.guard: +; BOOLEAN-NEXT: [[GUARD_B:%.*]] = phi i1 [ true, [[A]] ], [ false, [[C]] ], [ false, [[E]] ], [ false, [[G]] ], [ false, [[I]] ] +; BOOLEAN-NEXT: [[GUARD_D:%.*]] = phi i1 [ false, [[A]] ], [ true, [[C]] ], [ false, [[E]] ], [ false, [[G]] ], [ false, [[I]] ] +; BOOLEAN-NEXT: [[GUARD_F:%.*]] = phi i1 [ false, [[A]] ], [ false, [[C]] ], [ true, [[E]] ], [ false, [[G]] ], [ false, [[I]] ] +; BOOLEAN-NEXT: [[GUARD_H:%.*]] = phi i1 [ false, [[A]] ], [ false, [[C]] ], [ false, [[E]] ], [ true, [[G]] ], [ false, [[I]] ] +; BOOLEAN-NEXT: br i1 [[GUARD_B]], label [[B:%.*]], label [[LOOP_EXIT_GUARD1:%.*]] +; BOOLEAN: loop.exit.guard1: +; BOOLEAN-NEXT: br i1 [[GUARD_D]], label [[D:%.*]], label [[LOOP_EXIT_GUARD2:%.*]] +; BOOLEAN: loop.exit.guard2: +; BOOLEAN-NEXT: br i1 [[GUARD_F]], label [[F:%.*]], label [[LOOP_EXIT_GUARD3:%.*]] +; BOOLEAN: loop.exit.guard3: +; BOOLEAN-NEXT: br i1 [[GUARD_H]], label [[H:%.*]], label [[L]] +; +entry: + br i1 %PredEntry, label %A, label %L + +A: + %inc1 = phi i32 [ 0, %entry ], [ %inc2, %I ] + br i1 %PredA, label %B, label %C + +B: + tail call fastcc void @check(i32 1) #0 + br label %J + +C: + br i1 %PredB, label %D, label %E + +D: + tail call fastcc void @check(i32 2) #0 + br label %J + +E: + br i1 %PredC, label %F, label %G + +F: + tail call fastcc void @check(i32 3) #0 + br label %K + +G: + br i1 %PredD, label %H, label %I + +H: + tail call fastcc void @check(i32 4) #0 + br label %K + +I: + %inc2 = add i32 %inc1, 1 + %cmp = icmp ult i32 %inc2, 10 + br i1 %cmp, label %A, label %L + +J: + br label %L + +K: + br label %L + +L: + ret void +} + + +declare void @check(i32 noundef %i) #0 + +attributes #0 = { noreturn nounwind } + diff --git a/llvm/test/Transforms/UnifyLoopExits/nested.ll b/llvm/test/Transforms/UnifyLoopExits/nested.ll index 729e51db573034..9973aa305cdf1d 100644 --- a/llvm/test/Transforms/UnifyLoopExits/nested.ll +++ b/llvm/test/Transforms/UnifyLoopExits/nested.ll @@ -31,12 +31,12 @@ define void @nested(i1 %PredB3, i1 %PredB4, i1 %PredA4, i1 %PredA3, i32 %X, i32 ; CHECK-NEXT: [[EXIT_PHI:%.*]] = phi i32 [ [[Z:%.*]], [[C:%.*]] ], [ [[EXIT_PHI_MOVED:%.*]], [[LOOP_EXIT_GUARD]] ] ; CHECK-NEXT: ret void ; CHECK: loop.exit.guard: -; CHECK-NEXT: [[GUARD_C:%.*]] = phi i1 [ true, [[A4]] ], [ false, [[A5]] ] ; CHECK-NEXT: [[EXIT_PHI_MOVED]] = phi i32 [ undef, [[A4]] ], [ [[A4_PHI]], [[A5]] ] +; CHECK-NEXT: [[GUARD_C:%.*]] = phi i1 [ true, [[A4]] ], [ false, [[A5]] ] ; CHECK-NEXT: br i1 [[GUARD_C]], label [[C]], label [[EXIT]] ; CHECK: loop.exit.guard1: -; CHECK-NEXT: [[GUARD_A3:%.*]] = phi i1 [ true, [[B3]] ], [ false, [[B4]] ] ; CHECK-NEXT: [[X_INC_MOVED]] = phi i32 [ [[X_INC]], [[B3]] ], [ [[X_INC]], [[B4]] ] +; CHECK-NEXT: [[GUARD_A3:%.*]] = phi i1 [ true, [[B3]] ], [ false, [[B4]] ] ; CHECK-NEXT: br i1 [[GUARD_A3]], label [[A3]], label [[A2]] ; entry: diff --git a/llvm/test/Transforms/UnifyLoopExits/restore-ssa.ll b/llvm/test/Transforms/UnifyLoopExits/restore-ssa.ll index 5209979aa725b1..eb42858cbbf332 100644 --- a/llvm/test/Transforms/UnifyLoopExits/restore-ssa.ll +++ b/llvm/test/Transforms/UnifyLoopExits/restore-ssa.ll @@ -30,9 +30,9 @@ define i32 @exiting-used-in-exit(i32* %arg1, i32* %arg2) local_unnamed_addr alig ; CHECK-NEXT: [[PHI:%.*]] = phi i32 [ [[INC]], [[C:%.*]] ], [ [[PHI_MOVED:%.*]], [[LOOP_EXIT_GUARD]] ] ; CHECK-NEXT: ret i32 [[PHI]] ; CHECK: loop.exit.guard: -; CHECK-NEXT: [[GUARD_RETURN:%.*]] = phi i1 [ true, [[A]] ], [ false, [[B]] ] -; CHECK-NEXT: [[PHI_MOVED]] = phi i32 [ [[MYTMP42]], [[A]] ], [ undef, [[B]] ] ; CHECK-NEXT: [[MYTMP41_MOVED]] = phi i32 [ undef, [[A]] ], [ [[MYTMP41]], [[B]] ] +; CHECK-NEXT: [[PHI_MOVED]] = phi i32 [ [[MYTMP42]], [[A]] ], [ undef, [[B]] ] +; CHECK-NEXT: [[GUARD_RETURN:%.*]] = phi i1 [ true, [[A]] ], [ false, [[B]] ] ; CHECK-NEXT: br i1 [[GUARD_RETURN]], label [[RETURN]], label [[C]] ; entry: @@ -84,8 +84,8 @@ define i32 @internal-used-in-exit(i32* %arg1, i32* %arg2) local_unnamed_addr ali ; CHECK: return: ; CHECK-NEXT: ret i32 0 ; CHECK: loop.exit.guard: -; CHECK-NEXT: [[GUARD_RETURN:%.*]] = phi i1 [ true, [[A]] ], [ false, [[C]] ] ; CHECK-NEXT: [[MYTMP41_MOVED]] = phi i32 [ undef, [[A]] ], [ [[MYTMP41]], [[C]] ] +; CHECK-NEXT: [[GUARD_RETURN:%.*]] = phi i1 [ true, [[A]] ], [ false, [[C]] ] ; CHECK-NEXT: br i1 [[GUARD_RETURN]], label [[RETURN]], label [[D:%.*]] ; entry: @@ -141,9 +141,9 @@ define i32 @mixed-use-in-exit(i32* %arg1, i32* %arg2) local_unnamed_addr align 2 ; CHECK-NEXT: [[PHI:%.*]] = phi i32 [ [[MYTMP41_MOVED:%.*]], [[D:%.*]] ], [ [[MYTMP42]], [[ENTRY:%.*]] ], [ [[PHI_MOVED:%.*]], [[LOOP_EXIT_GUARD]] ] ; CHECK-NEXT: ret i32 [[PHI]] ; CHECK: loop.exit.guard: -; CHECK-NEXT: [[GUARD_RETURN:%.*]] = phi i1 [ true, [[A]] ], [ false, [[C]] ] -; CHECK-NEXT: [[PHI_MOVED]] = phi i32 [ [[MYTMP43]], [[A]] ], [ undef, [[C]] ] ; CHECK-NEXT: [[MYTMP41_MOVED]] = phi i32 [ undef, [[A]] ], [ [[MYTMP41]], [[C]] ] +; CHECK-NEXT: [[PHI_MOVED]] = phi i32 [ [[MYTMP43]], [[A]] ], [ undef, [[C]] ] +; CHECK-NEXT: [[GUARD_RETURN:%.*]] = phi i1 [ true, [[A]] ], [ false, [[C]] ] ; CHECK-NEXT: br i1 [[GUARD_RETURN]], label [[RETURN]], label [[D]] ; entry: @@ -206,8 +206,8 @@ define i32 @phi-via-external-block(i32* %arg1, i32* %arg2) local_unnamed_addr al ; CHECK-NEXT: [[PHI:%.*]] = phi i32 [ [[MYTMP41_MOVED:%.*]], [[D:%.*]] ], [ [[MYTMP42]], [[E:%.*]] ] ; CHECK-NEXT: ret i32 [[PHI]] ; CHECK: loop.exit.guard: -; CHECK-NEXT: [[GUARD_E:%.*]] = phi i1 [ true, [[A]] ], [ false, [[C]] ] ; CHECK-NEXT: [[MYTMP41_MOVED]] = phi i32 [ undef, [[A]] ], [ [[MYTMP41]], [[C]] ] +; CHECK-NEXT: [[GUARD_E:%.*]] = phi i1 [ true, [[A]] ], [ false, [[C]] ] ; CHECK-NEXT: br i1 [[GUARD_E]], label [[E]], label [[D]] ; entry: