diff --git a/llvm/lib/Transforms/Scalar/LoopDeletion.cpp b/llvm/lib/Transforms/Scalar/LoopDeletion.cpp index fd0e983b6199ac..2a080bfed67feb 100644 --- a/llvm/lib/Transforms/Scalar/LoopDeletion.cpp +++ b/llvm/lib/Transforms/Scalar/LoopDeletion.cpp @@ -17,6 +17,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" #include "llvm/Analysis/GlobalsModRef.h" +#include "llvm/Analysis/LoopIterator.h" #include "llvm/Analysis/LoopPass.h" #include "llvm/Analysis/MemorySSA.h" #include "llvm/Analysis/OptimizationRemarkEmitter.h" @@ -135,6 +136,155 @@ static bool isLoopNeverExecuted(Loop *L) { return true; } +static const SCEV * +getSCEVOnFirstIteration(Value *V, Loop *L, ScalarEvolution &SE, + DenseMap &FirstIterSCEV) { + // Fist, check in cache. + auto Existing = FirstIterSCEV.find(V); + if (Existing != FirstIterSCEV.end()) + return Existing->second; + const SCEV *S = nullptr; + // TODO: Once ScalarEvolution supports getValueOnNthIteration for anything + // else but AddRecs, it's a good use case for it. So far, just consider some + // simple cases, like arithmetic operations. + Value *LHS, *RHS; + using namespace PatternMatch; + if (match(V, m_Add(m_Value(LHS), m_Value(RHS)))) { + const SCEV *LHSS = getSCEVOnFirstIteration(LHS, L, SE, FirstIterSCEV); + const SCEV *RHSS = getSCEVOnFirstIteration(RHS, L, SE, FirstIterSCEV); + S = SE.getAddExpr(LHSS, RHSS); + } else if (match(V, m_Sub(m_Value(LHS), m_Value(RHS)))) { + const SCEV *LHSS = getSCEVOnFirstIteration(LHS, L, SE, FirstIterSCEV); + const SCEV *RHSS = getSCEVOnFirstIteration(RHS, L, SE, FirstIterSCEV); + S = SE.getMinusSCEV(LHSS, RHSS); + } else if (match(V, m_Mul(m_Value(LHS), m_Value(RHS)))) { + const SCEV *LHSS = getSCEVOnFirstIteration(LHS, L, SE, FirstIterSCEV); + const SCEV *RHSS = getSCEVOnFirstIteration(RHS, L, SE, FirstIterSCEV); + S = SE.getMulExpr(LHSS, RHSS); + } else + S = SE.getSCEV(V); + assert(S && "Case not handled?"); + FirstIterSCEV[V] = S; + return S; +} + +// Try to prove that one of conditions that dominates the latch must exit on 1st +// iteration. +static bool canProveExitOnFirstIteration(Loop *L, DominatorTree &DT, + ScalarEvolution &SE, LoopInfo &LI) { + BasicBlock *Latch = L->getLoopLatch(); + + if (!Latch) + return false; + + LoopBlocksRPO RPOT(L); + RPOT.perform(&LI); + + BasicBlock *Header = L->getHeader(); + // Blocks that are reachable on the 1st iteration. + SmallPtrSet LiveBlocks; + // Edges that are reachable on the 1st iteration. + DenseSet LiveEdges; + LiveBlocks.insert(L->getHeader()); + + auto MarkLiveEdge = [&](BasicBlock *From, BasicBlock *To) { + assert(LiveBlocks.count(From) && "Must be live!"); + LiveBlocks.insert(To); + LiveEdges.insert({ From, To }); + }; + + auto MarkAllSuccessorsLive = [&](BasicBlock *BB) { + for (auto *Succ : successors(BB)) + MarkLiveEdge(BB, Succ); + }; + + // Check if there is only one predecessor on 1st iteration. Note that because + // we iterate in RPOT, we have already visited all its (non-latch) + // predecessors. + auto GetSolePredecessorOnFirstIteration = [&](BasicBlock * BB)->BasicBlock * { + if (BB == Header) + return L->getLoopPredecessor(); + BasicBlock *OnlyPred = nullptr; + for (auto *Pred : predecessors(BB)) + if (OnlyPred != Pred && LiveEdges.count({ Pred, BB })) { + // 2 live preds. + if (OnlyPred) + return nullptr; + OnlyPred = Pred; + } + + assert(OnlyPred && "No live predecessors?"); + return OnlyPred; + }; + DenseMap FirstIterSCEV; + + // Use the following algorithm to prove we never take the latch on the 1st + // iteration: + // 1. Traverse in topological order, so that whenever we visit a block, all + // its predecessors are already visited. + // 2. If we can prove that the block may have only 1 predecessor on the 1st + // iteration, map all its phis onto input from this predecessor. + // 3a. If we can prove which successor of out block is taken on the 1st + // iteration, mark this successor live. + // 3b. If we cannot prove it, conservatively assume that all successors are + // live. + for (auto *BB : RPOT) { + // This block is not reachable on the 1st iterations. + if (!LiveBlocks.count(BB)) + continue; + + // Skip inner loops. + if (LI.getLoopFor(BB) != L) { + MarkAllSuccessorsLive(BB); + continue; + } + + // If this block has only one live pred, map its phis onto their SCEVs. + if (auto *OnlyPred = GetSolePredecessorOnFirstIteration(BB)) + for (auto &PN : BB->phis()) { + if (!SE.isSCEVable(PN.getType())) + continue; + auto *Incoming = PN.getIncomingValueForBlock(OnlyPred); + if (DT.dominates(Incoming, BB->getTerminator())) { + const SCEV *IncSCEV = + getSCEVOnFirstIteration(Incoming, L, SE, FirstIterSCEV); + FirstIterSCEV[&PN] = IncSCEV; + } + } + + using namespace PatternMatch; + ICmpInst::Predicate Pred; + Value *LHS, *RHS; + const BasicBlock *IfTrue, *IfFalse; + // TODO: Handle switches. + if (!match(BB->getTerminator(), + m_Br(m_ICmp(Pred, m_Value(LHS), m_Value(RHS)), + m_BasicBlock(IfTrue), m_BasicBlock(IfFalse)))) { + MarkAllSuccessorsLive(BB); + continue; + } + + if (!SE.isSCEVable(LHS->getType())) { + MarkAllSuccessorsLive(BB); + continue; + } + + // Can we prove constant true or false for this condition? + const SCEV *LHSS = getSCEVOnFirstIteration(LHS, L, SE, FirstIterSCEV); + const SCEV *RHSS = getSCEVOnFirstIteration(RHS, L, SE, FirstIterSCEV); + if (SE.isKnownPredicate(Pred, LHSS, RHSS)) + MarkLiveEdge(BB, BB->getTerminator()->getSuccessor(0)); + else if (SE.isKnownPredicate(ICmpInst::getInversePredicate(Pred), LHSS, + RHSS)) + MarkLiveEdge(BB, BB->getTerminator()->getSuccessor(1)); + else + MarkAllSuccessorsLive(BB); + } + + // We can break the latch if it wasn't live. + return !LiveEdges.count({ Latch, Header }); +} + /// If we can prove the backedge is untaken, remove it. This destroys the /// loop, but leaves the (now trivially loop invariant) control flow and /// side effects (if any) in place. @@ -148,7 +298,7 @@ breakBackedgeIfNotTaken(Loop *L, DominatorTree &DT, ScalarEvolution &SE, return LoopDeletionResult::Unmodified; auto *BTC = SE.getBackedgeTakenCount(L); - if (!BTC->isZero()) + if (!BTC->isZero() && !canProveExitOnFirstIteration(L, DT, SE, LI)) return LoopDeletionResult::Unmodified; breakLoopBackedge(L, DT, SE, LI, MSSA); diff --git a/llvm/test/Transforms/LoopDeletion/eval_first_iteration.ll b/llvm/test/Transforms/LoopDeletion/eval_first_iteration.ll index 98dafebe0df022..aa3bfb062ccfd7 100644 --- a/llvm/test/Transforms/LoopDeletion/eval_first_iteration.ll +++ b/llvm/test/Transforms/LoopDeletion/eval_first_iteration.ll @@ -7,7 +7,6 @@ ; and therefore prove that %sum.next = %sum + %sub = %sum + %limit - %sum = %limit, ; and predicate is false. -; TODO: We can break the backedge here. define i32 @test_ne(i32 %limit) { ; CHECK-LABEL: @test_ne( ; CHECK-NEXT: entry: @@ -60,7 +59,6 @@ failure: unreachable } -; TODO: We can break the backedge here. define i32 @test_slt(i32 %limit) { ; CHECK-LABEL: @test_slt( ; CHECK-NEXT: entry: @@ -113,7 +111,6 @@ failure: unreachable } -; TODO: We can break the backedge here. define i32 @test_ult(i32 %limit) { ; CHECK-LABEL: @test_ult( ; CHECK-NEXT: entry: @@ -166,7 +163,6 @@ failure: unreachable } -; TODO: We can break the backedge here. define i32 @test_sgt(i32 %limit) { ; CHECK-LABEL: @test_sgt( ; CHECK-NEXT: entry: @@ -219,7 +215,6 @@ failure: unreachable } -; TODO: We can break the backedge here. define i32 @test_ugt(i32 %limit) { ; CHECK-LABEL: @test_ugt( ; CHECK-NEXT: entry: @@ -272,7 +267,6 @@ failure: unreachable } -; TODO: We can break the backedge here. define i32 @test_multiple_pred(i32 %limit) { ; CHECK-LABEL: @test_multiple_pred( ; CHECK-NEXT: entry: @@ -344,17 +338,19 @@ define i32 @test_ne_const() { ; CHECK-NEXT: entry: ; CHECK-NEXT: br label [[LOOP:%.*]] ; CHECK: loop: -; CHECK-NEXT: [[SUM:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[SUM_NEXT:%.*]], [[BACKEDGE:%.*]] ] +; CHECK-NEXT: [[SUM:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ] ; CHECK-NEXT: [[SUB:%.*]] = sub i32 4, [[SUM]] ; CHECK-NEXT: [[IS_POSITIVE:%.*]] = icmp sgt i32 [[SUB]], 0 -; CHECK-NEXT: br i1 [[IS_POSITIVE]], label [[BACKEDGE]], label [[IF_FALSE:%.*]] +; CHECK-NEXT: br i1 [[IS_POSITIVE]], label [[BACKEDGE:%.*]], label [[IF_FALSE:%.*]] ; CHECK: if.false: ; CHECK-NEXT: br label [[BACKEDGE]] ; CHECK: backedge: ; CHECK-NEXT: [[MERGE_PHI:%.*]] = phi i32 [ 0, [[IF_FALSE]] ], [ [[SUB]], [[LOOP]] ] -; CHECK-NEXT: [[SUM_NEXT]] = add i32 [[SUM]], [[MERGE_PHI]] +; CHECK-NEXT: [[SUM_NEXT:%.*]] = add i32 [[SUM]], [[MERGE_PHI]] ; CHECK-NEXT: [[LOOP_COND:%.*]] = icmp ne i32 [[SUM_NEXT]], 4 -; CHECK-NEXT: br i1 [[LOOP_COND]], label [[LOOP]], label [[DONE:%.*]] +; CHECK-NEXT: br i1 [[LOOP_COND]], label [[BACKEDGE_LOOP_CRIT_EDGE:%.*]], label [[DONE:%.*]] +; CHECK: backedge.loop_crit_edge: +; CHECK-NEXT: unreachable ; CHECK: done: ; CHECK-NEXT: [[SUM_NEXT_LCSSA:%.*]] = phi i32 [ [[SUM_NEXT]], [[BACKEDGE]] ] ; CHECK-NEXT: ret i32 [[SUM_NEXT_LCSSA]] @@ -394,17 +390,19 @@ define i32 @test_slt_const() { ; CHECK-NEXT: entry: ; CHECK-NEXT: br label [[LOOP:%.*]] ; CHECK: loop: -; CHECK-NEXT: [[SUM:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[SUM_NEXT:%.*]], [[BACKEDGE:%.*]] ] +; CHECK-NEXT: [[SUM:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ] ; CHECK-NEXT: [[SUB:%.*]] = sub i32 4, [[SUM]] ; CHECK-NEXT: [[IS_POSITIVE:%.*]] = icmp sgt i32 [[SUB]], 0 -; CHECK-NEXT: br i1 [[IS_POSITIVE]], label [[BACKEDGE]], label [[IF_FALSE:%.*]] +; CHECK-NEXT: br i1 [[IS_POSITIVE]], label [[BACKEDGE:%.*]], label [[IF_FALSE:%.*]] ; CHECK: if.false: ; CHECK-NEXT: br label [[BACKEDGE]] ; CHECK: backedge: ; CHECK-NEXT: [[MERGE_PHI:%.*]] = phi i32 [ 0, [[IF_FALSE]] ], [ [[SUB]], [[LOOP]] ] -; CHECK-NEXT: [[SUM_NEXT]] = add i32 [[SUM]], [[MERGE_PHI]] +; CHECK-NEXT: [[SUM_NEXT:%.*]] = add i32 [[SUM]], [[MERGE_PHI]] ; CHECK-NEXT: [[LOOP_COND:%.*]] = icmp slt i32 [[SUM_NEXT]], 4 -; CHECK-NEXT: br i1 [[LOOP_COND]], label [[LOOP]], label [[DONE:%.*]] +; CHECK-NEXT: br i1 [[LOOP_COND]], label [[BACKEDGE_LOOP_CRIT_EDGE:%.*]], label [[DONE:%.*]] +; CHECK: backedge.loop_crit_edge: +; CHECK-NEXT: unreachable ; CHECK: done: ; CHECK-NEXT: [[SUM_NEXT_LCSSA:%.*]] = phi i32 [ [[SUM_NEXT]], [[BACKEDGE]] ] ; CHECK-NEXT: ret i32 [[SUM_NEXT_LCSSA]] @@ -444,17 +442,19 @@ define i32 @test_ult_const() { ; CHECK-NEXT: entry: ; CHECK-NEXT: br label [[LOOP:%.*]] ; CHECK: loop: -; CHECK-NEXT: [[SUM:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[SUM_NEXT:%.*]], [[BACKEDGE:%.*]] ] +; CHECK-NEXT: [[SUM:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ] ; CHECK-NEXT: [[SUB:%.*]] = sub i32 4, [[SUM]] ; CHECK-NEXT: [[IS_POSITIVE:%.*]] = icmp sgt i32 [[SUB]], 0 -; CHECK-NEXT: br i1 [[IS_POSITIVE]], label [[BACKEDGE]], label [[IF_FALSE:%.*]] +; CHECK-NEXT: br i1 [[IS_POSITIVE]], label [[BACKEDGE:%.*]], label [[IF_FALSE:%.*]] ; CHECK: if.false: ; CHECK-NEXT: br label [[BACKEDGE]] ; CHECK: backedge: ; CHECK-NEXT: [[MERGE_PHI:%.*]] = phi i32 [ 0, [[IF_FALSE]] ], [ [[SUB]], [[LOOP]] ] -; CHECK-NEXT: [[SUM_NEXT]] = add i32 [[SUM]], [[MERGE_PHI]] +; CHECK-NEXT: [[SUM_NEXT:%.*]] = add i32 [[SUM]], [[MERGE_PHI]] ; CHECK-NEXT: [[LOOP_COND:%.*]] = icmp ult i32 [[SUM_NEXT]], 4 -; CHECK-NEXT: br i1 [[LOOP_COND]], label [[LOOP]], label [[DONE:%.*]] +; CHECK-NEXT: br i1 [[LOOP_COND]], label [[BACKEDGE_LOOP_CRIT_EDGE:%.*]], label [[DONE:%.*]] +; CHECK: backedge.loop_crit_edge: +; CHECK-NEXT: unreachable ; CHECK: done: ; CHECK-NEXT: [[SUM_NEXT_LCSSA:%.*]] = phi i32 [ [[SUM_NEXT]], [[BACKEDGE]] ] ; CHECK-NEXT: ret i32 [[SUM_NEXT_LCSSA]] @@ -494,17 +494,19 @@ define i32 @test_sgt_const() { ; CHECK-NEXT: entry: ; CHECK-NEXT: br label [[LOOP:%.*]] ; CHECK: loop: -; CHECK-NEXT: [[SUM:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[SUM_NEXT:%.*]], [[BACKEDGE:%.*]] ] +; CHECK-NEXT: [[SUM:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ] ; CHECK-NEXT: [[SUB:%.*]] = sub i32 4, [[SUM]] ; CHECK-NEXT: [[IS_POSITIVE:%.*]] = icmp sgt i32 [[SUB]], 0 -; CHECK-NEXT: br i1 [[IS_POSITIVE]], label [[BACKEDGE]], label [[IF_FALSE:%.*]] +; CHECK-NEXT: br i1 [[IS_POSITIVE]], label [[BACKEDGE:%.*]], label [[IF_FALSE:%.*]] ; CHECK: if.false: ; CHECK-NEXT: br label [[BACKEDGE]] ; CHECK: backedge: ; CHECK-NEXT: [[MERGE_PHI:%.*]] = phi i32 [ 0, [[IF_FALSE]] ], [ [[SUB]], [[LOOP]] ] -; CHECK-NEXT: [[SUM_NEXT]] = add i32 [[SUM]], [[MERGE_PHI]] +; CHECK-NEXT: [[SUM_NEXT:%.*]] = add i32 [[SUM]], [[MERGE_PHI]] ; CHECK-NEXT: [[LOOP_COND:%.*]] = icmp sgt i32 [[SUM_NEXT]], 4 -; CHECK-NEXT: br i1 [[LOOP_COND]], label [[LOOP]], label [[DONE:%.*]] +; CHECK-NEXT: br i1 [[LOOP_COND]], label [[BACKEDGE_LOOP_CRIT_EDGE:%.*]], label [[DONE:%.*]] +; CHECK: backedge.loop_crit_edge: +; CHECK-NEXT: unreachable ; CHECK: done: ; CHECK-NEXT: [[SUM_NEXT_LCSSA:%.*]] = phi i32 [ [[SUM_NEXT]], [[BACKEDGE]] ] ; CHECK-NEXT: ret i32 [[SUM_NEXT_LCSSA]] @@ -544,17 +546,19 @@ define i32 @test_ugt_const() { ; CHECK-NEXT: entry: ; CHECK-NEXT: br label [[LOOP:%.*]] ; CHECK: loop: -; CHECK-NEXT: [[SUM:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[SUM_NEXT:%.*]], [[BACKEDGE:%.*]] ] +; CHECK-NEXT: [[SUM:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ] ; CHECK-NEXT: [[SUB:%.*]] = sub i32 4, [[SUM]] ; CHECK-NEXT: [[IS_POSITIVE:%.*]] = icmp sgt i32 [[SUB]], 0 -; CHECK-NEXT: br i1 [[IS_POSITIVE]], label [[BACKEDGE]], label [[IF_FALSE:%.*]] +; CHECK-NEXT: br i1 [[IS_POSITIVE]], label [[BACKEDGE:%.*]], label [[IF_FALSE:%.*]] ; CHECK: if.false: ; CHECK-NEXT: br label [[BACKEDGE]] ; CHECK: backedge: ; CHECK-NEXT: [[MERGE_PHI:%.*]] = phi i32 [ 0, [[IF_FALSE]] ], [ [[SUB]], [[LOOP]] ] -; CHECK-NEXT: [[SUM_NEXT]] = add i32 [[SUM]], [[MERGE_PHI]] +; CHECK-NEXT: [[SUM_NEXT:%.*]] = add i32 [[SUM]], [[MERGE_PHI]] ; CHECK-NEXT: [[LOOP_COND:%.*]] = icmp ugt i32 [[SUM_NEXT]], 4 -; CHECK-NEXT: br i1 [[LOOP_COND]], label [[LOOP]], label [[DONE:%.*]] +; CHECK-NEXT: br i1 [[LOOP_COND]], label [[BACKEDGE_LOOP_CRIT_EDGE:%.*]], label [[DONE:%.*]] +; CHECK: backedge.loop_crit_edge: +; CHECK-NEXT: unreachable ; CHECK: done: ; CHECK-NEXT: [[SUM_NEXT_LCSSA:%.*]] = phi i32 [ [[SUM_NEXT]], [[BACKEDGE]] ] ; CHECK-NEXT: ret i32 [[SUM_NEXT_LCSSA]] @@ -594,22 +598,24 @@ define i32 @test_multiple_pred_const() { ; CHECK-NEXT: entry: ; CHECK-NEXT: br label [[LOOP:%.*]] ; CHECK: loop: -; CHECK-NEXT: [[SUM:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[SUM_NEXT:%.*]], [[BACKEDGE:%.*]] ] +; CHECK-NEXT: [[SUM:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ] ; CHECK-NEXT: [[SUB:%.*]] = sub i32 4, [[SUM]] ; CHECK-NEXT: [[IS_POSITIVE:%.*]] = icmp sgt i32 [[SUB]], 0 ; CHECK-NEXT: br i1 [[IS_POSITIVE]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]] ; CHECK: if.true: ; CHECK-NEXT: switch i32 4, label [[FAILURE:%.*]] [ -; CHECK-NEXT: i32 100, label [[BACKEDGE]] +; CHECK-NEXT: i32 100, label [[BACKEDGE:%.*]] ; CHECK-NEXT: i32 200, label [[BACKEDGE]] ; CHECK-NEXT: ] ; CHECK: if.false: ; CHECK-NEXT: br label [[BACKEDGE]] ; CHECK: backedge: ; CHECK-NEXT: [[MERGE_PHI:%.*]] = phi i32 [ 0, [[IF_FALSE]] ], [ [[SUB]], [[IF_TRUE]] ], [ [[SUB]], [[IF_TRUE]] ] -; CHECK-NEXT: [[SUM_NEXT]] = add i32 [[SUM]], [[MERGE_PHI]] +; CHECK-NEXT: [[SUM_NEXT:%.*]] = add i32 [[SUM]], [[MERGE_PHI]] ; CHECK-NEXT: [[LOOP_COND:%.*]] = icmp ne i32 [[SUM_NEXT]], 4 -; CHECK-NEXT: br i1 [[LOOP_COND]], label [[LOOP]], label [[DONE:%.*]] +; CHECK-NEXT: br i1 [[LOOP_COND]], label [[BACKEDGE_LOOP_CRIT_EDGE:%.*]], label [[DONE:%.*]] +; CHECK: backedge.loop_crit_edge: +; CHECK-NEXT: unreachable ; CHECK: done: ; CHECK-NEXT: [[SUM_NEXT_LCSSA:%.*]] = phi i32 [ [[SUM_NEXT]], [[BACKEDGE]] ] ; CHECK-NEXT: ret i32 [[SUM_NEXT_LCSSA]] diff --git a/llvm/test/Transforms/LoopDeletion/zero-btc.ll b/llvm/test/Transforms/LoopDeletion/zero-btc.ll index 413b0a877dd481..958de0505af7fc 100644 --- a/llvm/test/Transforms/LoopDeletion/zero-btc.ll +++ b/llvm/test/Transforms/LoopDeletion/zero-btc.ll @@ -161,14 +161,16 @@ define void @test_multi_exit3(i1 %cond1) { ; CHECK-NEXT: entry: ; CHECK-NEXT: br label [[LOOP:%.*]] ; CHECK: loop: -; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_INC:%.*]], [[LATCH:%.*]] ] +; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ] ; CHECK-NEXT: store i32 0, i32* @G, align 4 -; CHECK-NEXT: br i1 [[COND1:%.*]], label [[LATCH]], label [[EXIT:%.*]] +; CHECK-NEXT: br i1 [[COND1:%.*]], label [[LATCH:%.*]], label [[EXIT:%.*]] ; CHECK: latch: ; CHECK-NEXT: store i32 1, i32* @G, align 4 -; CHECK-NEXT: [[IV_INC]] = add i32 [[IV]], 1 +; CHECK-NEXT: [[IV_INC:%.*]] = add i32 [[IV]], 1 ; CHECK-NEXT: [[BE_TAKEN:%.*]] = icmp ne i32 [[IV_INC]], 1 -; CHECK-NEXT: br i1 [[BE_TAKEN]], label [[LOOP]], label [[EXIT]] +; CHECK-NEXT: br i1 [[BE_TAKEN]], label [[LATCH_LOOP_CRIT_EDGE:%.*]], label [[EXIT]] +; CHECK: latch.loop_crit_edge: +; CHECK-NEXT: unreachable ; CHECK: exit: ; CHECK-NEXT: ret void ;