diff --git a/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp b/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp index 9e23fa9af786b..db7a6d5c63912 100644 --- a/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp +++ b/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp @@ -195,6 +195,9 @@ Value *SimplifyIndvar::foldIVUser(Instruction *UseInst, Instruction *IVOperand) bool SimplifyIndvar::makeIVComparisonInvariant(ICmpInst *ICmp, Instruction *IVOperand) { + auto *Preheader = L->getLoopPreheader(); + if (!Preheader) + return false; unsigned IVOperIdx = 0; ICmpInst::Predicate Pred = ICmp->getPredicate(); if (IVOperand != ICmp->getOperand(0)) { @@ -209,11 +212,6 @@ bool SimplifyIndvar::makeIVComparisonInvariant(ICmpInst *ICmp, const Loop *ICmpLoop = LI->getLoopFor(ICmp->getParent()); const SCEV *S = SE->getSCEVAtScope(ICmp->getOperand(IVOperIdx), ICmpLoop); const SCEV *X = SE->getSCEVAtScope(ICmp->getOperand(1 - IVOperIdx), ICmpLoop); - - auto *PN = dyn_cast(IVOperand); - if (!PN) - return false; - auto LIP = SE->getLoopInvariantPredicate(Pred, S, X, L, ICmp); if (!LIP) return false; @@ -221,40 +219,18 @@ bool SimplifyIndvar::makeIVComparisonInvariant(ICmpInst *ICmp, const SCEV *InvariantLHS = LIP->LHS; const SCEV *InvariantRHS = LIP->RHS; - // Rewrite the comparison to a loop invariant comparison if it can be done - // cheaply, where cheaply means "we don't need to emit any new - // instructions". - - SmallDenseMap CheapExpansions; - CheapExpansions[S] = ICmp->getOperand(IVOperIdx); - CheapExpansions[X] = ICmp->getOperand(1 - IVOperIdx); - - // TODO: Support multiple entry loops? (We currently bail out of these in - // the IndVarSimplify pass) - if (auto *BB = L->getLoopPredecessor()) { - const int Idx = PN->getBasicBlockIndex(BB); - if (Idx >= 0) { - Value *Incoming = PN->getIncomingValue(Idx); - const SCEV *IncomingS = SE->getSCEV(Incoming); - CheapExpansions[IncomingS] = Incoming; - } - } - Value *NewLHS = CheapExpansions[InvariantLHS]; - Value *NewRHS = CheapExpansions[InvariantRHS]; - - if (!NewLHS) - if (auto *ConstLHS = dyn_cast(InvariantLHS)) - NewLHS = ConstLHS->getValue(); - if (!NewRHS) - if (auto *ConstRHS = dyn_cast(InvariantRHS)) - NewRHS = ConstRHS->getValue(); - - if (!NewLHS || !NewRHS) - // We could not find an existing value to replace either LHS or RHS. - // Generating new instructions has subtler tradeoffs, so avoid doing that - // for now. + // Do not generate something ridiculous. + auto *PHTerm = Preheader->getTerminator(); + if (Rewriter.isHighCostExpansion(InvariantLHS, L, SCEVCheapExpansionBudget, + TTI, PHTerm)) return false; - + if (Rewriter.isHighCostExpansion(InvariantRHS, L, SCEVCheapExpansionBudget, + TTI, PHTerm)) + return false; + auto *NewLHS = + Rewriter.expandCodeFor(InvariantLHS, IVOperand->getType(), PHTerm); + auto *NewRHS = + Rewriter.expandCodeFor(InvariantRHS, IVOperand->getType(), PHTerm); LLVM_DEBUG(dbgs() << "INDVARS: Simplified comparison: " << *ICmp << '\n'); ICmp->setPredicate(InvariantPredicate); ICmp->setOperand(0, NewLHS); diff --git a/llvm/test/Transforms/IndVarSimplify/X86/loop-invariant-conditions.ll b/llvm/test/Transforms/IndVarSimplify/X86/loop-invariant-conditions.ll index 5ce4539e4bf29..a27ea770d1159 100644 --- a/llvm/test/Transforms/IndVarSimplify/X86/loop-invariant-conditions.ll +++ b/llvm/test/Transforms/IndVarSimplify/X86/loop-invariant-conditions.ll @@ -29,11 +29,10 @@ for.end: ; preds = %if.end, %entry define void @test1.next(i64 %start) { ; CHECK-LABEL: @test1.next( ; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = add nsw i64 [[START:%.*]], 1 ; CHECK-NEXT: br label [[LOOP:%.*]] ; CHECK: loop: -; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[START:%.*]], [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[LOOP]] ] -; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], 1 -; CHECK-NEXT: [[CMP1:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], 0 +; CHECK-NEXT: [[CMP1:%.*]] = icmp slt i64 [[TMP0]], 0 ; CHECK-NEXT: br i1 [[CMP1]], label [[FOR_END:%.*]], label [[LOOP]] ; CHECK: for.end: ; CHECK-NEXT: ret void @@ -77,11 +76,10 @@ for.end: ; preds = %if.end, %entry define void @test2.next(i64 %start) { ; CHECK-LABEL: @test2.next( ; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = add nsw i64 [[START:%.*]], 1 ; CHECK-NEXT: br label [[LOOP:%.*]] ; CHECK: loop: -; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[START:%.*]], [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[LOOP]] ] -; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], 1 -; CHECK-NEXT: [[CMP1:%.*]] = icmp sle i64 [[INDVARS_IV_NEXT]], 0 +; CHECK-NEXT: [[CMP1:%.*]] = icmp sle i64 [[TMP0]], 0 ; CHECK-NEXT: br i1 [[CMP1]], label [[FOR_END:%.*]], label [[LOOP]] ; CHECK: for.end: ; CHECK-NEXT: ret void @@ -138,15 +136,16 @@ for.end: ; preds = %if.end, %entry define void @test3.next(i64 %start) { ; CHECK-LABEL: @test3.next( ; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = add nsw i64 [[START:%.*]], 1 ; CHECK-NEXT: br label [[LOOP:%.*]] ; CHECK: loop: -; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[START:%.*]], [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[BACKEDGE:%.*]] ] +; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[START]], [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[BACKEDGE:%.*]] ] ; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], 1 ; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 25 ; CHECK-NEXT: br i1 [[CMP]], label [[BACKEDGE]], label [[FOR_END:%.*]] ; CHECK: backedge: ; CHECK-NEXT: call void @foo() -; CHECK-NEXT: [[CMP1:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], 0 +; CHECK-NEXT: [[CMP1:%.*]] = icmp slt i64 [[TMP0]], 0 ; CHECK-NEXT: br i1 [[CMP1]], label [[FOR_END]], label [[LOOP]] ; CHECK: for.end: ; CHECK-NEXT: ret void @@ -209,15 +208,16 @@ for.end: ; preds = %if.end, %entry define void @test4.next(i64 %start) { ; CHECK-LABEL: @test4.next( ; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = add nsw i64 [[START:%.*]], 1 ; CHECK-NEXT: br label [[LOOP:%.*]] ; CHECK: loop: -; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[START:%.*]], [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[BACKEDGE:%.*]] ] +; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[START]], [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[BACKEDGE:%.*]] ] ; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], 1 ; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 25 ; CHECK-NEXT: br i1 [[CMP]], label [[BACKEDGE]], label [[FOR_END:%.*]] ; CHECK: backedge: ; CHECK-NEXT: call void @foo() -; CHECK-NEXT: [[CMP1:%.*]] = icmp sgt i64 [[INDVARS_IV_NEXT]], 0 +; CHECK-NEXT: [[CMP1:%.*]] = icmp sgt i64 [[TMP0]], 0 ; CHECK-NEXT: br i1 [[CMP1]], label [[LOOP]], label [[FOR_END]] ; CHECK: for.end: ; CHECK-NEXT: ret void @@ -279,15 +279,16 @@ for.end: ; preds = %if.end, %entry define void @test5.next(i64 %start) { ; CHECK-LABEL: @test5.next( ; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = add nuw i64 [[START:%.*]], 1 ; CHECK-NEXT: br label [[LOOP:%.*]] ; CHECK: loop: -; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[START:%.*]], [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[BACKEDGE:%.*]] ] +; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[START]], [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[BACKEDGE:%.*]] ] ; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw i64 [[INDVARS_IV]], 1 ; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 25 ; CHECK-NEXT: br i1 [[CMP]], label [[BACKEDGE]], label [[FOR_END:%.*]] ; CHECK: backedge: ; CHECK-NEXT: call void @foo() -; CHECK-NEXT: [[CMP1:%.*]] = icmp ugt i64 [[INDVARS_IV_NEXT]], 101 +; CHECK-NEXT: [[CMP1:%.*]] = icmp ugt i64 [[TMP0]], 101 ; CHECK-NEXT: br i1 [[CMP1]], label [[LOOP]], label [[FOR_END]] ; CHECK: for.end: ; CHECK-NEXT: ret void @@ -350,15 +351,16 @@ for.end: ; preds = %if.end, %entry define void @test6.next(i64 %start) { ; CHECK-LABEL: @test6.next( ; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = add nuw i64 [[START:%.*]], 1 ; CHECK-NEXT: br label [[LOOP:%.*]] ; CHECK: loop: -; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[START:%.*]], [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[BACKEDGE:%.*]] ] +; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[START]], [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[BACKEDGE:%.*]] ] ; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw i64 [[INDVARS_IV]], 1 ; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 25 ; CHECK-NEXT: br i1 [[CMP]], label [[BACKEDGE]], label [[FOR_END:%.*]] ; CHECK: backedge: ; CHECK-NEXT: call void @foo() -; CHECK-NEXT: [[CMP1:%.*]] = icmp ult i64 [[INDVARS_IV_NEXT]], 101 +; CHECK-NEXT: [[CMP1:%.*]] = icmp ult i64 [[TMP0]], 101 ; CHECK-NEXT: br i1 [[CMP1]], label [[FOR_END]], label [[LOOP]] ; CHECK: for.end: ; CHECK-NEXT: ret void @@ -420,11 +422,10 @@ define void @test7.next(i64 %start, i64* %inc_ptr) { ; CHECK-NEXT: [[OK:%.*]] = icmp sge i64 [[INC]], 0 ; CHECK-NEXT: br i1 [[OK]], label [[LOOP_PREHEADER:%.*]], label [[FOR_END:%.*]] ; CHECK: loop.preheader: +; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INC]], [[START:%.*]] ; CHECK-NEXT: br label [[LOOP:%.*]] ; CHECK: loop: -; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[LOOP]] ], [ [[START:%.*]], [[LOOP_PREHEADER]] ] -; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], [[INC]] -; CHECK-NEXT: [[CMP1:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], 0 +; CHECK-NEXT: [[CMP1:%.*]] = icmp slt i64 [[TMP0]], 0 ; CHECK-NEXT: br i1 [[CMP1]], label [[FOR_END_LOOPEXIT:%.*]], label [[LOOP]] ; CHECK: for.end.loopexit: ; CHECK-NEXT: br label [[FOR_END]] diff --git a/llvm/test/Transforms/IndVarSimplify/X86/pr57187.ll b/llvm/test/Transforms/IndVarSimplify/X86/pr57187.ll index 08633344276c1..8dbb2a9126855 100644 --- a/llvm/test/Transforms/IndVarSimplify/X86/pr57187.ll +++ b/llvm/test/Transforms/IndVarSimplify/X86/pr57187.ll @@ -8,15 +8,16 @@ target triple = "x86_64-unknown-linux-gnu" define void @test(i32 %start) { ; CHECK-LABEL: @test( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[TMP0:%.*]] = zext i32 [[START:%.*]] to i64 +; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[START:%.*]], -1 +; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[START]] to i64 ; CHECK-NEXT: br label [[LOOP:%.*]] ; CHECK: backedge: ; CHECK-NEXT: br label [[LOOP]] ; CHECK: loop: -; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[BACKEDGE:%.*]] ], [ [[TMP0]], [[ENTRY:%.*]] ] +; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[BACKEDGE:%.*]] ], [ [[TMP1]], [[ENTRY:%.*]] ] ; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], -1 ; CHECK-NEXT: [[INDVARS:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32 -; CHECK-NEXT: [[LOOP_EXIT_COND:%.*]] = icmp slt i32 [[INDVARS]], 11 +; CHECK-NEXT: [[LOOP_EXIT_COND:%.*]] = icmp slt i32 [[TMP0]], 11 ; CHECK-NEXT: br i1 [[LOOP_EXIT_COND]], label [[EXIT:%.*]], label [[STUCK_PREHEADER:%.*]] ; CHECK: stuck.preheader: ; CHECK-NEXT: br label [[STUCK:%.*]]