diff --git a/llvm/include/llvm/Analysis/ScalarEvolution.h b/llvm/include/llvm/Analysis/ScalarEvolution.h index af3ad822e0b0d..0880f9c65aa45 100644 --- a/llvm/include/llvm/Analysis/ScalarEvolution.h +++ b/llvm/include/llvm/Analysis/ScalarEvolution.h @@ -1314,6 +1314,13 @@ class ScalarEvolution { void getPoisonGeneratingValues(SmallPtrSetImpl &Result, const SCEV *S); + /// Check whether it is poison-safe to represent the expression S using the + /// instruction I. If such a replacement is performed, the poison flags of + /// instructions in DropPoisonGeneratingInsts must be dropped. + bool canReuseInstruction( + const SCEV *S, Instruction *I, + SmallVectorImpl &DropPoisonGeneratingInsts); + class FoldID { const SCEV *Op = nullptr; const Type *Ty = nullptr; diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp index 2acb45837c480..4b2db80bc1ec3 100644 --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -4184,6 +4184,68 @@ void ScalarEvolution::getPoisonGeneratingValues( Result.insert(SU->getValue()); } +bool ScalarEvolution::canReuseInstruction( + const SCEV *S, Instruction *I, + SmallVectorImpl &DropPoisonGeneratingInsts) { + // If the instruction cannot be poison, it's always safe to reuse. + if (programUndefinedIfPoison(I)) + return true; + + // Otherwise, it is possible that I is more poisonous that S. Collect the + // poison-contributors of S, and then check whether I has any additional + // poison-contributors. Poison that is contributed through poison-generating + // flags is handled by dropping those flags instead. + SmallPtrSet PoisonVals; + getPoisonGeneratingValues(PoisonVals, S); + + SmallVector Worklist; + SmallPtrSet Visited; + Worklist.push_back(I); + while (!Worklist.empty()) { + Value *V = Worklist.pop_back_val(); + if (!Visited.insert(V).second) + continue; + + // Avoid walking large instruction graphs. + if (Visited.size() > 16) + return false; + + // Either the value can't be poison, or the S would also be poison if it + // is. + if (PoisonVals.contains(V) || isGuaranteedNotToBePoison(V)) + continue; + + auto *I = dyn_cast(V); + if (!I) + return false; + + // Disjoint or instructions are interpreted as adds by SCEV. However, we + // can't replace an arbitrary add with disjoint or, even if we drop the + // flag. We would need to convert the or into an add. + if (auto *PDI = dyn_cast(I)) + if (PDI->isDisjoint()) + return false; + + // FIXME: Ignore vscale, even though it technically could be poison. Do this + // because SCEV currently assumes it can't be poison. Remove this special + // case once we proper model when vscale can be poison. + if (auto *II = dyn_cast(I); + II && II->getIntrinsicID() == Intrinsic::vscale) + continue; + + if (canCreatePoison(cast(I), /*ConsiderFlagsAndMetadata*/ false)) + return false; + + // If the instruction can't create poison, we can recurse to its operands. + if (I->hasPoisonGeneratingFlagsOrMetadata()) + DropPoisonGeneratingInsts.push_back(I); + + for (Value *Op : I->operands()) + Worklist.push_back(Op); + } + return true; +} + const SCEV * ScalarEvolution::getSequentialMinMaxExpr(SCEVTypes Kind, SmallVectorImpl &Ops) { diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp index a1d7f0f9ba0f7..a3951fdf8a158 100644 --- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp +++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp @@ -1366,61 +1366,6 @@ Value *SCEVExpander::expandCodeFor(const SCEV *SH, Type *Ty) { return V; } -static bool -canReuseInstruction(ScalarEvolution &SE, const SCEV *S, Instruction *I, - SmallVectorImpl &DropPoisonGeneratingInsts) { - // If the instruction cannot be poison, it's always safe to reuse. - if (programUndefinedIfPoison(I)) - return true; - - // Otherwise, it is possible that I is more poisonous that S. Collect the - // poison-contributors of S, and then check whether I has any additional - // poison-contributors. Poison that is contributed through poison-generating - // flags is handled by dropping those flags instead. - SmallPtrSet PoisonVals; - SE.getPoisonGeneratingValues(PoisonVals, S); - - SmallVector Worklist; - SmallPtrSet Visited; - Worklist.push_back(I); - while (!Worklist.empty()) { - Value *V = Worklist.pop_back_val(); - if (!Visited.insert(V).second) - continue; - - // Avoid walking large instruction graphs. - if (Visited.size() > 16) - return false; - - // Either the value can't be poison, or the S would also be poison if it - // is. - if (PoisonVals.contains(V) || isGuaranteedNotToBePoison(V)) - continue; - - auto *I = dyn_cast(V); - if (!I) - return false; - - // FIXME: Ignore vscale, even though it technically could be poison. Do this - // because SCEV currently assumes it can't be poison. Remove this special - // case once we proper model when vscale can be poison. - if (auto *II = dyn_cast(I); - II && II->getIntrinsicID() == Intrinsic::vscale) - continue; - - if (canCreatePoison(cast(I), /*ConsiderFlagsAndMetadata*/ false)) - return false; - - // If the instruction can't create poison, we can recurse to its operands. - if (I->hasPoisonGeneratingFlagsOrMetadata()) - DropPoisonGeneratingInsts.push_back(I); - - for (Value *Op : I->operands()) - Worklist.push_back(Op); - } - return true; -} - Value *SCEVExpander::FindValueInExprValueMap( const SCEV *S, const Instruction *InsertPt, SmallVectorImpl &DropPoisonGeneratingInsts) { @@ -1448,7 +1393,7 @@ Value *SCEVExpander::FindValueInExprValueMap( continue; // Make sure reusing the instruction is poison-safe. - if (canReuseInstruction(SE, S, EntInst, DropPoisonGeneratingInsts)) + if (SE.canReuseInstruction(S, EntInst, DropPoisonGeneratingInsts)) return V; DropPoisonGeneratingInsts.clear(); } diff --git a/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp b/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp index 0ed3324a27b6c..1b142f14d8113 100644 --- a/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp +++ b/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp @@ -16,6 +16,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" #include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/ValueTracking.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/Instructions.h" @@ -713,8 +714,11 @@ bool SimplifyIndvar::replaceFloatIVWithIntegerIV(Instruction *UseInst) { bool SimplifyIndvar::eliminateIdentitySCEV(Instruction *UseInst, Instruction *IVOperand) { if (!SE->isSCEVable(UseInst->getType()) || - (UseInst->getType() != IVOperand->getType()) || - (SE->getSCEV(UseInst) != SE->getSCEV(IVOperand))) + UseInst->getType() != IVOperand->getType()) + return false; + + const SCEV *UseSCEV = SE->getSCEV(UseInst); + if (UseSCEV != SE->getSCEV(IVOperand)) return false; // getSCEV(X) == getSCEV(Y) does not guarantee that X and Y are related in the @@ -742,6 +746,16 @@ bool SimplifyIndvar::eliminateIdentitySCEV(Instruction *UseInst, if (!LI->replacementPreservesLCSSAForm(UseInst, IVOperand)) return false; + // Make sure the operand is not more poisonous than the instruction. + if (!impliesPoison(IVOperand, UseInst)) { + SmallVector DropPoisonGeneratingInsts; + if (!SE->canReuseInstruction(UseSCEV, IVOperand, DropPoisonGeneratingInsts)) + return false; + + for (Instruction *I : DropPoisonGeneratingInsts) + I->dropPoisonGeneratingFlagsAndMetadata(); + } + LLVM_DEBUG(dbgs() << "INDVARS: Eliminated identity: " << *UseInst << '\n'); SE->forgetValue(UseInst); diff --git a/llvm/test/Transforms/IndVarSimplify/pr55925.ll b/llvm/test/Transforms/IndVarSimplify/pr55925.ll index 420fc209949d4..312a8295ccdc9 100644 --- a/llvm/test/Transforms/IndVarSimplify/pr55925.ll +++ b/llvm/test/Transforms/IndVarSimplify/pr55925.ll @@ -18,9 +18,9 @@ define void @test(ptr %p) personality ptr undef { ; CHECK-NEXT: [[RES:%.*]] = invoke i32 @foo(i32 returned [[TMP0]]) ; CHECK-NEXT: to label [[LOOP_LATCH]] unwind label [[EXIT:%.*]] ; CHECK: loop.latch: -; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1 ; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[INDVARS_IV]] to i32 ; CHECK-NEXT: [[TMP2:%.*]] = call i32 @foo(i32 [[TMP1]]) +; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1 ; CHECK-NEXT: br label [[LOOP]] ; CHECK: exit: ; CHECK-NEXT: [[LP:%.*]] = landingpad { ptr, i32 } @@ -64,8 +64,8 @@ define void @test_critedge(i1 %c, ptr %p) personality ptr undef { ; CHECK-NEXT: br label [[LOOP_LATCH]] ; CHECK: loop.latch: ; CHECK-NEXT: [[PHI:%.*]] = phi i32 [ [[TMP1]], [[LOOP_INVOKE]] ], [ 0, [[LOOP_OTHER]] ] -; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1 ; CHECK-NEXT: [[TMP2:%.*]] = call i32 @foo(i32 [[PHI]]) +; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1 ; CHECK-NEXT: br label [[LOOP]] ; CHECK: exit: ; CHECK-NEXT: [[LP:%.*]] = landingpad { ptr, i32 } diff --git a/llvm/test/Transforms/IndVarSimplify/pr79861.ll b/llvm/test/Transforms/IndVarSimplify/pr79861.ll new file mode 100644 index 0000000000000..6625094496139 --- /dev/null +++ b/llvm/test/Transforms/IndVarSimplify/pr79861.ll @@ -0,0 +1,107 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -S -passes=indvars < %s | FileCheck %s + +target datalayout = "n64" + +declare void @use(i64) + +define void @or_disjoint() { +; CHECK-LABEL: define void @or_disjoint() { +; CHECK-NEXT: entry: +; CHECK-NEXT: br label [[LOOP:%.*]] +; CHECK: loop: +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 2, [[ENTRY:%.*]] ], [ [[IV_DEC:%.*]], [[LOOP]] ] +; CHECK-NEXT: [[OR:%.*]] = or disjoint i64 [[IV]], 1 +; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i64 [[IV]], 1 +; CHECK-NEXT: [[SEL:%.*]] = select i1 false, i64 [[OR]], i64 [[ADD]] +; CHECK-NEXT: call void @use(i64 [[SEL]]) +; CHECK-NEXT: [[IV_DEC]] = add nsw i64 [[IV]], -1 +; CHECK-NEXT: [[EXIT_COND:%.*]] = icmp eq i64 [[IV_DEC]], 0 +; CHECK-NEXT: br i1 [[EXIT_COND]], label [[EXIT:%.*]], label [[LOOP]] +; CHECK: exit: +; CHECK-NEXT: ret void +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 2, %entry ], [ %iv.dec, %loop ] + %or = or disjoint i64 %iv, 1 + %add = add nsw i64 %iv, 1 + %sel = select i1 false, i64 %or, i64 %add + call void @use(i64 %sel) + + %iv.dec = add nsw i64 %iv, -1 + %exit.cond = icmp eq i64 %iv.dec, 0 + br i1 %exit.cond, label %exit, label %loop + +exit: + ret void +} + +define void @add_nowrap_flags(i64 %n) { +; CHECK-LABEL: define void @add_nowrap_flags( +; CHECK-SAME: i64 [[N:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: br label [[LOOP:%.*]] +; CHECK: loop: +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_INC:%.*]], [[LOOP]] ] +; CHECK-NEXT: [[ADD1:%.*]] = add i64 [[IV]], 123 +; CHECK-NEXT: call void @use(i64 [[ADD1]]) +; CHECK-NEXT: [[IV_INC]] = add i64 [[IV]], 1 +; CHECK-NEXT: [[EXIT_COND:%.*]] = icmp eq i64 [[IV_INC]], [[N]] +; CHECK-NEXT: br i1 [[EXIT_COND]], label [[EXIT:%.*]], label [[LOOP]] +; CHECK: exit: +; CHECK-NEXT: ret void +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %iv.inc, %loop ] + %add1 = add nuw nsw i64 %iv, 123 + %add2 = add i64 %iv, 123 + %sel = select i1 false, i64 %add1, i64 %add2 + call void @use(i64 %sel) + + %iv.inc = add i64 %iv, 1 + %exit.cond = icmp eq i64 %iv.inc, %n + br i1 %exit.cond, label %exit, label %loop + +exit: + ret void +} + + +define void @expander_or_disjoint(i64 %n) { +; CHECK-LABEL: define void @expander_or_disjoint( +; CHECK-SAME: i64 [[N:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[OR:%.*]] = or disjoint i64 [[N]], 1 +; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1 +; CHECK-NEXT: br label [[LOOP:%.*]] +; CHECK: loop: +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_INC:%.*]], [[LOOP]] ] +; CHECK-NEXT: [[IV_INC]] = add i64 [[IV]], 1 +; CHECK-NEXT: [[ADD:%.*]] = add i64 [[IV]], [[OR]] +; CHECK-NEXT: call void @use(i64 [[ADD]]) +; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[IV_INC]], [[TMP0]] +; CHECK-NEXT: br i1 [[EXITCOND]], label [[LOOP]], label [[EXIT:%.*]] +; CHECK: exit: +; CHECK-NEXT: ret void +; +entry: + %or = or disjoint i64 %n, 1 + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %iv.inc, %loop ] + %iv.inc = add i64 %iv, 1 + %add = add i64 %iv, %or + call void @use(i64 %add) + %cmp = icmp ult i64 %iv, %n + br i1 %cmp, label %loop, label %exit + +exit: + ret void +}