diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h index 28546110ba04c1..91bc90d2cd389e 100644 --- a/llvm/include/llvm/Analysis/IVDescriptors.h +++ b/llvm/include/llvm/Analysis/IVDescriptors.h @@ -114,7 +114,7 @@ class RecurrenceDescriptor { /// compare instruction to the select instruction and stores this pointer in /// 'PatternLastInst' member of the returned struct. static InstDesc isRecurrenceInstr(Instruction *I, RecurKind Kind, - InstDesc &Prev, bool HasFunNoNaNAttr); + InstDesc &Prev, FastMathFlags FMF); /// Returns true if instruction I has multiple uses in Insts static bool hasMultipleUsesOf(Instruction *I, @@ -146,7 +146,7 @@ class RecurrenceDescriptor { /// non-null, the minimal bit width needed to compute the reduction will be /// computed. static bool AddReductionVar(PHINode *Phi, RecurKind Kind, Loop *TheLoop, - bool HasFunNoNaNAttr, + FastMathFlags FMF, RecurrenceDescriptor &RedDes, DemandedBits *DB = nullptr, AssumptionCache *AC = nullptr, diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp index 64b55b85f3a188..6204331a2e9b13 100644 --- a/llvm/lib/Analysis/IVDescriptors.cpp +++ b/llvm/lib/Analysis/IVDescriptors.cpp @@ -190,7 +190,7 @@ static void collectCastsToIgnore(Loop *TheLoop, Instruction *Exit, } bool RecurrenceDescriptor::AddReductionVar(PHINode *Phi, RecurKind Kind, - Loop *TheLoop, bool HasFunNoNaNAttr, + Loop *TheLoop, FastMathFlags FuncFMF, RecurrenceDescriptor &RedDes, DemandedBits *DB, AssumptionCache *AC, @@ -298,7 +298,7 @@ bool RecurrenceDescriptor::AddReductionVar(PHINode *Phi, RecurKind Kind, // the starting value (the Phi or an AND instruction if the Phi has been // type-promoted). if (Cur != Start) { - ReduxDesc = isRecurrenceInstr(Cur, Kind, ReduxDesc, HasFunNoNaNAttr); + ReduxDesc = isRecurrenceInstr(Cur, Kind, ReduxDesc, FuncFMF); if (!ReduxDesc.isRecurrence()) return false; // FIXME: FMF is allowed on phi, but propagation is not handled correctly. @@ -561,7 +561,7 @@ RecurrenceDescriptor::isConditionalRdxPattern(RecurKind Kind, Instruction *I) { RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(Instruction *I, RecurKind Kind, - InstDesc &Prev, bool HasFunNoNaNAttr) { + InstDesc &Prev, FastMathFlags FMF) { Instruction *UAI = Prev.getUnsafeAlgebraInst(); if (!UAI && isa(I) && !I->hasAllowReassoc()) UAI = I; // Found an unsafe (unvectorizable) algebra instruction. @@ -594,10 +594,11 @@ RecurrenceDescriptor::isRecurrenceInstr(Instruction *I, RecurKind Kind, LLVM_FALLTHROUGH; case Instruction::FCmp: case Instruction::ICmp: - if (!isIntMinMaxRecurrenceKind(Kind) && - (!HasFunNoNaNAttr || !isFPMinMaxRecurrenceKind(Kind))) - return InstDesc(false, I); - return isMinMaxSelectCmpPattern(I, Prev); + if (isIntMinMaxRecurrenceKind(Kind) || + (FMF.noNaNs() && FMF.noSignedZeros() && + isFPMinMaxRecurrenceKind(Kind))) + return isMinMaxSelectCmpPattern(I, Prev); + return InstDesc(false, I); } } @@ -622,71 +623,61 @@ bool RecurrenceDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop, BasicBlock *Header = TheLoop->getHeader(); Function &F = *Header->getParent(); - bool HasFunNoNaNAttr = - F.getFnAttribute("no-nans-fp-math").getValueAsString() == "true"; + FastMathFlags FMF; + FMF.setNoNaNs( + F.getFnAttribute("no-nans-fp-math").getValueAsString() == "true"); + FMF.setNoSignedZeros( + F.getFnAttribute("no-signed-zeros-fp-math").getValueAsString() == "true"); - if (AddReductionVar(Phi, RecurKind::Add, TheLoop, HasFunNoNaNAttr, RedDes, DB, - AC, DT)) { + if (AddReductionVar(Phi, RecurKind::Add, TheLoop, FMF, RedDes, DB, AC, DT)) { LLVM_DEBUG(dbgs() << "Found an ADD reduction PHI." << *Phi << "\n"); return true; } - if (AddReductionVar(Phi, RecurKind::Mul, TheLoop, HasFunNoNaNAttr, RedDes, DB, - AC, DT)) { + if (AddReductionVar(Phi, RecurKind::Mul, TheLoop, FMF, RedDes, DB, AC, DT)) { LLVM_DEBUG(dbgs() << "Found a MUL reduction PHI." << *Phi << "\n"); return true; } - if (AddReductionVar(Phi, RecurKind::Or, TheLoop, HasFunNoNaNAttr, RedDes, DB, - AC, DT)) { + if (AddReductionVar(Phi, RecurKind::Or, TheLoop, FMF, RedDes, DB, AC, DT)) { LLVM_DEBUG(dbgs() << "Found an OR reduction PHI." << *Phi << "\n"); return true; } - if (AddReductionVar(Phi, RecurKind::And, TheLoop, HasFunNoNaNAttr, RedDes, DB, - AC, DT)) { + if (AddReductionVar(Phi, RecurKind::And, TheLoop, FMF, RedDes, DB, AC, DT)) { LLVM_DEBUG(dbgs() << "Found an AND reduction PHI." << *Phi << "\n"); return true; } - if (AddReductionVar(Phi, RecurKind::Xor, TheLoop, HasFunNoNaNAttr, RedDes, DB, - AC, DT)) { + if (AddReductionVar(Phi, RecurKind::Xor, TheLoop, FMF, RedDes, DB, AC, DT)) { LLVM_DEBUG(dbgs() << "Found a XOR reduction PHI." << *Phi << "\n"); return true; } - if (AddReductionVar(Phi, RecurKind::SMax, TheLoop, HasFunNoNaNAttr, RedDes, - DB, AC, DT)) { + if (AddReductionVar(Phi, RecurKind::SMax, TheLoop, FMF, RedDes, DB, AC, DT)) { LLVM_DEBUG(dbgs() << "Found a SMAX reduction PHI." << *Phi << "\n"); return true; } - if (AddReductionVar(Phi, RecurKind::SMin, TheLoop, HasFunNoNaNAttr, RedDes, - DB, AC, DT)) { + if (AddReductionVar(Phi, RecurKind::SMin, TheLoop, FMF, RedDes, DB, AC, DT)) { LLVM_DEBUG(dbgs() << "Found a SMIN reduction PHI." << *Phi << "\n"); return true; } - if (AddReductionVar(Phi, RecurKind::UMax, TheLoop, HasFunNoNaNAttr, RedDes, - DB, AC, DT)) { + if (AddReductionVar(Phi, RecurKind::UMax, TheLoop, FMF, RedDes, DB, AC, DT)) { LLVM_DEBUG(dbgs() << "Found a UMAX reduction PHI." << *Phi << "\n"); return true; } - if (AddReductionVar(Phi, RecurKind::UMin, TheLoop, HasFunNoNaNAttr, RedDes, - DB, AC, DT)) { + if (AddReductionVar(Phi, RecurKind::UMin, TheLoop, FMF, RedDes, DB, AC, DT)) { LLVM_DEBUG(dbgs() << "Found a UMIN reduction PHI." << *Phi << "\n"); return true; } - if (AddReductionVar(Phi, RecurKind::FMul, TheLoop, HasFunNoNaNAttr, RedDes, - DB, AC, DT)) { + if (AddReductionVar(Phi, RecurKind::FMul, TheLoop, FMF, RedDes, DB, AC, DT)) { LLVM_DEBUG(dbgs() << "Found an FMult reduction PHI." << *Phi << "\n"); return true; } - if (AddReductionVar(Phi, RecurKind::FAdd, TheLoop, HasFunNoNaNAttr, RedDes, - DB, AC, DT)) { + if (AddReductionVar(Phi, RecurKind::FAdd, TheLoop, FMF, RedDes, DB, AC, DT)) { LLVM_DEBUG(dbgs() << "Found an FAdd reduction PHI." << *Phi << "\n"); return true; } - if (AddReductionVar(Phi, RecurKind::FMax, TheLoop, HasFunNoNaNAttr, RedDes, - DB, AC, DT)) { + if (AddReductionVar(Phi, RecurKind::FMax, TheLoop, FMF, RedDes, DB, AC, DT)) { LLVM_DEBUG(dbgs() << "Found a float MAX reduction PHI." << *Phi << "\n"); return true; } - if (AddReductionVar(Phi, RecurKind::FMin, TheLoop, HasFunNoNaNAttr, RedDes, - DB, AC, DT)) { + if (AddReductionVar(Phi, RecurKind::FMin, TheLoop, FMF, RedDes, DB, AC, DT)) { LLVM_DEBUG(dbgs() << "Found a float MIN reduction PHI." << *Phi << "\n"); return true; } diff --git a/llvm/test/Transforms/LoopVectorize/X86/reduction-fastmath.ll b/llvm/test/Transforms/LoopVectorize/X86/reduction-fastmath.ll index a52e9e35a93bb0..5d744044ba2d7b 100644 --- a/llvm/test/Transforms/LoopVectorize/X86/reduction-fastmath.ll +++ b/llvm/test/Transforms/LoopVectorize/X86/reduction-fastmath.ll @@ -292,17 +292,17 @@ define float @PR35538(float* nocapture readonly %a, i32 %N) #0 { ; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, float* [[TMP2]], i32 4 ; CHECK-NEXT: [[TMP7:%.*]] = bitcast float* [[TMP6]] to <4 x float>* ; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x float>, <4 x float>* [[TMP7]], align 4 -; CHECK-NEXT: [[TMP8:%.*]] = fcmp nnan ninf oge <4 x float> [[WIDE_LOAD]], [[VEC_PHI]] -; CHECK-NEXT: [[TMP9:%.*]] = fcmp nnan ninf oge <4 x float> [[WIDE_LOAD2]], [[VEC_PHI1]] +; CHECK-NEXT: [[TMP8:%.*]] = fcmp nnan ninf nsz oge <4 x float> [[WIDE_LOAD]], [[VEC_PHI]] +; CHECK-NEXT: [[TMP9:%.*]] = fcmp nnan ninf nsz oge <4 x float> [[WIDE_LOAD2]], [[VEC_PHI1]] ; CHECK-NEXT: [[TMP10]] = select <4 x i1> [[TMP8]], <4 x float> [[WIDE_LOAD]], <4 x float> [[VEC_PHI]] ; CHECK-NEXT: [[TMP11]] = select <4 x i1> [[TMP9]], <4 x float> [[WIDE_LOAD2]], <4 x float> [[VEC_PHI1]] ; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 8 ; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP8:!llvm.loop !.*]] ; CHECK: middle.block: -; CHECK-NEXT: [[RDX_MINMAX_CMP:%.*]] = fcmp nnan ninf ogt <4 x float> [[TMP10]], [[TMP11]] -; CHECK-NEXT: [[RDX_MINMAX_SELECT:%.*]] = select nnan ninf <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP10]], <4 x float> [[TMP11]] -; CHECK-NEXT: [[TMP13:%.*]] = call nnan ninf float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX_SELECT]]) +; CHECK-NEXT: [[RDX_MINMAX_CMP:%.*]] = fcmp nnan ninf nsz ogt <4 x float> [[TMP10]], [[TMP11]] +; CHECK-NEXT: [[RDX_MINMAX_SELECT:%.*]] = select nnan ninf nsz <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP10]], <4 x float> [[TMP11]] +; CHECK-NEXT: [[TMP13:%.*]] = call nnan ninf nsz float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX_SELECT]]) ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]] ; CHECK: scalar.ph: @@ -320,7 +320,7 @@ define float @PR35538(float* nocapture readonly %a, i32 %N) #0 { ; CHECK-NEXT: [[MAX_013:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[MAX_0_]], [[FOR_BODY]] ] ; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[INDVARS_IV]] ; CHECK-NEXT: [[TMP14:%.*]] = load float, float* [[ARRAYIDX]], align 4 -; CHECK-NEXT: [[CMP1_INV:%.*]] = fcmp nnan ninf oge float [[TMP14]], [[MAX_013]] +; CHECK-NEXT: [[CMP1_INV:%.*]] = fcmp nnan ninf nsz oge float [[TMP14]], [[MAX_013]] ; CHECK-NEXT: [[MAX_0_]] = select i1 [[CMP1_INV]], float [[TMP14]], float [[MAX_013]] ; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 ; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]] @@ -343,7 +343,7 @@ for.body: %max.013 = phi float [ -1.000000e+00, %for.body.lr.ph ], [ %max.0., %for.body ] %arrayidx = getelementptr inbounds float, float* %a, i64 %indvars.iv %0 = load float, float* %arrayidx, align 4 - %cmp1.inv = fcmp nnan ninf oge float %0, %max.013 + %cmp1.inv = fcmp nnan ninf nsz oge float %0, %max.013 %max.0. = select i1 %cmp1.inv, float %0, float %max.013 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count @@ -437,4 +437,4 @@ for.body: br i1 %exitcond, label %for.cond.cleanup, label %for.body } -attributes #0 = { "no-infs-fp-math"="true" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="false" "unsafe-fp-math"="false" } +attributes #0 = { "no-infs-fp-math"="true" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "unsafe-fp-math"="false" } diff --git a/llvm/test/Transforms/LoopVectorize/float-minmax-instruction-flag.ll b/llvm/test/Transforms/LoopVectorize/float-minmax-instruction-flag.ll index a891a756671a5c..f882ed08b4b297 100644 --- a/llvm/test/Transforms/LoopVectorize/float-minmax-instruction-flag.ll +++ b/llvm/test/Transforms/LoopVectorize/float-minmax-instruction-flag.ll @@ -148,6 +148,46 @@ out: ; preds = %loop ret float %t6 } +; This test is checking that we don't vectorize when only one of the required attributes is set. +; Note that this test should not vectorize even after switching to IR-level FMF. +define float @minloopmissingnsz(float* nocapture readonly %arg) #1 { +; CHECK-LABEL: @minloopmissingnsz( +; CHECK-NEXT: top: +; CHECK-NEXT: [[T:%.*]] = load float, float* [[ARG:%.*]], align 4 +; CHECK-NEXT: br label [[LOOP:%.*]] +; CHECK: loop: +; CHECK-NEXT: [[T1:%.*]] = phi i64 [ [[T7:%.*]], [[LOOP]] ], [ 1, [[TOP:%.*]] ] +; CHECK-NEXT: [[T2:%.*]] = phi float [ [[T6:%.*]], [[LOOP]] ], [ [[T]], [[TOP]] ] +; CHECK-NEXT: [[T3:%.*]] = getelementptr float, float* [[ARG]], i64 [[T1]] +; CHECK-NEXT: [[T4:%.*]] = load float, float* [[T3]], align 4 +; CHECK-NEXT: [[T5:%.*]] = fcmp olt float [[T2]], [[T4]] +; CHECK-NEXT: [[T6]] = select i1 [[T5]], float [[T2]], float [[T4]] +; CHECK-NEXT: [[T7]] = add i64 [[T1]], 1 +; CHECK-NEXT: [[T8:%.*]] = icmp eq i64 [[T7]], 65537 +; CHECK-NEXT: br i1 [[T8]], label [[OUT:%.*]], label [[LOOP]] +; CHECK: out: +; CHECK-NEXT: [[T6_LCSSA:%.*]] = phi float [ [[T6]], [[LOOP]] ] +; CHECK-NEXT: ret float [[T6_LCSSA]] +; +top: + %t = load float, float* %arg + br label %loop + +loop: ; preds = %loop, %top + %t1 = phi i64 [ %t7, %loop ], [ 1, %top ] + %t2 = phi float [ %t6, %loop ], [ %t, %top ] + %t3 = getelementptr float, float* %arg, i64 %t1 + %t4 = load float, float* %t3, align 4 + %t5 = fcmp olt float %t2, %t4 + %t6 = select i1 %t5, float %t2, float %t4 + %t7 = add i64 %t1, 1 + %t8 = icmp eq i64 %t7, 65537 + br i1 %t8, label %out, label %loop + +out: ; preds = %loop + ret float %t6 +} + ; This would assert on FMF propagation. define void @not_a_min_max() { @@ -177,4 +217,5 @@ end: ret void } -attributes #0 = { "no-nans-fp-math"="true" } +attributes #0 = { "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" } +attributes #1 = { "no-nans-fp-math"="true" } diff --git a/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll b/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll index dce179c78c1968..2f67b72ee40101 100644 --- a/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll +++ b/llvm/test/Transforms/LoopVectorize/minmax_reduction.ll @@ -852,7 +852,31 @@ for.end: ret float %max.red.0 } +; As above, with the no-signed-zeros-fp-math attribute missing +; CHECK-LABEL: @max_red_float_nsz( +; CHECK-NOT: <2 x float> + +define float @max_red_float_nsz(float %max) #1 { +entry: + br label %for.body + +for.body: + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] + %max.red.08 = phi float [ %max, %entry ], [ %max.red.0, %for.body ] + %arrayidx = getelementptr inbounds [1024 x float], [1024 x float]* @fA, i64 0, i64 %indvars.iv + %0 = load float, float* %arrayidx, align 4 + %cmp3 = fcmp fast ogt float %0, %max.red.08 + %max.red.0 = select i1 %cmp3, float %0, float %max.red.08 + %indvars.iv.next = add i64 %indvars.iv, 1 + %exitcond = icmp eq i64 %indvars.iv.next, 1024 + br i1 %exitcond, label %for.end, label %for.body + +for.end: + ret float %max.red.0 +} + ; Make sure any check-not directives are not triggered by function declarations. ; CHECK: declare -attributes #0 = { "no-nans-fp-math"="true" } +attributes #0 = { "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" } +attributes #1 = { "no-nans-fp-math"="true" }