diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp index 82954ee6f2457..c37b7ecd98b64 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp @@ -3347,6 +3347,80 @@ static Instruction *foldSelectWithFCmpToFabs(SelectInst &SI, return ChangedFMF ? &SI : nullptr; } +// Fold a select of an ordered fcmp using fabs of a NaN-scrubbed value: +// %s = select i1 (isnotnan T %x), T %x, T %y +// %a = call T @llvm.fabs.T(T %s) +// %c = fcmp T %a, %k +// %r = select i1 %c, T %s, T %y +// => +// %a2 = call T @llvm.fabs.T(T %x) +// %c2 = fcmp T %a2, %k +// %r2 = select i1 %c2, T %x, T %y +static Instruction * +foldSelectOfOrderedFAbsCmpOfNaNScrubbedValue(SelectInst &SI, + InstCombinerImpl &IC) { + Instruction *OuterCmpI; + Value *Cmp0, *Cmp1; + if (!match(SI.getCondition(), + m_OneUse(m_Instruction(OuterCmpI, + m_FCmp(m_Value(Cmp0), m_Value(Cmp1)))))) + return nullptr; + + auto *OuterCmp = cast(OuterCmpI); + CmpInst::Predicate Pred = OuterCmp->getPredicate(); + if (!FCmpInst::isOrdered(Pred)) + return nullptr; + + Value *Y = SI.getFalseValue(); + Value *InnerSel = SI.getTrueValue(); + + // Match a select that returns X when X is not NaN, and Y otherwise: + // select (fcmp ord X, 0.0), X, Y + Value *X; + if (!match(InnerSel, + m_Select(m_OneUse(m_SpecificFCmp(FCmpInst::FCMP_ORD, m_Value(X), + m_AnyZeroFP())), + m_Deferred(X), m_Specific(Y)))) + return nullptr; + + Instruction *FAbsI; + auto MatchFAbsOfInnerSel = [&](Value *V) { + return match(V, + m_OneUse(m_Instruction(FAbsI, m_FAbs(m_Specific(InnerSel))))); + }; + + if (!MatchFAbsOfInnerSel(Cmp0)) { + if (!MatchFAbsOfInnerSel(Cmp1)) + return nullptr; + + std::swap(Cmp0, Cmp1); + Pred = CmpInst::getSwappedPredicate(Pred); + } + + FastMathFlags FAbsFMF = FAbsI->getFastMathFlags(); + FastMathFlags CmpFMF = OuterCmp->getFastMathFlags(); + + FastMathFlags CommonRewriteFMF = + FastMathFlags::intersectRewrite(FAbsFMF, CmpFMF); + + // unionValue with FastMathFlags() drops all rewriter based flags + FastMathFlags NewFAbsFMF = + CommonRewriteFMF | FastMathFlags::unionValue(FAbsFMF, FastMathFlags()); + FastMathFlags NewCmpFMF = + CommonRewriteFMF | FastMathFlags::unionValue(CmpFMF, FastMathFlags()); + + // When X is NaN, the old code evaluated fabs(Y), while the new code evaluates + // fabs(X). Do not preserve nnan on either newly-created instruction. + NewFAbsFMF.setNoNaNs(false); + NewCmpFMF.setNoNaNs(false); + + Value *NewAbs = IC.Builder.CreateFAbs(X, FMFSource(NewFAbsFMF)); + Value *NewCmp = + IC.Builder.CreateFCmpFMF(Pred, NewAbs, Cmp1, FMFSource(NewCmpFMF)); + Value *NewSel = IC.Builder.CreateSelectFMF(NewCmp, X, Y, &SI); + return IC.replaceInstUsesWith(SI, NewSel); +} + // Match the following IR pattern: // %x.lowbits = and i8 %x, %lowbitmask // %x.lowbits.are.zero = icmp eq i8 %x.lowbits, 0 @@ -4599,6 +4673,9 @@ Instruction *InstCombinerImpl::visitSelectInst(SelectInst &SI) { if (Instruction *Fabs = foldSelectWithFCmpToFabs(SI, *this)) return Fabs; + if (Instruction *I = foldSelectOfOrderedFAbsCmpOfNaNScrubbedValue(SI, *this)) + return I; + // See if we are selecting two values based on a comparison of the two values. if (CmpInst *CI = dyn_cast(CondVal)) if (Instruction *NewSel = foldSelectValueEquivalence(SI, *CI)) diff --git a/llvm/test/Transforms/InstCombine/fcmp-select.ll b/llvm/test/Transforms/InstCombine/fcmp-select.ll index 268cd4675c6cc..6ec65d5b9aa8d 100644 --- a/llvm/test/Transforms/InstCombine/fcmp-select.ll +++ b/llvm/test/Transforms/InstCombine/fcmp-select.ll @@ -3,6 +3,8 @@ declare void @use(i1) declare void @usef64(double) +declare double @llvm.fabs.f64(double) +declare <2 x double> @llvm.fabs.v2f64(<2 x double>) ; X == 42.0 ? X : 42.0 --> 42.0 @@ -280,6 +282,252 @@ define i1 @test_fcmp_ord_select_fcmp_oeq_var_const(double %x) { ret i1 %cmp2 } +define double @test_fcmp_ord_select_fabs_fcmp_one_select_var_const(double %x) { +; CHECK-LABEL: @test_fcmp_ord_select_fabs_fcmp_one_select_var_const( +; CHECK-NEXT: [[ABS:%.*]] = call double @llvm.fabs.f64(double [[X:%.*]]) +; CHECK-NEXT: [[CMP:%.*]] = fcmp one double [[ABS]], 0x7FF0000000000000 +; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], double [[X]], double 0.000000e+00 +; CHECK-NEXT: ret double [[SEL]] +; + %cmp1 = fcmp ord double %x, 0.000000e+00 + %sel1 = select i1 %cmp1, double %x, double 0.000000e+00 + %abs = call double @llvm.fabs.f64(double %sel1) + %cmp2 = fcmp one double %abs, 0x7FF0000000000000 + %sel2 = select i1 %cmp2, double %sel1, double 0.000000e+00 + ret double %sel2 +} + +define double @test_fcmp_ord_commuted_select_fabs_fcmp_one_select_var_const(double %x) { +; CHECK-LABEL: @test_fcmp_ord_commuted_select_fabs_fcmp_one_select_var_const( +; CHECK-NEXT: [[ABS:%.*]] = call double @llvm.fabs.f64(double [[X:%.*]]) +; CHECK-NEXT: [[CMP:%.*]] = fcmp one double [[ABS]], 0x7FF0000000000000 +; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], double [[X]], double 0.000000e+00 +; CHECK-NEXT: ret double [[SEL]] +; + %cmp1 = fcmp ord double 0.000000e+00, %x + %sel1 = select i1 %cmp1, double %x, double 0.000000e+00 + %abs = call double @llvm.fabs.f64(double %sel1) + %cmp2 = fcmp one double %abs, 0x7FF0000000000000 + %sel2 = select i1 %cmp2, double %sel1, double 0.000000e+00 + ret double %sel2 +} + +define double @test_fcmp_ord_select_fabs_fcmp_one_select_nonnan_const(double %x, double %y) { +; CHECK-LABEL: @test_fcmp_ord_select_fabs_fcmp_one_select_nonnan_const( +; CHECK-NEXT: [[ABS:%.*]] = call double @llvm.fabs.f64(double [[X:%.*]]) +; CHECK-NEXT: [[CMP:%.*]] = fcmp one double [[ABS]], 0x7FF0000000000000 +; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], double [[X]], double [[Y:%.*]] +; CHECK-NEXT: ret double [[SEL]] +; + %cmp1 = fcmp ord double %x, 1.000000e+00 + %sel1 = select i1 %cmp1, double %x, double %y + %abs = call double @llvm.fabs.f64(double %sel1) + %cmp2 = fcmp one double %abs, 0x7FF0000000000000 + %sel2 = select i1 %cmp2, double %sel1, double %y + ret double %sel2 +} + +define double @test_fcmp_ord_self_select_fabs_fcmp_one_select_var_var(double %x, double %y) { +; CHECK-LABEL: @test_fcmp_ord_self_select_fabs_fcmp_one_select_var_var( +; CHECK-NEXT: [[ABS:%.*]] = call double @llvm.fabs.f64(double [[X:%.*]]) +; CHECK-NEXT: [[CMP:%.*]] = fcmp one double [[ABS]], 0x7FF0000000000000 +; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], double [[X]], double [[Y:%.*]] +; CHECK-NEXT: ret double [[SEL]] +; + %cmp1 = fcmp ord double %x, %x + %sel1 = select i1 %cmp1, double %x, double %y + %abs = call double @llvm.fabs.f64(double %sel1) + %cmp2 = fcmp one double %abs, 0x7FF0000000000000 + %sel2 = select i1 %cmp2, double %sel1, double %y + ret double %sel2 +} + +define double @test_fcmp_ord_select_fabs_fcmp_one_select_uitofp_nonnan(double %x, double %y, i32 %i) { +; CHECK-LABEL: @test_fcmp_ord_select_fabs_fcmp_one_select_uitofp_nonnan( +; CHECK: [[ABS:%.*]] = call double @llvm.fabs.f64(double [[X:%.*]]) +; CHECK-NEXT: [[CMP:%.*]] = fcmp one double [[ABS]], 0x7FF0000000000000 +; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], double [[X]], double [[Y:%.*]] +; CHECK-NEXT: ret double [[SEL]] +; + %nonnan = uitofp i32 %i to double + %cmp1 = fcmp ord double %x, %nonnan + %sel1 = select i1 %cmp1, double %x, double %y + %abs = call double @llvm.fabs.f64(double %sel1) + %cmp2 = fcmp one double %abs, 0x7FF0000000000000 + %sel2 = select i1 %cmp2, double %sel1, double %y + ret double %sel2 +} + +define double @test_fcmp_ord_commuted_select_fabs_fcmp_one_select_sitofp_nonnan(double %x, double %y, i32 %i) { +; CHECK-LABEL: @test_fcmp_ord_commuted_select_fabs_fcmp_one_select_sitofp_nonnan( +; CHECK: [[ABS:%.*]] = call double @llvm.fabs.f64(double [[X:%.*]]) +; CHECK-NEXT: [[CMP:%.*]] = fcmp one double [[ABS]], 0x7FF0000000000000 +; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], double [[X]], double [[Y:%.*]] +; CHECK-NEXT: ret double [[SEL]] +; + %nonnan = sitofp i32 %i to double + %cmp1 = fcmp ord double %nonnan, %x + %sel1 = select i1 %cmp1, double %x, double %y + %abs = call double @llvm.fabs.f64(double %sel1) + %cmp2 = fcmp one double %abs, 0x7FF0000000000000 + %sel2 = select i1 %cmp2, double %sel1, double %y + ret double %sel2 +} + +define double @test_fcmp_ord_select_fabs_rhs_fcmp_ogt_select_uitofp_nonnan(double %x, double %y, double %k, i32 %i) { +; CHECK-LABEL: @test_fcmp_ord_select_fabs_rhs_fcmp_ogt_select_uitofp_nonnan( +; CHECK: [[ABS:%.*]] = call double @llvm.fabs.f64(double [[X:%.*]]) +; CHECK-NEXT: [[CMP:%.*]] = fcmp olt double [[ABS]], [[K:%.*]] +; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], double [[X]], double [[Y:%.*]] +; CHECK-NEXT: ret double [[SEL]] +; + %nonnan = uitofp i32 %i to double + %cmp1 = fcmp ord double %x, %nonnan + %sel1 = select i1 %cmp1, double %x, double %y + %abs = call double @llvm.fabs.f64(double %sel1) + %cmp2 = fcmp ogt double %k, %abs + %sel2 = select i1 %cmp2, double %sel1, double %y + ret double %sel2 +} + +define double @test_fcmp_ord_select_fabs_fcmp_une_select_var_const(double %x) { +; CHECK-LABEL: @test_fcmp_ord_select_fabs_fcmp_une_select_var_const( +; CHECK-NEXT: [[CMP1:%.*]] = fcmp ord double [[X:%.*]], 0.000000e+00 +; CHECK-NEXT: [[SEL1:%.*]] = select i1 [[CMP1]], double [[X]], double 0.000000e+00 +; CHECK-NEXT: [[ABS:%.*]] = call double @llvm.fabs.f64(double [[SEL1]]) +; CHECK-NEXT: [[CMP2:%.*]] = fcmp une double [[ABS]], 0x7FF0000000000000 +; CHECK-NEXT: [[SEL2:%.*]] = select i1 [[CMP2]], double [[SEL1]], double 0.000000e+00 +; CHECK-NEXT: ret double [[SEL2]] +; + %cmp1 = fcmp ord double %x, 0.000000e+00 + %sel1 = select i1 %cmp1, double %x, double 0.000000e+00 + %abs = call double @llvm.fabs.f64(double %sel1) + %cmp2 = fcmp une double %abs, 0x7FF0000000000000 + %sel2 = select i1 %cmp2, double %sel1, double 0.000000e+00 + ret double %sel2 +} + +define double @test_fcmp_ord_select_fabs_fcmp_nnan_one_select_var_var(double %x, double %y) { +; CHECK-LABEL: @test_fcmp_ord_select_fabs_fcmp_nnan_one_select_var_var( +; CHECK-NEXT: [[ABS:%.*]] = call double @llvm.fabs.f64(double [[X:%.*]]) +; CHECK-NEXT: [[CMP:%.*]] = fcmp one double [[ABS]], 0x7FF0000000000000 +; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], double [[X]], double [[Y:%.*]] +; CHECK-NEXT: ret double [[SEL]] +; + %cmp1 = fcmp ord double %x, 0.000000e+00 + %sel1 = select i1 %cmp1, double %x, double %y + %abs = call double @llvm.fabs.f64(double %sel1) + %cmp2 = fcmp nnan one double %abs, 0x7FF0000000000000 + %sel2 = select i1 %cmp2, double %sel1, double %y + ret double %sel2 +} + +define double @test_fcmp_ord_select_fabs_rhs_fcmp_ogt_select_var_var(double %x, double %y, double %k) { +; CHECK-LABEL: @test_fcmp_ord_select_fabs_rhs_fcmp_ogt_select_var_var( +; CHECK-NEXT: [[ABS:%.*]] = call ninf double @llvm.fabs.f64(double [[X:%.*]]) +; CHECK-NEXT: [[CMP:%.*]] = fcmp ninf olt double [[ABS]], [[K:%.*]] +; CHECK-NEXT: [[SEL:%.*]] = select nnan i1 [[CMP]], double [[X]], double [[Y:%.*]] +; CHECK-NEXT: ret double [[SEL]] +; + %cmp1 = fcmp ord double %x, 0.000000e+00 + %sel1 = select i1 %cmp1, double %x, double %y + %abs = call ninf double @llvm.fabs.f64(double %sel1) + %cmp2 = fcmp ninf ogt double %k, %abs + %sel2 = select nnan i1 %cmp2, double %sel1, double %y + ret double %sel2 +} + +define <2 x double> @test_fcmp_ord_select_fabs_fcmp_one_select_v2f64(<2 x double> %x, <2 x double> %y) { +; CHECK-LABEL: @test_fcmp_ord_select_fabs_fcmp_one_select_v2f64( +; CHECK-NEXT: [[ABS:%.*]] = call <2 x double> @llvm.fabs.v2f64(<2 x double> [[X:%.*]]) +; CHECK-NEXT: [[CMP:%.*]] = fcmp one <2 x double> [[ABS]], splat (double 0x7FF0000000000000) +; CHECK-NEXT: [[SEL:%.*]] = select <2 x i1> [[CMP]], <2 x double> [[X]], <2 x double> [[Y:%.*]] +; CHECK-NEXT: ret <2 x double> [[SEL]] +; + %cmp1 = fcmp ord <2 x double> %x, zeroinitializer + %sel1 = select <2 x i1> %cmp1, <2 x double> %x, <2 x double> %y + %abs = call <2 x double> @llvm.fabs.v2f64(<2 x double> %sel1) + %cmp2 = fcmp one <2 x double> %abs, + %sel2 = select <2 x i1> %cmp2, <2 x double> %sel1, <2 x double> %y + ret <2 x double> %sel2 +} + +define <2 x double> @test_fcmp_ord_select_fabs_fcmp_one_select_uitofp_v2f64(<2 x double> %x, <2 x double> %y, <2 x i32> %i) { +; CHECK-LABEL: @test_fcmp_ord_select_fabs_fcmp_one_select_uitofp_v2f64( +; CHECK: [[ABS:%.*]] = call <2 x double> @llvm.fabs.v2f64(<2 x double> [[X:%.*]]) +; CHECK-NEXT: [[CMP:%.*]] = fcmp one <2 x double> [[ABS]], splat (double 0x7FF0000000000000) +; CHECK-NEXT: [[SEL:%.*]] = select <2 x i1> [[CMP]], <2 x double> [[X]], <2 x double> [[Y:%.*]] +; CHECK-NEXT: ret <2 x double> [[SEL]] +; + %nonnan = uitofp <2 x i32> %i to <2 x double> + %cmp1 = fcmp ord <2 x double> %x, %nonnan + %sel1 = select <2 x i1> %cmp1, <2 x double> %x, <2 x double> %y + %abs = call <2 x double> @llvm.fabs.v2f64(<2 x double> %sel1) + %cmp2 = fcmp one <2 x double> %abs, + %sel2 = select <2 x i1> %cmp2, <2 x double> %sel1, <2 x double> %y + ret <2 x double> %sel2 +} + +define <2 x double> @test_fcmp_ord_select_fabs_fcmp_nnan_one_select_v2f64(<2 x double> %x, <2 x double> %y) { +; CHECK-LABEL: @test_fcmp_ord_select_fabs_fcmp_nnan_one_select_v2f64( +; CHECK-NEXT: [[ABS:%.*]] = call <2 x double> @llvm.fabs.v2f64(<2 x double> [[X:%.*]]) +; CHECK-NEXT: [[CMP:%.*]] = fcmp one <2 x double> [[ABS]], splat (double 0x7FF0000000000000) +; CHECK-NEXT: [[SEL:%.*]] = select <2 x i1> [[CMP]], <2 x double> [[X]], <2 x double> [[Y:%.*]] +; CHECK-NEXT: ret <2 x double> [[SEL]] +; + %cmp1 = fcmp ord <2 x double> %x, zeroinitializer + %sel1 = select <2 x i1> %cmp1, <2 x double> %x, <2 x double> %y + %abs = call <2 x double> @llvm.fabs.v2f64(<2 x double> %sel1) + %cmp2 = fcmp nnan one <2 x double> %abs, + %sel2 = select <2 x i1> %cmp2, <2 x double> %sel1, <2 x double> %y + ret <2 x double> %sel2 +} + +define double @test_fcmp_ord_select_fabs_fcmp_fabs_only_ninf(double %x, double %y, double %k) { +; CHECK-LABEL: @test_fcmp_ord_select_fabs_fcmp_fabs_only_ninf( +; CHECK-NEXT: [[ABS:%.*]] = call ninf double @llvm.fabs.f64(double [[X:%.*]]) +; CHECK-NEXT: [[CMP:%.*]] = fcmp one double [[ABS]], [[K:%.*]] +; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], double [[X]], double [[Y:%.*]] +; CHECK-NEXT: ret double [[SEL]] +; + %cmp1 = fcmp ord double %x, 0.000000e+00 + %sel1 = select i1 %cmp1, double %x, double %y + %abs = call ninf double @llvm.fabs.f64(double %sel1) + %cmp2 = fcmp one double %abs, %k + %sel2 = select i1 %cmp2, double %sel1, double %y + ret double %sel2 +} + +define double @test_fcmp_ord_select_fabs_fcmp_cmp_only_ninf(double %x, double %y, double %k) { +; CHECK-LABEL: @test_fcmp_ord_select_fabs_fcmp_cmp_only_ninf( +; CHECK-NEXT: [[ABS:%.*]] = call double @llvm.fabs.f64(double [[X:%.*]]) +; CHECK-NEXT: [[CMP:%.*]] = fcmp ninf one double [[ABS]], [[K:%.*]] +; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], double [[X]], double [[Y:%.*]] +; CHECK-NEXT: ret double [[SEL]] +; + %cmp1 = fcmp ord double %x, 0.000000e+00 + %sel1 = select i1 %cmp1, double %x, double %y + %abs = call double @llvm.fabs.f64(double %sel1) + %cmp2 = fcmp ninf one double %abs, %k + %sel2 = select i1 %cmp2, double %sel1, double %y + ret double %sel2 +} + +define double @test_fcmp_ord_select_fabs_fcmp_intersect_drop_nnan(double %x, double %y, double %k) { +; CHECK-LABEL: @test_fcmp_ord_select_fabs_fcmp_intersect_drop_nnan( +; CHECK-NEXT: [[ABS:%.*]] = call ninf double @llvm.fabs.f64(double [[X:%.*]]) +; CHECK-NEXT: [[CMP:%.*]] = fcmp ninf one double [[ABS]], [[K:%.*]] +; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], double [[X]], double [[Y:%.*]] +; CHECK-NEXT: ret double [[SEL]] +; + %cmp1 = fcmp ord double %x, 0.000000e+00 + %sel1 = select i1 %cmp1, double %x, double %y + %abs = call nnan ninf double @llvm.fabs.f64(double %sel1) + %cmp2 = fcmp nnan ninf one double %abs, %k + %sel2 = select i1 %cmp2, double %sel1, double %y + ret double %sel2 +} + ; Make sure that we recognize the SPF correctly. define float @test_select_nnan_nsz_fcmp_olt(float %x) {