diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 1b418ae5185ee..9ed5b62f27d31 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -14400,7 +14400,8 @@ static InstructionCost canConvertToFMA(ArrayRef VL, "Can only convert to FMA for floating point types"); assert(S.isAddSubLikeOp() && "Can only convert to FMA for add/sub"); - auto CheckForContractable = [&](ArrayRef VL) { + auto CheckForContractable = [](ArrayRef VL, + const InstructionsState &S) { FastMathFlags FMF; FMF.set(); for (Value *V : VL) { @@ -14417,7 +14418,7 @@ static InstructionCost canConvertToFMA(ArrayRef VL, } return FMF.allowContract(); }; - if (!CheckForContractable(VL)) + if (!CheckForContractable(VL, S)) return InstructionCost::getInvalid(); // fmul also should be contractable InstructionsCompatibilityAnalysis Analysis(DT, DL, TTI, TLI); @@ -14429,7 +14430,7 @@ static InstructionCost canConvertToFMA(ArrayRef VL, if (OpS.isAltShuffle() || OpS.getOpcode() != Instruction::FMul) return InstructionCost::getInvalid(); - if (!CheckForContractable(Operands.front())) + if (!CheckForContractable(Operands.front(), OpS)) return InstructionCost::getInvalid(); // Compare the costs. InstructionCost FMulPlusFAddCost = 0; diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/recalc-copyable-deps-on-reorder.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/recalc-copyable-deps-on-reorder.ll index f7b2664e57b80..5bad17fa9c87d 100644 --- a/llvm/test/Transforms/SLPVectorizer/AArch64/recalc-copyable-deps-on-reorder.ll +++ b/llvm/test/Transforms/SLPVectorizer/AArch64/recalc-copyable-deps-on-reorder.ll @@ -7,89 +7,77 @@ define void @test(ptr %0, ptr %1, double %2, ptr %3) { ; CHECK-NEXT: br i1 true, label %[[VECTOR_PH:.*]], label %[[DOTPREHEADER511:.*]] ; CHECK: [[VECTOR_PH]]: ; CHECK-NEXT: [[TMP5:%.*]] = load double, ptr [[TMP0]], align 8 -; CHECK-NEXT: [[TMP6:%.*]] = fmul double [[TMP5]], 0.000000e+00 -; CHECK-NEXT: [[TMP7:%.*]] = fadd double [[TMP6]], 0.000000e+00 -; CHECK-NEXT: [[TMP8:%.*]] = fmul double [[TMP6]], 0.000000e+00 -; CHECK-NEXT: [[TMP9:%.*]] = fmul double [[TMP8]], 0.000000e+00 ; CHECK-NEXT: [[TMP10:%.*]] = fadd double [[TMP2]], 0.000000e+00 ; CHECK-NEXT: [[TMP11:%.*]] = fadd double [[TMP10]], 1.000000e+00 -; CHECK-NEXT: [[TMP12:%.*]] = fadd double [[TMP9]], [[TMP11]] -; CHECK-NEXT: [[TMP13:%.*]] = fmul double [[TMP2]], 0.000000e+00 -; CHECK-NEXT: [[TMP14:%.*]] = fadd contract double [[TMP13]], 0.000000e+00 -; CHECK-NEXT: [[TMP15:%.*]] = fmul double [[TMP6]], [[TMP2]] -; CHECK-NEXT: [[TMP16:%.*]] = fadd contract double [[TMP15]], 0.000000e+00 ; CHECK-NEXT: [[TMP17:%.*]] = load double, ptr [[TMP1]], align 8 ; CHECK-NEXT: [[TMP18:%.*]] = fmul double [[TMP17]], 0.000000e+00 +; CHECK-NEXT: [[TMP13:%.*]] = insertelement <2 x double> poison, double [[TMP2]], i64 1 +; CHECK-NEXT: [[TMP20:%.*]] = insertelement <2 x double> [[TMP13]], double [[TMP5]], i64 0 +; CHECK-NEXT: [[TMP12:%.*]] = fmul <2 x double> [[TMP20]], zeroinitializer ; CHECK-NEXT: [[TMP19:%.*]] = fmul double [[TMP18]], [[TMP2]] -; CHECK-NEXT: [[TMP20:%.*]] = fmul double [[TMP2]], [[TMP19]] -; CHECK-NEXT: [[TMP21:%.*]] = fadd double [[TMP20]], 0.000000e+00 +; CHECK-NEXT: [[TMP14:%.*]] = fadd <2 x double> [[TMP12]], zeroinitializer +; CHECK-NEXT: [[TMP15:%.*]] = insertelement <2 x double> [[TMP12]], double [[TMP19]], i64 1 +; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <2 x double> [[TMP20]], <2 x double> poison, <2 x i32> +; CHECK-NEXT: [[TMP36:%.*]] = fmul <2 x double> [[TMP15]], [[TMP16]] +; CHECK-NEXT: [[TMP49:%.*]] = fadd <2 x double> [[TMP36]], zeroinitializer ; CHECK-NEXT: [[TMP22:%.*]] = load double, ptr [[TMP0]], align 8 ; CHECK-NEXT: [[TMP23:%.*]] = fmul double [[TMP22]], 0.000000e+00 -; CHECK-NEXT: [[TMP24:%.*]] = fmul double [[TMP2]], [[TMP23]] -; CHECK-NEXT: [[TMP25:%.*]] = fadd double [[TMP24]], 0.000000e+00 -; CHECK-NEXT: [[TMP26:%.*]] = fmul double [[TMP5]], 0.000000e+00 -; CHECK-NEXT: [[TMP27:%.*]] = fmul double [[TMP26]], 0.000000e+00 -; CHECK-NEXT: [[TMP28:%.*]] = fadd double [[TMP27]], 0.000000e+00 -; CHECK-NEXT: [[TMP29:%.*]] = fmul double [[TMP2]], [[TMP17]] -; CHECK-NEXT: [[TMP30:%.*]] = fadd double [[TMP29]], 0.000000e+00 +; CHECK-NEXT: [[TMP21:%.*]] = insertelement <2 x double> [[TMP12]], double [[TMP5]], i64 1 +; CHECK-NEXT: [[TMP68:%.*]] = fmul <2 x double> [[TMP21]], zeroinitializer +; CHECK-NEXT: [[TMP69:%.*]] = fmul <2 x double> [[TMP68]], zeroinitializer +; CHECK-NEXT: [[TMP24:%.*]] = insertelement <2 x double> , double [[TMP11]], i64 0 +; CHECK-NEXT: [[TMP25:%.*]] = fadd <2 x double> [[TMP69]], [[TMP24]] +; CHECK-NEXT: [[TMP26:%.*]] = insertelement <2 x double> poison, double [[TMP2]], i64 0 +; CHECK-NEXT: [[TMP27:%.*]] = shufflevector <2 x double> [[TMP26]], <2 x double> poison, <2 x i32> zeroinitializer +; CHECK-NEXT: [[TMP28:%.*]] = insertelement <2 x double> poison, double [[TMP23]], i64 0 +; CHECK-NEXT: [[TMP29:%.*]] = insertelement <2 x double> [[TMP28]], double [[TMP17]], i64 1 +; CHECK-NEXT: [[TMP30:%.*]] = fmul <2 x double> [[TMP27]], [[TMP29]] +; CHECK-NEXT: [[TMP31:%.*]] = fadd <2 x double> [[TMP30]], zeroinitializer ; CHECK-NEXT: br label %[[DOTPREHEADER511]] ; CHECK: [[_PREHEADER511:.*:]] -; CHECK-NEXT: [[DOTSROA_9_0:%.*]] = phi double [ [[TMP30]], %[[VECTOR_PH]] ], [ 0.000000e+00, [[TMP4:%.*]] ] -; CHECK-NEXT: [[DOTSROA_7_0:%.*]] = phi double [ [[TMP28]], %[[VECTOR_PH]] ], [ 0.000000e+00, [[TMP4]] ] -; CHECK-NEXT: [[DOTSROA_6887_0:%.*]] = phi double [ [[TMP25]], %[[VECTOR_PH]] ], [ 0.000000e+00, [[TMP4]] ] -; CHECK-NEXT: [[DOTSROA_5_0:%.*]] = phi double [ [[TMP21]], %[[VECTOR_PH]] ], [ 0.000000e+00, [[TMP4]] ] -; CHECK-NEXT: [[DOTSROA_4886_0:%.*]] = phi double [ [[TMP16]], %[[VECTOR_PH]] ], [ 0.000000e+00, [[TMP4]] ] -; CHECK-NEXT: [[DOTSROA_3_0:%.*]] = phi double [ [[TMP14]], %[[VECTOR_PH]] ], [ 0.000000e+00, [[TMP4]] ] -; CHECK-NEXT: [[DOTSROA_2885_0:%.*]] = phi double [ [[TMP12]], %[[VECTOR_PH]] ], [ 0.000000e+00, [[TMP4]] ] -; CHECK-NEXT: [[DOTSROA_0884_0:%.*]] = phi double [ [[TMP7]], %[[VECTOR_PH]] ], [ 0.000000e+00, [[TMP4]] ] +; CHECK-NEXT: [[TMP32:%.*]] = phi <2 x double> [ [[TMP14]], %[[VECTOR_PH]] ], [ zeroinitializer, [[TMP4:%.*]] ] +; CHECK-NEXT: [[TMP33:%.*]] = phi <2 x double> [ [[TMP49]], %[[VECTOR_PH]] ], [ zeroinitializer, [[TMP4]] ] +; CHECK-NEXT: [[TMP34:%.*]] = phi <2 x double> [ [[TMP31]], %[[VECTOR_PH]] ], [ zeroinitializer, [[TMP4]] ] +; CHECK-NEXT: [[TMP35:%.*]] = phi <2 x double> [ [[TMP25]], %[[VECTOR_PH]] ], [ zeroinitializer, [[TMP4]] ] ; CHECK-NEXT: br i1 true, label %[[BB31:.*]], [[DOTPREHEADER498:label %.*]] ; CHECK: [[BB31]]: -; CHECK-NEXT: [[TMP32:%.*]] = getelementptr i8, ptr [[TMP3]], i64 8 -; CHECK-NEXT: [[TMP33:%.*]] = load double, ptr [[TMP3]], align 8 -; CHECK-NEXT: [[TMP34:%.*]] = fmul double [[DOTSROA_0884_0]], [[TMP33]] -; CHECK-NEXT: [[TMP35:%.*]] = load double, ptr [[TMP32]], align 8 -; CHECK-NEXT: [[TMP36:%.*]] = fmul double [[DOTSROA_2885_0]], [[TMP35]] -; CHECK-NEXT: [[TMP37:%.*]] = fadd double [[TMP34]], [[TMP36]] -; CHECK-NEXT: [[TMP38:%.*]] = fmul double [[DOTSROA_4886_0]], 0.000000e+00 -; CHECK-NEXT: [[TMP39:%.*]] = fadd double [[TMP37]], [[TMP38]] -; CHECK-NEXT: [[TMP40:%.*]] = fadd double [[TMP39]], 0.000000e+00 -; CHECK-NEXT: [[TMP41:%.*]] = fadd double [[TMP40]], 1.000000e+00 -; CHECK-NEXT: [[TMP42:%.*]] = fmul double [[DOTSROA_2885_0]], [[TMP33]] -; CHECK-NEXT: [[TMP43:%.*]] = fmul double [[DOTSROA_3_0]], [[TMP35]] -; CHECK-NEXT: [[TMP44:%.*]] = fadd double [[TMP42]], [[TMP43]] -; CHECK-NEXT: [[TMP45:%.*]] = fmul double [[DOTSROA_5_0]], 0.000000e+00 -; CHECK-NEXT: [[TMP46:%.*]] = fadd double [[TMP44]], [[TMP45]] -; CHECK-NEXT: [[TMP47:%.*]] = fadd double [[TMP46]], [[TMP2]] -; CHECK-NEXT: [[TMP48:%.*]] = fadd double [[TMP47]], 0.000000e+00 -; CHECK-NEXT: [[TMP49:%.*]] = fmul double [[DOTSROA_4886_0]], [[TMP33]] -; CHECK-NEXT: [[TMP50:%.*]] = fmul double [[DOTSROA_5_0]], [[TMP35]] -; CHECK-NEXT: [[TMP51:%.*]] = fadd double [[TMP49]], [[TMP50]] +; CHECK-NEXT: [[TMP37:%.*]] = fmul <2 x double> [[TMP33]], zeroinitializer +; CHECK-NEXT: [[TMP38:%.*]] = load <2 x double>, ptr [[TMP3]], align 8 +; CHECK-NEXT: [[TMP39:%.*]] = shufflevector <2 x double> [[TMP35]], <2 x double> poison, <2 x i32> zeroinitializer +; CHECK-NEXT: [[TMP40:%.*]] = fmul <2 x double> [[TMP39]], [[TMP38]] +; CHECK-NEXT: [[TMP41:%.*]] = shufflevector <2 x double> [[TMP40]], <2 x double> poison, <2 x i32> +; CHECK-NEXT: [[TMP42:%.*]] = fmul <2 x double> [[TMP32]], [[TMP38]] +; CHECK-NEXT: [[TMP43:%.*]] = fadd <2 x double> [[TMP41]], [[TMP42]] +; CHECK-NEXT: [[TMP44:%.*]] = fadd <2 x double> [[TMP43]], [[TMP37]] +; CHECK-NEXT: [[TMP45:%.*]] = insertelement <2 x double> , double [[TMP2]], i64 1 +; CHECK-NEXT: [[TMP46:%.*]] = fadd <2 x double> [[TMP44]], [[TMP45]] +; CHECK-NEXT: [[TMP47:%.*]] = fadd <2 x double> [[TMP46]], +; CHECK-NEXT: [[TMP48:%.*]] = extractelement <2 x double> [[TMP38]], i64 0 ; CHECK-NEXT: [[TMP52:%.*]] = load double, ptr [[TMP0]], align 8 -; CHECK-NEXT: [[TMP53:%.*]] = fmul double [[DOTSROA_6887_0]], [[TMP52]] -; CHECK-NEXT: [[TMP54:%.*]] = fadd double [[TMP51]], [[TMP53]] -; CHECK-NEXT: [[TMP55:%.*]] = fmul double [[DOTSROA_9_0]], [[TMP2]] -; CHECK-NEXT: [[TMP56:%.*]] = fadd double [[TMP54]], [[TMP55]] -; CHECK-NEXT: [[TMP57:%.*]] = fadd double [[TMP56]], 0.000000e+00 -; CHECK-NEXT: [[TMP58:%.*]] = fmul double [[DOTSROA_7_0]], [[TMP33]] -; CHECK-NEXT: [[TMP59:%.*]] = fadd double [[TMP58]], [[TMP2]] -; CHECK-NEXT: [[TMP60:%.*]] = fmul double [[DOTSROA_9_0]], 0.000000e+00 -; CHECK-NEXT: [[TMP61:%.*]] = fadd double [[TMP59]], [[TMP60]] -; CHECK-NEXT: [[TMP62:%.*]] = fadd double [[TMP61]], 0.000000e+00 -; CHECK-NEXT: [[TMP63:%.*]] = fadd double [[TMP62]], 1.000000e+00 +; CHECK-NEXT: [[TMP50:%.*]] = shufflevector <2 x double> [[TMP33]], <2 x double> [[TMP34]], <2 x i32> +; CHECK-NEXT: [[TMP51:%.*]] = shufflevector <2 x double> [[TMP38]], <2 x double> poison, <2 x i32> +; CHECK-NEXT: [[TMP70:%.*]] = insertelement <2 x double> [[TMP51]], double [[TMP2]], i64 1 +; CHECK-NEXT: [[TMP53:%.*]] = fmul <2 x double> [[TMP50]], [[TMP70]] +; CHECK-NEXT: [[TMP54:%.*]] = shufflevector <2 x double> [[TMP33]], <2 x double> [[TMP35]], <2 x i32> +; CHECK-NEXT: [[TMP55:%.*]] = shufflevector <2 x double> [[TMP38]], <2 x double> poison, <2 x i32> zeroinitializer +; CHECK-NEXT: [[TMP56:%.*]] = fmul <2 x double> [[TMP54]], [[TMP55]] +; CHECK-NEXT: [[TMP57:%.*]] = insertelement <2 x double> poison, double [[TMP2]], i64 1 +; CHECK-NEXT: [[TMP58:%.*]] = shufflevector <2 x double> [[TMP57]], <2 x double> [[TMP53]], <2 x i32> +; CHECK-NEXT: [[TMP59:%.*]] = fadd <2 x double> [[TMP56]], [[TMP58]] +; CHECK-NEXT: [[TMP60:%.*]] = insertelement <2 x double> , double [[TMP52]], i64 0 +; CHECK-NEXT: [[TMP61:%.*]] = fmul <2 x double> [[TMP34]], [[TMP60]] +; CHECK-NEXT: [[TMP62:%.*]] = fadd <2 x double> [[TMP59]], [[TMP61]] +; CHECK-NEXT: [[TMP63:%.*]] = shufflevector <2 x double> , <2 x double> [[TMP53]], <2 x i32> +; CHECK-NEXT: [[TMP64:%.*]] = fadd <2 x double> [[TMP62]], [[TMP63]] +; CHECK-NEXT: [[TMP65:%.*]] = fadd <2 x double> [[TMP64]], ; CHECK-NEXT: br [[DOTPREHEADER498]] ; CHECK: [[_PREHEADER498:.*:]] -; CHECK-NEXT: [[DOTSROA_0913_0:%.*]] = phi double [ [[TMP41]], %[[BB31]] ], [ 0.000000e+00, %[[DOTPREHEADER511]] ] -; CHECK-NEXT: [[DOTSROA_6915_0:%.*]] = phi double [ [[TMP48]], %[[BB31]] ], [ 0.000000e+00, %[[DOTPREHEADER511]] ] -; CHECK-NEXT: [[DOTSROA_10918_0:%.*]] = phi double [ [[TMP57]], %[[BB31]] ], [ 0.000000e+00, %[[DOTPREHEADER511]] ] -; CHECK-NEXT: [[DOTSROA_14921_0:%.*]] = phi double [ [[TMP63]], %[[BB31]] ], [ 0.000000e+00, %[[DOTPREHEADER511]] ] -; CHECK-NEXT: [[DOTSROA_18924_0:%.*]] = phi double [ [[TMP33]], %[[BB31]] ], [ 0.000000e+00, %[[DOTPREHEADER511]] ] -; CHECK-NEXT: store double [[DOTSROA_0913_0]], ptr [[TMP1]], align 8 -; CHECK-NEXT: [[DOTSROA_6915_0__SROA_IDX916:%.*]] = getelementptr i8, ptr [[TMP1]], i64 8 -; CHECK-NEXT: store double [[DOTSROA_6915_0]], ptr [[DOTSROA_6915_0__SROA_IDX916]], align 8 +; CHECK-NEXT: [[DOTSROA_18924_0:%.*]] = phi double [ [[TMP48]], %[[BB31]] ], [ 0.000000e+00, %[[DOTPREHEADER511]] ] +; CHECK-NEXT: [[TMP66:%.*]] = phi <2 x double> [ [[TMP47]], %[[BB31]] ], [ zeroinitializer, %[[DOTPREHEADER511]] ] +; CHECK-NEXT: [[TMP67:%.*]] = phi <2 x double> [ [[TMP65]], %[[BB31]] ], [ zeroinitializer, %[[DOTPREHEADER511]] ] +; CHECK-NEXT: store <2 x double> [[TMP66]], ptr [[TMP1]], align 8 ; CHECK-NEXT: [[DOTSROA_10918_0__SROA_IDX919:%.*]] = getelementptr i8, ptr [[TMP1]], i64 16 -; CHECK-NEXT: store double [[DOTSROA_10918_0]], ptr [[DOTSROA_10918_0__SROA_IDX919]], align 8 -; CHECK-NEXT: [[DOTSROA_14921_0__SROA_IDX922:%.*]] = getelementptr i8, ptr [[TMP1]], i64 24 -; CHECK-NEXT: store double [[DOTSROA_14921_0]], ptr [[DOTSROA_14921_0__SROA_IDX922]], align 8 +; CHECK-NEXT: store <2 x double> [[TMP67]], ptr [[DOTSROA_10918_0__SROA_IDX919]], align 8 ; CHECK-NEXT: store double [[DOTSROA_18924_0]], ptr [[TMP0]], align 8 ; CHECK-NEXT: ret void ; diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/recalc-copyable-node.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/recalc-copyable-node.ll index 0500c00537a32..c88179da10110 100644 --- a/llvm/test/Transforms/SLPVectorizer/AArch64/recalc-copyable-node.ll +++ b/llvm/test/Transforms/SLPVectorizer/AArch64/recalc-copyable-node.ll @@ -4,43 +4,38 @@ define { double, double } @test1(ptr %0, double %.unpack753, double %factor828) { ; CHECK-LABEL: define { double, double } @test1( ; CHECK-SAME: ptr [[TMP0:%.*]], double [[DOTUNPACK753:%.*]], double [[FACTOR828:%.*]]) { -; CHECK-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[TMP0]], align 8 -; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x double> , double [[DOTUNPACK753]], i64 0 -; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> poison, <2 x i32> zeroinitializer -; CHECK-NEXT: [[TMP5:%.*]] = fmul <2 x double> [[TMP3]], [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> , <2 x i32> -; CHECK-NEXT: [[TMP7:%.*]] = fmul <2 x double> [[TMP5]], [[TMP6]] -; CHECK-NEXT: [[TMP8:%.*]] = fmul <2 x double> [[TMP7]], zeroinitializer -; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <2 x double> [[TMP7]], <2 x double> [[TMP5]], <2 x i32> -; CHECK-NEXT: [[TMP10:%.*]] = insertelement <2 x double> , double [[FACTOR828]], i64 0 -; CHECK-NEXT: [[TMP11:%.*]] = fmul <2 x double> [[TMP9]], [[TMP10]] -; CHECK-NEXT: [[TMP12:%.*]] = fadd contract <2 x double> [[TMP11]], zeroinitializer -; CHECK-NEXT: [[TMP13:%.*]] = fmul <2 x double> [[TMP12]], zeroinitializer -; CHECK-NEXT: [[TMP14:%.*]] = fsub <2 x double> zeroinitializer, [[TMP13]] -; CHECK-NEXT: [[TMP15:%.*]] = shufflevector <2 x double> [[TMP7]], <2 x double> poison, <2 x i32> zeroinitializer -; CHECK-NEXT: [[TMP16:%.*]] = fmul <2 x double> [[TMP14]], [[TMP15]] -; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <2 x double> [[TMP16]], <2 x double> poison, <4 x i32> -; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <2 x double> [[TMP16]], <2 x double> poison, <4 x i32> -; CHECK-NEXT: [[TMP19:%.*]] = shufflevector <4 x double> [[TMP18]], <4 x double> , <4 x i32> -; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <2 x double> [[TMP8]], <2 x double> poison, <4 x i32> -; CHECK-NEXT: [[TMP21:%.*]] = shufflevector <4 x double> , <4 x double> [[TMP20]], <4 x i32> -; CHECK-NEXT: [[TMP22:%.*]] = shufflevector <4 x double> [[TMP21]], <4 x double> poison, <4 x i32> -; CHECK-NEXT: [[TMP23:%.*]] = fadd contract <4 x double> [[TMP19]], [[TMP22]] -; CHECK-NEXT: [[TMP24:%.*]] = insertelement <4 x double> , double [[DOTUNPACK753]], i64 0 -; CHECK-NEXT: [[TMP25:%.*]] = shufflevector <4 x double> [[TMP24]], <4 x double> poison, <4 x i32> -; CHECK-NEXT: [[TMP26:%.*]] = fmul <4 x double> [[TMP23]], [[TMP25]] -; CHECK-NEXT: [[TMP27:%.*]] = shufflevector <4 x double> [[TMP24]], <4 x double> , <4 x i32> -; CHECK-NEXT: [[TMP28:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> poison, <4 x i32> -; CHECK-NEXT: [[TMP29:%.*]] = shufflevector <4 x double> [[TMP27]], <4 x double> [[TMP28]], <4 x i32> -; CHECK-NEXT: [[TMP30:%.*]] = fadd <4 x double> [[TMP29]], [[TMP26]] -; CHECK-NEXT: [[TMP31:%.*]] = extractelement <4 x double> [[TMP30]], i64 0 +; CHECK-NEXT: [[DOTELT752:%.*]] = getelementptr i8, ptr [[TMP0]], i64 8 +; CHECK-NEXT: [[DOTUNPACK7532:%.*]] = load double, ptr [[DOTELT752]], align 8 +; CHECK-NEXT: [[DOTUNPACK1:%.*]] = load double, ptr [[TMP0]], align 8 +; CHECK-NEXT: [[TMP2:%.*]] = fmul double [[DOTUNPACK753]], [[DOTUNPACK1]] +; CHECK-NEXT: [[FACTOR8283:%.*]] = fmul double [[TMP2]], 0.000000e+00 +; CHECK-NEXT: [[TMP3:%.*]] = fmul double [[FACTOR8283]], 0.000000e+00 +; CHECK-NEXT: [[TMP4:%.*]] = fadd contract double [[TMP3]], 0.000000e+00 +; CHECK-NEXT: [[TMP5:%.*]] = fmul double [[TMP4]], [[DOTUNPACK753]] +; CHECK-NEXT: [[TMP31:%.*]] = fadd contract double [[TMP5]], 0.000000e+00 ; CHECK-NEXT: [[TMP32:%.*]] = call { double, double } @__divdc3(double [[TMP31]]) +; CHECK-NEXT: [[TMP8:%.*]] = fmul double [[DOTUNPACK1]], [[DOTUNPACK1]] +; CHECK-NEXT: [[TMP9:%.*]] = fmul double [[TMP8]], [[FACTOR828]] +; CHECK-NEXT: [[TMP10:%.*]] = fadd contract double [[TMP9]], 0.000000e+00 +; CHECK-NEXT: [[TMP11:%.*]] = fmul double [[TMP10]], 0.000000e+00 ; CHECK-NEXT: call void @llvm.stackrestore.p0(ptr null) -; CHECK-NEXT: [[TMP33:%.*]] = extractelement <4 x double> [[TMP30]], i64 1 +; CHECK-NEXT: [[TMP12:%.*]] = fsub double 0.000000e+00, [[TMP11]] +; CHECK-NEXT: [[TMP13:%.*]] = fmul double [[TMP12]], [[FACTOR8283]] +; CHECK-NEXT: [[TMP14:%.*]] = fmul double [[TMP8]], 0.000000e+00 +; CHECK-NEXT: [[TMP15:%.*]] = fadd contract double [[TMP13]], [[TMP14]] +; CHECK-NEXT: [[TMP16:%.*]] = fmul double [[TMP15]], [[DOTUNPACK753]] +; CHECK-NEXT: [[TMP33:%.*]] = fadd double [[DOTUNPACK753]], [[TMP16]] ; CHECK-NEXT: [[TMP34:%.*]] = call { double, double } @__divdc3(double [[TMP33]]) -; CHECK-NEXT: [[TMP35:%.*]] = extractelement <4 x double> [[TMP30]], i64 2 +; CHECK-NEXT: [[TMP35:%.*]] = fadd double [[DOTUNPACK1]], 0.000000e+00 ; CHECK-NEXT: [[TMP36:%.*]] = call { double, double } @__divdc3(double [[TMP35]]) -; CHECK-NEXT: [[TMP37:%.*]] = extractelement <4 x double> [[TMP30]], i64 3 +; CHECK-NEXT: [[TMP21:%.*]] = fmul double [[TMP2]], f0xC1502A98AE76C8B4 +; CHECK-NEXT: [[TMP22:%.*]] = fadd contract double [[TMP21]], 0.000000e+00 +; CHECK-NEXT: [[TMP23:%.*]] = fmul double [[TMP22]], 0.000000e+00 +; CHECK-NEXT: [[TMP24:%.*]] = fsub double 0.000000e+00, [[TMP23]] +; CHECK-NEXT: [[TMP25:%.*]] = fmul double [[TMP24]], [[FACTOR8283]] +; CHECK-NEXT: [[TMP26:%.*]] = fadd contract double [[TMP25]], [[TMP14]] +; CHECK-NEXT: [[TMP27:%.*]] = fmul double [[TMP26]], [[DOTUNPACK753]] +; CHECK-NEXT: [[TMP37:%.*]] = fadd double [[DOTUNPACK7532]], [[TMP27]] ; CHECK-NEXT: [[TMP38:%.*]] = call { double, double } @__divdc3(double [[TMP37]]) ; CHECK-NEXT: ret { double, double } [[TMP38]] ; diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/wide-store.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/wide-store.ll index 51c2040a86d64..37fc33318430b 100644 --- a/llvm/test/Transforms/SLPVectorizer/AArch64/wide-store.ll +++ b/llvm/test/Transforms/SLPVectorizer/AArch64/wide-store.ll @@ -5,20 +5,22 @@ define void @test(ptr %p, float %conv, i64 %n) { ; CHECK-LABEL: define void @test( ; CHECK-SAME: ptr [[P:%.*]], float [[CONV:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] { ; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x float> poison, float [[CONV]], i64 0 +; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <2 x float> [[TMP0]], <2 x float> poison, <2 x i32> zeroinitializer ; CHECK-NEXT: br label %[[FOR_BODY:.*]] ; CHECK: [[FOR_COND_CLEANUP:.*]]: ; CHECK-NEXT: ret void ; CHECK: [[FOR_BODY]]: ; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ] ; CHECK-NEXT: [[OUT:%.*]] = phi ptr [ [[P]], %[[ENTRY]] ], [ [[OUT_NEXT:%.*]], %[[FOR_BODY]] ] -; CHECK-NEXT: [[M2:%.*]] = fmul float [[CONV]], 0.000000e+00 -; CHECK-NEXT: [[M3:%.*]] = fmul float [[CONV]], 0.000000e+00 ; CHECK-NEXT: [[S0:%.*]] = fsub float 0.000000e+00, [[CONV]] -; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x float> , float [[S0]], i64 0 -; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> [[TMP0]], float [[M2]], i64 1 -; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x float> [[TMP1]], float [[M3]], i64 2 -; CHECK-NEXT: [[TMP3:%.*]] = fadd contract <4 x float> , [[TMP2]] -; CHECK-NEXT: store <4 x float> [[TMP3]], ptr [[OUT]], align 4 +; CHECK-NEXT: store float [[S0]], ptr [[OUT]], align 4 +; CHECK-NEXT: [[TMP2:%.*]] = fmul <2 x float> [[TMP1]], zeroinitializer +; CHECK-NEXT: [[O1:%.*]] = getelementptr inbounds i8, ptr [[OUT]], i64 4 +; CHECK-NEXT: [[TMP3:%.*]] = fadd contract <2 x float> [[TMP2]], zeroinitializer +; CHECK-NEXT: store <2 x float> [[TMP3]], ptr [[O1]], align 4 +; CHECK-NEXT: [[O3:%.*]] = getelementptr inbounds i8, ptr [[OUT]], i64 12 +; CHECK-NEXT: store float 0.000000e+00, ptr [[O3]], align 4 ; CHECK-NEXT: [[OUT_NEXT]] = getelementptr inbounds i8, ptr [[OUT]], i64 16 ; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1 ; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/fma-operand-contract-selection.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/fma-operand-contract-selection.ll new file mode 100644 index 0000000000000..9723bdd6bb369 --- /dev/null +++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/fma-operand-contract-selection.ll @@ -0,0 +1,265 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6 +; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a < %s | FileCheck %s +; RUN: opt -passes=slp-vectorizer -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -slp-threshold=15 < %s | FileCheck %s --check-prefix=THR15 + +; The fma check has to read the multiply's own fast math flags. Reading the +; add's instead makes every multiply look contractable, so a fusion that the +; backend will not perform gets paid for. mixed_contract_mul carries contract +; on one lane only and moves at the default threshold. + + +define void @nocontract_mul(ptr noalias %d, ptr noalias %a, ptr noalias %b, ptr noalias %c) { +; CHECK-LABEL: define void @nocontract_mul( +; CHECK-SAME: ptr noalias [[D:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: [[AP0:%.*]] = getelementptr inbounds float, ptr [[A]], i64 0 +; CHECK-NEXT: [[BP0:%.*]] = getelementptr inbounds float, ptr [[B]], i64 0 +; CHECK-NEXT: [[CP0:%.*]] = getelementptr inbounds float, ptr [[C]], i64 0 +; CHECK-NEXT: [[DP0:%.*]] = getelementptr inbounds float, ptr [[D]], i64 0 +; CHECK-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[AP0]], align 4 +; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[BP0]], align 4 +; CHECK-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[CP0]], align 4 +; CHECK-NEXT: [[TMP3:%.*]] = fmul <2 x float> [[TMP0]], [[TMP1]] +; CHECK-NEXT: [[TMP4:%.*]] = fadd contract <2 x float> [[TMP3]], [[TMP2]] +; CHECK-NEXT: store <2 x float> [[TMP4]], ptr [[DP0]], align 4 +; CHECK-NEXT: [[AP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 2 +; CHECK-NEXT: [[BP2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 2 +; CHECK-NEXT: [[CP2:%.*]] = getelementptr inbounds float, ptr [[C]], i64 2 +; CHECK-NEXT: [[DP2:%.*]] = getelementptr inbounds float, ptr [[D]], i64 2 +; CHECK-NEXT: [[TMP5:%.*]] = load <2 x float>, ptr [[AP2]], align 4 +; CHECK-NEXT: [[TMP6:%.*]] = load <2 x float>, ptr [[BP2]], align 4 +; CHECK-NEXT: [[TMP7:%.*]] = load <2 x float>, ptr [[CP2]], align 4 +; CHECK-NEXT: [[TMP8:%.*]] = fmul <2 x float> [[TMP5]], [[TMP6]] +; CHECK-NEXT: [[TMP9:%.*]] = fadd contract <2 x float> [[TMP8]], [[TMP7]] +; CHECK-NEXT: store <2 x float> [[TMP9]], ptr [[DP2]], align 4 +; CHECK-NEXT: ret void +; +; THR15-LABEL: define void @nocontract_mul( +; THR15-SAME: ptr noalias [[D:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0:[0-9]+]] { +; THR15-NEXT: [[ENTRY:.*:]] +; THR15-NEXT: [[AP0:%.*]] = getelementptr inbounds float, ptr [[A]], i64 0 +; THR15-NEXT: [[BP0:%.*]] = getelementptr inbounds float, ptr [[B]], i64 0 +; THR15-NEXT: [[CP0:%.*]] = getelementptr inbounds float, ptr [[C]], i64 0 +; THR15-NEXT: [[DP0:%.*]] = getelementptr inbounds float, ptr [[D]], i64 0 +; THR15-NEXT: [[TMP0:%.*]] = load <4 x float>, ptr [[AP0]], align 4 +; THR15-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[BP0]], align 4 +; THR15-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr [[CP0]], align 4 +; THR15-NEXT: [[TMP3:%.*]] = fmul <4 x float> [[TMP0]], [[TMP1]] +; THR15-NEXT: [[TMP4:%.*]] = fadd contract <4 x float> [[TMP3]], [[TMP2]] +; THR15-NEXT: store <4 x float> [[TMP4]], ptr [[DP0]], align 4 +; THR15-NEXT: ret void +; +entry: + %ap0 = getelementptr inbounds float, ptr %a, i64 0 + %bp0 = getelementptr inbounds float, ptr %b, i64 0 + %cp0 = getelementptr inbounds float, ptr %c, i64 0 + %dp0 = getelementptr inbounds float, ptr %d, i64 0 + %a0 = load float, ptr %ap0 + %b0 = load float, ptr %bp0 + %c0 = load float, ptr %cp0 + %m0 = fmul float %a0, %b0 + %r0 = fadd contract float %m0, %c0 + store float %r0, ptr %dp0 + %ap1 = getelementptr inbounds float, ptr %a, i64 1 + %bp1 = getelementptr inbounds float, ptr %b, i64 1 + %cp1 = getelementptr inbounds float, ptr %c, i64 1 + %dp1 = getelementptr inbounds float, ptr %d, i64 1 + %a1 = load float, ptr %ap1 + %b1 = load float, ptr %bp1 + %c1 = load float, ptr %cp1 + %m1 = fmul float %a1, %b1 + %r1 = fadd contract float %m1, %c1 + store float %r1, ptr %dp1 + %ap2 = getelementptr inbounds float, ptr %a, i64 2 + %bp2 = getelementptr inbounds float, ptr %b, i64 2 + %cp2 = getelementptr inbounds float, ptr %c, i64 2 + %dp2 = getelementptr inbounds float, ptr %d, i64 2 + %a2 = load float, ptr %ap2 + %b2 = load float, ptr %bp2 + %c2 = load float, ptr %cp2 + %m2 = fmul float %a2, %b2 + %r2 = fadd contract float %m2, %c2 + store float %r2, ptr %dp2 + %ap3 = getelementptr inbounds float, ptr %a, i64 3 + %bp3 = getelementptr inbounds float, ptr %b, i64 3 + %cp3 = getelementptr inbounds float, ptr %c, i64 3 + %dp3 = getelementptr inbounds float, ptr %d, i64 3 + %a3 = load float, ptr %ap3 + %b3 = load float, ptr %bp3 + %c3 = load float, ptr %cp3 + %m3 = fmul float %a3, %b3 + %r3 = fadd contract float %m3, %c3 + store float %r3, ptr %dp3 + ret void +} + +define void @contract_mul(ptr noalias %d, ptr noalias %a, ptr noalias %b, ptr noalias %c) { +; CHECK-LABEL: define void @contract_mul( +; CHECK-SAME: ptr noalias [[D:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: [[AP0:%.*]] = getelementptr inbounds float, ptr [[A]], i64 0 +; CHECK-NEXT: [[BP0:%.*]] = getelementptr inbounds float, ptr [[B]], i64 0 +; CHECK-NEXT: [[CP0:%.*]] = getelementptr inbounds float, ptr [[C]], i64 0 +; CHECK-NEXT: [[DP0:%.*]] = getelementptr inbounds float, ptr [[D]], i64 0 +; CHECK-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[AP0]], align 4 +; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[BP0]], align 4 +; CHECK-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[CP0]], align 4 +; CHECK-NEXT: [[TMP3:%.*]] = fmul contract <2 x float> [[TMP0]], [[TMP1]] +; CHECK-NEXT: [[TMP4:%.*]] = fadd contract <2 x float> [[TMP3]], [[TMP2]] +; CHECK-NEXT: store <2 x float> [[TMP4]], ptr [[DP0]], align 4 +; CHECK-NEXT: [[AP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 2 +; CHECK-NEXT: [[BP2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 2 +; CHECK-NEXT: [[CP2:%.*]] = getelementptr inbounds float, ptr [[C]], i64 2 +; CHECK-NEXT: [[DP2:%.*]] = getelementptr inbounds float, ptr [[D]], i64 2 +; CHECK-NEXT: [[TMP5:%.*]] = load <2 x float>, ptr [[AP2]], align 4 +; CHECK-NEXT: [[TMP6:%.*]] = load <2 x float>, ptr [[BP2]], align 4 +; CHECK-NEXT: [[TMP7:%.*]] = load <2 x float>, ptr [[CP2]], align 4 +; CHECK-NEXT: [[TMP8:%.*]] = fmul contract <2 x float> [[TMP5]], [[TMP6]] +; CHECK-NEXT: [[TMP9:%.*]] = fadd contract <2 x float> [[TMP8]], [[TMP7]] +; CHECK-NEXT: store <2 x float> [[TMP9]], ptr [[DP2]], align 4 +; CHECK-NEXT: ret void +; +; THR15-LABEL: define void @contract_mul( +; THR15-SAME: ptr noalias [[D:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] { +; THR15-NEXT: [[ENTRY:.*:]] +; THR15-NEXT: [[AP0:%.*]] = getelementptr inbounds float, ptr [[A]], i64 0 +; THR15-NEXT: [[BP0:%.*]] = getelementptr inbounds float, ptr [[B]], i64 0 +; THR15-NEXT: [[CP0:%.*]] = getelementptr inbounds float, ptr [[C]], i64 0 +; THR15-NEXT: [[DP0:%.*]] = getelementptr inbounds float, ptr [[D]], i64 0 +; THR15-NEXT: [[TMP0:%.*]] = load <4 x float>, ptr [[AP0]], align 4 +; THR15-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[BP0]], align 4 +; THR15-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr [[CP0]], align 4 +; THR15-NEXT: [[TMP3:%.*]] = fmul contract <4 x float> [[TMP0]], [[TMP1]] +; THR15-NEXT: [[TMP4:%.*]] = fadd contract <4 x float> [[TMP3]], [[TMP2]] +; THR15-NEXT: store <4 x float> [[TMP4]], ptr [[DP0]], align 4 +; THR15-NEXT: ret void +; +entry: + %ap0 = getelementptr inbounds float, ptr %a, i64 0 + %bp0 = getelementptr inbounds float, ptr %b, i64 0 + %cp0 = getelementptr inbounds float, ptr %c, i64 0 + %dp0 = getelementptr inbounds float, ptr %d, i64 0 + %a0 = load float, ptr %ap0 + %b0 = load float, ptr %bp0 + %c0 = load float, ptr %cp0 + %m0 = fmul contract float %a0, %b0 + %r0 = fadd contract float %m0, %c0 + store float %r0, ptr %dp0 + %ap1 = getelementptr inbounds float, ptr %a, i64 1 + %bp1 = getelementptr inbounds float, ptr %b, i64 1 + %cp1 = getelementptr inbounds float, ptr %c, i64 1 + %dp1 = getelementptr inbounds float, ptr %d, i64 1 + %a1 = load float, ptr %ap1 + %b1 = load float, ptr %bp1 + %c1 = load float, ptr %cp1 + %m1 = fmul contract float %a1, %b1 + %r1 = fadd contract float %m1, %c1 + store float %r1, ptr %dp1 + %ap2 = getelementptr inbounds float, ptr %a, i64 2 + %bp2 = getelementptr inbounds float, ptr %b, i64 2 + %cp2 = getelementptr inbounds float, ptr %c, i64 2 + %dp2 = getelementptr inbounds float, ptr %d, i64 2 + %a2 = load float, ptr %ap2 + %b2 = load float, ptr %bp2 + %c2 = load float, ptr %cp2 + %m2 = fmul contract float %a2, %b2 + %r2 = fadd contract float %m2, %c2 + store float %r2, ptr %dp2 + %ap3 = getelementptr inbounds float, ptr %a, i64 3 + %bp3 = getelementptr inbounds float, ptr %b, i64 3 + %cp3 = getelementptr inbounds float, ptr %c, i64 3 + %dp3 = getelementptr inbounds float, ptr %d, i64 3 + %a3 = load float, ptr %ap3 + %b3 = load float, ptr %bp3 + %c3 = load float, ptr %cp3 + %m3 = fmul contract float %a3, %b3 + %r3 = fadd contract float %m3, %c3 + store float %r3, ptr %dp3 + ret void +} + +define void @mixed_contract_mul(ptr noalias %d, ptr noalias %a, ptr noalias %b, ptr noalias %c) { +; CHECK-LABEL: define void @mixed_contract_mul( +; CHECK-SAME: ptr noalias [[D:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: [[AP0:%.*]] = getelementptr inbounds float, ptr [[A]], i64 0 +; CHECK-NEXT: [[BP0:%.*]] = getelementptr inbounds float, ptr [[B]], i64 0 +; CHECK-NEXT: [[CP0:%.*]] = getelementptr inbounds float, ptr [[C]], i64 0 +; CHECK-NEXT: [[DP0:%.*]] = getelementptr inbounds float, ptr [[D]], i64 0 +; CHECK-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[AP0]], align 4 +; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[BP0]], align 4 +; CHECK-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[CP0]], align 4 +; CHECK-NEXT: [[TMP3:%.*]] = fmul <2 x float> [[TMP0]], [[TMP1]] +; CHECK-NEXT: [[TMP4:%.*]] = fadd contract <2 x float> [[TMP3]], [[TMP2]] +; CHECK-NEXT: store <2 x float> [[TMP4]], ptr [[DP0]], align 4 +; CHECK-NEXT: [[AP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 2 +; CHECK-NEXT: [[BP2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 2 +; CHECK-NEXT: [[CP2:%.*]] = getelementptr inbounds float, ptr [[C]], i64 2 +; CHECK-NEXT: [[DP2:%.*]] = getelementptr inbounds float, ptr [[D]], i64 2 +; CHECK-NEXT: [[TMP5:%.*]] = load <2 x float>, ptr [[AP2]], align 4 +; CHECK-NEXT: [[TMP6:%.*]] = load <2 x float>, ptr [[BP2]], align 4 +; CHECK-NEXT: [[TMP7:%.*]] = load <2 x float>, ptr [[CP2]], align 4 +; CHECK-NEXT: [[TMP8:%.*]] = fmul <2 x float> [[TMP5]], [[TMP6]] +; CHECK-NEXT: [[TMP9:%.*]] = fadd contract <2 x float> [[TMP8]], [[TMP7]] +; CHECK-NEXT: store <2 x float> [[TMP9]], ptr [[DP2]], align 4 +; CHECK-NEXT: ret void +; +; THR15-LABEL: define void @mixed_contract_mul( +; THR15-SAME: ptr noalias [[D:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] { +; THR15-NEXT: [[ENTRY:.*:]] +; THR15-NEXT: [[AP0:%.*]] = getelementptr inbounds float, ptr [[A]], i64 0 +; THR15-NEXT: [[BP0:%.*]] = getelementptr inbounds float, ptr [[B]], i64 0 +; THR15-NEXT: [[CP0:%.*]] = getelementptr inbounds float, ptr [[C]], i64 0 +; THR15-NEXT: [[DP0:%.*]] = getelementptr inbounds float, ptr [[D]], i64 0 +; THR15-NEXT: [[TMP0:%.*]] = load <4 x float>, ptr [[AP0]], align 4 +; THR15-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[BP0]], align 4 +; THR15-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr [[CP0]], align 4 +; THR15-NEXT: [[TMP3:%.*]] = fmul <4 x float> [[TMP0]], [[TMP1]] +; THR15-NEXT: [[TMP4:%.*]] = fadd contract <4 x float> [[TMP3]], [[TMP2]] +; THR15-NEXT: store <4 x float> [[TMP4]], ptr [[DP0]], align 4 +; THR15-NEXT: ret void +; +entry: + %ap0 = getelementptr inbounds float, ptr %a, i64 0 + %bp0 = getelementptr inbounds float, ptr %b, i64 0 + %cp0 = getelementptr inbounds float, ptr %c, i64 0 + %dp0 = getelementptr inbounds float, ptr %d, i64 0 + %a0 = load float, ptr %ap0 + %b0 = load float, ptr %bp0 + %c0 = load float, ptr %cp0 + %m0 = fmul contract float %a0, %b0 + %r0 = fadd contract float %m0, %c0 + store float %r0, ptr %dp0 + %ap1 = getelementptr inbounds float, ptr %a, i64 1 + %bp1 = getelementptr inbounds float, ptr %b, i64 1 + %cp1 = getelementptr inbounds float, ptr %c, i64 1 + %dp1 = getelementptr inbounds float, ptr %d, i64 1 + %a1 = load float, ptr %ap1 + %b1 = load float, ptr %bp1 + %c1 = load float, ptr %cp1 + %m1 = fmul float %a1, %b1 + %r1 = fadd contract float %m1, %c1 + store float %r1, ptr %dp1 + %ap2 = getelementptr inbounds float, ptr %a, i64 2 + %bp2 = getelementptr inbounds float, ptr %b, i64 2 + %cp2 = getelementptr inbounds float, ptr %c, i64 2 + %dp2 = getelementptr inbounds float, ptr %d, i64 2 + %a2 = load float, ptr %ap2 + %b2 = load float, ptr %bp2 + %c2 = load float, ptr %cp2 + %m2 = fmul float %a2, %b2 + %r2 = fadd contract float %m2, %c2 + store float %r2, ptr %dp2 + %ap3 = getelementptr inbounds float, ptr %a, i64 3 + %bp3 = getelementptr inbounds float, ptr %b, i64 3 + %cp3 = getelementptr inbounds float, ptr %c, i64 3 + %dp3 = getelementptr inbounds float, ptr %d, i64 3 + %a3 = load float, ptr %ap3 + %b3 = load float, ptr %bp3 + %c3 = load float, ptr %cp3 + %m3 = fmul float %a3, %b3 + %r3 = fadd contract float %m3, %c3 + store float %r3, ptr %dp3 + ret void +} + diff --git a/llvm/test/Transforms/SLPVectorizer/X86/dot-product.ll b/llvm/test/Transforms/SLPVectorizer/X86/dot-product.ll index c499aea3819c7..ef1da6fbd8ca2 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/dot-product.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/dot-product.ll @@ -363,14 +363,11 @@ define double @dot2f64_fast(ptr dereferenceable(16) %ptrx, ptr dereferenceable(1 ; AVX-NEXT: ret double [[DOT01]] ; ; AVX2-LABEL: @dot2f64_fast( -; AVX2-NEXT: [[PTRX1:%.*]] = getelementptr inbounds double, ptr [[PTRX:%.*]], i64 1 -; AVX2-NEXT: [[PTRY1:%.*]] = getelementptr inbounds double, ptr [[PTRY:%.*]], i64 1 -; AVX2-NEXT: [[X0:%.*]] = load double, ptr [[PTRX]], align 4 -; AVX2-NEXT: [[Y0:%.*]] = load double, ptr [[PTRY]], align 4 -; AVX2-NEXT: [[X1:%.*]] = load double, ptr [[PTRX1]], align 4 -; AVX2-NEXT: [[Y1:%.*]] = load double, ptr [[PTRY1]], align 4 -; AVX2-NEXT: [[MUL0:%.*]] = fmul double [[X0]], [[Y0]] -; AVX2-NEXT: [[MUL1:%.*]] = fmul double [[X1]], [[Y1]] +; AVX2-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[PTRX:%.*]], align 4 +; AVX2-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[PTRY:%.*]], align 4 +; AVX2-NEXT: [[TMP3:%.*]] = fmul <2 x double> [[TMP1]], [[TMP2]] +; AVX2-NEXT: [[MUL0:%.*]] = extractelement <2 x double> [[TMP3]], i64 0 +; AVX2-NEXT: [[MUL1:%.*]] = extractelement <2 x double> [[TMP3]], i64 1 ; AVX2-NEXT: [[DOT01:%.*]] = fadd fast double [[MUL0]], [[MUL1]] ; AVX2-NEXT: ret double [[DOT01]] ; @@ -413,14 +410,11 @@ define float @dot2f32_fast(ptr dereferenceable(16) %ptrx, ptr dereferenceable(16 ; AVX-NEXT: ret float [[DOT01]] ; ; AVX2-LABEL: @dot2f32_fast( -; AVX2-NEXT: [[PTRX1:%.*]] = getelementptr inbounds float, ptr [[PTRX:%.*]], i64 1 -; AVX2-NEXT: [[PTRY1:%.*]] = getelementptr inbounds float, ptr [[PTRY:%.*]], i64 1 -; AVX2-NEXT: [[X0:%.*]] = load float, ptr [[PTRX]], align 4 -; AVX2-NEXT: [[Y0:%.*]] = load float, ptr [[PTRY]], align 4 -; AVX2-NEXT: [[X1:%.*]] = load float, ptr [[PTRX1]], align 4 -; AVX2-NEXT: [[Y1:%.*]] = load float, ptr [[PTRY1]], align 4 -; AVX2-NEXT: [[MUL0:%.*]] = fmul float [[X0]], [[Y0]] -; AVX2-NEXT: [[MUL1:%.*]] = fmul float [[X1]], [[Y1]] +; AVX2-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[PTRX:%.*]], align 4 +; AVX2-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[PTRY:%.*]], align 4 +; AVX2-NEXT: [[TMP3:%.*]] = fmul <2 x float> [[TMP1]], [[TMP2]] +; AVX2-NEXT: [[MUL0:%.*]] = extractelement <2 x float> [[TMP3]], i64 0 +; AVX2-NEXT: [[MUL1:%.*]] = extractelement <2 x float> [[TMP3]], i64 1 ; AVX2-NEXT: [[DOT01:%.*]] = fadd fast float [[MUL0]], [[MUL1]] ; AVX2-NEXT: ret float [[DOT01]] ;