Skip to content

Commit

Permalink
[SelectionDAG] Add legalizations for VECREDUCE_SEQ_FMUL
Browse files Browse the repository at this point in the history 
Hook up legalizations for VECREDUCE_SEQ_FMUL. This is following up on the VECREDUCE_SEQ_FADD work from D90247.

Differential Revision: https://reviews.llvm.org/D90644
  • Loading branch information
Cameron McInally committed Nov 4, 2020
1 parent 5dc4754 commit c126eb7
Show file tree
Hide file tree
Showing 10 changed files with 96 additions and 124 deletions.
3 changes: 3 additions & 0 deletions llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
View file
Expand Up @@ -141,6 +141,7 @@ void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
R = SoftenFloatRes_VECREDUCE(N);
break;
case ISD::VECREDUCE_SEQ_FADD:
case ISD::VECREDUCE_SEQ_FMUL:
R = SoftenFloatRes_VECREDUCE_SEQ(N);
break;
}
Expand Down Expand Up @@ -2261,6 +2262,7 @@ void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) {
R = PromoteFloatRes_VECREDUCE(N);
break;
case ISD::VECREDUCE_SEQ_FADD:
case ISD::VECREDUCE_SEQ_FMUL:
R = PromoteFloatRes_VECREDUCE_SEQ(N);
break;
}
Expand Down Expand Up @@ -2623,6 +2625,7 @@ void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
R = SoftPromoteHalfRes_VECREDUCE(N);
break;
case ISD::VECREDUCE_SEQ_FADD:
case ISD::VECREDUCE_SEQ_FMUL:
R = SoftPromoteHalfRes_VECREDUCE_SEQ(N);
break;
}
Expand Down
2 changes: 2 additions & 0 deletions llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
View file
Expand Up @@ -490,6 +490,7 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
Node->getOperand(0).getValueType());
break;
case ISD::VECREDUCE_SEQ_FADD:
case ISD::VECREDUCE_SEQ_FMUL:
Action = TLI.getOperationAction(Node->getOpcode(),
Node->getOperand(1).getValueType());
break;
Expand Down Expand Up @@ -875,6 +876,7 @@ void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
Results.push_back(TLI.expandVecReduce(Node, DAG));
return;
case ISD::VECREDUCE_SEQ_FADD:
case ISD::VECREDUCE_SEQ_FMUL:
Results.push_back(TLI.expandVecReduceSeq(Node, DAG));
return;
case ISD::SREM:
Expand Down
3 changes: 3 additions & 0 deletions llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
View file
Expand Up @@ -624,6 +624,7 @@ bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
Res = ScalarizeVecOp_VECREDUCE(N);
break;
case ISD::VECREDUCE_SEQ_FADD:
case ISD::VECREDUCE_SEQ_FMUL:
Res = ScalarizeVecOp_VECREDUCE_SEQ(N);
break;
}
Expand Down Expand Up @@ -2090,6 +2091,7 @@ bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
Res = SplitVecOp_VECREDUCE(N, OpNo);
break;
case ISD::VECREDUCE_SEQ_FADD:
case ISD::VECREDUCE_SEQ_FMUL:
Res = SplitVecOp_VECREDUCE_SEQ(N);
break;
}
Expand Down Expand Up @@ -4358,6 +4360,7 @@ bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
Res = WidenVecOp_VECREDUCE(N);
break;
case ISD::VECREDUCE_SEQ_FADD:
case ISD::VECREDUCE_SEQ_FMUL:
Res = WidenVecOp_VECREDUCE_SEQ(N);
break;
}
Expand Down
1 change: 1 addition & 0 deletions llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
View file
Expand Up @@ -341,6 +341,7 @@ ISD::NodeType ISD::getVecReduceBaseOpcode(unsigned VecReduceOpcode) {
case ISD::VECREDUCE_SEQ_FADD:
return ISD::FADD;
case ISD::VECREDUCE_FMUL:
case ISD::VECREDUCE_SEQ_FMUL:
return ISD::FMUL;
case ISD::VECREDUCE_ADD:
return ISD::ADD;
Expand Down
1 change: 1 addition & 0 deletions llvm/lib/CodeGen/TargetLoweringBase.cpp
View file
Expand Up @@ -734,6 +734,7 @@ void TargetLoweringBase::initActions() {
setOperationAction(ISD::VECREDUCE_FMAX, VT, Expand);
setOperationAction(ISD::VECREDUCE_FMIN, VT, Expand);
setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Expand);
setOperationAction(ISD::VECREDUCE_SEQ_FMUL, VT, Expand);
}

// Most targets ignore the @llvm.prefetch intrinsic.
Expand Down
12 changes: 1 addition & 11 deletions llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
View file
Expand Up @@ -221,17 +221,7 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
shouldConsiderAddressTypePromotion(const Instruction &I,
bool &AllowPromotionWithoutCommonHeader);

bool shouldExpandReduction(const IntrinsicInst *II) const {
switch (II->getIntrinsicID()) {
case Intrinsic::vector_reduce_fmul:
// We don't have legalization support for ordered FMUL reductions.
return !II->getFastMathFlags().allowReassoc();

default:
// Don't expand anything else, let legalization deal with it.
return false;
}
}
bool shouldExpandReduction(const IntrinsicInst *II) const { return false; }

unsigned getGISelRematGlobalCost() const {
return 2;
Expand Down
11 changes: 1 addition & 10 deletions llvm/lib/Target/ARM/ARMTargetTransformInfo.h
View file
Expand Up @@ -195,16 +195,7 @@ class ARMTTIImpl : public BasicTTIImplBase<ARMTTIImpl> {
bool preferPredicatedReductionSelect(unsigned Opcode, Type *Ty,
TTI::ReductionFlags Flags) const;

bool shouldExpandReduction(const IntrinsicInst *II) const {
switch (II->getIntrinsicID()) {
case Intrinsic::vector_reduce_fmul:
// We don't have legalization support for ordered FMUL reductions.
return !II->getFastMathFlags().allowReassoc();
default:
// Don't expand anything else, let legalization deal with it.
return false;
}
}
bool shouldExpandReduction(const IntrinsicInst *II) const { return false; }

int getCFInstrCost(unsigned Opcode,
TTI::TargetCostKind CostKind);
Expand Down
47 changes: 12 additions & 35 deletions llvm/test/CodeGen/AArch64/vecreduce-fmul-legalization-strict.ll
View file
Expand Up @@ -15,85 +15,62 @@ declare float @llvm.vector.reduce.fmul.f32.v16f32(float, <16 x float>)
define half @test_v1f16(<1 x half> %a) nounwind {
; CHECK-LABEL: test_v1f16:
; CHECK: // %bb.0:
; CHECK-NEXT: fcvt s0, h0
; CHECK-NEXT: fmov s1, wzr
; CHECK-NEXT: fmul s0, s0, s1
; CHECK-NEXT: fcvt h0, s0
; CHECK-NEXT: ret
%b = call half @llvm.vector.reduce.fmul.f16.v1f16(half 0.0, <1 x half> %a)
%b = call half @llvm.vector.reduce.fmul.f16.v1f16(half 1.0, <1 x half> %a)
ret half %b
}

define float @test_v1f32(<1 x float> %a) nounwind {
; CHECK-LABEL: test_v1f32:
; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
; CHECK-NEXT: fmov s1, wzr
; CHECK-NEXT: fmul s0, s1, v0.s[0]
; CHECK-NEXT: // kill: def $s0 killed $s0 killed $q0
; CHECK-NEXT: ret
%b = call float @llvm.vector.reduce.fmul.f32.v1f32(float 0.0, <1 x float> %a)
%b = call float @llvm.vector.reduce.fmul.f32.v1f32(float 1.0, <1 x float> %a)
ret float %b
}

define double @test_v1f64(<1 x double> %a) nounwind {
; CHECK-LABEL: test_v1f64:
; CHECK: // %bb.0:
; CHECK-NEXT: fmov d1, xzr
; CHECK-NEXT: fmul d0, d0, d1
; CHECK-NEXT: ret
%b = call double @llvm.vector.reduce.fmul.f64.v1f64(double 0.0, <1 x double> %a)
%b = call double @llvm.vector.reduce.fmul.f64.v1f64(double 1.0, <1 x double> %a)
ret double %b
}

define fp128 @test_v1f128(<1 x fp128> %a) nounwind {
; CHECK-LABEL: test_v1f128:
; CHECK: // %bb.0:
; CHECK-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
; CHECK-NEXT: adrp x8, .LCPI3_0
; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI3_0]
; CHECK-NEXT: bl __multf3
; CHECK-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
%b = call fp128 @llvm.vector.reduce.fmul.f128.v1f128(fp128 zeroinitializer, <1 x fp128> %a)
%b = call fp128 @llvm.vector.reduce.fmul.f128.v1f128(fp128 0xL00000000000000003fff00000000000000, <1 x fp128> %a)
ret fp128 %b
}

define float @test_v3f32(<3 x float> %a) nounwind {
; CHECK-LABEL: test_v3f32:
; CHECK: // %bb.0:
; CHECK-NEXT: fmov s1, wzr
; CHECK-NEXT: fmul s1, s1, v0.s[0]
; CHECK-NEXT: fmul s1, s1, v0.s[1]
; CHECK-NEXT: fmul s1, s0, v0.s[1]
; CHECK-NEXT: fmul s0, s1, v0.s[2]
; CHECK-NEXT: ret
%b = call float @llvm.vector.reduce.fmul.f32.v3f32(float 0.0, <3 x float> %a)
%b = call float @llvm.vector.reduce.fmul.f32.v3f32(float 1.0, <3 x float> %a)
ret float %b
}

define fp128 @test_v2f128(<2 x fp128> %a) nounwind {
; CHECK-LABEL: test_v2f128:
; CHECK: // %bb.0:
; CHECK-NEXT: sub sp, sp, #32 // =32
; CHECK-NEXT: adrp x8, .LCPI5_0
; CHECK-NEXT: str q1, [sp] // 16-byte Folded Spill
; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI5_0]
; CHECK-NEXT: str x30, [sp, #16] // 8-byte Folded Spill
; CHECK-NEXT: bl __multf3
; CHECK-NEXT: ldr q1, [sp] // 16-byte Folded Reload
; CHECK-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
; CHECK-NEXT: bl __multf3
; CHECK-NEXT: ldr x30, [sp, #16] // 8-byte Folded Reload
; CHECK-NEXT: add sp, sp, #32 // =32
; CHECK-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
; CHECK-NEXT: ret
%b = call fp128 @llvm.vector.reduce.fmul.f128.v2f128(fp128 zeroinitializer, <2 x fp128> %a)
%b = call fp128 @llvm.vector.reduce.fmul.f128.v2f128(fp128 0xL00000000000000003fff00000000000000, <2 x fp128> %a)
ret fp128 %b
}

define float @test_v16f32(<16 x float> %a) nounwind {
; CHECK-LABEL: test_v16f32:
; CHECK: // %bb.0:
; CHECK-NEXT: fmov s4, wzr
; CHECK-NEXT: fmul s4, s4, v0.s[0]
; CHECK-NEXT: fmul s4, s4, v0.s[1]
; CHECK-NEXT: fmul s4, s0, v0.s[1]
; CHECK-NEXT: fmul s4, s4, v0.s[2]
; CHECK-NEXT: fmul s0, s4, v0.s[3]
; CHECK-NEXT: fmul s0, s0, v1.s[0]
Expand All @@ -109,6 +86,6 @@ define float @test_v16f32(<16 x float> %a) nounwind {
; CHECK-NEXT: fmul s0, s0, v3.s[2]
; CHECK-NEXT: fmul s0, s0, v3.s[3]
; CHECK-NEXT: ret
%b = call float @llvm.vector.reduce.fmul.f32.v16f32(float 0.0, <16 x float> %a)
%b = call float @llvm.vector.reduce.fmul.f32.v16f32(float 1.0, <16 x float> %a)
ret float %b
}
53 changes: 53 additions & 0 deletions llvm/test/CodeGen/ARM/vecreduce-fmul-legalization-soft-float.ll
View file
Expand Up @@ -59,6 +59,24 @@ define float @test_v4f32(<4 x float> %a) nounwind {
ret float %b
}

define float @test_v4f32_strict(<4 x float> %a) nounwind {
; CHECK-LABEL: test_v4f32_strict:
; CHECK: @ %bb.0:
; CHECK-NEXT: .save {r4, r5, r11, lr}
; CHECK-NEXT: push {r4, r5, r11, lr}
; CHECK-NEXT: mov r4, r3
; CHECK-NEXT: mov r5, r2
; CHECK-NEXT: bl __aeabi_fmul
; CHECK-NEXT: mov r1, r5
; CHECK-NEXT: bl __aeabi_fmul
; CHECK-NEXT: mov r1, r4
; CHECK-NEXT: bl __aeabi_fmul
; CHECK-NEXT: pop {r4, r5, r11, lr}
; CHECK-NEXT: mov pc, lr
%b = call float @llvm.vector.reduce.fmul.f32.v4f32(float 1.0, <4 x float> %a)
ret float %b
}

define double @test_v2f64(<2 x double> %a) nounwind {
; CHECK-LABEL: test_v2f64:
; CHECK: @ %bb.0:
Expand All @@ -71,6 +89,18 @@ define double @test_v2f64(<2 x double> %a) nounwind {
ret double %b
}

define double @test_v2f64_strict(<2 x double> %a) nounwind {
; CHECK-LABEL: test_v2f64_strict:
; CHECK: @ %bb.0:
; CHECK-NEXT: .save {r11, lr}
; CHECK-NEXT: push {r11, lr}
; CHECK-NEXT: bl __aeabi_dmul
; CHECK-NEXT: pop {r11, lr}
; CHECK-NEXT: mov pc, lr
%b = call double @llvm.vector.reduce.fmul.f64.v2f64(double 1.0, <2 x double> %a)
ret double %b
}

define fp128 @test_v2f128(<2 x fp128> %a) nounwind {
; CHECK-LABEL: test_v2f128:
; CHECK: @ %bb.0:
Expand All @@ -93,3 +123,26 @@ define fp128 @test_v2f128(<2 x fp128> %a) nounwind {
%b = call fast fp128 @llvm.vector.reduce.fmul.f128.v2f128(fp128 0xL00000000000000003fff00000000000000, <2 x fp128> %a)
ret fp128 %b
}

define fp128 @test_v2f128_strict(<2 x fp128> %a) nounwind {
; CHECK-LABEL: test_v2f128_strict:
; CHECK: @ %bb.0:
; CHECK-NEXT: .save {r11, lr}
; CHECK-NEXT: push {r11, lr}
; CHECK-NEXT: .pad #16
; CHECK-NEXT: sub sp, sp, #16
; CHECK-NEXT: ldr r12, [sp, #36]
; CHECK-NEXT: str r12, [sp, #12]
; CHECK-NEXT: ldr r12, [sp, #32]
; CHECK-NEXT: str r12, [sp, #8]
; CHECK-NEXT: ldr r12, [sp, #28]
; CHECK-NEXT: str r12, [sp, #4]
; CHECK-NEXT: ldr r12, [sp, #24]
; CHECK-NEXT: str r12, [sp]
; CHECK-NEXT: bl __multf3
; CHECK-NEXT: add sp, sp, #16
; CHECK-NEXT: pop {r11, lr}
; CHECK-NEXT: mov pc, lr
%b = call fp128 @llvm.vector.reduce.fmul.f128.v2f128(fp128 0xL00000000000000003fff00000000000000, <2 x fp128> %a)
ret fp128 %b
}

0 comments on commit c126eb7

Please sign in to comment.