Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[AMDGPU] make v32i16/v32f16 legal #70484

Merged
merged 3 commits into from
Oct 27, 2023
Merged

[AMDGPU] make v32i16/v32f16 legal #70484

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f2dbe39
[AMDGPU] Make v32f16/v32i16 legal
changpeng Oct 27, 2023
940cb66
Merge branch 'llvm:main' into main
changpeng Oct 27, 2023
57b3c57
[AMDGPU] Make v32f16/v32i16 legal
changpeng Oct 27, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -384,7 +384,7 @@ AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM,
MVT::v12f32, MVT::v16f16, MVT::v16i16, MVT::v16f32, MVT::v16i32,
MVT::v32f32, MVT::v32i32, MVT::v2f64, MVT::v2i64, MVT::v3f64,
MVT::v3i64, MVT::v4f64, MVT::v4i64, MVT::v8f64, MVT::v8i64,
MVT::v16f64, MVT::v16i64},
MVT::v16f64, MVT::v16i64, MVT::v32i16, MVT::v32f16},
Custom);

setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand);
Expand Down
102 changes: 78 additions & 24 deletions llvm/lib/Target/AMDGPU/SIISelLowering.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,8 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
addRegisterClass(MVT::v8f16, &AMDGPU::SGPR_128RegClass);
addRegisterClass(MVT::v16i16, &AMDGPU::SGPR_256RegClass);
addRegisterClass(MVT::v16f16, &AMDGPU::SGPR_256RegClass);
addRegisterClass(MVT::v32i16, &AMDGPU::SGPR_512RegClass);
addRegisterClass(MVT::v32f16, &AMDGPU::SGPR_512RegClass);
}

addRegisterClass(MVT::v32i32, &AMDGPU::VReg_1024RegClass);
Expand Down Expand Up @@ -269,13 +271,13 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
// We only support LOAD/STORE and vector manipulation ops for vectors
// with > 4 elements.
for (MVT VT :
{MVT::v8i32, MVT::v8f32, MVT::v9i32, MVT::v9f32, MVT::v10i32,
MVT::v10f32, MVT::v11i32, MVT::v11f32, MVT::v12i32, MVT::v12f32,
MVT::v16i32, MVT::v16f32, MVT::v2i64, MVT::v2f64, MVT::v4i16,
MVT::v4f16, MVT::v3i64, MVT::v3f64, MVT::v6i32, MVT::v6f32,
MVT::v4i64, MVT::v4f64, MVT::v8i64, MVT::v8f64, MVT::v8i16,
MVT::v8f16, MVT::v16i16, MVT::v16f16, MVT::v16i64, MVT::v16f64,
MVT::v32i32, MVT::v32f32}) {
{MVT::v8i32, MVT::v8f32, MVT::v9i32, MVT::v9f32, MVT::v10i32,
MVT::v10f32, MVT::v11i32, MVT::v11f32, MVT::v12i32, MVT::v12f32,
MVT::v16i32, MVT::v16f32, MVT::v2i64, MVT::v2f64, MVT::v4i16,
MVT::v4f16, MVT::v3i64, MVT::v3f64, MVT::v6i32, MVT::v6f32,
MVT::v4i64, MVT::v4f64, MVT::v8i64, MVT::v8f64, MVT::v8i16,
MVT::v8f16, MVT::v16i16, MVT::v16f16, MVT::v16i64, MVT::v16f64,
MVT::v32i32, MVT::v32f32, MVT::v32i16, MVT::v32f16}) {
for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op) {
switch (Op) {
case ISD::LOAD:
Expand Down Expand Up @@ -553,8 +555,9 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
if (STI.hasMadF16())
setOperationAction(ISD::FMAD, MVT::f16, Legal);

for (MVT VT : {MVT::v2i16, MVT::v2f16, MVT::v4i16, MVT::v4f16, MVT::v8i16,
MVT::v8f16, MVT::v16i16, MVT::v16f16}) {
for (MVT VT :
{MVT::v2i16, MVT::v2f16, MVT::v4i16, MVT::v4f16, MVT::v8i16,
MVT::v8f16, MVT::v16i16, MVT::v16f16, MVT::v32i16, MVT::v32f16}) {
for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op) {
switch (Op) {
case ISD::LOAD:
Expand Down Expand Up @@ -640,6 +643,16 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
setOperationAction(ISD::STORE, MVT::v16f16, Promote);
AddPromotedToType(ISD::STORE, MVT::v16f16, MVT::v8i32);

setOperationAction(ISD::LOAD, MVT::v32i16, Promote);
AddPromotedToType(ISD::LOAD, MVT::v32i16, MVT::v16i32);
setOperationAction(ISD::LOAD, MVT::v32f16, Promote);
AddPromotedToType(ISD::LOAD, MVT::v32f16, MVT::v16i32);

setOperationAction(ISD::STORE, MVT::v32i16, Promote);
AddPromotedToType(ISD::STORE, MVT::v32i16, MVT::v16i32);
setOperationAction(ISD::STORE, MVT::v32f16, Promote);
AddPromotedToType(ISD::STORE, MVT::v32f16, MVT::v16i32);

setOperationAction({ISD::ANY_EXTEND, ISD::ZERO_EXTEND, ISD::SIGN_EXTEND},
MVT::v2i32, Expand);
setOperationAction(ISD::FP_EXTEND, MVT::v2f32, Expand);
Expand All @@ -662,12 +675,15 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
setOperationAction({ISD::FMAXNUM_IEEE, ISD::FMINNUM_IEEE}, MVT::f16, Legal);

setOperationAction({ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE},
{MVT::v4f16, MVT::v8f16, MVT::v16f16}, Custom);
{MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16},
Custom);

setOperationAction({ISD::FMINNUM, ISD::FMAXNUM},
{MVT::v4f16, MVT::v8f16, MVT::v16f16}, Expand);
{MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16},
Expand);

for (MVT Vec16 : {MVT::v8i16, MVT::v8f16, MVT::v16i16, MVT::v16f16}) {
for (MVT Vec16 : {MVT::v8i16, MVT::v8f16, MVT::v16i16, MVT::v16f16,
MVT::v32i16, MVT::v32f16}) {
setOperationAction(
{ISD::BUILD_VECTOR, ISD::EXTRACT_VECTOR_ELT, ISD::SCALAR_TO_VECTOR},
Vec16, Custom);
Expand All @@ -690,18 +706,18 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,

setOperationAction(ISD::VECTOR_SHUFFLE,
{MVT::v4f16, MVT::v4i16, MVT::v8f16, MVT::v8i16,
MVT::v16f16, MVT::v16i16},
MVT::v16f16, MVT::v16i16, MVT::v32f16, MVT::v32i16},
Custom);

for (MVT VT : {MVT::v4i16, MVT::v8i16, MVT::v16i16})
for (MVT VT : {MVT::v4i16, MVT::v8i16, MVT::v16i16, MVT::v32i16})
// Split vector operations.
setOperationAction({ISD::SHL, ISD::SRA, ISD::SRL, ISD::ADD, ISD::SUB,
ISD::MUL, ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX,
ISD::UADDSAT, ISD::SADDSAT, ISD::USUBSAT,
ISD::SSUBSAT},
VT, Custom);

for (MVT VT : {MVT::v4f16, MVT::v8f16, MVT::v16f16})
for (MVT VT : {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16})
// Split vector operations.
setOperationAction({ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FCANONICALIZE},
VT, Custom);
Expand Down Expand Up @@ -737,7 +753,8 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,

setOperationAction(ISD::SELECT,
{MVT::v4i16, MVT::v4f16, MVT::v2i8, MVT::v4i8, MVT::v8i8,
MVT::v8i16, MVT::v8f16, MVT::v16i16, MVT::v16f16},
MVT::v8i16, MVT::v8f16, MVT::v16i16, MVT::v16f16,
MVT::v32i16, MVT::v32f16},
Custom);

setOperationAction({ISD::SMULO, ISD::UMULO}, MVT::i64, Custom);
Expand Down Expand Up @@ -5107,7 +5124,7 @@ SDValue SITargetLowering::splitUnaryVectorOp(SDValue Op,
assert(VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4f32 ||
VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i16 ||
VT == MVT::v16f16 || VT == MVT::v8f32 || VT == MVT::v16f32 ||
VT == MVT::v32f32);
VT == MVT::v32f32 || VT == MVT::v32i16 || VT == MVT::v32f16);

SDValue Lo, Hi;
std::tie(Lo, Hi) = DAG.SplitVectorOperand(Op.getNode(), 0);
Expand All @@ -5130,7 +5147,7 @@ SDValue SITargetLowering::splitBinaryVectorOp(SDValue Op,
assert(VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4f32 ||
VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i16 ||
VT == MVT::v16f16 || VT == MVT::v8f32 || VT == MVT::v16f32 ||
VT == MVT::v32f32);
VT == MVT::v32f32 || VT == MVT::v32i16 || VT == MVT::v32f16);

SDValue Lo0, Hi0;
std::tie(Lo0, Hi0) = DAG.SplitVectorOperand(Op.getNode(), 0);
Expand Down Expand Up @@ -5897,7 +5914,8 @@ SDValue SITargetLowering::lowerFMINNUM_FMAXNUM(SDValue Op,
if (IsIEEEMode)
return expandFMINNUM_FMAXNUM(Op.getNode(), DAG);

if (VT == MVT::v4f16 || VT == MVT::v8f16 || VT == MVT::v16f16)
if (VT == MVT::v4f16 || VT == MVT::v8f16 || VT == MVT::v16f16 ||
VT == MVT::v16f16)
return splitBinaryVectorOp(Op, DAG);
return Op;
}
Expand Down Expand Up @@ -6415,7 +6433,7 @@ SDValue SITargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
if (SDValue Combined = performExtractVectorEltCombine(Op.getNode(), DCI))
return Combined;

if (VecSize == 128 || VecSize == 256) {
if (VecSize == 128 || VecSize == 256 || VecSize == 512) {
SDValue Lo, Hi;
EVT LoVT, HiVT;
std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VecVT);
Expand All @@ -6428,9 +6446,7 @@ SDValue SITargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
Hi = DAG.getBitcast(HiVT,
DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i64, V2,
DAG.getConstant(1, SL, MVT::i32)));
} else {
assert(VecSize == 256);

} else if (VecSize == 256) {
SDValue V2 = DAG.getBitcast(MVT::v4i64, Vec);
SDValue Parts[4];
for (unsigned P = 0; P < 4; ++P) {
Expand All @@ -6442,6 +6458,22 @@ SDValue SITargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
Parts[0], Parts[1]));
Hi = DAG.getBitcast(HiVT, DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i64,
Parts[2], Parts[3]));
} else {
assert(VecSize == 512);

SDValue V2 = DAG.getBitcast(MVT::v8i64, Vec);
SDValue Parts[8];
for (unsigned P = 0; P < 8; ++P) {
Parts[P] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i64, V2,
DAG.getConstant(P, SL, MVT::i32));
}

Lo = DAG.getBitcast(LoVT,
DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v4i64,
Parts[0], Parts[1], Parts[2], Parts[3]));
Hi = DAG.getBitcast(HiVT,
DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v4i64,
Parts[4], Parts[5],Parts[6], Parts[7]));
}

EVT IdxVT = Idx.getValueType();
Expand Down Expand Up @@ -6607,6 +6639,27 @@ SDValue SITargetLowering::lowerBUILD_VECTOR(SDValue Op,
return DAG.getNode(ISD::BITCAST, SL, VT, Blend);
}

if (VT == MVT::v32i16 || VT == MVT::v32f16) {
EVT QuarterVT = MVT::getVectorVT(VT.getVectorElementType().getSimpleVT(),
VT.getVectorNumElements() / 8);
MVT QuarterIntVT = MVT::getIntegerVT(QuarterVT.getSizeInBits());

SmallVector<SDValue, 8> Parts[8];
for (unsigned I = 0, E = VT.getVectorNumElements() / 8; I != E; ++I) {
for (unsigned P = 0; P < 8; ++P)
Parts[P].push_back(Op.getOperand(I + P * E));
}
SDValue Casts[8];
for (unsigned P = 0; P < 8; ++P) {
SDValue Vec = DAG.getBuildVector(QuarterVT, SL, Parts[P]);
Casts[P] = DAG.getNode(ISD::BITCAST, SL, QuarterIntVT, Vec);
}

SDValue Blend =
DAG.getBuildVector(MVT::getVectorVT(QuarterIntVT, 8), SL, Casts);
return DAG.getNode(ISD::BITCAST, SL, VT, Blend);
}

assert(VT == MVT::v2f16 || VT == MVT::v2i16);
assert(!Subtarget->hasVOP3PInsts() && "this should be legal");

Expand Down Expand Up @@ -9507,7 +9560,8 @@ SDValue SITargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {

SDValue SITargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
EVT VT = Op.getValueType();
if (VT.getSizeInBits() == 128 || VT.getSizeInBits() == 256)
if (VT.getSizeInBits() == 128 || VT.getSizeInBits() == 256 ||
VT.getSizeInBits() == 512)
return splitTernaryVectorOp(Op, DAG);

assert(VT.getSizeInBits() == 64);
Expand Down
10 changes: 10 additions & 0 deletions llvm/lib/Target/AMDGPU/SIInstructions.td
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1619,6 +1619,16 @@ def : BitConvert <v12i32, v12f32, VReg_384>;
def : BitConvert <v12f32, v12i32, VReg_384>;

// 512-bit bitcast
def : BitConvert <v32f16, v32i16, VReg_512>;
def : BitConvert <v32i16, v32f16, VReg_512>;
def : BitConvert <v32f16, v16i32, VReg_512>;
def : BitConvert <v32f16, v16f32, VReg_512>;
def : BitConvert <v16f32, v32f16, VReg_512>;
def : BitConvert <v16i32, v32f16, VReg_512>;
def : BitConvert <v32i16, v16i32, VReg_512>;
def : BitConvert <v32i16, v16f32, VReg_512>;
def : BitConvert <v16f32, v32i16, VReg_512>;
def : BitConvert <v16i32, v32i16, VReg_512>;
def : BitConvert <v16i32, v16f32, VReg_512>;
def : BitConvert <v16f32, v16i32, VReg_512>;
def : BitConvert <v8i64, v8f64, VReg_512>;
Expand Down
4 changes: 2 additions & 2 deletions llvm/lib/Target/AMDGPU/SIRegisterInfo.td
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -930,7 +930,7 @@ defm "" : SRegClass<11, [v11i32, v11f32], SGPR_352Regs, TTMP_352Regs>;
defm "" : SRegClass<12, [v12i32, v12f32], SGPR_384Regs, TTMP_384Regs>;

let GlobalPriority = true in {
defm "" : SRegClass<16, [v16i32, v16f32, v8i64, v8f64], SGPR_512Regs, TTMP_512Regs>;
defm "" : SRegClass<16, [v16i32, v16f32, v8i64, v8f64, v32i16, v32f16], SGPR_512Regs, TTMP_512Regs>;
defm "" : SRegClass<32, [v32i32, v32f32, v16i64, v16f64], SGPR_1024Regs>;
}

Expand Down Expand Up @@ -984,7 +984,7 @@ defm VReg_352 : VRegClass<11, [v11i32, v11f32], (add VGPR_352)>;
defm VReg_384 : VRegClass<12, [v12i32, v12f32], (add VGPR_384)>;

let GlobalPriority = true in {
defm VReg_512 : VRegClass<16, [v16i32, v16f32, v8i64, v8f64], (add VGPR_512)>;
defm VReg_512 : VRegClass<16, [v16i32, v16f32, v8i64, v8f64, v32i16, v32f16], (add VGPR_512)>;
defm VReg_1024 : VRegClass<32, [v32i32, v32f32, v16i64, v16f64], (add VGPR_1024)>;
}

Expand Down
4 changes: 2 additions & 2 deletions llvm/test/Analysis/CostModel/AMDGPU/add-sub.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ define amdgpu_kernel void @add_i16() #0 {
; FAST16-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v5i16 = add <5 x i16> undef, undef
; FAST16-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v6i16 = add <6 x i16> undef, undef
; FAST16-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i16 = add <16 x i16> undef, undef
; FAST16-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v17i16 = add <17 x i16> undef, undef
; FAST16-NEXT: Cost Model: Found an estimated cost of 48 for instruction: %v17i16 = add <17 x i16> undef, undef
; FAST16-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
; SLOW16-LABEL: 'add_i16'
Expand All @@ -98,7 +98,7 @@ define amdgpu_kernel void @add_i16() #0 {
; FAST16-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v5i16 = add <5 x i16> undef, undef
; FAST16-SIZE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v6i16 = add <6 x i16> undef, undef
; FAST16-SIZE-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i16 = add <16 x i16> undef, undef
; FAST16-SIZE-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v17i16 = add <17 x i16> undef, undef
; FAST16-SIZE-NEXT: Cost Model: Found an estimated cost of 48 for instruction: %v17i16 = add <17 x i16> undef, undef
; FAST16-SIZE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret void
;
; SLOW16-SIZE-LABEL: 'add_i16'
Expand Down
Loading
Loading