diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp index 82f708a0af661..0782818b1c9db 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp @@ -38,6 +38,7 @@ RegBankLegalizeHelper::RegBankLegalizeHelper( RBLRules(RBLRules), IsWave32(ST.isWave32()), SgprRB(&RBI.getRegBank(AMDGPU::SGPRRegBankID)), VgprRB(&RBI.getRegBank(AMDGPU::VGPRRegBankID)), + AgprRB(&RBI.getRegBank(AMDGPU::AGPRRegBankID)), VccRB(&RBI.getRegBank(AMDGPU::VCCRegBankID)) {} bool RegBankLegalizeHelper::findRuleAndApplyMapping(MachineInstr &MI) { @@ -1673,6 +1674,8 @@ RegBankLegalizeHelper::getRegBankFromID(RegBankLLTMappingApplyID ID) { case Sgpr32SExt: case Sgpr32ZExt: return SgprRB; + case AgprAnyTy: + return AgprRB; case Vgpr16: case Vgpr32: case Vgpr64: @@ -1703,6 +1706,7 @@ RegBankLegalizeHelper::getRegBankFromID(RegBankLLTMappingApplyID ID) { case VgprB256: case VgprB512: case VgprBRC: + case VgprAnyTy: case Vgpr32AExt: case Vgpr32SExt: case Vgpr32ZExt: @@ -1791,6 +1795,19 @@ bool RegBankLegalizeHelper::applyMappingDst( assert(RB == getRegBankFromID(MethodIDs[OpIdx])); break; } + case VgprAnyTy: { + assert(RB == VgprRB); + break; + } + case AgprAnyTy: { + if (RB == AgprRB) + break; + Register NewAgprDst = MRI.createVirtualRegister({AgprRB, Ty}); + Op.setReg(NewAgprDst); + if (!MRI.use_nodbg_empty(Reg)) + B.buildCopy(Reg, NewAgprDst); + break; + } // uniform in vcc/vgpr: scalars, vectors and B-types case UniInVcc: { assert(Ty == S1); @@ -1983,6 +2000,20 @@ bool RegBankLegalizeHelper::applyMappingSrc( } break; } + case VgprAnyTy: { + if (RB != VgprRB) { + auto CopyToVgpr = B.buildCopy({VgprRB, Ty}, Reg); + Op.setReg(CopyToVgpr.getReg(0)); + } + break; + } + case AgprAnyTy: { + if (RB != AgprRB) { + auto CopyToAgpr = B.buildCopy({AgprRB, Ty}, Reg); + Op.setReg(CopyToAgpr.getReg(0)); + } + break; + } // sgpr waterfall, scalars, and vectors case Sgpr32_WF: case SgprV4S32_WF: { diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.h b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.h index e4aeabc3ab77b..1f9708d5c784c 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.h +++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.h @@ -48,6 +48,7 @@ class RegBankLegalizeHelper { const bool IsWave32; const RegisterBank *SgprRB; const RegisterBank *VgprRB; + const RegisterBank *AgprRB; const RegisterBank *VccRB; static constexpr LLT S1 = LLT::scalar(1); diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp index f4fc15262a711..77802f5f4349a 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp @@ -98,6 +98,8 @@ bool matchUniformityAndLLT(Register Reg, UniformityLLTOpPredicateID UniID, return MRI.getType(Reg).getSizeInBits() == 256; case B512: return MRI.getType(Reg).getSizeInBits() == 512; + case DivAnyTy: + return MUI.isDivergent(Reg); case UniS1: return MRI.getType(Reg) == LLT::scalar(1) && MUI.isUniform(Reg); case UniS16: @@ -2029,6 +2031,22 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST, .Uni(S16, {{UniInVgprS16}, {IntrId, Vgpr32}}) .Div(S16, {{Vgpr16}, {IntrId, Vgpr32}}); + // TODO: Add handling for GFX90A+ which should use VGPRs instead of AGPRs. + bool HasGFX90AInsts = ST->hasGFX90AInsts(); + addRulesForIOpcs({amdgcn_mfma_f32_32x32x1f32, amdgcn_mfma_f32_16x16x1f32, + amdgcn_mfma_f32_4x4x1f32, amdgcn_mfma_f32_32x32x2f32, + amdgcn_mfma_f32_16x16x4f32, amdgcn_mfma_f32_32x32x4f16, + amdgcn_mfma_f32_16x16x4f16, amdgcn_mfma_f32_4x4x4f16, + amdgcn_mfma_f32_32x32x8f16, amdgcn_mfma_f32_16x16x16f16, + amdgcn_mfma_i32_32x32x4i8, amdgcn_mfma_i32_16x16x4i8, + amdgcn_mfma_i32_4x4x4i8, amdgcn_mfma_i32_32x32x8i8, + amdgcn_mfma_i32_16x16x16i8, amdgcn_mfma_f32_32x32x2bf16, + amdgcn_mfma_f32_16x16x2bf16, amdgcn_mfma_f32_4x4x2bf16, + amdgcn_mfma_f32_32x32x4bf16, amdgcn_mfma_f32_16x16x8bf16}) + .Any({{DivAnyTy}, + {{AgprAnyTy}, {IntrId, VgprAnyTy, VgprAnyTy, AgprAnyTy}}}, + !HasGFX90AInsts); + // WMMA/SWMMAC intrinsics: all register operands map to VGPR. addRulesForIOpcs( {// WMMA GFX11+ diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h index ec5f67bf62b0c..35bbefb54d29e 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h +++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h @@ -58,6 +58,9 @@ enum UniformityLLTOpPredicateID { DivS64, DivS128, + // any LLT, divergent-check only predicate + DivAnyTy, + // pointers P0, P1, @@ -149,6 +152,10 @@ enum RegBankLLTMappingApplyID { Imm, Vcc, + // any LLT, bank-only apply IDs + VgprAnyTy, + AgprAnyTy, + // sgpr scalars, pointers, vectors and B-types Sgpr16, Sgpr32, diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.mfma.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.mfma.mir index e34a62d29a102..06cdc075da42c 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.mfma.mir +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.mfma.mir @@ -1,6 +1,5 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py -# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs %s -o - | FileCheck %s -# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs %s -o - | FileCheck %s +# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass='amdgpu-regbankselect,amdgpu-regbanklegalize' -verify-machineinstrs %s -o - | FileCheck %s --- name: mfma_f32_32x32x1f32_vva @@ -15,9 +14,11 @@ body: | ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0 ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1 - ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(<32 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 - ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x1f32), [[COPY]](s32), [[COPY1]](s32), [[COPY2]](<32 x s32>), 0, 0, 0 - ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[INT]](<32 x s32>) + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<32 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<32 x s32>) = COPY [[COPY2]](<32 x s32>) + ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x1f32), [[COPY]](s32), [[COPY1]](s32), [[COPY3]](<32 x s32>), 0, 0, 0 + ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<32 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<32 x s32>) + ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[COPY4]](<32 x s32>) %0:_(s32) = COPY $vgpr0 %1:_(s32) = COPY $vgpr1 %2:_(<32 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 @@ -42,8 +43,9 @@ body: | ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32) ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32) ; CHECK-NEXT: [[COPY5:%[0-9]+]]:agpr(<32 x s32>) = COPY [[COPY2]](<32 x s32>) - ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x1f32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](<32 x s32>), 0, 0, 0 - ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[INT]](<32 x s32>) + ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x1f32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](<32 x s32>), 0, 0, 0 + ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(<32 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<32 x s32>) + ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[COPY6]](<32 x s32>) %0:_(s32) = COPY $sgpr32 %1:_(s32) = COPY $sgpr33 %2:_(<32 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31 @@ -64,9 +66,11 @@ body: | ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0 ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1 - ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 - ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x1f32), [[COPY]](s32), [[COPY1]](s32), [[COPY2]](<16 x s32>), 0, 0, 0 - ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>) + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>) + ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x1f32), [[COPY]](s32), [[COPY1]](s32), [[COPY3]](<16 x s32>), 0, 0, 0 + ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>) + ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY4]](<16 x s32>) %0:_(s32) = COPY $vgpr0 %1:_(s32) = COPY $vgpr1 %2:_(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 @@ -91,8 +95,9 @@ body: | ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32) ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32) ; CHECK-NEXT: [[COPY5:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>) - ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x1f32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](<16 x s32>), 0, 0, 0 - ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>) + ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x1f32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](<16 x s32>), 0, 0, 0 + ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>) + ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY6]](<16 x s32>) %0:_(s32) = COPY $sgpr32 %1:_(s32) = COPY $sgpr33 %2:_(<16 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15 @@ -113,9 +118,11 @@ body: | ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0 ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1 - ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3 - ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.4x4x1f32), [[COPY]](s32), [[COPY1]](s32), [[COPY2]](<4 x s32>), 0, 0, 0 - ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>) + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3 + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>) + ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.4x4x1f32), [[COPY]](s32), [[COPY1]](s32), [[COPY3]](<4 x s32>), 0, 0, 0 + ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>) + ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY4]](<4 x s32>) %0:_(s32) = COPY $vgpr0 %1:_(s32) = COPY $vgpr1 %2:_(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3 @@ -140,8 +147,9 @@ body: | ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32) ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32) ; CHECK-NEXT: [[COPY5:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>) - ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.4x4x1f32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](<4 x s32>), 0, 0, 0 - ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>) + ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.4x4x1f32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](<4 x s32>), 0, 0, 0 + ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>) + ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY6]](<4 x s32>) %0:_(s32) = COPY $sgpr32 %1:_(s32) = COPY $sgpr33 %2:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3 @@ -162,9 +170,11 @@ body: | ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0 ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1 - ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 - ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x2f32), [[COPY]](s32), [[COPY1]](s32), [[COPY2]](<16 x s32>), 0, 0, 0 - ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>) + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>) + ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x2f32), [[COPY]](s32), [[COPY1]](s32), [[COPY3]](<16 x s32>), 0, 0, 0 + ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>) + ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY4]](<16 x s32>) %0:_(s32) = COPY $vgpr0 %1:_(s32) = COPY $vgpr1 %2:_(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 @@ -189,8 +199,9 @@ body: | ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32) ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32) ; CHECK-NEXT: [[COPY5:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>) - ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x2f32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](<16 x s32>), 0, 0, 0 - ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>) + ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x2f32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](<16 x s32>), 0, 0, 0 + ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>) + ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY6]](<16 x s32>) %0:_(s32) = COPY $sgpr32 %1:_(s32) = COPY $sgpr33 %2:_(<16 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15 @@ -211,9 +222,11 @@ body: | ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0 ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1 - ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3 - ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x4f32), [[COPY]](s32), [[COPY1]](s32), [[COPY2]](<4 x s32>), 0, 0, 0 - ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>) + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3 + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>) + ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x4f32), [[COPY]](s32), [[COPY1]](s32), [[COPY3]](<4 x s32>), 0, 0, 0 + ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>) + ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY4]](<4 x s32>) %0:_(s32) = COPY $vgpr0 %1:_(s32) = COPY $vgpr1 %2:_(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3 @@ -238,8 +251,9 @@ body: | ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32) ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32) ; CHECK-NEXT: [[COPY5:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>) - ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x4f32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](<4 x s32>), 0, 0, 0 - ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>) + ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x4f32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](<4 x s32>), 0, 0, 0 + ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>) + ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY6]](<4 x s32>) %0:_(s32) = COPY $sgpr32 %1:_(s32) = COPY $sgpr33 %2:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3 @@ -260,9 +274,11 @@ body: | ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(<4 x s16>) = COPY $vgpr0_vgpr1 ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(<4 x s16>) = COPY $vgpr2_vgpr3 - ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(<32 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 - ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x4f16), [[COPY]](<4 x s16>), [[COPY1]](<4 x s16>), [[COPY2]](<32 x s32>), 0, 0, 0 - ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[INT]](<32 x s32>) + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<32 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<32 x s32>) = COPY [[COPY2]](<32 x s32>) + ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x4f16), [[COPY]](<4 x s16>), [[COPY1]](<4 x s16>), [[COPY3]](<32 x s32>), 0, 0, 0 + ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<32 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<32 x s32>) + ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[COPY4]](<32 x s32>) %0:_(<4 x s16>) = COPY $vgpr0_vgpr1 %1:_(<4 x s16>) = COPY $vgpr2_vgpr3 %2:_(<32 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 @@ -287,8 +303,9 @@ body: | ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(<4 x s16>) = COPY [[COPY]](<4 x s16>) ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<4 x s16>) = COPY [[COPY1]](<4 x s16>) ; CHECK-NEXT: [[COPY5:%[0-9]+]]:agpr(<32 x s32>) = COPY [[COPY2]](<32 x s32>) - ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x4f16), [[COPY3]](<4 x s16>), [[COPY4]](<4 x s16>), [[COPY5]](<32 x s32>), 0, 0, 0 - ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[INT]](<32 x s32>) + ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x4f16), [[COPY3]](<4 x s16>), [[COPY4]](<4 x s16>), [[COPY5]](<32 x s32>), 0, 0, 0 + ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(<32 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<32 x s32>) + ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[COPY6]](<32 x s32>) %0:_(<4 x s16>) = COPY $sgpr32_sgpr33 %1:_(<4 x s16>) = COPY $sgpr34_sgpr35 %2:_(<32 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31 @@ -309,9 +326,11 @@ body: | ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(<4 x s16>) = COPY $vgpr0_vgpr1 ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(<4 x s16>) = COPY $vgpr2_vgpr3 - ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 - ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x4f16), [[COPY]](<4 x s16>), [[COPY1]](<4 x s16>), [[COPY2]](<16 x s32>), 0, 0, 0 - ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>) + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>) + ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x4f16), [[COPY]](<4 x s16>), [[COPY1]](<4 x s16>), [[COPY3]](<16 x s32>), 0, 0, 0 + ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>) + ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY4]](<16 x s32>) %0:_(<4 x s16>) = COPY $vgpr0_vgpr1 %1:_(<4 x s16>) = COPY $vgpr2_vgpr3 %2:_(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 @@ -336,8 +355,9 @@ body: | ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(<4 x s16>) = COPY [[COPY]](<4 x s16>) ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<4 x s16>) = COPY [[COPY1]](<4 x s16>) ; CHECK-NEXT: [[COPY5:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>) - ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x4f16), [[COPY3]](<4 x s16>), [[COPY4]](<4 x s16>), [[COPY5]](<16 x s32>), 0, 0, 0 - ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>) + ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x4f16), [[COPY3]](<4 x s16>), [[COPY4]](<4 x s16>), [[COPY5]](<16 x s32>), 0, 0, 0 + ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>) + ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY6]](<16 x s32>) %0:_(<4 x s16>) = COPY $sgpr32_sgpr33 %1:_(<4 x s16>) = COPY $sgpr34_sgpr35 %2:_(<16 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15 @@ -358,9 +378,11 @@ body: | ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(<4 x s16>) = COPY $vgpr0_vgpr1 ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(<4 x s16>) = COPY $vgpr2_vgpr3 - ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3 - ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.4x4x4f16), [[COPY]](<4 x s16>), [[COPY1]](<4 x s16>), [[COPY2]](<4 x s32>), 0, 0, 0 - ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>) + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3 + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>) + ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.4x4x4f16), [[COPY]](<4 x s16>), [[COPY1]](<4 x s16>), [[COPY3]](<4 x s32>), 0, 0, 0 + ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>) + ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY4]](<4 x s32>) %0:_(<4 x s16>) = COPY $vgpr0_vgpr1 %1:_(<4 x s16>) = COPY $vgpr2_vgpr3 %2:_(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3 @@ -385,8 +407,9 @@ body: | ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(<4 x s16>) = COPY [[COPY]](<4 x s16>) ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<4 x s16>) = COPY [[COPY1]](<4 x s16>) ; CHECK-NEXT: [[COPY5:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>) - ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.4x4x4f16), [[COPY3]](<4 x s16>), [[COPY4]](<4 x s16>), [[COPY5]](<4 x s32>), 0, 0, 0 - ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>) + ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.4x4x4f16), [[COPY3]](<4 x s16>), [[COPY4]](<4 x s16>), [[COPY5]](<4 x s32>), 0, 0, 0 + ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>) + ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY6]](<4 x s32>) %0:_(<4 x s16>) = COPY $sgpr32_sgpr33 %1:_(<4 x s16>) = COPY $sgpr34_sgpr35 %2:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3 @@ -407,9 +430,11 @@ body: | ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(<4 x s16>) = COPY $vgpr0_vgpr1 ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(<4 x s16>) = COPY $vgpr2_vgpr3 - ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 - ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x8f16), [[COPY]](<4 x s16>), [[COPY1]](<4 x s16>), [[COPY2]](<16 x s32>), 0, 0, 0 - ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>) + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>) + ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x8f16), [[COPY]](<4 x s16>), [[COPY1]](<4 x s16>), [[COPY3]](<16 x s32>), 0, 0, 0 + ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>) + ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY4]](<16 x s32>) %0:_(<4 x s16>) = COPY $vgpr0_vgpr1 %1:_(<4 x s16>) = COPY $vgpr2_vgpr3 %2:_(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 @@ -434,8 +459,9 @@ body: | ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(<4 x s16>) = COPY [[COPY]](<4 x s16>) ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<4 x s16>) = COPY [[COPY1]](<4 x s16>) ; CHECK-NEXT: [[COPY5:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>) - ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x8f16), [[COPY3]](<4 x s16>), [[COPY4]](<4 x s16>), [[COPY5]](<16 x s32>), 0, 0, 0 - ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>) + ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x8f16), [[COPY3]](<4 x s16>), [[COPY4]](<4 x s16>), [[COPY5]](<16 x s32>), 0, 0, 0 + ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>) + ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY6]](<16 x s32>) %0:_(<4 x s16>) = COPY $sgpr32_sgpr33 %1:_(<4 x s16>) = COPY $sgpr34_sgpr35 %2:_(<16 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15 @@ -456,9 +482,11 @@ body: | ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(<4 x s16>) = COPY $vgpr0_vgpr1 ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(<4 x s16>) = COPY $vgpr2_vgpr3 - ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3 - ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x16f16), [[COPY]](<4 x s16>), [[COPY1]](<4 x s16>), [[COPY2]](<4 x s32>), 0, 0, 0 - ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>) + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3 + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>) + ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x16f16), [[COPY]](<4 x s16>), [[COPY1]](<4 x s16>), [[COPY3]](<4 x s32>), 0, 0, 0 + ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>) + ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY4]](<4 x s32>) %0:_(<4 x s16>) = COPY $vgpr0_vgpr1 %1:_(<4 x s16>) = COPY $vgpr2_vgpr3 %2:_(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3 @@ -483,8 +511,9 @@ body: | ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(<4 x s16>) = COPY [[COPY]](<4 x s16>) ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<4 x s16>) = COPY [[COPY1]](<4 x s16>) ; CHECK-NEXT: [[COPY5:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>) - ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x16f16), [[COPY3]](<4 x s16>), [[COPY4]](<4 x s16>), [[COPY5]](<4 x s32>), 0, 0, 0 - ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>) + ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x16f16), [[COPY3]](<4 x s16>), [[COPY4]](<4 x s16>), [[COPY5]](<4 x s32>), 0, 0, 0 + ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>) + ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY6]](<4 x s32>) %0:_(<4 x s16>) = COPY $sgpr32_sgpr33 %1:_(<4 x s16>) = COPY $sgpr34_sgpr35 %2:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3 @@ -505,9 +534,11 @@ body: | ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0 ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr2 - ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(<32 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 - ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.32x32x4i8), [[COPY]](s32), [[COPY1]](s32), [[COPY2]](<32 x s32>), 0, 0, 0 - ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[INT]](<32 x s32>) + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<32 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<32 x s32>) = COPY [[COPY2]](<32 x s32>) + ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.32x32x4i8), [[COPY]](s32), [[COPY1]](s32), [[COPY3]](<32 x s32>), 0, 0, 0 + ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<32 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<32 x s32>) + ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[COPY4]](<32 x s32>) %0:_(s32) = COPY $vgpr0 %1:_(s32) = COPY $vgpr2 %2:_(<32 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 @@ -532,8 +563,9 @@ body: | ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32) ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32) ; CHECK-NEXT: [[COPY5:%[0-9]+]]:agpr(<32 x s32>) = COPY [[COPY2]](<32 x s32>) - ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.32x32x4i8), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](<32 x s32>), 0, 0, 0 - ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[INT]](<32 x s32>) + ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.32x32x4i8), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](<32 x s32>), 0, 0, 0 + ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(<32 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<32 x s32>) + ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[COPY6]](<32 x s32>) %0:_(s32) = COPY $sgpr32 %1:_(s32) = COPY $sgpr33 %2:_(<32 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31 @@ -554,9 +586,11 @@ body: | ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0 ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr2 - ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 - ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.16x16x4i8), [[COPY]](s32), [[COPY1]](s32), [[COPY2]](<16 x s32>), 0, 0, 0 - ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>) + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>) + ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.16x16x4i8), [[COPY]](s32), [[COPY1]](s32), [[COPY3]](<16 x s32>), 0, 0, 0 + ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>) + ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY4]](<16 x s32>) %0:_(s32) = COPY $vgpr0 %1:_(s32) = COPY $vgpr2 %2:_(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 @@ -581,8 +615,9 @@ body: | ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32) ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32) ; CHECK-NEXT: [[COPY5:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>) - ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.16x16x4i8), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](<16 x s32>), 0, 0, 0 - ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>) + ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.16x16x4i8), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](<16 x s32>), 0, 0, 0 + ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>) + ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY6]](<16 x s32>) %0:_(s32) = COPY $sgpr32 %1:_(s32) = COPY $sgpr33 %2:_(<16 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15 @@ -603,9 +638,11 @@ body: | ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0 ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr2 - ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3 - ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.4x4x4i8), [[COPY]](s32), [[COPY1]](s32), [[COPY2]](<4 x s32>), 0, 0, 0 - ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>) + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3 + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>) + ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.4x4x4i8), [[COPY]](s32), [[COPY1]](s32), [[COPY3]](<4 x s32>), 0, 0, 0 + ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>) + ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY4]](<4 x s32>) %0:_(s32) = COPY $vgpr0 %1:_(s32) = COPY $vgpr2 %2:_(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3 @@ -630,8 +667,9 @@ body: | ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32) ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32) ; CHECK-NEXT: [[COPY5:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>) - ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.4x4x4i8), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](<4 x s32>), 0, 0, 0 - ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>) + ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.4x4x4i8), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](<4 x s32>), 0, 0, 0 + ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>) + ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY6]](<4 x s32>) %0:_(s32) = COPY $sgpr32 %1:_(s32) = COPY $sgpr33 %2:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3 @@ -652,9 +690,11 @@ body: | ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0 ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr2 - ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 - ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.32x32x8i8), [[COPY]](s32), [[COPY1]](s32), [[COPY2]](<16 x s32>), 0, 0, 0 - ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>) + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>) + ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.32x32x8i8), [[COPY]](s32), [[COPY1]](s32), [[COPY3]](<16 x s32>), 0, 0, 0 + ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>) + ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY4]](<16 x s32>) %0:_(s32) = COPY $vgpr0 %1:_(s32) = COPY $vgpr2 %2:_(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 @@ -679,8 +719,9 @@ body: | ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32) ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32) ; CHECK-NEXT: [[COPY5:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>) - ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.32x32x8i8), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](<16 x s32>), 0, 0, 0 - ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>) + ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.32x32x8i8), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](<16 x s32>), 0, 0, 0 + ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>) + ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY6]](<16 x s32>) %0:_(s32) = COPY $sgpr32 %1:_(s32) = COPY $sgpr33 %2:_(<16 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15 @@ -701,9 +742,11 @@ body: | ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0 ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr2 - ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3 - ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.16x16x16i8), [[COPY]](s32), [[COPY1]](s32), [[COPY2]](<4 x s32>), 0, 0, 0 - ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>) + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3 + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>) + ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.16x16x16i8), [[COPY]](s32), [[COPY1]](s32), [[COPY3]](<4 x s32>), 0, 0, 0 + ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>) + ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY4]](<4 x s32>) %0:_(s32) = COPY $vgpr0 %1:_(s32) = COPY $vgpr2 %2:_(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3 @@ -728,8 +771,9 @@ body: | ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32) ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32) ; CHECK-NEXT: [[COPY5:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>) - ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.16x16x16i8), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](<4 x s32>), 0, 0, 0 - ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>) + ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.i32.16x16x16i8), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](<4 x s32>), 0, 0, 0 + ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>) + ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY6]](<4 x s32>) %0:_(s32) = COPY $sgpr32 %1:_(s32) = COPY $sgpr33 %2:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3 @@ -750,9 +794,11 @@ body: | ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr0 ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr2 - ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(<32 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 - ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x2bf16), [[COPY]](<2 x s16>), [[COPY1]](<2 x s16>), [[COPY2]](<32 x s32>), 0, 0, 0 - ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[INT]](<32 x s32>) + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<32 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<32 x s32>) = COPY [[COPY2]](<32 x s32>) + ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x2bf16), [[COPY]](<2 x s16>), [[COPY1]](<2 x s16>), [[COPY3]](<32 x s32>), 0, 0, 0 + ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<32 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<32 x s32>) + ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[COPY4]](<32 x s32>) %0:_(<2 x s16>) = COPY $vgpr0 %1:_(<2 x s16>) = COPY $vgpr2 %2:_(<32 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 @@ -777,8 +823,9 @@ body: | ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(<2 x s16>) = COPY [[COPY]](<2 x s16>) ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<2 x s16>) = COPY [[COPY1]](<2 x s16>) ; CHECK-NEXT: [[COPY5:%[0-9]+]]:agpr(<32 x s32>) = COPY [[COPY2]](<32 x s32>) - ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x2bf16), [[COPY3]](<2 x s16>), [[COPY4]](<2 x s16>), [[COPY5]](<32 x s32>), 0, 0, 0 - ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[INT]](<32 x s32>) + ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<32 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x2bf16), [[COPY3]](<2 x s16>), [[COPY4]](<2 x s16>), [[COPY5]](<32 x s32>), 0, 0, 0 + ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(<32 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<32 x s32>) + ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = COPY [[COPY6]](<32 x s32>) %0:_(<2 x s16>) = COPY $sgpr32 %1:_(<2 x s16>) = COPY $sgpr33 %2:_(<32 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31 @@ -799,9 +846,11 @@ body: | ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr0 ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr2 - ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 - ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x2bf16), [[COPY]](<2 x s16>), [[COPY1]](<2 x s16>), [[COPY2]](<16 x s32>), 0, 0, 0 - ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>) + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>) + ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x2bf16), [[COPY]](<2 x s16>), [[COPY1]](<2 x s16>), [[COPY3]](<16 x s32>), 0, 0, 0 + ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>) + ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY4]](<16 x s32>) %0:_(<2 x s16>) = COPY $vgpr0 %1:_(<2 x s16>) = COPY $vgpr2 %2:_(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 @@ -826,8 +875,9 @@ body: | ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(<2 x s16>) = COPY [[COPY]](<2 x s16>) ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<2 x s16>) = COPY [[COPY1]](<2 x s16>) ; CHECK-NEXT: [[COPY5:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>) - ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x2bf16), [[COPY3]](<2 x s16>), [[COPY4]](<2 x s16>), [[COPY5]](<16 x s32>), 0, 0, 0 - ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>) + ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x2bf16), [[COPY3]](<2 x s16>), [[COPY4]](<2 x s16>), [[COPY5]](<16 x s32>), 0, 0, 0 + ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>) + ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY6]](<16 x s32>) %0:_(<2 x s16>) = COPY $sgpr32 %1:_(<2 x s16>) = COPY $sgpr33 %2:_(<16 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15 @@ -848,9 +898,11 @@ body: | ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr0 ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr2 - ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3 - ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.4x4x2bf16), [[COPY]](<2 x s16>), [[COPY1]](<2 x s16>), [[COPY2]](<4 x s32>), 0, 0, 0 - ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>) + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3 + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>) + ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.4x4x2bf16), [[COPY]](<2 x s16>), [[COPY1]](<2 x s16>), [[COPY3]](<4 x s32>), 0, 0, 0 + ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>) + ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY4]](<4 x s32>) %0:_(<2 x s16>) = COPY $vgpr0 %1:_(<2 x s16>) = COPY $vgpr2 %2:_(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3 @@ -875,8 +927,9 @@ body: | ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(<2 x s16>) = COPY [[COPY]](<2 x s16>) ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<2 x s16>) = COPY [[COPY1]](<2 x s16>) ; CHECK-NEXT: [[COPY5:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>) - ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.4x4x2bf16), [[COPY3]](<2 x s16>), [[COPY4]](<2 x s16>), [[COPY5]](<4 x s32>), 0, 0, 0 - ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>) + ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.4x4x2bf16), [[COPY3]](<2 x s16>), [[COPY4]](<2 x s16>), [[COPY5]](<4 x s32>), 0, 0, 0 + ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>) + ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY6]](<4 x s32>) %0:_(<2 x s16>) = COPY $sgpr32 %1:_(<2 x s16>) = COPY $sgpr33 %2:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3 @@ -897,9 +950,11 @@ body: | ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr0 ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr2 - ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 - ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x4bf16), [[COPY]](<2 x s16>), [[COPY1]](<2 x s16>), [[COPY2]](<16 x s32>), 0, 0, 0 - ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>) + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>) + ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x4bf16), [[COPY]](<2 x s16>), [[COPY1]](<2 x s16>), [[COPY3]](<16 x s32>), 0, 0, 0 + ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>) + ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY4]](<16 x s32>) %0:_(<2 x s16>) = COPY $vgpr0 %1:_(<2 x s16>) = COPY $vgpr2 %2:_(<16 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 @@ -924,8 +979,9 @@ body: | ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(<2 x s16>) = COPY [[COPY]](<2 x s16>) ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<2 x s16>) = COPY [[COPY1]](<2 x s16>) ; CHECK-NEXT: [[COPY5:%[0-9]+]]:agpr(<16 x s32>) = COPY [[COPY2]](<16 x s32>) - ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x4bf16), [[COPY3]](<2 x s16>), [[COPY4]](<2 x s16>), [[COPY5]](<16 x s32>), 0, 0, 0 - ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[INT]](<16 x s32>) + ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<16 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.32x32x4bf16), [[COPY3]](<2 x s16>), [[COPY4]](<2 x s16>), [[COPY5]](<16 x s32>), 0, 0, 0 + ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(<16 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<16 x s32>) + ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY6]](<16 x s32>) %0:_(<2 x s16>) = COPY $sgpr32 %1:_(<2 x s16>) = COPY $sgpr33 %2:_(<16 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15 @@ -946,9 +1002,11 @@ body: | ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr0 ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr2 - ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3 - ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x8bf16), [[COPY]](<2 x s16>), [[COPY1]](<2 x s16>), [[COPY2]](<4 x s32>), 0, 0, 0 - ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>) + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3 + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>) + ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x8bf16), [[COPY]](<2 x s16>), [[COPY1]](<2 x s16>), [[COPY3]](<4 x s32>), 0, 0, 0 + ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>) + ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY4]](<4 x s32>) %0:_(<2 x s16>) = COPY $vgpr0 %1:_(<2 x s16>) = COPY $vgpr2 %2:_(<4 x s32>) = COPY $agpr0_agpr1_agpr2_agpr3 @@ -973,8 +1031,9 @@ body: | ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(<2 x s16>) = COPY [[COPY]](<2 x s16>) ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(<2 x s16>) = COPY [[COPY1]](<2 x s16>) ; CHECK-NEXT: [[COPY5:%[0-9]+]]:agpr(<4 x s32>) = COPY [[COPY2]](<4 x s32>) - ; CHECK-NEXT: [[INT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x8bf16), [[COPY3]](<2 x s16>), [[COPY4]](<2 x s16>), [[COPY5]](<4 x s32>), 0, 0, 0 - ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INT]](<4 x s32>) + ; CHECK-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:agpr(<4 x s32>) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.mfma.f32.16x16x8bf16), [[COPY3]](<2 x s16>), [[COPY4]](<2 x s16>), [[COPY5]](<4 x s32>), 0, 0, 0 + ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(<4 x s32>) = COPY [[INTRINSIC_CONVERGENT]](<4 x s32>) + ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY6]](<4 x s32>) %0:_(<2 x s16>) = COPY $sgpr32 %1:_(<2 x s16>) = COPY $sgpr33 %2:_(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3 diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.bf16.ll index f0040b61efaae..45e5755675d83 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.bf16.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.bf16.ll @@ -1,6 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 -; RUN: llc -mtriple=amdgcn -mcpu=gfx908 < %s | FileCheck --check-prefix=GFX908 %s -; RUN: llc -mtriple=amdgcn -mcpu=gfx908 -mattr=-mfma-inline-literal-bug < %s | FileCheck --check-prefix=GFX908 %s +; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx908 < %s | FileCheck --check-prefixes=GFX908,GFX908-SDAG %s +; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx908 < %s | FileCheck --check-prefixes=GFX908,GFX908-GISEL %s +; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx908 -mattr=-mfma-inline-literal-bug < %s | FileCheck --check-prefixes=GFX908,GFX908-SDAG %s +; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx908 -mattr=-mfma-inline-literal-bug < %s | FileCheck --check-prefixes=GFX908,GFX908-GISEL %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -amdgpu-mfma-vgpr-form=0 < %s | FileCheck --check-prefix=GFX90A %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck --check-prefixes=GFX90A-VGPR %s @@ -12,140 +14,260 @@ declare <4 x float> @llvm.amdgcn.mfma.f32.16x16x8bf16(<2 x i16>, <2 x i16>, <4 x declare i32 @llvm.amdgcn.workitem.id.x() define amdgpu_kernel void @test_mfma_f32_32x32x2bf16(ptr addrspace(1) %arg) #0 { -; GFX908-LABEL: test_mfma_f32_32x32x2bf16: -; GFX908: ; %bb.0: ; %bb -; GFX908-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24 -; GFX908-NEXT: v_mov_b32_e32 v4, 0 -; GFX908-NEXT: s_waitcnt lgkmcnt(0) -; GFX908-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0 -; GFX908-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40 -; GFX908-NEXT: s_waitcnt lgkmcnt(0) -; GFX908-NEXT: v_mov_b32_e32 v0, s16 -; GFX908-NEXT: v_mov_b32_e32 v1, s17 -; GFX908-NEXT: v_mov_b32_e32 v2, s18 -; GFX908-NEXT: v_accvgpr_write_b32 a0, v0 -; GFX908-NEXT: v_accvgpr_write_b32 a1, v1 -; GFX908-NEXT: v_accvgpr_write_b32 a2, v2 -; GFX908-NEXT: v_mov_b32_e32 v0, s21 -; GFX908-NEXT: v_mov_b32_e32 v1, s22 -; GFX908-NEXT: v_mov_b32_e32 v2, s23 -; GFX908-NEXT: v_accvgpr_write_b32 a5, v0 -; GFX908-NEXT: v_accvgpr_write_b32 a6, v1 -; GFX908-NEXT: v_accvgpr_write_b32 a7, v2 -; GFX908-NEXT: v_mov_b32_e32 v0, s24 -; GFX908-NEXT: v_mov_b32_e32 v1, s25 -; GFX908-NEXT: v_mov_b32_e32 v2, s26 -; GFX908-NEXT: v_accvgpr_write_b32 a8, v0 -; GFX908-NEXT: v_accvgpr_write_b32 a9, v1 -; GFX908-NEXT: v_accvgpr_write_b32 a10, v2 -; GFX908-NEXT: v_mov_b32_e32 v0, s27 -; GFX908-NEXT: v_mov_b32_e32 v1, s28 -; GFX908-NEXT: v_mov_b32_e32 v2, s29 -; GFX908-NEXT: v_accvgpr_write_b32 a11, v0 -; GFX908-NEXT: v_accvgpr_write_b32 a12, v1 -; GFX908-NEXT: v_accvgpr_write_b32 a13, v2 -; GFX908-NEXT: v_mov_b32_e32 v0, s30 -; GFX908-NEXT: v_mov_b32_e32 v1, s31 -; GFX908-NEXT: v_mov_b32_e32 v2, s0 -; GFX908-NEXT: v_accvgpr_write_b32 a14, v0 -; GFX908-NEXT: v_accvgpr_write_b32 a15, v1 -; GFX908-NEXT: v_accvgpr_write_b32 a16, v2 -; GFX908-NEXT: v_mov_b32_e32 v0, s1 -; GFX908-NEXT: v_mov_b32_e32 v1, s2 -; GFX908-NEXT: v_mov_b32_e32 v2, s3 -; GFX908-NEXT: v_accvgpr_write_b32 a17, v0 -; GFX908-NEXT: v_accvgpr_write_b32 a18, v1 -; GFX908-NEXT: v_accvgpr_write_b32 a19, v2 -; GFX908-NEXT: v_mov_b32_e32 v0, s4 -; GFX908-NEXT: v_mov_b32_e32 v1, s5 -; GFX908-NEXT: v_mov_b32_e32 v2, s6 -; GFX908-NEXT: v_accvgpr_write_b32 a20, v0 -; GFX908-NEXT: v_accvgpr_write_b32 a21, v1 -; GFX908-NEXT: v_accvgpr_write_b32 a22, v2 -; GFX908-NEXT: v_mov_b32_e32 v0, s7 -; GFX908-NEXT: v_mov_b32_e32 v1, s8 -; GFX908-NEXT: v_mov_b32_e32 v2, s9 -; GFX908-NEXT: v_mov_b32_e32 v3, s19 -; GFX908-NEXT: v_accvgpr_write_b32 a23, v0 -; GFX908-NEXT: v_accvgpr_write_b32 a24, v1 -; GFX908-NEXT: v_accvgpr_write_b32 a25, v2 -; GFX908-NEXT: v_mov_b32_e32 v0, s10 -; GFX908-NEXT: v_mov_b32_e32 v1, s11 -; GFX908-NEXT: v_mov_b32_e32 v2, s12 -; GFX908-NEXT: v_mov_b32_e32 v5, s20 -; GFX908-NEXT: v_accvgpr_write_b32 a3, v3 -; GFX908-NEXT: v_accvgpr_write_b32 a26, v0 -; GFX908-NEXT: v_accvgpr_write_b32 a27, v1 -; GFX908-NEXT: v_accvgpr_write_b32 a28, v2 -; GFX908-NEXT: v_mov_b32_e32 v0, s13 -; GFX908-NEXT: v_mov_b32_e32 v1, s14 -; GFX908-NEXT: v_mov_b32_e32 v2, s15 -; GFX908-NEXT: v_mov_b32_e32 v3, 1 -; GFX908-NEXT: v_accvgpr_write_b32 a4, v5 -; GFX908-NEXT: v_accvgpr_write_b32 a29, v0 -; GFX908-NEXT: v_accvgpr_write_b32 a30, v1 -; GFX908-NEXT: v_accvgpr_write_b32 a31, v2 -; GFX908-NEXT: v_mov_b32_e32 v0, 2 -; GFX908-NEXT: s_nop 1 -; GFX908-NEXT: v_mfma_f32_32x32x2bf16 a[0:31], v3, v0, a[0:31] cbsz:1 abid:2 blgp:3 -; GFX908-NEXT: s_nop 15 -; GFX908-NEXT: s_nop 1 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a27 -; GFX908-NEXT: v_accvgpr_read_b32 v2, a26 -; GFX908-NEXT: v_accvgpr_read_b32 v1, a25 -; GFX908-NEXT: v_accvgpr_read_b32 v0, a24 -; GFX908-NEXT: s_nop 1 -; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:96 -; GFX908-NEXT: s_nop 0 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a31 -; GFX908-NEXT: v_accvgpr_read_b32 v2, a30 -; GFX908-NEXT: v_accvgpr_read_b32 v1, a29 -; GFX908-NEXT: v_accvgpr_read_b32 v0, a28 -; GFX908-NEXT: s_nop 1 -; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:112 -; GFX908-NEXT: s_nop 0 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a19 -; GFX908-NEXT: v_accvgpr_read_b32 v2, a18 -; GFX908-NEXT: v_accvgpr_read_b32 v1, a17 -; GFX908-NEXT: v_accvgpr_read_b32 v0, a16 -; GFX908-NEXT: s_nop 1 -; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:64 -; GFX908-NEXT: s_nop 0 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a23 -; GFX908-NEXT: v_accvgpr_read_b32 v2, a22 -; GFX908-NEXT: v_accvgpr_read_b32 v1, a21 -; GFX908-NEXT: v_accvgpr_read_b32 v0, a20 -; GFX908-NEXT: s_nop 1 -; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:80 -; GFX908-NEXT: s_nop 0 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a11 -; GFX908-NEXT: v_accvgpr_read_b32 v2, a10 -; GFX908-NEXT: v_accvgpr_read_b32 v1, a9 -; GFX908-NEXT: v_accvgpr_read_b32 v0, a8 -; GFX908-NEXT: s_nop 1 -; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:32 -; GFX908-NEXT: s_nop 0 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a15 -; GFX908-NEXT: v_accvgpr_read_b32 v2, a14 -; GFX908-NEXT: v_accvgpr_read_b32 v1, a13 -; GFX908-NEXT: v_accvgpr_read_b32 v0, a12 -; GFX908-NEXT: s_nop 1 -; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:48 -; GFX908-NEXT: s_nop 0 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a3 -; GFX908-NEXT: v_accvgpr_read_b32 v2, a2 -; GFX908-NEXT: v_accvgpr_read_b32 v1, a1 -; GFX908-NEXT: v_accvgpr_read_b32 v0, a0 -; GFX908-NEXT: s_nop 1 -; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] -; GFX908-NEXT: s_nop 0 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a7 -; GFX908-NEXT: v_accvgpr_read_b32 v2, a6 -; GFX908-NEXT: v_accvgpr_read_b32 v1, a5 -; GFX908-NEXT: v_accvgpr_read_b32 v0, a4 -; GFX908-NEXT: s_nop 1 -; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:16 -; GFX908-NEXT: s_endpgm +; GFX908-SDAG-LABEL: test_mfma_f32_32x32x2bf16: +; GFX908-SDAG: ; %bb.0: ; %bb +; GFX908-SDAG-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v4, 0 +; GFX908-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-SDAG-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0 +; GFX908-SDAG-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40 +; GFX908-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, s16 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s17 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s18 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a0, v0 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a1, v1 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a2, v2 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, s21 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s22 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s23 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a5, v0 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a6, v1 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a7, v2 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, s24 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s25 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s26 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a8, v0 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a9, v1 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a10, v2 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, s27 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s28 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s29 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a11, v0 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a12, v1 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a13, v2 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, s30 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s31 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s0 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a14, v0 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a15, v1 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a16, v2 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, s1 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s2 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s3 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a17, v0 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a18, v1 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a19, v2 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, s4 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s5 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s6 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a20, v0 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a21, v1 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a22, v2 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, s7 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s8 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s9 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v3, s19 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a23, v0 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a24, v1 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a25, v2 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, s10 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s11 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s12 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v5, s20 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a3, v3 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a26, v0 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a27, v1 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a28, v2 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, s13 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s14 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s15 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v3, 1 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a4, v5 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a29, v0 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a30, v1 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a31, v2 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, 2 +; GFX908-SDAG-NEXT: s_nop 1 +; GFX908-SDAG-NEXT: v_mfma_f32_32x32x2bf16 a[0:31], v3, v0, a[0:31] cbsz:1 abid:2 blgp:3 +; GFX908-SDAG-NEXT: s_nop 15 +; GFX908-SDAG-NEXT: s_nop 1 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a27 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a26 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a25 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a24 +; GFX908-SDAG-NEXT: s_nop 1 +; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:96 +; GFX908-SDAG-NEXT: s_nop 0 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a31 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a30 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a29 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a28 +; GFX908-SDAG-NEXT: s_nop 1 +; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:112 +; GFX908-SDAG-NEXT: s_nop 0 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a19 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a18 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a17 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a16 +; GFX908-SDAG-NEXT: s_nop 1 +; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:64 +; GFX908-SDAG-NEXT: s_nop 0 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a23 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a22 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a21 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a20 +; GFX908-SDAG-NEXT: s_nop 1 +; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:80 +; GFX908-SDAG-NEXT: s_nop 0 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a11 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a10 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a9 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a8 +; GFX908-SDAG-NEXT: s_nop 1 +; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:32 +; GFX908-SDAG-NEXT: s_nop 0 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a15 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a14 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a13 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a12 +; GFX908-SDAG-NEXT: s_nop 1 +; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:48 +; GFX908-SDAG-NEXT: s_nop 0 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-SDAG-NEXT: s_nop 1 +; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] +; GFX908-SDAG-NEXT: s_nop 0 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a7 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a6 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a5 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a4 +; GFX908-SDAG-NEXT: s_nop 1 +; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:16 +; GFX908-SDAG-NEXT: s_endpgm +; +; GFX908-GISEL-LABEL: test_mfma_f32_32x32x2bf16: +; GFX908-GISEL: ; %bb.0: ; %bb +; GFX908-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v32, 0 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0 +; GFX908-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s0 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s3 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s4 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a4, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s5 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s6 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s7 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a5, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a6, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a7, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s8 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s9 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s10 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a8, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a9, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a10, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s11 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s12 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s13 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a11, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a12, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a13, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s14 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s15 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s16 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a14, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a15, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a16, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s18 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s19 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a17, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a18, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a19, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s20 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s21 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s22 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a20, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a21, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a22, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s23 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s24 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s25 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a23, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a24, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a25, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s26 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s27 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s28 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a26, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a27, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a28, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s29 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s30 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s31 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a29, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a30, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a31, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2 +; GFX908-GISEL-NEXT: s_nop 1 +; GFX908-GISEL-NEXT: v_mfma_f32_32x32x2bf16 a[0:31], v0, v1, a[0:31] cbsz:1 abid:2 blgp:3 +; GFX908-GISEL-NEXT: s_nop 15 +; GFX908-GISEL-NEXT: s_nop 1 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v4, a4 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v5, a5 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v6, a6 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v7, a7 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v8, a8 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v9, a9 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v10, a10 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v11, a11 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v12, a12 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v13, a13 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v14, a14 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v15, a15 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v16, a16 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v17, a17 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v18, a18 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v19, a19 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v20, a20 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v21, a21 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v22, a22 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v23, a23 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v24, a24 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v25, a25 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v26, a26 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v27, a27 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v28, a28 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v29, a29 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v30, a30 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v31, a31 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35] +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112 +; GFX908-GISEL-NEXT: s_endpgm ; ; GFX90A-LABEL: test_mfma_f32_32x32x2bf16: ; GFX90A: ; %bb.0: ; %bb @@ -268,72 +390,138 @@ bb: } define amdgpu_kernel void @test_mfma_f32_16x16x2bf16(ptr addrspace(1) %arg) #0 { -; GFX908-LABEL: test_mfma_f32_16x16x2bf16: -; GFX908: ; %bb.0: ; %bb -; GFX908-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 -; GFX908-NEXT: v_mov_b32_e32 v0, 1 -; GFX908-NEXT: v_mov_b32_e32 v12, 0 -; GFX908-NEXT: s_waitcnt lgkmcnt(0) -; GFX908-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 -; GFX908-NEXT: s_waitcnt lgkmcnt(0) -; GFX908-NEXT: v_mov_b32_e32 v13, s0 -; GFX908-NEXT: v_mov_b32_e32 v1, s1 -; GFX908-NEXT: v_mov_b32_e32 v2, s2 -; GFX908-NEXT: v_accvgpr_write_b32 a0, v13 -; GFX908-NEXT: v_mov_b32_e32 v13, s3 -; GFX908-NEXT: v_accvgpr_write_b32 a1, v1 -; GFX908-NEXT: v_accvgpr_write_b32 a2, v2 -; GFX908-NEXT: v_accvgpr_write_b32 a3, v13 -; GFX908-NEXT: v_mov_b32_e32 v1, s4 -; GFX908-NEXT: v_mov_b32_e32 v2, s5 -; GFX908-NEXT: v_mov_b32_e32 v13, s6 -; GFX908-NEXT: v_accvgpr_write_b32 a4, v1 -; GFX908-NEXT: v_accvgpr_write_b32 a5, v2 -; GFX908-NEXT: v_accvgpr_write_b32 a6, v13 -; GFX908-NEXT: v_mov_b32_e32 v1, s7 -; GFX908-NEXT: v_mov_b32_e32 v2, s8 -; GFX908-NEXT: v_mov_b32_e32 v13, s9 -; GFX908-NEXT: v_accvgpr_write_b32 a7, v1 -; GFX908-NEXT: v_accvgpr_write_b32 a8, v2 -; GFX908-NEXT: v_accvgpr_write_b32 a9, v13 -; GFX908-NEXT: v_mov_b32_e32 v1, s10 -; GFX908-NEXT: v_mov_b32_e32 v2, s11 -; GFX908-NEXT: v_mov_b32_e32 v13, s12 -; GFX908-NEXT: v_accvgpr_write_b32 a10, v1 -; GFX908-NEXT: v_accvgpr_write_b32 a11, v2 -; GFX908-NEXT: v_accvgpr_write_b32 a12, v13 -; GFX908-NEXT: v_mov_b32_e32 v1, s13 -; GFX908-NEXT: v_mov_b32_e32 v2, s14 -; GFX908-NEXT: v_mov_b32_e32 v13, s15 -; GFX908-NEXT: v_accvgpr_write_b32 a13, v1 -; GFX908-NEXT: v_accvgpr_write_b32 a14, v2 -; GFX908-NEXT: v_accvgpr_write_b32 a15, v13 -; GFX908-NEXT: v_mov_b32_e32 v1, 2 -; GFX908-NEXT: s_nop 1 -; GFX908-NEXT: v_mfma_f32_16x16x2bf16 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3 -; GFX908-NEXT: s_nop 9 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a15 -; GFX908-NEXT: v_accvgpr_read_b32 v2, a14 -; GFX908-NEXT: v_accvgpr_read_b32 v1, a13 -; GFX908-NEXT: v_accvgpr_read_b32 v0, a12 -; GFX908-NEXT: v_accvgpr_read_b32 v7, a11 -; GFX908-NEXT: v_accvgpr_read_b32 v6, a10 -; GFX908-NEXT: v_accvgpr_read_b32 v5, a9 -; GFX908-NEXT: v_accvgpr_read_b32 v4, a8 -; GFX908-NEXT: v_accvgpr_read_b32 v11, a7 -; GFX908-NEXT: v_accvgpr_read_b32 v10, a6 -; GFX908-NEXT: v_accvgpr_read_b32 v9, a5 -; GFX908-NEXT: v_accvgpr_read_b32 v8, a4 -; GFX908-NEXT: global_store_dwordx4 v12, v[0:3], s[16:17] offset:48 -; GFX908-NEXT: global_store_dwordx4 v12, v[4:7], s[16:17] offset:32 -; GFX908-NEXT: global_store_dwordx4 v12, v[8:11], s[16:17] offset:16 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a3 -; GFX908-NEXT: v_accvgpr_read_b32 v2, a2 -; GFX908-NEXT: v_accvgpr_read_b32 v1, a1 -; GFX908-NEXT: v_accvgpr_read_b32 v0, a0 -; GFX908-NEXT: s_nop 1 -; GFX908-NEXT: global_store_dwordx4 v12, v[0:3], s[16:17] -; GFX908-NEXT: s_endpgm +; GFX908-SDAG-LABEL: test_mfma_f32_16x16x2bf16: +; GFX908-SDAG: ; %bb.0: ; %bb +; GFX908-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, 1 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v12, 0 +; GFX908-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX908-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-SDAG-NEXT: v_mov_b32_e32 v13, s0 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s1 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s2 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a0, v13 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v13, s3 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a1, v1 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a2, v2 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a3, v13 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s4 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s5 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v13, s6 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a4, v1 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a5, v2 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a6, v13 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s7 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s8 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v13, s9 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a7, v1 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a8, v2 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a9, v13 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s10 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s11 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v13, s12 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a10, v1 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a11, v2 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a12, v13 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s13 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s14 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v13, s15 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a13, v1 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a14, v2 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a15, v13 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, 2 +; GFX908-SDAG-NEXT: s_nop 1 +; GFX908-SDAG-NEXT: v_mfma_f32_16x16x2bf16 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3 +; GFX908-SDAG-NEXT: s_nop 9 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a15 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a14 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a13 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a12 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v7, a11 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v6, a10 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v5, a9 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v4, a8 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v11, a7 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v10, a6 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v9, a5 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v8, a4 +; GFX908-SDAG-NEXT: global_store_dwordx4 v12, v[0:3], s[16:17] offset:48 +; GFX908-SDAG-NEXT: global_store_dwordx4 v12, v[4:7], s[16:17] offset:32 +; GFX908-SDAG-NEXT: global_store_dwordx4 v12, v[8:11], s[16:17] offset:16 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-SDAG-NEXT: s_nop 1 +; GFX908-SDAG-NEXT: global_store_dwordx4 v12, v[0:3], s[16:17] +; GFX908-SDAG-NEXT: s_endpgm +; +; GFX908-GISEL-LABEL: test_mfma_f32_16x16x2bf16: +; GFX908-GISEL: ; %bb.0: ; %bb +; GFX908-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v16, 0 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s0 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s3 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v1 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s4 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s5 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s6 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a4, v1 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a5, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a6, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s7 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s8 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s9 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a7, v1 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a8, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a9, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s10 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s11 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s12 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a10, v1 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a11, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a12, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s13 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s14 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s15 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a13, v1 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a14, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a15, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2 +; GFX908-GISEL-NEXT: s_nop 1 +; GFX908-GISEL-NEXT: v_mfma_f32_16x16x2bf16 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3 +; GFX908-GISEL-NEXT: s_nop 9 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v4, a4 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v5, a5 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v6, a6 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v7, a7 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v8, a8 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v9, a9 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v10, a10 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v11, a11 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v12, a12 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v13, a13 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v14, a14 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v15, a15 +; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] +; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 +; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 +; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 +; GFX908-GISEL-NEXT: s_endpgm ; ; GFX90A-LABEL: test_mfma_f32_16x16x2bf16: ; GFX90A: ; %bb.0: ; %bb @@ -477,72 +665,139 @@ bb: } define amdgpu_kernel void @test_mfma_f32_32x32x4bf16(ptr addrspace(1) %arg) #0 { -; GFX908-LABEL: test_mfma_f32_32x32x4bf16: -; GFX908: ; %bb.0: ; %bb -; GFX908-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 -; GFX908-NEXT: v_mov_b32_e32 v0, 1 -; GFX908-NEXT: v_mov_b32_e32 v16, 0 -; GFX908-NEXT: s_waitcnt lgkmcnt(0) -; GFX908-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 -; GFX908-NEXT: s_waitcnt lgkmcnt(0) -; GFX908-NEXT: v_mov_b32_e32 v17, s0 -; GFX908-NEXT: v_mov_b32_e32 v1, s1 -; GFX908-NEXT: v_mov_b32_e32 v2, s2 -; GFX908-NEXT: v_accvgpr_write_b32 a0, v17 -; GFX908-NEXT: v_mov_b32_e32 v17, s3 -; GFX908-NEXT: v_accvgpr_write_b32 a1, v1 -; GFX908-NEXT: v_accvgpr_write_b32 a2, v2 -; GFX908-NEXT: v_accvgpr_write_b32 a3, v17 -; GFX908-NEXT: v_mov_b32_e32 v1, s4 -; GFX908-NEXT: v_mov_b32_e32 v2, s5 -; GFX908-NEXT: v_mov_b32_e32 v17, s6 -; GFX908-NEXT: v_accvgpr_write_b32 a4, v1 -; GFX908-NEXT: v_accvgpr_write_b32 a5, v2 -; GFX908-NEXT: v_accvgpr_write_b32 a6, v17 -; GFX908-NEXT: v_mov_b32_e32 v1, s7 -; GFX908-NEXT: v_mov_b32_e32 v2, s8 -; GFX908-NEXT: v_mov_b32_e32 v17, s9 -; GFX908-NEXT: v_accvgpr_write_b32 a7, v1 -; GFX908-NEXT: v_accvgpr_write_b32 a8, v2 -; GFX908-NEXT: v_accvgpr_write_b32 a9, v17 -; GFX908-NEXT: v_mov_b32_e32 v1, s10 -; GFX908-NEXT: v_mov_b32_e32 v2, s11 -; GFX908-NEXT: v_mov_b32_e32 v17, s12 -; GFX908-NEXT: v_accvgpr_write_b32 a10, v1 -; GFX908-NEXT: v_accvgpr_write_b32 a11, v2 -; GFX908-NEXT: v_accvgpr_write_b32 a12, v17 -; GFX908-NEXT: v_mov_b32_e32 v1, s13 -; GFX908-NEXT: v_mov_b32_e32 v2, s14 -; GFX908-NEXT: v_mov_b32_e32 v17, s15 -; GFX908-NEXT: v_accvgpr_write_b32 a13, v1 -; GFX908-NEXT: v_accvgpr_write_b32 a14, v2 -; GFX908-NEXT: v_accvgpr_write_b32 a15, v17 -; GFX908-NEXT: v_mov_b32_e32 v1, 2 -; GFX908-NEXT: s_nop 1 -; GFX908-NEXT: v_mfma_f32_32x32x4bf16 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3 -; GFX908-NEXT: s_nop 15 -; GFX908-NEXT: s_nop 1 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a15 -; GFX908-NEXT: v_accvgpr_read_b32 v2, a14 -; GFX908-NEXT: v_accvgpr_read_b32 v1, a13 -; GFX908-NEXT: v_accvgpr_read_b32 v0, a12 -; GFX908-NEXT: v_accvgpr_read_b32 v7, a11 -; GFX908-NEXT: v_accvgpr_read_b32 v6, a10 -; GFX908-NEXT: v_accvgpr_read_b32 v5, a9 -; GFX908-NEXT: v_accvgpr_read_b32 v4, a8 -; GFX908-NEXT: v_accvgpr_read_b32 v11, a7 -; GFX908-NEXT: v_accvgpr_read_b32 v10, a6 -; GFX908-NEXT: v_accvgpr_read_b32 v9, a5 -; GFX908-NEXT: v_accvgpr_read_b32 v8, a4 -; GFX908-NEXT: v_accvgpr_read_b32 v15, a3 -; GFX908-NEXT: v_accvgpr_read_b32 v14, a2 -; GFX908-NEXT: v_accvgpr_read_b32 v13, a1 -; GFX908-NEXT: v_accvgpr_read_b32 v12, a0 -; GFX908-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] offset:48 -; GFX908-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:32 -; GFX908-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:16 -; GFX908-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] -; GFX908-NEXT: s_endpgm +; GFX908-SDAG-LABEL: test_mfma_f32_32x32x4bf16: +; GFX908-SDAG: ; %bb.0: ; %bb +; GFX908-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, 1 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v16, 0 +; GFX908-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX908-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-SDAG-NEXT: v_mov_b32_e32 v17, s0 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s1 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s2 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a0, v17 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v17, s3 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a1, v1 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a2, v2 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a3, v17 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s4 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s5 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v17, s6 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a4, v1 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a5, v2 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a6, v17 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s7 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s8 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v17, s9 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a7, v1 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a8, v2 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a9, v17 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s10 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s11 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v17, s12 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a10, v1 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a11, v2 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a12, v17 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s13 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s14 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v17, s15 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a13, v1 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a14, v2 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a15, v17 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, 2 +; GFX908-SDAG-NEXT: s_nop 1 +; GFX908-SDAG-NEXT: v_mfma_f32_32x32x4bf16 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3 +; GFX908-SDAG-NEXT: s_nop 15 +; GFX908-SDAG-NEXT: s_nop 1 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a15 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a14 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a13 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a12 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v7, a11 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v6, a10 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v5, a9 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v4, a8 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v11, a7 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v10, a6 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v9, a5 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v8, a4 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v15, a3 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v14, a2 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v13, a1 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v12, a0 +; GFX908-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] offset:48 +; GFX908-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:32 +; GFX908-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:16 +; GFX908-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] +; GFX908-SDAG-NEXT: s_endpgm +; +; GFX908-GISEL-LABEL: test_mfma_f32_32x32x4bf16: +; GFX908-GISEL: ; %bb.0: ; %bb +; GFX908-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v16, 0 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s0 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s3 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v1 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s4 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s5 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s6 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a4, v1 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a5, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a6, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s7 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s8 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s9 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a7, v1 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a8, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a9, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s10 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s11 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s12 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a10, v1 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a11, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a12, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s13 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s14 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s15 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a13, v1 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a14, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a15, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2 +; GFX908-GISEL-NEXT: s_nop 1 +; GFX908-GISEL-NEXT: v_mfma_f32_32x32x4bf16 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3 +; GFX908-GISEL-NEXT: s_nop 15 +; GFX908-GISEL-NEXT: s_nop 1 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v4, a4 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v5, a5 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v6, a6 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v7, a7 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v8, a8 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v9, a9 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v10, a10 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v11, a11 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v12, a12 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v13, a13 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v14, a14 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v15, a15 +; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] +; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 +; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 +; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 +; GFX908-GISEL-NEXT: s_endpgm ; ; GFX90A-LABEL: test_mfma_f32_32x32x4bf16: ; GFX90A: ; %bb.0: ; %bb diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll index 61593a8a81a7c..4048f446371ef 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.i8.ll @@ -1,6 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 -; RUN: llc -mtriple=amdgcn -mcpu=gfx908 < %s | FileCheck --check-prefixes=GCN,GFX908 %s -; RUN: llc -mtriple=amdgcn -mcpu=gfx908 -mattr=-mfma-inline-literal-bug < %s | FileCheck --check-prefixes=GCN,GFX908 %s +; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx908 < %s | FileCheck --check-prefixes=GCN,GFX908,GFX908-SDAG %s +; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx908 < %s | FileCheck --check-prefixes=GCN,GFX908,GFX908-GISEL %s +; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx908 -mattr=-mfma-inline-literal-bug < %s | FileCheck --check-prefixes=GCN,GFX908,GFX908-SDAG %s +; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx908 -mattr=-mfma-inline-literal-bug < %s | FileCheck --check-prefixes=GCN,GFX908,GFX908-GISEL %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -amdgpu-mfma-vgpr-form=0 < %s | FileCheck --check-prefixes=GCN,GFX90A %s ; RUN: llc -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck --check-prefixes=GCN,GFX90A-VGPR %s @@ -8,73 +10,140 @@ declare <16 x i32> @llvm.amdgcn.mfma.i32.32x32x8i8(i32, i32, <16 x i32>, i32, i3 declare <4 x i32> @llvm.amdgcn.mfma.i32.16x16x16i8(i32, i32, <4 x i32>, i32, i32, i32) define amdgpu_kernel void @test_mfma_i32_32x32x8i8(ptr addrspace(1) %arg) #0 { -; GFX908-LABEL: test_mfma_i32_32x32x8i8: -; GFX908: ; %bb.0: ; %bb -; GFX908-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 -; GFX908-NEXT: v_mov_b32_e32 v0, 1 -; GFX908-NEXT: v_mov_b32_e32 v16, 0 -; GFX908-NEXT: s_waitcnt lgkmcnt(0) -; GFX908-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 -; GFX908-NEXT: s_waitcnt lgkmcnt(0) -; GFX908-NEXT: v_mov_b32_e32 v17, s0 -; GFX908-NEXT: v_mov_b32_e32 v1, s1 -; GFX908-NEXT: v_mov_b32_e32 v2, s2 -; GFX908-NEXT: v_accvgpr_write_b32 a0, v17 -; GFX908-NEXT: v_mov_b32_e32 v17, s3 -; GFX908-NEXT: v_accvgpr_write_b32 a1, v1 -; GFX908-NEXT: v_accvgpr_write_b32 a2, v2 -; GFX908-NEXT: v_accvgpr_write_b32 a3, v17 -; GFX908-NEXT: v_mov_b32_e32 v1, s4 -; GFX908-NEXT: v_mov_b32_e32 v2, s5 -; GFX908-NEXT: v_mov_b32_e32 v17, s6 -; GFX908-NEXT: v_accvgpr_write_b32 a4, v1 -; GFX908-NEXT: v_accvgpr_write_b32 a5, v2 -; GFX908-NEXT: v_accvgpr_write_b32 a6, v17 -; GFX908-NEXT: v_mov_b32_e32 v1, s7 -; GFX908-NEXT: v_mov_b32_e32 v2, s8 -; GFX908-NEXT: v_mov_b32_e32 v17, s9 -; GFX908-NEXT: v_accvgpr_write_b32 a7, v1 -; GFX908-NEXT: v_accvgpr_write_b32 a8, v2 -; GFX908-NEXT: v_accvgpr_write_b32 a9, v17 -; GFX908-NEXT: v_mov_b32_e32 v1, s10 -; GFX908-NEXT: v_mov_b32_e32 v2, s11 -; GFX908-NEXT: v_mov_b32_e32 v17, s12 -; GFX908-NEXT: v_accvgpr_write_b32 a10, v1 -; GFX908-NEXT: v_accvgpr_write_b32 a11, v2 -; GFX908-NEXT: v_accvgpr_write_b32 a12, v17 -; GFX908-NEXT: v_mov_b32_e32 v1, s13 -; GFX908-NEXT: v_mov_b32_e32 v2, s14 -; GFX908-NEXT: v_mov_b32_e32 v17, s15 -; GFX908-NEXT: v_accvgpr_write_b32 a13, v1 -; GFX908-NEXT: v_accvgpr_write_b32 a14, v2 -; GFX908-NEXT: v_accvgpr_write_b32 a15, v17 -; GFX908-NEXT: v_mov_b32_e32 v1, 2 -; GFX908-NEXT: s_nop 1 -; GFX908-NEXT: v_mfma_i32_32x32x8i8 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3 -; GFX908-NEXT: s_nop 15 -; GFX908-NEXT: s_nop 1 -; GFX908-NEXT: v_accvgpr_read_b32 v15, a15 -; GFX908-NEXT: v_accvgpr_read_b32 v14, a14 -; GFX908-NEXT: v_accvgpr_read_b32 v13, a13 -; GFX908-NEXT: v_accvgpr_read_b32 v12, a12 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a3 -; GFX908-NEXT: v_accvgpr_read_b32 v2, a2 -; GFX908-NEXT: v_accvgpr_read_b32 v1, a1 -; GFX908-NEXT: v_accvgpr_read_b32 v0, a0 -; GFX908-NEXT: v_accvgpr_read_b32 v7, a7 -; GFX908-NEXT: v_accvgpr_read_b32 v6, a6 -; GFX908-NEXT: v_accvgpr_read_b32 v5, a5 -; GFX908-NEXT: v_accvgpr_read_b32 v4, a4 -; GFX908-NEXT: v_accvgpr_read_b32 v11, a11 -; GFX908-NEXT: v_accvgpr_read_b32 v10, a10 -; GFX908-NEXT: v_accvgpr_read_b32 v9, a9 -; GFX908-NEXT: v_accvgpr_read_b32 v8, a8 -; GFX908-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 -; GFX908-NEXT: s_nop 0 -; GFX908-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 -; GFX908-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 -; GFX908-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] -; GFX908-NEXT: s_endpgm +; GFX908-SDAG-LABEL: test_mfma_i32_32x32x8i8: +; GFX908-SDAG: ; %bb.0: ; %bb +; GFX908-SDAG-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, 1 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v16, 0 +; GFX908-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX908-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-SDAG-NEXT: v_mov_b32_e32 v17, s0 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s1 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s2 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a0, v17 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v17, s3 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a1, v1 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a2, v2 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a3, v17 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s4 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s5 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v17, s6 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a4, v1 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a5, v2 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a6, v17 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s7 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s8 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v17, s9 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a7, v1 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a8, v2 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a9, v17 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s10 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s11 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v17, s12 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a10, v1 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a11, v2 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a12, v17 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s13 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s14 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v17, s15 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a13, v1 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a14, v2 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a15, v17 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, 2 +; GFX908-SDAG-NEXT: s_nop 1 +; GFX908-SDAG-NEXT: v_mfma_i32_32x32x8i8 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3 +; GFX908-SDAG-NEXT: s_nop 15 +; GFX908-SDAG-NEXT: s_nop 1 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v15, a15 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v14, a14 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v13, a13 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v12, a12 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v7, a7 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v6, a6 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v5, a5 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v4, a4 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v11, a11 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v10, a10 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v9, a9 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v8, a8 +; GFX908-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 +; GFX908-SDAG-NEXT: s_nop 0 +; GFX908-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 +; GFX908-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 +; GFX908-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] +; GFX908-SDAG-NEXT: s_endpgm +; +; GFX908-GISEL-LABEL: test_mfma_i32_32x32x8i8: +; GFX908-GISEL: ; %bb.0: ; %bb +; GFX908-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v16, 0 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s0 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s3 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v1 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s4 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s5 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s6 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a4, v1 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a5, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a6, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s7 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s8 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s9 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a7, v1 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a8, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a9, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s10 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s11 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s12 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a10, v1 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a11, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a12, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s13 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s14 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s15 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a13, v1 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a14, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a15, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2 +; GFX908-GISEL-NEXT: s_nop 1 +; GFX908-GISEL-NEXT: v_mfma_i32_32x32x8i8 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3 +; GFX908-GISEL-NEXT: s_nop 15 +; GFX908-GISEL-NEXT: s_nop 1 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v4, a4 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v5, a5 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v6, a6 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v7, a7 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v8, a8 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v9, a9 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v10, a10 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v11, a11 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v12, a12 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v13, a13 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v14, a14 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v15, a15 +; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] +; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 +; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 +; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 +; GFX908-GISEL-NEXT: s_endpgm ; ; GFX90A-LABEL: test_mfma_i32_32x32x8i8: ; GFX90A: ; %bb.0: ; %bb diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll index 0161cdf03deac..8152b5e38477c 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll @@ -1,6 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 -; RUN: llc -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx908 < %s | FileCheck -enable-var-scope --check-prefixes=GCN,NOLIT-SRCC,GFX908,GFX908_A %s -; RUN: llc -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx908 -mattr=-mfma-inline-literal-bug < %s | FileCheck -enable-var-scope --check-prefixes=GCN,LIT-SRCC,GFX908,GFX908_A %s +; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx908 < %s | FileCheck -enable-var-scope --check-prefixes=GCN,NOLIT-SRCC,GFX908,GFX908_A %s +; RUN: llc -global-isel=1 -new-reg-bank-select -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx908 < %s | FileCheck -enable-var-scope --check-prefixes=GFX908-GISEL,GFX908-GISEL-NOLIT-SRCC %s +; RUN: llc -global-isel=0 -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx908 -mattr=-mfma-inline-literal-bug < %s | FileCheck -enable-var-scope --check-prefixes=GCN,LIT-SRCC,GFX908,GFX908_A %s +; RUN: llc -global-isel=1 -new-reg-bank-select -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx908 -mattr=-mfma-inline-literal-bug < %s | FileCheck -enable-var-scope --check-prefixes=GFX908-GISEL,GFX908-GISEL-LIT-SRCC %s ; RUN: llc -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX90A,GFX908_A,GFX90A_42 %s ; RUN: llc -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GFX942,GFX90A_42 %s ; RUN: llc -verify-machineinstrs -mtriple=amdgcn -mcpu=gfx942 -amdgpu-mfma-vgpr-form < %s | FileCheck -enable-var-scope --check-prefix=GFX942-VGPR %s @@ -156,6 +158,126 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32(ptr addrspace(1) %arg) #0 { ; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:16 ; NOLIT-SRCC-NEXT: s_endpgm ; +; GFX908-GISEL-LABEL: test_mfma_f32_32x32x1f32: +; GFX908-GISEL: ; %bb.0: ; %bb +; GFX908-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1.0 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v32, 0 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0 +; GFX908-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s0 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s3 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s4 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a4, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s5 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s6 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s7 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a5, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a6, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a7, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s8 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s9 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s10 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a8, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a9, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a10, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s11 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s12 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s13 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a11, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a12, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a13, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s14 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s15 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s16 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a14, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a15, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a16, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s18 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s19 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a17, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a18, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a19, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s20 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s21 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s22 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a20, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a21, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a22, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s23 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s24 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s25 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a23, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a24, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a25, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s26 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s27 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s28 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a26, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a27, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a28, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s29 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s30 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s31 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a29, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a30, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a31, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2.0 +; GFX908-GISEL-NEXT: s_nop 1 +; GFX908-GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31] cbsz:1 abid:2 blgp:3 +; GFX908-GISEL-NEXT: s_nop 15 +; GFX908-GISEL-NEXT: s_nop 1 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v4, a4 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v5, a5 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v6, a6 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v7, a7 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v8, a8 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v9, a9 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v10, a10 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v11, a11 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v12, a12 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v13, a13 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v14, a14 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v15, a15 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v16, a16 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v17, a17 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v18, a18 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v19, a19 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v20, a20 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v21, a21 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v22, a22 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v23, a23 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v24, a24 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v25, a25 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v26, a26 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v27, a27 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v28, a28 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v29, a29 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v30, a30 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v31, a31 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35] +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112 +; GFX908-GISEL-NEXT: s_endpgm +; ; LIT-SRCC-LABEL: test_mfma_f32_32x32x1f32: ; LIT-SRCC: ; %bb.0: ; %bb ; LIT-SRCC-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24 @@ -533,6 +655,72 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32(ptr addrspace(1) %arg) #0 { ; NOLIT-SRCC-NEXT: global_store_dwordx4 v12, v[0:3], s[16:17] ; NOLIT-SRCC-NEXT: s_endpgm ; +; GFX908-GISEL-LABEL: test_mfma_f32_16x16x1f32: +; GFX908-GISEL: ; %bb.0: ; %bb +; GFX908-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1.0 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v16, 0 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s0 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s3 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v1 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s4 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s5 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s6 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a4, v1 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a5, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a6, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s7 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s8 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s9 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a7, v1 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a8, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a9, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s10 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s11 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s12 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a10, v1 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a11, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a12, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s13 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s14 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s15 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a13, v1 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a14, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a15, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2.0 +; GFX908-GISEL-NEXT: s_nop 1 +; GFX908-GISEL-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3 +; GFX908-GISEL-NEXT: s_nop 9 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v4, a4 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v5, a5 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v6, a6 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v7, a7 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v8, a8 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v9, a9 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v10, a10 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v11, a11 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v12, a12 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v13, a13 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v14, a14 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v15, a15 +; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] +; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 +; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 +; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 +; GFX908-GISEL-NEXT: s_endpgm +; ; LIT-SRCC-LABEL: test_mfma_f32_16x16x1f32: ; LIT-SRCC: ; %bb.0: ; %bb ; LIT-SRCC-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 @@ -713,6 +901,34 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32(ptr addrspace(1) %arg) #0 { ; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7] ; NOLIT-SRCC-NEXT: s_endpgm ; +; GFX908-GISEL-LABEL: test_mfma_f32_4x4x1f32: +; GFX908-GISEL: ; %bb.0: ; %bb +; GFX908-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1.0 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2.0 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v4, 0 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: v_mov_b32_e32 v5, s0 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v5 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v5, s3 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v3 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v5 +; GFX908-GISEL-NEXT: s_nop 0 +; GFX908-GISEL-NEXT: v_mfma_f32_4x4x1f32 a[0:3], v0, v1, a[0:3] cbsz:1 abid:2 blgp:3 +; GFX908-GISEL-NEXT: s_nop 3 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-GISEL-NEXT: s_nop 1 +; GFX908-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7] +; GFX908-GISEL-NEXT: s_endpgm +; ; LIT-SRCC-LABEL: test_mfma_f32_4x4x1f32: ; LIT-SRCC: ; %bb.0: ; %bb ; LIT-SRCC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 @@ -866,6 +1082,73 @@ define amdgpu_kernel void @test_mfma_f32_32x32x2f32(ptr addrspace(1) %arg) #0 { ; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] ; NOLIT-SRCC-NEXT: s_endpgm ; +; GFX908-GISEL-LABEL: test_mfma_f32_32x32x2f32: +; GFX908-GISEL: ; %bb.0: ; %bb +; GFX908-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1.0 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v16, 0 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s0 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s3 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v1 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s4 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s5 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s6 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a4, v1 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a5, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a6, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s7 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s8 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s9 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a7, v1 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a8, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a9, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s10 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s11 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s12 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a10, v1 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a11, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a12, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s13 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s14 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s15 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a13, v1 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a14, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a15, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2.0 +; GFX908-GISEL-NEXT: s_nop 1 +; GFX908-GISEL-NEXT: v_mfma_f32_32x32x2f32 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3 +; GFX908-GISEL-NEXT: s_nop 15 +; GFX908-GISEL-NEXT: s_nop 1 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v4, a4 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v5, a5 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v6, a6 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v7, a7 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v8, a8 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v9, a9 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v10, a10 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v11, a11 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v12, a12 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v13, a13 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v14, a14 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v15, a15 +; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] +; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 +; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 +; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 +; GFX908-GISEL-NEXT: s_endpgm +; ; LIT-SRCC-LABEL: test_mfma_f32_32x32x2f32: ; LIT-SRCC: ; %bb.0: ; %bb ; LIT-SRCC-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 @@ -1049,6 +1332,34 @@ define amdgpu_kernel void @test_mfma_f32_16x16x4f32(ptr addrspace(1) %arg) #0 { ; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7] ; NOLIT-SRCC-NEXT: s_endpgm ; +; GFX908-GISEL-LABEL: test_mfma_f32_16x16x4f32: +; GFX908-GISEL: ; %bb.0: ; %bb +; GFX908-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1.0 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2.0 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v4, 0 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: v_mov_b32_e32 v5, s0 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v5 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v5, s3 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v3 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v5 +; GFX908-GISEL-NEXT: s_nop 0 +; GFX908-GISEL-NEXT: v_mfma_f32_16x16x4f32 a[0:3], v0, v1, a[0:3] cbsz:1 abid:2 blgp:3 +; GFX908-GISEL-NEXT: s_nop 9 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-GISEL-NEXT: s_nop 1 +; GFX908-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7] +; GFX908-GISEL-NEXT: s_endpgm +; ; LIT-SRCC-LABEL: test_mfma_f32_16x16x4f32: ; LIT-SRCC: ; %bb.0: ; %bb ; LIT-SRCC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 @@ -1274,6 +1585,129 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a ; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:16 ; NOLIT-SRCC-NEXT: s_endpgm ; +; GFX908-GISEL-LABEL: test_mfma_f32_32x32x4f16: +; GFX908-GISEL: ; %bb.0: ; %bb +; GFX908-GISEL-NEXT: s_load_dwordx4 s[36:39], s[4:5], 0x24 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v32, 0 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: s_load_dwordx4 s[40:43], s[38:39], 0x0 +; GFX908-GISEL-NEXT: s_load_dwordx16 s[0:15], s[36:37], 0x0 +; GFX908-GISEL-NEXT: s_load_dwordx16 s[16:31], s[36:37], 0x40 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, s40 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s0 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v2 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s3 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s4 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v3 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a4, v2 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s5 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s6 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s7 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a5, v3 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a6, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a7, v2 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s8 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s9 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s10 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a8, v3 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a9, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a10, v2 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s11 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s12 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s13 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a11, v3 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a12, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a13, v2 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s14 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s15 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s16 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a14, v3 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a15, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a16, v2 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s18 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s19 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a17, v3 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a18, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a19, v2 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s20 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s21 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s22 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a20, v3 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a21, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a22, v2 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s23 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s24 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s25 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a23, v3 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a24, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a25, v2 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s26 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s27 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s28 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s41 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a26, v3 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a27, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a28, v2 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s29 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s30 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s31 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a29, v3 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a30, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a31, v2 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s42 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s43 +; GFX908-GISEL-NEXT: s_nop 1 +; GFX908-GISEL-NEXT: v_mfma_f32_32x32x4f16 a[0:31], v[0:1], v[2:3], a[0:31] cbsz:1 abid:2 blgp:3 +; GFX908-GISEL-NEXT: s_nop 15 +; GFX908-GISEL-NEXT: s_nop 1 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v4, a4 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v5, a5 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v6, a6 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v7, a7 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v8, a8 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v9, a9 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v10, a10 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v11, a11 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v12, a12 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v13, a13 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v14, a14 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v15, a15 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v16, a16 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v17, a17 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v18, a18 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v19, a19 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v20, a20 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v21, a21 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v22, a22 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v23, a23 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v24, a24 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v25, a25 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v26, a26 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v27, a27 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v28, a28 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v29, a29 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v30, a30 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v31, a31 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[36:37] +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[36:37] offset:16 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[36:37] offset:32 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[36:37] offset:48 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[36:37] offset:64 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[36:37] offset:80 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[36:37] offset:96 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[36:37] offset:112 +; GFX908-GISEL-NEXT: s_endpgm +; ; LIT-SRCC-LABEL: test_mfma_f32_32x32x4f16: ; LIT-SRCC: ; %bb.0: ; %bb ; LIT-SRCC-NEXT: s_load_dwordx4 s[36:39], s[4:5], 0x24 @@ -1673,6 +2107,75 @@ define amdgpu_kernel void @test_mfma_f32_16x16x4f16(ptr addrspace(1) %arg, ptr a ; NOLIT-SRCC-NEXT: global_store_dwordx4 v12, v[0:3], s[16:17] ; NOLIT-SRCC-NEXT: s_endpgm ; +; GFX908-GISEL-LABEL: test_mfma_f32_16x16x4f16: +; GFX908-GISEL: ; %bb.0: ; %bb +; GFX908-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v16, 0 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: s_load_dwordx4 s[20:23], s[18:19], 0x0 +; GFX908-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, s20 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s0 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s3 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v3 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s4 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s5 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s6 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a4, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a5, v3 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a6, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s7 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s8 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s9 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a7, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a8, v3 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a9, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s10 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s11 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s12 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s21 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a10, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a11, v3 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a12, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s13 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s14 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s15 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a13, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a14, v3 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a15, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s22 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s23 +; GFX908-GISEL-NEXT: s_nop 1 +; GFX908-GISEL-NEXT: v_mfma_f32_16x16x4f16 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX908-GISEL-NEXT: s_nop 9 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v4, a4 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v5, a5 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v6, a6 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v7, a7 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v8, a8 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v9, a9 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v10, a10 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v11, a11 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v12, a12 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v13, a13 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v14, a14 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v15, a15 +; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] +; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 +; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 +; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 +; GFX908-GISEL-NEXT: s_endpgm +; ; LIT-SRCC-LABEL: test_mfma_f32_16x16x4f16: ; LIT-SRCC: ; %bb.0: ; %bb ; LIT-SRCC-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24 @@ -1871,6 +2374,37 @@ define amdgpu_kernel void @test_mfma_f32_4x4x4f16(ptr addrspace(1) %arg, ptr add ; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] ; NOLIT-SRCC-NEXT: s_endpgm ; +; GFX908-GISEL-LABEL: test_mfma_f32_4x4x4f16: +; GFX908-GISEL: ; %bb.0: ; %bb +; GFX908-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0 +; GFX908-GISEL-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, s4 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v5, s8 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s5 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v4, s9 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v5 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v6, s10 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v5, s11 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s6 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v4 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v6 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v5 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s7 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v4, 0 +; GFX908-GISEL-NEXT: s_nop 0 +; GFX908-GISEL-NEXT: v_mfma_f32_4x4x4f16 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX908-GISEL-NEXT: s_nop 3 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-GISEL-NEXT: s_nop 1 +; GFX908-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] +; GFX908-GISEL-NEXT: s_endpgm +; ; LIT-SRCC-LABEL: test_mfma_f32_4x4x4f16: ; LIT-SRCC: ; %bb.0: ; %bb ; LIT-SRCC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 @@ -2042,6 +2576,76 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8f16(ptr addrspace(1) %arg, ptr a ; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] ; NOLIT-SRCC-NEXT: s_endpgm ; +; GFX908-GISEL-LABEL: test_mfma_f32_32x32x8f16: +; GFX908-GISEL: ; %bb.0: ; %bb +; GFX908-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v16, 0 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: s_load_dwordx4 s[20:23], s[18:19], 0x0 +; GFX908-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, s20 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s0 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s3 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v3 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s4 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s5 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s6 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a4, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a5, v3 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a6, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s7 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s8 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s9 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a7, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a8, v3 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a9, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s10 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s11 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s12 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s21 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a10, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a11, v3 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a12, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s13 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s14 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s15 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a13, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a14, v3 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a15, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s22 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s23 +; GFX908-GISEL-NEXT: s_nop 1 +; GFX908-GISEL-NEXT: v_mfma_f32_32x32x8f16 a[0:15], v[0:1], v[2:3], a[0:15] cbsz:1 abid:2 blgp:3 +; GFX908-GISEL-NEXT: s_nop 15 +; GFX908-GISEL-NEXT: s_nop 1 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v4, a4 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v5, a5 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v6, a6 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v7, a7 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v8, a8 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v9, a9 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v10, a10 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v11, a11 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v12, a12 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v13, a13 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v14, a14 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v15, a15 +; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] +; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 +; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 +; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 +; GFX908-GISEL-NEXT: s_endpgm +; ; LIT-SRCC-LABEL: test_mfma_f32_32x32x8f16: ; LIT-SRCC: ; %bb.0: ; %bb ; LIT-SRCC-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24 @@ -2241,6 +2845,37 @@ define amdgpu_kernel void @test_mfma_f32_16x16x16f16(ptr addrspace(1) %arg, ptr ; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] ; NOLIT-SRCC-NEXT: s_endpgm ; +; GFX908-GISEL-LABEL: test_mfma_f32_16x16x16f16: +; GFX908-GISEL: ; %bb.0: ; %bb +; GFX908-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0 +; GFX908-GISEL-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x0 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, s4 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v5, s8 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s5 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v4, s9 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v5 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v6, s10 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v5, s11 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s6 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v4 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v6 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v5 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s7 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v4, 0 +; GFX908-GISEL-NEXT: s_nop 0 +; GFX908-GISEL-NEXT: v_mfma_f32_16x16x16f16 a[0:3], v[0:1], v[2:3], a[0:3] cbsz:1 abid:2 blgp:3 +; GFX908-GISEL-NEXT: s_nop 9 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-GISEL-NEXT: s_nop 1 +; GFX908-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] +; GFX908-GISEL-NEXT: s_endpgm +; ; LIT-SRCC-LABEL: test_mfma_f32_16x16x16f16: ; LIT-SRCC: ; %bb.0: ; %bb ; LIT-SRCC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24 @@ -2463,6 +3098,126 @@ define amdgpu_kernel void @test_mfma_i32_32x32x4i8(ptr addrspace(1) %arg) #0 { ; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[0:3], s[34:35] offset:16 ; NOLIT-SRCC-NEXT: s_endpgm ; +; GFX908-GISEL-LABEL: test_mfma_i32_32x32x4i8: +; GFX908-GISEL: ; %bb.0: ; %bb +; GFX908-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v32, 0 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0 +; GFX908-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s0 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s3 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s4 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a4, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s5 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s6 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s7 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a5, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a6, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a7, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s8 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s9 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s10 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a8, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a9, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a10, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s11 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s12 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s13 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a11, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a12, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a13, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s14 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s15 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s16 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a14, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a15, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a16, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s18 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s19 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a17, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a18, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a19, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s20 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s21 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s22 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a20, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a21, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a22, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s23 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s24 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s25 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a23, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a24, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a25, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s26 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s27 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s28 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a26, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a27, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a28, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s29 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s30 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s31 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a29, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a30, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a31, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2 +; GFX908-GISEL-NEXT: s_nop 1 +; GFX908-GISEL-NEXT: v_mfma_i32_32x32x4i8 a[0:31], v0, v1, a[0:31] cbsz:1 abid:2 blgp:3 +; GFX908-GISEL-NEXT: s_nop 15 +; GFX908-GISEL-NEXT: s_nop 1 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v4, a4 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v5, a5 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v6, a6 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v7, a7 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v8, a8 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v9, a9 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v10, a10 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v11, a11 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v12, a12 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v13, a13 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v14, a14 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v15, a15 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v16, a16 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v17, a17 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v18, a18 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v19, a19 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v20, a20 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v21, a21 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v22, a22 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v23, a23 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v24, a24 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v25, a25 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v26, a26 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v27, a27 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v28, a28 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v29, a29 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v30, a30 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v31, a31 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35] +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112 +; GFX908-GISEL-NEXT: s_endpgm +; ; LIT-SRCC-LABEL: test_mfma_i32_32x32x4i8: ; LIT-SRCC: ; %bb.0: ; %bb ; LIT-SRCC-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24 @@ -2826,6 +3581,72 @@ define amdgpu_kernel void @test_mfma_i32_16x16x4i8(ptr addrspace(1) %arg) #0 { ; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] ; NOLIT-SRCC-NEXT: s_endpgm ; +; GFX908-GISEL-LABEL: test_mfma_i32_16x16x4i8: +; GFX908-GISEL: ; %bb.0: ; %bb +; GFX908-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v16, 0 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s0 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s3 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v1 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s4 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s5 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s6 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a4, v1 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a5, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a6, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s7 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s8 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s9 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a7, v1 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a8, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a9, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s10 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s11 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s12 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a10, v1 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a11, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a12, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s13 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s14 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s15 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a13, v1 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a14, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a15, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2 +; GFX908-GISEL-NEXT: s_nop 1 +; GFX908-GISEL-NEXT: v_mfma_i32_16x16x4i8 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3 +; GFX908-GISEL-NEXT: s_nop 9 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v4, a4 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v5, a5 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v6, a6 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v7, a7 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v8, a8 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v9, a9 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v10, a10 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v11, a11 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v12, a12 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v13, a13 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v14, a14 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v15, a15 +; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] +; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 +; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 +; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 +; GFX908-GISEL-NEXT: s_endpgm +; ; LIT-SRCC-LABEL: test_mfma_i32_16x16x4i8: ; LIT-SRCC: ; %bb.0: ; %bb ; LIT-SRCC-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 @@ -3025,6 +3846,53 @@ define amdgpu_kernel void @test_mfma_i32_16x16x4i8_splatimm_src2_64(ptr addrspac ; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1] ; NOLIT-SRCC-NEXT: s_endpgm ; +; GFX908-GISEL-NOLIT-SRCC-LABEL: test_mfma_i32_16x16x4i8_splatimm_src2_64: +; GFX908-GISEL-NOLIT-SRCC: ; %bb.0: ; %bb +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, 1 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a0, 64 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a1, 64 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a2, 64 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a3, 64 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a4, 64 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a5, 64 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a6, 64 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a7, 64 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a8, 64 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a9, 64 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a10, 64 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a11, 64 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a12, 64 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a13, 64 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a14, 64 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a15, 64 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, 2 +; GFX908-GISEL-NOLIT-SRCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mov_b32_e32 v16, 0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mfma_i32_16x16x4i8 a[0:15], v0, v1, a[0:15] cbsz:1 abid:2 blgp:3 +; GFX908-GISEL-NOLIT-SRCC-NEXT: s_nop 9 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a4 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a5 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a6 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a7 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a8 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a9 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a10 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a11 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a12 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a13 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a14 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a15 +; GFX908-GISEL-NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1] +; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16 +; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32 +; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48 +; GFX908-GISEL-NOLIT-SRCC-NEXT: s_endpgm +; ; LIT-SRCC-LABEL: test_mfma_i32_16x16x4i8_splatimm_src2_64: ; LIT-SRCC: ; %bb.0: ; %bb ; LIT-SRCC-NEXT: v_mov_b32_e32 v0, 1 @@ -3056,6 +3924,37 @@ define amdgpu_kernel void @test_mfma_i32_16x16x4i8_splatimm_src2_64(ptr addrspac ; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1] ; LIT-SRCC-NEXT: s_endpgm ; +; GFX908-GISEL-LIT-SRCC-LABEL: test_mfma_i32_16x16x4i8_splatimm_src2_64: +; GFX908-GISEL-LIT-SRCC: ; %bb.0: ; %bb +; GFX908-GISEL-LIT-SRCC-NEXT: v_mov_b32_e32 v0, 1 +; GFX908-GISEL-LIT-SRCC-NEXT: v_mov_b32_e32 v1, 2 +; GFX908-GISEL-LIT-SRCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX908-GISEL-LIT-SRCC-NEXT: v_mov_b32_e32 v16, 0 +; GFX908-GISEL-LIT-SRCC-NEXT: v_mfma_i32_16x16x4i8 a[0:15], v0, v1, 64 cbsz:1 abid:2 blgp:3 +; GFX908-GISEL-LIT-SRCC-NEXT: s_nop 9 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a4 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a5 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a6 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a7 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a8 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a9 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a10 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a11 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a12 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a13 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a14 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a15 +; GFX908-GISEL-LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1] +; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16 +; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32 +; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48 +; GFX908-GISEL-LIT-SRCC-NEXT: s_endpgm +; ; GFX90A-LABEL: test_mfma_i32_16x16x4i8_splatimm_src2_64: ; GFX90A: ; %bb.0: ; %bb ; GFX90A-NEXT: v_mov_b32_e32 v16, 1 @@ -3139,6 +4038,34 @@ define amdgpu_kernel void @test_mfma_i32_4x4x4i8(ptr addrspace(1) %arg) #0 { ; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7] ; NOLIT-SRCC-NEXT: s_endpgm ; +; GFX908-GISEL-LABEL: test_mfma_i32_4x4x4i8: +; GFX908-GISEL: ; %bb.0: ; %bb +; GFX908-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v4, 0 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: v_mov_b32_e32 v5, s0 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v5 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v5, s3 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v3 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v5 +; GFX908-GISEL-NEXT: s_nop 0 +; GFX908-GISEL-NEXT: v_mfma_i32_4x4x4i8 a[0:3], v0, v1, a[0:3] cbsz:1 abid:2 blgp:3 +; GFX908-GISEL-NEXT: s_nop 3 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-GISEL-NEXT: s_nop 1 +; GFX908-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7] +; GFX908-GISEL-NEXT: s_endpgm +; ; LIT-SRCC-LABEL: test_mfma_i32_4x4x4i8: ; LIT-SRCC: ; %bb.0: ; %bb ; LIT-SRCC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 @@ -3247,6 +4174,28 @@ define amdgpu_kernel void @test_mfma_i32_4x4x4i8_splat_imm_src2_1(ptr addrspace( ; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] ; NOLIT-SRCC-NEXT: s_endpgm ; +; GFX908-GISEL-NOLIT-SRCC-LABEL: test_mfma_i32_4x4x4i8_splat_imm_src2_1: +; GFX908-GISEL-NOLIT-SRCC: ; %bb.0: ; %bb +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, 1 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a0, 1 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a1, 1 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a2, 1 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a3, 1 +; GFX908-GISEL-NOLIT-SRCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, 2 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mov_b32_e32 v4, 0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: s_nop 0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mfma_i32_4x4x4i8 a[0:3], v0, v1, a[0:3] cbsz:1 abid:2 blgp:3 +; GFX908-GISEL-NOLIT-SRCC-NEXT: s_nop 3 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-GISEL-NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NOLIT-SRCC-NEXT: s_nop 0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] +; GFX908-GISEL-NOLIT-SRCC-NEXT: s_endpgm +; ; LIT-SRCC-LABEL: test_mfma_i32_4x4x4i8_splat_imm_src2_1: ; LIT-SRCC: ; %bb.0: ; %bb ; LIT-SRCC-NEXT: v_mov_b32_e32 v0, 1 @@ -3265,6 +4214,23 @@ define amdgpu_kernel void @test_mfma_i32_4x4x4i8_splat_imm_src2_1(ptr addrspace( ; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] ; LIT-SRCC-NEXT: s_endpgm ; +; GFX908-GISEL-LIT-SRCC-LABEL: test_mfma_i32_4x4x4i8_splat_imm_src2_1: +; GFX908-GISEL-LIT-SRCC: ; %bb.0: ; %bb +; GFX908-GISEL-LIT-SRCC-NEXT: v_mov_b32_e32 v0, 1 +; GFX908-GISEL-LIT-SRCC-NEXT: v_mov_b32_e32 v1, 2 +; GFX908-GISEL-LIT-SRCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX908-GISEL-LIT-SRCC-NEXT: v_mov_b32_e32 v4, 0 +; GFX908-GISEL-LIT-SRCC-NEXT: v_mfma_i32_4x4x4i8 a[0:3], v0, v1, 1 cbsz:1 abid:2 blgp:3 +; GFX908-GISEL-LIT-SRCC-NEXT: s_nop 3 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-GISEL-LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-LIT-SRCC-NEXT: s_nop 0 +; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] +; GFX908-GISEL-LIT-SRCC-NEXT: s_endpgm +; ; GFX90A-LABEL: test_mfma_i32_4x4x4i8_splat_imm_src2_1: ; GFX90A: ; %bb.0: ; %bb ; GFX90A-NEXT: v_mov_b32_e32 v0, 1 @@ -3334,6 +4300,33 @@ define amdgpu_kernel void @test_mfma_i32_4x4x4i8_splat_k_src2_1(ptr addrspace(1) ; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] ; NOLIT-SRCC-NEXT: s_endpgm ; +; GFX908-GISEL-LABEL: test_mfma_i32_4x4x4i8_splat_k_src2_1: +; GFX908-GISEL: ; %bb.0: +; GFX908-GISEL-NEXT: s_movk_i32 s2, 0x41 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v5, s2 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v5 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s2 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v5, s2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v3 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v5 +; GFX908-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v4, 0 +; GFX908-GISEL-NEXT: s_nop 0 +; GFX908-GISEL-NEXT: v_mfma_i32_4x4x4i8 a[0:3], v0, v1, a[0:3] cbsz:1 abid:2 blgp:3 +; GFX908-GISEL-NEXT: s_nop 3 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: s_nop 0 +; GFX908-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] +; GFX908-GISEL-NEXT: s_endpgm +; ; LIT-SRCC-LABEL: test_mfma_i32_4x4x4i8_splat_k_src2_1: ; LIT-SRCC: ; %bb.0: ; LIT-SRCC-NEXT: v_mov_b32_e32 v0, 0x41 @@ -3550,6 +4543,127 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_forward_acc(ptr addrspace(1) ; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:16 ; NOLIT-SRCC-NEXT: s_endpgm ; +; GFX908-GISEL-LABEL: test_mfma_f32_32x32x1f32_forward_acc: +; GFX908-GISEL: ; %bb.0: ; %bb +; GFX908-GISEL-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1.0 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v32, 0 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x0 +; GFX908-GISEL-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x40 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s0 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s3 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s4 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a4, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s5 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s6 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s7 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a5, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a6, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a7, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s8 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s9 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s10 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a8, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a9, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a10, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s11 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s12 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s13 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a11, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a12, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a13, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s14 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s15 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s16 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a14, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a15, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a16, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s18 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s19 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a17, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a18, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a19, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s20 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s21 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s22 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a20, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a21, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a22, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s23 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s24 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s25 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a23, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a24, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a25, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s26 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s27 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s28 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a26, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a27, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a28, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s29 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v33, s30 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s31 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a29, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a30, v33 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a31, v1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2.0 +; GFX908-GISEL-NEXT: s_nop 1 +; GFX908-GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31] +; GFX908-GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31] +; GFX908-GISEL-NEXT: s_nop 15 +; GFX908-GISEL-NEXT: s_nop 1 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v4, a4 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v5, a5 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v6, a6 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v7, a7 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v8, a8 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v9, a9 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v10, a10 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v11, a11 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v12, a12 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v13, a13 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v14, a14 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v15, a15 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v16, a16 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v17, a17 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v18, a18 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v19, a19 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v20, a20 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v21, a21 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v22, a22 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v23, a23 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v24, a24 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v25, a25 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v26, a26 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v27, a27 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v28, a28 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v29, a29 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v30, a30 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v31, a31 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35] +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:16 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:32 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:48 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:64 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:80 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] offset:96 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:112 +; GFX908-GISEL-NEXT: s_endpgm +; ; LIT-SRCC-LABEL: test_mfma_f32_32x32x1f32_forward_acc: ; LIT-SRCC: ; %bb.0: ; %bb ; LIT-SRCC-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24 @@ -3932,6 +5046,73 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_forward_acc(ptr addrspace(1) ; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] ; NOLIT-SRCC-NEXT: s_endpgm ; +; GFX908-GISEL-LABEL: test_mfma_f32_16x16x1f32_forward_acc: +; GFX908-GISEL: ; %bb.0: ; %bb +; GFX908-GISEL-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1.0 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v16, 0 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s0 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s3 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v1 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s4 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s5 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s6 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a4, v1 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a5, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a6, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s7 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s8 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s9 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a7, v1 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a8, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a9, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s10 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s11 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s12 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a10, v1 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a11, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a12, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, s13 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s14 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v17, s15 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a13, v1 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a14, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a15, v17 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2.0 +; GFX908-GISEL-NEXT: s_nop 1 +; GFX908-GISEL-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v0, v1, a[0:15] +; GFX908-GISEL-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v0, v1, a[0:15] +; GFX908-GISEL-NEXT: s_nop 9 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v4, a4 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v5, a5 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v6, a6 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v7, a7 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v8, a8 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v9, a9 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v10, a10 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v11, a11 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v12, a12 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v13, a13 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v14, a14 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v15, a15 +; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] +; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 +; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 +; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 +; GFX908-GISEL-NEXT: s_endpgm +; ; LIT-SRCC-LABEL: test_mfma_f32_16x16x1f32_forward_acc: ; LIT-SRCC: ; %bb.0: ; %bb ; LIT-SRCC-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24 @@ -4117,6 +5298,35 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_forward_acc(ptr addrspace(1) % ; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7] ; NOLIT-SRCC-NEXT: s_endpgm ; +; GFX908-GISEL-LABEL: test_mfma_f32_4x4x1f32_forward_acc: +; GFX908-GISEL: ; %bb.0: ; %bb +; GFX908-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1.0 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2.0 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v4, 0 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: v_mov_b32_e32 v5, s0 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s1 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v5 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v5, s3 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v3 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v5 +; GFX908-GISEL-NEXT: s_nop 0 +; GFX908-GISEL-NEXT: v_mfma_f32_4x4x1f32 a[0:3], v0, v1, a[0:3] +; GFX908-GISEL-NEXT: v_mfma_f32_4x4x1f32 a[0:3], v0, v1, a[0:3] +; GFX908-GISEL-NEXT: s_nop 3 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-GISEL-NEXT: s_nop 1 +; GFX908-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7] +; GFX908-GISEL-NEXT: s_endpgm +; ; LIT-SRCC-LABEL: test_mfma_f32_4x4x1f32_forward_acc: ; LIT-SRCC: ; %bb.0: ; %bb ; LIT-SRCC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 @@ -4232,6 +5442,28 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_imm_splat(ptr addrspace(1) %ar ; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] ; NOLIT-SRCC-NEXT: s_endpgm ; +; GFX908-GISEL-NOLIT-SRCC-LABEL: test_mfma_f32_4x4x1f32_imm_splat: +; GFX908-GISEL-NOLIT-SRCC: ; %bb.0: ; %bb +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, 1.0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a0, 1.0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a1, 1.0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a2, 1.0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a3, 1.0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, 2.0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mov_b32_e32 v4, 0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: s_nop 0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mfma_f32_4x4x1f32 a[0:3], v0, v1, a[0:3] +; GFX908-GISEL-NOLIT-SRCC-NEXT: s_nop 3 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-GISEL-NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NOLIT-SRCC-NEXT: s_nop 0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] +; GFX908-GISEL-NOLIT-SRCC-NEXT: s_endpgm +; ; LIT-SRCC-LABEL: test_mfma_f32_4x4x1f32_imm_splat: ; LIT-SRCC: ; %bb.0: ; %bb ; LIT-SRCC-NEXT: v_mov_b32_e32 v0, 1.0 @@ -4250,6 +5482,23 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_imm_splat(ptr addrspace(1) %ar ; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] ; LIT-SRCC-NEXT: s_endpgm ; +; GFX908-GISEL-LIT-SRCC-LABEL: test_mfma_f32_4x4x1f32_imm_splat: +; GFX908-GISEL-LIT-SRCC: ; %bb.0: ; %bb +; GFX908-GISEL-LIT-SRCC-NEXT: v_mov_b32_e32 v0, 1.0 +; GFX908-GISEL-LIT-SRCC-NEXT: v_mov_b32_e32 v1, 2.0 +; GFX908-GISEL-LIT-SRCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX908-GISEL-LIT-SRCC-NEXT: v_mov_b32_e32 v4, 0 +; GFX908-GISEL-LIT-SRCC-NEXT: v_mfma_f32_4x4x1f32 a[0:3], v0, v1, 1.0 +; GFX908-GISEL-LIT-SRCC-NEXT: s_nop 3 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-GISEL-LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-LIT-SRCC-NEXT: s_nop 0 +; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] +; GFX908-GISEL-LIT-SRCC-NEXT: s_endpgm +; ; GFX90A-LABEL: test_mfma_f32_4x4x1f32_imm_splat: ; GFX90A: ; %bb.0: ; %bb ; GFX90A-NEXT: v_mov_b32_e32 v0, 1.0 @@ -4349,6 +5598,53 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_imm_splat(ptr addrspace(1) % ; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] ; NOLIT-SRCC-NEXT: s_endpgm ; +; GFX908-GISEL-NOLIT-SRCC-LABEL: test_mfma_f32_16x16x1f32_imm_splat: +; GFX908-GISEL-NOLIT-SRCC: ; %bb.0: ; %bb +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, 1.0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a0, 1.0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a1, 1.0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a2, 1.0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a3, 1.0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a4, 1.0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a5, 1.0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a6, 1.0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a7, 1.0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a8, 1.0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a9, 1.0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a10, 1.0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a11, 1.0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a12, 1.0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a13, 1.0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a14, 1.0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a15, 1.0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, 2.0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mov_b32_e32 v16, 0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v0, v1, a[0:15] +; GFX908-GISEL-NOLIT-SRCC-NEXT: s_nop 9 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a4 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a5 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a6 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a7 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a8 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a9 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a10 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a11 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a12 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a13 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a14 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a15 +; GFX908-GISEL-NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1] +; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16 +; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32 +; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48 +; GFX908-GISEL-NOLIT-SRCC-NEXT: s_endpgm +; ; LIT-SRCC-LABEL: test_mfma_f32_16x16x1f32_imm_splat: ; LIT-SRCC: ; %bb.0: ; %bb ; LIT-SRCC-NEXT: v_mov_b32_e32 v0, 1.0 @@ -4383,6 +5679,37 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_imm_splat(ptr addrspace(1) % ; LIT-SRCC-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] ; LIT-SRCC-NEXT: s_endpgm ; +; GFX908-GISEL-LIT-SRCC-LABEL: test_mfma_f32_16x16x1f32_imm_splat: +; GFX908-GISEL-LIT-SRCC: ; %bb.0: ; %bb +; GFX908-GISEL-LIT-SRCC-NEXT: v_mov_b32_e32 v0, 1.0 +; GFX908-GISEL-LIT-SRCC-NEXT: v_mov_b32_e32 v1, 2.0 +; GFX908-GISEL-LIT-SRCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX908-GISEL-LIT-SRCC-NEXT: v_mov_b32_e32 v16, 0 +; GFX908-GISEL-LIT-SRCC-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v0, v1, 1.0 +; GFX908-GISEL-LIT-SRCC-NEXT: s_nop 9 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a4 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a5 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a6 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a7 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a8 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a9 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a10 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a11 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a12 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a13 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a14 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a15 +; GFX908-GISEL-LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1] +; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16 +; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32 +; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48 +; GFX908-GISEL-LIT-SRCC-NEXT: s_endpgm +; ; GFX90A-LABEL: test_mfma_f32_16x16x1f32_imm_splat: ; GFX90A: ; %bb.0: ; %bb ; GFX90A-NEXT: v_mov_b32_e32 v16, 1.0 @@ -4494,6 +5821,60 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8f16_imm_splat(ptr addrspace(1) % ; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] ; NOLIT-SRCC-NEXT: s_endpgm ; +; GFX908-GISEL-NOLIT-SRCC-LABEL: test_mfma_f32_32x32x8f16_imm_splat: +; GFX908-GISEL-NOLIT-SRCC: ; %bb.0: ; %bb +; GFX908-GISEL-NOLIT-SRCC-NEXT: s_mov_b32 s0, 0x3c003c00 +; GFX908-GISEL-NOLIT-SRCC-NEXT: s_mov_b32 s1, s0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s1 +; GFX908-GISEL-NOLIT-SRCC-NEXT: s_mov_b32 s2, 0x40004000 +; GFX908-GISEL-NOLIT-SRCC-NEXT: s_mov_b32 s3, s2 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s2 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a0, 1.0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a1, 1.0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a2, 1.0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a3, 1.0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a4, 1.0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a5, 1.0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a6, 1.0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a7, 1.0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a8, 1.0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a9, 1.0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a10, 1.0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a11, 1.0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a12, 1.0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a13, 1.0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a14, 1.0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a15, 1.0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mov_b32_e32 v3, s3 +; GFX908-GISEL-NOLIT-SRCC-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mov_b32_e32 v16, 0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mfma_f32_32x32x8f16 a[0:15], v[0:1], v[2:3], a[0:15] +; GFX908-GISEL-NOLIT-SRCC-NEXT: s_nop 15 +; GFX908-GISEL-NOLIT-SRCC-NEXT: s_nop 1 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a4 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a5 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a6 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a7 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a8 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a9 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a10 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a11 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a12 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a13 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a14 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a15 +; GFX908-GISEL-NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[0:3], s[6:7] +; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[4:7], s[6:7] offset:16 +; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:32 +; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[6:7] offset:48 +; GFX908-GISEL-NOLIT-SRCC-NEXT: s_endpgm +; ; LIT-SRCC-LABEL: test_mfma_f32_32x32x8f16_imm_splat: ; LIT-SRCC: ; %bb.0: ; %bb ; LIT-SRCC-NEXT: v_mov_b32_e32 v0, 0x3c003c00 @@ -4530,6 +5911,44 @@ define amdgpu_kernel void @test_mfma_f32_32x32x8f16_imm_splat(ptr addrspace(1) % ; LIT-SRCC-NEXT: global_store_dwordx4 v13, v[9:12], s[0:1] ; LIT-SRCC-NEXT: s_endpgm ; +; GFX908-GISEL-LIT-SRCC-LABEL: test_mfma_f32_32x32x8f16_imm_splat: +; GFX908-GISEL-LIT-SRCC: ; %bb.0: ; %bb +; GFX908-GISEL-LIT-SRCC-NEXT: s_mov_b32 s0, 0x3c003c00 +; GFX908-GISEL-LIT-SRCC-NEXT: s_mov_b32 s1, s0 +; GFX908-GISEL-LIT-SRCC-NEXT: v_mov_b32_e32 v0, s0 +; GFX908-GISEL-LIT-SRCC-NEXT: v_mov_b32_e32 v1, s1 +; GFX908-GISEL-LIT-SRCC-NEXT: s_mov_b32 s0, 0x40004000 +; GFX908-GISEL-LIT-SRCC-NEXT: s_mov_b32 s1, s0 +; GFX908-GISEL-LIT-SRCC-NEXT: v_mov_b32_e32 v3, s1 +; GFX908-GISEL-LIT-SRCC-NEXT: v_mov_b32_e32 v2, s0 +; GFX908-GISEL-LIT-SRCC-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24 +; GFX908-GISEL-LIT-SRCC-NEXT: v_mov_b32_e32 v16, 0 +; GFX908-GISEL-LIT-SRCC-NEXT: v_mfma_f32_32x32x8f16 a[0:15], v[0:1], v[2:3], 1.0 +; GFX908-GISEL-LIT-SRCC-NEXT: s_nop 15 +; GFX908-GISEL-LIT-SRCC-NEXT: s_nop 1 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a4 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a5 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a6 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a7 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a8 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a9 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a10 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a11 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a12 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a13 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a14 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a15 +; GFX908-GISEL-LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v16, v[0:3], s[2:3] +; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v16, v[4:7], s[2:3] offset:16 +; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v16, v[8:11], s[2:3] offset:32 +; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[2:3] offset:48 +; GFX908-GISEL-LIT-SRCC-NEXT: s_endpgm +; ; GFX90A-LABEL: test_mfma_f32_32x32x8f16_imm_splat: ; GFX90A: ; %bb.0: ; %bb ; GFX90A-NEXT: v_mov_b32_e32 v16, 0x3c003c00 @@ -4690,6 +6109,90 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm_splat(ptr addrspace(1) % ; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] ; NOLIT-SRCC-NEXT: s_endpgm ; +; GFX908-GISEL-NOLIT-SRCC-LABEL: test_mfma_f32_32x32x1f32_imm_splat: +; GFX908-GISEL-NOLIT-SRCC: ; %bb.0: ; %bb +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, 1.0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a0, 0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a1, 0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a2, 0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a3, 0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a4, 0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a5, 0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a6, 0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a7, 0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a8, 0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a9, 0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a10, 0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a11, 0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a12, 0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a13, 0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a14, 0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a15, 0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a16, 0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a17, 0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a18, 0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a19, 0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a20, 0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a21, 0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a22, 0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a23, 0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a24, 0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a25, 0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a26, 0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a27, 0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a28, 0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a29, 0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a30, 0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a31, 0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, 2.0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mov_b32_e32 v32, 0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31] +; GFX908-GISEL-NOLIT-SRCC-NEXT: s_nop 15 +; GFX908-GISEL-NOLIT-SRCC-NEXT: s_nop 1 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a4 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a5 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a6 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a7 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a8 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a9 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a10 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a11 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a12 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a13 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a14 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a15 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v16, a16 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v17, a17 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v18, a18 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v19, a19 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v20, a20 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v21, a21 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v22, a22 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v23, a23 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a24 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v25, a25 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v26, a26 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v27, a27 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v28, a28 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v29, a29 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v30, a30 +; GFX908-GISEL-NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v31, a31 +; GFX908-GISEL-NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] +; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 +; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 +; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 +; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64 +; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80 +; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96 +; GFX908-GISEL-NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112 +; GFX908-GISEL-NOLIT-SRCC-NEXT: s_endpgm +; ; LIT-SRCC-LABEL: test_mfma_f32_32x32x1f32_imm_splat: ; LIT-SRCC: ; %bb.0: ; %bb ; LIT-SRCC-NEXT: v_mov_b32_e32 v0, 1.0 @@ -4746,6 +6249,58 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm_splat(ptr addrspace(1) % ; LIT-SRCC-NEXT: global_store_dwordx4 v14, v[0:3], s[0:1] ; LIT-SRCC-NEXT: s_endpgm ; +; GFX908-GISEL-LIT-SRCC-LABEL: test_mfma_f32_32x32x1f32_imm_splat: +; GFX908-GISEL-LIT-SRCC: ; %bb.0: ; %bb +; GFX908-GISEL-LIT-SRCC-NEXT: v_mov_b32_e32 v0, 1.0 +; GFX908-GISEL-LIT-SRCC-NEXT: v_mov_b32_e32 v1, 2.0 +; GFX908-GISEL-LIT-SRCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX908-GISEL-LIT-SRCC-NEXT: v_mov_b32_e32 v32, 0 +; GFX908-GISEL-LIT-SRCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, 0 +; GFX908-GISEL-LIT-SRCC-NEXT: s_nop 15 +; GFX908-GISEL-LIT-SRCC-NEXT: s_nop 1 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a4 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a5 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a6 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a7 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a8 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a9 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a10 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a11 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a12 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a13 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a14 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a15 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v16, a16 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v17, a17 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v18, a18 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v19, a19 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v20, a20 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v21, a21 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v22, a22 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v23, a23 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a24 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v25, a25 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v26, a26 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v27, a27 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v28, a28 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v29, a29 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v30, a30 +; GFX908-GISEL-LIT-SRCC-NEXT: v_accvgpr_read_b32 v31, a31 +; GFX908-GISEL-LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] +; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 +; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 +; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 +; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64 +; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80 +; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96 +; GFX908-GISEL-LIT-SRCC-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112 +; GFX908-GISEL-LIT-SRCC-NEXT: s_endpgm +; ; GFX90A-LABEL: test_mfma_f32_32x32x1f32_imm_splat: ; GFX90A: ; %bb.0: ; %bb ; GFX90A-NEXT: v_mov_b32_e32 v32, 1.0 @@ -4835,6 +6390,28 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_imm(ptr addrspace(1) %arg) #0 ; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] ; NOLIT-SRCC-NEXT: s_endpgm ; +; GFX908-GISEL-LABEL: test_mfma_f32_4x4x1f32_imm: +; GFX908-GISEL: ; %bb.0: ; %bb +; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1.0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, 1.0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, 2.0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, 1.0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, 1.0 +; GFX908-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2.0 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v4, 0 +; GFX908-GISEL-NEXT: s_nop 0 +; GFX908-GISEL-NEXT: v_mfma_f32_4x4x1f32 a[0:3], v0, v1, a[0:3] +; GFX908-GISEL-NEXT: s_nop 3 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: s_nop 0 +; GFX908-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] +; GFX908-GISEL-NEXT: s_endpgm +; ; LIT-SRCC-LABEL: test_mfma_f32_4x4x1f32_imm: ; LIT-SRCC: ; %bb.0: ; %bb ; LIT-SRCC-NEXT: v_mov_b32_e32 v0, 1.0 @@ -4962,6 +6539,53 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32_imm(ptr addrspace(1) %arg) # ; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] ; NOLIT-SRCC-NEXT: s_endpgm ; +; GFX908-GISEL-LABEL: test_mfma_f32_16x16x1f32_imm: +; GFX908-GISEL: ; %bb.0: ; %bb +; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1.0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, 1.0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a15, 2.0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, 1.0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, 1.0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, 1.0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a4, 1.0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a5, 1.0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a6, 1.0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a7, 1.0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a8, 1.0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a9, 1.0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a10, 1.0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a11, 1.0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a12, 1.0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a13, 1.0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a14, 1.0 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2.0 +; GFX908-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v16, 0 +; GFX908-GISEL-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v0, v1, a[0:15] +; GFX908-GISEL-NEXT: s_nop 9 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v4, a4 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v5, a5 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v6, a6 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v7, a7 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v8, a8 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v9, a9 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v10, a10 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v11, a11 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v12, a12 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v13, a13 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v14, a14 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v15, a15 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[0:1] +; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[0:1] offset:16 +; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[0:1] offset:32 +; GFX908-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[0:1] offset:48 +; GFX908-GISEL-NEXT: s_endpgm +; ; LIT-SRCC-LABEL: test_mfma_f32_16x16x1f32_imm: ; LIT-SRCC: ; %bb.0: ; %bb ; LIT-SRCC-NEXT: v_mov_b32_e32 v0, 1.0 @@ -5208,6 +6832,90 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm(ptr addrspace(1) %arg) # ; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] ; NOLIT-SRCC-NEXT: s_endpgm ; +; GFX908-GISEL-LABEL: test_mfma_f32_32x32x1f32_imm: +; GFX908-GISEL: ; %bb.0: ; %bb +; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1.0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, 1.0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, 0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, 0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, 0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a4, 0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a5, 0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a6, 0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a7, 0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a8, 0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a9, 0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a10, 0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a11, 0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a12, 0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a13, 0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a14, 0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a15, 0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a16, 0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a17, 0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a18, 0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a19, 0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a20, 0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a21, 0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a22, 0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a23, 0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a24, 0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a25, 0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a26, 0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a27, 0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a28, 0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a29, 0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a30, 0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a31, 0 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2.0 +; GFX908-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v32, 0 +; GFX908-GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31] +; GFX908-GISEL-NEXT: s_nop 15 +; GFX908-GISEL-NEXT: s_nop 1 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v4, a4 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v5, a5 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v6, a6 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v7, a7 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v8, a8 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v9, a9 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v10, a10 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v11, a11 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v12, a12 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v13, a13 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v14, a14 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v15, a15 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v16, a16 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v17, a17 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v18, a18 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v19, a19 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v20, a20 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v21, a21 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v22, a22 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v23, a23 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v24, a24 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v25, a25 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v26, a26 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v27, a27 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v28, a28 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v29, a29 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v30, a30 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v31, a31 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112 +; GFX908-GISEL-NEXT: s_endpgm +; ; LIT-SRCC-LABEL: test_mfma_f32_32x32x1f32_imm: ; LIT-SRCC: ; %bb.0: ; %bb ; LIT-SRCC-NEXT: v_mov_b32_e32 v0, 1.0 @@ -5537,6 +7245,33 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_lit_splat(ptr addrspace(1) %ar ; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] ; NOLIT-SRCC-NEXT: s_endpgm ; +; GFX908-GISEL-LABEL: test_mfma_f32_4x4x1f32_lit_splat: +; GFX908-GISEL: ; %bb.0: ; %bb +; GFX908-GISEL-NEXT: s_mov_b32 s2, 0x42f60000 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v5, s2 +; GFX908-GISEL-NEXT: v_lshlrev_b32_e32 v4, 4, v0 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1.0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v5 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s2 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s2 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v5, s2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v3 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v5 +; GFX908-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2.0 +; GFX908-GISEL-NEXT: s_nop 1 +; GFX908-GISEL-NEXT: v_mfma_f32_4x4x1f32 a[0:3], v0, v1, a[0:3] +; GFX908-GISEL-NEXT: s_nop 3 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: s_nop 0 +; GFX908-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] +; GFX908-GISEL-NEXT: s_endpgm +; ; LIT-SRCC-LABEL: test_mfma_f32_4x4x1f32_lit_splat: ; LIT-SRCC: ; %bb.0: ; %bb ; LIT-SRCC-NEXT: v_mov_b32_e32 v1, 0x42f60000 @@ -5646,6 +7381,33 @@ define amdgpu_kernel void @test_mfma_f32_4x4x1f32_lit_splat_bad_code(ptr addrspa ; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] ; NOLIT-SRCC-NEXT: s_endpgm ; +; GFX908-GISEL-LABEL: test_mfma_f32_4x4x1f32_lit_splat_bad_code: +; GFX908-GISEL: ; %bb.0: ; %bb +; GFX908-GISEL-NEXT: s_mov_b32 s2, 0x42f60000 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v5, s2 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 1.0 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v2, s2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v5 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v3, s2 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v5, s2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v3 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v5 +; GFX908-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v1, 2.0 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v4, 0 +; GFX908-GISEL-NEXT: s_nop 0 +; GFX908-GISEL-NEXT: v_mfma_f32_4x4x1f32 a[0:3], v0, v1, a[0:3] +; GFX908-GISEL-NEXT: s_nop 3 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: s_nop 0 +; GFX908-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] +; GFX908-GISEL-NEXT: s_endpgm +; ; LIT-SRCC-LABEL: test_mfma_f32_4x4x1f32_lit_splat_bad_code: ; LIT-SRCC: ; %bb.0: ; %bb ; LIT-SRCC-NEXT: v_mov_b32_e32 v0, 0x42f60000 @@ -5824,6 +7586,101 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vecarg(ptr addrspace(1) %arg ; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:16 ; NOLIT-SRCC-NEXT: s_endpgm ; +; GFX908-GISEL-LABEL: test_mfma_f32_32x32x1f32_vecarg: +; GFX908-GISEL: ; %bb.0: ; %bb +; GFX908-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 +; GFX908-GISEL-NEXT: v_lshlrev_b32_e32 v32, 7, v0 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v16, 1.0 +; GFX908-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX908-GISEL-NEXT: global_load_dwordx4 v[0:3], v32, s[0:1] +; GFX908-GISEL-NEXT: global_load_dwordx4 v[4:7], v32, s[0:1] offset:16 +; GFX908-GISEL-NEXT: global_load_dwordx4 v[8:11], v32, s[0:1] offset:32 +; GFX908-GISEL-NEXT: global_load_dwordx4 v[12:15], v32, s[0:1] offset:48 +; GFX908-GISEL-NEXT: s_waitcnt vmcnt(0) +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a0, v0 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a1, v1 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a2, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a3, v3 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a4, v4 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a5, v5 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a6, v6 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a7, v7 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a8, v8 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a9, v9 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a10, v10 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a11, v11 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a12, v12 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a13, v13 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a14, v14 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a15, v15 +; GFX908-GISEL-NEXT: global_load_dwordx4 v[0:3], v32, s[0:1] offset:64 +; GFX908-GISEL-NEXT: global_load_dwordx4 v[4:7], v32, s[0:1] offset:80 +; GFX908-GISEL-NEXT: global_load_dwordx4 v[8:11], v32, s[0:1] offset:96 +; GFX908-GISEL-NEXT: global_load_dwordx4 v[12:15], v32, s[0:1] offset:112 +; GFX908-GISEL-NEXT: s_waitcnt vmcnt(0) +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a31, v15 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a30, v14 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a29, v13 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a28, v12 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a27, v11 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a26, v10 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a25, v9 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a24, v8 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a23, v7 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a22, v6 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a21, v5 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a20, v4 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a19, v3 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a18, v2 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a17, v1 +; GFX908-GISEL-NEXT: v_accvgpr_write_b32 a16, v0 +; GFX908-GISEL-NEXT: v_mov_b32_e32 v0, 2.0 +; GFX908-GISEL-NEXT: s_nop 1 +; GFX908-GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v16, v0, a[0:31] cbsz:1 abid:2 blgp:3 +; GFX908-GISEL-NEXT: s_nop 15 +; GFX908-GISEL-NEXT: s_nop 1 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v4, a4 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v5, a5 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v6, a6 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v7, a7 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v8, a8 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v9, a9 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v10, a10 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v11, a11 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v12, a12 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v13, a13 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v14, a14 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v15, a15 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v16, a16 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v17, a17 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v18, a18 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v19, a19 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v20, a20 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v21, a21 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v22, a22 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v23, a23 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v24, a24 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v25, a25 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v26, a26 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v27, a27 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v28, a28 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v29, a29 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v30, a30 +; GFX908-GISEL-NEXT: v_accvgpr_read_b32 v31, a31 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96 +; GFX908-GISEL-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112 +; GFX908-GISEL-NEXT: s_endpgm +; ; LIT-SRCC-LABEL: test_mfma_f32_32x32x1f32_vecarg: ; LIT-SRCC: ; %bb.0: ; %bb ; LIT-SRCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24