diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp index 726be8c7f0982..70fc4960f8ef1 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp @@ -523,14 +523,17 @@ void AMDGPUCoExecSchedStrategy::pickNodeFromQueue( ArrayRef Pressure = RPTracker.getRegSetPressureAtPos(); unsigned SGPRPressure = 0; unsigned VGPRPressure = 0; + unsigned AGPRPressure = 0; PickedPending = false; if (DAG->isTrackingPressure()) { if (!useGCNTrackers()) { SGPRPressure = Pressure[AMDGPU::RegisterPressureSets::SReg_32]; VGPRPressure = Pressure[AMDGPU::RegisterPressureSets::VGPR_32]; + AGPRPressure = Pressure[AMDGPU::RegisterPressureSets::AGPR_32]; } else { SGPRPressure = DownwardTracker.getPressure().getSGPRNum(); VGPRPressure = DownwardTracker.getPressure().getArchVGPRNum(); + AGPRPressure = DownwardTracker.getPressure().getAGPRNum(); } } @@ -538,7 +541,7 @@ void AMDGPUCoExecSchedStrategy::pickNodeFromQueue( for (SUnit *SU : Q) { SchedCandidate TryCand(ZonePolicy); initCandidate(TryCand, SU, Zone.isTop(), RPTracker, SRI, SGPRPressure, - VGPRPressure, IsBottomUp); + VGPRPressure, AGPRPressure, IsBottomUp); SchedBoundary *ZoneArg = Cand.AtTop == TryCand.AtTop ? &Zone : nullptr; tryCandidateCoexec(Cand, TryCand, ZoneArg); if (TryCand.Reason != NoCand) { diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp index 6cfdc8717ef64..05477393ac580 100644 --- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp +++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp @@ -151,6 +151,8 @@ void GCNSchedStrategy::initialize(ScheduleDAGMI *DAG) { Context->RegClassInfo->getNumAllocatableRegs(&AMDGPU::SGPR_32RegClass); VGPRExcessLimit = Context->RegClassInfo->getNumAllocatableRegs(&AMDGPU::VGPR_32RegClass); + AGPRExcessLimit = + Context->RegClassInfo->getNumAllocatableRegs(&AMDGPU::AGPR_32RegClass); SIMachineFunctionInfo &MFI = *MF->getInfo(); // Set the initial TargetOccupnacy to the maximum occupancy that we can @@ -182,6 +184,10 @@ void GCNSchedStrategy::initialize(ScheduleDAGMI *DAG) { VGPRBudget = std::max(VGPRBudget, Granule); VGPRCriticalLimit = std::min(VGPRBudget, VGPRExcessLimit); } + + // Reuse VGPR critical limit + AGPRCriticalLimit = std::min(VGPRCriticalLimit, AGPRExcessLimit); + // Apply VGPR excess threshold percentage if specified. if (VGPRThresholdPercentOpt > 0) { [[maybe_unused]] unsigned OriginalVGPRExcessLimit = VGPRExcessLimit; @@ -203,8 +209,14 @@ void GCNSchedStrategy::initialize(ScheduleDAGMI *DAG) { // Subtract error margin and bias from register limits and avoid overflow. SGPRCriticalLimit -= std::min(SGPRLimitBias + ErrorMargin, SGPRCriticalLimit); SGPRExcessLimit -= std::min(SGPRLimitBias + ErrorMargin, SGPRExcessLimit); + + AGPRExcessLimit -= std::min(VGPRLimitBias + ErrorMargin, AGPRExcessLimit); + AGPRCriticalLimit -= std::min(VGPRLimitBias + ErrorMargin, AGPRCriticalLimit); + LLVM_DEBUG(dbgs() << "VGPRCriticalLimit = " << VGPRCriticalLimit << ", VGPRExcessLimit = " << VGPRExcessLimit + << ", AGPRCriticalLimit = " << AGPRCriticalLimit + << ", AGPRExcessLimit = " << AGPRExcessLimit << ", SGPRCriticalLimit = " << SGPRCriticalLimit << ", SGPRExcessLimit = " << SGPRExcessLimit << "\n\n"); } @@ -316,7 +328,8 @@ void GCNSchedStrategy::initCandidate(SchedCandidate &Cand, SUnit *SU, const RegPressureTracker &RPTracker, const SIRegisterInfo *SRI, unsigned SGPRPressure, - unsigned VGPRPressure, bool IsBottomUp) { + unsigned VGPRPressure, + unsigned AGPRPressure, bool IsBottomUp) { Cand.SU = SU; Cand.AtTop = AtTop; @@ -345,6 +358,7 @@ void GCNSchedStrategy::initCandidate(SchedCandidate &Cand, SUnit *SU, Pressure.resize(4, 0); Pressure[AMDGPU::RegisterPressureSets::SReg_32] = SGPRPressure; Pressure[AMDGPU::RegisterPressureSets::VGPR_32] = VGPRPressure; + Pressure[AMDGPU::RegisterPressureSets::AGPR_32] = AGPRPressure; for (const auto &Diff : DAG->getPressureDiff(SU)) { if (!Diff.isValid()) @@ -362,7 +376,9 @@ void GCNSchedStrategy::initCandidate(SchedCandidate &Cand, SUnit *SU, if (Pressure[AMDGPU::RegisterPressureSets::SReg_32] != CheckPressure[AMDGPU::RegisterPressureSets::SReg_32] || Pressure[AMDGPU::RegisterPressureSets::VGPR_32] != - CheckPressure[AMDGPU::RegisterPressureSets::VGPR_32]) { + CheckPressure[AMDGPU::RegisterPressureSets::VGPR_32] || + Pressure[AMDGPU::RegisterPressureSets::AGPR_32] != + CheckPressure[AMDGPU::RegisterPressureSets::AGPR_32]) { errs() << "Register Pressure is inaccurate when calculated through " "PressureDiff\n" << "SGPR got " << Pressure[AMDGPU::RegisterPressureSets::SReg_32] @@ -370,12 +386,16 @@ void GCNSchedStrategy::initCandidate(SchedCandidate &Cand, SUnit *SU, << CheckPressure[AMDGPU::RegisterPressureSets::SReg_32] << "\n" << "VGPR got " << Pressure[AMDGPU::RegisterPressureSets::VGPR_32] << ", expected " - << CheckPressure[AMDGPU::RegisterPressureSets::VGPR_32] << "\n"; + << CheckPressure[AMDGPU::RegisterPressureSets::VGPR_32] << "\n" + << "AGPR got " << Pressure[AMDGPU::RegisterPressureSets::AGPR_32] + << ", expected " + << CheckPressure[AMDGPU::RegisterPressureSets::AGPR_32] << "\n"; report_fatal_error("inaccurate register pressure calculation"); } #endif } + unsigned NewAGPRPressure = Pressure[AMDGPU::RegisterPressureSets::AGPR_32]; unsigned NewSGPRPressure = Pressure[AMDGPU::RegisterPressureSets::SReg_32]; unsigned NewVGPRPressure = Pressure[AMDGPU::RegisterPressureSets::VGPR_32]; @@ -384,18 +404,19 @@ void GCNSchedStrategy::initCandidate(SchedCandidate &Cand, SUnit *SU, // instruction that increases the set with the least amount of registers, // which in our case would be SGPRs. This is rarely what we want, so // when we report excess/critical register pressure, we do it either - // only for VGPRs or only for SGPRs. + // only for VGPRs, AGPRs or SGPRs. Priority: VGPR > AGPR > SGPR. // FIXME: Better heuristics to determine whether to prefer SGPRs or VGPRs. const unsigned MaxVGPRPressureInc = 16; bool ShouldTrackVGPRs = VGPRPressure + MaxVGPRPressureInc >= VGPRExcessLimit; - bool ShouldTrackSGPRs = !ShouldTrackVGPRs && SGPRPressure >= SGPRExcessLimit; - + bool ShouldTrackAGPRs = AGPRExcessLimit > 0 && !ShouldTrackVGPRs && + AGPRPressure + MaxVGPRPressureInc >= AGPRExcessLimit; + bool ShouldTrackSGPRs = + !ShouldTrackVGPRs && !ShouldTrackAGPRs && SGPRPressure >= SGPRExcessLimit; // FIXME: We have to enter REG-EXCESS before we reach the actual threshold // to increase the likelihood we don't go over the limits. We should improve // the analysis to look through dependencies to find the path with the least // register pressure. - // We only need to update the RPDelta for instructions that increase register // pressure. Instructions that decrease or keep reg pressure the same will be // marked as RegExcess in tryCandidate() when they are compared with @@ -406,6 +427,12 @@ void GCNSchedStrategy::initCandidate(SchedCandidate &Cand, SUnit *SU, Cand.RPDelta.Excess.setUnitInc(NewVGPRPressure - VGPRExcessLimit); } + if (ShouldTrackAGPRs && NewAGPRPressure >= AGPRExcessLimit) { + HasHighPressure = true; + Cand.RPDelta.Excess = PressureChange(AMDGPU::RegisterPressureSets::AGPR_32); + Cand.RPDelta.Excess.setUnitInc(NewAGPRPressure - AGPRExcessLimit); + } + if (ShouldTrackSGPRs && NewSGPRPressure >= SGPRExcessLimit) { HasHighPressure = true; Cand.RPDelta.Excess = PressureChange(AMDGPU::RegisterPressureSets::SReg_32); @@ -414,22 +441,29 @@ void GCNSchedStrategy::initCandidate(SchedCandidate &Cand, SUnit *SU, // Register pressure is considered 'CRITICAL' if it is approaching a value // that would reduce the wave occupancy for the execution unit. When - // register pressure is 'CRITICAL', increasing SGPR and VGPR pressure both - // has the same cost, so we don't need to prefer one over the other. + // register pressure is 'CRITICAL', increasing SGPR, VGPR, and AGPR + // pressure all has the same cost, so we pick the most critical type. int SGPRDelta = NewSGPRPressure - SGPRCriticalLimit; int VGPRDelta = NewVGPRPressure - VGPRCriticalLimit; + int AGPRDelta = AGPRExcessLimit > 0 ? NewAGPRPressure - AGPRCriticalLimit + : std::numeric_limits::min(); - if (SGPRDelta >= 0 || VGPRDelta >= 0) { + if (SGPRDelta >= 0 || VGPRDelta >= 0 || AGPRDelta >= 0) { HasHighPressure = true; - if (SGPRDelta > VGPRDelta) { - Cand.RPDelta.CriticalMax = - PressureChange(AMDGPU::RegisterPressureSets::SReg_32); - Cand.RPDelta.CriticalMax.setUnitInc(SGPRDelta); - } else { + // Pick the most critical type. + if (VGPRDelta >= SGPRDelta && VGPRDelta >= AGPRDelta) { Cand.RPDelta.CriticalMax = PressureChange(AMDGPU::RegisterPressureSets::VGPR_32); Cand.RPDelta.CriticalMax.setUnitInc(VGPRDelta); + } else if (AGPRDelta >= SGPRDelta) { + Cand.RPDelta.CriticalMax = + PressureChange(AMDGPU::RegisterPressureSets::AGPR_32); + Cand.RPDelta.CriticalMax.setUnitInc(AGPRDelta); + } else { + Cand.RPDelta.CriticalMax = + PressureChange(AMDGPU::RegisterPressureSets::SReg_32); + Cand.RPDelta.CriticalMax.setUnitInc(SGPRDelta); } } } @@ -479,17 +513,20 @@ void GCNSchedStrategy::pickNodeFromQueue(SchedBoundary &Zone, ArrayRef Pressure = RPTracker.getRegSetPressureAtPos(); unsigned SGPRPressure = 0; unsigned VGPRPressure = 0; + unsigned AGPRPressure = 0; IsPending = false; if (DAG->isTrackingPressure()) { if (!useGCNTrackers()) { SGPRPressure = Pressure[AMDGPU::RegisterPressureSets::SReg_32]; VGPRPressure = Pressure[AMDGPU::RegisterPressureSets::VGPR_32]; + AGPRPressure = Pressure[AMDGPU::RegisterPressureSets::AGPR_32]; } else { GCNRPTracker *T = IsBottomUp ? static_cast(&UpwardTracker) : static_cast(&DownwardTracker); SGPRPressure = T->getPressure().getSGPRNum(); VGPRPressure = T->getPressure().getArchVGPRNum(); + AGPRPressure = T->getPressure().getAGPRNum(); } } LLVM_DEBUG(dbgs() << "Available Q:\n"); @@ -498,7 +535,7 @@ void GCNSchedStrategy::pickNodeFromQueue(SchedBoundary &Zone, SchedCandidate TryCand(ZonePolicy); initCandidate(TryCand, SU, Zone.isTop(), RPTracker, SRI, SGPRPressure, - VGPRPressure, IsBottomUp); + VGPRPressure, AGPRPressure, IsBottomUp); // Pass SchedBoundary only when comparing nodes from the same boundary. SchedBoundary *ZoneArg = Cand.AtTop == TryCand.AtTop ? &Zone : nullptr; tryCandidate(Cand, TryCand, ZoneArg); @@ -522,7 +559,7 @@ void GCNSchedStrategy::pickNodeFromQueue(SchedBoundary &Zone, SchedCandidate TryCand(ZonePolicy); initCandidate(TryCand, SU, Zone.isTop(), RPTracker, SRI, SGPRPressure, - VGPRPressure, IsBottomUp); + VGPRPressure, AGPRPressure, IsBottomUp); // Pass SchedBoundary only when comparing nodes from the same boundary. SchedBoundary *ZoneArg = Cand.AtTop == TryCand.AtTop ? &Zone : nullptr; tryPendingCandidate(Cand, TryCand, ZoneArg); diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h index 2059f4e6479ff..29393d035cc8c 100644 --- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h +++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h @@ -57,7 +57,8 @@ class GCNSchedStrategy : public GenericScheduler { void initCandidate(SchedCandidate &Cand, SUnit *SU, bool AtTop, const RegPressureTracker &RPTracker, const SIRegisterInfo *SRI, unsigned SGPRPressure, - unsigned VGPRPressure, bool IsBottomUp); + unsigned VGPRPressure, unsigned AGPRPressure, + bool IsBottomUp); /// Estimate how many cycles \p SU must wait due to structural hazards at the /// current boundary cycle. Returns zero when no stall is required. @@ -92,6 +93,8 @@ class GCNSchedStrategy : public GenericScheduler { unsigned VGPRExcessLimit; + unsigned AGPRExcessLimit; + unsigned TargetOccupancy; MachineFunction *MF; @@ -136,6 +139,8 @@ class GCNSchedStrategy : public GenericScheduler { unsigned VGPRCriticalLimit; + unsigned AGPRCriticalLimit; + unsigned SGPRLimitBias = 0; unsigned VGPRLimitBias = 0; diff --git a/llvm/test/CodeGen/AMDGPU/agpr-csr.ll b/llvm/test/CodeGen/AMDGPU/agpr-csr.ll index 016ac4eeb8867..c0805565190fb 100644 --- a/llvm/test/CodeGen/AMDGPU/agpr-csr.ll +++ b/llvm/test/CodeGen/AMDGPU/agpr-csr.ll @@ -180,34 +180,34 @@ define amdgpu_kernel void @test_call_empty() #0 { ; GFX908-NEXT: ;;#ASMSTART ; GFX908-NEXT: ; def a[0:31] ; GFX908-NEXT: ;;#ASMEND -; GFX908-NEXT: v_accvgpr_read_b32 v6, a3 -; GFX908-NEXT: v_accvgpr_read_b32 v5, a2 -; GFX908-NEXT: v_accvgpr_read_b32 v4, a1 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a0 -; GFX908-NEXT: v_accvgpr_read_b32 v10, a7 -; GFX908-NEXT: v_accvgpr_read_b32 v9, a6 -; GFX908-NEXT: v_accvgpr_read_b32 v8, a5 -; GFX908-NEXT: v_accvgpr_read_b32 v7, a4 -; GFX908-NEXT: v_accvgpr_read_b32 v14, a11 -; GFX908-NEXT: v_accvgpr_read_b32 v13, a10 -; GFX908-NEXT: v_accvgpr_read_b32 v12, a9 -; GFX908-NEXT: v_accvgpr_read_b32 v11, a8 -; GFX908-NEXT: v_accvgpr_read_b32 v18, a15 -; GFX908-NEXT: v_accvgpr_read_b32 v17, a14 -; GFX908-NEXT: v_accvgpr_read_b32 v16, a13 -; GFX908-NEXT: v_accvgpr_read_b32 v15, a12 -; GFX908-NEXT: v_accvgpr_read_b32 v22, a19 -; GFX908-NEXT: v_accvgpr_read_b32 v21, a18 -; GFX908-NEXT: v_accvgpr_read_b32 v20, a17 -; GFX908-NEXT: v_accvgpr_read_b32 v19, a16 -; GFX908-NEXT: v_accvgpr_read_b32 v26, a23 -; GFX908-NEXT: v_accvgpr_read_b32 v25, a22 -; GFX908-NEXT: v_accvgpr_read_b32 v24, a21 -; GFX908-NEXT: v_accvgpr_read_b32 v23, a20 -; GFX908-NEXT: v_accvgpr_read_b32 v30, a27 -; GFX908-NEXT: v_accvgpr_read_b32 v29, a26 -; GFX908-NEXT: v_accvgpr_read_b32 v28, a25 -; GFX908-NEXT: v_accvgpr_read_b32 v27, a24 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-NEXT: v_accvgpr_read_b32 v7, a7 +; GFX908-NEXT: v_accvgpr_read_b32 v11, a11 +; GFX908-NEXT: v_accvgpr_read_b32 v15, a15 +; GFX908-NEXT: v_accvgpr_read_b32 v19, a19 +; GFX908-NEXT: v_accvgpr_read_b32 v23, a23 +; GFX908-NEXT: v_accvgpr_read_b32 v27, a27 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-NEXT: v_accvgpr_read_b32 v6, a6 +; GFX908-NEXT: v_accvgpr_read_b32 v5, a5 +; GFX908-NEXT: v_accvgpr_read_b32 v4, a4 +; GFX908-NEXT: v_accvgpr_read_b32 v10, a10 +; GFX908-NEXT: v_accvgpr_read_b32 v9, a9 +; GFX908-NEXT: v_accvgpr_read_b32 v8, a8 +; GFX908-NEXT: v_accvgpr_read_b32 v14, a14 +; GFX908-NEXT: v_accvgpr_read_b32 v13, a13 +; GFX908-NEXT: v_accvgpr_read_b32 v12, a12 +; GFX908-NEXT: v_accvgpr_read_b32 v18, a18 +; GFX908-NEXT: v_accvgpr_read_b32 v17, a17 +; GFX908-NEXT: v_accvgpr_read_b32 v16, a16 +; GFX908-NEXT: v_accvgpr_read_b32 v22, a22 +; GFX908-NEXT: v_accvgpr_read_b32 v21, a21 +; GFX908-NEXT: v_accvgpr_read_b32 v20, a20 +; GFX908-NEXT: v_accvgpr_read_b32 v26, a26 +; GFX908-NEXT: v_accvgpr_read_b32 v25, a25 +; GFX908-NEXT: v_accvgpr_read_b32 v24, a24 ; GFX908-NEXT: v_accvgpr_read_b32 v35, a31 ; GFX908-NEXT: v_accvgpr_read_b32 v34, a30 ; GFX908-NEXT: v_accvgpr_read_b32 v33, a29 @@ -216,19 +216,19 @@ define amdgpu_kernel void @test_call_empty() #0 { ; GFX908-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GFX908-NEXT: global_store_dwordx4 v[0:1], v[32:35], off ; GFX908-NEXT: s_waitcnt vmcnt(0) -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[27:30], off +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[24:27], off ; GFX908-NEXT: s_waitcnt vmcnt(0) -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[23:26], off +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[20:23], off ; GFX908-NEXT: s_waitcnt vmcnt(0) -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[19:22], off +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[16:19], off ; GFX908-NEXT: s_waitcnt vmcnt(0) -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[15:18], off +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[12:15], off ; GFX908-NEXT: s_waitcnt vmcnt(0) -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[11:14], off +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[8:11], off ; GFX908-NEXT: s_waitcnt vmcnt(0) -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[7:10], off +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[4:7], off ; GFX908-NEXT: s_waitcnt vmcnt(0) -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[3:6], off +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off ; GFX908-NEXT: s_waitcnt vmcnt(0) ; GFX908-NEXT: s_endpgm bb: @@ -315,34 +315,34 @@ define amdgpu_kernel void @test_call_areg4() #0 { ; GFX908-NEXT: ;;#ASMSTART ; GFX908-NEXT: ; def a[0:31] ; GFX908-NEXT: ;;#ASMEND -; GFX908-NEXT: v_accvgpr_read_b32 v6, a3 -; GFX908-NEXT: v_accvgpr_read_b32 v5, a2 -; GFX908-NEXT: v_accvgpr_read_b32 v4, a1 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a0 -; GFX908-NEXT: v_accvgpr_read_b32 v10, a7 -; GFX908-NEXT: v_accvgpr_read_b32 v9, a6 -; GFX908-NEXT: v_accvgpr_read_b32 v8, a5 -; GFX908-NEXT: v_accvgpr_read_b32 v7, a4 -; GFX908-NEXT: v_accvgpr_read_b32 v14, a11 -; GFX908-NEXT: v_accvgpr_read_b32 v13, a10 -; GFX908-NEXT: v_accvgpr_read_b32 v12, a9 -; GFX908-NEXT: v_accvgpr_read_b32 v11, a8 -; GFX908-NEXT: v_accvgpr_read_b32 v18, a15 -; GFX908-NEXT: v_accvgpr_read_b32 v17, a14 -; GFX908-NEXT: v_accvgpr_read_b32 v16, a13 -; GFX908-NEXT: v_accvgpr_read_b32 v15, a12 -; GFX908-NEXT: v_accvgpr_read_b32 v22, a19 -; GFX908-NEXT: v_accvgpr_read_b32 v21, a18 -; GFX908-NEXT: v_accvgpr_read_b32 v20, a17 -; GFX908-NEXT: v_accvgpr_read_b32 v19, a16 -; GFX908-NEXT: v_accvgpr_read_b32 v26, a23 -; GFX908-NEXT: v_accvgpr_read_b32 v25, a22 -; GFX908-NEXT: v_accvgpr_read_b32 v24, a21 -; GFX908-NEXT: v_accvgpr_read_b32 v23, a20 -; GFX908-NEXT: v_accvgpr_read_b32 v30, a27 -; GFX908-NEXT: v_accvgpr_read_b32 v29, a26 -; GFX908-NEXT: v_accvgpr_read_b32 v28, a25 -; GFX908-NEXT: v_accvgpr_read_b32 v27, a24 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-NEXT: v_accvgpr_read_b32 v7, a7 +; GFX908-NEXT: v_accvgpr_read_b32 v11, a11 +; GFX908-NEXT: v_accvgpr_read_b32 v15, a15 +; GFX908-NEXT: v_accvgpr_read_b32 v19, a19 +; GFX908-NEXT: v_accvgpr_read_b32 v23, a23 +; GFX908-NEXT: v_accvgpr_read_b32 v27, a27 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-NEXT: v_accvgpr_read_b32 v6, a6 +; GFX908-NEXT: v_accvgpr_read_b32 v5, a5 +; GFX908-NEXT: v_accvgpr_read_b32 v4, a4 +; GFX908-NEXT: v_accvgpr_read_b32 v10, a10 +; GFX908-NEXT: v_accvgpr_read_b32 v9, a9 +; GFX908-NEXT: v_accvgpr_read_b32 v8, a8 +; GFX908-NEXT: v_accvgpr_read_b32 v14, a14 +; GFX908-NEXT: v_accvgpr_read_b32 v13, a13 +; GFX908-NEXT: v_accvgpr_read_b32 v12, a12 +; GFX908-NEXT: v_accvgpr_read_b32 v18, a18 +; GFX908-NEXT: v_accvgpr_read_b32 v17, a17 +; GFX908-NEXT: v_accvgpr_read_b32 v16, a16 +; GFX908-NEXT: v_accvgpr_read_b32 v22, a22 +; GFX908-NEXT: v_accvgpr_read_b32 v21, a21 +; GFX908-NEXT: v_accvgpr_read_b32 v20, a20 +; GFX908-NEXT: v_accvgpr_read_b32 v26, a26 +; GFX908-NEXT: v_accvgpr_read_b32 v25, a25 +; GFX908-NEXT: v_accvgpr_read_b32 v24, a24 ; GFX908-NEXT: v_accvgpr_read_b32 v35, a31 ; GFX908-NEXT: v_accvgpr_read_b32 v34, a30 ; GFX908-NEXT: v_accvgpr_read_b32 v33, a29 @@ -351,19 +351,19 @@ define amdgpu_kernel void @test_call_areg4() #0 { ; GFX908-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GFX908-NEXT: global_store_dwordx4 v[0:1], v[32:35], off ; GFX908-NEXT: s_waitcnt vmcnt(0) -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[27:30], off +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[24:27], off ; GFX908-NEXT: s_waitcnt vmcnt(0) -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[23:26], off +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[20:23], off ; GFX908-NEXT: s_waitcnt vmcnt(0) -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[19:22], off +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[16:19], off ; GFX908-NEXT: s_waitcnt vmcnt(0) -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[15:18], off +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[12:15], off ; GFX908-NEXT: s_waitcnt vmcnt(0) -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[11:14], off +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[8:11], off ; GFX908-NEXT: s_waitcnt vmcnt(0) -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[7:10], off +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[4:7], off ; GFX908-NEXT: s_waitcnt vmcnt(0) -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[3:6], off +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off ; GFX908-NEXT: s_waitcnt vmcnt(0) ; GFX908-NEXT: s_endpgm bb: @@ -450,34 +450,34 @@ define amdgpu_kernel void @test_call_areg32() #0 { ; GFX908-NEXT: ;;#ASMSTART ; GFX908-NEXT: ; def a[0:31] ; GFX908-NEXT: ;;#ASMEND -; GFX908-NEXT: v_accvgpr_read_b32 v6, a3 -; GFX908-NEXT: v_accvgpr_read_b32 v5, a2 -; GFX908-NEXT: v_accvgpr_read_b32 v4, a1 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a0 -; GFX908-NEXT: v_accvgpr_read_b32 v10, a7 -; GFX908-NEXT: v_accvgpr_read_b32 v9, a6 -; GFX908-NEXT: v_accvgpr_read_b32 v8, a5 -; GFX908-NEXT: v_accvgpr_read_b32 v7, a4 -; GFX908-NEXT: v_accvgpr_read_b32 v14, a11 -; GFX908-NEXT: v_accvgpr_read_b32 v13, a10 -; GFX908-NEXT: v_accvgpr_read_b32 v12, a9 -; GFX908-NEXT: v_accvgpr_read_b32 v11, a8 -; GFX908-NEXT: v_accvgpr_read_b32 v18, a15 -; GFX908-NEXT: v_accvgpr_read_b32 v17, a14 -; GFX908-NEXT: v_accvgpr_read_b32 v16, a13 -; GFX908-NEXT: v_accvgpr_read_b32 v15, a12 -; GFX908-NEXT: v_accvgpr_read_b32 v22, a19 -; GFX908-NEXT: v_accvgpr_read_b32 v21, a18 -; GFX908-NEXT: v_accvgpr_read_b32 v20, a17 -; GFX908-NEXT: v_accvgpr_read_b32 v19, a16 -; GFX908-NEXT: v_accvgpr_read_b32 v26, a23 -; GFX908-NEXT: v_accvgpr_read_b32 v25, a22 -; GFX908-NEXT: v_accvgpr_read_b32 v24, a21 -; GFX908-NEXT: v_accvgpr_read_b32 v23, a20 -; GFX908-NEXT: v_accvgpr_read_b32 v30, a27 -; GFX908-NEXT: v_accvgpr_read_b32 v29, a26 -; GFX908-NEXT: v_accvgpr_read_b32 v28, a25 -; GFX908-NEXT: v_accvgpr_read_b32 v27, a24 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-NEXT: v_accvgpr_read_b32 v7, a7 +; GFX908-NEXT: v_accvgpr_read_b32 v11, a11 +; GFX908-NEXT: v_accvgpr_read_b32 v15, a15 +; GFX908-NEXT: v_accvgpr_read_b32 v19, a19 +; GFX908-NEXT: v_accvgpr_read_b32 v23, a23 +; GFX908-NEXT: v_accvgpr_read_b32 v27, a27 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-NEXT: v_accvgpr_read_b32 v6, a6 +; GFX908-NEXT: v_accvgpr_read_b32 v5, a5 +; GFX908-NEXT: v_accvgpr_read_b32 v4, a4 +; GFX908-NEXT: v_accvgpr_read_b32 v10, a10 +; GFX908-NEXT: v_accvgpr_read_b32 v9, a9 +; GFX908-NEXT: v_accvgpr_read_b32 v8, a8 +; GFX908-NEXT: v_accvgpr_read_b32 v14, a14 +; GFX908-NEXT: v_accvgpr_read_b32 v13, a13 +; GFX908-NEXT: v_accvgpr_read_b32 v12, a12 +; GFX908-NEXT: v_accvgpr_read_b32 v18, a18 +; GFX908-NEXT: v_accvgpr_read_b32 v17, a17 +; GFX908-NEXT: v_accvgpr_read_b32 v16, a16 +; GFX908-NEXT: v_accvgpr_read_b32 v22, a22 +; GFX908-NEXT: v_accvgpr_read_b32 v21, a21 +; GFX908-NEXT: v_accvgpr_read_b32 v20, a20 +; GFX908-NEXT: v_accvgpr_read_b32 v26, a26 +; GFX908-NEXT: v_accvgpr_read_b32 v25, a25 +; GFX908-NEXT: v_accvgpr_read_b32 v24, a24 ; GFX908-NEXT: v_accvgpr_read_b32 v35, a31 ; GFX908-NEXT: v_accvgpr_read_b32 v34, a30 ; GFX908-NEXT: v_accvgpr_read_b32 v33, a29 @@ -486,19 +486,19 @@ define amdgpu_kernel void @test_call_areg32() #0 { ; GFX908-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GFX908-NEXT: global_store_dwordx4 v[0:1], v[32:35], off ; GFX908-NEXT: s_waitcnt vmcnt(0) -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[27:30], off +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[24:27], off ; GFX908-NEXT: s_waitcnt vmcnt(0) -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[23:26], off +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[20:23], off ; GFX908-NEXT: s_waitcnt vmcnt(0) -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[19:22], off +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[16:19], off ; GFX908-NEXT: s_waitcnt vmcnt(0) -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[15:18], off +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[12:15], off ; GFX908-NEXT: s_waitcnt vmcnt(0) -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[11:14], off +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[8:11], off ; GFX908-NEXT: s_waitcnt vmcnt(0) -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[7:10], off +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[4:7], off ; GFX908-NEXT: s_waitcnt vmcnt(0) -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[3:6], off +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off ; GFX908-NEXT: s_waitcnt vmcnt(0) ; GFX908-NEXT: s_endpgm bb: @@ -585,34 +585,34 @@ define amdgpu_kernel void @test_call_areg64() #0 { ; GFX908-NEXT: ;;#ASMSTART ; GFX908-NEXT: ; def a[0:31] ; GFX908-NEXT: ;;#ASMEND -; GFX908-NEXT: v_accvgpr_read_b32 v6, a3 -; GFX908-NEXT: v_accvgpr_read_b32 v5, a2 -; GFX908-NEXT: v_accvgpr_read_b32 v4, a1 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a0 -; GFX908-NEXT: v_accvgpr_read_b32 v10, a7 -; GFX908-NEXT: v_accvgpr_read_b32 v9, a6 -; GFX908-NEXT: v_accvgpr_read_b32 v8, a5 -; GFX908-NEXT: v_accvgpr_read_b32 v7, a4 -; GFX908-NEXT: v_accvgpr_read_b32 v14, a11 -; GFX908-NEXT: v_accvgpr_read_b32 v13, a10 -; GFX908-NEXT: v_accvgpr_read_b32 v12, a9 -; GFX908-NEXT: v_accvgpr_read_b32 v11, a8 -; GFX908-NEXT: v_accvgpr_read_b32 v18, a15 -; GFX908-NEXT: v_accvgpr_read_b32 v17, a14 -; GFX908-NEXT: v_accvgpr_read_b32 v16, a13 -; GFX908-NEXT: v_accvgpr_read_b32 v15, a12 -; GFX908-NEXT: v_accvgpr_read_b32 v22, a19 -; GFX908-NEXT: v_accvgpr_read_b32 v21, a18 -; GFX908-NEXT: v_accvgpr_read_b32 v20, a17 -; GFX908-NEXT: v_accvgpr_read_b32 v19, a16 -; GFX908-NEXT: v_accvgpr_read_b32 v26, a23 -; GFX908-NEXT: v_accvgpr_read_b32 v25, a22 -; GFX908-NEXT: v_accvgpr_read_b32 v24, a21 -; GFX908-NEXT: v_accvgpr_read_b32 v23, a20 -; GFX908-NEXT: v_accvgpr_read_b32 v30, a27 -; GFX908-NEXT: v_accvgpr_read_b32 v29, a26 -; GFX908-NEXT: v_accvgpr_read_b32 v28, a25 -; GFX908-NEXT: v_accvgpr_read_b32 v27, a24 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-NEXT: v_accvgpr_read_b32 v7, a7 +; GFX908-NEXT: v_accvgpr_read_b32 v11, a11 +; GFX908-NEXT: v_accvgpr_read_b32 v15, a15 +; GFX908-NEXT: v_accvgpr_read_b32 v19, a19 +; GFX908-NEXT: v_accvgpr_read_b32 v23, a23 +; GFX908-NEXT: v_accvgpr_read_b32 v27, a27 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-NEXT: v_accvgpr_read_b32 v6, a6 +; GFX908-NEXT: v_accvgpr_read_b32 v5, a5 +; GFX908-NEXT: v_accvgpr_read_b32 v4, a4 +; GFX908-NEXT: v_accvgpr_read_b32 v10, a10 +; GFX908-NEXT: v_accvgpr_read_b32 v9, a9 +; GFX908-NEXT: v_accvgpr_read_b32 v8, a8 +; GFX908-NEXT: v_accvgpr_read_b32 v14, a14 +; GFX908-NEXT: v_accvgpr_read_b32 v13, a13 +; GFX908-NEXT: v_accvgpr_read_b32 v12, a12 +; GFX908-NEXT: v_accvgpr_read_b32 v18, a18 +; GFX908-NEXT: v_accvgpr_read_b32 v17, a17 +; GFX908-NEXT: v_accvgpr_read_b32 v16, a16 +; GFX908-NEXT: v_accvgpr_read_b32 v22, a22 +; GFX908-NEXT: v_accvgpr_read_b32 v21, a21 +; GFX908-NEXT: v_accvgpr_read_b32 v20, a20 +; GFX908-NEXT: v_accvgpr_read_b32 v26, a26 +; GFX908-NEXT: v_accvgpr_read_b32 v25, a25 +; GFX908-NEXT: v_accvgpr_read_b32 v24, a24 ; GFX908-NEXT: v_accvgpr_read_b32 v35, a31 ; GFX908-NEXT: v_accvgpr_read_b32 v34, a30 ; GFX908-NEXT: v_accvgpr_read_b32 v33, a29 @@ -621,19 +621,19 @@ define amdgpu_kernel void @test_call_areg64() #0 { ; GFX908-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GFX908-NEXT: global_store_dwordx4 v[0:1], v[32:35], off ; GFX908-NEXT: s_waitcnt vmcnt(0) -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[27:30], off +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[24:27], off ; GFX908-NEXT: s_waitcnt vmcnt(0) -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[23:26], off +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[20:23], off ; GFX908-NEXT: s_waitcnt vmcnt(0) -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[19:22], off +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[16:19], off ; GFX908-NEXT: s_waitcnt vmcnt(0) -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[15:18], off +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[12:15], off ; GFX908-NEXT: s_waitcnt vmcnt(0) -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[11:14], off +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[8:11], off ; GFX908-NEXT: s_waitcnt vmcnt(0) -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[7:10], off +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[4:7], off ; GFX908-NEXT: s_waitcnt vmcnt(0) -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[3:6], off +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off ; GFX908-NEXT: s_waitcnt vmcnt(0) ; GFX908-NEXT: s_endpgm bb: @@ -720,34 +720,34 @@ define amdgpu_kernel void @test_call_areg31_63() #0 { ; GFX908-NEXT: ;;#ASMSTART ; GFX908-NEXT: ; def a[0:31] ; GFX908-NEXT: ;;#ASMEND -; GFX908-NEXT: v_accvgpr_read_b32 v6, a3 -; GFX908-NEXT: v_accvgpr_read_b32 v5, a2 -; GFX908-NEXT: v_accvgpr_read_b32 v4, a1 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a0 -; GFX908-NEXT: v_accvgpr_read_b32 v10, a7 -; GFX908-NEXT: v_accvgpr_read_b32 v9, a6 -; GFX908-NEXT: v_accvgpr_read_b32 v8, a5 -; GFX908-NEXT: v_accvgpr_read_b32 v7, a4 -; GFX908-NEXT: v_accvgpr_read_b32 v14, a11 -; GFX908-NEXT: v_accvgpr_read_b32 v13, a10 -; GFX908-NEXT: v_accvgpr_read_b32 v12, a9 -; GFX908-NEXT: v_accvgpr_read_b32 v11, a8 -; GFX908-NEXT: v_accvgpr_read_b32 v18, a15 -; GFX908-NEXT: v_accvgpr_read_b32 v17, a14 -; GFX908-NEXT: v_accvgpr_read_b32 v16, a13 -; GFX908-NEXT: v_accvgpr_read_b32 v15, a12 -; GFX908-NEXT: v_accvgpr_read_b32 v22, a19 -; GFX908-NEXT: v_accvgpr_read_b32 v21, a18 -; GFX908-NEXT: v_accvgpr_read_b32 v20, a17 -; GFX908-NEXT: v_accvgpr_read_b32 v19, a16 -; GFX908-NEXT: v_accvgpr_read_b32 v26, a23 -; GFX908-NEXT: v_accvgpr_read_b32 v25, a22 -; GFX908-NEXT: v_accvgpr_read_b32 v24, a21 -; GFX908-NEXT: v_accvgpr_read_b32 v23, a20 -; GFX908-NEXT: v_accvgpr_read_b32 v30, a27 -; GFX908-NEXT: v_accvgpr_read_b32 v29, a26 -; GFX908-NEXT: v_accvgpr_read_b32 v28, a25 -; GFX908-NEXT: v_accvgpr_read_b32 v27, a24 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a3 +; GFX908-NEXT: v_accvgpr_read_b32 v7, a7 +; GFX908-NEXT: v_accvgpr_read_b32 v11, a11 +; GFX908-NEXT: v_accvgpr_read_b32 v15, a15 +; GFX908-NEXT: v_accvgpr_read_b32 v19, a19 +; GFX908-NEXT: v_accvgpr_read_b32 v23, a23 +; GFX908-NEXT: v_accvgpr_read_b32 v27, a27 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a2 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a1 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-NEXT: v_accvgpr_read_b32 v6, a6 +; GFX908-NEXT: v_accvgpr_read_b32 v5, a5 +; GFX908-NEXT: v_accvgpr_read_b32 v4, a4 +; GFX908-NEXT: v_accvgpr_read_b32 v10, a10 +; GFX908-NEXT: v_accvgpr_read_b32 v9, a9 +; GFX908-NEXT: v_accvgpr_read_b32 v8, a8 +; GFX908-NEXT: v_accvgpr_read_b32 v14, a14 +; GFX908-NEXT: v_accvgpr_read_b32 v13, a13 +; GFX908-NEXT: v_accvgpr_read_b32 v12, a12 +; GFX908-NEXT: v_accvgpr_read_b32 v18, a18 +; GFX908-NEXT: v_accvgpr_read_b32 v17, a17 +; GFX908-NEXT: v_accvgpr_read_b32 v16, a16 +; GFX908-NEXT: v_accvgpr_read_b32 v22, a22 +; GFX908-NEXT: v_accvgpr_read_b32 v21, a21 +; GFX908-NEXT: v_accvgpr_read_b32 v20, a20 +; GFX908-NEXT: v_accvgpr_read_b32 v26, a26 +; GFX908-NEXT: v_accvgpr_read_b32 v25, a25 +; GFX908-NEXT: v_accvgpr_read_b32 v24, a24 ; GFX908-NEXT: v_accvgpr_read_b32 v35, a31 ; GFX908-NEXT: v_accvgpr_read_b32 v34, a30 ; GFX908-NEXT: v_accvgpr_read_b32 v33, a29 @@ -756,19 +756,19 @@ define amdgpu_kernel void @test_call_areg31_63() #0 { ; GFX908-NEXT: s_swappc_b64 s[30:31], s[16:17] ; GFX908-NEXT: global_store_dwordx4 v[0:1], v[32:35], off ; GFX908-NEXT: s_waitcnt vmcnt(0) -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[27:30], off +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[24:27], off ; GFX908-NEXT: s_waitcnt vmcnt(0) -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[23:26], off +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[20:23], off ; GFX908-NEXT: s_waitcnt vmcnt(0) -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[19:22], off +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[16:19], off ; GFX908-NEXT: s_waitcnt vmcnt(0) -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[15:18], off +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[12:15], off ; GFX908-NEXT: s_waitcnt vmcnt(0) -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[11:14], off +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[8:11], off ; GFX908-NEXT: s_waitcnt vmcnt(0) -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[7:10], off +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[4:7], off ; GFX908-NEXT: s_waitcnt vmcnt(0) -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[3:6], off +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[0:3], off ; GFX908-NEXT: s_waitcnt vmcnt(0) ; GFX908-NEXT: s_endpgm bb: @@ -856,30 +856,30 @@ define amdgpu_kernel void @test_call_unknown() #0 { ; GFX908-NEXT: ; def a[0:31] ; GFX908-NEXT: ;;#ASMEND ; GFX908-NEXT: v_accvgpr_read_b32 v43, a3 +; GFX908-NEXT: v_accvgpr_read_b32 v47, a7 +; GFX908-NEXT: v_accvgpr_read_b32 v59, a11 +; GFX908-NEXT: v_accvgpr_read_b32 v63, a15 +; GFX908-NEXT: v_accvgpr_read_b32 v75, a19 +; GFX908-NEXT: v_accvgpr_read_b32 v79, a23 +; GFX908-NEXT: v_accvgpr_read_b32 v91, a27 ; GFX908-NEXT: v_accvgpr_read_b32 v42, a2 ; GFX908-NEXT: v_accvgpr_read_b32 v41, a1 ; GFX908-NEXT: v_accvgpr_read_b32 v40, a0 -; GFX908-NEXT: v_accvgpr_read_b32 v47, a7 ; GFX908-NEXT: v_accvgpr_read_b32 v46, a6 ; GFX908-NEXT: v_accvgpr_read_b32 v45, a5 ; GFX908-NEXT: v_accvgpr_read_b32 v44, a4 -; GFX908-NEXT: v_accvgpr_read_b32 v59, a11 ; GFX908-NEXT: v_accvgpr_read_b32 v58, a10 ; GFX908-NEXT: v_accvgpr_read_b32 v57, a9 ; GFX908-NEXT: v_accvgpr_read_b32 v56, a8 -; GFX908-NEXT: v_accvgpr_read_b32 v63, a15 ; GFX908-NEXT: v_accvgpr_read_b32 v62, a14 ; GFX908-NEXT: v_accvgpr_read_b32 v61, a13 ; GFX908-NEXT: v_accvgpr_read_b32 v60, a12 -; GFX908-NEXT: v_accvgpr_read_b32 v75, a19 ; GFX908-NEXT: v_accvgpr_read_b32 v74, a18 ; GFX908-NEXT: v_accvgpr_read_b32 v73, a17 ; GFX908-NEXT: v_accvgpr_read_b32 v72, a16 -; GFX908-NEXT: v_accvgpr_read_b32 v79, a23 ; GFX908-NEXT: v_accvgpr_read_b32 v78, a22 ; GFX908-NEXT: v_accvgpr_read_b32 v77, a21 ; GFX908-NEXT: v_accvgpr_read_b32 v76, a20 -; GFX908-NEXT: v_accvgpr_read_b32 v91, a27 ; GFX908-NEXT: v_accvgpr_read_b32 v90, a26 ; GFX908-NEXT: v_accvgpr_read_b32 v89, a25 ; GFX908-NEXT: v_accvgpr_read_b32 v88, a24 diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-no-agprs-violations.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-no-agprs-violations.ll index c8f75373a55dd..1a3f1ef9223ae 100644 --- a/llvm/test/CodeGen/AMDGPU/amdgpu-no-agprs-violations.ll +++ b/llvm/test/CodeGen/AMDGPU/amdgpu-no-agprs-violations.ll @@ -34,7 +34,7 @@ define void @func_illegal_agpr_use_asm() #0 { ; GFX908: v_accvgpr_write_b32 ; GFX90A-NOT: v_accvgpr_write_b32 -; GFX908: NumVgprs: 5 +; GFX908: NumVgprs: 30 ; GFX908: NumAgprs: 32 ; GFX90A: NumVgprs: 35 ; GFX90A: NumAgprs: 0 diff --git a/llvm/test/CodeGen/AMDGPU/coexec-sched-agpr-excess.mir b/llvm/test/CodeGen/AMDGPU/coexec-sched-agpr-excess.mir new file mode 100644 index 0000000000000..7e65714c45d18 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/coexec-sched-agpr-excess.mir @@ -0,0 +1,71 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6 +# RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -run-pass=machine-scheduler -amdgpu-sched-strategy=coexec -amdgpu-stress-agpr=8 %s -o - | FileCheck --check-prefix=LIMIT %s +# RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -run-pass=machine-scheduler -amdgpu-sched-strategy=coexec %s -o - | FileCheck --check-prefix=NO_LIMIT %s + +# Test that AGPR pressure is tracked during scheduling. +# Two independent MFMA chains (3 MFMAs each), results written via DS_WRITE. +# With -amdgpu-stress-agpr=8, the scheduler should prefer scheduling +# one chain's consumer before starting the other chain to limit AGPR pressure. + +--- | + define void @agpr_excess_tracking() #0 { ret void } + attributes #0 = { "amdgpu-waves-per-eu"="1,1" "amdgpu-flat-work-group-size"="64,64" } +... + +--- +name: agpr_excess_tracking +tracksRegLiveness: true +machineFunctionInfo: + isEntryFunction: true +body: | + bb.0: + ; LIMIT-LABEL: name: agpr_excess_tracking + ; LIMIT: %s0a:areg_128_align2 = DS_READ_B128_gfx9 undef %base0:vgpr_32, 0, 0, implicit $exec :: (load (s128), addrspace 3) + ; LIMIT-NEXT: %s0b:areg_128_align2 = DS_READ_B128_gfx9 undef %base1:vgpr_32, 16, 0, implicit $exec :: (load (s128), addrspace 3) + ; LIMIT-NEXT: %res1:areg_128_align2 = DS_READ_B128_gfx9 undef %base3:vgpr_32, 54, 0, implicit $exec :: (load (s128), addrspace 3) + ; LIMIT-NEXT: %res1:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 %s0a, %s0b, %res1, 0, 0, 0, implicit $mode, implicit $exec + ; LIMIT-NEXT: %res1:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 %s0a, %s0b, %res1, 0, 0, 0, implicit $mode, implicit $exec + ; LIMIT-NEXT: %s1a:areg_128_align2 = DS_READ_B128_gfx9 undef %base2:vgpr_32, 32, 0, implicit $exec :: (load (s128), addrspace 3) + ; LIMIT-NEXT: %res2:areg_128_align2 = DS_READ_B128_gfx9 undef %base3:vgpr_32, 58, 0, implicit $exec :: (load (s128), addrspace 3) + ; LIMIT-NEXT: %s1b:areg_128_align2 = DS_READ_B128_gfx9 undef %base3:vgpr_32, 48, 0, implicit $exec :: (load (s128), addrspace 3) + ; LIMIT-NEXT: DS_WRITE_B128_gfx9 undef %addr0:vgpr_32, %res1, 0, 0, implicit $exec :: (store (s128), addrspace 3) + ; LIMIT-NEXT: %res2:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 %s1a, %s1b, %res2, 0, 0, 0, implicit $mode, implicit $exec + ; LIMIT-NEXT: %res2:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 %s1a, %s1b, %res2, 0, 0, 0, implicit $mode, implicit $exec + ; LIMIT-NEXT: DS_WRITE_B128_gfx9 undef %addr1:vgpr_32, %res2, 16, 0, implicit $exec :: (store (s128), addrspace 3) + ; LIMIT-NEXT: S_ENDPGM 0 + ; + ; NO_LIMIT-LABEL: name: agpr_excess_tracking + ; NO_LIMIT: %s0a:areg_128_align2 = DS_READ_B128_gfx9 undef %base0:vgpr_32, 0, 0, implicit $exec :: (load (s128), addrspace 3) + ; NO_LIMIT-NEXT: %s0b:areg_128_align2 = DS_READ_B128_gfx9 undef %base1:vgpr_32, 16, 0, implicit $exec :: (load (s128), addrspace 3) + ; NO_LIMIT-NEXT: %res1:areg_128_align2 = DS_READ_B128_gfx9 undef %base3:vgpr_32, 54, 0, implicit $exec :: (load (s128), addrspace 3) + ; NO_LIMIT-NEXT: %s1a:areg_128_align2 = DS_READ_B128_gfx9 undef %base2:vgpr_32, 32, 0, implicit $exec :: (load (s128), addrspace 3) + ; NO_LIMIT-NEXT: %res2:areg_128_align2 = DS_READ_B128_gfx9 undef %base3:vgpr_32, 58, 0, implicit $exec :: (load (s128), addrspace 3) + ; NO_LIMIT-NEXT: %s1b:areg_128_align2 = DS_READ_B128_gfx9 undef %base3:vgpr_32, 48, 0, implicit $exec :: (load (s128), addrspace 3) + ; NO_LIMIT-NEXT: %res1:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 %s0a, %s0b, %res1, 0, 0, 0, implicit $mode, implicit $exec + ; NO_LIMIT-NEXT: %res2:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 %s1a, %s1b, %res2, 0, 0, 0, implicit $mode, implicit $exec + ; NO_LIMIT-NEXT: %res1:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 %s0a, %s0b, %res1, 0, 0, 0, implicit $mode, implicit $exec + ; NO_LIMIT-NEXT: %res2:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 %s1a, %s1b, %res2, 0, 0, 0, implicit $mode, implicit $exec + ; NO_LIMIT-NEXT: DS_WRITE_B128_gfx9 undef %addr0:vgpr_32, %res1, 0, 0, implicit $exec :: (store (s128), addrspace 3) + ; NO_LIMIT-NEXT: DS_WRITE_B128_gfx9 undef %addr1:vgpr_32, %res2, 16, 0, implicit $exec :: (store (s128), addrspace 3) + ; NO_LIMIT-NEXT: S_ENDPGM 0 + %s0a:areg_128_align2 = DS_READ_B128_gfx9 undef %base0:vgpr_32, 0, 0, implicit $exec :: (load (s128), addrspace 3) + %s0b:areg_128_align2 = DS_READ_B128_gfx9 undef %base1:vgpr_32, 16, 0, implicit $exec :: (load (s128), addrspace 3) + %s1a:areg_128_align2 = DS_READ_B128_gfx9 undef %base2:vgpr_32, 32, 0, implicit $exec :: (load (s128), addrspace 3) + %s1b:areg_128_align2 = DS_READ_B128_gfx9 undef %base3:vgpr_32, 48, 0, implicit $exec :: (load (s128), addrspace 3) + + %res1:areg_128_align2 = DS_READ_B128_gfx9 undef %base3:vgpr_32, 54, 0, implicit $exec :: (load (s128), addrspace 3) + %res2:areg_128_align2 = DS_READ_B128_gfx9 undef %base3:vgpr_32, 58, 0, implicit $exec :: (load (s128), addrspace 3) + + ; Chain 0 + %res1:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 %s0a, %s0b, %res1, 0, 0, 0, implicit $mode, implicit $exec + %res1:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 %s0a, %s0b, %res1, 0, 0, 0, implicit $mode, implicit $exec + + ; Chain 1 + %res2:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 %s1a, %s1b, %res2, 0, 0, 0, implicit $mode, implicit $exec + %res2:areg_128_align2 = V_MFMA_F32_16X16X32_F16_e64 %s1a, %s1b, %res2, 0, 0, 0, implicit $mode, implicit $exec + + DS_WRITE_B128_gfx9 undef %addr0:vgpr_32, %res1, 0, 0, implicit $exec :: (store (s128), addrspace 3) + DS_WRITE_B128_gfx9 undef %addr1:vgpr_32, %res2, 16, 0, implicit $exec :: (store (s128), addrspace 3) + + S_ENDPGM 0 +... diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.ll index 5810d859f0cf4..ef04672556acc 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.opt.ll @@ -32,6 +32,77 @@ define amdgpu_kernel void @test_iglp_opt_mfma_gemm(ptr addrspace(3) noalias %in, ; SDAG-NEXT: ds_read_b128 a[8:11], v2 offset:57376 ; SDAG-NEXT: ds_read_b128 a[12:15], v2 offset:57392 ; SDAG-NEXT: v_mov_b32_e32 v2, 1.0 +; SDAG-NEXT: v_add_u32_e32 v0, s1, v0 +; SDAG-NEXT: s_waitcnt lgkmcnt(0) +; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31] +; SDAG-NEXT: ds_read_b128 a[60:63], v1 offset:112 +; SDAG-NEXT: ds_read_b128 a[56:59], v1 offset:96 +; SDAG-NEXT: ds_read_b128 a[52:55], v1 offset:80 +; SDAG-NEXT: ds_read_b128 a[48:51], v1 offset:64 +; SDAG-NEXT: ds_read_b128 a[32:35], v1 +; SDAG-NEXT: ds_read_b128 a[36:39], v1 offset:16 +; SDAG-NEXT: ds_read_b128 a[40:43], v1 offset:32 +; SDAG-NEXT: ds_read_b128 a[44:47], v1 offset:48 +; SDAG-NEXT: s_waitcnt lgkmcnt(0) +; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v2, v3, a[32:63] +; SDAG-NEXT: s_nop 15 +; SDAG-NEXT: s_nop 2 +; SDAG-NEXT: ds_write_b128 v0, a[60:63] offset:112 +; SDAG-NEXT: ds_write_b128 v0, a[56:59] offset:96 +; SDAG-NEXT: ds_write_b128 v0, a[52:55] offset:80 +; SDAG-NEXT: ds_write_b128 v0, a[48:51] offset:64 +; SDAG-NEXT: ds_write_b128 v0, a[44:47] offset:48 +; SDAG-NEXT: ds_write_b128 v0, a[40:43] offset:32 +; SDAG-NEXT: ds_write_b128 v0, a[36:39] offset:16 +; SDAG-NEXT: ds_write_b128 v0, a[32:35] +; SDAG-NEXT: ds_read_b128 a[60:63], v1 offset:8304 +; SDAG-NEXT: ds_read_b128 a[56:59], v1 offset:8288 +; SDAG-NEXT: ds_read_b128 a[52:55], v1 offset:8272 +; SDAG-NEXT: ds_read_b128 a[48:51], v1 offset:8256 +; SDAG-NEXT: ds_read_b128 a[44:47], v1 offset:8240 +; SDAG-NEXT: ds_read_b128 a[40:43], v1 offset:8224 +; SDAG-NEXT: ds_read_b128 a[36:39], v1 offset:8208 +; SDAG-NEXT: ds_read_b128 a[32:35], v1 offset:8192 +; SDAG-NEXT: s_waitcnt lgkmcnt(0) +; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v2, v3, a[32:63] +; SDAG-NEXT: v_mov_b32_e32 v0, s1 +; SDAG-NEXT: ds_write_b128 v0, a[24:27] offset:32864 +; SDAG-NEXT: ds_write_b128 v0, a[28:31] offset:32880 +; SDAG-NEXT: ds_write_b128 v0, a[16:19] offset:32832 +; SDAG-NEXT: ds_write_b128 v0, a[20:23] offset:32848 +; SDAG-NEXT: ds_write_b128 v0, a[8:11] offset:32800 +; SDAG-NEXT: ds_write_b128 v0, a[12:15] offset:32816 +; SDAG-NEXT: ds_write_b128 v0, a[0:3] offset:32768 +; SDAG-NEXT: ds_write_b128 v0, a[4:7] offset:32784 +; SDAG-NEXT: s_nop 9 +; SDAG-NEXT: ds_write_b128 v0, a[56:59] offset:8288 +; SDAG-NEXT: ds_write_b128 v0, a[60:63] offset:8304 +; SDAG-NEXT: ds_write_b128 v0, a[48:51] offset:8256 +; SDAG-NEXT: ds_write_b128 v0, a[52:55] offset:8272 +; SDAG-NEXT: ds_write_b128 v0, a[40:43] offset:8224 +; SDAG-NEXT: ds_write_b128 v0, a[44:47] offset:8240 +; SDAG-NEXT: ds_write_b128 v0, a[32:35] offset:8192 +; SDAG-NEXT: ds_write_b128 v0, a[36:39] offset:8208 +; SDAG-NEXT: ds_read_b128 a[60:63], v1 offset:24688 +; SDAG-NEXT: ds_read_b128 a[56:59], v1 offset:24672 +; SDAG-NEXT: ds_read_b128 a[52:55], v1 offset:24656 +; SDAG-NEXT: ds_read_b128 a[48:51], v1 offset:24640 +; SDAG-NEXT: ds_read_b128 a[44:47], v1 offset:24624 +; SDAG-NEXT: ds_read_b128 a[40:43], v1 offset:24608 +; SDAG-NEXT: ds_read_b128 a[36:39], v1 offset:24592 +; SDAG-NEXT: ds_read_b128 a[32:35], v1 offset:24576 +; SDAG-NEXT: s_waitcnt lgkmcnt(0) +; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v2, v3, a[32:63] +; SDAG-NEXT: s_nop 15 +; SDAG-NEXT: s_nop 2 +; SDAG-NEXT: ds_write_b128 v0, a[56:59] offset:16480 +; SDAG-NEXT: ds_write_b128 v0, a[60:63] offset:16496 +; SDAG-NEXT: ds_write_b128 v0, a[48:51] offset:16448 +; SDAG-NEXT: ds_write_b128 v0, a[52:55] offset:16464 +; SDAG-NEXT: ds_write_b128 v0, a[40:43] offset:16416 +; SDAG-NEXT: ds_write_b128 v0, a[44:47] offset:16432 +; SDAG-NEXT: ds_write_b128 v0, a[32:35] offset:16384 +; SDAG-NEXT: ds_write_b128 v0, a[36:39] offset:16400 ; SDAG-NEXT: ds_read_b128 a[60:63], v1 offset:49264 ; SDAG-NEXT: ds_read_b128 a[56:59], v1 offset:49248 ; SDAG-NEXT: ds_read_b128 a[52:55], v1 offset:49232 @@ -40,222 +111,149 @@ define amdgpu_kernel void @test_iglp_opt_mfma_gemm(ptr addrspace(3) noalias %in, ; SDAG-NEXT: ds_read_b128 a[40:43], v1 offset:49184 ; SDAG-NEXT: ds_read_b128 a[36:39], v1 offset:49168 ; SDAG-NEXT: ds_read_b128 a[32:35], v1 offset:49152 -; SDAG-NEXT: s_waitcnt lgkmcnt(8) -; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v3, a[0:31] -; SDAG-NEXT: ds_read_b128 a[156:159], v1 offset:112 -; SDAG-NEXT: ds_read_b128 a[152:155], v1 offset:96 -; SDAG-NEXT: ds_read_b128 a[68:71], v1 offset:24592 -; SDAG-NEXT: ds_read_b128 a[64:67], v1 offset:24576 -; SDAG-NEXT: v_add_u32_e32 v0, s1, v0 -; SDAG-NEXT: s_waitcnt lgkmcnt(4) -; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v2, v3, a[32:63] -; SDAG-NEXT: ds_read_b128 a[148:151], v1 offset:80 -; SDAG-NEXT: ds_read_b128 a[144:147], v1 offset:64 -; SDAG-NEXT: ds_read_b128 a[128:131], v1 -; SDAG-NEXT: ds_read_b128 a[132:135], v1 offset:16 -; SDAG-NEXT: ds_read_b128 a[136:139], v1 offset:32 -; SDAG-NEXT: ds_read_b128 a[140:143], v1 offset:48 -; SDAG-NEXT: s_waitcnt lgkmcnt(0) -; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v2, v3, a[128:159] -; SDAG-NEXT: ds_read_b128 a[124:127], v1 offset:8304 -; SDAG-NEXT: ds_read_b128 a[120:123], v1 offset:8288 -; SDAG-NEXT: ds_read_b128 a[116:119], v1 offset:8272 -; SDAG-NEXT: ds_read_b128 a[112:115], v1 offset:8256 -; SDAG-NEXT: ds_read_b128 a[108:111], v1 offset:8240 -; SDAG-NEXT: ds_read_b128 a[104:107], v1 offset:8224 -; SDAG-NEXT: ds_read_b128 a[100:103], v1 offset:8208 -; SDAG-NEXT: ds_read_b128 a[96:99], v1 offset:8192 ; SDAG-NEXT: s_waitcnt lgkmcnt(0) -; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v2, v3, a[96:127] -; SDAG-NEXT: ds_read_b128 a[92:95], v1 offset:24688 -; SDAG-NEXT: ds_read_b128 a[88:91], v1 offset:24672 -; SDAG-NEXT: ds_read_b128 a[84:87], v1 offset:24656 -; SDAG-NEXT: ds_read_b128 a[80:83], v1 offset:24640 -; SDAG-NEXT: ds_read_b128 a[76:79], v1 offset:24624 -; SDAG-NEXT: ds_read_b128 a[72:75], v1 offset:24608 +; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v2, v3, a[32:63] +; SDAG-NEXT: s_nop 15 ; SDAG-NEXT: s_nop 2 -; SDAG-NEXT: ds_write_b128 v0, a[156:159] offset:112 -; SDAG-NEXT: ds_write_b128 v0, a[152:155] offset:96 -; SDAG-NEXT: ds_write_b128 v0, a[148:151] offset:80 -; SDAG-NEXT: ds_write_b128 v0, a[144:147] offset:64 -; SDAG-NEXT: ds_write_b128 v0, a[140:143] offset:48 -; SDAG-NEXT: ds_write_b128 v0, a[136:139] offset:32 -; SDAG-NEXT: ds_write_b128 v0, a[132:135] offset:16 -; SDAG-NEXT: ds_write_b128 v0, a[128:131] -; SDAG-NEXT: v_mov_b32_e32 v0, s1 -; SDAG-NEXT: s_waitcnt lgkmcnt(8) -; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v2, v3, a[64:95] ; SDAG-NEXT: ds_write_b128 v0, a[56:59] offset:24672 ; SDAG-NEXT: ds_write_b128 v0, a[60:63] offset:24688 ; SDAG-NEXT: ds_write_b128 v0, a[48:51] offset:24640 -; SDAG-NEXT: ds_write_b128 v0, a[120:123] offset:8288 -; SDAG-NEXT: ds_write_b128 v0, a[124:127] offset:8304 -; SDAG-NEXT: ds_write_b128 v0, a[112:115] offset:8256 -; SDAG-NEXT: ds_write_b128 v0, a[116:119] offset:8272 -; SDAG-NEXT: ds_write_b128 v0, a[104:107] offset:8224 -; SDAG-NEXT: ds_write_b128 v0, a[108:111] offset:8240 -; SDAG-NEXT: ds_write_b128 v0, a[96:99] offset:8192 -; SDAG-NEXT: ds_write_b128 v0, a[100:103] offset:8208 ; SDAG-NEXT: ds_write_b128 v0, a[52:55] offset:24656 ; SDAG-NEXT: ds_write_b128 v0, a[40:43] offset:24608 ; SDAG-NEXT: ds_write_b128 v0, a[44:47] offset:24624 ; SDAG-NEXT: ds_write_b128 v0, a[32:35] offset:24576 ; SDAG-NEXT: ds_write_b128 v0, a[36:39] offset:24592 -; SDAG-NEXT: ds_write_b128 v0, a[24:27] offset:32864 -; SDAG-NEXT: ds_write_b128 v0, a[28:31] offset:32880 -; SDAG-NEXT: ds_write_b128 v0, a[16:19] offset:32832 -; SDAG-NEXT: ds_write_b128 v0, a[88:91] offset:16480 -; SDAG-NEXT: ds_write_b128 v0, a[92:95] offset:16496 -; SDAG-NEXT: ds_write_b128 v0, a[80:83] offset:16448 -; SDAG-NEXT: ds_write_b128 v0, a[84:87] offset:16464 -; SDAG-NEXT: ds_write_b128 v0, a[72:75] offset:16416 -; SDAG-NEXT: ds_write_b128 v0, a[76:79] offset:16432 -; SDAG-NEXT: ds_write_b128 v0, a[64:67] offset:16384 -; SDAG-NEXT: ds_write_b128 v0, a[68:71] offset:16400 -; SDAG-NEXT: ds_write_b128 v0, a[20:23] offset:32848 -; SDAG-NEXT: ds_write_b128 v0, a[8:11] offset:32800 -; SDAG-NEXT: ds_write_b128 v0, a[12:15] offset:32816 -; SDAG-NEXT: ds_write_b128 v0, a[0:3] offset:32768 -; SDAG-NEXT: ds_write_b128 v0, a[4:7] offset:32784 ; SDAG-NEXT: s_endpgm ; ; GISEL-LABEL: test_iglp_opt_mfma_gemm: ; GISEL: ; %bb.0: ; %entry ; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 ; GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0 -; GISEL-NEXT: v_lshlrev_b32_e32 v33, 7, v0 -; GISEL-NEXT: v_mov_b32_e32 v32, 1.0 -; GISEL-NEXT: v_mov_b32_e32 v34, 2.0 +; GISEL-NEXT: v_lshlrev_b32_e32 v0, 7, v0 +; GISEL-NEXT: v_mov_b32_e32 v68, 1.0 +; GISEL-NEXT: v_mov_b32_e32 v69, 2.0 ; GISEL-NEXT: s_waitcnt lgkmcnt(0) -; GISEL-NEXT: v_add_u32_e32 v0, s0, v33 -; GISEL-NEXT: ds_read_b128 a[96:99], v0 offset:8192 -; GISEL-NEXT: ds_read_b128 a[100:103], v0 offset:8208 -; GISEL-NEXT: ds_read_b128 a[104:107], v0 offset:8224 -; GISEL-NEXT: ds_read_b128 a[108:111], v0 offset:8240 -; GISEL-NEXT: ds_read_b128 a[112:115], v0 offset:8256 -; GISEL-NEXT: ds_read_b128 a[116:119], v0 offset:8272 -; GISEL-NEXT: ds_read_b128 a[120:123], v0 offset:8288 -; GISEL-NEXT: ds_read_b128 a[124:127], v0 offset:8304 +; GISEL-NEXT: v_add_u32_e32 v70, s0, v0 +; GISEL-NEXT: v_add_u32_e32 v1, 0xc000, v70 +; GISEL-NEXT: ds_read_b128 a[0:3], v1 offset:32768 +; GISEL-NEXT: ds_read_b128 a[4:7], v1 offset:32784 +; GISEL-NEXT: ds_read_b128 a[8:11], v1 offset:32800 +; GISEL-NEXT: ds_read_b128 a[12:15], v1 offset:32816 +; GISEL-NEXT: ds_read_b128 a[16:19], v1 offset:32832 +; GISEL-NEXT: ds_read_b128 a[20:23], v1 offset:32848 +; GISEL-NEXT: ds_read_b128 a[24:27], v1 offset:32864 +; GISEL-NEXT: ds_read_b128 a[28:31], v1 offset:32880 ; GISEL-NEXT: s_waitcnt lgkmcnt(0) -; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v32, v34, a[96:127] -; GISEL-NEXT: ds_read_b128 a[0:3], v0 offset:24576 -; GISEL-NEXT: ds_read_b128 a[4:7], v0 offset:24592 -; GISEL-NEXT: ds_read_b128 a[8:11], v0 offset:24608 -; GISEL-NEXT: ds_read_b128 a[12:15], v0 offset:24624 -; GISEL-NEXT: ds_read_b128 a[16:19], v0 offset:24640 -; GISEL-NEXT: ds_read_b128 a[20:23], v0 offset:24656 -; GISEL-NEXT: ds_read_b128 a[24:27], v0 offset:24672 -; GISEL-NEXT: ds_read_b128 a[28:31], v0 offset:24688 -; GISEL-NEXT: v_add_u32_e32 v1, 0xc000, v0 -; GISEL-NEXT: ds_read_b128 a[152:155], v0 offset:96 -; GISEL-NEXT: ds_read_b128 a[156:159], v0 offset:112 -; GISEL-NEXT: v_add_u32_e32 v33, s1, v33 +; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v68, v69, a[0:31] +; GISEL-NEXT: ds_read_b128 a[32:35], v70 +; GISEL-NEXT: ds_read_b128 a[36:39], v70 offset:16 +; GISEL-NEXT: ds_read_b128 a[40:43], v70 offset:32 +; GISEL-NEXT: ds_read_b128 a[44:47], v70 offset:48 +; GISEL-NEXT: ds_read_b128 a[48:51], v70 offset:64 +; GISEL-NEXT: ds_read_b128 a[52:55], v70 offset:80 +; GISEL-NEXT: ds_read_b128 a[56:59], v70 offset:96 +; GISEL-NEXT: ds_read_b128 a[60:63], v70 offset:112 +; GISEL-NEXT: v_add_u32_e32 v71, s1, v0 ; GISEL-NEXT: ; iglp_opt mask(0x00000000) -; GISEL-NEXT: s_waitcnt lgkmcnt(2) -; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v32, v34, a[0:31] -; GISEL-NEXT: ds_read_b128 a[32:35], v0 offset:49152 -; GISEL-NEXT: ds_read_b128 a[36:39], v0 offset:49168 -; GISEL-NEXT: ds_read_b128 a[40:43], v0 offset:49184 -; GISEL-NEXT: ds_read_b128 a[44:47], v0 offset:49200 -; GISEL-NEXT: ds_read_b128 a[48:51], v0 offset:49216 -; GISEL-NEXT: ds_read_b128 a[52:55], v0 offset:49232 -; GISEL-NEXT: ds_read_b128 a[56:59], v0 offset:49248 -; GISEL-NEXT: ds_read_b128 a[60:63], v0 offset:49264 -; GISEL-NEXT: s_waitcnt lgkmcnt(0) -; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v32, v34, a[32:63] -; GISEL-NEXT: ds_read_b128 a[64:67], v1 offset:32768 -; GISEL-NEXT: ds_read_b128 a[68:71], v1 offset:32784 -; GISEL-NEXT: ds_read_b128 a[72:75], v1 offset:32800 -; GISEL-NEXT: ds_read_b128 a[76:79], v1 offset:32816 -; GISEL-NEXT: ds_read_b128 a[80:83], v1 offset:32832 -; GISEL-NEXT: ds_read_b128 a[84:87], v1 offset:32848 -; GISEL-NEXT: ds_read_b128 a[88:91], v1 offset:32864 -; GISEL-NEXT: ds_read_b128 a[92:95], v1 offset:32880 ; GISEL-NEXT: s_waitcnt lgkmcnt(0) -; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v32, v34, a[64:95] -; GISEL-NEXT: ds_read_b128 a[128:131], v0 -; GISEL-NEXT: ds_read_b128 a[132:135], v0 offset:16 -; GISEL-NEXT: ds_read_b128 a[136:139], v0 offset:32 -; GISEL-NEXT: ds_read_b128 a[140:143], v0 offset:48 -; GISEL-NEXT: ds_read_b128 a[144:147], v0 offset:64 -; GISEL-NEXT: ds_read_b128 a[148:151], v0 offset:80 +; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v68, v69, a[32:63] +; GISEL-NEXT: s_nop 15 +; GISEL-NEXT: s_nop 2 +; GISEL-NEXT: v_accvgpr_read_b32 v4, a32 +; GISEL-NEXT: v_accvgpr_read_b32 v5, a33 +; GISEL-NEXT: v_accvgpr_read_b32 v6, a34 +; GISEL-NEXT: v_accvgpr_read_b32 v7, a35 +; GISEL-NEXT: v_accvgpr_read_b32 v8, a36 +; GISEL-NEXT: v_accvgpr_read_b32 v9, a37 +; GISEL-NEXT: v_accvgpr_read_b32 v10, a38 +; GISEL-NEXT: v_accvgpr_read_b32 v11, a39 +; GISEL-NEXT: v_accvgpr_read_b32 v12, a40 +; GISEL-NEXT: v_accvgpr_read_b32 v13, a41 +; GISEL-NEXT: v_accvgpr_read_b32 v14, a42 +; GISEL-NEXT: v_accvgpr_read_b32 v15, a43 +; GISEL-NEXT: v_accvgpr_read_b32 v16, a44 +; GISEL-NEXT: v_accvgpr_read_b32 v17, a45 +; GISEL-NEXT: v_accvgpr_read_b32 v18, a46 +; GISEL-NEXT: v_accvgpr_read_b32 v19, a47 +; GISEL-NEXT: v_accvgpr_read_b32 v20, a48 +; GISEL-NEXT: v_accvgpr_read_b32 v21, a49 +; GISEL-NEXT: v_accvgpr_read_b32 v22, a50 +; GISEL-NEXT: v_accvgpr_read_b32 v23, a51 +; GISEL-NEXT: v_accvgpr_read_b32 v24, a52 +; GISEL-NEXT: v_accvgpr_read_b32 v25, a53 +; GISEL-NEXT: v_accvgpr_read_b32 v26, a54 +; GISEL-NEXT: v_accvgpr_read_b32 v27, a55 +; GISEL-NEXT: v_accvgpr_read_b32 v28, a56 +; GISEL-NEXT: v_accvgpr_read_b32 v29, a57 +; GISEL-NEXT: v_accvgpr_read_b32 v30, a58 +; GISEL-NEXT: v_accvgpr_read_b32 v31, a59 +; GISEL-NEXT: v_accvgpr_read_b32 v32, a60 +; GISEL-NEXT: v_accvgpr_read_b32 v33, a61 +; GISEL-NEXT: v_accvgpr_read_b32 v34, a62 +; GISEL-NEXT: v_accvgpr_read_b32 v35, a63 +; GISEL-NEXT: ds_write_b128 v71, v[4:7] +; GISEL-NEXT: ds_write_b128 v71, v[8:11] offset:16 +; GISEL-NEXT: ds_write_b128 v71, v[12:15] offset:32 +; GISEL-NEXT: ds_write_b128 v71, v[16:19] offset:48 +; GISEL-NEXT: ds_write_b128 v71, v[20:23] offset:64 +; GISEL-NEXT: ds_write_b128 v71, v[24:27] offset:80 +; GISEL-NEXT: ds_write_b128 v71, v[28:31] offset:96 +; GISEL-NEXT: ds_read_b128 a[32:35], v70 offset:8192 +; GISEL-NEXT: ds_read_b128 a[36:39], v70 offset:8208 +; GISEL-NEXT: ds_read_b128 a[40:43], v70 offset:8224 +; GISEL-NEXT: ds_read_b128 a[44:47], v70 offset:8240 +; GISEL-NEXT: ds_read_b128 a[48:51], v70 offset:8256 +; GISEL-NEXT: ds_read_b128 a[52:55], v70 offset:8272 +; GISEL-NEXT: ds_read_b128 a[56:59], v70 offset:8288 +; GISEL-NEXT: ds_read_b128 a[60:63], v70 offset:8304 ; GISEL-NEXT: s_waitcnt lgkmcnt(0) -; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v32, v34, a[128:159] +; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v68, v69, a[32:63] +; GISEL-NEXT: ds_write_b128 v71, v[32:35] offset:112 ; GISEL-NEXT: v_mov_b32_e32 v32, s1 ; GISEL-NEXT: s_nop 15 -; GISEL-NEXT: s_nop 1 -; GISEL-NEXT: v_accvgpr_read_b32 v0, a128 -; GISEL-NEXT: v_accvgpr_read_b32 v1, a129 -; GISEL-NEXT: v_accvgpr_read_b32 v2, a130 -; GISEL-NEXT: v_accvgpr_read_b32 v3, a131 -; GISEL-NEXT: v_accvgpr_read_b32 v4, a132 -; GISEL-NEXT: v_accvgpr_read_b32 v5, a133 -; GISEL-NEXT: v_accvgpr_read_b32 v6, a134 -; GISEL-NEXT: v_accvgpr_read_b32 v7, a135 -; GISEL-NEXT: v_accvgpr_read_b32 v8, a136 -; GISEL-NEXT: v_accvgpr_read_b32 v9, a137 -; GISEL-NEXT: v_accvgpr_read_b32 v10, a138 -; GISEL-NEXT: v_accvgpr_read_b32 v11, a139 -; GISEL-NEXT: v_accvgpr_read_b32 v12, a140 -; GISEL-NEXT: v_accvgpr_read_b32 v13, a141 -; GISEL-NEXT: v_accvgpr_read_b32 v14, a142 -; GISEL-NEXT: v_accvgpr_read_b32 v15, a143 -; GISEL-NEXT: v_accvgpr_read_b32 v16, a144 -; GISEL-NEXT: v_accvgpr_read_b32 v17, a145 -; GISEL-NEXT: v_accvgpr_read_b32 v18, a146 -; GISEL-NEXT: v_accvgpr_read_b32 v19, a147 -; GISEL-NEXT: v_accvgpr_read_b32 v20, a148 -; GISEL-NEXT: v_accvgpr_read_b32 v21, a149 -; GISEL-NEXT: v_accvgpr_read_b32 v22, a150 -; GISEL-NEXT: v_accvgpr_read_b32 v23, a151 -; GISEL-NEXT: v_accvgpr_read_b32 v24, a152 -; GISEL-NEXT: v_accvgpr_read_b32 v25, a153 -; GISEL-NEXT: v_accvgpr_read_b32 v26, a154 -; GISEL-NEXT: v_accvgpr_read_b32 v27, a155 -; GISEL-NEXT: v_accvgpr_read_b32 v28, a156 -; GISEL-NEXT: v_accvgpr_read_b32 v29, a157 -; GISEL-NEXT: v_accvgpr_read_b32 v30, a158 -; GISEL-NEXT: v_accvgpr_read_b32 v31, a159 -; GISEL-NEXT: ds_write_b128 v33, v[0:3] -; GISEL-NEXT: ds_write_b128 v33, v[4:7] offset:16 -; GISEL-NEXT: ds_write_b128 v33, v[8:11] offset:32 -; GISEL-NEXT: ds_write_b128 v33, v[12:15] offset:48 -; GISEL-NEXT: ds_write_b128 v33, v[16:19] offset:64 -; GISEL-NEXT: ds_write_b128 v33, v[20:23] offset:80 -; GISEL-NEXT: ds_write_b128 v33, v[24:27] offset:96 -; GISEL-NEXT: ds_write_b128 v33, v[28:31] offset:112 -; GISEL-NEXT: v_accvgpr_read_b32 v0, a96 -; GISEL-NEXT: v_accvgpr_read_b32 v1, a97 -; GISEL-NEXT: v_accvgpr_read_b32 v2, a98 -; GISEL-NEXT: v_accvgpr_read_b32 v3, a99 -; GISEL-NEXT: v_accvgpr_read_b32 v4, a100 -; GISEL-NEXT: v_accvgpr_read_b32 v5, a101 -; GISEL-NEXT: v_accvgpr_read_b32 v6, a102 -; GISEL-NEXT: v_accvgpr_read_b32 v7, a103 -; GISEL-NEXT: v_accvgpr_read_b32 v8, a104 -; GISEL-NEXT: v_accvgpr_read_b32 v9, a105 -; GISEL-NEXT: v_accvgpr_read_b32 v10, a106 -; GISEL-NEXT: v_accvgpr_read_b32 v11, a107 -; GISEL-NEXT: v_accvgpr_read_b32 v12, a108 -; GISEL-NEXT: v_accvgpr_read_b32 v13, a109 -; GISEL-NEXT: v_accvgpr_read_b32 v14, a110 -; GISEL-NEXT: v_accvgpr_read_b32 v15, a111 -; GISEL-NEXT: v_accvgpr_read_b32 v16, a112 -; GISEL-NEXT: v_accvgpr_read_b32 v17, a113 -; GISEL-NEXT: v_accvgpr_read_b32 v18, a114 -; GISEL-NEXT: v_accvgpr_read_b32 v19, a115 -; GISEL-NEXT: v_accvgpr_read_b32 v20, a116 -; GISEL-NEXT: v_accvgpr_read_b32 v21, a117 -; GISEL-NEXT: v_accvgpr_read_b32 v22, a118 -; GISEL-NEXT: v_accvgpr_read_b32 v23, a119 -; GISEL-NEXT: v_accvgpr_read_b32 v24, a120 -; GISEL-NEXT: v_accvgpr_read_b32 v25, a121 -; GISEL-NEXT: v_accvgpr_read_b32 v26, a122 -; GISEL-NEXT: v_accvgpr_read_b32 v27, a123 -; GISEL-NEXT: v_accvgpr_read_b32 v28, a124 -; GISEL-NEXT: v_accvgpr_read_b32 v29, a125 -; GISEL-NEXT: v_accvgpr_read_b32 v30, a126 -; GISEL-NEXT: v_accvgpr_read_b32 v31, a127 +; GISEL-NEXT: s_nop 0 +; GISEL-NEXT: v_accvgpr_read_b32 v0, a32 +; GISEL-NEXT: v_accvgpr_read_b32 v1, a33 +; GISEL-NEXT: v_accvgpr_read_b32 v2, a34 +; GISEL-NEXT: v_accvgpr_read_b32 v3, a35 +; GISEL-NEXT: v_accvgpr_read_b32 v4, a36 +; GISEL-NEXT: v_accvgpr_read_b32 v5, a37 +; GISEL-NEXT: v_accvgpr_read_b32 v6, a38 +; GISEL-NEXT: v_accvgpr_read_b32 v7, a39 +; GISEL-NEXT: v_accvgpr_read_b32 v8, a40 +; GISEL-NEXT: v_accvgpr_read_b32 v9, a41 +; GISEL-NEXT: v_accvgpr_read_b32 v10, a42 +; GISEL-NEXT: v_accvgpr_read_b32 v11, a43 +; GISEL-NEXT: v_accvgpr_read_b32 v12, a44 +; GISEL-NEXT: v_accvgpr_read_b32 v13, a45 +; GISEL-NEXT: v_accvgpr_read_b32 v14, a46 +; GISEL-NEXT: v_accvgpr_read_b32 v15, a47 +; GISEL-NEXT: v_accvgpr_read_b32 v16, a48 +; GISEL-NEXT: v_accvgpr_read_b32 v17, a49 +; GISEL-NEXT: v_accvgpr_read_b32 v18, a50 +; GISEL-NEXT: v_accvgpr_read_b32 v19, a51 +; GISEL-NEXT: v_accvgpr_read_b32 v20, a52 +; GISEL-NEXT: v_accvgpr_read_b32 v21, a53 +; GISEL-NEXT: v_accvgpr_read_b32 v22, a54 +; GISEL-NEXT: v_accvgpr_read_b32 v23, a55 +; GISEL-NEXT: v_accvgpr_read_b32 v24, a56 +; GISEL-NEXT: v_accvgpr_read_b32 v25, a57 +; GISEL-NEXT: v_accvgpr_read_b32 v26, a58 +; GISEL-NEXT: v_accvgpr_read_b32 v27, a59 +; GISEL-NEXT: v_accvgpr_read_b32 v28, a60 +; GISEL-NEXT: v_accvgpr_read_b32 v29, a61 +; GISEL-NEXT: v_accvgpr_read_b32 v30, a62 +; GISEL-NEXT: v_accvgpr_read_b32 v31, a63 +; GISEL-NEXT: ds_read_b128 a[32:35], v70 offset:24576 +; GISEL-NEXT: ds_read_b128 a[36:39], v70 offset:24592 +; GISEL-NEXT: ds_read_b128 a[40:43], v70 offset:24608 +; GISEL-NEXT: ds_read_b128 a[44:47], v70 offset:24624 +; GISEL-NEXT: ds_read_b128 a[48:51], v70 offset:24640 +; GISEL-NEXT: ds_read_b128 a[52:55], v70 offset:24656 +; GISEL-NEXT: ds_read_b128 a[56:59], v70 offset:24672 +; GISEL-NEXT: ds_read_b128 a[60:63], v70 offset:24688 +; GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v68, v69, a[32:63] ; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:8192 ; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:8208 ; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:8224 @@ -275,6 +273,48 @@ define amdgpu_kernel void @test_iglp_opt_mfma_gemm(ptr addrspace(3) noalias %in, ; GISEL-NEXT: v_accvgpr_read_b32 v8, a8 ; GISEL-NEXT: v_accvgpr_read_b32 v9, a9 ; GISEL-NEXT: v_accvgpr_read_b32 v10, a10 +; GISEL-NEXT: v_accvgpr_read_b32 v67, a63 +; GISEL-NEXT: v_accvgpr_read_b32 v66, a62 +; GISEL-NEXT: v_accvgpr_read_b32 v65, a61 +; GISEL-NEXT: v_accvgpr_read_b32 v64, a60 +; GISEL-NEXT: v_accvgpr_read_b32 v63, a59 +; GISEL-NEXT: v_accvgpr_read_b32 v62, a58 +; GISEL-NEXT: v_accvgpr_read_b32 v61, a57 +; GISEL-NEXT: v_accvgpr_read_b32 v60, a56 +; GISEL-NEXT: v_accvgpr_read_b32 v59, a55 +; GISEL-NEXT: v_accvgpr_read_b32 v58, a54 +; GISEL-NEXT: v_accvgpr_read_b32 v57, a53 +; GISEL-NEXT: v_accvgpr_read_b32 v56, a52 +; GISEL-NEXT: v_accvgpr_read_b32 v55, a51 +; GISEL-NEXT: v_accvgpr_read_b32 v54, a50 +; GISEL-NEXT: v_accvgpr_read_b32 v53, a49 +; GISEL-NEXT: v_accvgpr_read_b32 v52, a48 +; GISEL-NEXT: v_accvgpr_read_b32 v51, a47 +; GISEL-NEXT: v_accvgpr_read_b32 v50, a46 +; GISEL-NEXT: v_accvgpr_read_b32 v49, a45 +; GISEL-NEXT: v_accvgpr_read_b32 v48, a44 +; GISEL-NEXT: v_accvgpr_read_b32 v47, a43 +; GISEL-NEXT: v_accvgpr_read_b32 v46, a42 +; GISEL-NEXT: v_accvgpr_read_b32 v45, a41 +; GISEL-NEXT: v_accvgpr_read_b32 v44, a40 +; GISEL-NEXT: v_accvgpr_read_b32 v43, a39 +; GISEL-NEXT: v_accvgpr_read_b32 v42, a38 +; GISEL-NEXT: v_accvgpr_read_b32 v41, a37 +; GISEL-NEXT: v_accvgpr_read_b32 v40, a36 +; GISEL-NEXT: v_accvgpr_read_b32 v39, a35 +; GISEL-NEXT: v_accvgpr_read_b32 v38, a34 +; GISEL-NEXT: v_accvgpr_read_b32 v37, a33 +; GISEL-NEXT: v_accvgpr_read_b32 v36, a32 +; GISEL-NEXT: ds_read_b128 a[32:35], v70 offset:49152 +; GISEL-NEXT: ds_read_b128 a[36:39], v70 offset:49168 +; GISEL-NEXT: ds_read_b128 a[40:43], v70 offset:49184 +; GISEL-NEXT: ds_read_b128 a[44:47], v70 offset:49200 +; GISEL-NEXT: ds_read_b128 a[48:51], v70 offset:49216 +; GISEL-NEXT: ds_read_b128 a[52:55], v70 offset:49232 +; GISEL-NEXT: ds_read_b128 a[56:59], v70 offset:49248 +; GISEL-NEXT: ds_read_b128 a[60:63], v70 offset:49264 +; GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v68, v69, a[32:63] ; GISEL-NEXT: v_accvgpr_read_b32 v11, a11 ; GISEL-NEXT: v_accvgpr_read_b32 v12, a12 ; GISEL-NEXT: v_accvgpr_read_b32 v13, a13 @@ -294,88 +334,56 @@ define amdgpu_kernel void @test_iglp_opt_mfma_gemm(ptr addrspace(3) noalias %in, ; GISEL-NEXT: v_accvgpr_read_b32 v27, a27 ; GISEL-NEXT: v_accvgpr_read_b32 v28, a28 ; GISEL-NEXT: v_accvgpr_read_b32 v29, a29 +; GISEL-NEXT: v_accvgpr_read_b32 v99, a63 +; GISEL-NEXT: v_accvgpr_read_b32 v98, a62 +; GISEL-NEXT: v_accvgpr_read_b32 v97, a61 +; GISEL-NEXT: v_accvgpr_read_b32 v96, a60 +; GISEL-NEXT: v_accvgpr_read_b32 v95, a59 +; GISEL-NEXT: v_accvgpr_read_b32 v94, a58 +; GISEL-NEXT: v_accvgpr_read_b32 v93, a57 +; GISEL-NEXT: v_accvgpr_read_b32 v92, a56 +; GISEL-NEXT: v_accvgpr_read_b32 v91, a55 +; GISEL-NEXT: v_accvgpr_read_b32 v90, a54 +; GISEL-NEXT: v_accvgpr_read_b32 v89, a53 +; GISEL-NEXT: v_accvgpr_read_b32 v88, a52 +; GISEL-NEXT: v_accvgpr_read_b32 v87, a51 +; GISEL-NEXT: v_accvgpr_read_b32 v86, a50 +; GISEL-NEXT: v_accvgpr_read_b32 v85, a49 +; GISEL-NEXT: v_accvgpr_read_b32 v84, a48 +; GISEL-NEXT: v_accvgpr_read_b32 v83, a47 +; GISEL-NEXT: v_accvgpr_read_b32 v82, a46 +; GISEL-NEXT: v_accvgpr_read_b32 v81, a45 +; GISEL-NEXT: v_accvgpr_read_b32 v80, a44 +; GISEL-NEXT: v_accvgpr_read_b32 v79, a43 +; GISEL-NEXT: v_accvgpr_read_b32 v78, a42 +; GISEL-NEXT: v_accvgpr_read_b32 v77, a41 +; GISEL-NEXT: v_accvgpr_read_b32 v76, a40 +; GISEL-NEXT: v_accvgpr_read_b32 v75, a39 +; GISEL-NEXT: v_accvgpr_read_b32 v74, a38 +; GISEL-NEXT: v_accvgpr_read_b32 v73, a37 +; GISEL-NEXT: v_accvgpr_read_b32 v72, a36 +; GISEL-NEXT: v_accvgpr_read_b32 v71, a35 +; GISEL-NEXT: v_accvgpr_read_b32 v70, a34 +; GISEL-NEXT: v_accvgpr_read_b32 v69, a33 +; GISEL-NEXT: v_accvgpr_read_b32 v68, a32 ; GISEL-NEXT: v_accvgpr_read_b32 v30, a30 ; GISEL-NEXT: v_accvgpr_read_b32 v31, a31 -; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:16384 -; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:16400 -; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:16416 -; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:16432 -; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:16448 -; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:16464 -; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:16480 -; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:16496 -; GISEL-NEXT: v_accvgpr_read_b32 v0, a32 -; GISEL-NEXT: v_accvgpr_read_b32 v1, a33 -; GISEL-NEXT: v_accvgpr_read_b32 v2, a34 -; GISEL-NEXT: v_accvgpr_read_b32 v3, a35 -; GISEL-NEXT: v_accvgpr_read_b32 v4, a36 -; GISEL-NEXT: v_accvgpr_read_b32 v5, a37 -; GISEL-NEXT: v_accvgpr_read_b32 v6, a38 -; GISEL-NEXT: v_accvgpr_read_b32 v7, a39 -; GISEL-NEXT: v_accvgpr_read_b32 v8, a40 -; GISEL-NEXT: v_accvgpr_read_b32 v9, a41 -; GISEL-NEXT: v_accvgpr_read_b32 v10, a42 -; GISEL-NEXT: v_accvgpr_read_b32 v11, a43 -; GISEL-NEXT: v_accvgpr_read_b32 v12, a44 -; GISEL-NEXT: v_accvgpr_read_b32 v13, a45 -; GISEL-NEXT: v_accvgpr_read_b32 v14, a46 -; GISEL-NEXT: v_accvgpr_read_b32 v15, a47 -; GISEL-NEXT: v_accvgpr_read_b32 v16, a48 -; GISEL-NEXT: v_accvgpr_read_b32 v17, a49 -; GISEL-NEXT: v_accvgpr_read_b32 v18, a50 -; GISEL-NEXT: v_accvgpr_read_b32 v19, a51 -; GISEL-NEXT: v_accvgpr_read_b32 v20, a52 -; GISEL-NEXT: v_accvgpr_read_b32 v21, a53 -; GISEL-NEXT: v_accvgpr_read_b32 v22, a54 -; GISEL-NEXT: v_accvgpr_read_b32 v23, a55 -; GISEL-NEXT: v_accvgpr_read_b32 v24, a56 -; GISEL-NEXT: v_accvgpr_read_b32 v25, a57 -; GISEL-NEXT: v_accvgpr_read_b32 v26, a58 -; GISEL-NEXT: v_accvgpr_read_b32 v27, a59 -; GISEL-NEXT: v_accvgpr_read_b32 v28, a60 -; GISEL-NEXT: v_accvgpr_read_b32 v29, a61 -; GISEL-NEXT: v_accvgpr_read_b32 v30, a62 -; GISEL-NEXT: v_accvgpr_read_b32 v31, a63 -; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:24576 -; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:24592 -; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:24608 -; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:24624 -; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:24640 -; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:24656 -; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:24672 -; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:24688 -; GISEL-NEXT: v_accvgpr_read_b32 v0, a64 -; GISEL-NEXT: v_accvgpr_read_b32 v1, a65 -; GISEL-NEXT: v_accvgpr_read_b32 v2, a66 -; GISEL-NEXT: v_accvgpr_read_b32 v3, a67 -; GISEL-NEXT: v_accvgpr_read_b32 v4, a68 -; GISEL-NEXT: v_accvgpr_read_b32 v5, a69 -; GISEL-NEXT: v_accvgpr_read_b32 v6, a70 -; GISEL-NEXT: v_accvgpr_read_b32 v7, a71 -; GISEL-NEXT: v_accvgpr_read_b32 v8, a72 -; GISEL-NEXT: v_accvgpr_read_b32 v9, a73 -; GISEL-NEXT: v_accvgpr_read_b32 v10, a74 -; GISEL-NEXT: v_accvgpr_read_b32 v11, a75 -; GISEL-NEXT: v_accvgpr_read_b32 v12, a76 -; GISEL-NEXT: v_accvgpr_read_b32 v13, a77 -; GISEL-NEXT: v_accvgpr_read_b32 v14, a78 -; GISEL-NEXT: v_accvgpr_read_b32 v15, a79 -; GISEL-NEXT: v_accvgpr_read_b32 v16, a80 -; GISEL-NEXT: v_accvgpr_read_b32 v17, a81 -; GISEL-NEXT: v_accvgpr_read_b32 v18, a82 -; GISEL-NEXT: v_accvgpr_read_b32 v19, a83 -; GISEL-NEXT: v_accvgpr_read_b32 v20, a84 -; GISEL-NEXT: v_accvgpr_read_b32 v21, a85 -; GISEL-NEXT: v_accvgpr_read_b32 v22, a86 -; GISEL-NEXT: v_accvgpr_read_b32 v23, a87 -; GISEL-NEXT: v_accvgpr_read_b32 v24, a88 -; GISEL-NEXT: v_accvgpr_read_b32 v25, a89 -; GISEL-NEXT: v_accvgpr_read_b32 v26, a90 -; GISEL-NEXT: v_accvgpr_read_b32 v27, a91 -; GISEL-NEXT: v_accvgpr_read_b32 v28, a92 -; GISEL-NEXT: v_accvgpr_read_b32 v29, a93 -; GISEL-NEXT: v_accvgpr_read_b32 v30, a94 -; GISEL-NEXT: v_accvgpr_read_b32 v31, a95 +; GISEL-NEXT: ds_write_b128 v32, v[36:39] offset:16384 +; GISEL-NEXT: ds_write_b128 v32, v[40:43] offset:16400 +; GISEL-NEXT: ds_write_b128 v32, v[44:47] offset:16416 +; GISEL-NEXT: ds_write_b128 v32, v[48:51] offset:16432 +; GISEL-NEXT: ds_write_b128 v32, v[52:55] offset:16448 +; GISEL-NEXT: ds_write_b128 v32, v[56:59] offset:16464 +; GISEL-NEXT: ds_write_b128 v32, v[60:63] offset:16480 +; GISEL-NEXT: ds_write_b128 v32, v[64:67] offset:16496 +; GISEL-NEXT: ds_write_b128 v32, v[68:71] offset:24576 +; GISEL-NEXT: ds_write_b128 v32, v[72:75] offset:24592 +; GISEL-NEXT: ds_write_b128 v32, v[76:79] offset:24608 +; GISEL-NEXT: ds_write_b128 v32, v[80:83] offset:24624 +; GISEL-NEXT: ds_write_b128 v32, v[84:87] offset:24640 +; GISEL-NEXT: ds_write_b128 v32, v[88:91] offset:24656 +; GISEL-NEXT: ds_write_b128 v32, v[92:95] offset:24672 +; GISEL-NEXT: ds_write_b128 v32, v[96:99] offset:24688 ; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:32768 ; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:32784 ; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:32800 @@ -527,42 +535,28 @@ define amdgpu_kernel void @test_iglp_opt_rev_mfma_gemm(ptr addrspace(3) noalias ; GISEL: ; %bb.0: ; %entry ; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 ; GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0 -; GISEL-NEXT: v_lshlrev_b32_e32 v32, 7, v0 -; GISEL-NEXT: v_mov_b32_e32 v33, 1.0 -; GISEL-NEXT: v_mov_b32_e32 v34, 2.0 +; GISEL-NEXT: v_lshlrev_b32_e32 v96, 7, v0 +; GISEL-NEXT: v_mov_b32_e32 v97, 1.0 +; GISEL-NEXT: v_mov_b32_e32 v98, 2.0 ; GISEL-NEXT: s_waitcnt lgkmcnt(0) -; GISEL-NEXT: v_add_u32_e32 v0, s0, v32 -; GISEL-NEXT: ds_read_b128 a[0:3], v0 -; GISEL-NEXT: ds_read_b128 a[4:7], v0 offset:16 -; GISEL-NEXT: ds_read_b128 a[8:11], v0 offset:32 -; GISEL-NEXT: ds_read_b128 a[12:15], v0 offset:48 -; GISEL-NEXT: ds_read_b128 a[16:19], v0 offset:64 -; GISEL-NEXT: ds_read_b128 a[20:23], v0 offset:80 -; GISEL-NEXT: ds_read_b128 a[24:27], v0 offset:96 -; GISEL-NEXT: ds_read_b128 a[28:31], v0 offset:112 +; GISEL-NEXT: v_add_u32_e32 v64, s0, v96 +; GISEL-NEXT: ds_read_b128 a[0:3], v64 +; GISEL-NEXT: ds_read_b128 a[4:7], v64 offset:16 +; GISEL-NEXT: ds_read_b128 a[8:11], v64 offset:32 +; GISEL-NEXT: ds_read_b128 a[12:15], v64 offset:48 +; GISEL-NEXT: ds_read_b128 a[16:19], v64 offset:64 +; GISEL-NEXT: ds_read_b128 a[20:23], v64 offset:80 +; GISEL-NEXT: ds_read_b128 a[24:27], v64 offset:96 +; GISEL-NEXT: ds_read_b128 a[28:31], v64 offset:112 ; GISEL-NEXT: s_waitcnt lgkmcnt(0) -; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v33, v34, a[0:31] -; GISEL-NEXT: ds_read_b128 a[32:35], v0 offset:8192 -; GISEL-NEXT: ds_read_b128 a[36:39], v0 offset:8208 -; GISEL-NEXT: ds_read_b128 a[40:43], v0 offset:8224 -; GISEL-NEXT: ds_read_b128 a[44:47], v0 offset:8240 -; GISEL-NEXT: ds_read_b128 a[48:51], v0 offset:8256 -; GISEL-NEXT: ds_read_b128 a[52:55], v0 offset:8272 -; GISEL-NEXT: ds_read_b128 a[56:59], v0 offset:8288 -; GISEL-NEXT: ds_read_b128 a[60:63], v0 offset:8304 -; GISEL-NEXT: v_add_u32_e32 v1, 0xc000, v0 -; GISEL-NEXT: v_add_u32_e32 v32, s1, v32 +; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v97, v98, a[0:31] +; GISEL-NEXT: v_add_u32_e32 v99, 0xc000, v64 +; GISEL-NEXT: v_add_u32_e32 v96, s1, v96 ; GISEL-NEXT: ; iglp_opt mask(0x00000001) -; GISEL-NEXT: s_waitcnt lgkmcnt(0) -; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v33, v34, a[32:63] -; GISEL-NEXT: ds_read_b128 a[64:67], v0 offset:24576 -; GISEL-NEXT: ds_read_b128 a[68:71], v0 offset:24592 -; GISEL-NEXT: ds_read_b128 a[72:75], v0 offset:24608 -; GISEL-NEXT: ds_read_b128 a[76:79], v0 offset:24624 -; GISEL-NEXT: ds_read_b128 a[80:83], v0 offset:24640 -; GISEL-NEXT: ds_read_b128 a[84:87], v0 offset:24656 -; GISEL-NEXT: ds_read_b128 a[88:91], v0 offset:24672 -; GISEL-NEXT: ds_read_b128 a[92:95], v0 offset:24688 +; GISEL-NEXT: s_nop 15 +; GISEL-NEXT: s_nop 0 +; GISEL-NEXT: v_accvgpr_read_b32 v0, a0 +; GISEL-NEXT: v_accvgpr_read_b32 v1, a1 ; GISEL-NEXT: v_accvgpr_read_b32 v2, a2 ; GISEL-NEXT: v_accvgpr_read_b32 v3, a3 ; GISEL-NEXT: v_accvgpr_read_b32 v4, a4 @@ -570,42 +564,18 @@ define amdgpu_kernel void @test_iglp_opt_rev_mfma_gemm(ptr addrspace(3) noalias ; GISEL-NEXT: v_accvgpr_read_b32 v6, a6 ; GISEL-NEXT: v_accvgpr_read_b32 v7, a7 ; GISEL-NEXT: v_accvgpr_read_b32 v8, a8 -; GISEL-NEXT: s_waitcnt lgkmcnt(0) -; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v33, v34, a[64:95] -; GISEL-NEXT: ds_read_b128 a[96:99], v0 offset:49152 -; GISEL-NEXT: ds_read_b128 a[100:103], v0 offset:49168 -; GISEL-NEXT: ds_read_b128 a[104:107], v0 offset:49184 -; GISEL-NEXT: ds_read_b128 a[108:111], v0 offset:49200 -; GISEL-NEXT: ds_read_b128 a[112:115], v0 offset:49216 -; GISEL-NEXT: ds_read_b128 a[116:119], v0 offset:49232 -; GISEL-NEXT: ds_read_b128 a[120:123], v0 offset:49248 -; GISEL-NEXT: ds_read_b128 a[124:127], v0 offset:49264 -; GISEL-NEXT: v_accvgpr_read_b32 v0, a0 ; GISEL-NEXT: v_accvgpr_read_b32 v9, a9 ; GISEL-NEXT: v_accvgpr_read_b32 v10, a10 ; GISEL-NEXT: v_accvgpr_read_b32 v11, a11 ; GISEL-NEXT: v_accvgpr_read_b32 v12, a12 ; GISEL-NEXT: v_accvgpr_read_b32 v13, a13 ; GISEL-NEXT: v_accvgpr_read_b32 v14, a14 -; GISEL-NEXT: s_waitcnt lgkmcnt(0) -; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v33, v34, a[96:127] -; GISEL-NEXT: ds_read_b128 a[128:131], v1 offset:32768 -; GISEL-NEXT: ds_read_b128 a[132:135], v1 offset:32784 -; GISEL-NEXT: ds_read_b128 a[136:139], v1 offset:32800 -; GISEL-NEXT: ds_read_b128 a[140:143], v1 offset:32816 -; GISEL-NEXT: ds_read_b128 a[144:147], v1 offset:32832 -; GISEL-NEXT: ds_read_b128 a[148:151], v1 offset:32848 -; GISEL-NEXT: ds_read_b128 a[152:155], v1 offset:32864 -; GISEL-NEXT: ds_read_b128 a[156:159], v1 offset:32880 -; GISEL-NEXT: v_accvgpr_read_b32 v1, a1 ; GISEL-NEXT: v_accvgpr_read_b32 v15, a15 ; GISEL-NEXT: v_accvgpr_read_b32 v16, a16 ; GISEL-NEXT: v_accvgpr_read_b32 v17, a17 ; GISEL-NEXT: v_accvgpr_read_b32 v18, a18 ; GISEL-NEXT: v_accvgpr_read_b32 v19, a19 ; GISEL-NEXT: v_accvgpr_read_b32 v20, a20 -; GISEL-NEXT: s_waitcnt lgkmcnt(0) -; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v33, v34, a[128:159] ; GISEL-NEXT: v_accvgpr_read_b32 v21, a21 ; GISEL-NEXT: v_accvgpr_read_b32 v22, a22 ; GISEL-NEXT: v_accvgpr_read_b32 v23, a23 @@ -617,14 +587,122 @@ define amdgpu_kernel void @test_iglp_opt_rev_mfma_gemm(ptr addrspace(3) noalias ; GISEL-NEXT: v_accvgpr_read_b32 v29, a29 ; GISEL-NEXT: v_accvgpr_read_b32 v30, a30 ; GISEL-NEXT: v_accvgpr_read_b32 v31, a31 -; GISEL-NEXT: ds_write_b128 v32, v[0:3] -; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:16 -; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:32 -; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:48 -; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:64 -; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:80 -; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:96 -; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:112 +; GISEL-NEXT: ds_read_b128 a[0:3], v64 offset:8192 +; GISEL-NEXT: ds_read_b128 a[4:7], v64 offset:8208 +; GISEL-NEXT: ds_read_b128 a[8:11], v64 offset:8224 +; GISEL-NEXT: ds_read_b128 a[12:15], v64 offset:8240 +; GISEL-NEXT: ds_read_b128 a[16:19], v64 offset:8256 +; GISEL-NEXT: ds_read_b128 a[20:23], v64 offset:8272 +; GISEL-NEXT: ds_read_b128 a[24:27], v64 offset:8288 +; GISEL-NEXT: ds_read_b128 a[28:31], v64 offset:8304 +; GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v97, v98, a[0:31] +; GISEL-NEXT: s_nop 15 +; GISEL-NEXT: s_nop 2 +; GISEL-NEXT: v_accvgpr_read_b32 v63, a31 +; GISEL-NEXT: v_accvgpr_read_b32 v62, a30 +; GISEL-NEXT: v_accvgpr_read_b32 v61, a29 +; GISEL-NEXT: v_accvgpr_read_b32 v60, a28 +; GISEL-NEXT: v_accvgpr_read_b32 v59, a27 +; GISEL-NEXT: v_accvgpr_read_b32 v58, a26 +; GISEL-NEXT: v_accvgpr_read_b32 v57, a25 +; GISEL-NEXT: v_accvgpr_read_b32 v56, a24 +; GISEL-NEXT: v_accvgpr_read_b32 v55, a23 +; GISEL-NEXT: v_accvgpr_read_b32 v54, a22 +; GISEL-NEXT: v_accvgpr_read_b32 v53, a21 +; GISEL-NEXT: v_accvgpr_read_b32 v52, a20 +; GISEL-NEXT: v_accvgpr_read_b32 v51, a19 +; GISEL-NEXT: v_accvgpr_read_b32 v50, a18 +; GISEL-NEXT: v_accvgpr_read_b32 v49, a17 +; GISEL-NEXT: v_accvgpr_read_b32 v48, a16 +; GISEL-NEXT: v_accvgpr_read_b32 v47, a15 +; GISEL-NEXT: v_accvgpr_read_b32 v46, a14 +; GISEL-NEXT: v_accvgpr_read_b32 v45, a13 +; GISEL-NEXT: v_accvgpr_read_b32 v44, a12 +; GISEL-NEXT: v_accvgpr_read_b32 v43, a11 +; GISEL-NEXT: v_accvgpr_read_b32 v42, a10 +; GISEL-NEXT: v_accvgpr_read_b32 v41, a9 +; GISEL-NEXT: v_accvgpr_read_b32 v40, a8 +; GISEL-NEXT: v_accvgpr_read_b32 v39, a7 +; GISEL-NEXT: v_accvgpr_read_b32 v38, a6 +; GISEL-NEXT: v_accvgpr_read_b32 v37, a5 +; GISEL-NEXT: v_accvgpr_read_b32 v36, a4 +; GISEL-NEXT: v_accvgpr_read_b32 v35, a3 +; GISEL-NEXT: v_accvgpr_read_b32 v34, a2 +; GISEL-NEXT: v_accvgpr_read_b32 v33, a1 +; GISEL-NEXT: v_accvgpr_read_b32 v32, a0 +; GISEL-NEXT: ds_read_b128 a[0:3], v64 offset:24576 +; GISEL-NEXT: ds_read_b128 a[4:7], v64 offset:24592 +; GISEL-NEXT: ds_read_b128 a[8:11], v64 offset:24608 +; GISEL-NEXT: ds_read_b128 a[12:15], v64 offset:24624 +; GISEL-NEXT: ds_read_b128 a[16:19], v64 offset:24640 +; GISEL-NEXT: ds_read_b128 a[20:23], v64 offset:24656 +; GISEL-NEXT: ds_read_b128 a[24:27], v64 offset:24672 +; GISEL-NEXT: ds_read_b128 a[28:31], v64 offset:24688 +; GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v97, v98, a[0:31] +; GISEL-NEXT: ds_read_b128 a[32:35], v64 offset:49152 +; GISEL-NEXT: ds_read_b128 a[36:39], v64 offset:49168 +; GISEL-NEXT: ds_read_b128 a[40:43], v64 offset:49184 +; GISEL-NEXT: ds_read_b128 a[44:47], v64 offset:49200 +; GISEL-NEXT: ds_read_b128 a[48:51], v64 offset:49216 +; GISEL-NEXT: ds_read_b128 a[52:55], v64 offset:49232 +; GISEL-NEXT: ds_read_b128 a[56:59], v64 offset:49248 +; GISEL-NEXT: ds_read_b128 a[60:63], v64 offset:49264 +; GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v97, v98, a[32:63] +; GISEL-NEXT: s_nop 8 +; GISEL-NEXT: v_accvgpr_read_b32 v95, a31 +; GISEL-NEXT: v_accvgpr_read_b32 v94, a30 +; GISEL-NEXT: v_accvgpr_read_b32 v93, a29 +; GISEL-NEXT: v_accvgpr_read_b32 v92, a28 +; GISEL-NEXT: v_accvgpr_read_b32 v91, a27 +; GISEL-NEXT: v_accvgpr_read_b32 v90, a26 +; GISEL-NEXT: v_accvgpr_read_b32 v89, a25 +; GISEL-NEXT: v_accvgpr_read_b32 v88, a24 +; GISEL-NEXT: v_accvgpr_read_b32 v87, a23 +; GISEL-NEXT: v_accvgpr_read_b32 v86, a22 +; GISEL-NEXT: v_accvgpr_read_b32 v85, a21 +; GISEL-NEXT: v_accvgpr_read_b32 v84, a20 +; GISEL-NEXT: v_accvgpr_read_b32 v83, a19 +; GISEL-NEXT: v_accvgpr_read_b32 v82, a18 +; GISEL-NEXT: v_accvgpr_read_b32 v81, a17 +; GISEL-NEXT: v_accvgpr_read_b32 v80, a16 +; GISEL-NEXT: v_accvgpr_read_b32 v79, a15 +; GISEL-NEXT: v_accvgpr_read_b32 v78, a14 +; GISEL-NEXT: v_accvgpr_read_b32 v77, a13 +; GISEL-NEXT: v_accvgpr_read_b32 v76, a12 +; GISEL-NEXT: v_accvgpr_read_b32 v75, a11 +; GISEL-NEXT: v_accvgpr_read_b32 v74, a10 +; GISEL-NEXT: v_accvgpr_read_b32 v73, a9 +; GISEL-NEXT: v_accvgpr_read_b32 v72, a8 +; GISEL-NEXT: v_accvgpr_read_b32 v71, a7 +; GISEL-NEXT: v_accvgpr_read_b32 v70, a6 +; GISEL-NEXT: v_accvgpr_read_b32 v69, a5 +; GISEL-NEXT: v_accvgpr_read_b32 v68, a4 +; GISEL-NEXT: v_accvgpr_read_b32 v67, a3 +; GISEL-NEXT: v_accvgpr_read_b32 v66, a2 +; GISEL-NEXT: v_accvgpr_read_b32 v65, a1 +; GISEL-NEXT: v_accvgpr_read_b32 v64, a0 +; GISEL-NEXT: ds_read_b128 a[0:3], v99 offset:32768 +; GISEL-NEXT: ds_read_b128 a[4:7], v99 offset:32784 +; GISEL-NEXT: ds_read_b128 a[8:11], v99 offset:32800 +; GISEL-NEXT: ds_read_b128 a[12:15], v99 offset:32816 +; GISEL-NEXT: ds_read_b128 a[16:19], v99 offset:32832 +; GISEL-NEXT: ds_read_b128 a[20:23], v99 offset:32848 +; GISEL-NEXT: ds_read_b128 a[24:27], v99 offset:32864 +; GISEL-NEXT: ds_read_b128 a[28:31], v99 offset:32880 +; GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v97, v98, a[0:31] +; GISEL-NEXT: ds_write_b128 v96, v[0:3] +; GISEL-NEXT: ds_write_b128 v96, v[4:7] offset:16 +; GISEL-NEXT: ds_write_b128 v96, v[8:11] offset:32 +; GISEL-NEXT: ds_write_b128 v96, v[12:15] offset:48 +; GISEL-NEXT: ds_write_b128 v96, v[16:19] offset:64 +; GISEL-NEXT: ds_write_b128 v96, v[20:23] offset:80 +; GISEL-NEXT: ds_write_b128 v96, v[24:27] offset:96 +; GISEL-NEXT: ds_write_b128 v96, v[28:31] offset:112 +; GISEL-NEXT: v_mov_b32_e32 v96, s1 ; GISEL-NEXT: v_accvgpr_read_b32 v0, a32 ; GISEL-NEXT: v_accvgpr_read_b32 v1, a33 ; GISEL-NEXT: v_accvgpr_read_b32 v2, a34 @@ -657,135 +735,70 @@ define amdgpu_kernel void @test_iglp_opt_rev_mfma_gemm(ptr addrspace(3) noalias ; GISEL-NEXT: v_accvgpr_read_b32 v29, a61 ; GISEL-NEXT: v_accvgpr_read_b32 v30, a62 ; GISEL-NEXT: v_accvgpr_read_b32 v31, a63 -; GISEL-NEXT: v_mov_b32_e32 v32, s1 -; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:8192 -; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:8208 -; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:8224 -; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:8240 -; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:8256 -; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:8272 -; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:8288 -; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:8304 -; GISEL-NEXT: v_accvgpr_read_b32 v0, a64 -; GISEL-NEXT: v_accvgpr_read_b32 v1, a65 -; GISEL-NEXT: v_accvgpr_read_b32 v2, a66 -; GISEL-NEXT: v_accvgpr_read_b32 v3, a67 -; GISEL-NEXT: v_accvgpr_read_b32 v4, a68 -; GISEL-NEXT: v_accvgpr_read_b32 v5, a69 -; GISEL-NEXT: v_accvgpr_read_b32 v6, a70 -; GISEL-NEXT: v_accvgpr_read_b32 v7, a71 -; GISEL-NEXT: v_accvgpr_read_b32 v8, a72 -; GISEL-NEXT: v_accvgpr_read_b32 v9, a73 -; GISEL-NEXT: v_accvgpr_read_b32 v10, a74 -; GISEL-NEXT: v_accvgpr_read_b32 v11, a75 -; GISEL-NEXT: v_accvgpr_read_b32 v12, a76 -; GISEL-NEXT: v_accvgpr_read_b32 v13, a77 -; GISEL-NEXT: v_accvgpr_read_b32 v14, a78 -; GISEL-NEXT: v_accvgpr_read_b32 v15, a79 -; GISEL-NEXT: v_accvgpr_read_b32 v16, a80 -; GISEL-NEXT: v_accvgpr_read_b32 v17, a81 -; GISEL-NEXT: v_accvgpr_read_b32 v18, a82 -; GISEL-NEXT: v_accvgpr_read_b32 v19, a83 -; GISEL-NEXT: v_accvgpr_read_b32 v20, a84 -; GISEL-NEXT: v_accvgpr_read_b32 v21, a85 -; GISEL-NEXT: v_accvgpr_read_b32 v22, a86 -; GISEL-NEXT: v_accvgpr_read_b32 v23, a87 -; GISEL-NEXT: v_accvgpr_read_b32 v24, a88 -; GISEL-NEXT: v_accvgpr_read_b32 v25, a89 -; GISEL-NEXT: v_accvgpr_read_b32 v26, a90 -; GISEL-NEXT: v_accvgpr_read_b32 v27, a91 -; GISEL-NEXT: v_accvgpr_read_b32 v28, a92 -; GISEL-NEXT: v_accvgpr_read_b32 v29, a93 -; GISEL-NEXT: v_accvgpr_read_b32 v30, a94 -; GISEL-NEXT: v_accvgpr_read_b32 v31, a95 -; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:16384 -; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:16400 -; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:16416 -; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:16432 -; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:16448 -; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:16464 -; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:16480 -; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:16496 -; GISEL-NEXT: v_accvgpr_read_b32 v0, a96 -; GISEL-NEXT: v_accvgpr_read_b32 v1, a97 -; GISEL-NEXT: v_accvgpr_read_b32 v2, a98 -; GISEL-NEXT: v_accvgpr_read_b32 v3, a99 -; GISEL-NEXT: v_accvgpr_read_b32 v4, a100 -; GISEL-NEXT: v_accvgpr_read_b32 v5, a101 -; GISEL-NEXT: v_accvgpr_read_b32 v6, a102 -; GISEL-NEXT: v_accvgpr_read_b32 v7, a103 -; GISEL-NEXT: v_accvgpr_read_b32 v8, a104 -; GISEL-NEXT: v_accvgpr_read_b32 v9, a105 -; GISEL-NEXT: v_accvgpr_read_b32 v10, a106 -; GISEL-NEXT: v_accvgpr_read_b32 v11, a107 -; GISEL-NEXT: v_accvgpr_read_b32 v12, a108 -; GISEL-NEXT: v_accvgpr_read_b32 v13, a109 -; GISEL-NEXT: v_accvgpr_read_b32 v14, a110 -; GISEL-NEXT: v_accvgpr_read_b32 v15, a111 -; GISEL-NEXT: v_accvgpr_read_b32 v16, a112 -; GISEL-NEXT: v_accvgpr_read_b32 v17, a113 -; GISEL-NEXT: v_accvgpr_read_b32 v18, a114 -; GISEL-NEXT: v_accvgpr_read_b32 v19, a115 -; GISEL-NEXT: v_accvgpr_read_b32 v20, a116 -; GISEL-NEXT: v_accvgpr_read_b32 v21, a117 -; GISEL-NEXT: v_accvgpr_read_b32 v22, a118 -; GISEL-NEXT: v_accvgpr_read_b32 v23, a119 -; GISEL-NEXT: v_accvgpr_read_b32 v24, a120 -; GISEL-NEXT: v_accvgpr_read_b32 v25, a121 -; GISEL-NEXT: v_accvgpr_read_b32 v26, a122 -; GISEL-NEXT: v_accvgpr_read_b32 v27, a123 -; GISEL-NEXT: v_accvgpr_read_b32 v28, a124 -; GISEL-NEXT: v_accvgpr_read_b32 v29, a125 -; GISEL-NEXT: v_accvgpr_read_b32 v30, a126 -; GISEL-NEXT: v_accvgpr_read_b32 v31, a127 -; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:24576 -; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:24592 -; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:24608 -; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:24624 -; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:24640 -; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:24656 -; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:24672 -; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:24688 -; GISEL-NEXT: v_accvgpr_read_b32 v0, a128 -; GISEL-NEXT: v_accvgpr_read_b32 v1, a129 -; GISEL-NEXT: v_accvgpr_read_b32 v2, a130 -; GISEL-NEXT: v_accvgpr_read_b32 v3, a131 -; GISEL-NEXT: v_accvgpr_read_b32 v4, a132 -; GISEL-NEXT: v_accvgpr_read_b32 v5, a133 -; GISEL-NEXT: v_accvgpr_read_b32 v6, a134 -; GISEL-NEXT: v_accvgpr_read_b32 v7, a135 -; GISEL-NEXT: v_accvgpr_read_b32 v8, a136 -; GISEL-NEXT: v_accvgpr_read_b32 v9, a137 -; GISEL-NEXT: v_accvgpr_read_b32 v10, a138 -; GISEL-NEXT: v_accvgpr_read_b32 v11, a139 -; GISEL-NEXT: v_accvgpr_read_b32 v12, a140 -; GISEL-NEXT: v_accvgpr_read_b32 v13, a141 -; GISEL-NEXT: v_accvgpr_read_b32 v14, a142 -; GISEL-NEXT: v_accvgpr_read_b32 v15, a143 -; GISEL-NEXT: v_accvgpr_read_b32 v16, a144 -; GISEL-NEXT: v_accvgpr_read_b32 v17, a145 -; GISEL-NEXT: v_accvgpr_read_b32 v18, a146 -; GISEL-NEXT: v_accvgpr_read_b32 v19, a147 -; GISEL-NEXT: v_accvgpr_read_b32 v20, a148 -; GISEL-NEXT: v_accvgpr_read_b32 v21, a149 -; GISEL-NEXT: v_accvgpr_read_b32 v22, a150 -; GISEL-NEXT: v_accvgpr_read_b32 v23, a151 -; GISEL-NEXT: v_accvgpr_read_b32 v24, a152 -; GISEL-NEXT: v_accvgpr_read_b32 v25, a153 -; GISEL-NEXT: v_accvgpr_read_b32 v26, a154 -; GISEL-NEXT: v_accvgpr_read_b32 v27, a155 -; GISEL-NEXT: v_accvgpr_read_b32 v28, a156 -; GISEL-NEXT: v_accvgpr_read_b32 v29, a157 -; GISEL-NEXT: v_accvgpr_read_b32 v30, a158 -; GISEL-NEXT: v_accvgpr_read_b32 v31, a159 -; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:32768 -; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:32784 -; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:32800 -; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:32816 -; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:32832 -; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:32848 -; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:32864 -; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:32880 +; GISEL-NEXT: ds_write_b128 v96, v[32:35] offset:8192 +; GISEL-NEXT: ds_write_b128 v96, v[36:39] offset:8208 +; GISEL-NEXT: ds_write_b128 v96, v[40:43] offset:8224 +; GISEL-NEXT: ds_write_b128 v96, v[44:47] offset:8240 +; GISEL-NEXT: ds_write_b128 v96, v[48:51] offset:8256 +; GISEL-NEXT: ds_write_b128 v96, v[52:55] offset:8272 +; GISEL-NEXT: ds_write_b128 v96, v[56:59] offset:8288 +; GISEL-NEXT: ds_write_b128 v96, v[60:63] offset:8304 +; GISEL-NEXT: ds_write_b128 v96, v[64:67] offset:16384 +; GISEL-NEXT: ds_write_b128 v96, v[68:71] offset:16400 +; GISEL-NEXT: ds_write_b128 v96, v[72:75] offset:16416 +; GISEL-NEXT: ds_write_b128 v96, v[76:79] offset:16432 +; GISEL-NEXT: ds_write_b128 v96, v[80:83] offset:16448 +; GISEL-NEXT: ds_write_b128 v96, v[84:87] offset:16464 +; GISEL-NEXT: ds_write_b128 v96, v[88:91] offset:16480 +; GISEL-NEXT: ds_write_b128 v96, v[92:95] offset:16496 +; GISEL-NEXT: ds_write_b128 v96, v[0:3] offset:24576 +; GISEL-NEXT: ds_write_b128 v96, v[4:7] offset:24592 +; GISEL-NEXT: ds_write_b128 v96, v[8:11] offset:24608 +; GISEL-NEXT: ds_write_b128 v96, v[12:15] offset:24624 +; GISEL-NEXT: ds_write_b128 v96, v[16:19] offset:24640 +; GISEL-NEXT: ds_write_b128 v96, v[20:23] offset:24656 +; GISEL-NEXT: ds_write_b128 v96, v[24:27] offset:24672 +; GISEL-NEXT: ds_write_b128 v96, v[28:31] offset:24688 +; GISEL-NEXT: v_accvgpr_read_b32 v0, a0 +; GISEL-NEXT: v_accvgpr_read_b32 v1, a1 +; GISEL-NEXT: v_accvgpr_read_b32 v2, a2 +; GISEL-NEXT: v_accvgpr_read_b32 v3, a3 +; GISEL-NEXT: v_accvgpr_read_b32 v4, a4 +; GISEL-NEXT: v_accvgpr_read_b32 v5, a5 +; GISEL-NEXT: v_accvgpr_read_b32 v6, a6 +; GISEL-NEXT: v_accvgpr_read_b32 v7, a7 +; GISEL-NEXT: v_accvgpr_read_b32 v8, a8 +; GISEL-NEXT: v_accvgpr_read_b32 v9, a9 +; GISEL-NEXT: v_accvgpr_read_b32 v10, a10 +; GISEL-NEXT: v_accvgpr_read_b32 v11, a11 +; GISEL-NEXT: v_accvgpr_read_b32 v12, a12 +; GISEL-NEXT: v_accvgpr_read_b32 v13, a13 +; GISEL-NEXT: v_accvgpr_read_b32 v14, a14 +; GISEL-NEXT: v_accvgpr_read_b32 v15, a15 +; GISEL-NEXT: v_accvgpr_read_b32 v16, a16 +; GISEL-NEXT: v_accvgpr_read_b32 v17, a17 +; GISEL-NEXT: v_accvgpr_read_b32 v18, a18 +; GISEL-NEXT: v_accvgpr_read_b32 v19, a19 +; GISEL-NEXT: v_accvgpr_read_b32 v20, a20 +; GISEL-NEXT: v_accvgpr_read_b32 v21, a21 +; GISEL-NEXT: v_accvgpr_read_b32 v22, a22 +; GISEL-NEXT: v_accvgpr_read_b32 v23, a23 +; GISEL-NEXT: v_accvgpr_read_b32 v24, a24 +; GISEL-NEXT: v_accvgpr_read_b32 v25, a25 +; GISEL-NEXT: v_accvgpr_read_b32 v26, a26 +; GISEL-NEXT: v_accvgpr_read_b32 v27, a27 +; GISEL-NEXT: v_accvgpr_read_b32 v28, a28 +; GISEL-NEXT: v_accvgpr_read_b32 v29, a29 +; GISEL-NEXT: v_accvgpr_read_b32 v30, a30 +; GISEL-NEXT: v_accvgpr_read_b32 v31, a31 +; GISEL-NEXT: ds_write_b128 v96, v[0:3] offset:32768 +; GISEL-NEXT: ds_write_b128 v96, v[4:7] offset:32784 +; GISEL-NEXT: ds_write_b128 v96, v[8:11] offset:32800 +; GISEL-NEXT: ds_write_b128 v96, v[12:15] offset:32816 +; GISEL-NEXT: ds_write_b128 v96, v[16:19] offset:32832 +; GISEL-NEXT: ds_write_b128 v96, v[20:23] offset:32848 +; GISEL-NEXT: ds_write_b128 v96, v[24:27] offset:32864 +; GISEL-NEXT: ds_write_b128 v96, v[28:31] offset:32880 ; GISEL-NEXT: s_endpgm entry: call void @llvm.amdgcn.iglp.opt(i32 1) diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.bf16.ll index 279de9b7899e0..c296e5615d4ae 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.bf16.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.bf16.ll @@ -17,7 +17,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x2bf16(ptr addrspace(1) %arg) #0 { ; GFX908-SDAG-LABEL: test_mfma_f32_32x32x2bf16: ; GFX908-SDAG: ; %bb.0: ; %bb ; GFX908-SDAG-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24 -; GFX908-SDAG-NEXT: v_mov_b32_e32 v4, 0 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v29, 0 ; GFX908-SDAG-NEXT: s_waitcnt lgkmcnt(0) ; GFX908-SDAG-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0 ; GFX908-SDAG-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40 @@ -27,13 +27,11 @@ define amdgpu_kernel void @test_mfma_f32_32x32x2bf16(ptr addrspace(1) %arg) #0 { ; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s18 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a0, v0 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a1, v1 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, s22 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s23 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a2, v2 -; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, s21 -; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s22 -; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s23 -; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a5, v0 -; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a6, v1 -; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a7, v2 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a6, v0 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a7, v1 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, s24 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s25 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s26 @@ -74,7 +72,8 @@ define amdgpu_kernel void @test_mfma_f32_32x32x2bf16(ptr addrspace(1) %arg) #0 { ; GFX908-SDAG-NEXT: v_mov_b32_e32 v0, s10 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s11 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s12 -; GFX908-SDAG-NEXT: v_mov_b32_e32 v5, s20 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v4, s20 +; GFX908-SDAG-NEXT: v_mov_b32_e32 v5, s21 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a3, v3 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a26, v0 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a27, v1 @@ -83,7 +82,8 @@ define amdgpu_kernel void @test_mfma_f32_32x32x2bf16(ptr addrspace(1) %arg) #0 { ; GFX908-SDAG-NEXT: v_mov_b32_e32 v1, s14 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v2, s15 ; GFX908-SDAG-NEXT: v_mov_b32_e32 v3, 1 -; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a4, v5 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a4, v4 +; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a5, v5 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a29, v0 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a30, v1 ; GFX908-SDAG-NEXT: v_accvgpr_write_b32 a31, v2 @@ -96,57 +96,42 @@ define amdgpu_kernel void @test_mfma_f32_32x32x2bf16(ptr addrspace(1) %arg) #0 { ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a26 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a25 ; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a24 -; GFX908-SDAG-NEXT: s_nop 1 -; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:96 -; GFX908-SDAG-NEXT: s_nop 0 -; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a31 -; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a30 -; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a29 -; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a28 -; GFX908-SDAG-NEXT: s_nop 1 -; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:112 -; GFX908-SDAG-NEXT: s_nop 0 -; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a19 -; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a18 -; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a17 -; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a16 -; GFX908-SDAG-NEXT: s_nop 1 -; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:64 -; GFX908-SDAG-NEXT: s_nop 0 -; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a23 -; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a22 -; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a21 -; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a20 -; GFX908-SDAG-NEXT: s_nop 1 -; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:80 -; GFX908-SDAG-NEXT: s_nop 0 -; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a11 -; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a10 -; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a9 -; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a8 -; GFX908-SDAG-NEXT: s_nop 1 -; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:32 -; GFX908-SDAG-NEXT: s_nop 0 -; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a15 -; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a14 -; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a13 -; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a12 -; GFX908-SDAG-NEXT: s_nop 1 -; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:48 -; GFX908-SDAG-NEXT: s_nop 0 -; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a3 -; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a2 -; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a1 -; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a0 -; GFX908-SDAG-NEXT: s_nop 1 -; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] -; GFX908-SDAG-NEXT: s_nop 0 -; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a7 -; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a6 -; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a5 -; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v0, a4 -; GFX908-SDAG-NEXT: s_nop 1 -; GFX908-SDAG-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:16 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v4, a31 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v8, a19 +; GFX908-SDAG-NEXT: global_store_dwordx4 v29, v[0:3], s[34:35] offset:96 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v12, a23 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v3, a30 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v2, a29 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v1, a28 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v16, a11 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v20, a15 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v24, a3 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v28, a7 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v7, a18 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v6, a17 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v5, a16 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v11, a22 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v10, a21 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v9, a20 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v15, a10 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v14, a9 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v13, a8 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v19, a14 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v18, a13 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v17, a12 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v23, a2 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v22, a1 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v21, a0 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v27, a6 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v26, a5 +; GFX908-SDAG-NEXT: v_accvgpr_read_b32 v25, a4 +; GFX908-SDAG-NEXT: global_store_dwordx4 v29, v[1:4], s[34:35] offset:112 +; GFX908-SDAG-NEXT: global_store_dwordx4 v29, v[5:8], s[34:35] offset:64 +; GFX908-SDAG-NEXT: global_store_dwordx4 v29, v[9:12], s[34:35] offset:80 +; GFX908-SDAG-NEXT: global_store_dwordx4 v29, v[13:16], s[34:35] offset:32 +; GFX908-SDAG-NEXT: global_store_dwordx4 v29, v[17:20], s[34:35] offset:48 +; GFX908-SDAG-NEXT: global_store_dwordx4 v29, v[21:24], s[34:35] +; GFX908-SDAG-NEXT: global_store_dwordx4 v29, v[25:28], s[34:35] offset:16 ; GFX908-SDAG-NEXT: s_endpgm ; ; GFX908-GISEL-LABEL: test_mfma_f32_32x32x2bf16: diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll index 086fc6f0ed4d7..4c13ebc3d590e 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll @@ -29,7 +29,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32(ptr addrspace(1) %arg) #0 { ; NOLIT-SRCC-LABEL: test_mfma_f32_32x32x1f32: ; NOLIT-SRCC: ; %bb.0: ; %bb ; NOLIT-SRCC-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24 -; NOLIT-SRCC-NEXT: v_mov_b32_e32 v4, 0 +; NOLIT-SRCC-NEXT: v_mov_b32_e32 v29, 0 ; NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0) ; NOLIT-SRCC-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0 ; NOLIT-SRCC-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40 @@ -39,13 +39,11 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32(ptr addrspace(1) %arg) #0 { ; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s18 ; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a0, v0 ; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a1, v1 +; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s22 +; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s23 ; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a2, v2 -; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s21 -; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s22 -; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s23 -; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v0 -; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a6, v1 -; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a7, v2 +; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a6, v0 +; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a7, v1 ; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s24 ; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s25 ; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s26 @@ -86,7 +84,8 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32(ptr addrspace(1) %arg) #0 { ; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s10 ; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s11 ; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s12 -; NOLIT-SRCC-NEXT: v_mov_b32_e32 v5, s20 +; NOLIT-SRCC-NEXT: v_mov_b32_e32 v4, s20 +; NOLIT-SRCC-NEXT: v_mov_b32_e32 v5, s21 ; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a3, v3 ; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a26, v0 ; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a27, v1 @@ -95,7 +94,8 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32(ptr addrspace(1) %arg) #0 { ; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s14 ; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s15 ; NOLIT-SRCC-NEXT: v_mov_b32_e32 v3, 1.0 -; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v5 +; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v4 +; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v5 ; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a29, v0 ; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a30, v1 ; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a31, v2 @@ -108,57 +108,42 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32(ptr addrspace(1) %arg) #0 { ; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26 ; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25 ; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a24 -; NOLIT-SRCC-NEXT: s_nop 1 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:96 -; NOLIT-SRCC-NEXT: s_nop 0 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a31 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a30 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a29 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a28 -; NOLIT-SRCC-NEXT: s_nop 1 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:112 -; NOLIT-SRCC-NEXT: s_nop 0 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a19 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a18 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a17 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a16 -; NOLIT-SRCC-NEXT: s_nop 1 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:64 -; NOLIT-SRCC-NEXT: s_nop 0 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a23 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a22 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a21 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a20 -; NOLIT-SRCC-NEXT: s_nop 1 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:80 -; NOLIT-SRCC-NEXT: s_nop 0 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a8 -; NOLIT-SRCC-NEXT: s_nop 1 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:32 -; NOLIT-SRCC-NEXT: s_nop 0 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12 -; NOLIT-SRCC-NEXT: s_nop 1 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:48 -; NOLIT-SRCC-NEXT: s_nop 0 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0 -; NOLIT-SRCC-NEXT: s_nop 1 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] -; NOLIT-SRCC-NEXT: s_nop 0 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4 -; NOLIT-SRCC-NEXT: s_nop 1 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:16 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a31 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a19 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[0:3], s[34:35] offset:96 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a23 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a30 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a29 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a28 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v16, a11 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v20, a15 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a3 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v28, a7 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a18 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a17 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a16 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a22 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a21 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a20 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a10 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a9 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a8 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v19, a14 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v18, a13 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v17, a12 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v23, a2 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v22, a1 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v21, a0 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v27, a6 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v26, a5 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v25, a4 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[1:4], s[34:35] offset:112 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[5:8], s[34:35] offset:64 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[9:12], s[34:35] offset:80 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[13:16], s[34:35] offset:32 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[17:20], s[34:35] offset:48 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[21:24], s[34:35] +; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[25:28], s[34:35] offset:16 ; NOLIT-SRCC-NEXT: s_endpgm ; ; GFX908-GISEL-LABEL: test_mfma_f32_32x32x1f32: @@ -284,7 +269,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32(ptr addrspace(1) %arg) #0 { ; LIT-SRCC-LABEL: test_mfma_f32_32x32x1f32: ; LIT-SRCC: ; %bb.0: ; %bb ; LIT-SRCC-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24 -; LIT-SRCC-NEXT: v_mov_b32_e32 v4, 0 +; LIT-SRCC-NEXT: v_mov_b32_e32 v29, 0 ; LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0) ; LIT-SRCC-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0 ; LIT-SRCC-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40 @@ -294,13 +279,11 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32(ptr addrspace(1) %arg) #0 { ; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s18 ; LIT-SRCC-NEXT: v_accvgpr_write_b32 a0, v0 ; LIT-SRCC-NEXT: v_accvgpr_write_b32 a1, v1 +; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s22 +; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s23 ; LIT-SRCC-NEXT: v_accvgpr_write_b32 a2, v2 -; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s21 -; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s22 -; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s23 -; LIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v0 -; LIT-SRCC-NEXT: v_accvgpr_write_b32 a6, v1 -; LIT-SRCC-NEXT: v_accvgpr_write_b32 a7, v2 +; LIT-SRCC-NEXT: v_accvgpr_write_b32 a6, v0 +; LIT-SRCC-NEXT: v_accvgpr_write_b32 a7, v1 ; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s24 ; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s25 ; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s26 @@ -341,7 +324,8 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32(ptr addrspace(1) %arg) #0 { ; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s10 ; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s11 ; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s12 -; LIT-SRCC-NEXT: v_mov_b32_e32 v5, s20 +; LIT-SRCC-NEXT: v_mov_b32_e32 v4, s20 +; LIT-SRCC-NEXT: v_mov_b32_e32 v5, s21 ; LIT-SRCC-NEXT: v_accvgpr_write_b32 a3, v3 ; LIT-SRCC-NEXT: v_accvgpr_write_b32 a26, v0 ; LIT-SRCC-NEXT: v_accvgpr_write_b32 a27, v1 @@ -350,7 +334,8 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32(ptr addrspace(1) %arg) #0 { ; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s14 ; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s15 ; LIT-SRCC-NEXT: v_mov_b32_e32 v3, 1.0 -; LIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v5 +; LIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v4 +; LIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v5 ; LIT-SRCC-NEXT: v_accvgpr_write_b32 a29, v0 ; LIT-SRCC-NEXT: v_accvgpr_write_b32 a30, v1 ; LIT-SRCC-NEXT: v_accvgpr_write_b32 a31, v2 @@ -363,57 +348,42 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32(ptr addrspace(1) %arg) #0 { ; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26 ; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25 ; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a24 -; LIT-SRCC-NEXT: s_nop 1 -; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:96 -; LIT-SRCC-NEXT: s_nop 0 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a31 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a30 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a29 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a28 -; LIT-SRCC-NEXT: s_nop 1 -; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:112 -; LIT-SRCC-NEXT: s_nop 0 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a19 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a18 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a17 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a16 -; LIT-SRCC-NEXT: s_nop 1 -; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:64 -; LIT-SRCC-NEXT: s_nop 0 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a23 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a22 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a21 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a20 -; LIT-SRCC-NEXT: s_nop 1 -; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:80 -; LIT-SRCC-NEXT: s_nop 0 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a8 -; LIT-SRCC-NEXT: s_nop 1 -; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:32 -; LIT-SRCC-NEXT: s_nop 0 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12 -; LIT-SRCC-NEXT: s_nop 1 -; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:48 -; LIT-SRCC-NEXT: s_nop 0 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0 -; LIT-SRCC-NEXT: s_nop 1 -; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] -; LIT-SRCC-NEXT: s_nop 0 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4 -; LIT-SRCC-NEXT: s_nop 1 -; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:16 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a31 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a19 +; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[0:3], s[34:35] offset:96 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a23 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a30 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a29 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a28 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v16, a11 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v20, a15 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a3 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v28, a7 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a18 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a17 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a16 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a22 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a21 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a20 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a10 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a9 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a8 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v19, a14 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v18, a13 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v17, a12 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v23, a2 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v22, a1 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v21, a0 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v27, a6 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v26, a5 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v25, a4 +; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[1:4], s[34:35] offset:112 +; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[5:8], s[34:35] offset:64 +; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[9:12], s[34:35] offset:80 +; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[13:16], s[34:35] offset:32 +; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[17:20], s[34:35] offset:48 +; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[21:24], s[34:35] +; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[25:28], s[34:35] offset:16 ; LIT-SRCC-NEXT: s_endpgm ; ; GFX90A-LABEL: test_mfma_f32_32x32x1f32: @@ -1833,7 +1803,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a ; NOLIT-SRCC-LABEL: test_mfma_f32_32x32x4f16: ; NOLIT-SRCC: ; %bb.0: ; %bb ; NOLIT-SRCC-NEXT: s_load_dwordx4 s[36:39], s[4:5], 0x24 -; NOLIT-SRCC-NEXT: v_mov_b32_e32 v4, 0 +; NOLIT-SRCC-NEXT: v_mov_b32_e32 v32, 0 ; NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0) ; NOLIT-SRCC-NEXT: s_load_dwordx16 s[16:31], s[36:37], 0x0 ; NOLIT-SRCC-NEXT: s_load_dwordx16 s[0:15], s[36:37], 0x40 @@ -1842,11 +1812,9 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a ; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s17 ; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s18 ; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a0, v0 -; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s22 ; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a1, v1 -; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s24 -; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a6, v0 ; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s23 +; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s24 ; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a2, v2 ; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s25 ; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a7, v0 @@ -1891,8 +1859,9 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a ; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a26, v1 ; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s12 ; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s13 -; NOLIT-SRCC-NEXT: v_mov_b32_e32 v5, s20 -; NOLIT-SRCC-NEXT: v_mov_b32_e32 v6, s21 +; NOLIT-SRCC-NEXT: v_mov_b32_e32 v4, s20 +; NOLIT-SRCC-NEXT: v_mov_b32_e32 v5, s21 +; NOLIT-SRCC-NEXT: v_mov_b32_e32 v6, s22 ; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a3, v3 ; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a27, v2 ; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a28, v0 @@ -1902,8 +1871,9 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a ; NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0) ; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s0 ; NOLIT-SRCC-NEXT: v_mov_b32_e32 v3, s1 -; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v5 -; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v6 +; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v4 +; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v5 +; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a6, v6 ; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a30, v0 ; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a31, v1 ; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s2 @@ -1916,57 +1886,42 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a ; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26 ; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25 ; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a24 -; NOLIT-SRCC-NEXT: s_nop 1 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:96 -; NOLIT-SRCC-NEXT: s_nop 0 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a31 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a30 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a29 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a28 -; NOLIT-SRCC-NEXT: s_nop 1 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:112 -; NOLIT-SRCC-NEXT: s_nop 0 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a19 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a18 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a17 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a16 -; NOLIT-SRCC-NEXT: s_nop 1 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:64 -; NOLIT-SRCC-NEXT: s_nop 0 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a23 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a22 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a21 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a20 -; NOLIT-SRCC-NEXT: s_nop 1 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:80 -; NOLIT-SRCC-NEXT: s_nop 0 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a8 -; NOLIT-SRCC-NEXT: s_nop 1 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:32 -; NOLIT-SRCC-NEXT: s_nop 0 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12 -; NOLIT-SRCC-NEXT: s_nop 1 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:48 -; NOLIT-SRCC-NEXT: s_nop 0 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0 -; NOLIT-SRCC-NEXT: s_nop 1 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] -; NOLIT-SRCC-NEXT: s_nop 0 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4 -; NOLIT-SRCC-NEXT: s_nop 1 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:16 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a31 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a19 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a23 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v19, a11 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v23, a15 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v27, a3 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v31, a7 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a30 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a29 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a28 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a18 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a17 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a16 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a22 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a21 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a20 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v18, a10 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v17, a9 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v16, a8 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v22, a14 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v21, a13 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v20, a12 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v26, a2 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v25, a1 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a0 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v30, a6 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v29, a5 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v28, a4 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[0:3], s[36:37] offset:96 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[4:7], s[36:37] offset:112 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[8:11], s[36:37] offset:64 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[12:15], s[36:37] offset:80 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[16:19], s[36:37] offset:32 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[20:23], s[36:37] offset:48 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[24:27], s[36:37] +; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[28:31], s[36:37] offset:16 ; NOLIT-SRCC-NEXT: s_endpgm ; ; GFX908-GISEL-LABEL: test_mfma_f32_32x32x4f16: @@ -2095,7 +2050,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a ; LIT-SRCC-LABEL: test_mfma_f32_32x32x4f16: ; LIT-SRCC: ; %bb.0: ; %bb ; LIT-SRCC-NEXT: s_load_dwordx4 s[36:39], s[4:5], 0x24 -; LIT-SRCC-NEXT: v_mov_b32_e32 v4, 0 +; LIT-SRCC-NEXT: v_mov_b32_e32 v32, 0 ; LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0) ; LIT-SRCC-NEXT: s_load_dwordx16 s[16:31], s[36:37], 0x0 ; LIT-SRCC-NEXT: s_load_dwordx16 s[0:15], s[36:37], 0x40 @@ -2104,11 +2059,9 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a ; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s17 ; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s18 ; LIT-SRCC-NEXT: v_accvgpr_write_b32 a0, v0 -; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s22 ; LIT-SRCC-NEXT: v_accvgpr_write_b32 a1, v1 -; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s24 -; LIT-SRCC-NEXT: v_accvgpr_write_b32 a6, v0 ; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s23 +; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s24 ; LIT-SRCC-NEXT: v_accvgpr_write_b32 a2, v2 ; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s25 ; LIT-SRCC-NEXT: v_accvgpr_write_b32 a7, v0 @@ -2153,8 +2106,9 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a ; LIT-SRCC-NEXT: v_accvgpr_write_b32 a26, v1 ; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s12 ; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s13 -; LIT-SRCC-NEXT: v_mov_b32_e32 v5, s20 -; LIT-SRCC-NEXT: v_mov_b32_e32 v6, s21 +; LIT-SRCC-NEXT: v_mov_b32_e32 v4, s20 +; LIT-SRCC-NEXT: v_mov_b32_e32 v5, s21 +; LIT-SRCC-NEXT: v_mov_b32_e32 v6, s22 ; LIT-SRCC-NEXT: v_accvgpr_write_b32 a3, v3 ; LIT-SRCC-NEXT: v_accvgpr_write_b32 a27, v2 ; LIT-SRCC-NEXT: v_accvgpr_write_b32 a28, v0 @@ -2164,8 +2118,9 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a ; LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0) ; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s0 ; LIT-SRCC-NEXT: v_mov_b32_e32 v3, s1 -; LIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v5 -; LIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v6 +; LIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v4 +; LIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v5 +; LIT-SRCC-NEXT: v_accvgpr_write_b32 a6, v6 ; LIT-SRCC-NEXT: v_accvgpr_write_b32 a30, v0 ; LIT-SRCC-NEXT: v_accvgpr_write_b32 a31, v1 ; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s2 @@ -2178,57 +2133,42 @@ define amdgpu_kernel void @test_mfma_f32_32x32x4f16(ptr addrspace(1) %arg, ptr a ; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26 ; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25 ; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a24 -; LIT-SRCC-NEXT: s_nop 1 -; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:96 -; LIT-SRCC-NEXT: s_nop 0 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a31 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a30 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a29 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a28 -; LIT-SRCC-NEXT: s_nop 1 -; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:112 -; LIT-SRCC-NEXT: s_nop 0 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a19 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a18 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a17 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a16 -; LIT-SRCC-NEXT: s_nop 1 -; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:64 -; LIT-SRCC-NEXT: s_nop 0 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a23 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a22 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a21 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a20 -; LIT-SRCC-NEXT: s_nop 1 -; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:80 -; LIT-SRCC-NEXT: s_nop 0 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a8 -; LIT-SRCC-NEXT: s_nop 1 -; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:32 -; LIT-SRCC-NEXT: s_nop 0 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12 -; LIT-SRCC-NEXT: s_nop 1 -; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:48 -; LIT-SRCC-NEXT: s_nop 0 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0 -; LIT-SRCC-NEXT: s_nop 1 -; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] -; LIT-SRCC-NEXT: s_nop 0 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4 -; LIT-SRCC-NEXT: s_nop 1 -; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[36:37] offset:16 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a31 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a19 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a23 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v19, a11 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v23, a15 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v27, a3 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v31, a7 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a30 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a29 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a28 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a18 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a17 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a16 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a22 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a21 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a20 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v18, a10 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v17, a9 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v16, a8 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v22, a14 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v21, a13 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v20, a12 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v26, a2 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v25, a1 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a0 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v30, a6 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v29, a5 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v28, a4 +; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[0:3], s[36:37] offset:96 +; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[4:7], s[36:37] offset:112 +; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[8:11], s[36:37] offset:64 +; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[12:15], s[36:37] offset:80 +; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[16:19], s[36:37] offset:32 +; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[20:23], s[36:37] offset:48 +; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[24:27], s[36:37] +; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[28:31], s[36:37] offset:16 ; LIT-SRCC-NEXT: s_endpgm ; ; GFX90A-LABEL: test_mfma_f32_32x32x4f16: @@ -3758,7 +3698,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x4i8(ptr addrspace(1) %arg) #0 { ; NOLIT-SRCC-LABEL: test_mfma_i32_32x32x4i8: ; NOLIT-SRCC: ; %bb.0: ; %bb ; NOLIT-SRCC-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24 -; NOLIT-SRCC-NEXT: v_mov_b32_e32 v16, 0 +; NOLIT-SRCC-NEXT: v_mov_b32_e32 v28, 0 ; NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0) ; NOLIT-SRCC-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0 ; NOLIT-SRCC-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40 @@ -3768,13 +3708,11 @@ define amdgpu_kernel void @test_mfma_i32_32x32x4i8(ptr addrspace(1) %arg) #0 { ; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s18 ; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a0, v0 ; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a1, v1 +; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s22 +; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s23 ; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a2, v2 -; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s21 -; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s22 -; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s23 -; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v0 -; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a6, v1 -; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a7, v2 +; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a6, v0 +; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a7, v1 ; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s24 ; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s25 ; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s26 @@ -3816,6 +3754,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x4i8(ptr addrspace(1) %arg) #0 { ; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s11 ; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s12 ; NOLIT-SRCC-NEXT: v_mov_b32_e32 v4, s20 +; NOLIT-SRCC-NEXT: v_mov_b32_e32 v5, s21 ; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a3, v3 ; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a26, v0 ; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a27, v1 @@ -3825,6 +3764,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x4i8(ptr addrspace(1) %arg) #0 { ; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s15 ; NOLIT-SRCC-NEXT: v_mov_b32_e32 v3, 1 ; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v4 +; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v5 ; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a29, v0 ; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a30, v1 ; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a31, v2 @@ -3833,47 +3773,47 @@ define amdgpu_kernel void @test_mfma_i32_32x32x4i8(ptr addrspace(1) %arg) #0 { ; NOLIT-SRCC-NEXT: v_mfma_i32_32x32x4i8 a[0:31], v3, v0, a[0:31] cbsz:1 abid:2 blgp:3 ; NOLIT-SRCC-NEXT: s_nop 15 ; NOLIT-SRCC-NEXT: s_nop 1 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a27 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a26 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a25 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a24 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v27, a27 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v26, a26 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v25, a25 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a24 ; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a3 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v28, v[24:27], s[34:35] offset:96 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a15 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v27, a31 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v26, a30 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v25, a29 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a28 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a11 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v19, a23 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v23, a19 ; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[34:35] offset:96 ; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a31 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a30 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a29 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a28 ; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a3 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[34:35] offset:112 ; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a2 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a19 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a18 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a17 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a16 ; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a1 ; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a0 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[34:35] offset:64 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a15 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a23 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a22 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a21 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a20 ; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a14 ; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a13 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[34:35] offset:80 ; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a12 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a11 ; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a10 ; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a9 ; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a8 -; NOLIT-SRCC-NEXT: s_nop 1 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[34:35] offset:32 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[8:11], s[34:35] offset:48 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[4:7], s[34:35] -; NOLIT-SRCC-NEXT: global_store_dwordx4 v16, v[0:3], s[34:35] offset:16 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v18, a22 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v17, a21 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v16, a20 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v22, a18 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v21, a17 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v20, a16 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v28, v[24:27], s[34:35] offset:112 +; NOLIT-SRCC-NEXT: s_nop 0 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v28, v[20:23], s[34:35] offset:64 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v28, v[16:19], s[34:35] offset:80 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v28, v[12:15], s[34:35] offset:32 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v28, v[8:11], s[34:35] offset:48 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v28, v[4:7], s[34:35] +; NOLIT-SRCC-NEXT: global_store_dwordx4 v28, v[0:3], s[34:35] offset:16 ; NOLIT-SRCC-NEXT: s_endpgm ; ; GFX908-GISEL-LABEL: test_mfma_i32_32x32x4i8: @@ -3999,7 +3939,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x4i8(ptr addrspace(1) %arg) #0 { ; LIT-SRCC-LABEL: test_mfma_i32_32x32x4i8: ; LIT-SRCC: ; %bb.0: ; %bb ; LIT-SRCC-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24 -; LIT-SRCC-NEXT: v_mov_b32_e32 v16, 0 +; LIT-SRCC-NEXT: v_mov_b32_e32 v28, 0 ; LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0) ; LIT-SRCC-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0 ; LIT-SRCC-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40 @@ -4009,13 +3949,11 @@ define amdgpu_kernel void @test_mfma_i32_32x32x4i8(ptr addrspace(1) %arg) #0 { ; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s18 ; LIT-SRCC-NEXT: v_accvgpr_write_b32 a0, v0 ; LIT-SRCC-NEXT: v_accvgpr_write_b32 a1, v1 +; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s22 +; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s23 ; LIT-SRCC-NEXT: v_accvgpr_write_b32 a2, v2 -; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s21 -; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s22 -; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s23 -; LIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v0 -; LIT-SRCC-NEXT: v_accvgpr_write_b32 a6, v1 -; LIT-SRCC-NEXT: v_accvgpr_write_b32 a7, v2 +; LIT-SRCC-NEXT: v_accvgpr_write_b32 a6, v0 +; LIT-SRCC-NEXT: v_accvgpr_write_b32 a7, v1 ; LIT-SRCC-NEXT: v_mov_b32_e32 v0, s24 ; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s25 ; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s26 @@ -4057,6 +3995,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x4i8(ptr addrspace(1) %arg) #0 { ; LIT-SRCC-NEXT: v_mov_b32_e32 v1, s11 ; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s12 ; LIT-SRCC-NEXT: v_mov_b32_e32 v4, s20 +; LIT-SRCC-NEXT: v_mov_b32_e32 v5, s21 ; LIT-SRCC-NEXT: v_accvgpr_write_b32 a3, v3 ; LIT-SRCC-NEXT: v_accvgpr_write_b32 a26, v0 ; LIT-SRCC-NEXT: v_accvgpr_write_b32 a27, v1 @@ -4066,6 +4005,7 @@ define amdgpu_kernel void @test_mfma_i32_32x32x4i8(ptr addrspace(1) %arg) #0 { ; LIT-SRCC-NEXT: v_mov_b32_e32 v2, s15 ; LIT-SRCC-NEXT: v_mov_b32_e32 v3, 1 ; LIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v4 +; LIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v5 ; LIT-SRCC-NEXT: v_accvgpr_write_b32 a29, v0 ; LIT-SRCC-NEXT: v_accvgpr_write_b32 a30, v1 ; LIT-SRCC-NEXT: v_accvgpr_write_b32 a31, v2 @@ -4074,47 +4014,47 @@ define amdgpu_kernel void @test_mfma_i32_32x32x4i8(ptr addrspace(1) %arg) #0 { ; LIT-SRCC-NEXT: v_mfma_i32_32x32x4i8 a[0:31], v3, v0, a[0:31] cbsz:1 abid:2 blgp:3 ; LIT-SRCC-NEXT: s_nop 15 ; LIT-SRCC-NEXT: s_nop 1 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a27 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a26 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a25 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a24 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v27, a27 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v26, a26 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v25, a25 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a24 ; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a3 +; LIT-SRCC-NEXT: global_store_dwordx4 v28, v[24:27], s[34:35] offset:96 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a15 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v27, a31 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v26, a30 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v25, a29 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a28 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a11 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v19, a23 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v23, a19 ; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6 -; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[34:35] offset:96 ; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a31 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a30 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a29 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a28 ; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a3 -; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[34:35] offset:112 ; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a2 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a19 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a18 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a17 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a16 ; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a1 ; LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a0 -; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[34:35] offset:64 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a15 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a23 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a22 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a21 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a20 ; LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a14 ; LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a13 -; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[34:35] offset:80 ; LIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a12 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a11 ; LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a10 ; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a9 ; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a8 -; LIT-SRCC-NEXT: s_nop 1 -; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[12:15], s[34:35] offset:32 -; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[8:11], s[34:35] offset:48 -; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[4:7], s[34:35] -; LIT-SRCC-NEXT: global_store_dwordx4 v16, v[0:3], s[34:35] offset:16 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v18, a22 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v17, a21 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v16, a20 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v22, a18 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v21, a17 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v20, a16 +; LIT-SRCC-NEXT: global_store_dwordx4 v28, v[24:27], s[34:35] offset:112 +; LIT-SRCC-NEXT: s_nop 0 +; LIT-SRCC-NEXT: global_store_dwordx4 v28, v[20:23], s[34:35] offset:64 +; LIT-SRCC-NEXT: global_store_dwordx4 v28, v[16:19], s[34:35] offset:80 +; LIT-SRCC-NEXT: global_store_dwordx4 v28, v[12:15], s[34:35] offset:32 +; LIT-SRCC-NEXT: global_store_dwordx4 v28, v[8:11], s[34:35] offset:48 +; LIT-SRCC-NEXT: global_store_dwordx4 v28, v[4:7], s[34:35] +; LIT-SRCC-NEXT: global_store_dwordx4 v28, v[0:3], s[34:35] offset:16 ; LIT-SRCC-NEXT: s_endpgm ; ; GFX90A-LABEL: test_mfma_i32_32x32x4i8: @@ -5672,6 +5612,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_forward_acc(ptr addrspace(1) ; NOLIT-SRCC-LABEL: test_mfma_f32_32x32x1f32_forward_acc: ; NOLIT-SRCC: ; %bb.0: ; %bb ; NOLIT-SRCC-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24 +; NOLIT-SRCC-NEXT: v_mov_b32_e32 v29, 0 ; NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0) ; NOLIT-SRCC-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0 ; NOLIT-SRCC-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40 @@ -5734,75 +5675,59 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_forward_acc(ptr addrspace(1) ; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a28, v2 ; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, s13 ; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, s14 -; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s15 -; NOLIT-SRCC-NEXT: v_mov_b32_e32 v3, 1.0 -; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v4 -; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v5 -; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a29, v0 -; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a30, v1 -; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a31, v2 -; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, 2.0 -; NOLIT-SRCC-NEXT: v_mov_b32_e32 v4, 0 -; NOLIT-SRCC-NEXT: s_nop 0 -; NOLIT-SRCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v3, v0, a[0:31] -; NOLIT-SRCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v3, v0, a[0:31] -; NOLIT-SRCC-NEXT: s_nop 15 -; NOLIT-SRCC-NEXT: s_nop 1 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a27 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a24 -; NOLIT-SRCC-NEXT: s_nop 1 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:96 -; NOLIT-SRCC-NEXT: s_nop 0 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a31 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a30 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a29 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a28 -; NOLIT-SRCC-NEXT: s_nop 1 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:112 -; NOLIT-SRCC-NEXT: s_nop 0 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a19 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a18 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a17 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a16 -; NOLIT-SRCC-NEXT: s_nop 1 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:64 -; NOLIT-SRCC-NEXT: s_nop 0 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a23 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a22 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a21 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a20 -; NOLIT-SRCC-NEXT: s_nop 1 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:80 -; NOLIT-SRCC-NEXT: s_nop 0 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a8 -; NOLIT-SRCC-NEXT: s_nop 1 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:32 -; NOLIT-SRCC-NEXT: s_nop 0 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12 -; NOLIT-SRCC-NEXT: s_nop 1 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:48 -; NOLIT-SRCC-NEXT: s_nop 0 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0 +; NOLIT-SRCC-NEXT: v_mov_b32_e32 v2, s15 +; NOLIT-SRCC-NEXT: v_mov_b32_e32 v3, 1.0 +; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a4, v4 +; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a5, v5 +; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a29, v0 +; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a30, v1 +; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a31, v2 +; NOLIT-SRCC-NEXT: v_mov_b32_e32 v0, 2.0 ; NOLIT-SRCC-NEXT: s_nop 1 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] -; NOLIT-SRCC-NEXT: s_nop 0 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4 +; NOLIT-SRCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v3, v0, a[0:31] +; NOLIT-SRCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v3, v0, a[0:31] +; NOLIT-SRCC-NEXT: s_nop 15 ; NOLIT-SRCC-NEXT: s_nop 1 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:16 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a27 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a24 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a31 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a19 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[0:3], s[34:35] offset:96 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a23 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a30 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a29 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a28 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v16, a11 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v20, a15 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a3 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v28, a7 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a18 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a17 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a16 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a22 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a21 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a20 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a10 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a9 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a8 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v19, a14 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v18, a13 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v17, a12 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v23, a2 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v22, a1 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v21, a0 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v27, a6 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v26, a5 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v25, a4 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[1:4], s[34:35] offset:112 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[5:8], s[34:35] offset:64 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[9:12], s[34:35] offset:80 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[13:16], s[34:35] offset:32 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[17:20], s[34:35] offset:48 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[21:24], s[34:35] +; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[25:28], s[34:35] offset:16 ; NOLIT-SRCC-NEXT: s_endpgm ; ; GFX908-GISEL-LABEL: test_mfma_f32_32x32x1f32_forward_acc: @@ -5929,6 +5854,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_forward_acc(ptr addrspace(1) ; LIT-SRCC-LABEL: test_mfma_f32_32x32x1f32_forward_acc: ; LIT-SRCC: ; %bb.0: ; %bb ; LIT-SRCC-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24 +; LIT-SRCC-NEXT: v_mov_b32_e32 v29, 0 ; LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0) ; LIT-SRCC-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0 ; LIT-SRCC-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40 @@ -5999,8 +5925,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_forward_acc(ptr addrspace(1) ; LIT-SRCC-NEXT: v_accvgpr_write_b32 a30, v1 ; LIT-SRCC-NEXT: v_accvgpr_write_b32 a31, v2 ; LIT-SRCC-NEXT: v_mov_b32_e32 v0, 2.0 -; LIT-SRCC-NEXT: v_mov_b32_e32 v4, 0 -; LIT-SRCC-NEXT: s_nop 0 +; LIT-SRCC-NEXT: s_nop 1 ; LIT-SRCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v3, v0, a[0:31] ; LIT-SRCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v3, v0, a[0:31] ; LIT-SRCC-NEXT: s_nop 15 @@ -6009,57 +5934,42 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_forward_acc(ptr addrspace(1) ; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26 ; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25 ; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a24 -; LIT-SRCC-NEXT: s_nop 1 -; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:96 -; LIT-SRCC-NEXT: s_nop 0 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a31 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a30 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a29 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a28 -; LIT-SRCC-NEXT: s_nop 1 -; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:112 -; LIT-SRCC-NEXT: s_nop 0 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a19 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a18 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a17 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a16 -; LIT-SRCC-NEXT: s_nop 1 -; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:64 -; LIT-SRCC-NEXT: s_nop 0 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a23 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a22 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a21 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a20 -; LIT-SRCC-NEXT: s_nop 1 -; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:80 -; LIT-SRCC-NEXT: s_nop 0 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a8 -; LIT-SRCC-NEXT: s_nop 1 -; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:32 -; LIT-SRCC-NEXT: s_nop 0 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12 -; LIT-SRCC-NEXT: s_nop 1 -; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:48 -; LIT-SRCC-NEXT: s_nop 0 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0 -; LIT-SRCC-NEXT: s_nop 1 -; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] -; LIT-SRCC-NEXT: s_nop 0 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4 -; LIT-SRCC-NEXT: s_nop 1 -; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:16 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a31 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a19 +; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[0:3], s[34:35] offset:96 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a23 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a30 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a29 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a28 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v16, a11 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v20, a15 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a3 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v28, a7 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a18 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a17 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a16 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a22 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a21 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a20 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a10 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a9 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a8 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v19, a14 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v18, a13 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v17, a12 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v23, a2 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v22, a1 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v21, a0 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v27, a6 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v26, a5 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v25, a4 +; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[1:4], s[34:35] offset:112 +; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[5:8], s[34:35] offset:64 +; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[9:12], s[34:35] offset:80 +; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[13:16], s[34:35] offset:32 +; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[17:20], s[34:35] offset:48 +; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[21:24], s[34:35] +; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[25:28], s[34:35] offset:16 ; LIT-SRCC-NEXT: s_endpgm ; ; GFX90A-LABEL: test_mfma_f32_32x32x1f32_forward_acc: @@ -7880,7 +7790,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm_splat(ptr addrspace(1) % ; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a31, 0 ; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, 2.0 ; NOLIT-SRCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 -; NOLIT-SRCC-NEXT: v_mov_b32_e32 v4, 0 +; NOLIT-SRCC-NEXT: v_mov_b32_e32 v29, 0 ; NOLIT-SRCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31] ; NOLIT-SRCC-NEXT: s_nop 15 ; NOLIT-SRCC-NEXT: s_nop 1 @@ -7888,58 +7798,43 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm_splat(ptr addrspace(1) % ; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a30 ; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a29 ; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a28 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a27 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a23 ; NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0) -; NOLIT-SRCC-NEXT: s_nop 0 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:112 -; NOLIT-SRCC-NEXT: s_nop 0 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a27 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a24 -; NOLIT-SRCC-NEXT: s_nop 1 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:96 -; NOLIT-SRCC-NEXT: s_nop 0 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a23 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a22 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a21 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a20 -; NOLIT-SRCC-NEXT: s_nop 1 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:80 -; NOLIT-SRCC-NEXT: s_nop 0 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a19 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a18 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a17 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a16 -; NOLIT-SRCC-NEXT: s_nop 1 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:64 -; NOLIT-SRCC-NEXT: s_nop 0 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12 -; NOLIT-SRCC-NEXT: s_nop 1 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:48 -; NOLIT-SRCC-NEXT: s_nop 0 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a8 -; NOLIT-SRCC-NEXT: s_nop 1 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:32 -; NOLIT-SRCC-NEXT: s_nop 0 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4 -; NOLIT-SRCC-NEXT: s_nop 1 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:16 -; NOLIT-SRCC-NEXT: s_nop 0 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0 -; NOLIT-SRCC-NEXT: s_nop 1 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] +; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[0:3], s[0:1] offset:112 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a19 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a26 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a25 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a24 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v16, a15 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v20, a11 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a7 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v28, a3 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a22 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a21 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a20 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a18 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a17 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a16 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a14 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a13 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a12 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v19, a10 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v18, a9 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v17, a8 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v23, a6 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v22, a5 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v21, a4 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v27, a2 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v26, a1 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v25, a0 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[1:4], s[0:1] offset:96 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[5:8], s[0:1] offset:80 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[9:12], s[0:1] offset:64 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[13:16], s[0:1] offset:48 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[17:20], s[0:1] offset:32 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[21:24], s[0:1] offset:16 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[25:28], s[0:1] ; NOLIT-SRCC-NEXT: s_endpgm ; ; GFX908-GISEL-NOLIT-SRCC-LABEL: test_mfma_f32_32x32x1f32_imm_splat: @@ -8031,7 +7926,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm_splat(ptr addrspace(1) % ; LIT-SRCC-NEXT: v_mov_b32_e32 v0, 1.0 ; LIT-SRCC-NEXT: v_mov_b32_e32 v1, 2.0 ; LIT-SRCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 -; LIT-SRCC-NEXT: v_mov_b32_e32 v14, 0 +; LIT-SRCC-NEXT: v_mov_b32_e32 v29, 0 ; LIT-SRCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, 0 ; LIT-SRCC-NEXT: s_nop 15 ; LIT-SRCC-NEXT: s_nop 1 @@ -8039,47 +7934,43 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm_splat(ptr addrspace(1) % ; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a30 ; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a29 ; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a28 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a27 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a26 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a25 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a24 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a23 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a22 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a21 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a20 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a27 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a23 ; LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0) -; LIT-SRCC-NEXT: global_store_dwordx4 v14, v[0:3], s[0:1] offset:112 -; LIT-SRCC-NEXT: global_store_dwordx4 v14, v[4:7], s[0:1] offset:96 -; LIT-SRCC-NEXT: global_store_dwordx4 v14, v[8:11], s[0:1] offset:80 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a19 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a18 -; LIT-SRCC-NEXT: global_store_dwordx4 v14, v[0:3], s[0:1] offset:48 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a17 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a8 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a16 -; LIT-SRCC-NEXT: s_nop 0 -; LIT-SRCC-NEXT: global_store_dwordx4 v14, v[0:3], s[0:1] offset:32 -; LIT-SRCC-NEXT: global_store_dwordx4 v14, v[10:13], s[0:1] offset:64 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4 -; LIT-SRCC-NEXT: s_nop 1 -; LIT-SRCC-NEXT: global_store_dwordx4 v14, v[0:3], s[0:1] offset:16 -; LIT-SRCC-NEXT: s_nop 0 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0 -; LIT-SRCC-NEXT: s_nop 1 -; LIT-SRCC-NEXT: global_store_dwordx4 v14, v[0:3], s[0:1] +; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[0:3], s[0:1] offset:112 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a19 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a26 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a25 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a24 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v16, a15 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v20, a11 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a7 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v28, a3 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a22 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a21 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a20 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a18 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a17 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a16 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a14 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a13 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a12 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v19, a10 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v18, a9 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v17, a8 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v23, a6 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v22, a5 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v21, a4 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v27, a2 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v26, a1 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v25, a0 +; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[1:4], s[0:1] offset:96 +; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[5:8], s[0:1] offset:80 +; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[9:12], s[0:1] offset:64 +; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[13:16], s[0:1] offset:48 +; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[17:20], s[0:1] offset:32 +; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[21:24], s[0:1] offset:16 +; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[25:28], s[0:1] ; LIT-SRCC-NEXT: s_endpgm ; ; GFX908-GISEL-LIT-SRCC-LABEL: test_mfma_f32_32x32x1f32_imm_splat: @@ -8841,7 +8732,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm(ptr addrspace(1) %arg) # ; NOLIT-SRCC-NEXT: v_accvgpr_write_b32 a31, 0 ; NOLIT-SRCC-NEXT: v_mov_b32_e32 v1, 2.0 ; NOLIT-SRCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 -; NOLIT-SRCC-NEXT: v_mov_b32_e32 v4, 0 +; NOLIT-SRCC-NEXT: v_mov_b32_e32 v29, 0 ; NOLIT-SRCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31] ; NOLIT-SRCC-NEXT: s_nop 15 ; NOLIT-SRCC-NEXT: s_nop 1 @@ -8849,58 +8740,43 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm(ptr addrspace(1) %arg) # ; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a30 ; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a29 ; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a28 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a27 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a23 ; NOLIT-SRCC-NEXT: s_waitcnt lgkmcnt(0) -; NOLIT-SRCC-NEXT: s_nop 0 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:112 -; NOLIT-SRCC-NEXT: s_nop 0 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a27 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a24 -; NOLIT-SRCC-NEXT: s_nop 1 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:96 -; NOLIT-SRCC-NEXT: s_nop 0 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a23 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a22 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a21 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a20 -; NOLIT-SRCC-NEXT: s_nop 1 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:80 -; NOLIT-SRCC-NEXT: s_nop 0 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a19 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a18 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a17 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a16 -; NOLIT-SRCC-NEXT: s_nop 1 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:64 -; NOLIT-SRCC-NEXT: s_nop 0 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12 -; NOLIT-SRCC-NEXT: s_nop 1 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:48 -; NOLIT-SRCC-NEXT: s_nop 0 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a8 -; NOLIT-SRCC-NEXT: s_nop 1 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:32 -; NOLIT-SRCC-NEXT: s_nop 0 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4 -; NOLIT-SRCC-NEXT: s_nop 1 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:16 -; NOLIT-SRCC-NEXT: s_nop 0 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0 -; NOLIT-SRCC-NEXT: s_nop 1 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] +; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[0:3], s[0:1] offset:112 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a19 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a26 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a25 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a24 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v16, a15 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v20, a11 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a7 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v28, a3 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a22 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a21 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a20 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a18 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a17 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a16 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a14 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a13 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a12 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v19, a10 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v18, a9 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v17, a8 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v23, a6 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v22, a5 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v21, a4 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v27, a2 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v26, a1 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v25, a0 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[1:4], s[0:1] offset:96 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[5:8], s[0:1] offset:80 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[9:12], s[0:1] offset:64 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[13:16], s[0:1] offset:48 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[17:20], s[0:1] offset:32 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[21:24], s[0:1] offset:16 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v29, v[25:28], s[0:1] ; NOLIT-SRCC-NEXT: s_endpgm ; ; GFX908-GISEL-LABEL: test_mfma_f32_32x32x1f32_imm: @@ -9024,7 +8900,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm(ptr addrspace(1) %arg) # ; LIT-SRCC-NEXT: v_accvgpr_write_b32 a31, 0 ; LIT-SRCC-NEXT: v_mov_b32_e32 v1, 2.0 ; LIT-SRCC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 -; LIT-SRCC-NEXT: v_mov_b32_e32 v4, 0 +; LIT-SRCC-NEXT: v_mov_b32_e32 v29, 0 ; LIT-SRCC-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31] ; LIT-SRCC-NEXT: s_nop 15 ; LIT-SRCC-NEXT: s_nop 1 @@ -9032,58 +8908,43 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_imm(ptr addrspace(1) %arg) # ; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a30 ; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a29 ; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a28 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a27 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a23 ; LIT-SRCC-NEXT: s_waitcnt lgkmcnt(0) -; LIT-SRCC-NEXT: s_nop 0 -; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:112 -; LIT-SRCC-NEXT: s_nop 0 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a27 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a24 -; LIT-SRCC-NEXT: s_nop 1 -; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:96 -; LIT-SRCC-NEXT: s_nop 0 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a23 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a22 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a21 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a20 -; LIT-SRCC-NEXT: s_nop 1 -; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:80 -; LIT-SRCC-NEXT: s_nop 0 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a19 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a18 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a17 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a16 -; LIT-SRCC-NEXT: s_nop 1 -; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:64 -; LIT-SRCC-NEXT: s_nop 0 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a15 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a14 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a13 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a12 -; LIT-SRCC-NEXT: s_nop 1 -; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:48 -; LIT-SRCC-NEXT: s_nop 0 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a11 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a10 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a9 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a8 -; LIT-SRCC-NEXT: s_nop 1 -; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:32 -; LIT-SRCC-NEXT: s_nop 0 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4 -; LIT-SRCC-NEXT: s_nop 1 -; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] offset:16 -; LIT-SRCC-NEXT: s_nop 0 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a3 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a2 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a1 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a0 -; LIT-SRCC-NEXT: s_nop 1 -; LIT-SRCC-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] +; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[0:3], s[0:1] offset:112 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a19 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a26 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a25 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a24 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v16, a15 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v20, a11 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a7 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v28, a3 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a22 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a21 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a20 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a18 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a17 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a16 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a14 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a13 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a12 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v19, a10 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v18, a9 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v17, a8 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v23, a6 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v22, a5 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v21, a4 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v27, a2 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v26, a1 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v25, a0 +; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[1:4], s[0:1] offset:96 +; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[5:8], s[0:1] offset:80 +; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[9:12], s[0:1] offset:64 +; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[13:16], s[0:1] offset:48 +; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[17:20], s[0:1] offset:32 +; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[21:24], s[0:1] offset:16 +; LIT-SRCC-NEXT: global_store_dwordx4 v29, v[25:28], s[0:1] ; LIT-SRCC-NEXT: s_endpgm ; ; GFX90A-LABEL: test_mfma_f32_32x32x1f32_imm: @@ -9940,43 +9801,42 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vecarg(ptr addrspace(1) %arg ; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26 ; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25 ; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a24 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a31 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a30 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a29 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a28 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a19 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a18 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a17 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a16 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a23 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a22 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a21 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a20 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v19, a11 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v18, a10 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v17, a9 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v16, a8 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v23, a15 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v22, a14 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v21, a13 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v20, a12 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v27, a3 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v26, a2 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v25, a1 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a0 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a31 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a19 ; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:96 -; NOLIT-SRCC-NEXT: s_nop 0 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5 -; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:112 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:64 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:80 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:48 -; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] -; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:16 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a23 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a30 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a29 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a28 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v16, a11 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v20, a15 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a3 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v28, a7 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a18 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a17 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a16 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a22 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a21 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a20 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a10 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a9 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a8 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v19, a14 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v18, a13 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v17, a12 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v23, a2 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v22, a1 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v21, a0 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v27, a6 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v26, a5 +; NOLIT-SRCC-NEXT: v_accvgpr_read_b32 v25, a4 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[1:4], s[0:1] offset:112 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[5:8], s[0:1] offset:64 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[9:12], s[0:1] offset:80 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[13:16], s[0:1] offset:32 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[17:20], s[0:1] offset:48 +; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[21:24], s[0:1] +; NOLIT-SRCC-NEXT: global_store_dwordx4 v32, v[25:28], s[0:1] offset:16 ; NOLIT-SRCC-NEXT: s_endpgm ; ; GFX908-GISEL-LABEL: test_mfma_f32_32x32x1f32_vecarg: @@ -10130,43 +9990,42 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vecarg(ptr addrspace(1) %arg ; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a26 ; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a25 ; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a24 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a31 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a30 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a29 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a28 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a19 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a18 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a17 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a16 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a23 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a22 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a21 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a20 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v19, a11 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v18, a10 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v17, a9 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v16, a8 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v23, a15 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v22, a14 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v21, a13 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v20, a12 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v27, a3 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v26, a2 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v25, a1 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a0 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v4, a31 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v8, a19 ; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:96 -; LIT-SRCC-NEXT: s_nop 0 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a7 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a6 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a5 -; LIT-SRCC-NEXT: v_accvgpr_read_b32 v0, a4 -; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:112 -; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:64 -; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:80 -; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 -; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:48 -; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] -; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:16 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v12, a23 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v3, a30 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v2, a29 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v1, a28 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v16, a11 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v20, a15 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v24, a3 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v28, a7 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v7, a18 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v6, a17 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v5, a16 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v11, a22 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v10, a21 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v9, a20 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v15, a10 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v14, a9 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v13, a8 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v19, a14 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v18, a13 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v17, a12 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v23, a2 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v22, a1 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v21, a0 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v27, a6 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v26, a5 +; LIT-SRCC-NEXT: v_accvgpr_read_b32 v25, a4 +; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[1:4], s[0:1] offset:112 +; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[5:8], s[0:1] offset:64 +; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[9:12], s[0:1] offset:80 +; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[13:16], s[0:1] offset:32 +; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[17:20], s[0:1] offset:48 +; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[21:24], s[0:1] +; LIT-SRCC-NEXT: global_store_dwordx4 v32, v[25:28], s[0:1] offset:16 ; LIT-SRCC-NEXT: s_endpgm ; ; GFX90A-LABEL: test_mfma_f32_32x32x1f32_vecarg: diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll index 96edd7876e981..5d20c7c874760 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll @@ -863,23 +863,23 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_cluster(ptr ad ; SDAG-NEXT: v_mov_b32_e32 v1, 2.0 ; SDAG-NEXT: s_waitcnt lgkmcnt(0) ; SDAG-NEXT: v_add_u32_e32 v3, s0, v0 -; SDAG-NEXT: ds_read_b128 a[60:63], v3 offset:112 -; SDAG-NEXT: ds_read_b128 a[56:59], v3 offset:96 -; SDAG-NEXT: ds_read_b128 a[52:55], v3 offset:80 -; SDAG-NEXT: ds_read_b128 a[48:51], v3 offset:64 -; SDAG-NEXT: ds_read_b128 a[32:35], v3 -; SDAG-NEXT: ds_read_b128 a[36:39], v3 offset:16 -; SDAG-NEXT: ds_read_b128 a[40:43], v3 offset:32 -; SDAG-NEXT: ds_read_b128 a[44:47], v3 offset:48 +; SDAG-NEXT: ds_read_b128 a[28:31], v3 offset:112 +; SDAG-NEXT: ds_read_b128 a[24:27], v3 offset:96 +; SDAG-NEXT: ds_read_b128 a[20:23], v3 offset:80 +; SDAG-NEXT: ds_read_b128 a[16:19], v3 offset:64 +; SDAG-NEXT: ds_read_b128 a[0:3], v3 +; SDAG-NEXT: ds_read_b128 a[4:7], v3 offset:16 +; SDAG-NEXT: ds_read_b128 a[8:11], v3 offset:32 +; SDAG-NEXT: ds_read_b128 a[12:15], v3 offset:48 ; SDAG-NEXT: v_add_u32_e32 v4, 0x6000, v3 -; SDAG-NEXT: ds_read_b128 a[28:31], v3 offset:8304 -; SDAG-NEXT: ds_read_b128 a[24:27], v3 offset:8288 -; SDAG-NEXT: ds_read_b128 a[20:23], v3 offset:8272 -; SDAG-NEXT: ds_read_b128 a[16:19], v3 offset:8256 -; SDAG-NEXT: ds_read_b128 a[12:15], v3 offset:8240 -; SDAG-NEXT: ds_read_b128 a[8:11], v3 offset:8224 -; SDAG-NEXT: ds_read_b128 a[4:7], v3 offset:8208 -; SDAG-NEXT: ds_read_b128 a[0:3], v3 offset:8192 +; SDAG-NEXT: ds_read_b128 a[60:63], v3 offset:8304 +; SDAG-NEXT: ds_read_b128 a[56:59], v3 offset:8288 +; SDAG-NEXT: ds_read_b128 a[52:55], v3 offset:8272 +; SDAG-NEXT: ds_read_b128 a[48:51], v3 offset:8256 +; SDAG-NEXT: ds_read_b128 a[44:47], v3 offset:8240 +; SDAG-NEXT: ds_read_b128 a[40:43], v3 offset:8224 +; SDAG-NEXT: ds_read_b128 a[36:39], v3 offset:8208 +; SDAG-NEXT: ds_read_b128 a[32:35], v3 offset:8192 ; SDAG-NEXT: ds_read_b128 a[92:95], v3 offset:24688 ; SDAG-NEXT: ds_read_b128 a[88:91], v3 offset:24672 ; SDAG-NEXT: ds_read_b128 a[84:87], v3 offset:24656 @@ -905,33 +905,33 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_cluster(ptr ad ; SDAG-NEXT: ds_read_b128 a[136:139], v4 offset:57376 ; SDAG-NEXT: ds_read_b128 a[140:143], v4 offset:57392 ; SDAG-NEXT: s_waitcnt lgkmcnt(14) -; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v2, v1, a[32:63] +; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v1, a[0:31] ; SDAG-NEXT: v_add_u32_e32 v0, s1, v0 ; SDAG-NEXT: ; sched_group_barrier mask(0x00000100) size(40) SyncID(0) +; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v2, v1, a[32:63] +; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v2, v1, a[64:95] ; SDAG-NEXT: s_waitcnt lgkmcnt(8) ; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v2, v1, a[96:127] -; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v2, v1, a[64:95] -; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v1, a[0:31] ; SDAG-NEXT: s_waitcnt lgkmcnt(0) ; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v2, v1, a[128:159] ; SDAG-NEXT: s_nop 11 -; SDAG-NEXT: ds_write_b128 v0, a[60:63] offset:112 -; SDAG-NEXT: ds_write_b128 v0, a[56:59] offset:96 -; SDAG-NEXT: ds_write_b128 v0, a[52:55] offset:80 -; SDAG-NEXT: ds_write_b128 v0, a[48:51] offset:64 -; SDAG-NEXT: ds_write_b128 v0, a[44:47] offset:48 -; SDAG-NEXT: ds_write_b128 v0, a[40:43] offset:32 -; SDAG-NEXT: ds_write_b128 v0, a[36:39] offset:16 -; SDAG-NEXT: ds_write_b128 v0, a[32:35] +; SDAG-NEXT: ds_write_b128 v0, a[28:31] offset:112 +; SDAG-NEXT: ds_write_b128 v0, a[24:27] offset:96 +; SDAG-NEXT: ds_write_b128 v0, a[20:23] offset:80 +; SDAG-NEXT: ds_write_b128 v0, a[16:19] offset:64 +; SDAG-NEXT: ds_write_b128 v0, a[12:15] offset:48 +; SDAG-NEXT: ds_write_b128 v0, a[8:11] offset:32 +; SDAG-NEXT: ds_write_b128 v0, a[4:7] offset:16 +; SDAG-NEXT: ds_write_b128 v0, a[0:3] ; SDAG-NEXT: v_mov_b32_e32 v0, s1 -; SDAG-NEXT: ds_write_b128 v0, a[24:27] offset:8288 -; SDAG-NEXT: ds_write_b128 v0, a[28:31] offset:8304 -; SDAG-NEXT: ds_write_b128 v0, a[16:19] offset:8256 -; SDAG-NEXT: ds_write_b128 v0, a[20:23] offset:8272 -; SDAG-NEXT: ds_write_b128 v0, a[8:11] offset:8224 -; SDAG-NEXT: ds_write_b128 v0, a[12:15] offset:8240 -; SDAG-NEXT: ds_write_b128 v0, a[0:3] offset:8192 -; SDAG-NEXT: ds_write_b128 v0, a[4:7] offset:8208 +; SDAG-NEXT: ds_write_b128 v0, a[56:59] offset:8288 +; SDAG-NEXT: ds_write_b128 v0, a[60:63] offset:8304 +; SDAG-NEXT: ds_write_b128 v0, a[48:51] offset:8256 +; SDAG-NEXT: ds_write_b128 v0, a[52:55] offset:8272 +; SDAG-NEXT: ds_write_b128 v0, a[40:43] offset:8224 +; SDAG-NEXT: ds_write_b128 v0, a[44:47] offset:8240 +; SDAG-NEXT: ds_write_b128 v0, a[32:35] offset:8192 +; SDAG-NEXT: ds_write_b128 v0, a[36:39] offset:8208 ; SDAG-NEXT: ds_write_b128 v0, a[88:91] offset:16480 ; SDAG-NEXT: ds_write_b128 v0, a[92:95] offset:16496 ; SDAG-NEXT: ds_write_b128 v0, a[80:83] offset:16448 @@ -964,145 +964,209 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_cluster(ptr ad ; GISEL: ; %bb.0: ; %entry ; GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24 ; GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0 -; GISEL-NEXT: v_lshlrev_b32_e32 v32, 7, v0 -; GISEL-NEXT: v_mov_b32_e32 v33, 1.0 -; GISEL-NEXT: v_mov_b32_e32 v34, 2.0 +; GISEL-NEXT: v_lshlrev_b32_e32 v0, 7, v0 +; GISEL-NEXT: v_mov_b32_e32 v96, 1.0 +; GISEL-NEXT: v_mov_b32_e32 v97, 2.0 ; GISEL-NEXT: s_waitcnt lgkmcnt(0) -; GISEL-NEXT: v_add_u32_e32 v0, s0, v32 -; GISEL-NEXT: ds_read_b128 a[96:99], v0 -; GISEL-NEXT: ds_read_b128 a[100:103], v0 offset:16 -; GISEL-NEXT: ds_read_b128 a[104:107], v0 offset:32 -; GISEL-NEXT: ds_read_b128 a[108:111], v0 offset:48 -; GISEL-NEXT: ds_read_b128 a[112:115], v0 offset:64 -; GISEL-NEXT: ds_read_b128 a[116:119], v0 offset:80 -; GISEL-NEXT: ds_read_b128 a[120:123], v0 offset:96 -; GISEL-NEXT: ds_read_b128 a[124:127], v0 offset:112 -; GISEL-NEXT: ds_read_b128 a[0:3], v0 offset:8192 -; GISEL-NEXT: ds_read_b128 a[4:7], v0 offset:8208 -; GISEL-NEXT: ds_read_b128 a[8:11], v0 offset:8224 -; GISEL-NEXT: ds_read_b128 a[12:15], v0 offset:8240 -; GISEL-NEXT: ds_read_b128 a[16:19], v0 offset:8256 -; GISEL-NEXT: ds_read_b128 a[20:23], v0 offset:8272 -; GISEL-NEXT: ds_read_b128 a[24:27], v0 offset:8288 -; GISEL-NEXT: ds_read_b128 a[28:31], v0 offset:8304 -; GISEL-NEXT: ds_read_b128 a[32:35], v0 offset:24576 -; GISEL-NEXT: ds_read_b128 a[36:39], v0 offset:24592 -; GISEL-NEXT: ds_read_b128 a[40:43], v0 offset:24608 -; GISEL-NEXT: ds_read_b128 a[44:47], v0 offset:24624 -; GISEL-NEXT: ds_read_b128 a[48:51], v0 offset:24640 -; GISEL-NEXT: ds_read_b128 a[52:55], v0 offset:24656 -; GISEL-NEXT: ds_read_b128 a[56:59], v0 offset:24672 -; GISEL-NEXT: ds_read_b128 a[60:63], v0 offset:24688 -; GISEL-NEXT: v_add_u32_e32 v1, 0xc000, v0 -; GISEL-NEXT: ds_read_b128 a[64:67], v0 offset:49152 -; GISEL-NEXT: ds_read_b128 a[68:71], v0 offset:49168 -; GISEL-NEXT: ds_read_b128 a[72:75], v0 offset:49184 -; GISEL-NEXT: ds_read_b128 a[76:79], v0 offset:49200 -; GISEL-NEXT: ds_read_b128 a[80:83], v0 offset:49216 -; GISEL-NEXT: ds_read_b128 a[84:87], v0 offset:49232 -; GISEL-NEXT: ds_read_b128 a[88:91], v0 offset:49248 -; GISEL-NEXT: ds_read_b128 a[92:95], v0 offset:49264 -; GISEL-NEXT: ds_read_b128 a[128:131], v1 offset:32768 -; GISEL-NEXT: ds_read_b128 a[132:135], v1 offset:32784 -; GISEL-NEXT: ds_read_b128 a[136:139], v1 offset:32800 -; GISEL-NEXT: ds_read_b128 a[140:143], v1 offset:32816 -; GISEL-NEXT: ds_read_b128 a[144:147], v1 offset:32832 -; GISEL-NEXT: ds_read_b128 a[148:151], v1 offset:32848 -; GISEL-NEXT: ds_read_b128 a[152:155], v1 offset:32864 -; GISEL-NEXT: ds_read_b128 a[156:159], v1 offset:32880 +; GISEL-NEXT: v_add_u32_e32 v1, s0, v0 +; GISEL-NEXT: ds_read_b128 a[0:3], v1 +; GISEL-NEXT: ds_read_b128 a[4:7], v1 offset:16 +; GISEL-NEXT: ds_read_b128 a[8:11], v1 offset:32 +; GISEL-NEXT: ds_read_b128 a[12:15], v1 offset:48 +; GISEL-NEXT: ds_read_b128 a[16:19], v1 offset:64 +; GISEL-NEXT: ds_read_b128 a[20:23], v1 offset:80 +; GISEL-NEXT: ds_read_b128 a[24:27], v1 offset:96 +; GISEL-NEXT: ds_read_b128 a[28:31], v1 offset:112 +; GISEL-NEXT: v_add_u32_e32 v2, 0xc000, v1 +; GISEL-NEXT: ds_read_b128 a[128:131], v1 offset:24576 +; GISEL-NEXT: ds_read_b128 a[132:135], v1 offset:24592 +; GISEL-NEXT: ds_read_b128 a[136:139], v1 offset:24608 +; GISEL-NEXT: ds_read_b128 a[140:143], v1 offset:24624 +; GISEL-NEXT: ds_read_b128 a[144:147], v1 offset:24640 +; GISEL-NEXT: ds_read_b128 a[148:151], v1 offset:24656 +; GISEL-NEXT: ds_read_b128 a[152:155], v1 offset:24672 +; GISEL-NEXT: ds_read_b128 a[156:159], v1 offset:24688 +; GISEL-NEXT: ds_read_b128 a[96:99], v1 offset:8192 +; GISEL-NEXT: ds_read_b128 a[100:103], v1 offset:8208 +; GISEL-NEXT: ds_read_b128 a[104:107], v1 offset:8224 +; GISEL-NEXT: ds_read_b128 a[108:111], v1 offset:8240 +; GISEL-NEXT: ds_read_b128 a[112:115], v1 offset:8256 +; GISEL-NEXT: ds_read_b128 a[116:119], v1 offset:8272 +; GISEL-NEXT: ds_read_b128 a[120:123], v1 offset:8288 +; GISEL-NEXT: ds_read_b128 a[124:127], v1 offset:8304 +; GISEL-NEXT: ds_read_b128 a[64:67], v1 offset:49152 +; GISEL-NEXT: ds_read_b128 a[68:71], v1 offset:49168 +; GISEL-NEXT: ds_read_b128 a[72:75], v1 offset:49184 +; GISEL-NEXT: ds_read_b128 a[76:79], v1 offset:49200 +; GISEL-NEXT: ds_read_b128 a[80:83], v1 offset:49216 +; GISEL-NEXT: ds_read_b128 a[84:87], v1 offset:49232 +; GISEL-NEXT: ds_read_b128 a[88:91], v1 offset:49248 +; GISEL-NEXT: ds_read_b128 a[92:95], v1 offset:49264 +; GISEL-NEXT: ds_read_b128 a[32:35], v2 offset:32768 +; GISEL-NEXT: ds_read_b128 a[36:39], v2 offset:32784 +; GISEL-NEXT: ds_read_b128 a[40:43], v2 offset:32800 +; GISEL-NEXT: ds_read_b128 a[44:47], v2 offset:32816 +; GISEL-NEXT: ds_read_b128 a[48:51], v2 offset:32832 +; GISEL-NEXT: ds_read_b128 a[52:55], v2 offset:32848 +; GISEL-NEXT: ds_read_b128 a[56:59], v2 offset:32864 +; GISEL-NEXT: ds_read_b128 a[60:63], v2 offset:32880 ; GISEL-NEXT: s_waitcnt lgkmcnt(14) -; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v33, v34, a[96:127] -; GISEL-NEXT: v_add_u32_e32 v32, s1, v32 +; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v96, v97, a[0:31] +; GISEL-NEXT: v_add_u32_e32 v98, s1, v0 ; GISEL-NEXT: ; sched_group_barrier mask(0x00000100) size(40) SyncID(0) -; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v33, v34, a[0:31] +; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v96, v97, a[128:159] ; GISEL-NEXT: s_nop 15 ; GISEL-NEXT: s_nop 0 -; GISEL-NEXT: v_accvgpr_read_b32 v0, a96 -; GISEL-NEXT: v_accvgpr_read_b32 v1, a97 -; GISEL-NEXT: v_accvgpr_read_b32 v2, a98 -; GISEL-NEXT: v_accvgpr_read_b32 v3, a99 -; GISEL-NEXT: v_accvgpr_read_b32 v4, a100 -; GISEL-NEXT: v_accvgpr_read_b32 v5, a101 -; GISEL-NEXT: v_accvgpr_read_b32 v6, a102 -; GISEL-NEXT: v_accvgpr_read_b32 v7, a103 -; GISEL-NEXT: v_accvgpr_read_b32 v8, a104 -; GISEL-NEXT: v_accvgpr_read_b32 v9, a105 -; GISEL-NEXT: v_accvgpr_read_b32 v10, a106 -; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v33, v34, a[32:63] -; GISEL-NEXT: v_accvgpr_read_b32 v11, a107 -; GISEL-NEXT: v_accvgpr_read_b32 v12, a108 -; GISEL-NEXT: v_accvgpr_read_b32 v13, a109 -; GISEL-NEXT: v_accvgpr_read_b32 v14, a110 -; GISEL-NEXT: v_accvgpr_read_b32 v15, a111 -; GISEL-NEXT: v_accvgpr_read_b32 v16, a112 -; GISEL-NEXT: v_accvgpr_read_b32 v17, a113 -; GISEL-NEXT: v_accvgpr_read_b32 v18, a114 -; GISEL-NEXT: v_accvgpr_read_b32 v19, a115 -; GISEL-NEXT: v_accvgpr_read_b32 v20, a116 -; GISEL-NEXT: v_accvgpr_read_b32 v21, a117 -; GISEL-NEXT: v_accvgpr_read_b32 v22, a118 -; GISEL-NEXT: v_accvgpr_read_b32 v23, a119 -; GISEL-NEXT: v_accvgpr_read_b32 v24, a120 -; GISEL-NEXT: v_accvgpr_read_b32 v25, a121 +; GISEL-NEXT: v_accvgpr_read_b32 v95, a31 +; GISEL-NEXT: v_accvgpr_read_b32 v94, a30 +; GISEL-NEXT: v_accvgpr_read_b32 v93, a29 +; GISEL-NEXT: v_accvgpr_read_b32 v92, a28 +; GISEL-NEXT: v_accvgpr_read_b32 v91, a27 +; GISEL-NEXT: v_accvgpr_read_b32 v90, a26 +; GISEL-NEXT: v_accvgpr_read_b32 v89, a25 +; GISEL-NEXT: v_accvgpr_read_b32 v88, a24 +; GISEL-NEXT: v_accvgpr_read_b32 v87, a23 +; GISEL-NEXT: v_accvgpr_read_b32 v86, a22 +; GISEL-NEXT: v_accvgpr_read_b32 v85, a21 +; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v96, v97, a[96:127] +; GISEL-NEXT: v_accvgpr_read_b32 v84, a20 +; GISEL-NEXT: v_accvgpr_read_b32 v83, a19 +; GISEL-NEXT: v_accvgpr_read_b32 v82, a18 +; GISEL-NEXT: v_accvgpr_read_b32 v81, a17 +; GISEL-NEXT: v_accvgpr_read_b32 v80, a16 +; GISEL-NEXT: v_accvgpr_read_b32 v79, a15 +; GISEL-NEXT: v_accvgpr_read_b32 v78, a14 +; GISEL-NEXT: v_accvgpr_read_b32 v77, a13 +; GISEL-NEXT: v_accvgpr_read_b32 v76, a12 +; GISEL-NEXT: v_accvgpr_read_b32 v75, a11 +; GISEL-NEXT: v_accvgpr_read_b32 v74, a10 +; GISEL-NEXT: v_accvgpr_read_b32 v73, a9 +; GISEL-NEXT: v_accvgpr_read_b32 v72, a8 +; GISEL-NEXT: v_accvgpr_read_b32 v71, a7 +; GISEL-NEXT: v_accvgpr_read_b32 v70, a6 ; GISEL-NEXT: s_waitcnt lgkmcnt(8) -; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v33, v34, a[64:95] -; GISEL-NEXT: v_accvgpr_read_b32 v26, a122 -; GISEL-NEXT: v_accvgpr_read_b32 v27, a123 -; GISEL-NEXT: v_accvgpr_read_b32 v28, a124 -; GISEL-NEXT: v_accvgpr_read_b32 v29, a125 -; GISEL-NEXT: v_accvgpr_read_b32 v30, a126 -; GISEL-NEXT: v_accvgpr_read_b32 v31, a127 +; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v96, v97, a[64:95] +; GISEL-NEXT: v_accvgpr_read_b32 v69, a5 +; GISEL-NEXT: v_accvgpr_read_b32 v68, a4 +; GISEL-NEXT: v_accvgpr_read_b32 v67, a3 +; GISEL-NEXT: v_accvgpr_read_b32 v66, a2 +; GISEL-NEXT: v_accvgpr_read_b32 v65, a1 +; GISEL-NEXT: v_accvgpr_read_b32 v64, a0 +; GISEL-NEXT: v_accvgpr_read_b32 v0, a128 +; GISEL-NEXT: v_accvgpr_read_b32 v1, a129 +; GISEL-NEXT: v_accvgpr_read_b32 v2, a130 +; GISEL-NEXT: v_accvgpr_read_b32 v3, a131 +; GISEL-NEXT: v_accvgpr_read_b32 v4, a132 +; GISEL-NEXT: v_accvgpr_read_b32 v5, a133 +; GISEL-NEXT: v_accvgpr_read_b32 v6, a134 +; GISEL-NEXT: v_accvgpr_read_b32 v7, a135 +; GISEL-NEXT: v_accvgpr_read_b32 v8, a136 ; GISEL-NEXT: s_waitcnt lgkmcnt(0) -; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v33, v34, a[128:159] -; GISEL-NEXT: ds_write_b128 v32, v[0:3] -; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:16 -; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:32 -; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:48 -; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:64 -; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:80 -; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:96 -; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:112 -; GISEL-NEXT: v_accvgpr_read_b32 v0, a0 -; GISEL-NEXT: v_accvgpr_read_b32 v1, a1 -; GISEL-NEXT: v_accvgpr_read_b32 v2, a2 -; GISEL-NEXT: v_accvgpr_read_b32 v3, a3 -; GISEL-NEXT: v_accvgpr_read_b32 v4, a4 -; GISEL-NEXT: v_accvgpr_read_b32 v5, a5 -; GISEL-NEXT: v_accvgpr_read_b32 v6, a6 -; GISEL-NEXT: v_accvgpr_read_b32 v7, a7 -; GISEL-NEXT: v_accvgpr_read_b32 v8, a8 -; GISEL-NEXT: v_accvgpr_read_b32 v9, a9 -; GISEL-NEXT: v_accvgpr_read_b32 v10, a10 -; GISEL-NEXT: v_accvgpr_read_b32 v11, a11 -; GISEL-NEXT: v_accvgpr_read_b32 v12, a12 -; GISEL-NEXT: v_accvgpr_read_b32 v13, a13 -; GISEL-NEXT: v_accvgpr_read_b32 v14, a14 -; GISEL-NEXT: v_accvgpr_read_b32 v15, a15 -; GISEL-NEXT: v_accvgpr_read_b32 v16, a16 -; GISEL-NEXT: v_accvgpr_read_b32 v17, a17 -; GISEL-NEXT: v_accvgpr_read_b32 v18, a18 -; GISEL-NEXT: v_accvgpr_read_b32 v19, a19 -; GISEL-NEXT: v_accvgpr_read_b32 v20, a20 -; GISEL-NEXT: v_accvgpr_read_b32 v21, a21 -; GISEL-NEXT: v_accvgpr_read_b32 v22, a22 -; GISEL-NEXT: v_accvgpr_read_b32 v23, a23 -; GISEL-NEXT: v_accvgpr_read_b32 v24, a24 -; GISEL-NEXT: v_accvgpr_read_b32 v25, a25 -; GISEL-NEXT: v_accvgpr_read_b32 v26, a26 -; GISEL-NEXT: v_accvgpr_read_b32 v27, a27 -; GISEL-NEXT: v_accvgpr_read_b32 v28, a28 -; GISEL-NEXT: v_accvgpr_read_b32 v29, a29 -; GISEL-NEXT: v_accvgpr_read_b32 v30, a30 -; GISEL-NEXT: v_accvgpr_read_b32 v31, a31 -; GISEL-NEXT: v_mov_b32_e32 v32, s1 -; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:8192 -; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:8208 -; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:8224 -; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:8240 -; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:8256 -; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:8272 -; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:8288 -; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:8304 +; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v96, v97, a[32:63] +; GISEL-NEXT: v_accvgpr_read_b32 v9, a137 +; GISEL-NEXT: v_accvgpr_read_b32 v10, a138 +; GISEL-NEXT: v_accvgpr_read_b32 v11, a139 +; GISEL-NEXT: v_accvgpr_read_b32 v12, a140 +; GISEL-NEXT: v_accvgpr_read_b32 v13, a141 +; GISEL-NEXT: v_accvgpr_read_b32 v14, a142 +; GISEL-NEXT: v_accvgpr_read_b32 v15, a143 +; GISEL-NEXT: v_accvgpr_read_b32 v16, a144 +; GISEL-NEXT: v_accvgpr_read_b32 v17, a145 +; GISEL-NEXT: v_accvgpr_read_b32 v18, a146 +; GISEL-NEXT: v_accvgpr_read_b32 v19, a147 +; GISEL-NEXT: v_accvgpr_read_b32 v20, a148 +; GISEL-NEXT: v_accvgpr_read_b32 v21, a149 +; GISEL-NEXT: v_accvgpr_read_b32 v22, a150 +; GISEL-NEXT: v_accvgpr_read_b32 v23, a151 +; GISEL-NEXT: v_accvgpr_read_b32 v24, a152 +; GISEL-NEXT: v_accvgpr_read_b32 v25, a153 +; GISEL-NEXT: v_accvgpr_read_b32 v26, a154 +; GISEL-NEXT: v_accvgpr_read_b32 v27, a155 +; GISEL-NEXT: v_accvgpr_read_b32 v28, a156 +; GISEL-NEXT: v_accvgpr_read_b32 v29, a157 +; GISEL-NEXT: v_accvgpr_read_b32 v30, a158 +; GISEL-NEXT: v_accvgpr_read_b32 v31, a159 +; GISEL-NEXT: v_mov_b32_e32 v96, s1 +; GISEL-NEXT: v_accvgpr_read_b32 v32, a96 +; GISEL-NEXT: v_accvgpr_read_b32 v33, a97 +; GISEL-NEXT: v_accvgpr_read_b32 v34, a98 +; GISEL-NEXT: v_accvgpr_read_b32 v35, a99 +; GISEL-NEXT: v_accvgpr_read_b32 v36, a100 +; GISEL-NEXT: v_accvgpr_read_b32 v37, a101 +; GISEL-NEXT: v_accvgpr_read_b32 v38, a102 +; GISEL-NEXT: v_accvgpr_read_b32 v39, a103 +; GISEL-NEXT: v_accvgpr_read_b32 v40, a104 +; GISEL-NEXT: v_accvgpr_read_b32 v41, a105 +; GISEL-NEXT: v_accvgpr_read_b32 v42, a106 +; GISEL-NEXT: v_accvgpr_read_b32 v43, a107 +; GISEL-NEXT: v_accvgpr_read_b32 v44, a108 +; GISEL-NEXT: v_accvgpr_read_b32 v45, a109 +; GISEL-NEXT: v_accvgpr_read_b32 v46, a110 +; GISEL-NEXT: v_accvgpr_read_b32 v47, a111 +; GISEL-NEXT: v_accvgpr_read_b32 v48, a112 +; GISEL-NEXT: v_accvgpr_read_b32 v49, a113 +; GISEL-NEXT: v_accvgpr_read_b32 v50, a114 +; GISEL-NEXT: v_accvgpr_read_b32 v51, a115 +; GISEL-NEXT: v_accvgpr_read_b32 v52, a116 +; GISEL-NEXT: v_accvgpr_read_b32 v53, a117 +; GISEL-NEXT: v_accvgpr_read_b32 v54, a118 +; GISEL-NEXT: v_accvgpr_read_b32 v55, a119 +; GISEL-NEXT: v_accvgpr_read_b32 v56, a120 +; GISEL-NEXT: v_accvgpr_read_b32 v57, a121 +; GISEL-NEXT: v_accvgpr_read_b32 v58, a122 +; GISEL-NEXT: v_accvgpr_read_b32 v59, a123 +; GISEL-NEXT: v_accvgpr_read_b32 v60, a124 +; GISEL-NEXT: v_accvgpr_read_b32 v61, a125 +; GISEL-NEXT: v_accvgpr_read_b32 v62, a126 +; GISEL-NEXT: v_accvgpr_read_b32 v63, a127 +; GISEL-NEXT: ds_write_b128 v98, v[64:67] +; GISEL-NEXT: ds_write_b128 v98, v[68:71] offset:16 +; GISEL-NEXT: ds_write_b128 v98, v[72:75] offset:32 +; GISEL-NEXT: ds_write_b128 v98, v[76:79] offset:48 +; GISEL-NEXT: ds_write_b128 v98, v[80:83] offset:64 +; GISEL-NEXT: ds_write_b128 v98, v[84:87] offset:80 +; GISEL-NEXT: ds_write_b128 v98, v[88:91] offset:96 +; GISEL-NEXT: ds_write_b128 v98, v[92:95] offset:112 +; GISEL-NEXT: v_accvgpr_read_b32 v64, a64 +; GISEL-NEXT: v_accvgpr_read_b32 v65, a65 +; GISEL-NEXT: v_accvgpr_read_b32 v66, a66 +; GISEL-NEXT: v_accvgpr_read_b32 v67, a67 +; GISEL-NEXT: v_accvgpr_read_b32 v68, a68 +; GISEL-NEXT: v_accvgpr_read_b32 v69, a69 +; GISEL-NEXT: v_accvgpr_read_b32 v70, a70 +; GISEL-NEXT: v_accvgpr_read_b32 v71, a71 +; GISEL-NEXT: v_accvgpr_read_b32 v72, a72 +; GISEL-NEXT: v_accvgpr_read_b32 v73, a73 +; GISEL-NEXT: v_accvgpr_read_b32 v74, a74 +; GISEL-NEXT: v_accvgpr_read_b32 v75, a75 +; GISEL-NEXT: v_accvgpr_read_b32 v76, a76 +; GISEL-NEXT: v_accvgpr_read_b32 v77, a77 +; GISEL-NEXT: v_accvgpr_read_b32 v78, a78 +; GISEL-NEXT: v_accvgpr_read_b32 v79, a79 +; GISEL-NEXT: v_accvgpr_read_b32 v80, a80 +; GISEL-NEXT: v_accvgpr_read_b32 v81, a81 +; GISEL-NEXT: v_accvgpr_read_b32 v82, a82 +; GISEL-NEXT: v_accvgpr_read_b32 v83, a83 +; GISEL-NEXT: v_accvgpr_read_b32 v84, a84 +; GISEL-NEXT: v_accvgpr_read_b32 v85, a85 +; GISEL-NEXT: v_accvgpr_read_b32 v86, a86 +; GISEL-NEXT: v_accvgpr_read_b32 v87, a87 +; GISEL-NEXT: v_accvgpr_read_b32 v88, a88 +; GISEL-NEXT: v_accvgpr_read_b32 v89, a89 +; GISEL-NEXT: v_accvgpr_read_b32 v90, a90 +; GISEL-NEXT: v_accvgpr_read_b32 v91, a91 +; GISEL-NEXT: v_accvgpr_read_b32 v92, a92 +; GISEL-NEXT: v_accvgpr_read_b32 v93, a93 +; GISEL-NEXT: v_accvgpr_read_b32 v94, a94 +; GISEL-NEXT: v_accvgpr_read_b32 v95, a95 +; GISEL-NEXT: ds_write_b128 v96, v[0:3] offset:16384 +; GISEL-NEXT: ds_write_b128 v96, v[4:7] offset:16400 +; GISEL-NEXT: ds_write_b128 v96, v[8:11] offset:16416 +; GISEL-NEXT: ds_write_b128 v96, v[12:15] offset:16432 +; GISEL-NEXT: ds_write_b128 v96, v[16:19] offset:16448 +; GISEL-NEXT: ds_write_b128 v96, v[20:23] offset:16464 +; GISEL-NEXT: ds_write_b128 v96, v[24:27] offset:16480 +; GISEL-NEXT: ds_write_b128 v96, v[28:31] offset:16496 ; GISEL-NEXT: v_accvgpr_read_b32 v0, a32 ; GISEL-NEXT: v_accvgpr_read_b32 v1, a33 ; GISEL-NEXT: v_accvgpr_read_b32 v2, a34 @@ -1135,94 +1199,30 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_cluster(ptr ad ; GISEL-NEXT: v_accvgpr_read_b32 v29, a61 ; GISEL-NEXT: v_accvgpr_read_b32 v30, a62 ; GISEL-NEXT: v_accvgpr_read_b32 v31, a63 -; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:16384 -; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:16400 -; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:16416 -; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:16432 -; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:16448 -; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:16464 -; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:16480 -; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:16496 -; GISEL-NEXT: v_accvgpr_read_b32 v0, a64 -; GISEL-NEXT: v_accvgpr_read_b32 v1, a65 -; GISEL-NEXT: v_accvgpr_read_b32 v2, a66 -; GISEL-NEXT: v_accvgpr_read_b32 v3, a67 -; GISEL-NEXT: v_accvgpr_read_b32 v4, a68 -; GISEL-NEXT: v_accvgpr_read_b32 v5, a69 -; GISEL-NEXT: v_accvgpr_read_b32 v6, a70 -; GISEL-NEXT: v_accvgpr_read_b32 v7, a71 -; GISEL-NEXT: v_accvgpr_read_b32 v8, a72 -; GISEL-NEXT: v_accvgpr_read_b32 v9, a73 -; GISEL-NEXT: v_accvgpr_read_b32 v10, a74 -; GISEL-NEXT: v_accvgpr_read_b32 v11, a75 -; GISEL-NEXT: v_accvgpr_read_b32 v12, a76 -; GISEL-NEXT: v_accvgpr_read_b32 v13, a77 -; GISEL-NEXT: v_accvgpr_read_b32 v14, a78 -; GISEL-NEXT: v_accvgpr_read_b32 v15, a79 -; GISEL-NEXT: v_accvgpr_read_b32 v16, a80 -; GISEL-NEXT: v_accvgpr_read_b32 v17, a81 -; GISEL-NEXT: v_accvgpr_read_b32 v18, a82 -; GISEL-NEXT: v_accvgpr_read_b32 v19, a83 -; GISEL-NEXT: v_accvgpr_read_b32 v20, a84 -; GISEL-NEXT: v_accvgpr_read_b32 v21, a85 -; GISEL-NEXT: v_accvgpr_read_b32 v22, a86 -; GISEL-NEXT: v_accvgpr_read_b32 v23, a87 -; GISEL-NEXT: v_accvgpr_read_b32 v24, a88 -; GISEL-NEXT: v_accvgpr_read_b32 v25, a89 -; GISEL-NEXT: v_accvgpr_read_b32 v26, a90 -; GISEL-NEXT: v_accvgpr_read_b32 v27, a91 -; GISEL-NEXT: v_accvgpr_read_b32 v28, a92 -; GISEL-NEXT: v_accvgpr_read_b32 v29, a93 -; GISEL-NEXT: v_accvgpr_read_b32 v30, a94 -; GISEL-NEXT: v_accvgpr_read_b32 v31, a95 -; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:24576 -; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:24592 -; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:24608 -; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:24624 -; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:24640 -; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:24656 -; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:24672 -; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:24688 -; GISEL-NEXT: v_accvgpr_read_b32 v0, a128 -; GISEL-NEXT: v_accvgpr_read_b32 v1, a129 -; GISEL-NEXT: v_accvgpr_read_b32 v2, a130 -; GISEL-NEXT: v_accvgpr_read_b32 v3, a131 -; GISEL-NEXT: v_accvgpr_read_b32 v4, a132 -; GISEL-NEXT: v_accvgpr_read_b32 v5, a133 -; GISEL-NEXT: v_accvgpr_read_b32 v6, a134 -; GISEL-NEXT: v_accvgpr_read_b32 v7, a135 -; GISEL-NEXT: v_accvgpr_read_b32 v8, a136 -; GISEL-NEXT: v_accvgpr_read_b32 v9, a137 -; GISEL-NEXT: v_accvgpr_read_b32 v10, a138 -; GISEL-NEXT: v_accvgpr_read_b32 v11, a139 -; GISEL-NEXT: v_accvgpr_read_b32 v12, a140 -; GISEL-NEXT: v_accvgpr_read_b32 v13, a141 -; GISEL-NEXT: v_accvgpr_read_b32 v14, a142 -; GISEL-NEXT: v_accvgpr_read_b32 v15, a143 -; GISEL-NEXT: v_accvgpr_read_b32 v16, a144 -; GISEL-NEXT: v_accvgpr_read_b32 v17, a145 -; GISEL-NEXT: v_accvgpr_read_b32 v18, a146 -; GISEL-NEXT: v_accvgpr_read_b32 v19, a147 -; GISEL-NEXT: v_accvgpr_read_b32 v20, a148 -; GISEL-NEXT: v_accvgpr_read_b32 v21, a149 -; GISEL-NEXT: v_accvgpr_read_b32 v22, a150 -; GISEL-NEXT: v_accvgpr_read_b32 v23, a151 -; GISEL-NEXT: v_accvgpr_read_b32 v24, a152 -; GISEL-NEXT: v_accvgpr_read_b32 v25, a153 -; GISEL-NEXT: v_accvgpr_read_b32 v26, a154 -; GISEL-NEXT: v_accvgpr_read_b32 v27, a155 -; GISEL-NEXT: v_accvgpr_read_b32 v28, a156 -; GISEL-NEXT: v_accvgpr_read_b32 v29, a157 -; GISEL-NEXT: v_accvgpr_read_b32 v30, a158 -; GISEL-NEXT: v_accvgpr_read_b32 v31, a159 -; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:32768 -; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:32784 -; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:32800 -; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:32816 -; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:32832 -; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:32848 -; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:32864 -; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:32880 +; GISEL-NEXT: ds_write_b128 v96, v[32:35] offset:8192 +; GISEL-NEXT: ds_write_b128 v96, v[36:39] offset:8208 +; GISEL-NEXT: ds_write_b128 v96, v[40:43] offset:8224 +; GISEL-NEXT: ds_write_b128 v96, v[44:47] offset:8240 +; GISEL-NEXT: ds_write_b128 v96, v[48:51] offset:8256 +; GISEL-NEXT: ds_write_b128 v96, v[52:55] offset:8272 +; GISEL-NEXT: ds_write_b128 v96, v[56:59] offset:8288 +; GISEL-NEXT: ds_write_b128 v96, v[60:63] offset:8304 +; GISEL-NEXT: ds_write_b128 v96, v[64:67] offset:24576 +; GISEL-NEXT: ds_write_b128 v96, v[68:71] offset:24592 +; GISEL-NEXT: ds_write_b128 v96, v[72:75] offset:24608 +; GISEL-NEXT: ds_write_b128 v96, v[76:79] offset:24624 +; GISEL-NEXT: ds_write_b128 v96, v[80:83] offset:24640 +; GISEL-NEXT: ds_write_b128 v96, v[84:87] offset:24656 +; GISEL-NEXT: ds_write_b128 v96, v[88:91] offset:24672 +; GISEL-NEXT: ds_write_b128 v96, v[92:95] offset:24688 +; GISEL-NEXT: ds_write_b128 v96, v[0:3] offset:32768 +; GISEL-NEXT: ds_write_b128 v96, v[4:7] offset:32784 +; GISEL-NEXT: ds_write_b128 v96, v[8:11] offset:32800 +; GISEL-NEXT: ds_write_b128 v96, v[12:15] offset:32816 +; GISEL-NEXT: ds_write_b128 v96, v[16:19] offset:32832 +; GISEL-NEXT: ds_write_b128 v96, v[20:23] offset:32848 +; GISEL-NEXT: ds_write_b128 v96, v[24:27] offset:32864 +; GISEL-NEXT: ds_write_b128 v96, v[28:31] offset:32880 ; GISEL-NEXT: ; sched_group_barrier mask(0x00000008) size(5) SyncID(0) ; GISEL-NEXT: ; sched_group_barrier mask(0x00000200) size(40) SyncID(0) ; GISEL-NEXT: s_endpgm @@ -1236,23 +1236,23 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_cluster(ptr ad ; EXACTCUTOFF-NEXT: v_mov_b32_e32 v1, 2.0 ; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0) ; EXACTCUTOFF-NEXT: v_add_u32_e32 v3, s0, v0 -; EXACTCUTOFF-NEXT: ds_read_b128 a[60:63], v3 offset:112 -; EXACTCUTOFF-NEXT: ds_read_b128 a[56:59], v3 offset:96 -; EXACTCUTOFF-NEXT: ds_read_b128 a[52:55], v3 offset:80 -; EXACTCUTOFF-NEXT: ds_read_b128 a[48:51], v3 offset:64 -; EXACTCUTOFF-NEXT: ds_read_b128 a[32:35], v3 -; EXACTCUTOFF-NEXT: ds_read_b128 a[36:39], v3 offset:16 -; EXACTCUTOFF-NEXT: ds_read_b128 a[40:43], v3 offset:32 -; EXACTCUTOFF-NEXT: ds_read_b128 a[44:47], v3 offset:48 +; EXACTCUTOFF-NEXT: ds_read_b128 a[28:31], v3 offset:112 +; EXACTCUTOFF-NEXT: ds_read_b128 a[24:27], v3 offset:96 +; EXACTCUTOFF-NEXT: ds_read_b128 a[20:23], v3 offset:80 +; EXACTCUTOFF-NEXT: ds_read_b128 a[16:19], v3 offset:64 +; EXACTCUTOFF-NEXT: ds_read_b128 a[0:3], v3 +; EXACTCUTOFF-NEXT: ds_read_b128 a[4:7], v3 offset:16 +; EXACTCUTOFF-NEXT: ds_read_b128 a[8:11], v3 offset:32 +; EXACTCUTOFF-NEXT: ds_read_b128 a[12:15], v3 offset:48 ; EXACTCUTOFF-NEXT: v_add_u32_e32 v4, 0x6000, v3 -; EXACTCUTOFF-NEXT: ds_read_b128 a[28:31], v3 offset:8304 -; EXACTCUTOFF-NEXT: ds_read_b128 a[24:27], v3 offset:8288 -; EXACTCUTOFF-NEXT: ds_read_b128 a[20:23], v3 offset:8272 -; EXACTCUTOFF-NEXT: ds_read_b128 a[16:19], v3 offset:8256 -; EXACTCUTOFF-NEXT: ds_read_b128 a[12:15], v3 offset:8240 -; EXACTCUTOFF-NEXT: ds_read_b128 a[8:11], v3 offset:8224 -; EXACTCUTOFF-NEXT: ds_read_b128 a[4:7], v3 offset:8208 -; EXACTCUTOFF-NEXT: ds_read_b128 a[0:3], v3 offset:8192 +; EXACTCUTOFF-NEXT: ds_read_b128 a[60:63], v3 offset:8304 +; EXACTCUTOFF-NEXT: ds_read_b128 a[56:59], v3 offset:8288 +; EXACTCUTOFF-NEXT: ds_read_b128 a[52:55], v3 offset:8272 +; EXACTCUTOFF-NEXT: ds_read_b128 a[48:51], v3 offset:8256 +; EXACTCUTOFF-NEXT: ds_read_b128 a[44:47], v3 offset:8240 +; EXACTCUTOFF-NEXT: ds_read_b128 a[40:43], v3 offset:8224 +; EXACTCUTOFF-NEXT: ds_read_b128 a[36:39], v3 offset:8208 +; EXACTCUTOFF-NEXT: ds_read_b128 a[32:35], v3 offset:8192 ; EXACTCUTOFF-NEXT: ds_read_b128 a[92:95], v3 offset:24688 ; EXACTCUTOFF-NEXT: ds_read_b128 a[88:91], v3 offset:24672 ; EXACTCUTOFF-NEXT: ds_read_b128 a[84:87], v3 offset:24656 @@ -1278,33 +1278,33 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_MFMA_cluster(ptr ad ; EXACTCUTOFF-NEXT: ds_read_b128 a[136:139], v4 offset:57376 ; EXACTCUTOFF-NEXT: ds_read_b128 a[140:143], v4 offset:57392 ; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(14) -; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v2, v1, a[32:63] +; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v1, a[0:31] ; EXACTCUTOFF-NEXT: v_add_u32_e32 v0, s1, v0 ; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000100) size(40) SyncID(0) +; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v2, v1, a[32:63] +; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v2, v1, a[64:95] ; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(8) ; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v2, v1, a[96:127] -; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v2, v1, a[64:95] -; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v1, a[0:31] ; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0) ; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v2, v1, a[128:159] ; EXACTCUTOFF-NEXT: s_nop 11 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[60:63] offset:112 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[56:59] offset:96 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[52:55] offset:80 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[48:51] offset:64 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[44:47] offset:48 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[40:43] offset:32 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[36:39] offset:16 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[32:35] +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[28:31] offset:112 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[24:27] offset:96 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[20:23] offset:80 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[16:19] offset:64 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[12:15] offset:48 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[8:11] offset:32 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[4:7] offset:16 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[0:3] ; EXACTCUTOFF-NEXT: v_mov_b32_e32 v0, s1 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[24:27] offset:8288 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[28:31] offset:8304 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[16:19] offset:8256 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[20:23] offset:8272 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[8:11] offset:8224 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[12:15] offset:8240 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[0:3] offset:8192 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[4:7] offset:8208 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[56:59] offset:8288 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[60:63] offset:8304 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[48:51] offset:8256 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[52:55] offset:8272 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[40:43] offset:8224 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[44:47] offset:8240 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[32:35] offset:8192 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[36:39] offset:8208 ; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[88:91] offset:16480 ; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[92:95] offset:16496 ; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[80:83] offset:16448 @@ -1979,49 +1979,65 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_interleave_EXP_MFMA ; SDAG-LABEL: test_sched_group_barrier_pipeline_interleave_EXP_MFMA: ; SDAG: ; %bb.0: ; %entry ; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x44 -; SDAG-NEXT: v_mov_b32_e32 v3, 0x3fb8aa3b +; SDAG-NEXT: v_mov_b32_e32 v2, 0x3fb8aa3b ; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 -; SDAG-NEXT: v_mov_b32_e32 v7, 0x32a5705f +; SDAG-NEXT: v_mov_b32_e32 v6, 0x32a5705f ; SDAG-NEXT: v_lshlrev_b32_e32 v0, 7, v0 ; SDAG-NEXT: s_waitcnt lgkmcnt(0) -; SDAG-NEXT: v_mul_f32_e32 v4, s0, v3 -; SDAG-NEXT: v_rndne_f32_e32 v5, v4 -; SDAG-NEXT: v_sub_f32_e32 v6, v4, v5 -; SDAG-NEXT: v_fma_f32 v4, s0, v3, -v4 -; SDAG-NEXT: v_fmac_f32_e32 v4, s0, v7 -; SDAG-NEXT: v_add_f32_e32 v4, v6, v4 -; SDAG-NEXT: v_exp_f32_e32 v4, v4 -; SDAG-NEXT: v_cvt_i32_f32_e32 v5, v5 +; SDAG-NEXT: v_mul_f32_e32 v3, s0, v2 +; SDAG-NEXT: v_rndne_f32_e32 v4, v3 +; SDAG-NEXT: v_sub_f32_e32 v5, v3, v4 +; SDAG-NEXT: v_fma_f32 v3, s0, v2, -v3 +; SDAG-NEXT: v_fmac_f32_e32 v3, s0, v6 +; SDAG-NEXT: v_add_f32_e32 v3, v5, v3 +; SDAG-NEXT: v_exp_f32_e32 v3, v3 +; SDAG-NEXT: v_cvt_i32_f32_e32 v4, v4 ; SDAG-NEXT: v_and_b32_e32 v0, 0x1ff80, v0 ; SDAG-NEXT: v_add_u32_e32 v1, s6, v0 -; SDAG-NEXT: ds_read_b128 a[60:63], v1 offset:112 -; SDAG-NEXT: ds_read_b128 a[56:59], v1 offset:96 -; SDAG-NEXT: ds_read_b128 a[52:55], v1 offset:80 -; SDAG-NEXT: ds_read_b128 a[48:51], v1 offset:64 -; SDAG-NEXT: ds_read_b128 a[32:35], v1 -; SDAG-NEXT: ds_read_b128 a[36:39], v1 offset:16 -; SDAG-NEXT: ds_read_b128 a[40:43], v1 offset:32 -; SDAG-NEXT: ds_read_b128 a[44:47], v1 offset:48 -; SDAG-NEXT: v_mov_b32_e32 v9, 1.0 -; SDAG-NEXT: v_ldexp_f32 v4, v4, v5 -; SDAG-NEXT: v_mov_b32_e32 v5, 0xc2ce8ed0 -; SDAG-NEXT: v_mul_f32_e32 v10, s1, v3 -; SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s0, v5 -; SDAG-NEXT: v_mov_b32_e32 v6, 0x42b17218 -; SDAG-NEXT: v_rndne_f32_e32 v11, v10 -; SDAG-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc -; SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s0, v6 +; SDAG-NEXT: ds_read_b128 a[28:31], v1 offset:112 +; SDAG-NEXT: ds_read_b128 a[24:27], v1 offset:96 +; SDAG-NEXT: ds_read_b128 a[20:23], v1 offset:80 +; SDAG-NEXT: ds_read_b128 a[16:19], v1 offset:64 +; SDAG-NEXT: ds_read_b128 a[0:3], v1 +; SDAG-NEXT: ds_read_b128 a[4:7], v1 offset:16 +; SDAG-NEXT: ds_read_b128 a[8:11], v1 offset:32 +; SDAG-NEXT: ds_read_b128 a[12:15], v1 offset:48 +; SDAG-NEXT: v_mov_b32_e32 v5, 1.0 +; SDAG-NEXT: v_ldexp_f32 v3, v3, v4 +; SDAG-NEXT: v_mov_b32_e32 v4, 0xc2ce8ed0 +; SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s0, v4 +; SDAG-NEXT: v_mov_b32_e32 v7, 0x42b17218 +; SDAG-NEXT: v_cndmask_b32_e32 v3, 0, v3, vcc +; SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s0, v7 ; SDAG-NEXT: v_mov_b32_e32 v8, 0x7f800000 -; SDAG-NEXT: v_sub_f32_e32 v12, v10, v11 -; SDAG-NEXT: v_fma_f32 v10, s1, v3, -v10 -; SDAG-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc -; SDAG-NEXT: v_fmac_f32_e32 v10, s1, v7 +; SDAG-NEXT: v_cndmask_b32_e32 v3, v8, v3, vcc +; SDAG-NEXT: v_add_u32_e32 v0, s7, v0 +; SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s1, v4 +; SDAG-NEXT: s_waitcnt lgkmcnt(0) +; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v5, v3, a[0:31] +; SDAG-NEXT: s_load_dword s8, s[4:5], 0x54 +; SDAG-NEXT: ; kill: killed $sgpr4_sgpr5 +; SDAG-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0) +; SDAG-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0) +; SDAG-NEXT: s_nop 15 +; SDAG-NEXT: s_nop 1 +; SDAG-NEXT: ds_write_b128 v0, a[28:31] offset:112 +; SDAG-NEXT: ds_write_b128 v0, a[24:27] offset:96 +; SDAG-NEXT: ds_write_b128 v0, a[20:23] offset:80 +; SDAG-NEXT: ds_write_b128 v0, a[16:19] offset:64 +; SDAG-NEXT: ds_write_b128 v0, a[12:15] offset:48 +; SDAG-NEXT: ds_write_b128 v0, a[8:11] offset:32 +; SDAG-NEXT: ds_write_b128 v0, a[4:7] offset:16 +; SDAG-NEXT: ds_write_b128 v0, a[0:3] +; SDAG-NEXT: v_mul_f32_e32 v0, s1, v2 +; SDAG-NEXT: v_rndne_f32_e32 v3, v0 +; SDAG-NEXT: v_sub_f32_e32 v9, v0, v3 +; SDAG-NEXT: v_fma_f32 v0, s1, v2, -v0 +; SDAG-NEXT: v_fmac_f32_e32 v0, s1, v6 +; SDAG-NEXT: v_add_f32_e32 v0, v9, v0 +; SDAG-NEXT: v_exp_f32_e32 v0, v0 +; SDAG-NEXT: v_cvt_i32_f32_e32 v3, v3 ; SDAG-NEXT: ds_read_b128 a[28:31], v1 offset:8304 -; SDAG-NEXT: s_waitcnt lgkmcnt(1) -; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v9, v4, a[32:63] -; SDAG-NEXT: v_add_f32_e32 v4, v12, v10 -; SDAG-NEXT: v_exp_f32_e32 v4, v4 -; SDAG-NEXT: v_cvt_i32_f32_e32 v10, v11 ; SDAG-NEXT: ds_read_b128 a[24:27], v1 offset:8288 ; SDAG-NEXT: ds_read_b128 a[20:23], v1 offset:8272 ; SDAG-NEXT: ds_read_b128 a[16:19], v1 offset:8256 @@ -2029,104 +2045,32 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_interleave_EXP_MFMA ; SDAG-NEXT: ds_read_b128 a[8:11], v1 offset:8224 ; SDAG-NEXT: ds_read_b128 a[4:7], v1 offset:8208 ; SDAG-NEXT: ds_read_b128 a[0:3], v1 offset:8192 -; SDAG-NEXT: v_ldexp_f32 v4, v4, v10 -; SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s1, v5 -; SDAG-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc -; SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s1, v6 -; SDAG-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc -; SDAG-NEXT: v_mul_f32_e32 v10, s2, v3 -; SDAG-NEXT: v_rndne_f32_e32 v11, v10 -; SDAG-NEXT: s_waitcnt lgkmcnt(0) -; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v9, v4, a[0:31] -; SDAG-NEXT: v_fma_f32 v4, s2, v3, -v10 -; SDAG-NEXT: v_sub_f32_e32 v12, v10, v11 -; SDAG-NEXT: v_fmac_f32_e32 v4, s2, v7 -; SDAG-NEXT: v_add_f32_e32 v4, v12, v4 -; SDAG-NEXT: v_exp_f32_e32 v4, v4 -; SDAG-NEXT: v_cvt_i32_f32_e32 v10, v11 -; SDAG-NEXT: ds_read_b128 a[92:95], v1 offset:24688 -; SDAG-NEXT: ds_read_b128 a[88:91], v1 offset:24672 -; SDAG-NEXT: ds_read_b128 a[84:87], v1 offset:24656 -; SDAG-NEXT: ds_read_b128 a[80:83], v1 offset:24640 -; SDAG-NEXT: ds_read_b128 a[76:79], v1 offset:24624 -; SDAG-NEXT: ds_read_b128 a[72:75], v1 offset:24608 -; SDAG-NEXT: ds_read_b128 a[68:71], v1 offset:24592 -; SDAG-NEXT: ds_read_b128 a[64:67], v1 offset:24576 -; SDAG-NEXT: v_add_u32_e32 v2, 0x6000, v1 -; SDAG-NEXT: ds_read_b128 a[124:127], v1 offset:49264 -; SDAG-NEXT: ds_read_b128 a[120:123], v1 offset:49248 -; SDAG-NEXT: ds_read_b128 a[116:119], v1 offset:49232 -; SDAG-NEXT: ds_read_b128 a[112:115], v1 offset:49216 -; SDAG-NEXT: ds_read_b128 a[108:111], v1 offset:49200 -; SDAG-NEXT: ds_read_b128 a[104:107], v1 offset:49184 -; SDAG-NEXT: ds_read_b128 a[100:103], v1 offset:49168 -; SDAG-NEXT: ds_read_b128 a[96:99], v1 offset:49152 -; SDAG-NEXT: v_ldexp_f32 v1, v4, v10 -; SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s2, v5 -; SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc -; SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s2, v6 -; SDAG-NEXT: v_mul_f32_e32 v4, s3, v3 -; SDAG-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc -; SDAG-NEXT: v_rndne_f32_e32 v10, v4 -; SDAG-NEXT: s_load_dword s8, s[4:5], 0x54 +; SDAG-NEXT: v_ldexp_f32 v0, v0, v3 +; SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc +; SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s1, v7 +; SDAG-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc +; SDAG-NEXT: v_mul_f32_e32 v3, s2, v2 +; SDAG-NEXT: v_rndne_f32_e32 v10, v3 ; SDAG-NEXT: s_waitcnt lgkmcnt(0) -; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v9, v1, a[64:95] -; SDAG-NEXT: v_sub_f32_e32 v1, v4, v10 -; SDAG-NEXT: v_fma_f32 v4, s3, v3, -v4 -; SDAG-NEXT: v_fmac_f32_e32 v4, s3, v7 -; SDAG-NEXT: v_add_f32_e32 v1, v1, v4 -; SDAG-NEXT: v_exp_f32_e32 v1, v1 -; SDAG-NEXT: v_cvt_i32_f32_e32 v4, v10 -; SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s3, v5 -; SDAG-NEXT: ds_read_b128 a[156:159], v2 offset:57456 -; SDAG-NEXT: ds_read_b128 a[152:155], v2 offset:57440 -; SDAG-NEXT: v_ldexp_f32 v1, v1, v4 -; SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc -; SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s3, v6 -; SDAG-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc -; SDAG-NEXT: v_mul_f32_e32 v4, s8, v3 -; SDAG-NEXT: v_fma_f32 v3, s8, v3, -v4 -; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v9, v1, a[96:127] -; SDAG-NEXT: v_rndne_f32_e32 v1, v4 -; SDAG-NEXT: v_sub_f32_e32 v10, v4, v1 -; SDAG-NEXT: v_fmac_f32_e32 v3, s8, v7 -; SDAG-NEXT: v_add_f32_e32 v3, v10, v3 +; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v5, v0, a[0:31] +; SDAG-NEXT: v_sub_f32_e32 v11, v3, v10 +; SDAG-NEXT: v_fma_f32 v3, s2, v2, -v3 +; SDAG-NEXT: v_fmac_f32_e32 v3, s2, v6 +; SDAG-NEXT: v_add_f32_e32 v3, v11, v3 ; SDAG-NEXT: v_exp_f32_e32 v3, v3 -; SDAG-NEXT: v_cvt_i32_f32_e32 v1, v1 -; SDAG-NEXT: ds_read_b128 a[148:151], v2 offset:57424 -; SDAG-NEXT: ds_read_b128 a[144:147], v2 offset:57408 -; SDAG-NEXT: ds_read_b128 a[128:131], v2 offset:57344 -; SDAG-NEXT: ds_read_b128 a[132:135], v2 offset:57360 -; SDAG-NEXT: ds_read_b128 a[136:139], v2 offset:57376 -; SDAG-NEXT: ds_read_b128 a[140:143], v2 offset:57392 -; SDAG-NEXT: v_ldexp_f32 v1, v3, v1 -; SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s8, v5 -; SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc -; SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s8, v6 -; SDAG-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc -; SDAG-NEXT: v_add_u32_e32 v0, s7, v0 -; SDAG-NEXT: ds_write_b128 v0, a[60:63] offset:112 -; SDAG-NEXT: s_waitcnt lgkmcnt(1) -; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v9, v1, a[128:159] -; SDAG-NEXT: ds_write_b128 v0, a[56:59] offset:96 -; SDAG-NEXT: ds_write_b128 v0, a[52:55] offset:80 -; SDAG-NEXT: ds_write_b128 v0, a[48:51] offset:64 -; SDAG-NEXT: ds_write_b128 v0, a[44:47] offset:48 -; SDAG-NEXT: ds_write_b128 v0, a[40:43] offset:32 -; SDAG-NEXT: ds_write_b128 v0, a[36:39] offset:16 -; SDAG-NEXT: ds_write_b128 v0, a[32:35] +; SDAG-NEXT: v_cvt_i32_f32_e32 v10, v10 ; SDAG-NEXT: v_mov_b32_e32 v0, s7 -; SDAG-NEXT: ; kill: killed $sgpr4_sgpr5 -; SDAG-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0) -; SDAG-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0) -; SDAG-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0) -; SDAG-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0) -; SDAG-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0) -; SDAG-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0) +; SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s2, v4 +; SDAG-NEXT: v_add_u32_e32 v9, 0x6000, v1 +; SDAG-NEXT: v_ldexp_f32 v3, v3, v10 +; SDAG-NEXT: v_cndmask_b32_e32 v3, 0, v3, vcc +; SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s2, v7 +; SDAG-NEXT: v_cndmask_b32_e32 v3, v8, v3, vcc +; SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s3, v4 ; SDAG-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0) ; SDAG-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0) ; SDAG-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0) -; SDAG-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0) +; SDAG-NEXT: s_nop 4 ; SDAG-NEXT: ds_write_b128 v0, a[24:27] offset:8288 ; SDAG-NEXT: ds_write_b128 v0, a[28:31] offset:8304 ; SDAG-NEXT: ds_write_b128 v0, a[16:19] offset:8256 @@ -2135,216 +2079,202 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_interleave_EXP_MFMA ; SDAG-NEXT: ds_write_b128 v0, a[12:15] offset:8240 ; SDAG-NEXT: ds_write_b128 v0, a[0:3] offset:8192 ; SDAG-NEXT: ds_write_b128 v0, a[4:7] offset:8208 -; SDAG-NEXT: ds_write_b128 v0, a[88:91] offset:16480 -; SDAG-NEXT: ds_write_b128 v0, a[92:95] offset:16496 -; SDAG-NEXT: ds_write_b128 v0, a[80:83] offset:16448 -; SDAG-NEXT: ds_write_b128 v0, a[84:87] offset:16464 -; SDAG-NEXT: ds_write_b128 v0, a[72:75] offset:16416 -; SDAG-NEXT: ds_write_b128 v0, a[76:79] offset:16432 -; SDAG-NEXT: ds_write_b128 v0, a[64:67] offset:16384 -; SDAG-NEXT: ds_write_b128 v0, a[68:71] offset:16400 -; SDAG-NEXT: ds_write_b128 v0, a[120:123] offset:24672 -; SDAG-NEXT: ds_write_b128 v0, a[124:127] offset:24688 -; SDAG-NEXT: ds_write_b128 v0, a[112:115] offset:24640 -; SDAG-NEXT: ds_write_b128 v0, a[116:119] offset:24656 -; SDAG-NEXT: ds_write_b128 v0, a[104:107] offset:24608 -; SDAG-NEXT: ds_write_b128 v0, a[108:111] offset:24624 -; SDAG-NEXT: ds_write_b128 v0, a[96:99] offset:24576 -; SDAG-NEXT: ds_write_b128 v0, a[100:103] offset:24592 -; SDAG-NEXT: ds_write_b128 v0, a[152:155] offset:32864 -; SDAG-NEXT: ds_write_b128 v0, a[156:159] offset:32880 -; SDAG-NEXT: ds_write_b128 v0, a[144:147] offset:32832 -; SDAG-NEXT: ds_write_b128 v0, a[148:151] offset:32848 -; SDAG-NEXT: ds_write_b128 v0, a[136:139] offset:32800 -; SDAG-NEXT: ds_write_b128 v0, a[140:143] offset:32816 -; SDAG-NEXT: ds_write_b128 v0, a[128:131] offset:32768 -; SDAG-NEXT: ds_write_b128 v0, a[132:135] offset:32784 -; SDAG-NEXT: s_endpgm -; -; GISEL-LABEL: test_sched_group_barrier_pipeline_interleave_EXP_MFMA: -; GISEL: ; %bb.0: ; %entry +; SDAG-NEXT: ds_read_b128 a[28:31], v1 offset:24688 +; SDAG-NEXT: ds_read_b128 a[24:27], v1 offset:24672 +; SDAG-NEXT: ds_read_b128 a[20:23], v1 offset:24656 +; SDAG-NEXT: ds_read_b128 a[16:19], v1 offset:24640 +; SDAG-NEXT: ds_read_b128 a[12:15], v1 offset:24624 +; SDAG-NEXT: ds_read_b128 a[8:11], v1 offset:24608 +; SDAG-NEXT: ds_read_b128 a[4:7], v1 offset:24592 +; SDAG-NEXT: ds_read_b128 a[0:3], v1 offset:24576 +; SDAG-NEXT: s_waitcnt lgkmcnt(0) +; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v5, v3, a[0:31] +; SDAG-NEXT: v_mul_f32_e32 v3, s3, v2 +; SDAG-NEXT: v_rndne_f32_e32 v10, v3 +; SDAG-NEXT: v_sub_f32_e32 v11, v3, v10 +; SDAG-NEXT: v_fma_f32 v3, s3, v2, -v3 +; SDAG-NEXT: v_fmac_f32_e32 v3, s3, v6 +; SDAG-NEXT: v_add_f32_e32 v3, v11, v3 +; SDAG-NEXT: v_exp_f32_e32 v3, v3 +; SDAG-NEXT: v_cvt_i32_f32_e32 v10, v10 +; SDAG-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0) +; SDAG-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0) +; SDAG-NEXT: s_nop 10 +; SDAG-NEXT: ds_write_b128 v0, a[24:27] offset:16480 +; SDAG-NEXT: ds_write_b128 v0, a[28:31] offset:16496 +; SDAG-NEXT: ds_write_b128 v0, a[16:19] offset:16448 +; SDAG-NEXT: ds_write_b128 v0, a[20:23] offset:16464 +; SDAG-NEXT: ds_write_b128 v0, a[8:11] offset:16416 +; SDAG-NEXT: ds_write_b128 v0, a[12:15] offset:16432 +; SDAG-NEXT: ds_write_b128 v0, a[0:3] offset:16384 +; SDAG-NEXT: ds_write_b128 v0, a[4:7] offset:16400 +; SDAG-NEXT: ds_read_b128 a[28:31], v1 offset:49264 +; SDAG-NEXT: ds_read_b128 a[24:27], v1 offset:49248 +; SDAG-NEXT: ds_read_b128 a[20:23], v1 offset:49232 +; SDAG-NEXT: ds_read_b128 a[16:19], v1 offset:49216 +; SDAG-NEXT: ds_read_b128 a[12:15], v1 offset:49200 +; SDAG-NEXT: ds_read_b128 a[8:11], v1 offset:49184 +; SDAG-NEXT: ds_read_b128 a[4:7], v1 offset:49168 +; SDAG-NEXT: ds_read_b128 a[0:3], v1 offset:49152 +; SDAG-NEXT: v_ldexp_f32 v1, v3, v10 +; SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc +; SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s3, v7 +; SDAG-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc +; SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s8, v4 +; SDAG-NEXT: s_waitcnt lgkmcnt(0) +; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v5, v1, a[0:31] +; SDAG-NEXT: v_mul_f32_e32 v1, s8, v2 +; SDAG-NEXT: v_rndne_f32_e32 v3, v1 +; SDAG-NEXT: v_sub_f32_e32 v10, v1, v3 +; SDAG-NEXT: v_fma_f32 v1, s8, v2, -v1 +; SDAG-NEXT: v_fmac_f32_e32 v1, s8, v6 +; SDAG-NEXT: v_add_f32_e32 v1, v10, v1 +; SDAG-NEXT: v_exp_f32_e32 v1, v1 +; SDAG-NEXT: v_cvt_i32_f32_e32 v2, v3 +; SDAG-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0) +; SDAG-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0) +; SDAG-NEXT: v_ldexp_f32 v1, v1, v2 +; SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc +; SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s8, v7 +; SDAG-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc +; SDAG-NEXT: s_nop 6 +; SDAG-NEXT: ds_write_b128 v0, a[24:27] offset:24672 +; SDAG-NEXT: ds_write_b128 v0, a[28:31] offset:24688 +; SDAG-NEXT: ds_write_b128 v0, a[16:19] offset:24640 +; SDAG-NEXT: ds_write_b128 v0, a[20:23] offset:24656 +; SDAG-NEXT: ds_write_b128 v0, a[8:11] offset:24608 +; SDAG-NEXT: ds_write_b128 v0, a[12:15] offset:24624 +; SDAG-NEXT: ds_write_b128 v0, a[0:3] offset:24576 +; SDAG-NEXT: ds_write_b128 v0, a[4:7] offset:24592 +; SDAG-NEXT: ds_read_b128 a[28:31], v9 offset:57456 +; SDAG-NEXT: ds_read_b128 a[24:27], v9 offset:57440 +; SDAG-NEXT: ds_read_b128 a[20:23], v9 offset:57424 +; SDAG-NEXT: ds_read_b128 a[16:19], v9 offset:57408 +; SDAG-NEXT: ds_read_b128 a[0:3], v9 offset:57344 +; SDAG-NEXT: ds_read_b128 a[4:7], v9 offset:57360 +; SDAG-NEXT: ds_read_b128 a[8:11], v9 offset:57376 +; SDAG-NEXT: ds_read_b128 a[12:15], v9 offset:57392 +; SDAG-NEXT: s_waitcnt lgkmcnt(0) +; SDAG-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v5, v1, a[0:31] +; SDAG-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0) +; SDAG-NEXT: s_nop 15 +; SDAG-NEXT: s_nop 2 +; SDAG-NEXT: ds_write_b128 v0, a[24:27] offset:32864 +; SDAG-NEXT: ds_write_b128 v0, a[28:31] offset:32880 +; SDAG-NEXT: ds_write_b128 v0, a[16:19] offset:32832 +; SDAG-NEXT: ds_write_b128 v0, a[20:23] offset:32848 +; SDAG-NEXT: ds_write_b128 v0, a[8:11] offset:32800 +; SDAG-NEXT: ds_write_b128 v0, a[12:15] offset:32816 +; SDAG-NEXT: ds_write_b128 v0, a[0:3] offset:32768 +; SDAG-NEXT: ds_write_b128 v0, a[4:7] offset:32784 +; SDAG-NEXT: s_endpgm +; +; GISEL-LABEL: test_sched_group_barrier_pipeline_interleave_EXP_MFMA: +; GISEL: ; %bb.0: ; %entry ; GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x44 -; GISEL-NEXT: v_mov_b32_e32 v2, 0x3fb8aa3b -; GISEL-NEXT: v_mov_b32_e32 v5, 0x32a5705f -; GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0 -; GISEL-NEXT: s_waitcnt lgkmcnt(0) -; GISEL-NEXT: v_mul_f32_e32 v3, s0, v2 -; GISEL-NEXT: v_fma_f32 v4, s0, v2, -v3 -; GISEL-NEXT: v_rndne_f32_e32 v6, v3 -; GISEL-NEXT: v_fmac_f32_e32 v4, s0, v5 -; GISEL-NEXT: v_sub_f32_e32 v3, v3, v6 -; GISEL-NEXT: v_add_f32_e32 v3, v3, v4 -; GISEL-NEXT: v_cvt_i32_f32_e32 v4, v6 -; GISEL-NEXT: v_exp_f32_e32 v3, v3 +; GISEL-NEXT: v_mov_b32_e32 v34, 0x3fb8aa3b ; GISEL-NEXT: v_lshlrev_b32_e32 v32, 7, v0 +; GISEL-NEXT: v_mov_b32_e32 v35, 0x32a5705f +; GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GISEL-NEXT: v_mul_f32_e32 v0, s0, v34 +; GISEL-NEXT: v_fma_f32 v1, s0, v34, -v0 +; GISEL-NEXT: v_rndne_f32_e32 v2, v0 +; GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 +; GISEL-NEXT: v_fmac_f32_e32 v1, s0, v35 +; GISEL-NEXT: v_sub_f32_e32 v0, v0, v2 +; GISEL-NEXT: v_add_f32_e32 v0, v0, v1 +; GISEL-NEXT: v_cvt_i32_f32_e32 v1, v2 +; GISEL-NEXT: v_exp_f32_e32 v0, v0 +; GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GISEL-NEXT: v_add_u32_e32 v33, s6, v32 +; GISEL-NEXT: v_mov_b32_e32 v36, 0xc2ce8ed0 +; GISEL-NEXT: ds_read_b128 a[0:3], v33 +; GISEL-NEXT: ds_read_b128 a[4:7], v33 offset:16 +; GISEL-NEXT: ds_read_b128 a[8:11], v33 offset:32 +; GISEL-NEXT: ds_read_b128 a[12:15], v33 offset:48 +; GISEL-NEXT: ds_read_b128 a[16:19], v33 offset:64 +; GISEL-NEXT: ds_read_b128 a[20:23], v33 offset:80 +; GISEL-NEXT: ds_read_b128 a[24:27], v33 offset:96 +; GISEL-NEXT: ds_read_b128 a[28:31], v33 offset:112 +; GISEL-NEXT: v_ldexp_f32 v0, v0, v1 +; GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v36 +; GISEL-NEXT: v_mov_b32_e32 v37, 0x42b17218 +; GISEL-NEXT: v_mov_b32_e32 v38, 1.0 ; GISEL-NEXT: s_load_dword s8, s[4:5], 0x54 ; GISEL-NEXT: ; kill: killed $sgpr4_sgpr5 -; GISEL-NEXT: v_add_u32_e32 v0, s6, v32 -; GISEL-NEXT: v_ldexp_f32 v3, v3, v4 -; GISEL-NEXT: v_readfirstlane_b32 s4, v3 -; GISEL-NEXT: v_mov_b32_e32 v3, 0xc2ce8ed0 -; GISEL-NEXT: ds_read_b128 a[64:67], v0 -; GISEL-NEXT: ds_read_b128 a[68:71], v0 offset:16 -; GISEL-NEXT: ds_read_b128 a[72:75], v0 offset:32 -; GISEL-NEXT: ds_read_b128 a[76:79], v0 offset:48 -; GISEL-NEXT: ds_read_b128 a[80:83], v0 offset:64 -; GISEL-NEXT: ds_read_b128 a[84:87], v0 offset:80 -; GISEL-NEXT: ds_read_b128 a[88:91], v0 offset:96 -; GISEL-NEXT: ds_read_b128 a[92:95], v0 offset:112 -; GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s0, v3 -; GISEL-NEXT: v_mov_b32_e32 v4, 0x42b17218 -; GISEL-NEXT: v_mov_b32_e32 v33, 1.0 +; GISEL-NEXT: v_readfirstlane_b32 s4, v0 ; GISEL-NEXT: s_cmp_lg_u64 vcc, 0 -; GISEL-NEXT: v_cmp_gt_f32_e32 vcc, s0, v4 -; GISEL-NEXT: v_mul_f32_e32 v6, s1, v2 +; GISEL-NEXT: v_cmp_gt_f32_e32 vcc, s0, v37 ; GISEL-NEXT: s_cselect_b32 s4, 0, s4 ; GISEL-NEXT: s_cmp_lg_u64 vcc, 0 -; GISEL-NEXT: v_fma_f32 v7, s1, v2, -v6 -; GISEL-NEXT: v_rndne_f32_e32 v8, v6 ; GISEL-NEXT: s_cselect_b32 s0, 0x7f800000, s4 -; GISEL-NEXT: v_fmac_f32_e32 v7, s1, v5 -; GISEL-NEXT: v_sub_f32_e32 v6, v6, v8 -; GISEL-NEXT: v_add_f32_e32 v6, v6, v7 -; GISEL-NEXT: v_cvt_i32_f32_e32 v7, v8 -; GISEL-NEXT: v_mov_b32_e32 v8, s0 -; GISEL-NEXT: ds_read_b128 a[0:3], v0 offset:8192 -; GISEL-NEXT: ds_read_b128 a[4:7], v0 offset:8208 +; GISEL-NEXT: v_mov_b32_e32 v0, s0 +; GISEL-NEXT: v_mul_f32_e32 v39, s1, v34 +; GISEL-NEXT: v_fma_f32 v40, s1, v34, -v39 ; GISEL-NEXT: s_waitcnt lgkmcnt(0) -; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v33, v8, a[64:95] -; GISEL-NEXT: v_exp_f32_e32 v6, v6 -; GISEL-NEXT: ds_read_b128 a[8:11], v0 offset:8224 -; GISEL-NEXT: ds_read_b128 a[12:15], v0 offset:8240 -; GISEL-NEXT: ds_read_b128 a[16:19], v0 offset:8256 -; GISEL-NEXT: ds_read_b128 a[20:23], v0 offset:8272 -; GISEL-NEXT: ds_read_b128 a[24:27], v0 offset:8288 -; GISEL-NEXT: ds_read_b128 a[28:31], v0 offset:8304 -; GISEL-NEXT: v_ldexp_f32 v6, v6, v7 -; GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s1, v3 -; GISEL-NEXT: v_readfirstlane_b32 s0, v6 +; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v38, v0, a[0:31] +; GISEL-NEXT: v_rndne_f32_e32 v41, v39 +; GISEL-NEXT: v_fmac_f32_e32 v40, s1, v35 +; GISEL-NEXT: v_sub_f32_e32 v39, v39, v41 +; GISEL-NEXT: v_add_f32_e32 v39, v39, v40 +; GISEL-NEXT: v_cvt_i32_f32_e32 v40, v41 +; GISEL-NEXT: v_exp_f32_e32 v39, v39 +; GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s1, v36 ; GISEL-NEXT: s_cmp_lg_u64 vcc, 0 -; GISEL-NEXT: v_cmp_gt_f32_e32 vcc, s1, v4 -; GISEL-NEXT: v_mul_f32_e32 v6, s2, v2 +; GISEL-NEXT: v_cmp_gt_f32_e32 vcc, s1, v37 +; GISEL-NEXT: v_ldexp_f32 v39, v39, v40 +; GISEL-NEXT: v_readfirstlane_b32 s0, v39 ; GISEL-NEXT: s_cselect_b32 s0, 0, s0 ; GISEL-NEXT: s_cmp_lg_u64 vcc, 0 -; GISEL-NEXT: v_fma_f32 v7, s2, v2, -v6 -; GISEL-NEXT: v_rndne_f32_e32 v8, v6 ; GISEL-NEXT: s_cselect_b32 s0, 0x7f800000, s0 -; GISEL-NEXT: v_fmac_f32_e32 v7, s2, v5 -; GISEL-NEXT: v_sub_f32_e32 v6, v6, v8 -; GISEL-NEXT: v_add_f32_e32 v6, v6, v7 -; GISEL-NEXT: v_cvt_i32_f32_e32 v7, v8 -; GISEL-NEXT: v_mov_b32_e32 v8, s0 -; GISEL-NEXT: ds_read_b128 a[32:35], v0 offset:24576 -; GISEL-NEXT: ds_read_b128 a[36:39], v0 offset:24592 -; GISEL-NEXT: s_waitcnt lgkmcnt(2) -; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v33, v8, a[0:31] -; GISEL-NEXT: v_exp_f32_e32 v6, v6 -; GISEL-NEXT: ds_read_b128 a[40:43], v0 offset:24608 -; GISEL-NEXT: ds_read_b128 a[44:47], v0 offset:24624 -; GISEL-NEXT: ds_read_b128 a[48:51], v0 offset:24640 -; GISEL-NEXT: ds_read_b128 a[52:55], v0 offset:24656 -; GISEL-NEXT: ds_read_b128 a[56:59], v0 offset:24672 -; GISEL-NEXT: ds_read_b128 a[60:63], v0 offset:24688 -; GISEL-NEXT: v_add_u32_e32 v1, 0xc000, v0 -; GISEL-NEXT: ds_read_b128 a[96:99], v0 offset:49152 -; GISEL-NEXT: ds_read_b128 a[100:103], v0 offset:49168 -; GISEL-NEXT: ds_read_b128 a[104:107], v0 offset:49184 -; GISEL-NEXT: ds_read_b128 a[108:111], v0 offset:49200 -; GISEL-NEXT: ds_read_b128 a[112:115], v0 offset:49216 -; GISEL-NEXT: ds_read_b128 a[116:119], v0 offset:49232 -; GISEL-NEXT: ds_read_b128 a[120:123], v0 offset:49248 -; GISEL-NEXT: ds_read_b128 a[124:127], v0 offset:49264 -; GISEL-NEXT: v_ldexp_f32 v0, v6, v7 -; GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v3 -; GISEL-NEXT: v_readfirstlane_b32 s0, v0 -; GISEL-NEXT: s_cmp_lg_u64 vcc, 0 -; GISEL-NEXT: v_cmp_gt_f32_e32 vcc, s2, v4 -; GISEL-NEXT: v_mul_f32_e32 v0, s3, v2 -; GISEL-NEXT: s_cselect_b32 s0, 0, s0 -; GISEL-NEXT: s_cmp_lg_u64 vcc, 0 -; GISEL-NEXT: v_fma_f32 v6, s3, v2, -v0 -; GISEL-NEXT: v_rndne_f32_e32 v7, v0 -; GISEL-NEXT: s_cselect_b32 s0, 0x7f800000, s0 -; GISEL-NEXT: v_fmac_f32_e32 v6, s3, v5 -; GISEL-NEXT: v_sub_f32_e32 v0, v0, v7 -; GISEL-NEXT: v_add_f32_e32 v0, v0, v6 -; GISEL-NEXT: v_cvt_i32_f32_e32 v6, v7 -; GISEL-NEXT: v_mov_b32_e32 v7, s0 -; GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s3, v3 -; GISEL-NEXT: s_cmp_lg_u64 vcc, 0 -; GISEL-NEXT: s_waitcnt lgkmcnt(8) -; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v33, v7, a[32:63] -; GISEL-NEXT: v_exp_f32_e32 v0, v0 -; GISEL-NEXT: v_cmp_gt_f32_e32 vcc, s3, v4 -; GISEL-NEXT: ds_read_b128 a[128:131], v1 offset:32768 -; GISEL-NEXT: ds_read_b128 a[132:135], v1 offset:32784 -; GISEL-NEXT: v_ldexp_f32 v0, v0, v6 -; GISEL-NEXT: v_readfirstlane_b32 s0, v0 -; GISEL-NEXT: v_mul_f32_e32 v0, s8, v2 -; GISEL-NEXT: v_fma_f32 v2, s8, v2, -v0 -; GISEL-NEXT: s_cselect_b32 s0, 0, s0 -; GISEL-NEXT: s_cmp_lg_u64 vcc, 0 -; GISEL-NEXT: v_fmac_f32_e32 v2, s8, v5 -; GISEL-NEXT: v_rndne_f32_e32 v5, v0 -; GISEL-NEXT: s_cselect_b32 s0, 0x7f800000, s0 -; GISEL-NEXT: v_sub_f32_e32 v0, v0, v5 -; GISEL-NEXT: v_add_f32_e32 v0, v0, v2 -; GISEL-NEXT: v_cvt_i32_f32_e32 v2, v5 -; GISEL-NEXT: v_mov_b32_e32 v5, s0 -; GISEL-NEXT: ds_read_b128 a[136:139], v1 offset:32800 -; GISEL-NEXT: ds_read_b128 a[140:143], v1 offset:32816 -; GISEL-NEXT: s_waitcnt lgkmcnt(4) -; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v33, v5, a[96:127] -; GISEL-NEXT: v_exp_f32_e32 v0, v0 -; GISEL-NEXT: ds_read_b128 a[144:147], v1 offset:32832 -; GISEL-NEXT: ds_read_b128 a[148:151], v1 offset:32848 -; GISEL-NEXT: ds_read_b128 a[152:155], v1 offset:32864 -; GISEL-NEXT: ds_read_b128 a[156:159], v1 offset:32880 -; GISEL-NEXT: v_ldexp_f32 v0, v0, v2 -; GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s8, v3 -; GISEL-NEXT: v_readfirstlane_b32 s0, v0 -; GISEL-NEXT: s_cmp_lg_u64 vcc, 0 -; GISEL-NEXT: v_cmp_gt_f32_e32 vcc, s8, v4 -; GISEL-NEXT: s_cselect_b32 s0, 0, s0 +; GISEL-NEXT: v_mov_b32_e32 v39, s0 +; GISEL-NEXT: v_add_u32_e32 v32, s7, v32 +; GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v36 ; GISEL-NEXT: s_cmp_lg_u64 vcc, 0 -; GISEL-NEXT: s_cselect_b32 s0, 0x7f800000, s0 -; GISEL-NEXT: v_mov_b32_e32 v34, s0 -; GISEL-NEXT: v_accvgpr_read_b32 v0, a64 -; GISEL-NEXT: v_accvgpr_read_b32 v1, a65 +; GISEL-NEXT: v_cmp_gt_f32_e32 vcc, s2, v37 +; GISEL-NEXT: v_accvgpr_read_b32 v0, a0 +; GISEL-NEXT: v_accvgpr_read_b32 v1, a1 +; GISEL-NEXT: v_accvgpr_read_b32 v2, a2 +; GISEL-NEXT: v_accvgpr_read_b32 v3, a3 +; GISEL-NEXT: v_accvgpr_read_b32 v4, a4 +; GISEL-NEXT: v_accvgpr_read_b32 v5, a5 +; GISEL-NEXT: v_accvgpr_read_b32 v6, a6 +; GISEL-NEXT: v_accvgpr_read_b32 v7, a7 +; GISEL-NEXT: v_accvgpr_read_b32 v8, a8 +; GISEL-NEXT: v_accvgpr_read_b32 v9, a9 +; GISEL-NEXT: v_accvgpr_read_b32 v10, a10 +; GISEL-NEXT: v_accvgpr_read_b32 v11, a11 +; GISEL-NEXT: v_accvgpr_read_b32 v12, a12 +; GISEL-NEXT: v_accvgpr_read_b32 v13, a13 +; GISEL-NEXT: v_accvgpr_read_b32 v14, a14 +; GISEL-NEXT: v_accvgpr_read_b32 v15, a15 +; GISEL-NEXT: v_accvgpr_read_b32 v16, a16 +; GISEL-NEXT: v_accvgpr_read_b32 v17, a17 +; GISEL-NEXT: v_accvgpr_read_b32 v18, a18 +; GISEL-NEXT: v_accvgpr_read_b32 v19, a19 +; GISEL-NEXT: v_accvgpr_read_b32 v20, a20 +; GISEL-NEXT: v_accvgpr_read_b32 v21, a21 +; GISEL-NEXT: v_accvgpr_read_b32 v22, a22 +; GISEL-NEXT: v_accvgpr_read_b32 v23, a23 +; GISEL-NEXT: v_accvgpr_read_b32 v24, a24 +; GISEL-NEXT: v_accvgpr_read_b32 v25, a25 +; GISEL-NEXT: v_accvgpr_read_b32 v26, a26 +; GISEL-NEXT: v_accvgpr_read_b32 v27, a27 +; GISEL-NEXT: v_accvgpr_read_b32 v28, a28 +; GISEL-NEXT: v_accvgpr_read_b32 v29, a29 +; GISEL-NEXT: v_accvgpr_read_b32 v30, a30 +; GISEL-NEXT: v_accvgpr_read_b32 v31, a31 +; GISEL-NEXT: ds_read_b128 a[0:3], v33 offset:8192 +; GISEL-NEXT: ds_read_b128 a[4:7], v33 offset:8208 +; GISEL-NEXT: ds_read_b128 a[8:11], v33 offset:8224 +; GISEL-NEXT: ds_read_b128 a[12:15], v33 offset:8240 +; GISEL-NEXT: ds_read_b128 a[16:19], v33 offset:8256 +; GISEL-NEXT: ds_read_b128 a[20:23], v33 offset:8272 +; GISEL-NEXT: ds_read_b128 a[24:27], v33 offset:8288 +; GISEL-NEXT: ds_read_b128 a[28:31], v33 offset:8304 ; GISEL-NEXT: s_waitcnt lgkmcnt(0) -; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v33, v34, a[128:159] -; GISEL-NEXT: v_accvgpr_read_b32 v2, a66 -; GISEL-NEXT: v_accvgpr_read_b32 v3, a67 -; GISEL-NEXT: v_accvgpr_read_b32 v4, a68 -; GISEL-NEXT: v_accvgpr_read_b32 v5, a69 -; GISEL-NEXT: v_accvgpr_read_b32 v6, a70 -; GISEL-NEXT: v_accvgpr_read_b32 v7, a71 -; GISEL-NEXT: v_accvgpr_read_b32 v8, a72 -; GISEL-NEXT: v_accvgpr_read_b32 v9, a73 -; GISEL-NEXT: v_accvgpr_read_b32 v10, a74 -; GISEL-NEXT: v_accvgpr_read_b32 v11, a75 -; GISEL-NEXT: v_accvgpr_read_b32 v12, a76 -; GISEL-NEXT: v_accvgpr_read_b32 v13, a77 -; GISEL-NEXT: v_accvgpr_read_b32 v14, a78 -; GISEL-NEXT: v_accvgpr_read_b32 v15, a79 -; GISEL-NEXT: v_accvgpr_read_b32 v16, a80 -; GISEL-NEXT: v_accvgpr_read_b32 v17, a81 -; GISEL-NEXT: v_accvgpr_read_b32 v18, a82 -; GISEL-NEXT: v_accvgpr_read_b32 v19, a83 -; GISEL-NEXT: v_accvgpr_read_b32 v20, a84 -; GISEL-NEXT: v_accvgpr_read_b32 v21, a85 -; GISEL-NEXT: v_accvgpr_read_b32 v22, a86 -; GISEL-NEXT: v_accvgpr_read_b32 v23, a87 -; GISEL-NEXT: v_accvgpr_read_b32 v24, a88 -; GISEL-NEXT: v_accvgpr_read_b32 v25, a89 -; GISEL-NEXT: v_accvgpr_read_b32 v26, a90 -; GISEL-NEXT: v_accvgpr_read_b32 v27, a91 -; GISEL-NEXT: v_accvgpr_read_b32 v28, a92 -; GISEL-NEXT: v_accvgpr_read_b32 v29, a93 -; GISEL-NEXT: v_accvgpr_read_b32 v30, a94 -; GISEL-NEXT: v_accvgpr_read_b32 v31, a95 -; GISEL-NEXT: v_add_u32_e32 v32, s7, v32 +; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v38, v39, a[0:31] ; GISEL-NEXT: ds_write_b128 v32, v[0:3] ; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:16 ; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:32 @@ -2353,6 +2283,22 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_interleave_EXP_MFMA ; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:80 ; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:96 ; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:112 +; GISEL-NEXT: v_mul_f32_e32 v32, s2, v34 +; GISEL-NEXT: v_fma_f32 v39, s2, v34, -v32 +; GISEL-NEXT: v_rndne_f32_e32 v40, v32 +; GISEL-NEXT: v_fmac_f32_e32 v39, s2, v35 +; GISEL-NEXT: v_sub_f32_e32 v32, v32, v40 +; GISEL-NEXT: v_add_f32_e32 v32, v32, v39 +; GISEL-NEXT: v_cvt_i32_f32_e32 v39, v40 +; GISEL-NEXT: v_exp_f32_e32 v32, v32 +; GISEL-NEXT: v_add_u32_e32 v40, 0xc000, v33 +; GISEL-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0) +; GISEL-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0) +; GISEL-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0) +; GISEL-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0) +; GISEL-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0) +; GISEL-NEXT: v_ldexp_f32 v32, v32, v39 +; GISEL-NEXT: s_nop 0 ; GISEL-NEXT: v_accvgpr_read_b32 v0, a0 ; GISEL-NEXT: v_accvgpr_read_b32 v1, a1 ; GISEL-NEXT: v_accvgpr_read_b32 v2, a2 @@ -2385,6 +2331,29 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_interleave_EXP_MFMA ; GISEL-NEXT: v_accvgpr_read_b32 v29, a29 ; GISEL-NEXT: v_accvgpr_read_b32 v30, a30 ; GISEL-NEXT: v_accvgpr_read_b32 v31, a31 +; GISEL-NEXT: ds_read_b128 a[0:3], v33 offset:24576 +; GISEL-NEXT: ds_read_b128 a[4:7], v33 offset:24592 +; GISEL-NEXT: ds_read_b128 a[8:11], v33 offset:24608 +; GISEL-NEXT: ds_read_b128 a[12:15], v33 offset:24624 +; GISEL-NEXT: ds_read_b128 a[16:19], v33 offset:24640 +; GISEL-NEXT: ds_read_b128 a[20:23], v33 offset:24656 +; GISEL-NEXT: ds_read_b128 a[24:27], v33 offset:24672 +; GISEL-NEXT: ds_read_b128 a[28:31], v33 offset:24688 +; GISEL-NEXT: v_readfirstlane_b32 s0, v32 +; GISEL-NEXT: s_cselect_b32 s0, 0, s0 +; GISEL-NEXT: s_cmp_lg_u64 vcc, 0 +; GISEL-NEXT: s_cselect_b32 s0, 0x7f800000, s0 +; GISEL-NEXT: v_mov_b32_e32 v32, s0 +; GISEL-NEXT: v_mul_f32_e32 v39, s3, v34 +; GISEL-NEXT: v_fma_f32 v41, s3, v34, -v39 +; GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v38, v32, a[0:31] +; GISEL-NEXT: v_rndne_f32_e32 v42, v39 +; GISEL-NEXT: v_fmac_f32_e32 v41, s3, v35 +; GISEL-NEXT: v_sub_f32_e32 v39, v39, v42 +; GISEL-NEXT: v_add_f32_e32 v39, v39, v41 +; GISEL-NEXT: v_cvt_i32_f32_e32 v41, v42 +; GISEL-NEXT: v_exp_f32_e32 v39, v39 ; GISEL-NEXT: v_mov_b32_e32 v32, s7 ; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:8192 ; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:8208 @@ -2394,128 +2363,171 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_interleave_EXP_MFMA ; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:8272 ; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:8288 ; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:8304 -; GISEL-NEXT: v_accvgpr_read_b32 v0, a32 -; GISEL-NEXT: v_accvgpr_read_b32 v1, a33 -; GISEL-NEXT: v_accvgpr_read_b32 v2, a34 -; GISEL-NEXT: v_accvgpr_read_b32 v3, a35 -; GISEL-NEXT: v_accvgpr_read_b32 v4, a36 -; GISEL-NEXT: v_accvgpr_read_b32 v5, a37 -; GISEL-NEXT: v_accvgpr_read_b32 v6, a38 -; GISEL-NEXT: v_accvgpr_read_b32 v7, a39 -; GISEL-NEXT: v_accvgpr_read_b32 v8, a40 -; GISEL-NEXT: v_accvgpr_read_b32 v9, a41 -; GISEL-NEXT: v_accvgpr_read_b32 v10, a42 -; GISEL-NEXT: v_accvgpr_read_b32 v11, a43 -; GISEL-NEXT: v_accvgpr_read_b32 v12, a44 -; GISEL-NEXT: v_accvgpr_read_b32 v13, a45 -; GISEL-NEXT: v_accvgpr_read_b32 v14, a46 -; GISEL-NEXT: v_accvgpr_read_b32 v15, a47 -; GISEL-NEXT: v_accvgpr_read_b32 v16, a48 -; GISEL-NEXT: v_accvgpr_read_b32 v17, a49 -; GISEL-NEXT: v_accvgpr_read_b32 v18, a50 -; GISEL-NEXT: v_accvgpr_read_b32 v19, a51 -; GISEL-NEXT: v_accvgpr_read_b32 v20, a52 -; GISEL-NEXT: v_accvgpr_read_b32 v21, a53 -; GISEL-NEXT: v_accvgpr_read_b32 v22, a54 -; GISEL-NEXT: v_accvgpr_read_b32 v23, a55 -; GISEL-NEXT: v_accvgpr_read_b32 v24, a56 -; GISEL-NEXT: v_accvgpr_read_b32 v25, a57 -; GISEL-NEXT: v_accvgpr_read_b32 v26, a58 -; GISEL-NEXT: v_accvgpr_read_b32 v27, a59 -; GISEL-NEXT: v_accvgpr_read_b32 v28, a60 -; GISEL-NEXT: v_accvgpr_read_b32 v29, a61 -; GISEL-NEXT: v_accvgpr_read_b32 v30, a62 -; GISEL-NEXT: v_accvgpr_read_b32 v31, a63 +; GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s3, v36 +; GISEL-NEXT: s_cmp_lg_u64 vcc, 0 +; GISEL-NEXT: v_cmp_gt_f32_e32 vcc, s3, v37 +; GISEL-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0) +; GISEL-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0) +; GISEL-NEXT: s_nop 0 +; GISEL-NEXT: v_accvgpr_read_b32 v0, a0 +; GISEL-NEXT: v_accvgpr_read_b32 v1, a1 +; GISEL-NEXT: v_accvgpr_read_b32 v2, a2 +; GISEL-NEXT: v_accvgpr_read_b32 v3, a3 +; GISEL-NEXT: v_accvgpr_read_b32 v4, a4 +; GISEL-NEXT: v_accvgpr_read_b32 v5, a5 +; GISEL-NEXT: v_accvgpr_read_b32 v6, a6 +; GISEL-NEXT: v_accvgpr_read_b32 v7, a7 +; GISEL-NEXT: v_accvgpr_read_b32 v8, a8 +; GISEL-NEXT: v_accvgpr_read_b32 v9, a9 +; GISEL-NEXT: v_accvgpr_read_b32 v10, a10 +; GISEL-NEXT: v_accvgpr_read_b32 v11, a11 +; GISEL-NEXT: v_accvgpr_read_b32 v12, a12 +; GISEL-NEXT: v_accvgpr_read_b32 v13, a13 +; GISEL-NEXT: v_accvgpr_read_b32 v14, a14 +; GISEL-NEXT: v_accvgpr_read_b32 v15, a15 +; GISEL-NEXT: v_accvgpr_read_b32 v16, a16 +; GISEL-NEXT: v_accvgpr_read_b32 v17, a17 +; GISEL-NEXT: v_accvgpr_read_b32 v18, a18 +; GISEL-NEXT: v_accvgpr_read_b32 v19, a19 +; GISEL-NEXT: v_accvgpr_read_b32 v20, a20 +; GISEL-NEXT: v_accvgpr_read_b32 v21, a21 +; GISEL-NEXT: v_accvgpr_read_b32 v22, a22 +; GISEL-NEXT: v_accvgpr_read_b32 v23, a23 +; GISEL-NEXT: v_accvgpr_read_b32 v24, a24 +; GISEL-NEXT: v_accvgpr_read_b32 v25, a25 +; GISEL-NEXT: v_accvgpr_read_b32 v26, a26 +; GISEL-NEXT: v_accvgpr_read_b32 v27, a27 +; GISEL-NEXT: v_accvgpr_read_b32 v28, a28 +; GISEL-NEXT: v_accvgpr_read_b32 v29, a29 +; GISEL-NEXT: v_accvgpr_read_b32 v30, a30 +; GISEL-NEXT: v_accvgpr_read_b32 v31, a31 +; GISEL-NEXT: ds_read_b128 a[0:3], v33 offset:49152 +; GISEL-NEXT: ds_read_b128 a[4:7], v33 offset:49168 +; GISEL-NEXT: ds_read_b128 a[8:11], v33 offset:49184 +; GISEL-NEXT: ds_read_b128 a[12:15], v33 offset:49200 +; GISEL-NEXT: ds_read_b128 a[16:19], v33 offset:49216 +; GISEL-NEXT: ds_read_b128 a[20:23], v33 offset:49232 +; GISEL-NEXT: ds_read_b128 a[24:27], v33 offset:49248 +; GISEL-NEXT: ds_read_b128 a[28:31], v33 offset:49264 +; GISEL-NEXT: v_ldexp_f32 v33, v39, v41 +; GISEL-NEXT: v_readfirstlane_b32 s0, v33 +; GISEL-NEXT: s_cselect_b32 s0, 0, s0 +; GISEL-NEXT: s_cmp_lg_u64 vcc, 0 +; GISEL-NEXT: s_cselect_b32 s0, 0x7f800000, s0 +; GISEL-NEXT: v_mov_b32_e32 v33, s0 ; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:16384 ; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:16400 +; GISEL-NEXT: s_waitcnt lgkmcnt(2) +; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v38, v33, a[0:31] +; GISEL-NEXT: v_mul_f32_e32 v33, s8, v34 +; GISEL-NEXT: v_fma_f32 v34, s8, v34, -v33 +; GISEL-NEXT: v_fmac_f32_e32 v34, s8, v35 +; GISEL-NEXT: v_rndne_f32_e32 v35, v33 +; GISEL-NEXT: v_sub_f32_e32 v33, v33, v35 +; GISEL-NEXT: v_add_f32_e32 v33, v33, v34 +; GISEL-NEXT: v_cvt_i32_f32_e32 v34, v35 +; GISEL-NEXT: v_exp_f32_e32 v33, v33 ; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:16416 ; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:16432 ; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:16448 ; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:16464 ; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:16480 ; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:16496 -; GISEL-NEXT: v_accvgpr_read_b32 v0, a96 -; GISEL-NEXT: v_accvgpr_read_b32 v1, a97 -; GISEL-NEXT: v_accvgpr_read_b32 v2, a98 -; GISEL-NEXT: v_accvgpr_read_b32 v3, a99 -; GISEL-NEXT: v_accvgpr_read_b32 v4, a100 -; GISEL-NEXT: v_accvgpr_read_b32 v5, a101 -; GISEL-NEXT: v_accvgpr_read_b32 v6, a102 -; GISEL-NEXT: v_accvgpr_read_b32 v7, a103 -; GISEL-NEXT: v_accvgpr_read_b32 v8, a104 -; GISEL-NEXT: v_accvgpr_read_b32 v9, a105 -; GISEL-NEXT: v_accvgpr_read_b32 v10, a106 -; GISEL-NEXT: v_accvgpr_read_b32 v11, a107 -; GISEL-NEXT: v_accvgpr_read_b32 v12, a108 -; GISEL-NEXT: v_accvgpr_read_b32 v13, a109 -; GISEL-NEXT: v_accvgpr_read_b32 v14, a110 -; GISEL-NEXT: v_accvgpr_read_b32 v15, a111 -; GISEL-NEXT: v_accvgpr_read_b32 v16, a112 -; GISEL-NEXT: v_accvgpr_read_b32 v17, a113 -; GISEL-NEXT: v_accvgpr_read_b32 v18, a114 -; GISEL-NEXT: v_accvgpr_read_b32 v19, a115 -; GISEL-NEXT: v_accvgpr_read_b32 v20, a116 -; GISEL-NEXT: v_accvgpr_read_b32 v21, a117 -; GISEL-NEXT: v_accvgpr_read_b32 v22, a118 -; GISEL-NEXT: v_accvgpr_read_b32 v23, a119 -; GISEL-NEXT: v_accvgpr_read_b32 v24, a120 -; GISEL-NEXT: v_accvgpr_read_b32 v25, a121 -; GISEL-NEXT: v_accvgpr_read_b32 v26, a122 -; GISEL-NEXT: v_accvgpr_read_b32 v27, a123 -; GISEL-NEXT: v_accvgpr_read_b32 v28, a124 -; GISEL-NEXT: v_accvgpr_read_b32 v29, a125 -; GISEL-NEXT: v_accvgpr_read_b32 v30, a126 -; GISEL-NEXT: v_accvgpr_read_b32 v31, a127 +; GISEL-NEXT: v_ldexp_f32 v33, v33, v34 +; GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s8, v36 +; GISEL-NEXT: v_readfirstlane_b32 s0, v33 +; GISEL-NEXT: s_cmp_lg_u64 vcc, 0 +; GISEL-NEXT: v_cmp_gt_f32_e32 vcc, s8, v37 +; GISEL-NEXT: v_accvgpr_read_b32 v0, a0 +; GISEL-NEXT: v_accvgpr_read_b32 v1, a1 +; GISEL-NEXT: v_accvgpr_read_b32 v2, a2 +; GISEL-NEXT: v_accvgpr_read_b32 v3, a3 +; GISEL-NEXT: v_accvgpr_read_b32 v4, a4 +; GISEL-NEXT: v_accvgpr_read_b32 v5, a5 +; GISEL-NEXT: v_accvgpr_read_b32 v6, a6 +; GISEL-NEXT: v_accvgpr_read_b32 v7, a7 +; GISEL-NEXT: v_accvgpr_read_b32 v8, a8 +; GISEL-NEXT: v_accvgpr_read_b32 v9, a9 +; GISEL-NEXT: v_accvgpr_read_b32 v10, a10 +; GISEL-NEXT: v_accvgpr_read_b32 v11, a11 +; GISEL-NEXT: v_accvgpr_read_b32 v12, a12 +; GISEL-NEXT: v_accvgpr_read_b32 v13, a13 +; GISEL-NEXT: v_accvgpr_read_b32 v14, a14 +; GISEL-NEXT: v_accvgpr_read_b32 v15, a15 +; GISEL-NEXT: v_accvgpr_read_b32 v16, a16 +; GISEL-NEXT: v_accvgpr_read_b32 v17, a17 +; GISEL-NEXT: v_accvgpr_read_b32 v18, a18 +; GISEL-NEXT: v_accvgpr_read_b32 v19, a19 +; GISEL-NEXT: v_accvgpr_read_b32 v20, a20 +; GISEL-NEXT: v_accvgpr_read_b32 v21, a21 +; GISEL-NEXT: v_accvgpr_read_b32 v22, a22 +; GISEL-NEXT: v_accvgpr_read_b32 v23, a23 +; GISEL-NEXT: v_accvgpr_read_b32 v24, a24 +; GISEL-NEXT: v_accvgpr_read_b32 v25, a25 +; GISEL-NEXT: v_accvgpr_read_b32 v26, a26 +; GISEL-NEXT: v_accvgpr_read_b32 v27, a27 +; GISEL-NEXT: v_accvgpr_read_b32 v28, a28 +; GISEL-NEXT: v_accvgpr_read_b32 v29, a29 +; GISEL-NEXT: v_accvgpr_read_b32 v30, a30 +; GISEL-NEXT: v_accvgpr_read_b32 v31, a31 +; GISEL-NEXT: ds_read_b128 a[0:3], v40 offset:32768 +; GISEL-NEXT: ds_read_b128 a[4:7], v40 offset:32784 +; GISEL-NEXT: ds_read_b128 a[8:11], v40 offset:32800 +; GISEL-NEXT: ds_read_b128 a[12:15], v40 offset:32816 +; GISEL-NEXT: ds_read_b128 a[16:19], v40 offset:32832 +; GISEL-NEXT: ds_read_b128 a[20:23], v40 offset:32848 +; GISEL-NEXT: ds_read_b128 a[24:27], v40 offset:32864 +; GISEL-NEXT: ds_read_b128 a[28:31], v40 offset:32880 +; GISEL-NEXT: s_cselect_b32 s0, 0, s0 +; GISEL-NEXT: s_cmp_lg_u64 vcc, 0 +; GISEL-NEXT: s_cselect_b32 s0, 0x7f800000, s0 +; GISEL-NEXT: v_mov_b32_e32 v33, s0 ; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:24576 ; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:24592 +; GISEL-NEXT: s_waitcnt lgkmcnt(2) +; GISEL-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v38, v33, a[0:31] ; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:24608 ; GISEL-NEXT: ds_write_b128 v32, v[12:15] offset:24624 ; GISEL-NEXT: ds_write_b128 v32, v[16:19] offset:24640 ; GISEL-NEXT: ds_write_b128 v32, v[20:23] offset:24656 ; GISEL-NEXT: ds_write_b128 v32, v[24:27] offset:24672 ; GISEL-NEXT: ds_write_b128 v32, v[28:31] offset:24688 -; GISEL-NEXT: v_accvgpr_read_b32 v0, a128 -; GISEL-NEXT: v_accvgpr_read_b32 v1, a129 -; GISEL-NEXT: v_accvgpr_read_b32 v2, a130 -; GISEL-NEXT: v_accvgpr_read_b32 v3, a131 -; GISEL-NEXT: v_accvgpr_read_b32 v4, a132 -; GISEL-NEXT: v_accvgpr_read_b32 v5, a133 -; GISEL-NEXT: v_accvgpr_read_b32 v6, a134 -; GISEL-NEXT: v_accvgpr_read_b32 v7, a135 -; GISEL-NEXT: v_accvgpr_read_b32 v8, a136 -; GISEL-NEXT: v_accvgpr_read_b32 v9, a137 -; GISEL-NEXT: v_accvgpr_read_b32 v10, a138 -; GISEL-NEXT: v_accvgpr_read_b32 v11, a139 -; GISEL-NEXT: v_accvgpr_read_b32 v12, a140 -; GISEL-NEXT: v_accvgpr_read_b32 v13, a141 -; GISEL-NEXT: v_accvgpr_read_b32 v14, a142 -; GISEL-NEXT: v_accvgpr_read_b32 v15, a143 -; GISEL-NEXT: v_accvgpr_read_b32 v16, a144 -; GISEL-NEXT: v_accvgpr_read_b32 v17, a145 -; GISEL-NEXT: v_accvgpr_read_b32 v18, a146 -; GISEL-NEXT: v_accvgpr_read_b32 v19, a147 -; GISEL-NEXT: v_accvgpr_read_b32 v20, a148 -; GISEL-NEXT: v_accvgpr_read_b32 v21, a149 -; GISEL-NEXT: v_accvgpr_read_b32 v22, a150 -; GISEL-NEXT: v_accvgpr_read_b32 v23, a151 -; GISEL-NEXT: v_accvgpr_read_b32 v24, a152 -; GISEL-NEXT: v_accvgpr_read_b32 v25, a153 -; GISEL-NEXT: v_accvgpr_read_b32 v26, a154 -; GISEL-NEXT: v_accvgpr_read_b32 v27, a155 -; GISEL-NEXT: v_accvgpr_read_b32 v28, a156 -; GISEL-NEXT: v_accvgpr_read_b32 v29, a157 -; GISEL-NEXT: v_accvgpr_read_b32 v30, a158 -; GISEL-NEXT: v_accvgpr_read_b32 v31, a159 -; GISEL-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0) -; GISEL-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0) -; GISEL-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0) -; GISEL-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0) -; GISEL-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0) -; GISEL-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0) -; GISEL-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0) ; GISEL-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0) ; GISEL-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0) ; GISEL-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0) +; GISEL-NEXT: s_nop 12 +; GISEL-NEXT: v_accvgpr_read_b32 v0, a0 +; GISEL-NEXT: v_accvgpr_read_b32 v1, a1 +; GISEL-NEXT: v_accvgpr_read_b32 v2, a2 +; GISEL-NEXT: v_accvgpr_read_b32 v3, a3 +; GISEL-NEXT: v_accvgpr_read_b32 v4, a4 +; GISEL-NEXT: v_accvgpr_read_b32 v5, a5 +; GISEL-NEXT: v_accvgpr_read_b32 v6, a6 +; GISEL-NEXT: v_accvgpr_read_b32 v7, a7 +; GISEL-NEXT: v_accvgpr_read_b32 v8, a8 +; GISEL-NEXT: v_accvgpr_read_b32 v9, a9 +; GISEL-NEXT: v_accvgpr_read_b32 v10, a10 +; GISEL-NEXT: v_accvgpr_read_b32 v11, a11 +; GISEL-NEXT: v_accvgpr_read_b32 v12, a12 +; GISEL-NEXT: v_accvgpr_read_b32 v13, a13 +; GISEL-NEXT: v_accvgpr_read_b32 v14, a14 +; GISEL-NEXT: v_accvgpr_read_b32 v15, a15 +; GISEL-NEXT: v_accvgpr_read_b32 v16, a16 +; GISEL-NEXT: v_accvgpr_read_b32 v17, a17 +; GISEL-NEXT: v_accvgpr_read_b32 v18, a18 +; GISEL-NEXT: v_accvgpr_read_b32 v19, a19 +; GISEL-NEXT: v_accvgpr_read_b32 v20, a20 +; GISEL-NEXT: v_accvgpr_read_b32 v21, a21 +; GISEL-NEXT: v_accvgpr_read_b32 v22, a22 +; GISEL-NEXT: v_accvgpr_read_b32 v23, a23 +; GISEL-NEXT: v_accvgpr_read_b32 v24, a24 +; GISEL-NEXT: v_accvgpr_read_b32 v25, a25 +; GISEL-NEXT: v_accvgpr_read_b32 v26, a26 +; GISEL-NEXT: v_accvgpr_read_b32 v27, a27 +; GISEL-NEXT: v_accvgpr_read_b32 v28, a28 +; GISEL-NEXT: v_accvgpr_read_b32 v29, a29 +; GISEL-NEXT: v_accvgpr_read_b32 v30, a30 +; GISEL-NEXT: v_accvgpr_read_b32 v31, a31 ; GISEL-NEXT: ds_write_b128 v32, v[0:3] offset:32768 ; GISEL-NEXT: ds_write_b128 v32, v[4:7] offset:32784 ; GISEL-NEXT: ds_write_b128 v32, v[8:11] offset:32800 @@ -2529,49 +2541,65 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_interleave_EXP_MFMA ; EXACTCUTOFF-LABEL: test_sched_group_barrier_pipeline_interleave_EXP_MFMA: ; EXACTCUTOFF: ; %bb.0: ; %entry ; EXACTCUTOFF-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x44 -; EXACTCUTOFF-NEXT: v_mov_b32_e32 v3, 0x3fb8aa3b +; EXACTCUTOFF-NEXT: v_mov_b32_e32 v2, 0x3fb8aa3b ; EXACTCUTOFF-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 -; EXACTCUTOFF-NEXT: v_mov_b32_e32 v7, 0x32a5705f +; EXACTCUTOFF-NEXT: v_mov_b32_e32 v6, 0x32a5705f ; EXACTCUTOFF-NEXT: v_lshlrev_b32_e32 v0, 7, v0 ; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0) -; EXACTCUTOFF-NEXT: v_mul_f32_e32 v4, s0, v3 -; EXACTCUTOFF-NEXT: v_rndne_f32_e32 v5, v4 -; EXACTCUTOFF-NEXT: v_sub_f32_e32 v6, v4, v5 -; EXACTCUTOFF-NEXT: v_fma_f32 v4, s0, v3, -v4 -; EXACTCUTOFF-NEXT: v_fmac_f32_e32 v4, s0, v7 -; EXACTCUTOFF-NEXT: v_add_f32_e32 v4, v6, v4 -; EXACTCUTOFF-NEXT: v_exp_f32_e32 v4, v4 -; EXACTCUTOFF-NEXT: v_cvt_i32_f32_e32 v5, v5 +; EXACTCUTOFF-NEXT: v_mul_f32_e32 v3, s0, v2 +; EXACTCUTOFF-NEXT: v_rndne_f32_e32 v4, v3 +; EXACTCUTOFF-NEXT: v_sub_f32_e32 v5, v3, v4 +; EXACTCUTOFF-NEXT: v_fma_f32 v3, s0, v2, -v3 +; EXACTCUTOFF-NEXT: v_fmac_f32_e32 v3, s0, v6 +; EXACTCUTOFF-NEXT: v_add_f32_e32 v3, v5, v3 +; EXACTCUTOFF-NEXT: v_exp_f32_e32 v3, v3 +; EXACTCUTOFF-NEXT: v_cvt_i32_f32_e32 v4, v4 ; EXACTCUTOFF-NEXT: v_and_b32_e32 v0, 0x1ff80, v0 ; EXACTCUTOFF-NEXT: v_add_u32_e32 v1, s6, v0 -; EXACTCUTOFF-NEXT: ds_read_b128 a[60:63], v1 offset:112 -; EXACTCUTOFF-NEXT: ds_read_b128 a[56:59], v1 offset:96 -; EXACTCUTOFF-NEXT: ds_read_b128 a[52:55], v1 offset:80 -; EXACTCUTOFF-NEXT: ds_read_b128 a[48:51], v1 offset:64 -; EXACTCUTOFF-NEXT: ds_read_b128 a[32:35], v1 -; EXACTCUTOFF-NEXT: ds_read_b128 a[36:39], v1 offset:16 -; EXACTCUTOFF-NEXT: ds_read_b128 a[40:43], v1 offset:32 -; EXACTCUTOFF-NEXT: ds_read_b128 a[44:47], v1 offset:48 -; EXACTCUTOFF-NEXT: v_mov_b32_e32 v9, 1.0 -; EXACTCUTOFF-NEXT: v_ldexp_f32 v4, v4, v5 -; EXACTCUTOFF-NEXT: v_mov_b32_e32 v5, 0xc2ce8ed0 -; EXACTCUTOFF-NEXT: v_mul_f32_e32 v10, s1, v3 -; EXACTCUTOFF-NEXT: v_cmp_nlt_f32_e32 vcc, s0, v5 -; EXACTCUTOFF-NEXT: v_mov_b32_e32 v6, 0x42b17218 -; EXACTCUTOFF-NEXT: v_rndne_f32_e32 v11, v10 -; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc -; EXACTCUTOFF-NEXT: v_cmp_ngt_f32_e32 vcc, s0, v6 +; EXACTCUTOFF-NEXT: ds_read_b128 a[28:31], v1 offset:112 +; EXACTCUTOFF-NEXT: ds_read_b128 a[24:27], v1 offset:96 +; EXACTCUTOFF-NEXT: ds_read_b128 a[20:23], v1 offset:80 +; EXACTCUTOFF-NEXT: ds_read_b128 a[16:19], v1 offset:64 +; EXACTCUTOFF-NEXT: ds_read_b128 a[0:3], v1 +; EXACTCUTOFF-NEXT: ds_read_b128 a[4:7], v1 offset:16 +; EXACTCUTOFF-NEXT: ds_read_b128 a[8:11], v1 offset:32 +; EXACTCUTOFF-NEXT: ds_read_b128 a[12:15], v1 offset:48 +; EXACTCUTOFF-NEXT: v_mov_b32_e32 v5, 1.0 +; EXACTCUTOFF-NEXT: v_ldexp_f32 v3, v3, v4 +; EXACTCUTOFF-NEXT: v_mov_b32_e32 v4, 0xc2ce8ed0 +; EXACTCUTOFF-NEXT: v_cmp_nlt_f32_e32 vcc, s0, v4 +; EXACTCUTOFF-NEXT: v_mov_b32_e32 v7, 0x42b17218 +; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v3, 0, v3, vcc +; EXACTCUTOFF-NEXT: v_cmp_ngt_f32_e32 vcc, s0, v7 ; EXACTCUTOFF-NEXT: v_mov_b32_e32 v8, 0x7f800000 -; EXACTCUTOFF-NEXT: v_sub_f32_e32 v12, v10, v11 -; EXACTCUTOFF-NEXT: v_fma_f32 v10, s1, v3, -v10 -; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc -; EXACTCUTOFF-NEXT: v_fmac_f32_e32 v10, s1, v7 +; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v3, v8, v3, vcc +; EXACTCUTOFF-NEXT: v_add_u32_e32 v0, s7, v0 +; EXACTCUTOFF-NEXT: v_cmp_nlt_f32_e32 vcc, s1, v4 +; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0) +; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v5, v3, a[0:31] +; EXACTCUTOFF-NEXT: s_load_dword s8, s[4:5], 0x54 +; EXACTCUTOFF-NEXT: ; kill: killed $sgpr4_sgpr5 +; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0) +; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0) +; EXACTCUTOFF-NEXT: s_nop 15 +; EXACTCUTOFF-NEXT: s_nop 1 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[28:31] offset:112 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[24:27] offset:96 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[20:23] offset:80 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[16:19] offset:64 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[12:15] offset:48 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[8:11] offset:32 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[4:7] offset:16 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[0:3] +; EXACTCUTOFF-NEXT: v_mul_f32_e32 v0, s1, v2 +; EXACTCUTOFF-NEXT: v_rndne_f32_e32 v3, v0 +; EXACTCUTOFF-NEXT: v_sub_f32_e32 v9, v0, v3 +; EXACTCUTOFF-NEXT: v_fma_f32 v0, s1, v2, -v0 +; EXACTCUTOFF-NEXT: v_fmac_f32_e32 v0, s1, v6 +; EXACTCUTOFF-NEXT: v_add_f32_e32 v0, v9, v0 +; EXACTCUTOFF-NEXT: v_exp_f32_e32 v0, v0 +; EXACTCUTOFF-NEXT: v_cvt_i32_f32_e32 v3, v3 ; EXACTCUTOFF-NEXT: ds_read_b128 a[28:31], v1 offset:8304 -; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(1) -; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v9, v4, a[32:63] -; EXACTCUTOFF-NEXT: v_add_f32_e32 v4, v12, v10 -; EXACTCUTOFF-NEXT: v_exp_f32_e32 v4, v4 -; EXACTCUTOFF-NEXT: v_cvt_i32_f32_e32 v10, v11 ; EXACTCUTOFF-NEXT: ds_read_b128 a[24:27], v1 offset:8288 ; EXACTCUTOFF-NEXT: ds_read_b128 a[20:23], v1 offset:8272 ; EXACTCUTOFF-NEXT: ds_read_b128 a[16:19], v1 offset:8256 @@ -2579,104 +2607,32 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_interleave_EXP_MFMA ; EXACTCUTOFF-NEXT: ds_read_b128 a[8:11], v1 offset:8224 ; EXACTCUTOFF-NEXT: ds_read_b128 a[4:7], v1 offset:8208 ; EXACTCUTOFF-NEXT: ds_read_b128 a[0:3], v1 offset:8192 -; EXACTCUTOFF-NEXT: v_ldexp_f32 v4, v4, v10 -; EXACTCUTOFF-NEXT: v_cmp_nlt_f32_e32 vcc, s1, v5 -; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc -; EXACTCUTOFF-NEXT: v_cmp_ngt_f32_e32 vcc, s1, v6 -; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc -; EXACTCUTOFF-NEXT: v_mul_f32_e32 v10, s2, v3 -; EXACTCUTOFF-NEXT: v_rndne_f32_e32 v11, v10 +; EXACTCUTOFF-NEXT: v_ldexp_f32 v0, v0, v3 +; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc +; EXACTCUTOFF-NEXT: v_cmp_ngt_f32_e32 vcc, s1, v7 +; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v0, v8, v0, vcc +; EXACTCUTOFF-NEXT: v_mul_f32_e32 v3, s2, v2 +; EXACTCUTOFF-NEXT: v_rndne_f32_e32 v10, v3 ; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0) -; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v9, v4, a[0:31] -; EXACTCUTOFF-NEXT: v_fma_f32 v4, s2, v3, -v10 -; EXACTCUTOFF-NEXT: v_sub_f32_e32 v12, v10, v11 -; EXACTCUTOFF-NEXT: v_fmac_f32_e32 v4, s2, v7 -; EXACTCUTOFF-NEXT: v_add_f32_e32 v4, v12, v4 -; EXACTCUTOFF-NEXT: v_exp_f32_e32 v4, v4 -; EXACTCUTOFF-NEXT: v_cvt_i32_f32_e32 v10, v11 -; EXACTCUTOFF-NEXT: ds_read_b128 a[92:95], v1 offset:24688 -; EXACTCUTOFF-NEXT: ds_read_b128 a[88:91], v1 offset:24672 -; EXACTCUTOFF-NEXT: ds_read_b128 a[84:87], v1 offset:24656 -; EXACTCUTOFF-NEXT: ds_read_b128 a[80:83], v1 offset:24640 -; EXACTCUTOFF-NEXT: ds_read_b128 a[76:79], v1 offset:24624 -; EXACTCUTOFF-NEXT: ds_read_b128 a[72:75], v1 offset:24608 -; EXACTCUTOFF-NEXT: ds_read_b128 a[68:71], v1 offset:24592 -; EXACTCUTOFF-NEXT: ds_read_b128 a[64:67], v1 offset:24576 -; EXACTCUTOFF-NEXT: v_add_u32_e32 v2, 0x6000, v1 -; EXACTCUTOFF-NEXT: ds_read_b128 a[124:127], v1 offset:49264 -; EXACTCUTOFF-NEXT: ds_read_b128 a[120:123], v1 offset:49248 -; EXACTCUTOFF-NEXT: ds_read_b128 a[116:119], v1 offset:49232 -; EXACTCUTOFF-NEXT: ds_read_b128 a[112:115], v1 offset:49216 -; EXACTCUTOFF-NEXT: ds_read_b128 a[108:111], v1 offset:49200 -; EXACTCUTOFF-NEXT: ds_read_b128 a[104:107], v1 offset:49184 -; EXACTCUTOFF-NEXT: ds_read_b128 a[100:103], v1 offset:49168 -; EXACTCUTOFF-NEXT: ds_read_b128 a[96:99], v1 offset:49152 -; EXACTCUTOFF-NEXT: v_ldexp_f32 v1, v4, v10 -; EXACTCUTOFF-NEXT: v_cmp_nlt_f32_e32 vcc, s2, v5 -; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc -; EXACTCUTOFF-NEXT: v_cmp_ngt_f32_e32 vcc, s2, v6 -; EXACTCUTOFF-NEXT: v_mul_f32_e32 v4, s3, v3 -; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc -; EXACTCUTOFF-NEXT: v_rndne_f32_e32 v10, v4 -; EXACTCUTOFF-NEXT: s_load_dword s8, s[4:5], 0x54 -; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0) -; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[64:95], v9, v1, a[64:95] -; EXACTCUTOFF-NEXT: v_sub_f32_e32 v1, v4, v10 -; EXACTCUTOFF-NEXT: v_fma_f32 v4, s3, v3, -v4 -; EXACTCUTOFF-NEXT: v_fmac_f32_e32 v4, s3, v7 -; EXACTCUTOFF-NEXT: v_add_f32_e32 v1, v1, v4 -; EXACTCUTOFF-NEXT: v_exp_f32_e32 v1, v1 -; EXACTCUTOFF-NEXT: v_cvt_i32_f32_e32 v4, v10 -; EXACTCUTOFF-NEXT: v_cmp_nlt_f32_e32 vcc, s3, v5 -; EXACTCUTOFF-NEXT: ds_read_b128 a[156:159], v2 offset:57456 -; EXACTCUTOFF-NEXT: ds_read_b128 a[152:155], v2 offset:57440 -; EXACTCUTOFF-NEXT: v_ldexp_f32 v1, v1, v4 -; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc -; EXACTCUTOFF-NEXT: v_cmp_ngt_f32_e32 vcc, s3, v6 -; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc -; EXACTCUTOFF-NEXT: v_mul_f32_e32 v4, s8, v3 -; EXACTCUTOFF-NEXT: v_fma_f32 v3, s8, v3, -v4 -; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[96:127], v9, v1, a[96:127] -; EXACTCUTOFF-NEXT: v_rndne_f32_e32 v1, v4 -; EXACTCUTOFF-NEXT: v_sub_f32_e32 v10, v4, v1 -; EXACTCUTOFF-NEXT: v_fmac_f32_e32 v3, s8, v7 -; EXACTCUTOFF-NEXT: v_add_f32_e32 v3, v10, v3 +; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v5, v0, a[0:31] +; EXACTCUTOFF-NEXT: v_sub_f32_e32 v11, v3, v10 +; EXACTCUTOFF-NEXT: v_fma_f32 v3, s2, v2, -v3 +; EXACTCUTOFF-NEXT: v_fmac_f32_e32 v3, s2, v6 +; EXACTCUTOFF-NEXT: v_add_f32_e32 v3, v11, v3 ; EXACTCUTOFF-NEXT: v_exp_f32_e32 v3, v3 -; EXACTCUTOFF-NEXT: v_cvt_i32_f32_e32 v1, v1 -; EXACTCUTOFF-NEXT: ds_read_b128 a[148:151], v2 offset:57424 -; EXACTCUTOFF-NEXT: ds_read_b128 a[144:147], v2 offset:57408 -; EXACTCUTOFF-NEXT: ds_read_b128 a[128:131], v2 offset:57344 -; EXACTCUTOFF-NEXT: ds_read_b128 a[132:135], v2 offset:57360 -; EXACTCUTOFF-NEXT: ds_read_b128 a[136:139], v2 offset:57376 -; EXACTCUTOFF-NEXT: ds_read_b128 a[140:143], v2 offset:57392 -; EXACTCUTOFF-NEXT: v_ldexp_f32 v1, v3, v1 -; EXACTCUTOFF-NEXT: v_cmp_nlt_f32_e32 vcc, s8, v5 -; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc -; EXACTCUTOFF-NEXT: v_cmp_ngt_f32_e32 vcc, s8, v6 -; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc -; EXACTCUTOFF-NEXT: v_add_u32_e32 v0, s7, v0 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[60:63] offset:112 -; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(1) -; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[128:159], v9, v1, a[128:159] -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[56:59] offset:96 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[52:55] offset:80 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[48:51] offset:64 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[44:47] offset:48 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[40:43] offset:32 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[36:39] offset:16 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[32:35] +; EXACTCUTOFF-NEXT: v_cvt_i32_f32_e32 v10, v10 ; EXACTCUTOFF-NEXT: v_mov_b32_e32 v0, s7 -; EXACTCUTOFF-NEXT: ; kill: killed $sgpr4_sgpr5 +; EXACTCUTOFF-NEXT: v_cmp_nlt_f32_e32 vcc, s2, v4 +; EXACTCUTOFF-NEXT: v_add_u32_e32 v9, 0x6000, v1 +; EXACTCUTOFF-NEXT: v_ldexp_f32 v3, v3, v10 +; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v3, 0, v3, vcc +; EXACTCUTOFF-NEXT: v_cmp_ngt_f32_e32 vcc, s2, v7 +; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v3, v8, v3, vcc +; EXACTCUTOFF-NEXT: v_cmp_nlt_f32_e32 vcc, s3, v4 ; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0) ; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0) ; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0) -; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0) -; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0) -; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0) -; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0) -; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0) -; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0) -; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0) +; EXACTCUTOFF-NEXT: s_nop 4 ; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[24:27] offset:8288 ; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[28:31] offset:8304 ; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[16:19] offset:8256 @@ -2685,30 +2641,94 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_interleave_EXP_MFMA ; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[12:15] offset:8240 ; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[0:3] offset:8192 ; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[4:7] offset:8208 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[88:91] offset:16480 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[92:95] offset:16496 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[80:83] offset:16448 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[84:87] offset:16464 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[72:75] offset:16416 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[76:79] offset:16432 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[64:67] offset:16384 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[68:71] offset:16400 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[120:123] offset:24672 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[124:127] offset:24688 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[112:115] offset:24640 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[116:119] offset:24656 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[104:107] offset:24608 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[108:111] offset:24624 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[96:99] offset:24576 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[100:103] offset:24592 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[152:155] offset:32864 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[156:159] offset:32880 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[144:147] offset:32832 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[148:151] offset:32848 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[136:139] offset:32800 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[140:143] offset:32816 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[128:131] offset:32768 -; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[132:135] offset:32784 +; EXACTCUTOFF-NEXT: ds_read_b128 a[28:31], v1 offset:24688 +; EXACTCUTOFF-NEXT: ds_read_b128 a[24:27], v1 offset:24672 +; EXACTCUTOFF-NEXT: ds_read_b128 a[20:23], v1 offset:24656 +; EXACTCUTOFF-NEXT: ds_read_b128 a[16:19], v1 offset:24640 +; EXACTCUTOFF-NEXT: ds_read_b128 a[12:15], v1 offset:24624 +; EXACTCUTOFF-NEXT: ds_read_b128 a[8:11], v1 offset:24608 +; EXACTCUTOFF-NEXT: ds_read_b128 a[4:7], v1 offset:24592 +; EXACTCUTOFF-NEXT: ds_read_b128 a[0:3], v1 offset:24576 +; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0) +; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v5, v3, a[0:31] +; EXACTCUTOFF-NEXT: v_mul_f32_e32 v3, s3, v2 +; EXACTCUTOFF-NEXT: v_rndne_f32_e32 v10, v3 +; EXACTCUTOFF-NEXT: v_sub_f32_e32 v11, v3, v10 +; EXACTCUTOFF-NEXT: v_fma_f32 v3, s3, v2, -v3 +; EXACTCUTOFF-NEXT: v_fmac_f32_e32 v3, s3, v6 +; EXACTCUTOFF-NEXT: v_add_f32_e32 v3, v11, v3 +; EXACTCUTOFF-NEXT: v_exp_f32_e32 v3, v3 +; EXACTCUTOFF-NEXT: v_cvt_i32_f32_e32 v10, v10 +; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0) +; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0) +; EXACTCUTOFF-NEXT: s_nop 10 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[24:27] offset:16480 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[28:31] offset:16496 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[16:19] offset:16448 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[20:23] offset:16464 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[8:11] offset:16416 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[12:15] offset:16432 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[0:3] offset:16384 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[4:7] offset:16400 +; EXACTCUTOFF-NEXT: ds_read_b128 a[28:31], v1 offset:49264 +; EXACTCUTOFF-NEXT: ds_read_b128 a[24:27], v1 offset:49248 +; EXACTCUTOFF-NEXT: ds_read_b128 a[20:23], v1 offset:49232 +; EXACTCUTOFF-NEXT: ds_read_b128 a[16:19], v1 offset:49216 +; EXACTCUTOFF-NEXT: ds_read_b128 a[12:15], v1 offset:49200 +; EXACTCUTOFF-NEXT: ds_read_b128 a[8:11], v1 offset:49184 +; EXACTCUTOFF-NEXT: ds_read_b128 a[4:7], v1 offset:49168 +; EXACTCUTOFF-NEXT: ds_read_b128 a[0:3], v1 offset:49152 +; EXACTCUTOFF-NEXT: v_ldexp_f32 v1, v3, v10 +; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc +; EXACTCUTOFF-NEXT: v_cmp_ngt_f32_e32 vcc, s3, v7 +; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc +; EXACTCUTOFF-NEXT: v_cmp_nlt_f32_e32 vcc, s8, v4 +; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0) +; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v5, v1, a[0:31] +; EXACTCUTOFF-NEXT: v_mul_f32_e32 v1, s8, v2 +; EXACTCUTOFF-NEXT: v_rndne_f32_e32 v3, v1 +; EXACTCUTOFF-NEXT: v_sub_f32_e32 v10, v1, v3 +; EXACTCUTOFF-NEXT: v_fma_f32 v1, s8, v2, -v1 +; EXACTCUTOFF-NEXT: v_fmac_f32_e32 v1, s8, v6 +; EXACTCUTOFF-NEXT: v_add_f32_e32 v1, v10, v1 +; EXACTCUTOFF-NEXT: v_exp_f32_e32 v1, v1 +; EXACTCUTOFF-NEXT: v_cvt_i32_f32_e32 v2, v3 +; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0) +; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000400) size(1) SyncID(0) +; EXACTCUTOFF-NEXT: v_ldexp_f32 v1, v1, v2 +; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc +; EXACTCUTOFF-NEXT: v_cmp_ngt_f32_e32 vcc, s8, v7 +; EXACTCUTOFF-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc +; EXACTCUTOFF-NEXT: s_nop 6 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[24:27] offset:24672 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[28:31] offset:24688 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[16:19] offset:24640 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[20:23] offset:24656 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[8:11] offset:24608 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[12:15] offset:24624 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[0:3] offset:24576 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[4:7] offset:24592 +; EXACTCUTOFF-NEXT: ds_read_b128 a[28:31], v9 offset:57456 +; EXACTCUTOFF-NEXT: ds_read_b128 a[24:27], v9 offset:57440 +; EXACTCUTOFF-NEXT: ds_read_b128 a[20:23], v9 offset:57424 +; EXACTCUTOFF-NEXT: ds_read_b128 a[16:19], v9 offset:57408 +; EXACTCUTOFF-NEXT: ds_read_b128 a[0:3], v9 offset:57344 +; EXACTCUTOFF-NEXT: ds_read_b128 a[4:7], v9 offset:57360 +; EXACTCUTOFF-NEXT: ds_read_b128 a[8:11], v9 offset:57376 +; EXACTCUTOFF-NEXT: ds_read_b128 a[12:15], v9 offset:57392 +; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0) +; EXACTCUTOFF-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v5, v1, a[0:31] +; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000008) size(1) SyncID(0) +; EXACTCUTOFF-NEXT: s_nop 15 +; EXACTCUTOFF-NEXT: s_nop 2 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[24:27] offset:32864 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[28:31] offset:32880 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[16:19] offset:32832 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[20:23] offset:32848 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[8:11] offset:32800 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[12:15] offset:32816 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[0:3] offset:32768 +; EXACTCUTOFF-NEXT: ds_write_b128 v0, a[4:7] offset:32784 ; EXACTCUTOFF-NEXT: s_endpgm entry: %idx = call i32 @llvm.amdgcn.workitem.id.x() diff --git a/llvm/test/CodeGen/AMDGPU/machine-scheduler-sink-trivial-remats-attr.mir b/llvm/test/CodeGen/AMDGPU/machine-scheduler-sink-trivial-remats-attr.mir index 8b928bc9defa5..c8c71709ad803 100644 --- a/llvm/test/CodeGen/AMDGPU/machine-scheduler-sink-trivial-remats-attr.mir +++ b/llvm/test/CodeGen/AMDGPU/machine-scheduler-sink-trivial-remats-attr.mir @@ -790,12 +790,6 @@ body: | ; GFX908-NEXT: [[DEF20:%[0-9]+]]:agpr_32 = IMPLICIT_DEF ; GFX908-NEXT: [[DEF21:%[0-9]+]]:agpr_32 = IMPLICIT_DEF ; GFX908-NEXT: [[DEF22:%[0-9]+]]:agpr_32 = IMPLICIT_DEF - ; GFX908-NEXT: [[DEF23:%[0-9]+]]:agpr_32 = IMPLICIT_DEF - ; GFX908-NEXT: [[DEF24:%[0-9]+]]:agpr_32 = IMPLICIT_DEF - ; GFX908-NEXT: [[DEF25:%[0-9]+]]:agpr_32 = IMPLICIT_DEF - ; GFX908-NEXT: [[DEF26:%[0-9]+]]:agpr_32 = IMPLICIT_DEF - ; GFX908-NEXT: [[DEF27:%[0-9]+]]:agpr_32 = IMPLICIT_DEF - ; GFX908-NEXT: [[DEF28:%[0-9]+]]:agpr_32 = IMPLICIT_DEF ; GFX908-NEXT: [[V_CVT_I32_F64_e32_1:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 1, implicit $exec, implicit $mode, implicit-def $m0 ; GFX908-NEXT: [[V_CVT_I32_F64_e32_2:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 2, implicit $exec, implicit $mode, implicit-def $m0 ; GFX908-NEXT: [[V_CVT_I32_F64_e32_3:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 3, implicit $exec, implicit $mode, implicit-def $m0 @@ -822,6 +816,12 @@ body: | ; GFX908-NEXT: [[V_CVT_I32_F64_e32_24:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 24, implicit $exec, implicit $mode, implicit-def $m0 ; GFX908-NEXT: [[V_CVT_I32_F64_e32_25:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 25, implicit $exec, implicit $mode, implicit-def $m0 ; GFX908-NEXT: [[V_CVT_I32_F64_e32_26:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 26, implicit $exec, implicit $mode, implicit-def $m0 + ; GFX908-NEXT: [[DEF23:%[0-9]+]]:agpr_32 = IMPLICIT_DEF + ; GFX908-NEXT: [[DEF24:%[0-9]+]]:agpr_32 = IMPLICIT_DEF + ; GFX908-NEXT: [[DEF25:%[0-9]+]]:agpr_32 = IMPLICIT_DEF + ; GFX908-NEXT: [[DEF26:%[0-9]+]]:agpr_32 = IMPLICIT_DEF + ; GFX908-NEXT: [[DEF27:%[0-9]+]]:agpr_32 = IMPLICIT_DEF + ; GFX908-NEXT: [[DEF28:%[0-9]+]]:agpr_32 = IMPLICIT_DEF ; GFX908-NEXT: [[V_CVT_I32_F64_e32_27:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 28, implicit $exec, implicit $mode ; GFX908-NEXT: [[V_CVT_I32_F64_e32_28:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 29, implicit $exec, implicit $mode ; GFX908-NEXT: [[V_CVT_I32_F64_e32_29:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 30, implicit $exec, implicit $mode @@ -2268,6 +2268,8 @@ body: | ; GFX90A-NEXT: [[DEF46:%[0-9]+]]:agpr_32 = IMPLICIT_DEF ; GFX90A-NEXT: [[DEF47:%[0-9]+]]:agpr_32 = IMPLICIT_DEF ; GFX90A-NEXT: [[DEF48:%[0-9]+]]:agpr_32 = IMPLICIT_DEF + ; GFX90A-NEXT: [[V_CVT_I32_F64_e32_:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 257, implicit $exec, implicit $mode + ; GFX90A-NEXT: [[V_CVT_I32_F64_e32_1:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 258, implicit $exec, implicit $mode ; GFX90A-NEXT: [[DEF49:%[0-9]+]]:agpr_32 = IMPLICIT_DEF ; GFX90A-NEXT: [[DEF50:%[0-9]+]]:agpr_32 = IMPLICIT_DEF ; GFX90A-NEXT: [[DEF51:%[0-9]+]]:agpr_32 = IMPLICIT_DEF @@ -2465,8 +2467,6 @@ body: | ; GFX90A-NEXT: [[DEF243:%[0-9]+]]:agpr_32 = IMPLICIT_DEF ; GFX90A-NEXT: [[DEF244:%[0-9]+]]:agpr_32 = IMPLICIT_DEF ; GFX90A-NEXT: [[DEF245:%[0-9]+]]:agpr_32 = IMPLICIT_DEF - ; GFX90A-NEXT: [[V_CVT_I32_F64_e32_:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 257, implicit $exec, implicit $mode - ; GFX90A-NEXT: [[V_CVT_I32_F64_e32_1:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 258, implicit $exec, implicit $mode ; GFX90A-NEXT: {{ $}} ; GFX90A-NEXT: bb.1: ; GFX90A-NEXT: [[DEF246:%[0-9]+]]:agpr_32 = IMPLICIT_DEF diff --git a/llvm/test/CodeGen/AMDGPU/mfma-cd-select.ll b/llvm/test/CodeGen/AMDGPU/mfma-cd-select.ll index fc1c2d7d16d21..f703874d174c3 100644 --- a/llvm/test/CodeGen/AMDGPU/mfma-cd-select.ll +++ b/llvm/test/CodeGen/AMDGPU/mfma-cd-select.ll @@ -11,7 +11,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vgpr(ptr addrspace(1) %arg) ; GFX908-LABEL: test_mfma_f32_32x32x1f32_vgpr: ; GFX908: ; %bb.0: ; %bb ; GFX908-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24 -; GFX908-NEXT: v_mov_b32_e32 v4, 0 +; GFX908-NEXT: v_mov_b32_e32 v29, 0 ; GFX908-NEXT: s_waitcnt lgkmcnt(0) ; GFX908-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0 ; GFX908-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40 @@ -21,13 +21,11 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vgpr(ptr addrspace(1) %arg) ; GFX908-NEXT: v_mov_b32_e32 v2, s18 ; GFX908-NEXT: v_accvgpr_write_b32 a0, v0 ; GFX908-NEXT: v_accvgpr_write_b32 a1, v1 +; GFX908-NEXT: v_mov_b32_e32 v0, s22 +; GFX908-NEXT: v_mov_b32_e32 v1, s23 ; GFX908-NEXT: v_accvgpr_write_b32 a2, v2 -; GFX908-NEXT: v_mov_b32_e32 v0, s21 -; GFX908-NEXT: v_mov_b32_e32 v1, s22 -; GFX908-NEXT: v_mov_b32_e32 v2, s23 -; GFX908-NEXT: v_accvgpr_write_b32 a5, v0 -; GFX908-NEXT: v_accvgpr_write_b32 a6, v1 -; GFX908-NEXT: v_accvgpr_write_b32 a7, v2 +; GFX908-NEXT: v_accvgpr_write_b32 a6, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a7, v1 ; GFX908-NEXT: v_mov_b32_e32 v0, s24 ; GFX908-NEXT: v_mov_b32_e32 v1, s25 ; GFX908-NEXT: v_mov_b32_e32 v2, s26 @@ -68,7 +66,8 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vgpr(ptr addrspace(1) %arg) ; GFX908-NEXT: v_mov_b32_e32 v0, s10 ; GFX908-NEXT: v_mov_b32_e32 v1, s11 ; GFX908-NEXT: v_mov_b32_e32 v2, s12 -; GFX908-NEXT: v_mov_b32_e32 v5, s20 +; GFX908-NEXT: v_mov_b32_e32 v4, s20 +; GFX908-NEXT: v_mov_b32_e32 v5, s21 ; GFX908-NEXT: v_accvgpr_write_b32 a3, v3 ; GFX908-NEXT: v_accvgpr_write_b32 a26, v0 ; GFX908-NEXT: v_accvgpr_write_b32 a27, v1 @@ -77,7 +76,8 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vgpr(ptr addrspace(1) %arg) ; GFX908-NEXT: v_mov_b32_e32 v1, s14 ; GFX908-NEXT: v_mov_b32_e32 v2, s15 ; GFX908-NEXT: v_mov_b32_e32 v3, 1.0 -; GFX908-NEXT: v_accvgpr_write_b32 a4, v5 +; GFX908-NEXT: v_accvgpr_write_b32 a4, v4 +; GFX908-NEXT: v_accvgpr_write_b32 a5, v5 ; GFX908-NEXT: v_accvgpr_write_b32 a29, v0 ; GFX908-NEXT: v_accvgpr_write_b32 a30, v1 ; GFX908-NEXT: v_accvgpr_write_b32 a31, v2 @@ -90,57 +90,42 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_vgpr(ptr addrspace(1) %arg) ; GFX908-NEXT: v_accvgpr_read_b32 v2, a26 ; GFX908-NEXT: v_accvgpr_read_b32 v1, a25 ; GFX908-NEXT: v_accvgpr_read_b32 v0, a24 -; GFX908-NEXT: s_nop 1 -; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:96 -; GFX908-NEXT: s_nop 0 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a31 -; GFX908-NEXT: v_accvgpr_read_b32 v2, a30 -; GFX908-NEXT: v_accvgpr_read_b32 v1, a29 -; GFX908-NEXT: v_accvgpr_read_b32 v0, a28 -; GFX908-NEXT: s_nop 1 -; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:112 -; GFX908-NEXT: s_nop 0 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a19 -; GFX908-NEXT: v_accvgpr_read_b32 v2, a18 -; GFX908-NEXT: v_accvgpr_read_b32 v1, a17 -; GFX908-NEXT: v_accvgpr_read_b32 v0, a16 -; GFX908-NEXT: s_nop 1 -; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:64 -; GFX908-NEXT: s_nop 0 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a23 -; GFX908-NEXT: v_accvgpr_read_b32 v2, a22 -; GFX908-NEXT: v_accvgpr_read_b32 v1, a21 -; GFX908-NEXT: v_accvgpr_read_b32 v0, a20 -; GFX908-NEXT: s_nop 1 -; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:80 -; GFX908-NEXT: s_nop 0 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a11 -; GFX908-NEXT: v_accvgpr_read_b32 v2, a10 -; GFX908-NEXT: v_accvgpr_read_b32 v1, a9 -; GFX908-NEXT: v_accvgpr_read_b32 v0, a8 -; GFX908-NEXT: s_nop 1 -; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:32 -; GFX908-NEXT: s_nop 0 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a15 -; GFX908-NEXT: v_accvgpr_read_b32 v2, a14 -; GFX908-NEXT: v_accvgpr_read_b32 v1, a13 -; GFX908-NEXT: v_accvgpr_read_b32 v0, a12 -; GFX908-NEXT: s_nop 1 -; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:48 -; GFX908-NEXT: s_nop 0 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a3 -; GFX908-NEXT: v_accvgpr_read_b32 v2, a2 -; GFX908-NEXT: v_accvgpr_read_b32 v1, a1 -; GFX908-NEXT: v_accvgpr_read_b32 v0, a0 -; GFX908-NEXT: s_nop 1 -; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] -; GFX908-NEXT: s_nop 0 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a7 -; GFX908-NEXT: v_accvgpr_read_b32 v2, a6 -; GFX908-NEXT: v_accvgpr_read_b32 v1, a5 -; GFX908-NEXT: v_accvgpr_read_b32 v0, a4 -; GFX908-NEXT: s_nop 1 -; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:16 +; GFX908-NEXT: v_accvgpr_read_b32 v4, a31 +; GFX908-NEXT: v_accvgpr_read_b32 v8, a19 +; GFX908-NEXT: global_store_dwordx4 v29, v[0:3], s[34:35] offset:96 +; GFX908-NEXT: v_accvgpr_read_b32 v12, a23 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a30 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a29 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a28 +; GFX908-NEXT: v_accvgpr_read_b32 v16, a11 +; GFX908-NEXT: v_accvgpr_read_b32 v20, a15 +; GFX908-NEXT: v_accvgpr_read_b32 v24, a3 +; GFX908-NEXT: v_accvgpr_read_b32 v28, a7 +; GFX908-NEXT: v_accvgpr_read_b32 v7, a18 +; GFX908-NEXT: v_accvgpr_read_b32 v6, a17 +; GFX908-NEXT: v_accvgpr_read_b32 v5, a16 +; GFX908-NEXT: v_accvgpr_read_b32 v11, a22 +; GFX908-NEXT: v_accvgpr_read_b32 v10, a21 +; GFX908-NEXT: v_accvgpr_read_b32 v9, a20 +; GFX908-NEXT: v_accvgpr_read_b32 v15, a10 +; GFX908-NEXT: v_accvgpr_read_b32 v14, a9 +; GFX908-NEXT: v_accvgpr_read_b32 v13, a8 +; GFX908-NEXT: v_accvgpr_read_b32 v19, a14 +; GFX908-NEXT: v_accvgpr_read_b32 v18, a13 +; GFX908-NEXT: v_accvgpr_read_b32 v17, a12 +; GFX908-NEXT: v_accvgpr_read_b32 v23, a2 +; GFX908-NEXT: v_accvgpr_read_b32 v22, a1 +; GFX908-NEXT: v_accvgpr_read_b32 v21, a0 +; GFX908-NEXT: v_accvgpr_read_b32 v27, a6 +; GFX908-NEXT: v_accvgpr_read_b32 v26, a5 +; GFX908-NEXT: v_accvgpr_read_b32 v25, a4 +; GFX908-NEXT: global_store_dwordx4 v29, v[1:4], s[34:35] offset:112 +; GFX908-NEXT: global_store_dwordx4 v29, v[5:8], s[34:35] offset:64 +; GFX908-NEXT: global_store_dwordx4 v29, v[9:12], s[34:35] offset:80 +; GFX908-NEXT: global_store_dwordx4 v29, v[13:16], s[34:35] offset:32 +; GFX908-NEXT: global_store_dwordx4 v29, v[17:20], s[34:35] offset:48 +; GFX908-NEXT: global_store_dwordx4 v29, v[21:24], s[34:35] +; GFX908-NEXT: global_store_dwordx4 v29, v[25:28], s[34:35] offset:16 ; GFX908-NEXT: s_endpgm bb: %in.1 = load <32 x float>, ptr addrspace(1) %arg @@ -153,7 +138,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_agpr(ptr addrspace(1) %arg) ; GFX908-LABEL: test_mfma_f32_32x32x1f32_agpr: ; GFX908: ; %bb.0: ; %bb ; GFX908-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24 -; GFX908-NEXT: v_mov_b32_e32 v4, 0 +; GFX908-NEXT: v_mov_b32_e32 v29, 0 ; GFX908-NEXT: s_waitcnt lgkmcnt(0) ; GFX908-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0 ; GFX908-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40 @@ -163,13 +148,11 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_agpr(ptr addrspace(1) %arg) ; GFX908-NEXT: v_mov_b32_e32 v2, s18 ; GFX908-NEXT: v_accvgpr_write_b32 a0, v0 ; GFX908-NEXT: v_accvgpr_write_b32 a1, v1 +; GFX908-NEXT: v_mov_b32_e32 v0, s22 +; GFX908-NEXT: v_mov_b32_e32 v1, s23 ; GFX908-NEXT: v_accvgpr_write_b32 a2, v2 -; GFX908-NEXT: v_mov_b32_e32 v0, s21 -; GFX908-NEXT: v_mov_b32_e32 v1, s22 -; GFX908-NEXT: v_mov_b32_e32 v2, s23 -; GFX908-NEXT: v_accvgpr_write_b32 a5, v0 -; GFX908-NEXT: v_accvgpr_write_b32 a6, v1 -; GFX908-NEXT: v_accvgpr_write_b32 a7, v2 +; GFX908-NEXT: v_accvgpr_write_b32 a6, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a7, v1 ; GFX908-NEXT: v_mov_b32_e32 v0, s24 ; GFX908-NEXT: v_mov_b32_e32 v1, s25 ; GFX908-NEXT: v_mov_b32_e32 v2, s26 @@ -210,7 +193,8 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_agpr(ptr addrspace(1) %arg) ; GFX908-NEXT: v_mov_b32_e32 v0, s10 ; GFX908-NEXT: v_mov_b32_e32 v1, s11 ; GFX908-NEXT: v_mov_b32_e32 v2, s12 -; GFX908-NEXT: v_mov_b32_e32 v5, s20 +; GFX908-NEXT: v_mov_b32_e32 v4, s20 +; GFX908-NEXT: v_mov_b32_e32 v5, s21 ; GFX908-NEXT: v_accvgpr_write_b32 a3, v3 ; GFX908-NEXT: v_accvgpr_write_b32 a26, v0 ; GFX908-NEXT: v_accvgpr_write_b32 a27, v1 @@ -219,7 +203,8 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_agpr(ptr addrspace(1) %arg) ; GFX908-NEXT: v_mov_b32_e32 v1, s14 ; GFX908-NEXT: v_mov_b32_e32 v2, s15 ; GFX908-NEXT: v_mov_b32_e32 v3, 1.0 -; GFX908-NEXT: v_accvgpr_write_b32 a4, v5 +; GFX908-NEXT: v_accvgpr_write_b32 a4, v4 +; GFX908-NEXT: v_accvgpr_write_b32 a5, v5 ; GFX908-NEXT: v_accvgpr_write_b32 a29, v0 ; GFX908-NEXT: v_accvgpr_write_b32 a30, v1 ; GFX908-NEXT: v_accvgpr_write_b32 a31, v2 @@ -232,57 +217,42 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_agpr(ptr addrspace(1) %arg) ; GFX908-NEXT: v_accvgpr_read_b32 v2, a26 ; GFX908-NEXT: v_accvgpr_read_b32 v1, a25 ; GFX908-NEXT: v_accvgpr_read_b32 v0, a24 -; GFX908-NEXT: s_nop 1 -; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:96 -; GFX908-NEXT: s_nop 0 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a31 -; GFX908-NEXT: v_accvgpr_read_b32 v2, a30 -; GFX908-NEXT: v_accvgpr_read_b32 v1, a29 -; GFX908-NEXT: v_accvgpr_read_b32 v0, a28 -; GFX908-NEXT: s_nop 1 -; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:112 -; GFX908-NEXT: s_nop 0 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a19 -; GFX908-NEXT: v_accvgpr_read_b32 v2, a18 -; GFX908-NEXT: v_accvgpr_read_b32 v1, a17 -; GFX908-NEXT: v_accvgpr_read_b32 v0, a16 -; GFX908-NEXT: s_nop 1 -; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:64 -; GFX908-NEXT: s_nop 0 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a23 -; GFX908-NEXT: v_accvgpr_read_b32 v2, a22 -; GFX908-NEXT: v_accvgpr_read_b32 v1, a21 -; GFX908-NEXT: v_accvgpr_read_b32 v0, a20 -; GFX908-NEXT: s_nop 1 -; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:80 -; GFX908-NEXT: s_nop 0 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a11 -; GFX908-NEXT: v_accvgpr_read_b32 v2, a10 -; GFX908-NEXT: v_accvgpr_read_b32 v1, a9 -; GFX908-NEXT: v_accvgpr_read_b32 v0, a8 -; GFX908-NEXT: s_nop 1 -; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:32 -; GFX908-NEXT: s_nop 0 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a15 -; GFX908-NEXT: v_accvgpr_read_b32 v2, a14 -; GFX908-NEXT: v_accvgpr_read_b32 v1, a13 -; GFX908-NEXT: v_accvgpr_read_b32 v0, a12 -; GFX908-NEXT: s_nop 1 -; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:48 -; GFX908-NEXT: s_nop 0 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a3 -; GFX908-NEXT: v_accvgpr_read_b32 v2, a2 -; GFX908-NEXT: v_accvgpr_read_b32 v1, a1 -; GFX908-NEXT: v_accvgpr_read_b32 v0, a0 -; GFX908-NEXT: s_nop 1 -; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] -; GFX908-NEXT: s_nop 0 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a7 -; GFX908-NEXT: v_accvgpr_read_b32 v2, a6 -; GFX908-NEXT: v_accvgpr_read_b32 v1, a5 -; GFX908-NEXT: v_accvgpr_read_b32 v0, a4 -; GFX908-NEXT: s_nop 1 -; GFX908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:16 +; GFX908-NEXT: v_accvgpr_read_b32 v4, a31 +; GFX908-NEXT: v_accvgpr_read_b32 v8, a19 +; GFX908-NEXT: global_store_dwordx4 v29, v[0:3], s[34:35] offset:96 +; GFX908-NEXT: v_accvgpr_read_b32 v12, a23 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a30 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a29 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a28 +; GFX908-NEXT: v_accvgpr_read_b32 v16, a11 +; GFX908-NEXT: v_accvgpr_read_b32 v20, a15 +; GFX908-NEXT: v_accvgpr_read_b32 v24, a3 +; GFX908-NEXT: v_accvgpr_read_b32 v28, a7 +; GFX908-NEXT: v_accvgpr_read_b32 v7, a18 +; GFX908-NEXT: v_accvgpr_read_b32 v6, a17 +; GFX908-NEXT: v_accvgpr_read_b32 v5, a16 +; GFX908-NEXT: v_accvgpr_read_b32 v11, a22 +; GFX908-NEXT: v_accvgpr_read_b32 v10, a21 +; GFX908-NEXT: v_accvgpr_read_b32 v9, a20 +; GFX908-NEXT: v_accvgpr_read_b32 v15, a10 +; GFX908-NEXT: v_accvgpr_read_b32 v14, a9 +; GFX908-NEXT: v_accvgpr_read_b32 v13, a8 +; GFX908-NEXT: v_accvgpr_read_b32 v19, a14 +; GFX908-NEXT: v_accvgpr_read_b32 v18, a13 +; GFX908-NEXT: v_accvgpr_read_b32 v17, a12 +; GFX908-NEXT: v_accvgpr_read_b32 v23, a2 +; GFX908-NEXT: v_accvgpr_read_b32 v22, a1 +; GFX908-NEXT: v_accvgpr_read_b32 v21, a0 +; GFX908-NEXT: v_accvgpr_read_b32 v27, a6 +; GFX908-NEXT: v_accvgpr_read_b32 v26, a5 +; GFX908-NEXT: v_accvgpr_read_b32 v25, a4 +; GFX908-NEXT: global_store_dwordx4 v29, v[1:4], s[34:35] offset:112 +; GFX908-NEXT: global_store_dwordx4 v29, v[5:8], s[34:35] offset:64 +; GFX908-NEXT: global_store_dwordx4 v29, v[9:12], s[34:35] offset:80 +; GFX908-NEXT: global_store_dwordx4 v29, v[13:16], s[34:35] offset:32 +; GFX908-NEXT: global_store_dwordx4 v29, v[17:20], s[34:35] offset:48 +; GFX908-NEXT: global_store_dwordx4 v29, v[21:24], s[34:35] +; GFX908-NEXT: global_store_dwordx4 v29, v[25:28], s[34:35] offset:16 ; GFX908-NEXT: s_endpgm bb: %in.1 = load <32 x float>, ptr addrspace(1) %arg @@ -351,43 +321,42 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_inline_asm_virtual_agpr(ptr ; GFX908-NEXT: v_accvgpr_read_b32 v2, a26 ; GFX908-NEXT: v_accvgpr_read_b32 v1, a25 ; GFX908-NEXT: v_accvgpr_read_b32 v0, a24 -; GFX908-NEXT: v_accvgpr_read_b32 v7, a31 -; GFX908-NEXT: v_accvgpr_read_b32 v6, a30 -; GFX908-NEXT: v_accvgpr_read_b32 v5, a29 -; GFX908-NEXT: v_accvgpr_read_b32 v4, a28 -; GFX908-NEXT: v_accvgpr_read_b32 v11, a19 -; GFX908-NEXT: v_accvgpr_read_b32 v10, a18 -; GFX908-NEXT: v_accvgpr_read_b32 v9, a17 -; GFX908-NEXT: v_accvgpr_read_b32 v8, a16 -; GFX908-NEXT: v_accvgpr_read_b32 v15, a23 -; GFX908-NEXT: v_accvgpr_read_b32 v14, a22 -; GFX908-NEXT: v_accvgpr_read_b32 v13, a21 -; GFX908-NEXT: v_accvgpr_read_b32 v12, a20 -; GFX908-NEXT: v_accvgpr_read_b32 v19, a11 -; GFX908-NEXT: v_accvgpr_read_b32 v18, a10 -; GFX908-NEXT: v_accvgpr_read_b32 v17, a9 -; GFX908-NEXT: v_accvgpr_read_b32 v16, a8 -; GFX908-NEXT: v_accvgpr_read_b32 v23, a15 -; GFX908-NEXT: v_accvgpr_read_b32 v22, a14 -; GFX908-NEXT: v_accvgpr_read_b32 v21, a13 -; GFX908-NEXT: v_accvgpr_read_b32 v20, a12 -; GFX908-NEXT: v_accvgpr_read_b32 v27, a3 -; GFX908-NEXT: v_accvgpr_read_b32 v26, a2 -; GFX908-NEXT: v_accvgpr_read_b32 v25, a1 -; GFX908-NEXT: v_accvgpr_read_b32 v24, a0 +; GFX908-NEXT: v_accvgpr_read_b32 v4, a31 +; GFX908-NEXT: v_accvgpr_read_b32 v8, a19 ; GFX908-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:96 -; GFX908-NEXT: s_nop 0 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a7 -; GFX908-NEXT: v_accvgpr_read_b32 v2, a6 -; GFX908-NEXT: v_accvgpr_read_b32 v1, a5 -; GFX908-NEXT: v_accvgpr_read_b32 v0, a4 -; GFX908-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:112 -; GFX908-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:64 -; GFX908-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:80 -; GFX908-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 -; GFX908-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:48 -; GFX908-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] -; GFX908-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:16 +; GFX908-NEXT: v_accvgpr_read_b32 v12, a23 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a30 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a29 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a28 +; GFX908-NEXT: v_accvgpr_read_b32 v16, a11 +; GFX908-NEXT: v_accvgpr_read_b32 v20, a15 +; GFX908-NEXT: v_accvgpr_read_b32 v24, a3 +; GFX908-NEXT: v_accvgpr_read_b32 v28, a7 +; GFX908-NEXT: v_accvgpr_read_b32 v7, a18 +; GFX908-NEXT: v_accvgpr_read_b32 v6, a17 +; GFX908-NEXT: v_accvgpr_read_b32 v5, a16 +; GFX908-NEXT: v_accvgpr_read_b32 v11, a22 +; GFX908-NEXT: v_accvgpr_read_b32 v10, a21 +; GFX908-NEXT: v_accvgpr_read_b32 v9, a20 +; GFX908-NEXT: v_accvgpr_read_b32 v15, a10 +; GFX908-NEXT: v_accvgpr_read_b32 v14, a9 +; GFX908-NEXT: v_accvgpr_read_b32 v13, a8 +; GFX908-NEXT: v_accvgpr_read_b32 v19, a14 +; GFX908-NEXT: v_accvgpr_read_b32 v18, a13 +; GFX908-NEXT: v_accvgpr_read_b32 v17, a12 +; GFX908-NEXT: v_accvgpr_read_b32 v23, a2 +; GFX908-NEXT: v_accvgpr_read_b32 v22, a1 +; GFX908-NEXT: v_accvgpr_read_b32 v21, a0 +; GFX908-NEXT: v_accvgpr_read_b32 v27, a6 +; GFX908-NEXT: v_accvgpr_read_b32 v26, a5 +; GFX908-NEXT: v_accvgpr_read_b32 v25, a4 +; GFX908-NEXT: global_store_dwordx4 v32, v[1:4], s[0:1] offset:112 +; GFX908-NEXT: global_store_dwordx4 v32, v[5:8], s[0:1] offset:64 +; GFX908-NEXT: global_store_dwordx4 v32, v[9:12], s[0:1] offset:80 +; GFX908-NEXT: global_store_dwordx4 v32, v[13:16], s[0:1] offset:32 +; GFX908-NEXT: global_store_dwordx4 v32, v[17:20], s[0:1] offset:48 +; GFX908-NEXT: global_store_dwordx4 v32, v[21:24], s[0:1] +; GFX908-NEXT: global_store_dwordx4 v32, v[25:28], s[0:1] offset:16 ; GFX908-NEXT: s_endpgm bb: %acc = call i32 asm sideeffect "; def $0", "={a0}"() @@ -458,43 +427,42 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_inline_asm_phys_agpr(ptr add ; GFX908-NEXT: v_accvgpr_read_b32 v2, a26 ; GFX908-NEXT: v_accvgpr_read_b32 v1, a25 ; GFX908-NEXT: v_accvgpr_read_b32 v0, a24 -; GFX908-NEXT: v_accvgpr_read_b32 v7, a31 -; GFX908-NEXT: v_accvgpr_read_b32 v6, a30 -; GFX908-NEXT: v_accvgpr_read_b32 v5, a29 -; GFX908-NEXT: v_accvgpr_read_b32 v4, a28 -; GFX908-NEXT: v_accvgpr_read_b32 v11, a19 -; GFX908-NEXT: v_accvgpr_read_b32 v10, a18 -; GFX908-NEXT: v_accvgpr_read_b32 v9, a17 -; GFX908-NEXT: v_accvgpr_read_b32 v8, a16 -; GFX908-NEXT: v_accvgpr_read_b32 v15, a23 -; GFX908-NEXT: v_accvgpr_read_b32 v14, a22 -; GFX908-NEXT: v_accvgpr_read_b32 v13, a21 -; GFX908-NEXT: v_accvgpr_read_b32 v12, a20 -; GFX908-NEXT: v_accvgpr_read_b32 v19, a11 -; GFX908-NEXT: v_accvgpr_read_b32 v18, a10 -; GFX908-NEXT: v_accvgpr_read_b32 v17, a9 -; GFX908-NEXT: v_accvgpr_read_b32 v16, a8 -; GFX908-NEXT: v_accvgpr_read_b32 v23, a15 -; GFX908-NEXT: v_accvgpr_read_b32 v22, a14 -; GFX908-NEXT: v_accvgpr_read_b32 v21, a13 -; GFX908-NEXT: v_accvgpr_read_b32 v20, a12 -; GFX908-NEXT: v_accvgpr_read_b32 v27, a3 -; GFX908-NEXT: v_accvgpr_read_b32 v26, a2 -; GFX908-NEXT: v_accvgpr_read_b32 v25, a1 -; GFX908-NEXT: v_accvgpr_read_b32 v24, a0 +; GFX908-NEXT: v_accvgpr_read_b32 v4, a31 +; GFX908-NEXT: v_accvgpr_read_b32 v8, a19 ; GFX908-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:96 -; GFX908-NEXT: s_nop 0 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a7 -; GFX908-NEXT: v_accvgpr_read_b32 v2, a6 -; GFX908-NEXT: v_accvgpr_read_b32 v1, a5 -; GFX908-NEXT: v_accvgpr_read_b32 v0, a4 -; GFX908-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:112 -; GFX908-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:64 -; GFX908-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:80 -; GFX908-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 -; GFX908-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:48 -; GFX908-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] -; GFX908-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:16 +; GFX908-NEXT: v_accvgpr_read_b32 v12, a23 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a30 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a29 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a28 +; GFX908-NEXT: v_accvgpr_read_b32 v16, a11 +; GFX908-NEXT: v_accvgpr_read_b32 v20, a15 +; GFX908-NEXT: v_accvgpr_read_b32 v24, a3 +; GFX908-NEXT: v_accvgpr_read_b32 v28, a7 +; GFX908-NEXT: v_accvgpr_read_b32 v7, a18 +; GFX908-NEXT: v_accvgpr_read_b32 v6, a17 +; GFX908-NEXT: v_accvgpr_read_b32 v5, a16 +; GFX908-NEXT: v_accvgpr_read_b32 v11, a22 +; GFX908-NEXT: v_accvgpr_read_b32 v10, a21 +; GFX908-NEXT: v_accvgpr_read_b32 v9, a20 +; GFX908-NEXT: v_accvgpr_read_b32 v15, a10 +; GFX908-NEXT: v_accvgpr_read_b32 v14, a9 +; GFX908-NEXT: v_accvgpr_read_b32 v13, a8 +; GFX908-NEXT: v_accvgpr_read_b32 v19, a14 +; GFX908-NEXT: v_accvgpr_read_b32 v18, a13 +; GFX908-NEXT: v_accvgpr_read_b32 v17, a12 +; GFX908-NEXT: v_accvgpr_read_b32 v23, a2 +; GFX908-NEXT: v_accvgpr_read_b32 v22, a1 +; GFX908-NEXT: v_accvgpr_read_b32 v21, a0 +; GFX908-NEXT: v_accvgpr_read_b32 v27, a6 +; GFX908-NEXT: v_accvgpr_read_b32 v26, a5 +; GFX908-NEXT: v_accvgpr_read_b32 v25, a4 +; GFX908-NEXT: global_store_dwordx4 v32, v[1:4], s[0:1] offset:112 +; GFX908-NEXT: global_store_dwordx4 v32, v[5:8], s[0:1] offset:64 +; GFX908-NEXT: global_store_dwordx4 v32, v[9:12], s[0:1] offset:80 +; GFX908-NEXT: global_store_dwordx4 v32, v[13:16], s[0:1] offset:32 +; GFX908-NEXT: global_store_dwordx4 v32, v[17:20], s[0:1] offset:48 +; GFX908-NEXT: global_store_dwordx4 v32, v[21:24], s[0:1] +; GFX908-NEXT: global_store_dwordx4 v32, v[25:28], s[0:1] offset:16 ; GFX908-NEXT: s_endpgm bb: call void asm sideeffect "; use $0", "{a[100:131]}"(<32 x float> poison) @@ -564,43 +532,42 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_inline_asm_no_agprs(ptr addr ; GFX908-NEXT: v_accvgpr_read_b32 v2, a26 ; GFX908-NEXT: v_accvgpr_read_b32 v1, a25 ; GFX908-NEXT: v_accvgpr_read_b32 v0, a24 -; GFX908-NEXT: v_accvgpr_read_b32 v7, a31 -; GFX908-NEXT: v_accvgpr_read_b32 v6, a30 -; GFX908-NEXT: v_accvgpr_read_b32 v5, a29 -; GFX908-NEXT: v_accvgpr_read_b32 v4, a28 -; GFX908-NEXT: v_accvgpr_read_b32 v11, a19 -; GFX908-NEXT: v_accvgpr_read_b32 v10, a18 -; GFX908-NEXT: v_accvgpr_read_b32 v9, a17 -; GFX908-NEXT: v_accvgpr_read_b32 v8, a16 -; GFX908-NEXT: v_accvgpr_read_b32 v15, a23 -; GFX908-NEXT: v_accvgpr_read_b32 v14, a22 -; GFX908-NEXT: v_accvgpr_read_b32 v13, a21 -; GFX908-NEXT: v_accvgpr_read_b32 v12, a20 -; GFX908-NEXT: v_accvgpr_read_b32 v19, a11 -; GFX908-NEXT: v_accvgpr_read_b32 v18, a10 -; GFX908-NEXT: v_accvgpr_read_b32 v17, a9 -; GFX908-NEXT: v_accvgpr_read_b32 v16, a8 -; GFX908-NEXT: v_accvgpr_read_b32 v23, a15 -; GFX908-NEXT: v_accvgpr_read_b32 v22, a14 -; GFX908-NEXT: v_accvgpr_read_b32 v21, a13 -; GFX908-NEXT: v_accvgpr_read_b32 v20, a12 -; GFX908-NEXT: v_accvgpr_read_b32 v27, a3 -; GFX908-NEXT: v_accvgpr_read_b32 v26, a2 -; GFX908-NEXT: v_accvgpr_read_b32 v25, a1 -; GFX908-NEXT: v_accvgpr_read_b32 v24, a0 +; GFX908-NEXT: v_accvgpr_read_b32 v4, a31 +; GFX908-NEXT: v_accvgpr_read_b32 v8, a19 ; GFX908-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:96 -; GFX908-NEXT: s_nop 0 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a7 -; GFX908-NEXT: v_accvgpr_read_b32 v2, a6 -; GFX908-NEXT: v_accvgpr_read_b32 v1, a5 -; GFX908-NEXT: v_accvgpr_read_b32 v0, a4 -; GFX908-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:112 -; GFX908-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:64 -; GFX908-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:80 -; GFX908-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:32 -; GFX908-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:48 -; GFX908-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] -; GFX908-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1] offset:16 +; GFX908-NEXT: v_accvgpr_read_b32 v12, a23 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a30 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a29 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a28 +; GFX908-NEXT: v_accvgpr_read_b32 v16, a11 +; GFX908-NEXT: v_accvgpr_read_b32 v20, a15 +; GFX908-NEXT: v_accvgpr_read_b32 v24, a3 +; GFX908-NEXT: v_accvgpr_read_b32 v28, a7 +; GFX908-NEXT: v_accvgpr_read_b32 v7, a18 +; GFX908-NEXT: v_accvgpr_read_b32 v6, a17 +; GFX908-NEXT: v_accvgpr_read_b32 v5, a16 +; GFX908-NEXT: v_accvgpr_read_b32 v11, a22 +; GFX908-NEXT: v_accvgpr_read_b32 v10, a21 +; GFX908-NEXT: v_accvgpr_read_b32 v9, a20 +; GFX908-NEXT: v_accvgpr_read_b32 v15, a10 +; GFX908-NEXT: v_accvgpr_read_b32 v14, a9 +; GFX908-NEXT: v_accvgpr_read_b32 v13, a8 +; GFX908-NEXT: v_accvgpr_read_b32 v19, a14 +; GFX908-NEXT: v_accvgpr_read_b32 v18, a13 +; GFX908-NEXT: v_accvgpr_read_b32 v17, a12 +; GFX908-NEXT: v_accvgpr_read_b32 v23, a2 +; GFX908-NEXT: v_accvgpr_read_b32 v22, a1 +; GFX908-NEXT: v_accvgpr_read_b32 v21, a0 +; GFX908-NEXT: v_accvgpr_read_b32 v27, a6 +; GFX908-NEXT: v_accvgpr_read_b32 v26, a5 +; GFX908-NEXT: v_accvgpr_read_b32 v25, a4 +; GFX908-NEXT: global_store_dwordx4 v32, v[1:4], s[0:1] offset:112 +; GFX908-NEXT: global_store_dwordx4 v32, v[5:8], s[0:1] offset:64 +; GFX908-NEXT: global_store_dwordx4 v32, v[9:12], s[0:1] offset:80 +; GFX908-NEXT: global_store_dwordx4 v32, v[13:16], s[0:1] offset:32 +; GFX908-NEXT: global_store_dwordx4 v32, v[17:20], s[0:1] offset:48 +; GFX908-NEXT: global_store_dwordx4 v32, v[21:24], s[0:1] +; GFX908-NEXT: global_store_dwordx4 v32, v[25:28], s[0:1] offset:16 ; GFX908-NEXT: s_endpgm bb: %acc = call i32 asm sideeffect "; def $0", "={v0}"() @@ -693,43 +660,42 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_call(ptr addrspace(1) %arg) ; GFX908-NEXT: v_accvgpr_read_b32 v2, a26 ; GFX908-NEXT: v_accvgpr_read_b32 v1, a25 ; GFX908-NEXT: v_accvgpr_read_b32 v0, a24 -; GFX908-NEXT: v_accvgpr_read_b32 v7, a31 -; GFX908-NEXT: v_accvgpr_read_b32 v6, a30 -; GFX908-NEXT: v_accvgpr_read_b32 v5, a29 -; GFX908-NEXT: v_accvgpr_read_b32 v4, a28 -; GFX908-NEXT: v_accvgpr_read_b32 v11, a19 -; GFX908-NEXT: v_accvgpr_read_b32 v10, a18 -; GFX908-NEXT: v_accvgpr_read_b32 v9, a17 -; GFX908-NEXT: v_accvgpr_read_b32 v8, a16 -; GFX908-NEXT: v_accvgpr_read_b32 v15, a23 -; GFX908-NEXT: v_accvgpr_read_b32 v14, a22 -; GFX908-NEXT: v_accvgpr_read_b32 v13, a21 -; GFX908-NEXT: v_accvgpr_read_b32 v12, a20 -; GFX908-NEXT: v_accvgpr_read_b32 v19, a11 -; GFX908-NEXT: v_accvgpr_read_b32 v18, a10 -; GFX908-NEXT: v_accvgpr_read_b32 v17, a9 -; GFX908-NEXT: v_accvgpr_read_b32 v16, a8 -; GFX908-NEXT: v_accvgpr_read_b32 v23, a15 -; GFX908-NEXT: v_accvgpr_read_b32 v22, a14 -; GFX908-NEXT: v_accvgpr_read_b32 v21, a13 -; GFX908-NEXT: v_accvgpr_read_b32 v20, a12 -; GFX908-NEXT: v_accvgpr_read_b32 v27, a3 -; GFX908-NEXT: v_accvgpr_read_b32 v26, a2 -; GFX908-NEXT: v_accvgpr_read_b32 v25, a1 -; GFX908-NEXT: v_accvgpr_read_b32 v24, a0 +; GFX908-NEXT: v_accvgpr_read_b32 v4, a31 +; GFX908-NEXT: v_accvgpr_read_b32 v8, a19 ; GFX908-NEXT: global_store_dwordx4 v40, v[0:3], s[34:35] offset:96 -; GFX908-NEXT: s_nop 0 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a7 -; GFX908-NEXT: v_accvgpr_read_b32 v2, a6 -; GFX908-NEXT: v_accvgpr_read_b32 v1, a5 -; GFX908-NEXT: v_accvgpr_read_b32 v0, a4 -; GFX908-NEXT: global_store_dwordx4 v40, v[4:7], s[34:35] offset:112 -; GFX908-NEXT: global_store_dwordx4 v40, v[8:11], s[34:35] offset:64 -; GFX908-NEXT: global_store_dwordx4 v40, v[12:15], s[34:35] offset:80 -; GFX908-NEXT: global_store_dwordx4 v40, v[16:19], s[34:35] offset:32 -; GFX908-NEXT: global_store_dwordx4 v40, v[20:23], s[34:35] offset:48 -; GFX908-NEXT: global_store_dwordx4 v40, v[24:27], s[34:35] -; GFX908-NEXT: global_store_dwordx4 v40, v[0:3], s[34:35] offset:16 +; GFX908-NEXT: v_accvgpr_read_b32 v12, a23 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a30 +; GFX908-NEXT: v_accvgpr_read_b32 v2, a29 +; GFX908-NEXT: v_accvgpr_read_b32 v1, a28 +; GFX908-NEXT: v_accvgpr_read_b32 v16, a11 +; GFX908-NEXT: v_accvgpr_read_b32 v20, a15 +; GFX908-NEXT: v_accvgpr_read_b32 v24, a3 +; GFX908-NEXT: v_accvgpr_read_b32 v28, a7 +; GFX908-NEXT: v_accvgpr_read_b32 v7, a18 +; GFX908-NEXT: v_accvgpr_read_b32 v6, a17 +; GFX908-NEXT: v_accvgpr_read_b32 v5, a16 +; GFX908-NEXT: v_accvgpr_read_b32 v11, a22 +; GFX908-NEXT: v_accvgpr_read_b32 v10, a21 +; GFX908-NEXT: v_accvgpr_read_b32 v9, a20 +; GFX908-NEXT: v_accvgpr_read_b32 v15, a10 +; GFX908-NEXT: v_accvgpr_read_b32 v14, a9 +; GFX908-NEXT: v_accvgpr_read_b32 v13, a8 +; GFX908-NEXT: v_accvgpr_read_b32 v19, a14 +; GFX908-NEXT: v_accvgpr_read_b32 v18, a13 +; GFX908-NEXT: v_accvgpr_read_b32 v17, a12 +; GFX908-NEXT: v_accvgpr_read_b32 v23, a2 +; GFX908-NEXT: v_accvgpr_read_b32 v22, a1 +; GFX908-NEXT: v_accvgpr_read_b32 v21, a0 +; GFX908-NEXT: v_accvgpr_read_b32 v27, a6 +; GFX908-NEXT: v_accvgpr_read_b32 v26, a5 +; GFX908-NEXT: v_accvgpr_read_b32 v25, a4 +; GFX908-NEXT: global_store_dwordx4 v40, v[1:4], s[34:35] offset:112 +; GFX908-NEXT: global_store_dwordx4 v40, v[5:8], s[34:35] offset:64 +; GFX908-NEXT: global_store_dwordx4 v40, v[9:12], s[34:35] offset:80 +; GFX908-NEXT: global_store_dwordx4 v40, v[13:16], s[34:35] offset:32 +; GFX908-NEXT: global_store_dwordx4 v40, v[17:20], s[34:35] offset:48 +; GFX908-NEXT: global_store_dwordx4 v40, v[21:24], s[34:35] +; GFX908-NEXT: global_store_dwordx4 v40, v[25:28], s[34:35] offset:16 ; GFX908-NEXT: s_endpgm bb: call void @foo() @@ -756,8 +722,8 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_call_multi_bb(ptr addrspace( ; GFX908-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24 ; GFX908-NEXT: s_load_dword s8, s[4:5], 0x2c ; GFX908-NEXT: v_mov_b32_e32 v6, 1.0 -; GFX908-NEXT: v_mov_b32_e32 v7, 0 ; GFX908-NEXT: s_addc_u32 s53, s53, 0 +; GFX908-NEXT: v_mov_b32_e32 v32, 0 ; GFX908-NEXT: s_waitcnt lgkmcnt(0) ; GFX908-NEXT: s_load_dwordx16 s[36:51], s[6:7], 0x0 ; GFX908-NEXT: s_load_dwordx16 s[16:31], s[6:7], 0x40 @@ -837,57 +803,42 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_call_multi_bb(ptr addrspace( ; GFX908-NEXT: v_accvgpr_read_b32 v5, a26 ; GFX908-NEXT: v_accvgpr_read_b32 v4, a25 ; GFX908-NEXT: v_accvgpr_read_b32 v3, a24 -; GFX908-NEXT: s_nop 1 -; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[6:7] offset:96 -; GFX908-NEXT: s_nop 0 -; GFX908-NEXT: v_accvgpr_read_b32 v6, a31 -; GFX908-NEXT: v_accvgpr_read_b32 v5, a30 -; GFX908-NEXT: v_accvgpr_read_b32 v4, a29 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a28 -; GFX908-NEXT: s_nop 1 -; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[6:7] offset:112 -; GFX908-NEXT: s_nop 0 -; GFX908-NEXT: v_accvgpr_read_b32 v6, a19 -; GFX908-NEXT: v_accvgpr_read_b32 v5, a18 -; GFX908-NEXT: v_accvgpr_read_b32 v4, a17 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a16 -; GFX908-NEXT: s_nop 1 -; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[6:7] offset:64 -; GFX908-NEXT: s_nop 0 -; GFX908-NEXT: v_accvgpr_read_b32 v6, a23 -; GFX908-NEXT: v_accvgpr_read_b32 v5, a22 -; GFX908-NEXT: v_accvgpr_read_b32 v4, a21 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a20 -; GFX908-NEXT: s_nop 1 -; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[6:7] offset:80 -; GFX908-NEXT: s_nop 0 -; GFX908-NEXT: v_accvgpr_read_b32 v6, a11 -; GFX908-NEXT: v_accvgpr_read_b32 v5, a10 -; GFX908-NEXT: v_accvgpr_read_b32 v4, a9 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a8 -; GFX908-NEXT: s_nop 1 -; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[6:7] offset:32 -; GFX908-NEXT: s_nop 0 -; GFX908-NEXT: v_accvgpr_read_b32 v6, a15 -; GFX908-NEXT: v_accvgpr_read_b32 v5, a14 -; GFX908-NEXT: v_accvgpr_read_b32 v4, a13 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a12 -; GFX908-NEXT: s_nop 1 -; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[6:7] offset:48 -; GFX908-NEXT: s_nop 0 -; GFX908-NEXT: v_accvgpr_read_b32 v6, a3 -; GFX908-NEXT: v_accvgpr_read_b32 v5, a2 -; GFX908-NEXT: v_accvgpr_read_b32 v4, a1 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a0 -; GFX908-NEXT: s_nop 1 -; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[6:7] -; GFX908-NEXT: s_nop 0 -; GFX908-NEXT: v_accvgpr_read_b32 v6, a7 -; GFX908-NEXT: v_accvgpr_read_b32 v5, a6 -; GFX908-NEXT: v_accvgpr_read_b32 v4, a5 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a4 -; GFX908-NEXT: s_nop 1 -; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[6:7] offset:16 +; GFX908-NEXT: v_accvgpr_read_b32 v7, a31 +; GFX908-NEXT: v_accvgpr_read_b32 v31, a7 +; GFX908-NEXT: global_store_dwordx4 v32, v[3:6], s[6:7] offset:96 +; GFX908-NEXT: v_accvgpr_read_b32 v30, a6 +; GFX908-NEXT: v_accvgpr_read_b32 v6, a30 +; GFX908-NEXT: v_accvgpr_read_b32 v5, a29 +; GFX908-NEXT: v_accvgpr_read_b32 v4, a28 +; GFX908-NEXT: v_accvgpr_read_b32 v29, a5 +; GFX908-NEXT: v_accvgpr_read_b32 v28, a4 +; GFX908-NEXT: v_accvgpr_read_b32 v11, a19 +; GFX908-NEXT: v_accvgpr_read_b32 v15, a23 +; GFX908-NEXT: v_accvgpr_read_b32 v19, a11 +; GFX908-NEXT: v_accvgpr_read_b32 v23, a15 +; GFX908-NEXT: v_accvgpr_read_b32 v27, a3 +; GFX908-NEXT: v_accvgpr_read_b32 v10, a18 +; GFX908-NEXT: v_accvgpr_read_b32 v9, a17 +; GFX908-NEXT: v_accvgpr_read_b32 v8, a16 +; GFX908-NEXT: v_accvgpr_read_b32 v14, a22 +; GFX908-NEXT: v_accvgpr_read_b32 v13, a21 +; GFX908-NEXT: v_accvgpr_read_b32 v12, a20 +; GFX908-NEXT: v_accvgpr_read_b32 v18, a10 +; GFX908-NEXT: v_accvgpr_read_b32 v17, a9 +; GFX908-NEXT: v_accvgpr_read_b32 v16, a8 +; GFX908-NEXT: v_accvgpr_read_b32 v22, a14 +; GFX908-NEXT: v_accvgpr_read_b32 v21, a13 +; GFX908-NEXT: v_accvgpr_read_b32 v20, a12 +; GFX908-NEXT: v_accvgpr_read_b32 v26, a2 +; GFX908-NEXT: v_accvgpr_read_b32 v25, a1 +; GFX908-NEXT: v_accvgpr_read_b32 v24, a0 +; GFX908-NEXT: global_store_dwordx4 v32, v[4:7], s[6:7] offset:112 +; GFX908-NEXT: global_store_dwordx4 v32, v[8:11], s[6:7] offset:64 +; GFX908-NEXT: global_store_dwordx4 v32, v[12:15], s[6:7] offset:80 +; GFX908-NEXT: global_store_dwordx4 v32, v[16:19], s[6:7] offset:32 +; GFX908-NEXT: global_store_dwordx4 v32, v[20:23], s[6:7] offset:48 +; GFX908-NEXT: global_store_dwordx4 v32, v[24:27], s[6:7] +; GFX908-NEXT: global_store_dwordx4 v32, v[28:31], s[6:7] offset:16 ; GFX908-NEXT: s_cbranch_scc1 .LBB6_2 ; GFX908-NEXT: ; %bb.1: ; %bb2 ; GFX908-NEXT: s_add_u32 s8, s4, 48 @@ -979,43 +930,42 @@ define void @test_mfma_f32_32x32x1f32_nonentry_noagpr(ptr addrspace(1) %arg) #0 ; GFX908-NEXT: v_accvgpr_read_b32 v4, a26 ; GFX908-NEXT: v_accvgpr_read_b32 v3, a25 ; GFX908-NEXT: v_accvgpr_read_b32 v2, a24 -; GFX908-NEXT: v_accvgpr_read_b32 v9, a31 -; GFX908-NEXT: v_accvgpr_read_b32 v8, a30 -; GFX908-NEXT: v_accvgpr_read_b32 v7, a29 -; GFX908-NEXT: v_accvgpr_read_b32 v6, a28 -; GFX908-NEXT: v_accvgpr_read_b32 v13, a19 -; GFX908-NEXT: v_accvgpr_read_b32 v12, a18 -; GFX908-NEXT: v_accvgpr_read_b32 v11, a17 -; GFX908-NEXT: v_accvgpr_read_b32 v10, a16 -; GFX908-NEXT: v_accvgpr_read_b32 v17, a23 -; GFX908-NEXT: v_accvgpr_read_b32 v16, a22 -; GFX908-NEXT: v_accvgpr_read_b32 v15, a21 -; GFX908-NEXT: v_accvgpr_read_b32 v14, a20 -; GFX908-NEXT: v_accvgpr_read_b32 v21, a11 -; GFX908-NEXT: v_accvgpr_read_b32 v20, a10 -; GFX908-NEXT: v_accvgpr_read_b32 v19, a9 -; GFX908-NEXT: v_accvgpr_read_b32 v18, a8 -; GFX908-NEXT: v_accvgpr_read_b32 v25, a15 -; GFX908-NEXT: v_accvgpr_read_b32 v24, a14 -; GFX908-NEXT: v_accvgpr_read_b32 v23, a13 -; GFX908-NEXT: v_accvgpr_read_b32 v22, a12 -; GFX908-NEXT: v_accvgpr_read_b32 v29, a3 -; GFX908-NEXT: v_accvgpr_read_b32 v28, a2 -; GFX908-NEXT: v_accvgpr_read_b32 v27, a1 -; GFX908-NEXT: v_accvgpr_read_b32 v26, a0 +; GFX908-NEXT: v_accvgpr_read_b32 v6, a31 +; GFX908-NEXT: v_accvgpr_read_b32 v10, a19 ; GFX908-NEXT: global_store_dwordx4 v[0:1], v[2:5], off offset:96 -; GFX908-NEXT: s_nop 0 -; GFX908-NEXT: v_accvgpr_read_b32 v5, a7 -; GFX908-NEXT: v_accvgpr_read_b32 v4, a6 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a5 -; GFX908-NEXT: v_accvgpr_read_b32 v2, a4 -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[6:9], off offset:112 -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[10:13], off offset:64 -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[14:17], off offset:80 -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[18:21], off offset:32 -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[22:25], off offset:48 -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[26:29], off -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[2:5], off offset:16 +; GFX908-NEXT: v_accvgpr_read_b32 v14, a23 +; GFX908-NEXT: v_accvgpr_read_b32 v5, a30 +; GFX908-NEXT: v_accvgpr_read_b32 v4, a29 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a28 +; GFX908-NEXT: v_accvgpr_read_b32 v18, a11 +; GFX908-NEXT: v_accvgpr_read_b32 v22, a15 +; GFX908-NEXT: v_accvgpr_read_b32 v26, a3 +; GFX908-NEXT: v_accvgpr_read_b32 v30, a7 +; GFX908-NEXT: v_accvgpr_read_b32 v9, a18 +; GFX908-NEXT: v_accvgpr_read_b32 v8, a17 +; GFX908-NEXT: v_accvgpr_read_b32 v7, a16 +; GFX908-NEXT: v_accvgpr_read_b32 v13, a22 +; GFX908-NEXT: v_accvgpr_read_b32 v12, a21 +; GFX908-NEXT: v_accvgpr_read_b32 v11, a20 +; GFX908-NEXT: v_accvgpr_read_b32 v17, a10 +; GFX908-NEXT: v_accvgpr_read_b32 v16, a9 +; GFX908-NEXT: v_accvgpr_read_b32 v15, a8 +; GFX908-NEXT: v_accvgpr_read_b32 v21, a14 +; GFX908-NEXT: v_accvgpr_read_b32 v20, a13 +; GFX908-NEXT: v_accvgpr_read_b32 v19, a12 +; GFX908-NEXT: v_accvgpr_read_b32 v25, a2 +; GFX908-NEXT: v_accvgpr_read_b32 v24, a1 +; GFX908-NEXT: v_accvgpr_read_b32 v23, a0 +; GFX908-NEXT: v_accvgpr_read_b32 v29, a6 +; GFX908-NEXT: v_accvgpr_read_b32 v28, a5 +; GFX908-NEXT: v_accvgpr_read_b32 v27, a4 +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[3:6], off offset:112 +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[7:10], off offset:64 +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[11:14], off offset:80 +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[15:18], off offset:32 +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[19:22], off offset:48 +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[23:26], off +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[27:30], off offset:16 ; GFX908-NEXT: s_waitcnt vmcnt(0) ; GFX908-NEXT: s_setpc_b64 s[30:31] bb: @@ -1080,43 +1030,42 @@ define void @test_mfma_f32_32x32x1f32_nonentry_with_agpr(ptr addrspace(1) %arg) ; GFX908-NEXT: v_accvgpr_read_b32 v4, a26 ; GFX908-NEXT: v_accvgpr_read_b32 v3, a25 ; GFX908-NEXT: v_accvgpr_read_b32 v2, a24 -; GFX908-NEXT: v_accvgpr_read_b32 v9, a31 -; GFX908-NEXT: v_accvgpr_read_b32 v8, a30 -; GFX908-NEXT: v_accvgpr_read_b32 v7, a29 -; GFX908-NEXT: v_accvgpr_read_b32 v6, a28 -; GFX908-NEXT: v_accvgpr_read_b32 v13, a19 -; GFX908-NEXT: v_accvgpr_read_b32 v12, a18 -; GFX908-NEXT: v_accvgpr_read_b32 v11, a17 -; GFX908-NEXT: v_accvgpr_read_b32 v10, a16 -; GFX908-NEXT: v_accvgpr_read_b32 v17, a23 -; GFX908-NEXT: v_accvgpr_read_b32 v16, a22 -; GFX908-NEXT: v_accvgpr_read_b32 v15, a21 -; GFX908-NEXT: v_accvgpr_read_b32 v14, a20 -; GFX908-NEXT: v_accvgpr_read_b32 v21, a11 -; GFX908-NEXT: v_accvgpr_read_b32 v20, a10 -; GFX908-NEXT: v_accvgpr_read_b32 v19, a9 -; GFX908-NEXT: v_accvgpr_read_b32 v18, a8 -; GFX908-NEXT: v_accvgpr_read_b32 v25, a15 -; GFX908-NEXT: v_accvgpr_read_b32 v24, a14 -; GFX908-NEXT: v_accvgpr_read_b32 v23, a13 -; GFX908-NEXT: v_accvgpr_read_b32 v22, a12 -; GFX908-NEXT: v_accvgpr_read_b32 v29, a3 -; GFX908-NEXT: v_accvgpr_read_b32 v28, a2 -; GFX908-NEXT: v_accvgpr_read_b32 v27, a1 -; GFX908-NEXT: v_accvgpr_read_b32 v26, a0 +; GFX908-NEXT: v_accvgpr_read_b32 v6, a31 +; GFX908-NEXT: v_accvgpr_read_b32 v10, a19 ; GFX908-NEXT: global_store_dwordx4 v[0:1], v[2:5], off offset:96 -; GFX908-NEXT: s_nop 0 -; GFX908-NEXT: v_accvgpr_read_b32 v5, a7 -; GFX908-NEXT: v_accvgpr_read_b32 v4, a6 -; GFX908-NEXT: v_accvgpr_read_b32 v3, a5 -; GFX908-NEXT: v_accvgpr_read_b32 v2, a4 -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[6:9], off offset:112 -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[10:13], off offset:64 -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[14:17], off offset:80 -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[18:21], off offset:32 -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[22:25], off offset:48 -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[26:29], off -; GFX908-NEXT: global_store_dwordx4 v[0:1], v[2:5], off offset:16 +; GFX908-NEXT: v_accvgpr_read_b32 v14, a23 +; GFX908-NEXT: v_accvgpr_read_b32 v5, a30 +; GFX908-NEXT: v_accvgpr_read_b32 v4, a29 +; GFX908-NEXT: v_accvgpr_read_b32 v3, a28 +; GFX908-NEXT: v_accvgpr_read_b32 v18, a11 +; GFX908-NEXT: v_accvgpr_read_b32 v22, a15 +; GFX908-NEXT: v_accvgpr_read_b32 v26, a3 +; GFX908-NEXT: v_accvgpr_read_b32 v30, a7 +; GFX908-NEXT: v_accvgpr_read_b32 v9, a18 +; GFX908-NEXT: v_accvgpr_read_b32 v8, a17 +; GFX908-NEXT: v_accvgpr_read_b32 v7, a16 +; GFX908-NEXT: v_accvgpr_read_b32 v13, a22 +; GFX908-NEXT: v_accvgpr_read_b32 v12, a21 +; GFX908-NEXT: v_accvgpr_read_b32 v11, a20 +; GFX908-NEXT: v_accvgpr_read_b32 v17, a10 +; GFX908-NEXT: v_accvgpr_read_b32 v16, a9 +; GFX908-NEXT: v_accvgpr_read_b32 v15, a8 +; GFX908-NEXT: v_accvgpr_read_b32 v21, a14 +; GFX908-NEXT: v_accvgpr_read_b32 v20, a13 +; GFX908-NEXT: v_accvgpr_read_b32 v19, a12 +; GFX908-NEXT: v_accvgpr_read_b32 v25, a2 +; GFX908-NEXT: v_accvgpr_read_b32 v24, a1 +; GFX908-NEXT: v_accvgpr_read_b32 v23, a0 +; GFX908-NEXT: v_accvgpr_read_b32 v29, a6 +; GFX908-NEXT: v_accvgpr_read_b32 v28, a5 +; GFX908-NEXT: v_accvgpr_read_b32 v27, a4 +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[3:6], off offset:112 +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[7:10], off offset:64 +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[11:14], off offset:80 +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[15:18], off offset:32 +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[19:22], off offset:48 +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[23:26], off +; GFX908-NEXT: global_store_dwordx4 v[0:1], v[27:30], off offset:16 ; GFX908-NEXT: s_waitcnt vmcnt(0) ; GFX908-NEXT: s_setpc_b64 s[30:31] bb: diff --git a/llvm/test/CodeGen/AMDGPU/mfma-no-register-aliasing.ll b/llvm/test/CodeGen/AMDGPU/mfma-no-register-aliasing.ll index 07d04ea789a81..7f787568b5a0f 100644 --- a/llvm/test/CodeGen/AMDGPU/mfma-no-register-aliasing.ll +++ b/llvm/test/CodeGen/AMDGPU/mfma-no-register-aliasing.ll @@ -19,6 +19,7 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32(ptr addrspace(1) %arg) #0 { ; GREEDY908-LABEL: test_mfma_f32_32x32x1f32: ; GREEDY908: ; %bb.0: ; %bb ; GREEDY908-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x24 +; GREEDY908-NEXT: v_mov_b32_e32 v32, 0 ; GREEDY908-NEXT: s_waitcnt lgkmcnt(0) ; GREEDY908-NEXT: s_load_dwordx16 s[16:31], s[34:35], 0x0 ; GREEDY908-NEXT: s_load_dwordx16 s[0:15], s[34:35], 0x40 @@ -89,71 +90,70 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32(ptr addrspace(1) %arg) #0 { ; GREEDY908-NEXT: v_accvgpr_write_b32 a30, v1 ; GREEDY908-NEXT: v_accvgpr_write_b32 a31, v2 ; GREEDY908-NEXT: v_mov_b32_e32 v0, 2.0 -; GREEDY908-NEXT: v_mov_b32_e32 v4, 0 -; GREEDY908-NEXT: s_nop 0 +; GREEDY908-NEXT: s_nop 1 ; GREEDY908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v3, v0, a[0:31] ; GREEDY908-NEXT: v_mfma_f32_32x32x1f32 a[32:63], v3, v0, a[0:31] ; GREEDY908-NEXT: s_nop 15 ; GREEDY908-NEXT: s_nop 1 ; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a32 -; GREEDY908-NEXT: v_accvgpr_read_b32 v6, a33 +; GREEDY908-NEXT: v_accvgpr_read_b32 v33, a33 ; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a34 ; GREEDY908-NEXT: v_accvgpr_write_b32 a2, v2 -; GREEDY908-NEXT: v_accvgpr_write_b32 a3, v6 +; GREEDY908-NEXT: v_accvgpr_write_b32 a3, v33 ; GREEDY908-NEXT: v_accvgpr_write_b32 a4, v1 ; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a35 -; GREEDY908-NEXT: v_accvgpr_read_b32 v6, a36 +; GREEDY908-NEXT: v_accvgpr_read_b32 v33, a36 ; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a37 ; GREEDY908-NEXT: v_accvgpr_write_b32 a5, v2 -; GREEDY908-NEXT: v_accvgpr_write_b32 a6, v6 +; GREEDY908-NEXT: v_accvgpr_write_b32 a6, v33 ; GREEDY908-NEXT: v_accvgpr_write_b32 a7, v1 ; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a38 -; GREEDY908-NEXT: v_accvgpr_read_b32 v6, a39 +; GREEDY908-NEXT: v_accvgpr_read_b32 v33, a39 ; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a40 ; GREEDY908-NEXT: v_accvgpr_write_b32 a8, v2 -; GREEDY908-NEXT: v_accvgpr_write_b32 a9, v6 +; GREEDY908-NEXT: v_accvgpr_write_b32 a9, v33 ; GREEDY908-NEXT: v_accvgpr_write_b32 a10, v1 ; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a41 -; GREEDY908-NEXT: v_accvgpr_read_b32 v6, a42 +; GREEDY908-NEXT: v_accvgpr_read_b32 v33, a42 ; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a43 ; GREEDY908-NEXT: v_accvgpr_write_b32 a11, v2 -; GREEDY908-NEXT: v_accvgpr_write_b32 a12, v6 +; GREEDY908-NEXT: v_accvgpr_write_b32 a12, v33 ; GREEDY908-NEXT: v_accvgpr_write_b32 a13, v1 ; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a44 -; GREEDY908-NEXT: v_accvgpr_read_b32 v6, a45 +; GREEDY908-NEXT: v_accvgpr_read_b32 v33, a45 ; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a46 ; GREEDY908-NEXT: v_accvgpr_write_b32 a14, v2 -; GREEDY908-NEXT: v_accvgpr_write_b32 a15, v6 +; GREEDY908-NEXT: v_accvgpr_write_b32 a15, v33 ; GREEDY908-NEXT: v_accvgpr_write_b32 a16, v1 ; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a47 -; GREEDY908-NEXT: v_accvgpr_read_b32 v6, a48 +; GREEDY908-NEXT: v_accvgpr_read_b32 v33, a48 ; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a49 ; GREEDY908-NEXT: v_accvgpr_write_b32 a17, v2 -; GREEDY908-NEXT: v_accvgpr_write_b32 a18, v6 +; GREEDY908-NEXT: v_accvgpr_write_b32 a18, v33 ; GREEDY908-NEXT: v_accvgpr_write_b32 a19, v1 ; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a50 -; GREEDY908-NEXT: v_accvgpr_read_b32 v6, a51 +; GREEDY908-NEXT: v_accvgpr_read_b32 v33, a51 ; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a52 ; GREEDY908-NEXT: v_accvgpr_write_b32 a20, v2 -; GREEDY908-NEXT: v_accvgpr_write_b32 a21, v6 +; GREEDY908-NEXT: v_accvgpr_write_b32 a21, v33 ; GREEDY908-NEXT: v_accvgpr_write_b32 a22, v1 ; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a53 -; GREEDY908-NEXT: v_accvgpr_read_b32 v6, a54 +; GREEDY908-NEXT: v_accvgpr_read_b32 v33, a54 ; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a55 ; GREEDY908-NEXT: v_accvgpr_write_b32 a23, v2 -; GREEDY908-NEXT: v_accvgpr_write_b32 a24, v6 +; GREEDY908-NEXT: v_accvgpr_write_b32 a24, v33 ; GREEDY908-NEXT: v_accvgpr_write_b32 a25, v1 ; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a56 -; GREEDY908-NEXT: v_accvgpr_read_b32 v6, a57 +; GREEDY908-NEXT: v_accvgpr_read_b32 v33, a57 ; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a58 ; GREEDY908-NEXT: v_accvgpr_write_b32 a26, v2 -; GREEDY908-NEXT: v_accvgpr_write_b32 a27, v6 +; GREEDY908-NEXT: v_accvgpr_write_b32 a27, v33 ; GREEDY908-NEXT: v_accvgpr_write_b32 a28, v1 ; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a59 -; GREEDY908-NEXT: v_accvgpr_read_b32 v6, a60 +; GREEDY908-NEXT: v_accvgpr_read_b32 v33, a60 ; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a61 ; GREEDY908-NEXT: v_accvgpr_write_b32 a29, v2 -; GREEDY908-NEXT: v_accvgpr_write_b32 a30, v6 +; GREEDY908-NEXT: v_accvgpr_write_b32 a30, v33 ; GREEDY908-NEXT: v_accvgpr_write_b32 a31, v1 ; GREEDY908-NEXT: s_nop 0 ; GREEDY908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v3, v0, a[0:31] @@ -163,57 +163,42 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32(ptr addrspace(1) %arg) #0 { ; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a26 ; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a25 ; GREEDY908-NEXT: v_accvgpr_read_b32 v0, a24 -; GREEDY908-NEXT: s_nop 1 -; GREEDY908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:96 -; GREEDY908-NEXT: s_nop 0 -; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a31 -; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a30 -; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a29 -; GREEDY908-NEXT: v_accvgpr_read_b32 v0, a28 -; GREEDY908-NEXT: s_nop 1 -; GREEDY908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:112 -; GREEDY908-NEXT: s_nop 0 -; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a19 -; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a18 -; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a17 -; GREEDY908-NEXT: v_accvgpr_read_b32 v0, a16 -; GREEDY908-NEXT: s_nop 1 -; GREEDY908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:64 -; GREEDY908-NEXT: s_nop 0 -; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a23 -; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a22 -; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a21 -; GREEDY908-NEXT: v_accvgpr_read_b32 v0, a20 -; GREEDY908-NEXT: s_nop 1 -; GREEDY908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:80 -; GREEDY908-NEXT: s_nop 0 -; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a11 -; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a10 -; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a9 -; GREEDY908-NEXT: v_accvgpr_read_b32 v0, a8 -; GREEDY908-NEXT: s_nop 1 -; GREEDY908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:32 -; GREEDY908-NEXT: s_nop 0 -; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a15 -; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a14 -; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a13 -; GREEDY908-NEXT: v_accvgpr_read_b32 v0, a12 -; GREEDY908-NEXT: s_nop 1 -; GREEDY908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:48 -; GREEDY908-NEXT: s_nop 0 -; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a3 -; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a2 -; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a1 -; GREEDY908-NEXT: v_accvgpr_read_b32 v0, a0 -; GREEDY908-NEXT: s_nop 1 -; GREEDY908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] -; GREEDY908-NEXT: s_nop 0 -; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a7 -; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a6 -; GREEDY908-NEXT: v_accvgpr_read_b32 v1, a5 -; GREEDY908-NEXT: v_accvgpr_read_b32 v0, a4 -; GREEDY908-NEXT: s_nop 1 -; GREEDY908-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35] offset:16 +; GREEDY908-NEXT: v_accvgpr_read_b32 v7, a31 +; GREEDY908-NEXT: v_accvgpr_read_b32 v11, a19 +; GREEDY908-NEXT: v_accvgpr_read_b32 v15, a23 +; GREEDY908-NEXT: v_accvgpr_read_b32 v19, a11 +; GREEDY908-NEXT: v_accvgpr_read_b32 v23, a15 +; GREEDY908-NEXT: v_accvgpr_read_b32 v27, a3 +; GREEDY908-NEXT: v_accvgpr_read_b32 v31, a7 +; GREEDY908-NEXT: v_accvgpr_read_b32 v6, a30 +; GREEDY908-NEXT: v_accvgpr_read_b32 v5, a29 +; GREEDY908-NEXT: v_accvgpr_read_b32 v4, a28 +; GREEDY908-NEXT: v_accvgpr_read_b32 v10, a18 +; GREEDY908-NEXT: v_accvgpr_read_b32 v9, a17 +; GREEDY908-NEXT: v_accvgpr_read_b32 v8, a16 +; GREEDY908-NEXT: v_accvgpr_read_b32 v14, a22 +; GREEDY908-NEXT: v_accvgpr_read_b32 v13, a21 +; GREEDY908-NEXT: v_accvgpr_read_b32 v12, a20 +; GREEDY908-NEXT: v_accvgpr_read_b32 v18, a10 +; GREEDY908-NEXT: v_accvgpr_read_b32 v17, a9 +; GREEDY908-NEXT: v_accvgpr_read_b32 v16, a8 +; GREEDY908-NEXT: v_accvgpr_read_b32 v22, a14 +; GREEDY908-NEXT: v_accvgpr_read_b32 v21, a13 +; GREEDY908-NEXT: v_accvgpr_read_b32 v20, a12 +; GREEDY908-NEXT: v_accvgpr_read_b32 v26, a2 +; GREEDY908-NEXT: v_accvgpr_read_b32 v25, a1 +; GREEDY908-NEXT: v_accvgpr_read_b32 v24, a0 +; GREEDY908-NEXT: v_accvgpr_read_b32 v30, a6 +; GREEDY908-NEXT: v_accvgpr_read_b32 v29, a5 +; GREEDY908-NEXT: v_accvgpr_read_b32 v28, a4 +; GREEDY908-NEXT: global_store_dwordx4 v32, v[0:3], s[34:35] offset:96 +; GREEDY908-NEXT: global_store_dwordx4 v32, v[4:7], s[34:35] offset:112 +; GREEDY908-NEXT: global_store_dwordx4 v32, v[8:11], s[34:35] offset:64 +; GREEDY908-NEXT: global_store_dwordx4 v32, v[12:15], s[34:35] offset:80 +; GREEDY908-NEXT: global_store_dwordx4 v32, v[16:19], s[34:35] offset:32 +; GREEDY908-NEXT: global_store_dwordx4 v32, v[20:23], s[34:35] offset:48 +; GREEDY908-NEXT: global_store_dwordx4 v32, v[24:27], s[34:35] +; GREEDY908-NEXT: global_store_dwordx4 v32, v[28:31], s[34:35] offset:16 ; GREEDY908-NEXT: s_endpgm ; ; GREEDY90A-LABEL: test_mfma_f32_32x32x1f32: @@ -582,48 +567,71 @@ define amdgpu_kernel void @test_mfma_f32_16x16x1f32(ptr addrspace(1) %arg) #0 { ; GREEDY908-NEXT: s_waitcnt lgkmcnt(0) ; GREEDY908-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0 ; GREEDY908-NEXT: s_waitcnt lgkmcnt(0) -; GREEDY908-NEXT: v_mov_b32_e32 v5, s15 -; GREEDY908-NEXT: v_mov_b32_e32 v2, s14 -; GREEDY908-NEXT: v_mov_b32_e32 v1, s13 -; GREEDY908-NEXT: v_accvgpr_write_b32 a33, v5 -; GREEDY908-NEXT: v_mov_b32_e32 v5, s12 -; GREEDY908-NEXT: v_accvgpr_write_b32 a32, v2 -; GREEDY908-NEXT: v_accvgpr_write_b32 a31, v1 -; GREEDY908-NEXT: v_accvgpr_write_b32 a30, v5 -; GREEDY908-NEXT: v_mov_b32_e32 v2, s11 -; GREEDY908-NEXT: v_mov_b32_e32 v1, s10 -; GREEDY908-NEXT: v_mov_b32_e32 v5, s9 -; GREEDY908-NEXT: v_accvgpr_write_b32 a29, v2 -; GREEDY908-NEXT: v_accvgpr_write_b32 a28, v1 -; GREEDY908-NEXT: v_accvgpr_write_b32 a27, v5 -; GREEDY908-NEXT: v_mov_b32_e32 v2, s8 -; GREEDY908-NEXT: v_mov_b32_e32 v1, s7 -; GREEDY908-NEXT: v_mov_b32_e32 v5, s6 -; GREEDY908-NEXT: v_accvgpr_write_b32 a26, v2 -; GREEDY908-NEXT: v_accvgpr_write_b32 a25, v1 -; GREEDY908-NEXT: v_accvgpr_write_b32 a24, v5 -; GREEDY908-NEXT: v_mov_b32_e32 v2, s5 -; GREEDY908-NEXT: v_mov_b32_e32 v1, s4 -; GREEDY908-NEXT: v_mov_b32_e32 v5, s3 -; GREEDY908-NEXT: v_accvgpr_write_b32 a23, v2 -; GREEDY908-NEXT: v_accvgpr_write_b32 a22, v1 -; GREEDY908-NEXT: v_accvgpr_write_b32 a21, v5 -; GREEDY908-NEXT: v_mov_b32_e32 v2, s2 -; GREEDY908-NEXT: v_mov_b32_e32 v1, s1 ; GREEDY908-NEXT: v_mov_b32_e32 v5, s0 -; GREEDY908-NEXT: v_accvgpr_write_b32 a20, v2 -; GREEDY908-NEXT: v_accvgpr_write_b32 a19, v1 -; GREEDY908-NEXT: v_accvgpr_write_b32 a18, v5 +; GREEDY908-NEXT: v_mov_b32_e32 v1, s1 +; GREEDY908-NEXT: v_mov_b32_e32 v2, s2 +; GREEDY908-NEXT: v_accvgpr_write_b32 a0, v5 +; GREEDY908-NEXT: v_mov_b32_e32 v5, s3 +; GREEDY908-NEXT: v_accvgpr_write_b32 a1, v1 +; GREEDY908-NEXT: v_accvgpr_write_b32 a2, v2 +; GREEDY908-NEXT: v_accvgpr_write_b32 a3, v5 +; GREEDY908-NEXT: v_mov_b32_e32 v1, s4 +; GREEDY908-NEXT: v_mov_b32_e32 v2, s5 +; GREEDY908-NEXT: v_mov_b32_e32 v5, s6 +; GREEDY908-NEXT: v_accvgpr_write_b32 a4, v1 +; GREEDY908-NEXT: v_accvgpr_write_b32 a5, v2 +; GREEDY908-NEXT: v_accvgpr_write_b32 a6, v5 +; GREEDY908-NEXT: v_mov_b32_e32 v1, s7 +; GREEDY908-NEXT: v_mov_b32_e32 v2, s8 +; GREEDY908-NEXT: v_mov_b32_e32 v5, s9 +; GREEDY908-NEXT: v_accvgpr_write_b32 a7, v1 +; GREEDY908-NEXT: v_accvgpr_write_b32 a8, v2 +; GREEDY908-NEXT: v_accvgpr_write_b32 a9, v5 +; GREEDY908-NEXT: v_mov_b32_e32 v1, s10 +; GREEDY908-NEXT: v_mov_b32_e32 v2, s11 +; GREEDY908-NEXT: v_mov_b32_e32 v5, s12 +; GREEDY908-NEXT: v_accvgpr_write_b32 a10, v1 +; GREEDY908-NEXT: v_accvgpr_write_b32 a11, v2 +; GREEDY908-NEXT: v_accvgpr_write_b32 a12, v5 +; GREEDY908-NEXT: v_mov_b32_e32 v1, s13 +; GREEDY908-NEXT: v_mov_b32_e32 v2, s14 +; GREEDY908-NEXT: v_mov_b32_e32 v5, s15 +; GREEDY908-NEXT: v_accvgpr_write_b32 a13, v1 +; GREEDY908-NEXT: v_accvgpr_write_b32 a14, v2 +; GREEDY908-NEXT: v_accvgpr_write_b32 a15, v5 ; GREEDY908-NEXT: v_mov_b32_e32 v1, 2.0 ; GREEDY908-NEXT: s_nop 1 -; GREEDY908-NEXT: v_mfma_f32_16x16x1f32 a[18:33], v0, v1, a[18:33] -; GREEDY908-NEXT: v_mfma_f32_16x16x1f32 a[2:17], v0, v1, a[18:33] -; GREEDY908-NEXT: s_nop 8 -; GREEDY908-NEXT: v_accvgpr_read_b32 v5, a18 -; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a19 -; GREEDY908-NEXT: s_nop 0 -; GREEDY908-NEXT: v_accvgpr_write_b32 a0, v5 -; GREEDY908-NEXT: v_accvgpr_write_b32 a1, v2 +; GREEDY908-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v0, v1, a[0:15] +; GREEDY908-NEXT: v_mfma_f32_16x16x1f32 a[16:31], v0, v1, a[0:15] +; GREEDY908-NEXT: s_nop 9 +; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a16 +; GREEDY908-NEXT: v_accvgpr_read_b32 v5, a17 +; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a18 +; GREEDY908-NEXT: v_accvgpr_write_b32 a2, v3 +; GREEDY908-NEXT: v_accvgpr_write_b32 a3, v5 +; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a19 +; GREEDY908-NEXT: v_accvgpr_read_b32 v5, a20 +; GREEDY908-NEXT: v_accvgpr_write_b32 a4, v2 +; GREEDY908-NEXT: v_accvgpr_write_b32 a5, v3 +; GREEDY908-NEXT: v_accvgpr_write_b32 a6, v5 +; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a21 +; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a22 +; GREEDY908-NEXT: v_accvgpr_read_b32 v5, a23 +; GREEDY908-NEXT: v_accvgpr_write_b32 a7, v2 +; GREEDY908-NEXT: v_accvgpr_write_b32 a8, v3 +; GREEDY908-NEXT: v_accvgpr_write_b32 a9, v5 +; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a24 +; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a25 +; GREEDY908-NEXT: v_accvgpr_read_b32 v5, a26 +; GREEDY908-NEXT: v_accvgpr_write_b32 a10, v2 +; GREEDY908-NEXT: v_accvgpr_write_b32 a11, v3 +; GREEDY908-NEXT: v_accvgpr_write_b32 a12, v5 +; GREEDY908-NEXT: v_accvgpr_read_b32 v2, a27 +; GREEDY908-NEXT: v_accvgpr_read_b32 v3, a28 +; GREEDY908-NEXT: v_accvgpr_read_b32 v5, a29 +; GREEDY908-NEXT: v_accvgpr_write_b32 a13, v2 +; GREEDY908-NEXT: v_accvgpr_write_b32 a14, v3 +; GREEDY908-NEXT: v_accvgpr_write_b32 a15, v5 ; GREEDY908-NEXT: s_nop 0 ; GREEDY908-NEXT: v_mfma_f32_16x16x1f32 a[0:15], v0, v1, a[0:15] ; GREEDY908-NEXT: s_nop 9 diff --git a/llvm/test/CodeGen/AMDGPU/spill-agpr.ll b/llvm/test/CodeGen/AMDGPU/spill-agpr.ll index ea6216ac4ac40..1ca643d90aac0 100644 --- a/llvm/test/CodeGen/AMDGPU/spill-agpr.ll +++ b/llvm/test/CodeGen/AMDGPU/spill-agpr.ll @@ -191,82 +191,81 @@ define amdgpu_kernel void @max_10_vgprs_used_9a() #1 { define amdgpu_kernel void @max_32regs_mfma32(ptr addrspace(1) %arg) #3 { ; GFX908-LABEL: max_32regs_mfma32: ; GFX908: ; %bb.0: ; %bb -; GFX908-NEXT: v_mov_b32_e32 v1, 0x40400000 -; GFX908-NEXT: v_mov_b32_e32 v2, 0x40c00000 -; GFX908-NEXT: v_mov_b32_e32 v3, 0x40e00000 -; GFX908-NEXT: v_accvgpr_write_b32 a2, v1 -; GFX908-NEXT: v_mov_b32_e32 v1, 0x40a00000 -; GFX908-NEXT: v_accvgpr_write_b32 a5, v2 -; GFX908-NEXT: v_accvgpr_write_b32 a6, v3 -; GFX908-NEXT: v_accvgpr_write_b32 a4, v1 -; GFX908-NEXT: v_mov_b32_e32 v1, 0x41000000 -; GFX908-NEXT: v_mov_b32_e32 v2, 0x41100000 -; GFX908-NEXT: v_mov_b32_e32 v3, 0x41200000 -; GFX908-NEXT: v_accvgpr_write_b32 a7, v1 -; GFX908-NEXT: v_accvgpr_write_b32 a8, v2 -; GFX908-NEXT: v_accvgpr_write_b32 a9, v3 -; GFX908-NEXT: v_mov_b32_e32 v1, 0x41300000 -; GFX908-NEXT: v_mov_b32_e32 v2, 0x41400000 -; GFX908-NEXT: v_mov_b32_e32 v3, 0x41500000 -; GFX908-NEXT: v_accvgpr_write_b32 a10, v1 -; GFX908-NEXT: v_accvgpr_write_b32 a11, v2 -; GFX908-NEXT: v_accvgpr_write_b32 a12, v3 -; GFX908-NEXT: v_mov_b32_e32 v1, 0x41600000 -; GFX908-NEXT: v_mov_b32_e32 v2, 0x41700000 -; GFX908-NEXT: v_mov_b32_e32 v3, 0x41800000 -; GFX908-NEXT: v_accvgpr_write_b32 a13, v1 -; GFX908-NEXT: v_accvgpr_write_b32 a14, v2 -; GFX908-NEXT: v_accvgpr_write_b32 a15, v3 -; GFX908-NEXT: v_mov_b32_e32 v1, 0x41880000 -; GFX908-NEXT: v_mov_b32_e32 v2, 0x41900000 -; GFX908-NEXT: v_mov_b32_e32 v3, 0x41980000 -; GFX908-NEXT: v_accvgpr_write_b32 a16, v1 -; GFX908-NEXT: v_accvgpr_write_b32 a17, v2 -; GFX908-NEXT: v_accvgpr_write_b32 a18, v3 -; GFX908-NEXT: v_mov_b32_e32 v1, 0x41a00000 -; GFX908-NEXT: v_mov_b32_e32 v2, 0x41a80000 -; GFX908-NEXT: v_mov_b32_e32 v3, 0x41b00000 -; GFX908-NEXT: v_accvgpr_write_b32 a19, v1 -; GFX908-NEXT: v_accvgpr_write_b32 a20, v2 -; GFX908-NEXT: v_accvgpr_write_b32 a21, v3 -; GFX908-NEXT: v_mov_b32_e32 v1, 0x41b80000 -; GFX908-NEXT: v_mov_b32_e32 v2, 0x41c00000 -; GFX908-NEXT: v_mov_b32_e32 v3, 0x41c80000 -; GFX908-NEXT: v_accvgpr_write_b32 a22, v1 -; GFX908-NEXT: v_accvgpr_write_b32 a23, v2 -; GFX908-NEXT: v_accvgpr_write_b32 a24, v3 -; GFX908-NEXT: v_mov_b32_e32 v1, 0x41d00000 -; GFX908-NEXT: v_mov_b32_e32 v2, 0x41d80000 -; GFX908-NEXT: v_mov_b32_e32 v3, 0x41e00000 -; GFX908-NEXT: v_accvgpr_write_b32 a25, v1 -; GFX908-NEXT: v_accvgpr_write_b32 a26, v2 -; GFX908-NEXT: v_accvgpr_write_b32 a27, v3 -; GFX908-NEXT: v_mov_b32_e32 v1, 0x41e80000 -; GFX908-NEXT: v_mov_b32_e32 v2, 0x41f00000 -; GFX908-NEXT: v_mov_b32_e32 v3, 0x41f80000 -; GFX908-NEXT: v_mov_b32_e32 v4, 1.0 +; GFX908-NEXT: v_mov_b32_e32 v0, 0x40400000 +; GFX908-NEXT: v_mov_b32_e32 v1, 0x40c00000 +; GFX908-NEXT: v_mov_b32_e32 v2, 0x40e00000 +; GFX908-NEXT: v_accvgpr_write_b32 a2, v0 +; GFX908-NEXT: v_mov_b32_e32 v0, 0x40a00000 +; GFX908-NEXT: v_accvgpr_write_b32 a5, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a6, v2 +; GFX908-NEXT: v_accvgpr_write_b32 a4, v0 +; GFX908-NEXT: v_mov_b32_e32 v0, 0x41000000 +; GFX908-NEXT: v_mov_b32_e32 v1, 0x41100000 +; GFX908-NEXT: v_mov_b32_e32 v2, 0x41200000 +; GFX908-NEXT: v_accvgpr_write_b32 a7, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a8, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a9, v2 +; GFX908-NEXT: v_mov_b32_e32 v0, 0x41300000 +; GFX908-NEXT: v_mov_b32_e32 v1, 0x41400000 +; GFX908-NEXT: v_mov_b32_e32 v2, 0x41500000 +; GFX908-NEXT: v_accvgpr_write_b32 a10, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a11, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a12, v2 +; GFX908-NEXT: v_mov_b32_e32 v0, 0x41600000 +; GFX908-NEXT: v_mov_b32_e32 v1, 0x41700000 +; GFX908-NEXT: v_mov_b32_e32 v2, 0x41800000 +; GFX908-NEXT: v_accvgpr_write_b32 a13, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a14, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a15, v2 +; GFX908-NEXT: v_mov_b32_e32 v0, 0x41880000 +; GFX908-NEXT: v_mov_b32_e32 v1, 0x41900000 +; GFX908-NEXT: v_mov_b32_e32 v2, 0x41980000 +; GFX908-NEXT: v_accvgpr_write_b32 a16, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a17, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a18, v2 +; GFX908-NEXT: v_mov_b32_e32 v0, 0x41a00000 +; GFX908-NEXT: v_mov_b32_e32 v1, 0x41a80000 +; GFX908-NEXT: v_mov_b32_e32 v2, 0x41b00000 +; GFX908-NEXT: v_accvgpr_write_b32 a19, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a20, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a21, v2 +; GFX908-NEXT: v_mov_b32_e32 v0, 0x41b80000 +; GFX908-NEXT: v_mov_b32_e32 v1, 0x41c00000 +; GFX908-NEXT: v_mov_b32_e32 v2, 0x41c80000 +; GFX908-NEXT: v_accvgpr_write_b32 a22, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a23, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a24, v2 +; GFX908-NEXT: v_mov_b32_e32 v0, 0x41d00000 +; GFX908-NEXT: v_mov_b32_e32 v1, 0x41d80000 +; GFX908-NEXT: v_mov_b32_e32 v2, 0x41e00000 +; GFX908-NEXT: v_accvgpr_write_b32 a25, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a26, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a27, v2 +; GFX908-NEXT: v_mov_b32_e32 v0, 0x41e80000 +; GFX908-NEXT: v_mov_b32_e32 v1, 0x41f00000 +; GFX908-NEXT: v_mov_b32_e32 v2, 0x41f80000 +; GFX908-NEXT: v_mov_b32_e32 v3, 1.0 ; GFX908-NEXT: ;;#ASMSTART ; GFX908-NEXT: ;;#ASMEND -; GFX908-NEXT: v_accvgpr_read_b32 v5, a0 +; GFX908-NEXT: v_accvgpr_read_b32 v4, a0 ; GFX908-NEXT: v_accvgpr_write_b32 a0, 1.0 ; GFX908-NEXT: v_accvgpr_write_b32 a1, 2.0 ; GFX908-NEXT: v_accvgpr_write_b32 a3, 4.0 -; GFX908-NEXT: v_accvgpr_write_b32 a28, v1 -; GFX908-NEXT: v_accvgpr_write_b32 a29, v2 -; GFX908-NEXT: v_accvgpr_write_b32 a30, v3 +; GFX908-NEXT: v_accvgpr_write_b32 a28, v0 +; GFX908-NEXT: v_accvgpr_write_b32 a29, v1 +; GFX908-NEXT: v_accvgpr_write_b32 a30, v2 ; GFX908-NEXT: v_accvgpr_write_b32 a31, 2.0 ; GFX908-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 -; GFX908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v4, v4, a[0:31] -; GFX908-NEXT: v_mov_b32_e32 v0, 0 -; GFX908-NEXT: s_nop 13 -; GFX908-NEXT: v_accvgpr_write_b32 a1, v5 +; GFX908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v3, v3, a[0:31] +; GFX908-NEXT: v_mov_b32_e32 v1, 0 +; GFX908-NEXT: s_nop 15 +; GFX908-NEXT: s_nop 0 +; GFX908-NEXT: v_accvgpr_read_b32 v0, a0 +; GFX908-NEXT: v_accvgpr_write_b32 a0, v4 ; GFX908-NEXT: ;;#ASMSTART ; GFX908-NEXT: ;;#ASMEND -; GFX908-NEXT: s_nop 1 -; GFX908-NEXT: v_accvgpr_read_b32 v1, a0 ; GFX908-NEXT: s_waitcnt lgkmcnt(0) -; GFX908-NEXT: s_nop 0 -; GFX908-NEXT: global_store_dword v0, v1, s[2:3] +; GFX908-NEXT: global_store_dword v1, v0, s[2:3] ; GFX908-NEXT: s_endpgm ; ; GFX90A-LABEL: max_32regs_mfma32: @@ -330,7 +329,8 @@ define amdgpu_kernel void @max_32regs_mfma32(ptr addrspace(1) %arg) #3 { ; GFX90A-NEXT: v_accvgpr_write_b32 a1, 2.0 ; GFX90A-NEXT: v_accvgpr_write_b32 a29, v1 ; GFX90A-NEXT: v_mov_b32_e32 v1, 0x41f80000 -; GFX90A-NEXT: v_accvgpr_read_b32 v3, a0 +; GFX90A-NEXT: ;;#ASMSTART +; GFX90A-NEXT: ;;#ASMEND ; GFX90A-NEXT: v_accvgpr_write_b32 a0, 1.0 ; GFX90A-NEXT: v_accvgpr_write_b32 a3, 4.0 ; GFX90A-NEXT: v_accvgpr_write_b32 a30, v1 @@ -338,12 +338,9 @@ define amdgpu_kernel void @max_32regs_mfma32(ptr addrspace(1) %arg) #3 { ; GFX90A-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 ; GFX90A-NEXT: v_mov_b32_e32 v0, 0 ; GFX90A-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v2, v2, a[0:31] -; GFX90A-NEXT: s_nop 15 -; GFX90A-NEXT: s_nop 2 -; GFX90A-NEXT: v_accvgpr_write_b32 a1, v3 -; GFX90A-NEXT: ;;#ASMSTART -; GFX90A-NEXT: ;;#ASMEND ; GFX90A-NEXT: s_waitcnt lgkmcnt(0) +; GFX90A-NEXT: s_nop 15 +; GFX90A-NEXT: s_nop 1 ; GFX90A-NEXT: global_store_dword v0, a0, s[2:3] ; GFX90A-NEXT: s_endpgm bb: