diff --git a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp index 6c95dc76f4f6f..04639f55e3935 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp @@ -2472,11 +2472,11 @@ bool SchedGroup::canAddMI(const MachineInstr &MI) const { Result = true; else if (((SGMask & SchedGroupMask::VMEM_READ) != SchedGroupMask::NONE) && - MI.mayLoad() && TII->isVMEM(MI)) + MI.mayLoad() && TII->isVMEM(MI) && !TII->isLDSDMA(MI)) Result = true; else if (((SGMask & SchedGroupMask::VMEM_WRITE) != SchedGroupMask::NONE) && - MI.mayStore() && TII->isVMEM(MI)) + MI.mayStore() && TII->isVMEM(MI) && !TII->isLDSDMA(MI)) Result = true; else if (((SGMask & SchedGroupMask::DS) != SchedGroupMask::NONE) && @@ -2679,9 +2679,10 @@ IGroupLPDAGMutation::invertSchedBarrierMask(SchedGroupMask Mask) const { if ((InvertedMask & SchedGroupMask::DS) == SchedGroupMask::NONE) InvertedMask &= ~SchedGroupMask::DS_READ & ~SchedGroupMask::DS_WRITE & ~SchedGroupMask::LDSDMA; - // DS_READ, DS_WRITE implies DS. + // DS_READ, DS_WRITE, LDSDMA implies DS. else if ((InvertedMask & SchedGroupMask::DS_READ) == SchedGroupMask::NONE || - (InvertedMask & SchedGroupMask::DS_WRITE) == SchedGroupMask::NONE) + (InvertedMask & SchedGroupMask::DS_WRITE) == SchedGroupMask::NONE || + (InvertedMask & SchedGroupMask::LDSDMA) == SchedGroupMask::NONE) InvertedMask &= ~SchedGroupMask::DS; LLVM_DEBUG(dbgs() << "After Inverting, SchedGroup Mask: " << (int)InvertedMask diff --git a/llvm/test/CodeGen/AMDGPU/sched-ldsdma-mask.mir b/llvm/test/CodeGen/AMDGPU/sched-ldsdma-mask.mir index 1ab3e349fc0a0..c1924706a0751 100644 --- a/llvm/test/CodeGen/AMDGPU/sched-ldsdma-mask.mir +++ b/llvm/test/CodeGen/AMDGPU/sched-ldsdma-mask.mir @@ -10,6 +10,7 @@ define amdgpu_kernel void @sched_group_barrier_mask_2048_async_loads() { ret void } define amdgpu_kernel void @sched_group_barrier_mask_16_async_loads() { ret void } define amdgpu_kernel void @sched_group_barrier_mask_2048_interleave_2load_1valu() { ret void } + define amdgpu_kernel void @sched_barrier_mask_2049_allows_ldsdma() { ret void } ... --- @@ -340,3 +341,40 @@ body: | S_ENDPGM 0, implicit %30, implicit %31, implicit %32 ... + +# A SCHED_BARRIER whose mask allows LDSDMA (bit 0x800) together with ALU +# (bit 0x1) must let an LDSDMA op be scheduled across it. The second +# async load is hoisted above the barrier. + +--- +name: sched_barrier_mask_2049_allows_ldsdma +tracksRegLiveness: true +body: | + bb.0: + ; CHECK-LABEL: name: sched_barrier_mask_2049_allows_ldsdma + ; CHECK: $exec = IMPLICIT_DEF + ; CHECK-NEXT: [[DEF:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF + ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF + ; CHECK-NEXT: [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF + ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 [[DEF1]], [[DEF]], 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt + ; CHECK-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 [[DEF2]], [[DEF]], 16, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt + ; CHECK-NEXT: [[DEF3:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF + ; CHECK-NEXT: [[DEF4:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF + ; CHECK-NEXT: [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[DEF3]], [[DEF4]], implicit $exec + ; CHECK-NEXT: [[V_ADD_U32_e32_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[V_ADD_U32_e32_]], [[DEF3]], implicit $exec + ; CHECK-NEXT: SCHED_BARRIER 2049 + ; CHECK-NEXT: S_ENDPGM 0, implicit [[V_ADD_U32_e32_1]] + $exec = IMPLICIT_DEF + %0:vreg_64_align2 = IMPLICIT_DEF + %1:vgpr_32 = IMPLICIT_DEF + %2:vgpr_32 = IMPLICIT_DEF + %3:vgpr_32 = IMPLICIT_DEF + %4:vgpr_32 = IMPLICIT_DEF + GLOBAL_LOAD_ASYNC_TO_LDS_B32 %1, %0, 0, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt + %10:vgpr_32 = V_ADD_U32_e32 %3, %4, implicit $exec + SCHED_BARRIER 2049 + GLOBAL_LOAD_ASYNC_TO_LDS_B32 %2, %0, 16, 0, implicit-def dead $asynccnt, implicit $exec, implicit $asynccnt + %11:vgpr_32 = V_ADD_U32_e32 %10, %3, implicit $exec + S_ENDPGM 0, implicit %11 + +... diff --git a/llvm/test/CodeGen/AMDGPU/sched.barrier.inverted.mask.ll b/llvm/test/CodeGen/AMDGPU/sched.barrier.inverted.mask.ll index ca802ac586668..71476ab29029b 100644 --- a/llvm/test/CodeGen/AMDGPU/sched.barrier.inverted.mask.ll +++ b/llvm/test/CodeGen/AMDGPU/sched.barrier.inverted.mask.ll @@ -41,8 +41,8 @@ entry: ret void } -; Inverted 1935: 011110001111 -; GCN: After Inverting, SchedGroup Mask: 1935 +; Inverted 1807: 011100001111 +; GCN: After Inverting, SchedGroup Mask: 1807 define amdgpu_kernel void @invert16() #0 { entry: call void @llvm.amdgcn.sched.barrier(i32 16) #1 @@ -104,8 +104,8 @@ entry: ret void } -; Inverted 2031: 011111101111 -; GCN: After Inverting, SchedGroup Mask: 2031 +; Inverted 1903: 011101101111 +; GCN: After Inverting, SchedGroup Mask: 1903 define amdgpu_kernel void @invert2048() #0 { entry: call void @llvm.amdgcn.sched.barrier(i32 2048) #1