Skip to content

Commit

Permalink
[AMDGPU] Mark GFX11 dual source blend export as strict-wqm
Browse files Browse the repository at this point in the history 
The instructions that generate the source of dual source blend export
should run in strict-wqm. That is if any lane in a quad is active,
we need to enable all four lanes of that quad to make the shuffling
operation before exporting to dual source blend target work correctly.

Differential Revision: https://reviews.llvm.org/D127981
  • Loading branch information
@ruiling @jayfoad
ruiling authored and jayfoad committed Jun 20, 2022
1 parent 29621c1 commit 732eed4
Show file tree
Hide file tree
Showing 3 changed files with 124 additions and 2 deletions.
8 changes: 8 additions & 0 deletions llvm/lib/Target/AMDGPU/SIInstrInfo.h
View file
Expand Up @@ -554,6 +554,14 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
return MI.getDesc().TSFlags & SIInstrFlags::EXP;
}

static bool isDualSourceBlendEXP(const MachineInstr &MI) {
if (!isEXP(MI))
return false;
unsigned Target = MI.getOperand(0).getImm();
return Target == AMDGPU::Exp::ET_DUAL_SRC_BLEND0 ||
Target == AMDGPU::Exp::ET_DUAL_SRC_BLEND1;
}

bool isEXP(uint16_t Opcode) const {
return get(Opcode).TSFlags & SIInstrFlags::EXP;
}
Expand Down
19 changes: 17 additions & 2 deletions llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp
View file
Expand Up @@ -534,13 +534,28 @@ char SIWholeQuadMode::scanInstructions(MachineFunction &MF,
GlobalFlags |= StateStrictWWM;
LowerToMovInstrs.push_back(&MI);
continue;
} else if (Opcode == AMDGPU::STRICT_WQM) {
} else if (Opcode == AMDGPU::STRICT_WQM ||
TII->isDualSourceBlendEXP(MI)) {
// STRICT_WQM is similar to STRICTWWM, but instead of enabling all
// threads of the wave like STRICTWWM, STRICT_WQM enables all threads in
// quads that have at least one active thread.
markInstructionUses(MI, StateStrictWQM, Worklist);
GlobalFlags |= StateStrictWQM;
LowerToMovInstrs.push_back(&MI);

if (Opcode == AMDGPU::STRICT_WQM) {
LowerToMovInstrs.push_back(&MI);
} else {
// Dual source blend export acts as implicit strict-wqm, its sources
// need to be shuffled in strict wqm, but the export itself needs to
// run in exact mode.
BBI.Needs |= StateExact;
if (!(BBI.InNeeds & StateExact)) {
BBI.InNeeds |= StateExact;
Worklist.push_back(MBB);
}
GlobalFlags |= StateExact;
III.Disabled = StateWQM | StateStrict;
}
continue;
} else if (Opcode == AMDGPU::LDS_PARAM_LOAD ||
Opcode == AMDGPU::LDS_DIRECT_LOAD) {
Expand Down
99 changes: 99 additions & 0 deletions llvm/test/CodeGen/AMDGPU/dual-source-blend-export.ll
View file
@@ -0,0 +1,99 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -march=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck %s -check-prefix=GCN

; This is a slightly modified IR from real case to make it concise.
define amdgpu_ps void @_amdgpu_ps_main(i32 inreg %PrimMask, <2 x float> %InterpCenter) #0 {
; GCN-LABEL: _amdgpu_ps_main:
; GCN: ; %bb.0: ; %.entry
; GCN-NEXT: s_mov_b32 s1, exec_lo
; GCN-NEXT: s_wqm_b32 exec_lo, exec_lo
; GCN-NEXT: s_mov_b32 m0, s0
; GCN-NEXT: v_mov_b32_e32 v2, v0
; GCN-NEXT: lds_param_load v3, attr1.x wait_vdst:15
; GCN-NEXT: lds_param_load v4, attr1.y wait_vdst:15
; GCN-NEXT: lds_param_load v5, attr1.z wait_vdst:15
; GCN-NEXT: lds_param_load v6, attr1.w wait_vdst:15
; GCN-NEXT: v_mbcnt_lo_u32_b32 v7, -1, 0
; GCN-NEXT: v_mbcnt_hi_u32_b32 v7, -1, v7
; GCN-NEXT: v_and_b32_e32 v7, 1, v7
; GCN-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7
; GCN-NEXT: v_interp_p10_f32 v8, v4, v2, v4 wait_exp:2
; GCN-NEXT: v_interp_p10_f32 v10, v5, v2, v5 wait_exp:1
; GCN-NEXT: v_interp_p10_f32 v9, v6, v2, v6
; GCN-NEXT: v_interp_p10_f32 v2, v3, v2, v3 wait_exp:7
; GCN-NEXT: v_interp_p2_f32 v4, v4, v1, v8 wait_exp:7
; GCN-NEXT: v_interp_p2_f32 v5, v5, v1, v10 wait_exp:7
; GCN-NEXT: v_interp_p2_f32 v6, v6, v1, v9 wait_exp:7
; GCN-NEXT: v_interp_p2_f32 v2, v3, v1, v2 wait_exp:7
; GCN-NEXT: v_mov_b32_dpp v4, v4 dpp8:[1,0,3,2,5,4,7,6]
; GCN-NEXT: v_mov_b32_dpp v6, v6 dpp8:[1,0,3,2,5,4,7,6]
; GCN-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc_lo
; GCN-NEXT: v_cndmask_b32_e32 v4, v5, v4, vcc_lo
; GCN-NEXT: v_cndmask_b32_e32 v5, v2, v6, vcc_lo
; GCN-NEXT: v_cndmask_b32_e32 v2, v6, v2, vcc_lo
; GCN-NEXT: v_mov_b32_dpp v4, v4 dpp8:[1,0,3,2,5,4,7,6]
; GCN-NEXT: v_mov_b32_dpp v5, v5 dpp8:[1,0,3,2,5,4,7,6]
; GCN-NEXT: s_mov_b32 exec_lo, s1
; GCN-NEXT: exp dual_src_blend0 v3, v2, off, off
; GCN-NEXT: exp dual_src_blend1 v4, v5, off, off done
; GCN-NEXT: s_endpgm
.entry:
%InterpCenter.i0 = extractelement <2 x float> %InterpCenter, i64 0
%InterpCenter.i1 = extractelement <2 x float> %InterpCenter, i64 1
%i6 = call float @llvm.amdgcn.lds.param.load(i32 immarg 0, i32 immarg 1, i32 %PrimMask)
%i7 = call float @llvm.amdgcn.lds.param.load(i32 immarg 1, i32 immarg 1, i32 %PrimMask)
%i8 = call float @llvm.amdgcn.lds.param.load(i32 immarg 2, i32 immarg 1, i32 %PrimMask)
%i9 = call float @llvm.amdgcn.lds.param.load(i32 immarg 3, i32 immarg 1, i32 %PrimMask)

%i14 = call float @llvm.amdgcn.interp.inreg.p10(float %i8, float %InterpCenter.i0, float %i8)
%i15 = call float @llvm.amdgcn.interp.inreg.p2(float %i8, float %InterpCenter.i1, float %i14)

%i16 = call float @llvm.amdgcn.interp.inreg.p10(float %i7, float %InterpCenter.i0, float %i7)
%i17 = call float @llvm.amdgcn.interp.inreg.p2(float %i7, float %InterpCenter.i1, float %i16)

%i18 = call float @llvm.amdgcn.interp.inreg.p10(float %i6, float %InterpCenter.i0, float %i6)
%i19 = call float @llvm.amdgcn.interp.inreg.p2(float %i6, float %InterpCenter.i1, float %i18)

%i20 = call float @llvm.amdgcn.interp.inreg.p10(float %i9, float %InterpCenter.i0, float %i9)
%i21 = call float @llvm.amdgcn.interp.inreg.p2(float %i9, float %InterpCenter.i1, float %i20)

%i34 = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%i35 = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %i34)
%i36 = and i32 %i35, 1
%.not = icmp eq i32 %i36, 0

%i37 = bitcast float %i15 to i32
%i38 = bitcast float %i17 to i32
%i39 = call i32 @llvm.amdgcn.mov.dpp8.i32(i32 %i38, i32 14570689)
%i40 = select i1 %.not, i32 %i37, i32 %i39
%i41 = bitcast i32 %i40 to float
%i42 = select i1 %.not, i32 %i39, i32 %i37
%i43 = call i32 @llvm.amdgcn.mov.dpp8.i32(i32 %i42, i32 14570689)
%i44 = bitcast i32 %i43 to float

%i45 = bitcast float %i19 to i32
%i46 = bitcast float %i21 to i32
%i47 = call i32 @llvm.amdgcn.mov.dpp8.i32(i32 %i46, i32 14570689)
%i48 = select i1 %.not, i32 %i45, i32 %i47
%i49 = bitcast i32 %i48 to float
%i50 = select i1 %.not, i32 %i47, i32 %i45
%i51 = call i32 @llvm.amdgcn.mov.dpp8.i32(i32 %i50, i32 14570689)
%i52 = bitcast i32 %i51 to float
call void @llvm.amdgcn.exp.f32(i32 immarg 21, i32 immarg 3, float %i41, float %i49, float undef, float undef, i1 immarg false, i1 immarg true)
call void @llvm.amdgcn.exp.f32(i32 immarg 22, i32 immarg 3, float %i44, float %i52, float undef, float undef, i1 immarg true, i1 immarg true)
ret void
}

declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32) #2
declare i32 @llvm.amdgcn.mbcnt.hi(i32, i32) #2
declare i32 @llvm.amdgcn.mov.dpp8.i32(i32, i32 immarg) #3
declare void @llvm.amdgcn.exp.f32(i32 immarg, i32 immarg, float, float, float, float, i1 immarg, i1 immarg) #4
declare float @llvm.amdgcn.interp.inreg.p10(float, float, float) #1
declare float @llvm.amdgcn.interp.inreg.p2(float, float, float) #1
declare float @llvm.amdgcn.lds.param.load(i32 immarg, i32 immarg, i32) #1

attributes #0 = { nounwind }
attributes #1 = { nounwind readnone speculatable willreturn }
attributes #2 = { nounwind readnone willreturn }
attributes #3 = { convergent nounwind readnone willreturn }
attributes #4 = { inaccessiblememonly nounwind willreturn writeonly }

0 comments on commit 732eed4

Please sign in to comment.