Skip to content

Commit

Permalink
[AMDGPU] Do not wait for vscnt on function entry and return
Browse files Browse the repository at this point in the history 
SIInsertWaitcnts inserts waitcnt instructions to resolve data
dependencies. The GFX10+ vscnt (VMEM store count) counter is never used
in this way. It is only used to resolve memory dependencies, and that is
handled by SIMemoryLegalizer. Hence there is no need to conservatively
wait for vscnt to be 0 on function entry and before returns.

Differential Revision: https://reviews.llvm.org/D153537
  • Loading branch information
@jayfoad
jayfoad committed Jul 4, 2023
1 parent 8f5a68a commit f2c164c
Show file tree
Hide file tree
Showing 205 changed files with 294 additions and 5,195 deletions.
18 changes: 4 additions & 14 deletions llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -419,10 +419,6 @@ class SIInsertWaitcnts : public MachineFunctionPass {
return false;
}

AMDGPU::Waitcnt allZeroWaitcnt() const {
return AMDGPU::Waitcnt::allZero(ST->hasVscnt());
}

void setForceEmitWaitcnt() {
// For non-debug builds, ForceEmitWaitcnt has been initialized to false;
// For debug builds, get the debug counter info and adjust if need be
Expand Down Expand Up @@ -1036,7 +1032,7 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(MachineInstr &MI,
MI.getOpcode() == AMDGPU::SI_RETURN ||
MI.getOpcode() == AMDGPU::S_SETPC_B64_return ||
(MI.isReturn() && MI.isCall() && !callWaitsOnFunctionEntry(MI))) {
Wait = Wait.combined(allZeroWaitcnt());
Wait = Wait.combined(AMDGPU::Waitcnt::allZeroExceptVsCnt());
}
// Identify S_ENDPGM instructions which may have to wait for outstanding VMEM
// stores. In this case it can be useful to send a message to explicitly
Expand Down Expand Up @@ -1232,7 +1228,7 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(MachineInstr &MI,
// cause an exception. Otherwise, insert an explicit S_WAITCNT 0 here.
if (MI.getOpcode() == AMDGPU::S_BARRIER &&
!ST->hasAutoWaitcntBeforeBarrier() && !ST->supportsBackOffBarrier()) {
Wait = Wait.combined(allZeroWaitcnt());
Wait = Wait.combined(AMDGPU::Waitcnt::allZero(ST->hasVscnt()));
}

// TODO: Remove this work-around, enable the assert for Bug 457939
Expand All @@ -1248,16 +1244,14 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(MachineInstr &MI,
ScoreBrackets.simplifyWaitcnt(Wait);

if (ForceEmitZeroWaitcnts)
Wait = allZeroWaitcnt();
Wait = AMDGPU::Waitcnt::allZeroExceptVsCnt();

if (ForceEmitWaitcnt[VM_CNT])
Wait.VmCnt = 0;
if (ForceEmitWaitcnt[EXP_CNT])
Wait.ExpCnt = 0;
if (ForceEmitWaitcnt[LGKM_CNT])
Wait.LgkmCnt = 0;
if (ForceEmitWaitcnt[VS_CNT])
Wait.VsCnt = 0;

if (FlushVmCnt) {
if (ScoreBrackets.hasPendingEvent(VM_CNT))
Expand Down Expand Up @@ -1480,7 +1474,7 @@ void SIInsertWaitcnts::updateEventWaitcntAfter(MachineInstr &Inst,
} else if (Inst.isCall()) {
if (callWaitsOnFunctionReturn(Inst)) {
// Act as a wait on everything
ScoreBrackets->applyWaitcnt(allZeroWaitcnt());
ScoreBrackets->applyWaitcnt(AMDGPU::Waitcnt::allZeroExceptVsCnt());
} else {
// May need to way wait for anything.
ScoreBrackets->applyWaitcnt(AMDGPU::Waitcnt());
Expand Down Expand Up @@ -1862,10 +1856,6 @@ bool SIInsertWaitcnts::runOnMachineFunction(MachineFunction &MF) {
I != E && (I->isPHI() || I->isMetaInstruction()); ++I)
;
BuildMI(EntryBB, I, DebugLoc(), TII->get(AMDGPU::S_WAITCNT)).addImm(0);
if (ST->hasVscnt())
BuildMI(EntryBB, I, DebugLoc(), TII->get(AMDGPU::S_WAITCNT_VSCNT))
.addReg(AMDGPU::SGPR_NULL, RegState::Undef)
.addImm(0);

Modified = true;
}
Expand Down
7 changes: 0 additions & 7 deletions llvm/test/CodeGen/AMDGPU/GlobalISel/add.v2i16.ll
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ define <2 x i16> @v_add_v2i16(<2 x i16> %a, <2 x i16> %b) {
; GFX10-LABEL: v_add_v2i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: v_pk_add_u16 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
%add = add <2 x i16> %a, %b
Expand Down Expand Up @@ -68,7 +67,6 @@ define <2 x i16> @v_add_v2i16_fneg_lhs(<2 x half> %a, <2 x i16> %b) {
; GFX10-LABEL: v_add_v2i16_fneg_lhs:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: v_pk_add_u16 v0, v0, v1 neg_lo:[1,0] neg_hi:[1,0]
; GFX10-NEXT: s_setpc_b64 s[30:31]
%neg.a = fneg <2 x half> %a
Expand Down Expand Up @@ -108,7 +106,6 @@ define <2 x i16> @v_add_v2i16_fneg_rhs(<2 x i16> %a, <2 x half> %b) {
; GFX10-LABEL: v_add_v2i16_fneg_rhs:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: v_pk_add_u16 v0, v0, v1 neg_lo:[0,1] neg_hi:[0,1]
; GFX10-NEXT: s_setpc_b64 s[30:31]
%neg.b = fneg <2 x half> %b
Expand Down Expand Up @@ -154,7 +151,6 @@ define <2 x i16> @v_add_v2i16_fneg_lhs_fneg_rhs(<2 x half> %a, <2 x half> %b) {
; GFX10-LABEL: v_add_v2i16_fneg_lhs_fneg_rhs:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: v_pk_add_u16 v0, v0, v1 neg_lo:[1,1] neg_hi:[1,1]
; GFX10-NEXT: s_setpc_b64 s[30:31]
%neg.a = fneg <2 x half> %a
Expand Down Expand Up @@ -193,7 +189,6 @@ define <2 x i16> @v_add_v2i16_neg_inline_imm_splat(<2 x i16> %a) {
; GFX10-LABEL: v_add_v2i16_neg_inline_imm_splat:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: v_pk_add_u16 v0, 0xffc0, v0 op_sel_hi:[0,1]
; GFX10-NEXT: s_setpc_b64 s[30:31]
%add = add <2 x i16> %a, <i16 -64, i16 -64>
Expand Down Expand Up @@ -227,7 +222,6 @@ define <2 x i16> @v_add_v2i16_neg_inline_imm_lo(<2 x i16> %a) {
; GFX10-LABEL: v_add_v2i16_neg_inline_imm_lo:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: v_pk_add_u16 v0, 0x4ffc0, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
%add = add <2 x i16> %a, <i16 -64, i16 4>
Expand Down Expand Up @@ -261,7 +255,6 @@ define <2 x i16> @v_add_v2i16_neg_inline_imm_hi(<2 x i16> %a) {
; GFX10-LABEL: v_add_v2i16_neg_inline_imm_hi:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: v_pk_add_u16 v0, 0xffc00004, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
%add = add <2 x i16> %a, <i16 4, i16 -64>
Expand Down
6 changes: 0 additions & 6 deletions llvm/test/CodeGen/AMDGPU/GlobalISel/andn2.ll
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,6 @@ define i32 @v_andn2_i32(i32 %src0, i32 %src1) {
; GFX10PLUS-LABEL: v_andn2_i32:
; GFX10PLUS: ; %bb.0:
; GFX10PLUS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10PLUS-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10PLUS-NEXT: v_not_b32_e32 v1, v1
; GFX10PLUS-NEXT: v_and_b32_e32 v0, v0, v1
; GFX10PLUS-NEXT: s_setpc_b64 s[30:31]
Expand Down Expand Up @@ -257,7 +256,6 @@ define i64 @v_andn2_i64(i64 %src0, i64 %src1) {
; GFX10PLUS-LABEL: v_andn2_i64:
; GFX10PLUS: ; %bb.0:
; GFX10PLUS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10PLUS-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10PLUS-NEXT: v_not_b32_e32 v2, v2
; GFX10PLUS-NEXT: v_not_b32_e32 v3, v3
; GFX10PLUS-NEXT: v_and_b32_e32 v0, v0, v2
Expand Down Expand Up @@ -452,7 +450,6 @@ define i16 @v_andn2_i16(i16 %src0, i16 %src1) {
; GFX10PLUS-LABEL: v_andn2_i16:
; GFX10PLUS: ; %bb.0:
; GFX10PLUS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10PLUS-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10PLUS-NEXT: v_xor_b32_e32 v1, -1, v1
; GFX10PLUS-NEXT: v_and_b32_e32 v0, v0, v1
; GFX10PLUS-NEXT: s_setpc_b64 s[30:31]
Expand Down Expand Up @@ -679,7 +676,6 @@ define <2 x i16> @v_andn2_v2i16(<2 x i16> %src0, <2 x i16> %src1) {
; GFX10PLUS-LABEL: v_andn2_v2i16:
; GFX10PLUS: ; %bb.0:
; GFX10PLUS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10PLUS-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10PLUS-NEXT: v_xor_b32_e32 v1, -1, v1
; GFX10PLUS-NEXT: v_and_b32_e32 v0, v0, v1
; GFX10PLUS-NEXT: s_setpc_b64 s[30:31]
Expand Down Expand Up @@ -904,7 +900,6 @@ define <3 x i16> @v_andn2_v3i16(<3 x i16> %src0, <3 x i16> %src1) {
; GFX10PLUS-LABEL: v_andn2_v3i16:
; GFX10PLUS: ; %bb.0:
; GFX10PLUS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10PLUS-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10PLUS-NEXT: v_xor_b32_e32 v2, -1, v2
; GFX10PLUS-NEXT: v_xor_b32_e32 v3, -11, v3
; GFX10PLUS-NEXT: v_and_b32_e32 v0, v0, v2
Expand Down Expand Up @@ -1141,7 +1136,6 @@ define <4 x i16> @v_andn2_v4i16(<4 x i16> %src0, <4 x i16> %src1) {
; GFX10PLUS-LABEL: v_andn2_v4i16:
; GFX10PLUS: ; %bb.0:
; GFX10PLUS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10PLUS-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10PLUS-NEXT: v_xor_b32_e32 v2, -1, v2
; GFX10PLUS-NEXT: v_xor_b32_e32 v3, -1, v3
; GFX10PLUS-NEXT: v_and_b32_e32 v0, v0, v2
Expand Down
2 changes: 0 additions & 2 deletions llvm/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-asserts.ll
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ define hidden <2 x i64> @icmp_v2i32_sext_to_v2i64(<2 x i32> %arg) {
; CHECK-LABEL: icmp_v2i32_sext_to_v2i64:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_waitcnt_vscnt null, 0x0
; CHECK-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
; CHECK-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
Expand All @@ -24,7 +23,6 @@ define hidden <2 x i64> @icmp_v2i32_zext_to_v2i64(<2 x i32> %arg) {
; CHECK-LABEL: icmp_v2i32_zext_to_v2i64:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_waitcnt_vscnt null, 0x0
; CHECK-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; CHECK-NEXT: v_mov_b32_e32 v3, 0
; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
Expand Down
Loading

0 comments on commit f2c164c

Please sign in to comment.