Skip to content

Commit

Permalink
AutoUpgrade: Use syncscope("agent") atomic.inc/dec intrinsic upgrade
Browse files Browse the repository at this point in the history 
The old syncscope parameter never really worked correctly, but
effectively gave "workgroup" scope. Use something faster than system
but more correct than before.

https://reviews.llvm.org/D157389
  • Loading branch information
@arsenm
arsenm committed Aug 10, 2023
1 parent 9e94f7d commit 204a417
Show file tree
Hide file tree
Showing 2 changed files with 33 additions and 29 deletions.
6 changes: 5 additions & 1 deletion llvm/lib/IR/AutoUpgrade.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2301,7 +2301,11 @@ static Value *UpgradeAMDGCNIntrinsicCall(StringRef Name, CallBase *CI,
Order == AtomicOrdering::Unordered)
Order = AtomicOrdering::SequentiallyConsistent;

AtomicRMWInst *RMW = Builder.CreateAtomicRMW(RMWOp, Ptr, Val, std::nullopt, Order);
// The scope argument never really worked correctly. Use agent as the most
// conservative option which should still always produce the instruction.
SyncScope::ID SSID = F->getContext().getOrInsertSyncScopeID("agent");
AtomicRMWInst *RMW =
Builder.CreateAtomicRMW(RMWOp, Ptr, Val, std::nullopt, Order, SSID);

if (!VolatileArg || !VolatileArg->isZero())
RMW->setVolatile(true);
Expand Down
56 changes: 28 additions & 28 deletions llvm/test/Bitcode/amdgcn-atomic.ll
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,98 +2,98 @@


define void @atomic_inc(ptr %ptr0, ptr addrspace(1) %ptr1, ptr addrspace(3) %ptr3) {
; CHECK: atomicrmw uinc_wrap ptr %ptr0, i32 42 seq_cst, align 4
; CHECK: atomicrmw uinc_wrap ptr %ptr0, i32 42 syncscope("agent") seq_cst, align 4
%result0 = call i32 @llvm.amdgcn.atomic.inc.i32.p0(ptr %ptr0, i32 42, i32 0, i32 0, i1 false)

; CHECK: atomicrmw uinc_wrap ptr addrspace(1) %ptr1, i32 43 seq_cst, align 4
; CHECK: atomicrmw uinc_wrap ptr addrspace(1) %ptr1, i32 43 syncscope("agent") seq_cst, align 4
%result1 = call i32 @llvm.amdgcn.atomic.inc.i32.p1(ptr addrspace(1) %ptr1, i32 43, i32 0, i32 0, i1 false)

; CHECK: atomicrmw uinc_wrap ptr addrspace(3) %ptr3, i32 46 seq_cst, align 4
; CHECK: atomicrmw uinc_wrap ptr addrspace(3) %ptr3, i32 46 syncscope("agent") seq_cst, align 4
%result2 = call i32 @llvm.amdgcn.atomic.inc.i32.p3(ptr addrspace(3) %ptr3, i32 46, i32 0, i32 0, i1 false)

; CHECK: atomicrmw uinc_wrap ptr %ptr0, i64 48 seq_cst, align 8
; CHECK: atomicrmw uinc_wrap ptr %ptr0, i64 48 syncscope("agent") seq_cst, align 8
%result3 = call i64 @llvm.amdgcn.atomic.inc.i64.p0(ptr %ptr0, i64 48, i32 0, i32 0, i1 false)

; CHECK: atomicrmw uinc_wrap ptr addrspace(1) %ptr1, i64 45 seq_cst, align 8
; CHECK: atomicrmw uinc_wrap ptr addrspace(1) %ptr1, i64 45 syncscope("agent") seq_cst, align 8
%result4 = call i64 @llvm.amdgcn.atomic.inc.i64.p1(ptr addrspace(1) %ptr1, i64 45, i32 0, i32 0, i1 false)

; CHECK: atomicrmw uinc_wrap ptr addrspace(3) %ptr3, i64 4345 seq_cst, align 8
; CHECK: atomicrmw uinc_wrap ptr addrspace(3) %ptr3, i64 4345 syncscope("agent") seq_cst, align 8
%result5 = call i64 @llvm.amdgcn.atomic.inc.i64.p3(ptr addrspace(3) %ptr3, i64 4345, i32 0, i32 0, i1 false)

; CHECK: atomicrmw volatile uinc_wrap ptr addrspace(3) %ptr3, i64 4345 seq_cst, align 8
; CHECK: atomicrmw volatile uinc_wrap ptr addrspace(3) %ptr3, i64 4345 syncscope("agent") seq_cst, align 8
%result6 = call i64 @llvm.amdgcn.atomic.inc.i64.p3(ptr addrspace(3) %ptr3, i64 4345, i32 0, i32 0, i1 true)
ret void
}

define void @atomic_dec(ptr %ptr0, ptr addrspace(1) %ptr1, ptr addrspace(3) %ptr3) {
; CHECK: atomicrmw udec_wrap ptr %ptr0, i32 42 seq_cst, align 4
; CHECK: atomicrmw udec_wrap ptr %ptr0, i32 42 syncscope("agent") seq_cst, align 4
%result0 = call i32 @llvm.amdgcn.atomic.dec.i32.p0(ptr %ptr0, i32 42, i32 0, i32 0, i1 false)

; CHECK: atomicrmw udec_wrap ptr addrspace(1) %ptr1, i32 43 seq_cst, align 4
; CHECK: atomicrmw udec_wrap ptr addrspace(1) %ptr1, i32 43 syncscope("agent") seq_cst, align 4
%result1 = call i32 @llvm.amdgcn.atomic.dec.i32.p1(ptr addrspace(1) %ptr1, i32 43, i32 0, i32 0, i1 false)

; CHECK: atomicrmw udec_wrap ptr addrspace(3) %ptr3, i32 46 seq_cst, align 4
; CHECK: atomicrmw udec_wrap ptr addrspace(3) %ptr3, i32 46 syncscope("agent") seq_cst, align 4
%result2 = call i32 @llvm.amdgcn.atomic.dec.i32.p3(ptr addrspace(3) %ptr3, i32 46, i32 0, i32 0, i1 false)

; CHECK: atomicrmw udec_wrap ptr %ptr0, i64 48 seq_cst, align 8
; CHECK: atomicrmw udec_wrap ptr %ptr0, i64 48 syncscope("agent") seq_cst, align 8
%result3 = call i64 @llvm.amdgcn.atomic.dec.i64.p0(ptr %ptr0, i64 48, i32 0, i32 0, i1 false)

; CHECK: atomicrmw udec_wrap ptr addrspace(1) %ptr1, i64 45 seq_cst, align 8
; CHECK: atomicrmw udec_wrap ptr addrspace(1) %ptr1, i64 45 syncscope("agent") seq_cst, align 8
%result4 = call i64 @llvm.amdgcn.atomic.dec.i64.p1(ptr addrspace(1) %ptr1, i64 45, i32 0, i32 0, i1 false)

; CHECK: atomicrmw udec_wrap ptr addrspace(3) %ptr3, i64 4345 seq_cst, align 8
; CHECK: atomicrmw udec_wrap ptr addrspace(3) %ptr3, i64 4345 syncscope("agent") seq_cst, align 8
%result5 = call i64 @llvm.amdgcn.atomic.dec.i64.p3(ptr addrspace(3) %ptr3, i64 4345, i32 0, i32 0, i1 false)

; CHECK: atomicrmw volatile udec_wrap ptr addrspace(3) %ptr3, i64 4345 seq_cst, align 8
; CHECK: atomicrmw volatile udec_wrap ptr addrspace(3) %ptr3, i64 4345 syncscope("agent") seq_cst, align 8
%result6 = call i64 @llvm.amdgcn.atomic.dec.i64.p3(ptr addrspace(3) %ptr3, i64 4345, i32 0, i32 0, i1 true)
ret void
}

; Test some invalid ordering handling
define void @ordering(ptr %ptr0, ptr addrspace(1) %ptr1, ptr addrspace(3) %ptr3) {
; CHECK: atomicrmw volatile uinc_wrap ptr %ptr0, i32 42 seq_cst, align 4
; CHECK: atomicrmw volatile uinc_wrap ptr %ptr0, i32 42 syncscope("agent") seq_cst, align 4
%result0 = call i32 @llvm.amdgcn.atomic.inc.i32.p0(ptr %ptr0, i32 42, i32 -1, i32 0, i1 true)

; CHECK: atomicrmw volatile uinc_wrap ptr addrspace(1) %ptr1, i32 43 seq_cst, align 4
; CHECK: atomicrmw volatile uinc_wrap ptr addrspace(1) %ptr1, i32 43 syncscope("agent") seq_cst, align 4
%result1 = call i32 @llvm.amdgcn.atomic.inc.i32.p1(ptr addrspace(1) %ptr1, i32 43, i32 0, i32 0, i1 true)

; CHECK: atomicrmw uinc_wrap ptr addrspace(1) %ptr1, i32 43 seq_cst, align 4
; CHECK: atomicrmw uinc_wrap ptr addrspace(1) %ptr1, i32 43 syncscope("agent") seq_cst, align 4
%result2 = call i32 @llvm.amdgcn.atomic.inc.i32.p1(ptr addrspace(1) %ptr1, i32 43, i32 1, i32 0, i1 false)

; CHECK: atomicrmw volatile uinc_wrap ptr addrspace(1) %ptr1, i32 43 monotonic, align 4
; CHECK: atomicrmw volatile uinc_wrap ptr addrspace(1) %ptr1, i32 43 syncscope("agent") monotonic, align 4
%result3 = call i32 @llvm.amdgcn.atomic.inc.i32.p1(ptr addrspace(1) %ptr1, i32 43, i32 2, i32 0, i1 true)

; CHECK: atomicrmw uinc_wrap ptr addrspace(1) %ptr1, i32 43 seq_cst, align 4
; CHECK: atomicrmw uinc_wrap ptr addrspace(1) %ptr1, i32 43 syncscope("agent") seq_cst, align 4
%result4 = call i32 @llvm.amdgcn.atomic.inc.i32.p1(ptr addrspace(1) %ptr1, i32 43, i32 3, i32 0, i1 false)

; CHECK: atomicrmw volatile udec_wrap ptr %ptr0, i32 42 seq_cst, align 4
; CHECK: atomicrmw volatile udec_wrap ptr %ptr0, i32 42 syncscope("agent") seq_cst, align 4
%result5 = call i32 @llvm.amdgcn.atomic.dec.i32.p0(ptr %ptr0, i32 42, i32 0, i32 4, i1 true)

; CHECK: atomicrmw udec_wrap ptr %ptr0, i32 42 seq_cst, align 4
; CHECK: atomicrmw udec_wrap ptr %ptr0, i32 42 syncscope("agent") seq_cst, align 4
%result6 = call i32 @llvm.amdgcn.atomic.dec.i32.p0(ptr %ptr0, i32 42, i32 0, i32 5, i1 false)

; CHECK: atomicrmw volatile udec_wrap ptr %ptr0, i32 42 seq_cst, align 4
; CHECK: atomicrmw volatile udec_wrap ptr %ptr0, i32 42 syncscope("agent") seq_cst, align 4
%result7 = call i32 @llvm.amdgcn.atomic.dec.i32.p0(ptr %ptr0, i32 42, i32 0, i32 6, i1 true)

; CHECK: atomicrmw udec_wrap ptr %ptr0, i32 42 seq_cst, align 4
; CHECK: atomicrmw udec_wrap ptr %ptr0, i32 42 syncscope("agent") seq_cst, align 4
%result8 = call i32 @llvm.amdgcn.atomic.dec.i32.p0(ptr %ptr0, i32 42, i32 0, i32 7, i1 false)

; CHECK:= atomicrmw volatile udec_wrap ptr %ptr0, i32 42 seq_cst, align 4
; CHECK:= atomicrmw volatile udec_wrap ptr %ptr0, i32 42 syncscope("agent") seq_cst, align 4
%result9 = call i32 @llvm.amdgcn.atomic.dec.i32.p0(ptr %ptr0, i32 42, i32 0, i32 8, i1 true)

; CHECK:= atomicrmw volatile udec_wrap ptr addrspace(1) %ptr1, i32 43 seq_cst, align 4
; CHECK:= atomicrmw volatile udec_wrap ptr addrspace(1) %ptr1, i32 43 syncscope("agent") seq_cst, align 4
%result10 = call i32 @llvm.amdgcn.atomic.dec.i32.p1(ptr addrspace(1) %ptr1, i32 43, i32 3, i32 0, i1 true)
ret void
}

define void @immarg_violations(ptr %ptr0, i32 %val32, i1 %val1) {
; CHECK: atomicrmw udec_wrap ptr %ptr0, i32 42 seq_cst, align 4
; CHECK: atomicrmw udec_wrap ptr %ptr0, i32 42 syncscope("agent") seq_cst, align 4
%result0 = call i32 @llvm.amdgcn.atomic.dec.i32.p0(ptr %ptr0, i32 42, i32 %val32, i32 0, i1 false)

; CHECK: atomicrmw udec_wrap ptr %ptr0, i32 42 monotonic, align 4
; CHECK: atomicrmw udec_wrap ptr %ptr0, i32 42 syncscope("agent") monotonic, align 4
%result1 = call i32 @llvm.amdgcn.atomic.dec.i32.p0(ptr %ptr0, i32 42, i32 2, i32 %val32, i1 false)

; CHECK: atomicrmw volatile udec_wrap ptr %ptr0, i32 42 monotonic, align 4
; CHECK: atomicrmw volatile udec_wrap ptr %ptr0, i32 42 syncscope("agent") monotonic, align 4
%result2 = call i32 @llvm.amdgcn.atomic.dec.i32.p0(ptr %ptr0, i32 42, i32 2, i32 0, i1 %val1)
ret void
}
Expand Down

0 comments on commit 204a417

Please sign in to comment.