63 changes: 63 additions & 0 deletions llvm/test/CodeGen/AMDGPU/default-flat-work-group-size-overrides-waves-per-eu.ll
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
; RUN: opt -S -mtriple=amdgcn-unknown-unknown -mcpu=tahiti -amdgpu-promote-alloca -disable-promote-alloca-to-vector < %s | FileCheck %s

; Both of these kernels have the same value for
; amdgpu-flat-work-group-size, except one explicitly sets it. This is
; a program visible property which should always take precedence over
; the amdgpu-waves-per-eu optimization hint.
;
; The range is incompatible with the amdgpu-waves-per-eu value, so the
; flat work group size should take precedence implying a requirement
; to support 1024 size workgroups (which exceeds the available LDS
; amount).

; CHECK-NOT: @no_flat_workgroup_size.stack
; CHECK-NOT: @explicit_default_workgroup_size.stack

; CHECK-LABEL: @no_flat_workgroup_size(
; CHECK: alloca [5 x i32]
; CHECK: store i32 4, i32 addrspace(5)* %arrayidx1, align 4
define amdgpu_kernel void @no_flat_workgroup_size(i32 addrspace(1)* nocapture %out, i32 addrspace(1)* nocapture %in) #0 {
entry:
%stack = alloca [5 x i32], align 4, addrspace(5)
%0 = load i32, i32 addrspace(1)* %in, align 4
%arrayidx1 = getelementptr inbounds [5 x i32], [5 x i32] addrspace(5)* %stack, i32 0, i32 %0
store i32 4, i32 addrspace(5)* %arrayidx1, align 4
%arrayidx2 = getelementptr inbounds i32, i32 addrspace(1)* %in, i32 1
%1 = load i32, i32 addrspace(1)* %arrayidx2, align 4
%arrayidx3 = getelementptr inbounds [5 x i32], [5 x i32] addrspace(5)* %stack, i32 0, i32 %1
store i32 5, i32 addrspace(5)* %arrayidx3, align 4
%arrayidx10 = getelementptr inbounds [5 x i32], [5 x i32] addrspace(5)* %stack, i32 0, i32 0
%2 = load i32, i32 addrspace(5)* %arrayidx10, align 4
store i32 %2, i32 addrspace(1)* %out, align 4
%arrayidx12 = getelementptr inbounds [5 x i32], [5 x i32] addrspace(5)* %stack, i32 0, i32 1
%3 = load i32, i32 addrspace(5)* %arrayidx12
%arrayidx13 = getelementptr inbounds i32, i32 addrspace(1)* %out, i32 1
store i32 %3, i32 addrspace(1)* %arrayidx13
ret void
}

; CHECK-LABEL: @explicit_default_workgroup_size(
; CHECK: alloca [5 x i32]
; CHECK: store i32 4, i32 addrspace(5)* %arrayidx1, align 4
define amdgpu_kernel void @explicit_default_workgroup_size(i32 addrspace(1)* nocapture %out, i32 addrspace(1)* nocapture %in) #1 {
entry:
%stack = alloca [5 x i32], align 4, addrspace(5)
%0 = load i32, i32 addrspace(1)* %in, align 4
%arrayidx1 = getelementptr inbounds [5 x i32], [5 x i32] addrspace(5)* %stack, i32 0, i32 %0
store i32 4, i32 addrspace(5)* %arrayidx1, align 4
%arrayidx2 = getelementptr inbounds i32, i32 addrspace(1)* %in, i32 1
%1 = load i32, i32 addrspace(1)* %arrayidx2, align 4
%arrayidx3 = getelementptr inbounds [5 x i32], [5 x i32] addrspace(5)* %stack, i32 0, i32 %1
store i32 5, i32 addrspace(5)* %arrayidx3, align 4
%arrayidx10 = getelementptr inbounds [5 x i32], [5 x i32] addrspace(5)* %stack, i32 0, i32 0
%2 = load i32, i32 addrspace(5)* %arrayidx10, align 4
store i32 %2, i32 addrspace(1)* %out, align 4
%arrayidx12 = getelementptr inbounds [5 x i32], [5 x i32] addrspace(5)* %stack, i32 0, i32 1
%3 = load i32, i32 addrspace(5)* %arrayidx12
%arrayidx13 = getelementptr inbounds i32, i32 addrspace(1)* %out, i32 1
store i32 %3, i32 addrspace(1)* %arrayidx13
ret void
}

attributes #0 = { "amdgpu-waves-per-eu"="1,1" }
attributes #1 = { "amdgpu-waves-per-eu"="1,1" "amdgpu-flat-work-group-size"="1,1024" }
2 changes: 1 addition & 1 deletion llvm/test/CodeGen/AMDGPU/occupancy-levels.ll
View file
Original file line number Diff line number Diff line change
Expand Up @@ -392,7 +392,7 @@ define amdgpu_kernel void @used_lds_8252_max_group_size_32() #10 {
ret void
}

attributes #0 = { "amdgpu-waves-per-eu"="2,3" }
attributes #0 = { "amdgpu-waves-per-eu"="2,3" "amdgpu-flat-work-group-size"="1,64" }
attributes #1 = { "amdgpu-waves-per-eu"="18,18" }
attributes #2 = { "amdgpu-waves-per-eu"="19,19" }
attributes #3 = { "amdgpu-flat-work-group-size"="1,64" }
Expand Down
49 changes: 0 additions & 49 deletions llvm/test/CodeGen/AMDGPU/propagate-attributes-flat-work-group-size.ll
View file

This file was deleted.

214 changes: 214 additions & 0 deletions llvm/test/CodeGen/AMDGPU/propagate-flat-work-group-size.ll
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,214 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --check-globals
; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -amdgpu-attributor %s | FileCheck %s

; Check propagation of amdgpu-flat-work-group-size attribute.

; Called from a single kernel with 1,256
define internal void @default_to_1_256() {
; CHECK-LABEL: define {{[^@]+}}@default_to_1_256
; CHECK-SAME: () #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: ret void
;
ret void
}

define amdgpu_kernel void @kernel_1_256() #0 {
; CHECK-LABEL: define {{[^@]+}}@kernel_1_256
; CHECK-SAME: () #[[ATTR0]] {
; CHECK-NEXT: call void @default_to_1_256()
; CHECK-NEXT: ret void
;
call void @default_to_1_256()
ret void
}

; Called from a single kernel with 64,128
define internal void @default_to_64_128() {
; CHECK-LABEL: define {{[^@]+}}@default_to_64_128
; CHECK-SAME: () #[[ATTR1:[0-9]+]] {
; CHECK-NEXT: ret void
;
ret void
}

define amdgpu_kernel void @kernel_64_128() #1 {
; CHECK-LABEL: define {{[^@]+}}@kernel_64_128
; CHECK-SAME: () #[[ATTR1]] {
; CHECK-NEXT: call void @default_to_64_128()
; CHECK-NEXT: call void @flat_group_64_64()
; CHECK-NEXT: call void @default_to_64_256()
; CHECK-NEXT: call void @flat_group_128_256()
; CHECK-NEXT: ret void
;
call void @default_to_64_128()
call void @flat_group_64_64()
call void @default_to_64_256()
call void @flat_group_128_256()
ret void
}

; Called from kernels with 128,512 and 512,512
define internal void @default_to_128_512() {
; CHECK-LABEL: define {{[^@]+}}@default_to_128_512
; CHECK-SAME: () #[[ATTR2:[0-9]+]] {
; CHECK-NEXT: ret void
;
ret void
}

; This already has a strict bounds, but called from kernels with wider
; bounds, and should not be changed.
define internal void @flat_group_64_64() #2 {
; CHECK-LABEL: define {{[^@]+}}@flat_group_64_64
; CHECK-SAME: () #[[ATTR3:[0-9]+]] {
; CHECK-NEXT: ret void
;
ret void
}

; 128,256 -> 128,128
define internal void @flat_group_128_256() #3 {
; CHECK-LABEL: define {{[^@]+}}@flat_group_128_256
; CHECK-SAME: () #[[ATTR4:[0-9]+]] {
; CHECK-NEXT: ret void
;
ret void
}

define internal void @flat_group_512_1024() #4 {
; CHECK-LABEL: define {{[^@]+}}@flat_group_512_1024
; CHECK-SAME: () #[[ATTR5:[0-9]+]] {
; CHECK-NEXT: ret void
;
ret void
}

define amdgpu_kernel void @kernel_128_512() #5 {
; CHECK-LABEL: define {{[^@]+}}@kernel_128_512
; CHECK-SAME: () #[[ATTR2]] {
; CHECK-NEXT: call void @default_to_128_512()
; CHECK-NEXT: call void @flat_group_64_64()
; CHECK-NEXT: ret void
;
call void @default_to_128_512()
call void @flat_group_64_64()
ret void
}

define amdgpu_kernel void @kernel_512_512() #6 {
; CHECK-LABEL: define {{[^@]+}}@kernel_512_512
; CHECK-SAME: () #[[ATTR5]] {
; CHECK-NEXT: call void @default_to_128_512()
; CHECK-NEXT: call void @flat_group_512_1024()
; CHECK-NEXT: ret void
;
call void @default_to_128_512()
call void @flat_group_512_1024()
ret void
}

; Called from kernels with 128,256 and 64,128 => 64,256
define internal void @default_to_64_256() {
; CHECK-LABEL: define {{[^@]+}}@default_to_64_256
; CHECK-SAME: () #[[ATTR6:[0-9]+]] {
; CHECK-NEXT: ret void
;
ret void
}

; The kernel's lower bound is higher than the callee's lower bound, so
; this should probably be illegal.
define amdgpu_kernel void @kernel_128_256() #3 {
; CHECK-LABEL: define {{[^@]+}}@kernel_128_256
; CHECK-SAME: () #[[ATTR7:[0-9]+]] {
; CHECK-NEXT: call void @default_to_64_256()
; CHECK-NEXT: ret void
;
call void @default_to_64_256()
ret void
}

; 64,128 -> 64,128
define internal void @merge_cycle_0() #1 {
; CHECK-LABEL: define {{[^@]+}}@merge_cycle_0
; CHECK-SAME: () #[[ATTR1]] {
; CHECK-NEXT: call void @merge_cycle_1()
; CHECK-NEXT: ret void
;
call void @merge_cycle_1()
ret void
}

; 128,256 -> 128,128
define internal void @merge_cycle_1() #3 {
; CHECK-LABEL: define {{[^@]+}}@merge_cycle_1
; CHECK-SAME: () #[[ATTR4]] {
; CHECK-NEXT: call void @merge_cycle_0()
; CHECK-NEXT: ret void
;
call void @merge_cycle_0()
ret void
}

define amdgpu_kernel void @kernel_64_256() #7 {
; CHECK-LABEL: define {{[^@]+}}@kernel_64_256
; CHECK-SAME: () #[[ATTR6]] {
; CHECK-NEXT: call void @merge_cycle_0()
; CHECK-NEXT: call void @default_captured_address()
; CHECK-NEXT: call void @externally_visible_default()
; CHECK-NEXT: [[F32:%.*]] = call float bitcast (i32 ()* @bitcasted_function to float ()*)()
; CHECK-NEXT: ret void
;
call void @merge_cycle_0()
call void @default_captured_address()
call void @externally_visible_default()
%f32 = call float bitcast (i32 ()* @bitcasted_function to float ()*)()
ret void
}

define internal void @default_captured_address() {
; CHECK-LABEL: define {{[^@]+}}@default_captured_address
; CHECK-SAME: () #[[ATTR8:[0-9]+]] {
; CHECK-NEXT: store volatile void ()* @default_captured_address, void ()** undef, align 8
; CHECK-NEXT: ret void
;
store volatile void ()* @default_captured_address, void ()** undef, align 8
ret void
}

define void @externally_visible_default() {
; CHECK-LABEL: define {{[^@]+}}@externally_visible_default
; CHECK-SAME: () #[[ATTR8]] {
; CHECK-NEXT: ret void
;
ret void
}

; 1,1024 -> 64,256
define internal i32 @bitcasted_function() {
; CHECK-LABEL: define {{[^@]+}}@bitcasted_function
; CHECK-SAME: () #[[ATTR6]] {
; CHECK-NEXT: ret i32 0
;
ret i32 0
}

attributes #0 = { "amdgpu-flat-work-group-size"="1,256" }
attributes #1 = { "amdgpu-flat-work-group-size"="64,128" }
attributes #2 = { "amdgpu-flat-work-group-size"="64,64" }
attributes #3 = { "amdgpu-flat-work-group-size"="128,256" }
attributes #4 = { "amdgpu-flat-work-group-size"="512,1024" }
attributes #5 = { "amdgpu-flat-work-group-size"="128,512" }
attributes #6 = { "amdgpu-flat-work-group-size"="512,512" }
attributes #7 = { "amdgpu-flat-work-group-size"="64,256" }
;.
; CHECK: attributes #[[ATTR0]] = { "amdgpu-flat-work-group-size"="1,256" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
; CHECK: attributes #[[ATTR1]] = { "amdgpu-flat-work-group-size"="64,128" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
; CHECK: attributes #[[ATTR2]] = { "amdgpu-flat-work-group-size"="128,512" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
; CHECK: attributes #[[ATTR3]] = { "amdgpu-flat-work-group-size"="64,64" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
; CHECK: attributes #[[ATTR4]] = { "amdgpu-flat-work-group-size"="128,128" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
; CHECK: attributes #[[ATTR5]] = { "amdgpu-flat-work-group-size"="512,512" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
; CHECK: attributes #[[ATTR6]] = { "amdgpu-flat-work-group-size"="64,256" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
; CHECK: attributes #[[ATTR7]] = { "amdgpu-flat-work-group-size"="128,256" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
; CHECK: attributes #[[ATTR8]] = { "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }
;.
2 changes: 1 addition & 1 deletion llvm/test/CodeGen/AMDGPU/schedule-ilp.ll
View file
Original file line number Diff line number Diff line change
Expand Up @@ -585,5 +585,5 @@ bb:
; Function Attrs: nounwind readnone
declare float @llvm.fmuladd.f32(float, float, float) #1

attributes #0 = { nounwind "amdgpu-waves-per-eu"="1,1" }
attributes #0 = { nounwind "amdgpu-waves-per-eu"="1,1" "amdgpu-flat-work-group-size"="1,256" }
attributes #1 = { nounwind readnone }
2 changes: 1 addition & 1 deletion llvm/test/CodeGen/AMDGPU/schedule-regpressure-limit3.ll
View file
Original file line number Diff line number Diff line change
Expand Up @@ -588,4 +588,4 @@ bb:
declare float @llvm.fmuladd.f32(float, float, float) #0

attributes #0 = { nounwind readnone }
attributes #1 = { "amdgpu-waves-per-eu"="1,1" }
attributes #1 = { "amdgpu-waves-per-eu"="1,1" "amdgpu-flat-work-group-size"="1,256" }
4 changes: 2 additions & 2 deletions llvm/test/CodeGen/AMDGPU/target-cpu.ll
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,5 +107,5 @@ attributes #1 = { nounwind readnone }
attributes #2 = { nounwind "target-cpu"="tahiti" }
attributes #3 = { nounwind "target-cpu"="bonaire" }
attributes #4 = { nounwind "target-cpu"="fiji" }
attributes #5 = { nounwind "target-features"="+promote-alloca" "amdgpu-waves-per-eu"="1,3" }
attributes #6 = { nounwind "target-features"="-promote-alloca" "amdgpu-waves-per-eu"="1,3" }
attributes #5 = { nounwind "target-features"="+promote-alloca" "amdgpu-waves-per-eu"="1,3" "amdgpu-flat-work-group-size"="1,256" }
attributes #6 = { nounwind "target-features"="-promote-alloca" "amdgpu-waves-per-eu"="1,3" "amdgpu-flat-work-group-size"="1,256" }