Int128 regression in Julia 1.12

[nhz2]

I don't have a AMD gpu that I can debug this on but something has broken with Int128 arrays in Julia 1.12.
This was found when testing a GPUArrays.jl PR that adds Int128 to the list of eltypes tested in the testsuite: https://buildkite.com/julialang/gpuarrays-dot-jl/builds/1828/canvas?sid=019f99ef-2d3f-4354-b5aa-41128d35a9f0

[simeonschaub]

Do you have a minimal example of a case that fails?

[nhz2]

No I only have the failed CI logs. The testsets that fail are "gpuarrays/linalg/core/axp{b}y/Int128", "gpuarrays/broadcasting/broadcast Int128", and "gpuarrays/broadcasting/ComposedFunction Int128". The error messages are:

GPU Kernel Exception:
Unknown exception

[luraess]

If you get a chance to post a MWE, that'd be helpful. Thanks!

[simeonschaub]

One example that fails:

Start Julia with julia --check-bounds=yes

julia> using AMDGPU

julia> function test_kernel(a::T, b) where T
           c = a
           for i=1:5
               c += T(b)
               c = a * T(2)
           end
           return c
       end
test_kernel (generic function with 1 method)

julia> test_kernel.(AMDGPU.rand(Int128, 10, 10), Int128(10))
10×10 ROCArray{Int128, 2, AMDGPU.Runtime.Mem.HIPBuffer}:
Error showing value of type ROCArray{Int128, 2, AMDGPU.Runtime.Mem.HIPBuffer}:

SYSTEM (REPL): showing an error caused an error
ERROR: 1-element ExceptionStack:
GPU Kernel Exception:
Unknown exception
workgroupIdx: (1, 1, 1)
Stacktrace:
  [1] error(s::String)
    @ Base ./error.jl:44
  [2] throw_if_exception
    @ ~/.julia/packages/AMDGPU/cLA6g/src/exception_handler.jl:35 [inlined]
  [3] synchronize(stm::HIPStream; blocking::Bool, stop_hostcalls::Bool)
    @ AMDGPU ~/.julia/packages/AMDGPU/cLA6g/src/highlevel.jl:40
  [4] synchronize
    @ ~/.julia/packages/AMDGPU/cLA6g/src/highlevel.jl:36 [inlined]
  [5] copyto!(dest::Matrix{Int128}, d_offset::Int64, source::ROCArray{Int128, 2, AMDGPU.Runtime.Mem.HIPBuffer}, s_offset::Int64, amount::Int64; async::Bool)
    @ AMDGPU ~/.julia/packages/AMDGPU/cLA6g/src/array.jl:225
  [6] copyto!
    @ ~/.julia/packages/AMDGPU/cLA6g/src/array.jl:215 [inlined]
  [7] copyto!
    @ ~/.julia/packages/GPUArrays/VNJ6B/src/host/abstractarray.jl:174 [inlined]
  [8] copyto_axcheck!
    @ ./abstractarray.jl:1176 [inlined]
  [9] Array
    @ ./array.jl:621 [inlined]
 [10] Array
    @ ./boot.jl:674 [inlined]
 [11] convert
    @ ./array.jl:614 [inlined]
 [12] adapt_storage
    @ ~/.julia/packages/GPUArrays/VNJ6B/src/host/abstractarray.jl:137 [inlined]
 [13] adapt_structure
    @ ~/.julia/packages/Adapt/YzfDi/src/Adapt.jl:50 [inlined]
 [14] adapt(to::GPUArrays.ToArray, x::ROCArray{Int128, 2, AMDGPU.Runtime.Mem.HIPBuffer})
    @ Adapt ~/.julia/packages/Adapt/YzfDi/src/Adapt.jl:40
 [15] print_array(io::IOContext{REPL.LimitIO{Base.TTY}}, X::AnyGPUArray)
    @ GPUArrays ~/.julia/packages/GPUArrays/VNJ6B/src/host/abstractarray.jl:141
 [16] show(io::IOContext{REPL.LimitIO{Base.TTY}}, ::MIME{Symbol("text/plain")}, X::ROCArray{Int128, 2, AMDGPU.Runtime.Mem.HIPBuffer})
    @ Base ./arrayshow.jl:399
 [17] show_repl(io::IO, mime::MIME{Symbol("text/plain")}, x::Any)
    @ REPL ~/.julia/juliaup/julia-1.12.6+0.x64.linux.gnu/share/julia/stdlib/v1.12/REPL/src/REPL.jl:548
 [18] show_limited(io::IO, mime::MIME, x::Any)
    @ REPL ~/.julia/juliaup/julia-1.12.6+0.x64.linux.gnu/share/julia/stdlib/v1.12/REPL/src/REPL.jl:517
 [19] (::REPL.var"#display##0#display##1"{REPL.REPLDisplay{REPL.LineEditREPL}, MIME{Symbol("text/plain")}, Base.RefValue{Any}})(io::Any)
    @ REPL ~/.julia/juliaup/julia-1.12.6+0.x64.linux.gnu/share/julia/stdlib/v1.12/REPL/src/REPL.jl:540
 [20] with_repl_linfo(f::Any, repl::REPL.LineEditREPL)
    @ REPL ~/.julia/juliaup/julia-1.12.6+0.x64.linux.gnu/share/julia/stdlib/v1.12/REPL/src/REPL.jl:833
 [21] display(d::REPL.REPLDisplay, mime::MIME{Symbol("text/plain")}, x::Any)
    @ REPL ~/.julia/juliaup/julia-1.12.6+0.x64.linux.gnu/share/julia/stdlib/v1.12/REPL/src/REPL.jl:526
 [22] display
    @ ~/.julia/juliaup/julia-1.12.6+0.x64.linux.gnu/share/julia/stdlib/v1.12/REPL/src/REPL.jl:546 [inlined]
 [23] display(x::Any)
    @ Base.Multimedia ./multimedia.jl:340
 [24] (::REPL.var"#print_response##2#print_response##3")()
    @ REPL ~/.julia/juliaup/julia-1.12.6+0.x64.linux.gnu/share/julia/stdlib/v1.12/REPL/src/REPL.jl:590
 [25] repl_backend_loop(backend::REPL.REPLBackend, get_module::Function)
    @ REPL ~/.julia/juliaup/julia-1.12.6+0.x64.linux.gnu/share/julia/stdlib/v1.12/REPL/src/REPL.jl:445
 [26] start_repl_backend(backend::REPL.REPLBackend, consumer::Any; get_module::Function)
    @ REPL ~/.julia/juliaup/julia-1.12.6+0.x64.linux.gnu/share/julia/stdlib/v1.12/REPL/src/REPL.jl:427
 [27] start_repl_backend
    @ ~/.julia/juliaup/julia-1.12.6+0.x64.linux.gnu/share/julia/stdlib/v1.12/REPL/src/REPL.jl:424 [inlined]
 [28] run_repl(repl::REPL.AbstractREPL, consumer::Any; backend_on_current_task::Bool, backend::Any)
    @ REPL ~/.julia/juliaup/julia-1.12.6+0.x64.linux.gnu/share/julia/stdlib/v1.12/REPL/src/REPL.jl:653
 [29] run_repl(repl::REPL.AbstractREPL, consumer::Any)
    @ REPL ~/.julia/juliaup/julia-1.12.6+0.x64.linux.gnu/share/julia/stdlib/v1.12/REPL/src/REPL.jl:639
 [30] run_std_repl(REPL::Module, quiet::Bool, banner::Symbol, history_file::Bool)
    @ Base ./client.jl:478
 [31] run_main_repl(interactive::Bool, quiet::Bool, banner::Symbol, history_file::Bool)
    @ Base ./client.jl:499
 [32] repl_main
    @ ./client.jl:586 [inlined]
 [33] _start()
    @ Base ./client.jl:561
Stacktrace:
  [1] print_response(errio::IO, response::Any, backend::Union{Nothing, REPL.REPLBackendRef}, show_value::Bool, have_color::Bool, specialdisplay::Union{Nothing, AbstractDisplay})
    @ REPL ~/.julia/juliaup/julia-1.12.6+0.x64.linux.gnu/share/julia/stdlib/v1.12/REPL/src/REPL.jl:599
  [2] (::REPL.var"#print_response##0#print_response##1"{REPL.LineEditREPL, Pair{Any, Bool}, Bool, Bool})(io::Any)
    @ REPL ~/.julia/juliaup/julia-1.12.6+0.x64.linux.gnu/share/julia/stdlib/v1.12/REPL/src/REPL.jl:558
  [3] with_repl_linfo(f::Any, repl::REPL.LineEditREPL)
    @ REPL ~/.julia/juliaup/julia-1.12.6+0.x64.linux.gnu/share/julia/stdlib/v1.12/REPL/src/REPL.jl:833
  [4] print_response(repl::REPL.AbstractREPL, response::Any, show_value::Bool, have_color::Bool)
    @ REPL ~/.julia/juliaup/julia-1.12.6+0.x64.linux.gnu/share/julia/stdlib/v1.12/REPL/src/REPL.jl:556
  [5] (::REPL.var"#do_respond#73"{Bool, Bool, REPL.var"#setup_interface##12#setup_interface##13"{REPL.LineEditREPL, REPL.REPLHistoryProvider}, REPL.LineEditREPL, REPL.LineEdit.Prompt})(s::REPL.LineEdit.MIState, buf::Any, ok::Bool)
    @ REPL ~/.julia/juliaup/julia-1.12.6+0.x64.linux.gnu/share/julia/stdlib/v1.12/REPL/src/REPL.jl:1192
  [6] (::REPL.var"#setup_interface##32#setup_interface##33"{Regex, Regex, Int64, Int64, REPL.LineEdit.Prompt, REPL.LineEdit.Prompt, REPL.LineEdit.Prompt})(::REPL.LineEdit.MIState, ::Any, ::Vararg{Any})
    @ REPL ~/.julia/juliaup/julia-1.12.6+0.x64.linux.gnu/share/julia/stdlib/v1.12/REPL/src/REPL.jl:1599
  [7] (::REPL.LineEdit.var"#match_input##0#match_input##1"{REPL.var"#setup_interface##32#setup_interface##33"{Regex, Regex, Int64, Int64, REPL.LineEdit.Prompt, REPL.LineEdit.Prompt, REPL.LineEdit.Prompt}, String})(s::Any, p::Any)
    @ REPL.LineEdit ~/.julia/juliaup/julia-1.12.6+0.x64.linux.gnu/share/julia/stdlib/v1.12/REPL/src/LineEdit.jl:1810
  [8] macro expansion
    @ ~/.julia/juliaup/julia-1.12.6+0.x64.linux.gnu/share/julia/stdlib/v1.12/REPL/src/LineEdit.jl:2960 [inlined]
  [9] macro expansion
    @ ./lock.jl:376 [inlined]
 [10] (::REPL.LineEdit.var"#prompt!##2#prompt!##3"{Base.Terminals.TTYTerminal, REPL.LineEdit.ModalInterface, REPL.LineEdit.MIState, REPL.LineEdit.Prompt})()
    @ REPL.LineEdit ~/.julia/juliaup/julia-1.12.6+0.x64.linux.gnu/share/julia/stdlib/v1.12/REPL/src/LineEdit.jl:2949

unfortunately, the error message and stacktrace is not all that helpful

[simeonschaub]

Doesn't happen if using either AMDGPU_LLVM_Backend_jll or #919, so it looks like a miscompilation with the in-tree backend, the LLVM v18 that Julia ships with. Not sure if it's worth bisecting and backporting a fix since we seem to be moving away from using the in-tree backend anyways

[nhz2]

Can this be reopened please. #1005 looks to have fixed two of the failing tests, but there are still two failing tests in "gpuarrays/linalg/core/axp{b}y/Int128"
Ref: JuliaGPU/GPUArrays.jl#723 https://buildkite.com/julialang/gpuarrays-dot-jl/builds/1843/list?jid=019fbeb8-fa84-4c6f-aef7-6b60fec37a5f&tab=output

[nhz2]

A fix to this should add an overload to supported_eltypes(AT) with all the types that are intended to work on AMDGPU.jl to more fully test things. Like what OneAPI.jl does at https://github.com/JuliaGPU/oneAPI.jl/blob/0a71d96a3e5c6fc5e42f6f6efb639c5d4f52aec2/test/runtests.jl#L119