mapreducedim! not threadsafe #588

danielwe · 2020-12-04T21:59:59Z

Describe the bug

CuArray reduction operations that eventually land in mapreducedim! may segfault when called from multiple host threads simultaneously.

To reproduce

$ julia --threads=2

julia> using CUDA

julia> function _sum(x::CuVector)  # My own sum function, see below for the actual definition
           # [...]
       end

julia> function sums(n, sum_f)
           xs = [CUDA.randn(1024) for _ in 1:n]
           for x in xs
               sum_f(x)
           end
       end
sums (generic function with 1 method)

julia> function sums_threaded(n, sum_f)
           xs = [CUDA.randn(1024) for _ in 1:n]
           Threads.@threads for x in xs
               sum_f(x)
           end
       end
sums_threaded (generic function with 1 method)

julia> sums(200000, _sum)

julia> sums(200000, sum)

julia> sums_threaded(200000, _sum)

julia> sums_threaded(200000, sum)

signal (11): Segmentation fault

signal (11): Segmentation fault
in expression starting at REPL[70]:1
in expression starting at REPL[70]:1
jl_get_cfunction_trampoline at ~/local/julia-1.5.3/bin/../lib/libjul
ia.so.1 (unknown line)
#launch_configuration#606 at ~/.julia/packages/CUDA/YeS8q/lib/cudadr
v/occupancy.jl:61 [inlined]
launch_configuration##kw at ~/.julia/packages/CUDA/YeS8q/lib/cudadrv
/occupancy.jl:55 [inlined]
#mapreducedim!#900 at ~/.julia/packages/CUDA/YeS8q/src/mapreduce.jl:
197
mapreducedim!##kw at ~/.julia/packages/CUDA/YeS8q/src/mapreduce.jl:1
43 [inlined]
#mapreducedim!#900 at ~/.julia/packages/CUDA/YeS8q/src/mapreduce.jl:
239
mapreducedim!##kw at ~/.julia/packages/CUDA/YeS8q/src/mapreduce.jl:1
43 [inlined]
#_mapreduce#17 at ~/.julia/packages/GPUArrays/jhRU7/src/host/mapredu
ce.jl:62
_mapreduce##kw at ~/.julia/packages/GPUArrays/jhRU7/src/host/mapredu
ce.jl:34 [inlined]
#mapreduce#15 at ~/.julia/packages/GPUArrays/jhRU7/src/host/mapreduc
e.jl:28 [inlined]
mapreduce at ~/.julia/packages/GPUArrays/jhRU7/src/host/mapreduce.jl
:28 [inlined]
_sum at ./reducedim.jl:727 [inlined]
_sum at ./reducedim.jl:726 [inlined]
#sum#627 at ./reducedim.jl:722 [inlined]
sum at ./reducedim.jl:722 [inlined]
macro expansion at ./REPL[66]:4 [inlined]
#11#threadsfor_fun at ./threadingconstructs.jl:81
#11#threadsfor_fun at ./threadingconstructs.jl:48
unknown function (ip: 0x7fec602a018c)
trampoline_alloc at /buildworker/worker/package_linux64/build/src/runtime_ccall
.cpp:228 [inlined]
jl_get_cfunction_trampoline at /buildworker/worker/package_linux64/build/src/ru
ntime_ccall.cpp:334
#launch_configuration#606 at ~/.julia/packages/CUDA/YeS8q/lib/cudadr
v/occupancy.jl:61 [inlined]
launch_configuration##kw at ~/.julia/packages/CUDA/YeS8q/lib/cudadrv
/occupancy.jl:55 [inlined]
#mapreducedim!#900 at ~/.julia/packages/CUDA/YeS8q/src/mapreduce.jl:
197
mapreducedim!##kw at ~/.julia/packages/CUDA/YeS8q/src/mapreduce.jl:1
43 [inlined]
#_mapreduce#17 at ~/.julia/packages/GPUArrays/jhRU7/src/host/mapredu
ce.jl:62
_mapreduce##kw at ~/.julia/packages/GPUArrays/jhRU7/src/host/mapredu
ce.jl:34 [inlined]
#mapreduce#15 at ~/.julia/packages/GPUArrays/jhRU7/src/host/mapreduc
e.jl:28 [inlined]
mapreduce at ~/.julia/packages/GPUArrays/jhRU7/src/host/mapreduce.jl
:28 [inlined]
_sum at ./reducedim.jl:727 [inlined]
_sum at ./reducedim.jl:726 [inlined]
#sum#627 at ./reducedim.jl:722 [inlined]
sum at ./reducedim.jl:722 [inlined]
macro expansion at ./REPL[66]:4 [inlined]
#11#threadsfor_fun at ./threadingconstructs.jl:81
#11#threadsfor_fun at ./threadingconstructs.jl:48
unknown function (ip: 0x7fec602a018c)
_jl_invoke at /buildworker/worker/package_linux64/build/src/gf.c:2231 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:2398
_jl_invoke at /buildworker/worker/package_linux64/build/src/gf.c:2231 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:2398
jl_apply at /buildworker/worker/package_linux64/build/src/julia.h:1690 [inlined
]
start_task at /buildworker/worker/package_linux64/build/src/task.c:705
unknown function (ip: (nil))
Allocations: 98869074 (Pool: 98853314; Big: 15760); GC: 61
Segmentation fault (core dumped)

Manifest.toml

# This file is machine-generated - editing it directly is not advised

[[AbstractFFTs]]
deps = ["LinearAlgebra"]
git-tree-sha1 = "051c95d6836228d120f5f4b984dd5aba1624f716"
uuid = "621f4979-c628-5d54-868e-fcf4e3e8185c"
version = "0.5.0"

[[Adapt]]
deps = ["LinearAlgebra"]
git-tree-sha1 = "42c42f2221906892ceb765dbcb1a51deeffd86d7"
uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e"
version = "2.3.0"

[[BFloat16s]]
deps = ["LinearAlgebra", "Test"]
git-tree-sha1 = "4af69e205efc343068dc8722b8dfec1ade89254a"
uuid = "ab4f0b2a-ad5b-11e8-123f-65d77653426b"
version = "0.1.0"

[[Base64]]
uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"

[[CEnum]]
git-tree-sha1 = "215a9aa4a1f23fbd05b92769fdd62559488d70e9"
uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82"
version = "0.4.1"

[[CUDA]]
deps = ["AbstractFFTs", "Adapt", "BFloat16s", "CEnum", "DataStructures", "ExprTools", "GPUArrays", "GPUCompiler", "LLVM", "Libdl", "LinearAlgebra", "Logging", "MacroTools", "NNlib", "Pkg", "Printf", "Random", "Reexport", "Requires", "SparseArrays", "Statistics", "TimerOutputs"]
git-tree-sha1 = "7663b61782b569b03fba91d330a5ed2f86cd4cb8"
uuid = "052768ef-5323-5732-b1bb-66c8b64840ba"
version = "2.3.0"

[[Compat]]
deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "SHA", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"]
git-tree-sha1 = "a706ff10f1cd8dab94f59fd09c0e657db8e77ff0"
uuid = "34da2185-b29b-5c13-b0c7-acf172513d20"
version = "3.23.0"

[[DataStructures]]
deps = ["Compat", "InteractiveUtils", "OrderedCollections"]
git-tree-sha1 = "fb0aa371da91c1ff9dc7fbed6122d3e411420b9c"
uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
version = "0.18.8"

[[Dates]]
deps = ["Printf"]
uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"

[[DelimitedFiles]]
deps = ["Mmap"]
uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab"

[[Distributed]]
deps = ["Random", "Serialization", "Sockets"]
uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"

[[ExprTools]]
git-tree-sha1 = "10407a39b87f29d47ebaca8edbc75d7c302ff93e"
uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04"
version = "0.1.3"

[[GPUArrays]]
deps = ["AbstractFFTs", "Adapt", "LinearAlgebra", "Printf", "Random", "Serialization"]
git-tree-sha1 = "2c1dd57bca7ba0b3b4bf81d9332aeb81b154ef4c"
uuid = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7"
version = "6.1.2"

[[GPUCompiler]]
deps = ["DataStructures", "InteractiveUtils", "LLVM", "Libdl", "Scratch", "Serialization", "TimerOutputs", "UUIDs"]
git-tree-sha1 = "c853c810b52a80f9aad79ab109207889e57f41ef"
uuid = "61eb1bfa-7361-4325-ad38-22787b887f55"
version = "0.8.3"

[[InteractiveUtils]]
deps = ["Markdown"]
uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"

[[LLVM]]
deps = ["CEnum", "Libdl", "Printf", "Unicode"]
git-tree-sha1 = "000a737732aa4eb996414c4685368f6a74b41d14"
uuid = "929cbde3-209d-540e-8aea-75f648917ca0"
version = "3.4.0"

[[LibGit2]]
deps = ["Printf"]
uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"

[[Libdl]]
uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"

[[LinearAlgebra]]
deps = ["Libdl"]
uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"

[[Logging]]
uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"

[[MacroTools]]
deps = ["Markdown", "Random"]
git-tree-sha1 = "6a8a2a625ab0dea913aba95c11370589e0239ff0"
uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
version = "0.5.6"

[[Markdown]]
deps = ["Base64"]
uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"

[[Mmap]]
uuid = "a63ad114-7e13-5084-954f-fe012c677804"

[[NNlib]]
deps = ["Compat", "Libdl", "LinearAlgebra", "Pkg", "Requires", "Statistics"]
git-tree-sha1 = "1ae42464fea5258fd2ff49f1c4a40fc41cba3860"
uuid = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
version = "0.7.7"

[[OrderedCollections]]
git-tree-sha1 = "cf59cfed2e2c12e8a2ff0a4f1e9b2cd8650da6db"
uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
version = "1.3.2"

[[Pkg]]
deps = ["Dates", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"]
uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"

[[Printf]]
deps = ["Unicode"]
uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"

[[REPL]]
deps = ["InteractiveUtils", "Markdown", "Sockets"]
uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"

[[Random]]
deps = ["Serialization"]
uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"

[[Reexport]]
deps = ["Pkg"]
git-tree-sha1 = "7b1d07f411bc8ddb7977ec7f377b97b158514fe0"
uuid = "189a3867-3050-52da-a836-e630ba90ab69"
version = "0.2.0"

[[Requires]]
deps = ["UUIDs"]
git-tree-sha1 = "e05c53ebc86933601d36212a93b39144a2733493"
uuid = "ae029012-a4dd-5104-9daa-d747884805df"
version = "1.1.1"

[[SHA]]
uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"

[[Scratch]]
deps = ["Dates"]
git-tree-sha1 = "ad4b278adb62d185bbcb6864dc24959ab0627bf6"
uuid = "6c6a2e73-6563-6170-7368-637461726353"
version = "1.0.3"

[[Serialization]]
uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"

[[SharedArrays]]
deps = ["Distributed", "Mmap", "Random", "Serialization"]
uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383"

[[Sockets]]
uuid = "6462fe0b-24de-5631-8697-dd941f90decc"

[[SparseArrays]]
deps = ["LinearAlgebra", "Random"]
uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"

[[Statistics]]
deps = ["LinearAlgebra", "SparseArrays"]
uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"

[[Test]]
deps = ["Distributed", "InteractiveUtils", "Logging", "Random"]
uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

[[TimerOutputs]]
deps = ["Printf"]
git-tree-sha1 = "3318281dd4121ecf9713ce1383b9ace7d7476fdd"
uuid = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f"
version = "0.5.7"

[[UUIDs]]
deps = ["Random", "SHA"]
uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"

[[Unicode]]
uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"

Expected behavior

No segfault

Version info

Details on Julia:

Julia Version 1.5.3
Commit 788b2c77c1 (2020-11-09 13:37 UTC)
Platform Info:
  OS: Linux (x86_64-pc-linux-gnu)
  CPU: Intel(R) Core(TM) i7-7700HQ CPU @ 2.80GHz
  WORD_SIZE: 64
  LIBM: libopenlibm
  LLVM: libLLVM-9.0.1 (ORCJIT, skylake)

Details on CUDA:

CUDA toolkit 11.1.1, artifact installation
CUDA driver 11.1.0
NVIDIA driver 455.45.1

Libraries: 
- CUBLAS: 11.3.0
- CURAND: 10.2.2
- CUFFT: 10.3.0
- CUSOLVER: 11.0.1
- CUSPARSE: 11.3.0
- CUPTI: 14.0.0
- NVML: 11.0.0+455.45.1
- CUDNN: 8.0.4 (for CUDA 11.1.0)
- CUTENSOR: 1.2.1 (for CUDA 11.1.0)

Toolchain:
- Julia: 1.5.3
- LLVM: 9.0.1
- PTX ISA support: 3.2, 4.0, 4.1, 4.2, 4.3, 5.0, 6.0, 6.1, 6.3, 6.4
- Device support: sm_35, sm_37, sm_50, sm_52, sm_53, sm_60, sm_61, sm_62, sm_70
, sm_72, sm_75

1 device:
  0: GeForce GTX 1050 (sm_61, 3.829 GiB / 3.948 GiB available)

Additional context

My own sum function

For completeness, here's my own sum function that does not seem to have thread safety issues. It's vanilla tree reduction, most of the code is kernel boilerplate.

using CUDA: @allowscalar, unsafe_free!

function _sum(x::CuVector{T}) where {T}
    function _sum_kernel!(out::CuDeviceVector{T}, x) where {T}
        ti, nt = threadIdx().x, blockDim().x
        shared = @cuDynamicSharedMem(T, nt)
        shared[ti] = 0
        @inbounds for i in ti:nt:length(x)
            shared[ti] += x[i]
        end
        reduce_sum(shared, nt, ti)
        if ti == 1
            out[1] = shared[1]
        end
        return nothing
    end

    N = length(x)
    if N == 1
        return @allowscalar a[1]
    end
    threads, next = 1, 2
    while next < N  # Find largest 2^k < N
        threads = next
        next <<= 1
    end
    out = CuVector{T}(undef, 1)
    shmem = threads * sizeof(T)
    @cuda threads=threads shmem=shmem _sum_kernel!(out, x)
    synchronize(CuDefaultStream())
    s = @allowscalar out[1]
    unsafe_free!(out)
    return s
end

function reduce_sum(x::CuDeviceVector, n, t)
    while n > 1
        n >>= 1
        sync_threads()
        if t <= n
            x[t] += x[t + n]
        end
    end
    return nothing
end

The text was updated successfully, but these errors were encountered:

vchuravy · 2020-12-04T23:13:34Z

Jup it is even noted in the code as a TODO: https://github.com/JuliaLang/julia/blob/8629e213eac0ffc26a9f18fae949f378e3defec5/src/runtime_ccall.cpp#L271

danielwe added the bug Something isn't working label Dec 4, 2020

vchuravy mentioned this issue Dec 4, 2020

jl_get_cfunction_trampoline is not thread-safe JuliaLang/julia#38709

Closed

vchuravy added the upstream Somebody else's problem. label Dec 4, 2020

vchuravy mentioned this issue Dec 4, 2020

Add lock around cfunction lookup #589

Merged

maleadt closed this as completed in #589 Dec 7, 2020

danielwe mentioned this issue Dec 8, 2020

Thread safety issue with free #595

Closed

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

mapreducedim! not threadsafe #588

mapreducedim! not threadsafe #588

danielwe commented Dec 4, 2020 •

edited

vchuravy commented Dec 4, 2020

mapreducedim! not threadsafe #588

mapreducedim! not threadsafe #588

Comments

danielwe commented Dec 4, 2020 • edited

vchuravy commented Dec 4, 2020

danielwe commented Dec 4, 2020 •

edited