Dummy PR to demonstrate that CI is failing on Pkg#master when using Julia nightly

[DilumAluthge]

No description provided.

[DilumAluthge]

@StefanKarpinski @staticfloat This demonstrates the failures on the latest Julia nightly. It seems that they are related to curl.

The failures only occur on Julia nightly. They are present on multiple operating systems (Linux, macOS, and Windows).

[DilumAluthge]

Okay, I have a reproducer, although it is not minimal.

First, save the following code into a file named script.jl: (click to expand)

using Pkg
using TOML
using Test
using UUIDs

const artifact_meta = Pkg.Artifacts.artifact_meta
const artifact_path = Pkg.Artifacts.artifact_path

const LOADED_DEPOT = joinpath(@__DIR__, "loaded_depot")

const REGISTRY_DEPOT = joinpath(@__DIR__, "registry_depot")
const REGISTRY_DIR = joinpath(REGISTRY_DEPOT, "registries", "General")

const GENERAL_UUID = UUID("23338594-aafe-5451-b93e-139f81909106")

function init_reg()
    mkpath(REGISTRY_DIR)
    if Pkg.Registry.registry_use_pkg_server()
        url = Pkg.Registry.pkg_server_registry_urls()[GENERAL_UUID]
        @info "Downloading General registry from $url"
        Pkg.PlatformEngines.download_verify_unpack(url, nothing, REGISTRY_DIR, ignore_existence = true, io = stderr)
        tree_info_file = joinpath(REGISTRY_DIR, ".tree_info.toml")
        hash = Pkg.Registry.pkg_server_url_hash(url)
        write(tree_info_file, "git-tree-sha1 = " * repr(string(hash)))
    else
        Base.shred!(LibGit2.CachedCredentials()) do creds
            LibGit2.with(Pkg.GitTools.clone(
                stderr,
                "https://github.com/JuliaRegistries/General.git",
                REGISTRY_DIR,
                credentials = creds)) do repo
            end
        end
    end
end

function isolate(fn::Function; loaded_depot=false, linked_reg=true)
    old_load_path = copy(LOAD_PATH)
    old_depot_path = copy(DEPOT_PATH)
    old_home_project = Base.HOME_PROJECT[]
    old_active_project = Base.ACTIVE_PROJECT[]
    old_working_directory = pwd()
    old_general_registry_url = Pkg.Registry.DEFAULT_REGISTRIES[1].url
    old_general_registry_path = Pkg.Registry.DEFAULT_REGISTRIES[1].path
    old_general_registry_linked = Pkg.Registry.DEFAULT_REGISTRIES[1].linked
    try
        # Clone/download the registry only once
        if !isdir(REGISTRY_DIR)
            init_reg()
        end

        empty!(LOAD_PATH)
        empty!(DEPOT_PATH)
        Base.HOME_PROJECT[] = nothing
        Base.ACTIVE_PROJECT[] = nothing
        Pkg.UPDATED_REGISTRY_THIS_SESSION[] = false
        Pkg.Registry.DEFAULT_REGISTRIES[1].url = nothing
        Pkg.Registry.DEFAULT_REGISTRIES[1].path = REGISTRY_DIR
        Pkg.Registry.DEFAULT_REGISTRIES[1].linked = linked_reg
        Pkg.REPLMode.TEST_MODE[] = false
        withenv("JULIA_PROJECT" => nothing,
                "JULIA_LOAD_PATH" => nothing,
                "JULIA_PKG_DEVDIR" => nothing) do
            target_depot = nothing
            try
                target_depot = mktempdir()
                push!(LOAD_PATH, "@", "@v#.#", "@stdlib")
                push!(DEPOT_PATH, target_depot)
                loaded_depot && push!(DEPOT_PATH, LOADED_DEPOT)
                fn()
            finally
                if target_depot !== nothing && isdir(target_depot)
                    try
                        Base.rm(target_depot; force=true, recursive=true)
                    catch err
                        @show err
                    end
                end
            end
        end
    finally
        empty!(LOAD_PATH)
        empty!(DEPOT_PATH)
        append!(LOAD_PATH, old_load_path)
        append!(DEPOT_PATH, old_depot_path)
        Base.HOME_PROJECT[] = old_home_project
        Base.ACTIVE_PROJECT[] = old_active_project
        cd(old_working_directory)
        Pkg.REPLMode.TEST_MODE[] = false # reset unconditionally
        Pkg.Registry.DEFAULT_REGISTRIES[1].path = old_general_registry_path
        Pkg.Registry.DEFAULT_REGISTRIES[1].url = old_general_registry_url
        Pkg.Registry.DEFAULT_REGISTRIES[1].linked = old_general_registry_linked
    end
end

@testset "Pkg.add() with julia_version" begin
    # A package with artifacts that went from normal package -> stdlib
    gmp_jll_uuid = "781609d7-10c4-51f6-84f2-b8444358ff6d"
    # A package that has always only ever been an stdlib
    linalg_uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
    # A package that went from normal package - >stdlib
    networkoptions_uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"

    function get_manifest_block(name)
        manifest_path = joinpath(dirname(Base.active_project()), "Manifest.toml")
        @test isfile(manifest_path)
        deps = Base.get_deps(TOML.parsefile(manifest_path))
        @test haskey(deps, name)
        return only(deps[name])
    end

    isolate(loaded_depot=true) do
        # Next, test that if we ask for `v1.5` it DOES have a version, and that GMP_jll installs v6.1.X
        Pkg.add(["NetworkOptions", "GMP_jll"]; julia_version=v"1.5")
        no_block = get_manifest_block("NetworkOptions")
        @test haskey(no_block, "uuid")
        @test no_block["uuid"] == networkoptions_uuid
        @test haskey(no_block, "version")

        gmp_block = get_manifest_block("GMP_jll")
        @test haskey(gmp_block, "uuid")
        @test gmp_block["uuid"] == gmp_jll_uuid
        @test haskey(gmp_block, "version")
        @test startswith(gmp_block["version"], "6.1.2")

        # Test that the artifact of GMP_jll contains the right library
        @test haskey(gmp_block, "git-tree-sha1")
        gmp_jll_dir = Pkg.Operations.find_installed("GMP_jll", Base.UUID(gmp_jll_uuid), Base.SHA1(gmp_block["git-tree-sha1"]))
        @test isdir(gmp_jll_dir)
        artifacts_toml = joinpath(gmp_jll_dir, "Artifacts.toml")
        @test isfile(artifacts_toml)
        meta = artifact_meta("GMP", artifacts_toml)
        @test meta !== nothing

        gmp_artifact_path = artifact_path(Base.SHA1(meta["git-tree-sha1"]))
        @test isdir(gmp_artifact_path)

        # On linux, we can check the filename to ensure it's grabbing the correct library
        if Sys.islinux()
            libgmp_filename = joinpath(gmp_artifact_path, "lib", "libgmp.so.10.3.2")
            @test isfile(libgmp_filename)
        end
    end

    # Next, test that if we ask for `v1.6`, GMP_jll gets `v6.2.0`, and for `v1.7`, it gets `v6.2.1`
    function do_gmp_test(julia_version, gmp_version)
        isolate(loaded_depot=true) do
            Pkg.add("GMP_jll"; julia_version)
            gmp_block = get_manifest_block("GMP_jll")
            @test haskey(gmp_block, "uuid")
            @test gmp_block["uuid"] == gmp_jll_uuid
            @test haskey(gmp_block, "version")
            @test startswith(gmp_block["version"], string(gmp_version))
        end
    end
    do_gmp_test(v"1.6", v"6.2.0")
    do_gmp_test(v"1.7", v"6.2.1")

    isolate(loaded_depot=true) do
        # Next, test that if we ask for `nothing`, NetworkOptions has a `version` but `LinearAlgebra` does not.
        Pkg.add(["LinearAlgebra", "NetworkOptions"]; julia_version=nothing)
        no_block = get_manifest_block("NetworkOptions")
        @test haskey(no_block, "uuid")
        @test no_block["uuid"] == networkoptions_uuid
        @test haskey(no_block, "version")
        linalg_block = get_manifest_block("LinearAlgebra")
        @test haskey(linalg_block, "uuid")
        @test linalg_block["uuid"] == linalg_uuid
        @test !haskey(linalg_block, "version")
    end

    isolate(loaded_depot=true) do
        # Next, test that stdlibs do not get dependencies from the registry
        # NOTE: this test depends on the fact that in Julia v1.6+ we added
        # "fake" JLLs that do not depend on Pkg while the "normal" p7zip_jll does.
        # A future p7zip_jll in the registry may not depend on Pkg, so be sure
        # to verify your assumptions when updating this test.
        Pkg.add("p7zip_jll")
        p7zip_jll_uuid = UUID("3f19e933-33d8-53b3-aaab-bd5110c3b7a0")
        @test !("Pkg" in keys(Pkg.dependencies()[p7zip_jll_uuid].dependencies))
    end
end

Next, run the following command in your shell. It will eventually segfault. Make sure that you are using Julia nightly.

while julia script.jl; do :; done

If you want to run it under rr, just download the rr_capture.jl script. Then, run the following command in your shell:

while julia rr_capture.jl julia script.jl; do :; done

I have an rr trace of the segfault. It's too big to upload to GitHub, so if you want it, just DM me on Slack, and I'll send it to you.

Here is an example log of the segfault:

signal (11): Segmentation fault
in expression starting at /home/dilum/Desktop/script.jl:96
curl_multi_socket_action at /home/dilum/.asdf/installs/julia/nightly/bin/../lib/julia/libcurl.so (unknown line)
unknown function (ip: 0x7f01257fb85f)
lock at ./lock.jl:190
do_multi at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.8/Downloads/src/Curl/Multi.jl:101 [inlined]
#33 at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.8/Downloads/src/Curl/Multi.jl:118 [inlined]
macro expansion at ./asyncevent.jl:256 [inlined]
#644 at ./task.jl:123
jfptr_YY.644_44131.clone_1 at /home/dilum/.asdf/installs/julia/nightly/lib/julia/sys.so (unknown line)
_jl_invoke at /buildworker/worker/package_linux64/build/src/gf.c:2256 [inlined]
ijl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:2438
jl_apply at /buildworker/worker/package_linux64/build/src/julia.h:1773 [inlined]
start_task at /buildworker/worker/package_linux64/build/src/task.c:924
Allocations: 4859879 (Pool: 4855536; Big: 4343); GC: 8
Segmentation fault

[DilumAluthge]

CI is now green on Pkg master.