Shards refactoring (#480)

* Update Artifacts.toml

* Update JLL environment and add experimental env support for C++

src/Clang.jl

@@ -1,6 +1,6 @@
 module Clang
 
-include("shards/JLLEnvs.jl")
+include("platform/JLLEnvs.jl")
 using .JLLEnvs
 
 llvm_version = if Base.libllvm_version < v"13"

src/platform/JLLEnvs.jl

@@ -0,0 +1,75 @@
+module JLLEnvs
+
+using Pkg
+using Pkg.Artifacts
+using Pkg.Artifacts: load_artifacts_toml, artifact_path
+using Base.BinaryPlatforms
+
+const JLL_ENV_SHARDS = Dict{String,Any}()
+
+const ARTIFACT_TOML_PATH = Ref{String}()
+
+function __init__()
+    if haskey(ENV, "JULIA_CLANG_SHARDS_URL") && !isempty(get(ENV, "JULIA_CLANG_SHARDS_URL", ""))
+        ARTIFACT_TOML_PATH[] = ENV["JULIA_CLANG_SHARDS_URL"]
+    else
+        ARTIFACT_TOML_PATH[] = normpath(joinpath(@__DIR__, "..", "..", "Artifacts.toml"))
+    end
+    merge!(JLL_ENV_SHARDS, Artifacts.load_artifacts_toml(ARTIFACT_TOML_PATH[]))
+end
+
+include("env.jl")
+include("version.jl")
+include("target.jl")
+include("types.jl")
+include("system.jl")
+
+function get_system_dirs(triple::String, version::VersionNumber=v"4.8.5", is_cxx=false)
+    env = get_env(parse(Platform, triple); version, is_cxx)
+    return get_system_includes(env)
+end
+
+function get_default_args(is_cxx=false, version=GCC_MIN_VER)
+    env = get_default_env(; is_cxx, version)
+    args = ["-isystem" * dir for dir in get_system_includes(env)]
+    push!(args, "--target=$(target(env.platform))")
+    return args
+end
+
+function get_pkg_artifact_dir(pkg::Module, target::String)
+    arftspath = Artifacts.find_artifacts_toml(Pkg.pathof(pkg))
+    arfts = first(values(Artifacts.load_artifacts_toml(arftspath)))
+    target_arch, target_os, target_libc = get_arch_os_libc(target)
+    candidates = Dict[]
+    for info in arfts
+        if info isa Dict
+            arch = get(info, "arch", "")
+            os = get(info, "os", "")
+            libc = get(info, "libc", "")
+            if arch == target_arch && os == target_os && libc == target_libc
+                push!(candidates, info)
+            end
+        else
+            # this could be an "Any"-platform JLL package
+            push!(candidates, arfts)
+            break
+        end
+    end
+    isempty(candidates) && return ""
+    length(candidates) > 1 && @warn "found more than one candidate artifacts, only use the first one: $(first(candidates))"
+    info = first(candidates)
+    name = info["git-tree-sha1"]  # this is not a real name but a hash
+    Artifacts.ensure_artifact_installed(name, info, arftspath)
+    return normpath(Artifacts.artifact_path(Base.SHA1(name)))
+end
+
+function get_pkg_include_dir(pkg::Module, target::String)
+    artifact_dir = get_pkg_artifact_dir(pkg, target)
+    if isempty(artifact_dir)
+        return ""
+    else
+        joinpath(artifact_dir, "include")
+    end
+end
+
+end # module

src/platform/env.jl

@@ -0,0 +1,46 @@
+const JLL_ENV_TRIPLES = String[
+    "aarch64-apple-darwin20",
+    "aarch64-linux-gnu",
+    "aarch64-linux-musl",
+    "armv7l-linux-gnueabihf",
+    "armv7l-linux-musleabihf",
+    "i686-linux-gnu",
+    "i686-linux-musl",
+    "i686-w64-mingw32",
+    "powerpc64le-linux-gnu",
+    "x86_64-apple-darwin14",
+    "x86_64-linux-gnu",
+    "x86_64-linux-musl",
+    "x86_64-unknown-freebsd",
+    "x86_64-w64-mingw32",
+]
+
+const HOST_TRIPLE = "x86_64-linux-musl"
+const GCC_SHARD_NAME = "GCCBootstrap"
+
+const JLL_ENV_PLATFORMS = [
+    Platform("aarch64", "macos"),
+    Platform("aarch64", "linux"),
+    Platform("x86_64", "linux"; libc="musl"),
+    Platform("armv7l", "linux"),
+    Platform("armv7l", "linux"; libc="musl"),
+    Platform("i686", "linux"),
+    Platform("i686", "linux"; libc="musl"),
+    Platform("i686", "windows"),
+    Platform("powerpc64le", "linux"),
+    Platform("x86_64", "macos"),
+    Platform("x86_64", "linux"),
+    Platform("x86_64", "linux"; libc="musl"),
+    Platform("x86_64", "freebsd"),
+    Platform("x86_64", "windows"),
+]
+
+get_gcc_shard_key(p::Platform, version::VersionNumber=GCC_MIN_VER) = "$GCC_SHARD_NAME-$(__triplet(p)).v$version.$HOST_TRIPLE.unpacked"
+get_gcc_shard_key(triple::String, version::VersionNumber=GCC_MIN_VER) = get_gcc_shard_key(parse(Platform, triple), version)
+
+function get_environment_info(p::Platform, version::VersionNumber=GCC_MIN_VER)
+    gcc = JLL_ENV_SHARDS[get_gcc_shard_key(p, version)][]
+    gcc_download = gcc["download"][]
+    return (id=gcc["git-tree-sha1"], url=gcc_download["url"], chk=gcc_download["sha256"])
+end
+get_environment_info(triple::String, version::VersionNumber=GCC_MIN_VER) = get_environment_info(parse(Platform, triple), version)

src/platform/system.jl

@@ -0,0 +1,192 @@
+function get_env(p::Platform; version::VersionNumber=GCC_MIN_VER, is_cxx=false)
+    if os(p) == "macos"
+        return MacEnv(p, version, is_cxx)
+    elseif os(p) == "windows"
+        return WindowsEnv(p, version, is_cxx)
+    elseif arch(p) == "armv7l"
+        return ArmEnv(p, version, is_cxx)
+    elseif libc(p) == "musl"
+        return MuslEnv(p, version, is_cxx)
+    elseif libc(p) == "glibc" || libc(p) == "gnu"
+        return GnuEnv(p, version, is_cxx)
+    else
+        @warn "unknown platform! using default GNU environment."
+        return GnuEnv(p, version, is_cxx)
+    end
+end
+
+get_default_env(; version::VersionNumber=GCC_MIN_VER, is_cxx=false) = get_env(HostPlatform(); version, is_cxx)
+
+function get_system_includes(env::AbstractJLLEnv=get_default_env())
+    gcc_info = get_environment_info(env.platform, env.gcc_version)
+
+    # download shards
+    if haskey(ENV, "JULIA_CLANG_SHARDS_URL") && !isempty(get(ENV, "JULIA_CLANG_SHARDS_URL", ""))
+        @info "Downloading artifact($(gcc_info.id))"
+    end
+
+    name = get_gcc_shard_key(env.platform, env.gcc_version)
+    Artifacts.ensure_artifact_installed(name, JLL_ENV_SHARDS[name][], ARTIFACT_TOML_PATH[])
+
+    # -isystem paths
+    gcc_prefix = artifact_path(Base.SHA1(gcc_info.id))
+
+    isys = String[]
+    get_system_includes!(env, gcc_prefix, isys)
+
+    for dir in isys
+        @assert isdir(dir) "failed to setup environment due to missing dir: $dir, please file an issue."
+    end
+
+    return normpath.(isys)
+end
+
+function get_system_includes!(env::MacEnv, prefix::String, isys::Vector{String})
+    p = env.platform
+    triple = __triplet(env.platform)
+    version = env.gcc_version
+    if os(p) == "macos" && arch(p) == "x86_64"
+        if env.is_cxx
+            push!(isys, joinpath(prefix, triple, "sys-root", "usr", "include", "c++", "v1"))
+            push!(isys, joinpath(prefix, triple, "sys-root", "usr", "include"))
+            push!(isys, joinpath(prefix, triple, "include", "c++", string(version)))
+            push!(isys, joinpath(prefix, triple, "include", "c++", string(version), triple))
+            push!(isys,
+                  joinpath(prefix, triple, "include", "c++", string(version), "backward"))
+            push!(isys, joinpath(prefix, triple, "include"))
+            push!(isys,
+                  joinpath(prefix, triple, "sys-root", "System", "Library", "Frameworks"))
+        else
+            push!(isys, joinpath(prefix, "lib", "gcc", triple, string(version), "include"))
+            push!(isys,
+                  joinpath(prefix, "lib", "gcc", triple, string(version), "include-fixed"))
+            push!(isys, joinpath(prefix, triple, "include"))
+            push!(isys, joinpath(prefix, triple, "sys-root", "usr", "include"))
+            push!(isys,
+                  joinpath(prefix, triple, "sys-root", "System", "Library", "Frameworks"))
+        end
+    else  # "aarch64-apple-darwin20"
+        if env.is_cxx
+            push!(isys, joinpath(prefix, triple, "sys-root", "usr", "include", "c++", "v1"))
+            push!(isys, joinpath(prefix, triple, "sys-root", "usr", "include"))
+            push!(isys, joinpath(prefix, triple, "include", "c++", "11.0.0"))
+            push!(isys, joinpath(prefix, triple, "include", "c++", "11.0.0", triple))
+            push!(isys, joinpath(prefix, triple, "include", "c++", "11.0.0", "backward"))
+        else
+            push!(isys, joinpath(prefix, "lib", "gcc", triple, "11.0.0", "include"))
+            push!(isys, joinpath(prefix, "lib", "gcc", triple, "11.0.0", "include-fixed"))
+            push!(isys, joinpath(prefix, triple, "include"))
+            push!(isys, joinpath(prefix, triple, "sys-root", "usr", "include"))
+            push!(isys,
+                  joinpath(prefix, triple, "sys-root", "System", "Library", "Frameworks"))
+        end
+    end
+end
+
+function get_system_includes!(env::WindowsEnv, prefix::String, isys::Vector{String})
+    triple = __triplet(env.platform)
+    version = env.gcc_version
+    if env.is_cxx
+        push!(isys, joinpath(prefix, triple, "include", "c++", string(version)))
+        push!(isys, joinpath(prefix, triple, "include", "c++", string(version), triple))
+        push!(isys, joinpath(prefix, triple, "include", "c++", string(version), "backward"))
+        push!(isys, joinpath(prefix, triple, "include"))
+        push!(isys, joinpath(prefix, triple, "sys-root", "include"))
+    else
+        push!(isys, joinpath(prefix, "lib", "gcc", triple, string(version), "include"))
+        push!(isys,
+              joinpath(prefix, "lib", "gcc", triple, string(version), "include-fixed"))
+        push!(isys, joinpath(prefix, triple, "include"))
+        push!(isys, joinpath(prefix, triple, "sys-root", "include"))
+    end
+end
+
+function get_system_includes!(env::GnuEnv, prefix::String, isys::Vector{String})
+    triple = __triplet(env.platform)
+    version = env.gcc_version
+    if env.is_cxx
+        push!(isys, joinpath(prefix, triple, "include", "c++", string(version)))
+        push!(isys, joinpath(prefix, triple, "include", "c++", string(version), triple))
+        push!(isys, joinpath(prefix, triple, "include", "c++", string(version), "backward"))
+        push!(isys, joinpath(prefix, triple, "include"))
+        push!(isys, joinpath(prefix, triple, "sys-root", "usr", "include"))
+    else
+        push!(isys, joinpath(prefix, "lib", "gcc", triple, string(version), "include"))
+        push!(isys,
+              joinpath(prefix, "lib", "gcc", triple, string(version), "include-fixed"))
+        push!(isys, joinpath(prefix, triple, "include"))
+        push!(isys, joinpath(prefix, triple, "sys-root", "usr", "include"))
+    end
+end
+
+function get_system_includes!(env::MuslEnv, prefix::String, isys::Vector{String})
+    triple = __triplet(env.platform)
+    version = env.gcc_version
+    if env.is_cxx
+        push!(isys, joinpath(prefix, triple, "include", "c++", string(version)))
+        push!(isys, joinpath(prefix, triple, "include", "c++", string(version), triple))
+        push!(isys, joinpath(prefix, triple, "include", "c++", string(version), "backward"))
+        push!(isys, joinpath(prefix, triple, "include"))
+        push!(isys, joinpath(prefix, triple, "sys-root", "usr", "include"))
+    else
+        push!(isys, joinpath(prefix, "lib", "gcc", triple, string(version), "include"))
+        push!(isys, joinpath(prefix, triple, "include"))
+        push!(isys, joinpath(prefix, triple, "sys-root", "usr", "include"))
+    end
+end
+
+function get_system_includes!(env::ArmEnv, prefix::String, isys::Vector{String})
+    triple = __triplet(env.platform)
+    version = env.gcc_version
+    if env.platform == Platform("armv7l", "linux")
+        if env.is_cxx
+            push!(isys,
+                  joinpath(prefix, "arm-linux-gnueabihf", "include", "c++",
+                           string(version)))
+            push!(isys,
+                  joinpath(prefix, "arm-linux-gnueabihf", "include", "c++", string(version),
+                           "arm-linux-gnueabihf"))
+            push!(isys,
+                  joinpath(prefix, "arm-linux-gnueabihf", "include", "c++", string(version),
+                           "backward"))
+            push!(isys, joinpath(prefix, "arm-linux-gnueabihf", "include"))
+            push!(isys,
+                  joinpath(prefix, "arm-linux-gnueabihf", "sys-root", "usr", "include"))
+        else
+            push!(isys,
+                  joinpath(prefix, "lib", "gcc", "arm-linux-gnueabihf", string(version),
+                           "include"))
+            push!(isys,
+                  joinpath(prefix, "lib", "gcc", "arm-linux-gnueabihf", string(version),
+                           "include-fixed"))
+            push!(isys, joinpath(prefix, "arm-linux-gnueabihf", "include"))
+            push!(isys,
+                  joinpath(prefix, "arm-linux-gnueabihf", "sys-root", "usr", "include"))
+        end
+    else  # "armv7l-linux-musleabihf"
+        if env.is_cxx
+            push!(isys,
+                  joinpath(prefix, "arm-linux-musleabihf", "include", "c++",
+                           string(version)))
+            push!(isys,
+                  joinpath(prefix, "arm-linux-musleabihf", "include", "c++",
+                           string(version), "arm-linux-musleabihf"))
+            push!(isys,
+                  joinpath(prefix, "arm-linux-musleabihf", "include", "c++",
+                           string(version), "backward"))
+            push!(isys, joinpath(prefix, "arm-linux-musleabihf", "include"))
+            push!(isys,
+                  joinpath(prefix, "arm-linux-musleabihf", "sys-root", "usr", "include"))
+        else
+            push!(isys,
+                  joinpath(prefix, "lib", "gcc", "arm-linux-musleabihf", string(version),
+                           "include"))
+            push!(isys,
+                  joinpath(prefix, "lib", "gcc", "arm-linux-musleabihf", string(version),
+                           "include-fixed"))
+            push!(isys, joinpath(prefix, "arm-linux-musleabihf", "include"))
+            push!(isys,
+                  joinpath(prefix, "arm-linux-musleabihf", "sys-root", "usr", "include"))
+        end
+    end
+end

src/platform/target.jl

@@ -0,0 +1,36 @@
+const JLL_ENV_CLANG_TARGETS_MAPPING = Dict(
+    "aarch64-apple-darwin20"=>"aarch64-apple-darwin20",
+    "aarch64-linux-gnu"=>"aarch64-unknown-linux-gnu",
+    "aarch64-linux-musl"=>"aarch64-unknown-linux-musl",
+    "armv7l-linux-gnueabihf"=>"armv7l-unknown-linux-gnueabihf",
+    "armv7l-linux-musleabihf"=>"armv7l-unknown-linux-musleabihf",
+    "i686-linux-gnu"=>"i686-unknown-linux-gnu",
+    "i686-linux-musl"=>"i686-unknown-linux-musl",
+    "i686-w64-mingw32"=>"i686-w64-windows-gnu",
+    "powerpc64le-linux-gnu"=>"powerpc64le-unknown-linux-gnu",
+    "x86_64-apple-darwin14"=>"x86_64-apple-darwin14",
+    "x86_64-linux-gnu"=>"x86_64-unknown-linux-gnu",
+    "x86_64-linux-musl"=>"x86_64-unknown-linux-musl",
+    "x86_64-unknown-freebsd"=>"x86_64-unknown-freebsd13.2",
+    "x86_64-w64-mingw32"=>"x86_64-w64-windows-gnu",
+)
+
+triple2target(triple::String) = get(JLL_ENV_CLANG_TARGETS_MAPPING, triple, "unknown")
+
+function __triplet(p::Platform)
+    for k in keys(p.tags)
+        if k != "arch" && k != "os" && k != "libc"
+            delete!(p.tags, k)
+        end
+    end
+    t = triplet(p)
+    if os(p) == "macos" && arch(p) == "x86_64"
+        t *= "14"
+    elseif os(p) == "macos" && arch(p) == "aarch64"
+        t *= "20"
+    end
+    return t
+end
+
+target(triple::String) = get(JLL_ENV_CLANG_TARGETS_MAPPING, triple, "unknown")
+target(p::Platform) = target(__triplet(p))

src/platform/types.jl

@@ -0,0 +1,31 @@
+abstract type AbstractJLLEnv end
+
+struct MacEnv <: AbstractJLLEnv
+    platform::Platform
+    gcc_version::VersionNumber
+    is_cxx::Bool
+end
+
+struct GnuEnv <: AbstractJLLEnv
+    platform::Platform
+    gcc_version::VersionNumber
+    is_cxx::Bool
+end
+
+struct MuslEnv <: AbstractJLLEnv
+    platform::Platform
+    gcc_version::VersionNumber
+    is_cxx::Bool
+end
+
+struct WindowsEnv <: AbstractJLLEnv
+    platform::Platform
+    gcc_version::VersionNumber
+    is_cxx::Bool
+end
+
+struct ArmEnv <: AbstractJLLEnv
+    platform::Platform
+    gcc_version::VersionNumber
+    is_cxx::Bool
+end