diff --git a/REQUIRE b/REQUIRE index 0ac2b855..19786bca 100644 --- a/REQUIRE +++ b/REQUIRE @@ -1,4 +1,3 @@ julia 0.7 ArgParse -SnoopCompile @windows WinRPM diff --git a/src/PackageCompiler.jl b/src/PackageCompiler.jl index 641bba17..7a73442f 100644 --- a/src/PackageCompiler.jl +++ b/src/PackageCompiler.jl @@ -1,6 +1,6 @@ module PackageCompiler -using Libdl, SnoopCompile +using Libdl include("compiler_flags.jl") include("static_julia.jl") @@ -111,7 +111,7 @@ use a toml instead. function compile_package( packages::Tuple{String, String}...; force = false, reuse = false, debug = false, cpu_target = nothing, - additional_packages = Symbol[] + additional_packages = Symbol[], verbose = false ) userimg = sysimg_folder("precompile.jl") if !reuse @@ -119,7 +119,7 @@ function compile_package( end !isfile(userimg) && reuse && error("Nothing to reuse. Please run `compile_package(reuse = true)`") image_path = sysimg_folder() - build_sysimg(image_path, userimg, cpu_target=cpu_target) + build_sysimg(image_path, userimg, cpu_target=cpu_target, verbose = verbose) imgfile = joinpath(image_path, "sys.$(Libdl.dlext)") syspath = joinpath(default_sysimg_path(debug), "sys.$(Libdl.dlext)") if force diff --git a/src/compiler_flags.jl b/src/compiler_flags.jl index d4857bc7..65d2b0bb 100644 --- a/src/compiler_flags.jl +++ b/src/compiler_flags.jl @@ -226,7 +226,7 @@ function run_julia(code::String; kw...) end function jl_command(flag, value) - isempty(value) && return "" + (value === nothing || isempty(value)) && return "" if is_short_flag(flag) string("-", flag, value) else @@ -234,6 +234,12 @@ function jl_command(flag, value) end end + +function push_command!(cmd, flag, value) + command = jl_command(flag, value) + isempty(command) || push!(cmd.exec, command) +end + function julia_code_cmd( code::String, jl_options = Base.JLOptions(); kw... @@ -244,13 +250,12 @@ function julia_code_cmd( for (k, v) in kw jl_key = jl_option_key(k) flag = jl_options_to_flag[jl_key] - push!(cmd.exec, jl_command(flag, v)) + push_command!(cmd, flag, v) end # add remaining commands from JLOptions for key in setdiff(keys(jl_options_to_flag), keys(kw)) flag = jl_options_to_flag[key] - command = jl_command(flag, jl_option_value(jl_options, key)) - isempty(command) || push!(cmd.exec, command) + push_command!(cmd, flag, jl_option_value(jl_options, key)) end # for better debug, let's not make a tmp file which would get lost! file = sysimg_folder("run_julia_code.jl") diff --git a/src/incremental.jl b/src/incremental.jl index bbf07d20..d78c303e 100644 --- a/src/incremental.jl +++ b/src/incremental.jl @@ -4,6 +4,7 @@ Init basic C libraries function InitBase() """ Base.__init__() + Sys.__init__() #fix https://github.com/JuliaLang/julia/issues/30479 """ end @@ -17,17 +18,6 @@ function InitREPL() """ end -""" -Fix for https://github.com/JuliaLang/julia/issues/30479 -""" -function Fix30479() - """ - _bindir = ccall(:jl_get_julia_bindir, Any, ())::String - @eval(Sys, BINDIR = \$(_bindir)) - @eval(Sys, STDLIB = joinpath(\$_bindir, "..", "share", "julia", "stdlib", string('v', (VERSION.major), '.', VERSION.minor))) - """ -end - function Include(path) """ M = Module() @@ -61,7 +51,7 @@ end The command to pass to julia --output-o, that runs the julia code in `path` during compilation. """ function PrecompileCommand(path) - ExitHooksStart() * InitBase() * Fix30479() * InitREPL() * Include(path) * ExitHooksEnd() + ExitHooksStart() * InitBase() * InitREPL() * Include(path) * ExitHooksEnd() end @@ -87,6 +77,7 @@ function compile_incremental( debug = false, cc_flags = nothing ) precompiles = package_folder("incremental_precompile.jl") + if snoopfile == nothing && precompile_file != nothing # we directly got a precompile_file isfile(precompile_file) || error("Need to pass an existing file to precompile_file. Found: $(repr(precompile_file))") @@ -96,7 +87,7 @@ function compile_incremental( elseif snoopfile == nothing && precompile_file == nothing # reuse precompiles else - snoop(toml_path, snoopfile, precompiles) + snoop(nothing, toml_path, snoopfile, precompiles) end systemp = sysimg_folder("sys.a") sysout = sysimg_folder("sys.$(Libdl.dlext)") @@ -138,19 +129,17 @@ function compile_incremental(packages::Symbol...; kw...) "compat" => Dict(), ) precompiles_all = package_folder("incremental_precompile.jl") - open(precompiles_all, "w") do io - println(io, "# Precompile file for $(join(packages, " "))") - end - for package in packages - precompiles = package_folder(string(package), "incremental_precompile.jl") - toml, testfile = package_toml(package) - snoop(toml, testfile, precompiles) - pkg_toml = TOML.parsefile(toml) - merge!(finaltoml["deps"], get(pkg_toml, "deps", Dict())) - merge!(finaltoml["compat"], get(pkg_toml, "compat", Dict())) - open(precompiles_all, "a") do io - println(io) - write(io, read(precompiles)) + open(precompiles_all, "w") do compile_io + println(compile_io, "# Precompile file for $(join(packages, " "))") + for package in packages + precompiles = package_folder(string(package), "incremental_precompile.jl") + toml, testfile = package_toml(package) + snoop(package, toml, testfile, precompiles) + pkg_toml = TOML.parsefile(toml) + merge!(finaltoml["deps"], get(pkg_toml, "deps", Dict())) + merge!(finaltoml["compat"], get(pkg_toml, "compat", Dict())) + println(compile_io) + write(compile_io, read(precompiles)) end end toml = package_folder("Project.toml") diff --git a/src/pkg.jl b/src/pkg.jl index 933fabd0..3252a415 100644 --- a/src/pkg.jl +++ b/src/pkg.jl @@ -25,54 +25,6 @@ function in_manifest(name::AbstractString, manifest::Dict = current_manifest()) end -""" - require_uninstalled(name::AbstractString, mod = Main) - -Loads a package, even if it isn't installed. -Call this only in a precompile file used for incremental compilation! -""" -function require_uninstalled(name::AbstractString, mod = Main) - pkg = Base.PkgId(package_uuid(name), name) - psym = Symbol(name) - @eval mod begin - if !isdefined($mod, $(QuoteNode(psym))) - const $psym = Base.require($pkg) - # we need to call the __init__ because of - # https://github.com/JuliaLang/julia/issues/22910 - if isdefined($psym, :__init__) - $psym.__init__() - end - else - @warn(string($name, " already defined")) - end - end -end - - -function extract_used_packages(file::String) - scope_regex = r"([\u00A0-\uFFFF\w_!´]*@?[\u00A0-\uFFFF\w_!´]+)\." - namespaces = unique(getindex.(eachmatch(scope_regex, read(file, String)), 1)) - # only use names that are also in current manifest - return filter(in_manifest, namespaces) -end - - -""" -Extracts using statements from a julia file. -""" -function extract_using(path, usings = Set{String}()) - src = read(path, String) - regex = r"using ([\u00A0-\uFFFF\w_!´]+)(,[ \u00A0-\uFFFF\w_!´]+)?" - for match in eachmatch(regex, src) - push!(usings, match[1]) - if match[2] !== nothing - pkgs = strip.(split(match[2], ',', keepempty = false)) - union!(usings, pkgs) - end - end - return usings -end - #= genfile & create_project_from_require have been taken from the PR https://github.com/JuliaLang/PkgDev.jl/pull/144 diff --git a/src/snooping.jl b/src/snooping.jl index 3d8a43e5..d79c9a88 100644 --- a/src/snooping.jl +++ b/src/snooping.jl @@ -1,17 +1,19 @@ using Pkg, Serialization -function snoop(tomlpath, snoopfile, outputfile, reuse = false) - packages = extract_using(snoopfile) +function snoop(package, tomlpath, snoopfile, outputfile, reuse = false) + command = """ using Pkg, PackageCompiler """ + if tomlpath != nothing command *= """ Pkg.activate($(repr(tomlpath))) Pkg.instantiate() """ end + command *= """ # let's wrap the snoop file in a try catch... # This way we still do some snooping even if there is an error in the tests! @@ -24,29 +26,48 @@ function snoop(tomlpath, snoopfile, outputfile, reuse = false) # let's use a file in the PackageCompiler dir, # so it doesn't get lost if later steps fail - tmp_file = sysimg_folder("precompile_tmp.jl") + tmp_file = package_folder("precompile_tmp.jl") if !reuse run_julia(command, compile = "all", O = 0, g = 1, trace_compile = tmp_file) end - actually_used = extract_used_packages(tmp_file) + used_packages = Set{String}() # e.g. from test/REQUIRE + if package != nothing + push!(used_packages, string(package)) + end if tomlpath != nothing # add toml packages, in case extract_used_packages misses a package deps = get(TOML.parsefile(tomlpath), "deps", Dict{String, Any}()) - union!(actually_used, string.(keys(deps))) + union!(used_packages, string.(keys(deps))) + end + usings = if isempty(used_packages) + "" + else + packages = join(used_packages, ", ") + """ + using $packages + for Mod in [$packages] + isdefined(Mod, :__init__) && Mod.__init__() + end + """ end line_idx = 0; missed = 0 open(outputfile, "w") do io println(io, """ - # We need to use all used packages in the precompile file for maximum - # usage of the precompile statements. - # Since this can be any recursive dependency of the package we AOT compile, - # we decided to just use them without installing them. An added - # benefit is, that we can call __init__ this way more easily, since - # incremental sysimage compilation won't call __init__ on `using` - # https://github.com/JuliaLang/julia/issues/22910 - using PackageCompiler - PackageCompiler.require_uninstalled.($(repr(actually_used)), (@__MODULE__,)) + # We need to use all used packages in the precompile file for maximum + # usage of the precompile statements. + # Since this can be any recursive dependency of the package we AOT compile, + # we decided to just use them without installing them. An added + # benefit is, that we can call __init__ this way more easily, since + # incremental sysimage compilation won't call __init__ on `using` + # https://github.com/JuliaLang/julia/issues/22910 + $usings + # bring recursive dependencies of used packages and standard libraries into namespace + for Mod in Base.loaded_modules_array() + if !Core.isdefined(@__MODULE__, nameof(Mod)) + Core.eval(@__MODULE__, Expr(:const, Expr(:(=), nameof(Mod), Mod))) + end + end """) for line in eachline(tmp_file) line_idx += 1 @@ -61,17 +82,47 @@ function snoop(tomlpath, snoopfile, outputfile, reuse = false) if expr.head != :incomplete # after all this, we still need to wrap into try catch, # since some anonymous symbols won't be found... - println(io, "try;", line, "; catch e; @warn \"couldn't precompile statement $line_idx\" exception = e; end") + println(io, "try;", line, "; catch e; @debug \"couldn't precompile statement $line_idx\" exception = e; end") else missed += 1 - @warn "Incomplete line in precompile file: $line" + @debug "Incomplete line in precompile file: $line" end catch e missed += 1 - @warn "Parse error in precompile file: $line" exception=e + @debug "Parse error in precompile file: $line" exception=e end end end @info "used $(line_idx - missed) out of $line_idx precompile statements" outputfile end + + +""" + snoop_userimg(userimg, packages::Tuple{String, String}...) + + Traces all function calls in packages and writes out `precompile` statements into the file `userimg` +""" +function snoop_userimg(userimg, packages::Tuple{String, String}...; additional_packages = Symbol[]) + snooped_precompiles = map(packages) do package_snoopfile + package, snoopfile = package_snoopfile + module_name = Symbol(package) + toml, runtests = package_toml(module_name) + pkg_root = normpath(joinpath(dirname(runtests), "..")) + file2snoop = if isfile(pkg_root, snoopfile) + joinpath(pkg_root, snoopfile) + else + joinpath(pkg_root, snoopfile) + end + precompile_file = package_folder(package, "precompile.jl") + snoop(package, toml, file2snoop, precompile_file) + return precompile_file + end + # merge all of the temporary files into a single output + open(userimg, "w") do output + for path in snooped_precompiles + open(input -> write(output, input), path) + end + end + nothing +end diff --git a/src/static_julia.jl b/src/static_julia.jl index 01506f66..685ef620 100644 --- a/src/static_julia.jl +++ b/src/static_julia.jl @@ -148,26 +148,13 @@ function static_julia( if snoopfile != nothing snoopfile = abspath(snoopfile) precompfile = joinpath(builddir, "precompiled.jl") - snoop(nothing, snoopfile, precompfile) + snoop(nothing, nothing, snoopfile, precompfile) jlmain = joinpath(builddir, "julia_main.jl") open(jlmain, "w") do io - juliaprog != nothing && println(io, "include(\"$(escape_string(relpath(juliaprog, builddir)))\")") - println(io, """ - let M = Module() # Prevent this from putting anything into the Main namespace - for m in Base.loaded_modules_array() - Core.isdefined(M, nameof(m)) || Core.eval(M, Expr(:const, Expr(:(=), nameof(m), m))) - end - for n in names(Main) - if Core.isdefined(Main, n) && isconst(Main, n) - m = getfield(Main, n) - m isa Module && (Core.isdefined(M, nameof(m)) || Core.eval(M, Expr(:const, Expr(:(=), nameof(m), m)))) - end - end - """) - println(io, "Base.include(M, ", repr(relpath(precompfile, builddir)), ")") - println(io, """ - end # let - """) + # this file will get included via @eval Module() include(...) + # but the juliaprog should be included in the main module + juliaprog != nothing && println(io, "Base.include(Main, $(repr(relpath(juliaprog, builddir))))") + println(io, "Base.include(@__MODULE__, $(repr(relpath(precompfile, builddir))))") end juliaprog = jlmain end @@ -249,32 +236,33 @@ function build_object( sysimage, home, startup_file, handle_signals, sysimage_native_code, compiled_modules, depwarn, warn_overwrite, compile, cpu_target, optimize, debug, inline, check_bounds, math_mode ) - Sys.iswindows() && (juliaprog != nothing) && (juliaprog = replace(juliaprog, "\\" => "\\\\")) - julia_cmd = build_julia_cmd( - sysimage, home, startup_file, handle_signals, sysimage_native_code, compiled_modules, - depwarn, warn_overwrite, compile, cpu_target, optimize, debug, inline, check_bounds, math_mode + # TODO really refactor this :D + build_object( + juliaprog, o_file, builddir, verbose; + sysimage = sysimage, startup_file = startup_file, + handle_signals = handle_signals, sysimage_native_code = sysimage_native_code, + compiled_modules = compiled_modules, + depwarn = depwarn, warn_overwrite = warn_overwrite, + compile = compile, cpu_target = cpu_target, optimize = optimize, + debug_level = debug, inline = inline, check_bounds = check_bounds, math_mode = math_mode ) - cache_dir = "cache_ji_v$VERSION" - # TODO: verify if this initialization is correct for Julia v1.0 - expr = """ - Base.__init__(); Sys.__init__() # initialize `Base` and `Sys` modules - pushfirst!(Base.DEPOT_PATH, $(repr(cache_dir))) # save precompiled modules locally - """ - juliaprog != nothing && (expr = expr * """ - include($(repr(juliaprog))) # include Julia program file - """) - # TODO: fix this for Julia v1.0, or how to precompile modules? - #if compiled_modules == "yes" - # command = `$julia_cmd -e $expr` - # verbose && println("Build \".ji\" local cache:\n $command") - # cd(builddir) do - # run(command) - # end - #end - command = `$julia_cmd --output-o $o_file -e $expr` +end + +function build_object( + juliaprog, o_file, builddir, verbose; julia_flags... + ) + Sys.iswindows() && (juliaprog != nothing) && (juliaprog = replace(juliaprog, "\\" => "\\\\")) + command = ExitHooksStart() * InitBase() * InitREPL() + if juliaprog != nothing + command *= Include(juliaprog) + end + command *= ExitHooksEnd() verbose && println("Build static library $(repr(o_file)):\n $command") cd(builddir) do - run(command) + run_julia( + command; julia_flags..., + output_o = o_file, track_allocation = "none", code_coverage = "none" + ) end end diff --git a/test/runtests.jl b/test/runtests.jl index c6bd8e33..513c3bb3 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,19 +1,44 @@ using PackageCompiler, Test -# This is the new compile_package -syso, syso_old = PackageCompiler.compile_incremental(:FixedPointNumbers) -test_code = """ -using FixedPointNumbers; N0f8(0.5); println("no segfaults, yay") -""" -cmd = PackageCompiler.julia_code_cmd(test_code, J = syso) -@test read(cmd, String) == "no segfaults, yay\n" +@testset "compilage_package" begin + @testset "FixedPointNumbers" begin + sysimage = PackageCompiler.compile_package("FixedPointNumbers", verbose = true) + test_code = """ + using FixedPointNumbers; N0f8(0.5); println("no segfaults, yay") + """ + cmd = PackageCompiler.julia_code_cmd(test_code, J = sysimage) + @test read(cmd, String) == "no segfaults, yay\n" + end + @testset "FixedPointNumbers ColorTypes" begin + sysimage = PackageCompiler.compile_package("FixedPointNumbers", "ColorTypes", verbose = true) + test_code = """ + using FixedPointNumbers, ColorTypes; N0f8(0.5); RGB(0.0, 0.0, 0.0); println("no segfaults, yay") + """ + cmd = PackageCompiler.julia_code_cmd(test_code, J = sysimage) + @test read(cmd, String) == "no segfaults, yay\n" + end +end + -syso, syso_old = PackageCompiler.compile_incremental(:FixedPointNumbers, :ColorTypes) -test_code = """ -using FixedPointNumbers, ColorTypes; N0f8(0.5); RGB(0.0, 0.0, 0.0); println("no segfaults, yay") -""" -cmd = PackageCompiler.julia_code_cmd(test_code, J = syso) -@test read(cmd, String) == "no segfaults, yay\n" +@testset "compile_incremental" begin + @testset "FixedPointNumbers" begin + # This is the new compile_package + syso, syso_old = PackageCompiler.compile_incremental(:FixedPointNumbers) + test_code = """ + using FixedPointNumbers; N0f8(0.5); println("no segfaults, yay") + """ + cmd = PackageCompiler.julia_code_cmd(test_code, J = syso) + @test read(cmd, String) == "no segfaults, yay\n" + end + @testset "FixedPointNumbers ColorTypes" begin + syso, syso_old = PackageCompiler.compile_incremental(:FixedPointNumbers, :ColorTypes) + test_code = """ + using FixedPointNumbers, ColorTypes; N0f8(0.5); RGB(0.0, 0.0, 0.0); println("no segfaults, yay") + """ + cmd = PackageCompiler.julia_code_cmd(test_code, J = syso) + @test read(cmd, String) == "no segfaults, yay\n" + end +end julia = Base.julia_cmd().exec[1] @@ -36,8 +61,10 @@ julia = Base.julia_cmd().exec[1] builddir = joinpath(basedir, relativebuilddir) @test isfile(joinpath(builddir, "hello.$(PackageCompiler.Libdl.dlext)")) @test isfile(joinpath(builddir, "hello$executable_ext")) - @test success(`$(joinpath(builddir, "hello$executable_ext"))`) + @test startswith(read(`$(joinpath(builddir, "hello$executable_ext"))`, String), "hello, world") for i = 1:100 + # Windows seems to have problems with removing files - it can error + # making this test fail. try rm(basedir, recursive = true) catch end sleep(1/100) end