build from vanilla backup image (#38)

* build from vanilla backup image

* turn debug nothing into bool

* get debug from julia

* fix tests

README.md

@@ -27,7 +27,7 @@ compile_package("Matcha", "relative/path/for_snooping.jl")
 revert()
 
 # Or if you simply want to get a native system image e.g. when you have downloaded the generic Julia install:
-build_native_image()
+force_native_image!()
 
 # building an executable
 

src/PackageCompiler.jl

@@ -50,36 +50,22 @@ function copy_system_image(src, dest, ignore_missing = false)
     end
 end
 
-"""
-Replaces the julia system image forcefully with a system image located at `image_path`
-"""
-function replace_jl_sysimg(image_path, debug = false)
-    syspath = default_sysimg_path(debug)
-    backup = sysimgbackup_folder()
-    # create a backup
-    # if syspath has missing files, ignore, since it will get replaced anyways
-    copy_system_image(syspath, backup, true)
-    info("Overwriting system image!")
-    copy_system_image(image_path, syspath)
+julia_cpu_target(x) = error("CPU target needs to be a string or `nothing`")
+julia_cpu_target(x::String) = x # TODO match against available targets
+function julia_cpu_target(::Void)
+    replace(Base.julia_cmd().exec[2], "-C", "")
 end
 
 
-
-
 """
 Reverts a forced compilation of the system image.
 This will restore any previously backed up system image files, or
 build a new, clean system image
 """
 function revert(debug = false)
     syspath = default_sysimg_path(debug)
-    sysimg_backup = sysimgbackup_folder()
-    if all(x-> isfile(joinpath(sysimg_backup, x)), sysimage_binaries) # we have a backup
-        copy_system_image(sysimg_backup, syspath)
-    else
-        warn("No backup found but restoring. Need to build a new system image from scratch")
-        build_native_image(debug)
-    end
+    sysimg_backup = dirname(get_backup!(debug))
+    copy_system_image(sysimg_backup, syspath)
 end
 
 function get_root_dir(path)
@@ -147,8 +133,10 @@ function compile_package(packages::Tuple{String, String}...; force = false, reus
     image_path = sysimg_folder()
     build_sysimg(image_path, userimg)
     imgfile = joinpath(image_path, "sys.$(Libdl.dlext)")
+    syspath = default_sysimg_path(debug)
     if force
         try
+            copy_system_image(image_path, syspath)
             replace_jl_sysimg(image_path, debug)
             info(
                 "Replaced system image successfully. Next start of julia will load the newly compiled system image.
@@ -159,7 +147,7 @@ function compile_package(packages::Tuple{String, String}...; force = false, reus
             warn(e)
             info("Recovering old system image from backup")
             # if any file is missing in default system image, revert!
-            syspath = default_sysimg_path(debug)
+
             for file in sysimage_binaries
                 if !isfile(joinpath(syspath, file))
                     info("$(joinpath(syspath, file)) missing. Reverting!")
@@ -183,11 +171,11 @@ end
 function __init__()
     if Base.julia_cmd().exec[2] != "-Cnative"
         warn("Your Julia system image is not compiled natively for this CPU architecture.
-        Please run `PackageCompiler.build_native_image()` for optimal Julia performance"
+        Please run `PackageCompiler.force_native_image!()` for optimal Julia performance"
         )
     end
 end
 
-export compile_package, revert, build_native_image, executable_ext, build_executable
+export compile_package, revert, force_native_image!, executable_ext, build_executable
 
 end # module

src/api.jl

@@ -17,6 +17,8 @@ function build_sysimg(sysimg_path, userimg_path = nothing;
         debug = nothing, inline = nothing, check_bounds = nothing,
         math_mode = nothing
     )
+    # build vanilla backup system image
+    clean_sysimg = get_backup!(contains(basename(Base.julia_cmd().exec[1]), "debug"), cpu_target)
     static_julia(
         userimg_path, juliaprog_basename = "sys",
 
@@ -25,7 +27,7 @@ function build_sysimg(sysimg_path, userimg_path = nothing;
         math_mode = math_mode, verbose = verbose, quiet = quiet,
 
         cprog = nothing, builddir = sysimg_path,
-        clean = false, sysimage = nothing,
+        clean = false, sysimage = clean_sysimg,
         compile = nothing, depwarn = nothing, autodeps = false,
         object = true, shared = true, executable = false, julialibs = false,
     )
@@ -106,13 +108,11 @@ end
 
 
 """
-    build_native_image()
+    force_native_image!()
 Builds a clean system image, similar to a fresh Julia install.
 Can also be used to build a native system image for a downloaded cross compiled julia binary.
 """
-function build_native_image(debug = false) # debug is ignored right now
-    backup = sysimgbackup_folder()
-    # Build in backup dir, so we have a clean backup!
-    compile_system_image(joinpath(backup, "sys"), "native")
-    copy_system_image(backup, default_sysimg_path(debug))
+function force_native_image!(debug = false) # debug is ignored right now
+    sysimg = get_backup!(debug, "native")
+    copy_system_image(dirname(sysimg), default_sysimg_path(debug))
 end

src/snooping.jl

@@ -1,13 +1,32 @@
+# Taken from SnoopCompile
+function snoop_vanilla(filename, path)
+    code_object = """
+    while !eof(STDIN)
+        eval(Main, deserialize(STDIN))
+    end
+    """
+    julia_cmd = build_julia_cmd(
+        get_backup!(false, nothing), nothing, nothing, nothing, nothing,
+        nothing, nothing, nothing, nothing, false
+    )
+    @show julia_cmd
+    in, io = open(`$julia_cmd --eval $code_object`, "w", STDOUT)
+    serialize(in, quote
+        import SnoopCompile
+    end)
+    # Now that the new process knows about SnoopCompile, it can
+    # expand the macro in this next expression
+    serialize(in, quote
+          SnoopCompile.@snoop1 $filename include($(escape_string(path)))
+    end)
+    close(in)
+    wait(io)
+    println("done.")
+    nothing
+end
 
 function snoop(path, compilationfile, csv)
-    cd(@__DIR__)
-    # Snoop compiler can't handle the path as a variable, so we just create a file
-    open(joinpath("snoopy.jl"), "w") do io
-        println(io, "include(\"$(escape_string(path))\")")
-    end
-    SnoopCompile.@snoop csv begin
-        include("snoopy.jl")
-    end
+    snoop_vanilla(abspath(csv), abspath(path))
     data = SnoopCompile.read(csv)
     pc = SnoopCompile.parcel(reverse!(data[2]))
     delims = r"([\{\} \n\(\),])_([\{\} \n\(\),])"

src/static_julia.jl

@@ -61,10 +61,10 @@ function static_julia(
         cprog = nothing, builddir = "builddir", juliaprog_basename = splitext(basename(juliaprog))[1],
         verbose = false, quiet = false, clean = false,
         autodeps = false, object = false, shared = false, executable = false, julialibs = false,
-	sysimage = nothing, compile = nothing, cpu_target = nothing,
-	optimize = nothing, debug = nothing, inline = nothing,
-	check_bounds = nothing, math_mode = nothing, depwarn = nothing,
-	cc = system_compiler()
+    	sysimage = nothing, compile = nothing, cpu_target = nothing,
+    	optimize = nothing, debug = nothing, inline = nothing,
+    	check_bounds = nothing, math_mode = nothing, depwarn = nothing,
+    	cc = system_compiler()
     )
 
     verbose && quiet && (quiet = false)
@@ -122,15 +122,14 @@ function static_julia(
     o_file = juliaprog_basename * ".o"
     s_file = juliaprog_basename * ".$(Libdl.dlext)"
     e_file = juliaprog_basename * executable_ext()
-    tmp_dir = "tmp_v$VERSION"
 
     object && build_object(
-        juliaprog, tmp_dir, o_file, verbose,
+        juliaprog, builddir, o_file, verbose,
         sysimage, compile, cpu_target, optimize, debug, inline, check_bounds,
         math_mode, depwarn
     )
 
-    shared && build_shared(s_file, joinpath(tmp_dir, o_file), verbose, optimize, debug)
+    shared && build_shared(s_file, joinpath(builddir, o_file), verbose, optimize, debug)
 
     executable && build_executable(s_file, e_file, cprog, verbose, optimize, debug)
 
@@ -143,32 +142,31 @@ end
 function julia_flags(optimize, debug)
     if julia_v07
         command = `$(Base.julia_cmd()) --startup-file=no $(joinpath(dirname(Sys.BINDIR), "share", "julia", "julia-config.jl"))`
-        flags = `$(Base.shell_split(read(\`$command --allflags\`, String)))`
+        flags = Base.shell_split(read(`$command --allflags`, String))
         optimize == nothing || (flags = `$flags -O$optimize`)
         debug != 2 || (flags = `$flags -g`)
         return flags
     else
         command = `$(Base.julia_cmd()) --startup-file=no $(joinpath(dirname(JULIA_HOME), "share", "julia", "julia-config.jl"))`
-        cflags = `$(Base.shell_split(readstring(\`$command --cflags\`)))`
+        cflags = Base.shell_split(readstring(`$command --cflags`))
         optimize == nothing || (cflags = `$cflags -O$optimize`)
         debug != 2 || (cflags = `$cflags -g`)
-        ldflags = `$(Base.shell_split(readstring(\`$command --ldflags\`)))`
-        ldlibs = `$(Base.shell_split(readstring(\`$command --ldlibs\`)))`
+        ldflags = Base.shell_split(readstring(`$command --ldflags`))
+        ldlibs = Base.shell_split(readstring(`$command --ldlibs`))
         return `$cflags $ldflags $ldlibs`
     end
 end
 
-function build_object(
-        juliaprog, builddir, o_file, verbose,
+function build_julia_cmd(
         sysimage, compile, cpu_target, optimize, debug, inline, check_bounds,
-        math_mode, depwarn
+        math_mode, depwarn, startupfile = false
     )
     julia_cmd = `$(Base.julia_cmd())`
     if length(julia_cmd.exec) != 5 || !all(startswith.(julia_cmd.exec[2:5], ["-C", "-J", "--compile", "--depwarn"]))
         error("Unexpected format of \"Base.julia_cmd()\", you may be using an incompatible version of Julia")
     end
     sysimage == nothing || (julia_cmd.exec[3] = "-J$sysimage")
-    push!(julia_cmd.exec, "--startup-file=no")
+    push!(julia_cmd.exec, string("--startup-file=", startupfile ? "yes" : "no"))
     compile == nothing || (julia_cmd.exec[4] = "--compile=$compile")
     cpu_target == nothing || (julia_cmd.exec[2] = "-C$cpu_target")
     optimize == nothing || push!(julia_cmd.exec, "-O$optimize")
@@ -177,18 +175,31 @@ function build_object(
     check_bounds == nothing || push!(julia_cmd.exec, "--check-bounds=$check_bounds")
     math_mode == nothing || push!(julia_cmd.exec, "--math-mode=$math_mode")
     depwarn == nothing || (julia_cmd.exec[5] = "--depwarn=$depwarn")
+    julia_cmd
+end
+
+function build_object(
+        juliaprog, builddir, o_file, verbose,
+        sysimage, compile, cpu_target, optimize, debug, inline, check_bounds,
+        math_mode, depwarn
+    )
+    julia_cmd = build_julia_cmd(
+        sysimage, compile, cpu_target, optimize, debug, inline, check_bounds,
+        math_mode, depwarn, false
+    )
+    builddir_esc = escape_string(builddir)
     if julia_v07
         iswindows() && (juliaprog = replace(juliaprog, "\\", "\\\\"))
         expr = "Base.init_depot_path() # initialize package depots
         Base.init_load_path() # initialize location of site-packages
         empty!(Base.LOAD_CACHE_PATH) # reset / remove any builtin paths
-        push!(Base.LOAD_CACHE_PATH, abspath(\"$builddir\")) # enable usage of precompiled files
+        push!(Base.LOAD_CACHE_PATH, abspath(\"$builddir_esc\")) # enable usage of precompiled files
         include(\"$juliaprog\") # include Julia program file
         empty!(Base.LOAD_CACHE_PATH) # reset / remove build-system-relative paths"
     else
         iswindows() && (juliaprog = replace(juliaprog, "\\", "\\\\"))
         expr = "empty!(Base.LOAD_CACHE_PATH) # reset / remove any builtin paths
-        push!(Base.LOAD_CACHE_PATH, abspath(\"$builddir\")) # enable usage of precompiled files
+        push!(Base.LOAD_CACHE_PATH, abspath(\"$builddir_esc\")) # enable usage of precompiled files
         include(\"$juliaprog\") # include Julia program file
         empty!(Base.LOAD_CACHE_PATH) # reset / remove build-system-relative paths"
     end

src/system_image.jl

@@ -7,13 +7,19 @@ function default_sysimg_path(debug = false)
     end
 end
 
-function compile_system_image(sysimg_path, cpu_target; debug = false)
+function compile_system_image(sysimg_path, cpu_target = nothing; debug = false)
     # Enter base and setup some useful paths
     base_dir = dirname(Base.find_source_file("sysimg.jl"))
     cd(base_dir) do
         # This can probably get merged with build_object.
         # At some point, I will need to understand build_object a bit better before doing that move, though!
-        julia = Base.julia_cmd().exec[1]
+        julia_cmd = Base.julia_cmd()
+        julia = julia_cmd.exec[1]
+        cpu_target = if cpu_target == nothing
+            replace(julia_cmd.exec[2], "-C", "")
+        else
+            cpu_target
+        end
         cc = system_compiler()
         # Ensure we have write-permissions to wherever we're trying to write to
         try
@@ -34,7 +40,26 @@ function compile_system_image(sysimg_path, cpu_target; debug = false)
         info("$julia -C $cpu_target --output-ji $sysimg_path.ji --output-o $sysimg_path.o -J $inference_path.ji --startup-file=no sysimg.jl")
         run(`$julia -C $cpu_target --output-ji $sysimg_path.ji --output-o $sysimg_path.o -J $inference_path.ji --startup-file=no sysimg.jl`)
 
-        build_shared("$sysimg_path.$(Libdl.dlext)", "$sysimg_path.o",
-                     true, nothing, (if debug 2 else nothing end))
+        build_shared(
+            "$sysimg_path.$(Libdl.dlext)", "$sysimg_path.o",
+            true, nothing, debug ? 2 : nothing
+        )
     end
 end
+
+"""
+Returns the system image file stored in the backup folder.
+If there is no backup, this function will automatically generate a system image
+in the backup folder.
+"""
+function get_backup!(debug, cpu_target = nothing)
+    target = julia_cpu_target(cpu_target)
+    sysimg_backup = sysimgbackup_folder(target)
+    isdir(sysimg_backup) || mkpath(sysimg_backup)
+    if !all(x-> isfile(joinpath(sysimg_backup, x)), sysimage_binaries) # we have a backup
+        compile_system_image(joinpath(sysimg_backup, "sys"), target; debug = debug)
+    end
+    return joinpath(sysimg_backup, "sys.$(Libdl.dlext)")
+end
+
+|