cleanup: some fixes from effort to break up monolithic sysimg build

From #38119

base/Base.jl

@@ -6,7 +6,7 @@ using Core.Intrinsics, Core.IR
 
 # to start, we're going to use a very simple definition of `include`
 # that doesn't require any function (except what we can get from the `Core` top-module)
-const _included_files = Array{Tuple{Module,String},1}()
+const _included_files = Array{Tuple{Module,String},1}(Core.undef, 1)
 function include(mod::Module, path::String)
     ccall(:jl_array_grow_end, Cvoid, (Any, UInt), _included_files, UInt(1))
     Core.arrayset(true, _included_files, (mod, ccall(:jl_prepend_cwd, Any, (Any,), path)), arraylen(_included_files))
@@ -606,5 +606,8 @@ end
 
 end
 
+# Ensure this file is also tracked
+@assert !isassigned(_included_files, 1)
+_included_files[1] = (parentmodule(Base), abspath(@__FILE__))
 
 end # baremodule Base

base/initdefs.jl

@@ -93,6 +93,7 @@ function append_default_depot_path!(DEPOT_PATH)
     path in DEPOT_PATH || push!(DEPOT_PATH, path)
     path = abspath(Sys.BINDIR, "..", "share", "julia")
     path in DEPOT_PATH || push!(DEPOT_PATH, path)
+    return DEPOT_PATH
 end
 
 function init_depot_path()
@@ -111,6 +112,7 @@ function init_depot_path()
     else
         append_default_depot_path!(DEPOT_PATH)
     end
+    nothing
 end
 
 ## LOAD_PATH & ACTIVE_PROJECT ##
@@ -220,9 +222,7 @@ function parse_load_path(str::String)
 end
 
 function init_load_path()
-    if Base.creating_sysimg
-        paths = ["@stdlib"]
-    elseif haskey(ENV, "JULIA_LOAD_PATH")
+    if haskey(ENV, "JULIA_LOAD_PATH")
         paths = parse_load_path(ENV["JULIA_LOAD_PATH"])
     else
         paths = filter!(env -> env !== nothing,

base/sysimg.jl

@@ -9,11 +9,9 @@ using Base.MainInclude # ans, err, and sometimes Out
 import Base.MainInclude: eval, include
 
 # Ensure this file is also tracked
-pushfirst!(Base._included_files, (@__MODULE__, joinpath(@__DIR__, "Base.jl")))
-pushfirst!(Base._included_files, (@__MODULE__, joinpath(@__DIR__, "sysimg.jl")))
+pushfirst!(Base._included_files, (@__MODULE__, @__FILE__))
 
 # set up depot & load paths to be able to find stdlib packages
-@eval Base creating_sysimg = true
 Base.init_depot_path()
 Base.init_load_path()
 
@@ -82,7 +80,7 @@ let
     m = Module()
     GC.@preserve m begin
         print_time = @eval m (mod, t) -> (print(rpad(string(mod) * "  ", $maxlen + 3, "─"));
-                                          Base.time_print(t * 10^9); println())
+                                          Base.time_print(stdout, t * 10^9); println())
         print_time(Base, (Base.end_base_include - Base.start_base_include) * 10^(-9))
 
         Base._track_dependencies[] = true
@@ -104,7 +102,6 @@ let
     empty!(Core.ARGS)
     empty!(Base.ARGS)
     empty!(LOAD_PATH)
-    @eval Base creating_sysimg = false
     Base.init_load_path() # want to be able to find external packages in userimg.jl
 
     ccall(:jl_clear_implicit_imports, Cvoid, (Any,), Main)
@@ -114,12 +111,12 @@ let
     tot_time = tot_time_base + tot_time_stdlib + tot_time_userimg
 
     println("Sysimage built. Summary:")
-    print("Base ──────── "); Base.time_print(tot_time_base    * 10^9); print(" "); show(IOContext(stdout, :compact=>true), (tot_time_base    / tot_time) * 100); println("%")
-    print("Stdlibs ───── "); Base.time_print(tot_time_stdlib  * 10^9); print(" "); show(IOContext(stdout, :compact=>true), (tot_time_stdlib  / tot_time) * 100); println("%")
+    print("Base ──────── "); Base.time_print(stdout, tot_time_base    * 10^9); print(" "); show(IOContext(stdout, :compact=>true), (tot_time_base    / tot_time) * 100); println("%")
+    print("Stdlibs ───── "); Base.time_print(stdout, tot_time_stdlib  * 10^9); print(" "); show(IOContext(stdout, :compact=>true), (tot_time_stdlib  / tot_time) * 100); println("%")
     if isfile("userimg.jl")
-    print("Userimg ───── "); Base.time_print(tot_time_userimg * 10^9); print(" "); show(IOContext(stdout, :compact=>true), (tot_time_userimg / tot_time) * 100); println("%")
+    print("Userimg ───── "); Base.time_print(stdout, tot_time_userimg * 10^9); print(" "); show(IOContext(stdout, :compact=>true), (tot_time_userimg / tot_time) * 100); println("%")
     end
-    print("Total ─────── "); Base.time_print(tot_time         * 10^9); println();
+    print("Total ─────── "); Base.time_print(stdout, tot_time         * 10^9); println();
 
     empty!(LOAD_PATH)
     empty!(DEPOT_PATH)

base/timing.jl

@@ -135,7 +135,7 @@ function format_bytes(bytes) # also used by InteractiveUtils
     end
 end
 
-function time_print(elapsedtime, bytes=0, gctime=0, allocs=0, compile_time=0, recompile_time=0, newline=false, _lpad=true)
+function time_print(io::IO, elapsedtime, bytes=0, gctime=0, allocs=0, compile_time=0, recompile_time=0, newline=false, _lpad=true)
     timestr = Ryu.writefixed(Float64(elapsedtime/1e9), 6)
     str = sprint() do io
         _lpad && print(io, length(timestr) < 10 ? (" "^(10 - length(timestr))) : "")
@@ -169,16 +169,17 @@ function time_print(elapsedtime, bytes=0, gctime=0, allocs=0, compile_time=0, re
             print(io, ": ", perc < 1 ? "<1" : Ryu.writefixed(perc, 0), "% of which was recompilation")
         end
         parens && print(io, ")")
+        newline && print(io, "\n")
     end
-    newline ? println(str) : print(str)
+    print(io, str)
     nothing
 end
 
 function timev_print(elapsedtime, diff::GC_Diff, compile_times, _lpad)
     allocs = gc_alloc_count(diff)
     compile_time = first(compile_times)
     recompile_time = last(compile_times)
-    time_print(elapsedtime, diff.allocd, diff.total_time, allocs, compile_time, recompile_time, true, _lpad)
+    time_print(stdout, elapsedtime, diff.allocd, diff.total_time, allocs, compile_time, recompile_time, true, _lpad)
     padded_nonzero_print(elapsedtime,       "elapsed time (ns)")
     padded_nonzero_print(diff.total_time,   "gc time (ns)")
     padded_nonzero_print(diff.allocd,       "bytes allocated")
@@ -279,7 +280,7 @@ macro time(msg, ex)
         local _msg = $(esc(msg))
         local has_msg = !isnothing(_msg)
         has_msg && print(_msg, ": ")
-        time_print(elapsedtime, diff.allocd, diff.total_time, gc_alloc_count(diff), first(compile_elapsedtimes), last(compile_elapsedtimes), true, !has_msg)
+        time_print(stdout, delapsedtime, diff.allocd, diff.total_time, gc_alloc_count(diff), first(compile_elapsedtimes), last(compile_elapsedtimes), true, !has_msg)
         val
     end
 end

contrib/generate_precompile.jl

@@ -1,13 +1,14 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
+# Prevent this from putting anyting into the Main namespace
+@eval Module() begin
+
 if Threads.maxthreadid() != 1
     @warn "Running this file with multiple Julia threads may lead to a build error" Threads.maxthreadid()
 end
 
 if Base.isempty(Base.ARGS) || Base.ARGS[1] !== "0"
 Sys.__init_build()
-# Prevent this from being put into the Main namespace
-@eval Module() begin
 if !isdefined(Base, :uv_eventloop)
     Base.reinit_stdio()
 end
@@ -238,6 +239,7 @@ ansi_disablecursor = "\e[?25l"
 generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printed
     start_time = time_ns()
     sysimg = Base.unsafe_string(Base.JLOptions().image_file)
+    blackhole = Sys.isunix() ? "/dev/null" : "nul"
 
     # Extract the precompile statements from the precompile file
     statements_step1 = Channel{String}(Inf)
@@ -285,7 +287,15 @@ generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printe
             Base.compilecache(Base.PkgId($(repr(pkgname))), $(repr(path)))
             $precompile_script
             """
-        run(`$(julia_exepath()) -O0 --sysimage $sysimg --trace-compile=$tmp_proc --startup-file=no -Cnative -e $s`)
+        p = withenv("JULIA_HISTORY" => blackhole,
+                    "JULIA_PROJECT" => nothing, # remove from environment
+                    "JULIA_LOAD_PATH" => "@stdlib",
+                    "JULIA_DEPOT_PATH" => ":",
+                    "TERM" => "") do
+            run(pipeline(`$(julia_exepath()) -O0 --trace-compile=$tmp_proc --sysimage $sysimg
+                         --cpu-target=native --startup-file=no --color=yes`,
+                         stdin=IOBuffer(s), stdout=debug_output))
+        end
         n_step1 = 0
         for f in (tmp_prec, tmp_proc)
             isfile(f) || continue
@@ -305,21 +315,19 @@ generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printe
         # Collect statements from running a REPL process and replaying our REPL script
         touch(precompile_file)
         pts, ptm = open_fake_pty()
-        blackhole = Sys.isunix() ? "/dev/null" : "nul"
         if have_repl
-            cmdargs = ```--color=yes
-                      -e 'import REPL; REPL.Terminals.is_precompiling[] = true'
-                      ```
+            cmdargs = `-e 'import REPL; REPL.Terminals.is_precompiling[] = true'`
         else
             cmdargs = `-e nothing`
         end
         p = withenv("JULIA_HISTORY" => blackhole,
                     "JULIA_PROJECT" => nothing, # remove from environment
-                    "JULIA_LOAD_PATH" => Sys.iswindows() ? "@;@stdlib" : "@:@stdlib",
+                    "JULIA_LOAD_PATH" => "@stdlib",
+                    "JULIA_DEPOT_PATH" => ":",
                     "JULIA_PKG_PRECOMPILE_AUTO" => "0",
                     "TERM" => "") do
             run(```$(julia_exepath()) -O0 --trace-compile=$precompile_file --sysimage $sysimg
-                   --cpu-target=native --startup-file=no -i $cmdargs```,
+                   --cpu-target=native --startup-file=no --color=yes -i $cmdargs```,
                    pts, pts, pts; wait=false)
         end
         Base.close_stdio(pts)
@@ -452,18 +460,16 @@ generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printe
     failed = length(statements) - n_succeeded
     print_state("step3" => string("F$n_succeeded", failed > 0 ? " ($failed failed)" : ""))
     println()
-    if have_repl
-        # Seems like a reasonable number right now, adjust as needed
-        # comment out if debugging script
-        n_succeeded > 1500 || @warn "Only $n_succeeded precompile statements"
-    end
+    # Seems like a reasonable number right now, adjust as needed
+    # comment out if debugging script
+    n_succeeded > (have_repl ? 900 : 90) || @warn "Only $n_succeeded precompile statements"
 
     fetch(step1) == :ok || throw("Step 1 of collecting precompiles failed.")
     fetch(step2) == :ok || throw("Step 2 of collecting precompiles failed.")
 
     tot_time = time_ns() - start_time
     println("Precompilation complete. Summary:")
-    print("Total ─────── "); Base.time_print(tot_time);     println()
+    print("Total ─────── "); Base.time_print(stdout, tot_time); println()
 finally
     fancyprint && print(ansi_enablecursor)
     return
@@ -474,22 +480,30 @@ generate_precompile_statements()
 # As a last step in system image generation,
 # remove some references to build time environment for a more reproducible build.
 Base.Filesystem.temp_cleanup_purge(force=true)
-@eval Base PROGRAM_FILE = ""
-@eval Sys begin
-    BINDIR = ""
-    STDLIB = ""
-end
-empty!(Base.ARGS)
-empty!(Core.ARGS)
 
-end # @eval
-end # if
+let stdout = Ref{IO}(stdout)
+    Base.PROGRAM_FILE = ""
+    Sys.BINDIR = ""
+    Sys.STDLIB = ""
+    empty!(Base.ARGS)
+    empty!(Core.ARGS)
+    empty!(Base.TOML_CACHE.d)
+    Base.TOML.reinit!(Base.TOML_CACHE.p, "")
+
+    println("Outputting sysimage file...")
+    Base.stdout = Core.stdout
+    Base.stderr = Core.stderr
 
-println("Outputting sysimage file...")
-let pre_output_time = time_ns()
     # Print report after sysimage has been saved so all time spent can be captured
+    pre_output_time = time_ns()
     Base.postoutput() do
         output_time = time_ns() - pre_output_time
-        print("Output ────── "); Base.time_print(output_time); println()
+        let stdout = stdout[]
+            print(stdout, "Output ────── "); Base.time_print(stdout, output_time); println(stdout)
+        end
+        stdout[] = Core.stdout
     end
 end
+
+end # if
+end # @eval

pkgimage.mk

@@ -5,7 +5,16 @@ include $(JULIAHOME)/Make.inc
 
 VERSDIR := v$(shell cut -d. -f1-2 < $(JULIAHOME)/VERSION)
 
+# set some influential environment variables
 export JULIA_DEPOT_PATH := $(build_prefix)/share/julia
+export JULIA_LOAD_PATH := @stdlib
+unexport JULIA_PROJECT :=
+unexport JULIA_BINDIR :=
+
+default: release
+release: all-release
+debug: all-debug
+all: release debug
 
 $(JULIA_DEPOT_PATH):
 	mkdir -p $@

sysimage.mk

@@ -76,8 +76,12 @@ $(build_private_libdir)/sys.ji: $(build_private_libdir)/corecompiler.ji $(JULIAH
 define sysimg_builder
 $$(build_private_libdir)/sys$1-o.a $$(build_private_libdir)/sys$1-bc.a : $$(build_private_libdir)/sys$1-%.a : $$(build_private_libdir)/sys.ji $$(JULIAHOME)/contrib/generate_precompile.jl
 	@$$(call PRINT_JULIA, cd $$(JULIAHOME)/base && \
-	if ! JULIA_BINDIR=$$(call cygpath_w,$(build_bindir)) WINEPATH="$$(call cygpath_w,$$(build_bindir));$$$$WINEPATH" \
-			JULIA_NUM_THREADS=1 \
+	if ! JULIA_BINDIR=$$(call cygpath_w,$(build_bindir)) \
+		 WINEPATH="$$(call cygpath_w,$$(build_bindir));$$$$WINEPATH" \
+		 JULIA_LOAD_PATH='@stdlib' \
+		 JULIA_PROJECT= \
+		 JULIA_DEPOT_PATH=':' \
+		 JULIA_NUM_THREADS=1 \
 			$$(call spawn, $3) $2 -C "$$(JULIA_CPU_TARGET)" --output-$$* $$(call cygpath_w,$$@).tmp $$(JULIA_SYSIMG_BUILD_FLAGS) \
 			--startup-file=no --warn-overwrite=yes --sysimage $$(call cygpath_w,$$<) $$(call cygpath_w,$$(JULIAHOME)/contrib/generate_precompile.jl) $(JULIA_PRECOMPILE); then \
 		echo '*** This error is usually fixed by running `make clean`. If the error persists$$(COMMA) try `make cleanall`. ***'; \

test/Makefile

@@ -6,6 +6,11 @@ VERSDIR := v$(shell cut -d. -f1-2 < $(JULIAHOME)/VERSION)
 STDLIBDIR := $(build_datarootdir)/julia/stdlib/$(VERSDIR)
 # TODO: this Makefile ignores BUILDDIR, except for computing JULIA_EXECUTABLE
 
+export JULIA_DEPOT_PATH := $(build_prefix)/share/julia
+export JULIA_LOAD_PATH := @stdlib
+unexport JULIA_PROJECT :=
+unexport JULIA_BINDIR :=
+
 TESTGROUPS = unicode strings compiler
 TESTS = all default stdlib $(TESTGROUPS) \
 		$(patsubst $(STDLIBDIR)/%/,%,$(dir $(wildcard $(STDLIBDIR)/*/.))) \

test/cmdlineargs.jl

@@ -134,10 +134,11 @@ end
 
 let exename = `$(Base.julia_cmd()) --startup-file=no --color=no`
     # tests for handling of ENV errors
-    let v = writereadpipeline("println(\"REPL: \", @which(less), @isdefined(InteractiveUtils))",
-                setenv(`$exename -i -E 'empty!(LOAD_PATH); @isdefined InteractiveUtils'`,
+    let v = writereadpipeline(
+            "println(\"REPL: \", @which(less), @isdefined(InteractiveUtils))",
+            setenv(`$exename -i -E '@assert isempty(LOAD_PATH); push!(LOAD_PATH, "@stdlib"); @isdefined InteractiveUtils'`,
                     "JULIA_LOAD_PATH" => "",
-                    "JULIA_DEPOT_PATH" => "",
+                    "JULIA_DEPOT_PATH" => ":",
                     "HOME" => homedir()))
         @test v == ("false\nREPL: InteractiveUtilstrue\n", true)
     end

test/loading.jl

@@ -692,7 +692,7 @@ mktempdir() do dir
     mkpath(vpath)
     script = "@assert startswith(Base.active_project(), $(repr(vpath)))"
     cmd = `$(Base.julia_cmd()) --startup-file=no -e $(script)`
-    cmd = addenv(cmd, "JULIA_DEPOT_PATH" => dir)
+    cmd = addenv(cmd, "JULIA_DEPOT_PATH" => dir, "JULIA_LOAD_PATH" => ":")
     cmd = pipeline(cmd; stdout, stderr)
     @test success(cmd)
 end

test/precompile.jl

@@ -4,6 +4,7 @@ original_depot_path = copy(Base.DEPOT_PATH)
 original_load_path = copy(Base.LOAD_PATH)
 
 using Test, Distributed, Random
+using REPL # doc lookup function
 
 Foo_module = :Foo4b3a94a1a081a8cb
 Foo2_module = :F2oo4b3a94a1a081a8cb